mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 08:51:58 +00:00
ThearchyHelios
a552f4feab
额度：judge.MAX_TOKENS 从 4096 改为 16384（核查的 verify_model_spec 沿用它）。经 OpenAI 兼容接入的推理模型收不到 thinking=off，思考算在额度里，4096 常常想完就没额度发查询、交结果；16384 是常见网关接受的输出上限里偏大的一档，cite_fields 的抽取也是这个数。服务方嫌大（400/422 且报错提到 max_tokens 一类）时退回 judge.FALLBACK_MAX_TOKENS（4096）再试一次，工具循环和收尾共用一个开关，之后都用小的；小额度的模型要退回时才建。 截断：工具循环里没有工具调用、finish_reason=length（或 stop_reason=max_tokens）不再当成「模型说完了」，一条查询都没成功时不再收尾（收尾只能交「查不到」，双盲复算就会记成「盲算没有查到这个数」）；查到过东西的照常收尾，取不到要的数时同样交回截断的原因，取得到的照常交、notes 里注明。收尾被截断（Anthropic 强制工具调用写到一半、openai 结构化输出抛 LengthFinishReasonError、普通调用只有思考）时交回截断的原因，不再退回普通调用多花一次额度；截断那次的用量从异常里取，照记账。给人看的原因写明「核查模型的输出达到上限（N token）被截断，核查未能完成。通常是推理占满了输出额度，可在「设置 → 偏好设置 → 核查」中换用不带推理的模型」，经 error 交给双盲复算的无法复算原因、反方审稿的「反查未完成」、质询的报错；节点里调用时另记一条 verify_truncated，llm.end 标出 truncated 和额度。没有截断时原有说法不变。 测试：新增 test_verify_truncation（假模型返回 finish_reason=length、只有思考、LengthFinishReasonError，覆盖工具循环、收尾结构化和普通调用、退回 4096、别的 400 不重试、三条渠道）；三处钉着 4096 的断言改成 16384。
701 lines
35 KiB
Python
701 lines
35 KiB
Python
"""按需裁判接口 POST /api/runs/{id}/evidence/judge（四期）。
|
||
|
||
探索运行不在节点里花裁判的钱：点开哪句才判哪句。
|
||
|
||
- 判定作为批注追加在封存之后（evidence.judged 事件，post_seal: true）：封存核对只覆盖 manifest_seq 之前的
|
||
事件，所以追加之后 verify 照旧一致；封存的报告文档一个字都不改
|
||
- 只在探索运行里可用：正式运行的裁判在节点内完成，按需裁判返回 409；还没封存的、升级前发起的、封存链断了的
|
||
也不判
|
||
- 每次点击的上限、全局每日上限按设置控制；触顶不是报错，照实说「已到上限」和怎么调
|
||
- 同一次运行里同一句判过就直接给已有的判定，不再调用、不再花钱（连点两下也一样）
|
||
- 裁判只看得到封存范围内的证据：人为插进工件表的、封存之后才追加事件引用的快照，一行都不进摘录；
|
||
数据源遮罩的列同样不给
|
||
- 片段接口和证据图带上判定，封存内的和封存后追加的分得开
|
||
|
||
所有裁判都用 mock 模型给剧本，不调真接口。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import re
|
||
import sqlite3
|
||
from types import SimpleNamespace
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
from langchain_core.messages import AIMessage
|
||
from sqlalchemy import delete, select, update
|
||
|
||
from app.core import artifact_store
|
||
from app.core.events import EventType
|
||
from app.data.engine import engines
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import DataSource, Run, RunEvent, Setting
|
||
from app.engine import judge as judge_mod
|
||
from app.engine import runner as runner_mod
|
||
from app.engine.evidence import iter_segments, iter_units
|
||
from app.engine.judge import JUDGE_ROLE, SPEND_KEY, daily_spend
|
||
from app.engine.runner import run_manager, verify_manifest
|
||
from app.main import app
|
||
from app.providers.mock_model import MockChatModel
|
||
|
||
PRICED = "claude-sonnet-5"
|
||
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
|
||
#: 一句挂了依据的结论、一句没挂依据的归因、一句连接性的话
|
||
TEXT = "## 本周\n\n本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。下面看细节。"
|
||
CITED, CAUSAL, LINK = "本周销售额 45,678.5元。", "增长主要来自新客首单。", "下面看细节。"
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
async def engine_up():
|
||
await run_manager.setup()
|
||
async with SessionLocal() as session:
|
||
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"])))
|
||
await session.commit()
|
||
yield
|
||
await run_manager.shutdown()
|
||
|
||
|
||
@pytest.fixture
|
||
async def client():
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
def node(nid, ntype, **config):
|
||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||
|
||
|
||
def chain(*nodes) -> dict:
|
||
ids = [n["id"] for n in nodes]
|
||
return {"nodes": list(nodes), "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
|
||
|
||
|
||
def weekly(*, gate=False, **report) -> dict:
|
||
return chain(
|
||
node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]),
|
||
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
|
||
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
|
||
node("write", "report", instructions="写周报", **report),
|
||
*([node("gate", "human", mode="approve", title="确认")] if gate else []),
|
||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
|
||
)
|
||
|
||
|
||
class Models:
|
||
"""写作者回 text；裁判按句子里的字给判定（verdicts：{句中片段: 判定}，没对上的是 supported）。
|
||
judges 记下每次交给裁判的请求原文，specs 记下每次要的裁判模型。"""
|
||
|
||
def __init__(self, monkeypatch, text=TEXT, verdicts=None, *, model=PRICED, fail_judge=None, delay=0.0):
|
||
self.judges: list[str] = []
|
||
self.specs: list = []
|
||
verdicts = verdicts or {}
|
||
|
||
def decide(chat, messages):
|
||
if JUDGE_ROLE in str(messages[0].content):
|
||
request = str(messages[-1].content)
|
||
self.judges.append(request)
|
||
if fail_judge:
|
||
raise RuntimeError(fail_judge)
|
||
items = [{"unit": uid,
|
||
"verdict": next((s for frag, s in verdicts.items() if frag in line), "supported"),
|
||
"rationale": f"判的是：{line.strip()[:16]}", "used": []}
|
||
for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M)]
|
||
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
|
||
return AIMessage(content=text)
|
||
|
||
monkeypatch.setattr(MockChatModel, "_decide", decide)
|
||
|
||
async def judge_model_of(session, spec):
|
||
self.specs.append(spec)
|
||
if delay:
|
||
await asyncio.sleep(delay)
|
||
return MockChatModel(model_name=spec.model or model), spec.model or model
|
||
|
||
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
|
||
|
||
|
||
async def wait(run_id: str, statuses=("succeeded", "failed")) -> Run:
|
||
for _ in range(400):
|
||
await asyncio.sleep(0.05)
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Run, run_id)
|
||
if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None):
|
||
return row
|
||
raise AssertionError(f"run {run_id} 没跑完：{row.status}")
|
||
|
||
|
||
async def finish(graph, *, run_class="exploratory", statuses=("succeeded", "failed")) -> Run:
|
||
run = await run_manager.start(graph=graph, input_payload={}, run_class=run_class)
|
||
row = await wait(run.id, statuses)
|
||
return row
|
||
|
||
|
||
def doc_of(row: Run) -> dict:
|
||
return artifact_store.load(row.output["_evidence"]["doc_artifact"])
|
||
|
||
|
||
def unit_ids(doc: dict) -> dict[str, str]:
|
||
md = doc["markdown"]
|
||
return {md[u["span"][0]:u["span"][1]].strip(): u["id"] for _, u in iter_units(doc)}
|
||
|
||
|
||
def unit_starting(doc: dict, prefix: str) -> str:
|
||
return next(uid for text, uid in unit_ids(doc).items() if text.startswith(prefix))
|
||
|
||
|
||
def seg_in(doc: dict, uid: str) -> str:
|
||
return next(s["id"] for _, u in iter_units(doc) if u["id"] == uid for s in u["segments"] if s["kind"] == "text")
|
||
|
||
|
||
async def judged_events(run_id: str) -> list[RunEvent]:
|
||
async with SessionLocal() as session:
|
||
return list((await session.execute(select(RunEvent).where(
|
||
RunEvent.run_id == run_id, RunEvent.type == EventType.EVIDENCE_JUDGED).order_by(RunEvent.seq))).scalars())
|
||
|
||
|
||
async def judge(client, run_id: str, units, **extra):
|
||
return await client.post(f"/api/runs/{run_id}/evidence/judge", json={"units": units, **extra})
|
||
|
||
|
||
async def save_setting(key: str, value) -> None:
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Setting, key)
|
||
if row is None:
|
||
session.add(Setting(key=key, value=value))
|
||
else:
|
||
row.value = value
|
||
await session.commit()
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 判定落在封存之后，封存核对照旧一致
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_an_on_demand_verdict_lands_after_the_seal_and_the_seal_still_holds(client, monkeypatch):
|
||
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||
row = await finish(weekly(claims="judge"))
|
||
assert row.status == "succeeded" and models.judges == [], "探索运行节点里不判"
|
||
doc = doc_of(row)
|
||
ids = unit_ids(doc)
|
||
|
||
res = await judge(client, row.id, [ids[CAUSAL]])
|
||
assert res.status_code == 200, res.text
|
||
body = res.json()
|
||
verdict = body["verdicts"][ids[CAUSAL]]
|
||
assert verdict["status"] == "contradicted" and verdict["post_seal"] is True and verdict["judge"] == PRICED
|
||
assert verdict["rationale"] and body["judged"] == [ids[CAUSAL]] and body["reused"] == []
|
||
assert body["report"] == {"node_id": "write", "doc_artifact": row.output["_evidence"]["doc_artifact"]}
|
||
assert body["post_seal"] is True and body["limited"] is False and body["calls"] == 1
|
||
assert len(models.judges) == 1 and CAUSAL in models.judges[0] and CITED not in models.judges[0]
|
||
|
||
[event] = await judged_events(row.id)
|
||
assert event.seq > row.manifest_seq and event.node_id == "write" and body["event"] == {"seq": event.seq}
|
||
data = event.data
|
||
assert data["post_seal"] is True and data["report"] == "write"
|
||
assert data["doc_artifact"] == body["report"]["doc_artifact"]
|
||
assert data["verdicts"] == {ids[CAUSAL]: verdict} and data["model"] == PRICED and data["calls"] == 1
|
||
assert set(data["keys"]) == {ids[CAUSAL]} and data["units"] == [ids[CAUSAL]]
|
||
|
||
check = await verify_manifest(row.id)
|
||
assert check["ok"] is True and check["sealed_at"] == row.manifest_seq
|
||
assert (await client.get(f"/api/runs/{row.id}/verify")).json()["ok"] is True
|
||
async with SessionLocal() as session:
|
||
after = await session.get(Run, row.id)
|
||
assert after.manifest_seq == row.manifest_seq and after.last_seq == event.seq
|
||
assert doc_of(row) == doc, "封存的文档一个字都不改"
|
||
csv = await client.get(f"/api/runs/{row.id}/evidence/audit", params={"format": "csv"})
|
||
assert csv.headers["x-evidence-seal"] == "ok"
|
||
|
||
|
||
async def test_any_claim_in_an_explore_report_can_be_judged(client, monkeypatch):
|
||
"""报告节点没开 claims: judge 也能按需判：结论句都是候选。"""
|
||
models = Models(monkeypatch)
|
||
row = await finish(weekly())
|
||
ids = unit_ids(doc_of(row))
|
||
body = (await judge(client, row.id, [ids[CITED], ids[CAUSAL]])).json()
|
||
assert {u: v["status"] for u, v in body["verdicts"].items()} == {ids[CITED]: "supported", ids[CAUSAL]: "supported"}
|
||
assert len(models.judges) == 1, "一次点击的几句合成一次调用"
|
||
assert "【m:gmv】" in models.judges[0], "挂的依据摘给裁判"
|
||
|
||
|
||
async def test_sequence_numbers_continue_after_a_restart(client, monkeypatch):
|
||
"""进程重启后内存里的序号从 0 起：追加的事件要接在库里最后一条后面，不能和已有的撞号。"""
|
||
from app.core.bus import bus
|
||
|
||
Models(monkeypatch)
|
||
row = await finish(weekly())
|
||
ids = unit_ids(doc_of(row))
|
||
bus._seq.pop(row.id, None)
|
||
await judge(client, row.id, [ids[CAUSAL]])
|
||
[event] = await judged_events(row.id)
|
||
async with SessionLocal() as session:
|
||
seqs = [s for (s,) in await session.execute(select(RunEvent.seq).where(RunEvent.run_id == row.id))]
|
||
assert event.seq == max(seqs) and seqs.count(event.seq) == 1 and event.seq > row.manifest_seq
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 同一句不重复判
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_asking_again_returns_the_verdict_without_another_call(client, monkeypatch):
|
||
models = Models(monkeypatch, verdicts={"新客": "partial"})
|
||
row = await finish(weekly(claims="judge"))
|
||
ids = unit_ids(doc_of(row))
|
||
first = (await judge(client, row.id, [ids[CAUSAL]])).json()
|
||
spent = (await daily_spend())["calls"]
|
||
again = (await judge(client, row.id, [ids[CAUSAL]])).json()
|
||
assert len(models.judges) == 1 and (await daily_spend())["calls"] == spent, "不再调用、不再计费"
|
||
assert again["verdicts"] == first["verdicts"] and again["reused"] == [ids[CAUSAL]] and again["judged"] == []
|
||
assert again["calls"] == 0 and again["event"] is None and again["message"] is None
|
||
assert len(await judged_events(row.id)) == 1
|
||
|
||
# 判过的和没判过的一起点：只判没判过的
|
||
both = (await judge(client, row.id, [ids[CAUSAL], ids[CITED]])).json()
|
||
assert both["reused"] == [ids[CAUSAL]] and both["judged"] == [ids[CITED]]
|
||
assert len(models.judges) == 2 and CAUSAL not in models.judges[1]
|
||
|
||
|
||
async def test_a_double_click_is_judged_once(client, monkeypatch):
|
||
models = Models(monkeypatch, delay=0.05)
|
||
row = await finish(weekly(claims="judge"))
|
||
uid = unit_ids(doc_of(row))[CAUSAL]
|
||
one, two = await asyncio.gather(judge(client, row.id, [uid]), judge(client, row.id, [uid]))
|
||
assert one.status_code == two.status_code == 200
|
||
assert len(models.judges) == 1 and len(await judged_events(row.id)) == 1
|
||
assert sorted([one.json()["reused"], two.json()["reused"]]) == [[], [uid]]
|
||
|
||
|
||
async def test_a_failed_attempt_can_be_retried(client, monkeypatch):
|
||
Models(monkeypatch, fail_judge="网关 502")
|
||
row = await finish(weekly(claims="judge"))
|
||
uid = unit_ids(doc_of(row))[CAUSAL]
|
||
failed = (await judge(client, row.id, [uid])).json()
|
||
assert failed["verdicts"][uid]["status"] == "unjudged" and failed["verdicts"][uid]["reason"] == "error"
|
||
assert "网关 502" in failed["message"] and failed["limited"] is False
|
||
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||
retried = (await judge(client, row.id, [uid])).json()
|
||
assert retried["verdicts"][uid]["status"] == "contradicted" and retried["reused"] == [] and len(models.judges) == 1
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 上限
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_hitting_the_click_limit_is_a_result_not_an_error(client, monkeypatch):
|
||
models = Models(monkeypatch)
|
||
await save_setting("judge", {"click_max_cost_usd": 1e-9})
|
||
row = await finish(weekly(claims="judge"))
|
||
uid = unit_ids(doc_of(row))[CAUSAL]
|
||
res = await judge(client, row.id, [uid])
|
||
assert res.status_code == 200
|
||
body = res.json()
|
||
assert body["limited"] is True and body["limits_hit"] == ["max_cost_usd"]
|
||
assert body["unjudged"] == {"max_cost_usd": 1}
|
||
assert body["message"].startswith("已达上限") and "这次点击" in body["message"]
|
||
assert any("设置" in how and "不限" in how for how in body["adjust"])
|
||
verdict = body["verdicts"][uid]
|
||
assert verdict["status"] == "unjudged" and verdict["reason"] == "max_cost_usd" and verdict["post_seal"] is True
|
||
assert models.judges == [] and body["calls"] == 0 and body["budget"]["max_cost_usd"] == 1e-9
|
||
assert body["event"] is None and await judged_events(row.id) == [], "没问模型就没有判定可记"
|
||
|
||
|
||
async def test_the_daily_limit_applies_to_clicks_too(client, monkeypatch):
|
||
models = Models(monkeypatch)
|
||
today = (await daily_spend())["date"]
|
||
await save_setting(SPEND_KEY, {"date": today, "usd": 2.0, "calls": 9, "unpriced_calls": 0})
|
||
row = await finish(weekly(claims="judge"))
|
||
body = (await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])).json()
|
||
assert body["limits_hit"] == ["daily_max_usd"] and "每日" in body["message"] and models.judges == []
|
||
assert body["spend"]["usd"] == 2.0 and body["spend"]["daily_max_usd"] == 2.0
|
||
|
||
|
||
async def test_unlimited_click_budget_judges(client, monkeypatch):
|
||
models = Models(monkeypatch)
|
||
await save_setting("judge", {"click_max_cost_usd": None, "daily_max_usd": None})
|
||
row = await finish(weekly(claims="judge"))
|
||
body = (await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])).json()
|
||
assert body["limited"] is False and len(models.judges) == 1
|
||
assert body["budget"]["max_cost_usd"] is None and body["budget"]["daily_max_usd"] is None
|
||
assert body["spend"]["calls"] == 1 and body["spend"]["daily_max_usd"] is None
|
||
|
||
|
||
async def test_the_sentence_limit_keeps_what_was_judged(client, monkeypatch):
|
||
models = Models(monkeypatch)
|
||
row = await finish(weekly(claims="judge", judge={"max_claims": 1}))
|
||
ids = unit_ids(doc_of(row))
|
||
body = (await judge(client, row.id, [ids[CITED], ids[CAUSAL]])).json()
|
||
statuses = {u: v["status"] for u, v in body["verdicts"].items()}
|
||
assert statuses == {ids[CITED]: "supported", ids[CAUSAL]: "unjudged"} and len(models.judges) == 1
|
||
assert body["limits_hit"] == ["max_claims"] and body["verdicts"][ids[CAUSAL]]["reason"] == "max_claims"
|
||
[event] = await judged_events(row.id)
|
||
assert set(event.data["verdicts"]) == {ids[CITED], ids[CAUSAL]}, "问过模型的这一次连同没判的一起记下"
|
||
# 没判的那句还能再点
|
||
again = (await judge(client, row.id, [ids[CAUSAL]])).json()
|
||
assert again["judged"] == [ids[CAUSAL]] and again["reused"] == []
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 用哪个模型
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_the_judge_model_follows_node_then_settings(client, monkeypatch):
|
||
models = Models(monkeypatch)
|
||
await save_setting("judge", {"model": "judge-from-settings"})
|
||
row = await finish(weekly(claims="judge"))
|
||
await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])
|
||
[spec] = models.specs
|
||
assert spec.model == "judge-from-settings" and spec.thinking == "off" and spec.max_tokens == 16384
|
||
|
||
row = await finish(weekly(claims="judge", judge={"model": "judge-from-node"}))
|
||
body = (await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])).json()
|
||
assert models.specs[-1].model == "judge-from-node" and body["model"] == "judge-from-node"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 什么时候不判
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_formal_runs_are_judged_inside_the_node_not_on_demand(client, monkeypatch):
|
||
models = Models(monkeypatch)
|
||
row = await finish(weekly(claims="judge", judge={"max_cost_usd": 0.05}), run_class="formal")
|
||
assert row.status == "succeeded", row.error
|
||
calls = len(models.judges)
|
||
res = await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])
|
||
assert res.status_code == 409 and res.json()["code"] == "evidence_judge_formal"
|
||
assert "正式运行的结论句已在报告撰写节点内完成裁判" in res.json()["detail"]
|
||
assert len(models.judges) == calls and await judged_events(row.id) == []
|
||
|
||
|
||
async def test_a_run_paused_at_an_approval_is_not_judged(client, monkeypatch):
|
||
"""停在审批上的运行还没封存，按需裁判不接。交给审批的报告探索运行也在节点里判过了
|
||
（test_approval_report_check），这里只看点击：不再多一次裁判调用，也不追加判定事件。"""
|
||
models = Models(monkeypatch)
|
||
row = await finish(weekly(claims="judge", gate=True), statuses=("interrupted",))
|
||
judged_in_node = list(models.judges)
|
||
assert judged_in_node
|
||
res = await judge(client, row.id, ["u1"])
|
||
assert res.status_code == 409 and res.json()["code"] == "evidence_judge_unsealed"
|
||
assert models.judges == judged_in_node and await judged_events(row.id) == []
|
||
|
||
|
||
async def test_runs_from_before_the_upgrade_are_not_judged(client, monkeypatch):
|
||
async def no_snapshot(session):
|
||
return None
|
||
|
||
monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot)
|
||
models = Models(monkeypatch)
|
||
row = await finish(weekly())
|
||
res = await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])
|
||
assert res.status_code == 409 and res.json()["code"] == "evidence_judge_legacy"
|
||
assert models.judges == [] and await judged_events(row.id) == []
|
||
|
||
|
||
async def test_a_broken_seal_is_not_judged(client, monkeypatch):
|
||
models = Models(monkeypatch)
|
||
row = await finish(weekly(claims="judge"))
|
||
async with SessionLocal() as session:
|
||
event = (await session.execute(select(RunEvent).where(
|
||
RunEvent.run_id == row.id, RunEvent.type == "node.finished", RunEvent.node_id == "fetch"))).scalar_one()
|
||
event.data = {**event.data, "tampered": True}
|
||
await session.commit()
|
||
res = await judge(client, row.id, [unit_ids(doc_of(row))[CAUSAL]])
|
||
assert res.status_code == 409 and res.json()["code"] == "evidence_seal_broken" and models.judges == []
|
||
|
||
|
||
@pytest.mark.parametrize("meanwhile", [
|
||
# 裁判的这几秒里有人点了「继续运行」（失败的运行可以接着跑）：运行又活了
|
||
{"status": "running", "error": None},
|
||
# 接着跑完、重新封存了：manifest_seq 和清单哈希都换了，这次点击依据的那份封存已经不是最新的
|
||
{"manifest_seq": 10_000, "manifest_hash": "0" * 64},
|
||
])
|
||
async def test_a_run_continued_during_the_judge_call_records_no_event(client, monkeypatch, meanwhile):
|
||
"""判定只接在点击开始时的那份封存后面。运行在裁判期间被接着跑了，再追加就落进一次活着的运行中间，
|
||
还会被封进下一份清单：不记进运行记录，判定照样交回给点的人，并说明为什么没记。"""
|
||
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||
row = await finish(weekly(claims="judge"))
|
||
uid = unit_ids(doc_of(row))[CAUSAL]
|
||
ask_model = judge_mod.get_chat_model
|
||
|
||
async def continued_while_judging(session, spec):
|
||
async with SessionLocal() as other:
|
||
await other.execute(update(Run).where(Run.id == row.id).values(**meanwhile))
|
||
await other.commit()
|
||
return await ask_model(session, spec)
|
||
|
||
monkeypatch.setattr(judge_mod, "get_chat_model", continued_while_judging)
|
||
res = await judge(client, row.id, [uid])
|
||
assert res.status_code == 200, res.text
|
||
body = res.json()
|
||
assert len(models.judges) == 1 and body["calls"] == 1, "模型问过了"
|
||
assert body["verdicts"][uid]["status"] == "contradicted" and body["judged"] == [uid], "判定照样交回"
|
||
assert body["event"] is None and "运行已继续执行" in body["message"] and "未写入运行记录" in body["message"]
|
||
assert await judged_events(row.id) == []
|
||
async with SessionLocal() as session:
|
||
after = await session.get(Run, row.id)
|
||
assert after.last_seq == row.last_seq, "运行记录一条都没多"
|
||
|
||
|
||
async def test_a_continue_that_wins_the_row_between_check_and_write_gets_no_event(client, monkeypatch):
|
||
"""核对过状态、还没写入的那一下，接着跑的条件更新（runner._claim）先提交了：追加判定的条件更新落空，
|
||
照样不记——核对和写入是同一行上的条件更新，不是先读后写。"""
|
||
from app.core.bus import bus
|
||
from app.core.config import settings
|
||
|
||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||
row = await finish(weekly(claims="judge"))
|
||
uid = unit_ids(doc_of(row))[CAUSAL]
|
||
set_seq = bus.set_seq
|
||
|
||
def claimed_in_between(run_id, value):
|
||
# _append_judged 读完运行、分配序号之前正好调用它：在这里用另一条连接把运行占成 running
|
||
if run_id == row.id:
|
||
db = sqlite3.connect(settings.db_path, timeout=5)
|
||
try:
|
||
db.execute("UPDATE runs SET status = 'running' WHERE id = ?", (row.id,))
|
||
db.commit()
|
||
finally:
|
||
db.close()
|
||
set_seq(run_id, value)
|
||
|
||
monkeypatch.setattr(bus, "set_seq", claimed_in_between)
|
||
body = (await judge(client, row.id, [uid])).json()
|
||
assert body["verdicts"][uid]["status"] == "contradicted" and body["event"] is None
|
||
assert "未写入运行记录" in body["message"] and await judged_events(row.id) == []
|
||
async with SessionLocal() as session:
|
||
after = await session.get(Run, row.id)
|
||
assert after.status == "running" and after.last_seq == row.last_seq
|
||
|
||
|
||
async def test_missing_runs_and_reports(client, monkeypatch):
|
||
Models(monkeypatch)
|
||
res = await judge(client, "nope", ["u1"])
|
||
assert res.status_code == 404 and res.json()["code"] == "run_not_found"
|
||
row = await finish(chain(node("start", "input"), node("out", "output", fields=[{"name": "a", "value": "x"}])))
|
||
res = await judge(client, row.id, ["u1"])
|
||
assert res.status_code == 404 and res.json()["code"] == "evidence_report_not_found"
|
||
|
||
|
||
@pytest.mark.parametrize("payload", [{}, {"units": []}, {"units": "u1"}, {"units": [1]}, {"units": [""]},
|
||
{"units": [f"u{i}" for i in range(201)]}, {"units": ["u1"], "report": 3}])
|
||
async def test_bad_requests_are_422(client, monkeypatch, payload):
|
||
Models(monkeypatch)
|
||
row = await finish(weekly())
|
||
res = await client.post(f"/api/runs/{row.id}/evidence/judge", json=payload)
|
||
assert res.status_code == 422 and res.json()["code"] == "evidence_judge_bad_units"
|
||
|
||
|
||
async def test_sentences_that_are_not_claims_are_skipped(client, monkeypatch):
|
||
models = Models(monkeypatch)
|
||
row = await finish(weekly(claims="judge"))
|
||
ids = unit_ids(doc_of(row))
|
||
body = (await judge(client, row.id, [ids["本周"], ids[LINK], "u999"])).json()
|
||
assert body["skipped"] == {ids["本周"]: "not_a_claim", ids[LINK]: "not_a_claim", "u999": "not_found"}
|
||
assert body["verdicts"] == {} and models.judges == [] and body["event"] is None
|
||
assert "不是结论句" in body["message"]
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 片段接口、证据图带上判定
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_the_segment_shows_the_verdict_and_whether_it_can_still_be_asked(client, monkeypatch):
|
||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||
row = await finish(weekly(claims="judge"))
|
||
doc = doc_of(row)
|
||
ids = unit_ids(doc)
|
||
before = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, ids[CAUSAL])}")).json()
|
||
assert before["unit"]["verdict"]["status"] == "unjudged" and before["unit"]["verdict"]["reason"] == "on_demand"
|
||
assert before["unit"]["on_demand"] == {"available": True, "reason": None, "message": None}
|
||
|
||
await judge(client, row.id, [ids[CAUSAL]])
|
||
after = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, ids[CAUSAL])}")).json()
|
||
assert after["unit"]["verdict"]["status"] == "contradicted" and after["unit"]["verdict"]["post_seal"] is True
|
||
assert after["unit"]["on_demand"]["available"] is False and after["unit"]["on_demand"]["reason"] == "judged"
|
||
assert "模型" in after["note"] and "后续版本" not in after["note"]
|
||
other = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, ids[CITED])}")).json()
|
||
assert other["unit"]["on_demand"]["available"] is True
|
||
assert "附有依据" in other["note"] and "模型" in other["note"] and "后续版本" not in other["note"]
|
||
link = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, ids[LINK])}")).json()
|
||
assert link["unit"]["on_demand"]["reason"] == "not_a_claim" and "verdict" not in link["unit"]
|
||
|
||
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
|
||
[report] = graph["reports"]
|
||
assert report["claims"] == "judge" and report["judge"]["mode"] == "on_demand"
|
||
assert report["post_seal_verdicts"] == {ids[CAUSAL]: after["unit"]["verdict"]}
|
||
audit = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json()
|
||
assert audit["reports"][0]["post_seal_verdicts"] == report["post_seal_verdicts"]
|
||
|
||
|
||
async def test_sealed_verdicts_of_a_formal_run_are_shown_as_sealed(client, monkeypatch):
|
||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||
row = await finish(weekly(claims="judge", judge={"max_cost_usd": 0.05}), run_class="formal")
|
||
doc = doc_of(row)
|
||
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, unit_ids(doc)[CAUSAL])}")).json()
|
||
assert body["unit"]["verdict"]["status"] == "contradicted" and body["unit"]["verdict"]["post_seal"] is False
|
||
assert body["unit"]["on_demand"]["reason"] == "formal"
|
||
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
|
||
assert graph["reports"][0]["post_seal_verdicts"] == {} and graph["reports"][0]["judge"]["mode"] == "inline"
|
||
|
||
|
||
async def test_a_forged_event_cannot_overwrite_a_sealed_verdict(client, monkeypatch):
|
||
"""封存后追加的判定只填补没判过的句子：封存内判过的，后面追加什么都盖不掉。"""
|
||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||
row = await finish(weekly(claims="judge", judge={"max_cost_usd": 0.05}), run_class="formal")
|
||
doc = doc_of(row)
|
||
uid = unit_ids(doc)[CAUSAL]
|
||
doc_artifact = row.output["_evidence"]["doc_artifact"]
|
||
await run_manager.note(row.id, EventType.EVIDENCE_JUDGED, node_id="write", report="write",
|
||
doc_artifact=doc_artifact, post_seal=True,
|
||
verdicts={uid: {"status": "supported", "rationale": "改好了", "judge": "x", "post_seal": True}})
|
||
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, uid)}")).json()
|
||
assert body["unit"]["verdict"]["status"] == "contradicted"
|
||
|
||
|
||
async def test_appended_verdicts_always_read_as_post_seal(client, monkeypatch):
|
||
"""追加的判定不管载荷里怎么写，展示出来都是「封存后追加」：只有随文档封存的判定才算封存内的。"""
|
||
Models(monkeypatch)
|
||
row = await finish(weekly(claims="judge"))
|
||
doc = doc_of(row)
|
||
uid = unit_ids(doc)[CAUSAL]
|
||
await run_manager.note(row.id, EventType.EVIDENCE_JUDGED, node_id="write", report="write",
|
||
doc_artifact=row.output["_evidence"]["doc_artifact"],
|
||
verdicts={uid: {"status": "supported", "rationale": "x", "judge": "x", "post_seal": False},
|
||
"u999": {"status": "supported", "rationale": "x", "judge": "x"}})
|
||
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, uid)}")).json()
|
||
assert body["unit"]["verdict"]["status"] == "supported" and body["unit"]["verdict"]["post_seal"] is True
|
||
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
|
||
assert set(graph["reports"][0]["post_seal_verdicts"]) == {uid}, "文档里没有的句子不认"
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 裁判只看得到封存范围内的证据
|
||
# --------------------------------------------------------------------------
|
||
|
||
SOURCE = "warehouse"
|
||
TOOL = f"db_query__{SOURCE}"
|
||
SQL = "SELECT region, SUM(amount) AS gmv, COUNT(*) AS order_cnt FROM orders GROUP BY region ORDER BY gmv DESC"
|
||
EAST, WEST = "东区销售额 [[v:Q1.r0.gmv]]。[[see:Q1]]", "西区销售额 [[v:Q1.r1.gmv]]。[[see:Q1]]"
|
||
|
||
|
||
@pytest.fixture
|
||
async def warehouse(tmp_path):
|
||
from app.data.introspect import introspect
|
||
|
||
path = tmp_path / "warehouse.db"
|
||
db = sqlite3.connect(path)
|
||
db.executescript("CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, amount REAL);")
|
||
db.executemany("INSERT INTO orders VALUES (?, ?, ?)", [(1, "east", 100.5), (2, "west", 200.0), (3, "east", 300.0)])
|
||
db.commit()
|
||
db.close()
|
||
probe = SimpleNamespace(id=f"probe-{path.name}", name=SOURCE, kind="sqlite", database=str(path), host=None,
|
||
port=None, username=None, password=None, options={}, readonly=True, schema_cache={})
|
||
cache = await introspect(probe)
|
||
await engines.invalidate(probe.id)
|
||
async with SessionLocal() as session:
|
||
row = (await session.execute(select(DataSource).where(DataSource.name == SOURCE))).scalar_one_or_none()
|
||
if row is None:
|
||
row = DataSource(name=SOURCE, kind="sqlite", database=str(path), readonly=True, enabled=True)
|
||
session.add(row)
|
||
row.database, row.kind, row.enabled, row.readonly, row.schema_cache = str(path), "sqlite", True, True, cache
|
||
row.options = {}
|
||
await session.commit()
|
||
source_id = row.id
|
||
await engines.invalidate(source_id)
|
||
yield source_id
|
||
await engines.invalidate(source_id)
|
||
|
||
|
||
def queried() -> dict:
|
||
return chain(node("start", "input"), node("fetch", "tool", tool=TOOL, args={"sql": SQL}),
|
||
node("write", "report", instructions="写周报"),
|
||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]))
|
||
|
||
|
||
async def reseal(run_id: str) -> None:
|
||
from app.core.artifact_store import manifest_hash
|
||
|
||
async with SessionLocal() as session:
|
||
run = await session.get(Run, run_id)
|
||
rows = [tuple(r) for r in await session.execute(
|
||
select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data)
|
||
.where(RunEvent.run_id == run_id, RunEvent.seq <= run.manifest_seq).order_by(RunEvent.seq))]
|
||
run.manifest_hash = manifest_hash(rows)
|
||
await session.commit()
|
||
|
||
|
||
async def unseal_queries(run_id: str) -> str:
|
||
"""把查询快照从封存的事件里摘掉、按改过的事件重新封存：快照还在工件库和工件表里，封存链完好，
|
||
只是没有哪条封存的事件引用它（等于「人为插进工件表」）。返回那件快照的 id。"""
|
||
async with SessionLocal() as session:
|
||
rows = list((await session.execute(select(RunEvent).where(
|
||
RunEvent.run_id == run_id, RunEvent.type.in_(["tool.end", "node.finished"])))).scalars())
|
||
artifact = None
|
||
for event in rows:
|
||
data = dict(event.data or {})
|
||
if event.type == "tool.end":
|
||
artifact = data.pop("query_artifact", None) or artifact
|
||
data.pop("schema_artifact", None)
|
||
elif data.get("evidence"):
|
||
data["evidence"] = [e for e in data["evidence"] if e.get("kind") not in ("query", "schema")]
|
||
event.data = data
|
||
await session.commit()
|
||
await reseal(run_id)
|
||
return artifact
|
||
|
||
|
||
def excerpt_lines(request: str) -> tuple[str | None, list[str]]:
|
||
header = re.search(r"^列：(.*)$", request, re.M)
|
||
return (header.group(1) if header else None), re.findall(r"^r\d+：.*$", request, re.M)
|
||
|
||
|
||
async def test_only_sealed_snapshots_reach_the_judge(client, monkeypatch, warehouse):
|
||
models = Models(monkeypatch, text=EAST + WEST)
|
||
control = await finish(queried())
|
||
assert control.status == "succeeded", control.error
|
||
await judge(client, control.id, [unit_starting(doc_of(control), "东区")])
|
||
assert "【Q1】查询结果" in models.judges[0] and "SQL：SELECT region" in models.judges[0]
|
||
assert excerpt_lines(models.judges[0])[1], "封存链完好时照常摘被引用的行"
|
||
|
||
row = await finish(queried())
|
||
doc = doc_of(row)
|
||
artifact = await unseal_queries(row.id)
|
||
assert artifact and artifact_store.load(artifact) is not None, "快照本身还在工件库里"
|
||
assert (await verify_manifest(row.id))["ok"] is True
|
||
res = await judge(client, row.id, [unit_starting(doc, "东区")])
|
||
assert res.status_code == 200, res.text
|
||
assert "【Q1】" not in models.judges[1] and "SELECT" not in models.judges[1]
|
||
assert excerpt_lines(models.judges[1]) == (None, [])
|
||
|
||
# 封存之后才追加的事件引用了它，也不算
|
||
async with SessionLocal() as session:
|
||
run = await session.get(Run, row.id)
|
||
seq = run.last_seq
|
||
from app.core.bus import bus
|
||
|
||
bus.set_seq(row.id, seq)
|
||
await run_manager.note(row.id, "tool.end", node_id="fetch", tool=TOOL, query_artifact=artifact)
|
||
await judge(client, row.id, [unit_starting(doc, "西区")])
|
||
assert "【Q1】" not in models.judges[2] and "SELECT" not in models.judges[2]
|
||
|
||
|
||
async def test_masked_columns_never_reach_the_judge(client, monkeypatch, warehouse):
|
||
models = Models(monkeypatch, text=EAST)
|
||
row = await finish(queried())
|
||
async with SessionLocal() as session:
|
||
source = await session.get(DataSource, warehouse)
|
||
source.options = {"mask_columns": ["order_cnt"]}
|
||
await session.commit()
|
||
await judge(client, row.id, [unit_starting(doc_of(row), "东区")])
|
||
header, lines = excerpt_lines(models.judges[0])
|
||
assert header is not None and "order_cnt" not in header and "gmv" in header
|
||
assert lines and all(line.count("|") == 1 for line in lines), lines
|
||
assert "遮罩" in models.judges[0]
|