mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 04:31:58 +00:00
ThearchyHelios
bd8306bf95
- 判定取值改为：支持 / 部分支持 / 证据相矛盾 / 证据不足 / 不是结论。只有证据和句子冲突才判相矛盾； 证据里没有相关信息判证据不足，并写明缺什么（比如 SQL、字段清单） - 出具判档不变松：证据相矛盾按 on_unsupported 降档或不予出具；证据不足记一条缺口，最多降档、不能判完整出具； 改造前的「证据不支持」照旧当相矛盾处理 - 补全证据摘录：表名附字段清单和类型（最多 60 个，快照不全时注明，遮罩列不列）；口径卡指标附上每个输入 取自哪次查询、哪一格，以及那次查询的 SQL；一律只读封存范围内的工件 - 理由上限从 60 字放宽到 120 字，在句子边界截断，不再截在半句中间 - 裁判模型：按需裁判也判断是否和写作模型相同、是否在价格表里，证据面板里醒目提示； 设置页没配裁判模型时写明「未单独配置，将使用助手的模型」 - 新判定带规则版本：改造前的判定在界面上标「按旧规则判定」，可以请模型按当前规则重新判断 - 界面：证据相矛盾用红色；证据不足用中性色和 ○，写明缺什么；讲方法、讲结构的句子在证据面板里直接列出 对应的 SQL 和字段清单；出具横幅、审计表、报告节点印章的计数都分开这两档 - 顺带写稳 check-publish 里一条等按钮状态的断言（并行时偶发）
487 lines
24 KiB
Python
487 lines
24 KiB
Python
"""结论句裁判改造（JR）的接口：片段接口的字段清单、按需裁判的模型标记、证据图的 judge_meta、审计行。
|
||
|
||
- 片段接口里表名实体那一步带上字段清单 fields: [{name, type}]（封存范围内的表结构快照，最多 60 个，
|
||
多出来的记 fields_more），遮罩的列不列
|
||
- 按需裁判也判断裁判模型和写作模型是否相同（写作模型取这份报告撰写节点实际用的模型），并带上
|
||
priced、writer_model；evidence.judged 事件同样记下
|
||
- GET /evidence 每份报告带 judge_meta：最近一次按需裁判的 {model, same_model, priced, writer_model}
|
||
- 交给裁判的新摘录（字段清单、指标输入的 SQL）只经封存范围内的 loader 读
|
||
- 审计行的 issue 新增 contradicted_claim、insufficient_claim
|
||
- 旧的 unsupported 判定照旧算「判过」：封存之后追加的判定盖不掉它
|
||
|
||
所有裁判都用 mock 模型给剧本，不调真接口。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import csv
|
||
import io
|
||
import json
|
||
import re
|
||
import sqlite3
|
||
from types import SimpleNamespace
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
from langchain_core.messages import AIMessage
|
||
from sqlalchemy import delete, select
|
||
|
||
from app.core import artifact_store
|
||
from app.core.events import EventType
|
||
from app.data.engine import engines
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import DataSource, Run, RunEvent, Setting
|
||
from app.engine import judge as judge_mod
|
||
from app.engine.evidence import iter_segments, iter_units
|
||
from app.engine.judge import JUDGE_ROLE, SPEND_KEY
|
||
from app.engine.runner import run_manager
|
||
from app.main import app
|
||
from app.providers.mock_model import MockChatModel
|
||
|
||
PRICED = "claude-sonnet-5"
|
||
WRITER = "mock-fast"
|
||
SOURCE = "jr_shop"
|
||
TOOL = f"db_query__{SOURCE}"
|
||
SQL = "SELECT region, SUM(amount) AS gmv FROM orders GROUP BY region ORDER BY gmv DESC"
|
||
WIDE_COLS = [f"f{i:02d}" for i in range(65)]
|
||
METRIC = "东区销售额最高，达 [[m:gmv]]。[[see:m:gmv]]"
|
||
METHOD = "这个数由 [[t:orders]] 按 [[c:orders.region]] 分组求和得出。"
|
||
WIDE = "明细在 [[t:wide_orders]] 里。"
|
||
TEXT = METRIC + METHOD + WIDE
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
async def engine_up():
|
||
await run_manager.setup()
|
||
async with SessionLocal() as session:
|
||
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"])))
|
||
await session.commit()
|
||
yield
|
||
await run_manager.shutdown()
|
||
|
||
|
||
@pytest.fixture
|
||
async def client():
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
async def shop(tmp_path):
|
||
"""探查过结构的示例库：orders（带手机号列，数据源设了遮罩）和一张 65 列的宽表。"""
|
||
from app.data.introspect import introspect
|
||
|
||
path = tmp_path / "jr_shop.db"
|
||
db = sqlite3.connect(path)
|
||
db.executescript(
|
||
"CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, amount REAL, phone TEXT);"
|
||
f"CREATE TABLE wide_orders ({', '.join(f'{c} TEXT' for c in WIDE_COLS)});")
|
||
db.executemany("INSERT INTO orders VALUES (?, ?, ?, ?)",
|
||
[(1, "east", 300.5, "p1"), (2, "west", 200.0, "p2"), (3, "east", 100.0, "p3")])
|
||
db.commit()
|
||
db.close()
|
||
probe = SimpleNamespace(id=f"probe-{path.name}", name=SOURCE, kind="sqlite", database=str(path), host=None,
|
||
port=None, username=None, password=None, options={}, readonly=True, schema_cache={})
|
||
cache = await introspect(probe)
|
||
await engines.invalidate(probe.id)
|
||
async with SessionLocal() as session:
|
||
row = (await session.execute(select(DataSource).where(DataSource.name == SOURCE))).scalar_one_or_none()
|
||
if row is None:
|
||
row = DataSource(name=SOURCE, kind="sqlite", database=str(path), readonly=True, enabled=True)
|
||
session.add(row)
|
||
row.database, row.kind, row.enabled, row.readonly, row.schema_cache = str(path), "sqlite", True, True, cache
|
||
row.options = {"mask_columns": "phone"}
|
||
await session.commit()
|
||
source_id = row.id
|
||
await engines.invalidate(source_id)
|
||
yield source_id
|
||
await engines.invalidate(source_id)
|
||
|
||
|
||
def node(nid, ntype, **config):
|
||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||
|
||
|
||
def weekly(**report) -> dict:
|
||
nodes = [
|
||
node("start", "input"),
|
||
node("fetch", "tool", tool=TOOL, args={"sql": SQL}),
|
||
node("card", "metrics", caliber="周报口径", caliber_version="v2", metrics=[
|
||
{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "cell(nodes.fetch, 0, 'gmv')"}]),
|
||
node("write", "report", instructions="写周报", on_violation="flag", claims="judge", **report),
|
||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
|
||
]
|
||
ids = [n["id"] for n in nodes]
|
||
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
|
||
|
||
|
||
class Models:
|
||
"""写作者回 text；裁判按句子里的字给判定：verdicts = {句中片段: (判定, missing)}。judges 记下请求原文。"""
|
||
|
||
def __init__(self, monkeypatch, verdicts=None, *, model=PRICED, text=TEXT):
|
||
self.judges: list[str] = []
|
||
verdicts = verdicts or {}
|
||
|
||
def decide(chat, messages):
|
||
if JUDGE_ROLE in str(messages[0].content):
|
||
request = str(messages[-1].content)
|
||
self.judges.append(request)
|
||
items = []
|
||
for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M):
|
||
status, missing = next((v for frag, v in verdicts.items() if frag in line), ("supported", ""))
|
||
items.append({"unit": uid, "verdict": status, "rationale": f"判的是：{line.strip()[:12]}",
|
||
"missing": missing, "used": []})
|
||
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
|
||
return AIMessage(content=text)
|
||
|
||
monkeypatch.setattr(MockChatModel, "_decide", decide)
|
||
self.model = model
|
||
|
||
async def judge_model_of(session, spec):
|
||
return MockChatModel(model_name=self.model), self.model
|
||
|
||
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
|
||
|
||
|
||
async def wait(run_id: str) -> Run:
|
||
for _ in range(400):
|
||
await asyncio.sleep(0.05)
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Run, run_id)
|
||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||
return row
|
||
raise AssertionError(f"run {run_id} 没跑完：{row.status}")
|
||
|
||
|
||
async def explore_run(graph=None) -> tuple[Run, dict]:
|
||
started = await run_manager.start(graph=graph or weekly(), input_payload={}, run_class="exploratory")
|
||
row = await wait(started.id)
|
||
assert row.status == "succeeded", row.error
|
||
return row, artifact_store.load(row.output["_evidence"]["doc_artifact"])
|
||
|
||
|
||
def unit_of(doc: dict, prefix: str) -> str:
|
||
md = doc["markdown"]
|
||
return next(u["id"] for _, u in iter_units(doc) if md[u["span"][0]:u["span"][1]].strip().startswith(prefix))
|
||
|
||
|
||
async def judged_events(run_id: str) -> list[RunEvent]:
|
||
async with SessionLocal() as session:
|
||
return list((await session.execute(select(RunEvent).where(
|
||
RunEvent.run_id == run_id, RunEvent.type == EventType.EVIDENCE_JUDGED).order_by(RunEvent.seq))).scalars())
|
||
|
||
|
||
async def ask(client, run_id: str, units: list[str]) -> dict:
|
||
res = await client.post(f"/api/runs/{run_id}/evidence/judge", json={"units": units})
|
||
assert res.status_code == 200, res.text
|
||
return res.json()
|
||
|
||
|
||
async def reseal(run_id: str) -> None:
|
||
async with SessionLocal() as session:
|
||
run = await session.get(Run, run_id)
|
||
rows = [tuple(r) for r in await session.execute(
|
||
select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data)
|
||
.where(RunEvent.run_id == run_id, RunEvent.seq <= run.manifest_seq).order_by(RunEvent.seq))]
|
||
run.manifest_hash = artifact_store.manifest_hash(rows)
|
||
await session.commit()
|
||
|
||
|
||
async def rewrite(run_id: str, etype: str, node_id: str, change) -> None:
|
||
async with SessionLocal() as session:
|
||
for row in (await session.execute(select(RunEvent).where(
|
||
RunEvent.run_id == run_id, RunEvent.type == etype, RunEvent.node_id == node_id))).scalars():
|
||
row.data = change(dict(row.data))
|
||
await session.commit()
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 片段接口：表名实体步骤的字段清单
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def open_entity(client, run_id: str, doc: dict, ref: str) -> dict:
|
||
seg = next(s for s in iter_segments(doc) if s.get("kind") == "entity" and s.get("ref") == ref)
|
||
res = await client.get(f"/api/runs/{run_id}/evidence/segments/{seg['id']}")
|
||
assert res.status_code == 200, res.text
|
||
[step] = res.json()["chain"]
|
||
return step
|
||
|
||
|
||
async def test_a_table_step_lists_its_fields_without_masked_columns(client, monkeypatch):
|
||
Models(monkeypatch)
|
||
row, doc = await explore_run()
|
||
step = await open_entity(client, row.id, doc, "t:orders")
|
||
assert step["step"] == "entity" and step["kind"] == "table"
|
||
assert step["fields"] == [{"name": "id", "type": "INTEGER"}, {"name": "region", "type": "TEXT"},
|
||
{"name": "amount", "type": "REAL"}], "遮罩的 phone 不列"
|
||
assert "fields_more" not in step and step["columns"] == 4
|
||
|
||
|
||
async def test_a_wide_table_step_stops_at_sixty_fields(client, monkeypatch):
|
||
Models(monkeypatch)
|
||
row, doc = await explore_run()
|
||
step = await open_entity(client, row.id, doc, "t:wide_orders")
|
||
assert [f["name"] for f in step["fields"]] == WIDE_COLS[:60]
|
||
assert step["fields_more"] == 5
|
||
|
||
|
||
async def test_a_table_step_outside_the_seal_has_no_fields(client, monkeypatch):
|
||
Models(monkeypatch)
|
||
row, doc = await explore_run()
|
||
await rewrite(row.id, "tool.end", "fetch", lambda d: {k: v for k, v in d.items() if k != "schema_artifact"})
|
||
await rewrite(row.id, "node.finished", "fetch",
|
||
lambda d: {**d, "evidence": [e for e in d.get("evidence") or [] if e.get("kind") != "schema"]})
|
||
await reseal(row.id)
|
||
step = await open_entity(client, row.id, doc, "t:orders")
|
||
assert "fields" not in step and "fields_more" not in step
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 按需裁判：新的判定、摘录、模型标记
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_on_demand_judging_sees_fields_and_metric_sql_and_reports_the_models(client, monkeypatch):
|
||
models = Models(monkeypatch, verdicts={"分组求和": ("insufficient", "字段清单"), "东区销售额": ("contradicted", "")})
|
||
row, doc = await explore_run()
|
||
method, metric = unit_of(doc, "这个数由"), unit_of(doc, "东区销售额")
|
||
|
||
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
|
||
assert graph["reports"][0]["judge_meta"] is None, "还没按需裁判过"
|
||
|
||
body = await ask(client, row.id, [method, metric])
|
||
request = models.judges[0]
|
||
assert "region TEXT" in request and "id INTEGER" in request, "表附上字段清单和类型"
|
||
assert "phone" not in request.lower(), "遮罩的列不出现"
|
||
assert "查询 Q1 的 SQL" in request and "SUM(amount)" in request and "r0" in request, "指标附上输入的查询和格子"
|
||
|
||
assert body["verdicts"][method]["status"] == "insufficient" and body["verdicts"][method]["missing"] == "字段清单"
|
||
assert body["verdicts"][metric]["status"] == "contradicted"
|
||
assert body["same_model"] is False and body["priced"] is True and body["writer_model"] == WRITER
|
||
|
||
[event] = await judged_events(row.id)
|
||
assert event.data["same_model"] is False and event.data["priced"] is True
|
||
assert event.data["writer_model"] == WRITER
|
||
|
||
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
|
||
assert graph["reports"][0]["judge_meta"] == {"model": PRICED, "same_model": False, "priced": True,
|
||
"writer_model": WRITER}
|
||
|
||
|
||
async def test_judging_with_the_writer_model_is_flagged(client, monkeypatch):
|
||
models = Models(monkeypatch, model=WRITER)
|
||
row, doc = await explore_run()
|
||
body = await ask(client, row.id, [unit_of(doc, "这个数由")])
|
||
assert models.judges and body["model"] == WRITER
|
||
assert body["same_model"] is True and body["writer_model"] == WRITER, "判定照常出，另带标记"
|
||
assert body["verdicts"][unit_of(doc, "这个数由")]["status"] == "supported"
|
||
[event] = await judged_events(row.id)
|
||
assert event.data["same_model"] is True and event.data["writer_model"] == WRITER
|
||
|
||
# judge_meta 取最近一次：换一个价格目录里没有的模型再判一句，就按新的这次
|
||
models.model = "house-judge-model"
|
||
body = await ask(client, row.id, [unit_of(doc, "东区销售额")])
|
||
assert body["same_model"] is False and body["priced"] is False, "目录里没有价格的模型：金额上限对它不生效"
|
||
meta = (await client.get(f"/api/runs/{row.id}/evidence")).json()["reports"][0]["judge_meta"]
|
||
assert meta == {"model": "house-judge-model", "same_model": False, "priced": False, "writer_model": WRITER}
|
||
|
||
|
||
async def test_the_new_excerpts_only_read_sealed_artifacts(client, monkeypatch):
|
||
"""表结构快照、查询快照被挪出封存范围（事件里追不到了）：字段清单、指标输入的 SQL 都不进裁判的摘录。"""
|
||
models = Models(monkeypatch)
|
||
row, doc = await explore_run()
|
||
await rewrite(row.id, "tool.end", "fetch",
|
||
lambda d: {k: v for k, v in d.items() if k not in ("schema_artifact", "query_artifact")})
|
||
await rewrite(row.id, "node.finished", "fetch",
|
||
lambda d: {**d, "evidence": [e for e in d.get("evidence") or []
|
||
if e.get("kind") not in ("schema", "query")]})
|
||
await reseal(row.id)
|
||
await ask(client, row.id, [unit_of(doc, "这个数由"), unit_of(doc, "东区销售额")])
|
||
request = models.judges[0]
|
||
assert "INTEGER" not in request and "region TEXT" not in request
|
||
assert "SELECT" not in request
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 审计行
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_audit_rows_list_contradicted_and_insufficient_claims(client, monkeypatch):
|
||
Models(monkeypatch, verdicts={"分组求和": ("insufficient", "字段清单"), "东区销售额": ("contradicted", "")})
|
||
row, doc = await explore_run()
|
||
method, metric = unit_of(doc, "这个数由"), unit_of(doc, "东区销售额")
|
||
await ask(client, row.id, [method, metric])
|
||
|
||
body = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json()
|
||
rows = [r for g in body["groups"] for r in g["rows"] if r["kind"] == "claim"]
|
||
by_issue = {r["issue"]: r for r in rows}
|
||
assert by_issue["contradicted_claim"]["unit"] == metric and by_issue["contradicted_claim"]["group"] == "none"
|
||
assert by_issue["insufficient_claim"]["unit"] == method
|
||
assert "字段清单" in by_issue["insufficient_claim"]["note"]
|
||
assert by_issue["insufficient_claim"]["verdict"]["missing"] == "字段清单"
|
||
assert by_issue["contradicted_claim"]["verdict"]["post_seal"] is True
|
||
|
||
text = (await client.get(f"/api/runs/{row.id}/evidence/audit?format=csv")).text
|
||
issues = {line[8] for line in csv.reader(io.StringIO(text.lstrip("")))}
|
||
assert {"证据相矛盾", "证据不足"} <= issues
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 旧的 unsupported：照旧算判过
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def test_an_old_unsupported_verdict_still_counts_as_judged():
|
||
from app.api.evidence import _effective_verdicts, _on_demand, _Report, _Sealed
|
||
from app.engine.evidence import compose_doc
|
||
|
||
doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r1")
|
||
[unit] = [u for _, u in iter_units(doc)]
|
||
unit["verdict"] = {"status": "unsupported", "rationale": "证据不支持", "judge": PRICED, "post_seal": False,
|
||
"used": []}
|
||
report = _Report(node_id="write", doc_artifact="d" * 64, ok=True, repairs=0, stats={}, doc=doc, hash_ok=True,
|
||
fields=[], claims="judge")
|
||
later = {"report": "write", "doc_artifact": "d" * 64,
|
||
"verdicts": {unit["id"]: {"status": "supported", "rationale": "有依据", "judge": PRICED}}}
|
||
sealed = _Sealed(run_id="r1", graph={}, seal={"sealed": True, "ok": True, "manifest_seq": 5, "legacy": False},
|
||
events=[], judged=[(9, "write", later)], run_class="exploratory", status="succeeded",
|
||
ledger_on=True)
|
||
effective, appended = _effective_verdicts(report, sealed)
|
||
assert effective[unit["id"]]["status"] == "unsupported" and appended == {}, "追加的判定盖不掉封存的判定"
|
||
assert _on_demand(sealed, doc, unit["id"], effective[unit["id"]])["reason"] == "judged"
|
||
|
||
|
||
def test_the_writer_model_of_an_old_report_comes_from_the_sealed_node_output():
|
||
"""改造前的文档裁判摘要里没有 writer_model、改造前的事件里没有 same_model：按报告撰写节点封存的产出里
|
||
实际用的模型补上。"""
|
||
from app.api.evidence import _judge_meta, _Report, _Sealed, _writer_model
|
||
from app.engine.evidence import compose_doc
|
||
|
||
text = artifact_store.canonical_json({"text": "增长主要来自新客首单。", "model": "writer-old"})
|
||
output = artifact_store.content_hash(text)
|
||
path = artifact_store._path_of(output)
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(text, encoding="utf-8")
|
||
|
||
doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r2")
|
||
doc["judge"] = {"mode": "on_demand", "model": None, "counts": {}}
|
||
report = _Report(node_id="write", doc_artifact="e" * 64, ok=True, repairs=0, stats={}, doc=doc, hash_ok=True,
|
||
fields=[], claims="judge")
|
||
old_event = {"report": "write", "doc_artifact": "e" * 64, "model": "writer-old", "priced": False,
|
||
"verdicts": {}}
|
||
sealed = _Sealed(run_id="r2", graph={}, seal={"sealed": True, "ok": True, "manifest_seq": 5, "legacy": False},
|
||
events=[(3, "node.finished", "write", {"artifact": output})], outputs={"write": output},
|
||
judged=[(9, "write", old_event)], run_class="exploratory", status="succeeded", ledger_on=True)
|
||
assert _writer_model(sealed, report) == "writer-old"
|
||
assert _judge_meta(report, sealed) == {"model": "writer-old", "same_model": True, "priced": False,
|
||
"writer_model": "writer-old"}
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 按需裁判：早于当前规则的判定可以重判，当前规则下判过的照旧复用
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
def on_demand_of(verdict: dict) -> dict:
|
||
from app.api.evidence import _on_demand, _Sealed
|
||
from app.engine.evidence import compose_doc
|
||
|
||
doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r3")
|
||
[unit] = [u for _, u in iter_units(doc)]
|
||
sealed = _Sealed(run_id="r3", graph={}, seal={"sealed": True, "ok": True, "manifest_seq": 5, "legacy": False},
|
||
events=[], run_class="exploratory", status="succeeded", ledger_on=True)
|
||
return _on_demand(sealed, doc, unit["id"], verdict)
|
||
|
||
|
||
@pytest.mark.parametrize("verdict", [
|
||
{"status": "supported", "rationale": "有依据", "post_seal": True},
|
||
{"status": "unsupported", "rationale": "证据不支持", "post_seal": True},
|
||
{"status": "insufficient", "rationale": "缺 SQL", "post_seal": True, "rule_version": 1},
|
||
])
|
||
def test_an_outdated_appended_verdict_can_be_asked_again(verdict):
|
||
got = on_demand_of(verdict)
|
||
assert got["available"] is True and got["reason"] == "outdated" and got["message"]
|
||
|
||
|
||
def test_a_current_verdict_is_not_asked_again():
|
||
got = on_demand_of({"status": "contradicted", "rationale": "对不上", "post_seal": True,
|
||
"rule_version": judge_mod.RULES_VERSION})
|
||
assert got["available"] is False and got["reason"] == "judged"
|
||
|
||
|
||
def test_a_sealed_verdict_is_never_reopened_by_version():
|
||
"""封存的判定（文档里的，post_seal 为 False）盖不掉：版本再旧也不开放重判。"""
|
||
got = on_demand_of({"status": "unsupported", "rationale": "证据不支持", "post_seal": False})
|
||
assert got["available"] is False and got["reason"] == "judged"
|
||
|
||
|
||
async def age(run_id: str, uid: str, **changes) -> None:
|
||
"""把封存之后追加的那条判定改成改造前的样子（封存之后的事件不在封存核对范围里）。"""
|
||
async with SessionLocal() as session:
|
||
for row in (await session.execute(select(RunEvent).where(
|
||
RunEvent.run_id == run_id, RunEvent.type == EventType.EVIDENCE_JUDGED))).scalars():
|
||
data = json.loads(json.dumps(row.data))
|
||
if uid in data.get("verdicts", {}):
|
||
verdict = {k: v for k, v in data["verdicts"][uid].items() if k != "rule_version"}
|
||
data["verdicts"][uid] = {**verdict, **changes}
|
||
row.data = data
|
||
await session.commit()
|
||
|
||
|
||
async def segment_of(client, run_id: str, doc: dict, uid: str) -> dict:
|
||
seg = next(s["id"] for _, u in iter_units(doc) if u["id"] == uid for s in u["segments"])
|
||
res = await client.get(f"/api/runs/{run_id}/evidence/segments/{seg}")
|
||
assert res.status_code == 200, res.text
|
||
return res.json()["unit"]
|
||
|
||
|
||
async def test_an_outdated_verdict_is_really_judged_again(client, monkeypatch):
|
||
models = Models(monkeypatch, verdicts={"东区销售额": ("contradicted", "")})
|
||
row, doc = await explore_run()
|
||
uid = unit_of(doc, "东区销售额")
|
||
first = await ask(client, row.id, [uid])
|
||
assert first["judged"] == [uid] and len(models.judges) == 1
|
||
assert first["verdicts"][uid]["rule_version"] == judge_mod.RULES_VERSION
|
||
|
||
# 当前规则下判过的：照旧复用，不再调用
|
||
again = await ask(client, row.id, [uid])
|
||
assert again["reused"] == [uid] and again["judged"] == [] and len(models.judges) == 1
|
||
assert (await segment_of(client, row.id, doc, uid))["on_demand"]["reason"] == "judged"
|
||
|
||
# 改造前判的（旧取值、没有规则版本）：按钮照常出现，POST 真的重判一次，新判定追加在封存之后
|
||
await age(row.id, uid, status="unsupported")
|
||
unit = await segment_of(client, row.id, doc, uid)
|
||
assert unit["verdict"]["status"] == "unsupported"
|
||
assert unit["on_demand"]["available"] is True and unit["on_demand"]["reason"] == "outdated"
|
||
redo = await ask(client, row.id, [uid])
|
||
assert len(models.judges) == 2 and redo["judged"] == [uid] and redo["reused"] == []
|
||
assert redo["verdicts"][uid]["status"] == "contradicted" and redo["event"] is not None
|
||
assert len(await judged_events(row.id)) == 2
|
||
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
|
||
latest = graph["reports"][0]["post_seal_verdicts"][uid]
|
||
assert latest["status"] == "contradicted" and latest["rule_version"] == judge_mod.RULES_VERSION
|
||
assert (await segment_of(client, row.id, doc, uid))["on_demand"]["reason"] == "judged"
|
||
|
||
|
||
async def test_a_failed_rejudge_keeps_the_older_verdict_in_force(client, monkeypatch):
|
||
"""重判没判成（模型不可用）：旧判定照旧生效，答复和证据图说的一样；按钮还在，可以再点。"""
|
||
models = Models(monkeypatch)
|
||
row, doc = await explore_run()
|
||
uid = unit_of(doc, "东区销售额")
|
||
await ask(client, row.id, [uid])
|
||
await age(row.id, uid)
|
||
|
||
async def broken(session, spec):
|
||
raise RuntimeError("网关 502")
|
||
|
||
monkeypatch.setattr(judge_mod, "get_chat_model", broken)
|
||
body = await ask(client, row.id, [uid])
|
||
assert body["judged"] == [] and "网关 502" in body["message"]
|
||
assert body["verdicts"][uid]["status"] == "supported" and "rule_version" not in body["verdicts"][uid]
|
||
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
|
||
assert graph["reports"][0]["post_seal_verdicts"][uid] == body["verdicts"][uid]
|
||
assert (await segment_of(client, row.id, doc, uid))["on_demand"]["reason"] == "outdated"
|
||
assert len(models.judges) == 1
|