AgentLab/backend/app/eval/judging.py

"""评测里的两种模型判断：结论裁判、文字要点（细案 01 做法 10、11；评测-D4）。都只在变体打开了裁判时做，都是概率性的判断，
单列「模型判断」，不进准确率；花费都记在评测那本账上（engine/spend.eval_spend），不占裁判自己的每日额度。

- 结论裁判（judge_claims）：走按需裁判同一条路——api/evidence 的 _judge_now（证据面板上点「请模型判断」就是它）：只取封存
  范围内的工件、遮罩列不给裁判、判定作为 evidence.judged 追加在封存之后（post_seal）、报告文档一个字不改。送哪几句由裁判
  自己的预筛定（judge.candidates：结论句，挂了依据的在前）。花费经 judge.charged_to 记在评测那本账上、按评测的每日上限。
  结果按 supported / partial / contradicted / insufficient / 未判 数句数。
- 文字要点（judge_points）：问法、答案原文和要点交给裁判模型（设置里的证据裁判模型，和按需裁判同一个取值链），逐条说答到
  了（covered）、答到一部分（partial）、没答到（missing），附一句理由。答案原文只是这次运行的成果（遮过的），要点是人写的。
  不放查询结果、不放 SQL：要点问的是「说没说到」，不是「数对不对」（数由确定性的评分判）。

判断出错（模型没配好、超时、格式不对）一律照实记「没有判」和原因，不挡评分。
"""
from __future__ import annotations

import json
import logging
import re
from collections.abc import Mapping, Sequence
from typing import Any

logger = logging.getLogger(__name__)

#: 结论裁判按句数分（judge.COUNT_KEYS 里评测关心的几种，unsupported 是旧数据，并进 contradicted）
CLAIM_KEYS = ("supported", "partial", "contradicted", "insufficient", "unjudged")
#: 文字要点一次最多交给裁判多少字的答案
ANSWER_CHARS = 6000
#: 文字要点的输出额度、时限
POINTS_MAX_TOKENS = 2048
POINTS_TIMEOUT_S = 90

POINTS_SYSTEM_PROMPT = """\
你是评测的裁判。下面给出一个问题、对它的回答，以及期望回答里应当说到的几条要点。逐条判断回答有没有说到这条要点：
- covered：说到了（意思一致即可，不要求字面相同）
- partial：说到了一部分，或者说得含糊
- missing：没有说到，或者说的和要点相反
只看回答里写了什么，不要自己补充事实，也不要判断回答里的数对不对（数另有确定性的核对）。
只输出一个 JSON 数组，每条要点一项：[{"point": 要点序号（从 1 起）, "verdict": "covered|partial|missing", "reason": "一句话理由"}]"""
POINTS_HUMAN_PROMPT = "问题：{question}\n\n回答：\n{answer}\n\n要点：\n{points}"


async def judge_claims(run_id: str, *, daily_max_usd: float | None) -> dict[str, Any]:
    """按需裁判这次作答的报告（结论句）。{counts, model, cost_usd, calls, reason}；判不了的照实写 reason。"""
    from fastapi import HTTPException

    from app.api import evidence
    from app.engine import judge as judging
    from app.engine.spend import eval_spend

    empty = {k: 0 for k in CLAIM_KEYS}
    try:
        async with evidence._JUDGING.hold(run_id):
            sealed = await evidence._sealed(run_id)
            if not sealed.sealed or not sealed.trusted or not sealed.ledger_on:
                return {"counts": empty, "reason": "作答的运行没有可信的封存，不判"}
            reports = evidence._reports(sealed)
            if not reports:
                return {"counts": empty, "reason": "答案没有报告文档，没有结论句可判"}
            chosen = evidence._choose(reports, None, sealed)
            if chosen.doc is None or chosen.hash_ok is False:
                return {"counts": empty, "reason": "报告文档读不出来或被改过，不判"}
            units = [c.unit for c in judging.candidates(chosen.doc)][:evidence.JUDGE_MAX_UNITS]
            if not units:
                return {"counts": empty, "reason": "报告里没有要判的结论句"}
            with judging.charged_to(eval_spend, daily_max_usd):
                result = await evidence._judge_now(sealed, chosen, units)
    except HTTPException as e:
        return {"counts": empty, "reason": str(e.detail)}
    except Exception as e:  # noqa: BLE001 - 裁判是附加的一步：出错照实记，不挡评分
        from app.core.errors import describe_exception

        logger.exception("评测：结论裁判出错（运行 %s）", run_id)
        return {"counts": empty, "reason": f"结论裁判出错：{describe_exception(e)}"}
    counts = dict(empty)
    for verdict in (result.get("verdicts") or {}).values():
        status = (verdict or {}).get("status")
        status = "contradicted" if status == judging.UNSUPPORTED else status
        if status in counts:
            counts[status] += 1
        elif status != "not_a_claim":
            counts["unjudged"] += 1
    return {"counts": counts, "units": len(units), "model": result.get("model"), "priced": result.get("priced"),
            "cost_usd": float(result.get("cost_usd") or 0.0), "calls": int(result.get("calls") or 0),
            "limited": bool(result.get("limited")), "reason": result.get("message")}


def _parse_points(text: str, n: int) -> list[dict[str, Any]] | None:
    match = re.search(r"\[.*\]", text or "", re.DOTALL)
    if match is None:
        return None
    try:
        items = json.loads(match.group(0))
    except ValueError:
        return None
    if not isinstance(items, list):
        return None
    out: dict[int, dict[str, Any]] = {}
    for item in items:
        if not isinstance(item, Mapping):
            continue
        index = item.get("point")
        verdict = item.get("verdict")
        if isinstance(index, int) and 1 <= index <= n and verdict in ("covered", "partial", "missing"):
            out[index] = {"point": index, "verdict": verdict, "reason": str(item.get("reason") or "")[:200]}
    if len(out) != n:
        return None
    return [out[i] for i in range(1, n + 1)]


async def judge_points(question: str, answer: str, expectations: Sequence[Mapping[str, Any]], *,
                       daily_max_usd: float | None, per_run_left: float | None = None) -> dict[str, Any]:
    """文字要点：{expectations: {期望 id: {points: [{point, verdict, reason}], reason?}}, model, cost_usd, calls, reason?}。
    先按目录价估一次、在评测那本账上预留，加上会超过每日上限（或这次评测剩下的钱）就不判、照实写。"""
    from langchain_core.messages import HumanMessage, SystemMessage

    from app.core.errors import describe_exception
    from app.db.base import SessionLocal
    from app.engine.judge import judge_model_spec
    from app.engine.spend import eval_spend
    from app.providers import usage as usage_meter
    from app.providers.catalog import estimate_cost, find_pricing
    from app.providers.factory import get_chat_model

    texts = [e for e in expectations if e.get("kind") == "text" and e.get("points")]
    if not texts:
        return {"expectations": {}}
    if not (answer or "").strip():
        return {"expectations": {}, "reason": "答案是空的，文字要点没有判"}
    try:
        async with SessionLocal() as session:
            spec = await judge_model_spec(session, None)
            model, model_id = await get_chat_model(session, spec.model_copy(update={
                "max_tokens": POINTS_MAX_TOKENS, "timeout": POINTS_TIMEOUT_S}))
    except Exception as e:  # noqa: BLE001 - 裁判模型没配好：照实写没有判
        return {"expectations": {}, "reason": f"裁判模型用不了：{describe_exception(e)}"}
    priced = find_pricing(model_id) is not None
    out: dict[str, Any] = {}
    meter = usage_meter.meter()
    reason = None
    for exp in texts:
        points = [str(p) for p in exp.get("points") or []]
        human = POINTS_HUMAN_PROMPT.format(question=question, answer=answer[:ANSWER_CHARS],
                                           points="\n".join(f"{i}. {p}" for i, p in enumerate(points, start=1)))
        estimate = estimate_cost(model_id, (len(POINTS_SYSTEM_PROMPT) + len(human)) // 2, 60 * len(points)) if priced \
            else 0.0
        if per_run_left is not None and priced and estimate > per_run_left:
            out[exp["id"]] = {"reason": "这次评测的金额上限不够再判文字要点"}
            continue
        day = await eval_spend.charge(estimate, limit=daily_max_usd if priced else None, priced=priced)
        if day is None:
            out[exp["id"]] = {"reason": "评测今天的金额已到上限，文字要点没有判"}
            continue
        before = meter.summary()["cost_usd"]
        try:
            with usage_meter.metering(meter):
                reply = await model.ainvoke([SystemMessage(POINTS_SYSTEM_PROMPT), HumanMessage(human)])
            text = reply.content if isinstance(reply.content, str) else json.dumps(reply.content, ensure_ascii=False)
            parsed = _parse_points(text, len(points))
            out[exp["id"]] = {"points": parsed} if parsed else {"reason": "裁判的回复格式不对，文字要点没有判"}
        except Exception as e:  # noqa: BLE001
            logger.exception("评测：判文字要点出错")
            reason = f"判文字要点出错：{describe_exception(e)}"
            out[exp["id"]] = {"reason": reason}
        spent = meter.summary()["cost_usd"] - before
        await eval_spend.settle(day, (spent if priced else 0.0) - estimate)
        if per_run_left is not None:
            per_run_left -= spent
    summary = meter.summary()
    return {"expectations": out, "model": model_id, "usage": summary, "cost_usd": summary["cost_usd"],
            "calls": summary["calls"], **({"reason": reason} if reason else {})}


__all__ = ["CLAIM_KEYS", "POINTS_HUMAN_PROMPT", "POINTS_SYSTEM_PROMPT", "judge_claims", "judge_points"]