mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 04:31:58 +00:00
ThearchyHelios
1477 lines
80 KiB
Python
1477 lines
80 KiB
Python
"""结论句裁判：另一个模型按证据逐句判断报告里的话站不站得住（可点击证据第四期）。
|
||
|
||
数字、实体、引文是确定性的，系统自己取值、自己核对。可「增长主要来自华东」说得对不对，规则判断不了，
|
||
只能交给模型——这是概率性的判断，所以这里每一步都说实话、留余地：
|
||
|
||
1. 预筛（claim_priority）：只把结论句送去判。挂了依据的排第一，有方向词、因果词的排第二，其余有数字、
|
||
实体这类实质内容的排第三；标题、表格单元格、代码、连接性的话、只有过渡词的、短句不送
|
||
2. 证据摘录（prepare）：句子挂的每件证据摘一段。指标写「名称 = 值」加原式，再附上输入取自哪次查询、
|
||
哪一格和那次查询的 SQL；查询给 SQL 前 300 字和被引用的行（最多 20 行，数据源遮罩的列不给；宽表先给
|
||
被引用的列，再补满 12 列）；表附上字段清单和类型（最多 60 个，遮罩的列不列，快照不完整要注明），
|
||
字段附上类型和所属的表，两样都附上出现过的查询 SQL；知识库片段最多 600 字。快照一律经 loader 取
|
||
（按需裁判传只认封存工件的那个）。规则里写明数字已经由系统核对过。带期 4 文档标记的报告，上传表格的
|
||
查询在 SQL 之后另有来历行（来源、口径、已接受、区域外文字，_provenance_lines）
|
||
3. 调用（judge_doc）：一份报告合并成一次调用，超过每批的句数就分批；结构化输出失败退回 JSON 解析，
|
||
再失败就记未裁判。thinking 关掉，max_tokens 4096
|
||
4. 预算：每份报告的句数、金额、时长和全局每日金额，每一项都能是 None（不限）。调用前按目录价估算，
|
||
超了按优先级截断；触顶就停，已判的保留，没判的记未裁判并说明是哪个上限——不报错
|
||
5. 每日计数落在设置表（SPEND_KEY），按本地日期滚动。目录里没有价格的模型估不出金额，金额上限对它
|
||
不起作用，照实说「按令牌估不出金额」
|
||
|
||
判定五选一：supported、partial、contradicted（证据和句子冲突）、insufficient（证据里没有判断所需的
|
||
信息，missing 写缺什么）、not_a_claim。旧数据里的 unsupported（当时没分矛盾和不足）原样保留，判档当
|
||
contradicted；新判的不再产出它。
|
||
|
||
判定只标注、不改写正文。报告节点在正式运行里调它（结果写进文档、随文档封存）；探索运行按需裁判的
|
||
接口也调它（post_seal=True，结果落在封存之后）。所有模型调用都经 get_chat_model，测试换成剧本。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import math
|
||
import re
|
||
import time
|
||
from collections.abc import Callable, Iterable
|
||
from dataclasses import asdict, dataclass
|
||
from datetime import datetime
|
||
from typing import Any
|
||
|
||
from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage
|
||
from sqlalchemy import select
|
||
|
||
from app.core.artifact_store import canonical_json, content_hash
|
||
from app.core.artifact_store import load as load_artifact
|
||
from app.core.errors import describe_exception, not_configured
|
||
from app.core.events import EventType
|
||
# 只用标准库的契约模块：取文档标记的版本号。识别器、清单模块（direct_select、provenance）在函数体内导入（_provenance_lines）
|
||
from app.data.provenance_types import DOC_PROVENANCE
|
||
from app.data.tabular import UNSHAPED_NOTE
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import Setting
|
||
from app.engine.evidence import (
|
||
_DIRECTIONAL,
|
||
ENTITY_KINDS,
|
||
MISSING,
|
||
RenderError,
|
||
_auto_entity,
|
||
_clean,
|
||
_mask_names,
|
||
iter_units,
|
||
render_cell,
|
||
schema_column,
|
||
schema_table,
|
||
table_fields,
|
||
)
|
||
from app.engine.expressions import CellError, column_kind, locate_cell
|
||
from app.engine.merge_query import MERGE_SOURCE
|
||
from app.engine.nodes.llm import _split_structured, _usage_of
|
||
from app.engine.state import message_text
|
||
from app.providers import catalog as pricing
|
||
from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 常量
|
||
# --------------------------------------------------------------------------
|
||
|
||
#: 模型给的五种判定；unjudged 是系统记的（没判成），不是模型说的
|
||
JUDGED = ("supported", "partial", "contradicted", "insufficient", "not_a_claim")
|
||
#: 旧的判定「证据不支持」：当时没分「证据相矛盾」和「证据不足」。已封存的文档、已有的事件里原样保留，
|
||
#: 判档照旧当 contradicted；新判的一律用新取值（模型照旧写了的，按 contradicted 收下）
|
||
UNSUPPORTED = "unsupported"
|
||
#: 读已有的判定时算「判过」的：新的五种加旧的 unsupported
|
||
SETTLED = (*JUDGED, UNSUPPORTED)
|
||
#: 判档按「证据相矛盾」处置的（on_unsupported：降档或不予出具）
|
||
CONTRADICTED = ("contradicted", UNSUPPORTED)
|
||
#: 判定对象里 status 的全部取值
|
||
VERDICTS = (*SETTLED, "unjudged")
|
||
#: 各判定的句数（stats、counts、出具横幅、审计导出都用这组键）。unsupported 保留给旧数据，新判的恒为 0
|
||
COUNT_KEYS = ("supported", "partial", "contradicted", "insufficient", "not_a_claim", "unjudged", UNSUPPORTED)
|
||
#: 模型的理由最多几个字（超长在句子边界截断加「…」）；裁判规则里要求的字数
|
||
RATIONALE_MAX = 120
|
||
RATIONALE_ASK = 80
|
||
#: insufficient 时「缺的是什么」最多几个字
|
||
MISSING_MAX = 30
|
||
#: 判定的规则版本：新判的判定带着它（rule_version），也进 request_key。报告节点的断点里换了规则的旧判定不复用；
|
||
#: 按需裁判遇到早于当前规则的判定（没有版本、版本更小、旧取值 unsupported）重判一次
|
||
RULES_VERSION = 2
|
||
#: 一次调用最多判几句。以前是 40：真实运行里推理模型拿着 40 句的摘录 30 秒没回来，整份报告一句都没判成
|
||
#: （同一个模型 5 句一批 3 秒多就回来了）。批小了，超时最多丢掉没回来的那几批
|
||
BATCH_SIZE = 8
|
||
#: 一波最多几批同时问：几批共用剩下的时长，句数上限内的句子一两波就判完。第一批单独问：接入调不通（鉴权、
|
||
#: 网关挂了）时只撞这一次，不让一整波一起失败
|
||
PARALLEL_BATCHES = 4
|
||
MAX_TOKENS = 4096
|
||
#: 证据摘录：SQL 前几个字、每次查询最多几行几列、每段知识库片段最多几个字、整份引用的检索给几段。
|
||
#: 列数上限只管没被引用的列：句子引用的格子所在的列一律给（遮罩的除外），再用别的列补满到这么多列
|
||
SQL_CHARS = 300
|
||
EXCERPT_ROWS = 20
|
||
EXCERPT_COLS = 12
|
||
CHUNK_CHARS = 600
|
||
#: 表名、字段名的摘录里附上它出现过的查询的 SQL，最多几次（「按会员号去重计数」这种句子，证据在 SQL 里）；
|
||
#: 口径卡指标的输入取自的查询也按这个数给 SQL
|
||
ENTITY_QUERIES = 3
|
||
EXCERPT_HITS = 3
|
||
#: 表的摘录里最多列几个字段（多出来的注明另有几个）；指标的摘录里最多列几个输入
|
||
FIELDS_MAX = 60
|
||
METRIC_INPUTS = 8
|
||
#: 短句：看得见的字（不算标点、空白）少于这么多，又没挂依据、没有方向词的，不送裁判
|
||
SHORT_CHARS = 6
|
||
#: 估算输出 token：每句一条判定（理由 80 字上下、缺什么，加上 JSON 的键）
|
||
_OUT_PER_UNIT = 120
|
||
_OUT_BASE = 20
|
||
|
||
#: 未裁判的原因。前四个是上限，报告里写「已到上限」
|
||
LIMITS = ("max_claims", "max_cost_usd", "daily_max_usd", "timeout_s")
|
||
REASONS = (*LIMITS, "error", "format", "missing", "on_demand")
|
||
|
||
JUDGE_ROLE = "你是报告核查员：逐句判断报告里的话，能不能被它挂的证据支持。你不写报告，也不改报告。"
|
||
|
||
JUDGE_RULES = f"""判定（verdict）五选一：
|
||
- supported：证据足以支持这句话的说法——方向、比较、范围、因果都对得上
|
||
- partial：一部分说法有证据支持，另一部分（原因、范围、程度）证据里没有
|
||
- contradicted：证据和这句话冲突——证据里的方向、比较、范围、排名，或者 SQL 的算法、字段，和句子说的对不上
|
||
- insufficient：证据里没有判断这句话所需的信息（比如摘录里没有 SQL、没有字段清单、没有取数的查询），或者这句在陈述数据事实却没有挂任何证据
|
||
- not_a_claim：这句不陈述数据事实（过渡、背景、建议、计划），不需要证据
|
||
规则：
|
||
1. 数字已经由系统核对过，不要复算：每个数都是系统从证据里取出来、按固定规则渲染的，不要因为数字本身或它的格式质疑这句话；你只判断句子的说法有没有证据支持。
|
||
2. 只看下面给的证据摘录，不用常识、行业经验或外部知识补证据；摘录里没有的就是没有。
|
||
3. 只有证据和句子冲突时才判 contradicted；证据里没有相关信息判 insufficient——「没有」不等于「矛盾」。
|
||
4. 摘录注明「表结构快照不完整」时，「某张表、某个字段不存在」这类说法只能判 insufficient。
|
||
5. rationale 用中文写，不超过 {RATIONALE_ASK} 个字，说清依据是什么，或者缺了什么。
|
||
6. 判 insufficient 时 missing 写缺的是什么，越短越好，比如「SQL」「字段清单」「取数的查询」；其余判定 missing 写空字符串。
|
||
7. used 写你实际用到的证据编号（摘录里【】中的那个，比如 m:gmv、Q1），没用到就写空数组。
|
||
8. 每句都要给判定，unit 照抄句子前面方括号里的编号。"""
|
||
|
||
#: 报告节点 claims 为 judge 时拼在写作规则后面：写作者要知道还有一道按证据的核对
|
||
JUDGE_RULE = ("这份报告的结论句还会由另一个模型逐句核对：它只看你挂的依据，判断这句话站不站得住。"
|
||
"说法不要超出证据——证据里没有的原因、范围、趋势，不要写成结论。")
|
||
|
||
JUDGE_SCHEMA: dict[str, Any] = {
|
||
# 走工具调用的结构化输出要求函数名只含字母、数字、下划线
|
||
"title": "claim_verdicts",
|
||
"description": "逐句判断报告里的话能不能被它挂的证据支持",
|
||
"type": "object",
|
||
"properties": {"items": {"type": "array", "items": {
|
||
"type": "object",
|
||
"properties": {
|
||
"unit": {"type": "string", "description": "句子编号，照抄 [u3] 里的 u3"},
|
||
"verdict": {"type": "string", "enum": list(JUDGED)},
|
||
"rationale": {"type": "string", "description": f"判定的理由，中文，不超过 {RATIONALE_ASK} 个字"},
|
||
"missing": {"type": "string",
|
||
"description": "判 insufficient 时写缺的是什么（比如 SQL、字段清单、取数的查询），其余判定写空字符串"},
|
||
"used": {"type": "array", "items": {"type": "string"}, "description": "实际用到的证据编号"},
|
||
},
|
||
"required": ["unit", "verdict", "rationale", "missing", "used"],
|
||
}}},
|
||
"required": ["items"],
|
||
}
|
||
|
||
#: 只有过渡作用的说法。一句话去掉它们和标点之后什么都不剩，就是连接性的话
|
||
_TRANSITIONS = re.compile(
|
||
r"综上所述|综上|总的来说|总体来看|总体而言|整体来看|整体而言|具体来看|具体如下|具体而言|如下所示|"
|
||
r"详见下表|见下表|主要原因如下|原因如下|原因有以下几点|情况如下|结论如下|分析如下|"
|
||
r"下面(?:来)?(?:看看|看|是|分析|说明|展开|列出)?|接下来(?:看)?|首先|其次|再次|最后|另外|此外|同时|"
|
||
r"因此|所以|总之|如下|以下|(?<![A-Za-z])(?:in summary|as follows|overall|next)(?![A-Za-z])",
|
||
re.I)
|
||
_INVISIBLE = re.compile(r"[\W_]+")
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 预筛
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Candidate:
|
||
"""一句要送裁判的结论。"""
|
||
|
||
unit: str
|
||
priority: int # 1 挂了依据 / 2 方向词、因果词 / 3 其余有实质内容的
|
||
order: int # 在文档里的先后
|
||
text: str # 句子（渲染后的正文，压成一行）
|
||
section: str # 所在小节的标题，没有就是空串
|
||
aliases: tuple[str, ...] # 这句挂的依据（目录键）
|
||
rows: tuple[tuple[str, int], ...] = () # 点名引用的查询行：(Q1, 3)
|
||
cols: tuple[tuple[str, str], ...] = () # 行内引用的格子所在的列：(Q1, region)
|
||
whole: tuple[str, ...] = () # 整份引用的查询、检索：Q1、K1
|
||
hits: tuple[tuple[str, int], ...] = () # 引文所在的检索片段：(K1, 0)
|
||
|
||
|
||
def claim_priority(unit: dict[str, Any], block: dict[str, Any], markdown: str) -> int | None:
|
||
"""这句结论送不送裁判、排第几：1 / 2 / 3，不送是 None。只看文档本身，同一份文档永远判得一样。
|
||
|
||
- 不是结论句（连接性的话、标题、代码）、或者在表格里（一格值不是一句话）：不送
|
||
- 写作者挂了依据（行内引用或 [[see:]]，自动链接的名字不算）：1
|
||
- 去掉过渡词和标点什么都不剩（「原因如下：」）：不送
|
||
- 有方向词、因果词：2
|
||
- 有数字、值、实体、引文，看得见的字又不少于 SHORT_CHARS 个：3
|
||
- 其余（短句）：不送
|
||
"""
|
||
if unit.get("kind") != "claim" or block.get("type") in ("table", "code", "heading"):
|
||
return None
|
||
if unit.get("cites"):
|
||
return 1
|
||
segments = unit.get("segments") or []
|
||
words = "".join(str(s.get("text") or "") for s in segments if s.get("kind") == "text" or _auto_entity(s))
|
||
body = _INVISIBLE.sub("", _TRANSITIONS.sub("", words))
|
||
substance = [s for s in segments if s.get("kind") in ("number", "value", "entity", "quote") and not _auto_entity(s)]
|
||
shown = body + "".join(_INVISIBLE.sub("", str(s.get("text") or "")) for s in substance)
|
||
if not shown:
|
||
return None
|
||
if _DIRECTIONAL.search(body):
|
||
return 2
|
||
if (substance or unit.get("see")) and len(shown) >= SHORT_CHARS:
|
||
return 3
|
||
return None
|
||
|
||
|
||
def candidates(doc: dict[str, Any], units: Iterable[str] | None = None) -> list[Candidate]:
|
||
"""送裁判的句子，按优先级、再按文档里的先后排好。
|
||
|
||
units 给了就是有人点名要判这几句（按需裁判）：预筛放掉的结论句照判（按最低优先级），不是结论句的
|
||
（标题、连接性的话、代码、表格单元格）和不存在的编号不判。
|
||
"""
|
||
wanted = None if units is None else {str(u) for u in units}
|
||
markdown = str(doc.get("markdown") or "")
|
||
out: list[Candidate] = []
|
||
section = ""
|
||
for order, (block, unit) in enumerate(iter_units(doc)):
|
||
text = _unit_text(unit, markdown)
|
||
if block.get("type") == "heading":
|
||
section = text
|
||
priority = claim_priority(unit, block, markdown)
|
||
if wanted is not None:
|
||
if unit.get("id") not in wanted or unit.get("kind") != "claim" or block.get("type") in ("table", "code"):
|
||
continue
|
||
priority = priority or 3
|
||
if priority is None or not text:
|
||
continue
|
||
out.append(_candidate(unit, priority, order, text, section))
|
||
return sorted(out, key=lambda c: (c.priority, c.order))
|
||
|
||
|
||
def _unit_text(unit: dict[str, Any], markdown: str) -> str:
|
||
span = unit.get("span")
|
||
if not (isinstance(span, list) and len(span) == 2 and all(isinstance(x, int) for x in span)):
|
||
return ""
|
||
return _clean(markdown[span[0]:span[1]])
|
||
|
||
|
||
def unit_texts(doc: dict[str, Any]) -> dict[str, str]:
|
||
"""{unit id: 这句的正文（压成一行）}，按文档里的先后；没有正文的不列。改写前后按句子认编号时用。"""
|
||
markdown = str(doc.get("markdown") or "")
|
||
out: dict[str, str] = {}
|
||
for _, unit in iter_units(doc):
|
||
if text := _unit_text(unit, markdown):
|
||
out[str(unit.get("id"))] = text
|
||
return out
|
||
|
||
|
||
def _candidate(unit: dict[str, Any], priority: int, order: int, text: str, section: str) -> Candidate:
|
||
rows: list[tuple[str, int]] = []
|
||
cols: list[tuple[str, str]] = []
|
||
whole: list[str] = []
|
||
hits: list[tuple[str, int]] = []
|
||
cites = [s["cite"] for s in unit.get("segments") or [] if isinstance(s.get("cite"), dict) and not s.get("auto")]
|
||
for cite in [*cites, *(unit.get("see") or [])]:
|
||
if not isinstance(cite, dict) or cite.get("status") != "resolved" or not cite.get("alias"):
|
||
continue
|
||
alias, kind, locator = str(cite["alias"]), cite.get("kind"), cite.get("locator") or {}
|
||
if kind == "cell" or (kind == "query" and isinstance(locator.get("row"), int)):
|
||
rows.append((alias, int(locator["row"])))
|
||
if kind == "cell" and locator.get("column") not in (None, ""):
|
||
cols.append((alias, str(locator["column"])))
|
||
elif kind in ("query", "retrieval"):
|
||
whole.append(alias)
|
||
elif kind == "quote" and isinstance(locator.get("hit"), int):
|
||
hits.append((alias, int(locator["hit"])))
|
||
return Candidate(unit=str(unit.get("id")), priority=priority, order=order, text=text, section=section,
|
||
aliases=tuple(dict.fromkeys(str(a) for a in unit.get("cites") or [])),
|
||
rows=tuple(dict.fromkeys(rows)), cols=tuple(dict.fromkeys(cols)),
|
||
whole=tuple(dict.fromkeys(whole)),
|
||
hits=tuple(dict.fromkeys(hits)))
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 证据摘录与请求
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
class _Fetch:
|
||
"""这一次请求里取过的工件（取回时复验哈希；取不到、被改过的一律当没有）。"""
|
||
|
||
def __init__(self, loader: Callable[[str], Any] | None) -> None:
|
||
self.loader = loader or load_artifact
|
||
self.memo: dict[str, Any] = {}
|
||
|
||
def __call__(self, artifact: Any) -> Any:
|
||
key = str(artifact or "")
|
||
if key not in self.memo:
|
||
try:
|
||
self.memo[key] = self.loader(key) if key else None
|
||
except Exception: # noqa: BLE001 - 取不回来、哈希不符：这段摘录不给，裁判照实看不到
|
||
self.memo[key] = None
|
||
return self.memo[key]
|
||
|
||
|
||
def _hidden(snapshot: dict[str, Any], source: Any, masked: dict[str, Any] | None) -> set[str]:
|
||
"""要遮住的列（小写）：查询当时记下的，加上数据源现在设的。合并查询的结果不是数据源，按它合并的那几个数据源算。"""
|
||
names = _mask_names(snapshot.get("mask_columns"))
|
||
wanted = {str(source or "")}
|
||
if snapshot.get("source") == MERGE_SOURCE and isinstance(snapshot.get("sources"), list):
|
||
wanted |= {str(s) for s in snapshot["sources"]}
|
||
for name, cols in (masked or {}).items():
|
||
if str(name) in wanted:
|
||
names |= _mask_names(cols)
|
||
return names
|
||
|
||
|
||
def _shown_columns(columns: list[str], cited: Iterable[str]) -> list[str]:
|
||
"""摘录给哪几列：被引用的列一律给（超过上限也给），再按原来的顺序用别的列补满到 EXCERPT_COLS 列。
|
||
|
||
columns 已经去掉了遮罩的列：遮罩的列被引用了也不给。列名不分大小写。
|
||
"""
|
||
wanted = {str(c).lower() for c in cited}
|
||
first = [c for c in columns if c.lower() in wanted]
|
||
rest = [c for c in columns if c.lower() not in wanted]
|
||
return first + rest[:max(0, EXCERPT_COLS - len(first))]
|
||
|
||
|
||
def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iterable[str], whole: bool,
|
||
fetch: _Fetch, masked: dict[str, Any] | None, *, provenance_on: bool = False) -> str | None:
|
||
"""一条查询的摘录：头、SQL、（期 4）来历行、列、被引用的行。
|
||
|
||
provenance_on：文档带期 4 标记时为真（prepare 算）。这时上传表格的查询快照（有 data_version）在 SQL 之后、「列：」
|
||
之前多几行来历（_provenance_lines）；其余情况一个字都不多，期 4 之前的文档、手工源的摘录和以前逐字相同。
|
||
"""
|
||
snapshot = fetch(entry.get("artifact"))
|
||
if not isinstance(snapshot, dict) or not isinstance(snapshot.get("rows"), list):
|
||
return None
|
||
total = len(snapshot["rows"])
|
||
# 旧形状的台账条目没记数据源：按快照自己记的找遮罩（证据面板同一个口径）
|
||
source = entry.get("source") or snapshot.get("source")
|
||
head = (f"【{alias}】合并查询结果（共 {total} 行）" if snapshot.get("source") == MERGE_SOURCE
|
||
else f"【{alias}】查询结果（{entry.get('tool') or '查询'} · 数据源 {source or '—'} · 共 {total} 行）")
|
||
sql = _clean(str(snapshot.get("sql") or ""))
|
||
lines = [head, f"SQL：{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"]
|
||
if snapshot.get("source") == MERGE_SOURCE:
|
||
lines.append(_merge_line(snapshot))
|
||
hidden = _hidden(snapshot, source, masked)
|
||
if provenance_on and snapshot.get("data_version"):
|
||
lines += _provenance_lines(snapshot, fetch, hidden)
|
||
columns = [str(c) for c in snapshot.get("columns") or [] if str(c).lower() not in hidden]
|
||
shown_cols = _shown_columns(columns, cols)
|
||
# 点名引用的行先给，整份引用的再从头补满，一共不超过 EXCERPT_ROWS 行
|
||
picked = sorted(r for r in rows if 0 <= r < total)[:EXCERPT_ROWS]
|
||
if whole:
|
||
picked += [r for r in range(total) if r not in picked][:EXCERPT_ROWS - len(picked)]
|
||
omitted = len(columns) - len(shown_cols)
|
||
lines.append("列：" + " | ".join(shown_cols) + (f" | …另有 {omitted} 列" if omitted > 0 else ""))
|
||
for r in sorted(picked):
|
||
cells = []
|
||
for c in shown_cols:
|
||
try:
|
||
raw, _ = locate_cell(snapshot, r, c)
|
||
cells.append(render_cell(raw, kind=column_kind(snapshot, c)))
|
||
except (CellError, RenderError):
|
||
cells.append(MISSING)
|
||
lines.append(f"r{r}：" + " | ".join(cells))
|
||
if hidden & {str(c).lower() for c in snapshot.get("columns") or []}:
|
||
lines.append("（有的列按数据源的设置遮罩了，没有给出）")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def _merge_line(snapshot: dict[str, Any]) -> str:
|
||
"""合并查询结果的摘录多一行：合并了哪几个输入（SQL 里的别名、节点、数据源、行数）。上面的 SQL 就是合并 SQL。"""
|
||
parts = []
|
||
for item in snapshot.get("inputs") or []:
|
||
if isinstance(item, dict):
|
||
parts.append(f"{item.get('alias')}（节点 {item.get('node_id') or '—'}，数据源 {item.get('source') or '—'}，"
|
||
f"{item.get('rows')} 行）")
|
||
return "合并的输入：" + ("、".join(parts) if parts else "—")
|
||
|
||
|
||
def _provenance_lines(snapshot: dict[str, Any], fetch: _Fetch, hidden: set[str]) -> list[str]:
|
||
"""上传表格的查询快照的来历行（P4-SPEC 3.2、3.3）：provenance.judge_lines 生成，这里只负责取冻结的表结构快照、
|
||
算 SQL 用到的表（direct_select.tables_in，和执行这条 SQL 的 SQLite 同一种读法，宁可多认），把现成的遮罩交过去。
|
||
|
||
- 两个模块在函数体内导入：judge 被设置接口等地方导入，不能因此拉进整套配方模块；也只有带标记的文档、上传
|
||
表格的快照才走到这里，期 4 之前的文档一次都不碰它们（测试用桩钉住）；
|
||
- 工件一律经同一个 fetch 取（按需裁判时是只认封存工件、带标记才扩展到哈希链上清单的那个 loader），取不到、
|
||
哈希不符的都当没有：judge_lines 据此写「导入清单无法读取…」那一行，不中断摘录；
|
||
- 确定：同样的工件永远得到同样的行（摘录进断点指纹）。生成过程中出了意外（工件形状不对）同样落到那一行，
|
||
报告节点的裁判不因一份坏清单整体失败。
|
||
"""
|
||
from app.data.provenance import SOURCE_BROKEN, judge_lines
|
||
from app.engine.direct_select import tables_in
|
||
|
||
schema_id = snapshot.get("schema_artifact")
|
||
schema = fetch(schema_id) if schema_id else None
|
||
schema = schema if isinstance(schema, dict) else None
|
||
frozen = schema.get("tables") if schema is not None else None
|
||
tables = {str(name): [str(c.get("name")) for c in (meta.get("columns") or []) if isinstance(c, dict)]
|
||
for name, meta in frozen.items() if isinstance(meta, dict)} if isinstance(frozen, dict) else {}
|
||
try:
|
||
used = tables_in(str(snapshot.get("sql") or ""), tables) if tables else []
|
||
return judge_lines(snapshot, schema, fetch, tables=used, hidden=hidden)
|
||
except Exception: # noqa: BLE001 - 坏形状的工件只让这条查询少了来历，不让整份报告判不了
|
||
logger.warning("judge: 生成来历行失败", exc_info=True)
|
||
return [SOURCE_BROKEN]
|
||
|
||
|
||
def _sql_line(label: str, sql: str) -> str:
|
||
return f"{label}：{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"
|
||
|
||
|
||
def _source_hidden(source: Any, catalog: dict[str, Any], fetch: _Fetch, masked: dict[str, Any] | None) -> set[str]:
|
||
"""一个数据源要遮的列（小写）：数据源现在设的，加上这次运行里它的查询快照当时记下的（数据源改名、删掉了
|
||
也照遮）。说不清是哪个数据源的，所有遮罩都算上：宁可多遮，不能露。"""
|
||
name = str(source or "")
|
||
names: set[str] = set()
|
||
for key, cols in (masked or {}).items():
|
||
if not name or str(key) == name:
|
||
names |= _mask_names(cols)
|
||
for entry in catalog.values():
|
||
if not isinstance(entry, dict) or entry.get("kind") != "query":
|
||
continue
|
||
snapshot = fetch(entry.get("artifact"))
|
||
if isinstance(snapshot, dict) and (not name or str(entry.get("source") or snapshot.get("source") or "") == name):
|
||
names |= _mask_names(snapshot.get("mask_columns"))
|
||
return names
|
||
|
||
|
||
def _query_alias(catalog: dict[str, Any], artifact: Any) -> str | None:
|
||
"""这份查询快照在报告目录里的编号（Q1）：口径卡输入记的是工件，靠它对上。"""
|
||
return next((a for a, e in catalog.items() if isinstance(e, dict) and e.get("kind") == "query"
|
||
and artifact and e.get("artifact") == artifact), None)
|
||
|
||
|
||
def _cell_words(locator: dict[str, Any], hidden: set[str]) -> str:
|
||
"""输入落在哪一格：「r0 行 gmv 列」，数组字段是一段行「r0–r4 行 amount 列」。遮罩的列不写列名。"""
|
||
rows = locator.get("rows")
|
||
if isinstance(locator.get("row"), int):
|
||
where = f"r{locator['row']} 行"
|
||
elif isinstance(rows, list) and len(rows) == 2 and all(isinstance(r, int) for r in rows):
|
||
where = f"r{rows[0]}–r{rows[1]} 行"
|
||
else:
|
||
where = ""
|
||
cols = [locator["column"]] if locator.get("column") else list((locator.get("columns") or {}).values())
|
||
cols = [str(c) for c in cols if c not in (None, "")]
|
||
shown = [c for c in cols if c.lower() not in hidden]
|
||
parts = [where] if where else []
|
||
if shown:
|
||
parts.append("、".join(shown) + " 列")
|
||
if len(shown) < len(cols):
|
||
parts.append("（所在的列按数据源的设置遮罩，不写列名）")
|
||
return " ".join(parts) or "某一格"
|
||
|
||
|
||
def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any],
|
||
masked: dict[str, Any] | None) -> list[str]:
|
||
"""指标的输入取自哪次查询、哪一格，再附上那次查询的 SQL（截到 SQL_CHARS，最多 ENTITY_QUERIES 次）。
|
||
|
||
「活跃用户按会员号去重」这种讲口径的句子，证据在取数的 SQL 里；只给原式，裁判只能判证据不足。
|
||
位置来自封存的口径卡；SQL 经 fetch 取，取不到（不在封存范围里）的不给。没追到查询的输入照实写。
|
||
"""
|
||
items = [i for i in metric.get("inputs") or [] if isinstance(i, dict)]
|
||
lines: list[str] = []
|
||
sqls: list[str] = []
|
||
seen: set[str] = set()
|
||
for item in items[:METRIC_INPUTS]:
|
||
path = _clean(str(item.get("path") or item.get("field") or "")) or "（未命名）"
|
||
artifact = item.get("artifact") if item.get("via") in ("tool_cell", "agent_field") else None
|
||
if not artifact:
|
||
origin = f"（来自节点 {item['node_id']}）" if item.get("node_id") else ""
|
||
lines.append(f"输入 {path}：未追溯到查询{origin}")
|
||
continue
|
||
alias = _query_alias(catalog, artifact)
|
||
snapshot = fetch(artifact)
|
||
snap = snapshot if isinstance(snapshot, dict) else {}
|
||
source = (catalog.get(alias) or {}).get("source") if alias else None
|
||
hidden = _source_hidden(source or snap.get("source"), catalog, fetch, masked) \
|
||
| _mask_names(snap.get("mask_columns"))
|
||
label = f"查询 {alias} " if alias else "一次查询（不在这份报告的证据目录中）"
|
||
lines.append(f"输入 {path}：取自{label}的 {_cell_words(item.get('locator') or {}, hidden)}")
|
||
sql = _clean(str(snap.get("sql") or ""))
|
||
if sql and str(artifact) not in seen and len(seen) < ENTITY_QUERIES:
|
||
seen.add(str(artifact))
|
||
sqls.append(_sql_line(f"查询 {alias} 的 SQL" if alias else "这次查询的 SQL", sql))
|
||
if len(items) > METRIC_INPUTS:
|
||
lines.append(f"…另有 {len(items) - METRIC_INPUTS} 个输入")
|
||
return lines + sqls
|
||
|
||
|
||
def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any],
|
||
masked: dict[str, Any] | None) -> str:
|
||
name = entry.get("name") or (entry.get("locator") or {}).get("metric") or alias
|
||
rendered = entry.get("rendered")
|
||
where = " ".join(v for v in (f"口径「{entry['caliber']}」" if entry.get("caliber") else "",
|
||
str(entry.get("version") or "")) if v)
|
||
value = f"{name}：这次没有值" if entry.get("value") is None or rendered in (None, MISSING) \
|
||
else f"{name} = {rendered}"
|
||
lines = [f"【{alias}】口径卡指标：{value}" + (f"（{where}）" if where else "")]
|
||
card = fetch(entry.get("artifact"))
|
||
wanted = (entry.get("locator") or {}).get("metric")
|
||
metric = next((m for m in (card.get("metrics") if isinstance(card, dict) else None) or []
|
||
if isinstance(m, dict) and m.get("id") == wanted), None)
|
||
if metric is not None:
|
||
if metric.get("expression"):
|
||
lines.append(f"原式：{_clean(str(metric['expression']))}")
|
||
lines.extend(_metric_inputs(metric, fetch, catalog, masked))
|
||
return "\n".join(lines)
|
||
|
||
|
||
def _table_lines(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch,
|
||
masked: dict[str, Any] | None) -> list[str]:
|
||
"""表的字段清单和类型：取自封存范围内（经 fetch）的表结构快照，最多 FIELDS_MAX 个，遮罩的列不列。
|
||
|
||
讲表结构的句子（「订单表按会员号记录」）只给表名，裁判只能判证据不足。快照不完整（数据源的表太多、
|
||
只存了一部分）要注明：这时「某字段不存在」这类说法只能判 insufficient。
|
||
"""
|
||
name = entry.get("name") or (entry.get("locator") or {}).get("table")
|
||
origins = [o for o in entry.get("sources") or [] if isinstance(o, dict) and o.get("kind") == "schema"
|
||
and o.get("artifact")]
|
||
truncated = any(o.get("truncated") for o in origins)
|
||
info: dict[str, Any] | None = None
|
||
snapshot: dict[str, Any] = {}
|
||
readable = False
|
||
for origin in origins:
|
||
content = fetch(origin["artifact"])
|
||
readable = readable or isinstance(content, dict)
|
||
if (found := schema_table(content, name)) is not None:
|
||
info, snapshot = found, content
|
||
break
|
||
lines: list[str] = []
|
||
if info is None:
|
||
lines.append("字段清单：没有（" + ("本次运行的表结构快照中没有这张表" if readable or not origins
|
||
else "无法读取这张表所在的表结构快照") + "）")
|
||
else:
|
||
truncated = truncated or bool(snapshot.get("truncated"))
|
||
hidden = _source_hidden(entry.get("source") or snapshot.get("source"), catalog, fetch, masked)
|
||
fields, more, masked_any = table_fields(info, hidden, limit=FIELDS_MAX)
|
||
listed = " | ".join(f"{f['name']} {f['type']}".strip() for f in fields) or "无"
|
||
lines.append(f"字段清单（名称 类型，共 {len(fields) + more} 个）：{listed}"
|
||
+ (f" | …另有 {more} 个字段" if more else ""))
|
||
if masked_any:
|
||
lines.append("（有的字段按数据源的设置遮罩了，没有列出）")
|
||
if info.get("comment") == UNSHAPED_NOTE:
|
||
# 上传时选了「按原样导入（未规整）」：对这张表的列直接求和得出的数，裁判要知道它不可靠（R4、D16）
|
||
lines.append(f"表的说明：{UNSHAPED_NOTE}")
|
||
elif info.get("kind") == "reported_total":
|
||
# 按配方导入另存的「原表写明的合计」：各合计项可能重叠，相加得出的数不可靠（AU-5）
|
||
lines.append("表的说明：原表写明的合计，各合计项可能互相重叠，不要彼此相加，也不要与明细相加")
|
||
if truncated:
|
||
lines.append("注意：表结构快照不完整（数据源的表太多，只保存了一部分），不能据此断定某张表或某个字段不存在")
|
||
return lines
|
||
|
||
|
||
def _column_lines(entry: dict[str, Any], fetch: _Fetch) -> list[str]:
|
||
"""字段所属的表和类型。类型只从封存范围内的表结构快照取；表结构里没有的列（聚合的别名）才取查询快照
|
||
记下的列类型——和证据面板同一个口径。"""
|
||
locator = entry.get("locator") or {}
|
||
table, column = locator.get("table"), locator.get("column") or entry.get("name")
|
||
owners = [str(table)] if table else [str(t) for t in entry.get("tables") or []]
|
||
origins = [o for o in entry.get("sources") or [] if isinstance(o, dict) and o.get("artifact")]
|
||
types: dict[str, str] = {}
|
||
for origin in origins:
|
||
if origin.get("kind") != "schema":
|
||
continue
|
||
content = fetch(origin["artifact"])
|
||
for owner in owners:
|
||
if (col := schema_column(schema_table(content, owner), column)) is not None and col.get("type"):
|
||
types.setdefault(owner, str(col["type"]))
|
||
lines = ["所属的表：" + "、".join(owners)] if owners else []
|
||
if len(set(types.values())) == 1:
|
||
lines.append(f"类型：{next(iter(types.values()))}（取自表结构快照）")
|
||
elif types:
|
||
lines.append("类型：" + "；".join(f"{t} 表中为 {k}" for t, k in types.items()) + "（取自表结构快照）")
|
||
elif not any(o.get("kind") == "schema" for o in origins):
|
||
for origin in origins:
|
||
content = fetch(origin["artifact"]) if origin.get("kind") == "result" else None
|
||
kinds = content.get("column_types") if isinstance(content, dict) else None
|
||
if isinstance(kinds, dict) and isinstance(kinds.get(str(column)), str):
|
||
lines.append(f"类型：{kinds[str(column)]}（取自查询结果）")
|
||
break
|
||
return lines
|
||
|
||
|
||
def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], whole: bool, fetch: _Fetch) -> str | None:
|
||
content = fetch(entry.get("artifact"))
|
||
hits = content.get("hits") if isinstance(content, dict) else None
|
||
if not isinstance(hits, list):
|
||
return None
|
||
chosen = [h for h in dict.fromkeys(picked) if 0 <= h < len(hits)]
|
||
if whole:
|
||
chosen += [h for h in range(len(hits)) if h not in chosen][:max(0, EXCERPT_HITS - len(chosen))]
|
||
lines = [f"【{alias}】知识库检索（{entry.get('source') or '—'} · 共 {len(hits)} 段）"]
|
||
for h in sorted(chosen):
|
||
hit = hits[h]
|
||
if not isinstance(hit, dict) or not isinstance(hit.get("content"), str):
|
||
continue
|
||
body = _clean(hit["content"])
|
||
title = f"《{_clean(str(hit['title']))}》" if hit.get("title") else ""
|
||
lines.append(f"片段 {h + 1}{title}：{body[:CHUNK_CHARS]}{'…' if len(body) > CHUNK_CHARS else ''}")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _Fetch,
|
||
masked: dict[str, Any] | None, catalog: dict[str, Any], *, provenance_on: bool = False) -> str | None:
|
||
"""一件证据的摘录。provenance_on 只交给查询摘录（_query_excerpt）：指标摘录里「输入取自查询 Qn」的那几行不加
|
||
来历（P4-SPEC 调整 1），表名、字段名的摘录照旧。"""
|
||
kind = entry.get("kind")
|
||
if kind == "metric":
|
||
return _metric_excerpt(alias, entry, fetch, catalog, masked)
|
||
if kind == "input":
|
||
field_name = (entry.get("locator") or {}).get("field") or alias
|
||
return f"【{alias}】运行输入：{field_name} = {entry.get('rendered')}"
|
||
if kind == "query":
|
||
rows = {r for c in cands for a, r in c.rows if a == alias}
|
||
cols = [col for c in cands for a, col in c.cols if a == alias]
|
||
return _query_excerpt(alias, entry, rows, cols, any(alias in c.whole for c in cands), fetch, masked,
|
||
provenance_on=provenance_on)
|
||
if kind == "retrieval":
|
||
picked = [h for c in cands for a, h in c.hits if a == alias]
|
||
return _retrieval_excerpt(alias, entry, picked, any(alias in c.whole for c in cands), fetch)
|
||
if kind == "table":
|
||
head = f"【{alias}】表 {entry.get('name')}" + (f"（数据源 {entry['source']}）" if entry.get("source") else "")
|
||
return "\n".join([head, *_table_lines(entry, catalog, fetch, masked), *_entity_sql(entry, catalog, fetch)])
|
||
if kind == "column":
|
||
table = (entry.get("locator") or {}).get("table")
|
||
head = f"【{alias}】字段 {f'{table}.' if table else ''}{(entry.get('locator') or {}).get('column') or entry.get('name')}"
|
||
return "\n".join([head, *_column_lines(entry, fetch), *_entity_sql(entry, catalog, fetch)])
|
||
return None
|
||
|
||
|
||
def _entity_sql(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch) -> list[str]:
|
||
"""表名、字段名出现过的查询的 SQL（每条截到 SQL_CHARS，最多 ENTITY_QUERIES 条）。
|
||
|
||
讲算法的句子（「按会员号去重计数得出」）证据在 SQL 里；只给名字，裁判只能判不支持。
|
||
查询用条目自己记的（表：SQL 里用到它；字段：它是结果列）。只在 SQL 里出现、不是结果列的字段自己没记：
|
||
有表条目就从它记的查询里找，没有（文档目录只留正文用到的实体）就看全部查询，都只留 SQL 里写了这个
|
||
字段名（有表名的还要写了表名）的。快照经 fetch 取：取不到（不在封存范围里）的不给。
|
||
"""
|
||
locator = entry.get("locator") or {}
|
||
column = locator.get("column") if entry.get("kind") == "column" else None
|
||
table = locator.get("table")
|
||
aliases = [q for q in entry.get("queries") or [] if isinstance(catalog.get(q), dict)]
|
||
need = [str(column)] if column else []
|
||
if not aliases and column:
|
||
owner = catalog.get(f"t:{table}") if table else None
|
||
aliases = [q for q in (owner.get("queries") or []) if isinstance(catalog.get(q), dict)] \
|
||
if isinstance(owner, dict) else \
|
||
[a for a, e in catalog.items() if isinstance(e, dict) and e.get("kind") == "query"]
|
||
need += [str(table)] if table and not isinstance(owner, dict) else []
|
||
lines: list[str] = []
|
||
for q in aliases:
|
||
snapshot = fetch(catalog[q].get("artifact"))
|
||
sql = _clean(str(snapshot.get("sql") or "")) if isinstance(snapshot, dict) else ""
|
||
if not sql or any(not re.search(rf"(?<![\w]){re.escape(n)}(?![\w])", sql, re.I) for n in need):
|
||
continue
|
||
lines.append(f"出现在查询 {q}：{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}")
|
||
if len(lines) == ENTITY_QUERIES:
|
||
break
|
||
return lines
|
||
|
||
|
||
def _tokens(text: str) -> int:
|
||
"""估算令牌数：中日韩的字一个字算一个，其余大约 3.5 个字符一个。宁可估多，不估少。"""
|
||
wide = sum(1 for ch in text if ord(ch) > 0x2E7F)
|
||
return wide + math.ceil((len(text) - wide) / 3.5)
|
||
|
||
|
||
@dataclass
|
||
class JudgeRequest:
|
||
"""一次裁判的全部输入：候选句、每件证据的摘录、预筛放掉的结论句、点名了却不能判的句子。"""
|
||
|
||
cands: list[Candidate]
|
||
excerpts: dict[str, str]
|
||
screened: list[str]
|
||
skipped: dict[str, str]
|
||
|
||
def unit_line(self, c: Candidate) -> str:
|
||
cited = "、".join(a for a in c.aliases if a in self.excerpts) or "无"
|
||
where = f" ｜ 小节：{c.section}" if c.section else ""
|
||
return f"[{c.unit}] {c.text}\n 依据：{cited}{where}"
|
||
|
||
def messages(self, units: list[Candidate]) -> list[BaseMessage]:
|
||
aliases = list(dict.fromkeys(a for c in units for a in c.aliases if a in self.excerpts))
|
||
evidence = "\n\n".join(self.excerpts[a] for a in aliases) or "（这些句子都没有挂依据）"
|
||
ordered = sorted(units, key=lambda c: c.order)
|
||
return [
|
||
SystemMessage(content=f"{JUDGE_ROLE}\n\n{JUDGE_RULES}"),
|
||
HumanMessage(content=(
|
||
f"证据摘录（【】里是编号，判定时 used 写它）：\n{evidence}\n\n"
|
||
"要判断的句子（每句都要给判定）：\n" + "\n".join(self.unit_line(c) for c in ordered) + "\n\n"
|
||
'只输出 JSON：{"items": [{"unit": "u1", "verdict": "supported", "rationale": "…", "missing": "", '
|
||
'"used": ["m:gmv"]}]}'
|
||
)),
|
||
]
|
||
|
||
def cost(self, model_id: str, units: list[Candidate]) -> float | None:
|
||
"""这几句合成一次调用，按目录价估多少钱；目录里没有这个模型的价格时是 None（估不出金额）。"""
|
||
if pricing.find_pricing(model_id) is None:
|
||
return None
|
||
prompt = "\n".join(str(m.content) for m in self.messages(units))
|
||
out = min(MAX_TOKENS, _OUT_BASE + _OUT_PER_UNIT * len(units))
|
||
return pricing.estimate_cost(model_id, _tokens(prompt), out)
|
||
|
||
def unit_key(self, c: Candidate, model_id: str) -> str:
|
||
"""同一个模型、同一句话、同样的证据摘录：判定可以直接复用（改写之后没动的句子不再花钱判）。"""
|
||
evidence = [self.excerpts.get(a) for a in c.aliases]
|
||
return content_hash(canonical_json({"model": model_id, "text": c.text, "section": c.section,
|
||
"evidence": evidence}))[:16]
|
||
|
||
|
||
def prepare(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, units: Iterable[str] | None = None,
|
||
loader: Callable[[str], Any] | None = None, masked: dict[str, Any] | None = None) -> JudgeRequest:
|
||
"""裁判的输入：候选句和证据摘录。纯函数（快照经 loader 取，缺省读工件库），不调模型。
|
||
|
||
catalog 缺省用文档自己存的目录；masked = {数据源名: 要遮的列}，和快照里记下的 mask_columns 合在一起。
|
||
|
||
摘录按文档标记分版本（P4-SPEC 1.4、3.3）：带 `provenance: DOC_PROVENANCE` 的文档，上传表格的查询摘录多几行
|
||
来历；没有标记的（期 4 之前组装的）和以前逐字相同，复用键、断点指纹都不变，已判过的句子照旧复用。
|
||
"""
|
||
catalog = catalog if catalog is not None else (doc.get("catalog") or {})
|
||
cands = candidates(doc, units)
|
||
fetch = _Fetch(loader)
|
||
provenance_on = doc.get("provenance") == DOC_PROVENANCE
|
||
excerpts: dict[str, str] = {}
|
||
for alias in dict.fromkeys(a for c in cands for a in c.aliases):
|
||
entry = catalog.get(alias)
|
||
if isinstance(entry, dict) and (text := _excerpt(alias, entry, cands, fetch, masked, catalog,
|
||
provenance_on=provenance_on)):
|
||
excerpts[alias] = text
|
||
picked = {c.unit for c in cands}
|
||
skipped: dict[str, str] = {}
|
||
screened: list[str] = []
|
||
ids = set()
|
||
for _, unit in iter_units(doc):
|
||
ids.add(unit.get("id"))
|
||
if units is None and unit.get("kind") == "claim" and unit.get("id") not in picked:
|
||
screened.append(str(unit.get("id")))
|
||
if units is not None:
|
||
for uid in dict.fromkeys(str(u) for u in units):
|
||
if uid not in picked:
|
||
skipped[uid] = "not_a_claim" if uid in ids else "not_found"
|
||
return JudgeRequest(cands=cands, excerpts=excerpts, screened=screened, skipped=skipped)
|
||
|
||
|
||
def request_key(request: JudgeRequest, spec: ModelSpec, budget: Budget, known: dict[str, Any] | None = None) -> str:
|
||
"""一次裁判的输入指纹：报告节点拿它认 checkpoint 里缓存的判定是不是这一稿的。"""
|
||
return content_hash(canonical_json({
|
||
"units": [[c.unit, c.priority, c.text, c.section, list(c.aliases)] for c in request.cands],
|
||
"excerpts": request.excerpts, "model": [spec.provider, spec.model], "budget": budget.as_dict(),
|
||
"known": sorted(known or {}), "rules": RULES_VERSION,
|
||
}))
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 预算与设置
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Budget:
|
||
"""一次裁判的上限。每一项都可以是 None：不限。"""
|
||
|
||
max_claims: int | None = 40
|
||
max_cost_usd: float | None = 0.05
|
||
timeout_s: float | None = 30
|
||
daily_max_usd: float | None = 2.0
|
||
|
||
def as_dict(self) -> dict[str, Any]:
|
||
return asdict(self)
|
||
|
||
|
||
#: 设置里 judge 这一组（DEFAULT_SETTINGS 引用这一份）。上限写 null 表示不限
|
||
SETTING_GROUP = "judge"
|
||
JUDGE_DEFAULTS: dict[str, Any] = {
|
||
"provider": None, # 证据裁判模型：空着依次用 Copilot 模型、默认 provider
|
||
"model": None,
|
||
"report_max_claims": 40, # 每份报告最多判几句
|
||
"report_max_cost_usd": 0.05, # 每份报告的裁判金额上限
|
||
"report_timeout_s": 30, # 每份报告的裁判时长上限（每次调用的超时就是剩下的时长）
|
||
"click_max_cost_usd": 0.01, # 探索运行里每点一次「请模型判断」的金额上限
|
||
"daily_max_usd": 2.0, # 全局每日裁判金额上限（按本地日期）
|
||
}
|
||
#: 每日计数存在设置表的这一行：{date: 本地日期, usd, calls, unpriced_calls}。不从设置接口进出
|
||
SPEND_KEY = "judge_spend"
|
||
_COUNTS = frozenset({"report_max_claims"})
|
||
_SECONDS = frozenset({"report_timeout_s"})
|
||
|
||
|
||
def _limit_ok(key: str, value: Any) -> bool:
|
||
if value is None:
|
||
return True
|
||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0:
|
||
return False
|
||
return not (key in _COUNTS and int(value) != value)
|
||
|
||
|
||
def _limit_words(key: str) -> str:
|
||
if key in _COUNTS:
|
||
return "正整数（句数）"
|
||
if key in _SECONDS:
|
||
return "大于 0 的秒数"
|
||
return "大于 0 的金额（美元）"
|
||
|
||
|
||
def settings_problem(value: Any) -> str | None:
|
||
"""设置页写进来的 judge 这一组有什么问题，没问题返回 None。"""
|
||
if not isinstance(value, dict):
|
||
return f"「证据裁判」设置格式有误，需要是一个对象，当前为「{value}」"
|
||
for key, item in value.items():
|
||
if key not in JUDGE_DEFAULTS:
|
||
return (f"「证据裁判」设置中不支持「{key}」，可设置的项："
|
||
+ "、".join(f"「{SETTING_LABEL.get(k, k)}」" for k in JUDGE_DEFAULTS))
|
||
label = SETTING_LABEL.get(key, key)
|
||
if key in ("provider", "model"):
|
||
if item is not None and not isinstance(item, str):
|
||
return f"「{label}」需要填写模型接入的名称或模型 ID，留空表示不指定，当前为「{item}」"
|
||
elif not _limit_ok(key, item):
|
||
return f"「{label}」需要填写{_limit_words(key)}，留空表示不限，当前为「{item}」"
|
||
return None
|
||
|
||
|
||
#: 设置页「证据裁判」一组各项的叫法（前端 lib/terms.ts 的 JUDGE_SETTING_TEXT）
|
||
SETTING_LABEL = {
|
||
"provider": "裁判模型 · 接入", "model": "裁判模型 · 模型",
|
||
"report_max_claims": "每份报告最多裁判句数", "report_max_cost_usd": "每份报告的金额上限（美元）",
|
||
"report_timeout_s": "每份报告的时长上限（秒）", "click_max_cost_usd": "每次点击的金额上限（美元）",
|
||
"daily_max_usd": "每日金额上限（美元）",
|
||
}
|
||
|
||
|
||
def sanitize_settings(stored: Any) -> dict[str, Any]:
|
||
"""库里存的 judge 一组 → 补齐默认的完整设置。存坏了的项（旧版本、手改）按默认，存了 null 的上限就是不限。"""
|
||
stored = stored if isinstance(stored, dict) else {}
|
||
out = dict(JUDGE_DEFAULTS)
|
||
for key in JUDGE_DEFAULTS:
|
||
if key not in stored:
|
||
continue
|
||
value = stored[key]
|
||
if key in ("provider", "model"):
|
||
out[key] = value.strip() or None if isinstance(value, str) else None
|
||
elif _limit_ok(key, value):
|
||
out[key] = value
|
||
return out
|
||
|
||
|
||
async def judge_settings(session: Any) -> dict[str, Any]:
|
||
"""设置里的 judge 一组，没存过的键补默认（读法同 run_defaults）。"""
|
||
row = await session.get(Setting, SETTING_GROUP)
|
||
return sanitize_settings(row.value if row else None)
|
||
|
||
|
||
def _pick(node: dict[str, Any], key: str, settings: dict[str, Any], setting_key: str) -> Any:
|
||
# 节点上写了（包括写 null：不限）就按节点，没写才取设置
|
||
return node[key] if key in node else settings.get(setting_key)
|
||
|
||
|
||
def report_budget(settings: dict[str, Any], node_judge: dict[str, Any] | None = None) -> Budget:
|
||
"""一份报告的裁判上限：节点 judge 里写了的键盖过设置；每日上限只在设置里。"""
|
||
node = node_judge if isinstance(node_judge, dict) else {}
|
||
return Budget(max_claims=_pick(node, "max_claims", settings, "report_max_claims"),
|
||
max_cost_usd=_pick(node, "max_cost_usd", settings, "report_max_cost_usd"),
|
||
timeout_s=_pick(node, "timeout_s", settings, "report_timeout_s"),
|
||
daily_max_usd=settings.get("daily_max_usd"))
|
||
|
||
|
||
def click_budget(settings: dict[str, Any], node_judge: dict[str, Any] | None = None) -> Budget:
|
||
"""探索运行里点一次「请模型判断」的上限：金额按每次点击的上限，句数、时长按每份报告的。"""
|
||
node = node_judge if isinstance(node_judge, dict) else {}
|
||
return Budget(max_claims=_pick(node, "max_claims", settings, "report_max_claims"),
|
||
max_cost_usd=settings.get("click_max_cost_usd"),
|
||
timeout_s=_pick(node, "timeout_s", settings, "report_timeout_s"),
|
||
daily_max_usd=settings.get("daily_max_usd"))
|
||
|
||
|
||
def _text(value: Any) -> str | None:
|
||
return value.strip() or None if isinstance(value, str) else None
|
||
|
||
|
||
async def judge_model_spec(session: Any, node_judge: dict[str, Any] | None = None, *,
|
||
settings: dict[str, Any] | None = None) -> ModelSpec:
|
||
"""裁判用哪个模型：节点上的 judge.model（连同 provider）> 设置里的证据裁判模型 > Copilot 模型 >
|
||
默认 provider。哪一级写了就整组用哪一级，不跨级拼 provider 和模型。thinking 关、max_tokens 4096。
|
||
|
||
建议和写作模型不同，否则等于自己审自己（报告节点发现相同会发 judge_same_model 警告）。
|
||
"""
|
||
from app.api.copilot import COPILOT_SETTING_KEY
|
||
|
||
node = node_judge if isinstance(node_judge, dict) else {}
|
||
provider, model = _text(node.get("provider")), _text(node.get("model"))
|
||
if not (provider or model):
|
||
chosen = settings if settings is not None else await judge_settings(session)
|
||
provider, model = _text(chosen.get("provider")), _text(chosen.get("model"))
|
||
if not (provider or model):
|
||
row = await session.get(Setting, COPILOT_SETTING_KEY)
|
||
saved = row.value if row is not None and isinstance(row.value, dict) else {}
|
||
provider, model = _text(saved.get("provider")), _text(saved.get("model"))
|
||
return ModelSpec(provider=provider, model=model, max_tokens=MAX_TOKENS, thinking="off")
|
||
|
||
|
||
async def source_masks(catalog: dict[str, Any], *, loader: Callable[[str], Any] | None = None
|
||
) -> dict[str, list[str]]:
|
||
"""目录里的证据来自的数据源现在设的遮罩：{数据源名: 列名}。和证据面板同一个口径。
|
||
|
||
查的数据源：查询条目的；表、字段条目的（字段清单里遮罩的列不列）；口径卡指标的输入取自的查询快照的。
|
||
条目没记数据源的，按快照自己记的数据源查（快照经 loader 取，缺省读工件库；按需裁判的接口要传只认
|
||
封存工件的 loader，和 prepare 同一个）。
|
||
"""
|
||
from app.data.engine import masked_columns
|
||
from app.db.models import DataSource
|
||
|
||
fetch = _Fetch(loader)
|
||
found: set[str] = set()
|
||
|
||
def source_of(artifact: Any) -> Any:
|
||
snapshot = fetch(artifact)
|
||
return snapshot.get("source") if isinstance(snapshot, dict) else None
|
||
|
||
for entry in catalog.values():
|
||
if not isinstance(entry, dict):
|
||
continue
|
||
kind = entry.get("kind")
|
||
sources: list[Any] = []
|
||
if kind == "query":
|
||
sources.append(entry.get("source") or source_of(entry.get("artifact")))
|
||
elif kind in ENTITY_KINDS:
|
||
sources.append(entry.get("source") or next(
|
||
(source_of(o.get("artifact")) for o in entry.get("sources") or []
|
||
if isinstance(o, dict) and o.get("kind") == "schema"), None))
|
||
elif kind == "metric":
|
||
card = fetch(entry.get("artifact"))
|
||
wanted = (entry.get("locator") or {}).get("metric")
|
||
for metric in (card.get("metrics") if isinstance(card, dict) else None) or []:
|
||
if isinstance(metric, dict) and metric.get("id") == wanted:
|
||
sources.extend(source_of(i.get("artifact")) for i in metric.get("inputs") or []
|
||
if isinstance(i, dict) and i.get("artifact"))
|
||
found.update(str(s) for s in sources if s)
|
||
names = sorted(found)
|
||
if not names:
|
||
return {}
|
||
async with SessionLocal() as session:
|
||
rows = (await session.execute(select(DataSource.name, DataSource.options)
|
||
.where(DataSource.name.in_(names)))).all()
|
||
return {str(name): cols for name, options in rows if (cols := masked_columns(options))}
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 每日计数
|
||
# --------------------------------------------------------------------------
|
||
|
||
_spend_lock = asyncio.Lock()
|
||
|
||
|
||
def _today() -> str:
|
||
"""本地日期：每日上限按它滚动。"""
|
||
return datetime.now().astimezone().date().isoformat()
|
||
|
||
|
||
def _spend_of(row: Any) -> dict[str, Any]:
|
||
value = row.value if row is not None and isinstance(row.value, dict) else {}
|
||
today = _today()
|
||
if value.get("date") != today:
|
||
return {"date": today, "usd": 0.0, "calls": 0, "unpriced_calls": 0}
|
||
return {"date": today, "usd": float(value.get("usd") or 0.0), "calls": int(value.get("calls") or 0),
|
||
"unpriced_calls": int(value.get("unpriced_calls") or 0)}
|
||
|
||
|
||
async def daily_spend() -> dict[str, Any]:
|
||
"""今天（本地日期）裁判已经花了多少：{date, usd, calls, unpriced_calls}。估不出金额的调用只计次数。"""
|
||
async with SessionLocal() as session:
|
||
return _spend_of(await session.get(Setting, SPEND_KEY))
|
||
|
||
|
||
async def _charge(amount: float, *, limit: float | None, priced: bool) -> str | None:
|
||
"""记一次调用、先按估算预留金额。加上这笔会超过每日上限就不记，返回 None；记下了返回记在哪天。"""
|
||
async with _spend_lock:
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Setting, SPEND_KEY)
|
||
spend = _spend_of(row)
|
||
if limit is not None and spend["usd"] + amount > limit + 1e-12:
|
||
return None
|
||
spend["usd"] = round(spend["usd"] + amount, 8)
|
||
spend["calls"] += 1
|
||
spend["unpriced_calls"] += 0 if priced else 1
|
||
if row is None:
|
||
session.add(Setting(key=SPEND_KEY, value=spend))
|
||
else:
|
||
row.value = spend
|
||
await session.commit()
|
||
return spend["date"]
|
||
|
||
|
||
async def _settle(date: str, delta: float) -> None:
|
||
"""调用结束后按实际用量找补预留的差额。跨了天就不找补：预留记在前一天，今天从零算。"""
|
||
if not delta:
|
||
return
|
||
async with _spend_lock:
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Setting, SPEND_KEY)
|
||
spend = _spend_of(row)
|
||
if row is None or spend["date"] != date:
|
||
return
|
||
spend["usd"] = max(0.0, round(spend["usd"] + delta, 8))
|
||
row.value = spend
|
||
await session.commit()
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 调用
|
||
# --------------------------------------------------------------------------
|
||
|
||
Emit = Callable[..., None]
|
||
|
||
|
||
def _quiet(*_args: Any, **_data: Any) -> None:
|
||
return None
|
||
|
||
|
||
def _json_in(text: str) -> Any:
|
||
from app.api.copilot import _extract_json
|
||
|
||
try:
|
||
return json.loads(text)
|
||
except ValueError:
|
||
pass
|
||
try:
|
||
return _extract_json(text)
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def _items_of(value: Any) -> list[Any] | None:
|
||
"""判定清单：{"items": [...]}，或者直接一个数组。有的模型走工具调用时把嵌套的数组写成一段 JSON 文本
|
||
（{"items": "[...]"}），也有把整个对象写进正文的：都认。"""
|
||
if isinstance(value, str):
|
||
value = _json_in(value)
|
||
if isinstance(value, dict) and isinstance(value.get("items"), str):
|
||
value = {**value, "items": _json_in(value["items"])}
|
||
if isinstance(value, dict) and isinstance(value.get("items"), list):
|
||
return value["items"]
|
||
return value if isinstance(value, list) else None
|
||
|
||
|
||
async def _ask(model: Any, messages: list[BaseMessage]) -> tuple[list[Any] | None, list[BaseMessage], str | None,
|
||
int]:
|
||
"""问一次裁判：(判定清单, 模型的原始回复, 没成的原因, 调了几次模型)。
|
||
|
||
先走结构化输出；不成（模型不支持、给的不是合法结构）先从原始回复里找 JSON，再退回纯文本调用、
|
||
从回复里抠 JSON。原因以 error: 开头的是调用本身失败，format 是格式不对。超时由调用方掐。
|
||
"""
|
||
raws: list[BaseMessage] = []
|
||
try:
|
||
got = await model.with_structured_output(JUDGE_SCHEMA, include_raw=True).ainvoke(messages)
|
||
raw, parsed = _split_structured(got)
|
||
if raw is not None:
|
||
raws.append(raw)
|
||
if (items := _items_of(parsed)) is not None:
|
||
return items, raws, None, 1
|
||
if raw is not None:
|
||
# 有的网关把 JSON 写在正文里、或者塞进了一个没按 schema 解析出来的工具调用
|
||
for found in [message_text(raw), *(c.get("args") for c in getattr(raw, "tool_calls", None) or [])]:
|
||
if (items := _items_of(found)) is not None:
|
||
return items, raws, None, 1
|
||
except Exception: # noqa: BLE001 - 不支持结构化输出的模型退回文本解析
|
||
pass
|
||
try:
|
||
reply = await model.ainvoke(messages)
|
||
except Exception as e: # noqa: BLE001
|
||
return None, raws, f"error:{describe_exception(e)}", 2
|
||
raws.append(reply)
|
||
items = _items_of(message_text(reply))
|
||
return items, raws, None if items is not None else "format", 2
|
||
|
||
|
||
def _unjudged(reason: str, rationale: str, *, judge: str | None, post_seal: bool) -> dict[str, Any]:
|
||
return {"status": "unjudged", "rationale": rationale, "judge": judge, "post_seal": post_seal, "reason": reason}
|
||
|
||
|
||
def _limit_label(reason: str, budget: Budget, *, click: bool = False) -> str:
|
||
if reason == "max_claims":
|
||
return f"每{'次点击' if click else '份报告'}最多裁判 {budget.max_claims} 句"
|
||
if reason == "max_cost_usd":
|
||
return f"{'这次点击' if click else '这份报告'}的裁判金额上限 {_usd(budget.max_cost_usd)}"
|
||
if reason == "daily_max_usd":
|
||
return f"全局每日裁判金额上限 {_usd(budget.daily_max_usd)}"
|
||
return f"裁判时长上限 {budget.timeout_s:g} 秒"
|
||
|
||
|
||
def _usd(amount: Any) -> str:
|
||
"""金额写两位小数（$0.05）；比一分钱还细的上限照实写，不四舍五入成别的数。"""
|
||
value = float(amount or 0)
|
||
return f"${value:.2f}" if round(value, 2) == value else f"${value:g}"
|
||
|
||
|
||
def _why_unjudged(reason: str, budget: Budget, detail: str = "", *, click: bool = False) -> str:
|
||
if reason in LIMITS:
|
||
return f"已达上限（{_limit_label(reason, budget, click=click)}），这句未裁判"
|
||
if reason == "error":
|
||
return f"裁判失败：{detail}"[:120]
|
||
if reason == "format":
|
||
return detail or "裁判返回的结果格式有误，这句未裁判"
|
||
if reason == "missing":
|
||
return "裁判未对这句给出判定"
|
||
return "探索运行按需裁判：点开这句并请模型判断后才会裁判"
|
||
|
||
|
||
async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget, known: dict[str, Any] | None = None,
|
||
post_seal: bool = False, emit: Emit | None = None, click: bool = False,
|
||
shown: Budget | None = None) -> dict[str, Any]:
|
||
"""按预算判一份请求。永远不抛：模型没配好、调用失败、格式不对、触顶，都落成未裁判和一条缺口。
|
||
|
||
返回 outcome：{model, priced, verdicts:{unit: verdict}, keys:{unit: 复用键}, candidates, screened,
|
||
skipped, counts, unjudged:{原因: 句数}, limits_hit, cost_usd, estimated_usd, calls, reused, duration_ms,
|
||
usage, budget, gaps, notes}。click=True 时上限的说法按「这次点击」写。
|
||
|
||
shown 只管说给人看的上限：同一份报告分几轮判时（rewrite_once 的第二轮），budget 是这一轮剩下的钱和
|
||
时间，未裁判的理由、缺口、judge_limit 事件里写的却该是配置的上限（「$0.05」，不是剩下的零头）。
|
||
"""
|
||
emit = emit or _quiet
|
||
label = shown or budget
|
||
started = time.monotonic()
|
||
spec = spec.model_copy(update={"thinking": "off", "max_tokens": MAX_TOKENS,
|
||
"timeout": math.ceil(budget.timeout_s) if budget.timeout_s else spec.timeout})
|
||
verdicts: dict[str, dict[str, Any]] = {}
|
||
details: dict[str, str] = {}
|
||
usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cost_usd": 0.0, "calls": 0}
|
||
notes: list[str] = []
|
||
estimated = 0.0
|
||
reused = 0
|
||
model = None
|
||
model_id = spec.model or ""
|
||
try:
|
||
async with SessionLocal() as session:
|
||
model, model_id = await get_chat_model(session, spec)
|
||
except Exception as e: # noqa: BLE001 - 裁判模型没配好：这回判不了，报告照常
|
||
why = not_configured(e) if isinstance(e, ProviderNotConfigured) else describe_exception(e)
|
||
details["error"] = f"裁判模型不可用（{why}）"
|
||
judge = model_id or None
|
||
priced = model is not None and pricing.find_pricing(model_id) is not None
|
||
keys = {c.unit: request.unit_key(c, model_id) for c in request.cands}
|
||
|
||
def stop(units: Iterable[Candidate], reason: str, detail: str = "") -> None:
|
||
for c in units:
|
||
verdicts.setdefault(c.unit, _unjudged(reason, _why_unjudged(reason, label, detail, click=click),
|
||
judge=judge, post_seal=post_seal))
|
||
|
||
ranked = list(request.cands)
|
||
if budget.max_claims is not None:
|
||
stop(ranked[budget.max_claims:], "max_claims")
|
||
ranked = ranked[:budget.max_claims]
|
||
pending: list[Candidate] = []
|
||
for c in ranked:
|
||
hit = (known or {}).get(keys[c.unit])
|
||
if isinstance(hit, dict) and hit.get("status") in SETTLED:
|
||
verdicts[c.unit] = dict(hit)
|
||
reused += 1
|
||
else:
|
||
pending.append(c)
|
||
|
||
if model is None:
|
||
stop(pending, "error", details["error"])
|
||
pending = []
|
||
elif pending and not priced and (budget.max_cost_usd is not None or budget.daily_max_usd is not None):
|
||
note = (f"模型「{model_id}」不在价格目录中，无法按 token 数估算金额：金额上限（每份报告、每次点击、每日）"
|
||
"对其无效，费用只受句数和时长上限约束")
|
||
notes.append(note)
|
||
emit(EventType.LOG, level="warn", code="judge_unpriced", message=note)
|
||
|
||
spent = 0.0
|
||
width = 1
|
||
while pending:
|
||
remaining = None if budget.timeout_s is None else budget.timeout_s - (time.monotonic() - started)
|
||
if remaining is not None and remaining <= 0:
|
||
stop(pending, "timeout_s")
|
||
break
|
||
# 一波最多 PARALLEL_BATCHES 批：逐批按金额上限定下来（这一波已经定下的按估算先占着），再一起问
|
||
wave: list[tuple[list[Candidate], float, Any]] = []
|
||
reserved = 0.0
|
||
while pending and len(wave) < width:
|
||
batch = pending[:BATCH_SIZE]
|
||
cost = request.cost(model_id, batch) if priced else None
|
||
if cost is not None:
|
||
caps = []
|
||
if budget.max_cost_usd is not None:
|
||
caps.append((budget.max_cost_usd - spent - reserved, "max_cost_usd"))
|
||
if budget.daily_max_usd is not None:
|
||
caps.append((budget.daily_max_usd - (await daily_spend())["usd"], "daily_max_usd"))
|
||
if caps:
|
||
cap, binding = min(caps, key=lambda x: x[0])
|
||
trimmed = False
|
||
while batch and (cost := request.cost(model_id, batch) or 0.0) > cap + 1e-12:
|
||
batch, trimmed = batch[:-1], True # 按优先级从后往前丢：先判挂了依据的
|
||
if not batch:
|
||
stop(pending, binding)
|
||
pending = []
|
||
break
|
||
if trimmed:
|
||
# 这一批已经是剩下的钱装得下的最多句子，后面的句子更装不下
|
||
stop(pending[len(batch):], binding)
|
||
pending = pending[:len(batch)]
|
||
pending = pending[len(batch):]
|
||
charged = cost or 0.0
|
||
day = await _charge(charged, limit=budget.daily_max_usd if priced else None, priced=priced)
|
||
if day is None: # 估算之后、记账之前别的裁判先花掉了今天剩下的钱
|
||
stop([*batch, *pending], "daily_max_usd")
|
||
pending = []
|
||
break
|
||
estimated += charged
|
||
reserved += charged
|
||
wave.append((batch, charged, day))
|
||
if not wave:
|
||
break
|
||
|
||
began = time.perf_counter()
|
||
for batch, _, _ in wave:
|
||
emit(EventType.LLM_START, model=model_id, structured=True, purpose="judge", message_count=2,
|
||
units=len(batch))
|
||
tasks = [asyncio.create_task(_ask(model, request.messages(batch))) for batch, _, _ in wave]
|
||
done, late = await asyncio.wait(tasks, timeout=remaining)
|
||
for task in late:
|
||
task.cancel()
|
||
if late:
|
||
await asyncio.gather(*late, return_exceptions=True)
|
||
failed: str | None = None
|
||
for (batch, charged, day), task in zip(wave, tasks):
|
||
items: list[Any] | None = None
|
||
raws: list[BaseMessage] = []
|
||
problem: str | None = None
|
||
calls = 1
|
||
if task in done:
|
||
items, raws, problem, calls = task.result()
|
||
else:
|
||
problem = "timeout_s"
|
||
tokens = [_usage_of(r, model_id) for r in raws]
|
||
inp, out = sum(t["input_tokens"] for t in tokens), sum(t["output_tokens"] for t in tokens)
|
||
# 回复没带用量（有的网关不给、调用超时）就按估算记：宁可多记，不能让每日上限漏掉
|
||
actual = (pricing.estimate_cost(model_id, inp, out) if inp or out else charged) if priced else 0.0
|
||
await _settle(day, actual - charged)
|
||
spent += actual
|
||
usage["input_tokens"] += inp
|
||
usage["output_tokens"] += out
|
||
usage["calls"] += calls
|
||
usage["cost_usd"] += actual
|
||
end = {"model": model_id, "purpose": "judge", "duration_ms": int((time.perf_counter() - began) * 1000),
|
||
"input_tokens": inp, "output_tokens": out, "total_tokens": inp + out,
|
||
"cost_usd": round(actual, 6), "calls": calls, "units": len(batch)}
|
||
if problem is not None and problem != "format":
|
||
reason = "timeout_s" if problem == "timeout_s" else "error"
|
||
detail = problem.partition(":")[2]
|
||
emit(EventType.LLM_END, **end, error=detail or _why_unjudged(reason, label, click=click))
|
||
if reason == "error":
|
||
# 调用本身失败（鉴权、网关挂了）：后面几批多半一样，不再一次次去撞
|
||
details["error"] = failed = detail
|
||
stop(batch, reason, detail)
|
||
continue
|
||
emit(EventType.LLM_END, **end)
|
||
if items is None:
|
||
stop(batch, "format")
|
||
continue
|
||
_take(items, batch, request, verdicts, judge=judge, post_seal=post_seal)
|
||
stop(batch, "missing")
|
||
if failed is not None:
|
||
stop(pending, "error", failed)
|
||
break
|
||
width = PARALLEL_BATCHES
|
||
|
||
for c in request.cands: # 兜底：每个候选都有判定
|
||
verdicts.setdefault(c.unit, _unjudged("missing", _why_unjudged("missing", label), judge=judge,
|
||
post_seal=post_seal))
|
||
usage["total_tokens"] = usage["input_tokens"] + usage["output_tokens"]
|
||
usage["cost_usd"] = round(usage["cost_usd"], 6)
|
||
outcome = _outcome(request, verdicts, budget, details, click=click, shown=label)
|
||
outcome.update({
|
||
"model": judge, "priced": priced, "keys": keys, "cost_usd": usage["cost_usd"],
|
||
"estimated_usd": round(estimated, 6), "calls": usage["calls"], "reused": reused,
|
||
"duration_ms": int((time.monotonic() - started) * 1000), "usage": usage, "notes": notes,
|
||
})
|
||
if hit := outcome["limits_hit"]:
|
||
labels = "、".join(_limit_label(r, label, click=click) for r in hit)
|
||
n = sum(outcome["unjudged"].get(r, 0) for r in hit)
|
||
emit(EventType.LOG, level="warn", code="judge_limit", limits=hit, unjudged=n,
|
||
message=f"结论句裁判已达上限（{labels}）：{n} 句未裁判，已裁判的结果保留")
|
||
if failed := [g for r, g in outcome["_gaps"] if r not in LIMITS]:
|
||
emit(EventType.LOG, level="warn", code="judge_failed", message="结论句裁判未完成：" + "；".join(failed))
|
||
outcome.pop("_gaps")
|
||
return outcome
|
||
|
||
|
||
#: 截理由时认的句子边界（全角、半角都认）
|
||
_BOUNDARY = "。；，;,！？!?"
|
||
|
||
|
||
def clip_text(text: str, limit: int = RATIONALE_MAX) -> str:
|
||
"""超过 limit 个字时截短：在句子边界（。；，）截断加「…」，总长不超过 limit。
|
||
|
||
边界离开头太近（不到三分之一）就不按它截——只剩几个字的理由没用；这时退到词的边界（不把一个英文词、
|
||
一个数截成两半）。没超长的原样返回。
|
||
"""
|
||
text = text.strip()
|
||
if len(text) <= limit:
|
||
return text
|
||
head = text[:limit - 1]
|
||
cut = max(head.rfind(ch) for ch in _BOUNDARY)
|
||
if cut < limit // 3:
|
||
cut = len(head)
|
||
while cut > 0 and head[cut - 1].isascii() and head[cut - 1].isalnum() and text[cut].isascii() \
|
||
and text[cut].isalnum():
|
||
cut -= 1
|
||
if cut < limit // 3:
|
||
cut = len(head)
|
||
return head[:cut].rstrip(" ，,；;、") + "…"
|
||
|
||
|
||
def _take(items: list[Any], batch: list[Candidate], request: JudgeRequest, verdicts: dict[str, dict[str, Any]], *,
|
||
judge: str | None, post_seal: bool) -> None:
|
||
"""模型给的判定逐条收下：不在这一批的编号不认，判定不认识的记格式不对，理由在句子边界截到 RATIONALE_MAX 字，
|
||
used 只留这一批摘录里真有的编号。判 insufficient 的另记 missing（缺的是什么，模型没写就不编）。
|
||
|
||
模型照旧写 unsupported 的按 contradicted 收下：新判的不再产出旧取值，而 contradicted 的处置和原来的
|
||
unsupported 一样（不能因为模型用了旧词就放过这一句）。"""
|
||
ids = {c.unit for c in batch}
|
||
aliases = {a for c in batch for a in c.aliases if a in request.excerpts}
|
||
for item in items:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
uid = str(item.get("unit") or "").strip().strip("[]")
|
||
if uid not in ids or uid in verdicts:
|
||
continue
|
||
status = str(item.get("verdict") or "").strip().lower()
|
||
if status == UNSUPPORTED:
|
||
status = "contradicted"
|
||
if status not in JUDGED:
|
||
verdicts[uid] = _unjudged("format", f"裁判返回的判定「{str(item.get('verdict'))[:20]}」无法识别，这句未裁判",
|
||
judge=judge, post_seal=post_seal)
|
||
continue
|
||
used = item.get("used") if isinstance(item.get("used"), list) else []
|
||
verdict: dict[str, Any] = {
|
||
"status": status, "rationale": clip_text(_clean(str(item.get("rationale") or ""))),
|
||
"judge": judge, "post_seal": post_seal,
|
||
"used": list(dict.fromkeys(u.strip() for u in used if isinstance(u, str) and u.strip() in aliases))}
|
||
missing = item.get("missing")
|
||
if status == "insufficient" and isinstance(missing, str) and (missing := clip_text(_clean(missing), MISSING_MAX)):
|
||
verdict["missing"] = missing
|
||
verdict["rule_version"] = RULES_VERSION
|
||
verdicts[uid] = verdict
|
||
|
||
|
||
def outdated(verdict: Any) -> bool:
|
||
"""这条判过的判定早于当前规则：没有规则版本（改造前判的）、版本更小，或者是旧取值 unsupported。
|
||
没判成的（unjudged）谈不上新旧，返回 False。"""
|
||
if not isinstance(verdict, dict) or verdict.get("status") not in SETTLED:
|
||
return False
|
||
version = verdict.get("rule_version")
|
||
return verdict.get("status") == UNSUPPORTED or isinstance(version, bool) or not isinstance(version, int) \
|
||
or version < RULES_VERSION
|
||
|
||
|
||
def _outcome(request: JudgeRequest, verdicts: dict[str, dict[str, Any]], budget: Budget, details: dict[str, str], *,
|
||
click: bool = False, shown: Budget | None = None) -> dict[str, Any]:
|
||
"""汇总判定。outcome.budget 是这一轮实际用的上限；缺口里写的上限按 shown（缺省同 budget）。"""
|
||
label = shown or budget
|
||
counts = {s: 0 for s in COUNT_KEYS}
|
||
why: dict[str, int] = {}
|
||
for c in request.cands:
|
||
v = verdicts[c.unit]
|
||
counts[v["status"]] = counts.get(v["status"], 0) + 1
|
||
if v["status"] == "unjudged":
|
||
why[v["reason"]] = why.get(v["reason"], 0) + 1
|
||
gaps: list[tuple[str, str]] = []
|
||
for reason in REASONS:
|
||
n = why.get(reason)
|
||
if not n or reason == "on_demand":
|
||
continue
|
||
if reason in LIMITS:
|
||
text = f"有 {n} 句结论未裁判：已达上限（{_limit_label(reason, label, click=click)}）"
|
||
elif reason == "error":
|
||
text = f"有 {n} 句结论未裁判：裁判调用失败（{details.get('error') or '原因不明'}）"
|
||
elif reason == "format":
|
||
text = f"有 {n} 句结论未裁判：裁判返回的结果格式有误"
|
||
else:
|
||
text = f"有 {n} 句结论未获得裁判结果"
|
||
gaps.append((reason, text))
|
||
return {
|
||
"verdicts": {c.unit: verdicts[c.unit] for c in sorted(request.cands, key=lambda c: c.order)},
|
||
"candidates": len(request.cands), "screened": list(request.screened), "skipped": dict(request.skipped),
|
||
"counts": counts, "unjudged": {r: why[r] for r in REASONS if r in why},
|
||
"limits_hit": [r for r in LIMITS if r in why], "budget": budget.as_dict(),
|
||
"gaps": [text for _, text in gaps], "_gaps": gaps,
|
||
}
|
||
|
||
|
||
async def judge_doc(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, spec: ModelSpec, budget: Budget,
|
||
units: Iterable[str] | None = None, loader: Callable[[str], Any] | None = None,
|
||
masked: dict[str, Any] | None = None, known: dict[str, Any] | None = None,
|
||
post_seal: bool = False, emit: Emit | None = None, click: bool = False) -> dict[str, Any]:
|
||
"""判一份报告文档（units 给了就只判这几句）。见 prepare 和 run_request。"""
|
||
request = prepare(doc, catalog, units=units, loader=loader, masked=masked)
|
||
return await run_request(request, spec=spec, budget=budget, known=known, post_seal=post_seal, emit=emit,
|
||
click=click)
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 写回文档
|
||
# --------------------------------------------------------------------------
|
||
|
||
#: 文档的 judge 摘要里、report.checked 和节点产出的 judge 里都有的键
|
||
SUMMARY_KEYS = ("mode", "model", "on_unsupported", "rewrite_once", "limits", "candidates", "screened", "counts",
|
||
"unjudged", "limits_hit", "complete", "priced", "cost_usd", "calls", "reused", "duration_ms",
|
||
"gaps", "notes", "same_model", "rewrite", "writer_model")
|
||
|
||
|
||
def summarize(outcome: dict[str, Any], *, mode: str, on_unsupported: str = "degrade", rewrite_once: bool = False,
|
||
same_model: bool = False, rewrite: dict[str, Any] | None = None,
|
||
writer_model: str | None = None) -> dict[str, Any]:
|
||
"""裁判摘要：写进文档的 judge，也原样放进 report.checked 和节点产出。
|
||
|
||
counts 补齐 COUNT_KEYS 的每个键（断点里缓存的旧判定只有旧的几个键）；writer_model 是这份报告的写作模型，
|
||
same_model 是裁判模型和它相同。"""
|
||
raw = outcome["counts"]
|
||
counts = {k: int(raw.get(k) or 0) for k in COUNT_KEYS}
|
||
return {
|
||
"mode": mode, "model": outcome.get("model"), "on_unsupported": on_unsupported, "rewrite_once": rewrite_once,
|
||
"limits": outcome.get("budget"), "candidates": outcome["candidates"], "screened": outcome["screened"],
|
||
"counts": counts, "unjudged": outcome["unjudged"], "limits_hit": outcome["limits_hit"],
|
||
"complete": mode == "inline" and counts["unjudged"] == 0,
|
||
"priced": outcome.get("priced"), "cost_usd": outcome.get("cost_usd", 0.0), "calls": outcome.get("calls", 0),
|
||
"reused": outcome.get("reused", 0), "duration_ms": outcome.get("duration_ms", 0),
|
||
"gaps": outcome["gaps"] if mode == "inline" else [], "notes": outcome.get("notes") or [],
|
||
"same_model": same_model, "rewrite": rewrite, "writer_model": writer_model,
|
||
}
|
||
|
||
|
||
def apply_judgement(doc: dict[str, Any], outcome: dict[str, Any], summary: dict[str, Any]) -> dict[str, Any]:
|
||
"""判定写进文档：每句的 unit.verdict、文档的 judge 摘要、stats 里各判定的句数。返回同一份文档。"""
|
||
units = {u.get("id"): u for _, u in iter_units(doc)}
|
||
for uid, verdict in outcome["verdicts"].items():
|
||
if uid in units:
|
||
units[uid]["verdict"] = verdict
|
||
doc["judge"] = summary
|
||
doc["stats"] = {**(doc.get("stats") or {}), **summary["counts"]}
|
||
return doc
|
||
|
||
|
||
def on_demand(doc: dict[str, Any], *, on_unsupported: str = "degrade", rewrite_once: bool = False,
|
||
writer_model: str | None = None) -> dict[str, Any]:
|
||
"""探索运行：节点里不花这笔钱。候选句标「未裁判 · 按需」，点开哪句再判哪句。返回摘要（已写进文档）。
|
||
|
||
writer_model 记进摘要：按需裁判时拿它判断裁判模型是不是和写作模型相同。"""
|
||
request = prepare(doc, {})
|
||
budget = Budget()
|
||
verdicts = {c.unit: _unjudged("on_demand", _why_unjudged("on_demand", budget), judge=None, post_seal=False)
|
||
for c in request.cands}
|
||
outcome = _outcome(request, verdicts, budget, {})
|
||
outcome.pop("_gaps")
|
||
outcome["budget"] = None
|
||
summary = summarize(outcome, mode="on_demand", on_unsupported=on_unsupported, rewrite_once=rewrite_once,
|
||
writer_model=writer_model)
|
||
apply_judgement(doc, outcome, summary)
|
||
return summary
|