mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-18 04:01:57 +00:00
Merge pull request #16 from ThearchyHelios/feat/judge-rework
结论句裁判改造:「不支持」拆成「证据相矛盾」和「证据不足」,补全交给裁判的证据,旧判定可按新规则重判
This commit is contained in:
@@ -209,11 +209,13 @@ NODE_REFERENCE = """\
|
||||
metrics_from 不写就取所有上游口径卡;on_violation 不写时探索运行是 flag(标出来、照常产出),正式运行是 fail;
|
||||
numbers 默认 strict(裸数字算违规)。claims 是结论句策略:off(默认,不管)/ require_citation(每句结论都要
|
||||
挂引用,没挂的计入缺口、出具降档)/ judge(在 require_citation 之外,再请另一个模型按每句挂的依据判断支不支持:
|
||||
正式运行在节点里判,证据不支持的按 judge.on_unsupported 处置——degrade 降档(默认)或 withhold 不予出具;
|
||||
正式运行在节点里判,证据相矛盾的按 judge.on_unsupported 处置——degrade 降档(默认)或 withhold 不予出具,
|
||||
证据不足的(摘录里没有判断所需的信息)只降档;
|
||||
探索运行不在节点里花钱,读的人点开哪句再按需判哪句,只标注)。judge 的子配置写在节点的 judge 里:judge.model、
|
||||
judge.provider 指定裁判模型(建议和写作模型不同,同一个模型等于自己审自己;不写就用设置里的「证据裁判模型」);
|
||||
judge.max_cost_usd、judge.max_claims、judge.timeout_s 是每份报告的裁判金额、句数、时长上限,写 null 表示不限,
|
||||
不写取设置里的默认;judge.rewrite_once 默认 false,打开后把证据不支持的句子交回写作者只改一次。entities 默认 link
|
||||
不写取设置里的默认;judge.rewrite_once 默认 false,打开后把证据相矛盾、证据不足的句子交回写作者只改一次。
|
||||
entities 默认 link
|
||||
(核对表名、字段名),报告里根本不提表名时才写 off
|
||||
- output 的出具契约:config.contract = {report_from: 报告节点id, metrics_from:[口径卡id],
|
||||
required:[必需指标id], expected:[期望指标id], strict: true}。成果字段写 {{ nodes.报告节点id.text }},
|
||||
|
||||
+142
-31
@@ -66,10 +66,13 @@ from app.engine.evidence import (
|
||||
normalize_quote,
|
||||
query_entry_fields,
|
||||
render_metric,
|
||||
schema_column,
|
||||
schema_table,
|
||||
sql_tables,
|
||||
table_fields,
|
||||
uncited_claims,
|
||||
)
|
||||
from app.engine.judge import JUDGED, VERDICTS, candidates
|
||||
from app.engine.judge import FIELDS_MAX, SETTLED, UNSUPPORTED, VERDICTS, candidates, outdated
|
||||
from app.engine.toolcalls import QUERY_PREFIX
|
||||
|
||||
router = APIRouter(prefix="/api/runs", tags=["evidence"])
|
||||
@@ -727,7 +730,8 @@ async def _chain(seg: dict[str, Any], doc: dict[str, Any], sealed: _Sealed,
|
||||
if seg.get("kind") == "entity" and seg.get("issue") in SUSPICIOUS:
|
||||
return [_suspicious_step(seg, cite, sealed, report_node)]
|
||||
if seg.get("kind") == "entity" and seg.get("state") == "deterministic" and cite.get("status") == "resolved":
|
||||
return [_entity_step(seg, cite, entry, sealed)]
|
||||
hidden = await _entity_hidden(entry, sealed, masks) if entry.get("kind") == "table" else set()
|
||||
return [_entity_step(seg, cite, entry, sealed, hidden=hidden)]
|
||||
if seg.get("kind") == "quote" and seg.get("state") == "deterministic" and cite.get("status") == "resolved":
|
||||
return [_quote_step(seg, cite, entry, sealed)]
|
||||
if _missing_input(cite, entry):
|
||||
@@ -928,22 +932,10 @@ def _fetch(artifact: str) -> tuple[Any, bool | None]:
|
||||
return content, (True if content is not None else None)
|
||||
|
||||
|
||||
def _schema_table(content: Any, name: Any) -> dict[str, Any] | None:
|
||||
"""表结构快照里的一张表:表名一字不差的优先,其次不分大小写、全名(schema.表)也认。"""
|
||||
tables = content.get("tables") if isinstance(content, dict) else None
|
||||
if not isinstance(tables, dict) or not name:
|
||||
return None
|
||||
if isinstance(tables.get(name), dict):
|
||||
return tables[name]
|
||||
lowered = str(name).lower()
|
||||
return next((t for key, t in tables.items() if isinstance(t, dict)
|
||||
and lowered in (str(key).lower(), str(t.get("qualified") or "").lower())), None)
|
||||
|
||||
|
||||
def _schema_column(table: dict[str, Any] | None, name: Any) -> dict[str, Any] | None:
|
||||
columns = [c for c in (table or {}).get("columns") or [] if isinstance(c, dict)]
|
||||
return next((c for c in columns if c.get("name") == name), None) \
|
||||
or next((c for c in columns if str(c.get("name") or "").lower() == str(name or "").lower()), None)
|
||||
#: 表结构快照里找表、找字段:和裁判的摘录同一套(engine/evidence.py)。表名实体步骤的字段清单也和裁判的
|
||||
#: 摘录同一个上限(judge.FIELDS_MAX)
|
||||
_schema_table = schema_table
|
||||
_schema_column = schema_column
|
||||
|
||||
|
||||
def _entity_present(content: Any, kind: str, name: str, table: Any, column: Any, tables: list[Any]) -> bool:
|
||||
@@ -963,12 +955,34 @@ def _query_present(content: Any, kind: str, name: str, column: Any) -> bool:
|
||||
return str(column) in [str(c) for c in content.get("columns") or []]
|
||||
|
||||
|
||||
def _entity_step(seg: dict[str, Any], cite: dict[str, Any], entry: dict[str, Any], sealed: _Sealed) -> dict[str, Any]:
|
||||
async def _entity_hidden(entry: dict[str, Any], sealed: _Sealed, masks: _Masks) -> set[str]:
|
||||
"""表的字段清单要遮的列(小写):数据源现在设的,加上这次运行封存的查询快照里、同一个数据源当时记下的
|
||||
(数据源改名、删掉了也照遮,和查询步骤同一个口径)。"""
|
||||
source = str(entry.get("source") or "")
|
||||
hidden, _ = await masks.of(source) if source else (set(), True)
|
||||
hidden = set(hidden)
|
||||
for artifact, info in sealed.queries.items():
|
||||
if not sealed.trusted:
|
||||
break
|
||||
snap, ok = _fetch(artifact)
|
||||
if ok is not True or not isinstance(snap, dict):
|
||||
continue
|
||||
if not source or str(info.get("source") or snap.get("source") or "") == source:
|
||||
recorded = snap.get("mask_columns")
|
||||
hidden |= {str(c).lower() for c in recorded if isinstance(c, str)} if isinstance(recorded, list) else set()
|
||||
return hidden
|
||||
|
||||
|
||||
def _entity_step(seg: dict[str, Any], cite: dict[str, Any], entry: dict[str, Any], sealed: _Sealed, *,
|
||||
hidden: set[str] | frozenset[str] = frozenset()) -> dict[str, Any]:
|
||||
"""实体步骤:这个名字出现在哪几次查询里(SQL 用到的表、结果的列)、表结构快照什么时候同步的、字段类型。
|
||||
|
||||
来源(entry.sources)逐个核对:表结构快照要出现在封存范围内的 tool.end.schema_artifact 或 schema 台账
|
||||
条目里,查询快照要是封存范围内交回过的那几件。类型、同步时间只从核对过、复验过哈希的表结构快照取,
|
||||
取不到就不给。eid 背后的那件工件核对通过、名字确实在里面,这一步才算已封存。
|
||||
|
||||
表另带字段清单 fields: [{name, type}](同一份核对过的快照,最多 FIELDS_MAX 个,多出来的记 fields_more):
|
||||
讲表结构的句子,读的人在「挂的依据」下直接看得到有哪些字段。hidden 里的列(遮罩)不列。
|
||||
"""
|
||||
kind = str(cite.get("kind") or entry.get("kind") or "")
|
||||
locator = dict(entry.get("locator") or cite.get("locator") or {})
|
||||
@@ -1022,7 +1036,8 @@ def _entity_step(seg: dict[str, Any], cite: dict[str, Any], entry: dict[str, Any
|
||||
step["sources"].append(item)
|
||||
step["sealed"] = sealed.trusted and backed and step["eid_ok"]
|
||||
if snapshots:
|
||||
_describe_entity(step, snapshots[0], kind, name, table, column, list(entry.get("tables") or []))
|
||||
_describe_entity(step, snapshots[0], kind, name, table, column, list(entry.get("tables") or []),
|
||||
hidden=hidden)
|
||||
if kind == "column" and not any(o.get("kind") == "schema" for o in step["sources"]):
|
||||
# 聚合的别名这类只在结果里出现的列:类型取查询时从驱动的原始值记下的列类型。表结构里有的列
|
||||
# 只认表结构快照的类型,快照核对不过就不给——不拿结果列的类型顶上
|
||||
@@ -1033,8 +1048,8 @@ def _entity_step(seg: dict[str, Any], cite: dict[str, Any], entry: dict[str, Any
|
||||
|
||||
|
||||
def _describe_entity(step: dict[str, Any], snap: dict[str, Any], kind: str, name: str, table: Any, column: Any,
|
||||
tables: list[Any]) -> None:
|
||||
"""从表结构快照里补上同步时间、表的概况、字段的类型。"""
|
||||
tables: list[Any], *, hidden: set[str] | frozenset[str] = frozenset()) -> None:
|
||||
"""从表结构快照里补上同步时间、表的概况(含字段清单)、字段的类型。"""
|
||||
if snap.get("synced_at"):
|
||||
step["synced_at"] = snap["synced_at"]
|
||||
if snap.get("truncated"):
|
||||
@@ -1045,6 +1060,12 @@ def _describe_entity(step: dict[str, Any], snap: dict[str, Any], kind: str, name
|
||||
step["is_view"] = bool(info.get("is_view"))
|
||||
if info.get("comment"):
|
||||
step["comment"] = info["comment"]
|
||||
fields, more, masked = table_fields(info, hidden, limit=FIELDS_MAX)
|
||||
step["fields"] = fields
|
||||
if more:
|
||||
step["fields_more"] = more
|
||||
if masked:
|
||||
step["fields_masked"] = True
|
||||
return
|
||||
owners = [table] if table else tables
|
||||
found = {str(t): (tinfo, _schema_column(tinfo, column)) for t in owners
|
||||
@@ -1200,6 +1221,8 @@ UNSEALED_JUDGE = "运行尚未结束并封存(仍在运行、停在人工审
|
||||
LEGACY_JUDGE = "这次运行发起时系统尚未记录证据,不支持按需裁判"
|
||||
BROKEN_JUDGE = "封存核对未通过:封存后有事件被修改、删除或插入,报告和证据已不可信,无法再请模型裁判"
|
||||
JUDGED_ALREADY = "这句已裁判"
|
||||
#: 这句封存后按需判过,但那次早于当前的裁判规则(没有规则版本,或者是旧取值「证据不支持」):可以按当前规则重判
|
||||
OUTDATED_JUDGE = "这句的判定早于当前的裁判规则,可以请模型按当前规则重新判断"
|
||||
#: 裁判期间运行被接着跑了(失败的运行可以继续),或者接着跑完重新封存了:判定照样交回,不记进运行记录
|
||||
NOT_RECORDED = "运行已继续执行,本次裁判结果未写入运行记录,刷新后将不再显示。请在运行结束并封存后重新裁判"
|
||||
NOT_A_CLAIM = "不是结论句(标题、连接性文字、表格单元格或代码),无需模型裁判"
|
||||
@@ -1232,7 +1255,8 @@ def _effective_verdicts(report: _Report, sealed: _Sealed
|
||||
doc = report.doc or {}
|
||||
ids = {str(u.get("id")) for _, u in iter_units(doc)}
|
||||
out = {str(u.get("id")): u["verdict"] for _, u in iter_units(doc) if _verdict_ok(u.get("verdict"))}
|
||||
sealed_judged = {uid for uid, v in out.items() if v.get("status") in JUDGED}
|
||||
# 旧数据里的 unsupported 同样算判过(SETTLED):封存的判定用什么取值,都不能被追加的判定盖掉
|
||||
sealed_judged = {uid for uid, v in out.items() if v.get("status") in SETTLED}
|
||||
later: dict[str, dict[str, Any]] = {}
|
||||
for _, _, data in sealed.judged:
|
||||
if data.get("report") != report.node_id or data.get("doc_artifact") != report.doc_artifact:
|
||||
@@ -1242,17 +1266,48 @@ def _effective_verdicts(report: _Report, sealed: _Sealed
|
||||
uid = str(uid)
|
||||
if uid not in ids or uid in sealed_judged or not _verdict_ok(verdict):
|
||||
continue
|
||||
if later.get(uid, {}).get("status") in JUDGED and verdict.get("status") not in JUDGED:
|
||||
if later.get(uid, {}).get("status") in SETTLED and verdict.get("status") not in SETTLED:
|
||||
continue
|
||||
later[uid] = out[uid] = {**verdict, "post_seal": True}
|
||||
return out, later
|
||||
|
||||
|
||||
def _judgement_of(report: _Report, sealed: _Sealed) -> dict[str, Any]:
|
||||
"""证据图、审计表里每份报告带的结论句情况:写作时的策略、封存的裁判摘要、封存之后按需追加的判定。"""
|
||||
"""证据图、审计表里每份报告带的结论句情况:写作时的策略、封存的裁判摘要、封存之后按需追加的判定,
|
||||
和最近一次按需裁判用的模型(judge_meta,没按需裁判过是 None)。"""
|
||||
doc = report.doc or {}
|
||||
return {"claims": report.claims, "judge": doc.get("judge") if isinstance(doc.get("judge"), dict) else None,
|
||||
"post_seal_verdicts": _effective_verdicts(report, sealed)[1] if report.doc is not None else {}}
|
||||
"post_seal_verdicts": _effective_verdicts(report, sealed)[1] if report.doc is not None else {},
|
||||
"judge_meta": _judge_meta(report, sealed)}
|
||||
|
||||
|
||||
def _writer_model(sealed: _Sealed, report: _Report) -> str | None:
|
||||
"""这份报告的写作模型:文档的裁判摘要里记的(写作时当场记下),没有就取报告撰写节点封存的产出里的 model
|
||||
(实际用的模型 id,老文档也有)。都取不到是 None。"""
|
||||
judge = (report.doc or {}).get("judge")
|
||||
if isinstance(judge, dict) and isinstance(judge.get("writer_model"), str) and judge["writer_model"]:
|
||||
return judge["writer_model"]
|
||||
payload = sealed.payload_of([report.node_id]) or {}
|
||||
model = payload.get("model")
|
||||
return model if isinstance(model, str) and model else None
|
||||
|
||||
|
||||
def _same_model(model: Any, writer: str | None) -> bool:
|
||||
return bool(model) and bool(writer) and model == writer
|
||||
|
||||
|
||||
def _judge_meta(report: _Report, sealed: _Sealed) -> dict[str, Any] | None:
|
||||
"""最近一次按需裁判(evidence.judged,指向这份文档的)用的模型:{model, same_model, priced, writer_model}。
|
||||
|
||||
事件里没记 same_model、writer_model 的(改造前追加的),按这份报告的写作模型补上。"""
|
||||
data = next((d for _, _, d in reversed(sealed.judged)
|
||||
if d.get("report") == report.node_id and d.get("doc_artifact") == report.doc_artifact), None)
|
||||
if data is None:
|
||||
return None
|
||||
writer = data.get("writer_model") if "writer_model" in data else _writer_model(sealed, report)
|
||||
same = data.get("same_model") if isinstance(data.get("same_model"), bool) else _same_model(data.get("model"), writer)
|
||||
return {"model": data.get("model"), "same_model": same,
|
||||
"priced": data.get("priced") if isinstance(data.get("priced"), bool) else None, "writer_model": writer}
|
||||
|
||||
|
||||
def _on_demand(sealed: _Sealed, doc: dict[str, Any], uid: str, verdict: dict[str, Any] | None) -> dict[str, Any]:
|
||||
@@ -1266,15 +1321,25 @@ def _on_demand(sealed: _Sealed, doc: dict[str, Any], uid: str, verdict: dict[str
|
||||
return no("legacy", LEGACY_JUDGE)
|
||||
if not candidates(doc, [uid]):
|
||||
return no("not_a_claim", f"这句{NOT_A_CLAIM}")
|
||||
if isinstance(verdict, dict) and verdict.get("status") in JUDGED:
|
||||
stale = _stale(verdict)
|
||||
if isinstance(verdict, dict) and verdict.get("status") in SETTLED and not stale:
|
||||
return no("judged", JUDGED_ALREADY)
|
||||
if not sealed.sealed or sealed.status not in _TERMINAL:
|
||||
return no("unsealed", UNSEALED_JUDGE)
|
||||
if not sealed.trusted:
|
||||
return no("seal_broken", BROKEN_JUDGE)
|
||||
if stale:
|
||||
return {"available": True, "reason": "outdated", "message": OUTDATED_JUDGE}
|
||||
return {"available": True, "reason": None, "message": None}
|
||||
|
||||
|
||||
def _stale(verdict: Any) -> bool:
|
||||
"""封存之后按需追加的判定早于当前规则:按需裁判当它还没按当前规则判过,重判一次(不按摘录内容判断)。
|
||||
|
||||
封存的判定(文档里的,post_seal 为 False)不在此列:追加的判定盖不掉它,版本再旧也不重开。"""
|
||||
return isinstance(verdict, dict) and bool(verdict.get("post_seal")) and outdated(verdict)
|
||||
|
||||
|
||||
def _sealed_loader(sealed: _Sealed) -> Callable[[str], Any]:
|
||||
"""给裁判取证据的 loader:只认封存范围内的事件交回过的工件(查询、表结构、检索快照和台账里记过的),
|
||||
取回时照旧复验哈希。别的一律当取不到——人为插进工件表的行、封存之后才追加的事件引用的快照,
|
||||
@@ -1371,13 +1436,15 @@ async def _judge_now(sealed: _Sealed, report: _Report, units: list[str]) -> dict
|
||||
doc = report.doc or {}
|
||||
catalog = doc.get("catalog") or {}
|
||||
known = _effective_verdicts(report, sealed)[0]
|
||||
reused = [u for u in units if known.get(u, {}).get("status") in JUDGED]
|
||||
# 当前规则下判过的直接给;早于当前规则的(封存之后追加的旧判定)重判一次,和片段接口的 outdated 同一条件
|
||||
reused = [u for u in units if known.get(u, {}).get("status") in SETTLED and not _stale(known.get(u))]
|
||||
pending = [u for u in units if u not in reused]
|
||||
verdicts = {u: known[u] for u in reused}
|
||||
node_judge = _node_judge(sealed, report.node_id)
|
||||
async with SessionLocal() as session:
|
||||
settings = await judging.judge_settings(session)
|
||||
budget = judging.click_budget(settings, node_judge)
|
||||
writer = _writer_model(sealed, report)
|
||||
outcome: dict[str, Any] | None = None
|
||||
skipped: dict[str, str] = {}
|
||||
event = None
|
||||
@@ -1391,7 +1458,13 @@ async def _judge_now(sealed: _Sealed, report: _Report, units: list[str]) -> dict
|
||||
async with SessionLocal() as session:
|
||||
spec = await judging.judge_model_spec(session, node_judge, settings=settings)
|
||||
outcome = await judging.run_request(request, spec=spec, budget=budget, post_seal=True, click=True)
|
||||
verdicts.update(outcome["verdicts"])
|
||||
for uid, verdict in outcome["verdicts"].items():
|
||||
# 重判没判成(触顶、模型不可用):旧判定照旧生效,和证据图、片段接口说的一样
|
||||
older = known.get(uid)
|
||||
keep = verdict.get("status") not in SETTLED and isinstance(older, dict) and older.get("status") in SETTLED
|
||||
verdicts[uid] = older if keep else verdict
|
||||
outcome["writer_model"] = writer
|
||||
outcome["same_model"] = _same_model(outcome.get("model"), writer)
|
||||
if outcome["calls"] or outcome["cost_usd"]:
|
||||
# 没问模型(触顶、模型没配好)就没有判定可记:说给点的人听,不往运行记录里追加
|
||||
event = await _append_judged(sealed, report, pending, outcome)
|
||||
@@ -1403,12 +1476,14 @@ async def _judge_now(sealed: _Sealed, report: _Report, units: list[str]) -> dict
|
||||
return {
|
||||
"run_id": sealed.run_id, "report": {"node_id": report.node_id, "doc_artifact": report.doc_artifact},
|
||||
"units": units, "verdicts": {u: verdicts[u] for u in units if u in verdicts}, "reused": reused,
|
||||
"judged": [u for u in pending if (outcome or {}).get("verdicts", {}).get(u, {}).get("status") in JUDGED],
|
||||
"judged": [u for u in pending if (outcome or {}).get("verdicts", {}).get(u, {}).get("status") in SETTLED],
|
||||
"skipped": skipped, "unjudged": dict(outcome["unjudged"]) if outcome else {},
|
||||
"limits_hit": limits, "limited": bool(limits), "message": message,
|
||||
"adjust": [JUDGE_ADJUST[r] for r in limits if r in JUDGE_ADJUST],
|
||||
"gaps": list(outcome["gaps"]) if outcome else [], "notes": list(outcome["notes"]) if outcome else [],
|
||||
"model": (outcome or {}).get("model"), "priced": (outcome or {}).get("priced"),
|
||||
# 裁判模型和这份报告的写作模型相同:判定照常出,界面上标出来(审查缺乏独立性)
|
||||
"same_model": bool((outcome or {}).get("same_model")), "writer_model": writer,
|
||||
"cost_usd": (outcome or {}).get("cost_usd", 0.0), "calls": (outcome or {}).get("calls", 0),
|
||||
"duration_ms": (outcome or {}).get("duration_ms", 0),
|
||||
"budget": budget.as_dict() if outcome else None, "event": event, "post_seal": True,
|
||||
@@ -1461,7 +1536,8 @@ async def _append_judged(sealed: _Sealed, report: _Report, units: list[str],
|
||||
"report": report.node_id, "doc_artifact": report.doc_artifact, "units": units,
|
||||
"verdicts": outcome["verdicts"],
|
||||
"keys": {u: outcome["keys"][u] for u in outcome["verdicts"] if u in outcome["keys"]},
|
||||
"model": outcome["model"], "priced": outcome["priced"], "cost_usd": outcome["cost_usd"],
|
||||
"model": outcome["model"], "priced": outcome["priced"], "same_model": outcome["same_model"],
|
||||
"writer_model": outcome["writer_model"], "cost_usd": outcome["cost_usd"],
|
||||
"calls": outcome["calls"], "duration_ms": outcome["duration_ms"], "budget": outcome["budget"],
|
||||
"limits_hit": outcome["limits_hit"], "unjudged": outcome["unjudged"], "gaps": outcome["gaps"],
|
||||
"notes": outcome["notes"], "skipped": outcome["skipped"], "post_seal": True})
|
||||
@@ -1501,6 +1577,13 @@ UNCITED_CLAIM_JUDGED = "这句结论没有附依据。本报告的结论句由
|
||||
UNCITED_CLAIM_JUDGE_REQUIRED = ("这句结论没有附依据。本报告的结论句由模型裁判,同样要求附依据,但目前没有出具契约"
|
||||
"据此判档(未配置契约,或尚未到出具步骤)")
|
||||
LEGACY_UNMATCHED = "旧版出具中这个数字与任何指标都不相符"
|
||||
#: 模型判定有问题的结论句各一行(生效的判定:封存的,没有就是封存后按需追加的):判定 → 审计行的 issue。
|
||||
#: 旧数据里的 unsupported(当时没分矛盾和不足)记 unsupported_claim,说法照旧是「证据不支持」
|
||||
CLAIM_ISSUE = {"contradicted": "contradicted_claim", "insufficient": "insufficient_claim",
|
||||
UNSUPPORTED: "unsupported_claim"}
|
||||
CLAIM_VERDICT_NOTE = {"contradicted": "模型判断:证据与这句结论相矛盾", "insufficient": "模型判断:证据不足,无法判断这句结论",
|
||||
UNSUPPORTED: "模型判断:证据不支持这句结论"}
|
||||
CLAIM_POST_SEAL = "该判定为封存后按需追加,不在封存范围内"
|
||||
|
||||
#: CSV 的列:(表头, 行里的键)
|
||||
CSV_COLUMNS = (
|
||||
@@ -1521,6 +1604,7 @@ _CSV_LABELS: dict[str, dict[Any, str]] = {
|
||||
"state": {"deterministic": "有出处", "none": "无证据", "candidate": "候选"},
|
||||
"issue": {"uncited_number": "没有出处的数字", "unresolved_ref": "引用无法解析", "unknown_entity": "疑似不存在的名称",
|
||||
"unverified_entity": "无法核实", "uncited_claim": "结论句未附依据", "no_candidate": "未找到候选",
|
||||
"contradicted_claim": "证据相矛盾", "insufficient_claim": "证据不足", "unsupported_claim": "证据不支持",
|
||||
**{code: "文档不一致" for code in _INTEGRITY}},
|
||||
"evidence_kind": {"metric": "口径卡指标", "cell": "查询单元格", "input": "运行输入", "table": "表", "column": "字段",
|
||||
"quote": "知识库原话", "query": "查询结果"},
|
||||
@@ -1670,6 +1754,7 @@ def _audit_doc(report: _Report, sealed: _Sealed, inputs: dict[str, Any],
|
||||
"issue": "uncited_claim", "ref": None, "alias": None, "evidence_kind": None, "evidence": None,
|
||||
"eid": None, "artifact": None, "node_id": None, "sealed": None, "span": claim["span"],
|
||||
"sentence": claim["text"], "note": claim_note})
|
||||
rows.extend(_verdict_rows(report, sealed, field_name, markdown))
|
||||
for v in doc.get("violations") or []:
|
||||
if not isinstance(v, dict) or (str(v.get("segment")) in listed and v.get("code") not in _INTEGRITY):
|
||||
continue
|
||||
@@ -1683,6 +1768,32 @@ def _audit_doc(report: _Report, sealed: _Sealed, inputs: dict[str, Any],
|
||||
return rows
|
||||
|
||||
|
||||
def _verdict_rows(report: _Report, sealed: _Sealed, field_name: str | None, markdown: str) -> list[dict[str, Any]]:
|
||||
"""模型判定证据相矛盾、证据不足(旧数据里的证据不支持)的结论句,各一行,进「无证据」一组(问题)。
|
||||
|
||||
判定是模型的判断,不是证据:sealed 为 None,说明里写明是模型判断;封存后按需追加的另外注明。"""
|
||||
effective, _ = _effective_verdicts(report, sealed)
|
||||
units = {str(u.get("id")): u for _, u in iter_units(report.doc or {})}
|
||||
rows = []
|
||||
for uid, verdict in effective.items():
|
||||
status = verdict.get("status")
|
||||
if status not in CLAIM_ISSUE or uid not in units:
|
||||
continue
|
||||
span = units[uid].get("span") or [0, 0]
|
||||
text = markdown[span[0]:span[1]]
|
||||
missing = verdict.get("missing") if status == "insufficient" and isinstance(verdict.get("missing"), str) else ""
|
||||
rationale = str(verdict.get("rationale") or "").rstrip("。")
|
||||
note = CLAIM_VERDICT_NOTE[status] + (f"(缺少:{missing})" if missing else "") \
|
||||
+ (f"。理由:{rationale}" if rationale else "") + (f"。{CLAIM_POST_SEAL}" if verdict.get("post_seal") else "")
|
||||
rows.append({"group": "none", "report": report.node_id, "field": field_name, "segment": None, "unit": uid,
|
||||
"kind": "claim", "text": text, "state": "none", "issue": CLAIM_ISSUE[status], "ref": None,
|
||||
"alias": None, "evidence_kind": None, "evidence": None, "eid": None, "artifact": None,
|
||||
"node_id": None, "sealed": None, "span": span, "sentence": text, "note": note,
|
||||
"verdict": {k: verdict[k] for k in ("status", "rationale", "missing", "judge", "post_seal")
|
||||
if k in verdict}})
|
||||
return rows
|
||||
|
||||
|
||||
def _segment_row(group: str, report: _Report, field_name: str | None, seg: dict[str, Any], unit: dict[str, Any],
|
||||
sentence: str, catalog: dict[str, Any], sealed: _Sealed, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
cite = seg.get("cite") or {}
|
||||
|
||||
@@ -413,6 +413,37 @@ def schema_entry_fields(content: Any) -> dict[str, Any] | None:
|
||||
return fields
|
||||
|
||||
|
||||
def schema_table(content: Any, name: Any) -> dict[str, Any] | None:
|
||||
"""表结构快照里的一张表:表名一字不差的优先,其次不分大小写、全名(schema.表)也认。"""
|
||||
tables = content.get("tables") if isinstance(content, dict) else None
|
||||
if not isinstance(tables, dict) or not name:
|
||||
return None
|
||||
if isinstance(tables.get(name), dict):
|
||||
return tables[name]
|
||||
lowered = str(name).lower()
|
||||
return next((t for key, t in tables.items() if isinstance(t, dict)
|
||||
and lowered in (str(key).lower(), str(t.get("qualified") or "").lower())), None)
|
||||
|
||||
|
||||
def schema_column(table: dict[str, Any] | None, name: Any) -> dict[str, Any] | None:
|
||||
"""表结构快照里一张表的一个字段:名字一字不差的优先,其次不分大小写。"""
|
||||
columns = [c for c in (table or {}).get("columns") or [] if isinstance(c, dict)]
|
||||
return next((c for c in columns if c.get("name") == name), None) \
|
||||
or next((c for c in columns if str(c.get("name") or "").lower() == str(name or "").lower()), None)
|
||||
|
||||
|
||||
def table_fields(table: dict[str, Any] | None, hidden: set[str] | frozenset[str] = frozenset(), *,
|
||||
limit: int = 60) -> tuple[list[dict[str, str]], int, bool]:
|
||||
"""一张表的字段清单:([{name, type}] 最多 limit 个, 另有几个没列, 有没有字段按遮罩没列)。
|
||||
|
||||
hidden 是要遮的列(小写):遮罩的字段不列,也不计入「另有几个」。证据面板和裁判的摘录同一个口径。
|
||||
"""
|
||||
columns = [c for c in (table or {}).get("columns") or [] if isinstance(c, dict) and c.get("name")]
|
||||
shown = [c for c in columns if str(c["name"]).lower() not in hidden]
|
||||
fields = [{"name": str(c["name"]), "type": str(c.get("type") or "")} for c in shown[:limit]]
|
||||
return fields, max(0, len(shown) - limit), len(shown) < len(columns)
|
||||
|
||||
|
||||
def schema_ledger_entry(artifact: str, *, node_id: str, exec_no: int,
|
||||
loader: Callable[[str], Any] | None = None) -> dict[str, Any] | None:
|
||||
"""一件表结构快照 → 台账的 schema 条目。快照取不回来(或者哈希对不上)返回 None。
|
||||
|
||||
+314
-54
@@ -5,9 +5,11 @@
|
||||
|
||||
1. 预筛(claim_priority):只把结论句送去判。挂了依据的排第一,有方向词、因果词的排第二,其余有数字、
|
||||
实体这类实质内容的排第三;标题、表格单元格、代码、连接性的话、只有过渡词的、短句不送
|
||||
2. 证据摘录(prepare):句子挂的每件证据摘一段。指标写「名称 = 值」加原式;查询给 SQL 前 300 字和被
|
||||
引用的行(最多 20 行,数据源遮罩的列不给;宽表先给被引用的列,再补满 12 列);知识库片段最多 600 字。
|
||||
规则里写明数字已经由系统核对过
|
||||
2. 证据摘录(prepare):句子挂的每件证据摘一段。指标写「名称 = 值」加原式,再附上输入取自哪次查询、
|
||||
哪一格和那次查询的 SQL;查询给 SQL 前 300 字和被引用的行(最多 20 行,数据源遮罩的列不给;宽表先给
|
||||
被引用的列,再补满 12 列);表附上字段清单和类型(最多 60 个,遮罩的列不列,快照不完整要注明),
|
||||
字段附上类型和所属的表,两样都附上出现过的查询 SQL;知识库片段最多 600 字。快照一律经 loader 取
|
||||
(按需裁判传只认封存工件的那个)。规则里写明数字已经由系统核对过
|
||||
3. 调用(judge_doc):一份报告合并成一次调用,超过每批的句数就分批;结构化输出失败退回 JSON 解析,
|
||||
再失败就记未裁判。thinking 关掉,max_tokens 4096
|
||||
4. 预算:每份报告的句数、金额、时长和全局每日金额,每一项都能是 None(不限)。调用前按目录价估算,
|
||||
@@ -15,6 +17,10 @@
|
||||
5. 每日计数落在设置表(SPEND_KEY),按本地日期滚动。目录里没有价格的模型估不出金额,金额上限对它
|
||||
不起作用,照实说「按令牌估不出金额」
|
||||
|
||||
判定五选一:supported、partial、contradicted(证据和句子冲突)、insufficient(证据里没有判断所需的
|
||||
信息,missing 写缺什么)、not_a_claim。旧数据里的 unsupported(当时没分矛盾和不足)原样保留,判档当
|
||||
contradicted;新判的不再产出它。
|
||||
|
||||
判定只标注、不改写正文。报告节点在正式运行里调它(结果写进文档、随文档封存);探索运行按需裁判的
|
||||
接口也调它(post_seal=True,结果落在封存之后)。所有模型调用都经 get_chat_model,测试换成剧本。
|
||||
"""
|
||||
@@ -41,6 +47,7 @@ from app.db.base import SessionLocal
|
||||
from app.db.models import Setting
|
||||
from app.engine.evidence import (
|
||||
_DIRECTIONAL,
|
||||
ENTITY_KINDS,
|
||||
MISSING,
|
||||
RenderError,
|
||||
_auto_entity,
|
||||
@@ -48,6 +55,9 @@ from app.engine.evidence import (
|
||||
_mask_names,
|
||||
iter_units,
|
||||
render_cell,
|
||||
schema_column,
|
||||
schema_table,
|
||||
table_fields,
|
||||
)
|
||||
from app.engine.expressions import CellError, column_kind, locate_cell
|
||||
from app.engine.nodes.llm import _split_structured, _usage_of
|
||||
@@ -59,11 +69,27 @@ from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_mod
|
||||
# 常量
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: 模型给的四种判定;unjudged 是系统记的(没判成),不是模型说的
|
||||
JUDGED = ("supported", "partial", "unsupported", "not_a_claim")
|
||||
VERDICTS = (*JUDGED, "unjudged")
|
||||
#: 模型的理由最多几个字
|
||||
RATIONALE_MAX = 60
|
||||
#: 模型给的五种判定;unjudged 是系统记的(没判成),不是模型说的
|
||||
JUDGED = ("supported", "partial", "contradicted", "insufficient", "not_a_claim")
|
||||
#: 旧的判定「证据不支持」:当时没分「证据相矛盾」和「证据不足」。已封存的文档、已有的事件里原样保留,
|
||||
#: 判档照旧当 contradicted;新判的一律用新取值(模型照旧写了的,按 contradicted 收下)
|
||||
UNSUPPORTED = "unsupported"
|
||||
#: 读已有的判定时算「判过」的:新的五种加旧的 unsupported
|
||||
SETTLED = (*JUDGED, UNSUPPORTED)
|
||||
#: 判档按「证据相矛盾」处置的(on_unsupported:降档或不予出具)
|
||||
CONTRADICTED = ("contradicted", UNSUPPORTED)
|
||||
#: 判定对象里 status 的全部取值
|
||||
VERDICTS = (*SETTLED, "unjudged")
|
||||
#: 各判定的句数(stats、counts、出具横幅、审计导出都用这组键)。unsupported 保留给旧数据,新判的恒为 0
|
||||
COUNT_KEYS = ("supported", "partial", "contradicted", "insufficient", "not_a_claim", "unjudged", UNSUPPORTED)
|
||||
#: 模型的理由最多几个字(超长在句子边界截断加「…」);裁判规则里要求的字数
|
||||
RATIONALE_MAX = 120
|
||||
RATIONALE_ASK = 80
|
||||
#: insufficient 时「缺的是什么」最多几个字
|
||||
MISSING_MAX = 30
|
||||
#: 判定的规则版本:新判的判定带着它(rule_version),也进 request_key。报告节点的断点里换了规则的旧判定不复用;
|
||||
#: 按需裁判遇到早于当前规则的判定(没有版本、版本更小、旧取值 unsupported)重判一次
|
||||
RULES_VERSION = 2
|
||||
#: 一次调用最多判几句:再多,4096 个输出令牌装不下每句一条判定加理由
|
||||
BATCH_SIZE = 40
|
||||
MAX_TOKENS = 4096
|
||||
@@ -73,13 +99,17 @@ SQL_CHARS = 300
|
||||
EXCERPT_ROWS = 20
|
||||
EXCERPT_COLS = 12
|
||||
CHUNK_CHARS = 600
|
||||
#: 表名、字段名的摘录里附上它出现过的查询的 SQL,最多几次(「按会员号去重计数」这种句子,证据在 SQL 里)
|
||||
#: 表名、字段名的摘录里附上它出现过的查询的 SQL,最多几次(「按会员号去重计数」这种句子,证据在 SQL 里);
|
||||
#: 口径卡指标的输入取自的查询也按这个数给 SQL
|
||||
ENTITY_QUERIES = 3
|
||||
EXCERPT_HITS = 3
|
||||
#: 表的摘录里最多列几个字段(多出来的注明另有几个);指标的摘录里最多列几个输入
|
||||
FIELDS_MAX = 60
|
||||
METRIC_INPUTS = 8
|
||||
#: 短句:看得见的字(不算标点、空白)少于这么多,又没挂依据、没有方向词的,不送裁判
|
||||
SHORT_CHARS = 6
|
||||
#: 估算输出令牌:每句一条判定(理由 60 字上下,加上 JSON 的键)
|
||||
_OUT_PER_UNIT = 90
|
||||
#: 估算输出 token:每句一条判定(理由 80 字上下、缺什么,加上 JSON 的键)
|
||||
_OUT_PER_UNIT = 120
|
||||
_OUT_BASE = 20
|
||||
|
||||
#: 未裁判的原因。前四个是上限,报告里写「已到上限」
|
||||
@@ -88,17 +118,21 @@ REASONS = (*LIMITS, "error", "format", "missing", "on_demand")
|
||||
|
||||
JUDGE_ROLE = "你是报告核查员:逐句判断报告里的话,能不能被它挂的证据支持。你不写报告,也不改报告。"
|
||||
|
||||
JUDGE_RULES = """判定(verdict)四选一:
|
||||
JUDGE_RULES = f"""判定(verdict)五选一:
|
||||
- supported:证据足以支持这句话的说法——方向、比较、范围、因果都对得上
|
||||
- partial:一部分说法有证据支持,另一部分(原因、范围、程度)证据里没有
|
||||
- unsupported:证据不支持、和证据矛盾,或者这句在陈述数据事实却没有任何证据
|
||||
- contradicted:证据和这句话冲突——证据里的方向、比较、范围、排名,或者 SQL 的算法、字段,和句子说的对不上
|
||||
- insufficient:证据里没有判断这句话所需的信息(比如摘录里没有 SQL、没有字段清单、没有取数的查询),或者这句在陈述数据事实却没有挂任何证据
|
||||
- not_a_claim:这句不陈述数据事实(过渡、背景、建议、计划),不需要证据
|
||||
规则:
|
||||
1. 数字已经由系统核对过,不要复算:每个数都是系统从证据里取出来、按固定规则渲染的,不要因为数字本身或它的格式质疑这句话;你只判断句子的说法有没有证据支持。
|
||||
2. 只看下面给的证据摘录,不用常识、行业经验或外部知识补证据;摘录里没有的就是没有。
|
||||
3. rationale 用中文写,不超过 60 个字,说清依据是什么,或者缺了什么。
|
||||
4. used 写你实际用到的证据编号(摘录里【】中的那个,比如 m:gmv、Q1),没用到就写空数组。
|
||||
5. 每句都要给判定,unit 照抄句子前面方括号里的编号。"""
|
||||
3. 只有证据和句子冲突时才判 contradicted;证据里没有相关信息判 insufficient——「没有」不等于「矛盾」。
|
||||
4. 摘录注明「表结构快照不完整」时,「某张表、某个字段不存在」这类说法只能判 insufficient。
|
||||
5. rationale 用中文写,不超过 {RATIONALE_ASK} 个字,说清依据是什么,或者缺了什么。
|
||||
6. 判 insufficient 时 missing 写缺的是什么,越短越好,比如「SQL」「字段清单」「取数的查询」;其余判定 missing 写空字符串。
|
||||
7. used 写你实际用到的证据编号(摘录里【】中的那个,比如 m:gmv、Q1),没用到就写空数组。
|
||||
8. 每句都要给判定,unit 照抄句子前面方括号里的编号。"""
|
||||
|
||||
#: 报告节点 claims 为 judge 时拼在写作规则后面:写作者要知道还有一道按证据的核对
|
||||
JUDGE_RULE = ("这份报告的结论句还会由另一个模型逐句核对:它只看你挂的依据,判断这句话站不站得住。"
|
||||
@@ -114,10 +148,12 @@ JUDGE_SCHEMA: dict[str, Any] = {
|
||||
"properties": {
|
||||
"unit": {"type": "string", "description": "句子编号,照抄 [u3] 里的 u3"},
|
||||
"verdict": {"type": "string", "enum": list(JUDGED)},
|
||||
"rationale": {"type": "string", "description": f"判定的理由,中文,不超过 {RATIONALE_MAX} 个字"},
|
||||
"rationale": {"type": "string", "description": f"判定的理由,中文,不超过 {RATIONALE_ASK} 个字"},
|
||||
"missing": {"type": "string",
|
||||
"description": "判 insufficient 时写缺的是什么(比如 SQL、字段清单、取数的查询),其余判定写空字符串"},
|
||||
"used": {"type": "array", "items": {"type": "string"}, "description": "实际用到的证据编号"},
|
||||
},
|
||||
"required": ["unit", "verdict", "rationale", "used"],
|
||||
"required": ["unit", "verdict", "rationale", "missing", "used"],
|
||||
}}},
|
||||
"required": ["items"],
|
||||
}
|
||||
@@ -324,7 +360,90 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch) -> str:
|
||||
def _sql_line(label: str, sql: str) -> str:
|
||||
return f"{label}:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"
|
||||
|
||||
|
||||
def _source_hidden(source: Any, catalog: dict[str, Any], fetch: _Fetch, masked: dict[str, Any] | None) -> set[str]:
|
||||
"""一个数据源要遮的列(小写):数据源现在设的,加上这次运行里它的查询快照当时记下的(数据源改名、删掉了
|
||||
也照遮)。说不清是哪个数据源的,所有遮罩都算上:宁可多遮,不能露。"""
|
||||
name = str(source or "")
|
||||
names: set[str] = set()
|
||||
for key, cols in (masked or {}).items():
|
||||
if not name or str(key) == name:
|
||||
names |= _mask_names(cols)
|
||||
for entry in catalog.values():
|
||||
if not isinstance(entry, dict) or entry.get("kind") != "query":
|
||||
continue
|
||||
snapshot = fetch(entry.get("artifact"))
|
||||
if isinstance(snapshot, dict) and (not name or str(entry.get("source") or snapshot.get("source") or "") == name):
|
||||
names |= _mask_names(snapshot.get("mask_columns"))
|
||||
return names
|
||||
|
||||
|
||||
def _query_alias(catalog: dict[str, Any], artifact: Any) -> str | None:
|
||||
"""这份查询快照在报告目录里的编号(Q1):口径卡输入记的是工件,靠它对上。"""
|
||||
return next((a for a, e in catalog.items() if isinstance(e, dict) and e.get("kind") == "query"
|
||||
and artifact and e.get("artifact") == artifact), None)
|
||||
|
||||
|
||||
def _cell_words(locator: dict[str, Any], hidden: set[str]) -> str:
|
||||
"""输入落在哪一格:「r0 行 gmv 列」,数组字段是一段行「r0–r4 行 amount 列」。遮罩的列不写列名。"""
|
||||
rows = locator.get("rows")
|
||||
if isinstance(locator.get("row"), int):
|
||||
where = f"r{locator['row']} 行"
|
||||
elif isinstance(rows, list) and len(rows) == 2 and all(isinstance(r, int) for r in rows):
|
||||
where = f"r{rows[0]}–r{rows[1]} 行"
|
||||
else:
|
||||
where = ""
|
||||
cols = [locator["column"]] if locator.get("column") else list((locator.get("columns") or {}).values())
|
||||
cols = [str(c) for c in cols if c not in (None, "")]
|
||||
shown = [c for c in cols if c.lower() not in hidden]
|
||||
parts = [where] if where else []
|
||||
if shown:
|
||||
parts.append("、".join(shown) + " 列")
|
||||
if len(shown) < len(cols):
|
||||
parts.append("(所在的列按数据源的设置遮罩,不写列名)")
|
||||
return " ".join(parts) or "某一格"
|
||||
|
||||
|
||||
def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any],
|
||||
masked: dict[str, Any] | None) -> list[str]:
|
||||
"""指标的输入取自哪次查询、哪一格,再附上那次查询的 SQL(截到 SQL_CHARS,最多 ENTITY_QUERIES 次)。
|
||||
|
||||
「活跃用户按会员号去重」这种讲口径的句子,证据在取数的 SQL 里;只给原式,裁判只能判证据不足。
|
||||
位置来自封存的口径卡;SQL 经 fetch 取,取不到(不在封存范围里)的不给。没追到查询的输入照实写。
|
||||
"""
|
||||
items = [i for i in metric.get("inputs") or [] if isinstance(i, dict)]
|
||||
lines: list[str] = []
|
||||
sqls: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in items[:METRIC_INPUTS]:
|
||||
path = _clean(str(item.get("path") or item.get("field") or "")) or "(未命名)"
|
||||
artifact = item.get("artifact") if item.get("via") in ("tool_cell", "agent_field") else None
|
||||
if not artifact:
|
||||
origin = f"(来自节点 {item['node_id']})" if item.get("node_id") else ""
|
||||
lines.append(f"输入 {path}:未追溯到查询{origin}")
|
||||
continue
|
||||
alias = _query_alias(catalog, artifact)
|
||||
snapshot = fetch(artifact)
|
||||
snap = snapshot if isinstance(snapshot, dict) else {}
|
||||
source = (catalog.get(alias) or {}).get("source") if alias else None
|
||||
hidden = _source_hidden(source or snap.get("source"), catalog, fetch, masked) \
|
||||
| _mask_names(snap.get("mask_columns"))
|
||||
label = f"查询 {alias} " if alias else "一次查询(不在这份报告的证据目录中)"
|
||||
lines.append(f"输入 {path}:取自{label}的 {_cell_words(item.get('locator') or {}, hidden)}")
|
||||
sql = _clean(str(snap.get("sql") or ""))
|
||||
if sql and str(artifact) not in seen and len(seen) < ENTITY_QUERIES:
|
||||
seen.add(str(artifact))
|
||||
sqls.append(_sql_line(f"查询 {alias} 的 SQL" if alias else "这次查询的 SQL", sql))
|
||||
if len(items) > METRIC_INPUTS:
|
||||
lines.append(f"…另有 {len(items) - METRIC_INPUTS} 个输入")
|
||||
return lines + sqls
|
||||
|
||||
|
||||
def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any],
|
||||
masked: dict[str, Any] | None) -> str:
|
||||
name = entry.get("name") or (entry.get("locator") or {}).get("metric") or alias
|
||||
rendered = entry.get("rendered")
|
||||
where = " ".join(v for v in (f"口径「{entry['caliber']}」" if entry.get("caliber") else "",
|
||||
@@ -334,13 +453,83 @@ def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch) -> str:
|
||||
lines = [f"【{alias}】口径卡指标:{value}" + (f"({where})" if where else "")]
|
||||
card = fetch(entry.get("artifact"))
|
||||
wanted = (entry.get("locator") or {}).get("metric")
|
||||
for metric in (card.get("metrics") if isinstance(card, dict) else None) or []:
|
||||
if isinstance(metric, dict) and metric.get("id") == wanted and metric.get("expression"):
|
||||
metric = next((m for m in (card.get("metrics") if isinstance(card, dict) else None) or []
|
||||
if isinstance(m, dict) and m.get("id") == wanted), None)
|
||||
if metric is not None:
|
||||
if metric.get("expression"):
|
||||
lines.append(f"原式:{_clean(str(metric['expression']))}")
|
||||
break
|
||||
lines.extend(_metric_inputs(metric, fetch, catalog, masked))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _table_lines(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch,
|
||||
masked: dict[str, Any] | None) -> list[str]:
|
||||
"""表的字段清单和类型:取自封存范围内(经 fetch)的表结构快照,最多 FIELDS_MAX 个,遮罩的列不列。
|
||||
|
||||
讲表结构的句子(「订单表按会员号记录」)只给表名,裁判只能判证据不足。快照不完整(数据源的表太多、
|
||||
只存了一部分)要注明:这时「某字段不存在」这类说法只能判 insufficient。
|
||||
"""
|
||||
name = entry.get("name") or (entry.get("locator") or {}).get("table")
|
||||
origins = [o for o in entry.get("sources") or [] if isinstance(o, dict) and o.get("kind") == "schema"
|
||||
and o.get("artifact")]
|
||||
truncated = any(o.get("truncated") for o in origins)
|
||||
info: dict[str, Any] | None = None
|
||||
snapshot: dict[str, Any] = {}
|
||||
readable = False
|
||||
for origin in origins:
|
||||
content = fetch(origin["artifact"])
|
||||
readable = readable or isinstance(content, dict)
|
||||
if (found := schema_table(content, name)) is not None:
|
||||
info, snapshot = found, content
|
||||
break
|
||||
lines: list[str] = []
|
||||
if info is None:
|
||||
lines.append("字段清单:没有(" + ("本次运行的表结构快照中没有这张表" if readable or not origins
|
||||
else "无法读取这张表所在的表结构快照") + ")")
|
||||
else:
|
||||
truncated = truncated or bool(snapshot.get("truncated"))
|
||||
hidden = _source_hidden(entry.get("source") or snapshot.get("source"), catalog, fetch, masked)
|
||||
fields, more, masked_any = table_fields(info, hidden, limit=FIELDS_MAX)
|
||||
listed = " | ".join(f"{f['name']} {f['type']}".strip() for f in fields) or "无"
|
||||
lines.append(f"字段清单(名称 类型,共 {len(fields) + more} 个):{listed}"
|
||||
+ (f" | …另有 {more} 个字段" if more else ""))
|
||||
if masked_any:
|
||||
lines.append("(有的字段按数据源的设置遮罩了,没有列出)")
|
||||
if truncated:
|
||||
lines.append("注意:表结构快照不完整(数据源的表太多,只保存了一部分),不能据此断定某张表或某个字段不存在")
|
||||
return lines
|
||||
|
||||
|
||||
def _column_lines(entry: dict[str, Any], fetch: _Fetch) -> list[str]:
|
||||
"""字段所属的表和类型。类型只从封存范围内的表结构快照取;表结构里没有的列(聚合的别名)才取查询快照
|
||||
记下的列类型——和证据面板同一个口径。"""
|
||||
locator = entry.get("locator") or {}
|
||||
table, column = locator.get("table"), locator.get("column") or entry.get("name")
|
||||
owners = [str(table)] if table else [str(t) for t in entry.get("tables") or []]
|
||||
origins = [o for o in entry.get("sources") or [] if isinstance(o, dict) and o.get("artifact")]
|
||||
types: dict[str, str] = {}
|
||||
for origin in origins:
|
||||
if origin.get("kind") != "schema":
|
||||
continue
|
||||
content = fetch(origin["artifact"])
|
||||
for owner in owners:
|
||||
if (col := schema_column(schema_table(content, owner), column)) is not None and col.get("type"):
|
||||
types.setdefault(owner, str(col["type"]))
|
||||
lines = ["所属的表:" + "、".join(owners)] if owners else []
|
||||
if len(set(types.values())) == 1:
|
||||
lines.append(f"类型:{next(iter(types.values()))}(取自表结构快照)")
|
||||
elif types:
|
||||
lines.append("类型:" + ";".join(f"{t} 表中为 {k}" for t, k in types.items()) + "(取自表结构快照)")
|
||||
elif not any(o.get("kind") == "schema" for o in origins):
|
||||
for origin in origins:
|
||||
content = fetch(origin["artifact"]) if origin.get("kind") == "result" else None
|
||||
kinds = content.get("column_types") if isinstance(content, dict) else None
|
||||
if isinstance(kinds, dict) and isinstance(kinds.get(str(column)), str):
|
||||
lines.append(f"类型:{kinds[str(column)]}(取自查询结果)")
|
||||
break
|
||||
return lines
|
||||
|
||||
|
||||
def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], whole: bool, fetch: _Fetch) -> str | None:
|
||||
content = fetch(entry.get("artifact"))
|
||||
hits = content.get("hits") if isinstance(content, dict) else None
|
||||
@@ -364,7 +553,7 @@ def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _
|
||||
masked: dict[str, Any] | None, catalog: dict[str, Any]) -> str | None:
|
||||
kind = entry.get("kind")
|
||||
if kind == "metric":
|
||||
return _metric_excerpt(alias, entry, fetch)
|
||||
return _metric_excerpt(alias, entry, fetch, catalog, masked)
|
||||
if kind == "input":
|
||||
field_name = (entry.get("locator") or {}).get("field") or alias
|
||||
return f"【{alias}】运行输入:{field_name} = {entry.get('rendered')}"
|
||||
@@ -377,11 +566,11 @@ def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _
|
||||
return _retrieval_excerpt(alias, entry, picked, any(alias in c.whole for c in cands), fetch)
|
||||
if kind == "table":
|
||||
head = f"【{alias}】表 {entry.get('name')}" + (f"(数据源 {entry['source']})" if entry.get("source") else "")
|
||||
return "\n".join([head, *_entity_sql(entry, catalog, fetch)])
|
||||
return "\n".join([head, *_table_lines(entry, catalog, fetch, masked), *_entity_sql(entry, catalog, fetch)])
|
||||
if kind == "column":
|
||||
table = (entry.get("locator") or {}).get("table")
|
||||
head = f"【{alias}】字段 {f'{table}.' if table else ''}{(entry.get('locator') or {}).get('column') or entry.get('name')}"
|
||||
return "\n".join([head, *_entity_sql(entry, catalog, fetch)])
|
||||
return "\n".join([head, *_column_lines(entry, fetch), *_entity_sql(entry, catalog, fetch)])
|
||||
return None
|
||||
|
||||
|
||||
@@ -445,7 +634,8 @@ class JudgeRequest:
|
||||
HumanMessage(content=(
|
||||
f"证据摘录(【】里是编号,判定时 used 写它):\n{evidence}\n\n"
|
||||
"要判断的句子(每句都要给判定):\n" + "\n".join(self.unit_line(c) for c in ordered) + "\n\n"
|
||||
'只输出 JSON:{"items": [{"unit": "u1", "verdict": "supported", "rationale": "…", "used": ["m:gmv"]}]}'
|
||||
'只输出 JSON:{"items": [{"unit": "u1", "verdict": "supported", "rationale": "…", "missing": "", '
|
||||
'"used": ["m:gmv"]}]}'
|
||||
)),
|
||||
]
|
||||
|
||||
@@ -498,7 +688,7 @@ def request_key(request: JudgeRequest, spec: ModelSpec, budget: Budget, known: d
|
||||
return content_hash(canonical_json({
|
||||
"units": [[c.unit, c.priority, c.text, c.section, list(c.aliases)] for c in request.cands],
|
||||
"excerpts": request.excerpts, "model": [spec.provider, spec.model], "budget": budget.as_dict(),
|
||||
"known": sorted(known or {}),
|
||||
"known": sorted(known or {}), "rules": RULES_VERSION,
|
||||
}))
|
||||
|
||||
|
||||
@@ -650,25 +840,41 @@ async def judge_model_spec(session: Any, node_judge: dict[str, Any] | None = Non
|
||||
|
||||
async def source_masks(catalog: dict[str, Any], *, loader: Callable[[str], Any] | None = None
|
||||
) -> dict[str, list[str]]:
|
||||
"""目录里的查询来自的数据源现在设的遮罩:{数据源名: 列名}。和证据面板同一个口径。
|
||||
"""目录里的证据来自的数据源现在设的遮罩:{数据源名: 列名}。和证据面板同一个口径。
|
||||
|
||||
旧形状的台账条目没记数据源的,按快照自己记的数据源查(快照经 loader 取,缺省读工件库;
|
||||
按需裁判的接口要传只认封存工件的 loader,和 prepare 同一个)。
|
||||
查的数据源:查询条目的;表、字段条目的(字段清单里遮罩的列不列);口径卡指标的输入取自的查询快照的。
|
||||
条目没记数据源的,按快照自己记的数据源查(快照经 loader 取,缺省读工件库;按需裁判的接口要传只认
|
||||
封存工件的 loader,和 prepare 同一个)。
|
||||
"""
|
||||
from app.data.engine import masked_columns
|
||||
from app.db.models import DataSource
|
||||
|
||||
fetch = _Fetch(loader)
|
||||
found: set[str] = set()
|
||||
|
||||
def source_of(artifact: Any) -> Any:
|
||||
snapshot = fetch(artifact)
|
||||
return snapshot.get("source") if isinstance(snapshot, dict) else None
|
||||
|
||||
for entry in catalog.values():
|
||||
if not isinstance(entry, dict) or entry.get("kind") != "query":
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
source = entry.get("source")
|
||||
if not source:
|
||||
snapshot = fetch(entry.get("artifact"))
|
||||
source = snapshot.get("source") if isinstance(snapshot, dict) else None
|
||||
if source:
|
||||
found.add(str(source))
|
||||
kind = entry.get("kind")
|
||||
sources: list[Any] = []
|
||||
if kind == "query":
|
||||
sources.append(entry.get("source") or source_of(entry.get("artifact")))
|
||||
elif kind in ENTITY_KINDS:
|
||||
sources.append(entry.get("source") or next(
|
||||
(source_of(o.get("artifact")) for o in entry.get("sources") or []
|
||||
if isinstance(o, dict) and o.get("kind") == "schema"), None))
|
||||
elif kind == "metric":
|
||||
card = fetch(entry.get("artifact"))
|
||||
wanted = (entry.get("locator") or {}).get("metric")
|
||||
for metric in (card.get("metrics") if isinstance(card, dict) else None) or []:
|
||||
if isinstance(metric, dict) and metric.get("id") == wanted:
|
||||
sources.extend(source_of(i.get("artifact")) for i in metric.get("inputs") or []
|
||||
if isinstance(i, dict) and i.get("artifact"))
|
||||
found.update(str(s) for s in sources if s)
|
||||
names = sorted(found)
|
||||
if not names:
|
||||
return {}
|
||||
@@ -885,7 +1091,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget,
|
||||
pending: list[Candidate] = []
|
||||
for c in ranked:
|
||||
hit = (known or {}).get(keys[c.unit])
|
||||
if isinstance(hit, dict) and hit.get("status") in JUDGED:
|
||||
if isinstance(hit, dict) and hit.get("status") in SETTLED:
|
||||
verdicts[c.unit] = dict(hit)
|
||||
reused += 1
|
||||
else:
|
||||
@@ -998,10 +1204,38 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget,
|
||||
return outcome
|
||||
|
||||
|
||||
#: 截理由时认的句子边界(全角、半角都认)
|
||||
_BOUNDARY = "。;,;,!?!?"
|
||||
|
||||
|
||||
def clip_text(text: str, limit: int = RATIONALE_MAX) -> str:
|
||||
"""超过 limit 个字时截短:在句子边界(。;,)截断加「…」,总长不超过 limit。
|
||||
|
||||
边界离开头太近(不到三分之一)就不按它截——只剩几个字的理由没用;这时退到词的边界(不把一个英文词、
|
||||
一个数截成两半)。没超长的原样返回。
|
||||
"""
|
||||
text = text.strip()
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
head = text[:limit - 1]
|
||||
cut = max(head.rfind(ch) for ch in _BOUNDARY)
|
||||
if cut < limit // 3:
|
||||
cut = len(head)
|
||||
while cut > 0 and head[cut - 1].isascii() and head[cut - 1].isalnum() and text[cut].isascii() \
|
||||
and text[cut].isalnum():
|
||||
cut -= 1
|
||||
if cut < limit // 3:
|
||||
cut = len(head)
|
||||
return head[:cut].rstrip(" ,,;;、") + "…"
|
||||
|
||||
|
||||
def _take(items: list[Any], batch: list[Candidate], request: JudgeRequest, verdicts: dict[str, dict[str, Any]], *,
|
||||
judge: str | None, post_seal: bool) -> None:
|
||||
"""模型给的判定逐条收下:不在这一批的编号不认,判定不认识的记格式不对,理由截到 60 字,
|
||||
used 只留这一批摘录里真有的编号。"""
|
||||
"""模型给的判定逐条收下:不在这一批的编号不认,判定不认识的记格式不对,理由在句子边界截到 RATIONALE_MAX 字,
|
||||
used 只留这一批摘录里真有的编号。判 insufficient 的另记 missing(缺的是什么,模型没写就不编)。
|
||||
|
||||
模型照旧写 unsupported 的按 contradicted 收下:新判的不再产出旧取值,而 contradicted 的处置和原来的
|
||||
unsupported 一样(不能因为模型用了旧词就放过这一句)。"""
|
||||
ids = {c.unit for c in batch}
|
||||
aliases = {a for c in batch for a in c.aliases if a in request.excerpts}
|
||||
for item in items:
|
||||
@@ -1011,26 +1245,43 @@ def _take(items: list[Any], batch: list[Candidate], request: JudgeRequest, verdi
|
||||
if uid not in ids or uid in verdicts:
|
||||
continue
|
||||
status = str(item.get("verdict") or "").strip().lower()
|
||||
if status == UNSUPPORTED:
|
||||
status = "contradicted"
|
||||
if status not in JUDGED:
|
||||
verdicts[uid] = _unjudged("format", f"裁判返回的判定「{str(item.get('verdict'))[:20]}」无法识别,这句未裁判",
|
||||
judge=judge, post_seal=post_seal)
|
||||
continue
|
||||
used = item.get("used") if isinstance(item.get("used"), list) else []
|
||||
verdicts[uid] = {"status": status, "rationale": _clean(str(item.get("rationale") or ""))[:RATIONALE_MAX],
|
||||
"judge": judge, "post_seal": post_seal,
|
||||
"used": list(dict.fromkeys(u.strip() for u in used if isinstance(u, str)
|
||||
and u.strip() in aliases))}
|
||||
verdict: dict[str, Any] = {
|
||||
"status": status, "rationale": clip_text(_clean(str(item.get("rationale") or ""))),
|
||||
"judge": judge, "post_seal": post_seal,
|
||||
"used": list(dict.fromkeys(u.strip() for u in used if isinstance(u, str) and u.strip() in aliases))}
|
||||
missing = item.get("missing")
|
||||
if status == "insufficient" and isinstance(missing, str) and (missing := clip_text(_clean(missing), MISSING_MAX)):
|
||||
verdict["missing"] = missing
|
||||
verdict["rule_version"] = RULES_VERSION
|
||||
verdicts[uid] = verdict
|
||||
|
||||
|
||||
def outdated(verdict: Any) -> bool:
|
||||
"""这条判过的判定早于当前规则:没有规则版本(改造前判的)、版本更小,或者是旧取值 unsupported。
|
||||
没判成的(unjudged)谈不上新旧,返回 False。"""
|
||||
if not isinstance(verdict, dict) or verdict.get("status") not in SETTLED:
|
||||
return False
|
||||
version = verdict.get("rule_version")
|
||||
return verdict.get("status") == UNSUPPORTED or isinstance(version, bool) or not isinstance(version, int) \
|
||||
or version < RULES_VERSION
|
||||
|
||||
|
||||
def _outcome(request: JudgeRequest, verdicts: dict[str, dict[str, Any]], budget: Budget, details: dict[str, str], *,
|
||||
click: bool = False, shown: Budget | None = None) -> dict[str, Any]:
|
||||
"""汇总判定。outcome.budget 是这一轮实际用的上限;缺口里写的上限按 shown(缺省同 budget)。"""
|
||||
label = shown or budget
|
||||
counts = {s: 0 for s in VERDICTS}
|
||||
counts = {s: 0 for s in COUNT_KEYS}
|
||||
why: dict[str, int] = {}
|
||||
for c in request.cands:
|
||||
v = verdicts[c.unit]
|
||||
counts[v["status"]] += 1
|
||||
counts[v["status"]] = counts.get(v["status"], 0) + 1
|
||||
if v["status"] == "unjudged":
|
||||
why[v["reason"]] = why.get(v["reason"], 0) + 1
|
||||
gaps: list[tuple[str, str]] = []
|
||||
@@ -1073,13 +1324,18 @@ async def judge_doc(doc: dict[str, Any], catalog: dict[str, Any] | None = None,
|
||||
#: 文档的 judge 摘要里、report.checked 和节点产出的 judge 里都有的键
|
||||
SUMMARY_KEYS = ("mode", "model", "on_unsupported", "rewrite_once", "limits", "candidates", "screened", "counts",
|
||||
"unjudged", "limits_hit", "complete", "priced", "cost_usd", "calls", "reused", "duration_ms",
|
||||
"gaps", "notes", "same_model", "rewrite")
|
||||
"gaps", "notes", "same_model", "rewrite", "writer_model")
|
||||
|
||||
|
||||
def summarize(outcome: dict[str, Any], *, mode: str, on_unsupported: str = "degrade", rewrite_once: bool = False,
|
||||
same_model: bool = False, rewrite: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
"""裁判摘要:写进文档的 judge,也原样放进 report.checked 和节点产出。"""
|
||||
counts = outcome["counts"]
|
||||
same_model: bool = False, rewrite: dict[str, Any] | None = None,
|
||||
writer_model: str | None = None) -> dict[str, Any]:
|
||||
"""裁判摘要:写进文档的 judge,也原样放进 report.checked 和节点产出。
|
||||
|
||||
counts 补齐 COUNT_KEYS 的每个键(断点里缓存的旧判定只有旧的几个键);writer_model 是这份报告的写作模型,
|
||||
same_model 是裁判模型和它相同。"""
|
||||
raw = outcome["counts"]
|
||||
counts = {k: int(raw.get(k) or 0) for k in COUNT_KEYS}
|
||||
return {
|
||||
"mode": mode, "model": outcome.get("model"), "on_unsupported": on_unsupported, "rewrite_once": rewrite_once,
|
||||
"limits": outcome.get("budget"), "candidates": outcome["candidates"], "screened": outcome["screened"],
|
||||
@@ -1088,7 +1344,7 @@ def summarize(outcome: dict[str, Any], *, mode: str, on_unsupported: str = "degr
|
||||
"priced": outcome.get("priced"), "cost_usd": outcome.get("cost_usd", 0.0), "calls": outcome.get("calls", 0),
|
||||
"reused": outcome.get("reused", 0), "duration_ms": outcome.get("duration_ms", 0),
|
||||
"gaps": outcome["gaps"] if mode == "inline" else [], "notes": outcome.get("notes") or [],
|
||||
"same_model": same_model, "rewrite": rewrite,
|
||||
"same_model": same_model, "rewrite": rewrite, "writer_model": writer_model,
|
||||
}
|
||||
|
||||
|
||||
@@ -1103,8 +1359,11 @@ def apply_judgement(doc: dict[str, Any], outcome: dict[str, Any], summary: dict[
|
||||
return doc
|
||||
|
||||
|
||||
def on_demand(doc: dict[str, Any], *, on_unsupported: str = "degrade", rewrite_once: bool = False) -> dict[str, Any]:
|
||||
"""探索运行:节点里不花这笔钱。候选句标「未裁判 · 按需」,点开哪句再判哪句。返回摘要(已写进文档)。"""
|
||||
def on_demand(doc: dict[str, Any], *, on_unsupported: str = "degrade", rewrite_once: bool = False,
|
||||
writer_model: str | None = None) -> dict[str, Any]:
|
||||
"""探索运行:节点里不花这笔钱。候选句标「未裁判 · 按需」,点开哪句再判哪句。返回摘要(已写进文档)。
|
||||
|
||||
writer_model 记进摘要:按需裁判时拿它判断裁判模型是不是和写作模型相同。"""
|
||||
request = prepare(doc, {})
|
||||
budget = Budget()
|
||||
verdicts = {c.unit: _unjudged("on_demand", _why_unjudged("on_demand", budget), judge=None, post_seal=False)
|
||||
@@ -1112,6 +1371,7 @@ def on_demand(doc: dict[str, Any], *, on_unsupported: str = "degrade", rewrite_o
|
||||
outcome = _outcome(request, verdicts, budget, {})
|
||||
outcome.pop("_gaps")
|
||||
outcome["budget"] = None
|
||||
summary = summarize(outcome, mode="on_demand", on_unsupported=on_unsupported, rewrite_once=rewrite_once)
|
||||
summary = summarize(outcome, mode="on_demand", on_unsupported=on_unsupported, rewrite_once=rewrite_once,
|
||||
writer_model=writer_model)
|
||||
apply_judgement(doc, outcome, summary)
|
||||
return summary
|
||||
|
||||
@@ -92,7 +92,7 @@ CONTRACT_KEY_LABEL: dict[str, str] = {
|
||||
JUDGE_FIELD_LABEL: dict[str, str] = {
|
||||
"provider": "裁判模型 · 接入", "model": "裁判模型", "max_claims": "最多裁判句数",
|
||||
"max_cost_usd": "金额上限(美元)", "timeout_s": "时长上限(秒)",
|
||||
"rewrite_once": "证据不支持的句子退回改写一次", "on_unsupported": "证据不支持时",
|
||||
"rewrite_once": "证据相矛盾或不足的句子退回改写一次", "on_unsupported": "证据相矛盾时",
|
||||
}
|
||||
|
||||
#: 取值的短说法:键 → {值: 叫法}。前端下拉的选项文字去掉后半句的用途说明
|
||||
|
||||
@@ -417,7 +417,7 @@ def _check_citations(
|
||||
|
||||
#: 没挂依据的结论句怎么处置,从松到严。认不出的写法按 degrade 算:写错一个词不能让缺口悄悄消失
|
||||
_ON_UNCITED = ("ignore", "degrade", "withhold")
|
||||
#: 证据不支持的结论句怎么处置(claims: judge),从松到严。认不出的写法同样按 degrade 算
|
||||
#: 证据相矛盾的结论句怎么处置(claims: judge 的 on_unsupported),从松到严。认不出的写法同样按 degrade 算
|
||||
_ON_UNSUPPORTED = ("degrade", "withhold")
|
||||
#: 要求结论句挂依据的策略:judge 在这一点上和 require_citation 一样严,另外再按裁判的判定判档
|
||||
_CITING = ("require_citation", "judge")
|
||||
@@ -434,9 +434,10 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
|
||||
里 ignore 一律当 degrade:没有哪道门禁查契约里的 claims,不能让它把用户要的底线拉低。
|
||||
|
||||
报告节点写的是 judge 时,判定在文档里(按哈希取回的那份:判定是模型给的,出口复算不了,只能信封存的
|
||||
文档)。正式运行在节点里判过:证据不支持的按 on_unsupported 判档(报告节点和契约取更严的),没判完的
|
||||
(裁判没跑成、触顶)把裁判摘要里的缺口照抄进来——不能判完整出具,也不当成「不支持」。探索运行按需裁判,
|
||||
这里只标注。契约要 judge、报告节点却没开的,照实记缺口:出口替不了报告节点去裁判。
|
||||
文档)。正式运行在节点里判过:证据相矛盾的(contradicted,旧文档里的 unsupported 同样算)按 on_unsupported
|
||||
判档(报告节点和契约取更严的);证据不足的(insufficient)记一条缺口——最多降档,不会不予出具,也不会
|
||||
完整出具;没判完的(裁判没跑成、触顶)把裁判摘要里的缺口照抄进来——不能判完整出具,也不当成「矛盾」。
|
||||
探索运行按需裁判,这里只标注。契约要 judge、报告节点却没开的,照实记缺口:出口替不了报告节点去裁判。
|
||||
|
||||
数的是这里重新核对出来的 uncited,不信文档里记的 cites。升级前写的报告(节点产出里没有 claims 这个键,
|
||||
包括跨着升级还没跑完的运行)一律不管,契约写了也只记一条缺口:报告生成于升级前,结论句未核对。
|
||||
@@ -472,6 +473,7 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
|
||||
return
|
||||
|
||||
from app.engine.evidence import iter_units
|
||||
from app.engine.judge import COUNT_KEYS, UNSUPPORTED
|
||||
|
||||
summary = (doc or {}).get("judge")
|
||||
if not isinstance(summary, dict):
|
||||
@@ -483,7 +485,7 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
|
||||
asked.append(declared.get("on_unsupported"))
|
||||
on_unsupported = max((a if a in _ON_UNSUPPORTED else "degrade" for a in asked), key=_ON_UNSUPPORTED.index)
|
||||
inline = summary.get("mode") == "inline"
|
||||
flagged: dict[str, list[dict[str, Any]]] = {"unsupported": [], "partial": []}
|
||||
flagged: dict[str, list[dict[str, Any]]] = {s: [] for s in ("contradicted", "insufficient", UNSUPPORTED, "partial")}
|
||||
if inline:
|
||||
# 探索运行按需裁判的判定不在文档里(在封存之后追加的 evidence.judged 事件里),这里只数正式运行的
|
||||
markdown = str((doc or {}).get("markdown") or "")
|
||||
@@ -493,16 +495,26 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
|
||||
if status in flagged:
|
||||
span = unit.get("span") or [0, 0]
|
||||
flagged[status].append({"unit": unit.get("id"), "span": span, "text": markdown[span[0]:span[1]],
|
||||
"rationale": verdict.get("rationale") or ""})
|
||||
"rationale": verdict.get("rationale") or "",
|
||||
**({"missing": verdict["missing"]} if status == "insufficient"
|
||||
and verdict.get("missing") else {})})
|
||||
out["gaps"].extend(f"报告「{title}」{gap}" for gap in summary.get("gaps") or [])
|
||||
counts = {k: int((summary.get("counts") or {}).get(k) or 0)
|
||||
for k in ("supported", "partial", "unsupported", "not_a_claim", "unjudged")}
|
||||
if lacking := flagged["insufficient"]:
|
||||
# 证据不足按缺口处理:最多降档,不因 on_unsupported 不予出具,也不能完整出具
|
||||
missing = list(dict.fromkeys(i["missing"] for i in lacking if i.get("missing")))
|
||||
what = f"(缺少:{'、'.join(missing[:5])}{'等' if len(missing) > 5 else ''})" if missing else ""
|
||||
out["gaps"].append(f"报告「{title}」有 {len(lacking)} 句结论证据不足{what}:裁判未能在证据中找到判断所需的"
|
||||
"信息,这些结论未经核实")
|
||||
counts = {k: int((summary.get("counts") or {}).get(k) or 0) for k in COUNT_KEYS}
|
||||
# 证据相矛盾的按 on_unsupported 判档;旧文档里的 unsupported(当时没分矛盾和不足)照旧当 contradicted
|
||||
contradicted = [*flagged["contradicted"], *flagged[UNSUPPORTED]]
|
||||
out["claims_policy"] = {"policy": "judge", "on_unsupported": on_unsupported, "on_uncited": on_uncited}
|
||||
out["uncited"], out["unsupported"] = uncited, flagged["unsupported"]
|
||||
out["uncited"], out["unsupported"] = uncited, contradicted
|
||||
out["claims"] = {
|
||||
"policy": "judge", "uncited_claims": len(uncited), "uncited": uncited[:_LISTED], **extra,
|
||||
"on_unsupported": on_unsupported, "mode": "inline" if inline else "on_demand", "counts": counts,
|
||||
"unsupported": flagged["unsupported"][:_LISTED], "partial": flagged["partial"][:_LISTED],
|
||||
"contradicted": flagged["contradicted"][:_LISTED], "insufficient": flagged["insufficient"][:_LISTED],
|
||||
"unsupported": flagged[UNSUPPORTED][:_LISTED], "partial": flagged["partial"][:_LISTED],
|
||||
"unjudged": dict(summary.get("unjudged") or {}), "limits_hit": list(summary.get("limits_hit") or []),
|
||||
"complete": bool(summary.get("complete")), "model": summary.get("model"),
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
- 正式运行在节点里判:写完、核对完之后判,判定写进文档的 unit.verdict,随文档一起封存。判定只标注、
|
||||
不改写正文;裁判没跑成、触顶了照样产出,没判的记未裁判并留下缺口,出口按 on_unsupported 判档
|
||||
- 探索运行不在节点里花这笔钱:候选句标「未裁判 · 按需」,点开哪句再判哪句(证据接口)
|
||||
- rewrite_once(默认关):证据不支持的句子连同理由交回写作者只改这几句,再判一次,最多一轮;
|
||||
- rewrite_once(默认关):证据相矛盾、证据不足的句子连同理由交回写作者只改这几句,再判一次,最多一轮;
|
||||
改写稿冒出原稿没有的违规(只标注的可疑实体也算)就不采用
|
||||
- 裁判调用包在 @task 里,排在节点所有已有调用之后:「接着跑」时直接取 checkpoint 里的判定,
|
||||
不重复调用、不重复计费。写作调用不进 checkpoint,接着跑时会重写一稿——所以判定带着输入指纹,
|
||||
@@ -450,7 +450,7 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
if await _run_class(ctx.run.run_id) != "formal":
|
||||
# 探索运行默认按需裁判:点开哪句才判哪句,节点里不花这笔钱
|
||||
judgement = judging.on_demand(doc, on_unsupported=judge_cfg["on_unsupported"],
|
||||
rewrite_once=judge_cfg["rewrite_once"])
|
||||
rewrite_once=judge_cfg["rewrite_once"], writer_model=model_id)
|
||||
elif not (blocking and on_violation == "fail"):
|
||||
# 反正要判失败的报告不值得再花钱判(节点紧接着就报错)
|
||||
doc, response, blocking, judgement, usages = await _judge_inline(
|
||||
@@ -505,13 +505,21 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
def _rewrite_request(sentences: list[tuple[str, str]]) -> str:
|
||||
listed = "\n".join(f"{i}.「{text}」——{why or '证据不支持这句话'}" for i, (text, why) in enumerate(sentences, 1))
|
||||
return (
|
||||
"另一个模型按证据逐句核对了你的报告,下面这几句它认为证据不支持:\n"
|
||||
"另一个模型按证据逐句核对了你的报告,下面这几句它认为证据不支持(和证据相矛盾,或者证据不足):\n"
|
||||
f"{listed}\n\n"
|
||||
"请只改这几句:补上证据目录里真有的依据 [[see:…]],或者把说法收窄到证据能支持的程度,或者删掉;"
|
||||
"其余部分一字不改。数字仍然只能用引用标记写。不要解释改了什么,只输出改好的整篇报告正文。"
|
||||
)
|
||||
|
||||
|
||||
def _rewrite_reason(verdict: dict[str, Any]) -> str:
|
||||
"""交回写作者的理由:裁判的理由,证据不足的再写上缺的是什么。"""
|
||||
why = str(verdict.get("rationale") or "")
|
||||
if verdict.get("status") == "insufficient" and verdict.get("missing"):
|
||||
why = f"{why}(缺少:{verdict['missing']})" if why else f"证据不足,缺少:{verdict['missing']}"
|
||||
return why
|
||||
|
||||
|
||||
def _merge(first: dict[str, Any], second: dict[str, Any], budget: judging.Budget) -> dict[str, Any]:
|
||||
"""改写前后两轮裁判合成一份:判定、句数、缺口按改写后这一稿,花费、调用、用时两轮相加。"""
|
||||
merged = dict(second)
|
||||
@@ -582,10 +590,11 @@ async def _judge_inline(ctx: NodeContext, doc: dict[str, Any], catalog: dict[str
|
||||
usages = [outcome["usage"]]
|
||||
response = messages[-1]
|
||||
rewrite: dict[str, Any] | None = None
|
||||
bad = [uid for uid, v in outcome["verdicts"].items() if v["status"] == "unsupported"]
|
||||
# 交回改写的:证据相矛盾的(含旧取值 unsupported)和证据不足的——两样原来都判作「证据不支持」
|
||||
bad = [uid for uid, v in outcome["verdicts"].items() if v["status"] in (*judging.CONTRADICTED, "insufficient")]
|
||||
if judge_cfg["rewrite_once"] and bad:
|
||||
texts = judging.unit_texts(doc)
|
||||
sentences = [(texts.get(uid, ""), outcome["verdicts"][uid].get("rationale") or "") for uid in bad]
|
||||
sentences = [(texts.get(uid, ""), _rewrite_reason(outcome["verdicts"][uid])) for uid in bad]
|
||||
ctx.emit(EventType.LOG, level="warn", code="report_rewrite",
|
||||
message=f"裁判认为 {len(bad)} 句结论证据不支持({_named([{'text': t} for t, _ in sentences])}),"
|
||||
"已退回写作模型仅修改这几句")
|
||||
@@ -620,7 +629,8 @@ async def _judge_inline(ctx: NodeContext, doc: dict[str, Any], catalog: dict[str
|
||||
rewrite.update(applied=True, units=[at.get(texts.get(uid, "")) for uid in bad],
|
||||
changed=[uid for uid, text in now.items() if text not in before])
|
||||
summary = judging.summarize(outcome, mode="inline", on_unsupported=judge_cfg["on_unsupported"],
|
||||
rewrite_once=judge_cfg["rewrite_once"], same_model=same_model, rewrite=rewrite)
|
||||
rewrite_once=judge_cfg["rewrite_once"], same_model=same_model, rewrite=rewrite,
|
||||
writer_model=model_id)
|
||||
judging.apply_judgement(doc, outcome, summary)
|
||||
return doc, response, blocking, summary, usages
|
||||
|
||||
|
||||
@@ -425,7 +425,7 @@ CLAIMS_JUDGE = "judge"
|
||||
CLAIMS_VALUES = (*REPORT_CLAIMS, CLAIMS_JUDGE)
|
||||
#: judge 子配置能写的键。三个上限写 null 表示不限;没写的键运行时取设置里的默认(设置 → judge 一组)
|
||||
JUDGE_KEYS = ("provider", "model", "max_claims", "max_cost_usd", "timeout_s", "rewrite_once", "on_unsupported")
|
||||
#: 证据不支持时正式运行怎么判档:degrade 降档(默认)/ withhold 不予出具。探索运行只标注
|
||||
#: 证据相矛盾时正式运行怎么判档:degrade 降档(默认)/ withhold 不予出具。探索运行只标注
|
||||
JUDGE_ON_UNSUPPORTED = ("degrade", "withhold")
|
||||
|
||||
|
||||
|
||||
@@ -152,24 +152,25 @@ async def test_every_claim_supported_is_a_full_issuance(monkeypatch):
|
||||
assert issuance["tier"] == "formal" and issuance["gaps"] == [], issuance
|
||||
claims = issuance["claims"]
|
||||
assert claims["policy"] == "judge" and claims["mode"] == "inline" and claims["complete"] is True
|
||||
assert claims["counts"] == {"supported": 2, "partial": 0, "unsupported": 0, "not_a_claim": 0, "unjudged": 0}
|
||||
assert claims["counts"] == {"supported": 2, "partial": 0, "contradicted": 0, "insufficient": 0, "not_a_claim": 0,
|
||||
"unjudged": 0, "unsupported": 0}
|
||||
assert claims["on_unsupported"] == "degrade" and claims["uncited_claims"] == 0
|
||||
assert claims["unsupported"] == [] and claims["model"] == JUDGE_MODEL
|
||||
assert claims["contradicted"] == [] and claims["insufficient"] == [] and claims["model"] == JUDGE_MODEL
|
||||
|
||||
|
||||
async def test_an_unsupported_claim_degrades_by_default(monkeypatch):
|
||||
Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
async def test_a_contradicted_claim_degrades_by_default(monkeypatch):
|
||||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
issuance = await issued(weekly())
|
||||
assert issuance["tier"] == "degraded", "不支持按缺口降档,不是不予出具(契约是 strict 的)"
|
||||
assert issuance["gaps"] == [], "判了就不是「没核」:档位来自判定,不来自缺口"
|
||||
[flagged] = issuance["claims"]["unsupported"]
|
||||
[flagged] = issuance["claims"]["contradicted"]
|
||||
assert flagged["text"] == "增长主要来自新客首单。" and flagged["rationale"] == "证据里没有新客维度"
|
||||
assert flagged["unit"] and len(flagged["span"]) == 2
|
||||
assert issuance["claims"]["counts"]["unsupported"] == 1
|
||||
assert issuance["claims"]["counts"]["contradicted"] == 1
|
||||
|
||||
|
||||
async def test_withhold_withholds(monkeypatch):
|
||||
Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
|
||||
assert issuance["tier"] == "withheld" and issuance["claims"]["on_unsupported"] == "withhold"
|
||||
|
||||
@@ -209,23 +210,24 @@ async def test_an_uncited_claim_still_counts_under_judge(monkeypatch):
|
||||
|
||||
|
||||
async def test_the_issuance_event_carries_the_counts(monkeypatch):
|
||||
Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
run = await run_manager.start(graph=weekly(), input_payload={}, run_class="formal")
|
||||
row = await wait(run.id)
|
||||
event = await issuance_event(row.id)
|
||||
assert event["tier"] == "degraded"
|
||||
assert event["claims"] == {"policy": "judge", "uncited_claims": 0, "on_unsupported": "degrade",
|
||||
"mode": "inline", "counts": {"supported": 1, "partial": 0, "unsupported": 1,
|
||||
"mode": "inline", "counts": {"supported": 1, "partial": 0, "contradicted": 1,
|
||||
"insufficient": 0, "unsupported": 0,
|
||||
"not_a_claim": 0, "unjudged": 0}}
|
||||
|
||||
|
||||
async def test_a_governed_formal_run_grades_by_the_verdicts(client, monkeypatch):
|
||||
Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
graph = weekly(report={"numbers": "strict", "on_violation": "fail",
|
||||
"judge": {"max_cost_usd": None, "on_unsupported": "withhold"}})
|
||||
row = await governed(client, graph)
|
||||
issuance = row.output["_issuance"]
|
||||
assert issuance["tier"] == "withheld" and issuance["claims"]["unsupported"][0]["text"] == "增长主要来自新客首单。"
|
||||
assert issuance["tier"] == "withheld" and issuance["claims"]["contradicted"][0]["text"] == "增长主要来自新客首单。"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -234,7 +236,7 @@ async def test_a_governed_formal_run_grades_by_the_verdicts(client, monkeypatch)
|
||||
|
||||
|
||||
async def test_explore_runs_only_annotate(monkeypatch):
|
||||
models = Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}),
|
||||
run_class="exploratory")
|
||||
assert models.judges == [], "探索运行节点里不花这笔钱"
|
||||
@@ -242,7 +244,7 @@ async def test_explore_runs_only_annotate(monkeypatch):
|
||||
claims = issuance["claims"]
|
||||
assert claims["mode"] == "on_demand" and claims["complete"] is False
|
||||
assert claims["counts"]["unjudged"] == 2 and claims["unjudged"] == {"on_demand": 2}
|
||||
assert claims["unsupported"] == []
|
||||
assert claims["contradicted"] == [] and claims["insufficient"] == []
|
||||
|
||||
|
||||
async def test_explore_runs_still_count_uncited_claims(monkeypatch):
|
||||
@@ -257,14 +259,14 @@ async def test_explore_runs_still_count_uncited_claims(monkeypatch):
|
||||
|
||||
|
||||
async def test_the_contract_can_tighten_on_unsupported(monkeypatch):
|
||||
Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
issuance = await issued(weekly(contract={"claims": {"policy": "judge", "on_unsupported": "withhold"}}))
|
||||
assert issuance["tier"] == "withheld" and issuance["claims"]["on_unsupported"] == "withhold"
|
||||
assert not any("不支持" in g and "这一版" in g for g in issuance["gaps"])
|
||||
|
||||
|
||||
async def test_the_contract_cannot_loosen_on_unsupported(monkeypatch):
|
||||
Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}},
|
||||
contract={"claims": {"policy": "judge", "on_unsupported": "degrade"}}))
|
||||
assert issuance["tier"] == "withheld"
|
||||
@@ -295,7 +297,7 @@ async def test_runs_from_before_the_upgrade_ignore_judge(monkeypatch):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot)
|
||||
models = Models(monkeypatch, text=UNCITED, verdicts={"新客": "unsupported"})
|
||||
models = Models(monkeypatch, text=UNCITED, verdicts={"新客": "contradicted"})
|
||||
issuance = await issued(weekly())
|
||||
assert models.judges == [] and issuance["tier"] == "formal" and "claims" not in issuance
|
||||
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
数字、实体是确定性的,系统自己能核对;「增长主要来自华东」这种话说得对不对,只能交给另一个模型按
|
||||
证据判断。这是概率性的判断,所以每一步都要留余地、说实话:
|
||||
|
||||
- 一份报告合并成一次调用(超过每批的句数就分批),每句给 supported / partial / unsupported / not_a_claim
|
||||
- 一份报告合并成一次调用(超过每批的句数就分批),每句给 supported / partial / contradicted / insufficient /
|
||||
not_a_claim
|
||||
- 模型不支持结构化输出、或者给的不是合法结构时退回 JSON 解析,再失败就记「未裁判」,不编判定
|
||||
- 裁判模型关掉 thinking,max_tokens 4096
|
||||
- 预算(每份报告的句数、金额、时长,全局每日金额)每一项都能写 null 表示不限;触顶就停、已判的保留、
|
||||
@@ -183,7 +184,7 @@ def status_of(outcome, doc, text):
|
||||
|
||||
|
||||
async def test_one_call_for_the_whole_report_and_the_verdict_shape(monkeypatch, stored):
|
||||
judge = Judge(monkeypatch, {"u3": "unsupported"})
|
||||
judge = Judge(monkeypatch, {"u3": "contradicted"})
|
||||
catalog = make_catalog(stored)
|
||||
doc = compose_doc(RAW, catalog)
|
||||
outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED)
|
||||
@@ -192,7 +193,8 @@ async def test_one_call_for_the_whole_report_and_the_verdict_shape(monkeypatch,
|
||||
v = status_of(outcome, doc, CITED)
|
||||
assert v["status"] == "supported" and v["judge"] == PRICED and v["post_seal"] is False
|
||||
assert v["rationale"] and v["used"] == ["m:gmv"]
|
||||
assert outcome["counts"] == {"supported": 3, "partial": 0, "unsupported": 1, "not_a_claim": 0, "unjudged": 0}
|
||||
assert outcome["counts"] == {"supported": 3, "partial": 0, "contradicted": 1, "insufficient": 0, "not_a_claim": 0,
|
||||
"unjudged": 0, "unsupported": 0}
|
||||
assert outcome["model"] == PRICED and outcome["calls"] == 1 and outcome["gaps"] == []
|
||||
assert units_of(doc)["下面看细节。"] not in outcome["verdicts"], "连接性的话不送裁判"
|
||||
|
||||
@@ -226,7 +228,7 @@ async def test_structured_output_failure_falls_back_to_json(monkeypatch, stored)
|
||||
async def test_an_array_written_as_json_text_is_still_read(monkeypatch, stored):
|
||||
"""有的模型走工具调用时把嵌套的数组写成一段 JSON 文本({"items": "[…]"}):照样认,不为它再花一次调用。"""
|
||||
def stringified(units):
|
||||
items = [{"unit": u, "verdict": "unsupported", "rationale": "对不上", "used": []} for u in units]
|
||||
items = [{"unit": u, "verdict": "contradicted", "rationale": "对不上", "used": []} for u in units]
|
||||
return json.dumps({"items": json.dumps(items, ensure_ascii=False)}, ensure_ascii=False)
|
||||
|
||||
judge = Judge(monkeypatch, structured=stringified)
|
||||
@@ -234,7 +236,7 @@ async def test_an_array_written_as_json_text_is_still_read(monkeypatch, stored):
|
||||
doc = compose_doc(RAW, catalog)
|
||||
outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED)
|
||||
assert [c["kind"] for c in judge.calls] == ["structured"]
|
||||
assert {v["status"] for v in outcome["verdicts"].values()} == {"unsupported"}
|
||||
assert {v["status"] for v in outcome["verdicts"].values()} == {"contradicted"}
|
||||
|
||||
|
||||
async def test_a_broken_provider_is_not_called_batch_after_batch(monkeypatch, stored):
|
||||
@@ -266,18 +268,18 @@ async def test_bad_items_are_not_trusted(monkeypatch, stored):
|
||||
|
||||
def messy(units):
|
||||
return json.dumps({"items": [
|
||||
{"unit": ids[CITED], "verdict": "supported", "rationale": "长" * 100, "used": ["m:gmv", "m:nope", "Q9"]},
|
||||
{"unit": ids[CITED], "verdict": "supported", "rationale": "长" * 200, "used": ["m:gmv", "m:nope", "Q9"]},
|
||||
{"unit": ids[CITED2], "verdict": "maybe", "rationale": "不认识的判定", "used": []},
|
||||
{"unit": ids[CAUSAL], "verdict": "unsupported", "rationale": "没有新客维度", "used": []},
|
||||
{"unit": ids[CAUSAL], "verdict": "contradicted", "rationale": "没有新客维度", "used": []},
|
||||
{"unit": "u999", "verdict": "supported", "rationale": "不存在的句子", "used": []},
|
||||
]}, ensure_ascii=False)
|
||||
|
||||
Judge(monkeypatch, structured=messy)
|
||||
outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED)
|
||||
v = outcome["verdicts"]
|
||||
assert v[ids[CITED]]["rationale"] == "长" * RATIONALE_MAX and v[ids[CITED]]["used"] == ["m:gmv"]
|
||||
assert v[ids[CITED]]["rationale"] == "长" * (RATIONALE_MAX - 1) + "…" and v[ids[CITED]]["used"] == ["m:gmv"]
|
||||
assert (v[ids[CITED2]]["status"], v[ids[CITED2]]["reason"]) == ("unjudged", "format")
|
||||
assert v[ids[CAUSAL]]["status"] == "unsupported"
|
||||
assert v[ids[CAUSAL]]["status"] == "contradicted"
|
||||
assert (v[ids[NUMBER]]["status"], v[ids[NUMBER]]["reason"]) == ("unjudged", "missing")
|
||||
assert "u999" not in v
|
||||
|
||||
@@ -532,7 +534,7 @@ async def test_a_model_without_a_price_ignores_money_limits(monkeypatch, stored)
|
||||
|
||||
|
||||
async def test_known_sentences_are_not_judged_again(monkeypatch, stored):
|
||||
judge = Judge(monkeypatch, {"u2": "unsupported"})
|
||||
judge = Judge(monkeypatch, {"u2": "contradicted"})
|
||||
catalog = make_catalog(stored)
|
||||
doc = compose_doc(RAW, catalog)
|
||||
first = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED)
|
||||
|
||||
@@ -175,7 +175,7 @@ async def save_setting(key: str, value) -> None:
|
||||
|
||||
|
||||
async def test_an_on_demand_verdict_lands_after_the_seal_and_the_seal_still_holds(client, monkeypatch):
|
||||
models = Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
row = await finish(weekly(claims="judge"))
|
||||
assert row.status == "succeeded" and models.judges == [], "探索运行节点里不判"
|
||||
doc = doc_of(row)
|
||||
@@ -185,7 +185,7 @@ async def test_an_on_demand_verdict_lands_after_the_seal_and_the_seal_still_hold
|
||||
assert res.status_code == 200, res.text
|
||||
body = res.json()
|
||||
verdict = body["verdicts"][ids[CAUSAL]]
|
||||
assert verdict["status"] == "unsupported" and verdict["post_seal"] is True and verdict["judge"] == PRICED
|
||||
assert verdict["status"] == "contradicted" and verdict["post_seal"] is True and verdict["judge"] == PRICED
|
||||
assert verdict["rationale"] and body["judged"] == [ids[CAUSAL]] and body["reused"] == []
|
||||
assert body["report"] == {"node_id": "write", "doc_artifact": row.output["_evidence"]["doc_artifact"]}
|
||||
assert body["post_seal"] is True and body["limited"] is False and body["calls"] == 1
|
||||
@@ -276,9 +276,9 @@ async def test_a_failed_attempt_can_be_retried(client, monkeypatch):
|
||||
failed = (await judge(client, row.id, [uid])).json()
|
||||
assert failed["verdicts"][uid]["status"] == "unjudged" and failed["verdicts"][uid]["reason"] == "error"
|
||||
assert "网关 502" in failed["message"] and failed["limited"] is False
|
||||
models = Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
retried = (await judge(client, row.id, [uid])).json()
|
||||
assert retried["verdicts"][uid]["status"] == "unsupported" and retried["reused"] == [] and len(models.judges) == 1
|
||||
assert retried["verdicts"][uid]["status"] == "contradicted" and retried["reused"] == [] and len(models.judges) == 1
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -414,7 +414,7 @@ async def test_a_broken_seal_is_not_judged(client, monkeypatch):
|
||||
async def test_a_run_continued_during_the_judge_call_records_no_event(client, monkeypatch, meanwhile):
|
||||
"""判定只接在点击开始时的那份封存后面。运行在裁判期间被接着跑了,再追加就落进一次活着的运行中间,
|
||||
还会被封进下一份清单:不记进运行记录,判定照样交回给点的人,并说明为什么没记。"""
|
||||
models = Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
row = await finish(weekly(claims="judge"))
|
||||
uid = unit_ids(doc_of(row))[CAUSAL]
|
||||
ask_model = judge_mod.get_chat_model
|
||||
@@ -430,7 +430,7 @@ async def test_a_run_continued_during_the_judge_call_records_no_event(client, mo
|
||||
assert res.status_code == 200, res.text
|
||||
body = res.json()
|
||||
assert len(models.judges) == 1 and body["calls"] == 1, "模型问过了"
|
||||
assert body["verdicts"][uid]["status"] == "unsupported" and body["judged"] == [uid], "判定照样交回"
|
||||
assert body["verdicts"][uid]["status"] == "contradicted" and body["judged"] == [uid], "判定照样交回"
|
||||
assert body["event"] is None and "运行已继续执行" in body["message"] and "未写入运行记录" in body["message"]
|
||||
assert await judged_events(row.id) == []
|
||||
async with SessionLocal() as session:
|
||||
@@ -444,7 +444,7 @@ async def test_a_continue_that_wins_the_row_between_check_and_write_gets_no_even
|
||||
from app.core.bus import bus
|
||||
from app.core.config import settings
|
||||
|
||||
Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
row = await finish(weekly(claims="judge"))
|
||||
uid = unit_ids(doc_of(row))[CAUSAL]
|
||||
set_seq = bus.set_seq
|
||||
@@ -462,7 +462,7 @@ async def test_a_continue_that_wins_the_row_between_check_and_write_gets_no_even
|
||||
|
||||
monkeypatch.setattr(bus, "set_seq", claimed_in_between)
|
||||
body = (await judge(client, row.id, [uid])).json()
|
||||
assert body["verdicts"][uid]["status"] == "unsupported" and body["event"] is None
|
||||
assert body["verdicts"][uid]["status"] == "contradicted" and body["event"] is None
|
||||
assert "未写入运行记录" in body["message"] and await judged_events(row.id) == []
|
||||
async with SessionLocal() as session:
|
||||
after = await session.get(Run, row.id)
|
||||
@@ -503,7 +503,7 @@ async def test_sentences_that_are_not_claims_are_skipped(client, monkeypatch):
|
||||
|
||||
|
||||
async def test_the_segment_shows_the_verdict_and_whether_it_can_still_be_asked(client, monkeypatch):
|
||||
Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
row = await finish(weekly(claims="judge"))
|
||||
doc = doc_of(row)
|
||||
ids = unit_ids(doc)
|
||||
@@ -513,7 +513,7 @@ async def test_the_segment_shows_the_verdict_and_whether_it_can_still_be_asked(c
|
||||
|
||||
await judge(client, row.id, [ids[CAUSAL]])
|
||||
after = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, ids[CAUSAL])}")).json()
|
||||
assert after["unit"]["verdict"]["status"] == "unsupported" and after["unit"]["verdict"]["post_seal"] is True
|
||||
assert after["unit"]["verdict"]["status"] == "contradicted" and after["unit"]["verdict"]["post_seal"] is True
|
||||
assert after["unit"]["on_demand"]["available"] is False and after["unit"]["on_demand"]["reason"] == "judged"
|
||||
assert "模型" in after["note"] and "后续版本" not in after["note"]
|
||||
other = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, ids[CITED])}")).json()
|
||||
@@ -531,11 +531,11 @@ async def test_the_segment_shows_the_verdict_and_whether_it_can_still_be_asked(c
|
||||
|
||||
|
||||
async def test_sealed_verdicts_of_a_formal_run_are_shown_as_sealed(client, monkeypatch):
|
||||
Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
row = await finish(weekly(claims="judge", judge={"max_cost_usd": 0.05}), run_class="formal")
|
||||
doc = doc_of(row)
|
||||
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, unit_ids(doc)[CAUSAL])}")).json()
|
||||
assert body["unit"]["verdict"]["status"] == "unsupported" and body["unit"]["verdict"]["post_seal"] is False
|
||||
assert body["unit"]["verdict"]["status"] == "contradicted" and body["unit"]["verdict"]["post_seal"] is False
|
||||
assert body["unit"]["on_demand"]["reason"] == "formal"
|
||||
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
|
||||
assert graph["reports"][0]["post_seal_verdicts"] == {} and graph["reports"][0]["judge"]["mode"] == "inline"
|
||||
@@ -543,7 +543,7 @@ async def test_sealed_verdicts_of_a_formal_run_are_shown_as_sealed(client, monke
|
||||
|
||||
async def test_a_forged_event_cannot_overwrite_a_sealed_verdict(client, monkeypatch):
|
||||
"""封存后追加的判定只填补没判过的句子:封存内判过的,后面追加什么都盖不掉。"""
|
||||
Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
row = await finish(weekly(claims="judge", judge={"max_cost_usd": 0.05}), run_class="formal")
|
||||
doc = doc_of(row)
|
||||
uid = unit_ids(doc)[CAUSAL]
|
||||
@@ -552,7 +552,7 @@ async def test_a_forged_event_cannot_overwrite_a_sealed_verdict(client, monkeypa
|
||||
doc_artifact=doc_artifact, post_seal=True,
|
||||
verdicts={uid: {"status": "supported", "rationale": "改好了", "judge": "x", "post_seal": True}})
|
||||
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, uid)}")).json()
|
||||
assert body["unit"]["verdict"]["status"] == "unsupported"
|
||||
assert body["unit"]["verdict"]["status"] == "contradicted"
|
||||
|
||||
|
||||
async def test_appended_verdicts_always_read_as_post_seal(client, monkeypatch):
|
||||
|
||||
@@ -249,7 +249,7 @@ WITHHOLD = {**JUDGED, "judge": {"max_cost_usd": 0.05, "on_unsupported": "withhol
|
||||
])
|
||||
def test_relaxing_on_unsupported_on_the_node_is_loosening(judge, dropped):
|
||||
[reason] = _changed(WITHHOLD, {**WITHHOLD, "judge": judge})
|
||||
assert "「报告撰写」" in reason and "「结论句裁判 · 证据不支持时」" in reason
|
||||
assert "「报告撰写」" in reason and "「结论句裁判 · 证据相矛盾时」" in reason
|
||||
assert "「不予出具」" in reason and "「出具降档」" in reason
|
||||
assert ("删除了已设置的值" in reason) is dropped, reason
|
||||
|
||||
@@ -258,7 +258,7 @@ def test_replacing_the_whole_judge_block_to_add_a_budget_is_loosening():
|
||||
"""补预算把 judge 整个换成 {max_cost_usd}:连同裁判模型、withhold 一起丢了。"""
|
||||
before = {**STRICT, "claims": "judge", "judge": {"model": "judge-model", "on_unsupported": "withhold"}}
|
||||
[reason] = _changed(before, {**before, "judge": {"max_cost_usd": 0.05}})
|
||||
assert "「结论句裁判 · 证据不支持时」" in reason and "「不予出具」" in reason
|
||||
assert "「结论句裁判 · 证据相矛盾时」" in reason and "「不予出具」" in reason
|
||||
assert _changed(before, {**before, "judge": {**before["judge"], "max_cost_usd": 0.05}}) == [], \
|
||||
"带着原来的键补预算不算放宽"
|
||||
|
||||
@@ -268,7 +268,7 @@ def test_a_withhold_from_the_graph_defaults_lost_by_a_node_judge_is_loosening():
|
||||
defaults = {"judge": {"on_unsupported": "withhold", "max_cost_usd": 0.05}}
|
||||
report = {**STRICT, "claims": "judge"}
|
||||
[reason] = _changed(report, {**report, "judge": {"max_cost_usd": 0.05}}, defaults=defaults)
|
||||
assert "「结论句裁判 · 证据不支持时」" in reason and "从「不予出具」(跟随工作流默认设置)" in reason, reason
|
||||
assert "「结论句裁判 · 证据相矛盾时」" in reason and "从「不予出具」(跟随工作流默认设置)" in reason, reason
|
||||
assert "不再跟随工作流默认设置" in reason, "说清楚为什么全图默认里的 withhold 不生效了"
|
||||
assert _changed(report, {**report, "judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}},
|
||||
defaults=defaults) == []
|
||||
@@ -280,7 +280,7 @@ def test_relaxing_on_unsupported_in_the_graph_defaults_is_loosening():
|
||||
new = copy.deepcopy(old)
|
||||
new["defaults"]["judge"]["on_unsupported"] = "degrade"
|
||||
[reason] = forbidden_changes(old, new)
|
||||
assert "「报告撰写」" in reason and "「结论句裁判 · 证据不支持时」" in reason
|
||||
assert "「报告撰写」" in reason and "「结论句裁判 · 证据相矛盾时」" in reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("before, after", [
|
||||
@@ -297,7 +297,7 @@ def test_tightening_or_keeping_on_unsupported_is_fine(before, after):
|
||||
def test_dropping_withhold_along_with_judge_is_one_reason():
|
||||
"""从 judge 退回 require_citation:claims 那一条已经说了,不再为 on_unsupported 另记一条。"""
|
||||
[reason] = _changed(WITHHOLD, {**STRICT, "judge": {"max_cost_usd": 0.05}})
|
||||
assert "「未附依据的结论句」" in reason and "证据不支持时" not in reason
|
||||
assert "「未附依据的结论句」" in reason and "证据相矛盾时" not in reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize("before", ["require_citation", "judge"])
|
||||
|
||||
@@ -0,0 +1,486 @@
|
||||
"""结论句裁判改造(JR)的接口:片段接口的字段清单、按需裁判的模型标记、证据图的 judge_meta、审计行。
|
||||
|
||||
- 片段接口里表名实体那一步带上字段清单 fields: [{name, type}](封存范围内的表结构快照,最多 60 个,
|
||||
多出来的记 fields_more),遮罩的列不列
|
||||
- 按需裁判也判断裁判模型和写作模型是否相同(写作模型取这份报告撰写节点实际用的模型),并带上
|
||||
priced、writer_model;evidence.judged 事件同样记下
|
||||
- GET /evidence 每份报告带 judge_meta:最近一次按需裁判的 {model, same_model, priced, writer_model}
|
||||
- 交给裁判的新摘录(字段清单、指标输入的 SQL)只经封存范围内的 loader 读
|
||||
- 审计行的 issue 新增 contradicted_claim、insufficient_claim
|
||||
- 旧的 unsupported 判定照旧算「判过」:封存之后追加的判定盖不掉它
|
||||
|
||||
所有裁判都用 mock 模型给剧本,不调真接口。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from langchain_core.messages import AIMessage
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.core.events import EventType
|
||||
from app.data.engine import engines
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource, Run, RunEvent, Setting
|
||||
from app.engine import judge as judge_mod
|
||||
from app.engine.evidence import iter_segments, iter_units
|
||||
from app.engine.judge import JUDGE_ROLE, SPEND_KEY
|
||||
from app.engine.runner import run_manager
|
||||
from app.main import app
|
||||
from app.providers.mock_model import MockChatModel
|
||||
|
||||
PRICED = "claude-sonnet-5"
|
||||
WRITER = "mock-fast"
|
||||
SOURCE = "jr_shop"
|
||||
TOOL = f"db_query__{SOURCE}"
|
||||
SQL = "SELECT region, SUM(amount) AS gmv FROM orders GROUP BY region ORDER BY gmv DESC"
|
||||
WIDE_COLS = [f"f{i:02d}" for i in range(65)]
|
||||
METRIC = "东区销售额最高,达 [[m:gmv]]。[[see:m:gmv]]"
|
||||
METHOD = "这个数由 [[t:orders]] 按 [[c:orders.region]] 分组求和得出。"
|
||||
WIDE = "明细在 [[t:wide_orders]] 里。"
|
||||
TEXT = METRIC + METHOD + WIDE
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def engine_up():
|
||||
await run_manager.setup()
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"])))
|
||||
await session.commit()
|
||||
yield
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def shop(tmp_path):
|
||||
"""探查过结构的示例库:orders(带手机号列,数据源设了遮罩)和一张 65 列的宽表。"""
|
||||
from app.data.introspect import introspect
|
||||
|
||||
path = tmp_path / "jr_shop.db"
|
||||
db = sqlite3.connect(path)
|
||||
db.executescript(
|
||||
"CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, amount REAL, phone TEXT);"
|
||||
f"CREATE TABLE wide_orders ({', '.join(f'{c} TEXT' for c in WIDE_COLS)});")
|
||||
db.executemany("INSERT INTO orders VALUES (?, ?, ?, ?)",
|
||||
[(1, "east", 300.5, "p1"), (2, "west", 200.0, "p2"), (3, "east", 100.0, "p3")])
|
||||
db.commit()
|
||||
db.close()
|
||||
probe = SimpleNamespace(id=f"probe-{path.name}", name=SOURCE, kind="sqlite", database=str(path), host=None,
|
||||
port=None, username=None, password=None, options={}, readonly=True, schema_cache={})
|
||||
cache = await introspect(probe)
|
||||
await engines.invalidate(probe.id)
|
||||
async with SessionLocal() as session:
|
||||
row = (await session.execute(select(DataSource).where(DataSource.name == SOURCE))).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = DataSource(name=SOURCE, kind="sqlite", database=str(path), readonly=True, enabled=True)
|
||||
session.add(row)
|
||||
row.database, row.kind, row.enabled, row.readonly, row.schema_cache = str(path), "sqlite", True, True, cache
|
||||
row.options = {"mask_columns": "phone"}
|
||||
await session.commit()
|
||||
source_id = row.id
|
||||
await engines.invalidate(source_id)
|
||||
yield source_id
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||||
|
||||
|
||||
def weekly(**report) -> dict:
|
||||
nodes = [
|
||||
node("start", "input"),
|
||||
node("fetch", "tool", tool=TOOL, args={"sql": SQL}),
|
||||
node("card", "metrics", caliber="周报口径", caliber_version="v2", metrics=[
|
||||
{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "cell(nodes.fetch, 0, 'gmv')"}]),
|
||||
node("write", "report", instructions="写周报", on_violation="flag", claims="judge", **report),
|
||||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
|
||||
]
|
||||
ids = [n["id"] for n in nodes]
|
||||
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
|
||||
|
||||
|
||||
class Models:
|
||||
"""写作者回 text;裁判按句子里的字给判定:verdicts = {句中片段: (判定, missing)}。judges 记下请求原文。"""
|
||||
|
||||
def __init__(self, monkeypatch, verdicts=None, *, model=PRICED, text=TEXT):
|
||||
self.judges: list[str] = []
|
||||
verdicts = verdicts or {}
|
||||
|
||||
def decide(chat, messages):
|
||||
if JUDGE_ROLE in str(messages[0].content):
|
||||
request = str(messages[-1].content)
|
||||
self.judges.append(request)
|
||||
items = []
|
||||
for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M):
|
||||
status, missing = next((v for frag, v in verdicts.items() if frag in line), ("supported", ""))
|
||||
items.append({"unit": uid, "verdict": status, "rationale": f"判的是:{line.strip()[:12]}",
|
||||
"missing": missing, "used": []})
|
||||
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
|
||||
return AIMessage(content=text)
|
||||
|
||||
monkeypatch.setattr(MockChatModel, "_decide", decide)
|
||||
self.model = model
|
||||
|
||||
async def judge_model_of(session, spec):
|
||||
return MockChatModel(model_name=self.model), self.model
|
||||
|
||||
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
|
||||
|
||||
|
||||
async def wait(run_id: str) -> Run:
|
||||
for _ in range(400):
|
||||
await asyncio.sleep(0.05)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run_id)
|
||||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||||
return row
|
||||
raise AssertionError(f"run {run_id} 没跑完:{row.status}")
|
||||
|
||||
|
||||
async def explore_run(graph=None) -> tuple[Run, dict]:
|
||||
started = await run_manager.start(graph=graph or weekly(), input_payload={}, run_class="exploratory")
|
||||
row = await wait(started.id)
|
||||
assert row.status == "succeeded", row.error
|
||||
return row, artifact_store.load(row.output["_evidence"]["doc_artifact"])
|
||||
|
||||
|
||||
def unit_of(doc: dict, prefix: str) -> str:
|
||||
md = doc["markdown"]
|
||||
return next(u["id"] for _, u in iter_units(doc) if md[u["span"][0]:u["span"][1]].strip().startswith(prefix))
|
||||
|
||||
|
||||
async def judged_events(run_id: str) -> list[RunEvent]:
|
||||
async with SessionLocal() as session:
|
||||
return list((await session.execute(select(RunEvent).where(
|
||||
RunEvent.run_id == run_id, RunEvent.type == EventType.EVIDENCE_JUDGED).order_by(RunEvent.seq))).scalars())
|
||||
|
||||
|
||||
async def ask(client, run_id: str, units: list[str]) -> dict:
|
||||
res = await client.post(f"/api/runs/{run_id}/evidence/judge", json={"units": units})
|
||||
assert res.status_code == 200, res.text
|
||||
return res.json()
|
||||
|
||||
|
||||
async def reseal(run_id: str) -> None:
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, run_id)
|
||||
rows = [tuple(r) for r in await session.execute(
|
||||
select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data)
|
||||
.where(RunEvent.run_id == run_id, RunEvent.seq <= run.manifest_seq).order_by(RunEvent.seq))]
|
||||
run.manifest_hash = artifact_store.manifest_hash(rows)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def rewrite(run_id: str, etype: str, node_id: str, change) -> None:
|
||||
async with SessionLocal() as session:
|
||||
for row in (await session.execute(select(RunEvent).where(
|
||||
RunEvent.run_id == run_id, RunEvent.type == etype, RunEvent.node_id == node_id))).scalars():
|
||||
row.data = change(dict(row.data))
|
||||
await session.commit()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 片段接口:表名实体步骤的字段清单
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def open_entity(client, run_id: str, doc: dict, ref: str) -> dict:
|
||||
seg = next(s for s in iter_segments(doc) if s.get("kind") == "entity" and s.get("ref") == ref)
|
||||
res = await client.get(f"/api/runs/{run_id}/evidence/segments/{seg['id']}")
|
||||
assert res.status_code == 200, res.text
|
||||
[step] = res.json()["chain"]
|
||||
return step
|
||||
|
||||
|
||||
async def test_a_table_step_lists_its_fields_without_masked_columns(client, monkeypatch):
|
||||
Models(monkeypatch)
|
||||
row, doc = await explore_run()
|
||||
step = await open_entity(client, row.id, doc, "t:orders")
|
||||
assert step["step"] == "entity" and step["kind"] == "table"
|
||||
assert step["fields"] == [{"name": "id", "type": "INTEGER"}, {"name": "region", "type": "TEXT"},
|
||||
{"name": "amount", "type": "REAL"}], "遮罩的 phone 不列"
|
||||
assert "fields_more" not in step and step["columns"] == 4
|
||||
|
||||
|
||||
async def test_a_wide_table_step_stops_at_sixty_fields(client, monkeypatch):
|
||||
Models(monkeypatch)
|
||||
row, doc = await explore_run()
|
||||
step = await open_entity(client, row.id, doc, "t:wide_orders")
|
||||
assert [f["name"] for f in step["fields"]] == WIDE_COLS[:60]
|
||||
assert step["fields_more"] == 5
|
||||
|
||||
|
||||
async def test_a_table_step_outside_the_seal_has_no_fields(client, monkeypatch):
|
||||
Models(monkeypatch)
|
||||
row, doc = await explore_run()
|
||||
await rewrite(row.id, "tool.end", "fetch", lambda d: {k: v for k, v in d.items() if k != "schema_artifact"})
|
||||
await rewrite(row.id, "node.finished", "fetch",
|
||||
lambda d: {**d, "evidence": [e for e in d.get("evidence") or [] if e.get("kind") != "schema"]})
|
||||
await reseal(row.id)
|
||||
step = await open_entity(client, row.id, doc, "t:orders")
|
||||
assert "fields" not in step and "fields_more" not in step
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 按需裁判:新的判定、摘录、模型标记
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_on_demand_judging_sees_fields_and_metric_sql_and_reports_the_models(client, monkeypatch):
|
||||
models = Models(monkeypatch, verdicts={"分组求和": ("insufficient", "字段清单"), "东区销售额": ("contradicted", "")})
|
||||
row, doc = await explore_run()
|
||||
method, metric = unit_of(doc, "这个数由"), unit_of(doc, "东区销售额")
|
||||
|
||||
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
|
||||
assert graph["reports"][0]["judge_meta"] is None, "还没按需裁判过"
|
||||
|
||||
body = await ask(client, row.id, [method, metric])
|
||||
request = models.judges[0]
|
||||
assert "region TEXT" in request and "id INTEGER" in request, "表附上字段清单和类型"
|
||||
assert "phone" not in request.lower(), "遮罩的列不出现"
|
||||
assert "查询 Q1 的 SQL" in request and "SUM(amount)" in request and "r0" in request, "指标附上输入的查询和格子"
|
||||
|
||||
assert body["verdicts"][method]["status"] == "insufficient" and body["verdicts"][method]["missing"] == "字段清单"
|
||||
assert body["verdicts"][metric]["status"] == "contradicted"
|
||||
assert body["same_model"] is False and body["priced"] is True and body["writer_model"] == WRITER
|
||||
|
||||
[event] = await judged_events(row.id)
|
||||
assert event.data["same_model"] is False and event.data["priced"] is True
|
||||
assert event.data["writer_model"] == WRITER
|
||||
|
||||
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
|
||||
assert graph["reports"][0]["judge_meta"] == {"model": PRICED, "same_model": False, "priced": True,
|
||||
"writer_model": WRITER}
|
||||
|
||||
|
||||
async def test_judging_with_the_writer_model_is_flagged(client, monkeypatch):
|
||||
models = Models(monkeypatch, model=WRITER)
|
||||
row, doc = await explore_run()
|
||||
body = await ask(client, row.id, [unit_of(doc, "这个数由")])
|
||||
assert models.judges and body["model"] == WRITER
|
||||
assert body["same_model"] is True and body["writer_model"] == WRITER, "判定照常出,另带标记"
|
||||
assert body["verdicts"][unit_of(doc, "这个数由")]["status"] == "supported"
|
||||
[event] = await judged_events(row.id)
|
||||
assert event.data["same_model"] is True and event.data["writer_model"] == WRITER
|
||||
|
||||
# judge_meta 取最近一次:换一个价格目录里没有的模型再判一句,就按新的这次
|
||||
models.model = "house-judge-model"
|
||||
body = await ask(client, row.id, [unit_of(doc, "东区销售额")])
|
||||
assert body["same_model"] is False and body["priced"] is False, "目录里没有价格的模型:金额上限对它不生效"
|
||||
meta = (await client.get(f"/api/runs/{row.id}/evidence")).json()["reports"][0]["judge_meta"]
|
||||
assert meta == {"model": "house-judge-model", "same_model": False, "priced": False, "writer_model": WRITER}
|
||||
|
||||
|
||||
async def test_the_new_excerpts_only_read_sealed_artifacts(client, monkeypatch):
|
||||
"""表结构快照、查询快照被挪出封存范围(事件里追不到了):字段清单、指标输入的 SQL 都不进裁判的摘录。"""
|
||||
models = Models(monkeypatch)
|
||||
row, doc = await explore_run()
|
||||
await rewrite(row.id, "tool.end", "fetch",
|
||||
lambda d: {k: v for k, v in d.items() if k not in ("schema_artifact", "query_artifact")})
|
||||
await rewrite(row.id, "node.finished", "fetch",
|
||||
lambda d: {**d, "evidence": [e for e in d.get("evidence") or []
|
||||
if e.get("kind") not in ("schema", "query")]})
|
||||
await reseal(row.id)
|
||||
await ask(client, row.id, [unit_of(doc, "这个数由"), unit_of(doc, "东区销售额")])
|
||||
request = models.judges[0]
|
||||
assert "INTEGER" not in request and "region TEXT" not in request
|
||||
assert "SELECT" not in request
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 审计行
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_audit_rows_list_contradicted_and_insufficient_claims(client, monkeypatch):
|
||||
Models(monkeypatch, verdicts={"分组求和": ("insufficient", "字段清单"), "东区销售额": ("contradicted", "")})
|
||||
row, doc = await explore_run()
|
||||
method, metric = unit_of(doc, "这个数由"), unit_of(doc, "东区销售额")
|
||||
await ask(client, row.id, [method, metric])
|
||||
|
||||
body = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json()
|
||||
rows = [r for g in body["groups"] for r in g["rows"] if r["kind"] == "claim"]
|
||||
by_issue = {r["issue"]: r for r in rows}
|
||||
assert by_issue["contradicted_claim"]["unit"] == metric and by_issue["contradicted_claim"]["group"] == "none"
|
||||
assert by_issue["insufficient_claim"]["unit"] == method
|
||||
assert "字段清单" in by_issue["insufficient_claim"]["note"]
|
||||
assert by_issue["insufficient_claim"]["verdict"]["missing"] == "字段清单"
|
||||
assert by_issue["contradicted_claim"]["verdict"]["post_seal"] is True
|
||||
|
||||
text = (await client.get(f"/api/runs/{row.id}/evidence/audit?format=csv")).text
|
||||
issues = {line[8] for line in csv.reader(io.StringIO(text.lstrip("")))}
|
||||
assert {"证据相矛盾", "证据不足"} <= issues
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 旧的 unsupported:照旧算判过
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_an_old_unsupported_verdict_still_counts_as_judged():
|
||||
from app.api.evidence import _effective_verdicts, _on_demand, _Report, _Sealed
|
||||
from app.engine.evidence import compose_doc
|
||||
|
||||
doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r1")
|
||||
[unit] = [u for _, u in iter_units(doc)]
|
||||
unit["verdict"] = {"status": "unsupported", "rationale": "证据不支持", "judge": PRICED, "post_seal": False,
|
||||
"used": []}
|
||||
report = _Report(node_id="write", doc_artifact="d" * 64, ok=True, repairs=0, stats={}, doc=doc, hash_ok=True,
|
||||
fields=[], claims="judge")
|
||||
later = {"report": "write", "doc_artifact": "d" * 64,
|
||||
"verdicts": {unit["id"]: {"status": "supported", "rationale": "有依据", "judge": PRICED}}}
|
||||
sealed = _Sealed(run_id="r1", graph={}, seal={"sealed": True, "ok": True, "manifest_seq": 5, "legacy": False},
|
||||
events=[], judged=[(9, "write", later)], run_class="exploratory", status="succeeded",
|
||||
ledger_on=True)
|
||||
effective, appended = _effective_verdicts(report, sealed)
|
||||
assert effective[unit["id"]]["status"] == "unsupported" and appended == {}, "追加的判定盖不掉封存的判定"
|
||||
assert _on_demand(sealed, doc, unit["id"], effective[unit["id"]])["reason"] == "judged"
|
||||
|
||||
|
||||
def test_the_writer_model_of_an_old_report_comes_from_the_sealed_node_output():
|
||||
"""改造前的文档裁判摘要里没有 writer_model、改造前的事件里没有 same_model:按报告撰写节点封存的产出里
|
||||
实际用的模型补上。"""
|
||||
from app.api.evidence import _judge_meta, _Report, _Sealed, _writer_model
|
||||
from app.engine.evidence import compose_doc
|
||||
|
||||
text = artifact_store.canonical_json({"text": "增长主要来自新客首单。", "model": "writer-old"})
|
||||
output = artifact_store.content_hash(text)
|
||||
path = artifact_store._path_of(output)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r2")
|
||||
doc["judge"] = {"mode": "on_demand", "model": None, "counts": {}}
|
||||
report = _Report(node_id="write", doc_artifact="e" * 64, ok=True, repairs=0, stats={}, doc=doc, hash_ok=True,
|
||||
fields=[], claims="judge")
|
||||
old_event = {"report": "write", "doc_artifact": "e" * 64, "model": "writer-old", "priced": False,
|
||||
"verdicts": {}}
|
||||
sealed = _Sealed(run_id="r2", graph={}, seal={"sealed": True, "ok": True, "manifest_seq": 5, "legacy": False},
|
||||
events=[(3, "node.finished", "write", {"artifact": output})], outputs={"write": output},
|
||||
judged=[(9, "write", old_event)], run_class="exploratory", status="succeeded", ledger_on=True)
|
||||
assert _writer_model(sealed, report) == "writer-old"
|
||||
assert _judge_meta(report, sealed) == {"model": "writer-old", "same_model": True, "priced": False,
|
||||
"writer_model": "writer-old"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 按需裁判:早于当前规则的判定可以重判,当前规则下判过的照旧复用
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def on_demand_of(verdict: dict) -> dict:
|
||||
from app.api.evidence import _on_demand, _Sealed
|
||||
from app.engine.evidence import compose_doc
|
||||
|
||||
doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r3")
|
||||
[unit] = [u for _, u in iter_units(doc)]
|
||||
sealed = _Sealed(run_id="r3", graph={}, seal={"sealed": True, "ok": True, "manifest_seq": 5, "legacy": False},
|
||||
events=[], run_class="exploratory", status="succeeded", ledger_on=True)
|
||||
return _on_demand(sealed, doc, unit["id"], verdict)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("verdict", [
|
||||
{"status": "supported", "rationale": "有依据", "post_seal": True},
|
||||
{"status": "unsupported", "rationale": "证据不支持", "post_seal": True},
|
||||
{"status": "insufficient", "rationale": "缺 SQL", "post_seal": True, "rule_version": 1},
|
||||
])
|
||||
def test_an_outdated_appended_verdict_can_be_asked_again(verdict):
|
||||
got = on_demand_of(verdict)
|
||||
assert got["available"] is True and got["reason"] == "outdated" and got["message"]
|
||||
|
||||
|
||||
def test_a_current_verdict_is_not_asked_again():
|
||||
got = on_demand_of({"status": "contradicted", "rationale": "对不上", "post_seal": True,
|
||||
"rule_version": judge_mod.RULES_VERSION})
|
||||
assert got["available"] is False and got["reason"] == "judged"
|
||||
|
||||
|
||||
def test_a_sealed_verdict_is_never_reopened_by_version():
|
||||
"""封存的判定(文档里的,post_seal 为 False)盖不掉:版本再旧也不开放重判。"""
|
||||
got = on_demand_of({"status": "unsupported", "rationale": "证据不支持", "post_seal": False})
|
||||
assert got["available"] is False and got["reason"] == "judged"
|
||||
|
||||
|
||||
async def age(run_id: str, uid: str, **changes) -> None:
|
||||
"""把封存之后追加的那条判定改成改造前的样子(封存之后的事件不在封存核对范围里)。"""
|
||||
async with SessionLocal() as session:
|
||||
for row in (await session.execute(select(RunEvent).where(
|
||||
RunEvent.run_id == run_id, RunEvent.type == EventType.EVIDENCE_JUDGED))).scalars():
|
||||
data = json.loads(json.dumps(row.data))
|
||||
if uid in data.get("verdicts", {}):
|
||||
verdict = {k: v for k, v in data["verdicts"][uid].items() if k != "rule_version"}
|
||||
data["verdicts"][uid] = {**verdict, **changes}
|
||||
row.data = data
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def segment_of(client, run_id: str, doc: dict, uid: str) -> dict:
|
||||
seg = next(s["id"] for _, u in iter_units(doc) if u["id"] == uid for s in u["segments"])
|
||||
res = await client.get(f"/api/runs/{run_id}/evidence/segments/{seg}")
|
||||
assert res.status_code == 200, res.text
|
||||
return res.json()["unit"]
|
||||
|
||||
|
||||
async def test_an_outdated_verdict_is_really_judged_again(client, monkeypatch):
|
||||
models = Models(monkeypatch, verdicts={"东区销售额": ("contradicted", "")})
|
||||
row, doc = await explore_run()
|
||||
uid = unit_of(doc, "东区销售额")
|
||||
first = await ask(client, row.id, [uid])
|
||||
assert first["judged"] == [uid] and len(models.judges) == 1
|
||||
assert first["verdicts"][uid]["rule_version"] == judge_mod.RULES_VERSION
|
||||
|
||||
# 当前规则下判过的:照旧复用,不再调用
|
||||
again = await ask(client, row.id, [uid])
|
||||
assert again["reused"] == [uid] and again["judged"] == [] and len(models.judges) == 1
|
||||
assert (await segment_of(client, row.id, doc, uid))["on_demand"]["reason"] == "judged"
|
||||
|
||||
# 改造前判的(旧取值、没有规则版本):按钮照常出现,POST 真的重判一次,新判定追加在封存之后
|
||||
await age(row.id, uid, status="unsupported")
|
||||
unit = await segment_of(client, row.id, doc, uid)
|
||||
assert unit["verdict"]["status"] == "unsupported"
|
||||
assert unit["on_demand"]["available"] is True and unit["on_demand"]["reason"] == "outdated"
|
||||
redo = await ask(client, row.id, [uid])
|
||||
assert len(models.judges) == 2 and redo["judged"] == [uid] and redo["reused"] == []
|
||||
assert redo["verdicts"][uid]["status"] == "contradicted" and redo["event"] is not None
|
||||
assert len(await judged_events(row.id)) == 2
|
||||
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
|
||||
latest = graph["reports"][0]["post_seal_verdicts"][uid]
|
||||
assert latest["status"] == "contradicted" and latest["rule_version"] == judge_mod.RULES_VERSION
|
||||
assert (await segment_of(client, row.id, doc, uid))["on_demand"]["reason"] == "judged"
|
||||
|
||||
|
||||
async def test_a_failed_rejudge_keeps_the_older_verdict_in_force(client, monkeypatch):
|
||||
"""重判没判成(模型不可用):旧判定照旧生效,答复和证据图说的一样;按钮还在,可以再点。"""
|
||||
models = Models(monkeypatch)
|
||||
row, doc = await explore_run()
|
||||
uid = unit_of(doc, "东区销售额")
|
||||
await ask(client, row.id, [uid])
|
||||
await age(row.id, uid)
|
||||
|
||||
async def broken(session, spec):
|
||||
raise RuntimeError("网关 502")
|
||||
|
||||
monkeypatch.setattr(judge_mod, "get_chat_model", broken)
|
||||
body = await ask(client, row.id, [uid])
|
||||
assert body["judged"] == [] and "网关 502" in body["message"]
|
||||
assert body["verdicts"][uid]["status"] == "supported" and "rule_version" not in body["verdicts"][uid]
|
||||
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
|
||||
assert graph["reports"][0]["post_seal_verdicts"][uid] == body["verdicts"][uid]
|
||||
assert (await segment_of(client, row.id, doc, uid))["on_demand"]["reason"] == "outdated"
|
||||
assert len(models.judges) == 1
|
||||
@@ -0,0 +1,455 @@
|
||||
"""结论句裁判改造(JR)的内核:判定拆档、补全证据摘录、理由不硬截断。
|
||||
|
||||
- 判定拆成 contradicted(证据相矛盾)和 insufficient(证据不足):只有证据和句子冲突才判 contradicted,
|
||||
证据里没有相关信息判 insufficient,并写明缺的是什么(missing)。新判的不再产出 unsupported:
|
||||
模型照旧写 unsupported 的,按更严的 contradicted 收下
|
||||
- 讲方法、讲结构的句子只挂了表名、字段名,裁判以前只拿到名字:表附上字段清单和类型(最多 60 个,
|
||||
快照不完整要注明),字段附上类型和所属的表,口径卡指标附上输入取自哪次查询、哪一格和那次查询的 SQL。
|
||||
一律只经封存范围内的 loader 读工件,遮罩的列不出现
|
||||
- 理由上限 120 个字,超长时在句子边界截断加「…」;裁判规则要求 80 个字以内
|
||||
|
||||
所有模型调用都是 mock 剧本,不碰真接口。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
import pytest
|
||||
from langchain_core.messages import AIMessage
|
||||
from sqlalchemy import delete
|
||||
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource, Setting
|
||||
from app.engine import judge as judge_mod
|
||||
from app.engine.evidence import build_catalog, compose_doc, iter_units
|
||||
from app.engine.judge import (
|
||||
JUDGE_ROLE,
|
||||
JUDGE_SCHEMA,
|
||||
JUDGED,
|
||||
SPEND_KEY,
|
||||
SQL_CHARS,
|
||||
Budget,
|
||||
judge_doc,
|
||||
prepare,
|
||||
source_masks,
|
||||
summarize,
|
||||
)
|
||||
from app.providers.factory import ModelSpec
|
||||
from app.providers.mock_model import MockChatModel
|
||||
|
||||
PRICED = "claude-sonnet-5"
|
||||
UNLIMITED = Budget(max_claims=None, max_cost_usd=None, timeout_s=None, daily_max_usd=None)
|
||||
SPEC = ModelSpec(model=PRICED)
|
||||
|
||||
SOURCE = "warehouse"
|
||||
SCHEMA_ART, Q1_ART, Q2_ART, CARD_ART = "5" * 64, "a" * 64, "b" * 64, "c" * 64
|
||||
SQL1 = "SELECT COUNT(DISTINCT member_no) AS active_users FROM orders WHERE created_at >= '2026-09-01'"
|
||||
SQL2 = "SELECT region, SUM(amount) AS gmv FROM orders GROUP BY region"
|
||||
ORDER_COLS = [("id", "INTEGER"), ("member_no", "TEXT"), ("region", "TEXT"), ("amount", "REAL"),
|
||||
("phone", "TEXT"), ("created_at", "TEXT")]
|
||||
|
||||
|
||||
def schema_snapshot(columns=ORDER_COLS, *, truncated=False) -> dict:
|
||||
return {"source": SOURCE, "synced_at": "2026-09-01T00:00:00+00:00", "schema": None,
|
||||
"tables": {"orders": {"columns": [{"name": n, "type": t, "nullable": True} for n, t in columns],
|
||||
"primary_key": ["id"], "comment": None, "is_view": False},
|
||||
"refunds": {"columns": [{"name": "id", "type": "INTEGER"}, {"name": "amount", "type": "REAL"}],
|
||||
"primary_key": ["id"], "is_view": False}},
|
||||
**({"truncated": True, "total": 500} if truncated else {})}
|
||||
|
||||
|
||||
def query_entry(art: str, columns: list[str], tables: list[str]) -> dict:
|
||||
return {"kind": "query", "node_id": "fetch", "exec": 1, "artifact": art, "tool": f"db_query__{SOURCE}",
|
||||
"source": SOURCE, "columns": columns, "rows": 1, "truncated": False, "schema_artifact": SCHEMA_ART,
|
||||
"tables": tables}
|
||||
|
||||
|
||||
def schema_entry(snapshot: dict) -> dict:
|
||||
tables = {name: [c["name"] for c in meta["columns"]] for name, meta in snapshot["tables"].items()}
|
||||
return {"kind": "schema", "node_id": "fetch", "exec": 1, "artifact": SCHEMA_ART, "source": SOURCE,
|
||||
"tables": tables, **({"truncated": True, "total": 500} if snapshot.get("truncated") else {})}
|
||||
|
||||
|
||||
def snaps(snapshot=None, **extra) -> dict:
|
||||
out = {SCHEMA_ART: snapshot or schema_snapshot(),
|
||||
Q1_ART: {"columns": ["active_users"], "rows": [[321]], "sql": SQL1, "source": SOURCE},
|
||||
Q2_ART: {"columns": ["region", "gmv"], "rows": [["华东", 100.0]], "sql": SQL2, "source": SOURCE}}
|
||||
out.update(extra)
|
||||
return out
|
||||
|
||||
|
||||
def entity_catalog(snapshot=None) -> dict:
|
||||
snapshot = snapshot or schema_snapshot()
|
||||
return build_catalog(nodes={}, ledger=[query_entry(Q1_ART, ["active_users"], ["orders"]),
|
||||
query_entry(Q2_ART, ["region", "gmv"], ["orders"]), schema_entry(snapshot)])
|
||||
|
||||
|
||||
def excerpts(text: str, *, snapshot=None, loader=None, masked=None) -> dict[str, str]:
|
||||
cat = entity_catalog(snapshot)
|
||||
doc = compose_doc(text, cat)
|
||||
store = snaps(snapshot)
|
||||
return prepare(doc, cat, loader=loader or store.get, masked=masked).excerpts
|
||||
|
||||
|
||||
def field_line(excerpt: str) -> str:
|
||||
[line] = [x for x in excerpt.splitlines() if x.startswith("字段清单")]
|
||||
return line
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 判定拆档:取值、规则、schema
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_schema_offers_contradicted_and_insufficient_instead_of_unsupported():
|
||||
item = JUDGE_SCHEMA["properties"]["items"]["items"]
|
||||
assert item["properties"]["verdict"]["enum"] == list(JUDGED) == [
|
||||
"supported", "partial", "contradicted", "insufficient", "not_a_claim"]
|
||||
assert "unsupported" not in JUDGED
|
||||
assert "missing" in item["properties"] and "missing" in item["required"]
|
||||
assert "80" in item["properties"]["rationale"]["description"]
|
||||
|
||||
|
||||
def test_the_rules_separate_conflict_from_missing_information():
|
||||
rules = judge_mod.JUDGE_RULES
|
||||
assert "unsupported" not in rules, "新判的一律用新取值"
|
||||
assert "contradicted" in rules and "insufficient" in rules and "missing" in rules
|
||||
# 只有冲突才判 contradicted;没有相关信息判 insufficient,并写明缺什么
|
||||
[conflict] = [line for line in rules.splitlines() if line.startswith("- contradicted")]
|
||||
assert "冲突" in conflict
|
||||
[lacking] = [line for line in rules.splitlines() if line.startswith("- insufficient")]
|
||||
assert "没有" in lacking
|
||||
assert "80 个字" in rules and "60 个字" not in rules
|
||||
assert "表结构快照不完整" in rules, "快照不完整时「某字段不存在」只能判 insufficient"
|
||||
assert "数字已经由系统核对过,不要复算" in rules, "旧规则保留"
|
||||
|
||||
|
||||
def test_count_keys_keep_unsupported_for_old_data():
|
||||
assert set(judge_mod.COUNT_KEYS) == {"supported", "partial", "contradicted", "insufficient", "unsupported", "not_a_claim",
|
||||
"unjudged"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 判定拆档:调用与收下
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
RAW = ("活跃用户数由 [[t:orders]] 按 [[c:orders.member_no]] 去重计数得出。"
|
||||
"各区销售额按 [[c:orders.region]] 汇总。[[see:Q2]]"
|
||||
"华东的销售额是全国最高的,领先第二名一倍。[[see:Q2]]"
|
||||
"订单表里没有退款字段。[[see:t:orders]]")
|
||||
|
||||
|
||||
class Judge:
|
||||
"""裁判剧本:items(units, request) 给出每句的条目;calls 记下每次交给裁判的请求原文。"""
|
||||
|
||||
def __init__(self, monkeypatch, items, *, model_id=PRICED):
|
||||
self.calls: list[str] = []
|
||||
|
||||
def decide(model, messages):
|
||||
assert JUDGE_ROLE in str(messages[0].content)
|
||||
request = str(messages[-1].content)
|
||||
self.calls.append("\n".join(str(m.content) for m in messages))
|
||||
units = re.findall(r"^\[(u\d+)\]", request, re.M)
|
||||
return AIMessage(content=json.dumps({"items": items(units)}, ensure_ascii=False))
|
||||
|
||||
monkeypatch.setattr(MockChatModel, "_decide", decide)
|
||||
|
||||
async def fake_model(session, spec):
|
||||
return MockChatModel(model_name=model_id), model_id
|
||||
|
||||
monkeypatch.setattr(judge_mod, "get_chat_model", fake_model)
|
||||
|
||||
|
||||
def ids_of(doc) -> dict[str, str]:
|
||||
md = doc["markdown"]
|
||||
return {md[u["span"][0]:u["span"][1]].strip(): u["id"] for _, u in iter_units(doc)}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def clean_settings():
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"])))
|
||||
await session.commit()
|
||||
yield
|
||||
|
||||
|
||||
async def test_contradicted_and_insufficient_come_back_with_missing(monkeypatch):
|
||||
cat = entity_catalog()
|
||||
doc = compose_doc(RAW, cat)
|
||||
ids = ids_of(doc)
|
||||
method, grouped, ranked, absent = (ids[t] for t in list(ids)[:4])
|
||||
script = {
|
||||
method: {"verdict": "supported", "rationale": "Q1 的 SQL 写的是 COUNT(DISTINCT member_no)", "missing": ""},
|
||||
grouped: {"verdict": "insufficient", "rationale": "摘录里没有这次汇总的 SQL", "missing": "SQL"},
|
||||
ranked: {"verdict": "contradicted", "rationale": "Q2 里华东不是最高", "missing": "应当被丢掉"},
|
||||
absent: {"verdict": "insufficient", "rationale": "", "missing": ""},
|
||||
}
|
||||
Judge(monkeypatch, lambda units: [{"unit": u, "used": [], **script[u]} for u in units])
|
||||
outcome = await judge_doc(doc, cat, spec=SPEC, budget=UNLIMITED, loader=snaps().get)
|
||||
v = outcome["verdicts"]
|
||||
assert v[grouped]["status"] == "insufficient" and v[grouped]["missing"] == "SQL"
|
||||
assert v[ranked]["status"] == "contradicted" and "missing" not in v[ranked], "只有 insufficient 带 missing"
|
||||
assert v[absent]["status"] == "insufficient" and "missing" not in v[absent], "没写缺什么就不编"
|
||||
assert v[method]["status"] == "supported" and "missing" not in v[method]
|
||||
assert set(v[grouped]) == {"status", "rationale", "judge", "post_seal", "used", "missing", "rule_version"}
|
||||
assert outcome["counts"] == {"supported": 1, "partial": 0, "contradicted": 1, "insufficient": 2,
|
||||
"not_a_claim": 0, "unjudged": 0, "unsupported": 0}
|
||||
|
||||
|
||||
async def test_a_model_still_saying_unsupported_is_taken_as_contradicted(monkeypatch):
|
||||
"""新判的不再产出 unsupported:模型照旧写 unsupported 的按更严的 contradicted 收下,不当格式错误放过。"""
|
||||
cat = entity_catalog()
|
||||
doc = compose_doc(RAW, cat)
|
||||
Judge(monkeypatch, lambda units: [{"unit": u, "verdict": "unsupported", "rationale": "对不上", "used": []}
|
||||
for u in units])
|
||||
outcome = await judge_doc(doc, cat, spec=SPEC, budget=UNLIMITED, loader=snaps().get)
|
||||
assert {x["status"] for x in outcome["verdicts"].values()} == {"contradicted"}
|
||||
assert outcome["counts"]["unsupported"] == 0 and outcome["counts"]["contradicted"] == 4
|
||||
|
||||
|
||||
def test_summary_counts_always_carry_every_key():
|
||||
"""缓存里的旧判定只有旧的五个键:摘要照样补齐 contradicted、insufficient。"""
|
||||
outcome = {"counts": {"supported": 1, "partial": 0, "unsupported": 1, "not_a_claim": 0, "unjudged": 0},
|
||||
"candidates": 2, "screened": [], "unjudged": {}, "limits_hit": [], "gaps": []}
|
||||
summary = summarize(outcome, mode="inline", writer_model="writer-x")
|
||||
assert summary["counts"] == {"supported": 1, "partial": 0, "contradicted": 0, "insufficient": 0,
|
||||
"not_a_claim": 0, "unjudged": 0, "unsupported": 1}
|
||||
assert summary["writer_model"] == "writer-x"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 理由不硬截断
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_rationale_under_the_cap_is_kept_whole():
|
||||
text = "摘录里" + "有对应的指标和查询" * 10 + "。" # 94 个字:旧规则会截在 60 字的半句中间
|
||||
assert 60 < len(text) <= judge_mod.RATIONALE_MAX == 120
|
||||
assert judge_mod.clip_text(text) == text
|
||||
|
||||
|
||||
def test_a_long_rationale_is_cut_at_a_sentence_boundary():
|
||||
first = "Q2 的 SQL 按 region 汇总,华东 100 元排第一"
|
||||
text = first + ";" + "第二名的数字摘录里没有给出," * 12
|
||||
got = judge_mod.clip_text(text)
|
||||
assert len(got) <= judge_mod.RATIONALE_MAX and got.endswith("…")
|
||||
body = got[:-1]
|
||||
assert text.startswith(body) and text[len(body)] in "。;,", f"截在句子边界:{got}"
|
||||
|
||||
|
||||
def test_a_long_run_without_punctuation_is_not_cut_inside_a_word():
|
||||
text = " ".join(["evidence"] * 30)
|
||||
got = judge_mod.clip_text(text)
|
||||
assert len(got) <= judge_mod.RATIONALE_MAX and got.endswith("…")
|
||||
assert got[:-1].rstrip().split(" ")[-1] == "evidence", got
|
||||
|
||||
|
||||
async def test_the_judge_keeps_a_long_rationale_up_to_the_cap(monkeypatch):
|
||||
cat = entity_catalog()
|
||||
doc = compose_doc("各区销售额按 [[c:orders.region]] 汇总。[[see:Q2]]", cat)
|
||||
long = "Q2 的 SQL 是按 region 分组求和," * 3 + "句子说的汇总方式和 SQL 一致," * 8
|
||||
Judge(monkeypatch, lambda units: [{"unit": u, "verdict": "supported", "rationale": long, "used": []}
|
||||
for u in units])
|
||||
outcome = await judge_doc(doc, cat, spec=SPEC, budget=UNLIMITED, loader=snaps().get)
|
||||
[v] = outcome["verdicts"].values()
|
||||
assert 60 < len(v["rationale"]) <= judge_mod.RATIONALE_MAX and v["rationale"].endswith("…")
|
||||
assert long.startswith(v["rationale"][:-1])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 补全证据摘录:表
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_table_excerpt_lists_its_fields_and_types():
|
||||
ex = excerpts("活跃用户数由 [[t:orders]] 去重计数得出。")
|
||||
table = ex["t:orders"]
|
||||
line = field_line(table)
|
||||
for name, kind in ORDER_COLS:
|
||||
assert f"{name} {kind}" in line, line
|
||||
assert "COUNT(DISTINCT member_no)" in table, "出现过的查询 SQL 保留"
|
||||
assert "表结构快照不完整" not in table
|
||||
|
||||
|
||||
def test_the_field_list_stops_at_sixty():
|
||||
columns = [(f"col_{i:02d}", "TEXT") for i in range(75)]
|
||||
ex = excerpts("活跃用户数由 [[t:orders]] 去重计数得出。", snapshot=schema_snapshot(columns))
|
||||
line = field_line(ex["t:orders"])
|
||||
listed = re.findall(r"col_\d\d", line)
|
||||
assert judge_mod.FIELDS_MAX == 60 and listed == [f"col_{i:02d}" for i in range(60)]
|
||||
assert "另有 15 个" in line
|
||||
|
||||
|
||||
def test_masked_fields_are_never_listed():
|
||||
ex = excerpts("活跃用户数由 [[t:orders]] 去重计数得出。", masked={SOURCE: ["Phone"]})
|
||||
table = ex["t:orders"]
|
||||
assert "phone" not in field_line(table).lower() and "member_no TEXT" in field_line(table)
|
||||
assert "遮罩" in table, "说明有字段按遮罩没列出:裁判不能据此说表里没有这个字段"
|
||||
|
||||
|
||||
def test_masks_recorded_in_a_query_snapshot_also_hide_fields():
|
||||
"""数据源改名、删掉了,现在查不到遮罩:查询当时记下的遮罩照样生效(证据面板同一个口径)。"""
|
||||
cat = entity_catalog()
|
||||
doc = compose_doc("活跃用户数由 [[t:orders]] 去重计数得出。", cat)
|
||||
store = snaps(**{Q1_ART: {"columns": ["active_users"], "rows": [[321]], "sql": SQL1, "source": SOURCE,
|
||||
"mask_columns": ["phone"]}})
|
||||
table = prepare(doc, cat, loader=store.get).excerpts["t:orders"]
|
||||
assert "phone" not in field_line(table).lower()
|
||||
|
||||
|
||||
def test_a_truncated_snapshot_is_called_out():
|
||||
ex = excerpts("活跃用户数由 [[t:orders]] 去重计数得出。", snapshot=schema_snapshot(truncated=True))
|
||||
table = ex["t:orders"]
|
||||
assert "表结构快照不完整" in table and "id INTEGER" in field_line(table)
|
||||
|
||||
|
||||
def test_only_the_sealed_loader_feeds_the_field_list():
|
||||
"""schema 快照不在封存范围里(loader 取不到):一个字段都不给,照实说没有字段清单。"""
|
||||
store = snaps()
|
||||
ex = excerpts("活跃用户数由 [[t:orders]] 去重计数得出。", loader=lambda a: None if a == SCHEMA_ART else store.get(a))
|
||||
table = ex["t:orders"]
|
||||
assert "INTEGER" not in table and "member_no TEXT" not in table
|
||||
assert "没有" in field_line(table)
|
||||
|
||||
|
||||
def test_new_excerpt_content_counts_toward_the_estimate():
|
||||
cat = entity_catalog()
|
||||
doc = compose_doc("活跃用户数由 [[t:orders]] 去重计数得出。", cat)
|
||||
store = snaps()
|
||||
full = prepare(doc, cat, loader=store.get)
|
||||
bare = prepare(doc, cat, loader=lambda a: None if a == SCHEMA_ART else store.get(a))
|
||||
assert full.cost(PRICED, full.cands) > bare.cost(PRICED, bare.cands)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 补全证据摘录:字段
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_column_excerpt_carries_its_type_and_table():
|
||||
column = excerpts("活跃用户数按 [[c:orders.member_no]] 去重计数得出。")["c:orders.member_no"]
|
||||
assert "类型:TEXT" in column and "所属的表:orders" in column
|
||||
assert "COUNT(DISTINCT member_no)" in column, "出现过的查询 SQL 保留"
|
||||
|
||||
|
||||
def test_a_result_column_takes_its_type_from_the_query_snapshot():
|
||||
"""聚合的别名(gmv)不在表结构里:类型取查询快照记下的列类型。"""
|
||||
cat = entity_catalog()
|
||||
doc = compose_doc("汇总在 [[c:gmv]] 这一列。[[see:Q2]]", cat)
|
||||
store = snaps(**{Q2_ART: {"columns": ["region", "gmv"], "rows": [["华东", 100.0]], "sql": SQL2, "source": SOURCE,
|
||||
"column_types": {"region": "text", "gmv": "number"}}})
|
||||
column = prepare(doc, cat, loader=store.get).excerpts["c:gmv"]
|
||||
assert "类型:number" in column
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 补全证据摘录:口径卡指标
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
LONG_SQL = "SELECT COUNT(DISTINCT member_no) AS active_users FROM orders WHERE " + " AND ".join(
|
||||
f"flag_{i} = 1" for i in range(80))
|
||||
CARD = {"kind": "metric_set", "caliber": "周报口径", "caliber_version": "v2", "metrics": [
|
||||
{"id": "active", "name": "活跃用户数", "unit": "人", "value": 321, "decimals": 0, "format": "plain", "status": "ok",
|
||||
"expression": "cell(nodes.fetch, 0, 'active_users')", "rendered": "321人",
|
||||
"inputs": [{"path": "cell(nodes.fetch, 0, 'active_users')", "value": 321, "node_id": None, "via": "tool_cell",
|
||||
"artifact": Q1_ART, "locator": {"row": 0, "column": "active_users"}, "status": "ok"},
|
||||
{"path": "vars.target", "value": 300, "node_id": "plan", "via": "transform", "status": "ok"}]}]}
|
||||
|
||||
|
||||
def metric_catalog():
|
||||
ledger = [{"kind": "metric_set", "node_id": "caliber", "exec": 1, "artifact": CARD_ART, "caliber": "周报口径",
|
||||
"version": "v2", "metrics": ["active"]}, query_entry(Q1_ART, ["active_users"], ["orders"])]
|
||||
return build_catalog(nodes={"caliber": {**CARD, "artifact": CARD_ART}}, ledger=ledger)
|
||||
|
||||
|
||||
def test_a_metric_excerpt_names_the_query_and_cell_of_its_input_and_that_sql():
|
||||
cat = metric_catalog()
|
||||
doc = compose_doc("活跃用户数 [[m:active]],按会员号去重计数。[[see:m:active]]", cat)
|
||||
store = {**snaps(), CARD_ART: CARD}
|
||||
metric = prepare(doc, cat, loader=store.get).excerpts["m:active"]
|
||||
assert "原式:cell(nodes.fetch, 0, 'active_users')" in metric
|
||||
[where] = [line for line in metric.splitlines() if "active_users" in line and "Q1" in line and "SQL" not in line]
|
||||
assert "r0" in where, where
|
||||
[sql] = [line for line in metric.splitlines() if line.startswith("查询 Q1 的 SQL")]
|
||||
assert "COUNT(DISTINCT member_no)" in sql
|
||||
assert any("vars.target" in line and "未追溯到查询" in line for line in metric.splitlines())
|
||||
|
||||
|
||||
def test_metric_input_sql_is_capped_and_needs_the_sealed_loader():
|
||||
cat = metric_catalog()
|
||||
doc = compose_doc("活跃用户数 [[m:active]],按会员号去重计数。[[see:m:active]]", cat)
|
||||
long = {**snaps(**{Q1_ART: {"columns": ["active_users"], "rows": [[321]], "sql": LONG_SQL, "source": SOURCE}}),
|
||||
CARD_ART: CARD}
|
||||
metric = prepare(doc, cat, loader=long.get).excerpts["m:active"]
|
||||
[sql] = [line for line in metric.splitlines() if line.startswith("查询 Q1 的 SQL")]
|
||||
assert LONG_SQL[:SQL_CHARS] in sql and LONG_SQL[:SQL_CHARS + 10] not in sql and sql.endswith("…")
|
||||
|
||||
sealed_only = {CARD_ART: CARD} # 查询快照不在封存范围里
|
||||
metric = prepare(doc, cat, loader=sealed_only.get).excerpts["m:active"]
|
||||
assert "SELECT" not in metric and "Q1" in metric, "位置照给,SQL 不给"
|
||||
|
||||
|
||||
def test_a_masked_input_column_is_not_named():
|
||||
cat = metric_catalog()
|
||||
doc = compose_doc("活跃用户数 [[m:active]],按会员号去重计数。[[see:m:active]]", cat)
|
||||
store = {**snaps(), CARD_ART: CARD}
|
||||
metric = prepare(doc, cat, loader=store.get, masked={SOURCE: ["active_users"]}).excerpts["m:active"]
|
||||
[where] = [line for line in metric.splitlines() if line.startswith("输入 cell(")]
|
||||
assert "r0" in where and "active_users" not in where.split(":", 1)[1]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 遮罩:表、字段的数据源也要查
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_source_masks_cover_the_sources_of_tables():
|
||||
name = "jr_mask_src"
|
||||
cat = build_catalog(nodes={}, ledger=[{**schema_entry(schema_snapshot()), "source": name}])
|
||||
assert cat and all(e["kind"] in ("table", "column") for e in cat.values())
|
||||
async with SessionLocal() as session:
|
||||
session.add(DataSource(name=name, kind="sqlite", database="unused.db", options={"mask_columns": "phone"}))
|
||||
await session.commit()
|
||||
try:
|
||||
assert await source_masks(cat) == {name: ["phone"]}
|
||||
finally:
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(delete(DataSource).where(DataSource.name == name))
|
||||
await session.commit()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 断点里的旧判定:规则换了就不复用
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_request_key_carries_the_rules_version(monkeypatch):
|
||||
"""报告节点按 request_key 认断点里缓存的判定:规则、取值换了(旧的 unsupported),旧判定不能原样复用。"""
|
||||
cat = entity_catalog()
|
||||
doc = compose_doc(RAW, cat)
|
||||
request = prepare(doc, cat, loader=snaps().get)
|
||||
before = judge_mod.request_key(request, SPEC, UNLIMITED)
|
||||
assert before == judge_mod.request_key(request, SPEC, UNLIMITED), "同一份输入同一个指纹"
|
||||
monkeypatch.setattr(judge_mod, "RULES_VERSION", judge_mod.RULES_VERSION + 1)
|
||||
assert judge_mod.request_key(request, SPEC, UNLIMITED) != before
|
||||
|
||||
|
||||
async def test_a_new_verdict_carries_the_rules_version(monkeypatch):
|
||||
"""新判的判定带上规则版本(和 request_key 里的同一个常量):按需裁判据此认出早于当前规则的旧判定。"""
|
||||
cat = entity_catalog()
|
||||
doc = compose_doc("各区销售额按 [[c:orders.region]] 汇总。[[see:Q2]]", cat)
|
||||
Judge(monkeypatch, lambda units: [{"unit": u, "verdict": "supported", "rationale": "有依据", "used": []}
|
||||
for u in units])
|
||||
outcome = await judge_doc(doc, cat, spec=SPEC, budget=UNLIMITED, loader=snaps().get)
|
||||
[v] = outcome["verdicts"].values()
|
||||
assert v["rule_version"] == judge_mod.RULES_VERSION
|
||||
|
||||
|
||||
def test_which_verdicts_are_older_than_the_current_rules():
|
||||
current = judge_mod.RULES_VERSION
|
||||
assert judge_mod.outdated({"status": "supported"}), "没有规则版本:改造前判的"
|
||||
assert judge_mod.outdated({"status": "unsupported", "rule_version": current}), "旧取值一律算旧"
|
||||
assert judge_mod.outdated({"status": "partial", "rule_version": current - 1})
|
||||
assert not judge_mod.outdated({"status": "contradicted", "rule_version": current})
|
||||
assert not judge_mod.outdated({"status": "unjudged", "reason": "error"}), "没判成的谈不上新旧"
|
||||
@@ -0,0 +1,298 @@
|
||||
"""结论句裁判改造(JR)的判档:contradicted 沿用原来 unsupported 的处置,insufficient 按缺口处理。
|
||||
|
||||
- contradicted(证据相矛盾):按 on_unsupported 降档(degrade)或不予出具(withhold),和原来的 unsupported 一样
|
||||
- insufficient(证据不足):按缺口处理,最多降档——不会不予出具,也不会判完整出具
|
||||
- 不许比现在更松:受管正式运行里,原来会因「证据不支持」降档的句子,改判成 contradicted 或 insufficient
|
||||
都不能完整出具
|
||||
- 旧文档里的 unsupported 判档照旧当 contradicted 处理
|
||||
- 统计键(stats、counts、出具横幅的结论句计数)都有 contradicted、insufficient,unsupported 保留
|
||||
- 报告节点:改写一次把证据相矛盾、证据不足的句子都交回写作者;文档的裁判摘要记下写作模型
|
||||
|
||||
所有模型调用都是 mock 剧本,不碰真接口。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from langchain_core.messages import AIMessage
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Run, RunEvent, Setting, Workflow, WorkflowVersion
|
||||
from app.engine import judge as judge_mod
|
||||
from app.engine.evidence import compose_doc, iter_units
|
||||
from app.engine.issuance import decide_tier
|
||||
from app.engine.judge import JUDGE_ROLE, SPEND_KEY
|
||||
from app.engine.nodes.io import _claims
|
||||
from app.engine.runner import run_manager
|
||||
from app.main import app
|
||||
from app.providers.mock_model import MockChatModel
|
||||
|
||||
JUDGE_MODEL = "claude-sonnet-5"
|
||||
WRITER = "mock-fast"
|
||||
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
|
||||
CITED = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。[[see:m:gmv]]下面看细节。"
|
||||
SUPPORTED, CAUSAL = "本周销售额 45,678.5元。", "增长主要来自新客首单。"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 出口判档(io._claims):直接喂文档
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def judged_doc(statuses: dict[str, dict], *, on_unsupported="degrade") -> dict:
|
||||
"""两句结论:SUPPORTED_TEXT、CAUSAL_TEXT;statuses 按句子给判定。counts 照判定数。"""
|
||||
doc = compose_doc("本周销售额稳定增长。增长主要来自新客首单。", {})
|
||||
md = doc["markdown"]
|
||||
counts = {k: 0 for k in ("supported", "partial", "unsupported", "not_a_claim", "unjudged")}
|
||||
for _, unit in iter_units(doc):
|
||||
text = md[unit["span"][0]:unit["span"][1]].strip()
|
||||
verdict = {"status": "supported", "rationale": "有依据", "judge": JUDGE_MODEL, "post_seal": False, "used": [],
|
||||
**statuses.get(text, {})}
|
||||
unit["verdict"] = verdict
|
||||
counts[verdict["status"]] = counts.get(verdict["status"], 0) + 1
|
||||
doc["judge"] = {"mode": "inline", "model": JUDGE_MODEL, "on_unsupported": on_unsupported, "counts": counts,
|
||||
"unjudged": {}, "limits_hit": [], "complete": True, "gaps": []}
|
||||
return doc
|
||||
|
||||
|
||||
def grade(doc: dict, *, governed=False, contract=None) -> tuple[str, dict]:
|
||||
out = {"gaps": [], "claims_policy": None, "uncited": None, "claims": None, "unsupported": None}
|
||||
_claims(out, contract or {}, {"claims": "judge", "text": doc["markdown"]}, evidence_on=True, governed=governed,
|
||||
uncited=[], doc=doc, title="write")
|
||||
tier = decide_tier(missing_required=[], missing_expected=[], unmatched=[], strict=True, gaps=out["gaps"],
|
||||
unsupported=out["unsupported"], uncited_claims=out["uncited"],
|
||||
claims_policy=out["claims_policy"])
|
||||
return tier, out
|
||||
|
||||
|
||||
TEXT2 = "增长主要来自新客首单。"
|
||||
|
||||
|
||||
def test_all_supported_is_still_a_full_issuance():
|
||||
tier, out = grade(judged_doc({}))
|
||||
assert tier == "formal" and out["gaps"] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("on_unsupported", "tier"), [("degrade", "degraded"), ("withhold", "withheld")])
|
||||
def test_contradicted_follows_on_unsupported(on_unsupported, tier):
|
||||
doc = judged_doc({TEXT2: {"status": "contradicted", "rationale": "证据里新客占比下降"}},
|
||||
on_unsupported=on_unsupported)
|
||||
got, out = grade(doc)
|
||||
assert got == tier
|
||||
claims = out["claims"]
|
||||
[item] = claims["contradicted"]
|
||||
assert item["text"] == TEXT2 and item["rationale"] == "证据里新客占比下降"
|
||||
assert claims["counts"]["contradicted"] == 1 and claims["counts"]["insufficient"] == 0
|
||||
assert out["gaps"] == [], "判档来自判定,不来自缺口"
|
||||
|
||||
|
||||
def test_insufficient_is_a_gap_and_never_withholds():
|
||||
doc = judged_doc({TEXT2: {"status": "insufficient", "rationale": "摘录里没有新客维度", "missing": "新客维度的查询"}},
|
||||
on_unsupported="withhold")
|
||||
tier, out = grade(doc, governed=True, contract={"claims": {"policy": "judge", "on_unsupported": "withhold"}})
|
||||
assert tier == "degraded", "证据不足最多降档:不能不予出具,也不能完整出具"
|
||||
[gap] = [g for g in out["gaps"] if "证据不足" in g]
|
||||
assert gap.startswith("报告「write」") and "1 句" in gap and "新客维度的查询" in gap
|
||||
[item] = out["claims"]["insufficient"]
|
||||
assert item["text"] == TEXT2 and item["missing"] == "新客维度的查询"
|
||||
assert out["claims"]["counts"]["insufficient"] == 1 and out["unsupported"] == []
|
||||
|
||||
|
||||
def test_an_old_unsupported_verdict_is_graded_as_contradicted():
|
||||
doc = judged_doc({TEXT2: {"status": "unsupported", "rationale": "证据不支持"}}, on_unsupported="withhold")
|
||||
tier, out = grade(doc)
|
||||
assert tier == "withheld"
|
||||
assert out["claims"]["counts"]["unsupported"] == 1
|
||||
assert [i["text"] for i in out["claims"]["unsupported"]] == [TEXT2]
|
||||
assert [i["text"] for i in out["unsupported"]] == [TEXT2]
|
||||
|
||||
|
||||
def test_the_issuance_counts_carry_every_key():
|
||||
_, out = grade(judged_doc({}))
|
||||
assert set(out["claims"]["counts"]) == set(judge_mod.COUNT_KEYS)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 端到端:正式运行、受管正式运行
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def engine_up():
|
||||
await run_manager.setup()
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"])))
|
||||
await session.commit()
|
||||
yield
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client(engine_up):
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
def node(nid, ntype, **config):
|
||||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||||
|
||||
|
||||
def weekly(*, report=None, contract=None) -> dict:
|
||||
nodes = [
|
||||
node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]),
|
||||
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
|
||||
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
|
||||
node("write", "report", instructions="写周报", **{"claims": "judge", "judge": {"max_cost_usd": 0.05},
|
||||
**(report or {})}),
|
||||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}],
|
||||
contract={"report_from": "write", "metrics_from": ["caliber"], "required": ["gmv"], "strict": True,
|
||||
**(contract or {})}),
|
||||
]
|
||||
ids = [n["id"] for n in nodes]
|
||||
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
|
||||
|
||||
|
||||
class Models:
|
||||
"""写作者依次回 writer 里的几段;裁判按句子里的字给判定:verdicts = {句中片段: (判定, missing)}。"""
|
||||
|
||||
def __init__(self, monkeypatch, writer=(CITED,), verdicts=None):
|
||||
self.writes: list[list] = []
|
||||
self.judges: list[str] = []
|
||||
verdicts = verdicts or {}
|
||||
|
||||
def decide(model, messages):
|
||||
if JUDGE_ROLE in str(messages[0].content):
|
||||
request = str(messages[-1].content)
|
||||
self.judges.append(request)
|
||||
items = []
|
||||
for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M):
|
||||
status, missing = next((v for frag, v in verdicts.items() if frag in line), ("supported", ""))
|
||||
items.append({"unit": uid, "verdict": status, "rationale": f"判的是:{line.strip()[:12]}",
|
||||
"missing": missing, "used": []})
|
||||
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
|
||||
self.writes.append(list(messages))
|
||||
return AIMessage(content=writer[min(len(self.writes), len(writer)) - 1])
|
||||
|
||||
monkeypatch.setattr(MockChatModel, "_decide", decide)
|
||||
|
||||
async def judge_model_of(session, spec):
|
||||
return MockChatModel(model_name=JUDGE_MODEL), JUDGE_MODEL
|
||||
|
||||
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
|
||||
|
||||
|
||||
async def wait(run_id: str) -> Run:
|
||||
for _ in range(400):
|
||||
await asyncio.sleep(0.05)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, run_id)
|
||||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||||
return row
|
||||
raise AssertionError(f"run {run_id} 没跑完:{row.status}")
|
||||
|
||||
|
||||
async def run(graph, *, run_class="formal") -> Run:
|
||||
started = await run_manager.start(graph=graph, input_payload={}, run_class=run_class)
|
||||
row = await wait(started.id)
|
||||
assert row.status == "succeeded", row.error
|
||||
return row
|
||||
|
||||
|
||||
async def checked_of(run_id: str) -> tuple[dict, dict]:
|
||||
async with SessionLocal() as session:
|
||||
[event] = (await session.execute(select(RunEvent).where(
|
||||
RunEvent.run_id == run_id, RunEvent.type == "report.checked"))).scalars()
|
||||
return event.data, artifact_store.load(event.data["doc_artifact"])
|
||||
|
||||
|
||||
async def issuance_event(run_id: str) -> dict:
|
||||
async with SessionLocal() as session:
|
||||
[event] = (await session.execute(select(RunEvent).where(
|
||||
RunEvent.run_id == run_id, RunEvent.type == "issuance"))).scalars()
|
||||
return event.data
|
||||
|
||||
|
||||
async def test_a_formal_run_withholds_a_contradicted_claim(engine_up, monkeypatch):
|
||||
Models(monkeypatch, verdicts={"新客": ("contradicted", "")})
|
||||
row = await run(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
|
||||
issuance = row.output["_issuance"]
|
||||
assert issuance["tier"] == "withheld"
|
||||
assert [i["text"] for i in issuance["claims"]["contradicted"]] == [CAUSAL]
|
||||
event = await issuance_event(row.id)
|
||||
assert event["claims"]["counts"] == {"supported": 1, "partial": 0, "contradicted": 1, "insufficient": 0,
|
||||
"not_a_claim": 0, "unjudged": 0, "unsupported": 0}
|
||||
|
||||
|
||||
async def test_a_formal_run_only_degrades_an_insufficient_claim(engine_up, monkeypatch):
|
||||
Models(monkeypatch, verdicts={"新客": ("insufficient", "新客维度的数据")})
|
||||
row = await run(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
|
||||
issuance = row.output["_issuance"]
|
||||
assert issuance["tier"] == "degraded", "证据不足:最多降档,不因 withhold 不予出具"
|
||||
assert any("证据不足" in g and "新客维度的数据" in g for g in issuance["gaps"]), issuance["gaps"]
|
||||
[item] = issuance["claims"]["insufficient"]
|
||||
assert item["text"] == CAUSAL and item["missing"] == "新客维度的数据"
|
||||
assert issuance["claims"]["counts"]["insufficient"] == 1
|
||||
|
||||
checked, doc = await checked_of(row.id)
|
||||
assert doc["stats"]["insufficient"] == 1 and doc["stats"]["contradicted"] == 0 and doc["stats"]["unsupported"] == 0
|
||||
assert checked["stats"]["insufficient"] == 1
|
||||
assert doc["judge"]["writer_model"] == WRITER and checked["judge"]["writer_model"] == WRITER
|
||||
[unit] = [u for _, u in iter_units(doc) if u.get("verdict", {}).get("status") == "insufficient"]
|
||||
assert unit["verdict"]["missing"] == "新客维度的数据"
|
||||
|
||||
|
||||
async def governed(client, graph) -> Run:
|
||||
async with SessionLocal() as session:
|
||||
wf = Workflow(name=f"jr-{uuid.uuid4().hex[:6]}", graph=graph, status="governed", published_version=1)
|
||||
session.add(wf)
|
||||
await session.flush()
|
||||
session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph))
|
||||
await session.commit()
|
||||
wf_id = wf.id
|
||||
res = await client.post("/api/runs", json={"workflow_id": wf_id, "run_class": "formal"})
|
||||
assert res.status_code == 201, res.text
|
||||
row = await wait(res.json()["id"])
|
||||
assert row.status == "succeeded", row.error
|
||||
return row
|
||||
|
||||
|
||||
@pytest.mark.parametrize("status", ["contradicted", "insufficient"])
|
||||
async def test_a_governed_formal_run_is_never_fully_issued_on_a_split_verdict(client, monkeypatch, status):
|
||||
"""不许比现在更松:原来判「证据不支持」要降档的句子,改判成哪一档都不能完整出具。"""
|
||||
Models(monkeypatch, verdicts={"新客": (status, "新客维度的数据" if status == "insufficient" else "")})
|
||||
graph = weekly(report={"numbers": "strict", "on_violation": "fail", "judge": {"max_cost_usd": None}})
|
||||
row = await governed(client, graph)
|
||||
assert row.output["_issuance"]["tier"] == "degraded"
|
||||
|
||||
|
||||
async def test_rewrite_once_hands_back_contradicted_and_insufficient_sentences(engine_up, monkeypatch):
|
||||
draft = ("本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。[[see:m:gmv]]"
|
||||
"华东贡献了大部分增长。[[see:m:gmv]]下面看细节。")
|
||||
rewritten = "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客、华东的贡献要等分维度的数据再看。下面看细节。"
|
||||
models = Models(monkeypatch, writer=(draft, rewritten),
|
||||
verdicts={"新客首单": ("contradicted", ""), "华东贡献": ("insufficient", "分区域的查询"),
|
||||
"要等分维度": ("not_a_claim", "")})
|
||||
row = await run(weekly(report={"judge": {"max_cost_usd": 0.05, "rewrite_once": True}}))
|
||||
assert len(models.writes) == 2
|
||||
request = str(models.writes[1][-1].content)
|
||||
assert "增长主要来自新客首单。" in request and "华东贡献了大部分增长。" in request
|
||||
assert "分区域的查询" in request, "证据不足的句子连同缺的是什么一起交回"
|
||||
assert SUPPORTED not in request
|
||||
checked, _ = await checked_of(row.id)
|
||||
assert checked["judge"]["rewrite"]["applied"] is True and len(checked["judge"]["rewrite"]["sentences"]) == 2
|
||||
|
||||
|
||||
async def test_an_explore_report_records_the_writer_model_too(engine_up, monkeypatch):
|
||||
models = Models(monkeypatch)
|
||||
row = await run(weekly(), run_class="exploratory")
|
||||
assert models.judges == []
|
||||
checked, doc = await checked_of(row.id)
|
||||
assert doc["judge"]["mode"] == "on_demand" and doc["judge"]["writer_model"] == WRITER
|
||||
assert set(judge_mod.COUNT_KEYS) <= set(doc["stats"])
|
||||
@@ -26,7 +26,7 @@ from app.engine import judge as judge_mod
|
||||
from app.engine import replay
|
||||
from app.engine import runner as runner_mod
|
||||
from app.engine.evidence import CLAIMS_RULE, iter_units
|
||||
from app.engine.judge import JUDGE_ROLE, JUDGE_RULE, SPEND_KEY, SUMMARY_KEYS, daily_spend
|
||||
from app.engine.judge import COUNT_KEYS, JUDGE_ROLE, JUDGE_RULE, SPEND_KEY, SUMMARY_KEYS, daily_spend
|
||||
from app.engine.nodes import report as report_mod
|
||||
from app.engine.runner import run_manager, verify_manifest
|
||||
from app.providers.mock_model import MockChatModel
|
||||
@@ -167,7 +167,7 @@ async def judge_ends(run_id: str) -> list[dict]:
|
||||
|
||||
|
||||
async def test_a_formal_run_judges_inside_the_node_and_seals_the_verdicts(monkeypatch):
|
||||
models = Models(monkeypatch, verdicts={"新客": "unsupported"})
|
||||
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
|
||||
row = await finish(weekly(claims="judge", judge={"on_unsupported": "withhold"}))
|
||||
assert row.status == "succeeded", row.error
|
||||
assert len(models.writes) == 1 and len(models.judges) == 1
|
||||
@@ -176,18 +176,19 @@ async def test_a_formal_run_judges_inside_the_node_and_seals_the_verdicts(monkey
|
||||
got = verdicts(doc)
|
||||
assert got[CITED]["status"] == "supported" and got[CITED]["judge"] == PRICED
|
||||
assert got[CITED]["post_seal"] is False and got[CITED]["rationale"]
|
||||
assert got[CAUSAL]["status"] == "unsupported"
|
||||
assert got[CAUSAL]["status"] == "contradicted"
|
||||
assert got[LINK] is None, "连接性的话不送裁判"
|
||||
assert CITED in models.judges[0] and CAUSAL in models.judges[0] and LINK not in models.judges[0]
|
||||
|
||||
summary = checked["judge"]
|
||||
assert tuple(summary) == SUMMARY_KEYS, "B 段照着这份键读:不能悄悄少一个、多一个"
|
||||
assert summary["mode"] == "inline" and summary["model"] == PRICED and summary["on_unsupported"] == "withhold"
|
||||
assert summary["counts"] == {"supported": 1, "partial": 0, "unsupported": 1, "not_a_claim": 0, "unjudged": 0}
|
||||
assert summary["counts"] == {"supported": 1, "partial": 0, "contradicted": 1, "insufficient": 0, "not_a_claim": 0,
|
||||
"unjudged": 0, "unsupported": 0}
|
||||
assert summary["complete"] is True and summary["gaps"] == []
|
||||
assert checked["claims"] == output["claims"] == "judge" and output["judge"] == summary
|
||||
assert doc["judge"]["mode"] == "inline" and doc["judge"]["counts"] == summary["counts"]
|
||||
assert {k: doc["stats"][k] for k in ("supported", "partial", "unsupported", "not_a_claim", "unjudged")} \
|
||||
assert {k: doc["stats"][k] for k in COUNT_KEYS} \
|
||||
== summary["counts"]
|
||||
|
||||
[end] = await judge_ends(row.id)
|
||||
@@ -277,7 +278,7 @@ REWRITTEN = "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客首单的贡献要
|
||||
|
||||
async def test_rewrite_once_hands_back_only_the_unsupported_sentences(monkeypatch):
|
||||
models = Models(monkeypatch, writer=(TEXT, REWRITTEN),
|
||||
verdicts={"主要来自新客": "unsupported", "要等新客维度": "not_a_claim"})
|
||||
verdicts={"主要来自新客": "contradicted", "要等新客维度": "not_a_claim"})
|
||||
row = await finish(weekly(claims="judge", judge={"rewrite_once": True}))
|
||||
assert row.status == "succeeded", row.error
|
||||
assert len(models.writes) == 2 and len(models.judges) == 2
|
||||
@@ -305,23 +306,23 @@ async def test_rewrite_once_hands_back_only_the_unsupported_sentences(monkeypatc
|
||||
|
||||
|
||||
async def test_rewrite_once_is_off_by_default(monkeypatch):
|
||||
models = Models(monkeypatch, writer=(TEXT, REWRITTEN), verdicts={"主要来自新客": "unsupported"})
|
||||
models = Models(monkeypatch, writer=(TEXT, REWRITTEN), verdicts={"主要来自新客": "contradicted"})
|
||||
row = await finish(weekly(claims="judge"))
|
||||
assert row.status == "succeeded", row.error
|
||||
assert len(models.writes) == 1 and len(models.judges) == 1
|
||||
checked, doc, _ = await report_of(row.id)
|
||||
assert verdicts(doc)[CAUSAL]["status"] == "unsupported" and checked["judge"].get("rewrite") is None
|
||||
assert verdicts(doc)[CAUSAL]["status"] == "contradicted" and checked["judge"].get("rewrite") is None
|
||||
|
||||
|
||||
async def test_a_rewrite_that_breaks_the_numbers_is_not_used(monkeypatch):
|
||||
models = Models(monkeypatch, writer=(TEXT, "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客贡献了 45678 元。下面看细节。"),
|
||||
verdicts={"主要来自新客": "unsupported"})
|
||||
verdicts={"主要来自新客": "contradicted"})
|
||||
row = await finish(weekly(claims="judge", judge={"rewrite_once": True}))
|
||||
assert row.status == "succeeded", row.error
|
||||
assert len(models.writes) == 2 and len(models.judges) == 1, "改坏了的那一稿不再花钱判"
|
||||
checked, doc, output = await report_of(row.id)
|
||||
assert CAUSAL in doc["markdown"] and "45678" not in doc["markdown"]
|
||||
assert verdicts(doc)[CAUSAL]["status"] == "unsupported"
|
||||
assert verdicts(doc)[CAUSAL]["status"] == "contradicted"
|
||||
rewrite = checked["judge"]["rewrite"]
|
||||
assert rewrite["applied"] is False and "45678" in rewrite["reason"]
|
||||
# 封存的是原稿:units 就是原稿(也就是封存文档)里的编号
|
||||
@@ -346,7 +347,7 @@ async def test_a_rewrite_that_makes_up_a_name_is_not_used(monkeypatch):
|
||||
with_schema(monkeypatch)
|
||||
models = Models(monkeypatch, writer=(TEXT, "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客首单的贡献要看 `refund_detail` 表。"
|
||||
"下面看细节。"),
|
||||
verdicts={"主要来自新客": "unsupported"})
|
||||
verdicts={"主要来自新客": "contradicted"})
|
||||
row = await finish(weekly(claims="judge", judge={"rewrite_once": True}))
|
||||
assert row.status == "succeeded", row.error
|
||||
assert len(models.writes) == 2 and len(models.judges) == 1
|
||||
@@ -362,7 +363,7 @@ async def test_a_rewrite_that_trades_one_violation_for_another_is_not_used(monke
|
||||
"""去掉一处违规、又冒出另一处:总数没变,可冒出来的是原稿没有的,一样不采用。"""
|
||||
draft = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单,共 1200 单。下面看细节。"
|
||||
swapped = "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客首单约占 3500 单。下面看细节。"
|
||||
models = Models(monkeypatch, writer=(draft, draft, swapped), verdicts={"主要来自新客": "unsupported"})
|
||||
models = Models(monkeypatch, writer=(draft, draft, swapped), verdicts={"主要来自新客": "contradicted"})
|
||||
row = await finish(weekly(claims="judge", on_violation="flag", max_repairs=1, judge={"rewrite_once": True}))
|
||||
assert row.status == "succeeded", row.error
|
||||
assert len(models.writes) == 3 and len(models.judges) == 1
|
||||
@@ -385,7 +386,7 @@ def test_new_violations_are_counted_one_by_one():
|
||||
|
||||
async def test_a_second_round_that_hits_a_limit_names_the_configured_limit(monkeypatch):
|
||||
"""第二轮用的是第一轮剩下的钱,可说给人看的上限是节点上配的那个数,不是剩下的零头。"""
|
||||
Models(monkeypatch, writer=(TEXT, REWRITTEN), verdicts={"主要来自新客": "unsupported"})
|
||||
Models(monkeypatch, writer=(TEXT, REWRITTEN), verdicts={"主要来自新客": "contradicted"})
|
||||
# 每句估 1 分钱、回复不带用量(实际就按估算记):第一轮两句花掉 2 分,第二轮只剩半分,一句也装不下
|
||||
monkeypatch.setattr(judge_mod.JudgeRequest, "cost", lambda self, model_id, units: 0.01 * len(units))
|
||||
monkeypatch.setattr(judge_mod, "_usage_of", lambda message, model_id: {"input_tokens": 0, "output_tokens": 0})
|
||||
|
||||
@@ -12,7 +12,7 @@ import { datasourceTools, modelOptions, providerOfModel, useCatalog, useDatasour
|
||||
import { api } from '../api/client'
|
||||
import { IconButton, JsonInput, Modal, isComposing, useRadioGroup } from '../components/ui'
|
||||
import { formatShortcut } from '../lib/keys'
|
||||
import { JUDGE_FIELD_LABEL, JUDGE_ON_UNSUPPORTED_LABEL, JUDGE_SETTING_TEXT, SUBGRAPH_UPGRADE_HELP, judgeByModelText, upgradeNewerText } from '../lib/terms'
|
||||
import { JUDGE_FIELD_LABEL, JUDGE_ON_UNSUPPORTED_HINT, JUDGE_ON_UNSUPPORTED_LABEL, JUDGE_SETTING_TEXT, SUBGRAPH_UPGRADE_HELP, judgeByModelText, upgradeNewerText } from '../lib/terms'
|
||||
import { TemplateText } from './TemplateText'
|
||||
import type { NodeType, ValidationIssue, WorkflowVersion } from '../types'
|
||||
|
||||
@@ -624,7 +624,8 @@ function FieldInput({ field, id, nodeId, syntax, value, config, invalid, describ
|
||||
const JUDGE_SUB_KEYS = new Set(['provider', 'model', 'max_claims', 'max_cost_usd', 'timeout_s', 'rewrite_once', 'on_unsupported'])
|
||||
|
||||
/**
|
||||
* 结论句裁判(claims: judge)的子配置:裁判模型、三项上限(各自可以设成不限)、改写一次、证据不支持时怎么判档。
|
||||
* 结论句裁判(claims: judge)的子配置:裁判模型、三项上限(各自可以设成不限)、改写一次、证据相矛盾时怎么判档
|
||||
* (键名仍是 on_unsupported;证据不足的句子最多降档)。
|
||||
*
|
||||
* 上限有三种写法,得分得清:没写(跟随设置里「证据裁判」的默认值)、写数、写 null(不限)。勾「不限」写 null,
|
||||
* 清空数字框回到没写。全都没写时整个 judge 去掉,不留一个空对象。受管级别发布要求写了 max_cost_usd(数或 null)
|
||||
@@ -705,7 +706,7 @@ function JudgeConfig({ id, value, config, issues, onChange }: {
|
||||
<span>
|
||||
{JUDGE_FIELD_LABEL.rewrite_once}
|
||||
<span className="mt-0.5 block text-2xs leading-snug text-faint">
|
||||
默认关闭。开启后,裁判判定为证据不支持的句子会连同理由交回撰写模型,仅改写这些句子并重新判断一次,最多一轮。
|
||||
默认关闭。开启后,裁判判定为证据相矛盾或证据不足的句子会连同理由交回撰写模型,仅改写这些句子并重新判断一次,最多一轮。
|
||||
会额外产生一次撰写和裁判的费用;改写稿如出现原稿没有的问题,则不予采用
|
||||
</span>
|
||||
</span>
|
||||
@@ -724,7 +725,7 @@ function JudgeConfig({ id, value, config, issues, onChange }: {
|
||||
<option value={judge.on_unsupported} disabled>{judge.on_unsupported}:无法识别的值,请从上方选项中选择</option>
|
||||
)}
|
||||
</select>
|
||||
<div className="mt-1 text-2xs leading-snug text-faint">仅对正式运行生效;探索运行只做标注,不拦截</div>
|
||||
<div className="mt-1 text-2xs leading-snug text-faint" data-judge-hint="on_unsupported">{JUDGE_ON_UNSUPPORTED_HINT}</div>
|
||||
{lines('on_unsupported')}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -13,7 +13,7 @@ import { edgeKey, topology } from '../run/derive'
|
||||
import { exhaustedOf } from '../run/decode'
|
||||
import { api } from '../api/client'
|
||||
import { isComposing, StatusBadge, toast } from '../components/ui'
|
||||
import { matchedSource } from '../lib/evidence'
|
||||
import { claimCountsOf, matchedSource } from '../lib/evidence'
|
||||
import { explainRunError } from '../lib/explain'
|
||||
import { formatDuration, formatLapse, formatNumber, formatTokens, NONE, shortId } from '../lib/format'
|
||||
import { statusMeta } from '../lib/status'
|
||||
@@ -189,11 +189,12 @@ function matchedLines(matched: unknown): string {
|
||||
return `追溯明细:\n${lines.join('\n')}${more}`
|
||||
}
|
||||
|
||||
/** stats 里的裁判句数(claims 为 judge 时才有)→「结论 4 句(支持 1 · 不支持 1 · 未裁判 2)」 */
|
||||
/**
|
||||
* stats 里的裁判句数(claims 为 judge 时才有)→「结论 5 句(有依据 1 · 证据相矛盾 1 · 证据不足 1 · 未裁判 2)」。
|
||||
* 旧文档的 stats 里是 unsupported(拆档之前的取值),照样计入、照旧叫「证据不支持」
|
||||
*/
|
||||
function claimsLine(st: ReportStamp['stats']): string {
|
||||
const n = (v: unknown) => (typeof v === 'number' && Number.isFinite(v) ? v : 0)
|
||||
const c = { supported: n(st.supported), partial: n(st.partial), unsupported: n(st.unsupported), unjudged: n(st.unjudged), uncited: 0 }
|
||||
return claimTally({ ...c, total: c.supported + c.partial + c.unsupported + c.unjudged })
|
||||
return claimTally(claimCountsOf(st as Record<string, unknown>))
|
||||
}
|
||||
|
||||
/** 报告节点章的悬停说明:两个数怎么来的,缺口里各有什么 */
|
||||
|
||||
@@ -28,6 +28,7 @@ import evidenceFixture from '../run/__tests__/evidence-doc.json'
|
||||
import evidenceQuery from '../run/__tests__/evidence-query.json'
|
||||
import evidenceEntity from '../run/__tests__/evidence-entity.json'
|
||||
import evidenceJudge from '../run/__tests__/evidence-judge.json'
|
||||
import evidenceVerdict from '../run/__tests__/evidence-verdict.json'
|
||||
import { EvidenceGuessView } from '../run/EvidenceGuess'
|
||||
import { IssuanceBanner } from '../run/AssistantStream'
|
||||
import mdPinned from '../run/__tests__/markdown-pinned.json'
|
||||
@@ -199,6 +200,12 @@ const EV_ENTITY = evidenceEntity.doc as unknown as EvidenceDocData
|
||||
const EV_JUDGE = evidenceJudge.formal.doc as unknown as EvidenceDocData
|
||||
const EV_JUDGE_EXPLORE = evidenceJudge.explore.doc as unknown as EvidenceDocData
|
||||
const EV_JUDGE_PLAIN = evidenceJudge.plain.doc as unknown as EvidenceDocData
|
||||
/**
|
||||
* 裁判拆档(JR):证据相矛盾、证据不足(写明缺什么)、讲方法的句子(只挂表名字段名和整份查询)、裁判模型和写作
|
||||
* 模型相同、模型不在价格表里。探索运行那份的证据图里带着一条拆档之前留下的「证据不支持」(evidence-verdict.json)
|
||||
*/
|
||||
const EV_VERDICT = evidenceVerdict.formal.doc as unknown as EvidenceDocData
|
||||
const EV_VERDICT_EXPLORE = evidenceVerdict.explore.doc as unknown as EvidenceDocData
|
||||
/** 同一份旧契约的出具,去掉位置:老运行就是这样,只能按字符串标 */
|
||||
const LEGACY_LOOSE = {
|
||||
matched: EV_LEGACY.matched.map(({ start: _s, end: _e, ...m }) => m),
|
||||
@@ -339,18 +346,38 @@ function EvidenceOnly() {
|
||||
<EvidenceDoc doc={EV_JUDGE_PLAIN} artifact={evidenceJudge.plain.doc_artifact} runId={evidenceJudge.plain.run_id}
|
||||
label="answer" tally />
|
||||
</div>
|
||||
{/* 裁判拆档:证据相矛盾、证据不足(缺什么)、讲方法的句子挂的 SQL 和字段清单、裁判模型的两条提醒 */}
|
||||
<div id="evidence-verdict" className="rounded-lg border p-3">
|
||||
<div className="mb-1 text-2xs text-faint">正式运行 · 证据相矛盾、证据不足、讲方法的句子(裁判模型和写作模型相同,模型不在价格表里)</div>
|
||||
<EvidenceDoc doc={EV_VERDICT} artifact={evidenceVerdict.formal.doc_artifact} runId={evidenceVerdict.formal.run_id}
|
||||
runClass="formal" label="answer" tally />
|
||||
</div>
|
||||
<div id="evidence-verdict-narrow" className="rounded-lg border bg-panel p-2.5" style={{ width: 360, maxWidth: '100%' }}>
|
||||
<EvidenceDoc doc={EV_VERDICT} artifact={evidenceVerdict.formal.doc_artifact} runId={evidenceVerdict.formal.run_id}
|
||||
runClass="formal" label="answer" dense tally />
|
||||
</div>
|
||||
<div id="evidence-verdict-explore" className="rounded-lg border p-3">
|
||||
<div className="mb-1 text-2xs text-faint">探索运行 · 按需裁判,证据图里有一条拆档之前的「证据不支持」</div>
|
||||
<EvidenceDoc doc={EV_VERDICT_EXPLORE} artifact={evidenceVerdict.explore.doc_artifact}
|
||||
runId={evidenceVerdict.explore.run_id} label="answer" tally />
|
||||
</div>
|
||||
{/* 出具横幅那一行多一段结论句:「结论 4 句(支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1)」 */}
|
||||
<div id="issuance-claims" className="flex flex-col gap-2 rounded-lg border p-3">
|
||||
{/*
|
||||
judge-uncited:开了裁判的文档,没挂依据的那句送了裁判、按判定数在「支持 3」里(或预筛放掉了不数),
|
||||
出具(io.py)照样按没挂依据记 1 句——横幅得另起「没挂依据的结论句 1」,不能被结论句那一段吞掉
|
||||
*/}
|
||||
{/*
|
||||
rework:裁判拆档之后的计数(证据相矛盾、证据不足);insufficient:只有证据不足一种问题,照样给「定位下一处」
|
||||
*/}
|
||||
{([
|
||||
['judged', { total: 4, supported: 1, partial: 1, unsupported: 1, unjudged: 1, uncited: 0 }, null],
|
||||
['cited', { total: 4, supported: 3, partial: 0, unsupported: 0, unjudged: 0, uncited: 1 }, null],
|
||||
['all', { total: 2, supported: 2, partial: 0, unsupported: 0, unjudged: 0, uncited: 0 }, null],
|
||||
['judge-uncited', { total: 3, supported: 3, partial: 0, unsupported: 0, unjudged: 0, uncited: 0 },
|
||||
['judged', { total: 4, supported: 1, partial: 1, contradicted: 0, insufficient: 0, unsupported: 1, unjudged: 1, uncited: 0 }, null],
|
||||
['cited', { total: 4, supported: 3, partial: 0, contradicted: 0, insufficient: 0, unsupported: 0, unjudged: 0, uncited: 1 }, null],
|
||||
['all', { total: 2, supported: 2, partial: 0, contradicted: 0, insufficient: 0, unsupported: 0, unjudged: 0, uncited: 0 }, null],
|
||||
['judge-uncited', { total: 3, supported: 3, partial: 0, contradicted: 0, insufficient: 0, unsupported: 0, unjudged: 0, uncited: 0 },
|
||||
{ policy: 'judge', uncited_claims: 1 }],
|
||||
['rework', { total: 7, supported: 1, partial: 1, contradicted: 2, insufficient: 1, unsupported: 1, unjudged: 1, uncited: 0 }, null],
|
||||
['insufficient', { total: 3, supported: 2, partial: 0, contradicted: 0, insufficient: 1, unsupported: 0, unjudged: 0, uncited: 0 }, null],
|
||||
] as const).map(([key, claims, issued]) => (
|
||||
<div key={key} data-issuance-case={key}>
|
||||
<IssuanceBanner
|
||||
@@ -659,6 +686,7 @@ function Harness() {
|
||||
evidenceQuery,
|
||||
evidenceEntity,
|
||||
evidenceJudge,
|
||||
evidenceVerdict,
|
||||
}
|
||||
|
||||
createRoot(document.getElementById('root')!).render(
|
||||
|
||||
+191
-13
@@ -14,8 +14,12 @@
|
||||
* 按数值猜的候选用最淡的点状线。
|
||||
*
|
||||
* 四期接上了概率性的几种:结论句的判定是模型给的,只在句末挂一枚小徽标(◆ 有依据、◇ 部分有依据、
|
||||
* ! 证据不支持且整句浅底、? 未裁判),整句的字不画线。判定有两个来源:封存的文档里节点当场判的,
|
||||
* ! 证据相矛盾且整句浅底、○ 证据不足、? 未裁判),整句的字不画线。判定有两个来源:封存的文档里节点当场判的,
|
||||
* 和探索运行里点开再判、封存之后追加的(后者盖过前者,另写「封存后追加」)。
|
||||
*
|
||||
* 裁判拆档(JR):原来的「证据不支持」拆成证据相矛盾(证据和原句冲突)和证据不足(证据里没有相关信息,
|
||||
* 裁判写明缺什么)。证据不足是中性的:不用红,字形用 ○,和未裁判的 ? 分得开。旧文档、旧事件里的
|
||||
* unsupported 原样保留,外观同证据相矛盾,叫法照旧「证据不支持」。
|
||||
*/
|
||||
|
||||
import {
|
||||
@@ -25,8 +29,9 @@ import {
|
||||
import { NONE, formatNumber, shortId } from './format'
|
||||
import type {
|
||||
EvidenceBlock, EvidenceCaliberSource, EvidenceCaliberUpgrade, EvidenceCandidate, EvidenceDocData, EvidenceFieldRef,
|
||||
EvidenceGraph, EvidenceGuess, EvidenceInput, EvidenceLocator, EvidenceSeal, EvidenceSegment, EvidenceSegmentDetail,
|
||||
EvidenceStats, EvidenceStep, EvidenceUnit, EvidenceVerdict, EvidenceViolation, ReviewResult,
|
||||
EvidenceGraph, EvidenceGuess, EvidenceInput, EvidenceJudgeMeta, EvidenceJudgeSummary, EvidenceLocator, EvidenceSeal,
|
||||
EvidenceSegment, EvidenceSegmentDetail, EvidenceStats, EvidenceStep, EvidenceUnit, EvidenceVerdict, EvidenceViolation,
|
||||
ReviewResult,
|
||||
} from '../types'
|
||||
import type { MarkSpec } from '../run/Markdown'
|
||||
|
||||
@@ -80,6 +85,21 @@ export const EVIDENCE_STATE: Record<EvidenceStateCode, EvidenceStateMeta> = {
|
||||
alert: true, phase: 4,
|
||||
hint: '裁判模型认为证据仅支持本句的一部分',
|
||||
},
|
||||
contradicted: {
|
||||
code: 'contradicted', label: EVIDENCE_STATE_LABEL.contradicted, line: 'badge', glyph: '!',
|
||||
color: 'var(--st-failed)', decoration: 'transparent', soft: 'var(--st-failed-soft)',
|
||||
alert: true, phase: 4,
|
||||
hint: '裁判模型认为附带的证据与本句相冲突',
|
||||
},
|
||||
// 证据不足:中性,不用红(不是证据说它错了,是摘录里没有能判断的信息)。字形用 ○,不和未裁判的 ? 混;
|
||||
// 仍算问题(进 n / N),排在证据相矛盾、部分有依据之后
|
||||
insufficient: {
|
||||
code: 'insufficient', label: EVIDENCE_STATE_LABEL.insufficient, line: 'badge', glyph: '○',
|
||||
color: 'var(--st-cancelled)', decoration: 'transparent', soft: 'var(--st-cancelled-soft)',
|
||||
alert: true, phase: 4,
|
||||
hint: '裁判模型认为附带的证据中没有判断本句所需的信息,既不能证实也不能否定',
|
||||
},
|
||||
// 拆档之前的取值:旧文档、旧事件里保留原样。外观同证据相矛盾
|
||||
unsupported: {
|
||||
code: 'unsupported', label: EVIDENCE_STATE_LABEL.unsupported, line: 'badge', glyph: '!',
|
||||
color: 'var(--st-failed)', decoration: 'transparent', soft: 'var(--st-failed-soft)',
|
||||
@@ -129,6 +149,19 @@ export const EVIDENCE_STATE: Record<EvidenceStateCode, EvidenceStateMeta> = {
|
||||
/** 图例、检查脚本按这个顺序列 */
|
||||
export const EVIDENCE_STATES = Object.keys(EVIDENCE_STATE) as EvidenceStateCode[]
|
||||
|
||||
/**
|
||||
* n / N 跳转按两轮走:先走第 1 轮(无证据、可疑名称、证据相矛盾、旧的证据不支持、部分有依据),再走第 2 轮
|
||||
* (证据不足),每一轮里按正文顺序。不进跳转的是 0
|
||||
*/
|
||||
export function alertTier(state: EvidenceStateCode | null | undefined): 0 | 1 | 2 {
|
||||
if (!state || !EVIDENCE_STATE[state].alert) return 0
|
||||
return state === 'insufficient' ? 2 : 1
|
||||
}
|
||||
|
||||
/** 证据和原句冲突的两种(新取值、拆档之前的旧取值):整句铺浅底 */
|
||||
export const conflicting = (state: EvidenceStateCode | null | undefined): boolean =>
|
||||
state === 'contradicted' || state === 'unsupported'
|
||||
|
||||
/**
|
||||
* 片段在正文里的状态。文字、结构片段(行首符号、表格竖线)是 null:不画线、不进键盘顺序。
|
||||
* 后端的 probabilistic 在裁判给出结论之前按「未裁判」画
|
||||
@@ -543,7 +576,7 @@ export function tallySummary(t: EvidenceTally, keys: boolean): string {
|
||||
t.unverified ? `${formatNumber(t.unverified)} 个名称${EVIDENCE_STATE.unverified.label}` : '',
|
||||
t.structural ? EVIDENCE_TEXT.structuralCount(t.structural) : '',
|
||||
t.noSegment ? EVIDENCE_TEXT.noSegmentCount(t.noSegment) : '',
|
||||
keys ? (t.claims?.unsupported || t.claims?.partial ? JUDGE_TEXT.keysHint : EVIDENCE_TEXT.keysHint) : '',
|
||||
keys ? (claimProblems(t.claims) ? JUDGE_TEXT.keysHint : EVIDENCE_TEXT.keysHint) : '',
|
||||
].filter(Boolean).join('。')
|
||||
}
|
||||
|
||||
@@ -843,8 +876,11 @@ export function stampCounts(stats: EvidenceStats | null | undefined, claims?: st
|
||||
// 四期:结论句裁判
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/** 判过了的四种(不含未裁判):同后端 judge.JUDGED。判过的不再给「请模型判断这句」 */
|
||||
const JUDGED = new Set(['supported', 'partial', 'unsupported', 'not_a_claim'])
|
||||
/**
|
||||
* 判过了的几种(不含未裁判):同后端 judge.JUDGED,另加拆档之前的 unsupported(旧文档、旧事件里还有)。
|
||||
* 判过的不再给「请模型判断这句」
|
||||
*/
|
||||
const JUDGED = new Set(['supported', 'partial', 'contradicted', 'insufficient', 'unsupported', 'not_a_claim'])
|
||||
/** 触顶的四个上限:同后端 judge.LIMITS,这几种没判的写「已到上限」和怎么调 */
|
||||
export const JUDGE_LIMITS: readonly string[] = ['max_claims', 'max_cost_usd', 'daily_max_usd', 'timeout_s']
|
||||
|
||||
@@ -862,12 +898,73 @@ export function verdictState(v: Pick<EvidenceVerdict, 'status'> | null | undefin
|
||||
switch (v?.status) {
|
||||
case 'supported': return 'supported'
|
||||
case 'partial': return 'partial'
|
||||
case 'contradicted': return 'contradicted'
|
||||
case 'insufficient': return 'insufficient'
|
||||
case 'unsupported': return 'unsupported'
|
||||
case 'unjudged': return 'unjudged'
|
||||
default: return null
|
||||
}
|
||||
}
|
||||
|
||||
/** 证据不足时缺的是什么(裁判给的);别的判定、没写的是空串 */
|
||||
export function verdictMissing(v: Pick<EvidenceVerdict, 'status' | 'missing'> | null | undefined): string {
|
||||
return v?.status === 'insufficient' && typeof v.missing === 'string' ? v.missing.trim() : ''
|
||||
}
|
||||
|
||||
/** 结论句的计数里有没有要提醒的:证据相矛盾、旧的证据不支持、部分有依据、证据不足(横幅、证据条的「定位下一处」) */
|
||||
export function claimProblems(c: ClaimTallyCounts | null | undefined): number {
|
||||
if (!c) return 0
|
||||
return (c.contradicted ?? 0) + (c.unsupported ?? 0) + (c.partial ?? 0) + (c.insufficient ?? 0)
|
||||
}
|
||||
|
||||
/** 计数表里认的几种判定(「不是结论句」不算进总数) */
|
||||
const COUNTED = ['supported', 'partial', 'contradicted', 'insufficient', 'unsupported', 'unjudged'] as const
|
||||
|
||||
/**
|
||||
* 按判定分好的句数(report.checked 的 judge.counts、文档 stats 里裁判那几个键、出具的 claims.counts)→ 横幅那种
|
||||
* 计数:总数不算「不是结论句」,旧数据的 unsupported 照样计入。一个判定都没有时总数是 0
|
||||
*/
|
||||
export function claimCountsOf(counts: Record<string, unknown> | null | undefined): ClaimTallyCounts {
|
||||
const n = (k: string) => {
|
||||
const v = counts?.[k]
|
||||
return typeof v === 'number' && Number.isFinite(v) && v > 0 ? v : 0
|
||||
}
|
||||
const c: ClaimTallyCounts = { total: 0, supported: 0, partial: 0, contradicted: 0, insufficient: 0, unsupported: 0,
|
||||
unjudged: 0, uncited: 0 }
|
||||
for (const k of COUNTED) {
|
||||
c[k] = n(k)
|
||||
c.total += c[k]
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
/**
|
||||
* 这条判定该带的两条提醒:裁判模型和写作模型相同、模型不在价格表里(金额上限对它不生效)。只对模型判过的判定。
|
||||
* 按从近到远取:判定自己带着的 → 封存之后追加的看这一次按需裁判的答复、再看证据图里最近一次按需裁判
|
||||
* (reports[].judge_meta);节点里当场判的看报告文档的 judge。都没说的不提醒
|
||||
*/
|
||||
export function judgeFlags(v: EvidenceVerdict | null | undefined, from: {
|
||||
doc?: Pick<EvidenceJudgeSummary, 'same_model' | 'priced' | 'writer_model' | 'model'> | null
|
||||
reply?: { same_model?: unknown; priced?: unknown; writer_model?: unknown; model?: unknown } | null
|
||||
meta?: EvidenceJudgeMeta | null
|
||||
} = {}): { sameModel: boolean; unpriced: boolean; writer: string | null; model: string | null } {
|
||||
const none = { sameModel: false, unpriced: false, writer: null, model: null }
|
||||
if (!v || !isJudged(v)) return none
|
||||
const layers = (v.post_seal ? [v, from.reply, from.meta] : [v, from.doc]).filter(Boolean) as Record<string, unknown>[]
|
||||
const pick = <T,>(key: string, ok: (x: unknown) => x is T): T | undefined => {
|
||||
for (const l of layers) if (ok(l[key])) return l[key] as T
|
||||
return undefined
|
||||
}
|
||||
const isBool = (x: unknown): x is boolean => typeof x === 'boolean'
|
||||
const isText = (x: unknown): x is string => typeof x === 'string' && !!x
|
||||
return {
|
||||
sameModel: pick('same_model', isBool) === true,
|
||||
unpriced: pick('priced', isBool) === false,
|
||||
writer: pick('writer_model', isText) ?? null,
|
||||
model: (isText(v.judge) ? v.judge : pick('model', isText)) ?? null,
|
||||
}
|
||||
}
|
||||
|
||||
/** 一句的判定:封存之后按需追加的盖过文档里的(探索运行里文档记的是「未裁判 · 按需」) */
|
||||
export function unitVerdict(
|
||||
unit: Pick<EvidenceUnit, 'id' | 'verdict'>, overlay?: Record<string, EvidenceVerdict> | null,
|
||||
@@ -891,15 +988,17 @@ export function claimCounts(
|
||||
doc: Pick<EvidenceDocData, 'blocks' | 'judge'>, overlay?: Record<string, EvidenceVerdict> | null,
|
||||
): ClaimTallyCounts | null {
|
||||
const judged = !!doc.judge && typeof doc.judge === 'object'
|
||||
const c: ClaimTallyCounts = { total: 0, supported: 0, partial: 0, unsupported: 0, unjudged: 0, uncited: 0 }
|
||||
const c: ClaimTallyCounts = { total: 0, supported: 0, partial: 0, contradicted: 0, insufficient: 0, unsupported: 0,
|
||||
unjudged: 0, uncited: 0 }
|
||||
let seen = judged
|
||||
for (const block of doc.blocks ?? []) {
|
||||
for (const unit of block.units ?? []) {
|
||||
const v = unitVerdict(unit, overlay)
|
||||
if (v) seen = true
|
||||
if (v) {
|
||||
if (v.status === 'supported' || v.status === 'partial' || v.status === 'unsupported' || v.status === 'unjudged') {
|
||||
c[v.status] += 1
|
||||
const k = (COUNTED as readonly string[]).includes(v.status) ? v.status as typeof COUNTED[number] : null
|
||||
if (k) {
|
||||
c[k] += 1
|
||||
c.total += 1
|
||||
}
|
||||
continue
|
||||
@@ -932,12 +1031,13 @@ const clip = (text: string, n = 40) => (text.length > n ? `${text.slice(0, n - 1
|
||||
|
||||
/**
|
||||
* 句末徽标的 aria-label:句子、判定都写全,封存后追加的另说。「结论句「增长主要来自新客首单…」,
|
||||
* 模型判断:证据不支持」;到上限没判的说「未裁判:已到上限」
|
||||
* 模型判断:证据相矛盾」;证据不足的接着说缺什么;到上限没判的说「未裁判:已到上限」
|
||||
*/
|
||||
export function claimLabel(unit: Pick<EvidenceUnit, 'segments'>, v: EvidenceVerdict): string {
|
||||
const state = verdictState(v)
|
||||
const said = state ? EVIDENCE_STATE[state].label : JUDGE_TEXT.notClaim
|
||||
const why = limitOf(v) ? `${said}:${JUDGE_TEXT.limit}` : said
|
||||
const missing = verdictMissing(v)
|
||||
const why = limitOf(v) ? `${said}:${JUDGE_TEXT.limit}` : missing ? `${said},${JUDGE_TEXT.missing(missing)}` : said
|
||||
return JUDGE_TEXT.badgeLabel(clip(unitText(unit).trim()), why, !!v.post_seal)
|
||||
}
|
||||
|
||||
@@ -954,6 +1054,7 @@ function asVerdict(v: unknown, fallback: { model?: unknown; postSeal: boolean })
|
||||
post_seal: typeof o.post_seal === 'boolean' ? o.post_seal : fallback.postSeal,
|
||||
used: Array.isArray(o.used) ? o.used.map(String) : undefined,
|
||||
reason: typeof o.reason === 'string' ? o.reason : undefined,
|
||||
missing: typeof o.missing === 'string' && o.missing.trim() ? o.missing : undefined,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1007,6 +1108,68 @@ export function rewriteOf(doc: Pick<EvidenceDocData, 'judge'>, uid: string):
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* 讲方法、讲结构的句子(JR 第 3 条):写作者只挂了表名、字段名或整份查询([[see:Qn]]),没挂数字、值、口径卡、
|
||||
* 引文。这类句子的「挂的依据」下直接列出对应的 SQL 和字段清单——裁判看的就是这些,读的人也该看得到。
|
||||
*
|
||||
* 返回要列的东西:表(字段清单)、字段(类型)各自在句中的片段(字段清单、类型从这个片段的证据链取),
|
||||
* 要列 SQL 的查询(直接挂的在前,再补表名字段名出现过的查询,最多 BASIS_QUERIES 个,多的只写个数)。
|
||||
* 挂了别的依据、或者一个表名字段名和查询都没挂的,返回 null(照旧只列编号)
|
||||
*/
|
||||
export const BASIS_QUERIES = 3
|
||||
export interface ClaimBasis {
|
||||
tables: { alias: string; name: string; seg?: string }[]
|
||||
columns: { alias: string; name: string; seg?: string }[]
|
||||
queries: string[]
|
||||
more: number
|
||||
}
|
||||
export function claimBasis(
|
||||
unit: Pick<EvidenceUnit, 'cites' | 'see' | 'segments'>, doc: Pick<EvidenceDocData, 'catalog' | 'blocks'>,
|
||||
): ClaimBasis | null {
|
||||
const catalog = doc.catalog ?? {}
|
||||
const cites = (unit.cites ?? []).filter((a): a is string => typeof a === 'string' && !!a)
|
||||
if (!cites.length) return null
|
||||
const entity = (kind: unknown) => kind === 'table' || kind === 'column'
|
||||
// 句中写作者标的引用(自动链接的名字不算依据):数字、值、单元格、口径卡、引文都算「挂了数据」
|
||||
if ((unit.segments ?? []).some((s) => s.cite && !s.auto && s.cite.status === 'resolved' && !entity(s.cite.kind))) return null
|
||||
if ((unit.see ?? []).some((c) => c?.status === 'resolved' && !entity(c.kind) && c.kind !== 'query')) return null
|
||||
const out: ClaimBasis = { tables: [], columns: [], queries: [], more: 0 }
|
||||
const segOf = (alias: string): string | undefined => {
|
||||
const hit = (u: Pick<EvidenceUnit, 'segments'>) =>
|
||||
(u.segments ?? []).find((s) => s.cite?.alias === alias && s.state === 'deterministic')?.id
|
||||
return hit(unit) ?? (doc.blocks ?? []).flatMap((b) => b.units ?? []).map(hit).find(Boolean)
|
||||
}
|
||||
const nameOf = (alias: string, entry: Record<string, any> | undefined) => {
|
||||
const loc = entry?.locator ?? {}
|
||||
if (entry?.kind === 'column' && loc.table && loc.column) return `${loc.table}.${loc.column}`
|
||||
return typeof entry?.name === 'string' && entry.name ? entry.name : alias.replace(/^[tc]:/, '')
|
||||
}
|
||||
const queries: string[] = []
|
||||
for (const alias of cites) {
|
||||
const entry = catalog[alias]
|
||||
if (entry?.kind === 'table') out.tables.push({ alias, name: nameOf(alias, entry), seg: segOf(alias) })
|
||||
else if (entry?.kind === 'column') out.columns.push({ alias, name: nameOf(alias, entry), seg: segOf(alias) })
|
||||
else if (entry?.kind === 'query') queries.push(alias)
|
||||
else return null
|
||||
}
|
||||
for (const e of [...out.tables, ...out.columns]) {
|
||||
for (const q of Array.isArray(catalog[e.alias]?.queries) ? catalog[e.alias].queries : []) {
|
||||
if (typeof q === 'string' && catalog[q]?.kind === 'query' && !queries.includes(q)) queries.push(q)
|
||||
}
|
||||
}
|
||||
out.queries = queries.slice(0, BASIS_QUERIES)
|
||||
out.more = Math.max(0, queries.length - BASIS_QUERIES)
|
||||
return out
|
||||
}
|
||||
|
||||
/** 查询快照工件里的 SQL:快照本身带 sql;工具调用的快照({tool, args, result})在 args 或 result 里 */
|
||||
export function sqlOf(content: unknown): string | null {
|
||||
if (!content || typeof content !== 'object') return null
|
||||
const o = content as { sql?: unknown; args?: { sql?: unknown }; result?: { sql?: unknown } }
|
||||
const sql = [o.sql, o.args?.sql, o.result?.sql].find((x) => typeof x === 'string' && x.trim())
|
||||
return typeof sql === 'string' ? sql : null
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// 旧运行的猜测
|
||||
// -------------------------------------------------------------------------
|
||||
@@ -1082,11 +1245,22 @@ export interface AuditRow {
|
||||
sentence: string
|
||||
/** 这件证据在不在封存范围内(封存核对也通过);没有证据的是 null */
|
||||
sealed: boolean | null
|
||||
/** 结论句那几行:是哪一句(正文里有句末徽标时点这一行打开它的「模型的解释」) */
|
||||
unit?: string
|
||||
}
|
||||
|
||||
const GROUP_OF: Record<EvidenceStateCode, AuditGroup> = {
|
||||
deterministic: 'cited', none: 'none', suspect: 'suspicious', unverified: 'suspicious', candidate: 'candidate',
|
||||
supported: 'cited', partial: 'none', unsupported: 'none', unjudged: 'cited', connective: 'cited',
|
||||
supported: 'cited', partial: 'none', contradicted: 'none', insufficient: 'none', unsupported: 'none', unjudged: 'cited',
|
||||
connective: 'cited',
|
||||
}
|
||||
/** 审计行带的判定 → 行的状态:有问题的几种才有自己的行,认不出的交给后面按 state 认 */
|
||||
const claimRowState = (status: unknown): EvidenceStateCode | undefined =>
|
||||
status === 'contradicted' || status === 'insufficient' || status === 'unsupported' || status === 'partial' ? status : undefined
|
||||
/** 审计行的 issue → 结论句的判定(后端按约定给证据相矛盾、证据不足的结论句各一行;旧取值也认) */
|
||||
const CLAIM_ISSUE: Record<string, EvidenceStateCode> = {
|
||||
contradicted_claim: 'contradicted', insufficient_claim: 'insufficient', unsupported_claim: 'unsupported',
|
||||
partial_claim: 'partial',
|
||||
}
|
||||
const AUDIT_GROUP_SET = new Set<string>(['cited', 'none', 'suspicious', 'candidate'])
|
||||
|
||||
@@ -1111,7 +1285,10 @@ export function auditFromApi(body: unknown): AuditRow[] {
|
||||
for (const [ri, r] of (Array.isArray(g?.rows) ? g.rows : []).entries()) {
|
||||
if (!r || typeof r !== 'object') continue
|
||||
const kind = String(r.kind ?? '')
|
||||
const state = segmentState({ kind, state: r.state ?? 'none', issue: r.issue ?? undefined })
|
||||
// 结论句的判定行:按 issue 认成对应的判定(外观、叫法同句末徽标);issue 认不出时看行上带的 verdict
|
||||
const state = (kind === 'claim' && typeof r.issue === 'string' ? CLAIM_ISSUE[r.issue] : undefined)
|
||||
?? (kind === 'claim' && r.verdict && typeof r.verdict === 'object' ? claimRowState(r.verdict.status) : undefined)
|
||||
?? segmentState({ kind, state: r.state ?? 'none', issue: r.issue ?? undefined })
|
||||
?? (r.group === 'candidate' ? 'candidate' : 'none')
|
||||
const group: AuditGroup = AUDIT_GROUP_SET.has(r.group) ? r.group : AUDIT_GROUP_SET.has(g?.key) ? g.key : GROUP_OF[state]
|
||||
const report = String(r.report ?? r.field ?? '')
|
||||
@@ -1126,6 +1303,7 @@ export function auditFromApi(body: unknown): AuditRow[] {
|
||||
ref: typeof r.ref === 'string' ? r.ref : undefined,
|
||||
sentence: String(r.sentence ?? '').trim(),
|
||||
sealed: typeof r.sealed === 'boolean' ? r.sealed : null,
|
||||
...(kind === 'claim' && typeof r.unit === 'string' && r.unit ? { unit: r.unit } : {}),
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
+52
-10
@@ -155,13 +155,16 @@ export const APPROVAL_POLICY_LABEL: Record<'dangerous' | 'always' | 'never', str
|
||||
* 可点击证据:片段状态的叫法。正文里片段的 aria-label、面板标题、图例都用这一份。
|
||||
* 外观(线型、字形、颜色)在 lib/evidence.ts,和这里一一对应。
|
||||
*
|
||||
* 概率性的四种(四期的结论句裁判)挂在句末的徽标上:有依据、部分有依据、证据不支持都是模型的判断,
|
||||
* 概率性的几种(四期的结论句裁判)挂在句末的徽标上:有依据、部分有依据、证据相矛盾、证据不足都是模型的判断,
|
||||
* 叫法里写明「模型判断」;未裁判不带「有引用」——没挂依据、只有方向词的结论句也会送裁判、也会没判。
|
||||
* 证据不支持是裁判拆档之前的取值:已封存的文档、已有的事件里保留原样,照旧这样叫(新判的不再出现)。
|
||||
*/
|
||||
export const EVIDENCE_STATE_LABEL = {
|
||||
deterministic: '有出处',
|
||||
supported: '模型判断:有依据',
|
||||
partial: '模型判断:部分有依据',
|
||||
contradicted: '模型判断:证据相矛盾',
|
||||
insufficient: '模型判断:证据不足',
|
||||
unsupported: '模型判断:证据不支持',
|
||||
unjudged: '未裁判',
|
||||
none: '无证据',
|
||||
@@ -313,6 +316,10 @@ export const EVIDENCE_TEXT = {
|
||||
checked: (schemas: number, queries: number) =>
|
||||
`已核对 ${formatNumber(schemas)} 份表结构快照、${formatNumber(queries)} 次查询`,
|
||||
tableColumns: (n: number, view: boolean) => `${view ? '视图' : '表'} · ${formatNumber(n)} 个字段`,
|
||||
/** 表名实体步骤的字段清单(来自封存范围内的表结构快照,最多列 60 个,遮罩的列不列入) */
|
||||
fields: (n: number) => `字段清单(${formatNumber(n)} 个)`,
|
||||
fieldsMore: (n: number) => `另有 ${formatNumber(n)} 个字段未列出`,
|
||||
fieldsMasked: '部分字段已按数据源的遮罩设置隐藏,未列出',
|
||||
columnTypes: (types: Record<string, string>) =>
|
||||
`各表的类型:${Object.entries(types).map(([t, v]) => `${t} ${v}`).join('、')}`,
|
||||
quoteBad: '按记录的位置从原文中截取的内容与这句引文不一致:原文可能已被修改',
|
||||
@@ -342,6 +349,11 @@ export interface ClaimTallyCounts {
|
||||
total: number
|
||||
supported: number
|
||||
partial: number
|
||||
/** 证据和原句冲突 */
|
||||
contradicted: number
|
||||
/** 证据里没有判断所需的信息 */
|
||||
insufficient: number
|
||||
/** 拆档之前的取值(旧文档、旧事件),照旧计入 */
|
||||
unsupported: number
|
||||
/** 送了裁判、没判成(到上限、调用失败)或还没请模型判断的 */
|
||||
unjudged: number
|
||||
@@ -356,8 +368,9 @@ export interface ClaimTallyCounts {
|
||||
export function claimTally(c: ClaimTallyCounts): string {
|
||||
if (!c.total) return ''
|
||||
const parts = [
|
||||
['有依据', c.supported], ['部分有依据', c.partial], ['证据不支持', c.unsupported], ['未裁判', c.unjudged], ['无证据', c.uncited],
|
||||
].filter(([, n]) => (n as number) > 0).map(([k, n]) => `${k} ${formatNumber(n as number)}`)
|
||||
['有依据', c.supported], ['部分有依据', c.partial], ['证据相矛盾', c.contradicted], ['证据不支持', c.unsupported],
|
||||
['证据不足', c.insufficient], ['未裁判', c.unjudged], ['无证据', c.uncited],
|
||||
].filter(([, n]) => typeof n === 'number' && n > 0).map(([k, n]) => `${k} ${formatNumber(n as number)}`)
|
||||
return `结论 ${formatNumber(c.total)} 句${parts.length ? `(${parts.join(' · ')})` : ''}`
|
||||
}
|
||||
|
||||
@@ -368,8 +381,11 @@ export function claimTally(c: ClaimTallyCounts): string {
|
||||
export const JUDGE_TEXT = {
|
||||
section: '模型的解释',
|
||||
claim: '结论句',
|
||||
/** 读屏摘要末尾的操作说明:有证据不支持、部分支持的结论句时 n 也跳到它们的句末徽标 */
|
||||
keysHint: '用左右方向键逐个查看,上下方向键按句切换,n 跳到下一处无证据或证据不支持的句子,回车打开证据,Esc 关闭',
|
||||
/**
|
||||
* 读屏摘要末尾的操作说明:结论句有问题(证据相矛盾、部分有依据、证据不足)时 n 也跳到它们的句末徽标。
|
||||
* 证据不足的排在其余问题之后
|
||||
*/
|
||||
keysHint: '用左右方向键逐个查看,上下方向键按句切换,n 跳到下一处无证据或证据有问题的句子(证据不足的排在最后),回车打开证据,Esc 关闭',
|
||||
/** 面板里判断的徽标:谁判的、而且不是确定的 */
|
||||
badge: (model: string | null | undefined) => `模型判断 · ${model || '裁判模型'} · 非确定`,
|
||||
postSeal: '封存后追加',
|
||||
@@ -389,11 +405,30 @@ export const JUDGE_TEXT = {
|
||||
notClaim: '模型判断:不是结论句',
|
||||
ask: '请模型判断本句',
|
||||
askAgain: '再次请模型判断',
|
||||
/** 封存后追加的判定早于当前的裁判规则(片段接口的 on_demand.reason 为 outdated):旁边注明,按钮照常给 */
|
||||
outdated: '按旧规则判定',
|
||||
asking: '正在请模型判断…',
|
||||
askHint: '探索运行按需裁判:每次点击裁判一句,费用计入单次点击上限和每日上限',
|
||||
notAsked: '尚未请模型判断本句',
|
||||
screened: '预筛判定本句不陈述数据事实(短句或过渡语),未送裁判',
|
||||
noPrice: '裁判模型不在价格目录中,无法按 token 数估算金额:金额上限对其不生效',
|
||||
/** 证据不足时缺的是什么(裁判给的,如「SQL」「字段清单」):面板、句末徽标的悬停说明和读屏都写 */
|
||||
missingLead: '缺少:',
|
||||
missing: (what: string) => `缺少:${what}`,
|
||||
/** 裁判模型的两条提醒:判定照常给出,读的人要知道分量 */
|
||||
sameModel: '裁判模型和写作模型相同,结果仅供参考',
|
||||
sameModelTitle: (model: string) => `裁判模型与撰写这份报告的模型都是 ${model}:同一模型较难发现自身的错误`,
|
||||
noPrice: '该模型不在价格表中,金额上限不生效',
|
||||
noPriceTitle: '无法按 token 数估算这个模型的费用,裁判的金额上限(每份报告、每次点击、每日)对它不起作用;句数和时长上限照常生效',
|
||||
/** 讲方法、讲结构的句子(只挂了表名、字段名、整份查询):「挂的依据」下直接列出 SQL 和字段清单 */
|
||||
basisSql: (alias: string) => `查询 ${alias} 的 SQL`,
|
||||
basisFields: (table: string) => `表 ${table} 的字段清单`,
|
||||
basisFieldsCount: (table: string, n: number) => `表 ${table} 的字段清单(${formatNumber(n)} 个)`,
|
||||
basisColumns: '字段类型',
|
||||
basisPending: '正在加载 SQL 和字段清单…',
|
||||
sqlMissing: '未能获取这次查询的 SQL',
|
||||
fieldsMissing: '未能获取字段清单',
|
||||
basisNoRun: '这份报告不属于任何运行,无法获取 SQL 和字段清单',
|
||||
basisMoreQueries: (n: number) => `另有 ${formatNumber(n)} 次查询,点开句中的表名查看`,
|
||||
limit: '已达上限',
|
||||
/** 触顶之后怎么调。按需裁判(每次点击)和正式运行(每份报告)调的地方不一样 */
|
||||
limitHow: {
|
||||
@@ -437,10 +472,14 @@ export const JUDGE_FIELD_LABEL: Record<string, string> = {
|
||||
max_claims: '最多裁判句数',
|
||||
max_cost_usd: '金额上限(美元)',
|
||||
timeout_s: '时长上限(秒)',
|
||||
rewrite_once: '证据不支持的句子退回改写一次',
|
||||
on_unsupported: '证据不支持时',
|
||||
rewrite_once: '证据相矛盾或不足的句子退回改写一次',
|
||||
// 键名仍是 on_unsupported(兼容已有配置);裁判拆档后它管的是「证据相矛盾」(旧取值证据不支持照旧按它判),
|
||||
// 证据不足的句子最多降档
|
||||
on_unsupported: '证据相矛盾时',
|
||||
}
|
||||
/** judge.on_unsupported:正式运行里证据不支持的结论句怎么判档(探索运行只标注) */
|
||||
/** judge.on_unsupported 的说明:检查器里选项下面那一行 */
|
||||
export const JUDGE_ON_UNSUPPORTED_HINT = '仅对正式运行生效;证据不足的句子最多降档,不会因此不予出具。探索运行只做标注,不拦截'
|
||||
/** judge.on_unsupported:正式运行里证据相矛盾的结论句怎么判档(探索运行只标注) */
|
||||
export const JUDGE_ON_UNSUPPORTED_LABEL: Record<string, string> = {
|
||||
degrade: '出具降档',
|
||||
withhold: '不予出具',
|
||||
@@ -459,6 +498,9 @@ export const JUDGE_SETTING_TEXT = {
|
||||
modelDefault: '留空:用接入的默认模型',
|
||||
modelFollow: '留空:跟随助手的模型',
|
||||
differ: '建议与撰写报告的模型不同:由同一模型审核自身输出,不易发现其中的错误',
|
||||
/** 接入和模型都没填:后端依次用助手的模型、默认接入。醒目地说一句,并给出建议 */
|
||||
notSet: '未单独配置,将使用助手的模型',
|
||||
notSetAdvice: '建议配置一个与撰写报告的模型不同、能力更强的模型:同一模型审核自身输出时,结果仅供参考',
|
||||
limits: '上限(每一项都可以设成不限)',
|
||||
reportClaims: '每份报告最多裁判句数',
|
||||
reportCost: '每份报告的金额上限(美元)',
|
||||
@@ -532,7 +574,7 @@ export const EVIDENCE_AUDIT_TEXT = {
|
||||
candidate: '按数值猜测',
|
||||
} as Record<string, string>,
|
||||
groupHint: {
|
||||
none: '裸数字、无法解析的引用、未附依据的结论句',
|
||||
none: '裸数字、无法解析的引用、未附依据或模型判为证据有问题的结论句',
|
||||
suspicious: '表名、字段名在本次运行的表结构和查询中找不到',
|
||||
cited: '系统从证据中取值并核对过的片段',
|
||||
candidate: '按数值猜测的可能来源,不能作为证据,默认收起',
|
||||
|
||||
@@ -2,7 +2,7 @@ import { useEffect, useRef, useState } from 'react'
|
||||
import type { KeyboardEvent as ReactKeyboardEvent, ReactNode } from 'react'
|
||||
import { Navigate, useParams } from 'react-router-dom'
|
||||
import {
|
||||
Check, Cpu, Download, KeyRound, Monitor, Moon, Plug, Plus, Settings as SettingsIcon, Sun, X,
|
||||
AlertTriangle, Check, Cpu, Download, KeyRound, Monitor, Moon, Plug, Plus, Settings as SettingsIcon, Sun, X,
|
||||
} from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { api } from '../api/client'
|
||||
@@ -777,6 +777,8 @@ function PrefsTab() {
|
||||
const judgeProviderMissing = !!cur.judgeProvider && providers.length > 0 && !judgeProvider
|
||||
const judgeModelName = cur.judgeModel.trim()
|
||||
const judgeByModel = !cur.judgeProvider && !!judgeModelName
|
||||
// 裁判模型没有单独配置(接入、模型都留空):后端依次用助手的模型、默认接入
|
||||
const judgeNotSet = !cur.judgeProvider && !judgeModelName
|
||||
const judgeOwner = judgeByModel ? providerOfModel(providers, judgeModelName) : undefined
|
||||
const judgeByModelHint = judgeByModel && providers.length > 0
|
||||
? judgeByModelText('settings', judgeModelName, judgeOwner && { name: judgeOwner.provider.name, enabled: judgeOwner.enabled },
|
||||
@@ -940,7 +942,21 @@ function PrefsTab() {
|
||||
<SectionBar title={JUDGE_SETTING_TEXT.label} hint={JUDGE_SETTING_TEXT.hint} />
|
||||
<div role="group" aria-labelledby="pref-judge-model-label" aria-describedby="pref-judge-model-hint" data-judge-model>
|
||||
<div id="pref-judge-model-label" className="text-xs font-medium">裁判模型</div>
|
||||
<p id="pref-judge-model-hint" className="mb-2 mt-0.5 text-2xs leading-relaxed text-faint">{JUDGE_SETTING_TEXT.differ}</p>
|
||||
{judgeNotSet ? (
|
||||
// 接入和模型都没填:裁判会用助手的模型(往往就是写报告的那个)。醒目地说,并给出建议;
|
||||
// 填了任何一项就换回平常那句建议
|
||||
<div id="pref-judge-model-hint" role="note" data-judge-not-set=""
|
||||
className="mb-2 mt-1 flex items-start gap-1.5 rounded border-l-2 py-0.5 pl-2 text-2xs leading-relaxed"
|
||||
style={{ borderColor: 'var(--st-waiting)' }}>
|
||||
<AlertTriangle size={12} aria-hidden className="mt-0.5 shrink-0" style={{ color: 'var(--st-waiting)' }} />
|
||||
<span>
|
||||
<span className="font-medium" data-judge-not-set-text="">{JUDGE_SETTING_TEXT.notSet}</span>
|
||||
<span className="block text-dim" data-judge-not-set-advice="">{JUDGE_SETTING_TEXT.notSetAdvice}</span>
|
||||
</span>
|
||||
</div>
|
||||
) : (
|
||||
<p id="pref-judge-model-hint" className="mb-2 mt-0.5 text-2xs leading-relaxed text-faint">{JUDGE_SETTING_TEXT.differ}</p>
|
||||
)}
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<Field label={JUDGE_SETTING_TEXT.provider} htmlFor="pref-judge-provider"
|
||||
error={judgeProviderMissing ? `模型接入「${cur.judgeProvider}」不存在:裁判调用将失败,所有结论句记为未裁判` : undefined}
|
||||
|
||||
@@ -8,8 +8,8 @@ import { ApiError, api } from '../../api/client'
|
||||
import { isUpgradeAdvice } from '../../canvas/issues'
|
||||
import { ErrorState, Skeleton, Spinner, StatusBadge, rovingTarget, toast, useRadioGroup } from '../../components/ui'
|
||||
import {
|
||||
EVIDENCE_STATE, PROBLEM_GROUPS, auditCsv, auditFromApi, auditRows, groupRows, guessRows, guessesOf, issuanceMarks,
|
||||
segmentState, type AuditFilter, type AuditGroup, type AuditRow,
|
||||
EVIDENCE_STATE, PROBLEM_GROUPS, auditCsv, auditFromApi, auditRows, graphVerdicts, groupRows, guessRows, guessesOf,
|
||||
issuanceMarks, segmentState, unitVerdict, verdictState, type AuditFilter, type AuditGroup, type AuditRow,
|
||||
} from '../../lib/evidence'
|
||||
import { formatNumber } from '../../lib/format'
|
||||
import { EVIDENCE_AUDIT_TEXT as T, UPGRADE_TEXT } from '../../lib/terms'
|
||||
@@ -87,20 +87,28 @@ export function EvidencePane({ run, output, labelOf, refreshKey }: {
|
||||
|
||||
// 哪些行能在正文里打开:`报告:片段` 的集合,一份文档算一次(长报告几千个片段,不能每行各扫一遍)。
|
||||
// 只收有状态的片段,和 EvidenceDoc 能打开的是同一批:违规行指着的文字片段(粗体、链接里的可疑名字)、
|
||||
// 结构片段(列表序号 100. 里的数字)在正文里没有能点的地方,画成按钮就是点了没反应
|
||||
// 结构片段(列表序号 100. 里的数字)在正文里没有能点的地方,画成按钮就是点了没反应。
|
||||
// 结论句的行(证据相矛盾、证据不足……)打开的是这一句的句末徽标(`报告:claim:句子`):句子有判定(封存的、
|
||||
// 证据图里封存后追加的)才有徽标
|
||||
const openable = useMemo(() => {
|
||||
const keys = new Set<string>()
|
||||
for (const r of reports) {
|
||||
const doc = docOf(r.node_id)
|
||||
for (const b of doc?.blocks ?? []) for (const u of b.units ?? []) for (const seg of u.segments ?? []) {
|
||||
if (segmentState(seg)) keys.add(`${r.node_id}:${seg.id}`)
|
||||
const late = graphVerdicts(graph, r.node_id)
|
||||
for (const b of doc?.blocks ?? []) for (const u of b.units ?? []) {
|
||||
for (const seg of u.segments ?? []) if (segmentState(seg)) keys.add(`${r.node_id}:${seg.id}`)
|
||||
if (verdictState(unitVerdict(u, late))) keys.add(`${r.node_id}:claim:${u.id}`)
|
||||
}
|
||||
}
|
||||
return keys
|
||||
}, [reports, docOf])
|
||||
const canOpen = useCallback((row: AuditRow) => !!row.seg && openable.has(`${row.report}:${row.seg}`), [openable])
|
||||
}, [reports, docOf, graph])
|
||||
const canOpen = useCallback((row: AuditRow) => {
|
||||
const key = openKey(row)
|
||||
return !!key && openable.has(`${row.report}:${key}`)
|
||||
}, [openable])
|
||||
const openRow = useCallback((row: AuditRow) => {
|
||||
if (row.seg) handles.current.get(row.report)?.open(row.seg)
|
||||
const key = openKey(row)
|
||||
if (key) handles.current.get(row.report)?.open(key)
|
||||
}, [])
|
||||
|
||||
const onView = useCallback((report: string, open: boolean) => {
|
||||
@@ -188,6 +196,10 @@ export function EvidencePane({ run, output, labelOf, refreshKey }: {
|
||||
)
|
||||
}
|
||||
|
||||
/** 审计行在正文里打开什么:片段的行是那个片段,结论句的行是那一句的句末徽标(EvidenceDoc 认 claim:<句子>) */
|
||||
const openKey = (row: AuditRow): string | undefined =>
|
||||
row.seg ?? (row.kind === 'claim' && row.unit ? `claim:${row.unit}` : undefined)
|
||||
|
||||
/** 已经收尾的运行:还在跑、停在审批上的,报告可能还没写,说「这次的报告没有逐段证据」太早 */
|
||||
const SETTLED = new Set(['succeeded', 'failed', 'cancelled'])
|
||||
|
||||
@@ -558,7 +570,7 @@ function AuditLine({ row, current, openable, multiReport, labelOf, onOpen, onFoc
|
||||
data-audit-seg={row.seg ?? ''}>
|
||||
<td className="px-3 py-1.5">
|
||||
{openable ? (
|
||||
<button type="button" {...common} aria-label={label} title={label} onClick={onOpen} data-audit-open={row.seg}
|
||||
<button type="button" {...common} aria-label={label} title={label} onClick={onOpen} data-audit-open={openKey(row)}
|
||||
className={clsx(common.className, 'block underline decoration-dotted underline-offset-2 hover:text-[var(--accent)]')}>
|
||||
{text}
|
||||
</button>
|
||||
|
||||
@@ -23,7 +23,7 @@ import {
|
||||
} from './decode'
|
||||
import { CODE_COLUMN, CopyChip, LEADING_ZERO, Markdown, type MarkSpec } from './Markdown'
|
||||
import { ClaimTally, EvidenceField, type EvidenceDocHandle } from './EvidenceDoc'
|
||||
import { EVIDENCE_STATE, docTally, evidenceFields, issuanceMarks, type EvidenceTally } from '../lib/evidence'
|
||||
import { EVIDENCE_STATE, claimProblems, docTally, evidenceFields, issuanceMarks, type EvidenceTally } from '../lib/evidence'
|
||||
import type { EvidenceDocData } from '../types'
|
||||
import { useRunClock } from './useRunClock'
|
||||
import type { ReviewResult } from '../types'
|
||||
@@ -2289,7 +2289,7 @@ export function IssuanceBanner({ issuance, runClass, evidence }: {
|
||||
{meta.hint && <div className="mt-1 text-2xs leading-relaxed text-dim">{meta.hint}</div>}
|
||||
{counts && (counts.total > 0 || counts.other > 0 || suspect > 0 || uncitedClaims > 0 || !!claims) && (
|
||||
// 逐段证据的计数:和报告核对那一行、证据条同一种说法。可疑名字、没挂依据的结论句不是数字,
|
||||
// 各自另起一句。有无证据的数字、可疑名字、证据不支持(或部分支持)的结论句时才给「定位下一处」——
|
||||
// 各自另起一句。有无证据的数字、可疑名字、证据有问题(相矛盾、部分有依据、证据不足)的结论句时才给「定位下一处」——
|
||||
// 结论句靠句末徽标定位;只是没挂依据的结论句没有徽标,定位不到
|
||||
<div className="mt-1 flex flex-wrap items-center gap-2 text-2xs" data-evidence-line="">
|
||||
{numbers && (
|
||||
@@ -2307,7 +2307,7 @@ export function IssuanceBanner({ issuance, runClass, evidence }: {
|
||||
{numbers || suspect > 0 || claims ? '· ' : ''}{EVIDENCE_TEXT.uncitedClaimsTag} {formatNumber(uncitedClaims)}
|
||||
</span>
|
||||
)}
|
||||
{(counts.none > 0 || counts.other > 0 || suspect > 0 || !!claims?.unsupported || !!claims?.partial) && evidence?.onNext && (
|
||||
{(counts.none > 0 || counts.other > 0 || suspect > 0 || claimProblems(claims) > 0) && evidence?.onNext && (
|
||||
<button type="button" className="btn btn-xs" data-evidence-next="" onClick={evidence.onNext}>
|
||||
{EVIDENCE_TEXT.locateNext}
|
||||
</button>
|
||||
|
||||
@@ -8,8 +8,9 @@ import { ListChecks } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { rovingTarget } from '../components/ui'
|
||||
import {
|
||||
EVIDENCE_STATE, claimCounts, claimLabel, docTally, segmentKind, segmentLabel, segmentState, tallySummary, unitText,
|
||||
isJudged, limitOf, unitVerdict, verdictState, withVerdicts, type EvidenceStateCode, type EvidenceTally as Tally,
|
||||
EVIDENCE_STATE, alertTier, claimCounts, claimLabel, claimProblems, conflicting, docTally, segmentKind, segmentLabel,
|
||||
segmentState, tallySummary, unitText, isJudged, limitOf, unitVerdict, verdictMissing, verdictState, withVerdicts,
|
||||
type EvidenceStateCode, type EvidenceTally as Tally,
|
||||
} from '../lib/evidence'
|
||||
import { formatNumber } from '../lib/format'
|
||||
import { EVIDENCE_STATE_LABEL, EVIDENCE_TEXT, JUDGE_TEXT, claimTally, evidenceTally, type ClaimTallyCounts } from '../lib/terms'
|
||||
@@ -26,13 +27,13 @@ import { BlockView, Markdown, StyledSlice, inlineStyles, numericColumns, type In
|
||||
* BlockView,和普通答案长得一模一样;有出处的数字画细实线,没有出处的画点状线、
|
||||
* 句末挂「无证据」。
|
||||
*
|
||||
* 结论句的判定(四期)只在句末挂一枚小徽标(◆ ◇ ! ?),整句不画线;证据不支持的整句铺一层浅底。
|
||||
* 徽标也是按钮:点开是这一句的「模型的解释」。判定是模型给的,颜色一律不用确定性的绿。
|
||||
* 结论句的判定(四期)只在句末挂一枚小徽标(◆ ◇ ! ○ ?),整句不画线;证据相矛盾(和旧的证据不支持)的整句
|
||||
* 铺一层浅底。徽标也是按钮:点开是这一句的「模型的解释」。判定是模型给的,颜色一律不用确定性的绿。
|
||||
*
|
||||
* 键盘:整份报告只占一个 Tab 位(roving tabindex)。←/→ 在有状态的片段、句末徽标间走,↑/↓ 按
|
||||
* 句子走(表格里按列走到上一行 / 下一行,出了表格接着按句子走),n / N 跳到下一处 /
|
||||
* 上一处无证据、证据不支持的地方,回车打开证据面板,Esc 关掉并留在原片段;面板开着时方向键走到哪、
|
||||
* 面板跟到哪,Tab 进面板。
|
||||
* 上一处有问题的地方(无证据、可疑名称、证据相矛盾、部分有依据;证据不足的排在这些之后),回车打开证据面板,
|
||||
* Esc 关掉并留在原片段;面板开着时方向键走到哪、面板跟到哪,Tab 进面板。
|
||||
*
|
||||
* 长报告:事件都挂在容器上(片段只带 data-seg),块用 memo,外加
|
||||
* content-visibility: auto;超过 TEXT_CAP 的部分先折叠,跳转到折叠里的片段时自动展开。
|
||||
@@ -66,8 +67,13 @@ interface Model {
|
||||
/** 每个有可点片段的句子里第一个可点片段,按正文顺序(↑/↓ 用) */
|
||||
unitFirst: string[]
|
||||
unitOf: Map<string, number>
|
||||
/** 无证据的片段,按正文顺序(n / N 用) */
|
||||
/**
|
||||
* 有问题的片段和句末徽标(n / N 用):先第 1 轮(无证据、可疑名称、证据相矛盾、部分有依据),再第 2 轮
|
||||
* (证据不足),每一轮里按正文顺序
|
||||
*/
|
||||
alerts: string[]
|
||||
/** alerts 里每一处在第几轮 */
|
||||
tier: Map<string, 1 | 2>
|
||||
/** 每块到它为止的累计字数(折叠用) */
|
||||
chars: number[]
|
||||
/** 表格里的片段在第几行第几列(表头 row = -1 不进来);↑/↓ 在表格里按列走 */
|
||||
@@ -83,7 +89,7 @@ function buildModel(blocks: EvidenceBlock[], overlay?: Record<string, EvidenceVe
|
||||
const badge = new Map<string, Badge>()
|
||||
const unitFirst: string[] = []
|
||||
const unitOf = new Map<string, number>()
|
||||
const alerts: string[] = []
|
||||
const tiered: [string, 1 | 2][] = []
|
||||
const chars: number[] = []
|
||||
const cell = new Map<string, { block: number; row: number; col: number }>()
|
||||
const grid = new Map<string, string>()
|
||||
@@ -101,14 +107,15 @@ function buildModel(blocks: EvidenceBlock[], overlay?: Record<string, EvidenceVe
|
||||
order.push(s.id)
|
||||
if (first) { unitFirst.push(s.id); first = false }
|
||||
unitOf.set(s.id, unitFirst.length - 1)
|
||||
if (EVIDENCE_STATE[state].alert) alerts.push(s.id)
|
||||
const t = alertTier(state)
|
||||
if (t) tiered.push([s.id, t])
|
||||
if (loc) {
|
||||
cell.set(s.id, { block: b, row: loc.row, col: loc.col })
|
||||
const key = `${b}:${loc.row}:${loc.col}`
|
||||
if (!grid.has(key)) grid.set(key, s.id)
|
||||
}
|
||||
}
|
||||
// 句末徽标排在这一句的片段后面:←/→ 走到句尾就是它,证据不支持的也进 n / N
|
||||
// 句末徽标排在这一句的片段后面:←/→ 走到句尾就是它,证据相矛盾、部分有依据、证据不足的也进 n / N
|
||||
const verdict = unitVerdict(unit, overlay)
|
||||
const state = verdictState(verdict)
|
||||
if (verdict && state) {
|
||||
@@ -118,12 +125,16 @@ function buildModel(blocks: EvidenceBlock[], overlay?: Record<string, EvidenceVe
|
||||
order.push(key)
|
||||
if (first) { unitFirst.push(key); first = false }
|
||||
unitOf.set(key, unitFirst.length - 1)
|
||||
if (EVIDENCE_STATE[state].alert) alerts.push(key)
|
||||
const t = alertTier(state)
|
||||
if (t) tiered.push([key, t])
|
||||
}
|
||||
}
|
||||
chars.push(total)
|
||||
})
|
||||
return { order, index, seg, badge, unitFirst, unitOf, alerts, chars, cell, grid }
|
||||
// 按(轮次、正文顺序)排:tiered 本来就是正文顺序,稳定排序只按轮次挪
|
||||
const sorted = [...tiered].sort((a, b) => a[1] - b[1])
|
||||
return { order, index, seg, badge, unitFirst, unitOf, alerts: sorted.map(([k]) => k), tier: new Map(sorted), chars, cell,
|
||||
grid }
|
||||
}
|
||||
|
||||
/** 片段或徽标在第几块 */
|
||||
@@ -311,11 +322,17 @@ export const EvidenceDoc = forwardRef<EvidenceDocHandle, {
|
||||
if (refocus && back) requestAnimationFrame(() => segEl(back)?.focus())
|
||||
}, [view, current, segEl])
|
||||
|
||||
/**
|
||||
* n / N 的下一处:按(轮次、正文位置)排的一圈,走到头接回开头。当前位置不是问题(普通片段、有依据的徽标)时
|
||||
* 按第 1 轮算——第 1 轮里它后面没有了,就接着走第 2 轮(证据不足)
|
||||
*/
|
||||
const step = useCallback((list: string[], from: string | null, dir: 1 | -1): string | null => {
|
||||
if (!list.length) return null
|
||||
const pos = from ? model.index.get(from) ?? -1 : -1
|
||||
if (dir > 0) return list.find((id) => (model.index.get(id) ?? 0) > pos) ?? list[0]
|
||||
for (let i = list.length - 1; i >= 0; i--) if ((model.index.get(list[i]) ?? 0) < pos) return list[i]
|
||||
const rank = (id: string | null): [number, number] => [id ? model.tier.get(id) ?? 1 : 1, id ? model.index.get(id) ?? -1 : -1]
|
||||
const cmp = (a: [number, number], b: [number, number]) => a[0] - b[0] || a[1] - b[1]
|
||||
const cur = rank(from)
|
||||
if (dir > 0) return list.find((id) => cmp(rank(id), cur) > 0) ?? list[0]
|
||||
for (let i = list.length - 1; i >= 0; i--) if (cmp(rank(list[i]), cur) < 0) return list[i]
|
||||
return list[list.length - 1]
|
||||
}, [model])
|
||||
|
||||
@@ -510,7 +527,7 @@ export function EvidenceTally({ counts, onNext, onList }: {
|
||||
}) {
|
||||
const suspect = counts.suspect ?? 0
|
||||
const claims = counts.claims
|
||||
const clean = !counts.none && !counts.other && !suspect && !claims?.unsupported && !claims?.partial
|
||||
const clean = !counts.none && !counts.other && !suspect && !claimProblems(claims)
|
||||
const numbers = evidenceTally(counts.cited, counts.total, counts.other)
|
||||
return (
|
||||
<div className="mb-1.5 flex flex-wrap items-center gap-x-2 gap-y-1 text-2xs" data-evidence-tally="">
|
||||
@@ -541,13 +558,14 @@ export function EvidenceTally({ counts, onNext, onList }: {
|
||||
}
|
||||
|
||||
/**
|
||||
* 「结论 4 句(支持 3 · 不支持 1)」:和数字那一段同一种说法。判定是模型给的:全都支持也不用确定性的绿,
|
||||
* 有不支持、部分支持、没挂依据的用提醒色,其余用暗字色(它是概率性的,不该和「有出处」一样笃定)
|
||||
* 「结论 4 句(有依据 3 · 证据相矛盾 1)」:和数字那一段同一种说法。判定是模型给的:全都支持也不用确定性的绿,
|
||||
* 有问题(证据相矛盾、旧的证据不支持、部分有依据、证据不足)、没挂依据的用提醒色,其余用暗字色(它是概率性的,
|
||||
* 不该和「有出处」一样笃定)
|
||||
*/
|
||||
export function ClaimTally({ claims, lead }: { claims: ClaimTallyCounts; lead: boolean }) {
|
||||
const text = claimTally(claims)
|
||||
if (!text) return null
|
||||
const warn = claims.unsupported > 0 || claims.partial > 0 || claims.uncited > 0
|
||||
const warn = claimProblems(claims) > 0 || claims.uncited > 0
|
||||
return (
|
||||
<span className="tnum" style={{ color: warn ? 'var(--st-waiting)' : 'var(--text-dim)' }} data-evidence-claims=""
|
||||
title={JUDGE_TEXT.badge(null)}>
|
||||
@@ -655,7 +673,7 @@ const DocBlock = memo(function DocBlock({ block, dense, doc, visual, current, ac
|
||||
/**
|
||||
* 一句话:文字按行内 Markdown 渲染(整句算一次样式,跨片段的 `code`、**粗体** 不会
|
||||
* 被切坏),有状态的片段包成按钮。含无证据片段的句子末尾挂「? 无证据」;有判定的结论句末尾挂
|
||||
* 一枚徽标(也是按钮),证据不支持的整句铺浅底——字本身不画线
|
||||
* 一枚徽标(也是按钮),证据相矛盾(和旧的证据不支持)的整句铺浅底——字本身不画线
|
||||
*/
|
||||
function UnitView({ unit, doc, current, active, panelId, code, tag, badge }: {
|
||||
unit: EvidenceUnit; doc: EvidenceDocData; current: string | null; active: string | null; panelId: string
|
||||
@@ -702,8 +720,8 @@ function UnitView({ unit, doc, current, active, panelId, code, tag, badge }: {
|
||||
const open = !!badge && active === key
|
||||
return (
|
||||
<>
|
||||
{badge?.state === 'unsupported'
|
||||
// 证据不支持:整句铺一层浅底(折行时每行各自带圆角),字色、字重都不动,也不画线
|
||||
{conflicting(badge?.state)
|
||||
// 证据相矛盾(和旧的证据不支持):整句铺一层浅底(折行时每行各自带圆角),字色、字重都不动,也不画线
|
||||
? <span className="ev-claim" data-ev-claim-shade={unit.id} style={{ '--ev-soft': meta!.soft } as CSSProperties}>{nodes}</span>
|
||||
: nodes}
|
||||
{tag && [...alerts].map((state) => (
|
||||
@@ -722,10 +740,11 @@ function UnitView({ unit, doc, current, active, panelId, code, tag, badge }: {
|
||||
aria-label={claimLabel(unit, badge.verdict)}
|
||||
aria-expanded={open}
|
||||
aria-controls={open ? panelId : undefined}
|
||||
// 模型判过的写明谁判的、非确定;模型没判过的(按需还没点、到上限、没跑成)只说这是什么状态——
|
||||
// 到上限没判的也记着裁判模型,但不能写成「(模型 · 非确定)」,像是它判过一样
|
||||
// 模型判过的写明谁判的、非确定(证据不足的接着写缺什么);模型没判过的(按需还没点、到上限、没跑成)只说
|
||||
// 这是什么状态——到上限没判的也记着裁判模型,但不能写成「(模型 · 非确定)」,像是它判过一样
|
||||
title={isJudged(badge.verdict)
|
||||
? `${meta.label}${badge.verdict.post_seal ? ` · ${JUDGE_TEXT.postSeal}` : ''}(${badge.verdict.judge || JUDGE_TEXT.judgeModel} · 非确定)`
|
||||
? `${meta.label}${badge.verdict.post_seal ? ` · ${JUDGE_TEXT.postSeal}` : ''}(${badge.verdict.judge || JUDGE_TEXT.judgeModel} · 非确定)${
|
||||
verdictMissing(badge.verdict) ? `。${JUDGE_TEXT.missing(verdictMissing(badge.verdict))}` : ''}`
|
||||
: limitOf(badge.verdict) ? `${meta.label}:${JUDGE_TEXT.limitNotJudged}` : `${meta.label}:${meta.hint}`}
|
||||
style={segStyle(badge.state)}
|
||||
>
|
||||
|
||||
@@ -4,16 +4,17 @@ import {
|
||||
} from 'react'
|
||||
import { createPortal } from 'react-dom'
|
||||
import { Link } from 'react-router-dom'
|
||||
import { ArrowLeft, CornerDownRight, ListChecks, Scale, ShieldCheck, X } from 'lucide-react'
|
||||
import { AlertTriangle, ArrowLeft, CornerDownRight, ListChecks, Scale, ShieldCheck, X } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { ApiError } from '../api/client'
|
||||
import { Spinner, StatusBadge, isComposing } from '../components/ui'
|
||||
import {
|
||||
EVIDENCE_KIND_STYLE, EVIDENCE_STATE, caliberSourceText, caliberUpgradeText, closestNames, docForeign, entitySources,
|
||||
evidenceTrace, evidenceValue as valueText, graphDoc, inputSource, integrityFailures, isJudged, isJudgeLimit, judgeLimitWords,
|
||||
judgeable, limitOf, pointsToSettings,
|
||||
EVIDENCE_KIND_STYLE, EVIDENCE_STATE, caliberSourceText, caliberUpgradeText, claimBasis, closestNames, docForeign,
|
||||
entitySources, evidenceTrace, evidenceValue as valueText, graphDoc, inputSource, integrityFailures, isJudged,
|
||||
isJudgeLimit, judgeFlags, judgeLimitWords, judgeable, judgedVerdicts, limitOf, pointsToSettings,
|
||||
locatable, locatorText, notableInput, queryOf, queryWindow, quoteWhere, quoteWindow, reasonOf, rewriteOf, sealVerdict,
|
||||
segName, segmentKind, segmentState, sourceOf, unitText, unitVerdict, verdictState, type SealStatus, type SourceTone,
|
||||
segName, segmentKind, segmentState, sourceOf, unitText, unitVerdict, verdictMissing, verdictState, type ClaimBasis as Basis,
|
||||
type SealStatus, type SourceTone,
|
||||
} from '../lib/evidence'
|
||||
import { humanizeError } from '../lib/errors'
|
||||
import { formatDateTime, formatNumber, NONE, shortId } from '../lib/format'
|
||||
@@ -54,6 +55,9 @@ import { CopyChip } from './Markdown'
|
||||
* 的徽标,封存之后按需追加的另写「封存后追加」。探索运行里还没判过的句子给「请模型判断这句」;到了上限
|
||||
* 写明是哪个上限、到哪里调。点句末徽标打开的是整句(kind: 'unit'):句子、模型的解释、挂的依据、封存。
|
||||
*
|
||||
* 裁判拆档(JR):证据不足的写明缺什么;裁判模型和写作模型相同、模型不在价格表里时在「模型的解释」下提醒;
|
||||
* 表名实体步骤列出字段清单;讲方法、讲结构的句子在「挂的依据」下直接列出 SQL 和字段清单。
|
||||
*
|
||||
* 四种摆法:side 从右侧弹出(宽屏),drawer 从底部抽出(窄屏),inline 在 360px 的
|
||||
* 画布右栏里直接栏内展开,dock 放进页面给的一块常驻位置(记录页的「证据」页签)。
|
||||
* 都不是模态的:开着面板照样能在正文里走。
|
||||
@@ -387,7 +391,7 @@ function SegmentBody({ panelId, doc, artifact, runId, runClass, seg, unit, block
|
||||
)}
|
||||
|
||||
<JudgePart doc={doc} unit={unit} block={block} runId={runId} runClass={runClass}
|
||||
onDemand={detail?.unit?.on_demand} pending={pending} />
|
||||
onDemand={detail?.unit?.on_demand} answered={detail?.unit?.verdict} pending={pending} />
|
||||
|
||||
{host.onNode && (trace.producers.length > 0 || trace.consumers.length > 0) && (
|
||||
// 画布右栏:这段证据经过的节点,点一下在画布上选中并对准
|
||||
@@ -465,6 +469,8 @@ function ClaimBody({ doc, artifact, runId, runClass, unit, block }: {
|
||||
const sealLabel = sealed.status !== 'done' ? sealed.label
|
||||
: verdict && !late && isJudged(verdict) ? JUDGE_TEXT.sealedWithDoc : JUDGE_TEXT.sealedDoc
|
||||
const cites = (unit.cites ?? []).map((alias) => ({ alias, label: doc.catalog?.[alias]?.label as string | undefined }))
|
||||
// 讲方法、讲结构的句子:SQL 和字段清单直接列在挂的依据下面。正文不是封存的那份时不列(链是另一份报告的)
|
||||
const basis = useMemo(() => (foreign ? null : claimBasis(unit, doc)), [foreign, unit, doc])
|
||||
return (
|
||||
<>
|
||||
{foreign && (
|
||||
@@ -476,6 +482,7 @@ function ClaimBody({ doc, artifact, runId, runClass, unit, block }: {
|
||||
</Part>
|
||||
<JudgePart doc={doc} unit={unit} block={block} runId={runId} runClass={runClass}
|
||||
onDemand={foreign ? undefined : answered?.unit?.on_demand}
|
||||
answered={foreign ? undefined : answered?.unit?.verdict}
|
||||
pending={!!runId && !!first && (!slot || slot.status === 'loading')} />
|
||||
<Part title={JUDGE_TEXT.cites} data-ev-cites="">
|
||||
{cites.length ? (
|
||||
@@ -488,6 +495,7 @@ function ClaimBody({ doc, artifact, runId, runClass, unit, block }: {
|
||||
))}
|
||||
</ul>
|
||||
) : <p className="text-dim">{JUDGE_TEXT.noCites}</p>}
|
||||
{basis && <BasisPart basis={basis} doc={doc} runId={runId} />}
|
||||
</Part>
|
||||
<Part title={EVIDENCE_TEXT.seal} data-ev-seal="">
|
||||
<SealLine status={sealed.status} label={sealLabel} />
|
||||
@@ -508,13 +516,15 @@ function ClaimBody({ doc, artifact, runId, runClass, unit, block }: {
|
||||
* 另写「封存后追加」;到上限没判的写明哪个上限、到哪里调;改写过一次的写明原句。探索运行里还没判过的
|
||||
* (没有判定、未裁判)给「请模型判断这句」。什么都说不上的(正式运行、没开裁判的文档)不画这一节
|
||||
*/
|
||||
function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
|
||||
function JudgePart({ doc, unit, block, runId, runClass, onDemand, answered, pending }: {
|
||||
doc: EvidenceDocData; unit: EvidenceUnit; block: EvidenceBlock; runId?: string; runClass?: string
|
||||
/**
|
||||
* 片段接口答的「这句能不能请模型判断」:有它就照它(运行还没跑完封存、封存被改过这些前端认不出来);
|
||||
* 老后端没有时按运行类别和文档自己认
|
||||
*/
|
||||
onDemand?: EvidenceOnDemand
|
||||
/** 片段接口答这句时的判定:接口说它「早于当前规则」(outdated),指的就是这一条 */
|
||||
answered?: EvidenceVerdict
|
||||
/** 片段接口还在取:先不按运行类别另取一次 */
|
||||
pending?: boolean
|
||||
}) {
|
||||
@@ -532,7 +542,10 @@ function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
|
||||
const late = !!verdict?.post_seal
|
||||
const rewrite = rewriteOf(doc, unit.id)
|
||||
const judgeDoc = !!doc.judge
|
||||
const canAsk = !!runId && !isJudged(verdict) && (known ? !!known.available : eligible && explore === true)
|
||||
// 片段接口说这句封存后追加的判定早于当前的裁判规则(旧取值、改造前判的):照常给按钮,旧判定旁边注明「按旧规则判定」。
|
||||
// 这一页里已经按当前规则重判过(眼前的判定不再是接口答的那一条)就不算了;重判没判成时答复交回的还是那条旧判定
|
||||
const outdated = known?.reason === 'outdated' && !!known.available && isJudged(verdict) && sameVerdict(verdict, answered)
|
||||
const canAsk = !!runId && (outdated || (!isJudged(verdict) && (known ? !!known.available : eligible && explore === true)))
|
||||
// 探索运行里暂时判不了(还没跑完封存、封存被改过、升级前的运行):照接口的原话说为什么没有按钮
|
||||
const blocked = known && !known.available && BLOCKED.has(known.reason ?? '') ? known.message || '' : ''
|
||||
// 这一次按需裁判的答复里没有这一句的判定:接口说了为什么(跳过了、触顶)就照说
|
||||
@@ -542,6 +555,16 @@ function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
|
||||
: (Array.isArray(res?.limits_hit) && res.limits_hit.find((l) => typeof l === 'string')) || null
|
||||
const shownLimit = limit ?? resLimit
|
||||
const judged = isJudged(verdict)
|
||||
const missing = verdictMissing(verdict)
|
||||
// 裁判模型的两条提醒(和写作模型相同、不在价格表里):封存后追加的看这一次的答复(答复里有这一句时)、再看证据图里
|
||||
// 最近一次按需裁判;节点里当场判的看文档的裁判摘要
|
||||
const replied = res && judgedVerdicts(res)[unit.id] ? res : null
|
||||
const graph = useEvidence((s) => (runId ? s.graphs[runId] : undefined))
|
||||
const loadGraph = useEvidence((s) => s.loadGraph)
|
||||
const needMeta = !!runId && judged && late && !replied && !graph
|
||||
useEffect(() => { if (needMeta && runId) void loadGraph(runId) }, [needMeta, runId, loadGraph])
|
||||
const meta0 = (graph?.data?.reports ?? []).find((r) => !report || !r?.node_id || r.node_id === report)?.judge_meta
|
||||
const flags = judgeFlags(verdict, { doc: doc.judge, reply: replied, meta: meta0 })
|
||||
// 这一次答复的那句话(message):判定之外接口还想说的,照原话写出来。最要紧的是「运行已经接着跑了,这次的
|
||||
// 判定没有记进运行记录」——判定照样交回来、也标着封存后追加,只看徽标会以为它记进去了。已经由别处说过的不重复:
|
||||
// 跳过的那一行就是它、上限的那一框说的就是它、和判定里写的理由一字不差
|
||||
@@ -573,11 +596,43 @@ function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
|
||||
</div>
|
||||
)}
|
||||
{verdict && (
|
||||
<p className="font-medium" style={{ color: meta?.color ?? 'var(--text-dim)' }} data-ev-verdict={verdict.status}>
|
||||
{meta?.glyph && <span aria-hidden className="mr-1">{meta.glyph}</span>}
|
||||
{meta ? meta.label : JUDGE_TEXT.notClaim}
|
||||
<div className="flex flex-wrap items-center gap-x-1.5 gap-y-0.5">
|
||||
<p className="font-medium" style={{ color: meta?.color ?? 'var(--text-dim)' }} data-ev-verdict={verdict.status}>
|
||||
{meta?.glyph && <span aria-hidden className="mr-1">{meta.glyph}</span>}
|
||||
{meta ? meta.label : JUDGE_TEXT.notClaim}
|
||||
</p>
|
||||
{outdated && (
|
||||
<span className="chip text-2xs text-faint" data-ev-judge-outdated="" title={known?.message ?? undefined}>
|
||||
{JUDGE_TEXT.outdated}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{missing && (
|
||||
// 证据不足:裁判写明缺的是什么(「SQL」「字段清单」)。说的是证据,不是结论错了:中性字色
|
||||
<p className="mt-0.5 [overflow-wrap:anywhere]" data-ev-missing="">
|
||||
<span className="text-faint">{JUDGE_TEXT.missingLead}</span>{missing}
|
||||
</p>
|
||||
)}
|
||||
{(flags.sameModel || flags.unpriced) && (
|
||||
// 判定照常给出,但读的人要知道它的分量:左侧一道提醒色的竖线加图标,字用正文色——醒目,不刺眼
|
||||
<div className="mt-1 space-y-0.5 border-l-2 py-px pl-2 text-2xs leading-relaxed" role="note" data-ev-judge-caution=""
|
||||
style={{ borderColor: 'var(--st-waiting)' }}>
|
||||
{flags.sameModel && (
|
||||
<p className="flex items-start gap-1" data-ev-judge-same=""
|
||||
title={flags.model ? JUDGE_TEXT.sameModelTitle(flags.writer ?? flags.model) : undefined}>
|
||||
<AlertTriangle size={11} aria-hidden className="mt-0.5 shrink-0" style={{ color: 'var(--st-waiting)' }} />
|
||||
<span>{JUDGE_TEXT.sameModel}</span>
|
||||
</p>
|
||||
)}
|
||||
{flags.unpriced && (
|
||||
<p className="flex items-start gap-1" data-ev-judge-unpriced="" title={JUDGE_TEXT.noPriceTitle}>
|
||||
<AlertTriangle size={11} aria-hidden className="mt-0.5 shrink-0" style={{ color: 'var(--st-waiting)' }} />
|
||||
<span>{JUDGE_TEXT.noPrice}</span>
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{showMessage && (
|
||||
<p className="mb-1 mt-0.5 text-2xs leading-relaxed" role="note" style={{ color: 'var(--st-waiting)' }} data-ev-judge-message="">
|
||||
{resMessage}
|
||||
@@ -588,8 +643,9 @@ function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
|
||||
<p className="mt-0.5 text-2xs text-faint" data-ev-judge-model="">{JUDGE_TEXT.notJudgedBy(verdict.judge)}</p>
|
||||
)}
|
||||
{verdict?.rationale && !shownLimit && (
|
||||
// 理由是模型写的话:斜体,和系统给的说明分开
|
||||
<p className="mt-0.5 italic text-dim" data-ev-rationale=""><em>{verdict.rationale}</em></p>
|
||||
// 理由是模型写的话:斜体,和系统给的说明分开。最长 120 字、可能夹着 SQL 片段和长的字段名:照原样折行,
|
||||
// 不截断,长串也在栏内断开
|
||||
<p className="mt-0.5 whitespace-pre-line italic text-dim [overflow-wrap:anywhere]" data-ev-rationale=""><em>{verdict.rationale}</em></p>
|
||||
)}
|
||||
{shownLimit && (
|
||||
<div className="mt-1 rounded border px-2 py-1" role="note" data-ev-judge-limit={shownLimit}
|
||||
@@ -656,6 +712,9 @@ function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
|
||||
{asking ? <Spinner size={10} /> : <Scale size={11} aria-hidden />}
|
||||
{asking ? JUDGE_TEXT.asking : verdict && verdict.reason !== 'on_demand' ? JUDGE_TEXT.askAgain : JUDGE_TEXT.ask}
|
||||
</button>
|
||||
{outdated && known?.message && (
|
||||
<span className="basis-full text-2xs text-dim" data-ev-judge-outdated-why="">{known.message}</span>
|
||||
)}
|
||||
<span className="basis-full text-2xs text-faint">{JUDGE_TEXT.askHint}</span>
|
||||
</div>
|
||||
)}
|
||||
@@ -663,6 +722,13 @@ function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
|
||||
)
|
||||
}
|
||||
|
||||
/** 两条判定是不是同一条(片段接口答的、页面上叠着的):按取值、理由、裁判模型、规则版本比,不按对象引用 */
|
||||
function sameVerdict(a: EvidenceVerdict | undefined, b: EvidenceVerdict | undefined): boolean {
|
||||
if (!a || !b) return false
|
||||
const sig = (v: EvidenceVerdict) => [v.status, v.rationale ?? '', v.judge ?? '', v.rule_version ?? ''].join('\u0001')
|
||||
return sig(a) === sig(b)
|
||||
}
|
||||
|
||||
/** 接口说这句暂时判不了、值得告诉人为什么的几种(正式运行、判过了、不是结论句不用说) */
|
||||
const BLOCKED = new Set(['unsealed', 'seal_broken', 'legacy'])
|
||||
|
||||
@@ -735,6 +801,9 @@ function EntityPart({ step, entry, cite, seal, pending, failed, catalog }: {
|
||||
<span className="text-2xs text-dim">{EVIDENCE_TEXT.tableColumns((step as any).columns, !!step?.is_view)}</span>
|
||||
)}
|
||||
</div>
|
||||
{kind === 'table' && !!step?.fields?.length && (
|
||||
<FieldList fields={step.fields} more={step.fields_more} masked={step.fields_masked === true} />
|
||||
)}
|
||||
{step?.comment && <p className="mt-0.5 text-2xs text-dim">{step.comment}</p>}
|
||||
{owners.length > 1 && <p className="mt-0.5 text-2xs text-dim">{EVIDENCE_TEXT.ownerTables(owners)}</p>}
|
||||
{step?.types && !step.type && Object.keys(step.types).length > 0 && (
|
||||
@@ -769,6 +838,125 @@ function EntityPart({ step, entry, cite, seal, pending, failed, catalog }: {
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 表的字段清单:证据接口给的 fields(封存范围内的表结构快照,最多 60 个,遮罩的列不列入),多出来的写「另有 N 个」,
|
||||
* 快照只存了部分表时注明(partial,实体步骤那一节另有这句时不重复)。长名字、长类型都在栏内折行,不截断
|
||||
*/
|
||||
function FieldList({ fields, more, partial, masked, label }: {
|
||||
fields: NonNullable<EvidenceStep['fields']>; more?: number; partial?: boolean; masked?: boolean; label?: (n: number) => string
|
||||
}) {
|
||||
const list = fields.filter((f) => f && typeof f.name === 'string' && f.name)
|
||||
const extra = typeof more === 'number' && Number.isFinite(more) && more > 0 ? more : 0
|
||||
const total = list.length + extra
|
||||
return (
|
||||
<div className="mt-1.5" data-ev-fields="">
|
||||
<div className="mb-0.5 text-2xs font-medium text-faint">{label ? label(total) : EVIDENCE_TEXT.fields(total)}</div>
|
||||
<ul className="grid max-h-44 grid-cols-[repeat(auto-fill,minmax(8.5rem,1fr))] gap-x-3 gap-y-px overflow-y-auto rounded bg-bg px-1.5 py-1 text-2xs">
|
||||
{list.map((f) => (
|
||||
<li key={f.name} className="min-w-0 [overflow-wrap:anywhere]" data-ev-field={f.name}>
|
||||
<span className="mono">{f.name}</span>
|
||||
{f.type && <>{' '}<span className="mono ml-1 text-faint" data-ev-field-type="">{f.type}</span></>}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
{extra > 0 && <p className="mt-0.5 text-2xs text-dim" data-ev-fields-more="">{EVIDENCE_TEXT.fieldsMore(extra)}</p>}
|
||||
{masked && <p className="mt-0.5 text-2xs text-dim" data-ev-fields-masked="">{EVIDENCE_TEXT.fieldsMasked}</p>}
|
||||
{partial && (
|
||||
<p className="mt-0.5 text-2xs" style={{ color: 'var(--st-waiting)' }} data-ev-fields-partial="">{EVIDENCE_TEXT.snapshotPartial}</p>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 讲方法、讲结构的句子挂的依据:表的字段清单、字段的类型(取句中那个名字片段的证据链:封存范围内的表结构快照)、
|
||||
* 对应的 SQL(查询快照工件,取回时后端复验哈希)。直接摆出来,不用再点开句中的名字
|
||||
*/
|
||||
function BasisPart({ basis, doc, runId }: { basis: Basis; doc: EvidenceDocData; runId?: string }) {
|
||||
const report = doc.node_id || undefined
|
||||
const segments = useEvidence((s) => s.segments)
|
||||
const sqls = useEvidence((s) => s.sqls)
|
||||
const loadSegment = useEvidence((s) => s.loadSegment)
|
||||
const loadSql = useEvidence((s) => s.loadSql)
|
||||
const entities = [...basis.tables, ...basis.columns]
|
||||
const artifacts = basis.queries.map((q) => {
|
||||
const a = doc.catalog?.[q]?.artifact
|
||||
return typeof a === 'string' && a ? a : ''
|
||||
})
|
||||
const segKey = entities.map((e) => e.seg ?? '').join(',')
|
||||
const artKey = artifacts.join(',')
|
||||
useEffect(() => {
|
||||
if (!runId) return
|
||||
for (const e of entities) if (e.seg) void loadSegment(runId, e.seg, report)
|
||||
for (const a of artifacts) if (a) void loadSql(a)
|
||||
// 按内容比:同一句不重复取
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [runId, report, segKey, artKey, loadSegment, loadSql])
|
||||
if (!runId) return <p className="mt-1.5 text-2xs text-faint" data-ev-basis="none">{JUDGE_TEXT.basisNoRun}</p>
|
||||
const stepOf = (seg?: string) => {
|
||||
const slot = seg ? segments[segmentKey(runId, seg, report)] : undefined
|
||||
const step = slot?.status === 'ok' ? (slot.data?.chain ?? []).find((c) => c?.step === 'entity') : undefined
|
||||
return { step, pending: !!seg && (!slot || slot.status === 'loading') }
|
||||
}
|
||||
return (
|
||||
<div className="mt-2 space-y-2 border-t pt-2" data-ev-basis="">
|
||||
{basis.tables.map((t) => {
|
||||
const { step, pending } = stepOf(t.seg)
|
||||
return (
|
||||
<div key={t.alias} data-ev-basis-table={t.alias}>
|
||||
{step?.fields?.length ? (
|
||||
<FieldList fields={step.fields} more={step.fields_more} partial={step.snapshot_truncated === true}
|
||||
masked={step.fields_masked === true} label={(n) => JUDGE_TEXT.basisFieldsCount(t.name, n)} />
|
||||
) : (
|
||||
<p className="text-2xs text-faint">
|
||||
{JUDGE_TEXT.basisFields(t.name)}:{pending ? JUDGE_TEXT.basisPending : JUDGE_TEXT.fieldsMissing}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
{basis.columns.length > 0 && (
|
||||
<div data-ev-basis-columns="">
|
||||
<div className="mb-0.5 text-2xs font-medium text-faint">{JUDGE_TEXT.basisColumns}</div>
|
||||
<ul className="space-y-px text-2xs">
|
||||
{basis.columns.map((c) => {
|
||||
const { step, pending } = stepOf(c.seg)
|
||||
const type = step?.type ?? (step?.types ? Object.values(step.types)[0] : undefined)
|
||||
return (
|
||||
<li key={c.alias} className="min-w-0 [overflow-wrap:anywhere]" data-ev-basis-column={c.alias}>
|
||||
<span className="mono">{c.name}</span>{' '}
|
||||
<span className="mono ml-1 text-faint">{type || (pending ? '…' : NONE)}</span>
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
{basis.queries.map((q, i) => {
|
||||
const slot = artifacts[i] ? sqls[artifacts[i]] : undefined
|
||||
const sql = slot?.status === 'ok' ? slot.data?.sql ?? null : null
|
||||
return (
|
||||
<div key={q} data-ev-basis-sql={q}>
|
||||
<div className="mb-0.5 flex items-center gap-2">
|
||||
<span className="min-w-0 flex-1 text-2xs font-medium text-faint">{JUDGE_TEXT.basisSql(q)}</span>
|
||||
{sql && <CopyChip label={EVIDENCE_TEXT.copySql} text={() => sql} />}
|
||||
</div>
|
||||
{sql ? (
|
||||
<pre className="mono max-h-28 overflow-auto whitespace-pre-wrap rounded bg-bg px-1.5 py-1 text-2xs leading-relaxed text-dim [overflow-wrap:anywhere]"
|
||||
data-ev-sql="">
|
||||
{sql}
|
||||
</pre>
|
||||
) : (
|
||||
<p className="text-2xs text-faint">{artifacts[i] && (!slot || slot.status === 'loading') ? JUDGE_TEXT.basisPending : JUDGE_TEXT.sqlMissing}</p>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
{basis.more > 0 && <p className="text-2xs text-faint" data-ev-basis-more="">{JUDGE_TEXT.basisMoreQueries(basis.more)}</p>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 逐字引文:原文所在的文档和片段,引文在原文里的位置高亮(前后带一截上下文)。检索快照哈希对不上时
|
||||
* 不摆原文——那一段不能当证据。原话对不上的引文也画这一节,只写目录里记着的是哪次检索
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+10
-10
@@ -1,8 +1,8 @@
|
||||
import type { RunEvent, TeamMember, TeamRound, TeamRun, ToolChange } from '../types'
|
||||
import type { NodeState, RunPhase } from './trace'
|
||||
import { formatDuration, formatNumber } from '../lib/format'
|
||||
import { TYPE_LABEL, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel } from '../lib/terms'
|
||||
import { statsTally } from '../lib/evidence'
|
||||
import { JUDGE_TEXT, TYPE_LABEL, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel } from '../lib/terms'
|
||||
import { claimCountsOf, isJudged, statsTally } from '../lib/evidence'
|
||||
|
||||
// 泳道数据画布也要用(supervisor 节点要展开成协作矩阵),所以类型放在
|
||||
// types.ts 里;这里再导出一遍,老引用不用改
|
||||
@@ -647,16 +647,13 @@ function queryTitle(sql: string, source: string): string {
|
||||
/** agent 开了 cite_fields 时循环结束后那次结构化抽取 */
|
||||
const EXTRACT_TITLE = '按出处抽取字段'
|
||||
/**
|
||||
* report.checked 带的裁判摘要 → 「结论 4 句(支持 1 · 不支持 1 · 未裁判 2)」,和出具横幅同一种说法。
|
||||
* 「不是结论句」不算;没有摘要(没开裁判、老后端)返回空串
|
||||
* report.checked 带的裁判摘要 → 「结论 4 句(有依据 1 · 证据相矛盾 1 · 未裁判 2)」,和出具横幅同一种说法。
|
||||
* 「不是结论句」不算,旧数据的 unsupported 照样计入;没有摘要(没开裁判、老后端)返回空串
|
||||
*/
|
||||
function judgeTally(judge: unknown): string {
|
||||
const c = judge && typeof judge === 'object' ? (judge as { counts?: Record<string, unknown> }).counts : undefined
|
||||
if (!c || typeof c !== 'object') return ''
|
||||
const n = (k: string) => (typeof c[k] === 'number' && Number.isFinite(c[k] as number) ? c[k] as number : 0)
|
||||
const counts = { supported: n('supported'), partial: n('partial'), unsupported: n('unsupported'), unjudged: n('unjudged'),
|
||||
uncited: 0 }
|
||||
return claimTally({ ...counts, total: counts.supported + counts.partial + counts.unsupported + counts.unjudged })
|
||||
return claimTally(claimCountsOf(c))
|
||||
}
|
||||
|
||||
/** 报告撰写节点请裁判模型判断结论句(四期,claims: judge):units 是这一批几句 */
|
||||
@@ -1785,14 +1782,17 @@ export function decodeRun(events: RunEvent[], final?: RunFinal, opts?: DecodeOpt
|
||||
case 'evidence.judged': {
|
||||
// 探索运行跑完、封存之后,有人点开结论句请模型判断:判定追加在封存之后,不改封存的报告
|
||||
const verdicts = d.verdicts && typeof d.verdicts === 'object' ? Object.values(d.verdicts as Record<string, any>) : []
|
||||
const judged = verdicts.filter((v) => ['supported', 'partial', 'unsupported', 'not_a_claim'].includes(v?.status)).length
|
||||
const judged = verdicts.filter((v) => isJudged(v)).length
|
||||
const hit: string[] = Array.isArray(d.limits_hit) ? d.limits_hit : []
|
||||
push({
|
||||
id: `ej-${seq}`, seq, kind: 'note', nodeId, status: 'done', code: 'evidence_judged',
|
||||
level: hit.length || (Array.isArray(d.gaps) && d.gaps.length) ? 'warn' : 'info',
|
||||
title: `封存后按需裁判了 ${formatNumber(judged)} 句结论${hit.length ? ',部分因达到上限未裁判' : ''}`,
|
||||
sub: '封存后追加 · 模型判断,非确定',
|
||||
...(d.model ? { detail: [`模型:${d.model}`, ...(Array.isArray(d.gaps) ? d.gaps.map(String) : [])].join('\n') } : {}),
|
||||
// 裁判模型和写作模型相同、模型不在价格表里:和证据面板「模型的解释」同样的两句提醒
|
||||
...(d.model ? { detail: [`模型:${d.model}`, ...(d.same_model === true ? [JUDGE_TEXT.sameModel] : []),
|
||||
...(d.priced === false ? [JUDGE_TEXT.noPrice] : []),
|
||||
...(Array.isArray(d.gaps) ? d.gaps.map(String) : [])].join('\n') } : {}),
|
||||
}, nodeId)
|
||||
break
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { useEffect, useMemo } from 'react'
|
||||
import { create } from 'zustand'
|
||||
import { ApiError, api } from '../api/client'
|
||||
import { graphVerdicts, judgedVerdicts } from '../lib/evidence'
|
||||
import { graphVerdicts, judgedVerdicts, sqlOf } from '../lib/evidence'
|
||||
import type {
|
||||
EvidenceDocData, EvidenceGraph, EvidenceJudgeResult, EvidenceSegmentDetail, EvidenceStats, EvidenceVerdict,
|
||||
} from '../types'
|
||||
@@ -44,6 +44,12 @@ interface EvidenceState {
|
||||
docs: Record<string, Slot<EvidenceDocData>>
|
||||
graphs: Record<string, Slot<EvidenceGraph>>
|
||||
segments: Record<string, Slot<EvidenceSegmentDetail>>
|
||||
/**
|
||||
* 查询快照里的 SQL,按工件 id 缓存(内容寻址,永久):讲方法、讲结构的句子在「挂的依据」下直接列出。
|
||||
* 只留 SQL,不留整份快照的行
|
||||
*/
|
||||
sqls: Record<string, Slot<{ sql: string | null }>>
|
||||
loadSql: (artifact: string) => Promise<void>
|
||||
/** 按工作流:最近一次运行的报告统计。画布没挂着运行时,报告节点卡的章从这里取 */
|
||||
latest: Record<string, Slot<LatestReports>>
|
||||
loadDoc: (artifact: string) => Promise<void>
|
||||
@@ -100,7 +106,7 @@ export const useEvidence = create<EvidenceState>((set, get) => {
|
||||
* 取一次、记进 table[key];正在取或已经取到的不再发。取的途中这一格被清掉或换了(dropLatest 之后
|
||||
* 又起了一次),回来的结果作废:不能让早先发出的请求把后来那份盖掉
|
||||
*/
|
||||
async function fill<K extends 'docs' | 'graphs' | 'segments' | 'latest'>(
|
||||
async function fill<K extends 'docs' | 'graphs' | 'segments' | 'latest' | 'sqls'>(
|
||||
table: K, key: string, fetcher: () => Promise<NonNullable<EvidenceState[K][string]['data']>>, force = false,
|
||||
) {
|
||||
const cur = get()[table][key]
|
||||
@@ -124,6 +130,8 @@ export const useEvidence = create<EvidenceState>((set, get) => {
|
||||
graphs: {},
|
||||
segments: {},
|
||||
latest: {},
|
||||
sqls: {},
|
||||
loadSql: (artifact) => fill('sqls', artifact, async () => ({ sql: sqlOf((await api.artifact(artifact))?.content) })),
|
||||
loadDoc: (artifact) => fill('docs', artifact, async () => {
|
||||
const res = await api.artifact(artifact)
|
||||
const doc = asDoc(res?.content)
|
||||
|
||||
+44
-6
@@ -755,9 +755,20 @@ export interface EvidenceUnit {
|
||||
* timeout_s,rationale 以「已到上限(…)」开头)、error、format、missing、on_demand(探索运行按需)
|
||||
*/
|
||||
export interface EvidenceVerdict {
|
||||
status: 'supported' | 'partial' | 'unsupported' | 'not_a_claim' | 'unjudged' | string
|
||||
/** 理由,最多 60 字 */
|
||||
/**
|
||||
* contradicted:证据和原句冲突;insufficient:证据里没有相关信息(missing 写缺的是什么)。
|
||||
* unsupported 是拆档之前的取值:旧文档、旧事件里保留原样,界面上照旧写「证据不支持」
|
||||
*/
|
||||
status: 'supported' | 'partial' | 'contradicted' | 'insufficient' | 'unsupported' | 'not_a_claim' | 'unjudged' | string
|
||||
/** 理由,最多 120 字(后端在句子边界截断,加「…」) */
|
||||
rationale?: string
|
||||
/** insufficient 时缺的是什么:「SQL」「字段清单」「取数的查询」…,由裁判给出 */
|
||||
missing?: string
|
||||
/** 按哪一版裁判规则判的(新判的都带;拆档之前判的没有) */
|
||||
rule_version?: number
|
||||
/** 判定自己带着的模型提醒(有的话盖过报告、答复、证据图里的) */
|
||||
same_model?: boolean
|
||||
priced?: boolean | null
|
||||
/** 裁判模型 id;探索运行还没判的是 null */
|
||||
judge?: string | null
|
||||
/** 封存之后按需追加的(evidence.judged 事件) */
|
||||
@@ -780,7 +791,8 @@ export interface EvidenceJudgeSummary {
|
||||
candidates?: number
|
||||
/** 预筛放掉、没送裁判的结论句 */
|
||||
screened?: string[]
|
||||
counts?: Partial<Record<'supported' | 'partial' | 'unsupported' | 'not_a_claim' | 'unjudged', number>>
|
||||
counts?: Partial<Record<'supported' | 'partial' | 'contradicted' | 'insufficient' | 'unsupported' | 'not_a_claim'
|
||||
| 'unjudged', number>>
|
||||
unjudged?: Record<string, number>
|
||||
limits_hit?: string[]
|
||||
complete?: boolean
|
||||
@@ -792,6 +804,8 @@ export interface EvidenceJudgeSummary {
|
||||
gaps?: string[]
|
||||
notes?: string[]
|
||||
same_model?: boolean
|
||||
/** 撰写这份报告实际用的模型 id(裁判模型和它相同时 same_model 为真) */
|
||||
writer_model?: string | null
|
||||
/**
|
||||
* 不支持的句子交回写作者改写一次(rewrite_once)。units:交回的句子在封存文档里的编号(采用了改写稿时,
|
||||
* 改掉的句子是 null);sentences:交回的原句;changed:改写稿里新写的句子的编号(没采用时为空);
|
||||
@@ -822,13 +836,28 @@ export interface EvidenceJudgeResult {
|
||||
message?: string | null
|
||||
/** 触顶时怎么调,一个上限一句 */
|
||||
adjust?: string[]
|
||||
/** 这一次裁判模型和写作模型相同、模型在不在价格表里、写作模型 id */
|
||||
same_model?: boolean
|
||||
priced?: boolean | null
|
||||
writer_model?: string | null
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
/** 证据图 reports[].judge_meta:这份报告最近一次按需裁判用的模型 */
|
||||
export interface EvidenceJudgeMeta {
|
||||
model?: string | null
|
||||
same_model?: boolean
|
||||
priced?: boolean | null
|
||||
writer_model?: string | null
|
||||
}
|
||||
|
||||
/** 这句能不能「请模型判断」:片段接口按运行的实际情况答(和按需裁判接口同一套条件) */
|
||||
export interface EvidenceOnDemand {
|
||||
available?: boolean
|
||||
/** formal / legacy / not_a_claim / judged / unsealed / seal_broken */
|
||||
/**
|
||||
* formal / legacy / not_a_claim / judged / unsealed / seal_broken;outdated 是封存后追加的判定早于当前的裁判规则
|
||||
* (available 为真,照常给按钮)
|
||||
*/
|
||||
reason?: string | null
|
||||
message?: string | null
|
||||
}
|
||||
@@ -875,9 +904,11 @@ export interface EvidenceStats {
|
||||
quotes?: number
|
||||
unknown_entities?: number
|
||||
unverified_entities?: number
|
||||
/** 四期:claims 为 judge 时追加,候选句按判定的句数 */
|
||||
/** 四期:claims 为 judge 时追加,候选句按判定的句数(unsupported 是拆档之前的取值,旧文档里才有) */
|
||||
supported?: number
|
||||
partial?: number
|
||||
contradicted?: number
|
||||
insufficient?: number
|
||||
unsupported?: number
|
||||
not_a_claim?: number
|
||||
unjudged?: number
|
||||
@@ -947,7 +978,9 @@ export interface EvidenceGraph {
|
||||
*/
|
||||
post_seal_verdicts?: Record<string, EvidenceVerdict> | null
|
||||
/** 封存的裁判摘要(文档的 judge) */
|
||||
judge?: EvidenceJudgeSummary | null }[]
|
||||
judge?: EvidenceJudgeSummary | null
|
||||
/** 最近一次按需裁判用的模型:和写作模型相同没有、在不在价格表里 */
|
||||
judge_meta?: EvidenceJudgeMeta | null }[]
|
||||
evidence?: { alias?: string; eid?: string; kind?: string; label?: string; node_id?: string; artifact?: string
|
||||
sealed?: boolean; cited_by?: string[]; report?: string }[]
|
||||
edges?: { from: string; to: string; rel: string }[]
|
||||
@@ -1107,6 +1140,11 @@ export interface EvidenceStep extends Omit<EvidenceInput, 'status'> {
|
||||
snapshot_truncated?: boolean
|
||||
/** 好几张表都有的同名字段:各表的类型 */
|
||||
types?: Record<string, string>
|
||||
/** 表:字段清单和类型(封存范围内的表结构快照,最多 60 个,遮罩的列不列入);多出来的个数 */
|
||||
fields?: { name?: string; type?: string | null }[]
|
||||
fields_more?: number
|
||||
/** 有列按遮罩没列进字段清单 */
|
||||
fields_masked?: boolean
|
||||
/** 表:几个字段、是不是视图;表或字段的注释 */
|
||||
is_view?: boolean
|
||||
comment?: string
|
||||
|
||||
@@ -84,7 +84,7 @@ const ORDER = [
|
||||
// 拆开的两项看 SPLIT 里各份的 est。2026-09-29 八道并行、关浏览器不干等之后的实测
|
||||
const ESTIMATE = {
|
||||
stream: 173, chat: 160, studio: 149, manage: 145, runs: 112, publish: 92, shell: 87,
|
||||
evidence: 75, 'ui-kit': 67, ui: 61, guards: 8, 'canvas-layout': 2, copy: 2, tokens: 1, trace: 1, decode: 1,
|
||||
evidence: 85, 'ui-kit': 70, ui: 61, guards: 8, 'canvas-layout': 2, copy: 2, tokens: 1, trace: 1, decode: 1,
|
||||
}
|
||||
// 最慢的两项按段拆成几份。用的是它们自己的段过滤:最后一份用 *_SKIP 兜住「其余全部」,以后新加的段
|
||||
// 不用改这里也跑得到(拆出去的那几份要是一段都没对上,那一份会报「一项都没跑」)。
|
||||
|
||||
+394
-8
@@ -33,7 +33,10 @@ const fxe = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-
|
||||
// 第四期:结论句裁判。文档由 compose_doc 真跑,候选句由 judge.prepare 真跑,判定是剧本(模拟裁判模型),
|
||||
// 经 judge 的 summarize / apply_judgement 写回:正式运行(节点里判好的)、探索运行(按需)、没开裁判的探索运行
|
||||
const fxj = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-judge.json`, 'utf8'))
|
||||
const JUDGE_SETS = [fxj.formal, fxj.explore, fxj.plain]
|
||||
// 裁判拆档(JR):证据相矛盾、证据不足(写明缺什么)、讲方法的句子、裁判模型和写作模型相同、模型不在价格表里。
|
||||
// 文档由 compose_doc 真跑、证据链由 api/evidence.py 的 _chain 真跑;判定和表名实体步骤的 fields 照前后端接口约定写
|
||||
const fxv = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-verdict.json`, 'utf8'))
|
||||
const JUDGE_SETS = [fxj.formal, fxj.explore, fxj.plain, fxv.formal, fxv.explore]
|
||||
/** 夹具里某段文字是哪个片段:检查按文字认片段,重新生成夹具时编号变了也不用改 */
|
||||
const segOf = (doc, text) => doc.blocks.flatMap((b) => b.units.flatMap((u) => u.segments)).find((s) => s.text === text)?.id
|
||||
|
||||
@@ -61,7 +64,7 @@ const browser = await chromium.launch({ executablePath: CHROME })
|
||||
* 夹具里按句子备好了答复的用夹具的;别的句子给一条「有依据」。正式运行回 409
|
||||
*/
|
||||
function judgeReply(set, body) {
|
||||
if (set === fxj.formal) return { status: 409, json: { detail: '正式运行的结论句已在报告撰写节点内完成裁判,结果随报告一起封存,不能再按需裁判', code: 'evidence_judge_formal' } }
|
||||
if (set === fxj.formal || set === fxv.formal) return { status: 409, json: { detail: '正式运行的结论句已在报告撰写节点内完成裁判,结果随报告一起封存,不能再按需裁判', code: 'evidence_judge_formal' } }
|
||||
const unit = body?.units?.[0]
|
||||
const named = Object.entries(set.units).find(([, u]) => u === unit)?.[0]
|
||||
if (named && set.judge?.[named]) return { json: set.judge[named] }
|
||||
@@ -92,13 +95,20 @@ async function routed(page, { judge = null, segPatch = null } = {}) {
|
||||
// 四期的报告工件:问数据页的答案(Output → EvidenceField)按工件 id 取文档
|
||||
const jart = JUDGE_SETS.find((x) => url.pathname === `/api/artifacts/${x.doc_artifact}`)
|
||||
if (jart) return r.fulfill({ json: { id: jart.doc_artifact, content: jart.doc } })
|
||||
// 裁判拆档:讲方法的句子在「挂的依据」下列出 SQL,取的是查询快照工件
|
||||
const vsnap = Object.entries(fxv.snapshots).find(([, x]) => url.pathname === `/api/artifacts/${x.artifact}`)
|
||||
if (vsnap) {
|
||||
hits.push(`artifact:V:${vsnap[0]}`)
|
||||
return r.fulfill({ json: { id: vsnap[1].artifact, content: vsnap[1].content } })
|
||||
}
|
||||
if (jset) {
|
||||
const seg = url.pathname.match(/\/evidence\/segments\/([^/]+)$/)
|
||||
if (seg) {
|
||||
const id = decodeURIComponent(seg[1])
|
||||
hits.push(`j:${id}`)
|
||||
// 判过之后片段接口按最新的 evidence.judged 叠上判定(封存后追加):夹具里备着判过那一句的答复
|
||||
const after = jset.segments_after && judged.has(`${jset.run_id}:${jset.units.bad}`) ? jset.segments_after : jset.segments
|
||||
const trigger = jset.units.bad ?? jset.units.method
|
||||
const after = jset.segments_after && judged.has(`${jset.run_id}:${trigger}`) ? jset.segments_after : jset.segments
|
||||
let body = structuredClone(after[id])
|
||||
if (body && segPatch) body = segPatch(jset, body)
|
||||
return body ? r.fulfill({ json: body })
|
||||
@@ -1214,7 +1224,7 @@ await section('studio', '报告撰写节点:画布认得、检查器能配(R
|
||||
const shown = await p.locator('[data-field="claims"] select').evaluate((el) => el.options[el.selectedIndex]?.textContent ?? '')
|
||||
check('config 里写着无法识别的值:下拉照实显示,不装成 off', shown.includes('sometimes') && shown.includes('无法识别'), shown)
|
||||
|
||||
// 四期:选模型裁判,高级选项里出现结论句裁判的子配置(裁判模型、三项上限各自能设不限、改写一次、证据不支持时)
|
||||
// 四期:选模型裁判,高级选项里出现结论句裁判的子配置(裁判模型、三项上限各自能设不限、改写一次、证据相矛盾时)
|
||||
check('没选模型裁判时不出结论句裁判那一组', await p.locator('[data-field="judge"]').count() === 0)
|
||||
await p.locator('[data-field="claims"] select').selectOption('judge')
|
||||
check('选「请模型逐句判断」写进 config.claims = judge', (await cfgOf())?.claims === 'judge')
|
||||
@@ -1223,12 +1233,16 @@ await section('studio', '报告撰写节点:画布认得、检查器能配(R
|
||||
const jlabel = await jf.locator('label').first().innerText().catch(() => '')
|
||||
check('高级选项里出现「结论句裁判」一组(中文标签)', await jf.count() === 1 && jlabel.includes('结论句裁判'), jlabel)
|
||||
const keys = await jf.locator('[data-judge-key]').evaluateAll((els) => els.map((e) => e.getAttribute('data-judge-key')))
|
||||
check('……裁判模型、三项上限、改写一次、证据不支持时', keys.join(',') === 'model,max_claims,max_cost_usd,timeout_s,rewrite_once,on_unsupported',
|
||||
check('……裁判模型、三项上限、改写一次、证据相矛盾时', keys.join(',') === 'model,max_claims,max_cost_usd,timeout_s,rewrite_once,on_unsupported',
|
||||
keys.join(','))
|
||||
const labels = await jf.innerText()
|
||||
// 细节只报缺了哪几个:整段文字里有沙箱的接入名(下拉的选项),日志常被贴进报告
|
||||
const missingLabels = ['裁判模型', '最多裁判句数', '金额上限(美元)', '时长上限(秒)', '退回改写一次', '证据不支持时'].filter((t) => !labels.includes(t))
|
||||
// 细节只报缺了哪几个:整段文字里有沙箱的接入名(下拉的选项),日志常被贴进报告。裁判拆档后 on_unsupported
|
||||
// 管的是证据相矛盾(键名不变),标签跟着改
|
||||
const missingLabels = ['裁判模型', '最多裁判句数', '金额上限(美元)', '时长上限(秒)', '退回改写一次', '证据相矛盾时'].filter((t) => !labels.includes(t))
|
||||
check('……每一项都有中文标签', !missingLabels.length, `缺:${missingLabels.join('、')}`)
|
||||
const onHint = await jf.locator('[data-judge-key="on_unsupported"] [data-judge-hint]').innerText().catch(() => '')
|
||||
check('……「证据相矛盾时」下面写明证据不足的句子最多降档、不会因此不予出具', onHint.includes('证据不足的句子最多降档')
|
||||
&& onHint.includes('不会因此不予出具'), onHint)
|
||||
check('……没写的上限说明跟着设置里的默认值', (await jf.locator('[data-judge-key="max_cost_usd"] [data-judge-note]').innerText()).includes('默认值'))
|
||||
const cost = jf.locator('[data-judge-key="max_cost_usd"]')
|
||||
await cost.locator('input[type="checkbox"]').click()
|
||||
@@ -1277,7 +1291,7 @@ await section('studio', '报告撰写节点:画布认得、检查器能配(R
|
||||
check('发布前修复的预览:judge.max_cost_usd 写成「结论句裁判 · 金额上限(美元)」', previews[0] === '结论句裁判 · 金额上限(美元)', previews[0])
|
||||
check('……null 写「不限」(没写才是「(空)」)、金额带单位', previews[1] === '不限' && previews[2] === '0.05 美元', JSON.stringify(previews.slice(1, 3)))
|
||||
check('……补预算按整份 judge 记时一键一行', previews[3] === '裁判模型:x;金额上限(美元):不限', previews[3])
|
||||
check('……证据不支持时写选项文字', previews[4] === '不予出具', previews[4])
|
||||
check('……证据相矛盾时写选项文字', previews[4] === '不予出具', previews[4])
|
||||
check('……别处的 model 不借用裁判的叫法', previews[5] === '工作流默认设置 · model', previews[5])
|
||||
if (SHOTS) {
|
||||
await jf.scrollIntoViewIfNeeded()
|
||||
@@ -2108,6 +2122,378 @@ await section('j-stream', '四期:问数据页、画布右栏的横幅(Outpu
|
||||
await s.ctx.close()
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 裁判拆档(JR):证据相矛盾 / 证据不足、缺什么、裁判模型的两条提醒、理由 120 字、字段清单、讲方法的句子
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const V = '#evidence-verdict'
|
||||
const VN = '#evidence-verdict-narrow'
|
||||
const VX = '#evidence-verdict-explore'
|
||||
/** 一组句末徽标的样子:判定、字形、计算色、读屏名、悬停说明,外加整句的浅底(没有浅底是 null) */
|
||||
const badgeLook = (pg, sel, units) => pg.evaluate(({ sel, units }) => Object.fromEntries(Object.entries(units).map(([k, id]) => {
|
||||
const el = document.querySelector(`${sel} [data-ev-claim="${id}"]`)
|
||||
if (!el) return [k, null]
|
||||
const shade = document.querySelector(`${sel} [data-ev-claim-shade="${id}"]`)
|
||||
return [k, { verdict: el.getAttribute('data-ev-verdict'), glyph: el.textContent, color: getComputedStyle(el).color,
|
||||
label: el.getAttribute('aria-label'), title: el.getAttribute('title'),
|
||||
shade: shade ? getComputedStyle(shade).backgroundColor : null }]
|
||||
})), { sel, units })
|
||||
const softOf = (pg, token) => tokenColor(pg, 'backgroundColor', `var(${token})`)
|
||||
const firstNumber = (doc, uid) => doc.blocks.flatMap((b) => b.units).find((x) => x.id === uid).segments.find((s) => s.kind === 'number').id
|
||||
|
||||
await section('jr', '裁判拆档:证据相矛盾、证据不足四通道分得开,证据不足不用红、不和未裁判混;n / N 先走相矛盾、再走证据不足', async () => {
|
||||
const u = fxv.formal.units
|
||||
await page.waitForSelector(`${V} [data-ev-claim="${u.insuff}"]`, { timeout: 4000 }).catch(() => {})
|
||||
const look = await badgeLook(page, V, u)
|
||||
check('新取值各挂各的徽标:证据相矛盾、证据不足(讲方法的那句也判成证据相矛盾)',
|
||||
look.contra?.verdict === 'contradicted' && look.insuff?.verdict === 'insufficient' && look.method?.verdict === 'contradicted',
|
||||
JSON.stringify(Object.fromEntries(Object.entries(look).map(([k, v]) => [k, v?.verdict]))))
|
||||
check('字形:证据相矛盾 !、证据不足 ○,和未裁判的 ? 分得开', look.contra?.glyph === '!' && look.insuff?.glyph === '○'
|
||||
&& look.limit?.glyph === '?' && look.insuff.glyph !== look.limit.glyph,
|
||||
[look.contra?.glyph, look.insuff?.glyph, look.limit?.glyph].join(' '))
|
||||
const [failed, waiting, neutral, done] = await Promise.all(['--st-failed', '--st-waiting', '--st-cancelled', '--st-done']
|
||||
.map((t) => colorOf(page, t)))
|
||||
check('颜色令牌:证据相矛盾 --st-failed(同原来的证据不支持),证据不足 --st-cancelled(中性)',
|
||||
look.contra?.color === failed && look.method?.color === failed && look.insuff?.color === neutral,
|
||||
JSON.stringify({ got: [look.contra?.color, look.insuff?.color], want: [failed, neutral] }))
|
||||
check('证据不足不用红,也不用提醒色和确定性的绿', !!look.insuff && ![failed, waiting, done].includes(look.insuff.color), look.insuff?.color)
|
||||
const soft = await softOf(page, '--st-failed-soft')
|
||||
check('证据相矛盾的整句铺浅底(--st-failed-soft)', look.contra?.shade === soft && look.method?.shade === soft && soft !== 'rgba(0, 0, 0, 0)',
|
||||
JSON.stringify({ contra: look.contra?.shade, method: look.method?.shade, soft }))
|
||||
check('证据不足不铺底(中性样式)', look.insuff?.shade === null, String(look.insuff?.shade))
|
||||
check('读屏名:证据相矛盾', look.contra?.label === '结论句「各渠道的订单数相加与总订单数一致。」,模型判断:证据相矛盾', look.contra?.label)
|
||||
check('读屏名:证据不足接着说缺什么', look.insuff?.label === `结论句「增长主要来自新客首单,老客复购持平。」,模型判断:证据不足,缺少:${fxv.missing}`,
|
||||
look.insuff?.label)
|
||||
check('悬停说明:证据不足写明谁判的、非确定、缺什么', look.insuff?.title === `模型判断:证据不足(${fxv.model} · 非确定)。缺少:${fxv.missing}`,
|
||||
look.insuff?.title)
|
||||
const tally = await page.locator(`${V} [data-evidence-claims]`).innerText().catch(() => '')
|
||||
check('证据条的结论句计数带上证据相矛盾、证据不足', tally.includes('结论 6 句(有依据 1 · 部分有依据 1 · 证据相矛盾 2 · 证据不足 1 · 未裁判 1)'), tally)
|
||||
const summary = await page.locator(`${V} [data-evidence-summary]`).innerText()
|
||||
check('读屏摘要的按键说明:证据有问题的句子 n 也跳到,证据不足的排在最后', summary.includes('证据有问题的句子(证据不足的排在最后)'), summary)
|
||||
|
||||
// n / N:第 1 轮是证据相矛盾、部分有依据(按正文顺序),第 2 轮是证据不足——它在正文里排第二,照样最后才到
|
||||
await page.locator(`${V} [data-seg="${fxv.formal.segs.gmv}"]`).focus()
|
||||
const jumps = []
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await page.keyboard.press('n')
|
||||
jumps.push(await activeKey(page))
|
||||
}
|
||||
const want = [u.contra, u.partial, u.method, u.insuff, u.contra].map((x) => `claim:${x}`)
|
||||
check('n:先按正文顺序走证据相矛盾、部分有依据,再走证据不足,走完接回开头', jumps.join(',') === want.join(','), jumps.join(','))
|
||||
const back = []
|
||||
for (let i = 0; i < 2; i++) {
|
||||
await page.keyboard.press('N')
|
||||
back.push(await activeKey(page))
|
||||
}
|
||||
check('N:倒着走,从第一处退到证据不足(最后一处),再退到第 1 轮的最后一处', back.join(',') === `claim:${u.insuff},claim:${u.method}`,
|
||||
back.join(','))
|
||||
check('只是走动不打开面板', await panel(page).count() === 0)
|
||||
|
||||
// 拆档之前的取值:探索运行证据图里的 unsupported 照旧显示「证据不支持」,样式同证据相矛盾
|
||||
const x = fxv.explore.units
|
||||
await page.waitForSelector(`${VX} [data-ev-claim="${x.contra}"][data-ev-verdict="unsupported"]`, { timeout: 4000 }).catch(() => {})
|
||||
const old = (await badgeLook(page, VX, { contra: x.contra, insuff: x.insuff }))
|
||||
check('旧取值 unsupported:照旧叫「证据不支持」(读屏名、悬停说明)', old.contra?.verdict === 'unsupported'
|
||||
&& old.contra.label?.endsWith(',模型判断:证据不支持,封存后追加') && old.contra.title?.startsWith('模型判断:证据不支持 · 封存后追加'),
|
||||
JSON.stringify(old.contra))
|
||||
check('……样式同证据相矛盾:!、--st-failed、整句浅底', old.contra?.glyph === '!' && old.contra.color === failed && old.contra.shade === soft,
|
||||
JSON.stringify(old.contra))
|
||||
check('……封存后追加的证据不足同样是 ○、中性色', old.insuff?.verdict === 'insufficient' && old.insuff.glyph === '○' && old.insuff.color === neutral,
|
||||
JSON.stringify(old.insuff))
|
||||
const xt = await page.locator(`${VX} [data-evidence-claims]`).innerText().catch(() => '')
|
||||
check('……计数里旧取值照样算进去,和证据不足并列', xt.includes('结论 7 句(证据不支持 1 · 证据不足 1 · 未裁判 5)'), xt)
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await page.waitForTimeout(200)
|
||||
await page.locator(V).screenshot({ path: `${SHOTS}/jr-doc-${theme}.png` })
|
||||
await page.locator(VX).screenshot({ path: `${SHOTS}/jr-doc-explore-${theme}.png` })
|
||||
}
|
||||
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
})
|
||||
|
||||
await section('jr-panel', '裁判拆档:面板写明缺什么、裁判模型的两条提醒、理由 120 字不截断、字段清单、讲方法的句子直接看到 SQL 和字段清单', async () => {
|
||||
const r = await open('/ui-harness.html?evidence=1')
|
||||
const pg = r.page
|
||||
const p = pg.locator('[data-evidence-panel]')
|
||||
const u = fxv.formal.units
|
||||
const [failed, waiting, neutral] = await Promise.all(['--st-failed', '--st-waiting', '--st-cancelled'].map((t) => colorOf(pg, t)))
|
||||
await badge(pg, V, u.insuff).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="insufficient"]')
|
||||
check('面板:证据不足(字形 ○)', (await p.locator('[data-ev-verdict]').innerText()).trim() === '○模型判断:证据不足',
|
||||
await p.locator('[data-ev-verdict]').innerText())
|
||||
const head = await p.locator('header [data-ev-badge]').evaluate((el) => ({ code: el.getAttribute('data-ev-badge'), color: getComputedStyle(el).color }))
|
||||
.catch(() => null)
|
||||
check('……面板标题的徽标是中性色,不用红', head?.code === 'insufficient' && head.color === neutral && head.color !== failed, JSON.stringify(head))
|
||||
const missing = await p.locator('[data-ev-judge] [data-ev-missing]').innerText().catch(() => '')
|
||||
check('……「模型的解释」里写明缺什么', missing === `缺少:${fxv.missing}`, missing)
|
||||
const same = await p.locator('[data-ev-judge] [data-ev-judge-same]').innerText().catch(() => '')
|
||||
const unpriced = await p.locator('[data-ev-judge] [data-ev-judge-unpriced]').innerText().catch(() => '')
|
||||
check('裁判模型和写作模型相同:「模型的解释」下写「裁判模型和写作模型相同,结果仅供参考」', same === '裁判模型和写作模型相同,结果仅供参考', same)
|
||||
check('模型不在价格表里:写「该模型不在价格表中,金额上限不生效」', unpriced === '该模型不在价格表中,金额上限不生效', unpriced)
|
||||
const caution = await p.locator('[data-ev-judge-caution]').evaluate((el) => {
|
||||
const cs = getComputedStyle(el)
|
||||
return { border: cs.borderLeftColor, width: cs.borderLeftWidth, style: cs.borderLeftStyle, color: cs.color, bg: cs.backgroundColor,
|
||||
icon: getComputedStyle(el.querySelector('svg')).color }
|
||||
}).catch(() => null)
|
||||
check('……醒目但不刺眼:左侧一道 2px 提醒色竖线、图标用提醒色,字不用红、不铺底', !!caution && caution.border === waiting
|
||||
&& caution.width === '2px' && caution.style === 'solid' && caution.icon === waiting && caution.color !== failed
|
||||
&& caution.bg === 'rgba(0, 0, 0, 0)', JSON.stringify(caution))
|
||||
const sameTitle = await p.locator('[data-ev-judge-same]').getAttribute('title').catch(() => '')
|
||||
check('……悬停写明是哪个模型', (sameTitle ?? '').includes(fxv.model), sameTitle ?? '')
|
||||
// 没判的句子、别的报告不挂
|
||||
await badge(pg, V, u.limit).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="unjudged"]')
|
||||
check('到上限没判的句子不挂这两条提醒(模型没看过这句)', await p.locator('[data-ev-judge-caution]').count() === 0)
|
||||
await badge(pg, J, fxj.formal.units.bad).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]')
|
||||
check('对照:裁判模型和写作模型不同、模型在价格表里的报告不挂提醒', await p.locator('[data-ev-judge-caution]').count() === 0)
|
||||
check('……旧取值的面板照旧写「证据不支持」', (await p.locator('[data-ev-verdict]').innerText()).includes('模型判断:证据不支持'))
|
||||
// 理由最长 120 字:整句显示,侧边面板里折行、不溢出
|
||||
await badge(pg, V, u.contra).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="contradicted"]')
|
||||
const why = await p.locator('[data-ev-rationale]').evaluate((el) => {
|
||||
const cs = getComputedStyle(el)
|
||||
return { text: el.textContent, sw: el.scrollWidth, cw: el.clientWidth, ellipsis: cs.textOverflow, clamp: cs.webkitLineClamp,
|
||||
overflow: cs.overflow, lines: Math.round(el.getBoundingClientRect().height / parseFloat(cs.lineHeight)) }
|
||||
}).catch(() => null)
|
||||
check('理由 120 字整句显示,不截断', why?.text === fxv.long_rationale && [...fxv.long_rationale].length === 120, why?.text)
|
||||
check('……在侧边面板里折成几行、不溢出、不加省略号', !!why && why.sw <= why.cw && why.ellipsis !== 'ellipsis'
|
||||
&& (why.clamp === 'none' || why.clamp === '') && why.lines >= 3, JSON.stringify(why))
|
||||
check('证据相矛盾:面板写「模型判断:证据相矛盾」,不写缺什么', (await p.locator('[data-ev-verdict]').innerText()).trim() === '!模型判断:证据相矛盾'
|
||||
&& await p.locator('[data-ev-missing]').count() === 0)
|
||||
const orders = fxv.formal.segments[fxv.formal.segs.orders].chain.find((c) => c.step === 'entity')
|
||||
const wantFields = orders.fields.map((f) => [f.name, f.type])
|
||||
// 讲方法的句子:「挂的依据」下直接列出 SQL 和字段清单(先于点开句中的表名:取数只能来自这一块)
|
||||
const hitsBefore = r.hits.length
|
||||
check('……此前没取过句中表名的片段', !r.hits.includes(`j:${fxv.formal.segs.orders}`))
|
||||
await badge(pg, V, u.method).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-cites] [data-ev-basis]')
|
||||
// SQL、字段清单没画出来时也往下走:让下面几条断言自己报没通过
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-cites] [data-ev-basis] [data-ev-sql]', { timeout: 3000 }).catch(() => {})
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-basis-table] [data-ev-field]', { timeout: 3000 }).catch(() => {})
|
||||
const basis = p.locator('[data-ev-cites] [data-ev-basis]')
|
||||
const sqls = await basis.locator('[data-ev-basis-sql]').evaluateAll((els) => els.map((e) =>
|
||||
[e.getAttribute('data-ev-basis-sql'), e.querySelector('[data-ev-sql]')?.textContent ?? '']))
|
||||
check('讲方法的句子:「挂的依据」下直接列出 SQL(直接挂的 Q1,再补表名出现过的 Q2)',
|
||||
JSON.stringify(sqls) === JSON.stringify([['Q1', fxv.snapshots.Q1.content.sql], ['Q2', fxv.snapshots.Q2.content.sql]]), JSON.stringify(sqls))
|
||||
const bf = await basis.locator('[data-ev-basis-table="t:orders"] [data-ev-field]').evaluateAll((els) => els.map((e) => e.getAttribute('data-ev-field')))
|
||||
check('……表的字段清单也直接列出', JSON.stringify(bf) === JSON.stringify(wantFields.map(([n]) => n)), JSON.stringify(bf))
|
||||
const bhead = await basis.locator('[data-ev-basis-table="t:orders"]').innerText().catch(() => '')
|
||||
check('……写明是哪张表的字段清单', bhead.startsWith(`表 orders 的字段清单(${wantFields.length} 个)`), bhead.split('\n')[0])
|
||||
const bc = await basis.locator('[data-ev-basis-column]').allInnerTexts()
|
||||
check('……挂的字段写类型', bc.some((t) => t.includes('orders.status') && t.includes('VARCHAR(12)'))
|
||||
&& bc.some((t) => t.includes('orders.created_at') && t.includes('TIMESTAMP')), bc.join(' | '))
|
||||
check('……SQL 带复制', await basis.locator('[data-ev-basis-sql="Q1"] button', { hasText: '复制 SQL' }).count() === 1)
|
||||
// 这一页只有「挂的依据」会取查询快照工件(完整快照要点按钮);这之前没点开过句中的表名,字段清单那次片段请求只能是
|
||||
// 它自己发的。Q1、Q2 的 SQL 在前面几句(只挂了整份查询)打开时已经取过,缓存着不再取
|
||||
const fresh = r.hits.slice(hitsBefore)
|
||||
check('……SQL 取自查询快照工件,字段清单取自表名片段的证据链', r.hits.includes('artifact:V:Q1') && r.hits.includes('artifact:V:Q2')
|
||||
&& fresh.includes(`j:${fxv.formal.segs.orders}`), fresh.join(','))
|
||||
check('……判定的理由里提到的 created_at 在字段清单里看得到', bf.includes('created_at'))
|
||||
await badge(pg, V, u.insuff).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="insufficient"]')
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-basis] [data-ev-sql]').catch(() => {})
|
||||
const insuffSql = await p.locator('[data-ev-basis] [data-ev-basis-sql]').evaluateAll((els) => els.map((e) => e.getAttribute('data-ev-basis-sql')))
|
||||
check('只挂了整份查询的句子(证据不足的那句):同样列出 SQL,读的人看得出缺的是什么', insuffSql.join(',') === 'Q1', insuffSql.join(','))
|
||||
await badge(pg, V, u.ok).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="supported"]')
|
||||
await pg.waitForTimeout(200)
|
||||
check('对照:挂了数字的结论句只列依据的编号,不另列 SQL', await p.locator('[data-ev-basis]').count() === 0
|
||||
&& (await p.locator('[data-ev-cites]').innerText()).includes('Q1'))
|
||||
// 表名实体步骤:字段清单和类型
|
||||
await pg.locator(`${V} [data-seg="${fxv.formal.segs.orders}"]`).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-entity] [data-ev-entity-queries]')
|
||||
// 字段清单没画出来时也往下走:让下面两条断言自己报没通过,而不是整节在等待上出错
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-entity] [data-ev-fields]', { timeout: 3000 }).catch(() => {})
|
||||
const fields = await p.locator('[data-ev-entity] [data-ev-field]').evaluateAll((els) => els.map((e) =>
|
||||
[e.getAttribute('data-ev-field'), e.querySelector('[data-ev-field-type]')?.textContent ?? '']))
|
||||
check('表名实体步骤列出字段清单和类型(照证据接口的 fields,顺序不变)', JSON.stringify(fields) === JSON.stringify(wantFields), JSON.stringify(fields))
|
||||
const fhead = await p.locator('[data-ev-entity] [data-ev-fields]').innerText().catch(() => '')
|
||||
check('……写明一共几个字段;没有多出来的不写「另有」', fhead.startsWith(`字段清单(${wantFields.length} 个)`)
|
||||
&& await p.locator('[data-ev-fields-more]').count() === 0, fhead.split('\n')[0])
|
||||
check('没有运行时报错(裁判拆档 · 面板)', r.errors.length === 0, r.errors.join(' | '))
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await pg.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
for (const [name, unit] of [['insufficient', u.insuff], ['contradicted', u.contra], ['method', u.method]]) {
|
||||
await badge(pg, V, unit).click()
|
||||
await pg.waitForTimeout(100)
|
||||
if (await p.count() === 0) await badge(pg, V, unit).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge]')
|
||||
await pg.waitForTimeout(300)
|
||||
await pg.screenshot({ path: `${SHOTS}/jr-panel-${name}-${theme}.png` })
|
||||
}
|
||||
await pg.locator(`${V} [data-seg="${fxv.formal.segs.orders}"]`).click()
|
||||
await pg.waitForSelector('[data-evidence-panel] [data-ev-fields]')
|
||||
await pg.waitForTimeout(300)
|
||||
await pg.screenshot({ path: `${SHOTS}/jr-panel-fields-${theme}.png` })
|
||||
await pg.keyboard.press('Escape')
|
||||
}
|
||||
await pg.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
await r.ctx.close()
|
||||
|
||||
// 字段多于 60 个、表结构快照只存了部分表
|
||||
const more = await open('/ui-harness.html?evidence=1', { segPatch: (set, body) => (set === fxv.formal && body?.segment?.id === fxv.formal.segs.orders
|
||||
? { ...body, chain: body.chain.map((c) => (c.step === 'entity' ? { ...c, fields: c.fields.slice(0, 5), fields_more: 55, snapshot_truncated: true,
|
||||
fields_masked: true } : c)) }
|
||||
: body) })
|
||||
const mp = more.page.locator('[data-evidence-panel]')
|
||||
await more.page.locator(`${V} [data-seg="${fxv.formal.segs.orders}"]`).click()
|
||||
await more.page.waitForSelector('[data-evidence-panel] [data-ev-fields-more]').catch(() => {})
|
||||
check('字段多于 60 个:列出的之外写「另有 N 个」', (await mp.locator('[data-ev-fields-more]').innerText().catch(() => '')) === '另有 55 个字段未列出',
|
||||
await mp.locator('[data-ev-fields-more]').innerText().catch(() => '(没写)'))
|
||||
check('……总数按列出的加另有的算', (await mp.locator('[data-ev-entity] [data-ev-fields]').innerText().catch(() => '')).startsWith('字段清单(60 个)')
|
||||
&& await mp.locator('[data-ev-entity] [data-ev-field]').count() === 5)
|
||||
check('有列按遮罩没列出(fields_masked):照实说一句', (await mp.locator('[data-ev-entity] [data-ev-fields-masked]').innerText().catch(() => ''))
|
||||
=== '部分字段已按数据源的遮罩设置隐藏,未列出')
|
||||
await badge(more.page, V, u.method).click()
|
||||
await more.page.waitForSelector('[data-evidence-panel] [data-ev-basis-table] [data-ev-fields-partial]').catch(() => {})
|
||||
check('讲方法的句子:表结构快照只存了部分表时,字段清单下照实注明', (await mp.locator('[data-ev-basis-table] [data-ev-fields-partial]')
|
||||
.innerText().catch(() => '')).includes('表结构快照仅包含部分表'))
|
||||
check('……另有的字段也写出来', (await mp.locator('[data-ev-basis-table] [data-ev-fields-more]').innerText().catch(() => '')) === '另有 55 个字段未列出')
|
||||
await more.ctx.close()
|
||||
|
||||
// 探索运行:封存后追加的证据不足、按需裁判答复里的 same_model / priced
|
||||
const x = fxv.explore.units
|
||||
const ex = await open('/ui-harness.html?evidence=1')
|
||||
const xp = ex.page.locator('[data-evidence-panel]')
|
||||
await ex.page.waitForSelector(`${VX} [data-ev-claim="${x.insuff}"][data-ev-verdict="insufficient"]`, { timeout: 4000 }).catch(() => {})
|
||||
await badge(ex.page, VX, x.insuff).click()
|
||||
await ex.page.waitForSelector('[data-evidence-panel] [data-ev-judge="insufficient"]')
|
||||
await ex.page.waitForTimeout(300)
|
||||
check('封存后追加的证据不足:同样写明缺什么', (await xp.locator('[data-ev-missing]').innerText().catch(() => '')) === `缺少:${fxv.missing}`)
|
||||
check('……证据图里最近一次按需裁判的模型和写作模型不同、在价格表里:不挂提醒', await xp.locator('[data-ev-judge-caution]').count() === 0)
|
||||
await badge(ex.page, VX, x.method).click()
|
||||
await xp.locator('[data-ev-judge-ask]').click()
|
||||
await ex.page.waitForSelector('[data-evidence-panel] [data-ev-judge="contradicted"]').catch(() => {})
|
||||
check('按需裁判回来:讲方法的那句判成证据相矛盾,正文徽标跟着变', await badge(ex.page, VX, x.method).getAttribute('data-ev-verdict') === 'contradicted')
|
||||
check('……这一次的答复说裁判模型和写作模型相同、不在价格表里:两条提醒都挂上',
|
||||
(await xp.locator('[data-ev-judge-same]').innerText().catch(() => '')) === '裁判模型和写作模型相同,结果仅供参考'
|
||||
&& (await xp.locator('[data-ev-judge-unpriced]').innerText().catch(() => '')) === '该模型不在价格表中,金额上限不生效')
|
||||
await ex.page.keyboard.press('Escape')
|
||||
await ex.page.waitForTimeout(150)
|
||||
await ex.page.locator(`${VX} [data-seg="${firstNumber(fxv.explore.doc, x.ok)}"]`).focus()
|
||||
const xj = []
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await ex.page.keyboard.press('n')
|
||||
xj.push(await activeKey(ex.page))
|
||||
}
|
||||
check('n:旧的证据不支持、证据相矛盾在前,证据不足在后(它在正文里更靠前也一样)',
|
||||
xj.join(',') === [x.contra, x.method, x.insuff].map((k) => `claim:${k}`).join(','), xj.join(','))
|
||||
// 封存后追加的旧判定(拆档之前的 unsupported):片段接口给 on_demand.reason = outdated、available 为真——照常给按钮,
|
||||
// 旧判定旁边注明「按旧规则判定」;按当前规则重判之后注明和按钮都收起
|
||||
const outdatedMsg = fxv.explore.segments[fxv.explore.doc.blocks.flatMap((b) => b.units).find((u2) => u2.id === x.contra)
|
||||
.segments.find((sg) => sg.kind !== 'structural').id].unit.on_demand
|
||||
// 面板可能还开着、正停在这一句上(上面 n 跟着走过来):点一下是收起,没开出来就再点一次
|
||||
await badge(ex.page, VX, x.contra).click()
|
||||
await ex.page.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]', { timeout: 1500 }).catch(() => {})
|
||||
if (await xp.locator('[data-ev-judge="unsupported"]').count() === 0) await badge(ex.page, VX, x.contra).click()
|
||||
await ex.page.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]', { timeout: 4000 }).catch(() => {})
|
||||
await ex.page.waitForSelector('[data-evidence-panel] [data-ev-judge-outdated]').catch(() => {})
|
||||
const chip = xp.locator('[data-ev-judge-outdated]')
|
||||
check('旧判定(片段接口说 outdated):旁边注明「按旧规则判定」,悬停是接口的原话', outdatedMsg?.reason === 'outdated'
|
||||
&& (await chip.innerText().catch(() => '')) === '按旧规则判定' && await chip.getAttribute('title').catch(() => '') === outdatedMsg.message,
|
||||
JSON.stringify(outdatedMsg))
|
||||
check('……照常给按钮(再次请模型判断),按钮旁写明为什么', (await xp.locator('[data-ev-judge-ask]').innerText().catch(() => '')).includes('再次请模型判断')
|
||||
&& (await xp.locator('[data-ev-judge-outdated-why]').innerText().catch(() => '')) === outdatedMsg.message)
|
||||
await xp.locator('[data-ev-judge-ask]').click().catch(() => {})
|
||||
await ex.page.waitForSelector('[data-evidence-panel] [data-ev-judge="contradicted"]').catch(() => {})
|
||||
check('……按当前规则重判:判成证据相矛盾,「按旧规则判定」和按钮都收起', await badge(ex.page, VX, x.contra).getAttribute('data-ev-verdict') === 'contradicted'
|
||||
&& await xp.locator('[data-ev-judge-outdated]').count() === 0 && await xp.locator('[data-ev-judge-ask]').count() === 0
|
||||
&& ex.hits.some((h) => h === `judge:${fxv.explore.run_id}:${x.contra}:write`), ex.hits.filter((h) => h.startsWith('judge:')).join(' '))
|
||||
if (SHOTS) {
|
||||
// 重新开一页截「按旧规则判定」那一刻(上面已经重判掉了)
|
||||
const shot = await open('/ui-harness.html?evidence=1')
|
||||
await shot.page.waitForSelector(`${VX} [data-ev-claim="${x.contra}"][data-ev-verdict="unsupported"]`).catch(() => {})
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await shot.page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await badge(shot.page, VX, x.contra).click()
|
||||
await shot.page.waitForTimeout(100)
|
||||
if (await shot.page.locator('[data-evidence-panel]').count() === 0) await badge(shot.page, VX, x.contra).click()
|
||||
await shot.page.waitForSelector('[data-evidence-panel] [data-ev-judge-outdated]').catch(() => {})
|
||||
await shot.page.waitForTimeout(300)
|
||||
await shot.page.screenshot({ path: `${SHOTS}/jr-panel-outdated-${theme}.png` })
|
||||
}
|
||||
await shot.ctx.close()
|
||||
}
|
||||
check('没有运行时报错(裁判拆档 · 探索运行)', ex.errors.length === 0, ex.errors.join(' | '))
|
||||
await ex.ctx.close()
|
||||
})
|
||||
|
||||
await section('jr-narrow', '裁判拆档在 360px:理由 120 字、字段清单、SQL 栏内展开不横向滚动;底部抽屉整页不横向滚动', async () => {
|
||||
const overflow = () => page.evaluate((sel) => {
|
||||
const el = document.querySelector(sel)
|
||||
return { sw: el.scrollWidth, cw: el.clientWidth, w: el.getBoundingClientRect().width }
|
||||
}, VN)
|
||||
const before = await overflow()
|
||||
await badge(page, VN, fxv.formal.units.contra).click()
|
||||
await page.waitForSelector(`${VN} [data-evidence-panel="inline"] [data-ev-rationale]`)
|
||||
const why = await page.locator(`${VN} [data-ev-rationale]`).evaluate((el) => ({ text: el.textContent, sw: el.scrollWidth, cw: el.clientWidth }))
|
||||
const after = await overflow()
|
||||
check('窄栏里理由 120 字整句显示、在栏内折行', why.text === fxv.long_rationale && why.sw <= why.cw, JSON.stringify({ sw: why.sw, cw: why.cw }))
|
||||
check('……面板打开前后都没有横向滚动', Math.round(before.w) === 360 && before.sw <= before.cw && after.sw <= after.cw, JSON.stringify({ before, after }))
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await page.waitForTimeout(200)
|
||||
await page.locator(VN).screenshot({ path: `${SHOTS}/jr-360-rationale-${theme}.png` })
|
||||
}
|
||||
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
await badge(page, VN, fxv.formal.units.method).click()
|
||||
await page.waitForSelector(`${VN} [data-evidence-panel="inline"] [data-ev-basis] [data-ev-sql]`).catch(() => {})
|
||||
await page.waitForSelector(`${VN} [data-evidence-panel="inline"] [data-ev-basis-table] [data-ev-field]`).catch(() => {})
|
||||
const basis = await overflow()
|
||||
const pre = await page.locator(`${VN} [data-ev-basis] [data-ev-sql]`).first().evaluate((el) => ({ sw: el.scrollWidth, cw: el.clientWidth }))
|
||||
.catch(() => null)
|
||||
check('窄栏里讲方法的句子:SQL、字段清单都在栏内,不横向滚动', basis.sw <= basis.cw && !!pre && pre.sw <= pre.cw
|
||||
&& await page.locator(`${VN} [data-ev-basis-table] [data-ev-field]`).count() > 0, JSON.stringify({ basis, pre }))
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await page.waitForTimeout(200)
|
||||
await page.locator(VN).screenshot({ path: `${SHOTS}/jr-360-basis-${theme}.png` })
|
||||
}
|
||||
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
await page.locator(`${VN} [data-evidence-panel] button`, { hasText: '回到正文' }).click()
|
||||
await page.waitForTimeout(150)
|
||||
const small = await open('/ui-harness.html?evidence=1', { w: 360, h: 780 })
|
||||
const scroll = () => small.page.evaluate(() => ({ sw: document.documentElement.scrollWidth, vw: innerWidth }))
|
||||
await badge(small.page, V, fxv.formal.units.contra).click()
|
||||
await small.page.waitForSelector('[data-evidence-panel] [data-ev-rationale]')
|
||||
const d = await scroll()
|
||||
const dwhy = await small.page.locator('[data-evidence-panel] [data-ev-rationale]').evaluate((el) => ({ text: el.textContent, sw: el.scrollWidth, cw: el.clientWidth }))
|
||||
check('360px 宽的屏幕:底部抽屉里理由整句显示、整页没有横向滚动',
|
||||
await small.page.locator('[data-evidence-panel]').getAttribute('data-evidence-panel') === 'drawer' && d.sw <= d.vw
|
||||
&& dwhy.text === fxv.long_rationale && dwhy.sw <= dwhy.cw, JSON.stringify({ d, sw: dwhy.sw, cw: dwhy.cw }))
|
||||
await small.ctx.close()
|
||||
})
|
||||
|
||||
await section('jr-banner', '裁判拆档:出具横幅的结论句计数带上证据相矛盾、证据不足和旧的证据不支持', async () => {
|
||||
const at = (key) => page.locator(`#issuance-claims [data-issuance-case="${key}"]`)
|
||||
const rework = await at('rework').locator('[data-evidence-line]').innerText().catch(() => '')
|
||||
check('计数:证据相矛盾、旧的证据不支持、证据不足各写各的', rework.includes('结论 7 句(有依据 1 · 部分有依据 1 · 证据相矛盾 2 · 证据不支持 1 · 证据不足 1 · 未裁判 1)'),
|
||||
rework)
|
||||
const [color, waiting] = [await at('rework').locator('[data-evidence-claims]').evaluate((el) => getComputedStyle(el).color).catch(() => ''),
|
||||
await colorOf(page, '--st-waiting')]
|
||||
check('……有问题的计数用提醒色', color === waiting, `${color} / ${waiting}`)
|
||||
check('只有证据不足一种问题:照样给「定位下一处」', await at('insufficient').locator('[data-evidence-next]').count() === 1)
|
||||
const only = await at('insufficient').locator('[data-evidence-claims]').evaluate((el) => getComputedStyle(el).color).catch(() => '')
|
||||
check('……计数也用提醒色(证据不足也算问题)', only === waiting, only)
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await page.waitForTimeout(200)
|
||||
await page.locator('#issuance-claims').screenshot({ path: `${SHOTS}/jr-banner-${theme}.png` })
|
||||
}
|
||||
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
})
|
||||
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
|
||||
@@ -626,9 +626,34 @@ await section('设置 · 证据裁判', async () => {
|
||||
const box = page.locator('[data-judge-settings]')
|
||||
const all = (await box.innerText().catch(() => '')).replace(/\s+/g, ' ')
|
||||
check('偏好里有「证据裁判」一组,写明判断是模型给的、不是系统核对', all.includes('证据裁判') && all.includes('非确定'), all.slice(0, 80))
|
||||
// 裁判拆档:接入和模型都没填时,平常那句建议换成醒目的「未单独配置」提示(建议写在提示里);填了再换回来(见下)
|
||||
check('裁判模型:接入和模型,提示建议和写作模型不同', await page.locator('#pref-judge-provider').count() === 1
|
||||
&& await page.locator('#pref-judge-model').count() === 1 && all.includes('建议与撰写报告的模型不同'))
|
||||
&& await page.locator('#pref-judge-model').count() === 1 && all.includes('建议配置一个与撰写报告的模型不同、能力更强的模型'))
|
||||
check('……接入留空时跟随助手的模型', (await page.locator('#pref-judge-provider option').first().innerText()).includes('跟随助手的模型'))
|
||||
const notSet = page.locator('[data-judge-model] [data-judge-not-set]')
|
||||
check('没配裁判模型(接入、模型都是 null):提示「未单独配置,将使用助手的模型」',
|
||||
(await notSet.locator('[data-judge-not-set-text]').innerText().catch(() => '')) === '未单独配置,将使用助手的模型',
|
||||
await notSet.innerText().catch(() => '(没有这条提示)'))
|
||||
const advice = await notSet.locator('[data-judge-not-set-advice]').innerText().catch(() => '')
|
||||
check('……并建议配一个和写作模型不同、能力更强的模型', advice.includes('与撰写报告的模型不同') && advice.includes('能力更强'), advice)
|
||||
const noteLook = await notSet.evaluate((el) => {
|
||||
const probe = document.createElement('span')
|
||||
probe.style.color = 'var(--st-waiting)'
|
||||
document.body.appendChild(probe)
|
||||
const waiting = getComputedStyle(probe).color
|
||||
probe.remove()
|
||||
const cs = getComputedStyle(el)
|
||||
return { border: cs.borderLeftColor, width: cs.borderLeftWidth, icon: getComputedStyle(el.querySelector('svg')).color, waiting,
|
||||
described: document.getElementById(el.closest('[data-judge-model]').getAttribute('aria-describedby'))?.hasAttribute('data-judge-not-set') }
|
||||
}).catch(() => null)
|
||||
check('……醒目:左侧提醒色竖线、提醒色图标', !!noteLook && noteLook.border === noteLook.waiting && noteLook.width === '2px'
|
||||
&& noteLook.icon === noteLook.waiting, JSON.stringify(noteLook))
|
||||
check('……读屏:模型那一组的说明就是这条提示(aria-describedby)', noteLook?.described === true)
|
||||
if (SHOTS) {
|
||||
await box.scrollIntoViewIfNeeded()
|
||||
await page.waitForTimeout(200)
|
||||
await box.screenshot({ path: `${SHOTS}/prefs-judge-not-set-${THEME}.png` })
|
||||
}
|
||||
const val = (id) => page.locator(`#${id}`).inputValue()
|
||||
check('读回各项上限:40 句、30 秒、每次点击 $0.01、每日 $2', await val('pref-judge-claims') === '40' && await val('pref-judge-timeout') === '30'
|
||||
&& await val('pref-judge-click') === '0.01' && await val('pref-judge-daily') === '2')
|
||||
@@ -654,6 +679,8 @@ await section('设置 · 证据裁判', async () => {
|
||||
await page.locator('#pref-judge-click').fill('0.02')
|
||||
check('……接入留空时模型框的占位也写跟随助手的模型', await page.locator('#pref-judge-model').getAttribute('placeholder') === '留空:跟随助手的模型')
|
||||
await page.locator('#pref-judge-model').fill('judge-model-b')
|
||||
check('填了裁判模型:「未单独配置」的提示收起,换回平常那句建议', await notSet.count() === 0
|
||||
&& (await box.innerText().catch(() => '')).includes('建议与撰写报告的模型不同'))
|
||||
// 只填了模型:设置这一级整组生效(judge_model_spec 不跨级拼),后端按模型名找接入——已经不跟随助手的模型
|
||||
const provFirst = await page.locator('#pref-judge-provider option').first().innerText().catch(() => '')
|
||||
check('只填了模型:接入留空那一项不再叫「跟随助手的模型」,改叫「按模型名找接入」', provFirst === '按模型名找接入', provFirst)
|
||||
@@ -690,6 +717,15 @@ await section('设置 · 证据裁判', async () => {
|
||||
}
|
||||
await close()
|
||||
|
||||
// 只填了接入(模型留空):也算单独配置了,不提示「未单独配置」
|
||||
const byProvider = { ...stored, judge: { ...stored.judge, provider: 'judge-provider-x', model: null } }
|
||||
const pv = await open('/settings/prefs', { handlers: [[/^GET \/settings$/, (route) => json(byProvider)(route)],
|
||||
[/^GET \/settings\/judge\/spend$/, (route) => json({ date: '2026-09-29', usd: 0, calls: 0, unpriced_calls: 0 })(route)]] })
|
||||
await pv.page.waitForSelector('[data-judge-settings]', { timeout: 6000 }).catch(() => {})
|
||||
check('只填了接入:不提示「未单独配置」', await pv.page.locator('[data-judge-settings]').count() === 1
|
||||
&& await pv.page.locator('[data-judge-not-set]').count() === 0)
|
||||
await pv.close()
|
||||
|
||||
// 老后端没有 judge 这一组:不显示,也不写
|
||||
const legacy = { ...stored }
|
||||
delete legacy.judge
|
||||
|
||||
@@ -712,6 +712,8 @@ await section('保存失败、接口不存在、画布改过:都有明确状
|
||||
bad.state.save = 'ok'
|
||||
await dialog(bad.page).getByRole('button', { name: '重试保存' }).click()
|
||||
await waitFor(bad.page, async () => bad.state.checks.length === 2)
|
||||
// 重查的请求发出去了,按钮的可用状态要等答复回来、界面更新之后才变:并行跑的时候机器挤,直接读会抢在前面
|
||||
await waitFor(bad.page, async () => submit(bad.page).isEnabled())
|
||||
check('重试保存成功后重新检查、发布可用', bad.state.patches.length === 2 && await submit(bad.page).isEnabled())
|
||||
|
||||
const nofix = await open({ backend: 'nofix' })
|
||||
|
||||
@@ -651,6 +651,9 @@ await section('提取模板', async () => {
|
||||
fakes.set('POST /api/copilot/from-run', () => ({
|
||||
status: 200, json: { workflow_id: wfs[0].id, name: '检查用草稿', nodes: 3, edges: 2, dropped_nodes: 1, source_run: timed?.id },
|
||||
}))
|
||||
// 跳进画布后,画布先找这张图的助手对话,没有就建一条(POST,会被探针拦下、算成没伪造的写请求)。
|
||||
// 沙箱里 wfs[0] 有没有对话要看之前谁跑过什么:给一条现成的,和升级横幅那一节同一个做法
|
||||
fakes.set('GET /api/conversations', () => ({ status: 200, json: [{ id: 'fake-conv-extract', title: '', kind: 'canvas', archived: false, turn_count: 0 }] }))
|
||||
// 画布上还有没保存的改动(store 里的画布离开编排页也还在):先问要不要放弃,再建草稿。
|
||||
// 以前建完才问,点了「取消」库里就多出一张没人要的草稿(3C REQ-15)
|
||||
const extractPosts = () => writes.filter((w) => w === 'POST /api/copilot/from-run').length
|
||||
@@ -684,6 +687,7 @@ await section('提取模板', async () => {
|
||||
&& new URL(page.url()).pathname === `/studio/${wfs[0].id}`, `提取 ${extractPosts() - postsBefore} 次 · 问了 ${asks} 遍`)
|
||||
check('提取成功后跳到新草稿', new URL(page.url()).pathname === `/studio/${wfs[0].id}`)
|
||||
fakes.delete('POST /api/copilot/from-run')
|
||||
fakes.delete('GET /api/conversations')
|
||||
})
|
||||
|
||||
// ------------------------------------------------------------------ 运行中:接流、停止、不能删
|
||||
@@ -1981,6 +1985,54 @@ await section('证据页签:同一段文字里好几条同样的违规、正
|
||||
}
|
||||
})
|
||||
|
||||
await section('证据页签:裁判拆档——证据相矛盾、证据不足的结论句各一行,点开是这一句的「模型的解释」', async () => {
|
||||
// 夹具是后端(JR-back 之后)真算的:审计行由 api/evidence.py 的 _audit_doc 给出,判定行 issue 为
|
||||
// contradicted_claim / insufficient_claim,带着 unit 和 verdict
|
||||
const fxv = JSON.parse(readFileSync(new URL('../frontend/src/run/__tests__/evidence-verdict.json', import.meta.url), 'utf8'))
|
||||
const v = fxv.formal
|
||||
const EV_V = 'fake0evverdict000000000000000'
|
||||
const output = { answer: v.doc.markdown, _evidence: { report_node: 'write', doc_artifact: v.doc_artifact, fields: ['answer'] } }
|
||||
evFakes(EV_V, { output, graph: v.graph, audit: v.audit })
|
||||
fakes.set(`GET /api/artifacts/${v.doc_artifact}`, () => ({ status: 200, json: { id: v.doc_artifact, content: v.doc } }))
|
||||
for (const snap of Object.values(fxv.snapshots)) {
|
||||
fakes.set(`GET /api/artifacts/${snap.artifact}`, () => ({ status: 200, json: { id: snap.artifact, content: snap.content } }))
|
||||
}
|
||||
await page.route(new RegExp(`/api/runs/${EV_V}/evidence/segments/`), (r) => {
|
||||
const sid = decodeURIComponent(new URL(r.request().url()).pathname.split('/').pop())
|
||||
return v.segments[sid] ? r.fulfill({ json: v.segments[sid] }) : r.fulfill({ status: 404, json: { detail: '没有这个片段' } })
|
||||
})
|
||||
try {
|
||||
await openEvidence(EV_V)
|
||||
await page.locator('[data-audit-table]').waitFor({ timeout: 5000 }).catch(() => {})
|
||||
const claimRows = await page.locator('[data-audit-table] tbody[data-audit-group=none] tr[data-audit-row]').evaluateAll((els) =>
|
||||
els.map((e) => ({ state: e.getAttribute('data-audit-state'), open: e.querySelector('[data-audit-open]')?.getAttribute('data-audit-open') ?? null,
|
||||
stateText: e.children[1]?.textContent ?? '', source: e.querySelector('[data-audit-source]')?.textContent ?? '' })))
|
||||
const byState = (st) => claimRows.filter((r) => r.state === st)
|
||||
check('审计表:证据相矛盾的结论句两行、证据不足一行,列在「无证据」一组', byState('contradicted').length === 2 && byState('insufficient').length === 1,
|
||||
JSON.stringify(claimRows.map((r) => r.state)))
|
||||
check('……状态照句末徽标的字形和叫法写', byState('contradicted').every((r) => r.stateText === '!模型判断:证据相矛盾')
|
||||
&& byState('insufficient')[0]?.stateText === '○模型判断:证据不足', JSON.stringify(claimRows.map((r) => r.stateText)))
|
||||
const note = v.audit.groups.flatMap((g) => g.rows).find((r) => r.issue === 'insufficient_claim')?.note ?? '(夹具里没有这一行)'
|
||||
check('……原因照后端的原话(缺什么、理由)', byState('insufficient')[0]?.source === note && note.includes(`缺少:${fxv.missing}`), note)
|
||||
check('……判定行点得开,打开的是这一句的句末徽标(claim:<句子>)', byState('insufficient')[0]?.open === `claim:${v.units.insuff}`
|
||||
&& byState('contradicted').map((r) => r.open).join(',') === [v.units.contra, v.units.method].map((u) => `claim:${u}`).join(','),
|
||||
JSON.stringify(claimRows.map((r) => r.open)))
|
||||
await page.locator(`[data-audit-table] [data-audit-open="claim:${v.units.insuff}"]`).click()
|
||||
await page.locator('[data-evidence-dock] [data-evidence-panel] [data-ev-judge="insufficient"]').waitFor({ timeout: 5000 }).catch(() => {})
|
||||
const dock = page.locator('[data-evidence-dock] [data-evidence-panel]')
|
||||
check('点开证据不足那一行:右边打开这一句的「模型的解释」,写明缺什么', (await dock.locator('[data-ev-missing]').innerText().catch(() => ''))
|
||||
=== `缺少:${fxv.missing}`)
|
||||
check('……裁判模型和写作模型相同、不在价格表里:两条提醒都在', await dock.locator('[data-ev-judge-same]').count() === 1
|
||||
&& await dock.locator('[data-ev-judge-unpriced]').count() === 1)
|
||||
const tally = await page.locator(`[data-evidence-report=write] [data-evidence-claims]`).innerText().catch(() => '')
|
||||
check('报告上方的证据条:结论句计数带上证据相矛盾、证据不足', tally.includes('结论 6 句(有依据 1 · 部分有依据 1 · 证据相矛盾 2 · 证据不足 1 · 未裁判 1)'), tally)
|
||||
await page.keyboard.press('Escape')
|
||||
} finally {
|
||||
for (const k of [...fakes.keys()]) if (k.includes(EV_V) || k.includes(v.doc_artifact)) fakes.delete(k)
|
||||
for (const snap of Object.values(fxv.snapshots)) fakes.delete(`GET /api/artifacts/${snap.artifact}`)
|
||||
}
|
||||
})
|
||||
|
||||
await section('证据页签:没有报告的运行照实说明(none / 旧运行猜测 / 老后端)', async () => {
|
||||
evFakes(EV_LEGACY, { output: fxe.legacy.output, graph: fxe.legacy.graph, audit: fxe.legacy.audit, report: false })
|
||||
await openEvidence(EV_LEGACY)
|
||||
|
||||
@@ -458,9 +458,9 @@ await section('lib/evidence:证据状态的四通道元数据、报告节点
|
||||
const ok = (name, cond, detail = '') => out.push([name, !!cond, cond ? '' : detail])
|
||||
const codes = ev.EVIDENCE_STATES
|
||||
// 方案 6.1 的八种(确定性、概率性四种、无证据、连接性、旧运行候选),加三期无证据的两个变体:
|
||||
// 可疑实体(疑似不存在的名称)、无法核实(表结构快照不全)
|
||||
ok('十种状态一种不少', ['deterministic', 'supported', 'partial', 'unsupported', 'unjudged', 'none', 'connective', 'candidate',
|
||||
'suspect', 'unverified'].every((c) => codes.includes(c)) && codes.length === 10, codes.join(','))
|
||||
// 可疑实体(疑似不存在的名称)、无法核实(表结构快照不全),加裁判拆档的两种:证据相矛盾、证据不足
|
||||
ok('十二种状态一种不少', ['deterministic', 'supported', 'partial', 'contradicted', 'insufficient', 'unsupported', 'unjudged', 'none',
|
||||
'connective', 'candidate', 'suspect', 'unverified'].every((c) => codes.includes(c)) && codes.length === 12, codes.join(','))
|
||||
eq('可疑实体:点状线(同无证据)、?!、疑似不存在的名称', `${ev.EVIDENCE_STATE.suspect.line}/${ev.EVIDENCE_STATE.suspect.glyph}/${ev.EVIDENCE_STATE.suspect.label}`,
|
||||
'dotted/?!/疑似不存在的名称')
|
||||
ok('可疑实体醒目(进 n / N),核对不了只是标注', ev.EVIDENCE_STATE.suspect.alert && !ev.EVIDENCE_STATE.unverified.alert)
|
||||
@@ -492,8 +492,12 @@ await section('lib/evidence:证据状态的四通道元数据、报告节点
|
||||
const d = ev.EVIDENCE_STATE[c].decoration
|
||||
return d === 'transparent' || /var\(--st-[\w-]+\)/.test(d)
|
||||
}), codes.map((c) => ev.EVIDENCE_STATE[c].decoration).join(' | '))
|
||||
const pairs = codes.map((c) => `${ev.EVIDENCE_STATE[c].line}/${ev.EVIDENCE_STATE[c].glyph}`)
|
||||
ok('去掉颜色也分得开:线型 + 字形两两不同', new Set(pairs).size === codes.length, pairs.join(' '))
|
||||
// 裁判拆档:旧取值「证据不支持」按规格和证据相矛盾长得一样(同一类判定,拆档前后的叫法不同,文字仍不同),
|
||||
// 除它之外线型 + 字形两两不同
|
||||
const pairOf = (c) => `${ev.EVIDENCE_STATE[c].line}/${ev.EVIDENCE_STATE[c].glyph}`
|
||||
const pairs = codes.filter((c) => c !== 'unsupported').map(pairOf)
|
||||
ok('去掉颜色也分得开:线型 + 字形两两不同(旧的证据不支持同证据相矛盾)', new Set(pairs).size === codes.length - 1
|
||||
&& pairOf('unsupported') === pairOf('contradicted'), codes.map(pairOf).join(' '))
|
||||
const labels = codes.map((c) => ev.EVIDENCE_STATE[c].label)
|
||||
ok('文字两两不同', new Set(labels).size === codes.length, labels.join(' '))
|
||||
eq('确定性:细实线、有出处', `${ev.EVIDENCE_STATE.deterministic.line}/${ev.EVIDENCE_STATE.deterministic.label}`, 'solid/有出处')
|
||||
@@ -663,6 +667,94 @@ await section('lib/evidence:结论句裁判(四期)——判定的徽标
|
||||
})) check(name, ok, detail)
|
||||
})
|
||||
|
||||
await section('lib/evidence:裁判拆档(JR)——证据相矛盾、证据不足、计数、跳转轮次、模型提醒、审计行、讲方法的句子', async () => {
|
||||
for (const [name, ok, detail] of await page.evaluate(() => {
|
||||
const { evidence: ev, terms: t } = window.__ui.lib
|
||||
const fv = window.__ui.evidenceVerdict
|
||||
const out = []
|
||||
const eq = (name, got, want) => out.push([name, got === want, got === want ? '' : `得到 ${JSON.stringify(got)},应为 ${JSON.stringify(want)}`])
|
||||
const ok = (name, cond, detail = '') => out.push([name, !!cond, cond ? '' : detail])
|
||||
const S = ev.EVIDENCE_STATE
|
||||
eq('证据相矛盾、证据不足:句末徽标、字形 ! ○', ['contradicted', 'insufficient'].map((c) => `${S[c].line}/${S[c].glyph}`).join(' '),
|
||||
'badge/! badge/○')
|
||||
eq('……颜色令牌:证据相矛盾同原来的证据不支持(--st-failed),证据不足中性(--st-cancelled)',
|
||||
['contradicted', 'insufficient', 'unsupported'].map((c) => S[c].color).join(' '), 'var(--st-failed) var(--st-cancelled) var(--st-failed)')
|
||||
ok('……证据不足不用红、不用提醒色,字形不和未裁判的 ? 撞', !/--st-(failed|waiting|done)/.test(S.insufficient.color + S.insufficient.soft)
|
||||
&& S.insufficient.glyph !== S.unjudged.glyph, JSON.stringify(S.insufficient))
|
||||
eq('叫法:证据相矛盾、证据不足;旧取值照旧叫证据不支持', [S.contradicted.label, S.insufficient.label, S.unsupported.label].join(' / '),
|
||||
'模型判断:证据相矛盾 / 模型判断:证据不足 / 模型判断:证据不支持')
|
||||
eq('判定 → 徽标:新取值各是各的', [ev.verdictState({ status: 'contradicted' }), ev.verdictState({ status: 'insufficient' }),
|
||||
ev.verdictState({ status: 'unsupported' })].join(','), 'contradicted,insufficient,unsupported')
|
||||
ok('判过的算上新取值(不再给「请模型判断」)', ev.isJudged({ status: 'contradicted' }) && ev.isJudged({ status: 'insufficient' }))
|
||||
eq('n / N 的轮次:证据相矛盾、旧的证据不支持、部分有依据、无证据在第 1 轮,证据不足在第 2 轮,有依据、未裁判不进',
|
||||
['contradicted', 'unsupported', 'partial', 'none', 'suspect', 'insufficient', 'supported', 'unjudged'].map((c) => ev.alertTier(c)).join(''),
|
||||
'11111200')
|
||||
ok('整句铺浅底的是证据相矛盾和旧的证据不支持', ev.conflicting('contradicted') && ev.conflicting('unsupported')
|
||||
&& !ev.conflicting('insufficient') && !ev.conflicting('partial'))
|
||||
eq('证据不足缺什么:只认 insufficient 的 missing', [ev.verdictMissing({ status: 'insufficient', missing: ' SQL ' }),
|
||||
ev.verdictMissing({ status: 'contradicted', missing: 'SQL' }), ev.verdictMissing({ status: 'insufficient' })].join('|'), 'SQL||')
|
||||
const k = (c) => (c ? `${c.total}:${c.supported}/${c.partial}/${c.contradicted}/${c.unsupported}/${c.insufficient}/${c.unjudged}/${c.uncited}` : 'null')
|
||||
eq('文档计数:证据相矛盾 2、证据不足 1,不是结论句不算', k(ev.claimCounts(fv.formal.doc)), '6:1/1/2/0/1/1/0')
|
||||
eq('……叠上封存后追加的判定(旧取值 unsupported 照样计入)', k(ev.claimCounts(fv.explore.doc,
|
||||
ev.graphVerdicts(fv.explore.graph, 'write'))), '7:0/0/0/1/1/5/0')
|
||||
eq('stats / counts → 计数(report.checked、报告节点的章):旧数据的 unsupported 照样计入,不是结论句不算',
|
||||
k(ev.claimCountsOf({ supported: 1, partial: 1, contradicted: 2, insufficient: 1, unsupported: 1, not_a_claim: 3, unjudged: 1,
|
||||
numbers: 9 })), '7:1/1/2/1/1/1/0')
|
||||
eq('……缺键、不是数的当 0', k(ev.claimCountsOf({ supported: 2, unsupported: 'x' })), '2:2/0/0/0/0/0/0')
|
||||
eq('计数的说法:证据相矛盾、证据不支持、证据不足各写各的', t.claimTally(ev.claimCountsOf({ supported: 1, contradicted: 2, unsupported: 1,
|
||||
insufficient: 1, unjudged: 1 })), '结论 6 句(有依据 1 · 证据相矛盾 2 · 证据不支持 1 · 证据不足 1 · 未裁判 1)')
|
||||
eq('……旧的计数对象(没有新键)照旧能说', t.claimTally({ total: 2, supported: 1, partial: 0, unsupported: 1, unjudged: 0, uncited: 0 }),
|
||||
'结论 2 句(有依据 1 · 证据不支持 1)')
|
||||
eq('有问题的句数(横幅、证据条的「定位下一处」):证据相矛盾、旧的证据不支持、部分有依据、证据不足', ev.claimProblems(ev.claimCountsOf(
|
||||
{ supported: 5, contradicted: 1, unsupported: 1, partial: 1, insufficient: 1, unjudged: 3 })), 4)
|
||||
const unit = fv.formal.doc.blocks.flatMap((b) => b.units).find((u) => u.id === fv.formal.units.insuff)
|
||||
eq('读屏名:证据不足接着说缺什么', ev.claimLabel(unit, unit.verdict),
|
||||
`结论句「增长主要来自新客首单,老客复购持平。」,模型判断:证据不足,缺少:${fv.missing}`)
|
||||
const sealed = fv.formal.doc.blocks.flatMap((b) => b.units).find((u) => u.id === fv.formal.units.contra).verdict
|
||||
const f1 = ev.judgeFlags(sealed, { doc: fv.formal.doc.judge })
|
||||
ok('模型提醒:节点里当场判的看报告文档的 judge(same_model 真、priced 假)', f1.sameModel && f1.unpriced && f1.writer === fv.model, JSON.stringify(f1))
|
||||
const late = { status: 'insufficient', post_seal: true, judge: 'm' }
|
||||
const f2 = ev.judgeFlags(late, { doc: { same_model: true, priced: false }, meta: { same_model: false, priced: true } })
|
||||
ok('……封存后追加的不看报告文档,看证据图里最近一次按需裁判(judge_meta)', !f2.sameModel && !f2.unpriced, JSON.stringify(f2))
|
||||
const f3 = ev.judgeFlags(late, { reply: { same_model: true, priced: false }, meta: { same_model: false, priced: true } })
|
||||
ok('……这一次的答复盖过证据图', f3.sameModel && f3.unpriced, JSON.stringify(f3))
|
||||
const f4 = ev.judgeFlags({ ...late, same_model: false }, { reply: { same_model: true, priced: false } })
|
||||
ok('……判定自己带着的盖过答复', !f4.sameModel && f4.unpriced, JSON.stringify(f4))
|
||||
const f5 = ev.judgeFlags({ status: 'unjudged', reason: 'max_claims' }, { doc: { same_model: true, priced: false } })
|
||||
ok('……没判的句子不提醒;priced 为 null(不知道)也不提醒', !f5.sameModel && !f5.unpriced
|
||||
&& !ev.judgeFlags(sealed, { doc: { priced: null } }).unpriced, JSON.stringify(f5))
|
||||
const rows = ev.auditFromApi(fv.formal.audit)
|
||||
const claim = (issue) => rows.find((r) => r.kind === 'claim' && fv.formal.audit.groups.flatMap((g) => g.rows)
|
||||
.some((x) => x.issue === issue && x.unit === r.unit))
|
||||
const c1 = claim('contradicted_claim')
|
||||
const c2 = claim('insufficient_claim')
|
||||
ok('审计行:issue 为 contradicted_claim 的结论句认成证据相矛盾,insufficient_claim 认成证据不足,带着是哪一句',
|
||||
c1?.state === 'contradicted' && c2?.state === 'insufficient' && c2.unit === fv.formal.units.insuff && c1.group === 'none',
|
||||
JSON.stringify([c1, c2].map((r) => r && { state: r.state, unit: r.unit, group: r.group })))
|
||||
ok('……旧取值 unsupported_claim 认成证据不支持;没挂依据的结论句照旧是无证据',
|
||||
ev.auditFromApi({ groups: [{ key: 'none', rows: [{ kind: 'claim', issue: 'unsupported_claim', unit: 'u2', text: 'x', group: 'none' },
|
||||
{ kind: 'claim', issue: 'uncited_claim', unit: 'u7', text: 'y', group: 'none', state: 'none' }] }] }).map((r) => r.state).join(',')
|
||||
=== 'unsupported,none')
|
||||
const doc = fv.formal.doc
|
||||
const units = Object.fromEntries(doc.blocks.flatMap((b) => b.units).map((u) => [u.id, u]))
|
||||
const b = ev.claimBasis(units[fv.formal.units.method], doc)
|
||||
ok('讲方法的句子(只挂了表名、字段名、整份查询):列表、字段、要列 SQL 的查询(直接挂的在前,再补表名出现过的)',
|
||||
!!b && b.tables.map((x) => `${x.alias}=${x.name}@${x.seg}`).join() === `t:orders=orders@${fv.formal.segs.orders}`
|
||||
&& b.columns.map((x) => x.name).join() === 'orders.status,orders.created_at' && b.queries.join() === 'Q1,Q2' && b.more === 0, JSON.stringify(b))
|
||||
eq('……只挂了整份查询的句子也算', ev.claimBasis(units[fv.formal.units.insuff], doc)?.queries.join(), 'Q1')
|
||||
eq('……挂了数字(单元格)的句子不算', ev.claimBasis(units[fv.formal.units.ok], doc), null)
|
||||
eq('……挂了运行输入的句子不算', ev.claimBasis(units[fv.formal.units.not], doc), null)
|
||||
eq('……一个依据都没挂的不算', ev.claimBasis(units[fv.formal.units.limit], doc), null)
|
||||
const many = { catalog: { 't:a': { kind: 'table', name: 'a', queries: ['Q1', 'Q2', 'Q3', 'Q4', 'Q5'] },
|
||||
...Object.fromEntries(['Q1', 'Q2', 'Q3', 'Q4', 'Q5'].map((q) => [q, { kind: 'query' }])) }, blocks: [] }
|
||||
const bm = ev.claimBasis({ cites: ['t:a'], segments: [] }, many)
|
||||
ok('……查询多于 3 次:只列前 3 次的 SQL,其余写个数', bm?.queries.join() === 'Q1,Q2,Q3' && bm.more === 2, JSON.stringify(bm))
|
||||
eq('查询快照里的 SQL:快照本身、工具调用快照的 args / result 都认', [ev.sqlOf({ sql: 'A' }), ev.sqlOf({ tool: 'x', args: { sql: 'B' }, result: {} }),
|
||||
ev.sqlOf({ result: { sql: 'C' } }), ev.sqlOf({ rows: [] }), ev.sqlOf(null)].join('|'), 'A|B|C||')
|
||||
return out
|
||||
})) check(name, ok, detail)
|
||||
})
|
||||
|
||||
await section('lib/actor:本机署名', async () => {
|
||||
for (const [name, ok, detail] of await page.evaluate(() => {
|
||||
const a = window.__ui.lib.actor
|
||||
|
||||
Reference in New Issue
Block a user