Merge pull request #16 from ThearchyHelios/feat/judge-rework

结论句裁判改造:「不支持」拆成「证据相矛盾」和「证据不足」,补全交给裁判的证据,旧判定可按新规则重判
This commit is contained in:
ThearchyHelios
2026-09-30 10:09:51 +08:00
committed by GitHub
36 changed files with 11713 additions and 284 deletions
+4 -2
View File
@@ -209,11 +209,13 @@ NODE_REFERENCE = """\
metrics_from 不写就取所有上游口径卡;on_violation 不写时探索运行是 flag(标出来、照常产出),正式运行是 fail;
numbers 默认 strict(裸数字算违规)。claims 是结论句策略:off(默认,不管)/ require_citation(每句结论都要
挂引用,没挂的计入缺口、出具降档)/ judge(在 require_citation 之外,再请另一个模型按每句挂的依据判断支不支持:
正式运行在节点里判,证据不支持的按 judge.on_unsupported 处置——degrade 降档(默认)或 withhold 不予出具;
正式运行在节点里判,证据相矛盾的按 judge.on_unsupported 处置——degrade 降档(默认)或 withhold 不予出具,
证据不足的(摘录里没有判断所需的信息)只降档;
探索运行不在节点里花钱,读的人点开哪句再按需判哪句,只标注)。judge 的子配置写在节点的 judge 里:judge.model、
judge.provider 指定裁判模型(建议和写作模型不同,同一个模型等于自己审自己;不写就用设置里的「证据裁判模型」);
judge.max_cost_usd、judge.max_claims、judge.timeout_s 是每份报告的裁判金额、句数、时长上限,写 null 表示不限,
不写取设置里的默认;judge.rewrite_once 默认 false,打开后把证据不支持的句子交回写作者只改一次。entities 默认 link
不写取设置里的默认;judge.rewrite_once 默认 false,打开后把证据相矛盾、证据不足的句子交回写作者只改一次。
entities 默认 link
(核对表名、字段名),报告里根本不提表名时才写 off
- output 的出具契约:config.contract = {report_from: 报告节点id, metrics_from:[口径卡id],
required:[必需指标id], expected:[期望指标id], strict: true}。成果字段写 {{ nodes.报告节点id.text }},
+142 -31
View File
@@ -66,10 +66,13 @@ from app.engine.evidence import (
normalize_quote,
query_entry_fields,
render_metric,
schema_column,
schema_table,
sql_tables,
table_fields,
uncited_claims,
)
from app.engine.judge import JUDGED, VERDICTS, candidates
from app.engine.judge import FIELDS_MAX, SETTLED, UNSUPPORTED, VERDICTS, candidates, outdated
from app.engine.toolcalls import QUERY_PREFIX
router = APIRouter(prefix="/api/runs", tags=["evidence"])
@@ -727,7 +730,8 @@ async def _chain(seg: dict[str, Any], doc: dict[str, Any], sealed: _Sealed,
if seg.get("kind") == "entity" and seg.get("issue") in SUSPICIOUS:
return [_suspicious_step(seg, cite, sealed, report_node)]
if seg.get("kind") == "entity" and seg.get("state") == "deterministic" and cite.get("status") == "resolved":
return [_entity_step(seg, cite, entry, sealed)]
hidden = await _entity_hidden(entry, sealed, masks) if entry.get("kind") == "table" else set()
return [_entity_step(seg, cite, entry, sealed, hidden=hidden)]
if seg.get("kind") == "quote" and seg.get("state") == "deterministic" and cite.get("status") == "resolved":
return [_quote_step(seg, cite, entry, sealed)]
if _missing_input(cite, entry):
@@ -928,22 +932,10 @@ def _fetch(artifact: str) -> tuple[Any, bool | None]:
return content, (True if content is not None else None)
def _schema_table(content: Any, name: Any) -> dict[str, Any] | None:
"""表结构快照里的一张表:表名一字不差的优先,其次不分大小写、全名(schema.表)也认。"""
tables = content.get("tables") if isinstance(content, dict) else None
if not isinstance(tables, dict) or not name:
return None
if isinstance(tables.get(name), dict):
return tables[name]
lowered = str(name).lower()
return next((t for key, t in tables.items() if isinstance(t, dict)
and lowered in (str(key).lower(), str(t.get("qualified") or "").lower())), None)
def _schema_column(table: dict[str, Any] | None, name: Any) -> dict[str, Any] | None:
columns = [c for c in (table or {}).get("columns") or [] if isinstance(c, dict)]
return next((c for c in columns if c.get("name") == name), None) \
or next((c for c in columns if str(c.get("name") or "").lower() == str(name or "").lower()), None)
#: 表结构快照里找表、找字段:和裁判的摘录同一套(engine/evidence.py)。表名实体步骤的字段清单也和裁判的
#: 摘录同一个上限(judge.FIELDS_MAX)
_schema_table = schema_table
_schema_column = schema_column
def _entity_present(content: Any, kind: str, name: str, table: Any, column: Any, tables: list[Any]) -> bool:
@@ -963,12 +955,34 @@ def _query_present(content: Any, kind: str, name: str, column: Any) -> bool:
return str(column) in [str(c) for c in content.get("columns") or []]
def _entity_step(seg: dict[str, Any], cite: dict[str, Any], entry: dict[str, Any], sealed: _Sealed) -> dict[str, Any]:
async def _entity_hidden(entry: dict[str, Any], sealed: _Sealed, masks: _Masks) -> set[str]:
"""表的字段清单要遮的列(小写):数据源现在设的,加上这次运行封存的查询快照里、同一个数据源当时记下的
(数据源改名、删掉了也照遮,和查询步骤同一个口径)。"""
source = str(entry.get("source") or "")
hidden, _ = await masks.of(source) if source else (set(), True)
hidden = set(hidden)
for artifact, info in sealed.queries.items():
if not sealed.trusted:
break
snap, ok = _fetch(artifact)
if ok is not True or not isinstance(snap, dict):
continue
if not source or str(info.get("source") or snap.get("source") or "") == source:
recorded = snap.get("mask_columns")
hidden |= {str(c).lower() for c in recorded if isinstance(c, str)} if isinstance(recorded, list) else set()
return hidden
def _entity_step(seg: dict[str, Any], cite: dict[str, Any], entry: dict[str, Any], sealed: _Sealed, *,
hidden: set[str] | frozenset[str] = frozenset()) -> dict[str, Any]:
"""实体步骤:这个名字出现在哪几次查询里(SQL 用到的表、结果的列)、表结构快照什么时候同步的、字段类型。
来源(entry.sources)逐个核对:表结构快照要出现在封存范围内的 tool.end.schema_artifact 或 schema 台账
条目里,查询快照要是封存范围内交回过的那几件。类型、同步时间只从核对过、复验过哈希的表结构快照取,
取不到就不给。eid 背后的那件工件核对通过、名字确实在里面,这一步才算已封存。
表另带字段清单 fields: [{name, type}](同一份核对过的快照,最多 FIELDS_MAX 个,多出来的记 fields_more):
讲表结构的句子,读的人在「挂的依据」下直接看得到有哪些字段。hidden 里的列(遮罩)不列。
"""
kind = str(cite.get("kind") or entry.get("kind") or "")
locator = dict(entry.get("locator") or cite.get("locator") or {})
@@ -1022,7 +1036,8 @@ def _entity_step(seg: dict[str, Any], cite: dict[str, Any], entry: dict[str, Any
step["sources"].append(item)
step["sealed"] = sealed.trusted and backed and step["eid_ok"]
if snapshots:
_describe_entity(step, snapshots[0], kind, name, table, column, list(entry.get("tables") or []))
_describe_entity(step, snapshots[0], kind, name, table, column, list(entry.get("tables") or []),
hidden=hidden)
if kind == "column" and not any(o.get("kind") == "schema" for o in step["sources"]):
# 聚合的别名这类只在结果里出现的列:类型取查询时从驱动的原始值记下的列类型。表结构里有的列
# 只认表结构快照的类型,快照核对不过就不给——不拿结果列的类型顶上
@@ -1033,8 +1048,8 @@ def _entity_step(seg: dict[str, Any], cite: dict[str, Any], entry: dict[str, Any
def _describe_entity(step: dict[str, Any], snap: dict[str, Any], kind: str, name: str, table: Any, column: Any,
tables: list[Any]) -> None:
"""从表结构快照里补上同步时间、表的概况、字段的类型。"""
tables: list[Any], *, hidden: set[str] | frozenset[str] = frozenset()) -> None:
"""从表结构快照里补上同步时间、表的概况(含字段清单)、字段的类型。"""
if snap.get("synced_at"):
step["synced_at"] = snap["synced_at"]
if snap.get("truncated"):
@@ -1045,6 +1060,12 @@ def _describe_entity(step: dict[str, Any], snap: dict[str, Any], kind: str, name
step["is_view"] = bool(info.get("is_view"))
if info.get("comment"):
step["comment"] = info["comment"]
fields, more, masked = table_fields(info, hidden, limit=FIELDS_MAX)
step["fields"] = fields
if more:
step["fields_more"] = more
if masked:
step["fields_masked"] = True
return
owners = [table] if table else tables
found = {str(t): (tinfo, _schema_column(tinfo, column)) for t in owners
@@ -1200,6 +1221,8 @@ UNSEALED_JUDGE = "运行尚未结束并封存(仍在运行、停在人工审
LEGACY_JUDGE = "这次运行发起时系统尚未记录证据,不支持按需裁判"
BROKEN_JUDGE = "封存核对未通过:封存后有事件被修改、删除或插入,报告和证据已不可信,无法再请模型裁判"
JUDGED_ALREADY = "这句已裁判"
#: 这句封存后按需判过,但那次早于当前的裁判规则(没有规则版本,或者是旧取值「证据不支持」):可以按当前规则重判
OUTDATED_JUDGE = "这句的判定早于当前的裁判规则,可以请模型按当前规则重新判断"
#: 裁判期间运行被接着跑了(失败的运行可以继续),或者接着跑完重新封存了:判定照样交回,不记进运行记录
NOT_RECORDED = "运行已继续执行,本次裁判结果未写入运行记录,刷新后将不再显示。请在运行结束并封存后重新裁判"
NOT_A_CLAIM = "不是结论句(标题、连接性文字、表格单元格或代码),无需模型裁判"
@@ -1232,7 +1255,8 @@ def _effective_verdicts(report: _Report, sealed: _Sealed
doc = report.doc or {}
ids = {str(u.get("id")) for _, u in iter_units(doc)}
out = {str(u.get("id")): u["verdict"] for _, u in iter_units(doc) if _verdict_ok(u.get("verdict"))}
sealed_judged = {uid for uid, v in out.items() if v.get("status") in JUDGED}
# 旧数据里的 unsupported 同样算判过(SETTLED):封存的判定用什么取值,都不能被追加的判定盖掉
sealed_judged = {uid for uid, v in out.items() if v.get("status") in SETTLED}
later: dict[str, dict[str, Any]] = {}
for _, _, data in sealed.judged:
if data.get("report") != report.node_id or data.get("doc_artifact") != report.doc_artifact:
@@ -1242,17 +1266,48 @@ def _effective_verdicts(report: _Report, sealed: _Sealed
uid = str(uid)
if uid not in ids or uid in sealed_judged or not _verdict_ok(verdict):
continue
if later.get(uid, {}).get("status") in JUDGED and verdict.get("status") not in JUDGED:
if later.get(uid, {}).get("status") in SETTLED and verdict.get("status") not in SETTLED:
continue
later[uid] = out[uid] = {**verdict, "post_seal": True}
return out, later
def _judgement_of(report: _Report, sealed: _Sealed) -> dict[str, Any]:
"""证据图、审计表里每份报告带的结论句情况:写作时的策略、封存的裁判摘要、封存之后按需追加的判定。"""
"""证据图、审计表里每份报告带的结论句情况:写作时的策略、封存的裁判摘要、封存之后按需追加的判定,
和最近一次按需裁判用的模型(judge_meta,没按需裁判过是 None)。"""
doc = report.doc or {}
return {"claims": report.claims, "judge": doc.get("judge") if isinstance(doc.get("judge"), dict) else None,
"post_seal_verdicts": _effective_verdicts(report, sealed)[1] if report.doc is not None else {}}
"post_seal_verdicts": _effective_verdicts(report, sealed)[1] if report.doc is not None else {},
"judge_meta": _judge_meta(report, sealed)}
def _writer_model(sealed: _Sealed, report: _Report) -> str | None:
"""这份报告的写作模型:文档的裁判摘要里记的(写作时当场记下),没有就取报告撰写节点封存的产出里的 model
(实际用的模型 id,老文档也有)。都取不到是 None。"""
judge = (report.doc or {}).get("judge")
if isinstance(judge, dict) and isinstance(judge.get("writer_model"), str) and judge["writer_model"]:
return judge["writer_model"]
payload = sealed.payload_of([report.node_id]) or {}
model = payload.get("model")
return model if isinstance(model, str) and model else None
def _same_model(model: Any, writer: str | None) -> bool:
return bool(model) and bool(writer) and model == writer
def _judge_meta(report: _Report, sealed: _Sealed) -> dict[str, Any] | None:
"""最近一次按需裁判(evidence.judged,指向这份文档的)用的模型:{model, same_model, priced, writer_model}。
事件里没记 same_model、writer_model 的(改造前追加的),按这份报告的写作模型补上。"""
data = next((d for _, _, d in reversed(sealed.judged)
if d.get("report") == report.node_id and d.get("doc_artifact") == report.doc_artifact), None)
if data is None:
return None
writer = data.get("writer_model") if "writer_model" in data else _writer_model(sealed, report)
same = data.get("same_model") if isinstance(data.get("same_model"), bool) else _same_model(data.get("model"), writer)
return {"model": data.get("model"), "same_model": same,
"priced": data.get("priced") if isinstance(data.get("priced"), bool) else None, "writer_model": writer}
def _on_demand(sealed: _Sealed, doc: dict[str, Any], uid: str, verdict: dict[str, Any] | None) -> dict[str, Any]:
@@ -1266,15 +1321,25 @@ def _on_demand(sealed: _Sealed, doc: dict[str, Any], uid: str, verdict: dict[str
return no("legacy", LEGACY_JUDGE)
if not candidates(doc, [uid]):
return no("not_a_claim", f"这句{NOT_A_CLAIM}")
if isinstance(verdict, dict) and verdict.get("status") in JUDGED:
stale = _stale(verdict)
if isinstance(verdict, dict) and verdict.get("status") in SETTLED and not stale:
return no("judged", JUDGED_ALREADY)
if not sealed.sealed or sealed.status not in _TERMINAL:
return no("unsealed", UNSEALED_JUDGE)
if not sealed.trusted:
return no("seal_broken", BROKEN_JUDGE)
if stale:
return {"available": True, "reason": "outdated", "message": OUTDATED_JUDGE}
return {"available": True, "reason": None, "message": None}
def _stale(verdict: Any) -> bool:
"""封存之后按需追加的判定早于当前规则:按需裁判当它还没按当前规则判过,重判一次(不按摘录内容判断)。
封存的判定(文档里的,post_seal 为 False)不在此列:追加的判定盖不掉它,版本再旧也不重开。"""
return isinstance(verdict, dict) and bool(verdict.get("post_seal")) and outdated(verdict)
def _sealed_loader(sealed: _Sealed) -> Callable[[str], Any]:
"""给裁判取证据的 loader:只认封存范围内的事件交回过的工件(查询、表结构、检索快照和台账里记过的),
取回时照旧复验哈希。别的一律当取不到——人为插进工件表的行、封存之后才追加的事件引用的快照,
@@ -1371,13 +1436,15 @@ async def _judge_now(sealed: _Sealed, report: _Report, units: list[str]) -> dict
doc = report.doc or {}
catalog = doc.get("catalog") or {}
known = _effective_verdicts(report, sealed)[0]
reused = [u for u in units if known.get(u, {}).get("status") in JUDGED]
# 当前规则下判过的直接给;早于当前规则的(封存之后追加的旧判定)重判一次,和片段接口的 outdated 同一条件
reused = [u for u in units if known.get(u, {}).get("status") in SETTLED and not _stale(known.get(u))]
pending = [u for u in units if u not in reused]
verdicts = {u: known[u] for u in reused}
node_judge = _node_judge(sealed, report.node_id)
async with SessionLocal() as session:
settings = await judging.judge_settings(session)
budget = judging.click_budget(settings, node_judge)
writer = _writer_model(sealed, report)
outcome: dict[str, Any] | None = None
skipped: dict[str, str] = {}
event = None
@@ -1391,7 +1458,13 @@ async def _judge_now(sealed: _Sealed, report: _Report, units: list[str]) -> dict
async with SessionLocal() as session:
spec = await judging.judge_model_spec(session, node_judge, settings=settings)
outcome = await judging.run_request(request, spec=spec, budget=budget, post_seal=True, click=True)
verdicts.update(outcome["verdicts"])
for uid, verdict in outcome["verdicts"].items():
# 重判没判成(触顶、模型不可用):旧判定照旧生效,和证据图、片段接口说的一样
older = known.get(uid)
keep = verdict.get("status") not in SETTLED and isinstance(older, dict) and older.get("status") in SETTLED
verdicts[uid] = older if keep else verdict
outcome["writer_model"] = writer
outcome["same_model"] = _same_model(outcome.get("model"), writer)
if outcome["calls"] or outcome["cost_usd"]:
# 没问模型(触顶、模型没配好)就没有判定可记:说给点的人听,不往运行记录里追加
event = await _append_judged(sealed, report, pending, outcome)
@@ -1403,12 +1476,14 @@ async def _judge_now(sealed: _Sealed, report: _Report, units: list[str]) -> dict
return {
"run_id": sealed.run_id, "report": {"node_id": report.node_id, "doc_artifact": report.doc_artifact},
"units": units, "verdicts": {u: verdicts[u] for u in units if u in verdicts}, "reused": reused,
"judged": [u for u in pending if (outcome or {}).get("verdicts", {}).get(u, {}).get("status") in JUDGED],
"judged": [u for u in pending if (outcome or {}).get("verdicts", {}).get(u, {}).get("status") in SETTLED],
"skipped": skipped, "unjudged": dict(outcome["unjudged"]) if outcome else {},
"limits_hit": limits, "limited": bool(limits), "message": message,
"adjust": [JUDGE_ADJUST[r] for r in limits if r in JUDGE_ADJUST],
"gaps": list(outcome["gaps"]) if outcome else [], "notes": list(outcome["notes"]) if outcome else [],
"model": (outcome or {}).get("model"), "priced": (outcome or {}).get("priced"),
# 裁判模型和这份报告的写作模型相同:判定照常出,界面上标出来(审查缺乏独立性)
"same_model": bool((outcome or {}).get("same_model")), "writer_model": writer,
"cost_usd": (outcome or {}).get("cost_usd", 0.0), "calls": (outcome or {}).get("calls", 0),
"duration_ms": (outcome or {}).get("duration_ms", 0),
"budget": budget.as_dict() if outcome else None, "event": event, "post_seal": True,
@@ -1461,7 +1536,8 @@ async def _append_judged(sealed: _Sealed, report: _Report, units: list[str],
"report": report.node_id, "doc_artifact": report.doc_artifact, "units": units,
"verdicts": outcome["verdicts"],
"keys": {u: outcome["keys"][u] for u in outcome["verdicts"] if u in outcome["keys"]},
"model": outcome["model"], "priced": outcome["priced"], "cost_usd": outcome["cost_usd"],
"model": outcome["model"], "priced": outcome["priced"], "same_model": outcome["same_model"],
"writer_model": outcome["writer_model"], "cost_usd": outcome["cost_usd"],
"calls": outcome["calls"], "duration_ms": outcome["duration_ms"], "budget": outcome["budget"],
"limits_hit": outcome["limits_hit"], "unjudged": outcome["unjudged"], "gaps": outcome["gaps"],
"notes": outcome["notes"], "skipped": outcome["skipped"], "post_seal": True})
@@ -1501,6 +1577,13 @@ UNCITED_CLAIM_JUDGED = "这句结论没有附依据。本报告的结论句由
UNCITED_CLAIM_JUDGE_REQUIRED = ("这句结论没有附依据。本报告的结论句由模型裁判,同样要求附依据,但目前没有出具契约"
"据此判档(未配置契约,或尚未到出具步骤)")
LEGACY_UNMATCHED = "旧版出具中这个数字与任何指标都不相符"
#: 模型判定有问题的结论句各一行(生效的判定:封存的,没有就是封存后按需追加的):判定 → 审计行的 issue。
#: 旧数据里的 unsupported(当时没分矛盾和不足)记 unsupported_claim,说法照旧是「证据不支持」
CLAIM_ISSUE = {"contradicted": "contradicted_claim", "insufficient": "insufficient_claim",
UNSUPPORTED: "unsupported_claim"}
CLAIM_VERDICT_NOTE = {"contradicted": "模型判断:证据与这句结论相矛盾", "insufficient": "模型判断:证据不足,无法判断这句结论",
UNSUPPORTED: "模型判断:证据不支持这句结论"}
CLAIM_POST_SEAL = "该判定为封存后按需追加,不在封存范围内"
#: CSV 的列:(表头, 行里的键)
CSV_COLUMNS = (
@@ -1521,6 +1604,7 @@ _CSV_LABELS: dict[str, dict[Any, str]] = {
"state": {"deterministic": "有出处", "none": "无证据", "candidate": "候选"},
"issue": {"uncited_number": "没有出处的数字", "unresolved_ref": "引用无法解析", "unknown_entity": "疑似不存在的名称",
"unverified_entity": "无法核实", "uncited_claim": "结论句未附依据", "no_candidate": "未找到候选",
"contradicted_claim": "证据相矛盾", "insufficient_claim": "证据不足", "unsupported_claim": "证据不支持",
**{code: "文档不一致" for code in _INTEGRITY}},
"evidence_kind": {"metric": "口径卡指标", "cell": "查询单元格", "input": "运行输入", "table": "表", "column": "字段",
"quote": "知识库原话", "query": "查询结果"},
@@ -1670,6 +1754,7 @@ def _audit_doc(report: _Report, sealed: _Sealed, inputs: dict[str, Any],
"issue": "uncited_claim", "ref": None, "alias": None, "evidence_kind": None, "evidence": None,
"eid": None, "artifact": None, "node_id": None, "sealed": None, "span": claim["span"],
"sentence": claim["text"], "note": claim_note})
rows.extend(_verdict_rows(report, sealed, field_name, markdown))
for v in doc.get("violations") or []:
if not isinstance(v, dict) or (str(v.get("segment")) in listed and v.get("code") not in _INTEGRITY):
continue
@@ -1683,6 +1768,32 @@ def _audit_doc(report: _Report, sealed: _Sealed, inputs: dict[str, Any],
return rows
def _verdict_rows(report: _Report, sealed: _Sealed, field_name: str | None, markdown: str) -> list[dict[str, Any]]:
"""模型判定证据相矛盾、证据不足(旧数据里的证据不支持)的结论句,各一行,进「无证据」一组(问题)。
判定是模型的判断,不是证据:sealed 为 None,说明里写明是模型判断;封存后按需追加的另外注明。"""
effective, _ = _effective_verdicts(report, sealed)
units = {str(u.get("id")): u for _, u in iter_units(report.doc or {})}
rows = []
for uid, verdict in effective.items():
status = verdict.get("status")
if status not in CLAIM_ISSUE or uid not in units:
continue
span = units[uid].get("span") or [0, 0]
text = markdown[span[0]:span[1]]
missing = verdict.get("missing") if status == "insufficient" and isinstance(verdict.get("missing"), str) else ""
rationale = str(verdict.get("rationale") or "").rstrip("。")
note = CLAIM_VERDICT_NOTE[status] + (f"(缺少:{missing})" if missing else "") \
+ (f"。理由:{rationale}" if rationale else "") + (f"。{CLAIM_POST_SEAL}" if verdict.get("post_seal") else "")
rows.append({"group": "none", "report": report.node_id, "field": field_name, "segment": None, "unit": uid,
"kind": "claim", "text": text, "state": "none", "issue": CLAIM_ISSUE[status], "ref": None,
"alias": None, "evidence_kind": None, "evidence": None, "eid": None, "artifact": None,
"node_id": None, "sealed": None, "span": span, "sentence": text, "note": note,
"verdict": {k: verdict[k] for k in ("status", "rationale", "missing", "judge", "post_seal")
if k in verdict}})
return rows
def _segment_row(group: str, report: _Report, field_name: str | None, seg: dict[str, Any], unit: dict[str, Any],
sentence: str, catalog: dict[str, Any], sealed: _Sealed, inputs: dict[str, Any]) -> dict[str, Any]:
cite = seg.get("cite") or {}
+31
View File
@@ -413,6 +413,37 @@ def schema_entry_fields(content: Any) -> dict[str, Any] | None:
return fields
def schema_table(content: Any, name: Any) -> dict[str, Any] | None:
"""表结构快照里的一张表:表名一字不差的优先,其次不分大小写、全名(schema.表)也认。"""
tables = content.get("tables") if isinstance(content, dict) else None
if not isinstance(tables, dict) or not name:
return None
if isinstance(tables.get(name), dict):
return tables[name]
lowered = str(name).lower()
return next((t for key, t in tables.items() if isinstance(t, dict)
and lowered in (str(key).lower(), str(t.get("qualified") or "").lower())), None)
def schema_column(table: dict[str, Any] | None, name: Any) -> dict[str, Any] | None:
"""表结构快照里一张表的一个字段:名字一字不差的优先,其次不分大小写。"""
columns = [c for c in (table or {}).get("columns") or [] if isinstance(c, dict)]
return next((c for c in columns if c.get("name") == name), None) \
or next((c for c in columns if str(c.get("name") or "").lower() == str(name or "").lower()), None)
def table_fields(table: dict[str, Any] | None, hidden: set[str] | frozenset[str] = frozenset(), *,
limit: int = 60) -> tuple[list[dict[str, str]], int, bool]:
"""一张表的字段清单:([{name, type}] 最多 limit 个, 另有几个没列, 有没有字段按遮罩没列)。
hidden 是要遮的列(小写):遮罩的字段不列,也不计入「另有几个」。证据面板和裁判的摘录同一个口径。
"""
columns = [c for c in (table or {}).get("columns") or [] if isinstance(c, dict) and c.get("name")]
shown = [c for c in columns if str(c["name"]).lower() not in hidden]
fields = [{"name": str(c["name"]), "type": str(c.get("type") or "")} for c in shown[:limit]]
return fields, max(0, len(shown) - limit), len(shown) < len(columns)
def schema_ledger_entry(artifact: str, *, node_id: str, exec_no: int,
loader: Callable[[str], Any] | None = None) -> dict[str, Any] | None:
"""一件表结构快照 → 台账的 schema 条目。快照取不回来(或者哈希对不上)返回 None。
+314 -54
View File
@@ -5,9 +5,11 @@
1. 预筛(claim_priority):只把结论句送去判。挂了依据的排第一,有方向词、因果词的排第二,其余有数字、
实体这类实质内容的排第三;标题、表格单元格、代码、连接性的话、只有过渡词的、短句不送
2. 证据摘录(prepare):句子挂的每件证据摘一段。指标写「名称 = 值」加原式;查询给 SQL 前 300 字和被
引用的行(最多 20 行,数据源遮罩的列不给;宽表先给被引用的列,再补满 12 列);知识库片段最多 600 字。
规则里写明数字已经由系统核对过
2. 证据摘录(prepare):句子挂的每件证据摘一段。指标写「名称 = 值」加原式,再附上输入取自哪次查询、
哪一格和那次查询的 SQL;查询给 SQL 前 300 字和被引用的行(最多 20 行,数据源遮罩的列不给;宽表先给
被引用的列,再补满 12 列);表附上字段清单和类型(最多 60 个,遮罩的列不列,快照不完整要注明),
字段附上类型和所属的表,两样都附上出现过的查询 SQL;知识库片段最多 600 字。快照一律经 loader 取
(按需裁判传只认封存工件的那个)。规则里写明数字已经由系统核对过
3. 调用(judge_doc):一份报告合并成一次调用,超过每批的句数就分批;结构化输出失败退回 JSON 解析,
再失败就记未裁判。thinking 关掉,max_tokens 4096
4. 预算:每份报告的句数、金额、时长和全局每日金额,每一项都能是 None(不限)。调用前按目录价估算,
@@ -15,6 +17,10 @@
5. 每日计数落在设置表(SPEND_KEY),按本地日期滚动。目录里没有价格的模型估不出金额,金额上限对它
不起作用,照实说「按令牌估不出金额」
判定五选一:supported、partial、contradicted(证据和句子冲突)、insufficient(证据里没有判断所需的
信息,missing 写缺什么)、not_a_claim。旧数据里的 unsupported(当时没分矛盾和不足)原样保留,判档当
contradicted;新判的不再产出它。
判定只标注、不改写正文。报告节点在正式运行里调它(结果写进文档、随文档封存);探索运行按需裁判的
接口也调它(post_seal=True,结果落在封存之后)。所有模型调用都经 get_chat_model,测试换成剧本。
"""
@@ -41,6 +47,7 @@ from app.db.base import SessionLocal
from app.db.models import Setting
from app.engine.evidence import (
_DIRECTIONAL,
ENTITY_KINDS,
MISSING,
RenderError,
_auto_entity,
@@ -48,6 +55,9 @@ from app.engine.evidence import (
_mask_names,
iter_units,
render_cell,
schema_column,
schema_table,
table_fields,
)
from app.engine.expressions import CellError, column_kind, locate_cell
from app.engine.nodes.llm import _split_structured, _usage_of
@@ -59,11 +69,27 @@ from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_mod
# 常量
# --------------------------------------------------------------------------
#: 模型给的四种判定;unjudged 是系统记的(没判成),不是模型说的
JUDGED = ("supported", "partial", "unsupported", "not_a_claim")
VERDICTS = (*JUDGED, "unjudged")
#: 模型的理由最多几个字
RATIONALE_MAX = 60
#: 模型给的五种判定;unjudged 是系统记的(没判成),不是模型说的
JUDGED = ("supported", "partial", "contradicted", "insufficient", "not_a_claim")
#: 旧的判定「证据不支持」:当时没分「证据相矛盾」和「证据不足」。已封存的文档、已有的事件里原样保留,
#: 判档照旧当 contradicted;新判的一律用新取值(模型照旧写了的,按 contradicted 收下)
UNSUPPORTED = "unsupported"
#: 读已有的判定时算「判过」的:新的五种加旧的 unsupported
SETTLED = (*JUDGED, UNSUPPORTED)
#: 判档按「证据相矛盾」处置的(on_unsupported:降档或不予出具)
CONTRADICTED = ("contradicted", UNSUPPORTED)
#: 判定对象里 status 的全部取值
VERDICTS = (*SETTLED, "unjudged")
#: 各判定的句数(stats、counts、出具横幅、审计导出都用这组键)。unsupported 保留给旧数据,新判的恒为 0
COUNT_KEYS = ("supported", "partial", "contradicted", "insufficient", "not_a_claim", "unjudged", UNSUPPORTED)
#: 模型的理由最多几个字(超长在句子边界截断加「…」);裁判规则里要求的字数
RATIONALE_MAX = 120
RATIONALE_ASK = 80
#: insufficient 时「缺的是什么」最多几个字
MISSING_MAX = 30
#: 判定的规则版本:新判的判定带着它(rule_version),也进 request_key。报告节点的断点里换了规则的旧判定不复用;
#: 按需裁判遇到早于当前规则的判定(没有版本、版本更小、旧取值 unsupported)重判一次
RULES_VERSION = 2
#: 一次调用最多判几句:再多,4096 个输出令牌装不下每句一条判定加理由
BATCH_SIZE = 40
MAX_TOKENS = 4096
@@ -73,13 +99,17 @@ SQL_CHARS = 300
EXCERPT_ROWS = 20
EXCERPT_COLS = 12
CHUNK_CHARS = 600
#: 表名、字段名的摘录里附上它出现过的查询的 SQL,最多几次(「按会员号去重计数」这种句子,证据在 SQL 里)
#: 表名、字段名的摘录里附上它出现过的查询的 SQL,最多几次(「按会员号去重计数」这种句子,证据在 SQL 里);
#: 口径卡指标的输入取自的查询也按这个数给 SQL
ENTITY_QUERIES = 3
EXCERPT_HITS = 3
#: 表的摘录里最多列几个字段(多出来的注明另有几个);指标的摘录里最多列几个输入
FIELDS_MAX = 60
METRIC_INPUTS = 8
#: 短句:看得见的字(不算标点、空白)少于这么多,又没挂依据、没有方向词的,不送裁判
SHORT_CHARS = 6
#: 估算输出令牌:每句一条判定(理由 60 字上下,加上 JSON 的键)
_OUT_PER_UNIT = 90
#: 估算输出 token:每句一条判定(理由 80 字上下、缺什么,加上 JSON 的键)
_OUT_PER_UNIT = 120
_OUT_BASE = 20
#: 未裁判的原因。前四个是上限,报告里写「已到上限」
@@ -88,17 +118,21 @@ REASONS = (*LIMITS, "error", "format", "missing", "on_demand")
JUDGE_ROLE = "你是报告核查员:逐句判断报告里的话,能不能被它挂的证据支持。你不写报告,也不改报告。"
JUDGE_RULES = """判定(verdict)四选一:
JUDGE_RULES = f"""判定(verdict)五选一:
- supported:证据足以支持这句话的说法——方向、比较、范围、因果都对得上
- partial:一部分说法有证据支持,另一部分(原因、范围、程度)证据里没有
- unsupported:证据不支持、和证据矛盾,或者这句在陈述数据事实却没有任何证据
- contradicted:证据和这句话冲突——证据里的方向、比较、范围、排名,或者 SQL 的算法、字段,和句子说的对不上
- insufficient:证据里没有判断这句话所需的信息(比如摘录里没有 SQL、没有字段清单、没有取数的查询),或者这句在陈述数据事实却没有挂任何证据
- not_a_claim:这句不陈述数据事实(过渡、背景、建议、计划),不需要证据
规则:
1. 数字已经由系统核对过,不要复算:每个数都是系统从证据里取出来、按固定规则渲染的,不要因为数字本身或它的格式质疑这句话;你只判断句子的说法有没有证据支持。
2. 只看下面给的证据摘录,不用常识、行业经验或外部知识补证据;摘录里没有的就是没有。
3. rationale 用中文写,不超过 60 个字,说清依据是什么,或者缺了什么。
4. used 写你实际用到的证据编号(摘录里【】中的那个,比如 m:gmv、Q1),没用到就写空数组。
5. 每句都要给判定,unit 照抄句子前面方括号里的编号。"""
3. 只有证据和句子冲突时才判 contradicted;证据里没有相关信息判 insufficient——「没有」不等于「矛盾」。
4. 摘录注明「表结构快照不完整」时,「某张表、某个字段不存在」这类说法只能判 insufficient。
5. rationale 用中文写,不超过 {RATIONALE_ASK} 个字,说清依据是什么,或者缺了什么。
6. 判 insufficient 时 missing 写缺的是什么,越短越好,比如「SQL」「字段清单」「取数的查询」;其余判定 missing 写空字符串。
7. used 写你实际用到的证据编号(摘录里【】中的那个,比如 m:gmv、Q1),没用到就写空数组。
8. 每句都要给判定,unit 照抄句子前面方括号里的编号。"""
#: 报告节点 claims 为 judge 时拼在写作规则后面:写作者要知道还有一道按证据的核对
JUDGE_RULE = ("这份报告的结论句还会由另一个模型逐句核对:它只看你挂的依据,判断这句话站不站得住。"
@@ -114,10 +148,12 @@ JUDGE_SCHEMA: dict[str, Any] = {
"properties": {
"unit": {"type": "string", "description": "句子编号,照抄 [u3] 里的 u3"},
"verdict": {"type": "string", "enum": list(JUDGED)},
"rationale": {"type": "string", "description": f"判定的理由,中文,不超过 {RATIONALE_MAX} 个字"},
"rationale": {"type": "string", "description": f"判定的理由,中文,不超过 {RATIONALE_ASK} 个字"},
"missing": {"type": "string",
"description": "判 insufficient 时写缺的是什么(比如 SQL、字段清单、取数的查询),其余判定写空字符串"},
"used": {"type": "array", "items": {"type": "string"}, "description": "实际用到的证据编号"},
},
"required": ["unit", "verdict", "rationale", "used"],
"required": ["unit", "verdict", "rationale", "missing", "used"],
}}},
"required": ["items"],
}
@@ -324,7 +360,90 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter
return "\n".join(lines)
def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch) -> str:
def _sql_line(label: str, sql: str) -> str:
return f"{label}:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"
def _source_hidden(source: Any, catalog: dict[str, Any], fetch: _Fetch, masked: dict[str, Any] | None) -> set[str]:
"""一个数据源要遮的列(小写):数据源现在设的,加上这次运行里它的查询快照当时记下的(数据源改名、删掉了
也照遮)。说不清是哪个数据源的,所有遮罩都算上:宁可多遮,不能露。"""
name = str(source or "")
names: set[str] = set()
for key, cols in (masked or {}).items():
if not name or str(key) == name:
names |= _mask_names(cols)
for entry in catalog.values():
if not isinstance(entry, dict) or entry.get("kind") != "query":
continue
snapshot = fetch(entry.get("artifact"))
if isinstance(snapshot, dict) and (not name or str(entry.get("source") or snapshot.get("source") or "") == name):
names |= _mask_names(snapshot.get("mask_columns"))
return names
def _query_alias(catalog: dict[str, Any], artifact: Any) -> str | None:
"""这份查询快照在报告目录里的编号(Q1):口径卡输入记的是工件,靠它对上。"""
return next((a for a, e in catalog.items() if isinstance(e, dict) and e.get("kind") == "query"
and artifact and e.get("artifact") == artifact), None)
def _cell_words(locator: dict[str, Any], hidden: set[str]) -> str:
"""输入落在哪一格:「r0 行 gmv 列」,数组字段是一段行「r0–r4 行 amount 列」。遮罩的列不写列名。"""
rows = locator.get("rows")
if isinstance(locator.get("row"), int):
where = f"r{locator['row']} 行"
elif isinstance(rows, list) and len(rows) == 2 and all(isinstance(r, int) for r in rows):
where = f"r{rows[0]}–r{rows[1]} 行"
else:
where = ""
cols = [locator["column"]] if locator.get("column") else list((locator.get("columns") or {}).values())
cols = [str(c) for c in cols if c not in (None, "")]
shown = [c for c in cols if c.lower() not in hidden]
parts = [where] if where else []
if shown:
parts.append("、".join(shown) + " 列")
if len(shown) < len(cols):
parts.append("(所在的列按数据源的设置遮罩,不写列名)")
return " ".join(parts) or "某一格"
def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any],
masked: dict[str, Any] | None) -> list[str]:
"""指标的输入取自哪次查询、哪一格,再附上那次查询的 SQL(截到 SQL_CHARS,最多 ENTITY_QUERIES 次)。
「活跃用户按会员号去重」这种讲口径的句子,证据在取数的 SQL 里;只给原式,裁判只能判证据不足。
位置来自封存的口径卡;SQL 经 fetch 取,取不到(不在封存范围里)的不给。没追到查询的输入照实写。
"""
items = [i for i in metric.get("inputs") or [] if isinstance(i, dict)]
lines: list[str] = []
sqls: list[str] = []
seen: set[str] = set()
for item in items[:METRIC_INPUTS]:
path = _clean(str(item.get("path") or item.get("field") or "")) or "(未命名)"
artifact = item.get("artifact") if item.get("via") in ("tool_cell", "agent_field") else None
if not artifact:
origin = f"(来自节点 {item['node_id']})" if item.get("node_id") else ""
lines.append(f"输入 {path}:未追溯到查询{origin}")
continue
alias = _query_alias(catalog, artifact)
snapshot = fetch(artifact)
snap = snapshot if isinstance(snapshot, dict) else {}
source = (catalog.get(alias) or {}).get("source") if alias else None
hidden = _source_hidden(source or snap.get("source"), catalog, fetch, masked) \
| _mask_names(snap.get("mask_columns"))
label = f"查询 {alias} " if alias else "一次查询(不在这份报告的证据目录中)"
lines.append(f"输入 {path}:取自{label}的 {_cell_words(item.get('locator') or {}, hidden)}")
sql = _clean(str(snap.get("sql") or ""))
if sql and str(artifact) not in seen and len(seen) < ENTITY_QUERIES:
seen.add(str(artifact))
sqls.append(_sql_line(f"查询 {alias} 的 SQL" if alias else "这次查询的 SQL", sql))
if len(items) > METRIC_INPUTS:
lines.append(f"…另有 {len(items) - METRIC_INPUTS} 个输入")
return lines + sqls
def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any],
masked: dict[str, Any] | None) -> str:
name = entry.get("name") or (entry.get("locator") or {}).get("metric") or alias
rendered = entry.get("rendered")
where = " ".join(v for v in (f"口径「{entry['caliber']}」" if entry.get("caliber") else "",
@@ -334,13 +453,83 @@ def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch) -> str:
lines = [f"【{alias}】口径卡指标:{value}" + (f"({where})" if where else "")]
card = fetch(entry.get("artifact"))
wanted = (entry.get("locator") or {}).get("metric")
for metric in (card.get("metrics") if isinstance(card, dict) else None) or []:
if isinstance(metric, dict) and metric.get("id") == wanted and metric.get("expression"):
metric = next((m for m in (card.get("metrics") if isinstance(card, dict) else None) or []
if isinstance(m, dict) and m.get("id") == wanted), None)
if metric is not None:
if metric.get("expression"):
lines.append(f"原式:{_clean(str(metric['expression']))}")
break
lines.extend(_metric_inputs(metric, fetch, catalog, masked))
return "\n".join(lines)
def _table_lines(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch,
masked: dict[str, Any] | None) -> list[str]:
"""表的字段清单和类型:取自封存范围内(经 fetch)的表结构快照,最多 FIELDS_MAX 个,遮罩的列不列。
讲表结构的句子(「订单表按会员号记录」)只给表名,裁判只能判证据不足。快照不完整(数据源的表太多、
只存了一部分)要注明:这时「某字段不存在」这类说法只能判 insufficient。
"""
name = entry.get("name") or (entry.get("locator") or {}).get("table")
origins = [o for o in entry.get("sources") or [] if isinstance(o, dict) and o.get("kind") == "schema"
and o.get("artifact")]
truncated = any(o.get("truncated") for o in origins)
info: dict[str, Any] | None = None
snapshot: dict[str, Any] = {}
readable = False
for origin in origins:
content = fetch(origin["artifact"])
readable = readable or isinstance(content, dict)
if (found := schema_table(content, name)) is not None:
info, snapshot = found, content
break
lines: list[str] = []
if info is None:
lines.append("字段清单:没有(" + ("本次运行的表结构快照中没有这张表" if readable or not origins
else "无法读取这张表所在的表结构快照") + ")")
else:
truncated = truncated or bool(snapshot.get("truncated"))
hidden = _source_hidden(entry.get("source") or snapshot.get("source"), catalog, fetch, masked)
fields, more, masked_any = table_fields(info, hidden, limit=FIELDS_MAX)
listed = " | ".join(f"{f['name']} {f['type']}".strip() for f in fields) or "无"
lines.append(f"字段清单(名称 类型,共 {len(fields) + more} 个):{listed}"
+ (f" | …另有 {more} 个字段" if more else ""))
if masked_any:
lines.append("(有的字段按数据源的设置遮罩了,没有列出)")
if truncated:
lines.append("注意:表结构快照不完整(数据源的表太多,只保存了一部分),不能据此断定某张表或某个字段不存在")
return lines
def _column_lines(entry: dict[str, Any], fetch: _Fetch) -> list[str]:
"""字段所属的表和类型。类型只从封存范围内的表结构快照取;表结构里没有的列(聚合的别名)才取查询快照
记下的列类型——和证据面板同一个口径。"""
locator = entry.get("locator") or {}
table, column = locator.get("table"), locator.get("column") or entry.get("name")
owners = [str(table)] if table else [str(t) for t in entry.get("tables") or []]
origins = [o for o in entry.get("sources") or [] if isinstance(o, dict) and o.get("artifact")]
types: dict[str, str] = {}
for origin in origins:
if origin.get("kind") != "schema":
continue
content = fetch(origin["artifact"])
for owner in owners:
if (col := schema_column(schema_table(content, owner), column)) is not None and col.get("type"):
types.setdefault(owner, str(col["type"]))
lines = ["所属的表:" + "、".join(owners)] if owners else []
if len(set(types.values())) == 1:
lines.append(f"类型:{next(iter(types.values()))}(取自表结构快照)")
elif types:
lines.append("类型:" + ";".join(f"{t} 表中为 {k}" for t, k in types.items()) + "(取自表结构快照)")
elif not any(o.get("kind") == "schema" for o in origins):
for origin in origins:
content = fetch(origin["artifact"]) if origin.get("kind") == "result" else None
kinds = content.get("column_types") if isinstance(content, dict) else None
if isinstance(kinds, dict) and isinstance(kinds.get(str(column)), str):
lines.append(f"类型:{kinds[str(column)]}(取自查询结果)")
break
return lines
def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], whole: bool, fetch: _Fetch) -> str | None:
content = fetch(entry.get("artifact"))
hits = content.get("hits") if isinstance(content, dict) else None
@@ -364,7 +553,7 @@ def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _
masked: dict[str, Any] | None, catalog: dict[str, Any]) -> str | None:
kind = entry.get("kind")
if kind == "metric":
return _metric_excerpt(alias, entry, fetch)
return _metric_excerpt(alias, entry, fetch, catalog, masked)
if kind == "input":
field_name = (entry.get("locator") or {}).get("field") or alias
return f"【{alias}】运行输入:{field_name} = {entry.get('rendered')}"
@@ -377,11 +566,11 @@ def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _
return _retrieval_excerpt(alias, entry, picked, any(alias in c.whole for c in cands), fetch)
if kind == "table":
head = f"【{alias}】表 {entry.get('name')}" + (f"(数据源 {entry['source']})" if entry.get("source") else "")
return "\n".join([head, *_entity_sql(entry, catalog, fetch)])
return "\n".join([head, *_table_lines(entry, catalog, fetch, masked), *_entity_sql(entry, catalog, fetch)])
if kind == "column":
table = (entry.get("locator") or {}).get("table")
head = f"【{alias}】字段 {f'{table}.' if table else ''}{(entry.get('locator') or {}).get('column') or entry.get('name')}"
return "\n".join([head, *_entity_sql(entry, catalog, fetch)])
return "\n".join([head, *_column_lines(entry, fetch), *_entity_sql(entry, catalog, fetch)])
return None
@@ -445,7 +634,8 @@ class JudgeRequest:
HumanMessage(content=(
f"证据摘录(【】里是编号,判定时 used 写它):\n{evidence}\n\n"
"要判断的句子(每句都要给判定):\n" + "\n".join(self.unit_line(c) for c in ordered) + "\n\n"
'只输出 JSON:{"items": [{"unit": "u1", "verdict": "supported", "rationale": "…", "used": ["m:gmv"]}]}'
'只输出 JSON:{"items": [{"unit": "u1", "verdict": "supported", "rationale": "…", "missing": "", '
'"used": ["m:gmv"]}]}'
)),
]
@@ -498,7 +688,7 @@ def request_key(request: JudgeRequest, spec: ModelSpec, budget: Budget, known: d
return content_hash(canonical_json({
"units": [[c.unit, c.priority, c.text, c.section, list(c.aliases)] for c in request.cands],
"excerpts": request.excerpts, "model": [spec.provider, spec.model], "budget": budget.as_dict(),
"known": sorted(known or {}),
"known": sorted(known or {}), "rules": RULES_VERSION,
}))
@@ -650,25 +840,41 @@ async def judge_model_spec(session: Any, node_judge: dict[str, Any] | None = Non
async def source_masks(catalog: dict[str, Any], *, loader: Callable[[str], Any] | None = None
) -> dict[str, list[str]]:
"""目录里的查询来自的数据源现在设的遮罩:{数据源名: 列名}。和证据面板同一个口径。
"""目录里的证据来自的数据源现在设的遮罩:{数据源名: 列名}。和证据面板同一个口径。
旧形状的台账条目没记数据源的,按快照自己记的数据源查(快照经 loader 取,缺省读工件库;
按需裁判的接口要传只认封存工件的 loader,和 prepare 同一个)。
查的数据源:查询条目的;表、字段条目的(字段清单里遮罩的列不列);口径卡指标的输入取自的查询快照的。
条目没记数据源的,按快照自己记的数据源查(快照经 loader 取,缺省读工件库;按需裁判的接口要传只认
封存工件的 loader,和 prepare 同一个)。
"""
from app.data.engine import masked_columns
from app.db.models import DataSource
fetch = _Fetch(loader)
found: set[str] = set()
def source_of(artifact: Any) -> Any:
snapshot = fetch(artifact)
return snapshot.get("source") if isinstance(snapshot, dict) else None
for entry in catalog.values():
if not isinstance(entry, dict) or entry.get("kind") != "query":
if not isinstance(entry, dict):
continue
source = entry.get("source")
if not source:
snapshot = fetch(entry.get("artifact"))
source = snapshot.get("source") if isinstance(snapshot, dict) else None
if source:
found.add(str(source))
kind = entry.get("kind")
sources: list[Any] = []
if kind == "query":
sources.append(entry.get("source") or source_of(entry.get("artifact")))
elif kind in ENTITY_KINDS:
sources.append(entry.get("source") or next(
(source_of(o.get("artifact")) for o in entry.get("sources") or []
if isinstance(o, dict) and o.get("kind") == "schema"), None))
elif kind == "metric":
card = fetch(entry.get("artifact"))
wanted = (entry.get("locator") or {}).get("metric")
for metric in (card.get("metrics") if isinstance(card, dict) else None) or []:
if isinstance(metric, dict) and metric.get("id") == wanted:
sources.extend(source_of(i.get("artifact")) for i in metric.get("inputs") or []
if isinstance(i, dict) and i.get("artifact"))
found.update(str(s) for s in sources if s)
names = sorted(found)
if not names:
return {}
@@ -885,7 +1091,7 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget,
pending: list[Candidate] = []
for c in ranked:
hit = (known or {}).get(keys[c.unit])
if isinstance(hit, dict) and hit.get("status") in JUDGED:
if isinstance(hit, dict) and hit.get("status") in SETTLED:
verdicts[c.unit] = dict(hit)
reused += 1
else:
@@ -998,10 +1204,38 @@ async def run_request(request: JudgeRequest, *, spec: ModelSpec, budget: Budget,
return outcome
#: 截理由时认的句子边界(全角、半角都认)
_BOUNDARY = "。;,;,!?!?"
def clip_text(text: str, limit: int = RATIONALE_MAX) -> str:
"""超过 limit 个字时截短:在句子边界(。;,)截断加「…」,总长不超过 limit。
边界离开头太近(不到三分之一)就不按它截——只剩几个字的理由没用;这时退到词的边界(不把一个英文词、
一个数截成两半)。没超长的原样返回。
"""
text = text.strip()
if len(text) <= limit:
return text
head = text[:limit - 1]
cut = max(head.rfind(ch) for ch in _BOUNDARY)
if cut < limit // 3:
cut = len(head)
while cut > 0 and head[cut - 1].isascii() and head[cut - 1].isalnum() and text[cut].isascii() \
and text[cut].isalnum():
cut -= 1
if cut < limit // 3:
cut = len(head)
return head[:cut].rstrip(" ,,;;、") + "…"
def _take(items: list[Any], batch: list[Candidate], request: JudgeRequest, verdicts: dict[str, dict[str, Any]], *,
judge: str | None, post_seal: bool) -> None:
"""模型给的判定逐条收下:不在这一批的编号不认,判定不认识的记格式不对,理由截到 60 字,
used 只留这一批摘录里真有的编号。"""
"""模型给的判定逐条收下:不在这一批的编号不认,判定不认识的记格式不对,理由在句子边界截到 RATIONALE_MAX 字,
used 只留这一批摘录里真有的编号。判 insufficient 的另记 missing(缺的是什么,模型没写就不编)。
模型照旧写 unsupported 的按 contradicted 收下:新判的不再产出旧取值,而 contradicted 的处置和原来的
unsupported 一样(不能因为模型用了旧词就放过这一句)。"""
ids = {c.unit for c in batch}
aliases = {a for c in batch for a in c.aliases if a in request.excerpts}
for item in items:
@@ -1011,26 +1245,43 @@ def _take(items: list[Any], batch: list[Candidate], request: JudgeRequest, verdi
if uid not in ids or uid in verdicts:
continue
status = str(item.get("verdict") or "").strip().lower()
if status == UNSUPPORTED:
status = "contradicted"
if status not in JUDGED:
verdicts[uid] = _unjudged("format", f"裁判返回的判定「{str(item.get('verdict'))[:20]}」无法识别,这句未裁判",
judge=judge, post_seal=post_seal)
continue
used = item.get("used") if isinstance(item.get("used"), list) else []
verdicts[uid] = {"status": status, "rationale": _clean(str(item.get("rationale") or ""))[:RATIONALE_MAX],
"judge": judge, "post_seal": post_seal,
"used": list(dict.fromkeys(u.strip() for u in used if isinstance(u, str)
and u.strip() in aliases))}
verdict: dict[str, Any] = {
"status": status, "rationale": clip_text(_clean(str(item.get("rationale") or ""))),
"judge": judge, "post_seal": post_seal,
"used": list(dict.fromkeys(u.strip() for u in used if isinstance(u, str) and u.strip() in aliases))}
missing = item.get("missing")
if status == "insufficient" and isinstance(missing, str) and (missing := clip_text(_clean(missing), MISSING_MAX)):
verdict["missing"] = missing
verdict["rule_version"] = RULES_VERSION
verdicts[uid] = verdict
def outdated(verdict: Any) -> bool:
"""这条判过的判定早于当前规则:没有规则版本(改造前判的)、版本更小,或者是旧取值 unsupported。
没判成的(unjudged)谈不上新旧,返回 False。"""
if not isinstance(verdict, dict) or verdict.get("status") not in SETTLED:
return False
version = verdict.get("rule_version")
return verdict.get("status") == UNSUPPORTED or isinstance(version, bool) or not isinstance(version, int) \
or version < RULES_VERSION
def _outcome(request: JudgeRequest, verdicts: dict[str, dict[str, Any]], budget: Budget, details: dict[str, str], *,
click: bool = False, shown: Budget | None = None) -> dict[str, Any]:
"""汇总判定。outcome.budget 是这一轮实际用的上限;缺口里写的上限按 shown(缺省同 budget)。"""
label = shown or budget
counts = {s: 0 for s in VERDICTS}
counts = {s: 0 for s in COUNT_KEYS}
why: dict[str, int] = {}
for c in request.cands:
v = verdicts[c.unit]
counts[v["status"]] += 1
counts[v["status"]] = counts.get(v["status"], 0) + 1
if v["status"] == "unjudged":
why[v["reason"]] = why.get(v["reason"], 0) + 1
gaps: list[tuple[str, str]] = []
@@ -1073,13 +1324,18 @@ async def judge_doc(doc: dict[str, Any], catalog: dict[str, Any] | None = None,
#: 文档的 judge 摘要里、report.checked 和节点产出的 judge 里都有的键
SUMMARY_KEYS = ("mode", "model", "on_unsupported", "rewrite_once", "limits", "candidates", "screened", "counts",
"unjudged", "limits_hit", "complete", "priced", "cost_usd", "calls", "reused", "duration_ms",
"gaps", "notes", "same_model", "rewrite")
"gaps", "notes", "same_model", "rewrite", "writer_model")
def summarize(outcome: dict[str, Any], *, mode: str, on_unsupported: str = "degrade", rewrite_once: bool = False,
same_model: bool = False, rewrite: dict[str, Any] | None = None) -> dict[str, Any]:
"""裁判摘要:写进文档的 judge,也原样放进 report.checked 和节点产出。"""
counts = outcome["counts"]
same_model: bool = False, rewrite: dict[str, Any] | None = None,
writer_model: str | None = None) -> dict[str, Any]:
"""裁判摘要:写进文档的 judge,也原样放进 report.checked 和节点产出。
counts 补齐 COUNT_KEYS 的每个键(断点里缓存的旧判定只有旧的几个键);writer_model 是这份报告的写作模型,
same_model 是裁判模型和它相同。"""
raw = outcome["counts"]
counts = {k: int(raw.get(k) or 0) for k in COUNT_KEYS}
return {
"mode": mode, "model": outcome.get("model"), "on_unsupported": on_unsupported, "rewrite_once": rewrite_once,
"limits": outcome.get("budget"), "candidates": outcome["candidates"], "screened": outcome["screened"],
@@ -1088,7 +1344,7 @@ def summarize(outcome: dict[str, Any], *, mode: str, on_unsupported: str = "degr
"priced": outcome.get("priced"), "cost_usd": outcome.get("cost_usd", 0.0), "calls": outcome.get("calls", 0),
"reused": outcome.get("reused", 0), "duration_ms": outcome.get("duration_ms", 0),
"gaps": outcome["gaps"] if mode == "inline" else [], "notes": outcome.get("notes") or [],
"same_model": same_model, "rewrite": rewrite,
"same_model": same_model, "rewrite": rewrite, "writer_model": writer_model,
}
@@ -1103,8 +1359,11 @@ def apply_judgement(doc: dict[str, Any], outcome: dict[str, Any], summary: dict[
return doc
def on_demand(doc: dict[str, Any], *, on_unsupported: str = "degrade", rewrite_once: bool = False) -> dict[str, Any]:
"""探索运行:节点里不花这笔钱。候选句标「未裁判 · 按需」,点开哪句再判哪句。返回摘要(已写进文档)。"""
def on_demand(doc: dict[str, Any], *, on_unsupported: str = "degrade", rewrite_once: bool = False,
writer_model: str | None = None) -> dict[str, Any]:
"""探索运行:节点里不花这笔钱。候选句标「未裁判 · 按需」,点开哪句再判哪句。返回摘要(已写进文档)。
writer_model 记进摘要:按需裁判时拿它判断裁判模型是不是和写作模型相同。"""
request = prepare(doc, {})
budget = Budget()
verdicts = {c.unit: _unjudged("on_demand", _why_unjudged("on_demand", budget), judge=None, post_seal=False)
@@ -1112,6 +1371,7 @@ def on_demand(doc: dict[str, Any], *, on_unsupported: str = "degrade", rewrite_o
outcome = _outcome(request, verdicts, budget, {})
outcome.pop("_gaps")
outcome["budget"] = None
summary = summarize(outcome, mode="on_demand", on_unsupported=on_unsupported, rewrite_once=rewrite_once)
summary = summarize(outcome, mode="on_demand", on_unsupported=on_unsupported, rewrite_once=rewrite_once,
writer_model=writer_model)
apply_judgement(doc, outcome, summary)
return summary
+1 -1
View File
@@ -92,7 +92,7 @@ CONTRACT_KEY_LABEL: dict[str, str] = {
JUDGE_FIELD_LABEL: dict[str, str] = {
"provider": "裁判模型 · 接入", "model": "裁判模型", "max_claims": "最多裁判句数",
"max_cost_usd": "金额上限(美元)", "timeout_s": "时长上限(秒)",
"rewrite_once": "证据不支持的句子退回改写一次", "on_unsupported": "证据不支持时",
"rewrite_once": "证据相矛盾或不足的句子退回改写一次", "on_unsupported": "证据相矛盾时",
}
#: 取值的短说法:键 → {值: 叫法}。前端下拉的选项文字去掉后半句的用途说明
+22 -10
View File
@@ -417,7 +417,7 @@ def _check_citations(
#: 没挂依据的结论句怎么处置,从松到严。认不出的写法按 degrade 算:写错一个词不能让缺口悄悄消失
_ON_UNCITED = ("ignore", "degrade", "withhold")
#: 证据不支持的结论句怎么处置(claims: judge),从松到严。认不出的写法同样按 degrade 算
#: 证据相矛盾的结论句怎么处置(claims: judge 的 on_unsupported),从松到严。认不出的写法同样按 degrade 算
_ON_UNSUPPORTED = ("degrade", "withhold")
#: 要求结论句挂依据的策略:judge 在这一点上和 require_citation 一样严,另外再按裁判的判定判档
_CITING = ("require_citation", "judge")
@@ -434,9 +434,10 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
里 ignore 一律当 degrade:没有哪道门禁查契约里的 claims,不能让它把用户要的底线拉低。
报告节点写的是 judge 时,判定在文档里(按哈希取回的那份:判定是模型给的,出口复算不了,只能信封存的
文档)。正式运行在节点里判过:证据不支持的按 on_unsupported 判档(报告节点和契约取更严的),没判完的
(裁判没跑成、触顶)把裁判摘要里的缺口照抄进来——不能判完整出具,也不当成「不支持」。探索运行按需裁判,
这里只标注。契约要 judge、报告节点却没开的,照实记缺口:出口替不了报告节点去裁判。
文档)。正式运行在节点里判过:证据相矛盾的(contradicted,旧文档里的 unsupported 同样算)按 on_unsupported
判档(报告节点和契约取更严的);证据不足的(insufficient)记一条缺口——最多降档,不会不予出具,也不会
完整出具;没判完的(裁判没跑成、触顶)把裁判摘要里的缺口照抄进来——不能判完整出具,也不当成「矛盾」。
探索运行按需裁判,这里只标注。契约要 judge、报告节点却没开的,照实记缺口:出口替不了报告节点去裁判。
数的是这里重新核对出来的 uncited,不信文档里记的 cites。升级前写的报告(节点产出里没有 claims 这个键,
包括跨着升级还没跑完的运行)一律不管,契约写了也只记一条缺口:报告生成于升级前,结论句未核对。
@@ -472,6 +473,7 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
return
from app.engine.evidence import iter_units
from app.engine.judge import COUNT_KEYS, UNSUPPORTED
summary = (doc or {}).get("judge")
if not isinstance(summary, dict):
@@ -483,7 +485,7 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
asked.append(declared.get("on_unsupported"))
on_unsupported = max((a if a in _ON_UNSUPPORTED else "degrade" for a in asked), key=_ON_UNSUPPORTED.index)
inline = summary.get("mode") == "inline"
flagged: dict[str, list[dict[str, Any]]] = {"unsupported": [], "partial": []}
flagged: dict[str, list[dict[str, Any]]] = {s: [] for s in ("contradicted", "insufficient", UNSUPPORTED, "partial")}
if inline:
# 探索运行按需裁判的判定不在文档里(在封存之后追加的 evidence.judged 事件里),这里只数正式运行的
markdown = str((doc or {}).get("markdown") or "")
@@ -493,16 +495,26 @@ def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, An
if status in flagged:
span = unit.get("span") or [0, 0]
flagged[status].append({"unit": unit.get("id"), "span": span, "text": markdown[span[0]:span[1]],
"rationale": verdict.get("rationale") or ""})
"rationale": verdict.get("rationale") or "",
**({"missing": verdict["missing"]} if status == "insufficient"
and verdict.get("missing") else {})})
out["gaps"].extend(f"报告「{title}」{gap}" for gap in summary.get("gaps") or [])
counts = {k: int((summary.get("counts") or {}).get(k) or 0)
for k in ("supported", "partial", "unsupported", "not_a_claim", "unjudged")}
if lacking := flagged["insufficient"]:
# 证据不足按缺口处理:最多降档,不因 on_unsupported 不予出具,也不能完整出具
missing = list(dict.fromkeys(i["missing"] for i in lacking if i.get("missing")))
what = f"(缺少:{'、'.join(missing[:5])}{'等' if len(missing) > 5 else ''})" if missing else ""
out["gaps"].append(f"报告「{title}」有 {len(lacking)} 句结论证据不足{what}:裁判未能在证据中找到判断所需的"
"信息,这些结论未经核实")
counts = {k: int((summary.get("counts") or {}).get(k) or 0) for k in COUNT_KEYS}
# 证据相矛盾的按 on_unsupported 判档;旧文档里的 unsupported(当时没分矛盾和不足)照旧当 contradicted
contradicted = [*flagged["contradicted"], *flagged[UNSUPPORTED]]
out["claims_policy"] = {"policy": "judge", "on_unsupported": on_unsupported, "on_uncited": on_uncited}
out["uncited"], out["unsupported"] = uncited, flagged["unsupported"]
out["uncited"], out["unsupported"] = uncited, contradicted
out["claims"] = {
"policy": "judge", "uncited_claims": len(uncited), "uncited": uncited[:_LISTED], **extra,
"on_unsupported": on_unsupported, "mode": "inline" if inline else "on_demand", "counts": counts,
"unsupported": flagged["unsupported"][:_LISTED], "partial": flagged["partial"][:_LISTED],
"contradicted": flagged["contradicted"][:_LISTED], "insufficient": flagged["insufficient"][:_LISTED],
"unsupported": flagged[UNSUPPORTED][:_LISTED], "partial": flagged["partial"][:_LISTED],
"unjudged": dict(summary.get("unjudged") or {}), "limits_hit": list(summary.get("limits_hit") or []),
"complete": bool(summary.get("complete")), "model": summary.get("model"),
}
+16 -6
View File
@@ -20,7 +20,7 @@
- 正式运行在节点里判:写完、核对完之后判,判定写进文档的 unit.verdict,随文档一起封存。判定只标注、
不改写正文;裁判没跑成、触顶了照样产出,没判的记未裁判并留下缺口,出口按 on_unsupported 判档
- 探索运行不在节点里花这笔钱:候选句标「未裁判 · 按需」,点开哪句再判哪句(证据接口)
- rewrite_once(默认关):证据不支持的句子连同理由交回写作者只改这几句,再判一次,最多一轮;
- rewrite_once(默认关):证据相矛盾、证据不足的句子连同理由交回写作者只改这几句,再判一次,最多一轮;
改写稿冒出原稿没有的违规(只标注的可疑实体也算)就不采用
- 裁判调用包在 @task 里,排在节点所有已有调用之后:「接着跑」时直接取 checkpoint 里的判定,
不重复调用、不重复计费。写作调用不进 checkpoint,接着跑时会重写一稿——所以判定带着输入指纹,
@@ -450,7 +450,7 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
if await _run_class(ctx.run.run_id) != "formal":
# 探索运行默认按需裁判:点开哪句才判哪句,节点里不花这笔钱
judgement = judging.on_demand(doc, on_unsupported=judge_cfg["on_unsupported"],
rewrite_once=judge_cfg["rewrite_once"])
rewrite_once=judge_cfg["rewrite_once"], writer_model=model_id)
elif not (blocking and on_violation == "fail"):
# 反正要判失败的报告不值得再花钱判(节点紧接着就报错)
doc, response, blocking, judgement, usages = await _judge_inline(
@@ -505,13 +505,21 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
def _rewrite_request(sentences: list[tuple[str, str]]) -> str:
listed = "\n".join(f"{i}.「{text}」——{why or '证据不支持这句话'}" for i, (text, why) in enumerate(sentences, 1))
return (
"另一个模型按证据逐句核对了你的报告,下面这几句它认为证据不支持:\n"
"另一个模型按证据逐句核对了你的报告,下面这几句它认为证据不支持(和证据相矛盾,或者证据不足):\n"
f"{listed}\n\n"
"请只改这几句:补上证据目录里真有的依据 [[see:…]],或者把说法收窄到证据能支持的程度,或者删掉;"
"其余部分一字不改。数字仍然只能用引用标记写。不要解释改了什么,只输出改好的整篇报告正文。"
)
def _rewrite_reason(verdict: dict[str, Any]) -> str:
"""交回写作者的理由:裁判的理由,证据不足的再写上缺的是什么。"""
why = str(verdict.get("rationale") or "")
if verdict.get("status") == "insufficient" and verdict.get("missing"):
why = f"{why}(缺少:{verdict['missing']})" if why else f"证据不足,缺少:{verdict['missing']}"
return why
def _merge(first: dict[str, Any], second: dict[str, Any], budget: judging.Budget) -> dict[str, Any]:
"""改写前后两轮裁判合成一份:判定、句数、缺口按改写后这一稿,花费、调用、用时两轮相加。"""
merged = dict(second)
@@ -582,10 +590,11 @@ async def _judge_inline(ctx: NodeContext, doc: dict[str, Any], catalog: dict[str
usages = [outcome["usage"]]
response = messages[-1]
rewrite: dict[str, Any] | None = None
bad = [uid for uid, v in outcome["verdicts"].items() if v["status"] == "unsupported"]
# 交回改写的:证据相矛盾的(含旧取值 unsupported)和证据不足的——两样原来都判作「证据不支持」
bad = [uid for uid, v in outcome["verdicts"].items() if v["status"] in (*judging.CONTRADICTED, "insufficient")]
if judge_cfg["rewrite_once"] and bad:
texts = judging.unit_texts(doc)
sentences = [(texts.get(uid, ""), outcome["verdicts"][uid].get("rationale") or "") for uid in bad]
sentences = [(texts.get(uid, ""), _rewrite_reason(outcome["verdicts"][uid])) for uid in bad]
ctx.emit(EventType.LOG, level="warn", code="report_rewrite",
message=f"裁判认为 {len(bad)} 句结论证据不支持({_named([{'text': t} for t, _ in sentences])}),"
"已退回写作模型仅修改这几句")
@@ -620,7 +629,8 @@ async def _judge_inline(ctx: NodeContext, doc: dict[str, Any], catalog: dict[str
rewrite.update(applied=True, units=[at.get(texts.get(uid, "")) for uid in bad],
changed=[uid for uid, text in now.items() if text not in before])
summary = judging.summarize(outcome, mode="inline", on_unsupported=judge_cfg["on_unsupported"],
rewrite_once=judge_cfg["rewrite_once"], same_model=same_model, rewrite=rewrite)
rewrite_once=judge_cfg["rewrite_once"], same_model=same_model, rewrite=rewrite,
writer_model=model_id)
judging.apply_judgement(doc, outcome, summary)
return doc, response, blocking, summary, usages
+1 -1
View File
@@ -425,7 +425,7 @@ CLAIMS_JUDGE = "judge"
CLAIMS_VALUES = (*REPORT_CLAIMS, CLAIMS_JUDGE)
#: judge 子配置能写的键。三个上限写 null 表示不限;没写的键运行时取设置里的默认(设置 → judge 一组)
JUDGE_KEYS = ("provider", "model", "max_claims", "max_cost_usd", "timeout_s", "rewrite_once", "on_unsupported")
#: 证据不支持时正式运行怎么判档:degrade 降档(默认)/ withhold 不予出具。探索运行只标注
#: 证据相矛盾时正式运行怎么判档:degrade 降档(默认)/ withhold 不予出具。探索运行只标注
JUDGE_ON_UNSUPPORTED = ("degrade", "withhold")
+18 -16
View File
@@ -152,24 +152,25 @@ async def test_every_claim_supported_is_a_full_issuance(monkeypatch):
assert issuance["tier"] == "formal" and issuance["gaps"] == [], issuance
claims = issuance["claims"]
assert claims["policy"] == "judge" and claims["mode"] == "inline" and claims["complete"] is True
assert claims["counts"] == {"supported": 2, "partial": 0, "unsupported": 0, "not_a_claim": 0, "unjudged": 0}
assert claims["counts"] == {"supported": 2, "partial": 0, "contradicted": 0, "insufficient": 0, "not_a_claim": 0,
"unjudged": 0, "unsupported": 0}
assert claims["on_unsupported"] == "degrade" and claims["uncited_claims"] == 0
assert claims["unsupported"] == [] and claims["model"] == JUDGE_MODEL
assert claims["contradicted"] == [] and claims["insufficient"] == [] and claims["model"] == JUDGE_MODEL
async def test_an_unsupported_claim_degrades_by_default(monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
async def test_a_contradicted_claim_degrades_by_default(monkeypatch):
Models(monkeypatch, verdicts={"新客": "contradicted"})
issuance = await issued(weekly())
assert issuance["tier"] == "degraded", "不支持按缺口降档,不是不予出具(契约是 strict 的)"
assert issuance["gaps"] == [], "判了就不是「没核」:档位来自判定,不来自缺口"
[flagged] = issuance["claims"]["unsupported"]
[flagged] = issuance["claims"]["contradicted"]
assert flagged["text"] == "增长主要来自新客首单。" and flagged["rationale"] == "证据里没有新客维度"
assert flagged["unit"] and len(flagged["span"]) == 2
assert issuance["claims"]["counts"]["unsupported"] == 1
assert issuance["claims"]["counts"]["contradicted"] == 1
async def test_withhold_withholds(monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
Models(monkeypatch, verdicts={"新客": "contradicted"})
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
assert issuance["tier"] == "withheld" and issuance["claims"]["on_unsupported"] == "withhold"
@@ -209,23 +210,24 @@ async def test_an_uncited_claim_still_counts_under_judge(monkeypatch):
async def test_the_issuance_event_carries_the_counts(monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
Models(monkeypatch, verdicts={"新客": "contradicted"})
run = await run_manager.start(graph=weekly(), input_payload={}, run_class="formal")
row = await wait(run.id)
event = await issuance_event(row.id)
assert event["tier"] == "degraded"
assert event["claims"] == {"policy": "judge", "uncited_claims": 0, "on_unsupported": "degrade",
"mode": "inline", "counts": {"supported": 1, "partial": 0, "unsupported": 1,
"mode": "inline", "counts": {"supported": 1, "partial": 0, "contradicted": 1,
"insufficient": 0, "unsupported": 0,
"not_a_claim": 0, "unjudged": 0}}
async def test_a_governed_formal_run_grades_by_the_verdicts(client, monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
Models(monkeypatch, verdicts={"新客": "contradicted"})
graph = weekly(report={"numbers": "strict", "on_violation": "fail",
"judge": {"max_cost_usd": None, "on_unsupported": "withhold"}})
row = await governed(client, graph)
issuance = row.output["_issuance"]
assert issuance["tier"] == "withheld" and issuance["claims"]["unsupported"][0]["text"] == "增长主要来自新客首单。"
assert issuance["tier"] == "withheld" and issuance["claims"]["contradicted"][0]["text"] == "增长主要来自新客首单。"
# --------------------------------------------------------------------------
@@ -234,7 +236,7 @@ async def test_a_governed_formal_run_grades_by_the_verdicts(client, monkeypatch)
async def test_explore_runs_only_annotate(monkeypatch):
models = Models(monkeypatch, verdicts={"新客": "unsupported"})
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}),
run_class="exploratory")
assert models.judges == [], "探索运行节点里不花这笔钱"
@@ -242,7 +244,7 @@ async def test_explore_runs_only_annotate(monkeypatch):
claims = issuance["claims"]
assert claims["mode"] == "on_demand" and claims["complete"] is False
assert claims["counts"]["unjudged"] == 2 and claims["unjudged"] == {"on_demand": 2}
assert claims["unsupported"] == []
assert claims["contradicted"] == [] and claims["insufficient"] == []
async def test_explore_runs_still_count_uncited_claims(monkeypatch):
@@ -257,14 +259,14 @@ async def test_explore_runs_still_count_uncited_claims(monkeypatch):
async def test_the_contract_can_tighten_on_unsupported(monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
Models(monkeypatch, verdicts={"新客": "contradicted"})
issuance = await issued(weekly(contract={"claims": {"policy": "judge", "on_unsupported": "withhold"}}))
assert issuance["tier"] == "withheld" and issuance["claims"]["on_unsupported"] == "withhold"
assert not any("不支持" in g and "这一版" in g for g in issuance["gaps"])
async def test_the_contract_cannot_loosen_on_unsupported(monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
Models(monkeypatch, verdicts={"新客": "contradicted"})
issuance = await issued(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}},
contract={"claims": {"policy": "judge", "on_unsupported": "degrade"}}))
assert issuance["tier"] == "withheld"
@@ -295,7 +297,7 @@ async def test_runs_from_before_the_upgrade_ignore_judge(monkeypatch):
return None
monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot)
models = Models(monkeypatch, text=UNCITED, verdicts={"新客": "unsupported"})
models = Models(monkeypatch, text=UNCITED, verdicts={"新客": "contradicted"})
issuance = await issued(weekly())
assert models.judges == [] and issuance["tier"] == "formal" and "claims" not in issuance
+12 -10
View File
@@ -3,7 +3,8 @@
数字、实体是确定性的,系统自己能核对;「增长主要来自华东」这种话说得对不对,只能交给另一个模型按
证据判断。这是概率性的判断,所以每一步都要留余地、说实话:
- 一份报告合并成一次调用(超过每批的句数就分批),每句给 supported / partial / unsupported / not_a_claim
- 一份报告合并成一次调用(超过每批的句数就分批),每句给 supported / partial / contradicted / insufficient /
not_a_claim
- 模型不支持结构化输出、或者给的不是合法结构时退回 JSON 解析,再失败就记「未裁判」,不编判定
- 裁判模型关掉 thinking,max_tokens 4096
- 预算(每份报告的句数、金额、时长,全局每日金额)每一项都能写 null 表示不限;触顶就停、已判的保留、
@@ -183,7 +184,7 @@ def status_of(outcome, doc, text):
async def test_one_call_for_the_whole_report_and_the_verdict_shape(monkeypatch, stored):
judge = Judge(monkeypatch, {"u3": "unsupported"})
judge = Judge(monkeypatch, {"u3": "contradicted"})
catalog = make_catalog(stored)
doc = compose_doc(RAW, catalog)
outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED)
@@ -192,7 +193,8 @@ async def test_one_call_for_the_whole_report_and_the_verdict_shape(monkeypatch,
v = status_of(outcome, doc, CITED)
assert v["status"] == "supported" and v["judge"] == PRICED and v["post_seal"] is False
assert v["rationale"] and v["used"] == ["m:gmv"]
assert outcome["counts"] == {"supported": 3, "partial": 0, "unsupported": 1, "not_a_claim": 0, "unjudged": 0}
assert outcome["counts"] == {"supported": 3, "partial": 0, "contradicted": 1, "insufficient": 0, "not_a_claim": 0,
"unjudged": 0, "unsupported": 0}
assert outcome["model"] == PRICED and outcome["calls"] == 1 and outcome["gaps"] == []
assert units_of(doc)["下面看细节。"] not in outcome["verdicts"], "连接性的话不送裁判"
@@ -226,7 +228,7 @@ async def test_structured_output_failure_falls_back_to_json(monkeypatch, stored)
async def test_an_array_written_as_json_text_is_still_read(monkeypatch, stored):
"""有的模型走工具调用时把嵌套的数组写成一段 JSON 文本({"items": "[…]"}):照样认,不为它再花一次调用。"""
def stringified(units):
items = [{"unit": u, "verdict": "unsupported", "rationale": "对不上", "used": []} for u in units]
items = [{"unit": u, "verdict": "contradicted", "rationale": "对不上", "used": []} for u in units]
return json.dumps({"items": json.dumps(items, ensure_ascii=False)}, ensure_ascii=False)
judge = Judge(monkeypatch, structured=stringified)
@@ -234,7 +236,7 @@ async def test_an_array_written_as_json_text_is_still_read(monkeypatch, stored):
doc = compose_doc(RAW, catalog)
outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED)
assert [c["kind"] for c in judge.calls] == ["structured"]
assert {v["status"] for v in outcome["verdicts"].values()} == {"unsupported"}
assert {v["status"] for v in outcome["verdicts"].values()} == {"contradicted"}
async def test_a_broken_provider_is_not_called_batch_after_batch(monkeypatch, stored):
@@ -266,18 +268,18 @@ async def test_bad_items_are_not_trusted(monkeypatch, stored):
def messy(units):
return json.dumps({"items": [
{"unit": ids[CITED], "verdict": "supported", "rationale": "长" * 100, "used": ["m:gmv", "m:nope", "Q9"]},
{"unit": ids[CITED], "verdict": "supported", "rationale": "长" * 200, "used": ["m:gmv", "m:nope", "Q9"]},
{"unit": ids[CITED2], "verdict": "maybe", "rationale": "不认识的判定", "used": []},
{"unit": ids[CAUSAL], "verdict": "unsupported", "rationale": "没有新客维度", "used": []},
{"unit": ids[CAUSAL], "verdict": "contradicted", "rationale": "没有新客维度", "used": []},
{"unit": "u999", "verdict": "supported", "rationale": "不存在的句子", "used": []},
]}, ensure_ascii=False)
Judge(monkeypatch, structured=messy)
outcome = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED)
v = outcome["verdicts"]
assert v[ids[CITED]]["rationale"] == "长" * RATIONALE_MAX and v[ids[CITED]]["used"] == ["m:gmv"]
assert v[ids[CITED]]["rationale"] == "长" * (RATIONALE_MAX - 1) + "…" and v[ids[CITED]]["used"] == ["m:gmv"]
assert (v[ids[CITED2]]["status"], v[ids[CITED2]]["reason"]) == ("unjudged", "format")
assert v[ids[CAUSAL]]["status"] == "unsupported"
assert v[ids[CAUSAL]]["status"] == "contradicted"
assert (v[ids[NUMBER]]["status"], v[ids[NUMBER]]["reason"]) == ("unjudged", "missing")
assert "u999" not in v
@@ -532,7 +534,7 @@ async def test_a_model_without_a_price_ignores_money_limits(monkeypatch, stored)
async def test_known_sentences_are_not_judged_again(monkeypatch, stored):
judge = Judge(monkeypatch, {"u2": "unsupported"})
judge = Judge(monkeypatch, {"u2": "contradicted"})
catalog = make_catalog(stored)
doc = compose_doc(RAW, catalog)
first = await judge_doc(doc, catalog, spec=SPEC, budget=UNLIMITED)
+14 -14
View File
@@ -175,7 +175,7 @@ async def save_setting(key: str, value) -> None:
async def test_an_on_demand_verdict_lands_after_the_seal_and_the_seal_still_holds(client, monkeypatch):
models = Models(monkeypatch, verdicts={"新客": "unsupported"})
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
row = await finish(weekly(claims="judge"))
assert row.status == "succeeded" and models.judges == [], "探索运行节点里不判"
doc = doc_of(row)
@@ -185,7 +185,7 @@ async def test_an_on_demand_verdict_lands_after_the_seal_and_the_seal_still_hold
assert res.status_code == 200, res.text
body = res.json()
verdict = body["verdicts"][ids[CAUSAL]]
assert verdict["status"] == "unsupported" and verdict["post_seal"] is True and verdict["judge"] == PRICED
assert verdict["status"] == "contradicted" and verdict["post_seal"] is True and verdict["judge"] == PRICED
assert verdict["rationale"] and body["judged"] == [ids[CAUSAL]] and body["reused"] == []
assert body["report"] == {"node_id": "write", "doc_artifact": row.output["_evidence"]["doc_artifact"]}
assert body["post_seal"] is True and body["limited"] is False and body["calls"] == 1
@@ -276,9 +276,9 @@ async def test_a_failed_attempt_can_be_retried(client, monkeypatch):
failed = (await judge(client, row.id, [uid])).json()
assert failed["verdicts"][uid]["status"] == "unjudged" and failed["verdicts"][uid]["reason"] == "error"
assert "网关 502" in failed["message"] and failed["limited"] is False
models = Models(monkeypatch, verdicts={"新客": "unsupported"})
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
retried = (await judge(client, row.id, [uid])).json()
assert retried["verdicts"][uid]["status"] == "unsupported" and retried["reused"] == [] and len(models.judges) == 1
assert retried["verdicts"][uid]["status"] == "contradicted" and retried["reused"] == [] and len(models.judges) == 1
# --------------------------------------------------------------------------
@@ -414,7 +414,7 @@ async def test_a_broken_seal_is_not_judged(client, monkeypatch):
async def test_a_run_continued_during_the_judge_call_records_no_event(client, monkeypatch, meanwhile):
"""判定只接在点击开始时的那份封存后面。运行在裁判期间被接着跑了,再追加就落进一次活着的运行中间,
还会被封进下一份清单:不记进运行记录,判定照样交回给点的人,并说明为什么没记。"""
models = Models(monkeypatch, verdicts={"新客": "unsupported"})
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
row = await finish(weekly(claims="judge"))
uid = unit_ids(doc_of(row))[CAUSAL]
ask_model = judge_mod.get_chat_model
@@ -430,7 +430,7 @@ async def test_a_run_continued_during_the_judge_call_records_no_event(client, mo
assert res.status_code == 200, res.text
body = res.json()
assert len(models.judges) == 1 and body["calls"] == 1, "模型问过了"
assert body["verdicts"][uid]["status"] == "unsupported" and body["judged"] == [uid], "判定照样交回"
assert body["verdicts"][uid]["status"] == "contradicted" and body["judged"] == [uid], "判定照样交回"
assert body["event"] is None and "运行已继续执行" in body["message"] and "未写入运行记录" in body["message"]
assert await judged_events(row.id) == []
async with SessionLocal() as session:
@@ -444,7 +444,7 @@ async def test_a_continue_that_wins_the_row_between_check_and_write_gets_no_even
from app.core.bus import bus
from app.core.config import settings
Models(monkeypatch, verdicts={"新客": "unsupported"})
Models(monkeypatch, verdicts={"新客": "contradicted"})
row = await finish(weekly(claims="judge"))
uid = unit_ids(doc_of(row))[CAUSAL]
set_seq = bus.set_seq
@@ -462,7 +462,7 @@ async def test_a_continue_that_wins_the_row_between_check_and_write_gets_no_even
monkeypatch.setattr(bus, "set_seq", claimed_in_between)
body = (await judge(client, row.id, [uid])).json()
assert body["verdicts"][uid]["status"] == "unsupported" and body["event"] is None
assert body["verdicts"][uid]["status"] == "contradicted" and body["event"] is None
assert "未写入运行记录" in body["message"] and await judged_events(row.id) == []
async with SessionLocal() as session:
after = await session.get(Run, row.id)
@@ -503,7 +503,7 @@ async def test_sentences_that_are_not_claims_are_skipped(client, monkeypatch):
async def test_the_segment_shows_the_verdict_and_whether_it_can_still_be_asked(client, monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
Models(monkeypatch, verdicts={"新客": "contradicted"})
row = await finish(weekly(claims="judge"))
doc = doc_of(row)
ids = unit_ids(doc)
@@ -513,7 +513,7 @@ async def test_the_segment_shows_the_verdict_and_whether_it_can_still_be_asked(c
await judge(client, row.id, [ids[CAUSAL]])
after = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, ids[CAUSAL])}")).json()
assert after["unit"]["verdict"]["status"] == "unsupported" and after["unit"]["verdict"]["post_seal"] is True
assert after["unit"]["verdict"]["status"] == "contradicted" and after["unit"]["verdict"]["post_seal"] is True
assert after["unit"]["on_demand"]["available"] is False and after["unit"]["on_demand"]["reason"] == "judged"
assert "模型" in after["note"] and "后续版本" not in after["note"]
other = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, ids[CITED])}")).json()
@@ -531,11 +531,11 @@ async def test_the_segment_shows_the_verdict_and_whether_it_can_still_be_asked(c
async def test_sealed_verdicts_of_a_formal_run_are_shown_as_sealed(client, monkeypatch):
Models(monkeypatch, verdicts={"新客": "unsupported"})
Models(monkeypatch, verdicts={"新客": "contradicted"})
row = await finish(weekly(claims="judge", judge={"max_cost_usd": 0.05}), run_class="formal")
doc = doc_of(row)
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, unit_ids(doc)[CAUSAL])}")).json()
assert body["unit"]["verdict"]["status"] == "unsupported" and body["unit"]["verdict"]["post_seal"] is False
assert body["unit"]["verdict"]["status"] == "contradicted" and body["unit"]["verdict"]["post_seal"] is False
assert body["unit"]["on_demand"]["reason"] == "formal"
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
assert graph["reports"][0]["post_seal_verdicts"] == {} and graph["reports"][0]["judge"]["mode"] == "inline"
@@ -543,7 +543,7 @@ async def test_sealed_verdicts_of_a_formal_run_are_shown_as_sealed(client, monke
async def test_a_forged_event_cannot_overwrite_a_sealed_verdict(client, monkeypatch):
"""封存后追加的判定只填补没判过的句子:封存内判过的,后面追加什么都盖不掉。"""
Models(monkeypatch, verdicts={"新客": "unsupported"})
Models(monkeypatch, verdicts={"新客": "contradicted"})
row = await finish(weekly(claims="judge", judge={"max_cost_usd": 0.05}), run_class="formal")
doc = doc_of(row)
uid = unit_ids(doc)[CAUSAL]
@@ -552,7 +552,7 @@ async def test_a_forged_event_cannot_overwrite_a_sealed_verdict(client, monkeypa
doc_artifact=doc_artifact, post_seal=True,
verdicts={uid: {"status": "supported", "rationale": "改好了", "judge": "x", "post_seal": True}})
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg_in(doc, uid)}")).json()
assert body["unit"]["verdict"]["status"] == "unsupported"
assert body["unit"]["verdict"]["status"] == "contradicted"
async def test_appended_verdicts_always_read_as_post_seal(client, monkeypatch):
+5 -5
View File
@@ -249,7 +249,7 @@ WITHHOLD = {**JUDGED, "judge": {"max_cost_usd": 0.05, "on_unsupported": "withhol
])
def test_relaxing_on_unsupported_on_the_node_is_loosening(judge, dropped):
[reason] = _changed(WITHHOLD, {**WITHHOLD, "judge": judge})
assert "「报告撰写」" in reason and "「结论句裁判 · 证据不支持时」" in reason
assert "「报告撰写」" in reason and "「结论句裁判 · 证据相矛盾时」" in reason
assert "「不予出具」" in reason and "「出具降档」" in reason
assert ("删除了已设置的值" in reason) is dropped, reason
@@ -258,7 +258,7 @@ def test_replacing_the_whole_judge_block_to_add_a_budget_is_loosening():
"""补预算把 judge 整个换成 {max_cost_usd}:连同裁判模型、withhold 一起丢了。"""
before = {**STRICT, "claims": "judge", "judge": {"model": "judge-model", "on_unsupported": "withhold"}}
[reason] = _changed(before, {**before, "judge": {"max_cost_usd": 0.05}})
assert "「结论句裁判 · 证据不支持时」" in reason and "「不予出具」" in reason
assert "「结论句裁判 · 证据相矛盾时」" in reason and "「不予出具」" in reason
assert _changed(before, {**before, "judge": {**before["judge"], "max_cost_usd": 0.05}}) == [], \
"带着原来的键补预算不算放宽"
@@ -268,7 +268,7 @@ def test_a_withhold_from_the_graph_defaults_lost_by_a_node_judge_is_loosening():
defaults = {"judge": {"on_unsupported": "withhold", "max_cost_usd": 0.05}}
report = {**STRICT, "claims": "judge"}
[reason] = _changed(report, {**report, "judge": {"max_cost_usd": 0.05}}, defaults=defaults)
assert "「结论句裁判 · 证据不支持时」" in reason and "从「不予出具」(跟随工作流默认设置)" in reason, reason
assert "「结论句裁判 · 证据相矛盾时」" in reason and "从「不予出具」(跟随工作流默认设置)" in reason, reason
assert "不再跟随工作流默认设置" in reason, "说清楚为什么全图默认里的 withhold 不生效了"
assert _changed(report, {**report, "judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}},
defaults=defaults) == []
@@ -280,7 +280,7 @@ def test_relaxing_on_unsupported_in_the_graph_defaults_is_loosening():
new = copy.deepcopy(old)
new["defaults"]["judge"]["on_unsupported"] = "degrade"
[reason] = forbidden_changes(old, new)
assert "「报告撰写」" in reason and "「结论句裁判 · 证据不支持时」" in reason
assert "「报告撰写」" in reason and "「结论句裁判 · 证据相矛盾时」" in reason
@pytest.mark.parametrize("before, after", [
@@ -297,7 +297,7 @@ def test_tightening_or_keeping_on_unsupported_is_fine(before, after):
def test_dropping_withhold_along_with_judge_is_one_reason():
"""从 judge 退回 require_citation:claims 那一条已经说了,不再为 on_unsupported 另记一条。"""
[reason] = _changed(WITHHOLD, {**STRICT, "judge": {"max_cost_usd": 0.05}})
assert "「未附依据的结论句」" in reason and "证据不支持时" not in reason
assert "「未附依据的结论句」" in reason and "证据相矛盾时" not in reason
@pytest.mark.parametrize("before", ["require_citation", "judge"])
+486
View File
@@ -0,0 +1,486 @@
"""结论句裁判改造(JR)的接口:片段接口的字段清单、按需裁判的模型标记、证据图的 judge_meta、审计行。
- 片段接口里表名实体那一步带上字段清单 fields: [{name, type}](封存范围内的表结构快照,最多 60 个,
多出来的记 fields_more),遮罩的列不列
- 按需裁判也判断裁判模型和写作模型是否相同(写作模型取这份报告撰写节点实际用的模型),并带上
priced、writer_model;evidence.judged 事件同样记下
- GET /evidence 每份报告带 judge_meta:最近一次按需裁判的 {model, same_model, priced, writer_model}
- 交给裁判的新摘录(字段清单、指标输入的 SQL)只经封存范围内的 loader 读
- 审计行的 issue 新增 contradicted_claim、insufficient_claim
- 旧的 unsupported 判定照旧算「判过」:封存之后追加的判定盖不掉它
所有裁判都用 mock 模型给剧本,不调真接口。
"""
from __future__ import annotations
import asyncio
import csv
import io
import json
import re
import sqlite3
from types import SimpleNamespace
import pytest
from httpx import ASGITransport, AsyncClient
from langchain_core.messages import AIMessage
from sqlalchemy import delete, select
from app.core import artifact_store
from app.core.events import EventType
from app.data.engine import engines
from app.db.base import SessionLocal
from app.db.models import DataSource, Run, RunEvent, Setting
from app.engine import judge as judge_mod
from app.engine.evidence import iter_segments, iter_units
from app.engine.judge import JUDGE_ROLE, SPEND_KEY
from app.engine.runner import run_manager
from app.main import app
from app.providers.mock_model import MockChatModel
PRICED = "claude-sonnet-5"
WRITER = "mock-fast"
SOURCE = "jr_shop"
TOOL = f"db_query__{SOURCE}"
SQL = "SELECT region, SUM(amount) AS gmv FROM orders GROUP BY region ORDER BY gmv DESC"
WIDE_COLS = [f"f{i:02d}" for i in range(65)]
METRIC = "东区销售额最高,达 [[m:gmv]]。[[see:m:gmv]]"
METHOD = "这个数由 [[t:orders]] 按 [[c:orders.region]] 分组求和得出。"
WIDE = "明细在 [[t:wide_orders]] 里。"
TEXT = METRIC + METHOD + WIDE
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
async with SessionLocal() as session:
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"])))
await session.commit()
yield
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
@pytest.fixture(autouse=True)
async def shop(tmp_path):
"""探查过结构的示例库:orders(带手机号列,数据源设了遮罩)和一张 65 列的宽表。"""
from app.data.introspect import introspect
path = tmp_path / "jr_shop.db"
db = sqlite3.connect(path)
db.executescript(
"CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, amount REAL, phone TEXT);"
f"CREATE TABLE wide_orders ({', '.join(f'{c} TEXT' for c in WIDE_COLS)});")
db.executemany("INSERT INTO orders VALUES (?, ?, ?, ?)",
[(1, "east", 300.5, "p1"), (2, "west", 200.0, "p2"), (3, "east", 100.0, "p3")])
db.commit()
db.close()
probe = SimpleNamespace(id=f"probe-{path.name}", name=SOURCE, kind="sqlite", database=str(path), host=None,
port=None, username=None, password=None, options={}, readonly=True, schema_cache={})
cache = await introspect(probe)
await engines.invalidate(probe.id)
async with SessionLocal() as session:
row = (await session.execute(select(DataSource).where(DataSource.name == SOURCE))).scalar_one_or_none()
if row is None:
row = DataSource(name=SOURCE, kind="sqlite", database=str(path), readonly=True, enabled=True)
session.add(row)
row.database, row.kind, row.enabled, row.readonly, row.schema_cache = str(path), "sqlite", True, True, cache
row.options = {"mask_columns": "phone"}
await session.commit()
source_id = row.id
await engines.invalidate(source_id)
yield source_id
await engines.invalidate(source_id)
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def weekly(**report) -> dict:
nodes = [
node("start", "input"),
node("fetch", "tool", tool=TOOL, args={"sql": SQL}),
node("card", "metrics", caliber="周报口径", caliber_version="v2", metrics=[
{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "cell(nodes.fetch, 0, 'gmv')"}]),
node("write", "report", instructions="写周报", on_violation="flag", claims="judge", **report),
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]),
]
ids = [n["id"] for n in nodes]
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
class Models:
"""写作者回 text;裁判按句子里的字给判定:verdicts = {句中片段: (判定, missing)}。judges 记下请求原文。"""
def __init__(self, monkeypatch, verdicts=None, *, model=PRICED, text=TEXT):
self.judges: list[str] = []
verdicts = verdicts or {}
def decide(chat, messages):
if JUDGE_ROLE in str(messages[0].content):
request = str(messages[-1].content)
self.judges.append(request)
items = []
for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M):
status, missing = next((v for frag, v in verdicts.items() if frag in line), ("supported", ""))
items.append({"unit": uid, "verdict": status, "rationale": f"判的是:{line.strip()[:12]}",
"missing": missing, "used": []})
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
return AIMessage(content=text)
monkeypatch.setattr(MockChatModel, "_decide", decide)
self.model = model
async def judge_model_of(session, spec):
return MockChatModel(model_name=self.model), self.model
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
async def wait(run_id: str) -> Run:
for _ in range(400):
await asyncio.sleep(0.05)
async with SessionLocal() as session:
row = await session.get(Run, run_id)
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
return row
raise AssertionError(f"run {run_id} 没跑完:{row.status}")
async def explore_run(graph=None) -> tuple[Run, dict]:
started = await run_manager.start(graph=graph or weekly(), input_payload={}, run_class="exploratory")
row = await wait(started.id)
assert row.status == "succeeded", row.error
return row, artifact_store.load(row.output["_evidence"]["doc_artifact"])
def unit_of(doc: dict, prefix: str) -> str:
md = doc["markdown"]
return next(u["id"] for _, u in iter_units(doc) if md[u["span"][0]:u["span"][1]].strip().startswith(prefix))
async def judged_events(run_id: str) -> list[RunEvent]:
async with SessionLocal() as session:
return list((await session.execute(select(RunEvent).where(
RunEvent.run_id == run_id, RunEvent.type == EventType.EVIDENCE_JUDGED).order_by(RunEvent.seq))).scalars())
async def ask(client, run_id: str, units: list[str]) -> dict:
res = await client.post(f"/api/runs/{run_id}/evidence/judge", json={"units": units})
assert res.status_code == 200, res.text
return res.json()
async def reseal(run_id: str) -> None:
async with SessionLocal() as session:
run = await session.get(Run, run_id)
rows = [tuple(r) for r in await session.execute(
select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data)
.where(RunEvent.run_id == run_id, RunEvent.seq <= run.manifest_seq).order_by(RunEvent.seq))]
run.manifest_hash = artifact_store.manifest_hash(rows)
await session.commit()
async def rewrite(run_id: str, etype: str, node_id: str, change) -> None:
async with SessionLocal() as session:
for row in (await session.execute(select(RunEvent).where(
RunEvent.run_id == run_id, RunEvent.type == etype, RunEvent.node_id == node_id))).scalars():
row.data = change(dict(row.data))
await session.commit()
# --------------------------------------------------------------------------
# 片段接口:表名实体步骤的字段清单
# --------------------------------------------------------------------------
async def open_entity(client, run_id: str, doc: dict, ref: str) -> dict:
seg = next(s for s in iter_segments(doc) if s.get("kind") == "entity" and s.get("ref") == ref)
res = await client.get(f"/api/runs/{run_id}/evidence/segments/{seg['id']}")
assert res.status_code == 200, res.text
[step] = res.json()["chain"]
return step
async def test_a_table_step_lists_its_fields_without_masked_columns(client, monkeypatch):
Models(monkeypatch)
row, doc = await explore_run()
step = await open_entity(client, row.id, doc, "t:orders")
assert step["step"] == "entity" and step["kind"] == "table"
assert step["fields"] == [{"name": "id", "type": "INTEGER"}, {"name": "region", "type": "TEXT"},
{"name": "amount", "type": "REAL"}], "遮罩的 phone 不列"
assert "fields_more" not in step and step["columns"] == 4
async def test_a_wide_table_step_stops_at_sixty_fields(client, monkeypatch):
Models(monkeypatch)
row, doc = await explore_run()
step = await open_entity(client, row.id, doc, "t:wide_orders")
assert [f["name"] for f in step["fields"]] == WIDE_COLS[:60]
assert step["fields_more"] == 5
async def test_a_table_step_outside_the_seal_has_no_fields(client, monkeypatch):
Models(monkeypatch)
row, doc = await explore_run()
await rewrite(row.id, "tool.end", "fetch", lambda d: {k: v for k, v in d.items() if k != "schema_artifact"})
await rewrite(row.id, "node.finished", "fetch",
lambda d: {**d, "evidence": [e for e in d.get("evidence") or [] if e.get("kind") != "schema"]})
await reseal(row.id)
step = await open_entity(client, row.id, doc, "t:orders")
assert "fields" not in step and "fields_more" not in step
# --------------------------------------------------------------------------
# 按需裁判:新的判定、摘录、模型标记
# --------------------------------------------------------------------------
async def test_on_demand_judging_sees_fields_and_metric_sql_and_reports_the_models(client, monkeypatch):
models = Models(monkeypatch, verdicts={"分组求和": ("insufficient", "字段清单"), "东区销售额": ("contradicted", "")})
row, doc = await explore_run()
method, metric = unit_of(doc, "这个数由"), unit_of(doc, "东区销售额")
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
assert graph["reports"][0]["judge_meta"] is None, "还没按需裁判过"
body = await ask(client, row.id, [method, metric])
request = models.judges[0]
assert "region TEXT" in request and "id INTEGER" in request, "表附上字段清单和类型"
assert "phone" not in request.lower(), "遮罩的列不出现"
assert "查询 Q1 的 SQL" in request and "SUM(amount)" in request and "r0" in request, "指标附上输入的查询和格子"
assert body["verdicts"][method]["status"] == "insufficient" and body["verdicts"][method]["missing"] == "字段清单"
assert body["verdicts"][metric]["status"] == "contradicted"
assert body["same_model"] is False and body["priced"] is True and body["writer_model"] == WRITER
[event] = await judged_events(row.id)
assert event.data["same_model"] is False and event.data["priced"] is True
assert event.data["writer_model"] == WRITER
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
assert graph["reports"][0]["judge_meta"] == {"model": PRICED, "same_model": False, "priced": True,
"writer_model": WRITER}
async def test_judging_with_the_writer_model_is_flagged(client, monkeypatch):
models = Models(monkeypatch, model=WRITER)
row, doc = await explore_run()
body = await ask(client, row.id, [unit_of(doc, "这个数由")])
assert models.judges and body["model"] == WRITER
assert body["same_model"] is True and body["writer_model"] == WRITER, "判定照常出,另带标记"
assert body["verdicts"][unit_of(doc, "这个数由")]["status"] == "supported"
[event] = await judged_events(row.id)
assert event.data["same_model"] is True and event.data["writer_model"] == WRITER
# judge_meta 取最近一次:换一个价格目录里没有的模型再判一句,就按新的这次
models.model = "house-judge-model"
body = await ask(client, row.id, [unit_of(doc, "东区销售额")])
assert body["same_model"] is False and body["priced"] is False, "目录里没有价格的模型:金额上限对它不生效"
meta = (await client.get(f"/api/runs/{row.id}/evidence")).json()["reports"][0]["judge_meta"]
assert meta == {"model": "house-judge-model", "same_model": False, "priced": False, "writer_model": WRITER}
async def test_the_new_excerpts_only_read_sealed_artifacts(client, monkeypatch):
"""表结构快照、查询快照被挪出封存范围(事件里追不到了):字段清单、指标输入的 SQL 都不进裁判的摘录。"""
models = Models(monkeypatch)
row, doc = await explore_run()
await rewrite(row.id, "tool.end", "fetch",
lambda d: {k: v for k, v in d.items() if k not in ("schema_artifact", "query_artifact")})
await rewrite(row.id, "node.finished", "fetch",
lambda d: {**d, "evidence": [e for e in d.get("evidence") or []
if e.get("kind") not in ("schema", "query")]})
await reseal(row.id)
await ask(client, row.id, [unit_of(doc, "这个数由"), unit_of(doc, "东区销售额")])
request = models.judges[0]
assert "INTEGER" not in request and "region TEXT" not in request
assert "SELECT" not in request
# --------------------------------------------------------------------------
# 审计行
# --------------------------------------------------------------------------
async def test_audit_rows_list_contradicted_and_insufficient_claims(client, monkeypatch):
Models(monkeypatch, verdicts={"分组求和": ("insufficient", "字段清单"), "东区销售额": ("contradicted", "")})
row, doc = await explore_run()
method, metric = unit_of(doc, "这个数由"), unit_of(doc, "东区销售额")
await ask(client, row.id, [method, metric])
body = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json()
rows = [r for g in body["groups"] for r in g["rows"] if r["kind"] == "claim"]
by_issue = {r["issue"]: r for r in rows}
assert by_issue["contradicted_claim"]["unit"] == metric and by_issue["contradicted_claim"]["group"] == "none"
assert by_issue["insufficient_claim"]["unit"] == method
assert "字段清单" in by_issue["insufficient_claim"]["note"]
assert by_issue["insufficient_claim"]["verdict"]["missing"] == "字段清单"
assert by_issue["contradicted_claim"]["verdict"]["post_seal"] is True
text = (await client.get(f"/api/runs/{row.id}/evidence/audit?format=csv")).text
issues = {line[8] for line in csv.reader(io.StringIO(text.lstrip("")))}
assert {"证据相矛盾", "证据不足"} <= issues
# --------------------------------------------------------------------------
# 旧的 unsupported:照旧算判过
# --------------------------------------------------------------------------
def test_an_old_unsupported_verdict_still_counts_as_judged():
from app.api.evidence import _effective_verdicts, _on_demand, _Report, _Sealed
from app.engine.evidence import compose_doc
doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r1")
[unit] = [u for _, u in iter_units(doc)]
unit["verdict"] = {"status": "unsupported", "rationale": "证据不支持", "judge": PRICED, "post_seal": False,
"used": []}
report = _Report(node_id="write", doc_artifact="d" * 64, ok=True, repairs=0, stats={}, doc=doc, hash_ok=True,
fields=[], claims="judge")
later = {"report": "write", "doc_artifact": "d" * 64,
"verdicts": {unit["id"]: {"status": "supported", "rationale": "有依据", "judge": PRICED}}}
sealed = _Sealed(run_id="r1", graph={}, seal={"sealed": True, "ok": True, "manifest_seq": 5, "legacy": False},
events=[], judged=[(9, "write", later)], run_class="exploratory", status="succeeded",
ledger_on=True)
effective, appended = _effective_verdicts(report, sealed)
assert effective[unit["id"]]["status"] == "unsupported" and appended == {}, "追加的判定盖不掉封存的判定"
assert _on_demand(sealed, doc, unit["id"], effective[unit["id"]])["reason"] == "judged"
def test_the_writer_model_of_an_old_report_comes_from_the_sealed_node_output():
"""改造前的文档裁判摘要里没有 writer_model、改造前的事件里没有 same_model:按报告撰写节点封存的产出里
实际用的模型补上。"""
from app.api.evidence import _judge_meta, _Report, _Sealed, _writer_model
from app.engine.evidence import compose_doc
text = artifact_store.canonical_json({"text": "增长主要来自新客首单。", "model": "writer-old"})
output = artifact_store.content_hash(text)
path = artifact_store._path_of(output)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text, encoding="utf-8")
doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r2")
doc["judge"] = {"mode": "on_demand", "model": None, "counts": {}}
report = _Report(node_id="write", doc_artifact="e" * 64, ok=True, repairs=0, stats={}, doc=doc, hash_ok=True,
fields=[], claims="judge")
old_event = {"report": "write", "doc_artifact": "e" * 64, "model": "writer-old", "priced": False,
"verdicts": {}}
sealed = _Sealed(run_id="r2", graph={}, seal={"sealed": True, "ok": True, "manifest_seq": 5, "legacy": False},
events=[(3, "node.finished", "write", {"artifact": output})], outputs={"write": output},
judged=[(9, "write", old_event)], run_class="exploratory", status="succeeded", ledger_on=True)
assert _writer_model(sealed, report) == "writer-old"
assert _judge_meta(report, sealed) == {"model": "writer-old", "same_model": True, "priced": False,
"writer_model": "writer-old"}
# --------------------------------------------------------------------------
# 按需裁判:早于当前规则的判定可以重判,当前规则下判过的照旧复用
# --------------------------------------------------------------------------
def on_demand_of(verdict: dict) -> dict:
from app.api.evidence import _on_demand, _Sealed
from app.engine.evidence import compose_doc
doc = compose_doc("增长主要来自新客首单。", {}, node_id="write", run_id="r3")
[unit] = [u for _, u in iter_units(doc)]
sealed = _Sealed(run_id="r3", graph={}, seal={"sealed": True, "ok": True, "manifest_seq": 5, "legacy": False},
events=[], run_class="exploratory", status="succeeded", ledger_on=True)
return _on_demand(sealed, doc, unit["id"], verdict)
@pytest.mark.parametrize("verdict", [
{"status": "supported", "rationale": "有依据", "post_seal": True},
{"status": "unsupported", "rationale": "证据不支持", "post_seal": True},
{"status": "insufficient", "rationale": "缺 SQL", "post_seal": True, "rule_version": 1},
])
def test_an_outdated_appended_verdict_can_be_asked_again(verdict):
got = on_demand_of(verdict)
assert got["available"] is True and got["reason"] == "outdated" and got["message"]
def test_a_current_verdict_is_not_asked_again():
got = on_demand_of({"status": "contradicted", "rationale": "对不上", "post_seal": True,
"rule_version": judge_mod.RULES_VERSION})
assert got["available"] is False and got["reason"] == "judged"
def test_a_sealed_verdict_is_never_reopened_by_version():
"""封存的判定(文档里的,post_seal 为 False)盖不掉:版本再旧也不开放重判。"""
got = on_demand_of({"status": "unsupported", "rationale": "证据不支持", "post_seal": False})
assert got["available"] is False and got["reason"] == "judged"
async def age(run_id: str, uid: str, **changes) -> None:
"""把封存之后追加的那条判定改成改造前的样子(封存之后的事件不在封存核对范围里)。"""
async with SessionLocal() as session:
for row in (await session.execute(select(RunEvent).where(
RunEvent.run_id == run_id, RunEvent.type == EventType.EVIDENCE_JUDGED))).scalars():
data = json.loads(json.dumps(row.data))
if uid in data.get("verdicts", {}):
verdict = {k: v for k, v in data["verdicts"][uid].items() if k != "rule_version"}
data["verdicts"][uid] = {**verdict, **changes}
row.data = data
await session.commit()
async def segment_of(client, run_id: str, doc: dict, uid: str) -> dict:
seg = next(s["id"] for _, u in iter_units(doc) if u["id"] == uid for s in u["segments"])
res = await client.get(f"/api/runs/{run_id}/evidence/segments/{seg}")
assert res.status_code == 200, res.text
return res.json()["unit"]
async def test_an_outdated_verdict_is_really_judged_again(client, monkeypatch):
models = Models(monkeypatch, verdicts={"东区销售额": ("contradicted", "")})
row, doc = await explore_run()
uid = unit_of(doc, "东区销售额")
first = await ask(client, row.id, [uid])
assert first["judged"] == [uid] and len(models.judges) == 1
assert first["verdicts"][uid]["rule_version"] == judge_mod.RULES_VERSION
# 当前规则下判过的:照旧复用,不再调用
again = await ask(client, row.id, [uid])
assert again["reused"] == [uid] and again["judged"] == [] and len(models.judges) == 1
assert (await segment_of(client, row.id, doc, uid))["on_demand"]["reason"] == "judged"
# 改造前判的(旧取值、没有规则版本):按钮照常出现,POST 真的重判一次,新判定追加在封存之后
await age(row.id, uid, status="unsupported")
unit = await segment_of(client, row.id, doc, uid)
assert unit["verdict"]["status"] == "unsupported"
assert unit["on_demand"]["available"] is True and unit["on_demand"]["reason"] == "outdated"
redo = await ask(client, row.id, [uid])
assert len(models.judges) == 2 and redo["judged"] == [uid] and redo["reused"] == []
assert redo["verdicts"][uid]["status"] == "contradicted" and redo["event"] is not None
assert len(await judged_events(row.id)) == 2
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
latest = graph["reports"][0]["post_seal_verdicts"][uid]
assert latest["status"] == "contradicted" and latest["rule_version"] == judge_mod.RULES_VERSION
assert (await segment_of(client, row.id, doc, uid))["on_demand"]["reason"] == "judged"
async def test_a_failed_rejudge_keeps_the_older_verdict_in_force(client, monkeypatch):
"""重判没判成(模型不可用):旧判定照旧生效,答复和证据图说的一样;按钮还在,可以再点。"""
models = Models(monkeypatch)
row, doc = await explore_run()
uid = unit_of(doc, "东区销售额")
await ask(client, row.id, [uid])
await age(row.id, uid)
async def broken(session, spec):
raise RuntimeError("网关 502")
monkeypatch.setattr(judge_mod, "get_chat_model", broken)
body = await ask(client, row.id, [uid])
assert body["judged"] == [] and "网关 502" in body["message"]
assert body["verdicts"][uid]["status"] == "supported" and "rule_version" not in body["verdicts"][uid]
graph = (await client.get(f"/api/runs/{row.id}/evidence")).json()
assert graph["reports"][0]["post_seal_verdicts"][uid] == body["verdicts"][uid]
assert (await segment_of(client, row.id, doc, uid))["on_demand"]["reason"] == "outdated"
assert len(models.judges) == 1
+455
View File
@@ -0,0 +1,455 @@
"""结论句裁判改造(JR)的内核:判定拆档、补全证据摘录、理由不硬截断。
- 判定拆成 contradicted(证据相矛盾)和 insufficient(证据不足):只有证据和句子冲突才判 contradicted,
证据里没有相关信息判 insufficient,并写明缺的是什么(missing)。新判的不再产出 unsupported:
模型照旧写 unsupported 的,按更严的 contradicted 收下
- 讲方法、讲结构的句子只挂了表名、字段名,裁判以前只拿到名字:表附上字段清单和类型(最多 60 个,
快照不完整要注明),字段附上类型和所属的表,口径卡指标附上输入取自哪次查询、哪一格和那次查询的 SQL。
一律只经封存范围内的 loader 读工件,遮罩的列不出现
- 理由上限 120 个字,超长时在句子边界截断加「…」;裁判规则要求 80 个字以内
所有模型调用都是 mock 剧本,不碰真接口。
"""
from __future__ import annotations
import json
import re
import pytest
from langchain_core.messages import AIMessage
from sqlalchemy import delete
from app.db.base import SessionLocal
from app.db.models import DataSource, Setting
from app.engine import judge as judge_mod
from app.engine.evidence import build_catalog, compose_doc, iter_units
from app.engine.judge import (
JUDGE_ROLE,
JUDGE_SCHEMA,
JUDGED,
SPEND_KEY,
SQL_CHARS,
Budget,
judge_doc,
prepare,
source_masks,
summarize,
)
from app.providers.factory import ModelSpec
from app.providers.mock_model import MockChatModel
PRICED = "claude-sonnet-5"
UNLIMITED = Budget(max_claims=None, max_cost_usd=None, timeout_s=None, daily_max_usd=None)
SPEC = ModelSpec(model=PRICED)
SOURCE = "warehouse"
SCHEMA_ART, Q1_ART, Q2_ART, CARD_ART = "5" * 64, "a" * 64, "b" * 64, "c" * 64
SQL1 = "SELECT COUNT(DISTINCT member_no) AS active_users FROM orders WHERE created_at >= '2026-09-01'"
SQL2 = "SELECT region, SUM(amount) AS gmv FROM orders GROUP BY region"
ORDER_COLS = [("id", "INTEGER"), ("member_no", "TEXT"), ("region", "TEXT"), ("amount", "REAL"),
("phone", "TEXT"), ("created_at", "TEXT")]
def schema_snapshot(columns=ORDER_COLS, *, truncated=False) -> dict:
return {"source": SOURCE, "synced_at": "2026-09-01T00:00:00+00:00", "schema": None,
"tables": {"orders": {"columns": [{"name": n, "type": t, "nullable": True} for n, t in columns],
"primary_key": ["id"], "comment": None, "is_view": False},
"refunds": {"columns": [{"name": "id", "type": "INTEGER"}, {"name": "amount", "type": "REAL"}],
"primary_key": ["id"], "is_view": False}},
**({"truncated": True, "total": 500} if truncated else {})}
def query_entry(art: str, columns: list[str], tables: list[str]) -> dict:
return {"kind": "query", "node_id": "fetch", "exec": 1, "artifact": art, "tool": f"db_query__{SOURCE}",
"source": SOURCE, "columns": columns, "rows": 1, "truncated": False, "schema_artifact": SCHEMA_ART,
"tables": tables}
def schema_entry(snapshot: dict) -> dict:
tables = {name: [c["name"] for c in meta["columns"]] for name, meta in snapshot["tables"].items()}
return {"kind": "schema", "node_id": "fetch", "exec": 1, "artifact": SCHEMA_ART, "source": SOURCE,
"tables": tables, **({"truncated": True, "total": 500} if snapshot.get("truncated") else {})}
def snaps(snapshot=None, **extra) -> dict:
out = {SCHEMA_ART: snapshot or schema_snapshot(),
Q1_ART: {"columns": ["active_users"], "rows": [[321]], "sql": SQL1, "source": SOURCE},
Q2_ART: {"columns": ["region", "gmv"], "rows": [["华东", 100.0]], "sql": SQL2, "source": SOURCE}}
out.update(extra)
return out
def entity_catalog(snapshot=None) -> dict:
snapshot = snapshot or schema_snapshot()
return build_catalog(nodes={}, ledger=[query_entry(Q1_ART, ["active_users"], ["orders"]),
query_entry(Q2_ART, ["region", "gmv"], ["orders"]), schema_entry(snapshot)])
def excerpts(text: str, *, snapshot=None, loader=None, masked=None) -> dict[str, str]:
cat = entity_catalog(snapshot)
doc = compose_doc(text, cat)
store = snaps(snapshot)
return prepare(doc, cat, loader=loader or store.get, masked=masked).excerpts
def field_line(excerpt: str) -> str:
[line] = [x for x in excerpt.splitlines() if x.startswith("字段清单")]
return line
# --------------------------------------------------------------------------
# 判定拆档:取值、规则、schema
# --------------------------------------------------------------------------
def test_the_schema_offers_contradicted_and_insufficient_instead_of_unsupported():
item = JUDGE_SCHEMA["properties"]["items"]["items"]
assert item["properties"]["verdict"]["enum"] == list(JUDGED) == [
"supported", "partial", "contradicted", "insufficient", "not_a_claim"]
assert "unsupported" not in JUDGED
assert "missing" in item["properties"] and "missing" in item["required"]
assert "80" in item["properties"]["rationale"]["description"]
def test_the_rules_separate_conflict_from_missing_information():
rules = judge_mod.JUDGE_RULES
assert "unsupported" not in rules, "新判的一律用新取值"
assert "contradicted" in rules and "insufficient" in rules and "missing" in rules
# 只有冲突才判 contradicted;没有相关信息判 insufficient,并写明缺什么
[conflict] = [line for line in rules.splitlines() if line.startswith("- contradicted")]
assert "冲突" in conflict
[lacking] = [line for line in rules.splitlines() if line.startswith("- insufficient")]
assert "没有" in lacking
assert "80 个字" in rules and "60 个字" not in rules
assert "表结构快照不完整" in rules, "快照不完整时「某字段不存在」只能判 insufficient"
assert "数字已经由系统核对过,不要复算" in rules, "旧规则保留"
def test_count_keys_keep_unsupported_for_old_data():
assert set(judge_mod.COUNT_KEYS) == {"supported", "partial", "contradicted", "insufficient", "unsupported", "not_a_claim",
"unjudged"}
# --------------------------------------------------------------------------
# 判定拆档:调用与收下
# --------------------------------------------------------------------------
RAW = ("活跃用户数由 [[t:orders]] 按 [[c:orders.member_no]] 去重计数得出。"
"各区销售额按 [[c:orders.region]] 汇总。[[see:Q2]]"
"华东的销售额是全国最高的,领先第二名一倍。[[see:Q2]]"
"订单表里没有退款字段。[[see:t:orders]]")
class Judge:
"""裁判剧本:items(units, request) 给出每句的条目;calls 记下每次交给裁判的请求原文。"""
def __init__(self, monkeypatch, items, *, model_id=PRICED):
self.calls: list[str] = []
def decide(model, messages):
assert JUDGE_ROLE in str(messages[0].content)
request = str(messages[-1].content)
self.calls.append("\n".join(str(m.content) for m in messages))
units = re.findall(r"^\[(u\d+)\]", request, re.M)
return AIMessage(content=json.dumps({"items": items(units)}, ensure_ascii=False))
monkeypatch.setattr(MockChatModel, "_decide", decide)
async def fake_model(session, spec):
return MockChatModel(model_name=model_id), model_id
monkeypatch.setattr(judge_mod, "get_chat_model", fake_model)
def ids_of(doc) -> dict[str, str]:
md = doc["markdown"]
return {md[u["span"][0]:u["span"][1]].strip(): u["id"] for _, u in iter_units(doc)}
@pytest.fixture(autouse=True)
async def clean_settings():
async with SessionLocal() as session:
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"])))
await session.commit()
yield
async def test_contradicted_and_insufficient_come_back_with_missing(monkeypatch):
cat = entity_catalog()
doc = compose_doc(RAW, cat)
ids = ids_of(doc)
method, grouped, ranked, absent = (ids[t] for t in list(ids)[:4])
script = {
method: {"verdict": "supported", "rationale": "Q1 的 SQL 写的是 COUNT(DISTINCT member_no)", "missing": ""},
grouped: {"verdict": "insufficient", "rationale": "摘录里没有这次汇总的 SQL", "missing": "SQL"},
ranked: {"verdict": "contradicted", "rationale": "Q2 里华东不是最高", "missing": "应当被丢掉"},
absent: {"verdict": "insufficient", "rationale": "", "missing": ""},
}
Judge(monkeypatch, lambda units: [{"unit": u, "used": [], **script[u]} for u in units])
outcome = await judge_doc(doc, cat, spec=SPEC, budget=UNLIMITED, loader=snaps().get)
v = outcome["verdicts"]
assert v[grouped]["status"] == "insufficient" and v[grouped]["missing"] == "SQL"
assert v[ranked]["status"] == "contradicted" and "missing" not in v[ranked], "只有 insufficient 带 missing"
assert v[absent]["status"] == "insufficient" and "missing" not in v[absent], "没写缺什么就不编"
assert v[method]["status"] == "supported" and "missing" not in v[method]
assert set(v[grouped]) == {"status", "rationale", "judge", "post_seal", "used", "missing", "rule_version"}
assert outcome["counts"] == {"supported": 1, "partial": 0, "contradicted": 1, "insufficient": 2,
"not_a_claim": 0, "unjudged": 0, "unsupported": 0}
async def test_a_model_still_saying_unsupported_is_taken_as_contradicted(monkeypatch):
"""新判的不再产出 unsupported:模型照旧写 unsupported 的按更严的 contradicted 收下,不当格式错误放过。"""
cat = entity_catalog()
doc = compose_doc(RAW, cat)
Judge(monkeypatch, lambda units: [{"unit": u, "verdict": "unsupported", "rationale": "对不上", "used": []}
for u in units])
outcome = await judge_doc(doc, cat, spec=SPEC, budget=UNLIMITED, loader=snaps().get)
assert {x["status"] for x in outcome["verdicts"].values()} == {"contradicted"}
assert outcome["counts"]["unsupported"] == 0 and outcome["counts"]["contradicted"] == 4
def test_summary_counts_always_carry_every_key():
"""缓存里的旧判定只有旧的五个键:摘要照样补齐 contradicted、insufficient。"""
outcome = {"counts": {"supported": 1, "partial": 0, "unsupported": 1, "not_a_claim": 0, "unjudged": 0},
"candidates": 2, "screened": [], "unjudged": {}, "limits_hit": [], "gaps": []}
summary = summarize(outcome, mode="inline", writer_model="writer-x")
assert summary["counts"] == {"supported": 1, "partial": 0, "contradicted": 0, "insufficient": 0,
"not_a_claim": 0, "unjudged": 0, "unsupported": 1}
assert summary["writer_model"] == "writer-x"
# --------------------------------------------------------------------------
# 理由不硬截断
# --------------------------------------------------------------------------
def test_a_rationale_under_the_cap_is_kept_whole():
text = "摘录里" + "有对应的指标和查询" * 10 + "。" # 94 个字:旧规则会截在 60 字的半句中间
assert 60 < len(text) <= judge_mod.RATIONALE_MAX == 120
assert judge_mod.clip_text(text) == text
def test_a_long_rationale_is_cut_at_a_sentence_boundary():
first = "Q2 的 SQL 按 region 汇总,华东 100 元排第一"
text = first + ";" + "第二名的数字摘录里没有给出," * 12
got = judge_mod.clip_text(text)
assert len(got) <= judge_mod.RATIONALE_MAX and got.endswith("…")
body = got[:-1]
assert text.startswith(body) and text[len(body)] in "。;,", f"截在句子边界:{got}"
def test_a_long_run_without_punctuation_is_not_cut_inside_a_word():
text = " ".join(["evidence"] * 30)
got = judge_mod.clip_text(text)
assert len(got) <= judge_mod.RATIONALE_MAX and got.endswith("…")
assert got[:-1].rstrip().split(" ")[-1] == "evidence", got
async def test_the_judge_keeps_a_long_rationale_up_to_the_cap(monkeypatch):
cat = entity_catalog()
doc = compose_doc("各区销售额按 [[c:orders.region]] 汇总。[[see:Q2]]", cat)
long = "Q2 的 SQL 是按 region 分组求和," * 3 + "句子说的汇总方式和 SQL 一致," * 8
Judge(monkeypatch, lambda units: [{"unit": u, "verdict": "supported", "rationale": long, "used": []}
for u in units])
outcome = await judge_doc(doc, cat, spec=SPEC, budget=UNLIMITED, loader=snaps().get)
[v] = outcome["verdicts"].values()
assert 60 < len(v["rationale"]) <= judge_mod.RATIONALE_MAX and v["rationale"].endswith("…")
assert long.startswith(v["rationale"][:-1])
# --------------------------------------------------------------------------
# 补全证据摘录:表
# --------------------------------------------------------------------------
def test_a_table_excerpt_lists_its_fields_and_types():
ex = excerpts("活跃用户数由 [[t:orders]] 去重计数得出。")
table = ex["t:orders"]
line = field_line(table)
for name, kind in ORDER_COLS:
assert f"{name} {kind}" in line, line
assert "COUNT(DISTINCT member_no)" in table, "出现过的查询 SQL 保留"
assert "表结构快照不完整" not in table
def test_the_field_list_stops_at_sixty():
columns = [(f"col_{i:02d}", "TEXT") for i in range(75)]
ex = excerpts("活跃用户数由 [[t:orders]] 去重计数得出。", snapshot=schema_snapshot(columns))
line = field_line(ex["t:orders"])
listed = re.findall(r"col_\d\d", line)
assert judge_mod.FIELDS_MAX == 60 and listed == [f"col_{i:02d}" for i in range(60)]
assert "另有 15 个" in line
def test_masked_fields_are_never_listed():
ex = excerpts("活跃用户数由 [[t:orders]] 去重计数得出。", masked={SOURCE: ["Phone"]})
table = ex["t:orders"]
assert "phone" not in field_line(table).lower() and "member_no TEXT" in field_line(table)
assert "遮罩" in table, "说明有字段按遮罩没列出:裁判不能据此说表里没有这个字段"
def test_masks_recorded_in_a_query_snapshot_also_hide_fields():
"""数据源改名、删掉了,现在查不到遮罩:查询当时记下的遮罩照样生效(证据面板同一个口径)。"""
cat = entity_catalog()
doc = compose_doc("活跃用户数由 [[t:orders]] 去重计数得出。", cat)
store = snaps(**{Q1_ART: {"columns": ["active_users"], "rows": [[321]], "sql": SQL1, "source": SOURCE,
"mask_columns": ["phone"]}})
table = prepare(doc, cat, loader=store.get).excerpts["t:orders"]
assert "phone" not in field_line(table).lower()
def test_a_truncated_snapshot_is_called_out():
ex = excerpts("活跃用户数由 [[t:orders]] 去重计数得出。", snapshot=schema_snapshot(truncated=True))
table = ex["t:orders"]
assert "表结构快照不完整" in table and "id INTEGER" in field_line(table)
def test_only_the_sealed_loader_feeds_the_field_list():
"""schema 快照不在封存范围里(loader 取不到):一个字段都不给,照实说没有字段清单。"""
store = snaps()
ex = excerpts("活跃用户数由 [[t:orders]] 去重计数得出。", loader=lambda a: None if a == SCHEMA_ART else store.get(a))
table = ex["t:orders"]
assert "INTEGER" not in table and "member_no TEXT" not in table
assert "没有" in field_line(table)
def test_new_excerpt_content_counts_toward_the_estimate():
cat = entity_catalog()
doc = compose_doc("活跃用户数由 [[t:orders]] 去重计数得出。", cat)
store = snaps()
full = prepare(doc, cat, loader=store.get)
bare = prepare(doc, cat, loader=lambda a: None if a == SCHEMA_ART else store.get(a))
assert full.cost(PRICED, full.cands) > bare.cost(PRICED, bare.cands)
# --------------------------------------------------------------------------
# 补全证据摘录:字段
# --------------------------------------------------------------------------
def test_a_column_excerpt_carries_its_type_and_table():
column = excerpts("活跃用户数按 [[c:orders.member_no]] 去重计数得出。")["c:orders.member_no"]
assert "类型:TEXT" in column and "所属的表:orders" in column
assert "COUNT(DISTINCT member_no)" in column, "出现过的查询 SQL 保留"
def test_a_result_column_takes_its_type_from_the_query_snapshot():
"""聚合的别名(gmv)不在表结构里:类型取查询快照记下的列类型。"""
cat = entity_catalog()
doc = compose_doc("汇总在 [[c:gmv]] 这一列。[[see:Q2]]", cat)
store = snaps(**{Q2_ART: {"columns": ["region", "gmv"], "rows": [["华东", 100.0]], "sql": SQL2, "source": SOURCE,
"column_types": {"region": "text", "gmv": "number"}}})
column = prepare(doc, cat, loader=store.get).excerpts["c:gmv"]
assert "类型:number" in column
# --------------------------------------------------------------------------
# 补全证据摘录:口径卡指标
# --------------------------------------------------------------------------
LONG_SQL = "SELECT COUNT(DISTINCT member_no) AS active_users FROM orders WHERE " + " AND ".join(
f"flag_{i} = 1" for i in range(80))
CARD = {"kind": "metric_set", "caliber": "周报口径", "caliber_version": "v2", "metrics": [
{"id": "active", "name": "活跃用户数", "unit": "人", "value": 321, "decimals": 0, "format": "plain", "status": "ok",
"expression": "cell(nodes.fetch, 0, 'active_users')", "rendered": "321人",
"inputs": [{"path": "cell(nodes.fetch, 0, 'active_users')", "value": 321, "node_id": None, "via": "tool_cell",
"artifact": Q1_ART, "locator": {"row": 0, "column": "active_users"}, "status": "ok"},
{"path": "vars.target", "value": 300, "node_id": "plan", "via": "transform", "status": "ok"}]}]}
def metric_catalog():
ledger = [{"kind": "metric_set", "node_id": "caliber", "exec": 1, "artifact": CARD_ART, "caliber": "周报口径",
"version": "v2", "metrics": ["active"]}, query_entry(Q1_ART, ["active_users"], ["orders"])]
return build_catalog(nodes={"caliber": {**CARD, "artifact": CARD_ART}}, ledger=ledger)
def test_a_metric_excerpt_names_the_query_and_cell_of_its_input_and_that_sql():
cat = metric_catalog()
doc = compose_doc("活跃用户数 [[m:active]],按会员号去重计数。[[see:m:active]]", cat)
store = {**snaps(), CARD_ART: CARD}
metric = prepare(doc, cat, loader=store.get).excerpts["m:active"]
assert "原式:cell(nodes.fetch, 0, 'active_users')" in metric
[where] = [line for line in metric.splitlines() if "active_users" in line and "Q1" in line and "SQL" not in line]
assert "r0" in where, where
[sql] = [line for line in metric.splitlines() if line.startswith("查询 Q1 的 SQL")]
assert "COUNT(DISTINCT member_no)" in sql
assert any("vars.target" in line and "未追溯到查询" in line for line in metric.splitlines())
def test_metric_input_sql_is_capped_and_needs_the_sealed_loader():
cat = metric_catalog()
doc = compose_doc("活跃用户数 [[m:active]],按会员号去重计数。[[see:m:active]]", cat)
long = {**snaps(**{Q1_ART: {"columns": ["active_users"], "rows": [[321]], "sql": LONG_SQL, "source": SOURCE}}),
CARD_ART: CARD}
metric = prepare(doc, cat, loader=long.get).excerpts["m:active"]
[sql] = [line for line in metric.splitlines() if line.startswith("查询 Q1 的 SQL")]
assert LONG_SQL[:SQL_CHARS] in sql and LONG_SQL[:SQL_CHARS + 10] not in sql and sql.endswith("…")
sealed_only = {CARD_ART: CARD} # 查询快照不在封存范围里
metric = prepare(doc, cat, loader=sealed_only.get).excerpts["m:active"]
assert "SELECT" not in metric and "Q1" in metric, "位置照给,SQL 不给"
def test_a_masked_input_column_is_not_named():
cat = metric_catalog()
doc = compose_doc("活跃用户数 [[m:active]],按会员号去重计数。[[see:m:active]]", cat)
store = {**snaps(), CARD_ART: CARD}
metric = prepare(doc, cat, loader=store.get, masked={SOURCE: ["active_users"]}).excerpts["m:active"]
[where] = [line for line in metric.splitlines() if line.startswith("输入 cell(")]
assert "r0" in where and "active_users" not in where.split(":", 1)[1]
# --------------------------------------------------------------------------
# 遮罩:表、字段的数据源也要查
# --------------------------------------------------------------------------
async def test_source_masks_cover_the_sources_of_tables():
name = "jr_mask_src"
cat = build_catalog(nodes={}, ledger=[{**schema_entry(schema_snapshot()), "source": name}])
assert cat and all(e["kind"] in ("table", "column") for e in cat.values())
async with SessionLocal() as session:
session.add(DataSource(name=name, kind="sqlite", database="unused.db", options={"mask_columns": "phone"}))
await session.commit()
try:
assert await source_masks(cat) == {name: ["phone"]}
finally:
async with SessionLocal() as session:
await session.execute(delete(DataSource).where(DataSource.name == name))
await session.commit()
# --------------------------------------------------------------------------
# 断点里的旧判定:规则换了就不复用
# --------------------------------------------------------------------------
def test_the_request_key_carries_the_rules_version(monkeypatch):
"""报告节点按 request_key 认断点里缓存的判定:规则、取值换了(旧的 unsupported),旧判定不能原样复用。"""
cat = entity_catalog()
doc = compose_doc(RAW, cat)
request = prepare(doc, cat, loader=snaps().get)
before = judge_mod.request_key(request, SPEC, UNLIMITED)
assert before == judge_mod.request_key(request, SPEC, UNLIMITED), "同一份输入同一个指纹"
monkeypatch.setattr(judge_mod, "RULES_VERSION", judge_mod.RULES_VERSION + 1)
assert judge_mod.request_key(request, SPEC, UNLIMITED) != before
async def test_a_new_verdict_carries_the_rules_version(monkeypatch):
"""新判的判定带上规则版本(和 request_key 里的同一个常量):按需裁判据此认出早于当前规则的旧判定。"""
cat = entity_catalog()
doc = compose_doc("各区销售额按 [[c:orders.region]] 汇总。[[see:Q2]]", cat)
Judge(monkeypatch, lambda units: [{"unit": u, "verdict": "supported", "rationale": "有依据", "used": []}
for u in units])
outcome = await judge_doc(doc, cat, spec=SPEC, budget=UNLIMITED, loader=snaps().get)
[v] = outcome["verdicts"].values()
assert v["rule_version"] == judge_mod.RULES_VERSION
def test_which_verdicts_are_older_than_the_current_rules():
current = judge_mod.RULES_VERSION
assert judge_mod.outdated({"status": "supported"}), "没有规则版本:改造前判的"
assert judge_mod.outdated({"status": "unsupported", "rule_version": current}), "旧取值一律算旧"
assert judge_mod.outdated({"status": "partial", "rule_version": current - 1})
assert not judge_mod.outdated({"status": "contradicted", "rule_version": current})
assert not judge_mod.outdated({"status": "unjudged", "reason": "error"}), "没判成的谈不上新旧"
+298
View File
@@ -0,0 +1,298 @@
"""结论句裁判改造(JR)的判档:contradicted 沿用原来 unsupported 的处置,insufficient 按缺口处理。
- contradicted(证据相矛盾):按 on_unsupported 降档(degrade)或不予出具(withhold),和原来的 unsupported 一样
- insufficient(证据不足):按缺口处理,最多降档——不会不予出具,也不会判完整出具
- 不许比现在更松:受管正式运行里,原来会因「证据不支持」降档的句子,改判成 contradicted 或 insufficient
都不能完整出具
- 旧文档里的 unsupported 判档照旧当 contradicted 处理
- 统计键(stats、counts、出具横幅的结论句计数)都有 contradicted、insufficient,unsupported 保留
- 报告节点:改写一次把证据相矛盾、证据不足的句子都交回写作者;文档的裁判摘要记下写作模型
所有模型调用都是 mock 剧本,不碰真接口。
"""
from __future__ import annotations
import asyncio
import json
import re
import uuid
import pytest
from httpx import ASGITransport, AsyncClient
from langchain_core.messages import AIMessage
from sqlalchemy import delete, select
from app.core import artifact_store
from app.db.base import SessionLocal
from app.db.models import Run, RunEvent, Setting, Workflow, WorkflowVersion
from app.engine import judge as judge_mod
from app.engine.evidence import compose_doc, iter_units
from app.engine.issuance import decide_tier
from app.engine.judge import JUDGE_ROLE, SPEND_KEY
from app.engine.nodes.io import _claims
from app.engine.runner import run_manager
from app.main import app
from app.providers.mock_model import MockChatModel
JUDGE_MODEL = "claude-sonnet-5"
WRITER = "mock-fast"
METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}]
CITED = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。[[see:m:gmv]]下面看细节。"
SUPPORTED, CAUSAL = "本周销售额 45,678.5元。", "增长主要来自新客首单。"
# --------------------------------------------------------------------------
# 出口判档(io._claims):直接喂文档
# --------------------------------------------------------------------------
def judged_doc(statuses: dict[str, dict], *, on_unsupported="degrade") -> dict:
"""两句结论:SUPPORTED_TEXT、CAUSAL_TEXT;statuses 按句子给判定。counts 照判定数。"""
doc = compose_doc("本周销售额稳定增长。增长主要来自新客首单。", {})
md = doc["markdown"]
counts = {k: 0 for k in ("supported", "partial", "unsupported", "not_a_claim", "unjudged")}
for _, unit in iter_units(doc):
text = md[unit["span"][0]:unit["span"][1]].strip()
verdict = {"status": "supported", "rationale": "有依据", "judge": JUDGE_MODEL, "post_seal": False, "used": [],
**statuses.get(text, {})}
unit["verdict"] = verdict
counts[verdict["status"]] = counts.get(verdict["status"], 0) + 1
doc["judge"] = {"mode": "inline", "model": JUDGE_MODEL, "on_unsupported": on_unsupported, "counts": counts,
"unjudged": {}, "limits_hit": [], "complete": True, "gaps": []}
return doc
def grade(doc: dict, *, governed=False, contract=None) -> tuple[str, dict]:
out = {"gaps": [], "claims_policy": None, "uncited": None, "claims": None, "unsupported": None}
_claims(out, contract or {}, {"claims": "judge", "text": doc["markdown"]}, evidence_on=True, governed=governed,
uncited=[], doc=doc, title="write")
tier = decide_tier(missing_required=[], missing_expected=[], unmatched=[], strict=True, gaps=out["gaps"],
unsupported=out["unsupported"], uncited_claims=out["uncited"],
claims_policy=out["claims_policy"])
return tier, out
TEXT2 = "增长主要来自新客首单。"
def test_all_supported_is_still_a_full_issuance():
tier, out = grade(judged_doc({}))
assert tier == "formal" and out["gaps"] == []
@pytest.mark.parametrize(("on_unsupported", "tier"), [("degrade", "degraded"), ("withhold", "withheld")])
def test_contradicted_follows_on_unsupported(on_unsupported, tier):
doc = judged_doc({TEXT2: {"status": "contradicted", "rationale": "证据里新客占比下降"}},
on_unsupported=on_unsupported)
got, out = grade(doc)
assert got == tier
claims = out["claims"]
[item] = claims["contradicted"]
assert item["text"] == TEXT2 and item["rationale"] == "证据里新客占比下降"
assert claims["counts"]["contradicted"] == 1 and claims["counts"]["insufficient"] == 0
assert out["gaps"] == [], "判档来自判定,不来自缺口"
def test_insufficient_is_a_gap_and_never_withholds():
doc = judged_doc({TEXT2: {"status": "insufficient", "rationale": "摘录里没有新客维度", "missing": "新客维度的查询"}},
on_unsupported="withhold")
tier, out = grade(doc, governed=True, contract={"claims": {"policy": "judge", "on_unsupported": "withhold"}})
assert tier == "degraded", "证据不足最多降档:不能不予出具,也不能完整出具"
[gap] = [g for g in out["gaps"] if "证据不足" in g]
assert gap.startswith("报告「write」") and "1 句" in gap and "新客维度的查询" in gap
[item] = out["claims"]["insufficient"]
assert item["text"] == TEXT2 and item["missing"] == "新客维度的查询"
assert out["claims"]["counts"]["insufficient"] == 1 and out["unsupported"] == []
def test_an_old_unsupported_verdict_is_graded_as_contradicted():
doc = judged_doc({TEXT2: {"status": "unsupported", "rationale": "证据不支持"}}, on_unsupported="withhold")
tier, out = grade(doc)
assert tier == "withheld"
assert out["claims"]["counts"]["unsupported"] == 1
assert [i["text"] for i in out["claims"]["unsupported"]] == [TEXT2]
assert [i["text"] for i in out["unsupported"]] == [TEXT2]
def test_the_issuance_counts_carry_every_key():
_, out = grade(judged_doc({}))
assert set(out["claims"]["counts"]) == set(judge_mod.COUNT_KEYS)
# --------------------------------------------------------------------------
# 端到端:正式运行、受管正式运行
# --------------------------------------------------------------------------
@pytest.fixture
async def engine_up():
await run_manager.setup()
async with SessionLocal() as session:
await session.execute(delete(Setting).where(Setting.key.in_([SPEND_KEY, "judge", "copilot"])))
await session.commit()
yield
await run_manager.shutdown()
@pytest.fixture
async def client(engine_up):
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def weekly(*, report=None, contract=None) -> dict:
nodes = [
node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]),
node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"),
node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS),
node("write", "report", instructions="写周报", **{"claims": "judge", "judge": {"max_cost_usd": 0.05},
**(report or {})}),
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}],
contract={"report_from": "write", "metrics_from": ["caliber"], "required": ["gmv"], "strict": True,
**(contract or {})}),
]
ids = [n["id"] for n in nodes]
return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]}
class Models:
"""写作者依次回 writer 里的几段;裁判按句子里的字给判定:verdicts = {句中片段: (判定, missing)}。"""
def __init__(self, monkeypatch, writer=(CITED,), verdicts=None):
self.writes: list[list] = []
self.judges: list[str] = []
verdicts = verdicts or {}
def decide(model, messages):
if JUDGE_ROLE in str(messages[0].content):
request = str(messages[-1].content)
self.judges.append(request)
items = []
for uid, line in re.findall(r"^\[(u\d+)\]([^\n]*)$", request, re.M):
status, missing = next((v for frag, v in verdicts.items() if frag in line), ("supported", ""))
items.append({"unit": uid, "verdict": status, "rationale": f"判的是:{line.strip()[:12]}",
"missing": missing, "used": []})
return AIMessage(content=json.dumps({"items": items}, ensure_ascii=False))
self.writes.append(list(messages))
return AIMessage(content=writer[min(len(self.writes), len(writer)) - 1])
monkeypatch.setattr(MockChatModel, "_decide", decide)
async def judge_model_of(session, spec):
return MockChatModel(model_name=JUDGE_MODEL), JUDGE_MODEL
monkeypatch.setattr(judge_mod, "get_chat_model", judge_model_of)
async def wait(run_id: str) -> Run:
for _ in range(400):
await asyncio.sleep(0.05)
async with SessionLocal() as session:
row = await session.get(Run, run_id)
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
return row
raise AssertionError(f"run {run_id} 没跑完:{row.status}")
async def run(graph, *, run_class="formal") -> Run:
started = await run_manager.start(graph=graph, input_payload={}, run_class=run_class)
row = await wait(started.id)
assert row.status == "succeeded", row.error
return row
async def checked_of(run_id: str) -> tuple[dict, dict]:
async with SessionLocal() as session:
[event] = (await session.execute(select(RunEvent).where(
RunEvent.run_id == run_id, RunEvent.type == "report.checked"))).scalars()
return event.data, artifact_store.load(event.data["doc_artifact"])
async def issuance_event(run_id: str) -> dict:
async with SessionLocal() as session:
[event] = (await session.execute(select(RunEvent).where(
RunEvent.run_id == run_id, RunEvent.type == "issuance"))).scalars()
return event.data
async def test_a_formal_run_withholds_a_contradicted_claim(engine_up, monkeypatch):
Models(monkeypatch, verdicts={"新客": ("contradicted", "")})
row = await run(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
issuance = row.output["_issuance"]
assert issuance["tier"] == "withheld"
assert [i["text"] for i in issuance["claims"]["contradicted"]] == [CAUSAL]
event = await issuance_event(row.id)
assert event["claims"]["counts"] == {"supported": 1, "partial": 0, "contradicted": 1, "insufficient": 0,
"not_a_claim": 0, "unjudged": 0, "unsupported": 0}
async def test_a_formal_run_only_degrades_an_insufficient_claim(engine_up, monkeypatch):
Models(monkeypatch, verdicts={"新客": ("insufficient", "新客维度的数据")})
row = await run(weekly(report={"judge": {"max_cost_usd": 0.05, "on_unsupported": "withhold"}}))
issuance = row.output["_issuance"]
assert issuance["tier"] == "degraded", "证据不足:最多降档,不因 withhold 不予出具"
assert any("证据不足" in g and "新客维度的数据" in g for g in issuance["gaps"]), issuance["gaps"]
[item] = issuance["claims"]["insufficient"]
assert item["text"] == CAUSAL and item["missing"] == "新客维度的数据"
assert issuance["claims"]["counts"]["insufficient"] == 1
checked, doc = await checked_of(row.id)
assert doc["stats"]["insufficient"] == 1 and doc["stats"]["contradicted"] == 0 and doc["stats"]["unsupported"] == 0
assert checked["stats"]["insufficient"] == 1
assert doc["judge"]["writer_model"] == WRITER and checked["judge"]["writer_model"] == WRITER
[unit] = [u for _, u in iter_units(doc) if u.get("verdict", {}).get("status") == "insufficient"]
assert unit["verdict"]["missing"] == "新客维度的数据"
async def governed(client, graph) -> Run:
async with SessionLocal() as session:
wf = Workflow(name=f"jr-{uuid.uuid4().hex[:6]}", graph=graph, status="governed", published_version=1)
session.add(wf)
await session.flush()
session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph))
await session.commit()
wf_id = wf.id
res = await client.post("/api/runs", json={"workflow_id": wf_id, "run_class": "formal"})
assert res.status_code == 201, res.text
row = await wait(res.json()["id"])
assert row.status == "succeeded", row.error
return row
@pytest.mark.parametrize("status", ["contradicted", "insufficient"])
async def test_a_governed_formal_run_is_never_fully_issued_on_a_split_verdict(client, monkeypatch, status):
"""不许比现在更松:原来判「证据不支持」要降档的句子,改判成哪一档都不能完整出具。"""
Models(monkeypatch, verdicts={"新客": (status, "新客维度的数据" if status == "insufficient" else "")})
graph = weekly(report={"numbers": "strict", "on_violation": "fail", "judge": {"max_cost_usd": None}})
row = await governed(client, graph)
assert row.output["_issuance"]["tier"] == "degraded"
async def test_rewrite_once_hands_back_contradicted_and_insufficient_sentences(engine_up, monkeypatch):
draft = ("本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单。[[see:m:gmv]]"
"华东贡献了大部分增长。[[see:m:gmv]]下面看细节。")
rewritten = "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客、华东的贡献要等分维度的数据再看。下面看细节。"
models = Models(monkeypatch, writer=(draft, rewritten),
verdicts={"新客首单": ("contradicted", ""), "华东贡献": ("insufficient", "分区域的查询"),
"要等分维度": ("not_a_claim", "")})
row = await run(weekly(report={"judge": {"max_cost_usd": 0.05, "rewrite_once": True}}))
assert len(models.writes) == 2
request = str(models.writes[1][-1].content)
assert "增长主要来自新客首单。" in request and "华东贡献了大部分增长。" in request
assert "分区域的查询" in request, "证据不足的句子连同缺的是什么一起交回"
assert SUPPORTED not in request
checked, _ = await checked_of(row.id)
assert checked["judge"]["rewrite"]["applied"] is True and len(checked["judge"]["rewrite"]["sentences"]) == 2
async def test_an_explore_report_records_the_writer_model_too(engine_up, monkeypatch):
models = Models(monkeypatch)
row = await run(weekly(), run_class="exploratory")
assert models.judges == []
checked, doc = await checked_of(row.id)
assert doc["judge"]["mode"] == "on_demand" and doc["judge"]["writer_model"] == WRITER
assert set(judge_mod.COUNT_KEYS) <= set(doc["stats"])
+14 -13
View File
@@ -26,7 +26,7 @@ from app.engine import judge as judge_mod
from app.engine import replay
from app.engine import runner as runner_mod
from app.engine.evidence import CLAIMS_RULE, iter_units
from app.engine.judge import JUDGE_ROLE, JUDGE_RULE, SPEND_KEY, SUMMARY_KEYS, daily_spend
from app.engine.judge import COUNT_KEYS, JUDGE_ROLE, JUDGE_RULE, SPEND_KEY, SUMMARY_KEYS, daily_spend
from app.engine.nodes import report as report_mod
from app.engine.runner import run_manager, verify_manifest
from app.providers.mock_model import MockChatModel
@@ -167,7 +167,7 @@ async def judge_ends(run_id: str) -> list[dict]:
async def test_a_formal_run_judges_inside_the_node_and_seals_the_verdicts(monkeypatch):
models = Models(monkeypatch, verdicts={"新客": "unsupported"})
models = Models(monkeypatch, verdicts={"新客": "contradicted"})
row = await finish(weekly(claims="judge", judge={"on_unsupported": "withhold"}))
assert row.status == "succeeded", row.error
assert len(models.writes) == 1 and len(models.judges) == 1
@@ -176,18 +176,19 @@ async def test_a_formal_run_judges_inside_the_node_and_seals_the_verdicts(monkey
got = verdicts(doc)
assert got[CITED]["status"] == "supported" and got[CITED]["judge"] == PRICED
assert got[CITED]["post_seal"] is False and got[CITED]["rationale"]
assert got[CAUSAL]["status"] == "unsupported"
assert got[CAUSAL]["status"] == "contradicted"
assert got[LINK] is None, "连接性的话不送裁判"
assert CITED in models.judges[0] and CAUSAL in models.judges[0] and LINK not in models.judges[0]
summary = checked["judge"]
assert tuple(summary) == SUMMARY_KEYS, "B 段照着这份键读:不能悄悄少一个、多一个"
assert summary["mode"] == "inline" and summary["model"] == PRICED and summary["on_unsupported"] == "withhold"
assert summary["counts"] == {"supported": 1, "partial": 0, "unsupported": 1, "not_a_claim": 0, "unjudged": 0}
assert summary["counts"] == {"supported": 1, "partial": 0, "contradicted": 1, "insufficient": 0, "not_a_claim": 0,
"unjudged": 0, "unsupported": 0}
assert summary["complete"] is True and summary["gaps"] == []
assert checked["claims"] == output["claims"] == "judge" and output["judge"] == summary
assert doc["judge"]["mode"] == "inline" and doc["judge"]["counts"] == summary["counts"]
assert {k: doc["stats"][k] for k in ("supported", "partial", "unsupported", "not_a_claim", "unjudged")} \
assert {k: doc["stats"][k] for k in COUNT_KEYS} \
== summary["counts"]
[end] = await judge_ends(row.id)
@@ -277,7 +278,7 @@ REWRITTEN = "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客首单的贡献要
async def test_rewrite_once_hands_back_only_the_unsupported_sentences(monkeypatch):
models = Models(monkeypatch, writer=(TEXT, REWRITTEN),
verdicts={"主要来自新客": "unsupported", "要等新客维度": "not_a_claim"})
verdicts={"主要来自新客": "contradicted", "要等新客维度": "not_a_claim"})
row = await finish(weekly(claims="judge", judge={"rewrite_once": True}))
assert row.status == "succeeded", row.error
assert len(models.writes) == 2 and len(models.judges) == 2
@@ -305,23 +306,23 @@ async def test_rewrite_once_hands_back_only_the_unsupported_sentences(monkeypatc
async def test_rewrite_once_is_off_by_default(monkeypatch):
models = Models(monkeypatch, writer=(TEXT, REWRITTEN), verdicts={"主要来自新客": "unsupported"})
models = Models(monkeypatch, writer=(TEXT, REWRITTEN), verdicts={"主要来自新客": "contradicted"})
row = await finish(weekly(claims="judge"))
assert row.status == "succeeded", row.error
assert len(models.writes) == 1 and len(models.judges) == 1
checked, doc, _ = await report_of(row.id)
assert verdicts(doc)[CAUSAL]["status"] == "unsupported" and checked["judge"].get("rewrite") is None
assert verdicts(doc)[CAUSAL]["status"] == "contradicted" and checked["judge"].get("rewrite") is None
async def test_a_rewrite_that_breaks_the_numbers_is_not_used(monkeypatch):
models = Models(monkeypatch, writer=(TEXT, "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客贡献了 45678 元。下面看细节。"),
verdicts={"主要来自新客": "unsupported"})
verdicts={"主要来自新客": "contradicted"})
row = await finish(weekly(claims="judge", judge={"rewrite_once": True}))
assert row.status == "succeeded", row.error
assert len(models.writes) == 2 and len(models.judges) == 1, "改坏了的那一稿不再花钱判"
checked, doc, output = await report_of(row.id)
assert CAUSAL in doc["markdown"] and "45678" not in doc["markdown"]
assert verdicts(doc)[CAUSAL]["status"] == "unsupported"
assert verdicts(doc)[CAUSAL]["status"] == "contradicted"
rewrite = checked["judge"]["rewrite"]
assert rewrite["applied"] is False and "45678" in rewrite["reason"]
# 封存的是原稿:units 就是原稿(也就是封存文档)里的编号
@@ -346,7 +347,7 @@ async def test_a_rewrite_that_makes_up_a_name_is_not_used(monkeypatch):
with_schema(monkeypatch)
models = Models(monkeypatch, writer=(TEXT, "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客首单的贡献要看 `refund_detail` 表。"
"下面看细节。"),
verdicts={"主要来自新客": "unsupported"})
verdicts={"主要来自新客": "contradicted"})
row = await finish(weekly(claims="judge", judge={"rewrite_once": True}))
assert row.status == "succeeded", row.error
assert len(models.writes) == 2 and len(models.judges) == 1
@@ -362,7 +363,7 @@ async def test_a_rewrite_that_trades_one_violation_for_another_is_not_used(monke
"""去掉一处违规、又冒出另一处:总数没变,可冒出来的是原稿没有的,一样不采用。"""
draft = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客首单,共 1200 单。下面看细节。"
swapped = "本周销售额 [[m:gmv]]。[[see:m:gmv]]新客首单约占 3500 单。下面看细节。"
models = Models(monkeypatch, writer=(draft, draft, swapped), verdicts={"主要来自新客": "unsupported"})
models = Models(monkeypatch, writer=(draft, draft, swapped), verdicts={"主要来自新客": "contradicted"})
row = await finish(weekly(claims="judge", on_violation="flag", max_repairs=1, judge={"rewrite_once": True}))
assert row.status == "succeeded", row.error
assert len(models.writes) == 3 and len(models.judges) == 1
@@ -385,7 +386,7 @@ def test_new_violations_are_counted_one_by_one():
async def test_a_second_round_that_hits_a_limit_names_the_configured_limit(monkeypatch):
"""第二轮用的是第一轮剩下的钱,可说给人看的上限是节点上配的那个数,不是剩下的零头。"""
Models(monkeypatch, writer=(TEXT, REWRITTEN), verdicts={"主要来自新客": "unsupported"})
Models(monkeypatch, writer=(TEXT, REWRITTEN), verdicts={"主要来自新客": "contradicted"})
# 每句估 1 分钱、回复不带用量(实际就按估算记):第一轮两句花掉 2 分,第二轮只剩半分,一句也装不下
monkeypatch.setattr(judge_mod.JudgeRequest, "cost", lambda self, model_id, units: 0.01 * len(units))
monkeypatch.setattr(judge_mod, "_usage_of", lambda message, model_id: {"input_tokens": 0, "output_tokens": 0})
+5 -4
View File
@@ -12,7 +12,7 @@ import { datasourceTools, modelOptions, providerOfModel, useCatalog, useDatasour
import { api } from '../api/client'
import { IconButton, JsonInput, Modal, isComposing, useRadioGroup } from '../components/ui'
import { formatShortcut } from '../lib/keys'
import { JUDGE_FIELD_LABEL, JUDGE_ON_UNSUPPORTED_LABEL, JUDGE_SETTING_TEXT, SUBGRAPH_UPGRADE_HELP, judgeByModelText, upgradeNewerText } from '../lib/terms'
import { JUDGE_FIELD_LABEL, JUDGE_ON_UNSUPPORTED_HINT, JUDGE_ON_UNSUPPORTED_LABEL, JUDGE_SETTING_TEXT, SUBGRAPH_UPGRADE_HELP, judgeByModelText, upgradeNewerText } from '../lib/terms'
import { TemplateText } from './TemplateText'
import type { NodeType, ValidationIssue, WorkflowVersion } from '../types'
@@ -624,7 +624,8 @@ function FieldInput({ field, id, nodeId, syntax, value, config, invalid, describ
const JUDGE_SUB_KEYS = new Set(['provider', 'model', 'max_claims', 'max_cost_usd', 'timeout_s', 'rewrite_once', 'on_unsupported'])
/**
* 结论句裁判(claims: judge)的子配置:裁判模型、三项上限(各自可以设成不限)、改写一次、证据不支持时怎么判档。
* 结论句裁判(claims: judge)的子配置:裁判模型、三项上限(各自可以设成不限)、改写一次、证据相矛盾时怎么判档
* (键名仍是 on_unsupported;证据不足的句子最多降档)。
*
* 上限有三种写法,得分得清:没写(跟随设置里「证据裁判」的默认值)、写数、写 null(不限)。勾「不限」写 null,
* 清空数字框回到没写。全都没写时整个 judge 去掉,不留一个空对象。受管级别发布要求写了 max_cost_usd(数或 null)
@@ -705,7 +706,7 @@ function JudgeConfig({ id, value, config, issues, onChange }: {
<span>
{JUDGE_FIELD_LABEL.rewrite_once}
<span className="mt-0.5 block text-2xs leading-snug text-faint">
默认关闭。开启后,裁判判定为证据不支持的句子会连同理由交回撰写模型,仅改写这些句子并重新判断一次,最多一轮。
默认关闭。开启后,裁判判定为证据相矛盾或证据不足的句子会连同理由交回撰写模型,仅改写这些句子并重新判断一次,最多一轮。
会额外产生一次撰写和裁判的费用;改写稿如出现原稿没有的问题,则不予采用
</span>
</span>
@@ -724,7 +725,7 @@ function JudgeConfig({ id, value, config, issues, onChange }: {
<option value={judge.on_unsupported} disabled>{judge.on_unsupported}:无法识别的值,请从上方选项中选择</option>
)}
</select>
<div className="mt-1 text-2xs leading-snug text-faint">仅对正式运行生效;探索运行只做标注,不拦截</div>
<div className="mt-1 text-2xs leading-snug text-faint" data-judge-hint="on_unsupported">{JUDGE_ON_UNSUPPORTED_HINT}</div>
{lines('on_unsupported')}
</div>
</div>
+6 -5
View File
@@ -13,7 +13,7 @@ import { edgeKey, topology } from '../run/derive'
import { exhaustedOf } from '../run/decode'
import { api } from '../api/client'
import { isComposing, StatusBadge, toast } from '../components/ui'
import { matchedSource } from '../lib/evidence'
import { claimCountsOf, matchedSource } from '../lib/evidence'
import { explainRunError } from '../lib/explain'
import { formatDuration, formatLapse, formatNumber, formatTokens, NONE, shortId } from '../lib/format'
import { statusMeta } from '../lib/status'
@@ -189,11 +189,12 @@ function matchedLines(matched: unknown): string {
return `追溯明细:\n${lines.join('\n')}${more}`
}
/** stats 里的裁判句数(claims 为 judge 时才有)→「结论 4 句(支持 1 · 不支持 1 · 未裁判 2)」 */
/**
* stats 里的裁判句数(claims 为 judge 时才有)→「结论 5 句(有依据 1 · 证据相矛盾 1 · 证据不足 1 · 未裁判 2)」。
* 旧文档的 stats 里是 unsupported(拆档之前的取值),照样计入、照旧叫「证据不支持」
*/
function claimsLine(st: ReportStamp['stats']): string {
const n = (v: unknown) => (typeof v === 'number' && Number.isFinite(v) ? v : 0)
const c = { supported: n(st.supported), partial: n(st.partial), unsupported: n(st.unsupported), unjudged: n(st.unjudged), uncited: 0 }
return claimTally({ ...c, total: c.supported + c.partial + c.unsupported + c.unjudged })
return claimTally(claimCountsOf(st as Record<string, unknown>))
}
/** 报告节点章的悬停说明:两个数怎么来的,缺口里各有什么 */
+32 -4
View File
@@ -28,6 +28,7 @@ import evidenceFixture from '../run/__tests__/evidence-doc.json'
import evidenceQuery from '../run/__tests__/evidence-query.json'
import evidenceEntity from '../run/__tests__/evidence-entity.json'
import evidenceJudge from '../run/__tests__/evidence-judge.json'
import evidenceVerdict from '../run/__tests__/evidence-verdict.json'
import { EvidenceGuessView } from '../run/EvidenceGuess'
import { IssuanceBanner } from '../run/AssistantStream'
import mdPinned from '../run/__tests__/markdown-pinned.json'
@@ -199,6 +200,12 @@ const EV_ENTITY = evidenceEntity.doc as unknown as EvidenceDocData
const EV_JUDGE = evidenceJudge.formal.doc as unknown as EvidenceDocData
const EV_JUDGE_EXPLORE = evidenceJudge.explore.doc as unknown as EvidenceDocData
const EV_JUDGE_PLAIN = evidenceJudge.plain.doc as unknown as EvidenceDocData
/**
* 裁判拆档(JR):证据相矛盾、证据不足(写明缺什么)、讲方法的句子(只挂表名字段名和整份查询)、裁判模型和写作
* 模型相同、模型不在价格表里。探索运行那份的证据图里带着一条拆档之前留下的「证据不支持」(evidence-verdict.json)
*/
const EV_VERDICT = evidenceVerdict.formal.doc as unknown as EvidenceDocData
const EV_VERDICT_EXPLORE = evidenceVerdict.explore.doc as unknown as EvidenceDocData
/** 同一份旧契约的出具,去掉位置:老运行就是这样,只能按字符串标 */
const LEGACY_LOOSE = {
matched: EV_LEGACY.matched.map(({ start: _s, end: _e, ...m }) => m),
@@ -339,18 +346,38 @@ function EvidenceOnly() {
<EvidenceDoc doc={EV_JUDGE_PLAIN} artifact={evidenceJudge.plain.doc_artifact} runId={evidenceJudge.plain.run_id}
label="answer" tally />
</div>
{/* 裁判拆档:证据相矛盾、证据不足(缺什么)、讲方法的句子挂的 SQL 和字段清单、裁判模型的两条提醒 */}
<div id="evidence-verdict" className="rounded-lg border p-3">
<div className="mb-1 text-2xs text-faint">正式运行 · 证据相矛盾、证据不足、讲方法的句子(裁判模型和写作模型相同,模型不在价格表里)</div>
<EvidenceDoc doc={EV_VERDICT} artifact={evidenceVerdict.formal.doc_artifact} runId={evidenceVerdict.formal.run_id}
runClass="formal" label="answer" tally />
</div>
<div id="evidence-verdict-narrow" className="rounded-lg border bg-panel p-2.5" style={{ width: 360, maxWidth: '100%' }}>
<EvidenceDoc doc={EV_VERDICT} artifact={evidenceVerdict.formal.doc_artifact} runId={evidenceVerdict.formal.run_id}
runClass="formal" label="answer" dense tally />
</div>
<div id="evidence-verdict-explore" className="rounded-lg border p-3">
<div className="mb-1 text-2xs text-faint">探索运行 · 按需裁判,证据图里有一条拆档之前的「证据不支持」</div>
<EvidenceDoc doc={EV_VERDICT_EXPLORE} artifact={evidenceVerdict.explore.doc_artifact}
runId={evidenceVerdict.explore.run_id} label="answer" tally />
</div>
{/* 出具横幅那一行多一段结论句:「结论 4 句(支持 1 · 部分支持 1 · 不支持 1 · 未裁判 1)」 */}
<div id="issuance-claims" className="flex flex-col gap-2 rounded-lg border p-3">
{/*
judge-uncited:开了裁判的文档,没挂依据的那句送了裁判、按判定数在「支持 3」里(或预筛放掉了不数),
出具(io.py)照样按没挂依据记 1 句——横幅得另起「没挂依据的结论句 1」,不能被结论句那一段吞掉
*/}
{/*
rework:裁判拆档之后的计数(证据相矛盾、证据不足);insufficient:只有证据不足一种问题,照样给「定位下一处」
*/}
{([
['judged', { total: 4, supported: 1, partial: 1, unsupported: 1, unjudged: 1, uncited: 0 }, null],
['cited', { total: 4, supported: 3, partial: 0, unsupported: 0, unjudged: 0, uncited: 1 }, null],
['all', { total: 2, supported: 2, partial: 0, unsupported: 0, unjudged: 0, uncited: 0 }, null],
['judge-uncited', { total: 3, supported: 3, partial: 0, unsupported: 0, unjudged: 0, uncited: 0 },
['judged', { total: 4, supported: 1, partial: 1, contradicted: 0, insufficient: 0, unsupported: 1, unjudged: 1, uncited: 0 }, null],
['cited', { total: 4, supported: 3, partial: 0, contradicted: 0, insufficient: 0, unsupported: 0, unjudged: 0, uncited: 1 }, null],
['all', { total: 2, supported: 2, partial: 0, contradicted: 0, insufficient: 0, unsupported: 0, unjudged: 0, uncited: 0 }, null],
['judge-uncited', { total: 3, supported: 3, partial: 0, contradicted: 0, insufficient: 0, unsupported: 0, unjudged: 0, uncited: 0 },
{ policy: 'judge', uncited_claims: 1 }],
['rework', { total: 7, supported: 1, partial: 1, contradicted: 2, insufficient: 1, unsupported: 1, unjudged: 1, uncited: 0 }, null],
['insufficient', { total: 3, supported: 2, partial: 0, contradicted: 0, insufficient: 1, unsupported: 0, unjudged: 0, uncited: 0 }, null],
] as const).map(([key, claims, issued]) => (
<div key={key} data-issuance-case={key}>
<IssuanceBanner
@@ -659,6 +686,7 @@ function Harness() {
evidenceQuery,
evidenceEntity,
evidenceJudge,
evidenceVerdict,
}
createRoot(document.getElementById('root')!).render(
+191 -13
View File
@@ -14,8 +14,12 @@
* 按数值猜的候选用最淡的点状线。
*
* 四期接上了概率性的几种:结论句的判定是模型给的,只在句末挂一枚小徽标(◆ 有依据、◇ 部分有依据、
* ! 证据不支持且整句浅底、? 未裁判),整句的字不画线。判定有两个来源:封存的文档里节点当场判的,
* ! 证据相矛盾且整句浅底、○ 证据不足、? 未裁判),整句的字不画线。判定有两个来源:封存的文档里节点当场判的,
* 和探索运行里点开再判、封存之后追加的(后者盖过前者,另写「封存后追加」)。
*
* 裁判拆档(JR):原来的「证据不支持」拆成证据相矛盾(证据和原句冲突)和证据不足(证据里没有相关信息,
* 裁判写明缺什么)。证据不足是中性的:不用红,字形用 ○,和未裁判的 ? 分得开。旧文档、旧事件里的
* unsupported 原样保留,外观同证据相矛盾,叫法照旧「证据不支持」。
*/
import {
@@ -25,8 +29,9 @@ import {
import { NONE, formatNumber, shortId } from './format'
import type {
EvidenceBlock, EvidenceCaliberSource, EvidenceCaliberUpgrade, EvidenceCandidate, EvidenceDocData, EvidenceFieldRef,
EvidenceGraph, EvidenceGuess, EvidenceInput, EvidenceLocator, EvidenceSeal, EvidenceSegment, EvidenceSegmentDetail,
EvidenceStats, EvidenceStep, EvidenceUnit, EvidenceVerdict, EvidenceViolation, ReviewResult,
EvidenceGraph, EvidenceGuess, EvidenceInput, EvidenceJudgeMeta, EvidenceJudgeSummary, EvidenceLocator, EvidenceSeal,
EvidenceSegment, EvidenceSegmentDetail, EvidenceStats, EvidenceStep, EvidenceUnit, EvidenceVerdict, EvidenceViolation,
ReviewResult,
} from '../types'
import type { MarkSpec } from '../run/Markdown'
@@ -80,6 +85,21 @@ export const EVIDENCE_STATE: Record<EvidenceStateCode, EvidenceStateMeta> = {
alert: true, phase: 4,
hint: '裁判模型认为证据仅支持本句的一部分',
},
contradicted: {
code: 'contradicted', label: EVIDENCE_STATE_LABEL.contradicted, line: 'badge', glyph: '!',
color: 'var(--st-failed)', decoration: 'transparent', soft: 'var(--st-failed-soft)',
alert: true, phase: 4,
hint: '裁判模型认为附带的证据与本句相冲突',
},
// 证据不足:中性,不用红(不是证据说它错了,是摘录里没有能判断的信息)。字形用 ○,不和未裁判的 ? 混;
// 仍算问题(进 n / N),排在证据相矛盾、部分有依据之后
insufficient: {
code: 'insufficient', label: EVIDENCE_STATE_LABEL.insufficient, line: 'badge', glyph: '○',
color: 'var(--st-cancelled)', decoration: 'transparent', soft: 'var(--st-cancelled-soft)',
alert: true, phase: 4,
hint: '裁判模型认为附带的证据中没有判断本句所需的信息,既不能证实也不能否定',
},
// 拆档之前的取值:旧文档、旧事件里保留原样。外观同证据相矛盾
unsupported: {
code: 'unsupported', label: EVIDENCE_STATE_LABEL.unsupported, line: 'badge', glyph: '!',
color: 'var(--st-failed)', decoration: 'transparent', soft: 'var(--st-failed-soft)',
@@ -129,6 +149,19 @@ export const EVIDENCE_STATE: Record<EvidenceStateCode, EvidenceStateMeta> = {
/** 图例、检查脚本按这个顺序列 */
export const EVIDENCE_STATES = Object.keys(EVIDENCE_STATE) as EvidenceStateCode[]
/**
* n / N 跳转按两轮走:先走第 1 轮(无证据、可疑名称、证据相矛盾、旧的证据不支持、部分有依据),再走第 2 轮
* (证据不足),每一轮里按正文顺序。不进跳转的是 0
*/
export function alertTier(state: EvidenceStateCode | null | undefined): 0 | 1 | 2 {
if (!state || !EVIDENCE_STATE[state].alert) return 0
return state === 'insufficient' ? 2 : 1
}
/** 证据和原句冲突的两种(新取值、拆档之前的旧取值):整句铺浅底 */
export const conflicting = (state: EvidenceStateCode | null | undefined): boolean =>
state === 'contradicted' || state === 'unsupported'
/**
* 片段在正文里的状态。文字、结构片段(行首符号、表格竖线)是 null:不画线、不进键盘顺序。
* 后端的 probabilistic 在裁判给出结论之前按「未裁判」画
@@ -543,7 +576,7 @@ export function tallySummary(t: EvidenceTally, keys: boolean): string {
t.unverified ? `${formatNumber(t.unverified)} 个名称${EVIDENCE_STATE.unverified.label}` : '',
t.structural ? EVIDENCE_TEXT.structuralCount(t.structural) : '',
t.noSegment ? EVIDENCE_TEXT.noSegmentCount(t.noSegment) : '',
keys ? (t.claims?.unsupported || t.claims?.partial ? JUDGE_TEXT.keysHint : EVIDENCE_TEXT.keysHint) : '',
keys ? (claimProblems(t.claims) ? JUDGE_TEXT.keysHint : EVIDENCE_TEXT.keysHint) : '',
].filter(Boolean).join('。')
}
@@ -843,8 +876,11 @@ export function stampCounts(stats: EvidenceStats | null | undefined, claims?: st
// 四期:结论句裁判
// -------------------------------------------------------------------------
/** 判过了的四种(不含未裁判):同后端 judge.JUDGED。判过的不再给「请模型判断这句」 */
const JUDGED = new Set(['supported', 'partial', 'unsupported', 'not_a_claim'])
/**
* 判过了的几种(不含未裁判):同后端 judge.JUDGED,另加拆档之前的 unsupported(旧文档、旧事件里还有)。
* 判过的不再给「请模型判断这句」
*/
const JUDGED = new Set(['supported', 'partial', 'contradicted', 'insufficient', 'unsupported', 'not_a_claim'])
/** 触顶的四个上限:同后端 judge.LIMITS,这几种没判的写「已到上限」和怎么调 */
export const JUDGE_LIMITS: readonly string[] = ['max_claims', 'max_cost_usd', 'daily_max_usd', 'timeout_s']
@@ -862,12 +898,73 @@ export function verdictState(v: Pick<EvidenceVerdict, 'status'> | null | undefin
switch (v?.status) {
case 'supported': return 'supported'
case 'partial': return 'partial'
case 'contradicted': return 'contradicted'
case 'insufficient': return 'insufficient'
case 'unsupported': return 'unsupported'
case 'unjudged': return 'unjudged'
default: return null
}
}
/** 证据不足时缺的是什么(裁判给的);别的判定、没写的是空串 */
export function verdictMissing(v: Pick<EvidenceVerdict, 'status' | 'missing'> | null | undefined): string {
return v?.status === 'insufficient' && typeof v.missing === 'string' ? v.missing.trim() : ''
}
/** 结论句的计数里有没有要提醒的:证据相矛盾、旧的证据不支持、部分有依据、证据不足(横幅、证据条的「定位下一处」) */
export function claimProblems(c: ClaimTallyCounts | null | undefined): number {
if (!c) return 0
return (c.contradicted ?? 0) + (c.unsupported ?? 0) + (c.partial ?? 0) + (c.insufficient ?? 0)
}
/** 计数表里认的几种判定(「不是结论句」不算进总数) */
const COUNTED = ['supported', 'partial', 'contradicted', 'insufficient', 'unsupported', 'unjudged'] as const
/**
* 按判定分好的句数(report.checked 的 judge.counts、文档 stats 里裁判那几个键、出具的 claims.counts)→ 横幅那种
* 计数:总数不算「不是结论句」,旧数据的 unsupported 照样计入。一个判定都没有时总数是 0
*/
export function claimCountsOf(counts: Record<string, unknown> | null | undefined): ClaimTallyCounts {
const n = (k: string) => {
const v = counts?.[k]
return typeof v === 'number' && Number.isFinite(v) && v > 0 ? v : 0
}
const c: ClaimTallyCounts = { total: 0, supported: 0, partial: 0, contradicted: 0, insufficient: 0, unsupported: 0,
unjudged: 0, uncited: 0 }
for (const k of COUNTED) {
c[k] = n(k)
c.total += c[k]
}
return c
}
/**
* 这条判定该带的两条提醒:裁判模型和写作模型相同、模型不在价格表里(金额上限对它不生效)。只对模型判过的判定。
* 按从近到远取:判定自己带着的 → 封存之后追加的看这一次按需裁判的答复、再看证据图里最近一次按需裁判
* (reports[].judge_meta);节点里当场判的看报告文档的 judge。都没说的不提醒
*/
export function judgeFlags(v: EvidenceVerdict | null | undefined, from: {
doc?: Pick<EvidenceJudgeSummary, 'same_model' | 'priced' | 'writer_model' | 'model'> | null
reply?: { same_model?: unknown; priced?: unknown; writer_model?: unknown; model?: unknown } | null
meta?: EvidenceJudgeMeta | null
} = {}): { sameModel: boolean; unpriced: boolean; writer: string | null; model: string | null } {
const none = { sameModel: false, unpriced: false, writer: null, model: null }
if (!v || !isJudged(v)) return none
const layers = (v.post_seal ? [v, from.reply, from.meta] : [v, from.doc]).filter(Boolean) as Record<string, unknown>[]
const pick = <T,>(key: string, ok: (x: unknown) => x is T): T | undefined => {
for (const l of layers) if (ok(l[key])) return l[key] as T
return undefined
}
const isBool = (x: unknown): x is boolean => typeof x === 'boolean'
const isText = (x: unknown): x is string => typeof x === 'string' && !!x
return {
sameModel: pick('same_model', isBool) === true,
unpriced: pick('priced', isBool) === false,
writer: pick('writer_model', isText) ?? null,
model: (isText(v.judge) ? v.judge : pick('model', isText)) ?? null,
}
}
/** 一句的判定:封存之后按需追加的盖过文档里的(探索运行里文档记的是「未裁判 · 按需」) */
export function unitVerdict(
unit: Pick<EvidenceUnit, 'id' | 'verdict'>, overlay?: Record<string, EvidenceVerdict> | null,
@@ -891,15 +988,17 @@ export function claimCounts(
doc: Pick<EvidenceDocData, 'blocks' | 'judge'>, overlay?: Record<string, EvidenceVerdict> | null,
): ClaimTallyCounts | null {
const judged = !!doc.judge && typeof doc.judge === 'object'
const c: ClaimTallyCounts = { total: 0, supported: 0, partial: 0, unsupported: 0, unjudged: 0, uncited: 0 }
const c: ClaimTallyCounts = { total: 0, supported: 0, partial: 0, contradicted: 0, insufficient: 0, unsupported: 0,
unjudged: 0, uncited: 0 }
let seen = judged
for (const block of doc.blocks ?? []) {
for (const unit of block.units ?? []) {
const v = unitVerdict(unit, overlay)
if (v) seen = true
if (v) {
if (v.status === 'supported' || v.status === 'partial' || v.status === 'unsupported' || v.status === 'unjudged') {
c[v.status] += 1
const k = (COUNTED as readonly string[]).includes(v.status) ? v.status as typeof COUNTED[number] : null
if (k) {
c[k] += 1
c.total += 1
}
continue
@@ -932,12 +1031,13 @@ const clip = (text: string, n = 40) => (text.length > n ? `${text.slice(0, n - 1
/**
* 句末徽标的 aria-label:句子、判定都写全,封存后追加的另说。「结论句「增长主要来自新客首单…」,
* 模型判断:证据不支持」;到上限没判的说「未裁判:已到上限」
* 模型判断:证据相矛盾」;证据不足的接着说缺什么;到上限没判的说「未裁判:已到上限」
*/
export function claimLabel(unit: Pick<EvidenceUnit, 'segments'>, v: EvidenceVerdict): string {
const state = verdictState(v)
const said = state ? EVIDENCE_STATE[state].label : JUDGE_TEXT.notClaim
const why = limitOf(v) ? `${said}:${JUDGE_TEXT.limit}` : said
const missing = verdictMissing(v)
const why = limitOf(v) ? `${said}:${JUDGE_TEXT.limit}` : missing ? `${said},${JUDGE_TEXT.missing(missing)}` : said
return JUDGE_TEXT.badgeLabel(clip(unitText(unit).trim()), why, !!v.post_seal)
}
@@ -954,6 +1054,7 @@ function asVerdict(v: unknown, fallback: { model?: unknown; postSeal: boolean })
post_seal: typeof o.post_seal === 'boolean' ? o.post_seal : fallback.postSeal,
used: Array.isArray(o.used) ? o.used.map(String) : undefined,
reason: typeof o.reason === 'string' ? o.reason : undefined,
missing: typeof o.missing === 'string' && o.missing.trim() ? o.missing : undefined,
}
}
@@ -1007,6 +1108,68 @@ export function rewriteOf(doc: Pick<EvidenceDocData, 'judge'>, uid: string):
return null
}
/**
* 讲方法、讲结构的句子(JR 第 3 条):写作者只挂了表名、字段名或整份查询([[see:Qn]]),没挂数字、值、口径卡、
* 引文。这类句子的「挂的依据」下直接列出对应的 SQL 和字段清单——裁判看的就是这些,读的人也该看得到。
*
* 返回要列的东西:表(字段清单)、字段(类型)各自在句中的片段(字段清单、类型从这个片段的证据链取),
* 要列 SQL 的查询(直接挂的在前,再补表名字段名出现过的查询,最多 BASIS_QUERIES 个,多的只写个数)。
* 挂了别的依据、或者一个表名字段名和查询都没挂的,返回 null(照旧只列编号)
*/
export const BASIS_QUERIES = 3
export interface ClaimBasis {
tables: { alias: string; name: string; seg?: string }[]
columns: { alias: string; name: string; seg?: string }[]
queries: string[]
more: number
}
export function claimBasis(
unit: Pick<EvidenceUnit, 'cites' | 'see' | 'segments'>, doc: Pick<EvidenceDocData, 'catalog' | 'blocks'>,
): ClaimBasis | null {
const catalog = doc.catalog ?? {}
const cites = (unit.cites ?? []).filter((a): a is string => typeof a === 'string' && !!a)
if (!cites.length) return null
const entity = (kind: unknown) => kind === 'table' || kind === 'column'
// 句中写作者标的引用(自动链接的名字不算依据):数字、值、单元格、口径卡、引文都算「挂了数据」
if ((unit.segments ?? []).some((s) => s.cite && !s.auto && s.cite.status === 'resolved' && !entity(s.cite.kind))) return null
if ((unit.see ?? []).some((c) => c?.status === 'resolved' && !entity(c.kind) && c.kind !== 'query')) return null
const out: ClaimBasis = { tables: [], columns: [], queries: [], more: 0 }
const segOf = (alias: string): string | undefined => {
const hit = (u: Pick<EvidenceUnit, 'segments'>) =>
(u.segments ?? []).find((s) => s.cite?.alias === alias && s.state === 'deterministic')?.id
return hit(unit) ?? (doc.blocks ?? []).flatMap((b) => b.units ?? []).map(hit).find(Boolean)
}
const nameOf = (alias: string, entry: Record<string, any> | undefined) => {
const loc = entry?.locator ?? {}
if (entry?.kind === 'column' && loc.table && loc.column) return `${loc.table}.${loc.column}`
return typeof entry?.name === 'string' && entry.name ? entry.name : alias.replace(/^[tc]:/, '')
}
const queries: string[] = []
for (const alias of cites) {
const entry = catalog[alias]
if (entry?.kind === 'table') out.tables.push({ alias, name: nameOf(alias, entry), seg: segOf(alias) })
else if (entry?.kind === 'column') out.columns.push({ alias, name: nameOf(alias, entry), seg: segOf(alias) })
else if (entry?.kind === 'query') queries.push(alias)
else return null
}
for (const e of [...out.tables, ...out.columns]) {
for (const q of Array.isArray(catalog[e.alias]?.queries) ? catalog[e.alias].queries : []) {
if (typeof q === 'string' && catalog[q]?.kind === 'query' && !queries.includes(q)) queries.push(q)
}
}
out.queries = queries.slice(0, BASIS_QUERIES)
out.more = Math.max(0, queries.length - BASIS_QUERIES)
return out
}
/** 查询快照工件里的 SQL:快照本身带 sql;工具调用的快照({tool, args, result})在 args 或 result 里 */
export function sqlOf(content: unknown): string | null {
if (!content || typeof content !== 'object') return null
const o = content as { sql?: unknown; args?: { sql?: unknown }; result?: { sql?: unknown } }
const sql = [o.sql, o.args?.sql, o.result?.sql].find((x) => typeof x === 'string' && x.trim())
return typeof sql === 'string' ? sql : null
}
// -------------------------------------------------------------------------
// 旧运行的猜测
// -------------------------------------------------------------------------
@@ -1082,11 +1245,22 @@ export interface AuditRow {
sentence: string
/** 这件证据在不在封存范围内(封存核对也通过);没有证据的是 null */
sealed: boolean | null
/** 结论句那几行:是哪一句(正文里有句末徽标时点这一行打开它的「模型的解释」) */
unit?: string
}
const GROUP_OF: Record<EvidenceStateCode, AuditGroup> = {
deterministic: 'cited', none: 'none', suspect: 'suspicious', unverified: 'suspicious', candidate: 'candidate',
supported: 'cited', partial: 'none', unsupported: 'none', unjudged: 'cited', connective: 'cited',
supported: 'cited', partial: 'none', contradicted: 'none', insufficient: 'none', unsupported: 'none', unjudged: 'cited',
connective: 'cited',
}
/** 审计行带的判定 → 行的状态:有问题的几种才有自己的行,认不出的交给后面按 state 认 */
const claimRowState = (status: unknown): EvidenceStateCode | undefined =>
status === 'contradicted' || status === 'insufficient' || status === 'unsupported' || status === 'partial' ? status : undefined
/** 审计行的 issue → 结论句的判定(后端按约定给证据相矛盾、证据不足的结论句各一行;旧取值也认) */
const CLAIM_ISSUE: Record<string, EvidenceStateCode> = {
contradicted_claim: 'contradicted', insufficient_claim: 'insufficient', unsupported_claim: 'unsupported',
partial_claim: 'partial',
}
const AUDIT_GROUP_SET = new Set<string>(['cited', 'none', 'suspicious', 'candidate'])
@@ -1111,7 +1285,10 @@ export function auditFromApi(body: unknown): AuditRow[] {
for (const [ri, r] of (Array.isArray(g?.rows) ? g.rows : []).entries()) {
if (!r || typeof r !== 'object') continue
const kind = String(r.kind ?? '')
const state = segmentState({ kind, state: r.state ?? 'none', issue: r.issue ?? undefined })
// 结论句的判定行:按 issue 认成对应的判定(外观、叫法同句末徽标);issue 认不出时看行上带的 verdict
const state = (kind === 'claim' && typeof r.issue === 'string' ? CLAIM_ISSUE[r.issue] : undefined)
?? (kind === 'claim' && r.verdict && typeof r.verdict === 'object' ? claimRowState(r.verdict.status) : undefined)
?? segmentState({ kind, state: r.state ?? 'none', issue: r.issue ?? undefined })
?? (r.group === 'candidate' ? 'candidate' : 'none')
const group: AuditGroup = AUDIT_GROUP_SET.has(r.group) ? r.group : AUDIT_GROUP_SET.has(g?.key) ? g.key : GROUP_OF[state]
const report = String(r.report ?? r.field ?? '')
@@ -1126,6 +1303,7 @@ export function auditFromApi(body: unknown): AuditRow[] {
ref: typeof r.ref === 'string' ? r.ref : undefined,
sentence: String(r.sentence ?? '').trim(),
sealed: typeof r.sealed === 'boolean' ? r.sealed : null,
...(kind === 'claim' && typeof r.unit === 'string' && r.unit ? { unit: r.unit } : {}),
})
}
})
+52 -10
View File
@@ -155,13 +155,16 @@ export const APPROVAL_POLICY_LABEL: Record<'dangerous' | 'always' | 'never', str
* 可点击证据:片段状态的叫法。正文里片段的 aria-label、面板标题、图例都用这一份。
* 外观(线型、字形、颜色)在 lib/evidence.ts,和这里一一对应。
*
* 概率性的四种(四期的结论句裁判)挂在句末的徽标上:有依据、部分有依据、证据不支持都是模型的判断,
* 概率性的几种(四期的结论句裁判)挂在句末的徽标上:有依据、部分有依据、证据相矛盾、证据不足都是模型的判断,
* 叫法里写明「模型判断」;未裁判不带「有引用」——没挂依据、只有方向词的结论句也会送裁判、也会没判。
* 证据不支持是裁判拆档之前的取值:已封存的文档、已有的事件里保留原样,照旧这样叫(新判的不再出现)。
*/
export const EVIDENCE_STATE_LABEL = {
deterministic: '有出处',
supported: '模型判断:有依据',
partial: '模型判断:部分有依据',
contradicted: '模型判断:证据相矛盾',
insufficient: '模型判断:证据不足',
unsupported: '模型判断:证据不支持',
unjudged: '未裁判',
none: '无证据',
@@ -313,6 +316,10 @@ export const EVIDENCE_TEXT = {
checked: (schemas: number, queries: number) =>
`已核对 ${formatNumber(schemas)} 份表结构快照、${formatNumber(queries)} 次查询`,
tableColumns: (n: number, view: boolean) => `${view ? '视图' : '表'} · ${formatNumber(n)} 个字段`,
/** 表名实体步骤的字段清单(来自封存范围内的表结构快照,最多列 60 个,遮罩的列不列入) */
fields: (n: number) => `字段清单(${formatNumber(n)} 个)`,
fieldsMore: (n: number) => `另有 ${formatNumber(n)} 个字段未列出`,
fieldsMasked: '部分字段已按数据源的遮罩设置隐藏,未列出',
columnTypes: (types: Record<string, string>) =>
`各表的类型:${Object.entries(types).map(([t, v]) => `${t} ${v}`).join('、')}`,
quoteBad: '按记录的位置从原文中截取的内容与这句引文不一致:原文可能已被修改',
@@ -342,6 +349,11 @@ export interface ClaimTallyCounts {
total: number
supported: number
partial: number
/** 证据和原句冲突 */
contradicted: number
/** 证据里没有判断所需的信息 */
insufficient: number
/** 拆档之前的取值(旧文档、旧事件),照旧计入 */
unsupported: number
/** 送了裁判、没判成(到上限、调用失败)或还没请模型判断的 */
unjudged: number
@@ -356,8 +368,9 @@ export interface ClaimTallyCounts {
export function claimTally(c: ClaimTallyCounts): string {
if (!c.total) return ''
const parts = [
['有依据', c.supported], ['部分有依据', c.partial], ['证据不支持', c.unsupported], ['未裁判', c.unjudged], ['无证据', c.uncited],
].filter(([, n]) => (n as number) > 0).map(([k, n]) => `${k} ${formatNumber(n as number)}`)
['有依据', c.supported], ['部分有依据', c.partial], ['证据相矛盾', c.contradicted], ['证据不支持', c.unsupported],
['证据不足', c.insufficient], ['未裁判', c.unjudged], ['无证据', c.uncited],
].filter(([, n]) => typeof n === 'number' && n > 0).map(([k, n]) => `${k} ${formatNumber(n as number)}`)
return `结论 ${formatNumber(c.total)} 句${parts.length ? `(${parts.join(' · ')})` : ''}`
}
@@ -368,8 +381,11 @@ export function claimTally(c: ClaimTallyCounts): string {
export const JUDGE_TEXT = {
section: '模型的解释',
claim: '结论句',
/** 读屏摘要末尾的操作说明:有证据不支持、部分支持的结论句时 n 也跳到它们的句末徽标 */
keysHint: '用左右方向键逐个查看,上下方向键按句切换,n 跳到下一处无证据或证据不支持的句子,回车打开证据,Esc 关闭',
/**
* 读屏摘要末尾的操作说明:结论句有问题(证据相矛盾、部分有依据、证据不足)时 n 也跳到它们的句末徽标。
* 证据不足的排在其余问题之后
*/
keysHint: '用左右方向键逐个查看,上下方向键按句切换,n 跳到下一处无证据或证据有问题的句子(证据不足的排在最后),回车打开证据,Esc 关闭',
/** 面板里判断的徽标:谁判的、而且不是确定的 */
badge: (model: string | null | undefined) => `模型判断 · ${model || '裁判模型'} · 非确定`,
postSeal: '封存后追加',
@@ -389,11 +405,30 @@ export const JUDGE_TEXT = {
notClaim: '模型判断:不是结论句',
ask: '请模型判断本句',
askAgain: '再次请模型判断',
/** 封存后追加的判定早于当前的裁判规则(片段接口的 on_demand.reason 为 outdated):旁边注明,按钮照常给 */
outdated: '按旧规则判定',
asking: '正在请模型判断…',
askHint: '探索运行按需裁判:每次点击裁判一句,费用计入单次点击上限和每日上限',
notAsked: '尚未请模型判断本句',
screened: '预筛判定本句不陈述数据事实(短句或过渡语),未送裁判',
noPrice: '裁判模型不在价格目录中,无法按 token 数估算金额:金额上限对其不生效',
/** 证据不足时缺的是什么(裁判给的,如「SQL」「字段清单」):面板、句末徽标的悬停说明和读屏都写 */
missingLead: '缺少:',
missing: (what: string) => `缺少:${what}`,
/** 裁判模型的两条提醒:判定照常给出,读的人要知道分量 */
sameModel: '裁判模型和写作模型相同,结果仅供参考',
sameModelTitle: (model: string) => `裁判模型与撰写这份报告的模型都是 ${model}:同一模型较难发现自身的错误`,
noPrice: '该模型不在价格表中,金额上限不生效',
noPriceTitle: '无法按 token 数估算这个模型的费用,裁判的金额上限(每份报告、每次点击、每日)对它不起作用;句数和时长上限照常生效',
/** 讲方法、讲结构的句子(只挂了表名、字段名、整份查询):「挂的依据」下直接列出 SQL 和字段清单 */
basisSql: (alias: string) => `查询 ${alias} 的 SQL`,
basisFields: (table: string) => `表 ${table} 的字段清单`,
basisFieldsCount: (table: string, n: number) => `表 ${table} 的字段清单(${formatNumber(n)} 个)`,
basisColumns: '字段类型',
basisPending: '正在加载 SQL 和字段清单…',
sqlMissing: '未能获取这次查询的 SQL',
fieldsMissing: '未能获取字段清单',
basisNoRun: '这份报告不属于任何运行,无法获取 SQL 和字段清单',
basisMoreQueries: (n: number) => `另有 ${formatNumber(n)} 次查询,点开句中的表名查看`,
limit: '已达上限',
/** 触顶之后怎么调。按需裁判(每次点击)和正式运行(每份报告)调的地方不一样 */
limitHow: {
@@ -437,10 +472,14 @@ export const JUDGE_FIELD_LABEL: Record<string, string> = {
max_claims: '最多裁判句数',
max_cost_usd: '金额上限(美元)',
timeout_s: '时长上限(秒)',
rewrite_once: '证据不支持的句子退回改写一次',
on_unsupported: '证据不支持时',
rewrite_once: '证据相矛盾或不足的句子退回改写一次',
// 键名仍是 on_unsupported(兼容已有配置);裁判拆档后它管的是「证据相矛盾」(旧取值证据不支持照旧按它判),
// 证据不足的句子最多降档
on_unsupported: '证据相矛盾时',
}
/** judge.on_unsupported:正式运行里证据不支持的结论句怎么判档(探索运行只标注) */
/** judge.on_unsupported 的说明:检查器里选项下面那一行 */
export const JUDGE_ON_UNSUPPORTED_HINT = '仅对正式运行生效;证据不足的句子最多降档,不会因此不予出具。探索运行只做标注,不拦截'
/** judge.on_unsupported:正式运行里证据相矛盾的结论句怎么判档(探索运行只标注) */
export const JUDGE_ON_UNSUPPORTED_LABEL: Record<string, string> = {
degrade: '出具降档',
withhold: '不予出具',
@@ -459,6 +498,9 @@ export const JUDGE_SETTING_TEXT = {
modelDefault: '留空:用接入的默认模型',
modelFollow: '留空:跟随助手的模型',
differ: '建议与撰写报告的模型不同:由同一模型审核自身输出,不易发现其中的错误',
/** 接入和模型都没填:后端依次用助手的模型、默认接入。醒目地说一句,并给出建议 */
notSet: '未单独配置,将使用助手的模型',
notSetAdvice: '建议配置一个与撰写报告的模型不同、能力更强的模型:同一模型审核自身输出时,结果仅供参考',
limits: '上限(每一项都可以设成不限)',
reportClaims: '每份报告最多裁判句数',
reportCost: '每份报告的金额上限(美元)',
@@ -532,7 +574,7 @@ export const EVIDENCE_AUDIT_TEXT = {
candidate: '按数值猜测',
} as Record<string, string>,
groupHint: {
none: '裸数字、无法解析的引用、未附依据的结论句',
none: '裸数字、无法解析的引用、未附依据或模型判为证据有问题的结论句',
suspicious: '表名、字段名在本次运行的表结构和查询中找不到',
cited: '系统从证据中取值并核对过的片段',
candidate: '按数值猜测的可能来源,不能作为证据,默认收起',
+18 -2
View File
@@ -2,7 +2,7 @@ import { useEffect, useRef, useState } from 'react'
import type { KeyboardEvent as ReactKeyboardEvent, ReactNode } from 'react'
import { Navigate, useParams } from 'react-router-dom'
import {
Check, Cpu, Download, KeyRound, Monitor, Moon, Plug, Plus, Settings as SettingsIcon, Sun, X,
AlertTriangle, Check, Cpu, Download, KeyRound, Monitor, Moon, Plug, Plus, Settings as SettingsIcon, Sun, X,
} from 'lucide-react'
import clsx from 'clsx'
import { api } from '../api/client'
@@ -777,6 +777,8 @@ function PrefsTab() {
const judgeProviderMissing = !!cur.judgeProvider && providers.length > 0 && !judgeProvider
const judgeModelName = cur.judgeModel.trim()
const judgeByModel = !cur.judgeProvider && !!judgeModelName
// 裁判模型没有单独配置(接入、模型都留空):后端依次用助手的模型、默认接入
const judgeNotSet = !cur.judgeProvider && !judgeModelName
const judgeOwner = judgeByModel ? providerOfModel(providers, judgeModelName) : undefined
const judgeByModelHint = judgeByModel && providers.length > 0
? judgeByModelText('settings', judgeModelName, judgeOwner && { name: judgeOwner.provider.name, enabled: judgeOwner.enabled },
@@ -940,7 +942,21 @@ function PrefsTab() {
<SectionBar title={JUDGE_SETTING_TEXT.label} hint={JUDGE_SETTING_TEXT.hint} />
<div role="group" aria-labelledby="pref-judge-model-label" aria-describedby="pref-judge-model-hint" data-judge-model>
<div id="pref-judge-model-label" className="text-xs font-medium">裁判模型</div>
<p id="pref-judge-model-hint" className="mb-2 mt-0.5 text-2xs leading-relaxed text-faint">{JUDGE_SETTING_TEXT.differ}</p>
{judgeNotSet ? (
// 接入和模型都没填:裁判会用助手的模型(往往就是写报告的那个)。醒目地说,并给出建议;
// 填了任何一项就换回平常那句建议
<div id="pref-judge-model-hint" role="note" data-judge-not-set=""
className="mb-2 mt-1 flex items-start gap-1.5 rounded border-l-2 py-0.5 pl-2 text-2xs leading-relaxed"
style={{ borderColor: 'var(--st-waiting)' }}>
<AlertTriangle size={12} aria-hidden className="mt-0.5 shrink-0" style={{ color: 'var(--st-waiting)' }} />
<span>
<span className="font-medium" data-judge-not-set-text="">{JUDGE_SETTING_TEXT.notSet}</span>
<span className="block text-dim" data-judge-not-set-advice="">{JUDGE_SETTING_TEXT.notSetAdvice}</span>
</span>
</div>
) : (
<p id="pref-judge-model-hint" className="mb-2 mt-0.5 text-2xs leading-relaxed text-faint">{JUDGE_SETTING_TEXT.differ}</p>
)}
<div className="grid grid-cols-2 gap-3">
<Field label={JUDGE_SETTING_TEXT.provider} htmlFor="pref-judge-provider"
error={judgeProviderMissing ? `模型接入「${cur.judgeProvider}」不存在:裁判调用将失败,所有结论句记为未裁判` : undefined}
+21 -9
View File
@@ -8,8 +8,8 @@ import { ApiError, api } from '../../api/client'
import { isUpgradeAdvice } from '../../canvas/issues'
import { ErrorState, Skeleton, Spinner, StatusBadge, rovingTarget, toast, useRadioGroup } from '../../components/ui'
import {
EVIDENCE_STATE, PROBLEM_GROUPS, auditCsv, auditFromApi, auditRows, groupRows, guessRows, guessesOf, issuanceMarks,
segmentState, type AuditFilter, type AuditGroup, type AuditRow,
EVIDENCE_STATE, PROBLEM_GROUPS, auditCsv, auditFromApi, auditRows, graphVerdicts, groupRows, guessRows, guessesOf,
issuanceMarks, segmentState, unitVerdict, verdictState, type AuditFilter, type AuditGroup, type AuditRow,
} from '../../lib/evidence'
import { formatNumber } from '../../lib/format'
import { EVIDENCE_AUDIT_TEXT as T, UPGRADE_TEXT } from '../../lib/terms'
@@ -87,20 +87,28 @@ export function EvidencePane({ run, output, labelOf, refreshKey }: {
// 哪些行能在正文里打开:`报告:片段` 的集合,一份文档算一次(长报告几千个片段,不能每行各扫一遍)。
// 只收有状态的片段,和 EvidenceDoc 能打开的是同一批:违规行指着的文字片段(粗体、链接里的可疑名字)、
// 结构片段(列表序号 100. 里的数字)在正文里没有能点的地方,画成按钮就是点了没反应
// 结构片段(列表序号 100. 里的数字)在正文里没有能点的地方,画成按钮就是点了没反应。
// 结论句的行(证据相矛盾、证据不足……)打开的是这一句的句末徽标(`报告:claim:句子`):句子有判定(封存的、
// 证据图里封存后追加的)才有徽标
const openable = useMemo(() => {
const keys = new Set<string>()
for (const r of reports) {
const doc = docOf(r.node_id)
for (const b of doc?.blocks ?? []) for (const u of b.units ?? []) for (const seg of u.segments ?? []) {
if (segmentState(seg)) keys.add(`${r.node_id}:${seg.id}`)
const late = graphVerdicts(graph, r.node_id)
for (const b of doc?.blocks ?? []) for (const u of b.units ?? []) {
for (const seg of u.segments ?? []) if (segmentState(seg)) keys.add(`${r.node_id}:${seg.id}`)
if (verdictState(unitVerdict(u, late))) keys.add(`${r.node_id}:claim:${u.id}`)
}
}
return keys
}, [reports, docOf])
const canOpen = useCallback((row: AuditRow) => !!row.seg && openable.has(`${row.report}:${row.seg}`), [openable])
}, [reports, docOf, graph])
const canOpen = useCallback((row: AuditRow) => {
const key = openKey(row)
return !!key && openable.has(`${row.report}:${key}`)
}, [openable])
const openRow = useCallback((row: AuditRow) => {
if (row.seg) handles.current.get(row.report)?.open(row.seg)
const key = openKey(row)
if (key) handles.current.get(row.report)?.open(key)
}, [])
const onView = useCallback((report: string, open: boolean) => {
@@ -188,6 +196,10 @@ export function EvidencePane({ run, output, labelOf, refreshKey }: {
)
}
/** 审计行在正文里打开什么:片段的行是那个片段,结论句的行是那一句的句末徽标(EvidenceDoc 认 claim:<句子>) */
const openKey = (row: AuditRow): string | undefined =>
row.seg ?? (row.kind === 'claim' && row.unit ? `claim:${row.unit}` : undefined)
/** 已经收尾的运行:还在跑、停在审批上的,报告可能还没写,说「这次的报告没有逐段证据」太早 */
const SETTLED = new Set(['succeeded', 'failed', 'cancelled'])
@@ -558,7 +570,7 @@ function AuditLine({ row, current, openable, multiReport, labelOf, onOpen, onFoc
data-audit-seg={row.seg ?? ''}>
<td className="px-3 py-1.5">
{openable ? (
<button type="button" {...common} aria-label={label} title={label} onClick={onOpen} data-audit-open={row.seg}
<button type="button" {...common} aria-label={label} title={label} onClick={onOpen} data-audit-open={openKey(row)}
className={clsx(common.className, 'block underline decoration-dotted underline-offset-2 hover:text-[var(--accent)]')}>
{text}
</button>
+3 -3
View File
@@ -23,7 +23,7 @@ import {
} from './decode'
import { CODE_COLUMN, CopyChip, LEADING_ZERO, Markdown, type MarkSpec } from './Markdown'
import { ClaimTally, EvidenceField, type EvidenceDocHandle } from './EvidenceDoc'
import { EVIDENCE_STATE, docTally, evidenceFields, issuanceMarks, type EvidenceTally } from '../lib/evidence'
import { EVIDENCE_STATE, claimProblems, docTally, evidenceFields, issuanceMarks, type EvidenceTally } from '../lib/evidence'
import type { EvidenceDocData } from '../types'
import { useRunClock } from './useRunClock'
import type { ReviewResult } from '../types'
@@ -2289,7 +2289,7 @@ export function IssuanceBanner({ issuance, runClass, evidence }: {
{meta.hint && <div className="mt-1 text-2xs leading-relaxed text-dim">{meta.hint}</div>}
{counts && (counts.total > 0 || counts.other > 0 || suspect > 0 || uncitedClaims > 0 || !!claims) && (
// 逐段证据的计数:和报告核对那一行、证据条同一种说法。可疑名字、没挂依据的结论句不是数字,
// 各自另起一句。有无证据的数字、可疑名字、证据不支持(或部分支持)的结论句时才给「定位下一处」——
// 各自另起一句。有无证据的数字、可疑名字、证据有问题(相矛盾、部分有依据、证据不足)的结论句时才给「定位下一处」——
// 结论句靠句末徽标定位;只是没挂依据的结论句没有徽标,定位不到
<div className="mt-1 flex flex-wrap items-center gap-2 text-2xs" data-evidence-line="">
{numbers && (
@@ -2307,7 +2307,7 @@ export function IssuanceBanner({ issuance, runClass, evidence }: {
{numbers || suspect > 0 || claims ? '· ' : ''}{EVIDENCE_TEXT.uncitedClaimsTag} {formatNumber(uncitedClaims)}
</span>
)}
{(counts.none > 0 || counts.other > 0 || suspect > 0 || !!claims?.unsupported || !!claims?.partial) && evidence?.onNext && (
{(counts.none > 0 || counts.other > 0 || suspect > 0 || claimProblems(claims) > 0) && evidence?.onNext && (
<button type="button" className="btn btn-xs" data-evidence-next="" onClick={evidence.onNext}>
{EVIDENCE_TEXT.locateNext}
</button>
+44 -25
View File
@@ -8,8 +8,9 @@ import { ListChecks } from 'lucide-react'
import clsx from 'clsx'
import { rovingTarget } from '../components/ui'
import {
EVIDENCE_STATE, claimCounts, claimLabel, docTally, segmentKind, segmentLabel, segmentState, tallySummary, unitText,
isJudged, limitOf, unitVerdict, verdictState, withVerdicts, type EvidenceStateCode, type EvidenceTally as Tally,
EVIDENCE_STATE, alertTier, claimCounts, claimLabel, claimProblems, conflicting, docTally, segmentKind, segmentLabel,
segmentState, tallySummary, unitText, isJudged, limitOf, unitVerdict, verdictMissing, verdictState, withVerdicts,
type EvidenceStateCode, type EvidenceTally as Tally,
} from '../lib/evidence'
import { formatNumber } from '../lib/format'
import { EVIDENCE_STATE_LABEL, EVIDENCE_TEXT, JUDGE_TEXT, claimTally, evidenceTally, type ClaimTallyCounts } from '../lib/terms'
@@ -26,13 +27,13 @@ import { BlockView, Markdown, StyledSlice, inlineStyles, numericColumns, type In
* BlockView,和普通答案长得一模一样;有出处的数字画细实线,没有出处的画点状线、
* 句末挂「无证据」。
*
* 结论句的判定(四期)只在句末挂一枚小徽标(◆ ◇ ! ?),整句不画线;证据不支持的整句铺一层浅底。
* 徽标也是按钮:点开是这一句的「模型的解释」。判定是模型给的,颜色一律不用确定性的绿。
* 结论句的判定(四期)只在句末挂一枚小徽标(◆ ◇ ! ○ ?),整句不画线;证据相矛盾(和旧的证据不支持)的整句
* 铺一层浅底。徽标也是按钮:点开是这一句的「模型的解释」。判定是模型给的,颜色一律不用确定性的绿。
*
* 键盘:整份报告只占一个 Tab 位(roving tabindex)。←/→ 在有状态的片段、句末徽标间走,↑/↓ 按
* 句子走(表格里按列走到上一行 / 下一行,出了表格接着按句子走),n / N 跳到下一处 /
* 上一处无证据、证据不支持的地方,回车打开证据面板,Esc 关掉并留在原片段;面板开着时方向键走到哪、
* 面板跟到哪,Tab 进面板。
* 上一处有问题的地方(无证据、可疑名称、证据相矛盾、部分有依据;证据不足的排在这些之后),回车打开证据面板,
* Esc 关掉并留在原片段;面板开着时方向键走到哪、面板跟到哪,Tab 进面板。
*
* 长报告:事件都挂在容器上(片段只带 data-seg),块用 memo,外加
* content-visibility: auto;超过 TEXT_CAP 的部分先折叠,跳转到折叠里的片段时自动展开。
@@ -66,8 +67,13 @@ interface Model {
/** 每个有可点片段的句子里第一个可点片段,按正文顺序(↑/↓ 用) */
unitFirst: string[]
unitOf: Map<string, number>
/** 无证据的片段,按正文顺序(n / N 用) */
/**
* 有问题的片段和句末徽标(n / N 用):先第 1 轮(无证据、可疑名称、证据相矛盾、部分有依据),再第 2 轮
* (证据不足),每一轮里按正文顺序
*/
alerts: string[]
/** alerts 里每一处在第几轮 */
tier: Map<string, 1 | 2>
/** 每块到它为止的累计字数(折叠用) */
chars: number[]
/** 表格里的片段在第几行第几列(表头 row = -1 不进来);↑/↓ 在表格里按列走 */
@@ -83,7 +89,7 @@ function buildModel(blocks: EvidenceBlock[], overlay?: Record<string, EvidenceVe
const badge = new Map<string, Badge>()
const unitFirst: string[] = []
const unitOf = new Map<string, number>()
const alerts: string[] = []
const tiered: [string, 1 | 2][] = []
const chars: number[] = []
const cell = new Map<string, { block: number; row: number; col: number }>()
const grid = new Map<string, string>()
@@ -101,14 +107,15 @@ function buildModel(blocks: EvidenceBlock[], overlay?: Record<string, EvidenceVe
order.push(s.id)
if (first) { unitFirst.push(s.id); first = false }
unitOf.set(s.id, unitFirst.length - 1)
if (EVIDENCE_STATE[state].alert) alerts.push(s.id)
const t = alertTier(state)
if (t) tiered.push([s.id, t])
if (loc) {
cell.set(s.id, { block: b, row: loc.row, col: loc.col })
const key = `${b}:${loc.row}:${loc.col}`
if (!grid.has(key)) grid.set(key, s.id)
}
}
// 句末徽标排在这一句的片段后面:←/→ 走到句尾就是它,证据不支持的也进 n / N
// 句末徽标排在这一句的片段后面:←/→ 走到句尾就是它,证据相矛盾、部分有依据、证据不足的也进 n / N
const verdict = unitVerdict(unit, overlay)
const state = verdictState(verdict)
if (verdict && state) {
@@ -118,12 +125,16 @@ function buildModel(blocks: EvidenceBlock[], overlay?: Record<string, EvidenceVe
order.push(key)
if (first) { unitFirst.push(key); first = false }
unitOf.set(key, unitFirst.length - 1)
if (EVIDENCE_STATE[state].alert) alerts.push(key)
const t = alertTier(state)
if (t) tiered.push([key, t])
}
}
chars.push(total)
})
return { order, index, seg, badge, unitFirst, unitOf, alerts, chars, cell, grid }
// 按(轮次、正文顺序)排:tiered 本来就是正文顺序,稳定排序只按轮次挪
const sorted = [...tiered].sort((a, b) => a[1] - b[1])
return { order, index, seg, badge, unitFirst, unitOf, alerts: sorted.map(([k]) => k), tier: new Map(sorted), chars, cell,
grid }
}
/** 片段或徽标在第几块 */
@@ -311,11 +322,17 @@ export const EvidenceDoc = forwardRef<EvidenceDocHandle, {
if (refocus && back) requestAnimationFrame(() => segEl(back)?.focus())
}, [view, current, segEl])
/**
* n / N 的下一处:按(轮次、正文位置)排的一圈,走到头接回开头。当前位置不是问题(普通片段、有依据的徽标)时
* 按第 1 轮算——第 1 轮里它后面没有了,就接着走第 2 轮(证据不足)
*/
const step = useCallback((list: string[], from: string | null, dir: 1 | -1): string | null => {
if (!list.length) return null
const pos = from ? model.index.get(from) ?? -1 : -1
if (dir > 0) return list.find((id) => (model.index.get(id) ?? 0) > pos) ?? list[0]
for (let i = list.length - 1; i >= 0; i--) if ((model.index.get(list[i]) ?? 0) < pos) return list[i]
const rank = (id: string | null): [number, number] => [id ? model.tier.get(id) ?? 1 : 1, id ? model.index.get(id) ?? -1 : -1]
const cmp = (a: [number, number], b: [number, number]) => a[0] - b[0] || a[1] - b[1]
const cur = rank(from)
if (dir > 0) return list.find((id) => cmp(rank(id), cur) > 0) ?? list[0]
for (let i = list.length - 1; i >= 0; i--) if (cmp(rank(list[i]), cur) < 0) return list[i]
return list[list.length - 1]
}, [model])
@@ -510,7 +527,7 @@ export function EvidenceTally({ counts, onNext, onList }: {
}) {
const suspect = counts.suspect ?? 0
const claims = counts.claims
const clean = !counts.none && !counts.other && !suspect && !claims?.unsupported && !claims?.partial
const clean = !counts.none && !counts.other && !suspect && !claimProblems(claims)
const numbers = evidenceTally(counts.cited, counts.total, counts.other)
return (
<div className="mb-1.5 flex flex-wrap items-center gap-x-2 gap-y-1 text-2xs" data-evidence-tally="">
@@ -541,13 +558,14 @@ export function EvidenceTally({ counts, onNext, onList }: {
}
/**
* 「结论 4 句(支持 3 · 不支持 1)」:和数字那一段同一种说法。判定是模型给的:全都支持也不用确定性的绿,
* 有不支持、部分支持、没挂依据的用提醒色,其余用暗字色(它是概率性的,不该和「有出处」一样笃定)
* 「结论 4 句(有依据 3 · 证据相矛盾 1)」:和数字那一段同一种说法。判定是模型给的:全都支持也不用确定性的绿,
* 有问题(证据相矛盾、旧的证据不支持、部分有依据、证据不足)、没挂依据的用提醒色,其余用暗字色(它是概率性的,
* 不该和「有出处」一样笃定)
*/
export function ClaimTally({ claims, lead }: { claims: ClaimTallyCounts; lead: boolean }) {
const text = claimTally(claims)
if (!text) return null
const warn = claims.unsupported > 0 || claims.partial > 0 || claims.uncited > 0
const warn = claimProblems(claims) > 0 || claims.uncited > 0
return (
<span className="tnum" style={{ color: warn ? 'var(--st-waiting)' : 'var(--text-dim)' }} data-evidence-claims=""
title={JUDGE_TEXT.badge(null)}>
@@ -655,7 +673,7 @@ const DocBlock = memo(function DocBlock({ block, dense, doc, visual, current, ac
/**
* 一句话:文字按行内 Markdown 渲染(整句算一次样式,跨片段的 `code`、**粗体** 不会
* 被切坏),有状态的片段包成按钮。含无证据片段的句子末尾挂「? 无证据」;有判定的结论句末尾挂
* 一枚徽标(也是按钮),证据不支持的整句铺浅底——字本身不画线
* 一枚徽标(也是按钮),证据相矛盾(和旧的证据不支持)的整句铺浅底——字本身不画线
*/
function UnitView({ unit, doc, current, active, panelId, code, tag, badge }: {
unit: EvidenceUnit; doc: EvidenceDocData; current: string | null; active: string | null; panelId: string
@@ -702,8 +720,8 @@ function UnitView({ unit, doc, current, active, panelId, code, tag, badge }: {
const open = !!badge && active === key
return (
<>
{badge?.state === 'unsupported'
// 证据不支持:整句铺一层浅底(折行时每行各自带圆角),字色、字重都不动,也不画线
{conflicting(badge?.state)
// 证据相矛盾(和旧的证据不支持):整句铺一层浅底(折行时每行各自带圆角),字色、字重都不动,也不画线
? <span className="ev-claim" data-ev-claim-shade={unit.id} style={{ '--ev-soft': meta!.soft } as CSSProperties}>{nodes}</span>
: nodes}
{tag && [...alerts].map((state) => (
@@ -722,10 +740,11 @@ function UnitView({ unit, doc, current, active, panelId, code, tag, badge }: {
aria-label={claimLabel(unit, badge.verdict)}
aria-expanded={open}
aria-controls={open ? panelId : undefined}
// 模型判过的写明谁判的、非确定;模型没判过的(按需还没点、到上限、没跑成)只说这是什么状态——
// 到上限没判的也记着裁判模型,但不能写成「(模型 · 非确定)」,像是它判过一样
// 模型判过的写明谁判的、非确定(证据不足的接着写缺什么);模型没判过的(按需还没点、到上限、没跑成)只说
// 这是什么状态——到上限没判的也记着裁判模型,但不能写成「(模型 · 非确定)」,像是它判过一样
title={isJudged(badge.verdict)
? `${meta.label}${badge.verdict.post_seal ? ` · ${JUDGE_TEXT.postSeal}` : ''}(${badge.verdict.judge || JUDGE_TEXT.judgeModel} · 非确定)`
? `${meta.label}${badge.verdict.post_seal ? ` · ${JUDGE_TEXT.postSeal}` : ''}(${badge.verdict.judge || JUDGE_TEXT.judgeModel} · 非确定)${
verdictMissing(badge.verdict) ? `。${JUDGE_TEXT.missing(verdictMissing(badge.verdict))}` : ''}`
: limitOf(badge.verdict) ? `${meta.label}:${JUDGE_TEXT.limitNotJudged}` : `${meta.label}:${meta.hint}`}
style={segStyle(badge.state)}
>
+201 -13
View File
@@ -4,16 +4,17 @@ import {
} from 'react'
import { createPortal } from 'react-dom'
import { Link } from 'react-router-dom'
import { ArrowLeft, CornerDownRight, ListChecks, Scale, ShieldCheck, X } from 'lucide-react'
import { AlertTriangle, ArrowLeft, CornerDownRight, ListChecks, Scale, ShieldCheck, X } from 'lucide-react'
import clsx from 'clsx'
import { ApiError } from '../api/client'
import { Spinner, StatusBadge, isComposing } from '../components/ui'
import {
EVIDENCE_KIND_STYLE, EVIDENCE_STATE, caliberSourceText, caliberUpgradeText, closestNames, docForeign, entitySources,
evidenceTrace, evidenceValue as valueText, graphDoc, inputSource, integrityFailures, isJudged, isJudgeLimit, judgeLimitWords,
judgeable, limitOf, pointsToSettings,
EVIDENCE_KIND_STYLE, EVIDENCE_STATE, caliberSourceText, caliberUpgradeText, claimBasis, closestNames, docForeign,
entitySources, evidenceTrace, evidenceValue as valueText, graphDoc, inputSource, integrityFailures, isJudged,
isJudgeLimit, judgeFlags, judgeLimitWords, judgeable, judgedVerdicts, limitOf, pointsToSettings,
locatable, locatorText, notableInput, queryOf, queryWindow, quoteWhere, quoteWindow, reasonOf, rewriteOf, sealVerdict,
segName, segmentKind, segmentState, sourceOf, unitText, unitVerdict, verdictState, type SealStatus, type SourceTone,
segName, segmentKind, segmentState, sourceOf, unitText, unitVerdict, verdictMissing, verdictState, type ClaimBasis as Basis,
type SealStatus, type SourceTone,
} from '../lib/evidence'
import { humanizeError } from '../lib/errors'
import { formatDateTime, formatNumber, NONE, shortId } from '../lib/format'
@@ -54,6 +55,9 @@ import { CopyChip } from './Markdown'
* 的徽标,封存之后按需追加的另写「封存后追加」。探索运行里还没判过的句子给「请模型判断这句」;到了上限
* 写明是哪个上限、到哪里调。点句末徽标打开的是整句(kind: 'unit'):句子、模型的解释、挂的依据、封存。
*
* 裁判拆档(JR):证据不足的写明缺什么;裁判模型和写作模型相同、模型不在价格表里时在「模型的解释」下提醒;
* 表名实体步骤列出字段清单;讲方法、讲结构的句子在「挂的依据」下直接列出 SQL 和字段清单。
*
* 四种摆法:side 从右侧弹出(宽屏),drawer 从底部抽出(窄屏),inline 在 360px 的
* 画布右栏里直接栏内展开,dock 放进页面给的一块常驻位置(记录页的「证据」页签)。
* 都不是模态的:开着面板照样能在正文里走。
@@ -387,7 +391,7 @@ function SegmentBody({ panelId, doc, artifact, runId, runClass, seg, unit, block
)}
<JudgePart doc={doc} unit={unit} block={block} runId={runId} runClass={runClass}
onDemand={detail?.unit?.on_demand} pending={pending} />
onDemand={detail?.unit?.on_demand} answered={detail?.unit?.verdict} pending={pending} />
{host.onNode && (trace.producers.length > 0 || trace.consumers.length > 0) && (
// 画布右栏:这段证据经过的节点,点一下在画布上选中并对准
@@ -465,6 +469,8 @@ function ClaimBody({ doc, artifact, runId, runClass, unit, block }: {
const sealLabel = sealed.status !== 'done' ? sealed.label
: verdict && !late && isJudged(verdict) ? JUDGE_TEXT.sealedWithDoc : JUDGE_TEXT.sealedDoc
const cites = (unit.cites ?? []).map((alias) => ({ alias, label: doc.catalog?.[alias]?.label as string | undefined }))
// 讲方法、讲结构的句子:SQL 和字段清单直接列在挂的依据下面。正文不是封存的那份时不列(链是另一份报告的)
const basis = useMemo(() => (foreign ? null : claimBasis(unit, doc)), [foreign, unit, doc])
return (
<>
{foreign && (
@@ -476,6 +482,7 @@ function ClaimBody({ doc, artifact, runId, runClass, unit, block }: {
</Part>
<JudgePart doc={doc} unit={unit} block={block} runId={runId} runClass={runClass}
onDemand={foreign ? undefined : answered?.unit?.on_demand}
answered={foreign ? undefined : answered?.unit?.verdict}
pending={!!runId && !!first && (!slot || slot.status === 'loading')} />
<Part title={JUDGE_TEXT.cites} data-ev-cites="">
{cites.length ? (
@@ -488,6 +495,7 @@ function ClaimBody({ doc, artifact, runId, runClass, unit, block }: {
))}
</ul>
) : <p className="text-dim">{JUDGE_TEXT.noCites}</p>}
{basis && <BasisPart basis={basis} doc={doc} runId={runId} />}
</Part>
<Part title={EVIDENCE_TEXT.seal} data-ev-seal="">
<SealLine status={sealed.status} label={sealLabel} />
@@ -508,13 +516,15 @@ function ClaimBody({ doc, artifact, runId, runClass, unit, block }: {
* 另写「封存后追加」;到上限没判的写明哪个上限、到哪里调;改写过一次的写明原句。探索运行里还没判过的
* (没有判定、未裁判)给「请模型判断这句」。什么都说不上的(正式运行、没开裁判的文档)不画这一节
*/
function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
function JudgePart({ doc, unit, block, runId, runClass, onDemand, answered, pending }: {
doc: EvidenceDocData; unit: EvidenceUnit; block: EvidenceBlock; runId?: string; runClass?: string
/**
* 片段接口答的「这句能不能请模型判断」:有它就照它(运行还没跑完封存、封存被改过这些前端认不出来);
* 老后端没有时按运行类别和文档自己认
*/
onDemand?: EvidenceOnDemand
/** 片段接口答这句时的判定:接口说它「早于当前规则」(outdated),指的就是这一条 */
answered?: EvidenceVerdict
/** 片段接口还在取:先不按运行类别另取一次 */
pending?: boolean
}) {
@@ -532,7 +542,10 @@ function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
const late = !!verdict?.post_seal
const rewrite = rewriteOf(doc, unit.id)
const judgeDoc = !!doc.judge
const canAsk = !!runId && !isJudged(verdict) && (known ? !!known.available : eligible && explore === true)
// 片段接口说这句封存后追加的判定早于当前的裁判规则(旧取值、改造前判的):照常给按钮,旧判定旁边注明「按旧规则判定」。
// 这一页里已经按当前规则重判过(眼前的判定不再是接口答的那一条)就不算了;重判没判成时答复交回的还是那条旧判定
const outdated = known?.reason === 'outdated' && !!known.available && isJudged(verdict) && sameVerdict(verdict, answered)
const canAsk = !!runId && (outdated || (!isJudged(verdict) && (known ? !!known.available : eligible && explore === true)))
// 探索运行里暂时判不了(还没跑完封存、封存被改过、升级前的运行):照接口的原话说为什么没有按钮
const blocked = known && !known.available && BLOCKED.has(known.reason ?? '') ? known.message || '' : ''
// 这一次按需裁判的答复里没有这一句的判定:接口说了为什么(跳过了、触顶)就照说
@@ -542,6 +555,16 @@ function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
: (Array.isArray(res?.limits_hit) && res.limits_hit.find((l) => typeof l === 'string')) || null
const shownLimit = limit ?? resLimit
const judged = isJudged(verdict)
const missing = verdictMissing(verdict)
// 裁判模型的两条提醒(和写作模型相同、不在价格表里):封存后追加的看这一次的答复(答复里有这一句时)、再看证据图里
// 最近一次按需裁判;节点里当场判的看文档的裁判摘要
const replied = res && judgedVerdicts(res)[unit.id] ? res : null
const graph = useEvidence((s) => (runId ? s.graphs[runId] : undefined))
const loadGraph = useEvidence((s) => s.loadGraph)
const needMeta = !!runId && judged && late && !replied && !graph
useEffect(() => { if (needMeta && runId) void loadGraph(runId) }, [needMeta, runId, loadGraph])
const meta0 = (graph?.data?.reports ?? []).find((r) => !report || !r?.node_id || r.node_id === report)?.judge_meta
const flags = judgeFlags(verdict, { doc: doc.judge, reply: replied, meta: meta0 })
// 这一次答复的那句话(message):判定之外接口还想说的,照原话写出来。最要紧的是「运行已经接着跑了,这次的
// 判定没有记进运行记录」——判定照样交回来、也标着封存后追加,只看徽标会以为它记进去了。已经由别处说过的不重复:
// 跳过的那一行就是它、上限的那一框说的就是它、和判定里写的理由一字不差
@@ -573,11 +596,43 @@ function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
</div>
)}
{verdict && (
<p className="font-medium" style={{ color: meta?.color ?? 'var(--text-dim)' }} data-ev-verdict={verdict.status}>
{meta?.glyph && <span aria-hidden className="mr-1">{meta.glyph}</span>}
{meta ? meta.label : JUDGE_TEXT.notClaim}
<div className="flex flex-wrap items-center gap-x-1.5 gap-y-0.5">
<p className="font-medium" style={{ color: meta?.color ?? 'var(--text-dim)' }} data-ev-verdict={verdict.status}>
{meta?.glyph && <span aria-hidden className="mr-1">{meta.glyph}</span>}
{meta ? meta.label : JUDGE_TEXT.notClaim}
</p>
{outdated && (
<span className="chip text-2xs text-faint" data-ev-judge-outdated="" title={known?.message ?? undefined}>
{JUDGE_TEXT.outdated}
</span>
)}
</div>
)}
{missing && (
// 证据不足:裁判写明缺的是什么(「SQL」「字段清单」)。说的是证据,不是结论错了:中性字色
<p className="mt-0.5 [overflow-wrap:anywhere]" data-ev-missing="">
<span className="text-faint">{JUDGE_TEXT.missingLead}</span>{missing}
</p>
)}
{(flags.sameModel || flags.unpriced) && (
// 判定照常给出,但读的人要知道它的分量:左侧一道提醒色的竖线加图标,字用正文色——醒目,不刺眼
<div className="mt-1 space-y-0.5 border-l-2 py-px pl-2 text-2xs leading-relaxed" role="note" data-ev-judge-caution=""
style={{ borderColor: 'var(--st-waiting)' }}>
{flags.sameModel && (
<p className="flex items-start gap-1" data-ev-judge-same=""
title={flags.model ? JUDGE_TEXT.sameModelTitle(flags.writer ?? flags.model) : undefined}>
<AlertTriangle size={11} aria-hidden className="mt-0.5 shrink-0" style={{ color: 'var(--st-waiting)' }} />
<span>{JUDGE_TEXT.sameModel}</span>
</p>
)}
{flags.unpriced && (
<p className="flex items-start gap-1" data-ev-judge-unpriced="" title={JUDGE_TEXT.noPriceTitle}>
<AlertTriangle size={11} aria-hidden className="mt-0.5 shrink-0" style={{ color: 'var(--st-waiting)' }} />
<span>{JUDGE_TEXT.noPrice}</span>
</p>
)}
</div>
)}
{showMessage && (
<p className="mb-1 mt-0.5 text-2xs leading-relaxed" role="note" style={{ color: 'var(--st-waiting)' }} data-ev-judge-message="">
{resMessage}
@@ -588,8 +643,9 @@ function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
<p className="mt-0.5 text-2xs text-faint" data-ev-judge-model="">{JUDGE_TEXT.notJudgedBy(verdict.judge)}</p>
)}
{verdict?.rationale && !shownLimit && (
// 理由是模型写的话:斜体,和系统给的说明分开
<p className="mt-0.5 italic text-dim" data-ev-rationale=""><em>{verdict.rationale}</em></p>
// 理由是模型写的话:斜体,和系统给的说明分开。最长 120 字、可能夹着 SQL 片段和长的字段名:照原样折行,
// 不截断,长串也在栏内断开
<p className="mt-0.5 whitespace-pre-line italic text-dim [overflow-wrap:anywhere]" data-ev-rationale=""><em>{verdict.rationale}</em></p>
)}
{shownLimit && (
<div className="mt-1 rounded border px-2 py-1" role="note" data-ev-judge-limit={shownLimit}
@@ -656,6 +712,9 @@ function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
{asking ? <Spinner size={10} /> : <Scale size={11} aria-hidden />}
{asking ? JUDGE_TEXT.asking : verdict && verdict.reason !== 'on_demand' ? JUDGE_TEXT.askAgain : JUDGE_TEXT.ask}
</button>
{outdated && known?.message && (
<span className="basis-full text-2xs text-dim" data-ev-judge-outdated-why="">{known.message}</span>
)}
<span className="basis-full text-2xs text-faint">{JUDGE_TEXT.askHint}</span>
</div>
)}
@@ -663,6 +722,13 @@ function JudgePart({ doc, unit, block, runId, runClass, onDemand, pending }: {
)
}
/** 两条判定是不是同一条(片段接口答的、页面上叠着的):按取值、理由、裁判模型、规则版本比,不按对象引用 */
function sameVerdict(a: EvidenceVerdict | undefined, b: EvidenceVerdict | undefined): boolean {
if (!a || !b) return false
const sig = (v: EvidenceVerdict) => [v.status, v.rationale ?? '', v.judge ?? '', v.rule_version ?? ''].join('\u0001')
return sig(a) === sig(b)
}
/** 接口说这句暂时判不了、值得告诉人为什么的几种(正式运行、判过了、不是结论句不用说) */
const BLOCKED = new Set(['unsealed', 'seal_broken', 'legacy'])
@@ -735,6 +801,9 @@ function EntityPart({ step, entry, cite, seal, pending, failed, catalog }: {
<span className="text-2xs text-dim">{EVIDENCE_TEXT.tableColumns((step as any).columns, !!step?.is_view)}</span>
)}
</div>
{kind === 'table' && !!step?.fields?.length && (
<FieldList fields={step.fields} more={step.fields_more} masked={step.fields_masked === true} />
)}
{step?.comment && <p className="mt-0.5 text-2xs text-dim">{step.comment}</p>}
{owners.length > 1 && <p className="mt-0.5 text-2xs text-dim">{EVIDENCE_TEXT.ownerTables(owners)}</p>}
{step?.types && !step.type && Object.keys(step.types).length > 0 && (
@@ -769,6 +838,125 @@ function EntityPart({ step, entry, cite, seal, pending, failed, catalog }: {
)
}
/**
* 表的字段清单:证据接口给的 fields(封存范围内的表结构快照,最多 60 个,遮罩的列不列入),多出来的写「另有 N 个」,
* 快照只存了部分表时注明(partial,实体步骤那一节另有这句时不重复)。长名字、长类型都在栏内折行,不截断
*/
function FieldList({ fields, more, partial, masked, label }: {
fields: NonNullable<EvidenceStep['fields']>; more?: number; partial?: boolean; masked?: boolean; label?: (n: number) => string
}) {
const list = fields.filter((f) => f && typeof f.name === 'string' && f.name)
const extra = typeof more === 'number' && Number.isFinite(more) && more > 0 ? more : 0
const total = list.length + extra
return (
<div className="mt-1.5" data-ev-fields="">
<div className="mb-0.5 text-2xs font-medium text-faint">{label ? label(total) : EVIDENCE_TEXT.fields(total)}</div>
<ul className="grid max-h-44 grid-cols-[repeat(auto-fill,minmax(8.5rem,1fr))] gap-x-3 gap-y-px overflow-y-auto rounded bg-bg px-1.5 py-1 text-2xs">
{list.map((f) => (
<li key={f.name} className="min-w-0 [overflow-wrap:anywhere]" data-ev-field={f.name}>
<span className="mono">{f.name}</span>
{f.type && <>{' '}<span className="mono ml-1 text-faint" data-ev-field-type="">{f.type}</span></>}
</li>
))}
</ul>
{extra > 0 && <p className="mt-0.5 text-2xs text-dim" data-ev-fields-more="">{EVIDENCE_TEXT.fieldsMore(extra)}</p>}
{masked && <p className="mt-0.5 text-2xs text-dim" data-ev-fields-masked="">{EVIDENCE_TEXT.fieldsMasked}</p>}
{partial && (
<p className="mt-0.5 text-2xs" style={{ color: 'var(--st-waiting)' }} data-ev-fields-partial="">{EVIDENCE_TEXT.snapshotPartial}</p>
)}
</div>
)
}
/**
* 讲方法、讲结构的句子挂的依据:表的字段清单、字段的类型(取句中那个名字片段的证据链:封存范围内的表结构快照)、
* 对应的 SQL(查询快照工件,取回时后端复验哈希)。直接摆出来,不用再点开句中的名字
*/
function BasisPart({ basis, doc, runId }: { basis: Basis; doc: EvidenceDocData; runId?: string }) {
const report = doc.node_id || undefined
const segments = useEvidence((s) => s.segments)
const sqls = useEvidence((s) => s.sqls)
const loadSegment = useEvidence((s) => s.loadSegment)
const loadSql = useEvidence((s) => s.loadSql)
const entities = [...basis.tables, ...basis.columns]
const artifacts = basis.queries.map((q) => {
const a = doc.catalog?.[q]?.artifact
return typeof a === 'string' && a ? a : ''
})
const segKey = entities.map((e) => e.seg ?? '').join(',')
const artKey = artifacts.join(',')
useEffect(() => {
if (!runId) return
for (const e of entities) if (e.seg) void loadSegment(runId, e.seg, report)
for (const a of artifacts) if (a) void loadSql(a)
// 按内容比:同一句不重复取
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [runId, report, segKey, artKey, loadSegment, loadSql])
if (!runId) return <p className="mt-1.5 text-2xs text-faint" data-ev-basis="none">{JUDGE_TEXT.basisNoRun}</p>
const stepOf = (seg?: string) => {
const slot = seg ? segments[segmentKey(runId, seg, report)] : undefined
const step = slot?.status === 'ok' ? (slot.data?.chain ?? []).find((c) => c?.step === 'entity') : undefined
return { step, pending: !!seg && (!slot || slot.status === 'loading') }
}
return (
<div className="mt-2 space-y-2 border-t pt-2" data-ev-basis="">
{basis.tables.map((t) => {
const { step, pending } = stepOf(t.seg)
return (
<div key={t.alias} data-ev-basis-table={t.alias}>
{step?.fields?.length ? (
<FieldList fields={step.fields} more={step.fields_more} partial={step.snapshot_truncated === true}
masked={step.fields_masked === true} label={(n) => JUDGE_TEXT.basisFieldsCount(t.name, n)} />
) : (
<p className="text-2xs text-faint">
{JUDGE_TEXT.basisFields(t.name)}:{pending ? JUDGE_TEXT.basisPending : JUDGE_TEXT.fieldsMissing}
</p>
)}
</div>
)
})}
{basis.columns.length > 0 && (
<div data-ev-basis-columns="">
<div className="mb-0.5 text-2xs font-medium text-faint">{JUDGE_TEXT.basisColumns}</div>
<ul className="space-y-px text-2xs">
{basis.columns.map((c) => {
const { step, pending } = stepOf(c.seg)
const type = step?.type ?? (step?.types ? Object.values(step.types)[0] : undefined)
return (
<li key={c.alias} className="min-w-0 [overflow-wrap:anywhere]" data-ev-basis-column={c.alias}>
<span className="mono">{c.name}</span>{' '}
<span className="mono ml-1 text-faint">{type || (pending ? '…' : NONE)}</span>
</li>
)
})}
</ul>
</div>
)}
{basis.queries.map((q, i) => {
const slot = artifacts[i] ? sqls[artifacts[i]] : undefined
const sql = slot?.status === 'ok' ? slot.data?.sql ?? null : null
return (
<div key={q} data-ev-basis-sql={q}>
<div className="mb-0.5 flex items-center gap-2">
<span className="min-w-0 flex-1 text-2xs font-medium text-faint">{JUDGE_TEXT.basisSql(q)}</span>
{sql && <CopyChip label={EVIDENCE_TEXT.copySql} text={() => sql} />}
</div>
{sql ? (
<pre className="mono max-h-28 overflow-auto whitespace-pre-wrap rounded bg-bg px-1.5 py-1 text-2xs leading-relaxed text-dim [overflow-wrap:anywhere]"
data-ev-sql="">
{sql}
</pre>
) : (
<p className="text-2xs text-faint">{artifacts[i] && (!slot || slot.status === 'loading') ? JUDGE_TEXT.basisPending : JUDGE_TEXT.sqlMissing}</p>
)}
</div>
)
})}
{basis.more > 0 && <p className="text-2xs text-faint" data-ev-basis-more="">{JUDGE_TEXT.basisMoreQueries(basis.more)}</p>}
</div>
)
}
/**
* 逐字引文:原文所在的文档和片段,引文在原文里的位置高亮(前后带一截上下文)。检索快照哈希对不上时
* 不摆原文——那一段不能当证据。原话对不上的引文也画这一节,只写目录里记着的是哪次检索
File diff suppressed because it is too large Load Diff
+10 -10
View File
@@ -1,8 +1,8 @@
import type { RunEvent, TeamMember, TeamRound, TeamRun, ToolChange } from '../types'
import type { NodeState, RunPhase } from './trace'
import { formatDuration, formatNumber } from '../lib/format'
import { TYPE_LABEL, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel } from '../lib/terms'
import { statsTally } from '../lib/evidence'
import { JUDGE_TEXT, TYPE_LABEL, claimTally, evidenceTally, issuanceLabel, nodeTypeLabel } from '../lib/terms'
import { claimCountsOf, isJudged, statsTally } from '../lib/evidence'
// 泳道数据画布也要用(supervisor 节点要展开成协作矩阵),所以类型放在
// types.ts 里;这里再导出一遍,老引用不用改
@@ -647,16 +647,13 @@ function queryTitle(sql: string, source: string): string {
/** agent 开了 cite_fields 时循环结束后那次结构化抽取 */
const EXTRACT_TITLE = '按出处抽取字段'
/**
* report.checked 带的裁判摘要 → 「结论 4 句(支持 1 · 不支持 1 · 未裁判 2)」,和出具横幅同一种说法。
* 「不是结论句」不算;没有摘要(没开裁判、老后端)返回空串
* report.checked 带的裁判摘要 → 「结论 4 句(有依据 1 · 证据相矛盾 1 · 未裁判 2)」,和出具横幅同一种说法。
* 「不是结论句」不算,旧数据的 unsupported 照样计入;没有摘要(没开裁判、老后端)返回空串
*/
function judgeTally(judge: unknown): string {
const c = judge && typeof judge === 'object' ? (judge as { counts?: Record<string, unknown> }).counts : undefined
if (!c || typeof c !== 'object') return ''
const n = (k: string) => (typeof c[k] === 'number' && Number.isFinite(c[k] as number) ? c[k] as number : 0)
const counts = { supported: n('supported'), partial: n('partial'), unsupported: n('unsupported'), unjudged: n('unjudged'),
uncited: 0 }
return claimTally({ ...counts, total: counts.supported + counts.partial + counts.unsupported + counts.unjudged })
return claimTally(claimCountsOf(c))
}
/** 报告撰写节点请裁判模型判断结论句(四期,claims: judge):units 是这一批几句 */
@@ -1785,14 +1782,17 @@ export function decodeRun(events: RunEvent[], final?: RunFinal, opts?: DecodeOpt
case 'evidence.judged': {
// 探索运行跑完、封存之后,有人点开结论句请模型判断:判定追加在封存之后,不改封存的报告
const verdicts = d.verdicts && typeof d.verdicts === 'object' ? Object.values(d.verdicts as Record<string, any>) : []
const judged = verdicts.filter((v) => ['supported', 'partial', 'unsupported', 'not_a_claim'].includes(v?.status)).length
const judged = verdicts.filter((v) => isJudged(v)).length
const hit: string[] = Array.isArray(d.limits_hit) ? d.limits_hit : []
push({
id: `ej-${seq}`, seq, kind: 'note', nodeId, status: 'done', code: 'evidence_judged',
level: hit.length || (Array.isArray(d.gaps) && d.gaps.length) ? 'warn' : 'info',
title: `封存后按需裁判了 ${formatNumber(judged)} 句结论${hit.length ? ',部分因达到上限未裁判' : ''}`,
sub: '封存后追加 · 模型判断,非确定',
...(d.model ? { detail: [`模型:${d.model}`, ...(Array.isArray(d.gaps) ? d.gaps.map(String) : [])].join('\n') } : {}),
// 裁判模型和写作模型相同、模型不在价格表里:和证据面板「模型的解释」同样的两句提醒
...(d.model ? { detail: [`模型:${d.model}`, ...(d.same_model === true ? [JUDGE_TEXT.sameModel] : []),
...(d.priced === false ? [JUDGE_TEXT.noPrice] : []),
...(Array.isArray(d.gaps) ? d.gaps.map(String) : [])].join('\n') } : {}),
}, nodeId)
break
}
+10 -2
View File
@@ -1,7 +1,7 @@
import { useEffect, useMemo } from 'react'
import { create } from 'zustand'
import { ApiError, api } from '../api/client'
import { graphVerdicts, judgedVerdicts } from '../lib/evidence'
import { graphVerdicts, judgedVerdicts, sqlOf } from '../lib/evidence'
import type {
EvidenceDocData, EvidenceGraph, EvidenceJudgeResult, EvidenceSegmentDetail, EvidenceStats, EvidenceVerdict,
} from '../types'
@@ -44,6 +44,12 @@ interface EvidenceState {
docs: Record<string, Slot<EvidenceDocData>>
graphs: Record<string, Slot<EvidenceGraph>>
segments: Record<string, Slot<EvidenceSegmentDetail>>
/**
* 查询快照里的 SQL,按工件 id 缓存(内容寻址,永久):讲方法、讲结构的句子在「挂的依据」下直接列出。
* 只留 SQL,不留整份快照的行
*/
sqls: Record<string, Slot<{ sql: string | null }>>
loadSql: (artifact: string) => Promise<void>
/** 按工作流:最近一次运行的报告统计。画布没挂着运行时,报告节点卡的章从这里取 */
latest: Record<string, Slot<LatestReports>>
loadDoc: (artifact: string) => Promise<void>
@@ -100,7 +106,7 @@ export const useEvidence = create<EvidenceState>((set, get) => {
* 取一次、记进 table[key];正在取或已经取到的不再发。取的途中这一格被清掉或换了(dropLatest 之后
* 又起了一次),回来的结果作废:不能让早先发出的请求把后来那份盖掉
*/
async function fill<K extends 'docs' | 'graphs' | 'segments' | 'latest'>(
async function fill<K extends 'docs' | 'graphs' | 'segments' | 'latest' | 'sqls'>(
table: K, key: string, fetcher: () => Promise<NonNullable<EvidenceState[K][string]['data']>>, force = false,
) {
const cur = get()[table][key]
@@ -124,6 +130,8 @@ export const useEvidence = create<EvidenceState>((set, get) => {
graphs: {},
segments: {},
latest: {},
sqls: {},
loadSql: (artifact) => fill('sqls', artifact, async () => ({ sql: sqlOf((await api.artifact(artifact))?.content) })),
loadDoc: (artifact) => fill('docs', artifact, async () => {
const res = await api.artifact(artifact)
const doc = asDoc(res?.content)
+44 -6
View File
@@ -755,9 +755,20 @@ export interface EvidenceUnit {
* timeout_s,rationale 以「已到上限(…)」开头)、error、format、missing、on_demand(探索运行按需)
*/
export interface EvidenceVerdict {
status: 'supported' | 'partial' | 'unsupported' | 'not_a_claim' | 'unjudged' | string
/** 理由,最多 60 字 */
/**
* contradicted:证据和原句冲突;insufficient:证据里没有相关信息(missing 写缺的是什么)。
* unsupported 是拆档之前的取值:旧文档、旧事件里保留原样,界面上照旧写「证据不支持」
*/
status: 'supported' | 'partial' | 'contradicted' | 'insufficient' | 'unsupported' | 'not_a_claim' | 'unjudged' | string
/** 理由,最多 120 字(后端在句子边界截断,加「…」) */
rationale?: string
/** insufficient 时缺的是什么:「SQL」「字段清单」「取数的查询」…,由裁判给出 */
missing?: string
/** 按哪一版裁判规则判的(新判的都带;拆档之前判的没有) */
rule_version?: number
/** 判定自己带着的模型提醒(有的话盖过报告、答复、证据图里的) */
same_model?: boolean
priced?: boolean | null
/** 裁判模型 id;探索运行还没判的是 null */
judge?: string | null
/** 封存之后按需追加的(evidence.judged 事件) */
@@ -780,7 +791,8 @@ export interface EvidenceJudgeSummary {
candidates?: number
/** 预筛放掉、没送裁判的结论句 */
screened?: string[]
counts?: Partial<Record<'supported' | 'partial' | 'unsupported' | 'not_a_claim' | 'unjudged', number>>
counts?: Partial<Record<'supported' | 'partial' | 'contradicted' | 'insufficient' | 'unsupported' | 'not_a_claim'
| 'unjudged', number>>
unjudged?: Record<string, number>
limits_hit?: string[]
complete?: boolean
@@ -792,6 +804,8 @@ export interface EvidenceJudgeSummary {
gaps?: string[]
notes?: string[]
same_model?: boolean
/** 撰写这份报告实际用的模型 id(裁判模型和它相同时 same_model 为真) */
writer_model?: string | null
/**
* 不支持的句子交回写作者改写一次(rewrite_once)。units:交回的句子在封存文档里的编号(采用了改写稿时,
* 改掉的句子是 null);sentences:交回的原句;changed:改写稿里新写的句子的编号(没采用时为空);
@@ -822,13 +836,28 @@ export interface EvidenceJudgeResult {
message?: string | null
/** 触顶时怎么调,一个上限一句 */
adjust?: string[]
/** 这一次裁判模型和写作模型相同、模型在不在价格表里、写作模型 id */
same_model?: boolean
priced?: boolean | null
writer_model?: string | null
[key: string]: unknown
}
/** 证据图 reports[].judge_meta:这份报告最近一次按需裁判用的模型 */
export interface EvidenceJudgeMeta {
model?: string | null
same_model?: boolean
priced?: boolean | null
writer_model?: string | null
}
/** 这句能不能「请模型判断」:片段接口按运行的实际情况答(和按需裁判接口同一套条件) */
export interface EvidenceOnDemand {
available?: boolean
/** formal / legacy / not_a_claim / judged / unsealed / seal_broken */
/**
* formal / legacy / not_a_claim / judged / unsealed / seal_broken;outdated 是封存后追加的判定早于当前的裁判规则
* (available 为真,照常给按钮)
*/
reason?: string | null
message?: string | null
}
@@ -875,9 +904,11 @@ export interface EvidenceStats {
quotes?: number
unknown_entities?: number
unverified_entities?: number
/** 四期:claims 为 judge 时追加,候选句按判定的句数 */
/** 四期:claims 为 judge 时追加,候选句按判定的句数(unsupported 是拆档之前的取值,旧文档里才有) */
supported?: number
partial?: number
contradicted?: number
insufficient?: number
unsupported?: number
not_a_claim?: number
unjudged?: number
@@ -947,7 +978,9 @@ export interface EvidenceGraph {
*/
post_seal_verdicts?: Record<string, EvidenceVerdict> | null
/** 封存的裁判摘要(文档的 judge) */
judge?: EvidenceJudgeSummary | null }[]
judge?: EvidenceJudgeSummary | null
/** 最近一次按需裁判用的模型:和写作模型相同没有、在不在价格表里 */
judge_meta?: EvidenceJudgeMeta | null }[]
evidence?: { alias?: string; eid?: string; kind?: string; label?: string; node_id?: string; artifact?: string
sealed?: boolean; cited_by?: string[]; report?: string }[]
edges?: { from: string; to: string; rel: string }[]
@@ -1107,6 +1140,11 @@ export interface EvidenceStep extends Omit<EvidenceInput, 'status'> {
snapshot_truncated?: boolean
/** 好几张表都有的同名字段:各表的类型 */
types?: Record<string, string>
/** 表:字段清单和类型(封存范围内的表结构快照,最多 60 个,遮罩的列不列入);多出来的个数 */
fields?: { name?: string; type?: string | null }[]
fields_more?: number
/** 有列按遮罩没列进字段清单 */
fields_masked?: boolean
/** 表:几个字段、是不是视图;表或字段的注释 */
is_view?: boolean
comment?: string
+1 -1
View File
@@ -84,7 +84,7 @@ const ORDER = [
// 拆开的两项看 SPLIT 里各份的 est。2026-09-29 八道并行、关浏览器不干等之后的实测
const ESTIMATE = {
stream: 173, chat: 160, studio: 149, manage: 145, runs: 112, publish: 92, shell: 87,
evidence: 75, 'ui-kit': 67, ui: 61, guards: 8, 'canvas-layout': 2, copy: 2, tokens: 1, trace: 1, decode: 1,
evidence: 85, 'ui-kit': 70, ui: 61, guards: 8, 'canvas-layout': 2, copy: 2, tokens: 1, trace: 1, decode: 1,
}
// 最慢的两项按段拆成几份。用的是它们自己的段过滤:最后一份用 *_SKIP 兜住「其余全部」,以后新加的段
// 不用改这里也跑得到(拆出去的那几份要是一段都没对上,那一份会报「一项都没跑」)。
+394 -8
View File
@@ -33,7 +33,10 @@ const fxe = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-
// 第四期:结论句裁判。文档由 compose_doc 真跑,候选句由 judge.prepare 真跑,判定是剧本(模拟裁判模型),
// 经 judge 的 summarize / apply_judgement 写回:正式运行(节点里判好的)、探索运行(按需)、没开裁判的探索运行
const fxj = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-judge.json`, 'utf8'))
const JUDGE_SETS = [fxj.formal, fxj.explore, fxj.plain]
// 裁判拆档(JR):证据相矛盾、证据不足(写明缺什么)、讲方法的句子、裁判模型和写作模型相同、模型不在价格表里。
// 文档由 compose_doc 真跑、证据链由 api/evidence.py 的 _chain 真跑;判定和表名实体步骤的 fields 照前后端接口约定写
const fxv = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-verdict.json`, 'utf8'))
const JUDGE_SETS = [fxj.formal, fxj.explore, fxj.plain, fxv.formal, fxv.explore]
/** 夹具里某段文字是哪个片段:检查按文字认片段,重新生成夹具时编号变了也不用改 */
const segOf = (doc, text) => doc.blocks.flatMap((b) => b.units.flatMap((u) => u.segments)).find((s) => s.text === text)?.id
@@ -61,7 +64,7 @@ const browser = await chromium.launch({ executablePath: CHROME })
* 夹具里按句子备好了答复的用夹具的;别的句子给一条「有依据」。正式运行回 409
*/
function judgeReply(set, body) {
if (set === fxj.formal) return { status: 409, json: { detail: '正式运行的结论句已在报告撰写节点内完成裁判,结果随报告一起封存,不能再按需裁判', code: 'evidence_judge_formal' } }
if (set === fxj.formal || set === fxv.formal) return { status: 409, json: { detail: '正式运行的结论句已在报告撰写节点内完成裁判,结果随报告一起封存,不能再按需裁判', code: 'evidence_judge_formal' } }
const unit = body?.units?.[0]
const named = Object.entries(set.units).find(([, u]) => u === unit)?.[0]
if (named && set.judge?.[named]) return { json: set.judge[named] }
@@ -92,13 +95,20 @@ async function routed(page, { judge = null, segPatch = null } = {}) {
// 四期的报告工件:问数据页的答案(Output → EvidenceField)按工件 id 取文档
const jart = JUDGE_SETS.find((x) => url.pathname === `/api/artifacts/${x.doc_artifact}`)
if (jart) return r.fulfill({ json: { id: jart.doc_artifact, content: jart.doc } })
// 裁判拆档:讲方法的句子在「挂的依据」下列出 SQL,取的是查询快照工件
const vsnap = Object.entries(fxv.snapshots).find(([, x]) => url.pathname === `/api/artifacts/${x.artifact}`)
if (vsnap) {
hits.push(`artifact:V:${vsnap[0]}`)
return r.fulfill({ json: { id: vsnap[1].artifact, content: vsnap[1].content } })
}
if (jset) {
const seg = url.pathname.match(/\/evidence\/segments\/([^/]+)$/)
if (seg) {
const id = decodeURIComponent(seg[1])
hits.push(`j:${id}`)
// 判过之后片段接口按最新的 evidence.judged 叠上判定(封存后追加):夹具里备着判过那一句的答复
const after = jset.segments_after && judged.has(`${jset.run_id}:${jset.units.bad}`) ? jset.segments_after : jset.segments
const trigger = jset.units.bad ?? jset.units.method
const after = jset.segments_after && judged.has(`${jset.run_id}:${trigger}`) ? jset.segments_after : jset.segments
let body = structuredClone(after[id])
if (body && segPatch) body = segPatch(jset, body)
return body ? r.fulfill({ json: body })
@@ -1214,7 +1224,7 @@ await section('studio', '报告撰写节点:画布认得、检查器能配(R
const shown = await p.locator('[data-field="claims"] select').evaluate((el) => el.options[el.selectedIndex]?.textContent ?? '')
check('config 里写着无法识别的值:下拉照实显示,不装成 off', shown.includes('sometimes') && shown.includes('无法识别'), shown)
// 四期:选模型裁判,高级选项里出现结论句裁判的子配置(裁判模型、三项上限各自能设不限、改写一次、证据不支持时)
// 四期:选模型裁判,高级选项里出现结论句裁判的子配置(裁判模型、三项上限各自能设不限、改写一次、证据相矛盾时)
check('没选模型裁判时不出结论句裁判那一组', await p.locator('[data-field="judge"]').count() === 0)
await p.locator('[data-field="claims"] select').selectOption('judge')
check('选「请模型逐句判断」写进 config.claims = judge', (await cfgOf())?.claims === 'judge')
@@ -1223,12 +1233,16 @@ await section('studio', '报告撰写节点:画布认得、检查器能配(R
const jlabel = await jf.locator('label').first().innerText().catch(() => '')
check('高级选项里出现「结论句裁判」一组(中文标签)', await jf.count() === 1 && jlabel.includes('结论句裁判'), jlabel)
const keys = await jf.locator('[data-judge-key]').evaluateAll((els) => els.map((e) => e.getAttribute('data-judge-key')))
check('……裁判模型、三项上限、改写一次、证据不支持时', keys.join(',') === 'model,max_claims,max_cost_usd,timeout_s,rewrite_once,on_unsupported',
check('……裁判模型、三项上限、改写一次、证据相矛盾时', keys.join(',') === 'model,max_claims,max_cost_usd,timeout_s,rewrite_once,on_unsupported',
keys.join(','))
const labels = await jf.innerText()
// 细节只报缺了哪几个:整段文字里有沙箱的接入名(下拉的选项),日志常被贴进报告
const missingLabels = ['裁判模型', '最多裁判句数', '金额上限(美元)', '时长上限(秒)', '退回改写一次', '证据不支持时'].filter((t) => !labels.includes(t))
// 细节只报缺了哪几个:整段文字里有沙箱的接入名(下拉的选项),日志常被贴进报告。裁判拆档后 on_unsupported
// 管的是证据相矛盾(键名不变),标签跟着改
const missingLabels = ['裁判模型', '最多裁判句数', '金额上限(美元)', '时长上限(秒)', '退回改写一次', '证据相矛盾时'].filter((t) => !labels.includes(t))
check('……每一项都有中文标签', !missingLabels.length, `缺:${missingLabels.join('、')}`)
const onHint = await jf.locator('[data-judge-key="on_unsupported"] [data-judge-hint]').innerText().catch(() => '')
check('……「证据相矛盾时」下面写明证据不足的句子最多降档、不会因此不予出具', onHint.includes('证据不足的句子最多降档')
&& onHint.includes('不会因此不予出具'), onHint)
check('……没写的上限说明跟着设置里的默认值', (await jf.locator('[data-judge-key="max_cost_usd"] [data-judge-note]').innerText()).includes('默认值'))
const cost = jf.locator('[data-judge-key="max_cost_usd"]')
await cost.locator('input[type="checkbox"]').click()
@@ -1277,7 +1291,7 @@ await section('studio', '报告撰写节点:画布认得、检查器能配(R
check('发布前修复的预览:judge.max_cost_usd 写成「结论句裁判 · 金额上限(美元)」', previews[0] === '结论句裁判 · 金额上限(美元)', previews[0])
check('……null 写「不限」(没写才是「(空)」)、金额带单位', previews[1] === '不限' && previews[2] === '0.05 美元', JSON.stringify(previews.slice(1, 3)))
check('……补预算按整份 judge 记时一键一行', previews[3] === '裁判模型:x;金额上限(美元):不限', previews[3])
check('……证据不支持时写选项文字', previews[4] === '不予出具', previews[4])
check('……证据相矛盾时写选项文字', previews[4] === '不予出具', previews[4])
check('……别处的 model 不借用裁判的叫法', previews[5] === '工作流默认设置 · model', previews[5])
if (SHOTS) {
await jf.scrollIntoViewIfNeeded()
@@ -2108,6 +2122,378 @@ await section('j-stream', '四期:问数据页、画布右栏的横幅(Outpu
await s.ctx.close()
})
// ---------------------------------------------------------------------------
// 裁判拆档(JR):证据相矛盾 / 证据不足、缺什么、裁判模型的两条提醒、理由 120 字、字段清单、讲方法的句子
// ---------------------------------------------------------------------------
const V = '#evidence-verdict'
const VN = '#evidence-verdict-narrow'
const VX = '#evidence-verdict-explore'
/** 一组句末徽标的样子:判定、字形、计算色、读屏名、悬停说明,外加整句的浅底(没有浅底是 null) */
const badgeLook = (pg, sel, units) => pg.evaluate(({ sel, units }) => Object.fromEntries(Object.entries(units).map(([k, id]) => {
const el = document.querySelector(`${sel} [data-ev-claim="${id}"]`)
if (!el) return [k, null]
const shade = document.querySelector(`${sel} [data-ev-claim-shade="${id}"]`)
return [k, { verdict: el.getAttribute('data-ev-verdict'), glyph: el.textContent, color: getComputedStyle(el).color,
label: el.getAttribute('aria-label'), title: el.getAttribute('title'),
shade: shade ? getComputedStyle(shade).backgroundColor : null }]
})), { sel, units })
const softOf = (pg, token) => tokenColor(pg, 'backgroundColor', `var(${token})`)
const firstNumber = (doc, uid) => doc.blocks.flatMap((b) => b.units).find((x) => x.id === uid).segments.find((s) => s.kind === 'number').id
await section('jr', '裁判拆档:证据相矛盾、证据不足四通道分得开,证据不足不用红、不和未裁判混;n / N 先走相矛盾、再走证据不足', async () => {
const u = fxv.formal.units
await page.waitForSelector(`${V} [data-ev-claim="${u.insuff}"]`, { timeout: 4000 }).catch(() => {})
const look = await badgeLook(page, V, u)
check('新取值各挂各的徽标:证据相矛盾、证据不足(讲方法的那句也判成证据相矛盾)',
look.contra?.verdict === 'contradicted' && look.insuff?.verdict === 'insufficient' && look.method?.verdict === 'contradicted',
JSON.stringify(Object.fromEntries(Object.entries(look).map(([k, v]) => [k, v?.verdict]))))
check('字形:证据相矛盾 !、证据不足 ○,和未裁判的 ? 分得开', look.contra?.glyph === '!' && look.insuff?.glyph === '○'
&& look.limit?.glyph === '?' && look.insuff.glyph !== look.limit.glyph,
[look.contra?.glyph, look.insuff?.glyph, look.limit?.glyph].join(' '))
const [failed, waiting, neutral, done] = await Promise.all(['--st-failed', '--st-waiting', '--st-cancelled', '--st-done']
.map((t) => colorOf(page, t)))
check('颜色令牌:证据相矛盾 --st-failed(同原来的证据不支持),证据不足 --st-cancelled(中性)',
look.contra?.color === failed && look.method?.color === failed && look.insuff?.color === neutral,
JSON.stringify({ got: [look.contra?.color, look.insuff?.color], want: [failed, neutral] }))
check('证据不足不用红,也不用提醒色和确定性的绿', !!look.insuff && ![failed, waiting, done].includes(look.insuff.color), look.insuff?.color)
const soft = await softOf(page, '--st-failed-soft')
check('证据相矛盾的整句铺浅底(--st-failed-soft)', look.contra?.shade === soft && look.method?.shade === soft && soft !== 'rgba(0, 0, 0, 0)',
JSON.stringify({ contra: look.contra?.shade, method: look.method?.shade, soft }))
check('证据不足不铺底(中性样式)', look.insuff?.shade === null, String(look.insuff?.shade))
check('读屏名:证据相矛盾', look.contra?.label === '结论句「各渠道的订单数相加与总订单数一致。」,模型判断:证据相矛盾', look.contra?.label)
check('读屏名:证据不足接着说缺什么', look.insuff?.label === `结论句「增长主要来自新客首单,老客复购持平。」,模型判断:证据不足,缺少:${fxv.missing}`,
look.insuff?.label)
check('悬停说明:证据不足写明谁判的、非确定、缺什么', look.insuff?.title === `模型判断:证据不足(${fxv.model} · 非确定)。缺少:${fxv.missing}`,
look.insuff?.title)
const tally = await page.locator(`${V} [data-evidence-claims]`).innerText().catch(() => '')
check('证据条的结论句计数带上证据相矛盾、证据不足', tally.includes('结论 6 句(有依据 1 · 部分有依据 1 · 证据相矛盾 2 · 证据不足 1 · 未裁判 1)'), tally)
const summary = await page.locator(`${V} [data-evidence-summary]`).innerText()
check('读屏摘要的按键说明:证据有问题的句子 n 也跳到,证据不足的排在最后', summary.includes('证据有问题的句子(证据不足的排在最后)'), summary)
// n / N:第 1 轮是证据相矛盾、部分有依据(按正文顺序),第 2 轮是证据不足——它在正文里排第二,照样最后才到
await page.locator(`${V} [data-seg="${fxv.formal.segs.gmv}"]`).focus()
const jumps = []
for (let i = 0; i < 5; i++) {
await page.keyboard.press('n')
jumps.push(await activeKey(page))
}
const want = [u.contra, u.partial, u.method, u.insuff, u.contra].map((x) => `claim:${x}`)
check('n:先按正文顺序走证据相矛盾、部分有依据,再走证据不足,走完接回开头', jumps.join(',') === want.join(','), jumps.join(','))
const back = []
for (let i = 0; i < 2; i++) {
await page.keyboard.press('N')
back.push(await activeKey(page))
}
check('N:倒着走,从第一处退到证据不足(最后一处),再退到第 1 轮的最后一处', back.join(',') === `claim:${u.insuff},claim:${u.method}`,
back.join(','))
check('只是走动不打开面板', await panel(page).count() === 0)
// 拆档之前的取值:探索运行证据图里的 unsupported 照旧显示「证据不支持」,样式同证据相矛盾
const x = fxv.explore.units
await page.waitForSelector(`${VX} [data-ev-claim="${x.contra}"][data-ev-verdict="unsupported"]`, { timeout: 4000 }).catch(() => {})
const old = (await badgeLook(page, VX, { contra: x.contra, insuff: x.insuff }))
check('旧取值 unsupported:照旧叫「证据不支持」(读屏名、悬停说明)', old.contra?.verdict === 'unsupported'
&& old.contra.label?.endsWith(',模型判断:证据不支持,封存后追加') && old.contra.title?.startsWith('模型判断:证据不支持 · 封存后追加'),
JSON.stringify(old.contra))
check('……样式同证据相矛盾:!、--st-failed、整句浅底', old.contra?.glyph === '!' && old.contra.color === failed && old.contra.shade === soft,
JSON.stringify(old.contra))
check('……封存后追加的证据不足同样是 ○、中性色', old.insuff?.verdict === 'insufficient' && old.insuff.glyph === '○' && old.insuff.color === neutral,
JSON.stringify(old.insuff))
const xt = await page.locator(`${VX} [data-evidence-claims]`).innerText().catch(() => '')
check('……计数里旧取值照样算进去,和证据不足并列', xt.includes('结论 7 句(证据不支持 1 · 证据不足 1 · 未裁判 5)'), xt)
if (SHOTS) {
for (const theme of ['dark', 'light']) {
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
await page.waitForTimeout(200)
await page.locator(V).screenshot({ path: `${SHOTS}/jr-doc-${theme}.png` })
await page.locator(VX).screenshot({ path: `${SHOTS}/jr-doc-explore-${theme}.png` })
}
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
}
})
await section('jr-panel', '裁判拆档:面板写明缺什么、裁判模型的两条提醒、理由 120 字不截断、字段清单、讲方法的句子直接看到 SQL 和字段清单', async () => {
const r = await open('/ui-harness.html?evidence=1')
const pg = r.page
const p = pg.locator('[data-evidence-panel]')
const u = fxv.formal.units
const [failed, waiting, neutral] = await Promise.all(['--st-failed', '--st-waiting', '--st-cancelled'].map((t) => colorOf(pg, t)))
await badge(pg, V, u.insuff).click()
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="insufficient"]')
check('面板:证据不足(字形 ○)', (await p.locator('[data-ev-verdict]').innerText()).trim() === '○模型判断:证据不足',
await p.locator('[data-ev-verdict]').innerText())
const head = await p.locator('header [data-ev-badge]').evaluate((el) => ({ code: el.getAttribute('data-ev-badge'), color: getComputedStyle(el).color }))
.catch(() => null)
check('……面板标题的徽标是中性色,不用红', head?.code === 'insufficient' && head.color === neutral && head.color !== failed, JSON.stringify(head))
const missing = await p.locator('[data-ev-judge] [data-ev-missing]').innerText().catch(() => '')
check('……「模型的解释」里写明缺什么', missing === `缺少:${fxv.missing}`, missing)
const same = await p.locator('[data-ev-judge] [data-ev-judge-same]').innerText().catch(() => '')
const unpriced = await p.locator('[data-ev-judge] [data-ev-judge-unpriced]').innerText().catch(() => '')
check('裁判模型和写作模型相同:「模型的解释」下写「裁判模型和写作模型相同,结果仅供参考」', same === '裁判模型和写作模型相同,结果仅供参考', same)
check('模型不在价格表里:写「该模型不在价格表中,金额上限不生效」', unpriced === '该模型不在价格表中,金额上限不生效', unpriced)
const caution = await p.locator('[data-ev-judge-caution]').evaluate((el) => {
const cs = getComputedStyle(el)
return { border: cs.borderLeftColor, width: cs.borderLeftWidth, style: cs.borderLeftStyle, color: cs.color, bg: cs.backgroundColor,
icon: getComputedStyle(el.querySelector('svg')).color }
}).catch(() => null)
check('……醒目但不刺眼:左侧一道 2px 提醒色竖线、图标用提醒色,字不用红、不铺底', !!caution && caution.border === waiting
&& caution.width === '2px' && caution.style === 'solid' && caution.icon === waiting && caution.color !== failed
&& caution.bg === 'rgba(0, 0, 0, 0)', JSON.stringify(caution))
const sameTitle = await p.locator('[data-ev-judge-same]').getAttribute('title').catch(() => '')
check('……悬停写明是哪个模型', (sameTitle ?? '').includes(fxv.model), sameTitle ?? '')
// 没判的句子、别的报告不挂
await badge(pg, V, u.limit).click()
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="unjudged"]')
check('到上限没判的句子不挂这两条提醒(模型没看过这句)', await p.locator('[data-ev-judge-caution]').count() === 0)
await badge(pg, J, fxj.formal.units.bad).click()
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]')
check('对照:裁判模型和写作模型不同、模型在价格表里的报告不挂提醒', await p.locator('[data-ev-judge-caution]').count() === 0)
check('……旧取值的面板照旧写「证据不支持」', (await p.locator('[data-ev-verdict]').innerText()).includes('模型判断:证据不支持'))
// 理由最长 120 字:整句显示,侧边面板里折行、不溢出
await badge(pg, V, u.contra).click()
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="contradicted"]')
const why = await p.locator('[data-ev-rationale]').evaluate((el) => {
const cs = getComputedStyle(el)
return { text: el.textContent, sw: el.scrollWidth, cw: el.clientWidth, ellipsis: cs.textOverflow, clamp: cs.webkitLineClamp,
overflow: cs.overflow, lines: Math.round(el.getBoundingClientRect().height / parseFloat(cs.lineHeight)) }
}).catch(() => null)
check('理由 120 字整句显示,不截断', why?.text === fxv.long_rationale && [...fxv.long_rationale].length === 120, why?.text)
check('……在侧边面板里折成几行、不溢出、不加省略号', !!why && why.sw <= why.cw && why.ellipsis !== 'ellipsis'
&& (why.clamp === 'none' || why.clamp === '') && why.lines >= 3, JSON.stringify(why))
check('证据相矛盾:面板写「模型判断:证据相矛盾」,不写缺什么', (await p.locator('[data-ev-verdict]').innerText()).trim() === '!模型判断:证据相矛盾'
&& await p.locator('[data-ev-missing]').count() === 0)
const orders = fxv.formal.segments[fxv.formal.segs.orders].chain.find((c) => c.step === 'entity')
const wantFields = orders.fields.map((f) => [f.name, f.type])
// 讲方法的句子:「挂的依据」下直接列出 SQL 和字段清单(先于点开句中的表名:取数只能来自这一块)
const hitsBefore = r.hits.length
check('……此前没取过句中表名的片段', !r.hits.includes(`j:${fxv.formal.segs.orders}`))
await badge(pg, V, u.method).click()
await pg.waitForSelector('[data-evidence-panel] [data-ev-cites] [data-ev-basis]')
// SQL、字段清单没画出来时也往下走:让下面几条断言自己报没通过
await pg.waitForSelector('[data-evidence-panel] [data-ev-cites] [data-ev-basis] [data-ev-sql]', { timeout: 3000 }).catch(() => {})
await pg.waitForSelector('[data-evidence-panel] [data-ev-basis-table] [data-ev-field]', { timeout: 3000 }).catch(() => {})
const basis = p.locator('[data-ev-cites] [data-ev-basis]')
const sqls = await basis.locator('[data-ev-basis-sql]').evaluateAll((els) => els.map((e) =>
[e.getAttribute('data-ev-basis-sql'), e.querySelector('[data-ev-sql]')?.textContent ?? '']))
check('讲方法的句子:「挂的依据」下直接列出 SQL(直接挂的 Q1,再补表名出现过的 Q2)',
JSON.stringify(sqls) === JSON.stringify([['Q1', fxv.snapshots.Q1.content.sql], ['Q2', fxv.snapshots.Q2.content.sql]]), JSON.stringify(sqls))
const bf = await basis.locator('[data-ev-basis-table="t:orders"] [data-ev-field]').evaluateAll((els) => els.map((e) => e.getAttribute('data-ev-field')))
check('……表的字段清单也直接列出', JSON.stringify(bf) === JSON.stringify(wantFields.map(([n]) => n)), JSON.stringify(bf))
const bhead = await basis.locator('[data-ev-basis-table="t:orders"]').innerText().catch(() => '')
check('……写明是哪张表的字段清单', bhead.startsWith(`表 orders 的字段清单(${wantFields.length} 个)`), bhead.split('\n')[0])
const bc = await basis.locator('[data-ev-basis-column]').allInnerTexts()
check('……挂的字段写类型', bc.some((t) => t.includes('orders.status') && t.includes('VARCHAR(12)'))
&& bc.some((t) => t.includes('orders.created_at') && t.includes('TIMESTAMP')), bc.join(' | '))
check('……SQL 带复制', await basis.locator('[data-ev-basis-sql="Q1"] button', { hasText: '复制 SQL' }).count() === 1)
// 这一页只有「挂的依据」会取查询快照工件(完整快照要点按钮);这之前没点开过句中的表名,字段清单那次片段请求只能是
// 它自己发的。Q1、Q2 的 SQL 在前面几句(只挂了整份查询)打开时已经取过,缓存着不再取
const fresh = r.hits.slice(hitsBefore)
check('……SQL 取自查询快照工件,字段清单取自表名片段的证据链', r.hits.includes('artifact:V:Q1') && r.hits.includes('artifact:V:Q2')
&& fresh.includes(`j:${fxv.formal.segs.orders}`), fresh.join(','))
check('……判定的理由里提到的 created_at 在字段清单里看得到', bf.includes('created_at'))
await badge(pg, V, u.insuff).click()
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="insufficient"]')
await pg.waitForSelector('[data-evidence-panel] [data-ev-basis] [data-ev-sql]').catch(() => {})
const insuffSql = await p.locator('[data-ev-basis] [data-ev-basis-sql]').evaluateAll((els) => els.map((e) => e.getAttribute('data-ev-basis-sql')))
check('只挂了整份查询的句子(证据不足的那句):同样列出 SQL,读的人看得出缺的是什么', insuffSql.join(',') === 'Q1', insuffSql.join(','))
await badge(pg, V, u.ok).click()
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge="supported"]')
await pg.waitForTimeout(200)
check('对照:挂了数字的结论句只列依据的编号,不另列 SQL', await p.locator('[data-ev-basis]').count() === 0
&& (await p.locator('[data-ev-cites]').innerText()).includes('Q1'))
// 表名实体步骤:字段清单和类型
await pg.locator(`${V} [data-seg="${fxv.formal.segs.orders}"]`).click()
await pg.waitForSelector('[data-evidence-panel] [data-ev-entity] [data-ev-entity-queries]')
// 字段清单没画出来时也往下走:让下面两条断言自己报没通过,而不是整节在等待上出错
await pg.waitForSelector('[data-evidence-panel] [data-ev-entity] [data-ev-fields]', { timeout: 3000 }).catch(() => {})
const fields = await p.locator('[data-ev-entity] [data-ev-field]').evaluateAll((els) => els.map((e) =>
[e.getAttribute('data-ev-field'), e.querySelector('[data-ev-field-type]')?.textContent ?? '']))
check('表名实体步骤列出字段清单和类型(照证据接口的 fields,顺序不变)', JSON.stringify(fields) === JSON.stringify(wantFields), JSON.stringify(fields))
const fhead = await p.locator('[data-ev-entity] [data-ev-fields]').innerText().catch(() => '')
check('……写明一共几个字段;没有多出来的不写「另有」', fhead.startsWith(`字段清单(${wantFields.length} 个)`)
&& await p.locator('[data-ev-fields-more]').count() === 0, fhead.split('\n')[0])
check('没有运行时报错(裁判拆档 · 面板)', r.errors.length === 0, r.errors.join(' | '))
if (SHOTS) {
for (const theme of ['dark', 'light']) {
await pg.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
for (const [name, unit] of [['insufficient', u.insuff], ['contradicted', u.contra], ['method', u.method]]) {
await badge(pg, V, unit).click()
await pg.waitForTimeout(100)
if (await p.count() === 0) await badge(pg, V, unit).click()
await pg.waitForSelector('[data-evidence-panel] [data-ev-judge]')
await pg.waitForTimeout(300)
await pg.screenshot({ path: `${SHOTS}/jr-panel-${name}-${theme}.png` })
}
await pg.locator(`${V} [data-seg="${fxv.formal.segs.orders}"]`).click()
await pg.waitForSelector('[data-evidence-panel] [data-ev-fields]')
await pg.waitForTimeout(300)
await pg.screenshot({ path: `${SHOTS}/jr-panel-fields-${theme}.png` })
await pg.keyboard.press('Escape')
}
await pg.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
}
await r.ctx.close()
// 字段多于 60 个、表结构快照只存了部分表
const more = await open('/ui-harness.html?evidence=1', { segPatch: (set, body) => (set === fxv.formal && body?.segment?.id === fxv.formal.segs.orders
? { ...body, chain: body.chain.map((c) => (c.step === 'entity' ? { ...c, fields: c.fields.slice(0, 5), fields_more: 55, snapshot_truncated: true,
fields_masked: true } : c)) }
: body) })
const mp = more.page.locator('[data-evidence-panel]')
await more.page.locator(`${V} [data-seg="${fxv.formal.segs.orders}"]`).click()
await more.page.waitForSelector('[data-evidence-panel] [data-ev-fields-more]').catch(() => {})
check('字段多于 60 个:列出的之外写「另有 N 个」', (await mp.locator('[data-ev-fields-more]').innerText().catch(() => '')) === '另有 55 个字段未列出',
await mp.locator('[data-ev-fields-more]').innerText().catch(() => '(没写)'))
check('……总数按列出的加另有的算', (await mp.locator('[data-ev-entity] [data-ev-fields]').innerText().catch(() => '')).startsWith('字段清单(60 个)')
&& await mp.locator('[data-ev-entity] [data-ev-field]').count() === 5)
check('有列按遮罩没列出(fields_masked):照实说一句', (await mp.locator('[data-ev-entity] [data-ev-fields-masked]').innerText().catch(() => ''))
=== '部分字段已按数据源的遮罩设置隐藏,未列出')
await badge(more.page, V, u.method).click()
await more.page.waitForSelector('[data-evidence-panel] [data-ev-basis-table] [data-ev-fields-partial]').catch(() => {})
check('讲方法的句子:表结构快照只存了部分表时,字段清单下照实注明', (await mp.locator('[data-ev-basis-table] [data-ev-fields-partial]')
.innerText().catch(() => '')).includes('表结构快照仅包含部分表'))
check('……另有的字段也写出来', (await mp.locator('[data-ev-basis-table] [data-ev-fields-more]').innerText().catch(() => '')) === '另有 55 个字段未列出')
await more.ctx.close()
// 探索运行:封存后追加的证据不足、按需裁判答复里的 same_model / priced
const x = fxv.explore.units
const ex = await open('/ui-harness.html?evidence=1')
const xp = ex.page.locator('[data-evidence-panel]')
await ex.page.waitForSelector(`${VX} [data-ev-claim="${x.insuff}"][data-ev-verdict="insufficient"]`, { timeout: 4000 }).catch(() => {})
await badge(ex.page, VX, x.insuff).click()
await ex.page.waitForSelector('[data-evidence-panel] [data-ev-judge="insufficient"]')
await ex.page.waitForTimeout(300)
check('封存后追加的证据不足:同样写明缺什么', (await xp.locator('[data-ev-missing]').innerText().catch(() => '')) === `缺少:${fxv.missing}`)
check('……证据图里最近一次按需裁判的模型和写作模型不同、在价格表里:不挂提醒', await xp.locator('[data-ev-judge-caution]').count() === 0)
await badge(ex.page, VX, x.method).click()
await xp.locator('[data-ev-judge-ask]').click()
await ex.page.waitForSelector('[data-evidence-panel] [data-ev-judge="contradicted"]').catch(() => {})
check('按需裁判回来:讲方法的那句判成证据相矛盾,正文徽标跟着变', await badge(ex.page, VX, x.method).getAttribute('data-ev-verdict') === 'contradicted')
check('……这一次的答复说裁判模型和写作模型相同、不在价格表里:两条提醒都挂上',
(await xp.locator('[data-ev-judge-same]').innerText().catch(() => '')) === '裁判模型和写作模型相同,结果仅供参考'
&& (await xp.locator('[data-ev-judge-unpriced]').innerText().catch(() => '')) === '该模型不在价格表中,金额上限不生效')
await ex.page.keyboard.press('Escape')
await ex.page.waitForTimeout(150)
await ex.page.locator(`${VX} [data-seg="${firstNumber(fxv.explore.doc, x.ok)}"]`).focus()
const xj = []
for (let i = 0; i < 3; i++) {
await ex.page.keyboard.press('n')
xj.push(await activeKey(ex.page))
}
check('n:旧的证据不支持、证据相矛盾在前,证据不足在后(它在正文里更靠前也一样)',
xj.join(',') === [x.contra, x.method, x.insuff].map((k) => `claim:${k}`).join(','), xj.join(','))
// 封存后追加的旧判定(拆档之前的 unsupported):片段接口给 on_demand.reason = outdated、available 为真——照常给按钮,
// 旧判定旁边注明「按旧规则判定」;按当前规则重判之后注明和按钮都收起
const outdatedMsg = fxv.explore.segments[fxv.explore.doc.blocks.flatMap((b) => b.units).find((u2) => u2.id === x.contra)
.segments.find((sg) => sg.kind !== 'structural').id].unit.on_demand
// 面板可能还开着、正停在这一句上(上面 n 跟着走过来):点一下是收起,没开出来就再点一次
await badge(ex.page, VX, x.contra).click()
await ex.page.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]', { timeout: 1500 }).catch(() => {})
if (await xp.locator('[data-ev-judge="unsupported"]').count() === 0) await badge(ex.page, VX, x.contra).click()
await ex.page.waitForSelector('[data-evidence-panel] [data-ev-judge="unsupported"]', { timeout: 4000 }).catch(() => {})
await ex.page.waitForSelector('[data-evidence-panel] [data-ev-judge-outdated]').catch(() => {})
const chip = xp.locator('[data-ev-judge-outdated]')
check('旧判定(片段接口说 outdated):旁边注明「按旧规则判定」,悬停是接口的原话', outdatedMsg?.reason === 'outdated'
&& (await chip.innerText().catch(() => '')) === '按旧规则判定' && await chip.getAttribute('title').catch(() => '') === outdatedMsg.message,
JSON.stringify(outdatedMsg))
check('……照常给按钮(再次请模型判断),按钮旁写明为什么', (await xp.locator('[data-ev-judge-ask]').innerText().catch(() => '')).includes('再次请模型判断')
&& (await xp.locator('[data-ev-judge-outdated-why]').innerText().catch(() => '')) === outdatedMsg.message)
await xp.locator('[data-ev-judge-ask]').click().catch(() => {})
await ex.page.waitForSelector('[data-evidence-panel] [data-ev-judge="contradicted"]').catch(() => {})
check('……按当前规则重判:判成证据相矛盾,「按旧规则判定」和按钮都收起', await badge(ex.page, VX, x.contra).getAttribute('data-ev-verdict') === 'contradicted'
&& await xp.locator('[data-ev-judge-outdated]').count() === 0 && await xp.locator('[data-ev-judge-ask]').count() === 0
&& ex.hits.some((h) => h === `judge:${fxv.explore.run_id}:${x.contra}:write`), ex.hits.filter((h) => h.startsWith('judge:')).join(' '))
if (SHOTS) {
// 重新开一页截「按旧规则判定」那一刻(上面已经重判掉了)
const shot = await open('/ui-harness.html?evidence=1')
await shot.page.waitForSelector(`${VX} [data-ev-claim="${x.contra}"][data-ev-verdict="unsupported"]`).catch(() => {})
for (const theme of ['dark', 'light']) {
await shot.page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
await badge(shot.page, VX, x.contra).click()
await shot.page.waitForTimeout(100)
if (await shot.page.locator('[data-evidence-panel]').count() === 0) await badge(shot.page, VX, x.contra).click()
await shot.page.waitForSelector('[data-evidence-panel] [data-ev-judge-outdated]').catch(() => {})
await shot.page.waitForTimeout(300)
await shot.page.screenshot({ path: `${SHOTS}/jr-panel-outdated-${theme}.png` })
}
await shot.ctx.close()
}
check('没有运行时报错(裁判拆档 · 探索运行)', ex.errors.length === 0, ex.errors.join(' | '))
await ex.ctx.close()
})
await section('jr-narrow', '裁判拆档在 360px:理由 120 字、字段清单、SQL 栏内展开不横向滚动;底部抽屉整页不横向滚动', async () => {
const overflow = () => page.evaluate((sel) => {
const el = document.querySelector(sel)
return { sw: el.scrollWidth, cw: el.clientWidth, w: el.getBoundingClientRect().width }
}, VN)
const before = await overflow()
await badge(page, VN, fxv.formal.units.contra).click()
await page.waitForSelector(`${VN} [data-evidence-panel="inline"] [data-ev-rationale]`)
const why = await page.locator(`${VN} [data-ev-rationale]`).evaluate((el) => ({ text: el.textContent, sw: el.scrollWidth, cw: el.clientWidth }))
const after = await overflow()
check('窄栏里理由 120 字整句显示、在栏内折行', why.text === fxv.long_rationale && why.sw <= why.cw, JSON.stringify({ sw: why.sw, cw: why.cw }))
check('……面板打开前后都没有横向滚动', Math.round(before.w) === 360 && before.sw <= before.cw && after.sw <= after.cw, JSON.stringify({ before, after }))
if (SHOTS) {
for (const theme of ['dark', 'light']) {
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
await page.waitForTimeout(200)
await page.locator(VN).screenshot({ path: `${SHOTS}/jr-360-rationale-${theme}.png` })
}
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
}
await badge(page, VN, fxv.formal.units.method).click()
await page.waitForSelector(`${VN} [data-evidence-panel="inline"] [data-ev-basis] [data-ev-sql]`).catch(() => {})
await page.waitForSelector(`${VN} [data-evidence-panel="inline"] [data-ev-basis-table] [data-ev-field]`).catch(() => {})
const basis = await overflow()
const pre = await page.locator(`${VN} [data-ev-basis] [data-ev-sql]`).first().evaluate((el) => ({ sw: el.scrollWidth, cw: el.clientWidth }))
.catch(() => null)
check('窄栏里讲方法的句子:SQL、字段清单都在栏内,不横向滚动', basis.sw <= basis.cw && !!pre && pre.sw <= pre.cw
&& await page.locator(`${VN} [data-ev-basis-table] [data-ev-field]`).count() > 0, JSON.stringify({ basis, pre }))
if (SHOTS) {
for (const theme of ['dark', 'light']) {
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
await page.waitForTimeout(200)
await page.locator(VN).screenshot({ path: `${SHOTS}/jr-360-basis-${theme}.png` })
}
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
}
await page.locator(`${VN} [data-evidence-panel] button`, { hasText: '回到正文' }).click()
await page.waitForTimeout(150)
const small = await open('/ui-harness.html?evidence=1', { w: 360, h: 780 })
const scroll = () => small.page.evaluate(() => ({ sw: document.documentElement.scrollWidth, vw: innerWidth }))
await badge(small.page, V, fxv.formal.units.contra).click()
await small.page.waitForSelector('[data-evidence-panel] [data-ev-rationale]')
const d = await scroll()
const dwhy = await small.page.locator('[data-evidence-panel] [data-ev-rationale]').evaluate((el) => ({ text: el.textContent, sw: el.scrollWidth, cw: el.clientWidth }))
check('360px 宽的屏幕:底部抽屉里理由整句显示、整页没有横向滚动',
await small.page.locator('[data-evidence-panel]').getAttribute('data-evidence-panel') === 'drawer' && d.sw <= d.vw
&& dwhy.text === fxv.long_rationale && dwhy.sw <= dwhy.cw, JSON.stringify({ d, sw: dwhy.sw, cw: dwhy.cw }))
await small.ctx.close()
})
await section('jr-banner', '裁判拆档:出具横幅的结论句计数带上证据相矛盾、证据不足和旧的证据不支持', async () => {
const at = (key) => page.locator(`#issuance-claims [data-issuance-case="${key}"]`)
const rework = await at('rework').locator('[data-evidence-line]').innerText().catch(() => '')
check('计数:证据相矛盾、旧的证据不支持、证据不足各写各的', rework.includes('结论 7 句(有依据 1 · 部分有依据 1 · 证据相矛盾 2 · 证据不支持 1 · 证据不足 1 · 未裁判 1)'),
rework)
const [color, waiting] = [await at('rework').locator('[data-evidence-claims]').evaluate((el) => getComputedStyle(el).color).catch(() => ''),
await colorOf(page, '--st-waiting')]
check('……有问题的计数用提醒色', color === waiting, `${color} / ${waiting}`)
check('只有证据不足一种问题:照样给「定位下一处」', await at('insufficient').locator('[data-evidence-next]').count() === 1)
const only = await at('insufficient').locator('[data-evidence-claims]').evaluate((el) => getComputedStyle(el).color).catch(() => '')
check('……计数也用提醒色(证据不足也算问题)', only === waiting, only)
if (SHOTS) {
for (const theme of ['dark', 'light']) {
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
await page.waitForTimeout(200)
await page.locator('#issuance-claims').screenshot({ path: `${SHOTS}/jr-banner-${theme}.png` })
}
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
}
})
if (SHOTS) {
for (const theme of ['dark', 'light']) {
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
+37 -1
View File
@@ -626,9 +626,34 @@ await section('设置 · 证据裁判', async () => {
const box = page.locator('[data-judge-settings]')
const all = (await box.innerText().catch(() => '')).replace(/\s+/g, ' ')
check('偏好里有「证据裁判」一组,写明判断是模型给的、不是系统核对', all.includes('证据裁判') && all.includes('非确定'), all.slice(0, 80))
// 裁判拆档:接入和模型都没填时,平常那句建议换成醒目的「未单独配置」提示(建议写在提示里);填了再换回来(见下)
check('裁判模型:接入和模型,提示建议和写作模型不同', await page.locator('#pref-judge-provider').count() === 1
&& await page.locator('#pref-judge-model').count() === 1 && all.includes('建议与撰写报告的模型不同'))
&& await page.locator('#pref-judge-model').count() === 1 && all.includes('建议配置一个与撰写报告的模型不同、能力更强的模型'))
check('……接入留空时跟随助手的模型', (await page.locator('#pref-judge-provider option').first().innerText()).includes('跟随助手的模型'))
const notSet = page.locator('[data-judge-model] [data-judge-not-set]')
check('没配裁判模型(接入、模型都是 null):提示「未单独配置,将使用助手的模型」',
(await notSet.locator('[data-judge-not-set-text]').innerText().catch(() => '')) === '未单独配置,将使用助手的模型',
await notSet.innerText().catch(() => '(没有这条提示)'))
const advice = await notSet.locator('[data-judge-not-set-advice]').innerText().catch(() => '')
check('……并建议配一个和写作模型不同、能力更强的模型', advice.includes('与撰写报告的模型不同') && advice.includes('能力更强'), advice)
const noteLook = await notSet.evaluate((el) => {
const probe = document.createElement('span')
probe.style.color = 'var(--st-waiting)'
document.body.appendChild(probe)
const waiting = getComputedStyle(probe).color
probe.remove()
const cs = getComputedStyle(el)
return { border: cs.borderLeftColor, width: cs.borderLeftWidth, icon: getComputedStyle(el.querySelector('svg')).color, waiting,
described: document.getElementById(el.closest('[data-judge-model]').getAttribute('aria-describedby'))?.hasAttribute('data-judge-not-set') }
}).catch(() => null)
check('……醒目:左侧提醒色竖线、提醒色图标', !!noteLook && noteLook.border === noteLook.waiting && noteLook.width === '2px'
&& noteLook.icon === noteLook.waiting, JSON.stringify(noteLook))
check('……读屏:模型那一组的说明就是这条提示(aria-describedby)', noteLook?.described === true)
if (SHOTS) {
await box.scrollIntoViewIfNeeded()
await page.waitForTimeout(200)
await box.screenshot({ path: `${SHOTS}/prefs-judge-not-set-${THEME}.png` })
}
const val = (id) => page.locator(`#${id}`).inputValue()
check('读回各项上限:40 句、30 秒、每次点击 $0.01、每日 $2', await val('pref-judge-claims') === '40' && await val('pref-judge-timeout') === '30'
&& await val('pref-judge-click') === '0.01' && await val('pref-judge-daily') === '2')
@@ -654,6 +679,8 @@ await section('设置 · 证据裁判', async () => {
await page.locator('#pref-judge-click').fill('0.02')
check('……接入留空时模型框的占位也写跟随助手的模型', await page.locator('#pref-judge-model').getAttribute('placeholder') === '留空:跟随助手的模型')
await page.locator('#pref-judge-model').fill('judge-model-b')
check('填了裁判模型:「未单独配置」的提示收起,换回平常那句建议', await notSet.count() === 0
&& (await box.innerText().catch(() => '')).includes('建议与撰写报告的模型不同'))
// 只填了模型:设置这一级整组生效(judge_model_spec 不跨级拼),后端按模型名找接入——已经不跟随助手的模型
const provFirst = await page.locator('#pref-judge-provider option').first().innerText().catch(() => '')
check('只填了模型:接入留空那一项不再叫「跟随助手的模型」,改叫「按模型名找接入」', provFirst === '按模型名找接入', provFirst)
@@ -690,6 +717,15 @@ await section('设置 · 证据裁判', async () => {
}
await close()
// 只填了接入(模型留空):也算单独配置了,不提示「未单独配置」
const byProvider = { ...stored, judge: { ...stored.judge, provider: 'judge-provider-x', model: null } }
const pv = await open('/settings/prefs', { handlers: [[/^GET \/settings$/, (route) => json(byProvider)(route)],
[/^GET \/settings\/judge\/spend$/, (route) => json({ date: '2026-09-29', usd: 0, calls: 0, unpriced_calls: 0 })(route)]] })
await pv.page.waitForSelector('[data-judge-settings]', { timeout: 6000 }).catch(() => {})
check('只填了接入:不提示「未单独配置」', await pv.page.locator('[data-judge-settings]').count() === 1
&& await pv.page.locator('[data-judge-not-set]').count() === 0)
await pv.close()
// 老后端没有 judge 这一组:不显示,也不写
const legacy = { ...stored }
delete legacy.judge
+2
View File
@@ -712,6 +712,8 @@ await section('保存失败、接口不存在、画布改过:都有明确状
bad.state.save = 'ok'
await dialog(bad.page).getByRole('button', { name: '重试保存' }).click()
await waitFor(bad.page, async () => bad.state.checks.length === 2)
// 重查的请求发出去了,按钮的可用状态要等答复回来、界面更新之后才变:并行跑的时候机器挤,直接读会抢在前面
await waitFor(bad.page, async () => submit(bad.page).isEnabled())
check('重试保存成功后重新检查、发布可用', bad.state.patches.length === 2 && await submit(bad.page).isEnabled())
const nofix = await open({ backend: 'nofix' })
+52
View File
@@ -651,6 +651,9 @@ await section('提取模板', async () => {
fakes.set('POST /api/copilot/from-run', () => ({
status: 200, json: { workflow_id: wfs[0].id, name: '检查用草稿', nodes: 3, edges: 2, dropped_nodes: 1, source_run: timed?.id },
}))
// 跳进画布后,画布先找这张图的助手对话,没有就建一条(POST,会被探针拦下、算成没伪造的写请求)。
// 沙箱里 wfs[0] 有没有对话要看之前谁跑过什么:给一条现成的,和升级横幅那一节同一个做法
fakes.set('GET /api/conversations', () => ({ status: 200, json: [{ id: 'fake-conv-extract', title: '', kind: 'canvas', archived: false, turn_count: 0 }] }))
// 画布上还有没保存的改动(store 里的画布离开编排页也还在):先问要不要放弃,再建草稿。
// 以前建完才问,点了「取消」库里就多出一张没人要的草稿(3C REQ-15)
const extractPosts = () => writes.filter((w) => w === 'POST /api/copilot/from-run').length
@@ -684,6 +687,7 @@ await section('提取模板', async () => {
&& new URL(page.url()).pathname === `/studio/${wfs[0].id}`, `提取 ${extractPosts() - postsBefore} 次 · 问了 ${asks} 遍`)
check('提取成功后跳到新草稿', new URL(page.url()).pathname === `/studio/${wfs[0].id}`)
fakes.delete('POST /api/copilot/from-run')
fakes.delete('GET /api/conversations')
})
// ------------------------------------------------------------------ 运行中:接流、停止、不能删
@@ -1981,6 +1985,54 @@ await section('证据页签:同一段文字里好几条同样的违规、正
}
})
await section('证据页签:裁判拆档——证据相矛盾、证据不足的结论句各一行,点开是这一句的「模型的解释」', async () => {
// 夹具是后端(JR-back 之后)真算的:审计行由 api/evidence.py 的 _audit_doc 给出,判定行 issue 为
// contradicted_claim / insufficient_claim,带着 unit 和 verdict
const fxv = JSON.parse(readFileSync(new URL('../frontend/src/run/__tests__/evidence-verdict.json', import.meta.url), 'utf8'))
const v = fxv.formal
const EV_V = 'fake0evverdict000000000000000'
const output = { answer: v.doc.markdown, _evidence: { report_node: 'write', doc_artifact: v.doc_artifact, fields: ['answer'] } }
evFakes(EV_V, { output, graph: v.graph, audit: v.audit })
fakes.set(`GET /api/artifacts/${v.doc_artifact}`, () => ({ status: 200, json: { id: v.doc_artifact, content: v.doc } }))
for (const snap of Object.values(fxv.snapshots)) {
fakes.set(`GET /api/artifacts/${snap.artifact}`, () => ({ status: 200, json: { id: snap.artifact, content: snap.content } }))
}
await page.route(new RegExp(`/api/runs/${EV_V}/evidence/segments/`), (r) => {
const sid = decodeURIComponent(new URL(r.request().url()).pathname.split('/').pop())
return v.segments[sid] ? r.fulfill({ json: v.segments[sid] }) : r.fulfill({ status: 404, json: { detail: '没有这个片段' } })
})
try {
await openEvidence(EV_V)
await page.locator('[data-audit-table]').waitFor({ timeout: 5000 }).catch(() => {})
const claimRows = await page.locator('[data-audit-table] tbody[data-audit-group=none] tr[data-audit-row]').evaluateAll((els) =>
els.map((e) => ({ state: e.getAttribute('data-audit-state'), open: e.querySelector('[data-audit-open]')?.getAttribute('data-audit-open') ?? null,
stateText: e.children[1]?.textContent ?? '', source: e.querySelector('[data-audit-source]')?.textContent ?? '' })))
const byState = (st) => claimRows.filter((r) => r.state === st)
check('审计表:证据相矛盾的结论句两行、证据不足一行,列在「无证据」一组', byState('contradicted').length === 2 && byState('insufficient').length === 1,
JSON.stringify(claimRows.map((r) => r.state)))
check('……状态照句末徽标的字形和叫法写', byState('contradicted').every((r) => r.stateText === '!模型判断:证据相矛盾')
&& byState('insufficient')[0]?.stateText === '○模型判断:证据不足', JSON.stringify(claimRows.map((r) => r.stateText)))
const note = v.audit.groups.flatMap((g) => g.rows).find((r) => r.issue === 'insufficient_claim')?.note ?? '(夹具里没有这一行)'
check('……原因照后端的原话(缺什么、理由)', byState('insufficient')[0]?.source === note && note.includes(`缺少:${fxv.missing}`), note)
check('……判定行点得开,打开的是这一句的句末徽标(claim:<句子>)', byState('insufficient')[0]?.open === `claim:${v.units.insuff}`
&& byState('contradicted').map((r) => r.open).join(',') === [v.units.contra, v.units.method].map((u) => `claim:${u}`).join(','),
JSON.stringify(claimRows.map((r) => r.open)))
await page.locator(`[data-audit-table] [data-audit-open="claim:${v.units.insuff}"]`).click()
await page.locator('[data-evidence-dock] [data-evidence-panel] [data-ev-judge="insufficient"]').waitFor({ timeout: 5000 }).catch(() => {})
const dock = page.locator('[data-evidence-dock] [data-evidence-panel]')
check('点开证据不足那一行:右边打开这一句的「模型的解释」,写明缺什么', (await dock.locator('[data-ev-missing]').innerText().catch(() => ''))
=== `缺少:${fxv.missing}`)
check('……裁判模型和写作模型相同、不在价格表里:两条提醒都在', await dock.locator('[data-ev-judge-same]').count() === 1
&& await dock.locator('[data-ev-judge-unpriced]').count() === 1)
const tally = await page.locator(`[data-evidence-report=write] [data-evidence-claims]`).innerText().catch(() => '')
check('报告上方的证据条:结论句计数带上证据相矛盾、证据不足', tally.includes('结论 6 句(有依据 1 · 部分有依据 1 · 证据相矛盾 2 · 证据不足 1 · 未裁判 1)'), tally)
await page.keyboard.press('Escape')
} finally {
for (const k of [...fakes.keys()]) if (k.includes(EV_V) || k.includes(v.doc_artifact)) fakes.delete(k)
for (const snap of Object.values(fxv.snapshots)) fakes.delete(`GET /api/artifacts/${snap.artifact}`)
}
})
await section('证据页签:没有报告的运行照实说明(none / 旧运行猜测 / 老后端)', async () => {
evFakes(EV_LEGACY, { output: fxe.legacy.output, graph: fxe.legacy.graph, audit: fxe.legacy.audit, report: false })
await openEvidence(EV_LEGACY)
+97 -5
View File
@@ -458,9 +458,9 @@ await section('lib/evidence:证据状态的四通道元数据、报告节点
const ok = (name, cond, detail = '') => out.push([name, !!cond, cond ? '' : detail])
const codes = ev.EVIDENCE_STATES
// 方案 6.1 的八种(确定性、概率性四种、无证据、连接性、旧运行候选),加三期无证据的两个变体:
// 可疑实体(疑似不存在的名称)、无法核实(表结构快照不全)
ok('十种状态一种不少', ['deterministic', 'supported', 'partial', 'unsupported', 'unjudged', 'none', 'connective', 'candidate',
'suspect', 'unverified'].every((c) => codes.includes(c)) && codes.length === 10, codes.join(','))
// 可疑实体(疑似不存在的名称)、无法核实(表结构快照不全),加裁判拆档的两种:证据相矛盾、证据不足
ok('十二种状态一种不少', ['deterministic', 'supported', 'partial', 'contradicted', 'insufficient', 'unsupported', 'unjudged', 'none',
'connective', 'candidate', 'suspect', 'unverified'].every((c) => codes.includes(c)) && codes.length === 12, codes.join(','))
eq('可疑实体:点状线(同无证据)、?!、疑似不存在的名称', `${ev.EVIDENCE_STATE.suspect.line}/${ev.EVIDENCE_STATE.suspect.glyph}/${ev.EVIDENCE_STATE.suspect.label}`,
'dotted/?!/疑似不存在的名称')
ok('可疑实体醒目(进 n / N),核对不了只是标注', ev.EVIDENCE_STATE.suspect.alert && !ev.EVIDENCE_STATE.unverified.alert)
@@ -492,8 +492,12 @@ await section('lib/evidence:证据状态的四通道元数据、报告节点
const d = ev.EVIDENCE_STATE[c].decoration
return d === 'transparent' || /var\(--st-[\w-]+\)/.test(d)
}), codes.map((c) => ev.EVIDENCE_STATE[c].decoration).join(' | '))
const pairs = codes.map((c) => `${ev.EVIDENCE_STATE[c].line}/${ev.EVIDENCE_STATE[c].glyph}`)
ok('去掉颜色也分得开:线型 + 字形两两不同', new Set(pairs).size === codes.length, pairs.join(' '))
// 裁判拆档:旧取值「证据不支持」按规格和证据相矛盾长得一样(同一类判定,拆档前后的叫法不同,文字仍不同),
// 除它之外线型 + 字形两两不同
const pairOf = (c) => `${ev.EVIDENCE_STATE[c].line}/${ev.EVIDENCE_STATE[c].glyph}`
const pairs = codes.filter((c) => c !== 'unsupported').map(pairOf)
ok('去掉颜色也分得开:线型 + 字形两两不同(旧的证据不支持同证据相矛盾)', new Set(pairs).size === codes.length - 1
&& pairOf('unsupported') === pairOf('contradicted'), codes.map(pairOf).join(' '))
const labels = codes.map((c) => ev.EVIDENCE_STATE[c].label)
ok('文字两两不同', new Set(labels).size === codes.length, labels.join(' '))
eq('确定性:细实线、有出处', `${ev.EVIDENCE_STATE.deterministic.line}/${ev.EVIDENCE_STATE.deterministic.label}`, 'solid/有出处')
@@ -663,6 +667,94 @@ await section('lib/evidence:结论句裁判(四期)——判定的徽标
})) check(name, ok, detail)
})
await section('lib/evidence:裁判拆档(JR)——证据相矛盾、证据不足、计数、跳转轮次、模型提醒、审计行、讲方法的句子', async () => {
for (const [name, ok, detail] of await page.evaluate(() => {
const { evidence: ev, terms: t } = window.__ui.lib
const fv = window.__ui.evidenceVerdict
const out = []
const eq = (name, got, want) => out.push([name, got === want, got === want ? '' : `得到 ${JSON.stringify(got)},应为 ${JSON.stringify(want)}`])
const ok = (name, cond, detail = '') => out.push([name, !!cond, cond ? '' : detail])
const S = ev.EVIDENCE_STATE
eq('证据相矛盾、证据不足:句末徽标、字形 ! ○', ['contradicted', 'insufficient'].map((c) => `${S[c].line}/${S[c].glyph}`).join(' '),
'badge/! badge/○')
eq('……颜色令牌:证据相矛盾同原来的证据不支持(--st-failed),证据不足中性(--st-cancelled)',
['contradicted', 'insufficient', 'unsupported'].map((c) => S[c].color).join(' '), 'var(--st-failed) var(--st-cancelled) var(--st-failed)')
ok('……证据不足不用红、不用提醒色,字形不和未裁判的 ? 撞', !/--st-(failed|waiting|done)/.test(S.insufficient.color + S.insufficient.soft)
&& S.insufficient.glyph !== S.unjudged.glyph, JSON.stringify(S.insufficient))
eq('叫法:证据相矛盾、证据不足;旧取值照旧叫证据不支持', [S.contradicted.label, S.insufficient.label, S.unsupported.label].join(' / '),
'模型判断:证据相矛盾 / 模型判断:证据不足 / 模型判断:证据不支持')
eq('判定 → 徽标:新取值各是各的', [ev.verdictState({ status: 'contradicted' }), ev.verdictState({ status: 'insufficient' }),
ev.verdictState({ status: 'unsupported' })].join(','), 'contradicted,insufficient,unsupported')
ok('判过的算上新取值(不再给「请模型判断」)', ev.isJudged({ status: 'contradicted' }) && ev.isJudged({ status: 'insufficient' }))
eq('n / N 的轮次:证据相矛盾、旧的证据不支持、部分有依据、无证据在第 1 轮,证据不足在第 2 轮,有依据、未裁判不进',
['contradicted', 'unsupported', 'partial', 'none', 'suspect', 'insufficient', 'supported', 'unjudged'].map((c) => ev.alertTier(c)).join(''),
'11111200')
ok('整句铺浅底的是证据相矛盾和旧的证据不支持', ev.conflicting('contradicted') && ev.conflicting('unsupported')
&& !ev.conflicting('insufficient') && !ev.conflicting('partial'))
eq('证据不足缺什么:只认 insufficient 的 missing', [ev.verdictMissing({ status: 'insufficient', missing: ' SQL ' }),
ev.verdictMissing({ status: 'contradicted', missing: 'SQL' }), ev.verdictMissing({ status: 'insufficient' })].join('|'), 'SQL||')
const k = (c) => (c ? `${c.total}:${c.supported}/${c.partial}/${c.contradicted}/${c.unsupported}/${c.insufficient}/${c.unjudged}/${c.uncited}` : 'null')
eq('文档计数:证据相矛盾 2、证据不足 1,不是结论句不算', k(ev.claimCounts(fv.formal.doc)), '6:1/1/2/0/1/1/0')
eq('……叠上封存后追加的判定(旧取值 unsupported 照样计入)', k(ev.claimCounts(fv.explore.doc,
ev.graphVerdicts(fv.explore.graph, 'write'))), '7:0/0/0/1/1/5/0')
eq('stats / counts → 计数(report.checked、报告节点的章):旧数据的 unsupported 照样计入,不是结论句不算',
k(ev.claimCountsOf({ supported: 1, partial: 1, contradicted: 2, insufficient: 1, unsupported: 1, not_a_claim: 3, unjudged: 1,
numbers: 9 })), '7:1/1/2/1/1/1/0')
eq('……缺键、不是数的当 0', k(ev.claimCountsOf({ supported: 2, unsupported: 'x' })), '2:2/0/0/0/0/0/0')
eq('计数的说法:证据相矛盾、证据不支持、证据不足各写各的', t.claimTally(ev.claimCountsOf({ supported: 1, contradicted: 2, unsupported: 1,
insufficient: 1, unjudged: 1 })), '结论 6 句(有依据 1 · 证据相矛盾 2 · 证据不支持 1 · 证据不足 1 · 未裁判 1)')
eq('……旧的计数对象(没有新键)照旧能说', t.claimTally({ total: 2, supported: 1, partial: 0, unsupported: 1, unjudged: 0, uncited: 0 }),
'结论 2 句(有依据 1 · 证据不支持 1)')
eq('有问题的句数(横幅、证据条的「定位下一处」):证据相矛盾、旧的证据不支持、部分有依据、证据不足', ev.claimProblems(ev.claimCountsOf(
{ supported: 5, contradicted: 1, unsupported: 1, partial: 1, insufficient: 1, unjudged: 3 })), 4)
const unit = fv.formal.doc.blocks.flatMap((b) => b.units).find((u) => u.id === fv.formal.units.insuff)
eq('读屏名:证据不足接着说缺什么', ev.claimLabel(unit, unit.verdict),
`结论句「增长主要来自新客首单,老客复购持平。」,模型判断:证据不足,缺少:${fv.missing}`)
const sealed = fv.formal.doc.blocks.flatMap((b) => b.units).find((u) => u.id === fv.formal.units.contra).verdict
const f1 = ev.judgeFlags(sealed, { doc: fv.formal.doc.judge })
ok('模型提醒:节点里当场判的看报告文档的 judge(same_model 真、priced 假)', f1.sameModel && f1.unpriced && f1.writer === fv.model, JSON.stringify(f1))
const late = { status: 'insufficient', post_seal: true, judge: 'm' }
const f2 = ev.judgeFlags(late, { doc: { same_model: true, priced: false }, meta: { same_model: false, priced: true } })
ok('……封存后追加的不看报告文档,看证据图里最近一次按需裁判(judge_meta)', !f2.sameModel && !f2.unpriced, JSON.stringify(f2))
const f3 = ev.judgeFlags(late, { reply: { same_model: true, priced: false }, meta: { same_model: false, priced: true } })
ok('……这一次的答复盖过证据图', f3.sameModel && f3.unpriced, JSON.stringify(f3))
const f4 = ev.judgeFlags({ ...late, same_model: false }, { reply: { same_model: true, priced: false } })
ok('……判定自己带着的盖过答复', !f4.sameModel && f4.unpriced, JSON.stringify(f4))
const f5 = ev.judgeFlags({ status: 'unjudged', reason: 'max_claims' }, { doc: { same_model: true, priced: false } })
ok('……没判的句子不提醒;priced 为 null(不知道)也不提醒', !f5.sameModel && !f5.unpriced
&& !ev.judgeFlags(sealed, { doc: { priced: null } }).unpriced, JSON.stringify(f5))
const rows = ev.auditFromApi(fv.formal.audit)
const claim = (issue) => rows.find((r) => r.kind === 'claim' && fv.formal.audit.groups.flatMap((g) => g.rows)
.some((x) => x.issue === issue && x.unit === r.unit))
const c1 = claim('contradicted_claim')
const c2 = claim('insufficient_claim')
ok('审计行:issue 为 contradicted_claim 的结论句认成证据相矛盾,insufficient_claim 认成证据不足,带着是哪一句',
c1?.state === 'contradicted' && c2?.state === 'insufficient' && c2.unit === fv.formal.units.insuff && c1.group === 'none',
JSON.stringify([c1, c2].map((r) => r && { state: r.state, unit: r.unit, group: r.group })))
ok('……旧取值 unsupported_claim 认成证据不支持;没挂依据的结论句照旧是无证据',
ev.auditFromApi({ groups: [{ key: 'none', rows: [{ kind: 'claim', issue: 'unsupported_claim', unit: 'u2', text: 'x', group: 'none' },
{ kind: 'claim', issue: 'uncited_claim', unit: 'u7', text: 'y', group: 'none', state: 'none' }] }] }).map((r) => r.state).join(',')
=== 'unsupported,none')
const doc = fv.formal.doc
const units = Object.fromEntries(doc.blocks.flatMap((b) => b.units).map((u) => [u.id, u]))
const b = ev.claimBasis(units[fv.formal.units.method], doc)
ok('讲方法的句子(只挂了表名、字段名、整份查询):列表、字段、要列 SQL 的查询(直接挂的在前,再补表名出现过的)',
!!b && b.tables.map((x) => `${x.alias}=${x.name}@${x.seg}`).join() === `t:orders=orders@${fv.formal.segs.orders}`
&& b.columns.map((x) => x.name).join() === 'orders.status,orders.created_at' && b.queries.join() === 'Q1,Q2' && b.more === 0, JSON.stringify(b))
eq('……只挂了整份查询的句子也算', ev.claimBasis(units[fv.formal.units.insuff], doc)?.queries.join(), 'Q1')
eq('……挂了数字(单元格)的句子不算', ev.claimBasis(units[fv.formal.units.ok], doc), null)
eq('……挂了运行输入的句子不算', ev.claimBasis(units[fv.formal.units.not], doc), null)
eq('……一个依据都没挂的不算', ev.claimBasis(units[fv.formal.units.limit], doc), null)
const many = { catalog: { 't:a': { kind: 'table', name: 'a', queries: ['Q1', 'Q2', 'Q3', 'Q4', 'Q5'] },
...Object.fromEntries(['Q1', 'Q2', 'Q3', 'Q4', 'Q5'].map((q) => [q, { kind: 'query' }])) }, blocks: [] }
const bm = ev.claimBasis({ cites: ['t:a'], segments: [] }, many)
ok('……查询多于 3 次:只列前 3 次的 SQL,其余写个数', bm?.queries.join() === 'Q1,Q2,Q3' && bm.more === 2, JSON.stringify(bm))
eq('查询快照里的 SQL:快照本身、工具调用快照的 args / result 都认', [ev.sqlOf({ sql: 'A' }), ev.sqlOf({ tool: 'x', args: { sql: 'B' }, result: {} }),
ev.sqlOf({ result: { sql: 'C' } }), ev.sqlOf({ rows: [] }), ev.sqlOf(null)].join('|'), 'A|B|C||')
return out
})) check(name, ok, detail)
})
await section('lib/actor:本机署名', async () => {
for (const [name, ok, detail] of await page.evaluate(() => {
const a = window.__ui.lib.actor