diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index ed79a3d9..dfe2971b 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -190,19 +190,33 @@ NODE_REFERENCE = """\ 写 2 只剩「2%」,0.004 这样的小比率还会显示不出来、报告引用不了。on_missing 默认 fail(缺输入整张卡失败);写 null 则缺输入的指标记为空值,交给出具契约判档 取 tool 节点查库的结果用 cell(nodes.取数节点id, 行, '列名')(行号从 0 数,列按列名):cell() 直接认查询工具 交回的结果,出处精确到快照里的那一格,不用再接 transform 解析。取 agent 的字段写 vars..字段 -- report:报告撰写(带引用)。config: {instructions, system, metrics_from:[口径卡id], on_violation, max_repairs, assign_to} +- report:报告撰写(带引用)。config: {instructions, system, metrics_from:[口径卡id], numbers, on_violation, max_repairs, + claims, entities, assign_to} **凡是给人看的、带数字的报告、分析、结论,都用 report,不要用 llm**。它自动收集上游的口径卡指标、查询结果 - (tool 节点和 agent 查过的库,按查询先后编成 Q1、Q2…)和运行输入,模型只能写引用标记,数值由系统从证据里 - 取出来渲染;模型自己写的数字会被打回重写: + (tool 节点和 agent 查过的库,按查询先后编成 Q1、Q2…)、查库当时的表结构、知识库检索(按先后编成 K1、K2…) + 和运行输入,模型只能写引用标记,数值由系统从证据里取出来渲染;模型自己写的数字会被打回重写: - [[m:指标id]]:口径卡指标。比率、增幅、占比、差值这类派生计算只能先在口径卡里登记再引用 - [[v:Q1.r0.列名]]:第 1 次查询结果里第 0 行那一列的原值(行号从 0 数) - [[table:Q1 cols=列a,列b rows=0-4]]:把查询结果的几行几列原样生成表格,每一格都点得开出处 - [[i:输入字段]]:运行输入 - metrics_from 不写就取所有上游口径卡;on_violation 不写时探索运行是 flag(标出来、照常产出),正式运行是 fail + - [[t:表名]]、[[c:表名.字段名]]:表和字段,点得开是哪次查询用到的、表结构里是什么类型。写在反引号里的名字 + (`orders`)也会被核对:只写本次运行查过的数据源里真实存在的名字,表结构、查询、结果列里都没有的会被标成 + 「可能是编造的名字」(受管模板的正式出具因此降档) + - [[q:K1|逐字引文]]:知识库检索的原话,必须和命中片段里的字逐字一致(空白不计,至少 4 个字),改一个字就算 + 没有出处;只是拿检索当依据时写 [[see:K1]]。要引原话,在 report 前面接 retrieve 节点 + - 句末的 [[see:m:指标id,Q1,K1]]:这句结论的依据,不显示 + metrics_from 不写就取所有上游口径卡;on_violation 不写时探索运行是 flag(标出来、照常产出),正式运行是 fail; + numbers 默认 strict(裸数字算违规)。claims 是结论句策略:off(默认,不管)/ require_citation(每句结论都要 + 挂引用,没挂的计入缺口、出具降档);judge(模型裁判)在后续版本支持,现在写了整张图跑不起来。entities 默认 link + (核对表名、字段名),报告里根本不提表名时才写 off - output 的出具契约:config.contract = {report_from: 报告节点id, metrics_from:[口径卡id], required:[必需指标id], expected:[期望指标id], strict: true}。成果字段写 {{ nodes.报告节点id.text }}, 前后不要拼别的字——拼了就没法逐段对应证据,出具会降档。受管模板的报告要直接引用查询单元格 ([[v:]] / [[table:]])的,契约里写 cells: true,否则这些引用按解析不了算 + 要按受管级别发布的模板(发布前检查不过就发不出去):报告节点写明 numbers: "strict"、on_violation: "fail"、 + claims: "require_citation";契约用 report_from(受管级别不收 narrative),成果字段只取报告节点的正文;给口径卡 + 供数的 agent 配 output_schema 和 cite_fields: true,code 节点喂口径卡的必须是真取数(evidence_role: "source"), + 否则把计算写进口径卡;llm、agent、协作团队写的文字不能绕过报告节点直接进出口,llm 也不能喂口径卡 搭图规则(有数字结论时必须遵守): 1. 取数(tool 节点查库;或者 agent 查库并配 output_schema + cite_fields)→ 口径卡 → report → output @@ -304,6 +318,10 @@ def _user_message(payload: GenerateIn, *, patch: bool) -> str: "直接引用 agent 查过的单元格,不必经过口径卡;要比率、增幅再在 agent 和 report 之间加口径卡,agent 配 " "output_schema + cite_fields,口径卡读 vars..字段。出口字段写 {{ nodes.report的id.text }}\n" "- 只问表结构、清单(有哪些表、有哪些字段、列出名单)的,不加 report,agent → output 就行\n" + # 三期起报告会核对写出来的表名、字段名:instructions 里让它「列出用到的表」时,编出来的名字会被标出来 + "- report 提到表名、字段名时写 [[t:表名]] / [[c:表名.字段名]] 或放进反引号,只写 agent 真查过的名字," + "编造的会被标出来;要引用知识库原话就在 report 前接 retrieve,用 [[q:K1|原话]] 逐字引用。问数是探索运行," + "report 的 claims 不用写\n" "- 用户要的是周报、要三档出具时,按搭图规则配口径卡和 report_from 契约\n" "- 不要用 transform 解析 agent / llm 的文字:要把 agent 查到的数交给下游,就给它配 output_schema + " "cite_fields\n" @@ -968,7 +986,7 @@ def _sse(obj: dict[str, Any]) -> str: def _blocking_issues( nodes: dict[str, dict[str, Any]], edges: list[dict[str, Any]], scope: set[str] | None = None, *, sources: list[Any] | None = None, baseline: list[dict[str, Any]] | None = None, - level: str | None = None, + level: str | None = None, defaults: dict[str, Any] | None = None, ) -> list[dict[str, Any]] | None: """按运行时同一套校验,挑出会挡住运行的问题。图本身不成形时返回 None。 @@ -983,9 +1001,13 @@ def _blocking_issues( 每条是 ValidationIssue 的字典形状(node_id、field 都在),和 final.issues 一样: 界面靠 node_id 定位卡片、靠 field 落到具体的输入框,不用从一行字里抠节点 id。 + + defaults 是全图默认(graph.defaults):报告撰写节点的 numbers / claims、审批策略都跟随它,不带上的话 + 自查看到的问题和真正的门禁对不上。 """ try: - spec = GraphSpec.model_validate({"nodes": list(nodes.values()), "edges": edges}) + spec = GraphSpec.model_validate({"nodes": list(nodes.values()), "edges": edges, + **({"defaults": defaults} if isinstance(defaults, dict) else {})}) except Exception: # noqa: BLE001 - 结构不合法留给收尾那一步报 return None out = ([i.model_dump() for i in validate_graph(spec).issues if i.level == "error"] @@ -1609,9 +1631,12 @@ _ASSIST_RULES = """\ - 不许降低要求:不删节点、不删连线,add_node 不许用图里已有的节点 id(那等于把旧节点删了重建),也不换节点类型; 不删掉或清空出具契约,不删 required 里的指标,不把 strict 改成 false,不往 allow_numbers 里加数,不替人打开 cells; 审批策略只能往严里改(每次调用都审批 > 仅危险工具需要审批 > 全部自动放行),不放宽、不删掉,更不改成 never; - 不取消子工作流钉住的版本;发布级别不是你能改的。结构要动(比如换掉协作团队),输出 question 说明怎么拆,不要自己动手 + 不取消子工作流钉住的版本;发布级别不是你能改的。结构要动(比如换掉协作团队),输出 question 说明怎么拆,不要自己动手; + 报告撰写节点的 numbers、on_violation、claims 只能往严里改(numbers: strict、on_violation: fail、 + claims: require_citation),不放宽、不删掉;契约里的 claims 也一样;不关 agent 的 cite_fields,不删它的 output_schema - 要人拿主意的,不替人选、不编:required 该包括哪些指标、几张口径卡该用哪张、钉哪个版本、协作团队拆成哪几个 - 固定步骤……每一处输出一行 {"op":"question","node_id":"节点 id","text":"要问的话,把候选列出来"},那一处不改 + 固定步骤、沙箱代码节点是在取数还是在计算(不许自己把 evidence_role 设成 source)……每一处输出一行 + {"op":"question","node_id":"节点 id","text":"要问的话,把候选列出来"},那一处不改 - 只修下面列出的问题,别的不要动 - 可用操作只有 update_node(config 只写要改的字段,没写的原样保留)、add_node、add_edge 和 question; 不要输出 remove_node、remove_edge、reply @@ -1648,7 +1673,8 @@ async def assist_publish_fix( before_nodes = {str(n["id"]): n for n in copy.deepcopy(base.get("nodes") or []) if isinstance(n, dict)} before_edges = [e for e in copy.deepcopy(base.get("edges") or []) if isinstance(e, dict)] baseline = list(copy.deepcopy(before_nodes).values()) - errors = _blocking_issues(before_nodes, before_edges, sources=sources, baseline=baseline, level=level) or [] + errors = _blocking_issues(before_nodes, before_edges, sources=sources, baseline=baseline, level=level, + defaults=base.get("defaults")) or [] out: dict[str, Any] = {"accepted": False, "reason": None, "summary": "", "questions": [], "graph": graph, "ops": []} if not errors: @@ -1705,7 +1731,7 @@ async def assist_publish_fix( proposed = _proposed_graph(base, nodes, edges) reasons = forbidden_changes(base, proposed, tried) after = _blocking_issues(_nodes_by_id(proposed), proposed.get("edges") or [], sources=sources, - baseline=baseline, level=level) + baseline=baseline, level=level, defaults=proposed.get("defaults")) if reasons: out["reason"] = "Copilot 的修改降低了要求,已作废:" + ";".join(reasons) elif after is None: diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index 1eb82cf9..5a8772b0 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -8,11 +8,14 @@ - 查询快照:tool.end 事件里的 query_artifact,或者 node.finished.evidence 里的 query 条目 - 成果(哪些字段是报告原文):出口节点 node.finished 事件里的 node_output 工件 - 运行输入:入口节点 node.finished 事件里的 node_output 工件 +- 表结构快照:tool.end 事件里的 schema_artifact,或者 node.finished.evidence 里的 schema 条目 +- 检索快照:retrieve.end 事件里的 artifact,或者 node.finished.evidence 里的 retrieval 条目 artifacts 表可以事后插行,run.output 是可以改写的一列,封存之后追加的事件不在 核对范围里——这三样都不当来源。解析数字层的链:口径卡指标 → 它的输入 → 查询快照里 -的那一格,或者单元格引用直接到查询快照;旧运行按旧契约的位置信息标出 matched -(legacy_contract),不按数值猜来源。 +的那一格,或者单元格引用直接到查询快照;表名、字段名到表结构快照和查询;原话到检索 +命中的那一段。旧运行按旧契约的位置信息标出 matched(legacy_contract);没有契约的旧答案 +按数值在封存的查询快照、口径卡里猜候选(legacy_text),只做展示,写明是猜的。 查询步骤只给被引用的行和前后各 2 行,完整快照仍走 /api/artifacts/{id}。数据源 options.mask_columns 里的列换成「已遮罩」——在有身份体系之前,这只减少暴露,不是 @@ -20,10 +23,16 @@ options.mask_columns 里的列换成「已遮罩」——在有身份体系之 """ from __future__ import annotations +import asyncio +import csv +import io +import re +from collections import OrderedDict from dataclasses import dataclass, field from typing import Any -from fastapi import APIRouter +from fastapi import APIRouter, Query +from fastapi.responses import JSONResponse, Response from sqlalchemy import select from app.api.coded import CodedHTTPException @@ -32,13 +41,27 @@ from app.data.engine import masked_columns from app.db.base import SessionLocal from app.db.models import DataSource, Run, RunEvent, Workflow from app.engine.evidence import ( + ENTITY_KINDS, + GUESS_NOTE, + GUESS_SCHEMA, + MISSING, + UNKNOWN_ENTITY_REASON, + UNVERIFIED_ENTITY_REASON, RenderError, + build_catalog, + closest_entities, find_segment, + guess_sources, input_eid, iter_units, make_eid, + normalize_quote, + query_entry_fields, render_metric, + sql_tables, + uncited_claims, ) +from app.engine.toolcalls import QUERY_PREFIX router = APIRouter(prefix="/api/runs", tags=["evidence"]) @@ -65,6 +88,9 @@ MAX_WINDOW_ROWS = 50 UNSEALED_QUERY = "这份查询快照不在封存范围内的任何事件里(没有哪次查询在封存前交回过它),不能当证据展示" TAMPERED_QUERY = "查询快照和它的哈希对不上,疑似被改过,不展示其中的行" MISSING_QUERY = "查询快照在工件库里取不回来" +UNSEALED_RETRIEVAL = "这份检索快照不在封存范围内的任何事件里(没有哪次检索在封存前交回过它),不能当证据展示" +TAMPERED_RETRIEVAL = "检索快照和它的哈希对不上,疑似被改过,不展示原文" +MISSING_RETRIEVAL = "检索快照在工件库里取不回来,或者里面没有这条命中" # -------------------------------------------------------------------------- @@ -86,6 +112,16 @@ class _Sealed: queries: dict[str, dict[str, Any]] = field(default_factory=dict) #: 口径卡节点 id → 封存范围内它的 caliber.upgrade 事件(升版处置) upgrades: dict[str, dict[str, Any]] = field(default_factory=dict) + #: 封存范围内 node.finished.evidence 的台账条目,按事件顺序(给可疑实体找「最接近的名字」重建目录用) + entries: list[dict[str, Any]] = field(default_factory=list) + #: 封存范围内交回过的表结构快照:工件 id → {node_id, source} + schemas: dict[str, dict[str, Any]] = field(default_factory=dict) + #: 封存范围内交回过的检索快照:工件 id → {node_id, source} + retrievals: dict[str, dict[str, Any]] = field(default_factory=dict) + #: 封存时记下的清单哈希(没封存的是 None):旧答案的猜测按 (run_id, 它) 缓存 + manifest_hash: str | None = None + #: verify_manifest 的完整结论(审计表照实附上) + verdict: dict[str, Any] = field(default_factory=dict) @property def sealed(self) -> bool: @@ -125,19 +161,25 @@ async def _sealed(run_id: str) -> _Sealed: select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data) .where(RunEvent.run_id == run_id).order_by(RunEvent.seq))] graph = run.graph if isinstance(run.graph, dict) else {} + manifest = run.manifest_hash verdict = await verify_manifest(run_id) bound = verdict.get("sealed_at") if verdict.get("sealed") else None # 没封存(还在跑、停在审批)的运行没有「封存之后」:现有的事件都算,但每件证据都标未封存 events = [r for r in rows if bound is None or r[0] <= bound] out = _Sealed(run_id=run_id, graph=graph, events=events, seal={ "sealed": bool(verdict.get("sealed")), "ok": verdict.get("ok"), "manifest_seq": bound, - "legacy": bool(verdict.get("legacy", False))}) + "legacy": bool(verdict.get("legacy", False))}, manifest_hash=manifest, verdict=verdict) for _, etype, node_id, data in events: if not node_id or not isinstance(data, dict): continue - if etype == "tool.end" and data.get("query_artifact"): - out.queries.setdefault(str(data["query_artifact"]), { - "node_id": node_id, "tool": data.get("tool"), "via": data.get("artifact")}) + if etype == "tool.end": + if data.get("query_artifact"): + out.queries.setdefault(str(data["query_artifact"]), { + "node_id": node_id, "tool": data.get("tool"), "via": data.get("artifact")}) + if data.get("schema_artifact"): + out.schemas.setdefault(str(data["schema_artifact"]), {"node_id": node_id}) + elif etype == "retrieve.end" and data.get("artifact"): + out.retrievals.setdefault(str(data["artifact"]), {"node_id": node_id, "source": data.get("collection")}) elif etype == "caliber.upgrade": out.upgrades[node_id] = data if etype != "node.finished": @@ -147,10 +189,16 @@ async def _sealed(run_id: str) -> _Sealed: for entry in data.get("evidence") or []: if isinstance(entry, dict) and entry.get("artifact"): out.ledger.add((node_id, entry["artifact"])) + out.entries.append({**entry, "node_id": node_id}) + artifact = str(entry["artifact"]) if entry.get("kind") == "query": - out.queries[str(entry["artifact"])] = { + out.queries[artifact] = { "node_id": node_id, **{k: entry[k] for k in ("tool", "source", "via", "call_id", "exec") if entry.get(k) is not None}} + elif entry.get("kind") == "schema": + out.schemas[artifact] = {"node_id": node_id, "source": entry.get("source")} + elif entry.get("kind") == "retrieval": + out.retrievals[artifact] = {"node_id": node_id, "source": entry.get("source")} return out @@ -170,6 +218,8 @@ class _Report: #: True 取回并复验通过;False 内容和哈希对不上(或者不是这次运行这个节点写的);None 文件不在了 hash_ok: bool | None fields: list[str] + #: 写这份报告时的结论句策略(report.checked 里记的,升级前的运行没有) + claims: str | None = None def _reports(sealed: _Sealed) -> list[_Report]: @@ -194,7 +244,8 @@ def _reports(sealed: _Sealed) -> list[_Report]: for f in r.get("fields") or []] out.append(_Report(node_id=node_id, doc_artifact=artifact, ok=data.get("ok"), repairs=data.get("repairs"), stats=(doc or {}).get("stats") or data.get("stats"), - doc=doc if isinstance(doc, dict) else None, hash_ok=hash_ok, fields=fields)) + doc=doc if isinstance(doc, dict) else None, hash_ok=hash_ok, fields=fields, + claims=data.get("claims") if isinstance(data.get("claims"), str) else None)) return out @@ -217,7 +268,20 @@ def _entry_sealed(sealed: _Sealed, entry: dict[str, Any], inputs: dict[str, Any] name = (entry.get("locator") or {}).get("field") return name in inputs and inputs[name] == entry.get("value") \ and entry.get("eid") == input_eid(name, entry.get("value")) - return (str(entry.get("node_id") or ""), str(entry.get("artifact") or "")) in sealed.ledger + artifact = str(entry.get("artifact") or "") + if entry.get("kind") in ENTITY_KINDS: + # 表和字段不属于哪个节点:eid 背后的表结构快照或查询快照是封存范围内交回过的就算 + return artifact in sealed.schemas or artifact in sealed.queries + if entry.get("kind") == "retrieval" and artifact in sealed.retrievals: + return True + return (str(entry.get("node_id") or ""), artifact) in sealed.ledger + + +def _origin_sealed(sealed: _Sealed, origin: dict[str, Any]) -> bool: + """实体的一个来历(schema / sql / result)背后的工件是不是封存范围内交回过的。""" + artifact = str(origin.get("artifact") or "") + pool = sealed.schemas if origin.get("kind") == "schema" else sealed.queries + return sealed.trusted and artifact in pool # -------------------------------------------------------------------------- @@ -227,21 +291,14 @@ def _entry_sealed(sealed: _Sealed, entry: dict[str, Any], inputs: dict[str, Any] @router.get("/{run_id}/evidence") async def evidence_graph(run_id: str) -> dict[str, Any]: - """整次运行的证据图:报告、每件证据、谁引用了谁。mode 是 cited / legacy_contract / none。""" + """整次运行的证据图:报告、每件证据、谁引用了谁。mode 是 cited / legacy_contract / legacy_text / none。""" sealed = await _sealed(run_id) reports = _reports(sealed) body: dict[str, Any] = {"run_id": run_id, "schema": SCHEMA, "seal": sealed.seal, "reports": [], "evidence": [], "edges": []} - if not reports: - issuance = _issuance_of(sealed) - if issuance is None: - return {**body, "mode": "none", "note": NONE_NOTE} - if issuance.get("mode") == "citations": - # 引用模式的契约,报告却被跳过、或者文档没落下来:不是旧版出具,照实说没有文档,并带上契约记的缘由 - gaps = [str(g) for g in issuance.get("gaps") or [] if g] - why = next((g for g in gaps if "报告" in g), gaps[0] if gaps else None) - return {**body, "mode": "none", "note": f"{NO_DOC_NOTE}:{why}" if why else NO_DOC_NOTE} - return {**body, "mode": "legacy_contract", "legacy": _legacy(sealed, issuance)} + mode, extra = await _mode_of(sealed, reports) + if mode != "cited": + return {**body, "mode": mode, **extra} inputs = _input_payloads(sealed) for report in reports: @@ -257,6 +314,27 @@ async def evidence_graph(run_id: str) -> dict[str, Any]: return {**body, "mode": "cited"} +async def _mode_of(sealed: _Sealed, reports: list[_Report]) -> tuple[str, dict[str, Any]]: + """这次运行按哪种方式展示证据:(mode, 附带的 note 或 legacy)。 + + 有封存的报告文档就是 cited;否则看成果上的契约:引用模式的契约却没有文档照实说没有(none); + 旧契约按它记下的 matched 标(legacy_contract);没有契约的答案按数值猜候选(legacy_text), + 答案里没有数字、或者封存范围里没有查询和口径卡可比时是 none。 + """ + if reports: + return "cited", {} + issuance = _issuance_of(sealed) + if issuance is None: + legacy = await _legacy_text(sealed) + return ("legacy_text", {"legacy": legacy}) if legacy is not None else ("none", {"note": NONE_NOTE}) + if issuance.get("mode") == "citations": + # 引用模式的契约,报告却被跳过、或者文档没落下来:不是旧版出具,照实说没有文档,并带上契约记的缘由 + gaps = [str(g) for g in issuance.get("gaps") or [] if g] + why = next((g for g in gaps if "报告" in g), gaps[0] if gaps else None) + return "none", {"note": f"{NO_DOC_NOTE}:{why}" if why else NO_DOC_NOTE} + return "legacy_contract", {"legacy": _legacy(sealed, issuance)} + + def _graph_of(report: _Report, sealed: _Sealed, inputs: dict[str, Any]) -> tuple[list, list]: doc = report.doc or {} cited_by: dict[str, list[str]] = {} @@ -294,6 +372,15 @@ def _graph_of(report: _Report, sealed: _Sealed, inputs: dict[str, Any]) -> tuple if entry.get("kind") in ("query", "retrieval"): # 查询、检索条目带上是什么、有多大:图上不点开也看得出这是哪次取数 item.update({k: entry[k] for k in ("tool", "source", "columns", "rows", "truncated") if k in entry}) + elif entry.get("kind") in ENTITY_KINDS: + # 表和字段:叫什么、出现在哪几次查询里、来历(表结构快照 / SQL / 结果列)各自封存了没有 + item.update({k: entry[k] for k in ("name", "table", "tables", "qualified", "source") if entry.get(k)}) + item["queries"] = [str(q) for q in entry.get("queries") or []] + item["sources"] = [{**{k: o[k] for k in ("kind", "artifact", "alias", "source", "truncated") + if o.get(k) is not None}, "sealed": _origin_sealed(sealed, o)} + for o in entry.get("sources") or [] if isinstance(o, dict)] + edges.extend({"from": alias, "to": q, "rel": "appears_in", "report": report.node_id} + for q in item["queries"]) evidence.append(item) if entry.get("kind") != "metric" or not entry.get("artifact"): continue @@ -367,7 +454,15 @@ def _legacy(sealed: _Sealed, issuance: dict[str, Any]) -> dict[str, Any]: where = isinstance(item.get("start"), int) and isinstance(item.get("end"), int) return {**item, "positioned": where} - matched = [positioned(m) for m in issuance.get("matched") or [] if isinstance(m, dict)] + # 指标的名字、值、口径只从封存的口径卡里取,而且只看这份契约当时收的那几张卡(_issuance.calibers) + cards = _metric_sources(sealed, [c.get("node") for c in issuance.get("calibers") or [] if isinstance(c, dict)]) + + def sources_of(item: dict[str, Any]) -> list[dict[str, Any]]: + ids = [item["metric"]] if item.get("metric") else list(item.get("candidates") or []) + return [dict(src) for mid in ids for src in cards.get(str(mid), [])] + + matched = [{**positioned(m), "sources": sources_of(m)} for m in issuance.get("matched") or [] + if isinstance(m, dict)] unmatched = [positioned(u) for u in issuance.get("unmatched_numbers") or [] if isinstance(u, dict)] placed = [m for m in matched + unmatched if m["positioned"]] fields = [name for name, value in output.items() @@ -377,6 +472,169 @@ def _legacy(sealed: _Sealed, issuance: dict[str, Any]) -> dict[str, Any]: "field": fields[0] if len(fields) == 1 else None} +def _sealed_cards(sealed: _Sealed) -> list[tuple[str, str, dict[str, Any]]]: + """封存范围内的口径卡:(节点, 工件, 指标集)。取回时复验哈希,对不上的、取不回来的不要。 + + 一期以后口径卡把指标集落成 metric_set 工件、记在台账里;一期以前没有,节点的产出(node.finished + 的 node_output 工件)本身就是指标集——只对画布上确实是口径卡的节点这样认。 + """ + out: list[tuple[str, str, dict[str, Any]]] = [] + seen: set[tuple[str, str]] = set() + calibers = set(sealed.nodes_of("metrics")) + for _, etype, node_id, data in sealed.events: + if etype != "node.finished" or not node_id or not isinstance(data, dict): + continue + arts = [str(e["artifact"]) for e in data.get("evidence") or [] + if isinstance(e, dict) and e.get("kind") == "metric_set" and e.get("artifact")] + if not arts and node_id in calibers and data.get("artifact"): + arts = [str(data["artifact"])] + for art in arts: + if (node_id, art) in seen: + continue + seen.add((node_id, art)) + content, ok = _fetch(art) + if ok is True and isinstance(content, dict) and isinstance(content.get("metrics"), list): + out.append((node_id, art, content)) + return out + + +def _metric_sources(sealed: _Sealed, nodes: list[Any]) -> dict[str, list[dict[str, Any]]]: + """指标 id → 它在封存的口径卡里的样子(名字、值、渲染、口径、工件、eid)。nodes 为空时不限卡。""" + wanted = {str(n) for n in nodes if n} + out: dict[str, list[dict[str, Any]]] = {} + for node_id, art, card in _sealed_cards(sealed): + if wanted and node_id not in wanted: + continue + for metric in card["metrics"]: + if not isinstance(metric, dict) or not metric.get("id"): + continue + mid = str(metric["id"]) + try: + rendered = render_metric(metric) + except RenderError: + rendered = MISSING + out.setdefault(mid, []).append({ + "kind": "metric", "ref": f"m:{mid}", "metric": mid, "name": metric.get("name") or mid, + "value": metric.get("value"), "rendered": rendered, "unit": metric.get("unit") or "", + "caliber": card.get("caliber") or "", "version": card.get("caliber_version") or "", + "node_id": node_id, "artifact": art, "eid": make_eid("metric", art, {"metric": mid}), + "sealed": sealed.trusted, + }) + return out + + +# -------------------------------------------------------------------------- +# 没有契约的旧答案:按数值猜候选(legacy_text) +# -------------------------------------------------------------------------- + +#: 猜测的缓存:(run_id, manifest_hash, 数据源遮罩的指纹) → legacy 对象(猜不出来的是 None)。 +#: 只缓存封存完好的运行:还在跑、停在审批的运行事件还会变,封存链断了的运行不该被一份旧结果盖住 +GUESS_CACHE_SIZE = 64 +_guess_cache: OrderedDict[tuple[Any, ...], dict[str, Any] | None] = OrderedDict() +NO_CANDIDATE = "旧答案里的这个数,在这次运行封存的查询结果和口径卡里没有找到相同的值" + + +async def _mask_fingerprint() -> dict[str, list[str]]: + """每个数据源现在设的遮罩(列名小写、排好序)。进缓存键:事后加了遮罩,缓存的猜测要作废—— + 候选的值、渲染、差值都会把遮住的原值带出去。""" + async with SessionLocal() as session: + rows = (await session.execute(select(DataSource.name, DataSource.options))).all() + masks = {str(name): sorted({c.lower() for c in masked_columns(options)}) for name, options in rows} + return {name: cols for name, cols in masks.items() if cols} + + +async def _legacy_text(sealed: _Sealed) -> dict[str, Any] | None: + """成果里每个文字字段的数字,去封存范围内的查询快照、口径卡里按数值找候选。只做展示,不是证据。 + + 查询快照、口径卡只认封存范围内的事件引用得到、取回时复验过哈希的(见 _legacy_pool);artifacts 表 + 里人为插的行、封存之后追加的事件都不看。猜测是同步的 CPU 活,连同读盘放进线程;结果按 + (run_id, manifest_hash) 加遮罩指纹缓存。 + """ + output = sealed.payload_of(sealed.nodes_of("output")) or {} + texts = [(name, value) for name, value in output.items() + if isinstance(name, str) and not name.startswith("_") and isinstance(value, str) and value.strip()] + if not texts: + return None + masks = await _mask_fingerprint() + key = (sealed.run_id, sealed.manifest_hash, tuple(sorted((n, tuple(c)) for n, c in masks.items()))) + cacheable = sealed.trusted and bool(sealed.manifest_hash) + if cacheable and key in _guess_cache: + _guess_cache.move_to_end(key) + return _guess_cache[key] + legacy = await asyncio.to_thread(_guess_legacy, sealed, texts, masks) + if cacheable: + _guess_cache[key] = legacy + while len(_guess_cache) > GUESS_CACHE_SIZE: + _guess_cache.popitem(last=False) + return legacy + + +def _guess_legacy(sealed: _Sealed, texts: list[tuple[str, str]], + masks: dict[str, list[str]]) -> dict[str, Any] | None: + pool = _legacy_pool(sealed) + if not pool: + return None + # 遮罩:查询当时记下的(快照里的 mask_columns,guess_sources 自己会用)加数据源现在设的 + masked = {item["artifact"]: masks.get(str(item["content"].get("source") or ""), []) + for item in pool if item["kind"] == "query"} + fields = [] + for name, text in texts: + guess = guess_sources(text, pool, masked=masked) + if guess["stats"]["numbers"]: + fields.append({"field": name, "markdown": guess["markdown"], "segments": guess["segments"], + "stats": guess["stats"]}) + if not fields: + return None + return {"schema": GUESS_SCHEMA, "mode": "legacy_text", "note": GUESS_NOTE, "fields": fields, + "stats": {k: sum(f["stats"][k] for f in fields) for k in fields[0]["stats"]}, + "sources": {"queries": sum(1 for i in pool if i["kind"] == "query"), + "metric_sets": sum(1 for i in pool if i["kind"] == "metric_set")}, + "sealed": sealed.trusted} + + +def _legacy_pool(sealed: _Sealed) -> list[dict[str, Any]]: + """猜候选用的已封存条目(guess_sources 的 sealed 参数),按在封存事件里第一次出现的顺序。 + + 查询快照三条路:tool.end.query_artifact、node.finished.evidence 的 query 条目,以及二期以前的运行 + ——那时 tool.end 只有 tool_snapshot,查询快照的 id 嵌在它的内容里(数据源工具交回的 JSON), + 内容寻址,所以同样追得到。只认数据源工具(db_query__):别的工具拼一个同样形状的 JSON,里面的 + id 不是我们落的。查询按这个顺序编 Q1、Q2…,只为候选好认,和哪份报告的目录都无关。 + """ + pool: list[dict[str, Any]] = [] + seen: set[str] = set() + + def query(artifact: Any, node_id: str, tool: Any) -> None: + artifact = str(artifact or "") + if not artifact or artifact in seen: + return + seen.add(artifact) + content, ok = _fetch(artifact) + if ok is True and isinstance(content, dict) and isinstance(content.get("rows"), list): + number = sum(1 for item in pool if item["kind"] == "query") + 1 + pool.append({"kind": "query", "artifact": artifact, "content": content, "alias": f"Q{number}", + "node_id": node_id, **({"tool": str(tool)} if tool else {})}) + + for _, etype, node_id, data in sealed.events: + if etype not in ("tool.end", "node.finished") or not node_id or not isinstance(data, dict): + continue + if etype == "tool.end": + tool = str(data.get("tool") or "") + if data.get("query_artifact"): + query(data["query_artifact"], node_id, tool) + elif tool.startswith(QUERY_PREFIX) and data.get("artifact"): + snap, ok = _fetch(str(data["artifact"])) + if ok is True and isinstance(snap, dict) and str(snap.get("tool") or "").startswith(QUERY_PREFIX): + if fields := query_entry_fields(snap.get("result")): + query(fields["artifact"], node_id, tool) + continue + for entry in data.get("evidence") or []: + if isinstance(entry, dict) and entry.get("kind") == "query": + query(entry.get("artifact"), node_id, entry.get("tool")) + pool.extend({"kind": "metric_set", "artifact": art, "content": card, "node_id": node_id} + for node_id, art, card in _sealed_cards(sealed)) + return pool + + # -------------------------------------------------------------------------- # GET /api/runs/{run_id}/evidence/segments/{segment_id} # -------------------------------------------------------------------------- @@ -405,7 +663,7 @@ async def evidence_segment(run_id: str, segment_id: str, report: str | None = No doc = chosen.doc markdown = str(doc.get("markdown") or "") start, end = unit.get("span") or [0, 0] - chain = await _chain(seg, doc, sealed) + chain = await _chain(seg, doc, sealed, chosen.node_id) covered = sealed.trusted and all(step.get("sealed") for step in chain if "sealed" in step) masked = list(dict.fromkeys(c for step in chain if step.get("step") == "query" for c in step.get("masked") or [])) return { @@ -435,10 +693,17 @@ def _choose(reports: list[_Report], wanted: str | None, sealed: _Sealed) -> _Rep return next((r for r in reports if r.node_id == primary.get("report_node")), reports[0]) -async def _chain(seg: dict[str, Any], doc: dict[str, Any], sealed: _Sealed) -> list[dict[str, Any]]: +async def _chain(seg: dict[str, Any], doc: dict[str, Any], sealed: _Sealed, + report_node: str) -> list[dict[str, Any]]: cite = seg.get("cite") or {} entry = (doc.get("catalog") or {}).get(cite.get("alias")) or {} masks = _Masks() + if seg.get("kind") == "entity" and seg.get("issue") in SUSPICIOUS: + return [_suspicious_step(seg, cite, sealed, report_node)] + if seg.get("kind") == "entity" and seg.get("state") == "deterministic" and cite.get("status") == "resolved": + return [_entity_step(seg, cite, entry, sealed)] + if seg.get("kind") == "quote" and seg.get("state") == "deterministic" and cite.get("status") == "resolved": + return [_quote_step(seg, cite, entry, sealed)] if _missing_input(cite, entry): # 指标在卡里、只是这次没有值(显示「—」):照样给出指标、输入和查询步骤,面板才指得出是哪个 # 输入空了、那一格在哪次查询里。片段上没记定位和 eid,用目录条目的(它们指的是同一个指标) @@ -620,14 +885,259 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str return step +# -------------------------------------------------------------------------- +# 表名、字段名、原话 +# -------------------------------------------------------------------------- + +#: 片段上的可疑实体:名字哪里都找不到(unknown_entity),或者表结构快照不全、核对不了(unverified_entity) +SUSPICIOUS = frozenset({"unknown_entity", "unverified_entity"}) + + +def _fetch(artifact: str) -> tuple[Any, bool | None]: + """取回一件工件:(内容, hash_ok)。hash_ok 为 True 复验通过,False 内容和哈希对不上,None 取不回来。""" + try: + content = load(artifact) + except ValueError: + return None, False + return content, (True if content is not None else None) + + +def _schema_table(content: Any, name: Any) -> dict[str, Any] | None: + """表结构快照里的一张表:表名一字不差的优先,其次不分大小写、全名(schema.表)也认。""" + tables = content.get("tables") if isinstance(content, dict) else None + if not isinstance(tables, dict) or not name: + return None + if isinstance(tables.get(name), dict): + return tables[name] + lowered = str(name).lower() + return next((t for key, t in tables.items() if isinstance(t, dict) + and lowered in (str(key).lower(), str(t.get("qualified") or "").lower())), None) + + +def _schema_column(table: dict[str, Any] | None, name: Any) -> dict[str, Any] | None: + columns = [c for c in (table or {}).get("columns") or [] if isinstance(c, dict)] + return next((c for c in columns if c.get("name") == name), None) \ + or next((c for c in columns if str(c.get("name") or "").lower() == str(name or "").lower()), None) + + +def _entity_present(content: Any, kind: str, name: str, table: Any, column: Any, tables: list[Any]) -> bool: + """目录条目说的表、字段在这份表结构快照里确实有:文档是内容寻址的,这一步防的是目录和快照张冠李戴。""" + if kind == "table": + return _schema_table(content, name) is not None + owners = [table] if table else list(tables or []) + return any(_schema_column(_schema_table(content, t), column) is not None for t in owners) + + +def _query_present(content: Any, kind: str, name: str, column: Any) -> bool: + """查询快照里确实有这个名字:表在它的 SQL 里(FROM / JOIN),列在它的结果列里。""" + if not isinstance(content, dict): + return False + if kind == "table": + return name.lower() in {t.lower() for t in sql_tables(str(content.get("sql") or ""))} + return str(column) in [str(c) for c in content.get("columns") or []] + + +def _entity_step(seg: dict[str, Any], cite: dict[str, Any], entry: dict[str, Any], sealed: _Sealed) -> dict[str, Any]: + """实体步骤:这个名字出现在哪几次查询里(SQL 用到的表、结果的列)、表结构快照什么时候同步的、字段类型。 + + 来源(entry.sources)逐个核对:表结构快照要出现在封存范围内的 tool.end.schema_artifact 或 schema 台账 + 条目里,查询快照要是封存范围内交回过的那几件。类型、同步时间只从核对过、复验过哈希的表结构快照取, + 取不到就不给。eid 背后的那件工件核对通过、名字确实在里面,这一步才算已封存。 + """ + kind = str(cite.get("kind") or entry.get("kind") or "") + locator = dict(entry.get("locator") or cite.get("locator") or {}) + table, column = locator.get("table"), locator.get("column") + name = str(entry.get("name") or cite.get("rendered") or "") + artifact = str(entry.get("artifact") or "") + step: dict[str, Any] = { + "step": "entity", "status": "resolved", "kind": kind, "alias": cite.get("alias"), "name": name, + **({"table": table} if table else {}), **({"column": column} if kind == "column" and column else {}), + **{k: entry[k] for k in ("qualified", "tables", "source") if entry.get(k)}, + "queries": [str(q) for q in entry.get("queries") or []], "sources": [], + "artifact": artifact or None, "eid": cite.get("eid"), + "eid_ok": bool(artifact) and cite.get("eid") == entry.get("eid") == make_eid(kind, artifact, locator), + } + for flag in ("code", "auto"): + if seg.get(flag): + step[flag] = True + snapshots: list[dict[str, Any]] = [] # 核对过、复验过哈希、名字确实在里面的表结构快照 + results: list[dict[str, Any]] = [] # 核对过、复验过哈希的查询快照(聚合别名的列类型从这里取) + backed = False + for origin in entry.get("sources") or []: + if not isinstance(origin, dict) or not origin.get("artifact"): + continue + art = str(origin["artifact"]) + item: dict[str, Any] = {k: origin[k] for k in ("kind", "artifact", "alias", "source", "truncated") + if origin.get(k) is not None} + if origin.get("kind") == "schema": + item["sealed"] = art in sealed.schemas and sealed.trusted + ok = False + if art in sealed.schemas: + content, item["hash_ok"] = _fetch(art) + item["present"] = item["hash_ok"] is True and _entity_present( + content, kind, name, table, column, list(entry.get("tables") or [])) + ok = item["present"] + if ok: + snapshots.append(content) + else: + info = sealed.queries.get(art) + item.update({k: info[k] for k in ("node_id", "tool") if info and info.get(k)}) + item["sealed"] = info is not None and sealed.trusted + ok = info is not None + if ok and (art == artifact or (kind == "column" and origin.get("kind") == "result")): + content, hash_ok = _fetch(art) + present = hash_ok is True and _query_present(content, kind, name, column) + if art == artifact: + item.update(hash_ok=hash_ok, present=present) + ok = present + if present and origin.get("kind") == "result": + results.append(content) + backed = backed or (art == artifact and ok) + step["sources"].append(item) + step["sealed"] = sealed.trusted and backed and step["eid_ok"] + if snapshots: + _describe_entity(step, snapshots[0], kind, name, table, column, list(entry.get("tables") or [])) + if kind == "column" and not any(o.get("kind") == "schema" for o in step["sources"]): + # 聚合的别名这类只在结果里出现的列:类型取查询时从驱动的原始值记下的列类型。表结构里有的列 + # 只认表结构快照的类型,快照核对不过就不给——不拿结果列的类型顶上 + kinds = [c.get("column_types", {}).get(str(column)) for c in results if isinstance(c.get("column_types"), dict)] + if kinds and isinstance(kinds[0], str): + step.update(type=kinds[0], type_source="result") + return step + + +def _describe_entity(step: dict[str, Any], snap: dict[str, Any], kind: str, name: str, table: Any, column: Any, + tables: list[Any]) -> None: + """从表结构快照里补上同步时间、表的概况、字段的类型。""" + if snap.get("synced_at"): + step["synced_at"] = snap["synced_at"] + if snap.get("truncated"): + step["snapshot_truncated"] = True + if kind == "table": + info = _schema_table(snap, name) or {} + step["columns"] = len([c for c in info.get("columns") or [] if isinstance(c, dict)]) + step["is_view"] = bool(info.get("is_view")) + if info.get("comment"): + step["comment"] = info["comment"] + return + owners = [table] if table else tables + found = {str(t): (tinfo, _schema_column(tinfo, column)) for t in owners + if (tinfo := _schema_table(snap, t)) is not None} + types = {t: str(c.get("type") or "") for t, (_, c) in found.items() if c is not None} + if table and table in found and found[table][1] is not None: + tinfo, col = found[table] + step.update(type=str(col.get("type") or ""), type_source="schema", nullable=bool(col.get("nullable", True)), + primary_key=col.get("name") in (tinfo.get("primary_key") or [])) + if col.get("comment"): + step["comment"] = col["comment"] + elif types: + # 好几张表都有的同名字段:各表的类型分别列出,一样的话再给一个 type + step["types"] = types + if len(set(types.values())) == 1: + step.update(type=next(iter(types.values())), type_source="schema") + + +def _scope_catalog(sealed: _Sealed, report_node: str) -> dict[str, Any]: + """用封存的台账按报告节点同一套参数(祖先范围、evidence_from、entities)重建的目录。 + + doc.catalog 只留了正文用到的实体,给「是不是想写…」找候选不够;运行状态(checkpoint)不在封存 + 范围里,也不能用。图读不出来、找不到报告节点时退回全部封存台账。 + """ + from app.engine.nodes.report import report_catalog + from app.engine.schema import GraphSpec + + try: + spec = GraphSpec.model_validate(sealed.graph) + node = spec.node_map().get(report_node) + except Exception: # noqa: BLE001 - 老运行的图可能已经不合现在的格式,退回不分范围 + node = None + if node is None: + return build_catalog(nodes={}, ledger=sealed.entries) + return report_catalog({"evidence": sealed.entries, "nodes": {}, "input": {}}, spec, node) # type: ignore[arg-type] + + +def _suspicious_step(seg: dict[str, Any], cite: dict[str, Any], sealed: _Sealed, report_node: str) -> dict[str, Any]: + """可疑实体:照实说查过哪些地方都没有这个名字,附上最接近的已知名字(最多 3 个)当提示。 + + 提示只从封存的台账里找——一个编造的名字,提示里再冒出一个封存链外的名字,就是在帮它圆。 + """ + unsure = seg.get("issue") == "unverified_entity" + name = str(seg.get("name") or cite.get("ref") or seg.get("text") or "").strip().strip("`") + catalog = _scope_catalog(sealed, report_node) + entities = [e for e in catalog.values() if isinstance(e, dict) and e.get("kind") in ENTITY_KINDS] + origins = [o for e in entities for o in e.get("sources") or [] if isinstance(o, dict)] + closest = [] + for alias in closest_entities(name, catalog) if name else []: + found = catalog[alias] + table = (found.get("locator") or {}).get("table") if found.get("kind") == "column" else None + closest.append({"alias": alias, "kind": found.get("kind"), "name": found.get("name"), + **({"table": table} if table else {})}) + return { + "step": "entity", "status": "unverified" if unsure else "unknown", "name": name, + **({"kind": cite["kind"]} if cite.get("kind") in ENTITY_KINDS else {}), + "reason": UNVERIFIED_ENTITY_REASON if unsure else UNKNOWN_ENTITY_REASON, + "closest": closest, + "checked": {"schemas": len({o.get("artifact") for o in origins if o.get("kind") == "schema"}), + "queries": len({o.get("alias") for o in origins if o.get("kind") in ("sql", "result") + and o.get("alias")})}, + } + + +def _quote_step(seg: dict[str, Any], cite: dict[str, Any], entry: dict[str, Any], sealed: _Sealed) -> dict[str, Any]: + """引文步骤:原话在哪次检索的哪条命中里、原文的 [起, 止),带上那条命中的全文好高亮。 + + 检索快照要出现在封存范围内的 retrieve.end.artifact 或 retrieval 台账条目里;取回时复验哈希, + 再按记下的位置把原文切出来和引文重新比一次(空白归一化后逐字相等)。 + """ + source = dict(cite.get("source") or {}) + artifact = str(entry.get("artifact") or source.get("artifact") or "") + locator = dict(cite.get("locator") or {}) + text = str(cite.get("rendered") or seg.get("text") or "") + info = sealed.retrievals.get(artifact) if artifact else None + step: dict[str, Any] = { + "step": "quote", "alias": cite.get("alias"), "text": text, "source": source, + "match": {k: locator.get(k) for k in ("hit", "start", "end")}, + "content": None, "collection": (info or {}).get("source") or entry.get("source"), "query": None, + "node_id": (info or {}).get("node_id") or entry.get("node_id"), "artifact": artifact or None, + "eid": cite.get("eid"), "eid_ok": bool(artifact) and cite.get("eid") == make_eid("quote", artifact, locator), + "hash_ok": None, "match_ok": None, "sealed": False, + } + if info is None: + return {**step, "note": UNSEALED_RETRIEVAL} + snap, step["hash_ok"] = _fetch(artifact) + hits = snap.get("hits") if isinstance(snap, dict) else None + at, start, end = locator.get("hit"), locator.get("start"), locator.get("end") + hit = hits[at] if isinstance(hits, list) and isinstance(at, int) and 0 <= at < len(hits) else None + content = hit.get("content") if isinstance(hit, dict) else None + if not isinstance(content, str) or not isinstance(start, int) or not isinstance(end, int): + return {**step, "note": TAMPERED_RETRIEVAL if step["hash_ok"] is False else MISSING_RETRIEVAL} + step.update(content=content, query=(snap or {}).get("query"), + collection=(snap or {}).get("collection") or step["collection"], + match_ok=0 <= start <= end <= len(content) + and normalize_quote(content[start:end]) == normalize_quote(text)) + step["sealed"] = sealed.trusted and step["hash_ok"] is True and step["match_ok"] and step["eid_ok"] + return step + + def _note(seg: dict[str, Any], unit: dict[str, Any]) -> str: """片段为什么是现在这个状态,一句人话。""" issue = seg.get("issue") + cite = seg.get("cite") or {} if issue == "uncited_number": return "这个数字没有出处:写作者直接写了数字,没有用引用标记,系统没法核对它" if issue == "unresolved_ref": - return f"引用解析不了:{(seg.get('cite') or {}).get('reason') or '证据目录里没有它'}" - if seg.get("state") == "deterministic" and (seg.get("cite") or {}).get("kind") == "cell": + return f"引用解析不了:{cite.get('reason') or '证据目录里没有它'}" + if issue == "unknown_entity": + return UNKNOWN_ENTITY_REASON + if issue == "unverified_entity": + return f"核对不了:{UNVERIFIED_ENTITY_REASON}" + if seg.get("state") == "deterministic" and seg.get("kind") == "entity": + lead = "系统在正文里认出的表名、字段名:" if seg.get("auto") else "" + return (f"{lead}这个名字在本次运行冻结的表结构快照、查询用到的表或查询结果列里确实存在。" + "只核对它存在、这次用过,不核对它的业务含义") + if seg.get("state") == "deterministic" and seg.get("kind") == "quote": + return "这段原话在知识库检索命中的片段里逐字出现(空白归一化后比对),由系统核对过" + if seg.get("state") == "deterministic" and cite.get("kind") == "cell": return "这个值由系统从查询快照里取出、按固定规则渲染,没有经过模型转写" if seg.get("state") == "deterministic": return "数字由系统从证据里取出、按口径卡的格式渲染,没有经过模型转写" @@ -638,4 +1148,304 @@ def _note(seg: dict[str, Any], unit: dict[str, Any]) -> str: return "这是标题" if unit.get("kind") == "connective": return "这是连接性文字,不陈述数据事实,所以不需要证据" - return "这是结论句里的文字。本期只核对数字,句子本身有没有依据要到后续版本由模型判断" + cites = [str(c) for c in unit.get("cites") or []] + if cites: + return (f"这是结论句里的文字,这句挂了依据:{'、'.join(cites[:4])}{' 等' if len(cites) > 4 else ''}。" + "依据支不支持这句话,要到后续版本由模型判断") + return "这是结论句里的文字,这句没有挂依据。句子本身有没有依据要到后续版本由模型判断" + + +# -------------------------------------------------------------------------- +# GET /api/runs/{run_id}/evidence/audit:记录页的审计表,可以导出 JSON / CSV +# -------------------------------------------------------------------------- + +AUDIT_SCHEMA = "agentlab.evidence.audit/1" +BAD_FORMAT = "evidence_bad_format" +BAD_GROUP = "evidence_bad_group" +#: 分组的顺序固定:有出处 / 无证据 / 可疑实体 / 旧运行猜测 +GROUPS = (("cited", "有出处"), ("none", "无证据"), ("suspicious", "可疑实体"), ("candidate", "旧运行猜测")) +#: 文档本身对不上的违规:不管落在哪个片段上都单独列一行,不能被那个片段「有出处」的行盖住 +_INTEGRITY = frozenset({"bad_schema", "segment_mismatch", "structural_text", "render_mismatch", "eid_mismatch", + "state_mismatch"}) +UNCITED_CLAIM = "这句结论没有挂依据(没有引用标记,也没有 [[see:]])" +UNCITED_CLAIM_COUNTED = "这句结论没有挂依据:这份报告要求结论句挂依据(claims: require_citation),出具时计入缺口" +UNCITED_CLAIM_WITHHELD = "这句结论没有挂依据:出具契约要求结论句挂依据,没挂的按契约不予出具(on_uncited: withhold)" +UNCITED_CLAIM_REQUIRED = ("这句结论没有挂依据:这份报告要求结论句挂依据(claims: require_citation),不过没有出具契约" + "按它判档(没配契约,或者还没到出具那一步)") +LEGACY_UNMATCHED = "旧版出具里这个数字对不上任何指标" + +#: CSV 的列:(表头, 行里的键) +CSV_COLUMNS = ( + ("分组", "group"), ("报告节点", "report"), ("成果字段", "field"), ("片段", "segment"), ("句子编号", "unit"), + ("种类", "kind"), ("原文", "text"), ("状态", "state"), ("问题", "issue"), ("引用", "ref"), ("证据", "evidence"), + ("证据种类", "evidence_kind"), ("证据编号", "eid"), ("工件", "artifact"), ("来源节点", "node_id"), + ("已封存", "sealed"), ("句子", "sentence"), ("说明", "note"), + # 整次运行的封存核对结果,每行都写:文件存下来以后响应头就没了,光看「已封存」分不出封存链断了还是没封存 + ("封存核对", "seal"), +) +SEAL_OK = "封存核对通过(封存于第 {seq} 条事件)" +SEAL_BROKEN = "封存核对没通过:封存之后有事件被改过、删过或插过" +SEAL_NONE = "未封存:运行还没跑完,或者停在人工审批" +_CSV_LABELS: dict[str, dict[Any, str]] = { + "group": dict(GROUPS), + "kind": {"number": "数字", "value": "值", "entity": "表名、字段名", "quote": "引文", "claim": "结论句", + "violation": "违规"}, + "state": {"deterministic": "有出处", "none": "无证据", "candidate": "候选"}, + "issue": {"uncited_number": "没有出处的数字", "unresolved_ref": "引用解析不了", "unknown_entity": "可能是编造的名字", + "unverified_entity": "核对不了", "uncited_claim": "结论句没挂依据", "no_candidate": "没找到候选", + **{code: "文档对不上" for code in _INTEGRITY}}, + "evidence_kind": {"metric": "口径卡指标", "cell": "查询单元格", "input": "运行输入", "table": "表", "column": "字段", + "quote": "知识库原话", "query": "查询结果"}, + "sealed": {True: "是", False: "否", None: "—"}, +} +#: 电子表格会把以这些字符开头的格子当公式算:报告、答案里的字来自模型,照原样写进去就能被人利用 +_FORMULA_LEAD = ("=", "+", "-", "@", "\t", "\r") +_PLAIN_NUMBER = re.compile(r"^[+-]?[\d,.]+%?$") + + +@router.get("/{run_id}/evidence/audit") +async def evidence_audit(run_id: str, fmt: str | None = Query(None, alias="format"), + groups: str | None = None) -> Any: + """审计视图:报告里每个有状态的片段按状态分组,附上封存核对的结果。 + + ?groups=none,suspicious 只要这几组;?format=json / csv 作为附件下载(不带 format 是给页面用的 JSON)。 + 每行的 sealed 是「背后的工件是封存范围内的事件交回过的,而且封存核对通过」,不逐件复验哈希—— + 点开片段时的证据链才逐件复验。 + """ + wanted = _wanted_groups(groups) + if fmt not in (None, "", "json", "csv"): + raise CodedHTTPException(422, f"导出格式只能是 json 或 csv,写的是 {fmt!r}", BAD_FORMAT) + sealed = await _sealed(run_id) + reports = _reports(sealed) + mode, extra = await _mode_of(sealed, reports) + rows: list[dict[str, Any]] = [] + if mode == "cited": + inputs = _input_payloads(sealed) + issuance = _issuance_of(sealed) + for report in reports: + if report.doc is not None: + rows.extend(_audit_doc(report, sealed, inputs, issuance)) + elif mode == "legacy_text": + rows = _audit_guess(extra["legacy"]) + elif mode == "legacy_contract": + legacy = extra["legacy"] + field_name = legacy.get("field") + text = (sealed.payload_of(sealed.nodes_of("output")) or {}).get(field_name) if field_name else None + rows = _audit_contract(legacy, text) + grouped = [{"key": key, "label": label, "count": len(members), "rows": members} + for key, label in GROUPS if key in wanted + for members in [[r for r in rows if r["group"] == key]]] + body: dict[str, Any] = { + "run_id": run_id, "schema": AUDIT_SCHEMA, "mode": mode, + "seal": {**sealed.seal, "events": sealed.verdict.get("events"), "message": sealed.verdict.get("message")}, + "reports": [{"node_id": r.node_id, "doc_artifact": r.doc_artifact, "hash_ok": r.hash_ok, + "doc_sealed": sealed.trusted, "fields": r.fields, "claims": r.claims, "stats": r.stats} + for r in reports], + "groups": grouped, "counts": {g["key"]: g["count"] for g in grouped}, + "total": sum(g["count"] for g in grouped), + **({"note": extra["note"]} if extra.get("note") else {}), + **({"legacy_note": extra["legacy"].get("note")} if isinstance(extra.get("legacy"), dict) else {}), + } + name = f"evidence-{run_id[:8]}" + if fmt == "csv": + seal = "ok" if sealed.trusted else "broken" if sealed.sealed else "unsealed" + label = SEAL_OK.format(seq=sealed.seal["manifest_seq"]) if sealed.trusted \ + else SEAL_BROKEN if sealed.sealed else SEAL_NONE + return Response(audit_csv([r for g in grouped for r in g["rows"]], seal=label), + media_type="text/csv; charset=utf-8", + headers={"Content-Disposition": f'attachment; filename="{name}.csv"', "X-Evidence-Seal": seal}) + if fmt == "json": + return JSONResponse(body, headers={"Content-Disposition": f'attachment; filename="{name}.json"'}) + return body + + +def _wanted_groups(raw: str | None) -> set[str]: + known = [key for key, _ in GROUPS] + if not raw: + return set(known) + picked = {part.strip() for part in raw.split(",") if part.strip()} + if unknown := picked - set(known): + raise CodedHTTPException(422, f"分组只能是 {' / '.join(known)},写的是 {'、'.join(sorted(unknown))}", BAD_GROUP) + return picked + + +def audit_csv(rows: list[dict[str, Any]], *, seal: str | None = None) -> str: + """审计行 → CSV 文本:中文表头,代码换成中文,RFC 4180 转义(逗号、引号、换行都进引号),开头带 BOM + (Excel 靠它认出 UTF-8)。以公式字符开头、又不是一个数的格子前面加 '。 + + seal 是整次运行的封存核对结论(SEAL_OK / SEAL_BROKEN / SEAL_NONE),填进每行的「封存核对」;不给就空着。""" + buf = io.StringIO() + writer = csv.writer(buf, lineterminator="\r\n") + writer.writerow([head for head, _ in CSV_COLUMNS]) + for row in rows: + line = [] + for _, key in CSV_COLUMNS: + value = seal if key == "seal" else row.get(key) + if value is None or isinstance(value, (str, bool)): + value = _CSV_LABELS.get(key, {}).get(value, value) + text = "" if value is None else str(value) + if text.startswith(_FORMULA_LEAD) and not _PLAIN_NUMBER.match(text): + text = "'" + text + line.append(text) + writer.writerow(line) + return "\ufeff" + buf.getvalue() + + +def _claim_note(report: _Report, issuance: dict[str, Any] | None) -> str: + """没挂依据的结论句那一行怎么说:按出具时实际生效的策略(_issuance.claims)说,和出具横幅一致。 + + 契约核对的就是这份报告时,以它记下的为准(策略可能来自契约、也可能被契约收紧);没有契约核对它时, + 报告节点自己要求了也只能说「要求了」,不能说出具时计入缺口。""" + if isinstance(issuance, dict) and issuance.get("mode") == "citations" \ + and (issuance.get("report") or {}).get("node_id") == report.node_id: + claims = issuance.get("claims") + if not isinstance(claims, dict) or claims.get("policy") != "require_citation": + return UNCITED_CLAIM + on_uncited = claims.get("on_uncited") or "degrade" + return UNCITED_CLAIM_WITHHELD if on_uncited == "withhold" else \ + UNCITED_CLAIM if on_uncited == "ignore" else UNCITED_CLAIM_COUNTED + return UNCITED_CLAIM_REQUIRED if report.claims == "require_citation" else UNCITED_CLAIM + + +def _audit_doc(report: _Report, sealed: _Sealed, inputs: dict[str, Any], + issuance: dict[str, Any] | None = None) -> list[dict[str, Any]]: + """一份报告的审计行:有出处、无证据、可疑实体的片段各一行;没挂依据的结论句一行;没有可画线文字的 + 违规(结构片段里的数字、粗体和链接里的可疑名字、[[see:]] 里解析不了的依据)一行。按在正文里的位置排。""" + doc = report.doc or {} + catalog = doc.get("catalog") or {} + markdown = str(doc.get("markdown") or "") + field_name = "、".join(report.fields) or None + rows: list[dict[str, Any]] = [] + listed: set[str] = set() + sentences: dict[str, str] = {} + for _, unit in iter_units(doc): + span = unit.get("span") or [0, 0] + sentence = markdown[span[0]:span[1]] + sentences[str(unit.get("id"))] = sentence + for seg in unit.get("segments") or []: + state = seg.get("state") + if state == "deterministic" and seg.get("cite"): + group = "cited" + elif state == "none" and seg.get("kind") != "structural": + group = "suspicious" if seg.get("issue") in SUSPICIOUS else "none" + else: + continue + listed.add(str(seg.get("id"))) + rows.append(_segment_row(group, report, field_name, seg, unit, sentence, catalog, sealed, inputs)) + claim_note = _claim_note(report, issuance) + for claim in uncited_claims(doc): + rows.append({"group": "none", "report": report.node_id, "field": field_name, "segment": None, + "unit": claim["unit"], "kind": "claim", "text": claim["text"], "state": "none", + "issue": "uncited_claim", "ref": None, "alias": None, "evidence_kind": None, "evidence": None, + "eid": None, "artifact": None, "node_id": None, "sealed": None, "span": claim["span"], + "sentence": claim["text"], "note": claim_note}) + for v in doc.get("violations") or []: + if not isinstance(v, dict) or (str(v.get("segment")) in listed and v.get("code") not in _INTEGRITY): + continue + rows.append({"group": "suspicious" if v.get("code") in SUSPICIOUS else "none", "report": report.node_id, + "field": field_name, "segment": v.get("segment"), "unit": v.get("unit"), "kind": "violation", + "text": v.get("text") or v.get("ref") or "", "state": "none", "issue": v.get("code"), + "ref": v.get("ref"), "alias": None, "evidence_kind": None, "evidence": None, "eid": None, + "artifact": None, "node_id": None, "sealed": None, "span": v.get("span"), + "sentence": sentences.get(str(v.get("unit")), v.get("context") or ""), "note": v.get("message")}) + rows.sort(key=lambda r: ((r["span"] or [len(markdown)])[0], r["kind"] != "claim")) + return rows + + +def _segment_row(group: str, report: _Report, field_name: str | None, seg: dict[str, Any], unit: dict[str, Any], + sentence: str, catalog: dict[str, Any], sealed: _Sealed, inputs: dict[str, Any]) -> dict[str, Any]: + cite = seg.get("cite") or {} + alias = cite.get("alias") + entry = (catalog.get(alias) or {}) if alias else {} + row: dict[str, Any] = { + "group": group, "report": report.node_id, "field": field_name, "segment": seg.get("id"), + "unit": unit.get("id"), "kind": seg.get("kind"), "text": seg.get("text"), "state": seg.get("state"), + "issue": seg.get("issue"), "ref": seg.get("ref"), "alias": alias, "evidence_kind": cite.get("kind"), + "evidence": None, "eid": cite.get("eid"), "artifact": None, "node_id": None, "sealed": None, + "span": seg.get("span"), "sentence": sentence, "note": _note(seg, unit), + } + if group != "cited": + return row + kind = cite.get("kind") + if kind == "cell": + artifact = str(entry.get("artifact") or "") + row.update(evidence=_cell_name(alias, cite.get("locator") or {}), artifact=artifact or None, + node_id=entry.get("node_id"), sealed=sealed.trusted and artifact in sealed.queries) + elif kind == "quote": + source = cite.get("source") or {} + artifact = str(source.get("artifact") or entry.get("artifact") or "") + title = source.get("title") + row.update(evidence=f"{alias} · {title}" if title else alias, artifact=artifact or None, + node_id=entry.get("node_id"), sealed=sealed.trusted and artifact in sealed.retrievals) + else: + row.update(evidence=entry.get("label") or alias, artifact=entry.get("artifact"), node_id=entry.get("node_id"), + sealed=_entry_sealed(sealed, entry, inputs) if entry else False) + return row + + +def _around(text: str, span: list[int]) -> str: + return text[max(0, span[0] - 18):span[1] + 18].replace("\n", " ") + + +def _audit_guess(legacy: dict[str, Any]) -> list[dict[str, Any]]: + """legacy_text 的审计行:有候选的数字进「旧运行猜测」,没有的进「无证据」。候选只是猜的, + 所以这里没有 eid、没有「已封存」的说法——一律写明是猜测。""" + rows = [] + for field_guess in legacy.get("fields") or []: + text = str(field_guess.get("markdown") or "") + for seg in field_guess.get("segments") or []: + if seg.get("kind") != "number": + continue + candidates = list(seg.get("candidates") or []) + top = candidates[0] if candidates else {} + rows.append({ + "group": "candidate" if candidates else "none", "report": None, "field": field_guess.get("field"), + "segment": seg.get("id"), "unit": None, "kind": "number", "text": seg.get("text"), + "state": seg.get("state"), "issue": None if candidates else "no_candidate", "ref": None, + "alias": None, "evidence_kind": top.get("kind"), + "evidence": ";".join(f"{c.get('ref') or c.get('alias') or c.get('kind')} = {c.get('rendered')}" + for c in candidates) or None, + "eid": None, "artifact": top.get("artifact"), "node_id": top.get("node_id"), "sealed": None, + "span": seg.get("span"), "sentence": _around(text, seg.get("span") or [0, 0]), + "note": GUESS_NOTE if candidates else NO_CANDIDATE, "candidates": candidates, + }) + return rows + + +def _audit_contract(legacy: dict[str, Any], text: Any = None) -> list[dict[str, Any]]: + """legacy_contract 的审计行:旧契约按数值对上的进「旧运行猜测」(不是显式引用),对不上的进「无证据」。 + + text 是认出来的那个成果字段的原文(legacy.field):句子一栏取数字前后的原文,和 legacy_text 一样。 + trace_numbers 只给对不上的数记了 context,对上的没有;认不出字段时退回 context。""" + def sentence(item: dict[str, Any]) -> str: + if isinstance(text, str) and item.get("positioned"): + return _around(text, [item["start"], item["end"]]) + return item.get("context") or "" + + rows = [] + for item in legacy.get("matched") or []: + sources = list(item.get("sources") or []) + if item.get("metric"): + shown = ";".join(f"{s['ref']} = {s['rendered']}" for s in sources) or f"m:{item['metric']}" + else: + shown = "出处不唯一:候选 " + "、".join(str(c) for c in item.get("candidates") or []) + rows.append({ + "group": "candidate", "report": None, "field": legacy.get("field"), "segment": None, "unit": None, + "kind": "number", "text": item.get("token"), "state": "candidate", "issue": None, "ref": None, + "alias": None, "evidence_kind": "metric", "evidence": shown, "eid": None, + "artifact": sources[0]["artifact"] if len(sources) == 1 else None, + "node_id": sources[0]["node_id"] if len(sources) == 1 else None, + "sealed": all(s.get("sealed") for s in sources) if sources else False, + "span": [item["start"], item["end"]] if item.get("positioned") else None, + "sentence": sentence(item), "note": LEGACY_NOTE, "candidates": sources, + }) + for item in legacy.get("unmatched") or []: + rows.append({ + "group": "none", "report": None, "field": legacy.get("field"), "segment": None, "unit": None, + "kind": "number", "text": item.get("token"), "state": "none", "issue": "uncited_number", "ref": None, + "alias": None, "evidence_kind": None, "evidence": None, "eid": None, "artifact": None, "node_id": None, + "sealed": None, "span": [item["start"], item["end"]] if item.get("positioned") else None, + "sentence": sentence(item), "note": LEGACY_UNMATCHED, + }) + return rows + diff --git a/backend/app/api/workflows.py b/backend/app/api/workflows.py index 123fc69c..5584b71e 100644 --- a/backend/app/api/workflows.py +++ b/backend/app/api/workflows.py @@ -444,11 +444,12 @@ async def autofix( bad = _unreadable(payload.level, e) return {"graph": graph, "changes": [], "ops": [], "applied": [], "rejected": [ {"fix_id": fid, "reason": "工作流的结构读不懂,没法应用修复"} for fid in payload.apply], - "remaining": bad["issues"], "fixes": [], "assist": None, "ok": False} + "remaining": bad["issues"], "fixes": [], "handoff": [], "assist": None, "ok": False} versions, cards = await _gate_context(session, spec) out = apply_fixes(graph, payload.apply, payload.choices, level=payload.level, versions=versions, cards=cards) out["assist"] = None - if payload.assist: + # 人在选项里选了「交给 Copilot」(handoff),和点「交给 Copilot」是一回事 + if payload.assist or out["handoff"]: if out["ok"]: out["assist"] = {"ok": True, "summary": "修复之后已经没有挡住发布的错误,没有再交给 Copilot", "questions": []} diff --git a/backend/app/data/engine.py b/backend/app/data/engine.py index 60376885..2733d24a 100644 --- a/backend/app/data/engine.py +++ b/backend/app/data/engine.py @@ -499,13 +499,14 @@ async def run_query( 时限没指定时按数据源自己的配置(query_timeout),由数据库执行(_server_deadline)。 """ limits = limits or QueryLimits(timeout_seconds=query_timeout(source)) - statement = check(sql, readonly=bool(source.readonly), source_name=source.name) kind = (source.kind or "").lower() + # 按数据源的方言扫:MySQL 的反斜杠转义、PG 的 $$ 字符串,守卫要和数据库读得一样 + statement = check(sql, readonly=bool(source.readonly), source_name=source.name, dialect=kind) engine = await engines.get(source) started = time.perf_counter() - if not source.readonly and is_write(statement): + if not source.readonly and is_write(statement, dialect=kind): return await _run_write(engine, kind, statement, limits, started) async def _read(conn: AsyncConnection) -> tuple[list[str], list[list[Any]], bool, dict[str, str]]: diff --git a/backend/app/data/guard.py b/backend/app/data/guard.py index f49f6616..6a100937 100644 --- a/backend/app/data/guard.py +++ b/backend/app/data/guard.py @@ -16,6 +16,12 @@ 刻意不引入 sqlparse 之类的重型依赖:完整解析 SQL 方言是个无底洞,而这里需要的 判断很窄——"第一个关键字是什么"和"有没有第二条语句"。窄问题用窄解法, 把复杂度留在能看懂的范围内。 + +所有判断都建立在同一次扫描上(_tokens):一遍从左到右,同时认注释、字符串、标识符引号和 +分号。以前先用正则去注释、再认字符串,字符串里的 `--`、`/*` 被当成注释删掉,执行的 SQL +和写的不一样。扫描要和数据库的读法一致,而读法跟方言、跟服务器设置走(MySQL 的 +NO_BACKSLASH_ESCAPES / ANSI_QUOTES,PG 的 standard_conforming_strings)。守卫不知道 +服务器开了哪个,就按每种可能的读法各扫一遍,读出来不一致的直接拒掉。 """ from __future__ import annotations @@ -35,9 +41,6 @@ _ALWAYS_DENIED = frozenset({ "attach", "detach", }) -_LINE_COMMENT = re.compile(r"--[^\n]*") -_BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.S) - # 首关键字是查询、正文却能写的语句。只看首关键字的话它们全都能混过去: # WITH k AS (SELECT 1) DELETE FROM t SQLite / MySQL / PG 都认 # WITH x AS (DELETE … RETURNING *) SELECT … PG 的数据修改 CTE @@ -87,89 +90,265 @@ class QueryLimits: timeout_seconds: float = 30 -def strip_comments(sql: str) -> str: - """去掉注释再判断。否则 `/*x*/ DROP TABLE t` 的首关键字会被看成注释内容。""" - return _BLOCK_COMMENT.sub(" ", _LINE_COMMENT.sub(" ", sql)) +@dataclass(frozen=True) +class _Reading: + """数据库怎么切分一段 SQL:哪些字符开始注释、字符串、带引号的标识符。""" + + name: str + #: '…' 里的反斜杠转义下一个字符(MySQL 默认;PG 关了 standard_conforming_strings 时) + backslash: bool = False + #: "…" 是字符串而不是标识符(MySQL 没开 ANSI_QUOTES 时),跟 '…' 一样认反斜杠 + dq_string: bool = False + #: `…` 是标识符(MySQL、SQLite) + backtick: bool = False + #: […] 是标识符(SQLite) + brackets: bool = False + #: $$…$$、$tag$…$tag$ 是字符串(PostgreSQL) + dollar: bool = False + #: E'…' 里认反斜杠转义(PostgreSQL) + e_strings: bool = False + #: /* /* */ */ 可以嵌套(PostgreSQL) + nested: bool = False + #: # 开始行注释(MySQL) + hash_comment: bool = False + #: q'[…]' 这类自选定界符的字符串(Oracle) + q_quote: bool = False -def split_statements(sql: str) -> list[str]: - """按分号切语句,但要认得字符串字面量里的分号。 +_GENERIC = _Reading("generic", backtick=True) +_SQLITE = _Reading("sqlite", backtick=True, brackets=True) +_PG = _Reading("postgresql", dollar=True, e_strings=True, nested=True) +_PG_OLD = _Reading("postgresql-backslash", backslash=True, dollar=True, e_strings=True, nested=True) +_MYSQL = _Reading("mysql", backslash=True, dq_string=True, backtick=True, hash_comment=True) +_MYSQL_NBE = _Reading("mysql-no-backslash", dq_string=True, backtick=True, hash_comment=True) +_MYSQL_ANSI = _Reading("mysql-ansi", backslash=True, backtick=True, hash_comment=True) +_MYSQL_BOTH = _Reading("mysql-ansi-no-backslash", backtick=True, hash_comment=True) +_ORACLE = _Reading("oracle", q_quote=True) - `SELECT ';'` 只是一条语句。简单 split(";") 会把它切成两半,然后把后半截 - 当成"第二条语句"拒掉——那是误杀,比漏杀更让人无从下手。 - """ - out: list[str] = [] - buf: list[str] = [] - quote: str | None = None +# 每种方言的读法:第一种是服务器默认设置下的读法,其余是服务器可能开着的其他设置。 +# 守卫看不到服务器的设置,所以每种都要扫,读出来不一样就拒 +_READINGS: dict[str, tuple[_Reading, ...]] = { + "sqlite": (_SQLITE,), + "postgresql": (_PG, _PG_OLD), + "postgres": (_PG, _PG_OLD), + "mysql": (_MYSQL, _MYSQL_NBE, _MYSQL_ANSI, _MYSQL_BOTH), + "mariadb": (_MYSQL, _MYSQL_NBE, _MYSQL_ANSI, _MYSQL_BOTH), + "oracle": (_ORACLE,), +} +# 不知道方言时每种读法都算上 +_ALL_READINGS: tuple[_Reading, ...] = (_GENERIC, *dict.fromkeys( + r for readings in _READINGS.values() for r in readings)) + +_AMBIGUOUS = ( + "这条 SQL 里的引号、反斜杠或注释,在数据库的不同设置下会读成不同的语句" + "(比如 MySQL 的 NO_BACKSLASH_ESCAPES、ANSI_QUOTES)。" + "字符串里的单引号写成两个单引号 '',不要用反斜杠转义;注释删掉再试" +) + +_DOLLAR_TAG = re.compile(r"\$(?:[^\W\d]\w*)?\$") +_Q_CLOSE = {"[": "]", "(": ")", "{": "}", "<": ">"} + + +def _readings(dialect: str | None) -> tuple[_Reading, ...]: + return _READINGS.get((dialect or "").lower(), _ALL_READINGS) + + +def _reading(dialect: str | None) -> _Reading: + """服务器默认设置下的读法。不知道方言时用通用读法(标准 SQL 加反引号)。""" + return _READINGS.get((dialect or "").lower(), (_GENERIC,))[0] + + +def _word_char(ch: str) -> bool: + return ch.isalnum() or ch in "_$" + + +def _prefixed(sql: str, i: int, letters: str) -> bool: + """i 处的引号前面紧挨着一个前缀字母(E'…'),而且这个字母不是某个标识符的尾巴。""" + return i >= 1 and sql[i - 1] in letters and not (i >= 2 and _word_char(sql[i - 2])) + + +def _q_quote_at(sql: str, i: int) -> bool: + """Oracle 的 q'…' / nq'…':引号后面紧跟的字符是定界符,不能是空白。""" + if i + 1 >= len(sql) or sql[i + 1].isspace() or not (i >= 1 and sql[i - 1] in "qQ"): + return False + j = i - 2 + if j >= 0 and sql[j] in "nN": + j -= 1 + return not (j >= 0 and _word_char(sql[j])) + + +@dataclass(frozen=True) +class _Token: + #: code / comment / string / ident / sep + kind: str + text: str + #: 开、闭定界符的长度,抹内容时保留它们 + open: int = 0 + close: int = 0 + + +def _quoted_end(sql: str, i: int, quote: str, backslash: bool) -> tuple[int, int]: + """从 i 处的开引号往后找闭引号。返回 (结束位置, 闭引号长度);没闭合的一直到末尾。""" + j = i + 1 + while j < len(sql): + ch = sql[j] + if backslash and ch == "\\": + j += 2 + continue + if ch == quote: + if j + 1 < len(sql) and sql[j + 1] == quote: # 两个引号是转义,不是结束 + j += 2 + continue + return j + 1, 1 + j += 1 + return len(sql), 0 + + +def _tokens(sql: str, reading: _Reading) -> list[_Token]: + """一遍扫完:注释、字符串、带引号的标识符、分号各自成段,其余是代码。""" + out: list[_Token] = [] + n = len(sql) + code = 0 # 当前这段代码的起点 i = 0 - text = strip_comments(sql) - while i < len(text): - ch = text[i] - if quote: - buf.append(ch) - if ch == quote: - # SQL 里连续两个引号是转义('' 表示一个单引号) - if i + 1 < len(text) and text[i + 1] == quote: - buf.append(text[i + 1]) - i += 2 - continue - quote = None - elif ch in ("'", '"', "`"): - quote = ch - buf.append(ch) + + def flush(upto: int) -> None: + if upto > code: + out.append(_Token("code", sql[code:upto])) + + while i < n: + ch = sql[i] + nxt = sql[i + 1] if i + 1 < n else "" + prev = sql[i - 1] if i else "" + token: _Token | None = None + end = i + if (ch == "-" and nxt == "-") or (ch == "#" and reading.hash_comment): + end = sql.find("\n", i) + end = n if end < 0 else end + token = _Token("comment", sql[i:end]) + elif ch == "/" and nxt == "*": + if reading.nested: + depth, end = 0, i + while end < n: + if sql.startswith("/*", end): + depth, end = depth + 1, end + 2 + elif sql.startswith("*/", end): + depth, end = depth - 1, end + 2 + if depth == 0: + break + else: + end += 1 + end = min(end, n) + else: + close = sql.find("*/", i + 2) + end = n if close < 0 else close + 2 + token = _Token("comment", sql[i:end]) elif ch == ";": - stmt = "".join(buf).strip() + end = i + 1 + token = _Token("sep", ";") + elif ch == "'": + if reading.q_quote and _q_quote_at(sql, i): + closer = _Q_CLOSE.get(nxt, nxt) + "'" + found = sql.find(closer, i + 2) + end, tail = (n, 0) if found < 0 else (found + 2, 2) + token = _Token("string", sql[i:end], 2, tail) + else: + e_string = reading.e_strings and _prefixed(sql, i, "eE") + end, tail = _quoted_end(sql, i, "'", reading.backslash or e_string) + token = _Token("string", sql[i:end], 1, tail) + elif ch == '"': + end, tail = _quoted_end(sql, i, '"', reading.backslash and reading.dq_string) + token = _Token("string" if reading.dq_string else "ident", sql[i:end], 1, tail) + elif ch == "`" and reading.backtick: + end, tail = _quoted_end(sql, i, "`", False) + token = _Token("ident", sql[i:end], 1, tail) + elif ch == "[" and reading.brackets: + close = sql.find("]", i + 1) + end, tail = (n, 0) if close < 0 else (close + 1, 1) + token = _Token("ident", sql[i:end], 1, tail) + elif ch == "$" and reading.dollar and not (prev and _word_char(prev)): + tag = _DOLLAR_TAG.match(sql, i) + if tag: + found = sql.find(tag.group(0), tag.end()) + width = len(tag.group(0)) + end, tail = (n, 0) if found < 0 else (found + width, width) + token = _Token("string", sql[i:end], width, tail) + if token is None: + i += 1 + continue + flush(i) + out.append(token) + i = code = end + flush(n) + return out + + +def _joined(tokens: list[_Token]) -> str: + """注释换成一个空格,其余原样拼回去。空格不能省:'a'/**/'b' 删成 'a''b' 就成了一个字符串。""" + return "".join(" " if t.kind == "comment" else t.text for t in tokens) + + +def _statements(sql: str, reading: _Reading) -> list[str]: + out: list[str] = [] + buf: list[_Token] = [] + for token in [*_tokens(sql, reading), _Token("sep", ";")]: + if token.kind == "sep": + stmt = _joined(buf).strip() if stmt: out.append(stmt) buf = [] else: - buf.append(ch) - i += 1 - tail = "".join(buf).strip() - if tail: - out.append(tail) + buf.append(token) return out -def first_verb(sql: str) -> str: +def strip_comments(sql: str, *, dialect: str | None = None) -> str: + """去掉注释再判断。否则 `/*x*/ DROP TABLE t` 的首关键字会被看成注释内容。 + + 字符串、带引号的标识符里的 `--`、`/*` 不是注释,原样保留。 + """ + return _joined(_tokens(sql, _reading(dialect))) + + +def split_statements(sql: str, *, dialect: str | None = None) -> list[str]: + """按分号切语句(去掉注释),但要认得字符串字面量里的分号。 + + `SELECT ';'` 只是一条语句。简单 split(";") 会把它切成两半,然后把后半截 + 当成"第二条语句"拒掉——那是误杀,比漏杀更让人无从下手。 + """ + return _statements(sql, _reading(dialect)) + + +def first_verb(sql: str, *, dialect: str | None = None) -> str: """取语句的首关键字(小写)。括号开头的 `(SELECT ...)` 也要认出来。""" - text = strip_comments(sql).lstrip().lstrip("(").lstrip() + text = strip_comments(sql, dialect=dialect).lstrip().lstrip("(").lstrip() match = re.match(r"[A-Za-z_]+", text) return match.group(0).lower() if match else "" -def _blank_quoted(sql: str, *, identifiers: bool = True) -> str: +def _blanked(tokens: list[_Token], *, identifiers: bool) -> str: + out: list[str] = [] + for t in tokens: + if t.kind == "comment": + out.append(" ") + elif t.kind == "string" or (t.kind == "ident" and identifiers): + inner = len(t.text) - t.open - t.close + out.append(t.text[:t.open] + " " * inner + t.text[len(t.text) - t.close:]) + else: + out.append(t.text) + return "".join(out) + + +def _blank_quoted(sql: str, *, identifiers: bool = True, dialect: str | None = None) -> str: """把引号里的内容换成空格:字符串 'please delete' 和标识符 "update" 都不是关键字。 identifiers=False 时只抹字符串、保留 "…" 和 `…`——找函数调用时要看得见 被引号括起来的函数名。 """ - quotes = ("'", '"', "`") if identifiers else ("'",) - out: list[str] = [] - quote: str | None = None - i = 0 - while i < len(sql): - ch = sql[i] - if quote: - if ch == quote: - if i + 1 < len(sql) and sql[i + 1] == quote: # '' 是转义,不是结束 - out.append(" ") - i += 2 - continue - quote = None - out.append(ch) - else: - out.append(" ") - else: - if ch in quotes: - quote = ch - out.append(ch) - i += 1 - return "".join(out) + return _blanked(_tokens(sql, _reading(dialect)), identifiers=identifiers) -def _pragma_write_reason(stmt: str) -> str | None: - match = _PRAGMA.match(strip_comments(stmt).strip()) +def _pragma_write_reason(stmt: str, reading: _Reading = _GENERIC) -> str | None: + match = _PRAGMA.match(_joined(_tokens(stmt, reading)).strip()) if not match: return "看不出这条 PRAGMA 读的是什么" name, rest = match.group(1).lower(), match.group(2).strip() @@ -183,40 +362,63 @@ def _pragma_write_reason(stmt: str) -> str | None: ) -def _write_reason(stmt: str) -> str | None: +def _write_reason(stmt: str, reading: _Reading = _GENERIC) -> str | None: """这条语句会不会写。会写就返回一句说给模型听的原因,不会写返回 None。 首关键字只是第一道:以查询开头的语句正文照样能写(见 _HIDDEN_WRITE), 所以 SELECT / WITH / EXPLAIN 还要把正文扫一遍。SHOW / DESCRIBE / VALUES 结构上写不了,不扫——`SHOW CREATE TABLE` 里的 CREATE 不是在建表。 """ - verb = first_verb(stmt) + tokens = _tokens(stmt, reading) + text = _joined(tokens).lstrip().lstrip("(").lstrip() + match = re.match(r"[A-Za-z_]+", text) + verb = match.group(0).lower() if match else "" if verb not in _READONLY_VERBS: return f"{verb.upper()} 会修改数据" if verb == "pragma": - return _pragma_write_reason(stmt) + return _pragma_write_reason(stmt, reading) if verb in ("select", "with", "explain"): - text = strip_comments(stmt) - hidden = _HIDDEN_WRITE.search(_blank_quoted(text)) + hidden = _HIDDEN_WRITE.search(_blanked(tokens, identifiers=True)) if hidden: word = " ".join(hidden.group(1).split()).upper() return f"语句里有 {word}——以查询开头的语句也能写数据" - call = _SIDE_EFFECT_CALL.search(_blank_quoted(text, identifiers=False)) + call = _SIDE_EFFECT_CALL.search(_blanked(tokens, identifiers=False)) if call: return f"{call.group(1)}() 有副作用" return None -def check(sql: str, *, readonly: bool, source_name: str = "") -> str: +def _write_reason_any(stmt: str, readings: tuple[_Reading, ...]) -> str | None: + """按每种读法都判一遍:哪种读法下会写,就当它会写。""" + reasons = [_write_reason(stmt, r) for r in readings] + if reasons[0]: + return reasons[0] + other = next((r for r in reasons[1:] if r), None) + return f"数据库换一种设置来读的话,{other}。{_AMBIGUOUS}" if other else None + + +def _one_view(sql: str, readings: tuple[_Reading, ...]) -> list[str] | None: + """每种读法切出来的语句都一样时返回它,不一样返回 None。""" + views = [_statements(sql, r) for r in readings] + return views[0] if all(v == views[0] for v in views[1:]) else None + + +def check(sql: str, *, readonly: bool, source_name: str = "", dialect: str | None = None) -> str: """校验并返回规范化后的单条 SQL。不通过就抛 SqlRejected。 - 返回值是去掉尾分号的那一条语句——调用方应该执行这个返回值, - 而不是原始输入,否则守卫等于没做。 + 返回值是去掉注释和尾分号的那一条语句——调用方应该执行这个返回值, + 而不是原始输入,否则守卫等于没做。字符串、带引号的标识符原样保留。 + + dialect 是数据源的 kind(sqlite / postgresql / mysql / oracle …)。 + 不知道的话按所有方言、所有设置的读法各判一遍,任何一种读法不过都拒。 """ if not sql or not sql.strip(): raise SqlRejected("SQL 是空的") - statements = split_statements(sql) + readings = _readings(dialect) + statements = _one_view(sql, readings) + if statements is None: + raise SqlRejected(_AMBIGUOUS) if not statements: raise SqlRejected("SQL 里没有可执行的语句") if len(statements) > 1: @@ -226,7 +428,11 @@ def check(sql: str, *, readonly: bool, source_name: str = "") -> str: ) stmt = statements[0] - verb = first_verb(stmt) + # 交给数据库的就是 stmt:它在每种读法下都得还是完整的一条,没有注释、没有分号 + if any(_tokens_have(stmt, r, ("comment", "sep")) for r in readings): + raise SqlRejected(_AMBIGUOUS) + + verb = first_verb(stmt, dialect=dialect) if not verb: raise SqlRejected("看不出这条 SQL 要做什么(没有识别到关键字)") @@ -237,7 +443,7 @@ def check(sql: str, *, readonly: bool, source_name: str = "") -> str: ) if readonly: - reason = _write_reason(stmt) + reason = _write_reason_any(stmt, readings) if reason: where = f"「{source_name}」" if source_name else "这个数据源" raise SqlRejected( @@ -249,11 +455,19 @@ def check(sql: str, *, readonly: bool, source_name: str = "") -> str: return stmt -def is_write(sql: str) -> bool: +def _tokens_have(sql: str, reading: _Reading, kinds: tuple[str, ...]) -> bool: + return any(t.kind in kinds for t in _tokens(sql, reading)) + + +def is_write(sql: str, *, dialect: str | None = None) -> bool: """是不是写操作。用来决定要不要走人工审批。 和只读判定用同一份 _write_reason:审批这道门要是只看首关键字, - `WITH … DELETE` 在可写源上就能不经审批直接写。 + `WITH … DELETE` 在可写源上就能不经审批直接写。几种读法切出来不一样的, + 一律当写:看不清的交给人批。 """ - statements = split_statements(sql) - return not statements or any(_write_reason(s) is not None for s in statements) + readings = _readings(dialect) + statements = _one_view(sql, readings) + if not statements: + return True + return any(_write_reason(s, r) is not None for s in statements for r in readings) diff --git a/backend/app/engine/autofix.py b/backend/app/engine/autofix.py index f6453c84..419f012f 100644 --- a/backend/app/engine/autofix.py +++ b/backend/app/engine/autofix.py @@ -18,20 +18,26 @@ from __future__ import annotations import copy +import re from typing import Any, Callable from pydantic import BaseModel -from app.engine.governance import publish_issues -from app.engine.schema import GraphNode, GraphSpec, NodeType, _ancestors +from app.engine.governance import REPORT_POLICY, exit_text_fields, publish_issues +from app.engine.schema import REPORT_CLAIMS, GraphNode, GraphSpec, NodeType, _ancestors, claims_problem #: 被选中的修复按这个顺序应用:先补契约骨架,再补契约里的各项,最后是节点上的开关。 -#: 前面的修复可能改变后面的候选(metrics_from 决定 required 从哪几张卡里挑) +#: 前面的修复可能改变后面的候选(metrics_from 决定 required 从哪几张卡里挑,report_from 决定 +#: 成果字段默认接哪个报告撰写节点) _ORDER = ( "governed.no_contract", "contract.metrics_from_missing", "contract.metrics_from_invalid", - "contract.report_from_missing", "contract.report_from_invalid", "report.metrics_from_invalid", - "contract.required_missing", "contract.strict_off", "governed.agent_approval_never", + "contract.report_from_missing", "contract.report_from_invalid", "governed.report_from_required", + "governed.exit_text_source", "report.metrics_from_invalid", + "contract.required_missing", "contract.strict_off", "contract.claims_ignored", "report.claims_invalid", + "governed.report_policy", + "governed.caliber_agent_cite_fields", "governed.caliber_compute_input", "governed.agent_approval_never", "governed.default_approval_never", "governed.subgraph_unpinned", "governed.supervisor", "contract.not_object", + "governed.text_bypass", "governed.caliber_agent_schema", "governed.caliber_model_input", ) #: 修复落在全图上、不落在任何节点上的问题:问题各挂在跟随全图默认的那几个节点上(点一条能定位), @@ -40,14 +46,17 @@ _GRAPH_FIXES = frozenset({"governed.default_approval_never"}) #: 除了 error,这几条警告也给修复(受管档位下)。已发布档位的门禁只给警告,不借「一键修复」 #: 替人把审批、strict 这些往受管的要求上拧 -_WARNING_FIXES = {"governed": frozenset({"contract.strict_off"})} +_WARNING_FIXES = {"governed": frozenset({"contract.strict_off", "contract.claims_ignored"})} + +#: choice 类修复里「交给 Copilot」那个候选的值。候选上另有 handoff: true 标记:只认标记,不认值—— +#: 别的 choice 拿节点 id 当候选值,节点 id 恰好叫 copilot 的报告撰写节点照样是个普通候选 +HANDOFF = "copilot" ASSIST_ONLY = "这一处是结构性的问题,要交给 Copilot:请求里带上 assist: true" STALE = "这处问题现在已经没有了(修复 id 过期),重新检查一次" SOLVED = "前面的修复已经一并解决了这一处" NEED_CHOICE = "这一处要你来选:{label}" -_TEXT_NODES = (NodeType.LLM, NodeType.AGENT) #: 修复和 Copilot 兜底只许改节点配置、补节点和连线。其余的图操作(删节点、删连线)和 #: 任何不认识的操作(发布、改级别…)一律当降低要求处理 _GRAPH_OPS = frozenset({"add_node", "update_node", "add_edge"}) @@ -59,11 +68,13 @@ _GRAPH_OPS = frozenset({"add_node", "update_node", "add_edge"}) class _Ctx: - def __init__(self, spec: GraphSpec, versions: dict[str, Any] | None, cards: dict[str, Any] | None) -> None: + def __init__(self, spec: GraphSpec, versions: dict[str, Any] | None, cards: dict[str, Any] | None, + level: str = "governed") -> None: self.spec = spec self.nodes = spec.node_map() self.versions = versions or {} self.cards = cards or {} + self.level = level def upstream(self, node_id: str, *types: NodeType) -> list[GraphNode]: """这个节点上游的某几类节点,按画布上的节点顺序。""" @@ -148,7 +159,10 @@ def _plan_metrics_from_invalid(ctx: _Ctx, issue: dict[str, Any], node: GraphNode return _cards_choice(ctx, node, before, "contract.metrics_from") -def _reports_choice(ctx: _Ctx, node: GraphNode, before: Any, *, narrative: bool) -> dict[str, Any]: +def _reports_choice(ctx: _Ctx, node: GraphNode, before: Any) -> dict[str, Any]: + """report_from 指向哪个报告撰写节点。以前没有报告撰写节点时退回让人选 narrative(旧的叙述模式), + 可受管级别的叙述模式过不了 G1(governed.report_from_required),已发布级别这几条又只是警告、不给 + 修复——那个候选修了也发不出去,所以没有报告撰写节点就交给 Copilot 加一个。""" field = "contract.report_from" reports = ctx.upstream(node.id, NodeType.REPORT) if len(reports) == 1: @@ -156,27 +170,120 @@ def _reports_choice(ctx: _Ctx, node: GraphNode, before: Any, *, narrative: bool) before, reports[0].id) if reports: return _choice("选出具契约核对哪个「报告撰写」节点的文档(report_from)", field, _node_options(reports)) - writers = ctx.upstream(node.id, *_TEXT_NODES) if narrative else [] - if writers: - # 没有报告撰写节点时退回旧的叙述模式:数字按叙述回指口径卡。只有一个候选也要人确认—— - # 它是不是那段该核对的叙述,只有作者知道 - options = [{"value": f"{{{{ nodes.{n.id}.text }}}}", "label": f"「{n.title}」写的文字", - "hint": f"narrative = {{{{ nodes.{n.id}.text }}}}"} for n in writers] - return _choice("选出具契约要核对的叙述来自哪个节点(narrative)", "contract.narrative", options, - default=options[0]["value"] if len(options) == 1 else None) - return _assist("出口上游没有「报告撰写」节点,也没有写叙述的模型节点", field) + return _assist("出口上游没有「报告撰写」节点:要在出口前面加一个,出具契约才有文档可核对", field) def _plan_report_from_missing(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: if node is None: return None - return _reports_choice(ctx, node, _contract(node).get("report_from"), narrative=True) + return _reports_choice(ctx, node, _contract(node).get("report_from")) + + +def _plan_report_from_required(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: + if node is None: + return None + return _reports_choice(ctx, node, _contract(node).get("report_from")) + + +def _report_text(report_id: str) -> str: + return f"{{{{ nodes.{report_id}.text }}}}" + + +def _plan_exit_fields(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: + """G1:成果字段改成取报告撰写节点的正文。就一个候选也给选项——字段里放什么是作者的事, + 改了就是换掉原来的内容;报告撰写节点一个都没有,交给 Copilot 加。""" + if node is None: + return None + bad = exit_text_fields(ctx.spec, node) + if not bad: + return None + named = [f"「{name}」" for _, name, _ in bad if name is not None] + action = f"把成果字段{'、'.join(named)}改成取" if named else "给出口配一个成果字段 result,取" + reports = ctx.upstream(node.id, NodeType.REPORT) + if not reports: + return _assist(f"「{node.title}」上游没有报告撰写节点:要在出口前面加一个,{action}它的正文", "fields") + options = [{**o, "hint": f"{o['hint']} · {_report_text(o['value'])}"} for o in _node_options(reports)] + pinned = _contract(node).get("report_from") + default = pinned if pinned in {r.id for r in reports} else reports[0].id if len(reports) == 1 else None + return _choice(f"{action}哪个报告撰写节点的正文", "fields", options, default=default, + _fields=[field for field, _, _ in bad]) + + +def _plan_report_policy(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: + """G3:报告撰写节点写明最严的三项。答案唯一,只往严里改;写了不支持的 claims 由 claims_invalid 管。""" + if node is None: + return None + writes: dict[str, Any] = {} + before: dict[str, Any] = {} + for key, want, _ in REPORT_POLICY: + value = node.config.get(key) + effective = ctx.spec.defaults.get(key) if value in (None, "") else value + if effective != want and not (key == "claims" and claims_problem(effective)): + writes[key], before[key] = want, copy.deepcopy(value) + if not writes: + return None + keys = list(writes) + preview = ({"field": keys[0], "before": before[keys[0]], "after": writes[keys[0]]} if len(keys) == 1 + else {"field": "、".join(keys), "before": before, "after": writes}) + return {"kind": "auto", "label": f"给「{node.title}」写明 " + "、".join(f"{k}: {v}" for k, v in writes.items()), + "field": keys[0], "preview": preview, "_set": writes} + + +def _plan_claims(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: + """claims 写了不支持的值。受管级别只认 require_citation,答案唯一;已发布级别 off 和 require_citation + 都行,要人选——写的是 judge(还不支持的模型裁判)时建议 require_citation:它离裁判最近。""" + if node is None: + return None + before = node.config.get("claims") + if ctx.level == "governed": + return _auto(f"把「{node.title}」的 claims 改成 require_citation:受管出具要求每句结论挂引用" + + ("(结论句裁判在后续版本支持)" if before == "judge" else ""), "claims", before, "require_citation") + hints = {"off": "结论句不参与判档,和以前一样", "require_citation": "没挂引用的结论句计入缺口,按出具档位降档"} + options = [{"value": v, "label": v, "hint": hints[v]} for v in REPORT_CLAIMS] + return _choice(f"选「{node.title}」的结论句策略(claims)", "claims", options, + default="require_citation" if before == "judge" else None) + + +def _plan_compute_input(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: + """G4:沙箱代码到底是在取数还是在算,只有作者知道。取数的标成 source;在算的交给 Copilot 把计算 + 挪进口径卡——两样都不替人选,也不给建议。""" + if node is None: + return None + options = [ + {"value": "source", "label": "它在取数:标成 source(取数)", + "hint": "evidence_role = source,口径卡读它时记得下出处"}, + {"value": HANDOFF, "label": "它在做计算:交给 Copilot 把计算挪进口径卡", + "hint": "Copilot 改的同样只是预览,要人拿主意的会原样问你", "handoff": True}, + ] + return _choice(f"「{node.title}」的产出喂给了口径卡:它是在取数,还是在做计算?", "evidence_role", options) + + +def _plan_cite_fields(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: + if node is None: + return None + return _auto(f"打开「{node.title}」的 cite_fields:每个字段都核对到查询结果里的那一格", "cite_fields", + node.config.get("cite_fields"), True) + + +def _plan_agent_schema(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: + return _assist(f"给「{_title(node)}」配 output_schema:交哪些字段、什么类型要你或 Copilot 定,自动修复不替你编;" + "配好之后再开 cite_fields", "output_schema") + + +def _plan_model_input(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: + return _assist(f"「{_title(node)}」写的数进了口径卡:要换成 Agent(配 output_schema、开 cite_fields)或「调用工具」" + "节点取数。修复不删节点,Copilot 会拟一个改法") + + +def _plan_text_bypass(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: + return _assist(f"「{_title(node)}」写的文字要改由报告撰写节点来写:在它和出口之间加一个报告撰写节点,出口改成取" + f"报告的正文。修复不删节点,「{_title(node)}」会留在图上") def _plan_report_from_invalid(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: if node is None: return None - return _reports_choice(ctx, node, _contract(node).get("report_from"), narrative=False) + return _reports_choice(ctx, node, _contract(node).get("report_from")) def _plan_report_metrics_from(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: @@ -226,6 +333,18 @@ def _plan_strict(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> di _contract(node).get("strict"), True) +def _plan_claims_ignored(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: + """受管契约的 claims 写了 on_uncited: ignore:受管正式运行本来就按 degrade 办,改成 degrade 答案唯一、 + 不降低要求,只是让写法和实际一致。""" + if node is None: + return None + claims = _contract(node).get("claims") + if not isinstance(claims, dict): + return None + return _auto("把出具契约 claims 的 on_uncited 改成 degrade:受管级别的正式运行本来就这么办,写法和实际一致", + "contract.claims.on_uncited", claims.get("on_uncited"), "degrade") + + def _plan_approval(ctx: _Ctx, issue: dict[str, Any], node: GraphNode | None) -> dict[str, Any] | None: if node is None: return None @@ -292,7 +411,18 @@ _PLANNERS: dict[str, Callable[[_Ctx, dict[str, Any], GraphNode | None], dict[str "contract.report_from_invalid": _plan_report_from_invalid, "contract.required_missing": _plan_required, "contract.strict_off": _plan_strict, + "contract.claims_ignored": _plan_claims_ignored, "report.metrics_from_invalid": _plan_report_metrics_from, + # 证据门禁 G1–G5 + "governed.report_from_required": _plan_report_from_required, + "governed.exit_text_source": _plan_exit_fields, + "governed.text_bypass": _plan_text_bypass, + "governed.report_policy": _plan_report_policy, + "report.claims_invalid": _plan_claims, + "governed.caliber_compute_input": _plan_compute_input, + "governed.caliber_agent_cite_fields": _plan_cite_fields, + "governed.caliber_agent_schema": _plan_agent_schema, + "governed.caliber_model_input": _plan_model_input, } @@ -320,7 +450,7 @@ def plan_fixes(spec: GraphSpec, issues: list[Any], *, level: str, versions: dict cards:钉在别处(caliber_from)的口径卡节点 id → 那一版里的指标定义,用来列 required 的候选。 返回的 Fix 里以 _ 开头的键是应用时要用的内部信息,交给前端前用 public() 去掉。 """ - ctx = _Ctx(spec, versions, cards) + ctx = _Ctx(spec, versions, cards, level) warned = _WARNING_FIXES.get(level, frozenset()) out: dict[str, dict[str, Any]] = {} for issue in map(_as_dict, issues): @@ -426,11 +556,41 @@ def _edit_defaults(graph: dict[str, Any], fix: dict[str, Any], value: Any) -> li for path, after in (fix.get("_set") or {}).items() if path.startswith("defaults.")] -#: 每类问题怎么改图。大多是同一种「按节点字段写值」,改全图默认的另走一格;留着这张表是为了哪天 +def _edit_exit_fields(graph: dict[str, Any], fix: dict[str, Any], value: Any) -> list[tuple[str, str, Any, Any]]: + """G1:文字不是来自报告撰写节点的成果字段,改成取选中那个报告的正文;只动门禁点了名的那几个字段。 + 出口没配字段的补一个 result——没配字段时成果的键就叫 result,下游照旧取得到。""" + node = _node(graph, fix["node_id"]) + if node is None: + return [] + config = _config(node) + text = _report_text(str(value)) + targets = list(fix.get("_fields") or []) + if targets == ["fields"]: + before = copy.deepcopy(config.get("fields")) + config["fields"] = [{"name": "result", "value": text}] + return [(str(node["id"]), "fields", before, copy.deepcopy(config["fields"]))] + fields = config.get("fields") if isinstance(config.get("fields"), list) else [] + out = [] + for path in targets: + m = re.fullmatch(r"fields\[(\d+)\]\.value", path) + field = fields[int(m.group(1))] if m and int(m.group(1)) < len(fields) else None + if isinstance(field, dict): + out.append((str(node["id"]), path, copy.deepcopy(field.get("value")), text)) + field["value"] = text + return out + + +#: 每类问题怎么改图。大多是同一种「按节点字段写值」,改全图默认、改成果字段的各走一格;留着这张表是为了 #: 某类修复要特殊处理时只换它自己的那一格(测试里也借它模拟一条出错的修复规则) _EDITORS: dict[str, Callable[[dict[str, Any], dict[str, Any], Any], list[tuple[str | None, str, Any, Any]]]] = { code: (_edit_defaults if code in _GRAPH_FIXES else _edit) for code in _PLANNERS } +_EDITORS["governed.exit_text_source"] = _edit_exit_fields + + +def _handoff(fix: dict[str, Any], value: Any) -> bool: + """选中的是不是「交给 Copilot」那个候选:只认候选上的 handoff 标记。""" + return any(o.get("handoff") is True and o.get("value") == value for o in fix.get("options") or []) def _chosen(fix: dict[str, Any], choices: dict[str, Any]) -> tuple[Any, str]: @@ -460,7 +620,7 @@ def _op(graph: dict[str, Any], node_id: str | None, edits: list[tuple[str | None keys = dict.fromkeys(path.split(".", 1)[1] for _, path, _, _ in edits if path.startswith("defaults.")) return {"op": "update_defaults", "defaults": {k: copy.deepcopy(defaults.get(k)) for k in keys}} config = _config(_node(graph, node_id) or {}) - tops = dict.fromkeys(path.split(".")[0] for _, path, _, _ in edits) + tops = dict.fromkeys(re.split(r"[.\[]", path, maxsplit=1)[0] for _, path, _, _ in edits) return {"op": "update_node", "id": node_id, "config": {k: copy.deepcopy(config.get(k)) for k in tops}} @@ -469,7 +629,8 @@ def apply_fixes(graph: dict[str, Any] | GraphSpec, fix_ids: list[str], choices: cards: dict[str, list[dict[str, Any]]] | None = None) -> dict[str, Any]: """在副本上逐个应用修复,每一个都复核:降低要求、错误没变少或者冒出新错误的,丢弃并写明原因。 - 返回 {graph, changes, ops, applied, rejected, remaining, fixes, ok}。graph 只是预览,不落库。 + 返回 {graph, changes, ops, applied, rejected, handoff, remaining, fixes, ok}。graph 只是预览,不落库。 + handoff 是人在 choice 里选了「交给 Copilot」的修复 id:图不动,调用方据此把剩下的 error 交给 Copilot。 """ raw = graph.model_dump(mode="json") if isinstance(graph, GraphSpec) else graph current = copy.deepcopy(raw) @@ -481,6 +642,7 @@ def apply_fixes(graph: dict[str, Any] | GraphSpec, fix_ids: list[str], choices: wanted = sorted(dict.fromkeys(fix_ids), key=lambda fid: rank.get(fid.split(":")[0], len(rank))) applied: list[str] = [] rejected: list[dict[str, str]] = [] + handoff: list[str] = [] changes: list[dict[str, Any]] = [] ops: list[dict[str, Any]] = [] for fid in wanted: @@ -495,6 +657,9 @@ def apply_fixes(graph: dict[str, Any] | GraphSpec, fix_ids: list[str], choices: if why: rejected.append({"fix_id": fid, "reason": why}) continue + if fix["kind"] == "choice" and _handoff(fix, value): + handoff.append(fid) + continue trial = copy.deepcopy(current) edits = _EDITORS.get(fix["code"], _edit)(trial, fix, value) try: @@ -503,7 +668,9 @@ def apply_fixes(graph: dict[str, Any] | GraphSpec, fix_ids: list[str], choices: rejected.append({"fix_id": fid, "reason": "改完的图结构读不懂,这条修复已丢弃"}) continue trial_issues = publish_issues(trial_spec, level=level) - why = ";".join(forbidden_changes(current, trial)) or judge( + # choice 里人亲手选的值不算「替人决定」(比如把沙箱代码标成取数);别的降低要求照样拦 + chosen = {(str(nid), path) for nid, path, _, _ in edits} if fix["kind"] == "choice" else set() + why = ";".join(forbidden_changes(current, trial, chosen=chosen)) or judge( issues, trial_issues, target=(fix["code"], fix["node_id"])) if why: rejected.append({"fix_id": fid, "reason": why}) @@ -517,7 +684,7 @@ def apply_fixes(graph: dict[str, Any] | GraphSpec, fix_ids: list[str], choices: ops.append(_op(current, fix["node_id"], edits)) plan = plan_fixes(spec, issues, level=level, versions=versions, cards=cards) return {"graph": current, "changes": changes, "ops": ops, "applied": applied, "rejected": rejected, - "remaining": annotate(issues, plan), "fixes": [public(f) for f in plan], + "handoff": handoff, "remaining": annotate(issues, plan), "fixes": [public(f) for f in plan], "ok": not any(i.level == "error" for i in issues)} @@ -643,11 +810,31 @@ def _contract_loosened(name: str, prior: Any, contract: Any) -> list[str]: out.append(f"往「{name}」出具契约的 allow_numbers 里加了 {'、'.join(extra)}(不带出处也能放行的数)") if contract.get("cells") is True and prior.get("cells") is not True: out.append(f"替你打开了「{name}」出具契约的 cells(允许报告直接引用查询单元格),这要你自己决定") + was, now = _claims_rank(prior.get("claims")), _claims_rank(contract.get("claims")) + if was is not None and now is not None and now < was: + out.append(f"把「{name}」出具契约的 claims 往宽里改了(没挂依据的结论句{_CLAIMS_WORDS[was]} → " + f"{_CLAIMS_WORDS[now]})") return out +#: 契约 claims 的严格程度:不查 < 查了不算缺口(ignore)< 计入缺口降档(degrade)< 不予出具(withhold) +_CLAIMS_WORDS = ("不检查", "只标出来", "计入缺口", "不予出具") + + +def _claims_rank(value: Any) -> int | None: + """契约里 claims 的严格程度,越大越严。judge 这类还不支持的写法不比较(validate 另报)。""" + name = value.get("policy") if isinstance(value, dict) else value + if name in (None, "", "off"): + return 0 + if name != "require_citation": + return None + on_uncited = value.get("on_uncited") if isinstance(value, dict) else None + return {"ignore": 1, "withhold": 3}.get(on_uncited, 2) + + def forbidden_changes(before: dict[str, Any], after: dict[str, Any], - ops: list[dict[str, Any]] | None = None) -> list[str]: + ops: list[dict[str, Any]] | None = None, *, + chosen: set[tuple[str, str]] | frozenset[tuple[str, str]] = frozenset()) -> list[str]: """降低要求的改动,一条一句。空列表表示没有。确定性修复和 Copilot 兜底过的是同一道检查。 禁止: @@ -657,6 +844,9 @@ def forbidden_changes(before: dict[str, Any], after: dict[str, Any], - 审批策略往宽里改(每次调用都审批 > 仅危险工具需要审批 > 全部自动放行)、删掉写明的审批策略, 把审批写成(或新加一个)「全部自动放行」;全图默认同理 - 取消子工作流钉住的版本 + - 报告撰写节点的 numbers、on_violation、claims 往宽里改;关掉 agent 的 cite_fields + - 替人把沙箱代码标成取数(evidence_role: source):它在取数还是在算只有作者知道。chosen 里的 + (节点 id, 字段) 是人在 choice 里亲手选的,不算 - 改发布级别——级别不在图里,修复接口也从不写工作流的 status,所以只可能以一个不认识的 操作出现,按不认识的操作拒掉 """ @@ -701,8 +891,21 @@ def forbidden_changes(before: dict[str, Any], after: dict[str, Any], reasons.append(why) contract, prior = cfg.get("contract"), prev.get("contract") reasons += _contract_loosened(name, prior, contract) + if node.get("type") == "code" and cfg.get("evidence_role") == "source" \ + and prev.get("evidence_role") != "source" and (nid, "evidence_role") not in chosen: + reasons.append((f"替你把「{name}」标成了取数(evidence_role: source)" if was + else f"新加的「{name}」标成了取数(evidence_role: source)") + + ",它是在取数还是在算要你自己决定") if was is None: continue + if was.get("type") == node.get("type") == "report": + reasons += _report_loosened(name, prev, cfg, before.get("defaults"), after.get("defaults")) + if was.get("type") == node.get("type") == "agent" and prev.get("cite_fields") is True \ + and cfg.get("cite_fields") is not True: + reasons.append(f"关掉了「{name}」的 cite_fields(字段不再逐个核对到查询结果)") + elif was.get("type") == node.get("type") == "agent" and prev.get("cite_fields") is True \ + and prev.get("output_schema") and not cfg.get("output_schema"): + reasons.append(f"删掉了「{name}」的 output_schema(cite_fields 随之失效,字段不再逐个核对到查询结果)") if prev.get("workflow_version") and not cfg.get("workflow_version"): reasons.append(f"取消了「{name}」钉住的版本") if not isinstance(prior, dict) or not prior: @@ -722,6 +925,30 @@ def forbidden_changes(before: dict[str, Any], after: dict[str, Any], return reasons +#: 报告撰写节点三项设置从严到宽的次序:没写时运行时的缺省值(on_violation 没写时探索运行按 flag) +_REPORT_STRICT = {"numbers": ("strict", "strict"), "on_violation": ("fail", "flag"), + "claims": ("require_citation", "off")} + + +def _report_loosened(name: str, prev: dict[str, Any], cfg: dict[str, Any], old_defaults: Any, + new_defaults: Any) -> list[str]: + """报告撰写节点的三项设置有没有往宽里改:原来(按节点、再按全图默认)是最严的值,改完不是了。 + claims 改成 judge 不算放宽——那是还不支持的更严的写法,validate 会报。""" + def effective(config: dict[str, Any], defaults: Any, key: str, fallback: str) -> Any: + value = config.get(key) + if value in (None, ""): + value = (defaults if isinstance(defaults, dict) else {}).get(key) + return fallback if value in (None, "") else value + + out = [] + for key, (strict, fallback) in _REPORT_STRICT.items(): + was, now = effective(prev, old_defaults, key, fallback), effective(cfg, new_defaults, key, fallback) + if was == strict and now != strict and not (key == "claims" and now == "judge"): + out.append(f"把「{name}」的 {key} 从 {strict} 放宽成了 {now}" + + ("(去掉了写明的值,运行时退回缺省)" if cfg.get(key) in (None, "") else "")) + return out + + def _type_word(node_type: Any, type_label: Callable[[Any], str]) -> str: try: return f"「{type_label(NodeType(node_type))}」" diff --git a/backend/app/engine/evidence.py b/backend/app/engine/evidence.py index 1f3daf3d..15678850 100644 --- a/backend/app/engine/evidence.py +++ b/backend/app/engine/evidence.py @@ -15,14 +15,23 @@ [[v:Q.r<行>.<列>]] 查询快照里的一格(取回时复验哈希),也能加 |万 这类换算 [[table:Q cols=a,b rows=0-4]] 整表:系统从快照生成 Markdown 表,每一格都是带出处的片段 [[i:<输入字段>]] 运行输入 + [[t:<表>]] / [[c:<表>.<列>]] 实体:表 / 字段,显示名字本身(反引号里写的已知名字也会自动链接) + [[q:K|<逐字引文>]] 引文:必须在那次检索命中的片段里逐字出现,显示引文本身 [[see:,…]] 依据:挂在句末,不渲染 -`[[t:]]` `[[c:]]` `[[q:]]` 能解析,本期一律判为解析不了,原因写「这种引用在后续版本支持」。 +实体只在这次运行冻结了表结构(schema_snapshot)时才核对:目录里没有表和字段条目的, +t / c 仍按一期判为解析不了(「这种引用在后续版本支持」),反引号里的名字也不查;报告节点 +写了 entities: off 的,照实说是它关掉的。表结构快照不全(库里的表太多、只存了一部分)时, +找不到的名字只说「核对不了」,不说「可能是编造的」。自动链接的名字只是标注:一句结论挂没挂 +依据,只看写作者自己写的 [[…]] 和 [[see:…]]。 沙箱代码节点的产出不能直接引用(`[[v:N:calc.x]]`),原因写「沙箱算出来的数要进口径卡」。 -单元格和整表要读快照:整个模块仍然不碰数据库、不调模型,快照经 loader 取(缺省是 +单元格、整表、引文要读快照:整个模块仍然不碰数据库、不调模型,快照经 loader 取(缺省是 artifact_store.load,只读文件)。测试和接口可以换成自己的 loader。 +旧运行没有报告文档时,guess_sources 按数值去已封存的证据里找「可能的来源」:那只是猜测, +不是证据,界面要照实这么说;遮罩的列不当候选。 + 偏移一律按 Unicode 码点算(Python 的 str 下标)。前端的 JS 字符串按 UTF-16 计, 碰到码点在 BMP 之外的字符(emoji)要自己换算。 """ @@ -30,6 +39,7 @@ artifact_store.load,只读文件)。测试和接口可以换成自己的 loa from __future__ import annotations import bisect +import difflib import json import math import re @@ -40,7 +50,7 @@ from typing import Any, Literal, TypedDict from app.core.artifact_store import canonical_json, content_hash from app.core.artifact_store import load as load_artifact from app.engine.expressions import CellError, cell_value, column_kind, locate_cell -from app.engine.issuance import extract_numbers, number_allowance +from app.engine.issuance import _tolerance, extract_numbers, number_allowance DOC_SCHEMA = "agentlab.report/1" @@ -69,20 +79,26 @@ class EvidenceEntry(TypedDict, total=False): code_sha: str # node_output(代码节点)专有:实际执行的代码的 sha256、 role: str # evidence_role(source / compute)、语言 language: str + schema_artifact: str # query 专有(有表结构快照时):查询当时数据源的表结构快照 + tables: Any # query:SQL 里 FROM / JOIN 的表名(list);schema:{表: [列…]} + schema: str | None # schema 专有:数据库里的 schema 名(全名 = schema.表)、同步时间 + synced_at: str | None class Citation(TypedDict, total=False): - ref: str # 标记里写的原文:gmv、week、Q4.r0.amount - alias: str # 目录里的键:m:gmv、i:week、Q4 + ref: str # 标记里写的原文:gmv、week、Q4.r0.amount、orders + alias: str # 目录里的键:m:gmv、i:week、Q4、t:orders、K1 locator: dict[str, Any] eid: str kind: str # metric / input / query / cell / table / column / quote role: Literal["value", "entity", "quote", "support"] status: Literal["resolved", "unresolved", "mismatch"] reason: str # unresolved 时为什么 + unknown: bool # 实体:本次运行哪里都没有这个名字(可疑实体,不是写法错了) conv: str # 换算:万 / 亿 / pct / int / .N value: Any rendered: str + source: dict[str, Any] # 引文:原文所在的检索快照、文档、片段 class Segment(TypedDict, total=False): @@ -90,11 +106,14 @@ class Segment(TypedDict, total=False): kind: Literal["text", "number", "value", "entity", "quote", "structural"] text: str span: list[int] # [start, end),在 doc.markdown 里的码点偏移 - ref: str # 带前缀的原始引用:m:gmv、i:week + ref: str # 带前缀的原始引用:m:gmv、i:week、t:orders cite: Citation state: Literal["deterministic", "probabilistic", "none", "neutral"] strong: bool - issue: str # 这一段本身的问题:uncited_number / unresolved_ref + code: bool # 实体写在反引号里:text 带着两个反引号,渲染成行内代码 + auto: bool # 实体是正文里自动链接出来的,不是 [[t:]] / [[c:]] 标记 + name: str # 可疑实体(没有 ref):反引号里写的那个名字 + issue: str # 这一段本身的问题:uncited_number / unresolved_ref / unknown_entity class Unit(TypedDict, total=False): @@ -198,8 +217,214 @@ def query_entry_fields(payload: Any) -> dict[str, Any] | None: artifact, columns, rows = data.get("artifact"), data.get("columns"), data.get("rows") if not (isinstance(artifact, str) and artifact and isinstance(columns, list) and isinstance(rows, list)): return None - return {"artifact": artifact, "source": data.get("source"), "columns": [str(c) for c in columns], - "rows": len(rows), "truncated": bool(data.get("truncated"))} + fields = {"artifact": artifact, "source": data.get("source"), "columns": [str(c) for c in columns], + "rows": len(rows), "truncated": bool(data.get("truncated"))} + schema = data.get("schema_artifact") + if isinstance(schema, str) and schema: + # 表结构快照和 SQL 里的表名成对出现:没探查过结构的数据源,只凭 SQL 和结果列核对名字, + # 反引号里写的真实表名(没进这条 SQL)会被误标成编造的,这种查询不启用实体核对 + fields.update(schema_artifact=schema, tables=sql_tables(str(data.get("sql") or ""))) + return fields + + +# -------------------------------------------------------------------------- +# 表结构:冻结的快照、SQL 里用到的表 +# -------------------------------------------------------------------------- + +#: 数据源工具每次查询时把当时的表结构存成这种工件 +SCHEMA_SNAPSHOT = "schema_snapshot" + +_SQL_IDENT = r'(?:"[^"\n]+"|`[^`\n]+`|\[[^\]\n]+\]|[A-Za-z_][A-Za-z0-9_$#]*)' +_SQL_NAME = re.compile(rf"\s*({_SQL_IDENT}(?:\s*\.\s*{_SQL_IDENT}){{0,2}})") +_SQL_PART = re.compile(_SQL_IDENT) +_SQL_FROM = re.compile(r"\b(?:from|join)\b", re.I) +#: WITH x AS (…), y AS (…):公用表表达式的名字不是真表 +_SQL_CTE = re.compile(rf"(?:\bwith\b(?:\s+recursive\b)?|,)\s*({_SQL_IDENT})\s*(?:\([^()]*\)\s*)?\bas\s*\(", re.I) +#: 跟在 FROM 后面、却不是表名的词 +_SQL_NOT_TABLE = frozenset({"select", "lateral", "unnest", "dual", "values", "table"}) +#: 表名后面紧跟这些词时,它们不是别名 +_SQL_CLAUSE = frozenset({ + "where", "group", "order", "having", "limit", "offset", "fetch", "join", "inner", "left", "right", "full", + "cross", "outer", "natural", "on", "using", "union", "except", "intersect", "window", "for", "as", "and", + "or", "not", "when", "then", "else", "end", "set", "returning", "into", "pivot", "unpivot", "qualify"}) +#: FROM 出现在这些函数的括号里时是语法的一部分:EXTRACT(YEAR FROM ts)、TRIM(' ' FROM name) +_SQL_FROM_FUNCS = frozenset({"extract", "substring", "trim", "overlay", "position"}) +#: SQLite / SQL Server 的方括号标识符:里面有引号的不是(那是 ARRAY['…'] 这样的数组字面量) +_SQL_BRACKET = re.compile(r"\[[^\]'\"\n]*\]") +#: PostgreSQL 的美元引号字符串:$$…$$、$tag$…$tag$($1 这种参数不是) +_SQL_DOLLAR = re.compile(r"\$(?:[A-Za-z_][A-Za-z0-9_]*)?\$") +_SQL_WORD_CHAR = re.compile(r"[A-Za-z0-9_$#]") + + +def _sql_scan(sql: str) -> tuple[str, str]: + """一趟扫完注释、字符串、带引号的标识符,返回两份和原文等长的文本: + + - code:注释换成空格,字符串字面量的内容换成空格(引号留着),标识符原样——取表名用 + - words:在 code 的基础上,带引号的标识符的内容也换成空格——找 FROM / JOIN 这些关键字用, + "FROM x" 这样的名字(MySQL 默认还把它当字符串)里的 FROM 不是关键字 + + 字符串和注释必须一起认:先去注释再抹字符串的话,'--x' 会把后半条语句当注释吞掉, + '/*' 和 '*/' 两个字符串之间的正文也会被当成注释。拿不准的地方一律往「少认一张表」偏: + 反斜杠当转义(MySQL、E'…'),块注释可以嵌套(PostgreSQL),# 开头到行尾是注释(MySQL)。 + 换行原样保留。 + """ + code, words = list(sql), list(sql) + n = len(sql) + + def blank(a: int, b: int, *targets: list[str]) -> None: + for k in range(a, min(b, n)): + if sql[k] != "\n": + for t in targets: + t[k] = " " + + i = 0 + while i < n: + ch, nxt = sql[i], sql[i + 1] if i + 1 < n else "" + word_before = i > 0 and bool(_SQL_WORD_CHAR.match(sql[i - 1])) + if (ch == "-" and nxt == "-") or (ch == "#" and not word_before): + end = sql.find("\n", i) + end = n if end < 0 else end + blank(i, end, code, words) + i = end + elif ch == "/" and nxt == "*": + depth, j = 1, i + 2 + while j < n and depth: + if sql.startswith("/*", j): + depth, j = depth + 1, j + 2 + elif sql.startswith("*/", j): + depth, j = depth - 1, j + 2 + else: + j += 1 + blank(i, j, code, words) + i = j + elif ch == "'": + j = i + 1 + while j < n: + if sql[j] == "\\": + j += 2 + elif sql[j] == "'" and j + 1 < n and sql[j + 1] == "'": + j += 2 + elif sql[j] == "'": + break + else: + j += 1 + blank(i + 1, j, code, words) + i = j + 1 + elif ch in ('"', "`"): + j = i + 1 + while j < n and not (sql[j] == ch and not (j + 1 < n and sql[j + 1] == ch)): + j += 2 if sql[j] == ch else 1 + blank(i + 1, j, words) + i = j + 1 + elif ch == "[" and (m := _SQL_BRACKET.match(sql, i)): + blank(i + 1, m.end() - 1, words) + i = m.end() + elif ch == "$" and not word_before and (m := _SQL_DOLLAR.match(sql, i)): + close = sql.find(m.group(0), m.end()) + end = n if close < 0 else close + blank(m.end(), end, code, words) + i = n if close < 0 else close + len(m.group(0)) + else: + i += 1 + return "".join(code), "".join(words) + + +def _unquote(name: str) -> str: + return ".".join(p[1:-1] if p[:1] in ('"', "`", "[") else p for p in _SQL_PART.findall(name)) + + +def _inside_call(code: str, pos: int) -> bool: + """pos 所在的最内层括号,是不是 EXTRACT( / TRIM( 这类把 FROM 当语法用的函数;IS DISTINCT FROM 同理。""" + if re.search(r"\bdistinct\s*$", code[:pos], re.I): + return True + depth = 0 + for i in range(pos - 1, -1, -1): + ch = code[i] + if ch == ")": + depth += 1 + elif ch == "(": + if depth == 0: + word = re.search(r"([A-Za-z_]+)\s*$", code[:i]) + return bool(word and word.group(1).lower() in _SQL_FROM_FUNCS) + depth -= 1 + return False + + +def sql_tables(sql: str) -> list[str]: + """SQL 里 FROM / JOIN 后面的表名,按第一次出现的顺序、不分大小写去重。 + + 注释、字符串字面量一趟扫掉(_sql_scan:`-- FROM ghost`、'JOIN ghost'、'--x' 后面的正文都 + 处理对)。双引号、反引号、方括号括起来的标识符照样认,去掉引号;引号里的 FROM 不算关键字。 + WITH 定义的公用表表达式、子查询、表函数(generate_series(…))不算表。 + """ + if not sql: + return [] + code, words = _sql_scan(sql) + ctes = {_unquote(code[m.start(1):m.end(1)]).lower() for m in _SQL_CTE.finditer(words)} + out: dict[str, str] = {} + for keyword in _SQL_FROM.finditer(words): + if _inside_call(words, keyword.start()): + continue + pos = keyword.end() + while True: + m = _SQL_NAME.match(code, pos) + if not m or re.match(r"\s*\(", code[m.end():]): + break # 子查询、表函数 + name = _unquote(m.group(1)) + lowered = name.lower() + if lowered == "only": # FROM ONLY t(PostgreSQL) + pos = m.end() + continue + if lowered in _SQL_NOT_TABLE: + break + if lowered not in ctes: + out.setdefault(lowered, name) + pos = m.end() + alias = re.match(rf"\s+(?:as\s+)?({_SQL_IDENT})", code[pos:], re.I) + if alias and alias.group(1).lower() not in _SQL_CLAUSE: + pos += alias.end() + comma = re.match(r"\s*,", code[pos:]) + if not comma: + break + pos += comma.end() + return list(out.values()) + + +def schema_entry_fields(content: Any) -> dict[str, Any] | None: + """表结构快照的内容 → 台账 schema 条目要的字段:{source, schema, synced_at, tables: {表: [列…]}}, + 快照不全(探查时表太多被截断)时另有 truncated: True 和 total(库里一共几张表)。 + + 只记名字,不记类型、注释:目录核对名字存不存在就够了,类型要看的时候去取快照本身 + (它在封存范围内,取回时复验哈希)。没有表的快照返回 None。 + """ + if not isinstance(content, dict) or not isinstance(content.get("tables"), dict) or not content["tables"]: + return None + tables: dict[str, list[str]] = {} + for name, meta in content["tables"].items(): + cols = meta.get("columns") if isinstance(meta, dict) else None + tables[str(name)] = [str(c["name"]) for c in cols or [] if isinstance(c, dict) and c.get("name")] + fields = {"source": content.get("source"), "schema": content.get("schema"), "synced_at": content.get("synced_at"), + "tables": tables} + if content.get("truncated"): + # 库里的表太多、探查只存了前一部分:没列出的名字可能是真的,实体核对据此只说「核对不了」 + total = content.get("total") + fields.update(truncated=True, **({"total": total} if isinstance(total, int) else {})) + return fields + + +def schema_ledger_entry(artifact: str, *, node_id: str, exec_no: int, + loader: Callable[[str], Any] | None = None) -> dict[str, Any] | None: + """一件表结构快照 → 台账的 schema 条目。快照取不回来(或者哈希对不上)返回 None。 + + 节点执行器在工具调用之后调它:快照是内容寻址的,重放时读到的是同一份,条目也就是同一条。 + """ + try: + fields = schema_entry_fields((loader or load_artifact)(artifact)) + except Exception: # noqa: BLE001 - 取不回来就不记,不能让一次查询因此失败 + return None + if fields is None: + return None + return {"kind": "schema", "node_id": node_id, "exec": exec_no, "artifact": artifact, **fields} # -------------------------------------------------------------------------- @@ -406,8 +631,10 @@ _MARKER_GUARD = rf"(?=[^\[\]\n]{{0,{MARKER_MAX - 4}}}\]\])" #: [[kind:body]]。body 里不许有方括号和换行——这样一个没闭合的 [[ 最多吞到行尾 MARKER_RE = re.compile( r"\[\[" + _MARKER_GUARD + r"[ \t]*(?P[A-Za-z]+)[ \t]*:(?P[^\[\]\n]*?)\]\]") -SUPPORTED_KINDS = frozenset({"m", "i", "see", "v", "table"}) -LATER_KINDS = frozenset({"t", "c", "q"}) +SUPPORTED_KINDS = frozenset({"m", "i", "see", "v", "table", "t", "c", "q"}) +#: 三期起没有「以后才支持」的标记了;留着这个名字给老代码 import +LATER_KINDS: frozenset[str] = frozenset() +#: 这次运行没冻结表结构(没查过库、升级前的运行)时,t / c 仍按一期判为解析不了,原因还是这一句 LATER_REASON = "这种引用在后续版本支持" #: 沙箱代码节点的产出不能直接引用:它能做任意计算,那种数要进口径卡、留下代入式 CODE_REASON = "沙箱算出来的数要进口径卡" @@ -507,6 +734,10 @@ def build_catalog( 沙箱代码节点的台账条目登记成 N:<节点 id>(kind node_output,code: True):不编号、不进 写作目录,只为了有人写 [[v:N:calc.x]] 时能说清为什么不行。 + + 表和字段(t:<表>、c:<表>.<列>、c:<列>)来自三处:表结构快照(schema 条目)、查询 SQL 用到的 + 表、查询结果的列,后两样只认带着表结构快照的查询条目(见 query_entry_fields)。二期的台账 + 里没有这些,编不出实体条目,实体核对也就不启用。 """ if not metrics_from: metrics_from = None @@ -547,9 +778,14 @@ def build_catalog( } queries = retrievals = 0 + schemas: list[dict[str, Any]] = [] + shaped: list[tuple[dict[str, Any], str]] = [] # 带表结构快照的查询条目和它的全局编号 for entry in entries: if allow is not None and entry.get("node_id") not in allow: continue + if entry.get("kind") == "schema": + schemas.append(entry) + continue rows = entry.get("rows") if entry.get("kind") == "node_output" and entry.get("code_sha"): # 沙箱代码节点:登记下来只为了引用它时能说清为什么不行,不编号、不进写作目录 @@ -565,6 +801,8 @@ def build_catalog( queries += 1 alias, kind = f"Q{queries}", "query" label = f"{entry.get('tool') or entry.get('source') or kind}" + (f" · {rows} 行" if isinstance(rows, int) else "") + if entry.get("schema_artifact") and isinstance(entry.get("tables"), list): + shaped.append((entry, alias)) elif entry.get("kind") == "retrieval": retrievals += 1 alias, kind = f"K{retrievals}", "retrieval" @@ -579,6 +817,8 @@ def build_catalog( "columns", "rows", "truncated") if k in entry}, "label": label, } + if schemas or shaped: + catalog.update(_entity_entries(schemas, shaped)) for key, value in (inputs or {}).items(): if not isinstance(key, str) or not key or isinstance(value, (dict, list, tuple)): @@ -592,6 +832,231 @@ def build_catalog( return catalog +# -------------------------------------------------------------------------- +# 实体:表和字段 +# -------------------------------------------------------------------------- + +ENTITY_KINDS = frozenset({"table", "column"}) +UNKNOWN_ENTITY_REASON = "本次运行的表结构快照、查询用到的表、查询结果列里都没有这个名字,可能是编造的名字" +#: 表结构快照不全(库里的表太多、只存了一部分)时找不到的名字:可能在没存下来的表里,只能说核对不了 +UNVERIFIED_ENTITY_REASON = ("这个数据源的表太多,表结构快照只存了一部分,查询用到的表、查询结果列里也没有这个名字," + "核对不了它存不存在") +#: 报告节点写了 entities: off:作者有意关掉了表名、字段名核对 +ENTITIES_OFF_REASON = "这个报告节点关掉了表名、字段名核对(entities: off),[[t:]] / [[c:]] 不解析:直接写名字就行" + + +def _add_origin(entry: dict[str, Any], origin: dict[str, Any]) -> None: + key = (origin.get("kind"), origin.get("artifact"), origin.get("alias")) + if all((o.get("kind"), o.get("artifact"), o.get("alias")) != key for o in entry["sources"]): + entry["sources"].append(origin) + alias = origin.get("alias") + if alias and alias not in entry["queries"]: + entry["queries"].append(alias) + + +def _entity_entries(schemas: list[dict[str, Any]], queries: list[tuple[dict[str, Any], str]]) -> dict[str, Any]: + """表条目 t:<表>、列条目 c:<表>.<列>,没有表名的结果列(聚合的别名)是 c:<列>。 + + 每个条目的 eid 取它第一次出现的那件工件:表结构快照里有的按快照,只在 SQL / 结果列里出现 + 的按那份查询快照。sources 记全三种来历(schema / sql / result),queries 是哪几次查询用到。 + 同名字段在好几张表里(id、amount)另编一条 c:<列>,tables 写明是哪几张:正文里只写字段名时 + 指的是哪张说不准,但这个名字确实存在。 + """ + tables: dict[str, dict[str, Any]] = {} + columns: dict[str, dict[str, Any]] = {} + by_lower: dict[str, str] = {} # 小写的表名、全名、全名的末段 → 条目里的表名 + owners: dict[str, list[str]] = {} # 小写的列名 → 表结构里有它的表 + + def table(name: str, artifact: Any, *, qualified: str | None = None, source: Any = None) -> str: + key = by_lower.get(name.lower()) or (by_lower.get(qualified.lower()) if qualified else None) + if key is None and not qualified and "." in name: + key = by_lower.get(name.rsplit(".", 1)[-1].lower()) # SQL 里写了别的 schema 前缀的同名表 + if key is None: + key = name + tables[key] = {"alias": f"t:{key}", "kind": "table", "eid": make_eid("table", artifact, {"table": key}), + "artifact": artifact, "locator": {"table": key}, "name": key, + **({"qualified": qualified} if qualified and qualified != key else {}), + **({"source": source} if source else {}), "sources": [], "queries": [], + "label": f"表 {key}"} + for k in (key, qualified, key.rsplit(".", 1)[-1]): + if k: + by_lower.setdefault(k.lower(), key) + return key + + def column(alias: str, name: str, artifact: Any, locator: dict[str, str], source: Any) -> dict[str, Any]: + if alias not in columns: + columns[alias] = {"alias": alias, "kind": "column", "eid": make_eid("column", artifact, locator), + "artifact": artifact, "locator": locator, "name": name, + **({"table": locator["table"]} if "table" in locator else {}), + **({"source": source} if source else {}), "sources": [], "queries": [], + "label": f"字段 {alias[2:]}"} + return columns[alias] + + for entry in schemas: + artifact, source, prefix = entry.get("artifact"), entry.get("source"), entry.get("schema") + origin = {"kind": "schema", "artifact": artifact, **({"source": source} if source else {}), + **({"truncated": True} if entry.get("truncated") else {})} + for name, cols in (entry.get("tables") or {}).items(): + key = table(str(name), artifact, qualified=f"{prefix}.{name}" if prefix else None, source=source) + _add_origin(tables[key], origin) + for col in cols or []: + col = str(col) + _add_origin(column(f"c:{key}.{col}", col, artifact, {"table": key, "column": col}, source), origin) + listed = owners.setdefault(col.lower(), []) + if key not in listed: + listed.append(key) + by_column = {a.lower(): a for a in columns} + for entry, alias in queries: + artifact, source = entry.get("artifact"), entry.get("source") + extra = {"source": source} if source else {} + used = [] + for written in entry.get("tables") or []: + key = table(str(written), artifact, source=source) + _add_origin(tables[key], {"kind": "sql", "artifact": artifact, "alias": alias, **extra}) + used.append(key) + for col in entry.get("columns") or []: + col = str(col) + mine = [a for t in used if (a := by_column.get(f"c:{t}.{col}".lower()))] + target = columns[mine[0]] if len(mine) == 1 else column(f"c:{col}", col, artifact, {"column": col}, source) + _add_origin(target, {"kind": "result", "artifact": artifact, "alias": alias, **extra}) + for lowered, keys in owners.items(): + if len(keys) < 2: + continue + first = columns[by_column[f"c:{keys[0]}.{lowered}".lower()]] + target = column(f"c:{first['name']}", first["name"], first["artifact"], {"column": first["name"]}, + first.get("source")) + target["tables"] = list(keys) + for key in keys: + for origin in columns[by_column[f"c:{key}.{lowered}".lower()]]["sources"]: + if origin["kind"] == "schema": + _add_origin(target, origin) + return {**{e["alias"]: e for e in tables.values()}, **{e["alias"]: e for e in columns.values()}} + + +#: 反引号里、正文里像标识符的名字:字母或下划线开头,最多四段用点连起来 +_ENTITY_NAME = r"[A-Za-z_][A-Za-z0-9_$#]*(?:\.[A-Za-z_][A-Za-z0-9_$#]*){0,3}" +_IDENTIFIER = re.compile(rf"^{_ENTITY_NAME}$") +#: 反引号里写了这些不算名字:SQL 关键字、字面量、常用函数 +_SQL_WORDS = frozenset({ + "null", "true", "false", "none", "nan", "select", "from", "where", "group", "order", "by", "having", "limit", + "join", "left", "right", "inner", "outer", "on", "as", "and", "or", "not", "in", "is", "like", "between", + "case", "when", "then", "else", "end", "distinct", "union", "all", "with", "sum", "count", "avg", "min", + "max", "asc", "desc", "date", "now", "offset", "insert", "update", "delete"}) +#: 正文里的裸名字:这些常用英文词就算恰好是表名也不自动链接,免得满屏下划线 +_COMMON_WORDS = frozenset({ + "id", "name", "date", "time", "type", "status", "value", "data", "info", "key", "code", "text", "title", + "count", "sum", "total", "year", "month", "day", "week", "user", "users", "item", "items", "list", "table", + "log", "note", "notes", "order", "group", "level", "rank", "score", "price", "amount", "region", + "city", "source", "target", "event", "events", "result", "results", "report", "detail", "details"}) + + +class _EntityIndex: + """按名字找目录里的表和字段。不分大小写;全名(schema.表)和末段都认。 + + 表和字段的查法分开(t: 只找表,c: 只找字段)。正文里的名字两样都试:先整名当表,再当 + 字段(表.字段、只有字段名),最后才把 schema.表 按末段认成表——「orders.amount」不能因为 + 恰好有一张叫 amount 的表就认成表。 + """ + + def __init__(self, catalog: dict[str, Any]) -> None: + self.catalog = catalog + self.tables: dict[str, str] = {} + self.columns: dict[str, str] = {} + self.owners: dict[str, list[str]] = {} + self.others: set[str] = set() + self.described: set[str] = set() # 表结构快照里列出了全部字段的表(目录键) + self.partial = False # 有表结构快照不全(探查时被截断) + for alias, entry in catalog.items(): + if not isinstance(entry, dict): + continue + kind, locator = entry.get("kind"), entry.get("locator") or {} + if kind == "table": + frozen = [o for o in entry.get("sources") or [] if isinstance(o, dict) and o.get("kind") == "schema"] + if frozen: + self.described.add(alias) + self.partial = self.partial or any(o.get("truncated") for o in frozen) + name = str(entry.get("name") or alias[2:]) + for key in (name, entry.get("qualified"), name.rsplit(".", 1)[-1]): + if key: + self.tables.setdefault(str(key).lower(), alias) + elif kind == "column": + col = str(locator.get("column") or entry.get("name") or "") + if locator.get("table"): + self.columns.setdefault(f"{locator['table']}.{col}".lower(), alias) + self.owners.setdefault(col.lower(), []).append(alias) + else: + self.columns.setdefault(col.lower(), alias) + elif kind == "metric": + self.others.add(str(locator.get("metric") or "").lower()) + elif kind == "input": + self.others.add(str(locator.get("field") or "").lower()) + self.others.add(alias.lower()) + self.active = bool(self.tables or self.columns) + + def table(self, name: str, *, tail: bool = True) -> str | None: + lowered = name.lower() + hit = self.tables.get(lowered) + if hit is None and tail and "." in lowered: + hit = self.tables.get(lowered.rsplit(".", 1)[-1]) + return hit + + def column(self, name: str) -> str | None: + lowered = name.lower() + if lowered in self.columns: + return self.columns[lowered] + if "." in lowered: + head, col = lowered.rsplit(".", 1) + owner = self.table(head) + if owner is None: + return None + return self.columns.get(f"{self.catalog[owner]['name']}.{col}".lower()) + mine = self.owners.get(lowered) or [] + return mine[0] if len(mine) == 1 else None + + def find(self, name: str, kind: str | None = None) -> tuple[str, str] | None: + """(table | column, alias),找不到返回 None。""" + if kind in (None, "table") and (hit := self.table(name, tail=kind == "table")): + return "table", hit + if kind in (None, "column") and (hit := self.column(name)): + return "column", hit + if kind is None and (hit := self.table(name)): + return "table", hit + return None + + def known(self, name: str) -> bool: + """这个名字在这次运行里有没有出处:表、字段,或者指标 id、运行输入、证据编号。""" + return self.find(name) is not None or name.lower() in self.others + + def unsure(self, name: str) -> bool: + """一个找不到的名字是不是只能说「核对不了」:表结构快照不全时,它可能在没存下来的表里。 + + 快照里列出的表,字段是全的:「orders.ghost」里的 orders 在快照里,ghost 找不到就是真没有。 + """ + if not self.partial: + return False + head = name.rsplit(".", 1)[0] if "." in name else "" + owner = self.table(head) if head else None + return owner is None or owner not in self.described + + +def find_entity(name: str, catalog: dict[str, Any], kind: str | None = None) -> str | None: + """按名字找表或字段的目录键(t:orders、c:orders.amount),找不到返回 None。kind 为 table / column 时只找那一种。""" + hit = _EntityIndex(catalog).find(name.strip(), kind) + return hit[1] if hit else None + + +def closest_entities(name: str, catalog: dict[str, Any], *, limit: int = 3) -> list[str]: + """和一个可疑的名字最像的已知表、字段(目录键,最多 limit 个),给「你是不是想写…」用。""" + names: dict[str, str] = {} + for alias, entry in catalog.items(): + if isinstance(entry, dict) and entry.get("kind") in ENTITY_KINDS: + names.setdefault(alias[2:].lower(), alias) + names.setdefault(str(entry.get("name") or "").lower(), alias) + names.pop("", None) + picked = difflib.get_close_matches(name.strip().lower(), list(names), n=limit * 3, cutoff=0.6) + return list(dict.fromkeys(names[p] for p in picked))[:limit] + + # -------------------------------------------------------------------------- # 解析一个引用 # -------------------------------------------------------------------------- @@ -626,42 +1091,69 @@ Loader = Callable[[str], Any] class _Snapshots: - """这一次解析里取过的查询快照。 + """这一次解析里取过的快照(查询、检索),以及按名字找表和字段的索引。 一张整表每一格都要读同一份快照,每格都读盘、复验一次哈希不值。缓存只活在这一次 compose / verify / 流式渲染里:出口复核是另一次调用,照样重新取、重新验。 """ - def __init__(self, loader: Loader | None = None) -> None: + def __init__(self, loader: Loader | None = None, *, entities: bool = True) -> None: self.loader = loader or load_artifact self.memo: dict[str, tuple[Any, str | None]] = {} + self._index: tuple[dict[str, Any], _EntityIndex] | None = None + #: False:报告节点写了 entities: off,目录里就算有表和字段也整层不管 + self.entities_on = entities + + def entities(self, catalog: dict[str, Any]) -> _EntityIndex: + if self._index is None or self._index[0] is not catalog: + shown = catalog if self.entities_on else \ + {a: e for a, e in catalog.items() if not (isinstance(e, dict) and e.get("kind") in ENTITY_KINDS)} + self._index = (catalog, _EntityIndex(shown)) + return self._index[1] def get(self, alias: str, artifact: Any) -> tuple[Any, str | None]: - """(快照内容, 取不到的原因)。原因以 alias 开头,能直接当 unresolved 的 reason。""" + """查询快照:(内容, 取不到的原因)。原因以 alias 开头,能直接当 unresolved 的 reason。""" + content, why = self._fetch(artifact, "查询快照") + if why is None and (not isinstance(content, dict) or not isinstance(content.get("rows"), list)): + why = "的快照里没有表格数据" + return (None, f"{alias} {why}") if why else (content, None) + + def hits(self, alias: str, artifact: Any) -> tuple[list[Any], str | None]: + """检索快照里的命中:(命中列表, 取不到的原因)。""" + content, why = self._fetch(artifact, "检索快照") + if why is None and (not isinstance(content, dict) or not isinstance(content.get("hits"), list)): + why = "的快照里没有检索命中" + return ([], f"{alias} {why}") if why else (content["hits"], None) + + def _fetch(self, artifact: Any, noun: str) -> tuple[Any, str | None]: key = str(artifact or "") if key not in self.memo: - self.memo[key] = self._fetch(key) - content, why = self.memo[key] - return content, (f"{alias} {why}" if why else None) + self.memo[key] = self._load(key) + content, problem = self.memo[key] + if problem is None: + return content, None + if problem == "missing": + return None, f"没有{noun}" + if problem == "tampered": + return None, f"的{noun}和哈希对不上,疑似被改过" + if problem == "absent": + return None, f"的{noun}取不回来(工件 {key[:12]}… 不存在)" + return None, f"的{noun}取不回来({problem})" - def _fetch(self, artifact: str) -> tuple[Any, str | None]: + def _load(self, artifact: str) -> tuple[Any, str | None]: if not artifact: - return None, "没有查询快照" + return None, "missing" try: content = self.loader(artifact) except ValueError: - return None, "的查询快照和哈希对不上,疑似被改过" + return None, "tampered" except Exception as e: # noqa: BLE001 - 取不回来就是解析不了,不能让整篇报告崩掉 - return None, f"的查询快照取不回来({type(e).__name__})" - if content is None: - return None, f"的查询快照取不回来(工件 {artifact[:12]}… 不存在)" - if not isinstance(content, dict) or not isinstance(content.get("rows"), list): - return None, "的快照里没有表格数据" - return content, None + return None, type(e).__name__ + return (None, "absent") if content is None else (content, None) -def _snapshots(loader: Loader | _Snapshots | None) -> _Snapshots: - return loader if isinstance(loader, _Snapshots) else _Snapshots(loader) +def _snapshots(loader: Loader | _Snapshots | None, *, entities: bool = True) -> _Snapshots: + return loader if isinstance(loader, _Snapshots) else _Snapshots(loader, entities=entities) _CELL_REF = re.compile(r"^(?P[QK]\d+)\.r(?P\d+)\.(?P.+)$") @@ -849,14 +1341,14 @@ def resolve_marker(marker: dict[str, Any], catalog: dict[str, Any], *, cells_all # 整篇渲染时整表已经按上下文展开过了,走到这里的是复核一个片段:给出不带上下文的样子 return {"ref": ref, "alias": ref, "locator": {}, "eid": catalog[ref]["eid"], "kind": "query", "role": "value", "status": "resolved", "rendered": _render(table or "", catalog, snaps, cells_allowed)} - if kind in LATER_KINDS: - alias = {"t": f"t:{ref}", "c": f"c:{ref}"}.get(kind, ref.split(".")[0]) - ev_kind = {"t": "table", "c": "column", "q": "quote"}[kind] - return _unresolved(marker, alias, ev_kind, LATER_REASON) + if kind in ("t", "c"): + return _resolve_entity(marker, catalog, _snapshots(loader)) + if kind == "q": + return _resolve_quote(marker, catalog, _snapshots(loader)) if kind not in ("m", "i"): return _unresolved(marker, f"{kind}:{ref}", kind, f"不认识的引用类型 {kind}:只支持 m(口径卡指标)、v(查询单元格)、table(整表)、" - "i(运行输入)、see(依据)") + "i(运行输入)、t / c(表 / 字段)、q(引文)、see(依据)") conv = marker.get("conv") if kind == "m": @@ -891,16 +1383,149 @@ def resolve_marker(marker: dict[str, Any], catalog: dict[str, Any], *, cells_all return cite +def _resolve_entity(marker: dict[str, Any], catalog: dict[str, Any], snaps: _Snapshots) -> dict[str, Any]: + """[[t:orders]] / [[c:orders.amount]]:显示名字本身(照写作者写的大小写),出处是目录里的表或字段。 + + 报告节点关掉了实体层(entities: off)时照实说关掉了;这次运行没冻结表结构时没有东西可以核对, + 按一期的样子判为解析不了。名字哪里都找不到的是可疑实体(unknown: True):它不是写法错了,是可能 + 编造了一个名字——出口按缺口处理,不拦。表结构快照不全时找不到的名字只是核对不了(unverified: True)。 + """ + kind, ref = marker["kind"], (marker.get("ref") or "").strip() + ev_kind, other = ("table", "column") if kind == "t" else ("column", "table") + alias = f"{kind}:{ref}" + index = snaps.entities(catalog) + if not snaps.entities_on: + return _unresolved(marker, alias, ev_kind, ENTITIES_OFF_REASON) + if not index.active: + return _unresolved(marker, alias, ev_kind, LATER_REASON) + if not ref or not _IDENTIFIER.match(ref): + return _unresolved(marker, alias, ev_kind, "表名、字段名要写成 [[t:表名]]、[[c:表名.字段名]]," + "名字是字母或下划线开头的标识符") + hit = index.find(ref, ev_kind) + if hit is None: + if index.find(ref, other): + right = "c" if kind == "t" else "t" + return _unresolved(marker, alias, ev_kind, f"{ref} 是{'字段' if kind == 't' else '表'}," + f"不是{'表' if kind == 't' else '字段'}:写成 [[{right}:{ref}]]") + if index.unsure(ref): + cite = _unresolved(marker, alias, ev_kind, UNVERIFIED_ENTITY_REASON, rendered=ref) + cite["unverified"] = True + return cite + cite = _unresolved(marker, alias, ev_kind, UNKNOWN_ENTITY_REASON, rendered=ref) + cite["unknown"] = True + return cite + entry = catalog[hit[1]] + return {"ref": ref, "alias": hit[1], "locator": dict(entry.get("locator") or {}), "eid": entry["eid"], + "kind": ev_kind, "role": "entity", "status": "resolved", "rendered": ref} + + +# -------------------------------------------------------------------------- +# 引文:[[q:K1|原话]] 要在检索快照的命中片段里逐字出现 +# -------------------------------------------------------------------------- + +#: 引文至少几个字(空白归一化之后):「东区」这种两个字的「原话」哪里都找得到,证明不了什么 +QUOTE_MIN = 4 + + +def _wide(ch: str) -> bool: + """中日韩文字和全角标点:它们之间的空白是排版(换行、PDF 断行),不是词的分隔。""" + code = ord(ch) + return (0x2E80 <= code <= 0x9FFF or 0xF900 <= code <= 0xFAFF or 0xFF00 <= code <= 0xFFEF + or 0x3000 <= code <= 0x303F) + + +def _normalized(text: str) -> tuple[str, list[int]]: + """空白归一化:连续空白并成一个空格、去掉首尾,挨着中文或全角字符的空白直接去掉。 + + 同时记下归一化后每个字在原文里的位置,命中之后能换算回原文的起止。 + """ + out: list[str] = [] + where: list[int] = [] + gap: int | None = None + for i, ch in enumerate(text): + if ch.isspace(): + if out and gap is None: + gap = i + continue + if gap is not None and not (_wide(out[-1]) or _wide(ch)): + out.append(" ") + where.append(gap) + gap = None + out.append(ch) + where.append(i) + return "".join(out), where + + +def normalize_quote(text: str) -> str: + """引文比对前的样子:见 _normalized。大小写、标点、全半角都不改——要的是逐字。""" + return _normalized(text or "")[0] + + +def find_quote(quote: str, hits: list[Any]) -> tuple[int, int, int] | None: + """引文落在哪条命中、原文里的 [起, 止)。按命中的顺序找第一处;找不到返回 None。""" + wanted = normalize_quote(quote) + if not wanted: + return None + for i, hit in enumerate(hits): + content = hit.get("content") if isinstance(hit, dict) else None + if not isinstance(content, str): + continue + flat, where = _normalized(content) + at = flat.find(wanted) + if at >= 0: + return i, where[at], where[at + len(wanted) - 1] + 1 + return None + + +def _resolve_quote(marker: dict[str, Any], catalog: dict[str, Any], snaps: _Snapshots) -> dict[str, Any]: + """引文显示引文本身;原话对不上时照样显示(写作者的话),只是标成没有出处。 + + 目录里没有这次检索、快照取不回来时没有东西可以对,和别的解析不了的引用一样显示占位。 + """ + alias = (marker.get("ref") or "").strip() + quote = _clean(str(marker.get("quote") or "")) + entry = catalog.get(alias) + if entry is None: + return _unresolved(marker, alias, "quote", f"目录里没有 {alias}(能引原话的知识库检索见证据目录)") + if entry.get("kind") != "retrieval": + return _unresolved(marker, alias, "quote", f"{alias} 不是知识库检索:引文只能引检索命中的原文") + if not quote: + return _unresolved(marker, alias, "quote", "引文是空的:写成 [[q:K1|原话]]") + hits, why = snaps.hits(alias, entry.get("artifact")) + if why: + return _unresolved(marker, alias, "quote", why) + if len(normalize_quote(quote)) < QUOTE_MIN: + return _unresolved(marker, alias, "quote", f"引文太短(至少 {QUOTE_MIN} 个字),看不出是不是原话", + rendered=quote) + found = find_quote(quote, hits) + if found is None: + return _unresolved(marker, alias, "quote", f"{alias} 的命中片段里找不到这句原话(空白归一化后逐字比对):" + "引文要一字不差地抄原文", rendered=quote) + at, start, end = found + hit = hits[at] + locator = {"hit": at, "start": start, "end": end} + source = {"artifact": entry.get("artifact"), "document": hit.get("document_id"), "chunk": hit.get("chunk_id"), + "title": hit.get("title"), "ordinal": hit.get("ordinal")} + return {"ref": alias, "alias": alias, "locator": locator, "eid": make_eid("quote", entry.get("artifact"), locator), + "kind": "quote", "role": "quote", "status": "resolved", "rendered": quote, + "source": {k: v for k, v in source.items() if v is not None}} + + _ROW_REF = re.compile(r"^(?P[QK]\d+)(?:\.r(?P\d+))?$") -def resolve_support(ref: str, catalog: dict[str, Any]) -> dict[str, Any]: - """[[see:]] 里的一个依据:m:gmv、i:week、Q4、Q4.r0、K2。只核对「这件证据存在」。""" +def resolve_support(ref: str, catalog: dict[str, Any], *, loader: Loader | _Snapshots | None = None) -> dict[str, Any]: + """[[see:]] 里的一个依据:m:gmv、i:week、Q4、Q4.r0、K2、t:orders、c:orders.amount。只核对「这件证据存在」。""" ref = ref.strip() kind, sep, body = ref.partition(":") base = {"ref": ref, "role": "support"} - if sep and kind in LATER_KINDS: - return {**base, "alias": ref, "kind": kind, "status": "unresolved", "reason": LATER_REASON} + if sep and kind in ("t", "c"): + cite = _resolve_entity({"kind": kind, "ref": body, "body": body}, catalog, _snapshots(loader)) + return {**base, **{k: cite[k] for k in ("alias", "kind", "status", "eid", "locator", "reason", "unknown", + "unverified") if k in cite}} + if sep and kind == "q": + return {**base, "alias": ref, "kind": "quote", "status": "unresolved", + "reason": "引文不能当依据:依据写那次检索的编号,比如 [[see:K1]]"} alias: str | None = None locator: dict[str, Any] = {} why = f"目录里没有 {ref}" @@ -1396,14 +2021,117 @@ def _normalize_strong(raw: str) -> tuple[str, list[tuple[int, int]]]: return "".join(out), ranges +# -------------------------------------------------------------------------- +# 实体的自动链接:反引号里的名字一律核对,正文里的裸名字只链接明显是标识符的 +# -------------------------------------------------------------------------- + +_CODE_NAME = re.compile(r"^`([^`\n]+)`$") +#: 复核时找反引号:粗体、链接里面的也算——组装时它们没切成片段(切开会拆坏粗体),违规照记 +_CODE_SPAN = re.compile(r"(? bool: + """反引号里的这段是不是要核对的名字:像标识符、不止一个字、不是 SQL 关键字和常用函数。""" + return len(name) >= 2 and bool(_IDENTIFIER.match(name)) and name.lower() not in _SQL_WORDS + + +def _bare_link(name: str, catalog: dict[str, Any], index: _EntityIndex) -> tuple[str, str] | None: + """正文里的裸名字要不要自动链接:保守,只链接一眼就是标识符的。 + + 表.列 这种全名、和表名一字不差的(大小写也一样)、带下划线或数字或大小写混排的(order_id、 + orderId)才链接;name、date、id 这类常用词,哪怕恰好是字段名、表名,也不链接,免得满屏下划线。 + """ + if name.lower() in _COMMON_WORDS: + return None + hit = index.find(name) + if hit is None: + return None + entry = catalog[hit[1]] + if "." in name or (hit[0] == "table" and name in (entry.get("name"), entry.get("qualified"))): + return hit + mixed = any(ch.isupper() for ch in name[1:]) and any(ch.islower() for ch in name) + if "_" in name or any(ch.isdigit() for ch in name) or mixed: + return hit + return None + + +def _entity_segment(name: str, hit: tuple[str, str], catalog: dict[str, Any], *, code: bool) -> dict[str, Any]: + kind, alias = hit + entry = catalog[alias] + cite = {"ref": name, "alias": alias, "locator": dict(entry.get("locator") or {}), "eid": entry["eid"], + "kind": kind, "role": "entity", "status": "resolved", "rendered": name} + seg = {"kind": "entity", "text": f"`{name}`" if code else name, "ref": f"{'t' if kind == 'table' else 'c'}:{name}", + "cite": cite, "state": "deterministic", "auto": True} + if code: + seg["code"] = True + return seg + + +def _mentions(text: str, catalog: dict[str, Any], index: _EntityIndex) -> list[tuple[int, int, dict[str, Any]]]: + """一段文字里要切出来的实体:(起, 止, 片段)。只切顶层的行内代码和粗体、链接外面的裸名字。""" + found: list[tuple[int, int, dict[str, Any]]] = [] + protected: list[tuple[int, int]] = [] + for m in _INLINE.finditer(text): + protected.append((m.start(), m.end())) + code = _CODE_NAME.match(m.group(0)) + name = code.group(1) if code else "" + if not code or not _checked_name(name): + continue + if hit := index.find(name): + found.append((m.start(), m.end(), _entity_segment(name, hit, catalog, code=True))) + elif not index.known(name): + issue = "unverified_entity" if index.unsure(name) else "unknown_entity" + found.append((m.start(), m.end(), {"kind": "entity", "text": m.group(0), "state": "none", + "issue": issue, "code": True, "name": name})) + for m in _BARE_NAME.finditer(text): + if any(a <= m.start() < b for a, b in protected): + continue + if hit := _bare_link(m.group(0), catalog, index): + found.append((m.start(), m.end(), _entity_segment(m.group(0), hit, catalog, code=False))) + return sorted(found, key=lambda f: f[0]) + + +def _link_entities(segments: list[dict[str, Any]], catalog: dict[str, Any], + index: _EntityIndex) -> list[dict[str, Any]]: + """把文字片段里的表名、字段名切成实体片段。只重新切片,正文一个字不改。""" + out: list[dict[str, Any]] = [] + for seg in segments: + found = _mentions(seg["text"], catalog, index) if seg["kind"] == "text" else [] + if not found: + out.append(seg) + continue + s0, text = seg["span"][0], seg["text"] + extra = {"strong": True} if seg.get("strong") else {} + cursor = 0 + for a, b, piece in found: + if cursor < a: + out.append({"kind": "text", "text": text[cursor:a], "span": [s0 + cursor, s0 + a], + "state": seg["state"], **extra}) + out.append({**piece, "span": [s0 + a, s0 + b], **extra}) + cursor = b + if cursor < len(text): + out.append({"kind": "text", "text": text[cursor:], "span": [s0 + cursor, s0 + len(text)], + "state": seg["state"], **extra}) + return out + + +def _auto_entity(seg: dict[str, Any]) -> bool: + """系统在正文里自动切出来的实体片段(自动链接的名字、反引号里的可疑名字),不是写作者写的标记。""" + return seg.get("kind") == "entity" and (bool(seg.get("auto")) or not seg.get("ref")) + + def _classify(kind: str | None, segments: list[dict[str, Any]], see: list[dict[str, Any]]) -> str: """unit 的种类。本期没有裁判,按确定的规则分:引用了证据、写了数字、或者有方向词、 - 因果词的算结论;其余的是连接性的话。""" + 因果词的算结论;其余的是连接性的话。 + + 自动链接的名字按文字算:它只是把正文重新切了一刀,句子算不算结论和开没开实体层无关。 + """ if kind: return kind - if see or any(s["kind"] != "text" and s["kind"] != "structural" for s in segments): + if see or any(s["kind"] not in ("text", "structural") and not _auto_entity(s) for s in segments): return "claim" - if any(s["kind"] == "text" and _DIRECTIONAL.search(s["text"]) for s in segments): + if any((s["kind"] == "text" or _auto_entity(s)) and _DIRECTIONAL.search(s["text"]) for s in segments): return "claim" return "connective" @@ -1417,15 +2145,19 @@ def compose_doc( allow_numbers: list[Any] | None = None, cells_allowed: bool = True, loader: Loader | None = None, + entities: bool = True, ) -> dict[str, Any]: """模型写的原文(带标记)→ 报告文档:渲染后的 markdown、块、句、片段、统计、违规。 + entities=False(报告节点 entities: off):表名、字段名整层不管——不链接、不核对反引号, + [[t:]] / [[c:]] 解析不了,原因是 ENTITIES_OFF_REASON。 + stats / violations 由 verify_doc 算出,和出口契约复核用的是同一个函数。 整表标记先展开成每格一个 [[v:]] 的 Markdown 表,之后和模型自己写的表格走同一条路; doc["source"] 仍是模型的原文。 """ source = (raw or "").replace("\r\n", "\n").replace("\r", "\n").strip() - snaps = _Snapshots(loader) + snaps = _Snapshots(loader, entities=entities) text, strong = _normalize_strong(_expand_tables(source, catalog, snaps, cells_allowed)) markers = parse_markers(text) for marker in markers: @@ -1485,7 +2217,7 @@ def compose_doc( plain(cursor, marker["start"]) cursor = marker["end"] if marker["kind"] == "see": - see.extend(resolve_support(r, catalog) for r in marker["refs"]) + see.extend(resolve_support(r, catalog, loader=snaps) for r in marker["refs"]) continue cite = resolve_marker(marker, catalog, cells_allowed=cells_allowed, loader=snaps) ok = cite["status"] == "resolved" @@ -1497,7 +2229,10 @@ def compose_doc( "cite": cite, "state": "deterministic" if ok else "none", } if not ok: - seg["issue"] = "unresolved_ref" + # 实体名字哪里都找不到:可疑实体,不是写法错了(出口只按缺口处理); + # 表结构快照不全时只是核对不了(出口只标注) + seg["issue"] = "unknown_entity" if cite.get("unknown") else \ + "unverified_entity" if cite.get("unverified") else "unresolved_ref" if marker["strong"]: seg["strong"] = True push(seg, segments) @@ -1514,6 +2249,14 @@ def compose_doc( out_blocks.append({"type": block.type, **block.meta, "units": units}) markdown = "".join(parts) + index = snaps.entities(catalog) + if index.active: + # 表名、字段名切成实体片段(代码块里的不算:那是 SQL 原文,不是在报告里提到一个名字) + for block in out_blocks: + if block["type"] != "code": + for unit in block["units"]: + unit["segments"] = _link_entities(unit["segments"], catalog, index) + all_segments = [s for block in out_blocks for unit in block["units"] for s in unit["segments"]] # 裸数字单独切成片段:前端要在那几个字底下画「无出处」 refs = [tuple(s["span"]) for s in all_segments if s.get("ref")] + _header_masks(out_blocks, catalog) syntax = [tuple(s["span"]) for s in all_segments if s["kind"] == "structural"] @@ -1535,8 +2278,11 @@ def compose_doc( anchor = unit.pop("_anchor") unit["span"] = [body[0]["span"][0], body[-1]["span"][1]] if body else [anchor, anchor] unit["kind"] = _classify(unit["kind"], unit["segments"], unit["see"]) + # 这句话挂的依据:只认写作者自己写的标记和 [[see:]]。自动链接的名字是系统加的标注, + # 提一个字段名不等于给结论挂了出处(claims: require_citation 按这个判) unit["cites"] = list(dict.fromkeys( - c["alias"] for c in [*(s["cite"] for s in unit["segments"] if s.get("cite")), *unit["see"]] + c["alias"] for c in [*(s["cite"] for s in unit["segments"] if s.get("cite") and not s.get("auto")), + *unit["see"]] if c["status"] == "resolved")) # 字段顺序固定,文档内容寻址,同样的输入要得到同样的哈希 ordered = {k: unit[k] for k in ("id", "kind", "span", "cites", "see", "segments", "loc", "depth") @@ -1544,9 +2290,15 @@ def compose_doc( unit.clear() unit.update(ordered) + # 表结构可能有几百张表、几千个字段:文档里只留正文用到的实体条目(复核用的是重建的目录,不看这份)。 + # 自动链接的名字不进 cites,但点开它要查这份目录,所以按片段自己的 cite 收 + used = {c["alias"] for block in out_blocks for unit in block["units"] + for c in [*(s["cite"] for s in unit["segments"] if s.get("cite")), *unit["see"]] + if c.get("status") == "resolved" and c.get("alias")} + kept = {a: e for a, e in catalog.items() if e.get("kind") not in ENTITY_KINDS or a in used} doc: dict[str, Any] = { "schema": DOC_SCHEMA, "run_id": run_id, "node_id": node_id, - "markdown": markdown, "source": source, "catalog": catalog, "blocks": out_blocks, + "markdown": markdown, "source": source, "catalog": kept, "blocks": out_blocks, } checked = verify_doc(doc, catalog, allow_numbers=allow_numbers, cells_allowed=cells_allowed, loader=snaps) doc["stats"], doc["violations"] = checked["stats"], checked["violations"] @@ -1653,11 +2405,13 @@ def verify_doc( allow_numbers: list[Any] | None = None, cells_allowed: bool = True, loader: Loader | _Snapshots | None = None, + entities: bool = True, ) -> dict[str, Any]: - """逐项复核一份报告文档:{ok, violations, stats}。 + """逐项复核一份报告文档:{ok, violations, stats, uncited}。 cells_allowed=False:单元格引用(含整表里的格)一律判解析不了,原因是 CELLS_REASON—— 受管级别的正式出具,契约没声明 cells 时出口复核这样调。快照经 loader 重新取、复验哈希。 + entities=False:报告节点写了 entities: off,和组装时一样整层不管表名、字段名。 报告节点自查用它(违规就让写作者重写),出口契约复核也用它(判档)——后者传的 是自己从状态里重建的目录,不用文档里存的那份。复核什么: @@ -1668,16 +2422,31 @@ def verify_doc( - [[see:]] 里的每个依据都能解析 - 引用涂白、结构片段只涂符号之后,正文里剩下的数字全部算裸数字(结构片段里的 「100. 」「```45678」也在内) + - 目录里有表和字段时(这次运行冻结了表结构):反引号里像标识符、却哪里都找不到的名字 + 全部算可疑实体(unknown_entity)——不管文档有没有把它切成片段,粗体、链接里的也算; + 表结构快照不全时,这样的名字只算核对不了(unverified_entity) + - 引文重新取检索快照、重新逐字比对 - 片段标的状态和核对结果一致(界面按状态画线,不能标着「有出处」其实没有) + + 另外返回 uncited:没挂依据的结论句 [{unit, span, text}](按这次核对的引用算,不信文档里的 cites; + 自动链接的名字不算依据)。 + + stats 里 entities、quotes、unknown_entities、unverified_entities 四个键只在目录里有表和字段、或者 + 有知识库检索时才有(它们才可能不是 0):升级前的运行统计的键和以前一模一样。 """ - snaps = _snapshots(loader) + snaps = _snapshots(loader, entities=entities) + index = snaps.entities(catalog) violations: list[dict[str, Any]] = [] + uncited: list[dict[str, Any]] = [] + layered = index.active or any(isinstance(e, dict) and e.get("kind") == "retrieval" for e in catalog.values()) stats = {"units": 0, "segments": 0, "claims": 0, "connective": 0, "headings": 0, "numbers": 0, - "numbers_cited": 0, "values": 0, "uncited_numbers": 0, "unresolved": 0, "see": 0, - "uncited_claims": 0, "violations": 0} + "numbers_cited": 0, "values": 0, **({"entities": 0, "quotes": 0} if layered else {}), + "uncited_numbers": 0, "unresolved": 0, + **({"unknown_entities": 0, "unverified_entities": 0} if layered else {}), + "see": 0, "uncited_claims": 0, "violations": 0} if not isinstance(doc, dict): violations.append(_violation("bad_schema", "报告文档不是一个对象")) - return {"ok": False, "violations": violations, "stats": stats} + return {"ok": False, "violations": violations, "stats": stats, "uncited": uncited} if doc.get("schema") != DOC_SCHEMA: violations.append(_violation("bad_schema", f"不认识的报告文档格式 {doc.get('schema')!r}")) @@ -1686,7 +2455,7 @@ def verify_doc( syntax: list[tuple[int, int]] = [] # 结构片段:只涂符号,数字照查 owners: list[tuple[int, int, str, str]] = [] # (start, end, segment id, unit id) pos, tiled = 0, True - for _, unit in iter_units(doc): + for block, unit in iter_units(doc): stats["units"] += 1 kind = unit.get("kind") if kind in ("claim", "connective"): @@ -1720,17 +2489,26 @@ def verify_doc( f"结构片段里夹带了内容「{text[:30]}」", **where)) elif seg.get("ref"): ref = str(seg["ref"]) - cite = resolve_ref(ref, catalog, cells_allowed=cells_allowed, loader=snaps) + cite = _phase_two_cite(seg, ref, text) \ + or resolve_ref(ref, catalog, cells_allowed=cells_allowed, loader=snaps) ok = cite["status"] == "resolved" - matches = cite["rendered"] == text + # 反引号里的实体:片段的字带着两个反引号 + shown = f"`{cite['rendered']}`" if seg.get("code") and seg.get("kind") == "entity" else cite["rendered"] + matches = shown == text if intact: masks.append((span[0], span[1])) - if not ok: + if not ok and cite.get("unknown"): + violations.append(_violation("unknown_entity", _unknown_message(cite.get("ref") or ref), + ref=ref, **where)) + elif not ok and cite.get("unverified"): + violations.append(_violation("unverified_entity", _unverified_message(cite.get("ref") or ref), + ref=ref, **where)) + elif not ok: violations.append(_violation("unresolved_ref", f"引用 [[{ref}]] 解析不了:{cite['reason']}", ref=ref, **where)) elif not matches: violations.append(_violation( - "render_mismatch", f"片段「{text}」和引用 [[{ref}]] 重新渲染出来的「{cite['rendered']}」" + "render_mismatch", f"片段「{text}」和引用 [[{ref}]] 重新渲染出来的「{shown}」" "对不上", ref=ref, **where)) elif (seg.get("cite") or {}).get("eid") not in (None, cite["eid"]): violations.append(_violation("eid_mismatch", f"引用 [[{ref}]] 记的证据标识和目录里的不一致," @@ -1741,13 +2519,30 @@ def verify_doc( stats["numbers"] += 1 stats["numbers_cited"] += int(good) elif good: - stats["values"] += 1 - if ok: + key = {"entity": "entities", "quote": "quotes"}.get(seg.get("kind"), "values") + stats[key if key in stats else "values"] += 1 + if ok and not seg.get("auto"): + # 自动链接的名字是标注,不是写作者给这句话挂的依据 cited.append(cite["alias"]) elif seg.get("kind") == "number": expected = "none" + elif seg.get("kind") == "entity": + # 可疑实体(没有引用):反引号里那个名字这次核对下来确实哪里都没有,才该标 none + code = _CODE_NAME.match(text) + expected = "none" if code and index.active and not index.known(code.group(1)) else "neutral" elif state == "deterministic": expected = "neutral" + if seg.get("kind") in ("text", "entity") and not seg.get("ref") and index.active \ + and block.get("type") != "code": + for m in _CODE_SPAN.finditer(text): + name = m.group(1) + if _checked_name(name) and not index.known(name): + at = [span[0] + m.start(1), span[0] + m.end(1)] if intact else None + unsure = index.unsure(name) + violations.append(_violation("unverified_entity" if unsure else "unknown_entity", + (_unverified_message if unsure else _unknown_message)(name), + span=at, text=name, segment=seg.get("id"), unit=unit.get("id"), + context=_context(markdown, *at) if at else None)) # 已经为这一段报过别的问题,状态不对是它的推论,不再重复 if expected is not None and state != expected and len(violations) == found: violations.append(_violation("state_mismatch", @@ -1755,9 +2550,16 @@ def verify_doc( **where)) for support in unit.get("see") or []: stats["see"] += 1 - again = resolve_support(str(support.get("ref") or ""), catalog) + again = resolve_support(str(support.get("ref") or ""), catalog, loader=snaps) if again["status"] == "resolved": cited.append(again["alias"]) + elif again.get("unknown") or again.get("unverified"): + unsure = bool(again.get("unverified")) + violations.append(_violation("unverified_entity" if unsure else "unknown_entity", + (_unverified_message if unsure else _unknown_message)( + again["ref"].partition(":")[2]), + ref=again["ref"], unit=unit.get("id"), + span=list(unit.get("span") or []) or None)) else: violations.append(_violation("unresolved_ref", f"依据 [[see:{again['ref']}]] 解析不了:{again['reason']}", @@ -1765,6 +2567,10 @@ def verify_doc( span=list(unit.get("span") or []) or None)) if kind == "claim" and not cited: stats["uncited_claims"] += 1 + where = unit.get("span") + ok_span = isinstance(where, list) and len(where) == 2 and all(isinstance(x, int) for x in where) + uncited.append({"unit": unit.get("id"), "span": list(where) if ok_span else None, + "text": markdown[where[0]:where[1]] if ok_span else ""}) if tiled and pos != len(markdown): violations.append(_violation("segment_mismatch", "正文末尾有一段不在任何片段里:文档可能被改过")) @@ -1781,8 +2587,48 @@ def verify_doc( stats["uncited_numbers"] = sum(1 for v in violations if v["code"] == "uncited_number") stats["numbers"] += stats["uncited_numbers"] stats["unresolved"] = sum(1 for v in violations if v["code"] == "unresolved_ref") + if layered: + stats["unknown_entities"] = sum(1 for v in violations if v["code"] == "unknown_entity") + stats["unverified_entities"] = sum(1 for v in violations if v["code"] == "unverified_entity") stats["violations"] = len(violations) - return {"ok": not violations, "violations": violations, "stats": stats} + return {"ok": not violations, "violations": violations, "stats": stats, "uncited": uncited} + + +def _phase_two_cite(seg: dict[str, Any], ref: str, text: str) -> dict[str, Any] | None: + """升级前(二期)组装的文档里的 t / c / q:那时一律判为解析不了,正文是占位,原因是 LATER_REASON。 + + 这样的文档升级后才被复核(运行停在报告和出口之间)时,按当时的规矩查:正文确实是占位、确实 + 没有出处,记解析不了——不能因为今天解析得了,就把它判成「渲染对不上」(完整性问题,出口记缺口)。 + 只认「原因是 LATER_REASON 并且正文恰好是占位」的片段:自称二期却显示了别的字的,照现在的规矩查。 + """ + kind, _, body = ref.partition(":") + stored = seg.get("cite") + if kind not in ("t", "c", "q") or not isinstance(stored, dict) or stored.get("reason") != LATER_REASON \ + or text != placeholder(kind, body): + return None + alias = stored.get("alias") if isinstance(stored.get("alias"), str) else ref + return {"ref": body, "alias": alias, "kind": stored.get("kind") or kind, "role": _ROLE.get(kind, "value"), + "status": "unresolved", "reason": LATER_REASON, "rendered": text} + + +def _unknown_message(name: str) -> str: + return f"「{name}」{UNKNOWN_ENTITY_REASON}:只写本次运行的表结构、查询里真实存在的表名和字段名" + + +def _unverified_message(name: str) -> str: + return f"「{name}」核对不了:{UNVERIFIED_ENTITY_REASON}" + + +def uncited_claims(doc: dict[str, Any]) -> list[dict[str, Any]]: + """文档里没挂依据的结论句:[{unit, span, text}]。按文档自己记的 cites 算,给界面用; + 出口判档要用 verify_doc 返回的 uncited(按重新核对的引用算)。""" + markdown = str(doc.get("markdown") or "") + out = [] + for _, unit in iter_units(doc): + span = unit.get("span") + if unit.get("kind") == "claim" and not unit.get("cites") and isinstance(span, list) and len(span) == 2: + out.append({"unit": unit.get("id"), "span": list(span), "text": markdown[span[0]:span[1]]}) + return out # -------------------------------------------------------------------------- @@ -1806,9 +2652,32 @@ CELL_RULES = ( "结论的依据可以写查询编号,比如 [[see:Q1]]。下面没有的查询、行、列不要编造。" ) +#: 表名、字段名的写法。只在这次运行冻结了表结构(目录里有表和字段)时出现 +ENTITY_RULES = ( + "表和字段(系统会核对):提到表名、字段名时写 [[t:表名]]、[[c:表名.字段名]],或者放进反引号(`表名`);" + "只写下面列出的、或者这次查过的数据源里真实存在的名字——本次运行的表结构快照、查询用到的表、" + "查询结果列里都没有的名字,会被标成「可能是编造的名字」。" +) +#: 引原话的写法。只跟着目录里的知识库检索出现 +QUOTE_RULES = ( + "知识库检索:引用原话写 [[q:K1|原话]],原话必须和下面片段里的字一字不差(系统逐字核对,可以只抄其中" + "连续的一段,至少 4 个字);只是拿检索当依据时写 [[see:K1]]。下面没有的检索、片段里没有的话不要编造。" +) +#: 报告节点 claims 为 require_citation 时,拼在写作规则后面 +CLAIMS_RULE = ( + "这次要求每句结论都挂依据:陈述数据、比较、变化、原因的句子,句末必须写 [[see:…]](指标、查询编号、" + "知识库编号都可以)。没挂依据的结论句会被记为缺口,报告因此降档出具;过渡、连接性的话不用挂。" +) + #: 写作目录里每次查询最多列出几行、几列。更多的行照样能按行号引用 _PROMPT_ROWS = 20 _PROMPT_COLS = 12 +#: 表和字段:最多列几张查询用到的表、几张别的表、几个结果列;每条检索最多几段、每段多长 +_PROMPT_TABLES = 20 +_PROMPT_OTHER_TABLES = 40 +_PROMPT_RESULT_COLS = 40 +_PROMPT_HITS = 8 +_PROMPT_HIT_CHARS = 300 def _query_prompt(entry: dict[str, Any], snaps: _Snapshots, cells_allowed: bool) -> list[str]: @@ -1884,16 +2753,70 @@ def catalog_prompt(catalog: dict[str, Any], *, budget: int = 12000, cells_allowe "\n查询结果(这次出具不允许直接引用单元格,只能写在 [[see:Q1]] 这样的依据里):") for e in queries: lines.extend(_query_prompt(e, snaps, cells_allowed)) + if (index := snaps.entities(catalog)).active: + lines.extend(_entity_prompt(catalog, partial=index.partial)) retrievals = [e for e in catalog.values() if e.get("kind") == "retrieval"] if retrievals: - lines.append("\n知识库检索(只能写在 [[see:K1]] 这样的依据里):") - lines.extend(f"- {e['alias']}:{e.get('label')}" for e in retrievals) + lines.append(f"\n{QUOTE_RULES}") + for e in retrievals: + lines.extend(_retrieval_prompt(e, snaps)) text = "\n".join(lines) if len(text) > budget: text = text[:budget].rsplit("\n", 1)[0] + "\n…(目录太长,后面的省略了)" return text +def _entity_prompt(catalog: dict[str, Any], *, partial: bool = False) -> list[str]: + """表和字段在写作目录里的样子:查询用到的表连同字段,查询结果里的列,别的表只列名字。""" + tables = [e for e in catalog.values() if e.get("kind") == "table"] + fields: dict[str, list[str]] = {} + loose: list[str] = [] + for e in catalog.values(): + if e.get("kind") != "column": + continue + table = (e.get("locator") or {}).get("table") + if table: + fields.setdefault(table, []).append(str(e.get("name"))) + elif any(o.get("kind") == "result" for o in e.get("sources") or []): + loose.append(str(e.get("name"))) + lines = [f"\n{ENTITY_RULES}"] + used = [t for t in tables if t.get("queries")] + for t in used[:_PROMPT_TABLES]: + cols = fields.get(t["name"], []) + more = f" 等 {len(cols)} 个" if len(cols) > _PROMPT_COLS else "" + lines.append(f"- {t['name']}({'、'.join(t['queries'])} 用到)" + (f":{', '.join(cols[:_PROMPT_COLS])}{more}" + if cols else "")) + if loose: + more = f" 等 {len(loose)} 个" if len(loose) > _PROMPT_RESULT_COLS else "" + lines.append(f"- 查询结果里的列:{', '.join(loose[:_PROMPT_RESULT_COLS])}{more}") + others = [str(t["name"]) for t in tables if not t.get("queries")] + if others: + more = f" 等 {len(others)} 张" if len(others) > _PROMPT_OTHER_TABLES else "" + lines.append(f"- 其他表:{'、'.join(others[:_PROMPT_OTHER_TABLES])}{more}") + if partial: + lines.append("- 这个数据源的表太多,表结构快照只存了一部分:没列出的名字系统核对不了,只写你确定存在的") + return lines + + +def _retrieval_prompt(entry: dict[str, Any], snaps: _Snapshots) -> list[str]: + """一次检索在写作目录里的样子:编号、来源,以及能引原话的片段(压成一行,太长的截断)。""" + alias = entry["alias"] + lines = [f"- {alias}:{entry.get('label')}"] + hits, why = snaps.hits(alias, entry.get("artifact")) + if why: + return [*lines, f" ({why},只能写 [[see:{alias}]],不要引原话)"] + for i, hit in enumerate(hits[:_PROMPT_HITS]): + if not isinstance(hit, dict) or not isinstance(hit.get("content"), str): + continue + body = _clean(hit["content"]) + body = body if len(body) <= _PROMPT_HIT_CHARS else body[:_PROMPT_HIT_CHARS] + "…" + title = f"《{_clean(str(hit['title']))}》" if hit.get("title") else "" + lines.append(f" 片段 {i + 1}{title}:{body}") + if len(hits) > _PROMPT_HITS: + lines.append(f" …共 {len(hits)} 段,这里只列出前 {_PROMPT_HITS} 段") + return lines + + def describe_violations(violations: list[dict[str, Any]], *, limit: int = 20) -> str: """违规清单的文字版:给写作者的修复指令、给节点的报错,都用这一份。""" lines = [] @@ -1903,3 +2826,195 @@ def describe_violations(violations: list[dict[str, Any]], *, limit: int = 20) -> if len(violations) > limit: lines.append(f"- …另有 {len(violations) - limit} 处") return "\n".join(lines) + + +# -------------------------------------------------------------------------- +# 旧运行:按数值猜「可能的来源」 +# +# 没有契约、没有报告文档的旧答案,界面上仍想让人看看每个数大概是从哪来的。这件事一期就 +# 核对过:按数值匹配巧合极多(168 个数里 131 个能在快照里找到相同的值,大量是 0、1、2), +# 所以它只做降级展示:state 记 candidate,界面默认折叠,写明「猜测的来源,不能当证据」。 +# -------------------------------------------------------------------------- + +GUESS_SCHEMA = "agentlab.guess/1" +GUESS_NOTE = "猜测的来源,不能当证据:按数值在这次运行已封存的查询结果和口径卡里找相同的值,同值的巧合很多" +#: 每个数字最多给几个候选 +MAX_CANDIDATES = 3 + + +#: 同一个值的格往往成千上万(0、1):一个值最多看这么多个,够挑出候选、又不必逐个过一遍 +_GUESS_GROUP_SCAN = 16 +_MASK_SPLIT = re.compile(r"[,,、;;\n]+") + + +def _mask_names(raw: Any) -> set[str]: + """遮罩列名 → 小写的集合。列表和「a, b」这样的文字都认(和 data.engine.masked_columns 一样); + 别的形状当作没遮——但字符串一律当列名处理,宁可多遮,不能因为写法不同露出原值。""" + items = _MASK_SPLIT.split(raw) if isinstance(raw, str) else raw if isinstance(raw, (list, tuple, set)) else [] + return {str(c).strip().lower() for c in items if isinstance(c, (str, int)) and str(c).strip()} + + +def _guess_pool(sealed: list[Any], masked: dict[str, Any] | None + ) -> tuple[list[tuple[float, int, tuple[Any, ...]]], list[dict[str, Any]]]: + """已封存的条目摊成 (数值, 顺序号, 出处) 和每个条目的上下文。 + + 只取值、不渲染:一份几千行的快照有几万格,渲染只留给最后挑中的候选(每个数字至多 limit 个)。 + 顺序号按传入顺序、行、列递增,距离相同时靠它排。遮罩的列(快照记下的 mask_columns,加上调用方 + 按 artifact 交来的数据源现有遮罩)整列不进池子:候选的值、渲染、差值都会把原值带出去。 + """ + pool: list[tuple[float, int, tuple[Any, ...]]] = [] + contexts: list[dict[str, Any]] = [] + + def add(value: Any, where: tuple[Any, ...]) -> None: + if not is_number(value): + return + try: + number = float(value) + except OverflowError: # 几百位的整数:换不成浮点数,也不会有人按它写报告 + return + if math.isfinite(number): + pool.append((number, len(pool), where)) + + for entry in sealed or []: + if not isinstance(entry, dict) or not isinstance(entry.get("artifact"), str) or not entry["artifact"]: + continue + artifact, content = entry["artifact"], entry.get("content") + if not isinstance(content, dict): + continue + at = len(contexts) + context = {"entry": entry, "artifact": artifact, "content": content, + "extra": {k: entry[k] for k in ("node_id", "tool") if entry.get(k)}} + if entry.get("kind") == "metric_set" and isinstance(content.get("metrics"), list): + contexts.append(context) + for metric in content["metrics"]: + if isinstance(metric, dict) and metric.get("id"): + add(metric.get("value"), (at, metric)) + elif entry.get("kind") == "query" and isinstance(content.get("rows"), list): + contexts.append(context) + hidden = _mask_names(content.get("mask_columns")) | _mask_names((masked or {}).get(artifact)) + names = [str(c) for c in content.get("columns") or []] + types = content.get("column_types") if isinstance(content.get("column_types"), dict) else {} + # 同名的列按第一个算(和 locate_cell 一样),只算一次 + columns = [(name, names.index(name), types.get(name) if isinstance(types.get(name), str) else None) + for name in dict.fromkeys(names) if name.lower() not in hidden] + for r, record in enumerate(content["rows"]): + for name, index, kind in columns: + if isinstance(record, dict): + if name not in record: + continue + raw = record[name] + elif isinstance(record, (list, tuple)) and index < len(record): + raw = record[index] + else: + continue + add(cell_value(raw, kind), (at, r, name, kind, raw)) + return pool, contexts + + +def _guess_candidate(where: tuple[Any, ...], contexts: list[dict[str, Any]]) -> dict[str, Any] | None: + """池子里的一项 → 候选。单元格渲染不出来(非零却显示成 0)的返回 None,名额让给下一个。""" + context = contexts[where[0]] + artifact, content, extra = context["artifact"], context["content"], context["extra"] + if len(where) == 2: + metric = where[1] + mid = str(metric["id"]) + try: + rendered = render_metric(metric) + except RenderError: + rendered = MISSING + return {"kind": "metric", "ref": f"m:{mid}", "artifact": artifact, "locator": {"metric": mid}, + "eid": make_eid("metric", artifact, {"metric": mid}), "value": metric.get("value"), + "rendered": rendered, "name": metric.get("name") or mid, + **{k: v for k, v in (("caliber", content.get("caliber")), + ("version", content.get("caliber_version"))) if v}, + **{k: v for k, v in extra.items() if k == "node_id"}} + _, r, name, kind, raw = where + try: + rendered = render_cell(raw, kind=kind) + except RenderError: + return None + alias = context["entry"].get("alias") + return {"kind": "cell", **({"ref": f"{alias}.r{r}.{name}", "alias": alias} if alias else {}), + "artifact": artifact, "locator": {"row": r, "column": name}, "eid": cell_eid(artifact, r, name), + "value": cell_value(raw, kind), "rendered": rendered, **extra} + + +def _guesses(token: Any, values: list[float], pool: list[tuple[float, int, tuple[Any, ...]]], + contexts: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: + """一个数字的候选:书写精度以内相等的值(写成 8.7% 的也和比率 0.087 比),按距离、再按传入顺序。""" + if limit <= 0: + return [] + tolerance = _tolerance(token) + 1e-9 + best: dict[int, tuple[float, int]] = {} + targets = [(token.value, 1.0)] + ([(token.value / 100, 100.0)] if token.is_percent else []) + for target, scale in targets: + k = bisect.bisect_left(values, target - tolerance / scale) + hi = bisect.bisect_right(values, target + tolerance / scale) + while k < hi: + # 同一个值的一串(池子按值、顺序号排好了):差值一样,只有前几个有机会入选 + group_end = bisect.bisect_right(values, values[k], k, hi) + for j in range(k, min(group_end, k + _GUESS_GROUP_SCAN)): + value, order, _ = pool[j] + diff = abs(token.value - value * scale) + if diff <= tolerance and (order not in best or diff < best[order][0]): + best[order] = (diff, j) + k = group_end + out: list[dict[str, Any]] = [] + for _, (diff, j) in sorted(best.items(), key=lambda item: (item[1][0], item[0])): + candidate = _guess_candidate(pool[j][2], contexts) + if candidate is not None: + out.append({**candidate, "diff": round(diff, 10)}) + if len(out) >= limit: + break + return out + + +def guess_sources(text: str, sealed: list[Any], *, limit: int = MAX_CANDIDATES, + masked: dict[str, Any] | None = None) -> dict[str, Any]: + """旧答案里的每个数字,去调用方交来的已封存条目里按数值找候选。只做展示,不是证据。 + + sealed 的每一项:{kind: "query" | "metric_set", artifact, content, alias?, node_id?, tool?} + - query:content 是查询快照(columns / rows / column_types / mask_columns),候选是单元格,ref 形如 + Q1.r0.gmv(给了 alias 才有);文本列里的数字样子的字(编号 "00123")不按数比 + - metric_set:content 是口径卡的指标集,候选是指标,ref 形如 m:gmv + 这个函数不读库:调用方只把封存范围内的事件引用得到、取回时复验过哈希的工件交进来。 + + 遮罩:快照自己记下的 mask_columns,加上 masked({查询快照工件: 数据源现在的遮罩列},调用方按 + masked_columns(source.options) 给)里的列,一律不当候选——候选的 value、rendered、diff 都会带出 + 原值,写成 8,123 的数也会对上 8,123.45。列名不分大小写。 + + 数字的抽取和容差沿用出具校验(日期、ISO 周、行首序号不算数字;写 12.3 的容差是 0.05)。 + 每个数字最多 limit 个候选,按距离从近到远,距离相同的按传入顺序(条目、行、列),同样的输入 + 永远同样的输出。返回 {schema, mode: "legacy_text", note, markdown, segments, stats}: + segments 铺满原文,数字片段有候选的 state 为 candidate(带 candidates),没有的为 none。 + """ + text = text or "" + pool, contexts = _guess_pool(sealed, masked if isinstance(masked, dict) else None) + pool.sort(key=lambda item: (item[0], item[1])) + values = [item[0] for item in pool] + segments: list[dict[str, Any]] = [] + stats = {"numbers": 0, "guessed": 0, "unguessed": 0, "candidates": 0} + cursor = 0 + + def push(seg: dict[str, Any]) -> None: + segments.append({"id": f"s{len(segments)}", **seg}) + + for token in sorted(extract_numbers(text), key=lambda t: t.start): + if token.start < cursor: + continue + if cursor < token.start: + push({"kind": "text", "text": text[cursor:token.start], "span": [cursor, token.start], "state": "neutral"}) + found = _guesses(token, values, pool, contexts, limit) + seg = {"kind": "number", "text": text[token.start:token.end], "span": [token.start, token.end], + "state": "candidate" if found else "none"} + if found: + seg["candidates"] = found + push(seg) + stats["numbers"] += 1 + stats["guessed" if found else "unguessed"] += 1 + stats["candidates"] += len(found) + cursor = token.end + if cursor < len(text): + push({"kind": "text", "text": text[cursor:], "span": [cursor, len(text)], "state": "neutral"}) + return {"schema": GUESS_SCHEMA, "mode": "legacy_text", "note": GUESS_NOTE, "markdown": text, + "segments": segments, "stats": stats} diff --git a/backend/app/engine/governance.py b/backend/app/engine/governance.py index ec44319c..01c29796 100644 --- a/backend/app/engine/governance.py +++ b/backend/app/engine/governance.py @@ -1,11 +1,22 @@ from __future__ import annotations -from typing import Any +import re +from collections import defaultdict +from typing import Any, Callable from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession -from app.engine.schema import GraphSpec, NodeType, ValidationResult, type_label, validate_graph +from app.engine.schema import ( + GraphNode, + GraphSpec, + NodeType, + ValidationResult, + _caliber_inputs, + claims_problem, + type_label, + validate_graph, +) # 口径升版的三种合法处置。没有默认值是刻意的:升版最容易静默翻车的地方 # 就是"上周和这周的数其实不可比,但报表看起来一切正常"。 @@ -26,7 +37,7 @@ def lint_for_publish(spec: GraphSpec, *, level: str) -> ValidationResult: 错误会挡住发布。 每条问题带机器编号(code):发布前自动修复(engine/autofix.py)按它认问题、出修复, - 文案可以改,编号不能随便改。 + 文案可以改,编号不能随便改。证据门禁 G1–G5 见 _lint_evidence。 """ result = ValidationResult() strict = level == "governed" @@ -88,6 +99,8 @@ def lint_for_publish(spec: GraphSpec, *, level: str) -> ValidationResult: flag(f"{who}打开了「允许联网」:受管模板建议把取数收敛到「调用工具」节点" "(那里有取数快照)", code="governed.code_network", node_id=node.id, field="network") + _lint_evidence(spec, flag, strict=strict) + if strict and not has_contract_output: result.add( "受管模板至少要有一个「成果 / 出具」节点声明出具契约,否则三档出具无从谈起", @@ -101,7 +114,28 @@ def publish_issues(spec: GraphSpec, *, level: str) -> list[Any]: 发布、发布前检查、自动修复复核都走这一个口径——检查说能发的,真发布时不会被拦。 """ - return [*validate_graph(spec).issues, *lint_for_publish(spec, level=level).issues] + gate = lint_for_publish(spec, level=level).issues + return [*_not_repeated(validate_graph(spec).issues, gate, spec), *gate] + + +#: validate 里和门禁 G4、G5 说同一件事的两条提示(schema.evidence_issues) +_REPEATS = frozenset({"evidence.caliber_compute_input", "evidence.cite_fields_off"}) + + +def _not_repeated(issues: list[Any], gate: list[Any], spec: GraphSpec) -> list[Any]: + """门禁已经点了名的,validate 那条同义的提示不再列:门禁那条带编号、带修复,一处问题只说一遍。 + + G5 落在 agent 上,validate 那条也落在 agent 上;G4 落在喂数的沙箱代码上,validate 那条落在口径卡读它的 + 那条指标上,按口径卡的输入对回去。没被门禁覆盖的(比如不给口径卡供数的 agent)照旧提示。 + """ + cited = {i.node_id for i in gate if i.code == "governed.caliber_agent_cite_fields"} + compute = {i.node_id for i in gate if i.code == "governed.caliber_compute_input"} + covered = {("evidence.cite_fields_off", nid, "output_schema") for nid in cited} + for card in spec.nodes: + if card.type == NodeType.METRICS and compute: + covered |= {("evidence.caliber_compute_input", card.id, f"metrics[{i}].expression") + for i, producer in _caliber_inputs(card, spec) if producer.id in compute} + return [i for i in issues if i.code not in _REPEATS or (i.code, i.node_id, i.field) not in covered] def _lint_contract( @@ -153,6 +187,342 @@ def _lint_contract( "受管模板建议把出具契约设为 strict:非 strict 下未回指的数字只降档不拦截", code="contract.strict_off", node_id=node_id, field="contract.strict", ) + claims = contract.get("claims") + if strict and isinstance(claims, dict) and claims.get("on_uncited") == "ignore": + # 受管级别的正式运行把 ignore 当 degrade(io._claims),要求没有降低;只是写法看上去像放宽了 + flag( + "出具契约的 claims 写了 on_uncited: ignore,但受管级别的正式运行里没挂依据的结论句照样计入缺口" + "(按 degrade 处理),这个写法不会生效。改成 degrade,或者更严的 withhold", + code="contract.claims_ignored", node_id=node_id, field="contract.claims", + ) + + +# -------------------------------------------------------------------------- +# 证据门禁 G1–G5:受管出具要在结构上保证「报告里的每个数都点得开出处」 +# +# G1 带契约的出口核对报告撰写节点的文档,成果字段的文字只能来自报告撰写节点 +# G2 模型写的文字不绕过报告撰写节点直接流进出口(从图上追) +# G3 报告撰写节点写明 numbers: strict、on_violation: fail、claims: require_citation +# G4 口径卡的输入不来自计算角色的沙箱代码 +# G5 给口径卡供数的 agent 配 output_schema 并开 cite_fields(模型调用节点不能给口径卡供数) +# +# 只看图和配置,不看运行结果。只在发布和发布前检查时起作用:已经发布的版本照常运行, +# 下次发布才按这些规则查。受管级别挡住发布,已发布级别只给警告。 +# -------------------------------------------------------------------------- + +#: 报告撰写节点在受管模板里要写明的三项:(配置键, 要求的值, 为什么) +REPORT_POLICY = ( + ("numbers", "strict", "没有出处的裸数字判为违规"), + ("on_violation", "fail", "重写之后还违规就让节点失败,不带着问题出具"), + ("claims", "require_citation", "没挂引用的结论句计入缺口,按出具档位降档"), +) + +#: 产出本身就是证据、或者由系统确定地算出来的节点。成果字段取它们的值,不算「别的节点写的字」 +_TRACEABLE = (NodeType.REPORT, NodeType.METRICS, NodeType.INPUT, NodeType.TOOL, NodeType.RETRIEVE) +_WRITERS = (NodeType.LLM, NodeType.AGENT, NodeType.SUPERVISOR) +#: 往对话里写消息的节点(emit_message 没关时),「最后一条消息」就是它们里最后跑的那个写的 +_EMITTERS = (*_WRITERS, NodeType.REPORT) +#: 把上游的值原样或加工后往下传的节点:追文字的来路要穿过它们 +_RELAYS = (NodeType.TRANSFORM, NodeType.CODE, NodeType.VALIDATE, NodeType.HUMAN, NodeType.LOOP) +#: 往回追口径卡的输入时穿过的整形节点。沙箱代码不穿过:它自己就是 G4 要判的 +_RESHAPERS = (NodeType.TRANSFORM, NodeType.VALIDATE) +#: 只决定走哪条路、不把值往下带的配置 +_CONTROL_KEYS = frozenset({"skip_if", "condition", "cases"}) + +_TEMPLATE = re.compile(r"\{\{(.*?)\}\}", re.S) +# 模板和表达式里对值的引用。nodes.…、vars.<名字>… 落得到具体节点;last_message、messages +# 是最后写消息的那个节点写的;output、loops(和整个 nodes / vars)说不准来自谁 +_REF = re.compile( + r"(?nodes|vars|last_message|messages|output|loops)(?![\w-])" + r"(?P(?:\s*(?:\.\s*[\w-]+|\[\s*(?:'[^']*'|\"[^\"]*\"|-?\d+)\s*\]))*)") +_SEGMENT = re.compile(r"\.\s*([\w-]+)|\[\s*(?:'([^']*)'|\"([^\"]*)\"|(-?\d+))\s*\]") + +#: (根, 名字, 往里取的路径)。last_message 这类没有名字 +Ref = tuple[str, str, tuple[str, ...]] +_LAST: Ref = ("last_message", "", ()) + + +def _refs(text: str, *, template: bool) -> list[Ref]: + out: list[Ref] = [] + for piece in [m.group(1) for m in _TEMPLATE.finditer(text)] if template else [text]: + for m in _REF.finditer(piece): + path = tuple(next(g for g in s.groups() if g is not None) for s in _SEGMENT.finditer(m.group("path"))) + root = m.group("root") + if root in ("nodes", "vars") and path: + out.append((root, path[0], path[1:])) + else: + out.append((root, "", path)) + return out + + +def _strings(value: Any) -> list[str]: + """配置里所有带 {{ }} 的字符串。""" + if isinstance(value, str): + return [value] if "{{" in value else [] + if isinstance(value, dict): + return [s for v in value.values() for s in _strings(v)] + if isinstance(value, list): + return [s for v in value for s in _strings(v)] + return [] + + +def _template_refs(value: Any) -> list[Ref]: + return [r for text in _strings(value) for r in _refs(text, template=True)] + + +def _flow_refs(node: GraphNode) -> list[Ref]: + """这个节点的产出读了哪些值。判断走哪条路的条件不算;校验节点没写 source 时校验的是最后一条消息。""" + out: list[Ref] = [] + for key, value in node.config.items(): + if key in _CONTROL_KEYS: + continue + out += _refs(value, template=False) if key == "expression" and isinstance(value, str) \ + else _template_refs(value) + if node.type == NodeType.VALIDATE and node.config.get("source") in (None, ""): + out.append(_LAST) + return out + + +def _effective(spec: GraphSpec, node: GraphNode, key: str, default: Any = None) -> Any: + """和运行时 NodeContext.cfg 同一个规则:节点上没写就取图级 defaults。""" + value = node.config.get(key) + return spec.defaults.get(key, default) if value in (None, "") else value + + +class _Graph: + """一张图上反复要用的查找:谁产出了某个值、最后一条消息是谁写的。""" + + def __init__(self, spec: GraphSpec) -> None: + self.spec = spec + self.nodes = spec.node_map() + self.parents: dict[str, list[str]] = defaultdict(list) + for e in spec.edges: + self.parents[e.target].append(e.source) + + def producers(self, root: str, name: str) -> list[GraphNode]: + """nodes. 就是那个节点;vars. 是用 assign_to 写它的节点、同名的入口字段、 + 每轮注入它的循环。""" + if not name or root not in ("nodes", "vars"): + return [] + if root == "nodes": + return [self.nodes[name]] if name in self.nodes else [] + out = [] + for n in self.spec.nodes: + cfg = n.config + if n.type == NodeType.INPUT: + writes = {str(f.get("name") or "").strip() for f in cfg.get("fields") or [] if isinstance(f, dict)} + elif n.type == NodeType.LOOP and cfg.get("mode", "foreach") == "foreach": + item = str(cfg.get("item_var") or "item").strip() + writes = {item, f"{item}_index"} + else: + writes = set() + if name in writes or str(cfg.get("assign_to") or "").strip() == name: + out.append(n) + return out + + def last_writers(self, node_id: str) -> list[GraphNode]: + """到这个节点时,最后一条消息可能是谁写的:沿连线往回走,每条路上遇到的第一个写消息的节点。""" + found: set[str] = set() + seen = {node_id} + stack = list(self.parents[node_id]) + while stack: + cur = stack.pop() + if cur in seen or cur not in self.nodes: + continue + seen.add(cur) + n = self.nodes[cur] + if n.type in _EMITTERS and _effective(self.spec, n, "emit_message", True): + found.add(cur) + continue + stack.extend(self.parents[cur]) + return [n for n in self.spec.nodes if n.id in found] + + def sources(self, consumer: str, ref: Ref) -> list[GraphNode]: + root, name, _ = ref + return self.last_writers(consumer) if root in ("last_message", "messages") else self.producers(root, name) + + +def _who(node: GraphNode) -> str: + return f"「{node.title}」({type_label(node.type)})" + + +def _traceable(node: GraphNode) -> bool: + return node.type in _TRACEABLE or (node.type == NodeType.CODE and node.config.get("evidence_role") == "source") + + +def _model_text(node: GraphNode, ref: Ref) -> bool: + """顺着这个引用拿到的是不是模型写的话。开了 cite_fields 的 agent,assign_to(和 nodes..data) + 拿到的是逐格核对过的字段,那是数据;它的原话、最后一条消息照样是模型写的。""" + if node.type in (NodeType.LLM, NodeType.SUPERVISOR): + return True + if node.type != NodeType.AGENT: + return False + root, _, path = ref + cited = bool(node.config.get("output_schema")) and node.config.get("cite_fields") is True + return not (cited and (root == "vars" or path[:1] == ("data",))) + + +def _exit_refs(node: GraphNode) -> list[tuple[str, str | None, list[Ref]]]: + """出口交出去的每个字段:(字段路径, 字段名, 引用)。没配字段时交的是最后一条消息(io.run_output)。""" + fields = node.config.get("fields") + if not fields: + return [("fields", None, [_LAST])] + return [(f"fields[{i}].value", str(f["name"]), _template_refs(f.get("value", ""))) + for i, f in enumerate(fields if isinstance(fields, list) else []) + if isinstance(f, dict) and f.get("name")] + + +def _untraceable_sources(graph: _Graph, exit_id: str, refs: list[Ref]) -> list[str]: + """一个成果字段里核对不了出处的来源,一个来源一句(「叙述」(模型调用)、最后一条消息…)。""" + out: list[str] = [] + for ref in refs: + root, name, _ = ref + if root in ("output", "loops") or (root in ("nodes", "vars") and not name): + out.append(f"{root} 里的值(说不准来自哪个节点)") + elif root in ("last_message", "messages"): + out += [f"最后一条消息({_who(w)}写的)" for w in graph.last_writers(exit_id) if w.type != NodeType.REPORT] + else: + # 开了 cite_fields 的 agent 交的字段逐格核对过,是数据不是模型写的话——和 G2(_model_text)同一口径 + out += [_who(p) for p in graph.producers(root, name) + if not _traceable(p) and (p.type != NodeType.AGENT or _model_text(p, ref))] + return list(dict.fromkeys(out)) + + +def exit_text_fields(spec: GraphSpec, exit_node: GraphNode, graph: _Graph | None = None, + ) -> list[tuple[str, str | None, list[str]]]: + """G1:出口里文字不是来自报告撰写节点的字段:(字段路径, 字段名, 来源)。字段名为 None 是没配字段。 + + 发布前自动修复(autofix)按同一份结果改字段,门禁和修复认的是同一批字段。 + """ + graph = graph or _Graph(spec) + out = [] + for field, name, refs in _exit_refs(exit_node): + if bad := _untraceable_sources(graph, exit_node.id, refs): + out.append((field, name, bad)) + return out + + +def _text_leaks(graph: _Graph, exit_node: GraphNode) -> dict[str, tuple[GraphNode, set[tuple[str, ...]]]]: + """流进这个出口、没经过报告撰写节点的模型原话:写字的节点 id → (节点, 途经的节点标题)。 + + 途经为空的是出口字段直接取的。沿着模板和表达式里的引用往回追,穿过整形、沙箱代码、校验、人工审批、 + 循环;碰到报告撰写节点就停——模型写的话进了报告,就只是写作者的素材。 + """ + found: dict[str, tuple[GraphNode, set[tuple[str, ...]]]] = {} + seen: set[str] = set() + todo: list[tuple[str, Ref, tuple[str, ...]]] = [ + (exit_node.id, ref, ()) for _, _, refs in _exit_refs(exit_node) for ref in refs] + while todo: + consumer, ref, via = todo.pop(0) + for source in graph.sources(consumer, ref): + if source.type == NodeType.REPORT or source.id == exit_node.id: + continue + if source.type in _WRITERS: + if _model_text(source, ref): + found.setdefault(source.id, (source, set()))[1].add(via) + elif source.type in _RELAYS and source.id not in seen: + seen.add(source.id) + todo += [(source.id, r, (*via, source.title)) for r in _flow_refs(source)] + return found + + +def _card_feeders(graph: _Graph, card: GraphNode) -> list[GraphNode]: + """口径卡的输入是谁产出的:穿过整形、校验节点往回追到真正的来源。""" + out: dict[str, GraphNode] = {} + seen: set[str] = set() + todo = [producer for _, producer in _caliber_inputs(card, graph.spec)] + while todo: + node = todo.pop(0) + if node.id in seen or node.id == card.id: + continue + seen.add(node.id) + if node.type in _RESHAPERS: + todo += [s for ref in _flow_refs(node) for s in graph.sources(node.id, ref)] + else: + out.setdefault(node.id, node) + return list(out.values()) + + +def _lint_evidence(spec: GraphSpec, flag: Callable[..., None], *, strict: bool) -> None: + graph = _Graph(spec) + outputs = [n for n in spec.nodes if n.type == NodeType.OUTPUT] + contracted = {n.id for n in outputs if isinstance(n.config.get("contract"), dict) and n.config["contract"]} + + # G1 + for node in outputs: + if node.id not in contracted: + continue + contract = node.config["contract"] + # 两样都没写的由 contract.report_from_missing 报,这里只管「写了叙述、没写 report_from」 + if contract.get("report_from") in (None, "") and str(contract.get("narrative") or "").strip(): + flag(f"{_who(node)}的出具契约用的是旧的叙述模式(narrative),没有声明 report_from:数字只能按数值" + "回头匹配,出处可能不唯一。受管出具核对「报告撰写」节点产出的文档,每个数都有唯一的出处——把 " + "report_from 指向出口上游的报告撰写节点", + code="governed.report_from_required", node_id=node.id, hard=True, field="contract.report_from") + for field, name, bad in exit_text_fields(spec, node, graph): + head = (f"{_who(node)}的成果字段「{name}」取的是{'、'.join(bad)}的产出" if name is not None + else f"{_who(node)}没有配成果字段,交出去的是{'、'.join(bad)}") + flag(f"{head}:受管出具里给人看的文字只能来自「报告撰写」节点,别的节点写的字核对不了出处。" + + ("把它改成取报告撰写节点的正文" if name is not None else "给出口配一个字段,取报告撰写节点的正文"), + code="governed.exit_text_source", node_id=node.id, hard=True, field=field) + + # G2:每个出口都算。受管级别是 error,已发布级别是警告——和别的 G 规则一样,级别只管轻重,不管范围 + flagged: set[str] = set() + for node in outputs: + for writer, vias in _text_leaks(graph, node).values(): + indirect = sorted(v for v in vias if v) + # 带契约的出口里字段直接取的,G1 已经点名了那个字段 + if writer.id in flagged or (node.id in contracted and not indirect): + continue + flagged.add(writer.id) + via = f"(经过{'、'.join(f'「{t}」' for t in indirect[0])})" if indirect else "" + flag(f"{_who(writer)}写的文字没经过「报告撰写」节点,就流进了出口「{node.title}」{via}:模型写的话不算" + "证据,里面的数字、表名都核对不了出处。改由报告撰写节点来写(它只写引用标记,数字由系统渲染)," + "出口取报告的正文", + code="governed.text_bypass", node_id=writer.id, hard=True) + + # G3 + for node in spec.nodes: + if node.type != NodeType.REPORT: + continue + loose = [] + for key, want, why in REPORT_POLICY: + value = _effective(spec, node, key) + if key == "claims" and claims_problem(value): + continue # 写了不支持的值(比如 judge),validate 的 report.claims_invalid 报 + if value != want: + now = "现在没写" if value in (None, "") else f"现在是 {value}" + loose.append((key, f"{key}: {want}({why};{now})")) + if loose: + flag(f"{_who(node)}要写明 {';'.join(text for _, text in loose)}。受管模板的报告撰写节点按最严的规则自查", + code="governed.report_policy", node_id=node.id, hard=True, field=loose[0][0]) + + # G4、G5 + feeders: dict[str, tuple[GraphNode, list[str]]] = {} + for card in spec.nodes: + if card.type == NodeType.METRICS: + for feeder in _card_feeders(graph, card): + feeders.setdefault(feeder.id, (feeder, []))[1].append(f"「{card.title}」") + for feeder, cards in feeders.values(): + to = f"口径卡{'、'.join(dict.fromkeys(cards))}" + cfg = feeder.config + if feeder.type == NodeType.CODE and cfg.get("evidence_role") != "source": + role = "是 compute" if cfg.get("evidence_role") else "没写,按计算算" + flag(f"{_who(feeder)}的产出喂给了{to},而它的角色是计算(evidence_role {role}):沙箱里算出来的数核对" + "不了出处。它其实是在取数的话,把 evidence_role 标成 source;是在做计算的话,把计算挪进口径卡的表达式", + code="governed.caliber_compute_input", node_id=feeder.id, hard=True, field="evidence_role") + elif feeder.type == NodeType.AGENT and not cfg.get("output_schema"): + flag(f"{_who(feeder)}给{to}供数,却没有配 output_schema:它交出来的是一段自由文字,口径卡读到的数核对不了" + "出处。给它配 output_schema 并开 cite_fields,每个字段都核对到查询结果里的那一格", + code="governed.caliber_agent_schema", node_id=feeder.id, hard=True, field="output_schema") + elif feeder.type == NodeType.AGENT and cfg.get("cite_fields") is not True: + flag(f"{_who(feeder)}给{to}供数,配了 output_schema 却没开 cite_fields:字段没有逐个核对到查询结果," + "口径卡读到的仍是模型报的数。打开 cite_fields", + code="governed.caliber_agent_cite_fields", node_id=feeder.id, hard=True, field="cite_fields") + elif feeder.type == NodeType.LLM: + flag(f"{_who(feeder)}给{to}供数:模型调用节点不调工具,写出来的数没有出处。改由 Agent(配 output_schema " + "并开 cite_fields)或「调用工具」节点取数", + code="governed.caliber_model_input", node_id=feeder.id, hard=True) async def unresolved_caliber_upgrades( diff --git a/backend/app/engine/nodes/io.py b/backend/app/engine/nodes/io.py index 3cb24daa..e20d0da0 100644 --- a/backend/app/engine/nodes/io.py +++ b/backend/app/engine/nodes/io.py @@ -57,12 +57,13 @@ async def run_output(state: GraphState, ctx: NodeContext) -> dict[str, Any]: reports = _report_fields(result, mapping, state, ctx) contract = ctx.cfg("contract") if contract: - cells = True + cells, governed = True, False if isinstance(contract, dict) and contract.get("report_from") not in (None, ""): from app.engine.governance import cells_allowed, governed_formal - cells = cells_allowed(contract, governed=await governed_formal(ctx.run.run_id)) - result["_issuance"] = _apply_contract(contract, state, ctx, reports=reports, cells=cells) + governed = await governed_formal(ctx.run.run_id) + cells = cells_allowed(contract, governed=governed) + result["_issuance"] = _apply_contract(contract, state, ctx, reports=reports, cells=cells, governed=governed) evidence = _evidence_of(reports, contract) if evidence: result["_evidence"] = evidence @@ -142,7 +143,7 @@ def _evidence_of(reports: list[dict[str, Any]], contract: Any) -> dict[str, Any] def _apply_contract( contract: dict[str, Any], state: GraphState, ctx: NodeContext, - *, reports: list[dict[str, Any]] | None = None, cells: bool = True, + *, reports: list[dict[str, Any]] | None = None, cells: bool = True, governed: bool = False, ) -> dict[str, Any]: from datetime import datetime, timezone @@ -195,7 +196,8 @@ def _apply_contract( trace = None cited: dict[str, Any] = {} if citations: - cited = _check_citations(str(report_from), contract, state, ctx, reports or [], cells=cells) + cited = _check_citations(str(report_from), contract, state, ctx, reports or [], cells=cells, + governed=governed) gaps.extend(cited["gaps"]) unmatched = cited["unmatched"] else: @@ -235,6 +237,8 @@ def _apply_contract( strict=bool(contract.get("strict")), gaps=gaps, unresolved=cited.get("unresolved"), + uncited_claims=cited.get("uncited"), + claims_policy=cited.get("claims_policy"), ) if citations: @@ -285,26 +289,31 @@ def _apply_contract( #: 文档本身对不上(不是某个数没出处):这次复核没法完成,记 gap _INTEGRITY = {"bad_schema", "segment_mismatch", "structural_text", "render_mismatch", "eid_mismatch", "state_mismatch"} +#: 出具声明里列多少个可疑名字、多少句没挂依据的结论(计数照实,列表只是给人看的样本) +_LISTED = 20 def _check_citations( report_from: str, contract: dict[str, Any], state: GraphState, ctx: NodeContext, - reports: list[dict[str, Any]], *, cells: bool = True, + reports: list[dict[str, Any]], *, cells: bool = True, governed: bool = False, ) -> dict[str, Any]: - """独立复核报告文档:{matched, unmatched, unresolved, gaps, doc_artifact, stats}。 + """独立复核报告文档:{matched, unmatched, unresolved, gaps, doc_artifact, stats, claims_policy, uncited, + claims, entities}。 不信报告节点自己的统计:文档按 id 从工件库取回(取回时复验哈希),目录按报告节点 同一套参数从状态里重建(不用文档里存的那份),每个带引用的片段重新解析、重新渲染、 逐字比对。一个配错的报告节点绕不过这里。 cells:受管级别的正式运行、契约没声明 cells 时为 False,单元格引用一律判解析不了。 + governed:受管级别的正式运行。可疑实体(名字哪里都找不到)只在这时按缺口降档,别处只标注。 """ from app.core.artifact_store import load - from app.engine.evidence import iter_units, resolve_ref, verify_doc - from app.engine.nodes.report import report_catalog + from app.engine.evidence import iter_units, ledger_enabled, resolve_ref, verify_doc + from app.engine.nodes.report import report_catalog, report_entities out: dict[str, Any] = {"matched": [], "unmatched": [], "unresolved": [], "gaps": [], - "doc_artifact": None, "stats": None} + "doc_artifact": None, "stats": None, "claims_policy": None, "uncited": None, + "claims": None, "entities": None} gaps = out["gaps"] spec = ctx.run.spec node = spec.node_map().get(report_from) @@ -332,10 +341,12 @@ def _check_citations( return out catalog = report_catalog(state, spec, node) - checked = verify_doc(doc, catalog, allow_numbers=contract.get("allow_numbers"), cells_allowed=cells) + checked = verify_doc(doc, catalog, allow_numbers=contract.get("allow_numbers"), cells_allowed=cells, + entities=report_entities(spec, node) == "link") out["stats"] = checked["stats"] broken: list[str] = [] flagged: set[str] = set() + suspicious: dict[str, list[dict[str, Any]]] = {"unknown_entity": [], "unverified_entity": []} for v in checked["violations"]: where = {k: v[k] for k in ("segment", "unit") if v.get(k)} if v.get("span"): @@ -346,11 +357,27 @@ def _check_citations( out["unmatched"].append({"token": v.get("text", ""), "context": v.get("context", ""), **where}) elif v["code"] == "unresolved_ref": out["unresolved"].append({"ref": v.get("ref", ""), "message": v["message"], **where}) + elif v["code"] in suspicious: + # 写 [[see:t:x]] 时违规上没有正文里的字,名字取自引用 + name = v.get("text") or str(v.get("ref") or "").partition(":")[2] + suspicious[v["code"]].append({"name": name, **({"ref": v["ref"]} if v.get("ref") else {}), **where}) elif v["code"] in _INTEGRITY: broken.append(v["message"]) if broken: more = f"等 {len(broken)} 处" if len(broken) > 1 else "" gaps.append(f"报告「{node.title}」的文档没通过复核:{broken[0]}{more}") + unknown, unverified = suspicious["unknown_entity"], suspicious["unverified_entity"] + if unknown and governed: + # 用户拍板:可疑实体在受管级别的正式出具里按「有缺口」降档,不拦截——反引号里的一个业务词 + # 也可能被当成名字,为它不予出具太重;探索运行、已发布级别只标注 + names = "".join(f"「{n}」" for n in dict.fromkeys(u["name"] for u in unknown[:5])) + more = "等" if len(unknown) > 5 else "" + gaps.append(f"报告「{node.title}」里有 {len(unknown)} 处可疑实体({names}{more}):本次运行的表结构、查询、" + "结果列里都没有这些名字,可能是编造的") + if unknown or unverified: + # 表结构快照不全时核对不了的名字(unverified)在哪个级别都只标注 + out["entities"] = {"unknown": unknown[:_LISTED], "unverified": unverified[:_LISTED], + "counted": governed and bool(unknown)} # 老前端要用的逐个数字出处,从复核通过的数字片段拼出来 for _, unit in iter_units(doc): @@ -375,10 +402,8 @@ def _check_citations( out["matched"].append({**hit, "segment": seg["id"], "unit": unit["id"], "start": start, "end": end, "span": [start, end], "eid": cite["eid"]}) - claims = contract.get("claims") - policy = claims.get("policy") if isinstance(claims, dict) else claims - if policy not in (None, "", "off"): - gaps.append("契约声明了结论句检查(claims),这一版还不支持,结论句没有核对") + _claims(out, contract, payload, evidence_on=ledger_enabled(ctx.run), governed=governed, + uncited=checked.get("uncited") or []) mine = next((r for r in reports if r["node"].id == report_from), None) edited = mine["edited"] if mine else [] @@ -389,6 +414,48 @@ def _check_citations( return out +#: 没挂依据的结论句怎么处置,从松到严。认不出的写法按 degrade 算:写错一个词不能让缺口悄悄消失 +_ON_UNCITED = ("ignore", "degrade", "withhold") + + +def _claims(out: dict[str, Any], contract: dict[str, Any], payload: dict[str, Any], *, evidence_on: bool, + governed: bool = False, uncited: list[dict[str, Any]]) -> None: + """结论句策略:没挂依据的结论句按 claims_policy 交给 decide_tier 判档。 + + 两处可以要求挂依据:报告节点写作时记下的 claims(节点产出里的,写作提示按它要求过),和契约自己写的 + require_citation(对象形式可以另写 on_uncited)。两处都写了取更严的那个——契约只能收紧、不能放松: + 报告节点要求了,契约写 ignore 或 off 也照旧计入缺口,写 withhold 就收紧成不予出具。受管级别的正式运行 + 里 ignore 一律当 degrade:没有哪道门禁查契约里的 claims,不能让它把用户要的底线拉低。 + + 数的是这里重新核对出来的 uncited,不信文档里记的 cites。升级前写的报告(节点产出里没有 claims 这个键, + 包括跨着升级还没跑完的运行)一律不管,契约写了也照旧记「这一版还不支持」。judge(裁判模型)是后续版本 + 的事,契约写了照实记缺口。 + """ + written = payload.get("claims") if isinstance(payload, dict) else None + # 报告节点升级后才在产出里记 claims(off 也记):没有这个键就是升级前写的报告 + upgraded = evidence_on and isinstance(payload, dict) and "claims" in payload + declared = contract.get("claims") + name = declared.get("policy") if isinstance(declared, dict) else declared + asks: list[str] = [] + if upgraded and written == "require_citation": + asks.append("degrade") + if upgraded and name == "require_citation": + wanted = declared.get("on_uncited") if isinstance(declared, dict) else None + asks.append(wanted if wanted in _ON_UNCITED else "degrade") + elif name not in (None, "", "off"): + out["gaps"].append("契约声明了结论句检查(claims),这一版还不支持,结论句没有核对") + if not asks: + return + on_uncited = max(asks, key=_ON_UNCITED.index) + if governed and on_uncited == "ignore": + on_uncited = "degrade" + policy: Any = "require_citation" if on_uncited == "degrade" else \ + {"policy": "require_citation", "on_uncited": on_uncited} + out["claims_policy"], out["uncited"] = policy, uncited + out["claims"] = {"policy": "require_citation", "uncited_claims": len(uncited), "uncited": uncited[:_LISTED], + **({"on_uncited": on_uncited} if on_uncited != "degrade" else {})} + + def _declare_citations( tier: str, cited: dict[str, Any], *, calibers: list[dict[str, str]], metrics: list[dict[str, Any]], gaps: list[str], missing_required: list[str], missing_expected: list[str], report_from: str, @@ -411,7 +478,11 @@ def _declare_citations( "stats": cited["stats"], "gaps": gaps, "declared_at": declared_at.isoformat(), + # 结论句策略和可疑实体:只在用上时才有这两个键,升级前的运行和没用上的契约形状不变 + **({"claims": cited["claims"]} if cited.get("claims") else {}), + **({"entities": cited["entities"]} if cited.get("entities") else {}), } + entities = cited.get("entities") ctx.emit( EventType.ISSUANCE, mode="citations", @@ -426,6 +497,13 @@ def _declare_citations( metrics_checked=len(metrics), matched_numbers=len(cited["matched"]), matched=cited["matched"][:50], + # on_uncited 只在不是默认的 degrade 时才有(收紧成 withhold、探索运行的契约写了 ignore) + **({"claims": {k: cited["claims"][k] for k in ("policy", "uncited_claims", "on_uncited") + if k in cited["claims"]}} if cited.get("claims") else {}), + # 计数取复核的统计(声明里的名单最多列 _LISTED 个) + **({"entities": {"unknown": (cited["stats"] or {}).get("unknown_entities", len(entities["unknown"])), + "unverified": (cited["stats"] or {}).get("unverified_entities", len(entities["unverified"])), + "counted": entities["counted"]}} if entities else {}), ) return issuance diff --git a/backend/app/engine/nodes/llm.py b/backend/app/engine/nodes/llm.py index 45569a8a..e31bf063 100644 --- a/backend/app/engine/nodes/llm.py +++ b/backend/app/engine/nodes/llm.py @@ -18,7 +18,14 @@ from app.db.base import SessionLocal from app.db.models import Skill from app.engine.approval import read_decision from app.engine.context import NodeContext, NodeError -from app.engine.evidence import cell_eid, ledger_enabled, make_eid, next_exec, query_entry_fields +from app.engine.evidence import ( + cell_eid, + ledger_enabled, + make_eid, + next_exec, + query_entry_fields, + schema_ledger_entry, +) from app.engine.expressions import CellError, cell_value, column_kind, locate_cell, numeric_text, same_value from app.engine.guards import FALLBACK_CONTEXT, Guard, legacy_hint, node_limits from app.engine.replay import ask, once @@ -503,6 +510,8 @@ async def run_agent(state: GraphState, ctx: NodeContext) -> dict[str, Any]: preview=content[:2000], artifact=snapshot_id, **({"query_artifact": query["artifact"]} if query else {}), + # 查询当时数据源的表结构快照:和查询快照一样落在封存范围内,证据接口从这里认它 + **({"schema_artifact": query["schema_artifact"]} if query and query.get("schema_artifact") else {}), **extra, ) outcome: dict[str, Any] = {"content": content, "ok": ok, "duration_ms": elapsed} @@ -821,10 +830,17 @@ def _record_call(node_id: str, exec_no: int, call_id: str, name: str, args: dict return content alias = f"Q{len(queries) + 1}" record.update(alias=alias, artifact=artifact) + # 表结构快照:同一件在本节点这次执行里只记一条 schema 条目。快照是内容寻址的,重放时读到的是同一份 + schema = fields.get("schema_artifact") + seen = schema and any(e.get("kind") == "schema" and e.get("artifact") == schema for e in entries) + frozen = None if not schema or seen else schema_ledger_entry(schema, node_id=node_id, exec_no=exec_no) + shaped = {"schema_artifact": schema, "tables": fields["tables"]} if seen or frozen else {} entries.append({"kind": "query", "node_id": node_id, "exec": exec_no, "artifact": artifact, **({"via": snapshot} if snapshot else {}), "call_id": call_id, "tool": name, "source": fields["source"], "columns": fields["columns"], "rows": fields["rows"], - "truncated": fields["truncated"]}) + "truncated": fields["truncated"], **shaped}) + if frozen: + entries.append(frozen) queries.append({"alias": alias, "call_id": call_id, "tool": name, "sql": str(args.get("sql") or ""), "artifact": artifact, "via": snapshot, "columns": fields["columns"], "rows": data.get("rows") or [], "truncated": fields["truncated"]}) diff --git a/backend/app/engine/nodes/report.py b/backend/app/engine/nodes/report.py index 99b55f45..8881d6e8 100644 --- a/backend/app/engine/nodes/report.py +++ b/backend/app/engine/nodes/report.py @@ -11,6 +11,12 @@ 5. 收尾:文档落成 report_doc 工件,report.checked 事件进封存;仍然违规时按 on_violation 失败(fail)或者照常产出、把违规片段标出来(flag) +三期起目录里还有表和字段(这次运行冻结的表结构)、知识库检索的原话。可疑实体(反引号里 +写了个哪里都找不到的名字)只标注:不重写、不失败,受管级别的正式出具按缺口降档;表结构快照 +不全时核对不了的名字(unverified_entity)只标注。entities: off 整层不管表名、字段名。 +结论句策略 claims:off(默认)不管;require_citation 在写作提示里要求结论句挂依据(自动链接的 +名字不算依据),没挂的计入缺口、出口降档;judge(模型裁判)是后续版本的事,现在写了直接报错。 + 出口契约复核(io.py)用 report_catalog 按同一套参数重建目录、独立重算,不信这里的自查。 """ from __future__ import annotations @@ -26,6 +32,8 @@ from app.db.base import SessionLocal from app.engine.context import NodeContext, NodeError from app.engine.evidence import ( CELLS_REASON, + CLAIMS_RULE, + ENTITY_KINDS, MARKER_RULES, StreamRenderer, build_catalog, @@ -35,13 +43,20 @@ from app.engine.evidence import ( ledger_enabled, ) from app.engine.nodes.llm import _invoke_streaming, _model_spec, _report_call -from app.engine.schema import GraphNode, GraphSpec, NodeType, _ancestors +from app.engine.schema import REPORT_CLAIMS, GraphNode, GraphSpec, NodeType, _ancestors, claims_problem from app.engine.state import GraphState, message_text, thinking_text from app.engine.toolcalls import leaked_markup, markup_warning from app.providers.factory import ProviderNotConfigured, get_chat_model ON_VIOLATION = ("fail", "flag") NUMBERS = ("strict", "off") +#: 实体层:link(默认)核对表名、字段名并自动链接;off 整层不管(目录里不收表和字段) +ENTITIES = ("link", "off") +#: 结论句策略,和画布校验同一份(schema.REPORT_CLAIMS);judge 是后续版本的事 +CLAIMS = REPORT_CLAIMS +#: 只标注、不让写作者重写也不判失败的违规:可疑实体(受管的正式出具按缺口降档)、表结构快照不全时 +#: 核对不了的名字(出口只标注) +SOFT = frozenset({"unknown_entity", "unverified_entity"}) #: 重写一次就是多一次整篇的模型调用,写不对的模型多给几次也大多写不对 MAX_REPAIRS = 3 @@ -79,16 +94,38 @@ def report_catalog(state: GraphState, spec: GraphSpec, node: GraphNode) -> dict[ if isinstance(sources, str) and sources != "ancestors": sources = [sources] if isinstance(sources, list): - # evidence_from 只管查询、检索这类证据从哪几个节点来;指标从哪来由 metrics_from 管 + # evidence_from 只管查询、检索这类证据(连同查询当时的表结构)从哪几个节点来;指标从哪来由 metrics_from 管 keep = {str(s) for s in sources} - ledger = [e for e in ledger if e.get("kind") not in ("query", "retrieval") or e.get("node_id") in keep] - return build_catalog( + ledger = [e for e in ledger + if e.get("kind") not in ("query", "retrieval", "schema") or e.get("node_id") in keep] + catalog = build_catalog( nodes=state.get("nodes") or {}, ledger=ledger, inputs=state.get("input") or {}, metrics_from=_setting(spec, node, "metrics_from") or None, allowed=_ancestors(spec, node.id), ) + if report_entities(spec, node) == "off": + # 目录里没有表和字段,写作目录也就不列;compose / verify 另传 entities=False,解析不了的原因照实说 + catalog = {alias: e for alias, e in catalog.items() if e.get("kind") not in ENTITY_KINDS} + return catalog + + +def report_entities(spec: GraphSpec, node: GraphNode) -> str: + """报告节点的实体层:link(默认)/ off。出口复核调 verify_doc 时传 entities=(这个值 == "link")。 + + 写错的值执行时节点会报错,这里只把 off 当 off,别的一律按默认。 + """ + return "off" if _setting(spec, node, "entities", "link") == "off" else "link" + + +def report_claims(spec: GraphSpec, node: GraphNode) -> str: + """报告节点的结论句策略:off / require_citation。出口判档按它(和节点执行时同一个取值规则)。 + + 写错的值(包括 judge)画布校验就拦了,执行时节点也会报错,这里只把没写当成 off。 + """ + value = _setting(spec, node, "claims", "off") + return value if isinstance(value, str) and value in CLAIMS else "off" def report_allowance(spec: GraphSpec, node: GraphNode) -> list[Any]: @@ -185,6 +222,12 @@ def _options(ctx: NodeContext) -> tuple[str, int]: numbers = str(ctx.cfg("numbers", "strict") or "strict") if numbers not in NUMBERS: raise NodeError(ctx.node.id, f"numbers 只能是 {' / '.join(NUMBERS)},写的是 {numbers!r}") + # 画布校验也拦这两项;执行时再守一道,报的是同一句话,免得写错的值被悄悄当成默认 + if problem := claims_problem(ctx.cfg("claims")): + raise NodeError(ctx.node.id, problem) + entities = ctx.cfg("entities", "link") or "link" + if entities not in ENTITIES: + raise NodeError(ctx.node.id, f"entities 只能是 {' / '.join(ENTITIES)},写的是 {entities!r}") raw = ctx.cfg("max_repairs", 1) try: repairs = int(raw) @@ -196,8 +239,13 @@ def _options(ctx: NodeContext) -> tuple[str, int]: def _blocking(violations: list[dict[str, Any]], numbers: str) -> list[dict[str, Any]]: - """要让写作者改的违规。numbers=off 时裸数字只记录不拦(文档和出口复核照样看得到)。""" - return [v for v in violations if not (numbers == "off" and v.get("code") == "uncited_number")] + """要让写作者改的违规。numbers=off 时裸数字只记录不拦(文档和出口复核照样看得到)。 + + 可疑实体只标注(SOFT):一个反引号里的业务词也可能被当成名字,为它整篇重写、判失败都不值; + 它留在文档和出口复核里,受管的正式出具按缺口降档。表结构快照不全时核对不了的名字同样只标注。 + """ + return [v for v in violations if v.get("code") not in SOFT + and not (numbers == "off" and v.get("code") == "uncited_number")] def _named(violations: list[dict[str, Any]], limit: int = 3) -> str: @@ -216,14 +264,16 @@ def _repair_request(violations: list[dict[str, Any]]) -> str: ) -def _messages(ctx: NodeContext, state: GraphState, catalog: dict[str, Any], cells: bool = True) -> list[BaseMessage]: +def _messages(ctx: NodeContext, state: GraphState, catalog: dict[str, Any], cells: bool = True, + claims: str = "off") -> list[BaseMessage]: system = ctx.render_str(ctx.cfg("system", ""), state) instructions = ctx.render_str(ctx.cfg("instructions", ""), state).strip() \ or "根据下面的证据写一份简洁的报告,先总后分。" + rules = f"{MARKER_RULES}\n{CLAIMS_RULE}" if claims == "require_citation" else MARKER_RULES return [ SystemMessage(content="\n\n".join(p for p in (system, ROLE) if p)), HumanMessage(content=( - f"{instructions}\n\n{catalog_prompt(catalog, cells_allowed=cells)}\n\n{MARKER_RULES}\n\n" + f"{instructions}\n\n{catalog_prompt(catalog, cells_allowed=cells)}\n\n{rules}\n\n" "只输出报告正文(Markdown),不要写撰写说明。" )), ] @@ -257,7 +307,10 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: catalog = report_catalog(state, spec, ctx.node) allow = report_allowance(spec, ctx.node) cells = True - if ledger_enabled(ctx.run): + # 结论句策略只对升级后发起的运行生效:升级前的运行(图里也不会有 claims)事件、产出一个字不加 + evidence_on = ledger_enabled(ctx.run) + claims = report_claims(spec, ctx.node) if evidence_on else "off" + if evidence_on: # 升级前发起的运行没有查询条目,单元格无从谈起,也就不多查这一次 from app.engine.governance import governed_formal @@ -265,7 +318,7 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: if warning := _no_evidence(catalog, cells): ctx.emit(EventType.LOG, level="warn", code="report_no_evidence", message=warning) - messages = _messages(ctx, state, catalog, cells) + messages = _messages(ctx, state, catalog, cells, claims) started = time.perf_counter() spent: list[dict[str, Any]] = [] nudged = False @@ -291,9 +344,11 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: message="模型只输出了思考内容,报告正文为空。把 max_tokens 调大,或把 thinking 设为 off。") return response, text + entities = report_entities(spec, ctx.node) == "link" + def compose(raw: str) -> dict[str, Any]: return compose_doc(raw, catalog, node_id=ctx.node.id, run_id=ctx.run.run_id, allow_numbers=allow, - cells_allowed=cells) + cells_allowed=cells, entities=entities) response, raw = await draft(messages) doc = compose(raw) @@ -310,7 +365,8 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: doc_artifact = await _store(doc, ctx) ctx.emit(EventType.REPORT_CHECKED, doc_artifact=doc_artifact, ok=not doc["violations"], stats=doc["stats"], violations=doc["violations"][:20], repairs=repairs, - on_violation=on_violation, failed=bool(blocking) and on_violation == "fail") + on_violation=on_violation, failed=bool(blocking) and on_violation == "fail", + **({"claims": claims} if evidence_on else {})) if blocking and on_violation == "fail": head = ";".join(v["message"] for v in blocking[:5]) more = f";…另有 {len(blocking) - 5} 处" if len(blocking) > 5 else "" @@ -331,6 +387,7 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: "repairs": repairs, "model": model_id, "duration_ms": int((time.perf_counter() - started) * 1000), + **({"claims": claims} if evidence_on else {}), } updates: dict[str, Any] = {"nodes": {ctx.node.id: result}, "usage": usage} if ctx.cfg("emit_message", True): diff --git a/backend/app/engine/nodes/tools.py b/backend/app/engine/nodes/tools.py index cf1cf13e..a3948b2f 100644 --- a/backend/app/engine/nodes/tools.py +++ b/backend/app/engine/nodes/tools.py @@ -12,7 +12,7 @@ from langgraph.func import task from app.core.artifact_store import canonical_json, content_hash from app.engine.approval import read_decision from app.engine.context import NodeContext, NodeError -from app.engine.evidence import ledger_enabled, next_exec, query_entry_fields +from app.engine.evidence import ledger_enabled, next_exec, query_entry_fields, schema_ledger_entry from app.engine.replay import ask, once from app.engine.state import GraphState from app.engine.toolcalls import ToolTimeout, limit_fields, limit_of, run_bounded @@ -175,9 +175,11 @@ async def run_tool(state: GraphState, ctx: NodeContext) -> dict[str, Any]: snapshot_id = None # 只认数据源工具交回的查询:别的工具(MCP、自定义)拼出同样形状的 JSON,里面的工件 id 不是我们落的 query = query_entry_fields(result) if evidence_on and name.startswith(QUERY_PREFIX) else None + schema = query.get("schema_artifact") if query else None ctx.emit(EventType.TOOL_END, tool=name, duration_ms=elapsed, preview=json.dumps(result, ensure_ascii=False, default=str)[:2000], - artifact=snapshot_id, **({"query_artifact": query["artifact"]} if query else {})) + artifact=snapshot_id, **({"query_artifact": query["artifact"]} if query else {}), + **({"schema_artifact": schema} if schema else {})) updates: dict[str, Any] = {"nodes": {ctx.node.id: result}} if evidence_on: @@ -188,9 +190,14 @@ async def run_tool(state: GraphState, ctx: NodeContext) -> dict[str, Any]: entries.append({"kind": "tool", "node_id": ctx.node.id, "exec": exec_no, "artifact": snapshot_id, "tool": name}) if query: + # 查询当时的表结构快照另记一条 schema 条目;取不回来的话查询条目也不带它(不启用实体核对) + frozen = schema_ledger_entry(schema, node_id=ctx.node.id, exec_no=exec_no) if schema else None entries.append({"kind": "query", "node_id": ctx.node.id, "exec": exec_no, "artifact": query["artifact"], **({"via": snapshot_id} if snapshot_id else {}), "tool": name, "source": query["source"], - "columns": query["columns"], "rows": query["rows"], "truncated": query["truncated"]}) + "columns": query["columns"], "rows": query["rows"], "truncated": query["truncated"], + **({"schema_artifact": schema, "tables": query["tables"]} if frozen else {})}) + if frozen: + entries.append(frozen) if entries: updates["evidence"] = entries var_name = ctx.cfg("assign_to", "") diff --git a/backend/app/engine/schema.py b/backend/app/engine/schema.py index f9eddc14..93a00aab 100644 --- a/backend/app/engine/schema.py +++ b/backend/app/engine/schema.py @@ -422,6 +422,34 @@ def _check_report_sources(node: GraphNode, spec: GraphSpec, result: ValidationRe node_id=node.id, field="metrics_from", code=code) +#: 报告撰写节点的结论句策略(claims)。off:结论句不参与判档(默认,保持以前的行为); +#: require_citation:没挂引用的结论句计入缺口、按出具档位降档。judge(模型裁判)是后续版本的 +REPORT_CLAIMS = ("off", "require_citation") +CLAIMS_LATER = "结论句裁判在后续版本支持" + + +def claims_problem(value: Any) -> str | None: + """claims 的取值有什么问题,没问题返回 None。校验和报告撰写节点执行时报的是同一句话。""" + if value in (None, "") or (isinstance(value, str) and value in REPORT_CLAIMS): + return None + if value == "judge": + return f"claims 写的是 judge:{CLAIMS_LATER},现在只能写 off 或 require_citation" + return f"claims 只能是 {' / '.join(REPORT_CLAIMS)},写的是 {value!r}" + + +def _check_report_claims(node: GraphNode, spec: GraphSpec, result: ValidationResult) -> None: + """claims 写错了(包括还不支持的 judge)挡住运行:执行时报告节点照样会报这一句, + 与其跑完取数、算完口径卡才死在写报告那一步,不如在画图时说破。 + + 和运行时同一个取值规则:节点上没写就取图级 defaults(NodeContext.cfg)。 + """ + value = node.config.get("claims") + if value in (None, ""): + value = spec.defaults.get("claims") + if problem := claims_problem(value): + result.add(problem, node_id=node.id, field="claims", code="report.claims_invalid") + + def _check_report_from(node: GraphNode, report_from: Any, spec: GraphSpec, result: ValidationResult) -> None: """引用模式的契约:report_from 必须是出口上游的报告撰写节点。""" @@ -506,6 +534,7 @@ def validate_graph(spec: GraphSpec) -> ValidationResult: node_id=node.id, field=f"metrics[{i}]") elif node.type == NodeType.REPORT: _check_report_sources(node, spec, result) + _check_report_claims(node, spec, result) elif node.type == NodeType.OUTPUT: contract = cfg.get("contract") if contract and not isinstance(contract, dict): @@ -982,15 +1011,16 @@ def evidence_issues(spec: GraphSpec) -> list[ValidationIssue]: out: list[ValidationIssue] = [] - def add(message: str, node: GraphNode, field: str, level: str = "warning") -> None: - out.append(ValidationIssue(level=level, node_id=node.id, field=field, message=message)) # type: ignore[arg-type] + def add(message: str, node: GraphNode, field: str, level: str = "warning", code: str | None = None) -> None: + out.append(ValidationIssue(level=level, node_id=node.id, field=field, message=message, # type: ignore[arg-type] + code=code)) for node in spec.nodes: cfg = node.config if node.type == NodeType.AGENT and cfg.get("output_schema") and cfg.get("cite_fields") is not True: add("output_schema 要开 cite_fields 才生效,会多一次抽取调用:开了之后循环结束再按 Schema " "抽一次字段,每个字段都核对到查询结果里的那一格;不开的话 assign_to 拿到的仍是自由文字", - node, "output_schema") + node, "output_schema", code="evidence.cite_fields_off") elif node.type == NodeType.CODE: role = cfg.get("evidence_role") if role not in (None, "") and role not in EVIDENCE_ROLES: @@ -1007,7 +1037,8 @@ def evidence_issues(spec: GraphSpec) -> list[ValidationIssue]: warned.add(producer.id) add(f"口径卡的输入来自沙箱代码「{producer.title}」,它的角色是计算(evidence_role 不是 source):" "沙箱里算出来的数核对不了出处。负责取数的代码节点把 evidence_role 标成 source;" - "业务计算挪进口径卡的表达式", node, f"metrics[{i}].expression") + "业务计算挪进口径卡的表达式", node, f"metrics[{i}].expression", + code="evidence.caliber_compute_input") return out diff --git a/backend/app/seed.py b/backend/app/seed.py index 21adb2d9..0583ed1a 100644 --- a/backend/app/seed.py +++ b/backend/app/seed.py @@ -233,7 +233,9 @@ TEMPLATES: list[dict[str, Any]] = [ "description": "报告里的每个数都点得开出处:tool 节点查库、口径卡登记指标、报告撰写节点只写引用" "标记,数字由系统从口径卡取出来渲染,出口按引用逐段核对后三档出具。取数节点查的是示例数据源 " "shop 的 orders 表(week、amount、refunded 三列):先接入你的库(或者上传一张表格、命名为 " - "shop),再把 SQL 换成你的口径。", + "shop),再把 SQL 换成你的口径。报告撰写节点按受管要求写明了 on_violation: fail 和 claims: " + "require_citation:探索运行里修不掉的数字违规也会让报告节点失败,而不只是标出来;正式运行里没挂引用" + "的结论句计入缺口、按档降级。", "tags": ["出具", "口径卡", "周报", "证据"], "nodes": [ _n("start", "input", "周期", fields=[ @@ -265,7 +267,9 @@ TEMPLATES: list[dict[str, Any]] = [ {"id": "refund_rate", "name": "退款率", "decimals": 4, "format": "percent_of_ratio", "expression": "vars.q.rows[0][3] / vars.q.rows[0][2] if vars.q.rows[0][2] else None"}, ]), + # 受管出具要求报告撰写节点写明最严的三项(发布门禁 G3),模板照着写,拿来就能发布成受管 _n("write", "report", "报告撰写", metrics_from=["caliber"], + numbers="strict", on_violation="fail", claims="require_citation", system="你是周报撰写人:结论先行,只写有数据支撑的判断。", instructions="为 {{ input.week }} 写一份简短的周报:先用一两句话总结本周," "再分点说销售额、订单、客单价和退款的变化。"), diff --git a/backend/app/tools/datasource.py b/backend/app/tools/datasource.py index 63bfea9d..769accaa 100644 --- a/backend/app/tools/datasource.py +++ b/backend/app/tools/datasource.py @@ -144,6 +144,10 @@ def _make_query_tool(source: DataSource, ctx: ToolContext) -> StructuredTool: payload = result.to_payload() payload["source"] = source.name + # 表结构快照:这次查询时数据源的结构冻结下来,报告里写的表名、字段名按它核对。事后数据源 + # 重新探查、改了结构,已经跑完的运行核对的还是当时那一份 + if schema := await _store_schema(source, ctx): + payload["schema_artifact"] = schema # 查询快照进工件库:出具体系的数字回指要能下钻到"这个数是哪条 SQL 查出来的" try: from app.core.artifact_store import put_json @@ -172,6 +176,28 @@ def _make_query_tool(source: DataSource, ctx: ToolContext) -> StructuredTool: ) +async def _store_schema(source: DataSource, ctx: ToolContext) -> str | None: + """把数据源此刻的 schema_cache 存成 schema_snapshot 工件,返回工件 id。 + + 内容寻址:结构没变的话,同一次运行里查多少次都是同一件,文件只有一份。没探查过结构 + (或者探查失败、一张表都没有)返回 None:没有东西可以冻结,报告也就不核对表名。 + """ + cache = source.schema_cache or {} + tables = cache.get("tables") + if not isinstance(tables, dict) or not tables: + return None + from app.core.artifact_store import put_json + from app.engine.evidence import SCHEMA_SNAPSHOT + + content = {"source": source.name, "tables": tables, + **{k: cache[k] for k in ("schema", "synced_at", "truncated", "total") if k in cache}} + try: + return await put_json(content, kind=SCHEMA_SNAPSHOT, run_id=ctx.run_id or "", node_id=ctx.node_id or "", + meta={"source": source.name, "tables": len(tables)}) + except Exception: # noqa: BLE001 - 存不下不影响查询本身,只是这次报告不核对表名 + return None + + def _make_schema_tool(source: DataSource) -> StructuredTool: async def _run(table: str | list[str] | None = None) -> str: wanted = [table] if isinstance(table, str) else list(table or []) @@ -206,4 +232,4 @@ def is_dangerous_call(source: DataSource, sql: str) -> bool: 只读源上的写不需要审批——守卫和连接层都会拒,批了也写不进去。 """ - return not source.readonly and is_write(sql) + return not source.readonly and is_write(sql, dialect=source.kind) diff --git a/backend/tests/test_approval_default.py b/backend/tests/test_approval_default.py index 89741419..705718d7 100644 --- a/backend/tests/test_approval_default.py +++ b/backend/tests/test_approval_default.py @@ -157,9 +157,11 @@ def _governable(tool: str): node("q", "tool", tool=tool, args={"sql": "DELETE FROM t WHERE x = 0"}), node("caliber", "metrics", caliber="删除口径", caliber_version="v1", assign_to="m", metrics=[{"id": "done", "name": "已执行", "unit": "次", "expression": "1"}]), + # 受管契约要核对报告撰写节点的文档(门禁 G1),叙述模式发不成受管 + node("write", "report", instructions="写执行结果", + numbers="strict", on_violation="fail", claims="require_citation"), node("out", "output", fields=[{"name": "结果", "value": "{{ nodes.q }}"}], - contract={"metrics_from": ["caliber"], "narrative": "执行了 {{ vars.m.done }} 次", - "required": ["done"], "strict": True}), + contract={"metrics_from": ["caliber"], "report_from": "write", "required": ["done"], "strict": True}), ) diff --git a/backend/tests/test_autofix_assist.py b/backend/tests/test_autofix_assist.py index 5360aa7e..9708eb1d 100644 --- a/backend/tests/test_autofix_assist.py +++ b/backend/tests/test_autofix_assist.py @@ -39,7 +39,8 @@ def two_exits() -> dict: node("fetch", "tool", "取数", tool="db_query__shop", args={"sql": "SELECT SUM(amount) AS gmv FROM orders"}), node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[ {"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}]), - node("write", "report", "报告撰写", instructions="写周报"), + node("write", "report", "报告撰写", instructions="写周报", + numbers="strict", on_violation="fail", claims="require_citation"), node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]), node("raw", "output", "原始数", fields=[{"name": "数", "value": "{{ nodes.fetch }}"}]), ] @@ -227,7 +228,8 @@ def guarded() -> dict: {"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}]), node("team", "supervisor", "复核团队", goal="复核", agents=[{"name": "checker", "tools": []}]), node("gate", "human", "人工把关", prompt="确认数字"), - node("write", "report", "报告撰写", instructions="写周报"), + node("write", "report", "报告撰写", instructions="写周报", + numbers="strict", on_violation="fail", claims="require_citation"), node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], contract=CONTRACT), ] edges = [{"id": f"e{i}", "source": a, "target": b} for i, (a, b) in enumerate( diff --git a/backend/tests/test_autofix_governance.py b/backend/tests/test_autofix_governance.py new file mode 100644 index 00000000..97e34897 --- /dev/null +++ b/backend/tests/test_autofix_governance.py @@ -0,0 +1,495 @@ +"""证据门禁 G1–G5 接进发布前自动修复(engine/autofix.py)。 + +照发布前自动修复的原则:答案唯一、只提高要求的一键修(auto);要人拿主意的列候选、不替人选(choice); +结构性的交给 Copilot(assist)。修完重跑 validate 和门禁,错误必须变少、不能冒出新的;降低要求的改动 +(numbers、on_violation、claims 往宽里改,关掉 cite_fields,替人把沙箱代码标成取数)一律拒绝。 +""" +from __future__ import annotations + +import copy +import json + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessageChunk + +from app.engine.autofix import HANDOFF, apply_fixes, forbidden_changes, plan_fixes +from app.engine.governance import publish_issues +from app.engine.schema import GraphSpec +from app.main import app + +STRICT = {"numbers": "strict", "on_violation": "fail", "claims": "require_citation"} +SCHEMA = {"type": "object", "properties": {"gmv": {"type": "number"}}} + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 10, "y": 20}, "width": 240, + "data": {"label": label or nid, "config": config}} + + +def traceable(*, report=None, **contract_overrides) -> dict: + """input → 查库 → 口径卡 → 报告撰写 → 出具:受管要求(含 G1–G5)全部满足。""" + contract = {"report_from": "write", "metrics_from": ["card"], "required": ["gmv"], "strict": True} + contract.update(contract_overrides) + nodes = [ + node("start", "input", "周期"), + node("fetch", "tool", "取数", tool="db_query__shop", + args={"sql": "SELECT SUM(amount) AS gmv, COUNT(*) AS orders FROM orders"}), + node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[ + {"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}]), + node("write", "report", "报告撰写", instructions="写周报", **(STRICT if report is None else report)), + node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}, + {"name": "指标清单", "value": "{{ nodes.card.text }}"}], + contract={k: v for k, v in contract.items() if v is not None}), + ] + edges = [{"id": f"e{i}", "source": a["id"], "target": b["id"]} for i, (a, b) in enumerate(zip(nodes, nodes[1:]))] + return {"nodes": nodes, "edges": edges, "viewport": {"x": 1, "y": 2, "zoom": 1}} + + +def find(graph: dict, nid: str) -> dict: + return next(n for n in graph["nodes"] if n["id"] == nid) + + +def config(graph: dict, nid: str) -> dict: + return find(graph, nid)["data"]["config"] + + +def insert(graph: dict, new: dict, *, before: str, after: str) -> dict: + graph["nodes"].insert(len(graph["nodes"]) - 1, new) + graph["edges"] += [{"source": after, "target": new["id"]}, {"source": new["id"], "target": before}] + return graph + + +def plan(graph: dict, level: str = "governed") -> dict[str, dict]: + spec = GraphSpec.model_validate(graph) + return {f["id"]: f for f in plan_fixes(spec, publish_issues(spec, level=level), level=level)} + + +def errors(graph: dict, level: str = "governed") -> list: + return [i for i in publish_issues(GraphSpec.model_validate(graph), level=level) if i.level == "error"] + + +def codes(issues) -> set[str]: + return {i["code"] if isinstance(i, dict) else i.code for i in issues} + + +def test_the_baseline_needs_no_fix(): + assert errors(traceable()) == [] and plan(traceable()) == {} + + +# -------------------------------------------------------------------------- +# G3:一键写明 numbers / on_violation / claims +# -------------------------------------------------------------------------- + + +def test_report_policy_is_filled_in_with_one_auto_fix(): + graph = traceable(report={"numbers": "off"}) + fid = "governed.report_policy:write" + fix = plan(graph)[fid] + assert fix["kind"] == "auto" and fix["node_id"] == "write" + assert fix["preview"]["after"] == STRICT and fix["preview"]["before"] == { + "numbers": "off", "on_violation": None, "claims": None} + out = apply_fixes(graph, [fid], level="governed") + assert out["applied"] == [fid] and out["rejected"] == [] and out["ok"] is True + assert {k: config(out["graph"], "write")[k] for k in STRICT} == STRICT + assert config(out["graph"], "write")["instructions"] == "写周报" # 别的配置原样留着 + assert [(c["field"], c["before"], c["after"]) for c in out["changes"]] == [ + ("numbers", "off", "strict"), ("on_violation", None, "fail"), ("claims", None, "require_citation")] + assert out["ops"] == [{"op": "update_node", "id": "write", "config": STRICT}] + # 坐标、视口这些前端的东西原样留着 + assert find(out["graph"], "write")["position"] == {"x": 10, "y": 20} and out["graph"]["viewport"]["zoom"] == 1 + + +def test_only_the_missing_settings_are_written(): + graph = traceable(report={"numbers": "strict", "claims": "require_citation"}) + fix = plan(graph)["governed.report_policy:write"] + assert fix["preview"] == {"field": "on_violation", "before": None, "after": "fail"} + out = apply_fixes(graph, [fix["id"]], level="governed") + assert [c["field"] for c in out["changes"]] == ["on_violation"] and out["ok"] is True + + +def test_published_level_warnings_get_no_fix(): + graph = traceable(report={}) + assert plan(graph, "published") == {} + + +# -------------------------------------------------------------------------- +# validate:claims 写了不支持的值 +# -------------------------------------------------------------------------- + + +def test_judge_becomes_require_citation_under_governed(): + graph = traceable(report={**STRICT, "claims": "judge"}) + fid = "report.claims_invalid:write" + fix = plan(graph)[fid] + assert fix["kind"] == "auto" and fix["preview"] == {"field": "claims", "before": "judge", + "after": "require_citation"} + out = apply_fixes(graph, [fid], level="governed") + assert out["ok"] is True and config(out["graph"], "write")["claims"] == "require_citation" + + +def test_an_invalid_claims_value_is_a_choice_under_published(): + graph = traceable(report={"claims": "judge"}) + fid = "report.claims_invalid:write" + fix = plan(graph, "published")[fid] + assert fix["kind"] == "choice" and [o["value"] for o in fix["options"]] == ["off", "require_citation"] + assert fix["default"] == "require_citation" # 离裁判最近的是「每句结论挂引用」,仍然要人确认 + assert apply_fixes(graph, [fid], level="published")["applied"] == [] + out = apply_fixes(graph, [fid], {fid: "off"}, level="published") + assert out["applied"] == [fid] and config(out["graph"], "write")["claims"] == "off" and out["ok"] is True + # 乱写的值没有「最接近的」,不给建议 + assert "default" not in plan(traceable(report={"claims": "strict"}), "published")[fid] + + +# -------------------------------------------------------------------------- +# G5:给口径卡供数的 agent +# -------------------------------------------------------------------------- + + +def feeding_agent(**config_) -> dict: + graph = insert(traceable(), node("ask", "agent", "查数员", tools=["db_query__shop"], prompt="查 orders", + assign_to="facts", **config_), before="card", after="start") + config(graph, "card")["metrics"].append({"id": "gmv2", "expression": "vars.facts.gmv"}) + return graph + + +def test_cite_fields_is_switched_on_when_the_schema_exists(): + graph = feeding_agent(output_schema=SCHEMA) + fid = "governed.caliber_agent_cite_fields:ask" + fix = plan(graph)[fid] + assert fix["kind"] == "auto" and fix["preview"] == {"field": "cite_fields", "before": None, "after": True} + out = apply_fixes(graph, [fid], level="governed") + assert out["ok"] is True and config(out["graph"], "ask")["cite_fields"] is True + assert config(out["graph"], "ask")["output_schema"] == SCHEMA + + +def test_a_missing_schema_is_not_invented(): + graph = feeding_agent() + fid = "governed.caliber_agent_schema:ask" + fix = plan(graph)[fid] + assert fix["kind"] == "assist" and "output_schema" in fix["label"] + out = apply_fixes(graph, [fid], level="governed") + assert out["applied"] == [] and out["rejected"][0]["fix_id"] == fid + assert "output_schema" not in config(out["graph"], "ask") and out["graph"] == graph + + +def test_a_model_call_feeding_a_card_goes_to_copilot(): + graph = insert(traceable(), node("guess", "llm", "估算", prompt="估一个数", assign_to="g"), before="card", + after="start") + config(graph, "card")["metrics"].append({"id": "g", "expression": "vars.g"}) + assert plan(graph)["governed.caliber_model_input:guess"]["kind"] == "assist" + + +# -------------------------------------------------------------------------- +# G4:沙箱代码喂口径卡——它到底是在取数还是在算,要人说 +# -------------------------------------------------------------------------- + + +def compute_feeding() -> dict: + graph = insert(traceable(), node("calc", "code", "计算", code="print(1)", assign_to="calc"), before="card", + after="fetch") + config(graph, "card")["metrics"].append({"id": "ratio", "expression": "vars.calc.ratio"}) + return graph + + +def test_compute_code_is_a_choice_not_an_auto_fix(): + graph = compute_feeding() + fid = "governed.caliber_compute_input:calc" + fix = plan(graph)[fid] + assert fix["kind"] == "choice" and fix["node_id"] == "calc" and fix["field"] == "evidence_role" + assert [o["value"] for o in fix["options"]] == ["source", HANDOFF] + assert "default" not in fix and "preview" not in fix + # 不选就不动 + out = apply_fixes(graph, [fid], level="governed") + assert out["applied"] == [] and "evidence_role" not in config(out["graph"], "calc") + assert out["rejected"][0]["fix_id"] == fid + + +def test_choosing_source_marks_the_node(): + graph = compute_feeding() + fid = "governed.caliber_compute_input:calc" + out = apply_fixes(graph, [fid], {fid: "source"}, level="governed") + assert out["applied"] == [fid] and out["ok"] is True + assert config(out["graph"], "calc")["evidence_role"] == "source" + + +def test_choosing_copilot_hands_the_node_over_without_touching_it(): + graph = compute_feeding() + fid = "governed.caliber_compute_input:calc" + out = apply_fixes(graph, [fid], {fid: HANDOFF}, level="governed") + assert out["applied"] == [] and out["rejected"] == [] and out["handoff"] == [fid] + assert out["graph"] == graph and out["ok"] is False + + +def test_copilot_may_not_mark_a_code_node_as_source_on_its_own(): + before = compute_feeding() + after = copy.deepcopy(before) + config(after, "calc")["evidence_role"] = "source" + [reason] = forbidden_changes(before, after) + assert "「计算」" in reason and "source" in reason + # 新加的沙箱代码直接标成取数也一样 + after = copy.deepcopy(before) + after["nodes"].append(node("raw", "code", "取原始数据", code="print(2)", evidence_role="source")) + assert any("「取原始数据」" in r for r in forbidden_changes(before, after)) + + +# -------------------------------------------------------------------------- +# G1:契约和成果字段接到报告撰写节点 +# -------------------------------------------------------------------------- + + +def narrative() -> dict: + return traceable(report_from=None, narrative="{{ nodes.write.text }}") + + +def test_report_from_takes_the_only_report_upstream(): + graph = narrative() + fid = "governed.report_from_required:done" + fix = plan(graph)[fid] + assert fix["kind"] == "auto" and fix["preview"] == {"field": "contract.report_from", "before": None, + "after": "write"} + out = apply_fixes(graph, [fid], level="governed") + assert out["ok"] is True and config(out["graph"], "done")["contract"]["report_from"] == "write" + + +def test_several_reports_make_report_from_a_choice(): + graph = insert(narrative(), node("write2", "report", "第二份报告", instructions="写月报", **STRICT), + before="done", after="card") + fid = "governed.report_from_required:done" + fix = plan(graph)[fid] + assert fix["kind"] == "choice" and [o["value"] for o in fix["options"]] == ["write", "write2"] + assert apply_fixes(graph, [fid], level="governed")["applied"] == [] + out = apply_fixes(graph, [fid], {fid: "write2"}, level="governed") + assert out["applied"] == [fid] and config(out["graph"], "done")["contract"]["report_from"] == "write2" + + +def test_without_a_report_node_report_from_goes_to_copilot(): + graph = narrative() + write = find(graph, "write") + write.update(type="llm", data={"label": "写周报", "config": {"prompt": "写周报"}}) + # 受管级别的叙述模式过不了 G1,不再给「选 narrative」这种修了也发不出去的候选 + for fid in ("governed.report_from_required:done",): + assert plan(graph)[fid]["kind"] == "assist" + bare = traceable(report_from=None) + find(bare, "write").update(type="llm", data={"label": "写周报", "config": {"prompt": "写周报"}}) + assert plan(bare)["contract.report_from_missing:done"]["kind"] == "assist" + + +def with_story(graph: dict) -> dict: + graph = insert(graph, node("narrate", "llm", "叙述", prompt="写周报", assign_to="story"), before="done", + after="card") + config(graph, "done")["fields"] += [{"name": "摘要", "value": "{{ vars.story }}"}, + {"name": "点评", "value": "点评:{{ nodes.narrate.text }}"}] + return graph + + +def test_exit_fields_are_repointed_at_the_chosen_report(): + graph = with_story(traceable()) + fid = "governed.exit_text_source:done" + fix = plan(graph)[fid] + assert fix["kind"] == "choice" and fix["field"] == "fields" + assert [o["value"] for o in fix["options"]] == ["write"] and fix["default"] == "write" + assert "「摘要」" in fix["label"] and "「点评」" in fix["label"] + # 就一个候选也不自动改:字段里放什么是作者的事 + out = apply_fixes(graph, [fid], level="governed") + assert out["applied"] == [] and out["graph"] == graph + out = apply_fixes(graph, [fid], {fid: "write"}, level="governed") + assert out["applied"] == [fid] and out["ok"] is True + fields = config(out["graph"], "done")["fields"] + assert fields == [{"name": "周报", "value": "{{ nodes.write.text }}"}, + {"name": "指标清单", "value": "{{ nodes.card.text }}"}, + {"name": "摘要", "value": "{{ nodes.write.text }}"}, + {"name": "点评", "value": "{{ nodes.write.text }}"}] + assert [(c["field"], c["before"], c["after"]) for c in out["changes"]] == [ + ("fields[2].value", "{{ vars.story }}", "{{ nodes.write.text }}"), + ("fields[3].value", "点评:{{ nodes.narrate.text }}", "{{ nodes.write.text }}")] + assert out["ops"] == [{"op": "update_node", "id": "done", "config": {"fields": fields}}] + + +def test_an_exit_without_fields_gets_one_that_takes_the_report(): + graph = traceable() + config(graph, "done")["fields"] = [] + graph = insert(graph, node("polish", "llm", "润色", prompt="{{ nodes.write.text }}"), before="done", + after="write") + graph["edges"] = [e for e in graph["edges"] if (e["source"], e["target"]) != ("write", "done")] + fid = "governed.exit_text_source:done" + out = apply_fixes(graph, [fid], {fid: "write"}, level="governed") + assert out["applied"] == [fid] and out["ok"] is True + assert config(out["graph"], "done")["fields"] == [{"name": "result", "value": "{{ nodes.write.text }}"}] + + +def test_exit_fields_without_any_report_go_to_copilot(): + graph = with_story(traceable()) + find(graph, "write").update(type="llm", data={"label": "写周报", "config": {"prompt": "写周报"}}) + config(graph, "done")["contract"].pop("report_from") + assert plan(graph)["governed.exit_text_source:done"]["kind"] == "assist" + + +# -------------------------------------------------------------------------- +# G2:结构性的,交给 Copilot +# -------------------------------------------------------------------------- + + +def test_text_bypass_is_an_assist(): + graph = traceable() + graph["nodes"] += [node("chat", "llm", "闲聊", prompt="你好"), + node("side", "output", "附带成果", fields=[{"name": "回答", "value": "{{ nodes.chat.text }}"}])] + graph["edges"] += [{"source": "start", "target": "chat"}, {"source": "chat", "target": "side"}] + fid = "governed.text_bypass:chat" + fix = plan(graph)[fid] + assert fix["kind"] == "assist" and "报告撰写" in fix["label"] + out = apply_fixes(graph, [fid], level="governed") + assert out["applied"] == [] and out["rejected"][0]["fix_id"] == fid + + +# -------------------------------------------------------------------------- +# 修完更好、不降低要求 +# -------------------------------------------------------------------------- + + +def test_all_auto_fixes_together_leave_only_the_decisions(): + graph = with_story(feeding_agent(output_schema=SCHEMA)) + graph = insert(graph, node("calc", "code", "计算", code="print(1)", assign_to="calc"), before="card", + after="fetch") + config(graph, "card")["metrics"].append({"id": "ratio", "expression": "vars.calc.ratio"}) + config(graph, "write").clear() + config(graph, "done")["contract"].update(report_from=None, narrative="{{ nodes.write.text }}") + fixes = plan(graph) + autos = [f["id"] for f in fixes.values() if f["kind"] == "auto"] + assert set(autos) == {"governed.report_policy:write", "governed.caliber_agent_cite_fields:ask", + "governed.report_from_required:done"} + before = len(errors(graph)) + out = apply_fixes(graph, autos, level="governed") + assert sorted(out["applied"]) == sorted(autos) and out["rejected"] == [] + assert len([i for i in out["remaining"] if i["level"] == "error"]) < before + left = {i["code"] for i in out["remaining"] if i["level"] == "error"} + assert left == {"governed.exit_text_source", "governed.caliber_compute_input"} + # 剩下的都要人拿主意,没有一条被替人应用 + assert all(fixes[f]["kind"] != "auto" for f in fixes if f.split(":")[0] in left) + + +@pytest.mark.parametrize("key, before, after", [ + ("numbers", "strict", "off"), ("on_violation", "fail", "flag"), ("on_violation", "fail", None), + ("claims", "require_citation", "off"), ("claims", "require_citation", None), +]) +def test_loosening_the_report_policy_is_refused(key, before, after): + old = traceable(report={**STRICT, key: before}) + new = copy.deepcopy(old) + if after is None: + config(new, "write").pop(key) + else: + config(new, "write")[key] = after + [reason] = forbidden_changes(old, new) + assert "「报告撰写」" in reason and key in reason + + +def test_turning_cite_fields_off_is_refused(): + old = feeding_agent(output_schema=SCHEMA, cite_fields=True) + new = copy.deepcopy(old) + config(new, "ask")["cite_fields"] = False + [reason] = forbidden_changes(old, new) + assert "「查数员」" in reason and "cite_fields" in reason + + +def test_tightening_is_not_refused(): + old = traceable(report={"numbers": "off", "on_violation": "flag", "claims": "off"}) + new = copy.deepcopy(old) + config(new, "write").update(STRICT) + assert forbidden_changes(old, new) == [] + + +# -------------------------------------------------------------------------- +# 接口:发布前检查带上 G 规则的修复;选了「交给 Copilot」就真的交给 Copilot +# -------------------------------------------------------------------------- + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +class Scripted: + """按剧本吐操作流的 Copilot 模型;记下每次收到的请求。""" + + def __init__(self, ops: list[dict]) -> None: + self.lines = [json.dumps(o, ensure_ascii=False) for o in ops] + self.calls: list[list] = [] + + async def astream(self, messages): + self.calls.append(messages) + for line in self.lines: + yield AIMessageChunk(content=line + "\n") + + +def script(monkeypatch, ops: list[dict]) -> Scripted: + import app.api.copilot as copilot + + model = Scripted([{"op": "plan", "summary": "改图"}, *ops, {"op": "done", "explanation": "改好了"}]) + + async def _model(*_a, **_k): + return model, "mock-fast" + + monkeypatch.setattr(copilot, "get_chat_model", _model) + return model + + +async def create(client, graph: dict) -> str: + return (await client.post("/api/workflows", json={"name": "证据门禁-修复", "graph": graph})).json()["id"] + + +async def test_publish_check_lists_the_g_fixes(client): + graph = with_story(compute_feeding()) + config(graph, "write").clear() + wf = await create(client, graph) + body = (await client.post(f"/api/workflows/{wf}/publish-check", json={"level": "governed"})).json() + assert body["ok"] is False + kinds = {f["id"]: f["kind"] for f in body["fixes"]} + assert kinds["governed.report_policy:write"] == "auto" + assert kinds["governed.caliber_compute_input:calc"] == "choice" + assert kinds["governed.exit_text_source:done"] == "choice" + assert all(not any(k.startswith("_") for k in f) for f in body["fixes"]) # 内部信息不外露 + by_code = {i["code"]: i for i in body["issues"]} + assert by_code["governed.report_policy"]["fix"] == "governed.report_policy:write" + assert by_code["governed.exit_text_source"]["fix"] == "governed.exit_text_source:done" + # 已发布级别:G 规则只是警告,不给修复 + body = (await client.post(f"/api/workflows/{wf}/publish-check", json={"level": "published"})).json() + assert {i["level"] for i in body["issues"] if (i["code"] or "").startswith("governed.")} == {"warning"} + assert not [f for f in body["fixes"] if f["code"].startswith("governed.")] + + +async def test_choosing_copilot_in_the_choice_runs_the_assist(client, monkeypatch): + # Copilot 把口径卡里读沙箱代码的那条指标改成直接读查询结果:计算挪进了口径卡 + card = [{"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}, + {"id": "ratio", "expression": "cell(nodes.fetch, 0, 'gmv') / 100"}] + model = script(monkeypatch, [{"op": "update_node", "id": "card", "config": {"metrics": card}}]) + wf = await create(client, compute_feeding()) + fid = "governed.caliber_compute_input:calc" + out = (await client.post(f"/api/workflows/{wf}/autofix", json={ + "level": "governed", "apply": [fid], "choices": {fid: HANDOFF}})).json() + assert out["handoff"] == [fid] and len(model.calls) == 1 + human = next(text for role, text in model.calls[0] if role == "human") + assert "「计算」" in human and "evidence_role" in human + assert out["assist"]["ok"] is True and out["applied"] == ["assist"] and out["ok"] is True, out + assert "evidence_role" not in config(out["graph"], "calc") + # 没选「交给 Copilot」、也没带 assist 的,不调模型 + model.calls.clear() + out = (await client.post(f"/api/workflows/{wf}/autofix", json={ + "level": "governed", "apply": [fid], "choices": {fid: "source"}})).json() + assert model.calls == [] and out["assist"] is None and out["applied"] == [fid] + + +@pytest.mark.parametrize("ops, word", [ + ([{"op": "update_node", "id": "calc", "config": {"evidence_role": "source"}}], "evidence_role"), + ([{"op": "update_node", "id": "write", "config": {"numbers": "off"}}, + {"op": "update_node", "id": "calc", "config": {"evidence_role": "source"}}], "numbers"), +]) +async def test_copilot_cannot_decide_for_the_author_or_loosen_the_report(client, monkeypatch, ops, word): + script(monkeypatch, ops) + wf = await create(client, compute_feeding()) + out = (await client.post(f"/api/workflows/{wf}/autofix", json={ + "level": "governed", "apply": [], "assist": True})).json() + assert out["assist"]["ok"] is False and out["applied"] == [] + [refused] = [r for r in out["rejected"] if r["fix_id"] == "assist"] + assert "降低了要求" in refused["reason"] and word in refused["reason"] + assert out["graph"] == (await client.get(f"/api/workflows/{wf}")).json()["graph"] diff --git a/backend/tests/test_autofix_graph_defaults.py b/backend/tests/test_autofix_graph_defaults.py index b2770d66..ad2863ee 100644 --- a/backend/tests/test_autofix_graph_defaults.py +++ b/backend/tests/test_autofix_graph_defaults.py @@ -39,7 +39,8 @@ def weekly(default: str | None = "never", **ask_config) -> dict: node("ask", "agent", "查数员", **ask), node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[ {"id": "gmv", "name": "销售额", "unit": "元", "expression": "vars.gmv"}]), - node("write", "report", "报告撰写", instructions="写周报"), + node("write", "report", "报告撰写", instructions="写周报", + numbers="strict", on_violation="fail", claims="require_citation"), node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], contract={"report_from": "write", "metrics_from": ["card"], "required": ["gmv"], "strict": True}), ] diff --git a/backend/tests/test_autofix_rules.py b/backend/tests/test_autofix_rules.py index 2c36caf0..a6276e82 100644 --- a/backend/tests/test_autofix_rules.py +++ b/backend/tests/test_autofix_rules.py @@ -32,7 +32,8 @@ def weekly(**contract_overrides) -> dict: node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[ {"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}, {"id": "orders", "name": "订单数", "unit": "单", "expression": "cell(nodes.fetch, 0, 'orders')"}]), - node("write", "report", "报告撰写", instructions="写周报"), + node("write", "report", "报告撰写", instructions="写周报", + numbers="strict", on_violation="fail", claims="require_citation"), node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], contract={k: v for k, v in contract.items() if v is not None}), ] @@ -70,7 +71,8 @@ def add_card(graph: dict, nid: str = "card2") -> dict: def add_report(graph: dict, nid: str = "write2") -> dict: - graph["nodes"].insert(4, node(nid, "report", "第二份报告", instructions="写月报")) + graph["nodes"].insert(4, node(nid, "report", "第二份报告", instructions="写月报", + numbers="strict", on_violation="fail", claims="require_citation")) graph["edges"] += [{"source": "card", "target": nid}, {"source": nid, "target": "done"}] return graph @@ -174,17 +176,17 @@ def test_a_wrong_report_from_takes_the_only_upstream_report(): assert out["ok"] is True and config(out["graph"], "done")["contract"]["report_from"] == "write" -def test_without_a_report_node_the_narrative_is_a_choice_over_model_nodes(): +def test_without_a_report_node_report_from_goes_to_copilot(): + # 以前这里让人在模型节点里选一段 narrative。受管级别的叙述模式过不了门禁 G1 + # (governed.report_from_required),选了也发不出去:没有报告撰写节点就交给 Copilot 加一个 graph = weekly(report_from=None) write = find(graph, "write") write.update(type="llm", data={"label": "写周报", "config": {"prompt": "写周报", "assign_to": "report"}}) fid = "contract.report_from_missing:done" fix = plan(graph)[fid] - assert fix["kind"] == "choice" and fix["field"] == "contract.narrative" - assert [o["value"] for o in fix["options"]] == ["{{ nodes.write.text }}"] + assert fix["kind"] == "assist" and fix["field"] == "contract.report_from" out = apply_fixes(graph, [fid], {fid: "{{ nodes.write.text }}"}, level="governed") - assert config(out["graph"], "done")["contract"]["narrative"] == "{{ nodes.write.text }}" - assert "contract.report_from_missing" not in codes(out["remaining"]) + assert out["applied"] == [] and "narrative" not in config(out["graph"], "done")["contract"] def test_strict_off_is_turned_on(): diff --git a/backend/tests/test_contract_claims.py b/backend/tests/test_contract_claims.py new file mode 100644 index 00000000..b3e9bd79 --- /dev/null +++ b/backend/tests/test_contract_claims.py @@ -0,0 +1,360 @@ +"""出口契约(引用模式)按报告节点的 claims 和可疑实体判档。 + +- claims: require_citation:没挂依据的结论句计入缺口,按 decide_tier 的 claims_policy 降档(不是不予出具, + strict 也一样);数的是出口重新核对出来的(verify_doc 的 uncited),不信文档里记的 cites。off(默认) + 照旧不管。升级前发起的运行不管图里写没写,一律按 off +- 可疑实体(unknown_entity):只在受管级别的正式运行里按「有缺口」降档,不拦截;探索运行、已发布级别 + 只在 _issuance.entities 里标注。表结构快照不全时核对不了的名字(unverified_entity)在哪都只标注 +- 契约自己写 claims: require_citation 的同样生效;写 judge 的仍是后续版本,照实记缺口 +- 契约只能收紧、不能放松:报告节点要求了,契约写 ignore / off 照旧降档,写 withhold 收紧成不予出具; + 受管级别的正式运行里 ignore 一律当 degrade(没有哪道门禁查契约里的 claims) +""" +from __future__ import annotations + +import asyncio +import sqlite3 +import uuid +from types import SimpleNamespace + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage +from sqlalchemy import select + +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource, Run, RunEvent, Workflow, WorkflowVersion +from app.engine import runner as runner_mod +from app.engine.runner import run_manager +from app.main import app + +SOURCE = "warehouse" +TOOL = f"db_query__{SOURCE}" +SQL = "SELECT region, SUM(amount) AS gmv, COUNT(*) AS order_cnt FROM orders GROUP BY region ORDER BY gmv DESC" +CLEAN = "东区销售额 [[m:gmv]],来自 `orders`。[[see:m:gmv]]" +UNCITED = CLEAN + "增长主要来自新客。" +SUSPICIOUS = "东区销售额 [[m:gmv]],来自 `orders`,`refund_log` 另算。[[see:m:gmv]]" + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def _warehouse(tmp_path, *, truncated: bool = False): + from app.data.introspect import introspect + + path = tmp_path / "warehouse.db" + db = sqlite3.connect(path) + db.executescript( + "CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, amount REAL);" + "CREATE TABLE refunds (id INTEGER PRIMARY KEY, order_id INTEGER, amount REAL);") + db.executemany("INSERT INTO orders VALUES (?, ?, ?)", [(1, "east", 100.5), (2, "west", 200.0), (3, "east", 300.0)]) + db.commit() + db.close() + probe = SimpleNamespace(id=f"probe-{path.name}", name=SOURCE, kind="sqlite", database=str(path), host=None, + port=None, username=None, password=None, options={}, readonly=True, schema_cache={}) + cache = await introspect(probe) + if truncated: + # 库里的表太多、探查只存了一部分:快照不全,找不到的名字只能说核对不了 + cache.update(truncated=True, total=500) + await engines.invalidate(probe.id) + async with SessionLocal() as session: + row = (await session.execute(select(DataSource).where(DataSource.name == SOURCE))).scalar_one_or_none() + if row is None: + row = DataSource(name=SOURCE, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + row.database, row.kind, row.enabled, row.readonly, row.schema_cache = str(path), "sqlite", True, True, cache + row.options = {} + await session.commit() + source_id = row.id + await engines.invalidate(source_id) + return source_id + + +@pytest.fixture +async def warehouse(tmp_path): + source_id = await _warehouse(tmp_path) + yield source_id + await engines.invalidate(source_id) + + +@pytest.fixture +async def partial_warehouse(tmp_path): + source_id = await _warehouse(tmp_path, truncated=True) + yield source_id + await engines.invalidate(source_id) + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def weekly(*, report=None, contract=None): + nodes = [ + node("start", "input"), + node("fetch", "tool", tool=TOOL, args={"sql": SQL}), + node("card", "metrics", caliber="周报口径", caliber_version="v2", metrics=[ + {"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "format": "thousands", + "expression": "cell(nodes.fetch, 0, 'gmv')"}]), + node("write", "report", instructions="写周报", on_violation="flag", max_repairs=0, **(report or {})), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], + contract={"report_from": "write", "metrics_from": ["card"], "required": ["gmv"], "strict": True, + **(contract or {})}), + ] + return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +def script(monkeypatch, text): + from app.providers import mock_model + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=text)) + + +async def wait(run_id: str) -> Run: + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + return row + raise AssertionError(f"run {run_id} 没跑完:{row.status}") + + +async def explore(graph) -> dict: + run = await run_manager.start(graph=graph, input_payload={}) + row = await wait(run.id) + assert row.status == "succeeded", row.error + return row.output["_issuance"] + + +async def formal(client, graph, *, status: str) -> Run: + """把这张图发布成 status 级别,再从发布版本发起正式运行。""" + async with SessionLocal() as session: + wf = Workflow(name=f"claims-{status}-{uuid.uuid4().hex[:6]}", graph=graph, status=status, published_version=1) + session.add(wf) + await session.flush() + session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph)) + await session.commit() + wf_id = wf.id + res = await client.post("/api/runs", json={"workflow_id": wf_id, "run_class": "formal"}) + assert res.status_code == 201, res.text + row = await wait(res.json()["id"]) + assert row.status == "succeeded", row.error + return row + + +async def issuance_event(run_id: str) -> dict: + async with SessionLocal() as session: + [event] = (await session.execute(select(RunEvent).where( + RunEvent.run_id == run_id, RunEvent.type == "issuance"))).scalars() + return event.data + + +# -------------------------------------------------------------------------- +# claims +# -------------------------------------------------------------------------- + + +async def test_require_citation_degrades_uncited_claims(warehouse, monkeypatch): + script(monkeypatch, UNCITED) + issuance = await explore(weekly(report={"claims": "require_citation"})) + assert issuance["tier"] == "degraded", "strict 下也只是降档:结论句没挂依据不是数没有出处" + claims = issuance["claims"] + assert claims["policy"] == "require_citation" and claims["uncited_claims"] == 1 + [item] = claims["uncited"] + assert item["text"] == "增长主要来自新客。" and item["unit"] and len(item["span"]) == 2 + assert issuance["unmatched_numbers"] == [] and issuance["unresolved"] == [] + assert not any("结论句" in g for g in issuance["gaps"]), "判了就不是「没核」" + + +async def test_require_citation_with_every_claim_cited_is_formal(warehouse, monkeypatch): + script(monkeypatch, CLEAN) + issuance = await explore(weekly(report={"claims": "require_citation"})) + assert issuance["tier"] == "formal" + assert issuance["claims"] == {"policy": "require_citation", "uncited_claims": 0, "uncited": []} + + +async def test_claims_off_changes_nothing(warehouse, monkeypatch): + script(monkeypatch, UNCITED) + issuance = await explore(weekly()) + assert issuance["tier"] == "formal" and "claims" not in issuance + + +async def test_an_auto_linked_name_does_not_count_as_a_citation(warehouse, monkeypatch): + """只提到表名的结论句仍算没挂依据:自动链接的名字是系统加的标注,不是写作者给的出处。""" + script(monkeypatch, CLEAN + "增长主要来自 `orders` 里的新客。") + issuance = await explore(weekly(report={"claims": "require_citation"})) + assert issuance["tier"] == "degraded" and issuance["claims"]["uncited_claims"] == 1 + + +async def test_runs_from_before_the_upgrade_ignore_claims(warehouse, monkeypatch): + async def no_snapshot(session): + return None + + monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot) + script(monkeypatch, UNCITED) + issuance = await explore(weekly(report={"claims": "require_citation"})) + assert issuance["tier"] == "formal" and "claims" not in issuance and "entities" not in issuance + + +async def test_the_contract_can_ask_for_citations_too(warehouse, monkeypatch): + script(monkeypatch, UNCITED) + issuance = await explore(weekly(contract={"claims": {"policy": "require_citation", "on_uncited": "withhold"}})) + assert issuance["tier"] == "withheld" and issuance["claims"]["uncited_claims"] == 1 + script(monkeypatch, CLEAN) + issuance = await explore(weekly(contract={"claims": "require_citation"})) + assert issuance["tier"] == "formal" and not any("结论句" in g for g in issuance["gaps"]) + + +LOOSE = {"claims": {"policy": "require_citation", "on_uncited": "ignore"}} + + +async def test_the_contract_can_tighten_the_report_policy_but_never_loosen_it(warehouse, monkeypatch): + """报告节点写了 require_citation:契约写 withhold 能收紧成不予出具;写 ignore、写 off 都放不松,照旧降档。""" + script(monkeypatch, UNCITED) + asks = {"claims": "require_citation"} + issuance = await explore(weekly(report=asks, contract=LOOSE)) + assert issuance["tier"] == "degraded", "契约的 ignore 不能把报告节点要求的缺口抹掉" + assert issuance["claims"]["uncited_claims"] == 1 and "on_uncited" not in issuance["claims"] + issuance = await explore(weekly(report=asks, contract={"claims": "off"})) + assert issuance["tier"] == "degraded" and issuance["claims"]["uncited_claims"] == 1 + issuance = await explore(weekly(report=asks, contract={"claims": {"policy": "require_citation", + "on_uncited": "withhold"}})) + assert issuance["tier"] == "withheld" and issuance["claims"]["on_uncited"] == "withhold" + + +async def test_ignore_cannot_lower_the_governed_floor(client, warehouse, monkeypatch): + """受管级别的正式运行:契约写 on_uncited: ignore 不算数(没有哪道门禁查契约里的 claims),没挂依据的结论句照样 + 计入缺口、降一档。""" + script(monkeypatch, UNCITED) + row = await formal(client, weekly(report={"claims": "require_citation"}, contract=LOOSE), status="governed") + issuance = row.output["_issuance"] + assert issuance["tier"] == "degraded" and issuance["gaps"] == [], issuance + assert issuance["claims"]["uncited_claims"] == 1 and "on_uncited" not in issuance["claims"] + assert (await issuance_event(row.id))["claims"] == {"policy": "require_citation", "uncited_claims": 1} + # 报告节点没写(门禁之前发布的版本)、只有契约写了 ignore:受管级别同样不认 ignore + row = await formal(client, weekly(contract=LOOSE), status="governed") + assert row.output["_issuance"]["tier"] == "degraded" + + +async def test_outside_governed_a_contract_may_ignore_its_own_policy(client, warehouse, monkeypatch): + """报告节点没要求时,契约自己写的 ignore 在探索运行、已发布级别照旧生效:不计入缺口,照实记下 ignore。""" + script(monkeypatch, UNCITED) + issuance = await explore(weekly(contract=LOOSE)) + assert issuance["tier"] == "formal" and issuance["claims"]["on_uncited"] == "ignore" + row = await formal(client, weekly(contract=LOOSE), status="published") + assert row.output["_issuance"]["tier"] == "formal" + assert (await issuance_event(row.id))["claims"] == {"policy": "require_citation", "uncited_claims": 1, + "on_uncited": "ignore"} + + +@pytest.mark.parametrize(("written", "declared", "governed", "effect"), [ + ("require_citation", None, False, "degrade"), + ("require_citation", "off", False, "degrade"), + ("require_citation", {"policy": "require_citation", "on_uncited": "ignore"}, False, "degrade"), + ("require_citation", {"policy": "require_citation", "on_uncited": "degrade"}, False, "degrade"), + ("require_citation", {"policy": "require_citation", "on_uncited": "withhold"}, False, "withhold"), + ("require_citation", {"policy": "require_citation", "on_uncited": "bogus"}, False, "degrade"), + ("off", "require_citation", False, "degrade"), + ("off", {"policy": "require_citation", "on_uncited": "withhold"}, True, "withhold"), + ("off", {"policy": "require_citation", "on_uncited": "ignore"}, False, "ignore"), + ("off", {"policy": "require_citation", "on_uncited": "ignore"}, True, "degrade"), + ("off", {"policy": "require_citation", "on_uncited": "bogus"}, False, "degrade"), + ("off", None, True, None), + ("off", "off", True, None), +]) +def test_the_effective_claims_policy_is_the_stricter_one(written, declared, governed, effect): + """报告节点记下的策略和契约写的取更严的那个;受管级别 ignore 当 degrade;认不出的写法不放松(当 degrade)。""" + from app.engine.issuance import decide_tier + from app.engine.nodes.io import _claims + + out = {"gaps": [], "claims_policy": None, "uncited": None, "claims": None} + uncited = [{"unit": "u1", "span": [0, 4], "text": "增长很快"}] + _claims(out, {"claims": declared} if declared is not None else {}, {"text": "x", "claims": written}, + evidence_on=True, governed=governed, uncited=uncited) + assert out["gaps"] == [] + if effect is None: + assert out["claims_policy"] is None and out["claims"] is None + return + assert out["claims"]["uncited_claims"] == 1 and out["claims"].get("on_uncited", "degrade") == effect + tier = decide_tier(missing_required=[], missing_expected=[], unmatched=[], strict=True, gaps=[], + uncited_claims=out["uncited"], claims_policy=out["claims_policy"]) + assert tier == {"degrade": "degraded", "withhold": "withheld", "ignore": "formal"}[effect] + + +def test_a_run_in_flight_across_the_upgrade_keeps_the_old_gap(): + """升级前写的报告(产出里没有 claims)碰上写了 require_citation 的契约:照旧记「这一版还不支持」,不判档。""" + from app.engine.nodes.io import _claims + + out = {"gaps": [], "claims_policy": None, "uncited": None, "claims": None} + _claims(out, {"claims": "require_citation"}, {"text": "x"}, evidence_on=True, governed=False, + uncited=[{"unit": "u1", "span": [0, 4], "text": "增长很快"}]) + assert out["claims_policy"] is None and out["claims"] is None + assert len(out["gaps"]) == 1 and "这一版还不支持" in out["gaps"][0] + + +# -------------------------------------------------------------------------- +# 可疑实体 +# -------------------------------------------------------------------------- + + +async def test_suspicious_names_are_only_annotated_in_exploratory_runs(warehouse, monkeypatch): + script(monkeypatch, SUSPICIOUS) + issuance = await explore(weekly()) + assert issuance["tier"] == "formal" and issuance["gaps"] == [] + [fake] = issuance["entities"]["unknown"] + assert fake["name"] == "refund_log" and fake["segment"] and fake["unit"] + assert issuance["entities"]["unverified"] == [] and issuance["entities"]["counted"] is False + + +async def test_suspicious_names_degrade_governed_formal_runs(client, warehouse, monkeypatch): + script(monkeypatch, SUSPICIOUS) + row = await formal(client, weekly(), status="governed") + issuance = row.output["_issuance"] + assert issuance["tier"] == "degraded", "按缺口降档,不是不予出具(契约是 strict 的)" + assert any("refund_log" in g and "可疑" in g for g in issuance["gaps"]), issuance["gaps"] + assert issuance["entities"]["counted"] is True and issuance["entities"]["unknown"][0]["name"] == "refund_log" + event = await issuance_event(row.id) + assert event["tier"] == "degraded" and event["entities"] == {"unknown": 1, "unverified": 0, "counted": True} + + +async def test_published_formal_runs_only_annotate(client, warehouse, monkeypatch): + script(monkeypatch, SUSPICIOUS) + issuance = (await formal(client, weekly(), status="published")).output["_issuance"] + assert issuance["tier"] == "formal" and issuance["gaps"] == [] + assert issuance["entities"]["unknown"][0]["name"] == "refund_log" and issuance["entities"]["counted"] is False + + +async def test_names_that_cannot_be_checked_are_only_annotated_even_when_governed(client, partial_warehouse, + monkeypatch): + script(monkeypatch, SUSPICIOUS) + issuance = (await formal(client, weekly(), status="governed")).output["_issuance"] + assert issuance["tier"] == "formal" and issuance["gaps"] == [] + assert issuance["entities"]["unknown"] == [] + assert [u["name"] for u in issuance["entities"]["unverified"]] == ["refund_log"] + + +async def test_the_exit_says_the_report_turned_entities_off(warehouse, monkeypatch): + """entities: off 的报告写了 [[t:]]:出口复核和报告节点一样照实说是这个节点关掉的,不说成「后续版本支持」。""" + from app.engine.evidence import ENTITIES_OFF_REASON, LATER_REASON + + script(monkeypatch, "东区销售额 [[m:gmv]],见 [[t:orders]]。[[see:m:gmv]]") + issuance = await explore(weekly(report={"entities": "off"})) + [unresolved] = issuance["unresolved"] + assert unresolved["ref"] == "t:orders" and ENTITIES_OFF_REASON in unresolved["message"] + assert LATER_REASON not in unresolved["message"] and "entities" not in issuance + + +async def test_clean_reports_carry_no_entity_annotation(warehouse, monkeypatch): + script(monkeypatch, CLEAN) + issuance = await explore(weekly()) + assert issuance["tier"] == "formal" and "entities" not in issuance diff --git a/backend/tests/test_copilot_entity_guide.py b/backend/tests/test_copilot_entity_guide.py new file mode 100644 index 00000000..fbbfb63b --- /dev/null +++ b/backend/tests/test_copilot_entity_guide.py @@ -0,0 +1,54 @@ +"""Copilot 的节点手册要教三期的写法:表名字段名、知识库原话、结论句策略,以及受管模板的报告节点怎么配。 + +手册里没写的配置,Copilot 就不会用;写错的写法,报告节点会一直被打回。所以这里不只查关键词, +还把手册里举的标记例子交给真正的解析器,确认写法和系统认的是同一套。 +""" +from __future__ import annotations + +import re + +from app.api.copilot import NODE_REFERENCE, GenerateIn, _user_message +from app.engine.evidence import QUOTE_MIN, SUPPORTED_KINDS, parse_markers +from app.engine.schema import REPORT_CLAIMS + + +def _report_section() -> str: + start = NODE_REFERENCE.index("- report:") + return NODE_REFERENCE[start:NODE_REFERENCE.index("\n- output", start)] + + +def test_the_report_entry_teaches_entities_quotes_and_claims(): + report = _report_section() + for words in ( + ["[[t:", "[[c:", "反引号", "可能是编造的名字"], # 表名、字段名 + ["[[q:K1|", "逐字", f"至少 {QUOTE_MIN} 个字", "[[see:K1]]"], # 知识库原话 + ["claims", "require_citation", "off", "judge", "后续版本"], # 结论句策略 + ["entities", "link"], # 实体层开关 + ): + assert all(w in report for w in words), [w for w in words if w not in report] + # 结论句策略的取值和校验认的是同一份 + assert all(value in report for value in REPORT_CLAIMS) + + +def test_the_markers_in_the_guide_parse_as_the_system_reads_them(): + kinds = {m["kind"] for m in parse_markers(_report_section())} + assert {"t", "c", "q", "m", "v", "table", "i", "see"} <= kinds + assert kinds <= SUPPORTED_KINDS + [quote] = [m for m in parse_markers(_report_section()) if m["kind"] == "q"] + assert quote["ref"] == "K1" and quote["quote"] + + +def test_governed_templates_spell_out_the_report_settings(): + ref = NODE_REFERENCE + governed = ref[ref.index("受管"):] + for words in (['numbers: "strict"', 'on_violation: "fail"', 'claims: "require_citation"'], + ["report_from", "narrative"], ["cite_fields: true"], ["evidence_role", "source"]): + assert all(w in governed for w in words), [w for w in words if w not in governed] + + +def test_the_answer_branch_mentions_names_and_quotes(): + text = _user_message(GenerateIn(instruction="上周哪张表的退款最多", intent="answer"), patch=False) + assert "[[t:" in text and "反引号" in text and "编造" in text + assert "[[q:K1|" in text and "逐字" in text + # 问数是探索运行:不要替人打开结论句策略 + assert not re.search(r"claims\s*[:=]\s*['\"]?require_citation", text) diff --git a/backend/tests/test_evidence_audit.py b/backend/tests/test_evidence_audit.py new file mode 100644 index 00000000..56011a97 --- /dev/null +++ b/backend/tests/test_evidence_audit.py @@ -0,0 +1,418 @@ +"""证据图里的表、字段、检索,和记录页审计表用的审计视图(可以导出 JSON / CSV)。 + +- 证据图的 evidence 列出报告用到的表、字段、知识库检索;它们算不算「已封存」要看背后的 + 表结构快照、查询快照、检索快照是不是封存范围内的事件交回过的 +- 审计视图把报告里每个有状态的片段按状态分组:有出处 / 无证据 / 可疑实体 / 旧运行猜测。 + 没挂依据的结论句、没有可画线文字的违规(结构片段里的数字、粗体里的可疑名字)也各占一行, + 键盘用户在表里能看全 +- 附上封存核对的结果;CSV 表头是中文,字段里的逗号、引号、换行按 RFC 4180 转义, + 以 = + - @ 开头的文字(不是数)前面加一个 ',免得被电子表格当成公式;每行末尾一列「封存核对」, + 文件存下来以后也看得出封存链是好的、断的还是没封存 +- 没挂依据的结论句那一行说的「计入缺口」按出具时实际生效的策略(_issuance.claims)说,和出具横幅一致 +""" +from __future__ import annotations + +import asyncio +import csv +import io +import sqlite3 +from types import SimpleNamespace + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage +from sqlalchemy import select + +from app.core import artifact_store +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource, Run, RunEvent +from app.engine.evidence import iter_segments +from app.engine.runner import run_manager +from app.main import app + +SOURCE = "warehouse" +TOOL = f"db_query__{SOURCE}" +SQL = "SELECT region, SUM(amount) AS gmv, COUNT(*) AS order_cnt FROM orders GROUP BY region ORDER BY gmv DESC" +#: 第一句有出处(反引号里的真表名、单元格),ASCII 逗号和引号是给 CSV 转义用的;第二句有编造的名字 +#: 和裸数字、没挂依据;第三句是没挂依据的结论 +WRITE = ('`orders` 里东区最多, 达 [[v:Q1.r0.gmv]] "含税"。[[see:Q1]]`refund_log` 另算,另有 999 笔待核。' + "增长主要来自新客。") +HEADER = ["分组", "报告节点", "成果字段", "片段", "句子编号", "种类", "原文", "状态", "问题", "引用", "证据", + "证据种类", "证据编号", "工件", "来源节点", "已封存", "句子", "说明", "封存核对"] + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +async def warehouse(tmp_path): + from app.data.introspect import introspect + + path = tmp_path / "warehouse.db" + db = sqlite3.connect(path) + db.executescript( + "CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, amount REAL, created_at TEXT);" + "CREATE TABLE refunds (id INTEGER PRIMARY KEY, order_id INTEGER, amount REAL);") + db.executemany("INSERT INTO orders VALUES (?, ?, ?, ?)", + [(1, "east", 100.5, "2026-09-01"), (2, "west", 200.0, "2026-09-02"), (3, "east", 300.0, "2026-09-03")]) + db.commit() + db.close() + probe = SimpleNamespace(id=f"probe-{path.name}", name=SOURCE, kind="sqlite", database=str(path), host=None, + port=None, username=None, password=None, options={}, readonly=True, schema_cache={}) + cache = await introspect(probe) + await engines.invalidate(probe.id) + async with SessionLocal() as session: + row = (await session.execute(select(DataSource).where(DataSource.name == SOURCE))).scalar_one_or_none() + if row is None: + row = DataSource(name=SOURCE, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + row.database, row.kind, row.enabled, row.readonly, row.schema_cache = str(path), "sqlite", True, True, cache + row.options = {} + await session.commit() + source_id = row.id + await engines.invalidate(source_id) + yield source_id + await engines.invalidate(source_id) + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +def script(monkeypatch, text): + from app.providers import mock_model + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=text)) + + +async def finish(graph) -> Run: + run = await run_manager.start(graph=graph, input_payload={}) + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + return row + raise AssertionError(f"run {run.id} 没跑完:{row.status}") + + +async def events(run_id: str, etype: str, node_id: str | None = None) -> list[RunEvent]: + async with SessionLocal() as session: + q = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype) + if node_id: + q = q.where(RunEvent.node_id == node_id) + return list((await session.execute(q.order_by(RunEvent.seq))).scalars()) + + +async def report_run(monkeypatch, text=WRITE) -> tuple[Run, dict]: + script(monkeypatch, text) + row = await finish(chain(node("start", "input"), node("fetch", "tool", tool=TOOL, args={"sql": SQL}), + node("write", "report", instructions="写一句", max_repairs=0), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.write.text }}"}]))) + assert row.status == "succeeded", row.error + [checked] = await events(row.id, "report.checked", "write") + return row, artifact_store.load(checked.data["doc_artifact"]) + + +def rows_of(body: dict, group: str) -> list[dict]: + return next(g["rows"] for g in body["groups"] if g["key"] == group) + + +# -------------------------------------------------------------------------- +# 证据图:表、字段、检索 +# -------------------------------------------------------------------------- + + +async def test_the_graph_lists_tables_and_columns(client, warehouse, monkeypatch): + row, doc = await report_run(monkeypatch, "`orders` 里东区最多,达 [[v:Q1.r0.gmv]],按 [[c:orders.region]] 汇总。" + "[[see:Q1]]") + [end] = await events(row.id, "tool.end", "fetch") + body = (await client.get(f"/api/runs/{row.id}/evidence")).json() + by_alias = {e["alias"]: e for e in body["evidence"]} + orders = by_alias["t:orders"] + assert orders["kind"] == "table" and orders["name"] == "orders" and orders["sealed"] is True + assert orders["artifact"] == end.data["schema_artifact"] and orders["queries"] == ["Q1"] + assert {s["kind"] for s in orders["sources"]} == {"schema", "sql"} + assert orders["cited_by"] == [next(s["id"] for s in iter_segments(doc) if s.get("ref") == "t:orders")] + region = by_alias["c:orders.region"] + assert region["kind"] == "column" and region["table"] == "orders" and region["sealed"] is True + assert {"from": "t:orders", "to": "Q1", "rel": "appears_in", "report": "write"} in body["edges"] + assert {"from": "c:orders.region", "to": "Q1", "rel": "appears_in", "report": "write"} in body["edges"] + + +async def test_the_graph_lists_retrievals_with_the_quotes_that_cite_them(client, monkeypatch): + from app.memory import kb + + async def fake_search(session, **kw): + return [{"chunk_id": "ch-1", "document_id": "doc-1", "title": "售后月报", "ordinal": 3, "score": 0.9, + "content": "九月退款主要集中在东区,原因是物流延误。"}] + + monkeypatch.setattr(kb, "search", fake_search) + script(monkeypatch, "售后说「[[q:K1|退款主要集中在东区]]」。[[see:K1]]") + row = await finish(chain(node("start", "input"), node("docs", "retrieve", query="退款", collection="kb"), + node("write", "report", instructions="引用原话"), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.write.text }}"}]))) + assert row.status == "succeeded", row.error + [checked] = await events(row.id, "report.checked", "write") + doc = artifact_store.load(checked.data["doc_artifact"]) + quote = next(s for s in iter_segments(doc) if s["kind"] == "quote") + body = (await client.get(f"/api/runs/{row.id}/evidence")).json() + k1 = next(e for e in body["evidence"] if e["alias"] == "K1") + assert k1["kind"] == "retrieval" and k1["sealed"] is True and k1["source"] == "kb" + assert k1["cited_by"] == [quote["id"]] + + audit = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json() + [cited] = rows_of(audit, "cited") + assert cited["kind"] == "quote" and cited["text"] == "退款主要集中在东区" and cited["evidence_kind"] == "quote" + assert cited["alias"] == "K1" and cited["sealed"] is True and cited["artifact"] == k1["artifact"] + + +# -------------------------------------------------------------------------- +# 审计视图 +# -------------------------------------------------------------------------- + + +async def test_the_audit_groups_every_stateful_piece(client, warehouse, monkeypatch): + row, doc = await report_run(monkeypatch) + body = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json() + assert body["run_id"] == row.id and body["schema"] == "agentlab.evidence.audit/1" and body["mode"] == "cited" + assert [g["key"] for g in body["groups"]] == ["cited", "none", "suspicious", "candidate"] + assert [g["label"] for g in body["groups"]] == ["有出处", "无证据", "可疑实体", "旧运行猜测"] + assert body["counts"] == {g["key"]: len(g["rows"]) for g in body["groups"]} + assert body["total"] == sum(body["counts"].values()) + + cited = rows_of(body, "cited") + assert [(r["kind"], r["text"]) for r in cited] == [("entity", "`orders`"), ("number", "400.5")] + cell = cited[1] + assert cell["ref"] == "v:Q1.r0.gmv" and cell["evidence"] == "Q1.r0.gmv" and cell["evidence_kind"] == "cell" + assert cell["node_id"] == "fetch" and cell["sealed"] is True and cell["state"] == "deterministic" + assert cell["segment"] and cell["unit"] and doc["markdown"][slice(*cell["span"])] == "400.5" + assert cited[0]["evidence_kind"] == "table" and cited[0]["sealed"] is True + + # 组内按在正文里的位置排:没挂依据的句子从句首算 + none = rows_of(body, "none") + assert [(r["kind"], r["issue"]) for r in none] == [ + ("claim", "uncited_claim"), ("number", "uncited_number"), ("claim", "uncited_claim")] + assert none[1]["text"] == "999" and none[1]["state"] == "none" + assert [none[0]["text"], none[2]["text"]] == ["`refund_log` 另算,另有 999 笔待核。", "增长主要来自新客。"] + assert none[0]["segment"] is None and none[0]["unit"] and none[0]["sentence"] == none[0]["text"] + + [fake] = rows_of(body, "suspicious") + assert fake["kind"] == "entity" and fake["issue"] == "unknown_entity" and fake["text"] == "`refund_log`" + assert "可能是编造的名字" in fake["note"] + assert rows_of(body, "candidate") == [] + + seal = body["seal"] + assert seal["sealed"] is True and seal["ok"] is True and seal["manifest_seq"] == row.manifest_seq + assert seal["events"] > 0 and seal["message"] == "事件流与封存时一致" + [report] = body["reports"] + assert report["node_id"] == "write" and report["hash_ok"] is True and report["doc_sealed"] is True + assert report["claims"] == "off" and report["stats"]["unknown_entities"] == 1 + + +async def test_violations_without_a_row_of_their_own_are_listed(client, warehouse, monkeypatch): + """粗体里的可疑名字、列表序号里的数字没有可以画线的片段,只在违规清单里:审计表照样各给一行。""" + row, _ = await report_run(monkeypatch, "**`refund_log`** 另算。\n\n100. 东区最多,达 [[v:Q1.r0.gmv]]。[[see:Q1]]") + body = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json() + [fake] = rows_of(body, "suspicious") + assert (fake["kind"], fake["issue"], fake["text"]) == ("violation", "unknown_entity", "refund_log") + assert fake["segment"] and "可能是编造的名字" in fake["note"] + [bare] = rows_of(body, "none") + assert (bare["kind"], bare["issue"], bare["text"]) == ("violation", "uncited_number", "100") + assert [r["text"] for r in rows_of(body, "cited")] == ["400.5"] + + +async def test_the_audit_can_show_only_some_groups(client, warehouse, monkeypatch): + row, _ = await report_run(monkeypatch) + body = (await client.get(f"/api/runs/{row.id}/evidence/audit", params={"groups": "none,suspicious"})).json() + assert [g["key"] for g in body["groups"]] == ["none", "suspicious"] + assert body["total"] == 4 + bad = await client.get(f"/api/runs/{row.id}/evidence/audit", params={"groups": "none,nope"}) + assert bad.status_code == 422 and bad.json()["code"] == "evidence_bad_group" + + +async def test_the_audit_exports_json_and_csv(client, warehouse, monkeypatch): + row, _ = await report_run(monkeypatch) + plain = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json() + + as_json = await client.get(f"/api/runs/{row.id}/evidence/audit", params={"format": "json"}) + assert as_json.status_code == 200 and as_json.json() == plain + assert "attachment" in as_json.headers["content-disposition"] + assert as_json.headers["content-disposition"].endswith('.json"') + + as_csv = await client.get(f"/api/runs/{row.id}/evidence/audit", params={"format": "csv"}) + assert as_csv.status_code == 200 and as_csv.headers["content-type"].startswith("text/csv") + assert as_csv.headers["content-disposition"].endswith('.csv"') + assert as_csv.headers["x-evidence-seal"] == "ok" + raw = as_csv.content.decode("utf-8") + assert raw.startswith(""), "Excel 要 BOM 才认 UTF-8" + table = list(csv.reader(io.StringIO(raw[1:]))) + assert table[0] == HEADER + flat = [r for g in plain["groups"] for r in g["rows"]] + assert len(table) - 1 == len(flat) == plain["total"] + cell = next(line for line in table[1:] if line[6] == "400.5") + assert cell[0] == "有出处" and cell[7] == "有出处" and cell[15] == "是" and cell[10] == "Q1.r0.gmv" + # 句子里的 ASCII 逗号和双引号原样读回来 + assert cell[16] == '`orders` 里东区最多, 达 400.5 "含税"。' + fake = next(line for line in table[1:] if line[8] == "可能是编造的名字") + assert fake[0] == "可疑实体" and fake[6] == "`refund_log`" + # 存下来的文件自己带着封存核对的结果(响应头离了浏览器就没了) + assert {line[18] for line in table[1:]} == {f"封存核对通过(封存于第 {row.manifest_seq} 条事件)"} + + bad = await client.get(f"/api/runs/{row.id}/evidence/audit", params={"format": "xlsx"}) + assert bad.status_code == 422 and bad.json()["code"] == "evidence_bad_format" + + +def test_csv_escapes_commas_quotes_newlines_and_formulas(): + from app.api.evidence import audit_csv + + rows = [{"group": "none", "text": 'a,b "c"\nd', "sentence": "=HYPERLINK(\"x\")", "note": "-3.2%", + "evidence": "+cmd", "ref": "@x", "sealed": None, "kind": "claim"}] + raw = audit_csv(rows) + assert raw.startswith("") and "\r\n" in raw + [header, line] = list(csv.reader(io.StringIO(raw[1:]))) + assert header == HEADER + got = dict(zip(header, line)) + assert got["原文"] == 'a,b "c"\nd' + assert got["句子"] == "'=HYPERLINK(\"x\")" and got["证据"] == "'+cmd" and got["引用"] == "'@x" + assert got["说明"] == "-3.2%", "负数不是公式,不加引号" + assert got["已封存"] == "—" and got["分组"] == "无证据" and got["种类"] == "结论句" + assert got["封存核对"] == "", "不给封存结论就空着,不瞎填" + [_, line] = list(csv.reader(io.StringIO(audit_csv(rows, seal="=封存核对没通过")[1:]))) + assert line[-1] == "'=封存核对没通过", "封存那一格同样过公式防护" + + +async def test_a_broken_seal_marks_every_row_unsealed(client, warehouse, monkeypatch): + row, _ = await report_run(monkeypatch) + async with SessionLocal() as session: + started = (await session.execute(select(RunEvent).where( + RunEvent.run_id == row.id, RunEvent.type == "node.started", RunEvent.node_id == "start"))).scalar_one() + started.data = {**(started.data or {}), "note": "事后改过"} + await session.commit() + body = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json() + assert body["seal"]["ok"] is False and "对不上" in body["seal"]["message"] + assert all(r["sealed"] is False for r in rows_of(body, "cited")) + as_csv = await client.get(f"/api/runs/{row.id}/evidence/audit", params={"format": "csv"}) + assert as_csv.headers["x-evidence-seal"] == "broken" + table = list(csv.reader(io.StringIO(as_csv.content.decode("utf-8")[1:]))) + assert {line[15] for line in table[1:] if line[0] == "有出处"} == {"否"} + assert {line[18] for line in table[1:]} == {"封存核对没通过:封存之后有事件被改过、删过或插过"} + + +async def test_a_run_waiting_for_approval_is_audited_as_unsealed(client, warehouse, monkeypatch): + script(monkeypatch, WRITE) + run = await run_manager.start(graph=chain( + node("start", "input"), node("fetch", "tool", tool=TOOL, args={"sql": SQL}), + node("write", "report", instructions="写一句", max_repairs=0), + node("gate", "human", mode="approve", title="确认"), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.write.text }}"}])), input_payload={}) + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("interrupted", "failed"): + break + assert row.status == "interrupted", row.error + body = (await client.get(f"/api/runs/{run.id}/evidence/audit")).json() + assert body["mode"] == "cited" and body["seal"]["sealed"] is False and body["seal"]["ok"] is None + assert rows_of(body, "cited") and all(r["sealed"] is False for r in rows_of(body, "cited")) + as_csv = await client.get(f"/api/runs/{run.id}/evidence/audit", params={"format": "csv"}) + assert as_csv.headers["x-evidence-seal"] == "unsealed" + table = list(csv.reader(io.StringIO(as_csv.content.decode("utf-8")[1:]))) + assert len(table) > 1 and {line[18] for line in table[1:]} == {"未封存:运行还没跑完,或者停在人工审批"} + + +CLAIMED = "`orders` 里东区最多,达 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客。" + + +async def claimed_run(monkeypatch, *, report=None, contract=None) -> Run: + """一句有出处、一句没挂依据的结论;contract 为 None 时出口不带契约。""" + script(monkeypatch, CLAIMED) + card = node("card", "metrics", caliber="周报口径", caliber_version="v1", metrics=[ + {"id": "gmv", "name": "销售额", "decimals": 1, "expression": "cell(nodes.fetch, 0, 'gmv')"}]) + out = node("out", "output", fields=[{"name": "r", "value": "{{ nodes.write.text }}"}], + **({"contract": {"report_from": "write", "metrics_from": ["card"], "required": ["gmv"], **contract}} + if contract is not None else {})) + row = await finish(chain(node("start", "input"), node("fetch", "tool", tool=TOOL, args={"sql": SQL}), card, + node("write", "report", instructions="写一句", max_repairs=0, **(report or {})), out)) + assert row.status == "succeeded", row.error + return row + + +async def test_uncited_claims_say_what_issuance_did_with_them(client, warehouse, monkeypatch): + """结论句那一行说的「计入缺口」要和出具横幅一致:按出具时实际生效的策略(_issuance.claims)说, + 不只看报告节点自己写的 claims。""" + from app.api.evidence import UNCITED_CLAIM, UNCITED_CLAIM_COUNTED, UNCITED_CLAIM_REQUIRED, UNCITED_CLAIM_WITHHELD + + async def claim_notes(row: Run) -> list[str]: + body = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json() + return [r["note"] for r in rows_of(body, "none") if r["kind"] == "claim"] + + # 报告节点没写、契约写了 require_citation:出具因为这句降档,审计表也得说计入缺口 + row = await claimed_run(monkeypatch, contract={"claims": "require_citation"}) + assert row.output["_issuance"]["tier"] == "degraded" + assert await claim_notes(row) == [UNCITED_CLAIM_COUNTED] + # 契约收紧成 withhold:说的是不予出具 + row = await claimed_run(monkeypatch, contract={"claims": {"policy": "require_citation", "on_uncited": "withhold"}}) + assert row.output["_issuance"]["tier"] == "withheld" + assert await claim_notes(row) == [UNCITED_CLAIM_WITHHELD] + # 探索运行里契约自己写 ignore:出具没算它,审计表也不说计入缺口 + row = await claimed_run(monkeypatch, contract={"claims": {"policy": "require_citation", "on_uncited": "ignore"}}) + assert row.output["_issuance"]["tier"] == "formal" + assert await claim_notes(row) == [UNCITED_CLAIM] + # 报告节点要求了、契约的 ignore 放不松:计入缺口 + row = await claimed_run(monkeypatch, report={"claims": "require_citation"}, + contract={"claims": {"policy": "require_citation", "on_uncited": "ignore"}}) + assert row.output["_issuance"]["tier"] == "degraded" + assert await claim_notes(row) == [UNCITED_CLAIM_COUNTED] + # 报告节点要求了、但出口没有契约:没有哪次出具按它判档,不能说「出具时计入缺口」 + row = await claimed_run(monkeypatch, report={"claims": "require_citation"}) + assert "_issuance" not in (row.output or {}) + assert await claim_notes(row) == [UNCITED_CLAIM_REQUIRED] + # 都没要求 + row = await claimed_run(monkeypatch, contract={}) + assert await claim_notes(row) == [UNCITED_CLAIM] + + +async def test_the_contract_policy_only_speaks_for_the_report_it_checked(client, warehouse, monkeypatch): + """两份报告,契约只核对 write:另一份(side)没挂依据的结论句不能跟着说「出具时计入缺口」。""" + from app.api.evidence import UNCITED_CLAIM, UNCITED_CLAIM_COUNTED + + script(monkeypatch, CLAIMED) + card = node("card", "metrics", caliber="周报口径", caliber_version="v1", metrics=[ + {"id": "gmv", "name": "销售额", "decimals": 1, "expression": "cell(nodes.fetch, 0, 'gmv')"}]) + row = await finish(chain( + node("start", "input"), node("fetch", "tool", tool=TOOL, args={"sql": SQL}), card, + node("write", "report", instructions="写一句", max_repairs=0), + node("side", "report", instructions="再写一句", max_repairs=0), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.write.text }}"}, + {"name": "s", "value": "{{ nodes.side.text }}"}], + contract={"report_from": "write", "metrics_from": ["card"], "claims": "require_citation"}))) + assert row.status == "succeeded", row.error + body = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json() + notes = {r["report"]: r["note"] for r in rows_of(body, "none") if r["kind"] == "claim"} + assert notes == {"write": UNCITED_CLAIM_COUNTED, "side": UNCITED_CLAIM} + + +async def test_runs_without_evidence_say_so(client): + row = await finish(chain(node("start", "input", fields=[{"name": "q", "default": "x"}]), + node("out", "output", fields=[{"name": "r", "value": "{{ input.q }}"}]))) + body = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json() + assert body["mode"] == "none" and body["total"] == 0 and body["note"] + resp = await client.get("/api/runs/no-such-run/evidence/audit") + assert resp.status_code == 404 and resp.json()["code"] == "run_not_found" diff --git a/backend/tests/test_evidence_entities.py b/backend/tests/test_evidence_entities.py new file mode 100644 index 00000000..b4df237b --- /dev/null +++ b/backend/tests/test_evidence_entities.py @@ -0,0 +1,685 @@ +"""实体层:报告里写的表名、字段名能点开出处,写了不存在的名字会被标出来。 + +以前 `refunds` 这种写在反引号里的名字,报告不管它存不存在:模型编一个听起来很像的表名, +读者无从分辨。三期: + +- 数据源工具每次查询都把当时的表结构冻结成 schema_snapshot 工件(内容寻址),台账记一条 + schema 条目;查询条目记下 SQL 里 FROM / JOIN 的表(注释和字符串里的不算) +- 目录编出表条目 t:<表> 和列条目 c:<表>.<列>,来源是表结构快照、SQL 用到的表、查询结果的列 +- [[t:]] [[c:]] 标记能解析,显示名字本身;正文里的名字按保守规则自动链接 +- 反引号里写了像标识符的名字、却哪里都找不到的,标成可疑实体(state none,issue + unknown_entity),记违规但不拦——报告节点不为它失败、不为它重写 +- 流式渲染和整篇渲染一致 +""" +from __future__ import annotations + +import asyncio +import json +import random +import sqlite3 +from types import SimpleNamespace + +import pytest +from langchain_core.messages import AIMessage, ToolMessage +from sqlalchemy import select + +from app.core import artifact_store +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource, Run, RunEvent +from app.engine import runner as runner_mod +from app.engine.evidence import ( + ENTITIES_OFF_REASON, + LATER_REASON, + StreamRenderer, + build_catalog, + catalog_prompt, + compose_doc, + iter_segments, + make_eid, + query_entry_fields, + render_markers, + schema_entry_fields, + sql_tables, + uncited_claims, + verify_doc, +) +from app.engine.runner import run_manager + +SOURCE = "warehouse" +TOOL = f"db_query__{SOURCE}" +SQL = "SELECT region, SUM(amount) AS gmv, COUNT(*) AS order_cnt FROM orders GROUP BY region ORDER BY gmv DESC" + +SCHEMA_ART = "5" * 64 +Q_ART = "a" * 64 +TOOL_SNAP = "b" * 64 +QUERY = {"kind": "query", "node_id": "fetch", "exec": 1, "artifact": Q_ART, "via": TOOL_SNAP, "tool": TOOL, + "source": SOURCE, "columns": ["region", "gmv", "order_cnt"], "rows": 2, "truncated": False, + "schema_artifact": SCHEMA_ART, "tables": ["orders"]} +SCHEMA = {"kind": "schema", "node_id": "fetch", "exec": 1, "artifact": SCHEMA_ART, "source": SOURCE, + "tables": {"orders": ["id", "region", "amount", "created_at", "name"], + "refunds": ["id", "order_id", "amount"]}} + + +@pytest.fixture +def catalog(): + return build_catalog(nodes={}, ledger=[QUERY, SCHEMA], inputs={"week": "2026-W37"}) + + +def entity_segments(doc): + return [s for s in iter_segments(doc) if s["kind"] == "entity"] + + +def codes(doc): + return [v["code"] for v in doc["violations"]] + + +# -------------------------------------------------------------------------- +# 从 SQL 里抽表名:注释、字符串里的不算 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("sql, tables", [ + ("-- FROM ghost_a\nSELECT o.region, 'JOIN ghost_b' AS note /* FROM ghost_c */ FROM orders o " + "LEFT JOIN \"refunds\" r ON r.order_id = o.id WHERE o.name <> 'FROM ghost_d'", ["orders", "refunds"]), + ("WITH recent AS (SELECT * FROM orders), top AS (SELECT * FROM recent) " + "SELECT * FROM top JOIN ANALYTICS.v_demo_table v ON 1 = 1", ["orders", "ANALYTICS.v_demo_table"]), + ("SELECT * FROM orders a, refunds AS b, `sales_daily` WHERE a.id = b.order_id", ["orders", "refunds", "sales_daily"]), + ("SELECT * FROM (SELECT id FROM orders) t JOIN refunds USING (id)", ["orders", "refunds"]), + ("SELECT * FROM generate_series(1, 3)", []), + ("select * from Orders join orders on 1=1", ["Orders"]), + ("SELECT 'it''s FROM ghost' AS x FROM orders", ["orders"]), + # 字符串里的 -- 和 /* 不是注释:先认字符串、再认注释,一趟扫完 + ("SELECT '--x' AS a FROM orders", ["orders"]), + ("SELECT '/*' AS a FROM orders WHERE note <> '*/ JOIN ghost_e'", ["orders"]), + ("SELECT \"a--b\" FROM orders JOIN refunds ON 1 = 1", ["orders", "refunds"]), + # 引号括起来的名字里的 FROM 不是关键字(MySQL 默认把 "…" 当字符串) + ("SELECT \"FROM ghost_f\" AS x FROM orders", ["orders"]), + # 反斜杠转义的引号(MySQL、PostgreSQL 的 E'…'):宁可漏认一张表,不能把字符串里的名字当表 + ("SELECT 'it\\'s FROM ghost_g' AS x FROM orders", ["orders"]), + ("SELECT $$ FROM ghost_h $$ AS a, $tag$ JOIN ghost_i $tag$ FROM orders WHERE id = $1", ["orders"]), + ("/* a /* nested */ FROM ghost_j */ SELECT * FROM orders", ["orders"]), + ("# FROM ghost_k\nSELECT * FROM orders", ["orders"]), + # SQLite 也认的方括号标识符;数组下标、数组字面量里的字符串不是 + ("SELECT * FROM [main].[orders] o JOIN [refunds] r ON r.order_id = o.id", ["main.orders", "refunds"]), + ("SELECT ARRAY['FROM ghost_l'] AS a, tags[1] FROM orders", ["orders"]), + ("", []), +]) +def test_sql_tables_ignore_comments_strings_and_cte_names(sql, tables): + assert sql_tables(sql) == tables + + +def test_query_entry_carries_tables_only_with_a_schema_snapshot(): + """没有表结构快照(数据源没探查过结构)的查询保持二期的形状:只凭 SQL 和结果列核对名字, + 反引号里写的真实表名(没进这条 SQL)会被误标成编造的,这种运行不启用实体核对。""" + payload = {"artifact": Q_ART, "columns": ["gmv"], "rows": [[1]], "sql": SQL, "source": SOURCE} + plain = query_entry_fields(payload) + assert "tables" not in plain and "schema_artifact" not in plain + fields = query_entry_fields(json.dumps({**payload, "schema_artifact": SCHEMA_ART})) + assert fields["schema_artifact"] == SCHEMA_ART and fields["tables"] == ["orders"] + + +# -------------------------------------------------------------------------- +# 目录 +# -------------------------------------------------------------------------- + + +def test_catalog_lists_tables_and_columns_from_all_three_sources(catalog): + orders = catalog["t:orders"] + assert orders["kind"] == "table" and orders["locator"] == {"table": "orders"} + assert orders["eid"] == make_eid("table", SCHEMA_ART, {"table": "orders"}) and orders["artifact"] == SCHEMA_ART + assert {(s["kind"], s.get("alias")) for s in orders["sources"]} == {("schema", None), ("sql", "Q1")} + assert orders["queries"] == ["Q1"] + assert catalog["t:refunds"]["queries"] == [] + + amount = catalog["c:orders.amount"] + assert amount["kind"] == "column" and amount["locator"] == {"table": "orders", "column": "amount"} + assert amount["eid"] == make_eid("column", SCHEMA_ART, {"table": "orders", "column": "amount"}) + # 结果列落在 SQL 用到的表里:同一个条目多一个来源 + region = catalog["c:orders.region"] + assert {(s["kind"], s.get("alias")) for s in region["sources"]} == {("schema", None), ("result", "Q1")} + # 没有表名的结果列(聚合的别名):c:<列>,出处是那份查询快照 + cnt = catalog["c:order_cnt"] + assert cnt["locator"] == {"column": "order_cnt"} and cnt["eid"] == make_eid("column", Q_ART, {"column": "order_cnt"}) + assert cnt["sources"] == [{"kind": "result", "artifact": Q_ART, "alias": "Q1", "source": SOURCE}] + # 同名字段在好几张表里:c:<列> 列出是哪几张 + assert catalog["c:amount"]["tables"] == ["orders", "refunds"] + + +def test_phase_two_ledgers_build_no_entities(): + old = {k: v for k, v in QUERY.items() if k not in ("schema_artifact", "tables")} + catalog = build_catalog(nodes={}, ledger=[old]) + assert not [a for a, e in catalog.items() if e["kind"] in ("table", "column")] + doc = compose_doc("看 `ghost_table`。", catalog) + assert doc["violations"] == [] and not entity_segments(doc) + + +# -------------------------------------------------------------------------- +# 标记 +# -------------------------------------------------------------------------- + + +def test_entity_markers_render_as_the_name(catalog): + doc = compose_doc("数据来自 [[t:orders]] 的 [[c:orders.amount]] 和 [[c:order_cnt]]。[[see:t:refunds]]", catalog) + assert doc["markdown"] == "数据来自 orders 的 orders.amount 和 order_cnt。" + segs = entity_segments(doc) + assert [(s["text"], s["cite"]["kind"], s["state"]) for s in segs] == [ + ("orders", "table", "deterministic"), ("orders.amount", "column", "deterministic"), + ("order_cnt", "column", "deterministic")] + assert segs[0]["cite"]["eid"] == catalog["t:orders"]["eid"] + assert doc["violations"] == [] and doc["stats"]["entities"] == 3 + assert doc["blocks"][0]["units"][0]["cites"] == ["t:orders", "c:orders.amount", "c:order_cnt", "t:refunds"] + + +def test_unknown_marker_is_suspicious_not_unresolved(catalog): + doc = compose_doc("退款看 [[t:refund_log]]。字段写错 [[t:amount]]。", catalog) + fake, wrong = entity_segments(doc) + assert fake["text"] == "refund_log" and fake["state"] == "none" and fake["issue"] == "unknown_entity" + assert "可能是编造的名字" in fake["cite"]["reason"] + # 字段当表写:名字是真的,写法错了——这是解析不了,要重写 + assert wrong["issue"] == "unresolved_ref" and wrong["text"].startswith("⟦?t:") + assert codes(doc) == ["unknown_entity", "unresolved_ref"] + assert doc["stats"]["unknown_entities"] == 1 and doc["stats"]["unresolved"] == 1 + + +def test_entity_markers_without_any_schema_keep_the_phase_one_answer(): + """这次运行没查过库(或者是升级前的运行):没有东西可以核对,t / c 仍按一期判为解析不了。""" + catalog = build_catalog(nodes={}, inputs={"week": "2026-W37"}) + doc = compose_doc("看 [[t:orders]],还有 `ghost_table`。", catalog) + [seg] = [s for s in iter_segments(doc) if s.get("ref")] + assert seg["state"] == "none" and LATER_REASON in seg["cite"]["reason"] + assert codes(doc) == ["unresolved_ref"] # 反引号里的名字不核对 + + +# -------------------------------------------------------------------------- +# 反引号:一律核对 +# -------------------------------------------------------------------------- + + +def test_backticked_names_are_linked_or_flagged(catalog): + doc = compose_doc("看 `orders` 和 `ghost_table`,`refunds.order_id` 与 `SUM(amount)`、`NULL`、`week`。", catalog) + segs = entity_segments(doc) + assert [(s["text"], s["state"], s.get("issue")) for s in segs] == [ + ("`orders`", "deterministic", None), ("`ghost_table`", "none", "unknown_entity"), + ("`refunds.order_id`", "deterministic", None)] + assert segs[0]["ref"] == "t:orders" and segs[0]["code"] is True and segs[0]["auto"] is True + assert segs[2]["cite"]["eid"] == catalog["c:refunds.order_id"]["eid"] + [v] = doc["violations"] + assert v["code"] == "unknown_entity" and v["text"] == "ghost_table" and v["segment"] == segs[1]["id"] + assert doc["markdown"][v["span"][0]:v["span"][1]] == "ghost_table" + assert doc["stats"]["unknown_entities"] == 1 + assert verify_doc(doc, catalog)["violations"] == doc["violations"] + + +def test_verify_finds_suspicious_names_even_if_the_doc_hides_them(catalog): + doc = compose_doc("看 `ghost_table` 的数。", catalog) + for s in iter_segments(doc): + if s.get("issue") == "unknown_entity": + s.update(kind="text", state="neutral") + s.pop("issue") + checked = verify_doc(doc, catalog) + assert [v["code"] for v in checked["violations"]] == ["unknown_entity"] + assert checked["stats"]["unknown_entities"] == 1 + + +def test_names_inside_code_blocks_are_not_checked(catalog): + doc = compose_doc("```sql\nSELECT `ghost_col` FROM ghost_table\n```\n\n正文。", catalog) + assert doc["violations"] == [] and not entity_segments(doc) + + +# -------------------------------------------------------------------------- +# 正文里的裸名字:保守 +# -------------------------------------------------------------------------- + + +def test_bare_names_link_conservatively(catalog): + doc = compose_doc("按 region 和 name 统计,orders 表的 order_id、created_at 与 refunds.amount;" + "amount、id、gmv 不链接,Orders 也不链接,order_cnt 链接。", catalog) + assert [s["text"] for s in entity_segments(doc)] == [ + "orders", "order_id", "created_at", "refunds.amount", "order_cnt"] + assert doc["violations"] == [] + assert all(s["state"] == "deterministic" and s.get("auto") and not s.get("code") for s in entity_segments(doc)) + + +def test_bare_names_inside_links_and_bold_are_left_alone(catalog): + doc = compose_doc("见 [orders 说明](http://example.com/orders) 和 **orders 汇总**。", catalog) + assert not entity_segments(doc) + assert "".join(s["text"] for s in iter_segments(doc)) == doc["markdown"] + + +# -------------------------------------------------------------------------- +# 一致:流式 = 整篇;组装 = 复核 +# -------------------------------------------------------------------------- + + +def test_streaming_equals_full_render(catalog): + text = ("## [[t:orders]]\n\n按 [[c:orders.region]] 看,[[c:order_cnt]] 最多;[[t:refund_log]] 没有。" + "`orders` 与 [[t:amount]] 以及 [[i:week]]。[[see:t:orders]]\n\n- [[c:refunds.order_id]]") + whole = render_markers(text, catalog) + assert "[[" not in whole and "orders" in whole + rng = random.Random(7) + for _ in range(300): + renderer = StreamRenderer(catalog) + out, i = [], 0 + while i < len(text): + step = rng.randint(1, 9) + out.append(renderer.feed(text[i:i + step])) + i += step + out.append(renderer.flush()) + assert "".join(out) == whole + + +def test_compose_and_verify_agree_and_doc_catalog_stays_small(catalog): + doc = compose_doc("[[t:orders]] 里 `refunds` 和 `ghost_table`,order_cnt。", catalog) + checked = verify_doc(doc, catalog) + assert checked["violations"] == doc["violations"] and checked["stats"] == doc["stats"] + kept = {a for a, e in doc["catalog"].items() if e["kind"] in ("table", "column")} + assert kept == {"t:orders", "t:refunds", "c:order_cnt"} # 只留正文里用到的实体 + assert "i:week" in doc["catalog"] and "Q1" in doc["catalog"] + + +# -------------------------------------------------------------------------- +# 表结构快照不全、实体层关掉、老目录:各说各的话 +# -------------------------------------------------------------------------- + + +def test_schema_entry_keeps_truncation(): + content = {"source": SOURCE, "tables": {"orders": {"columns": [{"name": "id"}]}}, "schema": None} + assert schema_entry_fields(content) == {"source": SOURCE, "schema": None, "synced_at": None, + "tables": {"orders": ["id"]}} + assert schema_entry_fields({**content, "truncated": True, "total": 250}) == { + "source": SOURCE, "schema": None, "synced_at": None, "tables": {"orders": ["id"]}, + "truncated": True, "total": 250} + + +def test_a_truncated_schema_cannot_call_a_name_made_up(catalog): + """库里的表太多、快照只存了前 200 张:没列出的名字可能是真的,只能说「核对不了」(软问题,出口 + 不记缺口),不能说「可能是编造的」。已列出的表的字段是全的,表.字段 写错了照样是可疑实体。""" + cut = build_catalog(nodes={}, ledger=[QUERY, {**SCHEMA, "truncated": True, "total": 250}]) + text = "看 `ghost_table`、[[t:ghost_two]]、`orders.ghost_col` 和 `orders`。[[see:c:ghost_three]]" + doc = compose_doc(text, cut) + segs = entity_segments(doc) + assert [(s["text"], s["state"], s.get("issue")) for s in segs] == [ + ("`ghost_table`", "none", "unverified_entity"), ("ghost_two", "none", "unverified_entity"), + ("`orders.ghost_col`", "none", "unknown_entity"), ("`orders`", "deterministic", None)] + assert segs[1]["cite"]["unverified"] is True and "unknown" not in segs[1]["cite"] + # 违规按位置排:句末依据 [[see:]] 的位置记的是整句,排在最前 + assert codes(doc) == ["unverified_entity", "unverified_entity", "unverified_entity", "unknown_entity"] + assert "核对不了" in doc["violations"][0]["message"] and "编造" not in doc["violations"][0]["message"] + assert doc["stats"]["unverified_entities"] == 3 and doc["stats"]["unknown_entities"] == 1 + checked = verify_doc(doc, cut) + assert checked["violations"] == doc["violations"] and checked["stats"] == doc["stats"] + assert "只存了一部分" in catalog_prompt(cut) and "只存了一部分" not in catalog_prompt(catalog) + # 同一篇放到完整的快照下:全都是可疑实体 + assert codes(compose_doc(text, catalog)) == ["unknown_entity"] * 4 + + +def test_old_catalogs_keep_the_old_stats_keys(catalog): + """升级前的运行(目录里没有表、字段、检索):统计的键和以前一模一样,事件一个字不加。""" + old = build_catalog(nodes={}, inputs={"week": "2026-W37"}) + doc = compose_doc("本周 [[i:week]],看 [[t:orders]]。", old) + assert list(doc["stats"]) == ["units", "segments", "claims", "connective", "headings", "numbers", + "numbers_cited", "values", "uncited_numbers", "unresolved", "see", + "uncited_claims", "violations"] + assert verify_doc(doc, old)["stats"] == doc["stats"] + for cat in (catalog, build_catalog(nodes={}, ledger=[{"kind": "retrieval", "node_id": "kb", "exec": 1, + "artifact": "c" * 64, "source": "kb", "rows": 1}])): + stats = compose_doc("本周。", cat)["stats"] + assert {"entities", "quotes", "unknown_entities", "unverified_entities"} <= set(stats) + + +def test_entities_off_says_the_node_turned_it_off(catalog): + """entities: off 是作者有意关掉的:[[t:]] 解析不了的原因照实说,不能说「后续版本支持」。 + 目录里就算还有表和字段,关掉了也不链接、不核对反引号。""" + text = "看 [[t:orders]]、`ghost_table` 和 order_cnt。" + doc = compose_doc(text, catalog, entities=False) + [seg] = [s for s in iter_segments(doc) if s.get("ref")] + assert seg["issue"] == "unresolved_ref" and seg["cite"]["reason"] == ENTITIES_OFF_REASON + assert not entity_segments(doc) or all(s.get("ref") for s in entity_segments(doc)) + assert codes(doc) == ["unresolved_ref"] and ENTITIES_OFF_REASON in doc["violations"][0]["message"] + assert verify_doc(doc, catalog, entities=False)["violations"] == doc["violations"] + stripped = {a: e for a, e in catalog.items() if e["kind"] not in ("table", "column")} + assert compose_doc(text, stripped, entities=False)["violations"] == doc["violations"] + + +# -------------------------------------------------------------------------- +# 自动链接只是标注,不是写作者声明的依据 +# -------------------------------------------------------------------------- + + +def units(doc): + return [u for block in doc["blocks"] for u in block["units"]] + + +@pytest.mark.parametrize("text", ["增长主要来自新客,见 order_id 字段。", "增长主要来自 `orders` 表的新客。", + "增长主要来自新客,见 orders.region。"]) +def test_an_auto_linked_name_is_not_a_citation(catalog, text): + """结论句的依据只认写作者自己写的 [[…]] 和 [[see:…]]:提一个字段名就算挂了依据的话, + claims: require_citation(受管级别必须写)形同虚设。""" + doc = compose_doc(text, catalog) + [unit] = units(doc) + assert entity_segments(doc) and all(s.get("auto") for s in entity_segments(doc)) + assert unit["kind"] == "claim" and unit["cites"] == [] + checked = verify_doc(doc, catalog) + assert [u["unit"] for u in checked["uncited"]] == [unit["id"]] + assert checked["uncited"] == uncited_claims(doc) and checked["stats"]["uncited_claims"] == 1 + # 链接照样在:点得开,文档目录里也留着它的条目(证据接口要用) + for seg in entity_segments(doc): + assert seg["state"] == "deterministic" and seg["cite"]["alias"] in doc["catalog"] + + +def test_a_marker_the_writer_wrote_is_a_citation(catalog): + doc = compose_doc("增长主要来自新客,见 [[c:refunds.order_id]]。退款来自 `refunds`。[[see:t:refunds]]", catalog) + first, second = units(doc) + assert first["cites"] == ["c:refunds.order_id"] and second["cites"] == ["t:refunds"] + assert verify_doc(doc, catalog)["uncited"] == [] == uncited_claims(doc) + + +def test_an_auto_linked_name_alone_does_not_make_a_claim(catalog): + """自动链接只重新切片:句子算不算结论,和它提没提表名无关——没开实体层时是连接性的话, + 开了也还是。""" + text = "下面按 order_id 和 `orders` 展开,`ghost_table` 另说。" + doc = compose_doc(text, catalog) + [unit] = units(doc) + assert unit["kind"] == "connective" and entity_segments(doc) + plain = compose_doc(text, build_catalog(nodes={}, inputs={"week": "2026-W37"})) + assert [u["kind"] for u in units(plain)] == ["connective"] + # 名字里带方向词的(growth_rate 是查询结果的列,会被自动链接),开不开实体层都一样判 + rated = build_catalog(nodes={}, ledger=[{**QUERY, "columns": ["growth_rate"]}, SCHEMA]) + doc = compose_doc("见 growth_rate。", rated) + assert [s["text"] for s in entity_segments(doc)] == ["growth_rate"] + for cat in (rated, build_catalog(nodes={})): + assert [u["kind"] for u in units(compose_doc("见 growth_rate。", cat))] == ["claim"] + + +def test_verify_recomputes_uncited_instead_of_trusting_the_doc(catalog): + doc = compose_doc("增长主要来自新客,见 order_id 字段。收入上升。[[see:t:orders]]", catalog) + first, second = units(doc) + checked = verify_doc(doc, catalog) + assert checked["uncited"] == uncited_claims(doc) == [ + {"unit": first["id"], "span": first["span"], "text": "增长主要来自新客,见 order_id 字段。"}] + first["cites"] = ["c:refunds.order_id"] # 文档自称挂了依据 + assert uncited_claims(doc) == [] # 界面用的这份只照文档说 + assert [u["unit"] for u in verify_doc(doc, catalog)["uncited"]] == [first["id"]] # 复核不信 + second["see"] = [] # 依据被拿掉了:复核也看得出来 + assert [u["unit"] for u in verify_doc(doc, catalog)["uncited"]] == [first["id"], second["id"]] + + +# -------------------------------------------------------------------------- +# 跑一次:表结构快照被冻结,台账记下它 +# -------------------------------------------------------------------------- + + +@pytest.fixture +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def warehouse(tmp_path): + """探查过结构的示例库:orders、refunds 两张表。""" + from app.data.introspect import introspect + + path = tmp_path / "warehouse.db" + db = sqlite3.connect(path) + db.executescript( + "CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, amount REAL, created_at TEXT);" + "CREATE TABLE refunds (id INTEGER PRIMARY KEY, order_id INTEGER, amount REAL);") + db.executemany("INSERT INTO orders VALUES (?, ?, ?, ?)", + [(1, "east", 100.5, "2026-09-01"), (2, "west", 200.0, "2026-09-02"), (3, "east", 300.0, "2026-09-03")]) + db.commit() + db.close() + probe = SimpleNamespace(id=f"probe-{path.name}", name=SOURCE, kind="sqlite", database=str(path), host=None, + port=None, username=None, password=None, options={}, readonly=True, schema_cache={}) + cache = await introspect(probe) + await engines.invalidate(probe.id) + async with SessionLocal() as session: + row = (await session.execute(select(DataSource).where(DataSource.name == SOURCE))).scalar_one_or_none() + if row is None: + row = DataSource(name=SOURCE, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + row.database, row.kind, row.enabled, row.readonly, row.schema_cache = str(path), "sqlite", True, True, cache + await session.commit() + source_id = row.id + await engines.invalidate(source_id) + yield source_id + await engines.invalidate(source_id) + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +def tool_graph(*after, sql=SQL): + return chain(node("start", "input"), node("fetch", "tool", tool=TOOL, args={"sql": sql}), *after, + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.fetch }}"}])) + + +async def finish(graph, statuses=("succeeded", "failed")) -> Run: + run = await run_manager.start(graph=graph, input_payload={}) + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None): + return row + raise AssertionError(f"run {run.id} 没跑完:{row.status}") + + +async def events(run_id: str, etype: str, node_id: str | None = None) -> list[RunEvent]: + async with SessionLocal() as session: + q = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype) + if node_id: + q = q.where(RunEvent.node_id == node_id) + return list((await session.execute(q.order_by(RunEvent.seq))).scalars()) + + +async def ledger(run_id: str) -> list[dict]: + tup = await run_manager.checkpointer.aget_tuple({"configurable": {"thread_id": run_id}}) + return list(tup.checkpoint["channel_values"].get("evidence") or []) + + +async def test_schema_snapshot_is_frozen_and_recorded(engine_up, warehouse): + row = await finish(tool_graph()) + assert row.status == "succeeded", row.error + [end] = await events(row.id, "tool.end", "fetch") + schema_art = end.data["schema_artifact"] + payload = json.loads(row.output["r"]) + assert payload["schema_artifact"] == schema_art + snap = artifact_store.load(schema_art) + assert snap["source"] == SOURCE and set(snap["tables"]) == {"orders", "refunds"} + assert artifact_store.load(payload["artifact"])["schema_artifact"] == schema_art + + entries = await ledger(row.id) + assert [e["kind"] for e in entries] == ["tool", "query", "schema"] + query, schema = entries[1], entries[2] + assert query["schema_artifact"] == schema_art and query["tables"] == ["orders"] + assert schema == {"kind": "schema", "node_id": "fetch", "exec": 1, "artifact": schema_art, "source": SOURCE, + "schema": snap.get("schema"), "synced_at": snap.get("synced_at"), + "tables": {"orders": ["id", "region", "amount", "created_at"], + "refunds": ["id", "order_id", "amount"]}} + [finished] = await events(row.id, "node.finished", "fetch") + assert finished.data["evidence"] == entries + + # 事后改了数据源的结构缓存:已有的快照、台账、目录都不变 + async with SessionLocal() as session: + src = await session.get(DataSource, warehouse) + cache = json.loads(json.dumps(src.schema_cache)) + cache["tables"]["ghost_new"] = {"qualified": "ghost_new", "columns": [{"name": "x", "type": "INT"}]} + del cache["tables"]["refunds"] + src.schema_cache = cache + await session.commit() + assert artifact_store.load(schema_art) == snap + catalog = build_catalog(nodes={}, ledger=await ledger(row.id)) + assert "t:refunds" in catalog and "t:ghost_new" not in catalog + + again = await finish(tool_graph()) + [end2] = await events(again.id, "tool.end", "fetch") + assert end2.data["schema_artifact"] != schema_art + assert "ghost_new" in artifact_store.load(end2.data["schema_artifact"])["tables"] + + +async def test_agent_records_one_schema_entry_per_snapshot(engine_up, warehouse, monkeypatch): + from app.providers import mock_model + + def decide(self, messages): + done = sum(isinstance(m, ToolMessage) for m in messages) + if done < 2: + sql = SQL if done == 0 else "SELECT COUNT(*) AS n FROM refunds -- FROM ghost_table" + return AIMessage(content="查。", tool_calls=[{"name": TOOL, "args": {"sql": sql}, "id": f"call_{done}"}]) + return AIMessage(content="查完了。") + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide) + row = await finish(chain(node("start", "input"), node("bot", "agent", prompt="查", tools=[TOOL], approval="never"), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.bot.text }}"}]))) + assert row.status == "succeeded", row.error + ends = await events(row.id, "tool.end", "bot") + assert len(ends) == 2 and ends[0].data["schema_artifact"] == ends[1].data["schema_artifact"] + entries = await ledger(row.id) + assert [e["kind"] for e in entries] == ["tool", "query", "schema", "tool", "query"] + assert [e["tables"] for e in entries if e["kind"] == "query"] == [["orders"], ["refunds"]] + catalog = build_catalog(nodes={}, ledger=entries) + assert catalog["t:refunds"]["queries"] == ["Q2"] and "t:ghost_table" not in catalog + + +async def test_runs_from_before_the_upgrade_record_no_schema(engine_up, warehouse, monkeypatch): + async def no_snapshot(session): + return None + + monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot) + row = await finish(tool_graph()) + assert row.status == "succeeded", row.error + [end] = await events(row.id, "tool.end", "fetch") + assert "schema_artifact" not in end.data and "query_artifact" not in end.data + [finished] = await events(row.id, "node.finished", "fetch") + assert "evidence" not in finished.data + + +async def test_report_flags_a_made_up_name_without_failing(engine_up, warehouse, monkeypatch): + """可疑实体只标注:on_violation=fail 的报告节点不为它失败,也不为它重写。""" + from app.providers import mock_model + + seen: list = [] + + def decide(self, messages): + seen.append(messages) + return AIMessage(content="`orders` 里东区最多,达 [[v:Q1.r0.gmv]];`refund_log` 另算。[[see:Q1]]") + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide) + row = await finish(tool_graph(node("write", "report", instructions="写一句", on_violation="fail"))) + assert row.status == "succeeded", row.error + assert len(seen) == 1, "可疑实体不该触发重写" + [checked] = await events(row.id, "report.checked", "write") + assert [v["code"] for v in checked.data["violations"]] == ["unknown_entity"] + assert checked.data["stats"]["unknown_entities"] == 1 and checked.data["stats"]["entities"] == 1 + prompt = "\n".join(str(m.content) for m in seen[0]) + assert "[[t:" in prompt and "可能是编造的名字" in prompt and "orders" in prompt + doc = artifact_store.load(checked.data["doc_artifact"]) + [linked] = [s for s in iter_segments(doc) if s.get("ref") == "t:orders"] + assert linked["code"] is True and linked["cite"]["eid"] == doc["catalog"]["t:orders"]["eid"] + schema_art = (await events(row.id, "tool.end", "fetch"))[0].data["schema_artifact"] + assert doc["catalog"]["t:orders"]["artifact"] == schema_art + + +async def test_report_with_entities_off_leaves_names_alone(engine_up, warehouse, monkeypatch): + """entities: off:目录里没有表和字段,写作目录不教 [[t:]],反引号不核对;写了 [[t:]] 的, + 解析不了的原因照实说是这个节点关掉的。""" + from app.providers import mock_model + + seen: list = [] + + def decide(self, messages): + seen.append(messages) + return AIMessage(content="`orders` 里东区最多,达 [[v:Q1.r0.gmv]];`refund_log` 另算,见 [[t:orders]]。[[see:Q1]]") + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide) + row = await finish(tool_graph(node("write", "report", instructions="写一句", entities="off", max_repairs=0))) + assert row.status == "succeeded", row.error + [checked] = await events(row.id, "report.checked", "write") + assert [v["code"] for v in checked.data["violations"]] == ["unresolved_ref"] + assert ENTITIES_OFF_REASON in checked.data["violations"][0]["message"] + assert "unknown_entities" not in checked.data["stats"] and "entities" not in checked.data["stats"] + doc = artifact_store.load(checked.data["doc_artifact"]) + assert not [a for a, e in doc["catalog"].items() if e["kind"] in ("table", "column")] + assert not [s for s in iter_segments(doc) if s["kind"] == "entity" and not s.get("ref")] + assert "[[t:" not in "\n".join(str(m.content) for m in seen[0]) + + +async def test_report_with_a_bad_entities_value_fails(engine_up, warehouse, monkeypatch): + from app.providers import mock_model + + seen: list = [] + monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: seen.append(1) or AIMessage("x")) + row = await finish(tool_graph(node("write", "report", instructions="写一句", entities="bogus"))) + assert row.status == "failed" + assert "entities 只能是 link / off,写的是 'bogus'" in (row.error or ""), row.error + assert seen == [], "写错的配置不该先花一次模型调用" + + +def test_soft_violations_never_ask_for_a_rewrite(): + from app.engine.nodes.report import SOFT, _blocking + + assert SOFT == {"unknown_entity", "unverified_entity"} + soft = [{"code": "unknown_entity"}, {"code": "unverified_entity"}] + assert _blocking(soft, "strict") == [] + assert _blocking([*soft, {"code": "unresolved_ref"}], "strict") == [{"code": "unresolved_ref"}] + + +# -------------------------------------------------------------------------- +# 给 B 段的导出:evidence_from 连带表结构、按名字找条目 +# -------------------------------------------------------------------------- + + +def test_evidence_from_also_filters_the_schema_entries(): + """evidence_from 只收某几个节点的证据:别的节点查询当时冻结的表结构也不收,否则报告能引用 + 一张它的证据范围里根本没查过的表。""" + from app.engine.nodes.report import report_catalog, report_entities + from app.engine.schema import GraphSpec + + other = {**QUERY, "node_id": "other", "artifact": "c" * 64, "columns": ["n"], + "schema_artifact": "6" * 64, "tables": ["refunds"]} + other_schema = {**SCHEMA, "node_id": "other", "artifact": "6" * 64, "tables": {"refunds": ["id", "order_id"]}} + state = {"nodes": {}, "input": {}, "evidence": [QUERY, SCHEMA, other, other_schema]} + + def catalog_for(**config): + spec = GraphSpec.model_validate(chain(node("start", "input"), node("fetch", "tool", tool=TOOL), + node("other", "tool", tool=TOOL), node("write", "report", **config))) + return report_catalog(state, spec, spec.node_map()["write"]), spec + + full, _ = catalog_for() + assert {"t:orders", "t:refunds", "c:orders.region"} <= set(full) + only, _ = catalog_for(evidence_from=["other"]) + assert [a for a in only if a[0] == "Q"] == ["Q1"] and only["Q1"]["artifact"] == "c" * 64 + tables = {a for a, e in only.items() if e["kind"] == "table"} + assert tables == {"t:refunds"} # fetch 的表结构(orders)不收 + assert all(o["artifact"] == "6" * 64 for o in only["t:refunds"]["sources"] if o["kind"] == "schema") + off, spec = catalog_for(entities="off") + assert not [a for a, e in off.items() if e["kind"] in ("table", "column")] and "Q1" in off + assert report_entities(spec, spec.node_map()["write"]) == "off" + + +def test_find_entity_and_closest_entities(catalog): + from app.engine.evidence import closest_entities, find_entity + + assert find_entity("ORDERS", catalog) == "t:orders" + assert find_entity(" orders.Amount ", catalog) == "c:orders.amount" + assert find_entity("amount", catalog) == "c:amount" # 好几张表都有的字段 + assert find_entity("order_cnt", catalog) == "c:order_cnt" # 没有表名的结果列 + assert find_entity("amount", catalog, "table") is None + assert find_entity("orders", catalog, "column") is None + assert find_entity("ghost_table", catalog) is None + qualified = build_catalog(nodes={}, ledger=[{**SCHEMA, "schema": "public"}]) + assert find_entity("public.orders", qualified) == "t:orders" == find_entity("PUBLIC.ORDERS", qualified, "table") + assert find_entity("public.orders.region", qualified) == "c:orders.region" + + assert closest_entities("ordrs", catalog)[0] == "t:orders" + assert closest_entities("refund", catalog, limit=1) == ["t:refunds"] + assert len(closest_entities("order", catalog, limit=2)) <= 2 + assert closest_entities("zzzz_qqq", catalog) == [] + assert closest_entities("ordrs", catalog) == closest_entities("ordrs", catalog) # 同样的输入同样的顺序 diff --git a/backend/tests/test_evidence_entity_chain.py b/backend/tests/test_evidence_entity_chain.py new file mode 100644 index 00000000..937bf98f --- /dev/null +++ b/backend/tests/test_evidence_entity_chain.py @@ -0,0 +1,368 @@ +"""证据接口的实体步骤和引文步骤:点开报告里的表名、字段名、原话,看得到它们的出处。 + +- 表名、字段名:出现在哪几次查询里(SQL 用到的表、结果的列),表结构快照是什么时候同步的、 + 字段是什么类型。表结构快照只认封存范围内的事件交回过的那几件(tool.end.schema_artifact、 + node.finished.evidence 的 schema 条目),取回时复验哈希 +- 可疑实体(写了一个哪里都找不到的名字):照实说本次运行的表结构、查询、结果列里都没有它, + 再附上最接近的几个已知名字当提示——提示只从封存的台账里找 +- 引文:原话落在哪次检索的哪条命中、原文里的起止,检索快照同样只认封存范围内的 +""" +from __future__ import annotations + +import asyncio +import json +import sqlite3 +from types import SimpleNamespace + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage +from sqlalchemy import select + +from app.core import artifact_store +from app.core.config import settings +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource, Run, RunEvent +from app.engine.evidence import ( + UNKNOWN_ENTITY_REASON, + UNVERIFIED_ENTITY_REASON, + iter_segments, + make_eid, + normalize_quote, +) +from app.engine.runner import run_manager +from app.main import app + +SOURCE = "warehouse" +TOOL = f"db_query__{SOURCE}" +SQL = "SELECT region, SUM(amount) AS gmv, COUNT(*) AS order_cnt FROM orders GROUP BY region ORDER BY gmv DESC" +WRITE = "`orders` 里东区最多,达 [[v:Q1.r0.gmv]],按 [[c:orders.region]] 汇总。`refund_log` 另算。[[see:Q1]]" +HIT = {"chunk_id": "ch-1", "document_id": "doc-1", "title": "售后月报", "ordinal": 3, + "content": "九月退款主要集中在\n东区,原因是物流延误。"} + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def _warehouse(tmp_path, *, truncated: bool = False) -> str: + """探查过结构的示例库:orders、refunds 两张表。truncated 装作库里的表太多、快照只存了一部分。""" + from app.data.introspect import introspect + + path = tmp_path / "warehouse.db" + db = sqlite3.connect(path) + db.executescript( + "CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, amount REAL, created_at TEXT);" + "CREATE TABLE refunds (id INTEGER PRIMARY KEY, order_id INTEGER, amount REAL);") + db.executemany("INSERT INTO orders VALUES (?, ?, ?, ?)", + [(1, "east", 100.5, "2026-09-01"), (2, "west", 200.0, "2026-09-02"), (3, "east", 300.0, "2026-09-03")]) + db.commit() + db.close() + probe = SimpleNamespace(id=f"probe-{path.name}", name=SOURCE, kind="sqlite", database=str(path), host=None, + port=None, username=None, password=None, options={}, readonly=True, schema_cache={}) + cache = await introspect(probe) + if truncated: + cache.update(truncated=True, total=500) + await engines.invalidate(probe.id) + async with SessionLocal() as session: + row = (await session.execute(select(DataSource).where(DataSource.name == SOURCE))).scalar_one_or_none() + if row is None: + row = DataSource(name=SOURCE, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + row.database, row.kind, row.enabled, row.readonly, row.schema_cache = str(path), "sqlite", True, True, cache + row.options = {} + await session.commit() + source_id = row.id + await engines.invalidate(source_id) + return source_id + + +@pytest.fixture +async def warehouse(tmp_path): + source_id = await _warehouse(tmp_path) + yield source_id + await engines.invalidate(source_id) + + +@pytest.fixture +async def partial_warehouse(tmp_path): + source_id = await _warehouse(tmp_path, truncated=True) + yield source_id + await engines.invalidate(source_id) + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +def script(monkeypatch, text): + from app.providers import mock_model + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=text)) + + +async def finish(graph) -> Run: + run = await run_manager.start(graph=graph, input_payload={}) + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + return row + raise AssertionError(f"run {run.id} 没跑完:{row.status}") + + +async def events(run_id: str, etype: str, node_id: str | None = None) -> list[RunEvent]: + async with SessionLocal() as session: + q = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype) + if node_id: + q = q.where(RunEvent.node_id == node_id) + return list((await session.execute(q.order_by(RunEvent.seq))).scalars()) + + +async def reseal(run_id: str) -> None: + """按改过之后的事件重算封存哈希:模拟「封存链完好、但事件里追不到这件工件」。""" + async with SessionLocal() as session: + run = await session.get(Run, run_id) + rows = [tuple(r) for r in await session.execute( + select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data) + .where(RunEvent.run_id == run_id, RunEvent.seq <= run.manifest_seq).order_by(RunEvent.seq))] + run.manifest_hash = artifact_store.manifest_hash(rows) + await session.commit() + + +async def rewrite(run_id: str, etype: str, node_id: str, change) -> None: + async with SessionLocal() as session: + for row in (await session.execute(select(RunEvent).where( + RunEvent.run_id == run_id, RunEvent.type == etype, RunEvent.node_id == node_id))).scalars(): + row.data = change(dict(row.data)) + await session.commit() + + +async def report_run(monkeypatch, text=WRITE) -> tuple[Run, dict]: + script(monkeypatch, text) + row = await finish(chain(node("start", "input"), node("fetch", "tool", tool=TOOL, args={"sql": SQL}), + node("write", "report", instructions="写一句"), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.write.text }}"}]))) + assert row.status == "succeeded", row.error + [checked] = await events(row.id, "report.checked", "write") + return row, artifact_store.load(checked.data["doc_artifact"]) + + +async def opened(client, run_id: str, seg: dict) -> dict: + resp = await client.get(f"/api/runs/{run_id}/evidence/segments/{seg['id']}") + assert resp.status_code == 200, resp.text + return resp.json() + + +def seg_where(doc, **match) -> dict: + return next(s for s in iter_segments(doc) if all(s.get(k) == v for k, v in match.items())) + + +# -------------------------------------------------------------------------- +# 表名、字段名 +# -------------------------------------------------------------------------- + + +async def test_a_table_opens_its_schema_snapshot_and_queries(client, warehouse, monkeypatch): + row, doc = await report_run(monkeypatch) + [end] = await events(row.id, "tool.end", "fetch") + schema_art, query_art = end.data["schema_artifact"], end.data["query_artifact"] + snap = artifact_store.load(schema_art) + + body = await opened(client, row.id, seg_where(doc, kind="entity", ref="t:orders")) + [step] = body["chain"] + assert step["step"] == "entity" and step["status"] == "resolved" + assert step["kind"] == "table" and step["name"] == "orders" and step["alias"] == "t:orders" + assert step["code"] is True and step["auto"] is True + assert step["queries"] == ["Q1"] and step["source"] == SOURCE + assert step["synced_at"] == snap["synced_at"] and step["columns"] == 4 and step["is_view"] is False + kinds = {s["kind"]: s for s in step["sources"]} + assert set(kinds) == {"schema", "sql"} + assert kinds["schema"]["artifact"] == schema_art and kinds["schema"]["sealed"] is True + assert kinds["schema"]["hash_ok"] is True and kinds["schema"]["present"] is True + assert kinds["sql"] == {"kind": "sql", "artifact": query_art, "alias": "Q1", "source": SOURCE, + "node_id": "fetch", "tool": TOOL, "sealed": True} + assert step["eid"] == make_eid("table", schema_art, {"table": "orders"}) and step["eid_ok"] is True + assert step["sealed"] is True and body["seal"]["covered"] is True + assert "表结构" in body["note"] + + +async def test_a_column_carries_its_type_and_result_source(client, warehouse, monkeypatch): + row, doc = await report_run(monkeypatch) + body = await opened(client, row.id, seg_where(doc, kind="entity", ref="c:orders.region")) + [step] = body["chain"] + assert step["kind"] == "column" and step["table"] == "orders" and step["column"] == "region" + assert step["type"] == "TEXT" and step["type_source"] == "schema" and step["nullable"] is True + assert step["primary_key"] is False and "code" not in step and "auto" not in step + assert {(s["kind"], s.get("alias")) for s in step["sources"]} == {("schema", None), ("result", "Q1")} + assert step["queries"] == ["Q1"] and step["sealed"] is True + + +async def test_an_aggregate_alias_takes_its_type_from_the_result(client, warehouse, monkeypatch): + """聚合的别名(gmv)不在表结构里:类型取查询快照记下的列类型,出处是那份查询快照。""" + row, doc = await report_run(monkeypatch, "东区最多,汇总在 `gmv` 这一列,见 [[v:Q1.r0.gmv]]。[[see:Q1]]") + [end] = await events(row.id, "tool.end", "fetch") + body = await opened(client, row.id, seg_where(doc, kind="entity", ref="c:gmv")) + [step] = body["chain"] + assert step["kind"] == "column" and step["column"] == "gmv" and step.get("table") is None + assert step["type"] == "number" and step["type_source"] == "result" + assert [(s["kind"], s["artifact"], s["sealed"]) for s in step["sources"]] == [ + ("result", end.data["query_artifact"], True)] + assert step["eid"] == make_eid("column", end.data["query_artifact"], {"column": "gmv"}) + assert step["sealed"] is True + + +async def test_a_made_up_name_explains_itself_and_suggests_real_ones(client, warehouse, monkeypatch): + row, doc = await report_run(monkeypatch) + seg = seg_where(doc, kind="entity", issue="unknown_entity") + body = await opened(client, row.id, seg) + [step] = body["chain"] + assert step["step"] == "entity" and step["status"] == "unknown" and step["name"] == "refund_log" + assert step["reason"] == UNKNOWN_ENTITY_REASON + assert 0 < len(step["closest"]) <= 3 + assert step["closest"][0] == {"alias": "t:refunds", "kind": "table", "name": "refunds"} + assert step["checked"] == {"schemas": 1, "queries": 1} + assert "可能是编造的名字" in body["note"] + assert [v["code"] for v in body["violations"]] == ["unknown_entity"] + + +async def test_a_made_up_marker_is_suspicious_too(client, warehouse, monkeypatch): + """[[t:编造]] 和反引号里编的名字一样是可疑实体:名字取自引用,不是占位。""" + row, doc = await report_run(monkeypatch, "退款看 [[t:refund_logs]],东区最多 [[v:Q1.r0.gmv]]。[[see:Q1]]") + body = await opened(client, row.id, seg_where(doc, kind="entity", issue="unknown_entity")) + [step] = body["chain"] + assert step["status"] == "unknown" and step["name"] == "refund_logs" and step["kind"] == "table" + assert step["closest"][0]["alias"] == "t:refunds" + + +async def test_a_name_a_partial_snapshot_cannot_check_is_not_called_made_up(client, partial_warehouse, monkeypatch): + row, doc = await report_run(monkeypatch) + body = await opened(client, row.id, seg_where(doc, kind="entity", issue="unverified_entity")) + [step] = body["chain"] + assert step["status"] == "unverified" and step["name"] == "refund_log" + assert step["reason"] == UNVERIFIED_ENTITY_REASON and "编造" not in body["note"] and "核对不了" in body["note"] + table = await opened(client, row.id, seg_where(doc, kind="entity", ref="t:orders")) + assert table["chain"][0]["snapshot_truncated"] is True + assert {s["kind"]: s for s in table["chain"][0]["sources"]}["schema"]["truncated"] is True + + +async def test_suggestions_come_only_from_the_sealed_ledger(client, warehouse, monkeypatch): + """台账里的 schema 条目在封存的事件里被去掉了:提示里就不能再有只在那份快照里的表。""" + row, doc = await report_run(monkeypatch) + await rewrite(row.id, "node.finished", "fetch", + lambda d: {**d, "evidence": [e for e in d["evidence"] if e["kind"] != "schema"]}) + await reseal(row.id) + body = await opened(client, row.id, seg_where(doc, kind="entity", issue="unknown_entity")) + [step] = body["chain"] + assert all(c["alias"] != "t:refunds" for c in step["closest"]), step["closest"] + assert step["checked"]["schemas"] == 0 + + +async def test_a_schema_snapshot_outside_the_seal_is_not_trusted(client, warehouse, monkeypatch): + row, doc = await report_run(monkeypatch) + [end] = await events(row.id, "tool.end", "fetch") + schema_art = end.data["schema_artifact"] + await rewrite(row.id, "tool.end", "fetch", lambda d: {k: v for k, v in d.items() if k != "schema_artifact"}) + await rewrite(row.id, "node.finished", "fetch", + lambda d: {**d, "evidence": [e for e in d["evidence"] if e["kind"] != "schema"]}) + await reseal(row.id) + + body = await opened(client, row.id, seg_where(doc, kind="entity", ref="c:orders.region")) + [step] = body["chain"] + schema = next(s for s in step["sources"] if s["kind"] == "schema") + assert schema["artifact"] == schema_art and schema["sealed"] is False + assert "hash_ok" not in schema and "present" not in schema + # 类型、同步时间只能从封存的快照取:取不到就不给,不从工件库里「顺手」读 + assert "type" not in step and "synced_at" not in step + assert step["sealed"] is False and body["seal"]["covered"] is False + graph = (await client.get(f"/api/runs/{row.id}/evidence")).json() + assert {e["alias"]: e["sealed"] for e in graph["evidence"]}["c:orders.region"] is False + + +async def test_a_tampered_schema_snapshot_is_reported(client, warehouse, monkeypatch): + row, doc = await report_run(monkeypatch) + [end] = await events(row.id, "tool.end", "fetch") + art = end.data["schema_artifact"] + path = settings.data_dir / "artifacts" / art[:2] / f"{art}.json" + content = json.loads(path.read_text("utf-8")) + content["tables"]["orders"]["columns"][1]["type"] = "INTEGER" + path.write_text(json.dumps(content, ensure_ascii=False), encoding="utf-8") + + body = await opened(client, row.id, seg_where(doc, kind="entity", ref="c:orders.region")) + [step] = body["chain"] + schema = next(s for s in step["sources"] if s["kind"] == "schema") + assert schema["sealed"] is True and schema["hash_ok"] is False + assert "type" not in step and step["sealed"] is False + + +# -------------------------------------------------------------------------- +# 引文 +# -------------------------------------------------------------------------- + + +async def quote_run(monkeypatch, text="售后说「[[q:K1|退款主要集中在东区]]」。[[see:K1]]") -> tuple[Run, dict]: + from app.memory import kb + + async def fake_search(session, **kw): + return [dict(HIT, score=0.9)] + + monkeypatch.setattr(kb, "search", fake_search) + script(monkeypatch, text) + row = await finish(chain(node("start", "input"), node("docs", "retrieve", query="退款", collection="kb"), + node("write", "report", instructions="引用售后的原话"), + node("out", "output", fields=[{"name": "r", "value": "{{ nodes.write.text }}"}]))) + assert row.status == "succeeded", row.error + [checked] = await events(row.id, "report.checked", "write") + return row, artifact_store.load(checked.data["doc_artifact"]) + + +async def test_a_quote_opens_the_hit_it_was_copied_from(client, monkeypatch): + row, doc = await quote_run(monkeypatch) + [end] = await events(row.id, "retrieve.end", "docs") + body = await opened(client, row.id, seg_where(doc, kind="quote")) + [step] = body["chain"] + assert step["step"] == "quote" and step["alias"] == "K1" and step["text"] == "退款主要集中在东区" + assert step["source"] == {"artifact": end.data["artifact"], "document": "doc-1", "chunk": "ch-1", + "title": "售后月报", "ordinal": 3} + match = step["match"] + assert match["hit"] == 0 and step["content"] == HIT["content"] + assert normalize_quote(step["content"][match["start"]:match["end"]]) == normalize_quote(step["text"]) + assert step["collection"] == "kb" and step["query"] == "退款" and step["node_id"] == "docs" + assert step["hash_ok"] is True and step["match_ok"] is True and step["eid_ok"] is True + assert step["sealed"] is True and body["seal"]["covered"] is True + assert "逐字" in body["note"] + + +async def test_a_retrieval_outside_the_seal_is_not_shown(client, monkeypatch): + row, doc = await quote_run(monkeypatch) + await rewrite(row.id, "retrieve.end", "docs", lambda d: {k: v for k, v in d.items() if k != "artifact"}) + await rewrite(row.id, "node.finished", "docs", + lambda d: {**d, "evidence": [e for e in d.get("evidence") or [] if e["kind"] != "retrieval"]}) + await reseal(row.id) + body = await opened(client, row.id, seg_where(doc, kind="quote")) + [step] = body["chain"] + assert step["sealed"] is False and step["content"] is None and step["note"] + assert body["seal"]["covered"] is False + + +async def test_an_unmatched_quote_has_no_chain(client, monkeypatch): + row, doc = await quote_run(monkeypatch, "售后说「[[q:K1|退款主要集中在西区]]」。[[see:K1]]") + seg = seg_where(doc, kind="quote") + assert seg["state"] == "none" + body = await opened(client, row.id, seg) + assert body["chain"] == [] and "找不到这句原话" in body["note"] diff --git a/backend/tests/test_evidence_legacy.py b/backend/tests/test_evidence_legacy.py new file mode 100644 index 00000000..c1d9029f --- /dev/null +++ b/backend/tests/test_evidence_legacy.py @@ -0,0 +1,298 @@ +"""旧运行的降级展示:候选只从已封存的工件里找。 + +- legacy_text:没有契约、没有报告的答案,把数字按数值和封存范围内的查询快照单元格、口径卡指标 + 比对,每个数字最多 3 个候选,state 记 candidate,写明「猜测的来源,不能当证据」。按 + (run_id, manifest_hash) 放进内存 LRU(再带上数据源遮罩的指纹:事后加了遮罩要重算),猜测放到 + 线程里跑,不堵事件循环 +- 二期以前的运行没有 tool.end.query_artifact:查询快照的 id 只嵌在 tool_snapshot 的内容里, + 顺着封存事件里的 tool_snapshot(取回时复验哈希)找 +- legacy_contract:旧契约 matched 的指标详情同样只从封存的口径卡里取 +- 人为插进 artifacts 表的行、封存之后追加的事件指向的工件,一律不当证据 +""" +from __future__ import annotations + +import asyncio +import sqlite3 +import threading + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select + +from app.api import evidence as evidence_api +from app.core import artifact_store +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import Artifact, DataSource, Run, RunEvent +from app.engine import runner as runner_mod +from app.engine.evidence import GUESS_NOTE, GUESS_SCHEMA +from app.engine.runner import run_manager +from app.main import app + +SOURCE = "warehouse" +TOOL = f"db_query__{SOURCE}" +SQL = "SELECT region, SUM(amount) AS gmv, COUNT(*) AS order_cnt FROM orders GROUP BY region ORDER BY gmv DESC" +ANSWER = "东区销售额 400.5,西区 200,一共 7 单。" + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +async def warehouse(tmp_path): + path = tmp_path / "warehouse.db" + db = sqlite3.connect(path) + db.executescript("CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, amount REAL);") + db.executemany("INSERT INTO orders VALUES (?, ?, ?)", [(1, "east", 100.5), (2, "west", 200.0), (3, "east", 300.0)]) + db.commit() + db.close() + async with SessionLocal() as session: + row = (await session.execute(select(DataSource).where(DataSource.name == SOURCE))).scalar_one_or_none() + if row is None: + row = DataSource(name=SOURCE, kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + row.database, row.kind, row.enabled, row.readonly, row.schema_cache = str(path), "sqlite", True, True, {} + row.options = {} + await session.commit() + source_id = row.id + await engines.invalidate(source_id) + yield source_id + await engines.invalidate(source_id) + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +def asked(answer=ANSWER): + return chain(node("start", "input"), node("fetch", "tool", tool=TOOL, args={"sql": SQL}), + node("out", "output", fields=[{"name": "answer", "value": answer}])) + + +async def finish(graph) -> Run: + run = await run_manager.start(graph=graph, input_payload={}) + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + return row + raise AssertionError(f"run {run.id} 没跑完:{row.status}") + + +async def events(run_id: str, etype: str, node_id: str | None = None) -> list[RunEvent]: + async with SessionLocal() as session: + q = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype) + if node_id: + q = q.where(RunEvent.node_id == node_id) + return list((await session.execute(q.order_by(RunEvent.seq))).scalars()) + + +async def reseal(run_id: str) -> None: + async with SessionLocal() as session: + run = await session.get(Run, run_id) + rows = [tuple(r) for r in await session.execute( + select(RunEvent.seq, RunEvent.type, RunEvent.node_id, RunEvent.data) + .where(RunEvent.run_id == run_id, RunEvent.seq <= run.manifest_seq).order_by(RunEvent.seq))] + run.manifest_hash = artifact_store.manifest_hash(rows) + await session.commit() + + +def numbers(field: dict) -> dict[str, dict]: + return {s["text"]: s for s in field["segments"] if s["kind"] == "number"} + + +def counting(monkeypatch) -> list[int]: + """数一数真正猜了几次,并记下在哪个线程里猜的。""" + calls: list[int] = [] + real = evidence_api.guess_sources + + def spy(*args, **kwargs): + calls.append(threading.get_ident()) + return real(*args, **kwargs) + + monkeypatch.setattr(evidence_api, "guess_sources", spy) + return calls + + +# -------------------------------------------------------------------------- +# legacy_text +# -------------------------------------------------------------------------- + + +async def test_an_answer_without_a_contract_gets_guessed_candidates(client, warehouse): + row = await finish(asked()) + assert row.status == "succeeded", row.error + [end] = await events(row.id, "tool.end", "fetch") + body = (await client.get(f"/api/runs/{row.id}/evidence")).json() + assert body["mode"] == "legacy_text" and body["reports"] == [] and body["evidence"] == [] + legacy = body["legacy"] + assert legacy["schema"] == GUESS_SCHEMA and legacy["mode"] == "legacy_text" and legacy["note"] == GUESS_NOTE + assert legacy["note"].startswith("猜测的来源,不能当证据") and legacy["sealed"] is True + [field] = legacy["fields"] + assert field["field"] == "answer" and field["markdown"] == ANSWER + found = numbers(field) + east = found["400.5"] + assert east["state"] == "candidate" and len(east["candidates"]) <= 3 + top = east["candidates"][0] + assert top["kind"] == "cell" and top["ref"] == "Q1.r0.gmv" and top["artifact"] == end.data["query_artifact"] + assert top["node_id"] == "fetch" and top["tool"] == TOOL + assert found["200"]["candidates"][0]["ref"] == "Q1.r1.gmv" + assert found["7"]["state"] == "none" and "candidates" not in found["7"] + assert legacy["stats"] == {"numbers": 3, "guessed": 2, "unguessed": 1, + "candidates": sum(len(s.get("candidates") or []) for s in found.values())} + + audit = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json() + assert audit["mode"] == "legacy_text" + guessed = next(g for g in audit["groups"] if g["key"] == "candidate")["rows"] + assert [(r["text"], r["field"], r["state"]) for r in guessed] == [("400.5", "answer", "candidate"), + ("200", "answer", "candidate")] + assert guessed[0]["evidence"].startswith("Q1.r0.gmv") and "猜测" in guessed[0]["note"] + assert len(guessed[0]["candidates"]) == len(east["candidates"]) + unguessed = next(g for g in audit["groups"] if g["key"] == "none")["rows"] + assert [(r["text"], r["kind"]) for r in unguessed] == [("7", "number")] + + +async def test_artifacts_outside_the_sealed_events_are_not_candidates(client, warehouse): + """人为往 artifacts 表插一行(内容里正好有 7),再在封存之后追加一条指向它的 tool.end:都不认。""" + row = await finish(asked()) + fake = await artifact_store.put_json( + {"columns": ["n"], "rows": [[7]], "row_count": 1, "truncated": False, "sql": "SELECT 7 AS n", + "source": SOURCE}, kind="query_snapshot", run_id=row.id, node_id="fetch") + async with SessionLocal() as session: + assert await session.get(Artifact, (fake, row.id)) is not None + session.add(RunEvent(run_id=row.id, seq=row.manifest_seq + 3, type="tool.end", node_id="fetch", ts=0.0, + data={"tool": TOOL, "query_artifact": fake})) + await session.commit() + body = (await client.get(f"/api/runs/{row.id}/evidence")).json() + assert body["seal"]["ok"] is True + found = numbers(body["legacy"]["fields"][0]) + assert found["7"]["state"] == "none" + assert all(c["artifact"] != fake for s in found.values() for c in s.get("candidates") or []) + + +async def test_guesses_are_cached_per_seal_and_run_off_the_event_loop(client, warehouse, monkeypatch): + row = await finish(asked()) + calls = counting(monkeypatch) + first = (await client.get(f"/api/runs/{row.id}/evidence")).json() + again = (await client.get(f"/api/runs/{row.id}/evidence")).json() + await client.get(f"/api/runs/{row.id}/evidence/audit") + assert len(calls) == 1, "同一次封存只猜一次,证据图和审计表共用" + assert again["legacy"] == first["legacy"] + assert calls[0] != threading.get_ident(), "猜测要放到线程里跑" + + +async def test_masked_columns_are_never_candidates_even_after_caching(client, warehouse, monkeypatch): + row = await finish(asked()) + calls = counting(monkeypatch) + before = numbers((await client.get(f"/api/runs/{row.id}/evidence")).json()["legacy"]["fields"][0]) + assert before["400.5"]["state"] == "candidate" + async with SessionLocal() as session: + src = await session.get(DataSource, warehouse) + src.options = {"mask_columns": ["gmv"]} + await session.commit() + after = numbers((await client.get(f"/api/runs/{row.id}/evidence")).json()["legacy"]["fields"][0]) + assert len(calls) == 2, "遮罩变了,缓存的猜测作废" + assert after["400.5"]["state"] == "none" and after["200"]["state"] == "none" + assert all(c["locator"]["column"] != "gmv" for s in after.values() for c in s.get("candidates") or []) + + +async def test_runs_from_before_phase_two_are_followed_through_the_tool_snapshot(client, warehouse, monkeypatch): + async def no_snapshot(session): + return None + + monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot) + row = await finish(asked()) + assert row.status == "succeeded", row.error + [end] = await events(row.id, "tool.end", "fetch") + assert "query_artifact" not in end.data and end.data["artifact"] + body = (await client.get(f"/api/runs/{row.id}/evidence")).json() + assert body["mode"] == "legacy_text" + top = numbers(body["legacy"]["fields"][0])["400.5"]["candidates"][0] + assert top["ref"] == "Q1.r0.gmv" and top["kind"] == "cell" + snapshot = artifact_store.load(end.data["artifact"]) + assert top["artifact"] in snapshot["result"] + + +async def test_answers_without_numbers_or_sources_stay_none(client, warehouse): + body = (await client.get(f"/api/runs/{(await finish(asked('东区最多'))).id}/evidence")).json() + assert body["mode"] == "none" and "legacy" not in body + plain = await finish(chain(node("start", "input"), node("out", "output", fields=[{"name": "r", "value": "共 7 单"}]))) + body = (await client.get(f"/api/runs/{plain.id}/evidence")).json() + assert body["mode"] == "none" + + +# -------------------------------------------------------------------------- +# legacy_contract:指标详情只从封存的口径卡里取 +# -------------------------------------------------------------------------- + +KPI = "{'gmv': 45678.5, 'orders': 1234}" +STANDARD = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}, + {"id": "orders", "name": "订单数", "unit": "单", "expression": "vars.kpi.orders"}] +STORY = "本周销售额 45678.5 元,订单 1234 单,另有 999 笔待核。" + + +def narrated(): + return chain(node("start", "input"), node("fetch", "transform", mode="expression", expression=KPI, assign_to="kpi"), + node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=STANDARD), + node("narr", "transform", mode="template", template=STORY, assign_to="story"), + node("out", "output", fields=[{"name": "r", "value": "{{ vars.story }}"}], + contract={"metrics_from": ["caliber"], "narrative": "{{ vars.story }}"})) + + +async def test_legacy_matches_name_their_sealed_metric(client): + row = await finish(narrated()) + assert row.status == "succeeded", row.error + [card] = [e for e in await events(row.id, "node.finished", "caliber")] + sealed_card = next(e["artifact"] for e in card.data["evidence"] if e["kind"] == "metric_set") + # 人为插一张同名指标的「口径卡」:值不一样,不能被拿来当出处 + await artifact_store.put_json({"kind": "metric_set", "caliber": "别的口径", "caliber_version": "v9", + "metrics": [{"id": "gmv", "name": "销售额", "value": 1.0}]}, + kind="metric_set", run_id=row.id, node_id="caliber") + body = (await client.get(f"/api/runs/{row.id}/evidence")).json() + assert body["mode"] == "legacy_contract" + gmv = next(m for m in body["legacy"]["matched"] if m["metric"] == "gmv") + [source] = gmv["sources"] + assert source["kind"] == "metric" and source["ref"] == "m:gmv" and source["artifact"] == sealed_card + assert source["value"] == 45678.5 and source["rendered"] == "45,678.5元" and source["name"] == "销售额" + assert (source["caliber"], source["version"], source["node_id"]) == ("周报口径", "v2", "caliber") + assert source["sealed"] is True + + audit = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json() + assert audit["mode"] == "legacy_contract" + guessed = next(g for g in audit["groups"] if g["key"] == "candidate")["rows"] + assert [r["text"] for r in guessed] == ["45678.5", "1234"] + assert guessed[0]["evidence"].startswith("m:gmv") and "按数值匹配" in guessed[0]["note"] + bare = next(g for g in audit["groups"] if g["key"] == "none")["rows"] + assert [r["text"] for r in bare] == ["999"] + # 句子一栏取成果字段里这个数前后的原文(对上的数 trace_numbers 不给 context,以前整列是空的) + for r in guessed + bare: + assert r["field"] == "r" and r["text"] in r["sentence"] and r["sentence"] in STORY, r + assert "销售额 45678.5 元" in guessed[0]["sentence"] + + +async def test_legacy_matches_without_a_sealed_card_have_no_source(client): + row = await finish(narrated()) + async with SessionLocal() as session: + finished = (await session.execute(select(RunEvent).where( + RunEvent.run_id == row.id, RunEvent.type == "node.finished", RunEvent.node_id == "caliber"))).scalar_one() + finished.data = {k: v for k, v in finished.data.items() if k not in ("evidence", "artifact")} + await session.commit() + await reseal(row.id) + body = (await client.get(f"/api/runs/{row.id}/evidence")).json() + assert body["seal"]["ok"] is True + assert all(m["sources"] == [] for m in body["legacy"]["matched"]) diff --git a/backend/tests/test_evidence_legacy_guess.py b/backend/tests/test_evidence_legacy_guess.py new file mode 100644 index 00000000..5f3c636f --- /dev/null +++ b/backend/tests/test_evidence_legacy_guess.py @@ -0,0 +1,187 @@ +"""旧运行的「猜测来源」:没有契约、没有报告文档的答案,按数值去已封存的证据里找可能的来源。 + +这不是证据:一期核对真实反例时,168 个数里有 131 个能在快照里找到相同的值,大量是 0、1、2 +这种巧合。所以它只做降级展示,而且规矩很死: + +- 只从调用方传进来的已封存条目里找(查询快照的单元格、口径卡的指标值),自己不读库—— + 人为插进工件表的行进不了这个函数 +- 每个数字最多 3 个候选,按接近程度排,距离相同的按传入顺序;同样的输入永远同样的输出 +- 片段化返回,有候选的数字 state 记 candidate,一个都没有的记 none +""" +from __future__ import annotations + +import copy + +import pytest + +from app.engine.evidence import ( + GUESS_NOTE, + GUESS_SCHEMA, + MAX_CANDIDATES, + cell_eid, + guess_sources, + make_eid, +) + +Q_ART, M_ART, OTHER = "a" * 64, "d" * 64, "e" * 64 +METRICS = {"kind": "metric_set", "artifact": M_ART, "node_id": "caliber", "content": { + "caliber": "周报口径", "caliber_version": "v2", "metrics": [ + {"id": "gmv", "name": "销售额", "value": 45678.5, "unit": "元", "decimals": 1}, + {"id": "wow", "name": "环比", "value": 8.7, "unit": "%", "decimals": 1, "format": "plain"}, + {"id": "gone", "name": "缺值", "value": None, "unit": ""}]}} +QUERY = {"kind": "query", "artifact": Q_ART, "alias": "Q1", "node_id": "fetch", "tool": "db_query__warehouse", + "content": { + "columns": ["region", "gmv", "rate", "code"], + "rows": [["east", 45678.5, 0.087, "00123"], ["west", "12004.0", 0.02, "7"], ["north", 45678.2, None, "x"]], + "column_types": {"region": "text", "gmv": "number", "rate": "number", "code": "text"}}} +TEXT = "本周销售额 45,678.5 元,环比 8.7%,西区 12,004,比率 2%,另有 999 单,编号 123,日期 2026-09-01。" + + +def numbers(result): + return [s for s in result["segments"] if s["kind"] == "number"] + + +def refs(seg): + return [c.get("ref") for c in seg.get("candidates") or []] + + +def test_segments_tile_the_text_and_carry_candidates(): + result = guess_sources(TEXT, [METRICS, QUERY]) + assert result["schema"] == GUESS_SCHEMA and result["mode"] == "legacy_text" and result["note"] == GUESS_NOTE + assert "猜测" in GUESS_NOTE and "不能当证据" in GUESS_NOTE + assert "".join(s["text"] for s in result["segments"]) == TEXT + assert [s["id"] for s in result["segments"]] == [f"s{i}" for i in range(len(result["segments"]))] + for s in result["segments"]: + assert TEXT[s["span"][0]:s["span"][1]] == s["text"] + found = {s["text"]: (s["state"], refs(s)) for s in numbers(result)} + assert found == { + "45,678.5": ("candidate", ["m:gmv", "Q1.r0.gmv"]), + "8.7%": ("candidate", ["m:wow", "Q1.r0.rate"]), + "12,004": ("candidate", ["Q1.r1.gmv"]), + "2%": ("candidate", ["Q1.r1.rate"]), + "999": ("none", []), + "123": ("none", []), # 文本列里的 "00123" 是编号,不按数比 + } + assert result["stats"] == {"numbers": 6, "guessed": 4, "unguessed": 2, "candidates": 6} + assert all(s["state"] == "neutral" for s in result["segments"] if s["kind"] == "text") + + +def test_candidate_shapes(): + result = guess_sources("销售额 45,678.5", [METRICS, QUERY]) + metric, cell = numbers(result)[0]["candidates"] + assert metric == {"kind": "metric", "ref": "m:gmv", "artifact": M_ART, "locator": {"metric": "gmv"}, + "eid": make_eid("metric", M_ART, {"metric": "gmv"}), "value": 45678.5, + "rendered": "45,678.5元", "name": "销售额", "caliber": "周报口径", "version": "v2", + "node_id": "caliber", "diff": 0.0} + assert cell == {"kind": "cell", "ref": "Q1.r0.gmv", "alias": "Q1", "artifact": Q_ART, + "locator": {"row": 0, "column": "gmv"}, "eid": cell_eid(Q_ART, 0, "gmv"), "value": 45678.5, + "rendered": "45,678.5", "node_id": "fetch", "tool": "db_query__warehouse", "diff": 0.0} + + +def test_closer_values_come_first_and_ties_keep_the_given_order(): + result = guess_sources("约 45,678", [QUERY, METRICS]) + [seg] = numbers(result) + # 写的是整数,容差 0.5:45678.2 比 45678.5 更近;两个 45678.5 按传入顺序(查询在前) + assert refs(seg) == ["Q1.r2.gmv", "Q1.r0.gmv", "m:gmv"] + assert [c["diff"] for c in seg["candidates"]] == pytest.approx([0.2, 0.5, 0.5]) + + +def test_at_most_three_candidates_per_number(): + zeros = {"kind": "query", "artifact": OTHER, "alias": "Q2", "content": { + "columns": ["a", "b"], "rows": [[0, 0], [0, 0], [0, 0]]}} + [seg] = numbers(guess_sources("退款 0 单", [zeros])) + assert MAX_CANDIDATES == 3 and len(seg["candidates"]) == 3 + assert refs(seg) == ["Q2.r0.a", "Q2.r0.b", "Q2.r1.a"] + [seg] = numbers(guess_sources("退款 0 单", [zeros], limit=1)) + assert refs(seg) == ["Q2.r0.a"] + + +def test_only_the_entries_passed_in_are_searched(): + """函数自己不读库:一份没传进来的快照里就算有这个值,也不会成为候选。""" + result = guess_sources(TEXT, [QUERY]) + assert refs(numbers(result)[0]) == ["Q1.r0.gmv"] + assert all(not r.startswith("m:") for s in numbers(result) for r in refs(s)) + # 不认识的种类、坏掉的内容一律跳过,不崩 + junk = [{"kind": "tool", "artifact": OTHER, "content": {"columns": ["x"], "rows": [[45678.5]]}}, + {"kind": "retrieval", "artifact": OTHER, "content": {"hits": [{"content": "45678.5"}]}}, + {"kind": "query", "artifact": OTHER, "content": None}, + {"kind": "query", "artifact": OTHER, "content": {"columns": ["x"], "rows": "oops"}}, + {"kind": "metric_set", "content": {"metrics": "oops"}}, + "not a dict"] + result = guess_sources(TEXT, junk) + assert all(s["state"] == "none" for s in numbers(result)) + + +def test_same_input_same_output_and_inputs_untouched(): + entries = [METRICS, QUERY] + before = copy.deepcopy(entries) + first = guess_sources(TEXT, entries) + assert guess_sources(TEXT, entries) == first + assert entries == before + + +def test_text_without_numbers_and_empty_text(): + assert guess_sources("", [QUERY])["segments"] == [] + result = guess_sources("没有数字,只有 v2 这种标识符。", [QUERY]) + assert numbers(result) == [] and result["stats"]["numbers"] == 0 + + +# -------------------------------------------------------------------------- +# 遮罩的列:原始值一律不露,候选里也不行 +# -------------------------------------------------------------------------- + +SALARY = {"kind": "query", "artifact": OTHER, "alias": "Q2", "content": { + "columns": ["region", "salary", "headcount"], + "rows": [["east", 8123.45, 12], ["west", 9001.0, 8]], + "column_types": {"region": "text", "salary": "number", "headcount": "number"}, + "mask_columns": ["salary"]}} + + +@pytest.mark.parametrize("text", ["东区人均 8,123.45 元", "东区人均 8,123 元", "西区人均 9,001 元"]) +def test_columns_masked_at_query_time_are_never_candidates(text): + """快照记下的遮罩(查询当时数据源的 mask_columns):按容差对得上也不给候选——候选的 value、 + rendered、diff 都会把原值带出去,写成 8,123 的也会露出 8,123.45。""" + result = guess_sources(text, [SALARY]) + [seg] = numbers(result) + assert seg["state"] == "none" and "candidates" not in seg + assert result["stats"]["candidates"] == 0 + # 没遮的列照样能猜 + [seg] = numbers(guess_sources("东区 12 人", [SALARY])) + assert refs(seg) == ["Q2.r0.headcount"] + + +def test_columns_masked_now_are_never_candidates(): + """数据源现在设的遮罩(调用方按 artifact 交进来,不分大小写):事后加的遮罩同样生效。""" + [seg] = numbers(guess_sources("销售额 45,678.5", [QUERY], masked={Q_ART: ["GMV"]})) + assert refs(seg) == [] + [seg] = numbers(guess_sources("销售额 45,678.5", [QUERY, METRICS], masked={Q_ART: ["gmv"]})) + assert refs(seg) == ["m:gmv"] # 口径卡的指标是公开口径,不受列遮罩影响 + [seg] = numbers(guess_sources("比率 2%", [QUERY], masked={OTHER: ["rate"]})) + assert refs(seg) == ["Q1.r1.rate"] # 遮罩只管它那一份快照 + [seg] = numbers(guess_sources("比率 2%", [QUERY], masked={Q_ART: "code, Rate"})) + assert refs(seg) == [] # 写成「a, b」的遮罩也认:宁可多遮 + [seg] = numbers(guess_sources("比率 2%", [QUERY], masked="rate")) + assert refs(seg) == ["Q1.r1.rate"] # 形状不对的整个参数不当真,也不崩 + + +def test_a_value_that_cannot_be_rendered_is_skipped_for_the_next_one(): + """渲染不出来的格(非零却显示成 0)不当候选,名额让给下一个:和逐格先渲染的结果一样。""" + tiny = {"kind": "query", "artifact": OTHER, "alias": "Q2", "content": { + "columns": ["a"], "rows": [[1e-12], [0.3], [0.2], [0.4]]}} + # 写的是整数 0,容差 0.5:最近的 1e-12 显示不出来,第四近的 0.4 顶上 + [seg] = numbers(guess_sources("退款 0 单", [tiny])) + assert refs(seg) == ["Q2.r2.a", "Q2.r1.a", "Q2.r3.a"] + assert all(c["rendered"] != "—" for c in seg["candidates"]) + + +def test_big_snapshots_are_searched_quickly(): + import time + + cols = [f"c{i}" for i in range(12)] + rows = [[(r * 12 + c) * 1.25 for c in range(12)] for r in range(5000)] + sealed = [{"kind": "query", "artifact": f"{i:064x}", "alias": f"Q{i + 1}", + "content": {"columns": cols, "rows": rows}} for i in range(4)] + started = time.perf_counter() + result = guess_sources("数 " + ",".join(str(v * 1.25) for v in range(0, 60000, 997)), sealed) + assert time.perf_counter() - started < 1.0 + assert result["stats"]["guessed"] == result["stats"]["numbers"] > 0 diff --git a/backend/tests/test_evidence_quotes.py b/backend/tests/test_evidence_quotes.py new file mode 100644 index 00000000..d1802d18 --- /dev/null +++ b/backend/tests/test_evidence_quotes.py @@ -0,0 +1,267 @@ +"""引文:[[q:K1|原话]] 必须在那次检索命中的片段里逐字出现。 + +知识库的结论以前只能写成 [[see:K1]]:读者知道依据是哪次检索,看不到是哪一句。引文把 +原话写进报告,系统拿检索快照(取回时复验哈希)逐字核对: + +- 空白归一化后逐字出现才算有出处;改一个字就是没有出处(state none,issue unresolved_ref) +- 正文里显示引文本身;目录里根本没有这次检索时显示占位,和别的解析不了的引用一样 +- eid 带上命中片段的定位:第几条命中、原文里的起止 +- 出口复核重新取快照、重新比对,不信文档里记的结论 +""" +from __future__ import annotations + +import asyncio +import random + +import pytest +from langchain_core.messages import AIMessage +from sqlalchemy import select + +from app.core import artifact_store +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent +from app.engine.evidence import ( + LATER_REASON, + StreamRenderer, + build_catalog, + catalog_prompt, + compose_doc, + iter_segments, + make_eid, + render_markers, + verify_doc, +) +from app.engine.runner import run_manager + +K_ART = "c" * 64 +HITS = [ + {"chunk_id": "ch-1", "document_id": "doc-1", "title": "售后月报", "ordinal": 3, + "content": "九月退款主要集中在\n东区,原因是物流延误。退款率 2.35%,高于上月。"}, + {"chunk_id": "ch-2", "document_id": "doc-2", "title": "Ops notes", "ordinal": 0, + "content": "Net revenue\nrose after the promotion ended. 物流延误已经解决。"}, +] +SNAPSHOT = {"query": "退款", "collection": "kb", "hits": HITS} + + +def loader_of(content): + def load(artifact): + return content if artifact == K_ART else None + return load + + +LOAD = loader_of(SNAPSHOT) + + +@pytest.fixture +def catalog(): + return build_catalog(nodes={}, ledger=[{"kind": "retrieval", "node_id": "kb", "exec": 1, "artifact": K_ART, + "source": "kb", "rows": len(HITS)}]) + + +def quotes(doc): + return [s for s in iter_segments(doc) if s["kind"] == "quote"] + + +def test_verbatim_quote_resolves_with_its_location(catalog): + doc = compose_doc("售后说「[[q:K1|退款主要集中在东区]]」。[[see:K1]]", catalog, loader=LOAD) + [seg] = quotes(doc) + assert doc["markdown"] == "售后说「退款主要集中在东区」。" + assert seg["text"] == "退款主要集中在东区" and seg["state"] == "deterministic" + cite = seg["cite"] + assert cite["kind"] == "quote" and cite["role"] == "quote" and cite["alias"] == "K1" + loc = cite["locator"] + assert loc["hit"] == 0 and HITS[0]["content"][loc["start"]:loc["end"]] == "退款主要集中在\n东区" + assert cite["eid"] == make_eid("quote", K_ART, loc) + assert cite["source"] == {"artifact": K_ART, "document": "doc-1", "chunk": "ch-1", "title": "售后月报", + "ordinal": 3} + assert doc["violations"] == [] and doc["stats"]["quotes"] == 1 + assert doc["blocks"][0]["units"][0]["cites"] == ["K1"] + + +def test_one_changed_character_is_not_a_quote(catalog): + doc = compose_doc("售后说「[[q:K1|退款主要集中在西区]]」。", catalog, loader=LOAD) + [seg] = quotes(doc) + # 正文照样显示引文本身,只是标成没有出处 + assert seg["text"] == "退款主要集中在西区" and seg["state"] == "none" and seg["issue"] == "unresolved_ref" + assert "逐字" in seg["cite"]["reason"] + assert [v["code"] for v in doc["violations"]] == ["unresolved_ref"] + assert doc["stats"]["quotes"] == 0 + + +@pytest.mark.parametrize("quote", [ + "Net revenue rose after the promotion ended.", + "Net revenue rose", + "退款率 2.35%,高于上月", + "退款主要集中在 东区", +]) +def test_whitespace_is_normalised(catalog, quote): + doc = compose_doc(f"原话:[[q:K1|{quote}]]", catalog, loader=LOAD) + [seg] = quotes(doc) + assert seg["state"] == "deterministic", seg["cite"].get("reason") + loc = seg["cite"]["locator"] + assert " ".join(HITS[loc["hit"]]["content"][loc["start"]:loc["end"]].split()).replace(" ", "") \ + == quote.replace(" ", "") + + +@pytest.mark.parametrize("quote", ["Netrevenue rose", "net revenue rose", "退款主要集中在东区。九月"]) +def test_normalising_is_not_fuzzy(catalog, quote): + doc = compose_doc(f"原话:[[q:K1|{quote}]]", catalog, loader=LOAD) + assert quotes(doc)[0]["state"] == "none" + + +def test_numbers_inside_a_verified_quote_are_not_bare(catalog): + doc = compose_doc("[[q:K1|退款率 2.35%,高于上月]]", catalog, loader=LOAD) + assert doc["violations"] == [] and doc["stats"]["uncited_numbers"] == 0 + + +@pytest.mark.parametrize("text, words", [ + ("[[q:K9|退款主要集中在东区]]", "目录里没有 K9"), + ("[[q:K1|]]", "引文是空的"), + ("[[q:K1|东区]]", "太短"), +]) +def test_quotes_that_cannot_be_checked(catalog, text, words): + doc = compose_doc(text, catalog, loader=LOAD) + [seg] = quotes(doc) + assert seg["state"] == "none" and words in seg["cite"]["reason"], seg["cite"]["reason"] + + +def test_missing_retrieval_shows_a_placeholder(catalog): + doc = compose_doc("[[q:K9|退款主要集中在东区]]。", catalog, loader=LOAD) + assert doc["markdown"] == "⟦?q:K9|退款主要集中在东区⟧。" + + +def test_tampered_snapshot_is_not_evidence(catalog): + def tampered(artifact): + raise ValueError("hash mismatch") + + doc = compose_doc("[[q:K1|退款主要集中在东区]]", catalog, loader=tampered) + [seg] = quotes(doc) + assert seg["state"] == "none" and "哈希" in seg["cite"]["reason"] + + +def test_exit_check_rereads_the_snapshot(catalog): + doc = compose_doc("[[q:K1|物流延误已经解决]]。", catalog, loader=LOAD) + assert doc["violations"] == [] and quotes(doc)[0]["cite"]["locator"]["hit"] == 1 + changed = {**SNAPSHOT, "hits": [HITS[0], {**HITS[1], "content": "物流延误仍在持续。"}]} + checked = verify_doc(doc, catalog, loader=loader_of(changed)) + assert [v["code"] for v in checked["violations"]] == ["unresolved_ref"] + assert verify_doc(doc, catalog, loader=LOAD)["ok"] is True + + +def phase_two(doc): + """把文档改回二期组装的样子:那时 q / t / c 一律判为解析不了,正文是占位,原因是 LATER_REASON。 + 占位的字和现在「目录里没有 K9」时一样,只有记下的原因不同。""" + for seg in iter_segments(doc): + if seg.get("issue") == "unresolved_ref" and seg["ref"][:2] in ("q:", "t:", "c:"): + seg["cite"]["reason"] = LATER_REASON + return doc + + +def test_quotes_from_before_the_upgrade_are_rechecked_the_old_way(catalog): + """二期的报告文档(升级前组装,停在报告和出口之间)升级后被复核:原话今天查得到也不能算 + 「渲染对不上」——那会被出口当成完整性问题记缺口。按当时的规矩:占位,解析不了。""" + doc = phase_two(compose_doc("售后说[[q:K1|退款主要集中在东区]]。", {}, loader=LOAD)) + [seg] = quotes(doc) + assert seg["text"] == "⟦?q:K1|退款主要集中在东区⟧" + checked = verify_doc(doc, catalog, loader=LOAD) + assert [v["code"] for v in checked["violations"]] == ["unresolved_ref"] + assert LATER_REASON in checked["violations"][0]["message"] + # 只认占位:正文被改成了引文本身、又自称是二期的,照样按现在的规矩查 + seg["text"] = "退款主要集中在东区" + doc["markdown"] = "售后说退款主要集中在东区。" + tail = [x for x in iter_segments(doc)][-1] + seg["span"] = [3, 3 + len(seg["text"])] + tail["span"] = [seg["span"][1], seg["span"][1] + len(tail["text"])] + [unit] = [u for b in doc["blocks"] for u in b["units"]] + unit["span"] = [0, tail["span"][1]] + assert [v["code"] for v in verify_doc(doc, catalog, loader=LOAD)["violations"]] == ["state_mismatch"] + + +def test_entity_markers_from_before_the_upgrade_are_rechecked_the_old_way(): + ledger = [{"kind": "query", "node_id": "f", "exec": 1, "artifact": "a" * 64, "tool": "db_query__w", "source": "w", + "columns": ["gmv"], "rows": 1, "truncated": False, "schema_artifact": "5" * 64, "tables": ["orders"]}, + {"kind": "schema", "node_id": "f", "exec": 1, "artifact": "5" * 64, "source": "w", + "tables": {"orders": ["id", "gmv"]}}] + doc = phase_two(compose_doc("看 [[t:orders]] 和 [[c:orders.gmv]]。", {})) + assert [s["text"] for s in iter_segments(doc) if s.get("ref")] == ["⟦?t:orders⟧", "⟦?c:orders.gmv⟧"] + checked = verify_doc(doc, build_catalog(nodes={}, ledger=ledger)) + assert [v["code"] for v in checked["violations"]] == ["unresolved_ref", "unresolved_ref"] + + +def test_streaming_equals_full_render(catalog): + text = "开头 [[q:K1|退款主要集中在东区]],又说 [[q:K1|退款主要集中在西区]];[[q:K9|无此检索]] 完。[[see:K1]]" + whole = render_markers(text, catalog, loader=LOAD) + assert whole == "开头 退款主要集中在东区,又说 退款主要集中在西区;⟦?q:K9|无此检索⟧ 完。" + rng = random.Random(11) + for _ in range(200): + renderer = StreamRenderer(catalog, loader=LOAD) + out, i = [], 0 + while i < len(text): + step = rng.randint(1, 7) + out.append(renderer.feed(text[i:i + step])) + i += step + out.append(renderer.flush()) + assert "".join(out) == whole + + +def test_catalog_prompt_shows_hits_to_quote_from(catalog): + prompt = catalog_prompt(catalog, loader=LOAD) + assert "[[q:K1|" in prompt and "逐字" in prompt and "[[see:K1]]" in prompt + assert "售后月报" in prompt and "退款主要集中在" in prompt and "Ops notes" in prompt + + +# -------------------------------------------------------------------------- +# 跑一次:检索 → 报告撰写,引错了原话会被要求重写 +# -------------------------------------------------------------------------- + + +@pytest.fixture +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +async def test_report_node_checks_quotes_against_the_retrieval(engine_up, monkeypatch): + from app.memory import kb + from app.providers import mock_model + + async def fake_search(session, **kw): + return [dict(HITS[0], score=0.9)] + + monkeypatch.setattr(kb, "search", fake_search) + replies = ["售后说「[[q:K1|退款主要集中在西区]]」。[[see:K1]]", "售后说「[[q:K1|退款主要集中在东区]]」。[[see:K1]]"] + seen: list = [] + + def decide(self, messages): + seen.append(list(messages)) + return AIMessage(content=replies[min(len(seen), len(replies)) - 1]) + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide) + nodes = [{"id": nid, "type": t, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": cfg}} for nid, t, cfg in [ + ("start", "input", {}), ("docs", "retrieve", {"query": "退款", "collection": "kb"}), + ("write", "report", {"instructions": "引用售后的原话"}), + ("out", "output", {"fields": [{"name": "r", "value": "{{ nodes.write.text }}"}]})]] + graph = {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + run = await run_manager.start(graph=graph, input_payload={}) + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + break + assert row.status == "succeeded", row.error + assert row.output["r"] == "售后说「退款主要集中在东区」。" + + first = "\n".join(str(m.content) for m in seen[0]) + assert "[[q:K1|" in first and "退款主要集中在" in first and "售后月报" in first + assert len(seen) == 2 and "找不到这句原话" in str(seen[1][-1].content) + async with SessionLocal() as session: + [checked] = (await session.execute(select(RunEvent).where( + RunEvent.run_id == run.id, RunEvent.type == "report.checked"))).scalars().all() + assert checked.data["ok"] is True and checked.data["repairs"] == 1 and checked.data["stats"]["quotes"] == 1 + doc = artifact_store.load(checked.data["doc_artifact"]) + [seg] = quotes(doc) + retrieval = doc["catalog"]["K1"]["artifact"] + assert seg["cite"]["eid"] == make_eid("quote", retrieval, seg["cite"]["locator"]) + assert seg["cite"]["source"]["chunk"] == "ch-1" diff --git a/backend/tests/test_gov_wrapup.py b/backend/tests/test_gov_wrapup.py new file mode 100644 index 00000000..263f3590 --- /dev/null +++ b/backend/tests/test_gov_wrapup.py @@ -0,0 +1,215 @@ +"""证据门禁 G1–G5 的收尾:验收留下的小问题和 B 段的交叉请求。 + +- G1 和 G2 对开了 cite_fields 的 agent 字段看法一致:逐格核对过的字段是数据,不是模型写的话; +- 选项里只有标了 handoff 的那一项才是「交给 Copilot」,节点 id 恰好叫 copilot 的报告照常应用; +- 自动修复接口读不懂图时也带 handoff 键; +- 已发布级别的 G2 对每个出口都给警告(用户拍板:G 规则受管挡、已发布警告,范围一样); +- G4、G5 挡住的地方,validate 那条同义的提示不再重复; +- 删掉开了 cite_fields 的 agent 的 output_schema 算降低要求; +- Copilot 自查带上全图默认,和真正的门禁口径一致; +- 受管契约的 claims 写 on_uncited: ignore 给警告,并能一键改成 degrade。 +""" +from __future__ import annotations + +import copy +import json + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.engine.autofix import HANDOFF, apply_fixes, forbidden_changes +from app.engine.governance import publish_issues +from app.engine.schema import GraphSpec, validate_graph +from app.main import app +from tests.test_autofix_governance import ( + SCHEMA, STRICT, codes, compute_feeding, config, create, errors, insert, node, plan, script, traceable, + with_story, +) + + +def issues(graph: dict, level: str = "governed") -> list: + return publish_issues(GraphSpec.model_validate(graph), level=level) + + +def cited_agent(graph: dict, *, cite: bool = True) -> dict: + ask = node("ask", "agent", "查数助手", prompt="查一下", tools=["db_query__shop"], approval="dangerous", + output_schema=SCHEMA, cite_fields=cite, assign_to="facts") + return insert(graph, ask, before="done", after="write") + + +# ---------------------------------------------------------------- G1 / G2 一致 + + +@pytest.mark.parametrize("ref", ["{{ vars.facts.gmv }}", "{{ nodes.ask.data.gmv }}"]) +def test_a_cited_agent_field_in_a_contract_exit_is_data(ref): + graph = cited_agent(traceable()) + config(graph, "done")["fields"].append({"name": "GMV", "value": ref}) + found = codes(errors(graph)) + assert "governed.exit_text_source" not in found and "governed.text_bypass" not in found, found + assert not [f for f in plan(graph) if f.startswith("governed.exit_text_source")] + + +def test_an_uncited_agent_field_in_a_contract_exit_is_still_text(): + graph = cited_agent(traceable(), cite=False) + config(graph, "done")["fields"].append({"name": "GMV", "value": "{{ vars.facts.gmv }}"}) + assert "governed.exit_text_source" in codes(errors(graph)) + + +# ---------------------------------------------------------------- handoff 只认标记 + + +def renamed(graph: dict, old: str, new: str) -> dict: + text = json.dumps(graph, ensure_ascii=False) + return json.loads(text.replace(f'"{old}"', f'"{new}"').replace(f"nodes.{old}.", f"nodes.{new}.")) + + +def test_the_handoff_option_is_marked(): + fix = plan(compute_feeding())["governed.caliber_compute_input:calc"] + marked = [o for o in fix["options"] if o.get("handoff")] + assert [o["value"] for o in marked] == [HANDOFF] + + +def test_a_report_whose_id_is_copilot_is_applied_not_handed_off(): + graph = renamed(with_story(traceable()), "write", HANDOFF) + fid = "governed.exit_text_source:done" + assert [o["value"] for o in plan(graph)[fid]["options"]] == [HANDOFF] + out = apply_fixes(graph, [fid], {fid: HANDOFF}, level="governed") + assert out["applied"] == [fid] and out["handoff"] == [], out + assert config(out["graph"], "done")["fields"][2]["value"] == "{{ nodes.copilot.text }}" + + +# ---------------------------------------------------------------- 接口的早退回包 + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def test_autofix_on_an_unreadable_graph_still_has_handoff(client): + wf = await create(client, traceable()) + body = (await client.post(f"/api/workflows/{wf}/autofix", json={ + "level": "governed", "graph": {"nodes": "坏的"}, "apply": ["x"]})).json() + assert body["ok"] is False and body["handoff"] == [] + + +# ---------------------------------------------------------------- G2 已发布级别的范围 + + +def test_published_g2_warns_on_every_exit(): + graph = {"nodes": [node("start", "input", "问题"), node("chat", "llm", "回答", prompt="{{ input.q }}"), + node("out", "output", "回答出口", fields=[{"name": "回答", "value": "{{ nodes.chat.text }}"}])], + "edges": [{"source": "start", "target": "chat"}, {"source": "chat", "target": "out"}]} + published = [i for i in issues(graph, "published") if i.code == "governed.text_bypass"] + assert [i.level for i in published] == ["warning"] and published[0].node_id == "chat" + governed = [i for i in issues(graph, "governed") if i.code == "governed.text_bypass"] + assert [i.level for i in governed] == ["error"] + + +# ---------------------------------------------------------------- G4 / G5 不重复 + + +def test_the_g4_error_is_not_repeated_by_a_validate_warning(): + graph = compute_feeding() + spec = GraphSpec.model_validate(graph) + # validate 自己照样提示(画布上没发布也看得到),带编号 + assert "evidence.caliber_compute_input" in {i.code for i in validate_graph(spec).issues} + for level in ("governed", "published"): + found = issues(graph, level) + assert "governed.caliber_compute_input" in {i.code for i in found} + assert "evidence.caliber_compute_input" not in {i.code for i in found}, level + + +def test_the_g5_error_is_not_repeated_by_a_validate_warning(): + graph = insert(traceable(), node("ask", "agent", "查数助手", prompt="查", tools=["db_query__shop"], + approval="dangerous", output_schema=SCHEMA, assign_to="facts"), + before="card", after="fetch") + config(graph, "card")["metrics"].append({"id": "n", "expression": "vars.facts.gmv"}) + spec = GraphSpec.model_validate(graph) + assert "evidence.cite_fields_off" in {i.code for i in validate_graph(spec).issues} + found = issues(graph, "governed") + assert "governed.caliber_agent_cite_fields" in {i.code for i in found} + assert "evidence.cite_fields_off" not in {i.code for i in found} + + +def test_the_validate_warning_stays_where_no_gate_rule_covers_it(): + graph = cited_agent(traceable(), cite=False) # 不给口径卡供数:G5 不管,validate 的提示留着 + assert "evidence.cite_fields_off" in {i.code for i in issues(graph, "governed")} + + +# ---------------------------------------------------------------- 删 output_schema 算降低要求 + + +def test_removing_the_schema_of_a_cited_agent_is_refused(): + before = cited_agent(traceable()) + after = copy.deepcopy(before) + config(after, "ask").pop("output_schema") + [reason] = forbidden_changes(before, after) + assert "「查数助手」" in reason and "output_schema" in reason + config(after, "ask")["output_schema"] = {} + assert forbidden_changes(before, after) + + +# ---------------------------------------------------------------- Copilot 自查带全图默认 + + +async def test_copilot_self_check_sees_graph_defaults(client, monkeypatch): + graph = compute_feeding() + config(graph, "write").clear() + config(graph, "write")["instructions"] = "写周报" + graph["defaults"] = dict(STRICT) + assert "governed.report_policy" not in codes(errors(graph)) # 真正的门禁认全图默认 + model = script(monkeypatch, []) + wf = await create(client, graph) + await client.post(f"/api/workflows/{wf}/autofix", json={"level": "governed", "apply": [], "assist": True}) + human = next(text for role, text in model.calls[0] if role == "human") + assert "「计算」" in human and "要写明" not in human + + +async def test_copilot_self_check_still_sees_report_policy_without_defaults(client, monkeypatch): + graph = compute_feeding() + config(graph, "write").clear() + config(graph, "write")["instructions"] = "写周报" + model = script(monkeypatch, []) + wf = await create(client, graph) + await client.post(f"/api/workflows/{wf}/autofix", json={"level": "governed", "apply": [], "assist": True}) + human = next(text for role, text in model.calls[0] if role == "human") + assert "要写明" in human + + +def test_the_assist_rules_forbid_marking_sources_and_loosening_reports(): + from app.api.copilot import _ASSIST_RULES + + assert "evidence_role" in _ASSIST_RULES and "source" in _ASSIST_RULES + for key in ("numbers", "on_violation", "claims", "cite_fields", "output_schema"): + assert key in _ASSIST_RULES, key + + +# ---------------------------------------------------------------- 契约 claims 写 ignore + + +def ignoring() -> dict: + return traceable(claims={"policy": "require_citation", "on_uncited": "ignore"}) + + +def test_contract_ignore_warns_under_governed_only(): + found = [i for i in issues(ignoring()) if i.code == "contract.claims_ignored"] + assert [(i.level, i.node_id, i.field) for i in found] == [("warning", "done", "contract.claims")] + assert not [i for i in issues(ignoring(), "published") if i.code == "contract.claims_ignored"] + assert not [i for i in issues(traceable(claims={"policy": "require_citation", "on_uncited": "withhold"})) + if i.code == "contract.claims_ignored"] + + +def test_contract_ignore_is_fixed_to_degrade(): + fid = "contract.claims_ignored:done" + fix = plan(ignoring())[fid] + assert fix["kind"] == "auto" and fix["preview"] == {"field": "contract.claims.on_uncited", + "before": "ignore", "after": "degrade"} + out = apply_fixes(ignoring(), [fid], level="governed") + assert out["applied"] == [fid], out["rejected"] + assert config(out["graph"], "done")["contract"]["claims"] == {"policy": "require_citation", + "on_uncited": "degrade"} + assert "contract.claims_ignored" not in {i["code"] for i in out["remaining"]} + # 反方向(degrade 改回 ignore)是放宽,要拒 + assert forbidden_changes(out["graph"], ignoring()) diff --git a/backend/tests/test_governance_report_lint.py b/backend/tests/test_governance_report_lint.py new file mode 100644 index 00000000..fc336a65 --- /dev/null +++ b/backend/tests/test_governance_report_lint.py @@ -0,0 +1,475 @@ +"""受管出具的证据门禁 G1–G5(governance.lint_for_publish)。 + +报告里的每个数都点得开出处,前提是这张图在结构上走得通:出口核对报告撰写节点的文档(G1)、 +模型写的话不绕过报告直接进出口(G2)、报告节点按最严的规则自查(G3)、口径卡的输入不是沙箱里 +算出来的(G4)、给口径卡供数的 agent 交的是逐格核对过的字段(G5)。 + +受管级别是硬错误、挡住发布;已发布级别只给警告。判定只看图和配置,不看运行结果;已经发布的 +版本照常运行,下次发布时按新规则查。 +""" +from __future__ import annotations + +import asyncio +import copy + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage + +from app.db.base import SessionLocal +from app.db.models import Run, Workflow, WorkflowVersion +from app.engine.governance import lint_for_publish, publish_issues +from app.engine.runner import run_manager +from app.engine.schema import CLAIMS_LATER, GraphSpec, validate_graph +from app.main import app +from app.seed import TEMPLATES + +#: 报告撰写节点在受管模板里要写明的三项 +STRICT = {"numbers": "strict", "on_violation": "fail", "claims": "require_citation"} + +G_CODES = { + "governed.report_from_required", "governed.exit_text_source", "governed.text_bypass", + "governed.report_policy", "governed.caliber_compute_input", "governed.caliber_agent_schema", + "governed.caliber_agent_cite_fields", "governed.caliber_model_input", +} + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +def traceable(*, report=None, fields=None, **contract_overrides) -> dict: + """input → 查库 → 口径卡 → 报告撰写 → 出具(引用模式契约):受管要求全部满足,包括 G1–G5。""" + contract = {"report_from": "write", "metrics_from": ["card"], "required": ["gmv"], "strict": True} + contract.update(contract_overrides) + nodes = [ + node("start", "input", "周期", fields=[{"name": "week", "default": "2026-W37"}]), + node("fetch", "tool", "取数", tool="db_query__shop", + args={"sql": "SELECT SUM(amount) AS gmv, COUNT(*) AS orders FROM orders"}), + node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[ + {"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}, + {"id": "orders", "name": "订单数", "unit": "单", "expression": "cell(nodes.fetch, 0, 'orders')"}]), + node("write", "report", "报告撰写", instructions="写周报", **(STRICT if report is None else report)), + node("done", "output", "出具", + fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}] if fields is None else fields, + contract={k: v for k, v in contract.items() if v is not None}), + ] + edges = [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])] + return {"nodes": nodes, "edges": edges} + + +def find(graph: dict, nid: str) -> dict: + return next(n for n in graph["nodes"] if n["id"] == nid) + + +def config(graph: dict, nid: str) -> dict: + return find(graph, nid)["data"]["config"] + + +def insert(graph: dict, new: dict, *, before: str, after: str) -> dict: + """把 new 接在 after 和 before 之间(原来的直连保留)。""" + graph["nodes"].insert(len(graph["nodes"]) - 1, new) + graph["edges"] += [{"source": after, "target": new["id"]}, {"source": new["id"], "target": before}] + return graph + + +def lint(graph: dict, level: str = "governed") -> list: + return lint_for_publish(GraphSpec.model_validate(graph), level=level).issues + + +def g_issues(graph: dict, level: str = "governed") -> list: + return [i for i in lint(graph, level) if i.code in G_CODES] + + +def by_code(graph: dict, code: str, level: str = "governed") -> list: + return [i for i in lint(graph, level) if i.code == code] + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def publish(client, graph: dict, level: str = "governed") -> dict: + wf = (await client.post("/api/workflows", json={"name": "证据门禁-受管", "graph": graph})).json() + body = (await client.post(f"/api/workflows/{wf['id']}/publish", json={"level": level})).json() + await client.delete(f"/api/workflows/{wf['id']}") + return body + + +def test_the_baseline_passes_every_rule(): + for level in ("governed", "published"): + issues = publish_issues(GraphSpec.model_validate(traceable()), level=level) + assert [i for i in issues if i.level == "error"] == [], [i.message for i in issues] + assert not {i.code for i in issues} & G_CODES + + +def test_every_rule_carries_a_code_and_points_at_a_node(): + graph = traceable(report={}, narrative="{{ nodes.write.text }}", report_from=None) + graph = insert(graph, node("calc", "code", "计算", code="print(1)", assign_to="calc"), before="card", + after="fetch") + config(graph, "card")["metrics"].append({"id": "extra", "expression": "vars.calc"}) + issues = g_issues(graph) + assert {i.code for i in issues} >= {"governed.report_from_required", "governed.report_policy", + "governed.caliber_compute_input"} + assert all(i.node_id and i.field for i in issues), [(i.code, i.node_id, i.field) for i in issues] + + +# -------------------------------------------------------------------------- +# G1:带契约的出口核对报告撰写节点的文档,成果字段的文字只能来自报告撰写节点 +# -------------------------------------------------------------------------- + + +async def test_a_governed_contract_without_report_from_is_refused(client): + # 旧的叙述模式:数字按数值回头匹配,出处可能不唯一 + graph = traceable(report_from=None, narrative="{{ nodes.write.text }}") + [issue] = by_code(graph, "governed.report_from_required") + assert issue.level == "error" and issue.node_id == "done" and issue.field == "contract.report_from" + assert "report_from" in issue.message and "narrative" in issue.message + body = await publish(client, graph) + assert body["ok"] is False + assert "governed.report_from_required" in {i["code"] for i in body["issues"] if i["level"] == "error"} + # 两样都没写的,照旧由 contract.report_from_missing 报,不重复报一条 + bare = traceable(report_from=None) + assert by_code(bare, "governed.report_from_required") == [] + assert (await publish(client, bare))["ok"] is False + + +@pytest.mark.parametrize("producer, label", [ + (node("narrate", "llm", "叙述", prompt="写周报", assign_to="story"), "模型调用"), + (node("shape", "transform", "整形", mode="template", template="{{ nodes.card.text }}", assign_to="story"), + "数据整形"), + (node("calc", "code", "计算", code="print('x')", assign_to="story"), "沙箱代码"), +]) +@pytest.mark.parametrize("ref", ["{{ nodes.%s.text }}", "{{ vars.story }}"]) +def test_an_exit_field_taking_text_from_a_non_report_node_is_named(producer, label, ref): + graph = insert(traceable(), producer, before="done", after="write") + config(graph, "done")["fields"].append({"name": "摘要", "value": "本周:" + ref % producer["id"] + if "%s" in ref else "本周:" + ref}) + [issue] = by_code(graph, "governed.exit_text_source") + assert issue.level == "error" and issue.node_id == "done" and issue.field == "fields[1].value" + assert "「摘要」" in issue.message and f"「{producer['data']['label']}」({label})" in issue.message + assert [i.level for i in by_code(graph, "governed.exit_text_source", "published")] == ["warning"] + + +@pytest.mark.parametrize("value", [ + "{{ nodes.write.text }}", "{{ vars.report }}", "{{ nodes.write.stats | json }}", "{{ nodes.card.text }}", + "{{ input.week }}", "{{ vars.week }}", "{{ nodes.fetch }}", "{{ usage.total_tokens }}", "由系统生成,仅供内部参考", + "{{ last_message }}", +]) +def test_deterministic_sources_and_report_text_are_fine(value): + graph = traceable() + config(graph, "write")["assign_to"] = "report" + config(graph, "done")["fields"].append({"name": "附", "value": value}) + assert g_issues(graph) == [], [i.message for i in g_issues(graph)] + + +def test_a_source_code_node_is_a_data_source_but_a_compute_one_is_not(): + graph = insert(traceable(), node("raw", "code", "取原始数据", code="print(1)", evidence_role="source"), + before="done", after="fetch") + config(graph, "done")["fields"].append({"name": "原始数据", "value": "{{ nodes.raw }}"}) + assert by_code(graph, "governed.exit_text_source") == [] + config(graph, "raw")["evidence_role"] = "compute" + assert len(by_code(graph, "governed.exit_text_source")) == 1 + + +def test_an_exit_without_fields_hands_over_the_last_message(): + # 没配字段就把最后一条消息当成果:最后一条是报告的,没问题;是模型调用写的,就绕过了报告 + graph = traceable(fields=[]) + assert g_issues(graph) == [] + graph = insert(graph, node("polish", "llm", "润色", prompt="{{ nodes.write.text }}"), before="done", + after="write") + graph["edges"] = [e for e in graph["edges"] if (e["source"], e["target"]) != ("write", "done")] + [issue] = by_code(graph, "governed.exit_text_source") + assert issue.field == "fields" and "最后一条消息" in issue.message and "「润色」" in issue.message + # 出口没有契约时 G1 不管,G2 照样追到这段绕过报告的文字 + config(graph, "done").pop("contract") + assert [i.node_id for i in by_code(graph, "governed.text_bypass")] == ["polish"] + + +# -------------------------------------------------------------------------- +# G2:模型写的文字没经过报告撰写节点,从图上追到出口 +# -------------------------------------------------------------------------- + + +def test_an_llm_bypassing_the_report_is_caught_through_intermediate_nodes(): + graph = traceable() + graph = insert(graph, node("narrate", "llm", "叙述", prompt="写周报", assign_to="story"), before="done", + after="card") + graph = insert(graph, node("shape", "transform", "整形", mode="template", + template="【周报】{{ vars.story }}", assign_to="shaped"), before="done", after="narrate") + config(graph, "done")["fields"].append({"name": "摘要", "value": "{{ vars.shaped }}"}) + [bypass] = by_code(graph, "governed.text_bypass") + assert bypass.level == "error" and bypass.node_id == "narrate" + assert "「叙述」(模型调用)" in bypass.message and "「出具」" in bypass.message and "「整形」" in bypass.message + # 出口字段点的是整形节点:G1 点名字段,G2 点名写文字的那个模型节点,两处各报各的 + assert [i.field for i in by_code(graph, "governed.exit_text_source")] == ["fields[1].value"] + + +def test_a_direct_reference_in_a_contract_exit_is_left_to_g1(): + graph = insert(traceable(), node("narrate", "llm", "叙述", prompt="写周报"), before="done", after="card") + config(graph, "done")["fields"].append({"name": "摘要", "value": "{{ nodes.narrate.text }}"}) + assert len(by_code(graph, "governed.exit_text_source")) == 1 + assert by_code(graph, "governed.text_bypass") == [] + + +def test_every_exit_counts_at_both_levels(): + graph = traceable() + graph["nodes"] += [node("chat", "llm", "闲聊", prompt="你好"), + node("side", "output", "附带成果", fields=[{"name": "回答", "value": "{{ nodes.chat.text }}"}])] + graph["edges"] += [{"source": "start", "target": "chat"}, {"source": "chat", "target": "side"}] + [issue] = by_code(graph, "governed.text_bypass") + assert issue.node_id == "chat" and "「附带成果」" in issue.message and issue.level == "error" + # 已发布级别范围一样,只是轻重不同:没带契约的出口也给一条警告(用户拍板:G 规则受管挡、已发布警告) + [issue] = by_code(graph, "governed.text_bypass", "published") + assert issue.node_id == "chat" and issue.level == "warning" + + +def test_text_that_goes_through_a_report_does_not_bypass_it(): + graph = insert(traceable(), node("notes", "agent", "查数员", tools=["db_query__shop"], prompt="查 orders"), + before="write", after="fetch") + config(graph, "write")["notes_from"] = ["notes"] + assert g_issues(graph) == [] + + +@pytest.mark.parametrize("value, flagged", [ + ("{{ vars.facts.gmv }}", False), # 开了 cite_fields:assign_to 拿到的是逐格核对过的字段 + ("{{ nodes.ask.data.gmv }}", False), + ("{{ nodes.ask.text }}", True), # 它的原话照样是模型写的 +]) +def test_cited_agent_fields_are_data_not_text(value, flagged): + graph = insert(traceable(), node("ask", "agent", "查数员", tools=["db_query__shop"], prompt="查 orders", + output_schema={"type": "object", "properties": {"gmv": {"type": "number"}}}, + cite_fields=True, assign_to="facts"), before="done", after="fetch") + graph = insert(graph, node("shape", "transform", "整形", mode="template", template=value, assign_to="x"), + before="done", after="ask") + graph["nodes"].append(node("side", "output", "附带成果", fields=[{"name": "值", "value": "{{ vars.x }}"}])) + graph["edges"].append({"source": "shape", "target": "side"}) + assert bool(by_code(graph, "governed.text_bypass")) is flagged + + +# -------------------------------------------------------------------------- +# G3:报告撰写节点按最严的规则自查,并显式声明结论句策略 +# -------------------------------------------------------------------------- + + +def test_a_report_node_must_spell_out_the_strict_settings(): + graph = traceable(report={}) + [issue] = by_code(graph, "governed.report_policy") + assert issue.level == "error" and issue.node_id == "write" and issue.field == "numbers" + for piece in ("numbers: strict", "on_violation: fail", "claims: require_citation"): + assert piece in issue.message + assert [i.level for i in by_code(graph, "governed.report_policy", "published")] == ["warning"] + + +@pytest.mark.parametrize("key, value", [("numbers", "off"), ("on_violation", "flag"), ("claims", "off")]) +def test_a_looser_setting_is_named(key, value): + graph = traceable(report={**STRICT, key: value}) + [issue] = by_code(graph, "governed.report_policy") + assert issue.field == key and f"现在是 {value}" in issue.message + for other in set(STRICT) - {key}: + assert f"{other}:" not in issue.message + + +def test_graph_defaults_count_like_they_do_at_run_time(): + graph = traceable(report={"claims": "require_citation"}) + graph["defaults"] = {"numbers": "strict", "on_violation": "fail"} + assert by_code(graph, "governed.report_policy") == [] + graph["defaults"]["on_violation"] = "flag" + assert [i.field for i in by_code(graph, "governed.report_policy")] == ["on_violation"] + + +def test_an_unsupported_claims_value_is_left_to_validate(): + graph = traceable(report={**STRICT, "claims": "judge"}) + assert by_code(graph, "governed.report_policy") == [] + [bad] = [i for i in validate_graph(GraphSpec.model_validate(graph)).issues if i.code == "report.claims_invalid"] + assert bad.level == "error" and CLAIMS_LATER in bad.message + + +# -------------------------------------------------------------------------- +# G4:口径卡的输入不能来自计算角色的沙箱代码 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("role", [None, "compute"]) +def test_compute_code_feeding_a_caliber_card_is_refused(role): + graph = traceable() + extra = {} if role is None else {"evidence_role": role} + graph = insert(graph, node("calc", "code", "计算", code="print(1)", assign_to="calc", **extra), before="card", + after="fetch") + config(graph, "card")["metrics"].append({"id": "ratio", "expression": "vars.calc.ratio"}) + [issue] = by_code(graph, "governed.caliber_compute_input") + assert issue.level == "error" and issue.node_id == "calc" and issue.field == "evidence_role" + assert "「周报口径卡」" in issue.message and "source" in issue.message + assert [i.level for i in by_code(graph, "governed.caliber_compute_input", "published")] == ["warning"] + config(graph, "calc")["evidence_role"] = "source" + assert by_code(graph, "governed.caliber_compute_input") == [] + + +def test_compute_code_is_caught_behind_a_transform(): + graph = insert(traceable(), node("calc", "code", "计算", code="print(1)"), before="card", after="fetch") + graph = insert(graph, node("shape", "transform", "整形", mode="json", template="{{ nodes.calc.stdout }}", + assign_to="shaped"), before="card", after="calc") + config(graph, "card")["metrics"].append({"id": "ratio", "expression": "vars.shaped.ratio"}) + assert [i.node_id for i in by_code(graph, "governed.caliber_compute_input")] == ["calc"] + + +# -------------------------------------------------------------------------- +# G5:给口径卡供数的 agent 配 output_schema 并开 cite_fields +# -------------------------------------------------------------------------- + + +SCHEMA = {"type": "object", "properties": {"gmv": {"type": "number"}}} + + +def feeding_agent(**config_) -> dict: + graph = insert(traceable(), node("ask", "agent", "查数员", tools=["db_query__shop"], prompt="查 orders", + assign_to="facts", **config_), before="card", after="start") + config(graph, "card")["metrics"].append({"id": "gmv2", "expression": "vars.facts.gmv"}) + return graph + + +def test_an_agent_feeding_a_card_needs_an_output_schema(): + [issue] = by_code(feeding_agent(), "governed.caliber_agent_schema") + assert issue.level == "error" and issue.node_id == "ask" and issue.field == "output_schema" + assert "「周报口径卡」" in issue.message + + +def test_an_agent_with_a_schema_needs_cite_fields(): + graph = feeding_agent(output_schema=SCHEMA) + [issue] = by_code(graph, "governed.caliber_agent_cite_fields") + assert issue.node_id == "ask" and issue.field == "cite_fields" + assert by_code(graph, "governed.caliber_agent_schema") == [] + assert g_issues(feeding_agent(output_schema=SCHEMA, cite_fields=True)) == [] + + +def test_an_agent_that_only_feeds_the_report_is_not_a_card_input(): + graph = insert(traceable(), node("ask", "agent", "查数员", tools=["db_query__shop"], prompt="查 orders"), + before="write", after="fetch") + assert by_code(graph, "governed.caliber_agent_schema") == [] + + +def test_a_model_call_feeding_a_card_is_refused(): + graph = insert(traceable(), node("guess", "llm", "估算", prompt="估一个数", assign_to="g"), before="card", + after="start") + config(graph, "card")["metrics"].append({"id": "g", "expression": "vars.g"}) + [issue] = by_code(graph, "governed.caliber_model_input") + assert issue.node_id == "guess" and "「估算」(模型调用)" in issue.message + + +# -------------------------------------------------------------------------- +# 已发布级别只给警告;已经发布的版本照常运行 +# -------------------------------------------------------------------------- + + +def everything_wrong() -> dict: + graph = feeding_agent() + graph = insert(graph, node("calc", "code", "计算", code="print(1)", assign_to="calc"), before="card", + after="fetch") + config(graph, "card")["metrics"].append({"id": "ratio", "expression": "vars.calc.ratio"}) + graph = insert(graph, node("narrate", "llm", "叙述", prompt="写周报", assign_to="story"), before="done", + after="card") + graph = insert(graph, node("shape", "transform", "整形", mode="template", template="{{ vars.story }}", + assign_to="shaped"), before="done", after="narrate") + config(graph, "done")["fields"].append({"name": "摘要", "value": "{{ vars.shaped }}"}) + config(graph, "done")["contract"].update(report_from=None, narrative="{{ nodes.write.text }}") + config(graph, "write").clear() + return graph + + +async def test_the_published_level_only_warns(client): + graph = everything_wrong() + assert {i.code for i in g_issues(graph)} == G_CODES - {"governed.caliber_agent_cite_fields", + "governed.caliber_model_input"} + assert all(i.level == "error" for i in g_issues(graph)) + warned = g_issues(graph, "published") + assert {i.code for i in warned} == {i.code for i in g_issues(graph)} + assert all(i.level == "warning" for i in warned) + assert (await publish(client, graph, "published"))["ok"] is True + assert (await publish(client, graph, "governed"))["ok"] is False + + +def test_template_nine_passes_once_completed(): + tpl = next(t for t in TEMPLATES if t["name"].startswith("⑨")) + graph = copy.deepcopy({"nodes": tpl["nodes"], "edges": tpl["edges"]}) + # 模板原样:G 规则里至多差报告撰写节点那三项 + assert {i.code for i in g_issues(graph)} <= {"governed.report_policy"} + next(n for n in graph["nodes"] if n["type"] == "report")["data"]["config"].update(STRICT) + issues = publish_issues(GraphSpec.model_validate(graph), level="governed") + assert [i for i in issues if i.level == "error"] == [], [i.message for i in issues if i.level == "error"] + assert not {i.code for i in issues} & G_CODES + + +@pytest.fixture +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +async def test_an_already_governed_version_keeps_running_formally(client, engine_up, monkeypatch): + """升级前按受管发布的版本(报告节点没写那三项)照常发起正式运行;再发布时才按新规则查。""" + from app.providers import mock_model + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", + lambda self, messages: AIMessage(content="本周销售额 [[m:gmv]]。[[see:m:gmv]]")) + graph = traceable(report={}) + card = config(graph, "card") + card["metrics"] = [{"id": "gmv", "name": "销售额", "unit": "元", "expression": "vars.kpi.gmv"}] + graph["nodes"][1] = node("fetch", "transform", "取数", mode="expression", expression="{'gmv': 300.5}", + assign_to="kpi") + async with SessionLocal() as session: + wf = Workflow(name="证据门禁-老受管版本", graph=graph, status="governed", published_version=1) + session.add(wf) + await session.flush() + session.add(WorkflowVersion(workflow_id=wf.id, version=1, graph=graph, level="governed")) + await session.commit() + wf_id = wf.id + r = await client.post("/api/runs", json={"workflow_id": wf_id, "run_class": "formal"}) + assert r.status_code == 201, r.text + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, r.json()["id"]) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + break + assert row.status == "succeeded", row.error + # 下次发布按新规则:报告节点那三项没写明,受管发布被拦 + check = (await client.post(f"/api/workflows/{wf_id}/publish-check", json={"level": "governed"})).json() + assert check["ok"] is False + assert "governed.report_policy" in {i["code"] for i in check["issues"] if i["level"] == "error"} + + +# -------------------------------------------------------------------------- +# validate:报告撰写节点的 claims 只接受 off / require_citation +# -------------------------------------------------------------------------- + + +def claims_issues(value, *, on_defaults: bool = False) -> list: + graph = traceable(report={} if on_defaults else {"claims": value}) + if on_defaults: + graph["defaults"] = {"claims": value} + return [i for i in validate_graph(GraphSpec.model_validate(graph)).issues if i.code == "report.claims_invalid"] + + +@pytest.mark.parametrize("value", [None, "", "off", "require_citation"]) +def test_supported_claims_values_pass(value): + assert claims_issues(value) == [] + + +def test_judge_is_a_later_version(): + [issue] = claims_issues("judge") + assert issue.level == "error" and issue.node_id == "write" and issue.field == "claims" + assert CLAIMS_LATER in issue.message and "off" in issue.message and "require_citation" in issue.message + + +@pytest.mark.parametrize("value", ["strict", "Require_Citation", 1, ["off"], {"policy": "judge"}]) +def test_other_claims_values_are_errors(value): + [issue] = claims_issues(value) + assert issue.level == "error" and "off / require_citation" in issue.message and repr(value) in issue.message + + +def test_claims_from_graph_defaults_are_checked_too(): + [issue] = claims_issues("judge", on_defaults=True) + assert issue.node_id == "write" diff --git a/backend/tests/test_lint_contract_report_from.py b/backend/tests/test_lint_contract_report_from.py index 4f6cf71c..96467be0 100644 --- a/backend/tests/test_lint_contract_report_from.py +++ b/backend/tests/test_lint_contract_report_from.py @@ -100,4 +100,6 @@ def test_a_narrative_contract_is_unchanged(): contract = contract_of(graph) del contract["report_from"] contract["narrative"] = "{{ nodes.write.text }}" - assert [i for i in lint(graph) if i.level == "error"] == [] + # _lint_contract 对叙述模式照旧;受管级别另有门禁 G1 要求核对报告撰写节点的文档 + assert [i for i in lint(graph, "published") if i.level == "error"] == [] + assert [i.code for i in lint(graph) if i.level == "error"] == ["governed.report_from_required"] diff --git a/backend/tests/test_publish_check_api.py b/backend/tests/test_publish_check_api.py index c97355f9..9bff08d2 100644 --- a/backend/tests/test_publish_check_api.py +++ b/backend/tests/test_publish_check_api.py @@ -37,7 +37,8 @@ def weekly(**contract_overrides) -> dict: node("card", "metrics", "周报口径卡", caliber="周报口径", metrics=[ {"id": "gmv", "name": "销售额", "unit": "元", "expression": "cell(nodes.fetch, 0, 'gmv')"}, {"id": "orders", "name": "订单数", "unit": "单", "expression": "cell(nodes.fetch, 0, 'orders')"}]), - node("write", "report", "报告撰写", instructions="写周报"), + node("write", "report", "报告撰写", instructions="写周报", + numbers="strict", on_violation="fail", claims="require_citation"), node("done", "output", "出具", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], contract={k: v for k, v in contract.items() if v is not None}), ] diff --git a/backend/tests/test_report_claims.py b/backend/tests/test_report_claims.py new file mode 100644 index 00000000..22a6ce64 --- /dev/null +++ b/backend/tests/test_report_claims.py @@ -0,0 +1,191 @@ +"""报告节点的结论句策略 claims:off(默认,和以前一样)/ require_citation(没挂依据的结论句计入缺口)。 + +结论句要靠裁判模型判断支不支持,那是后续版本的事;这一版只做确定性的一半: +- require_citation 时写作提示明确要求每句结论挂 [[see:…]];没挂的照常产出、计数, + 出口按出具档位降档(io.py),报告节点自己不为它失败、不为它重写 +- report.checked 事件和节点产出都带上 claims,出口据此判档 +- 写 judge 时节点直接报错「结论句裁判在后续版本支持」,不能悄悄当成 off +- 升级前发起的运行一个字都不变 +""" +from __future__ import annotations + +import asyncio + +import pytest +from langchain_core.messages import AIMessage +from sqlalchemy import select + +from app.core import artifact_store +from app.db.base import SessionLocal +from app.db.models import Run, RunEvent +from app.engine import runner as runner_mod +from app.engine.evidence import CLAIMS_RULE, build_catalog, compose_doc, uncited_claims +from app.engine.runner import run_manager +from app.engine.schema import CLAIMS_LATER, REPORT_CLAIMS, claims_problem + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +METRICS = [{"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "expression": "vars.kpi.gmv"}] +#: 第一句挂了依据,第二句是没挂依据的结论(有方向词),第三句是连接性的话 +TEXT = "本周销售额 [[m:gmv]]。[[see:m:gmv]]增长主要来自新客。下面看细节。" + + +def weekly(**report): + nodes = [ + node("start", "input", fields=[{"name": "week", "default": "2026-W37"}]), + node("fetch", "transform", mode="expression", expression="{'gmv': 45678.5}", assign_to="kpi"), + node("caliber", "metrics", caliber="周报口径", caliber_version="v2", metrics=METRICS), + node("write", "report", instructions="写周报", **report), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]), + ] + ids = [n["id"] for n in nodes] + return {"nodes": nodes, "edges": [{"source": a, "target": b} for a, b in zip(ids, ids[1:])]} + + +def script(monkeypatch, reply=TEXT) -> list: + from app.providers import mock_model + + seen: list = [] + + def _decide(self, messages): + seen.append(list(messages)) + return AIMessage(content=reply) + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", _decide) + return seen + + +def lenient(monkeypatch): + """装作画布校验没拦住(校验以后放宽、或者绕过校验发起的运行):节点执行时自己也要守住这一关, + 报的是同一句话。""" + original = runner_mod.validate_graph + + def permissive(spec): + report = original(spec) + report.issues = [i for i in report.issues if i.field != "claims"] + report.ok = not any(i.level == "error" for i in report.issues) + return report + + monkeypatch.setattr(runner_mod, "validate_graph", permissive) + + +async def finish(graph) -> Run: + run = await run_manager.start(graph=graph, input_payload={}) + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + return row + raise AssertionError(f"run {run.id} 没跑完:{row.status}") + + +async def events(run_id: str, etype: str) -> list[RunEvent]: + async with SessionLocal() as session: + q = select(RunEvent).where(RunEvent.run_id == run_id, RunEvent.type == etype).order_by(RunEvent.seq) + return list((await session.execute(q)).scalars()) + + +async def checked_and_output(run_id: str) -> tuple[dict, dict]: + [checked] = await events(run_id, "report.checked") + finished = [e for e in await events(run_id, "node.finished") if e.node_id == "write"][0] + return checked.data, artifact_store.load(finished.data["artifact"]) + + +def prompt_of(seen: list) -> str: + return "\n".join(str(m.content) for m in seen[0]) + + +async def test_default_is_off_and_changes_nothing(monkeypatch): + seen = script(monkeypatch) + row = await finish(weekly()) + assert row.status == "succeeded", row.error + checked, output = await checked_and_output(row.id) + assert checked["claims"] == "off" and output["claims"] == "off" + assert CLAIMS_RULE not in prompt_of(seen) + assert checked["stats"]["uncited_claims"] == 1 + + +async def test_require_citation_asks_for_support_but_does_not_block(monkeypatch): + seen = script(monkeypatch) + row = await finish(weekly(claims="require_citation", on_violation="fail")) + assert row.status == "succeeded", row.error + assert len(seen) == 1, "没挂依据的结论句不触发重写" + assert CLAIMS_RULE in prompt_of(seen) + checked, output = await checked_and_output(row.id) + assert checked["claims"] == "require_citation" and output["claims"] == "require_citation" + assert checked["ok"] is True and checked["violations"] == [] + assert checked["stats"]["uncited_claims"] == output["stats"]["uncited_claims"] == 1 + doc = artifact_store.load(checked["doc_artifact"]) + [flagged] = uncited_claims(doc) + assert flagged["text"] == "增长主要来自新客。" and doc["markdown"][slice(*flagged["span"])] == flagged["text"] + + +async def test_judge_is_refused_until_it_exists(monkeypatch): + seen = script(monkeypatch) + with pytest.raises(ValueError) as info: # 画布校验先拦(E3-gov 的 validate) + await run_manager.start(graph=weekly(claims="judge"), input_payload={}) + assert CLAIMS_LATER in str(info.value) + lenient(monkeypatch) + row = await finish(weekly(claims="judge")) + assert row.status == "failed" and claims_problem("judge") in (row.error or ""), row.error + assert seen == [], "不支持的策略不该先花一次模型调用" + + +@pytest.mark.parametrize("value", ["sometimes", 1, ["require_citation"]]) +async def test_other_values_are_refused(monkeypatch, value): + script(monkeypatch) + lenient(monkeypatch) + row = await finish(weekly(claims=value)) + assert row.status == "failed", row.error + assert claims_problem(value) in row.error and "require_citation" in row.error + + +async def test_runs_from_before_the_upgrade_are_untouched(monkeypatch): + async def no_snapshot(session): + return None + + monkeypatch.setattr(runner_mod, "_agent_limits", no_snapshot) + seen = script(monkeypatch) + row = await finish(weekly()) + assert row.status == "succeeded", row.error + checked, output = await checked_and_output(row.id) + assert "claims" not in checked and "claims" not in output + assert CLAIMS_RULE not in prompt_of(seen) + + +# -------------------------------------------------------------------------- +# 出口和界面共用的纯函数 +# -------------------------------------------------------------------------- + + +def test_the_supported_policies_match_the_validator(): + from app.engine.nodes.report import CLAIMS, report_claims + from app.engine.schema import GraphSpec + + assert CLAIMS == REPORT_CLAIMS == ("off", "require_citation") + spec = GraphSpec.model_validate(weekly(claims="require_citation")) + assert report_claims(spec, spec.node_map()["write"]) == "require_citation" + spec = GraphSpec.model_validate(weekly()) + assert report_claims(spec, spec.node_map()["write"]) == "off" + spec.defaults["claims"] = "require_citation" # 图级 defaults 同样生效,和执行时一个规则 + assert report_claims(spec, spec.node_map()["write"]) == "require_citation" + + +def test_uncited_claims_lists_claim_units_without_support(): + catalog = build_catalog(nodes={"caliber": {"kind": "metric_set", "caliber": "c", "caliber_version": "v1", + "artifact": "f" * 64, + "metrics": [{"id": "gmv", "name": "销售额", "value": 1, "unit": ""}]}}) + doc = compose_doc("销售额 [[m:gmv]]。环比下降。[[see:m:gmv]]主要来自东区。连接的话。", catalog) + assert [u["text"] for u in uncited_claims(doc)] == ["主要来自东区。"] + assert all(set(u) == {"unit", "span", "text"} for u in uncited_claims(doc)) diff --git a/backend/tests/test_sql_guard_scan.py b/backend/tests/test_sql_guard_scan.py new file mode 100644 index 00000000..6fac3df6 --- /dev/null +++ b/backend/tests/test_sql_guard_scan.py @@ -0,0 +1,167 @@ +"""SQL 守卫的单遍扫描:去注释要认得字符串和标识符引号。 + +以前先去注释、后认字符串:字符串里的 `--`、`/*` 被当成注释删掉,执行的 SQL 和写的不一样 +(run_query 执行的正是 guard.check 的返回值)。只改成执行原文又会让守卫看到的和数据库执行的 +不一致,所以扫描必须一遍走完,同时认得注释、字符串、标识符。 +""" +from __future__ import annotations + +import sqlite3 +import uuid +from types import SimpleNamespace + +import pytest + +from app.data.engine import engines, run_query +from app.data.guard import SqlRejected, check, is_write, split_statements, strip_comments + + +def rejected(sql: str, *, readonly: bool = True, dialect: str | None = None) -> str | None: + try: + check(sql, readonly=readonly, source_name="测试库", dialect=dialect) + return None + except SqlRejected as e: + return str(e) + + +# ---------------------------------------------------------------- 字符串原样保留 + + +@pytest.mark.parametrize("dialect", [None, "sqlite", "postgresql", "mysql", "oracle"]) +@pytest.mark.parametrize("sql", [ + "SELECT '--' AS a", + "SELECT '/* x */' AS a", + "SELECT 'a -- b', 'c /* d' AS a", + 'SELECT "col--x" FROM t', +]) +def test_comment_markers_inside_quotes_are_kept(sql, dialect): + assert check(sql, readonly=True, dialect=dialect) == sql + assert strip_comments(sql, dialect=dialect) == sql + + +def test_real_comments_are_still_removed(): + assert check("SELECT 1 -- 行注释", readonly=True).strip() == "SELECT 1" + assert check("SELECT /* 块注释 */ 1", readonly=True).split() == ["SELECT", "1"] + assert check("SELECT '--' /* 注释 */ AS a", readonly=True).split() == ["SELECT", "'--'", "AS", "a"] + # 删掉的注释要留一个空格:'x'/**/AS 删成 'x'AS 就粘成一个词了 + assert check("SELECT 'x'/**/AS a", readonly=True).split() == ["SELECT", "'x'", "AS", "a"] + + +@pytest.mark.parametrize("sql,dialect,kept", [ + ("SELECT 1 /* a /* b */ c */ AS n", "postgresql", "SELECT 1 AS n"), # PG 的块注释可以嵌套 + ("SELECT E'it\\'s -- x' AS a", "postgresql", None), # E'' 里认反斜杠 + ("SELECT [a--b] FROM t", "sqlite", None), # SQLite 的方括号标识符 + ("SELECT q'[it's -- x]' AS a FROM dual", "oracle", None), # Oracle 的 q'[…]' + ("SELECT a$b$ AS c -- $b$", "postgresql", "SELECT a$b$ AS c"), # 标识符里的 $ 不开字符串 +]) +def test_dialect_specific_quoting(sql, dialect, kept): + assert check(sql, readonly=True, dialect=dialect) == (kept if kept is not None else sql) + + +def test_doubled_quote_escape_is_one_string(): + sql = "SELECT 'it''s -- fine' AS a" + assert check(sql, readonly=True) == sql + + +def test_dollar_quoted_body_is_a_string_on_postgres(): + sql = "SELECT $$ -- not a comment $$ AS a, $tag$ /* nor this */ $tag$ AS b" + assert check(sql, readonly=True, dialect="postgresql") == sql + + +def test_mysql_backslash_escape_without_ambiguity_is_allowed(): + sql = r"SELECT 'O\'Brien' AS s" + assert check(sql, readonly=True, dialect="mysql") == sql + + +def test_mysql_backslash_that_moves_a_comment_boundary_is_rejected(): + """默认 sql_mode 下是一个字符串,NO_BACKSLASH_ESCAPES 下后半截是注释:两种读法不一致就拒。""" + reason = rejected(r"SELECT 'a\'b -- c' AS s", dialect="mysql") + assert reason and "反斜杠" in reason + + +@pytest.fixture +async def data_source(tmp_path): + path = tmp_path / "scan.db" + conn = sqlite3.connect(path) + conn.execute("CREATE TABLE t (id INTEGER, note TEXT)") + conn.executemany("INSERT INTO t VALUES (?, ?)", + [(1, "a -- b"), (2, "c /* d */"), (3, "plain")]) + conn.commit() + conn.close() + src = SimpleNamespace( + id=f"scan-{uuid.uuid4().hex[:8]}", kind="sqlite", database=str(path), readonly=True, + name="测试源", options={}, password=None, username="u", host="h", port=None, + ) + yield src + await engines.invalidate(src.id) + + +async def test_query_with_comment_markers_in_strings_runs_and_returns_right_rows(data_source): + """字符串里的 -- 和 /* 要原样到达数据库,结果才对。""" + result = await run_query(data_source, "SELECT id FROM t WHERE note = 'a -- b'") + assert result.rows == [[1]] + result = await run_query(data_source, "SELECT id FROM t WHERE note = 'c /* d */' -- 真注释") + assert result.rows == [[2]] + result = await run_query(data_source, "SELECT '--' AS a, '/*' AS b") + assert result.rows == [["--", "/*"]] + result = await run_query(data_source, "SELECT id FROM t WHERE note LIKE '%--%' /* ' */ ORDER BY id") + assert result.rows == [[1]] + + +# ---------------------------------------------------------------- 拦截 + + +@pytest.mark.parametrize("sql,dialect", [ + ("SELECT '--'; DROP TABLE t", None), + ("SELECT '--'; DROP TABLE t", "sqlite"), + ("SELECT '/*'; DELETE FROM t", None), + ("SELECT 1 /* ' */; DELETE FROM t", None), + ("SELECT 1 -- '\n; DELETE FROM t", None), + ("SELECT $$--$$; DROP TABLE t", "postgresql"), + ("SELECT $x$ ; $x$; DELETE FROM t", "postgresql"), + ("SELECT '\\'; DELETE FROM t", "sqlite"), + ("SELECT '\\'; DELETE FROM t", "postgresql"), + ("SELECT '\\\\'; DELETE FROM t", "mysql"), + ("SELECT 1 # '\n; DELETE FROM t", "mysql"), + ("SELECT \"--\"; DELETE FROM t", None), + ("SELECT `--`; DELETE FROM t", None), +]) +def test_statement_smuggling_is_rejected(sql, dialect): + reason = rejected(sql, dialect=dialect) + assert reason is not None, f"放过了:{sql!r} ({dialect})" + reason_rw = rejected(sql, readonly=False, dialect=dialect) + assert reason_rw is not None, f"可写源上放过了:{sql!r} ({dialect})" + + +@pytest.mark.parametrize("sql,dialect", [ + ("SELECT '--' AS a FROM t WHERE 1=1 /* ' */ AND 0 = 0 UNION SELECT 1 FROM t FOR UPDATE", None), + ("SELECT $$ ' $$ AS a FROM t FOR UPDATE", "postgresql"), +]) +def test_hidden_writes_are_seen_through_quotes_and_comments(sql, dialect): + reason = rejected(sql, dialect=dialect) + assert reason and "只读" in reason, f"漏了:{sql!r}" + + +@pytest.mark.parametrize("sql,dialect", [ + ("SELECT '--'; DELETE FROM t", None), + ("SELECT 1 /* ' */; DELETE FROM t", "sqlite"), + ("SELECT $$--$$; DELETE FROM t", "postgresql"), +]) +def test_is_write_counts_smuggled_statements(sql, dialect): + """审批这道门:看不清的一律当写,等人批。""" + assert is_write(sql, dialect=dialect) + + +def test_split_statements_respects_quotes_before_comments(): + assert len(split_statements("SELECT '--'; SELECT 2")) == 2 + assert len(split_statements("SELECT '-- ; x'")) == 1 + assert len(split_statements("SELECT $$;$$", dialect="postgresql")) == 1 + + +def test_a_write_word_seen_under_another_mysql_setting_counts(): + """默认设置下是字符串,关了反斜杠转义就成了代码:哪种读法下会写,就按会写处理。""" + sql = r"SELECT 'can\'t delete' AS a" + reason = rejected(sql, dialect="mysql") + assert reason and "只读" in reason and "反斜杠" in reason + assert is_write(sql, dialect="mysql") + assert rejected(sql, readonly=False, dialect="mysql") is None diff --git a/backend/tests/test_version_level.py b/backend/tests/test_version_level.py index 0b6ed7b4..ac9f7c2c 100644 --- a/backend/tests/test_version_level.py +++ b/backend/tests/test_version_level.py @@ -51,7 +51,8 @@ def weekly() -> dict: node("fetch", "transform", mode="expression", expression="{'gmv': 300.5}", assign_to="kpi"), node("card", "metrics", caliber="周报口径", metrics=[ {"id": "gmv", "name": "销售额", "unit": "元", "expression": "vars.kpi.gmv"}]), - node("write", "report", instructions="写周报"), + node("write", "report", instructions="写周报", + numbers="strict", on_violation="fail", claims="require_citation"), node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], contract={"report_from": "write", "metrics_from": ["card"], "required": ["gmv"], "strict": True}), ] diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 2ab01609..b88453bf 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,6 +1,6 @@ import type { Approval, AutofixResult, Conversation, ConversationDetail, ConversationTurn, CustomTool, DataSource, EvidenceGraph, - EvidenceSegmentDetail, GraphSpec, KbDocument, MemoryItem, Provider, PublishCheck, PublishLevel, ReviewResult, Run, + EvidenceAudit, EvidenceSegmentDetail, GraphSpec, KbDocument, MemoryItem, Provider, PublishCheck, PublishLevel, ReviewResult, Run, RunEvent, RunStatus, Skill, ToolChange, ToolInfo, ToolTrust, ValidationIssue, VarIssue, Variable, Workflow, WorkflowVersion, } from '../types' @@ -702,6 +702,31 @@ export const api = { get( `/runs/${encodeURIComponent(runId)}/evidence/segments/${encodeURIComponent(segmentId)}${qs({ report: opts?.report })}`, opts), + /** + * 审计表:每个有状态的片段按状态分组(有出处 / 无证据 / 可疑实体 / 旧运行猜测),附上封存核对的结果。 + * groups 只要这几组(none、suspicious…) + */ + audit: (runId: string, opts?: RequestOptions & { groups?: string[] }) => + get(`/runs/${encodeURIComponent(runId)}/evidence/audit${qs({ groups: opts?.groups?.join(',') || undefined })}`, opts), + /** + * 导出审计表:?format=json / csv 作为附件下载。回来的是文件本身和后端给的文件名 + * (Content-Disposition;没给就用 evidence-<运行号前 8 位>.<格式>) + */ + auditExport: async (runId: string, format: 'json' | 'csv', groups?: string[]): Promise<{ blob: Blob; name: string }> => { + const path = `/runs/${encodeURIComponent(runId)}/evidence/audit${qs({ format, groups: groups?.join(',') || undefined })}` + let res: Response + try { + res = await fetch(BASE + path, { headers: actorHeader() }) + } catch (e: any) { + const err = new ApiError(0, NETWORK_MESSAGE, { kind: 'network', raw: `${e?.name ?? 'Error'}: ${e?.message ?? e}` }) + report(false, err) + throw err + } + if (!res.ok) throw failure(res.status, res.statusText, await res.text().catch(() => ''), path) + report(true) + const named = /filename\*?=(?:UTF-8'')?"?([^";]+)"?/i.exec(res.headers.get('Content-Disposition') ?? '')?.[1] + return { blob: await res.blob(), name: named ? decodeURIComponent(named) : `evidence-${runId.slice(0, 8)}.${format}` } + }, }, // ---- 会话 ---- diff --git a/frontend/src/canvas/Inspector.tsx b/frontend/src/canvas/Inspector.tsx index eddc778d..f2051b29 100644 --- a/frontend/src/canvas/Inspector.tsx +++ b/frontend/src/canvas/Inspector.tsx @@ -522,6 +522,10 @@ function FieldInput({ field, id, nodeId, syntax, value, config, invalid, describ {(field.options ?? []).map((o) => ( ))} + {/* config 里是个不在选项里的值:照实列出来(不能选回去),不让它看着像第一个选项 */} + {field.unknownLabel && typeof value === 'string' && value && !(field.options ?? []).some((o) => o.value === value) && ( + + )} ) diff --git a/frontend/src/canvas/NodeCard.tsx b/frontend/src/canvas/NodeCard.tsx index f039291c..6d756e2d 100644 --- a/frontend/src/canvas/NodeCard.tsx +++ b/frontend/src/canvas/NodeCard.tsx @@ -14,9 +14,10 @@ import { api } from '../api/client' import { isComposing, StatusBadge, toast } from '../components/ui' import { matchedSource } from '../lib/evidence' import { explainRunError } from '../lib/explain' -import { formatDuration, formatLapse, formatNumber, formatTokens, NONE } from '../lib/format' +import { formatDuration, formatLapse, formatNumber, formatTokens, NONE, shortId } from '../lib/format' import { statusMeta } from '../lib/status' -import { issuanceLabel, RUN_CLASS_LABEL } from '../lib/terms' +import { issuanceLabel, reportStampText, RUN_CLASS_LABEL } from '../lib/terms' +import { useReportStamp, type ReportStamp } from './reportStamp' import { ApprovalCard } from '../run/RunPanel' import type { NodeState, NodeTrace } from '../run/trace' import { useRunClock } from '../run/useRunClock' @@ -187,6 +188,23 @@ function matchedLines(matched: unknown): string { return `回指明细:\n${lines.join('\n')}${more}` } +/** 报告节点章的悬停说明:两个数怎么来的,缺口里各有什么 */ +function reportStampTitle(r: ReportStamp): string { + const st = r.stats + const n = (v: unknown) => (typeof v === 'number' && Number.isFinite(v) ? v : 0) + const uncited = n(st.uncited_claims) + return [ + `报告核对(这张图最近一次运行${r.runId ? ` ${shortId(r.runId)}` : ''}):${reportStampText(r.cited, r.none)}`, + typeof st.numbers === 'number' ? `数字 ${formatNumber(n(st.numbers_cited))}/${formatNumber(st.numbers)} 有出处` : '', + n(st.entities) || n(st.quotes) ? `表名字段名 ${formatNumber(n(st.entities))} 处、引文 ${formatNumber(n(st.quotes))} 处有出处` : '', + n(st.unresolved) ? `引用解析不了 ${formatNumber(n(st.unresolved))} 处` : '', + n(st.unknown_entities) ? `可能是编造的名字 ${formatNumber(n(st.unknown_entities))} 个` : '', + n(st.unverified_entities) ? `核对不了的名字 ${formatNumber(n(st.unverified_entities))} 个(只标注,不计入)` : '', + uncited ? `没挂依据的结论句 ${formatNumber(uncited)} 句${r.claims === 'require_citation' ? '(计入缺口)' : '(结论句策略不要求,不计入)'}` : '', + '完整清单在记录页这次运行的「证据」页签', + ].filter(Boolean).join('\n') +} + /** * 每个节点有几路上游(不算循环的回边)。排队中的节点据此说清在等什么: * 汇合点是「等其余上游」,单入口的只是轮到它之前那一拍。按拓扑缓存 @@ -724,6 +742,8 @@ function NodeCardImpl({ id, data, selected, isConnectable }: NodeProps const issuanceDetail = useStudio((s) => (type === 'output' ? (s.run?.output as any)?._issuance as Record | undefined ?? (issuanceOf(s.events).id === id ? issuanceOf(s.events).data : undefined) : undefined)) + // 报告节点:这张图最近一次运行的核对统计,画成卡上的章(没有就不画) + const reportStamp = useReportStamp(id, type === 'report') // 只有排队中的卡片要知道自己有几路上游,别的卡片不必为改图重跑这一趟 const fanIn = useStudio((s) => (state === 'queued' ? fanInOf(s.nodes, s.edges)[id] ?? 0 : 0)) @@ -1075,6 +1095,17 @@ function NodeCardImpl({ id, data, selected, isConnectable }: NodeProps )} + {/* 报告节点的章:「引用 N · 无证据 M」,这张图最近一次运行的 report.checked。回放时游标还没走到 + 那一刻不画。章不是按钮:完整清单在记录页的「证据」页签 */} + {reportStamp && (reportStamp.at == null || view.at == null || view.at >= reportStamp.at) && ( + + + {reportStampText(reportStamp.cited, reportStamp.none)} + + )} + {/* 出口:标签画在出线第一段的上方,不压在线上。跑过之后命中的那条亮起来、 落空的压暗——六出口的分支节点跑完,不这样的话谁也说不清它到底选了哪条。 运行前一律中性色:绿色的「通过」挂在还没跑的分支上,读起来像已经通过了 */} diff --git a/frontend/src/canvas/PublishDialog.tsx b/frontend/src/canvas/PublishDialog.tsx index b50a17ef..5c9979a8 100644 --- a/frontend/src/canvas/PublishDialog.tsx +++ b/frontend/src/canvas/PublishDialog.tsx @@ -581,6 +581,8 @@ function FixControl({ fix, pf }: { fix: PublishFix; pf: PreflightState }) { const picked = (v: unknown) => (fix.multiple ? Array.isArray(value) && value.some((x) => same(x, v)) : value !== undefined && same(value, v)) + // 选的是「交给 Copilot」那一项:和点「交给 Copilot」一回事,请求带 assist,等待时说 Copilot 在修 + const handoff = !fix.multiple && (fix.options ?? []).some((o) => o.handoff === true && picked(o.value)) return (
{fix.label}{fix.multiple ? ` · ${T.multiple}` : ''} @@ -603,8 +605,10 @@ function FixControl({ fix, pf }: { fix: PublishFix; pf: PreflightState }) {
{!ready && {T.chooseFirst}}
diff --git a/frontend/src/canvas/issues.ts b/frontend/src/canvas/issues.ts index 5fe2e053..bea9ca82 100644 --- a/frontend/src/canvas/issues.ts +++ b/frontend/src/canvas/issues.ts @@ -220,7 +220,8 @@ function normalizeFix(raw: any): PublishFix | null { if (!kind) return null const options = asArray(raw.options) .filter((o) => o && typeof o === 'object' && 'value' in o) - .map((o) => ({ value: o.value, label: asText(o.label ?? o.value), hint: o.hint ? asText(o.hint) : null })) + .map((o) => ({ value: o.value, label: asText(o.label ?? o.value), hint: o.hint ? asText(o.hint) : null, + ...(o.handoff === true ? { handoff: true } : {}) })) // 选项类没有候选就没法选:不给控件,比给一个空下拉强 if (kind === 'choice' && !options.length) return null return { @@ -353,7 +354,13 @@ const EXTRA_FIELDS: Record = { workflow_version: '钉住版本', upgrade_policy: '上游发了新版本时', defaults: '全图默认', } /** 点号后面那一段的叫法:契约里的键,全图默认里的审批策略 */ -const SUB_KEYS: Record = { ...CONTRACT_KEYS, approval: '审批策略' } +const SUB_KEYS: Record = { + ...CONTRACT_KEYS, approval: '审批策略', claims: '没挂依据的结论句', on_uncited: '没挂依据时', +} +/** 契约 claims 的 on_uncited:没挂依据的结论句怎么算 */ +const ON_UNCITED_LABEL: Record = { + ignore: '只标出来,不算缺口', degrade: '计入缺口、出具降档', withhold: '不予出具', +} /** 值是节点 id 的几个键(报告、契约的 metrics_from,契约的 report_from):预览里写节点名,不写 id */ const NODE_REF_KEYS = new Set(['metrics_from', 'report_from']) @@ -386,6 +393,7 @@ export function fixValueText(value: unknown, field?: string | null, nameOf?: Nod return APPROVAL_POLICY_LABEL[value as keyof typeof APPROVAL_POLICY_LABEL] } if (typeof value === 'boolean') return value ? '是' : '否' + if (key === 'on_uncited' && typeof value === 'string' && value in ON_UNCITED_LABEL) return ON_UNCITED_LABEL[value] if (key === 'workflow_version' && (typeof value === 'number' || /^\d+$/.test(String(value)))) return `v${value}` if (key && NODE_REF_KEYS.has(key)) { const ids = Array.isArray(value) ? value : [value] diff --git a/frontend/src/canvas/nodeDefs.ts b/frontend/src/canvas/nodeDefs.ts index c3ce5250..b31ff9d8 100644 --- a/frontend/src/canvas/nodeDefs.ts +++ b/frontend/src/canvas/nodeDefs.ts @@ -43,6 +43,11 @@ export interface FieldDef { disabled?: (config: Record) => string | null /** nodeRefs:只能选这一类的上游节点(报告的 metrics_from 只收口径卡) */ refType?: NodeType + /** + * select:config 里写着一个不在选项里的值(手写的、Copilot 写的、后续版本才支持的)时, + * 下拉里照实多列一项、写明为什么不认。不给的话浏览器会把它显示成第一个选项——看着像默认值 + */ + unknownLabel?: (value: string) => string advanced?: boolean } @@ -558,6 +563,30 @@ export const NODE_DEFS: Record = { key: 'max_repairs', label: '最多重写几次', type: 'number', min: 0, max: 3, placeholder: '1', help: '有违规时把清单交回去让模型重写。每次都是一整篇的调用,写不对的模型多给几次也大多写不对', }, + { + // 后端只认 off / require_citation(schema.REPORT_CLAIMS);judge 是四期的模型裁判,写了会被校验拦下。 + // 没写等于 off:select 显示第一项,正好是默认 + key: 'claims', label: '没挂依据的结论句', type: 'select', + options: [ + { value: 'off', label: '不管(默认):不参与出具判档' }, + { value: 'require_citation', label: '计入缺口:出具按档位降档' }, + ], + help: '结论句是带数字、方向词或因果词的句子。选「计入缺口」时,写作提示要求每句结论挂上 [[see:…]] 依据,' + + '没挂的记为缺口、出具降档(系统自动链接的表名字段名不算依据)。受管级别发布必须显式选它。' + + '结论句裁判在后续版本支持', + unknownLabel: (v) => (v === 'judge' ? 'judge:结论句裁判在后续版本支持,现在不能用' : `${v}:不认识的值,只能选上面两项`), + }, + { + // 后端只认 link / off(report.py 的 ENTITIES) + key: 'entities', label: '表名、字段名', type: 'select', advanced: true, + options: [ + { value: 'link', label: '核对,对得上的可以点开(默认)' }, + { value: 'off', label: '不核对' }, + ], + help: '报告里写到的表名、字段名按本次运行的表结构快照和查询核对:对得上的能点开看出处,' + + '反引号里写了哪里都没有的名字标成「可能是编造的名字」', + unknownLabel: (v) => `${v}:不认识的值,只能选 link 或 off`, + }, ...MODEL_FIELDS, { key: 'system', label: '角色设定', type: 'textarea', advanced: true, placeholder: '你是…' }, ...COMMON_TAIL, diff --git a/frontend/src/canvas/reportStamp.ts b/frontend/src/canvas/reportStamp.ts new file mode 100644 index 00000000..a39dad54 --- /dev/null +++ b/frontend/src/canvas/reportStamp.ts @@ -0,0 +1,73 @@ +import { useEffect } from 'react' +import { stampCounts } from '../lib/evidence' +import { useEvidence } from '../store/evidence' +import { useStudio } from '../store/studio' +import type { EvidenceStats, RunEvent } from '../types' + +/** + * 报告节点卡上的章:「引用 N · 无证据 M」,数据取这张图最近一次运行的 report.checked。 + * + * - 画布上挂着一次运行(刚跑的、从记录页回放的):就用它的事件,跟着 report.checked 实时落下;回放时 + * 游标还没走到那一刻不画 + * - 没挂运行:取这张工作流最近一次运行的证据图(只认封存范围内的 report.checked) + * - 两边都没有这个节点的核对(没跑过、跑到一半、老后端):不画章 + */ +export interface ReportStamp { + cited: number + none: number + stats: EvidenceStats + claims?: string | null + /** report.checked 落下的时刻(毫秒,和航迹同一条时间轴);取自证据图的没有 */ + at: number | null + runId: string | null +} + +interface Checked { stats: EvidenceStats; claims?: string | null; at: number | null; event: RunEvent } +const checkedCache = new WeakMap>() +/** 同一条事件给出同一个对象:选择器返回稳定引用,别的事件进来时报告卡不跟着重渲染 */ +const byEvent = new WeakMap() + +/** 这次运行里每个报告节点最后一次核对,按事件数组缓存(每条事件都会把所有卡片的选择器跑一遍) */ +function checkedOf(events: RunEvent[]): Map { + const hit = checkedCache.get(events) + if (hit) return hit + const out = new Map() + for (const e of events) { + if (e.type !== 'report.checked' || !e.node_id || !e.data?.stats || typeof e.data.stats !== 'object') continue + let c = byEvent.get(e) + if (!c) { + c = { stats: e.data.stats, claims: e.data.claims, at: typeof e.ts === 'number' ? e.ts * 1000 : null, event: e } + byEvent.set(e, c) + } + out.set(e.node_id, c) + } + checkedCache.set(events, out) + return out +} + +export function useReportStamp(nodeId: string, enabled: boolean): ReportStamp | null { + const workflowId = useStudio((s) => s.workflow?.id ?? null) + const runId = useStudio((s) => s.run?.id ?? null) + const live = useStudio((s) => (enabled && s.run ? checkedOf(s.events).get(nodeId) ?? null : null)) + const latest = useEvidence((s) => (enabled && workflowId ? s.latest[workflowId] : undefined)) + const loadLatest = useEvidence((s) => s.loadLatest) + const dropLatest = useEvidence((s) => s.dropLatest) + // 没挂运行时才去取最近一次运行;挂着运行时它就是「最近一次」,之前取的那份随之作废—— + // 「清除」把运行摘下来(或者换一张图再回来)时重新取,不拿这次运行之前的旧统计冒充「最近一次」 + useEffect(() => { + if (!enabled || !workflowId) return + if (runId) dropLatest(workflowId) + else void loadLatest(workflowId) + }, [enabled, workflowId, runId, loadLatest, dropLatest]) + if (!enabled) return null + if (runId) { + if (!live) return null + const counts = stampCounts(live.stats, live.claims) + return counts ? { ...counts, stats: live.stats, claims: live.claims, at: live.at, runId } : null + } + const data = latest?.status === 'ok' ? latest.data : undefined + const report = data?.reports[nodeId] + if (!report?.stats) return null + const counts = stampCounts(report.stats, report.claims) + return counts ? { ...counts, stats: report.stats, claims: report.claims, at: null, runId: data?.runId ?? null } : null +} diff --git a/frontend/src/dev/ui-harness.tsx b/frontend/src/dev/ui-harness.tsx index f52d0b65..0d9279d1 100644 --- a/frontend/src/dev/ui-harness.tsx +++ b/frontend/src/dev/ui-harness.tsx @@ -26,8 +26,11 @@ import { EvidenceDoc, EvidenceField } from '../run/EvidenceDoc' import { Markdown } from '../run/Markdown' import evidenceFixture from '../run/__tests__/evidence-doc.json' import evidenceQuery from '../run/__tests__/evidence-query.json' +import evidenceEntity from '../run/__tests__/evidence-entity.json' +import { EvidenceGuessView } from '../run/EvidenceGuess' +import { IssuanceBanner } from '../run/AssistantStream' import mdPinned from '../run/__tests__/markdown-pinned.json' -import type { EvidenceDocData } from '../types' +import type { EvidenceDocData, EvidenceGraph } from '../types' import '../index.css' /** @@ -189,6 +192,8 @@ const EV_DOC = evidenceFixture.doc as unknown as EvidenceDocData /** 第二期:查询单元格、整表、口径卡来源(evidence-query.json,compose_doc 真跑出来的) */ const EV_QUERY = evidenceQuery.doc as unknown as EvidenceDocData const EV_LEGACY = evidenceFixture.legacy +/** 第三期:表名字段名、可疑实体、核对不了的名字、逐字引文(evidence-entity.json,compose_doc 真跑出来的) */ +const EV_ENTITY = evidenceEntity.doc as unknown as EvidenceDocData /** 同一份旧契约的出具,去掉位置:老运行就是这样,只能按字符串标 */ const LEGACY_LOOSE = { matched: EV_LEGACY.matched.map(({ start: _s, end: _e, ...m }) => m), @@ -269,6 +274,38 @@ function EvidenceOnly() { + {/* + 第三期:表名字段名(反引号里的、标记的、自动链接的)、可疑实体、核对不了的名字、逐字引文。 + 片段接口由 check-evidence 用 page.route 伪造(按 runId 认出这一份夹具) + */} +
+
+
表名字段名、可疑实体、逐字引文
+ +
+
+ +
+ {/* 没有契约的旧运行:按数值猜的候选(guess_sources 真跑出来的),默认折叠 */} +
+
旧运行 · 按数值猜的可能来源
+ +
+ {/* 出具横幅那一行:数字都有出处,可疑名字、没挂依据的结论句也要写出来,可疑名字也给「定位下一处」 */} +
+ {([['suspect', 3, 2, 1], ['clean', 3, 0, 0], ['names', 0, 1, 0]] as const).map(([key, total, suspect, claims]) => ( +
+ { const w = window as any; w.__issuanceNext = (w.__issuanceNext ?? 0) + 1 }, + }} /> +
+ ))} +
+
长报告:{EV_LONG.blocks.length} 块,块级 content-visibility,超过 3000 字先折叠
@@ -561,6 +598,7 @@ function Harness() { lib: { format, status, keys, terms, errors, explain, health, actor, validation, evidence }, evidenceFixture, evidenceQuery, + evidenceEntity, } createRoot(document.getElementById('root')!).render( diff --git a/frontend/src/index.css b/frontend/src/index.css index 78df8d47..406a1cce 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -976,6 +976,14 @@ font-weight: 500; letter-spacing: 0; } +/* 报告节点的章(引用 N · 无证据 M)放在卡片下沿:上沿左边是血缘标签(产出 / 引用),右边是校验角标 */ +.nc-stamp.is-evidence { + top: auto; + /* 只压住卡片下沿 4px:卡片最后一行(校验提示、耗时)不被章挡住 */ + bottom: -17px; + right: 14px; + letter-spacing: 0; +} .nc-stamp.is-formal { color: var(--st-done); } .nc-stamp.is-degraded { color: var(--st-waiting); } .nc-stamp.is-withheld { color: var(--st-failed); } @@ -1596,6 +1604,25 @@ .ev-seg[data-ev-line="solid"] { text-decoration-style: solid; } /* 点状线比实线细碎,同样 1px 在小字号下几乎看不见 */ .ev-seg[data-ev-line="dotted"] { text-decoration-style: dotted; text-decoration-thickness: 1.5px; } +/* 引文:前面挂一个引号(字形那一条通道),颜色随状态——有出处的绿、原话对不上的琥珀。 + 引号是装饰,读屏从 aria-label 里听到「引文」,这里的替代文字给空串 */ +.ev-seg[data-ev-kind="quote"]::before { + content: "“"; + content: "“" / ""; + margin-right: 1px; + color: var(--ev-glyph, currentColor); + font-weight: 600; +} +/* 旧运行的候选:比无证据的点状线更淡、更细,绝不像确定性的实线。正文里的候选是 Markdown 的标记(.ev-mark), + 不是按钮:旧答案保留原来的版式,候选的明细在下面的清单里逐条列出 */ +.ev-seg[data-ev-state="candidate"] { text-decoration-thickness: 1px; } +.ev-mark { + text-decoration-line: underline; + text-decoration-style: dotted; + text-decoration-thickness: 1px; + text-decoration-color: var(--ev-line, currentColor); + text-underline-offset: 3px; +} .ev-seg:hover, .ev-seg[aria-expanded="true"] { background: var(--ev-soft, transparent); } .ev-seg:focus-visible, diff --git a/frontend/src/lib/evidence.ts b/frontend/src/lib/evidence.ts index 7d4dc956..7ac7e060 100644 --- a/frontend/src/lib/evidence.ts +++ b/frontend/src/lib/evidence.ts @@ -7,16 +7,21 @@ * 「未裁判」借 --st-cancelled(暗字色),「猜测」借 --st-idle(最淡的字色)—— * 绝不用确定性的绿,模型的判断不能长得像系统核对过的事实。 * - * 本期(数字层)实际只会出现两种:确定性(引用解析成功)和无证据(裸数字、引用 - * 不存在)。其余几种先把外观定下,后续期接上时不用再各处补。 + * 一期(数字层)只会出现两种:确定性(引用解析成功)和无证据(裸数字、引用不存在)。 + * 三期多了表名字段名、逐字引文:有出处的实体、引文和数字同一套「有出处」线型(状态就是 + * 确定性,另有 EVIDENCE_KIND_STYLE 给它们各自的字形和说法);可疑实体(可能是编造的 + * 名字)和核对不了的名字是无证据的两个变体,线型同无证据、字形和文字各不相同;旧运行 + * 按数值猜的候选用最淡的点状线。概率性的几种先把外观定下,四期接上时不用再各处补。 */ -import { EVIDENCE_KIND_LABEL, EVIDENCE_STATE_LABEL, EVIDENCE_TEXT, UPGRADE_POLICY_LABEL, evidenceTally } from './terms' +import { + EVIDENCE_AUDIT_TEXT, EVIDENCE_KIND_LABEL, EVIDENCE_STATE_LABEL, EVIDENCE_TEXT, UPGRADE_POLICY_LABEL, evidenceTally, +} from './terms' import { NONE, formatNumber, shortId } from './format' import type { - EvidenceBlock, EvidenceCaliberSource, EvidenceCaliberUpgrade, EvidenceDocData, EvidenceFieldRef, EvidenceGraph, - EvidenceInput, EvidenceLocator, EvidenceSeal, EvidenceSegment, EvidenceSegmentDetail, EvidenceStep, EvidenceUnit, - EvidenceViolation, ReviewResult, + EvidenceBlock, EvidenceCaliberSource, EvidenceCaliberUpgrade, EvidenceCandidate, EvidenceDocData, EvidenceFieldRef, + EvidenceGraph, EvidenceGuess, EvidenceInput, EvidenceLocator, EvidenceSeal, EvidenceSegment, EvidenceSegmentDetail, + EvidenceStats, EvidenceStep, EvidenceUnit, EvidenceViolation, ReviewResult, } from '../types' import type { MarkSpec } from '../run/Markdown' @@ -100,6 +105,20 @@ export const EVIDENCE_STATE: Record = { alert: false, phase: 3, hint: '旧运行按数值猜的可能来源,不能当证据', }, + // 可疑实体:线型同无证据(点状),字形和文字另起,一眼分得出「没写出处」和「名字可能是编的」 + suspect: { + code: 'suspect', label: EVIDENCE_STATE_LABEL.suspect, line: 'dotted', glyph: '?!', + color: 'var(--st-waiting)', decoration: 'var(--st-waiting)', soft: 'var(--st-waiting-soft)', + alert: true, phase: 3, + hint: '本次运行的表结构快照、查询用到的表、查询结果列里都没有这个名字,可能是编造的', + }, + // 核对不了:表结构快照不全,找不到不等于不存在。只是标注,不进 n / N 的跳转,颜色压低 + unverified: { + code: 'unverified', label: EVIDENCE_STATE_LABEL.unverified, line: 'dotted', glyph: '…', + color: 'var(--st-cancelled)', decoration: mix('--st-cancelled', 80), soft: 'var(--st-cancelled-soft)', + alert: false, phase: 3, + hint: '这个数据源的表太多,表结构快照只存了一部分:找不到这个名字,也说不准它不存在', + }, } /** 图例、检查脚本按这个顺序列 */ @@ -109,17 +128,58 @@ export const EVIDENCE_STATES = Object.keys(EVIDENCE_STATE) as EvidenceStateCode[ * 片段在正文里的状态。文字、结构片段(行首符号、表格竖线)是 null:不画线、不进键盘顺序。 * 后端的 probabilistic 在裁判给出结论之前按「未裁判」画 */ -export function segmentState(seg: Pick): EvidenceStateCode | null { +export function segmentState( + seg: Pick & Partial>, +): EvidenceStateCode | null { if (seg.kind === 'structural') return null switch (seg.state) { case 'deterministic': return 'deterministic' - case 'none': return 'none' + case 'none': + // 无证据的两个变体:可疑实体(反引号里发现的没有 cite,[[t:编造]] 的 cite 带 unknown)、核对不了 + if (seg.issue === 'unknown_entity' || seg.cite?.unknown) return 'suspect' + if (seg.issue === 'unverified_entity' || seg.cite?.unverified) return 'unverified' + return 'none' case 'probabilistic': return 'unjudged' case 'candidate': return 'candidate' default: return null } } +/** + * 有出处的片段里,实体和引文另有一层外观:线型、颜色和数字同一套「有出处」,字形和文字 + * 各自不同——引文前面挂一个引号,实体写在反引号里的照行内代码画。四个通道同 EVIDENCE_STATE + */ +export type EvidenceKindCode = 'entity' | 'quote' +export interface EvidenceKindStyle { + code: EvidenceKindCode + line: EvidenceLine + glyph: string + color: string + label: string + hint: string +} +export const EVIDENCE_KIND_STYLE: Record = { + entity: { + code: 'entity', line: 'solid', glyph: '', color: 'var(--st-done)', label: '有出处 · 表或字段', + hint: '本次运行的表结构快照、查询用到的表或查询结果列里真实存在的名字', + }, + quote: { + code: 'quote', line: 'solid', glyph: '“', color: 'var(--st-done)', label: '有出处 · 逐字引文', + hint: '在知识库检索命中的原文里逐字出现(空白归一化后比对)', + }, +} + +/** 片段是实体还是引文(状态另看 segmentState):其余返回 null */ +export function segmentKind(seg: Pick): EvidenceKindCode | null { + return seg.kind === 'entity' ? 'entity' : seg.kind === 'quote' ? 'quote' : null +} + +/** 片段给人看的名字:反引号里的实体去掉两个反引号 */ +export function segName(seg: Pick): string { + if (seg.kind === 'entity' && (seg.code || /^`[^`]+`$/.test(seg.text))) return seg.name ?? seg.text.replace(/^`|`$/g, '') + return seg.text +} + /** 一个句子(单元格、列表项)的正文:结构片段不算 */ export function unitText(unit: Pick): string { return (unit.segments ?? []).filter((s) => s.kind !== 'structural').map((s) => s.text).join('') @@ -144,10 +204,30 @@ export function sourceOf(seg: EvidenceSegment, doc?: Pick | null): string { const state = segmentState(seg) if (!state) return seg.text - const why = state === 'none' ? reasonOf(seg) : sourceOf(seg, doc) - return `${seg.text},${EVIDENCE_STATE[state].label}${why ? `:${why}` : ''}` + const why = state === 'none' || state === 'suspect' || state === 'unverified' ? reasonOf(seg) : sourceOf(seg, doc) + // 可疑实体的原话本身就以「可能是编造的名字」收尾,状态那几个字不再重复一遍 + const label = state === 'suspect' && why.includes(EVIDENCE_STATE.suspect.label) ? '' : EVIDENCE_STATE[state].label + return `${segName(seg)},${[label, why].filter(Boolean).join(':')}` } /** 在正文里画得出线的违规:指向的片段存在,而且不是结构片段 */ @@ -364,6 +452,9 @@ export function locatable(doc: EvidenceDocData, v: EvidenceViolation): EvidenceS export interface EvidenceTally { /** 数字总数:带引用的数字片段 + 裸数字(含列表序号、代码块标签里的) */ total: number + /** 可疑实体(可能是编造的名字)、核对不了的名字:数字那半句之外另说 */ + suspect?: number + unverified?: number cited: number /** 没有出处的数字:总数 − 有出处(裸数字、解析不了的数字引用)。横幅上「无证据 M」的 M */ none: number @@ -409,7 +500,11 @@ export function docTally(doc: EvidenceDocData): EvidenceTally { : uncited != null && unresolved != null ? Math.max(0, uncited + unresolved - none) : segs.filter((s) => segmentState(s) === 'none' && s.kind !== 'number').length - return { total, cited, none, other, hidden: hidden.length, structural, noSegment: hidden.length - structural } + // 三期:可疑实体、核对不了的名字。stats 缺这两个键(老文档、没开实体层)时按违规清单数 + const suspect = num(st.unknown_entities) ?? all.filter((v) => v.code === 'unknown_entity').length + const unverified = num(st.unverified_entities) ?? all.filter((v) => v.code === 'unverified_entity').length + return { total, cited, none, other, hidden: hidden.length, structural, noSegment: hidden.length - structural, + suspect, unverified } } /** @@ -431,6 +526,8 @@ export function statsTally(stats: EvidenceDocData['stats'] | null | undefined): export function tallySummary(t: EvidenceTally, keys: boolean): string { return [ evidenceTally(t.cited, t.total, t.other), + t.suspect ? `${formatNumber(t.suspect)} 个${EVIDENCE_STATE.suspect.label}` : '', + t.unverified ? `${formatNumber(t.unverified)} 个名字${EVIDENCE_STATE.unverified.label}` : '', t.structural ? EVIDENCE_TEXT.structuralCount(t.structural) : '', t.noSegment ? EVIDENCE_TEXT.noSegmentCount(t.noSegment) : '', keys ? EVIDENCE_TEXT.keysHint : '', @@ -619,3 +716,320 @@ export function codePointIndex(text: string): (cp: number) => number { export function segmentsOf(blocks: EvidenceBlock[]): EvidenceSegment[] { return blocks.flatMap((b) => (b.units ?? []).flatMap((u) => u.segments ?? [])) } + +// ------------------------------------------------------------------------- +// 三期:实体步骤、引文步骤、画布上的证据路径 +// ------------------------------------------------------------------------- + +/** + * 可疑实体的「最接近的已知名字」:接口给目录键(c:orders.amount),或对象 {alias, kind, name, table?} + * (字段的 name 只是字段名,带着 table 时写成 表.字段)。统一成去掉 t: / c: 前缀的名字,最多 3 个 + */ +export function closestNames(list: unknown): string[] { + if (!Array.isArray(list)) return [] + const names = list.map((x) => { + if (typeof x === 'string') return x + if (!x || typeof x !== 'object') return '' + const o = x as { alias?: unknown; name?: unknown; table?: unknown } + if (typeof o.alias === 'string' && o.alias) return o.alias + const name = typeof o.name === 'string' ? o.name : '' + return typeof o.table === 'string' && o.table && name && !name.includes('.') ? `${o.table}.${name}` : name + }).map((n) => n.replace(/^[tc]:/, '').trim()).filter(Boolean) + return [...new Set(names)].slice(0, 3) +} + +/** 实体的来历一行一句:「表结构快照」「查询 Q2 · 查询 SQL 用到的表」「查询 Q1 · 查询结果列」 */ +export function entitySources(list: unknown): { kind: string; text: string; truncated: boolean }[] { + if (!Array.isArray(list)) return [] + const out: { kind: string; text: string; truncated: boolean }[] = [] + for (const src of list) { + if (!src || typeof src !== 'object') continue + const kind = String((src as any).kind ?? '') + const alias = typeof (src as any).alias === 'string' ? (src as any).alias : '' + const what = EVIDENCE_TEXT.entitySource[kind] ?? kind + const text = [alias ? EVIDENCE_TEXT.query(alias) : '', what].filter(Boolean).join(' · ') + if (text && !out.some((o) => o.text === text)) out.push({ kind, text, truncated: !!(src as any).truncated }) + } + return out +} + +/** + * 引文在原文里的位置,切成前、中、后三截给面板高亮。match 是码点偏移(后端 Python 的下标), + * 先换成 JS 的 UTF-16 下标。对不上(越界、那一截的字和引文对不上)时按引文在原文里找一次, + * 再找不到就不高亮——宁可不标,不标错地方。原文很长时前后各留 context 个字,多的写省略号 + */ +export function quoteWindow(original: string, match: { start?: number; end?: number } | null | undefined, + quote: string, context = 120): { before: string; hit: string; after: string; cutBefore: boolean; cutAfter: boolean } | null { + if (!original) return null + const norm = (t: string) => t.replace(/\s+/g, '') + const at = codePointIndex(original) + let a = typeof match?.start === 'number' ? at(match.start) : -1 + let b = typeof match?.end === 'number' ? at(match.end) : -1 + const fits = a >= 0 && b > a && b <= original.length && (!quote || norm(original.slice(a, b)) === norm(quote)) + if (!fits) { + const i = quote ? original.indexOf(quote) : -1 + if (i < 0) return null + a = i + b = i + quote.length + } + const from = Math.max(0, a - context) + const to = Math.min(original.length, b + context) + return { before: original.slice(from, a), hit: original.slice(a, b), after: original.slice(b, to), + cutBefore: from > 0, cutAfter: to < original.length } +} + +/** + * 点开一个片段时,画布上该亮哪些节点:产出证据的(查询、口径卡、检索)和用到它的(报告)。 + * 先按文档目录认(打开就有),证据链取回来后再补上链里每一步的节点(口径卡的输入、查询) + */ +export function evidenceTrace( + seg: EvidenceSegment, doc: Pick, chain: EvidenceStep[] = [], +): { producers: string[]; consumers: string[] } { + const catalog = doc.catalog ?? {} + const producers = new Set() + const add = (id: unknown) => { if (typeof id === 'string' && id) producers.add(id) } + const entry = seg.cite?.alias ? catalog[seg.cite.alias] : undefined + add(entry?.node_id) + // 实体:它出现在哪几次查询里,那几次查询是谁跑的 + const aliases = [ + ...(Array.isArray(entry?.queries) ? entry.queries : []), + ...(Array.isArray(entry?.sources) ? entry.sources.map((s: any) => s?.alias) : []), + ] + for (const a of aliases) if (typeof a === 'string') add(catalog[a]?.node_id) + for (const step of chain) { + add(step.node_id) + for (const inp of step.inputs ?? []) add(inp.node_id) + for (const q of step.queries ?? []) if (typeof q === 'string') add(catalog[q]?.node_id) + } + const consumers = doc.node_id ? [doc.node_id] : [] + for (const c of consumers) producers.delete(c) + return { producers: [...producers], consumers } +} + +// ------------------------------------------------------------------------- +// 报告节点卡上的章 +// ------------------------------------------------------------------------- + +/** + * 章上的两个数:「引用 N」是有出处的片段(数字、值、表名字段名、引文),「无证据 M」是没出处的 + * 数字、解析不了的引用、可疑实体,结论句策略为 require_citation 时再加上没挂依据的结论句。 + * 统计缺了数字那两项(别的版本的后端)返回 null,不画章 + */ +export function stampCounts(stats: EvidenceStats | null | undefined, claims?: string | null): + { cited: number; none: number } | null { + const t = statsTally(stats) + if (!t) return null + const cited = t.cited + (num(stats?.values) ?? 0) + (num(stats?.entities) ?? 0) + (num(stats?.quotes) ?? 0) + const none = t.none + t.other + (num(stats?.unknown_entities) ?? 0) + + (claims === 'require_citation' ? num(stats?.uncited_claims) ?? 0 : 0) + return { cited, none } +} + +// ------------------------------------------------------------------------- +// 旧运行的猜测 +// ------------------------------------------------------------------------- + +/** 证据图里的猜测(legacy_text):可能是一份、一个列表、或按字段分的对象。认不出的丢掉 */ +export function guessesOf(graph: EvidenceGraph | null | undefined): EvidenceGuess[] { + if (!graph) return [] + const isGuess = (g: unknown): g is EvidenceGuess => + !!g && typeof g === 'object' && Array.isArray((g as EvidenceGuess).segments) + const from = graph.guess ?? (graph.mode === 'legacy_text' ? graph.legacy : null) + if (!from) return [] + if (isGuess(from)) return [from] + if (Array.isArray(from)) return from.filter(isGuess) + if (typeof from === 'object') { + // {字段名: guess} 或 {fields: {…}} + // {note, fields: [{field, markdown, segments, stats}]}(后端 legacy_text 的样子),或 {字段名: guess} + const note = typeof (from as any).note === 'string' ? (from as any).note : undefined + const inner = (from as any).fields && typeof (from as any).fields === 'object' ? (from as any).fields : from + if (Array.isArray(inner)) return inner.filter(isGuess).map((g) => ({ note, ...g })) + return Object.entries(inner).filter(([, g]) => isGuess(g)).map(([field, g]) => ({ note, field, ...(g as EvidenceGuess) })) + } + return [] +} + +/** 一个候选说一句:「查询 Q1 · 第 1 行 · gmv」「口径卡指标 销售额」,差值不为 0 时补一句 */ +export function candidateText(c: EvidenceCandidate): string { + const loc = c.locator ?? {} + let where = '' + if (c.kind === 'metric' || typeof loc.metric === 'string') { + const name = c.name ?? loc.metric ?? c.ref?.replace(/^m:/, '') ?? '' + where = `${EVIDENCE_KIND_LABEL.metric} ${name}`.trim() + } else { + const alias = c.alias ?? (typeof c.ref === 'string' ? c.ref.split('.')[0] : '') + where = [alias ? EVIDENCE_TEXT.query(alias) : EVIDENCE_KIND_LABEL.cell, locatorText(loc)].filter(Boolean).join(' · ') + } + const diff = typeof c.diff === 'number' && Number.isFinite(c.diff) && c.diff !== 0 + ? EVIDENCE_TEXT.guessDiff(evidenceValue(Math.abs(c.diff))) : '' + return [where, c.rendered && c.rendered !== NONE ? `= ${c.rendered}` : '', diff].filter(Boolean).join(' ') +} + +// ------------------------------------------------------------------------- +// 记录页的审计表 +// ------------------------------------------------------------------------- + +/** 分组的键和后端 /evidence/audit 一致:有出处 / 无证据 / 可疑实体 / 旧运行猜测 */ +export type AuditGroup = 'cited' | 'none' | 'suspicious' | 'candidate' +/** 表里分组的顺序:有问题的在前(正常态安静,异常态醒目)。导出的文件按后端的顺序 */ +export const AUDIT_GROUPS: readonly AuditGroup[] = ['none', 'suspicious', 'cited', 'candidate'] +export type AuditFilter = 'all' | 'problems' +/** 「只看无证据 / 可疑实体」对应的分组,也是导出时 ?groups= 的值 */ +export const PROBLEM_GROUPS: readonly AuditGroup[] = ['none', 'suspicious'] + +export interface AuditRow { + /** + * 行的键,一张表里各不相同(React 的 key,也是键盘走表时找行的依据):片段的行 `报告:片段`; + * 违规、没挂依据的结论句 `报告:片段:问题:位置`——同一段文字里可以有好几条同样的违规(粗体、链接里 + * 两个可疑名字),片段和问题都一样,只有位置不同。万一还撞上,后来的带 `#序号` + */ + key: string + /** 报告节点 id;旧运行的猜测是成果字段名 */ + report: string + /** 能在正文里打开的片段 id */ + seg?: string + text: string + state: EvidenceStateCode + group: AuditGroup + /** number / value / entity / quote / claim / violation */ + kind: string + /** 出处(有出处的)或原因(没有的) */ + source: string + ref?: string + /** 所在的句子 */ + sentence: string + /** 这件证据在不在封存范围内(封存核对也通过);没有证据的是 null */ + sealed: boolean | null +} + +const GROUP_OF: Record = { + deterministic: 'cited', none: 'none', suspect: 'suspicious', unverified: 'suspicious', candidate: 'candidate', + supported: 'cited', partial: 'none', unsupported: 'none', unjudged: 'cited', connective: 'cited', +} +const AUDIT_GROUP_SET = new Set(['cited', 'none', 'suspicious', 'candidate']) + +/** 反引号里的名字去掉反引号(后端审计行的 text 照正文原样带着) */ +const bareName = (kind: string, text: string) => (kind === 'entity' && /^`[^`]+`$/.test(text) ? text.slice(1, -1) : text) + +/** + * 后端 /evidence/audit 的 JSON → 表里的行。形状照 api/evidence.py(groups[].rows[]),缺字段的宽容: + * 状态按 state + issue 认(可疑实体、核对不了是无证据的两个变体),出处写 evidence,原因写 note + */ +export function auditFromApi(body: unknown): AuditRow[] { + const groups = body && typeof body === 'object' && Array.isArray((body as any).groups) ? (body as any).groups : [] + const rows: AuditRow[] = [] + const taken = new Map() + /** 后端每条违规一行:同一片段、同一个 code 的违规可以有好几条,键带上位置;还撞的按出现顺序编号 */ + const unique = (base: string) => { + const n = taken.get(base) ?? 0 + taken.set(base, n + 1) + return n ? `${base}#${n}` : base + } + groups.forEach((g: any, gi: number) => { + for (const [ri, r] of (Array.isArray(g?.rows) ? g.rows : []).entries()) { + if (!r || typeof r !== 'object') continue + const kind = String(r.kind ?? '') + const state = segmentState({ kind, state: r.state ?? 'none', issue: r.issue ?? undefined }) + ?? (r.group === 'candidate' ? 'candidate' : 'none') + const group: AuditGroup = AUDIT_GROUP_SET.has(r.group) ? r.group : AUDIT_GROUP_SET.has(g?.key) ? g.key : GROUP_OF[state] + const report = String(r.report ?? r.field ?? '') + const seg = typeof r.segment === 'string' && r.segment ? r.segment : undefined + const at = Array.isArray(r.span) && Number.isInteger(r.span[0]) ? String(r.span[0]) : `${gi}.${ri}` + rows.push({ + key: unique(seg && kind !== 'violation' && kind !== 'claim' ? `${report}:${seg}` + : `${report}:${seg ?? '-'}:${r.issue ?? kind}:${at}`), + report, seg: kind === 'claim' ? undefined : seg, + text: bareName(kind, String(r.text ?? '')) || NONE, state, group, kind, + source: String((group === 'cited' || group === 'candidate') && r.evidence ? r.evidence : r.note ?? r.evidence ?? ''), + ref: typeof r.ref === 'string' ? r.ref : undefined, + sentence: String(r.sentence ?? '').trim(), + sealed: typeof r.sealed === 'boolean' ? r.sealed : null, + }) + } + }) + return rows +} + +/** + * 后端没有审计接口时(老后端),按正文这份文档自己拼:每个有状态的片段一行(连接性文字、结构片段不进), + * 另加画不了线的违规(列表序号、代码块标签里的数字,句末依据里写错的引用)——键盘用户在表里能看全每一处。 + * 封存状态按证据图里同一份报告、同一个别名的 sealed 取;没有证据图时是 null + */ +export function auditRows(doc: EvidenceDocData, opts: { report?: string; graph?: EvidenceGraph | null } = {}): AuditRow[] { + const report = opts.report ?? doc.node_id ?? '' + const sealedOf = (alias?: string): boolean | null => { + if (!alias || !opts.graph?.evidence) return null + const hit = opts.graph.evidence.find((e) => e?.alias === alias && (!e.report || !report || e.report === report)) + return typeof hit?.sealed === 'boolean' ? hit.sealed : null + } + const rows: AuditRow[] = [] + const shown = new Set() + for (const block of doc.blocks ?? []) { + for (const unit of block.units ?? []) { + const sentence = unitText(unit).trim() + for (const seg of unit.segments ?? []) { + const state = segmentState(seg) + if (!state) continue + shown.add(seg.id) + const bad = state === 'none' || state === 'suspect' || state === 'unverified' + rows.push({ + key: `${report}:${seg.id}`, report, seg: seg.id, text: segName(seg), state, group: GROUP_OF[state], + kind: seg.kind, source: bad ? reasonOf(seg) : sourceOf(seg, doc), ref: seg.ref, sentence, + sealed: bad ? null : sealedOf(seg.cite?.alias), + }) + } + } + } + ;(doc.violations ?? []).forEach((v, i) => { + if (!GAP_CODES.has(v.code) && v.code !== 'unknown_entity' && v.code !== 'unverified_entity') return + // 画得出线的违规已经是上面的一行;粗体、链接里的可疑名字没切片段(指着一段文字),也在这里补上 + if (v.segment && shown.has(v.segment)) return + const state: EvidenceStateCode = v.code === 'unknown_entity' ? 'suspect' : v.code === 'unverified_entity' ? 'unverified' : 'none' + rows.push({ + key: `${report}:v${i}`, report, text: v.text ?? (v.ref ? `[[${v.ref}]]` : NONE), state, group: GROUP_OF[state], + // 「正文里画不了线」由表格按行能不能打开来写(后端的行也一样),这里只放违规本身的话 + kind: 'violation', source: v.message ?? '', ref: v.ref, + sentence: (v.context ?? '').trim(), sealed: null, + }) + }) + return rows +} + +/** 旧运行的猜测 → 表里的行:有候选的数字进「旧运行猜测」,没有的进「无证据」(和后端一致) */ +export function guessRows(guess: EvidenceGuess): AuditRow[] { + const field = guess.field ?? '' + return (guess.segments ?? []).filter((s) => s.kind === 'number').map((s) => { + const hit = s.state === 'candidate' && !!s.candidates?.length + return { + key: `${field}:${s.id}`, report: field, text: s.text, state: hit ? 'candidate' as const : 'none' as const, + group: hit ? 'candidate' as const : 'none' as const, kind: 'number', + source: hit ? (s.candidates ?? []).map(candidateText).join(';') : EVIDENCE_TEXT.guessNone, + sentence: '', sealed: null, + } + }) +} + +/** 按组归拢;problems 只留无证据和可疑实体两组。空组不返回 */ +export function groupRows(rows: AuditRow[], filter: AuditFilter = 'all'): { group: AuditGroup; rows: AuditRow[] }[] { + const keep = filter === 'problems' ? new Set(PROBLEM_GROUPS) : null + return AUDIT_GROUPS.filter((g) => !keep || keep.has(g)) + .map((group) => ({ group, rows: rows.filter((r) => r.group === group) })) + .filter((g) => g.rows.length > 0) +} + +/** 后端没有导出接口时按表里的行导出的 CSV:中文表头,逗号、引号、换行转义,公式字符开头的格子前面加 ' */ +export function auditCsv(rows: AuditRow[]): string { + const cell = (v: unknown) => { + let t = v == null ? '' : String(v) + // 电子表格会把 = + - @ 开头的格子当公式算:报告里的字来自模型,不能照原样写进去(纯数字除外) + if (/^[=+\-@\t\r]/.test(t) && !/^[+-]?[\d,.]+%?$/.test(t)) t = `'${t}` + return /[",\n\r]/.test(t) ? `"${t.replace(/"/g, '""')}"` : t + } + const c = EVIDENCE_AUDIT_TEXT.cols + const head = ['分组', c.report, '片段', c.text, c.state, c.source, '引用', c.sentence, c.seal] + const seal = (v: boolean | null) => (v == null ? '' : v ? '是' : '否') + return [head, ...rows.map((r) => [ + EVIDENCE_AUDIT_TEXT.groups[r.group], r.report, r.seg ?? '', r.text, EVIDENCE_STATE[r.state].label, r.source, + r.ref ?? '', r.sentence, seal(r.sealed), + ])].map((r) => r.map(cell).join(',')).join('\r\n') +} diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 5a6460fc..da660422 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -144,6 +144,8 @@ export const EVIDENCE_STATE_LABEL = { none: '无证据', connective: '连接性文字', candidate: '猜测的来源', + suspect: '可能是编造的名字', + unverified: '核对不了', } as const /** @@ -188,9 +190,9 @@ export const EVIDENCE_TEXT = { input: '运行输入', seal: '封存', sealOk: '已封存 · 核对一致', - sealDocOk: '报告文档已封存 · 核对一致(这个数字本身没有证据可封存)', - /** 片段接口取不到、封存状态是从整次运行的证据图查的:只核对了文档在封存范围内,这个数的链没逐项核对 */ - sealDocChain: '报告文档已封存 · 核对一致(这个数字的证据链这次没取到,没有逐项核对)', + sealDocOk: '报告文档已封存 · 核对一致(这一段本身没有证据可封存)', + /** 片段接口取不到、封存状态是从整次运行的证据图查的:只核对了文档在封存范围内,这一段的链没逐项核对 */ + sealDocChain: '报告文档已封存 · 核对一致(这一段的证据链这次没取到,没有逐项核对)', sealBad: '已封存 · 核对不一致', sealOpen: '尚未封存', sealOutside: '不在封存范围内', @@ -264,8 +266,114 @@ export const EVIDENCE_TEXT = { `口径卡「${caliber || '—'}」${version},来自「${workflow}」${wfVersion}`, caliberUpgrade: (latest: string | null, policy: string) => `${latest ? `上游已有 ${latest}` : '上游有新版本'},按「${policy}」处置`, + + // ---- 第三期:表名字段名、逐字引文、旧运行的猜测 ---- + entity: '表结构', + table: '表', + column: '字段', + entityQueries: (aliases: string[]) => `出现在查询 ${aliases.join('、')}`, + entityNoQuery: '这次的查询里没有用到它,只在表结构快照里', + syncedAt: (at: string) => `表结构快照同步于 ${at}`, + snapshotPartial: '库里的表太多,表结构快照只存了一部分', + columnType: '字段类型', + ownerTables: (tables: string[]) => `好几张表都有这个字段:${tables.join('、')}`, + entitySource: { schema: '表结构快照', sql: '查询 SQL 用到的表', result: '查询结果列' } as Record, + entityPending: '正在取表结构…', + entityMissing: '字段类型、同步时间要从证据接口取,这一次没取到', + /** 反引号里写了个名字,哪里都没有(没有 cite 时的原因;有 cite 的用后端给的原话) */ + suspect: '本次运行的表结构快照、查询用到的表、查询结果列里都没有这个名字,可能是编造的名字', + unverified: '表结构快照只存了一部分,查询用到的表、查询结果列里也没有这个名字,核对不了它存不存在', + suspectTag: '可疑名字', + /** 出具横幅那一行:报告要求结论句挂依据(claims: require_citation)时,没挂的有几句 */ + uncitedClaimsTag: '没挂依据的结论句', + closest: '最接近的已知名字', + checked: (schemas: number, queries: number) => + `查过 ${formatNumber(schemas)} 份表结构快照、${formatNumber(queries)} 次查询`, + tableColumns: (n: number, view: boolean) => `${view ? '视图' : '表'} · ${formatNumber(n)} 个字段`, + columnTypes: (types: Record) => + `各表的类型:${Object.entries(types).map(([t, v]) => `${t} ${v}`).join('、')}`, + quoteBad: '按记下的位置从原文里切出来,和这句引文对不上:原文可能被改过', + quoteCollection: (c: string) => `知识库「${c}」`, + closestNone: '没有相近的已知名字', + quote: '逐字引文', + quoteFrom: (title: string) => `出自「${title}」`, + quoteChunk: (n: number) => `第 ${formatNumber(n)} 段`, + quoteHit: '引文在原文里的位置(高亮)', + quotePending: '正在取原文…', + quoteMissing: '原文要从证据接口取,这一次没取到', + quoteMiss: '原文里找不到这句话:写作者写的不是一字不差的原话', + quoteHash: '检索快照取回时哈希对不上,疑似被改过:原文不能当证据', + /** 旧运行:按数值猜的候选 */ + guessTitle: '猜测的来源', + guessNote: '猜测的来源,不能当证据:按数值在这次运行已封存的查询结果和口径卡里找相同的值,同值的巧合很多', + guessCount: (guessed: number, numbers: number) => + numbers ? `${formatNumber(numbers)} 个数字里 ${formatNumber(guessed)} 个找到了可能的来源` : '答案里没有数字', + guessNone: '没找到数值相同的格子', + guessDiff: (d: string) => `相差 ${d}`, + guessCandidates: '可能来自', } as const +/** + * 记录页「证据」页签:报告、常驻面板、审计表。审计表按状态分组,四组的说法和片段状态同一套 + */ +export const EVIDENCE_AUDIT_TEXT = { + tab: '证据', + tabTitle: '报告里每一段的出处:左边报告、右边证据、下面整张清单,可以只看有问题的、导出', + title: '证据清单', + groups: { + none: '无证据', + suspicious: '可疑实体', + cited: '有出处', + candidate: '旧运行猜测', + } as Record, + groupHint: { + none: '裸数字、解析不了的引用、没挂依据的结论句', + suspicious: '表名、字段名在本次运行的表结构和查询里找不到', + cited: '系统从证据里取值、核对过的片段', + candidate: '按数值猜的可能来源,不能当证据,默认收起', + } as Record, + filterLabel: '清单筛选', + filterAll: '全部', + filterProblems: '只看无证据 / 可疑实体', + cols: { text: '片段', state: '状态', source: '出处 / 原因', sentence: '所在的句子', report: '报告', seal: '封存' }, + sealIn: '在封存范围内', + sealOut: '不在封存范围内', + sealNa: '—', + exportJson: '导出 JSON', + exportCsv: '导出 CSV', + exported: (name: string) => `已导出 ${name}`, + exportLocal: '后端没有导出接口:按页面上这张清单导出(没有经过后端的封存核对)', + fallbackUnsupported: '后端还没有审计接口:清单按页面上的报告拼出来,每行的封存状态取自证据图', + fallbackError: (why: string) => `审计接口没取到(${why}):清单按页面上的报告拼出来`, + empty: '这一组没有片段', + emptyProblems: '没有无证据、可疑实体的片段', + hidden: '正文里画不了线,只在这里列出', + hiddenWhy: '这一处在列表序号、代码块标签、粗体或链接里,或者在句末依据里:正文里没有能画线、能点开的片段,只在这张清单里列出', + keys: '在清单里用上下方向键逐行走,回车在右边打开这一段的证据', + panelIdle: '点报告里的片段,或清单里的一行,这里显示它的出处', + sealTitle: '封存核对', + sealOk: (seq?: number) => `已封存 · 核对一致${seq != null ? `(封存到第 ${formatNumber(seq)} 条事件)` : ''}`, + sealBad: '已封存 · 核对不一致:封存之后有事件被改过、删过或插过', + sealOpen: '尚未封存:运行还没收尾,证据还可能变', + sealUnknown: '封存状态拿不到', + docOk: (node: string) => `报告「${node}」的文档哈希一致`, + docBad: (node: string) => `报告「${node}」的文档和哈希对不上,疑似被改过`, + docGone: (node: string) => `报告「${node}」的文档在工件库里取不回来`, + mode: { + none: '这次运行没有报告文档,也没有出具契约:没有可以逐段展示的证据', + legacy_contract: '旧版出具:按数值匹配口径卡,不是显式引用。同一个值对得上好几个指标时,出处不唯一', + legacy_text: '这次运行没有出具契约:下面是按数值猜的可能来源,默认收起', + } as Record, + modeLabel: { cited: '逐段引用', none: '没有证据', legacy_contract: '旧版出具', legacy_text: '没有契约的旧运行' } as Record, + legacyMatched: (n: number) => `按数值回指上 ${formatNumber(n)} 个数字`, + legacyUnmatched: (n: number) => `${formatNumber(n)} 个回指不上`, + legacyNoField: '回指的位置对不上成果里的任何一个字段,只列出数字,不在正文上标', +} as const + +/** 报告节点卡上的章:这张图最近一次运行的报告核对统计 */ +export const reportStampText = (cited: number, none: number) => + `引用 ${formatNumber(cited)} · 无证据 ${formatNumber(none)}` + /** * 口径升版的三种处置,和后端 governance.UPGRADE_POLICY_LABELS 同一套说法。子工作流和 * 钉版本的口径卡(caliber_from)共用 diff --git a/frontend/src/pages/runs/EvidencePane.tsx b/frontend/src/pages/runs/EvidencePane.tsx new file mode 100644 index 00000000..f05775c3 --- /dev/null +++ b/frontend/src/pages/runs/EvidencePane.tsx @@ -0,0 +1,532 @@ +import { + useCallback, useEffect, useId, useLayoutEffect, useMemo, useRef, useState, type KeyboardEvent as ReactKeyboardEvent, +} from 'react' +import { ChevronRight, Download, FileText } from 'lucide-react' +import clsx from 'clsx' +import { ApiError, api } from '../../api/client' +import { ErrorState, Skeleton, Spinner, StatusBadge, rovingTarget, toast, useRadioGroup } from '../../components/ui' +import { + EVIDENCE_STATE, PROBLEM_GROUPS, auditCsv, auditFromApi, auditRows, groupRows, guessRows, guessesOf, issuanceMarks, + segmentState, type AuditFilter, type AuditGroup, type AuditRow, +} from '../../lib/evidence' +import { formatNumber } from '../../lib/format' +import { EVIDENCE_AUDIT_TEXT as T } from '../../lib/terms' +import { EvidenceDoc, type EvidenceDocHandle } from '../../run/EvidenceDoc' +import { EvidenceGuessView } from '../../run/EvidenceGuess' +import { Markdown, inlineStyles } from '../../run/Markdown' +import { useEvidence } from '../../store/evidence' +import type { EvidenceAudit, EvidenceDocData, EvidenceGraph, Run } from '../../types' + +/** + * 记录页的「证据」页签(方案 6.4):左边报告,右边常驻的证据面板,报告下面是整张审计表。 + * + * - 报告按证据图里封存的那几份画(片段接口也只认它们):文档按工件 id 取、取回时后端复验哈希; + * 证据图说哈希对不上的,不画成可点的报告,醒目地说它疑似被改过。 + * - 审计表按状态分组(无证据、可疑实体在前),可以只看有问题的两组,可以导出 JSON / CSV + * (后端 /evidence/audit?format=…;老后端没有这个接口时按页面上的清单导出,并照实说)。 + * 键盘用户在表里能看全每一处:一个 Tab 位,↑/↓ 逐行走,回车在右边打开那一段。 + * - 没有报告的运行照实说是哪种:没有证据(none)、旧版出具按数值匹配(legacy_contract)、 + * 没有契约按数值猜的候选(legacy_text,默认收起,写明不能当证据)。 + * + * 窄的时候(详情栏不到 880px)不留右栏:面板从底部抽出,和问数据页一样。 + */ +export function EvidencePane({ run, output, labelOf, refreshKey }: { + run: Run + /** 这次运行的成果(旧契约按位置标数字时要取那个字段的原文) */ + output: Record | null + labelOf: (id?: string | null) => string | undefined + /** 运行又往前走了(事件数、状态变了):证据图和审计表重取 */ + refreshKey: string +}) { + const graphSlot = useEvidence((s) => s.graphs[run.id]) + const loadGraph = useEvidence((s) => s.loadGraph) + const reloadGraph = useEvidence((s) => s.reloadGraph) + const first = useRef(true) + useEffect(() => { + if (first.current) { first.current = false; void loadGraph(run.id); return } + void reloadGraph(run.id) + }, [run.id, refreshKey, loadGraph, reloadGraph]) + const audit = useAudit(run.id, refreshKey) + const graph = graphSlot?.data + + const [root, setRoot] = useState(null) + const wide = useWide(root, 880) + const [dock, setDock] = useState(null) + const [opened, setOpened] = useState>(() => new Set()) + const handles = useRef(new Map()) + + // 报告文档:证据图里封存的那几份,按工件 id 取(store 永久缓存) + const reports = useMemo(() => (graph?.mode === 'cited' ? graph.reports ?? [] : []) + .filter((r) => typeof r?.doc_artifact === 'string' && !!r.doc_artifact && r.hash_ok !== false), [graph]) + const docs = useEvidence((s) => s.docs) + const loadDoc = useEvidence((s) => s.loadDoc) + useEffect(() => { for (const r of reports) void loadDoc(r.doc_artifact!) }, [reports, loadDoc]) + const docOf = useCallback((node?: string): EvidenceDocData | undefined => { + const r = reports.find((x) => x.node_id === node) + const slot = r?.doc_artifact ? docs[r.doc_artifact] : undefined + return slot?.status === 'ok' ? slot.data : undefined + }, [reports, docs]) + const guesses = useMemo(() => guessesOf(graph), [graph]) + + // 审计表的行:后端的审计接口优先(每行的封存核对是后端做的);老后端没有它时按页面上的文档、猜测自己拼 + const fallback = audit.status === 'unsupported' || audit.status === 'error' + const rows = useMemo(() => { + if (audit.status === 'ok') return auditFromApi(audit.data) + if (!fallback || !graph) return [] + if (graph.mode === 'legacy_text') return guesses.flatMap(guessRows) + return reports.flatMap((r) => { + const doc = docOf(r.node_id) + return doc ? auditRows(doc, { report: r.node_id, graph }) : [] + }) + }, [audit, fallback, graph, guesses, reports, docOf]) + const [filter, setFilter] = useState('all') + + // 哪些行能在正文里打开:`报告:片段` 的集合,一份文档算一次(长报告几千个片段,不能每行各扫一遍)。 + // 只收有状态的片段,和 EvidenceDoc 能打开的是同一批:违规行指着的文字片段(粗体、链接里的可疑名字)、 + // 结构片段(列表序号 100. 里的数字)在正文里没有能点的地方,画成按钮就是点了没反应 + const openable = useMemo(() => { + const keys = new Set() + for (const r of reports) { + const doc = docOf(r.node_id) + for (const b of doc?.blocks ?? []) for (const u of b.units ?? []) for (const seg of u.segments ?? []) { + if (segmentState(seg)) keys.add(`${r.node_id}:${seg.id}`) + } + } + return keys + }, [reports, docOf]) + const canOpen = useCallback((row: AuditRow) => !!row.seg && openable.has(`${row.report}:${row.seg}`), [openable]) + const openRow = useCallback((row: AuditRow) => { + if (row.seg) handles.current.get(row.report)?.open(row.seg) + }, []) + + const onView = useCallback((report: string, open: boolean) => { + setOpened((prev) => { + if (prev.has(report) === open) return prev + const next = new Set(prev) + if (open) next.add(report) + else next.delete(report) + return next + }) + }, []) + + const mode = graph?.mode ?? audit.data?.mode + const seal = audit.data?.seal ?? graph?.seal + + if (!graph && graphSlot?.status !== 'error') { + return ( +
+ +
+ ) + } + if (!graph) { + return void reloadGraph(run.id)} className="flex-1" /> + } + + return ( +
+
+
+ + + {mode === 'cited' && reports.map((r) => { + const doc = docOf(r.node_id) + const slot = r.doc_artifact ? docs[r.doc_artifact] : undefined + return ( +
+
+ + + 报告「{labelOf(r.node_id) ?? r.node_id}」{r.fields?.length ? ` · 成果字段 ${r.fields.join('、')}` : ''} + +
+ {doc ? ( + { handles.current.set(r.node_id!, h) }} doc={doc} artifact={r.doc_artifact} + runId={run.id} label={r.fields?.join('、') || r.node_id} tally + panel={wide ? 'dock' : 'auto'} dock={dock} onView={(o) => onView(r.node_id!, o)} /> + ) : slot?.status === 'error' ? ( +

{T.docGone(labelOf(r.node_id) ?? r.node_id ?? '')}

+ ) : } +
+ ) + })} + {/* 证据图说哈希对不上的报告:不画成可点的,醒目地说 */} + {mode === 'cited' && (graph.reports ?? []).filter((r) => r.hash_ok === false).map((r) => ( +

+ {T.docBad(labelOf(r.node_id) ?? r.node_id ?? '')} +

+ ))} + + {mode !== 'cited' && ( + + )} + + 0} + unsupported={audit.status === 'unsupported'} error={audit.status === 'error' ? audit.error : undefined} + multiReport={reports.length > 1} labelOf={labelOf} mode={mode} /> +
+
+ {wide && ( + + )} +
+ ) +} + +/** 详情栏宽到能放下右栏吗:按容器量,不按屏幕(左边还有运行列表) */ +function useWide(el: HTMLElement | null, min: number): boolean { + const [wide, setWide] = useState(true) + useLayoutEffect(() => { + if (!el || typeof ResizeObserver !== 'function') return + const on = () => setWide(el.getBoundingClientRect().width >= min) + on() + const ro = new ResizeObserver(on) + ro.observe(el) + return () => ro.disconnect() + }, [el, min]) + return wide +} + +type AuditState = + | { status: 'loading'; data?: EvidenceAudit; error?: undefined } + | { status: 'ok'; data: EvidenceAudit; error?: undefined } + | { status: 'unsupported'; data?: undefined; error?: undefined } + | { status: 'error'; data?: undefined; error: unknown } + +/** 审计表的数据。接口不存在(老后端:404 且没有机读码)算「不支持」,退回页面自己拼 */ +function useAudit(runId: string, key: string): AuditState { + const [state, setState] = useState({ status: 'loading' }) + useEffect(() => { + const ctrl = new AbortController() + setState((s) => (s.status === 'ok' ? { status: 'loading', data: s.data } : { status: 'loading' })) + api.evidence.audit(runId, { signal: ctrl.signal }).then( + (data) => setState({ status: 'ok', data }), + (error) => { + if (ctrl.signal.aborted) return + const missing = error instanceof ApiError && (error.status === 404 || error.status === 405) && !error.code + setState(missing ? { status: 'unsupported' } : { status: 'error', error }) + }, + ) + return () => ctrl.abort() + }, [runId, key]) + return state +} + +/** 顶上一行:这次运行按哪种方式展示证据、封存核对的结果、每份报告文档的哈希 */ +function SummaryBar({ mode, seal, message, reports, labelOf }: { + mode?: string + seal?: EvidenceGraph['seal'] & { events?: number; message?: string } + message?: string + reports: NonNullable + labelOf: (id?: string | null) => string | undefined +}) { + const verdict: { status: 'done' | 'failed' | 'idle'; text: string } = !seal + ? { status: 'idle', text: T.sealUnknown } + : !seal.sealed ? { status: 'idle', text: T.sealOpen } + : seal.ok === false ? { status: 'failed', text: T.sealBad } + : { status: 'done', text: T.sealOk(seal.manifest_seq ?? seal.events) } + return ( +
+ {T.modeLabel[mode ?? 'none'] ?? mode} + + + {T.sealTitle}: + {verdict.text} + + {reports.map((r) => { + const name = labelOf(r.node_id) ?? r.node_id ?? '' + const bad = r.hash_ok === false + const gone = r.hash_ok == null + return ( + + {bad ? T.docBad(name) : gone ? T.docGone(name) : T.docOk(name)} + + ) + })} +
+ ) +} + +/** 没有逐段引用的运行:照实说是哪一种,旧契约把回指的数字按位置标在原文上,没有契约的给收起的猜测 */ +function ModeNote({ mode, graph, output, guesses }: { + mode?: string; graph: EvidenceGraph; output: Record | null; guesses: ReturnType +}) { + const legacy = graph.mode === 'legacy_contract' && graph.legacy && !Array.isArray(graph.legacy) ? graph.legacy : null + const field = typeof legacy?.field === 'string' ? legacy.field : null + const text = field && typeof output?.[field] === 'string' ? output[field] as string : null + const matched: any[] = Array.isArray(legacy?.matched) ? legacy!.matched : [] + const unmatched: any[] = Array.isArray(legacy?.unmatched) ? legacy!.unmatched : [] + return ( +
+

+ {T.mode[mode ?? 'none'] ?? T.mode.none} + {graph.note && graph.note !== T.mode[mode ?? 'none'] && {graph.note}} +

+ {legacy && ( +
+
+ {legacy.note ?? T.mode.legacy_contract} + · {T.legacyMatched(matched.length)} + {unmatched.length > 0 && · {T.legacyUnmatched(unmatched.length)}} +
+ {text + ? + :

{T.legacyNoField}

} +
+ )} + {mode === 'legacy_text' && guesses.length > 0 && ( +
+ +
+ )} +
+ ) +} + +/** + * 审计表:按状态分组,一组一个 tbody。「旧运行猜测」那组默认收起(猜的,不能当证据)。 + * 键盘:整张表一个 Tab 位(roving),↑/↓ 逐行走、Home/End 到头尾,回车或点击在右边打开那一段 + */ +function AuditTable({ + runId, rows, filter, onFilter, canOpen, onOpen, loading, fallback, unsupported, error, multiReport, labelOf, mode, +}: { + runId: string + rows: AuditRow[] + filter: AuditFilter + onFilter: (f: AuditFilter) => void + canOpen: (row: AuditRow) => boolean + onOpen: (row: AuditRow) => void + loading: boolean + /** 行是页面自己拼的(审计接口不支持或没取到) */ + fallback: boolean + unsupported: boolean + error?: unknown + multiReport: boolean + labelOf: (id?: string | null) => string | undefined + mode?: string +}) { + const groups = useMemo(() => groupRows(rows, filter), [rows, filter]) + const [collapsed, setCollapsed] = useState>(() => new Set(['candidate'])) + const visible = useMemo(() => groups.flatMap((g) => (collapsed.has(g.group) ? [] : g.rows)), [groups, collapsed]) + const [current, setCurrent] = useState(null) + const cur = current && visible.some((r) => r.key === current) ? current : visible[0]?.key ?? null + const tableRef = useRef(null) + const keysId = useId() + const radio = useRadioGroup(['all', 'problems'], filter, onFilter) + const [exporting, setExporting] = useState(null) + const problems = rows.filter((r) => r.group === 'none' || r.group === 'suspicious').length + + // 焦点当场挪过去(每一行都画着,只是 tabIndex=-1):等下一帧再挪的话,连按几下方向键会落在同一行上 + const focusRow = (key: string) => { + setCurrent(key) + tableRef.current?.querySelector(`[data-audit-focus="${CSS.escape(key)}"]`)?.focus() + } + const onKey = (e: ReactKeyboardEvent, key: string) => { + if (e.key !== 'ArrowDown' && e.key !== 'ArrowUp' && e.key !== 'Home' && e.key !== 'End') return + const i = visible.findIndex((r) => r.key === key) + // ↑/↓ 当作一维走(rovingTarget 按 both 认上下),到头停住、不首尾相接:表格读到底就是底 + const to = e.key === 'ArrowDown' ? Math.min(visible.length - 1, i + 1) + : e.key === 'ArrowUp' ? Math.max(0, i - 1) + : rovingTarget(e, i, visible.length, 'both') + if (to < 0 || to === i) { e.preventDefault(); return } + e.preventDefault() + focusRow(visible[to].key) + } + + const exportAs = async (format: 'json' | 'csv') => { + setExporting(format) + const wanted = filter === 'problems' ? [...PROBLEM_GROUPS] : undefined + try { + const { blob, name } = await api.evidence.auditExport(runId, format, wanted) + save(name, blob) + toast.ok(T.exported(name)) + } catch (e) { + // 老后端没有导出接口:按页面上这张清单导出,并照实说没经过后端的封存核对 + if (e instanceof ApiError && (e.status === 404 || e.status === 405) && !e.code) { + const shown = groupRows(rows, filter).flatMap((g) => g.rows) + const name = `evidence-${runId.slice(0, 8)}.${format}` + save(name, new Blob([format === 'csv' ? `${auditCsv(shown)}` : JSON.stringify({ + run_id: runId, mode, source: 'page', groups: groupRows(rows, filter).map((g) => ({ key: g.group, rows: g.rows })), + }, null, 2)], { type: format === 'csv' ? 'text/csv;charset=utf-8' : 'application/json' })) + toast.info(T.exportLocal) + } else { + toast.error(e, { key: 'evidence:export' }) + } + } finally { + setExporting(null) + } + } + + return ( +
+
+

{T.title}

+ {formatNumber(rows.length)} 处 +
+ {(['all', 'problems'] as const).map((f) => ( + + ))} +
+ + {(['json', 'csv'] as const).map((f) => ( + + ))} +
+ {fallback && ( +

+ {unsupported ? T.fallbackUnsupported : T.fallbackError(error instanceof Error ? error.message : '未知原因')} +

+ )} +

{T.keys}

+ {loading ?
+ : !groups.length ? ( +

{filter === 'problems' ? T.emptyProblems : T.empty}

+ ) : ( +
+ + {/* 定宽:出处是长文字,按内容分宽的话会被挤成一个字宽。所在的句子放在出处下面一行,不另占一列 */} + + + + + {multiReport && } + + + + + + + + {multiReport && } + + + + {groups.map((g) => { + const shut = collapsed.has(g.group) + return ( + + + + + {!shut && g.rows.map((r) => ( + { setCurrent(r.key); onOpen(r) }} + onFocus={() => setCurrent(r.key)} onKey={(e) => onKey(e, r.key)} /> + ))} + + ) + })} +
{T.cols.text}{T.cols.state}{T.cols.source}({T.cols.sentence}){T.cols.report}{T.cols.seal}
+ +
+
+ )} +
+ ) +} + +/** 表格里的字按正文显示的样子写:去掉行内 Markdown 的语法字符(反引号、星号、链接的括号) */ +function plain(text: string): string { + if (!/[`*_~[]/.test(text)) return text + const { hide } = inlineStyles(text) + let out = '' + for (let i = 0; i < text.length; i++) if (!hide[i]) out += text[i] + return out +} + +const groupColor = (g: AuditGroup): string => + g === 'none' || g === 'suspicious' ? 'var(--st-waiting)' : g === 'cited' ? 'var(--st-done)' : EVIDENCE_STATE.candidate.color + +function AuditLine({ row, current, openable, multiReport, labelOf, onOpen, onFocus, onKey }: { + row: AuditRow; current: boolean; openable: boolean; multiReport: boolean + labelOf: (id?: string | null) => string | undefined + onOpen: () => void; onFocus: () => void; onKey: (e: ReactKeyboardEvent) => void +}) { + const meta = EVIDENCE_STATE[row.state] + const text = plain(row.text) + const sentence = plain(row.sentence) + const label = `${text},${meta.label}${row.source ? `:${row.source}` : ''}` + const common = { + tabIndex: current ? 0 : -1, 'data-audit-focus': row.key, onFocus, onKeyDown: onKey, + className: clsx('mono max-w-full truncate rounded-sm text-left outline-none focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-1 focus-visible:outline-[color:var(--accent)]', + row.kind === 'claim' && 'font-sans'), + } + return ( + + + {openable ? ( + + ) : ( + {text} + )} + + + {meta.glyph && {meta.glyph}}{meta.label} + + + {row.source || '—'} + {/* 打不开的违规行(列表序号里的数字、粗体和链接里的可疑名字):说清为什么只在表里 */} + {row.kind === 'violation' && !openable && ( + {T.hidden} + )} + {sentence && sentence !== text && ( + + 「{sentence}」 + + )} + + {multiReport && {labelOf(row.report) ?? row.report}} + + {row.sealed == null ? {T.sealNa} + : row.sealed ? {T.sealIn} + : {T.sealOut}} + + + ) +} + +/** 存成文件:同 AssistantStream 的下载,CSV 由后端带了 BOM(Excel 按 GBK 猜编码,不带的话中文列名乱码) */ +function save(name: string, blob: Blob) { + const url = URL.createObjectURL(blob) + const a = document.createElement('a') + a.href = url + a.download = name + a.click() + setTimeout(() => URL.revokeObjectURL(url), 1000) +} diff --git a/frontend/src/pages/runs/RunDetailView.tsx b/frontend/src/pages/runs/RunDetailView.tsx index 0c3f9426..4dcefee1 100644 --- a/frontend/src/pages/runs/RunDetailView.tsx +++ b/frontend/src/pages/runs/RunDetailView.tsx @@ -24,6 +24,8 @@ import { asView, duplicateNames, graphShape, idTail, isLiveRun, runScope, type DetailView, } from './model' import { ArtifactsPane, useRunArtifacts } from './ArtifactsPane' +import { EvidencePane } from './EvidencePane' +import { EVIDENCE_AUDIT_TEXT } from '../../lib/terms' import { ClassChip, MoreMenu, RunTabs, TierChip, copyText, type MenuItem, type TabItem } from './parts' import { ProvenanceBar, Telemetry, runClocks } from './Telemetry' import { TracePane } from './TracePane' @@ -523,6 +525,7 @@ export function RunDetailView({ runId, onChange, onDeleted }: { const viewTabs: TabItem[] = [ { key: 'stream', label: '时间线', title: '逐步看每个节点做了什么、说了什么' }, { key: 'trace', label: '航迹', title: '按时间摊开:慢在哪、卡在哪、谁和谁同时;拖到任意一刻回放' }, + { key: 'evidence', label: EVIDENCE_AUDIT_TEXT.tab, title: EVIDENCE_AUDIT_TEXT.tabTitle }, { key: 'artifacts', label: '工件', count: artifactCount, unit: '件', tone: 'quiet', title: artifactCount == null ? '这次运行存下的证据和产出' : `${artifactCount} 件:查询和工具调用的原始结果、每个节点每一次的产出`, @@ -651,6 +654,11 @@ export function RunDetailView({ runId, onChange, onDeleted }: { labelOf={labelOf} nodeHref={hrefFor} goneFromCanvas={goneFromCanvas} onRevealStep={revealStep} />
)} + {view === 'evidence' && ( +
+ +
+ )} {view === 'artifacts' && (
(DETAIL_VIEWS as readonly string[]).includes(v ?? '') ? (v as DetailView) : 'stream' @@ -328,6 +331,7 @@ export const ARTIFACT_KIND_LABEL: Record = { node_output: '节点产出', metric_set: '口径卡指标集', report_doc: '报告文档', + schema_snapshot: '表结构快照', } export const artifactKindLabel = (kind: string): string => ARTIFACT_KIND_LABEL[kind] ?? kind @@ -342,6 +346,9 @@ export function artifactDescription(kind: string, meta?: Record | n case 'metric_set': return meta?.caliber ? `口径卡「${meta.caliber}」${meta.version ?? ''}` : '口径卡的指标、算式和输入' case 'report_doc': return '报告:逐段的片段和每个数字的出处' + // 查库时冻结的表结构(工件行的 meta 里有 {source, tables: 表数量}):报告里的表名、字段名按它核对 + case 'schema_snapshot': return meta?.source + ? `表结构:数据源「${meta.source}」${typeof meta.tables === 'number' ? ` · ${meta.tables} 张表` : ''}` : '查库时冻结的表结构' default: return '工件' } } diff --git a/frontend/src/run/AssistantPanel.tsx b/frontend/src/run/AssistantPanel.tsx index 375a5017..1111b4df 100644 --- a/frontend/src/run/AssistantPanel.tsx +++ b/frontend/src/run/AssistantPanel.tsx @@ -18,6 +18,7 @@ import { runClassLabel } from '../lib/terms' import type { RunUsage, ToolChange } from '../types' import { AssistantStream, type StreamTurn } from './AssistantStream' import { Composer } from './Composer' +import { EvidenceHostContext, type EvidenceHost } from './evidenceHost' import { copilotOutcome, decodeCopilot, decodeRun, exitLabels, issueLine, SELF_CHECK_ROUNDS, type CopilotIssue, type CopilotOutcome, } from './decode' @@ -541,6 +542,39 @@ function useExitLabels() { ), [sig]) } +/** + * 画布右栏里的证据面板能做的两件事(evidenceHost): + * - 点开片段时把证据路径交给血缘层(setLineage):产出证据的节点(查询、口径卡、检索)实线描边、 + * 报告虚线描边,两者之间的走线提亮。画布上已经没有的节点不交(对不到任何卡片)。面板关掉、换走时只 + * 收回自己画的那一条——悬停变量画的血缘不去动它 + * - 面板里的节点名点一下:选中并对准画布上的节点,和步骤行的「在画布上看」同一个动作 + */ +function useCanvasEvidenceHost(): EvidenceHost { + const select = useStudio((s) => s.select) + const focusNode = useStudio((s) => s.focusNode) + const drawn = useRef(null) + return useMemo(() => { + const exists = (id: string) => useStudio.getState().nodes.some((n) => n.id === id) + return { + onTrace: (trace) => { + const s = useStudio.getState() + if (trace) { + drawn.current = trace.label + s.setLineage({ var: trace.label, producers: trace.producers.filter(exists), consumers: trace.consumers.filter(exists) }) + } else if (drawn.current && s.lineage?.var === drawn.current) { + drawn.current = null + s.setLineage(null) + } + }, + onNode: (id) => { select(id); focusNode(id) }, + nodeLabel: (id) => { + const node = useStudio.getState().nodes.find((n) => n.id === id) + return node ? node.data.label || id : undefined + }, + } + }, [select, focusNode]) +} + function RunView({ onBack, reveal, onRevealed }: { onBack: () => void; reveal: Reveal | null; onRevealed: () => void }) { @@ -563,6 +597,7 @@ function RunView({ onBack, reveal, onRevealed }: { const [stopping, setStopping] = useState(false) const bodyRef = useRef(null) const exitLabelOf = useExitLabels() + const evidenceHost = useCanvasEvidenceHost() const turns = useMemo(() => { const finished = [...events].reverse().find((e) => e.type === 'run.finished') @@ -734,6 +769,8 @@ function RunView({ onBack, reveal, onRevealed }: {
{raw ? : ( + // 成果里的报告点开片段时:证据路径画到画布上,面板里的节点名点一下就对准那个节点 + ) : null)} /> + )}
diff --git a/frontend/src/run/AssistantStream.tsx b/frontend/src/run/AssistantStream.tsx index e41ec4a8..afe275d3 100644 --- a/frontend/src/run/AssistantStream.tsx +++ b/frontend/src/run/AssistantStream.tsx @@ -23,7 +23,7 @@ import { } from './decode' import { CODE_COLUMN, CopyChip, LEADING_ZERO, Markdown, type MarkSpec } from './Markdown' import { EvidenceField, type EvidenceDocHandle } from './EvidenceDoc' -import { docTally, evidenceFields, issuanceMarks, type EvidenceTally } from '../lib/evidence' +import { EVIDENCE_STATE, docTally, evidenceFields, issuanceMarks, type EvidenceTally } from '../lib/evidence' import type { EvidenceDocData } from '../types' import { useRunClock } from './useRunClock' import type { ReviewResult } from '../types' @@ -2256,6 +2256,12 @@ export function IssuanceBanner({ issuance, runClass, evidence }: { } const hitsOf = (c: any): number | undefined => perCaliber.get([c?.caliber, c?.version].filter(Boolean).join(' @ ')) + // 数字之外另说的两样:可疑名字(按报告文档数,和文档自己的证据条同一个数)、没挂依据的结论句(只在要求挂依据时有) + const counts = evidence?.counts + const suspect = counts?.suspect ?? 0 + const uncitedClaims = Number(issuance?.claims?.uncited_claims ?? 0) || 0 + const numbers = counts ? evidenceTally(counts.cited, counts.total, counts.other) : '' + const numbersOk = !counts?.none && !counts?.other return ( // data-issuance-banner:画布上成果节点的出具印章据此滚过来;tabIndex 让「去看出具」能把焦点带到这里 @@ -2275,14 +2281,24 @@ export function IssuanceBanner({ issuance, runClass, evidence }: {
{meta.hint &&
{meta.hint}
} - {evidence?.counts && (evidence.counts.total > 0 || evidence.counts.other > 0) && ( - // 逐段证据的计数:和报告核对那一行、证据条同一种说法。有无证据的地方才给「定位下一处」 + {counts && (counts.total > 0 || counts.other > 0 || suspect > 0 || uncitedClaims > 0) && ( + // 逐段证据的计数:和报告核对那一行、证据条同一种说法。可疑名字、没挂依据的结论句不是数字, + // 各自另起一句。有无证据的数字、可疑名字时才给「定位下一处」(结论句不是片段,定位不到)
- - {evidenceTally(evidence.counts.cited, evidence.counts.total, evidence.counts.other)} - - {(evidence.counts.none > 0 || evidence.counts.other > 0) && evidence.onNext && ( + {numbers && ( + {numbers} + )} + {suspect > 0 && ( + + {numbers ? '· ' : ''}{EVIDENCE_STATE.suspect.glyph} {EVIDENCE_TEXT.suspectTag} {formatNumber(suspect)} + + )} + {uncitedClaims > 0 && ( + + {numbers || suspect > 0 ? '· ' : ''}{EVIDENCE_TEXT.uncitedClaimsTag} {formatNumber(uncitedClaims)} + + )} + {(counts.none > 0 || counts.other > 0 || suspect > 0) && evidence?.onNext && ( diff --git a/frontend/src/run/EvidenceDoc.tsx b/frontend/src/run/EvidenceDoc.tsx index da9d1dae..dbe6d923 100644 --- a/frontend/src/run/EvidenceDoc.tsx +++ b/frontend/src/run/EvidenceDoc.tsx @@ -8,7 +8,7 @@ import { ListChecks } from 'lucide-react' import clsx from 'clsx' import { rovingTarget } from '../components/ui' import { - EVIDENCE_STATE, docTally, segmentLabel, segmentState, tallySummary, unitText, type EvidenceStateCode, + EVIDENCE_STATE, docTally, segmentKind, segmentLabel, segmentState, tallySummary, unitText, type EvidenceStateCode, type EvidenceTally as Tally, } from '../lib/evidence' import { formatNumber } from '../lib/format' @@ -40,6 +40,8 @@ export interface EvidenceDocHandle { next: (dir?: 1 | -1) => boolean /** 打开违规清单。没有违规返回 false */ list: () => boolean + /** 打开某个片段的面板(记录页的审计表点一行)。focus 为 true 时焦点也挪到正文里那一段;没有这个片段返回 false */ + open: (segId: string, opts?: { focus?: boolean }) => boolean } /** 和 AssistantStream 的长文本折叠同一个量级:先显示这么多字,后面的折起来 */ @@ -131,10 +133,16 @@ function verticalTarget(model: Model, id: string, down: boolean): string | null return model.unitFirst[(u + (down ? 1 : -1) + n) % n] ?? null } -/** 片段的外观:线型走 data 属性(CSS 里只管线型),颜色写成两个变量交给 index.css */ +/** 片段的外观:线型走 data 属性(CSS 里只管线型),颜色写成变量交给 index.css(引文前的引号用 --ev-glyph) */ function segStyle(state: EvidenceStateCode): CSSProperties { const meta = EVIDENCE_STATE[state] - return { '--ev-line': meta.decoration, '--ev-soft': meta.soft } as CSSProperties + return { '--ev-line': meta.decoration, '--ev-soft': meta.soft, '--ev-glyph': meta.color } as CSSProperties +} + +/** 句末小标签:哪几种异常态在句末挂字形加文字(去掉颜色也读得出)。可疑名字和无证据分开挂 */ +const TAG_TEXT: Partial> = { + none: EVIDENCE_STATE_LABEL.none, + suspect: EVIDENCE_TEXT.suspectTag, } export const EvidenceDoc = forwardRef void /** 自己显示「N/N 数字有出处」那一条。上面已经有出具横幅(它会写这一行)时不用 */ tally?: boolean -}>(function EvidenceDoc({ doc, artifact, runId, dense = false, label, panel = 'auto', tally = false }, ref) { +}>(function EvidenceDoc({ doc, artifact, runId, dense = false, label, panel = 'auto', dock, onView, tally = false }, ref) { const blocks = useMemo(() => doc.blocks ?? [], [doc]) const model = useMemo(() => buildModel(blocks), [blocks]) const counts = useMemo(() => docTally(doc), [doc]) @@ -246,7 +261,18 @@ export const EvidenceDoc = forwardRef { onView?.(open) }, [open, onView]) + useImperativeHandle(ref, () => ({ + open: (id, opts) => { + if (!model.seg.has(id)) return false + if (opts?.focus) focusSeg(id, { flash: true }) + else setCurrent(id) + openSeg(id) + return true + }, next: (dir = 1) => { const target = step(model.alerts, view?.kind === 'seg' ? view.id : current, dir) if (target) { @@ -300,7 +326,7 @@ export const EvidenceDoc = forwardRef

{summary}

{announce}
- {tally && (counts.total > 0 || counts.other > 0) && ( + {tally && (counts.total > 0 || counts.other > 0 || !!counts.suspect) && ( { const target = step(model.alerts, current, 1) if (target) { focusSeg(target, { flash: true }); openSeg(target) } else openViolations() @@ -381,7 +407,8 @@ export const EvidenceDoc = forwardRef setExpanded(false)}>收起
)} - {mode !== 'inline' && panelNode && createPortal(panelNode, document.body)} + {mode === 'dock' && panelNode && dock && createPortal(panelNode, dock)} + {mode !== 'inline' && mode !== 'dock' && panelNode && createPortal(panelNode, document.body)} ) }) @@ -394,6 +421,7 @@ function flashOnce(el: HTMLElement) { /** 面板放哪。auto:窄栏(dense)里栏内展开,宽屏侧边,窄屏底部抽屉;屏宽变了跟着变 */ function usePanelMode(panel: 'auto' | PanelMode, dense: boolean): PanelMode { + // dock 由页面自己定宽窄(记录页窄的时候换成 drawer 再传进来),这里原样用 const query = '(min-width: 900px)' const [wide, setWide] = useState(() => typeof matchMedia !== 'function' || matchMedia(query).matches) useEffect(() => { @@ -411,16 +439,26 @@ function usePanelMode(panel: 'auto' | PanelMode, dense: boolean): PanelMode { /** 证据条:没有出具横幅时由文档自己说「7/12 数字有出处 · 无证据 5 · 另有 1 处引用解析不了」 */ export function EvidenceTally({ counts, onNext, onList }: { - counts: Pick + counts: Pick onNext?: () => void onList?: () => void }) { - const clean = !counts.none && !counts.other + const suspect = counts.suspect ?? 0 + const clean = !counts.none && !counts.other && !suspect + const numbers = evidenceTally(counts.cited, counts.total, counts.other) return (
- - {evidenceTally(counts.cited, counts.total, counts.other)} - + {numbers && ( + + {numbers} + + )} + {/* 可疑名字另起一句:它不是数字,混进「无证据 M」就算不平了 */} + {suspect > 0 && ( + + {numbers ? '· ' : ''}{EVIDENCE_STATE.suspect.glyph} {EVIDENCE_TEXT.suspectTag} {formatNumber(suspect)} + + )} {!clean && onNext && ( +
+ {open && ( +

+ {note} +

+ )} + {guesses.map((g, i) => ( + + ))} +
+ + ) +} + +function GuessField({ guess, open, dense, title }: { + guess: EvidenceGuess; open: boolean; dense: boolean; title?: string +}) { + const segs = useMemo(() => (guess.segments ?? []).filter((s) => s.kind === 'number'), [guess]) + const guessed = segs.filter((s) => s.state === 'candidate' && s.candidates?.length) + const missed = segs.filter((s) => !(s.state === 'candidate' && s.candidates?.length)) + // 候选按位置标在原文上(码点偏移,Markdown 自己换算、核对切出来的字);收着的时候不标,就是原来的答案 + const marks = useMemo(() => (open ? guessed.map((s) => ({ + token: s.text, tone: 'candidate' as const, start: s.span?.[0], end: s.span?.[1], + title: (s.candidates ?? []).map(candidateText).join(';'), + })) : undefined), [open, guessed]) + const text = guess.markdown ?? '' + return ( +
+ {title &&
{title}
} + + {open && ( +
+
{EVIDENCE_TEXT.guessCandidates}
+ {guessed.length ? ( +
    + {guessed.map((s) => ( +
  1. + + {EVIDENCE_STATE.candidate.glyph}{s.text} + + + {(s.candidates ?? []).map(candidateText).join(';')} + +
  2. + ))} +
+ ) :

{EVIDENCE_TEXT.guessNone}

} + {missed.length > 0 && guessed.length > 0 && ( +

+ {EVIDENCE_TEXT.guessNone}:{missed.map((s) => s.text).join('、')}({formatNumber(missed.length)} 个) +

+ )} +
+ )} +
+ ) +} diff --git a/frontend/src/run/EvidencePanel.tsx b/frontend/src/run/EvidencePanel.tsx index d11d1afd..109b3b28 100644 --- a/frontend/src/run/EvidencePanel.tsx +++ b/frontend/src/run/EvidencePanel.tsx @@ -7,18 +7,21 @@ import { ArrowLeft, CornerDownRight, ListChecks, ShieldCheck, X } from 'lucide-r import clsx from 'clsx' import { StatusBadge, isComposing } from '../components/ui' import { - EVIDENCE_STATE, caliberSourceText, caliberUpgradeText, docForeign, evidenceValue as valueText, graphDoc, inputSource, - integrityFailures, locatable, locatorText, notableInput, queryOf, queryWindow, reasonOf, sealVerdict, segmentState, - sourceOf, type SealStatus, type SourceTone, + EVIDENCE_KIND_STYLE, EVIDENCE_STATE, caliberSourceText, caliberUpgradeText, closestNames, docForeign, entitySources, + evidenceTrace, evidenceValue as valueText, graphDoc, inputSource, integrityFailures, locatable, locatorText, notableInput, + queryOf, queryWindow, quoteWhere, quoteWindow, reasonOf, sealVerdict, segName, segmentKind, segmentState, sourceOf, + type SealStatus, type SourceTone, } from '../lib/evidence' -import { formatNumber, NONE, shortId } from '../lib/format' +import { formatDateTime, formatNumber, NONE, shortId } from '../lib/format' import { EVIDENCE_TEXT } from '../lib/terms' import { useCatalog } from '../store/catalog' import { segmentKey, useEvidence } from '../store/evidence' import type { - EvidenceDocData, EvidenceInput, EvidenceSeal, EvidenceSegment, EvidenceStep, EvidenceUnit, EvidenceViolation, + EvidenceDocData, EvidenceInput, EvidenceQuoteSource, EvidenceSeal, EvidenceSegment, EvidenceStep, EvidenceUnit, + EvidenceViolation, } from '../types' import { ArtifactViewer, ResultTable } from './AssistantStream' +import { useEvidenceHost } from './evidenceHost' import { CopyChip } from './Markdown' /** @@ -39,11 +42,16 @@ import { CopyChip } from './Markdown' * 事后插行),接口是从封存范围内的事件找的文档。两边不是同一份时,接口的封存状态和 * 证据链说的是另一份报告——不能挂到正文上画成绿的,要醒目地说正文不是封存的那一份。 * - * 三种摆法:side 从右侧弹出(宽屏),drawer 从底部抽出(窄屏),inline 在 360px 的 - * 画布右栏里直接栏内展开。都不是模态的:开着面板照样能在正文里走。 + * 三期多了两种步骤:表或字段(出现在哪几次查询里、表结构快照的同步时间、字段类型;可疑的 + * 名字给出最接近的已知名字),逐字引文(原文所在的文档和片段,引文在原文里的位置高亮)。 + * 画布右栏里点开片段时,把证据路径交给画布(evidenceHost),节点名点一下就对准那个节点。 + * + * 四种摆法:side 从右侧弹出(宽屏),drawer 从底部抽出(窄屏),inline 在 360px 的 + * 画布右栏里直接栏内展开,dock 放进页面给的一块常驻位置(记录页的「证据」页签)。 + * 都不是模态的:开着面板照样能在正文里走。 */ -export type PanelMode = 'side' | 'drawer' | 'inline' +export type PanelMode = 'side' | 'drawer' | 'inline' | 'dock' export type PanelView = { kind: 'seg'; id: string } | { kind: 'violations' } export function EvidencePanel({ id, mode, doc, artifact, runId, view, onClose, onLocate, onViolations }: { @@ -72,12 +80,18 @@ export function EvidencePanel({ id, mode, doc, artifact, runId, view, onClose, o onClose() } - const title = view.kind === 'violations' ? EVIDENCE_TEXT.violations : found?.seg.text ?? EVIDENCE_TEXT.panelTitle + const title = view.kind === 'violations' ? EVIDENCE_TEXT.violations + : found ? segName(found.seg) : EVIDENCE_TEXT.panelTitle + // 有出处的实体、引文:徽标写得更具体(「有出处 · 逐字引文」),颜色和字形照同一套 + const kind = found && state === 'deterministic' ? segmentKind(found.seg) : null + const badge = kind ? { ...meta!, label: EVIDENCE_KIND_STYLE[kind].label, glyph: EVIDENCE_KIND_STYLE[kind].glyph } : meta const frame = mode === 'side' ? 'ev-panel-side fixed bottom-0 right-0 top-0 z-40 flex flex-col border-l bg-panel shadow-elev-3' : mode === 'drawer' ? 'ev-panel-rise fixed bottom-0 left-0 right-0 z-40 flex flex-col rounded-t-lg border-t bg-panel shadow-elev-3' - : 'ev-panel-rise my-2 flex flex-col rounded-lg border bg-panel' + : mode === 'dock' + ? 'flex h-full min-h-0 flex-col bg-panel' + : 'ev-panel-rise my-2 flex flex-col rounded-lg border bg-panel' return (
{EVIDENCE_TEXT.back} )} - {meta && ( - - {meta.glyph && {meta.glyph}}{meta.label} + {badge && ( + + {badge.glyph && {badge.glyph}}{badge.label} )}

@@ -110,7 +124,8 @@ export function EvidencePanel({ id, mode, doc, artifact, runId, view, onClose, o )} -
+
{view.kind === 'violations' ? : found @@ -180,14 +195,37 @@ function SegmentBody({ panelId, doc, artifact, runId, seg, unit, onViolations }: const state = segmentState(seg) const cite = seg.cite const entry = cite?.alias ? doc.catalog?.[cite.alias] : undefined - const chain: EvidenceStep[] = Array.isArray(detail?.chain) ? detail.chain.filter((s) => s && typeof s === 'object') : [] + const chain: EvidenceStep[] = useMemo(() => (Array.isArray(detail?.chain) + ? detail.chain.filter((s) => s && typeof s === 'object') : []), [detail]) const metricStep = chain.find((s) => s.step === 'metric') const inputStep = chain.find((s) => s.step === 'run_input') + const entityStep = chain.find((s) => s.step === 'entity') + const quoteStep = chain.find((s) => s.step === 'quote') const queries = chain.filter((s) => s.step === 'query') const isMetric = cite?.kind === 'metric' || entry?.kind === 'metric' || !!metricStep const isInput = cite?.kind === 'input' || entry?.kind === 'input' || !!inputStep + // 三期:有出处的表名字段名、引文(原话对不上的引文也画这一节:说清原文在哪、对不上) + const isEntity = !isMetric && !isInput && seg.kind === 'entity' && state === 'deterministic' + const isQuote = !isMetric && !isInput && seg.kind === 'quote' // 报告直接引用的查询单元格([[v:Q3.r5.amount]]、整表里的一格):证据链就是那一次查询 - const isCell = !isMetric && !isInput && (cite?.kind === 'cell' || entry?.kind === 'query' || !!queries.length) + const isCell = !isMetric && !isInput && !isEntity && !isQuote + && (cite?.kind === 'cell' || entry?.kind === 'query' || !!queries.length) + const bad = state === 'none' || state === 'suspect' || state === 'unverified' + const closest = closestNames(detail?.closest ?? entityStep?.closest) + + // 画布右栏:点开片段时把证据路径交给画布(产出证据的节点实线、报告虚线),链取回来后再补全 + const host = useEvidenceHost() + const onTrace = host.onTrace + const trace = useMemo(() => evidenceTrace(seg, doc, chain), [seg, doc, chain]) + const traceKey = `${trace.producers.join(',')}|${trace.consumers.join(',')}` + useEffect(() => { + if (!onTrace) return + const text = segName(seg) + onTrace({ label: `证据 ${text.length > 16 ? `${text.slice(0, 15)}…` : text}`, ...trace }) + return () => onTrace(null) + // trace 按内容比(traceKey):同一条路径不重画 + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [onTrace, seg.id, traceKey]) const inputs: EvidenceInput[] = metricStep?.inputs?.length ? metricStep.inputs : chain.filter((s) => s.step === 'input').map((s) => s as EvidenceInput) @@ -218,6 +256,7 @@ function SegmentBody({ panelId, doc, artifact, runId, seg, unit, onViolations }: docOnly: state !== 'deterministic', viaGraph: slot?.status === 'error', }) + const pending = !!runId && !docIssue && (!slot || slot.status === 'loading') const chainFailed = !runId ? EVIDENCE_TEXT.noRun : docIssue ? EVIDENCE_TEXT.chainForeign : slot?.status === 'error' ? EVIDENCE_TEXT.chainMissing : '' @@ -236,20 +275,49 @@ function SegmentBody({ panelId, doc, artifact, runId, seg, unit, onViolations }: - {state === 'none' && ( - -

{reason}

+ {bad && ( + +

{reason}

{/* 违规的原话多半是「引用 … 解析不了:<原因>」,和上面那句重复的不再列;裸数字那条说的是怎么改,留着 */} {violations.filter((v) => v.message && !(reason && v.message.includes(reason))).map((v, i) => (

{v.message}

))} {seg.ref &&

[[{seg.ref}]]

} + {(state === 'suspect' || state === 'unverified') && runId && !docIssue && ( + // 可疑的名字:给最接近的已知名字,多半是写错了一个字(接口按封存的台账重建目录找的) +
+
{EVIDENCE_TEXT.closest}
+ {closest.length ? ( +
    + {closest.map((n) =>
  • {n}
  • )} +
+ ) : ( +

{pending ? EVIDENCE_TEXT.entityPending : EVIDENCE_TEXT.closestNone}

+ )} + {entityStep?.checked && ( +

+ {EVIDENCE_TEXT.checked(entityStep.checked.schemas ?? 0, entityStep.checked.queries ?? 0)} +

+ )} +
+ )}
)} + {isEntity && ( + + )} + + {isQuote && ( + + )} + {isMetric && ( )} @@ -283,10 +351,21 @@ function SegmentBody({ panelId, doc, artifact, runId, seg, unit, onViolations }: ))} - {state === 'deterministic' && !isMetric && !isInput && !isCell && ( + {state === 'deterministic' && !isMetric && !isInput && !isCell && !isEntity && !isQuote && (

{sourceOf(seg, doc) || detail?.note}

)} + {host.onNode && (trace.producers.length > 0 || trace.consumers.length > 0) && ( + // 画布右栏:这段证据经过的节点,点一下在画布上选中并对准 + +
+ {trace.producers.map((n) => )} + {trace.consumers.length > 0 && trace.producers.length > 0 && →} + {trace.consumers.map((n) => )} +
+
+ )} + @@ -309,6 +388,167 @@ function SegmentBody({ panelId, doc, artifact, runId, seg, unit, onViolations }: type InputLink = { index: number; alias: string; go: () => void } | undefined +/** + * 面板里提到的节点。画布右栏里是按钮:点一下在画布上选中并对准它(画布上已经没有这个节点 + * 时不给按钮,点了会落空);别处照旧写节点 id + */ +function NodeChip({ id, prefix }: { id?: string | null; prefix?: string }) { + const host = useEvidenceHost() + if (!id) return null + const label = host.nodeLabel?.(id) + if (host.onNode && label !== undefined) { + return ( + + ) + } + return {prefix}{id} +} + +/** + * 表或字段:是什么、字段类型、出现在哪几次查询里、从哪来的(表结构快照、查询 SQL、查询结果列)、 + * 表结构快照什么时候同步的。证据接口没取到时照文档目录里记着的来历写,缺的照实说 + */ +function EntityPart({ step, entry, cite, seal, pending, failed, catalog }: { + step?: EvidenceStep + entry?: Record + cite?: EvidenceSegment['cite'] + seal?: EvidenceSeal + pending: boolean + failed: string + catalog?: EvidenceDocData['catalog'] +}) { + const kind = step?.kind ?? entry?.kind ?? cite?.kind + const table = step?.table ?? entry?.table ?? entry?.locator?.table ?? cite?.locator?.table + const column = step?.column ?? entry?.locator?.column ?? cite?.locator?.column + const name = kind === 'table' ? (step?.name ?? entry?.name ?? table ?? cite?.ref) + : table && column ? `${table}.${column}` : (step?.name ?? entry?.name ?? column ?? cite?.ref) + const queries: string[] = (Array.isArray(step?.queries) ? step.queries : Array.isArray(entry?.queries) ? entry.queries : []) + .map(String).filter(Boolean) + const sources = entitySources(step?.sources ?? entry?.sources) + const truncated = sources.some((s) => s.truncated) || step?.snapshot_truncated === true || (step as any)?.truncated === true + const owners: string[] = Array.isArray(step?.tables) ? step.tables : Array.isArray(entry?.tables) ? entry.tables : [] + const nodes = [...new Set(queries.map((q) => catalog?.[q]?.node_id).filter((n): n is string => typeof n === 'string'))] + return ( + +
+ {kind === 'table' ? EVIDENCE_TEXT.table : EVIDENCE_TEXT.column} + {name || NONE} + {step?.type && ( + {EVIDENCE_TEXT.columnType} {step.type} + )} + {kind === 'table' && typeof (step as any)?.columns === 'number' && ( + {EVIDENCE_TEXT.tableColumns((step as any).columns, !!step?.is_view)} + )} +
+ {step?.comment &&

{step.comment}

} + {owners.length > 1 &&

{EVIDENCE_TEXT.ownerTables(owners)}

} + {step?.types && !step.type && Object.keys(step.types).length > 0 && ( +

{EVIDENCE_TEXT.columnTypes(step.types)}

+ )} +

+ {queries.length ? EVIDENCE_TEXT.entityQueries(queries) : EVIDENCE_TEXT.entityNoQuery} +

+ {nodes.length > 0 && ( +
+ {nodes.map((n) => )} +
+ )} + {sources.length > 0 && ( +
    + {sources.map((src) =>
  • · {src.text}
  • )} +
+ )} + {step?.synced_at && ( +

{EVIDENCE_TEXT.syncedAt(formatDateTime(step.synced_at))}

+ )} + {truncated && ( +

{EVIDENCE_TEXT.snapshotPartial}

+ )} + + {!step && ( +

+ {pending ? EVIDENCE_TEXT.entityPending : failed && failed !== EVIDENCE_TEXT.chainMissing ? failed : EVIDENCE_TEXT.entityMissing} +

+ )} +
+ ) +} + +/** + * 逐字引文:原文所在的文档和片段,引文在原文里的位置高亮(前后带一截上下文)。检索快照哈希对不上时 + * 不摆原文——那一段不能当证据。原话对不上的引文也画这一节,只写目录里记着的是哪次检索 + */ +function QuotePart({ step, cite, quote, resolved, seal, pending, failed, node, label }: { + step?: EvidenceStep + cite?: EvidenceSegment['cite'] + quote: string + resolved: boolean + seal?: EvidenceSeal + pending: boolean + failed: string + node?: string + /** 目录里这次检索的说明:「知识库「运营手册」 · 2 条」 */ + label?: string +}) { + const source = (step?.source && typeof step.source === 'object' ? step.source as EvidenceQuoteSource : undefined) + ?? cite?.source + const where = quoteWhere(source) + const bad = integrityFailures(step, seal) + // 原文:接口给在 content 里(text 是引文本身);content 为 null 是快照不在封存范围里、取不回来——不拿引文冒充原文 + const original = step && 'content' in step ? step.content ?? undefined : step?.text + const loc = cite?.locator + const mismatch = step?.match_ok === false + const win = original && !bad.includes('hash') && !mismatch + ? quoteWindow(original, step?.match ?? (loc ? { start: loc.start, end: loc.end } : null), step?.quote ?? quote) + : null + const collection = step?.collection + return ( + +
+ {step?.alias ?? cite?.alias ?? NONE} + {where ? {where} : label && {label}} + {collection && where && {EVIDENCE_TEXT.quoteCollection(collection)}} + {source?.document && ( + + {shortId(source.document, 12)} + + )} + +
+ {(bad.length > 0 || mismatch) && ( + (k === 'hash' ? 'quote-hash' : k)), ...(mismatch ? ['quote-match'] : [])].join(' ')} + items={[...bad.map((k) => (k === 'hash' ? EVIDENCE_TEXT.quoteHash : EVIDENCE_TEXT.integrity[k])), + ...(mismatch ? [EVIDENCE_TEXT.quoteBad] : [])]} /> + )} + {win && ( + <> +
+ {win.cutBefore && '…'}{win.before} + + {win.hit} + + {win.after}{win.cutAfter && '…'} +
+

{EVIDENCE_TEXT.quoteHit}

+ + )} + {resolved && !win && !bad.length && !mismatch && ( + // 接口说了为什么没有原文(不在封存范围里、取不回来)就照原话 +

+ {pending ? EVIDENCE_TEXT.quotePending : step?.note + || (failed && failed !== EVIDENCE_TEXT.chainMissing ? failed : EVIDENCE_TEXT.quoteMissing)} +

+ )} + {!resolved &&

{EVIDENCE_TEXT.quoteMiss}

} +
+ ) +} + /** 口径卡钉住的上游工作流 id:方案写在 caliber_from,接口给在 source 上 */ function pinnedWorkflow(step?: EvidenceStep): string | undefined { const from = step?.caliber_from ?? (step?.source && typeof step.source === 'object' ? step.source : undefined) @@ -323,12 +563,14 @@ function Sentence({ unit, active }: { unit: EvidenceUnit; active: string }) { const state = segmentState(s) const on = s.id === active const number = s.kind === 'number' || s.kind === 'value' + // 反引号里的表名字段名照行内代码写,不露反引号 + const code = s.kind === 'entity' && !!s.code return ( - {s.text} + {code ? segName(s) : s.text} ) })} @@ -354,6 +596,7 @@ function MetricPart({ step, entry, inputs, runId, seal, pending, failed, resolve /** 钉住的上游工作流在目录里的名字(接口没给名字时用) */ workflowName?: string }) { + const host = useEvidenceHost() const name = step?.name ?? entry?.name ?? step?.metric ?? entry?.locator?.metric ?? NONE const value = step?.value !== undefined ? step.value : entry?.value const rendered = step?.rendered ?? entry?.rendered @@ -382,6 +625,7 @@ function MetricPart({ step, entry, inputs, runId, seal, pending, failed, resolve ? {from} : (caliber || version) && 口径卡「{caliber || NONE}」{version}} {artifact && 工件 {shortId(artifact)}} + {host.onNode && }
)} {upgrade && ( @@ -508,7 +752,6 @@ function SourcesPart({ inputs, linkOf }: { inputs: EvidenceInput[]; linkOf: (inp // 那个节点内部的编号,和面板里的「查询 Qn」不是一回事,不拿来写 const alias = link?.alias || inp.query || (typeof inp.cell === 'string' ? inp.cell.split('.')[0] : '') const where = [ - inp.node_id && `节点 ${inp.node_id}`, inp.field && (inp.via === 'agent_field' ? `字段 ${inp.field}` : inp.field), [alias, locatorText(inp.locator)].filter(Boolean).join(' · '), ].filter(Boolean).join(' · ') @@ -525,7 +768,12 @@ function SourcesPart({ inputs, linkOf }: { inputs: EvidenceInput[]; linkOf: (inp )}
- {where &&
{where}
} + {(inp.node_id || where) && ( +
+ {inp.node_id && } + {inp.node_id && where && ' · '}{where} +
+ )} {said.text &&

{said.text}

} {said.reason &&

{said.reason}

} @@ -544,6 +792,7 @@ function SourcesPart({ inputs, linkOf }: { inputs: EvidenceInput[]; linkOf: (inp * 这时候这些行不能当证据,比「有出处」重要得多 */ function QueryPart({ id, step, seal, masked }: { id: string; step: EvidenceStep; seal?: EvidenceSeal; masked: string[] }) { + const host = useEvidenceHost() const win = useMemo(() => queryWindow(step), [step]) const [open, setOpen] = useState(false) const scroller = useRef(null) @@ -580,6 +829,7 @@ function QueryPart({ id, step, seal, masked }: { id: string; step: EvidenceStep; {EVIDENCE_TEXT.query(alias)} {step.tool && {step.tool}} + {host.onNode && } {step.sql && step.sql ?? ''} />}

{step.sql && ( @@ -671,6 +921,8 @@ function SealLine({ status, label }: { status: SealStatus; label: string }) { const CODE_LABEL: Record = { uncited_number: '裸数字', unresolved_ref: '引用解析不了', + unknown_entity: '可能是编造的名字', + unverified_entity: '核对不了的名字', } /** 违规清单:报告撰写节点核对出来的全部问题,画得出线的能定位,画不出的说清在哪 */ diff --git a/frontend/src/run/Markdown.tsx b/frontend/src/run/Markdown.tsx index 190d07a1..a1d9694c 100644 --- a/frontend/src/run/Markdown.tsx +++ b/frontend/src/run/Markdown.tsx @@ -1,10 +1,10 @@ import { - createContext, Fragment, useContext, useMemo, useState, type HTMLAttributes, type ReactNode, + createContext, Fragment, useContext, useMemo, useState, type CSSProperties, type HTMLAttributes, type ReactNode, } from 'react' import { Check, Copy } from 'lucide-react' import clsx from 'clsx' import { toast } from '../components/ui' -import { codePointIndex } from '../lib/evidence' +import { EVIDENCE_STATE, codePointIndex } from '../lib/evidence' /** * 模型输出的 Markdown 渲染。 @@ -60,8 +60,11 @@ export function Markdown({ text, dense, marks }: { export interface MarkSpec { token: string title: string - /** ambiguous:回指上了,但同一个值对得上好几个指标,出处不唯一 */ - tone: 'warn' | 'ok' | 'ambiguous' + /** + * ambiguous:回指上了,但同一个值对得上好几个指标,出处不唯一; + * candidate:没有契约的旧运行按数值猜的候选(最淡的点状线,读屏在字后面听到「猜测的来源:…」) + */ + tone: 'warn' | 'ok' | 'ambiguous' | 'candidate' /** 在原文里的码点偏移 [start, end) */ start?: number end?: number @@ -108,6 +111,16 @@ const shiftAt = (at: At | undefined, i: number): At | undefined => at == null ? undefined : typeof at === 'number' ? at + i : at.slice(i) function markSpan(spec: MarkSpec, text: string, key: string): ReactNode { + if (spec.tone === 'candidate') { + // 猜测:外观取证据状态表里「旧运行候选」那一档,绝不用确定性的实线和绿 + const meta = EVIDENCE_STATE.candidate + return ( + + {text}({meta.label}:{spec.title}) + + ) + } const color = spec.tone === 'ok' ? 'var(--st-done)' : 'var(--st-waiting)' return ( void + /** 面板里的节点名点一下:选中并对准画布上的这个节点 */ + onNode?: (nodeId: string) => void + /** 节点 id → 画布上的名字;画布上没有这个节点时返回 undefined(不给按钮,点了会落空) */ + nodeLabel?: (nodeId: string) => string | undefined +} + +export const EvidenceHostContext = createContext({}) + +export const useEvidenceHost = (): EvidenceHost => useContext(EvidenceHostContext) diff --git a/frontend/src/store/evidence.ts b/frontend/src/store/evidence.ts index 634a8cb1..3af05336 100644 --- a/frontend/src/store/evidence.ts +++ b/frontend/src/store/evidence.ts @@ -1,6 +1,6 @@ import { create } from 'zustand' -import { api } from '../api/client' -import type { EvidenceDocData, EvidenceGraph, EvidenceSegmentDetail } from '../types' +import { ApiError, api } from '../api/client' +import type { EvidenceDocData, EvidenceGraph, EvidenceSegmentDetail, EvidenceStats } from '../types' /** * 可点击证据的缓存:报告文档、证据图、点开过的片段。 @@ -20,6 +20,12 @@ export interface Slot { error?: unknown } +/** 一张图最近一次运行里每个报告节点的核对统计(画布上报告节点卡的章用) */ +export interface LatestReports { + runId: string | null + reports: Record +} + interface EvidenceState { /** * 哪一份报告的面板开着。同一屏上可能有好几份报告(问数据页的几轮、改写前的原件), @@ -30,9 +36,19 @@ interface EvidenceState { docs: Record> graphs: Record> segments: Record> + /** 按工作流:最近一次运行的报告统计。画布没挂着运行时,报告节点卡的章从这里取 */ + latest: Record> loadDoc: (artifact: string) => Promise loadGraph: (runId: string) => Promise + /** 证据图重新取一次:运行还在往前走(记录页接着流)时,封存状态和报告都还会变 */ + reloadGraph: (runId: string) => Promise loadSegment: (runId: string, segmentId: string, report?: string) => Promise + loadLatest: (workflowId: string) => Promise + /** + * 这张图的「最近一次运行」作废:画布上挂着这张图的一次运行(刚跑的、回放的)时,之前取的那份 + * 不一定还是最近的。清掉以后,运行摘下来(「清除」、换一张图再回来)时 loadLatest 重新取 + */ + dropLatest: (workflowId: string) => void } /** @@ -57,18 +73,24 @@ export class NotADocError extends Error { } export const useEvidence = create((set, get) => { - /** 取一次、记进 table[key];正在取或已经取到的不再发 */ - async function fill( - table: K, key: string, fetcher: () => Promise>, + /** + * 取一次、记进 table[key];正在取或已经取到的不再发。取的途中这一格被清掉或换了(dropLatest 之后 + * 又起了一次),回来的结果作废:不能让早先发出的请求把后来那份盖掉 + */ + async function fill( + table: K, key: string, fetcher: () => Promise>, force = false, ) { const cur = get()[table][key] - if (cur && cur.status !== 'error') return - set((s) => ({ [table]: { ...s[table], [key]: { status: 'loading' } } }) as Partial) + if (cur && (cur.status === 'loading' || (cur.status === 'ok' && !force))) return + // 重取时留着上一份:界面不闪回「正在取」 + const pending = (force && cur?.data ? { ...cur, status: 'loading' } : { status: 'loading' }) as Slot + set((s) => ({ [table]: { ...s[table], [key]: pending } }) as Partial) + const settle = (slot: Slot) => set((s) => (s[table][key] === pending + ? { [table]: { ...s[table], [key]: slot } } as Partial : s)) try { - const data = await fetcher() - set((s) => ({ [table]: { ...s[table], [key]: { status: 'ok', data } } }) as Partial) + settle({ status: 'ok', data: await fetcher() }) } catch (error) { - set((s) => ({ [table]: { ...s[table], [key]: { status: 'error', error } } }) as Partial) + settle({ status: 'error', error }) } } @@ -78,6 +100,7 @@ export const useEvidence = create((set, get) => { docs: {}, graphs: {}, segments: {}, + latest: {}, loadDoc: (artifact) => fill('docs', artifact, async () => { const res = await api.artifact(artifact) const doc = asDoc(res?.content) @@ -85,7 +108,31 @@ export const useEvidence = create((set, get) => { return doc }), loadGraph: (runId) => fill('graphs', runId, () => api.evidence.graph(runId)), + reloadGraph: (runId) => fill('graphs', runId, () => api.evidence.graph(runId), true), loadSegment: (runId, segmentId, report) => fill('segments', segmentKey(runId, segmentId, report), () => api.evidence.segment(runId, segmentId, { report })), + // 最近一次运行 → 它封存范围内的报告核对(report.checked)。审计接口的 reports 带着结论句策略(claims, + // 章上「无证据」要不要算没挂依据的结论句靠它);只要 reports,所以只取最小的「可疑实体」那一组行。 + // 老后端没有审计接口时退回证据图(没有 claims,没挂依据的结论句不计入)。没有运行、没有报告都是空表,不画章 + loadLatest: (workflowId) => fill('latest', workflowId, async () => { + const [run] = await api.runs.list({ workflow_id: workflowId, limit: 1 }) + if (!run) return { runId: null, reports: {} } + const list = await api.evidence.audit(run.id, { groups: ['suspicious'] }).then((a) => a.reports ?? [], async (e) => { + if (!(e instanceof ApiError && (e.status === 404 || e.status === 405) && !e.code)) throw e + await get().loadGraph(run.id) + return get().graphs[run.id]?.data?.reports ?? [] + }) + const reports: LatestReports['reports'] = {} + for (const r of list) if (r?.node_id) reports[r.node_id] = { stats: r.stats, claims: r.claims } + return { runId: run.id, reports } + }), + dropLatest: (workflowId) => { + if (!(workflowId in get().latest)) return + set((s) => { + const latest = { ...s.latest } + delete latest[workflowId] + return { latest } + }) + }, } }) diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 68f3575f..dff87883 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -325,7 +325,9 @@ export interface PublishFix { node_id?: string | null label: string preview?: { field?: string | null; before?: unknown; after?: unknown } | null - options?: { value: unknown; label: string; hint?: string | null }[] + /** handoff:选了这一项就是「交给 Copilot」(G4 的「它在做计算」)。只认这个标记,不认值—— + * 别的选项拿节点 id 当值,节点 id 恰好叫 copilot 的报告撰写节点照样是普通候选 */ + options?: { value: unknown; label: string; hint?: string | null; handoff?: boolean }[] multiple?: boolean /** 建议值:界面上标「建议」,仍然要人选 */ default?: unknown @@ -599,7 +601,9 @@ export interface EvidenceCitation { ref?: string /** 目录里的别名,如 m:gmv、i:week、Q1 */ alias?: string - locator?: { metric?: string; field?: string; row?: number; column?: string; table?: string } + /** 引文:hit 是检索快照里第几条命中,start / end 是引文在那条原文里的码点位置 */ + locator?: { metric?: string; field?: string; row?: number; column?: string; table?: string + hit?: number; start?: number; end?: number } eid?: string /** metric / input / cell / table / column / quote / query */ kind?: string @@ -610,6 +614,21 @@ export interface EvidenceCitation { conv?: string value?: unknown rendered?: string + /** 引文的原文出处:检索快照、文档、片段 */ + source?: EvidenceQuoteSource + /** [[t:]] / [[c:]] 写了一个哪里都找不到的名字:可能是编造的 */ + unknown?: boolean + /** 表结构快照不全,找不到的名字只能说核对不了 */ + unverified?: boolean +} + +/** 引文出自哪:检索快照工件、知识库里的文档和片段 */ +export interface EvidenceQuoteSource { + artifact?: string + document?: string + chunk?: string + title?: string + ordinal?: number } /** 可点的最小片段 */ @@ -625,8 +644,53 @@ export interface EvidenceSegment { ref?: string cite?: EvidenceCitation strong?: boolean - /** uncited_number / unresolved_ref */ + /** uncited_number / unresolved_ref / unknown_entity(可能是编造的名字)/ unverified_entity(核对不了) */ issue?: string + /** 实体写在反引号里:text 自带两个反引号,按行内代码画 */ + code?: boolean + /** 系统自动链接的名字,不是写作者写的标记(不算句子的依据) */ + auto?: boolean + /** 反引号里自动发现的可疑名字(没有 ref 时名字在这里) */ + name?: string + /** 旧运行按数值猜的候选(state 为 candidate) */ + candidates?: EvidenceCandidate[] +} + +/** + * 旧运行(没有契约)按数值猜的一个候选:查询快照的一格,或口径卡的一个指标。 + * 只做展示,不是证据 + */ +export interface EvidenceCandidate { + kind?: 'cell' | 'metric' | string + /** Q1.r0.gmv / m:gmv */ + ref?: string + alias?: string + artifact?: string + locator?: EvidenceLocator & { metric?: string } + eid?: string + value?: unknown + rendered?: string + /** 指标名 */ + name?: string + caliber?: string + version?: string + node_id?: string + tool?: string + /** 和答案里那个数差多少 */ + diff?: number +} + +/** 旧运行的猜测:guess_sources 的输出(agentlab.guess/1) */ +export interface EvidenceGuess { + schema?: string + mode?: string + /** 「猜测的来源,不能当证据:…」 */ + note?: string + markdown?: string + segments?: EvidenceSegment[] + stats?: { numbers?: number; guessed?: number; unguessed?: number; candidates?: number } + /** 猜的是成果里哪个字段 */ + field?: string } /** 句子、列表项或表格单元格 */ @@ -681,6 +745,11 @@ export interface EvidenceStats { see?: number uncited_claims?: number violations?: number + /** 三期:有出处的实体、引文片段数;可疑实体、核对不了的名字数。老文档没有这几个键,缺省当 0 */ + entities?: number + quotes?: number + unknown_entities?: number + unverified_entities?: number } /** report_doc 工件的内容 */ @@ -724,12 +793,23 @@ export interface EvidenceGraph { schema?: string /** cited / legacy_contract / legacy_text / none */ mode?: string + /** 没有报告时为什么(none)、旧版出具怎么认的(legacy_*):一句人话 */ + note?: string + /** + * legacy_contract:{note, tier, matched, unmatched, field},按旧契约的位置标出的数字; + * legacy_text:guess_sources 的输出(EvidenceGuess),可能按字段分成几份 + */ + legacy?: Record | Record[] | null + /** legacy_text 另一种放法:猜测单独放在这里 */ + guess?: EvidenceGuess | EvidenceGuess[] | Record | null seal?: EvidenceSeal reports?: { node_id?: string; doc_artifact?: string; doc_sealed?: boolean; fields?: string[]; stats?: EvidenceStats /** 文档取回时哈希复验:false 对不上(或不是这次运行这个节点写的),null 文件不在了 */ - hash_ok?: boolean | null }[] + hash_ok?: boolean | null + /** 写这份报告时的结论句策略(report.checked 里记的;老后端、升级前的运行没有) */ + claims?: string | null }[] evidence?: { alias?: string; eid?: string; kind?: string; label?: string; node_id?: string; artifact?: string - sealed?: boolean; cited_by?: string[] }[] + sealed?: boolean; cited_by?: string[]; report?: string }[] edges?: { from: string; to: string; rel: string }[] } @@ -864,6 +944,98 @@ export interface EvidenceStep extends Omit { note?: string /** 数据源改名或删掉了、按查询当时记下的遮罩处理时的那句说明。行照样有 */ mask_note?: string + // ---- entity:表或字段(三期)---- + /** entity:table / column */ + kind?: string + table?: string + column?: string + /** 字段类型(表结构快照里记的) */ + type?: string + /** 同名字段在好几张表里时,是哪几张 */ + tables?: string[] + /** 这个名字的来历:表结构快照、查询 SQL、查询结果列 */ + sources?: EvidenceEntitySource[] + /** 出现在哪几次查询里(报告目录的全局编号) */ + queries?: string[] + /** 表结构快照什么时候同步的 */ + synced_at?: string + /** 可疑实体:最接近的已知名字 */ + closest?: (string | { alias?: string; name?: string; table?: string; kind?: string })[] + /** 可疑实体:查过几份表结构快照、几次查询 */ + checked?: { schemas?: number; queries?: number } + /** 表结构快照只存了一部分(库里表太多) */ + snapshot_truncated?: boolean + /** 好几张表都有的同名字段:各表的类型 */ + types?: Record + /** 表:几个字段、是不是视图;表或字段的注释 */ + is_view?: boolean + comment?: string + nullable?: boolean + primary_key?: boolean + // ---- quote:逐字引文(三期)---- + /** quote:引文所在那条命中的原文(也可能放在 content 里) */ + text?: string + /** quote:引文所在那条命中的全文;快照不在封存范围里、取不回来时是 null */ + content?: string | null + quote?: string + match?: { hit?: number; start?: number; end?: number } + /** quote:按记下的位置把原文切出来,和引文逐字比对的结果 */ + match_ok?: boolean | null + /** quote:哪个知识库、检索的是什么 */ + collection?: string + query?: string | null +} + +/** 实体的一处来历 */ +export interface EvidenceEntitySource { + /** schema(表结构快照)/ sql(查询 SQL 用到的表)/ result(查询结果列) */ + kind?: string + artifact?: string + alias?: string + source?: string + truncated?: boolean +} + +/** GET /runs/{id}/evidence/audit 的一行(形状照 api/evidence.py;前端经 lib/evidence 的 auditFromApi 读) */ +export interface EvidenceAuditRow { + /** cited / none / suspicious / candidate */ + group?: string + report?: string | null + field?: string | null + segment?: string | null + unit?: string | null + /** number / value / entity / quote / claim / violation */ + kind?: string + text?: string + state?: string + issue?: string | null + ref?: string | null + alias?: string | null + evidence_kind?: string | null + evidence?: string | null + eid?: string | null + artifact?: string | null + node_id?: string | null + sealed?: boolean | null + span?: [number, number] | null + sentence?: string + note?: string | null + candidates?: EvidenceCandidate[] +} + +/** GET /runs/{id}/evidence/audit:记录页的审计表 */ +export interface EvidenceAudit { + run_id?: string + schema?: string + mode?: string + seal?: EvidenceSeal & { events?: number; message?: string } + reports?: { node_id?: string; doc_artifact?: string; hash_ok?: boolean | null; doc_sealed?: boolean + fields?: string[]; claims?: string | null; stats?: EvidenceStats }[] + groups?: { key?: string; label?: string; count?: number; rows?: EvidenceAuditRow[] }[] + counts?: Record + total?: number + note?: string + legacy_note?: string } /** GET /runs/{id}/evidence/segments/{sid}:点开一个片段 */ @@ -880,4 +1052,6 @@ export interface EvidenceSegmentDetail { seal?: EvidenceSeal /** 查询步骤里被遮罩的列(数据源 options.mask_columns)。遮罩只减少暴露,不是安全边界 */ redacted?: { columns?: string[] } + /** 可疑实体:最接近的已知名字(最多 3 个),给「是不是想写…」 */ + closest?: (string | { alias?: string; name?: string; table?: string; kind?: string })[] } diff --git a/scripts/check-all.mjs b/scripts/check-all.mjs index 91666e6c..2ee1002f 100644 --- a/scripts/check-all.mjs +++ b/scripts/check-all.mjs @@ -52,9 +52,10 @@ const ORDER = [ ] // 最近一次全量里各项的耗时(秒)。只决定排队顺序:长的先开跑,短的填空,结果不受影响。 // 新加的检查不在表里就按 60 秒算 +// 2026-09-29 三期收尾那一次四道并行的实测(秒):只用来排先后,长的先跑 const ESTIMATE = { - 'run-states': 220, 'canvas-fx': 181, stream: 162, studio: 125, chat: 124, manage: 115, runs: 91, - shell: 84, 'ui-kit': 69, ui: 61, evidence: 56, publish: 31, guards: 10, 'canvas-layout': 4, tokens: 1, trace: 1, decode: 1, + 'run-states': 299, 'canvas-fx': 264, stream: 176, chat: 164, studio: 153, manage: 142, runs: 105, + shell: 85, 'ui-kit': 74, evidence: 67, ui: 67, publish: 54, guards: 17, 'canvas-layout': 8, tokens: 1, trace: 1, decode: 1, } // 量时间最多、以前并行最容易挂的:同一时刻最多跑其中一项 const EXCLUSIVE = new Set(['run-states', 'canvas-fx']) diff --git a/scripts/check-evidence.mjs b/scripts/check-evidence.mjs index 7a29780d..a2e78cab 100644 --- a/scripts/check-evidence.mjs +++ b/scripts/check-evidence.mjs @@ -27,6 +27,11 @@ const fx = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-d // 第二期:报告直接引用查询单元格、整表,口径卡的输入核对到 agent 字段和查询快照。文档同样是 // compose_doc 真跑出来的(查询快照走 loader),片段接口的答复按方案第 5 节和 NOTES-A2 的形状拼 const fxq = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-query.json`, 'utf8')) +// 第三期:表名字段名(反引号里的、标记的、自动链接的)、可疑实体、核对不了的名字、逐字引文, +// 以及没有契约的旧运行按数值猜的候选。文档和猜测都是后端真跑出来的(compose_doc / guess_sources) +const fxe = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-entity.json`, 'utf8')) +/** 夹具里某段文字是哪个片段:检查按文字认片段,重新生成夹具时编号变了也不用改 */ +const segOf = (doc, text) => doc.blocks.flatMap((b) => b.units.flatMap((u) => u.segments)).find((s) => s.text === text)?.id let failed = 0 const check = (name, cond, detail = '') => { @@ -55,10 +60,10 @@ async function routed(page) { const url = new URL(req.url()) if (req.method() !== 'GET') return r.abort() const seg = url.pathname.match(/\/api\/runs\/([^/]+)\/evidence\/segments\/([^/]+)$/) - // 按运行认夹具:第二期那份(查询链)和第一期那份各有自己的片段 - const set = url.pathname.includes(`/runs/${fxq.run_id}/`) ? fxq : fx + // 按运行认夹具:第一期、第二期(查询链)、第三期(实体、引文)各有自己的片段 + const set = url.pathname.includes(`/runs/${fxq.run_id}/`) ? fxq : url.pathname.includes(`/runs/${fxe.run_id}/`) ? fxe : fx if (seg) { - hits.push(`${set === fxq ? 'q:' : ''}${decodeURIComponent(seg[2])}`) + hits.push(`${set === fxq ? 'q:' : set === fxe ? 'e:' : ''}${decodeURIComponent(seg[2])}`) const body = set.segments[decodeURIComponent(seg[2])] return body ? r.fulfill({ json: body }) : r.fulfill({ status: 404, json: { detail: '报告里没有这个片段', code: 'evidence_segment_not_found' } }) @@ -1125,6 +1130,48 @@ await section('studio', '报告撰写节点:画布认得、检查器能配(R check('「指标来自」只列上游的口径卡(下游那张不列)', opts.length === 1 && opts[0].includes('周报口径'), opts.join(' | ')) const onViolation = await p.locator('[data-field="on_violation"] option').allInnerTexts() check('违规时的默认写明按运行类别', onViolation[0]?.includes('探索运行照常产出'), onViolation.join(' | ')) + // 三期:结论句策略 claims(off / require_citation,judge 是四期的,不出现在选项里)和表名字段名核对 entities + // 表名字段名核对收在「高级选项」里(很少要改),先展开 + await p.locator('button[aria-expanded]', { hasText: '高级选项' }).click().catch(() => {}) + await p.waitForTimeout(100) + const all = await p.evaluate(() => [...document.querySelectorAll('[data-field]')].map((el) => el.getAttribute('data-field'))) + check('检查器有结论句策略、表名字段名核对(高级选项里)两项', fields.includes('claims') && all.includes('entities') + && !fields.includes('entities'), all.join(',')) + const claimVals = await p.locator('[data-field="claims"] option').evaluateAll((els) => els.map((e) => e.value)) + check('结论句策略只有 off / require_citation,没有 judge', claimVals.join(',') === 'off,require_citation', claimVals.join(',')) + const claimsLabel = await p.locator('[data-field="claims"] label').first().innerText().catch(() => '') + check('结论句策略有中文标签(发布前修复的预览也用它)', /结论句/.test(claimsLabel) && !/claims/.test(claimsLabel), claimsLabel) + const claimsHelp = await p.locator('[data-field="claims"]').innerText() + check('……说明写明「结论句裁判在后续版本支持」、受管级别要显式选', claimsHelp.includes('结论句裁判在后续版本支持') + && claimsHelp.includes('受管'), claimsHelp.replace(/\s+/g, ' ').slice(0, 160)) + check('没写 claims 时显示默认的 off', await p.locator('[data-field="claims"] select').inputValue() === 'off' + || (await p.locator('[data-field="claims"] select').evaluate((el) => el.options[el.selectedIndex]?.value)) === 'off') + const entityVals = await p.locator('[data-field="entities"] option').evaluateAll((els) => els.map((e) => e.value)) + check('表名字段名核对:link / off', entityVals.join(',') === 'link,off', entityVals.join(',')) + await p.locator('[data-field="claims"] select').selectOption('require_citation') + const cfgOf = () => p.evaluate(() => window.__studio.getState().nodes.find((n) => n.id === 'write')?.data.config) + check('选「要求挂依据」写进 config.claims', (await cfgOf())?.claims === 'require_citation') + // 手写或别处写进来的 judge:不能悄悄显示成 off,要照实写出来并说后续版本支持 + await p.evaluate(() => { + const st = window.__studio.getState() + const n = st.nodes.find((x) => x.id === 'write') + st.updateNode('write', { config: { ...n.data.config, claims: 'judge' } }) + }) + await p.waitForTimeout(150) + const shown = await p.locator('[data-field="claims"] select').evaluate((el) => el.options[el.selectedIndex]?.textContent ?? '') + check('config 里写着 judge:下拉照实显示 judge 并说后续版本支持,不装成 off', shown.includes('judge') && shown.includes('后续版本'), shown) + const fixLabel = await p.evaluate(async () => { + // 按页面自己加载时的地址 import:改过的模块地址带 ?t=,直接写 /src/… 会拿到另一份实例 + const url = performance.getEntriesByType('resource').map((e) => e.name) + .find((n) => { try { return new URL(n).pathname === '/src/canvas/issues.ts' } catch { return false } }) + const m = await import(url ?? '/src/canvas/issues.ts') + return [m.fixFieldLabel('claims', 'report'), m.fixFieldLabel('contract.claims.on_uncited', 'output'), + m.fixValueText('ignore', 'contract.claims.on_uncited'), m.fixValueText('degrade', 'contract.claims.on_uncited')] + }).catch((e) => [String(e)]) + check('发布前修复预览里 claims 这个字段写中文名', fixLabel[0] === claimsLabel.trim() && !/claims/.test(fixLabel[0]), fixLabel[0]) + check('契约 claims 的 on_uncited(ignore 改成 degrade 那条修复):字段和值都写中文', + !/claims|on_uncited/.test(fixLabel[1] ?? 'claims') && fixLabel[2] === '只标出来,不算缺口' && fixLabel[3] === '计入缺口、出具降档', + JSON.stringify(fixLabel.slice(1))) if (SHOTS) { for (const theme of ['dark', 'light']) { await p.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) @@ -1309,6 +1356,236 @@ await section('config', '配置项:agent 的 cite_fields、code 的证据角 await ctx.close() }) +// ------------------------------------------------------------------ 第三期:实体、引文、旧运行的猜测 + +const E = { + num: segOf(fxe.doc, '45,678.5'), orders: segOf(fxe.doc, 'orders'), week: segOf(fxe.doc, 'orders.week'), + refunds: segOf(fxe.doc, '`refunds`'), column: segOf(fxe.doc, '`refunds.refund_amount`'), + coupon: segOf(fxe.doc, '`orders.coupon_code`'), discount: segOf(fxe.doc, 'orders.discount'), + promo: segOf(fxe.doc, '`promo_events`'), customers: segOf(fxe.doc, 'customers'), bare: segOf(fxe.doc, '30'), + quote: segOf(fxe.doc, '退款金额以财务确认日为准,未确认的退款不计入当周'), miss: segOf(fxe.doc, '周报一律按自然周统计'), +} +const ent = '#evidence-entity' + +await section('entity', '三期:表名字段名、可疑实体、逐字引文——四通道分得开', async () => { + check('夹具里认得出每一种片段', Object.values(E).every(Boolean), JSON.stringify(E)) + const look = await page.evaluate(([sel, E]) => Object.fromEntries(Object.entries(E).map(([k, id]) => { + const el = document.querySelector(`${sel} [data-seg="${id}"]`) + if (!el) return [k, null] + const cs = getComputedStyle(el) + return [k, { style: cs.textDecorationStyle, color: cs.textDecorationColor, state: el.getAttribute('data-ev-state'), + kind: el.getAttribute('data-ev-kind'), label: el.getAttribute('aria-label') ?? '', text: el.innerText, + code: !!el.querySelector('code'), before: getComputedStyle(el, '::before').content }] + })), [ent, E]) + check('有出处的实体和数字同一套「有出处」线型:实线、同一个颜色', look.orders?.state === 'deterministic' + && look.orders.style === 'solid' && look.orders.color === look.num?.color, JSON.stringify([look.orders, look.num?.color])) + check('可疑实体用「无证据」的线型(点状),和有出处的实体线型不同', look.coupon?.state === 'suspect' + && look.coupon.style === 'dotted' && look.coupon.style !== look.orders?.style, JSON.stringify(look.coupon)) + check('标记写的可疑实体([[c:…]] 编造的字段)同样标成可疑', look.discount?.state === 'suspect' && look.discount.style === 'dotted', + JSON.stringify(look.discount)) + check('可疑实体的 aria-label 写「可能是编造的名字」,名字不带反引号', look.coupon?.label.startsWith('orders.coupon_code,') + && look.coupon.label.includes('可能是编造的名字'), look.coupon?.label) + check('核对不了的名字:点状线、自己的说法,不说「编造」', look.promo?.state === 'unverified' && look.promo.style === 'dotted' + && look.promo.label.includes('核对不了') && !look.promo.label.includes('编造'), look.promo?.label) + check('核对不了和可疑实体颜色不同(它只是标注,不醒目)', look.promo?.color !== look.coupon?.color, `${look.promo?.color} / ${look.coupon?.color}`) + check('反引号里的实体按行内代码画,不露反引号', look.refunds?.code && !look.refunds.text.includes('`') + && look.column?.code, JSON.stringify([look.refunds, look.column?.text])) + check('自动链接的名字(正文里的已知表名)也是有出处的实体', look.customers?.state === 'deterministic' && look.customers.kind === 'entity') + check('aria-label:实体说清是表还是字段、出现在哪几次查询', look.orders?.label === 'orders,有出处:表 orders · 出现在查询 Q1、Q2', + look.orders?.label) + check('引文片段用引用样式:前面挂引号、实线', look.quote?.kind === 'quote' && look.quote.before.includes('“') + && look.quote.style === 'solid' && look.quote.state === 'deterministic', JSON.stringify(look.quote)) + check('原话对不上的引文照样是引用样式,但线型换成点状(无证据)', look.miss?.kind === 'quote' && look.miss.before.includes('“') + && look.miss.style === 'dotted' && look.miss.state === 'none', JSON.stringify(look.miss)) + check('引文的 aria-label 写出出处', look.quote?.label.includes('有出处') && look.quote.label.includes('运营手册 · 退款'), look.quote?.label) + const tags = await page.locator(`${ent} .ev-tag`).allInnerTexts() + check('句末小标签:可疑名字挂「?!可疑名字」,和「?无证据」分开', tags.includes('?!可疑名字') && tags.includes('?无证据'), tags.join('|')) + const tally = await page.locator(`${ent} [data-evidence-tally]`).innerText().catch(() => '') + check('证据条另说可疑名字的个数', tally.includes('可疑名字 2'), tally) + const summary = await page.locator(`${ent} [data-evidence-summary]`).innerText().catch(() => '') + check('读屏摘要写出可疑实体、核对不了的个数', summary.includes('2 个可能是编造的名字') && summary.includes('1 个名字核对不了'), summary) + + // n / N:可疑实体进跳转(异常态),核对不了的不进(只是标注) + await page.locator(`${ent} [data-seg="${E.num}"]`).focus() + const jumps = [] + for (let i = 0; i < 5; i++) { + await page.keyboard.press('n') + jumps.push(await active(page)) + } + check('n 依次跳到裸数字、可疑实体、原话对不上的引文,跳过核对不了的名字', + jumps.join(',') === [E.bare, E.coupon, E.discount, E.miss, E.bare].join(','), jumps.join(',')) +}) + +await section('e-panel', '三期:面板里的实体步骤、可疑实体的提示、引文步骤', async () => { + const p = panel(page) + await page.locator(`${ent} [data-seg="${E.week}"]`).click() + await page.waitForSelector('[data-evidence-panel] [data-ev-entity-type]', { timeout: 4000 }).catch(() => {}) + check('面板标题是名字本身', (await p.locator('h3').innerText().catch(() => '')) === 'orders.week') + check('实体步骤写字段类型', (await p.locator('[data-ev-entity-type]').innerText().catch(() => '')).includes('VARCHAR(8)')) + check('实体步骤写表结构快照的同步时间', (await p.locator('[data-ev-entity-synced]').innerText().catch(() => '')).includes('2026-09-20'), + await p.locator('[data-ev-entity-synced]').innerText().catch(() => '')) + check('快照不全时照实说只存了一部分', (await p.innerText()).includes('只存了一部分')) + check('实体的来历:表结构快照', (await p.locator('[data-ev-entity-source]').allInnerTexts()).some((t) => t.includes('表结构快照'))) + await page.locator(`${ent} [data-seg="${E.orders}"]`).click() + await page.waitForTimeout(250) + const queries = await p.locator('[data-ev-entity-queries]').innerText().catch(() => '') + check('表:出现在哪几次查询里', queries.includes('Q1') && queries.includes('Q2'), queries) + const srcs = await p.locator('[data-ev-entity-source]').allInnerTexts() + check('表的来历逐条列:表结构快照、查询 SQL', srcs.some((t) => t.includes('查询 Q2') && t.includes('SQL')), srcs.join(' | ')) + check('实体步骤取自证据接口(按 runId:segId 取了一次)', harness.hits.includes(`e:${E.orders}`), harness.hits.filter((h) => h.startsWith('e:')).join(',')) + + await page.locator(`${ent} [data-seg="${E.discount}"]`).click() + // 提示要等片段接口回来(先显示「正在取」):等到名字出来再读 + await page.waitForSelector('[data-evidence-panel] [data-ev-closest-name]', { timeout: 4000 }).catch(() => {}) + const reason = await p.locator('[data-ev-reason]').innerText().catch(() => '') + check('可疑实体说清为什么:本次运行里哪里都没有这个名字', reason.includes('可能是编造的名字'), reason.slice(0, 80)) + const closest = await p.locator('[data-ev-closest-name]').allInnerTexts() + check('可疑实体给出最接近的已知名字(最多 3 个,不带 c: 前缀)', closest.length >= 1 && closest.length <= 3 + && closest.includes('orders.amount') && closest.every((n) => !n.startsWith('c:')), closest.join(',')) + check('可疑实体的面板徽标:?! 可能是编造的名字', (await p.locator('[data-ev-badge]').innerText().catch(() => '')).includes('可能是编造的名字')) + + await page.locator(`${ent} [data-seg="${E.promo}"]`).click() + await page.waitForTimeout(250) + const promo = await p.innerText() + check('核对不了的名字:说快照不全、核对不了,不说编造', promo.includes('核对不了') && !promo.includes('编造'), promo.slice(0, 120)) + + await page.locator(`${ent} [data-seg="${E.quote}"]`).click() + await page.waitForSelector('[data-evidence-panel] [data-ev-quote-hit]', { timeout: 4000 }).catch(() => {}) + const src = await p.locator('[data-ev-quote-source]').innerText().catch(() => '') + check('引文步骤:原文所在的文档和片段', src.includes('运营手册 · 退款') && src.includes('第 3 段'), src) + const hit = await p.locator('[data-ev-quote-hit]').innerText().catch(() => '') + check('引文在原文里的位置高亮出来,就是那句原话', hit === '退款金额以财务确认日为准,未确认的退款不计入当周', hit) + const ctx = await p.locator('[data-ev-quote]').innerText().catch(() => '') + check('高亮前后带着原文的上下文', ctx.includes('第三章 退款口径') && ctx.includes('部分退款'), ctx.slice(0, 80)) + await page.locator(`${ent} [data-seg="${E.miss}"]`).click() + await page.waitForTimeout(250) + const miss = await p.innerText() + check('原话对不上的引文:说清找不到这句原话', miss.includes('找不到这句原话'), miss.slice(0, 120)) + await page.keyboard.press('Escape') + if (SHOTS) { + for (const theme of ['dark', 'light']) { + await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + for (const [name, id] of [['entity', E.orders], ['suspect', E.discount], ['quote', E.quote]]) { + await page.locator(`${ent} [data-seg="${id}"]`).click() + await page.waitForTimeout(350) + await page.screenshot({ path: `${SHOTS}/evidence3-${name}-${theme}.png` }) + } + await page.keyboard.press('Escape') + } + await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark')) + } +}) + +await section('e-narrow', '三期在 360px:栏内展开不横向滚动;减少动效', async () => { + const narrow = '#evidence-entity-narrow' + const overflow = () => page.evaluate((sel) => { + const el = document.querySelector(sel) + return { sw: el.scrollWidth, cw: el.clientWidth } + }, narrow) + for (const id of [E.quote, E.discount, E.orders]) { + await page.locator(`${narrow} [data-seg="${id}"]`).click() + await page.waitForSelector(`${narrow} [data-evidence-panel="inline"]`, { timeout: 4000 }).catch(() => {}) + await page.waitForTimeout(250) + const o = await overflow() + check(`360px 栏内面板(${id})不出横向滚动`, o.sw <= o.cw, JSON.stringify(o)) + } + await page.keyboard.press('Escape') + const r = await open('/ui-harness.html?evidence=1', { reduced: true, w: 360, h: 780 }) + const scroll = () => r.page.evaluate(() => ({ sw: document.documentElement.scrollWidth, vw: innerWidth })) + await r.page.locator(`${ent} [data-seg="${E.quote}"]`).click() + await r.page.waitForSelector('[data-evidence-panel] [data-ev-quote]', { timeout: 4000 }).catch(() => {}) + const s1 = await scroll() + check('360px 屏幕:引文面板(底部抽屉)整页不横向滚动', s1.sw <= s1.vw, JSON.stringify(s1)) + const anim = await r.page.evaluate(() => document.querySelector('[data-evidence-panel]')?.getAnimations({ subtree: true }) + .filter((a) => !(a instanceof CSSTransition && Number(a.effect?.getTiming().duration) <= 0.01)).length ?? -1) + check('减少动效:引文面板没有动画', anim === 0, String(anim)) + await r.page.keyboard.press('Escape') + await r.page.locator('#evidence-guess [data-ev-guess-toggle]').click().catch(() => {}) + await r.page.waitForTimeout(150) + const s2 = await scroll() + check('360px 屏幕:展开猜测之后整页不横向滚动', s2.sw <= s2.vw + && await r.page.locator('#evidence-guess [data-ev-guess]').getAttribute('data-open').catch(() => null) === 'true', JSON.stringify(s2)) + const guessAnim = await r.page.evaluate(() => document.querySelector('#evidence-guess [data-ev-guess]')?.getAnimations({ subtree: true }) + .filter((a) => !(a instanceof CSSTransition && Number(a.effect?.getTiming().duration) <= 0.01)).length ?? -1) + check('减少动效:展开猜测没有动画', guessAnim === 0, String(guessAnim)) + check('没有运行时报错(三期 360px)', r.errors.length === 0, r.errors.join(' | ')) + await r.ctx.close() +}) + +await section('banner', '出具横幅那一行:数字都有出处时,可疑名字、没挂依据的结论句也写出来', async () => { + const at = (key) => page.locator(`#issuance-line [data-issuance-case="${key}"]`) + const line = await at('suspect').locator('[data-evidence-line]').innerText().catch(() => '') + check('数字那半句照旧:3 个数字都有出处', line.includes('3 个数字都有出处'), line) + check('可疑名字另起一句:?! 可疑名字 2', await at('suspect').locator('[data-evidence-line-suspect]').count() === 1 + && line.includes('?! 可疑名字 2'), line) + check('没挂依据的结论句另起一句:没挂依据的结论句 1', await at('suspect').locator('[data-evidence-line-claims]').count() === 1 + && line.includes('没挂依据的结论句 1'), line) + const next = at('suspect').locator('[data-evidence-next]') + check('数字都有出处、只有可疑名字时也给「定位下一处」', await next.count() === 1) + await page.evaluate(() => { window.__issuanceNext = 0 }) + await next.click() + check('点了它就去找下一处', await page.evaluate(() => window.__issuanceNext) === 1) + const clean = await at('clean').locator('[data-evidence-line]').innerText().catch(() => '') + check('都干净的那一份:只有数字那半句,不给「定位下一处」', clean.includes('3 个数字都有出处') + && await at('clean').locator('[data-evidence-line-suspect], [data-evidence-line-claims], [data-evidence-next]').count() === 0, clean) + const names = await at('names').locator('[data-evidence-line]').innerText().catch(() => '') + check('一个数字都没有、只有可疑名字:这一行照样出来,开头不带「·」,也给「定位下一处」', + names.trim().startsWith('?! 可疑名字 1') && await at('names').locator('[data-evidence-next]').count() === 1, names) +}) + +await section('guess', '旧运行按数值猜的候选:默认折叠,展开写明「猜测」,淡点状线', async () => { + const g = '#evidence-guess' + const box = page.locator(`${g} [data-ev-guess]`) + check('默认折叠', await box.getAttribute('data-open').catch(() => null) === 'false') + check('折叠时看不到候选(不在页面上,不只是藏起来)', await page.locator(`${g} [data-ev-state="candidate"]`).count() === 0) + const head = await page.locator(`${g} [data-ev-guess-toggle]`).innerText().catch(() => '') + check('折叠的标题就写明「猜测」', head.includes('猜测'), head) + const toggle = page.locator(`${g} [data-ev-guess-toggle]`) + check('展开按钮是 button,带 aria-expanded=false', await toggle.evaluate((el) => el.tagName).catch(() => '') === 'BUTTON' + && await toggle.getAttribute('aria-expanded').catch(() => null) === 'false') + const answer = await page.locator(`${g}`).innerText().catch(() => '') + check('折叠时答案本身照常显示', answer.includes('本周销售额 45,678.5 元'), answer.slice(0, 60)) + await toggle.click() + await page.waitForTimeout(150) + check('展开后 aria-expanded=true', await toggle.getAttribute('aria-expanded').catch(() => null) === 'true') + const note = await page.locator(`${g} [data-ev-guess-note]`).innerText().catch(() => '') + check('展开后写明「猜测的来源,不能当证据」', note.includes('猜测的来源,不能当证据'), note) + const cands = await page.evaluate((sel) => [...document.querySelectorAll(`${sel} [data-ev-state="candidate"]`)].map((el) => { + const cs = getComputedStyle(el) + // 候选是 Markdown 里的行内标记(保留旧答案的版式),读屏靠标记里那句 sr-only 的「猜测的来源:…」 + return { style: cs.textDecorationStyle, color: cs.textDecorationColor, line: el.getAttribute('data-ev-line'), + label: el.textContent ?? '', title: el.getAttribute('title') ?? '', text: el.firstChild?.textContent ?? '' } + }), g) + check('有候选的数字画成候选(4 个)', cands.length === 4, cands.map((c) => c.text).join(',')) + check('候选用淡点状线,不用确定性的实线', cands.length > 0 && cands.every((c) => c.style === 'dotted' && c.line === 'dotted'), + JSON.stringify(cands[0])) + const detColor = await page.evaluate((sel) => { + const el = document.querySelector(`${sel} [data-ev-state="deterministic"]`) + return el ? getComputedStyle(el).textDecorationColor : '' + }, ent) + check('候选的线和「有出处」不是一个颜色', cands.length > 0 && cands.every((c) => c.color !== detColor), `${cands[0]?.color} / ${detColor}`) + check('候选给读屏写「猜测的来源」和候选出处,悬停也看得到', cands[0]?.label.includes('猜测的来源') && cands[0].label.includes('Q1') + && cands[0].title.includes('Q1'), cands[0]?.label) + const list = await page.locator(`${g} [data-ev-candidates]`).innerText().catch(() => '') + check('候选清单:每个数字可能来自哪一格', list.includes('查询 Q1') && list.includes('gmv'), list.slice(0, 120)) + const plain = await page.evaluate((sel) => [...document.querySelectorAll(`${sel} [data-ev-guess-body] [data-ev-state]`)] + .map((el) => el.firstChild?.textContent ?? ''), g) + check('没猜到候选的数字不画线(40%、50,000)', !plain.includes('40%') && !plain.includes('50,000'), plain.join(',')) + await toggle.click() + await page.waitForTimeout(100) + check('再点一次收起', await box.getAttribute('data-open').catch(() => null) === 'false') + if (SHOTS) { + await toggle.click() + for (const theme of ['dark', 'light']) { + await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + await page.waitForTimeout(200) + await page.locator(g).screenshot({ path: `${SHOTS}/evidence3-guess-${theme}.png` }) + } + await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark')) + await toggle.click() + } +}) + if (SHOTS) { for (const theme of ['dark', 'light']) { await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) diff --git a/scripts/check-publish.mjs b/scripts/check-publish.mjs index c0ce2162..40df988b 100644 --- a/scripts/check-publish.mjs +++ b/scripts/check-publish.mjs @@ -100,6 +100,23 @@ const DUP = { edges: chain('in', 'ask1', 'ask2', 'caliber', 'done'), } +/** + * 两个 choice:沙箱代码喂口径卡(G4),候选里「交给 Copilot」那一项带 handoff 标记;成果字段接哪个报告 + * (G1),唯一的候选是一个 id 恰好叫 copilot 的报告撰写节点——它只是普通候选,不是「交给 Copilot」 + */ +const HANDOFF = { + nodes: [ + node('in', 'input', 0, '入口', { fields: [{ name: 'week', required: true }] }), + node('calc', 'code', 240, '计算', { code: 'print(1)', assign_to: 'calc' }), + node('caliber', 'metrics', 480, '周报口径', { caliber: '周报口径', caliber_version: 'v1', + metrics: [{ id: 'ratio', name: '比例', expression: 'vars.calc.ratio' }] }), + node('copilot', 'report', 720, '报告撰写', { instructions: '写周报', numbers: 'strict', on_violation: 'fail', claims: 'require_citation' }), + node('done', 'output', 960, '成果', { fields: [{ name: 'report', value: '{{ vars.story }}' }], + contract: { report_from: 'copilot', metrics_from: ['caliber'], required: ['ratio'], strict: true } }), + ], + edges: chain('in', 'calc', 'caliber', 'copilot', 'done'), +} + const wf = (id, name, graph, extra = {}) => ({ id, name, description: '检查脚本伪造的工作流', graph, tags: [], version: 3, is_template: false, status: 'draft', published_version: null, published_by: null, run_count: 0, @@ -111,6 +128,7 @@ const FAKES = { 'pf-lib': wf('pf-lib', '__publish_check_lib__', { nodes: [], edges: [] }, { status: 'published', published_version: 4 }), 'pf-bare': wf('pf-bare', '__publish_check_bare__', BARE, { status: 'governed', published_version: 2 }), 'pf-dup': wf('pf-dup', '__publish_check_dup__', DUP, { status: 'governed', published_version: 2 }), + 'pf-handoff': wf('pf-handoff', '__publish_check_handoff__', HANDOFF, { status: 'governed', published_version: 2 }), // 带全图默认的图:画布只管节点和连线,保存、检查、修复都得把 defaults 原样带着 'pf-defs': wf('pf-defs', '__publish_check_defaults__', { ...EASY, defaults: { approval: 'always', model: 'demo-model' } }), } @@ -162,6 +180,19 @@ function lint(graph, level, { followsNever = false } = {}) { add(n, 'governed.subgraph_unpinned', `${who(n)}(子工作流)没有钉住版本,口径会随上游最新版漂移`, { kind: 'auto', label: '钉到它最新的已发布版本 v4', preview: { field: 'workflow_version', before: null, after: 4 } }) } + if (n.type === 'code' && c.evidence_role !== 'source') { + add(n, 'governed.caliber_compute_input', `${who(n)}(沙箱代码)的产出喂给了口径卡,而它的角色是计算`, { + kind: 'choice', label: `${who(n)}的产出喂给了口径卡:它是在取数,还是在做计算?`, + options: [{ value: 'source', label: '它在取数:标成 source(取数)' }, + { value: 'copilot', label: '它在做计算:交给 Copilot 把计算挪进口径卡', handoff: true }], + }) + } + if (n.type === 'output' && c.contract?.report_from && !(c.fields ?? []).every((f) => f.value.includes(`nodes.${c.contract.report_from}.`))) { + add(n, 'governed.exit_text_source', `${who(n)}的成果字段取的不是报告撰写节点的正文`, { + kind: 'choice', label: '把成果字段改成取哪个报告撰写节点的正文', + options: graph.nodes.filter((r) => r.type === 'report').map((r) => ({ value: r.id, label: who(r), hint: '报告撰写' })), + }) + } if (n.type === 'supervisor') { add(n, 'governed.supervisor', `${who(n)}(多 Agent 协作):受管模板不允许全动态规划的节点`, { kind: 'assist', label: '换成固定编排是结构性的改动,交给 Copilot' }) @@ -202,6 +233,7 @@ function autofix(body, { unpublished = false, followsNever = false } = {}) { const changes = [] const rejected = [] const applied = [] + const handoff = [] const byId = (id) => graph.nodes.find((n) => n.id === id) for (const id of body.apply ?? []) { const fix = before.fixes.find((f) => f.id === id) @@ -230,6 +262,15 @@ function autofix(body, { unpublished = false, followsNever = false } = {}) { if (!Array.isArray(v) || !v.length) { rejected.push({ fix_id: id, reason: '要人选:没给选择' }); continue } const contract = { report_from: 'write', metrics_from: ['caliber'], strict: true, required: v } change('contract', null, contract); c.contract = contract + } else if (fix.code === 'governed.caliber_compute_input') { + const v = body.choices?.[id] + // 只认候选上的 handoff 标记,和后端 autofix._handoff 一样 + if (fix.options.some((o) => o.handoff && o.value === v)) { handoff.push(id); continue } + change('evidence_role', c.evidence_role ?? null, v); c.evidence_role = v + } else if (fix.code === 'governed.exit_text_source') { + const v = body.choices?.[id] + const to = `{{ nodes.${v}.text }}` + change('fields[0].value', c.fields[0].value, to); c.fields[0].value = to } else if (fix.code === 'contract.required_missing') { const v = body.choices?.[id] if (!Array.isArray(v) || !v.length) { rejected.push({ fix_id: id, reason: '要人选:没给选择' }); continue } @@ -238,7 +279,7 @@ function autofix(body, { unpublished = false, followsNever = false } = {}) { applied.push(id) } let assist = null - if (body.assist) { + if (body.assist || handoff.length) { const team = byId('team') if (team) { changes.push({ fix_id: 'assist', node_id: 'team', node_title: team.data.label, field: 'max_rounds', before: team.data.config.max_rounds, after: 2, @@ -249,7 +290,7 @@ function autofix(body, { unpublished = false, followsNever = false } = {}) { questions: ['复核团队要换成哪一个固定编排的 Agent?它现在只有一个成员 checker,保留它的职责吗?'] } } const remaining = lint(graph, body.level, { followsNever }) - return { graph, changes, applied, rejected, remaining: remaining.issues, assist, ok: remaining.ok, ops: [] } + return { graph, changes, applied, rejected, handoff, remaining: remaining.issues, assist, ok: remaining.ok, ops: [] } } // ---------------------------------------------------------------- 浏览器 @@ -261,7 +302,7 @@ const writes = [] * backend:'new'(两个接口都有)| 'old'(两个接口 404)| 'nofix'(有检查、没有 autofix) * save:'ok' | 'fail'(保存接口 500)| 'slow'(1.5 秒后才回,看保存进行中的样子) */ -async function open({ id = 'pf-hard', width = 1440, height = 900, backend = 'new', save = 'ok', unpublished = false } = {}) { +async function open({ id = 'pf-hard', width = 1440, height = 900, backend = 'new', save = 'ok', unpublished = false, fixDelay = 0 } = {}) { const followsNever = id === 'pf-dup' const ctx = await browser.newContext({ viewport: { width, height } }) opened.add(ctx) @@ -316,6 +357,7 @@ async function open({ id = 'pf-hard', width = 1440, height = 900, backend = 'new const body = req.postDataJSON() state.autofixes.push(body) if (state.backend !== 'new') return json(route, { detail: 'Not Found' }, 404) + if (fixDelay) await new Promise((r) => setTimeout(r, fixDelay)) return json(route, autofix(body, { unpublished, followsNever })) } if (method === 'POST' && sub === 'publish') { @@ -793,6 +835,48 @@ await section('全图默认不会被画布弄丢:保存、发布前检查、 void first }) +await section('choice 里选「交给 Copilot」:请求带 assist、等待时说 Copilot 在修;id 叫 copilot 的报告只是普通候选', async () => { + const { page, state, errors } = await open({ id: 'pf-handoff', fixDelay: 1200 }) + await openDialog(page) + const g4 = dialog(page).locator('[data-fix-choice="governed.caliber_compute_input:calc"]') + const g1 = dialog(page).locator('[data-fix-choice="governed.exit_text_source:done"]') + await g4.waitFor() + check('两个 choice 都画出来了', await g1.count() === 1) + check('没选之前,按钮不是「交给 Copilot」的样子', await g4.locator('[data-fix-handoff]').count() === 0) + await g4.getByText('它在做计算', { exact: false }).click() + const button = g4.locator('[data-fix-action="choice"]') + check('选了「交给 Copilot」那一项,按钮换成「交给 Copilot」', await button.getAttribute('data-fix-handoff') === '' + && (await button.innerText()).includes('交给 Copilot'), await button.innerText()) + await button.click() + const loading = dialog(page).locator('[data-fix-preview="loading"]') + await loading.waitFor() + check('等待时写「Copilot 正在试着修」,不是「正在生成修复预览」', (await loading.innerText()).includes('Copilot 正在试着修'), await loading.innerText()) + const body = state.autofixes.at(-1) + check('请求带 assist: true,选的值原样带上', body?.assist === true && body?.choices?.['governed.caliber_compute_input:calc'] === 'copilot' + && JSON.stringify(body?.apply) === JSON.stringify(['governed.caliber_compute_input:calc']), JSON.stringify(body)) + await dialog(page).locator('[data-fix-preview="ready"], [data-fix-preview]:not([data-fix-preview="loading"])').first().waitFor() + check('Copilot 的总结摆出来了', await waitFor(page, async () => (await dialog(page).locator('[data-fix-assist]').count()) > 0)) + await shoot(page, 'publish-dialog-handoff', dialog(page)) + + // 另一个 choice:唯一的候选是 id 叫 copilot 的报告撰写节点——普通候选,不交给 Copilot + const { page: page2, state: state2, errors: errors2 } = await open({ id: 'pf-handoff', fixDelay: 1200 }) + await openDialog(page2) + const exit = dialog(page2).locator('[data-fix-choice="governed.exit_text_source:done"]') + await exit.waitFor() + await exit.locator('input[type="radio"]').first().check() + const pick = exit.locator('[data-fix-action="choice"]') + check('选了 id 叫 copilot 的报告:按钮不变成「交给 Copilot」', await pick.getAttribute('data-fix-handoff') === null + && !(await pick.innerText()).includes('交给 Copilot'), await pick.innerText()) + await pick.click() + const wait2 = dialog(page2).locator('[data-fix-preview="loading"]') + await wait2.waitFor() + check('等待时写「正在生成修复预览」', (await wait2.innerText()).includes('正在生成修复预览'), await wait2.innerText()) + const body2 = state2.autofixes.at(-1) + check('请求不带 assist,选的值是 copilot(节点 id)', !body2?.assist && body2?.choices?.['governed.exit_text_source:done'] === 'copilot', + JSON.stringify(body2)) + check('没有页面错误', !errors.length && !errors2.length, [...errors, ...errors2].join(' | ')) +}) + await section('360px 下不横向滚动', async () => { const { page } = await open({ width: 360, height: 780 }) // 窄屏上「发布」收在「更多操作」里 diff --git a/scripts/check-runs.mjs b/scripts/check-runs.mjs index 5d5fb462..0d36e730 100644 --- a/scripts/check-runs.mjs +++ b/scripts/check-runs.mjs @@ -16,6 +16,8 @@ import { chromium } from '../frontend/node_modules/playwright-core/index.mjs' const WEB = process.env.AGENTLAB_WEB ?? 'http://localhost:5273' const API = process.env.AGENTLAB_API ?? 'http://localhost:8000/api' +// RUNS_SHOTS=<目录>:证据页签亮暗各截几张,供人眼复核 +const SHOTS = process.env.RUNS_SHOTS ?? '' const CHROME = process.env.CHROME_PATH ?? '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' @@ -47,7 +49,11 @@ const check = (name, cond, detail = '') => { * 不让一处卡住把后面的检查一起吞掉。页面是各节共用的: * 出错那一节停在哪,下一节就从哪接着 */ +// RUNS_ONLY=证据页签 只跑段名里含这些字的段(逗号分隔;改坏验证时省时间)。前面几段给后面留的数据 +// (航迹那一节的时刻、工件)只在整本跑时有,单跑依赖它们的段会报「中途出错」;check-all 不传它 +const ONLY = (process.env.RUNS_ONLY ?? '').split(',').filter(Boolean) async function section(name, fn) { + if (ONLY.length && !ONLY.some((k) => name.includes(k)) && name !== '收尾') return console.log(`\n=== ${name} ===`) try { await fn() @@ -1698,6 +1704,296 @@ await section('屏幕矮、节点多:读数区不被航迹坞压住', async () fakes.delete('GET /api/approvals') }) +// ------------------------------------------------------------------ 证据页签(可点击证据三期) + +// 夹具是后端真跑出来的(compose_doc、guess_sources、api/evidence.py 的审计函数),只用通用名 +const fxe = JSON.parse(readFileSync(new URL('../frontend/src/run/__tests__/evidence-entity.json', import.meta.url), 'utf8')) +const EV = 'fake0evidence0000000000000000' +const EV_LEGACY = 'fake0evlegacy0000000000000000' +const EV_NONE = 'fake0evnone000000000000000000' +const EV_OLD = 'fake0evoldback000000000000000' +const evGraph = { nodes: [ + { id: 'in', type: 'input', position: { x: 0, y: 0 }, data: { label: '输入', config: {} } }, + { id: 'fetch', type: 'agent', position: { x: 200, y: 0 }, data: { label: '取数', config: { tools: ['db_query__shop'] } } }, + { id: 'manual', type: 'retrieve', position: { x: 200, y: 150 }, data: { label: '查手册', config: {} } }, + { id: 'write', type: 'report', position: { x: 400, y: 0 }, data: { label: '写周报', config: {} } }, + { id: 'out', type: 'output', position: { x: 600, y: 0 }, data: { label: '成果', config: {} } }, +], edges: [{ source: 'in', target: 'fetch' }, { source: 'fetch', target: 'write' }, { source: 'manual', target: 'write' }, + { source: 'write', target: 'out' }] } +const evRun = (id, output) => ({ + ...base, id, workflow_id: null, workflow_name: '证据检查', status: 'succeeded', run_class: 'formal', + version: 3, version_hash: null, manifest_hash: 'f'.repeat(64), manifest_seq: 88, error: null, error_node_id: null, + input: { week: '2026-W37' }, output, usage: { input_tokens: 10, output_tokens: 10, cost_usd: 0, wall_ms: 4000, active_ms: 4000, wait_ms: 0 }, + created_at: new Date(Date.now() - 7200e3).toISOString(), started_at: new Date(Date.now() - 7200e3).toISOString(), + finished_at: new Date(Date.now() - 7196e3).toISOString(), +}) +const evEvents = (output, report = true) => { + const t0 = Date.now() / 1000 - 7200 + return [ + { seq: 1, type: 'run.started', node_id: null, ts: t0, data: { nodes: 5 } }, + { seq: 2, type: 'node.started', node_id: 'write', ts: t0 + 1, data: { node_type: 'report', label: '写周报' } }, + ...(report ? [{ seq: 3, type: 'report.checked', node_id: 'write', ts: t0 + 3, data: fxe.report_checked }] : []), + { seq: 4, type: 'node.finished', node_id: 'write', ts: t0 + 3.1, data: { duration_ms: 2100 } }, + { seq: 5, type: 'run.finished', node_id: null, ts: t0 + 4, data: { output, usage: {}, timing: { wall_ms: 4000, active_ms: 4000, wait_ms: 0 } } }, + ] +} +/** 证据相关的请求(审计、导出、片段)记下来:导出发出的请求要带对 format 和 groups */ +const evRequests = [] +page.on('request', (r) => { + const u = new URL(r.url()) + if (/\/api\/runs\/fake0ev[^/]*\/evidence/.test(u.pathname)) evRequests.push(`${u.pathname}${u.search}`) +}) +const auditFake = (body, csv) => (url) => { + const format = url.searchParams.get('format') + const groups = url.searchParams.get('groups')?.split(',') + const pick = groups ? { ...body, groups: body.groups.filter((g) => groups.includes(g.key)) } : body + if (format === 'csv') { + return { status: 200, body: csv, headers: { 'Content-Type': 'text/csv; charset=utf-8', + 'Content-Disposition': `attachment; filename="evidence-${body.run_id.slice(0, 8)}.csv"` } } + } + if (format === 'json') { + return { status: 200, body: JSON.stringify(pick), headers: { 'Content-Type': 'application/json', + 'Content-Disposition': `attachment; filename="evidence-${body.run_id.slice(0, 8)}.json"` } } + } + return { status: 200, json: pick } +} +const evFakes = (id, { output, graph, audit, csv, report = true }) => { + fakes.set(`GET /api/runs/${id}`, () => ({ status: 200, json: evRun(id, output) })) + fakes.set(`GET /api/runs/${id}/events`, () => ({ status: 200, json: evEvents(output, report) })) + fakes.set(`GET /api/runs/${id}/graph`, () => ({ status: 200, json: { graph: evGraph, workflow_id: null, version: 3 } })) + fakes.set(`GET /api/runs/${id}/artifacts`, () => ({ status: 200, json: [] })) + fakes.set(`GET /api/runs/${id}/evidence`, () => ({ status: 200, json: { ...graph, run_id: id } })) + fakes.set(`GET /api/runs/${id}/evidence/audit`, audit ? auditFake({ ...audit, run_id: id }, csv ?? '') + : () => ({ status: 404, json: { detail: 'Not Found' } })) +} +const openEvidence = async (id) => { + await page.goto(`${WEB}/runs/${id}?view=evidence`, { waitUntil: 'networkidle' }) + await page.locator('[data-view-pane=evidence] [data-evidence-pane]:not([data-evidence-pane=loading])').waitFor({ timeout: 8000 }) + await page.waitForTimeout(300) +} +const auditGroups = () => page.locator('[data-audit-table] tbody[data-audit-group]').evaluateAll((els) => els.map((e) => e.getAttribute('data-audit-group'))) +const auditRowsShown = () => page.locator('[data-audit-table] tr[data-audit-row]').count() + +await section('证据页签:左边报告、右边常驻面板、下方审计表', async () => { + evFakes(EV, { output: fxe.output, graph: fxe.graph, audit: fxe.audit, csv: fxe.audit_csv }) + fakes.set(`GET /api/artifacts/${fxe.doc_artifact}`, () => ({ status: 200, json: { id: fxe.doc_artifact, content: fxe.doc } })) + fakes.set(`GET /api/runs/${EV}/evidence/segments/`, () => ({ status: 404, json: {} })) + await page.route(new RegExp(`/api/runs/${EV}/evidence/segments/`), (r) => { + const sid = decodeURIComponent(new URL(r.request().url()).pathname.split('/').pop()) + return fxe.segments[sid] ? r.fulfill({ json: fxe.segments[sid] }) : r.fulfill({ status: 404, json: { detail: '没有这个片段' } }) + }) + await page.goto(`${WEB}/runs/${EV}`, { waitUntil: 'networkidle' }) + await page.locator('[data-run-detail]').waitFor() + check('详情多了「证据」页签(在航迹和工件之间)', await viewTab('evidence').count() === 1 + && (await page.locator('[data-run-detail] [role=tab]').evaluateAll((els) => els.map((e) => e.getAttribute('data-tab')))).join(',') === 'stream,trace,evidence,artifacts') + check('没打开证据页签之前不取证据图和审计表', !evRequests.some((u) => u.includes(EV)), evRequests.join(' ')) + await viewTab('evidence').click() + await page.locator('[data-view-pane=evidence] [data-evidence-pane=cited]').waitFor({ timeout: 8000 }).catch(() => {}) + await page.waitForTimeout(400) + check('证据页签能打开,地址记着 ?view=evidence', await viewTab('evidence').getAttribute('aria-selected') === 'true' + && new URL(page.url()).searchParams.get('view') === 'evidence') + check('左边是报告(逐段可点的文档)', await page.locator('[data-evidence-report=write] [data-evidence-doc] [data-seg]').count() > 5) + check('右边是常驻面板的位置,没点片段时说怎么用', await page.locator('[data-evidence-dock] [data-evidence-dock-idle]').count() === 1) + const seal = await page.locator('[data-evidence-seal]').innerText().catch(() => '') + check('显示封存核对的结果', await page.locator('[data-evidence-seal]').getAttribute('data-evidence-seal') === 'done' + && seal.includes('已封存 · 核对一致') && seal.includes('88'), seal) + check('……也说报告文档的哈希一致', (await page.locator('[data-evidence-doc-hash=ok]').innerText().catch(() => '')).includes('写周报')) + check('审计表按状态分组:无证据、可疑实体在前,有出处在后', (await auditGroups()).join(',') === 'none,suspicious,cited', + (await auditGroups()).join(',')) + const counts = await page.locator('[data-audit-group-toggle]').allInnerTexts() + check('每组写着条数(无证据 3、可疑实体 3、有出处 10)', counts.join('|').includes('无证据\n3') || (counts[0].includes('3') && counts[1].includes('3') && counts[2].includes('10')), + counts.join(' | ').replace(/\n/g, ' ')) + check('一共 16 行', await auditRowsShown() === 16, String(await auditRowsShown())) + // 这张图的出口没有出具契约:后端说「要求了,但没有契约按它判档」,不说「出具时计入缺口」。表里照后端的原话写 + const claim = page.locator('[data-audit-table] tbody[data-audit-group=none] tr[data-audit-row]', { hasText: '另外参考了' }) + const claimNote = fxe.audit.groups.find((g) => g.key === 'none')?.rows.find((r) => r.kind === 'claim')?.note ?? '(夹具里没有结论句那行)' + check('没挂依据的结论句也列在无证据里,原因照后端的原话(claims: require_citation、没有出具契约)', await claim.count() === 1 + && claimNote.includes('require_citation') && (await claim.innerText()).includes(claimNote), claimNote) + check('可疑实体那组有「可能是编造的名字」和「核对不了」', (await page.locator('[data-audit-table] tbody[data-audit-group=suspicious]').innerText()) + .includes('可能是编造的名字') && (await page.locator('[data-audit-table] tbody[data-audit-group=suspicious]').innerText()).includes('核对不了')) + check('有出处的行写着在封存范围内', await page.locator('[data-audit-table] tbody[data-audit-group=cited] [data-audit-sealed=yes]').count() === 10) + check('反引号里的名字在表里不带反引号', !(await page.locator('[data-audit-table]').innerText()).includes('`')) + + // 只看无证据 / 可疑实体 + await page.locator('[data-audit-filter-option=problems]').click() + await page.waitForTimeout(200) + check('筛选「只看无证据 / 可疑实体」:只剩这两组', (await auditGroups()).join(',') === 'none,suspicious' && await auditRowsShown() === 6, + `${(await auditGroups()).join(',')} ${await auditRowsShown()}`) + check('筛选是一组单选(radio),选中的那项 aria-checked', await page.locator('[data-audit-filter-option=problems]').getAttribute('aria-checked') === 'true') + + // 导出:发出的请求要带对 format,筛选了就带 groups + evRequests.length = 0 + const dl1 = page.waitForEvent('download', { timeout: 5000 }).catch(() => null) + await page.locator('[data-audit-export=csv]').click() + const d1 = await dl1 + check('导出 CSV:请求 /evidence/audit?format=csv,只要筛出来的两组', + evRequests.some((u) => u === `/api/runs/${EV}/evidence/audit?format=csv&groups=none%2Csuspicious`), evRequests.join(' ')) + check('……下载的文件名是后端给的', d1?.suggestedFilename() === `evidence-${EV.slice(0, 8)}.csv`, d1?.suggestedFilename() ?? '没有下载') + await page.locator('[data-audit-filter-option=all]').click() + evRequests.length = 0 + const dl2 = page.waitForEvent('download', { timeout: 5000 }).catch(() => null) + await page.locator('[data-audit-export=json]').click() + const d2 = await dl2 + check('导出 JSON:请求 ?format=json,不筛就不带 groups', evRequests.some((u) => u === `/api/runs/${EV}/evidence/audit?format=json`), + evRequests.join(' ')) + check('……下载了 .json', d2?.suggestedFilename()?.endsWith('.json'), d2?.suggestedFilename() ?? '没有下载') + + // 键盘:整张表一个 Tab 位,↑/↓ 逐行走完每一行,回车在右边打开 + const focusables = await page.locator('[data-audit-table] [data-audit-focus]').evaluateAll((els) => els.map((e) => e.tabIndex)) + check('表里只有一个 Tab 位(roving)', focusables.filter((t) => t === 0).length === 1 && focusables.length === 16, focusables.join(',')) + await page.locator('[data-audit-table] [data-audit-focus][tabindex="0"]').focus() + const seen = [await page.evaluate(() => document.activeElement?.getAttribute('data-audit-focus'))] + for (let i = 0; i < 20; i++) { + await page.keyboard.press('ArrowDown') + seen.push(await page.evaluate(() => document.activeElement?.getAttribute('data-audit-focus'))) + } + const uniq = [...new Set(seen.filter(Boolean))] + check('↓ 能逐行走完全部 16 行(包括没挂依据的结论句这种打不开的行)', uniq.length === 16, `${uniq.length} 行`) + check('走到底停住,不首尾相接', seen[seen.length - 1] === seen[seen.length - 2]) + await page.keyboard.press('Home') + const home = await page.evaluate(() => document.activeElement?.getAttribute('data-audit-focus')) + await page.keyboard.press('End') + const end = await page.evaluate(() => document.activeElement?.getAttribute('data-audit-focus')) + check('Home / End 到第一行、最后一行', home === uniq[0] && end === uniq[uniq.length - 1], `${home} / ${end}`) + // 找到「orders.week」那一行,回车打开 + const target = await page.locator('[data-audit-table] [data-audit-open]', { hasText: /^orders\.week$/ }).getAttribute('data-audit-focus') + await page.keyboard.press('Home') + for (let i = 0; i < 20; i++) { + if (await page.evaluate(() => document.activeElement?.getAttribute('data-audit-focus')) === target) break + await page.keyboard.press('ArrowDown') + } + await page.keyboard.press('Enter') + await page.locator('[data-evidence-dock] [data-evidence-panel=dock] [data-ev-entity-type]').waitFor({ timeout: 5000 }).catch(() => {}) + check('回车在右边的常驻面板里打开这一段:实体步骤、字段类型', (await page.locator('[data-evidence-dock] [data-evidence-panel] h3').innerText().catch(() => '')) === 'orders.week' + && (await page.locator('[data-evidence-dock] [data-ev-entity-type]').innerText().catch(() => '')).includes('VARCHAR')) + check('打开之后焦点还在表里(接着往下走)', await page.evaluate(() => document.activeElement?.getAttribute('data-audit-focus')) === target) + check('面板开着时空位提示收起', await page.locator('[data-evidence-dock] [data-evidence-dock-idle]').count() === 0) + check('正文里对应的片段标着展开', await page.locator('[data-evidence-report=write] [data-seg][aria-expanded=true]').innerText().catch(() => '') === 'orders.week') + // 点正文里的片段:面板跟着换 + await page.locator('[data-evidence-report=write] [data-seg]', { hasText: '退款金额以财务确认日为准' }).click() + await page.locator('[data-evidence-dock] [data-ev-quote-hit]').waitFor({ timeout: 5000 }).catch(() => {}) + check('点正文里的引文:右边换成引文步骤,原文里高亮', (await page.locator('[data-evidence-dock] [data-ev-quote-hit]').innerText().catch(() => '')) + .includes('退款金额以财务确认日为准')) + if (SHOTS) { + for (const theme of ['dark', 'light']) { + await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + await page.waitForTimeout(250) + await page.screenshot({ path: `${SHOTS}/runs-evidence-${theme}.png` }) + await page.locator('[data-evidence-audit]').scrollIntoViewIfNeeded() + await page.waitForTimeout(150) + await page.screenshot({ path: `${SHOTS}/runs-evidence-audit-${theme}.png` }) + } + await page.evaluate(() => document.documentElement.removeAttribute('data-theme')) + } + await page.keyboard.press('Escape') +}) + +await section('证据页签:同一段文字里好几条同样的违规、正文里点不开的行', async () => { + // 夹具同样是后端真跑的:粗体、链接里反引号写的可疑名字只记违规、不切片段——同一段文字上两条 unknown_entity、 + // 两条 unverified_entity(片段和 code 都相同),列表序号 100. 是结构片段上的裸数字。这几行正文里都点不开 + const dup = fxe.dup + const EV_DUP = 'fake0evdupviol000000000000000' + evFakes(EV_DUP, { output: dup.output, graph: dup.graph, audit: dup.audit }) + fakes.set(`GET /api/artifacts/${dup.doc_artifact}`, () => ({ status: 200, json: { id: dup.doc_artifact, content: dup.doc } })) + await page.route(new RegExp(`/api/runs/${EV_DUP}/evidence/segments/`), (r) => { + const sid = decodeURIComponent(new URL(r.request().url()).pathname.split('/').pop()) + return dup.segments[sid] ? r.fulfill({ json: dup.segments[sid] }) : r.fulfill({ status: 404, json: { detail: '没有这个片段' } }) + }) + const consoleErrors = [] + const onConsole = (m) => { if (m.type() === 'error') consoleErrors.push(m.text()) } + page.on('console', onConsole) + try { + await openEvidence(EV_DUP) + await page.locator('[data-audit-table]').waitFor({ timeout: 5000 }).catch(() => {}) + const texts = await page.locator('[data-audit-table] [data-audit-focus]').allInnerTexts() + const keys = await page.locator('[data-audit-table] tr[data-audit-row]').evaluateAll((els) => els.map((e) => e.getAttribute('data-audit-row'))) + const total = dup.audit.groups.reduce((n, g) => n + g.rows.length, 0) + check(`每条违规一行(${total} 行),同一片段同一种问题的几行键各不相同`, keys.length === total && new Set(keys).size === keys.length, + keys.join(' ')) + check('React 没有报「两个子元素键相同」', !consoleErrors.some((t) => /same key/i.test(t)), consoleErrors.find((t) => /same key/i.test(t))?.slice(0, 120) ?? '') + + await page.locator('[data-audit-table] [data-audit-focus][tabindex="0"]').focus() + const seen = [await page.evaluate(() => document.activeElement?.getAttribute('data-audit-focus'))] + for (let i = 0; i < total + 2; i++) { + await page.keyboard.press('ArrowDown') + seen.push(await page.evaluate(() => document.activeElement?.getAttribute('data-audit-focus'))) + } + const walked = [...new Set(seen.filter(Boolean))] + check(`↓ 逐行走完全部 ${total} 行,不卡在同一片段的第一条违规上`, walked.length === total + && seen.slice(0, total - 1).every((k, i) => k !== seen[i + 1]), `${walked.length} 行:${seen.join(' → ')}`) + const reached = await page.evaluate((ks) => ks.map((k) => document.querySelector(`[data-audit-focus="${CSS.escape(k)}"]`)?.textContent), walked) + check('……粗体、链接里的四个可疑名字都走得到', ['orders.coupon_id', 'orders.promo_code', 'campaign_tags', 'promo_rules'].every((n) => reached.includes(n)), + reached.join('、')) + + // 点不开的行不画成按钮:只有有状态的片段(这里是那个有出处的数字)能在右边打开 + const openers = await page.locator('[data-audit-table] [data-audit-open]').evaluateAll((els) => els.map((e) => e.getAttribute('data-audit-open'))) + check('只有正文里点得开的片段画成按钮(文字片段、结构片段上的违规不是)', openers.join(',') === 's1', openers.join(',') || '一个都没有') + const hidden = page.locator('[data-audit-table] tr[data-audit-row]:has([data-audit-hidden])') + check('……点不开的五行都说清只在表里列出', await hidden.count() === 5 + && (await hidden.first().innerText()).includes('只在这里列出'), String(await hidden.count())) + const deadKey = keys.find((k, i) => texts[i] === 'orders.promo_code') + await page.locator(`[data-audit-table] [data-audit-focus="${deadKey}"]`).focus() + await page.keyboard.press('Enter') + await page.waitForTimeout(300) + check('……在点不开的行上按回车:右边不开一个空面板,也不报错', await page.locator('[data-evidence-dock] [data-evidence-dock-idle]').count() === 1) + await page.locator('[data-audit-table] [data-audit-open="s1"]').click() + await page.locator('[data-evidence-dock] [data-evidence-panel]').waitFor({ timeout: 5000 }).catch(() => {}) + check('点得开的那行:右边打开这一段', (await page.locator('[data-evidence-dock] [data-evidence-panel] h3').innerText().catch(() => '')) === '45,678.5') + await page.keyboard.press('Escape') + } finally { + page.off('console', onConsole) + for (const k of [...fakes.keys()]) if (k.includes(EV_DUP)) fakes.delete(k) + fakes.delete(`GET /api/artifacts/${dup.doc_artifact}`) + } +}) + +await section('证据页签:没有报告的运行照实说明(none / 旧运行猜测 / 老后端)', async () => { + evFakes(EV_LEGACY, { output: fxe.legacy.output, graph: fxe.legacy.graph, audit: fxe.legacy.audit, report: false }) + await openEvidence(EV_LEGACY) + check('没有契约的旧运行:mode 是 legacy_text,照实说', await page.locator('[data-evidence-pane=legacy_text]').count() === 1 + && (await page.locator('[data-evidence-mode-note=legacy_text]').innerText()).includes('没有出具契约')) + check('猜测默认收起', await page.locator('[data-evidence-legacy-text] [data-ev-guess]').getAttribute('data-open') === 'false') + check('审计表里「旧运行猜测」那组默认收起', await page.locator('tbody[data-audit-group=candidate]').getAttribute('data-audit-collapsed') === '1' + && await page.locator('tbody[data-audit-group=candidate] tr[data-audit-row]').count() === 0) + await page.locator('[data-evidence-legacy-text] [data-ev-guess-toggle]').click() + check('展开猜测写明「猜测的来源,不能当证据」', (await page.locator('[data-evidence-legacy-text] [data-ev-guess-note]').innerText().catch(() => '')) + .includes('猜测的来源,不能当证据')) + await page.locator('[data-audit-group-toggle=candidate]').click() + check('展开那一组:有候选的 4 个数字,线型是候选那一档', await page.locator('tbody[data-audit-group=candidate] tr[data-audit-row]').count() === 4 + && (await page.locator('tbody[data-audit-group=candidate] tr[data-audit-row]').evaluateAll((els) => els.every((e) => e.getAttribute('data-audit-state') === 'candidate')))) + if (SHOTS) { + for (const theme of ['dark', 'light']) { + await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + await page.waitForTimeout(200) + await page.screenshot({ path: `${SHOTS}/runs-evidence-legacy-${theme}.png` }) + } + await page.evaluate(() => document.documentElement.removeAttribute('data-theme')) + } + + evFakes(EV_NONE, { output: { answer: '这次没有数字' }, report: false, + graph: { schema: 'agentlab.evidence/1', mode: 'none', note: '这次运行没有报告文档,也没有出具契约,没有可以展示的证据', seal: { sealed: true, ok: true }, + reports: [], evidence: [], edges: [] }, + audit: { schema: 'agentlab.evidence.audit/1', mode: 'none', seal: { sealed: true, ok: true }, reports: [], groups: [], counts: {}, total: 0 } }) + await openEvidence(EV_NONE) + check('mode 为 none:照实说没有证据', await page.locator('[data-evidence-pane=none] [data-evidence-mode-note=none]').count() === 1 + && (await page.locator('[data-evidence-mode-note=none]').innerText()).includes('没有')) + + // 老后端:审计接口不存在(404 没有机读码)——清单按页面上的报告自己拼,导出也按页面上的 + evFakes(EV_OLD, { output: fxe.output, graph: fxe.graph, audit: null }) + await openEvidence(EV_OLD) + await page.locator('[data-audit-fallback]').waitFor({ timeout: 5000 }).catch(() => {}) + check('老后端没有审计接口:照实说,清单按页面上的报告拼', await page.locator('[data-audit-fallback=unsupported]').count() === 1 + && await auditRowsShown() > 10, String(await auditRowsShown())) + const dl = page.waitForEvent('download', { timeout: 5000 }).catch(() => null) + await page.locator('[data-audit-export=csv]').click() + const d = await dl + check('……导出退回按页面上的清单,也照实说', !!d && (await page.locator('[data-toast], [role=status], [role=alert]').allInnerTexts()).join(' ').includes('后端没有导出接口')) + for (const id of [EV, EV_LEGACY, EV_NONE, EV_OLD]) { + for (const k of [...fakes.keys()]) if (k.includes(id)) fakes.delete(k) + } + fakes.delete(`GET /api/artifacts/${fxe.doc_artifact}`) +}) + await section('收尾', async () => { // 提取模板之后会跳进画布:画布一打开就拿图去做校验、变量分析,这两个是带图的只读 POST // (同样被探针拦下,不落库)。赶上它们发出来之前就离开了画布的话就没有 diff --git a/scripts/check-studio.mjs b/scripts/check-studio.mjs index 887726d0..6f760692 100644 --- a/scripts/check-studio.mjs +++ b/scripts/check-studio.mjs @@ -7,6 +7,7 @@ // // 跑之前前端得起着(./scripts/dev.sh),默认连 5273。对别的实例(比如一份沙箱拷贝)跑时 // 带上地址:AGENTLAB_WEB=http://localhost:<前端端口> node scripts/check-studio.mjs +import { readFileSync } from 'node:fs' import { chromium } from '../frontend/node_modules/playwright-core/index.mjs' const WEB = process.env.AGENTLAB_WEB ?? 'http://localhost:5273' @@ -100,6 +101,19 @@ const TEAM_BOUND = { ? { ...n, data: { ...n.data, config: { ...n.data.config, tools: [...QUERY_TOOLS] } } } : n)), } +// 可疑实体、引文、表名字段名的报告(可点击证据三期):夹具是后端真跑出来的,只用通用名 +const fxe = JSON.parse(readFileSync(new URL('../frontend/src/run/__tests__/evidence-entity.json', import.meta.url), 'utf8')) +const EV_GRAPH = { + nodes: [ + node('start', 'input', 0, 200, '输入', { fields: [{ name: 'week', required: true }] }), + node('fetch', 'agent', 300, 120, '取数', { prompt: '查 {{ input.week }} 的订单', tools: ['db_query__shop'], assign_to: 'rows' }), + node('manual', 'retrieve', 300, 320, '查手册', { query: '退款口径', collection: 'ops', assign_to: 'manual' }), + node('write', 'report', 640, 200, '写周报', { instructions: '写本周周报', numbers: 'strict', claims: 'require_citation' }), + node('done', 'output', 980, 200, '成果', { fields: [{ name: 'answer', value: '{{ nodes.write.text }}' }] }), + ], + edges: [edge('start', 'fetch'), edge('start', 'manual'), edge('fetch', 'write'), edge('manual', 'write'), edge('write', 'done')], +} + const wf = (id, extra) => ({ id, name: extra.name, description: extra.description ?? '检查脚本伪造的工作流', graph: extra.graph ?? GRAPH, tags: extra.tags ?? [], version: extra.version ?? 3, is_template: !!extra.is_template, @@ -113,6 +127,7 @@ const FAKES = { 'st-tpl': wf('st-tpl', { name: '__studio_check_tpl__', is_template: true, tags: ['extracted'] }), 'st-team': wf('st-team', { name: '__studio_check_team__', graph: TEAM_BOUND, version: 2, published_version: 2, status: 'published' }), 'st-lint': wf('st-lint', { name: '__studio_check_lint__', graph: TEAM_GRAPH, version: 1 }), + 'st-ev': wf('st-ev', { name: '__studio_check_ev__', graph: EV_GRAPH, version: 2 }), } const VERSIONS = [ { id: 'v3', version: 3, note: '', created_at: '2026-09-26T02:00:00Z' }, @@ -168,7 +183,7 @@ async function ready(page, fn, what, { arg = null, timeout = 15000, retry = true * only:目录里只放这几张伪造的(不混真实的),用来造「删完一张不剩」 * platform:伪装成别的平台('Windows'),看快捷键提示是不是跟着平台走 */ -async function open({ width = 1440, height = 900, path = '/studio/st-main', prefs = {}, stream, only, platform, init } = {}) { +async function open({ width = 1440, height = 900, path = '/studio/st-main', prefs = {}, stream, only, platform, init, before } = {}) { const ctx = await browser.newContext({ viewport: { width, height } }) opened.add(ctx) // init:[函数, 参数],在页面脚本之前跑(比如装数渲染次数的钩子) @@ -277,6 +292,8 @@ async function open({ width = 1440, height = 900, path = '/studio/st-main', pref input: {}, output: {}, error: '', usage: {}, created_at: '2026-09-26T02:00:00Z', })) + // before:页面第一次加载之前再装几条路由(画布一打开就发的请求,比如报告节点卡去取最近一次运行) + if (before) await before(page) await page.goto(`${WEB}${path}`, { waitUntil: 'networkidle' }) await ready(page, () => window.__studio?.getState().workflow != null, '画布的 store(window.__studio)') await page.waitForTimeout(500) @@ -1779,9 +1796,14 @@ await section('自动排版:请求在路上时画布锁了或改了,不套 const okToast = page.locator('[role="status"] > div').filter({ hasText: '已重新排版' }) const pos = () => S(page, () => JSON.stringify(window.__studio.getState().nodes.map((n) => [n.id, n.position.x, n.position.y]))) const layoutBtn = page.getByRole('button', { name: '自动排版' }) + // 等排版接口真的回来再看结果:固定等 1100ms 在四道并行、后端忙的时候不够(接口本身就压了 700ms) + const laidOut = () => page.waitForResponse((r) => new URL(r.url()).pathname.endsWith('/api/copilot/layout'), + { timeout: 15000 }).catch(() => null) + const settle = async (done) => { await done; await page.waitForTimeout(250) } const p0 = await pos() const past0 = (await st(page)).past + let done = laidOut() await layoutBtn.click() await page.waitForTimeout(150) await page.evaluate(() => window.__studio.setState((s) => ({ @@ -1789,7 +1811,7 @@ await section('自动排版:请求在路上时画布锁了或改了,不套 input: {}, output: {}, error: '', usage: {}, created_at: '2026-09-26T02:00:00Z' }, runPhase: 'running', trace: { ...s.trace, phase: 'running', runClass: 'formal' }, }))) - await page.waitForTimeout(1100) + await settle(done) check('正式运行半路开始:排版不套用', await pos() === p0 && (await st(page)).past === past0) check('也不报「已重新排版」(撤销按钮会撤掉别的改动)', await okToast.count() === 0) check('说清为什么没排', await page.locator('[role="status"] > div').filter({ hasText: '正式运行进行中' }).count() === 1) @@ -1798,17 +1820,19 @@ await section('自动排版:请求在路上时画布锁了或改了,不套 run: { ...s.run, status: 'succeeded' }, runPhase: 'succeeded', trace: { ...s.trace, phase: 'succeeded' }, }))) await page.waitForTimeout(150) + done = laidOut() await layoutBtn.click() await page.waitForTimeout(150) await S(page, () => window.__studio.getState().updateNode('lookup', { label: '背景检索(排版时改的)' })) - await page.waitForTimeout(1100) + await settle(done) const kept = await S(page, () => window.__studio.getState().nodes.find((n) => n.id === 'lookup')?.data.label) check('排版期间又改了画布:不拿旧图的排版盖掉刚才的改动', kept === '背景检索(排版时改的)', kept) check('说清这次排版没套用', await okToast.count() === 0 && await page.locator('[role="status"] > div').filter({ hasText: '排版期间画布又改过了' }).count() === 1) + done = laidOut() await layoutBtn.click() - await page.waitForTimeout(1100) + await settle(done) check('正常情况照样排版、给撤销', await okToast.count() === 1 && await okToast.locator('button:has-text("撤销")').count() === 1) check('没有运行时报错', errors.length === 0, errors.slice(0, 2).join(' | ')) await ctx.close() @@ -2120,6 +2144,212 @@ await section('发起就被拒(工具不存在)给修复入口;整形节 await run.ctx.close() }) +await section('证据路径:点开报告片段时画布高亮、节点名能对准;报告卡上的章', async () => { + // 运行号用夹具里文档记着的那个:成果字段取文档时核对「是不是这次运行写的」,对不上就按普通文本画 + const EV_RUN = fxe.run_id + const json = (route, body, status = 200) => route.fulfill({ status, contentType: 'application/json', body: JSON.stringify(body) }) + /** 伪造这张图的「最近一次运行」:latest 为 null 时一次都没跑过;传函数时每次请求现取(中途换成新跑的那次) */ + const routes = (latest) => async (page) => { + await page.route((u) => { const x = new URL(u); return x.pathname === '/api/runs' && x.searchParams.get('workflow_id') === 'st-ev' }, + (r) => { + const id = typeof latest === 'function' ? latest() : latest + return json(r, id ? [{ id, workflow_id: 'st-ev', status: 'succeeded', run_class: 'exploratory' }] : []) + }) + await page.route(new RegExp(`/api/runs/${fxe.run_id}/evidence$`), (r) => json(r, fxe.graph)) + await page.route(new RegExp(`/api/runs/${fxe.run_id}/evidence/audit(\\?.*)?$`), (r) => json(r, fxe.audit)) + await page.route(new RegExp(`/api/runs/${fxe.dup.run_id}/evidence/audit(\\?.*)?$`), (r) => json(r, fxe.dup.audit)) + await page.route(new RegExp(`/api/artifacts/${fxe.doc_artifact}$`), (r) => json(r, { id: fxe.doc_artifact, content: fxe.doc })) + await page.route(new RegExp(`/api/artifacts/${fxe.dup.doc_artifact}$`), (r) => json(r, { id: fxe.dup.doc_artifact, content: fxe.dup.doc })) + await page.route(new RegExp(`/api/runs/${fxe.run_id}/evidence/segments/`), (r) => { + const sid = decodeURIComponent(new URL(r.request().url()).pathname.split('/').pop()) + return fxe.segments[sid] ? json(r, fxe.segments[sid]) : json(r, { detail: '没有这个片段' }, 404) + }) + } + const stamp = (page) => page.locator('.react-flow__node[data-id="write"] [data-report-stamp]') + const st = fxe.report_checked.stats + const cited = st.numbers_cited + st.values + st.entities + st.quotes + const none = (st.numbers - st.numbers_cited) + Math.max(0, st.uncited_numbers + st.unresolved - (st.numbers - st.numbers_cited)) + + st.unknown_entities + st.uncited_claims + // 另一次运行(结论句策略 off):章上的数照 stampCounts 的口径算 + const sd = fxe.dup.report_checked.stats + const dupText = `引用 ${sd.numbers_cited + sd.values + (sd.entities ?? 0) + (sd.quotes ?? 0)} · 无证据 ${(sd.numbers - sd.numbers_cited) + + Math.max(0, sd.uncited_numbers + sd.unresolved - (sd.numbers - sd.numbers_cited)) + (sd.unknown_entities ?? 0)}` + /** 画布上把运行摘下来(RunHud「清除」):等它重新去取这张图的最近一次运行(取回来、画完再读章) */ + const clearAndRefetch = async (pg) => { + const refetch = pg.waitForRequest((r) => { const x = new URL(r.url()); return x.pathname === '/api/runs' && x.searchParams.get('workflow_id') === 'st-ev' }, + { timeout: 3000 }).catch(() => null) + await S(pg, () => window.__studio.getState().clearRun()) + const req = await refetch + await pg.waitForLoadState('networkidle').catch(() => {}) + await pg.waitForTimeout(400) + return !!req + } + + // 一次都没跑过:不画章 + let latestRun = null + const { page, errors } = await open({ path: '/studio/st-ev', before: routes(() => latestRun) }) + await page.waitForTimeout(400) + check('这张图没有运行:报告节点卡上不画章', await stamp(page).count() === 0) + + // 挂上一次运行:report.checked 落下就盖章,数字取它的统计(结论句策略为 require_citation,没挂依据的结论句计入) + await S(page, ({ id, rc, output }) => { + const s = window.__studio + const t = Date.now() / 1000 - 5 + s.setState({ run: { id, workflow_id: 'st-ev', workflow_name: '__studio_check_ev__', status: 'queued', input: { week: '2026-W37' }, + output: {}, error: null, usage: {}, run_class: 'exploratory', version: null } }) + const list = [['run.started', null, {}], ['node.started', 'write', { label: '写周报', node_type: 'report' }], + ['report.checked', 'write', rc], ['node.finished', 'write', { duration_ms: 1200 }], + ['run.finished', null, { output, usage: {}, timing: { wall_ms: 1500, active_ms: 1500, wait_ms: 0 } }]] + list.forEach((e, i) => s.getState().applyEvent({ seq: i + 1, type: e[0], node_id: e[1], data: e[2], ts: t + i * 0.2 })) + }, { id: EV_RUN, rc: fxe.report_checked, output: fxe.output }) + await stamp(page).waitFor({ timeout: 4000 }).catch(() => {}) + const text = await stamp(page).innerText().catch(() => '') + check(`报告卡上盖章「引用 ${cited} · 无证据 ${none}」`, text === `引用 ${cited} · 无证据 ${none}`, text) + const title = await stamp(page).getAttribute('title').catch(() => '') + check('章的悬停说明写出可疑名字、没挂依据的结论句计入缺口', (title ?? '').includes('可能是编造的名字 2') + && (title ?? '').includes('计入缺口'), (title ?? '').replace(/\n/g, ' / ')) + check('有缺口的章用提醒色,不是绿的', (await stamp(page).getAttribute('class')).includes('is-degraded')) + check('别的节点卡上没有章', await page.locator('[data-report-stamp]').count() === 1) + + // 右栏的报告点开片段:证据路径画到画布上(LineageLayer 的 style[data-lineage]) + const doc = page.locator('[data-assistant-panel] [data-evidence-doc]') + await doc.waitFor({ timeout: 6000 }).catch(() => {}) + check('右栏的成果是逐段可点的报告', await doc.locator('[data-seg]').count() > 5) + await doc.locator('[data-seg]', { hasText: /^orders$/ }).first().click() + await page.waitForSelector('[data-assistant-panel] [data-evidence-panel="inline"]', { timeout: 4000 }).catch(() => {}) + await page.waitForTimeout(300) + const lineage = await page.evaluate(() => { + const el = document.querySelector('style[data-lineage]') + return el ? { var: el.getAttribute('data-lineage'), css: el.textContent } : null + }) + check('点开片段:画布上画出证据路径(LineageLayer 的 data-lineage 是这段证据)', lineage?.var === '证据 orders', lineage?.var ?? '没有') + check('……产出证据的查询节点实线描边、报告节点虚线描边', !!lineage && /\[data-id="fetch"\][^{]*> \.nc \{\s*box-shadow/.test(lineage.css) + && /\[data-id="write"\][^{]*> \.nc \{\s*outline: 1\.5px dashed/.test(lineage.css), (lineage?.css ?? '').slice(0, 160)) + const state = await S(page, () => window.__studio.getState().lineage) + check('……用的是 setLineage 的原形状 {var, producers, consumers}', JSON.stringify(state) === JSON.stringify({ var: '证据 orders', producers: ['fetch'], consumers: ['write'] }), + JSON.stringify(state)) + await doc.locator('[data-seg]', { hasText: '退款金额以财务确认日为准' }).first().click() + await page.waitForTimeout(300) + const quote = await S(page, () => window.__studio.getState().lineage) + check('换一段(引文):路径换成检索节点 → 报告', quote?.producers?.join(',') === 'manual' && quote?.consumers?.join(',') === 'write', JSON.stringify(quote)) + // 面板里的节点名:点一下选中并对准画布上的节点 + const chip = page.locator('[data-assistant-panel] [data-evidence-panel] button[data-ev-node="manual"]').first() + check('面板里的节点名是按钮,写画布上的节点名', (await chip.innerText().catch(() => '')).includes('查手册')) + await chip.click().catch(() => {}) + await page.waitForTimeout(250) + const sel = await S(page, () => ({ selected: window.__studio.getState().selectedId, focus: window.__studio.getState().focusRequest?.id })) + check('点节点名:选中并对准画布上的那个节点', sel.selected === 'manual' && sel.focus === 'manual', JSON.stringify(sel)) + await S(page, () => window.__studio.getState().select(null)) + await page.waitForTimeout(250) + if (SHOTS) { + await doc.locator('[data-seg]', { hasText: /^orders$/ }).first().click() + await page.waitForTimeout(300) + for (const theme of ['dark', 'light']) { + await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + await page.waitForTimeout(250) + await page.screenshot({ path: `${SHOTS}/studio-evidence-path-${theme}.png` }) + } + await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark')) + } + // 收起面板:只收回自己画的那条路径 + const back = page.locator('[data-assistant-panel] [data-evidence-panel] button', { hasText: '回到正文' }) + await back.click().catch(() => {}) + await page.waitForTimeout(250) + check('收起面板:画布上的证据路径一起收掉', await page.locator('style[data-lineage]').count() === 0 + && await S(page, () => window.__studio.getState().lineage) === null) + // 「清除」把刚才那次运行摘下来:打开画布时这张图还没跑过(取到的是空的),现在最近一次就是刚才那次—— + // 章要跟着换,不能拿打开画布时的「没跑过」当最近一次 + latestRun = EV_RUN + const refetched = await clearAndRefetch(page) + check('「清除」摘下运行:重新取这张图的最近一次运行', refetched) + check('……章取刚才那次(打开画布时还没跑过,不能照旧不画)', (await stamp(page).innerText().catch(() => '没有章')) === `引用 ${cited} · 无证据 ${none}`, + await stamp(page).innerText().catch(() => '没有章')) + check('没有运行时报错(证据路径)', errors.length === 0, errors.slice(0, 2).join(' | ')) + await page.context().close() + + // 没挂运行、这张图最近一次运行有报告核对:章从那次运行的证据图取 + let latestNow = fxe.run_id + const latest = await open({ path: '/studio/st-ev', before: routes(() => latestNow) }) + await stamp(latest.page).waitFor({ timeout: 4000 }).catch(() => {}) + check('没挂运行:章取这张图最近一次运行的核对统计(和挂着运行时一样,结论句策略也算上)', + (await stamp(latest.page).innerText().catch(() => '')) === `引用 ${cited} · 无证据 ${none}`, + await stamp(latest.page).innerText().catch(() => '没有章')) + if (SHOTS) { + // 取景到报告节点(可读的缩放),截卡片连同下沿的章,四周留一圈 + await S(latest.page, () => window.__studio.getState().focusNode('write')) + await latest.page.waitForTimeout(600) + for (const theme of ['dark', 'light']) { + await latest.page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + await latest.page.waitForTimeout(250) + const box = await latest.page.locator('.react-flow__node[data-id="write"]').boundingBox() + if (box) { + await latest.page.screenshot({ path: `${SHOTS}/studio-report-stamp-${theme}.png`, + clip: { x: box.x - 40, y: box.y - 40, width: box.width + 80, height: box.height + 80 } }).catch(() => {}) + } + } + } + // 在画布上又跑了一次(另一份报告),跑完「清除」:最近一次换成了这次,章不能还是打开画布时取的那份 + await S(latest.page, ({ id, rc, output }) => { + const s = window.__studio + const t = Date.now() / 1000 - 3 + s.setState({ run: { id, workflow_id: 'st-ev', workflow_name: '__studio_check_ev__', status: 'queued', input: {}, + output: {}, error: null, usage: {}, run_class: 'exploratory', version: null } }) + const list = [['run.started', null, {}], ['node.started', 'write', { label: '写周报', node_type: 'report' }], + ['report.checked', 'write', rc], ['node.finished', 'write', { duration_ms: 900 }], + ['run.finished', null, { output, usage: {}, timing: { wall_ms: 1000, active_ms: 1000, wait_ms: 0 } }]] + list.forEach((e, i) => s.getState().applyEvent({ seq: i + 1, type: e[0], node_id: e[1], data: e[2], ts: t + i * 0.2 })) + }, { id: fxe.dup.run_id, rc: fxe.dup.report_checked, output: fxe.dup.output }) + await latest.page.waitForTimeout(300) + check(`挂上新跑的一次:章跟着它(${dupText})`, (await stamp(latest.page).innerText().catch(() => '')) === dupText, + await stamp(latest.page).innerText().catch(() => '没有章')) + latestNow = fxe.dup.run_id + const again = await clearAndRefetch(latest.page) + check('「清除」之后重新取最近一次运行(打开画布时取的那份作废)', again) + check(`……章换成新跑的那次(${dupText}),不是打开画布时的「引用 ${cited} · 无证据 ${none}」`, + (await stamp(latest.page).innerText().catch(() => '没有章')) === dupText, await stamp(latest.page).innerText().catch(() => '没有章')) + check('没有运行时报错(最近一次运行的章)', latest.errors.length === 0, latest.errors.slice(0, 2).join(' | ')) + await latest.ctx.close() + + // 取「最近一次运行」的请求还没回来,就在画布上跑完、清除了:先发的那次(那时还没跑过)晚回来,不能把后来 + // 取到的那份盖掉。页面加载等网络静下来才算完,慢请求只能在加载后从 store 发(和报告卡发的是同一个 loadLatest) + let raceRun = null + let raceHits = 0 + const race = await open({ path: '/studio/st-ev', before: async (pg) => { + await routes(() => raceRun)(pg) + await pg.route((u) => { const x = new URL(u); return x.pathname === '/api/runs' && x.searchParams.get('workflow_id') === 'st-ev' }, + async (r) => { + if (raceHits++ !== 1) return r.fallback() + const body = raceRun ? [{ id: raceRun, workflow_id: 'st-ev', status: 'succeeded', run_class: 'exploratory' }] : [] + await new Promise((res) => setTimeout(res, 1800)) + return json(r, body).catch(() => {}) + }) + } }) + await race.page.evaluate(async () => { + const { useEvidence } = await window.__appImport('/src/store/evidence.ts') + useEvidence.getState().dropLatest('st-ev') + void useEvidence.getState().loadLatest('st-ev') + }) + await race.page.waitForTimeout(150) + await S(race.page, ({ id, rc, output }) => { + const s = window.__studio + const t = Date.now() / 1000 - 2 + s.setState({ run: { id, workflow_id: 'st-ev', workflow_name: '__studio_check_ev__', status: 'queued', input: {}, + output: {}, error: null, usage: {}, run_class: 'exploratory', version: null } }) + const list = [['run.started', null, {}], ['report.checked', 'write', rc], + ['run.finished', null, { output, usage: {}, timing: { wall_ms: 1000, active_ms: 1000, wait_ms: 0 } }]] + list.forEach((e, i) => s.getState().applyEvent({ seq: i + 1, type: e[0], node_id: e[1], data: e[2], ts: t + i * 0.2 })) + }, { id: fxe.dup.run_id, rc: fxe.dup.report_checked, output: fxe.dup.output }) + raceRun = fxe.dup.run_id + await race.page.waitForTimeout(150) + await S(race.page, () => window.__studio.getState().clearRun()) + await race.page.waitForTimeout(2600) + check(`先发的请求晚回来(那时还没跑过):章照旧是清除后取到的那次(${dupText}),没被盖成不画`, + raceHits >= 3 && (await stamp(race.page).innerText().catch(() => '没有章')) === dupText, + `${raceHits} 次请求,${await stamp(race.page).innerText().catch(() => '没有章')}`) + check('没有运行时报错(请求先后颠倒)', race.errors.length === 0, race.errors.slice(0, 2).join(' | ')) + await race.ctx.close() +}) + check('整个检查没有弹出原生 confirm / prompt', dialogs === 0, `${dialogs} 次`) check('检查没有写库(兜底拦下的写请求)', writes.length === 0, writes.slice(0, 4).join('、')) diff --git a/scripts/check-ui-kit.mjs b/scripts/check-ui-kit.mjs index 368204e5..9803d29a 100644 --- a/scripts/check-ui-kit.mjs +++ b/scripts/check-ui-kit.mjs @@ -289,9 +289,28 @@ await section('lib/evidence:证据状态的四通道元数据、报告节点 const eq = (name, got, want) => out.push([name, got === want, got === want ? '' : `得到 ${JSON.stringify(got)},应为 ${JSON.stringify(want)}`]) const ok = (name, cond, detail = '') => out.push([name, !!cond, cond ? '' : detail]) const codes = ev.EVIDENCE_STATES - // 方案 6.1 的八种:确定性、概率性四种、无证据、连接性、旧运行候选 - ok('八种状态一种不少', ['deterministic', 'supported', 'partial', 'unsupported', 'unjudged', 'none', 'connective', 'candidate'] - .every((c) => codes.includes(c)) && codes.length === 8, codes.join(',')) + // 方案 6.1 的八种(确定性、概率性四种、无证据、连接性、旧运行候选),加三期无证据的两个变体: + // 可疑实体(可能是编造的名字)、核对不了(表结构快照不全) + ok('十种状态一种不少', ['deterministic', 'supported', 'partial', 'unsupported', 'unjudged', 'none', 'connective', 'candidate', + 'suspect', 'unverified'].every((c) => codes.includes(c)) && codes.length === 10, codes.join(',')) + eq('可疑实体:点状线(同无证据)、?!、可能是编造的名字', `${ev.EVIDENCE_STATE.suspect.line}/${ev.EVIDENCE_STATE.suspect.glyph}/${ev.EVIDENCE_STATE.suspect.label}`, + 'dotted/?!/可能是编造的名字') + ok('可疑实体醒目(进 n / N),核对不了只是标注', ev.EVIDENCE_STATE.suspect.alert && !ev.EVIDENCE_STATE.unverified.alert) + eq('片段状态:issue 为 unknown_entity 的是可疑实体', ev.segmentState({ kind: 'entity', state: 'none', issue: 'unknown_entity' }), 'suspect') + eq('片段状态:[[t:编造]] 的 cite 带 unknown 也是可疑实体', ev.segmentState({ kind: 'entity', state: 'none', cite: { unknown: true } }), 'suspect') + eq('片段状态:unverified_entity 是核对不了', ev.segmentState({ kind: 'entity', state: 'none', issue: 'unverified_entity' }), 'unverified') + eq('片段状态:原话对不上的引文仍是无证据', ev.segmentState({ kind: 'quote', state: 'none', issue: 'unresolved_ref' }), 'none') + ok('实体、引文另有字形和说法,线型颜色同「有出处」', ev.EVIDENCE_KIND_STYLE.entity.line === 'solid' && ev.EVIDENCE_KIND_STYLE.quote.line === 'solid' + && ev.EVIDENCE_KIND_STYLE.quote.glyph === '“' && ev.EVIDENCE_KIND_STYLE.entity.color === ev.EVIDENCE_STATE.deterministic.color + && /^var\(--st-[\w-]+\)$/.test(ev.EVIDENCE_KIND_STYLE.quote.color)) + eq('章的两个数:引用 = 有出处的数字、值、实体、引文;无证据含可疑实体,require_citation 时加没挂依据的结论句', + JSON.stringify(ev.stampCounts({ numbers: 4, numbers_cited: 3, values: 1, entities: 5, quotes: 1, uncited_numbers: 1, unresolved: 1, + unknown_entities: 2, uncited_claims: 1 }, 'require_citation')), JSON.stringify({ cited: 10, none: 5 })) + eq('……结论句策略是 off 时不算没挂依据的结论句', ev.stampCounts({ numbers: 4, numbers_cited: 3, uncited_numbers: 1, unresolved: 0, + uncited_claims: 3 }, 'off')?.none, 1) + eq('可疑实体的提示:目录键、{name, table} 都写成去掉前缀的名字,最多 3 个', + ev.closestNames(['c:orders.amount', { alias: 't:refunds' }, { name: 'region', table: 'orders', kind: 'column' }, 'c:x']).join(','), + 'orders.amount,refunds,orders.region') const lines = new Set(['solid', 'dotted', 'badge', 'none']) const bad = codes.filter((c) => { const m = ev.EVIDENCE_STATE[c]