diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index 52afad10..1ae739d8 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -21,6 +21,9 @@ artifacts 表可以事后插行，run.output 是可以改写的一列，封存 options.mask_columns 里的列换成「已遮罩」——在有身份体系之前，这只减少暴露，不是 安全边界：知道工件 id 的人照样能取到整份快照。 +期 4 的核查状态同样只读封存范围内的东西：指标步骤带双盲复算的结果（_dual_check_of，读双盲复算节点的产出和台账 +条目），句子带反方审稿的判断（_challenge_of，读反方审稿节点的产出和审稿记录）。它们是核查，不是证据：没有就不带这两个键。 + 结论句的判定（四期）是模型给的，不是证据：正式运行的判定在报告文档里、随文档封存；探索运行 按需裁判（POST …/evidence/judge），判定作为 evidence.judged 事件追加在封存之后，只填补没判过的 句子。交给裁判的摘录同样只取封存范围内的工件。 @@ -75,7 +78,7 @@ from app.data.provenance_types import ( from app.data.table_versions import SnapshotManifestMismatch, SnapshotMissing from app.db.base import SessionLocal from app.db.models import DataSource, Run, RunEvent, Workflow -from app.engine import direct_select +from app.engine import direct_select, recompute from app.engine.evidence import ( ENTITY_KINDS, GUESS_NOTE, @@ -103,6 +106,7 @@ from app.engine.evidence import ( ) from app.engine.judge import FIELDS_MAX, SETTLED, UNSUPPORTED, VERDICTS, candidates, outdated from app.engine.merge_query import MERGE_SOURCE, trace_cell +from app.engine.schema import GraphSpec, challenge_report, dual_check_card from app.engine.toolcalls import QUERY_PREFIX router = APIRouter(prefix="/api/runs", tags=["evidence"]) @@ -726,8 +730,11 @@ async def evidence_segment(run_id: str, segment_id: str, report: str | None = No start, end = unit.get("span") or [0, 0] chain = await _chain(seg, doc, sealed, chosen.node_id) covered = sealed.trusted and all(step.get("sealed") for step in chain if "sealed" in step) - masked = list(dict.fromkeys(c for step in chain if step.get("step") == "query" for c in step.get("masked") or [])) + # 遮罩列：查询步骤里遮着的，加上指标步骤里取自遮罩列的输入（按位置取的那一格可能没有对应的查询步骤） + masked = list(dict.fromkeys(c for step in chain if step.get("step") in ("query", "input") + for c in step.get("masked") or [])) verdict = _effective_verdicts(chosen, sealed)[0].get(str(unit.get("id"))) + challenged = _challenge_of(sealed, chosen, str(unit.get("id"))) return { "report": {"node_id": chosen.node_id, "doc_artifact": chosen.doc_artifact}, "segment": {"id": seg["id"], "text": seg.get("text"), "kind": seg.get("kind"), "state": seg.get("state"), @@ -736,6 +743,8 @@ async def evidence_segment(run_id: str, segment_id: str, report: str | None = No "unit": {"id": unit.get("id"), "kind": unit.get("kind"), "text": markdown[start:end], "span": unit.get("span"), "cites": unit.get("cites") or [], **({"verdict": verdict} if verdict is not None else {}), + # 期 4：反方审稿对这一句的判断，没审到这一句的没有这个键 + **({"challenge": challenged} if challenged is not None else {}), "on_demand": _on_demand(sealed, doc, str(unit.get("id")), verdict)}, "block": {"id": block.get("id"), "type": block.get("type")}, "chain": chain, @@ -855,10 +864,15 @@ async def _metric_chain(seg: dict[str, Any], cite: dict[str, Any], entry: dict[s "source": await _caliber_source((card or {}).get("source")), "caliber_upgrade": _caliber_upgrade(sealed.upgrades.get(str(entry.get("node_id") or ""))), } + # 期 4：这个指标做没做过双盲复算、结果如何。没有双盲复算复核过这份指标集时没有这个键，老运行一字不差 + if checked := _dual_check_of(sealed, str(entry.get("node_id") or ""), artifact, metric_id, metric): + step["dual_check"] = checked catalog = doc.get("catalog") or {} inputs: list[dict[str, Any]] = [] #: 输入指到的查询快照 → 被引用的格；同一份快照只给一个查询步骤，格合在一起高亮 wanted: dict[str, list[tuple[Any, Any]]] = {} + hide = _InputMasks(sealed, masks, str(entry.get("node_id") or ""), artifact) + withheld: set[str] = set() for item in (metric or {}).get("inputs") or []: one = {"step": "input", **item} artifact_of = item.get("artifact") @@ -867,11 +881,125 @@ async def _metric_chain(seg: dict[str, Any], cite: dict[str, Any], entry: dict[s if cell := _input_cell(item, catalog): one["cell"] = cell wanted.setdefault(str(artifact_of), []).extend(_located(item.get("locator") or {})) + if (hidden := await hide.check(item)) is not None: + one = _withheld(one, *hidden) + withheld.add(str(item.get("path"))) inputs.append(one) + if withheld: + # 代入式里那几处同样写「已遮罩」：不然标签里遮着、代入式里亮着，同一个值换个地方照样看得到 + step["substituted"] = recompute.substituted_without(metric or {}, withheld, MASKED) queries = [s for a, cells in wanted.items() for s in await _query_steps(a, cells, doc, sealed, masks)] return [step, *inputs, *queries] +def _withheld(step: dict[str, Any], columns: list[str], unverified: str | None) -> dict[str, Any]: + """不给值的输入：值写「已遮罩」（本来就没有值的照旧是空的，缺输入还得看得出来），模型报的值、快照里的值一并拿掉。 + 取自遮罩列的，masked 写明是哪几列；遮没遮核对不了的（_InputMasks），mask_unverified 写原因、不进 masked—— + 面板底下那句「这些列设置了遮罩」不能把它们也算进去。""" + out = {k: v for k, v in step.items() if k not in ("model_value", "snapshot_value")} + if out.get("value") is not None: + out["value"] = MASKED + if unverified is None: + out["masked"] = columns + else: + out["mask_unverified"] = unverified + return out + + +#: tool_cell、Agent 字段落在的查询快照读不出来：只知道数据源现在设的遮罩，查询当时记下的在快照里，遮没遮核对不了 +SNAPSHOT_UNVERIFIED = "查询快照无法读取，核对不了这一格是否取自设置了遮罩的列" + + +class _InputMasks: + """一个指标的输入里哪些不能给值。遮的列和查询步骤同一套（_query_step）：查询当时记下的加数据源现在设的，合并查询按它 + 合并的那几个数据源算。 + + 取自哪份查询结果的哪几列按 recompute.input_origin 认，推演、可验证报告是同一个认法：cell() 的格、Agent 交来的字段按 + 定位里的列认；vars.res.rows[0][2] 这种按位置取的一格卡上没记定位，顺着取值链认，从写变量的节点在口径卡算这一次之前 + 的产出起走（_card_outputs）。 + + 遮没遮核对不了的宁可多遮，和推演一致：快照读不出来（数据源现在没遮这一列，可查询当时记下的遮罩在快照里）、按位置取的 + 一格取值链上的产出读不出来、那份结果没存快照只带着数据源名。认得出、可结果既没带快照的 id 也没带数据源名的（数据整形 + 改过形状）不遮：认不出出处，卡上本来就只记着路径和值。 + + 一个指标里同一件工件只读一次（_fetch 记在 _read）：按位置取了十几格的指标，不必把写变量的节点那几 MB 的产出重读 + 十几遍。查询步骤（_query_step）另读一遍快照：它的接口好几条链都在用，读两遍比读十几遍是小事。 + """ + + def __init__(self, sealed: _Sealed, masks: _Masks, card_node: str, artifact: str) -> None: + self._sealed, self._masks = sealed, masks + self._card, self._artifact = card_node, artifact + self._outputs: dict[str, str] | None = None + self._memo: dict[tuple[str | None, str | None], set[str]] = {} + self._read: dict[str, tuple[Any, bool | None]] = {} + + def _fetch(self, artifact: str) -> tuple[Any, bool | None]: + if artifact not in self._read: + self._read[artifact] = _fetch(artifact) + return self._read[artifact] + + async def check(self, item: dict[str, Any]) -> tuple[list[str], str | None] | None: + """None：给值。(遮罩列, None)：取自这几列遮罩列（列名照原样，按出现的先后）。(用到的列, 原因)：遮没遮核对不了， + 原因给人看（认不出是哪一列时列为空）。""" + origin = recompute.input_origin(item, self._output) + if origin is None: + return None + if origin.unverified is not None: + return list(origin.columns), origin.unverified + used = list(dict.fromkeys(origin.columns)) + if not used: + return None + snap, ok = self._fetch(origin.artifact) if origin.artifact else (None, None) + readable = ok is True and isinstance(snap, dict) + hidden = await self._hidden(origin.artifact, origin.source, snap if readable else None) + if masked := [c for c in used if c.lower() in hidden]: + return masked, None + if origin.artifact and not readable: + return used, SNAPSHOT_UNVERIFIED + if not origin.artifact and origin.source: + return used, recompute.POSITION_NO_SNAPSHOT + return None + + def _output(self, node_id: Any) -> Any: + if self._outputs is None: + self._outputs = _card_outputs(self._sealed, self._card, self._artifact) + artifact = self._outputs.get(str(node_id or "")) + content, ok = self._fetch(artifact) if artifact else (None, None) + return content if ok is True else None + + async def _hidden(self, artifact: str | None, source: str | None, snap: dict[str, Any] | None) -> set[str]: + key = (artifact, source) + if key not in self._memo: + if snap is not None and snap.get("source") == MERGE_SOURCE: + hidden, _ = await _merge_masks(snap, self._masks) + else: + info = self._sealed.queries.get(artifact) if artifact else None + current, _ = await self._masks.of((info or {}).get("source") or source or (snap or {}).get("source")) + hidden = set(current) | (_recorded_masks(snap) if snap is not None else set()) + self._memo[key] = hidden + return self._memo[key] + + +def _card_outputs(sealed: _Sealed, card_node: str, artifact: str) -> dict[str, str]: + """口径卡算出这份指标集的那一次执行开始之前，每个节点最后一次的产出工件：卡上 vars.*、nodes.* 取到的就是它们。 + 认不出是哪一次执行（台账里没有这份指标集）时，取封存范围内每个节点最后一次的产出。""" + finished = next((seq for seq, etype, node_id, data in sealed.events + if etype == "node.finished" and node_id == card_node and isinstance(data, dict) + and any(isinstance(e, dict) and e.get("kind") == "metric_set" and e.get("artifact") == artifact + for e in data.get("evidence") or [])), None) + if finished is None: + return dict(sealed.outputs) + started = max((seq for seq, etype, node_id, _ in sealed.events + if etype == "node.started" and node_id == card_node and seq < finished), default=finished) + out: dict[str, str] = {} + for seq, etype, node_id, data in sealed.events: + if seq >= started: + break + if etype == "node.finished" and node_id and isinstance(data, dict) and data.get("artifact"): + out[node_id] = str(data["artifact"]) + return out + + def _located(locator: dict[str, Any]) -> list[tuple[Any, Any]]: """输入的定位 → 被引用的格。单个格 {row, column}；数组字段是一段行 {rows:[a,b], column | columns}。""" if isinstance(locator.get("row"), int): @@ -899,6 +1027,224 @@ def _caliber_upgrade(event: dict[str, Any] | None) -> dict[str, Any] | None: return {k: event.get(k) for k in ("workflow_id", "caliber_node", "pinned", "latest", "policy", "policy_label")} +#: 双盲复算节点产出里一个指标的结果（engine/nodes/dual_check.py 的 SAME / DIFFERENT / UNCHECKED），原样交给界面 +_DUAL_RESULTS = ("agreed", "disagreed", "unchecked") + + +def _spec_of(sealed: _Sealed) -> GraphSpec | None: + """这次运行的图（认双盲复算复算哪张卡、反方审稿审哪份报告，和执行时同一个认法）。图读不懂的是 None。""" + try: + return GraphSpec.model_validate(sealed.graph) + except Exception: # noqa: BLE001 - 图读不懂只是认不出来，证据照常给 + return None + + +def _dual_check_of(sealed: _Sealed, card_node: str, artifact: str, metric_id: Any, + metric: dict[str, Any] | None = None) -> dict[str, Any] | None: + """这个指标的双盲复算状态：{status, node_id, choice?, blind_rendered?, reason?}。没有双盲复算复核过这份指标集的是 None。 + metric 是卡上的这个指标（认它为什么没被复算用），读不到卡时为 None。 + + 认的是复核了口径卡的哪一次结果：台账条目（kind dual_check）记着它复算的指标集工件，和指标步骤的工件对上才算——口径卡 + 在循环里算了几次时，报告引用的那一次才是。直通了的（设置里关了、口径卡上没有来自查询的输入）没有台账条目：按画布上 + 它复算的那张卡、和它跑完之前那张卡最后一次的指标集认。几个双盲复算复核了同一份指标集时取最后跑完的那个。 + + - agreed / disagreed / unchecked：节点产出里这个指标的结果。不一致的带审核人的选择（a / b / neither，没选的不带）； + 一致、不一致的带盲算值（渲染好的）；无法复算的带原因 + - skipped：节点没复算这个指标，reason 说为什么（_not_checked）：关闭了（disabled，节点直通）；不在节点配置的 + 「复算的指标」里、或者值不是数（not_targeted）；这个指标没有一个输入来自查询快照（no_inputs）。节点直通时产出里的 + no_inputs 说的是整张卡，不原样套到每个指标上：卡上确有来自查询的指标、只是没被选上的，写它的说法是 not_targeted + 产出读不出来（文件不在、与哈希不一致）时按台账条目说状态和选择，不带盲算值和原因。 + """ + if not card_node or not artifact or metric_id is None: + return None + checkers = set(sealed.nodes_of("dual_check")) + if not checkers: + return None + spec: GraphSpec | None = None + spec_read = False + #: 口径卡节点 → 到这一刻为止它最后一次的指标集 + latest: dict[str, str] = {} + found: dict[str, Any] | None = None + for _, etype, node_id, data in sealed.events: + if etype != "node.finished" or not node_id or not isinstance(data, dict): + continue + ledger = [e for e in data.get("evidence") or [] if isinstance(e, dict)] + for e in ledger: + if e.get("kind") == "metric_set" and e.get("artifact"): + latest[node_id] = str(e["artifact"]) + if node_id not in checkers: + continue + output, ok = _fetch(str(data.get("artifact") or "")) + output = output if ok is True and isinstance(output, dict) else {} + entry = next((e for e in ledger if e.get("kind") == "dual_check"), None) + if not spec_read: + spec, spec_read = _spec_of(sealed), True + node = spec.node_map().get(node_id) if spec else None + if entry is not None: + card, checked = str(entry.get("card") or ""), str(entry.get("metric_set") or "") + elif output.get("status") == "skipped": + card = (dual_check_card(spec, node)[0] or "") if spec and node else "" + checked = latest.get(card, "") + else: + continue + if card != card_node or checked != artifact: + continue + found = _dual_status(node_id, output, entry, str(metric_id), metric, _picked(node)) + return found + + +def _picked(node: Any) -> list[str] | None: + """画布上这个双盲复算节点配的「复算的指标」，认法同 dual_check._options：没配、配了空列表都是 None（复算全部）。""" + wanted = node.config.get("metrics") if node is not None else None + picked = [str(m) for m in wanted if isinstance(m, str) and m.strip()] if isinstance(wanted, list) else [] + return picked or None + + +def _query_input(item: Any) -> bool: + """来自查询快照的输入（同 dual_check._from_query）：双盲复算只复算有这种输入的指标。""" + return isinstance(item, dict) and item.get("via") in ("tool_cell", "agent_field") \ + and isinstance(item.get("artifact"), str) and bool(item["artifact"]) + + +def _not_checked(node_reason: str, metric: dict[str, Any] | None, picked: list[str] | None, metric_id: str) -> str: + """节点没复算这个指标的原因。关闭了（disabled）照节点的说；别的按这个指标自己说：不在配置的「复算的指标」里是 + not_targeted；卡上读得到它、它没有一个输入来自查询快照是 no_inputs，有（那就是值不是数）是 not_targeted。读不到卡时 + 照节点直通的原因说，节点跑了的说 not_targeted。""" + if node_reason == "disabled": + return node_reason + if picked is not None and metric_id not in picked: + return "not_targeted" + if metric is not None: + return "not_targeted" if any(_query_input(i) for i in metric.get("inputs") or []) else "no_inputs" + return node_reason or "not_targeted" + + +def _dual_status(node_id: str, output: dict[str, Any], entry: dict[str, Any] | None, metric_id: str, + metric: dict[str, Any] | None, picked: list[str] | None) -> dict[str, Any]: + """一次双盲复算里这个指标的状态（_dual_check_of 的答复）。""" + if output.get("status") == "skipped": + reason = _not_checked(str(output.get("reason") or ""), metric, picked, metric_id) + return {"status": "skipped", "node_id": node_id, "reason": reason} + row = next((m for m in output.get("metrics") or [] if isinstance(m, dict) and str(m.get("metric_id")) == metric_id), + None) + if row is None and not output and entry is not None: + # 产出读不出来：退回台账条目，只有状态和选择 + brief = next((m for m in entry.get("metrics") or [] if isinstance(m, dict) and str(m.get("id")) == metric_id), + None) + row = {"status": brief.get("status"), **({"choice": brief["choice"]} if "choice" in brief else {})} \ + if brief is not None else None + if row is None: + return {"status": "skipped", "node_id": node_id, "reason": _not_checked("", metric, picked, metric_id)} + status = row.get("status") if row.get("status") in _DUAL_RESULTS else "unchecked" + out: dict[str, Any] = {"status": status, "node_id": node_id} + if status == "disagreed" and row.get("choice") in ("a", "b", "neither"): + out["choice"] = row["choice"] + rendered = row.get("blind_rendered") + if status != "unchecked" and isinstance(rendered, str) and rendered: + out["blind_rendered"] = rendered + if status == "unchecked" and isinstance(row.get("reason"), str) and row["reason"]: + out["reason"] = row["reason"] + return out + + +#: 反方审稿的四种判断（engine/nodes/challenge.py 的 VERDICTS）：审稿概况的计数照这个顺序 +_CHALLENGE_VERDICTS = ("stands", "weakened", "refuted", "inconclusive") + + +def _challenge_of(sealed: _Sealed, report: _Report, unit_id: str) -> dict[str, Any] | None: + """这一句的反方审稿判断：{verdict, node_id, artifact, on_refuted, report_label, counts, unreviewed?}。没审到这一句的 + 是 None。 + + 只认审的就是正文这一份报告文档的：审稿记录（challenge_artifact）里记着它审的报告节点和文档工件，和这份对上才算—— + 报告在审稿之后又重写了（循环、接着跑），同一个句子编号在新的一稿里说的可能是另一句话。记录没存上（产出里的 + challenge_artifact 为空）或读不出来时，按画布上它审的报告、和它跑完之前那份报告最后一次自查记下的文档认，判断取 + 产出里列的那几句（最多 10 句，被推翻、被削弱的排在前面），artifact 照产出写（没存上的为空）。几个反方审稿审了同一 + 份文档时取最后跑完、审到了这一句的那个。on_refuted 是实际生效的处置，report_label 是报告节点的名字。 + + counts 是这次审稿四种判断各几句——整份报告的，取节点产出里的那一份；unreviewed 是没能反查的句数，产出里有才带。 + 证据面板借审批卡上的展示画这一句，那一块的概况得是这份报告真实的数，不能凭这一句拼出一个「被推翻 0 句」。 + + 节点设为「报错并停止运行」、又有句子被推翻时，节点报错、没有 node.finished：审完那一刻的 challenge_done 日志记着 + 审稿记录和计数，按它认（只认审稿记录存上了的），on_refuted 为 fail。运行因此失败了，被推翻的是哪几句、反方凭什么， + 证据面板里照样看得到。 + """ + reviewers = set(sealed.nodes_of("challenge")) + if not reviewers or not unit_id: + return None + spec: GraphSpec | None = None + #: 报告节点 → 到这一刻为止它最后一次自查记下的文档 + drafts: dict[str, str] = {} + #: 反方审稿节点 → 这一次执行里审完时的日志：节点报错时它就是审稿结果 + done: dict[str, dict[str, Any]] = {} + found: dict[str, Any] | None = None + for _, etype, node_id, data in sealed.events: + if not node_id or not isinstance(data, dict): + continue + if etype == "report.checked" and data.get("doc_artifact"): + drafts[node_id] = str(data["doc_artifact"]) + continue + if node_id not in reviewers: + continue + if etype == "node.started": + done.pop(node_id, None) + continue + if etype == "log" and data.get("code") == "challenge_done": + done[node_id] = data + continue + if etype == "node.finished": + done.pop(node_id, None) + output, ok = _fetch(str(data.get("artifact") or "")) + if ok is not True or not isinstance(output, dict) or not isinstance(output.get("counts"), dict): + continue # 直通了（关闭、没有可审的句子）或者产出读不出来 + elif etype == "node.failed" and node_id in done: + log = done.pop(node_id) + if not isinstance(log.get("artifact"), str) or not log["artifact"]: + continue + output = {"challenge_artifact": log["artifact"], "counts": log.get("counts"), "on_refuted": "fail"} + else: + continue + artifact = output.get("challenge_artifact") if isinstance(output.get("challenge_artifact"), str) else None + record, saved = _fetch(artifact) if artifact else (None, None) + if saved is True and isinstance(record, dict): + if record.get("report") != report.node_id or record.get("doc_artifact") != report.doc_artifact: + continue + claims = record.get("claims") + elif etype == "node.failed": + continue # 节点报错时没有产出可退：认不出审的是哪一稿 + else: + spec = spec or _spec_of(sealed) + node = spec.node_map().get(node_id) if spec else None + target = challenge_report(spec, node)[0] if spec and node else None + if target != report.node_id or drafts.get(target) != report.doc_artifact: + continue + claims = output.get("items") + claims = [c for c in claims or [] if isinstance(c, dict)] + verdict = next((c.get("verdict") for c in claims if c.get("unit") == unit_id), None) + if not isinstance(verdict, str) or not verdict: + continue + spec = spec or _spec_of(sealed) + writer = spec.node_map().get(report.node_id) if spec else None + unreviewed = output.get("unreviewed") + found = {"verdict": verdict, "node_id": node_id, "artifact": artifact, + "on_refuted": str(output.get("on_refuted") or "degrade"), + "report_label": writer.title if writer else report.node_id, + "counts": _challenge_counts(output.get("counts"), claims), + **({"unreviewed": unreviewed} if isinstance(unreviewed, int) and not isinstance(unreviewed, bool) + and unreviewed >= 0 else {})} + return found + + +def _challenge_counts(raw: Any, claims: list[dict[str, Any]]) -> dict[str, int]: + """四种判断各几句：取节点产出（或审完时的日志）里的计数；一个数都认不出来时按认得的句子数（审稿记录里的是全部， + 记录没存上时只有产出里列的那几句）。""" + given = raw if isinstance(raw, dict) else {} + known = {v: given[v] for v in _CHALLENGE_VERDICTS + if isinstance(given.get(v), int) and not isinstance(given[v], bool) and given[v] >= 0} + if known: + return {v: known.get(v, 0) for v in _CHALLENGE_VERDICTS} + return {v: sum(1 for c in claims if c.get("verdict") == v) for v in _CHALLENGE_VERDICTS} + + class _Masks: """一次请求里查过的数据源遮罩：数据源名 → (要遮的列（小写）, 数据源现在还在不在)。""" diff --git a/backend/app/bundle/build.py b/backend/app/bundle/build.py index 5886bd22..90fca722 100644 --- a/backend/app/bundle/build.py +++ b/backend/app/bundle/build.py @@ -8,7 +8,9 @@ 浏览器先核它的哈希，再从里面读出报告文档、口径卡、查询快照的 id。其余事件只给哈希：工具返回、节点产出的 预览里可能有遮罩列的原值和提示词。 - 工件：每份报告文档（各报告节点最后一次自查记下的 doc_artifact）、它引用到的口径卡和查询快照，按原字节 - base64 嵌入，浏览器重算 sha256 和 id 比；带遮罩的快照不嵌内容、只给 id。没被报告引用的工件不嵌。 + base64 嵌入，浏览器重算 sha256 和 id 比；带遮罩的快照不嵌内容、只给 id。口径卡上记着每个输入的值和代入式， + 有输入取自遮罩列（cell() 的格、Agent 字段、按位置取的 vars.res.rows[0][2]）的整张卡同样只给 id，认法和证据 + 面板、推演同一个（recompute.input_origin）。没被报告引用的工件不嵌。 - 封存之后追加的事件（按需裁判、审批质询这类批注）单列：类型、序号、内容摘要，不进清单。 它只证明「内部一致」：数字、文档、运行记录彼此对得上，而且没被改过。真伪要拿页面上的封存哈希和系统里这次 @@ -34,6 +36,7 @@ from app.core.artifact_store import canonical_json, content_hash from app.core.events import EventType from app.engine.evidence import iter_units, query_entry_fields from app.engine.merge_query import MERGE_SOURCE +from app.engine.recompute import input_origin from app.engine.schema import NodeType from app.engine.toolcalls import QUERY_PREFIX @@ -106,6 +109,44 @@ class _Store: self._masked[artifact] = _snapshot_masked(self.content(artifact), self._source_masks) return self._masked[artifact] + def card_masked(self, artifact: str, events: Sequence[tuple[int, str, str | None, Any]]) -> bool: + """这张口径卡算不算带遮罩：卡上有一个输入取自遮罩列，或者遮没遮核对不了。卡上记着每个输入的值（cell() 取的格、 + Agent 字段、按位置取的 vars.res.rows[0][2]）和代入式，整张卡不随文件附带——没法只拿掉几个值，拿掉了哈希就对不上。 + + 认哪个输入取自哪份查询结果的哪几列，和证据面板、推演同一个认法（recompute.input_origin）；按位置取的一格顺着 + 取值链从写变量的节点在口径卡算这一次之前的产出起走。核对不了的宁可多挡：快照读不出来、取值链上的产出读不出来、 + 那份结果没存快照只带着数据源名、定位里没写列。认得出、可结果既没带快照的 id 也没带数据源名的（数据整形改过形状） + 认不出出处，不算。读不出来的卡本来就不嵌，这里不管。""" + key = f"card:{artifact}" + if key not in self._masked: + card = self.content(artifact) if self.intact(artifact) else None + outputs = _card_outputs(events, artifact) if isinstance(card, dict) else {} + + def output(node_id: Any) -> Any: + found = outputs.get(str(node_id or "")) + return self.content(found) if found and self.intact(found) else None + + items = [item for metric in (card or {}).get("metrics") or [] if isinstance(metric, dict) + for item in metric.get("inputs") or [] if isinstance(item, dict)] + self._masked[key] = any(self._input_masked(item, output) for item in items) + return self._masked[key] + + def _input_masked(self, item: dict[str, Any], output: Callable[[Any], Any]) -> bool: + origin = input_origin(item, output) + if origin is None: + return False + if origin.unverified is not None: + return True + if origin.artifact is None: + return origin.source is not None # 没存快照：查询当时记下的遮罩无从核对 + content = self.content(origin.artifact) + if not self.intact(origin.artifact) or not isinstance(content, dict): + return True + if not origin.columns: + return self.masked(origin.artifact) + hidden = _snapshot_hidden(content, self._source_masks) + return any(c.lower() in hidden for c in origin.columns) + def _snapshot_masked(content: Any, source_masks: Mapping[str, Any]) -> bool: if not isinstance(content, dict): @@ -117,6 +158,37 @@ def _snapshot_masked(content: Any, source_masks: Mapping[str, Any]) -> bool: return any(source_masks.get(str(name or "")) for name in sources or []) +def _snapshot_hidden(content: dict[str, Any], source_masks: Mapping[str, Any]) -> set[str]: + """一份读得出来的快照要遮的列（小写）：查询当时记下的，加上它的数据源（合并查询看它合并的那几个）现在设的。""" + recorded = content.get("mask_columns") + hidden = {c.lower() for c in recorded if isinstance(c, str)} if isinstance(recorded, list) else set() + sources = content.get("sources") if content.get("source") == MERGE_SOURCE else [content.get("source")] + for name in sources if isinstance(sources, list) else []: + now = source_masks.get(str(name or "")) + hidden |= {str(c).lower() for c in now} if isinstance(now, (list, tuple, set, frozenset)) else set() + return hidden + + +def _card_outputs(events: Sequence[tuple[int, str, str | None, Any]], card: str) -> dict[str, str]: + """算出这份指标集的那一次口径卡执行开始之前，每个节点最后一次的产出工件：卡上 vars.*、nodes.* 取到的就是它们 + （和证据接口 api/evidence._card_outputs 同一个认法）。认不出是哪一次执行时取封存范围内每个节点最后一次的产出。""" + found = next(((seq, node_id) for seq, etype, node_id, data in events + if etype == EventType.NODE_FINISHED and isinstance(data, dict) + and any(isinstance(e, dict) and e.get("kind") == "metric_set" and e.get("artifact") == card + for e in data.get("evidence") or [])), None) + bound = None + if found is not None: + bound = max((seq for seq, etype, node_id, _ in events + if etype == EventType.NODE_STARTED and node_id == found[1] and seq < found[0]), default=found[0]) + out: dict[str, str] = {} + for seq, etype, node_id, data in events: + if bound is not None and seq >= bound: + break + if etype == EventType.NODE_FINISHED and node_id and isinstance(data, dict) and data.get("artifact"): + out[node_id] = str(data["artifact"]) + return out + + def _query_artifacts(events: Iterable[tuple[int, str, str | None, Any]]) -> list[str]: """封存范围内交回过的查询快照：tool.end / merge.end 的 query_artifact，节点台账里的 query 条目。""" out: list[str] = [] @@ -234,10 +306,12 @@ def _cited_artifacts(doc: dict[str, Any]) -> list[tuple[str, str, str | None]]: class _Artifacts: - """嵌进页面的工件清单，按先后：报告文档在前，接着是它引用到的。同一份只列一次。""" + """嵌进页面的工件清单，按先后：报告文档在前，接着是它引用到的。同一份只列一次。events 是封存范围内的事件 + （认口径卡带不带遮罩时找写变量的节点当时的产出）。""" - def __init__(self, store: _Store) -> None: + def __init__(self, store: _Store, events: Sequence[tuple[int, str, str | None, Any]] = ()) -> None: self.store = store + self.events = events self.items: list[dict[str, Any]] = [] self._seen: set[str] = set() @@ -252,7 +326,8 @@ class _Artifacts: if raw is None: # 服务端的文件不在了：只给 id，浏览器照实说核对不了 item["missing"] = True - elif kind == "query" and self.store.masked(artifact): + elif (kind == "query" and self.store.masked(artifact)) \ + or (kind == "metric_set" and self.store.card_masked(artifact, self.events)): item["masked"] = True else: # 原样嵌入，不在这里复验哈希：被改过的文件照实交出去，浏览器自己算出对不上 @@ -380,7 +455,7 @@ def build_bundle( later = [e for e in events if e[0] > manifest_seq] store = _Store(read, source_masks or {}) - artifacts = _Artifacts(store) + artifacts = _Artifacts(store, sealed) reports = _report_events(sealed) cited_queries: list[str] = [] for report in reports: diff --git a/backend/app/bundle/verifier.js b/backend/app/bundle/verifier.js index a2bb6e42..8184979f 100644 --- a/backend/app/bundle/verifier.js +++ b/backend/app/bundle/verifier.js @@ -106,7 +106,7 @@ artifactUndecodable: '内容无法解码', artifactUnparsable: '哈希一致，但内容无法解析', masked: '含遮罩列，未随文件附带，只能核对到哈希', - maskedOdd: '只有查询快照会因遮罩不随文件附带，这份工件却标着遮罩：导出时不会出现这种情况，文件可能被改过', + maskedOdd: '只有查询快照和口径卡会因遮罩不随文件附带，这份工件却标着遮罩：导出时不会出现这种情况，文件可能被改过', artifactMissing: '导出时服务端已没有这份工件的文件', artifactAbsent: '内容未随文件附带：导出时不会出现这种情况，文件可能被改过', chainDoc: (label) => `报告「${label}」的文档`, @@ -148,6 +148,8 @@ statusMismatch: (claim, seq, what) => `文件里写的是「${claim}」，与封存范围内最后一条结束类事件（第 ${seq} 条，${what}）不符`, statusLegacy: '早期运行在结束事件落库之前封存，结束事件不在封存范围内，运行结果无法核对', statusNoEvent: '封存范围内没有运行结束、失败或取消的事件，运行结果无法核对', + statusContinued: (claim, seq, what) => `文件里写的是「${claim}」，但封存范围内第 ${seq} 条事件（${what}）之后运行又继续执行，` + + '它不是运行的结束，范围内也没有更晚的结束事件可以印证', classOk: (seq) => `与第 ${seq} 条事件（运行开始）记下的运行参数一致`, classMismatch: (claim, seq) => `文件里写的是「${claim}」，与第 ${seq} 条事件（运行开始）记下的运行参数不符`, classLegacy: (seq) => `第 ${seq} 条事件（运行开始）出自早期运行，没有记下区分运行类别的参数，运行类别无法核对`, @@ -483,7 +485,8 @@ } } } else if (a.masked) { - if (a.kind === 'query') { + // 查询快照带遮罩列；口径卡上有输入取自遮罩列（卡上记着输入的值和代入式）。两种都只给 id，报告文档不会 + if (a.kind === 'query' || a.kind === 'metric_set') { item.reason = T.masked } else { item.result = FAIL @@ -651,20 +654,24 @@ const label = (map, value) => map[value] || (value ? String(value) : '—') const what = (ev) => EVENT_LABEL[ev.type] || ev.type - // 运行结果：封存范围内最后一条结束类事件的类型。早期运行在 run.finished 落库之前封存，成功结束的那条在范围外， - // 范围内最后一条也就不是结束类事件；失败、取消的那条在范围内 + // 运行结果：按封存范围内真正截止的那条结束类事件核对——它之后再没有更晚的事件。新运行在结束事件落库之后马上封存， + // 它就是范围内的最后一条；早期运行在 run.finished 落库之前封存，成功结束的那条在范围外，失败、取消的那条在范围的最后。 + // 先失败、经继续运行接着跑的，范围里那条运行失败之后运行又继续了，它不是运行的结束：拿它去对，元信息改成「已失败」 + // 也会被标成已核对。真正截止的那条不在范围里时，只有早期运行成功结束说得通（无法核对），别的说法都没有事件印证 const statusClaim = label(STATUS_LABEL, meta.status) - const end = [...order].reverse().find((ev) => TERMINAL[ev.type]) - const lastEv = order[order.length - 1] + const lastSeq = order.length ? order[order.length - 1].seq : null + const final = [...order].reverse().find((ev) => TERMINAL[ev.type]) + // 同一序号的几条（老数据里有）都算最后：结束事件和别的事件撞了序号，不能因此说它之后运行又继续了 + const end = final && final.seq === lastSeq ? final : null if (!sealOk) add('status', statusClaim, statusClaim, NA, T.metaNeedsManifest) - else if (meta.legacy && meta.status === 'succeeded' && !(lastEv && TERMINAL[lastEv.type])) { - add('status', statusClaim, statusClaim, NA, T.statusLegacy) - } else if (!end) add('status', statusClaim, statusClaim, NA, T.statusNoEvent) - else { + else if (end) { const ok = TERMINAL[end.type] === meta.status add('status', label(STATUS_LABEL, TERMINAL[end.type]), statusClaim, ok ? PASS : FAIL, ok ? T.statusOk(end.seq, what(end)) : T.statusMismatch(statusClaim, end.seq, what(end))) - } + } else if (meta.legacy && meta.status === 'succeeded') add('status', statusClaim, statusClaim, NA, T.statusLegacy) + // 显示的值写「—」：范围里没有说明运行怎样结束的事件，不能把文件里写的当成核对出来的 + else if (final) add('status', '—', statusClaim, FAIL, T.statusContinued(statusClaim, final.seq, what(final))) + else add('status', statusClaim, statusClaim, NA, T.statusNoEvent) // 运行类别：运行开始事件记下的工具信任档。正式运行记 formal，探索运行记各工具的档位（一个对象），升级前的运行没有 const classClaim = label(CLASS_LABEL, meta.run_class) diff --git a/backend/app/engine/nodes/metrics.py b/backend/app/engine/nodes/metrics.py index 096aeed8..67d9effc 100644 --- a/backend/app/engine/nodes/metrics.py +++ b/backend/app/engine/nodes/metrics.py @@ -128,8 +128,7 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]: errors.append(f"指标「{metric_id}」：小数位（decimals）需要填写整数，当前为「{definition.get('decimals')}」") continue decimals = display["decimals"] - if isinstance(value, float) and decimals is not None: - value = round(value, decimals) + value = round_metric(value, decimals) substituted = substitute(tree, values) incomplete: dict[str, Any] = {} if use.sources: @@ -334,6 +333,17 @@ def _integer(raw: Any) -> int | None: return None +def round_metric(value: Any, decimals: Any) -> Any: + """口径卡的取整：只取整浮点，按小数位 round（Python 的 round，负数取整到十位、百位）；整数、文本、空值原样， + 小数位没写或不是整数（老卡上记着的 true、2.0 这类怪值）时也原样。 + + 推演重算（engine/recompute.py）、双盲复算的盲值都按卡重算，取整规则只能有这一处：两边各写一行，口径卡哪天换了 + 取整方式，推演就会对着同一张卡算出另一个数。""" + if isinstance(value, float) and isinstance(decimals, int) and not isinstance(decimals, bool): + return round(value, decimals) + return value + + def _rendered(value: Any, definition: dict[str, Any], display: dict[str, Any]) -> str: try: return render_number(value, unit=str(definition.get("unit") or ""), @@ -726,8 +736,7 @@ def _recompute(substituted: str, tctx: dict[str, Any], value: Any, decimals: int again = eval_expression(substituted, tctx) except Exception: # noqa: BLE001 - 代入式算不出来本身就是「对不上」 return False - if isinstance(again, float) and decimals is not None: - again = round(again, decimals) + again = round_metric(again, decimals) if is_number(again) and is_number(value): return math.isclose(again, value, rel_tol=1e-12, abs_tol=0.0) or again == value return again == value diff --git a/backend/app/engine/recompute.py b/backend/app/engine/recompute.py index 0f3a0f76..fa3b86d9 100644 --- a/backend/app/engine/recompute.py +++ b/backend/app/engine/recompute.py @@ -3,9 +3,9 @@ 推演永远不是出具。这里只算数：不读库、不写工件、不追加事件。接口（api/whatif.py）把这次运行封存范围内取得到的 工件交进来（Sources），结果原样交回给点开证据面板的人，运行记录和报告一个字都不动。 -口径卡怎么算的，这里就怎么算：同一条原式、同一个求值器（expressions.eval_expression），浮点结果按卡上记的小数位 -round（和 metrics.run_metrics 一样只 round 浮点），渲染走 evidence.render_metric。重算前先把每个输入当时的值还原 -出来，卡上记着的就用卡上的： +口径卡怎么算的，这里就怎么算：同一条原式、同一个求值器（expressions.eval_expression），取整用口径卡的同一个函数 +（metrics.round_metric，只取整浮点），渲染走 evidence.render_metric。重算前先把每个输入当时的值还原出来，卡上记着的 +就用卡上的： - 标量输入（数、文本、布尔、空值）：卡上记的 value 就是当时代进去的值。cell() 取的格、agent 的单格字段和数组里的 一个元素也是标量，它们落在查询快照的哪一行记在 locator 里：排除了那一行，这个输入就当缺失，指标标「排除后无法计算」 @@ -14,7 +14,8 @@ round（和 metrics.run_metrics 一样只 round 浮点），渲染走 evidence.r - nodes.* / input.*：那个节点在口径卡执行之前最后一次的产出工件 - vars.*：有 value_artifact（口径卡当时另存的完整值）才还原得出来；老运行照实说当时没有保存 - 容器本身是一份查询结果（带 artifact 的 {columns, rows}，或者和某份快照的 rows 一模一样的列表）：排除行就去掉那几行 -- 遮罩列取来的输入不给值、也不拿来推演：改一个输入、排除一行，新值和原值一比就能算出遮住的那一格 +- 遮罩列取来的输入不给值、也不拿来推演：改一个输入、排除一行，新值和原值一比就能算出遮住的那一格。vars.res.rows[0][2] + 这种按位置取自查询结果的标量没有定位，顺着取值链认出是哪一列（positional_cell，证据面板的指标步骤用同一个认法） - 落在读不出来（文件不在、与哈希不一致）的查询快照上的输入同样不给值、不推演：那一格遮没遮核对不了，宁可多遮 有的指标不论改什么都推演不了（_Plan.blocked）：卡上没记全输入、用到了遮罩列、最外层的输入还原不了（容器的完整值 @@ -52,7 +53,7 @@ from app.engine.expressions import ( parse_expression, substitute, ) -from app.engine.nodes.metrics import _REF_HEAD, _brief +from app.engine.nodes.metrics import _REF_HEAD, _brief, round_metric NO_LINEAGE = "这个指标生成于系统升级前，口径卡没有记录输入，不能推演" NO_FULL_VALUE = "当时没有保存这个输入的完整值" @@ -60,6 +61,13 @@ NOT_RESTORABLE = "这类输入当时没有保存完整值" MASKED_INPUT = "这一列在数据源中设置了遮罩，不能用于推演" MASKED_TABLE = "这份查询结果里有设置了遮罩的列，不能用于推演" MASKED_METRIC = "这个指标用到了设置了遮罩的列，不能推演" +#: 按位置取自查询结果的一格（vars.res.rows[0][2]）遮没遮核对不了：取值链上节点的产出读不出来，或者那份结果没存快照、 +#: 只带着数据源名（查询当时记下的遮罩在快照里）。和读不出来的快照一样宁可多遮（_positional） +POSITION_UNREADABLE = "查询结果无法读取，核对不了这一格是否取自设置了遮罩的列" +POSITION_NO_SNAPSHOT = "查询结果没有保存快照，核对不了这一格是否取自设置了遮罩的列" +#: recompute（双盲复算的预检、盲值）里整个指标重算不了的说法：同 _Plan.blocked，只是不说「推演」——那里不是推演 +NO_LINEAGE_PLAIN = "这个指标生成于系统升级前，口径卡没有记录输入" +MASKED_PLAIN = "这个指标用到了设置了遮罩的列" SELECTOR = "这个输入决定取哪一行、哪一项，修改它需要重新取数，不能推演" EXCLUDED = "排除后无法计算" UNREADABLE = "查询快照无法读取" @@ -169,6 +177,8 @@ class _Plan: tops: list[str] = field(default_factory=list) inputs: dict[str, _Input] = field(default_factory=dict) problem: str | None = None + #: problem 不说「推演」的说法（recompute 给双盲复算用）；和 problem 一样时不填 + plain: str | None = None #: 嵌在别的引用里的输入 → 包着它的最外层引用（_refs） within: dict[str, set[str]] = field(default_factory=dict) @@ -199,6 +209,20 @@ class _Plan: return "unsupported", f"无法推演：{path}（{problem}）" return None + @property + def stuck(self) -> str | None: + """blocked 的原因换成不说「推演」的说法：recompute 交给双盲复算的预检、盲值，那里说「不能推演」驴唇不对马嘴。 + 判断和先后与 blocked 一字不差，只换措辞；推演得了的是 None。""" + blocked = self.blocked + if blocked is None: + return None + if self.problem: + return self.plain or self.problem + if blocked[0] == "masked": + return MASKED_PLAIN + path = next(p for p in self.tops if self.inputs[p].problem) + return f"输入 {path} 无法还原（{self.inputs[path].problem}）" + # -------------------------------------------------------------------------- # 入口 @@ -231,8 +255,14 @@ def whatif(card: dict[str, Any], changes: Changes, sources: Sources, *, metric: def recompute(metric: dict[str, Any], changes: Changes, sources: Sources) -> dict[str, Any]: - """重算一个指标，不管改动碰没碰到它，也不校验改动。核对还原是否忠实时用：什么都不改，重算出来应当和卡上记的一样。""" - return _outcome(_plan(metric, _Store(sources)), changes, sources, force=True)[0] + """重算一个指标，不管改动碰没碰到它，也不校验改动。核对还原是否忠实时用：什么都不改，重算出来应当和卡上记的一样。 + + 调用方是双盲复算（预检、按盲算的输入代回原式），不是推演：整个指标重算不了时原因不说「推演」（_Plan.stuck）。""" + plan = _plan(metric, _Store(sources)) + result = _outcome(plan, changes, sources, force=True)[0] + if (plain := plan.stuck) is not None: + result["reason"] = plain + return result # -------------------------------------------------------------------------- @@ -244,7 +274,7 @@ def _plan(metric: dict[str, Any], store: _Store) -> _Plan: expr = str(metric.get("expression") or "").strip() items = metric.get("inputs") if not expr or not isinstance(items, list): - return _Plan(metric, problem=NO_LINEAGE) + return _Plan(metric, problem=NO_LINEAGE, plain=NO_LINEAGE_PLAIN) try: tree, _, _ = parse_expression(expr) except ExpressionError as e: @@ -257,7 +287,8 @@ def _plan(metric: dict[str, Any], store: _Store) -> _Plan: inputs[item["path"]] = _restore(item, remaining, set(within), store) plan = _Plan(metric, tree, tops, inputs, within=within) if lost := [p for p in tops if p not in inputs]: - plan.problem = f"口径卡缺少输入 {'、'.join(lost[:3])} 的记录，不能推演" + plan.plain = f"口径卡缺少输入 {'、'.join(lost[:3])} 的记录" + plan.problem = f"{plan.plain}，不能推演" return plan @@ -335,6 +366,8 @@ def _restore(item: dict[str, Any], remaining: set[str], nested: set[str], store: inp.unreadable = True inp.problem = inp.problem or (TAMPERED if trouble == "tampered" else UNREADABLE) inp.masked = _masked(inp, locator, store) + if not inp.masked: + _positional(inp, store) return inp @@ -416,6 +449,159 @@ def _masked(inp: _Input, locator: dict[str, Any], store: _Store) -> bool: return any(isinstance(c, str) and c.lower() in hidden for c in columns) +def _positional(inp: _Input, store: _Store) -> None: + """按位置取自查询结果的一格（vars.res.rows[0][2]）：卡上没记定位，顺着取值链找到那份结果、认出是哪一列（input_origin）。 + + 落在遮罩列上的按遮罩（masked）。遮没遮核对不了的——取值链上节点的产出读不出来，或者那份结果没存快照、只带着数据源名 + ——和读不出来的快照同一个处理：值不给、也不拿来推演（unreadable，原因记在 problem），宁可多遮。认得出、可那份结果 + 既没带快照的 id 也没带数据源名的（数据整形改过形状的结果）认不出出处，照常推演，见模块说明。""" + if inp.snapshot is not None or inp.shape != "scalar" or inp.item.get("via") in ("tool_cell", "agent_field"): + return + origin = input_origin(inp.item, lambda node_id: store.output(node_id)[0]) + if origin is None: + return + if origin.unverified is None and origin.artifact: + hidden = store.sources.masked.get(origin.artifact) + inp.masked = bool(hidden) and any(c.lower() in hidden for c in origin.columns) + return + if origin.unverified is not None or origin.source: + inp.unreadable = True + inp.problem = inp.problem or origin.unverified or POSITION_NO_SNAPSHOT + + +@dataclass(frozen=True) +class Origin: + """一个输入取自哪份查询结果的哪几列（input_origin）。 + + - artifact、source：那份查询结果的快照 id 和数据源名，结果里没带的是 None + - columns：用到的列（列名照原样）。按位置取的一格、取值链上的产出读不出来时认不出是哪一列，为空 + - unverified：遮没遮核对不了的原因（取值链上节点的产出读不出来）；核对得了的是 None + """ + + artifact: str | None + source: str | None + columns: tuple[str, ...] + unverified: str | None = None + + +def input_origin(item: dict[str, Any], output: Callable[[Any], Any]) -> Origin | None: + """卡上的一个输入取自哪份查询结果的哪几列，遮罩按列认时用（推演、证据面板的指标步骤、可验证报告是同一个认法）。 + + cell() 的格、Agent 交来的字段按定位里的列认；vars.res.rows[0][2] 这种按位置取的一格顺着取值链认（positional_cell， + output 同它）。不是取自查询结果的格（常量、运行输入、Agent 没按出处交的字段、加工过的数）是 None。""" + if item.get("via") in ("tool_cell", "agent_field") and isinstance(item.get("artifact"), str) and item["artifact"]: + locator = item.get("locator") if isinstance(item.get("locator"), dict) else {} + columns = [locator.get("column")] if locator.get("column") else list((locator.get("columns") or {}).values()) + return Origin(item["artifact"], None, tuple(c for c in columns if isinstance(c, str) and c)) + found = positional_cell(str(item.get("path") or ""), item.get("node_id"), output) + if found is None: + return None + if found.get("unreadable"): + return Origin(None, None, (), POSITION_UNREADABLE) + return Origin(found["artifact"], found["source"], (found["column"],)) + + +#: 取值链没走通 +_GONE = object() + + +def positional_cell(path: str, node_id: Any, output: Callable[[Any], Any]) -> dict[str, Any] | None: + """按位置取自一份查询结果的一格：vars.res.rows[0][2]、nodes.parse.rows[1]['amount'] 这种标量引用——查询结果经数据 + 整形解析成对象之后按行号、列号（或列名）取。返回 {artifact, source, row, column}：artifact、source 是那份结果里带着的 + （数据源工具、合并查询交回的结果都带，没有的是 None），row 是从 0 数的行号，column 是列名。认不出来的返回 None。 + + 遮罩要按列认（证据面板的指标步骤、推演都用这个）：cell() 和 Agent 字段有定位，这种引用卡上只记了路径和值。 + + output(节点 id) 交回那个节点当时的产出（读不出来给 None）。nodes.X 的根就是 X 的产出；vars.X 是写它的节点（输入记着 + 的 node_id）交给变量的那个值——数据整形、合并查询交的是产出本身，Agent、LLM、代码节点交的是产出里的 data，工具节点 + 交的是 JSON 文本，都试一遍，取值链走得通的那个就是。下标得是字面量：rows[vars.i] 要另取一个值才知道是哪一行，不认。 + + 路径就是按位置取一格的样子（以 .rows[行号][列] 结尾），可那个节点的产出读不出来（文件不在、与哈希不一致、不知道是 + 谁写的）时返回 {"unreadable": True}：这一格取自哪一列核对不了，调用方宁可多遮。卡上记着这个输入的值，算的时候 + 取值链是走得通的，现在读不出来只能是产出丢了或被改了。 + """ + try: + tree = ast.parse(path, mode="eval").body + except SyntaxError: + return None + keys: list[Any] = [] + while isinstance(tree, (ast.Attribute, ast.Subscript)): + if isinstance(tree, ast.Attribute): + keys.append(tree.attr) + else: + key = _literal_key(tree.slice) + if key is _GONE: + return None + keys.append(key) + tree = tree.value + keys.reverse() + if not isinstance(tree, ast.Name) or tree.id not in ("vars", "nodes") or len(keys) < 4 \ + or not isinstance(keys[0], str): + return None + base = output(keys[0] if tree.id == "nodes" else node_id) + if base is None: + positional = keys[-3] == "rows" and isinstance(keys[-2], int) and isinstance(keys[-1], (int, str)) + return {"unreadable": True} if positional else None + candidates = [base] + if isinstance(base, dict) and "data" in base: + candidates.append(base["data"]) + if isinstance(base, str): + try: + candidates.append(json.loads(base)) + except ValueError: + pass + for value in candidates: + if (found := _cell_in(value, keys[1:])) is not None: + return found + return None + + +def _literal_key(node: ast.AST) -> Any: + """下标里的字面量：整数（含负数）或者文本。别的（变量、算式）交回 _GONE。""" + if isinstance(node, ast.Constant) and isinstance(node.value, (int, str)) and not isinstance(node.value, bool): + return node.value + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.USub) and isinstance(node.operand, ast.Constant) \ + and isinstance(node.operand.value, int) and not isinstance(node.operand.value, bool): + return -node.operand.value + return _GONE + + +def _cell_in(value: Any, keys: list[Any]) -> dict[str, Any] | None: + """沿取值链往下走，走到一份查询结果（带 rows 列表的对象）、剩下的正好是 rows[行][列] 时交回那一格。""" + for i, key in enumerate(keys): + if isinstance(value, dict) and isinstance(value.get("rows"), list) and len(keys) - i == 3 and key == "rows": + return _cell_at(value, keys[i + 1], keys[i + 2]) + if isinstance(value, dict) and isinstance(key, str): + value = value.get(key, _GONE) + elif isinstance(value, list) and isinstance(key, int) and -len(value) <= key < len(value): + value = value[key] + else: + return None + if value is _GONE: + return None + return None + + +def _cell_at(result: dict[str, Any], row: Any, column: Any) -> dict[str, Any] | None: + rows = result["rows"] + if not isinstance(row, int) or not -len(rows) <= row < len(rows): + return None + row = row % len(rows) + record = rows[row] + columns = [str(c) for c in result.get("columns") or []] + if isinstance(column, int) and isinstance(record, (list, tuple)) and -len(columns) <= column < len(columns): + name: str | None = columns[column] + elif isinstance(column, str) and (column in columns or (isinstance(record, dict) and column in record)): + name = column + else: + name = None + if name is None: + return None + artifact, source = result.get("artifact"), result.get("source") + return {"artifact": artifact if isinstance(artifact, str) and artifact else None, + "source": source if isinstance(source, str) and source else None, "row": row, "column": name} + + def _static(path: str) -> bool: """取值路径的下标全是字面量（nodes.q.rows[0]），没有 nodes.q.rows[vars.i] 这种要另取一个值才知道是哪一项的。""" try: @@ -647,7 +833,7 @@ class _Bind(ast.NodeTransformer): def _evaluate(plan: _Plan, values: dict[str, Any]) -> tuple[Any, str | None]: - """按原式求值，报错的说法和口径卡一致。浮点结果按卡上的小数位 round（metrics.run_metrics 同样只 round 浮点）。""" + """按原式求值，报错的说法和口径卡一致。取整和口径卡共用 metrics.round_metric（只取整浮点）。""" names = {path: _PLACEHOLDER.format(i) for i, path in enumerate(plan.tops)} bound = ast.fix_missing_locations(_Bind(names).visit(copy.deepcopy(plan.tree))) ctx = {names[p]: values.get(p) for p in plan.tops} @@ -663,10 +849,8 @@ def _evaluate(plan: _Plan, values: dict[str, Any]) -> tuple[Any, str | None]: if isinstance(e, TypeError): return None, "输入的类型不正确，无法计算" return None, f"无法计算（{describe_exception(e)}）" - decimals = plan.metric.get("decimals") - if isinstance(value, float) and isinstance(decimals, int) and not isinstance(decimals, bool): - value = round(value, decimals) - return value, None + # 取整和口径卡是同一个函数：卡上怎么取整的，推演就怎么取整 + return round_metric(value, plan.metric.get("decimals")), None # -------------------------------------------------------------------------- @@ -723,11 +907,10 @@ def _delta(metric: dict[str, Any], old: Any, new: Any) -> tuple[Any, str | None] """变化量和它的写法。比率、百分数的变化写「个百分点」：8.7% 到 19.0% 是多了 10.3 个百分点，不是 10.3%。""" if not (_number(old) and _number(new)): return None, None - delta = new - old decimals = metric.get("decimals") places = decimals if isinstance(decimals, int) and not isinstance(decimals, bool) else None - if isinstance(delta, float) and places is not None: - delta = round(delta, places) + # 变化量按卡上的小数位取整，规则同指标本身（round_metric）：不然 8.7 到 19.0 会写成 +10.299999999999999 + delta = round_metric(new - old, places) if delta == 0: return delta, None fmt = metric.get("format") or DEFAULT_FORMAT @@ -843,6 +1026,30 @@ def _fixed(inp: _Input) -> str | None: return None if value is None or is_number(value) else NOT_NUMBER +def substituted_without(metric: dict[str, Any], paths: Iterable[str], token: str) -> str | None: + """卡上的代入式，只把 paths 这几个输入代成 token（证据面板给取自遮罩列的输入写「已遮罩」），别的照卡上代。 + + 按卡上记的输入值重新代一遍：先核对一个都不换时代出来的和卡上记的一字不差，对不上（卡上记的不是原值、老卡的写法） + 就整条不给，交回 None——宁可不显示代入式，也不能让遮着的值从对不上的那一处漏出来。卡上没代成字面量的（容器、太长的 + 文本）照旧留原路径。嵌在别的引用里的输入不单独换：外面那个引用代成了它的值，里面那一格本来就不出现。 + """ + expr, recorded = metric.get("expression"), metric.get("substituted") + if not isinstance(expr, str) or not isinstance(recorded, str): + return None + try: + tree, _, _ = parse_expression(expr) + kept = set(leaf_refs(parse_expression(recorded)[0])) + except ExpressionError: + return None + values = {str(i["path"]): i.get("value") for i in metric.get("inputs") or [] + if isinstance(i, dict) and isinstance(i.get("path"), str) and i["path"] not in kept} + if substitute(tree, values) != recorded: + return None + hidden = set(paths) + swapped = ast.fix_missing_locations(_Bind({p: token for p in hidden}).visit(copy.deepcopy(tree))) + return substitute(swapped, {p: v for p, v in values.items() if p not in hidden}) + + # -------------------------------------------------------------------------- # 报告里引用的地方、下游口径卡 # -------------------------------------------------------------------------- diff --git a/backend/tests/bundle_samples.py b/backend/tests/bundle_samples.py index aed8b030..2e87f8d8 100644 --- a/backend/tests/bundle_samples.py +++ b/backend/tests/bundle_samples.py @@ -14,8 +14,11 @@ - two-reports：另有一份月报，两份报告各自的引用链 - legacy：早期运行，封存范围按结束事件推算（运行结束事件在范围外，运行结果无法核对） - failed：失败结束的运行（没有出具判定），封存说明多一句「继续运行会重新封存」 +- continued：早期运行先失败、经继续运行跑到成功：范围里有那条运行失败，之后运行又继续了，成功结束的那条在范围外 - degraded：两条出具判定，先完整出具、后降档出具，认最后一条 - missing-doc / missing-card：导出时服务端已没有报告文档 / 口径卡的文件 +- masked-card：口径卡上一个指标的输入取自会员消费的号码列（设了遮罩），卡上记着号码原值：整张卡只给 id，引用它的 + 数字无法核对 - nan：运行开始事件和门店流水快照里有 json.dumps 写出的裸 NaN（快照里那一列报告没有引用） - stray：报告自查记下的文档是另一次运行写的 """ @@ -69,14 +72,25 @@ def shop(nan: bool = False) -> dict[str, Any]: "sql": "SELECT store, amount, ratio FROM orders ORDER BY id", "truncated": False} -class Sample: - """一次虚构的已封存运行：工件（原字节）、证据目录、报告文档、事件。""" +def masked_card(crm_id: str) -> dict[str, Any]: + """口径卡多一个指标：联系电话位数，输入是会员消费第 0 行的号码（phone 设了遮罩），卡上记着号码原值和代入式。""" + cell = "cell(nodes.members, 0, 'phone')" + contact = {"id": "contact", "name": "联系电话位数", "value": 8, "unit": "位", "decimals": 0, "format": "thousands", + "expression": f"len({cell})", "substituted": f"len('{CRM['rows'][0][1]}')", + "inputs": [{"path": cell, "value": CRM["rows"][0][1], "node_id": "members", "via": "tool_cell", + "artifact": crm_id, "locator": {"row": 0, "column": "phone"}, "status": "verified"}]} + return {**CARD, "metrics": [*CARD["metrics"], contact]} - def __init__(self, *, nan: bool = False) -> None: + +class Sample: + """一次虚构的已封存运行：工件（原字节）、证据目录、报告文档、事件。masked=True 时口径卡带着取自遮罩列的输入。""" + + def __init__(self, *, nan: bool = False, masked: bool = False) -> None: self.files: dict[str, bytes] = {} self.shop_id = self.put(shop(nan)) self.crm_id = self.put(CRM) - self.card_id = self.put(CARD) + self.card = masked_card(self.crm_id) if masked else CARD + self.card_id = self.put(self.card) self.ledger = [ {"kind": "query", "node_id": "fetch", "exec": 1, "artifact": self.shop_id, "tool": "db_query__shop", "source": "shop", "columns": ["store", "amount", "ratio"], "rows": len(STORES), "truncated": False}, @@ -84,7 +98,7 @@ class Sample: "source": "crm", "columns": CRM["columns"], "rows": len(CRM["rows"]), "truncated": False}, {"kind": "metric_set", "node_id": "card", "artifact": self.card_id}, ] - self.catalog = build_catalog(nodes={"card": CARD}, ledger=self.ledger) + self.catalog = build_catalog(nodes={"card": self.card}, ledger=self.ledger) def put(self, obj: Any) -> str: artifact, raw = stored(obj) @@ -133,6 +147,16 @@ class Sample: add("issuance", "out", {"tier": tier, "mode": "citations"}) add("node.finished", "out", {"duration_ms": 2, "attempt": 1, "preview": {"周报": "（报告正文）"}}) add("run.finished", None, {"output": {"周报": "（报告正文）"}}) + elif ending == "continued": + # 先失败，再从成果节点继续运行到成功（runner.continue_failed 记一条 run.resumed） + add("node.failed", "out", {"error": "出口复核没有通过"}) + add("run.failed", None, {"error": "出口复核没有通过"}) + add("run.resumed", None, {"from": ["out"], "message": "从「成果」继续运行，保留前面 6 个节点的结果", + "actor": "值班分析员"}) + for tier in tiers: + add("issuance", "out", {"tier": tier, "mode": "citations"}) + add("node.finished", "out", {"duration_ms": 2, "attempt": 1, "preview": {"周报": "（报告正文）"}}) + add("run.finished", None, {"output": {"周报": "（报告正文）"}}) else: add("node.failed", "out", {"error": "出口复核没有通过"}) add("run.failed", None, {"error": "出口复核没有通过"}) @@ -172,6 +196,11 @@ def pages() -> dict[str, str]: events, bound = s.events({"write": weekly}, ending="failed") out["failed"] = render(s, events, bound, run={**RUN, "status": "failed"}) + # 早期运行先失败、经继续运行跑到成功：封存范围同样按结束事件推算，到成功结束的前一条为止 + events, _ = s.events({"write": weekly}, ending="continued") + finished = next(seq for seq, etype, _, _ in events if etype == "run.finished") + out["continued"] = render(s, events, finished - 1, legacy=True) + events, bound = s.events({"write": weekly}, tiers=("formal", "degraded")) out["degraded"] = render(s, events, bound) @@ -182,6 +211,10 @@ def pages() -> dict[str, str]: n = Sample(nan=True) events, bound = n.events({"write": n.doc(WEEKLY, "write")}, nan=True) out["nan"] = render(n, events, bound) + + m = Sample(masked=True) + events, bound = m.events({"write": m.doc(WEEKLY, "write")}) + out["masked-card"] = render(m, events, bound) return out diff --git a/backend/tests/test_bundle_build.py b/backend/tests/test_bundle_build.py index 1cb16179..fff346d7 100644 --- a/backend/tests/test_bundle_build.py +++ b/backend/tests/test_bundle_build.py @@ -462,8 +462,8 @@ def test_the_sample_pages_for_the_browser_check_are_sealed_for_real(): from tests.bundle_samples import pages got = {name: json.loads(data_block(page)) for name, page in pages().items()} - assert set(got) == {"base", "two-reports", "legacy", "failed", "degraded", "missing-doc", "missing-card", "nan", - "stray"} + assert set(got) == {"base", "two-reports", "legacy", "failed", "continued", "degraded", "missing-doc", + "missing-card", "nan", "stray", "masked-card"} for name, bundle in got.items(): assert browser_manifest(bundle["events"]) == bundle["meta"]["manifest_hash"], name @@ -490,9 +490,18 @@ def test_the_sample_pages_for_the_browser_check_are_sealed_for_real(): failed = got["failed"] assert failed["meta"]["status"] == "failed" and failed["meta"]["tier"] is None assert max(failed["events"], key=lambda e: e["seq"])["type"] == "run.failed" + # 先失败、经继续运行跑到成功的早期运行：范围里有那条运行失败，之后运行又继续了，最后一条不是结束事件 + continued = got["continued"] + sealed_types = [e["type"] for e in sorted(continued["events"], key=lambda e: e["seq"])] + assert continued["meta"]["legacy"] is True and continued["meta"]["status"] == "succeeded" + assert "run.failed" in sealed_types and sealed_types[sealed_types.index("run.failed") + 1] == "run.resumed" + assert sealed_types[-1] == "node.finished" and "run.finished" in [a["type"] for a in continued["appended"]] issuances = [e["seq"] for e in got["degraded"]["events"] if e["type"] == "issuance"] assert len(issuances) == 2 and got["degraded"]["meta"]["tier"] == {"value": "degraded", "seq": issuances[-1]} assert [a["kind"] for a in got["stray"]["artifacts"]] == ["report_doc"] + # 口径卡上有输入取自遮罩列：整张卡只给 id，页面里找不到那个号码 + card = next(a for a in got["masked-card"]["artifacts"] if a["kind"] == "metric_set") + assert card.get("masked") is True and "b64" not in card nan = got["nan"] assert "NaN" in next(e["canonical"] for e in nan["events"] if e["type"] == "run.started") assert b"NaN" in base64.b64decode(next(a["b64"] for a in nan["artifacts"] if a.get("node_id") == "fetch")) diff --git a/backend/tests/test_bundle_masked_cards.py b/backend/tests/test_bundle_masked_cards.py new file mode 100644 index 00000000..11f8d917 --- /dev/null +++ b/backend/tests/test_bundle_masked_cards.py @@ -0,0 +1,107 @@ +"""可验证报告里的口径卡和遮罩（期 4 I2a 复核）：口径卡上记着每个输入的值和代入式，有输入取自遮罩列时整张卡只给 id。 + +以前只有查询快照会因遮罩不随文件附带，口径卡一律原样嵌入：证据面板上已经遮住的号码（cell() 取的遮罩格、按位置取的 +vars.res.rows[0][1]），在这份要发给别人的文件里照样看得到。认哪个输入取自哪份查询结果的哪几列，和证据面板、推演同一个 +认法（recompute.input_origin）；核对不了的宁可多挡。 + +工件、事件都是手搭的（会员、号码都是编的），照 tests/test_bundle_build.py 的样子。 +""" +from __future__ import annotations + +import base64 + +import pytest + +from app.bundle.build import build_bundle, render_html +from tests.test_bundle_build import GRAPH, MASKED_ID, RUN, SNAP_MASKED, STORE, doc_of, stored + +#: SNAP_MASKED 里的号码（phone 设了遮罩） +PHONE = SNAP_MASKED["rows"][0][1] +CELL = "cell(nodes.members, 0, 'phone')" + + +def card_with(inputs, substituted="len('000-0001')", expression=f"len({CELL})"): + """会员号码位数一张卡：报告引用的是它（m:gmv 指过来）。""" + return {"kind": "metric_set", "caliber": "会员口径", "caliber_version": "v1", "metrics": [ + {"id": "gmv", "name": "号码位数", "value": 1280.5, "unit": "元", "decimals": 1, "expression": expression, + "substituted": substituted, "inputs": inputs}]} + + +def bundle_of(card, *, outputs=None, source_masks=None): + """一次手搭的运行：会员查询（带遮罩的快照）→ 数据整形（outputs 里的产出）→ 口径卡 → 报告。""" + card_id, card_raw = stored(card) + doc = doc_of(extra_catalog={"m:gmv": {"alias": "m:gmv", "kind": "metric", "artifact": card_id, "node_id": "card", + "locator": {"metric": "gmv"}, "value": 1280.5}}) + doc_id, doc_raw = stored(doc) + files = {**STORE, card_id: card_raw, doc_id: doc_raw} + events = [ + (1, "run.started", None, {"nodes": 7}), + (2, "tool.end", "members", {"tool": "db_query__crm", "query_artifact": MASKED_ID}), + (3, "node.finished", "members", {"evidence": [{"kind": "query", "node_id": "members", "artifact": MASKED_ID}]}), + ] + for node_id, content in (outputs or {}).items(): + artifact, raw = stored(content) if content is not None else ("ee" * 32, None) + if raw is not None: + files[artifact] = raw + events.append((len(events) + 1, "node.finished", node_id, {"artifact": artifact})) + events += [ + (len(events) + 1, "node.started", "card", {}), + (len(events) + 2, "node.finished", "card", {"evidence": [{"kind": "metric_set", "node_id": "card", + "artifact": card_id}]}), + (len(events) + 3, "report.checked", "write", {"doc_artifact": doc_id, "ok": True}), + (len(events) + 4, "run.finished", None, {"output": {}}), + ] + bundle = build_bundle(RUN, events, manifest_seq=len(events), manifest_hash="0" * 64, graph=GRAPH, + source_masks=source_masks or {}, read=files.get, exported_at="2026-10-06T08:00:00+00:00") + entry = next(a for a in bundle["artifacts"] if a["id"] == card_id) + return bundle, entry + + +def cell_input(column, value): + return {"path": f"cell(nodes.members, 0, '{column}')", "value": value, "node_id": "members", "via": "tool_cell", + "artifact": MASKED_ID, "locator": {"row": 0, "column": column}, "status": "verified"} + + +def test_a_card_reading_a_masked_cell_ships_only_its_id(): + bundle, entry = bundle_of(card_with([cell_input("phone", PHONE)])) + assert entry == {"id": entry["id"], "kind": "metric_set", "node_id": "card", "masked": True} + assert PHONE not in render_html(bundle) + + +def test_a_card_reading_only_unmasked_columns_is_embedded(): + """同一份快照里没遮的列（消费额）：卡上没有遮罩列的值，照常嵌入、照常核对。""" + card = card_with([cell_input("spend", 320.0)], substituted="320.0", expression="cell(nodes.members, 0, 'spend')") + _, entry = bundle_of(card) + assert "masked" not in entry and base64.b64decode(entry["b64"]) == stored(card)[1] + + +def test_a_mask_set_on_the_source_now_counts_too(): + card = card_with([cell_input("spend", 320.0)], substituted="320.0", expression="cell(nodes.members, 0, 'spend')") + _, entry = bundle_of(card, source_masks={"crm": ["spend"]}) + assert entry.get("masked") is True and "b64" not in entry + + +PARSED = {"columns": SNAP_MASKED["columns"], "rows": SNAP_MASKED["rows"]} +POSITIONAL = "vars.res.rows[0][1]" + + +@pytest.mark.parametrize("parsed, masked", [ + # 数据整形原样解析的查询结果（带着快照的 id）：认得出取自 phone 列 + ({**PARSED, "artifact": MASKED_ID, "source": "crm"}, True), + # 写变量的节点的产出读不出来：核对不了，宁可多挡 + (None, True), + # 结果没存快照、只带着数据源名：查询当时记下的遮罩无从核对，宁可多挡 + ({**PARSED, "source": "crm"}, True), + # 改过形状、既没带快照的 id 也没带数据源名：认不出出处，照常嵌入（和证据面板、推演一样，是已知的缺口） + (PARSED, False), +]) +def test_a_card_reading_a_cell_by_position_is_judged_the_same_way(parsed, masked): + item = {"path": POSITIONAL, "value": PHONE, "node_id": "parse", "via": "transform", "field": "rows[0][1]", + "status": "ok"} + card = card_with([item], substituted=f"len('{PHONE}')", expression=f"len({POSITIONAL})") + bundle, entry = bundle_of(card, outputs={"parse": parsed}) + assert (entry.get("masked") is True) is masked and ("b64" in entry) is not masked + if masked: + assert PHONE not in render_html(bundle) + else: + assert PHONE in base64.b64decode(entry["b64"]).decode() diff --git a/backend/tests/test_evidence_metric_masks.py b/backend/tests/test_evidence_metric_masks.py new file mode 100644 index 00000000..f3e72094 --- /dev/null +++ b/backend/tests/test_evidence_metric_masks.py @@ -0,0 +1,416 @@ +"""证据面板指标步骤的遮罩（期 4 I2a）：取自遮罩列的输入，输入标签和代入式里都不显示它的值。 + +以前只有查询步骤的结果窗口把遮罩列换成「已遮罩」，指标步骤的输入（「值 ← 路径」）和代入式照样写着原值——同一个 +号码，表格里遮着，上面的标签里亮着。现在： + +- cell() 取的格、Agent 交来的字段：按定位里的列认 +- vars.res.rows[0][2] 这种按位置取自查询结果的标量引用：顺着取值链找到那份查询结果（写变量的节点当时的产出）， + 按列号认出是哪一列 +- 遮的是「查询当时记下的」加「数据源现在设的」，和查询步骤同一套；数据源事后去掉了遮罩，当时遮着的照样遮 +- 取自遮罩列的输入值写「已遮罩」，带上是哪几列（masked），模型报的值、快照里的值一并不给；代入式里那一处写「已遮罩」， + 遮罩列也进 redacted.columns（面板底下那句「遮罩不是安全边界」） +- 没有遮罩的指标和以前一字不差；推演（api/whatif.py）同样认得出按位置取的遮罩列，不给值、不推演 + +真跑一遍图（虚构的 SQLite 订单库，号码是编的），报告由 MockChatModel 照剧本写。 +""" +from __future__ import annotations + +import asyncio +import json +import sqlite3 + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage +from sqlalchemy import select + +from app.core import artifact_store +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource, Run +from app.engine.evidence import iter_segments +from app.engine.runner import run_manager +from app.main import app + +SQL = "SELECT id, region, phone, amount FROM orders ORDER BY id" +#: 号码写成带横线的文本：全是数字的文本 cell() 会按数读（口径卡在算数），len() 就算不了 +ROWS = [(1, "华东", "139-0000-0001", 120.5), (2, "华南", "139-0000-0002", 80.0), (3, "华北", "139-0000-0003", 99.5)] +PHONES = [r[2] for r in ROWS] +#: 号码那一格按位置取：vars.res 是数据整形解析好的查询结果，第 2 列是 phone +FIRST_PHONE = "vars.res.rows[0][2]" +SECOND_PHONE = "vars.res.rows[1][2]" +PHONE_CELL = "cell(nodes.fetch, 0, 'phone')" +METRICS = [ + {"id": "amt", "name": "首单金额", "unit": "元", "decimals": 1, "format": "thousands", + "expression": "cell(nodes.fetch, 0, 'amount')"}, + {"id": "plen", "name": "号码位数", "unit": "位", "format": "thousands", "expression": f"len({PHONE_CELL})"}, + {"id": "plen2", "name": "号码位数两倍", "unit": "位", "format": "thousands", "expression": f"len({FIRST_PHONE}) * 2"}, + {"id": "mix", "name": "混合", "unit": "分", "decimals": 1, "format": "thousands", + "expression": f"round(cell(nodes.fetch, 1, 'amount') + len({SECOND_PHONE}), 1)"}, +] +TEXT = "首单金额 [[m:amt]]，号码 [[m:plen]]，两倍 [[m:plen2]]，混合 [[m:mix]]。" + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +async def set_source(path, options=None): + async with SessionLocal() as session: + row = (await session.execute(select(DataSource).where(DataSource.name == "shop"))).scalar_one_or_none() + if row is None: + row = DataSource(name="shop", kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + row.database, row.kind, row.enabled, row.readonly = str(path), "sqlite", True, True + row.options = dict(options or {}) + await session.commit() + return row.id + + +@pytest.fixture +async def shop(tmp_path): + path = tmp_path / "shop.db" + db = sqlite3.connect(path) + db.executescript("CREATE TABLE orders (id INTEGER PRIMARY KEY, region TEXT, phone TEXT, amount REAL);") + db.executemany("INSERT INTO orders VALUES (?, ?, ?, ?)", ROWS) + db.commit() + db.close() + source_id = await set_source(path) + await engines.invalidate(source_id) + yield path + await engines.invalidate(source_id) + + +def node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def graph(): + nodes = [ + node("start", "input"), + node("fetch", "tool", tool="db_query__shop", args={"sql": SQL}), + node("parse", "transform", mode="json", template="{{ nodes.fetch }}", assign_to="res"), + node("card", "metrics", caliber="遮罩口径", caliber_version="v1", metrics=METRICS), + node("write", "report", instructions="写一句", on_violation="flag"), + node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}]), + ] + return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +async def run_graph(monkeypatch) -> Run: + from app.providers import mock_model + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=TEXT)) + run_id = (await run_manager.start(graph=graph(), input_payload={})).id + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + assert row.status == "succeeded", row.error + return row + raise AssertionError(f"运行 {run_id} 没跑完：{row.status}") + + +async def segment(client, run_id: str, text: str) -> dict: + graph_body = (await client.get(f"/api/runs/{run_id}/evidence")).json() + [report] = graph_body["reports"] + doc = artifact_store.load(report["doc_artifact"]) + seg = next(s for s in iter_segments(doc) if s["text"] == text) + res = await client.get(f"/api/runs/{run_id}/evidence/segments/{seg['id']}") + assert res.status_code == 200, res.text + return res.json() + + +def inputs_of(body: dict) -> dict[str, dict]: + return {s["path"]: s for s in body["chain"] if s["step"] == "input"} + + +def leaks(body: dict) -> list[str]: + raw = json.dumps(body, ensure_ascii=False) + return [p for p in PHONES if p in raw] + + +# -------------------------------------------------------------------------- + + +async def test_a_masked_cell_is_hidden_from_the_input_label_and_the_substituted_expression(client, monkeypatch, shop): + await set_source(shop, {"mask_columns": ["phone"]}) + row = await run_graph(monkeypatch) + body = await segment(client, row.id, "13位") + metric = body["chain"][0] + assert metric["step"] == "metric" and metric["expression"] == f"len({PHONE_CELL})" + assert metric["substituted"] == "len(已遮罩)" + [cell] = inputs_of(body).values() + assert cell["value"] == "已遮罩" and cell["masked"] == ["phone"] and cell["via"] == "tool_cell" + assert "snapshot_value" not in cell and "model_value" not in cell + # 那一格所在的查询步骤照旧遮着；面板底下的遮罩说明带上这一列 + [query] = [s for s in body["chain"] if s["step"] == "query"] + assert query["masked"] == ["phone"] and body["redacted"]["columns"] == ["phone"] + assert leaks(body) == [] + + +async def test_a_scalar_taken_by_position_from_a_masked_column_is_hidden(client, monkeypatch, shop): + """vars.res.rows[0][2]：没有 cell() 的定位，顺着取值链认出取自查询结果的 phone 列。""" + await set_source(shop, {"mask_columns": ["phone"]}) + row = await run_graph(monkeypatch) + body = await segment(client, row.id, "26位") + metric = body["chain"][0] + assert metric["substituted"] == "len(已遮罩) * 2" + [phone] = inputs_of(body).values() + assert phone["path"] == FIRST_PHONE and phone["value"] == "已遮罩" and phone["masked"] == ["phone"] + assert body["redacted"]["columns"] == ["phone"] + assert leaks(body) == [] + + +async def test_only_the_masked_input_is_hidden_when_inputs_are_mixed(client, monkeypatch, shop): + await set_source(shop, {"mask_columns": ["phone"]}) + row = await run_graph(monkeypatch) + body = await segment(client, row.id, "93.0分") + assert body["chain"][0]["substituted"] == "round(80.0 + len(已遮罩), 1)" + inputs = inputs_of(body) + amount = inputs["cell(nodes.fetch, 1, 'amount')"] + assert amount["value"] == 80.0 and "masked" not in amount + assert inputs[SECOND_PHONE]["value"] == "已遮罩" and inputs[SECOND_PHONE]["masked"] == ["phone"] + assert leaks(body) == [] + # 没用到遮罩列的指标和以前一样 + plain = await segment(client, row.id, "120.5元") + assert plain["chain"][0]["substituted"] == "120.5" + [amt] = inputs_of(plain).values() + assert amt["value"] == 120.5 and "masked" not in amt + + +async def test_masks_recorded_at_query_time_stay_after_the_source_drops_them(client, monkeypatch, shop): + await set_source(shop, {"mask_columns": ["phone"]}) + row = await run_graph(monkeypatch) + await set_source(shop, {}) + for text in ("13位", "26位", "93.0分"): + body = await segment(client, row.id, text) + assert "已遮罩" in body["chain"][0]["substituted"] and leaks(body) == [], text + + +async def test_masks_added_after_the_run_apply_too(client, monkeypatch, shop): + row = await run_graph(monkeypatch) + before = await segment(client, row.id, "26位") + assert before["chain"][0]["substituted"] == "len('139-0000-0001') * 2" + assert inputs_of(before)[FIRST_PHONE]["value"] == "139-0000-0001" and "masked" not in inputs_of(before)[FIRST_PHONE] + await set_source(shop, {"mask_columns": "phone"}) + after = await segment(client, row.id, "26位") + assert after["chain"][0]["substituted"] == "len(已遮罩) * 2" and leaks(after) == [] + + +async def test_without_masks_the_metric_step_is_unchanged(client, monkeypatch, shop): + row = await run_graph(monkeypatch) + body = await segment(client, row.id, "93.0分") + assert body["chain"][0]["substituted"] == "round(80.0 + len('139-0000-0002'), 1)" + assert all("masked" not in s for s in body["chain"] if s["step"] == "input") + assert body["redacted"] == {"columns": []} + + +async def test_the_whatif_withholds_a_masked_scalar_taken_by_position(client, monkeypatch, shop): + """推演同一套：按位置取自遮罩列的输入不给值，整个指标推演不了，和证据面板说的一致。""" + await set_source(shop, {"mask_columns": ["phone"]}) + row = await run_graph(monkeypatch) + res = await client.post(f"/api/runs/{row.id}/evidence/whatif", json={"card": "card", "metric": "plen2"}) + assert res.status_code == 200, res.text + [plen2] = res.json()["metrics"] + assert plen2["status"] == "masked" and plen2["substituted"] is None + [phone] = plen2["inputs"] + assert phone["path"] == FIRST_PHONE and phone["masked"] is True and phone["old"] is None + assert not [p for p in PHONES if p in res.text] + # 改它的值同样不行：遮罩列不能拿来推演 + res = await client.post(f"/api/runs/{row.id}/evidence/whatif", + json={"card": "card", "metric": "plen2", "overrides": {"inputs": {FIRST_PHONE: 1}}}) + assert res.status_code == 422 and "遮罩" in res.json()["detail"] + + +# -------------------------------------------------------------------------- +# 手搭的封存范围：模型报的值、快照里的值，以及遮没遮核对不了的几种 +# -------------------------------------------------------------------------- + +#: 一份带遮罩的快照：phone 在查询当时就设了遮罩（记在快照里）。数据源名是编的，库里没有这个数据源（现在的遮罩为空） +GHOST = "ghost_mask_src" +PHONE_SNAP = {"columns": ["id", "phone"], "rows": [[1, "139-0000-0009"]], "source": GHOST, + "sql": "SELECT id, phone FROM members", "mask_columns": ["phone"]} +#: 这几条用例里出现过的号码，答复里一个都不能有 +SECRETS = ["139-0000-0009", "139-0000-0008", "139-0000-0007", "139-0000-0006", "139-0000-0005", "139-0000-0004"] +CELL = "cell(nodes.fetch, 0, 'phone')" + + +def card_metric(mid, expression, substituted, inputs, value=13): + return {"id": mid, "name": mid, "unit": "位", "value": value, "rendered": f"{value}位", "format": "thousands", + "expression": expression, "substituted": substituted, "recompute_ok": True, "inputs": inputs, + "status": "ok"} + + +async def hand_chain(metrics, *, queries=(), outputs=None, metric_id=None): + """手搭一次封存了的运行（取数 → 数据整形 → 口径卡），直接走指标步骤（evidence._metric_chain）。queries 是封存范围内 + 交回过的查询快照，outputs 是口径卡算之前各节点的产出工件。""" + from app.api import evidence + from app.core.artifact_store import put_json + from app.engine.evidence import make_eid + + card = await put_json({"kind": "metric_set", "caliber": "遮罩口径", "caliber_version": "v1", "metrics": metrics}, + kind="metric_set") + events = [(i + 1, "node.finished", nid, {"artifact": art}) for i, (nid, art) in enumerate((outputs or {}).items())] + events += [(len(events) + 1, "node.started", "card", {}), + (len(events) + 2, "node.finished", "card", {"evidence": [ + {"kind": "metric_set", "node_id": "card", "artifact": card}]})] + sealed = evidence._Sealed(run_id="syn-mask", graph=graph(), events=events, + seal={"sealed": True, "ok": True, "manifest_seq": len(events), "legacy": False}) + sealed.queries = {a: {"node_id": "fetch", "tool": f"db_query__{GHOST}", "source": GHOST} for a in queries} + sealed.ledger = {("card", card)} + mid = metric_id or metrics[0]["id"] + eid = make_eid("metric", card, {"metric": mid}) + cite = {"alias": f"m:{mid}", "kind": "metric", "locator": {"metric": mid}, "status": "resolved", "eid": eid} + entry = {"alias": f"m:{mid}", "kind": "metric", "artifact": card, "node_id": "card", "locator": {"metric": mid}, + "eid": eid} + seg = {"id": "s1", "kind": "number", "text": "13位", "state": "deterministic", "cite": cite} + return await evidence._metric_chain(seg, cite, entry, {"catalog": {f"m:{mid}": entry}}, sealed, evidence._Masks()) + + +def said(chain) -> list[str]: + raw = json.dumps(chain, ensure_ascii=False) + return [p for p in SECRETS if p in raw] + + +async def test_what_the_model_reported_and_what_the_snapshot_held_are_withheld_too(): + """对不上的输入另记着两边的值：Agent 字段的 model_value（模型报的）、cell() 的 snapshot_value（快照里的）。取自遮罩列 + 时一并拿掉——界面上不显示它们，接口答复里同样不能带着。""" + from app.core.artifact_store import put_json + + snap = await put_json(PHONE_SNAP, kind="query") + cell = card_metric("plen", f"len({CELL})", "len('139-0000-0008')", [ + {"path": CELL, "value": "139-0000-0008", "node_id": "fetch", "via": "tool_cell", "artifact": snap, + "locator": {"row": 0, "column": "phone"}, "status": "mismatch", "snapshot_value": "139-0000-0009"}]) + agent = card_metric("alen", "len(vars.kpi.phone)", "len('139-0000-0009')", [ + {"path": "vars.kpi.phone", "value": "139-0000-0009", "node_id": "bot", "via": "agent_field", "field": "phone", + "artifact": snap, "locator": {"row": 0, "column": "phone"}, "status": "mismatch", + "model_value": "139-0000-0007"}]) + for metric in (cell, agent): + chain = await hand_chain([metric], queries=[snap]) + [step] = [s for s in chain if s["step"] == "input"] + assert step["value"] == "已遮罩" and step["masked"] == ["phone"], metric["id"] + assert "model_value" not in step and "snapshot_value" not in step, metric["id"] + assert chain[0]["substituted"] == "len(已遮罩)" and said(chain) == [], (metric["id"], said(chain)) + + +async def test_a_substituted_expression_that_does_not_reproduce_is_not_shown(): + """卡上记的代入式和按卡上的输入重新代出来的对不上（老卡、卡上记的不是原值）：整条代入式不给，宁可不显示，也不能让 + 遮着的值从对不上的那一处漏出来。""" + from app.core.artifact_store import put_json + from app.engine import recompute + + snap = await put_json(PHONE_SNAP, kind="query") + odd = card_metric("odd", f"len({CELL})", "len('139-0000-0006')", [ + {"path": CELL, "value": "139-0000-0005", "node_id": "fetch", "via": "tool_cell", "artifact": snap, + "locator": {"row": 0, "column": "phone"}, "status": "verified"}]) + assert recompute.substituted_without(odd, {CELL}, "已遮罩") is None + chain = await hand_chain([odd], queries=[snap]) + assert chain[0]["substituted"] is None and said(chain) == [], said(chain) + # 对得上的照常换掉那一处 + assert recompute.substituted_without({**odd, "substituted": "len('139-0000-0005')"}, {CELL}, "已遮罩") == "len(已遮罩)" + + +async def test_a_cell_on_a_snapshot_that_cannot_be_read_is_withheld(): + """cell() 落在的快照读不出来（文件不在了）：数据源现在没遮这一列，可查询当时记下的遮罩在快照里，遮没遮核对不了—— + 宁可多遮，和推演一致。说法另写，不说「这一列设置了遮罩」（不进 masked）。""" + gone = "dd" * 32 + metric = card_metric("plen", f"len({CELL})", "len('139-0000-0004')", [ + {"path": CELL, "value": "139-0000-0004", "node_id": "fetch", "via": "tool_cell", "artifact": gone, + "locator": {"row": 0, "column": "phone"}, "status": "verified"}]) + chain = await hand_chain([metric], queries=[gone]) + [step] = [s for s in chain if s["step"] == "input"] + assert step["value"] == "已遮罩" and "masked" not in step + assert step["mask_unverified"] == "查询快照无法读取，核对不了这一格是否取自设置了遮罩的列" + assert chain[0]["substituted"] == "len(已遮罩)" and said(chain) == [] + + +PARSED = {"columns": ["id", "phone"], "rows": [[1, "139-0000-0004"]]} +POSITIONAL = "vars.res.rows[0][1]" + + +@pytest.mark.parametrize("parsed, why", [ + # 写变量的节点的产出读不出来：取值链走不通，那一格取自哪一列核对不了 + (None, "查询结果无法读取，核对不了这一格是否取自设置了遮罩的列"), + # 结果没存快照、只带着数据源名：查询当时记下的遮罩在快照里，核对不了 + ({**PARSED, "source": GHOST}, "查询结果没有保存快照，核对不了这一格是否取自设置了遮罩的列"), +]) +async def test_a_positional_cell_that_cannot_be_traced_is_withheld(parsed, why): + from app.core.artifact_store import put_json + + out = await put_json(parsed, kind="node_output") if parsed is not None else "ee" * 32 + metric = card_metric("plen2", f"len({POSITIONAL}) * 2", "len('139-0000-0004') * 2", [ + {"path": POSITIONAL, "value": "139-0000-0004", "node_id": "parse", "via": "transform", "field": "rows[0][1]", + "status": "ok"}], value=26) + chain = await hand_chain([metric], outputs={"parse": out}) + [step] = [s for s in chain if s["step"] == "input"] + assert step["value"] == "已遮罩" and step["mask_unverified"] == why and "masked" not in step + assert chain[0]["substituted"] == "len(已遮罩) * 2" and said(chain) == [] + + +async def test_a_reshaped_result_without_a_snapshot_or_a_source_is_shown_as_is(): + """数据整形改过形状的结果（既没带快照的 id，也没带数据源名）认不出出处：照常显示。已知的缺口，写在交付说明里。""" + from app.core.artifact_store import put_json + + out = await put_json(PARSED, kind="node_output") + metric = card_metric("plen2", f"len({POSITIONAL}) * 2", "len('139-0000-0004') * 2", [ + {"path": POSITIONAL, "value": "139-0000-0004", "node_id": "parse", "via": "transform", "field": "rows[0][1]", + "status": "ok"}], value=26) + chain = await hand_chain([metric], outputs={"parse": out}) + [step] = [s for s in chain if s["step"] == "input"] + assert step["value"] == "139-0000-0004" and "mask_unverified" not in step and "masked" not in step + + +@pytest.mark.parametrize("parsed, withheld", [ + (None, True), + ({**PARSED, "source": GHOST}, True), + (PARSED, False), +]) +def test_the_whatif_withholds_the_same_untraceable_cells(parsed, withheld): + """推演同一个认法：取值链走不通、结果没存快照的那一格不给值、不推演；认不出出处的照常推演。""" + from app.engine import recompute + + store = {"out": parsed} + metric = card_metric("plen2", f"len({POSITIONAL}) * 2", "len('139-0000-0004') * 2", [ + {"path": POSITIONAL, "value": "139-0000-0004", "node_id": "parse", "via": "transform", "field": "rows[0][1]", + "status": "ok"}], value=26) + sources = recompute.Sources(load=store.get, outputs={"parse": "out"} if parsed is not None else {}) + [result] = recompute.whatif({"metrics": [metric]}, recompute.Changes(), sources)["metrics"] + [inp] = result["inputs"] + if withheld: + assert inp["old"] is None and inp.get("withheld") is True and result["status"] == "unsupported" + assert result["substituted"] is None and "139-0000-0004" not in json.dumps(result, ensure_ascii=False) + else: + assert inp["old"] == "139-0000-0004" and result["status"] == "unchanged" + + +async def test_the_writer_output_is_read_once_for_all_cells_taken_by_position(monkeypatch): + """一个指标按位置取了好几格：写变量的节点的产出（可能是几 MB 的查询结果）一次请求里只读一次，快照也只读一次。""" + from app.api import evidence + from app.core.artifact_store import put_json + + snap = await put_json({**PHONE_SNAP, "rows": [[1, "139-0000-0009"], [2, "139-0000-0008"]]}, kind="query") + out = await put_json({"columns": ["id", "phone"], "rows": [[1, "139-0000-0009"], [2, "139-0000-0008"]], + "artifact": snap, "source": GHOST}, kind="node_output") + paths = ["vars.res.rows[0][1]", "vars.res.rows[1][1]", "vars.res.rows[0][0]"] + metric = card_metric("many", " + ".join(f"len(str({p}))" for p in paths), "len('139-0000-0009') + 0", [ + {"path": p, "value": v, "node_id": "parse", "via": "transform", "field": p[9:], "status": "ok"} + for p, v in zip(paths, ["139-0000-0009", "139-0000-0008", 1])], value=27) + reads: list[str] = [] + real = evidence.load + monkeypatch.setattr(evidence, "load", lambda artifact: (reads.append(artifact), real(artifact))[1]) + chain = await hand_chain([metric], outputs={"parse": out}) + inputs = {s["path"]: s for s in chain if s["step"] == "input"} + assert inputs[paths[0]]["masked"] == ["phone"] and inputs[paths[1]]["masked"] == ["phone"] + assert inputs[paths[2]]["value"] == 1 and "masked" not in inputs[paths[2]] + assert reads.count(out) == 1 and reads.count(snap) == 1, reads diff --git a/backend/tests/test_evidence_verify_status.py b/backend/tests/test_evidence_verify_status.py new file mode 100644 index 00000000..280967da --- /dev/null +++ b/backend/tests/test_evidence_verify_status.py @@ -0,0 +1,504 @@ +"""证据接口的核查状态（期 4 I2a）：口径卡指标那一步带双盲复算的结果，句子带反方审稿的判断。 + +- 指标步骤 dual_check：{status, node_id, choice?, blind_rendered?, reason?}。status 是一致（agreed）、不一致（disagreed， + 带审核人的选择，没选的不带）、无法复算（unchecked，带原因）、跳过（skipped）。跳过的原因按这个指标自己说：关闭了 + （disabled）；不在节点配置的「复算的指标」里、或值不是数（not_targeted）；这个指标没有来自查询的输入（no_inputs）—— + 节点直通时产出里的 no_inputs 说的是整张卡，卡上确有来自查询的指标、只是没被选上的不能也说成没有。读双盲复算节点封存的 + 产出和它的台账条目，按台账里记的指标集工件认它复算的是口径卡的哪一次结果；直通的没有台账条目，按画布上它复算的卡、 + 和它跑完之前那张卡最后一次的结果认 +- 句子 unit.challenge：{verdict, node_id, artifact, on_refuted, report_label, counts, unreviewed?}。读反方审稿节点的产出 + 和审稿记录，只认审的就是这一份报告文档的；记录没存上时按画布上审的报告、和它跑完之前那份报告最后一次自查的文档认， + 判断取产出里列的那几句。counts 是整份报告的审稿计数（节点产出里的那一份），证据面板借审批卡的展示画这一句时，概况那 + 一行不能凭这一句拼。节点设为「报错并停止运行」、句子被推翻、运行因此失败时，按审完那一刻的日志认 +- 没有双盲复算、反方审稿的运行，答复里没有这两个键（老运行和以前一字不差） + +双盲复算、反方审稿各真跑一遍图（虚构的 SQLite 库，模型是 MockChatModel 剧本）；认「哪一次」的边界情形用手写的封存事件。 +""" +from __future__ import annotations + +import asyncio +import json +import sqlite3 +import uuid + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage, ToolMessage +from sqlalchemy import delete, select + +from app.api import evidence +from app.core import artifact_store +from app.core.artifact_store import put_json +from app.data import introspect +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import Approval, DataSource, Run, Setting +from app.engine import spend, verify_agent +from app.engine.evidence import iter_segments, iter_units +from app.engine.judge import SPEND_KEY as JUDGE_SPEND_KEY +from app.engine.nodes import challenge as challenge_mod +from app.engine.runner import run_manager +from app.main import app +from app.providers.mock_model import MockChatModel +from tests.fixtures.dual_check_shop import DEFINITION, RIGHT, WITH_REFUNDS, Blind, add_source, build +from tests.fixtures.sources import drop_source + + +@pytest.fixture(autouse=True) +async def engine_up(): + await run_manager.setup() + + async def clear() -> None: + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_( + ["verify", "judge", spend.SPEND_KEY, JUDGE_SPEND_KEY]))) + await session.commit() + + await clear() + yield + await clear() + await run_manager.shutdown() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def node(nid, ntype, label=None, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": label or nid, "config": config}} + + +def chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +async def wait(run_id: str, statuses=("succeeded", "failed", "interrupted")) -> Run: + for _ in range(600): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in statuses and (row.status == "interrupted" or row.manifest_seq is not None): + await run_manager.wait_idle(run_id) + return row + raise AssertionError(f"运行 {run_id} 停在 {row.status}：{row.error}") + + +async def start(spec: dict, **kwargs) -> Run: + return await wait((await run_manager.start(graph=spec, input_payload={}, **kwargs)).id) + + +async def report_of(client, run_id: str) -> dict: + body = (await client.get(f"/api/runs/{run_id}/evidence")).json() + [report] = body["reports"] + return artifact_store.load(report["doc_artifact"]) + + +async def open_segment(client, run_id: str, seg_id: str) -> dict: + res = await client.get(f"/api/runs/{run_id}/evidence/segments/{seg_id}") + assert res.status_code == 200, res.text + return res.json() + + +async def segment(client, run_id: str, text: str) -> dict: + doc = await report_of(client, run_id) + return await open_segment(client, run_id, next(s for s in iter_segments(doc) if s["text"] == text)["id"]) + + +async def sentence(client, run_id: str, text: str | None = None, *, kind: str | None = None) -> dict: + """点开一整句（句末徽标）时证据面板请求的那个片段：这一句里第一个不是结构性的片段。按原文认句子，或者按句子的 + 种类认（标题）。""" + doc = await report_of(client, run_id) + md = doc["markdown"] + unit = next(u for _, u in iter_units(doc) + if (kind is not None and u.get("kind") == kind) + or (text is not None and md[u["span"][0]:u["span"][1]].strip() == text)) + first = next(s for s in unit["segments"] if s["kind"] != "structural") + return await open_segment(client, run_id, first["id"]) + + +# -------------------------------------------------------------------------- +# 双盲复算：真跑 +# -------------------------------------------------------------------------- + +GMV = {"id": "gmv", "name": "销售额", "unit": "元", "decimals": 1, "format": "thousands", + "expression": "cell(nodes.fetch, 0, 'gmv')"} +#: 不在复算范围里的指标：没有来自查询的输入 +DAYS = {"id": "days", "name": "统计天数", "unit": "天", "format": "thousands", "expression": "7"} +WRITER = "本周销售额 [[m:gmv]]，统计 [[m:days]]。" + + +@pytest.fixture +async def shop(tmp_path): + name, source_id = await add_source(build(tmp_path / "shop.db")) + yield name + await drop_source(source_id) + + +def dual_graph(source: str, *, dual: bool = True, **config) -> dict: + """取数 → 口径卡 →（双盲复算）→ 报告 → 成果。""" + return chain( + node("start", "input"), + node("fetch", "tool", tool=f"db_query__{source}", args={"sql": RIGHT}), + node("card", "metrics", caliber="销售口径", caliber_version="v1", metrics=[GMV, DAYS]), + *([node("dual", "dual_check", "复核销售额", **{"definitions": {"gmv": DEFINITION}, **config})] if dual else []), + node("write", "report", "写周报", instructions="写一句周报", on_violation="flag"), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]), + ) + + +async def decide(client, run_id: str, choices: dict) -> Run: + async with SessionLocal() as session: + approval = (await session.execute(select(Approval).where( + Approval.run_id == run_id, Approval.status == "pending"))).scalar_one() + res = await client.post(f"/api/approvals/{approval.id}/decide", + json={"approved": True, "value": {"choices": choices, "note": ""}}) + assert res.status_code == 200, res.text + return await wait(run_id, ("succeeded", "failed")) + + +async def test_an_agreeing_dual_check_shows_on_the_metric_step(client, monkeypatch, shop): + Blind(monkeypatch, shop, {"销售额": (RIGHT, "gmv")}, writer=WRITER) + run = await start(dual_graph(shop)) + assert run.status == "succeeded", run.error + step = (await segment(client, run.id, "330.0元"))["chain"][0] + assert step["step"] == "metric" and step["metric"] == "gmv" + assert step["dual_check"] == {"status": "agreed", "node_id": "dual", "blind_rendered": "330.0元"} + # 节点跑了、这个指标没复算（它没有来自查询的输入）：照实说没复算、为什么，不是缺省 + days = (await segment(client, run.id, "7天"))["chain"][0] + assert days["dual_check"] == {"status": "skipped", "node_id": "dual", "reason": "no_inputs"} + + +async def test_a_metric_left_out_of_the_subset_is_not_said_to_lack_query_inputs(client, monkeypatch, shop): + """「复算的指标」只选了统计天数：它没有来自查询的输入，节点以 no_inputs 直通。销售额明明取自查询，只是没被选上—— + 它的说法是不在复算范围内，不能套用节点那句「没有来自查询的输入」。""" + Blind(monkeypatch, shop, {}, writer=WRITER) + run = await start(dual_graph(shop, metrics=["days"])) + assert run.status == "succeeded", run.error + gmv = (await segment(client, run.id, "330.0元"))["chain"][0]["dual_check"] + assert gmv == {"status": "skipped", "node_id": "dual", "reason": "not_targeted"} + days = (await segment(client, run.id, "7天"))["chain"][0]["dual_check"] + assert days == {"status": "skipped", "node_id": "dual", "reason": "no_inputs"} + + +@pytest.mark.parametrize("choice", ["b", None]) +async def test_a_disagreement_shows_what_the_reviewer_chose(client, monkeypatch, shop, choice): + Blind(monkeypatch, shop, {"销售额": (WITH_REFUNDS, "gmv")}, writer=WRITER) + run = await start(dual_graph(shop)) + assert run.status == "interrupted", run.error + run = await decide(client, run.id, {"gmv": choice} if choice else {}) + assert run.status == "succeeded", run.error + check = (await segment(client, run.id, "330.0元"))["chain"][0]["dual_check"] + assert check == {"status": "disagreed", "node_id": "dual", "blind_rendered": "380.0元", + **({"choice": choice} if choice else {})} + + +async def test_a_dual_check_turned_off_is_skipped(client, monkeypatch, shop): + Blind(monkeypatch, shop, {}, writer=WRITER) + run = await start(dual_graph(shop, enabled=False)) + assert run.status == "succeeded", run.error + for text in ("330.0元", "7天"): + step = (await segment(client, run.id, text))["chain"][0] + assert step["dual_check"] == {"status": "skipped", "node_id": "dual", "reason": "disabled"}, text + + +async def test_a_metric_the_blind_agent_could_not_compute_is_unchecked(client, monkeypatch, shop): + Blind(monkeypatch, shop, {}, writer=WRITER) + run = await start(dual_graph(shop)) + assert run.status == "succeeded", run.error + check = (await segment(client, run.id, "330.0元"))["chain"][0]["dual_check"] + assert check["status"] == "unchecked" and check["node_id"] == "dual" and check["reason"] + assert "choice" not in check and "blind_rendered" not in check + + +async def test_runs_without_verification_nodes_answer_as_before(client, monkeypatch, shop): + Blind(monkeypatch, shop, {}, writer=WRITER) + run = await start(dual_graph(shop, dual=False)) + assert run.status == "succeeded", run.error + body = await segment(client, run.id, "330.0元") + assert "dual_check" not in body["chain"][0] and "challenge" not in body["unit"] + + +# -------------------------------------------------------------------------- +# 反方审稿：真跑 +# -------------------------------------------------------------------------- + +PRICED = "claude-sonnet-5" +#: 各区域销售额（含退款）：华东 1500、华南 900、华北 400 +QUERY = "SELECT region AS 区域, SUM(amount) AS 销售额 FROM sales GROUP BY region ORDER BY 销售额 DESC" +NET = "SELECT region AS 区域, SUM(amount) AS 净销售额 FROM sales WHERE refunded = 0 GROUP BY region ORDER BY 净销售额 DESC" +REFUNDS = "SELECT COUNT(*) AS 退款笔数 FROM sales WHERE refunded = 1" +#: 一句有依据的结论、一句预埋的错误结论（华南其实低于华东），前面一个标题 +TEXT = "## 区域周报\n\n华东销售额 [[v:Q1.r0.销售额]]，居各区域之首。[[see:Q1]]华南销售额超过华东。[[see:Q1]]" +EAST, SOUTH = "华东销售额 1,500，居各区域之首。", "华南销售额超过华东。" +PLAYS = { + "华南销售额超过华东": ([NET], "refuted", "扣除退款后华南为 [[v:Q1.r1.净销售额]]，仍低于华东的 [[v:Q1.r0.净销售额]]。[[see:Q1]]"), + "居各区域之首": ([REFUNDS], "weakened", "华东含 [[v:Q1.r0.退款笔数]] 笔退款，扣除后仍居首，但销售额被高估。"), +} + + +@pytest.fixture +async def sales(tmp_path): + path = tmp_path / f"{uuid.uuid4().hex[:8]}.db" + db = sqlite3.connect(path) + db.executescript( + "CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, amount INTEGER, refunded INTEGER);" + "INSERT INTO sales VALUES (1, '华东', 1200, 0), (2, '华东', 300, 1), (3, '华南', 900, 0), (4, '华北', 400, 0);") + db.commit() + db.close() + name = f"sales_{uuid.uuid4().hex[:8]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True, options={}) + session.add(row) + await session.flush() + row.schema_cache = await introspect.introspect(row) + await session.commit() + source_id = row.id + yield name + await engines.invalidate(source_id) + + +def reviewer(monkeypatch, source: str) -> None: + """写作者交 TEXT；反方 agent 按任务里的结论认出是哪一句，照 PLAYS 查询、交结论和理由。""" + + def decide(model, messages): + if challenge_mod.CHALLENGE_SYSTEM not in str(messages[0].content): + return AIMessage(content=TEXT) + task = str(messages[1].content) if len(messages) > 1 else "" + queries, verdict, rationale = next((p for k, p in PLAYS.items() if k in task), ([], "stands", "没有找到反例。")) + if model.response_format: + return AIMessage(content=json.dumps({"verdict": verdict, "rationale": rationale}, ensure_ascii=False)) + done = sum(1 for m in messages if isinstance(m, ToolMessage)) + if done < len(queries): + call = {"name": f"db_query__{source}", "args": {"sql": queries[done]}, "id": f"q{done}-{uuid.uuid4().hex[:6]}"} + return AIMessage(content="查一下。", tool_calls=[call]) + return AIMessage(content="查完了。") + + monkeypatch.setattr(MockChatModel, "_decide", decide) + + async def verify_model(session, spec): + return MockChatModel(model_name=PRICED, latency=0.0), PRICED + + monkeypatch.setattr(verify_agent, "get_chat_model", verify_model) + + +def challenge_graph(source: str, *, challenge: bool = True) -> dict: + return chain( + node("start", "input"), + node("fetch", "tool", tool=f"db_query__{source}", args={"sql": QUERY}), + node("write", "report", "写周报", instructions="写周报"), + *([node("chal", "challenge", "审稿", report="write")] if challenge else []), + node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}], + contract={"report_from": "write"}), + ) + + +async def challenge_output(run_id: str) -> dict: + from app.db.models import RunEvent + + async with SessionLocal() as session: + [finished] = (await session.execute(select(RunEvent).where( + RunEvent.run_id == run_id, RunEvent.type == "node.finished", RunEvent.node_id == "chal"))).scalars() + return artifact_store.load(finished.data["artifact"]) + + +async def test_a_challenged_sentence_carries_the_verdict(client, monkeypatch, sales): + reviewer(monkeypatch, sales) + run = await start(challenge_graph(sales), run_class="formal") + assert run.status == "succeeded", run.error + output = await challenge_output(run.id) + record = output["challenge_artifact"] + south = (await sentence(client, run.id, SOUTH))["unit"]["challenge"] + # counts 是整份报告的审稿计数（节点产出里的那一份）：一句被推翻、一句被削弱，不是只数这一句 + assert south == {"verdict": "refuted", "node_id": "chal", "artifact": record, "on_refuted": "degrade", + "report_label": "写周报", "counts": {"stands": 0, "weakened": 1, "refuted": 1, "inconclusive": 0}} + assert south["counts"] == output["counts"] + # 点开句子里的数字，片段接口答的这一句同样带着判断 + east = (await segment(client, run.id, "1,500"))["unit"]["challenge"] + assert east["verdict"] == "weakened" and east["artifact"] == record and east["counts"] == output["counts"] + # 反方没审的（标题不是结论）：没有这个键 + heading = await sentence(client, run.id, kind="heading") + assert heading["unit"]["kind"] == "heading" and "challenge" not in heading["unit"] + + +async def test_a_lost_review_record_still_shows_the_listed_verdicts(client, monkeypatch, sales): + """审稿记录没存上（challenge_artifact 为空）：判断取产出里列的那几句，工件为空（理由和查询看不了）。""" + async def lost(self, record): + return None + + monkeypatch.setattr(challenge_mod._Review, "_store", lost) + reviewer(monkeypatch, sales) + run = await start(challenge_graph(sales), run_class="formal") + assert run.status == "succeeded", run.error + assert (await challenge_output(run.id))["challenge_artifact"] is None + south = (await sentence(client, run.id, SOUTH))["unit"]["challenge"] + assert south == {"verdict": "refuted", "node_id": "chal", "artifact": None, "on_refuted": "degrade", + "report_label": "写周报", "counts": {"stands": 0, "weakened": 1, "refuted": 1, "inconclusive": 0}} + + +async def test_a_review_that_stopped_the_run_still_marks_its_sentences(client, monkeypatch, sales): + """「结论被推翻时」设为报错并停止运行：节点报错、没有 node.finished，运行失败。被推翻的是哪几句、反方凭什么， + 证据面板里照样要看得到——按审完那一刻的日志（challenge_done）认审稿记录，处置写 fail。""" + reviewer(monkeypatch, sales) + graph = challenge_graph(sales) + next(n for n in graph["nodes"] if n["id"] == "chal")["data"]["config"]["on_refuted"] = "fail" + run = await start(graph, run_class="formal") + assert run.status == "failed" and "推翻" in (run.error or ""), run.error + south = (await sentence(client, run.id, SOUTH))["unit"]["challenge"] + assert south["verdict"] == "refuted" and south["on_refuted"] == "fail" and south["node_id"] == "chal" + assert south["artifact"] and south["counts"] == {"stands": 0, "weakened": 1, "refuted": 1, "inconclusive": 0} + east = (await segment(client, run.id, "1,500"))["unit"]["challenge"] + assert east["verdict"] == "weakened" and east["artifact"] == south["artifact"] + + +async def test_reports_without_a_challenge_have_no_verdicts(client, monkeypatch, sales): + reviewer(monkeypatch, sales) + run = await start(challenge_graph(sales, challenge=False), run_class="formal") + assert run.status == "succeeded", run.error + assert "challenge" not in (await sentence(client, run.id, SOUTH))["unit"] + + +# -------------------------------------------------------------------------- +# 认「哪一次」：手写的封存事件 +# -------------------------------------------------------------------------- + + +def sealed_of(graph: dict, events: list[tuple]) -> evidence._Sealed: + return evidence._Sealed(run_id="syn-run", graph=graph, events=events, + seal={"sealed": True, "ok": True, "manifest_seq": events[-1][0], "legacy": False}) + + +def card_set(artifact: str) -> dict: + return {"evidence": [{"kind": "metric_set", "node_id": "card", "artifact": artifact}]} + + +async def test_the_dual_check_belongs_to_the_card_result_it_checked(): + """口径卡在循环里算了两次：双盲复算只复算了第一次的结果，第二次的指标没有复算状态；之后又跑了一次直通的，第二次的 + 结果记成跳过，第一次的不受影响。""" + first = await put_json({"kind": "metric_set", "metrics": [{"id": "gmv", "value": 1.5}], "n": 1}, kind="metric_set") + second = await put_json({"kind": "metric_set", "metrics": [{"id": "gmv", "value": 2.5}], "n": 2}, kind="metric_set") + checked = await put_json({"status": "agreed", "counts": {"agreed": 1, "disagreed": 0, "unchecked": 0}, + "metrics": [{"metric_id": "gmv", "name": "销售额", "unit": "元", "status": "agreed", + "original_rendered": "1.5元", "blind_rendered": "1.5元", "delta_pct": 0.0}], + "model": "mock", "same_model": False}, kind="node_output") + skipped = await put_json({"status": "skipped", "reason": "disabled"}, kind="node_output") + graph = chain(node("card", "metrics"), node("dual", "dual_check", card="card")) + events = [ + (1, "node.started", "card", {}), + (2, "node.finished", "card", card_set(first)), + (3, "node.started", "dual", {}), + (4, "node.finished", "dual", {"artifact": checked, "evidence": [{ + "kind": "dual_check", "node_id": "dual", "exec": 1, "card": "card", "status": "agreed", + "counts": {"agreed": 1, "disagreed": 0, "unchecked": 0}, "metrics": [{"id": "gmv", "status": "agreed"}], + "metric_set": first}]}), + (5, "node.started", "card", {}), + (6, "node.finished", "card", card_set(second)), + ] + agreed = {"status": "agreed", "node_id": "dual", "blind_rendered": "1.5元"} + assert evidence._dual_check_of(sealed_of(graph, events), "card", first, "gmv") == agreed + assert evidence._dual_check_of(sealed_of(graph, events), "card", second, "gmv") is None + later = [*events, (7, "node.started", "dual", {}), (8, "node.finished", "dual", {"artifact": skipped})] + assert evidence._dual_check_of(sealed_of(graph, later), "card", second, "gmv") == { + "status": "skipped", "node_id": "dual", "reason": "disabled"} + assert evidence._dual_check_of(sealed_of(graph, later), "card", first, "gmv") == agreed + # 别的口径卡、卡上没有的指标：没有 + assert evidence._dual_check_of(sealed_of(graph, events), "other", first, "gmv") is None + assert evidence._dual_check_of(sealed_of(graph, events), "card", first, "ghost") == { + "status": "skipped", "node_id": "dual", "reason": "not_targeted"} + + +async def test_an_unreadable_dual_check_output_falls_back_to_its_ledger_entry(): + """产出的文件不在了：状态和审核人的选择按台账条目说，盲算值说不了就不给。""" + card = await put_json({"kind": "metric_set", "metrics": [{"id": "gmv", "value": 3.5}], "n": 3}, kind="metric_set") + graph = chain(node("card", "metrics"), node("dual", "dual_check", card="card")) + events = [ + (1, "node.finished", "card", card_set(card)), + (2, "node.finished", "dual", {"artifact": "f0" * 32, "evidence": [{ + "kind": "dual_check", "node_id": "dual", "exec": 1, "card": "card", "status": "resolved", + "metrics": [{"id": "gmv", "status": "disagreed", "choice": "a"}], "metric_set": card}]}), + ] + assert evidence._dual_check_of(sealed_of(graph, events), "card", card, "gmv") == { + "status": "disagreed", "node_id": "dual", "choice": "a"} + assert evidence._dual_check_of(sealed_of(graph, events), "card", card, "other") == { + "status": "skipped", "node_id": "dual", "reason": "not_targeted"} + + +def report(doc_artifact: str) -> evidence._Report: + return evidence._Report(node_id="write", doc_artifact=doc_artifact, ok=True, repairs=0, stats=None, doc=None, + hash_ok=True, fields=[]) + + +@pytest.mark.parametrize("saved", [True, False]) +async def test_a_review_of_an_earlier_draft_does_not_mark_the_rewritten_report(saved): + """报告在反方审稿之后又重写了一次（循环、接着跑）：审的是前一稿，新的一稿上同一个句子编号说的可能是另一句话。 + 审稿记录存上了按记录里的文档认，没存上按它跑完之前那份报告最后一次自查的文档认。""" + draft = await put_json({"markdown": "第一稿", "n": 1}, kind="report_doc") + final = await put_json({"markdown": "第二稿", "n": 2}, kind="report_doc") + claims = [{"unit": "u1", "text": "华南销售额超过华东。", "verdict": "refuted"}] + record = await put_json({"report": "write", "doc_artifact": draft, "model": "mock", + "claims": [{**c, "rationale": "", "rationale_doc": None, "queries": []} for c in claims]}, + kind="challenge") if saved else None + out = await put_json({"challenge_artifact": record, "counts": {"stands": 0, "weakened": 0, "refuted": 1, + "inconclusive": 0}, + "items": claims, "on_refuted": "flag"}, kind="node_output") + graph = chain(node("write", "report", "写周报"), node("chal", "challenge", "审稿", report="write")) + events = [ + (1, "report.checked", "write", {"doc_artifact": draft, "ok": True}), + (2, "node.finished", "write", {}), + (3, "node.finished", "chal", {"artifact": out}), + (4, "report.checked", "write", {"doc_artifact": final, "ok": True}), + ] + sealed = sealed_of(graph, events) + assert evidence._challenge_of(sealed, report(draft), "u1") == { + "verdict": "refuted", "node_id": "chal", "artifact": record, "on_refuted": "flag", "report_label": "写周报", + "counts": {"stands": 0, "weakened": 0, "refuted": 1, "inconclusive": 0}} + assert evidence._challenge_of(sealed, report(final), "u1") is None + assert evidence._challenge_of(sealed, report(draft), "u2") is None + + +async def test_the_counts_are_the_whole_reviews_and_unreviewed_rides_along(): + """counts 照产出（整份报告审了 12 句），没能反查的句数产出里有就带上；产出里的计数认不出来时按审稿记录数。""" + doc = await put_json({"markdown": "周报", "n": 3}, kind="report_doc") + claims = [{"unit": "u1", "text": "华南销售额超过华东。", "verdict": "weakened"}, + {"unit": "u2", "text": "华北持平。", "verdict": "stands"}] + record = await put_json({"report": "write", "doc_artifact": doc, "model": "mock", + "claims": [{**c, "rationale": "", "rationale_doc": None, "queries": []} for c in claims]}, + kind="challenge") + graph = chain(node("write", "report", "写周报"), node("chal", "challenge", "审稿", report="write")) + + async def answer(output: dict) -> dict: + out = await put_json({"challenge_artifact": record, "items": claims, "on_refuted": "degrade", **output}, + kind="node_output") + events = [(1, "report.checked", "write", {"doc_artifact": doc, "ok": True}), + (2, "node.finished", "chal", {"artifact": out})] + return evidence._challenge_of(sealed_of(graph, events), report(doc), "u1") + + got = await answer({"counts": {"stands": 7, "weakened": 3, "refuted": 2, "inconclusive": 0}, "unreviewed": 1}) + assert got["counts"] == {"stands": 7, "weakened": 3, "refuted": 2, "inconclusive": 0} and got["unreviewed"] == 1 + # 计数格式不对（一个数都认不出来）：按审稿记录里的句子数，不写四个 0 + got = await answer({"counts": {"stands": "7"}, "unreviewed": True}) + assert got["counts"] == {"stands": 1, "weakened": 1, "refuted": 0, "inconclusive": 0} and "unreviewed" not in got + + +async def test_a_review_that_failed_the_node_needs_its_record(): + """节点报错：认审完那一刻的日志。重跑（node.started）之后没再审完的不算；日志里没有审稿记录（保存失败）认不出审的 + 是哪一稿，也不算。""" + doc = await put_json({"markdown": "周报", "n": 4}, kind="report_doc") + claims = [{"unit": "u1", "text": "华南销售额超过华东。", "verdict": "refuted", "rationale": "", "rationale_doc": None, + "queries": []}] + record = await put_json({"report": "write", "doc_artifact": doc, "model": "mock", "claims": claims}, + kind="challenge") + graph = chain(node("write", "report", "写周报"), node("chal", "challenge", "审稿", report="write", + on_refuted="fail")) + counts = {"stands": 0, "weakened": 0, "refuted": 1, "inconclusive": 0} + head = [(1, "report.checked", "write", {"doc_artifact": doc, "ok": True}), (2, "node.started", "chal", {})] + done = (3, "log", "chal", {"code": "challenge_done", "counts": counts, "artifact": record}) + failed = (4, "node.failed", "chal", {"error": "反方审稿推翻了 1 句结论"}) + got = evidence._challenge_of(sealed_of(graph, [*head, done, failed]), report(doc), "u1") + assert got == {"verdict": "refuted", "node_id": "chal", "artifact": record, "on_refuted": "fail", + "report_label": "写周报", "counts": counts} + retried = [*head, done, (4, "node.started", "chal", {}), (5, "node.failed", "chal", {"error": "模型不可用"})] + assert evidence._challenge_of(sealed_of(graph, retried), report(doc), "u1") is None + lost = [*head, (3, "log", "chal", {"code": "challenge_done", "counts": counts, "artifact": None}), failed] + assert evidence._challenge_of(sealed_of(graph, lost), report(doc), "u1") is None diff --git a/backend/tests/test_positional_masks_elsewhere.py b/backend/tests/test_positional_masks_elsewhere.py new file mode 100644 index 00000000..654f822b --- /dev/null +++ b/backend/tests/test_positional_masks_elsewhere.py @@ -0,0 +1,113 @@ +"""按位置取自遮罩列的一格（vars.res.rows[1][2]、nodes.fetch.rows[0][0]）在推演之外的两处：期 4 I2a 把它接进了 +recompute 的还原（_positional），用 recompute 的地方都跟着变，这里把变化钉住。 + +- 双盲复算的预检（dual_check._plan_metric 什么都不改先重算一遍）：以前这种指标照常盲算，遮着的那一格还会作为字面量 + 写进给盲算模型的计算式；现在预检判它用到了遮罩列，按无法复算记，原因不说「推演」（这里不是推演） +- 运行对比（engine/compare.py）：这种输入两边的值都不给，标 masked，和 cell() 取的遮罩格一样 + +库和数字都是编的：号码写成带横线的文本，区域名是「东区」这类。 +""" +from __future__ import annotations + +import json + +from app.engine import compare +from app.engine.runner import run_manager +from tests.fixtures.compare_runs import BASE_ROWS, NEXT_ROWS, metric, scalar_in, week +from tests.test_evidence_metric_masks import ( # noqa: F401 - 夹具按名字注入（engine_up 自动生效） + METRICS, + PHONES, + SQL, + client, + engine_up, + node, + set_source, + shop, +) + + +def dual_graph(): + """取数 → 数据整形（vars.res）→ 口径卡 → 双盲复算（只复算 mix）→ 成果。""" + nodes = [ + node("start", "input"), + node("fetch", "tool", tool="db_query__shop", args={"sql": SQL}), + node("parse", "transform", mode="json", template="{{ nodes.fetch }}", assign_to="res"), + node("card", "metrics", caliber="遮罩口径", caliber_version="v1", metrics=METRICS), + node("dual", "dual_check", metrics=["mix"]), + node("out", "output", fields=[{"name": "结果", "value": "{{ nodes.card.text }}"}]), + ] + return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +async def dual_output(run_id: str) -> dict: + from sqlalchemy import select + + from app.core import artifact_store + from app.db.base import SessionLocal + from app.db.models import RunEvent + + async with SessionLocal() as session: + [finished] = (await session.execute(select(RunEvent).where( + RunEvent.run_id == run_id, RunEvent.type == "node.finished", RunEvent.node_id == "dual"))).scalars() + return artifact_store.load(finished.data["artifact"]) + + +async def test_the_dual_check_does_not_blind_compute_a_metric_reading_a_masked_cell_by_position(shop): + """mix = round(cell(nodes.fetch, 1, 'amount') + len(vars.res.rows[1][2]), 1)：cell() 那一格不在遮罩列上，按位置取的 + 号码在。预检判它用到了遮罩列，按无法复算记缺口，原因不说「推演」；没遮罩时预检照常通过、指标进入盲算。""" + import asyncio + + from app.db.base import SessionLocal + from app.db.models import Run + + async def run(options): + await set_source(shop, options) + run_id = (await run_manager.start(graph=dual_graph(), input_payload={})).id + for _ in range(400): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run_id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + await run_manager.wait_idle(run_id) + assert row.status == "succeeded", row.error + return await dual_output(run_id) + raise AssertionError(f"运行 {run_id} 没跑完：{row.status}") + + masked = await run({"mask_columns": ["phone"]}) + [mix] = masked["metrics"] + assert mix["metric_id"] == "mix" and mix["status"] == "unchecked" + assert mix["reason"] == "无法按原表达式重算这个指标（这个指标用到了设置了遮罩的列）" + assert "推演" not in json.dumps(masked, ensure_ascii=False) + assert not [p for p in PHONES if p in json.dumps(masked, ensure_ascii=False)] + assert any("遮罩" in gap for gap in masked["issuance_gaps"]) + + # 没遮罩：预检通过，原因不再是遮罩（盲算模型没配，按盲算没交回数记，但已经走到了盲算这一步） + plain = await run({}) + [mix] = plain["metrics"] + assert "遮罩" not in (mix.get("reason") or "") and "无法按原表达式重算" not in (mix.get("reason") or "") + + +def region_len(fetch, bot, rows, bot_rows): + """东区名称的字数：按位置取自取数结果的第 0 行第 0 列（region）。""" + return [metric("rlen", "len(nodes.fetch.rows[0][0])", len(rows[0][0]), + [scalar_in("nodes.fetch.rows[0][0]", rows[0][0], "fetch", "tool")], name="首个区域名字数", + unit="字", decimals=0)] + + +def test_the_compare_page_withholds_a_masked_cell_read_by_position(): + """运行对比：按位置取自遮罩列（region）的一格两边都不给值、标 masked，和 cell() 取的遮罩格一样；没遮罩时照给。""" + def masked(run): + run.masked = {a: frozenset({"region"}) for a in run.queries} + return run + + result = compare.compare(masked(week("r1", metrics=region_len)).side(), + masked(week("r2", NEXT_ROWS, metrics=region_len)).side()) + [rlen] = [m for m in result["metrics"] if m["metric_id"] == "rlen"] + [inp] = rlen["inputs"] + assert inp["masked"] is True and inp["base"] is None and inp["target"] is None + assert "东区" not in json.dumps(result, ensure_ascii=False) + + shown = compare.compare(week("r1", metrics=region_len).side(), week("r2", NEXT_ROWS, metrics=region_len).side()) + [rlen] = [m for m in shown["metrics"] if m["metric_id"] == "rlen"] + [inp] = rlen["inputs"] + assert not inp.get("masked") and inp["base"] == BASE_ROWS[0][0] and inp["target"] == NEXT_ROWS[0][0] diff --git a/backend/tests/test_recompute.py b/backend/tests/test_recompute.py index 1f5c4bcf..cb21fb40 100644 --- a/backend/tests/test_recompute.py +++ b/backend/tests/test_recompute.py @@ -29,6 +29,9 @@ BOT_OUT = "d4" * 32 FETCH_OUT = "e5" * 32 ITEMS_VALUE = "f6" * 32 RES_ROWS = "a7" * 32 +#: 数据整形节点 parse 解析好的 S1（数据源工具交回的结果原样解析，带着快照的 id）：vars.res 就是它。按位置取的 +#: vars.res.rows[0][1] 顺着它认得出取自 S1 的 amount 列；没有这份产出时那一格遮没遮核对不了，推演不给值 +PARSE_OUT = "c9" * 32 STORE = { **SNAPSHOTS, BOT_OUT: {"text": "四个区域合计 340", "data": {"amounts": [120.5, 80.0, 99.5, 40.0]}, @@ -37,6 +40,7 @@ STORE = { FETCH_OUT: {"items": ["甲", "乙", "丙"], "k": 2}, ITEMS_VALUE: {"path": "vars.kpi.items", "value": ["甲", "乙", "丙"]}, RES_ROWS: {"path": "vars.res.rows", "value": SNAPSHOTS[S1]["rows"]}, + PARSE_OUT: {**SNAPSHOTS[S1], "artifact": S1}, } @@ -47,7 +51,7 @@ def load(artifact): def sources(**kw): - base = {"load": load, "queries": (S1, S2, BAD), "outputs": {"bot": BOT_OUT, "fetch": FETCH_OUT}, + base = {"load": load, "queries": (S1, S2, BAD), "outputs": {"bot": BOT_OUT, "fetch": FETCH_OUT, "parse": PARSE_OUT}, "masked": {S2: frozenset({"phone"})}} return Sources(**{**base, **kw}) diff --git a/backend/tests/test_recompute_followups.py b/backend/tests/test_recompute_followups.py new file mode 100644 index 00000000..a3b8a802 --- /dev/null +++ b/backend/tests/test_recompute_followups.py @@ -0,0 +1,101 @@ +"""数字推演的几处补测（期 4 打磨）： + +- 取整只有一个函数：推演重算（engine/recompute.py）用的就是口径卡（engine/nodes/metrics.py）那一个，口径卡的取整 + 规则哪天改了，推演跟着变，不会一边改了、另一边还按旧的算 +- _doubt 里 moot 的那一支：一个输入既嵌在下标里、又直接参与计算，包着它的引用给了新值之后，它直接参与的那一处照常 + 拿它的值——排除行后它变没变说不准，指标仍得说不准，不能借着「嵌在里面的那一处已无关紧要」放过去 +- 决定取哪一格的单格照给行段：它自己那一行勾得了，勾掉它确定是排除后无法计算 + +夹具沿用 test_recompute 的虚构快照和卡。只导入夹具，不导入它的测试函数：导进来的 test_* 会在这里再收集一遍。 +""" +from __future__ import annotations + +import pytest + +from app.engine import recompute +from app.engine.nodes import metrics as metrics_node +from app.engine.recompute import Changes, whatif +from tests.test_recompute import AMOUNTS, FEE, REGION, S1, cell, lone, scalar, sources + +# -------------------------------------------------------------------------- +# 取整：和口径卡共用一个函数 +# -------------------------------------------------------------------------- + + +def test_the_whatif_rounds_with_the_cards_own_function(): + assert recompute.round_metric is metrics_node.round_metric + + +@pytest.mark.parametrize("value, decimals, rounded", [ + # 二进制里落在半中间的数照 Python 的 round：口径卡当时就是这么算的 + (2.675, 2, 2.67), + (0.125, 2, 0.12), + # 负的小数位：取整到十位、百位 + (1234.5, -1, 1230.0), + (-1250.0, -2, -1200.0), + # 只取整浮点：整数、文本、空值原样 + (7, 2, 7), + ("3.14159", 2, "3.14159"), + (None, 2, None), + # 没写小数位、小数位不是整数（老卡上的怪值）：不取整 + (1.25, None, 1.25), + (1.25, True, 1.25), + (1.25, 1.0, 1.25), +]) +def test_round_metric_rounds_only_floats_by_an_integer_number_of_places(value, decimals, rounded): + got = metrics_node.round_metric(value, decimals) + assert got == rounded and type(got) is type(rounded) + + +# -------------------------------------------------------------------------- +# _doubt 里 moot 的那一支 +# -------------------------------------------------------------------------- + +#: 按区域取费率，区域名本身还拿来数字数。区域名是查询交回之后才算出来的（label 节点，对不上快照） +RATE = "vars.kpi.rates[vars.kpi.region]" +TWICE = lone("fee_twice", f"round({RATE} * sum(vars.kpi.amounts), 1) + len(vars.kpi.region)", + "round(0.05 * sum(vars.kpi.amounts), 1) + len('华南')", 19.0, + [scalar(RATE, 0.05), scalar("vars.kpi.region", "华南", node="label"), AMOUNTS], decimals=1) +TIMED = {"returned": {S1: 10}, "finished": {"fetch": 5, "label": 12}} + + +def test_an_input_used_directly_and_as_an_index_stays_doubted_after_the_item_has_a_new_value(): + timed = sources(**TIMED) + assert recompute.recompute(TWICE["metrics"][0], Changes(), timed)["new"] == 19.0 + # 费率给了新值：区域名嵌在下标里的那一处已无关紧要。可它还直接参与计算（len），排除行后它变没变说不准 + [fee] = whatif(TWICE, Changes(inputs={RATE: 0.1}, exclude={S1: frozenset({1})}), timed)["metrics"] + assert (fee["status"], fee["new"], fee["changed"]) == ("unsupported", None, None), fee + assert "排除行对 vars.kpi.region 的影响无法确定" in fee["reason"] + # 对照：不排除行，费率的新值照常算进去 + [fee] = whatif(TWICE, Changes(inputs={RATE: 0.1}), timed)["metrics"] + assert (fee["status"], fee["new"]) == ("ok", round(0.1 * 340.0, 1) + 2), fee + + +def test_an_index_used_only_inside_the_item_is_still_moot(): + """对照：同一个区域名只嵌在下标里（不直接参与计算）时，费率给了新值就无关紧要，按剩下的行照常算。""" + card = lone("fee_once", f"round({RATE} * sum(vars.kpi.amounts), 1)", "round(0.05 * sum(vars.kpi.amounts), 1)", 17.0, + [scalar(RATE, 0.05), scalar("vars.kpi.region", "华南", node="label"), AMOUNTS], decimals=1) + [fee] = whatif(card, Changes(inputs={RATE: 0.1}, exclude={S1: frozenset({1})}), sources(**TIMED))["metrics"] + assert (fee["status"], fee["new"]) == ("ok", round(0.1 * (340.0 - 80.0), 1)), fee + + +# -------------------------------------------------------------------------- +# 决定取哪一格的单格照给行段 +# -------------------------------------------------------------------------- + + +def test_a_cell_that_picks_the_item_still_offers_its_own_row(): + """区域那一格决定取哪一项费率：它不能改（改它要重新取数），可它自己那一行勾得了——勾掉它，取哪一项不知道， + 确定是排除后无法计算，不是说不准。""" + card = lone("fee_pick", f"{FEE} * 100", "0.05 * 100", 5.0, [scalar(FEE, 0.05), cell("pull", 1, "region", "华南")], + decimals=1) + timed = sources(returned={S1: 10}, finished={"fetch": 5}) + baseline = whatif(card, Changes(), timed) + inputs = {i["path"]: i for i in baseline["metrics"][0]["inputs"]} + assert inputs[REGION]["row_span"] == [1, 1] and inputs[REGION]["editable"] is False + assert inputs[FEE]["editable"] is True and "row_span" not in inputs[FEE] + assert [(u["input"], u["reason"]) for u in baseline["unsupported"]] == [(REGION, recompute.SELECTOR)] + [fee] = whatif(card, Changes(exclude={S1: frozenset({1})}), timed)["metrics"] + assert (fee["status"], fee["changed"]) == ("excluded", True) and "第 2 行" in fee["reason"], fee + [row] = [i for i in fee["inputs"] if i["path"] == REGION] + assert row["excluded_rows"] == [1] diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 7d850bd5..5bdd7b37 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -473,6 +473,39 @@ export const EVIDENCE_TEXT = { loading: '正在推断来源…', failed: '未能获取推断的来源', }, + + // ---- 期 4：核查状态。指标步骤上的双盲复算、取自遮罩列的输入（反方审稿的判断复用审批卡的 CHALLENGE_TEXT）---- + /** + * 指标步骤上的一行：「双盲复算：状态（盲算 值），审核人的选择。后果」。状态词和同一节里按代入式重算的「复算一致 / + * 无法复算」分开说（两件事），用时间线上的说法（与盲算结果一致 / 不一致）；选项名照审批卡的 choices 原样加引号。 + * 运行已经结束：后果只说节点已经做了什么（记为缺口），降没降档看报告的出具横幅，这里不预告 + */ + dualCheck: { + label: '双盲复算', + status: { agreed: '与盲算结果一致', disagreed: '与盲算结果不一致', unchecked: '未能完成', skipped: '已跳过' } as Record, + /** 服务端给了一个界面不认识的状态：照写原值，不归到哪一类 */ + unknown: (status: string) => `未知状态「${status}」`, + /** 盲算的值，服务端按口径卡的格式渲染好的 */ + blind: (value: string) => `盲算 ${value}`, + /** choice 是审批卡上那个选项的原名（DUAL_CHECK_TEXT.approval.choices） */ + choice: (choice: string) => `审核人选择「${choice}」`, + noChoice: '审核人没有作出选择', + /** 采用盲算、两边都不对、没选：报告里的数没换，节点把它记成了缺口。正式运行叫出具缺口，探索运行不出具 */ + kept: (formal: boolean) => `报告中的数仍取自原流程，这个指标已记为${formal ? '出具缺口' : '缺口'}`, + /** 未能完成：同样记了缺口 */ + gap: (formal: boolean) => `这个指标已记为${formal ? '出具缺口' : '缺口'}`, + /** 跳过的原因，按这个指标说；认不出的原因代码不写 */ + skip: { + disabled: '节点或设置中已关闭', + no_inputs: '这个指标没有来自查询的输入', + not_targeted: '这个指标不在复算范围内', + } as Record, + }, + /** 指标的输入取自数据源设置了遮罩的列：标签和代入式里不显示值 */ + maskedInput: '取自设置了遮罩的列，不显示值', + /** 遮没遮核对不了（快照、取值链上的产出读不出来）：原因是服务端写好的一句 */ + unverifiedInput: (why: string) => `${why}，不显示值`, + maskedMismatch: '与快照不一致；取自设置了遮罩的列，不显示值', } as const /** @@ -3151,6 +3184,8 @@ export const RUN_COMPARE_TEXT = { pickLoading: '正在读取同一工作流的运行…', pickFailed: '无法读取同一工作流的运行', pickNone: '同一工作流没有其他运行', + /** 本次的运行记录不存在：不知道是哪个工作流，列不出候选，重试也是一样 */ + pickTargetGone: '本次运行的记录不存在，无法列出同一工作流的运行', pickUnsaved: '工作流未保存或已删除，没有可对比的运行', pickPlaceholder: '选择一次运行作为基准', /** 下拉里的一次运行：时间 · 类别和版本 · 短 id，没封存的另标 */ @@ -3182,6 +3217,8 @@ export const RUN_COMPARE_TEXT = { errNoBaseHint: '可在上方选择同一工作流的其他运行作为基准', /** 下拉里一次能选的都没有（只有本次、或者都没封存） */ errNoBaseNone: '同一工作流还没有其他已封存的运行', + /** 409、422、404 时下拉里除了本次和当前的基准，没有别的已封存的运行：换基准这条路走不通，照实说 */ + errNoOtherBase: '同一工作流没有其他可换作基准的已封存运行', viewBase: '查看基准运行', viewTarget: '查看本次运行', // 摘要 diff --git a/frontend/src/pages/runs/ComparePage.tsx b/frontend/src/pages/runs/ComparePage.tsx index 2e28c161..f56f49d7 100644 --- a/frontend/src/pages/runs/ComparePage.tsx +++ b/frontend/src/pages/runs/ComparePage.tsx @@ -68,7 +68,9 @@ export function ComparePage() { targetUnsealed: targetRun.state === 'ok' && !targetRun.data?.manifest_hash, targetGone: targetRun.state === 'error' && isGone(targetRun.error), baseGone: baseRun.state === 'error' && isGone(baseRun.error), - pickable: candidates.state === 'ok' ? (candidates.data ?? []).some((r) => r.id !== target && !!r.manifest_hash) : null, + // 能换的：已封存、不是本次、也不是当前的基准（409 时出问题的可能正是它） + pickable: candidates.state === 'ok' + ? (candidates.data ?? []).some((r) => r.id !== target && r.id !== base && !!r.manifest_hash) : null, } return ( diff --git a/frontend/src/pages/runs/compare/RunPair.tsx b/frontend/src/pages/runs/compare/RunPair.tsx index b32719d8..d04e27a0 100644 --- a/frontend/src/pages/runs/compare/RunPair.tsx +++ b/frontend/src/pages/runs/compare/RunPair.tsx @@ -120,7 +120,7 @@ function RetryButton({ onClick }: { onClick: () => void }) { * 写短 id 加原因，下拉如实反映当前选的是哪一次，其余每一项都能触发 change。 * * 列不出候选时说清楚、给重试，不停在「正在读取」：同一工作流的运行没取到；或者本次的记录没读到，不知道是哪个工作流 - * （候选的键是空的，不会自己去取） + * （候选的键是空的，不会自己去取）。本次的记录根本不存在（404）时重试也是一样，只说不存在、不给重试 */ function BasePicker({ target, base, baseRun, targetRun, candidates, unsaved, onPick }: { target: string; base: string | null; baseRun: Loaded; targetRun: Loaded @@ -129,6 +129,9 @@ function BasePicker({ target, base, baseRun, targetRun, candidates, unsaved, onP const id = useId() const now = useTicker(60_000) if (unsaved) return

{T.pickUnsaved}

+ if (candidates.state === 'idle' && targetRun.state === 'error' && isGone(targetRun.error)) { + return

{T.pickTargetGone}

+ } const failed = candidates.state === 'error' ? candidates.reload : candidates.state === 'idle' && targetRun.state === 'error' ? targetRun.reload : null if (failed) { diff --git a/frontend/src/pages/runs/compare/entry.tsx b/frontend/src/pages/runs/compare/entry.tsx index 22b64843..cee05d99 100644 --- a/frontend/src/pages/runs/compare/entry.tsx +++ b/frontend/src/pages/runs/compare/entry.tsx @@ -23,7 +23,8 @@ const NOTE_DELAY_MS = 300 * 里没保存就运行的，和工作流删掉之后 workflow_id 被置空的（服务端找不到同一工作流的运行，手动选也是 422）。 * * 菜单点完就收起：查找慢于一眨眼时挂一条提示，接口慢的时候不至于像没反应。离开运行详情（组件卸载）时取消请求、 - * 收起提示，晚到的答复不再把人带去对比页。 + * 收起提示，晚到的答复不再把人带去对比页。卸载取消不了全部：地址已经换了、新页面还没画出来（路由按过渡渲染）的那一下 + * 答复回来，组件还在，请求也没取消。所以答复回来时再看一眼地址：人已经不在点菜单时的那个运行详情上，就不跳、也不提示。 * * 写成 hook（返回造菜单项的函数）：运行详情里有提前返回，菜单项本身只能在拿到运行之后造，查找中的状态得在那之前就挂上 */ @@ -49,6 +50,9 @@ export function useCompareEntry(): (run: Run, live: boolean) => MenuItem { hint: orphan ? T.menuNoWorkflow : unsealed ? T.menuSealedOnly : busy ? T.menuFinding : undefined, onSelect: () => { const ctrl = new AbortController() + // 点菜单时所在的运行详情。只比路径：同一页里换视图、改列表筛选（改查询串）不算离开 + const from = window.location.pathname + const left = () => ctrl.signal.aborted || window.location.pathname !== from let note: number | null = null const timer = setTimeout(() => { note = toast.info(T.menuFinding, { sticky: true, key: `compare-finding:${run.id}` }) @@ -60,11 +64,11 @@ export function useCompareEntry(): (run: Run, live: boolean) => MenuItem { pending.current = { ctrl, stop } setBusy(true) api.compare.base(run.id, { signal: ctrl.signal }).then((found) => { - if (ctrl.signal.aborted) return + if (left()) return const via: CompareVia = { base: found.base_run_id, reason: found.reason } navigate(comparePath(run.id, found.base_run_id), { state: via }) }, (e: unknown) => { - if (ctrl.signal.aborted) return + if (left()) return if (e instanceof ApiError && e.kind === 'http' && e.status === 404) { toast.info(e.message, { action: { label: T.pickManually, onClick: () => navigate(comparePath(run.id)) } }) } else { diff --git a/frontend/src/pages/runs/compare/parts.tsx b/frontend/src/pages/runs/compare/parts.tsx index 944ef95a..67d4c103 100644 --- a/frontend/src/pages/runs/compare/parts.tsx +++ b/frontend/src/pages/runs/compare/parts.tsx @@ -67,10 +67,18 @@ export interface CompareErrorContext { /** 本次、基准的运行记录是不是 404 */ targetGone: boolean baseGone: boolean - /** 换基准的下拉里有没有能选的运行（已封存、不是本次）；还不知道（在读、读失败）时为 null */ + /** 换基准的下拉里有没有能换的运行（已封存、不是本次、也不是当前的基准）；还不知道（在读、读失败）时为 null */ pickable: boolean | null } +/** + * 「可在上方选择」这类下一步只在下拉里真有能换的运行时说：一次都没有时照实说没有（找默认基准时说「还没有」，换基准时说 + * 「没有其他可换的」），还不知道（在读、读失败）时先不说 + */ +function pickHint(ctx: CompareErrorContext, hint: string, none: string): string | null { + return ctx.pickable == null ? null : ctx.pickable ? hint : none +} + /** * 标题和下一步。标题说是哪一种；服务端的原因原样跟在标题后面（CompareError 里），下一步只写界面上真有的入口、 * 服务端已经写了的不再重复： @@ -80,6 +88,7 @@ export interface CompareErrorContext { * - 404：缺的是本次（targetGone）才是链接失效；缺的是基准（基准的记录 404，或者本次的记录读得到）就在上方另选； * 还不知道是哪一边时先不说； * - 找不到默认基准：本次不属于已保存的工作流同 422；下拉里有能选的才说可以在上方选，一次都没有就照实说 + * 叫人去上方换基准的几句（409、422、缺基准的 404、找不到默认基准）都先看下拉里有没有能换的（pickHint） */ function errorCopy(kind: Exclude, error: unknown, targetGone: boolean, ctx: CompareErrorContext): { title: string; hint: string | null @@ -88,18 +97,20 @@ function errorCopy(kind: Exclude, error: unknown, tar switch (kind) { case '409': { const title = code === SEAL_BROKEN ? T.errSealBroken : code === UNSEALED ? T.errUnsealed : T.errSealed - return { title, hint: code === UNSEALED && ctx.targetUnsealed ? null : T.errSealedHint } + return { title, hint: code === UNSEALED && ctx.targetUnsealed ? null : pickHint(ctx, T.errSealedHint, T.errNoOtherBase) } } case '422': - return ctx.targetUnsaved ? { title: T.errUnsaved, hint: null } : { title: T.errMismatch, hint: T.errMismatchHint } + return ctx.targetUnsaved ? { title: T.errUnsaved, hint: null } + : { title: T.errMismatch, hint: pickHint(ctx, T.errMismatchHint, T.errNoOtherBase) } case '404': return { title: T.errMissing, - hint: targetGone ? T.errMissingHint : ctx.baseGone || ctx.targetKnown ? T.errBaseMissingHint : null, + hint: targetGone ? T.errMissingHint + : ctx.baseGone || ctx.targetKnown ? pickHint(ctx, T.errBaseMissingHint, T.errNoOtherBase) : null, } case 'no-base': if (ctx.targetUnsaved) return { title: T.errUnsaved, hint: null } - return { title: T.errNoBase, hint: ctx.pickable == null ? null : ctx.pickable ? T.errNoBaseHint : T.errNoBaseNone } + return { title: T.errNoBase, hint: pickHint(ctx, T.errNoBaseHint, T.errNoBaseNone) } } } diff --git a/frontend/src/run/EvidencePanel.tsx b/frontend/src/run/EvidencePanel.tsx index 4d8ccc9c..2bc5574a 100644 --- a/frontend/src/run/EvidencePanel.tsx +++ b/frontend/src/run/EvidencePanel.tsx @@ -18,14 +18,18 @@ import { } from '../lib/evidence' import { humanizeError } from '../lib/errors' import { formatDateTime, formatNumber, NONE, shortId } from '../lib/format' -import { ABSENCE_CLAIM_TEXT, EVIDENCE_TEXT, JUDGE_TEXT, MERGE_TEXT, NODE_TYPE_LABEL, RUN_SQL_CHECK_TEXT, SQL_CHECK_TEXT, nodeTypeLabel } from '../lib/terms' +import { + ABSENCE_CLAIM_TEXT, DUAL_CHECK_TEXT, EVIDENCE_TEXT, JUDGE_TEXT, MERGE_TEXT, NODE_TYPE_LABEL, RUN_SQL_CHECK_TEXT, SQL_CHECK_TEXT, + nodeTypeLabel, +} from '../lib/terms' import { useCatalog } from '../store/catalog' import { askKey, segmentKey, useEvidence, useExplore, useVerdicts } from '../store/evidence' import type { - EvidenceBlock, EvidenceDocData, EvidenceInput, EvidenceMerge, EvidenceOnDemand, EvidenceQuoteSource, EvidenceSeal, - EvidenceSegment, EvidenceStep, EvidenceUnit, EvidenceVerdict, EvidenceViolation, + ApprovalReport, ChallengeCounts, EvidenceBlock, EvidenceDocData, EvidenceInput, EvidenceMerge, EvidenceOnDemand, + EvidenceQuoteSource, EvidenceSeal, EvidenceSegment, EvidenceStep, EvidenceUnit, EvidenceVerdict, EvidenceViolation, } from '../types' import { ArtifactViewer, ResultTable } from './AssistantStream' +import { ChallengeSummary } from './ChallengeSummary' import { useEvidenceHost } from './evidenceHost' import { EvidenceProvenance } from './EvidenceProvenance' import { CopyChip } from './Markdown' @@ -71,11 +75,81 @@ import { * 多少。推演不是出具，不写运行记录、不改报告。勾选框只在推演时出现在查询步骤的结果窗口里，只有这个指标用到的行能勾， * 表格底下带着推演结果。 * + * 核查状态（期 4）：指标步骤多一行双盲复算的结果（一致、不一致和审核人选了什么、未能完成、已跳过），选了盲算、两边都 + * 不对、没选、未能完成的写明报告里的数没换、已记为缺口；句子被反方审过的（点开整句、点开句中的数字都一样）多一块反方 + * 审稿，复用审批卡上的展示（run/ChallengeSummary），概况是整份报告真实的计数，点开这一句看反方的理由和查询，不说出具会 + * 不会降档（运行已经结束，看报告的出具横幅）。取自遮罩列的输入，服务端已把值换成「已遮罩」，标签照样写「已遮罩」，输入 + * 来源里只说核对结果、不写值；遮没遮核对不了的（快照读不出来）同样不显示值，写明原因。 + * * 四种摆法：side 从右侧弹出（宽屏），drawer 从底部抽出（窄屏），inline 在 360px 的 * 画布右栏里直接栏内展开，dock 放进页面给的一块常驻位置（记录页的「证据」页签）。 * 都不是模态的：开着面板照样能在正文里走。 */ +/** + * 期 4：证据接口在指标步骤上给的双盲复算状态（api/evidence.py 的 _dual_check_of）。没有双盲复算复核过这份指标集时 + * 没有这个键。只有证据面板读它，类型就近写在这里 + */ +interface DualCheckState { + status: 'agreed' | 'disagreed' | 'unchecked' | 'skipped' | (string & {}) + node_id?: string + /** 不一致时审核人的选择：a 采用原流程、b 采用盲算、neither 两边都不对。没选的没有 */ + choice?: string + /** 一致、不一致时的盲算值，服务端按口径卡的格式渲染好的 */ + blind_rendered?: string + /** 未能完成（unchecked）：原因（人话）；已跳过：disabled / no_inputs / not_targeted（按这个指标说），认不出的代码不显示 */ + reason?: string +} + +/** 期 4：证据接口在句子上给的反方审稿判断（api/evidence.py 的 _challenge_of）。没审到这一句时没有这个键 */ +interface UnitChallenge { + verdict: string + node_id?: string + /** 审稿记录（challenge_artifact）。服务端保存失败时为 null：判断照样有，理由和查询看不了 */ + artifact?: string | null + /** 实际生效的处置：degrade 被推翻的计入出具缺口；flag 只标注；fail 节点报错、运行停了 */ + on_refuted?: string + /** 报告撰写节点的名字 */ + report_label?: string + /** 整份报告的审稿计数（节点产出里的那一份），不是这一句的 */ + counts: ChallengeCounts + /** 没能反查的句数（期 4 起的产出才有） */ + unreviewed?: number +} + +function dualCheckOf(step?: EvidenceStep): DualCheckState | null { + const raw = (step as (EvidenceStep & { dual_check?: unknown }) | undefined)?.dual_check + return raw && typeof raw === 'object' && typeof (raw as DualCheckState).status === 'string' ? raw as DualCheckState : null +} + +const CHALLENGE_VERDICTS = ['stands', 'weakened', 'refuted', 'inconclusive'] as const + +/** + * 句子上的反方审稿判断认不认得：判断是一个词，计数四项都是数。没有计数的不画——审批卡的展示缺了计数会按列出的这一句 + * 拼一个「被推翻 1 句、其余 0 句」，读着像整份报告的结果 + */ +function challengeOf(unit: unknown): UnitChallenge | null { + const raw = unit && typeof unit === 'object' ? (unit as { challenge?: unknown }).challenge : undefined + if (!raw || typeof raw !== 'object') return null + const c = raw as UnitChallenge + const counts = c.counts && typeof c.counts === 'object' ? c.counts : null + if (typeof c.verdict !== 'string' || !c.verdict || !counts + || !CHALLENGE_VERDICTS.every((v) => typeof counts[v] === 'number' && Number.isFinite(counts[v]))) return null + return c +} + +/** 取自遮罩列的输入：服务端把值换成「已遮罩」，masked 写明是哪几列 */ +function maskedOf(input: EvidenceInput): string[] { + const raw = (input as EvidenceInput & { masked?: unknown }).masked + return Array.isArray(raw) ? raw.filter((c): c is string => typeof c === 'string' && !!c) : [] +} + +/** 遮没遮核对不了的输入（快照、取值链上的产出读不出来）：值同样换成「已遮罩」，mask_unverified 是原因 */ +function unverifiedOf(input: EvidenceInput): string { + const raw = (input as EvidenceInput & { mask_unverified?: unknown }).mask_unverified + return typeof raw === 'string' ? raw : '' +} + export type PanelMode = 'side' | 'drawer' | 'inline' | 'dock' export type PanelView = { kind: 'seg'; id: string } | { kind: 'unit'; id: string } | { kind: 'violations' } @@ -343,6 +417,7 @@ function SegmentBody({ panelId, doc, artifact, runId, runClass, seg, unit, block const chainFailed = !runId ? EVIDENCE_TEXT.noRun : docIssue ? EVIDENCE_TEXT.chainForeign : slot?.status === 'error' ? EVIDENCE_TEXT.chainMissing : '' + const segChallenge = docIssue ? null : challengeOf(detail?.unit) return ( <> @@ -399,7 +474,7 @@ function SegmentBody({ panelId, doc, artifact, runId, runClass, seg, unit, block )} {isMetric && ( - + {/* 反方审稿对这一句的判断：点句中的数字也看得到（没开结论句裁判的报告没有句末徽标，这里常是唯一的入口）。 + 正文不是封存的那一份时不用：服务端认的是封存的那份文档 */} + {segChallenge && } + {host.onNode && (trace.producers.length > 0 || trace.consumers.length > 0) && ( // 画布右栏：这段证据经过的节点，点一下在画布上选中并对准 @@ -537,6 +616,8 @@ function ClaimBody({ doc, artifact, runId, runClass, unit, block }: { const cites = (unit.cites ?? []).map((alias) => ({ alias, label: doc.catalog?.[alias]?.label as string | undefined })) // 讲方法、讲结构的句子：SQL 和字段清单直接列在挂的依据下面。正文不是封存的那份时不列（链是另一份报告的） const basis = useMemo(() => (foreign ? null : claimBasis(unit, doc)), [foreign, unit, doc]) + // 反方审稿对这一句的判断：服务端只认审的就是封存的这一份文档；正文不是那一份时不用 + const challenge = foreign ? null : challengeOf(answered?.unit) return ( <> {foreign && ( @@ -550,6 +631,7 @@ function ClaimBody({ doc, artifact, runId, runClass, unit, block }: { onDemand={foreign ? undefined : answered?.unit?.on_demand} answered={foreign ? undefined : answered?.unit?.verdict} pending={!!runId && !!first && (!slot || slot.status === 'loading')} /> + {challenge && } {cites.length ? (
    @@ -577,6 +659,32 @@ function ClaimBody({ doc, artifact, runId, runClass, unit, block }: { ) } +/** + * 反方审稿对这一句的判断（期 4），整句面板、片段面板共用。复用审批卡上的展示（run/ChallengeSummary，只读它的导出）： + * 那一块是报告级的——标题写报告名，概况是整份报告四种判断各几句（服务端给的真实计数，不凭这一句拼），条目只列这一句， + * 点开看反方的理由和它执行的查询，「显示全部」补出这份报告别的句子。判断是反方模型给的，那一块自己写明。 + * + * 不给运行 id：ChallengeSummary 只拿它查运行类别，好在审批卡上预告「出具会降档」。证据面板看的是已经结束的运行，降没 + * 降档、出没出具以报告的出具横幅（封存的出具判定）为准，这里再说一遍将来时的预告只会和横幅对不上（驳回了、不予出具、 + * 反方审稿不在成果节点上游……）。ChallengeSummary 取不到运行类别时一句和出具有关的话都不说。等它有了单句的展示 + * （只画这一句的判断和理由，不画报告级的标题、计数），换用那个 + */ +function ChallengePart({ challenge, unit, report }: { challenge: UnitChallenge; unit: EvidenceUnit; report?: string }) { + const reports = useMemo(() => [{ + node_id: report ?? '', label: challenge.report_label || report || '', + challenge: { + counts: challenge.counts, items: [{ unit: unit.id, text: unitText(unit).trim(), verdict: challenge.verdict }], + artifact: challenge.artifact ?? null, on_refuted: challenge.on_refuted, + ...(typeof challenge.unreviewed === 'number' ? { unreviewed: challenge.unreviewed } : {}), + } as ApprovalReport['challenge'], + }], [challenge, unit, report]) + return ( +
    + +
    + ) +} + /** * 模型的解释：只对结论句。判定、理由（斜体）、「模型判断 · 模型名 · 非确定」的徽标，封存之后按需追加的 * 另写「封存后追加」；到上限没判的写明哪个上限、到哪里调；改写过一次的写明原句。探索运行里还没判过的 @@ -1125,11 +1233,13 @@ function Sentence({ unit, active }: { unit: EvidenceUnit; active: string }) { } /** 指标步骤：名称、值、口径与版本（来源、升版处置）、原式、代入式（输入标签）、复算；可以展开数字推演 */ -function MetricPart({ step, entry, inputs, runId, seal, pending, failed, resolved, linkOf, workflowName, whatif, children }: { +function MetricPart({ step, entry, inputs, runId, runClass, seal, pending, failed, resolved, linkOf, workflowName, whatif, children }: { step?: EvidenceStep entry?: Record inputs: EvidenceInput[] runId?: string + /** 运行类别：双盲复算记的缺口，正式运行叫出具缺口 */ + runClass?: string /** 封存状态：逐项复核里「在不在封存范围内」只在封存完好时单独报 */ seal?: EvidenceSeal pending: boolean @@ -1168,6 +1278,8 @@ function MetricPart({ step, entry, inputs, runId, seal, pending, failed, resolve // 所依据的查询对照数据目录查出了错误级的问题：值照算，结果不可靠，出具同样按缺口降档。问题和改法在查询步骤里 const sqlFailed = step?.sql_check_failed === true || entry?.sql_check_failed === true const sqlReason = step?.sql_check_reason || entry?.sql_check_reason || EVIDENCE_TEXT.sqlCheckFallback + // 期 4：双盲复算复核过这个指标才有 + const dualCheck = dualCheckOf(step) return ( @@ -1200,6 +1312,7 @@ function MetricPart({ step, entry, inputs, runId, seal, pending, failed, resolve {upgrade && (

    {upgrade}

    )} + {dualCheck && } {missing &&

    {EVIDENCE_TEXT.missingValue}

    } {incomplete && ( @@ -1255,6 +1368,47 @@ function MetricPart({ step, entry, inputs, runId, seal, pending, failed, resolve ) } +/** + * 这个指标的双盲复算结果（期 4），一行：与盲算结果一致（带盲算值）、不一致（带盲算值和审核人选了什么）、未能完成（带原因）、 + * 已跳过（带原因）。状态词和同一节里按代入式重算的「复算」分开说：那是同一组输入再算一遍，这是另起一路独立取数， + * 标签上悬停、读屏都说得出是什么。 + * + * 选了盲算、两边都不对、没选、未能完成的，写明后果：报告里的数没换（仍取自原流程），节点把它记成了缺口——只看上面 + * 那个数，会以为已经换成了盲算值。运行已经结束，只说节点做了什么，降没降档看报告的出具横幅。 + * + * 结果都写成字，颜色只表达要不要多看一眼：一致用确定的绿（盲值是代码按原式代回算的、和原值比对的，不是模型的判断）； + * 不一致而审核人采用了原流程用提醒色，采用盲算、两边都不对、没有选的用失败色（记了缺口）；未能完成用提醒色（同样记了 + * 缺口）；跳过的中性色。认不出的跳过原因不写原因代码 + */ +function DualCheckLine({ check, formal }: { check: DualCheckState; formal: boolean }) { + const T = EVIDENCE_TEXT.dualCheck + const host = useEvidenceHost() + const { status } = check + const blind = (status === 'agreed' || status === 'disagreed') && check.blind_rendered ? T.blind(check.blind_rendered) : '' + const picked = check.choice ? DUAL_CHECK_TEXT.approval.choices[check.choice as 'a' | 'b' | 'neither'] : undefined + const detail = status === 'disagreed' ? (picked ? T.choice(picked) : T.noChoice) + : status === 'skipped' ? (check.reason ? T.skip[check.reason] ?? '' : '') + : status === 'unchecked' ? check.reason ?? '' : '' + const kept = status === 'disagreed' && check.choice !== 'a' + const after = kept ? T.kept(formal) : status === 'unchecked' ? T.gap(formal) : '' + const color = status === 'agreed' ? 'var(--st-done)' + : status === 'disagreed' ? (check.choice === 'a' ? 'var(--st-waiting)' : 'var(--st-failed)') + : status === 'unchecked' ? 'var(--st-waiting)' : 'var(--text-dim)' + return ( +

    + + {T.label}（{DUAL_CHECK_TEXT.description}）： + + {T.status[status] ?? T.unknown(String(status))} + {blind && （{blind}）} + {detail && ，{detail}} + {after && 。{after}} + {host.onNode && check.node_id && <> } +

    + ) +} + /** * 证据接口逐项复核的结果：eid 能重算、工件哈希复验、重新渲染一致、在封存范围内。 * 全过的时候不说（正常态安静）；哪一项明确没过就用失败色写出来——这时候这件证据 @@ -1283,27 +1437,38 @@ function IntegrityList({ code, items }: { code: string; items: string[] }) { function InputChip({ input, link }: { input: EvidenceInput; link?: InputLink }) { const host = useEvidenceHost() const missing = input.status === 'missing' || input.value == null + // 取自遮罩列（期 4）：服务端已把值换成「已遮罩」，这里照查询表格里遮罩格的样子写，读着不像一个真值。遮没遮核对不了的 + // （快照读不出来）一样不显示值，悬停说明原因 + const unverified = unverifiedOf(input) + const masked = !missing && (maskedOf(input).length > 0 || !!unverified) // Agent 交来、被截断切开的数组字段：对它计数、求和只算到了取回的那部分 const cut = !missing && input.truncated === true const node = input.node_id ? host.nodeLabel?.(input.node_id) || input.node_id : '' const via = input.via ? VIA_TEXT[input.via] ?? nodeTypeLabel(input.via) : '' const from = [node && `来自节点「${node}」`, via && `（${via}）`, input.role && ` · ${ROLE_TEXT[input.role] ?? input.role}`, - cut && ` · ${EVIDENCE_TEXT.inputTruncated}`].filter(Boolean).join('') + cut && ` · ${EVIDENCE_TEXT.inputTruncated}`, + masked && ` · ${unverified ? EVIDENCE_TEXT.unverifiedInput(unverified) : EVIDENCE_TEXT.maskedInput}`].filter(Boolean).join('') const style = missing || cut ? { color: 'var(--st-waiting)', borderColor: 'var(--st-waiting)' } : { color: 'var(--text)' } + const shown = masked ? EVIDENCE_TEXT.masked : valueText(input.value) const body = ( - {valueText(input.value)} ← {input.path ?? NONE}{missing ? '（缺失）' : cut ? '（已截断）' : ''} + + {masked ? {shown} : shown} ← {input.path ?? NONE} + {missing ? '（缺失）' : cut ? '（已截断）' : ''} + ) return (
  • {link ? ( ) : ( - + {body} )} @@ -1333,7 +1498,7 @@ function SourcesPart({ inputs, linkOf }: { inputs: EvidenceInput[]; linkOf: (inp
      {inputs.map((inp, i) => { - const said = inputSource(inp) + const said = maskedOf(inp).length || unverifiedOf(inp) ? maskedSource(inp) : inputSource(inp) const tone = TONE[said.tone] const link = linkOf(inp) // 查询编号写报告目录里的全局编号（接口补的 query、对上的查询步骤）；agent 字段自己的 ref 是 @@ -1372,6 +1537,18 @@ function SourcesPart({ inputs, linkOf }: { inputs: EvidenceInput[]; linkOf: (inp ) } +/** + * 取自遮罩列的输入在「输入来源」里怎么说（期 4）：核对结果照说，只是不写值——核对不一致时 inputSource 会把两边的值都写 + * 出来。另附一句为什么不显示值。遮没遮核对不了的（快照读不出来）直接说原因：卡上记的核对结果对着的那份快照已经读不出来了 + */ +function maskedSource(inp: EvidenceInput): { tone: SourceTone; text: string; reason?: string } { + const unverified = unverifiedOf(inp) + if (unverified) return { tone: 'warn', text: EVIDENCE_TEXT.unverifiedInput(unverified) } + if (inp.status === 'mismatch') return { tone: 'warn', text: EVIDENCE_TEXT.maskedMismatch } + const said = inputSource(inp) + return { ...said, reason: said.reason ? `${said.reason}；${EVIDENCE_TEXT.maskedInput}` : EVIDENCE_TEXT.maskedInput } +} + /** * 查询步骤：SQL（带复制）、结果窗口（被引用的行和格高亮）、窗口说明、打开完整快照、复核。 * diff --git a/scripts/check-bundle.mjs b/scripts/check-bundle.mjs index 087c8db6..08c0bf22 100644 --- a/scripts/check-bundle.mjs +++ b/scripts/check-bundle.mjs @@ -13,10 +13,11 @@ // 总览显示不通过；另做一次「哈希全部重算」的伪造：改报告里一个数字的引用值，文档、事件、清单、页面顶部的封存哈希 // 都重算一遍，①–④ 都对得上，只有 ⑤ 拿口径卡、快照对出来不通过，报告标题旁的徽标跟着不通过； // - 深浅色、窄屏：390 宽不横向溢出，点开的数字面板在窄屏上从底部抽出，深色下底是深的、字是浅的； -// - 样本页：tests/bundle_samples.py 用虚构数据真实生成的几份页面（遮罩快照、两份报告、早期运行、失败结束、两条出具 -// 判定、服务端缺文件、裸 NaN、别的运行的文档），逐份断言各项结果和原因文案；在上面篡改 ④ 引用链（换文档、拿掉报告、 -// 多一份没人引用的工件）、⑥ 运行信息（运行结果、类别、出具档位）、拿掉被引用的口径卡、把口径卡标成遮罩、只改页面顶部 -// 的封存哈希；宽屏点开表格里右对齐的数字不被面板盖住；不执行脚本时封存哈希、运行 ID 照样看得到，状态写「尚未核对」； +// - 样本页：tests/bundle_samples.py 用虚构数据真实生成的几份页面（遮罩快照、两份报告、早期运行、失败结束、先失败后 +// 经继续运行跑到成功的早期运行、两条出具 +// 判定、服务端缺文件、裸 NaN、别的运行的文档、口径卡有输入取自遮罩列），逐份断言各项结果和原因文案；在上面篡改 ④ +// 引用链（换文档、拿掉报告、多一份没人引用的工件）、⑥ 运行信息（运行结果、类别、出具档位）、拿掉被引用的口径卡、把 +// 报告文档标成遮罩、只改页面顶部的封存哈希；宽屏点开表格里右对齐的数字不被面板盖住；不执行脚本时封存哈希、运行 ID 照样看得到，状态写「尚未核对」； // - 记录页的导出按钮（预览用例 ?syn=bundle-provenance-*，page.route 假装导出接口）：已封存时可点、下载的文件名同服务端； // 未封存时不可点并说明原因（左边写着「已封存」却不能点时原因写在屏幕上）；服务端回 409 时提示原因；失败结束的运行 // 提示之后继续运行会重新封存。 @@ -486,7 +487,7 @@ await section('样本页：各种形状', async () => { check('生成样本页（tests/bundle_samples.py）', false, String(e?.message ?? e)) return } - check('生成样本页（tests/bundle_samples.py）', Object.keys(samples).length === 9, Object.keys(samples).join('、')) + check('生成样本页（tests/bundle_samples.py）', Object.keys(samples).length === 11, Object.keys(samples).join('、')) await sample('base', async (s, page) => { check('base：总览通过，有无法核对的项时写「可核对的项全部通过」', s.overall === 'pass' && s.text === '可核对的项全部通过', @@ -571,6 +572,12 @@ await section('样本页：各种形状', async () => { await sample('base', async (_, page) => { check('成功结束的运行不提重新封存', await page.locator('#seal-failed').isHidden()) }) + // 早期运行先失败、经继续运行跑到成功：范围里那条运行失败之后运行又继续了，它不是运行的结束；成功结束的那条在范围外 + await sample('continued', async (s) => { + check('continued：总览通过，运行结果无法核对（成功结束的那条不在封存范围内），不拿中途那条失败去对', + s.overall === 'pass' && s.meta.status?.result === 'na' && hasItem(s, 'meta', 'na', '早期运行在结束事件落库之前封存'), + JSON.stringify(s.meta.status)) + }) await sample('degraded', async (s) => { check('degraded：出具档位认最后一条出具判定（降档出具）', s.meta.tier?.result === 'pass' && s.meta.tier.text.includes('降档出具'), JSON.stringify(s.meta.tier)) @@ -594,6 +601,19 @@ await section('样本页：各种形状', async () => { check('missing-card：引用口径卡的数字无法核对，写明服务端已没有文件', hasItem(s, 'numbers', 'na', '导出时服务端已没有这份工件的文件')) check('missing-card：服务端缺文件是正常的无法核对，总览通过', s.overall === 'pass', `${s.overall}：${s.text}`) }) + // 口径卡上有输入取自遮罩列（会员号码）：卡上记着号码原值和代入式，整张卡只给 id。这是正常的无法核对，不是被改过 + await sample('masked-card', async (s, page) => { + const card = s.items.find((i) => i.check === 'artifacts' && i.text.includes('口径卡')) + check('masked-card：口径卡在 ③ 无法核对，写明「含遮罩列，未随文件附带，只能核对到哈希」，不判不通过', + card?.result === 'na' && card.text.includes('含遮罩列，未随文件附带，只能核对到哈希'), card?.text) + const metricNumber = s.items.find((i) => i.check === 'numbers' && i.text.includes('销售额')) + check('masked-card：引用这张卡的数字无法核对，原因同上', metricNumber?.result === 'na' + && metricNumber.text.includes('含遮罩列'), metricNumber?.text) + check('masked-card：总览通过（可核对的项全部通过），没有一项不通过', s.overall === 'pass' && s.text === '可核对的项全部通过' + && !s.items.some((i) => i.result === 'fail'), `${s.overall}：${s.text}`) + const html = await page.content() + check('masked-card：页面里找不到卡上那个遮罩列的号码', !html.includes('000-0001') && !html.includes('000-0002')) + }) await sample('nan', async (s) => { check('nan：内容里有裸 NaN 的事件和工件照常通过', s.overall === 'pass' && s.checks.events === 'pass' && !s.items.some((i) => i.result === 'fail'), describe(s)) @@ -648,12 +668,13 @@ await section('样本页：篡改', async () => { data.artifacts = data.artifacts.filter((a) => a.kind !== 'metric_set') return '删掉口径卡条目' }, (s) => [...turnsRed('numbers')(s), ['原因写明', hasItem(s, 'numbers', 'fail', '引用的工件不在文件的工件清单里')]]) - await tamper(base, '口径卡标成遮罩', (data) => { - const card = byKind(data, 'metric_set') - delete card.b64 - card.masked = true - return '口径卡去掉内容、标遮罩' - }, (s) => [...turnsRed('artifacts')(s), ['原因写明', hasItem(s, 'artifacts', 'fail', '只有查询快照会因遮罩不随文件附带')]]) + // 口径卡有输入取自遮罩列时服务端照样只给 id（见样本页 masked-card），报告文档从来不会：把它标成遮罩只能是改过 + await tamper(base, '报告文档标成遮罩', (data) => { + const doc = byKind(data, 'report_doc') + delete doc.b64 + doc.masked = true + return '报告文档去掉内容、标遮罩' + }, (s) => [...turnsRed('artifacts')(s), ['原因写明', hasItem(s, 'artifacts', 'fail', '只有查询快照和口径卡会因遮罩不随文件附带')]]) await tamper(base, '报告文档的一个字节', (data) => flipByte(byKind(data, 'report_doc')), (s) => [ ...turnsRed('artifacts')(s), ['数字不予核对', s.checks.numbers === 'na' && hasItem(s, 'numbers', 'na', '所在的报告文档没有通过核对')], @@ -666,6 +687,16 @@ await section('样本页：篡改', async () => { }, (s) => [...turnsRed('meta', ['events', 'manifest'])(s), ['页面显示事件里的「已完成」并标明文件里写的不同', s.meta.status?.result === 'fail' && s.meta.status.text.includes('已完成') && s.meta.status.text.includes('已失败'), JSON.stringify(s.meta.status)]]) + // 先失败、经继续运行跑到成功的早期运行，运行结果改成「已失败」：范围里那条运行失败之后运行又继续了，它不是运行的 + // 结束，不能拿它把「已失败」标成已核对 + for (const [status, label] of [['failed', '已失败'], ['cancelled', '已取消']]) { + await tamper(samples.continued, `继续运行过的早期运行的运行结果改成${label}`, (data) => { + data.meta.status = status + return `succeeded → ${status}` + }, (s) => [...turnsRed('meta', ['events', 'manifest'])(s), + ['运行结果不再标已核对，写明那条失败之后运行又继续了', s.meta.status?.result === 'fail' + && hasItem(s, 'meta', 'fail', '之后运行又继续执行'), JSON.stringify(s.meta.status)]]) + } await tamper(base, '运行类别', (data) => { data.meta.run_class = 'exploratory' return 'formal → exploratory' diff --git a/scripts/check-compare.mjs b/scripts/check-compare.mjs index 3abf62d8..f980e46d 100644 --- a/scripts/check-compare.mjs +++ b/scripts/check-compare.mjs @@ -7,10 +7,12 @@ // 404（找不到基准）只弹一句通用报错；窄屏（390px）被表格撑出横向滚动；浅色下分类标签、差异高亮看不清。 // 还有几处照服务端（engine/compare.py、api/compare.py）的实际输出才看得出来的：无法比较的指标原因只在 reasons 里； // 同一节点两条查询的 (node_id, ordinal) 相同；一边的 SQL 是 null；基准运行读不到；本次的记录读不到；不属于已保存 -// 工作流的运行；查找基准期间离开运行详情。下一步的提示要和页面上真能做的事对得上。 +// 工作流的运行；查找基准期间离开运行详情（新页面画好之后、或者地址刚换新页面还没画出来时答复才回来，走过的地址里 +// 都不能出现对比页）。下一步的提示要和页面上真能做的事对得上：下拉里没有能换的运行就不叫人去上方换，本次的记录 +// 不存在时不给注定失败的「重试」。 // // 全部用 page.route 假接口：虚构的「区域销售周报」三次已封存的正式运行和一次没封存的，运行详情、运行列表、 -// compare/base、compare 都在浏览器层答掉，别的 GET 放行（外壳要的工作流、待审批），非 GET 一律拦掉。 +// compare/base、compare、定时任务列表都在浏览器层答掉，别的 GET 放行（外壳要的工作流、待审批），非 GET 一律拦掉。 // 运行对比的两个接口只读，这里的假答复严格照接口约定的字段名和形状。 // // node scripts/check-compare.mjs @@ -181,6 +183,8 @@ const LABEL = { unchanged: '未变', data_only: '只数据变', caliber_and_data const ORDER = ['data_only', 'caliber_and_data', 'sql_changed', 'params_only', 'added', 'removed', 'incomparable', 'unchanged'] const REASON = '默认基准：同一定时任务的上一次运行' const FINDING = '正在查找上一次运行' +/** 409、422、缺基准的 404 时下拉里没有能换的运行：不叫人去上方换，照实说 */ +const NO_OTHER = '同一工作流没有其他可换作基准的已封存运行' // ---------------------------------------------------------------- 假接口 @@ -243,6 +247,8 @@ function defaults(opts) { if (m[2] === '/artifacts') return { json: [] } } if (p === '/api/approvals' && url.searchParams.get('run_id')) return { json: [] } + // 查找基准期间离开运行详情去的那一页（定时任务列表）：给空的列表，页面落定不靠服务端里有什么 + if (p === '/api/schedules') return { json: { items: [] } } return null } } @@ -356,37 +362,85 @@ await section('从运行详情进入', async () => { check('页面没有报错', errors.length === 0, errors.join('；')) }) -await section('查找基准期间：看得见在做事，离开之后不再跳走', async () => { +/** 查找基准期间离开去的那一页：定时任务列表。进去之后不再自己换地址（/data 会再换成 /data/databases），晚到的跳转盖不住 */ +const AWAY = '/schedules' +/** 那一页落定：列表读回来了（假接口给的空列表） */ +const awaySettled = (page) => page.locator('[data-schedules-page="list"] [data-schedules-empty]').waitFor({ timeout: 8000 }) +const NO_BASE = '同一工作流没有其他已封存的正式运行' + +/** + * 在运行详情里点「与上一次对比」，compare/base 的答复（answer，默认找到 B）先压住不放。交回页面、放行的函数、这一请求 + * 了结的 Promise（答完或者被取消，交回那个请求）、点了之后有没有挂出「正在查找」的提示，和点了之后走过的路径 + * （framenavigated 连 pushState、replaceState 都算）：晚到的答复哪怕只让页面在对比页一闪而过、又被别的跳转盖掉， + * 走过的路径里也留着 + */ +async function findingBase(answer = { json: { base_run_id: B, reason: 'same_schedule', base: refOf(B) } }) { let release const gate = new Promise((r) => { release = r }) - const { page, log, errors } = await open(`/runs/${T}`, - { base: () => ({ json: { base_run_id: B, reason: 'same_schedule', base: refOf(B) }, wait: gate }) }) - // compare/base 这一请求了结（答完，或者被取消） + const opened_ = await open(`/runs/${T}`, { base: () => ({ ...answer, wait: gate }) }) + const { page, log } = opened_ const ended = new Promise((resolve) => { - const done = (req) => { if (new URL(req.url()).pathname.endsWith('/compare/base')) resolve() } + const done = (req) => { if (new URL(req.url()).pathname.endsWith('/compare/base')) resolve(req) } page.on('requestfinished', done) page.on('requestfailed', done) }) await page.locator(`[data-run-detail="${T}"]`).waitFor({ timeout: 10000 }) + const visited = [] + page.on('framenavigated', (f) => { if (f === page.mainFrame()) visited.push(new URL(f.url()).pathname) }) await page.getByRole('button', { name: '更多操作' }).click() await page.locator('[data-menu=compare]').click() // 菜单点完就收起了：查找期间要有一条看得见的提示，接口慢的时候不像点了没反应 - check('点了之后有「正在查找上一次运行」的提示', await until(async () => - await page.locator('[data-toast]', { hasText: FINDING }).count() === 1, 3000)) + const noted = await until(async () => await page.locator('[data-toast]', { hasText: FINDING }).count() === 1, 3000) await until(async () => log.includes(`GET /api/runs/${T}/compare/base`), 3000) - // 改主意去了别的页面：答复晚到也不能把人拽回对比页 - await page.evaluate(() => { history.pushState({}, '', '/data'); dispatchEvent(new PopStateEvent('popstate')) }) - await page.waitForURL((u) => u.pathname.startsWith('/data'), { timeout: 5000 }) + return { ...opened_, release, ended, noted, visited } +} + +await section('查找基准期间：看得见在做事，离开之后不再跳走', async () => { + const { page, release, ended, noted, visited, errors } = await findingBase() + check('点了之后有「正在查找上一次运行」的提示', noted) + // 改主意去了别的页面。等那一页画好、运行详情卸下之后再放行：答复晚到也不能把人拽回对比页 + await page.evaluate((to) => { history.pushState({}, '', to); dispatchEvent(new PopStateEvent('popstate')) }, AWAY) + await awaySettled(page) + const unmounted = await until(async () => await page.locator('[data-run-detail]').count() === 0, 3000) release() - // 等请求了结，再给跳转留足时间：第一次进 /data 要现编那一页，主线程忙的时候晚到的跳转会落在固定的等待之后 + // 等请求了结，再给跳转留足时间：主线程忙的时候晚到的跳转会落在固定的等待之后 await Promise.race([ended, new Promise((r) => setTimeout(r, 4000))]) - const jumped = await until(async () => new URL(page.url()).pathname === '/runs/compare', 2500) + const jumped = await until(async () => visited.includes('/runs/compare'), 2500) const at = new URL(page.url()) - check('离开运行详情之后答复才回来：留在原地，不跳到对比页', !jumped && at.pathname.startsWith('/data'), at.pathname + at.search) + check('新页面画好之后答复才回来：一直留在新页面，走过的地址里从没有对比页', unmounted && !jumped && at.pathname === AWAY, + `${visited.join(' → ')}；现在 ${at.pathname}${at.search}`) check('离开之后「正在查找」的提示也收起', await page.locator('[data-toast]', { hasText: FINDING }).count() === 0) check('页面没有报错', errors.length === 0, errors.join('；')) }) +await section('查找基准期间离开、立刻放行：地址已换，新页面还没画出来', async () => { + // React Router 按过渡（startTransition）渲染换页：地址先换，新页面画好、运行详情卸下（这才取消请求）在后。这里只换 + // 地址、先不通知路由，把这段空当撑住，答复就在这时放行——检查不靠时机碰运气。找到了基准的不跳，找不到的不弹提示 + for (const [what, answer] of [['找到了基准', undefined], ['找不到基准', { status: 404, json: { detail: NO_BASE } }]]) { + const { page, release, ended, noted, visited, errors } = await findingBase(answer) + await page.evaluate((to) => { history.pushState({}, '', to) }, AWAY) + release() + const req = await Promise.race([ended, new Promise((r) => setTimeout(() => r(null), 4000))]) + // 答复处理完的标志：菜单收尾时收起「正在查找」的提示 + const handled = noted && await until(async () => await page.locator('[data-toast]', { hasText: FINDING }).count() === 0, 3000) + const held = !!req && !req.failure() && handled && await page.locator(`[data-run-detail="${T}"]`).count() === 1 + const jumped = await until(async () => visited.includes('/runs/compare'), 1000) + const told = await page.locator('[data-toast]', { hasText: NO_BASE }).count() + // 空当结束：通知路由，新页面画出来 + await page.evaluate(() => { dispatchEvent(new PopStateEvent('popstate')) }) + await awaySettled(page).catch(() => {}) + const at = new URL(page.url()) + check(`${what}：答复回来时运行详情还在、请求没被取消（撑住了空当）`, held, + `请求${req ? (req.failure() ? '被取消' : '已答完') : '没有了结'}，提示${handled ? '已收起' : '没收起'}`) + check(`${what}：不跳到对比页，走过的地址里从没有对比页，留在新页面`, !jumped && at.pathname === AWAY, + `${visited.join(' → ')}；现在 ${at.pathname}${at.search}`) + check(`${what}：不弹找不到基准的提示`, told === 0) + check(`${what}：页面没有报错`, errors.length === 0, errors.join('；')) + for (const c of opened) await c.close().catch(() => {}) + opened.clear() + } +}) + await section('不属于已保存工作流的运行：入口不可点并说明原因', async () => { const saved = RUNS[T] // 两种都没有 workflow_id：画布上没保存就运行的，和工作流删掉之后被置空的（名字还是原来的） @@ -765,6 +819,8 @@ await section('出错与找不到基准', async () => { const sealed = await open(comparePath(T, B), { compare: () => ({ status: 409, json: { detail: broken, code: 'compare_seal_broken' } }) }) await settled(sealed.page, 'error') const e409 = sealed.page.locator('[data-compare-error="409"]') + // 叫人去上方换基准的那句下一步，要等下拉读到同一工作流的运行、知道有没有能换的才写 + await until(() => has(sealed.page, '[data-compare-error="409"] [data-compare-error-hint]'), 3000) const t409 = (await e409.innerText().catch(() => '')).replace(/\s+/g, ' ') check('409 封存核对不一致：标题照实写，带上服务端的原因', (await e409.locator('[data-compare-error-title]').innerText()) === '无法对比：封存核对不一致' && t409.includes(broken), t409) @@ -790,13 +846,27 @@ await section('出错与找不到基准', async () => { RUNS[T] = savedT } - const other = await open(comparePath(T, B), { compare: () => ({ status: 422, - json: { detail: '两次运行不属于同一个工作流（「区域销售周报」与「门店日报」），不能对比', code: 'compare_mismatch' } }) }) + const mismatch = '两次运行不属于同一个工作流（「区域销售周报」与「门店日报」），不能对比' + const other = await open(comparePath(T, B), { compare: () => ({ status: 422, json: { detail: mismatch, code: 'compare_mismatch' } }) }) await settled(other.page, 'error') const e422 = other.page.locator('[data-compare-error="422"]') + await until(() => has(other.page, '[data-compare-error="422"] [data-compare-error-hint]'), 3000) check('422：写明无法对比，带上原因和下一步', (await e422.innerText()).includes('两次运行不属于同一个工作流') && (await e422.innerText()).includes('同一工作流的另一次运行'), (await e422.innerText().catch(() => '')).replace(/\s+/g, ' ')) + // 下拉里除了本次和当前的基准，没有别的已封存的运行：换基准这条路走不通，409、422 都不再叫人去上方换，照实说没有。 + // 409 的基准和本次同属一个工作流（下拉里有它，可它正是出问题的那一次），另一次没封存；422 的基准属于别的工作流 + for (const [code, answer, list] of [ + ['409', { status: 409, json: { detail: broken, code: 'compare_seal_broken' } }, [RUNS[T], RUNS[B], RUNS[U]]], + ['422', { status: 422, json: { detail: mismatch, code: 'compare_mismatch' } }, [RUNS[T], RUNS[U]]], + ]) { + const only = await open(comparePath(T, B), { compare: () => answer, list: () => ({ json: list }) }) + await settled(only.page, 'error') + const said = await until(async () => (await maybeText(only.page.locator(`[data-compare-error="${code}"] [data-compare-error-hint]`))).trim(), 3000) + const box = (await only.page.locator(`[data-compare-error="${code}"]`).innerText()).replace(/\s+/g, ' ') + check(`${code}、下拉里没有能换的：不叫人去上方换基准，写明没有其他可换作基准的已封存运行`, said === NO_OTHER && !box.includes('上方'), box) + } + const gone = await open(comparePath(T, B), { compare: () => RUN_GONE }) await settled(gone.page, 'error') check('404：写明运行记录不存在', (await gone.page.locator('[data-compare-error="404"]').innerText()).includes('运行记录不存在')) @@ -836,6 +906,17 @@ await section('出错与找不到基准', async () => { && (await missing.page.locator('[data-compare-error-title]').innerText()) === '运行记录不存在' && (await gone404.innerText()).includes('链接可能已失效') && await gone404.locator('a[href="/runs"]').count() === 1, (await gone404.innerText().catch(() => '')).replace(/\s+/g, ' ')) + // 本次的记录不存在：不知道是哪个工作流、列不出候选，重试几次都一样。换基准的位置照实说，不给「重试」；地址里带了 + // 基准的（对比回 404）也一样，整页都没有注定失败的「重试」 + const goneWithBase = await open(comparePath(T, B), { compare: () => RUN_GONE, run: (id) => (id === T ? RECORD_GONE : null) }) + await settled(goneWithBase.page, 'error') + for (const [what, p] of [['只带本次', missing.page], ['带着基准', goneWithBase.page]]) { + const area = p.locator('[data-compare-run="base"]') + const noted = await until(() => has(p, '[data-compare-run="base"] [data-compare-pick-note="target-gone"]'), 3000) + check(`本次不存在（${what}）：换基准的位置写明列不出同一工作流的运行，整页不给「重试」`, + noted && (await area.innerText()).includes('本次运行的记录不存在') && await p.getByRole('button', { name: '重试' }).count() === 0, + (await area.innerText()).replace(/\s+/g, ' ')) + } // 只带本次、找得到：自己补上基准，写明怎么选的 const auto = await open(comparePath(T)) @@ -877,6 +958,14 @@ await section('基准运行读不到（删了、链接过期）', async () => { await settled(page) check('另选一次之后照常对比', new URL(page.url()).searchParams.get('base') === B && compareHits(log, T, B) === 1) check('页面没有报错', errors.length === 0, errors.join('；')) + + // 同一工作流只剩本次和一次没封存的：下拉里没有能换的，缺基准时同样不叫人去上方另选 + const bare = await open(comparePath(T, GONE), { compare: () => ({ status: 404, + json: { detail: '作为对比基准的运行记录不存在', code: 'run_not_found' } }), list: () => ({ json: [RUNS[T], RUNS[U]] }) }) + await settled(bare.page, 'error') + const said = await until(async () => (await maybeText(bare.page.locator('[data-compare-error="404"] [data-compare-error-hint]'))).trim(), 3000) + const bareBox = (await bare.page.locator('[data-compare-error="404"]').innerText()).replace(/\s+/g, ' ') + check('缺的是基准、下拉里没有能换的：不叫人去上方另选，写明没有其他可换作基准的已封存运行', said === NO_OTHER && !bareBox.includes('上方'), bareBox) }) await section('本次的运行记录读不到：换基准不一直停在「正在读取」', async () => { diff --git a/scripts/check-evidence.mjs b/scripts/check-evidence.mjs index 3da07423..f38bee61 100644 --- a/scripts/check-evidence.mjs +++ b/scripts/check-evidence.mjs @@ -3289,6 +3289,298 @@ await section('copy-r2','按后端原文匹配的地方：后端现在的原文 got.suspect.join(' ｜ ')) }) +// ---------------------------------------------------------------- 期 4：核查状态 + +/** + * 双盲复算的几种状态，按片段挂到一期夹具的指标步骤上。形状照服务端（api/evidence.py 的 _dual_check_of）：一致、不一致 + * （审核人采用原流程 / 采用盲算 / 没选）、未能完成（带原因）、已跳过（带原因代码）。DUAL_MORE 是第二轮：跳过的另几种原因 + * （关闭了、没有来自查询的输入、认不出的代码）和「两边都不对」 + */ +const DUAL = { + s6: { status: 'agreed', node_id: 'dual', blind_rendered: '45,678.5元' }, + s8: { status: 'disagreed', node_id: 'dual', choice: 'a', blind_rendered: '9.1%' }, + s10: { status: 'disagreed', node_id: 'dual', choice: 'b', blind_rendered: '1,301单' }, + s45: { status: 'disagreed', node_id: 'dual', blind_rendered: '45,900.0元' }, + s12: { status: 'unchecked', node_id: 'dual', reason: '盲算没有交出可用的数：查询全部失败' }, + s23: { status: 'skipped', node_id: 'dual', reason: 'not_targeted' }, +} +const DUAL_MORE = { + s23: { status: 'skipped', node_id: 'dual', reason: 'disabled' }, + s49: { status: 'skipped', node_id: 'dual', reason: 'no_inputs' }, + s45: { status: 'skipped', node_id: 'dual', reason: 'no_such_reason' }, + s10: { status: 'disagreed', node_id: 'dual', choice: 'neither', blind_rendered: '1,301单' }, +} +/** 双盲复算那一行标签上的说明（DUAL_CHECK_TEXT.description）：悬停、读屏都说得出是什么 */ +const DUAL_ABOUT = '另起一路独立计算，与原结果逐项比对' +/** 虚构的反方审稿记录（challenge_artifact）：整句面板里点开那一句，按它取理由 */ +const CH_ART = 'c4'.repeat(32) +const CH_RATIONALE = '扣除退款后新客首单只占增长的一小部分，老客复购反而下降。' +/** 整份报告的审稿计数（服务端从节点产出里取）：审了 6 句，经受住 3、被削弱 1、被推翻 2 */ +const CH_COUNTS = { stands: 3, weakened: 1, refuted: 2, inconclusive: 0 } +/** 出具预告的几行（审批卡上说「出具会降档」那一套）：证据面板看的是已经结束的运行，一行都不该有 */ +const ISSUANCE_LINES = '[data-challenge-degrade], [data-challenge-degrade-note], [data-challenge-degrade-off], ' + + '[data-challenge-flag], [data-challenge-runclass]' + +/** WCAG 对比度：rgb()/rgba()/color(srgb …) 都认 */ +function contrastOf(fg, bg) { + const parse = (c) => { + const m = c.match(/color\(srgb ([\d.]+) ([\d.]+) ([\d.]+)/) + if (m) return [m[1], m[2], m[3]].map((x) => Number(x) * 255) + return (c.match(/[\d.]+/g) ?? []).slice(0, 3).map(Number) + } + const lum = (c) => { + const [r, g, b] = parse(c).map((v) => { + const x = v / 255 + return x <= 0.03928 ? x / 12.92 : ((x + 0.055) / 1.055) ** 2.4 + }) + return 0.2126 * r + 0.7152 * g + 0.0722 * b + } + const [a, b] = [lum(fg), lum(bg)].sort((x, y) => y - x) + return (a + 0.05) / (b + 0.05) +} + +await section('verify', '期 4：指标步骤的双盲复算、取自遮罩列的输入、反方审稿的判断', async () => { + // ---- 指标步骤多一行双盲复算的结果；没复核过的不画 + const r = await probe((id, body) => { + const metric = body.chain?.find((st) => st.step === 'metric') + if (metric && DUAL[id]) metric.dual_check = DUAL[id] + if (id === 's12' && metric) { + // 服务端遮罩之后的样子：订单数取自遮罩列，标签和代入式里都写「已遮罩」 + metric.substituted = metric.substituted.replace('1234', '已遮罩') + for (const st of body.chain) { + if (st.step === 'input' && st.path === 'vars.kpi.order_cnt') Object.assign(st, { value: '已遮罩', masked: ['order_cnt'] }) + } + body.redacted = { columns: ['order_cnt'], note: '这些列在数据源中设置了遮罩' } + } + if (id === 's23' && metric) { + // 遮没遮核对不了（快照读不出来）：同样不显示值，原因是服务端写好的一句，不进遮罩列 + for (const st of body.chain) { + if (st.step === 'input' && st.path === 'vars.kpi.refund_cnt') { + Object.assign(st, { value: '已遮罩', mask_unverified: '查询快照无法读取，核对不了这一格是否取自设置了遮罩的列' }) + } + } + } + return body + }) + const pg = r.page + const p = panel(pg) + const line = p.locator('[data-ev-dual-check]') + const lineText = async () => (await line.innerText().catch(() => '')).replace(/\s+/g, ' ') + for (const [id, status, words, token, none = []] of [ + ['s6', 'agreed', ['双盲复算', '与盲算结果一致', '盲算 45,678.5元'], '--st-done', ['缺口']], + ['s8', 'disagreed', ['与盲算结果不一致', '盲算 9.1%', '审核人选择「采用原流程的结果」'], '--st-waiting', ['缺口']], + ['s10', 'disagreed', ['与盲算结果不一致', '盲算 1,301单', '审核人选择「采用盲算的结果」', + '报告中的数仍取自原流程，这个指标已记为缺口'], '--st-failed', ['审核人采用盲算结果', '出具缺口', '降档']], + ['s45', 'disagreed', ['与盲算结果不一致', '盲算 45,900.0元', '审核人没有作出选择', '报告中的数仍取自原流程'], '--st-failed'], + ['s12', 'unchecked', ['未能完成', '查询全部失败', '这个指标已记为缺口'], '--st-waiting', ['无法复算']], + ['s23', 'skipped', ['已跳过', '这个指标不在复算范围内'], '--text-dim', ['缺口']], + ]) { + await openSeg(pg, id, '[data-ev-dual-check]') + const text = await lineText() + const got = await line.getAttribute('data-ev-dual-check').catch(() => null) + const color = await line.locator('span.font-medium').evaluate((el) => getComputedStyle(el).color).catch(() => '') + check(`双盲复算 ${status}（${words.slice(1).join('、') || words[0]}）：指标步骤写成一行，颜色按要不要多看一眼`, + got === status && words.every((w) => text.includes(w)) && !none.some((w) => text.includes(w)) + && color === await tokenColor(pg, 'color', `var(${token})`), `${got}：${text}（${color}）`) + await pg.keyboard.press('Escape') + await pg.waitForTimeout(80) + } + // 「双盲复算」和同一节里按代入式重算的「复算」是两件事：标签悬停、读屏都说明是什么，状态词也不同 + await openSeg(pg, 's12', '[data-ev-dual-check]') + const label = p.locator('[data-ev-dual-check-label]') + const recompute = await p.locator('[data-ev-recompute]').innerText().catch(() => '') + check('双盲复算的标签悬停写明是什么（另起一路独立计算），读屏同样读得到', await label.getAttribute('title') === DUAL_ABOUT + && (await label.locator('.sr-only').innerText().catch(() => '')).includes(DUAL_ABOUT)) + check('双盲复算的状态词和「复算」那一行分得开：未能完成 ≠ 无法复算', (await lineText()).includes('未能完成') + && !(await lineText()).includes('无法复算') && !!recompute, `${await lineText()} ｜ ${recompute}`) + await pg.keyboard.press('Escape') + for (const theme of ['light', 'dark']) { + await pg.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme) + let worst = Infinity + for (const id of ['s6', 's10', 's8', 's23']) { + await openSeg(pg, id, '[data-ev-dual-check]') + const pair = await line.evaluate((el) => { + const back = (node) => { + for (let n = node; n; n = n.parentElement) { + const c = getComputedStyle(n).backgroundColor + if (c && !/rgba\(0, 0, 0, 0\)|transparent/.test(c)) return c + } + return getComputedStyle(document.body).backgroundColor + } + return [...el.querySelectorAll('span')].filter((sp) => !sp.classList.contains('sr-only')) + .map((sp) => [getComputedStyle(sp).color, back(sp)]) + }).catch(() => []) + for (const [fg, bg] of pair) worst = Math.min(worst, contrastOf(fg, bg)) + await pg.keyboard.press('Escape') + await pg.waitForTimeout(80) + } + check(`${theme}：双盲复算那一行的字对比度都 ≥ 4.5`, worst >= 4.5, worst.toFixed(2)) + } + await pg.evaluate(() => document.documentElement.removeAttribute('data-theme')) + await openSeg(pg, 's49', '[data-ev-substituted]') + check('没有双盲复算复核过的指标：不画这一行', await line.count() === 0) + await pg.keyboard.press('Escape') + + // ---- 取自遮罩列的输入：标签写「已遮罩」（遮罩格的样子），代入式里也是，底下写明遮罩不是安全边界 + await openSeg(pg, 's12', '[data-ev-substituted]') + const masked = p.locator('[data-ev-input="vars.kpi.order_cnt"]') + const chip = (await masked.innerText().catch(() => '')).replace(/\s+/g, ' ') + const italic = await masked.locator('.italic').evaluate((el) => getComputedStyle(el).fontStyle).catch(() => '') + check('遮罩列的输入：标签写「已遮罩 ← 路径」，按遮罩格的样子画，不露出原值', + await masked.getAttribute('data-ev-input-masked') === '' && chip.includes('已遮罩 ← vars.kpi.order_cnt') + && !chip.includes('1,234') && italic === 'italic', `${chip}（${italic}）`) + const sub = await p.locator('[data-ev-substituted]').innerText().catch(() => '') + check('遮罩列的输入：代入式里那一处写「已遮罩」', sub === 'round(45678.5 / 已遮罩, 1)', sub) + check('没用到遮罩列的输入照常写值', (await p.locator('[data-ev-input="vars.kpi.gmv"]').innerText()).includes('45,678.5 ← vars.kpi.gmv') + && await p.locator('[data-ev-input="vars.kpi.gmv"]').getAttribute('data-ev-input-masked') === null) + check('面板底下写明遮罩的列和遮罩不是安全边界', (await p.locator('[data-ev-mask-note]').innerText().catch(() => '')).includes('order_cnt')) + await pg.keyboard.press('Escape') + // 遮没遮核对不了：同样写「已遮罩」，悬停说明原因（快照读不出来），不算进遮罩列 + await openSeg(pg, 's23', '[data-ev-substituted]') + const unsure = p.locator('[data-ev-input="vars.kpi.refund_cnt"]') + const unsureTitle = await unsure.getAttribute('title').catch(() => '') + check('遮没遮核对不了的输入：标签写「已遮罩」，悬停写明快照读不出来、不显示值', + await unsure.getAttribute('data-ev-input-masked') === '' && (await unsure.innerText()).includes('已遮罩') + && (unsureTitle ?? '').includes('查询快照无法读取，核对不了这一格是否取自设置了遮罩的列，不显示值'), unsureTitle ?? '') + check('……面板底下的遮罩说明不把它算进遮罩列', await p.locator('[data-ev-mask-note]').count() === 0) + await pg.keyboard.press('Escape') + check('核查状态那几页没有运行时报错', r.errors.length === 0, r.errors.join(' | ')) + await r.ctx.close() + + // ---- 第二轮：跳过的另几种原因、「两边都不对」。原因代码认不出来的不写进句子；关闭了的不说两遍「双盲复算」 + const r2 = await probe((id, body) => { + const metric = body.chain?.find((st) => st.step === 'metric') + if (metric && DUAL_MORE[id]) metric.dual_check = DUAL_MORE[id] + return body + }) + const line2 = panel(r2.page).locator('[data-ev-dual-check]') + for (const [id, words, none] of [ + ['s23', ['已跳过', '节点或设置中已关闭'], ['双盲复算已关闭']], + ['s49', ['已跳过', '这个指标没有来自查询的输入'], ['口径卡上没有']], + ['s45', ['已跳过'], ['no_such_reason', '，']], + ['s10', ['审核人选择「两边都不对」', '报告中的数仍取自原流程，这个指标已记为缺口'], ['审核人判断']], + ]) { + await openSeg(r2.page, id, '[data-ev-dual-check]') + const text = (await line2.innerText().catch(() => '')).replace(/\s+/g, ' ').replace(`（${DUAL_ABOUT}）`, '') + check(`双盲复算 ${id}：${words.join('、')}${none.length ? `，不写「${none.join('」「')}」` : ''}`, + words.every((w) => text.includes(w)) && !none.some((w) => text.includes(w)), text) + await r2.page.keyboard.press('Escape') + await r2.page.waitForTimeout(80) + } + check('第二轮没有运行时报错', r2.errors.length === 0, r2.errors.join(' | ')) + await r2.ctx.close() + + // ---- 输入来源：遮罩列的 Agent 字段核对不一致时只说不一致，不写两边的值 + const q = await probeQ((id, body) => { + if (body.segment?.text !== '37.0元') return body + for (const st of body.chain) { + if (st.step === 'input' && st.path === 'vars.kpi.order_cnt') { + delete st.model_value + Object.assign(st, { value: '已遮罩', masked: ['order_cnt'] }) + } + } + return body + }) + await openQ(q.page, '37.0元', { wait: '[data-ev-sources]' }) + const src = await panel(q.page).locator('[data-ev-source="vars.kpi.order_cnt"]').innerText().catch(() => '') + check('输入来源：遮罩列的字段核对不一致，只说不一致、不写值', src.includes('与快照不一致；取自设置了遮罩的列，不显示值') + && !src.includes('1,240') && !src.includes('1,234'), src.replace(/\s+/g, ' ')) + const verified = await panel(q.page).locator('[data-ev-source="vars.kpi.gmv"]').innerText().catch(() => '') + check('输入来源：没遮罩的字段照旧', verified.includes('与快照一致') && !verified.includes('遮罩'), verified.replace(/\s+/g, ' ')) + await q.ctx.close() + + // ---- 反方审稿：整句面板、片段面板都看得到这一句的判断。复用审批卡的展示，概况是整份报告的真实计数，不预告出具 + const u = fxj.formal.units + const challenge = (verdict) => ({ verdict, node_id: 'chal', artifact: CH_ART, on_refuted: 'degrade', report_label: '写周报', + counts: CH_COUNTS }) + const segPatch = (jset, body) => { + if (jset === fxj.formal && body.unit?.id === u.bad) body.unit.challenge = challenge('refuted') + if (jset === fxj.formal && body.unit?.id === u.ok) { + body.unit.challenge = challenge('weakened') + // 正式运行里的一个指标：双盲复算不一致、审核人采用盲算。记的是出具缺口 + if (body.segment?.id === 's6') { + body.chain = [{ step: 'metric', metric: 'gmv', name: '销售额', value: 45678.5, rendered: '45,678.5', unit: '元', + expression: 'vars.kpi.gmv', substituted: '45678.5', recompute_ok: true, + dual_check: { status: 'disagreed', node_id: 'dual', choice: 'b', blind_rendered: '45,900.0元' } }] + } + } + // 计数缺了（格式不对）：不画，免得审批卡的展示按这一句拼一个「被推翻 1 句、其余 0 句」 + if (jset === fxj.formal && body.unit?.id === u.limit) body.unit.challenge = { verdict: 'refuted', node_id: 'chal' } + // 没开结论句裁判的报告：没有句末徽标，点句中的数字也要看得到 + if (jset === fxj.plain && body.unit?.id === 'u3') body.unit.challenge = challenge('refuted') + return body + } + for (const [w, h] of [[1280, 1000], [390, 844]]) { + const c = await open('/ui-harness.html?evidence=1', { segPatch, w, h }) + const artifactHits = [] + await c.page.route((url) => new URL(url).pathname === `/api/artifacts/${CH_ART}`, (route) => { + artifactHits.push(route.request().url()) + return route.fulfill({ json: { id: CH_ART, content: { + report: 'write', doc_artifact: fxj.formal.doc_artifact, model: 'challenger-x', + claims: [{ unit: u.bad, text: '增长主要来自新客首单，老客复购持平。', verdict: 'refuted', rationale: CH_RATIONALE, + rationale_doc: null, queries: [] }], + } } }) + }) + const cp = panel(c.page) + const block = cp.locator('[data-ev-challenge]') + const blockText = async () => (await block.innerText().catch(() => '')).replace(/\s+/g, ' ') + await badge(c.page, J, u.bad).click() + await c.page.waitForSelector('[data-evidence-panel] [data-ev-challenge] [data-challenge-item]', { timeout: 4000 }).catch(() => {}) + const text = await blockText() + if (w === 1280) { + check('整句面板：反方审稿的判断（被推翻）挂在这一句下，写明是哪份报告的审稿', await block.getAttribute('data-ev-challenge', { timeout: 3000 }).catch(() => null) === 'refuted' + && text.includes('报告「写周报」的反方审稿') && text.includes('被推翻') && text.includes('新客首单'), text) + check('……概况是整份报告的真实计数（服务端给的），不凭这一句拼', text.includes('经受住 3 句') && text.includes('被削弱 1 句') + && text.includes('被推翻 2 句') && text.includes('无法判断 0 句') && await block.locator('[data-challenge-more="5"]').count() === 1, text) + check('……运行已经结束：不预告「出具会降档」，降没降档看报告的出具横幅', await block.locator(ISSUANCE_LINES).count() === 0 + && !text.includes('降档'), text) + check('……判断由反方模型给出，并非系统核对', text.includes('并非系统核对'), text) + await block.locator('[data-challenge-toggle]').click() + await c.page.waitForSelector('[data-evidence-panel] [data-challenge-detail-state="ok"]', { timeout: 4000 }).catch(() => {}) + const why = await block.locator('[data-challenge-rationale]').innerText().catch(() => '') + check('……点开看得到反方的理由（按审稿记录取）', why.includes(CH_RATIONALE) && artifactHits.length === 1, why) + // 被削弱的那一句：概况照样是整份报告的数，不出现凭这一句拼的「被推翻 0 句」 + await badge(c.page, J, u.ok).click() + await c.page.waitForSelector('[data-evidence-panel] [data-ev-challenge="weakened"]', { timeout: 4000 }).catch(() => {}) + const weak = await blockText() + check('被削弱的那一句：概况写整份报告的计数（被推翻 2 句），不写凭这一句拼的「被推翻 0 句」', weak.includes('被推翻 2 句') + && !weak.includes('被推翻 0 句') && !weak.includes('被削弱 1 句 · 被推翻 0 句'), weak) + // 片段面板：点这一句里的数字，同样看得到判断；正式运行里双盲复算记的是出具缺口 + await c.page.locator(`${J} [data-seg="s6"]`).click() + await c.page.waitForSelector('[data-evidence-panel] [data-ev-challenge="weakened"]', { timeout: 4000 }).catch(() => {}) + check('片段面板：点句中的数字，看得到这一句的反方审稿判断', await block.getAttribute('data-ev-challenge', { timeout: 3000 }).catch(() => null) === 'weakened' + && (await blockText()).includes('被削弱'), await blockText()) + const dual = (await cp.locator('[data-ev-dual-check]').innerText().catch(() => '')).replace(/\s+/g, ' ') + check('……正式运行：审核人采用盲算的，写明报告中的数仍取自原流程、已记为出具缺口', + dual.includes('审核人选择「采用盲算的结果」') && dual.includes('报告中的数仍取自原流程，这个指标已记为出具缺口'), dual) + // 计数格式不对的：不画 + await badge(c.page, J, u.limit).click() + await c.page.waitForSelector('[data-evidence-panel] [data-ev-judge]', { timeout: 4000 }).catch(() => {}) + await c.page.waitForTimeout(200) + check('服务端没给计数的判断：不画（不拼一个只数这一句的概况）', await block.count() === 0) + await badge(c.page, J, u.partial).click() + await c.page.waitForSelector('[data-evidence-panel] [data-ev-judge="partial"]', { timeout: 4000 }).catch(() => {}) + await c.page.waitForTimeout(200) + check('没审到的句子：不画反方审稿', await block.count() === 0) + // 没开结论句裁判的报告：没有句末徽标，点句中的数字照样看得到判断 + await c.page.keyboard.press('Escape') + await c.page.locator(`${JP} [data-seg="s14"]`).click() + await c.page.waitForSelector('[data-evidence-panel] [data-ev-challenge]', { timeout: 4000 }).catch(() => {}) + check('没开结论句裁判的报告：点句中的数字看得到反方审稿判断', await block.getAttribute('data-ev-challenge', { timeout: 3000 }).catch(() => null) === 'refuted' + && (await blockText()).includes('退款 29 单'), await blockText()) + } else { + const wide = await c.page.evaluate(() => { + const el = document.querySelector('[data-evidence-panel]') + return { page: document.documentElement.scrollWidth - document.documentElement.clientWidth, + panel: el ? el.scrollWidth - el.clientWidth : 99 } + }) + check('390px：整句面板带着反方审稿不横向溢出', !!text && wide.page <= 0 && wide.panel <= 1, JSON.stringify(wide)) + } + check(`${w}px：没有运行时报错`, c.errors.length === 0, c.errors.join(' | ')) + await c.ctx.close() + } +}) + check('没有运行时报错（组件预览）', harness.errors.length === 0, harness.errors.join(' | ')) await harness.ctx.close() await browser.close()