From 91aa34d01498811167d69000a2807b4050bafd40 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 12 Oct 2026 21:12:57 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9C=9F=205=20=E9=AA=8C=E6=94=B6=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=20N1=20=E7=AC=AC=203=20=E8=BD=AE=EF=BC=9A=E8=BD=AC?= =?UTF-8?q?=E6=8A=84=E4=B8=8A=E6=B8=B8=E5=8F=A3=E5=BE=84=E5=8D=A1=E7=9A=84?= =?UTF-8?q?=E9=81=AE=E7=BD=A9=E6=94=B9=E5=9C=A8=E6=8C=87=E6=A0=87=E4=BE=9D?= =?UTF-8?q?=E8=B5=96=E5=9B=BE=E4=B8=8A=E4=B8=80=E6=AC=A1=E7=AE=97=E5=87=BA?= =?UTF-8?q?=E6=9C=80=E5=B0=8F=E4=B8=8D=E5=8A=A8=E7=82=B9=EF=BC=9B=E6=98=BE?= =?UTF-8?q?=E7=A4=BA=E6=96=87=E5=AD=97=E6=A0=B8=E5=AF=B9=E7=B4=A7=E6=8C=A8?= =?UTF-8?q?=E7=9D=80=E7=9A=84=E5=AD=97=E6=94=B9=E6=88=90=E7=99=BD=E5=90=8D?= =?UTF-8?q?=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 复核第 2 轮(ce71f7d)找到两处 major、两处 minor:转抄的识别逐个往上递归、追够八层(CARD_DEPTH)就按遮罩算——没设任何遮罩 的工作流里,循环里累加自己上一轮产出的口径卡在写作目录里指回自己,一直追到八层,写成「已遮罩」,正式运行判失败、报错说 「取自设置了遮罩的列」;每个指标的护栏 nodes.acc 选中整张卡的全部指标,递归次数约为(指标数 + 1)的八次方,四个指标四十秒、 卡住事件循环;封存之后循环超过八轮的,证据面板写「已遮罩」,可验证报告把文档整份只给哈希。紧挨着改字仍是黑名单:西里尔、 希腊的 I、O、M、K,汉字「丨」,换行、制表、各种宽度的空格后接「万」,「(万元)」,负号前面放一个别的文字系统里长得像撇号的 字母,都还是绿色的「全部通过」、提示也不显示。另有:* 展开派生表、WITH 里没起别名的 UPPER(phone) 认不出是确定取自遮罩列, 号码进了写作目录和文档;上游卡的节点产出读不出来时转抄当作认不出出处,号码随卡 2 的节点完成事件出去了。 转抄的遮罩在指标依赖图上一次认(recompute.masked_closure,写作目录、证据面板、推演、可验证报告共用) - 先按转抄以外的输入认每个指标(写作目录、可验证报告认老文档只按确定取自遮罩列的 sure;证据面板、推演宁可多遮),再沿转抄的 边(nodes.<卡>.metrics[i].value、vars..metrics[i]、整张卡 / 清单文字 / 整份指标列表 / 下标不是字面量的连到那张卡 的全部指标)往下游传,直到不再变化。从「都照给」起往上收:环(循环里抄自己上一轮、几张卡互相转抄)本身不带来遮罩,环上有 一个指标别的输入取自遮罩格,整个环都不给。每个指标的结论只往严里变,工作量和边数成正比;删掉 CARD_DEPTH、CARD_TOO_DEEP 和逐层递归(_InputMasks._copied、_Store._copied_kind),可验证报告的缓存不再按 (卡, 指标) 混着深度记 - api/evidence.py 的 _CopyGraph(按请求记在 _Masks 上,键是 (指标集工件, 指标在卡上的位置)):写作目录只有最后一次的产出, 抄自己上一轮的卡是一个环;封存之后按每次执行认(_card_outputs,同一次执行只找一次),一次一次往回连到第一轮。bundle/build.py 的 _Store.metric_word 按同样的键各认法一张图 - 推演(api/whatif.py → copied_inputs)的转抄输入按证据面板同一张图认:转抄的是不给的指标就不给值、不能改(原因「转抄的口径卡 指标取自设置了遮罩的列」),核对不了的不给、不推演。以前只按值认,转抄来的三位数余数照给、还能改 - 上游卡当时的产出读不出来(文件不在、与哈希不一致)、卡上又记着这个输入出自口径卡、转抄来了值的,转抄核对不了(同按位置取的 一格取值链上的产出读不出来),可验证报告里卡 2 只给哈希;第一轮还没有上一轮(转抄来的是空值)的不算 - 可验证报告附带口径卡另看卡上记着的转抄值(_copies_masked):转抄的是不给、核对不了的指标,或者整张卡、清单文字转抄来的值里 写着号码(len(nodes.card.text) 的输入就是整段清单文字),卡只给 id。指标的值照给时以前照常附带 - 确定取自遮罩列(_sure)在整份算的快照里另按列名认(_named_from_masks):SQLite、MySQL 拿没起别名的表达式原文当列名,* 展开 派生表、WITH、UNION 一支时原样带出;按列名拼一条只选这一项的 SQL 交给同一套认,COUNT(phone)、FILTER、分档标签照旧不算。 单元格引用的自查、收遮罩值、裁判摘录跟着收紧 显示文字核对(verifier.js):紧挨着数字头尾的那一个字改成白名单(COMMON) - 名单:汉字(含「一」,去掉丨、丶、丿这类笔画字)、中文标点、半角和全角空格、ASCII 字母、常用半角标点和 = + / ~ * # < > 等 算式符号、× ÷ ≈ ≤ ≥、上一轮放行的破折号写法(——、—、–、―、﹘、ー、─、〜、〰);逗号、句点、连字符看上下文(句点在数字头 前面只认 NO. 这种字母缩写)。名单以外的(别的文字系统的字母、制表换行和各种宽度的空格、零宽字符、全角拉丁字母、表情)判 无法核对,原因「紧挨着这个数的字不在常见写法里,无法确认读者看到的数」,总览随之不是绿的、显示去系统里核对的提示 - 像数字的字母另收 o、竖线一类(|、|、丨、竖排破折号、制表符竖线、拉丁 / 西里尔 / 希腊里长得像 l、I 的)和圆圈一类(西里尔、 希腊的 O、空心圆);量级单位另收十、拾、佰、仟和当「万」用的 w、W,前面隔着的改成任何空白和左括号、左引号(「(万元)」) - 负号前面的「编号」例外只认 ASCII 字母数字;后面接 e 和数字(科学记数法)同紧挨着数字,判不通过 - 开发数据 39 份报告 1591 处、验收数据 58 份报告 1695 处引用按新规则扫过,没有一处判不通过或无法核对;prose、formats 样本原样 导出全绿,别的样本页总览和第 2 轮一字不差;复核者的 42 种伪造写法(forge.py、forge2.py)全部不是绿色、提示都显示 测试 - 新增 tests/test_report_masked_copy_closure.py:没设遮罩的循环自引用照常显示、正式运行成功(复核者的探针);循环三轮、十轮的 老文档照常附带;两张卡互相转抄照给;环上一个输入取自号码列整个环都不给、同循环里不相干的卡照给;上游产出读不出来三种转抄 写法都不漏号码;卡上记着转抄清单文字的卡不附带;推演按同一张图认转抄输入;* 展开里没起别名的号码表达式八种写法算确定取自 遮罩列、只数个数和分档标签的不算;纯函数的环、整张卡转抄、读不出来、两千个指标线性;写作目录和可验证报告在三张卡各 8 个 指标、循环十轮时各一秒以内(写作目录每个指标只认一次),另有真跑两张卡各 8 个指标十轮的计时 - test_report_masked_card_copies:认转抄补读不出来的几种;「追不到头按遮罩」改成环上认(没别的输入照给、有号码输入整个环不给) - test_bundle_shown:名单以外的字、像数字的字母、隔着各种空白和左括号的量级单位、别的文字系统字母后的负号、科学记数法各自判 无法核对或不通过、总览不是绿的;常见半角标点、算式符号照常通过;表情紧挨着数字改判无法核对(隔一个空格照常通过) - check-bundle 补长得像 I、M 的西里尔字母、汉字「丨」、数学粗体 l、隔着细空格的一位数、开尔文符号、隔着换行和 em 空格的「万」、 「(万元)」判无法核对,撇号字母后的负号、e3 判不通过,各自点名、总览不是绿的 - 每处改动都做了「去掉实现、测试变红」:整套退回第 2 轮(27 项红、写作目录计时十秒没认完、可验证报告长循环把文档只给哈希); 环按遮罩算、不沿边往下游传、读不出来当认不出出处、不看卡上的转抄值、不按列名认、推演不用依赖图、每个指标各建一张图;名单 不判、名单去掉等号、不去笔画字、像数字的字母不扩、量级单位前只隔空格、负号例外按文字区、科学记数法不算、单位不收十和 w、 句点不看前面是不是字母 --- backend/app/api/evidence.py | 316 ++++++-- backend/app/api/whatif.py | 14 +- backend/app/bundle/build.py | 187 +++-- backend/app/bundle/verifier.js | 108 ++- backend/app/engine/recompute.py | 222 ++++- backend/tests/test_bundle_shown.py | 62 +- .../tests/test_report_masked_card_copies.py | 77 +- .../tests/test_report_masked_copy_closure.py | 766 ++++++++++++++++++ scripts/check-bundle.mjs | 32 +- 9 files changed, 1541 insertions(+), 243 deletions(-) create mode 100644 backend/tests/test_report_masked_copy_closure.py diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index 611d0f30..b7747d75 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -22,7 +22,8 @@ options.mask_columns 里的列换成「已遮罩」,SQL 里给它们起了别 (_mask_sets,和可验证报告同一个函数,合并查询另按合并 SQL 认);用到这些列的指标,指标步骤的值、 证据图的标签也不给(_metric_masks,和推演同一个判断;报告撰写节点写作之前按同一个判断把这类指标在目录里写成「已遮罩」, withheld_metrics,只是遮罩列只认确定的那一套,和单元格引用的自查同一个口径;可验证报告认老文档也按这个口径, -certain_masks);SQL、合并警告里写着的遮罩值(这次运行里确定取自遮罩列的值) +certain_masks;转抄上游口径卡的指标在指标依赖图上一次认,_CopyGraph → recompute.masked_closure,推演的转抄输入按同一张图, +copied_inputs);SQL、合并警告里写着的遮罩值(这次运行里确定取自遮罩列的值) 展示时换掉。一份快照的遮罩分析、收到的值按快照工件记在进程里(_MEMO),之后的请求不再读盘、不再分析 ——在有身份体系之前,这只减少暴露,不是安全边界:知道工件 id 的人照样能取到整份快照。 @@ -413,7 +414,8 @@ async def _withheld_labels(evidence: list[dict[str, Any]], doc: dict[str, Any], metric = _metric_in(cards[artifact], (entry.get("locator") or {}).get("metric")) if metric is None or metric.get("value") is None: continue - _, word = await _metric_masks(metric, str(entry.get("node_id") or ""), artifact, sealed, masks) + _, word = await _metric_masks(metric, str(entry.get("node_id") or ""), artifact, sealed, masks, + card=cards[artifact]) if word is not None: item["label"] = f"{entry.get('name') or metric.get('name') or metric.get('id')} = {word}" @@ -918,7 +920,7 @@ async def _metric_chain(seg: dict[str, Any], cite: dict[str, Any], entry: dict[s inputs: list[dict[str, Any]] = [] #: 输入指到的查询快照 → 被引用的格;同一份快照只给一个查询步骤,格合在一起高亮 wanted: dict[str, list[tuple[Any, Any]]] = {} - held, word = await _metric_masks(metric or {}, str(entry.get("node_id") or ""), artifact, sealed, masks) + held, word = await _metric_masks(metric or {}, str(entry.get("node_id") or ""), artifact, sealed, masks, card=card) for item in (metric or {}).get("inputs") or []: one = {"step": "input", **item} artifact_of = item.get("artifact") @@ -980,51 +982,153 @@ def _held_word(held: _Held) -> str: async def _metric_masks(metric: dict[str, Any], card_node: str, artifact: str, sealed: _Sealed, - masks: _Masks, *, output: Callable[[Any], Any] | None = None, by_value: bool = True, - sure: bool = False, depth: int = 0) -> tuple[dict[str, _Held], str | None]: + masks: _Masks, *, card: dict[str, Any] | None = None, + output: Callable[[Any], Any] | None = None, by_value: bool = True, sure: bool = False + ) -> tuple[dict[str, _Held], str | None]: """一个指标的输入里哪些不给值(路径 → _Held),指标本身的值给不给(None 照给,否则是写在值那里的字:「已遮罩」 - 「无法核对」)。证据面板的指标步骤、证据图的标签、报告撰写节点的写作目录(withheld_metrics)都按这一个函数认,和推演 - 同一套分类、同一个判断(recompute.withheld_value): + 「无法核对」)。证据面板的指标步骤、证据图的标签、报告撰写节点的写作目录(withheld_metrics)、推演里转抄的输入 + (copied_inputs)都按这一个函数认,和推演同一套分类、同一个判断(recompute.withheld_value): - 输入取自哪份结果的哪几列按 _InputMasks 认(recompute.input_origin 的 wide:下标不是字面量的、整行、整份结果都认) - 认不出出处的输入(数据整形算出来再赋给变量的一格),值就是这次运行查到过的遮罩值的,按值认(masks.run_secrets); 运行输入是发起运行的人自己填的,不认 + - 转抄上游口径卡的输入(nodes.<卡>.metrics[0].value、nodes.<卡>.text,recompute.card_input)按上游那几个指标认: + 在这次请求的指标依赖图上一次算出(_CopyGraph → recompute.masked_closure),不逐个往上递归 - 指标的值:取自遮罩列的一格、一行的不给;最外层的输入遮没遮核对不了的写「无法核对」;整份结果当输入、有认不出出处 的输入的,值里有遮罩值才不给(len(vars.res.rows) 照给) - output(节点 id → 那个节点的产出)给了的,按位置取的一格顺着它走取值链,不去封存的事件里找口径卡算这一次之前的产出: - 报告撰写节点写作时运行还没封存,产出就在运行状态里。by_value 为假(报告撰写节点的写作目录):认不出出处的输入不按值认, - 也不因此按值认指标的值——只按出处认,见 withheld_metrics。sure 为真(写作目录、可验证报告认老文档):遮罩列只按确定的 - 那一套认(_InputMasks 的 sure),不跟着证据面板宁可多遮。 + card 是指标所在的那张卡(认它在卡上的位置,依赖图按 (指标集工件, 位置) 认指标),没给的按这个指标对象本身认。 + output(节点 id → 那个节点的产出)给了的,按位置取的一格、转抄的上游卡顺着它走取值链,不去封存的事件里找口径卡算这一次 + 之前的产出:报告撰写节点写作时运行还没封存,产出就在运行状态里。by_value 为假(报告撰写节点的写作目录):认不出出处的 + 输入不按值认,也不因此按值认指标的值——只按出处认,见 withheld_metrics。sure 为真(写作目录、可验证报告认老文档): + 遮罩列只按确定的那一套认(_InputMasks 的 sure),不跟着证据面板宁可多遮。""" + graph = masks.copies(sealed, output=output, by_value=by_value, sure=sure) + judged = await graph.judged(card_node, artifact, metric, card) + held = dict(graph.held(card_node, artifact, metric, card)) + for path, (kind, columns, reason) in judged.copied.items(): + if kind is not None: + held[path] = _copied_held(kind, columns, reason) + return held, {"masked": MASKED, "unverified": UNVERIFIED_VALUE}.get(judged.word or "") - 转抄上游口径卡的输入(nodes.<卡>.metrics[0].value、nodes.<卡>.text,recompute.card_input)按上游那几个指标认,同一个 - 函数往上追(depth 记追了几张卡,最多 recompute.CARD_DEPTH)。""" - hide = _InputMasks(sealed, masks, card_node, artifact, output=output, by_value=by_value, sure=sure, depth=depth) - held: dict[str, _Held] = {} - uncertain = False - for item in metric.get("inputs") or []: - if not isinstance(item, dict): - continue - path = str(item.get("path")) - found, traced = await hide.check(item) - if found is None and not traced and not by_value: - continue - if found is None and not traced: - uncertain = True - value = item.get("value") - if not path.startswith("input.") and recompute.maybe_masked_value(value) \ - and recompute.carries_masked(value, await masks.run_secrets()): - found = _Held("value", [], VALUE_MASKED) - if found is not None: - held[path] = found - uncertain = uncertain or found.kind == "table" - value = metric.get("value") - secrets = await masks.run_secrets() if uncertain and recompute.maybe_masked_value(value) else None - word = recompute.withheld_value( - metric, value, masked=[p for p, h in held.items() if h.kind in ("masked", "value")], - unverified=[p for p, h in held.items() if h.kind == "unverified"], uncertain=uncertain, - secrets=lambda: secrets) - return held, {"masked": MASKED, "unverified": UNVERIFIED_VALUE}.get(word or "") + +def _copied_held(kind: str, columns: tuple[str, ...], reason: str | None) -> _Held: + """转抄上游口径卡的输入不给值的原因(recompute.Judged.copied 的一项):上游有指标不给的是 masked(整张卡转抄的是 + table,指标的值另按值认),写上游让它不给的遮罩列;上游有指标核对不了的核对不了(整张卡转抄的同 table、带原因)。""" + if kind == "masked": + return _Held("masked", list(columns)) + if kind == "table": + return _Held("table", list(columns)) + if kind == "table_unverified": + return _Held("table", [], reason or CARD_UNVERIFIED) + return _Held("unverified", [], reason or CARD_UNVERIFIED) + + +def _card_key(node_id: Any, card: dict[str, Any]) -> Any: + """一张口径卡在依赖图上的键:它的指标集工件(每次执行一份,内容寻址);产出里没记工件的(落工件失败)按节点和这份 + 产出本身认。""" + return str(card.get("artifact") or "") or ("node", str(node_id or ""), id(card)) + + +def _metric_key(artifact: str, card_node: str, metric: dict[str, Any], card: dict[str, Any] | None) -> Any: + """一个指标在依赖图上的键:(指标集工件, 指标在卡上的位置),转抄的边按同样的键指到上游。认不出位置的(没给卡,或者 + 调用方另取来的指标对象)按这个指标对象本身认,不和别的指标混在一起(依赖图留着它,对象 id 在这次请求里不会被复用)。""" + metrics = card.get("metrics") if isinstance(card, dict) and isinstance(card.get("metrics"), list) else [] + index = next((i for i, m in enumerate(metrics) if m is metric), None) + if index is None: + return ("metric", id(metric)) + return (artifact or _card_key(card_node, card), index) # type: ignore[arg-type] + + +def _needs_secrets(links: recompute.MetricLinks) -> bool: + """依赖图上这个指标可能要按值认(整份结果当输入、认不出出处的输入、整张卡转抄),值又可能是遮罩值:认之前先把这次运行 + 查到过的遮罩值收好(masked_closure 在同步代码里取)。""" + return recompute.maybe_masked_value(links.metric.get("value")) and ( + links.uncertain or any(kind == "table" for kind, _ in links.direct.values()) + or any(edge.whole for edge in links.copies)) + + +class _CopyGraph: + """一次请求里认指标的值给不给(_metric_masks)用的指标依赖图。键是 (指标集工件, 指标在卡上的位置):转抄以外的输入按 + _InputMasks 认(取自遮罩格的、核对不了的、按值认出来的),转抄上游口径卡的输入(recompute.card_input)连成边,交给 + recompute.masked_closure 一次算出最小不动点。认过的指标记下来:证据图给每个指标认标签、同一次请求里别的指标转抄它, + 都不再重认,工作量和图的大小成正比。 + + 写作目录(output 给了:运行状态里每个节点最后一次的产出)里,循环中抄自己上一轮的卡指回自己,是一个环;封存之后 + (output 没给)按每次执行认:卡在这次执行之前读到的是自己上一次执行的产出(_card_outputs),一次一次往回连,连到 + 第一次为止。两种都只按图算,不按追了几层算:以前追够八层就按遮罩算,没设遮罩的循环累加也成了「已遮罩」。""" + + def __init__(self, sealed: _Sealed, masks: _Masks, *, output: Callable[[Any], Any] | None, by_value: bool, + sure: bool) -> None: + self.sealed, self.masks = sealed, masks + self.given, self.by_value, self.sure = output, by_value, sure + self._links: dict[Any, recompute.MetricLinks] = {} + self._held: dict[Any, dict[str, _Held]] = {} + self._judged: dict[Any, recompute.Judged] = {} + + def matches(self, sealed: _Sealed, output: Callable[[Any], Any] | None, by_value: bool, sure: bool) -> bool: + return self.sealed is sealed and self.given is output and self.by_value == by_value and self.sure == sure + + def held(self, card_node: str, artifact: str, metric: dict[str, Any], card: dict[str, Any] | None + ) -> dict[str, _Held]: + """认过的指标里转抄以外不给值的输入(judged 之后才有)。""" + return self._held[_metric_key(artifact, card_node, metric, card)] + + async def judged(self, card_node: str, artifact: str, metric: dict[str, Any], card: dict[str, Any] | None + ) -> recompute.Judged: + key = _metric_key(artifact, card_node, metric, card) + if key not in self._judged: + fresh = await self._build(key, card_node, artifact, metric) + secrets = await self.masks.run_secrets() if any(_needs_secrets(self._links[k]) for k in fresh) else None + self._judged.update(recompute.masked_closure({k: self._links[k] for k in fresh}, self._judged, + lambda: secrets)) + return self._judged[key] + + async def _build(self, key: Any, card_node: str, artifact: str, metric: dict[str, Any]) -> list[Any]: + """认好 key 和顺着转抄的边够得着、还没认过的指标(转抄以外的输入,_links、_held),交回这一批新认的键。上游的 + 指标按上游那张卡那一次执行的产出认(_InputMasks 按 _card_outputs 找它当时读到的产出)。卡上没记输入的上游指标 + (写死的数、升级前的老卡)不连边:照给。""" + fresh: list[Any] = [] + todo = [(key, card_node, artifact, metric)] + while todo: + key, node, art, item_metric = todo.pop() + if key in self._links: + continue + hide = _InputMasks(self.sealed, self.masks, node, art, output=self.given, sure=self.sure) + links = recompute.MetricLinks(item_metric) + held: dict[str, _Held] = {} + for item in item_metric.get("inputs") or []: + if not isinstance(item, dict): + continue + path = str(item.get("path")) + copied = hide.copied(item) + if copied is not None: + upstream: list[Any] = [] + if copied.card is not None: + for index in copied.picked: + up_metric = copied.card["metrics"][index] + if isinstance(up_metric, dict) and up_metric.get("inputs"): + up = (_card_key(copied.node_id, copied.card), index) + upstream.append(up) + todo.append((up, copied.node_id, str(copied.card.get("artifact") or ""), up_metric)) + links.copies.append(recompute.CopyEdge(path, tuple(upstream), copied.whole, copied.unreadable)) + continue + found, traced = await hide.check(item) + if found is None and not traced: + if not self.by_value: + continue + links.uncertain = True + value = item.get("value") + if not path.startswith("input.") and recompute.maybe_masked_value(value) \ + and recompute.carries_masked(value, await self.masks.run_secrets()): + found = _Held("value", [], VALUE_MASKED) + if found is not None: + held[path] = found + # 遮罩列只在确实取自遮罩列时往下游传(核对不了的那几列不算) + links.direct[path] = (found.kind, tuple(found.columns) if found.reason is None else ()) + self._links[key] = links + self._held[key] = held + fresh.append(key) + return fresh async def withheld_metrics(catalog: dict[str, Any], *, nodes: dict[str, Any], ledger: list[Any]) -> list[str]: @@ -1047,14 +1151,15 @@ async def withheld_metrics(catalog: dict[str, Any], *, nodes: dict[str, Any], le 遮罩列只按确定的那一套认(sure,和单元格引用的自查同一个口径:_cited_masks):SQL 认不准、证据面板整份遮的快照里, 没确定取自遮罩列的普通金额、计数、只装着分档标签的列照给。以前跟着证据面板宁可多遮,WITH 写了列名表、VALUES 当表的 销售额也成了「已遮罩」,正式运行引用它就判失败;同一格按 [[v:…]] 引用却不报违规。转抄上游口径卡的指标 - (nodes.<卡>.metrics[0].value、nodes.<卡>.text)按上游那个指标认(_InputMasks._copied)。""" + (nodes.<卡>.metrics[0].value、nodes.<卡>.text)按上游那个指标认:运行状态里每张卡只有最后一次的产出,循环里抄自己 + 上一轮的卡指回自己,在依赖图上是一个环(_CopyGraph),环上没有别的输入取自遮罩格就照给。""" queries = {str(e["artifact"]): {k: e[k] for k in ("node_id", "tool", "source", "via", "call_id", "exec") if e.get(k) is not None} for e in ledger if isinstance(e, dict) and e.get("kind") == "query" and e.get("artifact")} view = _Sealed(run_id="", graph={}, events=[], queries=queries, seal={"sealed": False, "ok": None, "manifest_seq": None, "legacy": False}) masks = _Masks(view) - found: list[tuple[str, str, str, dict[str, Any]]] = [] + found: list[tuple[str, str, str, dict[str, Any], dict[str, Any]]] = [] for alias, entry in catalog.items(): if not isinstance(entry, dict) or entry.get("kind") != "metric" or entry.get("value") is None: continue @@ -1062,13 +1167,13 @@ async def withheld_metrics(catalog: dict[str, Any], *, nodes: dict[str, Any], le card = nodes.get(card_node) metric = _metric_in(card if isinstance(card, dict) else None, (entry.get("locator") or {}).get("metric")) if isinstance(metric, dict) and metric.get("inputs"): - found.append((alias, card_node, str(entry.get("artifact") or ""), metric)) + found.append((alias, card_node, str(entry.get("artifact") or ""), metric, card)) if not found: return [] # cell()、Agent 字段落在的快照先在线程里读好(读盘、复验哈希):在运行里认,不占着事件循环。转抄上游口径卡的指标要顺着 # 上游卡的输入认,运行状态里每张口径卡的输入一并读好 cards = [card for card in nodes.values() if isinstance(card, dict) and card.get("kind") == "metric_set"] - await masks.prefetch(str(item["artifact"]) for metric in [*(m for *_, m in found), + await masks.prefetch(str(item["artifact"]) for metric in [*(m for *_, m, _ in found), *(m for card in cards for m in card.get("metrics") or [])] if isinstance(metric, dict) for item in metric.get("inputs") or [] if isinstance(item, dict) and isinstance(item.get("artifact"), str) and item["artifact"]) @@ -1077,9 +1182,9 @@ async def withheld_metrics(catalog: dict[str, Any], *, nodes: dict[str, Any], le return nodes.get(str(node_id or "")) out: list[str] = [] - for alias, card_node, artifact, metric in found: - _, word = await _metric_masks(metric, card_node, artifact, view, masks, output=output, by_value=False, - sure=True) + for alias, card_node, artifact, metric, card in found: + _, word = await _metric_masks(metric, card_node, artifact, view, masks, card=card, output=output, + by_value=False, sure=True) if word == MASKED: out.append(alias) return out @@ -1101,12 +1206,29 @@ async def certain_masks(sealed: _Sealed, masks: _Masks) -> tuple[dict[str, froze return out, await masks.run_secrets() +async def copied_inputs(sealed: _Sealed, card_node: str, artifact: str, card: dict[str, Any] + ) -> dict[int, dict[str, tuple[str | None, str | None]]]: + """推演(api/whatif.py)用:这张卡上每个指标转抄上游口径卡的输入给不给值,{指标在卡上的位置: {输入路径: (种类, 原因)}}, + 种类同 recompute.Judged.copied(None 照给)。和证据面板的指标步骤同一个判断、同一张依赖图(_metric_masks 的 + _CopyGraph,宁可多遮的那一套):推演据此不给转抄来的遮罩值、不让改,核对不了的不给,照给的照常推演。以前推演只按值认, + 转抄来的值不是够长的号码(短编码、从遮罩格算出来的数)就照给、还能改。""" + masks = _Masks(sealed) + graph = masks.copies(sealed, output=None, by_value=True, sure=False) + out: dict[int, dict[str, tuple[str | None, str | None]]] = {} + for index, metric in enumerate(card.get("metrics") or []): + if isinstance(metric, dict) and metric.get("inputs"): + judged = await graph.judged(card_node, artifact, metric, card) + if judged.copied: + out[index] = {path: (kind, reason or (CARD_UNVERIFIED if kind in ("unverified", "table_unverified") + else None)) + for path, (kind, _, reason) in judged.copied.items()} + return out + + #: tool_cell、Agent 字段落在的查询快照读不出来:只知道数据源现在设的遮罩,查询当时记下的在快照里,遮没遮核对不了 SNAPSHOT_UNVERIFIED = "查询快照无法读取,核对不了这一格是否取自设置了遮罩的列" #: 转抄的上游口径卡指标遮没遮核对不了(它的输入落在读不出来的快照上这类) CARD_UNVERIFIED = "转抄的上游口径卡指标核对不了是否取自设置了遮罩的列" -#: 口径卡一张抄一张,追了 recompute.CARD_DEPTH 张还没追到头(或者循环里抄自己上一轮的产出):宁可多遮 -CARD_TOO_DEEP = f"转抄上游口径卡超过 {recompute.CARD_DEPTH} 层,追不到这个值最初取自哪里,按取自设置了遮罩的列处理" class _InputMasks: @@ -1128,30 +1250,32 @@ class _InputMasks: 的 sure,再按血缘排除值确实不取自遮罩列的),读不出来的按数据源现在设的列名。证据面板、推演照旧按 hidden 宁可多遮。 转抄上游口径卡的输入(recompute.card_input:nodes.<卡>.metrics[0].value、vars.<卡的 assign_to>.metrics[0].value、 - nodes.<卡>.text)按上游那几个指标认(_copied,同一个 _metric_masks 往上追):上游的指标取自遮罩格,转抄它的这个输入 - 同样不给值。整张卡、清单文字这种不一定把哪个指标的值带进来的,同整份查询结果(table),指标的值另按值认。 + nodes.<卡>.text)这里只认出转抄了哪张卡的哪几个指标(copied),给不给按上游那几个指标在依赖图上认(_CopyGraph)。 """ def __init__(self, sealed: _Sealed, masks: _Masks, card_node: str, artifact: str, *, - output: Callable[[Any], Any] | None = None, by_value: bool = True, sure: bool = False, - depth: int = 0) -> None: + output: Callable[[Any], Any] | None = None, sure: bool = False) -> None: self._sealed, self._masks = sealed, masks self._card, self._artifact = card_node, artifact self._outputs: dict[str, str] | None = None # 报告撰写节点写作时(withheld_metrics):节点的产出就在运行状态里,不从封存的事件里找 self._given = output - self._by_value, self._sure, self._depth = by_value, sure, depth + self._sure = sure def _fetch(self, artifact: str) -> tuple[Any, bool | None]: return self._masks.fetch(artifact) + def copied(self, item: dict[str, Any]) -> recompute.CardInput | None: + """这个输入转抄了上游哪张口径卡的哪几个指标(recompute.card_input),不是转抄的为 None。卡当时的产出读不出来的 + (文件不在、与哈希不一致)交回 unreadable:核对不了,宁可多遮。""" + return recompute.card_input(item, self._output) + async def check(self, item: dict[str, Any]) -> tuple[_Held | None, bool]: """(不给值的原因, 认不认得出出处):给值的原因为 None。认不出出处的(常量、运行输入、数据整形算出来的一格)第二项 - 为假,值按值认(_metric_masks)。""" + 为假,值按值认(_metric_masks)。转抄上游口径卡的输入不在这里认(copied)。""" origin = recompute.input_origin(item, self._output, wide=True) if origin is None: - copied = recompute.card_input(item, self._output) - return (await self._copied(copied), True) if copied is not None else (None, False) + return None, False if origin.unverified is not None: return _Held("unverified", list(origin.columns), origin.unverified), True used = list(dict.fromkeys(origin.columns)) @@ -1172,7 +1296,7 @@ class _InputMasks: if self._given is not None: return self._given(node_id) if self._outputs is None: - self._outputs = _card_outputs(self._sealed, self._card, self._artifact) + self._outputs = self._masks.card_outputs(self._sealed, self._card, self._artifact) artifact = self._outputs.get(str(node_id or "")) content, ok = self._fetch(artifact) if artifact else (None, None) return content if ok is True else None @@ -1185,32 +1309,6 @@ class _InputMasks: sets = await self._masks.sets(artifact, name, snap) return sets.sure if self._sure else sets.hidden - async def _copied(self, copied: recompute.CardInput) -> _Held | None: - """转抄上游口径卡的输入不给值的原因:上游带着值的那几个指标里,有不给的(「已遮罩」)就不给,有遮没遮核对不了的 - 就核对不了;整张卡、清单文字(whole)同整份查询结果,记 table、指标的值另按值认。追得太深(卡抄卡超过 - recompute.CARD_DEPTH 张,循环里一张卡抄自己上一轮的产出)的追不到最初取自哪里,宁可多遮,按取自遮罩格算: - 核对不了的不给写作目录拦,号码就从这里出去了。""" - if self._depth >= recompute.CARD_DEPTH: - return _Held("masked", [], CARD_TOO_DEEP) - artifact = str(copied.card.get("artifact") or "") - words: list[str] = [] - columns: list[str] = [] - for index in copied.picked: - metric = copied.card["metrics"][index] - if not isinstance(metric, dict) or not metric.get("inputs"): - continue - held, word = await _metric_masks(metric, copied.node_id, artifact, self._sealed, self._masks, - output=self._given, by_value=self._by_value, sure=self._sure, - depth=self._depth + 1) - if word is not None: - words.append(word) - columns.extend(c for h in held.values() if h.reason is None for c in h.columns) - if MASKED in words: - return _Held("table" if copied.whole else "masked", list(dict.fromkeys(columns))) - if UNVERIFIED_VALUE in words: - return _Held("table" if copied.whole else "unverified", [], CARD_UNVERIFIED) - return None - def _card_outputs(sealed: _Sealed, card_node: str, artifact: str) -> dict[str, str]: """口径卡算出这份指标集的那一次执行开始之前,每个节点最后一次的产出工件:卡上 vars.*、nodes.* 取到的就是它们。 @@ -1525,6 +1623,26 @@ class _Masks: self._read: dict[str, tuple[Any, bool | None]] = {} self._sets: dict[tuple[str | None, str], _MaskSets] = {} self._cited: dict[tuple[str, str], set[str]] = {} + self._graphs: list[_CopyGraph] = [] + self._outputs: dict[tuple[int, str, str], dict[str, str]] = {} + + def copies(self, sealed: _Sealed, *, output: Callable[[Any], Any] | None, by_value: bool, sure: bool) -> _CopyGraph: + """这次请求里认指标的值给不给用的指标依赖图(_CopyGraph),按认法(output、by_value、sure)各一张:证据图给每个 + 指标认标签、转抄同一个上游指标的几个指标,都在同一张图上认,认过的不再认。""" + for graph in self._graphs: + if graph.matches(sealed, output, by_value, sure): + return graph + graph = _CopyGraph(sealed, self, output=output, by_value=by_value, sure=sure) + self._graphs.append(graph) + return graph + + def card_outputs(self, sealed: _Sealed, card_node: str, artifact: str) -> dict[str, str]: + """_card_outputs,一次请求里同一张卡的同一次执行只找一次:循环里每一次执行都转抄上一次的产出时,依赖图要逐次往回 + 认,每次都把整次运行的事件扫一遍就成了执行次数乘事件数。调用方只读不改。""" + key = (id(sealed), str(card_node), str(artifact)) + if key not in self._outputs: + self._outputs[key] = _card_outputs(sealed, card_node, artifact) + return self._outputs[key] def fetch(self, artifact: str) -> tuple[Any, bool | None]: """取回一件工件(_fetch):一次请求里同一件只读一次。证据图要给每个指标认一遍输入,同一份快照、同一个节点的 @@ -1831,7 +1949,9 @@ def _sure(snap: dict[str, Any], base: set[str], kind: str | None, 整份算,按位置认得出的列一个都收不到(合入 K3b 之后的复核)。 只装着分档标签的列(CASE WHEN phone IS NULL THEN '未留' ELSE '已留' END)格子里是 SQL 写死的字,不算;只数个数的 (COUNT(*) FILTER (WHERE phone IS NULL)、SUM(phone IS NULL))先改写成那一套认得出的样子再认(_counted)。 - 出了意外只认列名对上的。""" + 出了意外只认列名对上的。 + 整份算的快照里,列名本身就是取自遮罩列的表达式的(* 展开派生表、WITH 里没起别名的 UPPER(phone))另按列名认 + (_named_from_masks):那一套整份算之前认不出它落在哪个位置,collect 里没有,指标、单元格引用照给了号码。""" columns = {str(c).lower() for c in snap.get("columns") or []} direct = set(base) & columns if not base or not columns: @@ -1841,6 +1961,8 @@ def _sure(snap: dict[str, Any], base: set[str], kind: str | None, found = _derived_masks(_counted(snap, kind), set(base), kind, tables) sure = ((found.collect if found.whole else found.hidden) & columns) | direct + if found.whole: + sure |= _named_from_masks(snap, base, kind) labels = _snapshot_labels(snap, kind) return sure - {name for name in labels if name not in base}, labels except Exception: # noqa: BLE001 - 认不准:只认列名对上的 @@ -1848,6 +1970,32 @@ def _sure(snap: dict[str, Any], base: set[str], kind: str | None, return direct, {} +def _named_from_masks(snap: dict[str, Any], base: set[str], kind: str | None) -> set[str]: + """SQL 认不准、整份算的快照里,列名本身就是一个取自遮罩列的表达式的结果列(小写,_sure):没起别名的表达式,SQLite、 + MySQL 拿表达式的原文当列名(UPPER(phone)、phone || ''、SUBSTR(phone, 1, 11)),* 展开派生表、WITH、UNION 的一支时原样 + 带出来。认遮罩列的那一套(masking._derived_masks)认不出 * 展开到了哪个位置,整份算、collect 里没有它:写作目录、 + 单元格引用的自查就照给了号码。 + + 按列名拼一条只选这一项的 SQL,交给同一套认(同 _sure,只数个数的先改写,_counted):认得出取自遮罩列、又不是只装着 + 分档标签的(masking._label_columns)才算;COUNT(phone)、COUNT(*) FILTER (WHERE phone IS NULL)、LOWER(name) 这类不算。 + 只认列名里写着遮罩列名字的(masking._mentions),别的列不拼。Postgres 给这种列起的名字是函数名(upper),认不出, + 照旧只在证据面板、可验证报告那边宁可多遮。""" + from app.engine.masking import _derived_masks, _label_columns, _mentions + + out: set[str] = set() + for column in snap.get("columns") or []: + name = str(column) + lowered = name.lower() + if lowered in base or not _mentions(name, base): + continue + probe = _counted({"sql": f"SELECT {name} FROM t", "columns": [name]}, kind) + found = _derived_masks(probe, set(base), kind) + if found.whole or lowered not in found.hidden or lowered in _label_columns(probe["sql"], kind, [name]): + continue + out.add(lowered) + return out + + def _counted(snap: dict[str, Any], kind: str | None) -> dict[str, Any]: """报告自查认确定取自遮罩列的列之前(_sure),把只决定算不算进来的写法改写成认遮罩列的那一套认得出的样子: diff --git a/backend/app/api/whatif.py b/backend/app/api/whatif.py index 88d5e739..a9275906 100644 --- a/backend/app/api/whatif.py +++ b/backend/app/api/whatif.py @@ -40,6 +40,7 @@ from app.api.evidence import ( _sealed_cards, _sealed_loader, _snapshot_secrets, + copied_inputs, ) from app.core.artifact_store import load from app.engine import recompute @@ -82,7 +83,10 @@ async def evidence_whatif(run_id: str, payload: Any = Body(None)) -> dict[str, A if metric is not None and not any(isinstance(m, dict) and str(m.get("id")) == metric for m in card.get("metrics") or []): raise CodedHTTPException(404, f"口径卡中没有指标「{metric}」", METRIC_NOT_FOUND) - sources = _sources(sealed, cards, started, asyncio.get_running_loop()) + # 转抄上游口径卡的输入给不给值:和证据面板同一张指标依赖图(copied_inputs → recompute.masked_closure),在事件循环里 + # 认好交进去,重算的线程里不再查库 + copied = await copied_inputs(sealed, card_node, artifact, card) + sources = _sources(sealed, cards, started, asyncio.get_running_loop(), copied=copied) docs = [(r.node_id, r.doc) for r in reports if r.doc is not None and r.hash_ok is True and (report is None or r.node_id == report)] try: @@ -242,14 +246,16 @@ def _card_of(sealed: _Sealed, cards: list[tuple[str, str, dict[str, Any]]], node def _sources(sealed: _Sealed, cards: list[tuple[str, str, dict[str, Any]]], started: int, - loop: asyncio.AbstractEventLoop) -> recompute.Sources: + loop: asyncio.AbstractEventLoop, *, copied: Mapping[int, Mapping[str, Any]] | None = None + ) -> recompute.Sources: """重算要用的东西(recompute.Sources):封存范围内的工件、口径卡那次执行之前的节点产出和跑完的时刻、快照交回的 - 时刻、用到哪份才算哪份的遮罩。拿到的 Sources 要在工作线程里用(_Hidden 在线程里等事件循环查库)。""" + 时刻、用到哪份才算哪份的遮罩、转抄上游口径卡的输入给不给值(copied)。拿到的 Sources 要在工作线程里用(_Hidden + 在线程里等事件循环查库)。""" loader = _loader(sealed, cards) hidden = _Hidden(sealed, loader, loop) return recompute.Sources(load=loader, queries=tuple(sealed.queries), outputs=_outputs_before(sealed, started), masked=hidden, returned=_returned(sealed), finished=_finished_before(sealed, started), - secrets=hidden.secrets) + secrets=hidden.secrets, copied=dict(copied or {})) def _outputs_before(sealed: _Sealed, started: int) -> dict[str, str]: diff --git a/backend/app/bundle/build.py b/backend/app/bundle/build.py index b0a7d169..f6b3c446 100644 --- a/backend/app/bundle/build.py +++ b/backend/app/bundle/build.py @@ -13,7 +13,8 @@ 嵌入,别的不嵌内容、只给 id。口径卡上记着每个输入的值和代入式,有输入取自遮罩列(cell() 的格、Agent 字段、按位置 取的 vars.res.rows[0][2],SQL 里改了名字的遮罩列也算,SQL 认不出结果里哪一列取自哪一列的整份快照的列都算)的整张卡 同样只给 id,认法和证据面板、推演同一个(recompute.input_origin);有指标的值不给的(metric_word:值取自遮罩列的 - 一格、一行,整行、下标不是字面量的、转抄上游口径卡里这类指标的也算,遮罩列按这份文件宁可多遮的认法)也一样;服务端 + 一格、一行,整行、下标不是字面量的、转抄上游口径卡里这类指标的也算,转抄的在指标依赖图上一次认,recompute.masked_closure; + 遮罩列按这份文件宁可多遮的认法)、卡上记着的转抄来的值带遮罩的(_copies_masked)也一样;服务端 文件被改过的卡照样按卡上记的输入认,解析不出一张卡的整张只给 id。报告文档的证据目录收着上游口径卡的全部指标,现在的 报告撰写节点把值不给的写成「已遮罩」;还写着原值的老文档(升级前生成的,或者生成之后数据源才设的遮罩)整份只给 id、 不附带它引用的工件——哪些指标的值不给和写作目录同一个判断(metric_word 的 sure:只按确定取自遮罩列的认,由调用方按 @@ -48,7 +49,15 @@ from app.core.events import EventType from app.engine.evidence import iter_units, query_entry_fields from app.engine.masking import OPAQUE_SQL, SYSTEM_SCHEMAS, indirect_reads from app.engine.merge_query import MERGE_SOURCE -from app.engine.recompute import CARD_DEPTH, CardInput, card_input, carries_masked, input_origin, withheld_value +from app.engine.recompute import ( + CopyEdge, + Judged, + MetricLinks, + card_input, + carries_masked, + input_origin, + masked_closure, +) from app.engine.schema import NodeType from app.engine.toolcalls import QUERY_PREFIX @@ -103,7 +112,8 @@ class _Store: self._masked: dict[str, bool] = {} self._hidden: dict[str, set[str]] = {} self._seen: Any = None - self._words: dict[tuple[str, int, bool], str | None] = {} + self._graphs: dict[bool, _Graph] = {} + self._card_outputs: dict[str, dict[str, str]] = {} self._docs: dict[str, bool] = {} self._sources: dict[str, str] | None = None @@ -198,75 +208,124 @@ class _Store: if not isinstance(card, dict): self._masked[key] = self.raw(artifact) is not None return self._masked[key] - outputs = _card_outputs(events, artifact) - - def output(node_id: Any) -> Any: - found = outputs.get(str(node_id or "")) - return self.content(found) if found and self.intact(found) else None - + output = self._outputs_of(artifact, events) metrics = [metric for metric in card.get("metrics") or [] if isinstance(metric, dict)] items = [item for metric in metrics for item in metric.get("inputs") or [] if isinstance(item, dict)] # 另按指标认:值不给的指标(metric_word,遮罩列宁可多遮的那一套)——按位置取的整行、下标不是字面量的一格,整份 # 结果里带着遮罩列、值里就是号码的,转抄上游口径卡里这类指标的,上面只认字面量下标一格的那一套认不出,卡上记着的 - # 值照样是号码 + # 值照样是号码。转抄来的输入另看一遍(_copies_masked):卡上记着转抄来的值本身,指标的值照给时它也可能是号码 self._masked[key] = any(self._input_masked(item, output) for item in items) \ - or any(self.metric_word(artifact, metric, events) is not None for metric in metrics) + or any(self.metric_word(artifact, metric, events) is not None or self._copies_masked(artifact, metric) + for metric in metrics) return self._masked[key] + def _copies_masked(self, card: str, metric: dict[str, Any]) -> bool: + """卡上这个指标转抄来的输入带不带遮罩(metric_word 认过之后才看得到):转抄的是不给、核对不了的上游指标,卡上记着的 + 就是那个值(嵌在下标里、指标的值照给时也一样);整张卡、清单文字转抄(table)的,记着的值(清单文字逐条写着「指标 + (名称)= 值」)里有这次运行查到过的遮罩值才算,记成「[3 项]」这种大概的不算。""" + judged = self._graph(False).judged.get(_metric_key(card, metric, self.content(card))) + if judged is None: + return False + values = {str(i.get("path")): i.get("value") for i in metric.get("inputs") or [] if isinstance(i, dict)} + for path, (kind, _, _) in judged.copied.items(): + if kind in ("masked", "unverified"): + return True + if kind is not None and carries_masked(values.get(path), self.seen()): + return True + return False + def metric_word(self, card: str, metric: dict[str, Any], events: Sequence[tuple[int, str, str | None, Any]], *, - sure: bool = False, depth: int = 0) -> str | None: + sure: bool = False) -> str | None: """口径卡 card 上一个指标的值给不给:None 照给;"masked" 不给(值取自遮罩列的一格、一行,或者整份结果里带着遮罩列、 值里就有这次运行查到过的遮罩值);"unverified" 遮没遮核对不了。和证据面板指标步骤的值同一个判断(api/evidence.py - 的 _metric_masks,by_value=False:输入的出处按 recompute.input_origin 的 wide 认,转抄上游口径卡的按上游那个指标认, - 最后交给 recompute.withheld_value)。只按出处认:认不出出处的输入(写死的目标、数据整形算出来的数)值碰巧等于遮罩列 - 里的某一格,不因此算——不然普通报告里的这种指标在文件里也核对不了,老文档也整份只给哈希。 + 的 _metric_masks,by_value=False:输入的出处按 recompute.input_origin 的 wide 认,转抄上游口径卡的按上游那个指标 + 在依赖图上认,recompute.masked_closure,最后交给 recompute.withheld_value)。只按出处认:认不出出处的输入(写死的 + 目标、数据整形算出来的数)值碰巧等于遮罩列里的某一格,不因此算——不然普通报告里的这种指标在文件里也核对不了, + 老文档也整份只给哈希。 遮罩列两种认法: - 默认(认口径卡能不能随文件附带,card_masked):按这份文件自己的认法宁可多遮(hidden:SQL 认不准时整份的列都算; 快照读不出来、和 id 对不上的,按它的数据源现在设的遮罩),遮罩值用 seen。卡上记着输入的值,附带了就带出去了 - sure(认老文档,doc_withheld):只按确定取自遮罩列的认(sure,调用方按写作目录的口径认好交进来),遮罩值用 - sure_seen。和报告撰写节点写作时认的一样:现在的报告撰写节点照给的普通指标,老文档里写着它也照常附带""" - key = (card, id(metric), sure) - if key not in self._words: - outputs = _card_outputs(events, card) + sure_seen。和报告撰写节点写作时认的一样:现在的报告撰写节点照给的普通指标,老文档里写着它也照常附带 - def output(node_id: Any) -> Any: - found = outputs.get(str(node_id or "")) - return self.content(found) if found and self.intact(found) else None + 依赖图按 (指标集工件, 指标在卡上的位置) 认指标,两种认法各一张、构建里认过的记下来:卡转抄卡、循环里每一次执行都 + 转抄上一次的产出时,每个指标只认一次,和追了几层无关。""" + key = _metric_key(card, metric, self.content(card)) + graph = self._graph(sure) + if key not in graph.judged: + fresh = self._build(graph, key, card, metric, events, sure) + graph.judged.update(masked_closure({k: graph.links[k] for k in fresh}, graph.judged, + self.sure_seen if sure else self.seen)) + return graph.judged[key].word - masked: list[str] = [] - unverified: list[str] = [] - uncertain = False - for item in metric.get("inputs") or []: + def _graph(self, sure: bool) -> "_Graph": + if sure not in self._graphs: + self._graphs[sure] = _Graph() + return self._graphs[sure] + + def _build(self, graph: "_Graph", key: Any, card: str, metric: dict[str, Any], + events: Sequence[tuple[int, str, str | None, Any]], sure: bool) -> list[Any]: + """认好 key 和顺着转抄的边够得着、还没认过的指标(转抄以外的输入),交回这一批新认的键,同 api/evidence.py 的 + _CopyGraph._build。上游的指标按上游那张卡那一次执行之前的产出认(_card_outputs);卡上没记输入的上游指标不连边。""" + fresh: list[Any] = [] + todo = [(key, card, metric)] + while todo: + key, art, item_metric = todo.pop() + if key in graph.links: + continue + output = self._outputs_of(art, events) + links = MetricLinks(item_metric) + for item in item_metric.get("inputs") or []: if not isinstance(item, dict): continue path = str(item.get("path")) - kind, _ = self._input_kind(item, output, events, sure=sure, depth=depth) - if kind == "masked": - masked.append(path) - elif kind == "unverified": - unverified.append(path) - uncertain = uncertain or kind == "table" - self._words[key] = withheld_value(metric, metric.get("value"), masked=masked, unverified=unverified, - uncertain=uncertain, secrets=self.sure_seen if sure else self.seen) - return self._words[key] + copied = card_input(item, output) + if copied is not None: + upstream: list[Any] = [] + if copied.card is not None: + up_card = str(copied.card.get("artifact") or "") + for index in copied.picked: + up_metric = copied.card["metrics"][index] + if isinstance(up_metric, dict) and up_metric.get("inputs"): + up = (up_card or ("node", copied.node_id, id(copied.card)), index) + upstream.append(up) + todo.append((up, up_card, up_metric)) + links.copies.append(CopyEdge(path, tuple(upstream), copied.whole, copied.unreadable)) + continue + kind = self._input_kind(item, output, sure=sure) + if kind is not None: + links.direct[path] = (kind, ()) + graph.links[key] = links + fresh.append(key) + return fresh - def _input_kind(self, item: dict[str, Any], output: Callable[[Any], Any], - events: Sequence[tuple[int, str, str | None, Any]] = (), *, sure: bool = False, - depth: int = 0) -> tuple[str | None, bool]: - """(一个输入不给值的原因, 认不认得出出处),同 api/evidence.py 的 _InputMasks.check:取自遮罩列的一格、一行是 + def _outputs_of(self, card: str, events: Sequence[tuple[int, str, str | None, Any]]) -> Callable[[Any], Any]: + """口径卡 card 那一次执行之前每个节点的产出(_card_outputs):节点 id → 读出来的内容,读不出来、和 id 对不上的是 + None。同一张卡只找一次。""" + if card not in self._card_outputs: + self._card_outputs[card] = _card_outputs(events, card) + outputs = self._card_outputs[card] + + def output(node_id: Any) -> Any: + found = outputs.get(str(node_id or "")) + return self.content(found) if found and self.intact(found) else None + + return output + + def _input_kind(self, item: dict[str, Any], output: Callable[[Any], Any], *, sure: bool = False) -> str | None: + """一个输入(转抄上游口径卡的除外)不给值的原因,同 api/evidence.py 的 _InputMasks.check:取自遮罩列的一格、一行是 "masked",整份结果是 "table"(它不一定把哪一格带进指标,指标的值另按值认),遮没遮核对不了的是 "unverified" - (取值链上的产出读不出来、快照读不出来、结果没存快照只带着数据源名),给值的是 None。认不出出处的(常量、运行输入、 - 数据整形算出来的一格)第二项为假。转抄上游口径卡的(recompute.card_input)按上游那几个指标认(_copied_kind)。""" + (取值链上的产出读不出来、快照读不出来、结果没存快照只带着数据源名),给值的、认不出出处的(常量、运行输入、数据 + 整形算出来的一格)是 None。""" origin = input_origin(item, output, wide=True) if origin is None: - copied = card_input(item, output) - return (self._copied_kind(copied, events, sure, depth), True) if copied is not None else (None, False) + return None if origin.unverified is not None: - return "unverified", True + return "unverified" used = list(dict.fromkeys(origin.columns)) if not used: - return None, True + return None readable = bool(origin.artifact) and self.intact(str(origin.artifact)) \ and isinstance(self.content(str(origin.artifact)), dict) if readable: @@ -277,26 +336,9 @@ class _Store: now = self._source_masks.get(str(source or "")) hidden = {str(c).lower() for c in now} if isinstance(now, (list, tuple, set, frozenset)) else set() if any(c.lower() in hidden for c in used): - return ("table" if origin.table else "masked"), True + return "table" if origin.table else "masked" if (origin.artifact and not readable) or (not origin.artifact and origin.source): - return ("table" if origin.table else "unverified"), True - return None, True - - def _copied_kind(self, copied: CardInput, events: Sequence[tuple[int, str, str | None, Any]], sure: bool, - depth: int) -> str | None: - """转抄上游口径卡的输入(nodes.<卡>.metrics[0].value、nodes.<卡>.text)不给值的原因,同 api/evidence.py 的 - _InputMasks._copied:上游带着值的那几个指标有不给的就不给,有遮没遮核对不了的就核对不了;整张卡、清单文字同整份结果 - ("table")。追得太深(recompute.CARD_DEPTH)的追不到最初取自哪里,宁可多遮,按取自遮罩格算。""" - if depth >= CARD_DEPTH: - return "masked" - artifact = str(copied.card.get("artifact") or "") - words = [self.metric_word(artifact, metric, events, sure=sure, depth=depth + 1) - for metric in (copied.card["metrics"][i] for i in copied.picked) - if isinstance(metric, dict) and metric.get("inputs")] - if "masked" in words: - return "table" if copied.whole else "masked" - if "unverified" in words: - return "table" if copied.whole else "unverified" + return "table" if origin.table else "unverified" return None def query_source(self, artifact: str) -> str | None: @@ -350,6 +392,23 @@ class _Store: return any(c.lower() in hidden for c in origin.columns) +class _Graph: + """一种认法下这次构建认过的指标依赖图(_Store.metric_word):键 → 转抄以外的输入和转抄的边(MetricLinks)、认出来的 + 结论(Judged)。""" + + def __init__(self) -> None: + self.links: dict[Any, MetricLinks] = {} + self.judged: dict[Any, Judged] = {} + + +def _metric_key(card: str, metric: dict[str, Any], content: Any) -> Any: + """一个指标在依赖图上的键:(指标集工件, 指标在卡上的位置),转抄的边按同样的键指到上游;不是这张卡上的那个指标对象的 + (调用方另取来的)按对象本身认,不和别的指标混在一起,同 api/evidence.py 的 _metric_key。""" + metrics = content.get("metrics") if isinstance(content, dict) and isinstance(content.get("metrics"), list) else [] + index = next((i for i, m in enumerate(metrics) if m is metric), None) + return (card, index) if index is not None and card else ("metric", id(metric)) + + def _snapshot_masked(content: Any, source_masks: Mapping[str, Any], *, intact: bool, seen: Callable[[], Any] | None = None) -> bool: if not isinstance(content, dict): diff --git a/backend/app/bundle/verifier.js b/backend/app/bundle/verifier.js index 37c02b12..a17fea5e 100644 --- a/backend/app/bundle/verifier.js +++ b/backend/app/bundle/verifier.js @@ -195,6 +195,7 @@ shownScaled: '后面接着量级单位(万、亿、千、百、%、‰、k、M、B 这类)。系统显示这个数时不带它,读起来的数可能差了量级', shownDash: '前面紧挨着一道像负号的短横,读起来可能是负数', shownLookalike: '紧挨着长得像数字的字母或符号(l、I、O、上标数字、圈码这类),读起来可能多出一位', + shownOdd: '紧挨着这个数的字不在常见写法里,无法确认读者看到的数', numNotShown: '正文里没有显示这一处:导出时不会出现这种情况,文件可能被改过', numAria: (kind, text, result) => `${kind === 'value' ? '取值' : '数字'} ${text},${result},查看出处`, statusOk: (seq, what) => `与第 ${seq} 条事件(${what})一致`, @@ -226,8 +227,8 @@ shaJs: 'SHA-256 由本页内置的实现计算(浏览器未提供 crypto.subtle),结果与浏览器内置实现一致', renderNote: '正文里带出处的数字和取值,本页按系统的显示规则(千分位、小数位、单位、万 / 亿换算、百分数,文字压成一行)' + '把口径卡或快照里的值(运行输入按引用的值)重新渲染一遍,与显示的文字逐字比对;核对通过的字在正文里原样显示,' - + '紧挨着它写着数字、负号或小数点的判不通过;画着删除线、后面接着量级单位、紧挨着像负号的短横、长得像数字的字母' - + '或看不见的字,或者同一句里有改变文字方向的字,判无法核对。有一项无法核对,总览就不是绿色的通过。', + + '紧挨着它写着数字、负号或小数点的判不通过;画着删除线、后面接着量级单位、紧挨着像负号的短横、长得像数字的字母、' + + '看不见的字或其他不常见的字,或者同一句里有改变文字方向的字,判无法核对。有一项无法核对,总览就不是绿色的通过。', schema: (s) => `文件格式 ${s}`, } @@ -843,21 +844,27 @@ // - minus:负号。只认真当负号用的三个:半角的连字符减号、数学减号「−」、全角的「-」; // - dash:长得像负号、平时却不当负号用的短横:连字符、不断行连字符、数字宽的横线、小号的连字符减号、上下标负号、修饰 // 字母负号、粗减号、别的文字里的连字符。不判不通过:紧挨着数字开头、前面又不是字母数字时,读起来可能是负数,判无法 - // 核对。破折号(——、—、–、―,小号的、竖排的)、波浪线、汉字「一」和显示出来同样是一道横线的字(假名长音、制表符 - // 的横线、注音和谚文里的横这类)不在这里,是正文里的普通写法(「运费统一10元」「合计——7,036.50元」),当 other; + // 核对。破折号(——、—、–、―,小号的)、波浪线、汉字「一」、假名长音、制表符的横线不在这里,是正文里的普通写法 + // (「运费统一10元」「合计——7,036.50元」),在常见写法的名单里(COMMON); // - sep:半角逗号、句点和长得一样的窄标点(低位单引号、单点前导符、小号的逗号句点这类,取自易混字符表):「1,」后面 // 接着数字也算紧挨着。全角的「,」「.」不算:字宽,中文里两个数之间本来就常用「,」隔开; - // - unit:量级单位(万、亿、千、百、兆,繁体的萬、億,%、‰、‱,k、K、M、B,全角写法也算):接在数字后面,读者看到 - // 的数差了量级。系统显示一个数时单位、万 / 亿换算都在核对过的那段字里,不会另接在后面,判无法核对; - // - look:长得像数字、却不是数字的字:拉丁字母 l、I、O(全角的也算),上下标数字、圈码、罗马数字、汉字「〇」这类数字 - // 符号。紧挨着数字,读起来可能多出一位(「l74,613.65元」),判无法核对; - // - skip:显示时不占位置的字(零宽空格、连接符、变体选择符、韩文填充符这类格式字符和默认不显示的字),隔着它们照样 - // 算紧挨着; + // - unit:量级单位(万、亿、千、百、十、兆,繁体、大写的萬、億、拾、佰、仟,%、‰、‱,k、K、M、B,当「万」用的 w、W, + // 全角写法也算):接在数字后面,读者看到的数差了量级。系统显示一个数时单位、万 / 亿换算都在核对过的那段字里,不会另接 + // 在后面,判无法核对; + // - look:长得像数字、却不是数字的字:拉丁字母 l、I、O、o,竖线和长得像它们的字(|、丨、竖排的破折号、制表符的竖线、 + // 别的文字里的 I、O,全角的也算),上下标数字、圈码、罗马数字、汉字「〇」、空心圆这类数字符号。紧挨着数字,读起来 + // 可能多出一位(「l74,613.65元」),判无法核对; + // - skip:显示时不占位置的字(零宽空格、连接符、变体选择符、韩文填充符这类格式字符和默认不显示的字),找紧挨着的数字、 + // 负号时隔着它们照样算紧挨着;它自己紧挨着数字时不在常见写法的名单里,判无法核对; // - blur:显示成什么说不准的字:控制字符(制表、换行这几个空白除外)、未分配和私用区的字、孤立的代理项、组合附加符号 // (叠在前一个字上:叠在空格上能在数字前面画出一道横线)、可选连字符(在它那里换行时显示成连字符)、发丝空格、对象 // 替换符,以及在 Chrome 的常用字体里量出来几乎不占宽度的几个字。各家浏览器、字体把它们显示成数字、横线还是什么都 // 不显示说不准:隔着它们照样往外找数字、负号;找不到的也判无法核对,原因另写; - // - other:别的字(汉字、别的字母、空格、别的标点)把数字隔开。 + // - other:别的字。 + // 紧挨着数字头尾的那一个字(不越过 skip、blur)还要在常见写法的名单里(COMMON:汉字、中文标点、半角和全角空格、ASCII + // 字母和常用的半角标点、算式里的几个符号、上一轮放行的破折号写法),不在名单里的(别的文字系统的字母、制表换行和各种 + // 宽度的空格、零宽字符、全角拉丁字母、表情这类)判无法核对:伪造的写法换一个易混字、换一种空白就能绕开上面任何一条按 + // 字认的规则,名单以外的一律说不准。 // 改变文字方向的控制字符(BIDI)和从右往左书写的文字(RTL:希伯来文、阿拉伯文这类)另算:它们能把一位数挪到数字的 // 另一边显示(逻辑上在数字后面的「5」显示在数字前面),按字的先后看不出来,同一句里出现就判无法核对 const DIGIT = unicode('^\\p{Nd}$') || /^[0-9\uff10-\uff19]$/ @@ -868,18 +875,32 @@ || /^[\u058a\u1400\u1806\u2010-\u2012\u2e17\u2e1a\u2e40\u30a0\ufe63\u02d7\u207b\u208b\u2043\u2796]$/ const SEP = unicode('^[.,\\u00b8\\u201a\\u2024\\u2e12\\ua4f8\\ua4f9\\ua60e\\uabec\\ufe50\\ufe52\\u{16f53}\\u{1d16d}]$') || /^[.,\u00b8\u201a\u2024\u2e12\ua4f8\ua4f9\ua60e\uabec\ufe50\ufe52]$/ - const UNIT = /^[万亿千百兆萬億%\uff05\u2030\u2031kKMB\uff4b\uff2b\uff2d\uff22]$/ + const UNIT = /^[万亿千百十兆萬億拾佰仟%\uff05\u2030\u2031kKMBwW\uff4b\uff2b\uff2d\uff22\uff57\uff37]$/ // 量级单位里本身就能接在数后面的那几个汉字:核对过的字以它们结尾(换算成万的「4.57万」),再接一个也差了量级(「4.57万亿」) const UNIT_HAN = /^[万亿千百兆萬億]$/ - const LOOK = unicode('^[lIO\\uff4c\\uff29\\uff2f\\p{No}\\p{Nl}]$') - || new RegExp('^[lIO\\uff4c\\uff29\\uff2f\\u00b2\\u00b3\\u00b9\\u00bc-\\u00be\\u2070-\\u2079\\u2080-\\u2089' + // 竖线一类:半角、全角的竖线,汉字「丨」,竖排的破折号,制表符的竖线,拉丁、西里尔、希腊字母里长得像 l、I 的;圆圈一类: + // 西里尔、希腊的 O、o,空心圆 + const LOOK_EXTRA = 'o\\uff4f|\\uff5c\\u4e28\\ufe31\\ufe32\\u2502\\u01c0\\u04c0\\u0406\\u0399\\u041e\\u043e\\u039f\\u03bf\\u25cb' + const LOOK = unicode(`^[lIO\\uff4c\\uff29\\uff2f${LOOK_EXTRA}\\p{No}\\p{Nl}]$`) + || new RegExp(`^[lIO\\uff4c\\uff29\\uff2f${LOOK_EXTRA}\\u00b2\\u00b3\\u00b9\\u00bc-\\u00be\\u2070-\\u2079\\u2080-\\u2089` + '\\u2150-\\u2189\\u2460-\\u249b\\u24ea-\\u24ff\\u2776-\\u2793\\u3007\\u3021-\\u3029\\u3038-\\u303a]$') - // 连字符减号前面的字母数字(拉丁、希腊、西里尔字母,全角的也算):「SKU-12」「A-3」是编号的一部分,不是负号 - const WORDY = unicode('^[\\p{Script=Latin}\\p{Script=Greek}\\p{Script=Cyrillic}]$') - || /^[A-Za-z\u00c0-\u024f\u0370-\u03ff\u0400-\u04ff\uff21-\uff3a\uff41-\uff5a]$/ - // 量级单位前面可以隔着的空白:「45,678.5 万」照样读作四亿多 - const BLANK = /^[ \u00a0\u3000]$/ - // 变体选择符只换前一个字的字形(文字样式还是表情样式),算 skip:表情后面常带一个,不能因此把数字判成核对不了 + // 连字符减号前面的字母数字(只认 ASCII:别的文字系统里长得像撇号、点号、竖线的字母不算):「SKU-12」「A-3」是编号的一部分, + // 不是负号 + const WORDY = /^[A-Za-z0-9]$/ + // 量级单位前面可以隔着的:各种空白(制表、换行、不同宽度的空格),左括号、左引号(「45,678.5(万元)」读作四亿多) + const LEAD_IN = unicode('^[\\s\\p{Zs}(\\[{\\uff08\\uff3b\\uff5b\\u3008\\u300a\\u300c\\u300e\\u3010\\u3014\\u3016\\u3018' + + '\\u301a\\u201c\\u2018]$') + || /^[\s(\[{\uff08\uff3b\uff5b\u3008\u300a\u300c\u300e\u3010\u3014\u3016\u3018\u301a\u201c\u2018]$/ + // 常见写法的名单(紧挨着数字头尾的那一个字)。汉字只认表意文字(含「一」),去掉丨、丶、丿这类当笔画用的字;中文标点 + // 按全角的写;半角标点里不收撇号(有的地方拿它当千分位)、反斜杠、货币符号这类;逗号、句点、连字符另看上下文(usual)。 + // 破折号、波浪线、假名长音、制表符的横线是上一轮放行的写法 + const COMMON_REST = ' A-Za-z\\u3000\\u3001-\\u3003\\u3008-\\u3011\\u3014-\\u301f\\u3030\\u30fc()\\[\\]{}:;!?"=+/~*#<>' + + '\\uff01-\\uff03\\uff08-\\uff0c\\uff0f\\uff1a-\\uff1f\\uff3b\\uff3d\\uff5b\\uff5d\\uff5e' + + '\\u2013-\\u2015\\u2018\\u2019\\u201c\\u201d\\u2026\\u2e3a\\u2e3b\\ufe58\\u2500\\u00d7\\u00f7\\u2248\\u2264\\u2265' + const STROKES = '\\u4e28\\u4e36\\u4e3f\\u4e40\\u4e41\\u4e5a\\u4e5b\\u4e85' + const COMMON = unicode(`^(?:(?![${STROKES}])(?=\\p{Script=Han})\\p{Lo}|[${COMMON_REST}])$`) + || new RegExp(`^(?:(?![${STROKES}])[\\u3400-\\u4dbf\\u4e00-\\u9fff\\uf900-\\ufaff]|[${COMMON_REST}])$`) + // 变体选择符只换前一个字的字形(文字样式还是表情样式),算 skip const VARIATION = unicode('^[\\u180b-\\u180d\\u180f\\ufe00-\\ufe0f\\u{e0100}-\\u{e01ef}]$') || /^[\u180b-\u180d\u180f\ufe00-\ufe0f]$/ const BLUR_EXTRA = '\\u00ad\\u02dd\\u1843\\u1cd3\\u200a\\u2044\\ufffc' @@ -912,15 +933,16 @@ * 每一项记下正文里显示出来的字(onPage)。 * - 正文里根本不显示它(分隔线里、表格里不在各行各列上的格):伪造的字可以写在别处,读的人看不到核对过的那个数。 * 服务端导出时不会出现(分隔线不带句子,表格的格都在行列上),判不通过; - * - 紧挨着的字里写着数字、小数点(前面补一位、后面接一位或一段小数;隔着不占位置、显示成什么说不准的字,显示时藏起来 - * 的排版标记也算紧挨着),或者紧挨着数字开头的是负号、负号前面又不是字母数字(「SKU-12」的连字符是编号的一部分): - * 读者看到的是另一个数,判不通过。伪造者不动核对过的那段字、只改旁边的正文,重算全部哈希就做得到。服务端组装的文档 - * 不这样排:两个数之间补空格(evidence._glue),写作者自己写在引用前面的字(「统一」「——」「SKU-」)都不在这几样 - * 里,原样导出照样通过。两个数之间的连字符是区间,不算负号; - * - 这一句里有改变文字方向的字;核对过的字画着删除线,或者紧挨着画了删除线的字;后面接着量级单位(中间隔着空格也算); - * 前面紧挨着像负号的短横;紧挨着长得像数字的字母、数字符号;紧挨着显示成什么说不准的字:读起来可能不是核对过的那个 - * 数,判无法核对,原因各自写明——真报告里也可能有,不能说成被改过。总览里有一项无法核对就不是绿色的通过 - * (overallOf)。这几样不会把上一条的不通过降成无法核对。 + * - 紧挨着的字里写着数字、小数点(前面补一位、后面接一位或一段小数、接 e 和数字;隔着不占位置、显示成什么说不准的字, + * 显示时藏起来的排版标记也算紧挨着),或者紧挨着数字开头的是负号、负号前面又不是 ASCII 字母数字(「SKU-12」的连字符 + * 是编号的一部分):读者看到的是另一个数,判不通过。伪造者不动核对过的那段字、只改旁边的正文,重算全部哈希就做得到。 + * 服务端组装的文档不这样排:两个数之间补空格(evidence._glue),写作者自己写在引用前面的字(「统一」「——」「SKU-」) + * 都不在这几样里,原样导出照样通过。两个数之间的连字符是区间,不算负号; + * - 这一句里有改变文字方向的字;核对过的字画着删除线,或者紧挨着画了删除线的字;后面接着量级单位(中间隔着空白、左括号 + * 也算);前面紧挨着像负号的短横;紧挨着长得像数字的字母、数字符号;紧挨着显示成什么说不准的字;紧挨着数字头尾的那一个 + * 字不在常见写法的名单里(COMMON,白名单:换一个易混字、换一种空白就能绕开按字认的规则,名单以外的一律说不准):读起来 + * 可能不是核对过的那个数,判无法核对,原因各自写明——真报告里也可能有,不能说成被改过。总览里有一项无法核对就不是绿色的 + * 通过(overallOf),下面写明去系统里核对封存。这几样不会把上一条的不通过降成无法核对。 * 代码块按原文显示、不标数字,照旧 */ function checkLayout(doc, items) { @@ -981,8 +1003,17 @@ return { ...NOWHERE, blurred } } const struck = (j) => j >= 0 && (chars[j][2] & BIT.strike) !== 0 - // 前一个字算不算字母数字:负号前面是它的,是「SKU-12」这类编号的一部分 - const wordy = (at) => at >= 0 && (kinds[at] === 'digit' || kinds[at] === 'look' || WORDY.test(chars[at][0])) + // 前一个字算不算字母数字(只认 ASCII):负号前面是它的,是「SKU-12」这类编号的一部分 + const wordy = (at) => at >= 0 && WORDY.test(chars[at][0]) + // 紧挨着数字头尾的那一个字(j,不越过看不见的字)在不在常见写法里。逗号、连字符另看上下文:走到这里,数字头前面的 + // 连字符已是编号、区间的写法,逗号前面也不是数字(那两样判不通过);数字尾后面的是句末、区间。句点在数字头前面只认 + // 前面是字母的缩写(NO.12),别的读起来像小数点 + const usual = (j, left) => { + if (j < 0 || j >= chars.length) return true + const c = chars[j][0] + if (COMMON.test(c) || c === '-' || c === ',') return true + return c === '.' && (!left || (j > 0 && /^[A-Za-z]$/.test(chars[j - 1][0]))) + } for (const [seg, item] of items) { if (item.result !== PASS || !spans.has(seg)) continue const [s0, s1] = spans.get(seg) @@ -1001,16 +1032,23 @@ // 负号、像负号的短横紧挨着数字开头,前面不是字母数字、也不是另一个数(两个数之间是区间) const signed = lead && (l1.kind === 'minus' || l1.kind === 'dash') && !wordy(l2.at) && !(l2.at >= 0 && numeric(chars[l2.at][1])) - const after = tail && (r1.kind === 'digit' || (r1.kind === 'sep' && r2.kind === 'digit')) - // 后面的量级单位:中间隔着空格也算(「45,678.5 万」) + // 后面接着 e 和数字(科学记数法,「45,678.5e3」读作四千多万),同紧挨着数字 + const r3 = r2.at >= 0 ? look(r2.at, 1) : NOWHERE + const signs = (a) => a.kind === 'minus' || (a.at >= 0 && chars[a.at][0] === '+') + const exponent = tail && r1.at >= 0 && /^[eE]$/.test(chars[r1.at][0]) + && (r2.kind === 'digit' || (signs(r2) && r3.kind === 'digit')) + const after = tail && (r1.kind === 'digit' || (r1.kind === 'sep' && r2.kind === 'digit') || exponent) + // 后面的量级单位:中间隔着空白、左括号也算(「45,678.5 万」「45,678.5(万元)」) let ru = r1 - while (ru.at >= 0 && BLANK.test(chars[ru.at][0])) ru = look(ru.at, 1) + while (ru.at >= 0 && LEAD_IN.test(chars[ru.at][0])) ru = look(ru.at, 1) const scaled = (tail || UNIT_HAN.test(chars[last][0])) && ru.kind === 'unit' const crossed = chars.slice(s0, s1 + 1).some((_, j) => struck(s0 + j)) || struck(l1.at) || struck(r1.at) const lookalike = (kinds[first] === 'digit' && l1.kind === 'look') || (tail && r1.kind === 'look') // 数字头尾紧挨着(或隔着一个逗号、句点)的是说不准的字:它本身就可能显示成数字、负号 const near = (a, b) => a.blurred || (a.kind === 'sep' && b.blurred) const blurred = (lead && near(l1, l2)) || (tail && near(r1, r2)) + // 紧挨着数字头尾的那一个字不在常见写法里(别的文字系统的字母、不常见的空白、零宽字符、全角拉丁字母、表情……) + const odd = (lead && !usual(s0 - 1, true)) || (tail && !usual(s1 + 1, false)) if (before || (signed && l1.kind === 'minus') || after) { // 紧挨着写着数字、负号、小数点:读者看到的是另一个数(伪造者不动核对过的字、只改旁边的正文,重算全部哈希就 // 做得到),判不通过。排在下面那几样前面判:伪造者在同一句里再放一个说不准的字、改变方向的字,不能把不通过 @@ -1018,7 +1056,7 @@ Object.assign(item, { result: FAIL, reason: T.numGlued }) } else { const why = turned ? T.shownTurned : crossed ? T.shownStruck : scaled ? T.shownScaled - : signed ? T.shownDash : lookalike ? T.shownLookalike : blurred ? T.shownBlurred : null + : signed ? T.shownDash : lookalike ? T.shownLookalike : blurred ? T.shownBlurred : odd ? T.shownOdd : null if (why) Object.assign(item, { result: NA, reason: T.numShownUnknown(why) }) } } diff --git a/backend/app/engine/recompute.py b/backend/app/engine/recompute.py index a7a75d22..d3d008e2 100644 --- a/backend/app/engine/recompute.py +++ b/backend/app/engine/recompute.py @@ -17,6 +17,8 @@ - 遮罩列取来的输入不给值、也不拿来推演:改一个输入、排除一行,新值和原值一比就能算出遮住的那一格。vars.res.rows[0][2] 这种按位置取自查询结果的标量没有定位,顺着取值链认出是哪一列(positional_cell,证据面板的指标步骤用同一个认法); 下标不是字面量的(rows[vars.i][2])按列认,整行(rows[0])按整行的列认(wide)。 + 转抄上游口径卡的输入(nodes.<卡>.metrics[0].value、nodes.<卡>.text)按上游那几个指标认:调用方在指标依赖图上一次 + 认好(masked_closure,证据面板、写作目录、可验证报告同一个函数),交进来(Sources.copied),转抄的是取自遮罩格的指标就不给。 指标本身的原值给不给(withheld_value,证据面板指标步骤的值同一个判断):用到了遮罩列的一格、一行,原值写「已遮罩」—— 指标的值可能就是那一格,输入里拿掉了、指标上照给,等于没遮;最外层的输入遮没遮核对不了,写「无法核对」。整份结果 当输入(len(vars.res.rows))、认不出出处的输入(数据整形算出来再赋给变量的一格),按值认:值里有这次运行查到过的 @@ -66,6 +68,8 @@ NOT_RESTORABLE = "这类输入当时没有保存完整值" MASKED_INPUT = "这一列在数据源中设置了遮罩,不能用于推演" MASKED_TABLE = "这份查询结果里有设置了遮罩的列,不能用于推演" MASKED_METRIC = "这个指标用到了设置了遮罩的列,不能推演" +#: 转抄上游口径卡的输入,转抄的是取自遮罩列的指标(Sources.copied,和证据面板同一张依赖图) +MASKED_COPY = "转抄的口径卡指标取自设置了遮罩的列,不能用于推演" #: 认不出出处的输入(数据整形算出来再赋给变量的一格),值和这次运行查到过的遮罩值一样(按值认,Sources.secrets) MASKED_VALUE_INPUT = "这个输入的值与设置了遮罩的列中的值相同,不能用于推演" MASKED_VALUE_METRIC = "这个指标的值或输入与设置了遮罩的列中的值相同,不能推演" @@ -122,6 +126,9 @@ class Sources: 排除行只可能影响在那份快照交回之后才算出来的值(_doubt)。不知道的一律当可能受影响 - secrets:这次运行查到过的遮罩值(masking._Secrets,没有为 None),用到时才取:认不出出处的输入、整份结果当输入的 指标,值给不给按它认(withheld_value)。没给的(双盲复算)不按值认 + - copied:卡上转抄上游口径卡的输入给不给值,指标在卡上的位置 → 输入路径 → (种类, 原因),种类同 Judged.copied。 + 调用方按证据面板同一张指标依赖图认好(api/evidence.py 的 copied_inputs → masked_closure);没给的(双盲复算)当 + 认不出出处 """ load: Callable[[str], Any] @@ -131,6 +138,7 @@ class Sources: returned: Mapping[str, int] = field(default_factory=dict) finished: Mapping[str, int] = field(default_factory=dict) secrets: Callable[[], Any] | None = None + copied: Mapping[int, Mapping[str, tuple[str | None, str | None]]] = field(default_factory=dict) class _Store: @@ -182,9 +190,11 @@ class _Input: traced: bool = False #: 认不出出处,值里却有这次运行查到过的遮罩值(按值认的遮罩) by_value: bool = False - #: 遮着的是一整份结果(或者值里有遮罩值的容器):整个指标推演不了,可它不一定把遮罩值带进指标的值,指标的原值 - #: 按值认(len(vars.res.rows) 照给) + #: 遮着的是一整份结果(或者值里有遮罩值的容器、整张转抄的口径卡):整个指标推演不了,可它不一定把遮罩值带进指标的值, + #: 指标的原值按值认(len(vars.res.rows) 照给) whole: bool = False + #: 转抄上游口径卡(Sources.copied 里有它):给不给按依赖图认好了,不再按取值链、按值认 + copy: bool = False @property def cell(self) -> bool: @@ -274,12 +284,13 @@ def whatif(card: dict[str, Any], changes: Changes, sources: Sources, *, metric: 新值不知道)。推演不了的指标 changed 一律为 None,只看它分不出「什么都没改」。 """ store = _Store(sources) - metrics = [m for m in card.get("metrics") or [] if isinstance(m, dict) and m.get("id") is not None] + metrics = [(i, m) for i, m in enumerate(card.get("metrics") or []) + if isinstance(m, dict) and m.get("id") is not None] if metric is not None: - metrics = [m for m in metrics if str(m["id"]) == str(metric)] + metrics = [(i, m) for i, m in metrics if str(m["id"]) == str(metric)] if not metrics: raise WhatIfError(f"口径卡中没有指标「{metric}」") - plans = [_plan(m, store) for m in metrics] + plans = [_plan(m, store, copied=sources.copied.get(i)) for i, m in metrics] _check(changes, plans, store, metric=metric) outcomes = [_outcome(p, changes, sources) for p in plans] return {"metrics": [result for result, _ in outcomes], "unsupported": _unsupported(plans), @@ -302,7 +313,8 @@ def recompute(metric: dict[str, Any], changes: Changes, sources: Sources) -> dic # -------------------------------------------------------------------------- -def _plan(metric: dict[str, Any], store: _Store) -> _Plan: +def _plan(metric: dict[str, Any], store: _Store, *, copied: Mapping[str, tuple[str | None, str | None]] | None = None + ) -> _Plan: expr = str(metric.get("expression") or "").strip() items = metric.get("inputs") if not expr or not isinstance(items, list): @@ -319,7 +331,7 @@ def _plan(metric: dict[str, Any], store: _Store) -> _Plan: inputs: dict[str, _Input] = {} for item in items: if isinstance(item, dict) and isinstance(item.get("path"), str) and item["path"] not in inputs: - inputs[item["path"]] = _restore(item, remaining, set(within), store) + inputs[item["path"]] = _restore(item, remaining, set(within), store, (copied or {}).get(item["path"])) plan = _Plan(metric, tree, tops, inputs, within=within) if lost := [p for p in tops if p not in inputs]: plan.plain = f"口径卡缺少输入 {'、'.join(lost[:3])} 的记录" @@ -327,8 +339,8 @@ def _plan(metric: dict[str, Any], store: _Store) -> _Plan: # 指标的原值给不给:和证据面板的指标步骤同一个判断(withheld_value),输入按同一套分类交进去 plan.held = withheld_value( metric, metric.get("value"), masked=[p for p, i in inputs.items() if i.cell], - unverified=[p for p, i in inputs.items() if i.unreadable], - uncertain=any((i.masked and not i.cell) or not i.traced for i in inputs.values()), + unverified=[p for p, i in inputs.items() if i.unreadable and not i.whole], + uncertain=any((i.masked and not i.cell) or (i.unreadable and i.whole) or not i.traced for i in inputs.values()), secrets=store.sources.secrets) return plan @@ -379,7 +391,8 @@ def _remaining(substituted: Any, paths: list[str]) -> set[str]: return set(paths) -def _restore(item: dict[str, Any], remaining: set[str], nested: set[str], store: _Store) -> _Input: +def _restore(item: dict[str, Any], remaining: set[str], nested: set[str], store: _Store, + copied: tuple[str | None, str | None] | None = None) -> _Input: path = item["path"] inp = _Input(item=item, path=path, value=item.get("value"), selector=path in nested) locator = item.get("locator") if isinstance(item.get("locator"), dict) else {} @@ -406,6 +419,9 @@ def _restore(item: dict[str, Any], remaining: set[str], nested: set[str], store: # 也不拿来推演。数据源现在设的遮罩照样认(_masked),认出来了就按遮罩说 inp.unreadable = True inp.problem = inp.problem or (TAMPERED if trouble == "tampered" else UNREADABLE) + if copied is not None: + _copied(inp, *copied) + return inp inp.masked = _masked(inp, locator, store) inp.traced = inp.snapshot is not None or (item.get("via") in ("tool_cell", "agent_field") and artifact is not None) if not inp.masked: @@ -415,6 +431,19 @@ def _restore(item: dict[str, Any], remaining: set[str], nested: set[str], store: return inp +def _copied(inp: _Input, kind: str | None, reason: str | None) -> None: + """转抄上游口径卡的输入(Sources.copied,和证据面板同一张依赖图认的):转抄的是取自遮罩格的指标,不给值、不推演 + (整张卡转抄的同整份结果,指标的原值另按值认);上游指标核对不了的,同读不出来的快照,不给值、不推演(整张卡转抄的 + 指标的原值同样另按值认);照给的照常推演。认得出出处,不再按值认:以前只按值认,转抄来的值不是够长的号码就照给、 + 还能改。""" + inp.traced = inp.copy = True + if kind in ("masked", "table"): + inp.masked, inp.whole = True, kind == "table" + elif kind in ("unverified", "table_unverified"): + inp.unreadable, inp.whole = True, kind == "table_unverified" + inp.problem = inp.problem or reason or CARD_UNREADABLE + + def _full_value(item: dict[str, Any], store: _Store) -> tuple[Any, str | None]: """容器输入的完整值:(值, 找不回来的原因)。""" path = item["path"] @@ -575,37 +604,42 @@ _GONE = object() #: 下标不是字面量(rows[vars.i]、rows[len(vars.res.rows) - 1]):wide 时认作「哪一行、哪一列说不准」 _ANY = object() -#: 转抄上游口径卡的取值链最多往上追几张卡(卡 3 抄卡 2、卡 2 抄卡 1……):循环里一张卡抄自己上一轮的产出也就此打住。 -#: 追到头还没追完的宁可多遮,按取自遮罩格算 -CARD_DEPTH = 8 #: 口径卡产出里不带指标值的字段:取这些的输入不算转抄了指标的值 _CARD_PLAIN = frozenset({"kind", "caliber", "caliber_version", "artifact", "source"}) #: 一个指标里不带值的字段(名称、单位、表达式、显示格式、状态这类)。值、渲染、输入、代入式都带着值 _METRIC_PLAIN = frozenset({"id", "name", "unit", "expression", "decimals", "format", "status", "recompute_ok", "incomplete", "incomplete_reason", "sql_check_failed", "sql_check_reason"}) +#: 转抄的上游口径卡当时的产出读不出来(文件不在、与哈希不一致、不知道是谁写的):转抄来的是不是遮罩值核对不了 +CARD_UNREADABLE = "转抄的口径卡结果无法读取,核对不了这个值是否取自设置了遮罩的列" @dataclass(frozen=True) class CardInput: """卡上的一个输入转抄了上游口径卡的产出(card_input):node_id 是那张卡的节点,card 是它当时的产出(指标集加清单 文字),picked 是这个输入带着值的那几个指标(在 card["metrics"] 里的位置)。whole:整张卡、整份指标列表、清单文字、 - 下标不是字面量的一个指标——不一定把哪个指标的值带进来,同整份查询结果(Origin.table)。""" + 下标不是字面量的一个指标——不一定把哪个指标的值带进来,同整份查询结果(Origin.table)。 + + unreadable:那张卡当时的产出现在读不出来,card 为 None、picked 为空,这里写原因。卡上记着这个输入转抄来了值(不是空的), + 算的时候产出是在的,现在读不出来只能是产出丢了或被改了:同按位置取的一格取值链上的产出读不出来(positional_cell 的 + unreadable),遮没遮核对不了,宁可多遮。以前当作认不出出处,只按出处认的写作目录、可验证报告照给,号码随卡出去了。""" node_id: str - card: dict[str, Any] + card: dict[str, Any] | None picked: tuple[int, ...] whole: bool + unreadable: str | None = None def card_input(item: dict[str, Any], output: Callable[[Any], Any]) -> CardInput | None: """卡上的一个输入是不是转抄上游口径卡的产出:nodes.<卡>.metrics[0].value、vars.<卡的 assign_to>.metrics[0].value、 nodes.<卡>.text(清单文字逐条写着「指标(名称)= 值」)、整张卡。是的话交回那张卡和带着值的那几个指标,值给不给按 - 那几个指标认(证据面板、写作目录、可验证报告同一个判断):上游的指标取自遮罩格,转抄它的这个指标同样取自遮罩格。 - 以前这几种写法认不出出处,转抄来的号码照样进了写作目录、文档的证据目录和可验证报告。 + 那几个指标认(masked_closure:证据面板、写作目录、推演、可验证报告同一个判断):上游的指标取自遮罩格,转抄它的这个 + 指标同样取自遮罩格。 output 同 input_origin:节点 id → 那个节点当时的产出;nodes.X 是 X 的产出,vars.X 是写它的节点(输入记着的 node_id) - 的产出——口径卡的 assign_to 交给变量的就是它的产出本身。产出不是指标集的(别的节点)交回 None。只取口径名、版本、指标 - 的名称、单位这类不带值的字段的,也交回 None。""" + 的产出——口径卡的 assign_to 交给变量的就是它的产出本身。产出不是指标集的(别的节点)交回 None;卡上记着这个输入出自 + 口径卡(via 为 metric)、转抄来了值,产出却读不出来的交回 unreadable(见 CardInput)。只取口径名、版本、指标的名称、 + 单位这类不带值的字段的,也交回 None。""" try: tree = ast.parse(str(item.get("path") or ""), mode="eval").body except SyntaxError: @@ -622,26 +656,151 @@ def card_input(item: dict[str, Any], output: Callable[[Any], Any]) -> CardInput if not isinstance(tree, ast.Name) or tree.id not in ("vars", "nodes") or not keys or not isinstance(keys[0], str): return None node_id = keys[0] if tree.id == "nodes" else item.get("node_id") + rest = keys[1:] + whole = not rest or rest[0] == "text" or rest[0] is _ANY \ + or (rest[0] == "metrics" and (len(rest) == 1 or rest[1] is _ANY)) + plain = len(rest) > 2 and isinstance(rest[2], str) and rest[2] in _METRIC_PLAIN + if not whole and (rest[0] != "metrics" or plain): + # 口径名、版本、工件 id,指标的名称、单位这类,或者产出里本来没有的键:不带指标的值 + return None card = output(node_id) if node_id else None if not (isinstance(card, dict) and card.get("kind") == "metric_set" and isinstance(card.get("metrics"), list)): + if node_id and item.get("via") == "metric" and item.get("value") is not None: + return CardInput(str(node_id), None, (), whole, CARD_UNREADABLE) return None - metrics, rest = card["metrics"], keys[1:] - every = tuple(range(len(metrics))) - if not rest or rest[0] == "text" or rest[0] is _ANY or (rest[0] == "metrics" and len(rest) == 1): - return CardInput(str(node_id), card, every, True) - if rest[0] != "metrics": - # 口径名、版本、工件 id 这类,或者产出里本来没有的键:不带指标的值 - return None + metrics = card["metrics"] + if whole: + return CardInput(str(node_id), card, tuple(range(len(metrics))), True) index = rest[1] - if index is _ANY: - return CardInput(str(node_id), card, every, True) if not isinstance(index, int) or not -len(metrics) <= index < len(metrics): return None - if len(rest) > 2 and isinstance(rest[2], str) and rest[2] in _METRIC_PLAIN: - return None return CardInput(str(node_id), card, (index % len(metrics),), False) +@dataclass(frozen=True) +class CopyEdge: + """指标依赖图上的一条转抄边(masked_closure):卡上路径为 path 的输入转抄了上游 upstream 那几个指标(键由调用方定, + 和 MetricLinks 的键同一套)。whole、unreadable 同 CardInput;读不出来的 upstream 为空。""" + + path: str + upstream: tuple[Any, ...] + whole: bool + unreadable: str | None = None + + +@dataclass +class MetricLinks: + """指标依赖图上的一个指标(masked_closure 的输入):metric 是卡上的那个指标(认表达式的最外层引用、值);direct 是 + 转抄以外的输入里不给值的,路径 → (种类, 遮罩列),种类同 api/evidence.py 的 _Held(masked、table、unverified、value); + uncertain:有认不出出处、指标的值要按值认的输入;copies:转抄上游口径卡的输入。""" + + metric: dict[str, Any] + direct: dict[str, tuple[str, tuple[str, ...]]] = field(default_factory=dict) + uncertain: bool = False + copies: list[CopyEdge] = field(default_factory=list) + + +@dataclass(frozen=True) +class Judged: + """masked_closure 认出来的一个指标:word 是指标的值给不给(None 照给,masked 不给,unverified 核对不了,同 + withheld_value);copied 是每个转抄的输入:路径 → (种类, 遮罩列, 原因),种类 None 照给,masked、unverified 同直接的 + 输入,table 是整张卡转抄、上游有指标不给(同整份查询结果:指标的值另按值认),table_unverified 是整张卡转抄、上游有 + 指标核对不了,原因只在上游产出读不出来时有;columns 是让它不给的遮罩列,转抄它的输入上写这几列。""" + + word: str | None + copied: Mapping[str, tuple[str | None, tuple[str, ...], str | None]] = field(default_factory=dict) + columns: tuple[str, ...] = () + + +#: 指标的值给不给,从松到严:照给、核对不了、不给 +_STRICTNESS = {None: 0, "unverified": 1, "masked": 2} + + +def masked_closure(links: Mapping[Any, MetricLinks], known: Mapping[Any, Judged] | None = None, + secrets: Callable[[], Any] | None = None) -> dict[Any, Judged]: + """转抄上游口径卡的指标值给不给:在指标依赖图上一次算出的最小不动点。证据面板、写作目录、推演、可验证报告都按这一个 + 函数认(各自按自己的口径认好转抄以外的输入,见 MetricLinks.direct),不各写一套。 + + 先按转抄以外的输入认每个指标(取自遮罩格的、核对不了的),再沿转抄的边往下游传:上游有一个指标不给,转抄它的输入就是 + masked(整张卡、清单文字、整份指标列表、下标不是字面量的是 table,指标的值另按值认);上游只是核对不了的,转抄它的 + 输入核对不了。一直传到不再变化。环(循环里一张卡抄自己上一轮的产出、几张卡互相转抄)本身不带来遮罩:从「都照给」 + 起往上收,环上没有哪个指标别的输入取自遮罩格,整个环照给;有一个取自遮罩格,整个环都不给。以前逐个往上递归、追够 + 八层就按遮罩算:没设任何遮罩的工作流里,循环里累加上一轮结果的口径卡也成了「已遮罩」,引用它的正式运行判失败; + 每个指标的护栏 nodes.acc 选中整张卡的全部指标,递归次数随指标个数指数增长,卡住事件循环。 + + links:这次要认的指标(键 → MetricLinks),转抄的边指到的上游要么也在里面,要么在 known 里(之前认好的,结果不再变), + 都不在的(卡上没记输入的老指标、写死的数)照给。secrets 同 withheld_value,整张卡转抄、认不出出处的输入要按值认时才取。 + 每个指标的结论只会往严里变(最多两次),每变一次只重认转抄它的那几个:总的工作量和边数成正比。""" + known = known or {} + words: dict[Any, str | None] = {key: None for key in links} + columns: dict[Any, tuple[str, ...]] = {key: () for key in links} + below: dict[Any, set[Any]] = {} + for key, node in links.items(): + for edge in node.copies: + for up in edge.upstream: + if up in links: + below.setdefault(up, set()).add(key) + + def state(key: Any) -> tuple[str | None, tuple[str, ...]]: + if key in links: + return words[key], columns[key] + found = known.get(key) + return (found.word, found.columns) if found is not None else (None, ()) + + def judge(key: Any) -> Judged: + node = links[key] + masked = [p for p, (kind, _) in node.direct.items() if kind in ("masked", "value")] + unverified = [p for p, (kind, _) in node.direct.items() if kind == "unverified"] + uncertain = node.uncertain or any(kind == "table" for kind, _ in node.direct.values()) + because = [c for kind, cols in node.direct.values() if kind in ("masked", "table") for c in cols] + copied: dict[str, tuple[str | None, tuple[str, ...], str | None]] = {} + for edge in node.copies: + if edge.unreadable is not None: + copied[edge.path] = ("table_unverified" if edge.whole else "unverified", (), edge.unreadable) + else: + ups = [state(up) for up in edge.upstream] + if any(word == "masked" for word, _ in ups): + cols = tuple(dict.fromkeys(c for word, cs in ups if word == "masked" for c in cs)) + copied[edge.path] = ("table" if edge.whole else "masked", cols, None) + elif any(word == "unverified" for word, _ in ups): + copied[edge.path] = ("table_unverified" if edge.whole else "unverified", (), None) + else: + copied[edge.path] = (None, (), None) + kind, cols, _ = copied[edge.path] + if kind == "masked": + masked.append(edge.path) + elif kind == "unverified": + unverified.append(edge.path) + elif kind is not None: + uncertain = True + if kind in ("masked", "table"): + because.extend(cols) + word = withheld_value(node.metric, node.metric.get("value"), masked=masked, unverified=unverified, + uncertain=uncertain, secrets=secrets) + return Judged(word, copied, tuple(dict.fromkeys(because)) if word == "masked" else ()) + + queue = list(links) + waiting = set(queue) + while queue: + key = queue.pop() + waiting.discard(key) + got = judge(key) + # 只往严里收:withheld_value 先看取自遮罩格的、再看核对不了的,上游变严时这里也不会松回去 + word = got.word if _STRICTNESS[got.word] >= _STRICTNESS[words[key]] else words[key] + cols = tuple(dict.fromkeys((*columns[key], *got.columns))) if word == "masked" else () + if (word, cols) != (words[key], columns[key]): + words[key], columns[key] = word, cols + for down in below.get(key, ()): + if down not in waiting: + waiting.add(down) + queue.append(down) + out: dict[Any, Judged] = {} + for key in links: + got = judge(key) + out[key] = Judged(words[key], got.copied, columns[key]) + return out + + def positional_cell(path: str, node_id: Any, output: Callable[[Any], Any], *, wide: bool = False) -> dict[str, Any] | None: """按位置取自一份查询结果的一格:vars.res.rows[0][2]、nodes.parse.rows[1]['amount'] 这种标量引用——查询结果经数据 整形解析成对象之后按行号、列号(或列名)取。返回 {artifact, source, row, column}:artifact、source 是那份结果里带着的 @@ -1212,7 +1371,8 @@ def _unsupported(plans: list[_Plan]) -> list[dict[str, Any]]: for inp in plan.inputs.values(): if blocked and not (inp.masked or inp.problem): continue - reason = (MASKED_VALUE_INPUT if inp.by_value else MASKED_TABLE if inp.table else MASKED_INPUT) if inp.masked \ + reason = (MASKED_VALUE_INPUT if inp.by_value else MASKED_COPY if inp.copy else MASKED_TABLE if inp.table + else MASKED_INPUT) if inp.masked \ else f"无法推演:{inp.problem}" if inp.problem \ else SELECTOR if inp.selector else _fixed(inp) if reason: diff --git a/backend/tests/test_bundle_shown.py b/backend/tests/test_bundle_shown.py index 68c89459..551e4a7a 100644 --- a/backend/tests/test_bundle_shown.py +++ b/backend/tests/test_bundle_shown.py @@ -672,13 +672,13 @@ def test_characters_that_may_show_as_anything_next_to_a_value_cannot_pass(ref, c def test_uncertain_characters_where_they_cannot_change_the_value_still_pass(): - """说不准的字挨着的不是数字的头尾(销售额以「元」结尾,后面跟一个控制字符),表情后面的变体选择符,数字前面的波浪线, - 都改不了读到的数:照常通过。""" + """说不准的字挨着的不是数字的头尾(销售额以「元」结尾,后面跟一个控制字符),数字前面的波浪线,表情和数字之间隔着空格, + 都改不了读到的数:照常通过。表情紧挨着数字的见下面「不在常见写法里」的几条。""" from tests.bundle_samples import formats_bundle def change(doc: dict[str, Any]) -> None: by_id(doc, "s5")["text"] = chr(0x7F) + chr(0x0336) + by_id(doc, "s5")["text"] - by_id(doc, "s12")["text"] += chr(0x2705) + chr(0xFE0F) # 转化率紧前面:对勾表情加上表情样式的变体选择符 + by_id(doc, "s12")["text"] += chr(0x2705) + chr(0xFE0F) + " " # 转化率前面:对勾表情、变体选择符,再隔一个空格 by_id(doc, "s14")["text"] += chr(0x301C) # 环比紧前面:波浪线不是横线,不算负号 got = verdicts(rewritten(formats_bundle(), change)) @@ -809,6 +809,62 @@ def test_a_lookalike_digit_next_to_a_checked_value_cannot_pass(ref, change): assert got["overall"]["result"] == "partial", got["overall"] +ODD = UNSURE + "紧挨着这个数的字不在常见写法里,无法确认读者看到的数" + + +@pytest.mark.parametrize("ref, change, reason", [ + # 长得像 l、I、1 的字放在销售额前面(复核第 2 轮的伪造写法):西里尔、希腊、拉丁的竖线字母,ASCII 竖线,汉字「丨」, + # 全角竖线、竖排的破折号、制表符的竖线 + *[("m:gmv", _glue_before("s3", chr(cp)), LOOKALIKE) + for cp in (0x0406, 0x0399, 0x04C0, 0x01C0, 0x7C, 0x4E28, 0xFF5C, 0xFE31, 0x2502)], + # 长得像 O、0 的字接在流水后面:西里尔、希腊的 O,空心圆,小写 o + *[("v:Q1.r0.gmv", _glue_after("s34", chr(cp)), LOOKALIKE) for cp in (0x041E, 0x039F, 0x25CB, 0x6F)], + # 名单以外的字:数学粗体 l、谚文的 ㅣ、全角拉丁字母、日文假名、表情(带变体选择符)、零宽空格、各种窄空格和不断行空格, + # 当笔画用的汉字(丶、丿、亅),前面不是字母的句点(读起来像小数点) + *[("m:gmv", _glue_before("s3", text), ODD) + for text in (chr(0x1D425), chr(0x3163), chr(0xFF21), chr(0x3042), chr(0x2705) + chr(0xFE0F), chr(0x200B), + "1" + chr(0x2006), "1" + chr(0x2009), "1" + chr(0x2008), "1" + chr(0x202F), chr(0xA0), "\t", + chr(0x4E36), chr(0x4E3F), chr(0x4E85), ".")], + # 量级单位的易混字接在流水后面:西里尔、希腊的 M、K、B,开尔文符号,小号百分号,谚文的 ㅇ + *[("v:Q1.r0.gmv", _glue_after("s34", chr(cp)), ODD) + for cp in (0x041C, 0x039C, 0x212A, 0x041A, 0x0392, 0xFE6A, 0x3147)], + # 隔着各种空白、左括号再接量级单位:换行、制表、em / en / thin 空格、全角空格、不断行空格,「(万元)」「十万」「w」 + *[("v:Q1.r0.gmv", _glue_after("s34", text), SCALED) + for text in ("\n万", "\t万", chr(0x2003) + "万", chr(0x2002) + "万", chr(0x2009) + "万", chr(0x3000) + "万", + chr(0xA0) + "万", "\n亿元", "(万元)", " (万)", "十万", "w", "W")], + # 负号前面是别的文字系统里长得像标点的字母,不是 ASCII 字母数字:照旧是负号 + *[("m:conv_rate", lambda doc, t=text: by_id(doc, "s12").update(text=t), result) + for text, result in (("转化率\ua78f-", GLUED), ("转化率\u01c0-", GLUED), ("转化率\ua78c-", GLUED), + ("转化率\ua78f\u2212", GLUED), ("转化率\u0410-", GLUED), ("转化率\ua78f\u2010", DASHED))], + # 科学记数法:后面接 e 和数字 + ("v:Q1.r0.gmv", _glue_after("s34", "e3"), GLUED), + ("v:Q1.r0.gmv", _glue_after("s34", "e+3"), GLUED), +]) +def test_a_character_outside_the_common_writing_next_to_a_value_is_not_green(ref, change, reason): + """紧挨着核对过的数字头尾的那一个字改成白名单(COMMON):汉字、中文标点、半角和全角空格、ASCII 字母(l、I、O、o 照旧算 + 像数字的字母)、常用的半角标点和算式符号、上一轮放行的破折号写法以外的,一律判无法核对、写明原因,总览不再是绿色的 + 「全部通过」。复核第 2 轮用这些写法伪造(只改相邻的普通文字、重算全部哈希),以前全是绿的、提示也不显示。负号前面只认 + ASCII 字母数字是编号;后面接 e 和数字的读作科学记数法,判不通过。""" + result = "fail" if reason == GLUED else "na" + got = _only(ref, change, result, reason) + assert got["overall"]["result"] == ("fail" if result == "fail" else "partial"), got["overall"] + + +def test_common_punctuation_next_to_checked_values_still_passes(): + """常见的写法照常通过:等号、半角括号和冒号、加号、斜杠、百分比前面的加号、半角逗号和句点结句、约等号和乘号、编号后面的 + 句点(NO.)、全角的冒号括号顿号。""" + from tests.bundle_samples import formats_bundle + + text = ("## 算式\n\n客单价=[[m:avg]],销售额:[[m:gmv]],转化率(+[[m:conv_rate]]),订单 [[v:Q1.r0.order_no]], 另计。" + "流水约≈[[v:Q1.r0.gmv]]/月,退款×[[m:refund]],编号No.[[m:orders]],出口额:[[m:export]]、" + "年度([[m:annual|亿]])。合计 [[v:Q1.r1.gmv]].") + got = verdicts(formats_bundle(text=text)) + assert len(got["numbers"]) == 10, got["numbers"] + assert all(n["result"] == "pass" and n["onPage"] == n["text"] for n in got["numbers"]), \ + [n for n in got["numbers"] if n["result"] != "pass"] + assert got["overall"]["result"] == "pass", got["overall"] + + def test_ordinary_writing_next_to_checked_values_still_passes_as_exported(): """正文里的普通写法照常通过,原样导出总览全绿:「统一」后面紧跟数(汉字「一」不是负号)、破折号(——、—、–、―, 小号的)后面紧跟数、编号「SKU-」「A-」后面紧跟数(连字符前面是字母,是编号的一部分)、「共…家」、全角括号、区间、 diff --git a/backend/tests/test_report_masked_card_copies.py b/backend/tests/test_report_masked_card_copies.py index 32ba8350..5d5c1b23 100644 --- a/backend/tests/test_report_masked_card_copies.py +++ b/backend/tests/test_report_masked_card_copies.py @@ -4,9 +4,10 @@ nodes.card.text(清单文字逐条写着「指标(名称)= 值」),取值链认不出出处(recompute.input_origin 只认查询结果),只按出处认 时直接照给:写作目录写着号码,文档的证据目录里值、渲染、标签都是号码,可验证报告里报告文档和卡 2 的节点完成事件都带着号码。 -现在转抄上游口径卡的输入按上游那几个指标认(recompute.card_input,api/evidence.py 的 _InputMasks._copied,bundle/build.py -的 _Store._copied_kind):上游的指标取自遮罩格,转抄它的这个指标同样写「已遮罩」;整张卡、清单文字同整份查询结果,值里有 -遮罩值才不给(数一数有几个指标的照给)。写作目录、报告自查、可验证报告同一个判断。 +现在转抄上游口径卡的输入按上游那几个指标认(recompute.card_input 认出转抄了哪张卡的哪几个指标,recompute.masked_closure +在指标依赖图上一次认,api/evidence.py 的 _CopyGraph、bundle/build.py 的 _Store.metric_word 各自按自己的口径认好转抄以外的 +输入交进去):上游的指标取自遮罩格,转抄它的这个指标同样写「已遮罩」;整张卡、清单文字同整份查询结果,值里有遮罩值才不给 +(数一数有几个指标的照给)。写作目录、报告自查、可验证报告同一个判断。 真跑的部分用虚构的 SQLite 会员库(号码是编的),写作者是 MockChatModel 照剧本写。 """ @@ -205,10 +206,10 @@ async def test_the_panel_withholds_the_copied_number(client, monkeypatch, vip): # -------------------------------------------------------------------------- -# 手搭:转抄追不到头 +# 手搭:认转抄、循环里抄自己上一轮的产出 # -------------------------------------------------------------------------- -#: 一张卡转抄自己上一轮的产出(循环里写成这样):顺着取值链永远追不到最初取自哪里 +#: 一张卡转抄自己上一轮的产出(循环里写成这样):写作目录只有最后一次的产出,指回自己,是一个环 LOOP_PATH = "nodes.loop_card.metrics[0].value" LOOPING = {"kind": "metric_set", "caliber": "循环口径", "caliber_version": "v1", "artifact": "c" * 64, "metrics": [ {"id": "p", "name": "上一轮的号码", "value": PHONES[0], "format": "plain", "expression": LOOP_PATH, @@ -217,15 +218,18 @@ LOOPING = {"kind": "metric_set", "caliber": "循环口径", "caliber_version": " def test_recognizing_which_card_metrics_an_input_copies(): """认转抄:带值的字段(值、渲染、输入、整个指标)算转抄了那个指标;整张卡、清单文字、下标不是字面量的算整份;只取名称、 - 单位、口径名的不算;产出不是指标集的节点不算。""" - from app.engine.recompute import card_input + 单位、口径名的不算;产出不是指标集的节点不算。卡上记着出自口径卡(via 为 metric)、转抄来了值,那张卡的产出却读不出来 + 的,交回读不出来的原因(unreadable);转抄来的是空值(第一轮还没有上一轮)、不出自口径卡的不算。""" + from app.engine.recompute import CARD_UNREADABLE, card_input def output(node_id): return LOOPING if node_id == "loop_card" else {"rows": [[1]], "columns": ["a"]} if node_id == "q" else None - def picked(path, node_id=None): - got = card_input({"path": path, "node_id": node_id}, output) - return None if got is None else (got.picked, got.whole) + def picked(path, node_id=None, **item): + got = card_input({"path": path, "node_id": node_id, **item}, output) + if got is None: + return None + return (got.picked, got.whole) if got.unreadable is None else (got.unreadable, got.whole) assert picked("nodes.loop_card.metrics[0].value") == ((0,), False) assert picked("nodes.loop_card.metrics[-1]['rendered']") == ((0,), False) @@ -240,25 +244,56 @@ def test_recognizing_which_card_metrics_an_input_copies(): assert picked("nodes.loop_card.caliber") is None assert picked("nodes.loop_card.metrics[3].value") is None assert picked("nodes.q.rows[0][0]") is None + # 那张卡的产出读不出来 + assert picked("nodes.gone.metrics[0].value", via="metric", value=PHONES[0]) == (CARD_UNREADABLE, False) + assert picked("vars.kpi.metrics[0].value", node_id="gone", via="metric", value=1) == (CARD_UNREADABLE, False) + assert picked("nodes.gone.text", via="metric", value="清单") == (CARD_UNREADABLE, True) + assert picked("nodes.gone.metrics[0].value", via="metric", value=None) is None + assert picked("nodes.gone.metrics[0].value", via="transform", value=1) is None + assert picked("nodes.gone.caliber", via="metric", value="口径") is None + assert picked("nodes.gone.metrics[0].name", via="metric", value="名称") is None -async def test_a_copy_too_deep_to_follow_counts_as_masked(): - """转抄追了 recompute.CARD_DEPTH 张还没追到头(这里是一张卡转抄自己上一轮的产出):宁可多遮,按取自遮罩格算——写作目录 - 和可验证报告都不给值。核对不了就照给的话,号码就从这里出去了。""" +def _looping(masked: bool) -> tuple[dict, dict[str, bytes], str | None]: + """循环里抄自己上一轮产出的卡;masked 时另有一个输入取自号码列(第一轮还没有上一轮时取它)。交回 (卡, 工件, 快照 id)。""" + from app.core.artifact_store import _path_of, canonical_json, content_hash + + if not masked: + return LOOPING, {}, None + snap = {"columns": ["id", "phone"], "rows": [[1, PHONES[0]]], "sql": "SELECT id, phone FROM customers", + "mask_columns": ["phone"], "truncated": False} + text = canonical_json(snap) + snap_id = content_hash(text) + _path_of(snap_id).parent.mkdir(parents=True, exist_ok=True) + _path_of(snap_id).write_text(text, encoding="utf-8") + cell = "cell(nodes.q, 0, 'phone')" + metric = {**LOOPING["metrics"][0], "expression": f"{LOOP_PATH} if nodes.loop_card else {cell}", "inputs": [ + *LOOPING["metrics"][0]["inputs"], + {"path": cell, "value": PHONES[0], "node_id": "q", "via": "tool_cell", "artifact": snap_id, + "locator": {"row": 0, "column": "phone"}, "status": "verified"}]} + return {**LOOPING, "metrics": [metric]}, {snap_id: text.encode()}, snap_id + + +@pytest.mark.parametrize("masked", [False, True]) +async def test_a_card_copying_its_own_last_round_is_judged_on_the_cycle(masked): + """一张卡转抄自己上一轮的产出:写作目录(只有最后一次的产出,指回自己)和可验证报告(这里同样只有这一份产出)都在依赖 + 图上认这个环。环本身不带来遮罩:没有别的输入时照给(以前追够八层就按遮罩算);另有一个输入取自号码列时,整个环都不给。""" from app.api.evidence import MASKED, _Masks, _metric_masks, _Sealed from app.bundle.build import _Store from app.core.artifact_store import canonical_json, content_hash + looping, files, _ = _looping(masked) view = _Sealed(run_id="", graph={}, events=[], seal={"sealed": False, "ok": None, "manifest_seq": None, "legacy": False}) - metric = LOOPING["metrics"][0] - _, word = await _metric_masks(metric, "loop_card", LOOPING["artifact"], view, _Masks(view), - output=lambda node_id: LOOPING if node_id == "loop_card" else None, + metric = looping["metrics"][0] + _, word = await _metric_masks(metric, "loop_card", looping["artifact"], view, _Masks(view), card=looping, + output=lambda node_id: looping if node_id == "loop_card" else None, by_value=False, sure=True) - assert word == MASKED - raw = canonical_json(LOOPING).encode() + assert word == (MASKED if masked else None) + raw = canonical_json(looping).encode() out_id = content_hash(raw) - store = _Store({out_id: raw}.get, {}, [(1, "node.finished", "loop_card", {"artifact": out_id})]) + store = _Store({out_id: raw, **files}.get, {}, [(1, "node.finished", "loop_card", {"artifact": out_id})]) card = store.content(out_id) - assert store.metric_word(LOOPING["artifact"], card["metrics"][0], store._events, sure=True) == "masked" - assert store.metric_word(LOOPING["artifact"], card["metrics"][0], store._events) == "masked" + assert store.metric_word(looping["artifact"], card["metrics"][0], store._events, sure=True) == \ + ("masked" if masked else None) + assert store.metric_word(looping["artifact"], card["metrics"][0], store._events) == ("masked" if masked else None) diff --git a/backend/tests/test_report_masked_copy_closure.py b/backend/tests/test_report_masked_copy_closure.py new file mode 100644 index 00000000..375ea6b0 --- /dev/null +++ b/backend/tests/test_report_masked_copy_closure.py @@ -0,0 +1,766 @@ +"""转抄上游口径卡的指标给不给值:在指标依赖图上一次算出的最小不动点(recompute.masked_closure)。 + +复核第 2 轮发现:转抄的识别逐个往上递归、追够八层(CARD_DEPTH)就按遮罩算。循环里累加自己上一轮产出的口径卡 +(nodes.acc.metrics[0].value + cell(nodes.q, 0, 'amount')),写作目录里的产出指回自己,递归一直追到八层,没设任何遮罩的 +工作流里也写成「已遮罩」,正式运行判失败、报错说「取自设置了遮罩的列」;每个指标的护栏 nodes.acc 选中整张卡的全部指标, +递归次数约为(指标数 + 1)的八次方,四个指标就要四十秒,事件循环被占住。封存之后循环超过八轮的,证据面板写「已遮罩」, +可验证报告把整份文档只给哈希,说法都不实。 + +现在:先按转抄以外的输入认每个指标(只按确定取自遮罩列的那一套,写作目录、可验证报告认老文档;证据面板、推演宁可多遮), +再沿转抄的边往下游传,直到不再变化。环(循环里抄自己上一轮的产出、几张卡互相转抄)本身不带来遮罩,环上有一个指标别的 +输入取自遮罩格,整个环都不给。写作目录、证据面板、推演、可验证报告共用这一个函数,一次算完、按请求记下,和追了几层无关。 + +另有几处同属这一判断:上游卡当时的产出读不出来(文件不在、被改过)时转抄的值核对不了,可验证报告照样不给(以前当作认不出 +出处,号码随卡出去了);卡上记着转抄来的清单文字(里面写着号码)的卡不随文件附带,哪怕指标的值只是个字数;推演里转抄的输入 +按同一张依赖图认(以前只按值认);* 展开派生表、WITH 里没起别名的号码表达式(UPPER(phone))算确定取自遮罩列。 + +真跑的部分用虚构的 SQLite 库(号码是编的),写作者是 MockChatModel 照剧本写。 +""" +from __future__ import annotations + +import asyncio +import sqlite3 +import threading +import time + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select + +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource, RunEvent +from app.engine.evidence import iter_segments +from app.engine.runner import run_manager +from app.main import app +from tests.test_bundle_masked_metrics import data_of, event +from tests.test_report_masked_metrics import ( # noqa: F401 vip:号码设了遮罩的会员库 + PHONES, + doc_of, + events, + finish, + leaks, + metric_lines, + node, + script, + vip, +) + + +@pytest.fixture +async def engine_up(): + await run_manager.setup() + yield + await run_manager.shutdown() + + +@pytest.fixture +async def client(engine_up): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +async def plain(tmp_path, engine_up): + """没设任何遮罩的销售库:三个月的销售额,合计 425.5。""" + path = tmp_path / "plain.db" + db = sqlite3.connect(path) + db.executescript("CREATE TABLE sales (month INTEGER, amount REAL);") + db.executemany("INSERT INTO sales VALUES (?, ?)", [(1, 100.0), (2, 250.0), (3, 75.5)]) + db.commit() + db.close() + async with SessionLocal() as session: + row = (await session.execute(select(DataSource).where(DataSource.name == "plainsrc"))).scalar_one_or_none() + if row is None: + row = DataSource(name="plainsrc", kind="sqlite", database=str(path), readonly=True, enabled=True) + session.add(row) + row.database, row.kind, row.enabled, row.readonly, row.options = str(path), "sqlite", True, True, {} + await session.commit() + source_id = row.id + await engines.invalidate(source_id) + yield path + await engines.invalidate(source_id) + + +#: 循环里累加自己上一轮的产出:第一轮 nodes.acc 还没有,记 0 +ACC = "(nodes.acc.metrics[0].value if nodes.acc else 0) + cell(nodes.q, 0, 'amount')" + + +def loop_graph(rounds: int, cards: list[dict], *, sql: str = "SELECT SUM(amount) AS amount FROM sales", + source: str = "plainsrc", report: bool = True, **write) -> dict: + """入口 → 循环(foreach rounds 轮:查询 → 几张口径卡)→ 报告(report 为假时没有)→ 成果。cards 是 (节点 id, 指标) 列表。""" + start = node("start", "input") + loop = node("lp", "loop", mode="foreach", items=str(list(range(rounds)))) + query = node("q", "tool", tool=f"db_query__{source}", args={"sql": sql}) + built = [node(c["id"], "metrics", caliber=f"{c['id']} 口径", caliber_version="v1", metrics=c["metrics"]) + for c in cards] + tail = [node("write", "report", instructions="写一句", **{"on_violation": "fail", **write})] if report else [] + out = node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}] if report + else [{"name": "说明", "value": "完成"}]) + edges = [{"source": "start", "target": "lp"}, {"source": "lp", "target": "q", "sourceHandle": "body"}] + chain = ["q", *(c["id"] for c in cards)] + edges += [{"source": a, "target": b} for a, b in zip(chain, chain[1:])] + edges.append({"source": chain[-1], "target": "lp"}) + after = ["lp", *(n["id"] for n in tail), "out"] + edges += [{"source": a, "target": b, **({"sourceHandle": "done"} if a == "lp" else {})} + for a, b in zip(after, after[1:])] + return {"nodes": [start, loop, query, *built, *tail, out], "edges": edges} + + +def total(expression: str = ACC, metric_id: str = "total", name: str = "累计销售额") -> dict: + return {"id": metric_id, "name": name, "unit": "元", "decimals": 2, "format": "thousands", "expression": expression} + + +async def export(client, run_id: str) -> tuple[str, dict]: + resp = await client.get(f"/api/runs/{run_id}/bundle.html") + assert resp.status_code == 200, resp.text + return resp.text, data_of(resp.text) + + +def cards_of(bundle: dict) -> list[dict]: + return [a for a in bundle["artifacts"] if a["kind"] == "metric_set"] + + +# -------------------------------------------------------------------------- +# 环本身不带来遮罩:没设遮罩的循环累加照常显示、正式运行成功 +# -------------------------------------------------------------------------- + + +async def test_a_card_that_adds_its_own_last_round_shows_its_value_and_the_formal_run_succeeds(client, monkeypatch, + plain): + """复核者的探针:数据源没设遮罩,循环三轮,卡 acc 累加自己上一轮的产出,报告「违规处置」为判失败、引用 [[m:total]]。 + 写作目录写「累计销售额 = 1,276.50元」,正文写出这个数,运行成功;证据面板的标签、点开这一处都照给;可验证报告附带 + 报告文档和口径卡。以前写作目录写「已遮罩(取自设置了遮罩的列,不要引用)」,运行判失败。""" + seen = script(monkeypatch, "累计销售额 [[m:total]]。") + row = await finish(loop_graph(3, [{"id": "acc", "metrics": [total()]}])) + assert row.status == "succeeded", row.error + assert "累计销售额 = 1,276.50元" in metric_lines(seen[0]), metric_lines(seen[0]) + doc = await doc_of(row.id) + assert "1,276.50元" in "".join(s["text"] for s in iter_segments(doc)) and "masked" not in doc["catalog"]["m:total"] + [checked] = [e.data for e in await events(row.id) if e.type == "report.checked"] + assert checked["ok"] is True and not checked["violations"], checked + + body = (await client.get(f"/api/runs/{row.id}/evidence")).json() + labels = {e["alias"]: e["label"] for e in body["evidence"] if e["kind"] == "metric"} + assert labels["m:total"] == "累计销售额 = 1,276.50元", labels + seg = next(s for s in iter_segments(doc) if s.get("ref") == "m:total") + chain = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}")).json()["chain"] + assert chain[0]["value"] == 1276.5 and chain[0]["render_ok"], chain[0] + copied = next(i for i in chain if i.get("step") == "input" and i.get("path") == "nodes.acc.metrics[0].value") + assert copied["value"] == 851.0 and "masked" not in copied and "mask_unverified" not in copied, copied + + _, bundle = await export(client, row.id) + [art] = [a for a in bundle["artifacts"] if a["kind"] == "report_doc"] + assert "b64" in art and "masked" not in art, art + assert cards_of(bundle) and all("b64" in a and "masked" not in a for a in cards_of(bundle)), cards_of(bundle) + assert all("canonical" in e for e in bundle["events"] if e["type"] == "node.finished" and e["node_id"] == "acc") + + +@pytest.mark.parametrize("rounds", [3, 10]) +async def test_an_old_doc_from_a_long_loop_without_masks_is_embedded(client, monkeypatch, plain, rounds): + """封存之后按每次执行往回认:第 k 轮的卡读到的是第 k - 1 轮的产出,一轮一轮连到第一轮为止。循环十轮的老文档(写作前 + 不认遮罩,这里把那一步换成什么都不认)照常随文件附带,证据面板照给。以前超过八轮就按遮罩算:面板写「已遮罩」,文档 + 整份只给哈希,页面说「证据目录里写着取自遮罩列的指标的值」,总览「有 1 份报告未能核对」。""" + import app.api.evidence as evidence_api + + async def nothing(*_args, **_kwargs): + return [] + + monkeypatch.setattr(evidence_api, "withheld_metrics", nothing) + script(monkeypatch, "累计销售额 [[m:total]]。") + row = await finish(loop_graph(rounds, [{"id": "acc", "metrics": [total()]}])) + assert row.status == "succeeded", row.error + body = (await client.get(f"/api/runs/{row.id}/evidence")).json() + labels = {e["alias"]: e["label"] for e in body["evidence"] if e["kind"] == "metric"} + assert labels["m:total"] == f"累计销售额 = {425.5 * rounds:,.2f}元", labels + _, bundle = await export(client, row.id) + [art] = [a for a in bundle["artifacts"] if a["kind"] == "report_doc"] + assert "b64" in art and "masked" not in art, art + assert all("canonical" in e for e in bundle["events"] if e["type"] == "node.finished" and e["node_id"] == "write") + + +async def test_cards_that_copy_each_other_in_a_loop_show_their_values(client, monkeypatch, plain): + """几张卡互相转抄(acc 抄 peer 上一轮的、peer 抄 acc 这一轮的),没有哪个指标别的输入取自遮罩格:照给,正式运行成功。""" + seen = script(monkeypatch, "累计 [[m:total]],对照 [[m:mirror]]。") + cards = [ + {"id": "acc", "metrics": [ + total("(nodes.peer.metrics[0].value if nodes.peer else 0) + cell(nodes.q, 0, 'amount')")]}, + {"id": "peer", "metrics": [total("(nodes.acc.metrics[0].value if nodes.acc else 0) + 1", "mirror", "对照值")]}, + ] + row = await finish(loop_graph(3, cards)) + assert row.status == "succeeded", row.error + lines = metric_lines(seen[0]) + assert "累计销售额 = 1,278.50元" in lines and "对照值 = 1,279.50元" in lines, lines + _, bundle = await export(client, row.id) + assert all("b64" in a and "masked" not in a for a in cards_of(bundle)), cards_of(bundle) + + +# -------------------------------------------------------------------------- +# 环上有一个指标别的输入取自遮罩格:整个环都不给 +# -------------------------------------------------------------------------- + + +async def test_a_loop_with_a_masked_input_is_withheld_as_a_whole(client, monkeypatch, vip): + """会员库的号码设了遮罩。acc 每轮累加号码除以 7 的余数(输入取自遮罩格,值是算出来的数,宁可多遮),peer 抄 acc、acc 也抄 + peer:两张卡在一个环上,都写「已遮罩」,引用 peer 的正式运行判失败(masked_cell)。同一个循环里只累加自己、没碰号码的 + side 照给。可验证报告里 acc、peer 每一轮的节点完成事件都只给哈希(预览逐条写着「指标(名称)= 值」),side 的照常带 + 规范串。""" + seen = script(monkeypatch, "对照 [[m:mirror]],轮数 [[m:rounds]]。") + cards = [ + {"id": "acc", "metrics": [total("(nodes.peer.metrics[0].value if nodes.peer else 0) " + "+ cell(nodes.q, 0, 'phone') % 7", "digits", "号码余数累计")]}, + {"id": "peer", "metrics": [total("(nodes.acc.metrics[0].value if nodes.acc else 0) + 1", "mirror", "对照值")]}, + {"id": "side", "metrics": [{"id": "rounds", "name": "轮数", "unit": "轮", "format": "thousands", + "expression": "(nodes.side.metrics[0].value if nodes.side else 0) + 1"}]}, + ] + spec = loop_graph(3, cards, sql="SELECT id, phone FROM customers ORDER BY id", source="vip") + row = await finish(spec) + assert row.status == "failed" and "m:mirror" in (row.error or "") and "遮罩" in row.error, row.error + lines = metric_lines(seen[0]) + for name in ("号码余数累计", "对照值"): + assert f"{name} = 已遮罩(取自设置了遮罩的列,不要引用)" in lines, lines + assert "轮数 = 3轮" in lines, lines + [checked] = [e.data for e in await events(row.id) if e.type == "report.checked"][-1:] + assert [v["ref"] for v in checked["violations"] if v["code"] == "masked_cell"] == ["m:mirror"], checked + + spec = loop_graph(3, cards, sql="SELECT id, phone FROM customers ORDER BY id", source="vip", report=False) + row = await finish(spec) + assert row.status == "succeeded", row.error + _, bundle = await export(client, row.id) + finished = {n: [e for e in bundle["events"] if e["type"] == "node.finished" and e["node_id"] == n] + for n in ("acc", "peer", "side")} + assert all(len(found) == 3 for found in finished.values()), finished + assert not any("canonical" in e for n in ("acc", "peer") for e in finished[n]), finished + assert all("canonical" in e for e in finished["side"]), finished["side"] + + +# -------------------------------------------------------------------------- +# 上游卡当时的产出读不出来:转抄的值核对不了,可验证报告照样不给 +# -------------------------------------------------------------------------- + + +def chain(*nodes: dict) -> dict: + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +async def bundle_leaks(client, run_id: str) -> dict: + """导出一份可验证报告:整页里的号码、每份附带的工件里的号码、带规范串的事件里的号码(事件按 序号:类型:节点 记)。""" + import base64 + + html, bundle = await export(client, run_id) + out: dict = {"html": leaks(html), "artifacts": {}, "events": {}} + for art in bundle["artifacts"]: + if "b64" in art and (found := leaks(base64.b64decode(art["b64"]).decode("utf-8", "replace"))): + out["artifacts"][f"{art['kind']}:{art.get('node_id')}"] = found + for item in bundle["events"]: + if "canonical" in item and (found := leaks(item["canonical"])): + out["events"][f"{item['seq']}:{item['type']}:{item['node_id']}"] = found + return out + + +async def _drop_output(run_id: str, node_id: str) -> None: + """删掉一个节点产出的工件文件:服务端缺文件(missing-card 样本那种),导出时读不出来。""" + from app.core.artifact_store import _path_of + + async with SessionLocal() as session: + rows = (await session.execute(select(RunEvent).where(RunEvent.run_id == run_id))).scalars().all() + for row in rows: + if row.type == "node.finished" and row.node_id == node_id and (row.data or {}).get("artifact"): + _path_of(row.data["artifact"]).unlink(missing_ok=True) + + +@pytest.mark.parametrize("expr", ["nodes.card.metrics[0].value", "vars.kpi.metrics[0].value", "nodes.card.text"]) +async def test_a_copy_whose_upstream_output_is_gone_ships_without_the_number(client, monkeypatch, vip, expr): + """卡 2 转抄卡 1 的号码,导出之前卡 1 的节点产出工件不在了(服务端缺文件)。转抄的值核对不了:卡 2 的节点完成事件只给 + 哈希,整个文件里没有号码。以前当作认不出出处、照给,卡 2 的规范串和页面里都带着号码。""" + spec = chain( + node("start", "input"), + node("q_cust", "tool", tool="db_query__vip", args={"sql": "SELECT id, name, phone FROM customers ORDER BY id"}), + node("card", "metrics", caliber="会员口径", caliber_version="v1", assign_to="kpi", metrics=[ + {"id": "phone_raw", "name": "首位客户手机号", "format": "plain", + "expression": "cell(nodes.q_cust, 0, 'phone')"}]), + node("card2", "metrics", caliber="转抄口径", caliber_version="v1", metrics=[ + {"id": "p2", "name": "转抄", "format": "plain", "expression": expr}]), + node("out", "output", fields=[{"name": "说明", "value": "完成"}]), + ) + row = await finish(spec) + assert row.status == "succeeded", row.error + assert await bundle_leaks(client, row.id) == {"html": [], "artifacts": {}, "events": {}} + await _drop_output(row.id, "card") + got = await bundle_leaks(client, row.id) + assert got == {"html": [], "artifacts": {}, "events": {}}, got + _, bundle = await export(client, row.id) + assert "canonical" not in event(bundle, "node.finished", "card2") + + +async def test_a_copy_of_an_ordinary_metric_whose_upstream_output_is_gone_is_still_judged_safely(client, monkeypatch, + vip): + """对照:卡 2 转抄的是客户数(没碰号码),卡 1 的产出不在了也核对不了,卡 2 的节点完成事件同样只给哈希(宁可多挡); + 卡 1 的产出还在时照常带规范串。""" + spec = chain( + node("start", "input"), + node("q_cust", "tool", tool="db_query__vip", args={"sql": "SELECT id, name, phone FROM customers ORDER BY id"}), + node("parse", "transform", mode="json", template="{{ nodes.q_cust }}", assign_to="cust"), + node("card", "metrics", caliber="会员口径", caliber_version="v1", metrics=[ + {"id": "n_cust", "name": "客户数", "unit": "位", "format": "thousands", "expression": "len(vars.cust.rows)"}]), + node("card2", "metrics", caliber="转抄口径", caliber_version="v1", metrics=[ + {"id": "n2", "name": "转抄客户数", "unit": "位", "format": "thousands", + "expression": "nodes.card.metrics[0].value"}]), + node("out", "output", fields=[{"name": "说明", "value": "完成"}]), + ) + row = await finish(spec) + assert row.status == "succeeded", row.error + _, bundle = await export(client, row.id) + assert "canonical" in event(bundle, "node.finished", "card2") + await _drop_output(row.id, "card") + _, bundle = await export(client, row.id) + assert "canonical" not in event(bundle, "node.finished", "card2") + + +# -------------------------------------------------------------------------- +# 最小不动点本身(recompute.masked_closure,纯函数) +# -------------------------------------------------------------------------- + + +def _metric(expression: str, value=1.0) -> dict: + return {"id": "m", "name": "m", "value": value, "expression": expression} + + +def test_a_cycle_on_its_own_brings_no_mask(): + """自己指回自己、两张卡互相转抄:环上没有哪个指标别的输入取自遮罩格,都照给。""" + from app.engine.recompute import CopyEdge, MetricLinks, masked_closure + + a, b, c = ("a", 0), ("b", 0), ("c", 0) + got = masked_closure({ + a: MetricLinks(_metric("nodes.a.metrics[0].value + 1"), + copies=[CopyEdge("nodes.a.metrics[0].value", (a,), False)]), + b: MetricLinks(_metric("nodes.c.metrics[0].value"), copies=[CopyEdge("nodes.c.metrics[0].value", (c,), False)]), + c: MetricLinks(_metric("nodes.b.metrics[0].value + nodes.c"), copies=[ + CopyEdge("nodes.b.metrics[0].value", (b,), False), CopyEdge("nodes.c", (c,), True)]), + }) + assert {k: v.word for k, v in got.items()} == {a: None, b: None, c: None} + assert all(kind is None for v in got.values() for kind, _, _ in v.copied.values()) + + +def test_one_masked_input_on_a_cycle_withholds_the_whole_cycle(): + """环上 b 另有一个输入取自遮罩格(phone):b、c 都不给,转抄它们的输入上写着 phone;只转抄 c 的整张卡、值是个数的 d + 照给(整张卡转抄同整份结果,指标的值另按值认),环外没连上的 e 照给。""" + from app.engine.recompute import CopyEdge, MetricLinks, masked_closure + + b, c, d, e = ("b", 0), ("c", 0), ("d", 0), ("e", 0) + got = masked_closure({ + b: MetricLinks(_metric("nodes.c.metrics[0].value + cell(nodes.q, 0, 'phone')"), + direct={"cell(nodes.q, 0, 'phone')": ("masked", ("phone",))}, + copies=[CopyEdge("nodes.c.metrics[0].value", (c,), False)]), + c: MetricLinks(_metric("nodes.b.metrics[0].value"), copies=[CopyEdge("nodes.b.metrics[0].value", (b,), False)]), + d: MetricLinks(_metric("len(nodes.c.metrics)", 1), copies=[CopyEdge("nodes.c.metrics", (c,), True)]), + e: MetricLinks(_metric("nodes.e.metrics[0].value"), copies=[CopyEdge("nodes.e.metrics[0].value", (e,), False)]), + }, secrets=lambda: None) + assert {k: v.word for k, v in got.items()} == {b: "masked", c: "masked", d: None, e: None} + assert got[c].copied["nodes.b.metrics[0].value"] == ("masked", ("phone",), None) + assert got[b].copied["nodes.c.metrics[0].value"] == ("masked", ("phone",), None) + assert got[d].copied["nodes.c.metrics"] == ("table", ("phone",), None) + + +def test_a_whole_copy_is_withheld_when_its_value_carries_a_masked_value(): + """整张卡、清单文字转抄(table):上游有指标不给时,指标的值里有这次运行查到过的遮罩值才不给。""" + from app.engine.masking import _Secrets + from app.engine.recompute import CopyEdge, MetricLinks, masked_closure + + secrets = _Secrets() + secrets.add({"columns": ["phone"], "rows": [[PHONES[0]]]}, {"phone"}) + up, text, count = ("up", 0), ("text", 0), ("count", 0) + links = { + up: MetricLinks(_metric("cell(nodes.q, 0, 'phone')", PHONES[0]), + direct={"cell(nodes.q, 0, 'phone')": ("masked", ("phone",))}), + text: MetricLinks(_metric("nodes.up.text", f"- p(号码)= {PHONES[0]}"), + copies=[CopyEdge("nodes.up.text", (up,), True)]), + count: MetricLinks(_metric("len(nodes.up.metrics)", 1), copies=[CopyEdge("nodes.up.metrics", (up,), True)]), + } + got = masked_closure(links, secrets=lambda: secrets) + assert (got[up].word, got[text].word, got[count].word) == ("masked", "masked", None) + + +def test_an_unreadable_upstream_cannot_be_checked(): + """上游卡当时的产出读不出来(CopyEdge.unreadable):转抄的输入核对不了;它是最外层的引用时指标的值也核对不了。整张卡 + 转抄的同整份结果(table_unverified),指标的值另按值认。之前认好的上游(known)直接用,不再重认。""" + from app.engine.recompute import CopyEdge, Judged, MetricLinks, masked_closure + + why = "转抄的口径卡结果无法读取" + a, w, k, known = ("a", 0), ("w", 0), ("k", 0), ("known", 0) + got = masked_closure({ + a: MetricLinks(_metric("nodes.up.metrics[0].value"), + copies=[CopyEdge("nodes.up.metrics[0].value", (), False, why)]), + w: MetricLinks(_metric("len(nodes.up.text)", 3), copies=[CopyEdge("nodes.up.text", (), True, why)]), + k: MetricLinks(_metric("nodes.known.metrics[0].value"), + copies=[CopyEdge("nodes.known.metrics[0].value", (known,), False)]), + }, {known: Judged("masked", {}, ("phone",))}, secrets=lambda: None) + assert got[a].word == "unverified" and got[a].copied["nodes.up.metrics[0].value"] == ("unverified", (), why) + assert got[w].word is None and got[w].copied["nodes.up.text"] == ("table_unverified", (), why) + assert got[k].word == "masked" and got[k].copied["nodes.known.metrics[0].value"] == ("masked", ("phone",), None) + assert known not in got + + +def test_a_long_chain_with_many_cycles_is_solved_in_linear_time(): + """两千个指标,每个转抄前一个、自己、整张前一张卡(护栏),只有最前面那个取自遮罩格:全部不给,一秒以内。以前逐个 + 往上递归,次数随护栏选中的指标个数指数增长。""" + from app.engine.recompute import CopyEdge, MetricLinks, masked_closure + + links = {(0, 0): MetricLinks(_metric("cell(nodes.q, 0, 'phone')"), direct={"cell": ("masked", ("phone",))})} + for i in range(1, 2000): + links[(i, 0)] = MetricLinks(_metric("x"), copies=[ + CopyEdge("prev", ((i - 1, 0),), False), CopyEdge("self", ((i, 0),), False), + CopyEdge("guard", ((i - 1, 0), (i, 0)), True)]) + started = time.perf_counter() + got = masked_closure(links, secrets=lambda: None) + assert time.perf_counter() - started < 1.0 + assert all(v.word == "masked" for v in got.values()) + + +# -------------------------------------------------------------------------- +# 合成的长循环:写作目录、可验证报告的判断都在一秒内 +# -------------------------------------------------------------------------- + +#: 三张卡在一个循环里互相转抄:acc 抄 acc2、acc2 抄 acc3、acc3 抄 acc +LOOP_CARDS = ("acc", "acc2", "acc3") + + +def _put(obj) -> str: + """照工件库的样子落一份(规范 JSON,id 是 sha256),写作目录读快照时按 id 去读。""" + from app.core.artifact_store import _path_of, canonical_json, content_hash + + text = canonical_json(obj) + artifact = content_hash(text) + path = _path_of(artifact) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return artifact + + +def _snapshot() -> tuple[str, dict]: + snap = {"columns": ["amount", "phone"], "rows": [[425.5, PHONES[0]]], "sql": "SELECT amount, phone FROM sales", + "mask_columns": ["phone"], "truncated": False} + return _put(snap), snap + + +def _card(card: str, k: int, snap: str, seen: dict, masked: bool) -> dict: + """一轮里卡 card 的产出:k 个指标,第 i 个转抄自己上一轮的第 i 个、整张自己(护栏)、下一张卡的第 i 个和整张下一张卡, + 再加快照里的一格(masked 时 acc 的第一个指标取号码列)。seen 是这一刻每张卡最后一次的产出(节点 id → 产出)。""" + other = LOOP_CARDS[(LOOP_CARDS.index(card) + 1) % len(LOOP_CARDS)] + metrics = [] + for i in range(k): + column = "phone" if masked and card == "acc" and i == 0 else "amount" + inputs = [] + value = 425.5 + for source in (card, other): + before = seen.get(source) + copied = before["metrics"][i]["value"] if before else None + value += copied or 0 + inputs += [{"path": f"nodes.{source}.metrics[{i}].value", "value": copied, "node_id": source, + "via": "metric", "status": "ok" if before else "missing"}, + {"path": f"nodes.{source}", "value": "{6 个字段}" if before else None, "node_id": source, + "via": "metric", "status": "ok" if before else "missing"}] + inputs.append({"path": f"cell(nodes.q, 0, '{column}')", "value": 425.5, "node_id": "q", "via": "tool_cell", + "artifact": snap, "locator": {"row": 0, "column": column}, "status": "verified"}) + expression = (f"(nodes.{card}.metrics[{i}].value if nodes.{card} else 0) " + f"+ (nodes.{other}.metrics[{i}].value if nodes.{other} else 0) + cell(nodes.q, 0, '{column}')") + metrics.append({"id": f"t{i}", "name": f"累计{i}", "value": value, "unit": "元", "decimals": 2, + "format": "thousands", "expression": expression, "inputs": inputs}) + return {"kind": "metric_set", "caliber": f"{card} 口径", "caliber_version": "v1", "metrics": metrics} + + +def _within(seconds: float, work): + """在另一个线程里做 work,交回 (结果, 用时)。超过十秒还没做完就判失败:以前的递归卡住的是整个线程,等它没有意义。""" + box: dict = {} + + def run() -> None: + started = time.perf_counter() + try: + box["value"] = work() + except BaseException as e: # noqa: BLE001 - 交回主线程再抛 + box["error"] = e + box["elapsed"] = time.perf_counter() - started + + thread = threading.Thread(target=run, daemon=True) + thread.start() + thread.join(10) + assert not thread.is_alive(), "十秒还没认完" + if "error" in box: + raise box["error"] + assert box["elapsed"] < seconds, f"用了 {box['elapsed']:.2f} 秒" + return box["value"] + + +@pytest.mark.parametrize("masked", [False, True]) +def test_the_writer_catalog_judges_a_long_loop_of_copying_cards_within_a_second(monkeypatch, masked): + """写作目录(withheld_metrics,运行状态里每张卡只有最后一次的产出):三张卡、各 8 个指标,循环十轮后互相转抄、护栏选中 + 整张卡,一秒以内认完,每个指标只认一次(同一张依赖图上认,转抄它的指标不再重认)。没碰号码列的全部照给;acc 的第一个 + 指标取号码列时,第一个指标所在的环(三张卡的 t0)都不给,别的指标照给(整张卡转抄、值是数的不因此不给)。以前四个指标 + 就要四十秒。""" + import app.api.evidence as evidence_api + from app.api.evidence import withheld_metrics + + built: list[str] = [] + original = evidence_api._InputMasks.__init__ + + def counting(self, sealed, masks, card_node, artifact, **kwargs): + built.append(card_node) + original(self, sealed, masks, card_node, artifact, **kwargs) + + monkeypatch.setattr(evidence_api._InputMasks, "__init__", counting) + + snap, _ = _snapshot() + seen: dict = {} + for _ in range(10): + for card in LOOP_CARDS: + out = _card(card, 8, snap, seen, masked) + seen[card] = {**out, "artifact": _put(out), "text": "清单"} + catalog = {f"m:{card}.t{i}": {"kind": "metric", "node_id": card, "artifact": seen[card]["artifact"], + "locator": {"metric": f"t{i}"}, "value": seen[card]["metrics"][i]["value"]} + for card in LOOP_CARDS for i in range(8)} + got = _within(1.0, lambda: asyncio.run(withheld_metrics(catalog, nodes=seen, ledger=[]))) + assert sorted(got) == (sorted(f"m:{card}.t0" for card in LOOP_CARDS) if masked else []), got + assert len(built) == 3 * 8, len(built) + + +@pytest.mark.parametrize("masked", [False, True]) +def test_the_bundle_judges_a_long_loop_of_copying_cards_within_a_second(masked): + """可验证报告(按每次执行往回认:第 k 轮的卡读到的是各卡在它之前最后一次的产出):同样的三张卡、8 个指标、十轮,另有 + 一份报告文档,证据目录里写着最后一轮三张卡的全部指标(升级前的老文档那样写着原值)。导出一秒以内。没碰号码列的:文档 + 照常附带,每一轮节点完成事件都带规范串。acc 的第一个指标取号码列时:文档整份只给哈希(证据目录里写着取自、转抄自号码列 + 的 t0),卡上的 t0 取自或转抄自号码列的那几次执行都只给哈希——只有第一轮的 acc2 照常带规范串:它转抄的 acc3 那时还没跑, + 没连上号码。以前追够八层就按遮罩算:没碰号码列的十轮循环,文档也整份只给哈希。""" + from app.bundle.build import build_bundle + from app.core.artifact_store import canonical_json, content_hash + + store: dict[str, bytes] = {} + + def keep(obj) -> str: + raw = canonical_json(obj).encode() + store[content_hash(raw)] = raw + return content_hash(raw) + + snap = keep(_snapshot()[1]) + seen: dict = {} + events: list = [] + for _ in range(10): + for card in LOOP_CARDS: + out = _card(card, 8, snap, seen, masked) + art = keep(out) + seen[card] = {**out, "artifact": art, "text": "清单"} + events.append((len(events) + 1, "node.started", card, {})) + events.append((len(events) + 1, "node.finished", card, { + "artifact": keep(seen[card]), "preview": {"artifact": art}, + "evidence": [{"kind": "metric_set", "node_id": card, "artifact": art}]})) + catalog = {f"m:{card}.t{i}": {"alias": f"m:{card}.t{i}", "kind": "metric", "node_id": card, + "artifact": seen[card]["artifact"], "locator": {"metric": f"t{i}"}, + "value": seen[card]["metrics"][i]["value"]} for card in LOOP_CARDS for i in range(8)} + doc = keep({"schema": "agentlab.report/1", "run_id": "r1", "node_id": "write", "markdown": "", "blocks": [], + "catalog": catalog}) + events.append((len(events) + 1, "report.checked", "write", {"doc_artifact": doc})) + graph = {"nodes": [{"id": card, "type": "metrics", "data": {"label": card}} for card in LOOP_CARDS] + + [{"id": "write", "type": "report", "data": {"label": "write"}}]} + bundle = _within(1.0, lambda: build_bundle( + {"id": "r1", "status": "succeeded"}, events, manifest_seq=len(events), manifest_hash="0" * 64, + read=store.get, exported_at="2026-10-12T00:00:00+00:00", graph=graph, source_masks={})) + [art] = [a for a in bundle["artifacts"] if a["kind"] == "report_doc"] + embedded = "b64" in art and "masked" not in art + assert embedded if not masked else (art.get("masked") is True and "b64" not in art), art + shown = [("canonical" in e) for e in bundle["events"] if e["type"] == "node.finished"] + assert len(shown) == 30, shown + assert shown == ([True] * 30 if not masked else [False, True, False, *[False] * 27]), shown + + +# -------------------------------------------------------------------------- +# * 展开派生表、WITH 里没起别名的号码表达式:算确定取自遮罩列 +# -------------------------------------------------------------------------- + +#: 号码设了遮罩的会员表(号码是编的),外加一张订单表 +_MEMBERS_DDL = ("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, phone TEXT, amount REAL);" + "CREATE TABLE orders (oid INTEGER, cid INTEGER, total REAL);") + + +def _snap_of(sql: str) -> dict: + db = sqlite3.connect(":memory:") + db.executescript(_MEMBERS_DDL) + db.executemany("INSERT INTO customers VALUES (?, ?, ?, ?)", + [(i + 1, f"客户{i}", phone, 100.0 * (i + 1)) for i, phone in enumerate(PHONES)]) + db.executemany("INSERT INTO orders VALUES (?, ?, ?)", [(10, 1, 99.5), (11, 2, 15.0)]) + cur = db.execute(sql) + return {"sql": sql, "columns": [d[0] for d in cur.description], "rows": [list(r) for r in cur.fetchall()]} + + +@pytest.mark.parametrize("sql, column", [ + ("WITH a AS (SELECT id, LOWER(name), UPPER(phone) FROM customers) SELECT * FROM a ORDER BY id", "UPPER(phone)"), + ("SELECT * FROM (SELECT id, phone || '' FROM customers) ORDER BY id", "phone || ''"), + ("SELECT * FROM (SELECT id, SUBSTR(phone, 1, 11) FROM customers)", "SUBSTR(phone, 1, 11)"), + ("SELECT t.* FROM (SELECT UPPER(phone) FROM customers) t", "UPPER(phone)"), + ("SELECT a.id, b.* FROM customers a JOIN (SELECT id AS bid, UPPER(phone) FROM customers) b ON a.id = b.bid", + "UPPER(phone)"), + ("WITH t AS (SELECT id, phone FROM customers), u AS (SELECT id, UPPER(phone) FROM t) SELECT * FROM u", + "UPPER(phone)"), + ("SELECT * FROM (SELECT id, UPPER(phone) FROM customers UNION ALL SELECT oid, total FROM orders)", "UPPER(phone)"), + ("SELECT * FROM (SELECT id, name || phone, LOWER(name) FROM customers)", "name || phone"), +]) +def test_an_unaliased_masked_expression_expanded_by_a_star_is_surely_masked(sql, column): + """SQLite、MySQL 拿没起别名的表达式的原文当列名,* 展开派生表、WITH、UNION 的一支时原样带出来。认遮罩列的那一套认不出它 + 落在哪个位置、整份算,以前确定取自遮罩列的那几列里没有它:写作目录、单元格引用的自查照给了号码。现在按列名再认一遍: + 算确定取自遮罩列(_sure),别的列照旧不算。""" + from app.api.evidence import _sure + + snap = _snap_of(sql) + sure, _ = _sure(snap, {"phone"}, "sqlite") + assert column.lower() in sure, sure + assert not sure - {column.lower(), "phone"}, sure + + +@pytest.mark.parametrize("sql", [ + "SELECT * FROM (SELECT id, COUNT(phone) FROM customers GROUP BY id)", + "SELECT * FROM (SELECT id, COUNT(*) FILTER (WHERE phone IS NULL) FROM customers GROUP BY id)", + "SELECT * FROM (SELECT id, CASE WHEN phone IS NULL THEN '未留' ELSE '已留' END FROM customers)", + "SELECT * FROM (SELECT id, LOWER(name), amount * 2 FROM customers)", +]) +def test_columns_named_like_counts_or_labels_of_the_masked_column_stay_ordinary(sql): + """按列名认的只算值取自遮罩列的:只数个数的(COUNT(phone)、FILTER)、只装着分档标签的(CASE 写死的字)、没碰号码列的 + 照旧不算,指标照给、照常引用。""" + from app.api.evidence import _sure + + sure, _ = _sure(_snap_of(sql), {"phone"}, "sqlite") + assert not sure - {"phone"}, sure + + +async def test_a_metric_from_an_unaliased_masked_expression_is_withheld_everywhere(client, monkeypatch, vip): + """真跑:WITH 里没起别名的 UPPER(phone)、外层 * 展开,指标 cell(nodes.q, 0, 'UPPER(phone)')。写作目录写「已遮罩」、文档的 + 证据目录写「已遮罩」,可验证报告里没有号码。以前三处都写着号码。""" + seen = script(monkeypatch, "首位客户已登记。") + sql = "WITH a AS (SELECT id, LOWER(name), UPPER(phone) FROM customers) SELECT * FROM a ORDER BY id" + row = await finish(chain( + node("start", "input"), + node("q", "tool", tool="db_query__vip", args={"sql": sql}), + node("card", "metrics", caliber="会员口径", caliber_version="v1", metrics=[ + {"id": "p", "name": "首位客户联系方式", "format": "plain", "expression": "cell(nodes.q, 0, 'UPPER(phone)')"}]), + node("write", "report", instructions="写一句", on_violation="fail"), + node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}]), + )) + assert row.status == "succeeded", row.error + lines = metric_lines(seen[0]) + assert "首位客户联系方式 = 已遮罩(取自设置了遮罩的列,不要引用)" in lines and leaks(lines) == [], lines + doc = await doc_of(row.id) + assert doc["catalog"]["m:p"]["masked"] is True and leaks(doc) == [] + assert await bundle_leaks(client, row.id) == {"html": [], "artifacts": {}, "events": {}} + + +# -------------------------------------------------------------------------- +# 推演:转抄的输入按同一张依赖图认 +# -------------------------------------------------------------------------- + + +async def test_the_whatif_judges_copied_inputs_on_the_same_graph(client, monkeypatch, vip): + """卡 2 转抄卡 1 的两个指标:号码除以 1000 的余数(值是三位数,按值认不出,输入取自遮罩格)、客户编号。推演卡 2:转抄余数 + 的输入不给值、不能改,指标的原值写「已遮罩」,原因写「转抄的口径卡指标取自设置了遮罩的列」;转抄编号的照常推演。以前 + 推演只按值认,余数照给、还能改,改一下就看得出原值。证据面板点开卡 2 的指标同样写「已遮罩」。""" + script(monkeypatch, "卡二的对照 [[m:ten]],余数 [[m:tail2]]。") + row = await finish(chain( + node("start", "input"), + node("q_cust", "tool", tool="db_query__vip", args={"sql": "SELECT id, name, phone FROM customers ORDER BY id"}), + node("card", "metrics", caliber="会员口径", caliber_version="v1", metrics=[ + {"id": "tail", "name": "号码尾数", "format": "plain", "expression": "cell(nodes.q_cust, 0, 'phone') % 1000"}, + {"id": "first", "name": "首位编号", "format": "plain", "expression": "cell(nodes.q_cust, 0, 'id')"}]), + node("card2", "metrics", caliber="转抄口径", caliber_version="v1", metrics=[ + {"id": "tail2", "name": "转抄尾数", "format": "plain", "expression": "nodes.card.metrics[0].value + 1"}, + {"id": "ten", "name": "编号十倍", "format": "plain", "expression": "nodes.card.metrics[1].value * 10"}]), + node("write", "report", instructions="写一句", on_violation="flag"), + node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}]), + )) + assert row.status == "succeeded", row.error + res = await client.post(f"/api/runs/{row.id}/evidence/whatif", json={"card": "card2"}) + assert res.status_code == 200, res.text + body = res.json() + metrics = {m["id"]: m for m in body["metrics"]} + tail = metrics["tail2"] + assert (tail["status"], tail["old"], tail["withheld"]) == ("masked", "已遮罩", True), tail + [copied] = tail["inputs"] + assert copied["masked"] is True and copied["old"] is None and copied["editable"] is False, copied + assert {"metric": "tail2", "input": "nodes.card.metrics[0].value", + "reason": "转抄的口径卡指标取自设置了遮罩的列,不能用于推演"} in body["unsupported"], body["unsupported"] + ten = metrics["ten"] + assert ten["old"] == 10 and ten["inputs"][0]["editable"] is True and ten["inputs"][0]["old"] == 1, ten + res = await client.post(f"/api/runs/{row.id}/evidence/whatif", + json={"card": "card2", "overrides": {"inputs": {"nodes.card.metrics[1].value": 5}}}) + assert res.status_code == 200, res.text + assert {m["id"]: m for m in res.json()["metrics"]}["ten"]["new"] == 50 + res = await client.post(f"/api/runs/{row.id}/evidence/whatif", + json={"card": "card2", "overrides": {"inputs": {"nodes.card.metrics[0].value": 5}}}) + assert res.status_code == 422 and "遮罩" in res.json()["detail"], res.text + + doc = await doc_of(row.id) + seg = next(s for s in iter_segments(doc) if s.get("ref") == "m:tail2") + chain_ = (await client.get(f"/api/runs/{row.id}/evidence/segments/{seg['id']}")).json()["chain"] + assert chain_[0]["value"] == "已遮罩", chain_[0] + + +async def test_a_card_recording_a_copied_list_with_a_number_is_not_embedded(client, monkeypatch, vip): + """卡 2 数卡 1 清单文字的字数(len(nodes.card.text)):指标的值是个数,照给、照常引用;可卡 2 上记着转抄来的清单文字 + 本身,里面写着号码,卡 2 只给 id,文件里没有号码。以前指标的值照给,卡 2 就原样附带了。""" + script(monkeypatch, "清单字数 [[m:n_len]]。") + row = await finish(chain( + node("start", "input"), + node("q_cust", "tool", tool="db_query__vip", args={"sql": "SELECT id, name, phone FROM customers ORDER BY id"}), + node("card", "metrics", caliber="会员口径", caliber_version="v1", metrics=[ + {"id": "phone_raw", "name": "首位客户手机号", "format": "plain", + "expression": "cell(nodes.q_cust, 0, 'phone')"}]), + node("card2", "metrics", caliber="转抄口径", caliber_version="v1", metrics=[ + {"id": "n_len", "name": "清单字数", "unit": "字", "format": "thousands", "expression": "len(nodes.card.text)"}]), + node("write", "report", instructions="写一句", on_violation="fail"), + node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}]), + )) + assert row.status == "succeeded", row.error + doc = await doc_of(row.id) + assert "masked" not in doc["catalog"]["m:n_len"] and doc["catalog"]["m:n_len"]["value"] > 0 + _, bundle = await export(client, row.id) + cards = {a["node_id"]: a for a in cards_of(bundle)} + assert cards["card2"].get("masked") is True and "b64" not in cards["card2"], cards + assert await bundle_leaks(client, row.id) == {"html": [], "artifacts": {}, "events": {}} + + +async def test_a_real_loop_of_eight_metrics_is_judged_within_a_second(client, monkeypatch, plain): + """真跑:两张卡各 8 个指标,循环十轮,每个指标累加自己上一轮的同一个指标、另一张卡的同一个指标(护栏选中整张卡)。正式 + 运行成功、引用的数照常写出;写作目录的判断(withheld_metrics)、可验证报告的构建(build_bundle)各在一秒以内。以前每个 + 指标的护栏选中整张卡的全部指标,递归次数约为九的八次方,事件循环被占住、整个服务停住。""" + import app.api.bundle as bundle_api + import app.api.evidence as evidence_api + + spent: dict[str, float] = {} + writer, builder = evidence_api.withheld_metrics, bundle_api.build_bundle + + async def timed_writer(*args, **kwargs): + started = time.perf_counter() + try: + return await writer(*args, **kwargs) + finally: + spent["writer"] = max(spent.get("writer", 0.0), time.perf_counter() - started) + + def timed_builder(*args, **kwargs): + started = time.perf_counter() + try: + return builder(*args, **kwargs) + finally: + spent["bundle"] = time.perf_counter() - started + + monkeypatch.setattr(evidence_api, "withheld_metrics", timed_writer) + monkeypatch.setattr(bundle_api, "build_bundle", timed_builder) + + def metrics(card: str, other: str) -> list[dict]: + return [total(f"(nodes.{card}.metrics[{i}].value if nodes.{card} else 0) " + f"+ (nodes.{other}.metrics[{i}].value if nodes.{other} else 0) + cell(nodes.q, 0, 'amount')", + f"{card}_t{i}", f"{card} 累计{i}") for i in range(8)] + + seen = script(monkeypatch, "累计 [[m:acc_t0]],对照 [[m:peer_t7]]。") + row = await finish(loop_graph(10, [{"id": "acc", "metrics": metrics("acc", "peer")}, + {"id": "peer", "metrics": metrics("peer", "acc")}])) + assert row.status == "succeeded", row.error + assert "已遮罩" not in metric_lines(seen[0]), metric_lines(seen[0]) + _, bundle = await export(client, row.id) + [art] = [a for a in bundle["artifacts"] if a["kind"] == "report_doc"] + assert "b64" in art and "masked" not in art, art + assert spent["writer"] < 1.0 and spent["bundle"] < 1.0, spent diff --git a/scripts/check-bundle.mjs b/scripts/check-bundle.mjs index 16ee7d17..fb444be6 100644 --- a/scripts/check-bundle.mjs +++ b/scripts/check-bundle.mjs @@ -27,7 +27,9 @@ // 旁边的(链接地址藏起数字、整段改成分隔线、文字片段冒用数字的 id;数字前面补一位数、加负号(半角、数学减号、负号 // 前面是汉字的)、后面接「0 万」,隔着显示时不占位置的控制字符补一位数,这几种变红并点名「正文里紧挨着这个数的字改变了 // 读者看到的数」;后面接量级单位(万、亿元、%、k,订单号后面接万)、画上删除线另写一个数、长得像数字的字母(l、O)、 -// 长得像负号的连字符、叠在空格上的横线、从右往左书写的字母把一位数挪到数字前面判无法核对、写明原因,总览不再是绿的), +// 长得像负号的连字符、叠在空格上的横线、从右往左书写的字母把一位数挪到数字前面判无法核对、写明原因,总览不再是绿的; +// 紧挨着数字头尾的字改成白名单:别的文字系统里长得像 I、O、M、K 的字母、数学字母、隔着细空格的一位数、隔着换行和 em +// 空格的「万」、「(万元)」判无法核对,负号前面是别的文字系统的字母、后面接 e 和数字判不通过), // 读的人看到的数要就是核对过的那个;列表缩进写成十万级或负数、 // 表格的列号写成十万,核对通过的数字照样留在屏幕上(宽屏、窄屏都不横向溢出);在上面篡改 ④ // 引用链(换文档、拿掉报告、多一份没人引用的工件)、⑥ 运行信息(运行结果、类别、出具档位)、拿掉被引用的口径卡、把 @@ -1203,6 +1205,34 @@ await section('样本页:篡改', async () => { ...naBecause('这一句里有从右往左书写的文字')(s, forged), ] }, { seal: true }) + // 紧挨着数字头尾的那一个字改成白名单(复核第 2 轮的伪造写法:换一个易混字、换一种空白就能绕开按字认的规则,以前全是 + // 绿色的「全部通过」、提示也不显示):长得像数字的字母、名单以外的字、隔着各种空白和左括号的量级单位判无法核对,总览中性 + // 并写明去系统里核对;负号前面是别的文字系统里长得像标点的字母照旧算负号、后面接 e 和数字算科学记数法,判不通过 + for (const [label, at, change, target, why] of [ + ['数字前面加长得像 I 的西里尔字母', 's3', (t) => `${t}${cp(0x406)}`, 's4', '紧挨着长得像数字的字母或符号'], + ['数字前面加汉字「丨」', 's3', (t) => `${t}${cp(0x4e28)}`, 's4', '紧挨着长得像数字的字母或符号'], + ['数字前面加数学粗体 l', 's3', (t) => `${t}${cp(0x1d425)}`, 's4', '紧挨着这个数的字不在常见写法里'], + ['数字前面隔着细空格补一位数', 's3', (t) => `${t}1${cp(0x2009)}`, 's4', '紧挨着这个数的字不在常见写法里'], + ['数字后面接长得像 M 的西里尔字母', 's34', (t) => `${cp(0x41c)}${t}`, 's33', '紧挨着这个数的字不在常见写法里'], + ['数字后面接开尔文符号', 's34', (t) => `${cp(0x212a)}${t}`, 's33', '紧挨着这个数的字不在常见写法里'], + ['数字后面隔着换行接「万」', 's34', (t) => `\n万${t}`, 's33', '后面接着量级单位'], + ['数字后面隔着 em 空格接「万」', 's34', (t) => `${cp(0x2003)}万${t}`, 's33', '后面接着量级单位'], + ['数字后面接「(万元)」', 's34', (t) => `(万元)${t}`, 's33', '后面接着量级单位'], + ]) { + await tamper(samples.formats, label, (data) => rewriteDoc(data, (doc) => { + segById(doc, at).text = change(segById(doc, at).text) + return { what: label, seg: segById(doc, target) } + }), naBecause(why), { seal: true }) + } + for (const [label, at, change, target] of [ + ['负号前面是拉丁文里长得像撇号的字母', 's12', () => `转化率${cp(0xa78c)}-`, 's13'], + ['数字后面接 e 和数字', 's34', (t) => `e3${t}`, 's33'], + ]) { + await tamper(samples.formats, label, (data) => rewriteDoc(data, (doc) => { + segById(doc, at).text = change(segById(doc, at).text) + return { what: label, seg: segById(doc, target) } + }), failsBecause(GLUED), { seal: true }) + } // 列表缩进写得极大或写成负数(服务端按原文的缩进算级数、没有上限):最多缩进 8 级,列表里核对通过的数字都留在屏幕上、 // 照常通过,宽屏、窄屏都不横向溢出 for (const [label, depths] of [['列表缩进写成十万级', [100000, 3]], ['列表缩进写成负数', [-100000, -3]]]) {