diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index 581a6be3..95f7114d 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -51,6 +51,7 @@ from __future__ import annotations import asyncio import contextlib +import copy import csv import functools import io @@ -124,6 +125,7 @@ from app.engine.evidence import ( sql_tables, table_fields, uncited_claims, + withhold_metrics, ) from app.engine.judge import FIELDS_MAX, SETTLED, UNSUPPORTED, VERDICTS, candidates, outdated from app.engine.merge_query import MERGE_SOURCE, trace_cell @@ -2671,6 +2673,48 @@ def _export_cite_key(seg: dict[str, Any]) -> tuple | None: loc.get("metric"), loc.get("row"), loc.get("column"), loc.get("field"), seg.get("issue")) +async def _export_withheld(sealed: _Sealed, doc: dict[str, Any]) -> frozenset[str]: + """报告导出用:正文引用了、证据目录里却还带着原值的取自遮罩列的指标(目录里的别名)。升级前的报告撰写节点写的老文档 + (或者写完之后数据源才设的遮罩)才有;现在写的,这类指标在目录里已经是「已遮罩」。和可验证报告认老文档同一个判断 + (certain_masks + bundle/build.py 的 withheld_metric_aliases:只按确定取自遮罩列的认,值碰巧等于遮罩值的普通指标、SQL + 认不准时的普通金额照给)。导出的文件不带证据目录,只看正文引用了的那几个;一个都没有的不去认遮罩。""" + from app.bundle.build import withheld_metric_aliases + from app.core.artifact_store import read_bytes + + catalog = doc.get("catalog") if isinstance(doc.get("catalog"), dict) else {} + cited = {str(seg["cite"]["alias"]) for _, unit in iter_units(doc) for seg in unit.get("segments") or [] + if isinstance(seg, dict) and isinstance(seg.get("cite"), dict) and seg["cite"].get("kind") == "metric" + and seg["cite"].get("alias")} + entries = {alias: catalog[alias] for alias in sorted(cited) if isinstance(catalog.get(alias), dict) + and catalog[alias].get("kind") == "metric" and not catalog[alias].get("masked") + and catalog[alias].get("value") is not None} + if not entries: + return frozenset() + masks = await _mask_fingerprint() + sure, secrets = await certain_masks(sealed, _Masks(sealed)) + # 读工件、解析口径卡是同步的活,不该占着事件循环(同可验证报告) + return await asyncio.to_thread(lambda: withheld_metric_aliases( + {"catalog": entries}, read=lambda artifact: read_bytes(artifact, verify=False), events=sealed.events, + source_masks=masks, sure_columns=sure, sure_secrets=secrets)) + + +def _withhold_in_doc(doc: dict[str, Any], aliases: frozenset[str]) -> dict[str, Any]: + """老文档里值不该给的那几个指标照现在的报告撰写节点的写法改写,交回副本(原文档不动):目录条目的值、渲染写「已遮罩」、 + 记 masked(engine/evidence.withhold_metrics),引用它们的片段写「已遮罩」。出处链(_metric_chain)据此按目录里的那个字 + 比渲染、值也写「已遮罩」,正文和点开的面板里都不出现原值。""" + out = copy.deepcopy(doc) + out["catalog"] = withhold_metrics(out.get("catalog") or {}, sorted(aliases), MASKED) + for _, unit in iter_units(out): + for seg in unit.get("segments") or []: + cite = seg.get("cite") if isinstance(seg, dict) else None + if isinstance(cite, dict) and cite.get("kind") == "metric" and cite.get("alias") in aliases: + seg["text"] = MASKED + for key in ("value", "rendered"): + if key in cite: + cite[key] = MASKED + return out + + @router.get("/{run_id}/report.html") async def export_report_html(run_id: str, report: str | None = None) -> Response: """把一次运行的报告导出成一个自包含的 HTML 文件(app/report_export)。 @@ -2700,6 +2744,10 @@ async def export_report_html(run_id: str, report: str | None = None) -> Response if chosen.doc is None: raise CodedHTTPException(404, "无法读取报告文档", DOC_MISSING) doc = chosen.doc + # 升级前写的老文档:证据目录里还带着取自遮罩列的指标的原值,正文引用了的话正文里也是。可验证报告把这样的文档整份只给 + # 哈希;这份文件同样会拿给系统外的人,按同一个判断把引用了它们的片段照现在的写法换成「已遮罩」 + if withheld := await _export_withheld(sealed, doc): + doc = _withhold_in_doc(doc, withheld) nodes: dict[str, dict[str, str]] = {} for item in sealed.graph.get("nodes") or []: if isinstance(item, dict) and item.get("id"): diff --git a/backend/app/bundle/build.py b/backend/app/bundle/build.py index f7345ad9..806ecff7 100644 --- a/backend/app/bundle/build.py +++ b/backend/app/bundle/build.py @@ -409,14 +409,18 @@ class _Store: 目录里的口径卡读不出来或找不到这个指标的,按值认:值就是这次运行查到过的遮罩值才算(sure_seen)。按解析出来的内容认, 不看哈希(同 card_masked):服务端的文件被改过,嵌进页面的照样是这份原字节。""" if doc_id not in self._docs: - doc = self.content(doc_id) - catalog = doc.get("catalog") if isinstance(doc, dict) else None - self._docs[doc_id] = isinstance(catalog, dict) and any( - self._carries_withheld(entry) for entry in catalog.values() - if isinstance(entry, dict) and entry.get("kind") == "metric" and not entry.get("masked") - and entry.get("value") is not None) + self._docs[doc_id] = bool(self.withheld_aliases(self.content(doc_id))) return self._docs[doc_id] + def withheld_aliases(self, doc: Any) -> frozenset[str]: + """这份文档的证据目录里还带着原值、值却不给的那几个指标(目录里的别名),认法见 doc_withheld。""" + catalog = doc.get("catalog") if isinstance(doc, dict) else None + if not isinstance(catalog, dict): + return frozenset() + return frozenset(str(alias) for alias, entry in catalog.items() + if isinstance(entry, dict) and entry.get("kind") == "metric" and not entry.get("masked") + and entry.get("value") is not None and self._carries_withheld(entry)) + def _carries_withheld(self, entry: dict[str, Any]) -> bool: artifact = entry.get("artifact") card = self.content(artifact) if isinstance(artifact, str) and artifact else None @@ -1037,6 +1041,18 @@ def _card_input_queries(card: Any) -> list[tuple[str, str | None]]: return out +def withheld_metric_aliases(doc: Any, *, read: Reader, events: Sequence[tuple[int, str, str | None, Any]] = (), + source_masks: Mapping[str, Any] | None = None, + sure_columns: Mapping[str, Iterable[str]] | None = None, + sure_secrets: Any = None) -> frozenset[str]: + """报告文档的证据目录里还带着原值、值却不该给的指标(目录里的别名):可验证报告据此把老文档整份只给哈希 + (_Store.doc_withheld),报告导出成 HTML(api/evidence.export_report_html)据此把引用它们的片段写成「已遮罩」——两样 + 都要拿给系统外的人,同一个判断。参数同 build_bundle:events 是封存范围内的事件,sure_columns、sure_secrets 由调用方按 + api/evidence.py 的 certain_masks 认好交进来。""" + store = _Store(read, source_masks or {}, events, sure_columns=sure_columns, sure_secrets=sure_secrets) + return store.withheld_aliases(doc) + + class _Artifacts: """嵌进页面的工件清单,按先后:报告文档在前,接着是它引用到的。同一份只列一次。events 是封存范围内的事件 (认口径卡带不带遮罩时找写变量的节点当时的产出)。""" diff --git a/backend/tests/test_report_export_masked_metrics.py b/backend/tests/test_report_export_masked_metrics.py new file mode 100644 index 00000000..57bbe07b --- /dev/null +++ b/backend/tests/test_report_export_masked_metrics.py @@ -0,0 +1,85 @@ +"""报告导出成 HTML 时,取自遮罩格的口径卡指标不随文件出去(报告导出与可验证报告的遮罩改造合并时补上)。 + +报告导出照文档原样渲染正文、按证据面板的拼法嵌出处链。现在的报告撰写节点把这类指标写成「已遮罩」,导出的文件里本来就 +没有原值;升级前写的老文档(或写完之后数据源才设的遮罩)证据目录里还带着原值,正文引用了的话正文里也是——可验证报告把 +这样的文档整份只给哈希,报告导出同样会拿给系统外的人,按同一个判断把引用了它们的片段写成「已遮罩」,正文、出处链里都 +不出现原值;普通指标照常。 + +真跑的部分用 tests/test_report_masked_metrics.py 的虚构会员库(号码是编的),写作者是 MockChatModel 照剧本写。 +""" +from __future__ import annotations + +import json +import re + +from tests.test_report_masked_metrics import ( # noqa: F401 - 夹具按名字引入 + CITES, + CLEAN, + client, + engine_up, + finish, + graph, + leaks, + script, + vip, +) + + +async def _export(client, run_id: str) -> tuple[str, dict]: + res = await client.get(f"/api/runs/{run_id}/report.html") + assert res.status_code == 200, res.text + html = res.text + block = re.search(r'', html, re.S) + assert block, "页面里有数据块" + return html, json.loads(block.group(1)) + + +def _metric_step(data: dict, metric: str) -> dict: + return next(step for chain in data["chains"].values() for step in chain + if step.get("step") == "metric" and step.get("metric") == metric) + + +async def test_a_new_doc_exports_without_the_value(client, monkeypatch, vip): + """现在的报告撰写节点写的文档(写作者坚持引用了 [[m:phone_raw]],标注照常产出):正文、出处链里都是「已遮罩」。""" + script(monkeypatch, CITES) + row = await finish(graph(on_violation="flag")) + assert row.status == "succeeded", row.error + html, data = await _export(client, row.id) + assert "手机号 " in html and ">已遮罩" in html + step = _metric_step(data, "phone_raw") + assert (step["value"], step["rendered"], step["render_ok"]) == ("已遮罩", "已遮罩", True), step + assert leaks(html) == [] + + +async def test_an_old_doc_exports_the_masked_metric_as_masked(client, monkeypatch, vip): + """升级前的报告撰写节点写的文档(这里把写作前认遮罩的那一步换成什么都不认):目录、正文里写着号码。导出时引用它的那一处 + 写「已遮罩」,出处链的值、渲染也是,渲染一致;同一句里的普通指标(客户编号)照常;文件里一个号码都没有。""" + import app.api.evidence as evidence_api + + async def nothing(*_args, **_kwargs): + return [] + + real = evidence_api.withheld_metrics + monkeypatch.setattr(evidence_api, "withheld_metrics", nothing) + script(monkeypatch, CITES) + row = await finish(graph(on_violation="flag")) + assert row.status == "succeeded", row.error + monkeypatch.setattr(evidence_api, "withheld_metrics", real) + html, data = await _export(client, row.id) + assert ">已遮罩" in html and "title=\"数字「已遮罩」:点开查看出处\"" in html + step = _metric_step(data, "phone_raw") + assert (step["value"], step["rendered"], step["render_ok"]) == ("已遮罩", "已遮罩", True), step + first = _metric_step(data, "first_id") + assert (first["value"], first["rendered"], first["render_ok"]) == (1, "1", True), first + assert leaks(html) == [] + + +async def test_ordinary_metrics_export_as_before(client, monkeypatch, vip): + """只引用普通指标的报告:导出照旧,客户数、编号照写,出处链照常;文件里没有号码(会员查询的行里号码列写「已遮罩」)。""" + script(monkeypatch, CLEAN) + row = await finish(graph(on_violation="flag")) + assert row.status == "succeeded", row.error + html, data = await _export(client, row.id) + assert ">3位" in html and ">1" in html + assert _metric_step(data, "n_cust")["value"] == 3 + assert leaks(html) == []