From 03209d7db5f6ce2f888f61b6cc13f26690c346e4 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 22:38:15 +0800 Subject: [PATCH 01/12] =?UTF-8?q?K3b=EF=BC=9A=E5=B7=A5=E5=85=B7=E8=BE=B9?= =?UTF-8?q?=E7=95=8C=E7=9A=84=E9=81=AE=E7=BD=A9=E5=88=86=E6=9E=90=E5=87=BA?= =?UTF-8?q?=E4=BA=86=E6=84=8F=E5=A4=96=E6=97=B6=E6=95=B4=E4=BB=BD=E9=81=AE?= =?UTF-8?q?=EF=BC=8C=E5=85=9C=E5=BA=95=E4=B8=8D=E5=86=8D=E7=BB=8F=E9=80=90?= =?UTF-8?q?=E6=A0=BC=E6=8D=A2=E5=80=BC=E9=82=A3=E4=B8=80=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mask_query_result 的兜底原来又调一遍 _masked_view:出意外的正是它时兜底跟着出意外,结果交不出去,模块说明里「分析出意外就整份遮」也没有测试证明。改成 _masked_whole 直接把每一格换成「已遮罩」、类型记成文本,masked_at_query 记全部的列。补测试:认遮罩列、认标签列、逐格换值三处各自出意外时,交给模型的工具结果、落库的快照、通用工件接口都只有「已遮罩」,格子引用照实说是遮罩列;把兜底改回原样放行,这几项全部变红。 --- backend/app/engine/masking.py | 23 +++++- backend/tests/test_masking_tool_boundary.py | 92 +++++++++++++++++++++ 2 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 backend/tests/test_masking_tool_boundary.py diff --git a/backend/app/engine/masking.py b/backend/app/engine/masking.py index 58f3bac0..eee69f4a 100644 --- a/backend/app/engine/masking.py +++ b/backend/app/engine/masking.py @@ -2020,8 +2020,9 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str - 列名不换(模型自己写的,换了它引用的列名就对不上);结果里一列都没取自遮罩列、也没有可按值换的原样交回:数值的 汇总、只数个数的查询不受影响 - 交回的 secrets 只有这份结果里的遮罩值(known 调用方自己有)。分析本身出了意外,整份结果的列都遮:这一步挡在模型 - 前面,分析的缺陷不能把原值放过去(宁可多遮)。""" + 交回的 secrets 只有这份结果里的遮罩值(known 调用方自己有)。分析本身出了意外(认列、认标签、逐格换值哪一步都算), + 整份结果的每一格都遮、全部的列记成查询时遮掉的(_masked_whole):这一步挡在模型前面,分析的缺陷不能把原值放过去 + (宁可多遮)。""" base = {str(c).strip().lower() for c in masked if str(c).strip()} columns = [str(c) for c in payload.get("columns") or []] given = known is not None and bool(known.strong) @@ -2046,11 +2047,25 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str view, _ = _masked_view(payload, hidden, pool, embedded=True, rename=False) except Exception: # noqa: BLE001 - 认不准就整份遮掉,原值不交给模型 logger.warning("核查查询的遮罩分析出错,整份结果按遮罩处理", exc_info=True) - hidden, own = {c.lower() for c in columns}, _Secrets() - view, _ = _masked_view(payload, hidden, own, rename=False) + return QueryMask(_masked_whole(payload), columns, _Secrets()) return QueryMask(view, [c for c in columns if c.lower() in hidden], own) +def _masked_whole(payload: dict[str, Any]) -> dict[str, Any]: + """整份结果的每一格都换成「已遮罩」、类型都记成文本:遮罩分析出了意外时的兜底。不经 _masked_view——出意外的可能 + 就是它,兜底跟着出意外,原值就挡不住了。列名、SQL 是模型自己写的,照原样。""" + word = _masked_word() + rows: list[Any] = [] + for row in payload.get("rows") or []: + if isinstance(row, list): + rows.append([word] * len(row)) + elif isinstance(row, dict): + rows.append(dict.fromkeys(row, word)) + else: + rows.append(word) + return {**payload, "rows": rows, "column_types": {str(c): "text" for c in payload.get("columns") or []}} + + #: 查询报错里引号括起来的一段(Postgres、DuckDB、MySQL 把转换失败的值写在引号里:invalid input syntax for type #: bigint: "139-0000-0001")。引号前后紧挨着字母数字的不算(can't、isn't 里的撇号) _QUOTED_ERROR = re.compile(r"(? Date: Tue, 6 Oct 2026 22:44:48 +0800 Subject: [PATCH 02/12] =?UTF-8?q?K3b=EF=BC=9A=E5=B7=A5=E5=85=B7=E8=BE=B9?= =?UTF-8?q?=E7=95=8C=E7=9A=84=E6=8C=89=E5=80=BC=E9=81=AE=E5=8F=AA=E6=8D=A2?= =?UTF-8?q?=E6=96=87=E5=AD=97=E6=A0=BC=EF=BC=8C=E6=8D=A2=E8=BF=87=E7=9A=84?= =?UTF-8?q?=E6=A0=BC=E5=AD=90=E8=AE=B0=E8=BF=9B=E5=BF=AB=E7=85=A7=EF=BC=9B?= =?UTF-8?q?=E5=8D=A0=E4=BD=8D=E7=9A=84=E7=9F=AD=E8=AF=8D=E4=B8=8D=E5=BD=93?= =?UTF-8?q?=E9=81=AE=E7=BD=A9=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 以前工具边界按值遮扩到没设遮罩的数值列:访问次数正好等于某一笔被遮的金额,交给模型的、落库的都成了「已遮罩」,column_types 仍是 number、快照不留记录,盲算引用到时说成「不是数值」,存下的证据被改写。现在 _masked_view 在工具边界不按值换数值格(数、列类型记成 number 的格子),只换文字格(整格就是遮罩值的、文字里夹着的那一截),换过的格子随快照存下(masked_cells:[行, 列号],不交给模型),核查 agent 的格子引用指到它们时照实说是遮罩。回答、质询工件里的按值遮(第二道防线)照旧。遮罩列里占位的写法(未知、暂无、N/A……)不收进遮罩值,没设遮罩的区域、备注里的同一个词不再被换掉。补测试,分别去掉数值格的豁免、masked_cells 的记录、引用时的判断、占位词的过滤,对应的测试都变红。 --- backend/app/engine/inquiry.py | 11 ++- backend/app/engine/masking.py | 65 +++++++++++---- backend/app/engine/verify_agent.py | 26 +++++- backend/app/tools/datasource.py | 13 ++- backend/tests/test_masking_tool_boundary.py | 92 +++++++++++++++++++++ 5 files changed, 179 insertions(+), 28 deletions(-) diff --git a/backend/app/engine/inquiry.py b/backend/app/engine/inquiry.py index 696ad70d..3d974140 100644 --- a/backend/app/engine/inquiry.py +++ b/backend/app/engine/inquiry.py @@ -46,8 +46,10 @@ 已知局限(遮罩只减少暴露,不是安全边界,同证据面板): - 按值认有门槛,两头都有误差:短的数(年龄、等级这类不到 SECRET_DIGITS 位的)在回答正文、格子里不按值遮,agent 在 正文里写出某人的年龄遮不住,只靠 SQL 里的等值比较和按列遮;同一个短的数在 SQL 里被换掉了,别的列上撞上它的阈值、 - LIMIT 的行数照原样(age = 30 AND visits >= 30)。够长的数在格子和正文里按值遮,碰巧和某个遮罩值相同的数(访问次数 - 正好等于某一笔金额)也遮掉。这是产品取舍,不按上下文去猜 + LIMIT 的行数照原样(age = 30 AND visits >= 30)。够长的数在回答正文和回答引用的格子里按值遮,碰巧和某个遮罩值相同的 + 数(访问次数正好等于某一笔金额)也遮掉,这是产品取舍,不按上下文去猜;工具边界不按值换数值格(存下的是证据),交给 + agent 的、存下的查询快照里这样的数照原样,只在回答和质询工件里遮。占位的写法(未知、暂无……,masking.PLACEHOLDERS) + 不当遮罩值 - 遮罩数值列的 SUM / AVG / MIN / MAX 算派生自遮罩列,整列遮(COUNT 不算,见 _value_columns),CASE 里有一支取的是 遮罩列的也算:MIN / MAX 就是某一行的原值,只有一行的分组 SUM / AVG 也是,按列认不出哪些汇总值安全,宁可多遮。 核查 agent 的查询在工具边界照这个办法遮,所以这样的指标双盲复算不了(原 SQL 里认得出的,派盲算之前就记为无法 @@ -106,8 +108,9 @@ 从开头的引号、注释起整个去掉)。说明里码值表列出的码值和别的字一样按值换,恰好是遮罩值的码值也换(多遮) - 运行详情的「工件」页签、通用的工件接口按工件原样给:质询的查询快照落库时就遮好了(工具边界先遮),取自遮罩列的值、 报告快照里的遮罩值、agent 前面的查询里遮掉的值都看不到。只有一种照原样:先查了写着号码的备注(那时还不知道这个号码 - 是遮罩值),后来才查出号码——备注快照落库时原样,从通用接口打开照原样,质询工件接口照事件里记下的位置换。报告自己 - 的查询快照(写作时的查询)仍原样存着,从那里打开不经这里的遮罩 + 是遮罩值),后来才查出号码——备注快照落库时原样,从通用接口打开照原样,质询工件接口照事件里记下的位置换。没设遮罩的 + 数值格碰巧等于遮罩值的(见上),落库时照原样,从通用接口打开也照原样。报告自己的查询快照(写作时的查询)仍原样存着, + 从那里打开不经这里的遮罩 - 遮罩值最多收 SECRETS_MAX 种写法,再多的按值认不出(按列遮照旧)。agent 自己查到的值先收,报告快照里的后收:核查 agent 把两份分开记(masking.SeenValues),查询时按值遮、交回调用方、回答时合起来,都是 agent 查到的在前 - 换了写法的号码只认数字之间夹着一到四个分隔符(空格、制表符、全角空格、不换行空格、窄空格这类空白,连字符、减号、 diff --git a/backend/app/engine/masking.py b/backend/app/engine/masking.py index eee69f4a..a6fce84d 100644 --- a/backend/app/engine/masking.py +++ b/backend/app/engine/masking.py @@ -6,9 +6,10 @@ 质询(engine/inquiry.py)、双盲复算(nodes/dual_check.py,经 inquiry 的同名导出)共用: - 工具边界先遮(mask_query_result):核查 agent 的查询结果交给模型、落进快照之前,取自遮罩列的格子换成「已遮罩」, - 别的格子按值换:这份结果里的遮罩值,加上这个 agent 前面的查询遮掉的、调用方交进来的(报告快照里的)。模型看不到 + 别的文字格按值换:这份结果里的遮罩值,加上这个 agent 前面的查询遮掉的、调用方交进来的(报告快照里的)。模型看不到 遮罩值,就不会把它写进回答、理由、SQL——事后按值认每一轮复核都找到新的漏法,根治靠这一步;审批质询回答、反方 - 审稿理由里的按值遮降为第二道防线。查询失败时报错原话里的值也换(mask_error_text) + 审稿理由里的按值遮降为第二道防线。宁可多遮,但不改写没设遮罩的数值证据:数值格不按值换,按值换过的文字格记进 + 快照(masked_cells)。查询失败时报错原话里的值也换(mask_error_text)。分析本身出了意外,整份遮(_masked_whole) - 哪些列(_derived_hidden):从遮罩列本身起,按 SQL 一层层认出值取自遮罩列的结果列(别名、函数、子查询、WITH、UNION 的 另一支、和 * 写在同一支里没起别名的表达式、FROM 里拆开遮罩列的表值函数、函数参数里的 JSON 箭头 col->'path');* 把 @@ -21,7 +22,8 @@ TABLE x)的,整份快照的列都算。双盲复算按同一套认法在派盲算之前挡下取自遮罩列的指标(derived_columns) - 只装着分档标签的列(_label_columns):标签是 agent 写在 SQL 里的字,不当遮罩值;* 展开数据源里的表时按冻结的表结构 认它带不带同名的列(_schema_tables) -- 哪些值(_Secrets):遮罩列里出现过的值,按值认有门槛(SECRET_MIN、SECRET_DIGITS),换的时候先按词、按开头几个字筛出 +- 哪些值(_Secrets):遮罩列里出现过的值,按值认有门槛(SECRET_MIN、SECRET_DIGITS),占位的写法(PLACEHOLDERS:未知、 + 暂无……)不收;换的时候先按词、按开头几个字筛出 正文里真有的那几个值,不现编上万个分支的正则;正文、文字格、SQL 的字符串和注释里换了写法的长号码(199-1474-2785、 +86 …、全角数字)按纯数字写法在集合里查(loose_spans);已经摆在读者眼前的值免遮,之后并进来的照样免(protected) - 快照遮好的样子(_masked_view),回答时才认得出、事后要照位置重放的地方(view_edits、replay_edits) @@ -56,6 +58,12 @@ SECRET_MIN = 2 SECRET_DIGITS = 5 #: 遮罩列的值最多收这么多种写法(一份快照最多上千行):再多就按已经收到的换 SECRETS_MAX = 20_000 +#: 占位的写法:遮罩列里没填的格子常这样写,不是哪个人的信息。当遮罩值收了的话,没设遮罩的列、备注里同一个词都会被换掉 +#: (区域「未知」、「原因未知」变成「原因已遮罩」),一个都不收(不分大小写) +PLACEHOLDERS = frozenset({ + "未知", "暂无", "不详", "不明", "无", "空", "未填", "未填写", "未提供", "待定", "待补充", "暂缺", "缺失", "没有", + "其他", "其它", "保密", "匿名", "未登记", "未记录", "无数据", "null", "none", "nil", "nan", "n/a", "na", "-", "--", +}) #: 按词边界认值的字符:值的头尾是这样的字时,前后不能紧挨着同类的字(13900000001 不换掉 139000000012 的一截) @@ -1415,6 +1423,11 @@ def _forms(value: Any) -> set[str]: return forms +def _placeholder(value: Any) -> bool: + """一格是占位的写法(PLACEHOLDERS:未知、暂无、N/A……),不当遮罩值收。""" + return isinstance(value, str) and _plain_text(value).lower() in PLACEHOLDERS + + def _strong(form: str) -> bool: """按值认得出的写法(见 SECRET_MIN、SECRET_DIGITS)。""" if _NUMERIC.fullmatch(form): @@ -1522,7 +1535,8 @@ class _Secrets: def add(self, snapshot: Any, hidden: set[str], *, labels: dict[str, set[str]] | None = None, base: Iterable[str] = ()) -> None: """收一份快照里要遮的列(hidden,小写)的值。labels 是这份快照里只装着分档标签的列(_label_columns):这几列的 - 格子等于标签的不收;改了名字的遮罩列、别的派生列照收,base(数据源设了遮罩的列,小写)自己选出来的值也照收。""" + 格子等于标签的不收;改了名字的遮罩列、别的派生列照收,base(数据源设了遮罩的列,小写)自己选出来的值也照收。 + 占位的写法(PLACEHOLDERS:未知、暂无……)不收。""" if not _query_shaped(snapshot) or not hidden: return own = set(base) @@ -1541,7 +1555,7 @@ class _Secrets: else: continue for name, value in cells: - if value is None or isinstance(value, bool) or value == word: + if value is None or isinstance(value, bool) or value == word or _placeholder(value): continue forms = _forms(value) if forms & written.get(name, set()): @@ -1777,7 +1791,8 @@ def _scrubbed_source(text: str, secrets: _Secrets) -> str: def _masked_view(snapshot: dict[str, Any], hidden: set[str], secrets: _Secrets, *, embedded: bool = False, - rename: bool = True) -> tuple[dict[str, Any], list[str]]: + rename: bool = True, numbers: bool = True, changed: list[list[int]] | None = None + ) -> tuple[dict[str, Any], list[str]]: """一份查询快照遮好的样子和遮掉的列:要遮的列(hidden,小写)每一格都换成「已遮罩」、类型记成文本(渲染时不当数); 别的列里写着按值认得出的遮罩值的格子也换(另一张表里没设遮罩的同一个手机号、改了名字没认出来的遮罩列)。 列名里写着遮罩值的(agent 把遮罩值写进了别名,「13900000001 的单」),列名也换(同 SQL 里的名字,_scrub_sql)。 @@ -1788,6 +1803,9 @@ def _masked_view(snapshot: dict[str, Any], hidden: set[str], secrets: _Secrets, 里也没设遮罩。回答一路不这样换:渲染出来露了就整份不交(_leaked),不然回答被问的那一格看着像查到了别的东西。 rename 为假时列名不换(工具边界先遮时:列名是模型自己写的,换了它引用的列名就对不上)。列名按文字的办法换(换了写法的 号码也认,同 SQL 里带引号的名字)。 + numbers 为假时数值格(数、列类型记成 number 的格子)不按值换,只换文字格(工具边界先遮时:存下的就是证据,访问次数 + 正好等于某一笔被遮的金额,换掉就改写了没设遮罩的数,双盲复算也取不到它);要遮的列照旧整列换。changed 给了的话,按值 + 换过的格子(整格换的、文字里换掉一截的)按 [行, 列] 记进去,要遮的列里的不记。 工具边界遮过的快照(记着 masked_at_query)里,整格就是「已遮罩」的是查询时按值换掉的:这一列照样记进遮掉的列,和回答时 按值换掉的一样画。""" columns = [str(c) for c in snapshot.get("columns") or []] @@ -1801,8 +1819,10 @@ def _masked_view(snapshot: dict[str, Any], hidden: set[str], secrets: _Secrets, edited = False position = {c: i for i, c in enumerate(columns)} names = [secrets.scrub(c, loose=True) for c in columns] if by_value and rename else columns + kinds = snapshot.get("column_types") if isinstance(snapshot.get("column_types"), dict) else {} + counted = {i for i, c in enumerate(columns) if kinds.get(c) == "number"} if not numbers else set() - def cell(i: int | None, value: Any) -> Any: + def cell(r: int, i: int | None, value: Any) -> Any: nonlocal edited if i in index: return word @@ -1810,15 +1830,21 @@ def _masked_view(snapshot: dict[str, Any], hidden: set[str], secrets: _Secrets, if i is not None: touched.add(i) return value + if not numbers and (i in counted or (isinstance(value, (int, float)) and not isinstance(value, bool))): + return value if by_value and secrets.hit(value): if i is not None: touched.add(i) + if changed is not None: + changed.append([r, i]) return word if embedded and by_value and isinstance(value, (str, dict, list)): # JSON 列(对象、数组)按界面画出来的样子换,换过的照文字给 text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str) if (scrubbed := secrets.scrub(text, loose=True)) != text: edited = True + if changed is not None and i is not None: + changed.append([r, i]) return scrubbed return value @@ -1827,11 +1853,11 @@ def _masked_view(snapshot: dict[str, Any], hidden: set[str], secrets: _Secrets, return names[i] if i is not None else k rows: list[Any] = [] - for row in snapshot["rows"]: + for r, row in enumerate(snapshot["rows"]): if isinstance(row, list): - rows.append([cell(i, v) for i, v in enumerate(row)]) + rows.append([cell(r, i, v) for i, v in enumerate(row)]) elif isinstance(row, dict): - rows.append({key(k): word if str(k).lower() in hidden else cell(position.get(str(k)), v) + rows.append({key(k): word if str(k).lower() in hidden else cell(r, position.get(str(k)), v) for k, v in row.items()}) else: rows.append(row) @@ -1972,11 +1998,13 @@ class QueryMask: payload:遮好的结果,交给模型的、落进快照的都是它。columns:查询时就整列遮掉的结果列(结果里的列名,按先后)。 secrets:这份结果里遮掉的值,只在内存里交给调用方(审批质询按值遮的第二道防线:模型照着别处看到的号码写进回答、 - 查询条件时认得出),不进快照、不交给模型。""" + 查询条件时认得出),不进快照、不交给模型。cells:别的列里按值换过的格子 [行, 列](整格换的、文字里换掉一截的), + 随快照存下(masked_cells),格子引用指到它们照实说是遮罩,不说成「不是数值」。""" payload: dict[str, Any] columns: list[str] secrets: _Secrets + cells: list[list[int]] = field(default_factory=list) @dataclass @@ -2013,10 +2041,12 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str SELECT *(展开的列换了名字的也算)、表值函数、整行引用和函数参数里的 *、列名表、TABLE x;COUNT、只在条件里用到、 分档当分组键的不算);SQL 解析不了、又写着遮罩列(或者 *、TABLE x)的,整份结果的列都遮。这几列的格子换成 「已遮罩」,类型记成文本 - - 别的列按值换:整格就是遮罩值的换掉,文字格里夹着的那一截换掉(换了写法的号码也认,_Secrets.loose_spans)。遮罩值 - 是这份结果里的加上 known:先查号码、再查写着号码的备注、没设遮罩的联系方式表(别的库里的也一样)时,后一份同样 - 遮好。门槛同按值遮(SECRET_DIGITS、SECRET_MIN);只装着分档标签的列里的标签不算遮罩值,这份结果自己写的标签也不 - 按 known 换(前面的查询认不准、当遮罩值收了的同名标签,不能把这一份的档位列换掉) + - 别的列按值换,只换文字格:整格就是遮罩值的换掉,文字格里夹着的那一截换掉(换了写法的号码也认, + _Secrets.loose_spans),换过的格子记下来(QueryMask.cells)。遮罩值是这份结果里的加上 known:先查号码、再查写着 + 号码的备注、没设遮罩的联系方式表(别的库里的也一样)时,后一份同样遮好。门槛同按值遮(SECRET_DIGITS、SECRET_MIN); + 只装着分档标签的列里的标签不算遮罩值,这份结果自己写的标签也不按 known 换(前面的查询认不准、当遮罩值收了的同名 + 标签,不能把这一份的档位列换掉)。数值格(数、列类型是 number 的格子)不按值换:存下的快照就是证据,没设遮罩的 + 数值列(访问次数正好等于某一笔被遮的金额)换掉就改写了证据,双盲复算也取不到它 - 列名不换(模型自己写的,换了它引用的列名就对不上);结果里一列都没取自遮罩列、也没有可按值换的原样交回:数值的 汇总、只数个数的查询不受影响 @@ -2044,11 +2074,12 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str if not pool.strong and not any(c.lower() in hidden for c in columns): return QueryMask(payload, [], own) pool.prepare() - view, _ = _masked_view(payload, hidden, pool, embedded=True, rename=False) + cells: list[list[int]] = [] + view, _ = _masked_view(payload, hidden, pool, embedded=True, rename=False, numbers=False, changed=cells) except Exception: # noqa: BLE001 - 认不准就整份遮掉,原值不交给模型 logger.warning("核查查询的遮罩分析出错,整份结果按遮罩处理", exc_info=True) return QueryMask(_masked_whole(payload), columns, _Secrets()) - return QueryMask(view, [c for c in columns if c.lower() in hidden], own) + return QueryMask(view, [c for c in columns if c.lower() in hidden], own, cells) def _masked_whole(payload: dict[str, Any]) -> dict[str, Any]: diff --git a/backend/app/engine/verify_agent.py b/backend/app/engine/verify_agent.py index fd7b3bf8..229e9f0f 100644 --- a/backend/app/engine/verify_agent.py +++ b/backend/app/engine/verify_agent.py @@ -8,11 +8,12 @@ **先遮再交。** 建查询工具时还打开遮罩开关(tools/datasource 的 mask_results):数据源设了遮罩时,取自遮罩列的格子 (按 SQL 认,起别名、套函数、经子查询或 WITH 转一道、SELECT *(展开的列换了名字的也算)、表值函数拆开的都算)在交给 -模型、落进查询快照之前就换成「已遮罩」,别的格子按值换:这份结果里的遮罩值、前面的查询遮掉的、调用方交进来的 +模型、落进查询快照之前就换成「已遮罩」,别的文字格按值换:这份结果里的遮罩值、前面的查询遮掉的、调用方交进来的 (masked_values:审批质询、反方审稿收的报告快照里的值),文字格里夹着的换那一截(engine/masking.py 的 -mask_query_result);查询失败时报错原话里的值也换。模型看不到遮罩值,就不会把它写进回答、理由、SQL;以前是事后 -按值认,每一轮复核都找到新的漏法。快照里的 masked_at_query 记下查询时就遮了的列,格子引用指到这几列、指到按值换掉 -的格子的都不采信。遮掉的值只在内存里交回调用方(VerifyOutcome.masked_values,审批质询、反方审稿按值遮的第二道防线 +mask_query_result);数值格不按值换(存下的是证据,没设遮罩的访问次数正好等于某一笔被遮的金额也照原样);查询失败时 +报错原话里的值也换。模型看不到遮罩值,就不会把它写进回答、理由、SQL;以前是事后按值认,每一轮复核都找到新的漏法。 +快照里的 masked_at_query 记下查询时就遮了的列,masked_cells 记下按值换过的格子,格子引用指到这几列、这几格的都不采信, +照实说是遮罩。遮掉的值只在内存里交回调用方(VerifyOutcome.masked_values,审批质询、反方审稿按值遮的第二道防线 用),不进快照、不交给模型、不进 as_dict。调用方交进来的和查询时遮掉的分开记(masking.SeenValues),合起来时 agent 查到的在前:合起来到了上限 SECRETS_MAX,挤掉的是报告里的值。 @@ -438,6 +439,13 @@ class _Cells: "row": row, "column": name, "artifact": query["artifact"], "reason": f"{alias} 第 {row} 行「{name}」与设置了遮罩的值相同,查询时已遮罩,不采信"}) return None + if _masked_cell(content, row, name): + # 文字里有一截查询时按值换掉了(快照的 masked_cells 记着):这一格不是数据库里的原样,同样照实说 + self.cells.append({"path": path, "status": "unresolved", "ref": f"{alias}.r{row}.{name}", "alias": alias, + "row": row, "column": name, "artifact": query["artifact"], + "reason": f"{alias} 第 {row} 行「{name}」里有与设置了遮罩的值相同的部分,查询时已遮罩," + "不采信"}) + return None truth = _typed(raw, {"type": sorted(types)}, column_kind(content, name)) cited = {"ref": f"{alias}.r{row}.{name}", "alias": alias, "row": row, "column": name, "artifact": query["artifact"], "eid": cell_eid(query["artifact"], row, name), "value": truth} @@ -539,6 +547,16 @@ def _masked_at_query(content: Any) -> set[str]: return {c.lower() for c in listed if isinstance(c, str)} if isinstance(listed, list) else set() +def _masked_cell(content: Any, row: Any, name: str) -> bool: + """查询快照里第 row 行、name 这一列的格子查询时按值换过(tools/datasource 记下的 masked_cells:[行, 列号])。列名 + 同名的按第一列算,同 locate_cell。""" + listed = content.get("masked_cells") if isinstance(content, dict) else None + columns = [str(c) for c in content.get("columns") or []] if isinstance(content, dict) else [] + if not isinstance(listed, list) or name not in columns: + return False + return [row, columns.index(name)] in listed + + # -------------------------------------------------------------------------- # 花费 # -------------------------------------------------------------------------- diff --git a/backend/app/tools/datasource.py b/backend/app/tools/datasource.py index 1cf26969..e8b79057 100644 --- a/backend/app/tools/datasource.py +++ b/backend/app/tools/datasource.py @@ -470,7 +470,8 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, mask_results:数据源设了遮罩时,取自遮罩列的格子(按 SQL 认,改了名字的也算)在交给模型、落进快照之前就换成 「已遮罩」,见 engine/masking.py 的 mask_query_result;这一次遮掉的值并进 ctx.extra[MASKED_VALUES_KEY](有的话)。 那里已经有的值(这个 agent 前面的查询遮掉的、调用方交进来的)这一次也按值换,数据源没设遮罩也一样:先查号码、 - 再查写着号码的备注或别的库,后一份同样遮好。查询失败时报错原话里的值也换(masking.mask_error_text)。 + 再查写着号码的备注或别的库,后一份同样遮好。按值换只换文字格,数值格照原样(存下的是证据)。查询失败时报错原话里 + 的值也换(masking.mask_error_text)。 """ # 数据源在 options.query_timeout_s 里配了就用它,否则取缺省。数据库按它停下语句, # 引擎按它告诉界面上限是多少(metadata["timeout_s"]) @@ -530,8 +531,10 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, payload = result.to_payload() payload["source"] = source.name - # 查询时就整列遮掉的结果列;没开遮罩开关、数据源没设遮罩又没有可按值换的时为 None(快照和原来一字不差) + # 查询时就整列遮掉的结果列;没开遮罩开关、数据源没设遮罩又没有可按值换的时为 None(快照和原来一字不差)。 + # 别的列里按值换过的格子另记(masked_cells),没有就不记 masked_now: list[str] | None = None + masked_cells: list[list[int]] = [] if mask_results and (mask or known is not None): from app.engine.masking import mask_query_result @@ -539,7 +542,7 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, done = await asyncio.to_thread(mask_query_result, payload, mask, source.kind, known, schema=source.schema_cache) if mask or done.payload is not payload: - payload, masked_now = done.payload, done.columns + payload, masked_now, masked_cells = done.payload, done.columns, done.cells if sink is not None: # 并进 agent 查到的那一份(不和调用方交进来的混在一起,上限收满了也不丢),也放进线程 await asyncio.to_thread(sink.add, done.secrets) @@ -562,6 +565,8 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, # 同源各份快照记的这一项并起来,从表的字段清单里去掉,混进结果列的别名会把同名的真实字段也去掉; # - masked_at_query:开了遮罩开关时,查询时就整列遮掉的结果列(这几列的格子是「已遮罩」,不是数据源里的值, # 改了名字的也在这里)。有这一项的快照,别的格子整格是「已遮罩」的是查询时按值换掉的; + # - masked_cells:别的列里查询时按值换过的文字格 [行, 列号](整格换的、换掉一截的),格子引用指到它们时照实 + # 说是遮罩。数值格不按值换,存下的数和数据库里的一样; # - data_version:上传表格查的是哪个快照,事后据此认得出这个数出自哪一版数据。 # 手工源、没设遮罩的,快照和原来一字不差 extra: dict[str, Any] = {} @@ -569,6 +574,8 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, extra["mask_columns"] = mask if masked_now is not None: extra["masked_at_query"] = masked_now + if masked_cells: + extra["masked_cells"] = masked_cells if version := getattr(source, "snapshot_id", None): extra["data_version"] = version # 工件引用的 meta 记下源和 SQL 里用到的表:数据目录按它数每张表被运行查询过几次 diff --git a/backend/tests/test_masking_tool_boundary.py b/backend/tests/test_masking_tool_boundary.py index b5db6244..1f44345e 100644 --- a/backend/tests/test_masking_tool_boundary.py +++ b/backend/tests/test_masking_tool_boundary.py @@ -90,3 +90,95 @@ async def test_the_model_and_the_snapshot_see_only_masks_when_the_analysis_fails assert got.status_code == 200 and _clean(got.text) and "1200" not in got.text assert out.result["value"] is None assert out.cells[0]["reason"] == "Q1 的「amount」取自设置了遮罩的列,查询时已遮罩,不采信" + + +# -------------------------------------------------------------------------- +# 多遮不改写没设遮罩的数值证据;占位的短词不当遮罩值 +# -------------------------------------------------------------------------- + + +#: 金额设了遮罩,都是五位数;访问次数没设遮罩,第二、三单正好等于别的订单的金额(20000、30000)。联系表的访问次数存成 +#: 文字('20000'),客户名设了遮罩、里面有占位的「未知」,区域和备注里也写着「未知」 +BIG = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, amount INTEGER, visits INTEGER, buyer_name TEXT," + " note TEXT);" + "INSERT INTO sales VALUES (1, '华东', 10000, 25000, '张伟', '原因未知'), (2, '华东', 12000, 20000, '未知', '')," + " (3, '华南', 20000, 30000, '李娜', '客户未知来源'), (4, '未知', 30000, 22000, '王芳', '')," + " (5, '华北', 50000, 9000, '刘洋', '');" + "CREATE TABLE contacts (id INTEGER PRIMARY KEY, visits_text TEXT, remark TEXT);" + "INSERT INTO contacts VALUES (1, '20000', '上月消费 30000 元'), (2, '8', '例行回访');") + + +@pytest.fixture +async def big_shop(tmp_path): + name, source_id = await _source(tmp_path, {"mask_columns": "amount, buyer_name"}, script=BIG) + yield name + await engines.invalidate(source_id) + + +async def test_an_unmasked_numeric_column_is_kept_even_when_it_equals_a_masked_value(monkeypatch, big_shop): + """访问次数没设遮罩,第二、三单正好等于别的订单被遮的金额:交给模型的、落库的都是数据库里的原数,类型照旧是数, + 引用它照常取值(双盲复算取得到 20000)。以前按值整格换成「已遮罩」,快照里不留记录,引用时说成「不是数值」。""" + name = big_shop + sql = "SELECT id, amount, visits FROM sales ORDER BY id" + script = Script(monkeypatch, [q(name, sql, "c1")], {"value": {"alias": "Q1", "row": 1, "column": "visits"}, + "rationale": "x"}) + out, _ = await ask(name) + seen = json.loads(script.tool_results()["c1"].split("\n", 1)[1]) + assert seen["rows"] == [[1, MASKED, 25000], [2, MASKED, 20000], [3, MASKED, 30000], [4, MASKED, 22000], + [5, MASKED, 9000]] + snapshot = artifact_store.load(out.queries[0]["artifact"]) + assert snapshot["rows"] == seen["rows"] and snapshot["column_types"]["visits"] == "number" + assert snapshot["masked_at_query"] == ["amount"] and "masked_cells" not in snapshot + assert out.result["value"] == 20000 and out.cells[0]["status"] == "resolved" + + +async def test_text_cells_masked_by_value_are_recorded_and_cited_as_masked(monkeypatch, big_shop): + """存成文字的数('20000')、备注里夹着的金额(30000)照样按值换:交给模型的、落库的都换掉,换过的格子记进快照 + (masked_cells),引用到它们照实说是遮罩,不说成「不是数值」。""" + name = big_shop + steps = [q(name, "SELECT amount FROM sales ORDER BY id", "c1"), + q(name, "SELECT id, visits_text, remark FROM contacts ORDER BY id", "c2")] + script = Script(monkeypatch, steps, lambda messages: {"value": {"alias": "Q2", "row": 0, "column": "visits_text"}, + "rationale": "x"}) + out, _ = await ask(name) + seen = json.loads(script.tool_results()["c2"].split("\n", 1)[1]) + assert seen["rows"] == [[1, MASKED, f"上月消费 {MASKED} 元"], [2, "8", "例行回访"]] + assert "masked_cells" not in seen, "记录只在快照里,不交给模型" + snapshot = artifact_store.load(out.queries[1]["artifact"]) + assert snapshot["rows"] == seen["rows"] and snapshot["masked_cells"] == [[0, 1], [0, 2]] + assert snapshot["masked_at_query"] == [] + assert out.result["value"] is None + assert out.cells[0]["reason"] == "Q2 第 0 行「visits_text」与设置了遮罩的值相同,查询时已遮罩,不采信" + + +async def test_a_partly_masked_text_cell_is_cited_as_masked(monkeypatch, big_shop): + """引用指到文字里换掉了一截的格子:同样照实说是遮罩。""" + name = big_shop + steps = [q(name, "SELECT amount FROM sales ORDER BY id", "c1"), + q(name, "SELECT id, remark FROM contacts ORDER BY id", "c2")] + Script(monkeypatch, steps, {"value": {"alias": "Q2", "row": 0, "column": "remark"}, "rationale": "x"}) + out, _ = await ask(name) + assert out.result["value"] is None + assert out.cells[0]["reason"] == "Q2 第 0 行「remark」里有与设置了遮罩的值相同的部分,查询时已遮罩,不采信" + + +async def test_placeholder_words_are_not_masked_elsewhere(monkeypatch, big_shop): + """客户名设了遮罩,里面有占位的「未知」:客户名整列照旧遮;没设遮罩的区域里的「未知」、备注里的「原因未知」「客户 + 未知来源」照原样。以前「未知」当遮罩值收了,这几格都被换掉。""" + name = big_shop + sql = "SELECT id, buyer_name, region, note FROM sales ORDER BY id" + script = Script(monkeypatch, [q(name, sql, "c1")], {"value": {"alias": "Q1", "row": 0, "column": "id"}, + "rationale": "x"}) + out, _ = await ask(name) + seen = json.loads(script.tool_results()["c1"].split("\n", 1)[1]) + assert [row[1] for row in seen["rows"]] == [MASKED] * 5 + assert [row[2:] for row in seen["rows"]] == [["华东", "原因未知"], ["华东", ""], ["华南", "客户未知来源"], + ["未知", ""], ["华北", ""]] + assert out.masked_values.hit("张伟") and not out.masked_values.hit("未知") + + +def test_placeholders_are_never_collected(): + secrets = masking._Secrets() + secrets.add({"columns": ["name"], "rows": [["未知"], ["N/A"], ["暂无"], ["张伟"], ["NULL"]]}, {"name"}) + assert secrets.strong == {"张伟"} and secrets.weak == {"张伟"} + assert secrets.scrub("原因未知,暂无记录,N/A,张伟") == f"原因未知,暂无记录,N/A,{MASKED}" From 8c229c3d1aab92f175922665f2743dec9d1c8771 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 22:52:28 +0800 Subject: [PATCH 03/12] =?UTF-8?q?K3b=EF=BC=9A=E5=88=86=E6=A1=A3=E5=86=99?= =?UTF-8?q?=E5=9C=A8=E6=B4=BE=E7=94=9F=E6=9D=A5=E6=BA=90=E9=87=8C=E3=80=81?= =?UTF-8?q?=E5=A4=96=E5=B1=82=E6=8C=89=E5=AE=83=E5=88=86=E7=BB=84=E7=9A=84?= =?UTF-8?q?=E6=A0=87=E7=AD=BE=E5=88=97=E7=85=A7=E5=B8=B8=E6=98=BE=E7=A4=BA?= =?UTF-8?q?=EF=BC=8C=E5=92=8C=E5=BD=93=E5=88=86=E7=BB=84=E9=94=AE=E7=9A=84?= =?UTF-8?q?=20CASE=20=E4=B8=80=E6=A0=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 以前分档写在子查询、WITH 里(SELECT *, CASE … END AS 档位 FROM sales),外层再按档位分组,工具边界把档位整列遮成「已遮罩」,模型分不清哪个数属于哪一档,质询回答成了「已遮罩有 3 笔,已遮罩有 2 笔」。现在 _derived_hidden 认出只装着标签(_label_columns)、又一档一行的结果列(这一层或派生来源里那一层的 GROUP BY 键,按位置、别名、照原样写都算,_grouped_labels),不算取自遮罩列:格子里只有标签、和遮罩列不同名才算,认不出的照旧整列遮;SQL 里照旧认得出它取自金额。工具边界、质询回答、质询工件用的是同一套。* 展开的表在冻结的表结构里查得到、没有同名列时标签照常认(原来就有),不知道表结构时照旧当它带得出(多遮)。测试 test_inquiry_masking 里「外层再分组照旧算」的两个用例按新规则改;补六种写法的工具边界用例、逐行和混了别的值的反例、质询端到端用例,去掉豁免后全部变红。_tainted 多交回一个「认不出、整份都算」的标记,整份遮时不豁免。 --- backend/app/engine/inquiry.py | 3 +- backend/app/engine/masking.py | 240 +++++++++++++++++++- backend/tests/test_inquiry_masking.py | 16 +- backend/tests/test_inquiry_masking_e2e.py | 63 +++++ backend/tests/test_masking_tool_boundary.py | 71 ++++++ 5 files changed, 374 insertions(+), 19 deletions(-) create mode 100644 backend/tests/test_inquiry_masking_e2e.py diff --git a/backend/app/engine/inquiry.py b/backend/app/engine/inquiry.py index 3d974140..894f6a41 100644 --- a/backend/app/engine/inquiry.py +++ b/backend/app/engine/inquiry.py @@ -75,7 +75,8 @@ UNPIVOT、MATCH_RECOGNIZE 读到遮罩列的,整份快照的列都遮(多遮)。SQL 解析不了的只看文字:写着遮罩列、*(COUNT(*) 除外)、 TABLE x 的整份遮,整行引用这类不写列名的写法认不出 - 按遮罩列分档的标签(CASE、IIF 的结果全是写死的常量):标签文字是 agent 写在 SQL 里的,不当遮罩值,SQL、回答正文里 - 照原样;标签列当分组键(每一档一行)照常显示,不分组的逐行标签列整列遮(格子里看不出某一笔落在哪一档),可 agent + 照原样;标签列当分组键(每一档一行)照常显示,分档写在子查询、WITH 里、外层(或中间一层)按它分组的也算 + (masking._grouped_labels),不分组的逐行标签列整列遮(格子里看不出某一笔落在哪一档),可 agent 在正文里写出「这一笔是大单」遮不住,按 id 分组再按档数个数也看得出(同按阈值查出某几行)。只有只装着标签的列不收: 结果的这一列在 UNION 的每一支里都是这样的 CASE / IIF,或者按作用域认得出取自子查询、CTE 里只装着标签的一列;有一支 取自数据源里的表(遮罩列原样选出、经子查询按原名传出,哪怕起了标签的名字)就照收。认不准的也照收(多遮),比如 diff --git a/backend/app/engine/masking.py b/backend/app/engine/masking.py index a6fce84d..e7caa989 100644 --- a/backend/app/engine/masking.py +++ b/backend/app/engine/masking.py @@ -21,7 +21,8 @@ 不算(_value_columns、_band_only)。认不出落在哪一列的(PIVOT、UNPIVOT 这类也算)、SQL 解析不了又写着遮罩列(或者 *、 TABLE x)的,整份快照的列都算。双盲复算按同一套认法在派盲算之前挡下取自遮罩列的指标(derived_columns) - 只装着分档标签的列(_label_columns):标签是 agent 写在 SQL 里的字,不当遮罩值;* 展开数据源里的表时按冻结的表结构 - 认它带不带同名的列(_schema_tables) + 认它带不带同名的列(_schema_tables),不知道表结构的按带得出算(多遮)。分档写在派生来源里、外层按它分组的(一档 + 一行,_grouped_labels)同当分组键的 CASE 一样照常显示(_shown_labels) - 哪些值(_Secrets):遮罩列里出现过的值,按值认有门槛(SECRET_MIN、SECRET_DIGITS),占位的写法(PLACEHOLDERS:未知、 暂无……)不收;换的时候先按词、按开头几个字筛出 正文里真有的那几个值,不现编上万个分支的正则;正文、文字格、SQL 的字符串和注释里换了写法的长号码(199-1474-2785、 @@ -496,6 +497,191 @@ def _all_labels(texts: list[set[str] | None]) -> set[str] | None: return set().union(*texts) if all(t is not None for t in texts) else None +def _grouped_labels(sql: Any, kind: str | None, columns: list[str], labels: dict[str, set[str]], + tables: dict[str, set[str]] | None) -> set[str]: + """结果里只装着分档标签(labels,_label_columns 认出的)、又是一档一行的列(列名小写):标签列当分组键照常显示,同 + _band_only,只是分档写在派生来源里、外层才按它分组(SELECT 档位, COUNT(*) FROM (SELECT *, CASE … END AS 档位 FROM + sales) GROUP BY 档位、WITH 里的一段算档位、外层分组)。一档一行:结果的这一列在每一支里都是那一层 SELECT 的分组键, + 或者取自派生来源里一档一行的那一列(_item_grouped)。同名的几列都要是。解析不了、认不准的一个都不算(照旧整列遮)。""" + from app.data.sql_semantics import _parse, _read + + if not labels: + return set() + try: + root, _ = _parse(sql, _read(kind)) + if root is None: + return set() + _normalized(root, _read(kind), None) + result = next((top for top, role, _listed in _blocks(root) if role == "result"), None) + if result is None: + return set() + memo = _LabelMemo() + memo.tables = tables + out: set[str] = set() + for name in labels: + places = [i for i, c in enumerate(columns) if c.lower() == name] + if places and all(_result_grouped(result, place, name, len(columns), memo) for place in places): + out.add(name) + return out + except Exception: # noqa: BLE001 - 分析本身的缺陷:一个都不算(照旧整列遮,宁可多遮) + logger.warning("核查查询的分档标签分组分析失败", exc_info=True) + return set() + + +def _result_grouped(top: Any, place: int, name: str, width: int, memo: _LabelMemo) -> bool: + """结果(width 列)的第 place 列(列名 name)是不是一档一行:UNION 这类的每一支都是。落在 * 里的按列名经 * 往下找, + 只有一支时才找(同 _result_label)。""" + from sqlglot import exp + + leaves = _leaves(top) + if not all(isinstance(leaf, exp.Select) for leaf in leaves): + return False + for select in leaves: + items = list(select.expressions) + item = next((i for i, p in zip(items, _places(items, width)) if p == place), None) + if item is not None: + grouped = _item_grouped(item, select, memo) + elif len(leaves) == 1: + grouped = _star_grouped(select, name, memo) is True + else: + return False + if not grouped: + return False + return True + + +def _item_grouped(item: Any, select: Any, memo: _LabelMemo) -> bool: + """select 的一个选出项一档一行:它就是这一层的分组键(_group_key),或者是单独一列、取自 FROM 里派生来源中一档一行的 + 那一列(_source_grouped)。""" + from sqlglot import exp + + if _group_key(select, item): + return True + node = item + while isinstance(node, (exp.Alias, exp.Paren)): + node = node.this + if isinstance(node, exp.Column) and not isinstance(node.this, exp.Star) and node.name: + return _source_grouped(select, node.table.lower() or None, node.name.lower(), memo) is True + return False + + +def _group_key(select: Any, item: Any) -> bool: + """select 的选出项 item 是它 GROUP BY 的一个键:按位置、按别名、照原样再写一遍(同一列写不写表名都算,FROM 里只有 + 一个来源时)。ROLLUP、GROUPING SETS 这类不认。""" + from sqlglot import exp + + group = select.args.get("group") + if group is None: + return False + node = item + while isinstance(node, (exp.Alias, exp.Paren)): + node = node.this + place = next((str(i + 1) for i, e in enumerate(select.expressions) if e is item), None) + alias = item.alias.lower() if isinstance(item, exp.Alias) and item.alias else None + single = len(_from_sources(select)) == 1 + for key in group.expressions: + while isinstance(key, exp.Paren): + key = key.this + if (isinstance(key, exp.Literal) and not key.is_string and key.this == place) or key == node: + return True + if not isinstance(key, exp.Column) or isinstance(key.this, exp.Star): + continue + if alias and not key.table and key.name.lower() == alias: + return True + if isinstance(node, exp.Column) and key.name.lower() == node.name.lower() and ( + key.table.lower() == node.table.lower() or (single and not (key.table and node.table))): + return True + return False + + +def _source_grouped(select: Any, qualifier: str | None, name: str, memo: _LabelMemo) -> Any: + """select 的 FROM、JOIN 里读到的那一列(qualifier 同 _source_label)是不是一档一行:取自派生来源、那里一档一行的为 + True;没有哪个来源出这一列为 None;别的(数据源里的表可能出这一列、按这一列 USING / NATURAL 连起来、派生来源里不是 + 一档一行)为 False。""" + sources = _from_sources(select) + if qualifier is not None: + sources = [(alias, source) for alias, source in sources if alias == qualifier] + if len(sources) != 1: + return False + found = False + for _alias, source in sources: + derived = _derived_block(source) + if derived is None: + if not _table_lacks(source, name, memo): + return False + continue + grouped = _block_grouped(derived[0], derived[1], name, memo) + if grouped is None: + continue + if not grouped: + return False + found = True + if not found: + return None + return False if _merging(select, name) else True + + +def _block_grouped(block: Any, listed: list[str] | None, name: str, memo: _LabelMemo) -> Any: + """派生来源的那块查询(listed 是写的列名表)出的 name 这一列是不是一档一行;不出这一列为 None。查法同 _block_label_of: + 一条 SELECT 按选出项的名字找,没有的经 * 往下找;UNION 这类按位置看每一支。""" + from sqlglot import exp + + key = ("grouped", id(block), name) + if key in memo: + return memo[key] + # 先记成 False:递归的 CTE 绕回自己时按不是算 + memo[key] = False + leaves = _leaves(block) + result: Any = False + if all(isinstance(leaf, exp.Select) for leaf in leaves): + if listed is None and len(leaves) == 1: + items = list(leaves[0].expressions) + named = [i for i, item in enumerate(items) if _same_name(item, name)] + if not named: + result = _star_grouped(leaves[0], name, memo) + elif _star_grouped(leaves[0], name, memo, before=named[0]) is None: + result = all(_item_grouped(items[i], leaves[0], memo) for i in named) + else: + width: int | None = None + place: int | None = None + if listed is not None: + if name not in listed: + result = None + elif listed.count(name) == 1: + place, width = listed.index(name), len(listed) + else: + first = list(leaves[0].expressions) + places = [p for item, p in zip(first, _places(first, None)) if _same_name(item, name)] + if not places and not any(_is_star(item) for item in first): + result = None + elif len(places) == 1: + place = places[0] + if place is not None: + picked = [next((i for i, p in zip(list(s.expressions), _places(list(s.expressions), width)) + if p == place), None) for s in leaves] + result = all(i is not None and _item_grouped(i, s, memo) for i, s in zip(picked, leaves)) + memo[key] = result + return result + + +def _star_grouped(select: Any, name: str, memo: _LabelMemo, *, before: int | None = None) -> Any: + """select 经 *(t.* 只看 t)带出的 name 这一列是不是一档一行;没有 * 带出这一列为 None。before 同 _star_label。""" + from sqlglot import exp + + found = False + for item in select.expressions[:before]: + if not _is_star(item): + continue + qualifier = (item.table.lower() or None) if isinstance(item, exp.Column) else None + grouped = _source_grouped(select, qualifier, name, memo) + if grouped is None: + continue + if not grouped: + return False + found = True + return True if found else None + + def _from_sources(select: Any) -> list[tuple[str, Any]]: """一条 SELECT 的 FROM、JOIN 里的来源:(名字, 来源节点),名字是别名,没起别名的是表名(小写)。""" from_ = select.args.get("from_") @@ -1090,8 +1276,9 @@ def _source_names(top: Any, listed: list[str] | None, names: set[str], def _tainted(sql: str, kind: str | None, masked: set[str], width: int, tables: dict[str, list[str]] | None = None - ) -> tuple[set[str], set[int]] | None: - """SQL 里指代遮罩列的名字(小写),和结果(width 列)里值取自遮罩列的那几列的位置。解析不了返回 None。 + ) -> tuple[set[str], set[int], bool] | None: + """SQL 里指代遮罩列的名字(小写)、结果(width 列)里值取自遮罩列的那几列的位置,以及是不是认不出、整份都算(这时 + 位置是全部的列)。解析不了返回 None。 名字从遮罩列本身起,一层层传下去:选出列的值取自这些名字的(buyer_phone AS 手机号、UPPER(buyer_phone) AS x),它的 名字也算;派生来源(子查询、WITH 里的一段)按位置对上列名表、UNION 第一支的列名,也算,直到不再多出名字。同名 @@ -1162,7 +1349,7 @@ def _tainted(sql: str, kind: str | None, masked: set[str], width: int, tables: d everything = True else: positions |= spread - return names, set(range(width)) if everything else positions + return names, set(range(width)) if everything else positions, everything except Exception: # noqa: BLE001 - 分析本身的缺陷不能让回答出错:按解析不了处理(宁可多遮) logger.warning("质询的 SQL 遮罩分析失败", exc_info=True) return None @@ -1186,14 +1373,17 @@ def _implicit(sql: str) -> bool: def _derived_hidden(snapshot: dict[str, Any], hidden: set[str], kind: str | None, - tables: dict[str, list[str]] | None = None) -> tuple[set[str], set[str]]: + tables: dict[str, list[str]] | None = None, *, labels: dict[str, set[str]] | None = None + ) -> tuple[set[str], set[str]]: """(这份快照要遮的列, SQL 里指代遮罩列的名字),都是小写。tables 是查询当时冻结的表结构(_schema_columns),* 展开 数据源里的表时按它数位置;没给的按快照记着的表结构快照(schema_artifact)认,和工具边界遮的时候是同一份。 证据面板只按列名遮(hidden);agent 的查询起了别名、套了函数,结果列名就对不上。本仓库的查询惯例是起中文别名 (region AS 区域),所以按 SQL 认出改了名字的遮罩列(_tainted)。SQL 解析不了、里面又写着遮罩列的(*、TABLE x 不写 列名也会把遮罩列带进别的列名,同样算,_implicit),整份快照的列都遮掉;合并查询的快照不看 SQL,它记着自己的来历 - (_hidden_columns 已按 _merge_masks 遮好)。""" + (_hidden_columns 已按 _merge_masks 遮好)。 + 分档写在派生来源里、外层按它分组的标签列(一档一行,_grouped_labels)照常显示,同 _band_only;格子里装着标签以外的、 + 和遮罩列同名的照旧遮(_shown_labels)。labels 是调用方已经认好的标签列(_label_columns),没给的按快照认。""" base = set(hidden) sql = snapshot.get("sql") if not base or not isinstance(sql, str) or not sql.strip() or snapshot.get("source") == MERGE_SOURCE: @@ -1203,8 +1393,36 @@ def _derived_hidden(snapshot: dict[str, Any], hidden: set[str], kind: str | None if found is None: hides = _mentions(sql, base) or _implicit(sql) return (base | {c.lower() for c in columns}, base) if hides else (base, base) - names, positions = found - return names | {columns[i].lower() for i in positions if i < len(columns)}, names + names, positions, everything = found + out = names | {columns[i].lower() for i in positions if i < len(columns)} + if not everything and out & {c.lower() for c in columns}: + out -= _shown_labels(snapshot, kind, columns, base, tables, labels) + return out, names + + +def _shown_labels(snapshot: dict[str, Any], kind: str | None, columns: list[str], base: set[str], + tables: dict[str, list[str]] | None, labels: dict[str, set[str]] | None) -> set[str]: + """派生自遮罩列、却当分组键照常显示的标签列(列名小写,_grouped_labels):只装着标签(labels,没给的按快照认)、和 + 数据源设的遮罩列不同名、格子里只有标签(数的写法对不上的、查询时已经遮掉的都不算)、一档一行。表结构给了 tables 的 + 按它,没给的按快照记着的表结构快照。""" + sets = {name: set(cols) for name, cols in tables.items()} if tables is not None else ( + _frozen_tables(snapshot["schema_artifact"]) if isinstance(snapshot.get("schema_artifact"), str) else None) + if labels is None: + labels = _snapshot_labels(snapshot, kind, sets) + wanted = {name: texts for name, texts in labels.items() if name not in base and _only_labels(snapshot, name, texts)} + return _grouped_labels(snapshot["sql"], kind, columns, wanted, sets) if wanted else set() + + +def _only_labels(snapshot: dict[str, Any], name: str, texts: set[str]) -> bool: + """快照里叫 name 的列,格子不是空就是标签的一种写法(_forms 对得上 texts)。""" + columns = [str(c).lower() for c in snapshot.get("columns") or []] + index = [i for i, c in enumerate(columns) if c == name] + for row in snapshot.get("rows") or []: + cells = [row[i] for i in index if i < len(row)] if isinstance(row, list) else ( + [v for k, v in row.items() if str(k).lower() == name] if isinstance(row, dict) else [row]) + if any(v is not None and not (_forms(v) & texts) for v in cells): + return False + return True def derived_columns(snapshot: dict[str, Any], hidden: set[str], kind: str | None) -> set[str] | None: @@ -1218,7 +1436,7 @@ def derived_columns(snapshot: dict[str, Any], hidden: set[str], kind: str | None found = _tainted(sql, kind, set(hidden), len(columns), _snapshot_columns(snapshot)) if found is None: return None - names, positions = found + names, positions, _everything = found return {c.lower() for i, c in enumerate(columns) if c.lower() in names or i in positions} @@ -2059,9 +2277,9 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str if not (base or given) or not columns or not _query_shaped(payload): return QueryMask(payload, [], _Secrets()) try: - hidden = _derived_hidden(payload, base, kind, _schema_columns(schema))[0] if base else set() - own = _Secrets() labels = _snapshot_labels(payload, kind, _schema_tables(schema)) + hidden = _derived_hidden(payload, base, kind, _schema_columns(schema), labels=labels)[0] if base else set() + own = _Secrets() if any(c.lower() in hidden for c in columns): own.add(payload, hidden, labels=labels, base=base) pool = own diff --git a/backend/tests/test_inquiry_masking.py b/backend/tests/test_inquiry_masking.py index a6dc4a42..2e0f3e4b 100644 --- a/backend/tests/test_inquiry_masking.py +++ b/backend/tests/test_inquiry_masking.py @@ -183,10 +183,13 @@ BANDS = "CASE WHEN amount BETWEEN 0 AND 20000 THEN '小单' WHEN amount BETWEEN "SELECT CASE WHEN amount <= 20000 THEN 1 WHEN amount <= 60000 THEN 2 ELSE NULL END AS 档位, COUNT(*) AS 笔数 " "FROM sales GROUP BY 1", "SELECT IIF(amount > 20000, '大单', '小单') AS 档位, COUNT(*) AS 笔数 FROM sales GROUP BY 1", + # 分档写在子查询、WITH 里,外层再按它分组(K3b):同样每一档一行 + f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT {BANDS} AS 档位 FROM sales) t GROUP BY 档位", + f"WITH t AS (SELECT {BANDS} AS 档位 FROM sales) SELECT 档位, COUNT(*) AS 笔数 FROM t GROUP BY 1", ]) def test_a_band_label_used_as_the_group_key_is_not_a_masked_value(sql): - """按遮罩列分档、每一档一行(CASE、IIF 的结果全是写死的标签,又是分组键):标签只说一行落在哪一档,带不出金额, - 档位列不遮,才分得清哪一档是几笔。笔数照样只是计数。""" + """按遮罩列分档、每一档一行(CASE、IIF 的结果全是写死的标签,又是分组键;分档写在派生来源里、外层按它分组的也算): + 标签只说一行落在哪一档,带不出金额,档位列不遮,才分得清哪一档是几笔。笔数照样只是计数。""" assert hidden_of(sql, ["档位", "笔数"], {"amount"}) == {"amount"} @@ -195,8 +198,8 @@ def test_a_band_label_used_as_the_group_key_is_not_a_masked_value(sql): f"SELECT id, {BANDS} AS 档位, 1 AS 笔数 FROM sales", # 结果里有一支取的是金额本身 "SELECT CASE WHEN amount > 20000 THEN amount ELSE 0 END AS 档位, COUNT(*) AS 笔数 FROM sales GROUP BY 1", - # 分档写在子查询里,外层再分组:子查询里的档位逐行带出来,照旧算 - f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT {BANDS} AS 档位 FROM sales) t GROUP BY 档位", + # 分档写在子查询里,外层不分组:子查询里的档位逐行带出来,照旧算 + f"SELECT 档位, 1 AS 笔数 FROM (SELECT {BANDS} AS 档位 FROM sales) t", ]) def test_band_labels_row_by_row_still_count_as_derived(sql): assert hidden_of(sql, ["档位", "笔数"], {"amount"}) >= {"amount", "档位"} @@ -265,9 +268,8 @@ def test_a_renamed_masked_column_keeps_values_that_match_labels_written_elsewher @pytest.mark.parametrize("sql, columns, rows", [ - # 标签列经子查询、CTE 原样传下去(外层再分组也一样:每一行的档位照旧算派生值,见上) - (f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT {BANDS} AS 档位 FROM sales) t GROUP BY 档位", - ["档位", "笔数"], [["大单", 2], ["小单", 3]]), + # 标签列经子查询、CTE 原样传下去,外层不分组:每一行的档位照旧算派生值(外层按它分组的不遮,见上) + (f"SELECT 档位, 1 AS 笔数 FROM (SELECT {BANDS} AS 档位 FROM sales) t", ["档位", "笔数"], [["大单", 1], ["小单", 1]]), (f"WITH t AS (SELECT {BANDS} AS 档位 FROM sales) SELECT * FROM t", ["档位"], [["小单"], ["大单"]]), # 没起别名的标签列:按位置认 (f"SELECT id, {BANDS} FROM sales", ["id", BANDS], [[1, "小单"], [2, "大单"]]), diff --git a/backend/tests/test_inquiry_masking_e2e.py b/backend/tests/test_inquiry_masking_e2e.py new file mode 100644 index 00000000..f791f120 --- /dev/null +++ b/backend/tests/test_inquiry_masking_e2e.py @@ -0,0 +1,63 @@ +"""审批质询的遮罩收口,端到端(engine/inquiry.py 经核查 agent 的工具边界): + +- 分档写在子查询、WITH 里、外层按档位分组:档位列交给模型、存下、渲染进回答都照常,标签不当遮罩值,SQL 照原样封存; + 以前回答成了「已遮罩有 3 笔,已遮罩有 2 笔。」 +- 交给质询 agent 的证据摘录先遮:报告取数把遮罩列改了名字选出来(buyer_phone AS 手机号),摘录里不给这一列;别的列、 + SQL 里写着的遮罩值(换了写法的号码也算)换成「已遮罩」。以前摘录只按列名遮,号码原样交给模型 + +模型一律是 MockChatModel 剧本,数据源是现造的 SQLite 文件,区域、金额、手机号都是虚构的。 +""" +from __future__ import annotations + +import json + +import pytest + +from app.core import artifact_store +from tests.test_inquiry_api import ( # noqa: F401 - engine_up 是自动用上的夹具,client、big_shop 是夹具 + AMOUNTS, + Models, + Play, + ask, + big_shop, + client, + engine_up, + events, + fetching, + pause, + pending, +) + +BAND = "CASE WHEN amount >= 20000 THEN '大单' ELSE '小单' END" + + +@pytest.mark.parametrize("sql", [ + pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT *, {BAND} AS 档位 FROM sales) GROUP BY 档位 ORDER BY 档位", + id="子查询里 * 加分档"), + pytest.param(f"WITH b AS (SELECT {BAND} AS 档位 FROM sales) SELECT 档位, COUNT(*) AS 笔数 FROM b GROUP BY 档位 " + "ORDER BY 档位", id="WITH 里只有分档"), +]) +async def test_a_band_grouped_outside_its_derived_source_reads_as_written(monkeypatch, big_shop, client, sql): + """金额设了遮罩,问两万元上下各有几笔:分档写在派生来源里、外层按档位分组。模型看得到档位、回答里引用的档位和它写的 + 「大单」「小单」都照原样,SQL 照原样封存,质询工件里档位列照常显示。""" + said = "[[v:Q1.r0.档位]]有 [[v:Q1.r0.笔数]] 笔,[[v:Q1.r1.档位]]有 [[v:Q1.r1.笔数]] 笔,大单少于小单。[[see:Q1]]" + models = Models(monkeypatch, big_shop, writer="华东首单金额 [[v:Q1.r0.金额]]。[[see:Q1]]", + plays={"两万元上下": Play([sql], said)}) + paused = await pause(big_shop, graph=fetching(big_shop, AMOUNTS)) + [approval] = await pending(paused.id) + r = await ask(client, paused.id, report="write", unit=None, question="两万元上下各有几笔?", + approval_id=approval.id) + assert r.status_code == 200, r.text + item = r.json()["inquiry"] + assert (item["answer"]["ok"], item["answer"]["text"]) == (True, "大单有 3 笔,小单有 2 笔,大单少于小单。") + [query] = item["queries"] + assert query["sql"] == sql, "SQL 里的标签不当遮罩值换" + [event] = await events(paused.id, "approval.inquiry") + assert event.data["queries"][0]["sql"] == sql + snapshot = artifact_store.load(query["artifact"]) + assert snapshot["rows"] == [["大单", 3], ["小单", 2]] and snapshot["masked_at_query"] == [] + seen = [str(m.content) for messages in models.finals for m in messages] + assert any('r0:档位="大单",笔数=3' in text for text in seen), "模型看得到档位" + got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{query['artifact']}") + assert got.status_code == 200, got.text + assert (got.json()["content"]["rows"], got.json()["masked"]) == ([["大单", 3], ["小单", 2]], []) diff --git a/backend/tests/test_masking_tool_boundary.py b/backend/tests/test_masking_tool_boundary.py index 1f44345e..311fcf75 100644 --- a/backend/tests/test_masking_tool_boundary.py +++ b/backend/tests/test_masking_tool_boundary.py @@ -182,3 +182,74 @@ def test_placeholders_are_never_collected(): secrets.add({"columns": ["name"], "rows": [["未知"], ["N/A"], ["暂无"], ["张伟"], ["NULL"]]}, {"name"}) assert secrets.strong == {"张伟"} and secrets.weak == {"张伟"} assert secrets.scrub("原因未知,暂无记录,N/A,张伟") == f"原因未知,暂无记录,N/A,{MASKED}" + + +# -------------------------------------------------------------------------- +# 分档写在派生来源里、外层按它分组:标签列照常显示,标签不当遮罩值 +# -------------------------------------------------------------------------- + + +LAYOUT = ["id", "region", "amount", "visits"] +#: 探查得到的表结构:sales 里没有「档位」这一列 +SCHEMA = {"tables": {"sales": {"qualified": "sales", "schema": None, "columns": [{"name": c} for c in LAYOUT]}}} +BAND = "CASE WHEN amount >= 20000 THEN '大单' ELSE '小单' END" +GROUPED = [ + pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT *, {BAND} AS 档位 FROM sales) GROUP BY 档位 ORDER BY 档位", + id="子查询里 * 加分档"), + pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT s.*, {BAND} AS 档位 FROM sales s) t GROUP BY 档位 " + "ORDER BY 档位", id="子查询里 s.* 加分档"), + pytest.param(f"WITH b AS (SELECT *, {BAND} AS 档位 FROM sales) SELECT 档位, COUNT(*) AS 笔数 FROM b GROUP BY 档位 " + "ORDER BY 档位", id="WITH 里 * 加分档"), + pytest.param(f"WITH b AS (SELECT {BAND} AS 档位 FROM sales) SELECT 档位, COUNT(*) AS 笔数 FROM b GROUP BY 档位 " + "ORDER BY 档位", id="WITH 里只有分档"), + pytest.param(f"WITH b AS (SELECT {BAND} AS 档位 FROM sales) SELECT b.档位, COUNT(*) AS 笔数 FROM b GROUP BY 1 " + "ORDER BY 1", id="按位置分组"), + pytest.param(f"WITH b AS (SELECT {BAND} AS 档位 FROM sales), c AS (SELECT 档位, COUNT(*) AS 笔数 FROM b " + "GROUP BY 档位) SELECT * FROM c ORDER BY 档位", id="分组在中间一层"), +] +ROWS = [["大单", 3], ["小单", 2]] + + +@pytest.mark.parametrize("sql", GROUPED) +def test_a_band_grouped_outside_its_derived_source_is_shown(sql): + """分档(按遮罩的金额写死的标签)写在子查询、WITH 里,外层按它分组:每档一行,档位列照常显示,标签不当遮罩值收; + 以前整列遮成「已遮罩」,模型分不清哪个数属于哪一档。""" + payload = {"sql": sql, "columns": ["档位", "笔数"], "rows": [list(r) for r in ROWS], + "column_types": {"档位": "text", "笔数": "number"}} + done = masking.mask_query_result(payload, ["amount"], "sqlite", schema=SCHEMA) + assert done.payload["rows"] == ROWS and done.columns == [] + assert not done.secrets.strong + hidden, names = masking._derived_hidden({**payload, "rows": ROWS}, {"amount"}, "sqlite", + masking._schema_columns(SCHEMA)) + assert "档位" not in hidden and "档位" in names, "SQL 里照旧认得出它取自金额" + + +@pytest.mark.parametrize("sql", [ + pytest.param(f"SELECT id, 档位 FROM (SELECT id, {BAND} AS 档位 FROM sales) ORDER BY id", id="逐行的档位"), + pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT {BAND} AS 档位 FROM sales UNION ALL " + "SELECT region FROM sales) GROUP BY 档位", id="另一支是区域"), + pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT id, {BAND} AS 档位 FROM sales) x " + "FULL JOIN (SELECT id, amount AS 档位 FROM sales) y USING (档位) GROUP BY 档位", id="按档位连起来"), +]) +def test_a_band_that_is_not_one_row_per_band_is_still_masked(sql): + """逐行的档位(看得出每一笔落在哪一档)、和别的值混在一列里、按它连起来又不写表名的(外连接时取的可能是另一边的 + 金额),照旧整列遮。""" + payload = {"sql": sql, "columns": ["档位", "笔数"] if "笔数" in sql else ["id", "档位"], + "rows": [["大单", 3], ["小单", 2]] if "笔数" in sql else [[1, "小单"], [3, "大单"]]} + done = masking.mask_query_result(payload, ["amount"], "sqlite", schema=SCHEMA) + assert "档位" in done.columns + + +def test_a_grouped_band_with_cells_other_than_its_labels_is_still_masked(): + """格子里有标签以外的值(认错了、数的写法对不上):照旧整列遮。""" + sql = GROUPED[3].values[0] + payload = {"sql": sql, "columns": ["档位", "笔数"], "rows": [["大单", 3], ["13900000001", 2]]} + assert masking.mask_query_result(payload, ["amount"], "sqlite", schema=SCHEMA).columns == ["档位"] + + +def test_without_a_schema_a_star_next_to_the_band_is_still_masked(): + """不知道表结构时,* 展开的表可能就有「档位」这一列(外层取到的是它):照旧整列遮,标签照收(多遮)。""" + sql = GROUPED[0].values[0] + payload = {"sql": sql, "columns": ["档位", "笔数"], "rows": [list(r) for r in ROWS]} + done = masking.mask_query_result(payload, ["amount"], "sqlite") + assert done.columns == ["档位"] and done.payload["rows"] == [[MASKED, 3], [MASKED, 2]] From 6ba7babdf00e55e4ddb9d37d238c5e76ff5fba37 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 23:03:35 +0800 Subject: [PATCH 04/12] =?UTF-8?q?K3b=EF=BC=9A=E8=AE=A4=E4=B8=8D=E5=85=A8?= =?UTF-8?q?=E7=9A=84=20SQL=20=E5=86=99=E6=B3=95=E6=95=B4=E4=BB=BD=E9=81=AE?= =?UTF-8?q?=E2=80=94=E2=80=94(=E8=A1=A8=E8=BE=BE=E5=BC=8F).*=20=E5=B1=95?= =?UTF-8?q?=E5=BC=80=E3=80=81=E8=A1=A8=E5=90=8D=E5=92=8C=20SQL=20=E5=86=99?= =?UTF-8?q?=E5=9C=A8=E5=AD=97=E7=AC=A6=E4=B8=B2=E9=87=8C=E7=9A=84=E5=87=BD?= =?UTF-8?q?=E6=95=B0=E3=80=81=E5=88=86=E6=9E=90=E5=87=BA=E6=84=8F=E5=A4=96?= =?UTF-8?q?=E6=88=96=E8=B6=85=E5=87=BA=E5=B7=A5=E4=BD=9C=E9=87=8F=E4=B8=8A?= =?UTF-8?q?=E9=99=90=EF=BC=9BLATERAL=20=E7=9C=8B=E4=B8=8D=E8=A7=81?= =?UTF-8?q?=E5=AE=83=E8=87=AA=E5=B7=B1=EF=BC=9B=E4=B8=80=E9=95=BF=E4=B8=B2?= =?UTF-8?q?=20CTE=20=E4=B8=8D=E5=86=8D=E5=8D=A1=E4=BD=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - (表达式).* 展开(复合类型的列 (p.profile).*、记录函数 (jsonb_each_text(pj)).*、(jsonb_populate_record(…)).*、子查询里的匿名 ROW (q.r).*)以前被改写成对不上来源的 *,最外层、UNION 第一支、没写列名表的派生来源里按「照原名带出」认,字段名对不上遮罩列,号码原样交给模型、落库(K3 第 3 轮引入的回退,15 条新漏)。现在 _normalized 只把 (s).*、s 是这一层 FROM 里的来源时改写成 s.*,别的先记下表达式的值取自哪些列(整行引用已改写),读到遮罩列的整份遮。 - query_to_xml、table_to_xml、cursor_to_xml、schema_to_xml、database_to_xml 这一族、dblink、crosstab、DBMS_XMLGEN、DuckDB 的 query、OPENQUERY,按页按文件读原始数据的,pg_stats、sqlite_stat4、pg_stat_statements 这类存着值的统计、会话视图:按名字认出整份遮(_INDIRECT),解析不了的也一样。 - 整行引用由里往外找列时,LATERAL 里只看排在它前面的来源(_visible_sources),它自己的结果列碰巧叫 s 也不把整行 s 当成列。 - 一次分析里记下派生来源对应的那块查询和一项的值取自哪些列(_analysis,按线程分开),250 段一段套一段地整行引用的 CTE 从二十几秒降到零点几秒;超出工作量上限(ANALYSIS_STEPS)的整份遮。解析之后的分析出了意外也整份遮,不再退回只看文字(没写遮罩列、没有 * 的整行引用以前就放过去了)。 补测试:三种展开 × 最外层、子查询、CTE、LATERAL,端到端经核查 agent 的查询工具,九种函数和视图,LATERAL 自身,工作量上限、分析意外、长 CTE 的耗时;分别去掉每一处实现,对应的测试都变红。 --- backend/app/engine/inquiry.py | 13 +- backend/app/engine/masking.py | 292 +++++++++++++++++------ backend/tests/test_masking_opaque_sql.py | 282 ++++++++++++++++++++++ 3 files changed, 511 insertions(+), 76 deletions(-) create mode 100644 backend/tests/test_masking_opaque_sql.py diff --git a/backend/app/engine/inquiry.py b/backend/app/engine/inquiry.py index 894f6a41..f9796a0e 100644 --- a/backend/app/engine/inquiry.py +++ b/backend/app/engine/inquiry.py @@ -71,9 +71,16 @@ 一律算取自遮罩列。EXCEPT 后一支只用来去掉行,也照 UNION 的后一支算(多遮) - 整行和函数参数里的 *(Postgres 的 row_to_json(s)、json_agg(t)、SELECT s、s::text、to_json(s.*)、concat_ws(',', s.*), Oracle 的 JSON_OBJECT(*))按 * 展开的来源算:取自子查询、CTE 的看它选出的列,取自数据源里的表的一律算取自遮罩列(整列 - 遮)。Postgres 先按列认:表结构里查得到和表同名的列(visits 表的 visits 列)才当列,查不到时按整行算(多遮)。PIVOT、 - UNPIVOT、MATCH_RECOGNIZE 读到遮罩列的,整份快照的列都遮(多遮)。SQL 解析不了的只看文字:写着遮罩列、*(COUNT(*) 除外)、 - TABLE x 的整份遮,整行引用这类不写列名的写法认不出 + 遮)。Postgres 先按列认:表结构里查得到和表同名的列(visits 表的 visits 列)才当列,查不到时按整行算(多遮);LATERAL + 里只看排在它前面的来源,它自己的结果列不算。PIVOT、UNPIVOT、MATCH_RECOGNIZE 读到遮罩列的,整份快照的列都遮(多遮)。 + SQL 解析不了的只看文字:写着遮罩列、*(COUNT(*) 除外)、TABLE x、下面那几类函数和视图的整份遮,整行引用这类不写列名的 + 写法认不出 +- 认不全的写法整份遮(多遮,同一份里没碰遮罩列的编号、金额也遮):(表达式).* 展开读到遮罩列的(复合类型的列 + (p.profile).*、函数交回的记录 (jsonb_each_text(pj)).*、子查询里的 ROW (q.r).*,展开成几列、各叫什么 SQL 里看不出); + 表名、SQL 写在字符串里交给函数执行、整表导出的(query_to_xml、table_to_xml、cursor_to_xml、schema_to_xml、 + database_to_xml 这一族,dblink、crosstab、ts_stat,Oracle 的 DBMS_XMLGEN、XMLTYPE,DuckDB 的 query,SQL Server 的 + OPENQUERY)、按页按文件读原始数据的、统计信息和会话记录里存着值的(pg_stats、sqlite_stat4、pg_stat_statements), + 按名字认(masking._INDIRECT),自己建的同类函数认不出;分析出了意外、一长串 CTE 超出工作量上限(ANALYSIS_STEPS)的 - 按遮罩列分档的标签(CASE、IIF 的结果全是写死的常量):标签文字是 agent 写在 SQL 里的,不当遮罩值,SQL、回答正文里 照原样;标签列当分组键(每一档一行)照常显示,分档写在子查询、WITH 里、外层(或中间一层)按它分组的也算 (masking._grouped_labels),不分组的逐行标签列整列遮(格子里看不出某一笔落在哪一档),可 agent diff --git a/backend/app/engine/masking.py b/backend/app/engine/masking.py index e7caa989..2e5283dc 100644 --- a/backend/app/engine/masking.py +++ b/backend/app/engine/masking.py @@ -18,14 +18,17 @@ 查不到)。整行和函数参数里的 *(Postgres 的 row_to_json(s)、SELECT s、to_json(s.*),Oracle 的 JSON_OBJECT(*))按 * 展开的来源算;表的列名表(FROM sales AS s(i, r, p))、TABLE x、套了括号的 * 先改写成认得出的样子(_normalized)。 COUNT 只数个数、窗口的分区键只决定次序、按遮罩列分档又当分组键或套在汇总里数个数的、汇总里的 CASE 只在条件里用到的 - 不算(_value_columns、_band_only)。认不出落在哪一列的(PIVOT、UNPIVOT 这类也算)、SQL 解析不了又写着遮罩列(或者 *、 - TABLE x)的,整份快照的列都算。双盲复算按同一套认法在派盲算之前挡下取自遮罩列的指标(derived_columns) + 不算(_value_columns、_band_only)。认不全的写法整份快照的列都算,不再逐个形状补规则:认不出落在哪一列的(PIVOT、 + UNPIVOT 这类也算),(表达式).* 展开读到遮罩列的(复合类型的列、函数交回的记录,展开成几列、叫什么说不清),表名、SQL + 写在字符串里交给函数、统计信息里存着值的(_INDIRECT:query_to_xml 这一族、dblink、pg_stats……),SQL 解析不了又写着 + 遮罩列(或者 *、TABLE x、_INDIRECT 那几类)的,分析出了意外、超出工作量上限(ANALYSIS_STEPS)的。一次分析里派生来源 + 对应的那块查询、一项的值取自哪些列记下来(_analysis),一长串 CTE 不再卡住。双盲复算按同一套认法在派盲算之前挡下 + 取自遮罩列的指标(derived_columns) - 只装着分档标签的列(_label_columns):标签是 agent 写在 SQL 里的字,不当遮罩值;* 展开数据源里的表时按冻结的表结构 认它带不带同名的列(_schema_tables),不知道表结构的按带得出算(多遮)。分档写在派生来源里、外层按它分组的(一档 一行,_grouped_labels)同当分组键的 CASE 一样照常显示(_shown_labels) - 哪些值(_Secrets):遮罩列里出现过的值,按值认有门槛(SECRET_MIN、SECRET_DIGITS),占位的写法(PLACEHOLDERS:未知、 - 暂无……)不收;换的时候先按词、按开头几个字筛出 - 正文里真有的那几个值,不现编上万个分支的正则;正文、文字格、SQL 的字符串和注释里换了写法的长号码(199-1474-2785、 + 暂无……)不收;换的时候先按词、按开头几个字筛出正文里真有的那几个值,不现编上万个分支的正则;正文、文字格、SQL 的字符串和注释里换了写法的长号码(199-1474-2785、 +86 …、全角数字)按纯数字写法在集合里查(loose_spans);已经摆在读者眼前的值免遮,之后并进来的照样免(protected) - 快照遮好的样子(_masked_view),回答时才认得出、事后要照位置重放的地方(view_edits、replay_edits) - 一次核查里认出的遮罩值(SeenValues):调用方交进来的和 agent 查询时遮掉的分开记,合起来时 agent 查到的在前 @@ -40,6 +43,7 @@ import functools import json import logging import re +import threading from collections.abc import Callable, Iterable, Iterator from dataclasses import dataclass, field from decimal import Decimal @@ -89,6 +93,49 @@ def _masked_word() -> str: #: _value_columns 里「值可能取自数据源里一张表的任何一列」:套在里面的子查询里的 * 展开了表,SQL 里看不出这张表有哪几列。 #: 按它认(_reads)一律算取自遮罩列(宁可多遮);真实的列名不会是这个 _ANY_COLUMN = "*" +#: 一次遮罩分析(_tainted 认遮罩列)最多走这么多步:展开一个派生来源、认一项的值取自哪些列各算一步,记下的结果再用不算。 +#: 一般的查询几百到几千步,两百段的普通 CTE 几万步;一长串 CTE 一段套一段地整行引用、展开 *(2 万字的上限里写得下几百段) +#: 以前要几十秒,超出就不再分析,整份遮(工具边界每次查询都要分析,不能为一条病态的 SQL 卡住) +ANALYSIS_STEPS = 400_000 + + +class _Overworked(Exception): + """遮罩分析超出了工作量上限(ANALYSIS_STEPS)。""" + + +class _Analysis: + """一次遮罩分析里记下的中间结果:派生来源对应的那块查询(_derived_block)、一项的值取自哪些列(_value_columns), + 按语法树节点记(分析期间树不再改写,节点都活着);走过的步数(超出 ANALYSIS_STEPS 抛 _Overworked)。""" + + def __init__(self) -> None: + self.steps = 0 + self.blocks: dict[int, Any] = {} + self.values: dict[int, set[str]] = {} + + def step(self) -> None: + self.steps += 1 + if self.steps > ANALYSIS_STEPS: + raise _Overworked() + + +#: 当前线程上正在做的那次分析(_analysis)。按线程分开:工具边界的遮罩在线程里做,几个核查 agent 同时查询时互不相干 +_CURRENT = threading.local() + + +@contextlib.contextmanager +def _analysis() -> Iterator[_Analysis]: + """在一次分析(一棵解析、改写完的语法树)里记下中间结果、数步数;出来就丢掉。里面再开一次的另起一份,出来还原: + 记下的结果按节点的 id 认,不同的树不能混用。""" + outer = getattr(_CURRENT, "state", None) + state = _CURRENT.state = _Analysis() + try: + yield state + finally: + _CURRENT.state = outer + + +def _current() -> _Analysis | None: + return getattr(_CURRENT, "state", None) def _value_columns(node: Any) -> set[str]: @@ -104,12 +151,21 @@ def _value_columns(node: Any) -> set[str]: col->'path'(JSON 箭头)被解析器当成以 col 为参数的 lambda,参数名照列名收(真的 lambda 的参数多算一列无害)。""" from sqlglot import exp + state = _current() + if state is not None and id(node) in state.values: + return state.values[id(node)] out: set[str] = set() stack = [node] # 已经展开过的派生来源:递归的 CTE 里 * 展开它自己时不再绕回去 expanded = {id(node)} while stack: item = stack.pop() + if state is not None: + state.step() + if item is not node and id(item) in state.values: + # 这一项前面认过:直接并进来(它自己展开过的不必再走一遍) + out |= state.values[id(item)] + continue if isinstance(item, exp.Star) or (isinstance(item, exp.Column) and isinstance(item.this, exp.Star)): inputs = _scoped_star_inputs(item) if inputs is None: @@ -145,6 +201,8 @@ def _value_columns(node: Any) -> set[str]: stack.extend(fresh) else: stack.extend(item.iter_expressions()) + if state is not None: + state.values[id(node)] = out return out @@ -693,6 +751,18 @@ def _from_sources(select: Any) -> list[tuple[str, Any]]: def _derived_block(source: Any) -> tuple[Any, list[str] | None] | None: """FROM、JOIN 里的一个来源 → (它的那块查询, 写的列名表):FROM 里的子查询、LATERAL、指向 WITH 里一段的表名。数据源里 的表、VALUES、函数这类为 None(出哪些列、装着什么,SQL 里看不出)。""" + state = _current() + if state is None: + return _derived_block_of(source) + if id(source) not in state.blocks: + state.step() + state.blocks[id(source)] = _derived_block_of(source) + return state.blocks[id(source)] + + +def _derived_block_of(source: Any) -> tuple[Any, list[str] | None] | None: + """_derived_block 真去找的那一步(不看记下的结果):指向 WITH 的名字要由里往外数看得见的 CTE(_visible_ctes),一长串 + CTE 时每次都数一遍很慢,所以分析里记下来。""" from sqlglot import exp from app.data.sql_semantics import _visible_ctes @@ -1053,20 +1123,26 @@ def _star_places(select: Any, width: int | None, names: set[str], tables: dict[s _ROW_DIALECTS = frozenset({"postgres", "duckdb"}) -def _normalized(root: Any, read: str | None, tables: dict[str, list[str]] | None) -> None: +def _normalized(root: Any, read: str | None, tables: dict[str, list[str]] | None) -> list[set[str]]: """几种写法先就地改写成认得出的样子,再认遮罩列、标签列(_tainted、_label_columns)。只改解析出来的语法树,原文不动; 改写前后结果有哪几列、每一列的值取自哪里都一样: - TABLE x(Postgres、MySQL):解析器认不出,CTE、子查询里的读成了一个叫 TABLE 的列、表。改成 SELECT * FROM x;带库名、 ORDER BY 的解析不了,按解析不了处理(_implicit) - - 选出项套了括号的 *((s.*))、(s).*:Postgres 照样展开成好几列,改成 s.*,不然按一列数位置。括号里不是来源的 - ((t.profile).* 展开一个复合类型的列)改成展开一个对不上任何来源的 *:说不清展开成几列,整段遮 + - 选出项套了括号的 *((s.*))、(s).*(s 是这一层 FROM 里的来源,不是哪个来源的列):Postgres 照样展开成好几列,改成 + s.*,不然按一列数位置 - 带列名表的表(Postgres 的 FROM sales AS s(i, r, p),引用 CTE 时再写列名表的也算):改成 (SELECT * FROM sales) AS s(i, r, p),按派生来源的列名表认改了名的列。Postgres 的列名表可以比列少(只改前几列的名字,后面的照原名),这时 * 展开的列数和列名表对不上,列名表里的几列整段算(_star_places),后面照原名的按名字认。不按表结构补全列名表: 别的方言里列名表比列少只能是表结构过期了,补全就丢了这一道核对 - 整行引用(read 是 _ROW_DIALECTS 之一,_row_source):row_to_json(s)、json_agg(t)、SELECT s、s::text 里的 s 改成 ROW(s.*),按 * 展开的来源认(_value_columns) - tables 是查询当时冻结的表结构(_schema_columns,按次序的列名),认整行引用时看来源出不出某一列;没有为 None。""" + - 别的 (表达式).*(复合类型的列 (p.profile).*、函数交回的记录 (jsonb_each_text(pj)).*、子查询里的 ROW (q.r).*):展开 + 成几列、各叫什么(复合类型的字段名、函数的出列名)SQL 里看不出,改成一个对不上任何来源的 *(_star_sources 为 + None),只用来数别的项的位置。表达式的值取自哪些列(_value_columns,整行引用已经改写过)记下来交回:读到遮罩列的, + 整份快照都算(_tainted)——以前只改成对不上来源的 *,最外层、第一支、派生来源里没写列名表的 * 按「照原名带出」认, + 字段名和遮罩列对不上,号码原样交了出去 + tables 是查询当时冻结的表结构(_schema_columns,按次序的列名),认整行引用时看来源出不出某一列;没有为 None。 + 交回每个 (表达式).* 的值取自哪些列。""" from sqlglot import exp for cte in list(root.find_all(exp.CTE)): @@ -1083,7 +1159,7 @@ def _normalized(root: Any, read: str | None, tables: dict[str, list[str]] | None selects = list(root.find_all(exp.Select)) for select in selects: for item in list(select.expressions): - if (star := _bare_star(item)) is not None: + if (star := _bare_star(item, select, tables)) is not None: item.replace(star) for select in selects: for _name, source in _from_sources(select): @@ -1104,6 +1180,18 @@ def _normalized(root: Any, read: str | None, tables: dict[str, list[str]] | None # 直接是选出项的起回原名:结果的列名照旧(SELECT s 的列叫 s) bare = isinstance(column.parent, exp.Select) and column.arg_key == "expressions" column.replace(exp.alias_(row, column.this.copy()) if bare else row) + opaque: list[set[str]] = [] + for select in list(root.find_all(exp.Select)): + for item in list(select.expressions): + if isinstance(item, exp.Dot) and isinstance(item.expression, exp.Star): + # 先认值取自哪些列(还挂在树上,* 和整行引用按作用域认),再换掉 + opaque.append(set(_value_columns(item.this))) + item.replace(exp.Column(this=exp.Star(), table=exp.to_identifier(_OPAQUE_STAR))) + return opaque + + +#: (表达式).* 改写成的 * 写的「表名」:对不上任何来源(_star_sources 为 None),真实的表名、别名不会是它 +_OPAQUE_STAR = "*" def _table_keyword(node: Any) -> bool: @@ -1114,8 +1202,9 @@ def _table_keyword(node: Any) -> bool: return isinstance(ident, exp.Identifier) and not ident.quoted and ident.name.upper() == "TABLE" -def _bare_star(item: Any) -> Any: - """选出项是套了括号的 *((s.*)、((*)))或 (s).* 时,改写成的 *、s.*;别的为 None(_normalized)。""" +def _bare_star(item: Any, select: Any, tables: dict[str, list[str]] | None) -> Any: + """select 的选出项是套了括号的 *((s.*)、((*))),或者 (s).*、s 是这一层 FROM 里一个来源的名字、又不是哪个来源说得准 + 出的一列(Postgres 先按列认,_has_column)时,改写成的 *、s.*;别的为 None(_normalized,别的 (表达式).* 另外处理)。""" from sqlglot import exp node = item @@ -1128,10 +1217,14 @@ def _bare_star(item: Any) -> Any: inner = item.this while isinstance(inner, exp.Paren): inner = inner.this - if isinstance(inner, exp.Column) and not inner.table and isinstance(inner.this, exp.Identifier): - return exp.Column(this=exp.Star(), table=inner.this.copy()) - # 展开的不是 FROM 里的来源(复合类型的列、函数交回的记录):记成一个对不上任何来源的 *(_star_sources 为 None) - return exp.Column(this=exp.Star(), table=exp.to_identifier("*")) + if not (isinstance(inner, exp.Column) and not inner.table and isinstance(inner.this, exp.Identifier)): + return None + name = inner.name.lower() + sources = _from_sources(select) + [((lateral.alias_or_name or "").lower(), lateral) + for lateral in select.args.get("laterals") or []] + if not any(alias == name for alias, _ in sources) or any(_has_column(s, name, tables) for _, s in sources): + return None + return exp.Column(this=exp.Star(), table=inner.this.copy()) def _carries_value(column: Any) -> bool: @@ -1149,13 +1242,13 @@ def _carries_value(column: Any) -> bool: def _row_source(column: Any, tables: dict[str, list[str]] | None) -> str | None: """Postgres 里不带表名的名字 column 指整行时,它指的来源的名字(别名,没起别名的是表名);是列、不是任何来源的为 None。 - Postgres 先按列认:由里往外看得见的每一层(_outer_selects)的来源里,有一个出得了这一列(_has_column)就是列;都不是、 - 又有一个来源叫这个名字,才是整行(取最里面的那个)。说不清出不出这一列的(表结构里查不到的表、* 展开的列)按出不了算, - 当整行(宁可多遮)。""" + Postgres 先按列认:由里往外看得见的每一层(_outer_selects)的来源里(LATERAL 里只看排在它前面的,_visible_sources), + 有一个出得了这一列(_has_column)就是列;都不是、又有一个来源叫这个名字,才是整行(取最里面的那个)。说不清出不出这一列 + 的(表结构里查不到的表、* 展开的列)按出不了算,当整行(宁可多遮)。""" name = column.name.lower() found: str | None = None for select in _outer_selects(column): - for alias, source in _from_sources(select): + for alias, source in _visible_sources(select, column): if _has_column(source, name, tables): return None if found is None and alias == name: @@ -1163,6 +1256,27 @@ def _row_source(column: Any, tables: dict[str, list[str]] | None) -> str | None: return found +def _visible_sources(select: Any, node: Any) -> list[tuple[str, Any]]: + """select 的 FROM、JOIN 里,node(select 里面的一个节点)看得见的来源(_from_sources 的样子)。node 在 LATERAL 里的,只看 + 得见排在它前面的来源:看不见 LATERAL 自己(它的结果列不是 node 能取的列),也看不见后面的。以前把 LATERAL 自己的结果列 + (SELECT s …、1 AS s)当成看得见的列,里面的整行引用 s 就被认成了列,号码原样交了出去。""" + from sqlglot import exp + + sources = _from_sources(select) + child = node + while child.parent is not None and child.parent is not select: + child = child.parent + if child.parent is not select: + return sources + if isinstance(child, exp.From): + return [] + if isinstance(child, exp.Join): + joins = list(select.args.get("joins") or []) + at = next((i for i, join in enumerate(joins) if join is child), None) + return sources[:1 + at] if at is not None else sources + return sources + + def _has_column(source: Any, name: str, tables: dict[str, list[str]] | None) -> bool: """FROM 里的这个来源说得准出 name 这一列:派生来源看列名表、第一支里起了名字的选出项(列名表改了名的不算),数据源里 的表看表结构(_table_columns),表值函数这类看它出的列名(_function_columns)。* 展开的列说不准,不算。""" @@ -1293,9 +1407,9 @@ def _tainted(sql: str, kind: str | None, masked: set[str], width: int, tables: d JSON_OBJECT(*))、整行引用(row_to_json(s))按 _value_columns 算进外层的表达式。表的列名表、TABLE x、套了括号的 * 先改写成上面这些认得出的样子(_normalized)。 PIVOT、UNPIVOT、MATCH_RECOGNIZE 读到这些名字的:值转进了它们自己起名的列(UNPIVOT 的值列、PIVOT 按 IN 里的值起名的列、 - MEASURES),出的是哪几列不好认,整份快照的列都算。""" - from sqlglot import exp - + MEASURES),出的是哪几列不好认,整份快照的列都算。(表达式).* 读到这些名字的、_INDIRECT 那几类写法同样整份算。 + 解析之后的分析出了意外、超出工作量上限(ANALYSIS_STEPS)的也整份算——不退回「解析不了」只看文字:没写遮罩列、 + 没有 * 的整行引用(SELECT s)那样就放过去了。""" from app.data.sql_semantics import _parse, _read try: @@ -1305,54 +1419,69 @@ def _tainted(sql: str, kind: str | None, masked: set[str], width: int, tables: d if root is None: return None try: - _normalized(root, _read(kind), tables) - names = set(masked) - selects = list(root.find_all(exp.Select)) - blocks = _blocks(root) - sources = [(top, listed) for top, role, listed in blocks if role == "source"] - functions = [source for select_ in selects for source in _function_sources(select_)] - unreadable = False - for _ in range(len(selects) + len(sources) + len(functions) + 1): - before = len(names) - for select_ in selects: - for item in select_.expressions: - name = item.alias_or_name.lower() - if name and not _is_star(item) and name not in names and _reads(names, _value_columns(item)): - names.add(name) - for top, listed in sources: - names |= _source_names(top, listed, names, tables)[0] - for source in functions: - if _reads(names, _value_columns(source)): - columns = _function_columns(source) - unreadable = unreadable or columns is None - names |= columns or set() - if len(names) == before: - break - everything = unreadable or any(_source_names(top, listed, names, tables)[1] for top, listed in sources) \ - or any(_reads(names, _value_columns(node)) for node in root.find_all(exp.Pivot, exp.MatchRecognize)) - positions: set[int] = set() - for top, role, _listed in blocks: - if role != "result": - continue - branches = [leaf for leaf in _leaves(top) if isinstance(leaf, exp.Select)] - for k, select_ in enumerate(branches): - branch = list(select_.expressions) - for item, place in zip(branch, _places(branch, width)): - if not _is_star(item) and _reads(names, _value_columns(item)): - if place is None or place >= width: - everything = True - else: - positions.add(place) - # UNION 后一支的 * 按位置落进第一支起的列名里;第一支、单独一条 SELECT 的 * 照原名带出,按名字认得出 - spread = _star_places(select_, width, names, tables) if k else set() - if spread is None or any(place >= width for place in spread): - everything = True - else: - positions |= spread - return names, set(range(width)) if everything else positions, everything - except Exception: # noqa: BLE001 - 分析本身的缺陷不能让回答出错:按解析不了处理(宁可多遮) - logger.warning("质询的 SQL 遮罩分析失败", exc_info=True) - return None + opaque = _normalized(root, _read(kind), tables) + with _analysis(): + return _tainted_in(root, sql, masked, width, tables, opaque) + except _Overworked: + # 一长串 CTE 一段套一段地整行引用、展开 *:分析不过来,认不准,整份遮 + logger.warning("SQL 遮罩分析超出工作量上限(%d 步),整份快照按遮罩处理", ANALYSIS_STEPS) + except Exception: # noqa: BLE001 - 分析本身的缺陷不能让回答出错,也不能把原值放过去:整份遮(宁可多遮) + logger.warning("SQL 遮罩分析失败,整份快照按遮罩处理", exc_info=True) + return set(masked), set(range(width)), True + + +def _tainted_in(root: Any, sql: str, masked: set[str], width: int, tables: dict[str, list[str]] | None, + opaque: list[set[str]]) -> tuple[set[str], set[int], bool]: + """_tainted 解析、改写(_normalized)之后真去认的那一步。opaque 是 (表达式).* 的值取自哪些列(_normalized 交回的)。""" + from sqlglot import exp + + names = set(masked) + selects = list(root.find_all(exp.Select)) + blocks = _blocks(root) + sources = [(top, listed) for top, role, listed in blocks if role == "source"] + functions = [source for select_ in selects for source in _function_sources(select_)] + unreadable = False + for _ in range(len(selects) + len(sources) + len(functions) + 1): + before = len(names) + for select_ in selects: + for item in select_.expressions: + name = item.alias_or_name.lower() + if name and not _is_star(item) and name not in names and _reads(names, _value_columns(item)): + names.add(name) + for top, listed in sources: + names |= _source_names(top, listed, names, tables)[0] + for source in functions: + if _reads(names, _value_columns(source)): + columns = _function_columns(source) + unreadable = unreadable or columns is None + names |= columns or set() + if len(names) == before: + break + # 认不出的整份遮:表值函数出哪几列说不清、派生来源里有认不出名字的列、PIVOT 这类读到遮罩列,(表达式).* 读到遮罩列 + # (展开成几列、叫什么都说不清),SQL 写在字符串里交给函数执行、整表导出、统计信息里的值(_INDIRECT) + everything = unreadable or any(_source_names(top, listed, names, tables)[1] for top, listed in sources) \ + or any(_reads(names, _value_columns(node)) for node in root.find_all(exp.Pivot, exp.MatchRecognize)) \ + or any(_reads(names, cols) for cols in opaque) or bool(_INDIRECT.search(sql)) + positions: set[int] = set() + for top, role, _listed in blocks: + if role != "result": + continue + branches = [leaf for leaf in _leaves(top) if isinstance(leaf, exp.Select)] + for k, select_ in enumerate(branches): + branch = list(select_.expressions) + for item, place in zip(branch, _places(branch, width)): + if not _is_star(item) and _reads(names, _value_columns(item)): + if place is None or place >= width: + everything = True + else: + positions.add(place) + # UNION 后一支的 * 按位置落进第一支起的列名里;第一支、单独一条 SELECT 的 * 照原名带出,按名字认得出 + spread = _star_places(select_, width, names, tables) if k else set() + if spread is None or any(place >= width for place in spread): + everything = True + else: + positions |= spread + return names, set(range(width)) if everything else positions, everything def _mentions(sql: str, names: Iterable[str]) -> bool: @@ -1366,10 +1495,27 @@ _COUNT_STAR = re.compile(r"\bcount\s*\(\s*(?:all\s+|distinct\s+)?\*\s*\)", re.I) _IMPLICIT = re.compile(r"(?:\bselect|\bdistinct|\ball|[,(.])\s*\*|\btable\s+[\w\"`]", re.I) +#: 不点列名也读得出表里内容的写法,SQL 里认不出读的是哪一列:表名、SQL 写在字符串里交给函数执行或整表导出(Postgres 的 +#: query_to_xml、table_to_xml 这一族、ts_stat、tablefunc 的 crosstab / connectby、dblink,Oracle 的 DBMS_XMLGEN、XMLTYPE、 +#: XQuery,DuckDB 的 query / query_table,SQL Server 的 OPENQUERY、OPENROWSET),按页、按文件读出的原始数据,统计信息、 +#: 会话记录里存着的值(pg_stats 的常见值、sqlite_stat4 的样本、pg_stat_statements 记下的 SQL 原文)。出现就整份遮 +#: (_tainted、_implicit),写在字符串、注释里的也算(多遮)。函数要跟着左括号才算(列名碰巧叫 query 的不算)。 +#: information_schema、pg_catalog 只有表结构,不在里面 +_INDIRECT = re.compile(r"(? bool: - """解析不了的 SQL 里有没有不写列名也会带出遮罩列的写法(*、TABLE x):它们能把遮罩列带进别的列名(列名表、UNION 的 - 后一支),同写着遮罩列一样整份遮(_derived_hidden)。只看文字,字符串、注释里的也算(多遮)。""" - return bool(_IMPLICIT.search(_COUNT_STAR.sub("", sql))) + """解析不了的 SQL 里有没有不写列名也会带出遮罩列的写法(*、TABLE x,_INDIRECT 那几类):它们能把遮罩列带进别的列名 + (列名表、UNION 的后一支)、整表交出来,同写着遮罩列一样整份遮(_derived_hidden)。只看文字,字符串、注释里的也算 + (多遮)。""" + return bool(_IMPLICIT.search(_COUNT_STAR.sub("", sql)) or _INDIRECT.search(sql)) def _derived_hidden(snapshot: dict[str, Any], hidden: set[str], kind: str | None, diff --git a/backend/tests/test_masking_opaque_sql.py b/backend/tests/test_masking_opaque_sql.py new file mode 100644 index 00000000..5e58f16a --- /dev/null +++ b/backend/tests/test_masking_opaque_sql.py @@ -0,0 +1,282 @@ +"""认不全的 SQL 写法整份遮(engine/masking.py 的 _tainted),不再逐个形状补规则(K3 第 3 轮复验留下的几处): + +- (表达式).* 展开:复合类型的列 (p.profile).*、函数交回的记录 (jsonb_each_text(pj)).*、(jsonb_populate_record(…)).*、 + 子查询里的匿名 ROW (q.r).*。展开成几列、各叫什么 SQL 里看不出,表达式读到遮罩列的,整份结果都遮——最外层、子查询、 + CTE、LATERAL 里都一样。以前改写成一个对不上来源的 *,最外层、没写列名表的派生来源里按「照原名带出」认,字段名 + (tel、key/value、f1/f2)和遮罩列对不上,号码原样交给模型、落库。(s).*、s 是 FROM 里的来源时照旧按 s.* 认 +- 表名、SQL 写在字符串里交给函数执行、整表导出的(query_to_xml、table_to_xml、cursor_to_xml、schema_to_xml、 + database_to_xml、dblink,Oracle 的 DBMS_XMLGEN),统计信息里存着的值(pg_stats):整份遮,解析不了的也一样 +- LATERAL 看不见它自己:里面的整行引用 s 不因为 LATERAL 自己的结果列叫 s 就被当成列 +- 分析不过来、分析出了意外:整份遮。派生来源对应的那块查询、一项的值取自哪些列在一次分析里记下来,一长串 CTE 一段套 + 一段地整行引用(250 段,2 万字的上限里写得下)以前要二十几秒,现在零点几秒;超出工作量上限(ANALYSIS_STEPS)的整份遮 + +这里没有真的 Postgres:结果按它交回的样子造(复合类型、记录展开成几列)。数据都是虚构的:手机号、地址、区域。 +""" +from __future__ import annotations + +import json +import time +import uuid + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.core import artifact_store +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource +from app.engine import masking +from app.main import app +from tests.test_verify_agent import Script, q +from tests.test_verify_agent_masking import ask, clean_spend # noqa: F401 - clean_spend 是自动用上的夹具 + +MASKED_WORD = "已遮罩" +#: people.profile 是复合类型(tel、addr),people2.pj 是 JSON(tel、addr),sales 是订单:设了遮罩的是 profile、pj、buyer_phone +SCHEMA = {"tables": { + "people": {"qualified": "people", "schema": None, "columns": [{"name": c} for c in ["id", "profile", "address"]]}, + "people2": {"qualified": "people2", "schema": None, "columns": [{"name": c} for c in ["id", "pj", "city"]]}, + "sales": {"qualified": "sales", "schema": None, + "columns": [{"name": c} for c in ["id", "region", "buyer_phone", "amount"]]}, +}} +MASKED = ["profile", "pj", "buyer_phone"] +PHONE = "13900000001" + + +def _leaks(done: masking.QueryMask) -> bool: + return "1390000000" in json.dumps(done.payload, ensure_ascii=False) + + +def masked(sql: str, columns: list[str], rows: list[list], kind: str = "postgresql") -> masking.QueryMask: + payload = {"columns": columns, "rows": rows, "sql": sql} + return masking.mask_query_result(payload, MASKED, kind, schema=SCHEMA) + + +# -------------------------------------------------------------------------- +# (表达式).* 展开 +# -------------------------------------------------------------------------- + + +#: 三种展开(复合类型的列、函数交回的记录、子查询里的匿名 ROW)× 四个位置(最外层、子查询、CTE、LATERAL): +#: (SQL, 结果列, 一行) +EXPANSIONS = [ + pytest.param("SELECT p.id AS n, (p.profile).* FROM people p", ["n", "tel", "addr"], [1, PHONE, "某路 1 号"], + id="复合类型-最外层"), + pytest.param("SELECT x.* FROM (SELECT p.id AS n, (p.profile).* FROM people p) x", ["n", "tel", "addr"], + [1, PHONE, "某路 1 号"], id="复合类型-子查询"), + pytest.param("WITH x AS (SELECT p.id AS n, (p.profile).* FROM people p) SELECT * FROM x", ["n", "tel", "addr"], + [1, PHONE, "某路 1 号"], id="复合类型-CTE"), + pytest.param("SELECT x.* FROM people p, LATERAL (SELECT p.id AS n, (p.profile).*) x", ["n", "tel", "addr"], + [1, PHONE, "某路 1 号"], id="复合类型-LATERAL"), + pytest.param("SELECT (jsonb_each_text(pj)).* FROM people2", ["key", "value"], ["tel", PHONE], + id="记录函数-最外层"), + pytest.param("SELECT kv.value AS tel FROM (SELECT (jsonb_each_text(pj)).* FROM people2) kv WHERE kv.key = 'tel'", + ["tel"], [PHONE], id="记录函数-子查询"), + pytest.param("WITH kv AS (SELECT (jsonb_each_text(pj)).* FROM people2) SELECT value AS 号码 FROM kv " + "WHERE key = 'tel'", ["号码"], [PHONE], id="记录函数-CTE"), + pytest.param("SELECT x.value FROM people2 p, LATERAL (SELECT (jsonb_each_text(p.pj)).*) x", ["value"], [PHONE], + id="记录函数-LATERAL"), + pytest.param("SELECT (q.r).* FROM (SELECT ROW(buyer_phone, region) AS r FROM sales) q", ["f1", "f2"], + [PHONE, "华东"], id="匿名 ROW-最外层"), + pytest.param("SELECT y.* FROM (SELECT (q.r).* FROM (SELECT ROW(buyer_phone, region) AS r FROM sales) q) y", + ["f1", "f2"], [PHONE, "华东"], id="匿名 ROW-子查询"), + pytest.param("WITH q AS (SELECT ROW(buyer_phone, region) AS r FROM sales) SELECT (q.r).* FROM q", ["f1", "f2"], + [PHONE, "华东"], id="匿名 ROW-CTE"), + pytest.param("SELECT x.* FROM (SELECT ROW(buyer_phone, region) AS r FROM sales) q, LATERAL (SELECT (q.r).*) x", + ["f1", "f2"], [PHONE, "华东"], id="匿名 ROW-LATERAL"), + pytest.param("SELECT p.id AS n, (jsonb_populate_record(NULL::contact, p.pj)).* FROM people2 p", + ["n", "tel", "addr"], [1, PHONE, "某路 1 号"], id="按类型展开 JSON"), +] + + +@pytest.mark.parametrize("sql, columns, row", EXPANSIONS) +def test_an_expression_star_reading_a_masked_column_masks_the_whole_result(sql, columns, row): + """(表达式).* 读到遮罩列:展开成几列、叫什么说不清,整份结果的每一列都遮(同一份里的编号也遮,多遮)。""" + done = masked(sql, columns, [list(row)]) + assert done.columns == columns + assert done.payload["rows"] == [[MASKED_WORD] * len(columns)] and not _leaks(done) + + +@pytest.mark.parametrize("sql, columns, row", [ + pytest.param("SELECT p.id AS n, (p.address).* FROM people p", ["n", "street", "city"], [1, "某路", "某市"], + id="没设遮罩的复合类型"), + pytest.param("SELECT (jsonb_each_text(to_jsonb(c))).* FROM (SELECT id, region FROM sales) c", ["key", "value"], + ["region", "华东"], id="记录里没有遮罩列"), +]) +def test_an_expression_star_that_reads_no_masked_column_stays_visible(sql, columns, row): + done = masked(sql, columns, [list(row)]) + assert done.columns == [] and done.payload["rows"] == [list(row)] + + +def test_a_parenthesised_source_star_is_still_read_as_its_columns(): + """(s).*、s 是 FROM 里的来源:照旧当 s.* 认,只遮号码那一列,区域、金额照常。""" + done = masked("SELECT (s).* FROM sales s", ["id", "region", "buyer_phone", "amount"], [[1, "华东", PHONE, 1200]]) + assert done.columns == ["buyer_phone"] and done.payload["rows"] == [[1, "华东", MASKED_WORD, 1200]] + + +#: 造出来的 Postgres 数据源:表结构同上,查询交给换掉的 run_query(不连库) +@pytest.fixture +async def composite_shop(): + made: list[int] = [] + + async def make() -> str: + name = f"pg_{uuid.uuid4().hex[:8]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="postgresql", host="127.0.0.1", port=5432, username="probe", + database="shop", readonly=True, enabled=True, + options={"mask_columns": ", ".join(MASKED)}, schema_cache=SCHEMA) + session.add(row) + await session.commit() + made.append(row.id) + return name + + yield make + for source_id in made: + await engines.invalidate(source_id) + + +@pytest.mark.parametrize("sql, columns, rows", [ + ("SELECT p.id AS n, (p.profile).* FROM people p", ["n", "tel", "addr"], [[1, PHONE, "某路 1 号"]]), + ("SELECT x.* FROM (SELECT p.id AS n, (p.profile).* FROM people p) x", ["n", "tel", "addr"], + [[1, PHONE, "某路 1 号"]]), + ("SELECT p.id AS n, (jsonb_populate_record(NULL::contact, p.pj)).* FROM people2 p", ["n", "tel", "addr"], + [[1, PHONE, "某路 1 号"]]), +], ids=["复合类型", "子查询里的复合类型", "按类型展开 JSON"]) +async def test_an_expression_star_never_reaches_the_model_or_the_store(monkeypatch, composite_shop, sql, columns, + rows): + """端到端(核查 agent 的真实查询工具):交给模型的、落库的、通用工件接口给的都只有「已遮罩」,masked_at_query 是 + 全部的列。以前三条的 masked_at_query 都是空的,号码原样交给模型、落库。""" + from app.data.engine import QueryResult + from app.tools import datasource + + async def driver(source, statement, **kwargs): + assert statement == sql + return QueryResult(columns=list(columns), rows=[list(r) for r in rows], row_count=len(rows), truncated=False, + elapsed_ms=1, sql=statement) + + monkeypatch.setattr(datasource, "run_query", driver) + name = await composite_shop() + script = Script(monkeypatch, [q(name, sql, "c1")], {"value": {"alias": "Q1", "row": 0, "column": "n"}, + "rationale": "x"}) + out, _ = await ask(name) + seen = script.tool_results()["c1"] + assert MASKED_WORD in seen and "1390000000" not in seen, seen + snapshot = artifact_store.load(out.queries[0]["artifact"]) + assert snapshot["masked_at_query"] == columns and "1390000000" not in json.dumps(snapshot, ensure_ascii=False) + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + got = await client.get(f"/api/artifacts/{out.queries[0]['artifact']}") + assert got.status_code == 200 and "1390000000" not in got.text + + +# -------------------------------------------------------------------------- +# LATERAL 看不见它自己 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("sql, columns, row", [ + pytest.param("SELECT y.* FROM sales t2, LATERAL (SELECT s FROM sales s LIMIT 1) y", ["s"], + [f"(1,华东,{PHONE},1200)"], id="选出整行"), + pytest.param("SELECT y.j FROM sales t2 CROSS JOIN LATERAL (SELECT 1 AS s, row_to_json(s) AS j FROM sales s " + "LIMIT 1) y", ["j"], [json.dumps({"id": 1, "buyer_phone": PHONE})], id="结果列也叫 s"), + pytest.param("SELECT y.j FROM sales t2, LATERAL (SELECT s.id AS s, row_to_json(s) AS j FROM sales s LIMIT 1) y", + ["j"], [json.dumps({"id": 1, "buyer_phone": PHONE})], id="编号起名叫 s"), +]) +def test_a_whole_row_inside_a_lateral_is_not_taken_for_its_own_column(sql, columns, row): + """LATERAL 里的 s 是它自己 FROM 里的整行:LATERAL 的结果列碰巧叫 s,也不把它当成列(Postgres 里 LATERAL 看不见 + 自己)。以前当成了列,整行原样交了出去。""" + done = masked(sql, columns, [list(row)]) + assert done.columns == columns and not _leaks(done) + + +def test_a_lateral_still_sees_the_sources_before_it(): + """排在 LATERAL 前面的来源照样看得见:前面的表有一列叫 s 时,s 是那一列(没设遮罩),不是整行。""" + schema = {"tables": {**SCHEMA["tables"], "codes": {"qualified": "codes", "schema": None, + "columns": [{"name": "id"}, {"name": "s"}]}}} + payload = {"columns": ["v"], "rows": [["A1"]], + "sql": "SELECT y.v FROM codes c, LATERAL (SELECT s AS v FROM sales x WHERE x.id = c.id) y"} + done = masking.mask_query_result(payload, MASKED, "postgresql", schema=schema) + assert done.columns == [] and done.payload["rows"] == [["A1"]] + + +# -------------------------------------------------------------------------- +# 表名、SQL 写在字符串里交给函数,统计信息里的值 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("sql", [ + "SELECT query_to_xml('select * from sales', true, false, '') AS x", + "SELECT table_to_xml('sales', true, false, '') AS x", + "SELECT query_to_xml_and_xmlschema('select buyer_phone from sales', true, false, '') AS x", + "SELECT cursor_to_xml('c1', 10, false, true, '') AS x", + "SELECT schema_to_xml('public', true, false, '') AS x", + "SELECT database_to_xml(true, false, '') AS x", + "SELECT x FROM dblink('dbname=shop', 'select buyer_phone from sales') AS t(x text)", + "SELECT DBMS_XMLGEN.GETXML('select * from sales') AS x FROM dual", + "SELECT array_to_string(most_common_vals, ',') AS x FROM pg_stats WHERE tablename = 'sales'", +]) +def test_tables_and_sql_handed_to_functions_as_text_mask_the_whole_result(sql): + """表名、SQL 写在字符串里交给函数执行、整表导出,统计信息里存着的值:读的是哪一列认不出,整份遮。以前 masked=[], + 号码原样。""" + done = masked(sql, ["x"], [[f"{PHONE}"]], kind="oracle" + if "DBMS" in sql else "postgresql") + assert done.columns == ["x"] and not _leaks(done) + + +def test_a_table_handed_to_a_function_masks_the_whole_result_even_when_the_sql_cannot_be_read(): + sql = "SELECT table_to_xml('sales', true, false, '') AS x WHERE (" + hidden, _ = masking._derived_hidden({"sql": sql, "columns": ["x"], "rows": []}, {"buyer_phone"}, "postgresql") + assert "x" in hidden + + +# -------------------------------------------------------------------------- +# 分析不过来、分析出了意外:整份遮;一长串 CTE 不再卡住 +# -------------------------------------------------------------------------- + + +def _chain(k: int, tail: str) -> str: + """一长串 CTE,一段套一段地整行引用、展开上一段(复验时 250 段要二十几秒)。""" + parts = ["c0 AS (SELECT * FROM sales)"] + parts += [f"c{i} AS (SELECT row_to_json(c{i - 1}) AS j, c{i - 1}.* FROM c{i - 1})" for i in range(1, k)] + return "WITH " + ", ".join(parts) + f" {tail.format(last=f'c{k - 1}')}" + + +def _plain_chain(k: int) -> str: + parts = ["c0 AS (SELECT id, region, amount FROM sales)"] + parts += [f"c{i} AS (SELECT id, region, amount + {i} AS amount FROM c{i - 1})" for i in range(1, k)] + return "WITH " + ", ".join(parts) + f" SELECT region, SUM(amount) AS total FROM c{k - 1} GROUP BY region" + + +def test_a_long_chain_of_whole_rows_is_read_quickly_and_completely(): + """250 段一段套一段地整行引用:零点几秒认完(以前二十几秒),不靠工作量上限兜底——只数个数的结果照常不遮。""" + columns = masking._schema_columns(SCHEMA) + sql = _chain(250, "SELECT COUNT(*) AS n FROM {last}") + assert len(sql) < 20_000 + started = time.perf_counter() + hidden, _ = masking._derived_hidden({"sql": sql, "columns": ["n"], "rows": []}, {"buyer_phone"}, "postgresql", + columns) + assert time.perf_counter() - started < 5 + assert "n" not in hidden + hidden, _ = masking._derived_hidden({"sql": _chain(250, "SELECT j FROM {last}"), "columns": ["j"], "rows": []}, + {"buyer_phone"}, "postgresql", columns) + assert "j" in hidden + + +def test_an_analysis_over_the_work_limit_masks_the_whole_result(monkeypatch): + """超出工作量上限:不再分析,整份遮(结果本来没碰遮罩列也一样)。没超的照常。""" + sql = _plain_chain(40) + snapshot = {"sql": sql, "columns": ["region", "total"], "rows": []} + assert masking._derived_hidden(snapshot, {"buyer_phone"}, "postgresql")[0] == {"buyer_phone"} + monkeypatch.setattr(masking, "ANALYSIS_STEPS", 50) + assert masking._derived_hidden(snapshot, {"buyer_phone"}, "postgresql")[0] == {"buyer_phone", "region", "total"} + payload = {"sql": sql, "columns": ["region", "total"], "rows": [["华东", 1200]]} + done = masking.mask_query_result(payload, ["buyer_phone"], "postgresql") + assert done.columns == ["region", "total"] and done.payload["rows"] == [[MASKED_WORD, MASKED_WORD]] + + +def test_a_failing_analysis_masks_the_whole_result_instead_of_reading_the_text(monkeypatch): + """解析之后的分析出了意外:整份遮。以前按「解析不了」只看文字,没写遮罩列、没有 * 的整行引用(SELECT s)就放过去了。""" + def broken(*args, **kwargs): + raise RuntimeError("分析里的意外") + + monkeypatch.setattr(masking, "_tainted_in", broken) + snapshot = {"sql": "SELECT s AS r FROM sales s", "columns": ["r"], "rows": []} + assert "r" in masking._derived_hidden(snapshot, {"buyer_phone"}, "postgresql")[0] From 888f746d31225174ee7ca892c3eeb121ee2b362f Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 23:11:22 +0800 Subject: [PATCH 05/12] =?UTF-8?q?K3b=EF=BC=9A=E4=BA=A4=E7=BB=99=E6=A0=B8?= =?UTF-8?q?=E6=9F=A5=20agent=20=E7=9A=84=E8=AF=81=E6=8D=AE=E6=91=98?= =?UTF-8?q?=E5=BD=95=E5=85=88=E9=81=AE=EF=BC=9B=E5=8F=8D=E6=96=B9=E5=AE=A1?= =?UTF-8?q?=E7=A8=BF=E7=9A=84=E7=90=86=E7=94=B1=E8=BF=87=E6=9C=80=E5=90=8E?= =?UTF-8?q?=E4=B8=80=E9=81=93=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 摘录由 judge.prepare 生成,以前 _hidden 只按列名遮:报告取数把遮罩列改了名字选出来(buyer_phone AS 手机号),质询、反方 agent 的任务里四个号码都在,模型照抄进理由。现在 prepare 收 kinds({数据源名: 类型},只有核查 agent 的摘录给),查询摘录、指标输入按 masking._derived_hidden 认出改了名字的遮罩列(和工具边界同一套规则,表结构按快照冻结的那份),认的过程出了意外整份的列都不给;结论句裁判自己的摘录不给 kinds,和以前逐字相同、复用键不变。反方审稿、审批质询再把摘录里别的列、SQL 里写着的遮罩值按值换成「已遮罩」(换了写法的号码也认),值同交给查询工具的那一份(报告快照里的,反方去掉这句结论原文里写着的);解析 SQL、按值换放进线程。 - 反方审稿的理由按值遮之后,正文、理由文档(正文、原文、目录里的列名)里还认得出遮罩值的,这句记为无法判断、理由换成系统说明、不存理由文档(_kept,同审批质询的 _leaked),遮不住的「被推翻」不再让出具降档。 补测试:质询、反方各一条端到端(任务里的摘录没有号码,手机号列不给,SQL 里的号码、备注里换了写法的号码换掉),反方理由里 agent 自己查出的号码换着写法也换掉,最后一道关;分别去掉按 SQL 认列、两处按值换、agent 查到的值、最后一道关,对应的测试都变红。 --- backend/app/engine/inquiry.py | 35 ++++--- backend/app/engine/judge.py | 52 +++++++--- backend/app/engine/nodes/challenge.py | 82 ++++++++++++--- backend/tests/test_challenge_masking.py | 120 ++++++++++++++++++++++ backend/tests/test_inquiry_masking_e2e.py | 29 +++++- 5 files changed, 276 insertions(+), 42 deletions(-) create mode 100644 backend/tests/test_challenge_masking.py diff --git a/backend/app/engine/inquiry.py b/backend/app/engine/inquiry.py index f9796a0e..1556e6f5 100644 --- a/backend/app/engine/inquiry.py +++ b/backend/app/engine/inquiry.py @@ -7,8 +7,9 @@ 运行已经结束封存、核对一致、是探索运行——问答追加在封存之后(post_seal),是批注不是证据。正式运行封存之后不再追加 (同按需裁判);设置里关了「审批质询 · 启用」一律不答。同一次运行同时只答一条:进程内登记(_ANSWERING),登记期间 这次运行的审批回复、继续运行也不受理(api/runs.py),免得运行往下走了、问答还在路上 -2. 回答(_answer):交给 agent 被问的那句原文、它挂的证据摘录(照 judge.prepare 的办法取,数据源遮罩的列照旧不给,原报告 - 的编号标明「原报告」)、审批人选中的原文和问题;工具是这次运行用过的查询工具对应的数据源(同名重建的、没固定过版本的 +2. 回答(_answer):交给 agent 被问的那句原文、它挂的证据摘录(照 judge.prepare 的办法取,原报告的编号标明「原报告」; + 摘录先遮:遮罩列不给,改了名字的遮罩列按 SQL 认出来也不给,别的列、SQL 里写着的遮罩值按值换掉,见 _excerpts)、 + 审批人选中的原文和问题;工具是这次运行用过的查询工具对应的数据源(同名重建的、没固定过版本的 上传表格不算,见 _usable_sources);上限按设置(每次的查询次数、时长,全局每日金额)。回答先给结论,数字只能引用 agent 自己查询的格子,或者报告里已有的指标、运行输入;回答文档由 compose_doc 组装,裸数字被标出来(提问人自己写的数除外)。 封存之后的追问查的是数据源现在的数据,任务里给报告生成的时间,回答末尾注明;触顶、查询失败、引用解析不了也在末尾注明 @@ -125,9 +126,9 @@ 点、括号,含全角),带 +86 / 0086 / 86 前缀、全角数字:写成汉字(一三九……)、中间夹着别的字、拆成几处写的、跨行 写的认不出;和别的数字只隔着一个连字符或点连着的(13900000001-2、19914.74)不算号码;SQL 里不带引号的数不这样认 (199-1474-2785 在 SQL 里是三个数相减),字符串、带引号的名字、注释里照样认 -- 质询 agent 的任务里的证据摘录照 judge.prepare 按列名遮:报告取数把遮罩列改了名字选出来(buyer_phone AS 手机号)时, - 摘录里有原值,agent 看得到;它写进回答、SQL 的,按值认得出的照样换掉(报告快照里的值都收了),封存的是换过的。摘录 - 按 SQL 认改了名字的遮罩列要改 judge.py,另行处理 +- 质询 agent 的任务里的证据摘录先遮(_excerpts):遮罩列、按 SQL 认出改了名字的遮罩列(和工具边界同一套规则)不给, + 别的列、SQL 里写着的遮罩值按值换(报告快照里的值),按值认的门槛同上:短的数(年龄、等级)写在 SQL 的条件里、别的列 + 里照原样。没指明是哪一句时任务里给的是报告正文,正文里写着的值照原样(读者本来就看得到) """ from __future__ import annotations @@ -582,9 +583,13 @@ def _table_text(block: dict[str, Any], markdown: str) -> str: def _excerpts(doc: dict[str, Any], catalog: dict[str, Any], cand: judging.Candidate, loader: Callable[[str], Any], - masked: dict[str, Any]) -> list[str]: - """这句挂的每件证据的摘录:和 judge.prepare 一样逐件摘(数据源遮罩的列不给,带期 4 标记的文档多来历行), - 指向原报告编号的写法标上「原报告」——agent 自己的查询也编 Q1、Q2…,不分开的话会把它们混成一回事。""" + masked: dict[str, Any], kinds: dict[str, Any] | None = None, secrets: _Secrets | None = None + ) -> list[str]: + """这句挂的每件证据的摘录:和 judge.prepare 一样逐件摘(带期 4 标记的文档多来历行),指向原报告编号的写法标上 + 「原报告」——agent 自己的查询也编 Q1、Q2…,不分开的话会把它们混成一回事。 + 摘录先遮,模型从任务里也看不到遮罩值:数据源遮罩的列不给,改了名字的(buyer_phone AS 手机号)按 SQL 认出来也不给 + (kinds:{数据源名: 类型},和工具边界同一套规则,judge._derived);别的列、SQL 里写着的遮罩值按值换成「已遮罩」, + 换了写法的号码也认(secrets:报告快照里的遮罩值,同交给查询工具的那一份)。解析 SQL、按值换是纯计算,调用方放进线程。""" fetch = judging._Fetch(loader) provenance_on = doc.get("provenance") == DOC_PROVENANCE queries = {a for a, e in catalog.items() if isinstance(e, dict) and e.get("kind") == "query"} @@ -592,7 +597,9 @@ def _excerpts(doc: dict[str, Any], catalog: dict[str, Any], cand: judging.Candid for alias in cand.aliases: entry = catalog.get(alias) if isinstance(entry, dict) and (text := judging._excerpt(alias, entry, [cand], fetch, masked, catalog, - provenance_on=provenance_on)): + provenance_on=provenance_on, kinds=kinds)): + if secrets is not None and secrets.strong: + text = secrets.scrub(text, loose=True) out.append(mark_original(text, alias, queries)) return out @@ -1811,7 +1818,6 @@ async def _answer(op: _Opening, ask: Ask, target: _Target | None, settings: dict doc = op.doc catalog = doc.get("catalog") if isinstance(doc.get("catalog"), dict) else {} masked = await judging.source_masks(catalog, loader=op.loader) - excerpts = _excerpts(doc, catalog, target.cand, op.loader, masked) if target is not None else [] refs = _report_refs(catalog) wanted = _run_sources(op.events) usable = await _usable_sources(op.run_id, wanted, op.versions) @@ -1820,15 +1826,20 @@ async def _answer(op: _Opening, ask: Ask, target: _Target | None, settings: dict async with SessionLocal() as session: spec = await verify_model_spec(session, None, settings=settings) # 报告自己的查询快照:被问的那句可能直接写着遮罩列的值,agent 会照着复述;里面的遮罩值在派 agent 之前就交给它的 - # 查询工具(masked_values),agent 查到写着同一个号码的备注、别的表时,交给模型之前就换掉 + # 查询工具(masked_values),agent 查到写着同一个号码的备注、别的表时,交给模型之前就换掉;证据摘录也按它们先遮 given = [ask.question, ask.quote, *_ref_values(refs)] others = [snapshot for entry in catalog.values() if isinstance(entry, dict) and entry.get("kind") == "query" and _query_shaped(snapshot := _quiet_load(op.loader, entry.get("artifact")))] + seed = await snapshot_secrets(others, given) + excerpts: list[str] = [] + if target is not None: + kinds = await _source_kinds([*masked, *(str(s.get("source") or "") for s in others)]) + excerpts = await asyncio.to_thread(_excerpts, doc, catalog, target.cand, op.loader, masked, kinds, seed) outcome = await run_verify_agent( spec=spec, system=INQUIRY_SYSTEM, task=_task(op, ask, target, excerpts, refs, live, lost), sources=list(live), output_schema=INQUIRY_SCHEMA, max_queries=part.get("max_queries"), timeout_s=part.get("timeout_s"), daily_max_usd=settings.get("daily_max_usd"), run_id=op.run_id, data_versions=op.versions, - masked_values=await snapshot_secrets(others, given)) + masked_values=seed) # 遮罩:agent 自己的查询快照(查询时就遮好了,遮掉的值在 outcome.masked_values 里)、报告自己的查询快照 own: dict[str, dict[str, Any]] = {} diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index b5808583..f4f17ef2 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -342,6 +342,25 @@ def _hidden(snapshot: dict[str, Any], source: Any, masked: dict[str, Any] | None return names +def _derived(snapshot: dict[str, Any], hidden: set[str], fetch: _Fetch, kinds: dict[str, Any] | None) -> set[str]: + """核查 agent 的摘录(kinds 给了:{数据源名: 类型})要遮的列:按列名的(hidden),加上按 SQL 认出改了名字的遮罩列 + (buyer_phone AS 手机号、套函数、经子查询转一道……),和核查 agent 的查询在工具边界遮的是同一套规则 + (masking._derived_hidden,表结构按快照冻结的那份、经 fetch 取)。认的过程出了意外,整份快照的列都算(宁可多遮)。 + kinds 为 None(结论句裁判自己的摘录)照旧只按列名,摘录、复用键都和以前一样。""" + if kinds is None or not hidden: + return hidden + from app.engine.masking import _derived_hidden, _schema_columns # masking 导入了别的引擎模块:用到时再导入 + + try: + schema = fetch(snapshot.get("schema_artifact")) if snapshot.get("schema_artifact") else None + tables = _schema_columns(schema) if isinstance(schema, dict) else None + kind = kinds.get(str(snapshot.get("source") or "")) + return set(hidden) | _derived_hidden(dict(snapshot), set(hidden), kind, tables)[0] + except Exception: # noqa: BLE001 - 认不准就整份遮:摘录少几列,不让遮罩值交给核查 agent + logger.warning("judge: 按 SQL 认摘录里改了名字的遮罩列时出错,整份快照的列都按遮罩处理", exc_info=True) + return set(hidden) | {str(c).lower() for c in snapshot.get("columns") or []} + + def _shown_columns(columns: list[str], cited: Iterable[str]) -> list[str]: """摘录给哪几列:被引用的列一律给(超过上限也给),再按原来的顺序用别的列补满到 EXCERPT_COLS 列。 @@ -354,11 +373,13 @@ def _shown_columns(columns: list[str], cited: Iterable[str]) -> list[str]: def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iterable[str], whole: bool, - fetch: _Fetch, masked: dict[str, Any] | None, *, provenance_on: bool = False) -> str | None: + fetch: _Fetch, masked: dict[str, Any] | None, *, provenance_on: bool = False, + kinds: dict[str, Any] | None = None) -> str | None: """一条查询的摘录:头、SQL、(期 4)来历行、列、被引用的行。 provenance_on:文档带期 4 标记时为真(prepare 算)。这时上传表格的查询快照(有 data_version)在 SQL 之后、「列:」 之前多几行来历(_provenance_lines);其余情况一个字都不多,期 4 之前的文档、手工源的摘录和以前逐字相同。 + kinds:给核查 agent 的摘录才给,改了名字的遮罩列也不给(_derived)。 """ snapshot = fetch(entry.get("artifact")) if not isinstance(snapshot, dict) or not isinstance(snapshot.get("rows"), list): @@ -372,7 +393,7 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter lines = [head, f"SQL:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"] if snapshot.get("source") == MERGE_SOURCE: lines.append(_merge_line(snapshot)) - hidden = _hidden(snapshot, source, masked) + hidden = _derived(snapshot, _hidden(snapshot, source, masked), fetch, kinds) if provenance_on and snapshot.get("data_version"): lines += _provenance_lines(snapshot, fetch, hidden) columns = [str(c) for c in snapshot.get("columns") or [] if str(c).lower() not in hidden] @@ -483,7 +504,7 @@ def _cell_words(locator: dict[str, Any], hidden: set[str]) -> str: def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any], - masked: dict[str, Any] | None) -> list[str]: + masked: dict[str, Any] | None, kinds: dict[str, Any] | None = None) -> list[str]: """指标的输入取自哪次查询、哪一格,再附上那次查询的 SQL(截到 SQL_CHARS,最多 ENTITY_QUERIES 次)。 「活跃用户按会员号去重」这种讲口径的句子,证据在取数的 SQL 里;只给原式,裁判只能判证据不足。 @@ -504,8 +525,8 @@ def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any snapshot = fetch(artifact) snap = snapshot if isinstance(snapshot, dict) else {} source = (catalog.get(alias) or {}).get("source") if alias else None - hidden = _source_hidden(source or snap.get("source"), catalog, fetch, masked) \ - | _mask_names(snap.get("mask_columns")) + hidden = _derived(snap, _source_hidden(source or snap.get("source"), catalog, fetch, masked) + | _mask_names(snap.get("mask_columns")), fetch, kinds) label = f"查询 {alias} " if alias else "一次查询(不在这份报告的证据目录中)" lines.append(f"输入 {path}:取自{label}的 {_cell_words(item.get('locator') or {}, hidden)}") sql = _clean(str(snap.get("sql") or "")) @@ -518,7 +539,7 @@ def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any], - masked: dict[str, Any] | None) -> str: + masked: dict[str, Any] | None, kinds: dict[str, Any] | None = None) -> str: name = entry.get("name") or (entry.get("locator") or {}).get("metric") or alias rendered = entry.get("rendered") where = " ".join(v for v in (f"口径「{entry['caliber']}」" if entry.get("caliber") else "", @@ -533,7 +554,7 @@ def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch, catalog: d if metric is not None: if metric.get("expression"): lines.append(f"原式:{_clean(str(metric['expression']))}") - lines.extend(_metric_inputs(metric, fetch, catalog, masked)) + lines.extend(_metric_inputs(metric, fetch, catalog, masked, kinds)) return "\n".join(lines) @@ -631,12 +652,13 @@ def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], who def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _Fetch, - masked: dict[str, Any] | None, catalog: dict[str, Any], *, provenance_on: bool = False) -> str | None: + masked: dict[str, Any] | None, catalog: dict[str, Any], *, provenance_on: bool = False, + kinds: dict[str, Any] | None = None) -> str | None: """一件证据的摘录。provenance_on 只交给查询摘录(_query_excerpt):指标摘录里「输入取自查询 Qn」的那几行不加 - 来历(P4-SPEC 调整 1),表名、字段名的摘录照旧。""" + 来历(P4-SPEC 调整 1),表名、字段名的摘录照旧。kinds 只给核查 agent 的摘录(见 _derived)。""" kind = entry.get("kind") if kind == "metric": - return _metric_excerpt(alias, entry, fetch, catalog, masked) + return _metric_excerpt(alias, entry, fetch, catalog, masked, kinds) if kind == "input": field_name = (entry.get("locator") or {}).get("field") or alias return f"【{alias}】运行输入:{field_name} = {entry.get('rendered')}" @@ -644,7 +666,7 @@ def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _ rows = {r for c in cands for a, r in c.rows if a == alias} cols = [col for c in cands for a, col in c.cols if a == alias] return _query_excerpt(alias, entry, rows, cols, any(alias in c.whole for c in cands), fetch, masked, - provenance_on=provenance_on) + provenance_on=provenance_on, kinds=kinds) if kind == "retrieval": picked = [h for c in cands for a, h in c.hits if a == alias] return _retrieval_excerpt(alias, entry, picked, any(alias in c.whole for c in cands), fetch) @@ -739,10 +761,14 @@ class JudgeRequest: def prepare(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, units: Iterable[str] | None = None, - loader: Callable[[str], Any] | None = None, masked: dict[str, Any] | None = None) -> JudgeRequest: + loader: Callable[[str], Any] | None = None, masked: dict[str, Any] | None = None, + kinds: dict[str, Any] | None = None) -> JudgeRequest: """裁判的输入:候选句和证据摘录。纯函数(快照经 loader 取,缺省读工件库),不调模型。 catalog 缺省用文档自己存的目录;masked = {数据源名: 要遮的列},和快照里记下的 mask_columns 合在一起。 + kinds = {数据源名: 类型} 只在给核查 agent(反方审稿、审批质询)准备摘录时给:查询摘录、指标输入里改了名字的遮罩列 + (buyer_phone AS 手机号)按 SQL 认出来也不给,和核查 agent 的查询在工具边界遮的是同一套规则(_derived)。结论句裁判 + 自己的摘录不给它,和以前逐字相同。 摘录按文档标记分版本(P4-SPEC 1.4、3.3):带 `provenance: DOC_PROVENANCE` 的文档,上传表格的查询摘录多几行 来历;没有标记的(期 4 之前组装的)和以前逐字相同,复用键、断点指纹都不变,已判过的句子照旧复用。 @@ -755,7 +781,7 @@ def prepare(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, units for alias in dict.fromkeys(a for c in cands for a in c.aliases): entry = catalog.get(alias) if isinstance(entry, dict) and (text := _excerpt(alias, entry, cands, fetch, masked, catalog, - provenance_on=provenance_on)): + provenance_on=provenance_on, kinds=kinds)): excerpts[alias] = text picked = {c.unit for c in cands} skipped: dict[str, str] = {} diff --git a/backend/app/engine/nodes/challenge.py b/backend/app/engine/nodes/challenge.py index 00f0c22d..f9ddfff3 100644 --- a/backend/app/engine/nodes/challenge.py +++ b/backend/app/engine/nodes/challenge.py @@ -11,9 +11,10 @@ 的 Q1 分开),要它先想清楚「如果这句不成立,数据里会看到什么」,写只读查询去找,最后交结论(经受住 / 被削弱 / 被推翻 / 无法判断)和理由。理由里的数字只能引用它自己查询的格子,裸数字由 compose_doc 标出来。 遮罩:agent 的查询结果在工具边界就遮好了;这份报告引用的查询快照里的遮罩值(按 SQL 认出改了名字的遮罩列,同审批 - 质询)反查之前就交给查询工具,agent 查到写着同一个号码的别的表时也看不到。摘录按列名遮,改了名字的遮罩列原值 - agent 看得到,会照着写进理由:理由正文和理由文档按值换成「已遮罩」(换了写法的号码也认),这句结论原文里写着的 - 不换(_masked_rationale) + 质询)反查之前就交给查询工具,agent 查到写着同一个号码的别的表时也看不到。任务里的证据摘录先遮:遮罩列、按 SQL + 认出改了名字的遮罩列(buyer_phone AS 手机号)不给,别的列、SQL 里写着的遮罩值按值换(_task)。理由正文和理由文档 + 仍按值换成「已遮罩」(换了写法的号码也认,agent 查询时遮掉的值也算),这句结论原文里写着的不换(_masked_rationale); + 换完还认得出的,这句记为无法判断、理由不记(_kept,同审批质询的最后一道关) 3. 出处:一条查询都没执行成功的,判什么都记为无法判断;判「被推翻」的,理由里还得引用到自己的查询结果——它要让 正式出具降档,审稿记录里得查得到凭什么。不采信时系统写明原因,反方原本的结论和理由照录在后面(纯文字:解析 不了的引用写成「(引用无效)」,裸数字注明「(无出处)」)。这句该查的数据源都不存在或已停用、核查模型本身用 @@ -49,6 +50,8 @@ from __future__ import annotations import asyncio +import json +import logging import re from collections.abc import Callable, Collection, Iterable from dataclasses import replace @@ -76,8 +79,12 @@ from app.engine.state import GraphState from app.engine.verify_agent import RATIONALE, VerifyOutcome, run_verify_agent from app.tools.datasource import run_versions +logger = logging.getLogger(__name__) + #: 四种结论:经受住 / 被削弱 / 被推翻 / 无法判断。counts 的键照这个顺序 VERDICTS = ("stands", "weakened", "refuted", "inconclusive") +#: 理由按值遮之后还认得出遮罩值时,这句的理由换成这段系统说明(_kept) +LEAKED = "反方的理由里有设置了遮罩的值,无法遮住,这句的结论不予采信,记为无法判断" #: 节点产出、审批卡里最多列几句(审稿记录里一句不少) ITEMS_MAX = 10 #: 出具缺口、报错里最多点几句的原文,每句最多几个字 @@ -378,6 +385,12 @@ def _claim_of(cand: judging.Candidate, outcome: VerifyOutcome) -> tuple[dict[str return _claim(cand, verdict, rationale, doc=outcome.rationale, queries=outcome.queries, usage=outcome.usage), None +def _scrubbed_excerpt(text: str, known: Any) -> str: + """证据摘录里的遮罩值换成「已遮罩」:别的列、SQL 里写着的(换了写法的号码也认),交给核查 agent 之前就换掉。值是 + 报告快照里的遮罩值去掉这句结论原文里写着的(_spared),同交给查询工具的那一份。""" + return known.scrub(text, loose=True) if known is not None and known.strong else text + + def _spared(seed: Any, claim: str) -> Any: """报告快照里的遮罩值(seed)去掉这句结论原文里写着的,交给这一句的反查。seed 本身不动(下一句另算)。""" known = seed.copy() @@ -391,7 +404,7 @@ def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[s 原文里写着的(given)已经摆在读者眼前,不换。反方 agent 的查询结果在工具边界就遮好了,可证据摘录按列名遮,改了名字 的遮罩列(buyer_phone AS 手机号)的原值它看得到,会写进理由。 换的办法同审批质询的回答(_scrubbed_source):原文里的值先换掉,再照原来的目录重新组装理由文档;引用到的格子按 - 遮好的快照渲染(文字格里夹着的遮罩值换掉那一截)。理由里认不出遮罩值的原样交回。""" + 遮好的快照渲染(文字格里夹着的遮罩值换掉那一截)。理由里认不出遮罩值的原样交回。换完还认得出的过最后一道关(_kept)。""" from app.engine.evidence import compose_doc from app.engine.masking import _masked_view, _scrubbed_source, _Secrets @@ -406,7 +419,7 @@ def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[s pool.prepare() source = str(doc.get("source") or "") if not (pool.spans(str(doc.get("markdown") or ""), prose=True) or pool.spans(source, prose=True)): - return outcome + return _kept(outcome, pool) views: dict[str, Any] = {} fetch = _fetcher() for query in outcome.queries: @@ -422,8 +435,23 @@ def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[s except Exception: # noqa: BLE001 - 重新组装出了意外,值也不能留着 # 正文按值换掉,理由文档不要了(界面照正文显示;判「被推翻」的因此查不到出处,按无法判断记) said = pool.scrub(str(doc.get("markdown") or ""), prose=True) - return replace(outcome, rationale=None, result={**result, RATIONALE: said}) - return replace(outcome, rationale=again, result={**result, RATIONALE: again["markdown"]}) + return _kept(replace(outcome, rationale=None, result={**result, RATIONALE: said}), pool) + return _kept(replace(outcome, rationale=again, result={**result, RATIONALE: again["markdown"]}), pool) + + +def _kept(outcome: VerifyOutcome, pool: Any) -> VerifyOutcome: + """最后一道关(同审批质询的 _leaked):按值遮之后,理由正文、理由文档(正文、原文、目录里的列名)里还认得出遮罩值的 + (换了写法的号码也算),这句的结论不采信、理由不记:记为无法判断,理由换成系统的说明(LEAKED)。遮不住的「被推翻」 + 不能让出具降档,也不能把号码存进审稿记录、随运行封存。""" + doc = outcome.rationale if isinstance(outcome.rationale, dict) else {} + texts = [str((outcome.result or {}).get(RATIONALE) or ""), str(doc.get("markdown") or ""), + str(doc.get("source") or ""), + json.dumps(doc.get("catalog") or {}, ensure_ascii=False, default=str)] + if not any(pool.spans(text, prose=True) for text in texts): + return outcome + logger.warning("反方审稿的理由里有遮罩值遮不住,这句记为无法判断(运行 %s)", doc.get("run_id")) + return replace(outcome, rationale=None, result={**(outcome.result or {}), "verdict": "inconclusive", + RATIONALE: LEAKED}) def _cites(doc: dict[str, Any] | None) -> bool: @@ -573,6 +601,8 @@ class _Review: max_claims, max_queries, timeout_s, daily_max_usd self.spec, self.masks, self.writer_model = spec, masks, writer_model self.fetch = _fetcher() + #: 报告用到的数据源的类型(摘录按 SQL 认改了名字的遮罩列时按它的方言解析)。run() 里查好 + self.kinds: dict[str, Any] = {} #: 这一段执行的总时长(engine/deadline.py):有句子的时长按剩余时间收紧过、或者不再开始时记下,说明和日志里写 self.run_seconds: float | None = None #: 这份报告目录里的查询编号:摘录里指向它们的写法标上「原报告」(mark_original) @@ -597,13 +627,20 @@ class _Review: """当天的核查金额用完、这句一次模型都没调用时的理由:派出去就撞上的和后面不再派出的,是同一回事。""" return f"已达上限({self._limit_words('daily_max_usd')}),这句未反查" - def _task(self, cand: judging.Candidate, sources: list[str], lost: list[str], *, own: bool) -> str: - """交给核查 agent 的任务:这句结论、它挂的证据摘录(遮罩照旧,原报告的编号标明,见 mark_original)、可以查的 - 数据源;该查的源有停用了的,也告诉它查不了哪几个,免得它把查不到当成没有反例。own:源是这句自己引用的, - 不是没挂依据、按整份报告用到的源查的。""" - request = judging.prepare(self.doc, self.catalog, units=[cand.unit], loader=self.fetch, masked=self.masks) + def _task(self, cand: judging.Candidate, sources: list[str], lost: list[str], *, own: bool, + known: Any = None) -> str: + """交给核查 agent 的任务:这句结论、它挂的证据摘录(原报告的编号标明,见 mark_original)、可以查的数据源;该查的 + 源有停用了的,也告诉它查不了哪几个,免得它把查不到当成没有反例。own:源是这句自己引用的,不是没挂依据、按整份 + 报告用到的源查的。 + 摘录先遮,模型从任务里也看不到遮罩值:遮罩列不给,改了名字的(buyer_phone AS 手机号)按 SQL 认出来也不给 + (judge.prepare 的 kinds,和工具边界同一套规则);别的列、SQL 里写着的遮罩值按值换成「已遮罩」,换了写法的号码 + 也认(known:报告快照里的遮罩值去掉这句结论原文里写着的,同交给查询工具的那一份)。解析 SQL、按值换是纯计算, + 在线程里做。""" + request = judging.prepare(self.doc, self.catalog, units=[cand.unit], loader=self.fetch, masked=self.masks, + kinds=self.kinds) aliases = dict.fromkeys(a for c in request.cands for a in c.aliases if a in request.excerpts) - evidence = "\n\n".join(mark_original(request.excerpts[a], a, self.queries) for a in aliases) + evidence = "\n\n".join(mark_original(_scrubbed_excerpt(request.excerpts[a], known), a, self.queries) + for a in aliases) gone = f"({'这句还引用了' if own else '这份报告还用到了'}{_quoted(lost)},已不存在或已停用,无法查询)" \ if lost else "" return CHALLENGE_TASK_PROMPT.format( @@ -620,6 +657,7 @@ class _Review: # 这句没挂依据时在整份报告引用的数据源里查:写作者看得到的就是这几个库 everywhere = claim_sources(self.catalog, self.catalog, self.fetch) usable = await _usable_sources(everywhere) + self.kinds = await self._kinds(everywhere) # 报告引用的查询快照里遮罩列的值:反查之前交给查询工具(agent 查到写着同一个号码的别的表时看不到),反查之后 # 据此换掉理由里的遮罩值(_masked_rationale) seed = await self._seed() @@ -653,12 +691,15 @@ class _Review: claims.append(_claim(cand, "inconclusive", late[0])) overdue += 1 continue + # 这句结论原文里写着的值已经摆在读者眼前,不当遮罩值。上万个值过一遍要几十毫秒,放进线程 + known = await asyncio.to_thread(_spared, seed, cand.text) try: own = claim_sources(cand.aliases, self.catalog, self.fetch) sources = own or everywhere live = [s for s in sources if s in usable] lost = [s for s in sources if s not in usable] - task_text = self._task(cand, live, lost, own=bool(own)) if live else "" + task_text = await asyncio.to_thread(self._task, cand, live, lost, own=bool(own), known=known) \ + if live else "" except Exception as e: # noqa: BLE001 - 一句的摘录出了意外(坏形状的工件),只让这一句没审成 why = f"整理证据摘录时出错({describe_exception(e)})" unreached(cand, f"反查未完成:{why}", why) @@ -686,8 +727,6 @@ class _Review: if allot.tightened: self.run_seconds = allot.seconds # 收紧过的:核查 agent 到点自己发的 verify_limit 改说运行时长上限将至(deadline.relabel),不写收紧后的秒数 - # 这句结论原文里写着的值已经摆在读者眼前,不当遮罩值。上万个值过一遍要几十毫秒,放进线程 - known = await asyncio.to_thread(_spared, seed, cand.text) outcome = await run_verify_agent( spec=self.spec, system=CHALLENGE_SYSTEM, task=task_text, sources=live, output_schema=CHALLENGE_SCHEMA, max_queries=self.max_queries, timeout_s=allot.timeout_s, @@ -777,6 +816,17 @@ class _Review: return await snapshot_secrets(self.fetch(entry.get("artifact")) for entry in self.catalog.values() if isinstance(entry, dict) and entry.get("kind") == "query") + async def _kinds(self, names: Iterable[str]) -> dict[str, Any]: + """这几个数据源的类型(摘录按 SQL 认改了名字的遮罩列时用)。快照记的数据源也算上(旧台账条目没记数据源)。""" + from app.engine.inquiry import _source_kinds # inquiry 导入了这个模块,用到时再导入 + + wanted = set(names) + for entry in self.catalog.values(): + snapshot = self.fetch(entry.get("artifact")) if isinstance(entry, dict) else None + if isinstance(snapshot, dict) and snapshot.get("source"): + wanted.add(str(snapshot["source"])) + return await _source_kinds(wanted) + async def _store(self, record: dict[str, Any]) -> str | None: try: return await artifact_store.put_json(record, kind="challenge", run_id=self.ctx.run.run_id, diff --git a/backend/tests/test_challenge_masking.py b/backend/tests/test_challenge_masking.py new file mode 100644 index 00000000..b102023c --- /dev/null +++ b/backend/tests/test_challenge_masking.py @@ -0,0 +1,120 @@ +"""反方审稿的遮罩收口(engine/nodes/challenge.py): + +- 交给反方 agent 的证据摘录先遮:报告取数把遮罩列改了名字选出来(buyer_phone AS 手机号),摘录里不给这一列(按 SQL + 认,和工具边界同一套规则);别的列(备注里换了写法的号码)、SQL 里写着的遮罩值换成「已遮罩」。以前摘录只按列名遮, + 四个号码原样交给模型,模型照抄进理由 +- 理由正文按值遮(报告快照里的值、agent 查询时遮掉的值、换了写法的号码),端到端;组装之后还认得出遮罩值的(第二道 + 防线漏了),整句降为无法判断,理由不记,同审批质询的最后一道关 + +模型一律是 MockChatModel 剧本,数据源是现造的 SQLite 文件,区域、金额、手机号都是虚构的。 +""" +from __future__ import annotations + +import json +import sqlite3 +import uuid + +import pytest + +from app.core import artifact_store +from app.data import introspect +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource +from app.engine import masking +from app.engine.nodes import challenge as challenge_mod +from tests.test_challenge_node import Models, Play, engine_up, finish, graph, output_of # noqa: F401 - engine_up 是自动用上的夹具 + +#: 四笔订单,买家手机号设了遮罩;备注没设遮罩,其中一条写着换了写法的号码 +SCRIPT = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_phone TEXT, amount INTEGER, refunded INTEGER," + " note TEXT);" + "INSERT INTO sales VALUES (1, '华东', '13900000001', 1200, 0, '客户 139-0000-0002 代下单')," + " (2, '华东', '13900000002', 300, 1, ''), (3, '华南', '13900000003', 900, 0, '例行回访')," + " (4, '华北', '13900000004', 400, 0, '');") +SECRETS = ("1390000000", "0000-0002", "139 0000") + + +@pytest.fixture +async def masked_shop(tmp_path): + path = tmp_path / f"{uuid.uuid4().hex[:8]}.db" + db = sqlite3.connect(path) + db.executescript(SCRIPT) + db.commit() + db.close() + name = f"shop_{uuid.uuid4().hex[:8]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True, + options={"mask_columns": ["buyer_phone"]}) + session.add(row) + await session.flush() + row.schema_cache = await introspect.introspect(row) + await session.commit() + source_id = row.id + yield name + await engines.invalidate(source_id) + + +def _clean(text: str) -> bool: + return not any(s in text for s in SECRETS) + + +#: 报告取数:手机号改了名字选出来,备注照选,按号码排序时把号码写进了 SQL +REPORT_SQL = ("SELECT buyer_phone AS 手机号, note AS 备注, amount AS 金额 FROM sales " + "ORDER BY buyer_phone = '13900000001' DESC, amount DESC") +WRITER = ("金额最高的一笔 [[v:Q1.r0.金额]]。[[see:Q1]]",) +CLAIM = "金额最高的一笔" + + +async def test_the_excerpt_in_the_task_is_masked_before_the_agent_sees_it(monkeypatch, masked_shop): + """反方 agent 收到的任务里:改了名字的遮罩列(手机号)不给,备注里换了写法的号码、SQL 里写着的号码换成「已遮罩」, + 金额照常。以前「列:金额 | 手机号」「r0:1,200 | 13900000001」…,四个号码都在。""" + name = masked_shop + plays = {CLAIM: Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "stands", "共 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]")} + models = Models(monkeypatch, name, writer=WRITER, plays=plays) + row = await finish(graph(name, sql=REPORT_SQL), run_class="exploratory") + assert row.status == "succeeded", row.error + task = models.tasks()[CLAIM] + assert "金额最高的一笔 1,200。" in task and "【原报告 Q1】" in task + excerpt = task[task.index("【原报告 Q1】"):] + assert _clean(excerpt), excerpt + columns = next(line for line in excerpt.splitlines() if line.startswith("列:")) + assert "手机号" not in columns and "金额" in columns and "备注" in columns + assert "客户 已遮罩 代下单" in excerpt and "buyer_phone = '已遮罩'" in excerpt + assert "(有的列按数据源的设置遮罩了,没有给出)" in excerpt + + +async def test_masked_values_the_agent_found_itself_are_masked_in_the_rationale(monkeypatch, masked_shop): + """端到端:反方 agent 自己查出的号码(查询时在工具边界遮掉、只在内存里)被它换着写法写进理由:审稿记录里的理由正文、 + 理由文档都换成「已遮罩」。报告没有选出号码,值只来自 agent 查询时遮掉的那一份。""" + name = masked_shop + top = "SELECT buyer_phone AS 电话, amount AS 金额 FROM sales WHERE region = '华南'" + said = "华南那笔来自 +86 139 0000 0003,金额 [[v:Q1.r0.金额]]。[[see:Q1]]" + plays = {CLAIM: Play([top], "stands", said)} + Models(monkeypatch, name, writer=WRITER, plays=plays) + row = await finish(graph(name, sql="SELECT region AS 区域, amount AS 金额 FROM sales ORDER BY amount DESC"), + run_class="exploratory") + assert row.status == "succeeded", row.error + record = artifact_store.load((await output_of(row.id, "chal"))["challenge_artifact"]) + [claim] = [c for c in record["claims"] if CLAIM in c["text"]] + assert claim["verdict"] == "stands" and claim["rationale"] == "华南那笔来自 已遮罩,金额 900。" + assert claim["rationale_doc"]["markdown"] == claim["rationale"] + assert "0000 0003" not in json.dumps(record, ensure_ascii=False) + + +async def test_a_rationale_still_showing_a_masked_value_is_not_kept(monkeypatch, masked_shop): + """最后一道关:按值遮之后组装出来的理由里还认得出遮罩值(这里让按值换失手),这句降为无法判断,理由不记、不存, + 原值不进审稿记录。同审批质询的回答。""" + name = masked_shop + monkeypatch.setattr(masking, "_scrubbed_source", lambda text, secrets: text) + said = "号码 13900000001 的那笔属实,金额 [[v:Q1.r0.笔数]]。[[see:Q1]]" + plays = {CLAIM: Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "refuted", said)} + Models(monkeypatch, name, writer=WRITER, plays=plays) + row = await finish(graph(name, sql=REPORT_SQL), run_class="exploratory") + assert row.status == "succeeded", row.error + record = artifact_store.load((await output_of(row.id, "chal"))["challenge_artifact"]) + [claim] = [c for c in record["claims"] if CLAIM in c["text"]] + assert claim["verdict"] == "inconclusive" and claim["rationale_doc"] is None + assert claim["rationale"] == challenge_mod.LEAKED + assert _clean(json.dumps(record, ensure_ascii=False)) + out = await output_of(row.id, "chal") + assert out["counts"]["refuted"] == 0, "遮不住的理由不能让出具降档" diff --git a/backend/tests/test_inquiry_masking_e2e.py b/backend/tests/test_inquiry_masking_e2e.py index f791f120..ffae833f 100644 --- a/backend/tests/test_inquiry_masking_e2e.py +++ b/backend/tests/test_inquiry_masking_e2e.py @@ -14,7 +14,7 @@ import json import pytest from app.core import artifact_store -from tests.test_inquiry_api import ( # noqa: F401 - engine_up 是自动用上的夹具,client、big_shop 是夹具 +from tests.test_inquiry_api import ( # noqa: F401 - engine_up 是自动用上的夹具,client、big_shop、masked_shop 是夹具 AMOUNTS, Models, Play, @@ -24,8 +24,11 @@ from tests.test_inquiry_api import ( # noqa: F401 - engine_up 是自动用上 engine_up, events, fetching, + masked_shop, pause, pending, + report_doc, + unit_of, ) BAND = "CASE WHEN amount >= 20000 THEN '大单' ELSE '小单' END" @@ -61,3 +64,27 @@ async def test_a_band_grouped_outside_its_derived_source_reads_as_written(monkey got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{query['artifact']}") assert got.status_code == 200, got.text assert (got.json()["content"]["rows"], got.json()["masked"]) == ([["大单", 3], ["小单", 2]], []) + + +#: 报告取数:手机号改了名字选出来,按号码排序时把号码写进了 SQL +RENAMED = "SELECT buyer_phone AS 手机号, amount AS 金额 FROM sales ORDER BY buyer_phone = '13900000001' DESC, amount DESC" + + +async def test_the_excerpt_in_the_task_is_masked_before_the_agent_sees_it(monkeypatch, masked_shop, client): + """针对「金额最高的一笔 1,200。」发起质询:任务里这句挂的证据摘录不给改了名字的遮罩列(手机号),SQL 里写着的号码 + 换成「已遮罩」,金额照常。以前「列:金额 | 手机号」「r0:1,200 | 13900000001」…,四个号码都在。""" + models = Models(monkeypatch, masked_shop, writer="金额最高的一笔 [[v:Q1.r0.金额]]。[[see:Q1]]", + plays={"这笔是谁买的": Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "共 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]")}) + paused = await pause(masked_shop, graph=fetching(masked_shop, RENAMED)) + [approval] = await pending(paused.id) + doc = await report_doc(paused.id) + r = await ask(client, paused.id, report="write", unit=unit_of(doc, "金额最高的一笔 1,200。"), + question="这笔是谁买的?", approval_id=approval.id) + assert r.status_code == 200, r.text + [task] = models.tasks + excerpt = task[task.index("【原报告 Q1】"):] + assert "1390000000" not in task, excerpt + columns = next(line for line in excerpt.splitlines() if line.startswith("列:")) + assert "手机号" not in columns and "金额" in columns + assert "buyer_phone = '已遮罩'" in excerpt and "(有的列按数据源的设置遮罩了,没有给出)" in excerpt + assert r.json()["inquiry"]["answer"]["ok"] is True From 4b461e86e62065acb8f3c9059338d9ee994ffb0b Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 23:17:15 +0800 Subject: [PATCH 06/12] =?UTF-8?q?K3b=EF=BC=9A=E8=AF=B4=E6=98=8E=E5=86=99?= =?UTF-8?q?=E6=B8=85=E5=B7=A5=E5=85=B7=E8=BE=B9=E7=95=8C=E4=B8=8D=E6=8C=89?= =?UTF-8?q?=E5=80=BC=E6=8D=A2=E6=95=B0=E5=80=BC=E6=A0=BC=E7=9A=84=E5=8F=96?= =?UTF-8?q?=E8=88=8D=EF=BC=9BLATERAL=20=E7=9A=84=20ON=20=E6=9D=A1=E4=BB=B6?= =?UTF-8?q?=E9=87=8C=E7=9C=8B=E5=BE=97=E8=A7=81=E5=AE=83=E8=87=AA=E5=B7=B1?= =?UTF-8?q?=E8=BF=9E=E7=9A=84=E6=9D=A5=E6=BA=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 整行引用找列时,在连接的来源本身里(LATERAL)只看得见前面的来源,在 ON 条件里连这个来源自己也看得见(_visible_sources),和 Postgres 的作用域一致。质询的已知局限补上:没设遮罩的列里存成数的号码工具边界不按值换、agent 看得到,要遮就在那个数据源上把那一列也设成遮罩;mask_query_result、模块说明按这一轮的改动重写,长行重新折好。 --- backend/app/engine/inquiry.py | 5 +++-- backend/app/engine/masking.py | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/backend/app/engine/inquiry.py b/backend/app/engine/inquiry.py index 1556e6f5..19610299 100644 --- a/backend/app/engine/inquiry.py +++ b/backend/app/engine/inquiry.py @@ -49,8 +49,9 @@ 正文里写出某人的年龄遮不住,只靠 SQL 里的等值比较和按列遮;同一个短的数在 SQL 里被换掉了,别的列上撞上它的阈值、 LIMIT 的行数照原样(age = 30 AND visits >= 30)。够长的数在回答正文和回答引用的格子里按值遮,碰巧和某个遮罩值相同的 数(访问次数正好等于某一笔金额)也遮掉,这是产品取舍,不按上下文去猜;工具边界不按值换数值格(存下的是证据),交给 - agent 的、存下的查询快照里这样的数照原样,只在回答和质询工件里遮。占位的写法(未知、暂无……,masking.PLACEHOLDERS) - 不当遮罩值 + agent 的、存下的查询快照里这样的数照原样,只在回答和质询工件里遮。反过来,没设遮罩的列里存成数的号码(别的库里的 + 联系方式表把手机号存成整数)工具边界也不按值换,agent 看得到:要遮就在那个数据源上把那一列也设成遮罩。存成文字的照换。 + 占位的写法(未知、暂无……,masking.PLACEHOLDERS)不当遮罩值 - 遮罩数值列的 SUM / AVG / MIN / MAX 算派生自遮罩列,整列遮(COUNT 不算,见 _value_columns),CASE 里有一支取的是 遮罩列的也算:MIN / MAX 就是某一行的原值,只有一行的分组 SUM / AVG 也是,按列认不出哪些汇总值安全,宁可多遮。 核查 agent 的查询在工具边界照这个办法遮,所以这样的指标双盲复算不了(原 SQL 里认得出的,派盲算之前就记为无法 diff --git a/backend/app/engine/masking.py b/backend/app/engine/masking.py index 2e5283dc..30490636 100644 --- a/backend/app/engine/masking.py +++ b/backend/app/engine/masking.py @@ -28,8 +28,9 @@ 认它带不带同名的列(_schema_tables),不知道表结构的按带得出算(多遮)。分档写在派生来源里、外层按它分组的(一档 一行,_grouped_labels)同当分组键的 CASE 一样照常显示(_shown_labels) - 哪些值(_Secrets):遮罩列里出现过的值,按值认有门槛(SECRET_MIN、SECRET_DIGITS),占位的写法(PLACEHOLDERS:未知、 - 暂无……)不收;换的时候先按词、按开头几个字筛出正文里真有的那几个值,不现编上万个分支的正则;正文、文字格、SQL 的字符串和注释里换了写法的长号码(199-1474-2785、 - +86 …、全角数字)按纯数字写法在集合里查(loose_spans);已经摆在读者眼前的值免遮,之后并进来的照样免(protected) + 暂无……)不收;换的时候先按词、按开头几个字筛出正文里真有的那几个值,不现编上万个分支的正则;正文、文字格、SQL 的 + 字符串和注释里换了写法的长号码(199-1474-2785、+86 …、全角数字)按纯数字写法在集合里查(loose_spans);已经摆在 + 读者眼前的值免遮,之后并进来的照样免(protected) - 快照遮好的样子(_masked_view),回答时才认得出、事后要照位置重放的地方(view_edits、replay_edits) - 一次核查里认出的遮罩值(SeenValues):调用方交进来的和 agent 查询时遮掉的分开记,合起来时 agent 查到的在前 @@ -1263,9 +1264,9 @@ def _visible_sources(select: Any, node: Any) -> list[tuple[str, Any]]: from sqlglot import exp sources = _from_sources(select) - child = node + child, below = node, None while child.parent is not None and child.parent is not select: - child = child.parent + child, below = child.parent, child if child.parent is not select: return sources if isinstance(child, exp.From): @@ -1273,7 +1274,10 @@ def _visible_sources(select: Any, node: Any) -> list[tuple[str, Any]]: if isinstance(child, exp.Join): joins = list(select.args.get("joins") or []) at = next((i for i, join in enumerate(joins) if join is child), None) - return sources[:1 + at] if at is not None else sources + if at is None: + return sources + # 在连接的来源本身里(LATERAL)只看得见前面的;在 ON 条件里连它自己也看得见 + return sources[:1 + at] if below is child.this else sources[:2 + at] return sources @@ -2403,8 +2407,9 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str - 整列遮:列名直接对上遮罩列的,加上按 SQL 认出值取自遮罩列的(_derived_hidden:别名、函数、子查询、WITH、 SELECT *(展开的列换了名字的也算)、表值函数、整行引用和函数参数里的 *、列名表、TABLE x;COUNT、只在条件里用到、 - 分档当分组键的不算);SQL 解析不了、又写着遮罩列(或者 *、TABLE x)的,整份结果的列都遮。这几列的格子换成 - 「已遮罩」,类型记成文本 + 分档当分组键的不算,分档写在派生来源里、外层按它分组的也不算);认不全的(SQL 解析不了又写着遮罩列、*、TABLE x, + (表达式).* 读到遮罩列,表名、SQL 写在字符串里交给函数,分析出意外、超出工作量上限)整份结果的列都遮。这几列的 + 格子换成「已遮罩」,类型记成文本 - 别的列按值换,只换文字格:整格就是遮罩值的换掉,文字格里夹着的那一截换掉(换了写法的号码也认, _Secrets.loose_spans),换过的格子记下来(QueryMask.cells)。遮罩值是这份结果里的加上 known:先查号码、再查写着 号码的备注、没设遮罩的联系方式表(别的库里的也一样)时,后一份同样遮好。门槛同按值遮(SECRET_DIGITS、SECRET_MIN); From 6ba4d9f0992c69eb5274bf454f9bbff13f8e9bde Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 23:23:40 +0800 Subject: [PATCH 07/12] =?UTF-8?q?K3b=EF=BC=9A=E5=8F=8D=E6=96=B9=E5=AE=A1?= =?UTF-8?q?=E7=A8=BF=E6=9F=A5=E6=95=B0=E6=8D=AE=E6=BA=90=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E6=97=B6=E7=9B=B4=E6=8E=A5=E7=94=A8=E6=95=B4=E4=BB=BD=E6=8A=A5?= =?UTF-8?q?=E5=91=8A=E7=94=A8=E5=88=B0=E7=9A=84=E6=BA=90=EF=BC=8C=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E9=80=90=E4=BB=B6=E5=8F=96=E5=B7=A5=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit claim_sources 已经认全了查询的、合并查询合并的、指标输入取自的、表和字段的数据源,_kinds 不必再把目录里每件工件都取一遍。 --- backend/app/engine/nodes/challenge.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/backend/app/engine/nodes/challenge.py b/backend/app/engine/nodes/challenge.py index f9ddfff3..6a2ea01c 100644 --- a/backend/app/engine/nodes/challenge.py +++ b/backend/app/engine/nodes/challenge.py @@ -817,15 +817,11 @@ class _Review: if isinstance(entry, dict) and entry.get("kind") == "query") async def _kinds(self, names: Iterable[str]) -> dict[str, Any]: - """这几个数据源的类型(摘录按 SQL 认改了名字的遮罩列时用)。快照记的数据源也算上(旧台账条目没记数据源)。""" + """这几个数据源的类型(摘录按 SQL 认改了名字的遮罩列时按它的方言解析)。names 是整份报告用到的数据源 + (claim_sources:查询的、合并查询合并的、指标输入取自的、表和字段的)。""" from app.engine.inquiry import _source_kinds # inquiry 导入了这个模块,用到时再导入 - wanted = set(names) - for entry in self.catalog.values(): - snapshot = self.fetch(entry.get("artifact")) if isinstance(entry, dict) else None - if isinstance(snapshot, dict) and snapshot.get("source"): - wanted.add(str(snapshot["source"])) - return await _source_kinds(wanted) + return await _source_kinds(names) async def _store(self, record: dict[str, Any]) -> str | None: try: From c8127852a7bf3c1019d030b6bc6bb3728298ac8f Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 23:25:16 +0800 Subject: [PATCH 08/12] =?UTF-8?q?K3b=EF=BC=9A=E6=96=B0=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=8E=BB=E6=8E=89=E6=B2=A1=E7=94=A8=E5=88=B0?= =?UTF-8?q?=E7=9A=84=E5=AF=BC=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/tests/test_inquiry_masking_e2e.py | 2 -- backend/tests/test_masking_tool_boundary.py | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/backend/tests/test_inquiry_masking_e2e.py b/backend/tests/test_inquiry_masking_e2e.py index ffae833f..af233ad2 100644 --- a/backend/tests/test_inquiry_masking_e2e.py +++ b/backend/tests/test_inquiry_masking_e2e.py @@ -9,8 +9,6 @@ """ from __future__ import annotations -import json - import pytest from app.core import artifact_store diff --git a/backend/tests/test_masking_tool_boundary.py b/backend/tests/test_masking_tool_boundary.py index 311fcf75..acb2b2b4 100644 --- a/backend/tests/test_masking_tool_boundary.py +++ b/backend/tests/test_masking_tool_boundary.py @@ -21,11 +21,8 @@ from app.data.engine import engines from app.engine import masking from app.main import app from tests.test_verify_agent import Script, q -from tests.test_verify_agent_masking import ( # noqa: F401 - clean_spend 是自动用上的夹具 +from tests.test_verify_agent_masking import ( # noqa: F401 - clean_spend 是自动用上的夹具,masked_shop 是夹具 MASKED, - SALES, - VALUE_SCHEMA, - _cells, _clean, _source, ask, From 79c9559079a3adf6535c17ccc7d4839bebd18d80 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 6 Oct 2026 23:25:59 +0800 Subject: [PATCH 09/12] =?UTF-8?q?K3b=EF=BC=9A=E8=B4=A8=E8=AF=A2=E6=A8=A1?= =?UTF-8?q?=E5=9D=97=E8=AF=B4=E6=98=8E=E7=9A=84=E9=81=AE=E7=BD=A9=E4=B8=80?= =?UTF-8?q?=E8=8A=82=E8=A1=A5=E4=B8=8A=E6=B4=BE=E7=94=9F=E6=9D=A5=E6=BA=90?= =?UTF-8?q?=E9=87=8C=E5=88=86=E6=A1=A3=E3=80=81=E5=A4=96=E5=B1=82=E5=88=86?= =?UTF-8?q?=E7=BB=84=E7=9A=84=E6=A0=87=E7=AD=BE=E5=88=97=E4=B8=8D=E7=AE=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/engine/inquiry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/app/engine/inquiry.py b/backend/app/engine/inquiry.py index 19610299..b811e4b4 100644 --- a/backend/app/engine/inquiry.py +++ b/backend/app/engine/inquiry.py @@ -22,8 +22,8 @@ masking.view_edits),取工件时照着再换一遍;SQL 检查结果照事件里记下的 SQL 换(_recorded_swaps)。 哪些列要遮照证据面板的规则(查询当时记下的,加上数据源现在设的),另按 SQL 认出改了名字的遮罩列 (起别名、套函数、经子查询转一道、和 * 写在同一支里,_derived_hidden;认不出落在哪一列的,整份快照遮掉;按遮罩列 - 分档、当分组键或套在汇总里数个数的不算,_band_only);快照按列遮,再按值遮(别的列里写着遮罩值的格子,_masked_view;质询 - 工件里,文字格里夹着的遮罩值另按子串换)。回答正文、查询清单里的遮罩值换成「已遮罩」(_Secrets、_scrub_sql): + 分档、当分组键或套在汇总里数个数的不算,_band_only,分档写在派生来源里、外层按它分组的也不算,_grouped_labels); + 快照按列遮,再按值遮(别的列里写着遮罩值的格子,_masked_view;质询工件里,文字格里夹着的遮罩值另按子串换)。回答正文、查询清单里的遮罩值换成「已遮罩」(_Secrets、_scrub_sql): 按值认有门槛,只装着分档标签的列(agent 写在 SQL 里的标签)不收进遮罩值(_label_columns);SQL 里的字面量按语法树 认用在哪(_literal_uses),数字照原样留着的只有范围比较的边界(没把某一个值夹住的,_pinned)和 LIMIT 的行数,而且 同一个数在这条 SQL 别处没被换掉,封存的是实际执行的语句;在找某一个值的(等值、IN、简单 CASE、夹住某一个值的 From 169c500526bae91c861643a72420a798a0098016 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 7 Oct 2026 01:09:36 +0800 Subject: [PATCH 10/12] =?UTF-8?q?K3b=20=E5=A4=8D=E6=A0=B8=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=EF=BC=88=E7=AC=AC=201=20=E8=BD=AE=EF=BC=89=EF=BC=9A?= =?UTF-8?q?=E6=95=B0=E5=80=BC=E6=A0=BC=E6=8C=89=E4=BD=8D=E7=BD=AE=E8=AE=A4?= =?UTF-8?q?=E3=80=81(=E8=A1=A8=E8=BE=BE=E5=BC=8F).*=20=E5=B8=A6=E5=A4=96?= =?UTF-8?q?=E6=8B=AC=E5=8F=B7=E4=B9=9F=E6=95=B4=E4=BB=BD=E9=81=AE=E3=80=81?= =?UTF-8?q?=E5=90=8C=E5=90=8D=20CTE=20=E6=8C=89=E4=BD=9C=E7=94=A8=E5=9F=9F?= =?UTF-8?q?=E8=AE=A4=E3=80=81=E8=AE=A4=E4=B8=8D=E5=85=A8=E7=9A=84=E5=86=99?= =?UTF-8?q?=E6=B3=95=E5=90=8D=E5=8D=95=E5=90=88=E4=B8=80=E3=80=81=E6=95=B4?= =?UTF-8?q?=E4=BB=BD=E9=81=AE=E4=B8=8D=E5=86=8D=E6=8A=8A=E6=AF=8F=E4=B8=80?= =?UTF-8?q?=E6=A0=BC=E6=94=B6=E6=88=90=E9=81=AE=E7=BD=A9=E5=80=BC=E3=80=81?= =?UTF-8?q?=E8=AF=81=E6=8D=AE=E6=91=98=E5=BD=95=E5=88=86=E6=AE=B5=E9=81=AE?= =?UTF-8?q?=E3=80=81=E5=8F=8D=E6=96=B9=E6=9C=80=E5=90=8E=E4=B8=80=E9=81=93?= =?UTF-8?q?=E5=85=B3=E5=8F=AA=E7=9C=8B=E6=AD=A3=E6=96=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 同名列(回退):column_types 按列名记,同名的几列只留得下后一列的类型,文字的备注被当成数值列,号码不再按值换、 原样交给模型并落库。现在只在列名唯一时按列类型认数值列(_typed_numbers),同名的按格子本身认:整数、小数才算数, 写成字符串的数按文字照换、记进 masked_cells;列名唯一、列类型是 number 的 NUMERIC / DECIMAL 字符串照原样 - (表达式).* 外面套了括号、起了别名(((p.card).*)、(p.card).* AS x,Postgres 实测照样展开):以前只遮第一列。选出项 先去掉外括号、别名再认(_unwrapped),(s).*、(s.*) AS x 照旧当 s.* - WITH 里和数据源里的表同名的一段(WITH sales AS (SELECT * FROM sales WHERE …)):不带 RECURSIVE 的一段看不见它自己和 排在后面的段(_scoped_cte),同名的按数据源里的表认;以前认成它自己,* 展开什么都不带出,整行引用的号码原样交出 - 认不全的写法:和可验证报告合成一份名单(masking 的 OPAQUE_SQL、INDIRECT_FUNCTIONS、INDIRECT_NAMES,bundle 引用后另加 系统 schema),补上 MySQL 会执行的注释、U&"…"、pg_stat_get_activity、逻辑解码、MySQL 的 column_statistics / general_log / PROCESSLIST / performance_schema、MariaDB 的 column_stats、Oracle 的直方图 / *_tab_columns / V$SQL 一族; 去掉注释再认一遍(函数名和括号之间夹注释);去掉 DuckDB、SQL Server 的函数(query 误伤同名 CTE、字符串);没有 * 的 一支选出项和结果列数对不上的、Postgres 解析不了的整份遮 - 整份遮只收确定取自遮罩列的那几列的值(_derived_masks 的 collect,报告快照收值同一规则):以前每一格都收,区域名、 列名在同一次核查里处处被换掉,封存的 SQL 也被改写。快照另记 masked_whole,格子引用照实说写法认不全 - 证据摘录分段遮(inquiry.ExcerptMask,judge.prepare 的 mask 取代 kinds):SQL 照封存 SQL 的办法换,范围阈值、LIMIT 照 原样,和遮罩列做等值、IN、LIKE 比较的字面量不论收没收到都换;格子逐格遮,数值格照原样;质询把被问的那句(连同表格) 当作已经摆在眼前,摘录、查询工具、回答里都不换 - 反方审稿:理由文档的目录列名照快照换(同质询的 _renamed_entry),agent 自己 SQL 里照原样留着的阈值、行数在正文里 不换(_shown),最后一道关只看正文和原文,不再把 agent 起的列名、阈值当成露了遮罩值整句改判 补测试:同名列一文一数、DECIMAL 字符串(含端到端引用取值),外括号 × 复合类型 / 记录函数 / 匿名 ROW × 最外层 / UNION 前后支 / 和 * 写在一起,同名 CTE 四种写法和 RECURSIVE 对照,十七种认不全的写法和三种不该误伤的写法,整份遮后的 收值与封存 SQL,Postgres 解析不了的整行引用,摘录的阈值、数值格、IN / NOT IN 字面量、被问那句(反方、质询各端到端), 反方别名里的阈值(经受住、被推翻各一条)、找某个值的别名、正文里的阈值。逐项去掉实现,对应的测试都变红。 改了文件所有权以外的 bundle/build.py:_OPAQUE_SQL、_INDIRECT_READS 改为引用 masking 的同一份名单(另加系统 schema, 覆盖面不小于原来)。 --- backend/app/bundle/build.py | 44 +-- backend/app/engine/inquiry.py | 153 ++++++-- backend/app/engine/judge.py | 101 ++--- backend/app/engine/masking.py | 386 +++++++++++++++----- backend/app/engine/nodes/challenge.py | 97 +++-- backend/app/engine/verify_agent.py | 17 +- backend/app/tools/datasource.py | 6 + backend/tests/test_challenge_masking.py | 126 +++++++ backend/tests/test_inquiry_masking_e2e.py | 82 +++++ backend/tests/test_masking_excerpts.py | 111 ++++++ backend/tests/test_masking_opaque_sql.py | 234 +++++++++++- backend/tests/test_masking_tool_boundary.py | 130 ++++++- backend/tests/test_masking_whole_rows.py | 12 +- 13 files changed, 1251 insertions(+), 248 deletions(-) create mode 100644 backend/tests/test_masking_excerpts.py diff --git a/backend/app/bundle/build.py b/backend/app/bundle/build.py index a0214469..a04386d5 100644 --- a/backend/app/bundle/build.py +++ b/backend/app/bundle/build.py @@ -39,6 +39,7 @@ from urllib.parse import quote from app.core.artifact_store import canonical_json, content_hash from app.core.events import EventType from app.engine.evidence import iter_units, query_entry_fields +from app.engine.masking import OPAQUE_SQL, SYSTEM_SCHEMAS, indirect_reads from app.engine.merge_query import MERGE_SOURCE from app.engine.recompute import input_origin from app.engine.schema import NodeType @@ -231,43 +232,12 @@ _QUERY_KEYS = frozenset({"columns", "rows", "row_count", "truncated", "elapsed_m #: 导出时只知道数据源设的遮罩、不知道是哪种库:SQL 按每种支持的库的读法各解析一遍(None 是 sqlglot 的通用读法), #: 至少一种读得通,读得通的每一种都要过关 _READINGS = (None, "sqlite", "mysql", "postgres", "oracle") -#: 语法树和数据库实际执行的对不上的写法:MySQL、MariaDB 会执行的注释(/*! … */、/*M! … */,解析器当注释扔掉), -#: Postgres 按 Unicode 转义写的名字(U&"…",原文里看不出是哪一列) -_OPAQUE_SQL = re.compile(r'/\*m?!|u&"', re.I) -#: 不点列名也读得出表里内容的写法。SQL 原文里出现就按碰了遮罩列算,写在字符串里的也算(宁可多挡)。函数要跟着左括号 -#: 才算(列名碰巧叫 crosstab 的不算),表、视图、schema 按名字认 -_INDIRECT_READS = re.compile(r"(? str: def _excerpts(doc: dict[str, Any], catalog: dict[str, Any], cand: judging.Candidate, loader: Callable[[str], Any], - masked: dict[str, Any], kinds: dict[str, Any] | None = None, secrets: _Secrets | None = None - ) -> list[str]: + masked: dict[str, Any], mask: ExcerptMask | None = None) -> list[str]: """这句挂的每件证据的摘录:和 judge.prepare 一样逐件摘(带期 4 标记的文档多来历行),指向原报告编号的写法标上 「原报告」——agent 自己的查询也编 Q1、Q2…,不分开的话会把它们混成一回事。 - 摘录先遮,模型从任务里也看不到遮罩值:数据源遮罩的列不给,改了名字的(buyer_phone AS 手机号)按 SQL 认出来也不给 - (kinds:{数据源名: 类型},和工具边界同一套规则,judge._derived);别的列、SQL 里写着的遮罩值按值换成「已遮罩」, - 换了写法的号码也认(secrets:报告快照里的遮罩值,同交给查询工具的那一份)。解析 SQL、按值换是纯计算,调用方放进线程。""" + 摘录先遮,模型从任务里也看不到遮罩值(mask,见 ExcerptMask):数据源遮罩的列、按 SQL 认出改了名字的 + (buyer_phone AS 手机号)不给;文字格、SQL 里的字符串和注释里的遮罩值按值换成「已遮罩」,换了写法的号码也认,和遮罩列 + 做等值比较的字面量都换;数值格、范围比较的阈值、LIMIT 照原样。解析 SQL、按值换是纯计算,调用方放进线程。""" fetch = judging._Fetch(loader) provenance_on = doc.get("provenance") == DOC_PROVENANCE queries = {a for a, e in catalog.items() if isinstance(e, dict) and e.get("kind") == "query"} @@ -598,9 +615,7 @@ def _excerpts(doc: dict[str, Any], catalog: dict[str, Any], cand: judging.Candid for alias in cand.aliases: entry = catalog.get(alias) if isinstance(entry, dict) and (text := judging._excerpt(alias, entry, [cand], fetch, masked, catalog, - provenance_on=provenance_on, kinds=kinds)): - if secrets is not None and secrets.strong: - text = secrets.scrub(text, loose=True) + provenance_on=provenance_on, mask=mask)): out.append(mark_original(text, alias, queries)) return out @@ -1479,6 +1494,78 @@ def _scrub_sql(sql: str, secrets: _Secrets, compared: set[str], kind: str | None raw, tokens[i].text, kinds[i], uses[i], secrets, compared)[0] if i in uses else raw) +def _scrub_excerpt_sql(sql: str, secrets: _Secrets, compared: set[str], kind: str | None) -> str: + """交给核查 agent 的证据摘录里的 SQL(报告取数的那条):照封存 SQL 的办法换(_scrub_sql:字符串、注释、名字里的遮罩值 + 换,范围比较的边界、LIMIT 的行数照原样,别处够长的遮罩值换),另外和遮罩列(compared:_derived_hidden 认出的名字, + 含改了名字的)做等值、IN、LIKE 一类比较的字面量,不论认不认得出是遮罩值都换成「已遮罩」:报告没选出遮罩列时快照里 + 收不到这些值(WHERE buyer_phone NOT IN ('13900000002', …)),摘录又不是封存的 SQL,可以多遮。同一个数写成阈值、行数的 + 也换(同 _literals)。已经摆在读者眼前的值(问题、被问的那句、这句结论原文里写着的,_Secrets.spared)照旧不换。 + 以前对渲染好的整段摘录按值换,范围比较的阈值(amount >= 20000)也成了「已遮罩」,agent 看不到报告按什么取数。 + 分词出错的整条按值换(同 _scrub_sql)。""" + if not sql or not (secrets.strong or secrets.weak or compared): + return sql + split = _literals(sql, kind, secrets, compared) + if split is None: + return secrets.scrub(sql) + tokens, kinds, uses = split + word = _masked_word() + pinned = {i for i, (how, columns, _) in uses.items() + if how == "equal" and columns & compared and not secrets.spared(tokens[i].text)} + numbers = _number_forms(v for i in pinned for v in (tokens[i].text, *uses[i][2])) + + def literal(i: int, raw: str) -> str: + if i not in uses: + return raw + if i in pinned or (kinds[i] == "NUMBER" and uses[i][0] in _KEPT and _digits_of(tokens[i].text) in numbers): + return _requote(raw, lambda _: word) + return _swap_literal(raw, tokens[i].text, kinds[i], uses[i], secrets, compared)[0] + + return _rebuild(sql, tokens, kinds, secrets, literal) + + +@dataclass +class ExcerptMask: + """交给核查 agent(反方审稿、审批质询)的证据摘录怎么遮:judge.prepare 的 mask,judge 只按这几个方法用它。口径和工具 + 边界一样,模型从任务里也看不到遮罩值,又不改写没设遮罩的数值证据: + - columns:哪几列不给。按列名的,加上按 SQL 认出改了名字的(buyer_phone AS 手机号,masking._derived_masks,表结构按 + 快照冻结的那份);认的过程出了意外,整份的列都不给 + - cell:格子逐格遮(masking.masked_cell):数值格照原样,文字格按值换(整格的、夹着的那一截,换了写法的号码也认) + - sql:SQL 照 _scrub_excerpt_sql 换:范围比较的边界、LIMIT 照原样,和遮罩列做等值比较的字面量都换 + - text:别的文字(列名、检索片段、来历行)按值换,换了写法的号码也认,同 SQL 里的名字 + 报告里列出的指标值、原式、运行输入的值不经它(报告本来就摆在读者眼前,同 _ref_values 免遮)。以前先渲染好整段摘录、 + 再对整段按值换,分不出格子和 SQL 里的用法:阈值、没设遮罩的访问次数正好等于某一笔被遮的金额,都成了「已遮罩」。 + kinds:{数据源名: 类型}(SQL 按它的方言解析);secrets:按值换的值(报告快照里的遮罩值去掉已经摆在读者眼前的,同交给 + 查询工具的那一份),没有为 None(只按列遮、只换和遮罩列做等值比较的字面量)。""" + + kinds: dict[str, Any] + secrets: _Secrets | None = None + + def _kind(self, snapshot: dict[str, Any]) -> str | None: + return self.kinds.get(str(snapshot.get("source") or "")) + + def columns(self, snapshot: dict[str, Any], hidden: set[str], schema: Any) -> tuple[set[str], set[str]]: + """(不给的列, SQL 里指代遮罩列的名字),都是小写。schema 是快照冻结的表结构快照(经 judge 的 fetch 取),没有为 + None(按快照记着的认)。""" + if not hidden: + return set(hidden), set(hidden) + try: + tables = _schema_columns(schema) if isinstance(schema, dict) else None + found = _derived_masks(dict(snapshot), set(hidden), self._kind(snapshot), tables) + return set(hidden) | found.hidden, found.names + except Exception: # noqa: BLE001 - 认不准就整份不给:摘录少几列,不让遮罩值交给核查 agent + logger.warning("按 SQL 认摘录里改了名字的遮罩列时出错,整份快照的列都按遮罩处理", exc_info=True) + return set(hidden) | {str(c).lower() for c in snapshot.get("columns") or []}, set(hidden) + + def cell(self, value: Any, *, numeric: bool) -> Any: + return masked_cell(value, self.secrets, numeric=numeric) + + def sql(self, sql: str, snapshot: dict[str, Any], names: set[str]) -> str: + return _scrub_excerpt_sql(sql, self.secrets or _Secrets(), set(names), self._kind(snapshot)) + + def text(self, text: str) -> str: + return self.secrets.scrub(text, loose=True) if self.secrets is not None and self.secrets.strong else text + + def _rebuild(sql: str, tokens: list[Any], kinds: list[str], secrets: _Secrets, literal: Callable[[int, str], str] ) -> str: """按词重写一段 SQL:字面量交给 literal(第几个词、原文)定换成什么,名字(带不带引号)、注释和别的字按值换。 @@ -1689,8 +1776,10 @@ def _settle_plan(own: dict[str, dict[str, Any]], named: dict[str, tuple[set[str] tainted: dict[str, set[str]] = {} for artifact, snapshot in own.items(): kind, base = kinds.get(str(snapshot.get("source") or "")), named[artifact][0] - hidden[artifact], tainted[artifact] = _derived_hidden(snapshot, base, kind) - secrets.add(snapshot, hidden[artifact], labels=_snapshot_labels(snapshot, kind), base=base) + found = _derived_masks(snapshot, base, kind) + hidden[artifact], tainted[artifact] = found.hidden, found.names + # 收值只收确定取自遮罩列的那几列(整份遮的不是每一列都收,见 masking._derived_masks) + secrets.add(snapshot, found.collect, labels=_snapshot_labels(snapshot, kind), base=base) # agent 查询时遮掉的值先收(报告快照里的值收满了上限 SECRETS_MAX,也不丢 agent 自己查到的);不带它们的一份另算, # 比出事后取质询工件时认不出、要照位置重放的地方 baseline = secrets.copy() if seen is not None and seen.strong else None @@ -1715,10 +1804,11 @@ def _settle_plan(own: dict[str, dict[str, Any]], named: dict[str, tuple[set[str] def _add_based(secrets: _Secrets, based: list[tuple[dict[str, Any], set[str]]], kinds: dict[str, str | None]) -> None: - """收报告自己的查询快照里遮罩列的值(based:快照和它要遮的列),按 SQL 认出改了名字的遮罩列(_derived_hidden)。""" + """收报告自己的查询快照里遮罩列的值(based:快照和它要遮的列),按 SQL 认出改了名字的遮罩列(masking._derived_masks)。 + 写法认不全、整份遮的快照只收确定取自遮罩列的那几列:以前每一格都收,区域名、列名在这次核查里处处被换掉。""" for snapshot, base in based: kind = kinds.get(str(snapshot.get("source") or "")) - secrets.add(snapshot, _derived_hidden(snapshot, base, kind)[0], labels=_snapshot_labels(snapshot, kind), + secrets.add(snapshot, _derived_masks(snapshot, base, kind).collect, labels=_snapshot_labels(snapshot, kind), base=base) @@ -1827,15 +1917,20 @@ async def _answer(op: _Opening, ask: Ask, target: _Target | None, settings: dict async with SessionLocal() as session: spec = await verify_model_spec(session, None, settings=settings) # 报告自己的查询快照:被问的那句可能直接写着遮罩列的值,agent 会照着复述;里面的遮罩值在派 agent 之前就交给它的 - # 查询工具(masked_values),agent 查到写着同一个号码的备注、别的表时,交给模型之前就换掉;证据摘录也按它们先遮 + # 查询工具(masked_values),agent 查到写着同一个号码的备注、别的表时,交给模型之前就换掉;证据摘录也按它们先遮。 + # 被问的那句(在表格里的连同整张表)就写在任务开头、摆在审批人眼前,里面的值同问题一样哪里都不换(同反方审稿的 + # _spared):以前不算,这句引用的那一格在摘录里成了「已遮罩」,和任务开头写着的原句对不上 given = [ask.question, ask.quote, *_ref_values(refs)] + if target is not None: + given += [target.cand.text, target.table] others = [snapshot for entry in catalog.values() if isinstance(entry, dict) and entry.get("kind") == "query" and _query_shaped(snapshot := _quiet_load(op.loader, entry.get("artifact")))] seed = await snapshot_secrets(others, given) excerpts: list[str] = [] if target is not None: kinds = await _source_kinds([*masked, *(str(s.get("source") or "") for s in others)]) - excerpts = await asyncio.to_thread(_excerpts, doc, catalog, target.cand, op.loader, masked, kinds, seed) + excerpts = await asyncio.to_thread(_excerpts, doc, catalog, target.cand, op.loader, masked, + ExcerptMask(kinds, seed)) outcome = await run_verify_agent( spec=spec, system=INQUIRY_SYSTEM, task=_task(op, ask, target, excerpts, refs, live, lost), sources=list(live), output_schema=INQUIRY_SCHEMA, max_queries=part.get("max_queries"), timeout_s=part.get("timeout_s"), diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index f4f17ef2..53c6dff4 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -342,23 +342,22 @@ def _hidden(snapshot: dict[str, Any], source: Any, masked: dict[str, Any] | None return names -def _derived(snapshot: dict[str, Any], hidden: set[str], fetch: _Fetch, kinds: dict[str, Any] | None) -> set[str]: - """核查 agent 的摘录(kinds 给了:{数据源名: 类型})要遮的列:按列名的(hidden),加上按 SQL 认出改了名字的遮罩列 - (buyer_phone AS 手机号、套函数、经子查询转一道……),和核查 agent 的查询在工具边界遮的是同一套规则 - (masking._derived_hidden,表结构按快照冻结的那份、经 fetch 取)。认的过程出了意外,整份快照的列都算(宁可多遮)。 - kinds 为 None(结论句裁判自己的摘录)照旧只按列名,摘录、复用键都和以前一样。""" - if kinds is None or not hidden: - return hidden - from app.engine.masking import _derived_hidden, _schema_columns # masking 导入了别的引擎模块:用到时再导入 +def _derived(snapshot: dict[str, Any], hidden: set[str], fetch: _Fetch, mask: Any) -> tuple[set[str], set[str]]: + """(要遮的列, SQL 里指代遮罩列的名字),都是小写。核查 agent 的摘录(mask 给了,engine/inquiry.py 的 ExcerptMask)另按 + SQL 认出改了名字的遮罩列(buyer_phone AS 手机号、套函数、经子查询转一道……),和核查 agent 的查询在工具边界遮的是同一 + 套规则(表结构按快照冻结的那份、经 fetch 取)。mask 为 None(结论句裁判自己的摘录)照旧只按列名,摘录、复用键都和 + 以前一样。""" + if mask is None or not hidden: + return hidden, hidden + schema = fetch(snapshot.get("schema_artifact")) if snapshot.get("schema_artifact") else None + return mask.columns(snapshot, hidden, schema) - try: - schema = fetch(snapshot.get("schema_artifact")) if snapshot.get("schema_artifact") else None - tables = _schema_columns(schema) if isinstance(schema, dict) else None - kind = kinds.get(str(snapshot.get("source") or "")) - return set(hidden) | _derived_hidden(dict(snapshot), set(hidden), kind, tables)[0] - except Exception: # noqa: BLE001 - 认不准就整份遮:摘录少几列,不让遮罩值交给核查 agent - logger.warning("judge: 按 SQL 认摘录里改了名字的遮罩列时出错,整份快照的列都按遮罩处理", exc_info=True) - return set(hidden) | {str(c).lower() for c in snapshot.get("columns") or []} + +def _unique_number(snapshot: dict[str, Any], column: str) -> bool: + """这一列按列类型是数值列:column_types 记成 number,列名在结果里只有这一列(同名的几列说不清是哪一列的类型, + masking._typed_numbers)。""" + columns = [str(c) for c in snapshot.get("columns") or []] + return column_kind(snapshot, column) == "number" and columns.count(column) == 1 def _shown_columns(columns: list[str], cited: Iterable[str]) -> list[str]: @@ -374,12 +373,13 @@ def _shown_columns(columns: list[str], cited: Iterable[str]) -> list[str]: def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iterable[str], whole: bool, fetch: _Fetch, masked: dict[str, Any] | None, *, provenance_on: bool = False, - kinds: dict[str, Any] | None = None) -> str | None: + mask: Any = None) -> str | None: """一条查询的摘录:头、SQL、(期 4)来历行、列、被引用的行。 provenance_on:文档带期 4 标记时为真(prepare 算)。这时上传表格的查询快照(有 data_version)在 SQL 之后、「列:」 之前多几行来历(_provenance_lines);其余情况一个字都不多,期 4 之前的文档、手工源的摘录和以前逐字相同。 - kinds:给核查 agent 的摘录才给,改了名字的遮罩列也不给(_derived)。 + mask:给核查 agent 的摘录才给(engine/inquiry.py 的 ExcerptMask),分段遮:改了名字的遮罩列也不给(_derived);SQL + 先遮再截(截在半截的值认不出);列名、来历行按值换;格子逐格遮,数值格照原样。 """ snapshot = fetch(entry.get("artifact")) if not isinstance(snapshot, dict) or not isinstance(snapshot.get("rows"), list): @@ -389,13 +389,15 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter source = entry.get("source") or snapshot.get("source") head = (f"【{alias}】合并查询结果(共 {total} 行)" if snapshot.get("source") == MERGE_SOURCE else f"【{alias}】查询结果({entry.get('tool') or '查询'} · 数据源 {source or '—'} · 共 {total} 行)") + hidden, names = _derived(snapshot, _hidden(snapshot, source, masked), fetch, mask) sql = _clean(str(snapshot.get("sql") or "")) + if mask is not None: + sql = mask.sql(sql, snapshot, names) lines = [head, f"SQL:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"] if snapshot.get("source") == MERGE_SOURCE: lines.append(_merge_line(snapshot)) - hidden = _derived(snapshot, _hidden(snapshot, source, masked), fetch, kinds) if provenance_on and snapshot.get("data_version"): - lines += _provenance_lines(snapshot, fetch, hidden) + lines += [mask.text(line) if mask is not None else line for line in _provenance_lines(snapshot, fetch, hidden)] columns = [str(c) for c in snapshot.get("columns") or [] if str(c).lower() not in hidden] shown_cols = _shown_columns(columns, cols) # 点名引用的行先给,整份引用的再从头补满,一共不超过 EXCERPT_ROWS 行 @@ -403,13 +405,16 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter if whole: picked += [r for r in range(total) if r not in picked][:EXCERPT_ROWS - len(picked)] omitted = len(columns) - len(shown_cols) - lines.append("列:" + " | ".join(shown_cols) + (f" | …另有 {omitted} 列" if omitted > 0 else "")) + heads = [mask.text(c) for c in shown_cols] if mask is not None else shown_cols + lines.append("列:" + " | ".join(heads) + (f" | …另有 {omitted} 列" if omitted > 0 else "")) for r in sorted(picked): cells = [] for c in shown_cols: try: raw, _ = locate_cell(snapshot, r, c) - cells.append(render_cell(raw, kind=column_kind(snapshot, c))) + shown = mask.cell(raw, numeric=_unique_number(snapshot, c)) if mask is not None else raw + # 按值换过的格子照文字给:「已遮罩」不能再按这一列的类型当数渲染 + cells.append(render_cell(raw, kind=column_kind(snapshot, c)) if shown is raw else str(shown)) except (CellError, RenderError): cells.append(MISSING) lines.append(f"r{r}:" + " | ".join(cells)) @@ -504,11 +509,12 @@ def _cell_words(locator: dict[str, Any], hidden: set[str]) -> str: def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any], - masked: dict[str, Any] | None, kinds: dict[str, Any] | None = None) -> list[str]: + masked: dict[str, Any] | None, mask: Any = None) -> list[str]: """指标的输入取自哪次查询、哪一格,再附上那次查询的 SQL(截到 SQL_CHARS,最多 ENTITY_QUERIES 次)。 「活跃用户按会员号去重」这种讲口径的句子,证据在取数的 SQL 里;只给原式,裁判只能判证据不足。 位置来自封存的口径卡;SQL 经 fetch 取,取不到(不在封存范围里)的不给。没追到查询的输入照实写。 + mask 同 _query_excerpt:改了名字的遮罩列不写列名,SQL 先遮再截。 """ items = [i for i in metric.get("inputs") or [] if isinstance(i, dict)] lines: list[str] = [] @@ -525,11 +531,13 @@ def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any snapshot = fetch(artifact) snap = snapshot if isinstance(snapshot, dict) else {} source = (catalog.get(alias) or {}).get("source") if alias else None - hidden = _derived(snap, _source_hidden(source or snap.get("source"), catalog, fetch, masked) - | _mask_names(snap.get("mask_columns")), fetch, kinds) + hidden, names = _derived(snap, _source_hidden(source or snap.get("source"), catalog, fetch, masked) + | _mask_names(snap.get("mask_columns")), fetch, mask) label = f"查询 {alias} " if alias else "一次查询(不在这份报告的证据目录中)" lines.append(f"输入 {path}:取自{label}的 {_cell_words(item.get('locator') or {}, hidden)}") sql = _clean(str(snap.get("sql") or "")) + if sql and mask is not None: + sql = mask.sql(sql, snap, names) if sql and str(artifact) not in seen and len(seen) < ENTITY_QUERIES: seen.add(str(artifact)) sqls.append(_sql_line(f"查询 {alias} 的 SQL" if alias else "这次查询的 SQL", sql)) @@ -539,7 +547,7 @@ def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any], - masked: dict[str, Any] | None, kinds: dict[str, Any] | None = None) -> str: + masked: dict[str, Any] | None, mask: Any = None) -> str: name = entry.get("name") or (entry.get("locator") or {}).get("metric") or alias rendered = entry.get("rendered") where = " ".join(v for v in (f"口径「{entry['caliber']}」" if entry.get("caliber") else "", @@ -554,7 +562,7 @@ def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch, catalog: d if metric is not None: if metric.get("expression"): lines.append(f"原式:{_clean(str(metric['expression']))}") - lines.extend(_metric_inputs(metric, fetch, catalog, masked, kinds)) + lines.extend(_metric_inputs(metric, fetch, catalog, masked, mask)) return "\n".join(lines) @@ -632,7 +640,8 @@ def _column_lines(entry: dict[str, Any], fetch: _Fetch) -> list[str]: return lines -def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], whole: bool, fetch: _Fetch) -> str | None: +def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], whole: bool, fetch: _Fetch, + mask: Any = None) -> str | None: content = fetch(entry.get("artifact")) hits = content.get("hits") if isinstance(content, dict) else None if not isinstance(hits, list): @@ -646,6 +655,8 @@ def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], who if not isinstance(hit, dict) or not isinstance(hit.get("content"), str): continue body = _clean(hit["content"]) + # 核查 agent 的摘录(mask):知识库片段里写着的遮罩值按值换(先换再截,截在半截的值认不出) + body = mask.text(body) if mask is not None else body title = f"《{_clean(str(hit['title']))}》" if hit.get("title") else "" lines.append(f"片段 {h + 1}{title}:{body[:CHUNK_CHARS]}{'…' if len(body) > CHUNK_CHARS else ''}") return "\n".join(lines) @@ -653,12 +664,12 @@ def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], who def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _Fetch, masked: dict[str, Any] | None, catalog: dict[str, Any], *, provenance_on: bool = False, - kinds: dict[str, Any] | None = None) -> str | None: + mask: Any = None) -> str | None: """一件证据的摘录。provenance_on 只交给查询摘录(_query_excerpt):指标摘录里「输入取自查询 Qn」的那几行不加 - 来历(P4-SPEC 调整 1),表名、字段名的摘录照旧。kinds 只给核查 agent 的摘录(见 _derived)。""" + 来历(P4-SPEC 调整 1),表名、字段名的摘录照旧。mask 只给核查 agent 的摘录(见 prepare)。""" kind = entry.get("kind") if kind == "metric": - return _metric_excerpt(alias, entry, fetch, catalog, masked, kinds) + return _metric_excerpt(alias, entry, fetch, catalog, masked, mask) if kind == "input": field_name = (entry.get("locator") or {}).get("field") or alias return f"【{alias}】运行输入:{field_name} = {entry.get('rendered')}" @@ -666,27 +677,30 @@ def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _ rows = {r for c in cands for a, r in c.rows if a == alias} cols = [col for c in cands for a, col in c.cols if a == alias] return _query_excerpt(alias, entry, rows, cols, any(alias in c.whole for c in cands), fetch, masked, - provenance_on=provenance_on, kinds=kinds) + provenance_on=provenance_on, mask=mask) if kind == "retrieval": picked = [h for c in cands for a, h in c.hits if a == alias] - return _retrieval_excerpt(alias, entry, picked, any(alias in c.whole for c in cands), fetch) + return _retrieval_excerpt(alias, entry, picked, any(alias in c.whole for c in cands), fetch, mask) if kind == "table": head = f"【{alias}】表 {entry.get('name')}" + (f"(数据源 {entry['source']})" if entry.get("source") else "") - return "\n".join([head, *_table_lines(entry, catalog, fetch, masked), *_entity_sql(entry, catalog, fetch)]) + return "\n".join([head, *_table_lines(entry, catalog, fetch, masked), + *_entity_sql(entry, catalog, fetch, masked, mask)]) if kind == "column": table = (entry.get("locator") or {}).get("table") head = f"【{alias}】字段 {f'{table}.' if table else ''}{(entry.get('locator') or {}).get('column') or entry.get('name')}" - return "\n".join([head, *_column_lines(entry, fetch), *_entity_sql(entry, catalog, fetch)]) + return "\n".join([head, *_column_lines(entry, fetch), *_entity_sql(entry, catalog, fetch, masked, mask)]) return None -def _entity_sql(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch) -> list[str]: +def _entity_sql(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch, masked: dict[str, Any] | None = None, + mask: Any = None) -> list[str]: """表名、字段名出现过的查询的 SQL(每条截到 SQL_CHARS,最多 ENTITY_QUERIES 条)。 讲算法的句子(「按会员号去重计数得出」)证据在 SQL 里;只给名字,裁判只能判不支持。 查询用条目自己记的(表:SQL 里用到它;字段:它是结果列)。只在 SQL 里出现、不是结果列的字段自己没记: 有表条目就从它记的查询里找,没有(文档目录只留正文用到的实体)就看全部查询,都只留 SQL 里写了这个 字段名(有表名的还要写了表名)的。快照经 fetch 取:取不到(不在封存范围里)的不给。 + mask 同 _query_excerpt:SQL 先遮再截(要遮的列按那条查询的数据源认,masked 同 _query_excerpt)。 """ locator = entry.get("locator") or {} column = locator.get("column") if entry.get("kind") == "column" else None @@ -705,6 +719,9 @@ def _entity_sql(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch) - sql = _clean(str(snapshot.get("sql") or "")) if isinstance(snapshot, dict) else "" if not sql or any(not re.search(rf"(? SQL_CHARS else ''}") if len(lines) == ENTITY_QUERIES: break @@ -762,13 +779,15 @@ class JudgeRequest: def prepare(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, units: Iterable[str] | None = None, loader: Callable[[str], Any] | None = None, masked: dict[str, Any] | None = None, - kinds: dict[str, Any] | None = None) -> JudgeRequest: + mask: Any = None) -> JudgeRequest: """裁判的输入:候选句和证据摘录。纯函数(快照经 loader 取,缺省读工件库),不调模型。 catalog 缺省用文档自己存的目录;masked = {数据源名: 要遮的列},和快照里记下的 mask_columns 合在一起。 - kinds = {数据源名: 类型} 只在给核查 agent(反方审稿、审批质询)准备摘录时给:查询摘录、指标输入里改了名字的遮罩列 - (buyer_phone AS 手机号)按 SQL 认出来也不给,和核查 agent 的查询在工具边界遮的是同一套规则(_derived)。结论句裁判 - 自己的摘录不给它,和以前逐字相同。 + mask 只在给核查 agent(反方审稿、审批质询)准备摘录时给(engine/inquiry.py 的 ExcerptMask,这里只按它的 columns、 + cell、sql、text 几个方法用):摘录分段遮,和核查 agent 的查询在工具边界遮的是同一套口径——改了名字的遮罩列 + (buyer_phone AS 手机号)按 SQL 认出来也不给;SQL 照质询封存 SQL 的办法换,范围比较的阈值、LIMIT 照原样;格子逐格 + 遮,数值格照原样;列名、知识库片段、来历行按值换。报告里的指标值、原式、运行输入不遮(报告本来就摆在读者眼前)。 + 以前先渲染好整段、再整段按值换,阈值、没设遮罩的数也成了「已遮罩」。结论句裁判自己的摘录不给它,和以前逐字相同。 摘录按文档标记分版本(P4-SPEC 1.4、3.3):带 `provenance: DOC_PROVENANCE` 的文档,上传表格的查询摘录多几行 来历;没有标记的(期 4 之前组装的)和以前逐字相同,复用键、断点指纹都不变,已判过的句子照旧复用。 @@ -781,7 +800,7 @@ def prepare(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, units for alias in dict.fromkeys(a for c in cands for a in c.aliases): entry = catalog.get(alias) if isinstance(entry, dict) and (text := _excerpt(alias, entry, cands, fetch, masked, catalog, - provenance_on=provenance_on, kinds=kinds)): + provenance_on=provenance_on, mask=mask)): excerpts[alias] = text picked = {c.unit for c in cands} skipped: dict[str, str] = {} diff --git a/backend/app/engine/masking.py b/backend/app/engine/masking.py index 30490636..79f73294 100644 --- a/backend/app/engine/masking.py +++ b/backend/app/engine/masking.py @@ -8,29 +8,35 @@ - 工具边界先遮(mask_query_result):核查 agent 的查询结果交给模型、落进快照之前,取自遮罩列的格子换成「已遮罩」, 别的文字格按值换:这份结果里的遮罩值,加上这个 agent 前面的查询遮掉的、调用方交进来的(报告快照里的)。模型看不到 遮罩值,就不会把它写进回答、理由、SQL——事后按值认每一轮复核都找到新的漏法,根治靠这一步;审批质询回答、反方 - 审稿理由里的按值遮降为第二道防线。宁可多遮,但不改写没设遮罩的数值证据:数值格不按值换,按值换过的文字格记进 - 快照(masked_cells)。查询失败时报错原话里的值也换(mask_error_text)。分析本身出了意外,整份遮(_masked_whole) - -- 哪些列(_derived_hidden):从遮罩列本身起,按 SQL 一层层认出值取自遮罩列的结果列(别名、函数、子查询、WITH、UNION 的 + 审稿理由里的按值遮降为第二道防线。宁可多遮,但不改写没设遮罩的数值证据:数值格(数,列类型是 number 而且列名唯一 + 的,_typed_numbers)不按值换,按值换过的文字格记进快照(masked_cells),认不全整份遮的也记一笔(masked_whole)。查询 + 失败时报错原话里的值也换(mask_error_text)。分析本身出了意外,整份遮(_masked_whole) +- 证据摘录(engine/inquiry.py 的 ExcerptMask,judge.prepare 按它逐段遮):交给反方审稿、审批质询 agent 的报告摘录同一套 + 口径,格子逐格遮(masked_cell),SQL 只换在找某一个值的字面量,阈值照原样 +- 哪些列(_derived_masks):从遮罩列本身起,按 SQL 一层层认出值取自遮罩列的结果列(别名、函数、子查询、WITH、UNION 的 另一支、和 * 写在同一支里没起别名的表达式、FROM 里拆开遮罩列的表值函数、函数参数里的 JSON 箭头 col->'path');* 把 遮罩列带进别的列名的(写了列名表的派生来源、UNION 后一支的 *、标量子查询里的 *)按它展开落到的位置算(_star_places, 展开数据源里的表的按查询当时冻结的表结构数位置,_schema_columns;写了库名的表按「库名.表名」查,另一个库里的同名表 查不到)。整行和函数参数里的 *(Postgres 的 row_to_json(s)、SELECT s、to_json(s.*),Oracle 的 JSON_OBJECT(*))按 * 展开的来源算;表的列名表(FROM sales AS s(i, r, p))、TABLE x、套了括号的 * 先改写成认得出的样子(_normalized)。 + WITH 里的名字按作用域认(_scoped_cte):不带 RECURSIVE 的一段看不见它自己和排在后面的段,同名的是数据源里的表。 COUNT 只数个数、窗口的分区键只决定次序、按遮罩列分档又当分组键或套在汇总里数个数的、汇总里的 CASE 只在条件里用到的 不算(_value_columns、_band_only)。认不全的写法整份快照的列都算,不再逐个形状补规则:认不出落在哪一列的(PIVOT、 - UNPIVOT 这类也算),(表达式).* 展开读到遮罩列的(复合类型的列、函数交回的记录,展开成几列、叫什么说不清),表名、SQL - 写在字符串里交给函数、统计信息里存着值的(_INDIRECT:query_to_xml 这一族、dblink、pg_stats……),SQL 解析不了又写着 - 遮罩列(或者 *、TABLE x、_INDIRECT 那几类)的,分析出了意外、超出工作量上限(ANALYSIS_STEPS)的。一次分析里派生来源 - 对应的那块查询、一项的值取自哪些列记下来(_analysis),一长串 CTE 不再卡住。双盲复算按同一套认法在派盲算之前挡下 - 取自遮罩列的指标(derived_columns) + UNPIVOT 这类也算),(表达式).* 展开读到遮罩列的(复合类型的列、函数交回的记录,外面套了括号、起了别名的也算,展开成 + 几列、叫什么说不清),语法树和实际执行的对不上的(OPAQUE_SQL:MySQL 会执行的注释、U&"…";没有 * 的一支选出项和结果 + 的列数对不上),表名、SQL 写在字符串里交给函数、统计信息和会话记录里存着值的(INDIRECT_FUNCTIONS、INDIRECT_NAMES: + query_to_xml 这一族、dblink、pg_stats、MySQL 的 column_statistics、Oracle 的直方图和 V$SQL……,可验证报告用的同一份), + SQL 解析不了又写着遮罩列(或者 *、TABLE x、上面那几类,或者是 Postgres 的)的,分析出了意外、超出工作量上限 + (ANALYSIS_STEPS)的;这些只收确定取自遮罩列的那几列的值(_Masks.collect)。一次分析里派生来源对应的那块查询、一项 + 的值取自哪些列记下来(_analysis),一长串 CTE 不再卡住。双盲复算按同一套认法在派盲算之前挡下取自遮罩列的指标 + (derived_columns) - 只装着分档标签的列(_label_columns):标签是 agent 写在 SQL 里的字,不当遮罩值;* 展开数据源里的表时按冻结的表结构 认它带不带同名的列(_schema_tables),不知道表结构的按带得出算(多遮)。分档写在派生来源里、外层按它分组的(一档 一行,_grouped_labels)同当分组键的 CASE 一样照常显示(_shown_labels) - 哪些值(_Secrets):遮罩列里出现过的值,按值认有门槛(SECRET_MIN、SECRET_DIGITS),占位的写法(PLACEHOLDERS:未知、 暂无……)不收;换的时候先按词、按开头几个字筛出正文里真有的那几个值,不现编上万个分支的正则;正文、文字格、SQL 的 字符串和注释里换了写法的长号码(199-1474-2785、+86 …、全角数字)按纯数字写法在集合里查(loose_spans);已经摆在 - 读者眼前的值免遮,之后并进来的照样免(protected) + 读者眼前的值免遮,之后并进来的照样免(protected、spared) - 快照遮好的样子(_masked_view),回答时才认得出、事后要照位置重放的地方(view_edits、replay_edits) - 一次核查里认出的遮罩值(SeenValues):调用方交进来的和 agent 查询时遮掉的分开记,合起来时 agent 查到的在前 @@ -45,10 +51,11 @@ import json import logging import re import threading +from collections import Counter from collections.abc import Callable, Iterable, Iterator from dataclasses import dataclass, field from decimal import Decimal -from typing import Any +from typing import Any, NamedTuple from app.engine.evidence import MARKER_RE, RenderError, render_cell from app.engine.expressions import numeric_text @@ -762,19 +769,17 @@ def _derived_block(source: Any) -> tuple[Any, list[str] | None] | None: def _derived_block_of(source: Any) -> tuple[Any, list[str] | None] | None: - """_derived_block 真去找的那一步(不看记下的结果):指向 WITH 的名字要由里往外数看得见的 CTE(_visible_ctes),一长串 + """_derived_block 真去找的那一步(不看记下的结果):指向 WITH 的名字要由里往外数看得见的 CTE(_scoped_cte),一长串 CTE 时每次都数一遍很慢,所以分析里记下来。""" from sqlglot import exp - from app.data.sql_semantics import _visible_ctes - node, alias = source, None if isinstance(node, exp.Lateral): node, alias = node.this, node.args.get("alias") if isinstance(node, exp.Table): if node.args.get("db") or node.args.get("catalog") or not isinstance(node.this, exp.Identifier): return None - cte = _visible_ctes(node).get(node.name.lower()) + cte = _scoped_cte(node, node.name.lower()) if cte is None: return None node, alias = cte.this, cte.args.get("alias") @@ -786,6 +791,33 @@ def _derived_block_of(source: Any) -> tuple[Any, list[str] | None] | None: return node, ([c.name.lower() for c in alias.columns] if alias is not None and alias.columns else None) +def _scoped_cte(node: Any, name: str) -> Any: + """node 所在的位置按名字 name(小写)指到的那一段 CTE,没有为 None(是数据源里的表)。由里往外找,里层的 WITH 遮住 + 外层的:一层查询的正文看得见它的 WITH 里的每一段;一段 CTE 的正文里,不带 RECURSIVE 的 WITH 只看得见排在这一段 + 前面的几段——Postgres、MySQL、Oracle 里看不见它自己,同名的是数据源里的表(WITH sales AS (SELECT * FROM sales + WHERE …) 过滤同名的表是常见写法);SQLite 写自己是循环引用、报错,往后指的它看得见,这里照看不见算,当成数据源里的 + 表(宁可多遮)。带 RECURSIVE 的每一段都看得见(递归的那一支指向它自己)。以前一律当看得见(sql_semantics 的 + _visible_ctes),sales 认成了这一段自己,* 展开绕回自己什么列都不带出,整行引用、列名表改名的号码原样交了出去。""" + from sqlglot import exp + + child, cur = node, node.parent + inside: Any = None # 往外走时经过的、离 cur 最近的那一段 CTE + while cur is not None: + if isinstance(child, exp.CTE): + inside = child + with_ = cur.args.get("with_") + if isinstance(with_, exp.With): + ctes = list(with_.expressions) + at = next((i for i, cte in enumerate(ctes) if cte is inside), None) if child is with_ else None + if at is not None and not with_.args.get("recursive"): + ctes = ctes[:at] + found = next((cte for cte in reversed(ctes) if cte.alias_or_name.lower() == name), None) + if found is not None: + return found + child, cur = cur, cur.parent + return None + + #: 表值函数常见的出列名:SQLite 的 json_each / json_tree(key、value、atom、fullkey、path),Postgres 的 jsonb_each #: (key、value),Spark 的 explode / posexplode(col、pos),Snowflake 的 FLATTEN(key、value、path、index、this)。 #: 结构性的几列(id、parent、type)带不出值,不算——算上的话,同一条 SQL 里别的 id 列都会被遮 @@ -1129,19 +1161,19 @@ def _normalized(root: Any, read: str | None, tables: dict[str, list[str]] | None 改写前后结果有哪几列、每一列的值取自哪里都一样: - TABLE x(Postgres、MySQL):解析器认不出,CTE、子查询里的读成了一个叫 TABLE 的列、表。改成 SELECT * FROM x;带库名、 ORDER BY 的解析不了,按解析不了处理(_implicit) - - 选出项套了括号的 *((s.*))、(s).*(s 是这一层 FROM 里的来源,不是哪个来源的列):Postgres 照样展开成好几列,改成 - s.*,不然按一列数位置 + - 选出项套了括号、起了别名的 *((s.*)、(s.*) AS x)、(s).*(s 是这一层 FROM 里的来源,不是哪个来源的列;外面也可以 + 套括号、起别名):Postgres 照样展开成好几列(别名不起作用),改成 s.*,不然按一列数位置 - 带列名表的表(Postgres 的 FROM sales AS s(i, r, p),引用 CTE 时再写列名表的也算):改成 (SELECT * FROM sales) AS s(i, r, p),按派生来源的列名表认改了名的列。Postgres 的列名表可以比列少(只改前几列的名字,后面的照原名),这时 * 展开的列数和列名表对不上,列名表里的几列整段算(_star_places),后面照原名的按名字认。不按表结构补全列名表: 别的方言里列名表比列少只能是表结构过期了,补全就丢了这一道核对 - 整行引用(read 是 _ROW_DIALECTS 之一,_row_source):row_to_json(s)、json_agg(t)、SELECT s、s::text 里的 s 改成 ROW(s.*),按 * 展开的来源认(_value_columns) - - 别的 (表达式).*(复合类型的列 (p.profile).*、函数交回的记录 (jsonb_each_text(pj)).*、子查询里的 ROW (q.r).*):展开 - 成几列、各叫什么(复合类型的字段名、函数的出列名)SQL 里看不出,改成一个对不上任何来源的 *(_star_sources 为 - None),只用来数别的项的位置。表达式的值取自哪些列(_value_columns,整行引用已经改写过)记下来交回:读到遮罩列的, - 整份快照都算(_tainted)——以前只改成对不上来源的 *,最外层、第一支、派生来源里没写列名表的 * 按「照原名带出」认, - 字段名和遮罩列对不上,号码原样交了出去 + - 别的 (表达式).*(复合类型的列 (p.profile).*、函数交回的记录 (jsonb_each_text(pj)).*、子查询里的 ROW (q.r).*,外面 + 套了括号、起了别名的 ((p.profile).*)、(p.profile).* AS x 也算,_unwrapped):展开成几列、各叫什么(复合类型的字段名、 + 函数的出列名)SQL 里看不出,改成一个对不上任何来源的 *(_star_sources 为 None),只用来数别的项的位置。表达式的值 + 取自哪些列(_value_columns,整行引用已经改写过)记下来交回:读到遮罩列的,整份快照都算(_tainted)——以前只改成对 + 不上来源的 *,最外层、第一支、派生来源里没写列名表的 * 按「照原名带出」认,字段名和遮罩列对不上,号码原样交了出去 tables 是查询当时冻结的表结构(_schema_columns,按次序的列名),认整行引用时看来源出不出某一列;没有为 None。 交回每个 (表达式).* 的值取自哪些列。""" from sqlglot import exp @@ -1184,13 +1216,25 @@ def _normalized(root: Any, read: str | None, tables: dict[str, list[str]] | None opaque: list[set[str]] = [] for select in list(root.find_all(exp.Select)): for item in list(select.expressions): - if isinstance(item, exp.Dot) and isinstance(item.expression, exp.Star): + node = _unwrapped(item) + if isinstance(node, exp.Dot) and isinstance(node.expression, exp.Star): # 先认值取自哪些列(还挂在树上,* 和整行引用按作用域认),再换掉 - opaque.append(set(_value_columns(item.this))) + opaque.append(set(_value_columns(node.this))) item.replace(exp.Column(this=exp.Star(), table=exp.to_identifier(_OPAQUE_STAR))) return opaque +def _unwrapped(item: Any) -> Any: + """选出项去掉套在外面的括号、别名:Postgres 里 (s.*)、((p.card).*)、(p.card).* AS x 照样展开成好几列(别名不起作用, + 实测如此)。以前只认不带外括号的 (表达式).*,((p.card).*) 按一项数,只遮了第一列,后面的号码原样交了出去。""" + from sqlglot import exp + + node = item + while isinstance(node, (exp.Paren, exp.Alias)): + node = node.this + return node + + #: (表达式).* 改写成的 * 写的「表名」:对不上任何来源(_star_sources 为 None),真实的表名、别名不会是它 _OPAQUE_STAR = "*" @@ -1204,18 +1248,17 @@ def _table_keyword(node: Any) -> bool: def _bare_star(item: Any, select: Any, tables: dict[str, list[str]] | None) -> Any: - """select 的选出项是套了括号的 *((s.*)、((*))),或者 (s).*、s 是这一层 FROM 里一个来源的名字、又不是哪个来源说得准 - 出的一列(Postgres 先按列认,_has_column)时,改写成的 *、s.*;别的为 None(_normalized,别的 (表达式).* 另外处理)。""" + """select 的选出项是套了括号、起了别名的 *((s.*)、((*))、(s.*) AS x),或者 (s).*(外面也可以套括号、起别名)、s 是 + 这一层 FROM 里一个来源的名字、又不是哪个来源说得准出的一列(Postgres 先按列认,_has_column)时,改写成的 *、s.*;别的 + 为 None(_normalized,别的 (表达式).* 另外处理)。""" from sqlglot import exp - node = item - while isinstance(node, exp.Paren): - node = node.this + node = _unwrapped(item) if node is not item and _is_star(node): return node.copy() - if not (isinstance(item, exp.Dot) and isinstance(item.expression, exp.Star)): + if not (isinstance(node, exp.Dot) and isinstance(node.expression, exp.Star)): return None - inner = item.this + inner = node.this while isinstance(inner, exp.Paren): inner = inner.this if not (isinstance(inner, exp.Column) and not inner.table and isinstance(inner.this, exp.Identifier)): @@ -1393,10 +1436,20 @@ def _source_names(top: Any, listed: list[str] | None, names: set[str], return found, unknown +class _Taint(NamedTuple): + """_tainted 认出来的:SQL 里指代遮罩列的名字(小写);结果里值取自遮罩列的那几列的位置(认不全时是全部的列);是不是 + 认不全、整份都算;确定取自遮罩列的位置(整份算之前认出的那几列)——整份遮时只收这几列的值当遮罩值(_derived_masks)。""" + + names: set[str] + positions: set[int] + whole: bool + sure: set[int] + + def _tainted(sql: str, kind: str | None, masked: set[str], width: int, tables: dict[str, list[str]] | None = None - ) -> tuple[set[str], set[int], bool] | None: + ) -> _Taint | None: """SQL 里指代遮罩列的名字(小写)、结果(width 列)里值取自遮罩列的那几列的位置,以及是不是认不出、整份都算(这时 - 位置是全部的列)。解析不了返回 None。 + 位置是全部的列,另交回整份算之前确定认出的位置,见 _Taint)。解析不了返回 None。 名字从遮罩列本身起,一层层传下去:选出列的值取自这些名字的(buyer_phone AS 手机号、UPPER(buyer_phone) AS x),它的 名字也算;派生来源(子查询、WITH 里的一段)按位置对上列名表、UNION 第一支的列名,也算,直到不再多出名字。同名 @@ -1411,9 +1464,10 @@ def _tainted(sql: str, kind: str | None, masked: set[str], width: int, tables: d JSON_OBJECT(*))、整行引用(row_to_json(s))按 _value_columns 算进外层的表达式。表的列名表、TABLE x、套了括号的 * 先改写成上面这些认得出的样子(_normalized)。 PIVOT、UNPIVOT、MATCH_RECOGNIZE 读到这些名字的:值转进了它们自己起名的列(UNPIVOT 的值列、PIVOT 按 IN 里的值起名的列、 - MEASURES),出的是哪几列不好认,整份快照的列都算。(表达式).* 读到这些名字的、_INDIRECT 那几类写法同样整份算。 - 解析之后的分析出了意外、超出工作量上限(ANALYSIS_STEPS)的也整份算——不退回「解析不了」只看文字:没写遮罩列、 - 没有 * 的整行引用(SELECT s)那样就放过去了。""" + MEASURES),出的是哪几列不好认,整份快照的列都算。(表达式).* 读到这些名字的、OPAQUE_SQL 和 _INDIRECT 那几类写法 + (_opaque)、结果的一支没有 * 而选出项和结果的列数对不上的(MySQL 会执行的注释里多写了几项这类,语法树和实际执行的 + 对不上)同样整份算。解析之后的分析出了意外、超出工作量上限(ANALYSIS_STEPS)的也整份算——不退回「解析不了」只看 + 文字:没写遮罩列、没有 * 的整行引用(SELECT s)那样就放过去了。""" from app.data.sql_semantics import _parse, _read try: @@ -1425,17 +1479,17 @@ def _tainted(sql: str, kind: str | None, masked: set[str], width: int, tables: d try: opaque = _normalized(root, _read(kind), tables) with _analysis(): - return _tainted_in(root, sql, masked, width, tables, opaque) + return _tainted_in(root, sql, kind, masked, width, tables, opaque) except _Overworked: # 一长串 CTE 一段套一段地整行引用、展开 *:分析不过来,认不准,整份遮 logger.warning("SQL 遮罩分析超出工作量上限(%d 步),整份快照按遮罩处理", ANALYSIS_STEPS) except Exception: # noqa: BLE001 - 分析本身的缺陷不能让回答出错,也不能把原值放过去:整份遮(宁可多遮) logger.warning("SQL 遮罩分析失败,整份快照按遮罩处理", exc_info=True) - return set(masked), set(range(width)), True + return _Taint(set(masked), set(range(width)), True, set()) -def _tainted_in(root: Any, sql: str, masked: set[str], width: int, tables: dict[str, list[str]] | None, - opaque: list[set[str]]) -> tuple[set[str], set[int], bool]: +def _tainted_in(root: Any, sql: str, kind: str | None, masked: set[str], width: int, + tables: dict[str, list[str]] | None, opaque: list[set[str]]) -> _Taint: """_tainted 解析、改写(_normalized)之后真去认的那一步。opaque 是 (表达式).* 的值取自哪些列(_normalized 交回的)。""" from sqlglot import exp @@ -1462,10 +1516,11 @@ def _tainted_in(root: Any, sql: str, masked: set[str], width: int, tables: dict[ if len(names) == before: break # 认不出的整份遮:表值函数出哪几列说不清、派生来源里有认不出名字的列、PIVOT 这类读到遮罩列,(表达式).* 读到遮罩列 - # (展开成几列、叫什么都说不清),SQL 写在字符串里交给函数执行、整表导出、统计信息里的值(_INDIRECT) + # (展开成几列、叫什么都说不清),语法树和实际执行的对不上、SQL 写在字符串里交给函数执行、整表导出、统计信息里的值 + # (_opaque) everything = unreadable or any(_source_names(top, listed, names, tables)[1] for top, listed in sources) \ or any(_reads(names, _value_columns(node)) for node in root.find_all(exp.Pivot, exp.MatchRecognize)) \ - or any(_reads(names, cols) for cols in opaque) or bool(_INDIRECT.search(sql)) + or any(_reads(names, cols) for cols in opaque) or _opaque(sql, kind) positions: set[int] = set() for top, role, _listed in blocks: if role != "result": @@ -1473,6 +1528,10 @@ def _tainted_in(root: Any, sql: str, masked: set[str], width: int, tables: dict[ branches = [leaf for leaf in _leaves(top) if isinstance(leaf, exp.Select)] for k, select_ in enumerate(branches): branch = list(select_.expressions) + if len(branch) != width and not any(_is_star(item) for item in branch): + # 没有 * 的一支,选出项和结果的列数对不上:语法树和数据库实际执行的不是一回事(解析器扔掉的注释里多写 + # 了几项、方言的写法读岔了),哪一列取自哪里都说不准 + everything = True for item, place in zip(branch, _places(branch, width)): if not _is_star(item) and _reads(names, _value_columns(item)): if place is None or place >= width: @@ -1485,7 +1544,7 @@ def _tainted_in(root: Any, sql: str, masked: set[str], width: int, tables: dict[ everything = True else: positions |= spread - return names, set(range(width)) if everything else positions, everything + return _Taint(names, set(range(width)) if everything else positions, everything, positions) def _mentions(sql: str, names: Iterable[str]) -> bool: @@ -1499,55 +1558,144 @@ _COUNT_STAR = re.compile(r"\bcount\s*\(\s*(?:all\s+|distinct\s+)?\*\s*\)", re.I) _IMPLICIT = re.compile(r"(?:\bselect|\bdistinct|\ball|[,(.])\s*\*|\btable\s+[\w\"`]", re.I) -#: 不点列名也读得出表里内容的写法,SQL 里认不出读的是哪一列:表名、SQL 写在字符串里交给函数执行或整表导出(Postgres 的 -#: query_to_xml、table_to_xml 这一族、ts_stat、tablefunc 的 crosstab / connectby、dblink,Oracle 的 DBMS_XMLGEN、XMLTYPE、 -#: XQuery,DuckDB 的 query / query_table,SQL Server 的 OPENQUERY、OPENROWSET),按页、按文件读出的原始数据,统计信息、 -#: 会话记录里存着的值(pg_stats 的常见值、sqlite_stat4 的样本、pg_stat_statements 记下的 SQL 原文)。出现就整份遮 -#: (_tainted、_implicit),写在字符串、注释里的也算(多遮)。函数要跟着左括号才算(列名碰巧叫 query 的不算)。 -#: information_schema、pg_catalog 只有表结构,不在里面 -_INDIRECT = re.compile(r"(? bool: - """解析不了的 SQL 里有没有不写列名也会带出遮罩列的写法(*、TABLE x,_INDIRECT 那几类):它们能把遮罩列带进别的列名 - (列名表、UNION 的后一支)、整表交出来,同写着遮罩列一样整份遮(_derived_hidden)。只看文字,字符串、注释里的也算 - (多遮)。""" - return bool(_IMPLICIT.search(_COUNT_STAR.sub("", sql)) or _INDIRECT.search(sql)) +def indirect_reads(*names: str) -> re.Pattern[str]: + """认「不点列名也读得出表里内容」的写法的正则:INDIRECT_FUNCTIONS 跟着左括号(函数名和括号之间可以有引号、空白), + INDIRECT_NAMES 加上 names 按名字。工具边界(_INDIRECT)和可验证报告(bundle/build.py 的 _INDIRECT_READS,另加 + SYSTEM_SCHEMAS)共用这一份名单:以前两份各写各的,挡在模型前面的这一份反而少了一大截。""" + return re.compile(r"(? bool: + """SQL 里有认不全的写法:OPAQUE_SQL(MySQL 会执行的注释、U&"…"),或者 _INDIRECT 那几类。按原文认(字符串、注释里 + 写着的也算,多遮),再按去掉注释的认:函数名和括号之间夹着注释(query_to_xml/**/(…)、query_to_xml --换行(…))解析得了, + 语法树上又不认函数名,原文里名字后面不紧跟着括号,以前就放过去了。去注释照守卫按数据源的方言切(data/guard.py)。""" + if OPAQUE_SQL.search(sql) or _INDIRECT.search(sql): + return True + from app.data.guard import strip_comments + + try: + bare = strip_comments(sql, dialect=kind) + except Exception: # noqa: BLE001 - 守卫切不开的写法:认不准,按认不全算(宁可多遮) + return True + return bare != sql and bool(_INDIRECT.search(bare)) + + +def _implicit(sql: str, kind: str | None = None) -> bool: + """解析不了的 SQL 里有没有不写列名也会带出遮罩列的写法(*、TABLE x,OPAQUE_SQL、_INDIRECT 那几类,_opaque):它们能 + 把遮罩列带进别的列名(列名表、UNION 的后一支)、整表交出来,同写着遮罩列一样整份遮(_derived_hidden)。只看文字, + 字符串、注释里的也算(多遮)。""" + return bool(_IMPLICIT.search(_COUNT_STAR.sub("", sql))) or _opaque(sql, kind) + + +class _Masks(NamedTuple): + """一份快照按 SQL 认出来的遮罩(_derived_masks):要遮的列(小写)、SQL 里指代遮罩列的名字(小写)、要收值的列(小写, + collect:收进遮罩值的只有这几列,见 _derived_masks)、是不是认不全整份遮的(whole)。""" + + hidden: set[str] + names: set[str] + collect: set[str] + whole: bool def _derived_hidden(snapshot: dict[str, Any], hidden: set[str], kind: str | None, tables: dict[str, list[str]] | None = None, *, labels: dict[str, set[str]] | None = None ) -> tuple[set[str], set[str]]: - """(这份快照要遮的列, SQL 里指代遮罩列的名字),都是小写。tables 是查询当时冻结的表结构(_schema_columns),* 展开 - 数据源里的表时按它数位置;没给的按快照记着的表结构快照(schema_artifact)认,和工具边界遮的时候是同一份。 + """(这份快照要遮的列, SQL 里指代遮罩列的名字),都是小写,见 _derived_masks。""" + found = _derived_masks(snapshot, hidden, kind, tables, labels=labels) + return found.hidden, found.names + + +def _derived_masks(snapshot: dict[str, Any], hidden: set[str], kind: str | None, + tables: dict[str, list[str]] | None = None, *, labels: dict[str, set[str]] | None = None) -> _Masks: + """一份快照要遮的列、SQL 里指代遮罩列的名字、要收值的列、是不是整份遮(_Masks),都是小写。tables 是查询当时冻结的 + 表结构(_schema_columns),* 展开数据源里的表时按它数位置;没给的按快照记着的表结构快照(schema_artifact)认,和 + 工具边界遮的时候是同一份。 证据面板只按列名遮(hidden);agent 的查询起了别名、套了函数,结果列名就对不上。本仓库的查询惯例是起中文别名 (region AS 区域),所以按 SQL 认出改了名字的遮罩列(_tainted)。SQL 解析不了、里面又写着遮罩列的(*、TABLE x 不写 - 列名也会把遮罩列带进别的列名,同样算,_implicit),整份快照的列都遮掉;合并查询的快照不看 SQL,它记着自己的来历 - (_hidden_columns 已按 _merge_masks 遮好)。 + 列名也会把遮罩列带进别的列名,OPAQUE_SQL、_INDIRECT 那几类整表交出来,同样算,_implicit),或者数据源是有整行引用的 + 方言(Postgres:row_to_json(s)、SELECT s 不写列名、也没有 *,只看文字认不出),整份快照的列都遮掉;合并查询的快照 + 不看 SQL,它记着自己的来历(_hidden_columns 已按 _merge_masks 遮好)。 分档写在派生来源里、外层按它分组的标签列(一档一行,_grouped_labels)照常显示,同 _band_only;格子里装着标签以外的、 - 和遮罩列同名的照旧遮(_shown_labels)。labels 是调用方已经认好的标签列(_label_columns),没给的按快照认。""" + 和遮罩列同名的照旧遮(_shown_labels)。labels 是调用方已经认好的标签列(_label_columns),没给的按快照认。 + 收值(collect):认得清的,要遮的列都收;整份遮的只收确定取自遮罩列的那几列——列名对得上遮罩列和 SQL 里指代遮罩列的 + 名字的,加上整份算之前认出的位置(_Taint.sure)。以前整份遮时每一格都收成遮罩值,区域名、列名在同一次核查里处处被 + 换掉,封存的 SQL 也被改写(WHERE 已遮罩 = '华东');认不全的那几列的值本来就没交给模型,不收也不会从别处露出来。""" base = set(hidden) sql = snapshot.get("sql") if not base or not isinstance(sql, str) or not sql.strip() or snapshot.get("source") == MERGE_SOURCE: - return base, base + return _Masks(base, base, base, False) columns = [str(c) for c in snapshot.get("columns") or []] + lowered = {c.lower() for c in columns} found = _tainted(sql, kind, base, len(columns), _snapshot_columns(snapshot) if tables is None else tables) if found is None: - hides = _mentions(sql, base) or _implicit(sql) - return (base | {c.lower() for c in columns}, base) if hides else (base, base) - names, positions, everything = found - out = names | {columns[i].lower() for i in positions if i < len(columns)} - if not everything and out & {c.lower() for c in columns}: + from app.data.sql_semantics import _read + + if _mentions(sql, base) or _implicit(sql, kind) or _read(kind) in _ROW_DIALECTS: + return _Masks(base | lowered, base, base, True) + return _Masks(base, base, base, False) + out = found.names | {columns[i].lower() for i in found.positions if i < len(columns)} + if found.whole: + sure = found.names | {columns[i].lower() for i in found.sure if i < len(columns)} + return _Masks(out, found.names, sure, True) + if out & lowered: out -= _shown_labels(snapshot, kind, columns, base, tables, labels) - return out, names + return _Masks(out, found.names, out, False) def _shown_labels(snapshot: dict[str, Any], kind: str | None, columns: list[str], base: set[str], @@ -1586,8 +1734,7 @@ def derived_columns(snapshot: dict[str, Any], hidden: set[str], kind: str | None found = _tainted(sql, kind, set(hidden), len(columns), _snapshot_columns(snapshot)) if found is None: return None - names, positions, _everything = found - return {c.lower() for i, c in enumerate(columns) if c.lower() in names or i in positions} + return {c.lower() for i, c in enumerate(columns) if c.lower() in found.names or i in found.positions} def _snapshot_labels(snapshot: dict[str, Any], kind: str | None, tables: dict[str, set[str]] | None = None @@ -1976,6 +2123,24 @@ class _Secrets: self.weak = {f for f in self.weak if not asked(f)} self._index, self._compiled = None, {} + def spared(self, value: str) -> bool: + """value(SQL 里的一个字面量、一格的文字)写在已经摆在读者眼前的文字里(protected):认法同 _spare,数按去掉 + 千分位、正负号的数字认,换了写法的号码按纯数字写法认,别的按原文。value 不必是收过的遮罩值——证据摘录里和遮罩列 + 做等值比较的字面量不论认不认得出都换,提问人、这句结论原文里写着的照旧不换(inquiry._scrub_excerpt_sql)。""" + text = _plain_text(value) + if not text or not self.protected: + return False + given = " ".join(self.protected) + if _digits_of(text) in {_digits_of(n) for n in _DIGITS.findall(given)} if _NUMERIC.fullmatch(text) \ + else text in given: + return True + keys = set(_loose_keys(text)) + if not keys: + return False + runs = {"".join(_LOOSE_DIGITS.findall(run.group())).translate(_HALF_DIGITS) + for run in _LOOSE_RUN.finditer(given)} + return any(key in keys for run in runs for key in _loose_candidates(run)) + def show(self, numbers: Iterable[str]) -> None: """SQL 里照原样留着的阈值、行数(_shown_literals):回答正文里复述它们不换(「金额不低于 30000 元」),换了只会 让回答和 SQL 对不上。只免正文:格子、列名、SQL 里照旧按值换——格子里的值正好是这个数,可能就是那个人 @@ -2187,8 +2352,7 @@ def _masked_view(snapshot: dict[str, Any], hidden: set[str], secrets: _Secrets, edited = False position = {c: i for i, c in enumerate(columns)} names = [secrets.scrub(c, loose=True) for c in columns] if by_value and rename else columns - kinds = snapshot.get("column_types") if isinstance(snapshot.get("column_types"), dict) else {} - counted = {i for i, c in enumerate(columns) if kinds.get(c) == "number"} if not numbers else set() + counted = _typed_numbers(snapshot, columns) if not numbers else set() def cell(r: int, i: int | None, value: Any) -> Any: nonlocal edited @@ -2198,7 +2362,7 @@ def _masked_view(snapshot: dict[str, Any], hidden: set[str], secrets: _Secrets, if i is not None: touched.add(i) return value - if not numbers and (i in counted or (isinstance(value, (int, float)) and not isinstance(value, bool))): + if not numbers and (i in counted or _number_cell(value)): return value if by_value and secrets.hit(value): if i is not None: @@ -2239,6 +2403,38 @@ def _masked_view(snapshot: dict[str, Any], hidden: set[str], secrets: _Secrets, return view, [names[i] for i in sorted(touched)] +def _number_cell(value: Any) -> bool: + """格子本身是数(整数、小数,真假不算)。写成字符串的数(Postgres 的 NUMERIC、MySQL 的 DECIMAL 交回来是 "20000.00") + 单看格子分不出是数还是文字,要靠列类型(_typed_numbers)。""" + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _typed_numbers(snapshot: dict[str, Any], columns: list[str]) -> set[int]: + """按列类型认作数值列的列号:column_types 记成 number、而且结果里只有这一列叫这个名字。column_types 按列名记(查询层 + 逐列数类型,同名的几列后面的盖掉前面的),同名的几列说不清记的是哪一列的类型:备注是文字、另一张表里同名的评分是 + 整数,记下的是 number,以前按名字查,文字列也当成了数值列,里面的号码不再按值换,原样交给模型、落了库。同名的几列 + 按格子本身认(_number_cell:整数、小数才算数,写成字符串的数按文字处理,宁可多遮)。""" + kinds = snapshot.get("column_types") if isinstance(snapshot.get("column_types"), dict) else {} + counts = Counter(columns) + return {i for i, c in enumerate(columns) if kinds.get(c) == "number" and counts[c] == 1} + + +def masked_cell(value: Any, secrets: _Secrets | None, *, numeric: bool = False) -> Any: + """一格按值遮好的样子,口径同工具边界(_masked_view 的 numbers=False、embedded):数值格(格子是数,或者 numeric: + 列类型是 number、列名唯一的列,见 _typed_numbers)照原样,不改写没设遮罩的数值证据;文字格整格就是遮罩值的换成 + 「已遮罩」,文字里夹着的换掉那一截(换了写法的号码也认),JSON(对象、数组)按界面画出来的样子换、换过的照文字给。 + 交给核查 agent 的证据摘录逐格这样遮(judge.prepare 的 mask),不再对渲染好的整段文字按值换:那样分不出数值格和文字 + 格,没设遮罩的访问次数正好等于某一笔金额也被换掉。""" + if secrets is None or not secrets.strong or value is None or numeric or _number_cell(value): + return value + if secrets.hit(value): + return _masked_word() + text = _cell_text(value) + if text is not None and (scrubbed := secrets.scrub(text, loose=True)) != text: + return scrubbed + return value + + def _cell_text(value: Any) -> str | None: """一格按文字换的样子(_masked_view 的 embedded):文字照原样,JSON(对象、数组)按界面画出来的样子;别的为 None。""" if isinstance(value, str): @@ -2373,6 +2569,9 @@ class QueryMask: columns: list[str] secrets: _Secrets cells: list[list[int]] = field(default_factory=list) + #: 认不全、整份结果都遮了(_derived_masks 的 whole,分析出了意外的兜底也算):随快照存下(masked_whole,不交给模型), + #: 格子引用指到这份结果时照实说写法认不全,不说成「取自设置了遮罩的列」 + whole: bool = False @dataclass @@ -2405,23 +2604,25 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str (schema_cache,和随快照冻结的是同一份):认分档标签(_label_columns 的 tables)、数 * 展开数据源里的表落在第几列 (_derived_hidden 的 tables)时用。 - - 整列遮:列名直接对上遮罩列的,加上按 SQL 认出值取自遮罩列的(_derived_hidden:别名、函数、子查询、WITH、 + - 整列遮:列名直接对上遮罩列的,加上按 SQL 认出值取自遮罩列的(_derived_masks:别名、函数、子查询、WITH、 SELECT *(展开的列换了名字的也算)、表值函数、整行引用和函数参数里的 *、列名表、TABLE x;COUNT、只在条件里用到、 分档当分组键的不算,分档写在派生来源里、外层按它分组的也不算);认不全的(SQL 解析不了又写着遮罩列、*、TABLE x, - (表达式).* 读到遮罩列,表名、SQL 写在字符串里交给函数,分析出意外、超出工作量上限)整份结果的列都遮。这几列的 - 格子换成「已遮罩」,类型记成文本 + 或者是 Postgres 的;(表达式).* 读到遮罩列,MySQL 会执行的注释、U&"…",表名、SQL 写在字符串里交给函数,统计信息、 + 会话记录里存着值的视图,选出项和结果的列数对不上,分析出意外、超出工作量上限)整份结果的列都遮(QueryMask.whole)。 + 这几列的格子换成「已遮罩」,类型记成文本 - 别的列按值换,只换文字格:整格就是遮罩值的换掉,文字格里夹着的那一截换掉(换了写法的号码也认, _Secrets.loose_spans),换过的格子记下来(QueryMask.cells)。遮罩值是这份结果里的加上 known:先查号码、再查写着 号码的备注、没设遮罩的联系方式表(别的库里的也一样)时,后一份同样遮好。门槛同按值遮(SECRET_DIGITS、SECRET_MIN); 只装着分档标签的列里的标签不算遮罩值,这份结果自己写的标签也不按 known 换(前面的查询认不准、当遮罩值收了的同名 - 标签,不能把这一份的档位列换掉)。数值格(数、列类型是 number 的格子)不按值换:存下的快照就是证据,没设遮罩的 - 数值列(访问次数正好等于某一笔被遮的金额)换掉就改写了证据,双盲复算也取不到它 + 标签,不能把这一份的档位列换掉)。数值格(数、列类型是 number 而且列名唯一的格子,_typed_numbers)不按值换:存下的 + 快照就是证据,没设遮罩的数值列(访问次数正好等于某一笔被遮的金额)换掉就改写了证据,双盲复算也取不到它 - 列名不换(模型自己写的,换了它引用的列名就对不上);结果里一列都没取自遮罩列、也没有可按值换的原样交回:数值的 汇总、只数个数的查询不受影响 - 交回的 secrets 只有这份结果里的遮罩值(known 调用方自己有)。分析本身出了意外(认列、认标签、逐格换值哪一步都算), - 整份结果的每一格都遮、全部的列记成查询时遮掉的(_masked_whole):这一步挡在模型前面,分析的缺陷不能把原值放过去 - (宁可多遮)。""" + 交回的 secrets 只有这份结果里的遮罩值(known 调用方自己有);整份遮的只收确定取自遮罩列的那几列(_Masks.collect), + 不然区域名、列名都成了遮罩值,后面的查询、封存的 SQL 跟着被换掉。分析本身出了意外(认列、认标签、逐格换值哪一步都算), + 整份结果的每一格都遮、全部的列记成查询时遮掉的(_masked_whole),一个值都不收:这一步挡在模型前面,分析的缺陷不能 + 把原值放过去(宁可多遮)。""" base = {str(c).strip().lower() for c in masked if str(c).strip()} columns = [str(c) for c in payload.get("columns") or []] given = known is not None and bool(known.strong) @@ -2429,10 +2630,13 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str return QueryMask(payload, [], _Secrets()) try: labels = _snapshot_labels(payload, kind, _schema_tables(schema)) - hidden = _derived_hidden(payload, base, kind, _schema_columns(schema), labels=labels)[0] if base else set() + found = _derived_masks(payload, base, kind, _schema_columns(schema), labels=labels) if base \ + else _Masks(set(), set(), set(), False) + hidden = found.hidden own = _Secrets() - if any(c.lower() in hidden for c in columns): - own.add(payload, hidden, labels=labels, base=base) + # 收值只收确定取自遮罩列的那几列(整份遮时不是每一列都收,见 _derived_masks) + if any(c.lower() in found.collect for c in columns): + own.add(payload, found.collect, labels=labels, base=base) pool = own if given: pool = own.copy() @@ -2447,8 +2651,8 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str view, _ = _masked_view(payload, hidden, pool, embedded=True, rename=False, numbers=False, changed=cells) except Exception: # noqa: BLE001 - 认不准就整份遮掉,原值不交给模型 logger.warning("核查查询的遮罩分析出错,整份结果按遮罩处理", exc_info=True) - return QueryMask(_masked_whole(payload), columns, _Secrets()) - return QueryMask(view, [c for c in columns if c.lower() in hidden], own, cells) + return QueryMask(_masked_whole(payload), columns, _Secrets(), whole=True) + return QueryMask(view, [c for c in columns if c.lower() in hidden], own, cells, whole=found.whole) def _masked_whole(payload: dict[str, Any]) -> dict[str, Any]: diff --git a/backend/app/engine/nodes/challenge.py b/backend/app/engine/nodes/challenge.py index 6a2ea01c..a0c0ca81 100644 --- a/backend/app/engine/nodes/challenge.py +++ b/backend/app/engine/nodes/challenge.py @@ -11,10 +11,12 @@ 的 Q1 分开),要它先想清楚「如果这句不成立,数据里会看到什么」,写只读查询去找,最后交结论(经受住 / 被削弱 / 被推翻 / 无法判断)和理由。理由里的数字只能引用它自己查询的格子,裸数字由 compose_doc 标出来。 遮罩:agent 的查询结果在工具边界就遮好了;这份报告引用的查询快照里的遮罩值(按 SQL 认出改了名字的遮罩列,同审批 - 质询)反查之前就交给查询工具,agent 查到写着同一个号码的别的表时也看不到。任务里的证据摘录先遮:遮罩列、按 SQL - 认出改了名字的遮罩列(buyer_phone AS 手机号)不给,别的列、SQL 里写着的遮罩值按值换(_task)。理由正文和理由文档 - 仍按值换成「已遮罩」(换了写法的号码也认,agent 查询时遮掉的值也算),这句结论原文里写着的不换(_masked_rationale); - 换完还认得出的,这句记为无法判断、理由不记(_kept,同审批质询的最后一道关) + 质询)反查之前就交给查询工具,agent 查到写着同一个号码的别的表时也看不到。任务里的证据摘录先遮(_task,和工具 + 边界同一套口径):遮罩列、按 SQL 认出改了名字的遮罩列(buyer_phone AS 手机号)不给,文字格、SQL 的字符串和注释里 + 的遮罩值按值换,和遮罩列做等值比较的字面量都换;数值格、范围比较的阈值、LIMIT 照原样。理由正文和理由文档仍按值换成 + 「已遮罩」(换了写法的号码也认,agent 查询时遮掉的值也算),这句结论原文里写着的、agent 自己 SQL 里照原样留着的 + 阈值和行数不换,目录里的列名同质询照快照换(_masked_rationale);换完正文、原文里还认得出的,这句记为无法判断、 + 理由不记(_kept,同审批质询的最后一道关) 3. 出处:一条查询都没执行成功的,判什么都记为无法判断;判「被推翻」的,理由里还得引用到自己的查询结果——它要让 正式出具降档,审稿记录里得查得到凭什么。不采信时系统写明原因,反方原本的结论和理由照录在后面(纯文字:解析 不了的引用写成「(引用无效)」,裸数字注明「(无出处)」)。这句该查的数据源都不存在或已停用、核查模型本身用 @@ -50,7 +52,6 @@ from __future__ import annotations import asyncio -import json import logging import re from collections.abc import Callable, Collection, Iterable @@ -385,12 +386,6 @@ def _claim_of(cand: judging.Candidate, outcome: VerifyOutcome) -> tuple[dict[str return _claim(cand, verdict, rationale, doc=outcome.rationale, queries=outcome.queries, usage=outcome.usage), None -def _scrubbed_excerpt(text: str, known: Any) -> str: - """证据摘录里的遮罩值换成「已遮罩」:别的列、SQL 里写着的(换了写法的号码也认),交给核查 agent 之前就换掉。值是 - 报告快照里的遮罩值去掉这句结论原文里写着的(_spared),同交给查询工具的那一份。""" - return known.scrub(text, loose=True) if known is not None and known.strong else text - - def _spared(seed: Any, claim: str) -> Any: """报告快照里的遮罩值(seed)去掉这句结论原文里写着的,交给这一句的反查。seed 本身不动(下一句另算)。""" known = seed.copy() @@ -398,14 +393,18 @@ def _spared(seed: Any, claim: str) -> Any: return known -def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[str | None]) -> VerifyOutcome: +def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[str | None], + kinds: dict[str, Any] | None = None, masks: dict[str, Any] | None = None) -> VerifyOutcome: """反方的理由里的遮罩值换成「已遮罩」,换了写法的号码也认(199-1474-2785、+86 …)。值是这份报告引用的查询快照里的 (secrets,反查之前就交给了查询工具),加上 agent 查询时在工具边界遮掉的(outcome.masked_values);被审的那句结论 - 原文里写着的(given)已经摆在读者眼前,不换。反方 agent 的查询结果在工具边界就遮好了,可证据摘录按列名遮,改了名字 - 的遮罩列(buyer_phone AS 手机号)的原值它看得到,会写进理由。 - 换的办法同审批质询的回答(_scrubbed_source):原文里的值先换掉,再照原来的目录重新组装理由文档;引用到的格子按 - 遮好的快照渲染(文字格里夹着的遮罩值换掉那一截)。理由里认不出遮罩值的原样交回。换完还认得出的过最后一道关(_kept)。""" + 原文里写着的(given)已经摆在读者眼前,不换。agent 自己 SQL 里照原样留着的范围边界、LIMIT 的行数(_shown:kinds 是 + 数据源的类型,masks 是数据源现在设的遮罩列)在正文里复述也不换,同审批质询的回答(「金额不低于 20000 元」)。 + 反方 agent 的查询结果在工具边界就遮好了、证据摘录也先遮了,这里是第二道防线。 + 换的办法同审批质询的回答(_scrubbed_source):原文里的值先换掉,再照换过列名的目录重新组装理由文档(同质询的 + _renamed_entry:快照遮好的样子列名换过了,引用对得上);引用到的格子按遮好的快照渲染(文字格里夹着的遮罩值换掉 + 那一截)。理由、目录列名里都认不出遮罩值的原样交回。换完还认得出的过最后一道关(_kept)。""" from app.engine.evidence import compose_doc + from app.engine.inquiry import _leaked, _renamed_entry from app.engine.masking import _masked_view, _scrubbed_source, _Secrets doc, result = outcome.rationale, outcome.result @@ -416,9 +415,13 @@ def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[s pool.protect(given) if result is None or not isinstance(doc, dict) or not pool.strong: return outcome + pool.show(_shown(outcome, pool, kinds or {}, masks or {})) pool.prepare() source = str(doc.get("source") or "") - if not (pool.spans(str(doc.get("markdown") or ""), prose=True) or pool.spans(source, prose=True)): + catalog = doc.get("catalog") if isinstance(doc.get("catalog"), dict) else {} + renamed = {alias: _renamed_entry(entry, pool) for alias, entry in catalog.items()} + # 正文里认得出遮罩值的(照原样留着的阈值写进单元格引用里的也算,_leaked)、原文、目录列名里有的,才重新组装 + if renamed == catalog and not (_leaked(doc, pool) or pool.spans(source, prose=True)): return _kept(outcome, pool) views: dict[str, Any] = {} fetch = _fetcher() @@ -430,8 +433,8 @@ def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[s return views[artifact] if artifact in views else artifact_store.load(artifact) try: - again = compose_doc(_scrubbed_source(source, pool), doc.get("catalog") or {}, - node_id=str(doc.get("node_id") or ""), run_id=doc.get("run_id"), loader=load) + again = compose_doc(_scrubbed_source(source, pool), renamed, node_id=str(doc.get("node_id") or ""), + run_id=doc.get("run_id"), loader=load) except Exception: # noqa: BLE001 - 重新组装出了意外,值也不能留着 # 正文按值换掉,理由文档不要了(界面照正文显示;判「被推翻」的因此查不到出处,按无法判断记) said = pool.scrub(str(doc.get("markdown") or ""), prose=True) @@ -439,15 +442,39 @@ def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[s return _kept(replace(outcome, rationale=again, result={**result, RATIONALE: again["markdown"]}), pool) +def _shown(outcome: VerifyOutcome, pool: Any, kinds: dict[str, Any], masks: dict[str, Any]) -> set[str]: + """反方 agent 自己的 SQL 里照原样留着的数(范围比较的边界、LIMIT 的行数,审批质询的 _shown_literals):理由正文里 + 复述它们不换,最后一道关也不算露了遮罩值——它们随 SQL 存进审稿记录、摆在读者眼前,换掉只会让理由和 SQL 对不上。 + 哪些名字是遮罩列按那份快照认(_derived_hidden 的名字,含改了名字的),快照取不到的按数据源现在设的遮罩列。""" + from app.engine.inquiry import _shown_literals + from app.engine.masking import _derived_hidden + + fetch = _fetcher() + sqls: list[tuple[str, str | None, set[str]]] = [] + for query in outcome.queries: + name = str(query.get("source") or "") + base = {str(c).lower() for c in masks.get(name) or []} + snapshot = fetch(query.get("artifact")) + if isinstance(snapshot, dict): + base |= {str(c).lower() for c in snapshot.get("mask_columns") or [] if isinstance(c, str)} + compared = _derived_hidden(snapshot, base, kinds.get(name))[1] if isinstance(snapshot, dict) and base \ + else base + sqls.append((str(query.get("sql") or ""), kinds.get(name), compared)) + return _shown_literals(sqls, pool) + + def _kept(outcome: VerifyOutcome, pool: Any) -> VerifyOutcome: - """最后一道关(同审批质询的 _leaked):按值遮之后,理由正文、理由文档(正文、原文、目录里的列名)里还认得出遮罩值的 - (换了写法的号码也算),这句的结论不采信、理由不记:记为无法判断,理由换成系统的说明(LEAKED)。遮不住的「被推翻」 - 不能让出具降档,也不能把号码存进审稿记录、随运行封存。""" + """最后一道关(同审批质询的 _leaked):按值遮之后,理由正文、理由文档的正文和原文里还认得出遮罩值的(换了写法的号码 + 也算),这句的结论不采信、理由不记:记为无法判断,理由换成系统的说明(LEAKED)。遮不住的「被推翻」不能让出具降档, + 也不能把号码存进审稿记录、随运行封存。只看正文和原文:目录里的列名是 agent 自己起的名字(不低于20000的笔数), + 已经按值换过(_masked_rationale),再拿来认会把恰好等于遮罩值的阈值当成露了值,整句改判;agent 自己 SQL 里照原样 + 留着的阈值写在正文里不算(pool.show),渲染进单元格引用里的照算(同质询的 _leaked)。""" + from app.engine.inquiry import _leaked + doc = outcome.rationale if isinstance(outcome.rationale, dict) else {} - texts = [str((outcome.result or {}).get(RATIONALE) or ""), str(doc.get("markdown") or ""), - str(doc.get("source") or ""), - json.dumps(doc.get("catalog") or {}, ensure_ascii=False, default=str)] - if not any(pool.spans(text, prose=True) for text in texts): + said = str((outcome.result or {}).get(RATIONALE) or "") + if not (pool.spans(said, prose=True) or pool.spans(str(doc.get("source") or ""), prose=True) + or (doc and _leaked(doc, pool))): return outcome logger.warning("反方审稿的理由里有遮罩值遮不住,这句记为无法判断(运行 %s)", doc.get("run_id")) return replace(outcome, rationale=None, result={**(outcome.result or {}), "verdict": "inconclusive", @@ -632,15 +659,16 @@ class _Review: """交给核查 agent 的任务:这句结论、它挂的证据摘录(原报告的编号标明,见 mark_original)、可以查的数据源;该查的 源有停用了的,也告诉它查不了哪几个,免得它把查不到当成没有反例。own:源是这句自己引用的,不是没挂依据、按整份 报告用到的源查的。 - 摘录先遮,模型从任务里也看不到遮罩值:遮罩列不给,改了名字的(buyer_phone AS 手机号)按 SQL 认出来也不给 - (judge.prepare 的 kinds,和工具边界同一套规则);别的列、SQL 里写着的遮罩值按值换成「已遮罩」,换了写法的号码 - 也认(known:报告快照里的遮罩值去掉这句结论原文里写着的,同交给查询工具的那一份)。解析 SQL、按值换是纯计算, - 在线程里做。""" + 摘录先遮,模型从任务里也看不到遮罩值(judge.prepare 的 mask,inquiry.ExcerptMask,和工具边界同一套口径):遮罩列、 + 按 SQL 认出改了名字的(buyer_phone AS 手机号)不给;文字格、SQL 的字符串和注释里的遮罩值按值换成「已遮罩」,换了 + 写法的号码也认,和遮罩列做等值比较的字面量都换;数值格、范围比较的阈值、LIMIT 照原样(known:报告快照里的遮罩值 + 去掉这句结论原文里写着的,同交给查询工具的那一份)。解析 SQL、按值换是纯计算,在线程里做。""" + from app.engine.inquiry import ExcerptMask # inquiry 导入了这个模块,用到时再导入 + request = judging.prepare(self.doc, self.catalog, units=[cand.unit], loader=self.fetch, masked=self.masks, - kinds=self.kinds) + mask=ExcerptMask(self.kinds, known)) aliases = dict.fromkeys(a for c in request.cands for a in c.aliases if a in request.excerpts) - evidence = "\n\n".join(mark_original(_scrubbed_excerpt(request.excerpts[a], known), a, self.queries) - for a in aliases) + evidence = "\n\n".join(mark_original(request.excerpts[a], a, self.queries) for a in aliases) gone = f"({'这句还引用了' if own else '这份报告还用到了'}{_quoted(lost)},已不存在或已停用,无法查询)" \ if lost else "" return CHALLENGE_TASK_PROMPT.format( @@ -732,7 +760,8 @@ class _Review: output_schema=CHALLENGE_SCHEMA, max_queries=self.max_queries, timeout_s=allot.timeout_s, daily_max_usd=self.daily_max_usd, emit=deadline.relabel(ctx.emit, allot), run_id=ctx.run.run_id, node_id=ctx.node.id, data_versions=run_versions(ctx.run), masked_values=known) - outcome = await asyncio.to_thread(_masked_rationale, outcome, known, given=[cand.text]) + outcome = await asyncio.to_thread(_masked_rationale, outcome, known, given=[cand.text], kinds=self.kinds, + masks=self.masks) if outcome.error and not outcome.model: broken = outcome.error # 连模型都没建成(没配好、接入出错):换一句也一样 model = model or outcome.model diff --git a/backend/app/engine/verify_agent.py b/backend/app/engine/verify_agent.py index 229e9f0f..3ae088f4 100644 --- a/backend/app/engine/verify_agent.py +++ b/backend/app/engine/verify_agent.py @@ -12,8 +12,8 @@ (masked_values:审批质询、反方审稿收的报告快照里的值),文字格里夹着的换那一截(engine/masking.py 的 mask_query_result);数值格不按值换(存下的是证据,没设遮罩的访问次数正好等于某一笔被遮的金额也照原样);查询失败时 报错原话里的值也换。模型看不到遮罩值,就不会把它写进回答、理由、SQL;以前是事后按值认,每一轮复核都找到新的漏法。 -快照里的 masked_at_query 记下查询时就遮了的列,masked_cells 记下按值换过的格子,格子引用指到这几列、这几格的都不采信, -照实说是遮罩。遮掉的值只在内存里交回调用方(VerifyOutcome.masked_values,审批质询、反方审稿按值遮的第二道防线 +快照里的 masked_at_query 记下查询时就遮了的列,masked_cells 记下按值换过的格子,masked_whole 记下写法认不全、整份 +都遮了,格子引用指到这几列、这几格的都不采信,照实说是遮罩(整份遮的说写法认不全)。遮掉的值只在内存里交回调用方(VerifyOutcome.masked_values,审批质询、反方审稿按值遮的第二道防线 用),不进快照、不交给模型、不进 as_dict。调用方交进来的和查询时遮掉的分开记(masking.SeenValues),合起来时 agent 查到的在前:合起来到了上限 SECRETS_MAX,挤掉的是报告里的值。 @@ -428,10 +428,12 @@ class _Cells: "reason": f"{alias} {e}"}) return None if str(name).lower() in _masked_at_query(content): - # 查询时就遮了的列:格子里是「已遮罩」,不是数据源里的值。照实说是遮罩列,不说成「不是数值」 + # 查询时就遮了的列:格子里是「已遮罩」,不是数据源里的值。照实说是遮罩列,不说成「不是数值」;认不全、整份 + # 遮的(快照记着 masked_whole)说是写法认不全——这一列不一定取自遮罩列,说成遮罩列会让审批人以为它设了遮罩 + why = f"{alias} 的写法认不全,整份结果查询时已遮罩,不采信" if _masked_whole(content) \ + else f"{alias} 的「{name}」取自设置了遮罩的列,查询时已遮罩,不采信" self.cells.append({"path": path, "status": "unresolved", "ref": f"{alias}.r{row}.{name}", "alias": alias, - "row": row, "column": name, "artifact": query["artifact"], - "reason": f"{alias} 的「{name}」取自设置了遮罩的列,查询时已遮罩,不采信"}) + "row": row, "column": name, "artifact": query["artifact"], "reason": why}) return None if raw == _masked_word() and isinstance(content, dict) and "masked_at_query" in content: # 这一格查询时按值换掉了(和别的查询、报告里的遮罩值相同):同样照实说 @@ -547,6 +549,11 @@ def _masked_at_query(content: Any) -> set[str]: return {c.lower() for c in listed if isinstance(c, str)} if isinstance(listed, list) else set() +def _masked_whole(content: Any) -> bool: + """查询快照记着这条 SQL 的写法认不全、整份结果查询时都遮了(tools/datasource 记下的 masked_whole)。""" + return isinstance(content, dict) and content.get("masked_whole") is True + + def _masked_cell(content: Any, row: Any, name: str) -> bool: """查询快照里第 row 行、name 这一列的格子查询时按值换过(tools/datasource 记下的 masked_cells:[行, 列号])。列名 同名的按第一列算,同 locate_cell。""" diff --git a/backend/app/tools/datasource.py b/backend/app/tools/datasource.py index e8b79057..5fa69dac 100644 --- a/backend/app/tools/datasource.py +++ b/backend/app/tools/datasource.py @@ -535,6 +535,7 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, # 别的列里按值换过的格子另记(masked_cells),没有就不记 masked_now: list[str] | None = None masked_cells: list[list[int]] = [] + masked_whole = False if mask_results and (mask or known is not None): from app.engine.masking import mask_query_result @@ -543,6 +544,7 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, schema=source.schema_cache) if mask or done.payload is not payload: payload, masked_now, masked_cells = done.payload, done.columns, done.cells + masked_whole = done.whole if sink is not None: # 并进 agent 查到的那一份(不和调用方交进来的混在一起,上限收满了也不丢),也放进线程 await asyncio.to_thread(sink.add, done.secrets) @@ -567,6 +569,8 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, # 改了名字的也在这里)。有这一项的快照,别的格子整格是「已遮罩」的是查询时按值换掉的; # - masked_cells:别的列里查询时按值换过的文字格 [行, 列号](整格换的、换掉一截的),格子引用指到它们时照实 # 说是遮罩。数值格不按值换,存下的数和数据库里的一样; + # - masked_whole:这条 SQL 的写法认不全,整份结果查询时都遮了(masked_at_query 是全部的列),格子引用指到它时 + # 照实说写法认不全,不说成取自遮罩列; # - data_version:上传表格查的是哪个快照,事后据此认得出这个数出自哪一版数据。 # 手工源、没设遮罩的,快照和原来一字不差 extra: dict[str, Any] = {} @@ -576,6 +580,8 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, extra["masked_at_query"] = masked_now if masked_cells: extra["masked_cells"] = masked_cells + if masked_whole: + extra["masked_whole"] = True if version := getattr(source, "snapshot_id", None): extra["data_version"] = version # 工件引用的 meta 记下源和 SQL 里用到的表:数据目录按它数每张表被运行查询过几次 diff --git a/backend/tests/test_challenge_masking.py b/backend/tests/test_challenge_masking.py index b102023c..b1d6b89b 100644 --- a/backend/tests/test_challenge_masking.py +++ b/backend/tests/test_challenge_masking.py @@ -32,6 +32,7 @@ SCRIPT = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_phone " (2, '华东', '13900000002', 300, 1, ''), (3, '华南', '13900000003', 900, 0, '例行回访')," " (4, '华北', '13900000004', 400, 0, '');") SECRETS = ("1390000000", "0000-0002", "139 0000") +MASKED = "已遮罩" @pytest.fixture @@ -118,3 +119,128 @@ async def test_a_rationale_still_showing_a_masked_value_is_not_kept(monkeypatch, assert _clean(json.dumps(record, ensure_ascii=False)) out = await output_of(row.id, "chal") assert out["counts"]["refuted"] == 0, "遮不住的理由不能让出具降档" + + +# -------------------------------------------------------------------------- +# 摘录分段遮:阈值、没设遮罩的数值格照原样,和遮罩列做等值比较的字面量都换 +# -------------------------------------------------------------------------- + + +#: 金额、客户名设了遮罩,金额都是五位数;访问次数没设遮罩,第二、三、四单正好等于别的订单的金额 +BIG = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_name TEXT, amount INTEGER, visits INTEGER);" + "INSERT INTO sales VALUES (1, '华东', '张伟', 10000, 25000), (2, '华东', '李娜', 12000, 20000)," + " (3, '华南', '王芳', 20000, 30000), (4, '华北', '刘洋', 30000, 50000), (5, '华北', '赵敏', 50000, 9000);") + + +@pytest.fixture +async def big_shop(tmp_path): + path = tmp_path / f"{uuid.uuid4().hex[:8]}.db" + db = sqlite3.connect(path) + db.executescript(BIG) + db.commit() + db.close() + name = f"shop_{uuid.uuid4().hex[:8]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True, + options={"mask_columns": ["amount", "buyer_name"]}) + session.add(row) + await session.flush() + row.schema_cache = await introspect.introspect(row) + await session.commit() + source_id = row.id + yield name + await engines.invalidate(source_id) + + +def _excerpt_of(task: str) -> str: + return task[task.index("【原报告 Q1】"):] + + +async def test_thresholds_and_unmasked_numbers_in_the_excerpt_are_kept(monkeypatch, big_shop): + """报告按金额阈值取数(amount >= 20000,正好等于某一笔被遮的金额),访问次数没设遮罩、第二行正好等于另一笔金额: + 摘录里 SQL 的阈值、访问次数都照原样,金额列不给。以前对渲染好的整段按值换,成了「amount >= 已遮罩」「r1:4 | 华北 | + 已遮罩」,agent 看不到报告按什么取数,没设遮罩的数也被改写。""" + report = "SELECT id, region, amount, visits FROM sales WHERE amount >= 20000 ORDER BY id" + claim = "大额订单里访问最多的一笔在华南" + plays = {claim: Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "stands", "共 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]")} + models = Models(monkeypatch, big_shop, writer=(f"{claim},访问 [[v:Q1.r0.visits]] 次。[[see:Q1]]",), + plays=plays) + row = await finish(graph(big_shop, sql=report), run_class="exploratory") + assert row.status == "succeeded", row.error + excerpt = _excerpt_of(models.tasks()[claim]) + assert f"SQL:{report}" in excerpt, excerpt + assert "列:visits | id | region" in excerpt + assert "r1:50,000 | 4 | 华北" in excerpt and MASKED not in excerpt.split("列:", 1)[1].split("(", 1)[0] + + +async def test_literals_compared_with_a_masked_column_are_masked_even_when_never_collected(monkeypatch, masked_shop): + """报告没选出号码,只在条件里排除了两个(一个换了写法):报告快照里收不到这两个值,摘录的 SQL 照样换掉——和遮罩列 + 做等值、IN 比较的字面量不论认不认得出都换。以前按值换只认快照里收到的,号码原样交给 agent。""" + report = ("SELECT region AS 区域, COUNT(*) AS 笔数 FROM sales WHERE buyer_phone NOT IN ('13900000002', " + "'139-0000-0003') GROUP BY region ORDER BY 笔数 DESC") + claim = "笔数最多的区域" + plays = {claim: Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "stands", "共 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]")} + models = Models(monkeypatch, masked_shop, writer=(f"{claim}有 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]",), plays=plays) + row = await finish(graph(masked_shop, sql=report), run_class="exploratory") + assert row.status == "succeeded", row.error + task = models.tasks()[claim] + assert _clean(task), task + assert f"buyer_phone NOT IN ('{MASKED}', '{MASKED}')" in _excerpt_of(task) + + +# -------------------------------------------------------------------------- +# 最后一道关只看正文和原文:agent 起的列名、照原样留着的阈值不算露了遮罩值 +# -------------------------------------------------------------------------- + + +AMOUNTS = "SELECT region AS 区域, amount AS 金额 FROM sales ORDER BY id" +FIRST = "华东首单金额" + + +@pytest.mark.parametrize("verdict", ["stands", "refuted"]) +async def test_a_threshold_in_an_alias_does_not_void_the_verdict(monkeypatch, big_shop, verdict): + """反方 agent 把阈值写进别名(不低于20000的笔数),20000 恰好是某一笔被遮的金额:结论照常采信,理由照常记。以前最后 + 一道关把目录里的列名当成露了遮罩值,整句改判无法判断,「被推翻」也不再让出具降档,日志里多一条误报。""" + top = "SELECT COUNT(*) AS 不低于20000的笔数 FROM sales WHERE amount >= 20000" + said = "金额不低于两万的订单有 [[v:Q1.r0.不低于20000的笔数]] 笔,与结论" + ("不矛盾" if verdict == "stands" else "矛盾") \ + + "。[[see:Q1]]" + Models(monkeypatch, big_shop, writer=(f"{FIRST} [[v:Q1.r0.金额]]。[[see:Q1]]",), + plays={FIRST: Play([top], verdict, said)}) + row = await finish(graph(big_shop, sql=AMOUNTS), run_class="exploratory") + assert row.status == "succeeded", row.error + out = await output_of(row.id, "chal") + record = artifact_store.load(out["challenge_artifact"]) + [claim] = [c for c in record["claims"] if FIRST in c["text"]] + assert claim["verdict"] == verdict, claim["rationale"] + assert claim["rationale"].startswith("金额不低于两万的订单有 3 笔") + assert out["counts"][verdict] == 1 + + +async def test_a_threshold_the_agent_wrote_in_its_sql_stays_in_the_rationale(monkeypatch, big_shop): + """理由正文复述 agent 自己 SQL 里照原样留着的阈值(「不低于 20000 元」,恰好等于某一笔被遮的金额):照原样,同审批质询 + 的回答;它随 SQL 存进审稿记录、摆在读者眼前。以前换成「已遮罩」,理由和 SQL 对不上。""" + top = "SELECT COUNT(*) AS 笔数 FROM sales WHERE amount >= 20000" + said = "金额不低于 20000 元的订单有 [[v:Q1.r0.笔数]] 笔,与结论不矛盾。[[see:Q1]]" + Models(monkeypatch, big_shop, writer=(f"{FIRST} [[v:Q1.r0.金额]]。[[see:Q1]]",), + plays={FIRST: Play([top], "stands", said)}) + row = await finish(graph(big_shop, sql=AMOUNTS), run_class="exploratory") + assert row.status == "succeeded", row.error + record = artifact_store.load((await output_of(row.id, "chal"))["challenge_artifact"]) + [claim] = [c for c in record["claims"] if FIRST in c["text"]] + assert (claim["verdict"], claim["rationale"]) == ("stands", "金额不低于 20000 元的订单有 3 笔,与结论不矛盾。") + + +async def test_an_alias_naming_a_masked_value_is_renamed_like_the_snapshot(monkeypatch, big_shop): + """agent 把找的那个值写进了别名(金额为20000的笔数,20000 是一笔被遮的金额,不是阈值):理由文档的目录列名照快照遮好的 + 样子换(同质询的 _renamed_entry),引用照样对得上,结论照常采信,存下的理由文档里没有这个值。以前目录原样传进重新 + 组装,最后一道关又把目录拿去认,整句改判无法判断。""" + top = "SELECT COUNT(*) AS 金额为20000的笔数 FROM sales WHERE amount = 20000" + said = "金额为两万的订单有 [[v:Q1.r0.金额为20000的笔数]] 笔,与结论不矛盾。[[see:Q1]]" + Models(monkeypatch, big_shop, writer=(f"{FIRST} [[v:Q1.r0.金额]]。[[see:Q1]]",), + plays={FIRST: Play([top], "stands", said)}) + row = await finish(graph(big_shop, sql=AMOUNTS), run_class="exploratory") + assert row.status == "succeeded", row.error + record = artifact_store.load((await output_of(row.id, "chal"))["challenge_artifact"]) + [claim] = [c for c in record["claims"] if FIRST in c["text"]] + assert (claim["verdict"], claim["rationale"]) == ("stands", "金额为两万的订单有 1 笔,与结论不矛盾。") + assert "20000" not in json.dumps(claim["rationale_doc"], ensure_ascii=False) diff --git a/backend/tests/test_inquiry_masking_e2e.py b/backend/tests/test_inquiry_masking_e2e.py index af233ad2..aa6c0043 100644 --- a/backend/tests/test_inquiry_masking_e2e.py +++ b/backend/tests/test_inquiry_masking_e2e.py @@ -86,3 +86,85 @@ async def test_the_excerpt_in_the_task_is_masked_before_the_agent_sees_it(monkey assert "手机号" not in columns and "金额" in columns assert "buyer_phone = '已遮罩'" in excerpt and "(有的列按数据源的设置遮罩了,没有给出)" in excerpt assert r.json()["inquiry"]["answer"]["ok"] is True + + +# -------------------------------------------------------------------------- +# 摘录分段遮:阈值、没设遮罩的数值格、被问那句里的值照原样 +# -------------------------------------------------------------------------- + + +#: 报告按金额阈值取数(12000 正好是某一笔被遮的金额),访问次数没设遮罩 +THRESHOLD = ("SELECT id AS 编号, region AS 区域, amount AS 金额, visits AS 访问 FROM sales WHERE amount >= 12000 " + "ORDER BY id") + + +async def test_the_excerpt_keeps_thresholds_and_unmasked_numbers(monkeypatch, big_shop, client): + """针对「华南首单访问 30,000 次。」发起质询:摘录里 SQL 的阈值(12000,恰好是一笔被遮的金额)照原样,访问次数(没设 + 遮罩,20000、30000 恰好等于别的订单的金额)照原样,金额列不给。以前对渲染好的整段按值换:「amount >= 已遮罩」 + 「r0:已遮罩 | 2 | 华东」,被问的那一格也成了「已遮罩」,任务开头的原句却写着 30,000。""" + models = Models(monkeypatch, big_shop, writer="华南首单访问 [[v:Q1.r1.访问]] 次。[[see:Q1]]", + plays={"访问为什么这么高": Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "共 [[v:Q1.r0.笔数]] 笔。")}) + paused = await pause(big_shop, graph=fetching(big_shop, THRESHOLD)) + [approval] = await pending(paused.id) + doc = await report_doc(paused.id) + r = await ask(client, paused.id, report="write", unit=unit_of(doc, "华南首单访问 30,000 次。"), + question="访问为什么这么高?", approval_id=approval.id) + assert r.status_code == 200, r.text + [task] = models.tasks + excerpt = task[task.index("【原报告 Q1】"):] + assert f"SQL:{THRESHOLD}" in excerpt, excerpt + assert "列:访问 | 编号 | 区域" in excerpt and "r0:20,000 | 2 | 华东" in excerpt and "r1:30,000 | 3 | 华南" in excerpt + + +#: 买家姓名设了遮罩;推荐人没设遮罩,写着的恰好是另一位买家的名字 +REFERRALS = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_name TEXT, referrer TEXT, amount INTEGER);" + "INSERT INTO sales VALUES (1, '华东', '张伟', '李娜', 1200), (2, '华南', '李娜', '张伟', 900)," + " (3, '华北', '王芳', '刘洋', 400);") + + +@pytest.fixture +async def referral_shop(tmp_path): + from app.data.engine import engines + from tests.test_inquiry_api import _source + + name, source_id = await _source(tmp_path, options={"mask_columns": "buyer_name"}, script=REFERRALS) + yield name + await engines.invalidate(source_id) + + +async def test_values_in_the_asked_sentence_stay_in_the_excerpt_and_the_answer(monkeypatch, referral_shop, client): + """被问的那句写着「李娜」(推荐人这一格,恰好也是另一笔的买家):它就在任务开头、摆在审批人眼前,摘录里这一格、回答里 + 复述的都照原样(同反方审稿的 _spared);别的买家名照旧换。以前质询不把这句原文算作已经摆在眼前的,这一格成了 + 「已遮罩」,和任务开头写着的原句对不上。""" + report = "SELECT buyer_name AS 买家, referrer AS 推荐人, amount AS 金额 FROM sales ORDER BY id" + said = "推荐人是李娜,第二单的推荐人是 [[v:Q1.r1.推荐人]]。[[see:Q1]]" + models = Models(monkeypatch, referral_shop, writer="首单由 [[v:Q1.r0.推荐人]] 推荐。[[see:Q1]]", + plays={"谁推荐的": Play(["SELECT id, referrer AS 推荐人 FROM sales ORDER BY id"], said)}) + paused = await pause(referral_shop, graph=fetching(referral_shop, report)) + [approval] = await pending(paused.id) + doc = await report_doc(paused.id) + r = await ask(client, paused.id, report="write", unit=unit_of(doc, "首单由 李娜 推荐。"), + question="谁推荐的?", approval_id=approval.id) + assert r.status_code == 200, r.text + [task] = models.tasks + excerpt = task[task.index("【原报告 Q1】"):] + assert "列:推荐人 | 金额" in excerpt and "r0:李娜 | 1,200" in excerpt and "r1:已遮罩 | 900" in excerpt, excerpt + item = r.json()["inquiry"] + assert item["answer"]["text"] == "推荐人是李娜,第二单的推荐人是 已遮罩。" + + +async def test_literals_compared_with_a_masked_column_are_masked_in_the_excerpt(monkeypatch, masked_shop, client): + """报告没选出号码,只在条件里排除了两个(一个换了写法):报告快照里收不到这两个值,摘录的 SQL 照样换掉。""" + report = ("SELECT region AS 区域, COUNT(*) AS 笔数 FROM sales WHERE buyer_phone NOT IN ('13900000002', " + "'139-0000-0003') GROUP BY region ORDER BY 笔数 DESC") + models = Models(monkeypatch, masked_shop, writer="笔数最多的区域有 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]", + plays={"按什么排除": Play([], "查不到判断所需的数据。")}) + paused = await pause(masked_shop, graph=fetching(masked_shop, report)) + [approval] = await pending(paused.id) + doc = await report_doc(paused.id) + r = await ask(client, paused.id, report="write", unit=unit_of(doc, "笔数最多的区域有 1 笔。"), + question="按什么排除的?", approval_id=approval.id) + assert r.status_code == 200, r.text + [task] = models.tasks + assert "13900000002" not in task and "0000-0003" not in task, task + assert "buyer_phone NOT IN ('已遮罩', '已遮罩')" in task diff --git a/backend/tests/test_masking_excerpts.py b/backend/tests/test_masking_excerpts.py new file mode 100644 index 00000000..a19605d8 --- /dev/null +++ b/backend/tests/test_masking_excerpts.py @@ -0,0 +1,111 @@ +"""交给核查 agent 的证据摘录分段遮(engine/inquiry.py 的 ExcerptMask、_scrub_excerpt_sql,engine/masking.py 的 +masked_cell): + +- SQL 照质询封存 SQL 的办法换:范围比较的阈值、LIMIT 的行数照原样(恰好等于某个遮罩值也一样),字符串、注释、名字里的 + 遮罩值换;和遮罩列(含改了名字的)做等值、IN、LIKE 一类比较的字面量不论认不认得出都换(报告没选出遮罩列时快照里收 + 不到这些值),同一个数写成阈值的跟着换;已经摆在读者眼前的值(问题、被问的那句)照旧不换 +- 格子逐格遮:数值格(数、列类型是 number 的)照原样,文字格整格的、夹着的那一截换掉(换了写法的号码也认) + +以前对渲染好的整段摘录按值换,分不出格子和 SQL 里的用法:阈值、没设遮罩的访问次数恰好等于某一笔被遮的金额,都成了 +「已遮罩」。端到端的用例在 test_challenge_masking、test_inquiry_masking_e2e。号码、姓名、金额都是虚构的。 +""" +from __future__ import annotations + +import pytest + +from app.engine import masking +from app.engine.inquiry import ExcerptMask, _scrub_excerpt_sql + +MASKED = "已遮罩" + + +def secrets_of(*values, protect: tuple[str, ...] = ()) -> masking._Secrets: + """报告快照里收到的遮罩值(金额、号码),去掉已经摆在读者眼前的(protect)。""" + secrets = masking._Secrets() + secrets.add({"columns": ["v"], "rows": [[v] for v in values]}, {"v"}) + secrets.protect(protect) + return secrets + + +AMOUNTS = secrets_of(20000, 30000, 50000) + + +@pytest.mark.parametrize("sql", [ + "SELECT id, visits FROM sales WHERE amount >= 20000 ORDER BY id", + "SELECT id FROM sales WHERE amount BETWEEN 20000 AND 50000 LIMIT 30000", + "SELECT region, COUNT(*) AS n FROM sales WHERE visits > 30000 GROUP BY region", +]) +def test_thresholds_and_limits_stay_as_written(sql): + """范围比较的边界、LIMIT 的行数是报告自己定的条件,恰好等于某一笔被遮的金额也照原样:agent 要看得到报告按什么取数。""" + assert _scrub_excerpt_sql(sql, AMOUNTS, {"amount"}, "sqlite") == sql + + +def test_a_value_the_report_looked_for_is_masked(): + """在找某一笔的(等值比较)照旧换:这是在点名一个遮罩值。""" + assert _scrub_excerpt_sql("SELECT id FROM sales WHERE amount = 50000", AMOUNTS, {"amount"}, "sqlite") \ + == f"SELECT id FROM sales WHERE amount = {MASKED}" + + +@pytest.mark.parametrize("sql, expected", [ + ("SELECT region FROM sales WHERE buyer_phone NOT IN ('13900000002', '139-0000-0003')", + f"SELECT region FROM sales WHERE buyer_phone NOT IN ('{MASKED}', '{MASKED}')"), + ("SELECT region FROM sales WHERE buyer_phone = '13900000009' OR buyer_phone LIKE '%0001'", + f"SELECT region FROM sales WHERE buyer_phone = '{MASKED}' OR buyer_phone LIKE '{MASKED}'"), + ("SELECT region FROM (SELECT region, buyer_phone AS 手机号 FROM sales) t WHERE 手机号 IN ('13900000002')", + f"SELECT region FROM (SELECT region, buyer_phone AS 手机号 FROM sales) t WHERE 手机号 IN ('{MASKED}')"), + ("SELECT region FROM sales WHERE region = '华东'", "SELECT region FROM sales WHERE region = '华东'"), +]) +def test_literals_compared_with_a_masked_column_are_masked_even_when_never_collected(sql, expected): + """和遮罩列(含改了名字的)做等值、IN、LIKE 比较的字面量:报告快照里收不到这些值(报告没选出号码),照样换掉;和没设 + 遮罩的列比较的照原样。""" + assert _scrub_excerpt_sql(sql, masking._Secrets(), {"buyer_phone", "手机号"}, "sqlite") == expected + + +def test_a_threshold_equal_to_a_masked_literal_goes_with_it(): + """同一个数既在找某一笔、又写成阈值:阈值留着就把换掉的值露了,跟着换(同封存 SQL 的 _literals)。""" + sql = "SELECT id FROM sales WHERE buyer_phone = 13900000009 OR id >= 13900000009" + assert _scrub_excerpt_sql(sql, masking._Secrets(), {"buyer_phone"}, "sqlite") \ + == f"SELECT id FROM sales WHERE buyer_phone = {MASKED} OR id >= {MASKED}" + + +def test_values_already_in_front_of_the_reader_are_not_masked(): + """提问人、被问的那句里写着的值(protect)照旧不换,换了写法写的也认得出。""" + secrets = secrets_of(protect=("139-0000-0009 这位客户的单",)) + sql = "SELECT id FROM sales WHERE buyer_phone = '13900000009' OR buyer_phone = '13900000008'" + assert _scrub_excerpt_sql(sql, secrets, {"buyer_phone"}, "sqlite") \ + == f"SELECT id FROM sales WHERE buyer_phone = '13900000009' OR buyer_phone = '{MASKED}'" + + +def test_strings_and_comments_are_masked_by_value(): + """字符串、注释里写着的遮罩值按值换(换了写法的号码也认),同封存 SQL。""" + secrets = secrets_of("13900000001") + sql = "SELECT id FROM sales /* 客户 139-0000-0001 */ WHERE note LIKE '%13900000001%'" + assert _scrub_excerpt_sql(sql, secrets, set(), "sqlite") \ + == f"SELECT id FROM sales /* 客户 {MASKED} */ WHERE note LIKE '%{MASKED}%'" + + +@pytest.mark.parametrize("value, numeric, expected", [ + (20000, False, 20000), + (20000.0, False, 20000.0), + ("20000.00", True, "20000.00"), + ("20000", False, MASKED), + ("上月消费 20000 元", False, f"上月消费 {MASKED} 元"), + ("回访:联系 139-0000-0003", False, f"回访:联系 {MASKED}"), + ({"phone": "13900000003"}, False, '{"phone": "' + MASKED + '"}'), + ("例行回访", False, "例行回访"), + (None, False, None), +]) +def test_cells_are_masked_like_the_tool_boundary(value, numeric, expected): + """格子同工具边界:数(整数、小数,列类型是 number 的字符串)照原样;文字格整格是遮罩值的、夹着的那一截换掉,JSON + 按画出来的样子换。""" + secrets = secrets_of(20000, "13900000003") + assert masking.masked_cell(value, secrets, numeric=numeric) == expected + + +def test_an_excerpt_mask_without_values_only_masks_by_column(): + """没有可按值换的(报告快照里没收到值):格子、文字照原样,SQL 里和遮罩列做等值比较的照样换。""" + mask = ExcerptMask({"shop": "sqlite"}) + assert mask.cell("13900000001", numeric=False) == "13900000001" + assert mask.text("客户 13900000001") == "客户 13900000001" + assert mask.sql("SELECT 1 FROM sales WHERE buyer_phone = '13900000001'", {"source": "shop"}, {"buyer_phone"}) \ + == f"SELECT 1 FROM sales WHERE buyer_phone = '{MASKED}'" diff --git a/backend/tests/test_masking_opaque_sql.py b/backend/tests/test_masking_opaque_sql.py index 5e58f16a..4e81a02f 100644 --- a/backend/tests/test_masking_opaque_sql.py +++ b/backend/tests/test_masking_opaque_sql.py @@ -140,11 +140,13 @@ async def composite_shop(): [[1, PHONE, "某路 1 号"]]), ("SELECT p.id AS n, (jsonb_populate_record(NULL::contact, p.pj)).* FROM people2 p", ["n", "tel", "addr"], [[1, PHONE, "某路 1 号"]]), -], ids=["复合类型", "子查询里的复合类型", "按类型展开 JSON"]) + ("SELECT p.id AS n, ((p.profile).*) FROM people p", ["n", "tel", "addr"], [[1, PHONE, "某路 1 号"]]), +], ids=["复合类型", "子查询里的复合类型", "按类型展开 JSON", "外面套了括号"]) async def test_an_expression_star_never_reaches_the_model_or_the_store(monkeypatch, composite_shop, sql, columns, rows): """端到端(核查 agent 的真实查询工具):交给模型的、落库的、通用工件接口给的都只有「已遮罩」,masked_at_query 是 - 全部的列。以前三条的 masked_at_query 都是空的,号码原样交给模型、落库。""" + 全部的列,快照记着整份遮(masked_whole),格子引用照实说写法认不全。以前前三条的 masked_at_query 都是空的,套了外括号 + 的只遮了第一列,号码原样交给模型、落库。""" from app.data.engine import QueryResult from app.tools import datasource @@ -162,6 +164,8 @@ async def test_an_expression_star_never_reaches_the_model_or_the_store(monkeypat assert MASKED_WORD in seen and "1390000000" not in seen, seen snapshot = artifact_store.load(out.queries[0]["artifact"]) assert snapshot["masked_at_query"] == columns and "1390000000" not in json.dumps(snapshot, ensure_ascii=False) + assert snapshot["masked_whole"] is True and "masked_whole" not in seen + assert out.cells[0]["reason"] == "Q1 的写法认不全,整份结果查询时已遮罩,不采信" async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: got = await client.get(f"/api/artifacts/{out.queries[0]['artifact']}") assert got.status_code == 200 and "1390000000" not in got.text @@ -280,3 +284,229 @@ def test_a_failing_analysis_masks_the_whole_result_instead_of_reading_the_text(m monkeypatch.setattr(masking, "_tainted_in", broken) snapshot = {"sql": "SELECT s AS r FROM sales s", "columns": ["r"], "rows": []} assert "r" in masking._derived_hidden(snapshot, {"buyer_phone"}, "postgresql")[0] + + +# -------------------------------------------------------------------------- +# (表达式).* 套了外括号、起了别名 +# -------------------------------------------------------------------------- + + +#: Postgres 实测:((p.card).*)、(((p.card)).*)、(((p.card).*))、(p.card).* AS x 都照样展开成好几列(别名不起作用) +WRAPPED = [ + pytest.param("SELECT ((p.profile).*) FROM people p", ["tel", "addr"], [PHONE, "某路 1 号"], id="复合类型-外括号"), + pytest.param("SELECT (((p.profile)).*) FROM people p", ["tel", "addr"], [PHONE, "某路 1 号"], + id="复合类型-两层外括号"), + pytest.param("SELECT (((p.profile).*)) FROM people p", ["tel", "addr"], [PHONE, "某路 1 号"], + id="复合类型-套两层"), + pytest.param("SELECT (p.profile).* AS x FROM people p", ["tel", "addr"], [PHONE, "某路 1 号"], id="复合类型-别名"), + pytest.param("SELECT ((jsonb_each_text(pj)).*) FROM people2", ["key", "value"], ["tel", PHONE], + id="记录函数-外括号"), + pytest.param("SELECT ((q.r).*) FROM (SELECT ROW(region, buyer_phone) AS r FROM sales) q", ["f1", "f2"], + ["华东", PHONE], id="匿名 ROW-外括号"), + pytest.param("SELECT ((jsonb_each_text(pj)).*) FROM people2 UNION ALL SELECT 'a', 'b'", ["key", "value"], + ["tel", PHONE], id="UNION 第一支"), + pytest.param("SELECT 'a' AS k, 'b' AS v UNION ALL SELECT ((jsonb_each_text(pj)).*) FROM people2", ["k", "v"], + ["tel", PHONE], id="UNION 后一支"), + pytest.param("SELECT 'a' AS k, 'b' AS v UNION ALL SELECT ((p.profile).*) FROM people p", ["k", "v"], + [PHONE, "某路 1 号"], id="UNION 后一支-复合类型"), + pytest.param("SELECT x.* FROM (SELECT ((q.r).*) FROM (SELECT ROW(buyer_phone, region) AS r FROM sales) q) x", + ["f1", "f2"], [PHONE, "华东"], id="子查询里-外括号"), + # 和 * 写在同一支里:选出项和结果的列数本来就对不上(* 展开成几列),只能靠认出外括号里的 (表达式).* + pytest.param("SELECT p.*, ((p.profile).*) FROM people p", ["id", "profile", "address", "tel", "addr"], + [1, f"({PHONE},某路 1 号)", "某路 1 号", PHONE, "某路 1 号"], id="和 * 写在一起-外括号"), + pytest.param("SELECT p.*, (p.profile).* AS x FROM people p", ["id", "profile", "address", "tel", "addr"], + [1, f"({PHONE},某路 1 号)", "某路 1 号", PHONE, "某路 1 号"], id="和 * 写在一起-别名"), +] + + +@pytest.mark.parametrize("sql, columns, row", WRAPPED) +def test_a_wrapped_expression_star_reading_a_masked_column_masks_the_whole_result(sql, columns, row): + """(表达式).* 外面套了括号、起了别名,Postgres 照样展开成好几列:整份遮。以前只认不带外括号的写法,((p.card).*) 按一项 + 数,只遮了第一列,号码原样交给模型、落库。""" + done = masked(sql, columns, [list(row)]) + assert done.whole and done.columns == columns + assert done.payload["rows"] == [[MASKED_WORD] * len(columns)] and not _leaks(done) + + +def test_a_wrapped_expression_star_with_a_numeric_phone_is_masked_whole(): + """号码存成整数(bigint)、前面的查询已经认出它:数值格不按值换,(表达式).* 认不出就全靠整份遮挡住。""" + known = masking._Secrets() + known.add({"columns": ["p"], "rows": [[13900000001]]}, {"p"}) + payload = {"columns": ["f1", "f2"], "rows": [[1200, 13900000001]], + "sql": "SELECT ((q.r).*) FROM (SELECT ROW(amount, buyer_phone) AS r FROM sales) q"} + done = masking.mask_query_result(payload, MASKED, "postgresql", known, schema=SCHEMA) + assert done.payload["rows"] == [[MASKED_WORD, MASKED_WORD]] + + +@pytest.mark.parametrize("sql", ["SELECT ((s).*) FROM sales s", "SELECT (s.*) AS x FROM sales s"]) +def test_a_wrapped_source_star_is_still_read_as_its_columns(sql): + """((s).*)、(s.*) AS x、s 是 FROM 里的来源:照旧当 s.* 认,只遮号码那一列。""" + done = masked(sql, ["id", "region", "buyer_phone", "amount"], [[1, "华东", PHONE, 1200]]) + assert not done.whole and done.columns == ["buyer_phone"] + assert done.payload["rows"] == [[1, "华东", MASKED_WORD, 1200]] + + +# -------------------------------------------------------------------------- +# 和数据源里的表同名的 CTE +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("sql, columns, row", [ + pytest.param("WITH sales AS (SELECT * FROM sales WHERE region = '华东') SELECT row_to_json(sales) AS j FROM sales", + ["j"], [json.dumps({"id": 1, "buyer_phone": PHONE})], id="整行引用"), + pytest.param("WITH sales AS (SELECT * FROM sales WHERE region = '华东') SELECT s AS r FROM sales s", ["r"], + [f"(1,华东,{PHONE},1200)"], id="选出整行"), + pytest.param("WITH orders AS (SELECT * FROM sales), sales AS (SELECT * FROM sales WHERE amount > 0) " + "SELECT to_jsonb(x) AS j FROM sales x", ["j"], [json.dumps({"buyer_phone": PHONE})], + id="排在后面的同名段"), + pytest.param("WITH sales AS (SELECT * FROM sales) SELECT * FROM sales t(a, b, c, d)", ["a", "b", "c", "d"], + [1, "华东", PHONE, 1200], id="列名表"), +]) +def test_a_cte_named_like_a_table_reads_the_table(sql, columns, row): + """WITH sales AS (SELECT * FROM sales WHERE …) 里的 sales 是数据源里的表,不是这一段自己(不带 RECURSIVE 的 WITH 里, + 一段只看得见排在它前面的段)。以前认成它自己,* 展开绕回自己什么列都不带出,号码原样交了出去。""" + done = masked(sql, columns, [list(row)]) + assert done.columns and not _leaks(done) + assert done.payload["rows"] == [[MASKED_WORD if c in done.columns else v for c, v in zip(columns, row)]] + + +def test_a_cte_named_like_a_table_masks_like_any_other_name(): + """对照:同一段 CTE 改叫 t,同样遮;列名表按表结构对上位置,只遮号码那一列。""" + for sql in ("WITH t AS (SELECT * FROM sales WHERE region = '华东') SELECT row_to_json(t) AS j FROM t", + "WITH sales AS (SELECT * FROM sales WHERE region = '华东') SELECT row_to_json(sales) AS j FROM sales"): + assert masked(sql, ["j"], [[json.dumps({"buyer_phone": PHONE})]]).columns == ["j"] + done = masked("WITH sales AS (SELECT * FROM sales) SELECT * FROM sales t(a, b, c, d)", ["a", "b", "c", "d"], + [[1, "华东", PHONE, 1200]]) + assert done.columns == ["c"] + + +def test_a_recursive_cte_still_sees_itself(): + """带 RECURSIVE 的 WITH 里递归的那一支指向它自己:取自号码的那一列照遮,只数步数的照常。""" + body = ("WITH RECURSIVE t(n, p) AS (SELECT 1, buyer_phone FROM sales UNION ALL SELECT n + 1, p FROM t " + "WHERE n < 3) ") + assert masked(body + "SELECT n FROM t", ["n"], [[1]]).columns == [] + assert masked(body + "SELECT p FROM t", ["p"], [[PHONE]]).columns == ["p"] + + +# -------------------------------------------------------------------------- +# 语法树和实际执行的对不上、存着值的视图:和可验证报告共用一份名单 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("sql, kind, columns", [ + pytest.param("SELECT id /*!, buyer_phone AS x */ FROM sales", "mysql", ["id", "x"], id="MySQL 会执行的注释"), + pytest.param("SELECT /*! buyer_phone AS */ region FROM sales", "mysql", ["region"], + id="MySQL 会执行的注释改了列的来历"), + pytest.param('SELECT U&"\\0062uyer_phone" AS x FROM sales', "postgresql", ["x"], id="U&转义的名字"), + pytest.param("SELECT id, U&\"buyer!005fphone\" UESCAPE '!' AS p FROM sales", "postgresql", ["id", "p"], + id="U&带 UESCAPE"), + pytest.param("SELECT query FROM pg_stat_get_activity(NULL)", "postgresql", ["query"], id="pg_stat_get_activity"), + pytest.param("SELECT data FROM pg_logical_slot_peek_changes('s', NULL, NULL)", "postgresql", ["data"], + id="逻辑解码"), + pytest.param("SELECT HISTOGRAM AS h FROM information_schema.COLUMN_STATISTICS WHERE TABLE_NAME = 'sales'", "mysql", + ["h"], id="MySQL 直方图"), + pytest.param("SELECT sql_text AS v FROM performance_schema.events_statements_history_long", "mysql", ["v"], + id="MySQL 语句历史"), + pytest.param("SELECT info AS v FROM information_schema.PROCESSLIST", "mysql", ["v"], id="MySQL 会话列表"), + pytest.param("SELECT argument AS v FROM mysql.general_log", "mysql", ["v"], id="MySQL 通用日志"), + pytest.param("SELECT min_value AS v FROM mysql.column_stats", "mariadb", ["v"], id="MariaDB 列统计"), + pytest.param("SELECT sql_fulltext AS v FROM v$sql", "oracle", ["v"], id="Oracle V$SQL"), + pytest.param("SELECT value_string AS v FROM v$sql_bind_capture", "oracle", ["v"], id="Oracle 绑定变量"), + pytest.param("SELECT endpoint_actual_value AS v FROM all_tab_histograms WHERE table_name = 'SALES'", "oracle", + ["v"], id="Oracle 直方图"), + pytest.param("SELECT * FROM all_tab_columns WHERE table_name = 'SALES'", "oracle", ["column_name", "low_value"], + id="Oracle 列的最小最大值"), + pytest.param("SELECT query_to_xml/**/('select * from sales', true, false, '') AS x", "postgresql", ["x"], + id="函数名后夹块注释"), + pytest.param("SELECT query_to_xml --c\n('select * from sales', true, false, '') AS x", "postgresql", ["x"], + id="函数名后夹行注释"), + pytest.param("SELECT id FROM sales", "mysql", ["id", "x"], id="选出项和结果列数对不上"), +]) +def test_writings_the_analysis_cannot_follow_mask_the_whole_result(sql, kind, columns): + """语法树和实际执行的对不上(MySQL 会执行的注释、U&"…"、选出项和结果的列数对不上),统计信息、会话记录里存着值的视图和 + 函数,函数名和括号之间夹着注释:整份遮。以前工具边界的名单比可验证报告的少一大截,这些写法号码原样交给模型、落库。""" + row = [PHONE] * len(columns) + done = masked(sql, columns, [row], kind=kind) + assert done.whole and done.payload["rows"] == [[MASKED_WORD] * len(columns)] + + +@pytest.mark.parametrize("sql, columns, row", [ + pytest.param("WITH query(r, n) AS (SELECT region, COUNT(*) FROM sales GROUP BY region) SELECT r AS region, n " + "FROM query", ["region", "n"], ["华东", 2], id="叫 query 的 CTE"), + pytest.param("SELECT region, 'Query (x)' AS q FROM sales", ["region", "q"], ["华东", "Query (x)"], + id="字符串里的 Query ("), + pytest.param("SELECT column_name FROM information_schema.columns WHERE table_name = 'sales'", ["column_name"], + ["buyer_phone"], id="information_schema 的列清单"), +]) +def test_names_of_unsupported_databases_and_plain_schema_views_are_not_masked(sql, columns, row): + """不支持的库(DuckDB 的 query()、SQL Server 的 OPENQUERY)的函数不再收:名叫 query 的 CTE、字符串里的「Query (x)」不再 + 整份遮;information_schema 里只有表结构的视图照常(核查 agent 查列名、类型常用)。""" + done = masked(sql, columns, [list(row)]) + assert not done.whole and done.payload["rows"] == [list(row)] + + +def test_the_tool_boundary_and_the_bundle_share_one_list(): + """工具边界和可验证报告共用一份名单:报告那边另把系统 schema 整个挡(只决定附不附快照)。""" + from app.bundle import build + + assert build._OPAQUE_SQL is masking.OPAQUE_SQL + for sql in ("SELECT HISTOGRAM FROM information_schema.COLUMN_STATISTICS", "SELECT * FROM v$sql", + "SELECT data FROM pg_logical_slot_peek_changes('s', NULL, NULL)"): + assert build._INDIRECT_READS.search(sql) and masking._INDIRECT.search(sql) + plain = "SELECT column_name FROM information_schema.columns" + assert build._INDIRECT_READS.search(plain) and not masking._INDIRECT.search(plain) + + +@pytest.mark.parametrize("sql", [ + "SELECT row_to_json(s) AS j FROM sales s ORDER BY s.id USING >", + "SELECT s AS j FROM sales s WHERE s.region IS NOT NFC NORMALIZED OR true", + "SELECT to_jsonb(s) - 'id' AS j FROM sales s ORDER BY s.id USING <", +]) +def test_postgres_sql_that_cannot_be_read_masks_the_whole_result(sql): + """Postgres 合法、解析器解析不了的写法:整行引用不写列名、也没有 *,只看文字认不出,整份遮。以前 masked=[],整行连同 + 号码交给模型、落库。""" + done = masked(sql, ["j"], [[json.dumps({"id": 1, "buyer_phone": PHONE})]]) + assert done.whole and not _leaks(done) + + +# -------------------------------------------------------------------------- +# 整份遮的结果不把每一格都收成遮罩值 +# -------------------------------------------------------------------------- + + +def test_a_whole_masked_result_does_not_turn_every_cell_into_a_masked_value(): + """统计视图整份遮之后,里面的列名、区域名不收成遮罩值:后一条查询的区域照常,封存的 SQL 照原样。以前每一格都收, + 区域名、列名在同一次核查里处处被换掉,封存的 SQL 成了「WHERE 已遮罩 = '华东'」。""" + from app.engine.inquiry import _scrub_sql + + stats = masked("SELECT attname, n_distinct FROM pg_stats WHERE tablename = 'sales'", ["attname", "n_distinct"], + [["amount", 5], ["buyer_phone", -1], ["region", 3]]) + assert stats.whole and not stats.secrets.strong + payload = {"columns": ["id", "region", "amount"], "rows": [[1, "华东", 1200]], + "sql": "SELECT id, region, amount FROM sales ORDER BY id"} + later = masking.mask_query_result(payload, MASKED, "postgresql", stats.secrets, schema=SCHEMA) + assert later.payload["rows"] == [[1, "华东", 1200]] + sql = "SELECT region, COUNT(*) AS n FROM sales WHERE region = '华东' GROUP BY region" + assert _scrub_sql(sql, stats.secrets, {"buyer_phone"}, "postgresql") == sql + + +def test_a_whole_masked_result_still_collects_what_surely_came_from_a_masked_column(): + """整份遮时,认得出取自遮罩列的那几列照收(号码改了名字选出来、又夹着认不全的写法),别的列不收。""" + sql = "SELECT buyer_phone AS 手机号, region FROM sales UNION ALL SELECT query_to_xml('x', true, false, ''), 'y'" + done = masked(sql, ["手机号", "region"], [[PHONE, "华东"]]) + assert done.whole and done.secrets.hit(PHONE) and not done.secrets.hit("华东") + + +def test_report_snapshots_that_were_masked_whole_give_only_their_sure_values(): + """报告自己的查询快照(报告取数时没遮)收遮罩值时同一条规则:写法认不全的只收确定取自遮罩列的那几列,区域名、列名不收 + (审批质询、反方审稿把它们交给查询工具、拿来换回答和理由,收了就处处被换掉)。""" + from app.engine.inquiry import _add_based + + secrets = masking._Secrets() + stats = {"source": "pg", "sql": "SELECT attname, n_distinct FROM pg_stats WHERE tablename = 'sales'", + "columns": ["attname", "n_distinct"], "rows": [["region", 3], ["buyer_phone", -1]]} + mixed = {"source": "pg", "columns": ["手机号", "region"], "rows": [[PHONE, "华东"]], + "sql": "SELECT buyer_phone AS 手机号, region FROM sales UNION ALL SELECT query_to_xml('x', true, false, ''), " + "'y'"} + _add_based(secrets, [(stats, set(MASKED)), (mixed, set(MASKED))], {"pg": "postgresql"}) + assert secrets.hit(PHONE) and not secrets.hit("华东") and not secrets.hit("region") diff --git a/backend/tests/test_masking_tool_boundary.py b/backend/tests/test_masking_tool_boundary.py index acb2b2b4..9f2797ad 100644 --- a/backend/tests/test_masking_tool_boundary.py +++ b/backend/tests/test_masking_tool_boundary.py @@ -45,7 +45,7 @@ PAYLOAD = {"sql": "SELECT id, buyer_phone, note FROM sales", "columns": ["id", " "column_types": {"id": "number", "buyer_phone": "text", "note": "text"}} -@pytest.mark.parametrize("step", ["_derived_hidden", "_snapshot_labels", "_masked_view"]) +@pytest.mark.parametrize("step", ["_derived_masks", "_snapshot_labels", "_masked_view"]) def test_a_failing_analysis_masks_every_cell(monkeypatch, step): """认遮罩列、认标签列、逐格换值,哪一步出了意外都整份遮:每一格都是「已遮罩」、类型记成文本,整列遮掉的是全部的列。""" monkeypatch.setattr(masking, step, _broken) @@ -55,6 +55,7 @@ def test_a_failing_analysis_masks_every_cell(monkeypatch, step): assert done.columns == ["id", "buyer_phone", "note"] assert done.payload["sql"] == PAYLOAD["sql"], "SQL 是模型自己写的,照原样" assert not done.secrets.strong, "认不准,不收值" + assert done.whole, "快照记下是整份遮的" def test_a_failing_analysis_masks_rows_of_any_shape(monkeypatch): @@ -66,10 +67,10 @@ def test_a_failing_analysis_masks_rows_of_any_shape(monkeypatch): assert done.payload["rows"] == [{"id": MASKED, "buyer_phone": MASKED}] -@pytest.mark.parametrize("step", ["_derived_hidden", "_masked_view"]) +@pytest.mark.parametrize("step", ["_derived_masks", "_masked_view"]) async def test_the_model_and_the_snapshot_see_only_masks_when_the_analysis_fails(monkeypatch, masked_shop, step): """端到端:分析出了意外,交给模型的工具结果、落库的快照、通用接口给的都只有「已遮罩」,masked_at_query 是全部的列; - 格子引用照实说是遮罩列,不采信。""" + 格子引用照实说写法认不全、整份遮了,不采信——金额在这里没设遮罩,不说成「取自设置了遮罩的列」。""" monkeypatch.setattr(masking, step, _broken) name = masked_shop sql = "SELECT id, region, buyer_phone AS 手机号, amount FROM sales ORDER BY id" @@ -80,13 +81,14 @@ async def test_the_model_and_the_snapshot_see_only_masks_when_the_analysis_fails assert seen["rows"] == [[MASKED] * 4] * 4, seen snapshot = artifact_store.load(out.queries[0]["artifact"]) assert snapshot["rows"] == [[MASKED] * 4] * 4 - assert snapshot["masked_at_query"] == ["id", "region", "手机号", "amount"] + assert snapshot["masked_at_query"] == ["id", "region", "手机号", "amount"] and snapshot["masked_whole"] is True + assert "masked_whole" not in seen, "记录只在快照里,不交给模型" assert _clean(json.dumps(snapshot, ensure_ascii=False)) async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: got = await client.get(f"/api/artifacts/{out.queries[0]['artifact']}") assert got.status_code == 200 and _clean(got.text) and "1200" not in got.text assert out.result["value"] is None - assert out.cells[0]["reason"] == "Q1 的「amount」取自设置了遮罩的列,查询时已遮罩,不采信" + assert out.cells[0]["reason"] == "Q1 的写法认不全,整份结果查询时已遮罩,不采信" # -------------------------------------------------------------------------- @@ -250,3 +252,121 @@ def test_without_a_schema_a_star_next_to_the_band_is_still_masked(): payload = {"sql": sql, "columns": ["档位", "笔数"], "rows": [list(r) for r in ROWS]} done = masking.mask_query_result(payload, ["amount"], "sqlite") assert done.columns == ["档位"] and done.payload["rows"] == [[MASKED, 3], [MASKED, 2]] + + +# -------------------------------------------------------------------------- +# 数值格按位置认:同名的列、写成字符串的数 +# -------------------------------------------------------------------------- + + +def _known(*values: str) -> masking._Secrets: + """此前认出的遮罩值(前面的查询遮掉的、报告快照里的)。""" + secrets = masking._Secrets() + secrets.add({"columns": ["v"], "rows": [[v] for v in values]}, {"v"}) + return secrets + + +def test_a_text_column_sharing_its_name_with_a_number_column_is_still_masked_by_value(): + """column_types 按列名记,同名的几列只留下后面那一列的类型:两张表里都叫 phone,前一列是文字(写着号码)、后一列是 + 整数,记成 number。以前按列名认数值列,文字列也当成了数值列,号码原样交给模型、落库;现在同名的几列按格子本身认, + 文字格照换、记进 masked_cells,真正的整数照原样。写成字符串的数在同名列里也按文字处理(宁可多遮)。""" + payload = {"sql": "SELECT a.region, a.phone, b.phone FROM contacts a JOIN legacy b ON a.id = b.id", + "columns": ["region", "phone", "phone"], + "rows": [["华东", "联系人 13900000001", 13900000002], ["华南", "13900000001", "13900000001"]], + "column_types": {"region": "text", "phone": "number"}} + done = masking.mask_query_result(payload, [], "sqlite", _known("13900000001")) + assert done.payload["rows"] == [["华东", f"联系人 {MASKED}", 13900000002], ["华南", MASKED, MASKED]] + assert done.cells == [[0, 1], [1, 1], [1, 2]] + + +def test_a_numeric_column_written_as_decimal_strings_is_kept(): + """Postgres 的 NUMERIC、MySQL 的 DECIMAL 交回来是字符串("20000.00"),列类型记着 number:没设遮罩的这一列正好等于某个 + 遮罩值(金额也是 NUMERIC)也照原样,不记进 masked_cells——数值证据不改写。""" + payload = {"sql": "SELECT id, visits FROM sales2 ORDER BY id", "columns": ["id", "visits"], + "rows": [[1, "20000.00"], [2, "9000.00"]], "column_types": {"id": "number", "visits": "number"}} + done = masking.mask_query_result(payload, [], "postgresql", _known("20000.00", "30000.00")) + assert done.payload["rows"] == [[1, "20000.00"], [2, "9000.00"]] and done.cells == [] + + +#: 两张表里都有 remark:工单的是文字(写着号码),评分的是整数 +DUPLICATES = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_phone TEXT, amount INTEGER);" + "INSERT INTO sales VALUES (1, '华东', '13900000001', 1200), (2, '华南', '13900000002', 900);" + "CREATE TABLE tickets (id INTEGER PRIMARY KEY, remark TEXT);" + "INSERT INTO tickets VALUES (1, '客户 13900000001 来电投诉'), (2, '回访 13900000002');" + "CREATE TABLE scores (id INTEGER PRIMARY KEY, remark INTEGER);" + "INSERT INTO scores VALUES (1, 5), (2, 4);") + + +@pytest.fixture +async def ticket_shop(tmp_path): + name, source_id = await _source(tmp_path, {"mask_columns": "buyer_phone"}, script=DUPLICATES) + yield name + await engines.invalidate(source_id) + + +@pytest.mark.parametrize("sql", [ + "SELECT t.remark, s.remark FROM tickets t JOIN scores s ON s.id = t.id ORDER BY t.id", + "SELECT * FROM tickets t JOIN scores s ON s.id = t.id ORDER BY t.id", +]) +async def test_duplicate_names_do_not_let_numbers_shield_text(monkeypatch, ticket_shop, sql): + """端到端:先查出号码(查询时遮掉),再把工单和评分连起来查,两列都叫 remark(* 连两张表时 id 也同名)。工单备注里的 + 号码交给模型、落库之前照样换掉,评分照原样;以前评分的 number 套到了同名的备注上,号码原样交了出去。""" + name = ticket_shop + steps = [q(name, "SELECT id, buyer_phone FROM sales ORDER BY id", "c1"), q(name, sql, "c2")] + script = Script(monkeypatch, steps, {"value": None, "rationale": "x"}) + out, _ = await ask(name) + seen = script.tool_results()["c2"] + assert "1390000000" not in seen and f"客户 {MASKED} 来电投诉" in seen, seen + snapshot = artifact_store.load(out.queries[1]["artifact"]) + remarks = [i for i, c in enumerate(snapshot["columns"]) if c == "remark"] + assert [[row[i] for i in remarks] for row in snapshot["rows"]] == [[f"客户 {MASKED} 来电投诉", 5], + [f"回访 {MASKED}", 4]] + assert snapshot["masked_cells"] == [[0, remarks[0]], [1, remarks[0]]] + + +async def test_a_decimal_column_is_cited_as_a_number(monkeypatch, composite_numbers): + """端到端:NUMERIC 的访问次数(字符串 "20000.00"、列类型 number)正好等于某一笔被遮的金额:交给模型的、落库的照原样, + 格子引用照常取到 20000(双盲复算拿得到它)。""" + name, sql = composite_numbers + script = Script(monkeypatch, [q(name, "SELECT id, amount FROM sales ORDER BY id", "c1"), q(name, sql, "c2")], + {"value": {"alias": "Q2", "row": 0, "column": "visits"}, "rationale": "x"}) + out, _ = await ask(name) + seen = json.loads(script.tool_results()["c2"].split("\n", 1)[1]) + assert seen["rows"] == [[1, "20000.00"], [2, "9000.00"]] + snapshot = artifact_store.load(out.queries[1]["artifact"]) + assert snapshot["rows"] == seen["rows"] and "masked_cells" not in snapshot + assert out.result["value"] == 20000 and out.cells[0]["status"] == "resolved" + + +@pytest.fixture +async def composite_numbers(monkeypatch): + """造一个 Postgres 数据源(查询交给换掉的 run_query,不连库):金额设了遮罩,金额、访问次数都是 NUMERIC,驱动交回 + Decimal,查询层转成字符串、列类型记 number。交回 (源名, 查访问次数的 SQL)。""" + import uuid + + from app.data.engine import QueryResult + from app.db.base import SessionLocal + from app.db.models import DataSource + from app.tools import datasource + + visits = "SELECT id, visits FROM sales ORDER BY id" + results = {"SELECT id, amount FROM sales ORDER BY id": (["id", "amount"], [[1, "20000.00"], [2, "30000.00"]]), + visits: (["id", "visits"], [[1, "20000.00"], [2, "9000.00"]])} + + async def driver(source, statement, **kwargs): + columns, rows = results[statement] + return QueryResult(columns=columns, rows=[list(r) for r in rows], row_count=len(rows), truncated=False, + elapsed_ms=1, sql=statement, column_types={c: "number" for c in columns}) + + monkeypatch.setattr(datasource, "run_query", driver) + name = f"pg_{uuid.uuid4().hex[:8]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="postgresql", host="127.0.0.1", port=5432, username="probe", database="shop", + readonly=True, enabled=True, options={"mask_columns": "amount"}, + schema_cache={"tables": {"sales": {"columns": [{"name": c} for c in ("id", "amount", + "visits")]}}}) + session.add(row) + await session.commit() + source_id = row.id + yield name, visits + await engines.invalidate(source_id) diff --git a/backend/tests/test_masking_whole_rows.py b/backend/tests/test_masking_whole_rows.py index b3fc11a3..9db5aa1f 100644 --- a/backend/tests/test_masking_whole_rows.py +++ b/backend/tests/test_masking_whole_rows.py @@ -111,8 +111,9 @@ def test_a_whole_row_of_an_outer_query_is_found_from_a_nested_derived_table(sql) "SELECT EXISTS(SELECT * FROM sales s2 WHERE s2.id = s.id) AS j FROM sales s", ]) def test_rows_and_stars_that_bring_no_masked_values_stay_visible(sql): - """整行取自没有遮罩列的子查询、count(s.*) 只数个数:不算。""" - assert hidden_of(sql, ["j", "total"]) == set(MASKED) + """整行取自没有遮罩列的子查询、count(s.*) 只数个数:不算。结果的列数照选出项数(列数对不上的整份遮,见 + test_masking_opaque_sql)。""" + assert hidden_of(sql, ["j", "total"] if "total" in sql else ["j"]) == set(MASKED) def test_a_column_named_like_its_table_is_a_column_where_the_schema_says_so(): @@ -188,8 +189,11 @@ def test_sql_that_cannot_be_read_but_has_a_star_or_a_table_statement_hides_every "SELECT amount * 2 AS j FROM sales WHERE (", ]) def test_sql_that_cannot_be_read_with_only_a_count_or_a_product_shows_the_rest(sql): - """只有 COUNT(*)、乘号:不会把遮罩列带出来,照旧只遮遮罩列本身。""" - assert hidden_of(sql, ["j"]) == set(MASKED) + """只有 COUNT(*)、乘号:不会把遮罩列带出来,照旧只遮遮罩列本身(没有整行引用的方言)。Postgres 里解析不了的一律 + 整份遮:整行引用(row_to_json(s)、SELECT s)不写列名、也没有 *,只看文字认不出(见 test_masking_opaque_sql)。""" + assert hidden_of(sql, ["j"], kind="sqlite") == set(MASKED) + assert hidden_of(sql, ["j"], kind="mysql") == set(MASKED) + assert hidden_of(sql, ["j"]) == set(MASKED) | {"j"} # -------------------------------------------------------------------------- From 2b5989ef008bb2451282c9708ec95916c4f72df4 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 12 Oct 2026 10:50:40 +0800 Subject: [PATCH 11/12] =?UTF-8?q?K3b=20=E5=A4=8D=E6=A0=B8=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=EF=BC=88=E7=AC=AC=202=20=E8=BD=AE=EF=BC=89=EF=BC=9AVA?= =?UTF-8?q?LUES=E3=80=81=E8=A1=8C=E6=9E=84=E9=80=A0=E5=99=A8=E9=87=8C?= =?UTF-8?q?=E7=9A=84=20(=E8=A1=A8=E8=BE=BE=E5=BC=8F).*=20=E5=92=8C?= =?UTF-8?q?=E4=BA=A4=E5=9B=9E=E5=A4=8D=E5=90=88=E7=B1=BB=E5=9E=8B=E7=9A=84?= =?UTF-8?q?=E8=A1=A8=E5=80=BC=E5=87=BD=E6=95=B0=E6=95=B4=E4=BB=BD=E9=81=AE?= =?UTF-8?q?=EF=BC=9B=E8=AE=B0=E7=9D=80=E5=88=AB=E7=9A=84=E4=BC=9A=E8=AF=9D?= =?UTF-8?q?=20SQL=20=E7=9A=84=E8=A7=86=E5=9B=BE=E3=80=81=E4=B8=8D=E5=8F=AA?= =?UTF-8?q?=E8=AF=BB=E8=A1=A8=E7=BB=93=E6=9E=84=E7=9A=84=20SHOW=20?= =?UTF-8?q?=E5=92=8C=20EXPLAIN=20=E6=95=B4=E4=BB=BD=E9=81=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - VALUES 里的 (表达式).*(基线就有):Postgres 在 VALUES 里同样把复合类型、记录展开成好几列,LATERAL VALUES 按表值 函数认时一个元组项只算一列,column2 起的号码原样交给模型、落库。现在 (表达式).* 不只看选出项:树上任何位置的(VALUES、 行构造器、函数参数里的),以及 VALUES 里的 s.*、(s.*),读到遮罩列就整份遮。(表达式).* 的值只按那个表达式认,不再把 * 当成展开 FROM 的来源(没设遮罩的复合类型 (p.address).* 以前因此被整列遮) - 同一类、复核没点到的:Postgres 里 FROM 中交回复合类型的表值函数。unnest 的数组元素是复合类型时展开成各个字段 (unnest(ARRAY[p.card]) 出 addr、tel,写了列名表也只改前几列的名字),不认得的函数只写了列名、没写类型的列名表 (jsonb_populate_record(NULL::contact, …) AS r(a) 后面的字段照原名出),读到遮罩列的整份遮(_spreads_record);拆普通 数组的 unnest 也一样(元素类型 SQL 里看不出,多遮)。写了带类型列定义的照写的认 - Oracle 的 JSON 点号、对象类型的属性(t.profile.tel,解析器读成库、表、列):前面几段也算列,以前只认 tel,号码原样交出 - 记着别的会话 SQL 原文、被锁那一行的值的来源补进 INDIRECT_NAMES(工具边界和可验证报告共用一份):MySQL 的 INNODB_TRX、 OPTIMIZER_TRACE,MariaDB 的 INNODB_LOCKS、QUERY_CACHE_INFO,Oracle 的 V$ / GV$ / X$ 一律算、AWR 历史、SQL 调优集和 调优顾问、固定的执行计划和 SQL Profile、闪回事务,DBMS_XPLAN、DBMS_SQLTUNE 这类出报告的包 - 不是查询的语句(_reads_beyond):只读表结构的 SHOW(表、列、索引、库、表状态、建库语句)、DESCRIBE / EXPLAIN 一张表 照常,别的 SHOW(ENGINE INNODB STATUS、BINLOG / RELAYLOG EVENTS、REPLICA STATUS、WARNINGS、PROFILES、CREATE TABLE / VIEW、TRIGGERS……)、EXPLAIN 一条语句(MySQL 把只有一行的表的值代进计划里的条件,8.0 实测)不逐个认,一律整份遮。 SHOW CREATE TABLE 不算只读表结构:对视图交回的是视图定义(WHERE 里写死的号码,8.0 实测),以前因为写着 TABLE x 本来 就整份遮,照旧;SHOW TABLE STATUS 以前同样因此整份遮,现在照常 - 质询模块说明的已知局限同步;补一条:视图、函数、约束定义里写死的字面量(information_schema.views、pg_views、 pg_get_viewdef()……)不认,照常交给 agent 补测试:VALUES × LATERAL / JOIN LATERAL / CTE / FROM / 匿名 ROW / 整行 / 表和子查询的 *,函数参数里的 (表达式).*, 没碰遮罩列的对照;交回复合类型的表值函数九种写法和三种照旧按名字认的对照;Oracle 点号四种和两种对照;十九种视图和包、 十三种 SHOW、九种 EXPLAIN、十二种只读表结构的对照;核查 agent 真实查询工具上的端到端(Postgres 的 VALUES 展开、MySQL 的 INNODB_TRX 和 SHOW ENGINE INNODB STATUS)。逐项去掉实现((表达式).* 的取值、树上的 Dot、VALUES 里的 *、新名字、 _reads_beyond、只读表结构的豁免、SHOW CREATE TABLE、unnest、只写列名的列名表、三段名),对应的测试都变红。另在 Postgres 18、MySQL 8.0 临时实例上经核查 agent 的真实查询工具复跑复核的复现和上面这些写法(44 条),交给模型的、落库的、 通用工件接口给的都没有号码;169c500 上其中 23 条漏号码。 --- backend/app/engine/inquiry.py | 37 ++- backend/app/engine/masking.py | 170 ++++++++-- backend/tests/test_masking_table_functions.py | 90 +++++ backend/tests/test_masking_values_and_show.py | 308 ++++++++++++++++++ 4 files changed, 558 insertions(+), 47 deletions(-) create mode 100644 backend/tests/test_masking_values_and_show.py diff --git a/backend/app/engine/inquiry.py b/backend/app/engine/inquiry.py index f6f1c2a9..ffbd7d7a 100644 --- a/backend/app/engine/inquiry.py +++ b/backend/app/engine/inquiry.py @@ -63,8 +63,13 @@ IS NOT NULL THEN amount ELSE 0 END))同写在 WHERE 里,不算 - FROM、JOIN、LATERAL 里的表值函数(json_each、json_tree、unnest、jsonb_each_text、JSON_TABLE、LATERAL VALUES)拆开 遮罩列的,按出列名认它出的列(常见函数的出列名、写的列名表、别名、函数名,_function_columns),同名的别的列一并遮 - (多遮);不认得的函数又没写列名表的,整份快照都遮。函数参数里没写表名的 col->'path'(JSON 箭头,解析器当成以 col - 为参数的 lambda)按 col 认,真的 lambda(transform(arr, x -> …))的参数和遮罩列同名时也算(多遮) + (多遮);不认得的函数又没写列名表的,整份快照都遮。Postgres 里可能交回复合类型、展开成各个字段的也整份遮 + (masking._spreads_record):unnest(数组的元素是复合类型时展开成 addr、tel 这些字段,写了列名表也只改前几列的 + 名字),不认得的函数只写了列名、没写类型的列名表(jsonb_populate_record(NULL::contact, …) AS r(a) 后面的字段 + 照原名出);拆的是普通数组的 unnest 也一样整份遮(元素是什么类型 SQL 里看不出,多遮)。函数参数里没写表名的 + col->'path'(JSON 箭头,解析器当成以 col 为参数的 lambda)按 col 认,真的 lambda(transform(arr, x -> …))的参数 + 和遮罩列同名时也算(多遮)。三段以上的名字(Oracle 的 JSON 点号、对象类型的属性 t.profile.tel,解析器读成库、表、列) + 前面几段也算列,写了库名、表名的列,库名、表名碰巧和遮罩列同名时也算(多遮) - * 把遮罩列带进别的列名(WITH t(a, b, …) AS (SELECT * …) 这类列名表、表的列名表 FROM sales AS s(i, r, p)、UNION 后一支 的 *、TABLE x、标量子查询里的 *):只展开一个来源、那里每一列都数得清的,按位置只遮取自遮罩列的那几列——子查询按它的 选出项数,数据源里的表按查询当时冻结的表结构数(几张列一样的月度表 SELECT * 连起来汇总,金额照常)。表结构里查不到 @@ -85,18 +90,26 @@ 按看不见算(当成数据源里的表,多遮) - 认不全的写法整份遮(多遮,同一份里没碰遮罩列的编号、金额也遮):(表达式).* 展开读到遮罩列的(复合类型的列 (p.profile).*、函数交回的记录 (jsonb_each_text(pj)).*、子查询里的 ROW (q.r).*,外面套了括号、起了别名的也算,展开成 - 几列、各叫什么 SQL 里看不出);语法树和实际执行的对不上的(MySQL、MariaDB 会执行的注释 /*! … */,Postgres 的 + 几列、各叫什么 SQL 里看不出;不只选出项,VALUES 里的、行构造器和函数参数里的也算),VALUES 里的 s.*、(s.*) 读到遮罩列 + 的(同样展开成好几列);语法树和实际执行的对不上的(MySQL、MariaDB 会执行的注释 /*! … */,Postgres 的 U&"…" 名字,没有 * 的一支选出项和结果的列数对不上);表名、SQL 写在字符串里交给函数执行、整表导出的(query_to_xml、 table_to_xml、cursor_to_xml、schema_to_xml、database_to_xml 这一族,dblink、crosstab、ts_stat,Oracle 的 - DBMS_XMLGEN、XMLTYPE)、按页按文件按日志读原始数据的(pageinspect、逻辑解码、LOAD_FILE)、统计信息和会话记录里存着值 - 的(Postgres 的 pg_stats、pg_stat_statements、pg_stat_activity;SQLite 的 sqlite_stat4;MySQL 的 column_statistics、 - general_log、PROCESSLIST、performance_schema;MariaDB 的 column_stats;Oracle 的直方图、记着最小最大值的 - *_tab_columns、V$SQL 一族、审计记录),按名字认(masking 的 INDIRECT_FUNCTIONS、INDIRECT_NAMES,可验证报告用的同一份 - 名单),去掉注释再认一遍(函数名和括号之间夹着注释),自己建的同类函数、视图认不出;information_schema、pg_catalog 里 - 只有表结构的视图不算(核查 agent 查列名、类型常用)。只收支持的几种库的:DuckDB、SQL Server 的同类函数不收;分析出了 - 意外、一长串 CTE 超出工作量上限(ANALYSIS_STEPS)的。整份遮的只收确定取自遮罩列的那几列的值当遮罩值(别的列的值 - 本来就没交给模型,收了会把区域名、列名在同一次核查里处处换掉);快照记下整份遮(masked_whole),格子引用指到它时照实 - 说写法认不全 + DBMS_XMLGEN、XMLTYPE)、按页按文件按日志读原始数据的(pageinspect、逻辑解码、LOAD_FILE)、统计信息、会话记录、锁和 + 日志里存着值的(Postgres 的 pg_stats、pg_stat_statements、pg_stat_activity;SQLite 的 sqlite_stat4;MySQL 的 + column_statistics、general_log、PROCESSLIST、INNODB_TRX、OPTIMIZER_TRACE、performance_schema;MariaDB 的 + column_stats、INNODB_LOCKS、QUERY_CACHE_INFO;Oracle 的直方图、记着最小最大值的 *_tab_columns、V$SQL 一族、 + SQL 调优集、固定下来的执行计划、闪回事务、LogMiner、审计记录),按名字认(masking 的 INDIRECT_FUNCTIONS、 + INDIRECT_NAMES,可验证报告用的同一份名单),去掉注释再认一遍(函数名和括号之间夹着注释),自己建的同类函数、视图 + 认不出;information_schema、pg_catalog 里只有表结构的视图不算(核查 agent 查列名、类型常用)。不是查询的语句里, + 只读表结构的 SHOW(表、列、索引、库、表状态、建库语句)、DESCRIBE 一张表照常,别的 SHOW(ENGINE INNODB STATUS、 + BINLOG EVENTS、WARNINGS、CREATE TABLE / VIEW……)、EXPLAIN 一条语句(MySQL 把只有一行的表的值代进计划里的条件) + 不逐个认,一律整份遮。只收支持的几种库的:DuckDB、SQL Server 的同类函数不收;分析出了意外、一长串 CTE 超出工作量 + 上限(ANALYSIS_STEPS)的。整份遮的只收确定取自遮罩列的那几列的值当遮罩值(别的列的值本来就没交给模型,收了会把区域 + 名、列名在同一次核查里处处换掉);快照记下整份遮(masked_whole),格子引用指到它时照实说写法认不全 +- 定义里写死的字面量不认:视图、函数、触发器、约束、默认值的定义经 information_schema(views、routines、triggers、 + columns 的默认值)、pg_views、pg_get_viewdef() 这类查出来照常交给 agent,写死在定义里的号码(排除测试账号的 WHERE) + 只在已经认得的遮罩值里时按值换掉;SHOW CREATE TABLE(对视图交回的是视图定义)、SHOW CREATE VIEW、SHOW TRIGGERS 不在 + 只读表结构的几种里,跟着整份遮 - 按遮罩列分档的标签(CASE、IIF 的结果全是写死的常量):标签文字是 agent 写在 SQL 里的,不当遮罩值,SQL、回答正文里 照原样;标签列当分组键(每一档一行)照常显示,分档写在子查询、WITH 里、外层(或中间一层)按它分组的也算 (masking._grouped_labels),不分组的逐行标签列整列遮(格子里看不出某一笔落在哪一档),可 agent diff --git a/backend/app/engine/masking.py b/backend/app/engine/masking.py index 79f73294..263c000a 100644 --- a/backend/app/engine/masking.py +++ b/backend/app/engine/masking.py @@ -14,22 +14,25 @@ - 证据摘录(engine/inquiry.py 的 ExcerptMask,judge.prepare 按它逐段遮):交给反方审稿、审批质询 agent 的报告摘录同一套 口径,格子逐格遮(masked_cell),SQL 只换在找某一个值的字面量,阈值照原样 - 哪些列(_derived_masks):从遮罩列本身起,按 SQL 一层层认出值取自遮罩列的结果列(别名、函数、子查询、WITH、UNION 的 - 另一支、和 * 写在同一支里没起别名的表达式、FROM 里拆开遮罩列的表值函数、函数参数里的 JSON 箭头 col->'path');* 把 - 遮罩列带进别的列名的(写了列名表的派生来源、UNION 后一支的 *、标量子查询里的 *)按它展开落到的位置算(_star_places, - 展开数据源里的表的按查询当时冻结的表结构数位置,_schema_columns;写了库名的表按「库名.表名」查,另一个库里的同名表 - 查不到)。整行和函数参数里的 *(Postgres 的 row_to_json(s)、SELECT s、to_json(s.*),Oracle 的 JSON_OBJECT(*))按 * - 展开的来源算;表的列名表(FROM sales AS s(i, r, p))、TABLE x、套了括号的 * 先改写成认得出的样子(_normalized)。 + 另一支、和 * 写在同一支里没起别名的表达式、FROM 里拆开遮罩列的表值函数、函数参数里的 JSON 箭头 col->'path'、Oracle + 的 JSON 点号 t.col.path);* 把遮罩列带进别的列名的(写了列名表的派生来源、UNION 后一支的 *、标量子查询里的 *)按它 + 展开落到的位置算(_star_places,展开数据源里的表的按查询当时冻结的表结构数位置,_schema_columns;写了库名的表按 + 「库名.表名」查,另一个库里的同名表查不到)。整行和函数参数里的 *(Postgres 的 row_to_json(s)、SELECT s、 + to_json(s.*),Oracle 的 JSON_OBJECT(*))按 * 展开的来源算;表的列名表(FROM sales AS s(i, r, p))、TABLE x、 + 套了括号的 * 先改写成认得出的样子(_normalized)。 WITH 里的名字按作用域认(_scoped_cte):不带 RECURSIVE 的一段看不见它自己和排在后面的段,同名的是数据源里的表。 COUNT 只数个数、窗口的分区键只决定次序、按遮罩列分档又当分组键或套在汇总里数个数的、汇总里的 CASE 只在条件里用到的 不算(_value_columns、_band_only)。认不全的写法整份快照的列都算,不再逐个形状补规则:认不出落在哪一列的(PIVOT、 UNPIVOT 这类也算),(表达式).* 展开读到遮罩列的(复合类型的列、函数交回的记录,外面套了括号、起了别名的也算,展开成 - 几列、叫什么说不清),语法树和实际执行的对不上的(OPAQUE_SQL:MySQL 会执行的注释、U&"…";没有 * 的一支选出项和结果 - 的列数对不上),表名、SQL 写在字符串里交给函数、统计信息和会话记录里存着值的(INDIRECT_FUNCTIONS、INDIRECT_NAMES: - query_to_xml 这一族、dblink、pg_stats、MySQL 的 column_statistics、Oracle 的直方图和 V$SQL……,可验证报告用的同一份), - SQL 解析不了又写着遮罩列(或者 *、TABLE x、上面那几类,或者是 Postgres 的)的,分析出了意外、超出工作量上限 - (ANALYSIS_STEPS)的;这些只收确定取自遮罩列的那几列的值(_Masks.collect)。一次分析里派生来源对应的那块查询、一项 - 的值取自哪些列记下来(_analysis),一长串 CTE 不再卡住。双盲复算按同一套认法在派盲算之前挡下取自遮罩列的指标 - (derived_columns) + 几列、叫什么说不清;不只选出项,VALUES、行构造器、函数参数里的也算;Postgres 里 FROM 中可能交回复合类型的表值函数 + 同理,_spreads_record),语法树和实际执行的对不上的(OPAQUE_SQL:MySQL 会执行的注释、U&"…";没有 * 的一支选出项 + 和结果的列数对不上),表名、SQL 写在字符串里交给函数、统计信息和会话记录里存着值的(INDIRECT_FUNCTIONS、 + INDIRECT_NAMES:query_to_xml 这一族、dblink、pg_stats、MySQL 的 column_statistics、INNODB_TRX、Oracle 的直方图和 + V$SQL……,可验证报告用的同一份),不只读表结构的 SHOW、EXPLAIN 一条语句(_reads_beyond:引擎状态、日志、出错信息、 + 视图定义里的原文,MySQL 把只有一行的表的值代进计划里的条件),SQL 解析不了又写着遮罩列(或者 *、TABLE x、上面那几类, + 或者是 Postgres 的)的,分析出了意外、超出工作量上限(ANALYSIS_STEPS)的;这些只收确定取自遮罩列的那几列的值 + (_Masks.collect)。一次分析里派生来源对应的那块查询、一项的值取自哪些列记下来(_analysis),一长串 CTE 不再卡住。 + 双盲复算按同一套认法在派盲算之前挡下取自遮罩列的指标(derived_columns) - 只装着分档标签的列(_label_columns):标签是 agent 写在 SQL 里的字,不当遮罩值;* 展开数据源里的表时按冻结的表结构 认它带不带同名的列(_schema_tables),不知道表结构的按带得出算(多遮)。分档写在派生来源里、外层按它分组的(一档 一行,_grouped_labels)同当分组键的 CASE 一样照常显示(_shown_labels) @@ -185,6 +188,17 @@ def _value_columns(node: Any) -> set[str]: elif isinstance(item, exp.Column): if item.name: out.add(item.name.lower()) + if item.args.get("db") is not None: + # 三段以上的名字(t.profile.tel):Oracle 的 JSON 点号、对象类型的属性写法里中间几段才是列(t 是表的别名), + # 解析器读成「库 t、表 profile、列 tel」,只认 tel 时号码原样交了出去。前面几段都算(多遮:写了库名、表名 + # 的列,库名、表名碰巧和遮罩列同名时也算) + out.update(part.name.lower() for key in ("catalog", "db", "table") + if (part := item.args.get(key)) is not None and part.name) + elif isinstance(item, exp.Dot) and isinstance(item.expression, exp.Star): + # (表达式).*(VALUES、行构造器、函数参数里的):展开的各个字段都取自那个表达式。这个 * 不是在展开 FROM 里的 + # 来源,不按 _scoped_star_inputs 算(以前算了,没设遮罩的复合类型 (p.address).* 也因为 FROM 里有表被当成 + # 取自遮罩列)。读到遮罩列的整份遮,见 _tainted_in + stack.append(item.this) elif isinstance(item, exp.Count) or (isinstance(item, (exp.Case, exp.If)) and _band_only(item)): continue elif isinstance(item, (exp.Case, exp.If)) and (results := _branch_results(item)) is not None \ @@ -880,6 +894,29 @@ def _function_columns(source: Any) -> set[str] | None: return names if listed or name in _TABLE_FUNCTIONS else None +def _spreads_record(source: Any) -> bool: + """Postgres 里这个表值函数这类来源可能交回复合类型、展开成它的各个字段(字段名 SQL 里看不出,同 (表达式).*): + - unnest:数组的元素是复合类型时展开成各个字段(unnest(ARRAY[p.card]) 出 addr、tel),元素是什么类型 SQL 里看不出; + - 不在 _TABLE_FUNCTIONS 里的函数写了只有名字、没写类型的列名表:交回复合类型的(jsonb_populate_record(NULL::contact, + …))列名表只改前几列的名字,后面的字段照原名出(Postgres 18 实测,unnest 的列名表也一样)。 + 写了带类型的列定义(jsonb_to_record(…) AS r(tel text))的出的就是这几列;VALUES、JSON_TABLE 的列写在 SQL 里。""" + from sqlglot import exp + + node = source.this if isinstance(source, exp.Lateral) else source + alias = source.args.get("alias") or (node.args.get("alias") if node is not source else None) + func = node.this if isinstance(node, exp.Table) else node + while isinstance(func, exp.Subquery): + func = func.this + if isinstance(func, (exp.Values, exp.JSONTable)): + return False + name = (func.name if isinstance(func, exp.Anonymous) else func.sql_name() if isinstance(func, exp.Func) + else "").lower() + if name == "unnest": + return True + listed = list(alias.args.get("columns") or []) if alias is not None else [] + return bool(listed) and name not in _TABLE_FUNCTIONS and not all(isinstance(c, exp.ColumnDef) for c in listed) + + def _merging(select: Any, name: str) -> bool: """FROM 里按同名的 name 这一列连起来(USING 里写着它,或者 NATURAL):几个来源的这一列合成一列。""" return any(str(join.args.get("method") or "").upper() == "NATURAL" @@ -1173,7 +1210,8 @@ def _normalized(root: Any, read: str | None, tables: dict[str, list[str]] | None 套了括号、起了别名的 ((p.profile).*)、(p.profile).* AS x 也算,_unwrapped):展开成几列、各叫什么(复合类型的字段名、 函数的出列名)SQL 里看不出,改成一个对不上任何来源的 *(_star_sources 为 None),只用来数别的项的位置。表达式的值 取自哪些列(_value_columns,整行引用已经改写过)记下来交回:读到遮罩列的,整份快照都算(_tainted)——以前只改成对 - 不上来源的 *,最外层、第一支、派生来源里没写列名表的 * 按「照原名带出」认,字段名和遮罩列对不上,号码原样交了出去 + 不上来源的 *,最外层、第一支、派生来源里没写列名表的 * 按「照原名带出」认,字段名和遮罩列对不上,号码原样交了出去。 + 选出项以外的(VALUES、行构造器、函数参数里的)不换,_tainted_in 按它的值取自哪些列认 tables 是查询当时冻结的表结构(_schema_columns,按次序的列名),认整行引用时看来源出不出某一列;没有为 None。 交回每个 (表达式).* 的值取自哪些列。""" from sqlglot import exp @@ -1458,16 +1496,17 @@ def _tainted(sql: str, kind: str | None, masked: set[str], width: int, tables: d 后面的从末尾倒着数。认不出的——两个 * 之间的项、派生来源里没起别名的表达式(外层的 * 会把它原样带出来,列名 又认不准)——整份快照的列都算(同解析不了)。 FROM、JOIN、LATERAL 里的表值函数(json_each、unnest、JSON_TABLE、LATERAL VALUES)参数里写着这些名字的,它出的列 - 都算(_function_columns):遮罩列是 JSON 时,json_each 拆开是最自然的写法。出哪几列说不清的整份快照都算。 + 都算(_function_columns):遮罩列是 JSON 时,json_each 拆开是最自然的写法。出哪几列说不清的整份快照都算,Postgres + 里可能交回复合类型、展开成各个字段的(unnest、只写了列名的列名表,_spreads_record)也算。 * 展开的列换了名字(写了列名表的派生来源、UNION 后一支的 *):* 带出这些名字的值时,它展开落到的那几列照位置算 (_star_places,展开数据源里的表的按 tables 数位置,数不清的整段算);标量子查询里的 *、函数参数里的 *(to_json(s.*)、 JSON_OBJECT(*))、整行引用(row_to_json(s))按 _value_columns 算进外层的表达式。表的列名表、TABLE x、套了括号的 * 先改写成上面这些认得出的样子(_normalized)。 PIVOT、UNPIVOT、MATCH_RECOGNIZE 读到这些名字的:值转进了它们自己起名的列(UNPIVOT 的值列、PIVOT 按 IN 里的值起名的列、 - MEASURES),出的是哪几列不好认,整份快照的列都算。(表达式).* 读到这些名字的、OPAQUE_SQL 和 _INDIRECT 那几类写法 - (_opaque)、结果的一支没有 * 而选出项和结果的列数对不上的(MySQL 会执行的注释里多写了几项这类,语法树和实际执行的 - 对不上)同样整份算。解析之后的分析出了意外、超出工作量上限(ANALYSIS_STEPS)的也整份算——不退回「解析不了」只看 - 文字:没写遮罩列、没有 * 的整行引用(SELECT s)那样就放过去了。""" + MEASURES),出的是哪几列不好认,整份快照的列都算。(表达式).* 读到这些名字的(树上任何位置的,VALUES 里的也算)、 + OPAQUE_SQL 和 _INDIRECT 那几类写法(_opaque)、结果的一支没有 * 而选出项和结果的列数对不上的(MySQL 会执行的注释里 + 多写了几项这类,语法树和实际执行的对不上)同样整份算。解析之后的分析出了意外、超出工作量上限(ANALYSIS_STEPS)的 + 也整份算——不退回「解析不了」只看文字:没写遮罩列、没有 * 的整行引用(SELECT s)那样就放过去了。""" from app.data.sql_semantics import _parse, _read try: @@ -1493,11 +1532,16 @@ def _tainted_in(root: Any, sql: str, kind: str | None, masked: set[str], width: """_tainted 解析、改写(_normalized)之后真去认的那一步。opaque 是 (表达式).* 的值取自哪些列(_normalized 交回的)。""" from sqlglot import exp + from app.data.sql_semantics import _read + names = set(masked) selects = list(root.find_all(exp.Select)) blocks = _blocks(root) sources = [(top, listed) for top, role, listed in blocks if role == "source"] functions = [source for select_ in selects for source in _function_sources(select_)] + # Postgres 里可能交回复合类型、展开成各个字段的表值函数(_spreads_record):读到遮罩列时出哪几列说不清,整份遮 + spreading = {id(source) for source in functions if _spreads_record(source)} \ + if _read(kind) in _ROW_DIALECTS else set() unreadable = False for _ in range(len(selects) + len(sources) + len(functions) + 1): before = len(names) @@ -1510,17 +1554,24 @@ def _tainted_in(root: Any, sql: str, kind: str | None, masked: set[str], width: names |= _source_names(top, listed, names, tables)[0] for source in functions: if _reads(names, _value_columns(source)): - columns = _function_columns(source) + columns = None if id(source) in spreading else _function_columns(source) unreadable = unreadable or columns is None names |= columns or set() if len(names) == before: break # 认不出的整份遮:表值函数出哪几列说不清、派生来源里有认不出名字的列、PIVOT 这类读到遮罩列,(表达式).* 读到遮罩列 # (展开成几列、叫什么都说不清),语法树和实际执行的对不上、SQL 写在字符串里交给函数执行、整表导出、统计信息里的值 - # (_opaque) + # (_opaque)。(表达式).* 不只看选出项(那几个 _normalized 已经换掉,值取自哪些列在 opaque 里):VALUES 里同样展开成 + # 好几列(Postgres 文档:选出列表、VALUES、行构造器里的 composite.* 都展开),VALUES 里的一项是 s.*、(s.*) 的也一样; + # LATERAL VALUES 按表值函数认时一个元组项只算一列,column2 起的号码原样交了出去。不逐个位置认:(表达式).* 在树上任何 + # 位置(行构造器、函数参数里的不展开成几列,也一样,多遮)、VALUES 里的 *,读到遮罩列就整份遮 + expanded = [node.this for node in root.find_all(exp.Dot) if isinstance(node.expression, exp.Star)] + expanded += [item for values in root.find_all(exp.Values) for row in values.expressions + if isinstance(row, exp.Tuple) for item in row.expressions if _is_star(_unwrapped(item))] everything = unreadable or any(_source_names(top, listed, names, tables)[1] for top, listed in sources) \ or any(_reads(names, _value_columns(node)) for node in root.find_all(exp.Pivot, exp.MatchRecognize)) \ - or any(_reads(names, cols) for cols in opaque) or _opaque(sql, kind) + or any(_reads(names, cols) for cols in opaque) \ + or any(_reads(names, _value_columns(node)) for node in expanded) or _opaque(sql, kind) positions: set[int] = set() for top, role, _listed in blocks: if role != "result": @@ -1580,26 +1631,38 @@ INDIRECT_FUNCTIONS = ( r"pg_stat_get_(?:backend_)?activity", r"audit_log_read", ) #: 不点列名也读得出表里内容的表、视图、包(按名字认): -#: - Oracle 的 DBMS_XMLGEN 这一族(包名后面跟着点),XQuery 里的 ora:view 和 oradb: 表 +#: - Oracle 的 DBMS_XMLGEN 这一族(包名后面跟着点),XQuery 里的 ora:view 和 oradb: 表;读游标、出报告的包:DBMS_XPLAN 的 +#: DISPLAY_CURSOR(带 PEEKED_BINDS 时有绑定变量的值)、DBMS_SQLTUNE、DBMS_SQL_MONITOR 的报告、DBMS_WORKLOAD_REPOSITORY +#: 的 AWR 报告(排在前面的语句原文) #: - 统计信息里的采样值、全文索引里的词:Postgres 的 pg_stats、pg_statistic 一族,SQLite 的 sqlite_stat*,MySQL 的 #: column_statistics(直方图里是列的原值)、innodb_ft_*,MariaDB 的 column_stats,Oracle 的直方图、列统计和记着最小最大值 #: 的 *_tab_columns、*_tab_cols(LOW_VALUE、HIGH_VALUE 按字节交回来就是号码、姓名的原文),以及它们的底表 histgrm$、 #: hist_head$ #: - 别的会话执行过的 SQL 原文:Postgres 的 pg_stat_activity、pg_stat_statements、本连接预备过的语句;MySQL 的 general_log、 -#: slow_log、PROCESSLIST、performance_schema(语句历史 events_statements_*、线程正在执行的 SQL),sys 库里建在 -#: performance_schema 上的会话、锁等待视图(带 x$ 的是不排版的那一份;只认这几张,Oracle 常写的 sys.dual 不算);Oracle 的 -#: V$SQL 一族(含 v$sql_bind_capture)、打开着的游标、审计记录 +#: slow_log、PROCESSLIST、正在执行的事务(information_schema.INNODB_TRX 的 trx_query)、performance_schema(语句历史 +#: events_statements_*、线程正在执行的 SQL),sys 库里建在 performance_schema 上的会话、锁等待视图(只认这几张,Oracle +#: 常写的 sys.dual 不算;带 x$ 的不排版的那一份按下面的 X$ 算);MariaDB 的查询缓存(QUERY_CACHE_INFO);Oracle 的动态视图 +#: V$ / GV$ / X$ 一律算,不逐个认(V$SQL 一族、v$sql_bind_capture、打开着的游标、LogMiner 的 V$LOGMNR_CONTENTS、XML 的 +#: 审计记录都在里面),AWR 历史 DBA_HIST_*、审计记录,SQL 调优集里的语句和绑定变量(*_sqlset_*)、调优顾问记下的 +#: (*_sqltune_*)、SQL Access Advisor 的工作负载(*_advisor_sqlw_*)、固定下来的执行计划和 SQL Profile、Patch +#: - 被锁那一行、改动前后那一行的值:MariaDB(和 MySQL 5.7)的 INNODB_LOCKS 的 lock_data 是被锁那一行的键值;Oracle 的 +#: 闪回事务(FLASHBACK_TRANSACTION_QUERY 的 UNDO_SQL)、LogMiner(V$LOGMNR_CONTENTS 的 SQL_REDO / SQL_UNDO)写着整行的值 +#: - MySQL、MariaDB 的 OPTIMIZER_TRACE:同 EXPLAIN 一条语句(_reads_beyond),优化器把只有一行的表的值代进条件写在里面 +#: (要开着优化器跟踪才有内容) #: - SQLite 按页读出的原始数据 #: information_schema、pg_catalog 不整个收:大多只有表结构,核查 agent 查列名、类型时常用,里面存着值的那几张上面逐个列了 -#: (可验证报告那边宁可多挡,整个收,见 SYSTEM_SCHEMAS) +#: (可验证报告那边宁可多挡,整个收,见 SYSTEM_SCHEMAS)。SHOW、EXPLAIN 这类不是查询的语句另按 _reads_beyond 认 INDIRECT_NAMES = ( - r"dbms_xml\w*", r"ora:view", r"oradb", + r"dbms_xml\w*", r"ora:view", r"oradb", r"dbms_(?:xplan|sqltune|sql_monitor|workload_repository)", r"pg_stat(?:s|istic)\w*", r"sqlite_stat\d", r"column_statistics", r"innodb_ft_\w+", r"column_stats", r"(?:all|dba|user|cdb)_(?:\w+_)?(?:histograms|col_statistics|tab_cols|tab_columns)", r"(?:histgrm|hist_head)\$", r"pg_stat_activity", r"pg_stat_statements", r"pg_prepared_statements", r"general_log", r"slow_log", - r"processlist", r"performance_schema", r"events_statements_\w+", + r"processlist", r"innodb_trx", r"innodb_locks", r"query_cache_info", r"optimizer_trace", r"performance_schema", + r"events_statements_\w+", r"sys[\"`]?\s*\.\s*[\"`]?(?:x\$)?(?:session|processlist|innodb_lock_waits|schema_table_lock_waits)", - r"g?v\$sql\w*", r"(?:dba|cdb)_hist_sql\w*", r"g?v\$open_cursor", + r"(?:g?v|x)\$\w+", r"(?:dba|cdb)_hist_\w+", + r"(?:all|dba|user|cdb)_(?:sqlset|sqltune|advisor_sqlw)_\w+", r"(?:dba|cdb)_sql_(?:plan_baselines|profiles|patches)", + r"flashback_transaction_query", r"(?:dba|cdb|user)_(?:\w+_)?audit_trail", r"unified_audit_trail", r"aud\$\w*", r"fga_log\$", r"sqlite_dbpage", ) @@ -1620,10 +1683,43 @@ def indirect_reads(*names: str) -> re.Pattern[str]: _INDIRECT = indirect_reads() +#: 守卫放行的、不是查询的语句里只读得出表结构、库清单的几种(MySQL、MariaDB):SHOW 表、列、索引、库、表状态、建库语句, +#: DESCRIBE / EXPLAIN 一张表(看它的列,后面可以跟一个列名或通配的写法)。核查 agent 探查表结构靠它们,照常。 +#: SHOW CREATE TABLE 不在里面:对视图交回的是视图定义(MySQL 8.0 实测,同 SHOW CREATE VIEW,WHERE 里写死的号码原样), +#: 建表语句里还有默认值、CHECK 约束、分区的取值;以前因为写着 TABLE x 就整份遮,现在照旧 +_STRUCTURE_ONLY = re.compile( + r"show\s+(?:(?:extended|full)\s+)*(?:tables|columns|fields|index|indexes|keys|databases|schemas|table\s+status" + r"|create\s+(?:database|schema))\b" + r"|(?:explain|describe|desc)\s+(?!(?:select|with|table|values|analyze|format|for|extended|partitions|plan|query" + r"|verbose)\b)[\w$`\".]+(?:\s+[\w$`\"'%.]+)?\s*;?\s*$", + re.I) +#: 不是查询、读得出别的东西的语句:SHOW、EXPLAIN、DESCRIBE(EXPLAIN 一条语句也写作 DESCRIBE) +_STATEMENT_VERBS = frozenset({"show", "explain", "describe", "desc"}) + + +def _reads_beyond(bare: str, kind: str | None) -> bool: + """去掉注释的 SQL(bare)是 SHOW、EXPLAIN 这类不是查询的语句,读的又不只是表结构(_STRUCTURE_ONLY 以外的): + - SHOW ENGINE INNODB STATUS(最近的死锁、正在执行的事务带着别的会话的 SQL)、BINLOG / RELAYLOG EVENTS(语句格式的 + 日志就是 SQL 原文)、REPLICA STATUS 的出错信息、PROFILES、WARNINGS(EXPLAIN 之后改写过的语句里,只有一行的表的 + 每一列都代成了常量)、视图和触发器的定义、建表语句(对视图交回的是视图定义)、MariaDB 的 SHOW EXPLAIN FOR 别的 + 连接……交回的是原文,可能带着号码; + - EXPLAIN 一条语句:MySQL 优化时把只有一行的表(按主键取一行)读出来、把值当常量代进条件,FORMAT=TREE / JSON、 + EXPLAIN ANALYZE 写出的条件里就是那一行的值(MySQL 8.0 实测)。FOR CONNECTION 看别的连接正在执行的语句,也算。 + 守卫放行所有 SHOW、EXPLAIN(只读),这里不逐个认哪一种带着值:只读表结构的几种以外一律整份遮。Postgres 解析不了的 + 本来就整份遮(_derived_masks),SQLite 的 EXPLAIN 只有字节码和语句里写的字面量,照常。首关键字的认法同守卫的 + first_verb(括号开头的也认)。""" + text = bare.strip() + verb = re.match(r"[A-Za-z_]+", text.lstrip("(").lstrip()) + if verb is None or verb.group(0).lower() not in _STATEMENT_VERBS or (kind or "").lower() == "sqlite": + return False + return not _STRUCTURE_ONLY.match(text) + + def _opaque(sql: str, kind: str | None) -> bool: """SQL 里有认不全的写法:OPAQUE_SQL(MySQL 会执行的注释、U&"…"),或者 _INDIRECT 那几类。按原文认(字符串、注释里 写着的也算,多遮),再按去掉注释的认:函数名和括号之间夹着注释(query_to_xml/**/(…)、query_to_xml --换行(…))解析得了, - 语法树上又不认函数名,原文里名字后面不紧跟着括号,以前就放过去了。去注释照守卫按数据源的方言切(data/guard.py)。""" + 语法树上又不认函数名,原文里名字后面不紧跟着括号,以前就放过去了。去注释照守卫按数据源的方言切(data/guard.py)。 + 不只读表结构的 SHOW、EXPLAIN 一条语句也算(_reads_beyond)。""" if OPAQUE_SQL.search(sql) or _INDIRECT.search(sql): return True from app.data.guard import strip_comments @@ -1632,13 +1728,16 @@ def _opaque(sql: str, kind: str | None) -> bool: bare = strip_comments(sql, dialect=kind) except Exception: # noqa: BLE001 - 守卫切不开的写法:认不准,按认不全算(宁可多遮) return True - return bare != sql and bool(_INDIRECT.search(bare)) + return (bare != sql and bool(_INDIRECT.search(bare))) or _reads_beyond(bare, kind) def _implicit(sql: str, kind: str | None = None) -> bool: """解析不了的 SQL 里有没有不写列名也会带出遮罩列的写法(*、TABLE x,OPAQUE_SQL、_INDIRECT 那几类,_opaque):它们能 把遮罩列带进别的列名(列名表、UNION 的后一支)、整表交出来,同写着遮罩列一样整份遮(_derived_hidden)。只看文字, - 字符串、注释里的也算(多遮)。""" + 字符串、注释里的也算(多遮)。只读表结构的 SHOW、DESCRIBE(_STRUCTURE_ONLY)不按 *、TABLE x 认:SHOW TABLE STATUS + 里的 TABLE 不是 TABLE x 语句,以前因此整份遮,表状态看不到(SHOW CREATE TABLE 照旧整份遮,见 _STRUCTURE_ONLY)。""" + if _STRUCTURE_ONLY.match(sql.strip()): + return _opaque(sql, kind) return bool(_IMPLICIT.search(_COUNT_STAR.sub("", sql))) or _opaque(sql, kind) @@ -2607,9 +2706,10 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str - 整列遮:列名直接对上遮罩列的,加上按 SQL 认出值取自遮罩列的(_derived_masks:别名、函数、子查询、WITH、 SELECT *(展开的列换了名字的也算)、表值函数、整行引用和函数参数里的 *、列名表、TABLE x;COUNT、只在条件里用到、 分档当分组键的不算,分档写在派生来源里、外层按它分组的也不算);认不全的(SQL 解析不了又写着遮罩列、*、TABLE x, - 或者是 Postgres 的;(表达式).* 读到遮罩列,MySQL 会执行的注释、U&"…",表名、SQL 写在字符串里交给函数,统计信息、 - 会话记录里存着值的视图,选出项和结果的列数对不上,分析出意外、超出工作量上限)整份结果的列都遮(QueryMask.whole)。 - 这几列的格子换成「已遮罩」,类型记成文本 + 或者是 Postgres 的;(表达式).* 读到遮罩列(VALUES 里的也算)、Postgres 里可能交回复合类型的表值函数读到遮罩列, + MySQL 会执行的注释、U&"…",表名、SQL 写在字符串里交给函数,统计信息、会话记录里存着值的视图,不只读表结构的 + SHOW、EXPLAIN 一条语句,选出项和结果的列数对不上,分析出意外、超出工作量上限)整份结果的列都遮 + (QueryMask.whole)。这几列的格子换成「已遮罩」,类型记成文本 - 别的列按值换,只换文字格:整格就是遮罩值的换掉,文字格里夹着的那一截换掉(换了写法的号码也认, _Secrets.loose_spans),换过的格子记下来(QueryMask.cells)。遮罩值是这份结果里的加上 known:先查号码、再查写着 号码的备注、没设遮罩的联系方式表(别的库里的也一样)时,后一份同样遮好。门槛同按值遮(SECRET_DIGITS、SECRET_MIN); diff --git a/backend/tests/test_masking_table_functions.py b/backend/tests/test_masking_table_functions.py index f37bb5e1..c0f072be 100644 --- a/backend/tests/test_masking_table_functions.py +++ b/backend/tests/test_masking_table_functions.py @@ -146,6 +146,27 @@ def test_a_json_arrow_over_an_unmasked_column_stays_visible(): assert hidden_of("SELECT transform(scores, x -> x + 1) AS 分 FROM sales", ["分"], "postgres") == MASKED +@pytest.mark.parametrize("sql, columns, shown", [ + pytest.param("SELECT t.profile.tel AS 电话 FROM sales t", ["电话"], ["电话"], id="JSON 点号"), + pytest.param("SELECT t.profile.contact.tel FROM sales t", ["tel"], ["tel"], id="JSON 点号两层"), + pytest.param("SELECT UPPER(t.profile.tel) AS 电话, t.region FROM sales t", ["电话", "region"], ["电话"], + id="套函数"), + pytest.param("SELECT x.电话 FROM (SELECT t.profile.tel AS 电话 FROM sales t) x", ["电话"], ["电话"], + id="子查询里取出再选"), +]) +def test_oracle_dot_notation_into_a_masked_json_column_is_read_as_that_column(sql, columns, shown): + """Oracle 的 JSON 点号、对象类型的属性写法(t.profile.tel):解析器读成「库 t、表 profile、列 tel」,以前只认 tel, + 取出来的号码原样交给模型、落库。中间几段也算列(多遮:写了库名、表名的列,表名碰巧和遮罩列同名时也算)。""" + hidden = hidden_of(sql, columns, "oracle") + assert {c.lower() for c in shown} <= hidden and "region" not in hidden + + +def test_oracle_dot_notation_into_an_unmasked_column_stays_visible(): + """点号取的不是遮罩列、写了库名表名的普通列:一列都不遮。""" + assert hidden_of("SELECT t.tags.name AS 标签 FROM sales t", ["标签"], "oracle") == MASKED + assert hidden_of("SELECT shop.sales.region FROM shop.sales", ["region"], "oracle") == MASKED + + def test_a_json_arrow_inside_json_each_is_masked_when_really_run(): """SQLite 里真跑一遍:json_each(profile->'$.tels') 拆出来的号码在交给模型之前就遮掉。""" import sqlite3 @@ -159,3 +180,72 @@ def test_a_json_arrow_inside_json_each_is_masked_when_really_run(): db.close() done = masking.mask_query_result(payload, ["profile"], "sqlite") assert done.columns == ["v"] and done.payload["rows"] == [["已遮罩"], ["已遮罩"]] + + +# -------------------------------------------------------------------------- +# 交回复合类型的表值函数(K3b):Postgres 把复合类型展开成它的各个字段 +# -------------------------------------------------------------------------- + + +#: people3.card 是设了遮罩的复合类型(addr, tel),people2.pj 是设了遮罩的 jsonb(照着 contact 类型拆),sales 是订单 +COMPOSITE_SCHEMA = {"tables": { + "people3": {"qualified": "people3", "schema": None, "columns": [{"name": c} for c in ["id", "card", "city"]]}, + "people2": {"qualified": "people2", "schema": None, "columns": [{"name": c} for c in ["id", "pj"]]}, + "sales": {"qualified": "sales", "schema": None, "columns": [{"name": c} for c in SALES]}, +}} +COMPOSITE_MASKED = ["card", "pj", "buyer_phone"] + + +def _boundary(sql: str, columns: list[str], row: list) -> masking.QueryMask: + payload = {"columns": columns, "rows": [list(row)], "sql": sql} + return masking.mask_query_result(payload, COMPOSITE_MASKED, "postgresql", schema=COMPOSITE_SCHEMA) + + +#: (SQL, 结果列, 一行):结果的样子照 Postgres 18 实测。unnest 的数组元素是复合类型时,FROM 里的 unnest 展开成复合类型的 +#: 各个字段(addr、tel),写了列名表也只改前几列的名字;交回复合类型的函数(jsonb_populate_record(NULL::contact, …)) +#: 写了只有名字、没写类型的列名表时同样只改前几列,后面的字段照原名出 +COMPOSITE_RESULTS = [ + pytest.param("SELECT v.* FROM people3 p, unnest(ARRAY[p.card]) v", ["addr", "tel"], ["某路 1 号", "13900000001"], + id="unnest 复合类型的数组"), + pytest.param("SELECT * FROM unnest(ARRAY[(SELECT card FROM people3 WHERE id = 1)])", ["addr", "tel"], + ["某路 1 号", "13900000001"], id="参数是子查询"), + pytest.param("SELECT v.tel FROM people3 p, LATERAL unnest(ARRAY[p.card]) v", ["tel"], ["13900000001"], + id="LATERAL 取一个字段"), + pytest.param("SELECT * FROM people3 p, unnest(ARRAY[p.card]) AS u(a)", ["id", "card", "city", "a", "tel"], + [1, "(某路 1 号,13900000001)", "杭州", "某路 1 号", "13900000001"], id="列名表比字段少"), + pytest.param("SELECT u.* FROM people3 p, unnest(ARRAY[p.card], ARRAY[p.city]) AS u", ["addr", "tel", "unnest"], + ["某路 1 号", "13900000001", "杭州"], id="unnest 几个数组"), + pytest.param("SELECT u.* FROM people3 p, unnest(ARRAY[p.card]) WITH ORDINALITY AS u(a)", + ["a", "tel", "ordinality"], ["某路 1 号", "13900000001", 1], id="WITH ORDINALITY"), + pytest.param("SELECT * FROM (SELECT p.card FROM people3 p) s, LATERAL unnest(ARRAY[s.card]) WITH ORDINALITY u", + ["card", "addr", "tel", "ordinality"], ["(某路 1 号,13900000001)", "某路 1 号", "13900000001", 1], + id="参数取自派生来源"), + pytest.param("SELECT r.* FROM people2 p, jsonb_populate_record(NULL::contact, p.pj) AS r(a)", ["a", "tel"], + ["丙路", "13900000021"], id="populate_record 列名表比字段少"), + pytest.param("SELECT r.* FROM people2 p, LATERAL jsonb_populate_recordset(NULL::contact, jsonb_build_array(p.pj)) " + "AS r(a)", ["a", "tel"], ["丙路", "13900000021"], id="populate_recordset 列名表比字段少"), +] + + +@pytest.mark.parametrize("sql, columns, row", COMPOSITE_RESULTS) +def test_a_table_function_spreading_a_composite_masks_the_whole_result(sql, columns, row): + """FROM 里的函数交回复合类型、读到遮罩列:展开成哪几个字段、各叫什么 SQL 里看不出(同 (表达式).*),整份遮。以前 + unnest 按「出一列,名字是别名或函数名」认,列名表按写全了认,tel 这一列的号码原样交给模型、落库。""" + done = _boundary(sql, columns, row) + assert done.whole and done.columns == columns + assert done.payload["rows"] == [["已遮罩"] * len(columns)] + + +@pytest.mark.parametrize("sql, columns, row, hidden", [ + pytest.param("SELECT r.* FROM people2 p, jsonb_to_record(p.pj) AS r(tel text, addr text)", ["tel", "addr"], + ["13900000021", "丙路"], ["tel", "addr"], id="写了带类型的列定义"), + pytest.param("SELECT k, v FROM people2 p, jsonb_each_text(p.pj) AS x(k, v)", ["k", "v"], ["tel", "13900000021"], + ["k", "v"], id="出列认得全的函数写了列名表"), + pytest.param("SELECT s.id, u FROM sales s, unnest(ARRAY[s.region]) AS u", ["id", "u"], [1, "华东"], [], + id="unnest 没碰遮罩列"), +]) +def test_a_table_function_whose_columns_are_spelled_out_is_read_as_before(sql, columns, row, hidden): + """列写全了的(带类型的列定义、出列认得全的函数)照旧按名字认,没碰遮罩列的照常。""" + done = _boundary(sql, columns, row) + assert not done.whole and done.columns == hidden + assert done.payload["rows"] == [["已遮罩" if c in hidden else v for c, v in zip(columns, row)]] diff --git a/backend/tests/test_masking_values_and_show.py b/backend/tests/test_masking_values_and_show.py new file mode 100644 index 00000000..d7b5b6ea --- /dev/null +++ b/backend/tests/test_masking_values_and_show.py @@ -0,0 +1,308 @@ +"""认不全的写法整份遮,K3b 复核第 2 轮补的两类(engine/masking.py 的 _tainted_in、_opaque): + +- VALUES 里的 (表达式).*:Postgres 在 VALUES 里同样把复合类型的值、记录展开成好几列(文档:选出列表、VALUES、行构造器 + 里的 composite.* 都展开),LATERAL VALUES 按表值函数认时一个元组项只算一列,column2 起的号码原样交给模型、落库。现在 + 树上任何位置的 (表达式).* 读到遮罩列,整份结果都遮——LATERAL、JOIN LATERAL、CTE、FROM 里的 VALUES 都一样 +- 记着别的会话的 SQL 原文、被锁那一行的值的来源:MySQL 的 information_schema.INNODB_TRX、OPTIMIZER_TRACE,MariaDB 的 + INNODB_LOCKS、QUERY_CACHE_INFO,Oracle 的 SQL 调优集、固定下来的执行计划、闪回事务、LogMiner;不只读表结构的 SHOW + (ENGINE INNODB STATUS、BINLOG / RELAYLOG EVENTS、REPLICA STATUS、WARNINGS、CREATE TABLE / VIEW……)和 EXPLAIN 一条 + 语句。以前工具边界的名单里没有,可验证报告那份整个挡 information_schema、不附带解析不了的快照,反而挡得住 + +这里没有真的数据库:结果按它交回的样子造(复合类型展开成几列、SHOW 交回的几列,照 Postgres 18、MySQL 8.0 实测的样子)。 +数据都是虚构的:手机号、地址、区域。 +""" +from __future__ import annotations + +import json +import uuid + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.core import artifact_store +from app.data.engine import engines +from app.db.base import SessionLocal +from app.db.models import DataSource +from app.engine import masking +from app.main import app +from tests.test_verify_agent import Script, q +from tests.test_verify_agent_masking import ask, clean_spend # noqa: F401 - clean_spend 是自动用上的夹具 + +MASKED_WORD = "已遮罩" +#: people3.card 是复合类型(addr, tel),people.address 是没设遮罩的复合类型(street, city),sales 是订单。设了遮罩的是 +#: card、buyer_phone +SCHEMA = {"tables": { + "people3": {"qualified": "people3", "schema": None, "columns": [{"name": c} for c in ["id", "card", "city"]]}, + "people": {"qualified": "people", "schema": None, "columns": [{"name": c} for c in ["id", "address"]]}, + "sales": {"qualified": "sales", "schema": None, + "columns": [{"name": c} for c in ["id", "region", "buyer_phone", "amount"]]}, +}} +MASKED = ["card", "buyer_phone"] +PHONE = "13900000001" + + +def _leaks(done: masking.QueryMask) -> bool: + return "1390000000" in json.dumps(done.payload, ensure_ascii=False) + + +def masked(sql: str, columns: list[str], rows: list[list], kind: str = "postgresql") -> masking.QueryMask: + payload = {"columns": columns, "rows": rows, "sql": sql} + return masking.mask_query_result(payload, MASKED, kind, schema=SCHEMA) + + +# -------------------------------------------------------------------------- +# VALUES 里的 (表达式).* +# -------------------------------------------------------------------------- + + +#: 复合类型的列、子查询交回的复合值、匿名 ROW、整行 × LATERAL、JOIN LATERAL、CTE、FROM 里的 VALUES:(SQL, 结果列, 一行)。 +#: 号码都在第二列起:以前 LATERAL VALUES 一个元组项只算 column1,column2 起的号码原样交了出去 +VALUES_EXPANSIONS = [ + pytest.param("SELECT v.* FROM people3 p, LATERAL (VALUES ((p.card).*)) v", ["column1", "column2"], + ["某路 1 号", PHONE], id="LATERAL-取 *"), + pytest.param("SELECT v.column2 AS c FROM people3 p, LATERAL (VALUES ((p.card).*)) v", ["c"], [PHONE], + id="LATERAL-取第二列"), + pytest.param("SELECT x.column2 FROM people3 p JOIN LATERAL (VALUES ((p.card).*)) x ON true", ["column2"], [PHONE], + id="JOIN LATERAL"), + pytest.param("WITH v AS (VALUES ((SELECT card FROM people3 WHERE id = 1).*)) SELECT * FROM v", + ["column1", "column2"], ["某路 1 号", PHONE], id="CTE 里的 VALUES"), + pytest.param("SELECT t FROM (VALUES ((SELECT card FROM people3 WHERE id = 1).*)) v(a, t)", ["t"], [PHONE], + id="FROM 里的 VALUES-列名表"), + pytest.param("SELECT v.column2 FROM sales s, LATERAL (VALUES ((ROW(s.region, s.buyer_phone)).*)) v", ["column2"], + [PHONE], id="匿名 ROW"), + pytest.param("SELECT v.* FROM sales s, LATERAL (VALUES ((s).*)) v", ["column1", "column2", "column3", "column4"], + [1, "华东", PHONE, 1200], id="整行"), + pytest.param("SELECT v.column3 FROM people3 p, LATERAL (VALUES (p.id, (p.card).*)) v", ["column3"], [PHONE], + id="和别的项写在一个元组里"), + # VALUES 里的 s.*、(s.*) 同样展开成好几列(s 是表,或者取了遮罩列的子查询) + pytest.param("SELECT v.* FROM sales s, LATERAL (VALUES (s.*)) v", ["column1", "column2", "column3", "column4"], + [1, "华东", PHONE, 1200], id="表的 *"), + pytest.param("SELECT v.column3 FROM sales s, LATERAL (VALUES ((s.*))) v", ["column3"], [PHONE], + id="套了括号的表的 *"), + pytest.param("SELECT v.column2 FROM (SELECT id, buyer_phone AS p FROM sales) q, LATERAL (VALUES (q.*)) v", + ["column2"], [PHONE], id="子查询的 *"), +] + + +@pytest.mark.parametrize("sql, columns, row", VALUES_EXPANSIONS) +def test_an_expression_star_inside_values_masks_the_whole_result(sql, columns, row): + """VALUES 里的 (表达式).* 读到遮罩列:展开成几列说不清,整份结果的每一格都遮。以前只遮了 column1(或一列不遮), + 号码原样交给模型、落库。""" + done = masked(sql, columns, [list(row)]) + assert done.whole and done.columns == columns + assert done.payload["rows"] == [[MASKED_WORD] * len(columns)] and not _leaks(done) + + +@pytest.mark.parametrize("sql, columns, row, hidden", [ + pytest.param("SELECT v.* FROM people p, LATERAL (VALUES ((p.address).*)) v", ["column1", "column2"], + ["某路", "某市"], [], id="没设遮罩的复合类型"), + pytest.param("SELECT v.column1 AS r FROM sales s, LATERAL (VALUES (s.region)) v", ["r"], ["华东"], [], + id="没碰遮罩列的 VALUES"), + pytest.param("SELECT v.column1 AS p FROM sales s, LATERAL (VALUES (s.buyer_phone)) v", ["p"], [PHONE], ["p"], + id="VALUES 里直接写遮罩列"), + pytest.param("SELECT v.column2 AS r FROM (SELECT id, region FROM sales) q, LATERAL (VALUES (q.*)) v", ["r"], + ["华东"], [], id="没取遮罩列的子查询的 *"), +]) +def test_values_without_an_expression_star_on_a_masked_column_is_read_as_before(sql, columns, row, hidden): + """VALUES 里没有读到遮罩列的 (表达式).*:照旧按列认,没碰遮罩列的照常,直接写遮罩列的只遮那一列。""" + done = masked(sql, columns, [list(row)]) + assert not done.whole and done.columns == hidden + assert done.payload["rows"] == [[MASKED_WORD if c in hidden else v for c, v in zip(columns, row)]] + + +def test_an_expression_star_inside_a_function_argument_masks_the_whole_result_too(): + """行构造器、函数参数里的 (表达式).*:不逐个位置认展开不展开,读到遮罩列的一样整份遮(多遮,编号也遮)。""" + done = masked("SELECT p.id AS n, row_to_json(ROW((p.card).*)) AS j FROM people3 p", ["n", "j"], + [[1, json.dumps({"f1": "某路 1 号", "f2": PHONE})]]) + assert done.whole and done.payload["rows"] == [[MASKED_WORD, MASKED_WORD]] and not _leaks(done) + + +# -------------------------------------------------------------------------- +# 端到端:核查 agent 的真实查询工具 +# -------------------------------------------------------------------------- + + +@pytest.fixture +async def fake_source(): + """造出来的数据源(不连库,查询交给换掉的 run_query):make(kind) 建一个、交回源名。""" + made: list[str] = [] + + async def make(kind: str) -> str: + name = f"{kind[:2]}_{uuid.uuid4().hex[:8]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind=kind, host="127.0.0.1", port=5432, username="probe", database="shop", + readonly=True, enabled=True, options={"mask_columns": ", ".join(MASKED)}, + schema_cache=SCHEMA) + session.add(row) + await session.commit() + made.append(row.id) + return name + + yield make + for source_id in made: + await engines.invalidate(source_id) + + +async def _asked(monkeypatch, fake_source, kind: str, sql: str, columns: list[str], rows: list[list]): + """按剧本让核查 agent 查一次(结果由换掉的 run_query 交回),交回 (工具结果原文, 落库的快照, 通用工件接口给的原文)。""" + from app.data.engine import QueryResult + from app.tools import datasource + + async def driver(source, statement, **kwargs): + assert statement == sql + return QueryResult(columns=list(columns), rows=[list(r) for r in rows], row_count=len(rows), truncated=False, + elapsed_ms=1, sql=statement) + + monkeypatch.setattr(datasource, "run_query", driver) + name = await fake_source(kind) + script = Script(monkeypatch, [q(name, sql, "c1")], {"value": None, "rationale": "x"}) + out, _ = await ask(name) + snapshot = artifact_store.load(out.queries[0]["artifact"]) + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + got = await client.get(f"/api/artifacts/{out.queries[0]['artifact']}") + assert got.status_code == 200 + return script.tool_results()["c1"], snapshot, got.text + + +@pytest.mark.parametrize("sql, columns, rows", [ + ("SELECT v.* FROM people3 p, LATERAL (VALUES ((p.card).*)) v", ["column1", "column2"], + [["某路 1 号", PHONE], ["某路 2 号", "13900000002"]]), + ("SELECT v.column2 AS c FROM people3 p, LATERAL (VALUES ((p.card).*)) v", ["c"], [[PHONE], ["13900000002"]]), +], ids=["取 *", "取第二列"]) +async def test_values_expansions_never_reach_the_model_or_the_store(monkeypatch, fake_source, sql, columns, rows): + """交给模型的、落库的、通用工件接口给的都只有「已遮罩」,masked_at_query 是全部的列,快照记着整份遮。以前 + masked_at_query 是 ['column1'] 或空的,号码原样交给模型、落库。""" + seen, snapshot, api = await _asked(monkeypatch, fake_source, "postgresql", sql, columns, rows) + assert MASKED_WORD in seen and "1390000000" not in seen, seen + assert snapshot["masked_at_query"] == columns and snapshot["masked_whole"] is True + assert "1390000000" not in json.dumps(snapshot, ensure_ascii=False) and "1390000000" not in api + + +# -------------------------------------------------------------------------- +# 记着别的会话的 SQL、被锁那一行的值的视图 +# -------------------------------------------------------------------------- + + +#: 另一个会话正在执行的语句(号码写在条件里):会话、事务、锁、日志一类的视图交回的就是它 +BUSY = f"SELECT id, SLEEP(4) FROM sales WHERE buyer_phone = '{PHONE}' FOR UPDATE" + + +@pytest.mark.parametrize("sql, kind", [ + pytest.param("SELECT trx_query AS t FROM information_schema.INNODB_TRX", "mysql", id="MySQL 正在执行的事务"), + pytest.param("SELECT trx_query FROM INFORMATION_SCHEMA.innodb_trx WHERE trx_state = 'RUNNING'", "mysql", + id="大小写混写"), + pytest.param("SELECT lock_data AS d FROM information_schema.INNODB_LOCKS", "mariadb", id="MariaDB 被锁的行"), + pytest.param("SELECT statement_text AS t FROM information_schema.QUERY_CACHE_INFO", "mariadb", + id="MariaDB 查询缓存"), + pytest.param("SELECT sql_text AS t FROM dba_sqlset_statements", "oracle", id="Oracle SQL 调优集"), + pytest.param("SELECT value_string AS t FROM dba_sqlset_binds", "oracle", id="Oracle 调优集的绑定变量"), + pytest.param("SELECT sql_text AS t FROM dba_sql_plan_baselines", "oracle", id="Oracle 固定的执行计划"), + pytest.param("SELECT sql_text AS t FROM dba_sql_profiles", "oracle", id="Oracle SQL Profile"), + pytest.param("SELECT value AS t FROM dba_sqltune_binds", "oracle", id="Oracle 调优顾问的绑定变量"), + pytest.param("SELECT undo_sql AS t FROM flashback_transaction_query", "oracle", id="Oracle 闪回事务"), + pytest.param("SELECT sql_redo AS t FROM v$logmnr_contents", "oracle", id="Oracle LogMiner"), + pytest.param("SELECT sql_text AS t FROM v$xml_audit_trail", "oracle", id="Oracle XML 审计"), + pytest.param("SELECT trace AS t FROM information_schema.OPTIMIZER_TRACE", "mysql", id="MySQL 优化器跟踪"), + # 动态视图 V$ / GV$ / X$、AWR 历史不逐个认,一律算 + pytest.param("SELECT client_info AS t FROM v$session", "oracle", id="Oracle V$ 动态视图"), + pytest.param("SELECT kglnaobj AS t FROM x$kglob", "oracle", id="Oracle X$ 底表"), + pytest.param("SELECT sql_text AS t FROM gv$sql_monitor", "oracle", id="Oracle GV$"), + pytest.param("SELECT event AS t FROM dba_hist_active_sess_history", "oracle", id="Oracle AWR 历史"), + pytest.param("SELECT plan_table_output AS t FROM TABLE(DBMS_XPLAN.DISPLAY_CURSOR('a1', NULL, 'ADVANCED " + "+PEEKED_BINDS'))", "oracle", id="Oracle 游标的执行计划和绑定变量"), + pytest.param("SELECT DBMS_SQLTUNE.REPORT_SQL_MONITOR(sql_id => 'a1', type => 'TEXT') AS t FROM dual", "oracle", + id="Oracle SQL 监控报告"), +]) +def test_views_that_keep_other_sessions_sql_mask_the_whole_result(sql, kind): + """别的会话执行过的 SQL 原文、被锁那一行的键值、闪回和日志里的改动前后的值:读的是哪一列认不出,整份遮。以前 + masked=[],条件里的号码原样交给模型、落库。""" + done = masked(sql, ["t"], [[BUSY]], kind=kind) + assert done.whole and done.payload["rows"] == [[MASKED_WORD]] + + +@pytest.mark.parametrize("sql, columns", [ + pytest.param("SHOW ENGINE INNODB STATUS", ["Type", "Name", "Status"], id="InnoDB 状态"), + pytest.param("show engine innodb status", ["Type", "Name", "Status"], id="小写"), + pytest.param("SHOW BINLOG EVENTS IN 'binlog.000002'", ["Log_name", "Pos", "Info"], id="binlog 事件"), + pytest.param("SHOW RELAYLOG EVENTS", ["Log_name", "Pos", "Info"], id="relay log 事件"), + pytest.param("SHOW REPLICA STATUS", ["Last_SQL_Error"], id="复制状态"), + pytest.param("SHOW SLAVE STATUS", ["Last_SQL_Error"], id="复制状态-旧写法"), + pytest.param("SHOW PROFILES", ["Query_ID", "Duration", "Query"], id="PROFILES"), + pytest.param("SHOW WARNINGS", ["Level", "Code", "Message"], id="WARNINGS"), + pytest.param("SHOW CREATE VIEW vip_sales", ["View", "Create View"], id="视图定义"), + # SHOW CREATE TABLE 对视图交回的是视图定义(MySQL 8.0 实测),建表语句里还有默认值、CHECK 约束、分区的取值 + pytest.param("SHOW CREATE TABLE vip_sales", ["View", "Create View"], id="建表语句-视图"), + pytest.param("SHOW CREATE TABLE sales", ["Table", "Create Table"], id="建表语句"), + pytest.param("SHOW TRIGGERS", ["Trigger", "Statement"], id="触发器"), + pytest.param("SHOW EXPLAIN FOR 12", ["id", "Extra"], id="MariaDB 别的连接的计划"), +]) +def test_show_statements_beyond_the_table_structure_mask_the_whole_result(sql, columns): + """SHOW 里只交回表结构、库清单的几种以外(引擎状态、日志、复制、出错信息、视图和触发器的定义、建表语句……):交回的 + 是原文,可能带着别的会话 SQL 里的号码、定义里写死的号码,整份遮。以前守卫放行(SHOW 是只读的),遮罩认不出,号码 + 原样。""" + row = [BUSY] * len(columns) + done = masked(sql, columns, [row], kind="mysql") + assert done.whole and done.payload["rows"] == [[MASKED_WORD] * len(columns)] + + +@pytest.mark.parametrize("sql, columns, row", [ + pytest.param("SHOW TABLES", ["Tables_in_shop"], ["sales"], id="表清单"), + pytest.param("SHOW FULL TABLES FROM shop", ["Tables_in_shop", "Table_type"], ["sales", "BASE TABLE"], + id="FULL TABLES"), + pytest.param("SHOW COLUMNS FROM sales", ["Field", "Type"], ["region", "varchar(20)"], id="列"), + pytest.param("SHOW FULL FIELDS FROM sales", ["Field", "Type"], ["region", "varchar(20)"], id="FULL FIELDS"), + pytest.param("SHOW INDEX FROM sales", ["Table", "Key_name"], ["sales", "PRIMARY"], id="索引"), + pytest.param("SHOW KEYS FROM sales", ["Table", "Key_name"], ["sales", "PRIMARY"], id="KEYS"), + pytest.param("SHOW DATABASES", ["Database"], ["shop"], id="库清单"), + pytest.param("SHOW CREATE DATABASE shop", ["Database", "Create Database"], ["shop", "CREATE DATABASE `shop`"], + id="建库语句"), + pytest.param("SHOW TABLE STATUS LIKE 'sales'", ["Name", "Engine", "Rows"], ["sales", "InnoDB", 3], id="表状态"), + pytest.param("DESCRIBE sales", ["Field", "Type"], ["region", "varchar(20)"], id="DESCRIBE 表"), + pytest.param("EXPLAIN sales", ["Field", "Type"], ["region", "varchar(20)"], id="EXPLAIN 表"), + pytest.param("DESC sales region", ["Field", "Type"], ["region", "varchar(20)"], id="DESC 表的一列"), +]) +def test_show_and_describe_of_the_table_structure_stay_visible(sql, columns, row): + """只交回表结构、库清单的 SHOW、DESCRIBE(核查 agent 探查表结构靠它们)照常。""" + done = masked(sql, columns, [list(row)], kind="mysql") + assert not done.whole and done.payload["rows"] == [list(row)] + + +@pytest.mark.parametrize("sql", [ + "EXPLAIN FORMAT=TREE SELECT s.id FROM sales s JOIN people3 p ON p.id = s.id WHERE s.id = 1", + "EXPLAIN FORMAT = JSON SELECT id FROM sales WHERE id = 1", + "EXPLAIN ANALYZE SELECT id FROM sales WHERE id = 1", + "DESCRIBE SELECT id FROM sales WHERE id = 1", + "EXPLAIN FOR CONNECTION 12", + "EXPLAIN FORMAT=JSON FOR CONNECTION 12", + "EXPLAIN (SELECT id FROM sales)", + "EXPLAIN WITH t AS (SELECT id FROM sales) SELECT * FROM t", + "EXPLAIN EXTENDED SELECT id FROM sales", +]) +def test_explaining_a_statement_masks_the_whole_result(sql): + """EXPLAIN 一条语句:MySQL 优化时把只有一行的表读出来、把值当常量代进条件,FORMAT=TREE / JSON、EXPLAIN ANALYZE 写出的 + 条件里就是那一行的值;FOR CONNECTION 读的是别的连接正在执行的语句。整份遮。""" + done = masked(sql, ["EXPLAIN"], [[f"-> Filter: (s.buyer_phone = '{PHONE}')"]], kind="mysql") + assert done.whole and done.payload["rows"] == [[MASKED_WORD]] + + +@pytest.mark.parametrize("sql, columns, rows", [ + ("SELECT trx_query AS t FROM information_schema.INNODB_TRX", ["t"], [[BUSY]]), + ("SHOW ENGINE INNODB STATUS", ["Type", "Name", "Status"], + [["InnoDB", "", f"---TRANSACTION 1234, ACTIVE 2 sec\nMySQL thread id 9, query id 70 localhost root\n{BUSY}"]]), +], ids=["INNODB_TRX", "SHOW ENGINE INNODB STATUS"]) +async def test_other_sessions_sql_never_reaches_the_model_or_the_store(monkeypatch, fake_source, sql, columns, rows): + """端到端(MySQL):另一个会话正在执行的 SQL 里写着号码,交给模型的、落库的、通用工件接口给的都只有「已遮罩」, + 快照记着整份遮。以前 masked_at_query 为空,SQL 原文连同号码原样交出。""" + seen, snapshot, api = await _asked(monkeypatch, fake_source, "mysql", sql, columns, rows) + assert MASKED_WORD in seen and "1390000000" not in seen, seen + assert snapshot["masked_at_query"] == columns and snapshot["masked_whole"] is True + assert "1390000000" not in json.dumps(snapshot, ensure_ascii=False) and "1390000000" not in api + + +def test_the_new_views_are_on_the_list_the_bundle_shares(): + """可验证报告和工具边界共用一份名单:新补的视图两边都认(报告那边本来就整个挡 information_schema)。""" + from app.bundle import build + + for sql in ("SELECT trx_query FROM information_schema.INNODB_TRX", "SELECT sql_text FROM dba_sqlset_statements", + "SELECT undo_sql FROM flashback_transaction_query", "SELECT sql_redo FROM v$logmnr_contents"): + assert build._INDIRECT_READS.search(sql) and masking._INDIRECT.search(sql), sql From 45987748ef39cc98a54d865b52b5878f32e548e3 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Mon, 12 Oct 2026 12:18:34 +0800 Subject: [PATCH 12/12] =?UTF-8?q?K3b=20=E5=A4=8D=E6=A0=B8=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=EF=BC=88=E7=AC=AC=203=20=E8=BD=AE=EF=BC=89=EF=BC=9AVA?= =?UTF-8?q?LUES=20=E5=BD=93=E4=B8=80=E5=BC=A0=E8=A1=A8=E7=94=A8=E5=8F=88?= =?UTF-8?q?=E8=AF=BB=E5=88=B0=E9=81=AE=E7=BD=A9=E5=88=97=E7=9A=84=E6=95=B4?= =?UTF-8?q?=E4=BB=BD=E9=81=AE=EF=BC=8C=E5=88=97=E5=90=8D=E6=8C=89=E6=96=B9?= =?UTF-8?q?=E8=A8=80=E8=AE=A4=EF=BC=9Bsys=20=E5=BA=93=E8=AF=BB=20performan?= =?UTF-8?q?ce=5Fschema=20=E7=9A=84=20ps=5F*=20=E5=87=BD=E6=95=B0=E8=BF=9B?= =?UTF-8?q?=E8=AE=A4=E4=B8=8D=E5=85=A8=E7=9A=84=E5=90=8D=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - VALUES(复核 major,基线就有):_function_columns 按 Postgres、SQLite 的习惯把 VALUES 的列记成 column1…,MySQL 8.0 实际叫 column_0、column_1…(真库实测),派生表、CTE、LATERAL 里的 VALUES ROW 取了号码,外层按名字取、照原名带出时 对不上,号码原样交给模型、落库;MariaDB 拿第一行的值当列名,同样对不上。现在 VALUES 当一张表用(FROM、JOIN、 LATERAL 里的,WITH 里的一段,UNION 的一支,整条就是它,_values_table)又读到遮罩列(含派生出来的名字)的,不论哪个 库、列怎么起名,整份遮。套了括号的 VALUES(整条、UNION 的一支)解析器原样挂在树上、不在任何 SELECT 的 FROM 里,以前 哪个库都没认(Postgres 也漏),一并收口。IN / ANY / EXISTS、标量子查询里的 VALUES 不算:值经外层的表达式带出,或者 只用来筛行 - 列名按方言认(_values_names,_function_columns 用同一套):Postgres、SQLite 叫 column1…,MySQL 叫 column_0…(一行 ROW(…) 摊开数列),写了列名表的按列名表(Postgres 的列名表只改前几列),MariaDB、Oracle 认不准记 None。整份遮时只收 值取自遮罩列的那几列的值(_values_taken 按位置):以前 VALUES 按表值函数认,出的每一列(连同 key、value、别名)都算, 同一行里没碰遮罩列的区域收成遮罩值,在同一次核查里处处被换掉 - 行为变化:Postgres、SQLite 的 LATERAL VALUES 读到遮罩列,以前只遮那几列,现在整份遮(多遮,同一份里的编号也遮); 原来「VALUES 里直接写遮罩列只遮那一列」的用例改成整份遮 - sys 库(复核 minor):INDIRECT_NAMES 的 sys 规则补上按 sys.ps_ 开头认的函数(ps_thread_trx_info 交回线程的事务里执行 过的语句原文,ps_thread_stack 同类;只交回编号、开关的 ps_thread_id、ps_is_* 一并算,多遮;不写 sys. 的要默认库就是 sys 才执行得了,不认,同 sys 的视图)。工具边界和可验证报告共用这一份名单,报告不再判它没碰遮罩列、照常附带快照 - 质询模块说明的已知局限同步 补测试:VALUES 当一张表用 × MySQL(派生表、LATERAL、CTE、外层改名、JOIN LATERAL、LATERAL 子查询、标量子查询、派生 出来的名字、UNION 第一支、整条套括号、派生表里 UNION 的一支)、MariaDB 两种、Postgres 三种、SQLite 一种;三种没读到 遮罩列的对照;四种写法的收值(号码收、区域不收)和 (表达式).* 之后不按位置收;_function_columns 四种列名、两种认不准; 核查 agent 真实查询工具上的端到端(复核点名的三条 MySQL 写法);sys.ps_* 六种写法、三种不该误伤的对照、可验证报告 两种、端到端一条。逐项去掉实现(整份遮、MySQL 的起名、按位置收值、当一张表用的位置、整条就是 VALUES、VALUES 不按表值 函数认、摊开 ROW、列名表、_function_columns 的起名、(表达式).* 之后停、MariaDB / Oracle 认不准、sys.ps_),对应的测试 都变红。另在 MySQL 8.0.46、Postgres 18 临时实例上经核查 agent 的真实查询工具复跑 28 条(MySQL 的 VALUES 14 条、对照 4 条、sys 函数 4 条,Postgres 的 VALUES 6 条):交给模型的、落库的、通用工件接口给的都没有号码;2b5989e 上其中 16 条漏 号码(sys.ps_thread_stack 在只读事务里执行不了,前后一样)。 --- backend/app/engine/inquiry.py | 27 ++- backend/app/engine/masking.py | 172 ++++++++++--- backend/tests/test_masking_values_and_show.py | 225 +++++++++++++++++- 3 files changed, 368 insertions(+), 56 deletions(-) diff --git a/backend/app/engine/inquiry.py b/backend/app/engine/inquiry.py index ffbd7d7a..02988e3a 100644 --- a/backend/app/engine/inquiry.py +++ b/backend/app/engine/inquiry.py @@ -61,9 +61,9 @@ 复算),反方审稿核对这类汇总数的结论也拿不到数。汇总里套着结果全是写死常量的 CASE(SUM(CASE WHEN amount <= 20000 THEN 1 ELSE 0 END))只是按条件数个数,同 COUNT;汇总里的 CASE 只在条件里用到遮罩列的(SUM(CASE WHEN buyer_phone IS NOT NULL THEN amount ELSE 0 END))同写在 WHERE 里,不算 -- FROM、JOIN、LATERAL 里的表值函数(json_each、json_tree、unnest、jsonb_each_text、JSON_TABLE、LATERAL VALUES)拆开 - 遮罩列的,按出列名认它出的列(常见函数的出列名、写的列名表、别名、函数名,_function_columns),同名的别的列一并遮 - (多遮);不认得的函数又没写列名表的,整份快照都遮。Postgres 里可能交回复合类型、展开成各个字段的也整份遮 +- FROM、JOIN、LATERAL 里的表值函数(json_each、json_tree、unnest、jsonb_each_text、JSON_TABLE)拆开遮罩列的,按 + 出列名认它出的列(常见函数的出列名、写的列名表、别名、函数名,_function_columns),同名的别的列一并遮(多遮); + 不认得的函数又没写列名表的,整份快照都遮。Postgres 里可能交回复合类型、展开成各个字段的也整份遮 (masking._spreads_record):unnest(数组的元素是复合类型时展开成 addr、tel 这些字段,写了列名表也只改前几列的 名字),不认得的函数只写了列名、没写类型的列名表(jsonb_populate_record(NULL::contact, …) AS r(a) 后面的字段 照原名出);拆的是普通数组的 unnest 也一样整份遮(元素是什么类型 SQL 里看不出,多遮)。函数参数里没写表名的 @@ -90,15 +90,18 @@ 按看不见算(当成数据源里的表,多遮) - 认不全的写法整份遮(多遮,同一份里没碰遮罩列的编号、金额也遮):(表达式).* 展开读到遮罩列的(复合类型的列 (p.profile).*、函数交回的记录 (jsonb_each_text(pj)).*、子查询里的 ROW (q.r).*,外面套了括号、起了别名的也算,展开成 - 几列、各叫什么 SQL 里看不出;不只选出项,VALUES 里的、行构造器和函数参数里的也算),VALUES 里的 s.*、(s.*) 读到遮罩列 - 的(同样展开成好几列);语法树和实际执行的对不上的(MySQL、MariaDB 会执行的注释 /*! … */,Postgres 的 - U&"…" 名字,没有 * 的一支选出项和结果的列数对不上);表名、SQL 写在字符串里交给函数执行、整表导出的(query_to_xml、 - table_to_xml、cursor_to_xml、schema_to_xml、database_to_xml 这一族,dblink、crosstab、ts_stat,Oracle 的 - DBMS_XMLGEN、XMLTYPE)、按页按文件按日志读原始数据的(pageinspect、逻辑解码、LOAD_FILE)、统计信息、会话记录、锁和 - 日志里存着值的(Postgres 的 pg_stats、pg_stat_statements、pg_stat_activity;SQLite 的 sqlite_stat4;MySQL 的 - column_statistics、general_log、PROCESSLIST、INNODB_TRX、OPTIMIZER_TRACE、performance_schema;MariaDB 的 - column_stats、INNODB_LOCKS、QUERY_CACHE_INFO;Oracle 的直方图、记着最小最大值的 *_tab_columns、V$SQL 一族、 - SQL 调优集、固定下来的执行计划、闪回事务、LogMiner、审计记录),按名字认(masking 的 INDIRECT_FUNCTIONS、 + 几列、各叫什么 SQL 里看不出;不只选出项,VALUES 里的、行构造器和函数参数里的也算),VALUES 当一张表用(FROM、JOIN、 + LATERAL 里的,WITH 里的一段,UNION 的一支)又读到遮罩列的(直接写遮罩列、经子查询或派生来源改了名字的、里面写着 + s.*、(s.*) 的都算;各库给 VALUES 的列起名不一样:Postgres、SQLite 叫 column1…,MySQL 叫 column_0…,MariaDB 拿第一行 + 的值当列名,不逐个认,同一份里的编号、区域也遮,只收值取自遮罩列的那几列的值);语法树和实际执行的对不上的(MySQL、 + MariaDB 会执行的注释 /*! … */,Postgres 的 U&"…" 名字,没有 * 的一支选出项和结果的列数对不上);表名、SQL 写在字符串里 + 交给函数执行、整表导出的(query_to_xml、table_to_xml、cursor_to_xml、schema_to_xml、database_to_xml 这一族,dblink、 + crosstab、ts_stat,Oracle 的 DBMS_XMLGEN、XMLTYPE)、按页按文件按日志读原始数据的(pageinspect、逻辑解码、 + LOAD_FILE)、统计信息、会话记录、锁和日志里存着值的(Postgres 的 pg_stats、pg_stat_statements、pg_stat_activity; + SQLite 的 sqlite_stat4;MySQL 的 column_statistics、general_log、PROCESSLIST、INNODB_TRX、OPTIMIZER_TRACE、 + performance_schema,sys 库里读它的函数(按 sys.ps_ 开头认:ps_thread_trx_info、ps_thread_stack 交回别的线程执行过的 + 语句原文);MariaDB 的 column_stats、INNODB_LOCKS、QUERY_CACHE_INFO;Oracle 的直方图、记着最小最大值的 *_tab_columns、 + V$SQL 一族、SQL 调优集、固定下来的执行计划、闪回事务、LogMiner、审计记录),按名字认(masking 的 INDIRECT_FUNCTIONS、 INDIRECT_NAMES,可验证报告用的同一份名单),去掉注释再认一遍(函数名和括号之间夹着注释),自己建的同类函数、视图 认不出;information_schema、pg_catalog 里只有表结构的视图不算(核查 agent 查列名、类型常用)。不是查询的语句里, 只读表结构的 SHOW(表、列、索引、库、表状态、建库语句)、DESCRIBE 一张表照常,别的 SHOW(ENGINE INNODB STATUS、 diff --git a/backend/app/engine/masking.py b/backend/app/engine/masking.py index 263c000a..50526a4c 100644 --- a/backend/app/engine/masking.py +++ b/backend/app/engine/masking.py @@ -25,7 +25,9 @@ 不算(_value_columns、_band_only)。认不全的写法整份快照的列都算,不再逐个形状补规则:认不出落在哪一列的(PIVOT、 UNPIVOT 这类也算),(表达式).* 展开读到遮罩列的(复合类型的列、函数交回的记录,外面套了括号、起了别名的也算,展开成 几列、叫什么说不清;不只选出项,VALUES、行构造器、函数参数里的也算;Postgres 里 FROM 中可能交回复合类型的表值函数 - 同理,_spreads_record),语法树和实际执行的对不上的(OPAQUE_SQL:MySQL 会执行的注释、U&"…";没有 * 的一支选出项 + 同理,_spreads_record),VALUES 当一张表用(FROM、JOIN、LATERAL 里的,WITH 里的一段,UNION 的一支)又读到遮罩列的 + (各库给它的列起名不一样:Postgres、SQLite 叫 column1…,MySQL 叫 column_0…,MariaDB 拿第一行的值当列名, + _values_names),语法树和实际执行的对不上的(OPAQUE_SQL:MySQL 会执行的注释、U&"…";没有 * 的一支选出项 和结果的列数对不上),表名、SQL 写在字符串里交给函数、统计信息和会话记录里存着值的(INDIRECT_FUNCTIONS、 INDIRECT_NAMES:query_to_xml 这一族、dblink、pg_stats、MySQL 的 column_statistics、INNODB_TRX、Oracle 的直方图和 V$SQL……,可验证报告用的同一份),不只读表结构的 SHOW、EXPLAIN 一条语句(_reads_beyond:引擎状态、日志、出错信息、 @@ -862,10 +864,23 @@ def _function_sources(select: Any) -> list[Any]: return out -def _function_columns(source: Any) -> set[str] | None: +def _function_node(source: Any) -> Any: + """表值函数这类来源里真正出行的那个节点(函数、VALUES、JSON_TABLE……):套在外面的 LATERAL、表、括号先去掉。""" + from sqlglot import exp + + node = source.this if isinstance(source, exp.Lateral) else source + func = node.this if isinstance(node, exp.Table) else node + while isinstance(func, exp.Subquery): + func = func.this + return func + + +def _function_columns(source: Any, kind: str | None = None) -> set[str] | None: """表值函数这类来源出的列名(小写):写的列名表、别名(Postgres 里一列的函数出的列就叫别名,没起别名的叫函数名)、 - 函数名、常见的出列名(_FUNCTION_COLUMNS);JSON_TABLE 按 COLUMNS 里写的,VALUES 按 column1、column2…。不认得的函数 - 又没写列名表的,说不清出哪几列,为 None。""" + 函数名、常见的出列名(_FUNCTION_COLUMNS);JSON_TABLE 按 COLUMNS 里写的,VALUES 按数据源的类型 kind 认(_values_names: + Postgres、SQLite 叫 column1…,MySQL 叫 column_0…)。不认得的函数又没写列名表的、VALUES 的列名认不准又没写列名表的 + (MariaDB、Oracle),说不清出哪几列,为 None。kind 没给的按 Postgres、SQLite 的叫法:_has_column 只在 Postgres 的 + 整行引用、(s).* 里要知道来源出不出某一列。""" from sqlglot import exp node = source.this if isinstance(source, exp.Lateral) else source @@ -877,16 +892,14 @@ def _function_columns(source: Any) -> set[str] | None: names.add(alias.name.lower()) listed = [c.name.lower() for c in alias.args.get("columns") or [] if c.name] names |= set(listed) - func = node.this if isinstance(node, exp.Table) else node - while isinstance(func, exp.Subquery): - func = func.this + func = _function_node(source) if isinstance(func, exp.JSONTable): schema = func.args.get("schema") defined = [str(c.name).lower() for c in (schema.expressions if schema is not None else []) if c.name] return names | set(defined) if defined or listed else None if isinstance(func, exp.Values): - width = max((len(row.expressions) for row in func.expressions if isinstance(row, exp.Tuple)), default=1) - return names | {f"column{i}" for i in range(1, width + 1)} + found = _values_names(func, kind, listed) + return names | {n for n in found if n} if listed or all(found) else None name = (func.name if isinstance(func, exp.Anonymous) else func.sql_name() if isinstance(func, exp.Func) else "").lower() if name: @@ -894,6 +907,85 @@ def _function_columns(source: Any) -> set[str] | None: return names if listed or name in _TABLE_FUNCTIONS else None +def _values_rows(values: Any, kind: str | None) -> list[list[Any]]: + """一个 VALUES 的每一行按列摊开的各项。MySQL 的一行写作 ROW(…),解析器读成一个元组里套着一个 ROW 函数,摊开它的 + 参数;Postgres 的 VALUES (ROW(…)) 是一列复合类型,不摊。""" + from sqlglot import exp + + from app.data.sql_semantics import _read + + mysql = _read(kind) == "mysql" + rows: list[list[Any]] = [] + for row in values.expressions: + items = list(row.expressions) if isinstance(row, exp.Tuple) else [row] + if mysql and len(items) == 1 and isinstance(items[0], exp.Anonymous) and items[0].name.upper() == "ROW": + items = list(items[0].expressions) + rows.append(items) + return rows + + +def _values_names(values: Any, kind: str | None, listed: list[str]) -> list[str | None]: + """一个 VALUES 按次序的列名(小写):写了列名表的前几列按列名表(Postgres 的列名表可以只写前几列),后面的按各库的 + 叫法——Postgres、SQLite 叫 column1、column2…,MySQL 8.0 叫 column_0、column_1…(从 0 数起);MariaDB 拿第一行各项的 + 写法当列名,Oracle 的文档只有写了列名表的写法,认不准,记 None。kind 是数据源的类型(sqlite / postgresql / mysql / + mariadb / oracle)或解析用的方言名,没给的按 Postgres、SQLite 的叫法。以前一律按 column1… 认,MySQL 的 column_0 对不上, + 外层按名字取、照原名带出的号码原样交了出去。""" + from app.data.sql_semantics import _read + + width = max((len(row) for row in _values_rows(values, kind)), default=1) + read = _read(kind) + if (kind or "").strip().lower() == "mariadb" or read == "oracle": + defaults: list[str | None] = [None] * width + elif read == "mysql": + defaults = [f"column_{i}" for i in range(width)] + else: + defaults = [f"column{i}" for i in range(1, width + 1)] + return [*listed, *defaults[len(listed):]] + + +def _values_listed(values: Any) -> list[str]: + """一个 VALUES 写的列名表(小写):写在它自己、套在外面的括号或 LATERAL 上((VALUES …) AS v(a, b))。WITH v(a, b) 的 + 列名表记在 WITH 的那一段上,按派生来源认(_source_names),不在这里。""" + from sqlglot import exp + + node, alias = values, values.args.get("alias") + while alias is None and isinstance(node.parent, (exp.Subquery, exp.Lateral)) and node.arg_key == "this": + node = node.parent + alias = node.args.get("alias") + return [c.name.lower() for c in alias.args.get("columns") or [] if c.name] if alias is not None else [] + + +def _values_table(values: Any) -> bool: + """这个 VALUES 当一张表用,交出的是行、外层按列名取或照原名带出:FROM、JOIN、LATERAL 里的,WITH 里的一段、UNION 这类 + 的一支(不套括号的解析器改写成 SELECT * FROM (VALUES …) AS _values,套了括号的原样挂着),整条就是它。标量子查询、 + IN / ANY / EXISTS 里的不算:值经外层的表达式带出(_value_columns 已经算上),或者只用来筛行。""" + from sqlglot import exp + + node = values + while isinstance(node.parent, exp.Subquery): + node = node.parent + return node.parent is None or isinstance(node.parent, (exp.From, exp.Join, exp.Lateral, exp.CTE, exp.SetOperation)) + + +def _values_taken(values: Any, names: set[str], kind: str | None) -> set[str]: + """一个 VALUES 里值取自 names 的那几列的列名(_values_names):按位置,任何一行的那一项取自 names 就算(同 UNION 的 + 几支)。一行里 (表达式).*、s.* 展开成几列说不清,它和排在后面的项落在第几列认不出,不算;列名认不准的(None)不算。 + 这些名字照传下去(外层取它们的列、SQL 里拿它们做比较的字面量都按指代遮罩列认),整份遮时也只收这几列的值:同一行里 + 没碰遮罩列的区域不收(以前 VALUES 出的每一列都算,区域名在同一次核查里处处被换掉)。""" + from sqlglot import exp + + labels = _values_names(values, kind, _values_listed(values)) + out: set[str] = set() + for row in _values_rows(values, kind): + for i, item in enumerate(row): + node = _unwrapped(item) + if _is_star(node) or (isinstance(node, exp.Dot) and isinstance(node.expression, exp.Star)): + break + if i < len(labels) and labels[i] and _reads(names, _value_columns(item)): + out.add(labels[i]) + return out + + def _spreads_record(source: Any) -> bool: """Postgres 里这个表值函数这类来源可能交回复合类型、展开成它的各个字段(字段名 SQL 里看不出,同 (表达式).*): - unnest:数组的元素是复合类型时展开成各个字段(unnest(ARRAY[p.card]) 出 addr、tel),元素是什么类型 SQL 里看不出; @@ -904,9 +996,7 @@ def _spreads_record(source: Any) -> bool: node = source.this if isinstance(source, exp.Lateral) else source alias = source.args.get("alias") or (node.args.get("alias") if node is not source else None) - func = node.this if isinstance(node, exp.Table) else node - while isinstance(func, exp.Subquery): - func = func.this + func = _function_node(source) if isinstance(func, (exp.Values, exp.JSONTable)): return False name = (func.name if isinstance(func, exp.Anonymous) else func.sql_name() if isinstance(func, exp.Func) @@ -1495,9 +1585,11 @@ def _tainted(sql: str, kind: str | None, masked: set[str], width: int, tables: d 位置是给没起别名的表达式用的(UPPER(buyer_phone) 的结果列名对不上任何名字):和 * 写在同一支里的,* 前面的从头数、 后面的从末尾倒着数。认不出的——两个 * 之间的项、派生来源里没起别名的表达式(外层的 * 会把它原样带出来,列名 又认不准)——整份快照的列都算(同解析不了)。 - FROM、JOIN、LATERAL 里的表值函数(json_each、unnest、JSON_TABLE、LATERAL VALUES)参数里写着这些名字的,它出的列 - 都算(_function_columns):遮罩列是 JSON 时,json_each 拆开是最自然的写法。出哪几列说不清的整份快照都算,Postgres - 里可能交回复合类型、展开成各个字段的(unnest、只写了列名的列名表,_spreads_record)也算。 + FROM、JOIN、LATERAL 里的表值函数(json_each、unnest、JSON_TABLE)参数里写着这些名字的,它出的列都算 + (_function_columns):遮罩列是 JSON 时,json_each 拆开是最自然的写法。出哪几列说不清的整份快照都算,Postgres + 里可能交回复合类型、展开成各个字段的(unnest、只写了列名的列名表,_spreads_record)也算。VALUES 当一张表用(FROM、 + JOIN、LATERAL 里的,WITH 里的一段,UNION 的一支,_values_table)读到这些名字的也整份算:各库给它的列起名不一样 + (_values_names),不逐个方言、逐个位置认;值取自这些名字的那几列,按方言认得出名字的照传下去(_values_taken)。 * 展开的列换了名字(写了列名表的派生来源、UNION 后一支的 *):* 带出这些名字的值时,它展开落到的那几列照位置算 (_star_places,展开数据源里的表的按 tables 数位置,数不清的整段算);标量子查询里的 *、函数参数里的 *(to_json(s.*)、 JSON_OBJECT(*))、整行引用(row_to_json(s))按 _value_columns 算进外层的表达式。表的列名表、TABLE x、套了括号的 * @@ -1538,12 +1630,20 @@ def _tainted_in(root: Any, sql: str, kind: str | None, masked: set[str], width: selects = list(root.find_all(exp.Select)) blocks = _blocks(root) sources = [(top, listed) for top, role, listed in blocks if role == "source"] - functions = [source for select_ in selects for source in _function_sources(select_)] + # VALUES 不按表值函数认(下面另认):按出列名认时它出的每一列都算,同一行里没碰遮罩列的区域也收成了遮罩值 + functions = [source for select_ in selects for source in _function_sources(select_) + if not isinstance(_function_node(source), exp.Values)] # Postgres 里可能交回复合类型、展开成各个字段的表值函数(_spreads_record):读到遮罩列时出哪几列说不清,整份遮 spreading = {id(source) for source in functions if _spreads_record(source)} \ if _read(kind) in _ROW_DIALECTS else set() - unreadable = False - for _ in range(len(selects) + len(sources) + len(functions) + 1): + # VALUES 当一张表用(FROM、JOIN、LATERAL 里的,WITH 里的一段,UNION 的一支,整条就是它,_values_table)又读到遮罩列 + # (含派生出来的名字):各库给它的列起名不一样(Postgres、SQLite 叫 column1…,MySQL 叫 column_0…,MariaDB 拿第一行的 + # 值当列名),外层按名字取、照原名带出时对不上,以前 MySQL 派生表、CTE、LATERAL 里的 VALUES ROW 取了号码,原样交了 + # 出去;套了括号的 VALUES(整条就是它、UNION 的一支)不在任何 SELECT 的 FROM 里,哪个库都没认。不再逐个方言、逐个 + # 位置认,整份遮;按方言认得出的、值取自遮罩列的那几列的名字照传下去(_values_taken),整份遮时只收它们的值 + tables_of_values = [values for values in root.find_all(exp.Values) if _values_table(values)] + unreadable = tabled = False + for _ in range(len(selects) + len(sources) + len(functions) + len(tables_of_values) + 1): before = len(names) for select_ in selects: for item in select_.expressions: @@ -1554,21 +1654,24 @@ def _tainted_in(root: Any, sql: str, kind: str | None, masked: set[str], width: names |= _source_names(top, listed, names, tables)[0] for source in functions: if _reads(names, _value_columns(source)): - columns = None if id(source) in spreading else _function_columns(source) + columns = None if id(source) in spreading else _function_columns(source, kind) unreadable = unreadable or columns is None names |= columns or set() + for values in tables_of_values: + if _reads(names, _value_columns(values)): + tabled = True + names |= _values_taken(values, names, kind) if len(names) == before: break - # 认不出的整份遮:表值函数出哪几列说不清、派生来源里有认不出名字的列、PIVOT 这类读到遮罩列,(表达式).* 读到遮罩列 - # (展开成几列、叫什么都说不清),语法树和实际执行的对不上、SQL 写在字符串里交给函数执行、整表导出、统计信息里的值 - # (_opaque)。(表达式).* 不只看选出项(那几个 _normalized 已经换掉,值取自哪些列在 opaque 里):VALUES 里同样展开成 - # 好几列(Postgres 文档:选出列表、VALUES、行构造器里的 composite.* 都展开),VALUES 里的一项是 s.*、(s.*) 的也一样; - # LATERAL VALUES 按表值函数认时一个元组项只算一列,column2 起的号码原样交了出去。不逐个位置认:(表达式).* 在树上任何 - # 位置(行构造器、函数参数里的不展开成几列,也一样,多遮)、VALUES 里的 *,读到遮罩列就整份遮 + # 认不出的整份遮:表值函数出哪几列说不清、VALUES 当一张表用(见上)、派生来源里有认不出名字的列、PIVOT 这类读到遮罩列, + # (表达式).* 读到遮罩列(展开成几列、叫什么都说不清),语法树和实际执行的对不上、SQL 写在字符串里交给函数执行、整表 + # 导出、统计信息里的值(_opaque)。(表达式).* 不只看选出项(那几个 _normalized 已经换掉,值取自哪些列在 opaque 里): + # VALUES 里同样展开成好几列(Postgres 文档:选出列表、VALUES、行构造器里的 composite.* 都展开),LATERAL VALUES 按表值 + # 函数认时一个元组项只算一列,column2 起的号码原样交了出去。不逐个位置认:(表达式).* 在树上任何位置(行构造器、函数 + # 参数里的不展开成几列,也一样,多遮)读到遮罩列就整份遮;VALUES 里的 *、(s.*) 读到遮罩列的,那个 VALUES 就读到了 + # 遮罩列,按上面整份遮 expanded = [node.this for node in root.find_all(exp.Dot) if isinstance(node.expression, exp.Star)] - expanded += [item for values in root.find_all(exp.Values) for row in values.expressions - if isinstance(row, exp.Tuple) for item in row.expressions if _is_star(_unwrapped(item))] - everything = unreadable or any(_source_names(top, listed, names, tables)[1] for top, listed in sources) \ + everything = unreadable or tabled or any(_source_names(top, listed, names, tables)[1] for top, listed in sources) \ or any(_reads(names, _value_columns(node)) for node in root.find_all(exp.Pivot, exp.MatchRecognize)) \ or any(_reads(names, cols) for cols in opaque) \ or any(_reads(names, _value_columns(node)) for node in expanded) or _opaque(sql, kind) @@ -1641,7 +1744,10 @@ INDIRECT_FUNCTIONS = ( #: - 别的会话执行过的 SQL 原文:Postgres 的 pg_stat_activity、pg_stat_statements、本连接预备过的语句;MySQL 的 general_log、 #: slow_log、PROCESSLIST、正在执行的事务(information_schema.INNODB_TRX 的 trx_query)、performance_schema(语句历史 #: events_statements_*、线程正在执行的 SQL),sys 库里建在 performance_schema 上的会话、锁等待视图(只认这几张,Oracle -#: 常写的 sys.dual 不算;带 x$ 的不排版的那一份按下面的 X$ 算);MariaDB 的查询缓存(QUERY_CACHE_INFO);Oracle 的动态视图 +#: 常写的 sys.dual 不算;带 x$ 的不排版的那一份按下面的 X$ 算)和读 performance_schema 的函数(按 sys.ps_ 开头认: +#: ps_thread_trx_info 交回那个线程的事务里执行过的语句原文,ps_thread_stack 交回它的语句栈,MySQL 8.0 实测带着条件里 +#: 的号码;同一族只交回编号、开关的 ps_thread_id、ps_is_* 也算进来,多遮。不写 sys. 的要数据源的默认库就是 sys 才 +#: 执行得了,不认,同上面的视图);MariaDB 的查询缓存(QUERY_CACHE_INFO);Oracle 的动态视图 #: V$ / GV$ / X$ 一律算,不逐个认(V$SQL 一族、v$sql_bind_capture、打开着的游标、LogMiner 的 V$LOGMNR_CONTENTS、XML 的 #: 审计记录都在里面),AWR 历史 DBA_HIST_*、审计记录,SQL 调优集里的语句和绑定变量(*_sqlset_*)、调优顾问记下的 #: (*_sqltune_*)、SQL Access Advisor 的工作负载(*_advisor_sqlw_*)、固定下来的执行计划和 SQL Profile、Patch @@ -1659,7 +1765,7 @@ INDIRECT_NAMES = ( r"pg_stat_activity", r"pg_stat_statements", r"pg_prepared_statements", r"general_log", r"slow_log", r"processlist", r"innodb_trx", r"innodb_locks", r"query_cache_info", r"optimizer_trace", r"performance_schema", r"events_statements_\w+", - r"sys[\"`]?\s*\.\s*[\"`]?(?:x\$)?(?:session|processlist|innodb_lock_waits|schema_table_lock_waits)", + r"sys[\"`]?\s*\.\s*[\"`]?(?:(?:x\$)?(?:session|processlist|innodb_lock_waits|schema_table_lock_waits)|ps_\w+)", r"(?:g?v|x)\$\w+", r"(?:dba|cdb)_hist_\w+", r"(?:all|dba|user|cdb)_(?:sqlset|sqltune|advisor_sqlw)_\w+", r"(?:dba|cdb)_sql_(?:plan_baselines|profiles|patches)", r"flashback_transaction_query", @@ -2707,9 +2813,9 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str SELECT *(展开的列换了名字的也算)、表值函数、整行引用和函数参数里的 *、列名表、TABLE x;COUNT、只在条件里用到、 分档当分组键的不算,分档写在派生来源里、外层按它分组的也不算);认不全的(SQL 解析不了又写着遮罩列、*、TABLE x, 或者是 Postgres 的;(表达式).* 读到遮罩列(VALUES 里的也算)、Postgres 里可能交回复合类型的表值函数读到遮罩列, - MySQL 会执行的注释、U&"…",表名、SQL 写在字符串里交给函数,统计信息、会话记录里存着值的视图,不只读表结构的 - SHOW、EXPLAIN 一条语句,选出项和结果的列数对不上,分析出意外、超出工作量上限)整份结果的列都遮 - (QueryMask.whole)。这几列的格子换成「已遮罩」,类型记成文本 + VALUES 当一张表用又读到遮罩列,MySQL 会执行的注释、U&"…",表名、SQL 写在字符串里交给函数,统计信息、会话记录里 + 存着值的视图和函数,不只读表结构的 SHOW、EXPLAIN 一条语句,选出项和结果的列数对不上,分析出意外、超出工作量 + 上限)整份结果的列都遮(QueryMask.whole)。这几列的格子换成「已遮罩」,类型记成文本 - 别的列按值换,只换文字格:整格就是遮罩值的换掉,文字格里夹着的那一截换掉(换了写法的号码也认, _Secrets.loose_spans),换过的格子记下来(QueryMask.cells)。遮罩值是这份结果里的加上 known:先查号码、再查写着 号码的备注、没设遮罩的联系方式表(别的库里的也一样)时,后一份同样遮好。门槛同按值遮(SECRET_DIGITS、SECRET_MIN); diff --git a/backend/tests/test_masking_values_and_show.py b/backend/tests/test_masking_values_and_show.py index d7b5b6ea..acdd22c9 100644 --- a/backend/tests/test_masking_values_and_show.py +++ b/backend/tests/test_masking_values_and_show.py @@ -8,6 +8,15 @@ (ENGINE INNODB STATUS、BINLOG / RELAYLOG EVENTS、REPLICA STATUS、WARNINGS、CREATE TABLE / VIEW……)和 EXPLAIN 一条 语句。以前工具边界的名单里没有,可验证报告那份整个挡 information_schema、不附带解析不了的快照,反而挡得住 +复核第 3 轮补的同一类: +- VALUES 当一张表用(FROM、JOIN、LATERAL 里的,WITH 里的一段,UNION 的一支)又读到遮罩列(含派生出来的名字):各库给 + VALUES 的列起名不一样(Postgres、SQLite 叫 column1…,MySQL 8.0 叫 column_0…,MariaDB 拿第一行的值当列名),以前一律 + 按 column1… 认,MySQL 派生表、CTE、LATERAL 里的 VALUES ROW 取了号码,原样交给模型、落库;套了括号的 VALUES(整条 + 就是它、UNION 的一支)解析器原样挂在树上,哪个库都漏。现在不论怎么起名都整份遮;列名按方言认,整份遮时只收值取自 + 遮罩列的那几列的值 +- MySQL sys 库里读 performance_schema 的函数(sys.ps_thread_trx_info 交回线程的事务里执行过的语句原文,ps_thread_stack + 同类):按 sys.ps_ 开头认,工具边界和可验证报告共用一份名单 + 这里没有真的数据库:结果按它交回的样子造(复合类型展开成几列、SHOW 交回的几列,照 Postgres 18、MySQL 8.0 实测的样子)。 数据都是虚构的:手机号、地址、区域。 """ @@ -93,21 +102,19 @@ def test_an_expression_star_inside_values_masks_the_whole_result(sql, columns, r assert done.payload["rows"] == [[MASKED_WORD] * len(columns)] and not _leaks(done) -@pytest.mark.parametrize("sql, columns, row, hidden", [ +@pytest.mark.parametrize("sql, columns, row", [ pytest.param("SELECT v.* FROM people p, LATERAL (VALUES ((p.address).*)) v", ["column1", "column2"], - ["某路", "某市"], [], id="没设遮罩的复合类型"), - pytest.param("SELECT v.column1 AS r FROM sales s, LATERAL (VALUES (s.region)) v", ["r"], ["华东"], [], + ["某路", "某市"], id="没设遮罩的复合类型"), + pytest.param("SELECT v.column1 AS r FROM sales s, LATERAL (VALUES (s.region)) v", ["r"], ["华东"], id="没碰遮罩列的 VALUES"), - pytest.param("SELECT v.column1 AS p FROM sales s, LATERAL (VALUES (s.buyer_phone)) v", ["p"], [PHONE], ["p"], - id="VALUES 里直接写遮罩列"), pytest.param("SELECT v.column2 AS r FROM (SELECT id, region FROM sales) q, LATERAL (VALUES (q.*)) v", ["r"], - ["华东"], [], id="没取遮罩列的子查询的 *"), + ["华东"], id="没取遮罩列的子查询的 *"), ]) -def test_values_without_an_expression_star_on_a_masked_column_is_read_as_before(sql, columns, row, hidden): - """VALUES 里没有读到遮罩列的 (表达式).*:照旧按列认,没碰遮罩列的照常,直接写遮罩列的只遮那一列。""" +def test_values_that_reads_no_masked_column_is_read_as_before(sql, columns, row): + """VALUES 没读到遮罩列(没设遮罩的复合类型、没碰遮罩列的列、没取遮罩列的子查询的 *):照常,一列都不遮。读到遮罩列的 + (直接写遮罩列的也算)当一张表用就整份遮,见下一节。""" done = masked(sql, columns, [list(row)]) - assert not done.whole and done.columns == hidden - assert done.payload["rows"] == [[MASKED_WORD if c in hidden else v for c, v in zip(columns, row)]] + assert not done.whole and done.columns == [] and done.payload["rows"] == [list(row)] def test_an_expression_star_inside_a_function_argument_masks_the_whole_result_too(): @@ -117,6 +124,142 @@ def test_an_expression_star_inside_a_function_argument_masks_the_whole_result_to assert done.whole and done.payload["rows"] == [[MASKED_WORD, MASKED_WORD]] and not _leaks(done) +# -------------------------------------------------------------------------- +# VALUES 当一张表用(复核第 3 轮) +# -------------------------------------------------------------------------- + + +#: VALUES 当一张表用、读到遮罩列:(方言, SQL, 结果列, 各行),号码在最后一列。MySQL 8.0 给 VALUES 的列起名 column_0、 +#: column_1…(从 0 数起),MariaDB 拿第一行的值当列名;以前一律按 column1… 认,MySQL、MariaDB 的这几种号码原样交了出去 +VALUES_TABLES = [ + pytest.param("mysql", "SELECT * FROM (VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1))) AS v", ["column_0"], + [[PHONE]], id="MySQL-派生表"), + pytest.param("mysql", "SELECT x.column_0 FROM sales s, LATERAL (VALUES ROW(s.buyer_phone)) AS x", ["column_0"], + [[PHONE]], id="MySQL-LATERAL"), + pytest.param("mysql", "WITH v AS (VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1))) SELECT * FROM v", + ["column_0"], [[PHONE]], id="MySQL-CTE"), + pytest.param("mysql", "SELECT v.column_0 AS c FROM (VALUES ROW((SELECT buyer_phone FROM sales WHERE id = 1)), " + "ROW((SELECT buyer_phone FROM sales WHERE id = 2))) AS v", ["c"], [[PHONE], ["13900000002"]], + id="MySQL-外层改了名字"), + pytest.param("mysql", "SELECT s.id, t.column_1 FROM sales s JOIN LATERAL (VALUES ROW(s.region, s.buyer_phone)) " + "AS t ON TRUE", ["id", "column_1"], [[1, PHONE]], id="MySQL-JOIN LATERAL 连同编号"), + pytest.param("mysql", "SELECT w.* FROM sales s, LATERAL (SELECT * FROM (VALUES ROW(s.buyer_phone)) v) AS w", + ["column_0"], [[PHONE]], id="MySQL-LATERAL 子查询里的 VALUES"), + pytest.param("mysql", "SELECT (SELECT column_0 FROM (VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1))) v) AS p", + ["p"], [[PHONE]], id="MySQL-标量子查询里的 VALUES"), + pytest.param("mysql", "WITH q AS (SELECT buyer_phone AS p FROM sales) " + "SELECT v.column_0 FROM q, LATERAL (VALUES ROW(q.p)) AS v", ["column_0"], [[PHONE]], + id="MySQL-取派生出来的名字"), + pytest.param("mysql", "VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1)) UNION ALL SELECT region FROM sales", + ["column_0"], [[PHONE], ["华东"]], id="MySQL-UNION 第一支"), + # 套了括号的 VALUES 解析器不改写成 FROM 里的 VALUES:整条就是它、UNION 的一支都原样挂在树上,以前不认,Postgres + # 里也漏 + pytest.param("mysql", "(VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1)))", ["column_0"], [[PHONE]], + id="MySQL-整条套括号"), + pytest.param("postgresql", "(VALUES ((SELECT buyer_phone FROM sales LIMIT 1))) UNION ALL " + "(SELECT region FROM sales)", ["column1"], [[PHONE], ["华东"]], id="Postgres-UNION 第一支套括号"), + pytest.param("mysql", "SELECT * FROM ((VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1))) UNION ALL " + "(SELECT 'a')) t", ["column_0"], [[PHONE], ["a"]], id="MySQL-派生表里 UNION 的一支"), + pytest.param("mariadb", "SELECT * FROM (VALUES ((SELECT buyer_phone FROM sales LIMIT 1))) AS v", + ["(SELECT buyer_phone FROM sales LIMIT 1)"], [[PHONE]], id="MariaDB-派生表"), + pytest.param("mariadb", "SELECT * FROM (VALUES ('华东'), ((SELECT buyer_phone FROM sales LIMIT 1))) AS v", + ["华东"], [["华东"], [PHONE]], id="MariaDB-第一行的值当列名"), + pytest.param("postgresql", "SELECT v.column1 AS p FROM sales s, LATERAL (VALUES (s.buyer_phone)) v", ["p"], + [[PHONE]], id="Postgres-LATERAL"), + pytest.param("postgresql", "SELECT s.id, v.k, v.val FROM sales s CROSS JOIN LATERAL (VALUES ('phone', " + "s.buyer_phone), ('region', s.region)) AS v(k, val)", ["id", "k", "val"], + [[1, "phone", PHONE], [1, "region", "华东"]], id="Postgres-行转列"), + pytest.param("sqlite", "WITH v AS (VALUES ((SELECT buyer_phone FROM sales LIMIT 1))) SELECT * FROM v", + ["column1"], [[PHONE]], id="SQLite-CTE"), +] + + +@pytest.mark.parametrize("kind, sql, columns, rows", VALUES_TABLES) +def test_a_values_table_on_a_masked_column_masks_the_whole_result(kind, sql, columns, rows): + """VALUES 当一张表用、读到遮罩列(含派生出来的名字):不论各库给它的列起什么名字,整份结果的每一格都遮,同一份里 + 没碰遮罩列的编号、区域也遮(多遮)。以前 MySQL、MariaDB 的这几种和套了括号的 VALUES masked=[],号码原样交给模型、 + 落库;Postgres、SQLite 的按列名认得出,只遮了那几列。""" + done = masked(sql, columns, [list(r) for r in rows], kind=kind) + assert done.whole and done.columns == columns + assert done.payload["rows"] == [[MASKED_WORD] * len(columns)] * len(rows) and not _leaks(done) + + +@pytest.mark.parametrize("kind, sql, columns, row", [ + pytest.param("mysql", "SELECT x.column_0 AS r FROM sales s, LATERAL (VALUES ROW(s.region)) AS x", ["r"], ["华东"], + id="MySQL-没碰遮罩列"), + pytest.param("mysql", "SELECT s.region, b.column_1 AS 档 FROM sales s JOIN (VALUES ROW(0, '小单'), " + "ROW(20000, '大单')) AS b ON s.amount >= b.column_0", ["region", "档"], ["华东", "大单"], + id="MySQL-写死的分档表"), + pytest.param("mysql", "SELECT region FROM sales UNION ALL VALUES ROW('其他')", ["region"], ["其他"], + id="MySQL-UNION 一支写死的值"), +]) +def test_a_values_table_reading_no_masked_column_stays_visible(kind, sql, columns, row): + """VALUES 当一张表用、却没读到遮罩列(写死的分档表、没碰遮罩列的 LATERAL、UNION 里写死的一支):照常。""" + done = masked(sql, columns, [list(row)], kind=kind) + assert not done.whole and done.columns == [] and done.payload["rows"] == [list(row)] + + +@pytest.mark.parametrize("kind, sql", [ + # 号码在 ROW(…) 的第二项:MySQL 的一行解析成一个元组里套着一个 ROW 函数,要摊开数位置 + pytest.param("mysql", "SELECT x.column_1 AS p, x.column_0 AS r FROM sales s, " + "LATERAL (VALUES ROW(s.region, s.buyer_phone)) AS x", id="MySQL"), + pytest.param("postgresql", "SELECT v.column1 AS p, v.column2 AS r FROM sales s, " + "LATERAL (VALUES (s.buyer_phone, s.region)) v", id="Postgres"), + pytest.param("postgresql", "SELECT v.a AS p, v.b AS r FROM sales s, LATERAL (VALUES (s.buyer_phone, s.region)) " + "v(a, b)", id="Postgres-列名表"), + pytest.param("sqlite", "WITH v AS (VALUES ((SELECT buyer_phone FROM sales LIMIT 1), '华东')) " + "SELECT column1 AS p, column2 AS r FROM v", id="SQLite-CTE"), +]) +def test_only_the_values_columns_taken_from_a_masked_column_are_collected(kind, sql): + """VALUES 的列名按方言认(MySQL 叫 column_0…,Postgres、SQLite 叫 column1…,写了列名表的按列名表):整份遮时只收值 + 取自遮罩列的那几列的值当遮罩值,后面的查询、回答里同一个号码照样按值换掉;同一行里的区域不收。以前 MySQL 按 column1 + 认,号码一个都没收;Postgres、SQLite 把 VALUES 出的每一列都收了,区域名在同一次核查里处处被换掉。""" + done = masked(sql, ["p", "r"], [[PHONE, "华东"]], kind=kind) + assert done.whole and done.payload["rows"] == [[MASKED_WORD, MASKED_WORD]] + assert PHONE in done.secrets.strong and "华东" not in done.secrets.strong + + +def test_items_after_an_expanding_star_in_a_values_row_are_not_collected_by_place(): + """一行里的 (表达式).* 展开成几列说不清:排在它后面的号码实际落在第三列,按位置数成第二列就收了没设遮罩的城市。 + 整份照遮,这一行不按位置收值(少收不露:这份结果的值本来就没交给模型)。""" + done = masked("SELECT v.column2 AS c, v.column3 AS p FROM people p, sales s, " + "LATERAL (VALUES ((p.address).*, s.buyer_phone)) v", ["c", "p"], [["某市", PHONE]]) + assert done.whole and done.payload["rows"] == [[MASKED_WORD, MASKED_WORD]] + assert "某市" not in done.secrets.strong + + +@pytest.mark.parametrize("kind, sql, named, unnamed", [ + pytest.param("mysql", "SELECT * FROM sales s, LATERAL (VALUES ROW(s.id, s.region)) AS x", {"column_0", "column_1"}, + {"column1", "column2"}, id="MySQL"), + pytest.param("postgresql", "SELECT * FROM sales s, LATERAL (VALUES (s.id, s.region)) AS x", {"column1", "column2"}, + {"column_0"}, id="Postgres"), + pytest.param("sqlite", "SELECT * FROM (VALUES (1, '华东')) AS x", {"column1", "column2"}, {"column_0"}, + id="SQLite"), + # 列名表只改前几列的名字:第一列叫 n,不再叫 column1(Postgres 里按 column1 取会报错) + pytest.param("postgresql", "SELECT * FROM (VALUES (1, '华东')) AS x(n)", {"n", "column2"}, {"column1"}, + id="Postgres-列名表只写前几列"), +]) +def test_values_columns_are_named_the_way_each_database_names_them(kind, sql, named, unnamed): + """表值函数这类来源出的列名(_function_columns)里,VALUES 的列按方言起名:别处按列名认来源出不出某一列时对得上。""" + from app.data.sql_semantics import _parse, _read + + root, _ = _parse(sql, _read(kind)) + columns = masking._function_columns(masking._function_sources(root)[-1], kind) + assert named <= columns and not unnamed & columns + + +@pytest.mark.parametrize("kind, sql", [ + pytest.param("mariadb", "SELECT * FROM (VALUES (1, '华东')) AS x", id="MariaDB 拿第一行的值当列名"), + pytest.param("oracle", "SELECT * FROM (VALUES (1, '华东')) x", id="Oracle 没写列名表"), +]) +def test_values_columns_that_cannot_be_named_are_unknown(kind, sql): + """VALUES 的列名认不准(MariaDB 拿第一行的值当列名,Oracle 的文档只有写了列名表的写法)又没写列名表:说不清出哪几列。""" + from app.data.sql_semantics import _parse, _read + + root, _ = _parse(sql, _read(kind)) + assert masking._function_columns(masking._function_sources(root)[-1], kind) is None + + # -------------------------------------------------------------------------- # 端到端:核查 agent 的真实查询工具 # -------------------------------------------------------------------------- @@ -178,6 +321,20 @@ async def test_values_expansions_never_reach_the_model_or_the_store(monkeypatch, assert "1390000000" not in json.dumps(snapshot, ensure_ascii=False) and "1390000000" not in api +@pytest.mark.parametrize("sql", [ + "SELECT * FROM (VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1))) AS v", + "SELECT x.column_0 FROM sales s, LATERAL (VALUES ROW(s.buyer_phone)) AS x", + "WITH v AS (VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1))) SELECT * FROM v", +], ids=["派生表", "LATERAL", "CTE"]) +async def test_mysql_values_tables_never_reach_the_model_or_the_store(monkeypatch, fake_source, sql): + """端到端(MySQL,VALUES 的列叫 column_0):交给模型的、落库的、通用工件接口给的都只有「已遮罩」,masked_at_query + 是全部的列,快照记着整份遮。以前 masked_at_query 是空的,号码原样交出。""" + seen, snapshot, api = await _asked(monkeypatch, fake_source, "mysql", sql, ["column_0"], [[PHONE], ["13900000002"]]) + assert MASKED_WORD in seen and "1390000000" not in seen, seen + assert snapshot["masked_at_query"] == ["column_0"] and snapshot["masked_whole"] is True + assert "1390000000" not in json.dumps(snapshot, ensure_ascii=False) and "1390000000" not in api + + # -------------------------------------------------------------------------- # 记着别的会话的 SQL、被锁那一行的值的视图 # -------------------------------------------------------------------------- @@ -185,6 +342,10 @@ async def test_values_expansions_never_reach_the_model_or_the_store(monkeypatch, #: 另一个会话正在执行的语句(号码写在条件里):会话、事务、锁、日志一类的视图交回的就是它 BUSY = f"SELECT id, SLEEP(4) FROM sales WHERE buyer_phone = '{PHONE}' FOR UPDATE" +#: MySQL sys 库的 ps_thread_trx_info 交回的 JSON:那个线程的事务里执行过的语句原文在 statements_executed[].sql_text 里 +#: (MySQL 8.0 实测) +TRX_INFO = json.dumps([{"trx_id": "1234", "state": "ACTIVE", "statements_executed": [{"sql_text": BUSY}]}], + ensure_ascii=False) @pytest.mark.parametrize("sql, kind", [ @@ -220,6 +381,32 @@ def test_views_that_keep_other_sessions_sql_mask_the_whole_result(sql, kind): assert done.whole and done.payload["rows"] == [[MASKED_WORD]] +@pytest.mark.parametrize("sql, column", [ + pytest.param("SELECT sys.ps_thread_trx_info(81) AS t", "t", id="事务里执行过的语句"), + pytest.param("SELECT sys.ps_thread_trx_info(81)", "sys.ps_thread_trx_info(81)", id="不起别名"), + pytest.param("SELECT `sys`.`ps_thread_trx_info`(81) AS t", "t", id="反引号"), + pytest.param("SELECT SYS . PS_THREAD_TRX_INFO (81) AS t", "t", id="大写、隔着空格"), + pytest.param("SELECT sys.ps_thread_stack(81, FALSE) AS t", "t", id="语句栈"), + pytest.param("SELECT sys.ps_thread_trx_info(sys.ps_thread_id(NULL)) AS t", "t", id="套着取线程号"), +]) +def test_sys_functions_reading_performance_schema_mask_the_whole_result(sql, column): + """MySQL sys 库里读 performance_schema 的函数交回别的会话执行过的 SQL 原文(条件里的号码):按 sys.ps_ 开头认,整份 + 遮。以前名单里 sys 库只认会话、锁等待那几张视图,号码原样交给模型、落库。""" + done = masked(sql, [column], [[TRX_INFO]], kind="mysql") + assert done.whole and done.payload["rows"] == [[MASKED_WORD]] and not _leaks(done) + + +@pytest.mark.parametrize("kind, sql, column, value", [ + pytest.param("mysql", "SELECT sys.format_bytes(2048) AS b", "b", "2.00 KiB", id="sys 库里只排版的函数"), + pytest.param("mysql", "SELECT ps_total AS t FROM stats", "t", 12, id="ps_ 开头的列"), + pytest.param("oracle", "SELECT 1 AS n FROM sys.dual", "n", 1, id="Oracle 的 sys.dual"), +]) +def test_other_sys_names_stay_visible(kind, sql, column, value): + """sys 库里别的函数(只排版、换单位的)、不写 sys. 的 ps_ 开头的列名、Oracle 的 sys.dual:照常。""" + done = masked(sql, [column], [[value]], kind=kind) + assert not done.whole and done.payload["rows"] == [[value]] + + @pytest.mark.parametrize("sql, columns", [ pytest.param("SHOW ENGINE INNODB STATUS", ["Type", "Name", "Status"], id="InnoDB 状态"), pytest.param("show engine innodb status", ["Type", "Name", "Status"], id="小写"), @@ -289,7 +476,8 @@ def test_explaining_a_statement_masks_the_whole_result(sql): ("SELECT trx_query AS t FROM information_schema.INNODB_TRX", ["t"], [[BUSY]]), ("SHOW ENGINE INNODB STATUS", ["Type", "Name", "Status"], [["InnoDB", "", f"---TRANSACTION 1234, ACTIVE 2 sec\nMySQL thread id 9, query id 70 localhost root\n{BUSY}"]]), -], ids=["INNODB_TRX", "SHOW ENGINE INNODB STATUS"]) + ("SELECT sys.ps_thread_trx_info(81) AS t", ["t"], [[TRX_INFO]]), +], ids=["INNODB_TRX", "SHOW ENGINE INNODB STATUS", "sys.ps_thread_trx_info"]) async def test_other_sessions_sql_never_reaches_the_model_or_the_store(monkeypatch, fake_source, sql, columns, rows): """端到端(MySQL):另一个会话正在执行的 SQL 里写着号码,交给模型的、落库的、通用工件接口给的都只有「已遮罩」, 快照记着整份遮。以前 masked_at_query 为空,SQL 原文连同号码原样交出。""" @@ -306,3 +494,18 @@ def test_the_new_views_are_on_the_list_the_bundle_shares(): for sql in ("SELECT trx_query FROM information_schema.INNODB_TRX", "SELECT sql_text FROM dba_sqlset_statements", "SELECT undo_sql FROM flashback_transaction_query", "SELECT sql_redo FROM v$logmnr_contents"): assert build._INDIRECT_READS.search(sql) and masking._INDIRECT.search(sql), sql + + +@pytest.mark.parametrize("sql", [ + "SELECT sys.ps_thread_trx_info(81) AS t", + "SELECT `sys`.`ps_thread_stack`(81, FALSE) AS t", +]) +def test_the_sys_functions_are_on_the_list_the_bundle_shares(sql): + """可验证报告和工具边界共用一份名单:sys.ps_ 开头的函数两边都认,报告不附带这样的快照(只给哈希),它的列整份算取自 + 遮罩列。以前报告判它没碰遮罩列,照常附带快照,别的会话 SQL 里的号码随文件交出去。""" + from app.bundle import build + + snapshot = {"source": "shop", "sql": sql, "columns": ["t"], "rows": [[TRX_INFO]]} + assert build._INDIRECT_READS.search(sql) and masking._INDIRECT.search(sql) + assert build._untouched(snapshot, {"buyer_phone"}) is False + assert build._derived_columns(snapshot, {"buyer_phone"}) == {"buyer_phone", "t"}