diff --git a/backend/app/bundle/build.py b/backend/app/bundle/build.py index 30a9c648..a0214469 100644 --- a/backend/app/bundle/build.py +++ b/backend/app/bundle/build.py @@ -125,7 +125,7 @@ class _Store: def hidden(self, artifact: str) -> set[str]: """一份读得出来的查询快照里值取自遮罩列的列(小写):查询当时记下的、数据源现在设的遮罩列,加上 SQL 里给它们 - 改了名字的;SQL 认不出结果里哪一列取自哪一列的,整份的列(_derived_columns)。""" + 改了名字的;SQL 认不出结果里哪一列取自哪一列的(解析不了的也算),整份的列(_derived_columns)。""" if artifact not in self._hidden: content = self.content(artifact) base = _snapshot_hidden(content, self._source_masks) if isinstance(content, dict) else set() @@ -156,9 +156,9 @@ class _Store: 认哪个输入取自哪份查询结果的哪几列,和证据面板、推演同一个认法(recompute.input_origin);按位置取的一格顺着 取值链从写变量的节点在口径卡算这一次之前的产出起走。那几列算不算遮罩列,除了列名对上的,SQL 里给遮罩列起了别名、 套了函数、经子查询或 WITH 转了一道的也算(hidden,同质询、双盲复算的认法);SQL 认不出结果里哪一列取自哪一列的 - (列名表、整行引用这类,快照本身因此只给哈希),整份快照的列都算,两条路的判断一致。核对不了的宁可多挡:快照读不出来、 - 取值链上的产出读不出来、那份结果没存快照只带着数据源名、定位里没写列。认得出、可结果既没带快照的 id 也没带数据源名 - 的(数据整形改过形状)认不出出处,不算。 + (解析不了、列名表、整行引用这类,快照本身因此只给哈希),整份快照的列都算,两条路的判断一致。核对不了的宁可多挡: + 快照读不出来、取值链上的产出读不出来、那份结果没存快照只带着数据源名、定位里没写列。认得出、可结果既没带快照的 id + 也没带数据源名的(数据整形改过形状)认不出出处,不算。 卡本身按解析出来的内容认,不看哈希(同 content):服务端的文件被改过,嵌进页面的照样是这份原字节,卡上记着的号码 也照样在里面。读得出字节、解析不出一张卡的(截断的 JSON、不是对象)认不出有哪些输入,整张按带遮罩算,和读不出来的 @@ -241,27 +241,41 @@ _INDIRECT_READS = re.compile(r"(? bool: @@ -307,10 +321,11 @@ def _untouched(snapshot: dict[str, Any], hidden: set[str], seen: Callable[[], An def _derived_columns(snapshot: dict[str, Any], base: set[str]) -> set[str]: """一份快照里值取自遮罩列的结果列(小写):遮罩列本身(base),加上 SQL 里给它们起别名、套函数、经子查询或 WITH - 转一道得来的(inquiry._derived_hidden,质询、双盲复算同一个认法)。SQL 按每种读得通的读法各认一遍、合起来;都读不通 - 的照那一套的办法,写着遮罩列就整份的列都算。认不出结果里哪一列取自哪一列的,整份的列都算:语法树认不准、不点列名也 - 读得出表里内容的写法(_OPAQUE_SQL、_INDIRECT_READS),以及哪一种读法下有列名表、整行引用、不照原名展开的 *、吃进 - 遮罩列的表函数的(_renamed)——快照本身因此只给哈希,口径卡上取自它的值也一样。合并查询照那一套的办法不看 SQL。 + 转一道得来的(inquiry._derived_hidden,质询、双盲复算同一个认法)。SQL 按每种读得通的读法各认一遍、合起来。认不出 + 结果里哪一列取自哪一列的,整份的列都算:每种读法都解析不了、没有 SQL 的,语法树认不准、不点列名也读得出表里内容的 + 写法(_OPAQUE_SQL、_INDIRECT_READS),以及哪一种读法下有列名表、整行引用、不照原名展开的 *、吃进遮罩列的表函数的 + (_renamed)——快照本身因此只给哈希,口径卡上取自它的值也一样。解析不了的不退回那一套:它解析不了时只看 SQL 里写没 + 写着遮罩列,没写就只认遮罩列本身,卡上取自整行的值认不出。合并查询照那一套的办法不看 SQL(它记着自己的来历)。 出错时整份的列都算(宁可多挡)。""" columns = snapshot.get("columns") every = set(base) | ({str(c).lower() for c in columns} if isinstance(columns, list) else set()) @@ -318,16 +333,18 @@ def _derived_columns(snapshot: dict[str, Any], base: set[str]) -> set[str]: from app.data.sql_semantics import _parse, _read from app.engine.inquiry import _derived_hidden + if snapshot.get("source") == MERGE_SOURCE: + return set(base) | {c.lower() for c in _derived_hidden(dict(snapshot), set(base), None)[0]} sql = snapshot.get("sql") - own = isinstance(sql, str) and snapshot.get("source") != MERGE_SOURCE - if own and (_OPAQUE_SQL.search(sql) or _INDIRECT_READS.search(sql)): + if not isinstance(sql, str) or _OPAQUE_SQL.search(sql) or _INDIRECT_READS.search(sql): + return every + readings = [(kind, tree) for kind in _READINGS for tree in [_parse(sql, _read(kind))[0]] if tree is not None] + if not readings: return every - readings = [(kind, tree) for kind in _READINGS for tree in [_parse(sql, _read(kind))[0] if own else None] - if tree is not None] out = set(base) - for kind, tree in readings or [(None, None)]: + for kind, tree in readings: found, names = _derived_hidden(dict(snapshot), set(base), kind) - if tree is not None and _renamed(tree, {n.lower() for n in names}): + if _renamed(tree, {n.lower() for n in names}): return every out |= {c.lower() for c in found} return out @@ -467,11 +484,14 @@ def _whole_row(column: Any) -> bool: """一个名字是不是整行引用(Postgres 的 SELECT m、to_json(m),函数记法的 m.to_jsonb:一整行装进一列)。 没写表名的,Postgres 先按列认:所在 SELECT 的派生来源明写出了叫这个名字的列,就是那一列;不然对上它看得见的某个来源 - 的名字(别名、表名、WITH 里的一段)就算。写了表名的,列名是 _ROW_FUNCTIONS 里的、表名对上看得见的来源才算。 + 的名字(别名、表名、WITH 里的一段)就算。名字写在 FROM、JOIN 里的表函数的参数里时,只有排在这个表函数前面的派生 + 来源算数:它看不见后面的来源,(SELECT 1 AS m) s 排在后面,m 还是整行(_holder)。写了表名的,列名是 _ROW_FUNCTIONS + 里的、表名对上看得见的来源才算,同样先按列认:那个来源是明写出了这一列的派生来源,t.max 就是那一列,不是 max(t)。 看得见的是所在 SELECT 和外层 SELECT 的 FROM、JOIN 里的来源。FROM、JOIN 里的表函数(UNNEST、ROWS FROM 这类)和 LATERAL 看得见同一层的来源:Postgres 的函数不写 LATERAL 也看得见(逗号、CROSS JOIN、JOIN … ON TRUE 都一样),整行 交给 jsonb_each_text 这类就一列列竖着摊开了;括号括起来的一组连接按同一层算(_sources 摊开)。FROM、JOIN 里的子查询 - 不写 LATERAL 看不见,WITH 里的一段也看不见。数据源里的表有哪些列这里不知道:碰巧和表、别名同名的列也算(宁可多挡)。""" + 不写 LATERAL 看不见,WITH 里的一段也看不见。数据源里的表有哪些列这里不知道:碰巧和表、别名同名的列,碰巧叫 max、min + 这类名字的列也算(宁可多挡)。""" from sqlglot import exp select = column.find_ancestor(exp.Select) @@ -480,27 +500,48 @@ def _whole_row(column: Any) -> bool: if column.table: if column.name.lower() not in _ROW_FUNCTIONS: return False - name = column.table.lower() + name, field = column.table.lower(), column.name.lower() else: - name = column.name.lower() - for body in filter(None, map(_derived, _sources(select))): - first = _leaves(body)[0] - if isinstance(first, exp.Select) and any( - isinstance(item, (exp.Alias, exp.Column)) and not isinstance(item.this, exp.Star) - and item.alias_or_name.lower() == name for item in first.expressions): - return False + name, field = column.name.lower(), None + sources = _sources(select) + if any(_names_column(_derived(source), name) for source in sources[:_holder(column, select, sources)]): + return False child, node = column, column.parent while node is not None: # FROM、JOIN 里真的子查询(括号里是一条查询,不是一组连接)看不见同一层,WITH 里的一段也看不见 subquery = isinstance(child.this, exp.Subquery) and _derived(child.this) is not None sees = child.arg_key != "with_" and not (child.arg_key in ("from_", "joins") and subquery) - if isinstance(node, exp.Select) and sees \ - and any((source.alias_or_name or "").lower() == name for source in _sources(node)): - return True + found = [source for source in _sources(node) if (source.alias_or_name or "").lower() == name] \ + if isinstance(node, exp.Select) and sees else [] + if found: + return field is None or not any(_names_column(_derived(source), field) for source in found) child, node = node, node.parent return False +def _holder(column: Any, select: Any, sources: list[Any]) -> int | None: + """名字写在所在 SELECT 的哪一个表函数(_table_function)的参数里:交回它在 sources 里的位置;写在选出项、条件、 + ON 里的是 None(同一层的来源都看得见)。""" + places = {id(source): place for place, source in enumerate(sources) if _table_function(source)} + node = column.parent + while node is not None and node is not select: + if id(node) in places: + return places[id(node)] + node = node.parent + return None + + +def _names_column(block: Any, name: str) -> bool: + """一个派生来源(_derived 交回的那块查询,None 是数据源里的表这类)明写出了叫这个名字的列:第一支的选出项里起了 + 这个别名、或者直接写着这一列(* 不算,展开出哪些列这里看不出)。""" + from sqlglot import exp + + first = _leaves(block)[0] if block is not None else None + return isinstance(first, exp.Select) and any( + isinstance(item, (exp.Alias, exp.Column)) and not isinstance(item.this, exp.Star) + and item.alias_or_name.lower() == name for item in first.expressions) + + def _stars(tree: Any) -> list[Any]: """语法树里的 *(t.* 取包着它的 Column),COUNT(*)、COUNT(t.*) 不算:只数个数。""" from sqlglot import exp diff --git a/backend/tests/test_bundle_masked_cards.py b/backend/tests/test_bundle_masked_cards.py index 03775566..78d104dd 100644 --- a/backend/tests/test_bundle_masked_cards.py +++ b/backend/tests/test_bundle_masked_cards.py @@ -120,6 +120,9 @@ def test_a_card_reading_a_renamed_masked_column_ships_only_its_id(sql): # 不点列名也读得出表里内容;语法树认不准数据库实际执行什么 "SELECT name, table_to_xml('members', true, false, '')::text AS tel, spend FROM members", "SELECT name, /*!50000 phone AS */ tel, spend FROM members", + # 函数记法交出整行的文字写法;表函数看不见排在它后面的派生来源里叫 m 的列,m 还是整行 + "SELECT name, m.record_out AS tel, spend FROM members m", + "SELECT name, value AS tel, spend FROM members m, jsonb_each_text(to_jsonb(m)), (SELECT 1 AS m) s", ]) def test_a_card_reading_a_column_whose_origin_the_sql_hides_ships_only_its_id(sql): """快照的 SQL 认不出结果里哪一列取自哪一列(快照本身因此只给哈希):卡上取的 tel 可能就是号码,整份快照的列都按 @@ -133,6 +136,32 @@ def test_a_card_reading_a_column_whose_origin_the_sql_hides_ships_only_its_id(sq assert PHONE not in render_html(bundle) +@pytest.mark.parametrize("sql", [ + # 每种读法都解析不了:PG17 起的 JSON_QUERY … WITH WRAPPER(这里把整行装进了 tel)、ORDER BY … USING + "SELECT name, JSON_QUERY(to_jsonb(m), '$.*' WITH WRAPPER) AS tel, spend FROM members m", + "SELECT name, value AS tel, spend FROM members m, jsonb_each_text(to_jsonb(m)) ORDER BY value USING <", + "SELECT name, tel, spend FROM (SELECT * FROM members) AS x(name, tel, spend) ORDER BY spend USING <", + # 没有 SQL(数据源查询工具落的快照都带着)、SQL 是空的 + None, + "", +]) +def test_a_card_reading_a_snapshot_whose_sql_does_not_parse_ships_only_its_id(sql): + """快照的 SQL 每种读法都解析不了(或者根本没有):快照本身只给哈希(_untouched),认不出结果里哪一列取自哪一列, + 卡上取自这份快照任一列的输入都按遮罩列算,整张卡只给 id——和快照同一个结论。以前这条路退回按 SQL 认遮罩列的那一套, + 解析不了时它只看 SQL 里写没写着 phone:没写就只认 phone 本身,卡上取的 tel 是整行、号码原样进了文件。""" + snap = {**SNAP_MASKED, "columns": ["name", "tel", "spend"], "sql": sql} + if sql is None: + del snap["sql"] + snap_id, raw = stored(snap) + for column, value in (("tel", PHONE), ("spend", 320.0)): + item = {**cell_input(column, value), "artifact": snap_id, "path": f"cell(nodes.members, 0, '{column}')"} + bundle, entry = bundle_of(card_with([item], substituted=repr(value), + expression=f"len(cell(nodes.members, 0, '{column}'))"), + files_changed={snap_id: raw}) + assert entry.get("masked") is True and "b64" not in entry, (sql, column) + assert PHONE not in render_html(bundle) + + @pytest.mark.parametrize("sql, columns", [ ("SELECT name, phone AS tel, spend FROM members", ["name", "tel", "spend"]), # 最外层的 * 照原来的名字展开:号码还在 phone 那一列 diff --git a/backend/tests/test_bundle_masked_queries.py b/backend/tests/test_bundle_masked_queries.py index 0d631b53..e5e200dc 100644 --- a/backend/tests/test_bundle_masked_queries.py +++ b/backend/tests/test_bundle_masked_queries.py @@ -150,6 +150,16 @@ def test_an_aggregate_by_region_on_a_masked_source_is_embedded(masks, recorded): # 括号括起来的一组连接,按名字取列 ("SELECT m.region, SUM(o.amount) AS spend FROM (members m JOIN orders o ON o.member_id = m.id) GROUP BY m.region", ["region", "spend"], [["华东", 508.5]]), + # 派生来源明写出了叫 max 的列:t.max 是那一列,不是函数记法的 max(t) + ("SELECT t.region, t.max AS spend FROM (SELECT region, MAX(spend) AS max FROM members GROUP BY region) t", + ["region", "spend"], [["华东", 508.5]]), + # 表函数的参数没写表名、和它自己的别名同名:排在它前面的派生来源明写出了这一列,是列不是整行引用 + ("SELECT p.region, SUM(p.spend) AS spend FROM (SELECT region, spend, attrs FROM members) p, " + "jsonb_each_text(attrs) AS attrs WHERE attrs.key = 'vip' GROUP BY p.region", ["region", "spend"], + [["华东", 508.5]]), + # Oracle 的 sys.dual 不是 MySQL sys 库里记着别的会话 SQL 的视图 + ("SELECT region, SUM(spend) AS spend FROM members CROSS JOIN sys.dual GROUP BY region", ["region", "spend"], + [["华东", 508.5]]), ]) def test_queries_that_only_read_unmasked_columns_are_embedded(sql, columns, rows): bundle, entry = bundle_of(snapshot(sql, columns, rows)) @@ -233,9 +243,22 @@ def test_queries_that_use_a_masked_column_ship_only_their_id(sql, columns): # WITH 里的 * 只按名字往外传,可外层把那一段整行交给了表函数 ("WITH c AS (SELECT * FROM members) SELECT region, value AS spend FROM c, json_each_text(row_to_json(c))", ["region", "spend"]), - # 函数记法:m.to_jsonb 就是 to_jsonb(m) + # 函数记法:m.to_jsonb 就是 to_jsonb(m)。整行的文字、二进制写法(record_out、record_send),汇总函数也能这么写 + # (any_value、*_agg_strict,max、min 交回最大、最小的那一整行);WITH 里的 * 只按名字往外传,可外层整行交了出去 ("SELECT m.to_jsonb AS spend FROM members m", ["spend"]), ("SELECT m.row_to_json AS spend FROM members m", ["spend"]), + ("SELECT m.record_out AS spend FROM members m", ["spend"]), + ("SELECT m.record_send AS spend FROM members m", ["spend"]), + ("SELECT m.any_value AS spend FROM members m", ["spend"]), + ("SELECT m.json_agg_strict AS spend FROM members m", ["spend"]), + ("SELECT m.jsonb_agg_strict AS spend FROM members m", ["spend"]), + ("SELECT m.max AS spend FROM members m", ["spend"]), + ("SELECT region, m.min AS spend FROM members m GROUP BY region", ["region", "spend"]), + ("WITH c AS (SELECT * FROM members) SELECT region, c.record_out AS spend FROM c", ["region", "spend"]), + # 表函数只看得见排在它前面的来源:后面那个派生来源里叫 m 的列它看不见,m 还是整行 + ("SELECT value AS spend FROM members m, jsonb_each_text(to_jsonb(m)), (SELECT 1 AS m) s", ["spend"]), + ("SELECT value AS spend FROM (members m CROSS JOIN jsonb_each_text(to_jsonb(m))) AS x, (SELECT 1 AS m) s", + ["spend"]), # 列名表按位置给列改名(Postgres 的表别名、子查询、WITH 都能写) ("SELECT c AS spend FROM members AS m(a, b, c)", ["spend"]), ("WITH t(a, b) AS (SELECT * FROM members2) SELECT b AS spend FROM t", ["spend"]), @@ -285,6 +308,31 @@ def test_profiling_every_column_of_a_masked_table_ships_only_its_id(): ("SELECT LOAD_FILE('/var/lib/mysql/crm/members.ibd') AS spend", ["spend"]), ("SELECT data AS spend FROM pg_logical_slot_peek_changes('cdc', NULL, NULL)", ["spend"]), ("SELECT argument AS spend FROM mysql.general_log", ["spend"]), + # 别的会话执行过的 SQL 原文的另几种写法:Postgres 里 pg_stat_activity 背后的函数和按会话取的那一个、 + # pg_stat_statements、本连接预备过的语句(连接池里的连接留着前几次查询的原文);MySQL sys 库建在 + # performance_schema 上的视图(带 x$ 的是不排版的那一份)、企业版的审计日志;Oracle 打开着的游标和审计记录 + ("SELECT query AS spend FROM pg_stat_get_activity(NULL)", ["spend"]), + ("SELECT pg_stat_get_backend_activity(1) AS spend", ["spend"]), + ("SELECT query AS spend FROM pg_stat_statements", ["spend"]), + ("SELECT statement AS spend FROM pg_prepared_statements", ["spend"]), + ("SELECT last_statement AS spend FROM sys.session", ["spend"]), + ("SELECT current_statement AS spend FROM sys.x$processlist", ["spend"]), + ("SELECT `last_statement` AS spend FROM `sys`.`x$session`", ["spend"]), + ("SELECT waiting_query AS spend FROM sys.innodb_lock_waits", ["spend"]), + ("SELECT waiting_query AS spend FROM sys.x$schema_table_lock_waits", ["spend"]), + ("SELECT audit_log_read() AS spend", ["spend"]), + ("SELECT sql_text AS spend FROM v$open_cursor", ["spend"]), + ("SELECT sql_text AS spend FROM dba_audit_trail", ["spend"]), + ("SELECT sql_text AS spend FROM unified_audit_trail", ["spend"]), + ("SELECT lsqltext AS spend FROM sys.fga_log$", ["spend"]), + # 按页、按日志、按文件读的另几种写法:pageinspect 按索引名读 B 树页,pg_walinspect 读 WAL 里的块,SQLite 的 + # readfile;统计信息的另几处:Oracle 直方图和列统计的底表,MariaDB 的 column_stats + ("SELECT data AS spend FROM bt_page_items('members_phone_idx', 1)", ["spend"]), + ("SELECT block_data AS spend FROM pg_get_wal_block_info('0/0', '0/FFFFFFFF')", ["spend"]), + ("SELECT readfile('crm.db') AS spend", ["spend"]), + ("SELECT epvalue AS spend FROM sys.histgrm$", ["spend"]), + ("SELECT lowval AS spend FROM sys.hist_head$", ["spend"]), + ("SELECT min_value AS spend FROM mysql.column_stats WHERE table_name = 'members'", ["spend"]), # 系统 schema ("SELECT relname AS region, reltuples AS spend FROM pg_catalog.pg_class", ["region", "spend"]), ])