K3 复核修正(第 3 轮):写了库名的表按「库名.表名」查表结构;UNION 第一支按表结构数出的列名要和后面每一支一样宽才用;整行引用、函数参数里的 *、表的列名表、TABLE x、PIVOT 这类认得出

- 写了库名的表(复核 major 1):表结构以前按表名查,查询写的是另一个库里的同名表(archive.sales,列序不同)时照探查
  的那张数位置,号码原样交给模型、落库。现在表结构另按「库名.表名」记一份(库名取探查时记下的 schema、qualified 的
  前缀,masking._schema_keys),写了库名的引用按它查(_table_key),查不到的按数不清整段遮;探查时没设 schema、查询又
  写了库名的也查不到(多遮)。数位置(_star_columns、_branch_names)、认标签列(_table_lacks)都照这个查法
- 过期的表结构(复核 major 2,对 099473c 的回退):UNION 第一支是 * 时,按表结构数出的列名以前不核对列数就拿来对后一支
  的位置,表结构过期、列数已经变了也照对,遮罩值记到错的列名上。现在后面每一支数得出的列数都和它一样宽才用
  (_source_names、_width),对不上、数不清的退回整段遮
- 整行和别的 * 写法(复核 major 3):写在函数参数、表达式里的 *、s.*(to_json(s.*)、concat_ws(',', s.*)、ROW(s.*)、
  Oracle 的 JSON_OBJECT(*))按它展开的来源认(_value_columns、_scoped_star_inputs,相关子查询、FROM 里的子查询照样
  找得到外层的来源)。几种写法先改写语法树再认(_normalized,原文不动):Postgres 的整行引用(row_to_json(s)、
  json_agg(t)、SELECT s、s::text)按 Postgres 先认列的规矩,不是列的改成 ROW(s.*);带列名表的表(FROM sales AS
  s(i, r, p),引用 CTE 时写的也算)改成派生来源;解析器读错的 TABLE x 改成 SELECT * FROM x;选出项套了括号的 *
  ((s.*)、(s).*)按 * 数位置。PIVOT、UNPIVOT、MATCH_RECOGNIZE 读到遮罩列的整份遮。SQL 解析不了、里面有 *(COUNT(*)
  除外)或 TABLE x 的(UNION ALL TABLE x、带库名的 TABLE),同写着遮罩列一样整份遮(_implicit)
- Postgres 的列名表只写前几列时,不按表结构补全:别的方言里列名表比列少只能是表结构过期了,补全就丢了列数这道核对;
  这时改了名的几列整段遮(多遮),照原名的按名字认
- inquiry 模块说明的已知局限补上:写了库名的表、过期表结构的列数核对、Postgres 列名表只写前几列时整段遮、探查的不是
  默认库时没写库名的同名表可能数错、整行引用按表结构先认列(查不到时按整行多遮)、解析不了的 SQL 里的整行引用认不出

测试:新增 test_masking_whole_rows(整行、函数参数里的 *、列名表、TABLE x、Oracle 的 JSON_OBJECT(*) / UNPIVOT / PIVOT /
MATCH_RECOGNIZE、解析不了的 SQL、不碰遮罩列的照常);test_masking_star_renames 补写了库名的表(SQLite ATTACH 真跑)、
过期的表结构;test_verify_agent_masking 补核查 agent 的真实查询工具:过期的表结构,Postgres / Oracle 形状的结果(换掉
驱动)交给模型、落库、通用工件接口都遮好。每处实现都做过去掉之后测试变红的验证;复核给的 Postgres 18 探针、端到端
在临时库上重跑,都不再漏。
This commit is contained in:
ThearchyHelios
2026-10-06 21:50:48 +08:00
parent 6a7e51b81b
commit b0fdf56eed
5 changed files with 776 additions and 32 deletions
+127
View File
@@ -249,3 +249,130 @@ async def test_a_snapshot_is_read_with_its_frozen_schema(months):
assert masking._derived_hidden(snapshot, {"buyer_phone"}, "sqlite")[0] == {"buyer_phone"}
assert masking.derived_columns({**snapshot, "schema_artifact": "missing"}, {"buyer_phone"}, "sqlite") == {
"region", "合计"}
# --------------------------------------------------------------------------
# 写了库名的表、过期的表结构(复核第 3 轮)
# --------------------------------------------------------------------------
#: 同名的两张表:主库的 sales 和另一个库(archive)里的 sales,号码、区域换了次序。表结构探查的是主库
TWO_SALES = ("CREATE TABLE sales (id INTEGER, region TEXT, buyer_phone TEXT, buyer_name TEXT, amount INTEGER, "
"note TEXT);"
"INSERT INTO sales VALUES (1, '华东', '13900000001', '张伟', 1200, '改址');"
"CREATE TABLE archive.sales (id INTEGER, buyer_phone TEXT, region TEXT, buyer_name TEXT, amount INTEGER, "
"note TEXT);"
"INSERT INTO archive.sales VALUES (7, '13900000007', '华北', '王芳', 500, '回访');")
SALES_LAYOUT = ["id", "region", "buyer_phone", "buyer_name", "amount", "note"]
@pytest.fixture
def two_sales():
conn = sqlite3.connect(":memory:")
conn.execute("ATTACH DATABASE ':memory:' AS archive")
conn.executescript(TWO_SALES)
yield conn
conn.close()
def probed(schema: str | None = None) -> dict:
"""探查主库得到的表结构(schema_cache 的样子):没设 schema 的探查当前库,qualified 就是表名。"""
meta = {"qualified": f"{schema}.sales" if schema else "sales", "schema": schema,
"columns": [{"name": c} for c in SALES_LAYOUT]}
return {"tables": {"sales": meta}, "schema": schema}
def run_two(conn, sql: str, schema: dict | None) -> masking.QueryMask:
cursor = conn.execute(sql)
payload = {"columns": [d[0] for d in cursor.description], "rows": [list(r) for r in cursor.fetchall()], "sql": sql}
return masking.mask_query_result(payload, MASKED, "sqlite", schema=schema)
@pytest.mark.parametrize("sql", [
pytest.param("WITH t(a, b, c, d, e, f) AS (SELECT * FROM archive.sales) SELECT b AS j, a AS n FROM t",
id="列名表套 * 展开另一个库里的同名表"),
pytest.param("SELECT 0 AS a, '' AS b, '' AS c, '' AS d, 0 AS e, '' AS f WHERE 0 UNION ALL "
"SELECT * FROM archive.sales", id="UNION 后一支的 * 展开另一个库里的同名表"),
pytest.param("WITH t AS (SELECT * FROM sales UNION ALL SELECT * FROM archive.sales) SELECT region AS 区域 FROM t",
id="派生来源里 UNION 后一支展开另一个库里的同名表"),
])
def test_a_table_in_another_schema_is_not_counted_by_the_probed_one(two_sales, sql):
"""表结构探查的是主库:查询写了别的库名的同名表(archive.sales,号码排在第二列),不能按主库那张表的列序数位置,
按数不清处理,* 展开落到的那一段整段遮。以前按表名查表结构,号码落进的那一列被当成区域,原样交给模型、落库。"""
done = run_two(two_sales, sql, probed())
assert not _leaks(done), done.payload["rows"]
def test_a_table_named_with_its_probed_schema_is_still_counted(two_sales):
"""库名和表结构记下的对得上(探查时设了 schema、qualified 带着它):照样按表结构数位置,只遮号码那一列。没写库名的
照旧按表名查。库名对不上表结构记下的(这里探查时没设 schema,查询写了 main.)按数不清处理,整段遮(多遮)。"""
precise = "WITH t(a, b, c, d, e, f) AS (SELECT * FROM {}) SELECT c AS j, a AS n FROM t"
assert run_two(two_sales, precise.format("main.sales"), probed("main")).columns == ["j"]
assert run_two(two_sales, precise.format("sales"), probed()).columns == ["j"]
assert set(run_two(two_sales, precise.format("main.sales"), probed()).columns) == {"j", "n"}
def test_schema_columns_are_also_keyed_by_the_probed_schema():
"""表结构按「库名.表名」另记一份(库名取 schema、qualified 的前缀);写了别的库名的查不到。"""
tables = masking._schema_columns(probed("public"))
assert tables["sales"] == tables["public.sales"] == SALES_LAYOUT and "archive.sales" not in tables
assert "public.sales" not in masking._schema_columns(probed())
assert "archive.sales" not in (masking._schema_tables(probed("public")) or {})
#: 一月的表结构是早先探查的(那时还没有 extra 这一列),表里现在多了一列,排在第二;二月那张号码排在第二列
STALE = ("CREATE TABLE jan (id INTEGER, extra TEXT, region TEXT, buyer_phone TEXT, amount INTEGER);"
"CREATE TABLE feb (id INTEGER, buyer_phone TEXT, region TEXT, amount INTEGER, extra TEXT);"
"INSERT INTO jan VALUES (1, '加急', '华东', '13900000001', 1200);"
"INSERT INTO feb VALUES (2, '13900000002', '华南', 300, '普通');")
@pytest.fixture
def stale():
conn = sqlite3.connect(":memory:")
conn.executescript(STALE)
yield conn
conn.close()
def layout(**tables: list[str]) -> dict:
return {"tables": {name: {"qualified": name, "columns": [{"name": c} for c in columns]}
for name, columns in tables.items()}}
#: 过期的表结构:一月那张少了 extra(列数和表里实际的不一样)
OLD_JAN = layout(jan=["id", "region", "buyer_phone", "amount"], feb=["id", "buyer_phone", "region", "amount", "extra"])
#: 准的表结构
NOW = layout(jan=["id", "extra", "region", "buyer_phone", "amount"],
feb=["id", "buyer_phone", "region", "amount", "extra"])
@pytest.mark.parametrize("sql", [
pytest.param("WITH t AS (SELECT * FROM jan UNION ALL SELECT id, buyer_phone, region, amount, extra FROM feb) "
"SELECT extra AS 备注 FROM t", id="后一支明写各列"),
pytest.param("WITH t AS (SELECT * FROM jan UNION ALL SELECT * FROM feb) SELECT extra AS 备注 FROM t",
id="后一支也是 *"),
])
def test_a_first_branch_counted_by_a_stale_schema_is_not_trusted(stale, sql):
"""UNION 第一支是 *、按表结构数出的列名(表结构过期了,少一列)和后一支的列数对不上:不按它对列名,* 所在的位置按认不出
算,整份遮。以前照过期的列名表对,二月的号码落进的「extra」没认出来,原样交回。"""
cursor = stale.execute(sql)
payload = {"columns": [d[0] for d in cursor.description], "rows": [list(r) for r in cursor.fetchall()], "sql": sql}
for schema in (OLD_JAN, None, NOW):
done = masking.mask_query_result(payload, ["buyer_phone"], "sqlite", schema=schema)
assert done.columns == ["备注"] and not _leaks(done), (schema, done.payload["rows"])
def test_a_first_branch_counted_by_an_accurate_schema_still_places_the_values(stale):
"""表结构准、每一支数得出的列数都对得上:照样按第一支的列名对上位置,只遮号码落进的那一列,金额照常。"""
sql = ("WITH t AS (SELECT * FROM jan UNION ALL SELECT id, buyer_phone, region, buyer_phone, amount FROM feb) "
"SELECT region AS 区域, amount AS 金额 FROM t")
cursor = stale.execute(sql)
payload = {"columns": [d[0] for d in cursor.description], "rows": [list(r) for r in cursor.fetchall()], "sql": sql}
assert masking.mask_query_result(payload, ["buyer_phone"], "sqlite", schema=NOW).columns == []
moved = sql.replace("SELECT id, buyer_phone, region,", "SELECT id, region, buyer_phone,")
cursor = stale.execute(moved)
payload = {"columns": [d[0] for d in cursor.description], "rows": [list(r) for r in cursor.fetchall()],
"sql": moved}
done = masking.mask_query_result(payload, ["buyer_phone"], "sqlite", schema=NOW)
assert done.columns == ["区域"] and [row[1] for row in done.payload["rows"]] == [1200, 300]
+244
View File
@@ -0,0 +1,244 @@
"""按 SQL 认取自遮罩列的结果列(engine/masking.py 的 _derived_hidden):整行、函数参数里的 *、列名表把遮罩列带进别的列名
(复核第 3 轮)。
Postgres 里不带表名、名字是 FROM 里一个来源的列引用指它的整行(row_to_json(s)、json_agg(t)、SELECT s、s::text),函数
参数里的 s.*(to_json(s.*)、concat_ws(',', s.*)、ROW(s.*))也把整行交给函数;数据源里的表可以带列名表(FROM sales AS
s(i, r, p),只改前几列的名字),TABLE sales 等于 SELECT * FROM sales;Oracle 的 JSON_OBJECT(*) 把整行拼成 JSON,UNPIVOT、
PIVOT、MATCH_RECOGNIZE 把遮罩列的值转进它们自己起名的列。这些写法以前都认不出:遮罩列的值落进了别的列名,原样交给核查
agent、落进快照。现在整行、函数参数里的 * 按 * 展开的来源认(数据源里的表一律算带出了遮罩列),列名表按位置对上(表结构
查得到这张表的只遮号码改了名字的那一列),TABLE x 当 SELECT * FROM x,PIVOT 这类读到遮罩列的整份遮。
这里没有真的 Postgres、Oracle:结果按它们交回的样子造(行转成 JSON、记录转成文字),只看遮完的结果里还有没有原值。数据
都是虚构的:区域、手机号、姓名、金额。
"""
from __future__ import annotations
import json
import sqlite3
import pytest
from app.engine import masking
MASKED_WORD = "已遮罩"
MASKED = ["buyer_phone", "buyer_name"]
SECRETS = ("1390000000", "张伟", "李娜")
LAYOUT = ["id", "region", "buyer_phone", "buyer_name", "amount", "note"]
#: 探查得到的表结构(没设 schema,探查的是当前库)
SCHEMA = {"tables": {"sales": {"qualified": "sales", "schema": None, "columns": [{"name": c} for c in LAYOUT]}}}
ROWS = [[1, "华东", "13900000001", "张伟", 1200, "改址"], [2, "华南", "13900000002", "李娜", 300, "回访"]]
def as_json(row: list) -> str:
return json.dumps(dict(zip(LAYOUT, row)), ensure_ascii=False)
def as_record(row: list) -> str:
return "(" + ",".join(str(v) for v in row) + ")"
def hidden_of(sql: str, columns: list[str], kind: str = "postgres", schema: dict | None = None) -> set[str]:
snapshot = {"source": "shop", "sql": sql, "columns": columns, "rows": []}
return masking._derived_hidden(snapshot, set(MASKED), kind, masking._schema_columns(schema))[0]
def masked(sql: str, columns: list[str], rows: list[list], kind: str = "postgres",
schema: dict | None = SCHEMA) -> masking.QueryMask:
payload = {"columns": columns, "rows": rows, "sql": sql}
return masking.mask_query_result(payload, MASKED, kind, schema=schema)
def leaks(done: masking.QueryMask) -> bool:
text = json.dumps(done.payload["rows"], ensure_ascii=False)
return any(s in text for s in SECRETS)
# --------------------------------------------------------------------------
# 整行引用、函数参数里的 *
# --------------------------------------------------------------------------
@pytest.mark.parametrize("sql, cells", [
pytest.param("SELECT row_to_json(s) AS j, s.id AS n FROM sales s", [as_json(r) for r in ROWS], id="row_to_json(s)"),
pytest.param("SELECT to_jsonb(s) AS j, s.id AS n FROM sales s", [as_json(r) for r in ROWS], id="to_jsonb(s)"),
pytest.param("SELECT s AS j, s.id AS n FROM sales s", [as_record(r) for r in ROWS], id="SELECT s"),
pytest.param("SELECT s::text AS j, s.id AS n FROM sales s", [as_record(r) for r in ROWS], id="s::text"),
pytest.param("SELECT to_json(s.*) AS j, s.id AS n FROM sales s", [as_json(r) for r in ROWS], id="to_json(s.*)"),
pytest.param("SELECT concat_ws(',', s.*) AS j, s.id AS n FROM sales s",
[",".join(str(v) for v in r) for r in ROWS], id="concat_ws(',', s.*)"),
pytest.param("SELECT ROW(s.*) AS j, s.id AS n FROM sales s", [as_record(r) for r in ROWS], id="ROW(s.*)"),
pytest.param("SELECT (s).buyer_phone AS j, s.id AS n FROM sales s", [r[2] for r in ROWS], id="(s).buyer_phone"),
pytest.param("SELECT (s.*).buyer_name AS j, s.id AS n FROM sales s", [r[3] for r in ROWS], id="(s.*).buyer_name"),
pytest.param("SELECT (SELECT row_to_json(s)) AS j, s.id AS n FROM sales s", [as_json(r) for r in ROWS],
id="相关子查询里的整行"),
pytest.param("SELECT kv.value AS j, s.id AS n FROM sales s, LATERAL jsonb_each_text(to_jsonb(s)) AS kv "
"WHERE kv.key = 'buyer_phone'", [r[2] for r in ROWS], id="LATERAL 拆开整行"),
])
def test_a_whole_row_carrying_masked_columns_is_masked(sql, cells):
"""整行、函数参数里的 s.* 把遮罩列的值交给了别的列:这一列遮,编号照常。"""
done = masked(sql, ["j", "n"], [[cell, row[0]] for cell, row in zip(cells, ROWS)])
assert done.columns == ["j"] and not leaks(done), done.payload["rows"]
assert [row[1] for row in done.payload["rows"]] == [1, 2]
@pytest.mark.parametrize("sql", [
"SELECT json_agg(t) AS j FROM (SELECT id, buyer_phone FROM sales) t",
"SELECT row_to_json(t) AS j FROM (SELECT buyer_name AS 名 FROM sales) t",
"WITH t AS (SELECT id, buyer_phone AS p FROM sales) SELECT json_agg(t) AS j FROM t",
])
def test_a_whole_row_of_a_derived_source_is_read_through_its_columns(sql):
"""整行取自子查询、CTE:按它选出的列认,里面有遮罩列(改了名字的也算)就算。"""
assert "j" in hidden_of(sql, ["j"])
@pytest.mark.parametrize("sql", [
"SELECT (SELECT j FROM (SELECT row_to_json(s) AS j) z) AS j FROM sales s",
"SELECT (SELECT j FROM (SELECT json_agg(t) AS j) z) AS j FROM (SELECT id, buyer_phone FROM sales) t",
"SELECT (WITH w AS (SELECT row_to_json(s) AS j) SELECT j FROM w) AS j FROM sales s",
"SELECT (SELECT j FROM (SELECT to_json(s.*) AS j) z) AS j FROM sales s",
])
def test_a_whole_row_of_an_outer_query_is_found_from_a_nested_derived_table(sql):
"""FROM 里的子查询、CTE 看不见同一层 FROM 里的别的来源,却看得见再外层的查询:写在里面的整行引用、s.* 指外层的来源,
照样认得出。"""
assert "j" in hidden_of(sql, ["j"])
@pytest.mark.parametrize("sql", [
"SELECT json_agg(t) AS j FROM (SELECT id, region, amount FROM sales) t",
"SELECT count(s.*) AS j FROM sales s",
"SELECT COUNT(*) AS j FROM sales s",
"SELECT region AS j, SUM(amount) AS total FROM sales s GROUP BY region",
"SELECT EXISTS(SELECT * FROM sales s2 WHERE s2.id = s.id) AS j FROM sales s",
])
def test_rows_and_stars_that_bring_no_masked_values_stay_visible(sql):
"""整行取自没有遮罩列的子查询、count(s.*) 只数个数:不算。"""
assert hidden_of(sql, ["j", "total"]) == set(MASKED)
def test_a_column_named_like_its_table_is_a_column_where_the_schema_says_so():
"""Postgres 先按列认:表里有一列和表同名(visits 表的 visits 列),SELECT visits 取的是这一列,不是整行。表结构里查得到
这一列就不按整行算;SQLite、MySQL 没有整行引用,不看表结构也不按整行算。"""
schema = {"tables": {"visits": {"columns": [{"name": c} for c in ("id", "visits", "buyer_phone")]}}}
sql = "SELECT visits AS j FROM visits"
assert hidden_of(sql, ["j"], schema=schema) == set(MASKED)
assert hidden_of(sql, ["j"], kind="sqlite") == set(MASKED)
assert "j" in hidden_of(sql, ["j"]), "表结构里查不到这一列:可能是整行,按整行算(多遮)"
def test_a_star_inside_a_function_reads_every_source_it_expands():
"""Oracle 的 JSON_OBJECT(*) 把整行拼成 JSON:同 to_json(s.*)。"""
done = masked("SELECT JSON_OBJECT(*) AS j FROM sales", ["j"], [[as_json(r)] for r in ROWS], kind="oracle")
assert done.columns == ["j"] and not leaks(done)
# --------------------------------------------------------------------------
# 列名表、TABLE x
# --------------------------------------------------------------------------
def test_a_table_with_a_column_list_masks_the_renamed_masked_column():
"""数据源里的表带列名表:号码改叫 p。列名表写全了的,按表结构对上位置,只遮 p 和姓名改叫的 q,编号照常;Postgres 的
列名表只写前几列(后面的照原名)时,列名表和表的列数对不上,改了名的几列整段遮(多遮),照原名的姓名按名字遮。"""
full = "SELECT p AS j, i AS n FROM sales AS s(i, r, p, q, a, o)"
done = masked(full, ["j", "n"], [[r[2], r[0]] for r in ROWS])
assert done.columns == ["j"] and not leaks(done)
assert set(masked(full, ["j", "n"], [[r[2], r[0]] for r in ROWS], schema=None).columns) == {"j", "n"}, \
"表结构里查不到这张表:说不清哪几列改了名,都遮"
short = masked("SELECT p AS j, i AS n FROM sales AS s(i, r, p)", ["j", "n"], [[r[2], r[0]] for r in ROWS])
assert set(short.columns) == {"j", "n"} and not leaks(short)
star = masked("SELECT * FROM sales AS s(i, r, p)", ["i", "r", "p", *LAYOUT[3:]], [list(r) for r in ROWS])
assert star.columns == ["i", "r", "p", "buyer_name"] and not leaks(star)
def test_a_column_list_on_a_cte_reference_renames_it_too():
"""引用 CTE 时再写列名表(FROM t AS x(p, a)):同样按位置对上。"""
sql = "WITH t AS (SELECT buyer_phone, amount FROM sales) SELECT p AS j, a AS total FROM t AS x(p, a)"
assert hidden_of(sql, ["j", "total"]) >= {"j"}
assert "total" not in hidden_of(sql, ["j", "total"])
@pytest.mark.parametrize("sql, columns, shown", [
pytest.param("WITH t(a, b, c, d, e, f) AS (TABLE sales) SELECT c AS j, a AS n FROM t", ["j", "n"], {"n"},
id="CTE 里的 TABLE"),
pytest.param("SELECT * FROM (TABLE sales) AS t(a, b, c, d, e, f)", ["a", "b", "c", "d", "e", "f"], {"a", "b", "e"},
id="子查询里的 TABLE"),
pytest.param("WITH t(a, b, c) AS (TABLE sales) SELECT c AS j FROM t", ["j"], set(), id="列名表只写前几列"),
])
def test_a_table_statement_is_read_as_select_star(sql, columns, shown):
"""TABLE sales(Postgres、MySQL)就是 SELECT * FROM sales:列名表照样按位置对上,号码改叫 c 的那一列遮。"""
hidden = hidden_of(sql, columns, schema=SCHEMA)
assert {"j", "c"} & hidden and not shown & hidden, hidden
@pytest.mark.parametrize("sql", [
"SELECT 0 AS a, '' AS b, '' AS c, '' AS d, 0 AS e, '' AS f WHERE false UNION ALL TABLE sales",
"WITH t(a, b, c) AS (TABLE archive.sales) SELECT c AS j FROM t",
"WITH t(a, b, c) AS (TABLE sales ORDER BY id LIMIT 1) SELECT c AS j FROM t",
"WITH t(a, b, c, d, e, f) AS (SELECT * FROM sales) SELECT c AS j FROM t WHERE (",
])
def test_sql_that_cannot_be_read_but_has_a_star_or_a_table_statement_hides_everything(sql):
"""解析器认不出的写法(UNION 后一支的 TABLE、带库名或 ORDER BY 的 TABLE):SQL 里没写遮罩列,可 TABLE 语句、* 会把它
带进别的列名,整份快照的列都算,同解析不了又写着遮罩列的。"""
columns = ["a", "b", "c", "j"]
assert hidden_of(sql, columns) >= set(columns)
@pytest.mark.parametrize("sql", [
"SELECT COUNT(*) AS j FROM sales WHERE (",
"SELECT amount * 2 AS j FROM sales WHERE (",
])
def test_sql_that_cannot_be_read_with_only_a_count_or_a_product_shows_the_rest(sql):
"""只有 COUNT(*)、乘号:不会把遮罩列带出来,照旧只遮遮罩列本身。"""
assert hidden_of(sql, ["j"]) == set(MASKED)
# --------------------------------------------------------------------------
# UNPIVOT、PIVOT、MATCH_RECOGNIZE
# --------------------------------------------------------------------------
@pytest.mark.parametrize("sql, columns", [
pytest.param("SELECT id, val FROM sales UNPIVOT (val FOR col IN (buyer_phone, region))", ["id", "val"],
id="UNPIVOT"),
pytest.param("SELECT * FROM (SELECT region, buyer_phone FROM sales) PIVOT (MAX(buyer_phone) FOR region IN "
"('华东' AS e, '华南' AS s))", ["e", "s"], id="PIVOT"),
pytest.param("SELECT * FROM sales MATCH_RECOGNIZE (PARTITION BY region ORDER BY id MEASURES buyer_phone AS x "
"ONE ROW PER MATCH PATTERN (a) DEFINE a AS amount > 0)", ["region", "x"], id="MATCH_RECOGNIZE"),
])
def test_pivots_reading_masked_columns_hide_everything(sql, columns):
"""UNPIVOT 的值列、PIVOT 按 IN 里的值起的列、MATCH_RECOGNIZE 的 MEASURES 装着遮罩列的值:出的是哪几列不好认,整份遮。"""
assert hidden_of(sql, columns, kind="oracle") >= {c.lower() for c in columns}
@pytest.mark.parametrize("sql, columns", [
("SELECT id, val FROM sales UNPIVOT (val FOR col IN (region, note))", ["id", "val"]),
("SELECT * FROM (SELECT region, buyer_phone FROM sales) PIVOT (COUNT(buyer_phone) FOR region IN "
"('华东' AS e, '华南' AS s))", ["e", "s"]),
])
def test_pivots_that_bring_no_masked_values_stay_visible(sql, columns):
assert hidden_of(sql, columns, kind="oracle") == set(MASKED)
# --------------------------------------------------------------------------
# 选出项套了括号的 *
# --------------------------------------------------------------------------
def test_a_star_in_parentheses_is_counted_as_a_star():
"""Postgres 里选出项 (s.*)、(s).* 照样展开成好几列:后面没起别名的表达式按结果末尾倒着数,不按一列数位置。"""
columns = [*LAYOUT, "upper"]
for item in ("(s.*)", "(s).*"):
sql = f"SELECT {item}, UPPER(s.buyer_name) FROM sales s"
assert hidden_of(sql, columns) == {*MASKED, "upper"}, item
def test_sqlite_runs_of_the_same_shapes_are_unchanged():
"""SQLite 没有整行引用、列名表:同样的写法照旧(表和列同名时按列认,不多遮)。"""
conn = sqlite3.connect(":memory:")
conn.executescript("CREATE TABLE region (id INTEGER, region TEXT, buyer_phone TEXT);"
"INSERT INTO region VALUES (1, '华东', '13900000001');")
sql = "SELECT region AS 区域, COUNT(*) AS n FROM region GROUP BY region"
cursor = conn.execute(sql)
payload = {"columns": [d[0] for d in cursor.description], "rows": [list(r) for r in cursor.fetchall()], "sql": sql}
assert masking.mask_query_result(payload, MASKED, "sqlite").payload is payload
conn.close()
+133
View File
@@ -490,3 +490,136 @@ async def test_a_union_of_monthly_tables_is_counted_with_the_sources_schema(monk
assert rows["masked_at_query"] == ["buyer_phone"] and _cells(rows, "amount") == [1200, 300, 900]
finally:
await engines.invalidate(source_id)
# --------------------------------------------------------------------------
# 过期的表结构、写了库名的表、整行与列名表(复核第 3 轮)
# --------------------------------------------------------------------------
#: 一月那张表后来多了一列 extra(排在第二),数据源的表结构是那之前探查的;二月那张号码排在第二列
REBUILT = ("CREATE TABLE jan (id INTEGER PRIMARY KEY, extra TEXT, region TEXT, buyer_phone TEXT, amount INTEGER);"
"CREATE TABLE feb (id INTEGER PRIMARY KEY, buyer_phone TEXT, region TEXT, amount INTEGER, extra TEXT);"
"INSERT INTO jan VALUES (1, '加急', '华东', '13900000001', 1200);"
"INSERT INTO feb VALUES (2, '13900000002', '华南', 300, '普通');")
async def test_a_stale_schema_does_not_place_masked_values_by_its_old_layout(monkeypatch, tmp_path):
"""数据源的表结构过期了(一月那张少了 extra):UNION 第一支的 * 按它数出的列名和后一支的列数对不上,不按它对位置,整份
遮。以前照过期的列名对,二月的号码落进的「extra」认不出,原样交给模型、落库。"""
name, source_id = await _source(tmp_path, {"mask_columns": "buyer_phone"}, script=REBUILT)
async with SessionLocal() as session:
row = await session.get(DataSource, source_id)
cache = dict(row.schema_cache)
jan = dict(cache["tables"]["jan"])
jan["columns"] = [c for c in jan["columns"] if c["name"] != "extra"]
row.schema_cache = {**cache, "tables": {**cache["tables"], "jan": jan}}
await session.commit()
try:
sql = ("WITH t AS (SELECT * FROM jan UNION ALL SELECT id, buyer_phone, region, amount, extra FROM feb) "
"SELECT extra AS 备注, id AS n FROM t ORDER BY id")
script = Script(monkeypatch, [q(name, sql, "c1")], {"value": {"alias": "Q1", "row": 0, "column": "n"},
"rationale": "x"})
out, _ = await ask(name)
seen = script.tool_results()["c1"]
assert MASKED in seen and _clean(seen), seen
snapshot = artifact_store.load(out.queries[0]["artifact"])
assert "备注" in snapshot["masked_at_query"] and _clean(json.dumps(snapshot, ensure_ascii=False))
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
got = await client.get(f"/api/artifacts/{out.queries[0]['artifact']}")
assert got.status_code == 200 and _clean(got.text)
finally:
await engines.invalidate(source_id)
#: 造出来的 Postgres、Oracle 数据源的表结构:探查的是当前库(没设 schema),sales 一张表
PG_LAYOUT = ["id", "region", "buyer_phone", "buyer_name", "amount", "note"]
PG_SCHEMA = {"tables": {"sales": {"qualified": "sales", "schema": None,
"columns": [{"name": c, "type": "TEXT"} for c in PG_LAYOUT]}}, "schema": None}
PG_ROWS = [[1, "华东", "13900000001", "张伟", 1200, "改址"], [2, "华南", "13900000002", "李娜", 300, "回访"]]
def _json_row(row: list) -> str:
return json.dumps(dict(zip(PG_LAYOUT, row)), ensure_ascii=False)
#: (数据源类型, SQL, 数据库交回的行, 查询时就整列遮掉的列, 编号那一格取到的值)。结果列按 SQL 里起的别名(j、n)。行照
#: 驱动交回的样子造:JSON 是字符串,记录转成文字;archive.sales 是另一个库里的同名表,号码排在第二列。表的列名表只写了
#: 前几列(Postgres 后面的列照原名)、另一个库里的同名表:说不清号码改叫了哪个名字,那一段整段遮,编号也遮
DRIVER_CASES = [
("postgresql", "SELECT json_agg(t) AS j, 1 AS n FROM (SELECT id, buyer_phone FROM sales) t",
[[json.dumps([{"id": r[0], "buyer_phone": r[2]} for r in PG_ROWS]), 1]], ["j"], 1),
("postgresql", "SELECT row_to_json(s) AS j, s.id AS n FROM sales s ORDER BY s.id",
[[_json_row(r), r[0]] for r in PG_ROWS], ["j"], 1),
("postgresql", "SELECT to_json(s.*) AS j, s.id AS n FROM sales s ORDER BY s.id",
[[_json_row(r), r[0]] for r in PG_ROWS], ["j"], 1),
("postgresql", "SELECT s AS j, s.id AS n FROM sales s ORDER BY s.id",
[["(" + ",".join(map(str, r)) + ")", r[0]] for r in PG_ROWS], ["j"], 1),
("postgresql", "SELECT p AS j, i AS n FROM sales AS s(i, r, p) ORDER BY i", [[r[2], r[0]] for r in PG_ROWS],
["j", "n"], None),
("postgresql", "WITH t(a, b, c, d, e, f) AS (TABLE sales) SELECT c AS j, a AS n FROM t ORDER BY a",
[[r[2], r[0]] for r in PG_ROWS], ["j"], 1),
("postgresql", "WITH t(a, b, c, d, e, f) AS (SELECT * FROM archive.sales) SELECT b AS j, a AS n FROM t",
[["13900000007", 7]], ["j", "n"], None),
("postgresql", "WITH t(a, b, c, d, e, f) AS (SELECT * FROM sales) SELECT c AS j, a AS n FROM t ORDER BY a",
[[r[2], r[0]] for r in PG_ROWS], ["j"], 1),
("oracle", "SELECT JSON_OBJECT(*) AS j, id AS n FROM sales ORDER BY id", [[_json_row(r), r[0]] for r in PG_ROWS],
["j"], 1),
("oracle", "SELECT id AS n, val AS j FROM sales UNPIVOT (val FOR col IN (buyer_phone, region)) ORDER BY id",
[[1, "13900000001"], [1, "华东"], [2, "13900000002"], [2, "华南"]], ["n", "j"], None),
]
@pytest.fixture
async def driver_shop():
"""造一个 Postgres / Oracle 数据源(查询交给下面换掉的 run_query,不连库):买家手机号、姓名设了遮罩。返回建源的函数。"""
made: list[int] = []
async def make(kind: str) -> str:
name = f"pg_{uuid.uuid4().hex[:8]}"
async with SessionLocal() as session:
row = DataSource(name=name, kind=kind, host="127.0.0.1", port=5432, username="probe", database="shop",
readonly=True, enabled=True, options={"mask_columns": "buyer_phone, buyer_name"},
schema_cache=PG_SCHEMA)
session.add(row)
await session.commit()
made.append(row.id)
return name
yield make
for source_id in made:
await engines.invalidate(source_id)
@pytest.mark.parametrize("kind, sql, rows, masked, value", DRIVER_CASES,
ids=["json_agg(t)", "row_to_json(s)", "to_json(s.*)", "SELECT s", "表的列名表", "CTE 里的 TABLE",
"另一个库里的同名表", "本库的表按位置数", "JSON_OBJECT(*)", "UNPIVOT"])
async def test_whole_rows_and_renamed_columns_from_other_databases_are_masked(monkeypatch, driver_shop, kind, sql,
rows, masked, value):
"""Postgres 的整行(json_agg(t)、row_to_json(s)、to_json(s.*)、SELECT s)、表的列名表、CTE 里的 TABLE sales、另一个库里的
同名表,Oracle 的 JSON_OBJECT(*)、UNPIVOT:交给模型的、落库的、通用接口给的都是遮好的,编号照常(认不出位置的整份遮)。
以前这些写法 masked_at_query 是空的,号码、姓名原样交给模型、落库。"""
from app.data.engine import QueryResult
from app.tools import datasource
async def driver(source, statement, **kwargs):
assert statement == sql
columns = ["n", "j"] if statement.startswith("SELECT id AS n") else ["j", "n"]
return QueryResult(columns=columns, rows=[list(r) for r in rows], row_count=len(rows), truncated=False,
elapsed_ms=1, sql=statement)
monkeypatch.setattr(datasource, "run_query", driver)
name = await driver_shop(kind)
script = Script(monkeypatch, [q(name, sql, "c1")], {"value": {"alias": "Q1", "row": 0, "column": "n"},
"rationale": "x"})
out, events = await ask(name)
seen = script.tool_results()["c1"]
assert MASKED in seen and _clean(seen), seen
snapshot = artifact_store.load(out.queries[0]["artifact"])
assert snapshot["masked_at_query"] == masked and _clean(json.dumps(snapshot, ensure_ascii=False))
assert out.result["value"] == value
[end] = [d for k, d in events if k == EventType.TOOL_END]
assert _clean(end["preview"])
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
got = await client.get(f"/api/artifacts/{out.queries[0]['artifact']}")
assert got.status_code == 200 and _clean(got.text)