Files
AgentLab/backend/tests/test_masking_star_renames.py
T
ThearchyHelios b0fdf56eed K3 复核修正(第 3 轮):写了库名的表按「库名.表名」查表结构;UNION 第一支按表结构数出的列名要和后面每一支一样宽才用;整行引用、函数参数里的 *、表的列名表、TABLE x、PIVOT 这类认得出
- 写了库名的表(复核 major 1):表结构以前按表名查,查询写的是另一个库里的同名表(archive.sales,列序不同)时照探查
  的那张数位置,号码原样交给模型、落库。现在表结构另按「库名.表名」记一份(库名取探查时记下的 schema、qualified 的
  前缀,masking._schema_keys),写了库名的引用按它查(_table_key),查不到的按数不清整段遮;探查时没设 schema、查询又
  写了库名的也查不到(多遮)。数位置(_star_columns、_branch_names)、认标签列(_table_lacks)都照这个查法
- 过期的表结构(复核 major 2,对 099473c 的回退):UNION 第一支是 * 时,按表结构数出的列名以前不核对列数就拿来对后一支
  的位置,表结构过期、列数已经变了也照对,遮罩值记到错的列名上。现在后面每一支数得出的列数都和它一样宽才用
  (_source_names、_width),对不上、数不清的退回整段遮
- 整行和别的 * 写法(复核 major 3):写在函数参数、表达式里的 *、s.*(to_json(s.*)、concat_ws(',', s.*)、ROW(s.*)、
  Oracle 的 JSON_OBJECT(*))按它展开的来源认(_value_columns、_scoped_star_inputs,相关子查询、FROM 里的子查询照样
  找得到外层的来源)。几种写法先改写语法树再认(_normalized,原文不动):Postgres 的整行引用(row_to_json(s)、
  json_agg(t)、SELECT s、s::text)按 Postgres 先认列的规矩,不是列的改成 ROW(s.*);带列名表的表(FROM sales AS
  s(i, r, p),引用 CTE 时写的也算)改成派生来源;解析器读错的 TABLE x 改成 SELECT * FROM x;选出项套了括号的 *
  ((s.*)、(s).*)按 * 数位置。PIVOT、UNPIVOT、MATCH_RECOGNIZE 读到遮罩列的整份遮。SQL 解析不了、里面有 *(COUNT(*)
  除外)或 TABLE x 的(UNION ALL TABLE x、带库名的 TABLE),同写着遮罩列一样整份遮(_implicit)
- Postgres 的列名表只写前几列时,不按表结构补全:别的方言里列名表比列少只能是表结构过期了,补全就丢了列数这道核对;
  这时改了名的几列整段遮(多遮),照原名的按名字认
- inquiry 模块说明的已知局限补上:写了库名的表、过期表结构的列数核对、Postgres 列名表只写前几列时整段遮、探查的不是
  默认库时没写库名的同名表可能数错、整行引用按表结构先认列(查不到时按整行多遮)、解析不了的 SQL 里的整行引用认不出

测试:新增 test_masking_whole_rows(整行、函数参数里的 *、列名表、TABLE x、Oracle 的 JSON_OBJECT(*) / UNPIVOT / PIVOT /
MATCH_RECOGNIZE、解析不了的 SQL、不碰遮罩列的照常);test_masking_star_renames 补写了库名的表(SQLite ATTACH 真跑)、
过期的表结构;test_verify_agent_masking 补核查 agent 的真实查询工具:过期的表结构,Postgres / Oracle 形状的结果(换掉
驱动)交给模型、落库、通用工件接口都遮好。每处实现都做过去掉之后测试变红的验证;复核给的 Postgres 18 探针、端到端
在临时库上重跑,都不再漏。
2026-10-06 21:50:48 +08:00

379 lines
23 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""按 SQL 认取自遮罩列的结果列(engine/masking.py 的 _derived_hidden):* 把遮罩列带进了别的列名(复核第 2 轮的 blocker)。
* 展开的列按位置落进结果:WITH t(a, b, …) AS (SELECT * FROM sales) 的列名表、FROM 里子查询的列名表会给它们换上别的
名字,UNION 后一支的 * 落进第一支起的列名里,标量子查询里的 * 则把值交给外层的表达式。以前 * 本身不算取自哪一列,这些
写法的原值原样交给核查 agent、落进快照,审批质询按值遮的第二道防线也认不出。现在 * 带出的值取自遮罩列时,它展开落到的
那几列都算:只展开一个来源、那里的每一列都数得清的(子查询按选出项,数据源里的表按查询当时的表结构),按位置只算取自
遮罩列的那几列;数不清的(没有表结构、列数对不上)整段都算(多遮)。不改名的 *(单独一条 SELECT 的 *、UNION 第一支的 *)
照旧按名字认;* 只带出没碰遮罩列的值、EXISTS 里的 * 都不算。
查询都在内存里的 SQLite 上真跑一遍,看遮完的结果里还有没有原值。数据都是虚构的:区域、手机号、姓名、金额。
"""
from __future__ import annotations
import json
import sqlite3
import pytest
from app.db import models # noqa: F401 - 建表(conftest)之前先登记模型:表结构快照要存进工件库
from app.engine import masking
MASKED_WORD = "已遮罩"
#: 设了遮罩的列
MASKED = ["buyer_phone", "buyer_name"]
#: 遮罩列里的值:遮完的结果里不能出现
SECRETS = ("1390000000", "张伟", "李娜", "王芳")
SCRIPT = ("CREATE TABLE sales (id INTEGER, region TEXT, buyer_phone TEXT, buyer_name TEXT, amount INTEGER, note TEXT);"
"INSERT INTO sales VALUES (1, '华东', '13900000001', '张伟', 1200, '改址'),"
" (2, '华南', '13900000002', '李娜', 300, '回访'), (3, '华北', '13900000003', '王芳', 900, '');"
# 只有一列号码的表:标量子查询、UNION 后一支的 * 直接展开它
"CREATE TABLE phones (buyer_phone TEXT);"
"INSERT INTO phones VALUES ('13900000009');")
@pytest.fixture(scope="module")
def db():
conn = sqlite3.connect(":memory:")
conn.executescript(SCRIPT)
yield conn
conn.close()
def masked_run(db, sql: str) -> masking.QueryMask:
"""真跑一遍 SQL,按工具边界的办法遮(mask_query_result)。"""
cursor = db.execute(sql)
columns = [d[0] for d in cursor.description]
rows = [list(r) for r in cursor.fetchall()]
payload = {"columns": columns, "rows": rows, "row_count": len(rows), "truncated": False, "sql": sql}
return masking.mask_query_result(payload, MASKED, "sqlite")
def _leaks(done: masking.QueryMask) -> bool:
text = json.dumps(done.payload["rows"], ensure_ascii=False)
return any(s in text for s in SECRETS)
@pytest.mark.parametrize("sql, hidden, shown", [
pytest.param("WITH t(a, b, c, d, e, f) AS (SELECT * FROM sales) SELECT * FROM t", {"c", "d"}, set(),
id="CTE 列名表套 * 展开表"),
pytest.param("WITH t(a, b, c, d, e, f) AS (SELECT * FROM sales) SELECT c AS 号码, e AS 金额 FROM t "
"ORDER BY e DESC LIMIT 1", {"号码"}, set(), id="CTE 列名表套 * 再改名"),
pytest.param("SELECT 0 AS a, '' AS b, '' AS c, '' AS d, 0 AS e, '' AS f UNION ALL SELECT * FROM sales",
{"c", "d"}, set(), id="UNION 后一支的 * 展开表"),
pytest.param("SELECT region FROM sales UNION ALL SELECT * FROM phones", {"region"}, set(),
id="UNION 后一支的 * 展开只有号码的表"),
pytest.param("WITH a AS (SELECT buyer_phone FROM sales), b(x) AS (SELECT * FROM a) SELECT x FROM b", {"x"},
set(), id="列名表套 * 展开另一段 CTE"),
pytest.param("SELECT r FROM (SELECT region AS r FROM sales UNION ALL "
"SELECT * FROM (SELECT buyer_phone FROM sales))", {"r"}, set(), id="子查询里 UNION 后一支的 *"),
pytest.param("SELECT * FROM (SELECT region FROM sales UNION ALL SELECT * FROM (SELECT buyer_phone FROM sales))",
{"region"}, set(), id="外层 * 取子查询里 UNION 后一支的 *"),
pytest.param("WITH t(k) AS (SELECT * FROM (SELECT buyer_name FROM sales)) SELECT k AS 名 FROM t", {"名"}, set(),
id="列名表套 * 展开子查询"),
pytest.param("SELECT region FROM sales UNION ALL SELECT s.* FROM (SELECT buyer_name FROM sales) s", {"region"},
set(), id="UNION 后一支的 t.*"),
pytest.param("SELECT region FROM sales UNION SELECT * FROM (SELECT buyer_phone FROM sales)", {"region"}, set(),
id="UNION 去重"),
# 只展开一个派生来源、每一列都数得清:按位置只遮取自遮罩列的那一列,金额照常
pytest.param("SELECT '合计' AS 区域, SUM(amount) AS 金额 FROM sales UNION ALL SELECT * FROM "
"(SELECT buyer_phone, amount FROM sales ORDER BY amount DESC LIMIT 2)", {"区域"}, {"金额"},
id="合计行 UNION 后一支的 * 按位置"),
pytest.param("SELECT region AS 区域, amount AS 金额 FROM sales UNION ALL SELECT * FROM "
"(SELECT * FROM (SELECT buyer_name, amount FROM sales))", {"区域"}, {"金额"},
id="套了两层的 * 按位置"),
pytest.param("WITH t(r, x) AS (SELECT * FROM (SELECT buyer_phone, amount FROM sales)) SELECT r AS 号码, x AS 金额 "
"FROM t", {"号码"}, {"金额"}, id="列名表套 * 按位置"),
pytest.param("SELECT (SELECT * FROM (SELECT buyer_phone FROM sales) LIMIT 1) AS x", {"x"}, set(),
id="标量子查询里的 * 展开子查询"),
pytest.param("SELECT region, (SELECT * FROM (SELECT s2.buyer_name FROM sales s2 WHERE s2.id = s.id)) AS 名 "
"FROM sales s", {"名"}, {"region"}, id="相关子查询里的 *"),
pytest.param("SELECT (SELECT * FROM phones LIMIT 1) AS x", {"x"}, set(), id="标量子查询里的 * 展开表"),
])
def test_a_star_that_renames_a_masked_column_is_masked(db, sql, hidden, shown):
"""* 把遮罩列的值带进了别的列名:这几列整列遮,原值不交出去。按位置认得出的,别的列照常。"""
done = masked_run(db, sql)
columns = {c.lower() for c in done.columns}
assert hidden <= columns, done.columns
assert not shown & columns, done.columns
assert not _leaks(done), done.payload["rows"]
assert done.secrets.strong, "遮掉的值交回调用方(后面的查询、回答里按值遮)"
@pytest.mark.parametrize("sql", [
"SELECT * FROM (SELECT region, amount FROM sales) t",
"WITH t AS (SELECT * FROM sales) SELECT region, SUM(amount) AS 合计 FROM t GROUP BY region",
"SELECT region AS 区域 FROM sales UNION ALL SELECT * FROM (SELECT region FROM sales)",
"WITH t(a, b) AS (SELECT * FROM (SELECT region, amount FROM sales)) SELECT a, SUM(b) AS 合计 FROM t GROUP BY a",
"SELECT region, EXISTS(SELECT * FROM sales s2 WHERE s2.id = s.id AND s2.amount > 500) AS 大单 FROM sales s",
"SELECT '合计' AS 区域, SUM(amount) AS 金额 FROM sales UNION ALL SELECT * FROM "
"(SELECT region, SUM(amount) FROM sales GROUP BY region)",
])
def test_a_star_that_brings_no_masked_values_stays_visible(db, sql):
"""* 只带出没碰遮罩列的值、EXISTS 里的 *(只说有没有):一列都不遮,结果原样。"""
done = masked_run(db, sql)
assert done.columns == [] and MASKED_WORD not in json.dumps(done.payload, ensure_ascii=False)
def test_a_plain_star_still_masks_by_name(db):
"""不改名的 *(单独一条 SELECT)照旧只遮名字对上遮罩列的那几列。"""
done = masked_run(db, "SELECT * FROM sales")
assert done.columns == ["buyer_phone", "buyer_name"] and not _leaks(done)
@pytest.mark.parametrize("sql, columns, hidden", [
pytest.param("SELECT x FROM (SELECT * FROM (SELECT buyer_phone FROM sales) s) AS q(x)", ["x"], {"x"},
id="子查询列名表套 *"),
pytest.param("SELECT l.x FROM sales s CROSS JOIN LATERAL (SELECT * FROM (SELECT s.buyer_phone) z) AS l(x)", ["x"],
{"x"}, id="LATERAL 列名表套 *"),
pytest.param("SELECT ARRAY(SELECT * FROM (SELECT buyer_phone FROM sales) s) AS arr", ["arr"], {"arr"},
id="ARRAY 子查询里的 *"),
])
def test_a_star_renaming_in_postgres_is_masked(sql, columns, hidden):
snapshot = {"source": "shop", "sql": sql, "columns": columns, "rows": []}
assert hidden <= masking._derived_hidden(snapshot, set(MASKED), "postgres")[0]
def test_a_metric_summed_over_a_star_renamed_masked_column_is_not_sent_to_blind_check():
"""双盲复算按同一套认法挡下取自遮罩列的指标(derived_columns):遮罩的金额经 CTE 列名表套 * 改名之后汇总,照样认得出,
区域照常。"""
sql = "WITH q(r, x) AS (SELECT * FROM (SELECT region, amount FROM sales)) SELECT r AS 区域, SUM(x) AS gmv FROM q " \
"GROUP BY r"
snapshot = {"source": "shop", "sql": sql, "columns": ["区域", "gmv"], "rows": []}
assert masking.derived_columns(snapshot, {"amount"}, "sqlite") == {"gmv"}
def test_a_recursive_cte_with_a_star_is_read_without_looping(db):
"""递归的 CTE 里 * 展开它自己:照样认得出、不会绕不出来。"""
sql = ("WITH RECURSIVE r(n, p) AS (SELECT 1, buyer_phone FROM sales WHERE id = 1 UNION ALL "
"SELECT n + 1, p FROM r WHERE n < 2) SELECT * FROM (SELECT * FROM r) t")
done = masked_run(db, sql)
assert "p" in {c.lower() for c in done.columns} and not _leaks(done)
loop = "WITH RECURSIVE r AS (SELECT buyer_phone AS p FROM sales UNION ALL SELECT * FROM r) SELECT p AS 号 FROM r"
snapshot = {"source": "shop", "sql": loop, "columns": ["号"], "rows": []}
assert "号" in masking._derived_hidden(snapshot, set(MASKED), "sqlite")[0]
def test_a_star_with_a_replace_is_not_counted_by_position():
"""* 带着 REPLACE / EXCLUDE 这类改动(DuckDB 一类的写法):展开成的列不再是来源那几列的原样,不按位置数,整段都算。
这里 REPLACE 把手机号换进了「amount」那一列。"""
sql = ("SELECT region, amount FROM sales UNION ALL SELECT * REPLACE (buyer_phone AS amount) FROM "
"(SELECT buyer_phone, amount FROM sales) t")
snapshot = {"source": "shop", "sql": sql, "columns": ["region", "amount"], "rows": []}
assert {"region", "amount"} <= masking._derived_hidden(snapshot, set(MASKED), "duckdb")[0]
# --------------------------------------------------------------------------
# * 展开数据源里的表:按查询当时的表结构数位置
# --------------------------------------------------------------------------
#: 两个月的订单表列一样、次序一样(上传表格里的月度工作表常这样);三月那张的号码和区域换了次序
MONTHS = ("CREATE TABLE jan (id INTEGER, region TEXT, buyer_phone TEXT, amount INTEGER);"
"CREATE TABLE feb (id INTEGER, region TEXT, buyer_phone TEXT, amount INTEGER);"
"CREATE TABLE mar (id INTEGER, buyer_phone TEXT, region TEXT, amount INTEGER);"
"INSERT INTO jan VALUES (1, '华东', '13900000001', 1200), (2, '华南', '13900000002', 300);"
"INSERT INTO feb VALUES (3, '华东', '13900000003', 900);"
"INSERT INTO mar VALUES (4, '13900000004', '华北', 400);")
@pytest.fixture(scope="module")
def months():
conn = sqlite3.connect(":memory:")
conn.executescript(MONTHS)
yield conn
conn.close()
def schema_of(conn) -> dict:
"""数据源的表结构(schema_cache 的样子):每张表的列按表里的次序。"""
names = [name for (name,) in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name")]
return {"tables": {name: {"columns": [{"name": row[1]} for row in conn.execute(f"PRAGMA table_info({name})")]}
for name in names}}
def run_months(conn, sql: str, schema: dict | None) -> masking.QueryMask:
cursor = conn.execute(sql)
payload = {"columns": [d[0] for d in cursor.description], "rows": [list(r) for r in cursor.fetchall()], "sql": sql}
return masking.mask_query_result(payload, ["buyer_phone"], "sqlite", schema=schema)
def test_a_union_of_tables_with_the_same_layout_is_masked_by_name_with_the_schema(months):
"""UNION 的几支都是 SELECT * FROM 表、表的列一样:按表结构数得出号码落在哪一列,只遮那一列,汇总照常。没有表结构时
说不清号码排在第几,* 展开落到的那一段整段遮(多遮)。"""
schema = schema_of(months)
total = "SELECT region, SUM(amount) AS 合计 FROM (SELECT * FROM jan UNION ALL SELECT * FROM feb) t GROUP BY region"
done = run_months(months, total, schema)
assert done.columns == [] and done.payload["rows"] == [["华东", 2100], ["华南", 300]]
assert {"region", "合计"} <= set(run_months(months, total, None).columns)
both = run_months(months, "SELECT * FROM jan UNION ALL SELECT * FROM feb", schema)
assert both.columns == ["buyer_phone"] and not _leaks(both)
def test_a_union_of_tables_in_another_order_masks_where_the_values_land(months):
"""三月那张表号码排在第二列:UNION 之后它落进第一支的「region」,按表结构认得出,这一列遮;金额照常。"""
schema = schema_of(months)
both = run_months(months, "SELECT * FROM jan UNION ALL SELECT * FROM mar", schema)
assert {"region", "buyer_phone"} <= set(both.columns) and "amount" not in both.columns and not _leaks(both)
total = "SELECT region AS 区域, SUM(amount) AS 合计 FROM (SELECT * FROM jan UNION ALL SELECT * FROM mar) t " \
"GROUP BY region"
done = run_months(months, total, schema)
assert done.columns == ["区域"] and not _leaks(done)
def test_a_column_list_over_a_table_is_masked_by_position_with_the_schema(months):
"""列名表套 * 展开数据源里的表:按表结构只遮号码换了名字的那一列。"""
done = run_months(months, "WITH t(a, b, c, d) AS (SELECT * FROM jan) SELECT * FROM t", schema_of(months))
assert done.columns == ["c"] and [row[3] for row in done.payload["rows"]] == [1200, 300]
def test_a_schema_that_does_not_match_the_table_masks_the_whole_stretch(months):
"""表结构里没有这张表,或者列数和实际展开的对不上(表结构过期了):照旧整段遮。"""
stale = schema_of(months)
stale["tables"]["feb"]["columns"] = stale["tables"]["feb"]["columns"][:3]
for schema in (stale, {"tables": {"jan": schema_of(months)["tables"]["jan"]}}):
both = run_months(months, "SELECT * FROM jan UNION ALL SELECT * FROM feb", schema)
assert set(both.columns) == {"id", "region", "buyer_phone", "amount"} and not _leaks(both)
async def test_a_snapshot_is_read_with_its_frozen_schema(months):
"""事后按快照认(双盲复算挡指标、审批质询收值)用快照记着的表结构快照(schema_artifact),和查询当时工具边界的认法
一样:几张同样的月度表 UNION 起来汇总的金额不算取自遮罩列。"""
from app.core import artifact_store
sql = "SELECT region, SUM(amount) AS 合计 FROM (SELECT * FROM jan UNION ALL SELECT * FROM feb) t GROUP BY region"
schema = await artifact_store.put_json({"source": "shop", **schema_of(months)}, kind="schema_snapshot",
run_id="r-star", node_id="n1")
snapshot = {"source": "shop", "sql": sql, "columns": ["region", "合计"], "rows": [], "schema_artifact": schema}
assert masking.derived_columns(snapshot, {"buyer_phone"}, "sqlite") == set()
assert masking._derived_hidden(snapshot, {"buyer_phone"}, "sqlite")[0] == {"buyer_phone"}
assert masking.derived_columns({**snapshot, "schema_artifact": "missing"}, {"buyer_phone"}, "sqlite") == {
"region", "合计"}
# --------------------------------------------------------------------------
# 写了库名的表、过期的表结构(复核第 3 轮)
# --------------------------------------------------------------------------
#: 同名的两张表:主库的 sales 和另一个库(archive)里的 sales,号码、区域换了次序。表结构探查的是主库
TWO_SALES = ("CREATE TABLE sales (id INTEGER, region TEXT, buyer_phone TEXT, buyer_name TEXT, amount INTEGER, "
"note TEXT);"
"INSERT INTO sales VALUES (1, '华东', '13900000001', '张伟', 1200, '改址');"
"CREATE TABLE archive.sales (id INTEGER, buyer_phone TEXT, region TEXT, buyer_name TEXT, amount INTEGER, "
"note TEXT);"
"INSERT INTO archive.sales VALUES (7, '13900000007', '华北', '王芳', 500, '回访');")
SALES_LAYOUT = ["id", "region", "buyer_phone", "buyer_name", "amount", "note"]
@pytest.fixture
def two_sales():
conn = sqlite3.connect(":memory:")
conn.execute("ATTACH DATABASE ':memory:' AS archive")
conn.executescript(TWO_SALES)
yield conn
conn.close()
def probed(schema: str | None = None) -> dict:
"""探查主库得到的表结构(schema_cache 的样子):没设 schema 的探查当前库,qualified 就是表名。"""
meta = {"qualified": f"{schema}.sales" if schema else "sales", "schema": schema,
"columns": [{"name": c} for c in SALES_LAYOUT]}
return {"tables": {"sales": meta}, "schema": schema}
def run_two(conn, sql: str, schema: dict | None) -> masking.QueryMask:
cursor = conn.execute(sql)
payload = {"columns": [d[0] for d in cursor.description], "rows": [list(r) for r in cursor.fetchall()], "sql": sql}
return masking.mask_query_result(payload, MASKED, "sqlite", schema=schema)
@pytest.mark.parametrize("sql", [
pytest.param("WITH t(a, b, c, d, e, f) AS (SELECT * FROM archive.sales) SELECT b AS j, a AS n FROM t",
id="列名表套 * 展开另一个库里的同名表"),
pytest.param("SELECT 0 AS a, '' AS b, '' AS c, '' AS d, 0 AS e, '' AS f WHERE 0 UNION ALL "
"SELECT * FROM archive.sales", id="UNION 后一支的 * 展开另一个库里的同名表"),
pytest.param("WITH t AS (SELECT * FROM sales UNION ALL SELECT * FROM archive.sales) SELECT region AS 区域 FROM t",
id="派生来源里 UNION 后一支展开另一个库里的同名表"),
])
def test_a_table_in_another_schema_is_not_counted_by_the_probed_one(two_sales, sql):
"""表结构探查的是主库:查询写了别的库名的同名表(archive.sales,号码排在第二列),不能按主库那张表的列序数位置,
按数不清处理,* 展开落到的那一段整段遮。以前按表名查表结构,号码落进的那一列被当成区域,原样交给模型、落库。"""
done = run_two(two_sales, sql, probed())
assert not _leaks(done), done.payload["rows"]
def test_a_table_named_with_its_probed_schema_is_still_counted(two_sales):
"""库名和表结构记下的对得上(探查时设了 schema、qualified 带着它):照样按表结构数位置,只遮号码那一列。没写库名的
照旧按表名查。库名对不上表结构记下的(这里探查时没设 schema,查询写了 main.)按数不清处理,整段遮(多遮)。"""
precise = "WITH t(a, b, c, d, e, f) AS (SELECT * FROM {}) SELECT c AS j, a AS n FROM t"
assert run_two(two_sales, precise.format("main.sales"), probed("main")).columns == ["j"]
assert run_two(two_sales, precise.format("sales"), probed()).columns == ["j"]
assert set(run_two(two_sales, precise.format("main.sales"), probed()).columns) == {"j", "n"}
def test_schema_columns_are_also_keyed_by_the_probed_schema():
"""表结构按「库名.表名」另记一份(库名取 schema、qualified 的前缀);写了别的库名的查不到。"""
tables = masking._schema_columns(probed("public"))
assert tables["sales"] == tables["public.sales"] == SALES_LAYOUT and "archive.sales" not in tables
assert "public.sales" not in masking._schema_columns(probed())
assert "archive.sales" not in (masking._schema_tables(probed("public")) or {})
#: 一月的表结构是早先探查的(那时还没有 extra 这一列),表里现在多了一列,排在第二;二月那张号码排在第二列
STALE = ("CREATE TABLE jan (id INTEGER, extra TEXT, region TEXT, buyer_phone TEXT, amount INTEGER);"
"CREATE TABLE feb (id INTEGER, buyer_phone TEXT, region TEXT, amount INTEGER, extra TEXT);"
"INSERT INTO jan VALUES (1, '加急', '华东', '13900000001', 1200);"
"INSERT INTO feb VALUES (2, '13900000002', '华南', 300, '普通');")
@pytest.fixture
def stale():
conn = sqlite3.connect(":memory:")
conn.executescript(STALE)
yield conn
conn.close()
def layout(**tables: list[str]) -> dict:
return {"tables": {name: {"qualified": name, "columns": [{"name": c} for c in columns]}
for name, columns in tables.items()}}
#: 过期的表结构:一月那张少了 extra(列数和表里实际的不一样)
OLD_JAN = layout(jan=["id", "region", "buyer_phone", "amount"], feb=["id", "buyer_phone", "region", "amount", "extra"])
#: 准的表结构
NOW = layout(jan=["id", "extra", "region", "buyer_phone", "amount"],
feb=["id", "buyer_phone", "region", "amount", "extra"])
@pytest.mark.parametrize("sql", [
pytest.param("WITH t AS (SELECT * FROM jan UNION ALL SELECT id, buyer_phone, region, amount, extra FROM feb) "
"SELECT extra AS 备注 FROM t", id="后一支明写各列"),
pytest.param("WITH t AS (SELECT * FROM jan UNION ALL SELECT * FROM feb) SELECT extra AS 备注 FROM t",
id="后一支也是 *"),
])
def test_a_first_branch_counted_by_a_stale_schema_is_not_trusted(stale, sql):
"""UNION 第一支是 *、按表结构数出的列名(表结构过期了,少一列)和后一支的列数对不上:不按它对列名,* 所在的位置按认不出
算,整份遮。以前照过期的列名表对,二月的号码落进的「extra」没认出来,原样交回。"""
cursor = stale.execute(sql)
payload = {"columns": [d[0] for d in cursor.description], "rows": [list(r) for r in cursor.fetchall()], "sql": sql}
for schema in (OLD_JAN, None, NOW):
done = masking.mask_query_result(payload, ["buyer_phone"], "sqlite", schema=schema)
assert done.columns == ["备注"] and not _leaks(done), (schema, done.payload["rows"])
def test_a_first_branch_counted_by_an_accurate_schema_still_places_the_values(stale):
"""表结构准、每一支数得出的列数都对得上:照样按第一支的列名对上位置,只遮号码落进的那一列,金额照常。"""
sql = ("WITH t AS (SELECT * FROM jan UNION ALL SELECT id, buyer_phone, region, buyer_phone, amount FROM feb) "
"SELECT region AS 区域, amount AS 金额 FROM t")
cursor = stale.execute(sql)
payload = {"columns": [d[0] for d in cursor.description], "rows": [list(r) for r in cursor.fetchall()], "sql": sql}
assert masking.mask_query_result(payload, ["buyer_phone"], "sqlite", schema=NOW).columns == []
moved = sql.replace("SELECT id, buyer_phone, region,", "SELECT id, region, buyer_phone,")
cursor = stale.execute(moved)
payload = {"columns": [d[0] for d in cursor.description], "rows": [list(r) for r in cursor.fetchall()],
"sql": moved}
done = masking.mask_query_result(payload, ["buyer_phone"], "sqlite", schema=NOW)
assert done.columns == ["区域"] and [row[1] for row in done.payload["rows"]] == [1200, 300]