mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 03:41:58 +00:00
合入期 4 整合 J2：审批质询遮罩收口——阈值只在同一个数在别处确实被换掉时才遮、夹住某一个值的边界按条件与作用域认、质询工件里未遮罩文字格夹着的遮罩值按值换掉；算式边界、分档标签列、CAST 类型参数等变体照实处理，已知局限写进模块说明
This commit is contained in:
ThearchyHelios
@@ -3,8 +3,11 @@
|
||||
- POST /api/runs/{run_id}/inquiries：{report, unit, quote?, question, approval_id?} → {inquiry, recorded, note}
|
||||
- GET /api/runs/{run_id}/inquiries?report=&unit=：{items: [Inquiry]}，封存前后的都算，按先后
|
||||
- GET /api/runs/{run_id}/inquiries/artifacts/{artifact_id}：质询引用过的工件 {id, content, masked[, mask_note]}——
|
||||
查询快照（queries[].artifact）照回答时的办法遮好：遮罩列（含起了别名、套了函数的）整列遮，别的列里写着遮罩值的格子
|
||||
也遮，masked 是有格子被遮掉的列名；content.sql 就是事件里记下的那条（遮罩值已换掉），两处一字不差。回答文档
|
||||
查询快照（queries[].artifact）照回答时的办法遮好：遮罩列（含起了别名、套了函数的）整列遮，别的列里整格是遮罩值的
|
||||
格子也遮，masked 是有格子整格被遮掉的列名（界面把这几列整列画成「已遮罩」）；一格文字里夹着遮罩值的（备注里写着
|
||||
手机号），只把那一截换成「已遮罩」，这一列不记进 masked。content.sql 就是事件里记下的那条（遮罩值已换掉），两处
|
||||
一字不差；content.checks（SQL 检查结果）的字段照证据面板，片段 sql_excerpt、说明 message 里的遮罩值照整条 SQL
|
||||
的换法换掉（截断处半截的值去掉）。回答文档
|
||||
（answer.doc_artifact）组装时就遮好了，原样给；回答里引用的报告指标（doc.catalog 里 m: 条目的 artifact，口径卡）
|
||||
也从这里取，界面一律按 doc.catalog[alias].artifact 取。没写进运行记录的回答（recorded 为假）在本进程里留一小时，
|
||||
这段时间里它引用的工件照样能取；之后、或者服务重启过，回 404。别的工件一律 404
|
||||
|
||||
+1208
-197
File diff suppressed because it is too large
Load Diff
@@ -12,9 +12,14 @@
|
||||
事件里记的报告文档不是这次运行的 422；输入不合法 422，指明是哪一项
|
||||
- 回答：先给结论，数字只能引用自己查询的格子或报告已有的指标，裸数字被标出（提问人自己写的数不标）；遮罩列（起了
|
||||
别名的、和 * 写在一起没起别名的表达式也算）的格子渲染成「已遮罩」，回答正文、查询清单、事件、质询工件里都不出现
|
||||
遮罩列的值；换着写法找某一个客户（简单 CASE、IN (VALUES …)、两头相同的范围、行值、选出列里写着同一个号码……）
|
||||
也一样；遮罩值不误伤阈值、天数、LIMIT（长的数也一样）、提问人自己写的数、报告口径卡里的指标；SQL 里照原样留着的
|
||||
阈值只在回答正文里不遮，格子照旧按值遮，回答和质询工件对得上；质询工件按值遮只出现在报告快照里的遮罩值
|
||||
遮罩列的值；换着写法找某一个客户（简单 CASE、IN (VALUES …)、两头相同的范围、行值、选出列里写着同一个号码，两半
|
||||
拆到 ON 和 WHERE、WHERE 和 HAVING、INTERSECT 两支、CTE 里，边界、等值写成算式，两头不是整数，范围再排除一头，
|
||||
号码前面补零……）也一样，年龄这类短的值换着写法找（套了 CAST、汇总值两头相同、补零、取模和整数相除算出来的边界……）
|
||||
SQL 里也遮；遮罩值不误伤阈值、天数、LIMIT（长的数也一样）、提问人自己写的数、报告口径卡里的指标，分档共用的边界、
|
||||
和日期、天数修饰、标签里写着的同一个数也不牵连阈值，分档的文字标签不当遮罩值（当分组键的档位列照常显示），改了
|
||||
名字的遮罩列、UNION 另一支取的遮罩列撞上标签照样遮；SQL 里照原样留着的阈值只在回答正文里不遮，格子照旧按值遮，
|
||||
回答和质询工件对得上；质询工件按值遮只出现在报告快照里的遮罩值；一格文字里夹着遮罩值时回答整份不交，质询工件里
|
||||
只换掉那一截；质询工件里 SQL 检查结果的片段、说明照整条 SQL 换（短的数也换），给模型的那句改法不交出去
|
||||
- 接线：设置里的上限、运行固定的数据版本交到 agent 手里（上传表格重传之后查的仍是运行固定的那一版）；同名重建的
|
||||
数据源、没固定过版本的上传表格不查
|
||||
- 触顶、失败照样记一条事件，说法给提问人看得懂、说下一步；轮数用完不算查询次数到顶，查询失败、引用解析不了在末尾注明
|
||||
@@ -989,6 +994,180 @@ async def test_numbers_that_merely_match_a_short_masked_value_are_left_alone(mon
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
async def test_a_threshold_matching_numbers_left_alone_elsewhere_is_sealed_as_written(monkeypatch, tmp_path, client):
|
||||
"""遮罩列是年龄：阈值 30、31 和天数修饰、日期、年龄段标签里写着的同一个数撞在一起。那些地方一个字没换，阈值也
|
||||
照原样封存（回答正文本来就写着「30 岁」），质询工件的 SQL 和事件里一字不差；找某一个年龄的等值照样遮，同一条
|
||||
SQL 里的阈值也跟着遮——留着就把等值里换掉的露了。"""
|
||||
name, source_id = await _source(tmp_path, options={"mask_columns": "age"}, script=AGED)
|
||||
graph = chain(node("start", "input"),
|
||||
node("fetch", "tool", tool=f"db_query__{name}",
|
||||
args={"sql": "SELECT region AS 区域, age, amount AS 金额 FROM sales ORDER BY id"}),
|
||||
node("write", "report", "写周报", instructions="写周报"),
|
||||
node("gate", "human", mode="approve", title="周报可以出具吗"),
|
||||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]))
|
||||
recent = "SELECT COUNT(*) AS 人数 FROM sales WHERE age >= 30 AND paid_on >= DATE('2026-02-03', '-30 days')"
|
||||
older = "SELECT COUNT(*) AS 人数 FROM sales WHERE age > 31 AND paid_on <= '2026-01-31'"
|
||||
# 规格原句：按年龄分档的标签里写着同样的 30。标签是 agent 写在 SQL 里的字，不是遮罩值，阈值和标签都照原样
|
||||
bands = ("SELECT CASE WHEN age < 30 THEN '30岁以下' ELSE '30岁及以上' END AS 年龄段, COUNT(*) AS 人数 FROM sales "
|
||||
"GROUP BY 1 ORDER BY 1")
|
||||
exact = "SELECT COUNT(*) AS 人数 FROM sales WHERE age >= 30 AND age = 30"
|
||||
plays = {"年龄大的": Play([recent, older, bands, exact],
|
||||
"近 30 天、30 岁及以上的有 [[v:Q1.r0.人数]] 人，31 岁以上、1 月 31 日前付款的有 "
|
||||
"[[v:Q2.r0.人数]] 人；[[v:Q3.r0.年龄段]]的有 [[v:Q3.r0.人数]] 人。[[see:Q1]]")}
|
||||
Models(monkeypatch, name, writer="华东首单金额 [[v:Q1.r0.金额]]。[[see:Q1]]", plays=plays)
|
||||
try:
|
||||
paused = await pause(name, graph=graph)
|
||||
[approval] = await pending(paused.id)
|
||||
r = await ask(client, paused.id, report="write", unit=None, question="近期年龄大的客户多不多？",
|
||||
approval_id=approval.id)
|
||||
assert r.status_code == 200, r.text
|
||||
item = r.json()["inquiry"]
|
||||
assert item["answer"]["text"] == ("近 30 天、30 岁及以上的有 2 人，31 岁以上、1 月 31 日前付款的有 0 人；"
|
||||
"30岁以下的有 2 人。")
|
||||
sealed = [recent, older, bands, "SELECT COUNT(*) AS 人数 FROM sales WHERE age >= 已遮罩 AND age = 已遮罩"]
|
||||
assert [q["sql"] for q in item["queries"]] == sealed
|
||||
[event] = await events(paused.id, "approval.inquiry")
|
||||
assert [q["sql"] for q in event.data["queries"]] == sealed
|
||||
for query, sql in zip(item["queries"], sealed):
|
||||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{query['artifact']}")
|
||||
assert got.status_code == 200 and got.json()["content"]["sql"] == sql, "质询工件的 SQL 和事件里的一字不差"
|
||||
finally:
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, sealed", [
|
||||
pytest.param("SELECT COUNT(*) AS 人数 FROM sales WHERE CAST(age AS INTEGER) = 25",
|
||||
"SELECT COUNT(*) AS 人数 FROM sales WHERE CAST(age AS INTEGER) = 已遮罩", id="套了 CAST"),
|
||||
pytest.param("SELECT COUNT(*) AS 人数 FROM sales GROUP BY age HAVING MAX(age) BETWEEN 25 AND 25",
|
||||
"SELECT COUNT(*) AS 人数 FROM sales GROUP BY age HAVING MAX(age) BETWEEN 已遮罩 AND 已遮罩",
|
||||
id="汇总值两头相同"),
|
||||
pytest.param("SELECT COUNT(*) AS 人数 FROM sales WHERE age = 25.0",
|
||||
"SELECT COUNT(*) AS 人数 FROM sales WHERE age = 已遮罩", id="补零的写法"),
|
||||
pytest.param("SELECT COUNT(*) AS 人数 FROM sales WHERE age > 24.5 AND age < 25.5",
|
||||
"SELECT COUNT(*) AS 人数 FROM sales WHERE age > 已遮罩 AND age < 已遮罩", id="两头不是整数"),
|
||||
pytest.param("SELECT COUNT(*) AS 人数 FROM sales WHERE age >= 25 AND age <= 26 AND age <> 26",
|
||||
"SELECT COUNT(*) AS 人数 FROM sales WHERE age >= 已遮罩 AND age <= 26 AND age <> 已遮罩",
|
||||
id="范围再排除一头"),
|
||||
pytest.param("SELECT COUNT(*) AS 人数 FROM sales WHERE age BETWEEN 20 + 5 AND 30 - 5",
|
||||
"SELECT COUNT(*) AS 人数 FROM sales WHERE age BETWEEN 已遮罩 + 已遮罩 AND 已遮罩 - 已遮罩",
|
||||
id="算出来的边界"),
|
||||
# 第 3 轮复核：取模、整数相除（SQLite 里只留整数部分）算出来的边界
|
||||
pytest.param("SELECT COUNT(*) AS 人数 FROM sales WHERE age >= 25 AND age <= 125 % 100",
|
||||
"SELECT COUNT(*) AS 人数 FROM sales WHERE age >= 已遮罩 AND age <= 已遮罩 % 已遮罩",
|
||||
id="取模算出来的边界"),
|
||||
pytest.param("SELECT COUNT(*) AS 人数 FROM sales WHERE age >= 25 AND age < 53 / 2",
|
||||
"SELECT COUNT(*) AS 人数 FROM sales WHERE age >= 已遮罩 AND age < 已遮罩 / 已遮罩",
|
||||
id="整数相除算出来的边界"),
|
||||
])
|
||||
async def test_a_short_masked_value_looked_up_in_other_shapes_is_sealed_masked(monkeypatch, tmp_path, client, sql,
|
||||
sealed):
|
||||
"""年龄设了遮罩，agent 换着写法找某一个年龄（套了 CAST、汇总值两头相同、补零、两头不是整数、范围再排除一头、
|
||||
算出来的边界，取模、整数相除算出来的边界）：第 2、3 轮复核时都照原样封存。现在查询清单、事件、质询工件里的 SQL
|
||||
都换掉这个年龄，三处一字不差。"""
|
||||
name, source_id = await _source(tmp_path, options={"mask_columns": "age"}, script=AGED)
|
||||
graph = chain(node("start", "input"),
|
||||
node("fetch", "tool", tool=f"db_query__{name}",
|
||||
args={"sql": "SELECT region AS 区域, age, amount AS 金额 FROM sales ORDER BY id"}),
|
||||
node("write", "report", "写周报", instructions="写周报"),
|
||||
node("gate", "human", mode="approve", title="周报可以出具吗"),
|
||||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]))
|
||||
plays = {"这个年龄": Play([sql], "这个年龄的有 [[v:Q1.r0.人数]] 人。[[see:Q1]]")}
|
||||
Models(monkeypatch, name, writer="华东首单金额 [[v:Q1.r0.金额]]。[[see:Q1]]", plays=plays)
|
||||
try:
|
||||
paused = await pause(name, graph=graph)
|
||||
[approval] = await pending(paused.id)
|
||||
r = await ask(client, paused.id, report="write", unit=None, question="这个年龄的客户有几位？",
|
||||
approval_id=approval.id)
|
||||
assert r.status_code == 200, r.text
|
||||
item = r.json()["inquiry"]
|
||||
assert (item["answer"]["ok"], item["answer"]["text"]) == (True, "这个年龄的有 1 人。")
|
||||
[query] = item["queries"]
|
||||
assert query["sql"] == sealed
|
||||
[event] = await events(paused.id, "approval.inquiry")
|
||||
assert event.data["queries"][0]["sql"] == sealed
|
||||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{query['artifact']}")
|
||||
assert got.status_code == 200 and got.json()["content"]["sql"] == sealed, "质询工件的 SQL 和事件里的一字不差"
|
||||
finally:
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
#: 会员等级设了遮罩：等级是 1、2、3 这样的短码值，正好撞上按条件数个数写的 THEN 1 ELSE 0
|
||||
LEVELS = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, vip_level INTEGER, amount INTEGER);"
|
||||
"INSERT INTO sales VALUES (1, '华东', 1, 1200), (2, '华东', 2, 300), (3, '华南', 3, 900),"
|
||||
" (4, '华北', 1, 1500);")
|
||||
|
||||
|
||||
async def test_a_renamed_masked_column_keeps_values_that_match_labels_written_elsewhere(monkeypatch, tmp_path, client):
|
||||
"""第 2 轮复核：先按等级数大单（vip_level AS 等级，SUM(CASE … THEN 1 ELSE 0 END)），再找等级为 1、2 的。以前分档标签
|
||||
1、0 不收进遮罩值，改了名字的等级列也跟着不收，WHERE vip_level = 1 照原样封存，= 2 却遮了。现在只有只装着标签的
|
||||
列不收，两条都遮；按等级数大单的那份快照里，等级整列遮，大单数照常显示。"""
|
||||
name, source_id = await _source(tmp_path, options={"mask_columns": "vip_level"}, script=LEVELS)
|
||||
by_level = ("SELECT vip_level AS 等级, SUM(CASE WHEN amount > 1000 THEN 1 ELSE 0 END) AS 大单数 FROM sales "
|
||||
"GROUP BY vip_level ORDER BY vip_level")
|
||||
first, second = (f"SELECT COUNT(*) AS 人数 FROM sales WHERE vip_level = {level}" for level in (1, 2))
|
||||
plays = {"各等级": Play([by_level, first, second],
|
||||
"大单最多的等级有 [[v:Q1.r0.大单数]] 笔大单；问到的两个等级各有 [[v:Q2.r0.人数]] 人、"
|
||||
"[[v:Q3.r0.人数]] 人。[[see:Q1]]")}
|
||||
Models(monkeypatch, name, writer="华东首单金额 [[v:Q1.r0.金额]]。[[see:Q1]]", plays=plays)
|
||||
try:
|
||||
paused = await pause(name, graph=fetching(name, AMOUNTS))
|
||||
[approval] = await pending(paused.id)
|
||||
r = await ask(client, paused.id, report="write", unit=None, question="各等级的大单多不多？",
|
||||
approval_id=approval.id)
|
||||
assert r.status_code == 200, r.text
|
||||
item = r.json()["inquiry"]
|
||||
assert (item["answer"]["ok"], item["answer"]["text"]) == (
|
||||
True, "大单最多的等级有 2 笔大单；问到的两个等级各有 2 人、1 人。")
|
||||
sealed = [by_level, *["SELECT COUNT(*) AS 人数 FROM sales WHERE vip_level = 已遮罩"] * 2]
|
||||
assert [q["sql"] for q in item["queries"]] == sealed
|
||||
[event] = await events(paused.id, "approval.inquiry")
|
||||
assert [q["sql"] for q in event.data["queries"]] == sealed
|
||||
for query, sql in zip(item["queries"], sealed):
|
||||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{query['artifact']}")
|
||||
assert got.status_code == 200 and got.json()["content"]["sql"] == sql, "质询工件的 SQL 和事件里的一字不差"
|
||||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{item['queries'][0]['artifact']}")
|
||||
assert got.json()["content"]["rows"] == [["已遮罩", 2], ["已遮罩", 0], ["已遮罩", 0]]
|
||||
assert got.json()["masked"] == ["等级"]
|
||||
finally:
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mixed", [
|
||||
pytest.param("SELECT CASE WHEN amount > 1000 THEN 1 ELSE 0 END AS 等级 FROM sales UNION ALL "
|
||||
"SELECT vip_level FROM sales", id="标签在前、遮罩列在后"),
|
||||
pytest.param("SELECT CASE WHEN amount > 1000 THEN 1 ELSE 0 END AS 等级 FROM sales UNION ALL "
|
||||
"SELECT b.vip_level AS 等级 FROM (SELECT vip_level FROM sales) b", id="遮罩列经子查询按原名传出"),
|
||||
])
|
||||
async def test_a_masked_column_after_a_label_in_a_union_keeps_its_values(monkeypatch, tmp_path, client, mixed):
|
||||
"""第 3 轮复核：UNION 的第一支是按条件数大单的标签（THEN 1 ELSE 0），另一支取的是等级（遮罩列原样选出，或者经按
|
||||
原名选出的子查询、以标签的别名传出）。以前这一列被当成只装着标签，等级 1 撞上标签 1 不收，之后 WHERE vip_level = 1
|
||||
在查询清单、事件、质询工件里都照原样，= 2 却遮了。现在这一列在一支里取的是遮罩列，就照收，两条都遮。"""
|
||||
name, source_id = await _source(tmp_path, options={"mask_columns": "vip_level"}, script=LEVELS)
|
||||
first, second = (f"SELECT COUNT(*) AS 人数 FROM sales WHERE vip_level = {level}" for level in (1, 2))
|
||||
plays = {"各等级": Play([mixed, first, second],
|
||||
"问到的两个等级各有 [[v:Q2.r0.人数]] 人、[[v:Q3.r0.人数]] 人。[[see:Q2]]")}
|
||||
Models(monkeypatch, name, writer="华东首单金额 [[v:Q1.r0.金额]]。[[see:Q1]]", plays=plays)
|
||||
try:
|
||||
paused = await pause(name, graph=fetching(name, AMOUNTS))
|
||||
[approval] = await pending(paused.id)
|
||||
r = await ask(client, paused.id, report="write", unit=None, question="各等级的客户有几位？",
|
||||
approval_id=approval.id)
|
||||
assert r.status_code == 200, r.text
|
||||
item = r.json()["inquiry"]
|
||||
assert (item["answer"]["ok"], item["answer"]["text"]) == (True, "问到的两个等级各有 2 人、1 人。")
|
||||
sealed = [mixed, *["SELECT COUNT(*) AS 人数 FROM sales WHERE vip_level = 已遮罩"] * 2]
|
||||
assert [q["sql"] for q in item["queries"]] == sealed
|
||||
[event] = await events(paused.id, "approval.inquiry")
|
||||
assert [q["sql"] for q in event.data["queries"]] == sealed
|
||||
for query, sql in zip(item["queries"], sealed):
|
||||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{query['artifact']}")
|
||||
assert got.status_code == 200 and got.json()["content"]["sql"] == sql, "质询工件的 SQL 和事件里的一字不差"
|
||||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{item['queries'][0]['artifact']}")
|
||||
assert {cell for [cell] in got.json()["content"]["rows"]} == {"已遮罩"} and got.json()["masked"] == ["等级"]
|
||||
finally:
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
STAR_TAIL = "SELECT *, SUBSTR(buyer_phone, 4) FROM sales ORDER BY amount DESC LIMIT 1"
|
||||
|
||||
|
||||
@@ -1116,6 +1295,53 @@ async def test_a_metric_from_the_report_that_equals_a_masked_value_can_be_cited(
|
||||
assert got.json()["content"]["rows"] == [["已遮罩"]] and got.json()["masked"] == ["金额"]
|
||||
|
||||
|
||||
#: 规格原句：按金额分档的文字标签，两个 WHEN 分支共用边界 20000
|
||||
BANDS = "CASE WHEN amount BETWEEN 0 AND 20000 THEN '小单' WHEN amount BETWEEN 20000 AND 60000 THEN '大单' END"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, answer, text, rows, masked", [
|
||||
# 档位当分组键（每一档一行）：标签是 agent 写在 SQL 里的字，不是遮罩值，格子照常显示，正文、SQL 里照原样
|
||||
pytest.param(f"SELECT {BANDS} AS 档位, COUNT(*) AS 笔数 FROM sales GROUP BY 1 ORDER BY 1",
|
||||
"以 20000 元为界，[[v:Q1.r0.档位]] [[v:Q1.r0.笔数]] 笔，[[v:Q1.r1.档位]] [[v:Q1.r1.笔数]] 笔，"
|
||||
"小单更多。[[see:Q1]]",
|
||||
"以 20000 元为界，大单 2 笔，小单 3 笔，小单更多。", [["大单", 2], ["小单", 3]], [], id="两个 WHEN 分支"),
|
||||
pytest.param("SELECT COUNT(CASE WHEN amount <= 20000 THEN 1 END) AS 小单, "
|
||||
"COUNT(CASE WHEN amount >= 20000 THEN 1 END) AS 大单 FROM sales",
|
||||
"不超过 20000 元的有 [[v:Q1.r0.小单]] 笔，不低于 20000 元的有 [[v:Q1.r0.大单]] 笔。[[see:Q1]]",
|
||||
"不超过 20000 元的有 3 笔，不低于 20000 元的有 3 笔。", [[3, 3]], [], id="两个选出列"),
|
||||
# 规格原句的 SUM(CASE …)：边界照原样封存、正文里不遮；汇总的是写死的 1 和 0，就是按条件数个数（同 COUNT），
|
||||
# 格子照常显示
|
||||
pytest.param("SELECT SUM(CASE WHEN amount <= 20000 THEN 1 ELSE 0 END) AS 小单, "
|
||||
"SUM(CASE WHEN amount >= 20000 THEN 1 ELSE 0 END) AS 大单 FROM sales",
|
||||
"以 20000 元为界：不超过的 [[v:Q1.r0.小单]] 笔，不低于的 [[v:Q1.r0.大单]] 笔。[[see:Q1]]",
|
||||
"以 20000 元为界：不超过的 3 笔，不低于的 3 笔。", [[3, 3]], [], id="两个 SUM(CASE…) 选出列"),
|
||||
pytest.param("SELECT '小单' AS 档位, COUNT(*) AS 笔数 FROM sales WHERE amount <= 20000 "
|
||||
"UNION ALL SELECT '大单', COUNT(*) FROM sales WHERE amount >= 20000",
|
||||
"不超过 20000 元的有 [[v:Q1.r0.笔数]] 笔，不低于 20000 元的有 [[v:Q1.r1.笔数]] 笔。[[see:Q1]]",
|
||||
"不超过 20000 元的有 3 笔，不低于 20000 元的有 3 笔。", [["小单", 3], ["大单", 3]], [], id="UNION 两支"),
|
||||
])
|
||||
async def test_band_edges_shared_by_separate_conditions_are_sealed_and_shown_as_written(
|
||||
monkeypatch, big_shop, client, sql, answer, text, rows, masked):
|
||||
"""金额设了遮罩，分档的边界 20000 正好是某一笔的金额：两个 WHEN 分支、两个选出列、UNION 两支各写一边，各是各的
|
||||
条件，没有夹住哪一笔。SQL 照原样封存，回答正文里复述的边界、档位标签也不遮，质询工件的 SQL 和事件里一字不差。"""
|
||||
plays = {"两万元上下": Play([sql], answer)}
|
||||
Models(monkeypatch, big_shop, writer="华东首单金额 [[v:Q1.r0.金额]]。[[see:Q1]]", plays=plays)
|
||||
paused = await pause(big_shop, graph=fetching(big_shop, AMOUNTS))
|
||||
[approval] = await pending(paused.id)
|
||||
r = await ask(client, paused.id, report="write", unit=None, question="两万元上下各有几笔？",
|
||||
approval_id=approval.id)
|
||||
assert r.status_code == 200, r.text
|
||||
item = r.json()["inquiry"]
|
||||
assert (item["answer"]["ok"], item["answer"]["text"]) == (True, text)
|
||||
[query] = item["queries"]
|
||||
assert query["sql"] == sql
|
||||
[event] = await events(paused.id, "approval.inquiry")
|
||||
assert event.data["queries"][0]["sql"] == sql
|
||||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{query['artifact']}")
|
||||
assert got.status_code == 200 and got.json()["content"]["sql"] == sql, "质询工件的 SQL 和事件里的一字不差"
|
||||
assert (got.json()["content"]["rows"], got.json()["masked"]) == (rows, masked)
|
||||
|
||||
|
||||
async def test_a_masked_value_the_asker_wrote_stays_in_the_sql(monkeypatch, tmp_path, client):
|
||||
"""提问人自己写的数恰好是遮罩列里的值（金额设了遮罩，问的就是 300 那一笔）：问题里原样写着，SQL 里和金额做等值
|
||||
比较也不换，事件、质询工件里封存的就是实际执行的那条。"""
|
||||
@@ -1170,6 +1396,114 @@ async def test_a_value_seen_only_in_the_reports_snapshot_is_masked_in_the_artifa
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
#: 另有一张回访记录：备注里写着手机号，备注这一列没设遮罩
|
||||
CALLBACKS = SALES + ("CREATE TABLE callbacks (id INTEGER PRIMARY KEY, region TEXT, note TEXT);"
|
||||
"INSERT INTO callbacks VALUES (1, '华东', '客户13900000001来电改址'), (2, '华南', '例行回访');")
|
||||
#: 回答中含有遮罩值、整份不交时给提问人的说法
|
||||
LEAKED = "回答中含有设了遮罩的值，无法遮住，已不予展示，请换个问法重新提问"
|
||||
|
||||
|
||||
async def test_a_masked_value_inside_a_text_cell_is_cut_out_of_the_artifact(monkeypatch, tmp_path, client):
|
||||
"""备注里写着手机号（备注没设遮罩，手机号设了遮罩、被报告取数选中）：agent 引用这一格，回答照旧整份不交；
|
||||
质询工件里这份快照照样记着、点得开，那一格里的号码换成「已遮罩」，别的字、别的格子照原样，不然「不予展示」
|
||||
旁边一点就能看到原值。"""
|
||||
name, source_id = await _source(tmp_path, options={"mask_columns": "buyer_phone"}, script=CALLBACKS)
|
||||
notes = "SELECT region AS 区域, note AS 备注 FROM callbacks ORDER BY id"
|
||||
plays = {"回访": Play([notes], "华东的回访备注是 [[v:Q1.r0.备注]]。[[see:Q1]]")}
|
||||
Models(monkeypatch, name, writer="金额最高的一笔 [[v:Q1.r0.金额]]。[[see:Q1]]", plays=plays)
|
||||
try:
|
||||
report_sql = "SELECT buyer_phone, amount AS 金额 FROM sales ORDER BY amount DESC"
|
||||
paused = await pause(name, graph=fetching(name, report_sql))
|
||||
[approval] = await pending(paused.id)
|
||||
r = await ask(client, paused.id, report="write", unit=None, question="华东的回访说了什么？",
|
||||
approval_id=approval.id)
|
||||
assert r.status_code == 200, r.text
|
||||
assert "1390000000" not in r.text
|
||||
item = r.json()["inquiry"]
|
||||
assert (item["answer"]["ok"], item["answer"]["text"], item["answer"]["doc_artifact"]) == (False, "", None)
|
||||
assert item["answer"]["error"] == LEAKED
|
||||
[query] = item["queries"]
|
||||
assert query["sql"] == notes
|
||||
[event] = await events(paused.id, "approval.inquiry")
|
||||
assert event.data["inquiry_artifacts"] == [query["artifact"]]
|
||||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{query['artifact']}")
|
||||
assert got.status_code == 200, got.text
|
||||
assert got.json()["content"]["rows"] == [["华东", "客户已遮罩来电改址"], ["华南", "例行回访"]]
|
||||
assert got.json()["masked"] == [] and "1390000000" not in got.text
|
||||
finally:
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
async def test_sql_check_excerpts_in_the_artifact_are_masked_like_the_sql(monkeypatch, masked_shop, client):
|
||||
"""agent 的查询带着 SQL 检查结果（对照数据目录查出的问题，片段摘自这条 SQL）：质询工件里片段照查询清单的 SQL
|
||||
换掉遮罩值，给模型的那句改法照抄了片段，不交出去（同证据面板）。"""
|
||||
from app.tools import datasource
|
||||
|
||||
excerpt = "buyer_phone = '13900000001'"
|
||||
|
||||
async def flagged(checker, sql):
|
||||
return [{"code": "wrong_date_column", "level": "warning", "table": "sales", "column": "buyer_phone",
|
||||
"message": "查询按「买家手机号」筛选", "for_model": f"这条 SQL 按 {excerpt} 筛选",
|
||||
"sql_excerpt": excerpt}] if excerpt in sql else []
|
||||
|
||||
monkeypatch.setattr(datasource, "_sql_checks", flagged)
|
||||
plays = {"买得最多": Play([TOP_BUYER, BUYER_ORDERS], "这位客户共 [[v:Q2.r0.笔数]] 笔。[[see:Q2]]")}
|
||||
Models(monkeypatch, masked_shop, plays=plays)
|
||||
paused = await pause(masked_shop)
|
||||
[approval] = await pending(paused.id)
|
||||
r = await ask(client, paused.id, report="write", unit=None, question="买得最多的客户买过几笔？",
|
||||
approval_id=approval.id)
|
||||
assert r.status_code == 200, r.text
|
||||
_, orders = r.json()["inquiry"]["queries"]
|
||||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{orders['artifact']}")
|
||||
assert got.status_code == 200, got.text
|
||||
assert got.json()["content"]["checks"] == [
|
||||
{"code": "wrong_date_column", "level": "warning", "table": "sales", "column": "buyer_phone",
|
||||
"message": "查询按「买家手机号」筛选", "sql_excerpt": "buyer_phone = '已遮罩'"}]
|
||||
assert got.json()["content"]["sql"] == orders["sql"] and "1390000000" not in got.text
|
||||
|
||||
|
||||
async def test_sql_check_results_in_the_artifact_do_not_show_short_masked_values(monkeypatch, tmp_path, client):
|
||||
"""检查结果的片段是检查器按语法树重新写出的一段表达式，单独拿出来认不出在找哪一个值；说明里还照引 SQL 写的取值
|
||||
（「没有码值「25」」）。年龄设了遮罩：整条 SQL 里换掉的 25，片段、说明里一样换掉，码值表里列出的照原样。"""
|
||||
from app.tools import datasource
|
||||
|
||||
name, source_id = await _source(tmp_path, options={"mask_columns": "age"}, script=AGED)
|
||||
graph = chain(node("start", "input"),
|
||||
node("fetch", "tool", tool=f"db_query__{name}",
|
||||
args={"sql": "SELECT region AS 区域, age, amount AS 金额 FROM sales ORDER BY id"}),
|
||||
node("write", "report", "写周报", instructions="写周报"),
|
||||
node("gate", "human", mode="approve", title="周报可以出具吗"),
|
||||
node("out", "output", fields=[{"name": "周报", "value": "{{ nodes.write.text }}"}]))
|
||||
pinned = "SELECT COUNT(*) AS 人数 FROM sales WHERE age BETWEEN 25 AND 25"
|
||||
listed = "（数据目录中已列出的全部码值：18=成年、60=退休）。请核对筛选条件里的取值"
|
||||
|
||||
async def flagged(checker, sql):
|
||||
return [{"code": "unknown_code", "level": "warning", "table": "sales", "column": "age",
|
||||
"message": f"「订单」的「年龄」没有码值「25」{listed}", "for_model": "SQL 里写的 25 不在其中",
|
||||
"sql_excerpt": "age BETWEEN 25 AND 25"}] if "BETWEEN" in sql else []
|
||||
|
||||
monkeypatch.setattr(datasource, "_sql_checks", flagged)
|
||||
plays = {"这个年龄": Play([pinned], "这个年龄的有 [[v:Q1.r0.人数]] 人。[[see:Q1]]")}
|
||||
Models(monkeypatch, name, writer="华东首单金额 [[v:Q1.r0.金额]]。[[see:Q1]]", plays=plays)
|
||||
try:
|
||||
paused = await pause(name, graph=graph)
|
||||
[approval] = await pending(paused.id)
|
||||
r = await ask(client, paused.id, report="write", unit=None, question="这个年龄的客户有几位？",
|
||||
approval_id=approval.id)
|
||||
assert r.status_code == 200, r.text
|
||||
[query] = r.json()["inquiry"]["queries"]
|
||||
assert query["sql"] == "SELECT COUNT(*) AS 人数 FROM sales WHERE age BETWEEN 已遮罩 AND 已遮罩"
|
||||
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{query['artifact']}")
|
||||
assert got.status_code == 200, got.text
|
||||
assert got.json()["content"]["checks"] == [
|
||||
{"code": "unknown_code", "level": "warning", "table": "sales", "column": "age",
|
||||
"message": f"「订单」的「年龄」没有码值「已遮罩」{listed}", "sql_excerpt": "age BETWEEN 已遮罩 AND 已遮罩"}]
|
||||
assert got.json()["content"]["sql"] == query["sql"]
|
||||
finally:
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
#: 同样的四笔订单，手机号存成整数：遮罩值在 SQL 里写成数字
|
||||
PHONES = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_phone INTEGER, amount INTEGER, is_test INTEGER);"
|
||||
"INSERT INTO sales VALUES (1, '华东', 13900000001, 1200, 0), (2, '华东', 13900000002, 300, 1),"
|
||||
@@ -1180,7 +1514,7 @@ _COUNT = "SELECT COUNT(*) AS 笔数 FROM sales WHERE "
|
||||
|
||||
|
||||
@pytest.mark.parametrize("script, sql, sealed", [
|
||||
# 数个数用 COUNT：SUM(CASE buyer_phone …) 的结果取自遮罩列，整列按列遮
|
||||
# 简单 CASE 里找某一个号码，只数个数（COUNT，或者 SUM 套写死的 1）：号码换掉，笔数照常显示
|
||||
pytest.param(PHONES, "SELECT COUNT(CASE buyer_phone WHEN 13900000001 THEN 1 END) AS 笔数 FROM sales",
|
||||
"SELECT COUNT(CASE buyer_phone WHEN 已遮罩 THEN 1 END) AS 笔数 FROM sales", id="简单 CASE"),
|
||||
pytest.param(PHONES, _COUNT + "buyer_phone IN (VALUES (13900000001))", _COUNT + "buyer_phone IN (VALUES (已遮罩))",
|
||||
@@ -1193,6 +1527,11 @@ _COUNT = "SELECT COUNT(*) AS 笔数 FROM sales WHERE "
|
||||
_COUNT + "NOT (buyer_phone > 已遮罩 OR buyer_phone < 已遮罩)", id="既不大于也不小于"),
|
||||
pytest.param(PHONES, _COUNT + "buyer_phone >= 13900000001 AND buyer_phone < 13900000002",
|
||||
_COUNT + "buyer_phone >= 已遮罩 AND buyer_phone < 已遮罩", id="整数的半开区间"),
|
||||
# 两头都不是那个号码，夹住的是中间那一个：留着 13900000000 就等于写出了号码
|
||||
pytest.param(PHONES, _COUNT + "buyer_phone > 13900000000 AND buyer_phone < 13900000002",
|
||||
_COUNT + "buyer_phone > 已遮罩 AND buyer_phone < 已遮罩", id="整数的开区间"),
|
||||
pytest.param(PHONES, _COUNT + "region = '华东' AND buyer_phone >= 13900000001 AND buyer_phone <= 13900000001",
|
||||
_COUNT + "region = '华东' AND buyer_phone >= 已遮罩 AND buyer_phone <= 已遮罩", id="同一条件里还有别的条件"),
|
||||
pytest.param(PHONES, _COUNT + "(buyer_phone, region) = (13900000001, '华东')",
|
||||
_COUNT + "(buyer_phone, region) = (已遮罩, '华东')", id="行值"),
|
||||
pytest.param(PHONES, _COUNT + "buyer_phone IS NOT DISTINCT FROM 13900000001",
|
||||
@@ -1204,6 +1543,41 @@ _COUNT = "SELECT COUNT(*) AS 笔数 FROM sales WHERE "
|
||||
_COUNT + "buyer_phone >= '已遮罩' AND buyer_phone <= '已遮罩'", id="字符串范围两头相同"),
|
||||
pytest.param(SALES, _COUNT + "buyer_phone BETWEEN '13900000001' AND '13900000001'",
|
||||
_COUNT + "buyer_phone BETWEEN '已遮罩' AND '已遮罩'", id="字符串 BETWEEN 两头相同"),
|
||||
# 两半拆到同一行集的不同地方：第 4 轮只在同一个条件里认，号码照原样封存，回答正文里也露了出来
|
||||
pytest.param(PHONES, "SELECT COUNT(*) AS 笔数 FROM sales s JOIN sales b ON b.id = s.id "
|
||||
"AND b.buyer_phone >= 13900000001 WHERE b.buyer_phone <= 13900000001",
|
||||
"SELECT COUNT(*) AS 笔数 FROM sales s JOIN sales b ON b.id = s.id "
|
||||
"AND b.buyer_phone >= 已遮罩 WHERE b.buyer_phone <= 已遮罩", id="ON 和 WHERE 各写一边"),
|
||||
pytest.param(PHONES, _COUNT + "buyer_phone >= 13900000001 GROUP BY buyer_phone HAVING buyer_phone <= 13900000001",
|
||||
_COUNT + "buyer_phone >= 已遮罩 GROUP BY buyer_phone HAVING buyer_phone <= 已遮罩",
|
||||
id="WHERE 和 HAVING 各写一边"),
|
||||
pytest.param(PHONES, "SELECT COUNT(*) AS 笔数 FROM (SELECT id FROM sales WHERE buyer_phone >= 13900000001 "
|
||||
"INTERSECT SELECT id FROM sales WHERE buyer_phone <= 13900000001) t",
|
||||
"SELECT COUNT(*) AS 笔数 FROM (SELECT id FROM sales WHERE buyer_phone >= 已遮罩 "
|
||||
"INTERSECT SELECT id FROM sales WHERE buyer_phone <= 已遮罩) t", id="INTERSECT 两支"),
|
||||
pytest.param(PHONES, _COUNT + "buyer_phone >= 13900000001 AND CASE WHEN buyer_phone <= 13900000001 THEN 1 "
|
||||
"ELSE 0 END = 1",
|
||||
_COUNT + "buyer_phone >= 已遮罩 AND CASE WHEN buyer_phone <= 已遮罩 THEN 1 ELSE 0 END = 1",
|
||||
id="一半包在 CASE 里"),
|
||||
pytest.param(PHONES, "WITH a AS (SELECT * FROM sales WHERE buyer_phone >= 13900000001) "
|
||||
"SELECT COUNT(*) AS 笔数 FROM a WHERE buyer_phone <= 13900000001",
|
||||
"WITH a AS (SELECT * FROM sales WHERE buyer_phone >= 已遮罩) "
|
||||
"SELECT COUNT(*) AS 笔数 FROM a WHERE buyer_phone <= 已遮罩", id="拆进 CTE"),
|
||||
# 整数上相邻的两头把中间那一个隔开
|
||||
pytest.param(PHONES, _COUNT + "NOT (buyer_phone <= 13900000000 OR buyer_phone >= 13900000002)",
|
||||
_COUNT + "NOT (buyer_phone <= 已遮罩 OR buyer_phone >= 已遮罩)", id="相邻的整数隔开"),
|
||||
# 第 2 轮复核：一头是算式、等值的值是算式、两头不是整数、范围再用不等去掉一头、前面补零的号码。以前都照原样
|
||||
# 封存，后两种的号码还被当成阈值，回答正文跟着不遮
|
||||
pytest.param(PHONES, _COUNT + "buyer_phone BETWEEN 13900000000 + 1 AND 13900000009 - 8",
|
||||
_COUNT + "buyer_phone BETWEEN 已遮罩 + 已遮罩 AND 已遮罩 - 已遮罩", id="算出来的边界"),
|
||||
pytest.param(PHONES, _COUNT + "buyer_phone = 13900000000 + 1", _COUNT + "buyer_phone = 已遮罩 + 已遮罩",
|
||||
id="算出来的等值"),
|
||||
pytest.param(PHONES, _COUNT + "buyer_phone > 13900000000.5 AND buyer_phone < 13900000001.5",
|
||||
_COUNT + "buyer_phone > 已遮罩 AND buyer_phone < 已遮罩", id="两头不是整数"),
|
||||
pytest.param(PHONES, _COUNT + "buyer_phone >= 13900000001 AND buyer_phone <= 13900000002 "
|
||||
"AND buyer_phone <> 13900000002",
|
||||
_COUNT + "buyer_phone >= 已遮罩 AND buyer_phone <= 已遮罩 AND buyer_phone <> 已遮罩", id="范围再排除一头"),
|
||||
pytest.param(PHONES, _COUNT + "buyer_phone = 013900000001", _COUNT + "buyer_phone = 已遮罩", id="前面补零的号码"),
|
||||
])
|
||||
async def test_a_customer_looked_up_in_any_shape_never_shows(monkeypatch, tmp_path, client, script, sql, sealed):
|
||||
"""agent 换着写法找某一个客户（简单 CASE、IN (VALUES …)、两头相同的范围、行值、选出列里写着同一个号码……），回答里
|
||||
|
||||
@@ -8,11 +8,30 @@
|
||||
格子里的同一个数照旧换
|
||||
- SQL 里数字照原样留着的只有范围比较的边界（没把某一个值夹住的）和 LIMIT / OFFSET 的行数；在找某一个值的（等值、IN、
|
||||
LIKE、简单 CASE、IS [NOT] DISTINCT FROM，套了括号、写成行值、VALUES、ARRAY 的也算，还有 BETWEEN v AND v、
|
||||
>= v AND <= v 这类夹住某一个值的范围）和遮罩列比较时短的数也换；选出列、函数参数、算式里够长的遮罩值也换；字符串
|
||||
（范围边界也一样）、注释、名字按值换；SQL 解析不了时认不准哪个是阈值，一律不免
|
||||
- 快照按列遮，另按值遮：别的列里写着遮罩值的格子也换成「已遮罩」；列名里写着遮罩值的，列名也换
|
||||
>= v AND <= v 这类夹住某一个值的范围）和遮罩列比较时短的数也换，被比较的一边套了 CAST、汇总也算；同一个数换个
|
||||
写法（25.0、025、2.5e1）、写成几个数的算式（25 + 0、13900000000 + 1、125 % 100、50 DIV 2，整数相除照数据源的算法）
|
||||
照样认，算式里的数都换（CAST 的类型参数不算）；选出列、函数参数、算式里够长的遮罩值也换；字符串（范围边界也一样）、
|
||||
注释、名字按值换；SQL 解析不了时认不准哪个是阈值，一律不免
|
||||
- 「夹住某一个值」在整条 SQL 里认，只有分档的几种写法各算各的：CASE 的不同分支、不同的选出列（连同 GROUP BY、
|
||||
ORDER BY）、UNION 的不同支，共用的边界是阈值；同一行集上的条件（ON 和 WHERE、WHERE 和 HAVING、INTERSECT 两支、
|
||||
包在 CASE / 函数 / 算式里的一半、子查询、CTE）照样认得出在找哪一个值。同一个条件里按合起来的真假认（旁边的别的
|
||||
条件当成不知道真假），BETWEEN v AND v、>= v AND <= v 这类本身就只认一个值的，条件里哪一层都算；整数上 <= v、
|
||||
>= v 也隔开了相邻的整数，两头不是整数的（age > 24.5 AND age < 25.5）也夹住中间那一个整数；范围再用不等、NOT IN
|
||||
去掉一头的（age >= 25 AND age <= 26 AND age <> 26）也认，不等、IN 只在同一个条件里和范围合起来看；列名认不出的
|
||||
只和写得一样的表达式配
|
||||
- 阈值只在同一个数在这条 SQL 别处确实被换掉时才不留（换不换和 SQL 里换字面量是同一套判断）：日期、天数、标签里写着
|
||||
同一个短的数，没设遮罩的列上的等值，都不牵连阈值；不留的照等值的办法换，别的列上的短阈值、LIMIT 撞上别处换掉的
|
||||
短的数照原样（已知局限）
|
||||
- 按遮罩列分档的标签（CASE、IIF 的结果都是写死的常量）是 agent 自己写在 SQL 里的，不当遮罩值：SQL、回答正文里照原样；
|
||||
当分组键的（每一档一行）不算派生值，格子照常显示，逐行的标签列照旧整列遮。只有只装着标签的列不收（按作用域认取自
|
||||
哪个来源：经别名、子查询、CTE 传下去、没起别名的也算），改了名字的遮罩列撞上同一条 SQL 里的 THEN 1 ELSE 0 照收，
|
||||
UNION 另一支、子查询里的遮罩列流进标签列也照收
|
||||
- 快照按列遮，另按值遮：别的列里写着遮罩值的格子也换成「已遮罩」；列名里写着遮罩值的，列名也换；质询工件里，
|
||||
没设遮罩的列里一格文字夹着的遮罩值按子串换（回答一路照旧整份不交），SQL 检查结果的片段、说明照整条 SQL 换（截断的
|
||||
片段从没闭合的引号、注释起去掉，说明里的码值表和别的字一样按值换）
|
||||
- 组装回答之后还认得出遮罩值的（正文里复述的阈值除外），不交出去
|
||||
- 遮罩值上万个时换一遍也不卡：不现编上万个分支的正则
|
||||
- 遮罩值上万个时换一遍也不卡：不现编上万个分支的正则；上千个条件连成一长串、上千项的连加照样按结构认，成百个分档
|
||||
边界看不过来的按夹住算，也不卡
|
||||
|
||||
数据都是虚构的：手机号、姓名、年龄、区域、金额。
|
||||
"""
|
||||
@@ -124,11 +143,23 @@ def test_a_lateral_subquery_is_read_like_a_derived_table():
|
||||
("SELECT * FROM (SELECT region, COUNT(DISTINCT buyer_phone) FROM sales GROUP BY region) t",
|
||||
["region", "COUNT(DISTINCT buyer_phone)"]),
|
||||
("SELECT region FROM sales WHERE amount > (SELECT AVG(LENGTH(buyer_phone)) FROM sales)", ["region"]),
|
||||
# 汇总里套着结果全是写死的 CASE / IIF：按条件数个数、算占比，同 COUNT
|
||||
("SELECT SUM(CASE WHEN buyer_phone LIKE '139%' THEN 1 ELSE 0 END) AS 笔数 FROM sales", ["笔数"]),
|
||||
("SELECT region, AVG(IIF(buyer_phone LIKE '139%', 1.0, 0)) AS 占比 FROM sales GROUP BY region", ["region", "占比"]),
|
||||
])
|
||||
def test_counting_filtering_and_ordering_by_a_masked_column_shows_nothing(sql, columns):
|
||||
assert hidden_of(sql, columns) == {"buyer_phone"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, columns", [
|
||||
# 汇总的是遮罩列的值本身（已知局限：SUM / AVG / MIN / MAX 算派生），CASE 里有一支取的是它也一样
|
||||
("SELECT region, MAX(buyer_phone) AS 最大号码 FROM sales GROUP BY region", ["region", "最大号码"]),
|
||||
("SELECT SUM(CASE WHEN region = '华东' THEN buyer_phone END) AS 合计 FROM sales", ["合计"]),
|
||||
])
|
||||
def test_aggregating_the_masked_values_themselves_is_still_derived(sql, columns):
|
||||
assert hidden_of(sql, columns) >= {"buyer_phone", columns[-1].lower()}
|
||||
|
||||
|
||||
def test_sql_that_cannot_be_read_but_names_a_masked_column_hides_everything():
|
||||
assert hidden_of("SELECT buyer_phone AS 号码 FROM sales WHERE (", ["号码", "金额"]) == {"buyer_phone", "号码", "金额"}
|
||||
assert hidden_of("SELECT region FROM sales WHERE (", ["region"]) == {"buyer_phone"}
|
||||
@@ -140,6 +171,184 @@ def test_names_that_point_at_a_masked_column_are_known_for_comparisons():
|
||||
assert inquiry_mod._derived_hidden(snapshot, {"buyer_phone"}, "sqlite")[1] == {"buyer_phone", "p"}
|
||||
|
||||
|
||||
BANDS = "CASE WHEN amount BETWEEN 0 AND 20000 THEN '小单' WHEN amount BETWEEN 20000 AND 60000 THEN '大单' END"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql", [
|
||||
# 按位置、按别名、照原样再写一遍分组
|
||||
f"SELECT {BANDS} AS 档位, COUNT(*) AS 笔数 FROM sales GROUP BY 1",
|
||||
f"SELECT {BANDS} AS 档位, COUNT(*) AS 笔数 FROM sales GROUP BY 档位",
|
||||
f"SELECT {BANDS} AS 档位, COUNT(*) AS 笔数 FROM sales GROUP BY {BANDS}",
|
||||
# 档位编号、没写 ELSE、ELSE NULL、IIF 的两个分支也是写死的
|
||||
"SELECT CASE WHEN amount <= 20000 THEN 1 WHEN amount <= 60000 THEN 2 ELSE NULL END AS 档位, COUNT(*) AS 笔数 "
|
||||
"FROM sales GROUP BY 1",
|
||||
"SELECT IIF(amount > 20000, '大单', '小单') AS 档位, COUNT(*) AS 笔数 FROM sales GROUP BY 1",
|
||||
])
|
||||
def test_a_band_label_used_as_the_group_key_is_not_a_masked_value(sql):
|
||||
"""按遮罩列分档、每一档一行（CASE、IIF 的结果全是写死的标签，又是分组键）：标签只说一行落在哪一档，带不出金额，
|
||||
档位列不遮，才分得清哪一档是几笔。笔数照样只是计数。"""
|
||||
assert hidden_of(sql, ["档位", "笔数"], {"amount"}) == {"amount"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql", [
|
||||
# 不分组：每一行的档位就是这一笔落在哪一档
|
||||
f"SELECT id, {BANDS} AS 档位, 1 AS 笔数 FROM sales",
|
||||
# 结果里有一支取的是金额本身
|
||||
"SELECT CASE WHEN amount > 20000 THEN amount ELSE 0 END AS 档位, COUNT(*) AS 笔数 FROM sales GROUP BY 1",
|
||||
# 分档写在子查询里，外层再分组：子查询里的档位逐行带出来，照旧算
|
||||
f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT {BANDS} AS 档位 FROM sales) t GROUP BY 档位",
|
||||
])
|
||||
def test_band_labels_row_by_row_still_count_as_derived(sql):
|
||||
assert hidden_of(sql, ["档位", "笔数"], {"amount"}) >= {"amount", "档位"}
|
||||
|
||||
|
||||
def plan_of(snapshot: dict, base: set[str], kind: str = "sqlite") -> inquiry_mod._Plan:
|
||||
"""一份 agent 的查询快照按回答时的办法收遮罩值、遮好（数据源设了遮罩的列是 base）。"""
|
||||
return inquiry_mod._settle_plan({"a1": snapshot}, {"a1": (base, [])}, [], {"shop": kind}, [], [], {},
|
||||
frozenset())
|
||||
|
||||
|
||||
def test_band_labels_written_in_the_sql_are_not_masked_values():
|
||||
"""分档的标签是 agent 自己写在 SQL 里的字，不是数据源里的值：当分组键的整列照常显示；逐行的标签列照旧整列遮
|
||||
（格子里看不出这一笔落在哪一档），但标签文字也不当遮罩值——SQL 里、回答正文里照原样，换掉了封存的就不是实际
|
||||
执行的语句，「以 20000 元为界，小单 3 笔」也会变成满屏「已遮罩」。"""
|
||||
grouped = {"source": "shop", "sql": f"SELECT {BANDS} AS 档位, COUNT(*) AS 笔数 FROM sales GROUP BY 1 ORDER BY 1",
|
||||
"columns": ["档位", "笔数"], "rows": [["大单", 2], ["小单", 3]]}
|
||||
plan = plan_of(grouped, {"amount"})
|
||||
assert plan.views["a1"]["rows"] == [["大单", 2], ["小单", 3]] and plan.masked["a1"] == []
|
||||
assert plan.secrets.scrub("以 20000 元为界，小单 3 笔、大单 2 笔") == "以 20000 元为界，小单 3 笔、大单 2 笔"
|
||||
assert inquiry_mod._scrub_sql(grouped["sql"], plan.secrets, plan.tainted["a1"], "sqlite") == grouped["sql"]
|
||||
|
||||
rows = {"source": "shop", "sql": "SELECT id, CASE WHEN age < 30 THEN '30岁以下' ELSE '30岁及以上' END AS 年龄段 "
|
||||
"FROM sales ORDER BY id",
|
||||
"columns": ["id", "年龄段"], "rows": [[1, "30岁以下"], [2, "30岁及以上"]]}
|
||||
plan = plan_of(rows, {"age"})
|
||||
assert plan.views["a1"]["rows"] == [[1, MASKED], [2, MASKED]] and plan.masked["a1"] == ["年龄段"]
|
||||
assert plan.secrets.scrub("30岁以下的有 1 人") == "30岁以下的有 1 人"
|
||||
assert inquiry_mod._scrub_sql(rows["sql"], plan.secrets, plan.tainted["a1"], "sqlite") == rows["sql"]
|
||||
assert inquiry_mod._shown_literals([(rows["sql"], "sqlite", plan.tainted["a1"])], plan.secrets) == {"30"}
|
||||
|
||||
|
||||
def test_a_value_of_the_masked_column_itself_stays_masked_even_if_written_as_a_label():
|
||||
"""只有派生列里的标签不收：遮罩列自己选出来的值照收，哪怕 agent 在 SQL 里把同样的字写成了标签。"""
|
||||
snapshot = {"source": "shop", "sql": "SELECT level, CASE WHEN level = 'VIP会员' THEN 'VIP会员' ELSE '普通会员' END "
|
||||
"AS 等级 FROM customers",
|
||||
"columns": ["level", "等级"], "rows": [["VIP会员", "VIP会员"], ["铂金会员", "普通会员"]]}
|
||||
plan = plan_of(snapshot, {"level"})
|
||||
assert plan.views["a1"]["rows"] == [[MASKED, MASKED], [MASKED, MASKED]]
|
||||
assert plan.secrets.scrub("VIP会员、铂金会员、普通会员") == "已遮罩、已遮罩、普通会员"
|
||||
|
||||
|
||||
#: 按会员等级数大单：等级设了遮罩、改了名字，汇总里按条件数个数写着 THEN 1 ELSE 0
|
||||
LEVEL_COUNTS = ("SELECT vip_level AS 等级, SUM(CASE WHEN amount > 1000 THEN 1 ELSE 0 END) AS 大单数 FROM sales "
|
||||
"GROUP BY vip_level")
|
||||
|
||||
|
||||
def test_a_renamed_masked_column_keeps_values_that_match_labels_written_elsewhere():
|
||||
"""第 2 轮复核：分档标签不收进遮罩值，以前对改了名字的遮罩列也一样——vip_level AS 等级 的 1、2、3 撞上同一条 SQL
|
||||
里按条件数个数的 THEN 1 ELSE 0，1 就不算遮罩值，之后 WHERE vip_level = 1 照原样封存。现在只跳过只装着标签的
|
||||
那几列的格子，改了名字的遮罩列照收。"""
|
||||
snapshot = {"source": "shop", "sql": LEVEL_COUNTS, "columns": ["等级", "大单数"],
|
||||
"rows": [[1, 2], [2, 0], [3, 0]]}
|
||||
plan = plan_of(snapshot, {"vip_level"})
|
||||
assert plan.views["a1"]["rows"] == [[MASKED, 2], [MASKED, 0], [MASKED, 0]] and plan.masked["a1"] == ["等级"]
|
||||
assert {"1", "2", "3"} <= plan.secrets.weak
|
||||
for level in (1, 2, 3):
|
||||
assert inquiry_mod._scrub_sql(f"SELECT COUNT(*) AS 人数 FROM sales WHERE vip_level = {level}", plan.secrets,
|
||||
plan.tainted["a1"], "sqlite") == (
|
||||
"SELECT COUNT(*) AS 人数 FROM sales WHERE vip_level = 已遮罩")
|
||||
# 同一个名字既装着标签又装着遮罩列的值（UNION 两支）：不算只装着标签的列，照收
|
||||
mixed = {"source": "shop", "sql": "SELECT vip_level AS 等级 FROM sales UNION ALL "
|
||||
"SELECT CASE WHEN amount > 1000 THEN 1 ELSE 0 END FROM sales",
|
||||
"columns": ["等级"], "rows": [[1], [3], [0]]}
|
||||
assert {"0", "1", "3"} <= plan_of(mixed, {"vip_level"}).secrets.weak
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, columns, rows", [
|
||||
# 标签列经子查询、CTE 原样传下去（外层再分组也一样：每一行的档位照旧算派生值，见上）
|
||||
(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT {BANDS} AS 档位 FROM sales) t GROUP BY 档位",
|
||||
["档位", "笔数"], [["大单", 2], ["小单", 3]]),
|
||||
(f"WITH t AS (SELECT {BANDS} AS 档位 FROM sales) SELECT * FROM t", ["档位"], [["小单"], ["大单"]]),
|
||||
# 没起别名的标签列：按位置认
|
||||
(f"SELECT id, {BANDS} FROM sales", ["id", BANDS], [[1, "小单"], [2, "大单"]]),
|
||||
])
|
||||
def test_labels_passed_on_under_their_own_name_are_still_not_masked_values(sql, columns, rows):
|
||||
"""只装着标签的列经别名、子查询、CTE 原样传下去，或者没起别名：照旧整列遮（逐行的档位），标签文字照旧不当遮罩值。"""
|
||||
plan = plan_of({"source": "shop", "sql": sql, "columns": columns, "rows": rows}, {"amount"})
|
||||
label = columns.index("档位") if "档位" in columns else 1
|
||||
assert all(row[label] == MASKED for row in plan.views["a1"]["rows"])
|
||||
assert plan.secrets.scrub("小单 3 笔、大单 2 笔") == "小单 3 笔、大单 2 笔"
|
||||
|
||||
|
||||
#: 按条件数大单写的 1、0：和会员等级（1、2、3）撞在一起
|
||||
BIG_ONE = "CASE WHEN amount > 1000 THEN 1 ELSE 0 END"
|
||||
#: 标签那一支的 1、0 在前，等级 1、2、3 在后
|
||||
LEVEL_ROWS = [[1], [0], [0], [1], [1], [2], [3], [1]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, columns, rows", [
|
||||
# UNION 标签在前、遮罩列在后：遮罩列没起别名，或者起了别的名字
|
||||
(f"SELECT {BIG_ONE} AS 等级 FROM sales UNION ALL SELECT vip_level FROM sales", ["等级"], LEVEL_ROWS),
|
||||
(f"SELECT {BIG_ONE} AS 等级 FROM sales UNION ALL SELECT vip_level AS lv FROM sales", ["等级"], LEVEL_ROWS),
|
||||
# 子查询里的 UNION，外层按名字取；写了列名表的 CTE 里的 UNION
|
||||
(f"SELECT 等级, COUNT(*) AS 人数 FROM (SELECT {BIG_ONE} AS 等级 FROM sales UNION ALL SELECT vip_level FROM sales) t "
|
||||
"GROUP BY 等级", ["等级", "人数"], [[0, 2], [1, 4], [2, 1], [3, 1]]),
|
||||
(f"WITH t(lv) AS (SELECT {BIG_ONE} FROM sales UNION ALL SELECT vip_level FROM sales) SELECT lv AS 等级 FROM t",
|
||||
["等级"], LEVEL_ROWS),
|
||||
# 标签写在 CTE 里，结果的 UNION 另一支取遮罩列
|
||||
(f"WITH b AS (SELECT {BIG_ONE} AS lvl FROM sales) SELECT lvl FROM b UNION ALL SELECT vip_level FROM sales",
|
||||
["lvl"], LEVEL_ROWS),
|
||||
# 遮罩列经按原名选出的子查询、以标签的别名传出
|
||||
(f"SELECT {BIG_ONE} AS 档 FROM sales UNION ALL SELECT b.vip_level AS 档 FROM (SELECT vip_level FROM sales) b",
|
||||
["档"], LEVEL_ROWS),
|
||||
# 不用 UNION：两个子查询连起来，一边是标签，一边是改了名字的遮罩列，名字相同
|
||||
(f"SELECT y.档 AS 档 FROM (SELECT id, {BIG_ONE} AS 档 FROM sales) x "
|
||||
"JOIN (SELECT id, vip_level AS 档 FROM (SELECT id, vip_level FROM sales) z) y ON x.id = y.id", ["档"],
|
||||
[[1], [2], [3], [1]]),
|
||||
# USING 连起来的同名列：右连接时，左边没对上的行取的是右边那张表里的值
|
||||
(f"WITH b AS (SELECT id, {BIG_ONE} AS vip_level FROM sales) SELECT vip_level AS 等级 FROM b "
|
||||
"RIGHT JOIN members USING (vip_level)", ["等级"], [[1], [2], [3]]),
|
||||
])
|
||||
def test_a_column_that_also_carries_the_masked_column_is_not_a_label_column(sql, columns, rows):
|
||||
"""第 3 轮复核：只装着标签的列以前按名字认，从「所有名字都只装着标签」起往下剔，遮罩列原样选出（SELECT vip_level）
|
||||
时名字就是它自己，永远剔不掉；结果的 UNION 按位置那一遍只往里加标签，另一支取的是遮罩列也不剔，派生来源里的
|
||||
UNION 根本不按位置看。于是标签在前、遮罩列在后时，等级 1 撞上 THEN 1 就不收，之后 WHERE vip_level = 1 照原样
|
||||
封存。现在按作用域认每一项取自哪个来源，结果的这一列在每一支里都只装着标签才算，取自数据源里的表的一律不算。"""
|
||||
plan = plan_of({"source": "shop", "sql": sql, "columns": columns, "rows": rows}, {"vip_level"})
|
||||
assert inquiry_mod._label_columns(sql, "sqlite", columns) == {}
|
||||
assert {"1", "2", "3"} <= plan.secrets.weak
|
||||
for level in (1, 2, 3):
|
||||
assert inquiry_mod._scrub_sql(f"SELECT COUNT(*) AS 人数 FROM sales WHERE vip_level = {level}", plan.secrets,
|
||||
plan.tainted["a1"], "sqlite") == (
|
||||
"SELECT COUNT(*) AS 人数 FROM sales WHERE vip_level = 已遮罩")
|
||||
|
||||
|
||||
_EXTRA = "CASE WHEN amount > 60000 THEN '特大单' END"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, columns, texts", [
|
||||
# CTE 里分档，外层连上数据源里的表、按档位汇总：档位没写表名也认得出取自 CTE（SQL 能执行，别的表就没有同名的列，
|
||||
# 不然会报列名有歧义）
|
||||
(f"WITH banded AS (SELECT id, {BANDS} AS 档位 FROM sales) SELECT 档位, SUM(s.visits) AS 访问 FROM banded b "
|
||||
"JOIN sales s ON s.id = b.id GROUP BY 档位", ["档位", "访问"], {"小单", "大单"}),
|
||||
# 按别的列连起来（USING (id)）：档位没合成一列，照样取自 CTE
|
||||
(f"WITH banded AS (SELECT id, {BANDS} AS 档位 FROM sales) SELECT 档位, SUM(o.amount) AS 金额 FROM banded "
|
||||
"JOIN orders o USING (id) GROUP BY 档位", ["档位", "金额"], {"小单", "大单"}),
|
||||
# 写了表名、经两层子查询传下来
|
||||
(f"SELECT t.档位 FROM (SELECT b.档位 FROM (SELECT {BANDS} AS 档位 FROM sales) b) t", ["档位"], {"小单", "大单"}),
|
||||
# UNION 两支都是标签（第二支没写 ELSE、没起别名）：两支的标签都算
|
||||
(f"SELECT {BANDS} AS 档位 FROM sales UNION ALL SELECT {_EXTRA} FROM sales", ["档位"], {"小单", "大单", "特大单"}),
|
||||
# 派生来源里的 UNION 两支都是标签，外层经 * 带出来
|
||||
(f"SELECT * FROM (SELECT {BANDS} AS 档位 FROM sales UNION ALL SELECT {_EXTRA} FROM sales) t", ["档位"],
|
||||
{"小单", "大单", "特大单"}),
|
||||
])
|
||||
def test_label_columns_are_traced_to_where_they_are_written(sql, columns, texts):
|
||||
"""改成按作用域认之后，写在别处、原样传下来的标签照样认得出：外层连上数据源里的表没写表名、按别的列 USING、写了
|
||||
表名经两层子查询；UNION 两支都是标签的，每一支的标签都算（以前派生来源里的 UNION 按名字认，只算起了这个名字的
|
||||
第一支）。"""
|
||||
assert inquiry_mod._label_columns(sql, "sqlite", columns) == {"档位": texts}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 按值认：门槛、边界
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -216,12 +425,16 @@ def people() -> inquiry_mod._Secrets:
|
||||
("SELECT region FROM sales WHERE buyer_phone NOT IN ('13900000009') AND age <> 40",
|
||||
"SELECT region FROM sales WHERE buyer_phone NOT IN ('13900000009') AND age <> 40"),
|
||||
("SELECT amount FROM sales WHERE amount = 13900000001", "SELECT amount FROM sales WHERE amount = 已遮罩"),
|
||||
# 和遮罩列做范围比较：边界是阈值，不是在找某个人，照原样（同一个数只从一边隔开：25 的几处都是「不超过 25」那一侧）
|
||||
("SELECT COUNT(*) FROM sales WHERE age > 31 AND age <= 25 AND 25 < age AND age NOT BETWEEN 20 AND 25",
|
||||
"SELECT COUNT(*) FROM sales WHERE age > 31 AND age <= 25 AND 25 < age AND age NOT BETWEEN 20 AND 25"),
|
||||
# 和遮罩列做范围比较：边界是阈值，不是在找某个人，照原样。25 两边都有边界（<= 25、NOT BETWEEN 25 …），可这几个
|
||||
# 条件合起来谁也找不着（> 31 和 <= 25 不会同时成立），夹不住 25
|
||||
("SELECT COUNT(*) FROM sales WHERE age > 31 AND age <= 25 AND 25 < age AND age NOT BETWEEN 25 AND 31",
|
||||
"SELECT COUNT(*) FROM sales WHERE age > 31 AND age <= 25 AND 25 < age AND age NOT BETWEEN 25 AND 31"),
|
||||
# 不等、SQLite 的 IS、NOT LIKE 也是在找某一个值
|
||||
("SELECT COUNT(*) FROM sales WHERE age != 25 AND age IS NOT 31 AND buyer_phone NOT LIKE '13900000001'",
|
||||
"SELECT COUNT(*) FROM sales WHERE age != 已遮罩 AND age IS NOT 已遮罩 AND buyer_phone NOT LIKE '已遮罩'"),
|
||||
# IN 列表里写着遮罩值，换的是它；旁边的范围边界合起来不只认一个值（25、26 都要），照原样
|
||||
("SELECT COUNT(*) FROM sales WHERE age > 24 AND age IN (25, 26)",
|
||||
"SELECT COUNT(*) FROM sales WHERE age > 24 AND age IN (已遮罩, 26)"),
|
||||
])
|
||||
def test_sql_keeps_everything_but_the_masked_values(people, sql, shown):
|
||||
assert inquiry_mod._scrub_sql(sql, people, {"buyer_phone", "age"}, "sqlite") == shown
|
||||
@@ -314,12 +527,11 @@ def test_a_value_looked_up_in_any_shape_is_masked_in_the_sql(phones, kind, sql,
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone >= '已遮罩' AND buyer_phone <= '已遮罩'"),
|
||||
("SELECT COUNT(*) FROM sales WHERE buyer_phone BETWEEN '13900000001' AND '13900000001'",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone BETWEEN '已遮罩' AND '已遮罩'"),
|
||||
# 一头是几个数算出来的（13900000002 - 1 就是 13900000001）：照样夹住了
|
||||
# 一头是几个数算出来的（13900000002 - 1 就是 13900000001）：照样夹住了，算式里的数都换（留着哪个都可能算得出号码）
|
||||
("SELECT COUNT(*) FROM sales WHERE buyer_phone BETWEEN 13900000001 AND 13900000002 - 1",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone BETWEEN 已遮罩 AND 已遮罩 - 1"),
|
||||
# 同一个数在算式里被换掉了，边界上的也不留：留着就把算式里换掉的露了
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone BETWEEN 已遮罩 AND 已遮罩 - 已遮罩"),
|
||||
("SELECT COUNT(*) FROM sales WHERE buyer_phone BETWEEN 13900000001 AND 13900000001 + 0",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone BETWEEN 已遮罩 AND 已遮罩 + 0"),
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone BETWEEN 已遮罩 AND 已遮罩 + 已遮罩"),
|
||||
])
|
||||
def test_a_range_that_pins_a_single_value_is_a_lookup(phones, sql, shown):
|
||||
"""两头是同一个值的范围（BETWEEN v AND v、>= v AND <= v）、大于或小于同一个值、整数的半开区间，都只夹住了一个值，
|
||||
@@ -327,6 +539,332 @@ def test_a_range_that_pins_a_single_value_is_a_lookup(phones, sql, shown):
|
||||
assert inquiry_mod._scrub_sql(sql, phones, {"buyer_phone", "age"}, "sqlite") == shown
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, sealed", [
|
||||
# 范围的一头是几个数算出来的：算式就是那个值，夹住了一样是在找它，算式里的数都换
|
||||
("SELECT COUNT(*) FROM sales WHERE age >= 25 AND age <= 25 + 0",
|
||||
"SELECT COUNT(*) FROM sales WHERE age >= 已遮罩 AND age <= 已遮罩 + 已遮罩"),
|
||||
("SELECT COUNT(*) FROM sales WHERE age BETWEEN 20 + 5 AND 30 - 5",
|
||||
"SELECT COUNT(*) FROM sales WHERE age BETWEEN 已遮罩 + 已遮罩 AND 已遮罩 - 已遮罩"),
|
||||
("SELECT COUNT(*) FROM sales WHERE age >= 50 / 2 AND age <= (5 * 5)",
|
||||
"SELECT COUNT(*) FROM sales WHERE age >= 已遮罩 / 已遮罩 AND age <= (已遮罩 * 已遮罩)"),
|
||||
("SELECT COUNT(*) FROM sales WHERE buyer_phone BETWEEN 13900000000 + 1 AND 13900000009 - 8",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone BETWEEN 已遮罩 + 已遮罩 AND 已遮罩 - 已遮罩"),
|
||||
# 等值、IN 里的值是算出来的：按算出来的值认
|
||||
("SELECT COUNT(*) FROM sales WHERE age = 20 + 5 OR buyer_phone IN (13900000000 + 2, 7)",
|
||||
"SELECT COUNT(*) FROM sales WHERE age = 已遮罩 + 已遮罩 OR buyer_phone IN (已遮罩 + 已遮罩, 7)"),
|
||||
# 选出列里算出来的号码：同选出列里直接写着号码，够长的遮罩值换
|
||||
("SELECT 13900000000 + 1 AS 号码, COUNT(*) AS 笔数 FROM sales",
|
||||
"SELECT 已遮罩 + 已遮罩 AS 号码, COUNT(*) AS 笔数 FROM sales"),
|
||||
])
|
||||
def test_a_value_worked_out_from_several_numbers_is_read_as_that_value(phones, sql, sealed):
|
||||
"""第 2 轮复核：范围的一头写成算式（25 + 0、13900000000 + 1），夹住的值照样认得出，可算式里的数照原样留着，
|
||||
一算就是那个号码。算式按算出来的值认：夹住某一个值的、等值、IN 里的，算式里的数都换；正文里也不免遮。"""
|
||||
assert inquiry_mod._scrub_sql(sql, phones, {"buyer_phone", "age"}, "sqlite") == sealed
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite", {"buyer_phone", "age"})], phones) == set()
|
||||
|
||||
|
||||
def test_a_threshold_worked_out_from_several_numbers_is_kept_like_one_number(amounts):
|
||||
"""算出来的阈值（20000 * 1.5）和写成一个数的阈值一样：没夹住某一个值就照原样，正文里复述它（连同算出来的
|
||||
30000）也不遮；算出来的值在别处确实被换掉了，算式也不留（同写成一个数的阈值）。"""
|
||||
sql = "SELECT COUNT(*) FROM sales WHERE amount >= 20000 * 1.5 AND amount < 20000 * 2.5"
|
||||
assert inquiry_mod._scrub_sql(sql, amounts, {"amount"}, "sqlite") == sql
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite", {"amount"})], amounts) == {
|
||||
"20000", "1.5", "2.5", "30000", "50000"}
|
||||
swapped = "SELECT COUNT(*) FROM sales WHERE amount >= 15000 + 15000 OR amount = 30000"
|
||||
assert inquiry_mod._scrub_sql(swapped, amounts, {"amount"}, "sqlite") == (
|
||||
"SELECT COUNT(*) FROM sales WHERE amount >= 已遮罩 + 已遮罩 OR amount = 已遮罩")
|
||||
assert inquiry_mod._shown_literals([(swapped, "sqlite", {"amount"})], amounts) == set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind, sql, sealed", [
|
||||
# 取模、整除算出来的边界和等值：125 % 100、MOD(125, 100) 是 25，50 DIV 2、51 // 2 也是
|
||||
("sqlite", "SELECT COUNT(*) FROM sales WHERE age >= 25 AND age <= 125 % 100",
|
||||
"SELECT COUNT(*) FROM sales WHERE age >= 已遮罩 AND age <= 已遮罩 % 已遮罩"),
|
||||
("postgresql", "SELECT COUNT(*) FROM sales WHERE age = MOD(125, 100)",
|
||||
"SELECT COUNT(*) FROM sales WHERE age = MOD(已遮罩, 已遮罩)"),
|
||||
("mysql", "SELECT COUNT(*) FROM sales WHERE age >= 25 AND age <= 50 DIV 2",
|
||||
"SELECT COUNT(*) FROM sales WHERE age >= 已遮罩 AND age <= 已遮罩 DIV 已遮罩"),
|
||||
("duckdb", "SELECT COUNT(*) FROM sales WHERE age >= 25 AND age <= 51 // 2",
|
||||
"SELECT COUNT(*) FROM sales WHERE age >= 已遮罩 AND age <= 已遮罩 // 已遮罩"),
|
||||
# SQLite、Postgres 里两个整数相除只留整数部分：53 / 2 是 26，age < 26 和 age >= 25 夹住 25
|
||||
("sqlite", "SELECT COUNT(*) FROM sales WHERE age >= 25 AND age < 53 / 2",
|
||||
"SELECT COUNT(*) FROM sales WHERE age >= 已遮罩 AND age < 已遮罩 / 已遮罩"),
|
||||
("postgresql", "SELECT COUNT(*) FROM sales WHERE buyer_phone >= 13900000001 AND buyer_phone < 27800000005 / 2",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone >= 已遮罩 AND buyer_phone < 已遮罩 / 已遮罩"),
|
||||
])
|
||||
def test_a_value_worked_out_with_remainders_or_whole_division_is_read_as_that_value(phones, kind, sql, sealed):
|
||||
"""第 3 轮复核：取模（%、MOD）、整除（MySQL 的 DIV、DuckDB 的 //）写成的算式以前不按算出来的值认，age >= 25 AND
|
||||
age <= 125 % 100 照原样封存；整数相除在 SQLite、Postgres 里只留整数部分，以前按小数算（53 / 2 算成 26.5），
|
||||
夹住的值也认不出。现在都按数据源算出来的那一个值认，算式里的数都换，正文里也不免遮。"""
|
||||
assert inquiry_mod._scrub_sql(sql, phones, {"buyer_phone", "age"}, kind) == sealed
|
||||
assert not inquiry_mod._shown_literals([(sql, kind, {"buyer_phone", "age"})], phones) & {"25", "13900000001"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind, divided", [("mysql", "53 / 2"), ("duckdb", "53 / 2"), ("sqlite", "53.0 / 2")])
|
||||
def test_division_that_keeps_the_fraction_is_read_as_written(phones, kind, divided):
|
||||
"""MySQL、DuckDB 的 / 照小数算，SQLite 里写成小数的也是：53 / 2、53.0 / 2 是 26.5，age >= 25 AND age < 26.5 里
|
||||
25、26 都成立，没夹住，照原样。"""
|
||||
sql = f"SELECT COUNT(*) FROM sales WHERE age >= 25 AND age < {divided}"
|
||||
assert inquiry_mod._scrub_sql(sql, phones, {"buyer_phone", "age"}, kind) == sql
|
||||
|
||||
|
||||
def test_type_parameters_of_a_cast_bound_are_left_alone(amounts):
|
||||
"""第 3 轮复核：夹住某一个值的边界写成 CAST(v AS DECIMAL(10, 2)) 时，类型参数 10、2 以前也当成算出边界的数换掉，
|
||||
封存的 SQL 成了 DECIMAL(已遮罩, 已遮罩)，SQL 检查结果的片段里别处的 10、2 也跟着换。只换算出边界的那个数；Oracle
|
||||
的 CAST … DEFAULT 0 ON CONVERSION ERROR 里的 0 也不算出值，照原样。"""
|
||||
sql = ("SELECT COUNT(*) FROM sales WHERE amount >= CAST(30000 AS DECIMAL(10, 2)) "
|
||||
"AND amount <= CAST(30000 AS DECIMAL(10, 2))")
|
||||
assert inquiry_mod._scrub_sql(sql, amounts, {"amount"}, "sqlite") == sql.replace("30000", MASKED)
|
||||
assert inquiry_mod._literal_swaps(sql, amounts, {"amount"}, "sqlite") == {"30000"}
|
||||
checks = [{"code": "stock_summed", "level": "warning", "table": "sales", "message": "存量跨日期求和",
|
||||
"sql_excerpt": "ROUND(SUM(amount), 2) >= 10"}]
|
||||
[check] = inquiry_mod._masked_checks(checks, sql, amounts, {"amount"}, "sqlite")
|
||||
assert check["sql_excerpt"] == "ROUND(SUM(amount), 2) >= 10"
|
||||
fallback = ("SELECT COUNT(*) FROM sales WHERE amount >= CAST('30000' AS NUMBER DEFAULT 0 ON CONVERSION ERROR) "
|
||||
"AND amount <= CAST('30000' AS NUMBER DEFAULT 0 ON CONVERSION ERROR)")
|
||||
assert inquiry_mod._scrub_sql(fallback, amounts, {"amount"}, "oracle") == fallback.replace("30000", MASKED)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, sealed", [
|
||||
# 遮罩列套了 CAST、汇总：被比较的一边取的还是它的值，短的数照样换
|
||||
("SELECT COUNT(*) FROM sales WHERE CAST(age AS INTEGER) = 25",
|
||||
"SELECT COUNT(*) FROM sales WHERE CAST(age AS INTEGER) = 已遮罩"),
|
||||
("SELECT COUNT(*) FROM sales WHERE CAST(age AS INTEGER) BETWEEN 25 AND 25",
|
||||
"SELECT COUNT(*) FROM sales WHERE CAST(age AS INTEGER) BETWEEN 已遮罩 AND 已遮罩"),
|
||||
("SELECT region FROM sales GROUP BY region HAVING MAX(age) BETWEEN 25 AND 25",
|
||||
"SELECT region FROM sales GROUP BY region HAVING MAX(age) BETWEEN 已遮罩 AND 已遮罩"),
|
||||
# 同一个数换个写法（小数点后补零、前面补零、科学计数法）
|
||||
("SELECT COUNT(*) FROM sales WHERE age = 25.0 OR age = 025 OR age IN (2.5e1)",
|
||||
"SELECT COUNT(*) FROM sales WHERE age = 已遮罩 OR age = 已遮罩 OR age IN (已遮罩)"),
|
||||
("SELECT COUNT(*) FROM sales WHERE buyer_phone = 013900000001 OR buyer_phone = 1.3900000001e10",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone = 已遮罩 OR buyer_phone = 已遮罩"),
|
||||
# 两头不是整数的范围：整数上只夹住中间那一个
|
||||
("SELECT COUNT(*) FROM sales WHERE age > 24.5 AND age < 25.5",
|
||||
"SELECT COUNT(*) FROM sales WHERE age > 已遮罩 AND age < 已遮罩"),
|
||||
("SELECT COUNT(*) FROM sales WHERE age BETWEEN 25 AND 25.9",
|
||||
"SELECT COUNT(*) FROM sales WHERE age BETWEEN 已遮罩 AND 已遮罩"),
|
||||
("SELECT COUNT(*) FROM sales WHERE buyer_phone > 13900000000.5 AND buyer_phone < 13900000001.5",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone > 已遮罩 AND buyer_phone < 已遮罩"),
|
||||
# 范围再用不等、NOT IN 去掉一头：同一个条件里合起来只剩一个值
|
||||
("SELECT COUNT(*) FROM sales WHERE age >= 25 AND age <= 26 AND age <> 26",
|
||||
"SELECT COUNT(*) FROM sales WHERE age >= 已遮罩 AND age <= 26 AND age <> 已遮罩"),
|
||||
("SELECT COUNT(*) FROM sales WHERE buyer_phone >= 13900000001 AND buyer_phone <= 13900000002 "
|
||||
"AND buyer_phone <> 13900000002",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone >= 已遮罩 AND buyer_phone <= 已遮罩 AND buyer_phone <> 已遮罩"),
|
||||
("SELECT COUNT(*) FROM sales WHERE age BETWEEN 25 AND 27 AND age NOT IN (26, 27)",
|
||||
"SELECT COUNT(*) FROM sales WHERE age BETWEEN 已遮罩 AND 27 AND age NOT IN (已遮罩, 27)"),
|
||||
])
|
||||
def test_other_ways_of_looking_up_one_value_are_masked_too(phones, sql, sealed):
|
||||
"""第 2 轮复核留下的几种找值写法：遮罩列套了 CAST、汇总，同一个数换个写法，整数列上两头不是整数的范围，范围再用
|
||||
不等、NOT IN 去掉一头。以前都照原样封存，长号码的那几种还因为照原样留着、被当成阈值，回答正文跟着不遮。"""
|
||||
assert inquiry_mod._scrub_sql(sql, phones, {"buyer_phone", "age"}, "sqlite") == sealed
|
||||
shown = inquiry_mod._shown_literals([(sql, "sqlite", {"buyer_phone", "age"})], phones)
|
||||
assert not shown & {"25", "13900000001"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql", [
|
||||
# 范围加上不等、IN，合起来不只认一个值：照原样
|
||||
"SELECT COUNT(*) FROM sales WHERE age >= 25 AND age <= 30 AND age <> 26",
|
||||
"SELECT COUNT(*) FROM sales WHERE age > 0 AND age IN (26, 27)",
|
||||
"SELECT COUNT(*) FROM sales WHERE age >= 25 OR age = 26",
|
||||
# 两头不是整数、中间不只一个整数
|
||||
"SELECT COUNT(*) FROM sales WHERE age > 24.5 AND age < 26.5",
|
||||
# 不等、IN 在别的条件里（选出列的 CASE 和 WHERE）：只有范围比较跨条件配对
|
||||
"SELECT SUM(CASE WHEN age = 26 THEN 1 ELSE 0 END) AS 人数 FROM sales WHERE age >= 25",
|
||||
])
|
||||
def test_ranges_next_to_other_comparisons_that_do_not_pin_a_value_stay(phones, sql):
|
||||
assert inquiry_mod._scrub_sql(sql, phones, {"buyer_phone", "age"}, "sqlite") == sql
|
||||
|
||||
|
||||
def test_a_long_sum_of_numbers_does_not_stop_the_reading(ages):
|
||||
"""visits <= 1+1+…（上千项）：以前算式递归求值，超过递归深度，整条 SQL 退回按词认，同一句里夹住的年龄就照原样
|
||||
封存。现在不递归，照样按语法树认。"""
|
||||
total = "+".join(["1"] * 1500)
|
||||
sql = f"SELECT COUNT(*) FROM sales WHERE visits <= {total} AND age >= 25 AND age <= 25"
|
||||
assert inquiry_mod._literal_uses(sql, "sqlite") is not None
|
||||
started = time.perf_counter()
|
||||
assert inquiry_mod._scrub_sql(sql, ages, {"age"}, "sqlite") == sql.replace(
|
||||
"age >= 25 AND age <= 25", "age >= 已遮罩 AND age <= 已遮罩")
|
||||
elapsed = time.perf_counter() - started
|
||||
assert elapsed < 5, f"用了 {elapsed:.2f} 秒"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind, sql, shown", [
|
||||
# 同一个条件里还有别的条件（且、或）：照样夹住了
|
||||
("sqlite", "SELECT COUNT(*) FROM sales WHERE region = '华东' AND buyer_phone >= 13900000001 "
|
||||
"AND buyer_phone <= 13900000001",
|
||||
"SELECT COUNT(*) FROM sales WHERE region = '华东' AND buyer_phone >= 已遮罩 AND buyer_phone <= 已遮罩"),
|
||||
("sqlite", "SELECT COUNT(*) FROM sales WHERE (age >= 25 AND age <= 25) OR region = '华南'",
|
||||
"SELECT COUNT(*) FROM sales WHERE (age >= 已遮罩 AND age <= 已遮罩) OR region = '华南'"),
|
||||
# CASE WHEN 里的一个条件
|
||||
("sqlite", "SELECT COUNT(CASE WHEN buyer_phone BETWEEN 13900000001 AND 13900000001 THEN 1 END) AS 笔数 FROM sales",
|
||||
"SELECT COUNT(CASE WHEN buyer_phone BETWEEN 已遮罩 AND 已遮罩 THEN 1 END) AS 笔数 FROM sales"),
|
||||
# 整数的开区间：两头都不是那个值，夹住的是中间那一个，两头一样换（留着 24、26 就等于写出了 25）
|
||||
("sqlite", "SELECT COUNT(*) FROM sales WHERE age > 24 AND age < 26",
|
||||
"SELECT COUNT(*) FROM sales WHERE age > 已遮罩 AND age < 已遮罩"),
|
||||
("sqlite", "SELECT COUNT(*) FROM sales WHERE age >= 25 AND age < 26",
|
||||
"SELECT COUNT(*) FROM sales WHERE age >= 已遮罩 AND age < 已遮罩"),
|
||||
("sqlite", "SELECT COUNT(*) FROM sales WHERE buyer_phone > 13900000000 AND buyer_phone < 13900000002",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone > 已遮罩 AND buyer_phone < 已遮罩"),
|
||||
# 不低于、大于同一个值，二者只成立一个（MySQL 的 XOR）：就是等于它
|
||||
("mysql", "SELECT COUNT(*) FROM sales WHERE age >= 25 XOR age > 25",
|
||||
"SELECT COUNT(*) FROM sales WHERE age >= 已遮罩 XOR age > 已遮罩"),
|
||||
])
|
||||
def test_a_value_pinned_within_one_condition_is_still_a_lookup(phones, kind, sql, shown):
|
||||
"""同一个条件里（从比较往上只越过且、或、非、异或、括号）夹住的，按条件合起来的真假认出来照样换：旁边还有别的
|
||||
条件、写在 CASE WHEN 里、整数的开区间、XOR。"""
|
||||
assert inquiry_mod._scrub_sql(sql, phones, {"buyer_phone", "age"}, kind) == shown
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, shown", [
|
||||
("SELECT region FROM sales GROUP BY region HAVING SUM(amount) BETWEEN 20000 AND 20000",
|
||||
"SELECT region FROM sales GROUP BY region HAVING SUM(amount) BETWEEN 已遮罩 AND 已遮罩"),
|
||||
("SELECT region FROM sales GROUP BY region HAVING SUM(amount) >= 20000 AND SUM(amount) <= 20000",
|
||||
"SELECT region FROM sales GROUP BY region HAVING SUM(amount) >= 已遮罩 AND SUM(amount) <= 已遮罩"),
|
||||
])
|
||||
def test_the_same_aggregate_pinned_from_both_sides_is_a_lookup(amounts, sql, shown):
|
||||
"""列名认不出的（汇总值、函数）只和写得一样的表达式配：HAVING 里同一个汇总值两头相同，照样是在找它。"""
|
||||
assert inquiry_mod._scrub_sql(sql, amounts, {"amount"}, "sqlite") == shown
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql", [
|
||||
# 两个 WHEN 分支共用一个边界：各是各的条件，20000 只是分档的界
|
||||
f"SELECT {BANDS} AS 档位, COUNT(*) AS 笔数 FROM sales GROUP BY 1",
|
||||
# 分组时把 CASE 原样再写一遍：选出列和 GROUP BY 是同一个算法的两处写法，都不筛行
|
||||
f"SELECT {BANDS} AS 档位, COUNT(*) AS 笔数 FROM sales GROUP BY {BANDS}",
|
||||
# 两个选出列各按一边数（写成汇总、写成标量子查询都一样）
|
||||
"SELECT SUM(CASE WHEN amount <= 20000 THEN 1 ELSE 0 END) AS 小单, "
|
||||
"SUM(CASE WHEN amount >= 20000 THEN 1 ELSE 0 END) AS 大单 FROM sales",
|
||||
"SELECT (SELECT COUNT(*) FROM sales WHERE amount <= 20000) AS 小单, "
|
||||
"(SELECT COUNT(*) FROM sales WHERE amount >= 20000) AS 大单",
|
||||
# UNION 的两支
|
||||
"SELECT '小单' AS 档位, COUNT(*) AS 笔数 FROM sales WHERE amount <= 20000 "
|
||||
"UNION ALL SELECT '大单', COUNT(*) FROM sales WHERE amount >= 20000",
|
||||
# IIF 的两个分支：华东按一边数，别的区域按另一边数
|
||||
"SELECT SUM(IIF(region = '华东', amount <= 20000, amount >= 20000)) AS 笔数 FROM sales",
|
||||
# WHERE 里一笔的上限、HAVING 里汇总值的下限：比的不是同一个东西（列名认不出的只和写得一样的表达式配）
|
||||
"SELECT region, COUNT(*) AS 笔数 FROM sales WHERE amount <= 20000 GROUP BY region HAVING SUM(amount) >= 20000",
|
||||
# 同一个条件里，算式不和金额这一列配：金额在 15000 到 20000 之间
|
||||
"SELECT COUNT(*) FROM sales WHERE amount <= 20000 AND amount + 5000 >= 20000",
|
||||
# 边界两边都有比较，可合起来不只认 20000 这一个值：不超过 20000 的，或 20000 以上访问又多的
|
||||
"SELECT COUNT(*) FROM sales WHERE amount <= 20000 OR (amount >= 20000 AND visits > 25000)",
|
||||
])
|
||||
def test_band_edges_shared_by_separate_conditions_are_thresholds(amounts, sql):
|
||||
"""第 3 轮收全句所有范围比较一起配对，列名认不出的（SUM(amount)、函数）还和哪一列都配：分档的两个 WHEN 分支、
|
||||
两个选出列、UNION 两支各写一边，共用的那个边界就被当成「夹住了某一笔」遮掉，回答里「以 20000 元为界」也跟着遮。
|
||||
这些都是阈值：照原样封存，正文里复述也不遮。"""
|
||||
assert inquiry_mod._scrub_sql(sql, amounts, {"amount"}, "sqlite") == sql
|
||||
assert "20000" in inquiry_mod._shown_literals([(sql, "sqlite", {"amount"})], amounts)
|
||||
|
||||
|
||||
_PHONE, _NEXT = "13900000001", "13900000002"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind, sql", [
|
||||
# 内连接的 ON 和 WHERE 筛的是同一批行：各写一边照样夹住
|
||||
("sqlite", "SELECT o.region FROM sales o JOIN buyers b ON b.id = o.bid AND b.buyer_phone >= {v} "
|
||||
"WHERE b.buyer_phone <= {v}"),
|
||||
# WHERE 和 HAVING 比的是同一列（分组的键）
|
||||
("sqlite", "SELECT buyer_phone, COUNT(*) FROM sales WHERE buyer_phone >= {v} GROUP BY buyer_phone "
|
||||
"HAVING buyer_phone <= {v}"),
|
||||
# INTERSECT 两支都要成立；EXCEPT 是前一支成立、后一支不成立（不低于 25 又不高于 25）
|
||||
("sqlite", "SELECT id FROM sales WHERE age >= {a} INTERSECT SELECT id FROM sales WHERE age <= {a}"),
|
||||
("sqlite", "SELECT id FROM sales WHERE age >= {a} EXCEPT SELECT id FROM sales WHERE age > {a}"),
|
||||
# 一半包在 CASE、布尔比较、函数、算式里
|
||||
("sqlite", "SELECT COUNT(*) FROM sales WHERE age >= {a} AND CASE WHEN age <= {a} THEN 1 ELSE 0 END = 1"),
|
||||
("sqlite", "SELECT COUNT(*) FROM sales WHERE age >= {a} AND (age <= {a}) = TRUE"),
|
||||
("sqlite", "SELECT COUNT(*) FROM sales WHERE age >= {a} AND COALESCE(age <= {a}, 0)"),
|
||||
("sqlite", "SELECT COUNT(*) FROM sales WHERE IIF(age >= {a}, 1, 0) + IIF(age <= {a}, 1, 0) = 2"),
|
||||
# 嵌套的 CASE：里层只在外层成立时才算
|
||||
("sqlite", "SELECT SUM(CASE WHEN age >= {a} THEN CASE WHEN age <= {a} THEN 1 END END) AS 人数 FROM sales"),
|
||||
# 选出列里的 FILTER 数的是 WHERE 筛过的行
|
||||
("duckdb", "SELECT COUNT(*) FILTER (WHERE age >= {a}) AS 人数 FROM sales WHERE age <= {a}"),
|
||||
# 两个汇总值相减：不低于 25 的减去高于 25 的，就是正好 25 的
|
||||
("sqlite", "SELECT SUM(age >= {a}) - SUM(age > {a}) AS 人数 FROM sales"),
|
||||
# 拆进子查询、CTE
|
||||
("sqlite", "SELECT COUNT(*) FROM sales WHERE age >= {a} AND id IN (SELECT id FROM sales WHERE age <= {a})"),
|
||||
("sqlite", "WITH a AS (SELECT * FROM sales WHERE buyer_phone >= {v}) SELECT COUNT(*) FROM a "
|
||||
"WHERE buyer_phone <= {v}"),
|
||||
])
|
||||
def test_a_lookup_split_across_clauses_is_still_a_lookup(phones, kind, sql):
|
||||
"""把找某一个值的两半拆到同一行集的不同地方（ON 和 WHERE、WHERE 和 HAVING、INTERSECT 两支、包在 CASE 或函数里、
|
||||
子查询、CTE）：第 4 轮只在同一个条件里认，两半各自当阈值照原样留着，长的号码随即被当成「摆在读者眼前」，回答
|
||||
正文里也不遮，最后一道关跟着放行。只有分档的几种写法（CASE 的不同分支、不同的选出列、UNION 的不同支）各算各的，
|
||||
别的都按同一行集看：两半照样换掉，正文里也不免遮。"""
|
||||
written = sql.format(v=_PHONE, a=25)
|
||||
assert inquiry_mod._scrub_sql(written, phones, {"buyer_phone", "age"}, kind) == sql.format(v=MASKED, a=MASKED)
|
||||
assert inquiry_mod._shown_literals([(written, kind, {"buyer_phone", "age"})], phones) == set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, sealed", [
|
||||
# 整数上 age <= 24 和 age < 25 一样，从下面把 25 隔开；age >= 26 和 age > 25 一样，从上面隔开
|
||||
("SELECT COUNT(*) FROM sales WHERE NOT (age <= 24 OR age >= 26)",
|
||||
"SELECT COUNT(*) FROM sales WHERE NOT (age <= 已遮罩 OR age >= 已遮罩)"),
|
||||
("SELECT COUNT(*) FROM sales WHERE age >= 25 AND NOT age >= 26",
|
||||
"SELECT COUNT(*) FROM sales WHERE age >= 已遮罩 AND NOT age >= 已遮罩"),
|
||||
(f"SELECT COUNT(*) FROM sales WHERE buyer_phone >= {_PHONE} AND NOT buyer_phone >= {_NEXT}",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone >= 已遮罩 AND NOT buyer_phone >= 已遮罩"),
|
||||
# 就是 age <> 25
|
||||
("SELECT COUNT(*) FROM sales WHERE age <= 24 OR age >= 26",
|
||||
"SELECT COUNT(*) FROM sales WHERE age <= 已遮罩 OR age >= 已遮罩"),
|
||||
])
|
||||
def test_neighbouring_whole_numbers_pin_the_value_between_them(phones, sql, sealed):
|
||||
assert inquiry_mod._scrub_sql(sql, phones, {"buyer_phone", "age"}, "sqlite") == sealed
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite", {"buyer_phone", "age"})], phones) == set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql", [
|
||||
"SELECT COUNT(*) FROM sales WHERE age >= 18 AND age <= 30",
|
||||
"SELECT CASE WHEN age <= 24 THEN '24岁及以下' WHEN age >= 26 THEN '26岁及以上' END AS 年龄段, COUNT(*) AS 人数 "
|
||||
"FROM sales GROUP BY 1",
|
||||
"SELECT COUNT(*) FROM sales WHERE age BETWEEN 18 AND 24",
|
||||
])
|
||||
def test_ordinary_ranges_are_not_pinned_by_neighbouring_whole_numbers(phones, sql):
|
||||
assert inquiry_mod._scrub_sql(sql, phones, {"buyer_phone", "age"}, "sqlite") == sql
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, sealed", [
|
||||
# 被更宽的范围吸收了，条件合起来不只认这一个号码，可 BETWEEN v AND v 本身就写出了它
|
||||
(f"SELECT COUNT(*) FROM sales WHERE buyer_phone BETWEEN {_PHONE} AND {_PHONE} OR buyer_phone > 13000000000",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone BETWEEN 已遮罩 AND 已遮罩 OR buyer_phone > 13000000000"),
|
||||
# 旁边还有矛盾的条件（谁也查不着）：两半紧挨着、隔着别的条件写都一样
|
||||
(f"SELECT COUNT(*) FROM sales WHERE buyer_phone >= {_PHONE} AND buyer_phone <= {_PHONE} "
|
||||
"AND buyer_phone > 13900000005",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone >= 已遮罩 AND buyer_phone <= 已遮罩 AND buyer_phone > 13900000005"),
|
||||
(f"SELECT COUNT(*) FROM sales WHERE buyer_phone >= {_PHONE} AND buyer_phone > 13900000005 "
|
||||
f"AND buyer_phone <= {_PHONE}",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone >= 已遮罩 AND buyer_phone > 13900000005 AND buyer_phone <= 已遮罩"),
|
||||
# 大于或小于同一个值（就是不等于），旁边又或上一个更宽的范围
|
||||
(f"SELECT COUNT(*) FROM sales WHERE buyer_phone > {_PHONE} OR buyer_phone > 13000000000 OR buyer_phone < {_PHONE}",
|
||||
"SELECT COUNT(*) FROM sales WHERE buyer_phone > 已遮罩 OR buyer_phone > 13000000000 OR buyer_phone < 已遮罩"),
|
||||
])
|
||||
def test_a_range_that_names_one_value_is_a_lookup_whatever_surrounds_it(phones, sql, sealed):
|
||||
"""BETWEEN v AND v、>= v AND <= v、> v OR < v 不论整数都只认一个值：条件里哪一层这样写都算在找它，旁边的条件把它
|
||||
吸收了、和它矛盾了也一样（规格要求这几种必须继续遮）。整数上的半开、开区间只看整个条件，和相邻的分档连成一段的
|
||||
不算（见下面成百个分档的那条）。"""
|
||||
assert inquiry_mod._scrub_sql(sql, phones, {"buyer_phone", "age"}, "sqlite") == sealed
|
||||
assert _PHONE not in inquiry_mod._shown_literals([(sql, "sqlite", {"buyer_phone", "age"})], phones)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def ages() -> inquiry_mod._Secrets:
|
||||
"""年龄设了遮罩，都是两位数：和天数修饰、日期、标签、别的列上的数撞在一起。"""
|
||||
return secrets_of({"columns": ["age"], "rows": [[25], [30], [20], [31]]}, {"age"})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, shown", [
|
||||
("SELECT CASE WHEN age BETWEEN 18 AND 30 THEN '青年' WHEN age BETWEEN 30 AND 45 THEN '中年' END AS 年龄段, "
|
||||
"COUNT(*) AS 人数 FROM sales GROUP BY 1", {"18", "30", "45"}),
|
||||
# 标签里写着同样的数：短的数在字符串里不换，阈值也不跟着遮
|
||||
("SELECT CASE WHEN age BETWEEN 18 AND 30 THEN '18-30岁' WHEN age BETWEEN 30 AND 45 THEN '30-45岁' END AS 年龄段, "
|
||||
"COUNT(*) AS 人数 FROM sales GROUP BY 1", {"18", "30", "45"}),
|
||||
])
|
||||
def test_overlapping_age_bands_are_thresholds(ages, sql, shown):
|
||||
"""年龄段的两个 WHEN 分支共用边界 30：同上，是分档的界。"""
|
||||
assert inquiry_mod._scrub_sql(sql, ages, {"age"}, "sqlite") == sql
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite", {"age"})], ages) == shown
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, shown", [
|
||||
("SELECT COUNT(*) AS 笔数, 13900000001 AS 号码 FROM sales WHERE buyer_phone >= 13900000001",
|
||||
"SELECT COUNT(*) AS 笔数, 已遮罩 AS 号码 FROM sales WHERE buyer_phone >= 已遮罩"),
|
||||
@@ -336,7 +874,77 @@ def test_a_range_that_pins_a_single_value_is_a_lookup(phones, sql, shown):
|
||||
def test_a_number_masked_elsewhere_in_the_sql_is_not_kept_as_a_threshold(phones, sql, shown):
|
||||
"""同一个号码在选出列、算式里被换掉了，条件里的单边范围也不留：留着就把那里换掉的露了。"""
|
||||
assert inquiry_mod._scrub_sql(sql, phones, {"buyer_phone", "age"}, "sqlite") == shown
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite")]) <= {"5"}
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite", {"buyer_phone", "age"})], phones) <= {"5"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, shown", [
|
||||
# 天数修饰、日期里的数：短的数在字符串里从来不换，日期里的一截也不认
|
||||
("SELECT COUNT(*) AS 人数 FROM sales WHERE age >= 30 AND paid_on >= DATE('2026-02-03', '-30 days')", {"30"}),
|
||||
("SELECT COUNT(*) AS 人数 FROM sales WHERE age > 31 AND paid_on <= '2026-01-31'", {"31"}),
|
||||
# 标签里的数
|
||||
("SELECT CASE WHEN age < 30 THEN '30岁以下' ELSE '30岁及以上' END AS 年龄段, COUNT(*) AS 人数 FROM sales GROUP BY 1",
|
||||
{"30"}),
|
||||
# 没设遮罩的列上的等值：短的数只在和遮罩列比较时换
|
||||
("SELECT COUNT(*) AS 人数 FROM sales WHERE age >= 30 AND amount = 30", {"30"}),
|
||||
])
|
||||
def test_a_threshold_stays_when_the_same_number_is_left_alone_elsewhere(ages, sql, shown):
|
||||
"""第 3 轮把「这条 SQL 别的字面量里写着同一个数」一律当成那里换掉了：日期、天数、标签里的数，没设遮罩的列上的
|
||||
等值，那里其实一个字没换，阈值却跟着遮了，回答正文又照样写着它。现在只有别处确实换掉了才不留。"""
|
||||
assert inquiry_mod._scrub_sql(sql, ages, {"age"}, "sqlite") == sql
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite", {"age"})], ages) == shown
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, sealed", [
|
||||
# 找某一个年龄的等值把 30 换掉了：阈值留着就把它露了
|
||||
("SELECT COUNT(*) AS 人数 FROM sales WHERE age >= 30 AND age = 30",
|
||||
"SELECT COUNT(*) AS 人数 FROM sales WHERE age >= 已遮罩 AND age = 已遮罩"),
|
||||
("SELECT COUNT(*) AS 人数 FROM sales WHERE age > 31 OR age IN ('31', '99')",
|
||||
"SELECT COUNT(*) AS 人数 FROM sales WHERE age > 已遮罩 OR age IN ('已遮罩', '99')"),
|
||||
])
|
||||
def test_a_threshold_goes_when_the_same_number_is_masked_elsewhere(ages, sql, sealed):
|
||||
assert inquiry_mod._scrub_sql(sql, ages, {"age"}, "sqlite") == sealed
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite", {"age"})], ages) == set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, sealed", [
|
||||
("SELECT COUNT(*) AS 人数 FROM sales WHERE age = 30 AND visits >= 30",
|
||||
"SELECT COUNT(*) AS 人数 FROM sales WHERE age = 已遮罩 AND visits >= 30"),
|
||||
("SELECT region FROM sales WHERE age IN (30, 31) AND visits > 30 LIMIT 31",
|
||||
"SELECT region FROM sales WHERE age IN (已遮罩, 已遮罩) AND visits > 30 LIMIT 31"),
|
||||
# 开区间夹住的 25 两头都换了，别的列上的 25 照原样
|
||||
("SELECT COUNT(*) AS 人数 FROM sales WHERE age > 24 AND age < 26 AND visits >= 25",
|
||||
"SELECT COUNT(*) AS 人数 FROM sales WHERE age > 已遮罩 AND age < 已遮罩 AND visits >= 25"),
|
||||
])
|
||||
def test_a_short_number_masked_elsewhere_leaves_other_columns_and_limits_alone(ages, sql, sealed):
|
||||
"""同一个短的数在别处换掉了，别的列上的阈值、LIMIT 撞上它照原样（已知局限）：短的数本来就不按值遮，撞上只是巧合，
|
||||
一律换会把阈值、行数改得面目全非（第 2 轮起「age = 25 AND amount > 25 … LIMIT 25」就照原样留着阈值和行数）。
|
||||
正文里不免遮——反正短的数正文里也不按值遮。够长的数在别的列上照样换（visits = 20000 那条）。"""
|
||||
assert inquiry_mod._scrub_sql(sql, ages, {"age"}, "sqlite") == sealed
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite", {"age"})], ages) == set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, sealed", [
|
||||
# 批次号里的 20000 是日期一样的写法，按值认不出、没换：阈值照原样
|
||||
("SELECT COUNT(*) FROM sales WHERE amount >= 20000 AND batch = '20000-01'",
|
||||
"SELECT COUNT(*) FROM sales WHERE amount >= 20000 AND batch = '20000-01'"),
|
||||
# 够长的遮罩值在别的列的等值里也换：换掉了，阈值也不留
|
||||
("SELECT COUNT(*) FROM sales WHERE amount >= 20000 AND visits = 20000",
|
||||
"SELECT COUNT(*) FROM sales WHERE amount >= 已遮罩 AND visits = 已遮罩"),
|
||||
])
|
||||
def test_long_numbers_go_by_what_was_actually_masked_elsewhere(amounts, sql, sealed):
|
||||
assert inquiry_mod._scrub_sql(sql, amounts, {"amount"}, "sqlite") == sealed
|
||||
|
||||
|
||||
def test_numbers_shown_across_queries_go_by_what_was_actually_masked(amounts):
|
||||
"""几条查询之间同理：别的查询里同一个数只写在批次号里、没换，正文里复述阈值照样不遮；别的查询里确实换掉了（和
|
||||
金额做等值比较、分词都分不了整条按值换），正文里也遮。"""
|
||||
shown = inquiry_mod._shown_literals
|
||||
threshold = ("SELECT COUNT(*) FROM sales WHERE amount >= 20000 LIMIT 5", "sqlite", {"amount"})
|
||||
assert shown([threshold, ("SELECT COUNT(*) FROM sales WHERE batch = '20000-01'", "sqlite", {"amount"})],
|
||||
amounts) == {"20000", "5"}
|
||||
assert shown([threshold, ("SELECT COUNT(*) FROM sales WHERE amount = 20000", "sqlite", {"amount"})],
|
||||
amounts) == {"5"}
|
||||
assert shown([threshold, ("SELECT 'x FROM sales WHERE amount = 20000", "sqlite", {"amount"})], amounts) == {"5"}
|
||||
|
||||
|
||||
def test_a_threshold_named_in_an_alias_or_a_comment_stays(amounts):
|
||||
@@ -344,7 +952,7 @@ def test_a_threshold_named_in_an_alias_or_a_comment_stays(amounts):
|
||||
sql = "SELECT SUM(amount >= 30000) AS 超过30000的笔数 FROM sales -- 30000 以上算大单"
|
||||
assert inquiry_mod._scrub_sql(sql, amounts, {"amount"}, "sqlite") == (
|
||||
"SELECT SUM(amount >= 30000) AS 超过已遮罩的笔数 FROM sales -- 已遮罩 以上算大单")
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite")]) == {"30000"}
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite", {"amount"})], amounts) == {"30000"}
|
||||
|
||||
|
||||
def test_sql_starting_with_blank_lines_is_read_at_the_right_places():
|
||||
@@ -354,7 +962,7 @@ def test_sql_starting_with_blank_lines_is_read_at_the_right_places():
|
||||
sql = "\n SELECT COUNT(*) FROM sales WHERE amount >= 30000 AND buyer_phone IN (13900000001, 1) LIMIT 30000"
|
||||
assert inquiry_mod._scrub_sql(sql, secrets, {"buyer_phone", "amount"}, "sqlite") == (
|
||||
"\n SELECT COUNT(*) FROM sales WHERE amount >= 30000 AND buyer_phone IN (已遮罩, 1) LIMIT 30000")
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite")]) == {"30000"}
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite", {"buyer_phone", "amount"})], secrets) == {"30000"}
|
||||
|
||||
|
||||
def test_string_range_bounds_are_masked_by_value(people):
|
||||
@@ -372,20 +980,22 @@ def test_sql_that_cannot_be_parsed_keeps_no_thresholds():
|
||||
sql = "SELECT COUNT(*) FROM sales WHERE age = 25 AND amount >= 30000 AND visits > 31 LIMIT 10000 GROUP"
|
||||
assert inquiry_mod._scrub_sql(sql, secrets, {"age", "amount"}, "sqlite") == (
|
||||
"SELECT COUNT(*) FROM sales WHERE age = 已遮罩 AND amount >= 已遮罩 AND visits > 31 LIMIT 已遮罩 GROUP")
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite")]) == set()
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite", {"age", "amount"})], secrets) == set()
|
||||
|
||||
|
||||
def test_only_thresholds_and_row_counts_count_as_shown():
|
||||
def test_only_thresholds_and_row_counts_count_as_shown(amounts):
|
||||
"""照原样留在 SQL 里的只有范围边界（没夹住某一个值的）和 LIMIT / OFFSET 的行数；选出列、函数参数、CASE、夹住
|
||||
某一个值的范围里的数不算，在别处（这一条或别的查询里）被换掉的数也不算。"""
|
||||
shown = inquiry_mod._shown_literals
|
||||
assert shown([("SELECT COUNT(*) FROM sales WHERE amount >= 30000 AND visits > 20000 LIMIT 10000 OFFSET 12000",
|
||||
"sqlite")]) == {"30000", "20000", "10000", "12000"}
|
||||
assert shown([("SELECT 50000 AS 门槛, SUM(CASE amount WHEN 12000 THEN 1 END), ROUND(amount, 10000) FROM sales "
|
||||
"WHERE amount BETWEEN 20000 AND 20000", "sqlite")]) == set()
|
||||
assert shown([("SELECT COUNT(*) FROM sales WHERE amount >= 30000", "sqlite"),
|
||||
("SELECT COUNT(*) FROM sales WHERE amount = 30000", "sqlite")]) == set()
|
||||
assert shown([("SELECT COUNT(*) FROM sales WHERE amount >= 30000 AND note LIKE '%30000%'", "sqlite")]) == set()
|
||||
def shown(*sqls: str) -> set[str]:
|
||||
return inquiry_mod._shown_literals([(sql, "sqlite", {"amount"}) for sql in sqls], amounts)
|
||||
|
||||
assert shown("SELECT COUNT(*) FROM sales WHERE amount >= 30000 AND visits > 20000 LIMIT 10000 OFFSET 12000") == {
|
||||
"30000", "20000", "10000", "12000"}
|
||||
assert shown("SELECT 50000 AS 门槛, SUM(CASE amount WHEN 12000 THEN 1 END), ROUND(amount, 10000) FROM sales "
|
||||
"WHERE amount BETWEEN 20000 AND 20000") == set()
|
||||
assert shown("SELECT COUNT(*) FROM sales WHERE amount >= 30000", "SELECT COUNT(*) FROM sales WHERE amount = 30000"
|
||||
) == set()
|
||||
assert shown("SELECT COUNT(*) FROM sales WHERE amount >= 30000 AND note LIKE '%30000%'") == set()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind, sql, shown", [
|
||||
@@ -415,8 +1025,8 @@ def test_numbers_left_as_written_in_the_sql_are_not_masked_in_the_answer(amounts
|
||||
"""范围比较的边界、LIMIT 照原样留在 SQL 里，回答正文里同样的数再遮也遮不住什么，只会和 SQL 对不上；等值比较
|
||||
里换掉的照旧遮。"""
|
||||
sql = "SELECT COUNT(*) FROM sales WHERE amount >= 30000 AND amount <> 50000 LIMIT 10000"
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite")]) == {"30000", "10000"}
|
||||
amounts.show(inquiry_mod._shown_literals([(sql, "sqlite")]))
|
||||
assert inquiry_mod._shown_literals([(sql, "sqlite", {"amount"})], amounts) == {"30000", "10000"}
|
||||
amounts.show(inquiry_mod._shown_literals([(sql, "sqlite", {"amount"})], amounts))
|
||||
assert amounts.scrub("金额不低于 30000 元、不是 50000 元的，取前 10000 条", prose=True) == (
|
||||
"金额不低于 30000 元、不是 已遮罩 元的，取前 10000 条")
|
||||
assert inquiry_mod._scrub_sql(sql, amounts, {"amount"}, "sqlite") == (
|
||||
@@ -430,7 +1040,7 @@ def test_a_threshold_left_in_the_sql_is_shown_in_the_prose_but_not_in_the_cells(
|
||||
"""SQL 里照原样留着的阈值只在回答正文里不遮：格子里的同一个数照旧按值遮（col >= v 再取第一行，查出来的可能就是
|
||||
那一笔），列名、SQL 里的名字也照旧换。组装出来正文里的不算露出，格子里渲染出来的算。"""
|
||||
sql = "SELECT region AS 区域, visits AS 访问 FROM sales WHERE visits >= 20000 ORDER BY visits LIMIT 1"
|
||||
amounts.show(inquiry_mod._shown_literals([(sql, "sqlite")]))
|
||||
amounts.show(inquiry_mod._shown_literals([(sql, "sqlite", {"amount"})], amounts))
|
||||
assert amounts.scrub("访问不少于 20000 次", prose=True) == "访问不少于 20000 次"
|
||||
assert amounts.scrub("访问不少于 20000 次") == "访问不少于 已遮罩 次"
|
||||
snapshot = {"columns": ["区域", "访问"], "rows": [["华东", 20000]]}
|
||||
@@ -490,6 +1100,129 @@ def test_an_answer_still_showing_a_masked_value_is_not_handed_out():
|
||||
assert doc["markdown"] == "没有遮罩值。"
|
||||
|
||||
|
||||
def test_a_masked_value_inside_a_text_cell_is_cut_out_of_the_artifact_view():
|
||||
"""质询工件这一路（embedded）：没设遮罩的列里一格文字夹着遮罩值，把那一截换成「已遮罩」，这一格别的字、别的格子
|
||||
照原样；这一列不记进 masked（界面会把记进去的列整列画成「已遮罩」，这一列在数据源里也没设遮罩）。门槛同回答
|
||||
一路：短的数不按值换。回答一路不这样换，渲染出来露了就整份不交（上一条测试）。"""
|
||||
secrets = secrets_of({"columns": ["phone", "age"], "rows": [["13900000001", 25]]}, {"phone", "age"})
|
||||
snapshot = {"columns": ["区域", "备注", "金额"], "column_types": {"金额": "number"},
|
||||
"rows": [["华东", "客户13900000001来电改址", 1200], ["华南", "例行回访，25 岁", 900]]}
|
||||
view, masked = inquiry_mod._masked_view(snapshot, set(), secrets, embedded=True)
|
||||
assert view["rows"] == [["华东", "客户已遮罩来电改址", 1200], ["华南", "例行回访，25 岁", 900]]
|
||||
assert masked == [] and view["column_types"] == {"金额": "number"}
|
||||
assert snapshot["rows"][0][1] == "客户13900000001来电改址", "原快照不动"
|
||||
as_dicts = {**snapshot, "rows": [{"区域": "华东", "备注": "客户13900000001来电改址", "金额": 1200}]}
|
||||
assert inquiry_mod._masked_view(as_dicts, set(), secrets, embedded=True)[0]["rows"] == [
|
||||
{"区域": "华东", "备注": "客户已遮罩来电改址", "金额": 1200}]
|
||||
# JSON 列（对象）按界面画出来的样子换，换过的照文字给
|
||||
nested = {"columns": ["联系人"], "rows": [[{"电话": "13900000001", "年龄": 25}]]}
|
||||
assert inquiry_mod._masked_view(nested, set(), secrets, embedded=True)[0]["rows"] == [
|
||||
['{"电话": "已遮罩", "年龄": 25}']]
|
||||
# 整格就是遮罩值的照旧整格换、记进 masked；按列遮的照旧
|
||||
whole = {"columns": ["联系方式", "备注"], "rows": [["13900000001", "回电 13900000001"]]}
|
||||
view, masked = inquiry_mod._masked_view(whole, set(), secrets, embedded=True)
|
||||
assert view["rows"] == [[MASKED, "回电 已遮罩"]] and masked == ["联系方式"]
|
||||
# 回答一路照旧：文字格不动，交给最后一道关
|
||||
assert inquiry_mod._masked_view(snapshot, set(), secrets)[0] is snapshot
|
||||
|
||||
|
||||
def test_sql_check_excerpts_in_the_artifact_are_masked_like_the_sql():
|
||||
"""质询工件里快照记下的 SQL 检查结果：片段（sql_excerpt）摘自同一条 SQL，照查询清单的 SQL 换掉遮罩值（日期这样的
|
||||
字也认得出是在找哪一天）；给模型的那句改法（for_model）照抄了片段，不交出去（同证据面板）。"""
|
||||
secrets = secrets_of({"columns": ["birth_date"], "rows": [["1990-05-01"]]}, {"birth_date"})
|
||||
sql = "SELECT SUM(balance) FROM customers c JOIN balances b ON b.cid = c.id WHERE birth_date = '1990-05-01'"
|
||||
message = "「客户」的业务日期按「注册日期」计，查询却按「生日」筛选"
|
||||
checks = [{"code": "wrong_date_column", "level": "warning", "table": "customers", "column": "birth_date",
|
||||
"message": message, "for_model": "这条 SQL 按 birth_date = '1990-05-01' 筛选",
|
||||
"sql_excerpt": "birth_date = '1990-05-01'"},
|
||||
{"code": "stock_summed", "level": "info", "table": "balances", "message": "存量跨日期求和",
|
||||
"sql_excerpt": "SUM(balance)"}]
|
||||
assert inquiry_mod._masked_checks(checks, sql, secrets, {"birth_date"}, "sqlite") == [
|
||||
{"code": "wrong_date_column", "level": "warning", "table": "customers", "column": "birth_date",
|
||||
"message": message, "sql_excerpt": "birth_date = '已遮罩'"},
|
||||
{"code": "stock_summed", "level": "info", "table": "balances", "message": "存量跨日期求和",
|
||||
"sql_excerpt": "SUM(balance)"}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, excerpt, shown", [
|
||||
# 片段只是一段表达式，单独拿出来认不出在找哪一个值：按整条 SQL 里换掉的字面量换
|
||||
("SELECT SUM(CASE age WHEN 25 THEN balance END) AS 余额 FROM accounts",
|
||||
"SUM(CASE age WHEN 25 THEN balance END)", "SUM(CASE age WHEN 已遮罩 THEN balance END)"),
|
||||
("SELECT SUM(balance) AS 余额 FROM accounts WHERE (age, region) = (25, '华东')",
|
||||
"(age, region) = (25, '华东')", "(age, region) = (已遮罩, '华东')"),
|
||||
("SELECT SUM(CASE WHEN age BETWEEN 25 AND 25 THEN balance END) AS 余额 FROM accounts",
|
||||
"SUM(CASE WHEN age BETWEEN 25 AND 25 THEN balance END)",
|
||||
"SUM(CASE WHEN age BETWEEN 已遮罩 AND 已遮罩 THEN balance END)"),
|
||||
# 整条 SQL 里照原样留着的阈值，片段里也照原样
|
||||
("SELECT SUM(CASE WHEN amount >= 20000 THEN balance END) AS 余额 FROM accounts",
|
||||
"SUM(CASE WHEN amount >= 20000 THEN balance END)", "SUM(CASE WHEN amount >= 20000 THEN balance END)"),
|
||||
# 检查器截过的片段：截在号码中间，那半截按值认不出，去掉
|
||||
("SELECT SUM(balance) AS 余额 FROM accounts WHERE region = '华东' AND phone = '13900000001'",
|
||||
"region = '华东' AND phone = '1390000…", "region = '华东' AND phone = …"),
|
||||
])
|
||||
def test_sql_check_excerpts_follow_what_the_whole_sql_masked(sql, excerpt, shown):
|
||||
secrets = secrets_of({"columns": ["phone", "age", "amount"], "rows": [["13900000001", 25, 20000]]},
|
||||
{"phone", "age", "amount"})
|
||||
checks = [{"code": "stock_summed", "level": "warning", "table": "accounts", "message": "存量跨日期求和",
|
||||
"sql_excerpt": excerpt}]
|
||||
[check] = inquiry_mod._masked_checks(checks, sql, secrets, {"phone", "age", "amount"}, "sqlite")
|
||||
assert check["sql_excerpt"] == shown
|
||||
|
||||
|
||||
def test_sql_check_messages_do_not_quote_masked_values():
|
||||
"""检查结果的说明照引 SQL 里写的取值（码值表已列全时「没有码值「3」」）：整条 SQL 里换掉的取值，说明里一样换成
|
||||
「已遮罩」，短的码值、文字码值都一样；码值表里列出的码值是数据目录写明的，照原样。"""
|
||||
secrets = secrets_of({"columns": ["status", "channel_type"], "rows": [[3, "线下合作"], [1, "线上"]]},
|
||||
{"status", "channel_type"})
|
||||
sql = ("SELECT COUNT(*) FROM visits v JOIN channels c ON c.id = v.cid WHERE status IN (3, -3) "
|
||||
"AND channel_type = '线下合作'")
|
||||
listed = "（数据目录中已列出的全部码值：1=有效、0=作废）。请核对筛选条件里的取值"
|
||||
checks = [{"code": "unknown_code", "level": "warning", "table": "visits", "column": "status",
|
||||
"message": f"「入园记录」的「检票状态」没有码值「3」、「-3」{listed}", "sql_excerpt": "status IN (3, -3)"},
|
||||
{"code": "unknown_code", "level": "warning", "table": "channels", "column": "channel_type",
|
||||
"message": f"「渠道」的「渠道类型」没有码值「线下合作」{listed}",
|
||||
"sql_excerpt": "channel_type = '线下合作'"}]
|
||||
out = inquiry_mod._masked_checks(checks, sql, secrets, {"status", "channel_type"}, "sqlite")
|
||||
assert [c["message"] for c in out] == [f"「入园记录」的「检票状态」没有码值「已遮罩」、「-已遮罩」{listed}",
|
||||
f"「渠道」的「渠道类型」没有码值「已遮罩」{listed}"]
|
||||
assert [c["sql_excerpt"] for c in out] == ["status IN (已遮罩, -已遮罩)", "channel_type = '已遮罩'"]
|
||||
|
||||
|
||||
def test_codes_listed_in_a_check_message_are_masked_by_value_like_any_text():
|
||||
"""说明里码值表列出的码值是数据目录写明的，不从 SQL 摘，也不另外免遮：和说明里别的字一样按值换，恰好是按值认得出
|
||||
的遮罩值（两个字以上的文字、够长的数）的码值换成「已遮罩」（多遮，不漏），短的码值、别的码值照原样。"""
|
||||
secrets = secrets_of({"columns": ["channel_type", "status"], "rows": [["线上直销", 1], ["分销", 0]]},
|
||||
{"channel_type", "status"})
|
||||
message = ("「渠道」的「渠道类型」没有码值「线下合作」（数据目录中已列出的全部码值：线上直销=官网和小程序、"
|
||||
"线下=窗口售票、分销=旅行社、1=有效）。请核对筛选条件里的取值")
|
||||
checks = [{"code": "unknown_code", "level": "warning", "table": "channels", "column": "channel_type",
|
||||
"message": message, "sql_excerpt": "channel_type = '线下合作'"}]
|
||||
[check] = inquiry_mod._masked_checks(checks, "SELECT COUNT(*) FROM channels WHERE channel_type = '线下合作'",
|
||||
secrets, {"channel_type", "status"}, "sqlite")
|
||||
assert check["message"] == ("「渠道」的「渠道类型」没有码值「线下合作」（数据目录中已列出的全部码值：已遮罩=官网和小程序、"
|
||||
"线下=窗口售票、已遮罩=旅行社、1=有效）。请核对筛选条件里的取值")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("excerpt, shown", [
|
||||
# 截在带空格的地址中间：从没闭合的引号起整个去掉，不只去掉最后一段
|
||||
("region = '华东' AND addr = '上海市 浦东新区 某…", "region = '华东' AND addr = …"),
|
||||
# 写成两个单引号的引号不算闭合
|
||||
("region = 'O''Neil 的店' AND addr = '上海市 浦东…", "region = 'O''Neil 的店' AND addr = …"),
|
||||
# 截在带引号的名字、注释中间也一样
|
||||
('region = \'华东\' AND "上海市 浦东新区…', "region = '华东' AND …"),
|
||||
("region = '华东' /* 上海市 浦东…", "region = '华东' …"),
|
||||
])
|
||||
def test_a_cut_off_excerpt_drops_the_whole_unfinished_value(excerpt, shown):
|
||||
"""检查器截过的片段停在一个带空格的遮罩文字值中间（地址）：以前只去掉「…」前最后一段连续的字，前面那几段还在；
|
||||
现在从没闭合的引号、注释起整个去掉。"""
|
||||
secrets = secrets_of({"columns": ["addr"], "rows": [["上海市 浦东新区 某某路 1 号"]]}, {"addr"})
|
||||
sql = "SELECT SUM(balance) AS 余额 FROM accounts WHERE region = '华东' AND addr = '上海市 浦东新区 某某路 1 号'"
|
||||
checks = [{"code": "stock_summed", "level": "warning", "table": "accounts", "message": "存量跨日期求和",
|
||||
"sql_excerpt": excerpt}]
|
||||
[check] = inquiry_mod._masked_checks(checks, sql, secrets, {"addr"}, "sqlite")
|
||||
assert check["sql_excerpt"] == shown
|
||||
|
||||
|
||||
def test_report_values_that_happen_to_be_masked_values_are_shown():
|
||||
"""报告口径卡里的指标恰好和遮罩列里的某个值一样（单笔目标 20000，金额也有 20000 的单）：报告里本来就看得到，
|
||||
引用它的回答照常交出去。"""
|
||||
@@ -541,3 +1274,28 @@ def test_twenty_thousand_masked_values_are_scrubbed_without_stalling():
|
||||
elapsed = (time.perf_counter() - started) / 5
|
||||
assert "13900000042" not in out and "客户00042" not in out and "近 30 天" in out
|
||||
assert elapsed < 0.5, f"换一遍用了 {elapsed:.2f} 秒"
|
||||
|
||||
|
||||
def test_a_long_chain_of_conditions_is_still_read_by_its_structure(ages):
|
||||
"""上千个条件连成一长串（一层套一层的且）：照样按条件合起来看，夹住的值照样换，不因为层数太深退回到按词认——
|
||||
按词认不出范围夹住了值，短的数就漏了。"""
|
||||
chain = " AND ".join(f"visits{i} >= {i}" for i in range(1500))
|
||||
sql = f"SELECT COUNT(*) FROM sales WHERE age >= 25 AND age <= 25 AND {chain}"
|
||||
assert inquiry_mod._literal_uses(sql, "sqlite") is not None
|
||||
assert inquiry_mod._scrub_sql(sql, ages, {"age"}, "sqlite") == sql.replace(
|
||||
"age >= 25 AND age <= 25", "age >= 已遮罩 AND age <= 已遮罩")
|
||||
|
||||
|
||||
def test_hundreds_of_band_edges_are_read_within_a_budget():
|
||||
"""成百个分档边界连成一长串：看得过来的照条件合起来认（一百个相邻的分档连成一段，谁也没夹住）；多到看不过来的，
|
||||
两边都隔开就按夹住算（宁可多换），也不会一个个细看算上好几秒。年龄从 0 到 599 都有，按夹住算的哪一档都会换。"""
|
||||
every = secrets_of({"columns": ["age"], "rows": [[i] for i in range(600)]}, {"age"})
|
||||
|
||||
def bins(n: int) -> str:
|
||||
return "SELECT COUNT(*) FROM sales WHERE " + " OR ".join(f"(age >= {i} AND age < {i + 1})" for i in range(n))
|
||||
|
||||
started = time.perf_counter()
|
||||
assert inquiry_mod._scrub_sql(bins(100), every, {"age"}, "sqlite") == bins(100)
|
||||
assert "已遮罩" in inquiry_mod._scrub_sql(bins(600), every, {"age"}, "sqlite")
|
||||
elapsed = time.perf_counter() - started
|
||||
assert elapsed < 5, f"用了 {elapsed:.2f} 秒"
|
||||
|
||||
Reference in New Issue
Block a user