Files
AgentLab/backend/tests/test_masking_tool_boundary.py
T
ThearchyHelios 8c229c3d1a K3b:分档写在派生来源里、外层按它分组的标签列照常显示,和当分组键的 CASE 一样
以前分档写在子查询、WITH 里(SELECT *, CASE … END AS 档位 FROM sales),外层再按档位分组,工具边界把档位整列遮成「已遮罩」,模型分不清哪个数属于哪一档,质询回答成了「已遮罩有 3 笔,已遮罩有 2 笔」。现在 _derived_hidden 认出只装着标签(_label_columns)、又一档一行的结果列(这一层或派生来源里那一层的 GROUP BY 键,按位置、别名、照原样写都算,_grouped_labels),不算取自遮罩列:格子里只有标签、和遮罩列不同名才算,认不出的照旧整列遮;SQL 里照旧认得出它取自金额。工具边界、质询回答、质询工件用的是同一套。* 展开的表在冻结的表结构里查得到、没有同名列时标签照常认(原来就有),不知道表结构时照旧当它带得出(多遮)。测试 test_inquiry_masking 里「外层再分组照旧算」的两个用例按新规则改;补六种写法的工具边界用例、逐行和混了别的值的反例、质询端到端用例,去掉豁免后全部变红。_tainted 多交回一个「认不出、整份都算」的标记,整份遮时不豁免。
2026-10-06 22:52:28 +08:00

256 lines
16 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""工具边界先遮(engine/masking.py 的 mask_query_result)的收口:认不准就整份遮,多遮也不改写没设遮罩的数值证据。
- 遮罩分析本身出了意外(认遮罩列、标签列、逐格换值的哪一步都算):整份结果的每一格都是「已遮罩」,交给模型的、落库的
都一样,masked_at_query 记全部的列。兜底不再经出意外的那一步(以前兜底又调了一遍逐格换值的那一步,它出意外时兜底跟着
出意外,结果没交出去,也没有测试证明「整份遮」这句保证)
- 按值遮只换文字格:没设遮罩的数值列(访问次数正好等于某一笔被遮的金额)原样,存下的证据和数据库里的一样,双盲复算照常
取得到;文字格里整格就是遮罩值、夹着遮罩值的照换,换过的格子记进快照(masked_cells),格子引用指到它们照实说是遮罩。
占位的短词(未知、暂无……)不当遮罩值,没设遮罩的列、备注里的同一个词照原样
模型一律是 MockChatModel 剧本,数据源是现造的 SQLite 文件,手机号、姓名、金额、访问次数都是虚构的。
"""
from __future__ import annotations
import json
import pytest
from httpx import ASGITransport, AsyncClient
from app.core import artifact_store
from app.data.engine import engines
from app.engine import masking
from app.main import app
from tests.test_verify_agent import Script, q
from tests.test_verify_agent_masking import ( # noqa: F401 - clean_spend 是自动用上的夹具
MASKED,
SALES,
VALUE_SCHEMA,
_cells,
_clean,
_source,
ask,
clean_spend,
masked_shop,
)
# --------------------------------------------------------------------------
# 分析出意外:整份遮
# --------------------------------------------------------------------------
def _broken(*args, **kwargs):
raise RuntimeError("遮罩分析里的意外")
PAYLOAD = {"sql": "SELECT id, buyer_phone, note FROM sales", "columns": ["id", "buyer_phone", "note"],
"rows": [[1, "13900000001", "客户来电"], [2, "13900000002", "例行回访"]],
"column_types": {"id": "number", "buyer_phone": "text", "note": "text"}}
@pytest.mark.parametrize("step", ["_derived_hidden", "_snapshot_labels", "_masked_view"])
def test_a_failing_analysis_masks_every_cell(monkeypatch, step):
"""认遮罩列、认标签列、逐格换值,哪一步出了意外都整份遮:每一格都是「已遮罩」、类型记成文本,整列遮掉的是全部的列。"""
monkeypatch.setattr(masking, step, _broken)
done = masking.mask_query_result(json.loads(json.dumps(PAYLOAD)), ["buyer_phone"], "sqlite")
assert done.payload["rows"] == [[MASKED] * 3, [MASKED] * 3]
assert done.payload["column_types"] == {"id": "text", "buyer_phone": "text", "note": "text"}
assert done.columns == ["id", "buyer_phone", "note"]
assert done.payload["sql"] == PAYLOAD["sql"], "SQL 是模型自己写的,照原样"
assert not done.secrets.strong, "认不准,不收值"
def test_a_failing_analysis_masks_rows_of_any_shape(monkeypatch):
"""行是字典的结果同样整份遮。"""
monkeypatch.setattr(masking, "_masked_view", _broken)
payload = {"sql": "SELECT id, buyer_phone FROM sales", "columns": ["id", "buyer_phone"],
"rows": [{"id": 1, "buyer_phone": "13900000001"}]}
done = masking.mask_query_result(payload, ["buyer_phone"], "sqlite")
assert done.payload["rows"] == [{"id": MASKED, "buyer_phone": MASKED}]
@pytest.mark.parametrize("step", ["_derived_hidden", "_masked_view"])
async def test_the_model_and_the_snapshot_see_only_masks_when_the_analysis_fails(monkeypatch, masked_shop, step):
"""端到端:分析出了意外,交给模型的工具结果、落库的快照、通用接口给的都只有「已遮罩」,masked_at_query 是全部的列;
格子引用照实说是遮罩列,不采信。"""
monkeypatch.setattr(masking, step, _broken)
name = masked_shop
sql = "SELECT id, region, buyer_phone AS 手机号, amount FROM sales ORDER BY id"
script = Script(monkeypatch, [q(name, sql, "c1")], {"value": {"alias": "Q1", "row": 0, "column": "amount"},
"rationale": "x"})
out, _ = await ask(name)
seen = json.loads(script.tool_results()["c1"].split("\n", 1)[1])
assert seen["rows"] == [[MASKED] * 4] * 4, seen
snapshot = artifact_store.load(out.queries[0]["artifact"])
assert snapshot["rows"] == [[MASKED] * 4] * 4
assert snapshot["masked_at_query"] == ["id", "region", "手机号", "amount"]
assert _clean(json.dumps(snapshot, ensure_ascii=False))
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
got = await client.get(f"/api/artifacts/{out.queries[0]['artifact']}")
assert got.status_code == 200 and _clean(got.text) and "1200" not in got.text
assert out.result["value"] is None
assert out.cells[0]["reason"] == "Q1 的「amount」取自设置了遮罩的列,查询时已遮罩,不采信"
# --------------------------------------------------------------------------
# 多遮不改写没设遮罩的数值证据;占位的短词不当遮罩值
# --------------------------------------------------------------------------
#: 金额设了遮罩,都是五位数;访问次数没设遮罩,第二、三单正好等于别的订单的金额(20000、30000)。联系表的访问次数存成
#: 文字('20000'),客户名设了遮罩、里面有占位的「未知」,区域和备注里也写着「未知」
BIG = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, amount INTEGER, visits INTEGER, buyer_name TEXT,"
" note TEXT);"
"INSERT INTO sales VALUES (1, '华东', 10000, 25000, '张伟', '原因未知'), (2, '华东', 12000, 20000, '未知', ''),"
" (3, '华南', 20000, 30000, '李娜', '客户未知来源'), (4, '未知', 30000, 22000, '王芳', ''),"
" (5, '华北', 50000, 9000, '刘洋', '');"
"CREATE TABLE contacts (id INTEGER PRIMARY KEY, visits_text TEXT, remark TEXT);"
"INSERT INTO contacts VALUES (1, '20000', '上月消费 30000 元'), (2, '8', '例行回访');")
@pytest.fixture
async def big_shop(tmp_path):
name, source_id = await _source(tmp_path, {"mask_columns": "amount, buyer_name"}, script=BIG)
yield name
await engines.invalidate(source_id)
async def test_an_unmasked_numeric_column_is_kept_even_when_it_equals_a_masked_value(monkeypatch, big_shop):
"""访问次数没设遮罩,第二、三单正好等于别的订单被遮的金额:交给模型的、落库的都是数据库里的原数,类型照旧是数,
引用它照常取值(双盲复算取得到 20000)。以前按值整格换成「已遮罩」,快照里不留记录,引用时说成「不是数值」。"""
name = big_shop
sql = "SELECT id, amount, visits FROM sales ORDER BY id"
script = Script(monkeypatch, [q(name, sql, "c1")], {"value": {"alias": "Q1", "row": 1, "column": "visits"},
"rationale": "x"})
out, _ = await ask(name)
seen = json.loads(script.tool_results()["c1"].split("\n", 1)[1])
assert seen["rows"] == [[1, MASKED, 25000], [2, MASKED, 20000], [3, MASKED, 30000], [4, MASKED, 22000],
[5, MASKED, 9000]]
snapshot = artifact_store.load(out.queries[0]["artifact"])
assert snapshot["rows"] == seen["rows"] and snapshot["column_types"]["visits"] == "number"
assert snapshot["masked_at_query"] == ["amount"] and "masked_cells" not in snapshot
assert out.result["value"] == 20000 and out.cells[0]["status"] == "resolved"
async def test_text_cells_masked_by_value_are_recorded_and_cited_as_masked(monkeypatch, big_shop):
"""存成文字的数('20000')、备注里夹着的金额(30000)照样按值换:交给模型的、落库的都换掉,换过的格子记进快照
(masked_cells),引用到它们照实说是遮罩,不说成「不是数值」。"""
name = big_shop
steps = [q(name, "SELECT amount FROM sales ORDER BY id", "c1"),
q(name, "SELECT id, visits_text, remark FROM contacts ORDER BY id", "c2")]
script = Script(monkeypatch, steps, lambda messages: {"value": {"alias": "Q2", "row": 0, "column": "visits_text"},
"rationale": "x"})
out, _ = await ask(name)
seen = json.loads(script.tool_results()["c2"].split("\n", 1)[1])
assert seen["rows"] == [[1, MASKED, f"上月消费 {MASKED} 元"], [2, "8", "例行回访"]]
assert "masked_cells" not in seen, "记录只在快照里,不交给模型"
snapshot = artifact_store.load(out.queries[1]["artifact"])
assert snapshot["rows"] == seen["rows"] and snapshot["masked_cells"] == [[0, 1], [0, 2]]
assert snapshot["masked_at_query"] == []
assert out.result["value"] is None
assert out.cells[0]["reason"] == "Q2 第 0 行「visits_text」与设置了遮罩的值相同,查询时已遮罩,不采信"
async def test_a_partly_masked_text_cell_is_cited_as_masked(monkeypatch, big_shop):
"""引用指到文字里换掉了一截的格子:同样照实说是遮罩。"""
name = big_shop
steps = [q(name, "SELECT amount FROM sales ORDER BY id", "c1"),
q(name, "SELECT id, remark FROM contacts ORDER BY id", "c2")]
Script(monkeypatch, steps, {"value": {"alias": "Q2", "row": 0, "column": "remark"}, "rationale": "x"})
out, _ = await ask(name)
assert out.result["value"] is None
assert out.cells[0]["reason"] == "Q2 第 0 行「remark」里有与设置了遮罩的值相同的部分,查询时已遮罩,不采信"
async def test_placeholder_words_are_not_masked_elsewhere(monkeypatch, big_shop):
"""客户名设了遮罩,里面有占位的「未知」:客户名整列照旧遮;没设遮罩的区域里的「未知」、备注里的「原因未知」「客户
未知来源」照原样。以前「未知」当遮罩值收了,这几格都被换掉。"""
name = big_shop
sql = "SELECT id, buyer_name, region, note FROM sales ORDER BY id"
script = Script(monkeypatch, [q(name, sql, "c1")], {"value": {"alias": "Q1", "row": 0, "column": "id"},
"rationale": "x"})
out, _ = await ask(name)
seen = json.loads(script.tool_results()["c1"].split("\n", 1)[1])
assert [row[1] for row in seen["rows"]] == [MASKED] * 5
assert [row[2:] for row in seen["rows"]] == [["华东", "原因未知"], ["华东", ""], ["华南", "客户未知来源"],
["未知", ""], ["华北", ""]]
assert out.masked_values.hit("张伟") and not out.masked_values.hit("未知")
def test_placeholders_are_never_collected():
secrets = masking._Secrets()
secrets.add({"columns": ["name"], "rows": [["未知"], ["N/A"], ["暂无"], ["张伟"], ["NULL"]]}, {"name"})
assert secrets.strong == {"张伟"} and secrets.weak == {"张伟"}
assert secrets.scrub("原因未知,暂无记录,N/A,张伟") == f"原因未知,暂无记录,N/A,{MASKED}"
# --------------------------------------------------------------------------
# 分档写在派生来源里、外层按它分组:标签列照常显示,标签不当遮罩值
# --------------------------------------------------------------------------
LAYOUT = ["id", "region", "amount", "visits"]
#: 探查得到的表结构:sales 里没有「档位」这一列
SCHEMA = {"tables": {"sales": {"qualified": "sales", "schema": None, "columns": [{"name": c} for c in LAYOUT]}}}
BAND = "CASE WHEN amount >= 20000 THEN '大单' ELSE '小单' END"
GROUPED = [
pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT *, {BAND} AS 档位 FROM sales) GROUP BY 档位 ORDER BY 档位",
id="子查询里 * 加分档"),
pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT s.*, {BAND} AS 档位 FROM sales s) t GROUP BY 档位 "
"ORDER BY 档位", id="子查询里 s.* 加分档"),
pytest.param(f"WITH b AS (SELECT *, {BAND} AS 档位 FROM sales) SELECT 档位, COUNT(*) AS 笔数 FROM b GROUP BY 档位 "
"ORDER BY 档位", id="WITH 里 * 加分档"),
pytest.param(f"WITH b AS (SELECT {BAND} AS 档位 FROM sales) SELECT 档位, COUNT(*) AS 笔数 FROM b GROUP BY 档位 "
"ORDER BY 档位", id="WITH 里只有分档"),
pytest.param(f"WITH b AS (SELECT {BAND} AS 档位 FROM sales) SELECT b.档位, COUNT(*) AS 笔数 FROM b GROUP BY 1 "
"ORDER BY 1", id="按位置分组"),
pytest.param(f"WITH b AS (SELECT {BAND} AS 档位 FROM sales), c AS (SELECT 档位, COUNT(*) AS 笔数 FROM b "
"GROUP BY 档位) SELECT * FROM c ORDER BY 档位", id="分组在中间一层"),
]
ROWS = [["大单", 3], ["小单", 2]]
@pytest.mark.parametrize("sql", GROUPED)
def test_a_band_grouped_outside_its_derived_source_is_shown(sql):
"""分档(按遮罩的金额写死的标签)写在子查询、WITH 里,外层按它分组:每档一行,档位列照常显示,标签不当遮罩值收;
以前整列遮成「已遮罩」,模型分不清哪个数属于哪一档。"""
payload = {"sql": sql, "columns": ["档位", "笔数"], "rows": [list(r) for r in ROWS],
"column_types": {"档位": "text", "笔数": "number"}}
done = masking.mask_query_result(payload, ["amount"], "sqlite", schema=SCHEMA)
assert done.payload["rows"] == ROWS and done.columns == []
assert not done.secrets.strong
hidden, names = masking._derived_hidden({**payload, "rows": ROWS}, {"amount"}, "sqlite",
masking._schema_columns(SCHEMA))
assert "档位" not in hidden and "档位" in names, "SQL 里照旧认得出它取自金额"
@pytest.mark.parametrize("sql", [
pytest.param(f"SELECT id, 档位 FROM (SELECT id, {BAND} AS 档位 FROM sales) ORDER BY id", id="逐行的档位"),
pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT {BAND} AS 档位 FROM sales UNION ALL "
"SELECT region FROM sales) GROUP BY 档位", id="另一支是区域"),
pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT id, {BAND} AS 档位 FROM sales) x "
"FULL JOIN (SELECT id, amount AS 档位 FROM sales) y USING (档位) GROUP BY 档位", id="按档位连起来"),
])
def test_a_band_that_is_not_one_row_per_band_is_still_masked(sql):
"""逐行的档位(看得出每一笔落在哪一档)、和别的值混在一列里、按它连起来又不写表名的(外连接时取的可能是另一边的
金额),照旧整列遮。"""
payload = {"sql": sql, "columns": ["档位", "笔数"] if "笔数" in sql else ["id", "档位"],
"rows": [["大单", 3], ["小单", 2]] if "笔数" in sql else [[1, "小单"], [3, "大单"]]}
done = masking.mask_query_result(payload, ["amount"], "sqlite", schema=SCHEMA)
assert "档位" in done.columns
def test_a_grouped_band_with_cells_other_than_its_labels_is_still_masked():
"""格子里有标签以外的值(认错了、数的写法对不上):照旧整列遮。"""
sql = GROUPED[3].values[0]
payload = {"sql": sql, "columns": ["档位", "笔数"], "rows": [["大单", 3], ["13900000001", 2]]}
assert masking.mask_query_result(payload, ["amount"], "sqlite", schema=SCHEMA).columns == ["档位"]
def test_without_a_schema_a_star_next_to_the_band_is_still_masked():
"""不知道表结构时,* 展开的表可能就有「档位」这一列(外层取到的是它):照旧整列遮,标签照收(多遮)。"""
sql = GROUPED[0].values[0]
payload = {"sql": sql, "columns": ["档位", "笔数"], "rows": [list(r) for r in ROWS]}
done = masking.mask_query_result(payload, ["amount"], "sqlite")
assert done.columns == ["档位"] and done.payload["rows"] == [[MASKED, 3], [MASKED, 2]]