合入期 5 前置修复 K3b:工具边界遮罩收口——分析出意外、(表达式).* 展开、把 SQL 或表名写成字符串参数的函数、记着别的会话 SQL 的系统视图与 SHOW / EXPLAIN、当一张表用的 VALUES 读到遮罩列时整份遮;按值遮只换文字格、数值证据原样并记下换过的格子;分档标签一档一行时照常显示;交给核查 agent 的证据摘录先遮,反方理由加最后一道关;长 CTE 链的分析有缓存与步数上限

This commit is contained in:
ThearchyHelios
2026-10-12 13:12:10 +08:00
16 changed files with 3384 additions and 279 deletions
+7 -37
View File
@@ -39,6 +39,7 @@ from urllib.parse import quote
from app.core.artifact_store import canonical_json, content_hash
from app.core.events import EventType
from app.engine.evidence import iter_units, query_entry_fields
from app.engine.masking import OPAQUE_SQL, SYSTEM_SCHEMAS, indirect_reads
from app.engine.merge_query import MERGE_SOURCE
from app.engine.recompute import input_origin
from app.engine.schema import NodeType
@@ -231,43 +232,12 @@ _QUERY_KEYS = frozenset({"columns", "rows", "row_count", "truncated", "elapsed_m
#: 导出时只知道数据源设的遮罩、不知道是哪种库:SQL 按每种支持的库的读法各解析一遍(None 是 sqlglot 的通用读法),
#: 至少一种读得通,读得通的每一种都要过关
_READINGS = (None, "sqlite", "mysql", "postgres", "oracle")
#: 语法树和数据库实际执行的对不上的写法:MySQL、MariaDB 会执行的注释(/*! … */、/*M! … */,解析器当注释扔掉),
#: Postgres 按 Unicode 转义写的名字(U&"…",原文里看不出是哪一列)
_OPAQUE_SQL = re.compile(r'/\*m?!|u&"', re.I)
#: 不点列名也读得出表里内容的写法。SQL 原文里出现就按碰了遮罩列算,写在字符串里的也算(宁可多挡)。函数要跟着左括号
#: 才算(列名碰巧叫 crosstab 的不算),表、视图、schema 按名字认
_INDIRECT_READS = re.compile(r"(?<![0-9A-Za-z_$])(?:(?:" + "|".join((
# 表名、SQL 写在字符串里交给函数执行或整表导出:Postgres 的 XML 一族、ts_stat、tablefunc 的 crosstab / connectby、
# dblink,Oracle 的 XMLTYPE、XQuery(XMLQUERY、XMLTABLE)
r"(?:query|table|cursor|schema|database)_to_xml\w*", r"ts_stat", r"crosstab\d*", r"connectby", r"dblink\w*",
r"xmltype", r"xmlquery", r"xmltable",
# 按页、按文件、按日志读出的原始数据:Postgres 的 pageinspect(按索引名读 B 树页的 bt_page_items 也算)、服务器文件、
# 逻辑解码、pg_walinspect 读出的 WAL 块,MySQL 的 LOAD_FILE,Oracle 的 BFILENAME,SQLite 的 readfile
r"get_raw_page", r"bt_page_items", r"pg_read_(?:binary_)?file", r"pg_logical_slot_\w+", r"pg_get_wal_block_info",
r"load_file", r"bfilename", r"readfile",
# 别的会话执行过的 SQL 原文:Postgres 的 pg_stat_activity 背后的函数和按会话取的那一个,MySQL 企业版的审计日志
r"pg_stat_get_(?:backend_)?activity", r"audit_log_read",
)) + r")[\"`]?\s*\(|(?:" + "|".join((
# Oracle 的 DBMS_XMLGEN 这一族(包名后面跟着点),XQuery 里的 ora:view 和 oradb: 表
r"dbms_xml\w*", r"ora:view", r"oradb",
# 统计信息里的采样值、全文索引里的词:Postgres 的 pg_stats、pg_statistic 一族,SQLite 的 sqlite_stat*,MySQL 的
# column_statistics、innodb_ft_*,MariaDB 的 column_stats,Oracle 的直方图、列统计和记着最小最大值的 *_tab_columns,
# 以及它们的底表 histgrm$、hist_head$
r"pg_stat(?:s|istic)\w*", r"sqlite_stat\d", r"column_statistics", r"innodb_ft_\w+", r"column_stats",
r"(?:all|dba|user|cdb)_(?:\w+_)?(?:histograms|col_statistics|tab_cols|tab_columns)", r"(?:histgrm|hist_head)\$",
# 别的会话执行过的 SQL 原文(写入语句里就是值):Postgres 的 pg_stat_activity、pg_stat_statements,本连接预备过的
# 语句(连接池里的连接留着前几次查询的原文);MySQL 的 general_log、slow_log,sys 库里建在 performance_schema 上的
# 会话、锁等待视图(带 x$ 的是不排版的那一份;只认这几张,Oracle 常写的 sys.dual 不算);Oracle 的 V$SQL 一族、
# 打开着的游标、审计记录
r"pg_stat_activity", r"pg_stat_statements", r"pg_prepared_statements", r"general_log", r"slow_log",
r"sys[\"`]?\s*\.\s*[\"`]?(?:x\$)?(?:session|processlist|innodb_lock_waits|schema_table_lock_waits)",
r"g?v\$sql\w*", r"(?:dba|cdb)_hist_sql\w*", r"g?v\$open_cursor",
r"(?:dba|cdb|user)_(?:\w+_)?audit_trail", r"unified_audit_trail", r"aud\$\w*", r"fga_log\$",
# SQLite 按页读出的原始数据
r"sqlite_dbpage",
# 系统 schema:哪几张表采样了列值、记着别的会话的 SQL,一张张数靠不住,整个不认
r"pg_catalog", r"information_schema", r"performance_schema",
)) + r")(?![0-9A-Za-z_$]))", re.I)
#: 语法树和数据库实际执行的对不上的写法(MySQL 会执行的注释、Postgres 的 U&"…"),不点列名也读得出表里内容的写法(表名、
#: SQL 写在字符串里交给函数,按页按文件读的原始数据,统计信息、会话记录里存着值的):和核查 agent 的工具边界共用一份名单
#: (engine/masking.py 的 OPAQUE_SQL、indirect_reads),这里另把系统 schema 整个算上(只决定附不附快照,宁可多挡)。SQL
#: 原文里出现就按碰了遮罩列算,写在字符串里的也算
_OPAQUE_SQL = OPAQUE_SQL
_INDIRECT_READS = indirect_reads(*SYSTEM_SCHEMAS)
#: Postgres 的函数记法(m.to_jsonb 就是 to_jsonb(m))下能把一整行交出来的内置函数(_whole_row):只收一个参数、收得下
#: 整行(record、anyelement、"any" 这类),交回的值里带着整行。汇总函数也能这么写:any_value、*_agg 交回整行,max、min
#: 交回最大、最小的那一整行。照 PG 18 的 pg_proc 逐个对过:只交回个数、大小、类型、哈希的不算,这么写会报错的(伪类型的
+168 -34
View File
@@ -7,8 +7,10 @@
运行已经结束封存、核对一致、是探索运行——问答追加在封存之后(post_seal),是批注不是证据。正式运行封存之后不再追加
(同按需裁判);设置里关了「审批质询 · 启用」一律不答。同一次运行同时只答一条:进程内登记(_ANSWERING),登记期间
这次运行的审批回复、继续运行也不受理(api/runs.py),免得运行往下走了、问答还在路上
2. 回答(_answer):交给 agent 被问的那句原文、它挂的证据摘录(照 judge.prepare 的办法取,数据源遮罩的列照旧不给,原报告
的编号标明「原报告」)、审批人选中的原文和问题;工具是这次运行用过的查询工具对应的数据源(同名重建的、没固定过版本的
2. 回答(_answer):交给 agent 被问的那句原文、它挂的证据摘录(照 judge.prepare 的办法取,原报告的编号标明「原报告」;
摘录先遮,和工具边界同一套口径:遮罩列、改了名字的遮罩列按 SQL 认出来都不给,文字格、SQL 的字符串和注释里的遮罩值
按值换,和遮罩列做等值比较的字面量都换,数值格、范围比较的阈值、LIMIT 照原样,见 _excerpts、ExcerptMask)、
审批人选中的原文和问题;工具是这次运行用过的查询工具对应的数据源(同名重建的、没固定过版本的
上传表格不算,见 _usable_sources);上限按设置(每次的查询次数、时长,全局每日金额)。回答先给结论,数字只能引用 agent
自己查询的格子,或者报告里已有的指标、运行输入;回答文档由 compose_doc 组装,裸数字被标出来(提问人自己写的数除外)。
封存之后的追问查的是数据源现在的数据,任务里给报告生成的时间,回答末尾注明;触顶、查询失败、引用解析不了也在末尾注明
@@ -21,8 +23,8 @@
masking.view_edits),取工件时照着再换一遍;SQL 检查结果照事件里记下的 SQL 换(_recorded_swaps)。
哪些列要遮照证据面板的规则(查询当时记下的,加上数据源现在设的),另按 SQL 认出改了名字的遮罩列
(起别名、套函数、经子查询转一道、和 * 写在同一支里,_derived_hidden;认不出落在哪一列的,整份快照遮掉;按遮罩列
分档、当分组键或套在汇总里数个数的不算,_band_only);快照按列遮,再按值遮(别的列里写着遮罩值的格子,_masked_view;质询
工件里,文字格里夹着的遮罩值另按子串换)。回答正文、查询清单里的遮罩值换成「已遮罩」(_Secrets、_scrub_sql):
分档、当分组键或套在汇总里数个数的不算,_band_only,分档写在派生来源里、外层按它分组的也不算,_grouped_labels);
快照按列遮,再按值遮(别的列里写着遮罩值的格子,_masked_view;质询工件里,文字格里夹着的遮罩值另按子串换)。回答正文、查询清单里的遮罩值换成「已遮罩」(_Secrets、_scrub_sql):
按值认有门槛,只装着分档标签的列(agent 写在 SQL 里的标签)不收进遮罩值(_label_columns);SQL 里的字面量按语法树
认用在哪(_literal_uses),数字照原样留着的只有范围比较的边界(没把某一个值夹住的,_pinned)和 LIMIT 的行数,而且
同一个数在这条 SQL 别处没被换掉,封存的是实际执行的语句;在找某一个值的(等值、IN、简单 CASE、夹住某一个值的
@@ -46,18 +48,28 @@
已知局限(遮罩只减少暴露,不是安全边界,同证据面板):
- 按值认有门槛,两头都有误差:短的数(年龄、等级这类不到 SECRET_DIGITS 位的)在回答正文、格子里不按值遮,agent 在
正文里写出某人的年龄遮不住,只靠 SQL 里的等值比较和按列遮;同一个短的数在 SQL 里被换掉了,别的列上撞上它的阈值、
LIMIT 的行数照原样(age = 30 AND visits >= 30)。够长的数在格子和正文里按值遮,碰巧和某个遮罩值相同的数(访问次数
正好等于某一笔金额)也遮掉。这是产品取舍,不按上下文去猜
LIMIT 的行数照原样(age = 30 AND visits >= 30)。够长的数在回答正文和回答引用的格子里按值遮,碰巧和某个遮罩值相同的
数(访问次数正好等于某一笔金额)也遮掉,这是产品取舍,不按上下文去猜;工具边界不按值换数值格(存下的是证据),交给
agent 的、存下的查询快照里这样的数照原样,只在回答和质询工件里遮。反过来,没设遮罩的列里存成数的号码(别的库里的
联系方式表把手机号存成整数)工具边界也不按值换,agent 看得到:要遮就在那个数据源上把那一列也设成遮罩。存成文字的照换。
数值格按列类型认(Postgres 的 NUMERIC、MySQL 的 DECIMAL 交回来是字符串),列类型按列名记,结果里同名的几列说不清是
哪一列的类型,按格子本身认:整数、小数才算数,写成字符串的数按文字照换(多遮,换过的记进 masked_cells)。
占位的写法(未知、暂无……,masking.PLACEHOLDERS)不当遮罩值
- 遮罩数值列的 SUM / AVG / MIN / MAX 算派生自遮罩列,整列遮(COUNT 不算,见 _value_columns),CASE 里有一支取的是
遮罩列的也算:MIN / MAX 就是某一行的原值,只有一行的分组 SUM / AVG 也是,按列认不出哪些汇总值安全,宁可多遮。
核查 agent 的查询在工具边界照这个办法遮,所以这样的指标双盲复算不了(原 SQL 里认得出的,派盲算之前就记为无法
复算),反方审稿核对这类汇总数的结论也拿不到数。汇总里套着结果全是写死常量的 CASE(SUM(CASE WHEN amount <= 20000
THEN 1 ELSE 0 END))只是按条件数个数,同 COUNT;汇总里的 CASE 只在条件里用到遮罩列的(SUM(CASE WHEN buyer_phone
IS NOT NULL THEN amount ELSE 0 END))同写在 WHERE 里,不算
- FROM、JOIN、LATERAL 里的表值函数(json_each、json_tree、unnest、jsonb_each_text、JSON_TABLE、LATERAL VALUES)拆开
遮罩列的,按出列名认它出的列(常见函数的出列名、写的列名表、别名、函数名,_function_columns),同名的别的列一并遮
(多遮);不认得的函数又没写列名表的,整份快照都遮。函数参数里没写表名的 col->'path'(JSON 箭头,解析器当成以 col
为参数的 lambda)按 col 认,真的 lambda(transform(arr, x -> …))的参数和遮罩列同名时也算(多遮)
- FROM、JOIN、LATERAL 里的表值函数(json_each、json_tree、unnest、jsonb_each_text、JSON_TABLE)拆开遮罩列的,按
出列名认它出的列(常见函数的出列名、写的列名表、别名、函数名,_function_columns),同名的别的列一并遮(多遮);
不认得的函数又没写列名表的,整份快照都遮。Postgres 里可能交回复合类型、展开成各个字段的也整份遮
(masking._spreads_record):unnest(数组的元素是复合类型时展开成 addr、tel 这些字段,写了列名表也只改前几列的
名字),不认得的函数只写了列名、没写类型的列名表(jsonb_populate_record(NULL::contact, …) AS r(a) 后面的字段
照原名出);拆的是普通数组的 unnest 也一样整份遮(元素是什么类型 SQL 里看不出,多遮)。函数参数里没写表名的
col->'path'(JSON 箭头,解析器当成以 col 为参数的 lambda)按 col 认,真的 lambda(transform(arr, x -> …))的参数
和遮罩列同名时也算(多遮)。三段以上的名字(Oracle 的 JSON 点号、对象类型的属性 t.profile.tel,解析器读成库、表、列)
前面几段也算列,写了库名、表名的列,库名、表名碰巧和遮罩列同名时也算(多遮)
- * 把遮罩列带进别的列名(WITH t(a, b, …) AS (SELECT * …) 这类列名表、表的列名表 FROM sales AS s(i, r, p)、UNION 后一支
的 *、TABLE x、标量子查询里的 *):只展开一个来源、那里每一列都数得清的,按位置只遮取自遮罩列的那几列——子查询按它的
选出项数,数据源里的表按查询当时冻结的表结构数(几张列一样的月度表 SELECT * 连起来汇总,金额照常)。表结构里查不到
@@ -69,11 +81,41 @@
一律算取自遮罩列。EXCEPT 后一支只用来去掉行,也照 UNION 的后一支算(多遮)
- 整行和函数参数里的 *(Postgres 的 row_to_json(s)、json_agg(t)、SELECT s、s::text、to_json(s.*)、concat_ws(',', s.*),
Oracle 的 JSON_OBJECT(*))按 * 展开的来源算:取自子查询、CTE 的看它选出的列,取自数据源里的表的一律算取自遮罩列(整列
遮)。Postgres 先按列认:表结构里查得到和表同名的列(visits 表的 visits 列)才当列,查不到时按整行算(多遮)。PIVOT、
UNPIVOT、MATCH_RECOGNIZE 读到遮罩列的,整份快照的列都遮(多遮)。SQL 解析不了的只看文字:写着遮罩列、*(COUNT(*) 除外)、
TABLE x 的整份遮,整行引用这类不写列名的写法认不出
遮)。Postgres 先按列认:表结构里查得到和表同名的列(visits 表的 visits 列)才当列,查不到时按整行算(多遮);LATERAL
里只看排在它前面的来源,它自己的结果列不算。PIVOT、UNPIVOT、MATCH_RECOGNIZE 读到遮罩列的,整份快照的列都遮(多遮)。
SQL 解析不了的:Postgres 的一律整份遮(整行引用不写列名、也没有 *,只看文字认不出);别的库只看文字,写着遮罩列、
*(COUNT(*) 除外)、TABLE x、下面那几类写法的整份遮
- WITH 里和数据源里的表同名的一段(WITH sales AS (SELECT * FROM sales WHERE …)):不带 RECURSIVE 的 WITH 里一段只看得见
排在它前面的段,里面的 sales 是数据源里的表(按表算,整行引用、* 展开的照遮);SQLite 里一段看得见排在后面的段,这里
按看不见算(当成数据源里的表,多遮)
- 认不全的写法整份遮(多遮,同一份里没碰遮罩列的编号、金额也遮):(表达式).* 展开读到遮罩列的(复合类型的列
(p.profile).*、函数交回的记录 (jsonb_each_text(pj)).*、子查询里的 ROW (q.r).*,外面套了括号、起了别名的也算,展开成
几列、各叫什么 SQL 里看不出;不只选出项,VALUES 里的、行构造器和函数参数里的也算),VALUES 当一张表用(FROM、JOIN、
LATERAL 里的,WITH 里的一段,UNION 的一支)又读到遮罩列的(直接写遮罩列、经子查询或派生来源改了名字的、里面写着
s.*、(s.*) 的都算;各库给 VALUES 的列起名不一样:Postgres、SQLite 叫 column1…,MySQL 叫 column_0…,MariaDB 拿第一行
的值当列名,不逐个认,同一份里的编号、区域也遮,只收值取自遮罩列的那几列的值);语法树和实际执行的对不上的(MySQL、
MariaDB 会执行的注释 /*! … */,Postgres 的 U&"…" 名字,没有 * 的一支选出项和结果的列数对不上);表名、SQL 写在字符串里
交给函数执行、整表导出的(query_to_xml、table_to_xml、cursor_to_xml、schema_to_xml、database_to_xml 这一族,dblink、
crosstab、ts_stat,Oracle 的 DBMS_XMLGEN、XMLTYPE)、按页按文件按日志读原始数据的(pageinspect、逻辑解码、
LOAD_FILE)、统计信息、会话记录、锁和日志里存着值的(Postgres 的 pg_stats、pg_stat_statements、pg_stat_activity;
SQLite 的 sqlite_stat4;MySQL 的 column_statistics、general_log、PROCESSLIST、INNODB_TRX、OPTIMIZER_TRACE、
performance_schema,sys 库里读它的函数(按 sys.ps_ 开头认:ps_thread_trx_info、ps_thread_stack 交回别的线程执行过的
语句原文);MariaDB 的 column_stats、INNODB_LOCKS、QUERY_CACHE_INFO;Oracle 的直方图、记着最小最大值的 *_tab_columns、
V$SQL 一族、SQL 调优集、固定下来的执行计划、闪回事务、LogMiner、审计记录),按名字认(masking 的 INDIRECT_FUNCTIONS、
INDIRECT_NAMES,可验证报告用的同一份名单),去掉注释再认一遍(函数名和括号之间夹着注释),自己建的同类函数、视图
认不出;information_schema、pg_catalog 里只有表结构的视图不算(核查 agent 查列名、类型常用)。不是查询的语句里,
只读表结构的 SHOW(表、列、索引、库、表状态、建库语句)、DESCRIBE 一张表照常,别的 SHOW(ENGINE INNODB STATUS、
BINLOG EVENTS、WARNINGS、CREATE TABLE / VIEW……)、EXPLAIN 一条语句(MySQL 把只有一行的表的值代进计划里的条件)
不逐个认,一律整份遮。只收支持的几种库的:DuckDB、SQL Server 的同类函数不收;分析出了意外、一长串 CTE 超出工作量
上限(ANALYSIS_STEPS)的。整份遮的只收确定取自遮罩列的那几列的值当遮罩值(别的列的值本来就没交给模型,收了会把区域
名、列名在同一次核查里处处换掉);快照记下整份遮(masked_whole),格子引用指到它时照实说写法认不全
- 定义里写死的字面量不认:视图、函数、触发器、约束、默认值的定义经 information_schema(views、routines、triggers、
columns 的默认值)、pg_views、pg_get_viewdef() 这类查出来照常交给 agent,写死在定义里的号码(排除测试账号的 WHERE)
只在已经认得的遮罩值里时按值换掉;SHOW CREATE TABLE(对视图交回的是视图定义)、SHOW CREATE VIEW、SHOW TRIGGERS 不在
只读表结构的几种里,跟着整份遮
- 按遮罩列分档的标签(CASE、IIF 的结果全是写死的常量):标签文字是 agent 写在 SQL 里的,不当遮罩值,SQL、回答正文里
照原样;标签列当分组键(每一档一行)照常显示,不分组的逐行标签列整列遮(格子里看不出某一笔落在哪一档),可 agent
照原样;标签列当分组键(每一档一行)照常显示,分档写在子查询、WITH 里、外层(或中间一层)按它分组的也算
(masking._grouped_labels),不分组的逐行标签列整列遮(格子里看不出某一笔落在哪一档),可 agent
在正文里写出「这一笔是大单」遮不住,按 id 分组再按档数个数也看得出(同按阈值查出某几行)。只有只装着标签的列不收:
结果的这一列在 UNION 的每一支里都是这样的 CASE / IIF,或者按作用域认得出取自子查询、CTE 里只装着标签的一列;有一支
取自数据源里的表(遮罩列原样选出、经子查询按原名传出,哪怕起了标签的名字)就照收。认不准的也照收(多遮),比如
@@ -106,17 +148,21 @@
从开头的引号、注释起整个去掉)。说明里码值表列出的码值和别的字一样按值换,恰好是遮罩值的码值也换(多遮)
- 运行详情的「工件」页签、通用的工件接口按工件原样给:质询的查询快照落库时就遮好了(工具边界先遮),取自遮罩列的值、
报告快照里的遮罩值、agent 前面的查询里遮掉的值都看不到。只有一种照原样:先查了写着号码的备注(那时还不知道这个号码
是遮罩值),后来才查出号码——备注快照落库时原样,从通用接口打开照原样,质询工件接口照事件里记下的位置换。报告自己
的查询快照(写作时的查询)仍原样存着,从那里打开不经这里的遮罩
是遮罩值),后来才查出号码——备注快照落库时原样,从通用接口打开照原样,质询工件接口照事件里记下的位置换。没设遮罩的
数值格碰巧等于遮罩值的(见上),落库时照原样,从通用接口打开也照原样。报告自己的查询快照(写作时的查询)仍原样存着,
从那里打开不经这里的遮罩
- 遮罩值最多收 SECRETS_MAX 种写法,再多的按值认不出(按列遮照旧)。agent 自己查到的值先收,报告快照里的后收:核查
agent 把两份分开记(masking.SeenValues),查询时按值遮、交回调用方、回答时合起来,都是 agent 查到的在前
- 换了写法的号码只认数字之间夹着一到四个分隔符(空格、制表符、全角空格、不换行空格、窄空格这类空白,连字符、减号、
点、括号,含全角),带 +86 / 0086 / 86 前缀、全角数字:写成汉字(一三九……)、中间夹着别的字、拆成几处写的、跨行
写的认不出;和别的数字只隔着一个连字符或点连着的(13900000001-2、19914.74)不算号码;SQL 里不带引号的数不这样认
(199-1474-2785 在 SQL 里是三个数相减),字符串、带引号的名字、注释里照样认
- 质询 agent 的任务里的证据摘录照 judge.prepare 按列名遮:报告取数把遮罩列改了名字选出来(buyer_phone AS 手机号)时,
摘录里有原值,agent 看得到;它写进回答、SQL 的,按值认得出的照样换掉(报告快照里的值都收了),封存的是换过的。摘录
按 SQL 认改了名字的遮罩列要改 judge.py,另行处理
- 质询 agent 的任务里的证据摘录先遮(_excerpts、ExcerptMask,和工具边界同一套口径):遮罩列、按 SQL 认出改了名字的
遮罩列不给;文字格、SQL 的字符串和注释、列名里的遮罩值按值换(报告快照里的值,门槛同上);SQL 里和遮罩列做等值、
IN、LIKE 比较的字面量不论认不认得出都换(报告没选出遮罩列时快照里收不到这些值);数值格、范围比较的阈值、LIMIT 照
原样,没设遮罩的数值格恰好等于某个遮罩值也照原样(同工具边界不改写数值证据)。被问的那句(在表格里的连同整张表)、
问题、报告列出的指标和运行输入里写着的值不换,回答里复述也不换。没指明是哪一句时任务里给的是报告正文,正文里写着
的值照原样(读者本来就看得到)
"""
from __future__ import annotations
@@ -152,10 +198,11 @@ from app.engine.masking import ( # noqa: F401
SECRETS_MAX, SECRET_DIGITS, SECRET_MIN, _ABSENT, _DIGITS, _EXPONENT_MAX, _LabelMemo, _NUMERIC, _PLAIN_NUMBER,
_PREFIX, _Secrets, _WORD, _WORDS, _all_labels, _arithmetic, _band_only, _block_label, _block_label_of, _blocks,
_branch_results, _branches, _canonical, _constant, _constants, _decimal, _derived_block, _derived_hidden,
_digits_of, _fixed, _forms, _from_sources, _is_star, _item_label, _label_columns, _leaves, _literal_types,
_masked_view, _masked_word, _mentions, _merging, _numeric, _output_name, _pattern, _places, _plain_text,
_point_text, _query_shaped, _result_label, _scrubbed_source, _snapshot_labels, _source_label, _source_names,
_star_label, _strong, _tainted, _value_columns, _worked_out, replay_edits, view_edits,
_derived_masks, _digits_of, _fixed, _forms, _from_sources, _is_star, _item_label, _label_columns, _leaves,
_literal_types, _masked_view, _masked_word, _mentions, _merging, _numeric, _output_name, _pattern, _places,
_plain_text, _point_text, _query_shaped, _result_label, _schema_columns, _scrubbed_source, _snapshot_labels,
_source_label, _source_names, _star_label, _strong, _tainted, _value_columns, _worked_out, masked_cell,
replay_edits, view_edits,
)
from app.engine.merge_query import MERGE_SOURCE
from app.engine.nodes.challenge import mark_original, with_note
@@ -571,9 +618,12 @@ def _table_text(block: dict[str, Any], markdown: str) -> str:
def _excerpts(doc: dict[str, Any], catalog: dict[str, Any], cand: judging.Candidate, loader: Callable[[str], Any],
masked: dict[str, Any]) -> list[str]:
"""这句挂的每件证据的摘录:和 judge.prepare 一样逐件摘(数据源遮罩的列不给,带期 4 标记的文档多来历行),
指向原报告编号的写法标上「原报告」——agent 自己的查询也编 Q1、Q2…,不分开的话会把它们混成一回事。"""
masked: dict[str, Any], mask: ExcerptMask | None = None) -> list[str]:
"""这句挂的每件证据的摘录:和 judge.prepare 一样逐件摘(带期 4 标记的文档多来历行),指向原报告编号的写法标上
「原报告」——agent 自己的查询也编 Q1、Q2…,不分开的话会把它们混成一回事。
摘录先遮,模型从任务里也看不到遮罩值(mask,见 ExcerptMask):数据源遮罩的列、按 SQL 认出改了名字的
(buyer_phone AS 手机号)不给;文字格、SQL 里的字符串和注释里的遮罩值按值换成「已遮罩」,换了写法的号码也认,和遮罩列
做等值比较的字面量都换;数值格、范围比较的阈值、LIMIT 照原样。解析 SQL、按值换是纯计算,调用方放进线程。"""
fetch = judging._Fetch(loader)
provenance_on = doc.get("provenance") == DOC_PROVENANCE
queries = {a for a, e in catalog.items() if isinstance(e, dict) and e.get("kind") == "query"}
@@ -581,7 +631,7 @@ def _excerpts(doc: dict[str, Any], catalog: dict[str, Any], cand: judging.Candid
for alias in cand.aliases:
entry = catalog.get(alias)
if isinstance(entry, dict) and (text := judging._excerpt(alias, entry, [cand], fetch, masked, catalog,
provenance_on=provenance_on)):
provenance_on=provenance_on, mask=mask)):
out.append(mark_original(text, alias, queries))
return out
@@ -1460,6 +1510,78 @@ def _scrub_sql(sql: str, secrets: _Secrets, compared: set[str], kind: str | None
raw, tokens[i].text, kinds[i], uses[i], secrets, compared)[0] if i in uses else raw)
def _scrub_excerpt_sql(sql: str, secrets: _Secrets, compared: set[str], kind: str | None) -> str:
"""交给核查 agent 的证据摘录里的 SQL(报告取数的那条):照封存 SQL 的办法换(_scrub_sql:字符串、注释、名字里的遮罩值
换,范围比较的边界、LIMIT 的行数照原样,别处够长的遮罩值换),另外和遮罩列(compared:_derived_hidden 认出的名字,
含改了名字的)做等值、IN、LIKE 一类比较的字面量,不论认不认得出是遮罩值都换成「已遮罩」:报告没选出遮罩列时快照里
收不到这些值(WHERE buyer_phone NOT IN ('13900000002', …)),摘录又不是封存的 SQL,可以多遮。同一个数写成阈值、行数的
也换(同 _literals)。已经摆在读者眼前的值(问题、被问的那句、这句结论原文里写着的,_Secrets.spared)照旧不换。
以前对渲染好的整段摘录按值换,范围比较的阈值(amount >= 20000)也成了「已遮罩」,agent 看不到报告按什么取数。
分词出错的整条按值换(同 _scrub_sql)。"""
if not sql or not (secrets.strong or secrets.weak or compared):
return sql
split = _literals(sql, kind, secrets, compared)
if split is None:
return secrets.scrub(sql)
tokens, kinds, uses = split
word = _masked_word()
pinned = {i for i, (how, columns, _) in uses.items()
if how == "equal" and columns & compared and not secrets.spared(tokens[i].text)}
numbers = _number_forms(v for i in pinned for v in (tokens[i].text, *uses[i][2]))
def literal(i: int, raw: str) -> str:
if i not in uses:
return raw
if i in pinned or (kinds[i] == "NUMBER" and uses[i][0] in _KEPT and _digits_of(tokens[i].text) in numbers):
return _requote(raw, lambda _: word)
return _swap_literal(raw, tokens[i].text, kinds[i], uses[i], secrets, compared)[0]
return _rebuild(sql, tokens, kinds, secrets, literal)
@dataclass
class ExcerptMask:
"""交给核查 agent(反方审稿、审批质询)的证据摘录怎么遮:judge.prepare 的 mask,judge 只按这几个方法用它。口径和工具
边界一样,模型从任务里也看不到遮罩值,又不改写没设遮罩的数值证据:
- columns:哪几列不给。按列名的,加上按 SQL 认出改了名字的(buyer_phone AS 手机号,masking._derived_masks,表结构按
快照冻结的那份);认的过程出了意外,整份的列都不给
- cell:格子逐格遮(masking.masked_cell):数值格照原样,文字格按值换(整格的、夹着的那一截,换了写法的号码也认)
- sql:SQL 照 _scrub_excerpt_sql 换:范围比较的边界、LIMIT 照原样,和遮罩列做等值比较的字面量都换
- text:别的文字(列名、检索片段、来历行)按值换,换了写法的号码也认,同 SQL 里的名字
报告里列出的指标值、原式、运行输入的值不经它(报告本来就摆在读者眼前,同 _ref_values 免遮)。以前先渲染好整段摘录、
再对整段按值换,分不出格子和 SQL 里的用法:阈值、没设遮罩的访问次数正好等于某一笔被遮的金额,都成了「已遮罩」。
kinds:{数据源名: 类型}(SQL 按它的方言解析);secrets:按值换的值(报告快照里的遮罩值去掉已经摆在读者眼前的,同交给
查询工具的那一份),没有为 None(只按列遮、只换和遮罩列做等值比较的字面量)。"""
kinds: dict[str, Any]
secrets: _Secrets | None = None
def _kind(self, snapshot: dict[str, Any]) -> str | None:
return self.kinds.get(str(snapshot.get("source") or ""))
def columns(self, snapshot: dict[str, Any], hidden: set[str], schema: Any) -> tuple[set[str], set[str]]:
"""(不给的列, SQL 里指代遮罩列的名字),都是小写。schema 是快照冻结的表结构快照(经 judge 的 fetch 取),没有为
None(按快照记着的认)。"""
if not hidden:
return set(hidden), set(hidden)
try:
tables = _schema_columns(schema) if isinstance(schema, dict) else None
found = _derived_masks(dict(snapshot), set(hidden), self._kind(snapshot), tables)
return set(hidden) | found.hidden, found.names
except Exception: # noqa: BLE001 - 认不准就整份不给:摘录少几列,不让遮罩值交给核查 agent
logger.warning("按 SQL 认摘录里改了名字的遮罩列时出错,整份快照的列都按遮罩处理", exc_info=True)
return set(hidden) | {str(c).lower() for c in snapshot.get("columns") or []}, set(hidden)
def cell(self, value: Any, *, numeric: bool) -> Any:
return masked_cell(value, self.secrets, numeric=numeric)
def sql(self, sql: str, snapshot: dict[str, Any], names: set[str]) -> str:
return _scrub_excerpt_sql(sql, self.secrets or _Secrets(), set(names), self._kind(snapshot))
def text(self, text: str) -> str:
return self.secrets.scrub(text, loose=True) if self.secrets is not None and self.secrets.strong else text
def _rebuild(sql: str, tokens: list[Any], kinds: list[str], secrets: _Secrets, literal: Callable[[int, str], str]
) -> str:
"""按词重写一段 SQL:字面量交给 literal(第几个词、原文)定换成什么,名字(带不带引号)、注释和别的字按值换。
@@ -1670,8 +1792,10 @@ def _settle_plan(own: dict[str, dict[str, Any]], named: dict[str, tuple[set[str]
tainted: dict[str, set[str]] = {}
for artifact, snapshot in own.items():
kind, base = kinds.get(str(snapshot.get("source") or "")), named[artifact][0]
hidden[artifact], tainted[artifact] = _derived_hidden(snapshot, base, kind)
secrets.add(snapshot, hidden[artifact], labels=_snapshot_labels(snapshot, kind), base=base)
found = _derived_masks(snapshot, base, kind)
hidden[artifact], tainted[artifact] = found.hidden, found.names
# 收值只收确定取自遮罩列的那几列(整份遮的不是每一列都收,见 masking._derived_masks)
secrets.add(snapshot, found.collect, labels=_snapshot_labels(snapshot, kind), base=base)
# agent 查询时遮掉的值先收(报告快照里的值收满了上限 SECRETS_MAX,也不丢 agent 自己查到的);不带它们的一份另算,
# 比出事后取质询工件时认不出、要照位置重放的地方
baseline = secrets.copy() if seen is not None and seen.strong else None
@@ -1696,10 +1820,11 @@ def _settle_plan(own: dict[str, dict[str, Any]], named: dict[str, tuple[set[str]
def _add_based(secrets: _Secrets, based: list[tuple[dict[str, Any], set[str]]], kinds: dict[str, str | None]) -> None:
"""收报告自己的查询快照里遮罩列的值(based:快照和它要遮的列),按 SQL 认出改了名字的遮罩列(_derived_hidden)。"""
"""收报告自己的查询快照里遮罩列的值(based:快照和它要遮的列),按 SQL 认出改了名字的遮罩列(masking._derived_masks)。
写法认不全、整份遮的快照只收确定取自遮罩列的那几列:以前每一格都收,区域名、列名在这次核查里处处被换掉。"""
for snapshot, base in based:
kind = kinds.get(str(snapshot.get("source") or ""))
secrets.add(snapshot, _derived_hidden(snapshot, base, kind)[0], labels=_snapshot_labels(snapshot, kind),
secrets.add(snapshot, _derived_masks(snapshot, base, kind).collect, labels=_snapshot_labels(snapshot, kind),
base=base)
@@ -1800,7 +1925,6 @@ async def _answer(op: _Opening, ask: Ask, target: _Target | None, settings: dict
doc = op.doc
catalog = doc.get("catalog") if isinstance(doc.get("catalog"), dict) else {}
masked = await judging.source_masks(catalog, loader=op.loader)
excerpts = _excerpts(doc, catalog, target.cand, op.loader, masked) if target is not None else []
refs = _report_refs(catalog)
wanted = _run_sources(op.events)
usable = await _usable_sources(op.run_id, wanted, op.versions)
@@ -1809,15 +1933,25 @@ async def _answer(op: _Opening, ask: Ask, target: _Target | None, settings: dict
async with SessionLocal() as session:
spec = await verify_model_spec(session, None, settings=settings)
# 报告自己的查询快照:被问的那句可能直接写着遮罩列的值,agent 会照着复述;里面的遮罩值在派 agent 之前就交给它的
# 查询工具(masked_values),agent 查到写着同一个号码的备注、别的表时,交给模型之前就换掉
# 查询工具(masked_values),agent 查到写着同一个号码的备注、别的表时,交给模型之前就换掉;证据摘录也按它们先遮。
# 被问的那句(在表格里的连同整张表)就写在任务开头、摆在审批人眼前,里面的值同问题一样哪里都不换(同反方审稿的
# _spared):以前不算,这句引用的那一格在摘录里成了「已遮罩」,和任务开头写着的原句对不上
given = [ask.question, ask.quote, *_ref_values(refs)]
if target is not None:
given += [target.cand.text, target.table]
others = [snapshot for entry in catalog.values() if isinstance(entry, dict) and entry.get("kind") == "query"
and _query_shaped(snapshot := _quiet_load(op.loader, entry.get("artifact")))]
seed = await snapshot_secrets(others, given)
excerpts: list[str] = []
if target is not None:
kinds = await _source_kinds([*masked, *(str(s.get("source") or "") for s in others)])
excerpts = await asyncio.to_thread(_excerpts, doc, catalog, target.cand, op.loader, masked,
ExcerptMask(kinds, seed))
outcome = await run_verify_agent(
spec=spec, system=INQUIRY_SYSTEM, task=_task(op, ask, target, excerpts, refs, live, lost), sources=list(live),
output_schema=INQUIRY_SCHEMA, max_queries=part.get("max_queries"), timeout_s=part.get("timeout_s"),
daily_max_usd=settings.get("daily_max_usd"), run_id=op.run_id, data_versions=op.versions,
masked_values=await snapshot_secrets(others, given))
masked_values=seed)
# 遮罩:agent 自己的查询快照(查询时就遮好了,遮掉的值在 outcome.masked_values 里)、报告自己的查询快照
own: dict[str, dict[str, Any]] = {}
+66 -21
View File
@@ -342,6 +342,24 @@ def _hidden(snapshot: dict[str, Any], source: Any, masked: dict[str, Any] | None
return names
def _derived(snapshot: dict[str, Any], hidden: set[str], fetch: _Fetch, mask: Any) -> tuple[set[str], set[str]]:
"""(要遮的列, SQL 里指代遮罩列的名字),都是小写。核查 agent 的摘录(mask 给了,engine/inquiry.py 的 ExcerptMask)另按
SQL 认出改了名字的遮罩列(buyer_phone AS 手机号、套函数、经子查询转一道……),和核查 agent 的查询在工具边界遮的是同一
套规则(表结构按快照冻结的那份、经 fetch 取)。mask 为 None(结论句裁判自己的摘录)照旧只按列名,摘录、复用键都和
以前一样。"""
if mask is None or not hidden:
return hidden, hidden
schema = fetch(snapshot.get("schema_artifact")) if snapshot.get("schema_artifact") else None
return mask.columns(snapshot, hidden, schema)
def _unique_number(snapshot: dict[str, Any], column: str) -> bool:
"""这一列按列类型是数值列:column_types 记成 number,列名在结果里只有这一列(同名的几列说不清是哪一列的类型,
masking._typed_numbers)。"""
columns = [str(c) for c in snapshot.get("columns") or []]
return column_kind(snapshot, column) == "number" and columns.count(column) == 1
def _shown_columns(columns: list[str], cited: Iterable[str]) -> list[str]:
"""摘录给哪几列:被引用的列一律给(超过上限也给),再按原来的顺序用别的列补满到 EXCERPT_COLS 列。
@@ -354,11 +372,14 @@ def _shown_columns(columns: list[str], cited: Iterable[str]) -> list[str]:
def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iterable[str], whole: bool,
fetch: _Fetch, masked: dict[str, Any] | None, *, provenance_on: bool = False) -> str | None:
fetch: _Fetch, masked: dict[str, Any] | None, *, provenance_on: bool = False,
mask: Any = None) -> str | None:
"""一条查询的摘录:头、SQL、(期 4)来历行、列、被引用的行。
provenance_on:文档带期 4 标记时为真(prepare 算)。这时上传表格的查询快照(有 data_version)在 SQL 之后、「列:」
之前多几行来历(_provenance_lines);其余情况一个字都不多,期 4 之前的文档、手工源的摘录和以前逐字相同。
mask:给核查 agent 的摘录才给(engine/inquiry.py 的 ExcerptMask),分段遮:改了名字的遮罩列也不给(_derived);SQL
先遮再截(截在半截的值认不出);列名、来历行按值换;格子逐格遮,数值格照原样。
"""
snapshot = fetch(entry.get("artifact"))
if not isinstance(snapshot, dict) or not isinstance(snapshot.get("rows"), list):
@@ -368,13 +389,15 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter
source = entry.get("source") or snapshot.get("source")
head = (f"【{alias}】合并查询结果(共 {total} 行)" if snapshot.get("source") == MERGE_SOURCE
else f"【{alias}】查询结果({entry.get('tool') or '查询'} · 数据源 {source or '—'} · 共 {total} 行)")
hidden, names = _derived(snapshot, _hidden(snapshot, source, masked), fetch, mask)
sql = _clean(str(snapshot.get("sql") or ""))
if mask is not None:
sql = mask.sql(sql, snapshot, names)
lines = [head, f"SQL:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"]
if snapshot.get("source") == MERGE_SOURCE:
lines.append(_merge_line(snapshot))
hidden = _hidden(snapshot, source, masked)
if provenance_on and snapshot.get("data_version"):
lines += _provenance_lines(snapshot, fetch, hidden)
lines += [mask.text(line) if mask is not None else line for line in _provenance_lines(snapshot, fetch, hidden)]
columns = [str(c) for c in snapshot.get("columns") or [] if str(c).lower() not in hidden]
shown_cols = _shown_columns(columns, cols)
# 点名引用的行先给,整份引用的再从头补满,一共不超过 EXCERPT_ROWS 行
@@ -382,13 +405,16 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter
if whole:
picked += [r for r in range(total) if r not in picked][:EXCERPT_ROWS - len(picked)]
omitted = len(columns) - len(shown_cols)
lines.append("列:" + " | ".join(shown_cols) + (f" | …另有 {omitted} 列" if omitted > 0 else ""))
heads = [mask.text(c) for c in shown_cols] if mask is not None else shown_cols
lines.append("列:" + " | ".join(heads) + (f" | …另有 {omitted} 列" if omitted > 0 else ""))
for r in sorted(picked):
cells = []
for c in shown_cols:
try:
raw, _ = locate_cell(snapshot, r, c)
cells.append(render_cell(raw, kind=column_kind(snapshot, c)))
shown = mask.cell(raw, numeric=_unique_number(snapshot, c)) if mask is not None else raw
# 按值换过的格子照文字给:「已遮罩」不能再按这一列的类型当数渲染
cells.append(render_cell(raw, kind=column_kind(snapshot, c)) if shown is raw else str(shown))
except (CellError, RenderError):
cells.append(MISSING)
lines.append(f"r{r}:" + " | ".join(cells))
@@ -483,11 +509,12 @@ def _cell_words(locator: dict[str, Any], hidden: set[str]) -> str:
def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any],
masked: dict[str, Any] | None) -> list[str]:
masked: dict[str, Any] | None, mask: Any = None) -> list[str]:
"""指标的输入取自哪次查询、哪一格,再附上那次查询的 SQL(截到 SQL_CHARS,最多 ENTITY_QUERIES 次)。
「活跃用户按会员号去重」这种讲口径的句子,证据在取数的 SQL 里;只给原式,裁判只能判证据不足。
位置来自封存的口径卡;SQL 经 fetch 取,取不到(不在封存范围里)的不给。没追到查询的输入照实写。
mask 同 _query_excerpt:改了名字的遮罩列不写列名,SQL 先遮再截。
"""
items = [i for i in metric.get("inputs") or [] if isinstance(i, dict)]
lines: list[str] = []
@@ -504,11 +531,13 @@ def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any
snapshot = fetch(artifact)
snap = snapshot if isinstance(snapshot, dict) else {}
source = (catalog.get(alias) or {}).get("source") if alias else None
hidden = _source_hidden(source or snap.get("source"), catalog, fetch, masked) \
| _mask_names(snap.get("mask_columns"))
hidden, names = _derived(snap, _source_hidden(source or snap.get("source"), catalog, fetch, masked)
| _mask_names(snap.get("mask_columns")), fetch, mask)
label = f"查询 {alias} " if alias else "一次查询(不在这份报告的证据目录中)"
lines.append(f"输入 {path}:取自{label}的 {_cell_words(item.get('locator') or {}, hidden)}")
sql = _clean(str(snap.get("sql") or ""))
if sql and mask is not None:
sql = mask.sql(sql, snap, names)
if sql and str(artifact) not in seen and len(seen) < ENTITY_QUERIES:
seen.add(str(artifact))
sqls.append(_sql_line(f"查询 {alias} 的 SQL" if alias else "这次查询的 SQL", sql))
@@ -518,7 +547,7 @@ def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any
def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any],
masked: dict[str, Any] | None) -> str:
masked: dict[str, Any] | None, mask: Any = None) -> str:
name = entry.get("name") or (entry.get("locator") or {}).get("metric") or alias
rendered = entry.get("rendered")
where = " ".join(v for v in (f"口径「{entry['caliber']}」" if entry.get("caliber") else "",
@@ -533,7 +562,7 @@ def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch, catalog: d
if metric is not None:
if metric.get("expression"):
lines.append(f"原式:{_clean(str(metric['expression']))}")
lines.extend(_metric_inputs(metric, fetch, catalog, masked))
lines.extend(_metric_inputs(metric, fetch, catalog, masked, mask))
return "\n".join(lines)
@@ -611,7 +640,8 @@ def _column_lines(entry: dict[str, Any], fetch: _Fetch) -> list[str]:
return lines
def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], whole: bool, fetch: _Fetch) -> str | None:
def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], whole: bool, fetch: _Fetch,
mask: Any = None) -> str | None:
content = fetch(entry.get("artifact"))
hits = content.get("hits") if isinstance(content, dict) else None
if not isinstance(hits, list):
@@ -625,18 +655,21 @@ def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], who
if not isinstance(hit, dict) or not isinstance(hit.get("content"), str):
continue
body = _clean(hit["content"])
# 核查 agent 的摘录(mask):知识库片段里写着的遮罩值按值换(先换再截,截在半截的值认不出)
body = mask.text(body) if mask is not None else body
title = f"《{_clean(str(hit['title']))}》" if hit.get("title") else ""
lines.append(f"片段 {h + 1}{title}:{body[:CHUNK_CHARS]}{'…' if len(body) > CHUNK_CHARS else ''}")
return "\n".join(lines)
def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _Fetch,
masked: dict[str, Any] | None, catalog: dict[str, Any], *, provenance_on: bool = False) -> str | None:
masked: dict[str, Any] | None, catalog: dict[str, Any], *, provenance_on: bool = False,
mask: Any = None) -> str | None:
"""一件证据的摘录。provenance_on 只交给查询摘录(_query_excerpt):指标摘录里「输入取自查询 Qn」的那几行不加
来历(P4-SPEC 调整 1),表名、字段名的摘录照旧。"""
来历(P4-SPEC 调整 1),表名、字段名的摘录照旧。mask 只给核查 agent 的摘录(见 prepare)。"""
kind = entry.get("kind")
if kind == "metric":
return _metric_excerpt(alias, entry, fetch, catalog, masked)
return _metric_excerpt(alias, entry, fetch, catalog, masked, mask)
if kind == "input":
field_name = (entry.get("locator") or {}).get("field") or alias
return f"【{alias}】运行输入:{field_name} = {entry.get('rendered')}"
@@ -644,27 +677,30 @@ def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _
rows = {r for c in cands for a, r in c.rows if a == alias}
cols = [col for c in cands for a, col in c.cols if a == alias]
return _query_excerpt(alias, entry, rows, cols, any(alias in c.whole for c in cands), fetch, masked,
provenance_on=provenance_on)
provenance_on=provenance_on, mask=mask)
if kind == "retrieval":
picked = [h for c in cands for a, h in c.hits if a == alias]
return _retrieval_excerpt(alias, entry, picked, any(alias in c.whole for c in cands), fetch)
return _retrieval_excerpt(alias, entry, picked, any(alias in c.whole for c in cands), fetch, mask)
if kind == "table":
head = f"【{alias}】表 {entry.get('name')}" + (f"(数据源 {entry['source']})" if entry.get("source") else "")
return "\n".join([head, *_table_lines(entry, catalog, fetch, masked), *_entity_sql(entry, catalog, fetch)])
return "\n".join([head, *_table_lines(entry, catalog, fetch, masked),
*_entity_sql(entry, catalog, fetch, masked, mask)])
if kind == "column":
table = (entry.get("locator") or {}).get("table")
head = f"【{alias}】字段 {f'{table}.' if table else ''}{(entry.get('locator') or {}).get('column') or entry.get('name')}"
return "\n".join([head, *_column_lines(entry, fetch), *_entity_sql(entry, catalog, fetch)])
return "\n".join([head, *_column_lines(entry, fetch), *_entity_sql(entry, catalog, fetch, masked, mask)])
return None
def _entity_sql(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch) -> list[str]:
def _entity_sql(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch, masked: dict[str, Any] | None = None,
mask: Any = None) -> list[str]:
"""表名、字段名出现过的查询的 SQL(每条截到 SQL_CHARS,最多 ENTITY_QUERIES 条)。
讲算法的句子(「按会员号去重计数得出」)证据在 SQL 里;只给名字,裁判只能判不支持。
查询用条目自己记的(表:SQL 里用到它;字段:它是结果列)。只在 SQL 里出现、不是结果列的字段自己没记:
有表条目就从它记的查询里找,没有(文档目录只留正文用到的实体)就看全部查询,都只留 SQL 里写了这个
字段名(有表名的还要写了表名)的。快照经 fetch 取:取不到(不在封存范围里)的不给。
mask 同 _query_excerpt:SQL 先遮再截(要遮的列按那条查询的数据源认,masked 同 _query_excerpt)。
"""
locator = entry.get("locator") or {}
column = locator.get("column") if entry.get("kind") == "column" else None
@@ -683,6 +719,9 @@ def _entity_sql(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch) -
sql = _clean(str(snapshot.get("sql") or "")) if isinstance(snapshot, dict) else ""
if not sql or any(not re.search(rf"(?<![\w]){re.escape(n)}(?![\w])", sql, re.I) for n in need):
continue
if mask is not None:
source = catalog[q].get("source") or snapshot.get("source")
sql = mask.sql(sql, snapshot, _derived(snapshot, _hidden(snapshot, source, masked), fetch, mask)[1])
lines.append(f"出现在查询 {q}:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}")
if len(lines) == ENTITY_QUERIES:
break
@@ -739,10 +778,16 @@ class JudgeRequest:
def prepare(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, units: Iterable[str] | None = None,
loader: Callable[[str], Any] | None = None, masked: dict[str, Any] | None = None) -> JudgeRequest:
loader: Callable[[str], Any] | None = None, masked: dict[str, Any] | None = None,
mask: Any = None) -> JudgeRequest:
"""裁判的输入:候选句和证据摘录。纯函数(快照经 loader 取,缺省读工件库),不调模型。
catalog 缺省用文档自己存的目录;masked = {数据源名: 要遮的列},和快照里记下的 mask_columns 合在一起。
mask 只在给核查 agent(反方审稿、审批质询)准备摘录时给(engine/inquiry.py 的 ExcerptMask,这里只按它的 columns、
cell、sql、text 几个方法用):摘录分段遮,和核查 agent 的查询在工具边界遮的是同一套口径——改了名字的遮罩列
(buyer_phone AS 手机号)按 SQL 认出来也不给;SQL 照质询封存 SQL 的办法换,范围比较的阈值、LIMIT 照原样;格子逐格
遮,数值格照原样;列名、知识库片段、来历行按值换。报告里的指标值、原式、运行输入不遮(报告本来就摆在读者眼前)。
以前先渲染好整段、再整段按值换,阈值、没设遮罩的数也成了「已遮罩」。结论句裁判自己的摘录不给它,和以前逐字相同。
摘录按文档标记分版本(P4-SPEC 1.4、3.3):带 `provenance: DOC_PROVENANCE` 的文档,上传表格的查询摘录多几行
来历;没有标记的(期 4 之前组装的)和以前逐字相同,复用键、断点指纹都不变,已判过的句子照旧复用。
@@ -755,7 +800,7 @@ def prepare(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, units
for alias in dict.fromkeys(a for c in cands for a in c.aliases):
entry = catalog.get(alias)
if isinstance(entry, dict) and (text := _excerpt(alias, entry, cands, fetch, masked, catalog,
provenance_on=provenance_on)):
provenance_on=provenance_on, mask=mask)):
excerpts[alias] = text
picked = {c.unit for c in cands}
skipped: dict[str, str] = {}
File diff suppressed because it is too large Load Diff
+98 -23
View File
@@ -11,9 +11,12 @@
的 Q1 分开),要它先想清楚「如果这句不成立,数据里会看到什么」,写只读查询去找,最后交结论(经受住 / 被削弱 /
被推翻 / 无法判断)和理由。理由里的数字只能引用它自己查询的格子,裸数字由 compose_doc 标出来。
遮罩:agent 的查询结果在工具边界就遮好了;这份报告引用的查询快照里的遮罩值(按 SQL 认出改了名字的遮罩列,同审批
质询)反查之前就交给查询工具,agent 查到写着同一个号码的别的表时也看不到。摘录按列名遮,改了名字的遮罩列原值
agent 看得到,会照着写进理由:理由正文和理由文档按值换成「已遮罩」(换了写法的号码也认),这句结论原文里写着的
不换(_masked_rationale)
质询)反查之前就交给查询工具,agent 查到写着同一个号码的别的表时也看不到。任务里的证据摘录先遮(_task,和工具
边界同一套口径):遮罩列、按 SQL 认出改了名字的遮罩列(buyer_phone AS 手机号)不给,文字格、SQL 的字符串和注释里
的遮罩值按值换,和遮罩列做等值比较的字面量都换;数值格、范围比较的阈值、LIMIT 照原样。理由正文和理由文档仍按值换成
「已遮罩」(换了写法的号码也认,agent 查询时遮掉的值也算),这句结论原文里写着的、agent 自己 SQL 里照原样留着的
阈值和行数不换,目录里的列名同质询照快照换(_masked_rationale);换完正文、原文里还认得出的,这句记为无法判断、
理由不记(_kept,同审批质询的最后一道关)
3. 出处:一条查询都没执行成功的,判什么都记为无法判断;判「被推翻」的,理由里还得引用到自己的查询结果——它要让
正式出具降档,审稿记录里得查得到凭什么。不采信时系统写明原因,反方原本的结论和理由照录在后面(纯文字:解析
不了的引用写成「(引用无效)」,裸数字注明「(无出处)」)。这句该查的数据源都不存在或已停用、核查模型本身用
@@ -49,6 +52,7 @@
from __future__ import annotations
import asyncio
import logging
import re
from collections.abc import Callable, Collection, Iterable
from dataclasses import replace
@@ -76,8 +80,12 @@ from app.engine.state import GraphState
from app.engine.verify_agent import RATIONALE, VerifyOutcome, run_verify_agent
from app.tools.datasource import run_versions
logger = logging.getLogger(__name__)
#: 四种结论:经受住 / 被削弱 / 被推翻 / 无法判断。counts 的键照这个顺序
VERDICTS = ("stands", "weakened", "refuted", "inconclusive")
#: 理由按值遮之后还认得出遮罩值时,这句的理由换成这段系统说明(_kept)
LEAKED = "反方的理由里有设置了遮罩的值,无法遮住,这句的结论不予采信,记为无法判断"
#: 节点产出、审批卡里最多列几句(审稿记录里一句不少)
ITEMS_MAX = 10
#: 出具缺口、报错里最多点几句的原文,每句最多几个字
@@ -385,14 +393,18 @@ def _spared(seed: Any, claim: str) -> Any:
return known
def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[str | None]) -> VerifyOutcome:
def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[str | None],
kinds: dict[str, Any] | None = None, masks: dict[str, Any] | None = None) -> VerifyOutcome:
"""反方的理由里的遮罩值换成「已遮罩」,换了写法的号码也认(199-1474-2785、+86 …)。值是这份报告引用的查询快照里的
(secrets,反查之前就交给了查询工具),加上 agent 查询时在工具边界遮掉的(outcome.masked_values);被审的那句结论
原文里写着的(given)已经摆在读者眼前,不换。反方 agent 的查询结果在工具边界就遮好了,可证据摘录按列名遮,改了名字
的遮罩列(buyer_phone AS 手机号)的原值它看得到,会写进理由。
换的办法同审批质询的回答(_scrubbed_source):原文里的值先换掉,再照原来的目录重新组装理由文档;引用到的格子按
遮好的快照渲染(文字格里夹着的遮罩值换掉那一截)。理由里认不出遮罩值的原样交回。"""
原文里写着的(given)已经摆在读者眼前,不换。agent 自己 SQL 里照原样留着的范围边界、LIMIT 的行数(_shown:kinds 是
数据源的类型,masks 是数据源现在设的遮罩列)在正文里复述也不换,同审批质询的回答(「金额不低于 20000 元」)。
反方 agent 的查询结果在工具边界就遮好了、证据摘录也先遮了,这里是第二道防线。
换的办法同审批质询的回答(_scrubbed_source):原文里的值先换掉,再照换过列名的目录重新组装理由文档(同质询的
_renamed_entry:快照遮好的样子列名换过了,引用对得上);引用到的格子按遮好的快照渲染(文字格里夹着的遮罩值换掉
那一截)。理由、目录列名里都认不出遮罩值的原样交回。换完还认得出的过最后一道关(_kept)。"""
from app.engine.evidence import compose_doc
from app.engine.inquiry import _leaked, _renamed_entry
from app.engine.masking import _masked_view, _scrubbed_source, _Secrets
doc, result = outcome.rationale, outcome.result
@@ -403,10 +415,14 @@ def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[s
pool.protect(given)
if result is None or not isinstance(doc, dict) or not pool.strong:
return outcome
pool.show(_shown(outcome, pool, kinds or {}, masks or {}))
pool.prepare()
source = str(doc.get("source") or "")
if not (pool.spans(str(doc.get("markdown") or ""), prose=True) or pool.spans(source, prose=True)):
return outcome
catalog = doc.get("catalog") if isinstance(doc.get("catalog"), dict) else {}
renamed = {alias: _renamed_entry(entry, pool) for alias, entry in catalog.items()}
# 正文里认得出遮罩值的(照原样留着的阈值写进单元格引用里的也算,_leaked)、原文、目录列名里有的,才重新组装
if renamed == catalog and not (_leaked(doc, pool) or pool.spans(source, prose=True)):
return _kept(outcome, pool)
views: dict[str, Any] = {}
fetch = _fetcher()
for query in outcome.queries:
@@ -417,13 +433,52 @@ def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[s
return views[artifact] if artifact in views else artifact_store.load(artifact)
try:
again = compose_doc(_scrubbed_source(source, pool), doc.get("catalog") or {},
node_id=str(doc.get("node_id") or ""), run_id=doc.get("run_id"), loader=load)
again = compose_doc(_scrubbed_source(source, pool), renamed, node_id=str(doc.get("node_id") or ""),
run_id=doc.get("run_id"), loader=load)
except Exception: # noqa: BLE001 - 重新组装出了意外,值也不能留着
# 正文按值换掉,理由文档不要了(界面照正文显示;判「被推翻」的因此查不到出处,按无法判断记)
said = pool.scrub(str(doc.get("markdown") or ""), prose=True)
return replace(outcome, rationale=None, result={**result, RATIONALE: said})
return replace(outcome, rationale=again, result={**result, RATIONALE: again["markdown"]})
return _kept(replace(outcome, rationale=None, result={**result, RATIONALE: said}), pool)
return _kept(replace(outcome, rationale=again, result={**result, RATIONALE: again["markdown"]}), pool)
def _shown(outcome: VerifyOutcome, pool: Any, kinds: dict[str, Any], masks: dict[str, Any]) -> set[str]:
"""反方 agent 自己的 SQL 里照原样留着的数(范围比较的边界、LIMIT 的行数,审批质询的 _shown_literals):理由正文里
复述它们不换,最后一道关也不算露了遮罩值——它们随 SQL 存进审稿记录、摆在读者眼前,换掉只会让理由和 SQL 对不上。
哪些名字是遮罩列按那份快照认(_derived_hidden 的名字,含改了名字的),快照取不到的按数据源现在设的遮罩列。"""
from app.engine.inquiry import _shown_literals
from app.engine.masking import _derived_hidden
fetch = _fetcher()
sqls: list[tuple[str, str | None, set[str]]] = []
for query in outcome.queries:
name = str(query.get("source") or "")
base = {str(c).lower() for c in masks.get(name) or []}
snapshot = fetch(query.get("artifact"))
if isinstance(snapshot, dict):
base |= {str(c).lower() for c in snapshot.get("mask_columns") or [] if isinstance(c, str)}
compared = _derived_hidden(snapshot, base, kinds.get(name))[1] if isinstance(snapshot, dict) and base \
else base
sqls.append((str(query.get("sql") or ""), kinds.get(name), compared))
return _shown_literals(sqls, pool)
def _kept(outcome: VerifyOutcome, pool: Any) -> VerifyOutcome:
"""最后一道关(同审批质询的 _leaked):按值遮之后,理由正文、理由文档的正文和原文里还认得出遮罩值的(换了写法的号码
也算),这句的结论不采信、理由不记:记为无法判断,理由换成系统的说明(LEAKED)。遮不住的「被推翻」不能让出具降档,
也不能把号码存进审稿记录、随运行封存。只看正文和原文:目录里的列名是 agent 自己起的名字(不低于20000的笔数),
已经按值换过(_masked_rationale),再拿来认会把恰好等于遮罩值的阈值当成露了值,整句改判;agent 自己 SQL 里照原样
留着的阈值写在正文里不算(pool.show),渲染进单元格引用里的照算(同质询的 _leaked)。"""
from app.engine.inquiry import _leaked
doc = outcome.rationale if isinstance(outcome.rationale, dict) else {}
said = str((outcome.result or {}).get(RATIONALE) or "")
if not (pool.spans(said, prose=True) or pool.spans(str(doc.get("source") or ""), prose=True)
or (doc and _leaked(doc, pool))):
return outcome
logger.warning("反方审稿的理由里有遮罩值遮不住,这句记为无法判断(运行 %s)", doc.get("run_id"))
return replace(outcome, rationale=None, result={**(outcome.result or {}), "verdict": "inconclusive",
RATIONALE: LEAKED})
def _cites(doc: dict[str, Any] | None) -> bool:
@@ -573,6 +628,8 @@ class _Review:
max_claims, max_queries, timeout_s, daily_max_usd
self.spec, self.masks, self.writer_model = spec, masks, writer_model
self.fetch = _fetcher()
#: 报告用到的数据源的类型(摘录按 SQL 认改了名字的遮罩列时按它的方言解析)。run() 里查好
self.kinds: dict[str, Any] = {}
#: 这一段执行的总时长(engine/deadline.py):有句子的时长按剩余时间收紧过、或者不再开始时记下,说明和日志里写
self.run_seconds: float | None = None
#: 这份报告目录里的查询编号:摘录里指向它们的写法标上「原报告」(mark_original)
@@ -597,11 +654,19 @@ class _Review:
"""当天的核查金额用完、这句一次模型都没调用时的理由:派出去就撞上的和后面不再派出的,是同一回事。"""
return f"已达上限({self._limit_words('daily_max_usd')}),这句未反查"
def _task(self, cand: judging.Candidate, sources: list[str], lost: list[str], *, own: bool) -> str:
"""交给核查 agent 的任务:这句结论、它挂的证据摘录(遮罩照旧,原报告的编号标明,见 mark_original)、可以查的
数据源;该查的源有停用了的,也告诉它查不了哪几个,免得它把查不到当成没有反例。own:源是这句自己引用的,
不是没挂依据、按整份报告用到的源查的。"""
request = judging.prepare(self.doc, self.catalog, units=[cand.unit], loader=self.fetch, masked=self.masks)
def _task(self, cand: judging.Candidate, sources: list[str], lost: list[str], *, own: bool,
known: Any = None) -> str:
"""交给核查 agent 的任务:这句结论、它挂的证据摘录(原报告的编号标明,见 mark_original)、可以查的数据源;该查的
源有停用了的,也告诉它查不了哪几个,免得它把查不到当成没有反例。own:源是这句自己引用的,不是没挂依据、按整份
报告用到的源查的。
摘录先遮,模型从任务里也看不到遮罩值(judge.prepare 的 mask,inquiry.ExcerptMask,和工具边界同一套口径):遮罩列、
按 SQL 认出改了名字的(buyer_phone AS 手机号)不给;文字格、SQL 的字符串和注释里的遮罩值按值换成「已遮罩」,换了
写法的号码也认,和遮罩列做等值比较的字面量都换;数值格、范围比较的阈值、LIMIT 照原样(known:报告快照里的遮罩值
去掉这句结论原文里写着的,同交给查询工具的那一份)。解析 SQL、按值换是纯计算,在线程里做。"""
from app.engine.inquiry import ExcerptMask # inquiry 导入了这个模块,用到时再导入
request = judging.prepare(self.doc, self.catalog, units=[cand.unit], loader=self.fetch, masked=self.masks,
mask=ExcerptMask(self.kinds, known))
aliases = dict.fromkeys(a for c in request.cands for a in c.aliases if a in request.excerpts)
evidence = "\n\n".join(mark_original(request.excerpts[a], a, self.queries) for a in aliases)
gone = f"({'这句还引用了' if own else '这份报告还用到了'}{_quoted(lost)},已不存在或已停用,无法查询)" \
@@ -620,6 +685,7 @@ class _Review:
# 这句没挂依据时在整份报告引用的数据源里查:写作者看得到的就是这几个库
everywhere = claim_sources(self.catalog, self.catalog, self.fetch)
usable = await _usable_sources(everywhere)
self.kinds = await self._kinds(everywhere)
# 报告引用的查询快照里遮罩列的值:反查之前交给查询工具(agent 查到写着同一个号码的别的表时看不到),反查之后
# 据此换掉理由里的遮罩值(_masked_rationale)
seed = await self._seed()
@@ -653,12 +719,15 @@ class _Review:
claims.append(_claim(cand, "inconclusive", late[0]))
overdue += 1
continue
# 这句结论原文里写着的值已经摆在读者眼前,不当遮罩值。上万个值过一遍要几十毫秒,放进线程
known = await asyncio.to_thread(_spared, seed, cand.text)
try:
own = claim_sources(cand.aliases, self.catalog, self.fetch)
sources = own or everywhere
live = [s for s in sources if s in usable]
lost = [s for s in sources if s not in usable]
task_text = self._task(cand, live, lost, own=bool(own)) if live else ""
task_text = await asyncio.to_thread(self._task, cand, live, lost, own=bool(own), known=known) \
if live else ""
except Exception as e: # noqa: BLE001 - 一句的摘录出了意外(坏形状的工件),只让这一句没审成
why = f"整理证据摘录时出错({describe_exception(e)})"
unreached(cand, f"反查未完成:{why}", why)
@@ -686,14 +755,13 @@ class _Review:
if allot.tightened:
self.run_seconds = allot.seconds
# 收紧过的:核查 agent 到点自己发的 verify_limit 改说运行时长上限将至(deadline.relabel),不写收紧后的秒数
# 这句结论原文里写着的值已经摆在读者眼前,不当遮罩值。上万个值过一遍要几十毫秒,放进线程
known = await asyncio.to_thread(_spared, seed, cand.text)
outcome = await run_verify_agent(
spec=self.spec, system=CHALLENGE_SYSTEM, task=task_text, sources=live,
output_schema=CHALLENGE_SCHEMA, max_queries=self.max_queries, timeout_s=allot.timeout_s,
daily_max_usd=self.daily_max_usd, emit=deadline.relabel(ctx.emit, allot), run_id=ctx.run.run_id,
node_id=ctx.node.id, data_versions=run_versions(ctx.run), masked_values=known)
outcome = await asyncio.to_thread(_masked_rationale, outcome, known, given=[cand.text])
outcome = await asyncio.to_thread(_masked_rationale, outcome, known, given=[cand.text], kinds=self.kinds,
masks=self.masks)
if outcome.error and not outcome.model:
broken = outcome.error # 连模型都没建成(没配好、接入出错):换一句也一样
model = model or outcome.model
@@ -777,6 +845,13 @@ class _Review:
return await snapshot_secrets(self.fetch(entry.get("artifact")) for entry in self.catalog.values()
if isinstance(entry, dict) and entry.get("kind") == "query")
async def _kinds(self, names: Iterable[str]) -> dict[str, Any]:
"""这几个数据源的类型(摘录按 SQL 认改了名字的遮罩列时按它的方言解析)。names 是整份报告用到的数据源
(claim_sources:查询的、合并查询合并的、指标输入取自的、表和字段的)。"""
from app.engine.inquiry import _source_kinds # inquiry 导入了这个模块,用到时再导入
return await _source_kinds(names)
async def _store(self, record: dict[str, Any]) -> str | None:
try:
return await artifact_store.put_json(record, kind="challenge", run_id=self.ctx.run.run_id,
+32 -7
View File
@@ -8,11 +8,12 @@
**先遮再交。** 建查询工具时还打开遮罩开关(tools/datasource 的 mask_results):数据源设了遮罩时,取自遮罩列的格子
(按 SQL 认,起别名、套函数、经子查询或 WITH 转一道、SELECT *(展开的列换了名字的也算)、表值函数拆开的都算)在交给
模型、落进查询快照之前就换成「已遮罩」,别的格子按值换:这份结果里的遮罩值、前面的查询遮掉的、调用方交进来的
模型、落进查询快照之前就换成「已遮罩」,别的文字格按值换:这份结果里的遮罩值、前面的查询遮掉的、调用方交进来的
(masked_values:审批质询、反方审稿收的报告快照里的值),文字格里夹着的换那一截(engine/masking.py 的
mask_query_result);查询失败时报错原话里的值也换。模型看不到遮罩值,就不会把它写进回答、理由、SQL;以前是事后
按值认,每一轮复核都找到新的漏法。快照里的 masked_at_query 记下查询时就遮了的列,格子引用指到这几列、指到按值换掉
的格子的都不采信。遮掉的值只在内存里交回调用方(VerifyOutcome.masked_values,审批质询、反方审稿按值遮的第二道防线
mask_query_result);数值格不按值换(存下的是证据,没设遮罩的访问次数正好等于某一笔被遮的金额也照原样);查询失败时
报错原话里的值也换。模型看不到遮罩值,就不会把它写进回答、理由、SQL;以前是事后按值认,每一轮复核都找到新的漏法。
快照里的 masked_at_query 记下查询时就遮了的列,masked_cells 记下按值换过的格子,masked_whole 记下写法认不全、整份
都遮了,格子引用指到这几列、这几格的都不采信,照实说是遮罩(整份遮的说写法认不全)。遮掉的值只在内存里交回调用方(VerifyOutcome.masked_values,审批质询、反方审稿按值遮的第二道防线
用),不进快照、不交给模型、不进 as_dict。调用方交进来的和查询时遮掉的分开记(masking.SeenValues),合起来时 agent
查到的在前:合起来到了上限 SECRETS_MAX,挤掉的是报告里的值。
@@ -427,10 +428,12 @@ class _Cells:
"reason": f"{alias} {e}"})
return None
if str(name).lower() in _masked_at_query(content):
# 查询时就遮了的列:格子里是「已遮罩」,不是数据源里的值。照实说是遮罩列,不说成「不是数值」
# 查询时就遮了的列:格子里是「已遮罩」,不是数据源里的值。照实说是遮罩列,不说成「不是数值」;认不全、整份
# 遮的(快照记着 masked_whole)说是写法认不全——这一列不一定取自遮罩列,说成遮罩列会让审批人以为它设了遮罩
why = f"{alias} 的写法认不全,整份结果查询时已遮罩,不采信" if _masked_whole(content) \
else f"{alias} 的「{name}」取自设置了遮罩的列,查询时已遮罩,不采信"
self.cells.append({"path": path, "status": "unresolved", "ref": f"{alias}.r{row}.{name}", "alias": alias,
"row": row, "column": name, "artifact": query["artifact"],
"reason": f"{alias} 的「{name}」取自设置了遮罩的列,查询时已遮罩,不采信"})
"row": row, "column": name, "artifact": query["artifact"], "reason": why})
return None
if raw == _masked_word() and isinstance(content, dict) and "masked_at_query" in content:
# 这一格查询时按值换掉了(和别的查询、报告里的遮罩值相同):同样照实说
@@ -438,6 +441,13 @@ class _Cells:
"row": row, "column": name, "artifact": query["artifact"],
"reason": f"{alias} 第 {row} 行「{name}」与设置了遮罩的值相同,查询时已遮罩,不采信"})
return None
if _masked_cell(content, row, name):
# 文字里有一截查询时按值换掉了(快照的 masked_cells 记着):这一格不是数据库里的原样,同样照实说
self.cells.append({"path": path, "status": "unresolved", "ref": f"{alias}.r{row}.{name}", "alias": alias,
"row": row, "column": name, "artifact": query["artifact"],
"reason": f"{alias} 第 {row} 行「{name}」里有与设置了遮罩的值相同的部分,查询时已遮罩,"
"不采信"})
return None
truth = _typed(raw, {"type": sorted(types)}, column_kind(content, name))
cited = {"ref": f"{alias}.r{row}.{name}", "alias": alias, "row": row, "column": name,
"artifact": query["artifact"], "eid": cell_eid(query["artifact"], row, name), "value": truth}
@@ -539,6 +549,21 @@ def _masked_at_query(content: Any) -> set[str]:
return {c.lower() for c in listed if isinstance(c, str)} if isinstance(listed, list) else set()
def _masked_whole(content: Any) -> bool:
"""查询快照记着这条 SQL 的写法认不全、整份结果查询时都遮了(tools/datasource 记下的 masked_whole)。"""
return isinstance(content, dict) and content.get("masked_whole") is True
def _masked_cell(content: Any, row: Any, name: str) -> bool:
"""查询快照里第 row 行、name 这一列的格子查询时按值换过(tools/datasource 记下的 masked_cells:[行, 列号])。列名
同名的按第一列算,同 locate_cell。"""
listed = content.get("masked_cells") if isinstance(content, dict) else None
columns = [str(c) for c in content.get("columns") or []] if isinstance(content, dict) else []
if not isinstance(listed, list) or name not in columns:
return False
return [row, columns.index(name)] in listed
# --------------------------------------------------------------------------
# 花费
# --------------------------------------------------------------------------
+16 -3
View File
@@ -470,7 +470,8 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False,
mask_results:数据源设了遮罩时,取自遮罩列的格子(按 SQL 认,改了名字的也算)在交给模型、落进快照之前就换成
「已遮罩」,见 engine/masking.py 的 mask_query_result;这一次遮掉的值并进 ctx.extra[MASKED_VALUES_KEY](有的话)。
那里已经有的值(这个 agent 前面的查询遮掉的、调用方交进来的)这一次也按值换,数据源没设遮罩也一样:先查号码、
再查写着号码的备注或别的库,后一份同样遮好。查询失败时报错原话里的值也换(masking.mask_error_text)。
再查写着号码的备注或别的库,后一份同样遮好。按值换只换文字格,数值格照原样(存下的是证据)。查询失败时报错原话里
的值也换(masking.mask_error_text)。
"""
# 数据源在 options.query_timeout_s 里配了就用它,否则取缺省。数据库按它停下语句,
# 引擎按它告诉界面上限是多少(metadata["timeout_s"])
@@ -530,8 +531,11 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False,
payload = result.to_payload()
payload["source"] = source.name
# 查询时就整列遮掉的结果列;没开遮罩开关、数据源没设遮罩又没有可按值换的时为 None(快照和原来一字不差)
# 查询时就整列遮掉的结果列;没开遮罩开关、数据源没设遮罩又没有可按值换的时为 None(快照和原来一字不差)。
# 别的列里按值换过的格子另记(masked_cells),没有就不记
masked_now: list[str] | None = None
masked_cells: list[list[int]] = []
masked_whole = False
if mask_results and (mask or known is not None):
from app.engine.masking import mask_query_result
@@ -539,7 +543,8 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False,
done = await asyncio.to_thread(mask_query_result, payload, mask, source.kind, known,
schema=source.schema_cache)
if mask or done.payload is not payload:
payload, masked_now = done.payload, done.columns
payload, masked_now, masked_cells = done.payload, done.columns, done.cells
masked_whole = done.whole
if sink is not None:
# 并进 agent 查到的那一份(不和调用方交进来的混在一起,上限收满了也不丢),也放进线程
await asyncio.to_thread(sink.add, done.secrets)
@@ -562,6 +567,10 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False,
# 同源各份快照记的这一项并起来,从表的字段清单里去掉,混进结果列的别名会把同名的真实字段也去掉;
# - masked_at_query:开了遮罩开关时,查询时就整列遮掉的结果列(这几列的格子是「已遮罩」,不是数据源里的值,
# 改了名字的也在这里)。有这一项的快照,别的格子整格是「已遮罩」的是查询时按值换掉的;
# - masked_cells:别的列里查询时按值换过的文字格 [行, 列号](整格换的、换掉一截的),格子引用指到它们时照实
# 说是遮罩。数值格不按值换,存下的数和数据库里的一样;
# - masked_whole:这条 SQL 的写法认不全,整份结果查询时都遮了(masked_at_query 是全部的列),格子引用指到它时
# 照实说写法认不全,不说成取自遮罩列;
# - data_version:上传表格查的是哪个快照,事后据此认得出这个数出自哪一版数据。
# 手工源、没设遮罩的,快照和原来一字不差
extra: dict[str, Any] = {}
@@ -569,6 +578,10 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False,
extra["mask_columns"] = mask
if masked_now is not None:
extra["masked_at_query"] = masked_now
if masked_cells:
extra["masked_cells"] = masked_cells
if masked_whole:
extra["masked_whole"] = True
if version := getattr(source, "snapshot_id", None):
extra["data_version"] = version
# 工件引用的 meta 记下源和 SQL 里用到的表:数据目录按它数每张表被运行查询过几次
+246
View File
@@ -0,0 +1,246 @@
"""反方审稿的遮罩收口(engine/nodes/challenge.py):
- 交给反方 agent 的证据摘录先遮:报告取数把遮罩列改了名字选出来(buyer_phone AS 手机号),摘录里不给这一列(按 SQL
认,和工具边界同一套规则);别的列(备注里换了写法的号码)、SQL 里写着的遮罩值换成「已遮罩」。以前摘录只按列名遮,
四个号码原样交给模型,模型照抄进理由
- 理由正文按值遮(报告快照里的值、agent 查询时遮掉的值、换了写法的号码),端到端;组装之后还认得出遮罩值的(第二道
防线漏了),整句降为无法判断,理由不记,同审批质询的最后一道关
模型一律是 MockChatModel 剧本,数据源是现造的 SQLite 文件,区域、金额、手机号都是虚构的。
"""
from __future__ import annotations
import json
import sqlite3
import uuid
import pytest
from app.core import artifact_store
from app.data import introspect
from app.data.engine import engines
from app.db.base import SessionLocal
from app.db.models import DataSource
from app.engine import masking
from app.engine.nodes import challenge as challenge_mod
from tests.test_challenge_node import Models, Play, engine_up, finish, graph, output_of # noqa: F401 - engine_up 是自动用上的夹具
#: 四笔订单,买家手机号设了遮罩;备注没设遮罩,其中一条写着换了写法的号码
SCRIPT = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_phone TEXT, amount INTEGER, refunded INTEGER,"
" note TEXT);"
"INSERT INTO sales VALUES (1, '华东', '13900000001', 1200, 0, '客户 139-0000-0002 代下单'),"
" (2, '华东', '13900000002', 300, 1, ''), (3, '华南', '13900000003', 900, 0, '例行回访'),"
" (4, '华北', '13900000004', 400, 0, '');")
SECRETS = ("1390000000", "0000-0002", "139 0000")
MASKED = "已遮罩"
@pytest.fixture
async def masked_shop(tmp_path):
path = tmp_path / f"{uuid.uuid4().hex[:8]}.db"
db = sqlite3.connect(path)
db.executescript(SCRIPT)
db.commit()
db.close()
name = f"shop_{uuid.uuid4().hex[:8]}"
async with SessionLocal() as session:
row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True,
options={"mask_columns": ["buyer_phone"]})
session.add(row)
await session.flush()
row.schema_cache = await introspect.introspect(row)
await session.commit()
source_id = row.id
yield name
await engines.invalidate(source_id)
def _clean(text: str) -> bool:
return not any(s in text for s in SECRETS)
#: 报告取数:手机号改了名字选出来,备注照选,按号码排序时把号码写进了 SQL
REPORT_SQL = ("SELECT buyer_phone AS 手机号, note AS 备注, amount AS 金额 FROM sales "
"ORDER BY buyer_phone = '13900000001' DESC, amount DESC")
WRITER = ("金额最高的一笔 [[v:Q1.r0.金额]]。[[see:Q1]]",)
CLAIM = "金额最高的一笔"
async def test_the_excerpt_in_the_task_is_masked_before_the_agent_sees_it(monkeypatch, masked_shop):
"""反方 agent 收到的任务里:改了名字的遮罩列(手机号)不给,备注里换了写法的号码、SQL 里写着的号码换成「已遮罩」,
金额照常。以前「列:金额 | 手机号」「r0:1,200 | 13900000001」…,四个号码都在。"""
name = masked_shop
plays = {CLAIM: Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "stands", "共 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]")}
models = Models(monkeypatch, name, writer=WRITER, plays=plays)
row = await finish(graph(name, sql=REPORT_SQL), run_class="exploratory")
assert row.status == "succeeded", row.error
task = models.tasks()[CLAIM]
assert "金额最高的一笔 1,200。" in task and "【原报告 Q1】" in task
excerpt = task[task.index("【原报告 Q1】"):]
assert _clean(excerpt), excerpt
columns = next(line for line in excerpt.splitlines() if line.startswith("列:"))
assert "手机号" not in columns and "金额" in columns and "备注" in columns
assert "客户 已遮罩 代下单" in excerpt and "buyer_phone = '已遮罩'" in excerpt
assert "(有的列按数据源的设置遮罩了,没有给出)" in excerpt
async def test_masked_values_the_agent_found_itself_are_masked_in_the_rationale(monkeypatch, masked_shop):
"""端到端:反方 agent 自己查出的号码(查询时在工具边界遮掉、只在内存里)被它换着写法写进理由:审稿记录里的理由正文、
理由文档都换成「已遮罩」。报告没有选出号码,值只来自 agent 查询时遮掉的那一份。"""
name = masked_shop
top = "SELECT buyer_phone AS 电话, amount AS 金额 FROM sales WHERE region = '华南'"
said = "华南那笔来自 +86 139 0000 0003,金额 [[v:Q1.r0.金额]]。[[see:Q1]]"
plays = {CLAIM: Play([top], "stands", said)}
Models(monkeypatch, name, writer=WRITER, plays=plays)
row = await finish(graph(name, sql="SELECT region AS 区域, amount AS 金额 FROM sales ORDER BY amount DESC"),
run_class="exploratory")
assert row.status == "succeeded", row.error
record = artifact_store.load((await output_of(row.id, "chal"))["challenge_artifact"])
[claim] = [c for c in record["claims"] if CLAIM in c["text"]]
assert claim["verdict"] == "stands" and claim["rationale"] == "华南那笔来自 已遮罩,金额 900。"
assert claim["rationale_doc"]["markdown"] == claim["rationale"]
assert "0000 0003" not in json.dumps(record, ensure_ascii=False)
async def test_a_rationale_still_showing_a_masked_value_is_not_kept(monkeypatch, masked_shop):
"""最后一道关:按值遮之后组装出来的理由里还认得出遮罩值(这里让按值换失手),这句降为无法判断,理由不记、不存,
原值不进审稿记录。同审批质询的回答。"""
name = masked_shop
monkeypatch.setattr(masking, "_scrubbed_source", lambda text, secrets: text)
said = "号码 13900000001 的那笔属实,金额 [[v:Q1.r0.笔数]]。[[see:Q1]]"
plays = {CLAIM: Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "refuted", said)}
Models(monkeypatch, name, writer=WRITER, plays=plays)
row = await finish(graph(name, sql=REPORT_SQL), run_class="exploratory")
assert row.status == "succeeded", row.error
record = artifact_store.load((await output_of(row.id, "chal"))["challenge_artifact"])
[claim] = [c for c in record["claims"] if CLAIM in c["text"]]
assert claim["verdict"] == "inconclusive" and claim["rationale_doc"] is None
assert claim["rationale"] == challenge_mod.LEAKED
assert _clean(json.dumps(record, ensure_ascii=False))
out = await output_of(row.id, "chal")
assert out["counts"]["refuted"] == 0, "遮不住的理由不能让出具降档"
# --------------------------------------------------------------------------
# 摘录分段遮:阈值、没设遮罩的数值格照原样,和遮罩列做等值比较的字面量都换
# --------------------------------------------------------------------------
#: 金额、客户名设了遮罩,金额都是五位数;访问次数没设遮罩,第二、三、四单正好等于别的订单的金额
BIG = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_name TEXT, amount INTEGER, visits INTEGER);"
"INSERT INTO sales VALUES (1, '华东', '张伟', 10000, 25000), (2, '华东', '李娜', 12000, 20000),"
" (3, '华南', '王芳', 20000, 30000), (4, '华北', '刘洋', 30000, 50000), (5, '华北', '赵敏', 50000, 9000);")
@pytest.fixture
async def big_shop(tmp_path):
path = tmp_path / f"{uuid.uuid4().hex[:8]}.db"
db = sqlite3.connect(path)
db.executescript(BIG)
db.commit()
db.close()
name = f"shop_{uuid.uuid4().hex[:8]}"
async with SessionLocal() as session:
row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True,
options={"mask_columns": ["amount", "buyer_name"]})
session.add(row)
await session.flush()
row.schema_cache = await introspect.introspect(row)
await session.commit()
source_id = row.id
yield name
await engines.invalidate(source_id)
def _excerpt_of(task: str) -> str:
return task[task.index("【原报告 Q1】"):]
async def test_thresholds_and_unmasked_numbers_in_the_excerpt_are_kept(monkeypatch, big_shop):
"""报告按金额阈值取数(amount >= 20000,正好等于某一笔被遮的金额),访问次数没设遮罩、第二行正好等于另一笔金额:
摘录里 SQL 的阈值、访问次数都照原样,金额列不给。以前对渲染好的整段按值换,成了「amount >= 已遮罩」「r1:4 | 华北 |
已遮罩」,agent 看不到报告按什么取数,没设遮罩的数也被改写。"""
report = "SELECT id, region, amount, visits FROM sales WHERE amount >= 20000 ORDER BY id"
claim = "大额订单里访问最多的一笔在华南"
plays = {claim: Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "stands", "共 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]")}
models = Models(monkeypatch, big_shop, writer=(f"{claim},访问 [[v:Q1.r0.visits]] 次。[[see:Q1]]",),
plays=plays)
row = await finish(graph(big_shop, sql=report), run_class="exploratory")
assert row.status == "succeeded", row.error
excerpt = _excerpt_of(models.tasks()[claim])
assert f"SQL:{report}" in excerpt, excerpt
assert "列:visits | id | region" in excerpt
assert "r1:50,000 | 4 | 华北" in excerpt and MASKED not in excerpt.split("列:", 1)[1].split("(", 1)[0]
async def test_literals_compared_with_a_masked_column_are_masked_even_when_never_collected(monkeypatch, masked_shop):
"""报告没选出号码,只在条件里排除了两个(一个换了写法):报告快照里收不到这两个值,摘录的 SQL 照样换掉——和遮罩列
做等值、IN 比较的字面量不论认不认得出都换。以前按值换只认快照里收到的,号码原样交给 agent。"""
report = ("SELECT region AS 区域, COUNT(*) AS 笔数 FROM sales WHERE buyer_phone NOT IN ('13900000002', "
"'139-0000-0003') GROUP BY region ORDER BY 笔数 DESC")
claim = "笔数最多的区域"
plays = {claim: Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "stands", "共 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]")}
models = Models(monkeypatch, masked_shop, writer=(f"{claim}有 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]",), plays=plays)
row = await finish(graph(masked_shop, sql=report), run_class="exploratory")
assert row.status == "succeeded", row.error
task = models.tasks()[claim]
assert _clean(task), task
assert f"buyer_phone NOT IN ('{MASKED}', '{MASKED}')" in _excerpt_of(task)
# --------------------------------------------------------------------------
# 最后一道关只看正文和原文:agent 起的列名、照原样留着的阈值不算露了遮罩值
# --------------------------------------------------------------------------
AMOUNTS = "SELECT region AS 区域, amount AS 金额 FROM sales ORDER BY id"
FIRST = "华东首单金额"
@pytest.mark.parametrize("verdict", ["stands", "refuted"])
async def test_a_threshold_in_an_alias_does_not_void_the_verdict(monkeypatch, big_shop, verdict):
"""反方 agent 把阈值写进别名(不低于20000的笔数),20000 恰好是某一笔被遮的金额:结论照常采信,理由照常记。以前最后
一道关把目录里的列名当成露了遮罩值,整句改判无法判断,「被推翻」也不再让出具降档,日志里多一条误报。"""
top = "SELECT COUNT(*) AS 不低于20000的笔数 FROM sales WHERE amount >= 20000"
said = "金额不低于两万的订单有 [[v:Q1.r0.不低于20000的笔数]] 笔,与结论" + ("不矛盾" if verdict == "stands" else "矛盾") \
+ "。[[see:Q1]]"
Models(monkeypatch, big_shop, writer=(f"{FIRST} [[v:Q1.r0.金额]]。[[see:Q1]]",),
plays={FIRST: Play([top], verdict, said)})
row = await finish(graph(big_shop, sql=AMOUNTS), run_class="exploratory")
assert row.status == "succeeded", row.error
out = await output_of(row.id, "chal")
record = artifact_store.load(out["challenge_artifact"])
[claim] = [c for c in record["claims"] if FIRST in c["text"]]
assert claim["verdict"] == verdict, claim["rationale"]
assert claim["rationale"].startswith("金额不低于两万的订单有 3 笔")
assert out["counts"][verdict] == 1
async def test_a_threshold_the_agent_wrote_in_its_sql_stays_in_the_rationale(monkeypatch, big_shop):
"""理由正文复述 agent 自己 SQL 里照原样留着的阈值(「不低于 20000 元」,恰好等于某一笔被遮的金额):照原样,同审批质询
的回答;它随 SQL 存进审稿记录、摆在读者眼前。以前换成「已遮罩」,理由和 SQL 对不上。"""
top = "SELECT COUNT(*) AS 笔数 FROM sales WHERE amount >= 20000"
said = "金额不低于 20000 元的订单有 [[v:Q1.r0.笔数]] 笔,与结论不矛盾。[[see:Q1]]"
Models(monkeypatch, big_shop, writer=(f"{FIRST} [[v:Q1.r0.金额]]。[[see:Q1]]",),
plays={FIRST: Play([top], "stands", said)})
row = await finish(graph(big_shop, sql=AMOUNTS), run_class="exploratory")
assert row.status == "succeeded", row.error
record = artifact_store.load((await output_of(row.id, "chal"))["challenge_artifact"])
[claim] = [c for c in record["claims"] if FIRST in c["text"]]
assert (claim["verdict"], claim["rationale"]) == ("stands", "金额不低于 20000 元的订单有 3 笔,与结论不矛盾。")
async def test_an_alias_naming_a_masked_value_is_renamed_like_the_snapshot(monkeypatch, big_shop):
"""agent 把找的那个值写进了别名(金额为20000的笔数,20000 是一笔被遮的金额,不是阈值):理由文档的目录列名照快照遮好的
样子换(同质询的 _renamed_entry),引用照样对得上,结论照常采信,存下的理由文档里没有这个值。以前目录原样传进重新
组装,最后一道关又把目录拿去认,整句改判无法判断。"""
top = "SELECT COUNT(*) AS 金额为20000的笔数 FROM sales WHERE amount = 20000"
said = "金额为两万的订单有 [[v:Q1.r0.金额为20000的笔数]] 笔,与结论不矛盾。[[see:Q1]]"
Models(monkeypatch, big_shop, writer=(f"{FIRST} [[v:Q1.r0.金额]]。[[see:Q1]]",),
plays={FIRST: Play([top], "stands", said)})
row = await finish(graph(big_shop, sql=AMOUNTS), run_class="exploratory")
assert row.status == "succeeded", row.error
record = artifact_store.load((await output_of(row.id, "chal"))["challenge_artifact"])
[claim] = [c for c in record["claims"] if FIRST in c["text"]]
assert (claim["verdict"], claim["rationale"]) == ("stands", "金额为两万的订单有 1 笔,与结论不矛盾。")
assert "20000" not in json.dumps(claim["rationale_doc"], ensure_ascii=False)
+9 -7
View File
@@ -183,10 +183,13 @@ BANDS = "CASE WHEN amount BETWEEN 0 AND 20000 THEN '小单' WHEN amount BETWEEN
"SELECT CASE WHEN amount <= 20000 THEN 1 WHEN amount <= 60000 THEN 2 ELSE NULL END AS 档位, COUNT(*) AS 笔数 "
"FROM sales GROUP BY 1",
"SELECT IIF(amount > 20000, '大单', '小单') AS 档位, COUNT(*) AS 笔数 FROM sales GROUP BY 1",
# 分档写在子查询、WITH 里,外层再按它分组(K3b):同样每一档一行
f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT {BANDS} AS 档位 FROM sales) t GROUP BY 档位",
f"WITH t AS (SELECT {BANDS} AS 档位 FROM sales) SELECT 档位, COUNT(*) AS 笔数 FROM t GROUP BY 1",
])
def test_a_band_label_used_as_the_group_key_is_not_a_masked_value(sql):
"""按遮罩列分档、每一档一行(CASE、IIF 的结果全是写死的标签,又是分组键):标签只说一行落在哪一档,带不出金额,
档位列不遮,才分得清哪一档是几笔。笔数照样只是计数。"""
"""按遮罩列分档、每一档一行(CASE、IIF 的结果全是写死的标签,又是分组键;分档写在派生来源里、外层按它分组的也算):
标签只说一行落在哪一档,带不出金额,档位列不遮,才分得清哪一档是几笔。笔数照样只是计数。"""
assert hidden_of(sql, ["档位", "笔数"], {"amount"}) == {"amount"}
@@ -195,8 +198,8 @@ def test_a_band_label_used_as_the_group_key_is_not_a_masked_value(sql):
f"SELECT id, {BANDS} AS 档位, 1 AS 笔数 FROM sales",
# 结果里有一支取的是金额本身
"SELECT CASE WHEN amount > 20000 THEN amount ELSE 0 END AS 档位, COUNT(*) AS 笔数 FROM sales GROUP BY 1",
# 分档写在子查询里,外层再分组:子查询里的档位逐行带出来,照旧算
f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT {BANDS} AS 档位 FROM sales) t GROUP BY 档位",
# 分档写在子查询里,外层不分组:子查询里的档位逐行带出来,照旧算
f"SELECT 档位, 1 AS 笔数 FROM (SELECT {BANDS} AS 档位 FROM sales) t",
])
def test_band_labels_row_by_row_still_count_as_derived(sql):
assert hidden_of(sql, ["档位", "笔数"], {"amount"}) >= {"amount", "档位"}
@@ -265,9 +268,8 @@ def test_a_renamed_masked_column_keeps_values_that_match_labels_written_elsewher
@pytest.mark.parametrize("sql, columns, rows", [
# 标签列经子查询、CTE 原样传下去(外层再分组也一样:每一行的档位照旧算派生值,见上)
(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT {BANDS} AS 档位 FROM sales) t GROUP BY 档位",
["档位", "笔数"], [["大单", 2], ["小单", 3]]),
# 标签列经子查询、CTE 原样传下去,外层不分组:每一行的档位照旧算派生值(外层按它分组的不遮,见上)
(f"SELECT 档位, 1 AS 笔数 FROM (SELECT {BANDS} AS 档位 FROM sales) t", ["档位", "笔数"], [["大单", 1], ["小单", 1]]),
(f"WITH t AS (SELECT {BANDS} AS 档位 FROM sales) SELECT * FROM t", ["档位"], [["小单"], ["大单"]]),
# 没起别名的标签列:按位置认
(f"SELECT id, {BANDS} FROM sales", ["id", BANDS], [[1, "小单"], [2, "大单"]]),
+170
View File
@@ -0,0 +1,170 @@
"""审批质询的遮罩收口,端到端(engine/inquiry.py 经核查 agent 的工具边界):
- 分档写在子查询、WITH 里、外层按档位分组:档位列交给模型、存下、渲染进回答都照常,标签不当遮罩值,SQL 照原样封存;
以前回答成了「已遮罩有 3 笔,已遮罩有 2 笔。」
- 交给质询 agent 的证据摘录先遮:报告取数把遮罩列改了名字选出来(buyer_phone AS 手机号),摘录里不给这一列;别的列、
SQL 里写着的遮罩值(换了写法的号码也算)换成「已遮罩」。以前摘录只按列名遮,号码原样交给模型
模型一律是 MockChatModel 剧本,数据源是现造的 SQLite 文件,区域、金额、手机号都是虚构的。
"""
from __future__ import annotations
import pytest
from app.core import artifact_store
from tests.test_inquiry_api import ( # noqa: F401 - engine_up 是自动用上的夹具,client、big_shop、masked_shop 是夹具
AMOUNTS,
Models,
Play,
ask,
big_shop,
client,
engine_up,
events,
fetching,
masked_shop,
pause,
pending,
report_doc,
unit_of,
)
BAND = "CASE WHEN amount >= 20000 THEN '大单' ELSE '小单' END"
@pytest.mark.parametrize("sql", [
pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT *, {BAND} AS 档位 FROM sales) GROUP BY 档位 ORDER BY 档位",
id="子查询里 * 加分档"),
pytest.param(f"WITH b AS (SELECT {BAND} AS 档位 FROM sales) SELECT 档位, COUNT(*) AS 笔数 FROM b GROUP BY 档位 "
"ORDER BY 档位", id="WITH 里只有分档"),
])
async def test_a_band_grouped_outside_its_derived_source_reads_as_written(monkeypatch, big_shop, client, sql):
"""金额设了遮罩,问两万元上下各有几笔:分档写在派生来源里、外层按档位分组。模型看得到档位、回答里引用的档位和它写的
「大单」「小单」都照原样,SQL 照原样封存,质询工件里档位列照常显示。"""
said = "[[v:Q1.r0.档位]]有 [[v:Q1.r0.笔数]] 笔,[[v:Q1.r1.档位]]有 [[v:Q1.r1.笔数]] 笔,大单少于小单。[[see:Q1]]"
models = Models(monkeypatch, big_shop, writer="华东首单金额 [[v:Q1.r0.金额]]。[[see:Q1]]",
plays={"两万元上下": Play([sql], said)})
paused = await pause(big_shop, graph=fetching(big_shop, AMOUNTS))
[approval] = await pending(paused.id)
r = await ask(client, paused.id, report="write", unit=None, question="两万元上下各有几笔?",
approval_id=approval.id)
assert r.status_code == 200, r.text
item = r.json()["inquiry"]
assert (item["answer"]["ok"], item["answer"]["text"]) == (True, "大单有 3 笔,小单有 2 笔,大单少于小单。")
[query] = item["queries"]
assert query["sql"] == sql, "SQL 里的标签不当遮罩值换"
[event] = await events(paused.id, "approval.inquiry")
assert event.data["queries"][0]["sql"] == sql
snapshot = artifact_store.load(query["artifact"])
assert snapshot["rows"] == [["大单", 3], ["小单", 2]] and snapshot["masked_at_query"] == []
seen = [str(m.content) for messages in models.finals for m in messages]
assert any('r0:档位="大单",笔数=3' in text for text in seen), "模型看得到档位"
got = await client.get(f"/api/runs/{paused.id}/inquiries/artifacts/{query['artifact']}")
assert got.status_code == 200, got.text
assert (got.json()["content"]["rows"], got.json()["masked"]) == ([["大单", 3], ["小单", 2]], [])
#: 报告取数:手机号改了名字选出来,按号码排序时把号码写进了 SQL
RENAMED = "SELECT buyer_phone AS 手机号, amount AS 金额 FROM sales ORDER BY buyer_phone = '13900000001' DESC, amount DESC"
async def test_the_excerpt_in_the_task_is_masked_before_the_agent_sees_it(monkeypatch, masked_shop, client):
"""针对「金额最高的一笔 1,200。」发起质询:任务里这句挂的证据摘录不给改了名字的遮罩列(手机号),SQL 里写着的号码
换成「已遮罩」,金额照常。以前「列:金额 | 手机号」「r0:1,200 | 13900000001」…,四个号码都在。"""
models = Models(monkeypatch, masked_shop, writer="金额最高的一笔 [[v:Q1.r0.金额]]。[[see:Q1]]",
plays={"这笔是谁买的": Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "共 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]")})
paused = await pause(masked_shop, graph=fetching(masked_shop, RENAMED))
[approval] = await pending(paused.id)
doc = await report_doc(paused.id)
r = await ask(client, paused.id, report="write", unit=unit_of(doc, "金额最高的一笔 1,200。"),
question="这笔是谁买的?", approval_id=approval.id)
assert r.status_code == 200, r.text
[task] = models.tasks
excerpt = task[task.index("【原报告 Q1】"):]
assert "1390000000" not in task, excerpt
columns = next(line for line in excerpt.splitlines() if line.startswith("列:"))
assert "手机号" not in columns and "金额" in columns
assert "buyer_phone = '已遮罩'" in excerpt and "(有的列按数据源的设置遮罩了,没有给出)" in excerpt
assert r.json()["inquiry"]["answer"]["ok"] is True
# --------------------------------------------------------------------------
# 摘录分段遮:阈值、没设遮罩的数值格、被问那句里的值照原样
# --------------------------------------------------------------------------
#: 报告按金额阈值取数(12000 正好是某一笔被遮的金额),访问次数没设遮罩
THRESHOLD = ("SELECT id AS 编号, region AS 区域, amount AS 金额, visits AS 访问 FROM sales WHERE amount >= 12000 "
"ORDER BY id")
async def test_the_excerpt_keeps_thresholds_and_unmasked_numbers(monkeypatch, big_shop, client):
"""针对「华南首单访问 30,000 次。」发起质询:摘录里 SQL 的阈值(12000,恰好是一笔被遮的金额)照原样,访问次数(没设
遮罩,20000、30000 恰好等于别的订单的金额)照原样,金额列不给。以前对渲染好的整段按值换:「amount >= 已遮罩」
「r0:已遮罩 | 2 | 华东」,被问的那一格也成了「已遮罩」,任务开头的原句却写着 30,000。"""
models = Models(monkeypatch, big_shop, writer="华南首单访问 [[v:Q1.r1.访问]] 次。[[see:Q1]]",
plays={"访问为什么这么高": Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "共 [[v:Q1.r0.笔数]] 笔。")})
paused = await pause(big_shop, graph=fetching(big_shop, THRESHOLD))
[approval] = await pending(paused.id)
doc = await report_doc(paused.id)
r = await ask(client, paused.id, report="write", unit=unit_of(doc, "华南首单访问 30,000 次。"),
question="访问为什么这么高?", approval_id=approval.id)
assert r.status_code == 200, r.text
[task] = models.tasks
excerpt = task[task.index("【原报告 Q1】"):]
assert f"SQL:{THRESHOLD}" in excerpt, excerpt
assert "列:访问 | 编号 | 区域" in excerpt and "r0:20,000 | 2 | 华东" in excerpt and "r1:30,000 | 3 | 华南" in excerpt
#: 买家姓名设了遮罩;推荐人没设遮罩,写着的恰好是另一位买家的名字
REFERRALS = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_name TEXT, referrer TEXT, amount INTEGER);"
"INSERT INTO sales VALUES (1, '华东', '张伟', '李娜', 1200), (2, '华南', '李娜', '张伟', 900),"
" (3, '华北', '王芳', '刘洋', 400);")
@pytest.fixture
async def referral_shop(tmp_path):
from app.data.engine import engines
from tests.test_inquiry_api import _source
name, source_id = await _source(tmp_path, options={"mask_columns": "buyer_name"}, script=REFERRALS)
yield name
await engines.invalidate(source_id)
async def test_values_in_the_asked_sentence_stay_in_the_excerpt_and_the_answer(monkeypatch, referral_shop, client):
"""被问的那句写着「李娜」(推荐人这一格,恰好也是另一笔的买家):它就在任务开头、摆在审批人眼前,摘录里这一格、回答里
复述的都照原样(同反方审稿的 _spared);别的买家名照旧换。以前质询不把这句原文算作已经摆在眼前的,这一格成了
「已遮罩」,和任务开头写着的原句对不上。"""
report = "SELECT buyer_name AS 买家, referrer AS 推荐人, amount AS 金额 FROM sales ORDER BY id"
said = "推荐人是李娜,第二单的推荐人是 [[v:Q1.r1.推荐人]]。[[see:Q1]]"
models = Models(monkeypatch, referral_shop, writer="首单由 [[v:Q1.r0.推荐人]] 推荐。[[see:Q1]]",
plays={"谁推荐的": Play(["SELECT id, referrer AS 推荐人 FROM sales ORDER BY id"], said)})
paused = await pause(referral_shop, graph=fetching(referral_shop, report))
[approval] = await pending(paused.id)
doc = await report_doc(paused.id)
r = await ask(client, paused.id, report="write", unit=unit_of(doc, "首单由 李娜 推荐。"),
question="谁推荐的?", approval_id=approval.id)
assert r.status_code == 200, r.text
[task] = models.tasks
excerpt = task[task.index("【原报告 Q1】"):]
assert "列:推荐人 | 金额" in excerpt and "r0:李娜 | 1,200" in excerpt and "r1:已遮罩 | 900" in excerpt, excerpt
item = r.json()["inquiry"]
assert item["answer"]["text"] == "推荐人是李娜,第二单的推荐人是 已遮罩。"
async def test_literals_compared_with_a_masked_column_are_masked_in_the_excerpt(monkeypatch, masked_shop, client):
"""报告没选出号码,只在条件里排除了两个(一个换了写法):报告快照里收不到这两个值,摘录的 SQL 照样换掉。"""
report = ("SELECT region AS 区域, COUNT(*) AS 笔数 FROM sales WHERE buyer_phone NOT IN ('13900000002', "
"'139-0000-0003') GROUP BY region ORDER BY 笔数 DESC")
models = Models(monkeypatch, masked_shop, writer="笔数最多的区域有 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]",
plays={"按什么排除": Play([], "查不到判断所需的数据。")})
paused = await pause(masked_shop, graph=fetching(masked_shop, report))
[approval] = await pending(paused.id)
doc = await report_doc(paused.id)
r = await ask(client, paused.id, report="write", unit=unit_of(doc, "笔数最多的区域有 1 笔。"),
question="按什么排除的?", approval_id=approval.id)
assert r.status_code == 200, r.text
[task] = models.tasks
assert "13900000002" not in task and "0000-0003" not in task, task
assert "buyer_phone NOT IN ('已遮罩', '已遮罩')" in task
+111
View File
@@ -0,0 +1,111 @@
"""交给核查 agent 的证据摘录分段遮(engine/inquiry.py 的 ExcerptMask、_scrub_excerpt_sql,engine/masking.py 的
masked_cell):
- SQL 照质询封存 SQL 的办法换:范围比较的阈值、LIMIT 的行数照原样(恰好等于某个遮罩值也一样),字符串、注释、名字里的
遮罩值换;和遮罩列(含改了名字的)做等值、IN、LIKE 一类比较的字面量不论认不认得出都换(报告没选出遮罩列时快照里收
不到这些值),同一个数写成阈值的跟着换;已经摆在读者眼前的值(问题、被问的那句)照旧不换
- 格子逐格遮:数值格(数、列类型是 number 的)照原样,文字格整格的、夹着的那一截换掉(换了写法的号码也认)
以前对渲染好的整段摘录按值换,分不出格子和 SQL 里的用法:阈值、没设遮罩的访问次数恰好等于某一笔被遮的金额,都成了
「已遮罩」。端到端的用例在 test_challenge_masking、test_inquiry_masking_e2e。号码、姓名、金额都是虚构的。
"""
from __future__ import annotations
import pytest
from app.engine import masking
from app.engine.inquiry import ExcerptMask, _scrub_excerpt_sql
MASKED = "已遮罩"
def secrets_of(*values, protect: tuple[str, ...] = ()) -> masking._Secrets:
"""报告快照里收到的遮罩值(金额、号码),去掉已经摆在读者眼前的(protect)。"""
secrets = masking._Secrets()
secrets.add({"columns": ["v"], "rows": [[v] for v in values]}, {"v"})
secrets.protect(protect)
return secrets
AMOUNTS = secrets_of(20000, 30000, 50000)
@pytest.mark.parametrize("sql", [
"SELECT id, visits FROM sales WHERE amount >= 20000 ORDER BY id",
"SELECT id FROM sales WHERE amount BETWEEN 20000 AND 50000 LIMIT 30000",
"SELECT region, COUNT(*) AS n FROM sales WHERE visits > 30000 GROUP BY region",
])
def test_thresholds_and_limits_stay_as_written(sql):
"""范围比较的边界、LIMIT 的行数是报告自己定的条件,恰好等于某一笔被遮的金额也照原样:agent 要看得到报告按什么取数。"""
assert _scrub_excerpt_sql(sql, AMOUNTS, {"amount"}, "sqlite") == sql
def test_a_value_the_report_looked_for_is_masked():
"""在找某一笔的(等值比较)照旧换:这是在点名一个遮罩值。"""
assert _scrub_excerpt_sql("SELECT id FROM sales WHERE amount = 50000", AMOUNTS, {"amount"}, "sqlite") \
== f"SELECT id FROM sales WHERE amount = {MASKED}"
@pytest.mark.parametrize("sql, expected", [
("SELECT region FROM sales WHERE buyer_phone NOT IN ('13900000002', '139-0000-0003')",
f"SELECT region FROM sales WHERE buyer_phone NOT IN ('{MASKED}', '{MASKED}')"),
("SELECT region FROM sales WHERE buyer_phone = '13900000009' OR buyer_phone LIKE '%0001'",
f"SELECT region FROM sales WHERE buyer_phone = '{MASKED}' OR buyer_phone LIKE '{MASKED}'"),
("SELECT region FROM (SELECT region, buyer_phone AS 手机号 FROM sales) t WHERE 手机号 IN ('13900000002')",
f"SELECT region FROM (SELECT region, buyer_phone AS 手机号 FROM sales) t WHERE 手机号 IN ('{MASKED}')"),
("SELECT region FROM sales WHERE region = '华东'", "SELECT region FROM sales WHERE region = '华东'"),
])
def test_literals_compared_with_a_masked_column_are_masked_even_when_never_collected(sql, expected):
"""和遮罩列(含改了名字的)做等值、IN、LIKE 比较的字面量:报告快照里收不到这些值(报告没选出号码),照样换掉;和没设
遮罩的列比较的照原样。"""
assert _scrub_excerpt_sql(sql, masking._Secrets(), {"buyer_phone", "手机号"}, "sqlite") == expected
def test_a_threshold_equal_to_a_masked_literal_goes_with_it():
"""同一个数既在找某一笔、又写成阈值:阈值留着就把换掉的值露了,跟着换(同封存 SQL 的 _literals)。"""
sql = "SELECT id FROM sales WHERE buyer_phone = 13900000009 OR id >= 13900000009"
assert _scrub_excerpt_sql(sql, masking._Secrets(), {"buyer_phone"}, "sqlite") \
== f"SELECT id FROM sales WHERE buyer_phone = {MASKED} OR id >= {MASKED}"
def test_values_already_in_front_of_the_reader_are_not_masked():
"""提问人、被问的那句里写着的值(protect)照旧不换,换了写法写的也认得出。"""
secrets = secrets_of(protect=("139-0000-0009 这位客户的单",))
sql = "SELECT id FROM sales WHERE buyer_phone = '13900000009' OR buyer_phone = '13900000008'"
assert _scrub_excerpt_sql(sql, secrets, {"buyer_phone"}, "sqlite") \
== f"SELECT id FROM sales WHERE buyer_phone = '13900000009' OR buyer_phone = '{MASKED}'"
def test_strings_and_comments_are_masked_by_value():
"""字符串、注释里写着的遮罩值按值换(换了写法的号码也认),同封存 SQL。"""
secrets = secrets_of("13900000001")
sql = "SELECT id FROM sales /* 客户 139-0000-0001 */ WHERE note LIKE '%13900000001%'"
assert _scrub_excerpt_sql(sql, secrets, set(), "sqlite") \
== f"SELECT id FROM sales /* 客户 {MASKED} */ WHERE note LIKE '%{MASKED}%'"
@pytest.mark.parametrize("value, numeric, expected", [
(20000, False, 20000),
(20000.0, False, 20000.0),
("20000.00", True, "20000.00"),
("20000", False, MASKED),
("上月消费 20000 元", False, f"上月消费 {MASKED} 元"),
("回访:联系 139-0000-0003", False, f"回访:联系 {MASKED}"),
({"phone": "13900000003"}, False, '{"phone": "' + MASKED + '"}'),
("例行回访", False, "例行回访"),
(None, False, None),
])
def test_cells_are_masked_like_the_tool_boundary(value, numeric, expected):
"""格子同工具边界:数(整数、小数,列类型是 number 的字符串)照原样;文字格整格是遮罩值的、夹着的那一截换掉,JSON
按画出来的样子换。"""
secrets = secrets_of(20000, "13900000003")
assert masking.masked_cell(value, secrets, numeric=numeric) == expected
def test_an_excerpt_mask_without_values_only_masks_by_column():
"""没有可按值换的(报告快照里没收到值):格子、文字照原样,SQL 里和遮罩列做等值比较的照样换。"""
mask = ExcerptMask({"shop": "sqlite"})
assert mask.cell("13900000001", numeric=False) == "13900000001"
assert mask.text("客户 13900000001") == "客户 13900000001"
assert mask.sql("SELECT 1 FROM sales WHERE buyer_phone = '13900000001'", {"source": "shop"}, {"buyer_phone"}) \
== f"SELECT 1 FROM sales WHERE buyer_phone = '{MASKED}'"
+512
View File
@@ -0,0 +1,512 @@
"""认不全的 SQL 写法整份遮(engine/masking.py 的 _tainted),不再逐个形状补规则(K3 第 3 轮复验留下的几处):
- (表达式).* 展开:复合类型的列 (p.profile).*、函数交回的记录 (jsonb_each_text(pj)).*、(jsonb_populate_record(…)).*、
子查询里的匿名 ROW (q.r).*。展开成几列、各叫什么 SQL 里看不出,表达式读到遮罩列的,整份结果都遮——最外层、子查询、
CTE、LATERAL 里都一样。以前改写成一个对不上来源的 *,最外层、没写列名表的派生来源里按「照原名带出」认,字段名
(tel、key/value、f1/f2)和遮罩列对不上,号码原样交给模型、落库。(s).*、s 是 FROM 里的来源时照旧按 s.* 认
- 表名、SQL 写在字符串里交给函数执行、整表导出的(query_to_xml、table_to_xml、cursor_to_xml、schema_to_xml、
database_to_xml、dblink,Oracle 的 DBMS_XMLGEN),统计信息里存着的值(pg_stats):整份遮,解析不了的也一样
- LATERAL 看不见它自己:里面的整行引用 s 不因为 LATERAL 自己的结果列叫 s 就被当成列
- 分析不过来、分析出了意外:整份遮。派生来源对应的那块查询、一项的值取自哪些列在一次分析里记下来,一长串 CTE 一段套
一段地整行引用(250 段,2 万字的上限里写得下)以前要二十几秒,现在零点几秒;超出工作量上限(ANALYSIS_STEPS)的整份遮
这里没有真的 Postgres:结果按它交回的样子造(复合类型、记录展开成几列)。数据都是虚构的:手机号、地址、区域。
"""
from __future__ import annotations
import json
import time
import uuid
import pytest
from httpx import ASGITransport, AsyncClient
from app.core import artifact_store
from app.data.engine import engines
from app.db.base import SessionLocal
from app.db.models import DataSource
from app.engine import masking
from app.main import app
from tests.test_verify_agent import Script, q
from tests.test_verify_agent_masking import ask, clean_spend # noqa: F401 - clean_spend 是自动用上的夹具
MASKED_WORD = "已遮罩"
#: people.profile 是复合类型(tel、addr),people2.pj 是 JSON(tel、addr),sales 是订单:设了遮罩的是 profile、pj、buyer_phone
SCHEMA = {"tables": {
"people": {"qualified": "people", "schema": None, "columns": [{"name": c} for c in ["id", "profile", "address"]]},
"people2": {"qualified": "people2", "schema": None, "columns": [{"name": c} for c in ["id", "pj", "city"]]},
"sales": {"qualified": "sales", "schema": None,
"columns": [{"name": c} for c in ["id", "region", "buyer_phone", "amount"]]},
}}
MASKED = ["profile", "pj", "buyer_phone"]
PHONE = "13900000001"
def _leaks(done: masking.QueryMask) -> bool:
return "1390000000" in json.dumps(done.payload, ensure_ascii=False)
def masked(sql: str, columns: list[str], rows: list[list], kind: str = "postgresql") -> masking.QueryMask:
payload = {"columns": columns, "rows": rows, "sql": sql}
return masking.mask_query_result(payload, MASKED, kind, schema=SCHEMA)
# --------------------------------------------------------------------------
# (表达式).* 展开
# --------------------------------------------------------------------------
#: 三种展开(复合类型的列、函数交回的记录、子查询里的匿名 ROW)× 四个位置(最外层、子查询、CTE、LATERAL):
#: (SQL, 结果列, 一行)
EXPANSIONS = [
pytest.param("SELECT p.id AS n, (p.profile).* FROM people p", ["n", "tel", "addr"], [1, PHONE, "某路 1 号"],
id="复合类型-最外层"),
pytest.param("SELECT x.* FROM (SELECT p.id AS n, (p.profile).* FROM people p) x", ["n", "tel", "addr"],
[1, PHONE, "某路 1 号"], id="复合类型-子查询"),
pytest.param("WITH x AS (SELECT p.id AS n, (p.profile).* FROM people p) SELECT * FROM x", ["n", "tel", "addr"],
[1, PHONE, "某路 1 号"], id="复合类型-CTE"),
pytest.param("SELECT x.* FROM people p, LATERAL (SELECT p.id AS n, (p.profile).*) x", ["n", "tel", "addr"],
[1, PHONE, "某路 1 号"], id="复合类型-LATERAL"),
pytest.param("SELECT (jsonb_each_text(pj)).* FROM people2", ["key", "value"], ["tel", PHONE],
id="记录函数-最外层"),
pytest.param("SELECT kv.value AS tel FROM (SELECT (jsonb_each_text(pj)).* FROM people2) kv WHERE kv.key = 'tel'",
["tel"], [PHONE], id="记录函数-子查询"),
pytest.param("WITH kv AS (SELECT (jsonb_each_text(pj)).* FROM people2) SELECT value AS 号码 FROM kv "
"WHERE key = 'tel'", ["号码"], [PHONE], id="记录函数-CTE"),
pytest.param("SELECT x.value FROM people2 p, LATERAL (SELECT (jsonb_each_text(p.pj)).*) x", ["value"], [PHONE],
id="记录函数-LATERAL"),
pytest.param("SELECT (q.r).* FROM (SELECT ROW(buyer_phone, region) AS r FROM sales) q", ["f1", "f2"],
[PHONE, "华东"], id="匿名 ROW-最外层"),
pytest.param("SELECT y.* FROM (SELECT (q.r).* FROM (SELECT ROW(buyer_phone, region) AS r FROM sales) q) y",
["f1", "f2"], [PHONE, "华东"], id="匿名 ROW-子查询"),
pytest.param("WITH q AS (SELECT ROW(buyer_phone, region) AS r FROM sales) SELECT (q.r).* FROM q", ["f1", "f2"],
[PHONE, "华东"], id="匿名 ROW-CTE"),
pytest.param("SELECT x.* FROM (SELECT ROW(buyer_phone, region) AS r FROM sales) q, LATERAL (SELECT (q.r).*) x",
["f1", "f2"], [PHONE, "华东"], id="匿名 ROW-LATERAL"),
pytest.param("SELECT p.id AS n, (jsonb_populate_record(NULL::contact, p.pj)).* FROM people2 p",
["n", "tel", "addr"], [1, PHONE, "某路 1 号"], id="按类型展开 JSON"),
]
@pytest.mark.parametrize("sql, columns, row", EXPANSIONS)
def test_an_expression_star_reading_a_masked_column_masks_the_whole_result(sql, columns, row):
"""(表达式).* 读到遮罩列:展开成几列、叫什么说不清,整份结果的每一列都遮(同一份里的编号也遮,多遮)。"""
done = masked(sql, columns, [list(row)])
assert done.columns == columns
assert done.payload["rows"] == [[MASKED_WORD] * len(columns)] and not _leaks(done)
@pytest.mark.parametrize("sql, columns, row", [
pytest.param("SELECT p.id AS n, (p.address).* FROM people p", ["n", "street", "city"], [1, "某路", "某市"],
id="没设遮罩的复合类型"),
pytest.param("SELECT (jsonb_each_text(to_jsonb(c))).* FROM (SELECT id, region FROM sales) c", ["key", "value"],
["region", "华东"], id="记录里没有遮罩列"),
])
def test_an_expression_star_that_reads_no_masked_column_stays_visible(sql, columns, row):
done = masked(sql, columns, [list(row)])
assert done.columns == [] and done.payload["rows"] == [list(row)]
def test_a_parenthesised_source_star_is_still_read_as_its_columns():
"""(s).*、s 是 FROM 里的来源:照旧当 s.* 认,只遮号码那一列,区域、金额照常。"""
done = masked("SELECT (s).* FROM sales s", ["id", "region", "buyer_phone", "amount"], [[1, "华东", PHONE, 1200]])
assert done.columns == ["buyer_phone"] and done.payload["rows"] == [[1, "华东", MASKED_WORD, 1200]]
#: 造出来的 Postgres 数据源:表结构同上,查询交给换掉的 run_query(不连库)
@pytest.fixture
async def composite_shop():
made: list[int] = []
async def make() -> str:
name = f"pg_{uuid.uuid4().hex[:8]}"
async with SessionLocal() as session:
row = DataSource(name=name, kind="postgresql", host="127.0.0.1", port=5432, username="probe",
database="shop", readonly=True, enabled=True,
options={"mask_columns": ", ".join(MASKED)}, schema_cache=SCHEMA)
session.add(row)
await session.commit()
made.append(row.id)
return name
yield make
for source_id in made:
await engines.invalidate(source_id)
@pytest.mark.parametrize("sql, columns, rows", [
("SELECT p.id AS n, (p.profile).* FROM people p", ["n", "tel", "addr"], [[1, PHONE, "某路 1 号"]]),
("SELECT x.* FROM (SELECT p.id AS n, (p.profile).* FROM people p) x", ["n", "tel", "addr"],
[[1, PHONE, "某路 1 号"]]),
("SELECT p.id AS n, (jsonb_populate_record(NULL::contact, p.pj)).* FROM people2 p", ["n", "tel", "addr"],
[[1, PHONE, "某路 1 号"]]),
("SELECT p.id AS n, ((p.profile).*) FROM people p", ["n", "tel", "addr"], [[1, PHONE, "某路 1 号"]]),
], ids=["复合类型", "子查询里的复合类型", "按类型展开 JSON", "外面套了括号"])
async def test_an_expression_star_never_reaches_the_model_or_the_store(monkeypatch, composite_shop, sql, columns,
rows):
"""端到端(核查 agent 的真实查询工具):交给模型的、落库的、通用工件接口给的都只有「已遮罩」,masked_at_query 是
全部的列,快照记着整份遮(masked_whole),格子引用照实说写法认不全。以前前三条的 masked_at_query 都是空的,套了外括号
的只遮了第一列,号码原样交给模型、落库。"""
from app.data.engine import QueryResult
from app.tools import datasource
async def driver(source, statement, **kwargs):
assert statement == sql
return QueryResult(columns=list(columns), rows=[list(r) for r in rows], row_count=len(rows), truncated=False,
elapsed_ms=1, sql=statement)
monkeypatch.setattr(datasource, "run_query", driver)
name = await composite_shop()
script = Script(monkeypatch, [q(name, sql, "c1")], {"value": {"alias": "Q1", "row": 0, "column": "n"},
"rationale": "x"})
out, _ = await ask(name)
seen = script.tool_results()["c1"]
assert MASKED_WORD in seen and "1390000000" not in seen, seen
snapshot = artifact_store.load(out.queries[0]["artifact"])
assert snapshot["masked_at_query"] == columns and "1390000000" not in json.dumps(snapshot, ensure_ascii=False)
assert snapshot["masked_whole"] is True and "masked_whole" not in seen
assert out.cells[0]["reason"] == "Q1 的写法认不全,整份结果查询时已遮罩,不采信"
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
got = await client.get(f"/api/artifacts/{out.queries[0]['artifact']}")
assert got.status_code == 200 and "1390000000" not in got.text
# --------------------------------------------------------------------------
# LATERAL 看不见它自己
# --------------------------------------------------------------------------
@pytest.mark.parametrize("sql, columns, row", [
pytest.param("SELECT y.* FROM sales t2, LATERAL (SELECT s FROM sales s LIMIT 1) y", ["s"],
[f"(1,华东,{PHONE},1200)"], id="选出整行"),
pytest.param("SELECT y.j FROM sales t2 CROSS JOIN LATERAL (SELECT 1 AS s, row_to_json(s) AS j FROM sales s "
"LIMIT 1) y", ["j"], [json.dumps({"id": 1, "buyer_phone": PHONE})], id="结果列也叫 s"),
pytest.param("SELECT y.j FROM sales t2, LATERAL (SELECT s.id AS s, row_to_json(s) AS j FROM sales s LIMIT 1) y",
["j"], [json.dumps({"id": 1, "buyer_phone": PHONE})], id="编号起名叫 s"),
])
def test_a_whole_row_inside_a_lateral_is_not_taken_for_its_own_column(sql, columns, row):
"""LATERAL 里的 s 是它自己 FROM 里的整行:LATERAL 的结果列碰巧叫 s,也不把它当成列(Postgres 里 LATERAL 看不见
自己)。以前当成了列,整行原样交了出去。"""
done = masked(sql, columns, [list(row)])
assert done.columns == columns and not _leaks(done)
def test_a_lateral_still_sees_the_sources_before_it():
"""排在 LATERAL 前面的来源照样看得见:前面的表有一列叫 s 时,s 是那一列(没设遮罩),不是整行。"""
schema = {"tables": {**SCHEMA["tables"], "codes": {"qualified": "codes", "schema": None,
"columns": [{"name": "id"}, {"name": "s"}]}}}
payload = {"columns": ["v"], "rows": [["A1"]],
"sql": "SELECT y.v FROM codes c, LATERAL (SELECT s AS v FROM sales x WHERE x.id = c.id) y"}
done = masking.mask_query_result(payload, MASKED, "postgresql", schema=schema)
assert done.columns == [] and done.payload["rows"] == [["A1"]]
# --------------------------------------------------------------------------
# 表名、SQL 写在字符串里交给函数,统计信息里的值
# --------------------------------------------------------------------------
@pytest.mark.parametrize("sql", [
"SELECT query_to_xml('select * from sales', true, false, '') AS x",
"SELECT table_to_xml('sales', true, false, '') AS x",
"SELECT query_to_xml_and_xmlschema('select buyer_phone from sales', true, false, '') AS x",
"SELECT cursor_to_xml('c1', 10, false, true, '') AS x",
"SELECT schema_to_xml('public', true, false, '') AS x",
"SELECT database_to_xml(true, false, '') AS x",
"SELECT x FROM dblink('dbname=shop', 'select buyer_phone from sales') AS t(x text)",
"SELECT DBMS_XMLGEN.GETXML('select * from sales') AS x FROM dual",
"SELECT array_to_string(most_common_vals, ',') AS x FROM pg_stats WHERE tablename = 'sales'",
])
def test_tables_and_sql_handed_to_functions_as_text_mask_the_whole_result(sql):
"""表名、SQL 写在字符串里交给函数执行、整表导出,统计信息里存着的值:读的是哪一列认不出,整份遮。以前 masked=[],
号码原样。"""
done = masked(sql, ["x"], [[f"<row><buyer_phone>{PHONE}</buyer_phone></row>"]], kind="oracle"
if "DBMS" in sql else "postgresql")
assert done.columns == ["x"] and not _leaks(done)
def test_a_table_handed_to_a_function_masks_the_whole_result_even_when_the_sql_cannot_be_read():
sql = "SELECT table_to_xml('sales', true, false, '') AS x WHERE ("
hidden, _ = masking._derived_hidden({"sql": sql, "columns": ["x"], "rows": []}, {"buyer_phone"}, "postgresql")
assert "x" in hidden
# --------------------------------------------------------------------------
# 分析不过来、分析出了意外:整份遮;一长串 CTE 不再卡住
# --------------------------------------------------------------------------
def _chain(k: int, tail: str) -> str:
"""一长串 CTE,一段套一段地整行引用、展开上一段(复验时 250 段要二十几秒)。"""
parts = ["c0 AS (SELECT * FROM sales)"]
parts += [f"c{i} AS (SELECT row_to_json(c{i - 1}) AS j, c{i - 1}.* FROM c{i - 1})" for i in range(1, k)]
return "WITH " + ", ".join(parts) + f" {tail.format(last=f'c{k - 1}')}"
def _plain_chain(k: int) -> str:
parts = ["c0 AS (SELECT id, region, amount FROM sales)"]
parts += [f"c{i} AS (SELECT id, region, amount + {i} AS amount FROM c{i - 1})" for i in range(1, k)]
return "WITH " + ", ".join(parts) + f" SELECT region, SUM(amount) AS total FROM c{k - 1} GROUP BY region"
def test_a_long_chain_of_whole_rows_is_read_quickly_and_completely():
"""250 段一段套一段地整行引用:零点几秒认完(以前二十几秒),不靠工作量上限兜底——只数个数的结果照常不遮。"""
columns = masking._schema_columns(SCHEMA)
sql = _chain(250, "SELECT COUNT(*) AS n FROM {last}")
assert len(sql) < 20_000
started = time.perf_counter()
hidden, _ = masking._derived_hidden({"sql": sql, "columns": ["n"], "rows": []}, {"buyer_phone"}, "postgresql",
columns)
assert time.perf_counter() - started < 5
assert "n" not in hidden
hidden, _ = masking._derived_hidden({"sql": _chain(250, "SELECT j FROM {last}"), "columns": ["j"], "rows": []},
{"buyer_phone"}, "postgresql", columns)
assert "j" in hidden
def test_an_analysis_over_the_work_limit_masks_the_whole_result(monkeypatch):
"""超出工作量上限:不再分析,整份遮(结果本来没碰遮罩列也一样)。没超的照常。"""
sql = _plain_chain(40)
snapshot = {"sql": sql, "columns": ["region", "total"], "rows": []}
assert masking._derived_hidden(snapshot, {"buyer_phone"}, "postgresql")[0] == {"buyer_phone"}
monkeypatch.setattr(masking, "ANALYSIS_STEPS", 50)
assert masking._derived_hidden(snapshot, {"buyer_phone"}, "postgresql")[0] == {"buyer_phone", "region", "total"}
payload = {"sql": sql, "columns": ["region", "total"], "rows": [["华东", 1200]]}
done = masking.mask_query_result(payload, ["buyer_phone"], "postgresql")
assert done.columns == ["region", "total"] and done.payload["rows"] == [[MASKED_WORD, MASKED_WORD]]
def test_a_failing_analysis_masks_the_whole_result_instead_of_reading_the_text(monkeypatch):
"""解析之后的分析出了意外:整份遮。以前按「解析不了」只看文字,没写遮罩列、没有 * 的整行引用(SELECT s)就放过去了。"""
def broken(*args, **kwargs):
raise RuntimeError("分析里的意外")
monkeypatch.setattr(masking, "_tainted_in", broken)
snapshot = {"sql": "SELECT s AS r FROM sales s", "columns": ["r"], "rows": []}
assert "r" in masking._derived_hidden(snapshot, {"buyer_phone"}, "postgresql")[0]
# --------------------------------------------------------------------------
# (表达式).* 套了外括号、起了别名
# --------------------------------------------------------------------------
#: Postgres 实测:((p.card).*)、(((p.card)).*)、(((p.card).*))、(p.card).* AS x 都照样展开成好几列(别名不起作用)
WRAPPED = [
pytest.param("SELECT ((p.profile).*) FROM people p", ["tel", "addr"], [PHONE, "某路 1 号"], id="复合类型-外括号"),
pytest.param("SELECT (((p.profile)).*) FROM people p", ["tel", "addr"], [PHONE, "某路 1 号"],
id="复合类型-两层外括号"),
pytest.param("SELECT (((p.profile).*)) FROM people p", ["tel", "addr"], [PHONE, "某路 1 号"],
id="复合类型-套两层"),
pytest.param("SELECT (p.profile).* AS x FROM people p", ["tel", "addr"], [PHONE, "某路 1 号"], id="复合类型-别名"),
pytest.param("SELECT ((jsonb_each_text(pj)).*) FROM people2", ["key", "value"], ["tel", PHONE],
id="记录函数-外括号"),
pytest.param("SELECT ((q.r).*) FROM (SELECT ROW(region, buyer_phone) AS r FROM sales) q", ["f1", "f2"],
["华东", PHONE], id="匿名 ROW-外括号"),
pytest.param("SELECT ((jsonb_each_text(pj)).*) FROM people2 UNION ALL SELECT 'a', 'b'", ["key", "value"],
["tel", PHONE], id="UNION 第一支"),
pytest.param("SELECT 'a' AS k, 'b' AS v UNION ALL SELECT ((jsonb_each_text(pj)).*) FROM people2", ["k", "v"],
["tel", PHONE], id="UNION 后一支"),
pytest.param("SELECT 'a' AS k, 'b' AS v UNION ALL SELECT ((p.profile).*) FROM people p", ["k", "v"],
[PHONE, "某路 1 号"], id="UNION 后一支-复合类型"),
pytest.param("SELECT x.* FROM (SELECT ((q.r).*) FROM (SELECT ROW(buyer_phone, region) AS r FROM sales) q) x",
["f1", "f2"], [PHONE, "华东"], id="子查询里-外括号"),
# 和 * 写在同一支里:选出项和结果的列数本来就对不上(* 展开成几列),只能靠认出外括号里的 (表达式).*
pytest.param("SELECT p.*, ((p.profile).*) FROM people p", ["id", "profile", "address", "tel", "addr"],
[1, f"({PHONE},某路 1 号)", "某路 1 号", PHONE, "某路 1 号"], id="和 * 写在一起-外括号"),
pytest.param("SELECT p.*, (p.profile).* AS x FROM people p", ["id", "profile", "address", "tel", "addr"],
[1, f"({PHONE},某路 1 号)", "某路 1 号", PHONE, "某路 1 号"], id="和 * 写在一起-别名"),
]
@pytest.mark.parametrize("sql, columns, row", WRAPPED)
def test_a_wrapped_expression_star_reading_a_masked_column_masks_the_whole_result(sql, columns, row):
"""(表达式).* 外面套了括号、起了别名,Postgres 照样展开成好几列:整份遮。以前只认不带外括号的写法,((p.card).*) 按一项
数,只遮了第一列,号码原样交给模型、落库。"""
done = masked(sql, columns, [list(row)])
assert done.whole and done.columns == columns
assert done.payload["rows"] == [[MASKED_WORD] * len(columns)] and not _leaks(done)
def test_a_wrapped_expression_star_with_a_numeric_phone_is_masked_whole():
"""号码存成整数(bigint)、前面的查询已经认出它:数值格不按值换,(表达式).* 认不出就全靠整份遮挡住。"""
known = masking._Secrets()
known.add({"columns": ["p"], "rows": [[13900000001]]}, {"p"})
payload = {"columns": ["f1", "f2"], "rows": [[1200, 13900000001]],
"sql": "SELECT ((q.r).*) FROM (SELECT ROW(amount, buyer_phone) AS r FROM sales) q"}
done = masking.mask_query_result(payload, MASKED, "postgresql", known, schema=SCHEMA)
assert done.payload["rows"] == [[MASKED_WORD, MASKED_WORD]]
@pytest.mark.parametrize("sql", ["SELECT ((s).*) FROM sales s", "SELECT (s.*) AS x FROM sales s"])
def test_a_wrapped_source_star_is_still_read_as_its_columns(sql):
"""((s).*)、(s.*) AS x、s 是 FROM 里的来源:照旧当 s.* 认,只遮号码那一列。"""
done = masked(sql, ["id", "region", "buyer_phone", "amount"], [[1, "华东", PHONE, 1200]])
assert not done.whole and done.columns == ["buyer_phone"]
assert done.payload["rows"] == [[1, "华东", MASKED_WORD, 1200]]
# --------------------------------------------------------------------------
# 和数据源里的表同名的 CTE
# --------------------------------------------------------------------------
@pytest.mark.parametrize("sql, columns, row", [
pytest.param("WITH sales AS (SELECT * FROM sales WHERE region = '华东') SELECT row_to_json(sales) AS j FROM sales",
["j"], [json.dumps({"id": 1, "buyer_phone": PHONE})], id="整行引用"),
pytest.param("WITH sales AS (SELECT * FROM sales WHERE region = '华东') SELECT s AS r FROM sales s", ["r"],
[f"(1,华东,{PHONE},1200)"], id="选出整行"),
pytest.param("WITH orders AS (SELECT * FROM sales), sales AS (SELECT * FROM sales WHERE amount > 0) "
"SELECT to_jsonb(x) AS j FROM sales x", ["j"], [json.dumps({"buyer_phone": PHONE})],
id="排在后面的同名段"),
pytest.param("WITH sales AS (SELECT * FROM sales) SELECT * FROM sales t(a, b, c, d)", ["a", "b", "c", "d"],
[1, "华东", PHONE, 1200], id="列名表"),
])
def test_a_cte_named_like_a_table_reads_the_table(sql, columns, row):
"""WITH sales AS (SELECT * FROM sales WHERE …) 里的 sales 是数据源里的表,不是这一段自己(不带 RECURSIVE 的 WITH 里,
一段只看得见排在它前面的段)。以前认成它自己,* 展开绕回自己什么列都不带出,号码原样交了出去。"""
done = masked(sql, columns, [list(row)])
assert done.columns and not _leaks(done)
assert done.payload["rows"] == [[MASKED_WORD if c in done.columns else v for c, v in zip(columns, row)]]
def test_a_cte_named_like_a_table_masks_like_any_other_name():
"""对照:同一段 CTE 改叫 t,同样遮;列名表按表结构对上位置,只遮号码那一列。"""
for sql in ("WITH t AS (SELECT * FROM sales WHERE region = '华东') SELECT row_to_json(t) AS j FROM t",
"WITH sales AS (SELECT * FROM sales WHERE region = '华东') SELECT row_to_json(sales) AS j FROM sales"):
assert masked(sql, ["j"], [[json.dumps({"buyer_phone": PHONE})]]).columns == ["j"]
done = masked("WITH sales AS (SELECT * FROM sales) SELECT * FROM sales t(a, b, c, d)", ["a", "b", "c", "d"],
[[1, "华东", PHONE, 1200]])
assert done.columns == ["c"]
def test_a_recursive_cte_still_sees_itself():
"""带 RECURSIVE 的 WITH 里递归的那一支指向它自己:取自号码的那一列照遮,只数步数的照常。"""
body = ("WITH RECURSIVE t(n, p) AS (SELECT 1, buyer_phone FROM sales UNION ALL SELECT n + 1, p FROM t "
"WHERE n < 3) ")
assert masked(body + "SELECT n FROM t", ["n"], [[1]]).columns == []
assert masked(body + "SELECT p FROM t", ["p"], [[PHONE]]).columns == ["p"]
# --------------------------------------------------------------------------
# 语法树和实际执行的对不上、存着值的视图:和可验证报告共用一份名单
# --------------------------------------------------------------------------
@pytest.mark.parametrize("sql, kind, columns", [
pytest.param("SELECT id /*!, buyer_phone AS x */ FROM sales", "mysql", ["id", "x"], id="MySQL 会执行的注释"),
pytest.param("SELECT /*! buyer_phone AS */ region FROM sales", "mysql", ["region"],
id="MySQL 会执行的注释改了列的来历"),
pytest.param('SELECT U&"\\0062uyer_phone" AS x FROM sales', "postgresql", ["x"], id="U&转义的名字"),
pytest.param("SELECT id, U&\"buyer!005fphone\" UESCAPE '!' AS p FROM sales", "postgresql", ["id", "p"],
id="U&带 UESCAPE"),
pytest.param("SELECT query FROM pg_stat_get_activity(NULL)", "postgresql", ["query"], id="pg_stat_get_activity"),
pytest.param("SELECT data FROM pg_logical_slot_peek_changes('s', NULL, NULL)", "postgresql", ["data"],
id="逻辑解码"),
pytest.param("SELECT HISTOGRAM AS h FROM information_schema.COLUMN_STATISTICS WHERE TABLE_NAME = 'sales'", "mysql",
["h"], id="MySQL 直方图"),
pytest.param("SELECT sql_text AS v FROM performance_schema.events_statements_history_long", "mysql", ["v"],
id="MySQL 语句历史"),
pytest.param("SELECT info AS v FROM information_schema.PROCESSLIST", "mysql", ["v"], id="MySQL 会话列表"),
pytest.param("SELECT argument AS v FROM mysql.general_log", "mysql", ["v"], id="MySQL 通用日志"),
pytest.param("SELECT min_value AS v FROM mysql.column_stats", "mariadb", ["v"], id="MariaDB 列统计"),
pytest.param("SELECT sql_fulltext AS v FROM v$sql", "oracle", ["v"], id="Oracle V$SQL"),
pytest.param("SELECT value_string AS v FROM v$sql_bind_capture", "oracle", ["v"], id="Oracle 绑定变量"),
pytest.param("SELECT endpoint_actual_value AS v FROM all_tab_histograms WHERE table_name = 'SALES'", "oracle",
["v"], id="Oracle 直方图"),
pytest.param("SELECT * FROM all_tab_columns WHERE table_name = 'SALES'", "oracle", ["column_name", "low_value"],
id="Oracle 列的最小最大值"),
pytest.param("SELECT query_to_xml/**/('select * from sales', true, false, '') AS x", "postgresql", ["x"],
id="函数名后夹块注释"),
pytest.param("SELECT query_to_xml --c\n('select * from sales', true, false, '') AS x", "postgresql", ["x"],
id="函数名后夹行注释"),
pytest.param("SELECT id FROM sales", "mysql", ["id", "x"], id="选出项和结果列数对不上"),
])
def test_writings_the_analysis_cannot_follow_mask_the_whole_result(sql, kind, columns):
"""语法树和实际执行的对不上(MySQL 会执行的注释、U&"…"、选出项和结果的列数对不上),统计信息、会话记录里存着值的视图和
函数,函数名和括号之间夹着注释:整份遮。以前工具边界的名单比可验证报告的少一大截,这些写法号码原样交给模型、落库。"""
row = [PHONE] * len(columns)
done = masked(sql, columns, [row], kind=kind)
assert done.whole and done.payload["rows"] == [[MASKED_WORD] * len(columns)]
@pytest.mark.parametrize("sql, columns, row", [
pytest.param("WITH query(r, n) AS (SELECT region, COUNT(*) FROM sales GROUP BY region) SELECT r AS region, n "
"FROM query", ["region", "n"], ["华东", 2], id="叫 query 的 CTE"),
pytest.param("SELECT region, 'Query (x)' AS q FROM sales", ["region", "q"], ["华东", "Query (x)"],
id="字符串里的 Query ("),
pytest.param("SELECT column_name FROM information_schema.columns WHERE table_name = 'sales'", ["column_name"],
["buyer_phone"], id="information_schema 的列清单"),
])
def test_names_of_unsupported_databases_and_plain_schema_views_are_not_masked(sql, columns, row):
"""不支持的库(DuckDB 的 query()、SQL Server 的 OPENQUERY)的函数不再收:名叫 query 的 CTE、字符串里的「Query (x)」不再
整份遮;information_schema 里只有表结构的视图照常(核查 agent 查列名、类型常用)。"""
done = masked(sql, columns, [list(row)])
assert not done.whole and done.payload["rows"] == [list(row)]
def test_the_tool_boundary_and_the_bundle_share_one_list():
"""工具边界和可验证报告共用一份名单:报告那边另把系统 schema 整个挡(只决定附不附快照)。"""
from app.bundle import build
assert build._OPAQUE_SQL is masking.OPAQUE_SQL
for sql in ("SELECT HISTOGRAM FROM information_schema.COLUMN_STATISTICS", "SELECT * FROM v$sql",
"SELECT data FROM pg_logical_slot_peek_changes('s', NULL, NULL)"):
assert build._INDIRECT_READS.search(sql) and masking._INDIRECT.search(sql)
plain = "SELECT column_name FROM information_schema.columns"
assert build._INDIRECT_READS.search(plain) and not masking._INDIRECT.search(plain)
@pytest.mark.parametrize("sql", [
"SELECT row_to_json(s) AS j FROM sales s ORDER BY s.id USING >",
"SELECT s AS j FROM sales s WHERE s.region IS NOT NFC NORMALIZED OR true",
"SELECT to_jsonb(s) - 'id' AS j FROM sales s ORDER BY s.id USING <",
])
def test_postgres_sql_that_cannot_be_read_masks_the_whole_result(sql):
"""Postgres 合法、解析器解析不了的写法:整行引用不写列名、也没有 *,只看文字认不出,整份遮。以前 masked=[],整行连同
号码交给模型、落库。"""
done = masked(sql, ["j"], [[json.dumps({"id": 1, "buyer_phone": PHONE})]])
assert done.whole and not _leaks(done)
# --------------------------------------------------------------------------
# 整份遮的结果不把每一格都收成遮罩值
# --------------------------------------------------------------------------
def test_a_whole_masked_result_does_not_turn_every_cell_into_a_masked_value():
"""统计视图整份遮之后,里面的列名、区域名不收成遮罩值:后一条查询的区域照常,封存的 SQL 照原样。以前每一格都收,
区域名、列名在同一次核查里处处被换掉,封存的 SQL 成了「WHERE 已遮罩 = '华东'」。"""
from app.engine.inquiry import _scrub_sql
stats = masked("SELECT attname, n_distinct FROM pg_stats WHERE tablename = 'sales'", ["attname", "n_distinct"],
[["amount", 5], ["buyer_phone", -1], ["region", 3]])
assert stats.whole and not stats.secrets.strong
payload = {"columns": ["id", "region", "amount"], "rows": [[1, "华东", 1200]],
"sql": "SELECT id, region, amount FROM sales ORDER BY id"}
later = masking.mask_query_result(payload, MASKED, "postgresql", stats.secrets, schema=SCHEMA)
assert later.payload["rows"] == [[1, "华东", 1200]]
sql = "SELECT region, COUNT(*) AS n FROM sales WHERE region = '华东' GROUP BY region"
assert _scrub_sql(sql, stats.secrets, {"buyer_phone"}, "postgresql") == sql
def test_a_whole_masked_result_still_collects_what_surely_came_from_a_masked_column():
"""整份遮时,认得出取自遮罩列的那几列照收(号码改了名字选出来、又夹着认不全的写法),别的列不收。"""
sql = "SELECT buyer_phone AS 手机号, region FROM sales UNION ALL SELECT query_to_xml('x', true, false, ''), 'y'"
done = masked(sql, ["手机号", "region"], [[PHONE, "华东"]])
assert done.whole and done.secrets.hit(PHONE) and not done.secrets.hit("华东")
def test_report_snapshots_that_were_masked_whole_give_only_their_sure_values():
"""报告自己的查询快照(报告取数时没遮)收遮罩值时同一条规则:写法认不全的只收确定取自遮罩列的那几列,区域名、列名不收
(审批质询、反方审稿把它们交给查询工具、拿来换回答和理由,收了就处处被换掉)。"""
from app.engine.inquiry import _add_based
secrets = masking._Secrets()
stats = {"source": "pg", "sql": "SELECT attname, n_distinct FROM pg_stats WHERE tablename = 'sales'",
"columns": ["attname", "n_distinct"], "rows": [["region", 3], ["buyer_phone", -1]]}
mixed = {"source": "pg", "columns": ["手机号", "region"], "rows": [[PHONE, "华东"]],
"sql": "SELECT buyer_phone AS 手机号, region FROM sales UNION ALL SELECT query_to_xml('x', true, false, ''), "
"'y'"}
_add_based(secrets, [(stats, set(MASKED)), (mixed, set(MASKED))], {"pg": "postgresql"})
assert secrets.hit(PHONE) and not secrets.hit("华东") and not secrets.hit("region")
@@ -146,6 +146,27 @@ def test_a_json_arrow_over_an_unmasked_column_stays_visible():
assert hidden_of("SELECT transform(scores, x -> x + 1) AS 分 FROM sales", ["分"], "postgres") == MASKED
@pytest.mark.parametrize("sql, columns, shown", [
pytest.param("SELECT t.profile.tel AS 电话 FROM sales t", ["电话"], ["电话"], id="JSON 点号"),
pytest.param("SELECT t.profile.contact.tel FROM sales t", ["tel"], ["tel"], id="JSON 点号两层"),
pytest.param("SELECT UPPER(t.profile.tel) AS 电话, t.region FROM sales t", ["电话", "region"], ["电话"],
id="套函数"),
pytest.param("SELECT x.电话 FROM (SELECT t.profile.tel AS 电话 FROM sales t) x", ["电话"], ["电话"],
id="子查询里取出再选"),
])
def test_oracle_dot_notation_into_a_masked_json_column_is_read_as_that_column(sql, columns, shown):
"""Oracle 的 JSON 点号、对象类型的属性写法(t.profile.tel):解析器读成「库 t、表 profile、列 tel」,以前只认 tel,
取出来的号码原样交给模型、落库。中间几段也算列(多遮:写了库名、表名的列,表名碰巧和遮罩列同名时也算)。"""
hidden = hidden_of(sql, columns, "oracle")
assert {c.lower() for c in shown} <= hidden and "region" not in hidden
def test_oracle_dot_notation_into_an_unmasked_column_stays_visible():
"""点号取的不是遮罩列、写了库名表名的普通列:一列都不遮。"""
assert hidden_of("SELECT t.tags.name AS 标签 FROM sales t", ["标签"], "oracle") == MASKED
assert hidden_of("SELECT shop.sales.region FROM shop.sales", ["region"], "oracle") == MASKED
def test_a_json_arrow_inside_json_each_is_masked_when_really_run():
"""SQLite 里真跑一遍:json_each(profile->'$.tels') 拆出来的号码在交给模型之前就遮掉。"""
import sqlite3
@@ -159,3 +180,72 @@ def test_a_json_arrow_inside_json_each_is_masked_when_really_run():
db.close()
done = masking.mask_query_result(payload, ["profile"], "sqlite")
assert done.columns == ["v"] and done.payload["rows"] == [["已遮罩"], ["已遮罩"]]
# --------------------------------------------------------------------------
# 交回复合类型的表值函数(K3b):Postgres 把复合类型展开成它的各个字段
# --------------------------------------------------------------------------
#: people3.card 是设了遮罩的复合类型(addr, tel),people2.pj 是设了遮罩的 jsonb(照着 contact 类型拆),sales 是订单
COMPOSITE_SCHEMA = {"tables": {
"people3": {"qualified": "people3", "schema": None, "columns": [{"name": c} for c in ["id", "card", "city"]]},
"people2": {"qualified": "people2", "schema": None, "columns": [{"name": c} for c in ["id", "pj"]]},
"sales": {"qualified": "sales", "schema": None, "columns": [{"name": c} for c in SALES]},
}}
COMPOSITE_MASKED = ["card", "pj", "buyer_phone"]
def _boundary(sql: str, columns: list[str], row: list) -> masking.QueryMask:
payload = {"columns": columns, "rows": [list(row)], "sql": sql}
return masking.mask_query_result(payload, COMPOSITE_MASKED, "postgresql", schema=COMPOSITE_SCHEMA)
#: (SQL, 结果列, 一行):结果的样子照 Postgres 18 实测。unnest 的数组元素是复合类型时,FROM 里的 unnest 展开成复合类型的
#: 各个字段(addr、tel),写了列名表也只改前几列的名字;交回复合类型的函数(jsonb_populate_record(NULL::contact, …))
#: 写了只有名字、没写类型的列名表时同样只改前几列,后面的字段照原名出
COMPOSITE_RESULTS = [
pytest.param("SELECT v.* FROM people3 p, unnest(ARRAY[p.card]) v", ["addr", "tel"], ["某路 1 号", "13900000001"],
id="unnest 复合类型的数组"),
pytest.param("SELECT * FROM unnest(ARRAY[(SELECT card FROM people3 WHERE id = 1)])", ["addr", "tel"],
["某路 1 号", "13900000001"], id="参数是子查询"),
pytest.param("SELECT v.tel FROM people3 p, LATERAL unnest(ARRAY[p.card]) v", ["tel"], ["13900000001"],
id="LATERAL 取一个字段"),
pytest.param("SELECT * FROM people3 p, unnest(ARRAY[p.card]) AS u(a)", ["id", "card", "city", "a", "tel"],
[1, "(某路 1 号,13900000001)", "杭州", "某路 1 号", "13900000001"], id="列名表比字段少"),
pytest.param("SELECT u.* FROM people3 p, unnest(ARRAY[p.card], ARRAY[p.city]) AS u", ["addr", "tel", "unnest"],
["某路 1 号", "13900000001", "杭州"], id="unnest 几个数组"),
pytest.param("SELECT u.* FROM people3 p, unnest(ARRAY[p.card]) WITH ORDINALITY AS u(a)",
["a", "tel", "ordinality"], ["某路 1 号", "13900000001", 1], id="WITH ORDINALITY"),
pytest.param("SELECT * FROM (SELECT p.card FROM people3 p) s, LATERAL unnest(ARRAY[s.card]) WITH ORDINALITY u",
["card", "addr", "tel", "ordinality"], ["(某路 1 号,13900000001)", "某路 1 号", "13900000001", 1],
id="参数取自派生来源"),
pytest.param("SELECT r.* FROM people2 p, jsonb_populate_record(NULL::contact, p.pj) AS r(a)", ["a", "tel"],
["丙路", "13900000021"], id="populate_record 列名表比字段少"),
pytest.param("SELECT r.* FROM people2 p, LATERAL jsonb_populate_recordset(NULL::contact, jsonb_build_array(p.pj)) "
"AS r(a)", ["a", "tel"], ["丙路", "13900000021"], id="populate_recordset 列名表比字段少"),
]
@pytest.mark.parametrize("sql, columns, row", COMPOSITE_RESULTS)
def test_a_table_function_spreading_a_composite_masks_the_whole_result(sql, columns, row):
"""FROM 里的函数交回复合类型、读到遮罩列:展开成哪几个字段、各叫什么 SQL 里看不出(同 (表达式).*),整份遮。以前
unnest 按「出一列,名字是别名或函数名」认,列名表按写全了认,tel 这一列的号码原样交给模型、落库。"""
done = _boundary(sql, columns, row)
assert done.whole and done.columns == columns
assert done.payload["rows"] == [["已遮罩"] * len(columns)]
@pytest.mark.parametrize("sql, columns, row, hidden", [
pytest.param("SELECT r.* FROM people2 p, jsonb_to_record(p.pj) AS r(tel text, addr text)", ["tel", "addr"],
["13900000021", "丙路"], ["tel", "addr"], id="写了带类型的列定义"),
pytest.param("SELECT k, v FROM people2 p, jsonb_each_text(p.pj) AS x(k, v)", ["k", "v"], ["tel", "13900000021"],
["k", "v"], id="出列认得全的函数写了列名表"),
pytest.param("SELECT s.id, u FROM sales s, unnest(ARRAY[s.region]) AS u", ["id", "u"], [1, "华东"], [],
id="unnest 没碰遮罩列"),
])
def test_a_table_function_whose_columns_are_spelled_out_is_read_as_before(sql, columns, row, hidden):
"""列写全了的(带类型的列定义、出列认得全的函数)照旧按名字认,没碰遮罩列的照常。"""
done = _boundary(sql, columns, row)
assert not done.whole and done.columns == hidden
assert done.payload["rows"] == [["已遮罩" if c in hidden else v for c, v in zip(columns, row)]]
+372
View File
@@ -0,0 +1,372 @@
"""工具边界先遮(engine/masking.py 的 mask_query_result)的收口:认不准就整份遮,多遮也不改写没设遮罩的数值证据。
- 遮罩分析本身出了意外(认遮罩列、标签列、逐格换值的哪一步都算):整份结果的每一格都是「已遮罩」,交给模型的、落库的
都一样,masked_at_query 记全部的列。兜底不再经出意外的那一步(以前兜底又调了一遍逐格换值的那一步,它出意外时兜底跟着
出意外,结果没交出去,也没有测试证明「整份遮」这句保证)
- 按值遮只换文字格:没设遮罩的数值列(访问次数正好等于某一笔被遮的金额)原样,存下的证据和数据库里的一样,双盲复算照常
取得到;文字格里整格就是遮罩值、夹着遮罩值的照换,换过的格子记进快照(masked_cells),格子引用指到它们照实说是遮罩。
占位的短词(未知、暂无……)不当遮罩值,没设遮罩的列、备注里的同一个词照原样
模型一律是 MockChatModel 剧本,数据源是现造的 SQLite 文件,手机号、姓名、金额、访问次数都是虚构的。
"""
from __future__ import annotations
import json
import pytest
from httpx import ASGITransport, AsyncClient
from app.core import artifact_store
from app.data.engine import engines
from app.engine import masking
from app.main import app
from tests.test_verify_agent import Script, q
from tests.test_verify_agent_masking import ( # noqa: F401 - clean_spend 是自动用上的夹具,masked_shop 是夹具
MASKED,
_clean,
_source,
ask,
clean_spend,
masked_shop,
)
# --------------------------------------------------------------------------
# 分析出意外:整份遮
# --------------------------------------------------------------------------
def _broken(*args, **kwargs):
raise RuntimeError("遮罩分析里的意外")
PAYLOAD = {"sql": "SELECT id, buyer_phone, note FROM sales", "columns": ["id", "buyer_phone", "note"],
"rows": [[1, "13900000001", "客户来电"], [2, "13900000002", "例行回访"]],
"column_types": {"id": "number", "buyer_phone": "text", "note": "text"}}
@pytest.mark.parametrize("step", ["_derived_masks", "_snapshot_labels", "_masked_view"])
def test_a_failing_analysis_masks_every_cell(monkeypatch, step):
"""认遮罩列、认标签列、逐格换值,哪一步出了意外都整份遮:每一格都是「已遮罩」、类型记成文本,整列遮掉的是全部的列。"""
monkeypatch.setattr(masking, step, _broken)
done = masking.mask_query_result(json.loads(json.dumps(PAYLOAD)), ["buyer_phone"], "sqlite")
assert done.payload["rows"] == [[MASKED] * 3, [MASKED] * 3]
assert done.payload["column_types"] == {"id": "text", "buyer_phone": "text", "note": "text"}
assert done.columns == ["id", "buyer_phone", "note"]
assert done.payload["sql"] == PAYLOAD["sql"], "SQL 是模型自己写的,照原样"
assert not done.secrets.strong, "认不准,不收值"
assert done.whole, "快照记下是整份遮的"
def test_a_failing_analysis_masks_rows_of_any_shape(monkeypatch):
"""行是字典的结果同样整份遮。"""
monkeypatch.setattr(masking, "_masked_view", _broken)
payload = {"sql": "SELECT id, buyer_phone FROM sales", "columns": ["id", "buyer_phone"],
"rows": [{"id": 1, "buyer_phone": "13900000001"}]}
done = masking.mask_query_result(payload, ["buyer_phone"], "sqlite")
assert done.payload["rows"] == [{"id": MASKED, "buyer_phone": MASKED}]
@pytest.mark.parametrize("step", ["_derived_masks", "_masked_view"])
async def test_the_model_and_the_snapshot_see_only_masks_when_the_analysis_fails(monkeypatch, masked_shop, step):
"""端到端:分析出了意外,交给模型的工具结果、落库的快照、通用接口给的都只有「已遮罩」,masked_at_query 是全部的列;
格子引用照实说写法认不全、整份遮了,不采信——金额在这里没设遮罩,不说成「取自设置了遮罩的列」。"""
monkeypatch.setattr(masking, step, _broken)
name = masked_shop
sql = "SELECT id, region, buyer_phone AS 手机号, amount FROM sales ORDER BY id"
script = Script(monkeypatch, [q(name, sql, "c1")], {"value": {"alias": "Q1", "row": 0, "column": "amount"},
"rationale": "x"})
out, _ = await ask(name)
seen = json.loads(script.tool_results()["c1"].split("\n", 1)[1])
assert seen["rows"] == [[MASKED] * 4] * 4, seen
snapshot = artifact_store.load(out.queries[0]["artifact"])
assert snapshot["rows"] == [[MASKED] * 4] * 4
assert snapshot["masked_at_query"] == ["id", "region", "手机号", "amount"] and snapshot["masked_whole"] is True
assert "masked_whole" not in seen, "记录只在快照里,不交给模型"
assert _clean(json.dumps(snapshot, ensure_ascii=False))
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
got = await client.get(f"/api/artifacts/{out.queries[0]['artifact']}")
assert got.status_code == 200 and _clean(got.text) and "1200" not in got.text
assert out.result["value"] is None
assert out.cells[0]["reason"] == "Q1 的写法认不全,整份结果查询时已遮罩,不采信"
# --------------------------------------------------------------------------
# 多遮不改写没设遮罩的数值证据;占位的短词不当遮罩值
# --------------------------------------------------------------------------
#: 金额设了遮罩,都是五位数;访问次数没设遮罩,第二、三单正好等于别的订单的金额(20000、30000)。联系表的访问次数存成
#: 文字('20000'),客户名设了遮罩、里面有占位的「未知」,区域和备注里也写着「未知」
BIG = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, amount INTEGER, visits INTEGER, buyer_name TEXT,"
" note TEXT);"
"INSERT INTO sales VALUES (1, '华东', 10000, 25000, '张伟', '原因未知'), (2, '华东', 12000, 20000, '未知', ''),"
" (3, '华南', 20000, 30000, '李娜', '客户未知来源'), (4, '未知', 30000, 22000, '王芳', ''),"
" (5, '华北', 50000, 9000, '刘洋', '');"
"CREATE TABLE contacts (id INTEGER PRIMARY KEY, visits_text TEXT, remark TEXT);"
"INSERT INTO contacts VALUES (1, '20000', '上月消费 30000 元'), (2, '8', '例行回访');")
@pytest.fixture
async def big_shop(tmp_path):
name, source_id = await _source(tmp_path, {"mask_columns": "amount, buyer_name"}, script=BIG)
yield name
await engines.invalidate(source_id)
async def test_an_unmasked_numeric_column_is_kept_even_when_it_equals_a_masked_value(monkeypatch, big_shop):
"""访问次数没设遮罩,第二、三单正好等于别的订单被遮的金额:交给模型的、落库的都是数据库里的原数,类型照旧是数,
引用它照常取值(双盲复算取得到 20000)。以前按值整格换成「已遮罩」,快照里不留记录,引用时说成「不是数值」。"""
name = big_shop
sql = "SELECT id, amount, visits FROM sales ORDER BY id"
script = Script(monkeypatch, [q(name, sql, "c1")], {"value": {"alias": "Q1", "row": 1, "column": "visits"},
"rationale": "x"})
out, _ = await ask(name)
seen = json.loads(script.tool_results()["c1"].split("\n", 1)[1])
assert seen["rows"] == [[1, MASKED, 25000], [2, MASKED, 20000], [3, MASKED, 30000], [4, MASKED, 22000],
[5, MASKED, 9000]]
snapshot = artifact_store.load(out.queries[0]["artifact"])
assert snapshot["rows"] == seen["rows"] and snapshot["column_types"]["visits"] == "number"
assert snapshot["masked_at_query"] == ["amount"] and "masked_cells" not in snapshot
assert out.result["value"] == 20000 and out.cells[0]["status"] == "resolved"
async def test_text_cells_masked_by_value_are_recorded_and_cited_as_masked(monkeypatch, big_shop):
"""存成文字的数('20000')、备注里夹着的金额(30000)照样按值换:交给模型的、落库的都换掉,换过的格子记进快照
(masked_cells),引用到它们照实说是遮罩,不说成「不是数值」。"""
name = big_shop
steps = [q(name, "SELECT amount FROM sales ORDER BY id", "c1"),
q(name, "SELECT id, visits_text, remark FROM contacts ORDER BY id", "c2")]
script = Script(monkeypatch, steps, lambda messages: {"value": {"alias": "Q2", "row": 0, "column": "visits_text"},
"rationale": "x"})
out, _ = await ask(name)
seen = json.loads(script.tool_results()["c2"].split("\n", 1)[1])
assert seen["rows"] == [[1, MASKED, f"上月消费 {MASKED} 元"], [2, "8", "例行回访"]]
assert "masked_cells" not in seen, "记录只在快照里,不交给模型"
snapshot = artifact_store.load(out.queries[1]["artifact"])
assert snapshot["rows"] == seen["rows"] and snapshot["masked_cells"] == [[0, 1], [0, 2]]
assert snapshot["masked_at_query"] == []
assert out.result["value"] is None
assert out.cells[0]["reason"] == "Q2 第 0 行「visits_text」与设置了遮罩的值相同,查询时已遮罩,不采信"
async def test_a_partly_masked_text_cell_is_cited_as_masked(monkeypatch, big_shop):
"""引用指到文字里换掉了一截的格子:同样照实说是遮罩。"""
name = big_shop
steps = [q(name, "SELECT amount FROM sales ORDER BY id", "c1"),
q(name, "SELECT id, remark FROM contacts ORDER BY id", "c2")]
Script(monkeypatch, steps, {"value": {"alias": "Q2", "row": 0, "column": "remark"}, "rationale": "x"})
out, _ = await ask(name)
assert out.result["value"] is None
assert out.cells[0]["reason"] == "Q2 第 0 行「remark」里有与设置了遮罩的值相同的部分,查询时已遮罩,不采信"
async def test_placeholder_words_are_not_masked_elsewhere(monkeypatch, big_shop):
"""客户名设了遮罩,里面有占位的「未知」:客户名整列照旧遮;没设遮罩的区域里的「未知」、备注里的「原因未知」「客户
未知来源」照原样。以前「未知」当遮罩值收了,这几格都被换掉。"""
name = big_shop
sql = "SELECT id, buyer_name, region, note FROM sales ORDER BY id"
script = Script(monkeypatch, [q(name, sql, "c1")], {"value": {"alias": "Q1", "row": 0, "column": "id"},
"rationale": "x"})
out, _ = await ask(name)
seen = json.loads(script.tool_results()["c1"].split("\n", 1)[1])
assert [row[1] for row in seen["rows"]] == [MASKED] * 5
assert [row[2:] for row in seen["rows"]] == [["华东", "原因未知"], ["华东", ""], ["华南", "客户未知来源"],
["未知", ""], ["华北", ""]]
assert out.masked_values.hit("张伟") and not out.masked_values.hit("未知")
def test_placeholders_are_never_collected():
secrets = masking._Secrets()
secrets.add({"columns": ["name"], "rows": [["未知"], ["N/A"], ["暂无"], ["张伟"], ["NULL"]]}, {"name"})
assert secrets.strong == {"张伟"} and secrets.weak == {"张伟"}
assert secrets.scrub("原因未知,暂无记录,N/A,张伟") == f"原因未知,暂无记录,N/A,{MASKED}"
# --------------------------------------------------------------------------
# 分档写在派生来源里、外层按它分组:标签列照常显示,标签不当遮罩值
# --------------------------------------------------------------------------
LAYOUT = ["id", "region", "amount", "visits"]
#: 探查得到的表结构:sales 里没有「档位」这一列
SCHEMA = {"tables": {"sales": {"qualified": "sales", "schema": None, "columns": [{"name": c} for c in LAYOUT]}}}
BAND = "CASE WHEN amount >= 20000 THEN '大单' ELSE '小单' END"
GROUPED = [
pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT *, {BAND} AS 档位 FROM sales) GROUP BY 档位 ORDER BY 档位",
id="子查询里 * 加分档"),
pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT s.*, {BAND} AS 档位 FROM sales s) t GROUP BY 档位 "
"ORDER BY 档位", id="子查询里 s.* 加分档"),
pytest.param(f"WITH b AS (SELECT *, {BAND} AS 档位 FROM sales) SELECT 档位, COUNT(*) AS 笔数 FROM b GROUP BY 档位 "
"ORDER BY 档位", id="WITH 里 * 加分档"),
pytest.param(f"WITH b AS (SELECT {BAND} AS 档位 FROM sales) SELECT 档位, COUNT(*) AS 笔数 FROM b GROUP BY 档位 "
"ORDER BY 档位", id="WITH 里只有分档"),
pytest.param(f"WITH b AS (SELECT {BAND} AS 档位 FROM sales) SELECT b.档位, COUNT(*) AS 笔数 FROM b GROUP BY 1 "
"ORDER BY 1", id="按位置分组"),
pytest.param(f"WITH b AS (SELECT {BAND} AS 档位 FROM sales), c AS (SELECT 档位, COUNT(*) AS 笔数 FROM b "
"GROUP BY 档位) SELECT * FROM c ORDER BY 档位", id="分组在中间一层"),
]
ROWS = [["大单", 3], ["小单", 2]]
@pytest.mark.parametrize("sql", GROUPED)
def test_a_band_grouped_outside_its_derived_source_is_shown(sql):
"""分档(按遮罩的金额写死的标签)写在子查询、WITH 里,外层按它分组:每档一行,档位列照常显示,标签不当遮罩值收;
以前整列遮成「已遮罩」,模型分不清哪个数属于哪一档。"""
payload = {"sql": sql, "columns": ["档位", "笔数"], "rows": [list(r) for r in ROWS],
"column_types": {"档位": "text", "笔数": "number"}}
done = masking.mask_query_result(payload, ["amount"], "sqlite", schema=SCHEMA)
assert done.payload["rows"] == ROWS and done.columns == []
assert not done.secrets.strong
hidden, names = masking._derived_hidden({**payload, "rows": ROWS}, {"amount"}, "sqlite",
masking._schema_columns(SCHEMA))
assert "档位" not in hidden and "档位" in names, "SQL 里照旧认得出它取自金额"
@pytest.mark.parametrize("sql", [
pytest.param(f"SELECT id, 档位 FROM (SELECT id, {BAND} AS 档位 FROM sales) ORDER BY id", id="逐行的档位"),
pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT {BAND} AS 档位 FROM sales UNION ALL "
"SELECT region FROM sales) GROUP BY 档位", id="另一支是区域"),
pytest.param(f"SELECT 档位, COUNT(*) AS 笔数 FROM (SELECT id, {BAND} AS 档位 FROM sales) x "
"FULL JOIN (SELECT id, amount AS 档位 FROM sales) y USING (档位) GROUP BY 档位", id="按档位连起来"),
])
def test_a_band_that_is_not_one_row_per_band_is_still_masked(sql):
"""逐行的档位(看得出每一笔落在哪一档)、和别的值混在一列里、按它连起来又不写表名的(外连接时取的可能是另一边的
金额),照旧整列遮。"""
payload = {"sql": sql, "columns": ["档位", "笔数"] if "笔数" in sql else ["id", "档位"],
"rows": [["大单", 3], ["小单", 2]] if "笔数" in sql else [[1, "小单"], [3, "大单"]]}
done = masking.mask_query_result(payload, ["amount"], "sqlite", schema=SCHEMA)
assert "档位" in done.columns
def test_a_grouped_band_with_cells_other_than_its_labels_is_still_masked():
"""格子里有标签以外的值(认错了、数的写法对不上):照旧整列遮。"""
sql = GROUPED[3].values[0]
payload = {"sql": sql, "columns": ["档位", "笔数"], "rows": [["大单", 3], ["13900000001", 2]]}
assert masking.mask_query_result(payload, ["amount"], "sqlite", schema=SCHEMA).columns == ["档位"]
def test_without_a_schema_a_star_next_to_the_band_is_still_masked():
"""不知道表结构时,* 展开的表可能就有「档位」这一列(外层取到的是它):照旧整列遮,标签照收(多遮)。"""
sql = GROUPED[0].values[0]
payload = {"sql": sql, "columns": ["档位", "笔数"], "rows": [list(r) for r in ROWS]}
done = masking.mask_query_result(payload, ["amount"], "sqlite")
assert done.columns == ["档位"] and done.payload["rows"] == [[MASKED, 3], [MASKED, 2]]
# --------------------------------------------------------------------------
# 数值格按位置认:同名的列、写成字符串的数
# --------------------------------------------------------------------------
def _known(*values: str) -> masking._Secrets:
"""此前认出的遮罩值(前面的查询遮掉的、报告快照里的)。"""
secrets = masking._Secrets()
secrets.add({"columns": ["v"], "rows": [[v] for v in values]}, {"v"})
return secrets
def test_a_text_column_sharing_its_name_with_a_number_column_is_still_masked_by_value():
"""column_types 按列名记,同名的几列只留下后面那一列的类型:两张表里都叫 phone,前一列是文字(写着号码)、后一列是
整数,记成 number。以前按列名认数值列,文字列也当成了数值列,号码原样交给模型、落库;现在同名的几列按格子本身认,
文字格照换、记进 masked_cells,真正的整数照原样。写成字符串的数在同名列里也按文字处理(宁可多遮)。"""
payload = {"sql": "SELECT a.region, a.phone, b.phone FROM contacts a JOIN legacy b ON a.id = b.id",
"columns": ["region", "phone", "phone"],
"rows": [["华东", "联系人 13900000001", 13900000002], ["华南", "13900000001", "13900000001"]],
"column_types": {"region": "text", "phone": "number"}}
done = masking.mask_query_result(payload, [], "sqlite", _known("13900000001"))
assert done.payload["rows"] == [["华东", f"联系人 {MASKED}", 13900000002], ["华南", MASKED, MASKED]]
assert done.cells == [[0, 1], [1, 1], [1, 2]]
def test_a_numeric_column_written_as_decimal_strings_is_kept():
"""Postgres 的 NUMERIC、MySQL 的 DECIMAL 交回来是字符串("20000.00"),列类型记着 number:没设遮罩的这一列正好等于某个
遮罩值(金额也是 NUMERIC)也照原样,不记进 masked_cells——数值证据不改写。"""
payload = {"sql": "SELECT id, visits FROM sales2 ORDER BY id", "columns": ["id", "visits"],
"rows": [[1, "20000.00"], [2, "9000.00"]], "column_types": {"id": "number", "visits": "number"}}
done = masking.mask_query_result(payload, [], "postgresql", _known("20000.00", "30000.00"))
assert done.payload["rows"] == [[1, "20000.00"], [2, "9000.00"]] and done.cells == []
#: 两张表里都有 remark:工单的是文字(写着号码),评分的是整数
DUPLICATES = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_phone TEXT, amount INTEGER);"
"INSERT INTO sales VALUES (1, '华东', '13900000001', 1200), (2, '华南', '13900000002', 900);"
"CREATE TABLE tickets (id INTEGER PRIMARY KEY, remark TEXT);"
"INSERT INTO tickets VALUES (1, '客户 13900000001 来电投诉'), (2, '回访 13900000002');"
"CREATE TABLE scores (id INTEGER PRIMARY KEY, remark INTEGER);"
"INSERT INTO scores VALUES (1, 5), (2, 4);")
@pytest.fixture
async def ticket_shop(tmp_path):
name, source_id = await _source(tmp_path, {"mask_columns": "buyer_phone"}, script=DUPLICATES)
yield name
await engines.invalidate(source_id)
@pytest.mark.parametrize("sql", [
"SELECT t.remark, s.remark FROM tickets t JOIN scores s ON s.id = t.id ORDER BY t.id",
"SELECT * FROM tickets t JOIN scores s ON s.id = t.id ORDER BY t.id",
])
async def test_duplicate_names_do_not_let_numbers_shield_text(monkeypatch, ticket_shop, sql):
"""端到端:先查出号码(查询时遮掉),再把工单和评分连起来查,两列都叫 remark(* 连两张表时 id 也同名)。工单备注里的
号码交给模型、落库之前照样换掉,评分照原样;以前评分的 number 套到了同名的备注上,号码原样交了出去。"""
name = ticket_shop
steps = [q(name, "SELECT id, buyer_phone FROM sales ORDER BY id", "c1"), q(name, sql, "c2")]
script = Script(monkeypatch, steps, {"value": None, "rationale": "x"})
out, _ = await ask(name)
seen = script.tool_results()["c2"]
assert "1390000000" not in seen and f"客户 {MASKED} 来电投诉" in seen, seen
snapshot = artifact_store.load(out.queries[1]["artifact"])
remarks = [i for i, c in enumerate(snapshot["columns"]) if c == "remark"]
assert [[row[i] for i in remarks] for row in snapshot["rows"]] == [[f"客户 {MASKED} 来电投诉", 5],
[f"回访 {MASKED}", 4]]
assert snapshot["masked_cells"] == [[0, remarks[0]], [1, remarks[0]]]
async def test_a_decimal_column_is_cited_as_a_number(monkeypatch, composite_numbers):
"""端到端:NUMERIC 的访问次数(字符串 "20000.00"、列类型 number)正好等于某一笔被遮的金额:交给模型的、落库的照原样,
格子引用照常取到 20000(双盲复算拿得到它)。"""
name, sql = composite_numbers
script = Script(monkeypatch, [q(name, "SELECT id, amount FROM sales ORDER BY id", "c1"), q(name, sql, "c2")],
{"value": {"alias": "Q2", "row": 0, "column": "visits"}, "rationale": "x"})
out, _ = await ask(name)
seen = json.loads(script.tool_results()["c2"].split("\n", 1)[1])
assert seen["rows"] == [[1, "20000.00"], [2, "9000.00"]]
snapshot = artifact_store.load(out.queries[1]["artifact"])
assert snapshot["rows"] == seen["rows"] and "masked_cells" not in snapshot
assert out.result["value"] == 20000 and out.cells[0]["status"] == "resolved"
@pytest.fixture
async def composite_numbers(monkeypatch):
"""造一个 Postgres 数据源(查询交给换掉的 run_query,不连库):金额设了遮罩,金额、访问次数都是 NUMERIC,驱动交回
Decimal,查询层转成字符串、列类型记 number。交回 (源名, 查访问次数的 SQL)。"""
import uuid
from app.data.engine import QueryResult
from app.db.base import SessionLocal
from app.db.models import DataSource
from app.tools import datasource
visits = "SELECT id, visits FROM sales ORDER BY id"
results = {"SELECT id, amount FROM sales ORDER BY id": (["id", "amount"], [[1, "20000.00"], [2, "30000.00"]]),
visits: (["id", "visits"], [[1, "20000.00"], [2, "9000.00"]])}
async def driver(source, statement, **kwargs):
columns, rows = results[statement]
return QueryResult(columns=columns, rows=[list(r) for r in rows], row_count=len(rows), truncated=False,
elapsed_ms=1, sql=statement, column_types={c: "number" for c in columns})
monkeypatch.setattr(datasource, "run_query", driver)
name = f"pg_{uuid.uuid4().hex[:8]}"
async with SessionLocal() as session:
row = DataSource(name=name, kind="postgresql", host="127.0.0.1", port=5432, username="probe", database="shop",
readonly=True, enabled=True, options={"mask_columns": "amount"},
schema_cache={"tables": {"sales": {"columns": [{"name": c} for c in ("id", "amount",
"visits")]}}})
session.add(row)
await session.commit()
source_id = row.id
yield name, visits
await engines.invalidate(source_id)
@@ -0,0 +1,511 @@
"""认不全的写法整份遮,K3b 复核第 2 轮补的两类(engine/masking.py 的 _tainted_in、_opaque):
- VALUES 里的 (表达式).*:Postgres 在 VALUES 里同样把复合类型的值、记录展开成好几列(文档:选出列表、VALUES、行构造器
里的 composite.* 都展开),LATERAL VALUES 按表值函数认时一个元组项只算一列,column2 起的号码原样交给模型、落库。现在
树上任何位置的 (表达式).* 读到遮罩列,整份结果都遮——LATERAL、JOIN LATERAL、CTE、FROM 里的 VALUES 都一样
- 记着别的会话的 SQL 原文、被锁那一行的值的来源:MySQL 的 information_schema.INNODB_TRX、OPTIMIZER_TRACE,MariaDB 的
INNODB_LOCKS、QUERY_CACHE_INFO,Oracle 的 SQL 调优集、固定下来的执行计划、闪回事务、LogMiner;不只读表结构的 SHOW
(ENGINE INNODB STATUS、BINLOG / RELAYLOG EVENTS、REPLICA STATUS、WARNINGS、CREATE TABLE / VIEW……)和 EXPLAIN 一条
语句。以前工具边界的名单里没有,可验证报告那份整个挡 information_schema、不附带解析不了的快照,反而挡得住
复核第 3 轮补的同一类:
- VALUES 当一张表用(FROM、JOIN、LATERAL 里的,WITH 里的一段,UNION 的一支)又读到遮罩列(含派生出来的名字):各库给
VALUES 的列起名不一样(Postgres、SQLite 叫 column1…,MySQL 8.0 叫 column_0…,MariaDB 拿第一行的值当列名),以前一律
按 column1… 认,MySQL 派生表、CTE、LATERAL 里的 VALUES ROW 取了号码,原样交给模型、落库;套了括号的 VALUES(整条
就是它、UNION 的一支)解析器原样挂在树上,哪个库都漏。现在不论怎么起名都整份遮;列名按方言认,整份遮时只收值取自
遮罩列的那几列的值
- MySQL sys 库里读 performance_schema 的函数(sys.ps_thread_trx_info 交回线程的事务里执行过的语句原文,ps_thread_stack
同类):按 sys.ps_ 开头认,工具边界和可验证报告共用一份名单
这里没有真的数据库:结果按它交回的样子造(复合类型展开成几列、SHOW 交回的几列,照 Postgres 18、MySQL 8.0 实测的样子)。
数据都是虚构的:手机号、地址、区域。
"""
from __future__ import annotations
import json
import uuid
import pytest
from httpx import ASGITransport, AsyncClient
from app.core import artifact_store
from app.data.engine import engines
from app.db.base import SessionLocal
from app.db.models import DataSource
from app.engine import masking
from app.main import app
from tests.test_verify_agent import Script, q
from tests.test_verify_agent_masking import ask, clean_spend # noqa: F401 - clean_spend 是自动用上的夹具
MASKED_WORD = "已遮罩"
#: people3.card 是复合类型(addr, tel),people.address 是没设遮罩的复合类型(street, city),sales 是订单。设了遮罩的是
#: card、buyer_phone
SCHEMA = {"tables": {
"people3": {"qualified": "people3", "schema": None, "columns": [{"name": c} for c in ["id", "card", "city"]]},
"people": {"qualified": "people", "schema": None, "columns": [{"name": c} for c in ["id", "address"]]},
"sales": {"qualified": "sales", "schema": None,
"columns": [{"name": c} for c in ["id", "region", "buyer_phone", "amount"]]},
}}
MASKED = ["card", "buyer_phone"]
PHONE = "13900000001"
def _leaks(done: masking.QueryMask) -> bool:
return "1390000000" in json.dumps(done.payload, ensure_ascii=False)
def masked(sql: str, columns: list[str], rows: list[list], kind: str = "postgresql") -> masking.QueryMask:
payload = {"columns": columns, "rows": rows, "sql": sql}
return masking.mask_query_result(payload, MASKED, kind, schema=SCHEMA)
# --------------------------------------------------------------------------
# VALUES 里的 (表达式).*
# --------------------------------------------------------------------------
#: 复合类型的列、子查询交回的复合值、匿名 ROW、整行 × LATERAL、JOIN LATERAL、CTE、FROM 里的 VALUES:(SQL, 结果列, 一行)。
#: 号码都在第二列起:以前 LATERAL VALUES 一个元组项只算 column1,column2 起的号码原样交了出去
VALUES_EXPANSIONS = [
pytest.param("SELECT v.* FROM people3 p, LATERAL (VALUES ((p.card).*)) v", ["column1", "column2"],
["某路 1 号", PHONE], id="LATERAL-取 *"),
pytest.param("SELECT v.column2 AS c FROM people3 p, LATERAL (VALUES ((p.card).*)) v", ["c"], [PHONE],
id="LATERAL-取第二列"),
pytest.param("SELECT x.column2 FROM people3 p JOIN LATERAL (VALUES ((p.card).*)) x ON true", ["column2"], [PHONE],
id="JOIN LATERAL"),
pytest.param("WITH v AS (VALUES ((SELECT card FROM people3 WHERE id = 1).*)) SELECT * FROM v",
["column1", "column2"], ["某路 1 号", PHONE], id="CTE 里的 VALUES"),
pytest.param("SELECT t FROM (VALUES ((SELECT card FROM people3 WHERE id = 1).*)) v(a, t)", ["t"], [PHONE],
id="FROM 里的 VALUES-列名表"),
pytest.param("SELECT v.column2 FROM sales s, LATERAL (VALUES ((ROW(s.region, s.buyer_phone)).*)) v", ["column2"],
[PHONE], id="匿名 ROW"),
pytest.param("SELECT v.* FROM sales s, LATERAL (VALUES ((s).*)) v", ["column1", "column2", "column3", "column4"],
[1, "华东", PHONE, 1200], id="整行"),
pytest.param("SELECT v.column3 FROM people3 p, LATERAL (VALUES (p.id, (p.card).*)) v", ["column3"], [PHONE],
id="和别的项写在一个元组里"),
# VALUES 里的 s.*、(s.*) 同样展开成好几列(s 是表,或者取了遮罩列的子查询)
pytest.param("SELECT v.* FROM sales s, LATERAL (VALUES (s.*)) v", ["column1", "column2", "column3", "column4"],
[1, "华东", PHONE, 1200], id="表的 *"),
pytest.param("SELECT v.column3 FROM sales s, LATERAL (VALUES ((s.*))) v", ["column3"], [PHONE],
id="套了括号的表的 *"),
pytest.param("SELECT v.column2 FROM (SELECT id, buyer_phone AS p FROM sales) q, LATERAL (VALUES (q.*)) v",
["column2"], [PHONE], id="子查询的 *"),
]
@pytest.mark.parametrize("sql, columns, row", VALUES_EXPANSIONS)
def test_an_expression_star_inside_values_masks_the_whole_result(sql, columns, row):
"""VALUES 里的 (表达式).* 读到遮罩列:展开成几列说不清,整份结果的每一格都遮。以前只遮了 column1(或一列不遮),
号码原样交给模型、落库。"""
done = masked(sql, columns, [list(row)])
assert done.whole and done.columns == columns
assert done.payload["rows"] == [[MASKED_WORD] * len(columns)] and not _leaks(done)
@pytest.mark.parametrize("sql, columns, row", [
pytest.param("SELECT v.* FROM people p, LATERAL (VALUES ((p.address).*)) v", ["column1", "column2"],
["某路", "某市"], id="没设遮罩的复合类型"),
pytest.param("SELECT v.column1 AS r FROM sales s, LATERAL (VALUES (s.region)) v", ["r"], ["华东"],
id="没碰遮罩列的 VALUES"),
pytest.param("SELECT v.column2 AS r FROM (SELECT id, region FROM sales) q, LATERAL (VALUES (q.*)) v", ["r"],
["华东"], id="没取遮罩列的子查询的 *"),
])
def test_values_that_reads_no_masked_column_is_read_as_before(sql, columns, row):
"""VALUES 没读到遮罩列(没设遮罩的复合类型、没碰遮罩列的列、没取遮罩列的子查询的 *):照常,一列都不遮。读到遮罩列的
(直接写遮罩列的也算)当一张表用就整份遮,见下一节。"""
done = masked(sql, columns, [list(row)])
assert not done.whole and done.columns == [] and done.payload["rows"] == [list(row)]
def test_an_expression_star_inside_a_function_argument_masks_the_whole_result_too():
"""行构造器、函数参数里的 (表达式).*:不逐个位置认展开不展开,读到遮罩列的一样整份遮(多遮,编号也遮)。"""
done = masked("SELECT p.id AS n, row_to_json(ROW((p.card).*)) AS j FROM people3 p", ["n", "j"],
[[1, json.dumps({"f1": "某路 1 号", "f2": PHONE})]])
assert done.whole and done.payload["rows"] == [[MASKED_WORD, MASKED_WORD]] and not _leaks(done)
# --------------------------------------------------------------------------
# VALUES 当一张表用(复核第 3 轮)
# --------------------------------------------------------------------------
#: VALUES 当一张表用、读到遮罩列:(方言, SQL, 结果列, 各行),号码在最后一列。MySQL 8.0 给 VALUES 的列起名 column_0、
#: column_1…(从 0 数起),MariaDB 拿第一行的值当列名;以前一律按 column1… 认,MySQL、MariaDB 的这几种号码原样交了出去
VALUES_TABLES = [
pytest.param("mysql", "SELECT * FROM (VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1))) AS v", ["column_0"],
[[PHONE]], id="MySQL-派生表"),
pytest.param("mysql", "SELECT x.column_0 FROM sales s, LATERAL (VALUES ROW(s.buyer_phone)) AS x", ["column_0"],
[[PHONE]], id="MySQL-LATERAL"),
pytest.param("mysql", "WITH v AS (VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1))) SELECT * FROM v",
["column_0"], [[PHONE]], id="MySQL-CTE"),
pytest.param("mysql", "SELECT v.column_0 AS c FROM (VALUES ROW((SELECT buyer_phone FROM sales WHERE id = 1)), "
"ROW((SELECT buyer_phone FROM sales WHERE id = 2))) AS v", ["c"], [[PHONE], ["13900000002"]],
id="MySQL-外层改了名字"),
pytest.param("mysql", "SELECT s.id, t.column_1 FROM sales s JOIN LATERAL (VALUES ROW(s.region, s.buyer_phone)) "
"AS t ON TRUE", ["id", "column_1"], [[1, PHONE]], id="MySQL-JOIN LATERAL 连同编号"),
pytest.param("mysql", "SELECT w.* FROM sales s, LATERAL (SELECT * FROM (VALUES ROW(s.buyer_phone)) v) AS w",
["column_0"], [[PHONE]], id="MySQL-LATERAL 子查询里的 VALUES"),
pytest.param("mysql", "SELECT (SELECT column_0 FROM (VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1))) v) AS p",
["p"], [[PHONE]], id="MySQL-标量子查询里的 VALUES"),
pytest.param("mysql", "WITH q AS (SELECT buyer_phone AS p FROM sales) "
"SELECT v.column_0 FROM q, LATERAL (VALUES ROW(q.p)) AS v", ["column_0"], [[PHONE]],
id="MySQL-取派生出来的名字"),
pytest.param("mysql", "VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1)) UNION ALL SELECT region FROM sales",
["column_0"], [[PHONE], ["华东"]], id="MySQL-UNION 第一支"),
# 套了括号的 VALUES 解析器不改写成 FROM 里的 VALUES:整条就是它、UNION 的一支都原样挂在树上,以前不认,Postgres
# 里也漏
pytest.param("mysql", "(VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1)))", ["column_0"], [[PHONE]],
id="MySQL-整条套括号"),
pytest.param("postgresql", "(VALUES ((SELECT buyer_phone FROM sales LIMIT 1))) UNION ALL "
"(SELECT region FROM sales)", ["column1"], [[PHONE], ["华东"]], id="Postgres-UNION 第一支套括号"),
pytest.param("mysql", "SELECT * FROM ((VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1))) UNION ALL "
"(SELECT 'a')) t", ["column_0"], [[PHONE], ["a"]], id="MySQL-派生表里 UNION 的一支"),
pytest.param("mariadb", "SELECT * FROM (VALUES ((SELECT buyer_phone FROM sales LIMIT 1))) AS v",
["(SELECT buyer_phone FROM sales LIMIT 1)"], [[PHONE]], id="MariaDB-派生表"),
pytest.param("mariadb", "SELECT * FROM (VALUES ('华东'), ((SELECT buyer_phone FROM sales LIMIT 1))) AS v",
["华东"], [["华东"], [PHONE]], id="MariaDB-第一行的值当列名"),
pytest.param("postgresql", "SELECT v.column1 AS p FROM sales s, LATERAL (VALUES (s.buyer_phone)) v", ["p"],
[[PHONE]], id="Postgres-LATERAL"),
pytest.param("postgresql", "SELECT s.id, v.k, v.val FROM sales s CROSS JOIN LATERAL (VALUES ('phone', "
"s.buyer_phone), ('region', s.region)) AS v(k, val)", ["id", "k", "val"],
[[1, "phone", PHONE], [1, "region", "华东"]], id="Postgres-行转列"),
pytest.param("sqlite", "WITH v AS (VALUES ((SELECT buyer_phone FROM sales LIMIT 1))) SELECT * FROM v",
["column1"], [[PHONE]], id="SQLite-CTE"),
]
@pytest.mark.parametrize("kind, sql, columns, rows", VALUES_TABLES)
def test_a_values_table_on_a_masked_column_masks_the_whole_result(kind, sql, columns, rows):
"""VALUES 当一张表用、读到遮罩列(含派生出来的名字):不论各库给它的列起什么名字,整份结果的每一格都遮,同一份里
没碰遮罩列的编号、区域也遮(多遮)。以前 MySQL、MariaDB 的这几种和套了括号的 VALUES masked=[],号码原样交给模型、
落库;Postgres、SQLite 的按列名认得出,只遮了那几列。"""
done = masked(sql, columns, [list(r) for r in rows], kind=kind)
assert done.whole and done.columns == columns
assert done.payload["rows"] == [[MASKED_WORD] * len(columns)] * len(rows) and not _leaks(done)
@pytest.mark.parametrize("kind, sql, columns, row", [
pytest.param("mysql", "SELECT x.column_0 AS r FROM sales s, LATERAL (VALUES ROW(s.region)) AS x", ["r"], ["华东"],
id="MySQL-没碰遮罩列"),
pytest.param("mysql", "SELECT s.region, b.column_1 AS 档 FROM sales s JOIN (VALUES ROW(0, '小单'), "
"ROW(20000, '大单')) AS b ON s.amount >= b.column_0", ["region", "档"], ["华东", "大单"],
id="MySQL-写死的分档表"),
pytest.param("mysql", "SELECT region FROM sales UNION ALL VALUES ROW('其他')", ["region"], ["其他"],
id="MySQL-UNION 一支写死的值"),
])
def test_a_values_table_reading_no_masked_column_stays_visible(kind, sql, columns, row):
"""VALUES 当一张表用、却没读到遮罩列(写死的分档表、没碰遮罩列的 LATERAL、UNION 里写死的一支):照常。"""
done = masked(sql, columns, [list(row)], kind=kind)
assert not done.whole and done.columns == [] and done.payload["rows"] == [list(row)]
@pytest.mark.parametrize("kind, sql", [
# 号码在 ROW(…) 的第二项:MySQL 的一行解析成一个元组里套着一个 ROW 函数,要摊开数位置
pytest.param("mysql", "SELECT x.column_1 AS p, x.column_0 AS r FROM sales s, "
"LATERAL (VALUES ROW(s.region, s.buyer_phone)) AS x", id="MySQL"),
pytest.param("postgresql", "SELECT v.column1 AS p, v.column2 AS r FROM sales s, "
"LATERAL (VALUES (s.buyer_phone, s.region)) v", id="Postgres"),
pytest.param("postgresql", "SELECT v.a AS p, v.b AS r FROM sales s, LATERAL (VALUES (s.buyer_phone, s.region)) "
"v(a, b)", id="Postgres-列名表"),
pytest.param("sqlite", "WITH v AS (VALUES ((SELECT buyer_phone FROM sales LIMIT 1), '华东')) "
"SELECT column1 AS p, column2 AS r FROM v", id="SQLite-CTE"),
])
def test_only_the_values_columns_taken_from_a_masked_column_are_collected(kind, sql):
"""VALUES 的列名按方言认(MySQL 叫 column_0…,Postgres、SQLite 叫 column1…,写了列名表的按列名表):整份遮时只收值
取自遮罩列的那几列的值当遮罩值,后面的查询、回答里同一个号码照样按值换掉;同一行里的区域不收。以前 MySQL 按 column1
认,号码一个都没收;Postgres、SQLite 把 VALUES 出的每一列都收了,区域名在同一次核查里处处被换掉。"""
done = masked(sql, ["p", "r"], [[PHONE, "华东"]], kind=kind)
assert done.whole and done.payload["rows"] == [[MASKED_WORD, MASKED_WORD]]
assert PHONE in done.secrets.strong and "华东" not in done.secrets.strong
def test_items_after_an_expanding_star_in_a_values_row_are_not_collected_by_place():
"""一行里的 (表达式).* 展开成几列说不清:排在它后面的号码实际落在第三列,按位置数成第二列就收了没设遮罩的城市。
整份照遮,这一行不按位置收值(少收不露:这份结果的值本来就没交给模型)。"""
done = masked("SELECT v.column2 AS c, v.column3 AS p FROM people p, sales s, "
"LATERAL (VALUES ((p.address).*, s.buyer_phone)) v", ["c", "p"], [["某市", PHONE]])
assert done.whole and done.payload["rows"] == [[MASKED_WORD, MASKED_WORD]]
assert "某市" not in done.secrets.strong
@pytest.mark.parametrize("kind, sql, named, unnamed", [
pytest.param("mysql", "SELECT * FROM sales s, LATERAL (VALUES ROW(s.id, s.region)) AS x", {"column_0", "column_1"},
{"column1", "column2"}, id="MySQL"),
pytest.param("postgresql", "SELECT * FROM sales s, LATERAL (VALUES (s.id, s.region)) AS x", {"column1", "column2"},
{"column_0"}, id="Postgres"),
pytest.param("sqlite", "SELECT * FROM (VALUES (1, '华东')) AS x", {"column1", "column2"}, {"column_0"},
id="SQLite"),
# 列名表只改前几列的名字:第一列叫 n,不再叫 column1(Postgres 里按 column1 取会报错)
pytest.param("postgresql", "SELECT * FROM (VALUES (1, '华东')) AS x(n)", {"n", "column2"}, {"column1"},
id="Postgres-列名表只写前几列"),
])
def test_values_columns_are_named_the_way_each_database_names_them(kind, sql, named, unnamed):
"""表值函数这类来源出的列名(_function_columns)里,VALUES 的列按方言起名:别处按列名认来源出不出某一列时对得上。"""
from app.data.sql_semantics import _parse, _read
root, _ = _parse(sql, _read(kind))
columns = masking._function_columns(masking._function_sources(root)[-1], kind)
assert named <= columns and not unnamed & columns
@pytest.mark.parametrize("kind, sql", [
pytest.param("mariadb", "SELECT * FROM (VALUES (1, '华东')) AS x", id="MariaDB 拿第一行的值当列名"),
pytest.param("oracle", "SELECT * FROM (VALUES (1, '华东')) x", id="Oracle 没写列名表"),
])
def test_values_columns_that_cannot_be_named_are_unknown(kind, sql):
"""VALUES 的列名认不准(MariaDB 拿第一行的值当列名,Oracle 的文档只有写了列名表的写法)又没写列名表:说不清出哪几列。"""
from app.data.sql_semantics import _parse, _read
root, _ = _parse(sql, _read(kind))
assert masking._function_columns(masking._function_sources(root)[-1], kind) is None
# --------------------------------------------------------------------------
# 端到端:核查 agent 的真实查询工具
# --------------------------------------------------------------------------
@pytest.fixture
async def fake_source():
"""造出来的数据源(不连库,查询交给换掉的 run_query):make(kind) 建一个、交回源名。"""
made: list[str] = []
async def make(kind: str) -> str:
name = f"{kind[:2]}_{uuid.uuid4().hex[:8]}"
async with SessionLocal() as session:
row = DataSource(name=name, kind=kind, host="127.0.0.1", port=5432, username="probe", database="shop",
readonly=True, enabled=True, options={"mask_columns": ", ".join(MASKED)},
schema_cache=SCHEMA)
session.add(row)
await session.commit()
made.append(row.id)
return name
yield make
for source_id in made:
await engines.invalidate(source_id)
async def _asked(monkeypatch, fake_source, kind: str, sql: str, columns: list[str], rows: list[list]):
"""按剧本让核查 agent 查一次(结果由换掉的 run_query 交回),交回 (工具结果原文, 落库的快照, 通用工件接口给的原文)。"""
from app.data.engine import QueryResult
from app.tools import datasource
async def driver(source, statement, **kwargs):
assert statement == sql
return QueryResult(columns=list(columns), rows=[list(r) for r in rows], row_count=len(rows), truncated=False,
elapsed_ms=1, sql=statement)
monkeypatch.setattr(datasource, "run_query", driver)
name = await fake_source(kind)
script = Script(monkeypatch, [q(name, sql, "c1")], {"value": None, "rationale": "x"})
out, _ = await ask(name)
snapshot = artifact_store.load(out.queries[0]["artifact"])
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
got = await client.get(f"/api/artifacts/{out.queries[0]['artifact']}")
assert got.status_code == 200
return script.tool_results()["c1"], snapshot, got.text
@pytest.mark.parametrize("sql, columns, rows", [
("SELECT v.* FROM people3 p, LATERAL (VALUES ((p.card).*)) v", ["column1", "column2"],
[["某路 1 号", PHONE], ["某路 2 号", "13900000002"]]),
("SELECT v.column2 AS c FROM people3 p, LATERAL (VALUES ((p.card).*)) v", ["c"], [[PHONE], ["13900000002"]]),
], ids=["取 *", "取第二列"])
async def test_values_expansions_never_reach_the_model_or_the_store(monkeypatch, fake_source, sql, columns, rows):
"""交给模型的、落库的、通用工件接口给的都只有「已遮罩」,masked_at_query 是全部的列,快照记着整份遮。以前
masked_at_query 是 ['column1'] 或空的,号码原样交给模型、落库。"""
seen, snapshot, api = await _asked(monkeypatch, fake_source, "postgresql", sql, columns, rows)
assert MASKED_WORD in seen and "1390000000" not in seen, seen
assert snapshot["masked_at_query"] == columns and snapshot["masked_whole"] is True
assert "1390000000" not in json.dumps(snapshot, ensure_ascii=False) and "1390000000" not in api
@pytest.mark.parametrize("sql", [
"SELECT * FROM (VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1))) AS v",
"SELECT x.column_0 FROM sales s, LATERAL (VALUES ROW(s.buyer_phone)) AS x",
"WITH v AS (VALUES ROW((SELECT buyer_phone FROM sales LIMIT 1))) SELECT * FROM v",
], ids=["派生表", "LATERAL", "CTE"])
async def test_mysql_values_tables_never_reach_the_model_or_the_store(monkeypatch, fake_source, sql):
"""端到端(MySQL,VALUES 的列叫 column_0):交给模型的、落库的、通用工件接口给的都只有「已遮罩」,masked_at_query
是全部的列,快照记着整份遮。以前 masked_at_query 是空的,号码原样交出。"""
seen, snapshot, api = await _asked(monkeypatch, fake_source, "mysql", sql, ["column_0"], [[PHONE], ["13900000002"]])
assert MASKED_WORD in seen and "1390000000" not in seen, seen
assert snapshot["masked_at_query"] == ["column_0"] and snapshot["masked_whole"] is True
assert "1390000000" not in json.dumps(snapshot, ensure_ascii=False) and "1390000000" not in api
# --------------------------------------------------------------------------
# 记着别的会话的 SQL、被锁那一行的值的视图
# --------------------------------------------------------------------------
#: 另一个会话正在执行的语句(号码写在条件里):会话、事务、锁、日志一类的视图交回的就是它
BUSY = f"SELECT id, SLEEP(4) FROM sales WHERE buyer_phone = '{PHONE}' FOR UPDATE"
#: MySQL sys 库的 ps_thread_trx_info 交回的 JSON:那个线程的事务里执行过的语句原文在 statements_executed[].sql_text 里
#: (MySQL 8.0 实测)
TRX_INFO = json.dumps([{"trx_id": "1234", "state": "ACTIVE", "statements_executed": [{"sql_text": BUSY}]}],
ensure_ascii=False)
@pytest.mark.parametrize("sql, kind", [
pytest.param("SELECT trx_query AS t FROM information_schema.INNODB_TRX", "mysql", id="MySQL 正在执行的事务"),
pytest.param("SELECT trx_query FROM INFORMATION_SCHEMA.innodb_trx WHERE trx_state = 'RUNNING'", "mysql",
id="大小写混写"),
pytest.param("SELECT lock_data AS d FROM information_schema.INNODB_LOCKS", "mariadb", id="MariaDB 被锁的行"),
pytest.param("SELECT statement_text AS t FROM information_schema.QUERY_CACHE_INFO", "mariadb",
id="MariaDB 查询缓存"),
pytest.param("SELECT sql_text AS t FROM dba_sqlset_statements", "oracle", id="Oracle SQL 调优集"),
pytest.param("SELECT value_string AS t FROM dba_sqlset_binds", "oracle", id="Oracle 调优集的绑定变量"),
pytest.param("SELECT sql_text AS t FROM dba_sql_plan_baselines", "oracle", id="Oracle 固定的执行计划"),
pytest.param("SELECT sql_text AS t FROM dba_sql_profiles", "oracle", id="Oracle SQL Profile"),
pytest.param("SELECT value AS t FROM dba_sqltune_binds", "oracle", id="Oracle 调优顾问的绑定变量"),
pytest.param("SELECT undo_sql AS t FROM flashback_transaction_query", "oracle", id="Oracle 闪回事务"),
pytest.param("SELECT sql_redo AS t FROM v$logmnr_contents", "oracle", id="Oracle LogMiner"),
pytest.param("SELECT sql_text AS t FROM v$xml_audit_trail", "oracle", id="Oracle XML 审计"),
pytest.param("SELECT trace AS t FROM information_schema.OPTIMIZER_TRACE", "mysql", id="MySQL 优化器跟踪"),
# 动态视图 V$ / GV$ / X$、AWR 历史不逐个认,一律算
pytest.param("SELECT client_info AS t FROM v$session", "oracle", id="Oracle V$ 动态视图"),
pytest.param("SELECT kglnaobj AS t FROM x$kglob", "oracle", id="Oracle X$ 底表"),
pytest.param("SELECT sql_text AS t FROM gv$sql_monitor", "oracle", id="Oracle GV$"),
pytest.param("SELECT event AS t FROM dba_hist_active_sess_history", "oracle", id="Oracle AWR 历史"),
pytest.param("SELECT plan_table_output AS t FROM TABLE(DBMS_XPLAN.DISPLAY_CURSOR('a1', NULL, 'ADVANCED "
"+PEEKED_BINDS'))", "oracle", id="Oracle 游标的执行计划和绑定变量"),
pytest.param("SELECT DBMS_SQLTUNE.REPORT_SQL_MONITOR(sql_id => 'a1', type => 'TEXT') AS t FROM dual", "oracle",
id="Oracle SQL 监控报告"),
])
def test_views_that_keep_other_sessions_sql_mask_the_whole_result(sql, kind):
"""别的会话执行过的 SQL 原文、被锁那一行的键值、闪回和日志里的改动前后的值:读的是哪一列认不出,整份遮。以前
masked=[],条件里的号码原样交给模型、落库。"""
done = masked(sql, ["t"], [[BUSY]], kind=kind)
assert done.whole and done.payload["rows"] == [[MASKED_WORD]]
@pytest.mark.parametrize("sql, column", [
pytest.param("SELECT sys.ps_thread_trx_info(81) AS t", "t", id="事务里执行过的语句"),
pytest.param("SELECT sys.ps_thread_trx_info(81)", "sys.ps_thread_trx_info(81)", id="不起别名"),
pytest.param("SELECT `sys`.`ps_thread_trx_info`(81) AS t", "t", id="反引号"),
pytest.param("SELECT SYS . PS_THREAD_TRX_INFO (81) AS t", "t", id="大写、隔着空格"),
pytest.param("SELECT sys.ps_thread_stack(81, FALSE) AS t", "t", id="语句栈"),
pytest.param("SELECT sys.ps_thread_trx_info(sys.ps_thread_id(NULL)) AS t", "t", id="套着取线程号"),
])
def test_sys_functions_reading_performance_schema_mask_the_whole_result(sql, column):
"""MySQL sys 库里读 performance_schema 的函数交回别的会话执行过的 SQL 原文(条件里的号码):按 sys.ps_ 开头认,整份
遮。以前名单里 sys 库只认会话、锁等待那几张视图,号码原样交给模型、落库。"""
done = masked(sql, [column], [[TRX_INFO]], kind="mysql")
assert done.whole and done.payload["rows"] == [[MASKED_WORD]] and not _leaks(done)
@pytest.mark.parametrize("kind, sql, column, value", [
pytest.param("mysql", "SELECT sys.format_bytes(2048) AS b", "b", "2.00 KiB", id="sys 库里只排版的函数"),
pytest.param("mysql", "SELECT ps_total AS t FROM stats", "t", 12, id="ps_ 开头的列"),
pytest.param("oracle", "SELECT 1 AS n FROM sys.dual", "n", 1, id="Oracle 的 sys.dual"),
])
def test_other_sys_names_stay_visible(kind, sql, column, value):
"""sys 库里别的函数(只排版、换单位的)、不写 sys. 的 ps_ 开头的列名、Oracle 的 sys.dual:照常。"""
done = masked(sql, [column], [[value]], kind=kind)
assert not done.whole and done.payload["rows"] == [[value]]
@pytest.mark.parametrize("sql, columns", [
pytest.param("SHOW ENGINE INNODB STATUS", ["Type", "Name", "Status"], id="InnoDB 状态"),
pytest.param("show engine innodb status", ["Type", "Name", "Status"], id="小写"),
pytest.param("SHOW BINLOG EVENTS IN 'binlog.000002'", ["Log_name", "Pos", "Info"], id="binlog 事件"),
pytest.param("SHOW RELAYLOG EVENTS", ["Log_name", "Pos", "Info"], id="relay log 事件"),
pytest.param("SHOW REPLICA STATUS", ["Last_SQL_Error"], id="复制状态"),
pytest.param("SHOW SLAVE STATUS", ["Last_SQL_Error"], id="复制状态-旧写法"),
pytest.param("SHOW PROFILES", ["Query_ID", "Duration", "Query"], id="PROFILES"),
pytest.param("SHOW WARNINGS", ["Level", "Code", "Message"], id="WARNINGS"),
pytest.param("SHOW CREATE VIEW vip_sales", ["View", "Create View"], id="视图定义"),
# SHOW CREATE TABLE 对视图交回的是视图定义(MySQL 8.0 实测),建表语句里还有默认值、CHECK 约束、分区的取值
pytest.param("SHOW CREATE TABLE vip_sales", ["View", "Create View"], id="建表语句-视图"),
pytest.param("SHOW CREATE TABLE sales", ["Table", "Create Table"], id="建表语句"),
pytest.param("SHOW TRIGGERS", ["Trigger", "Statement"], id="触发器"),
pytest.param("SHOW EXPLAIN FOR 12", ["id", "Extra"], id="MariaDB 别的连接的计划"),
])
def test_show_statements_beyond_the_table_structure_mask_the_whole_result(sql, columns):
"""SHOW 里只交回表结构、库清单的几种以外(引擎状态、日志、复制、出错信息、视图和触发器的定义、建表语句……):交回的
是原文,可能带着别的会话 SQL 里的号码、定义里写死的号码,整份遮。以前守卫放行(SHOW 是只读的),遮罩认不出,号码
原样。"""
row = [BUSY] * len(columns)
done = masked(sql, columns, [row], kind="mysql")
assert done.whole and done.payload["rows"] == [[MASKED_WORD] * len(columns)]
@pytest.mark.parametrize("sql, columns, row", [
pytest.param("SHOW TABLES", ["Tables_in_shop"], ["sales"], id="表清单"),
pytest.param("SHOW FULL TABLES FROM shop", ["Tables_in_shop", "Table_type"], ["sales", "BASE TABLE"],
id="FULL TABLES"),
pytest.param("SHOW COLUMNS FROM sales", ["Field", "Type"], ["region", "varchar(20)"], id="列"),
pytest.param("SHOW FULL FIELDS FROM sales", ["Field", "Type"], ["region", "varchar(20)"], id="FULL FIELDS"),
pytest.param("SHOW INDEX FROM sales", ["Table", "Key_name"], ["sales", "PRIMARY"], id="索引"),
pytest.param("SHOW KEYS FROM sales", ["Table", "Key_name"], ["sales", "PRIMARY"], id="KEYS"),
pytest.param("SHOW DATABASES", ["Database"], ["shop"], id="库清单"),
pytest.param("SHOW CREATE DATABASE shop", ["Database", "Create Database"], ["shop", "CREATE DATABASE `shop`"],
id="建库语句"),
pytest.param("SHOW TABLE STATUS LIKE 'sales'", ["Name", "Engine", "Rows"], ["sales", "InnoDB", 3], id="表状态"),
pytest.param("DESCRIBE sales", ["Field", "Type"], ["region", "varchar(20)"], id="DESCRIBE 表"),
pytest.param("EXPLAIN sales", ["Field", "Type"], ["region", "varchar(20)"], id="EXPLAIN 表"),
pytest.param("DESC sales region", ["Field", "Type"], ["region", "varchar(20)"], id="DESC 表的一列"),
])
def test_show_and_describe_of_the_table_structure_stay_visible(sql, columns, row):
"""只交回表结构、库清单的 SHOW、DESCRIBE(核查 agent 探查表结构靠它们)照常。"""
done = masked(sql, columns, [list(row)], kind="mysql")
assert not done.whole and done.payload["rows"] == [list(row)]
@pytest.mark.parametrize("sql", [
"EXPLAIN FORMAT=TREE SELECT s.id FROM sales s JOIN people3 p ON p.id = s.id WHERE s.id = 1",
"EXPLAIN FORMAT = JSON SELECT id FROM sales WHERE id = 1",
"EXPLAIN ANALYZE SELECT id FROM sales WHERE id = 1",
"DESCRIBE SELECT id FROM sales WHERE id = 1",
"EXPLAIN FOR CONNECTION 12",
"EXPLAIN FORMAT=JSON FOR CONNECTION 12",
"EXPLAIN (SELECT id FROM sales)",
"EXPLAIN WITH t AS (SELECT id FROM sales) SELECT * FROM t",
"EXPLAIN EXTENDED SELECT id FROM sales",
])
def test_explaining_a_statement_masks_the_whole_result(sql):
"""EXPLAIN 一条语句:MySQL 优化时把只有一行的表读出来、把值当常量代进条件,FORMAT=TREE / JSON、EXPLAIN ANALYZE 写出的
条件里就是那一行的值;FOR CONNECTION 读的是别的连接正在执行的语句。整份遮。"""
done = masked(sql, ["EXPLAIN"], [[f"-> Filter: (s.buyer_phone = '{PHONE}')"]], kind="mysql")
assert done.whole and done.payload["rows"] == [[MASKED_WORD]]
@pytest.mark.parametrize("sql, columns, rows", [
("SELECT trx_query AS t FROM information_schema.INNODB_TRX", ["t"], [[BUSY]]),
("SHOW ENGINE INNODB STATUS", ["Type", "Name", "Status"],
[["InnoDB", "", f"---TRANSACTION 1234, ACTIVE 2 sec\nMySQL thread id 9, query id 70 localhost root\n{BUSY}"]]),
("SELECT sys.ps_thread_trx_info(81) AS t", ["t"], [[TRX_INFO]]),
], ids=["INNODB_TRX", "SHOW ENGINE INNODB STATUS", "sys.ps_thread_trx_info"])
async def test_other_sessions_sql_never_reaches_the_model_or_the_store(monkeypatch, fake_source, sql, columns, rows):
"""端到端(MySQL):另一个会话正在执行的 SQL 里写着号码,交给模型的、落库的、通用工件接口给的都只有「已遮罩」,
快照记着整份遮。以前 masked_at_query 为空,SQL 原文连同号码原样交出。"""
seen, snapshot, api = await _asked(monkeypatch, fake_source, "mysql", sql, columns, rows)
assert MASKED_WORD in seen and "1390000000" not in seen, seen
assert snapshot["masked_at_query"] == columns and snapshot["masked_whole"] is True
assert "1390000000" not in json.dumps(snapshot, ensure_ascii=False) and "1390000000" not in api
def test_the_new_views_are_on_the_list_the_bundle_shares():
"""可验证报告和工具边界共用一份名单:新补的视图两边都认(报告那边本来就整个挡 information_schema)。"""
from app.bundle import build
for sql in ("SELECT trx_query FROM information_schema.INNODB_TRX", "SELECT sql_text FROM dba_sqlset_statements",
"SELECT undo_sql FROM flashback_transaction_query", "SELECT sql_redo FROM v$logmnr_contents"):
assert build._INDIRECT_READS.search(sql) and masking._INDIRECT.search(sql), sql
@pytest.mark.parametrize("sql", [
"SELECT sys.ps_thread_trx_info(81) AS t",
"SELECT `sys`.`ps_thread_stack`(81, FALSE) AS t",
])
def test_the_sys_functions_are_on_the_list_the_bundle_shares(sql):
"""可验证报告和工具边界共用一份名单:sys.ps_ 开头的函数两边都认,报告不附带这样的快照(只给哈希),它的列整份算取自
遮罩列。以前报告判它没碰遮罩列,照常附带快照,别的会话 SQL 里的号码随文件交出去。"""
from app.bundle import build
snapshot = {"source": "shop", "sql": sql, "columns": ["t"], "rows": [[TRX_INFO]]}
assert build._INDIRECT_READS.search(sql) and masking._INDIRECT.search(sql)
assert build._untouched(snapshot, {"buyer_phone"}) is False
assert build._derived_columns(snapshot, {"buyer_phone"}) == {"buyer_phone", "t"}
+8 -4
View File
@@ -111,8 +111,9 @@ def test_a_whole_row_of_an_outer_query_is_found_from_a_nested_derived_table(sql)
"SELECT EXISTS(SELECT * FROM sales s2 WHERE s2.id = s.id) AS j FROM sales s",
])
def test_rows_and_stars_that_bring_no_masked_values_stay_visible(sql):
"""整行取自没有遮罩列的子查询、count(s.*) 只数个数:不算。"""
assert hidden_of(sql, ["j", "total"]) == set(MASKED)
"""整行取自没有遮罩列的子查询、count(s.*) 只数个数:不算。结果的列数照选出项数(列数对不上的整份遮,见
test_masking_opaque_sql)。"""
assert hidden_of(sql, ["j", "total"] if "total" in sql else ["j"]) == set(MASKED)
def test_a_column_named_like_its_table_is_a_column_where_the_schema_says_so():
@@ -188,8 +189,11 @@ def test_sql_that_cannot_be_read_but_has_a_star_or_a_table_statement_hides_every
"SELECT amount * 2 AS j FROM sales WHERE (",
])
def test_sql_that_cannot_be_read_with_only_a_count_or_a_product_shows_the_rest(sql):
"""只有 COUNT(*)、乘号:不会把遮罩列带出来,照旧只遮遮罩列本身。"""
assert hidden_of(sql, ["j"]) == set(MASKED)
"""只有 COUNT(*)、乘号:不会把遮罩列带出来,照旧只遮遮罩列本身(没有整行引用的方言)。Postgres 里解析不了的一律
整份遮:整行引用(row_to_json(s)、SELECT s)不写列名、也没有 *,只看文字认不出(见 test_masking_opaque_sql)。"""
assert hidden_of(sql, ["j"], kind="sqlite") == set(MASKED)
assert hidden_of(sql, ["j"], kind="mysql") == set(MASKED)
assert hidden_of(sql, ["j"]) == set(MASKED) | {"j"}
# --------------------------------------------------------------------------