mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:21:58 +00:00
K3b 复核修正(第 1 轮):数值格按位置认、(表达式).* 带外括号也整份遮、同名 CTE 按作用域认、认不全的写法名单合一、整份遮不再把每一格收成遮罩值、证据摘录分段遮、反方最后一道关只看正文
- 同名列(回退):column_types 按列名记,同名的几列只留得下后一列的类型,文字的备注被当成数值列,号码不再按值换、 原样交给模型并落库。现在只在列名唯一时按列类型认数值列(_typed_numbers),同名的按格子本身认:整数、小数才算数, 写成字符串的数按文字照换、记进 masked_cells;列名唯一、列类型是 number 的 NUMERIC / DECIMAL 字符串照原样 - (表达式).* 外面套了括号、起了别名(((p.card).*)、(p.card).* AS x,Postgres 实测照样展开):以前只遮第一列。选出项 先去掉外括号、别名再认(_unwrapped),(s).*、(s.*) AS x 照旧当 s.* - WITH 里和数据源里的表同名的一段(WITH sales AS (SELECT * FROM sales WHERE …)):不带 RECURSIVE 的一段看不见它自己和 排在后面的段(_scoped_cte),同名的按数据源里的表认;以前认成它自己,* 展开什么都不带出,整行引用的号码原样交出 - 认不全的写法:和可验证报告合成一份名单(masking 的 OPAQUE_SQL、INDIRECT_FUNCTIONS、INDIRECT_NAMES,bundle 引用后另加 系统 schema),补上 MySQL 会执行的注释、U&"…"、pg_stat_get_activity、逻辑解码、MySQL 的 column_statistics / general_log / PROCESSLIST / performance_schema、MariaDB 的 column_stats、Oracle 的直方图 / *_tab_columns / V$SQL 一族; 去掉注释再认一遍(函数名和括号之间夹注释);去掉 DuckDB、SQL Server 的函数(query 误伤同名 CTE、字符串);没有 * 的 一支选出项和结果列数对不上的、Postgres 解析不了的整份遮 - 整份遮只收确定取自遮罩列的那几列的值(_derived_masks 的 collect,报告快照收值同一规则):以前每一格都收,区域名、 列名在同一次核查里处处被换掉,封存的 SQL 也被改写。快照另记 masked_whole,格子引用照实说写法认不全 - 证据摘录分段遮(inquiry.ExcerptMask,judge.prepare 的 mask 取代 kinds):SQL 照封存 SQL 的办法换,范围阈值、LIMIT 照 原样,和遮罩列做等值、IN、LIKE 比较的字面量不论收没收到都换;格子逐格遮,数值格照原样;质询把被问的那句(连同表格) 当作已经摆在眼前,摘录、查询工具、回答里都不换 - 反方审稿:理由文档的目录列名照快照换(同质询的 _renamed_entry),agent 自己 SQL 里照原样留着的阈值、行数在正文里 不换(_shown),最后一道关只看正文和原文,不再把 agent 起的列名、阈值当成露了遮罩值整句改判 补测试:同名列一文一数、DECIMAL 字符串(含端到端引用取值),外括号 × 复合类型 / 记录函数 / 匿名 ROW × 最外层 / UNION 前后支 / 和 * 写在一起,同名 CTE 四种写法和 RECURSIVE 对照,十七种认不全的写法和三种不该误伤的写法,整份遮后的 收值与封存 SQL,Postgres 解析不了的整行引用,摘录的阈值、数值格、IN / NOT IN 字面量、被问那句(反方、质询各端到端), 反方别名里的阈值(经受住、被推翻各一条)、找某个值的别名、正文里的阈值。逐项去掉实现,对应的测试都变红。 改了文件所有权以外的 bundle/build.py:_OPAQUE_SQL、_INDIRECT_READS 改为引用 masking 的同一份名单(另加系统 schema, 覆盖面不小于原来)。
This commit is contained in:
@@ -39,6 +39,7 @@ from urllib.parse import quote
|
||||
from app.core.artifact_store import canonical_json, content_hash
|
||||
from app.core.events import EventType
|
||||
from app.engine.evidence import iter_units, query_entry_fields
|
||||
from app.engine.masking import OPAQUE_SQL, SYSTEM_SCHEMAS, indirect_reads
|
||||
from app.engine.merge_query import MERGE_SOURCE
|
||||
from app.engine.recompute import input_origin
|
||||
from app.engine.schema import NodeType
|
||||
@@ -231,43 +232,12 @@ _QUERY_KEYS = frozenset({"columns", "rows", "row_count", "truncated", "elapsed_m
|
||||
#: 导出时只知道数据源设的遮罩、不知道是哪种库:SQL 按每种支持的库的读法各解析一遍(None 是 sqlglot 的通用读法),
|
||||
#: 至少一种读得通,读得通的每一种都要过关
|
||||
_READINGS = (None, "sqlite", "mysql", "postgres", "oracle")
|
||||
#: 语法树和数据库实际执行的对不上的写法:MySQL、MariaDB 会执行的注释(/*! … */、/*M! … */,解析器当注释扔掉),
|
||||
#: Postgres 按 Unicode 转义写的名字(U&"…",原文里看不出是哪一列)
|
||||
_OPAQUE_SQL = re.compile(r'/\*m?!|u&"', re.I)
|
||||
#: 不点列名也读得出表里内容的写法。SQL 原文里出现就按碰了遮罩列算,写在字符串里的也算(宁可多挡)。函数要跟着左括号
|
||||
#: 才算(列名碰巧叫 crosstab 的不算),表、视图、schema 按名字认
|
||||
_INDIRECT_READS = re.compile(r"(?<![0-9A-Za-z_$])(?:(?:" + "|".join((
|
||||
# 表名、SQL 写在字符串里交给函数执行或整表导出:Postgres 的 XML 一族、ts_stat、tablefunc 的 crosstab / connectby、
|
||||
# dblink,Oracle 的 XMLTYPE、XQuery(XMLQUERY、XMLTABLE)
|
||||
r"(?:query|table|cursor|schema|database)_to_xml\w*", r"ts_stat", r"crosstab\d*", r"connectby", r"dblink\w*",
|
||||
r"xmltype", r"xmlquery", r"xmltable",
|
||||
# 按页、按文件、按日志读出的原始数据:Postgres 的 pageinspect(按索引名读 B 树页的 bt_page_items 也算)、服务器文件、
|
||||
# 逻辑解码、pg_walinspect 读出的 WAL 块,MySQL 的 LOAD_FILE,Oracle 的 BFILENAME,SQLite 的 readfile
|
||||
r"get_raw_page", r"bt_page_items", r"pg_read_(?:binary_)?file", r"pg_logical_slot_\w+", r"pg_get_wal_block_info",
|
||||
r"load_file", r"bfilename", r"readfile",
|
||||
# 别的会话执行过的 SQL 原文:Postgres 的 pg_stat_activity 背后的函数和按会话取的那一个,MySQL 企业版的审计日志
|
||||
r"pg_stat_get_(?:backend_)?activity", r"audit_log_read",
|
||||
)) + r")[\"`]?\s*\(|(?:" + "|".join((
|
||||
# Oracle 的 DBMS_XMLGEN 这一族(包名后面跟着点),XQuery 里的 ora:view 和 oradb: 表
|
||||
r"dbms_xml\w*", r"ora:view", r"oradb",
|
||||
# 统计信息里的采样值、全文索引里的词:Postgres 的 pg_stats、pg_statistic 一族,SQLite 的 sqlite_stat*,MySQL 的
|
||||
# column_statistics、innodb_ft_*,MariaDB 的 column_stats,Oracle 的直方图、列统计和记着最小最大值的 *_tab_columns,
|
||||
# 以及它们的底表 histgrm$、hist_head$
|
||||
r"pg_stat(?:s|istic)\w*", r"sqlite_stat\d", r"column_statistics", r"innodb_ft_\w+", r"column_stats",
|
||||
r"(?:all|dba|user|cdb)_(?:\w+_)?(?:histograms|col_statistics|tab_cols|tab_columns)", r"(?:histgrm|hist_head)\$",
|
||||
# 别的会话执行过的 SQL 原文(写入语句里就是值):Postgres 的 pg_stat_activity、pg_stat_statements,本连接预备过的
|
||||
# 语句(连接池里的连接留着前几次查询的原文);MySQL 的 general_log、slow_log,sys 库里建在 performance_schema 上的
|
||||
# 会话、锁等待视图(带 x$ 的是不排版的那一份;只认这几张,Oracle 常写的 sys.dual 不算);Oracle 的 V$SQL 一族、
|
||||
# 打开着的游标、审计记录
|
||||
r"pg_stat_activity", r"pg_stat_statements", r"pg_prepared_statements", r"general_log", r"slow_log",
|
||||
r"sys[\"`]?\s*\.\s*[\"`]?(?:x\$)?(?:session|processlist|innodb_lock_waits|schema_table_lock_waits)",
|
||||
r"g?v\$sql\w*", r"(?:dba|cdb)_hist_sql\w*", r"g?v\$open_cursor",
|
||||
r"(?:dba|cdb|user)_(?:\w+_)?audit_trail", r"unified_audit_trail", r"aud\$\w*", r"fga_log\$",
|
||||
# SQLite 按页读出的原始数据
|
||||
r"sqlite_dbpage",
|
||||
# 系统 schema:哪几张表采样了列值、记着别的会话的 SQL,一张张数靠不住,整个不认
|
||||
r"pg_catalog", r"information_schema", r"performance_schema",
|
||||
)) + r")(?![0-9A-Za-z_$]))", re.I)
|
||||
#: 语法树和数据库实际执行的对不上的写法(MySQL 会执行的注释、Postgres 的 U&"…"),不点列名也读得出表里内容的写法(表名、
|
||||
#: SQL 写在字符串里交给函数,按页按文件读的原始数据,统计信息、会话记录里存着值的):和核查 agent 的工具边界共用一份名单
|
||||
#: (engine/masking.py 的 OPAQUE_SQL、indirect_reads),这里另把系统 schema 整个算上(只决定附不附快照,宁可多挡)。SQL
|
||||
#: 原文里出现就按碰了遮罩列算,写在字符串里的也算
|
||||
_OPAQUE_SQL = OPAQUE_SQL
|
||||
_INDIRECT_READS = indirect_reads(*SYSTEM_SCHEMAS)
|
||||
#: Postgres 的函数记法(m.to_jsonb 就是 to_jsonb(m))下能把一整行交出来的内置函数(_whole_row):只收一个参数、收得下
|
||||
#: 整行(record、anyelement、"any" 这类),交回的值里带着整行。汇总函数也能这么写:any_value、*_agg 交回整行,max、min
|
||||
#: 交回最大、最小的那一整行。照 PG 18 的 pg_proc 逐个对过:只交回个数、大小、类型、哈希的不算,这么写会报错的(伪类型的
|
||||
|
||||
+124
-29
@@ -8,7 +8,8 @@
|
||||
(同按需裁判);设置里关了「审批质询 · 启用」一律不答。同一次运行同时只答一条:进程内登记(_ANSWERING),登记期间
|
||||
这次运行的审批回复、继续运行也不受理(api/runs.py),免得运行往下走了、问答还在路上
|
||||
2. 回答(_answer):交给 agent 被问的那句原文、它挂的证据摘录(照 judge.prepare 的办法取,原报告的编号标明「原报告」;
|
||||
摘录先遮:遮罩列不给,改了名字的遮罩列按 SQL 认出来也不给,别的列、SQL 里写着的遮罩值按值换掉,见 _excerpts)、
|
||||
摘录先遮,和工具边界同一套口径:遮罩列、改了名字的遮罩列按 SQL 认出来都不给,文字格、SQL 的字符串和注释里的遮罩值
|
||||
按值换,和遮罩列做等值比较的字面量都换,数值格、范围比较的阈值、LIMIT 照原样,见 _excerpts、ExcerptMask)、
|
||||
审批人选中的原文和问题;工具是这次运行用过的查询工具对应的数据源(同名重建的、没固定过版本的
|
||||
上传表格不算,见 _usable_sources);上限按设置(每次的查询次数、时长,全局每日金额)。回答先给结论,数字只能引用 agent
|
||||
自己查询的格子,或者报告里已有的指标、运行输入;回答文档由 compose_doc 组装,裸数字被标出来(提问人自己写的数除外)。
|
||||
@@ -51,6 +52,8 @@
|
||||
数(访问次数正好等于某一笔金额)也遮掉,这是产品取舍,不按上下文去猜;工具边界不按值换数值格(存下的是证据),交给
|
||||
agent 的、存下的查询快照里这样的数照原样,只在回答和质询工件里遮。反过来,没设遮罩的列里存成数的号码(别的库里的
|
||||
联系方式表把手机号存成整数)工具边界也不按值换,agent 看得到:要遮就在那个数据源上把那一列也设成遮罩。存成文字的照换。
|
||||
数值格按列类型认(Postgres 的 NUMERIC、MySQL 的 DECIMAL 交回来是字符串),列类型按列名记,结果里同名的几列说不清是
|
||||
哪一列的类型,按格子本身认:整数、小数才算数,写成字符串的数按文字照换(多遮,换过的记进 masked_cells)。
|
||||
占位的写法(未知、暂无……,masking.PLACEHOLDERS)不当遮罩值
|
||||
- 遮罩数值列的 SUM / AVG / MIN / MAX 算派生自遮罩列,整列遮(COUNT 不算,见 _value_columns),CASE 里有一支取的是
|
||||
遮罩列的也算:MIN / MAX 就是某一行的原值,只有一行的分组 SUM / AVG 也是,按列认不出哪些汇总值安全,宁可多遮。
|
||||
@@ -75,14 +78,25 @@
|
||||
Oracle 的 JSON_OBJECT(*))按 * 展开的来源算:取自子查询、CTE 的看它选出的列,取自数据源里的表的一律算取自遮罩列(整列
|
||||
遮)。Postgres 先按列认:表结构里查得到和表同名的列(visits 表的 visits 列)才当列,查不到时按整行算(多遮);LATERAL
|
||||
里只看排在它前面的来源,它自己的结果列不算。PIVOT、UNPIVOT、MATCH_RECOGNIZE 读到遮罩列的,整份快照的列都遮(多遮)。
|
||||
SQL 解析不了的只看文字:写着遮罩列、*(COUNT(*) 除外)、TABLE x、下面那几类函数和视图的整份遮,整行引用这类不写列名的
|
||||
写法认不出
|
||||
SQL 解析不了的:Postgres 的一律整份遮(整行引用不写列名、也没有 *,只看文字认不出);别的库只看文字,写着遮罩列、
|
||||
*(COUNT(*) 除外)、TABLE x、下面那几类写法的整份遮
|
||||
- WITH 里和数据源里的表同名的一段(WITH sales AS (SELECT * FROM sales WHERE …)):不带 RECURSIVE 的 WITH 里一段只看得见
|
||||
排在它前面的段,里面的 sales 是数据源里的表(按表算,整行引用、* 展开的照遮);SQLite 里一段看得见排在后面的段,这里
|
||||
按看不见算(当成数据源里的表,多遮)
|
||||
- 认不全的写法整份遮(多遮,同一份里没碰遮罩列的编号、金额也遮):(表达式).* 展开读到遮罩列的(复合类型的列
|
||||
(p.profile).*、函数交回的记录 (jsonb_each_text(pj)).*、子查询里的 ROW (q.r).*,展开成几列、各叫什么 SQL 里看不出);
|
||||
表名、SQL 写在字符串里交给函数执行、整表导出的(query_to_xml、table_to_xml、cursor_to_xml、schema_to_xml、
|
||||
database_to_xml 这一族,dblink、crosstab、ts_stat,Oracle 的 DBMS_XMLGEN、XMLTYPE,DuckDB 的 query,SQL Server 的
|
||||
OPENQUERY)、按页按文件读原始数据的、统计信息和会话记录里存着值的(pg_stats、sqlite_stat4、pg_stat_statements),
|
||||
按名字认(masking._INDIRECT),自己建的同类函数认不出;分析出了意外、一长串 CTE 超出工作量上限(ANALYSIS_STEPS)的
|
||||
(p.profile).*、函数交回的记录 (jsonb_each_text(pj)).*、子查询里的 ROW (q.r).*,外面套了括号、起了别名的也算,展开成
|
||||
几列、各叫什么 SQL 里看不出);语法树和实际执行的对不上的(MySQL、MariaDB 会执行的注释 /*! … */,Postgres 的
|
||||
U&"…" 名字,没有 * 的一支选出项和结果的列数对不上);表名、SQL 写在字符串里交给函数执行、整表导出的(query_to_xml、
|
||||
table_to_xml、cursor_to_xml、schema_to_xml、database_to_xml 这一族,dblink、crosstab、ts_stat,Oracle 的
|
||||
DBMS_XMLGEN、XMLTYPE)、按页按文件按日志读原始数据的(pageinspect、逻辑解码、LOAD_FILE)、统计信息和会话记录里存着值
|
||||
的(Postgres 的 pg_stats、pg_stat_statements、pg_stat_activity;SQLite 的 sqlite_stat4;MySQL 的 column_statistics、
|
||||
general_log、PROCESSLIST、performance_schema;MariaDB 的 column_stats;Oracle 的直方图、记着最小最大值的
|
||||
*_tab_columns、V$SQL 一族、审计记录),按名字认(masking 的 INDIRECT_FUNCTIONS、INDIRECT_NAMES,可验证报告用的同一份
|
||||
名单),去掉注释再认一遍(函数名和括号之间夹着注释),自己建的同类函数、视图认不出;information_schema、pg_catalog 里
|
||||
只有表结构的视图不算(核查 agent 查列名、类型常用)。只收支持的几种库的:DuckDB、SQL Server 的同类函数不收;分析出了
|
||||
意外、一长串 CTE 超出工作量上限(ANALYSIS_STEPS)的。整份遮的只收确定取自遮罩列的那几列的值当遮罩值(别的列的值
|
||||
本来就没交给模型,收了会把区域名、列名在同一次核查里处处换掉);快照记下整份遮(masked_whole),格子引用指到它时照实
|
||||
说写法认不全
|
||||
- 按遮罩列分档的标签(CASE、IIF 的结果全是写死的常量):标签文字是 agent 写在 SQL 里的,不当遮罩值,SQL、回答正文里
|
||||
照原样;标签列当分组键(每一档一行)照常显示,分档写在子查询、WITH 里、外层(或中间一层)按它分组的也算
|
||||
(masking._grouped_labels),不分组的逐行标签列整列遮(格子里看不出某一笔落在哪一档),可 agent
|
||||
@@ -127,9 +141,12 @@
|
||||
点、括号,含全角),带 +86 / 0086 / 86 前缀、全角数字:写成汉字(一三九……)、中间夹着别的字、拆成几处写的、跨行
|
||||
写的认不出;和别的数字只隔着一个连字符或点连着的(13900000001-2、19914.74)不算号码;SQL 里不带引号的数不这样认
|
||||
(199-1474-2785 在 SQL 里是三个数相减),字符串、带引号的名字、注释里照样认
|
||||
- 质询 agent 的任务里的证据摘录先遮(_excerpts):遮罩列、按 SQL 认出改了名字的遮罩列(和工具边界同一套规则)不给,
|
||||
别的列、SQL 里写着的遮罩值按值换(报告快照里的值),按值认的门槛同上:短的数(年龄、等级)写在 SQL 的条件里、别的列
|
||||
里照原样。没指明是哪一句时任务里给的是报告正文,正文里写着的值照原样(读者本来就看得到)
|
||||
- 质询 agent 的任务里的证据摘录先遮(_excerpts、ExcerptMask,和工具边界同一套口径):遮罩列、按 SQL 认出改了名字的
|
||||
遮罩列不给;文字格、SQL 的字符串和注释、列名里的遮罩值按值换(报告快照里的值,门槛同上);SQL 里和遮罩列做等值、
|
||||
IN、LIKE 比较的字面量不论认不认得出都换(报告没选出遮罩列时快照里收不到这些值);数值格、范围比较的阈值、LIMIT 照
|
||||
原样,没设遮罩的数值格恰好等于某个遮罩值也照原样(同工具边界不改写数值证据)。被问的那句(在表格里的连同整张表)、
|
||||
问题、报告列出的指标和运行输入里写着的值不换,回答里复述也不换。没指明是哪一句时任务里给的是报告正文,正文里写着
|
||||
的值照原样(读者本来就看得到)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -165,10 +182,11 @@ from app.engine.masking import ( # noqa: F401
|
||||
SECRETS_MAX, SECRET_DIGITS, SECRET_MIN, _ABSENT, _DIGITS, _EXPONENT_MAX, _LabelMemo, _NUMERIC, _PLAIN_NUMBER,
|
||||
_PREFIX, _Secrets, _WORD, _WORDS, _all_labels, _arithmetic, _band_only, _block_label, _block_label_of, _blocks,
|
||||
_branch_results, _branches, _canonical, _constant, _constants, _decimal, _derived_block, _derived_hidden,
|
||||
_digits_of, _fixed, _forms, _from_sources, _is_star, _item_label, _label_columns, _leaves, _literal_types,
|
||||
_masked_view, _masked_word, _mentions, _merging, _numeric, _output_name, _pattern, _places, _plain_text,
|
||||
_point_text, _query_shaped, _result_label, _scrubbed_source, _snapshot_labels, _source_label, _source_names,
|
||||
_star_label, _strong, _tainted, _value_columns, _worked_out, replay_edits, view_edits,
|
||||
_derived_masks, _digits_of, _fixed, _forms, _from_sources, _is_star, _item_label, _label_columns, _leaves,
|
||||
_literal_types, _masked_view, _masked_word, _mentions, _merging, _numeric, _output_name, _pattern, _places,
|
||||
_plain_text, _point_text, _query_shaped, _result_label, _schema_columns, _scrubbed_source, _snapshot_labels,
|
||||
_source_label, _source_names, _star_label, _strong, _tainted, _value_columns, _worked_out, masked_cell,
|
||||
replay_edits, view_edits,
|
||||
)
|
||||
from app.engine.merge_query import MERGE_SOURCE
|
||||
from app.engine.nodes.challenge import mark_original, with_note
|
||||
@@ -584,13 +602,12 @@ def _table_text(block: dict[str, Any], markdown: str) -> str:
|
||||
|
||||
|
||||
def _excerpts(doc: dict[str, Any], catalog: dict[str, Any], cand: judging.Candidate, loader: Callable[[str], Any],
|
||||
masked: dict[str, Any], kinds: dict[str, Any] | None = None, secrets: _Secrets | None = None
|
||||
) -> list[str]:
|
||||
masked: dict[str, Any], mask: ExcerptMask | None = None) -> list[str]:
|
||||
"""这句挂的每件证据的摘录:和 judge.prepare 一样逐件摘(带期 4 标记的文档多来历行),指向原报告编号的写法标上
|
||||
「原报告」——agent 自己的查询也编 Q1、Q2…,不分开的话会把它们混成一回事。
|
||||
摘录先遮,模型从任务里也看不到遮罩值:数据源遮罩的列不给,改了名字的(buyer_phone AS 手机号)按 SQL 认出来也不给
|
||||
(kinds:{数据源名: 类型},和工具边界同一套规则,judge._derived);别的列、SQL 里写着的遮罩值按值换成「已遮罩」,
|
||||
换了写法的号码也认(secrets:报告快照里的遮罩值,同交给查询工具的那一份)。解析 SQL、按值换是纯计算,调用方放进线程。"""
|
||||
摘录先遮,模型从任务里也看不到遮罩值(mask,见 ExcerptMask):数据源遮罩的列、按 SQL 认出改了名字的
|
||||
(buyer_phone AS 手机号)不给;文字格、SQL 里的字符串和注释里的遮罩值按值换成「已遮罩」,换了写法的号码也认,和遮罩列
|
||||
做等值比较的字面量都换;数值格、范围比较的阈值、LIMIT 照原样。解析 SQL、按值换是纯计算,调用方放进线程。"""
|
||||
fetch = judging._Fetch(loader)
|
||||
provenance_on = doc.get("provenance") == DOC_PROVENANCE
|
||||
queries = {a for a, e in catalog.items() if isinstance(e, dict) and e.get("kind") == "query"}
|
||||
@@ -598,9 +615,7 @@ def _excerpts(doc: dict[str, Any], catalog: dict[str, Any], cand: judging.Candid
|
||||
for alias in cand.aliases:
|
||||
entry = catalog.get(alias)
|
||||
if isinstance(entry, dict) and (text := judging._excerpt(alias, entry, [cand], fetch, masked, catalog,
|
||||
provenance_on=provenance_on, kinds=kinds)):
|
||||
if secrets is not None and secrets.strong:
|
||||
text = secrets.scrub(text, loose=True)
|
||||
provenance_on=provenance_on, mask=mask)):
|
||||
out.append(mark_original(text, alias, queries))
|
||||
return out
|
||||
|
||||
@@ -1479,6 +1494,78 @@ def _scrub_sql(sql: str, secrets: _Secrets, compared: set[str], kind: str | None
|
||||
raw, tokens[i].text, kinds[i], uses[i], secrets, compared)[0] if i in uses else raw)
|
||||
|
||||
|
||||
def _scrub_excerpt_sql(sql: str, secrets: _Secrets, compared: set[str], kind: str | None) -> str:
|
||||
"""交给核查 agent 的证据摘录里的 SQL(报告取数的那条):照封存 SQL 的办法换(_scrub_sql:字符串、注释、名字里的遮罩值
|
||||
换,范围比较的边界、LIMIT 的行数照原样,别处够长的遮罩值换),另外和遮罩列(compared:_derived_hidden 认出的名字,
|
||||
含改了名字的)做等值、IN、LIKE 一类比较的字面量,不论认不认得出是遮罩值都换成「已遮罩」:报告没选出遮罩列时快照里
|
||||
收不到这些值(WHERE buyer_phone NOT IN ('13900000002', …)),摘录又不是封存的 SQL,可以多遮。同一个数写成阈值、行数的
|
||||
也换(同 _literals)。已经摆在读者眼前的值(问题、被问的那句、这句结论原文里写着的,_Secrets.spared)照旧不换。
|
||||
以前对渲染好的整段摘录按值换,范围比较的阈值(amount >= 20000)也成了「已遮罩」,agent 看不到报告按什么取数。
|
||||
分词出错的整条按值换(同 _scrub_sql)。"""
|
||||
if not sql or not (secrets.strong or secrets.weak or compared):
|
||||
return sql
|
||||
split = _literals(sql, kind, secrets, compared)
|
||||
if split is None:
|
||||
return secrets.scrub(sql)
|
||||
tokens, kinds, uses = split
|
||||
word = _masked_word()
|
||||
pinned = {i for i, (how, columns, _) in uses.items()
|
||||
if how == "equal" and columns & compared and not secrets.spared(tokens[i].text)}
|
||||
numbers = _number_forms(v for i in pinned for v in (tokens[i].text, *uses[i][2]))
|
||||
|
||||
def literal(i: int, raw: str) -> str:
|
||||
if i not in uses:
|
||||
return raw
|
||||
if i in pinned or (kinds[i] == "NUMBER" and uses[i][0] in _KEPT and _digits_of(tokens[i].text) in numbers):
|
||||
return _requote(raw, lambda _: word)
|
||||
return _swap_literal(raw, tokens[i].text, kinds[i], uses[i], secrets, compared)[0]
|
||||
|
||||
return _rebuild(sql, tokens, kinds, secrets, literal)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExcerptMask:
|
||||
"""交给核查 agent(反方审稿、审批质询)的证据摘录怎么遮:judge.prepare 的 mask,judge 只按这几个方法用它。口径和工具
|
||||
边界一样,模型从任务里也看不到遮罩值,又不改写没设遮罩的数值证据:
|
||||
- columns:哪几列不给。按列名的,加上按 SQL 认出改了名字的(buyer_phone AS 手机号,masking._derived_masks,表结构按
|
||||
快照冻结的那份);认的过程出了意外,整份的列都不给
|
||||
- cell:格子逐格遮(masking.masked_cell):数值格照原样,文字格按值换(整格的、夹着的那一截,换了写法的号码也认)
|
||||
- sql:SQL 照 _scrub_excerpt_sql 换:范围比较的边界、LIMIT 照原样,和遮罩列做等值比较的字面量都换
|
||||
- text:别的文字(列名、检索片段、来历行)按值换,换了写法的号码也认,同 SQL 里的名字
|
||||
报告里列出的指标值、原式、运行输入的值不经它(报告本来就摆在读者眼前,同 _ref_values 免遮)。以前先渲染好整段摘录、
|
||||
再对整段按值换,分不出格子和 SQL 里的用法:阈值、没设遮罩的访问次数正好等于某一笔被遮的金额,都成了「已遮罩」。
|
||||
kinds:{数据源名: 类型}(SQL 按它的方言解析);secrets:按值换的值(报告快照里的遮罩值去掉已经摆在读者眼前的,同交给
|
||||
查询工具的那一份),没有为 None(只按列遮、只换和遮罩列做等值比较的字面量)。"""
|
||||
|
||||
kinds: dict[str, Any]
|
||||
secrets: _Secrets | None = None
|
||||
|
||||
def _kind(self, snapshot: dict[str, Any]) -> str | None:
|
||||
return self.kinds.get(str(snapshot.get("source") or ""))
|
||||
|
||||
def columns(self, snapshot: dict[str, Any], hidden: set[str], schema: Any) -> tuple[set[str], set[str]]:
|
||||
"""(不给的列, SQL 里指代遮罩列的名字),都是小写。schema 是快照冻结的表结构快照(经 judge 的 fetch 取),没有为
|
||||
None(按快照记着的认)。"""
|
||||
if not hidden:
|
||||
return set(hidden), set(hidden)
|
||||
try:
|
||||
tables = _schema_columns(schema) if isinstance(schema, dict) else None
|
||||
found = _derived_masks(dict(snapshot), set(hidden), self._kind(snapshot), tables)
|
||||
return set(hidden) | found.hidden, found.names
|
||||
except Exception: # noqa: BLE001 - 认不准就整份不给:摘录少几列,不让遮罩值交给核查 agent
|
||||
logger.warning("按 SQL 认摘录里改了名字的遮罩列时出错,整份快照的列都按遮罩处理", exc_info=True)
|
||||
return set(hidden) | {str(c).lower() for c in snapshot.get("columns") or []}, set(hidden)
|
||||
|
||||
def cell(self, value: Any, *, numeric: bool) -> Any:
|
||||
return masked_cell(value, self.secrets, numeric=numeric)
|
||||
|
||||
def sql(self, sql: str, snapshot: dict[str, Any], names: set[str]) -> str:
|
||||
return _scrub_excerpt_sql(sql, self.secrets or _Secrets(), set(names), self._kind(snapshot))
|
||||
|
||||
def text(self, text: str) -> str:
|
||||
return self.secrets.scrub(text, loose=True) if self.secrets is not None and self.secrets.strong else text
|
||||
|
||||
|
||||
def _rebuild(sql: str, tokens: list[Any], kinds: list[str], secrets: _Secrets, literal: Callable[[int, str], str]
|
||||
) -> str:
|
||||
"""按词重写一段 SQL:字面量交给 literal(第几个词、原文)定换成什么,名字(带不带引号)、注释和别的字按值换。
|
||||
@@ -1689,8 +1776,10 @@ def _settle_plan(own: dict[str, dict[str, Any]], named: dict[str, tuple[set[str]
|
||||
tainted: dict[str, set[str]] = {}
|
||||
for artifact, snapshot in own.items():
|
||||
kind, base = kinds.get(str(snapshot.get("source") or "")), named[artifact][0]
|
||||
hidden[artifact], tainted[artifact] = _derived_hidden(snapshot, base, kind)
|
||||
secrets.add(snapshot, hidden[artifact], labels=_snapshot_labels(snapshot, kind), base=base)
|
||||
found = _derived_masks(snapshot, base, kind)
|
||||
hidden[artifact], tainted[artifact] = found.hidden, found.names
|
||||
# 收值只收确定取自遮罩列的那几列(整份遮的不是每一列都收,见 masking._derived_masks)
|
||||
secrets.add(snapshot, found.collect, labels=_snapshot_labels(snapshot, kind), base=base)
|
||||
# agent 查询时遮掉的值先收(报告快照里的值收满了上限 SECRETS_MAX,也不丢 agent 自己查到的);不带它们的一份另算,
|
||||
# 比出事后取质询工件时认不出、要照位置重放的地方
|
||||
baseline = secrets.copy() if seen is not None and seen.strong else None
|
||||
@@ -1715,10 +1804,11 @@ def _settle_plan(own: dict[str, dict[str, Any]], named: dict[str, tuple[set[str]
|
||||
|
||||
|
||||
def _add_based(secrets: _Secrets, based: list[tuple[dict[str, Any], set[str]]], kinds: dict[str, str | None]) -> None:
|
||||
"""收报告自己的查询快照里遮罩列的值(based:快照和它要遮的列),按 SQL 认出改了名字的遮罩列(_derived_hidden)。"""
|
||||
"""收报告自己的查询快照里遮罩列的值(based:快照和它要遮的列),按 SQL 认出改了名字的遮罩列(masking._derived_masks)。
|
||||
写法认不全、整份遮的快照只收确定取自遮罩列的那几列:以前每一格都收,区域名、列名在这次核查里处处被换掉。"""
|
||||
for snapshot, base in based:
|
||||
kind = kinds.get(str(snapshot.get("source") or ""))
|
||||
secrets.add(snapshot, _derived_hidden(snapshot, base, kind)[0], labels=_snapshot_labels(snapshot, kind),
|
||||
secrets.add(snapshot, _derived_masks(snapshot, base, kind).collect, labels=_snapshot_labels(snapshot, kind),
|
||||
base=base)
|
||||
|
||||
|
||||
@@ -1827,15 +1917,20 @@ async def _answer(op: _Opening, ask: Ask, target: _Target | None, settings: dict
|
||||
async with SessionLocal() as session:
|
||||
spec = await verify_model_spec(session, None, settings=settings)
|
||||
# 报告自己的查询快照:被问的那句可能直接写着遮罩列的值,agent 会照着复述;里面的遮罩值在派 agent 之前就交给它的
|
||||
# 查询工具(masked_values),agent 查到写着同一个号码的备注、别的表时,交给模型之前就换掉;证据摘录也按它们先遮
|
||||
# 查询工具(masked_values),agent 查到写着同一个号码的备注、别的表时,交给模型之前就换掉;证据摘录也按它们先遮。
|
||||
# 被问的那句(在表格里的连同整张表)就写在任务开头、摆在审批人眼前,里面的值同问题一样哪里都不换(同反方审稿的
|
||||
# _spared):以前不算,这句引用的那一格在摘录里成了「已遮罩」,和任务开头写着的原句对不上
|
||||
given = [ask.question, ask.quote, *_ref_values(refs)]
|
||||
if target is not None:
|
||||
given += [target.cand.text, target.table]
|
||||
others = [snapshot for entry in catalog.values() if isinstance(entry, dict) and entry.get("kind") == "query"
|
||||
and _query_shaped(snapshot := _quiet_load(op.loader, entry.get("artifact")))]
|
||||
seed = await snapshot_secrets(others, given)
|
||||
excerpts: list[str] = []
|
||||
if target is not None:
|
||||
kinds = await _source_kinds([*masked, *(str(s.get("source") or "") for s in others)])
|
||||
excerpts = await asyncio.to_thread(_excerpts, doc, catalog, target.cand, op.loader, masked, kinds, seed)
|
||||
excerpts = await asyncio.to_thread(_excerpts, doc, catalog, target.cand, op.loader, masked,
|
||||
ExcerptMask(kinds, seed))
|
||||
outcome = await run_verify_agent(
|
||||
spec=spec, system=INQUIRY_SYSTEM, task=_task(op, ask, target, excerpts, refs, live, lost), sources=list(live),
|
||||
output_schema=INQUIRY_SCHEMA, max_queries=part.get("max_queries"), timeout_s=part.get("timeout_s"),
|
||||
|
||||
+60
-41
@@ -342,23 +342,22 @@ def _hidden(snapshot: dict[str, Any], source: Any, masked: dict[str, Any] | None
|
||||
return names
|
||||
|
||||
|
||||
def _derived(snapshot: dict[str, Any], hidden: set[str], fetch: _Fetch, kinds: dict[str, Any] | None) -> set[str]:
|
||||
"""核查 agent 的摘录(kinds 给了:{数据源名: 类型})要遮的列:按列名的(hidden),加上按 SQL 认出改了名字的遮罩列
|
||||
(buyer_phone AS 手机号、套函数、经子查询转一道……),和核查 agent 的查询在工具边界遮的是同一套规则
|
||||
(masking._derived_hidden,表结构按快照冻结的那份、经 fetch 取)。认的过程出了意外,整份快照的列都算(宁可多遮)。
|
||||
kinds 为 None(结论句裁判自己的摘录)照旧只按列名,摘录、复用键都和以前一样。"""
|
||||
if kinds is None or not hidden:
|
||||
return hidden
|
||||
from app.engine.masking import _derived_hidden, _schema_columns # masking 导入了别的引擎模块:用到时再导入
|
||||
def _derived(snapshot: dict[str, Any], hidden: set[str], fetch: _Fetch, mask: Any) -> tuple[set[str], set[str]]:
|
||||
"""(要遮的列, SQL 里指代遮罩列的名字),都是小写。核查 agent 的摘录(mask 给了,engine/inquiry.py 的 ExcerptMask)另按
|
||||
SQL 认出改了名字的遮罩列(buyer_phone AS 手机号、套函数、经子查询转一道……),和核查 agent 的查询在工具边界遮的是同一
|
||||
套规则(表结构按快照冻结的那份、经 fetch 取)。mask 为 None(结论句裁判自己的摘录)照旧只按列名,摘录、复用键都和
|
||||
以前一样。"""
|
||||
if mask is None or not hidden:
|
||||
return hidden, hidden
|
||||
schema = fetch(snapshot.get("schema_artifact")) if snapshot.get("schema_artifact") else None
|
||||
return mask.columns(snapshot, hidden, schema)
|
||||
|
||||
try:
|
||||
schema = fetch(snapshot.get("schema_artifact")) if snapshot.get("schema_artifact") else None
|
||||
tables = _schema_columns(schema) if isinstance(schema, dict) else None
|
||||
kind = kinds.get(str(snapshot.get("source") or ""))
|
||||
return set(hidden) | _derived_hidden(dict(snapshot), set(hidden), kind, tables)[0]
|
||||
except Exception: # noqa: BLE001 - 认不准就整份遮:摘录少几列,不让遮罩值交给核查 agent
|
||||
logger.warning("judge: 按 SQL 认摘录里改了名字的遮罩列时出错,整份快照的列都按遮罩处理", exc_info=True)
|
||||
return set(hidden) | {str(c).lower() for c in snapshot.get("columns") or []}
|
||||
|
||||
def _unique_number(snapshot: dict[str, Any], column: str) -> bool:
|
||||
"""这一列按列类型是数值列:column_types 记成 number,列名在结果里只有这一列(同名的几列说不清是哪一列的类型,
|
||||
masking._typed_numbers)。"""
|
||||
columns = [str(c) for c in snapshot.get("columns") or []]
|
||||
return column_kind(snapshot, column) == "number" and columns.count(column) == 1
|
||||
|
||||
|
||||
def _shown_columns(columns: list[str], cited: Iterable[str]) -> list[str]:
|
||||
@@ -374,12 +373,13 @@ def _shown_columns(columns: list[str], cited: Iterable[str]) -> list[str]:
|
||||
|
||||
def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iterable[str], whole: bool,
|
||||
fetch: _Fetch, masked: dict[str, Any] | None, *, provenance_on: bool = False,
|
||||
kinds: dict[str, Any] | None = None) -> str | None:
|
||||
mask: Any = None) -> str | None:
|
||||
"""一条查询的摘录:头、SQL、(期 4)来历行、列、被引用的行。
|
||||
|
||||
provenance_on:文档带期 4 标记时为真(prepare 算)。这时上传表格的查询快照(有 data_version)在 SQL 之后、「列:」
|
||||
之前多几行来历(_provenance_lines);其余情况一个字都不多,期 4 之前的文档、手工源的摘录和以前逐字相同。
|
||||
kinds:给核查 agent 的摘录才给,改了名字的遮罩列也不给(_derived)。
|
||||
mask:给核查 agent 的摘录才给(engine/inquiry.py 的 ExcerptMask),分段遮:改了名字的遮罩列也不给(_derived);SQL
|
||||
先遮再截(截在半截的值认不出);列名、来历行按值换;格子逐格遮,数值格照原样。
|
||||
"""
|
||||
snapshot = fetch(entry.get("artifact"))
|
||||
if not isinstance(snapshot, dict) or not isinstance(snapshot.get("rows"), list):
|
||||
@@ -389,13 +389,15 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter
|
||||
source = entry.get("source") or snapshot.get("source")
|
||||
head = (f"【{alias}】合并查询结果(共 {total} 行)" if snapshot.get("source") == MERGE_SOURCE
|
||||
else f"【{alias}】查询结果({entry.get('tool') or '查询'} · 数据源 {source or '—'} · 共 {total} 行)")
|
||||
hidden, names = _derived(snapshot, _hidden(snapshot, source, masked), fetch, mask)
|
||||
sql = _clean(str(snapshot.get("sql") or ""))
|
||||
if mask is not None:
|
||||
sql = mask.sql(sql, snapshot, names)
|
||||
lines = [head, f"SQL:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"]
|
||||
if snapshot.get("source") == MERGE_SOURCE:
|
||||
lines.append(_merge_line(snapshot))
|
||||
hidden = _derived(snapshot, _hidden(snapshot, source, masked), fetch, kinds)
|
||||
if provenance_on and snapshot.get("data_version"):
|
||||
lines += _provenance_lines(snapshot, fetch, hidden)
|
||||
lines += [mask.text(line) if mask is not None else line for line in _provenance_lines(snapshot, fetch, hidden)]
|
||||
columns = [str(c) for c in snapshot.get("columns") or [] if str(c).lower() not in hidden]
|
||||
shown_cols = _shown_columns(columns, cols)
|
||||
# 点名引用的行先给,整份引用的再从头补满,一共不超过 EXCERPT_ROWS 行
|
||||
@@ -403,13 +405,16 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter
|
||||
if whole:
|
||||
picked += [r for r in range(total) if r not in picked][:EXCERPT_ROWS - len(picked)]
|
||||
omitted = len(columns) - len(shown_cols)
|
||||
lines.append("列:" + " | ".join(shown_cols) + (f" | …另有 {omitted} 列" if omitted > 0 else ""))
|
||||
heads = [mask.text(c) for c in shown_cols] if mask is not None else shown_cols
|
||||
lines.append("列:" + " | ".join(heads) + (f" | …另有 {omitted} 列" if omitted > 0 else ""))
|
||||
for r in sorted(picked):
|
||||
cells = []
|
||||
for c in shown_cols:
|
||||
try:
|
||||
raw, _ = locate_cell(snapshot, r, c)
|
||||
cells.append(render_cell(raw, kind=column_kind(snapshot, c)))
|
||||
shown = mask.cell(raw, numeric=_unique_number(snapshot, c)) if mask is not None else raw
|
||||
# 按值换过的格子照文字给:「已遮罩」不能再按这一列的类型当数渲染
|
||||
cells.append(render_cell(raw, kind=column_kind(snapshot, c)) if shown is raw else str(shown))
|
||||
except (CellError, RenderError):
|
||||
cells.append(MISSING)
|
||||
lines.append(f"r{r}:" + " | ".join(cells))
|
||||
@@ -504,11 +509,12 @@ def _cell_words(locator: dict[str, Any], hidden: set[str]) -> str:
|
||||
|
||||
|
||||
def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any],
|
||||
masked: dict[str, Any] | None, kinds: dict[str, Any] | None = None) -> list[str]:
|
||||
masked: dict[str, Any] | None, mask: Any = None) -> list[str]:
|
||||
"""指标的输入取自哪次查询、哪一格,再附上那次查询的 SQL(截到 SQL_CHARS,最多 ENTITY_QUERIES 次)。
|
||||
|
||||
「活跃用户按会员号去重」这种讲口径的句子,证据在取数的 SQL 里;只给原式,裁判只能判证据不足。
|
||||
位置来自封存的口径卡;SQL 经 fetch 取,取不到(不在封存范围里)的不给。没追到查询的输入照实写。
|
||||
mask 同 _query_excerpt:改了名字的遮罩列不写列名,SQL 先遮再截。
|
||||
"""
|
||||
items = [i for i in metric.get("inputs") or [] if isinstance(i, dict)]
|
||||
lines: list[str] = []
|
||||
@@ -525,11 +531,13 @@ def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any
|
||||
snapshot = fetch(artifact)
|
||||
snap = snapshot if isinstance(snapshot, dict) else {}
|
||||
source = (catalog.get(alias) or {}).get("source") if alias else None
|
||||
hidden = _derived(snap, _source_hidden(source or snap.get("source"), catalog, fetch, masked)
|
||||
| _mask_names(snap.get("mask_columns")), fetch, kinds)
|
||||
hidden, names = _derived(snap, _source_hidden(source or snap.get("source"), catalog, fetch, masked)
|
||||
| _mask_names(snap.get("mask_columns")), fetch, mask)
|
||||
label = f"查询 {alias} " if alias else "一次查询(不在这份报告的证据目录中)"
|
||||
lines.append(f"输入 {path}:取自{label}的 {_cell_words(item.get('locator') or {}, hidden)}")
|
||||
sql = _clean(str(snap.get("sql") or ""))
|
||||
if sql and mask is not None:
|
||||
sql = mask.sql(sql, snap, names)
|
||||
if sql and str(artifact) not in seen and len(seen) < ENTITY_QUERIES:
|
||||
seen.add(str(artifact))
|
||||
sqls.append(_sql_line(f"查询 {alias} 的 SQL" if alias else "这次查询的 SQL", sql))
|
||||
@@ -539,7 +547,7 @@ def _metric_inputs(metric: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any
|
||||
|
||||
|
||||
def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch, catalog: dict[str, Any],
|
||||
masked: dict[str, Any] | None, kinds: dict[str, Any] | None = None) -> str:
|
||||
masked: dict[str, Any] | None, mask: Any = None) -> str:
|
||||
name = entry.get("name") or (entry.get("locator") or {}).get("metric") or alias
|
||||
rendered = entry.get("rendered")
|
||||
where = " ".join(v for v in (f"口径「{entry['caliber']}」" if entry.get("caliber") else "",
|
||||
@@ -554,7 +562,7 @@ def _metric_excerpt(alias: str, entry: dict[str, Any], fetch: _Fetch, catalog: d
|
||||
if metric is not None:
|
||||
if metric.get("expression"):
|
||||
lines.append(f"原式:{_clean(str(metric['expression']))}")
|
||||
lines.extend(_metric_inputs(metric, fetch, catalog, masked, kinds))
|
||||
lines.extend(_metric_inputs(metric, fetch, catalog, masked, mask))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@@ -632,7 +640,8 @@ def _column_lines(entry: dict[str, Any], fetch: _Fetch) -> list[str]:
|
||||
return lines
|
||||
|
||||
|
||||
def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], whole: bool, fetch: _Fetch) -> str | None:
|
||||
def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], whole: bool, fetch: _Fetch,
|
||||
mask: Any = None) -> str | None:
|
||||
content = fetch(entry.get("artifact"))
|
||||
hits = content.get("hits") if isinstance(content, dict) else None
|
||||
if not isinstance(hits, list):
|
||||
@@ -646,6 +655,8 @@ def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], who
|
||||
if not isinstance(hit, dict) or not isinstance(hit.get("content"), str):
|
||||
continue
|
||||
body = _clean(hit["content"])
|
||||
# 核查 agent 的摘录(mask):知识库片段里写着的遮罩值按值换(先换再截,截在半截的值认不出)
|
||||
body = mask.text(body) if mask is not None else body
|
||||
title = f"《{_clean(str(hit['title']))}》" if hit.get("title") else ""
|
||||
lines.append(f"片段 {h + 1}{title}:{body[:CHUNK_CHARS]}{'…' if len(body) > CHUNK_CHARS else ''}")
|
||||
return "\n".join(lines)
|
||||
@@ -653,12 +664,12 @@ def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], who
|
||||
|
||||
def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _Fetch,
|
||||
masked: dict[str, Any] | None, catalog: dict[str, Any], *, provenance_on: bool = False,
|
||||
kinds: dict[str, Any] | None = None) -> str | None:
|
||||
mask: Any = None) -> str | None:
|
||||
"""一件证据的摘录。provenance_on 只交给查询摘录(_query_excerpt):指标摘录里「输入取自查询 Qn」的那几行不加
|
||||
来历(P4-SPEC 调整 1),表名、字段名的摘录照旧。kinds 只给核查 agent 的摘录(见 _derived)。"""
|
||||
来历(P4-SPEC 调整 1),表名、字段名的摘录照旧。mask 只给核查 agent 的摘录(见 prepare)。"""
|
||||
kind = entry.get("kind")
|
||||
if kind == "metric":
|
||||
return _metric_excerpt(alias, entry, fetch, catalog, masked, kinds)
|
||||
return _metric_excerpt(alias, entry, fetch, catalog, masked, mask)
|
||||
if kind == "input":
|
||||
field_name = (entry.get("locator") or {}).get("field") or alias
|
||||
return f"【{alias}】运行输入:{field_name} = {entry.get('rendered')}"
|
||||
@@ -666,27 +677,30 @@ def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _
|
||||
rows = {r for c in cands for a, r in c.rows if a == alias}
|
||||
cols = [col for c in cands for a, col in c.cols if a == alias]
|
||||
return _query_excerpt(alias, entry, rows, cols, any(alias in c.whole for c in cands), fetch, masked,
|
||||
provenance_on=provenance_on, kinds=kinds)
|
||||
provenance_on=provenance_on, mask=mask)
|
||||
if kind == "retrieval":
|
||||
picked = [h for c in cands for a, h in c.hits if a == alias]
|
||||
return _retrieval_excerpt(alias, entry, picked, any(alias in c.whole for c in cands), fetch)
|
||||
return _retrieval_excerpt(alias, entry, picked, any(alias in c.whole for c in cands), fetch, mask)
|
||||
if kind == "table":
|
||||
head = f"【{alias}】表 {entry.get('name')}" + (f"(数据源 {entry['source']})" if entry.get("source") else "")
|
||||
return "\n".join([head, *_table_lines(entry, catalog, fetch, masked), *_entity_sql(entry, catalog, fetch)])
|
||||
return "\n".join([head, *_table_lines(entry, catalog, fetch, masked),
|
||||
*_entity_sql(entry, catalog, fetch, masked, mask)])
|
||||
if kind == "column":
|
||||
table = (entry.get("locator") or {}).get("table")
|
||||
head = f"【{alias}】字段 {f'{table}.' if table else ''}{(entry.get('locator') or {}).get('column') or entry.get('name')}"
|
||||
return "\n".join([head, *_column_lines(entry, fetch), *_entity_sql(entry, catalog, fetch)])
|
||||
return "\n".join([head, *_column_lines(entry, fetch), *_entity_sql(entry, catalog, fetch, masked, mask)])
|
||||
return None
|
||||
|
||||
|
||||
def _entity_sql(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch) -> list[str]:
|
||||
def _entity_sql(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch, masked: dict[str, Any] | None = None,
|
||||
mask: Any = None) -> list[str]:
|
||||
"""表名、字段名出现过的查询的 SQL(每条截到 SQL_CHARS,最多 ENTITY_QUERIES 条)。
|
||||
|
||||
讲算法的句子(「按会员号去重计数得出」)证据在 SQL 里;只给名字,裁判只能判不支持。
|
||||
查询用条目自己记的(表:SQL 里用到它;字段:它是结果列)。只在 SQL 里出现、不是结果列的字段自己没记:
|
||||
有表条目就从它记的查询里找,没有(文档目录只留正文用到的实体)就看全部查询,都只留 SQL 里写了这个
|
||||
字段名(有表名的还要写了表名)的。快照经 fetch 取:取不到(不在封存范围里)的不给。
|
||||
mask 同 _query_excerpt:SQL 先遮再截(要遮的列按那条查询的数据源认,masked 同 _query_excerpt)。
|
||||
"""
|
||||
locator = entry.get("locator") or {}
|
||||
column = locator.get("column") if entry.get("kind") == "column" else None
|
||||
@@ -705,6 +719,9 @@ def _entity_sql(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch) -
|
||||
sql = _clean(str(snapshot.get("sql") or "")) if isinstance(snapshot, dict) else ""
|
||||
if not sql or any(not re.search(rf"(?<![\w]){re.escape(n)}(?![\w])", sql, re.I) for n in need):
|
||||
continue
|
||||
if mask is not None:
|
||||
source = catalog[q].get("source") or snapshot.get("source")
|
||||
sql = mask.sql(sql, snapshot, _derived(snapshot, _hidden(snapshot, source, masked), fetch, mask)[1])
|
||||
lines.append(f"出现在查询 {q}:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}")
|
||||
if len(lines) == ENTITY_QUERIES:
|
||||
break
|
||||
@@ -762,13 +779,15 @@ class JudgeRequest:
|
||||
|
||||
def prepare(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, units: Iterable[str] | None = None,
|
||||
loader: Callable[[str], Any] | None = None, masked: dict[str, Any] | None = None,
|
||||
kinds: dict[str, Any] | None = None) -> JudgeRequest:
|
||||
mask: Any = None) -> JudgeRequest:
|
||||
"""裁判的输入:候选句和证据摘录。纯函数(快照经 loader 取,缺省读工件库),不调模型。
|
||||
|
||||
catalog 缺省用文档自己存的目录;masked = {数据源名: 要遮的列},和快照里记下的 mask_columns 合在一起。
|
||||
kinds = {数据源名: 类型} 只在给核查 agent(反方审稿、审批质询)准备摘录时给:查询摘录、指标输入里改了名字的遮罩列
|
||||
(buyer_phone AS 手机号)按 SQL 认出来也不给,和核查 agent 的查询在工具边界遮的是同一套规则(_derived)。结论句裁判
|
||||
自己的摘录不给它,和以前逐字相同。
|
||||
mask 只在给核查 agent(反方审稿、审批质询)准备摘录时给(engine/inquiry.py 的 ExcerptMask,这里只按它的 columns、
|
||||
cell、sql、text 几个方法用):摘录分段遮,和核查 agent 的查询在工具边界遮的是同一套口径——改了名字的遮罩列
|
||||
(buyer_phone AS 手机号)按 SQL 认出来也不给;SQL 照质询封存 SQL 的办法换,范围比较的阈值、LIMIT 照原样;格子逐格
|
||||
遮,数值格照原样;列名、知识库片段、来历行按值换。报告里的指标值、原式、运行输入不遮(报告本来就摆在读者眼前)。
|
||||
以前先渲染好整段、再整段按值换,阈值、没设遮罩的数也成了「已遮罩」。结论句裁判自己的摘录不给它,和以前逐字相同。
|
||||
|
||||
摘录按文档标记分版本(P4-SPEC 1.4、3.3):带 `provenance: DOC_PROVENANCE` 的文档,上传表格的查询摘录多几行
|
||||
来历;没有标记的(期 4 之前组装的)和以前逐字相同,复用键、断点指纹都不变,已判过的句子照旧复用。
|
||||
@@ -781,7 +800,7 @@ def prepare(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, units
|
||||
for alias in dict.fromkeys(a for c in cands for a in c.aliases):
|
||||
entry = catalog.get(alias)
|
||||
if isinstance(entry, dict) and (text := _excerpt(alias, entry, cands, fetch, masked, catalog,
|
||||
provenance_on=provenance_on, kinds=kinds)):
|
||||
provenance_on=provenance_on, mask=mask)):
|
||||
excerpts[alias] = text
|
||||
picked = {c.unit for c in cands}
|
||||
skipped: dict[str, str] = {}
|
||||
|
||||
+295
-91
@@ -8,29 +8,35 @@
|
||||
- 工具边界先遮(mask_query_result):核查 agent 的查询结果交给模型、落进快照之前,取自遮罩列的格子换成「已遮罩」,
|
||||
别的文字格按值换:这份结果里的遮罩值,加上这个 agent 前面的查询遮掉的、调用方交进来的(报告快照里的)。模型看不到
|
||||
遮罩值,就不会把它写进回答、理由、SQL——事后按值认每一轮复核都找到新的漏法,根治靠这一步;审批质询回答、反方
|
||||
审稿理由里的按值遮降为第二道防线。宁可多遮,但不改写没设遮罩的数值证据:数值格不按值换,按值换过的文字格记进
|
||||
快照(masked_cells)。查询失败时报错原话里的值也换(mask_error_text)。分析本身出了意外,整份遮(_masked_whole)
|
||||
|
||||
- 哪些列(_derived_hidden):从遮罩列本身起,按 SQL 一层层认出值取自遮罩列的结果列(别名、函数、子查询、WITH、UNION 的
|
||||
审稿理由里的按值遮降为第二道防线。宁可多遮,但不改写没设遮罩的数值证据:数值格(数,列类型是 number 而且列名唯一
|
||||
的,_typed_numbers)不按值换,按值换过的文字格记进快照(masked_cells),认不全整份遮的也记一笔(masked_whole)。查询
|
||||
失败时报错原话里的值也换(mask_error_text)。分析本身出了意外,整份遮(_masked_whole)
|
||||
- 证据摘录(engine/inquiry.py 的 ExcerptMask,judge.prepare 按它逐段遮):交给反方审稿、审批质询 agent 的报告摘录同一套
|
||||
口径,格子逐格遮(masked_cell),SQL 只换在找某一个值的字面量,阈值照原样
|
||||
- 哪些列(_derived_masks):从遮罩列本身起,按 SQL 一层层认出值取自遮罩列的结果列(别名、函数、子查询、WITH、UNION 的
|
||||
另一支、和 * 写在同一支里没起别名的表达式、FROM 里拆开遮罩列的表值函数、函数参数里的 JSON 箭头 col->'path');* 把
|
||||
遮罩列带进别的列名的(写了列名表的派生来源、UNION 后一支的 *、标量子查询里的 *)按它展开落到的位置算(_star_places,
|
||||
展开数据源里的表的按查询当时冻结的表结构数位置,_schema_columns;写了库名的表按「库名.表名」查,另一个库里的同名表
|
||||
查不到)。整行和函数参数里的 *(Postgres 的 row_to_json(s)、SELECT s、to_json(s.*),Oracle 的 JSON_OBJECT(*))按 *
|
||||
展开的来源算;表的列名表(FROM sales AS s(i, r, p))、TABLE x、套了括号的 * 先改写成认得出的样子(_normalized)。
|
||||
WITH 里的名字按作用域认(_scoped_cte):不带 RECURSIVE 的一段看不见它自己和排在后面的段,同名的是数据源里的表。
|
||||
COUNT 只数个数、窗口的分区键只决定次序、按遮罩列分档又当分组键或套在汇总里数个数的、汇总里的 CASE 只在条件里用到的
|
||||
不算(_value_columns、_band_only)。认不全的写法整份快照的列都算,不再逐个形状补规则:认不出落在哪一列的(PIVOT、
|
||||
UNPIVOT 这类也算),(表达式).* 展开读到遮罩列的(复合类型的列、函数交回的记录,展开成几列、叫什么说不清),表名、SQL
|
||||
写在字符串里交给函数、统计信息里存着值的(_INDIRECT:query_to_xml 这一族、dblink、pg_stats……),SQL 解析不了又写着
|
||||
遮罩列(或者 *、TABLE x、_INDIRECT 那几类)的,分析出了意外、超出工作量上限(ANALYSIS_STEPS)的。一次分析里派生来源
|
||||
对应的那块查询、一项的值取自哪些列记下来(_analysis),一长串 CTE 不再卡住。双盲复算按同一套认法在派盲算之前挡下
|
||||
取自遮罩列的指标(derived_columns)
|
||||
UNPIVOT 这类也算),(表达式).* 展开读到遮罩列的(复合类型的列、函数交回的记录,外面套了括号、起了别名的也算,展开成
|
||||
几列、叫什么说不清),语法树和实际执行的对不上的(OPAQUE_SQL:MySQL 会执行的注释、U&"…";没有 * 的一支选出项和结果
|
||||
的列数对不上),表名、SQL 写在字符串里交给函数、统计信息和会话记录里存着值的(INDIRECT_FUNCTIONS、INDIRECT_NAMES:
|
||||
query_to_xml 这一族、dblink、pg_stats、MySQL 的 column_statistics、Oracle 的直方图和 V$SQL……,可验证报告用的同一份),
|
||||
SQL 解析不了又写着遮罩列(或者 *、TABLE x、上面那几类,或者是 Postgres 的)的,分析出了意外、超出工作量上限
|
||||
(ANALYSIS_STEPS)的;这些只收确定取自遮罩列的那几列的值(_Masks.collect)。一次分析里派生来源对应的那块查询、一项
|
||||
的值取自哪些列记下来(_analysis),一长串 CTE 不再卡住。双盲复算按同一套认法在派盲算之前挡下取自遮罩列的指标
|
||||
(derived_columns)
|
||||
- 只装着分档标签的列(_label_columns):标签是 agent 写在 SQL 里的字,不当遮罩值;* 展开数据源里的表时按冻结的表结构
|
||||
认它带不带同名的列(_schema_tables),不知道表结构的按带得出算(多遮)。分档写在派生来源里、外层按它分组的(一档
|
||||
一行,_grouped_labels)同当分组键的 CASE 一样照常显示(_shown_labels)
|
||||
- 哪些值(_Secrets):遮罩列里出现过的值,按值认有门槛(SECRET_MIN、SECRET_DIGITS),占位的写法(PLACEHOLDERS:未知、
|
||||
暂无……)不收;换的时候先按词、按开头几个字筛出正文里真有的那几个值,不现编上万个分支的正则;正文、文字格、SQL 的
|
||||
字符串和注释里换了写法的长号码(199-1474-2785、+86 …、全角数字)按纯数字写法在集合里查(loose_spans);已经摆在
|
||||
读者眼前的值免遮,之后并进来的照样免(protected)
|
||||
读者眼前的值免遮,之后并进来的照样免(protected、spared)
|
||||
- 快照遮好的样子(_masked_view),回答时才认得出、事后要照位置重放的地方(view_edits、replay_edits)
|
||||
- 一次核查里认出的遮罩值(SeenValues):调用方交进来的和 agent 查询时遮掉的分开记,合起来时 agent 查到的在前
|
||||
|
||||
@@ -45,10 +51,11 @@ import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
from collections import Counter
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
from app.engine.evidence import MARKER_RE, RenderError, render_cell
|
||||
from app.engine.expressions import numeric_text
|
||||
@@ -762,19 +769,17 @@ def _derived_block(source: Any) -> tuple[Any, list[str] | None] | None:
|
||||
|
||||
|
||||
def _derived_block_of(source: Any) -> tuple[Any, list[str] | None] | None:
|
||||
"""_derived_block 真去找的那一步(不看记下的结果):指向 WITH 的名字要由里往外数看得见的 CTE(_visible_ctes),一长串
|
||||
"""_derived_block 真去找的那一步(不看记下的结果):指向 WITH 的名字要由里往外数看得见的 CTE(_scoped_cte),一长串
|
||||
CTE 时每次都数一遍很慢,所以分析里记下来。"""
|
||||
from sqlglot import exp
|
||||
|
||||
from app.data.sql_semantics import _visible_ctes
|
||||
|
||||
node, alias = source, None
|
||||
if isinstance(node, exp.Lateral):
|
||||
node, alias = node.this, node.args.get("alias")
|
||||
if isinstance(node, exp.Table):
|
||||
if node.args.get("db") or node.args.get("catalog") or not isinstance(node.this, exp.Identifier):
|
||||
return None
|
||||
cte = _visible_ctes(node).get(node.name.lower())
|
||||
cte = _scoped_cte(node, node.name.lower())
|
||||
if cte is None:
|
||||
return None
|
||||
node, alias = cte.this, cte.args.get("alias")
|
||||
@@ -786,6 +791,33 @@ def _derived_block_of(source: Any) -> tuple[Any, list[str] | None] | None:
|
||||
return node, ([c.name.lower() for c in alias.columns] if alias is not None and alias.columns else None)
|
||||
|
||||
|
||||
def _scoped_cte(node: Any, name: str) -> Any:
|
||||
"""node 所在的位置按名字 name(小写)指到的那一段 CTE,没有为 None(是数据源里的表)。由里往外找,里层的 WITH 遮住
|
||||
外层的:一层查询的正文看得见它的 WITH 里的每一段;一段 CTE 的正文里,不带 RECURSIVE 的 WITH 只看得见排在这一段
|
||||
前面的几段——Postgres、MySQL、Oracle 里看不见它自己,同名的是数据源里的表(WITH sales AS (SELECT * FROM sales
|
||||
WHERE …) 过滤同名的表是常见写法);SQLite 写自己是循环引用、报错,往后指的它看得见,这里照看不见算,当成数据源里的
|
||||
表(宁可多遮)。带 RECURSIVE 的每一段都看得见(递归的那一支指向它自己)。以前一律当看得见(sql_semantics 的
|
||||
_visible_ctes),sales 认成了这一段自己,* 展开绕回自己什么列都不带出,整行引用、列名表改名的号码原样交了出去。"""
|
||||
from sqlglot import exp
|
||||
|
||||
child, cur = node, node.parent
|
||||
inside: Any = None # 往外走时经过的、离 cur 最近的那一段 CTE
|
||||
while cur is not None:
|
||||
if isinstance(child, exp.CTE):
|
||||
inside = child
|
||||
with_ = cur.args.get("with_")
|
||||
if isinstance(with_, exp.With):
|
||||
ctes = list(with_.expressions)
|
||||
at = next((i for i, cte in enumerate(ctes) if cte is inside), None) if child is with_ else None
|
||||
if at is not None and not with_.args.get("recursive"):
|
||||
ctes = ctes[:at]
|
||||
found = next((cte for cte in reversed(ctes) if cte.alias_or_name.lower() == name), None)
|
||||
if found is not None:
|
||||
return found
|
||||
child, cur = cur, cur.parent
|
||||
return None
|
||||
|
||||
|
||||
#: 表值函数常见的出列名:SQLite 的 json_each / json_tree(key、value、atom、fullkey、path),Postgres 的 jsonb_each
|
||||
#: (key、value),Spark 的 explode / posexplode(col、pos),Snowflake 的 FLATTEN(key、value、path、index、this)。
|
||||
#: 结构性的几列(id、parent、type)带不出值,不算——算上的话,同一条 SQL 里别的 id 列都会被遮
|
||||
@@ -1129,19 +1161,19 @@ def _normalized(root: Any, read: str | None, tables: dict[str, list[str]] | None
|
||||
改写前后结果有哪几列、每一列的值取自哪里都一样:
|
||||
- TABLE x(Postgres、MySQL):解析器认不出,CTE、子查询里的读成了一个叫 TABLE 的列、表。改成 SELECT * FROM x;带库名、
|
||||
ORDER BY 的解析不了,按解析不了处理(_implicit)
|
||||
- 选出项套了括号的 *((s.*))、(s).*(s 是这一层 FROM 里的来源,不是哪个来源的列):Postgres 照样展开成好几列,改成
|
||||
s.*,不然按一列数位置
|
||||
- 选出项套了括号、起了别名的 *((s.*)、(s.*) AS x)、(s).*(s 是这一层 FROM 里的来源,不是哪个来源的列;外面也可以
|
||||
套括号、起别名):Postgres 照样展开成好几列(别名不起作用),改成 s.*,不然按一列数位置
|
||||
- 带列名表的表(Postgres 的 FROM sales AS s(i, r, p),引用 CTE 时再写列名表的也算):改成 (SELECT * FROM sales) AS
|
||||
s(i, r, p),按派生来源的列名表认改了名的列。Postgres 的列名表可以比列少(只改前几列的名字,后面的照原名),这时
|
||||
* 展开的列数和列名表对不上,列名表里的几列整段算(_star_places),后面照原名的按名字认。不按表结构补全列名表:
|
||||
别的方言里列名表比列少只能是表结构过期了,补全就丢了这一道核对
|
||||
- 整行引用(read 是 _ROW_DIALECTS 之一,_row_source):row_to_json(s)、json_agg(t)、SELECT s、s::text 里的 s 改成
|
||||
ROW(s.*),按 * 展开的来源认(_value_columns)
|
||||
- 别的 (表达式).*(复合类型的列 (p.profile).*、函数交回的记录 (jsonb_each_text(pj)).*、子查询里的 ROW (q.r).*):展开
|
||||
成几列、各叫什么(复合类型的字段名、函数的出列名)SQL 里看不出,改成一个对不上任何来源的 *(_star_sources 为
|
||||
None),只用来数别的项的位置。表达式的值取自哪些列(_value_columns,整行引用已经改写过)记下来交回:读到遮罩列的,
|
||||
整份快照都算(_tainted)——以前只改成对不上来源的 *,最外层、第一支、派生来源里没写列名表的 * 按「照原名带出」认,
|
||||
字段名和遮罩列对不上,号码原样交了出去
|
||||
- 别的 (表达式).*(复合类型的列 (p.profile).*、函数交回的记录 (jsonb_each_text(pj)).*、子查询里的 ROW (q.r).*,外面
|
||||
套了括号、起了别名的 ((p.profile).*)、(p.profile).* AS x 也算,_unwrapped):展开成几列、各叫什么(复合类型的字段名、
|
||||
函数的出列名)SQL 里看不出,改成一个对不上任何来源的 *(_star_sources 为 None),只用来数别的项的位置。表达式的值
|
||||
取自哪些列(_value_columns,整行引用已经改写过)记下来交回:读到遮罩列的,整份快照都算(_tainted)——以前只改成对
|
||||
不上来源的 *,最外层、第一支、派生来源里没写列名表的 * 按「照原名带出」认,字段名和遮罩列对不上,号码原样交了出去
|
||||
tables 是查询当时冻结的表结构(_schema_columns,按次序的列名),认整行引用时看来源出不出某一列;没有为 None。
|
||||
交回每个 (表达式).* 的值取自哪些列。"""
|
||||
from sqlglot import exp
|
||||
@@ -1184,13 +1216,25 @@ def _normalized(root: Any, read: str | None, tables: dict[str, list[str]] | None
|
||||
opaque: list[set[str]] = []
|
||||
for select in list(root.find_all(exp.Select)):
|
||||
for item in list(select.expressions):
|
||||
if isinstance(item, exp.Dot) and isinstance(item.expression, exp.Star):
|
||||
node = _unwrapped(item)
|
||||
if isinstance(node, exp.Dot) and isinstance(node.expression, exp.Star):
|
||||
# 先认值取自哪些列(还挂在树上,* 和整行引用按作用域认),再换掉
|
||||
opaque.append(set(_value_columns(item.this)))
|
||||
opaque.append(set(_value_columns(node.this)))
|
||||
item.replace(exp.Column(this=exp.Star(), table=exp.to_identifier(_OPAQUE_STAR)))
|
||||
return opaque
|
||||
|
||||
|
||||
def _unwrapped(item: Any) -> Any:
|
||||
"""选出项去掉套在外面的括号、别名:Postgres 里 (s.*)、((p.card).*)、(p.card).* AS x 照样展开成好几列(别名不起作用,
|
||||
实测如此)。以前只认不带外括号的 (表达式).*,((p.card).*) 按一项数,只遮了第一列,后面的号码原样交了出去。"""
|
||||
from sqlglot import exp
|
||||
|
||||
node = item
|
||||
while isinstance(node, (exp.Paren, exp.Alias)):
|
||||
node = node.this
|
||||
return node
|
||||
|
||||
|
||||
#: (表达式).* 改写成的 * 写的「表名」:对不上任何来源(_star_sources 为 None),真实的表名、别名不会是它
|
||||
_OPAQUE_STAR = "*"
|
||||
|
||||
@@ -1204,18 +1248,17 @@ def _table_keyword(node: Any) -> bool:
|
||||
|
||||
|
||||
def _bare_star(item: Any, select: Any, tables: dict[str, list[str]] | None) -> Any:
|
||||
"""select 的选出项是套了括号的 *((s.*)、((*))),或者 (s).*、s 是这一层 FROM 里一个来源的名字、又不是哪个来源说得准
|
||||
出的一列(Postgres 先按列认,_has_column)时,改写成的 *、s.*;别的为 None(_normalized,别的 (表达式).* 另外处理)。"""
|
||||
"""select 的选出项是套了括号、起了别名的 *((s.*)、((*))、(s.*) AS x),或者 (s).*(外面也可以套括号、起别名)、s 是
|
||||
这一层 FROM 里一个来源的名字、又不是哪个来源说得准出的一列(Postgres 先按列认,_has_column)时,改写成的 *、s.*;别的
|
||||
为 None(_normalized,别的 (表达式).* 另外处理)。"""
|
||||
from sqlglot import exp
|
||||
|
||||
node = item
|
||||
while isinstance(node, exp.Paren):
|
||||
node = node.this
|
||||
node = _unwrapped(item)
|
||||
if node is not item and _is_star(node):
|
||||
return node.copy()
|
||||
if not (isinstance(item, exp.Dot) and isinstance(item.expression, exp.Star)):
|
||||
if not (isinstance(node, exp.Dot) and isinstance(node.expression, exp.Star)):
|
||||
return None
|
||||
inner = item.this
|
||||
inner = node.this
|
||||
while isinstance(inner, exp.Paren):
|
||||
inner = inner.this
|
||||
if not (isinstance(inner, exp.Column) and not inner.table and isinstance(inner.this, exp.Identifier)):
|
||||
@@ -1393,10 +1436,20 @@ def _source_names(top: Any, listed: list[str] | None, names: set[str],
|
||||
return found, unknown
|
||||
|
||||
|
||||
class _Taint(NamedTuple):
|
||||
"""_tainted 认出来的:SQL 里指代遮罩列的名字(小写);结果里值取自遮罩列的那几列的位置(认不全时是全部的列);是不是
|
||||
认不全、整份都算;确定取自遮罩列的位置(整份算之前认出的那几列)——整份遮时只收这几列的值当遮罩值(_derived_masks)。"""
|
||||
|
||||
names: set[str]
|
||||
positions: set[int]
|
||||
whole: bool
|
||||
sure: set[int]
|
||||
|
||||
|
||||
def _tainted(sql: str, kind: str | None, masked: set[str], width: int, tables: dict[str, list[str]] | None = None
|
||||
) -> tuple[set[str], set[int], bool] | None:
|
||||
) -> _Taint | None:
|
||||
"""SQL 里指代遮罩列的名字(小写)、结果(width 列)里值取自遮罩列的那几列的位置,以及是不是认不出、整份都算(这时
|
||||
位置是全部的列)。解析不了返回 None。
|
||||
位置是全部的列,另交回整份算之前确定认出的位置,见 _Taint)。解析不了返回 None。
|
||||
|
||||
名字从遮罩列本身起,一层层传下去:选出列的值取自这些名字的(buyer_phone AS 手机号、UPPER(buyer_phone) AS x),它的
|
||||
名字也算;派生来源(子查询、WITH 里的一段)按位置对上列名表、UNION 第一支的列名,也算,直到不再多出名字。同名
|
||||
@@ -1411,9 +1464,10 @@ def _tainted(sql: str, kind: str | None, masked: set[str], width: int, tables: d
|
||||
JSON_OBJECT(*))、整行引用(row_to_json(s))按 _value_columns 算进外层的表达式。表的列名表、TABLE x、套了括号的 *
|
||||
先改写成上面这些认得出的样子(_normalized)。
|
||||
PIVOT、UNPIVOT、MATCH_RECOGNIZE 读到这些名字的:值转进了它们自己起名的列(UNPIVOT 的值列、PIVOT 按 IN 里的值起名的列、
|
||||
MEASURES),出的是哪几列不好认,整份快照的列都算。(表达式).* 读到这些名字的、_INDIRECT 那几类写法同样整份算。
|
||||
解析之后的分析出了意外、超出工作量上限(ANALYSIS_STEPS)的也整份算——不退回「解析不了」只看文字:没写遮罩列、
|
||||
没有 * 的整行引用(SELECT s)那样就放过去了。"""
|
||||
MEASURES),出的是哪几列不好认,整份快照的列都算。(表达式).* 读到这些名字的、OPAQUE_SQL 和 _INDIRECT 那几类写法
|
||||
(_opaque)、结果的一支没有 * 而选出项和结果的列数对不上的(MySQL 会执行的注释里多写了几项这类,语法树和实际执行的
|
||||
对不上)同样整份算。解析之后的分析出了意外、超出工作量上限(ANALYSIS_STEPS)的也整份算——不退回「解析不了」只看
|
||||
文字:没写遮罩列、没有 * 的整行引用(SELECT s)那样就放过去了。"""
|
||||
from app.data.sql_semantics import _parse, _read
|
||||
|
||||
try:
|
||||
@@ -1425,17 +1479,17 @@ def _tainted(sql: str, kind: str | None, masked: set[str], width: int, tables: d
|
||||
try:
|
||||
opaque = _normalized(root, _read(kind), tables)
|
||||
with _analysis():
|
||||
return _tainted_in(root, sql, masked, width, tables, opaque)
|
||||
return _tainted_in(root, sql, kind, masked, width, tables, opaque)
|
||||
except _Overworked:
|
||||
# 一长串 CTE 一段套一段地整行引用、展开 *:分析不过来,认不准,整份遮
|
||||
logger.warning("SQL 遮罩分析超出工作量上限(%d 步),整份快照按遮罩处理", ANALYSIS_STEPS)
|
||||
except Exception: # noqa: BLE001 - 分析本身的缺陷不能让回答出错,也不能把原值放过去:整份遮(宁可多遮)
|
||||
logger.warning("SQL 遮罩分析失败,整份快照按遮罩处理", exc_info=True)
|
||||
return set(masked), set(range(width)), True
|
||||
return _Taint(set(masked), set(range(width)), True, set())
|
||||
|
||||
|
||||
def _tainted_in(root: Any, sql: str, masked: set[str], width: int, tables: dict[str, list[str]] | None,
|
||||
opaque: list[set[str]]) -> tuple[set[str], set[int], bool]:
|
||||
def _tainted_in(root: Any, sql: str, kind: str | None, masked: set[str], width: int,
|
||||
tables: dict[str, list[str]] | None, opaque: list[set[str]]) -> _Taint:
|
||||
"""_tainted 解析、改写(_normalized)之后真去认的那一步。opaque 是 (表达式).* 的值取自哪些列(_normalized 交回的)。"""
|
||||
from sqlglot import exp
|
||||
|
||||
@@ -1462,10 +1516,11 @@ def _tainted_in(root: Any, sql: str, masked: set[str], width: int, tables: dict[
|
||||
if len(names) == before:
|
||||
break
|
||||
# 认不出的整份遮:表值函数出哪几列说不清、派生来源里有认不出名字的列、PIVOT 这类读到遮罩列,(表达式).* 读到遮罩列
|
||||
# (展开成几列、叫什么都说不清),SQL 写在字符串里交给函数执行、整表导出、统计信息里的值(_INDIRECT)
|
||||
# (展开成几列、叫什么都说不清),语法树和实际执行的对不上、SQL 写在字符串里交给函数执行、整表导出、统计信息里的值
|
||||
# (_opaque)
|
||||
everything = unreadable or any(_source_names(top, listed, names, tables)[1] for top, listed in sources) \
|
||||
or any(_reads(names, _value_columns(node)) for node in root.find_all(exp.Pivot, exp.MatchRecognize)) \
|
||||
or any(_reads(names, cols) for cols in opaque) or bool(_INDIRECT.search(sql))
|
||||
or any(_reads(names, cols) for cols in opaque) or _opaque(sql, kind)
|
||||
positions: set[int] = set()
|
||||
for top, role, _listed in blocks:
|
||||
if role != "result":
|
||||
@@ -1473,6 +1528,10 @@ def _tainted_in(root: Any, sql: str, masked: set[str], width: int, tables: dict[
|
||||
branches = [leaf for leaf in _leaves(top) if isinstance(leaf, exp.Select)]
|
||||
for k, select_ in enumerate(branches):
|
||||
branch = list(select_.expressions)
|
||||
if len(branch) != width and not any(_is_star(item) for item in branch):
|
||||
# 没有 * 的一支,选出项和结果的列数对不上:语法树和数据库实际执行的不是一回事(解析器扔掉的注释里多写
|
||||
# 了几项、方言的写法读岔了),哪一列取自哪里都说不准
|
||||
everything = True
|
||||
for item, place in zip(branch, _places(branch, width)):
|
||||
if not _is_star(item) and _reads(names, _value_columns(item)):
|
||||
if place is None or place >= width:
|
||||
@@ -1485,7 +1544,7 @@ def _tainted_in(root: Any, sql: str, masked: set[str], width: int, tables: dict[
|
||||
everything = True
|
||||
else:
|
||||
positions |= spread
|
||||
return names, set(range(width)) if everything else positions, everything
|
||||
return _Taint(names, set(range(width)) if everything else positions, everything, positions)
|
||||
|
||||
|
||||
def _mentions(sql: str, names: Iterable[str]) -> bool:
|
||||
@@ -1499,55 +1558,144 @@ _COUNT_STAR = re.compile(r"\bcount\s*\(\s*(?:all\s+|distinct\s+)?\*\s*\)", re.I)
|
||||
_IMPLICIT = re.compile(r"(?:\bselect|\bdistinct|\ball|[,(.])\s*\*|\btable\s+[\w\"`]", re.I)
|
||||
|
||||
|
||||
#: 不点列名也读得出表里内容的写法,SQL 里认不出读的是哪一列:表名、SQL 写在字符串里交给函数执行或整表导出(Postgres 的
|
||||
#: query_to_xml、table_to_xml 这一族、ts_stat、tablefunc 的 crosstab / connectby、dblink,Oracle 的 DBMS_XMLGEN、XMLTYPE、
|
||||
#: XQuery,DuckDB 的 query / query_table,SQL Server 的 OPENQUERY、OPENROWSET),按页、按文件读出的原始数据,统计信息、
|
||||
#: 会话记录里存着的值(pg_stats 的常见值、sqlite_stat4 的样本、pg_stat_statements 记下的 SQL 原文)。出现就整份遮
|
||||
#: (_tainted、_implicit),写在字符串、注释里的也算(多遮)。函数要跟着左括号才算(列名碰巧叫 query 的不算)。
|
||||
#: information_schema、pg_catalog 只有表结构,不在里面
|
||||
_INDIRECT = re.compile(r"(?<![0-9A-Za-z_$])(?:(?:" + "|".join((
|
||||
#: 语法树和数据库实际执行的对不上的写法:MySQL、MariaDB 会执行的注释(/*! … */、/*M! … */,解析器当注释扔掉,选出项
|
||||
#: 就少了几项),Postgres 按 Unicode 转义写的名字(U&"…",带不带 UESCAPE 都是,原文里看不出是哪一列,解析器读成 U 和一个
|
||||
#: 带引号的名字按位与)。出现就整份遮(_tainted、_implicit)。可验证报告(bundle/build.py)用的是同一个
|
||||
OPAQUE_SQL = re.compile(r'/\*m?!|u&"', re.I)
|
||||
#: 不点列名也读得出表里内容的写法里的函数(后面要跟着左括号才算,列名碰巧叫 crosstab 的不算):
|
||||
#: - 表名、SQL 写在字符串里交给函数执行或整表导出:Postgres 的 XML 一族(query_to_xml、table_to_xml、cursor_to_xml、
|
||||
#: schema_to_xml、database_to_xml 和带 _and_xmlschema 的)、ts_stat、tablefunc 的 crosstab / connectby、dblink,Oracle 的
|
||||
#: XMLTYPE、XQuery(XMLQUERY、XMLTABLE)
|
||||
#: - 按页、按文件、按日志读出的原始数据:Postgres 的 pageinspect(get_raw_page、按索引名读 B 树页的 bt_page_items)、服务器
|
||||
#: 文件、逻辑解码、pg_walinspect 读出的 WAL 块,MySQL 的 LOAD_FILE,Oracle 的 BFILENAME,SQLite 的 readfile
|
||||
#: - 别的会话执行过的 SQL 原文(写入语句里就是值):Postgres 的 pg_stat_activity 背后的函数和按会话取的那一个,MySQL 企业版的
|
||||
#: 审计日志
|
||||
#: 只收支持的几种库(SQLite、MySQL / MariaDB、Postgres、Oracle)的:DuckDB 的 query()、SQL Server 的 OPENQUERY 这类以前也收,
|
||||
#: 名叫 query 的 CTE、字符串里写着的「Query (x)」因此被当成了它们,整份遮了不该遮的结果
|
||||
INDIRECT_FUNCTIONS = (
|
||||
r"(?:query|table|cursor|schema|database)_to_xml\w*", r"ts_stat", r"crosstab\d*", r"connectby", r"dblink\w*",
|
||||
r"xmlquery", r"xmltable", r"xmltype", r"query_table", r"query", r"openquery", r"openrowset", r"opendatasource",
|
||||
r"get_raw_page", r"bt_page_items", r"pg_read_(?:binary_)?file", r"load_file", r"readfile", r"bfilename",
|
||||
)) + r")[\"`]?\s*\(|(?:" + "|".join((
|
||||
r"dbms_xml\w*", r"pg_stats\w*", r"pg_statistic\w*", r"sqlite_stat\d", r"sqlite_dbpage", r"pg_stat_statements\w*",
|
||||
r"pg_stat_activity", r"pg_prepared_statements",
|
||||
)) + r")(?![0-9A-Za-z_$]))", re.I)
|
||||
r"xmltype", r"xmlquery", r"xmltable",
|
||||
r"get_raw_page", r"bt_page_items", r"pg_read_(?:binary_)?file", r"pg_logical_slot_\w+", r"pg_get_wal_block_info",
|
||||
r"load_file", r"bfilename", r"readfile",
|
||||
r"pg_stat_get_(?:backend_)?activity", r"audit_log_read",
|
||||
)
|
||||
#: 不点列名也读得出表里内容的表、视图、包(按名字认):
|
||||
#: - Oracle 的 DBMS_XMLGEN 这一族(包名后面跟着点),XQuery 里的 ora:view 和 oradb: 表
|
||||
#: - 统计信息里的采样值、全文索引里的词:Postgres 的 pg_stats、pg_statistic 一族,SQLite 的 sqlite_stat*,MySQL 的
|
||||
#: column_statistics(直方图里是列的原值)、innodb_ft_*,MariaDB 的 column_stats,Oracle 的直方图、列统计和记着最小最大值
|
||||
#: 的 *_tab_columns、*_tab_cols(LOW_VALUE、HIGH_VALUE 按字节交回来就是号码、姓名的原文),以及它们的底表 histgrm$、
|
||||
#: hist_head$
|
||||
#: - 别的会话执行过的 SQL 原文:Postgres 的 pg_stat_activity、pg_stat_statements、本连接预备过的语句;MySQL 的 general_log、
|
||||
#: slow_log、PROCESSLIST、performance_schema(语句历史 events_statements_*、线程正在执行的 SQL),sys 库里建在
|
||||
#: performance_schema 上的会话、锁等待视图(带 x$ 的是不排版的那一份;只认这几张,Oracle 常写的 sys.dual 不算);Oracle 的
|
||||
#: V$SQL 一族(含 v$sql_bind_capture)、打开着的游标、审计记录
|
||||
#: - SQLite 按页读出的原始数据
|
||||
#: information_schema、pg_catalog 不整个收:大多只有表结构,核查 agent 查列名、类型时常用,里面存着值的那几张上面逐个列了
|
||||
#: (可验证报告那边宁可多挡,整个收,见 SYSTEM_SCHEMAS)
|
||||
INDIRECT_NAMES = (
|
||||
r"dbms_xml\w*", r"ora:view", r"oradb",
|
||||
r"pg_stat(?:s|istic)\w*", r"sqlite_stat\d", r"column_statistics", r"innodb_ft_\w+", r"column_stats",
|
||||
r"(?:all|dba|user|cdb)_(?:\w+_)?(?:histograms|col_statistics|tab_cols|tab_columns)", r"(?:histgrm|hist_head)\$",
|
||||
r"pg_stat_activity", r"pg_stat_statements", r"pg_prepared_statements", r"general_log", r"slow_log",
|
||||
r"processlist", r"performance_schema", r"events_statements_\w+",
|
||||
r"sys[\"`]?\s*\.\s*[\"`]?(?:x\$)?(?:session|processlist|innodb_lock_waits|schema_table_lock_waits)",
|
||||
r"g?v\$sql\w*", r"(?:dba|cdb)_hist_sql\w*", r"g?v\$open_cursor",
|
||||
r"(?:dba|cdb|user)_(?:\w+_)?audit_trail", r"unified_audit_trail", r"aud\$\w*", r"fga_log\$",
|
||||
r"sqlite_dbpage",
|
||||
)
|
||||
#: 系统 schema:哪几张表采样了列值、记着别的会话的 SQL,一张张数靠不住。可验证报告整个挡(只决定附不附快照),工具边界
|
||||
#: 不挡(见 INDIRECT_NAMES)
|
||||
SYSTEM_SCHEMAS = (r"pg_catalog", r"information_schema")
|
||||
|
||||
|
||||
def _implicit(sql: str) -> bool:
|
||||
"""解析不了的 SQL 里有没有不写列名也会带出遮罩列的写法(*、TABLE x,_INDIRECT 那几类):它们能把遮罩列带进别的列名
|
||||
(列名表、UNION 的后一支)、整表交出来,同写着遮罩列一样整份遮(_derived_hidden)。只看文字,字符串、注释里的也算
|
||||
(多遮)。"""
|
||||
return bool(_IMPLICIT.search(_COUNT_STAR.sub("", sql)) or _INDIRECT.search(sql))
|
||||
def indirect_reads(*names: str) -> re.Pattern[str]:
|
||||
"""认「不点列名也读得出表里内容」的写法的正则:INDIRECT_FUNCTIONS 跟着左括号(函数名和括号之间可以有引号、空白),
|
||||
INDIRECT_NAMES 加上 names 按名字。工具边界(_INDIRECT)和可验证报告(bundle/build.py 的 _INDIRECT_READS,另加
|
||||
SYSTEM_SCHEMAS)共用这一份名单:以前两份各写各的,挡在模型前面的这一份反而少了一大截。"""
|
||||
return re.compile(r"(?<![0-9A-Za-z_$])(?:(?:" + "|".join(INDIRECT_FUNCTIONS) + r")[\"`]?\s*\(|(?:"
|
||||
+ "|".join((*INDIRECT_NAMES, *names)) + r")(?![0-9A-Za-z_$]))", re.I)
|
||||
|
||||
|
||||
#: 工具边界认的那一份(不含 SYSTEM_SCHEMAS)。出现就整份遮(_tainted、_implicit),写在字符串、注释里的也算(多遮)
|
||||
_INDIRECT = indirect_reads()
|
||||
|
||||
|
||||
def _opaque(sql: str, kind: str | None) -> bool:
|
||||
"""SQL 里有认不全的写法:OPAQUE_SQL(MySQL 会执行的注释、U&"…"),或者 _INDIRECT 那几类。按原文认(字符串、注释里
|
||||
写着的也算,多遮),再按去掉注释的认:函数名和括号之间夹着注释(query_to_xml/**/(…)、query_to_xml --换行(…))解析得了,
|
||||
语法树上又不认函数名,原文里名字后面不紧跟着括号,以前就放过去了。去注释照守卫按数据源的方言切(data/guard.py)。"""
|
||||
if OPAQUE_SQL.search(sql) or _INDIRECT.search(sql):
|
||||
return True
|
||||
from app.data.guard import strip_comments
|
||||
|
||||
try:
|
||||
bare = strip_comments(sql, dialect=kind)
|
||||
except Exception: # noqa: BLE001 - 守卫切不开的写法:认不准,按认不全算(宁可多遮)
|
||||
return True
|
||||
return bare != sql and bool(_INDIRECT.search(bare))
|
||||
|
||||
|
||||
def _implicit(sql: str, kind: str | None = None) -> bool:
|
||||
"""解析不了的 SQL 里有没有不写列名也会带出遮罩列的写法(*、TABLE x,OPAQUE_SQL、_INDIRECT 那几类,_opaque):它们能
|
||||
把遮罩列带进别的列名(列名表、UNION 的后一支)、整表交出来,同写着遮罩列一样整份遮(_derived_hidden)。只看文字,
|
||||
字符串、注释里的也算(多遮)。"""
|
||||
return bool(_IMPLICIT.search(_COUNT_STAR.sub("", sql))) or _opaque(sql, kind)
|
||||
|
||||
|
||||
class _Masks(NamedTuple):
|
||||
"""一份快照按 SQL 认出来的遮罩(_derived_masks):要遮的列(小写)、SQL 里指代遮罩列的名字(小写)、要收值的列(小写,
|
||||
collect:收进遮罩值的只有这几列,见 _derived_masks)、是不是认不全整份遮的(whole)。"""
|
||||
|
||||
hidden: set[str]
|
||||
names: set[str]
|
||||
collect: set[str]
|
||||
whole: bool
|
||||
|
||||
|
||||
def _derived_hidden(snapshot: dict[str, Any], hidden: set[str], kind: str | None,
|
||||
tables: dict[str, list[str]] | None = None, *, labels: dict[str, set[str]] | None = None
|
||||
) -> tuple[set[str], set[str]]:
|
||||
"""(这份快照要遮的列, SQL 里指代遮罩列的名字),都是小写。tables 是查询当时冻结的表结构(_schema_columns),* 展开
|
||||
数据源里的表时按它数位置;没给的按快照记着的表结构快照(schema_artifact)认,和工具边界遮的时候是同一份。
|
||||
"""(这份快照要遮的列, SQL 里指代遮罩列的名字),都是小写,见 _derived_masks。"""
|
||||
found = _derived_masks(snapshot, hidden, kind, tables, labels=labels)
|
||||
return found.hidden, found.names
|
||||
|
||||
|
||||
def _derived_masks(snapshot: dict[str, Any], hidden: set[str], kind: str | None,
|
||||
tables: dict[str, list[str]] | None = None, *, labels: dict[str, set[str]] | None = None) -> _Masks:
|
||||
"""一份快照要遮的列、SQL 里指代遮罩列的名字、要收值的列、是不是整份遮(_Masks),都是小写。tables 是查询当时冻结的
|
||||
表结构(_schema_columns),* 展开数据源里的表时按它数位置;没给的按快照记着的表结构快照(schema_artifact)认,和
|
||||
工具边界遮的时候是同一份。
|
||||
|
||||
证据面板只按列名遮(hidden);agent 的查询起了别名、套了函数,结果列名就对不上。本仓库的查询惯例是起中文别名
|
||||
(region AS 区域),所以按 SQL 认出改了名字的遮罩列(_tainted)。SQL 解析不了、里面又写着遮罩列的(*、TABLE x 不写
|
||||
列名也会把遮罩列带进别的列名,同样算,_implicit),整份快照的列都遮掉;合并查询的快照不看 SQL,它记着自己的来历
|
||||
(_hidden_columns 已按 _merge_masks 遮好)。
|
||||
列名也会把遮罩列带进别的列名,OPAQUE_SQL、_INDIRECT 那几类整表交出来,同样算,_implicit),或者数据源是有整行引用的
|
||||
方言(Postgres:row_to_json(s)、SELECT s 不写列名、也没有 *,只看文字认不出),整份快照的列都遮掉;合并查询的快照
|
||||
不看 SQL,它记着自己的来历(_hidden_columns 已按 _merge_masks 遮好)。
|
||||
分档写在派生来源里、外层按它分组的标签列(一档一行,_grouped_labels)照常显示,同 _band_only;格子里装着标签以外的、
|
||||
和遮罩列同名的照旧遮(_shown_labels)。labels 是调用方已经认好的标签列(_label_columns),没给的按快照认。"""
|
||||
和遮罩列同名的照旧遮(_shown_labels)。labels 是调用方已经认好的标签列(_label_columns),没给的按快照认。
|
||||
收值(collect):认得清的,要遮的列都收;整份遮的只收确定取自遮罩列的那几列——列名对得上遮罩列和 SQL 里指代遮罩列的
|
||||
名字的,加上整份算之前认出的位置(_Taint.sure)。以前整份遮时每一格都收成遮罩值,区域名、列名在同一次核查里处处被
|
||||
换掉,封存的 SQL 也被改写(WHERE 已遮罩 = '华东');认不全的那几列的值本来就没交给模型,不收也不会从别处露出来。"""
|
||||
base = set(hidden)
|
||||
sql = snapshot.get("sql")
|
||||
if not base or not isinstance(sql, str) or not sql.strip() or snapshot.get("source") == MERGE_SOURCE:
|
||||
return base, base
|
||||
return _Masks(base, base, base, False)
|
||||
columns = [str(c) for c in snapshot.get("columns") or []]
|
||||
lowered = {c.lower() for c in columns}
|
||||
found = _tainted(sql, kind, base, len(columns), _snapshot_columns(snapshot) if tables is None else tables)
|
||||
if found is None:
|
||||
hides = _mentions(sql, base) or _implicit(sql)
|
||||
return (base | {c.lower() for c in columns}, base) if hides else (base, base)
|
||||
names, positions, everything = found
|
||||
out = names | {columns[i].lower() for i in positions if i < len(columns)}
|
||||
if not everything and out & {c.lower() for c in columns}:
|
||||
from app.data.sql_semantics import _read
|
||||
|
||||
if _mentions(sql, base) or _implicit(sql, kind) or _read(kind) in _ROW_DIALECTS:
|
||||
return _Masks(base | lowered, base, base, True)
|
||||
return _Masks(base, base, base, False)
|
||||
out = found.names | {columns[i].lower() for i in found.positions if i < len(columns)}
|
||||
if found.whole:
|
||||
sure = found.names | {columns[i].lower() for i in found.sure if i < len(columns)}
|
||||
return _Masks(out, found.names, sure, True)
|
||||
if out & lowered:
|
||||
out -= _shown_labels(snapshot, kind, columns, base, tables, labels)
|
||||
return out, names
|
||||
return _Masks(out, found.names, out, False)
|
||||
|
||||
|
||||
def _shown_labels(snapshot: dict[str, Any], kind: str | None, columns: list[str], base: set[str],
|
||||
@@ -1586,8 +1734,7 @@ def derived_columns(snapshot: dict[str, Any], hidden: set[str], kind: str | None
|
||||
found = _tainted(sql, kind, set(hidden), len(columns), _snapshot_columns(snapshot))
|
||||
if found is None:
|
||||
return None
|
||||
names, positions, _everything = found
|
||||
return {c.lower() for i, c in enumerate(columns) if c.lower() in names or i in positions}
|
||||
return {c.lower() for i, c in enumerate(columns) if c.lower() in found.names or i in found.positions}
|
||||
|
||||
|
||||
def _snapshot_labels(snapshot: dict[str, Any], kind: str | None, tables: dict[str, set[str]] | None = None
|
||||
@@ -1976,6 +2123,24 @@ class _Secrets:
|
||||
self.weak = {f for f in self.weak if not asked(f)}
|
||||
self._index, self._compiled = None, {}
|
||||
|
||||
def spared(self, value: str) -> bool:
|
||||
"""value(SQL 里的一个字面量、一格的文字)写在已经摆在读者眼前的文字里(protected):认法同 _spare,数按去掉
|
||||
千分位、正负号的数字认,换了写法的号码按纯数字写法认,别的按原文。value 不必是收过的遮罩值——证据摘录里和遮罩列
|
||||
做等值比较的字面量不论认不认得出都换,提问人、这句结论原文里写着的照旧不换(inquiry._scrub_excerpt_sql)。"""
|
||||
text = _plain_text(value)
|
||||
if not text or not self.protected:
|
||||
return False
|
||||
given = " ".join(self.protected)
|
||||
if _digits_of(text) in {_digits_of(n) for n in _DIGITS.findall(given)} if _NUMERIC.fullmatch(text) \
|
||||
else text in given:
|
||||
return True
|
||||
keys = set(_loose_keys(text))
|
||||
if not keys:
|
||||
return False
|
||||
runs = {"".join(_LOOSE_DIGITS.findall(run.group())).translate(_HALF_DIGITS)
|
||||
for run in _LOOSE_RUN.finditer(given)}
|
||||
return any(key in keys for run in runs for key in _loose_candidates(run))
|
||||
|
||||
def show(self, numbers: Iterable[str]) -> None:
|
||||
"""SQL 里照原样留着的阈值、行数(_shown_literals):回答正文里复述它们不换(「金额不低于 30000 元」),换了只会
|
||||
让回答和 SQL 对不上。只免正文:格子、列名、SQL 里照旧按值换——格子里的值正好是这个数,可能就是那个人
|
||||
@@ -2187,8 +2352,7 @@ def _masked_view(snapshot: dict[str, Any], hidden: set[str], secrets: _Secrets,
|
||||
edited = False
|
||||
position = {c: i for i, c in enumerate(columns)}
|
||||
names = [secrets.scrub(c, loose=True) for c in columns] if by_value and rename else columns
|
||||
kinds = snapshot.get("column_types") if isinstance(snapshot.get("column_types"), dict) else {}
|
||||
counted = {i for i, c in enumerate(columns) if kinds.get(c) == "number"} if not numbers else set()
|
||||
counted = _typed_numbers(snapshot, columns) if not numbers else set()
|
||||
|
||||
def cell(r: int, i: int | None, value: Any) -> Any:
|
||||
nonlocal edited
|
||||
@@ -2198,7 +2362,7 @@ def _masked_view(snapshot: dict[str, Any], hidden: set[str], secrets: _Secrets,
|
||||
if i is not None:
|
||||
touched.add(i)
|
||||
return value
|
||||
if not numbers and (i in counted or (isinstance(value, (int, float)) and not isinstance(value, bool))):
|
||||
if not numbers and (i in counted or _number_cell(value)):
|
||||
return value
|
||||
if by_value and secrets.hit(value):
|
||||
if i is not None:
|
||||
@@ -2239,6 +2403,38 @@ def _masked_view(snapshot: dict[str, Any], hidden: set[str], secrets: _Secrets,
|
||||
return view, [names[i] for i in sorted(touched)]
|
||||
|
||||
|
||||
def _number_cell(value: Any) -> bool:
|
||||
"""格子本身是数(整数、小数,真假不算)。写成字符串的数(Postgres 的 NUMERIC、MySQL 的 DECIMAL 交回来是 "20000.00")
|
||||
单看格子分不出是数还是文字,要靠列类型(_typed_numbers)。"""
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool)
|
||||
|
||||
|
||||
def _typed_numbers(snapshot: dict[str, Any], columns: list[str]) -> set[int]:
|
||||
"""按列类型认作数值列的列号:column_types 记成 number、而且结果里只有这一列叫这个名字。column_types 按列名记(查询层
|
||||
逐列数类型,同名的几列后面的盖掉前面的),同名的几列说不清记的是哪一列的类型:备注是文字、另一张表里同名的评分是
|
||||
整数,记下的是 number,以前按名字查,文字列也当成了数值列,里面的号码不再按值换,原样交给模型、落了库。同名的几列
|
||||
按格子本身认(_number_cell:整数、小数才算数,写成字符串的数按文字处理,宁可多遮)。"""
|
||||
kinds = snapshot.get("column_types") if isinstance(snapshot.get("column_types"), dict) else {}
|
||||
counts = Counter(columns)
|
||||
return {i for i, c in enumerate(columns) if kinds.get(c) == "number" and counts[c] == 1}
|
||||
|
||||
|
||||
def masked_cell(value: Any, secrets: _Secrets | None, *, numeric: bool = False) -> Any:
|
||||
"""一格按值遮好的样子,口径同工具边界(_masked_view 的 numbers=False、embedded):数值格(格子是数,或者 numeric:
|
||||
列类型是 number、列名唯一的列,见 _typed_numbers)照原样,不改写没设遮罩的数值证据;文字格整格就是遮罩值的换成
|
||||
「已遮罩」,文字里夹着的换掉那一截(换了写法的号码也认),JSON(对象、数组)按界面画出来的样子换、换过的照文字给。
|
||||
交给核查 agent 的证据摘录逐格这样遮(judge.prepare 的 mask),不再对渲染好的整段文字按值换:那样分不出数值格和文字
|
||||
格,没设遮罩的访问次数正好等于某一笔金额也被换掉。"""
|
||||
if secrets is None or not secrets.strong or value is None or numeric or _number_cell(value):
|
||||
return value
|
||||
if secrets.hit(value):
|
||||
return _masked_word()
|
||||
text = _cell_text(value)
|
||||
if text is not None and (scrubbed := secrets.scrub(text, loose=True)) != text:
|
||||
return scrubbed
|
||||
return value
|
||||
|
||||
|
||||
def _cell_text(value: Any) -> str | None:
|
||||
"""一格按文字换的样子(_masked_view 的 embedded):文字照原样,JSON(对象、数组)按界面画出来的样子;别的为 None。"""
|
||||
if isinstance(value, str):
|
||||
@@ -2373,6 +2569,9 @@ class QueryMask:
|
||||
columns: list[str]
|
||||
secrets: _Secrets
|
||||
cells: list[list[int]] = field(default_factory=list)
|
||||
#: 认不全、整份结果都遮了(_derived_masks 的 whole,分析出了意外的兜底也算):随快照存下(masked_whole,不交给模型),
|
||||
#: 格子引用指到这份结果时照实说写法认不全,不说成「取自设置了遮罩的列」
|
||||
whole: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -2405,23 +2604,25 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str
|
||||
(schema_cache,和随快照冻结的是同一份):认分档标签(_label_columns 的 tables)、数 * 展开数据源里的表落在第几列
|
||||
(_derived_hidden 的 tables)时用。
|
||||
|
||||
- 整列遮:列名直接对上遮罩列的,加上按 SQL 认出值取自遮罩列的(_derived_hidden:别名、函数、子查询、WITH、
|
||||
- 整列遮:列名直接对上遮罩列的,加上按 SQL 认出值取自遮罩列的(_derived_masks:别名、函数、子查询、WITH、
|
||||
SELECT *(展开的列换了名字的也算)、表值函数、整行引用和函数参数里的 *、列名表、TABLE x;COUNT、只在条件里用到、
|
||||
分档当分组键的不算,分档写在派生来源里、外层按它分组的也不算);认不全的(SQL 解析不了又写着遮罩列、*、TABLE x,
|
||||
(表达式).* 读到遮罩列,表名、SQL 写在字符串里交给函数,分析出意外、超出工作量上限)整份结果的列都遮。这几列的
|
||||
格子换成「已遮罩」,类型记成文本
|
||||
或者是 Postgres 的;(表达式).* 读到遮罩列,MySQL 会执行的注释、U&"…",表名、SQL 写在字符串里交给函数,统计信息、
|
||||
会话记录里存着值的视图,选出项和结果的列数对不上,分析出意外、超出工作量上限)整份结果的列都遮(QueryMask.whole)。
|
||||
这几列的格子换成「已遮罩」,类型记成文本
|
||||
- 别的列按值换,只换文字格:整格就是遮罩值的换掉,文字格里夹着的那一截换掉(换了写法的号码也认,
|
||||
_Secrets.loose_spans),换过的格子记下来(QueryMask.cells)。遮罩值是这份结果里的加上 known:先查号码、再查写着
|
||||
号码的备注、没设遮罩的联系方式表(别的库里的也一样)时,后一份同样遮好。门槛同按值遮(SECRET_DIGITS、SECRET_MIN);
|
||||
只装着分档标签的列里的标签不算遮罩值,这份结果自己写的标签也不按 known 换(前面的查询认不准、当遮罩值收了的同名
|
||||
标签,不能把这一份的档位列换掉)。数值格(数、列类型是 number 的格子)不按值换:存下的快照就是证据,没设遮罩的
|
||||
数值列(访问次数正好等于某一笔被遮的金额)换掉就改写了证据,双盲复算也取不到它
|
||||
标签,不能把这一份的档位列换掉)。数值格(数、列类型是 number 而且列名唯一的格子,_typed_numbers)不按值换:存下的
|
||||
快照就是证据,没设遮罩的数值列(访问次数正好等于某一笔被遮的金额)换掉就改写了证据,双盲复算也取不到它
|
||||
- 列名不换(模型自己写的,换了它引用的列名就对不上);结果里一列都没取自遮罩列、也没有可按值换的原样交回:数值的
|
||||
汇总、只数个数的查询不受影响
|
||||
|
||||
交回的 secrets 只有这份结果里的遮罩值(known 调用方自己有)。分析本身出了意外(认列、认标签、逐格换值哪一步都算),
|
||||
整份结果的每一格都遮、全部的列记成查询时遮掉的(_masked_whole):这一步挡在模型前面,分析的缺陷不能把原值放过去
|
||||
(宁可多遮)。"""
|
||||
交回的 secrets 只有这份结果里的遮罩值(known 调用方自己有);整份遮的只收确定取自遮罩列的那几列(_Masks.collect),
|
||||
不然区域名、列名都成了遮罩值,后面的查询、封存的 SQL 跟着被换掉。分析本身出了意外(认列、认标签、逐格换值哪一步都算),
|
||||
整份结果的每一格都遮、全部的列记成查询时遮掉的(_masked_whole),一个值都不收:这一步挡在模型前面,分析的缺陷不能
|
||||
把原值放过去(宁可多遮)。"""
|
||||
base = {str(c).strip().lower() for c in masked if str(c).strip()}
|
||||
columns = [str(c) for c in payload.get("columns") or []]
|
||||
given = known is not None and bool(known.strong)
|
||||
@@ -2429,10 +2630,13 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str
|
||||
return QueryMask(payload, [], _Secrets())
|
||||
try:
|
||||
labels = _snapshot_labels(payload, kind, _schema_tables(schema))
|
||||
hidden = _derived_hidden(payload, base, kind, _schema_columns(schema), labels=labels)[0] if base else set()
|
||||
found = _derived_masks(payload, base, kind, _schema_columns(schema), labels=labels) if base \
|
||||
else _Masks(set(), set(), set(), False)
|
||||
hidden = found.hidden
|
||||
own = _Secrets()
|
||||
if any(c.lower() in hidden for c in columns):
|
||||
own.add(payload, hidden, labels=labels, base=base)
|
||||
# 收值只收确定取自遮罩列的那几列(整份遮时不是每一列都收,见 _derived_masks)
|
||||
if any(c.lower() in found.collect for c in columns):
|
||||
own.add(payload, found.collect, labels=labels, base=base)
|
||||
pool = own
|
||||
if given:
|
||||
pool = own.copy()
|
||||
@@ -2447,8 +2651,8 @@ def mask_query_result(payload: dict[str, Any], masked: Iterable[str], kind: str
|
||||
view, _ = _masked_view(payload, hidden, pool, embedded=True, rename=False, numbers=False, changed=cells)
|
||||
except Exception: # noqa: BLE001 - 认不准就整份遮掉,原值不交给模型
|
||||
logger.warning("核查查询的遮罩分析出错,整份结果按遮罩处理", exc_info=True)
|
||||
return QueryMask(_masked_whole(payload), columns, _Secrets())
|
||||
return QueryMask(view, [c for c in columns if c.lower() in hidden], own, cells)
|
||||
return QueryMask(_masked_whole(payload), columns, _Secrets(), whole=True)
|
||||
return QueryMask(view, [c for c in columns if c.lower() in hidden], own, cells, whole=found.whole)
|
||||
|
||||
|
||||
def _masked_whole(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
|
||||
@@ -11,10 +11,12 @@
|
||||
的 Q1 分开),要它先想清楚「如果这句不成立,数据里会看到什么」,写只读查询去找,最后交结论(经受住 / 被削弱 /
|
||||
被推翻 / 无法判断)和理由。理由里的数字只能引用它自己查询的格子,裸数字由 compose_doc 标出来。
|
||||
遮罩:agent 的查询结果在工具边界就遮好了;这份报告引用的查询快照里的遮罩值(按 SQL 认出改了名字的遮罩列,同审批
|
||||
质询)反查之前就交给查询工具,agent 查到写着同一个号码的别的表时也看不到。任务里的证据摘录先遮:遮罩列、按 SQL
|
||||
认出改了名字的遮罩列(buyer_phone AS 手机号)不给,别的列、SQL 里写着的遮罩值按值换(_task)。理由正文和理由文档
|
||||
仍按值换成「已遮罩」(换了写法的号码也认,agent 查询时遮掉的值也算),这句结论原文里写着的不换(_masked_rationale);
|
||||
换完还认得出的,这句记为无法判断、理由不记(_kept,同审批质询的最后一道关)
|
||||
质询)反查之前就交给查询工具,agent 查到写着同一个号码的别的表时也看不到。任务里的证据摘录先遮(_task,和工具
|
||||
边界同一套口径):遮罩列、按 SQL 认出改了名字的遮罩列(buyer_phone AS 手机号)不给,文字格、SQL 的字符串和注释里
|
||||
的遮罩值按值换,和遮罩列做等值比较的字面量都换;数值格、范围比较的阈值、LIMIT 照原样。理由正文和理由文档仍按值换成
|
||||
「已遮罩」(换了写法的号码也认,agent 查询时遮掉的值也算),这句结论原文里写着的、agent 自己 SQL 里照原样留着的
|
||||
阈值和行数不换,目录里的列名同质询照快照换(_masked_rationale);换完正文、原文里还认得出的,这句记为无法判断、
|
||||
理由不记(_kept,同审批质询的最后一道关)
|
||||
3. 出处:一条查询都没执行成功的,判什么都记为无法判断;判「被推翻」的,理由里还得引用到自己的查询结果——它要让
|
||||
正式出具降档,审稿记录里得查得到凭什么。不采信时系统写明原因,反方原本的结论和理由照录在后面(纯文字:解析
|
||||
不了的引用写成「(引用无效)」,裸数字注明「(无出处)」)。这句该查的数据源都不存在或已停用、核查模型本身用
|
||||
@@ -50,7 +52,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable, Collection, Iterable
|
||||
@@ -385,12 +386,6 @@ def _claim_of(cand: judging.Candidate, outcome: VerifyOutcome) -> tuple[dict[str
|
||||
return _claim(cand, verdict, rationale, doc=outcome.rationale, queries=outcome.queries, usage=outcome.usage), None
|
||||
|
||||
|
||||
def _scrubbed_excerpt(text: str, known: Any) -> str:
|
||||
"""证据摘录里的遮罩值换成「已遮罩」:别的列、SQL 里写着的(换了写法的号码也认),交给核查 agent 之前就换掉。值是
|
||||
报告快照里的遮罩值去掉这句结论原文里写着的(_spared),同交给查询工具的那一份。"""
|
||||
return known.scrub(text, loose=True) if known is not None and known.strong else text
|
||||
|
||||
|
||||
def _spared(seed: Any, claim: str) -> Any:
|
||||
"""报告快照里的遮罩值(seed)去掉这句结论原文里写着的,交给这一句的反查。seed 本身不动(下一句另算)。"""
|
||||
known = seed.copy()
|
||||
@@ -398,14 +393,18 @@ def _spared(seed: Any, claim: str) -> Any:
|
||||
return known
|
||||
|
||||
|
||||
def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[str | None]) -> VerifyOutcome:
|
||||
def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[str | None],
|
||||
kinds: dict[str, Any] | None = None, masks: dict[str, Any] | None = None) -> VerifyOutcome:
|
||||
"""反方的理由里的遮罩值换成「已遮罩」,换了写法的号码也认(199-1474-2785、+86 …)。值是这份报告引用的查询快照里的
|
||||
(secrets,反查之前就交给了查询工具),加上 agent 查询时在工具边界遮掉的(outcome.masked_values);被审的那句结论
|
||||
原文里写着的(given)已经摆在读者眼前,不换。反方 agent 的查询结果在工具边界就遮好了,可证据摘录按列名遮,改了名字
|
||||
的遮罩列(buyer_phone AS 手机号)的原值它看得到,会写进理由。
|
||||
换的办法同审批质询的回答(_scrubbed_source):原文里的值先换掉,再照原来的目录重新组装理由文档;引用到的格子按
|
||||
遮好的快照渲染(文字格里夹着的遮罩值换掉那一截)。理由里认不出遮罩值的原样交回。换完还认得出的过最后一道关(_kept)。"""
|
||||
原文里写着的(given)已经摆在读者眼前,不换。agent 自己 SQL 里照原样留着的范围边界、LIMIT 的行数(_shown:kinds 是
|
||||
数据源的类型,masks 是数据源现在设的遮罩列)在正文里复述也不换,同审批质询的回答(「金额不低于 20000 元」)。
|
||||
反方 agent 的查询结果在工具边界就遮好了、证据摘录也先遮了,这里是第二道防线。
|
||||
换的办法同审批质询的回答(_scrubbed_source):原文里的值先换掉,再照换过列名的目录重新组装理由文档(同质询的
|
||||
_renamed_entry:快照遮好的样子列名换过了,引用对得上);引用到的格子按遮好的快照渲染(文字格里夹着的遮罩值换掉
|
||||
那一截)。理由、目录列名里都认不出遮罩值的原样交回。换完还认得出的过最后一道关(_kept)。"""
|
||||
from app.engine.evidence import compose_doc
|
||||
from app.engine.inquiry import _leaked, _renamed_entry
|
||||
from app.engine.masking import _masked_view, _scrubbed_source, _Secrets
|
||||
|
||||
doc, result = outcome.rationale, outcome.result
|
||||
@@ -416,9 +415,13 @@ def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[s
|
||||
pool.protect(given)
|
||||
if result is None or not isinstance(doc, dict) or not pool.strong:
|
||||
return outcome
|
||||
pool.show(_shown(outcome, pool, kinds or {}, masks or {}))
|
||||
pool.prepare()
|
||||
source = str(doc.get("source") or "")
|
||||
if not (pool.spans(str(doc.get("markdown") or ""), prose=True) or pool.spans(source, prose=True)):
|
||||
catalog = doc.get("catalog") if isinstance(doc.get("catalog"), dict) else {}
|
||||
renamed = {alias: _renamed_entry(entry, pool) for alias, entry in catalog.items()}
|
||||
# 正文里认得出遮罩值的(照原样留着的阈值写进单元格引用里的也算,_leaked)、原文、目录列名里有的,才重新组装
|
||||
if renamed == catalog and not (_leaked(doc, pool) or pool.spans(source, prose=True)):
|
||||
return _kept(outcome, pool)
|
||||
views: dict[str, Any] = {}
|
||||
fetch = _fetcher()
|
||||
@@ -430,8 +433,8 @@ def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[s
|
||||
return views[artifact] if artifact in views else artifact_store.load(artifact)
|
||||
|
||||
try:
|
||||
again = compose_doc(_scrubbed_source(source, pool), doc.get("catalog") or {},
|
||||
node_id=str(doc.get("node_id") or ""), run_id=doc.get("run_id"), loader=load)
|
||||
again = compose_doc(_scrubbed_source(source, pool), renamed, node_id=str(doc.get("node_id") or ""),
|
||||
run_id=doc.get("run_id"), loader=load)
|
||||
except Exception: # noqa: BLE001 - 重新组装出了意外,值也不能留着
|
||||
# 正文按值换掉,理由文档不要了(界面照正文显示;判「被推翻」的因此查不到出处,按无法判断记)
|
||||
said = pool.scrub(str(doc.get("markdown") or ""), prose=True)
|
||||
@@ -439,15 +442,39 @@ def _masked_rationale(outcome: VerifyOutcome, secrets: Any, *, given: Iterable[s
|
||||
return _kept(replace(outcome, rationale=again, result={**result, RATIONALE: again["markdown"]}), pool)
|
||||
|
||||
|
||||
def _shown(outcome: VerifyOutcome, pool: Any, kinds: dict[str, Any], masks: dict[str, Any]) -> set[str]:
|
||||
"""反方 agent 自己的 SQL 里照原样留着的数(范围比较的边界、LIMIT 的行数,审批质询的 _shown_literals):理由正文里
|
||||
复述它们不换,最后一道关也不算露了遮罩值——它们随 SQL 存进审稿记录、摆在读者眼前,换掉只会让理由和 SQL 对不上。
|
||||
哪些名字是遮罩列按那份快照认(_derived_hidden 的名字,含改了名字的),快照取不到的按数据源现在设的遮罩列。"""
|
||||
from app.engine.inquiry import _shown_literals
|
||||
from app.engine.masking import _derived_hidden
|
||||
|
||||
fetch = _fetcher()
|
||||
sqls: list[tuple[str, str | None, set[str]]] = []
|
||||
for query in outcome.queries:
|
||||
name = str(query.get("source") or "")
|
||||
base = {str(c).lower() for c in masks.get(name) or []}
|
||||
snapshot = fetch(query.get("artifact"))
|
||||
if isinstance(snapshot, dict):
|
||||
base |= {str(c).lower() for c in snapshot.get("mask_columns") or [] if isinstance(c, str)}
|
||||
compared = _derived_hidden(snapshot, base, kinds.get(name))[1] if isinstance(snapshot, dict) and base \
|
||||
else base
|
||||
sqls.append((str(query.get("sql") or ""), kinds.get(name), compared))
|
||||
return _shown_literals(sqls, pool)
|
||||
|
||||
|
||||
def _kept(outcome: VerifyOutcome, pool: Any) -> VerifyOutcome:
|
||||
"""最后一道关(同审批质询的 _leaked):按值遮之后,理由正文、理由文档(正文、原文、目录里的列名)里还认得出遮罩值的
|
||||
(换了写法的号码也算),这句的结论不采信、理由不记:记为无法判断,理由换成系统的说明(LEAKED)。遮不住的「被推翻」
|
||||
不能让出具降档,也不能把号码存进审稿记录、随运行封存。"""
|
||||
"""最后一道关(同审批质询的 _leaked):按值遮之后,理由正文、理由文档的正文和原文里还认得出遮罩值的(换了写法的号码
|
||||
也算),这句的结论不采信、理由不记:记为无法判断,理由换成系统的说明(LEAKED)。遮不住的「被推翻」不能让出具降档,
|
||||
也不能把号码存进审稿记录、随运行封存。只看正文和原文:目录里的列名是 agent 自己起的名字(不低于20000的笔数),
|
||||
已经按值换过(_masked_rationale),再拿来认会把恰好等于遮罩值的阈值当成露了值,整句改判;agent 自己 SQL 里照原样
|
||||
留着的阈值写在正文里不算(pool.show),渲染进单元格引用里的照算(同质询的 _leaked)。"""
|
||||
from app.engine.inquiry import _leaked
|
||||
|
||||
doc = outcome.rationale if isinstance(outcome.rationale, dict) else {}
|
||||
texts = [str((outcome.result or {}).get(RATIONALE) or ""), str(doc.get("markdown") or ""),
|
||||
str(doc.get("source") or ""),
|
||||
json.dumps(doc.get("catalog") or {}, ensure_ascii=False, default=str)]
|
||||
if not any(pool.spans(text, prose=True) for text in texts):
|
||||
said = str((outcome.result or {}).get(RATIONALE) or "")
|
||||
if not (pool.spans(said, prose=True) or pool.spans(str(doc.get("source") or ""), prose=True)
|
||||
or (doc and _leaked(doc, pool))):
|
||||
return outcome
|
||||
logger.warning("反方审稿的理由里有遮罩值遮不住,这句记为无法判断(运行 %s)", doc.get("run_id"))
|
||||
return replace(outcome, rationale=None, result={**(outcome.result or {}), "verdict": "inconclusive",
|
||||
@@ -632,15 +659,16 @@ class _Review:
|
||||
"""交给核查 agent 的任务:这句结论、它挂的证据摘录(原报告的编号标明,见 mark_original)、可以查的数据源;该查的
|
||||
源有停用了的,也告诉它查不了哪几个,免得它把查不到当成没有反例。own:源是这句自己引用的,不是没挂依据、按整份
|
||||
报告用到的源查的。
|
||||
摘录先遮,模型从任务里也看不到遮罩值:遮罩列不给,改了名字的(buyer_phone AS 手机号)按 SQL 认出来也不给
|
||||
(judge.prepare 的 kinds,和工具边界同一套规则);别的列、SQL 里写着的遮罩值按值换成「已遮罩」,换了写法的号码
|
||||
也认(known:报告快照里的遮罩值去掉这句结论原文里写着的,同交给查询工具的那一份)。解析 SQL、按值换是纯计算,
|
||||
在线程里做。"""
|
||||
摘录先遮,模型从任务里也看不到遮罩值(judge.prepare 的 mask,inquiry.ExcerptMask,和工具边界同一套口径):遮罩列、
|
||||
按 SQL 认出改了名字的(buyer_phone AS 手机号)不给;文字格、SQL 的字符串和注释里的遮罩值按值换成「已遮罩」,换了
|
||||
写法的号码也认,和遮罩列做等值比较的字面量都换;数值格、范围比较的阈值、LIMIT 照原样(known:报告快照里的遮罩值
|
||||
去掉这句结论原文里写着的,同交给查询工具的那一份)。解析 SQL、按值换是纯计算,在线程里做。"""
|
||||
from app.engine.inquiry import ExcerptMask # inquiry 导入了这个模块,用到时再导入
|
||||
|
||||
request = judging.prepare(self.doc, self.catalog, units=[cand.unit], loader=self.fetch, masked=self.masks,
|
||||
kinds=self.kinds)
|
||||
mask=ExcerptMask(self.kinds, known))
|
||||
aliases = dict.fromkeys(a for c in request.cands for a in c.aliases if a in request.excerpts)
|
||||
evidence = "\n\n".join(mark_original(_scrubbed_excerpt(request.excerpts[a], known), a, self.queries)
|
||||
for a in aliases)
|
||||
evidence = "\n\n".join(mark_original(request.excerpts[a], a, self.queries) for a in aliases)
|
||||
gone = f"({'这句还引用了' if own else '这份报告还用到了'}{_quoted(lost)},已不存在或已停用,无法查询)" \
|
||||
if lost else ""
|
||||
return CHALLENGE_TASK_PROMPT.format(
|
||||
@@ -732,7 +760,8 @@ class _Review:
|
||||
output_schema=CHALLENGE_SCHEMA, max_queries=self.max_queries, timeout_s=allot.timeout_s,
|
||||
daily_max_usd=self.daily_max_usd, emit=deadline.relabel(ctx.emit, allot), run_id=ctx.run.run_id,
|
||||
node_id=ctx.node.id, data_versions=run_versions(ctx.run), masked_values=known)
|
||||
outcome = await asyncio.to_thread(_masked_rationale, outcome, known, given=[cand.text])
|
||||
outcome = await asyncio.to_thread(_masked_rationale, outcome, known, given=[cand.text], kinds=self.kinds,
|
||||
masks=self.masks)
|
||||
if outcome.error and not outcome.model:
|
||||
broken = outcome.error # 连模型都没建成(没配好、接入出错):换一句也一样
|
||||
model = model or outcome.model
|
||||
|
||||
@@ -12,8 +12,8 @@
|
||||
(masked_values:审批质询、反方审稿收的报告快照里的值),文字格里夹着的换那一截(engine/masking.py 的
|
||||
mask_query_result);数值格不按值换(存下的是证据,没设遮罩的访问次数正好等于某一笔被遮的金额也照原样);查询失败时
|
||||
报错原话里的值也换。模型看不到遮罩值,就不会把它写进回答、理由、SQL;以前是事后按值认,每一轮复核都找到新的漏法。
|
||||
快照里的 masked_at_query 记下查询时就遮了的列,masked_cells 记下按值换过的格子,格子引用指到这几列、这几格的都不采信,
|
||||
照实说是遮罩。遮掉的值只在内存里交回调用方(VerifyOutcome.masked_values,审批质询、反方审稿按值遮的第二道防线
|
||||
快照里的 masked_at_query 记下查询时就遮了的列,masked_cells 记下按值换过的格子,masked_whole 记下写法认不全、整份
|
||||
都遮了,格子引用指到这几列、这几格的都不采信,照实说是遮罩(整份遮的说写法认不全)。遮掉的值只在内存里交回调用方(VerifyOutcome.masked_values,审批质询、反方审稿按值遮的第二道防线
|
||||
用),不进快照、不交给模型、不进 as_dict。调用方交进来的和查询时遮掉的分开记(masking.SeenValues),合起来时 agent
|
||||
查到的在前:合起来到了上限 SECRETS_MAX,挤掉的是报告里的值。
|
||||
|
||||
@@ -428,10 +428,12 @@ class _Cells:
|
||||
"reason": f"{alias} {e}"})
|
||||
return None
|
||||
if str(name).lower() in _masked_at_query(content):
|
||||
# 查询时就遮了的列:格子里是「已遮罩」,不是数据源里的值。照实说是遮罩列,不说成「不是数值」
|
||||
# 查询时就遮了的列:格子里是「已遮罩」,不是数据源里的值。照实说是遮罩列,不说成「不是数值」;认不全、整份
|
||||
# 遮的(快照记着 masked_whole)说是写法认不全——这一列不一定取自遮罩列,说成遮罩列会让审批人以为它设了遮罩
|
||||
why = f"{alias} 的写法认不全,整份结果查询时已遮罩,不采信" if _masked_whole(content) \
|
||||
else f"{alias} 的「{name}」取自设置了遮罩的列,查询时已遮罩,不采信"
|
||||
self.cells.append({"path": path, "status": "unresolved", "ref": f"{alias}.r{row}.{name}", "alias": alias,
|
||||
"row": row, "column": name, "artifact": query["artifact"],
|
||||
"reason": f"{alias} 的「{name}」取自设置了遮罩的列,查询时已遮罩,不采信"})
|
||||
"row": row, "column": name, "artifact": query["artifact"], "reason": why})
|
||||
return None
|
||||
if raw == _masked_word() and isinstance(content, dict) and "masked_at_query" in content:
|
||||
# 这一格查询时按值换掉了(和别的查询、报告里的遮罩值相同):同样照实说
|
||||
@@ -547,6 +549,11 @@ def _masked_at_query(content: Any) -> set[str]:
|
||||
return {c.lower() for c in listed if isinstance(c, str)} if isinstance(listed, list) else set()
|
||||
|
||||
|
||||
def _masked_whole(content: Any) -> bool:
|
||||
"""查询快照记着这条 SQL 的写法认不全、整份结果查询时都遮了(tools/datasource 记下的 masked_whole)。"""
|
||||
return isinstance(content, dict) and content.get("masked_whole") is True
|
||||
|
||||
|
||||
def _masked_cell(content: Any, row: Any, name: str) -> bool:
|
||||
"""查询快照里第 row 行、name 这一列的格子查询时按值换过(tools/datasource 记下的 masked_cells:[行, 列号])。列名
|
||||
同名的按第一列算,同 locate_cell。"""
|
||||
|
||||
@@ -535,6 +535,7 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False,
|
||||
# 别的列里按值换过的格子另记(masked_cells),没有就不记
|
||||
masked_now: list[str] | None = None
|
||||
masked_cells: list[list[int]] = []
|
||||
masked_whole = False
|
||||
if mask_results and (mask or known is not None):
|
||||
from app.engine.masking import mask_query_result
|
||||
|
||||
@@ -543,6 +544,7 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False,
|
||||
schema=source.schema_cache)
|
||||
if mask or done.payload is not payload:
|
||||
payload, masked_now, masked_cells = done.payload, done.columns, done.cells
|
||||
masked_whole = done.whole
|
||||
if sink is not None:
|
||||
# 并进 agent 查到的那一份(不和调用方交进来的混在一起,上限收满了也不丢),也放进线程
|
||||
await asyncio.to_thread(sink.add, done.secrets)
|
||||
@@ -567,6 +569,8 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False,
|
||||
# 改了名字的也在这里)。有这一项的快照,别的格子整格是「已遮罩」的是查询时按值换掉的;
|
||||
# - masked_cells:别的列里查询时按值换过的文字格 [行, 列号](整格换的、换掉一截的),格子引用指到它们时照实
|
||||
# 说是遮罩。数值格不按值换,存下的数和数据库里的一样;
|
||||
# - masked_whole:这条 SQL 的写法认不全,整份结果查询时都遮了(masked_at_query 是全部的列),格子引用指到它时
|
||||
# 照实说写法认不全,不说成取自遮罩列;
|
||||
# - data_version:上传表格查的是哪个快照,事后据此认得出这个数出自哪一版数据。
|
||||
# 手工源、没设遮罩的,快照和原来一字不差
|
||||
extra: dict[str, Any] = {}
|
||||
@@ -576,6 +580,8 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False,
|
||||
extra["masked_at_query"] = masked_now
|
||||
if masked_cells:
|
||||
extra["masked_cells"] = masked_cells
|
||||
if masked_whole:
|
||||
extra["masked_whole"] = True
|
||||
if version := getattr(source, "snapshot_id", None):
|
||||
extra["data_version"] = version
|
||||
# 工件引用的 meta 记下源和 SQL 里用到的表:数据目录按它数每张表被运行查询过几次
|
||||
|
||||
@@ -32,6 +32,7 @@ SCRIPT = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_phone
|
||||
" (2, '华东', '13900000002', 300, 1, ''), (3, '华南', '13900000003', 900, 0, '例行回访'),"
|
||||
" (4, '华北', '13900000004', 400, 0, '');")
|
||||
SECRETS = ("1390000000", "0000-0002", "139 0000")
|
||||
MASKED = "已遮罩"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -118,3 +119,128 @@ async def test_a_rationale_still_showing_a_masked_value_is_not_kept(monkeypatch,
|
||||
assert _clean(json.dumps(record, ensure_ascii=False))
|
||||
out = await output_of(row.id, "chal")
|
||||
assert out["counts"]["refuted"] == 0, "遮不住的理由不能让出具降档"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 摘录分段遮:阈值、没设遮罩的数值格照原样,和遮罩列做等值比较的字面量都换
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
#: 金额、客户名设了遮罩,金额都是五位数;访问次数没设遮罩,第二、三、四单正好等于别的订单的金额
|
||||
BIG = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_name TEXT, amount INTEGER, visits INTEGER);"
|
||||
"INSERT INTO sales VALUES (1, '华东', '张伟', 10000, 25000), (2, '华东', '李娜', 12000, 20000),"
|
||||
" (3, '华南', '王芳', 20000, 30000), (4, '华北', '刘洋', 30000, 50000), (5, '华北', '赵敏', 50000, 9000);")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def big_shop(tmp_path):
|
||||
path = tmp_path / f"{uuid.uuid4().hex[:8]}.db"
|
||||
db = sqlite3.connect(path)
|
||||
db.executescript(BIG)
|
||||
db.commit()
|
||||
db.close()
|
||||
name = f"shop_{uuid.uuid4().hex[:8]}"
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=name, kind="sqlite", database=str(path), readonly=True, enabled=True,
|
||||
options={"mask_columns": ["amount", "buyer_name"]})
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
row.schema_cache = await introspect.introspect(row)
|
||||
await session.commit()
|
||||
source_id = row.id
|
||||
yield name
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
def _excerpt_of(task: str) -> str:
|
||||
return task[task.index("【原报告 Q1】"):]
|
||||
|
||||
|
||||
async def test_thresholds_and_unmasked_numbers_in_the_excerpt_are_kept(monkeypatch, big_shop):
|
||||
"""报告按金额阈值取数(amount >= 20000,正好等于某一笔被遮的金额),访问次数没设遮罩、第二行正好等于另一笔金额:
|
||||
摘录里 SQL 的阈值、访问次数都照原样,金额列不给。以前对渲染好的整段按值换,成了「amount >= 已遮罩」「r1:4 | 华北 |
|
||||
已遮罩」,agent 看不到报告按什么取数,没设遮罩的数也被改写。"""
|
||||
report = "SELECT id, region, amount, visits FROM sales WHERE amount >= 20000 ORDER BY id"
|
||||
claim = "大额订单里访问最多的一笔在华南"
|
||||
plays = {claim: Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "stands", "共 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]")}
|
||||
models = Models(monkeypatch, big_shop, writer=(f"{claim},访问 [[v:Q1.r0.visits]] 次。[[see:Q1]]",),
|
||||
plays=plays)
|
||||
row = await finish(graph(big_shop, sql=report), run_class="exploratory")
|
||||
assert row.status == "succeeded", row.error
|
||||
excerpt = _excerpt_of(models.tasks()[claim])
|
||||
assert f"SQL:{report}" in excerpt, excerpt
|
||||
assert "列:visits | id | region" in excerpt
|
||||
assert "r1:50,000 | 4 | 华北" in excerpt and MASKED not in excerpt.split("列:", 1)[1].split("(", 1)[0]
|
||||
|
||||
|
||||
async def test_literals_compared_with_a_masked_column_are_masked_even_when_never_collected(monkeypatch, masked_shop):
|
||||
"""报告没选出号码,只在条件里排除了两个(一个换了写法):报告快照里收不到这两个值,摘录的 SQL 照样换掉——和遮罩列
|
||||
做等值、IN 比较的字面量不论认不认得出都换。以前按值换只认快照里收到的,号码原样交给 agent。"""
|
||||
report = ("SELECT region AS 区域, COUNT(*) AS 笔数 FROM sales WHERE buyer_phone NOT IN ('13900000002', "
|
||||
"'139-0000-0003') GROUP BY region ORDER BY 笔数 DESC")
|
||||
claim = "笔数最多的区域"
|
||||
plays = {claim: Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "stands", "共 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]")}
|
||||
models = Models(monkeypatch, masked_shop, writer=(f"{claim}有 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]",), plays=plays)
|
||||
row = await finish(graph(masked_shop, sql=report), run_class="exploratory")
|
||||
assert row.status == "succeeded", row.error
|
||||
task = models.tasks()[claim]
|
||||
assert _clean(task), task
|
||||
assert f"buyer_phone NOT IN ('{MASKED}', '{MASKED}')" in _excerpt_of(task)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 最后一道关只看正文和原文:agent 起的列名、照原样留着的阈值不算露了遮罩值
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
AMOUNTS = "SELECT region AS 区域, amount AS 金额 FROM sales ORDER BY id"
|
||||
FIRST = "华东首单金额"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("verdict", ["stands", "refuted"])
|
||||
async def test_a_threshold_in_an_alias_does_not_void_the_verdict(monkeypatch, big_shop, verdict):
|
||||
"""反方 agent 把阈值写进别名(不低于20000的笔数),20000 恰好是某一笔被遮的金额:结论照常采信,理由照常记。以前最后
|
||||
一道关把目录里的列名当成露了遮罩值,整句改判无法判断,「被推翻」也不再让出具降档,日志里多一条误报。"""
|
||||
top = "SELECT COUNT(*) AS 不低于20000的笔数 FROM sales WHERE amount >= 20000"
|
||||
said = "金额不低于两万的订单有 [[v:Q1.r0.不低于20000的笔数]] 笔,与结论" + ("不矛盾" if verdict == "stands" else "矛盾") \
|
||||
+ "。[[see:Q1]]"
|
||||
Models(monkeypatch, big_shop, writer=(f"{FIRST} [[v:Q1.r0.金额]]。[[see:Q1]]",),
|
||||
plays={FIRST: Play([top], verdict, said)})
|
||||
row = await finish(graph(big_shop, sql=AMOUNTS), run_class="exploratory")
|
||||
assert row.status == "succeeded", row.error
|
||||
out = await output_of(row.id, "chal")
|
||||
record = artifact_store.load(out["challenge_artifact"])
|
||||
[claim] = [c for c in record["claims"] if FIRST in c["text"]]
|
||||
assert claim["verdict"] == verdict, claim["rationale"]
|
||||
assert claim["rationale"].startswith("金额不低于两万的订单有 3 笔")
|
||||
assert out["counts"][verdict] == 1
|
||||
|
||||
|
||||
async def test_a_threshold_the_agent_wrote_in_its_sql_stays_in_the_rationale(monkeypatch, big_shop):
|
||||
"""理由正文复述 agent 自己 SQL 里照原样留着的阈值(「不低于 20000 元」,恰好等于某一笔被遮的金额):照原样,同审批质询
|
||||
的回答;它随 SQL 存进审稿记录、摆在读者眼前。以前换成「已遮罩」,理由和 SQL 对不上。"""
|
||||
top = "SELECT COUNT(*) AS 笔数 FROM sales WHERE amount >= 20000"
|
||||
said = "金额不低于 20000 元的订单有 [[v:Q1.r0.笔数]] 笔,与结论不矛盾。[[see:Q1]]"
|
||||
Models(monkeypatch, big_shop, writer=(f"{FIRST} [[v:Q1.r0.金额]]。[[see:Q1]]",),
|
||||
plays={FIRST: Play([top], "stands", said)})
|
||||
row = await finish(graph(big_shop, sql=AMOUNTS), run_class="exploratory")
|
||||
assert row.status == "succeeded", row.error
|
||||
record = artifact_store.load((await output_of(row.id, "chal"))["challenge_artifact"])
|
||||
[claim] = [c for c in record["claims"] if FIRST in c["text"]]
|
||||
assert (claim["verdict"], claim["rationale"]) == ("stands", "金额不低于 20000 元的订单有 3 笔,与结论不矛盾。")
|
||||
|
||||
|
||||
async def test_an_alias_naming_a_masked_value_is_renamed_like_the_snapshot(monkeypatch, big_shop):
|
||||
"""agent 把找的那个值写进了别名(金额为20000的笔数,20000 是一笔被遮的金额,不是阈值):理由文档的目录列名照快照遮好的
|
||||
样子换(同质询的 _renamed_entry),引用照样对得上,结论照常采信,存下的理由文档里没有这个值。以前目录原样传进重新
|
||||
组装,最后一道关又把目录拿去认,整句改判无法判断。"""
|
||||
top = "SELECT COUNT(*) AS 金额为20000的笔数 FROM sales WHERE amount = 20000"
|
||||
said = "金额为两万的订单有 [[v:Q1.r0.金额为20000的笔数]] 笔,与结论不矛盾。[[see:Q1]]"
|
||||
Models(monkeypatch, big_shop, writer=(f"{FIRST} [[v:Q1.r0.金额]]。[[see:Q1]]",),
|
||||
plays={FIRST: Play([top], "stands", said)})
|
||||
row = await finish(graph(big_shop, sql=AMOUNTS), run_class="exploratory")
|
||||
assert row.status == "succeeded", row.error
|
||||
record = artifact_store.load((await output_of(row.id, "chal"))["challenge_artifact"])
|
||||
[claim] = [c for c in record["claims"] if FIRST in c["text"]]
|
||||
assert (claim["verdict"], claim["rationale"]) == ("stands", "金额为两万的订单有 1 笔,与结论不矛盾。")
|
||||
assert "20000" not in json.dumps(claim["rationale_doc"], ensure_ascii=False)
|
||||
|
||||
@@ -86,3 +86,85 @@ async def test_the_excerpt_in_the_task_is_masked_before_the_agent_sees_it(monkey
|
||||
assert "手机号" not in columns and "金额" in columns
|
||||
assert "buyer_phone = '已遮罩'" in excerpt and "(有的列按数据源的设置遮罩了,没有给出)" in excerpt
|
||||
assert r.json()["inquiry"]["answer"]["ok"] is True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 摘录分段遮:阈值、没设遮罩的数值格、被问那句里的值照原样
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
#: 报告按金额阈值取数(12000 正好是某一笔被遮的金额),访问次数没设遮罩
|
||||
THRESHOLD = ("SELECT id AS 编号, region AS 区域, amount AS 金额, visits AS 访问 FROM sales WHERE amount >= 12000 "
|
||||
"ORDER BY id")
|
||||
|
||||
|
||||
async def test_the_excerpt_keeps_thresholds_and_unmasked_numbers(monkeypatch, big_shop, client):
|
||||
"""针对「华南首单访问 30,000 次。」发起质询:摘录里 SQL 的阈值(12000,恰好是一笔被遮的金额)照原样,访问次数(没设
|
||||
遮罩,20000、30000 恰好等于别的订单的金额)照原样,金额列不给。以前对渲染好的整段按值换:「amount >= 已遮罩」
|
||||
「r0:已遮罩 | 2 | 华东」,被问的那一格也成了「已遮罩」,任务开头的原句却写着 30,000。"""
|
||||
models = Models(monkeypatch, big_shop, writer="华南首单访问 [[v:Q1.r1.访问]] 次。[[see:Q1]]",
|
||||
plays={"访问为什么这么高": Play(["SELECT COUNT(*) AS 笔数 FROM sales"], "共 [[v:Q1.r0.笔数]] 笔。")})
|
||||
paused = await pause(big_shop, graph=fetching(big_shop, THRESHOLD))
|
||||
[approval] = await pending(paused.id)
|
||||
doc = await report_doc(paused.id)
|
||||
r = await ask(client, paused.id, report="write", unit=unit_of(doc, "华南首单访问 30,000 次。"),
|
||||
question="访问为什么这么高?", approval_id=approval.id)
|
||||
assert r.status_code == 200, r.text
|
||||
[task] = models.tasks
|
||||
excerpt = task[task.index("【原报告 Q1】"):]
|
||||
assert f"SQL:{THRESHOLD}" in excerpt, excerpt
|
||||
assert "列:访问 | 编号 | 区域" in excerpt and "r0:20,000 | 2 | 华东" in excerpt and "r1:30,000 | 3 | 华南" in excerpt
|
||||
|
||||
|
||||
#: 买家姓名设了遮罩;推荐人没设遮罩,写着的恰好是另一位买家的名字
|
||||
REFERRALS = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_name TEXT, referrer TEXT, amount INTEGER);"
|
||||
"INSERT INTO sales VALUES (1, '华东', '张伟', '李娜', 1200), (2, '华南', '李娜', '张伟', 900),"
|
||||
" (3, '华北', '王芳', '刘洋', 400);")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def referral_shop(tmp_path):
|
||||
from app.data.engine import engines
|
||||
from tests.test_inquiry_api import _source
|
||||
|
||||
name, source_id = await _source(tmp_path, options={"mask_columns": "buyer_name"}, script=REFERRALS)
|
||||
yield name
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
async def test_values_in_the_asked_sentence_stay_in_the_excerpt_and_the_answer(monkeypatch, referral_shop, client):
|
||||
"""被问的那句写着「李娜」(推荐人这一格,恰好也是另一笔的买家):它就在任务开头、摆在审批人眼前,摘录里这一格、回答里
|
||||
复述的都照原样(同反方审稿的 _spared);别的买家名照旧换。以前质询不把这句原文算作已经摆在眼前的,这一格成了
|
||||
「已遮罩」,和任务开头写着的原句对不上。"""
|
||||
report = "SELECT buyer_name AS 买家, referrer AS 推荐人, amount AS 金额 FROM sales ORDER BY id"
|
||||
said = "推荐人是李娜,第二单的推荐人是 [[v:Q1.r1.推荐人]]。[[see:Q1]]"
|
||||
models = Models(monkeypatch, referral_shop, writer="首单由 [[v:Q1.r0.推荐人]] 推荐。[[see:Q1]]",
|
||||
plays={"谁推荐的": Play(["SELECT id, referrer AS 推荐人 FROM sales ORDER BY id"], said)})
|
||||
paused = await pause(referral_shop, graph=fetching(referral_shop, report))
|
||||
[approval] = await pending(paused.id)
|
||||
doc = await report_doc(paused.id)
|
||||
r = await ask(client, paused.id, report="write", unit=unit_of(doc, "首单由 李娜 推荐。"),
|
||||
question="谁推荐的?", approval_id=approval.id)
|
||||
assert r.status_code == 200, r.text
|
||||
[task] = models.tasks
|
||||
excerpt = task[task.index("【原报告 Q1】"):]
|
||||
assert "列:推荐人 | 金额" in excerpt and "r0:李娜 | 1,200" in excerpt and "r1:已遮罩 | 900" in excerpt, excerpt
|
||||
item = r.json()["inquiry"]
|
||||
assert item["answer"]["text"] == "推荐人是李娜,第二单的推荐人是 已遮罩。"
|
||||
|
||||
|
||||
async def test_literals_compared_with_a_masked_column_are_masked_in_the_excerpt(monkeypatch, masked_shop, client):
|
||||
"""报告没选出号码,只在条件里排除了两个(一个换了写法):报告快照里收不到这两个值,摘录的 SQL 照样换掉。"""
|
||||
report = ("SELECT region AS 区域, COUNT(*) AS 笔数 FROM sales WHERE buyer_phone NOT IN ('13900000002', "
|
||||
"'139-0000-0003') GROUP BY region ORDER BY 笔数 DESC")
|
||||
models = Models(monkeypatch, masked_shop, writer="笔数最多的区域有 [[v:Q1.r0.笔数]] 笔。[[see:Q1]]",
|
||||
plays={"按什么排除": Play([], "查不到判断所需的数据。")})
|
||||
paused = await pause(masked_shop, graph=fetching(masked_shop, report))
|
||||
[approval] = await pending(paused.id)
|
||||
doc = await report_doc(paused.id)
|
||||
r = await ask(client, paused.id, report="write", unit=unit_of(doc, "笔数最多的区域有 1 笔。"),
|
||||
question="按什么排除的?", approval_id=approval.id)
|
||||
assert r.status_code == 200, r.text
|
||||
[task] = models.tasks
|
||||
assert "13900000002" not in task and "0000-0003" not in task, task
|
||||
assert "buyer_phone NOT IN ('已遮罩', '已遮罩')" in task
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
"""交给核查 agent 的证据摘录分段遮(engine/inquiry.py 的 ExcerptMask、_scrub_excerpt_sql,engine/masking.py 的
|
||||
masked_cell):
|
||||
|
||||
- SQL 照质询封存 SQL 的办法换:范围比较的阈值、LIMIT 的行数照原样(恰好等于某个遮罩值也一样),字符串、注释、名字里的
|
||||
遮罩值换;和遮罩列(含改了名字的)做等值、IN、LIKE 一类比较的字面量不论认不认得出都换(报告没选出遮罩列时快照里收
|
||||
不到这些值),同一个数写成阈值的跟着换;已经摆在读者眼前的值(问题、被问的那句)照旧不换
|
||||
- 格子逐格遮:数值格(数、列类型是 number 的)照原样,文字格整格的、夹着的那一截换掉(换了写法的号码也认)
|
||||
|
||||
以前对渲染好的整段摘录按值换,分不出格子和 SQL 里的用法:阈值、没设遮罩的访问次数恰好等于某一笔被遮的金额,都成了
|
||||
「已遮罩」。端到端的用例在 test_challenge_masking、test_inquiry_masking_e2e。号码、姓名、金额都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from app.engine import masking
|
||||
from app.engine.inquiry import ExcerptMask, _scrub_excerpt_sql
|
||||
|
||||
MASKED = "已遮罩"
|
||||
|
||||
|
||||
def secrets_of(*values, protect: tuple[str, ...] = ()) -> masking._Secrets:
|
||||
"""报告快照里收到的遮罩值(金额、号码),去掉已经摆在读者眼前的(protect)。"""
|
||||
secrets = masking._Secrets()
|
||||
secrets.add({"columns": ["v"], "rows": [[v] for v in values]}, {"v"})
|
||||
secrets.protect(protect)
|
||||
return secrets
|
||||
|
||||
|
||||
AMOUNTS = secrets_of(20000, 30000, 50000)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql", [
|
||||
"SELECT id, visits FROM sales WHERE amount >= 20000 ORDER BY id",
|
||||
"SELECT id FROM sales WHERE amount BETWEEN 20000 AND 50000 LIMIT 30000",
|
||||
"SELECT region, COUNT(*) AS n FROM sales WHERE visits > 30000 GROUP BY region",
|
||||
])
|
||||
def test_thresholds_and_limits_stay_as_written(sql):
|
||||
"""范围比较的边界、LIMIT 的行数是报告自己定的条件,恰好等于某一笔被遮的金额也照原样:agent 要看得到报告按什么取数。"""
|
||||
assert _scrub_excerpt_sql(sql, AMOUNTS, {"amount"}, "sqlite") == sql
|
||||
|
||||
|
||||
def test_a_value_the_report_looked_for_is_masked():
|
||||
"""在找某一笔的(等值比较)照旧换:这是在点名一个遮罩值。"""
|
||||
assert _scrub_excerpt_sql("SELECT id FROM sales WHERE amount = 50000", AMOUNTS, {"amount"}, "sqlite") \
|
||||
== f"SELECT id FROM sales WHERE amount = {MASKED}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, expected", [
|
||||
("SELECT region FROM sales WHERE buyer_phone NOT IN ('13900000002', '139-0000-0003')",
|
||||
f"SELECT region FROM sales WHERE buyer_phone NOT IN ('{MASKED}', '{MASKED}')"),
|
||||
("SELECT region FROM sales WHERE buyer_phone = '13900000009' OR buyer_phone LIKE '%0001'",
|
||||
f"SELECT region FROM sales WHERE buyer_phone = '{MASKED}' OR buyer_phone LIKE '{MASKED}'"),
|
||||
("SELECT region FROM (SELECT region, buyer_phone AS 手机号 FROM sales) t WHERE 手机号 IN ('13900000002')",
|
||||
f"SELECT region FROM (SELECT region, buyer_phone AS 手机号 FROM sales) t WHERE 手机号 IN ('{MASKED}')"),
|
||||
("SELECT region FROM sales WHERE region = '华东'", "SELECT region FROM sales WHERE region = '华东'"),
|
||||
])
|
||||
def test_literals_compared_with_a_masked_column_are_masked_even_when_never_collected(sql, expected):
|
||||
"""和遮罩列(含改了名字的)做等值、IN、LIKE 比较的字面量:报告快照里收不到这些值(报告没选出号码),照样换掉;和没设
|
||||
遮罩的列比较的照原样。"""
|
||||
assert _scrub_excerpt_sql(sql, masking._Secrets(), {"buyer_phone", "手机号"}, "sqlite") == expected
|
||||
|
||||
|
||||
def test_a_threshold_equal_to_a_masked_literal_goes_with_it():
|
||||
"""同一个数既在找某一笔、又写成阈值:阈值留着就把换掉的值露了,跟着换(同封存 SQL 的 _literals)。"""
|
||||
sql = "SELECT id FROM sales WHERE buyer_phone = 13900000009 OR id >= 13900000009"
|
||||
assert _scrub_excerpt_sql(sql, masking._Secrets(), {"buyer_phone"}, "sqlite") \
|
||||
== f"SELECT id FROM sales WHERE buyer_phone = {MASKED} OR id >= {MASKED}"
|
||||
|
||||
|
||||
def test_values_already_in_front_of_the_reader_are_not_masked():
|
||||
"""提问人、被问的那句里写着的值(protect)照旧不换,换了写法写的也认得出。"""
|
||||
secrets = secrets_of(protect=("139-0000-0009 这位客户的单",))
|
||||
sql = "SELECT id FROM sales WHERE buyer_phone = '13900000009' OR buyer_phone = '13900000008'"
|
||||
assert _scrub_excerpt_sql(sql, secrets, {"buyer_phone"}, "sqlite") \
|
||||
== f"SELECT id FROM sales WHERE buyer_phone = '13900000009' OR buyer_phone = '{MASKED}'"
|
||||
|
||||
|
||||
def test_strings_and_comments_are_masked_by_value():
|
||||
"""字符串、注释里写着的遮罩值按值换(换了写法的号码也认),同封存 SQL。"""
|
||||
secrets = secrets_of("13900000001")
|
||||
sql = "SELECT id FROM sales /* 客户 139-0000-0001 */ WHERE note LIKE '%13900000001%'"
|
||||
assert _scrub_excerpt_sql(sql, secrets, set(), "sqlite") \
|
||||
== f"SELECT id FROM sales /* 客户 {MASKED} */ WHERE note LIKE '%{MASKED}%'"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, numeric, expected", [
|
||||
(20000, False, 20000),
|
||||
(20000.0, False, 20000.0),
|
||||
("20000.00", True, "20000.00"),
|
||||
("20000", False, MASKED),
|
||||
("上月消费 20000 元", False, f"上月消费 {MASKED} 元"),
|
||||
("回访:联系 139-0000-0003", False, f"回访:联系 {MASKED}"),
|
||||
({"phone": "13900000003"}, False, '{"phone": "' + MASKED + '"}'),
|
||||
("例行回访", False, "例行回访"),
|
||||
(None, False, None),
|
||||
])
|
||||
def test_cells_are_masked_like_the_tool_boundary(value, numeric, expected):
|
||||
"""格子同工具边界:数(整数、小数,列类型是 number 的字符串)照原样;文字格整格是遮罩值的、夹着的那一截换掉,JSON
|
||||
按画出来的样子换。"""
|
||||
secrets = secrets_of(20000, "13900000003")
|
||||
assert masking.masked_cell(value, secrets, numeric=numeric) == expected
|
||||
|
||||
|
||||
def test_an_excerpt_mask_without_values_only_masks_by_column():
|
||||
"""没有可按值换的(报告快照里没收到值):格子、文字照原样,SQL 里和遮罩列做等值比较的照样换。"""
|
||||
mask = ExcerptMask({"shop": "sqlite"})
|
||||
assert mask.cell("13900000001", numeric=False) == "13900000001"
|
||||
assert mask.text("客户 13900000001") == "客户 13900000001"
|
||||
assert mask.sql("SELECT 1 FROM sales WHERE buyer_phone = '13900000001'", {"source": "shop"}, {"buyer_phone"}) \
|
||||
== f"SELECT 1 FROM sales WHERE buyer_phone = '{MASKED}'"
|
||||
@@ -140,11 +140,13 @@ async def composite_shop():
|
||||
[[1, PHONE, "某路 1 号"]]),
|
||||
("SELECT p.id AS n, (jsonb_populate_record(NULL::contact, p.pj)).* FROM people2 p", ["n", "tel", "addr"],
|
||||
[[1, PHONE, "某路 1 号"]]),
|
||||
], ids=["复合类型", "子查询里的复合类型", "按类型展开 JSON"])
|
||||
("SELECT p.id AS n, ((p.profile).*) FROM people p", ["n", "tel", "addr"], [[1, PHONE, "某路 1 号"]]),
|
||||
], ids=["复合类型", "子查询里的复合类型", "按类型展开 JSON", "外面套了括号"])
|
||||
async def test_an_expression_star_never_reaches_the_model_or_the_store(monkeypatch, composite_shop, sql, columns,
|
||||
rows):
|
||||
"""端到端(核查 agent 的真实查询工具):交给模型的、落库的、通用工件接口给的都只有「已遮罩」,masked_at_query 是
|
||||
全部的列。以前三条的 masked_at_query 都是空的,号码原样交给模型、落库。"""
|
||||
全部的列,快照记着整份遮(masked_whole),格子引用照实说写法认不全。以前前三条的 masked_at_query 都是空的,套了外括号
|
||||
的只遮了第一列,号码原样交给模型、落库。"""
|
||||
from app.data.engine import QueryResult
|
||||
from app.tools import datasource
|
||||
|
||||
@@ -162,6 +164,8 @@ async def test_an_expression_star_never_reaches_the_model_or_the_store(monkeypat
|
||||
assert MASKED_WORD in seen and "1390000000" not in seen, seen
|
||||
snapshot = artifact_store.load(out.queries[0]["artifact"])
|
||||
assert snapshot["masked_at_query"] == columns and "1390000000" not in json.dumps(snapshot, ensure_ascii=False)
|
||||
assert snapshot["masked_whole"] is True and "masked_whole" not in seen
|
||||
assert out.cells[0]["reason"] == "Q1 的写法认不全,整份结果查询时已遮罩,不采信"
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||
got = await client.get(f"/api/artifacts/{out.queries[0]['artifact']}")
|
||||
assert got.status_code == 200 and "1390000000" not in got.text
|
||||
@@ -280,3 +284,229 @@ def test_a_failing_analysis_masks_the_whole_result_instead_of_reading_the_text(m
|
||||
monkeypatch.setattr(masking, "_tainted_in", broken)
|
||||
snapshot = {"sql": "SELECT s AS r FROM sales s", "columns": ["r"], "rows": []}
|
||||
assert "r" in masking._derived_hidden(snapshot, {"buyer_phone"}, "postgresql")[0]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# (表达式).* 套了外括号、起了别名
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
#: Postgres 实测:((p.card).*)、(((p.card)).*)、(((p.card).*))、(p.card).* AS x 都照样展开成好几列(别名不起作用)
|
||||
WRAPPED = [
|
||||
pytest.param("SELECT ((p.profile).*) FROM people p", ["tel", "addr"], [PHONE, "某路 1 号"], id="复合类型-外括号"),
|
||||
pytest.param("SELECT (((p.profile)).*) FROM people p", ["tel", "addr"], [PHONE, "某路 1 号"],
|
||||
id="复合类型-两层外括号"),
|
||||
pytest.param("SELECT (((p.profile).*)) FROM people p", ["tel", "addr"], [PHONE, "某路 1 号"],
|
||||
id="复合类型-套两层"),
|
||||
pytest.param("SELECT (p.profile).* AS x FROM people p", ["tel", "addr"], [PHONE, "某路 1 号"], id="复合类型-别名"),
|
||||
pytest.param("SELECT ((jsonb_each_text(pj)).*) FROM people2", ["key", "value"], ["tel", PHONE],
|
||||
id="记录函数-外括号"),
|
||||
pytest.param("SELECT ((q.r).*) FROM (SELECT ROW(region, buyer_phone) AS r FROM sales) q", ["f1", "f2"],
|
||||
["华东", PHONE], id="匿名 ROW-外括号"),
|
||||
pytest.param("SELECT ((jsonb_each_text(pj)).*) FROM people2 UNION ALL SELECT 'a', 'b'", ["key", "value"],
|
||||
["tel", PHONE], id="UNION 第一支"),
|
||||
pytest.param("SELECT 'a' AS k, 'b' AS v UNION ALL SELECT ((jsonb_each_text(pj)).*) FROM people2", ["k", "v"],
|
||||
["tel", PHONE], id="UNION 后一支"),
|
||||
pytest.param("SELECT 'a' AS k, 'b' AS v UNION ALL SELECT ((p.profile).*) FROM people p", ["k", "v"],
|
||||
[PHONE, "某路 1 号"], id="UNION 后一支-复合类型"),
|
||||
pytest.param("SELECT x.* FROM (SELECT ((q.r).*) FROM (SELECT ROW(buyer_phone, region) AS r FROM sales) q) x",
|
||||
["f1", "f2"], [PHONE, "华东"], id="子查询里-外括号"),
|
||||
# 和 * 写在同一支里:选出项和结果的列数本来就对不上(* 展开成几列),只能靠认出外括号里的 (表达式).*
|
||||
pytest.param("SELECT p.*, ((p.profile).*) FROM people p", ["id", "profile", "address", "tel", "addr"],
|
||||
[1, f"({PHONE},某路 1 号)", "某路 1 号", PHONE, "某路 1 号"], id="和 * 写在一起-外括号"),
|
||||
pytest.param("SELECT p.*, (p.profile).* AS x FROM people p", ["id", "profile", "address", "tel", "addr"],
|
||||
[1, f"({PHONE},某路 1 号)", "某路 1 号", PHONE, "某路 1 号"], id="和 * 写在一起-别名"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, columns, row", WRAPPED)
|
||||
def test_a_wrapped_expression_star_reading_a_masked_column_masks_the_whole_result(sql, columns, row):
|
||||
"""(表达式).* 外面套了括号、起了别名,Postgres 照样展开成好几列:整份遮。以前只认不带外括号的写法,((p.card).*) 按一项
|
||||
数,只遮了第一列,号码原样交给模型、落库。"""
|
||||
done = masked(sql, columns, [list(row)])
|
||||
assert done.whole and done.columns == columns
|
||||
assert done.payload["rows"] == [[MASKED_WORD] * len(columns)] and not _leaks(done)
|
||||
|
||||
|
||||
def test_a_wrapped_expression_star_with_a_numeric_phone_is_masked_whole():
|
||||
"""号码存成整数(bigint)、前面的查询已经认出它:数值格不按值换,(表达式).* 认不出就全靠整份遮挡住。"""
|
||||
known = masking._Secrets()
|
||||
known.add({"columns": ["p"], "rows": [[13900000001]]}, {"p"})
|
||||
payload = {"columns": ["f1", "f2"], "rows": [[1200, 13900000001]],
|
||||
"sql": "SELECT ((q.r).*) FROM (SELECT ROW(amount, buyer_phone) AS r FROM sales) q"}
|
||||
done = masking.mask_query_result(payload, MASKED, "postgresql", known, schema=SCHEMA)
|
||||
assert done.payload["rows"] == [[MASKED_WORD, MASKED_WORD]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql", ["SELECT ((s).*) FROM sales s", "SELECT (s.*) AS x FROM sales s"])
|
||||
def test_a_wrapped_source_star_is_still_read_as_its_columns(sql):
|
||||
"""((s).*)、(s.*) AS x、s 是 FROM 里的来源:照旧当 s.* 认,只遮号码那一列。"""
|
||||
done = masked(sql, ["id", "region", "buyer_phone", "amount"], [[1, "华东", PHONE, 1200]])
|
||||
assert not done.whole and done.columns == ["buyer_phone"]
|
||||
assert done.payload["rows"] == [[1, "华东", MASKED_WORD, 1200]]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 和数据源里的表同名的 CTE
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, columns, row", [
|
||||
pytest.param("WITH sales AS (SELECT * FROM sales WHERE region = '华东') SELECT row_to_json(sales) AS j FROM sales",
|
||||
["j"], [json.dumps({"id": 1, "buyer_phone": PHONE})], id="整行引用"),
|
||||
pytest.param("WITH sales AS (SELECT * FROM sales WHERE region = '华东') SELECT s AS r FROM sales s", ["r"],
|
||||
[f"(1,华东,{PHONE},1200)"], id="选出整行"),
|
||||
pytest.param("WITH orders AS (SELECT * FROM sales), sales AS (SELECT * FROM sales WHERE amount > 0) "
|
||||
"SELECT to_jsonb(x) AS j FROM sales x", ["j"], [json.dumps({"buyer_phone": PHONE})],
|
||||
id="排在后面的同名段"),
|
||||
pytest.param("WITH sales AS (SELECT * FROM sales) SELECT * FROM sales t(a, b, c, d)", ["a", "b", "c", "d"],
|
||||
[1, "华东", PHONE, 1200], id="列名表"),
|
||||
])
|
||||
def test_a_cte_named_like_a_table_reads_the_table(sql, columns, row):
|
||||
"""WITH sales AS (SELECT * FROM sales WHERE …) 里的 sales 是数据源里的表,不是这一段自己(不带 RECURSIVE 的 WITH 里,
|
||||
一段只看得见排在它前面的段)。以前认成它自己,* 展开绕回自己什么列都不带出,号码原样交了出去。"""
|
||||
done = masked(sql, columns, [list(row)])
|
||||
assert done.columns and not _leaks(done)
|
||||
assert done.payload["rows"] == [[MASKED_WORD if c in done.columns else v for c, v in zip(columns, row)]]
|
||||
|
||||
|
||||
def test_a_cte_named_like_a_table_masks_like_any_other_name():
|
||||
"""对照:同一段 CTE 改叫 t,同样遮;列名表按表结构对上位置,只遮号码那一列。"""
|
||||
for sql in ("WITH t AS (SELECT * FROM sales WHERE region = '华东') SELECT row_to_json(t) AS j FROM t",
|
||||
"WITH sales AS (SELECT * FROM sales WHERE region = '华东') SELECT row_to_json(sales) AS j FROM sales"):
|
||||
assert masked(sql, ["j"], [[json.dumps({"buyer_phone": PHONE})]]).columns == ["j"]
|
||||
done = masked("WITH sales AS (SELECT * FROM sales) SELECT * FROM sales t(a, b, c, d)", ["a", "b", "c", "d"],
|
||||
[[1, "华东", PHONE, 1200]])
|
||||
assert done.columns == ["c"]
|
||||
|
||||
|
||||
def test_a_recursive_cte_still_sees_itself():
|
||||
"""带 RECURSIVE 的 WITH 里递归的那一支指向它自己:取自号码的那一列照遮,只数步数的照常。"""
|
||||
body = ("WITH RECURSIVE t(n, p) AS (SELECT 1, buyer_phone FROM sales UNION ALL SELECT n + 1, p FROM t "
|
||||
"WHERE n < 3) ")
|
||||
assert masked(body + "SELECT n FROM t", ["n"], [[1]]).columns == []
|
||||
assert masked(body + "SELECT p FROM t", ["p"], [[PHONE]]).columns == ["p"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 语法树和实际执行的对不上、存着值的视图:和可验证报告共用一份名单
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, kind, columns", [
|
||||
pytest.param("SELECT id /*!, buyer_phone AS x */ FROM sales", "mysql", ["id", "x"], id="MySQL 会执行的注释"),
|
||||
pytest.param("SELECT /*! buyer_phone AS */ region FROM sales", "mysql", ["region"],
|
||||
id="MySQL 会执行的注释改了列的来历"),
|
||||
pytest.param('SELECT U&"\\0062uyer_phone" AS x FROM sales', "postgresql", ["x"], id="U&转义的名字"),
|
||||
pytest.param("SELECT id, U&\"buyer!005fphone\" UESCAPE '!' AS p FROM sales", "postgresql", ["id", "p"],
|
||||
id="U&带 UESCAPE"),
|
||||
pytest.param("SELECT query FROM pg_stat_get_activity(NULL)", "postgresql", ["query"], id="pg_stat_get_activity"),
|
||||
pytest.param("SELECT data FROM pg_logical_slot_peek_changes('s', NULL, NULL)", "postgresql", ["data"],
|
||||
id="逻辑解码"),
|
||||
pytest.param("SELECT HISTOGRAM AS h FROM information_schema.COLUMN_STATISTICS WHERE TABLE_NAME = 'sales'", "mysql",
|
||||
["h"], id="MySQL 直方图"),
|
||||
pytest.param("SELECT sql_text AS v FROM performance_schema.events_statements_history_long", "mysql", ["v"],
|
||||
id="MySQL 语句历史"),
|
||||
pytest.param("SELECT info AS v FROM information_schema.PROCESSLIST", "mysql", ["v"], id="MySQL 会话列表"),
|
||||
pytest.param("SELECT argument AS v FROM mysql.general_log", "mysql", ["v"], id="MySQL 通用日志"),
|
||||
pytest.param("SELECT min_value AS v FROM mysql.column_stats", "mariadb", ["v"], id="MariaDB 列统计"),
|
||||
pytest.param("SELECT sql_fulltext AS v FROM v$sql", "oracle", ["v"], id="Oracle V$SQL"),
|
||||
pytest.param("SELECT value_string AS v FROM v$sql_bind_capture", "oracle", ["v"], id="Oracle 绑定变量"),
|
||||
pytest.param("SELECT endpoint_actual_value AS v FROM all_tab_histograms WHERE table_name = 'SALES'", "oracle",
|
||||
["v"], id="Oracle 直方图"),
|
||||
pytest.param("SELECT * FROM all_tab_columns WHERE table_name = 'SALES'", "oracle", ["column_name", "low_value"],
|
||||
id="Oracle 列的最小最大值"),
|
||||
pytest.param("SELECT query_to_xml/**/('select * from sales', true, false, '') AS x", "postgresql", ["x"],
|
||||
id="函数名后夹块注释"),
|
||||
pytest.param("SELECT query_to_xml --c\n('select * from sales', true, false, '') AS x", "postgresql", ["x"],
|
||||
id="函数名后夹行注释"),
|
||||
pytest.param("SELECT id FROM sales", "mysql", ["id", "x"], id="选出项和结果列数对不上"),
|
||||
])
|
||||
def test_writings_the_analysis_cannot_follow_mask_the_whole_result(sql, kind, columns):
|
||||
"""语法树和实际执行的对不上(MySQL 会执行的注释、U&"…"、选出项和结果的列数对不上),统计信息、会话记录里存着值的视图和
|
||||
函数,函数名和括号之间夹着注释:整份遮。以前工具边界的名单比可验证报告的少一大截,这些写法号码原样交给模型、落库。"""
|
||||
row = [PHONE] * len(columns)
|
||||
done = masked(sql, columns, [row], kind=kind)
|
||||
assert done.whole and done.payload["rows"] == [[MASKED_WORD] * len(columns)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql, columns, row", [
|
||||
pytest.param("WITH query(r, n) AS (SELECT region, COUNT(*) FROM sales GROUP BY region) SELECT r AS region, n "
|
||||
"FROM query", ["region", "n"], ["华东", 2], id="叫 query 的 CTE"),
|
||||
pytest.param("SELECT region, 'Query (x)' AS q FROM sales", ["region", "q"], ["华东", "Query (x)"],
|
||||
id="字符串里的 Query ("),
|
||||
pytest.param("SELECT column_name FROM information_schema.columns WHERE table_name = 'sales'", ["column_name"],
|
||||
["buyer_phone"], id="information_schema 的列清单"),
|
||||
])
|
||||
def test_names_of_unsupported_databases_and_plain_schema_views_are_not_masked(sql, columns, row):
|
||||
"""不支持的库(DuckDB 的 query()、SQL Server 的 OPENQUERY)的函数不再收:名叫 query 的 CTE、字符串里的「Query (x)」不再
|
||||
整份遮;information_schema 里只有表结构的视图照常(核查 agent 查列名、类型常用)。"""
|
||||
done = masked(sql, columns, [list(row)])
|
||||
assert not done.whole and done.payload["rows"] == [list(row)]
|
||||
|
||||
|
||||
def test_the_tool_boundary_and_the_bundle_share_one_list():
|
||||
"""工具边界和可验证报告共用一份名单:报告那边另把系统 schema 整个挡(只决定附不附快照)。"""
|
||||
from app.bundle import build
|
||||
|
||||
assert build._OPAQUE_SQL is masking.OPAQUE_SQL
|
||||
for sql in ("SELECT HISTOGRAM FROM information_schema.COLUMN_STATISTICS", "SELECT * FROM v$sql",
|
||||
"SELECT data FROM pg_logical_slot_peek_changes('s', NULL, NULL)"):
|
||||
assert build._INDIRECT_READS.search(sql) and masking._INDIRECT.search(sql)
|
||||
plain = "SELECT column_name FROM information_schema.columns"
|
||||
assert build._INDIRECT_READS.search(plain) and not masking._INDIRECT.search(plain)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql", [
|
||||
"SELECT row_to_json(s) AS j FROM sales s ORDER BY s.id USING >",
|
||||
"SELECT s AS j FROM sales s WHERE s.region IS NOT NFC NORMALIZED OR true",
|
||||
"SELECT to_jsonb(s) - 'id' AS j FROM sales s ORDER BY s.id USING <",
|
||||
])
|
||||
def test_postgres_sql_that_cannot_be_read_masks_the_whole_result(sql):
|
||||
"""Postgres 合法、解析器解析不了的写法:整行引用不写列名、也没有 *,只看文字认不出,整份遮。以前 masked=[],整行连同
|
||||
号码交给模型、落库。"""
|
||||
done = masked(sql, ["j"], [[json.dumps({"id": 1, "buyer_phone": PHONE})]])
|
||||
assert done.whole and not _leaks(done)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 整份遮的结果不把每一格都收成遮罩值
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_whole_masked_result_does_not_turn_every_cell_into_a_masked_value():
|
||||
"""统计视图整份遮之后,里面的列名、区域名不收成遮罩值:后一条查询的区域照常,封存的 SQL 照原样。以前每一格都收,
|
||||
区域名、列名在同一次核查里处处被换掉,封存的 SQL 成了「WHERE 已遮罩 = '华东'」。"""
|
||||
from app.engine.inquiry import _scrub_sql
|
||||
|
||||
stats = masked("SELECT attname, n_distinct FROM pg_stats WHERE tablename = 'sales'", ["attname", "n_distinct"],
|
||||
[["amount", 5], ["buyer_phone", -1], ["region", 3]])
|
||||
assert stats.whole and not stats.secrets.strong
|
||||
payload = {"columns": ["id", "region", "amount"], "rows": [[1, "华东", 1200]],
|
||||
"sql": "SELECT id, region, amount FROM sales ORDER BY id"}
|
||||
later = masking.mask_query_result(payload, MASKED, "postgresql", stats.secrets, schema=SCHEMA)
|
||||
assert later.payload["rows"] == [[1, "华东", 1200]]
|
||||
sql = "SELECT region, COUNT(*) AS n FROM sales WHERE region = '华东' GROUP BY region"
|
||||
assert _scrub_sql(sql, stats.secrets, {"buyer_phone"}, "postgresql") == sql
|
||||
|
||||
|
||||
def test_a_whole_masked_result_still_collects_what_surely_came_from_a_masked_column():
|
||||
"""整份遮时,认得出取自遮罩列的那几列照收(号码改了名字选出来、又夹着认不全的写法),别的列不收。"""
|
||||
sql = "SELECT buyer_phone AS 手机号, region FROM sales UNION ALL SELECT query_to_xml('x', true, false, ''), 'y'"
|
||||
done = masked(sql, ["手机号", "region"], [[PHONE, "华东"]])
|
||||
assert done.whole and done.secrets.hit(PHONE) and not done.secrets.hit("华东")
|
||||
|
||||
|
||||
def test_report_snapshots_that_were_masked_whole_give_only_their_sure_values():
|
||||
"""报告自己的查询快照(报告取数时没遮)收遮罩值时同一条规则:写法认不全的只收确定取自遮罩列的那几列,区域名、列名不收
|
||||
(审批质询、反方审稿把它们交给查询工具、拿来换回答和理由,收了就处处被换掉)。"""
|
||||
from app.engine.inquiry import _add_based
|
||||
|
||||
secrets = masking._Secrets()
|
||||
stats = {"source": "pg", "sql": "SELECT attname, n_distinct FROM pg_stats WHERE tablename = 'sales'",
|
||||
"columns": ["attname", "n_distinct"], "rows": [["region", 3], ["buyer_phone", -1]]}
|
||||
mixed = {"source": "pg", "columns": ["手机号", "region"], "rows": [[PHONE, "华东"]],
|
||||
"sql": "SELECT buyer_phone AS 手机号, region FROM sales UNION ALL SELECT query_to_xml('x', true, false, ''), "
|
||||
"'y'"}
|
||||
_add_based(secrets, [(stats, set(MASKED)), (mixed, set(MASKED))], {"pg": "postgresql"})
|
||||
assert secrets.hit(PHONE) and not secrets.hit("华东") and not secrets.hit("region")
|
||||
|
||||
@@ -45,7 +45,7 @@ PAYLOAD = {"sql": "SELECT id, buyer_phone, note FROM sales", "columns": ["id", "
|
||||
"column_types": {"id": "number", "buyer_phone": "text", "note": "text"}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("step", ["_derived_hidden", "_snapshot_labels", "_masked_view"])
|
||||
@pytest.mark.parametrize("step", ["_derived_masks", "_snapshot_labels", "_masked_view"])
|
||||
def test_a_failing_analysis_masks_every_cell(monkeypatch, step):
|
||||
"""认遮罩列、认标签列、逐格换值,哪一步出了意外都整份遮:每一格都是「已遮罩」、类型记成文本,整列遮掉的是全部的列。"""
|
||||
monkeypatch.setattr(masking, step, _broken)
|
||||
@@ -55,6 +55,7 @@ def test_a_failing_analysis_masks_every_cell(monkeypatch, step):
|
||||
assert done.columns == ["id", "buyer_phone", "note"]
|
||||
assert done.payload["sql"] == PAYLOAD["sql"], "SQL 是模型自己写的,照原样"
|
||||
assert not done.secrets.strong, "认不准,不收值"
|
||||
assert done.whole, "快照记下是整份遮的"
|
||||
|
||||
|
||||
def test_a_failing_analysis_masks_rows_of_any_shape(monkeypatch):
|
||||
@@ -66,10 +67,10 @@ def test_a_failing_analysis_masks_rows_of_any_shape(monkeypatch):
|
||||
assert done.payload["rows"] == [{"id": MASKED, "buyer_phone": MASKED}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("step", ["_derived_hidden", "_masked_view"])
|
||||
@pytest.mark.parametrize("step", ["_derived_masks", "_masked_view"])
|
||||
async def test_the_model_and_the_snapshot_see_only_masks_when_the_analysis_fails(monkeypatch, masked_shop, step):
|
||||
"""端到端:分析出了意外,交给模型的工具结果、落库的快照、通用接口给的都只有「已遮罩」,masked_at_query 是全部的列;
|
||||
格子引用照实说是遮罩列,不采信。"""
|
||||
格子引用照实说写法认不全、整份遮了,不采信——金额在这里没设遮罩,不说成「取自设置了遮罩的列」。"""
|
||||
monkeypatch.setattr(masking, step, _broken)
|
||||
name = masked_shop
|
||||
sql = "SELECT id, region, buyer_phone AS 手机号, amount FROM sales ORDER BY id"
|
||||
@@ -80,13 +81,14 @@ async def test_the_model_and_the_snapshot_see_only_masks_when_the_analysis_fails
|
||||
assert seen["rows"] == [[MASKED] * 4] * 4, seen
|
||||
snapshot = artifact_store.load(out.queries[0]["artifact"])
|
||||
assert snapshot["rows"] == [[MASKED] * 4] * 4
|
||||
assert snapshot["masked_at_query"] == ["id", "region", "手机号", "amount"]
|
||||
assert snapshot["masked_at_query"] == ["id", "region", "手机号", "amount"] and snapshot["masked_whole"] is True
|
||||
assert "masked_whole" not in seen, "记录只在快照里,不交给模型"
|
||||
assert _clean(json.dumps(snapshot, ensure_ascii=False))
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||
got = await client.get(f"/api/artifacts/{out.queries[0]['artifact']}")
|
||||
assert got.status_code == 200 and _clean(got.text) and "1200" not in got.text
|
||||
assert out.result["value"] is None
|
||||
assert out.cells[0]["reason"] == "Q1 的「amount」取自设置了遮罩的列,查询时已遮罩,不采信"
|
||||
assert out.cells[0]["reason"] == "Q1 的写法认不全,整份结果查询时已遮罩,不采信"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -250,3 +252,121 @@ def test_without_a_schema_a_star_next_to_the_band_is_still_masked():
|
||||
payload = {"sql": sql, "columns": ["档位", "笔数"], "rows": [list(r) for r in ROWS]}
|
||||
done = masking.mask_query_result(payload, ["amount"], "sqlite")
|
||||
assert done.columns == ["档位"] and done.payload["rows"] == [[MASKED, 3], [MASKED, 2]]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 数值格按位置认:同名的列、写成字符串的数
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _known(*values: str) -> masking._Secrets:
|
||||
"""此前认出的遮罩值(前面的查询遮掉的、报告快照里的)。"""
|
||||
secrets = masking._Secrets()
|
||||
secrets.add({"columns": ["v"], "rows": [[v] for v in values]}, {"v"})
|
||||
return secrets
|
||||
|
||||
|
||||
def test_a_text_column_sharing_its_name_with_a_number_column_is_still_masked_by_value():
|
||||
"""column_types 按列名记,同名的几列只留下后面那一列的类型:两张表里都叫 phone,前一列是文字(写着号码)、后一列是
|
||||
整数,记成 number。以前按列名认数值列,文字列也当成了数值列,号码原样交给模型、落库;现在同名的几列按格子本身认,
|
||||
文字格照换、记进 masked_cells,真正的整数照原样。写成字符串的数在同名列里也按文字处理(宁可多遮)。"""
|
||||
payload = {"sql": "SELECT a.region, a.phone, b.phone FROM contacts a JOIN legacy b ON a.id = b.id",
|
||||
"columns": ["region", "phone", "phone"],
|
||||
"rows": [["华东", "联系人 13900000001", 13900000002], ["华南", "13900000001", "13900000001"]],
|
||||
"column_types": {"region": "text", "phone": "number"}}
|
||||
done = masking.mask_query_result(payload, [], "sqlite", _known("13900000001"))
|
||||
assert done.payload["rows"] == [["华东", f"联系人 {MASKED}", 13900000002], ["华南", MASKED, MASKED]]
|
||||
assert done.cells == [[0, 1], [1, 1], [1, 2]]
|
||||
|
||||
|
||||
def test_a_numeric_column_written_as_decimal_strings_is_kept():
|
||||
"""Postgres 的 NUMERIC、MySQL 的 DECIMAL 交回来是字符串("20000.00"),列类型记着 number:没设遮罩的这一列正好等于某个
|
||||
遮罩值(金额也是 NUMERIC)也照原样,不记进 masked_cells——数值证据不改写。"""
|
||||
payload = {"sql": "SELECT id, visits FROM sales2 ORDER BY id", "columns": ["id", "visits"],
|
||||
"rows": [[1, "20000.00"], [2, "9000.00"]], "column_types": {"id": "number", "visits": "number"}}
|
||||
done = masking.mask_query_result(payload, [], "postgresql", _known("20000.00", "30000.00"))
|
||||
assert done.payload["rows"] == [[1, "20000.00"], [2, "9000.00"]] and done.cells == []
|
||||
|
||||
|
||||
#: 两张表里都有 remark:工单的是文字(写着号码),评分的是整数
|
||||
DUPLICATES = ("CREATE TABLE sales (id INTEGER PRIMARY KEY, region TEXT, buyer_phone TEXT, amount INTEGER);"
|
||||
"INSERT INTO sales VALUES (1, '华东', '13900000001', 1200), (2, '华南', '13900000002', 900);"
|
||||
"CREATE TABLE tickets (id INTEGER PRIMARY KEY, remark TEXT);"
|
||||
"INSERT INTO tickets VALUES (1, '客户 13900000001 来电投诉'), (2, '回访 13900000002');"
|
||||
"CREATE TABLE scores (id INTEGER PRIMARY KEY, remark INTEGER);"
|
||||
"INSERT INTO scores VALUES (1, 5), (2, 4);")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def ticket_shop(tmp_path):
|
||||
name, source_id = await _source(tmp_path, {"mask_columns": "buyer_phone"}, script=DUPLICATES)
|
||||
yield name
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql", [
|
||||
"SELECT t.remark, s.remark FROM tickets t JOIN scores s ON s.id = t.id ORDER BY t.id",
|
||||
"SELECT * FROM tickets t JOIN scores s ON s.id = t.id ORDER BY t.id",
|
||||
])
|
||||
async def test_duplicate_names_do_not_let_numbers_shield_text(monkeypatch, ticket_shop, sql):
|
||||
"""端到端:先查出号码(查询时遮掉),再把工单和评分连起来查,两列都叫 remark(* 连两张表时 id 也同名)。工单备注里的
|
||||
号码交给模型、落库之前照样换掉,评分照原样;以前评分的 number 套到了同名的备注上,号码原样交了出去。"""
|
||||
name = ticket_shop
|
||||
steps = [q(name, "SELECT id, buyer_phone FROM sales ORDER BY id", "c1"), q(name, sql, "c2")]
|
||||
script = Script(monkeypatch, steps, {"value": None, "rationale": "x"})
|
||||
out, _ = await ask(name)
|
||||
seen = script.tool_results()["c2"]
|
||||
assert "1390000000" not in seen and f"客户 {MASKED} 来电投诉" in seen, seen
|
||||
snapshot = artifact_store.load(out.queries[1]["artifact"])
|
||||
remarks = [i for i, c in enumerate(snapshot["columns"]) if c == "remark"]
|
||||
assert [[row[i] for i in remarks] for row in snapshot["rows"]] == [[f"客户 {MASKED} 来电投诉", 5],
|
||||
[f"回访 {MASKED}", 4]]
|
||||
assert snapshot["masked_cells"] == [[0, remarks[0]], [1, remarks[0]]]
|
||||
|
||||
|
||||
async def test_a_decimal_column_is_cited_as_a_number(monkeypatch, composite_numbers):
|
||||
"""端到端:NUMERIC 的访问次数(字符串 "20000.00"、列类型 number)正好等于某一笔被遮的金额:交给模型的、落库的照原样,
|
||||
格子引用照常取到 20000(双盲复算拿得到它)。"""
|
||||
name, sql = composite_numbers
|
||||
script = Script(monkeypatch, [q(name, "SELECT id, amount FROM sales ORDER BY id", "c1"), q(name, sql, "c2")],
|
||||
{"value": {"alias": "Q2", "row": 0, "column": "visits"}, "rationale": "x"})
|
||||
out, _ = await ask(name)
|
||||
seen = json.loads(script.tool_results()["c2"].split("\n", 1)[1])
|
||||
assert seen["rows"] == [[1, "20000.00"], [2, "9000.00"]]
|
||||
snapshot = artifact_store.load(out.queries[1]["artifact"])
|
||||
assert snapshot["rows"] == seen["rows"] and "masked_cells" not in snapshot
|
||||
assert out.result["value"] == 20000 and out.cells[0]["status"] == "resolved"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def composite_numbers(monkeypatch):
|
||||
"""造一个 Postgres 数据源(查询交给换掉的 run_query,不连库):金额设了遮罩,金额、访问次数都是 NUMERIC,驱动交回
|
||||
Decimal,查询层转成字符串、列类型记 number。交回 (源名, 查访问次数的 SQL)。"""
|
||||
import uuid
|
||||
|
||||
from app.data.engine import QueryResult
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
from app.tools import datasource
|
||||
|
||||
visits = "SELECT id, visits FROM sales ORDER BY id"
|
||||
results = {"SELECT id, amount FROM sales ORDER BY id": (["id", "amount"], [[1, "20000.00"], [2, "30000.00"]]),
|
||||
visits: (["id", "visits"], [[1, "20000.00"], [2, "9000.00"]])}
|
||||
|
||||
async def driver(source, statement, **kwargs):
|
||||
columns, rows = results[statement]
|
||||
return QueryResult(columns=columns, rows=[list(r) for r in rows], row_count=len(rows), truncated=False,
|
||||
elapsed_ms=1, sql=statement, column_types={c: "number" for c in columns})
|
||||
|
||||
monkeypatch.setattr(datasource, "run_query", driver)
|
||||
name = f"pg_{uuid.uuid4().hex[:8]}"
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=name, kind="postgresql", host="127.0.0.1", port=5432, username="probe", database="shop",
|
||||
readonly=True, enabled=True, options={"mask_columns": "amount"},
|
||||
schema_cache={"tables": {"sales": {"columns": [{"name": c} for c in ("id", "amount",
|
||||
"visits")]}}})
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
source_id = row.id
|
||||
yield name, visits
|
||||
await engines.invalidate(source_id)
|
||||
|
||||
@@ -111,8 +111,9 @@ def test_a_whole_row_of_an_outer_query_is_found_from_a_nested_derived_table(sql)
|
||||
"SELECT EXISTS(SELECT * FROM sales s2 WHERE s2.id = s.id) AS j FROM sales s",
|
||||
])
|
||||
def test_rows_and_stars_that_bring_no_masked_values_stay_visible(sql):
|
||||
"""整行取自没有遮罩列的子查询、count(s.*) 只数个数:不算。"""
|
||||
assert hidden_of(sql, ["j", "total"]) == set(MASKED)
|
||||
"""整行取自没有遮罩列的子查询、count(s.*) 只数个数:不算。结果的列数照选出项数(列数对不上的整份遮,见
|
||||
test_masking_opaque_sql)。"""
|
||||
assert hidden_of(sql, ["j", "total"] if "total" in sql else ["j"]) == set(MASKED)
|
||||
|
||||
|
||||
def test_a_column_named_like_its_table_is_a_column_where_the_schema_says_so():
|
||||
@@ -188,8 +189,11 @@ def test_sql_that_cannot_be_read_but_has_a_star_or_a_table_statement_hides_every
|
||||
"SELECT amount * 2 AS j FROM sales WHERE (",
|
||||
])
|
||||
def test_sql_that_cannot_be_read_with_only_a_count_or_a_product_shows_the_rest(sql):
|
||||
"""只有 COUNT(*)、乘号:不会把遮罩列带出来,照旧只遮遮罩列本身。"""
|
||||
assert hidden_of(sql, ["j"]) == set(MASKED)
|
||||
"""只有 COUNT(*)、乘号:不会把遮罩列带出来,照旧只遮遮罩列本身(没有整行引用的方言)。Postgres 里解析不了的一律
|
||||
整份遮:整行引用(row_to_json(s)、SELECT s)不写列名、也没有 *,只看文字认不出(见 test_masking_opaque_sql)。"""
|
||||
assert hidden_of(sql, ["j"], kind="sqlite") == set(MASKED)
|
||||
assert hidden_of(sql, ["j"], kind="mysql") == set(MASKED)
|
||||
assert hidden_of(sql, ["j"]) == set(MASKED) | {"j"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user