Files
AgentLab/backend/tests/test_report_masked_values.py
T
ThearchyHelios 65d6a1e63c M2 复核第 3 轮:报告自查只查引用、别的表里同名的列不算;裁判摘录退回只按数据源设的遮罩列;收遮罩值只收确定取自遮罩列的值;遮罩分析和收到的值按快照工件记在进程里
第 2 轮的按值自查和面板那套宁可多遮的认法带进了误判,正式运行因此失败,代价远大于收益,这一轮回到规格原意:

- 报告自查去掉按值在正文里找遮罩值(engine/evidence.py 的 masked_values、report.py 的收值):两个字的姓名会撞上「销售
  高峰」这类常用词,同名列的值会撞上门店名;照抄成正文的号码本来就会被「数字没有出处」拦下。时间线日志 report_masked_cell
  不再带 written,decode.ts、terms.ts、证据审计的 CSV 回到「引用了遮罩列」一种说法,check-decode 同步
- 自查认引用时从 _mask_sets 的 sure 起,再按 SQL 血缘(sqlglot.lineage,按快照里冻结的表结构)核一遍值取自哪张表的哪一列
  (api/evidence.py 的 _cited_masks / _owned_columns):遮罩列不带表名,同名的列在几张表里都有、其中有的表另有别的遮罩列时
  (customers 有 name 和 phone,stores 只有 name),只认那几张表,门店名不算客户姓名;值根本不取自遮罩列的(region AS
  phone)也不算。认不出出处的照算,没探查过表结构的照列名认。违规说明点名它取自的那一列遮罩列
- 交给裁判的摘录退回 M2 之前的做法,只按数据源设的列名遮(删掉 _judge_masks):并进面板那套宁可多遮的列后,整份遮的快照、
  只数个数的列从摘录里消失,被引用的金额、人数裁判看不到。按需裁判和节点里当场裁判都改回
- 证据面板、推演收「这次运行查到过的遮罩值」只收 sure 的列(_snapshot_secrets,同 masking 收值的口径):SQL 认不准、整份
  遮的快照只遮显示,里面的区域名、金额不再收成遮罩值
- 按快照工件 id 在进程里记下遮罩分析和收到的值(_SnapshotMemo,键里带数据源现在设的遮罩、在不在、类型,设置改了不拿旧的;
  只记读得出来的快照,存取加锁,分析结果冻结、交出去的值另抄一份):之后点开片段、推演都不再把全部快照读盘、复验、分析,
  只读这一步要显示的那一份。30 份 500 行的快照,第一次约 0.7 秒,之后不到 1 毫秒

测试:新增 test_report_masked_owners.py(高峰、门店名写成文字照常出具,照抄号码由数字核对拦下,引用门店名不算、引用客户名算,
同一条查询里两张表的同名列、只设 name、* 展开、WITH、UNION、没有表结构、表值函数)、test_evidence_mask_memo.py(整份遮
的快照不收值、只读一次只分析一次、设置改了重新收、推演共用、再点开片段只读本步快照);裁判摘录改测被引用的金额和计数
都在摘录里;按值自查的用例改成不按值认的回归
2026-10-12 12:33:32 +08:00

264 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""报告自查:只数个数的列不算取自遮罩列;正文里写着的值不按值认。
复核发现:
- 只数个数、只决定算不算进来的写法——COUNT(*) FILTER (WHERE phone IS NULL)、SUM(phone IS NULL)、AVG(phone IS NOT NULL)、
SUM(amount) FILTER (WHERE phone IS NOT NULL)——值里没有号码,可认遮罩列的那一套(masking._value_columns)把 FILTER 的
条件、汇总里的真假条件当成值的来历,自查跟着判 masked_cell,正式运行(默认违规即失败)因此判失败。等价的
SUM(CASE WHEN phone IS NULL THEN 1 ELSE 0 END) 本来就不算。现在自查认之前先把这几种写法改写成认得出的样子
(api/evidence.py 的 _counted),证据面板、推演照可验证报告的认法多遮(展示上多遮无妨)
- 第 2 轮另按值认过正文里写着的遮罩值(照抄的号码、姓名,经指标带出来的值),第 3 轮去掉了:两个字的姓名会撞上「销售
高峰」这类常用词,同名列的值会撞上门店名,正式运行因此判失败,代价远大于收益;照抄的号码本来就会被「数字没有出处」
拦下(test_report_masked_owners)。自查只查 [[v:]] 引用,这里留着不按值认的回归
真跑的部分用虚构的 SQLite 商城库(号码、姓名是编的),写作者是 MockChatModel 照剧本写。
"""
from __future__ import annotations
import asyncio
import sqlite3
import pytest
from langchain_core.messages import AIMessage
from sqlalchemy import select
from app.core import artifact_store
from app.data.engine import engines
from app.db.base import SessionLocal
from app.db.models import DataSource, Run, RunEvent
from app.engine.evidence import compose_doc
from app.engine.runner import run_manager
CUSTOMERS = [(1, "陈思远", "19900000701"), (2, "林若溪", "19900000702"), (3, "周以宁", "19900000703")]
ORDERS = [(1, 1, 120.5), (2, 2, 300.0), (3, 2, 80.0), (4, 3, 210.25), (5, 1, 99.5)]
PHONES = [c[2] for c in CUSTOMERS]
NAMES = [c[1] for c in CUSTOMERS]
@pytest.fixture
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
async def set_source(path, options=None) -> str:
async with SessionLocal() as session:
row = (await session.execute(select(DataSource).where(DataSource.name == "mall"))).scalar_one_or_none()
if row is None:
row = DataSource(name="mall", kind="sqlite", database=str(path), readonly=True, enabled=True)
session.add(row)
row.database, row.kind, row.enabled, row.readonly = str(path), "sqlite", True, True
row.options = dict(options or {})
await session.commit()
return row.id
@pytest.fixture
async def mall(tmp_path, engine_up):
path = tmp_path / "mall.db"
db = sqlite3.connect(path)
db.executescript("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, phone TEXT);"
"CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_id INTEGER, amount REAL);")
db.executemany("INSERT INTO customers VALUES (?, ?, ?)", CUSTOMERS)
db.executemany("INSERT INTO orders VALUES (?, ?, ?)", ORDERS)
db.commit()
db.close()
source_id = await set_source(path, {"mask_columns": ["phone"]})
await engines.invalidate(source_id)
yield path
await engines.invalidate(source_id)
def masked_of(violations: list[dict]) -> list[dict]:
return [v for v in violations if v["code"] == "masked_cell"]
# --------------------------------------------------------------------------
# 只数个数的列
# --------------------------------------------------------------------------
#: 值里没有号码的写法:条件只决定这一行算不算进来,汇总里的真假条件只数个数
COUNTING = [
"SELECT COUNT(*) FILTER (WHERE phone IS NULL) AS missing FROM customers",
"SELECT SUM(phone IS NULL) AS missing FROM customers",
"SELECT AVG(phone IS NOT NULL) AS missing FROM customers",
"SELECT SUM(LENGTH(phone) = 11) AS missing FROM customers",
"SELECT MAX((phone LIKE '199%')) AS missing FROM customers",
"SELECT SUM(o.amount) FILTER (WHERE c.phone IS NOT NULL) AS missing FROM orders o "
"JOIN customers c ON c.id = o.customer_id",
"SELECT COUNT(*) FILTER (WHERE phone IS NULL) OVER () AS missing FROM customers",
]
@pytest.mark.parametrize("sql", COUNTING)
async def test_counting_columns_are_not_surely_masked(engine_up, mall, sql):
"""自查不为它们判违规(sure 里没有);证据面板照可验证报告的认法多遮(hidden 里有,只影响展示)。"""
from app.api.evidence import _mask_sets, _Masks
sets = await _mask_sets({"sql": sql, "columns": ["missing"], "rows": [[1]], "source": "mall"}, "mall", _Masks())
assert sets.sure == set() and "missing" in sets.hidden, (sets.sure, sets.hidden)
@pytest.mark.parametrize("sql", [
# 值本身取自遮罩列的汇总、逐行的函数照旧算
"SELECT MAX(phone) AS missing FROM customers",
"SELECT SUBSTR(phone, 1, 3) AS missing FROM customers",
"SELECT MAX(phone) FILTER (WHERE id > 1) AS missing FROM customers",
# 逐行的真假条件(不汇总)照旧算:每一行是真是假本身就是这一行的信息,同逐行的 CASE
"SELECT phone IS NULL AS missing FROM customers",
])
async def test_values_taken_from_the_masked_column_stay_surely_masked(engine_up, mall, sql):
from app.api.evidence import _mask_sets, _Masks
sets = await _mask_sets({"sql": sql, "columns": ["missing"], "rows": [[1]], "source": "mall"}, "mall", _Masks())
assert sets.sure == {"missing"}, sets.sure
@pytest.mark.parametrize("sql, columns", [
# SQL 解析不了:认不出哪一列取自哪一列,面板整份遮,自查一列都不算(列名就是遮罩列的除外)
("SELECT name, phone AS p FROM customers WHERE ((", ["name", "p"]),
# 派生来源里没起别名的表达式:认遮罩列的那一套整份算,自查同样一列都不算
("SELECT name, amount FROM (SELECT c.name, UPPER(c.phone), o.amount FROM customers c "
"JOIN orders o ON o.customer_id = c.id) t", ["name", "amount"]),
])
async def test_columns_only_counted_because_the_sql_is_unclear_are_not_surely_masked(engine_up, mall, sql, columns):
from app.api.evidence import _mask_sets, _Masks
sets = await _mask_sets({"sql": sql, "columns": columns, "rows": [], "source": "mall"}, "mall", _Masks())
assert sets.sure == set() and set(columns) <= sets.hidden, (sets.sure, sets.hidden)
direct = await _mask_sets({"sql": "SELECT name, phone FROM customers WHERE ((", "columns": ["name", "phone"],
"rows": [], "source": "mall"}, "mall", _Masks())
assert direct.sure == {"phone"}, direct.sure
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def chain(*nodes) -> dict:
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
async def finish(monkeypatch, spec: dict, text: str, payload: dict | None = None) -> Run:
from app.providers import mock_model
monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=text))
run_id = (await run_manager.start(graph=spec, input_payload=payload or {})).id
for _ in range(400):
await asyncio.sleep(0.05)
async with SessionLocal() as session:
row = await session.get(Run, run_id)
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
return row
raise AssertionError(f"运行 {run_id} 没跑完:{row.status}")
async def events(run_id: str) -> list[RunEvent]:
async with SessionLocal() as session:
q = select(RunEvent).where(RunEvent.run_id == run_id).order_by(RunEvent.seq)
return list((await session.execute(q)).scalars())
def one_query(sql: str, **report) -> dict:
return chain(node("start", "input"), node("top", "tool", tool="db_query__mall", args={"sql": sql}),
node("write", "report", instructions="写一句", **report),
node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}]))
@pytest.mark.parametrize("sql", COUNTING[:2])
async def test_a_formal_report_citing_a_count_of_missing_phones_succeeds(monkeypatch, mall, sql):
"""违规即失败(正式运行的默认值):数据质量报告引用「手机号缺失的人数」照常出具,不判违规。"""
row = await finish(monkeypatch, one_query(sql, on_violation="fail"), "手机号缺失的客户有 [[v:Q1.r0.missing]] 位。")
assert row.status == "succeeded", row.error
checked = next(e.data for e in await events(row.id) if e.type == "report.checked")
assert masked_of(checked["violations"]) == [] and checked["ok"] is True, checked["violations"]
# --------------------------------------------------------------------------
# 正文里写着的值不按值认
# --------------------------------------------------------------------------
async def masked_catalog(sql: str, columns: list[str], rows: list[list]) -> dict:
art = await artifact_store.put_json({"columns": columns, "rows": rows, "source": "mall", "sql": sql}, kind="query")
return {"Q1": {"alias": "Q1", "kind": "query", "artifact": art, "columns": columns, "rows": len(rows),
"node_id": "top", "source": "mall"}}
SQL_TOP = "SELECT c.name AS customer, c.phone AS tel, 1 AS spend FROM customers c"
TOP_ROWS = [[NAMES[1], PHONES[1], 380.0], [NAMES[0], PHONES[0], 220.0]]
@pytest.mark.parametrize("text", [
f"消费最高的客户电话是 {PHONES[1]}。",
"消费最高的客户电话是 199-0000-0702。",
f"消费最高的客户电话是 +86 {PHONES[1][:3]} {PHONES[1][3:7]} {PHONES[1][7:]}。",
])
async def test_a_value_copied_into_the_text_is_not_a_masked_cell(engine_up, mall, text):
"""写作者把号码照抄成正文(没写成引用):自查不按值认,不记 masked_cell;裸数字由数字核对记 uncited_number。"""
from app.engine.nodes.report import _MaskedCells
catalog = await masked_catalog(SQL_TOP, ["customer", "tel", "spend"], TOP_ROWS)
doc = await _MaskedCells(catalog)(compose_doc(text, catalog, node_id="write", run_id="r1"))
assert masked_of(doc["violations"]) == [] and any(v["code"] == "uncited_number" for v in doc["violations"])
async def test_a_name_copied_as_text_is_not_flagged(engine_up, mall):
"""文字类的遮罩列(姓名)照抄成正文:同样不按值认(两个字的姓名会撞上常用词)。"""
from app.engine.nodes.report import _MaskedCells
await set_source(mall, {"mask_columns": ["name"]})
catalog = await masked_catalog(SQL_TOP, ["customer", "tel", "spend"], TOP_ROWS)
doc = await _MaskedCells(catalog)(compose_doc(f"消费最高的是{NAMES[1]},花了 [[v:Q1.r0.spend]]。", catalog,
node_id="write", run_id="r1"))
assert masked_of(doc["violations"]) == [], doc["violations"]
async def test_a_cited_masked_cell_is_flagged_once(engine_up, mall):
"""[[v:Q1.r0.tel]]:按引用记一条,没有按值另记的那一类(by_value)。"""
from app.engine.nodes.report import _MaskedCells
catalog = await masked_catalog(SQL_TOP, ["customer", "tel", "spend"], TOP_ROWS)
doc = await _MaskedCells(catalog)(compose_doc("电话 [[v:Q1.r0.tel]]。", catalog, node_id="write", run_id="r1"))
[v] = masked_of(doc["violations"])
assert v["ref"] == "v:Q1.r0.tel" and "by_value" not in v, v
async def test_no_snapshot_is_read_without_a_cited_cell(engine_up, mall, monkeypatch):
"""正文没引用格子:数据源设了遮罩也一份快照都不读(以前按值认要把设了遮罩的快照都读一遍)。"""
from app.core import artifact_store as store
from app.engine.nodes.report import _MaskedCells
catalog = await masked_catalog(SQL_TOP, ["customer", "tel", "spend"], TOP_ROWS)
read: list[str] = []
real = store.load
monkeypatch.setattr(store, "load", lambda a: read.append(a) or real(a))
await _MaskedCells(catalog)(compose_doc(f"一共三位客户,电话 {PHONES[0]}。", catalog, node_id="write", run_id="r1"))
assert read == [], read
METRICS = [
{"id": "phone_raw", "name": "首位客户手机号", "format": "plain", "expression": "cell(nodes.top, 0, 'tel')"},
{"id": "id3", "name": "第三位客户编号", "format": "plain", "expression": "cell(nodes.top, 2, 'id')"},
]
def card_graph(**report) -> dict:
return chain(node("start", "input"),
node("top", "tool", tool="db_query__mall",
args={"sql": "SELECT id, name AS customer, phone AS tel FROM customers ORDER BY id"}),
node("card", "metrics", caliber="遮罩对照", caliber_version="v1", metrics=METRICS),
node("write", "report", instructions="写一句", **report),
node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}]))
async def test_a_metric_reference_is_not_checked_by_value(monkeypatch, mall):
"""自查只查 [[v:]] 引用:[[m:phone_raw]] 的值就是号码,自查不按值认(指标的值在证据面板、推演里另有判断,写「已遮罩」,
见 test_masked_metric_values)。违规即失败的运行照常成功,时间线上没有「引用了遮罩列」那一行。"""
row = await finish(monkeypatch, card_graph(on_violation="fail"), "第三位客户编号 [[m:id3]],首位客户电话 [[m:phone_raw]]。")
assert row.status == "succeeded", row.error
rows = await events(row.id)
checked = next(e.data for e in rows if e.type == "report.checked")
assert masked_of(checked["violations"]) == [], checked["violations"]
assert not [e for e in rows if e.type == "log" and e.data.get("code") == "report_masked_cell"]