Files
AgentLab/backend/tests/test_masked_merge_queries.py
T
ThearchyHelios ce4be9cc8f M2 复核第 2 轮:报告自查按值认出正文里写着的遮罩值,只数个数的列不再判违规;遮没遮核对不了的输入写「无法核对」;整句面板挂的依据按证据图的标签写
- 自查按值认(engine/evidence.py 的 masked_values,report.py 的 _MaskedCells):写作目录里确定取自遮罩列的值出现在正文里——照抄成文字(姓名这类文字列数字核对管不到)、写成数字(换了写法的长号码组装时被拆成几个数字片段,按整句认)、经指标带出来([[m:phone_raw]] 的值就是号码)——记一条 masked_cell(by_value),不带值,交回写作者时附上值换成「已遮罩」的那一句。只收确定取自遮罩列的列、不收分档标签;运行输入里写着的值是发起运行的人自己填的,不算。只读数据源现在设了遮罩的快照,认引用时读过的不再读,没设遮罩的运行一份都不读
- 时间线:日志 report_masked_cell 多带 written(其中按值认出的几处),标题分「引用了…格子」「写进了…值」「引用、写进了…」;缺口、日志按两种分别说;违规清单、CSV 的叫法改成「写进了遮罩列的值」
- 只数个数的列(COUNT(*) FILTER (WHERE phone IS NULL)、SUM(phone IS NULL)、AVG(phone IS NOT NULL)、SUM(amount) FILTER (WHERE phone IS NOT NULL)):认遮罩列的那一套把 FILTER 条件、汇总里的真假条件当成值的来历,数据质量报告引用「手机号缺失的人数」在正式运行里判失败。自查认之前先改写(_counted:FILTER 同 WHERE 去掉,汇总里的真假条件改成 CASE WHEN … THEN 1 ELSE 0 END),不判违规;证据面板、推演照可验证报告的认法多遮(只影响展示),根上的改法在 masking 里
- 证据面板:遮没遮核对不了的输入(快照、取值链上的产出读不出来),值和代入式里写「无法核对」,不写「已遮罩」,数据源可能根本没设遮罩
- 整句面板「挂的依据」不再直接读文档目录里的「名称 = 渲染值」:按证据图的标签写(服务端已把用到遮罩列的指标值换成「已遮罩」),证据图没回来时指标只写名字
- on_demand:当场裁判过、文档里却压根没有这句的判定时,不说「原因见判定」
- 测试:合并查询自查的用例补上拼接、upper、WITH;推演对整行、下标不是字面量的一格说「用到了遮罩列」(顺着取值链认出来的,不是按值猜的);计数指标 cnt 的原值照给;SQL 认不准、只数个数的反例
2026-10-12 10:48:19 +08:00

280 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""合并查询里对遮罩列改名、套函数、拼接、汇总、经 WITH 转一道得来的结果列,证据面板、勾行、报告自查、指标都认得出。
复核发现:合并查询的遮罩以前只按合并当时记下的列名和逐格来历认,逐格来历又只认「单层 SELECT、选取项是输入表的列」
这一种写法。合并 SQL 写 SELECT a.tel AS contact, SUM(b.amount) … GROUP BY a.tel、substr(a.phone, 1, 11) AS p2、
a.tel || '' AS p2、WITH x AS (SELECT a.tel AS p2 FROM a) … 时没有逐格来历,号码在片段接口的查询步骤、推演的勾行接口里
原样给出,报告自查也认不出 [[v:Q3.r0.contact]];可验证报告对同一份快照只给哈希。现在合并 SQL 本身也按 SQL 认
(api/evidence.py 的 _merge_mask_sets):各输入里要遮的列名当作遮罩列,按 SQLite 的读法认合并 SQL。另外:
- 输入快照读不出来、又可能带着遮罩的:它的列全部当遮罩列(宁可多遮)
- 合并警告里举的例子(类型不一致的键、重复的键)是号码的,换成「已遮罩」
真跑一遍图(虚构的 SQLite 商城库,号码是编的),报告由 MockChatModel 照剧本写。
"""
from __future__ import annotations
import asyncio
import json
import sqlite3
import pytest
from httpx import ASGITransport, AsyncClient
from langchain_core.messages import AIMessage
from sqlalchemy import select
from app.core import artifact_store
from app.data.engine import engines
from app.db.base import SessionLocal
from app.db.models import DataSource, Run, RunEvent
from app.engine.evidence import iter_segments
from app.engine.runner import run_manager
from app.main import app
CUSTOMERS = [(1, "客户甲", "19900000101"), (2, "客户乙", "19900000102"), (3, "客户丙", "19900000103")]
ORDERS = [(1, 1, 120.5), (2, 2, 300.0), (3, 2, 80.0), (4, 3, 210.25), (5, 1, 99.5)]
PHONES = [c[2] for c in CUSTOMERS]
#: 输入里给号码起了别名(tel);另一份按客户汇总金额
Q_TEL = "SELECT id, phone AS tel FROM customers ORDER BY id"
Q_PHONE = "SELECT id, phone FROM customers ORDER BY id"
Q_ORD = "SELECT customer_id AS id, SUM(amount) AS amount FROM orders GROUP BY customer_id ORDER BY id"
@pytest.fixture
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
@pytest.fixture
async def client(engine_up):
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
async def set_source(path, options=None):
async with SessionLocal() as session:
row = (await session.execute(select(DataSource).where(DataSource.name == "mall"))).scalar_one_or_none()
if row is None:
row = DataSource(name="mall", kind="sqlite", database=str(path), readonly=True, enabled=True)
session.add(row)
row.database, row.kind, row.enabled, row.readonly = str(path), "sqlite", True, True
row.options = dict(options or {})
await session.commit()
return row.id
@pytest.fixture
async def mall(tmp_path, engine_up):
path = tmp_path / "mall.db"
db = sqlite3.connect(path)
db.executescript("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, phone TEXT);"
"CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_id INTEGER, amount REAL);")
db.executemany("INSERT INTO customers VALUES (?, ?, ?)", CUSTOMERS)
db.executemany("INSERT INTO orders VALUES (?, ?, ?)", ORDERS)
db.commit()
db.close()
source_id = await set_source(path, {"mask_columns": ["phone"]})
await engines.invalidate(source_id)
yield path
await engines.invalidate(source_id)
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def chain(*nodes) -> dict:
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
def merge_graph(cust_sql: str, merge_sql: str, *, metrics: list | None = None, **report) -> dict:
nodes = [node("start", "input"),
node("q_cust", "tool", tool="db_query__mall", args={"sql": cust_sql}),
node("q_ord", "tool", tool="db_query__mall", args={"sql": Q_ORD}),
node("merged", "merge", inputs={"a": "q_cust", "b": "q_ord"}, sql=merge_sql)]
if metrics:
nodes.append(node("card", "metrics", caliber="合并口径", caliber_version="v1", metrics=metrics))
nodes += [node("write", "report", instructions="写一句", **{"on_violation": "flag", **report}),
node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}])]
return chain(*nodes)
async def finish(monkeypatch, spec: dict, text: str) -> Run:
from app.providers import mock_model
monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=text))
run_id = (await run_manager.start(graph=spec, input_payload={})).id
for _ in range(400):
await asyncio.sleep(0.05)
async with SessionLocal() as session:
row = await session.get(Run, run_id)
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
assert row.status == "succeeded", row.error
return row
raise AssertionError(f"运行 {run_id} 没跑完:{row.status}")
def leaks(value) -> list[str]:
"""写着的号码,按千分位写的(19,900,000,101)也算。"""
raw = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
return [p for p in PHONES if p in raw or p in raw.replace(",", "")]
async def doc_of(client, run_id: str) -> dict:
[report] = (await client.get(f"/api/runs/{run_id}/evidence")).json()["reports"]
return artifact_store.load(report["doc_artifact"])
async def segment(client, run_id: str, ref: str) -> dict:
doc = await doc_of(client, run_id)
seg = next(s for s in iter_segments(doc) if s.get("ref") == ref)
res = await client.get(f"/api/runs/{run_id}/evidence/segments/{seg['id']}")
assert res.status_code == 200, res.text
return res.json()
def merged_step(body: dict) -> dict:
return next(s for s in body["chain"] if s["step"] == "query" and s.get("merge"))
async def checked_of(run_id: str) -> dict:
async with SessionLocal() as session:
rows = list((await session.execute(select(RunEvent).where(RunEvent.run_id == run_id,
RunEvent.type == "report.checked"))).scalars())
return rows[-1].data
#: (输入查询, 合并 SQL, 取自遮罩列的结果列):都没有逐格来历,或者来历指不到这一列
CASES = [
# 改名再按它汇总(验收配方的合并写法)
(Q_TEL, "SELECT a.tel AS contact, SUM(b.amount) AS spend FROM a JOIN b ON a.id = b.id GROUP BY a.tel "
"ORDER BY spend DESC", "contact"),
# 输入没改名,合并里改名、按它分组
(Q_PHONE, "SELECT a.phone AS contact, SUM(b.amount) AS spend FROM a JOIN b ON a.id = b.id GROUP BY a.phone "
"ORDER BY spend DESC", "contact"),
# 套函数
(Q_PHONE, "SELECT a.id, substr(a.phone, 1, 11) AS contact, b.amount AS spend FROM a JOIN b ON a.id = b.id "
"ORDER BY a.id", "contact"),
(Q_TEL, "SELECT a.id, upper(a.tel) AS contact, b.amount AS spend FROM a JOIN b ON a.id = b.id ORDER BY a.id",
"contact"),
# 拼接
(Q_TEL, "SELECT a.id, a.tel || '' AS contact, b.amount AS spend FROM a JOIN b ON a.id = b.id ORDER BY a.id",
"contact"),
# 经 WITH 转一道
(Q_TEL, "WITH x AS (SELECT a.id AS id, a.tel AS contact FROM a) SELECT x.id, x.contact, b.amount AS spend "
"FROM x JOIN b ON x.id = b.id ORDER BY x.id", "contact"),
]
@pytest.mark.parametrize("cust_sql, merge_sql, column", CASES)
async def test_the_query_step_and_the_rows_page_mask_a_column_derived_in_the_merge_sql(
client, monkeypatch, mall, cust_sql, merge_sql, column):
"""点开正文里引用的金额:合并步骤里那一列遮掉,勾行接口同样遮,号码一个都不交出来。"""
row = await finish(monkeypatch, merge_graph(cust_sql, merge_sql), "合并后第一行金额 [[v:Q3.r0.spend]]。")
body = await segment(client, row.id, "v:Q3.r0.spend")
step = merged_step(body)
assert column in step["masked"], step["masked"]
at = step["columns"].index(column)
assert all(r[at] == "已遮罩" for r in step["rows"]) and leaks(body) == [], step["rows"]
page = (await client.get(f"/api/runs/{row.id}/evidence/whatif/rows", params={"artifact": step["artifact"]})).json()
assert column in page["masked"] and leaks(page) == [], page["masked"]
@pytest.mark.parametrize("cust_sql, merge_sql, column", CASES)
async def test_the_self_check_flags_a_cell_derived_in_the_merge_sql(monkeypatch, mall, cust_sql, merge_sql, column):
"""写作者直接引用合并结果里取自遮罩列的那一格:自查记 masked_cell,别的格照常。"""
row = await finish(monkeypatch, merge_graph(cust_sql, merge_sql),
f"联系方式 [[v:Q3.r0.{column}]],金额 [[v:Q3.r0.spend]]。")
checked = await checked_of(row.id)
assert [v["ref"] for v in checked["violations"] if v["code"] == "masked_cell"] == [f"v:Q3.r0.{column}"], \
checked["violations"]
assert leaks(checked["violations"]) == []
async def test_a_metric_taking_a_cell_derived_in_the_merge_sql_is_withheld(client, monkeypatch, mall):
"""口径卡用 cell() 取合并结果里那一列:指标步骤的输入、指标的值都不给,推演按遮罩处理、原值也不给。"""
metrics = [{"id": "contact_len", "name": "联系方式位数", "unit": "位", "format": "thousands",
"expression": "len(str(cell(nodes.merged, 0, 'contact')))"},
{"id": "top_spend", "name": "最高消费", "unit": "元", "decimals": 2, "format": "thousands",
"expression": "cell(nodes.merged, 0, 'spend')"}]
row = await finish(monkeypatch, merge_graph(Q_TEL, CASES[0][1], metrics=metrics),
"位数 [[m:contact_len]],最高消费 [[m:top_spend]]。")
body = await segment(client, row.id, "m:contact_len")
assert body["chain"][0]["value"] == "已遮罩" and leaks(body) == []
[cell] = [s for s in body["chain"] if s["step"] == "input"]
assert cell["value"] == "已遮罩" and cell["masked"] == ["contact"], cell
res = await client.post(f"/api/runs/{row.id}/evidence/whatif", json={"card": "card"})
metrics_out = {m["id"]: m for m in res.json()["metrics"]}
assert metrics_out["contact_len"]["status"] == "masked" and metrics_out["contact_len"]["old"] == "已遮罩"
assert metrics_out["top_spend"]["status"] == "unchanged" and metrics_out["top_spend"]["old"] == 380.0
assert leaks(res.json()) == []
async def test_without_masks_the_merge_step_is_unchanged(client, monkeypatch, mall):
await set_source(mall, {})
row = await finish(monkeypatch, merge_graph(Q_TEL, CASES[0][1]), "合并后第一行金额 [[v:Q3.r0.spend]]。")
step = merged_step(await segment(client, row.id, "v:Q3.r0.spend"))
assert step["masked"] == [] and step["rows"][0][0] in PHONES
# --------------------------------------------------------------------------
# 输入快照读不出来:它的列全部当遮罩列
# --------------------------------------------------------------------------
async def test_an_unreadable_input_masks_every_column_that_may_come_from_it(engine_up, mall):
"""输入快照读不出来(文件丢了、被改了),它的数据源又设了遮罩:认不出输入里哪一列改过名字,它的每一列都当遮罩列,
合并结果里取自它的列(逐格来历指向它的、合并 SQL 里取自它的)都遮;另一个读得出、没遮罩的输入照常。"""
from app.api.evidence import _mask_sets, _Masks
ord_snap = await artifact_store.put_json({"columns": ["id", "amount"], "rows": [[1, 220.0], [2, 380.0]],
"source": "mall", "sql": Q_ORD}, kind="query")
snap = {"columns": ["contact", "spend"], "rows": [["已脱敏", 380.0]], "source": "合并查询",
"sql": "SELECT a.tel AS contact, SUM(b.amount) AS spend FROM a JOIN b ON a.id = b.id GROUP BY a.tel",
"sources": ["mall"],
"inputs": [{"alias": "a", "artifact": "0" * 64, "columns": ["id", "tel"], "source": "mall"},
{"alias": "b", "artifact": ord_snap, "columns": ["id", "amount"], "source": "mall"}]}
sets = await _mask_sets(snap, "合并查询", _Masks())
assert "contact" in sets.hidden and "spend" not in sets.hidden, sets.hidden
# 确定取自遮罩列的只认数据源设的列名:tel 是不是号码核对不了,报告自查不为它判违规
assert "contact" not in sets.sure
# 连输入有哪几列都不知道(老快照):整份结果的列都遮
old = {**snap, "inputs": [{"alias": "a", "artifact": "0" * 64, "source": "mall"}, snap["inputs"][1]]}
assert {"contact", "spend"} <= (await _mask_sets(old, "合并查询", _Masks())).hidden
# --------------------------------------------------------------------------
# 合并警告举的例子
# --------------------------------------------------------------------------
async def test_merge_warnings_do_not_quote_masked_values(client, monkeypatch, tmp_path, engine_up):
"""按号码合并、两边类型不一致又有重复的键:合并警告举的例子是号码,查询步骤里换成「已遮罩」,说明写清只是展示时换。"""
path = tmp_path / "keys.db"
db = sqlite3.connect(path)
db.executescript("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, phone TEXT);"
"CREATE TABLE orders (id INTEGER PRIMARY KEY, phone INTEGER, amount REAL);")
db.executemany("INSERT INTO customers VALUES (?, ?, ?)",
[(1, "客户甲", "19900000101"), (2, "客户甲家属", "19900000101"), (3, "客户乙", "19900000102")])
db.executemany("INSERT INTO orders VALUES (?, ?, ?)",
[(1, 19900000101, 120.5), (2, 19900000101, 30.0), (3, 19900000102, 80.0)])
db.commit()
db.close()
source_id = await set_source(path, {"mask_columns": ["phone"]})
await engines.invalidate(source_id)
spec = chain(node("start", "input"),
node("q_cust", "tool", tool="db_query__mall", args={"sql": "SELECT name, phone FROM customers ORDER BY id"}),
node("q_ord", "tool", tool="db_query__mall", args={"sql": "SELECT phone, amount FROM orders ORDER BY id"}),
node("merged", "merge", inputs={"a": "q_cust", "b": "q_ord"},
sql="SELECT a.name, b.amount FROM a JOIN b ON a.phone = b.phone"),
node("write", "report", instructions="写一句", on_violation="flag"),
node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}]))
row = await finish(monkeypatch, spec, "第一行金额 [[v:Q3.r0.amount]]。")
step = merged_step(await segment(client, row.id, "v:Q3.r0.amount"))
warnings = step["merge"]["warnings"]
assert {w["code"] for w in warnings} >= {"key_type_mismatch", "rows_grew"}, warnings
assert leaks(warnings) == [] and all("已遮罩" in w["message"] for w in warnings), warnings
assert "仅在展示时" in step["mask_note"], step.get("mask_note")
await engines.invalidate(source_id)