Files
AgentLab/backend/tests/test_evidence_masked_aliases.py
T
ThearchyHelios abb2707904 期 5 验收修复 N1:取自遮罩格的口径卡指标不进报告文档和可验证报告;显示文字核对堵住紧挨着改字的绕过
验收第二轮发现:口径卡 phone_raw = cell(q_cust, 0, 'phone') 被 [[m:phone_raw]] 引用时自查不拦,号码写进正文;写作目录、
文档的证据目录收着上游口径卡的全部指标和原值;可验证报告里口径卡的节点完成事件(预览逐条写着「指标(名称)= 值」)一律
附规范串,没有报告的运行也照样带出号码;另外只改紧挨着被引用数字的普通正文(前面补一位数、加负号、后面接「0 万」)、重算
全部哈希,读者看到的是另一个数,核对页只标无法核对,总览仍是绿的。

报告撰写节点
- 写作之前按证据面板指标步骤的那一套认出值取自遮罩格的指标(api/evidence.py 的 withheld_metrics → _metric_masks,
  recompute.withheld_value):cell() 取的遮罩格、改了名字的遮罩列、按位置取的一格一行、整份结果里带着遮罩值的。写作
  目录、存进文档的证据目录里这些指标的值、渲染、标签都写「已遮罩」并记 masked(evidence.withhold_metrics,没被引用的
  也一样),写作目录另说不要引用
- 只按出处认(by_value=False):认不出出处、值碰巧等于遮罩列某一格的普通指标(写死的目标 20000)不算,不然它在报告里
  成了「已遮罩」、引用它的正式运行判失败、写作者也改不掉(自查第 3 轮去掉按值认是同一个道理);证据面板展示上照旧多遮
- 引用这类指标照样解析、正文写「已遮罩」(换算不再算),自查记 masked_cell(evidence.masked_metrics),和引用遮罩格同一
  套处置:交回重写,仍不改按「违规处置」办;缺口、日志、判失败的下一步说「指标」
- 写成「已遮罩」的别名记进节点产出(masked_metrics),出口契约复核按同一份重建目录(report_catalog),正文里的「已遮罩」
  对得上,不说文档被改过;写作时按上游状态建目录,循环里重跑不沿用上一次记下的那份
- 证据面板点开这一处:值、渲染都是「已遮罩」,按文档写的比渲染,不报渲染不一致

可验证报告(bundle/build.py、verifier.js)
- 口径卡的节点完成事件:卡只给 id(有输入取自遮罩列,或者有指标的值不给)的只给哈希,和报告引不引用无关;没有台账的
  老运行按预览里的 artifact、节点产出工件认,都读不出来的按运行里有没有带遮罩的快照
- 卡只给 id 的判断另按指标认(_Store.metric_word,和写作目录同一个判断):整行、下标不是字面量的一格、整份结果转成的
  文字,以前只认字面量下标的一格,卡原样嵌了进去
- 证据目录里还写着这类原值的老文档(升级前生成的,或者生成之后数据源才设的遮罩)整份只给哈希、不附带它引用的工件,
  报告节点的完成事件也只给哈希;快照读不出来时按它的数据源现在设的遮罩认,卡读不出来时按这次运行查到过的遮罩值认。
  核对页照实说明原因(中性色),这份报告算未能核对,总览不是绿的;文档里写成「已遮罩」的指标所在的卡一律只给 id
- 被引用的数字紧挨着(中间没有空白、标点)数字、负号、小数点判不通过,点名「正文里紧挨着这个数的字改变了读者看到的
  数」;排在看不见、改变方向的字前面判,同一句里再放一个这种字也降不成无法核对。服务端组装的文档两个数之间补了空格,
  原样导出照旧全绿(开发数据里 39 份报告、505 处引用,没有一处紧挨着)

测试
- 新增 tests/test_report_masked_metrics.py(目录、自查、重写、标注 / 失败、出口契约、证据面板、循环)、
  tests/test_bundle_masked_metrics.py(没有报告的运行、新文档、老文档连同核对页、普通卡、和写作目录同一个判断、手搭的
  几条分支);样本页 masked-card 改成现在的报告撰写节点写的文档,新增 masked-old-doc;check-bundle 补数字前加数字、加
  负号、后面接「0 万」三种篡改,各自变红、点名,原有的紧挨着补位、连字符两条改成不通过,报告文档标成遮罩那条改成总览
  不是绿的
- test_report_masked_values 的 [[m:phone_raw]] 改成按出处拦、写死的热线号码照常引用;几处按旧的报告正文(「13位」
  「11位」「26位」)找片段的测试改成按引用找(test_evidence_metric_masks、test_evidence_masked_aliases、
  test_whatif_masked_values),断言照新的正文写
2026-10-12 17:23:27 +08:00

311 lines
16 KiB
Python

"""证据面板认得出改了名字的遮罩列:查询步骤、指标的输入、推演的勾行和遮罩,按 SQL 认。
验收发现:正式运行的报告没写号码,可验证报告也把 top 查询(c.phone AS tel)的快照只给了哈希,点开正文里的消费额,
片段接口的查询步骤却是 columns=[customer, tel, spend]、masked=[],三个号码原样交出来。证据面板以前只遮列名恰好等于
遮罩列的那几列;可验证报告、核查 agent 早已按 SQL 认出别名、函数、子查询、WITH 转一道得来的结果列。现在证据面板
按同一套认法(engine/masking.py 的 _derived_hidden):
- 查询步骤:c.phone AS tel 的 tel 列遮掉,面板底下的遮罩说明带上它
- 指标的输入:cell() 取 tel 那一格、按位置取的 vars.res.rows[0][1] 都不给值;指标本身的值也不给
- 推演:勾行接口遮 tel,用到 tel 的指标推演不了、原值不给
- SQL 解析不了的,整份快照的列都遮(保守规则,和可验证报告同一个函数)
- 合并查询:输入快照里改了名字的遮罩列,按逐格来历遮到合并结果里
- 没设遮罩时一切照旧
真跑一遍图(虚构的 SQLite 商城库,号码是编的),报告由 MockChatModel 照剧本写。
"""
from __future__ import annotations
import asyncio
import json
import sqlite3
import pytest
from httpx import ASGITransport, AsyncClient
from langchain_core.messages import AIMessage
from sqlalchemy import select
from app.core import artifact_store
from app.data.engine import engines
from app.db.base import SessionLocal
from app.db.models import DataSource, Run
from app.engine.evidence import iter_segments
from app.engine.runner import run_manager
from app.main import app
CUSTOMERS = [(1, "客户甲", "199-0000-0101"), (2, "客户乙", "199-0000-0102"), (3, "客户丙", "199-0000-0103"),
(4, "客户丁", "199-0000-0104")]
ORDERS = [(1, 1, 120.5), (2, 2, 300.0), (3, 2, 80.0), (4, 3, 210.25), (5, 4, 50.0), (6, 1, 99.5)]
PHONES = [c[2] for c in CUSTOMERS]
#: 验收原样的写法:号码起了别名 tel,和消费额排在同一份结果里
SQL_TOP = ("SELECT c.name AS customer, c.phone AS tel, ROUND(SUM(o.amount), 2) AS spend "
"FROM orders o JOIN customers c ON c.id = o.customer_id GROUP BY c.id, c.name, c.phone ORDER BY spend DESC")
TEL_CELL = "cell(nodes.top, 0, 'tel')"
#: 号码那一格按位置取:vars.res 是数据整形解析好的查询结果,第 1 列是 tel
TEL_POS = "vars.res.rows[0][1]"
METRICS = [
{"id": "top_spend", "name": "最高消费", "unit": "元", "decimals": 2, "format": "thousands",
"expression": "cell(nodes.top, 0, 'spend')"},
{"id": "tel_len", "name": "号码位数", "unit": "位", "format": "thousands", "expression": f"len({TEL_CELL})"},
{"id": "tel_pos2", "name": "号码位数两倍", "unit": "位", "format": "thousands", "expression": f"len({TEL_POS}) * 2"},
]
TEXT = "消费最高的客户花了 [[v:Q1.r0.spend]],口径卡记 [[m:top_spend]];号码 [[m:tel_len]],两倍 [[m:tel_pos2]]。"
@pytest.fixture(autouse=True)
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
async def set_source(path, options=None):
async with SessionLocal() as session:
row = (await session.execute(select(DataSource).where(DataSource.name == "mall"))).scalar_one_or_none()
if row is None:
row = DataSource(name="mall", kind="sqlite", database=str(path), readonly=True, enabled=True)
session.add(row)
row.database, row.kind, row.enabled, row.readonly = str(path), "sqlite", True, True
row.options = dict(options or {})
await session.commit()
return row.id
@pytest.fixture
async def mall(tmp_path):
path = tmp_path / "mall.db"
db = sqlite3.connect(path)
db.executescript("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, phone TEXT);"
"CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_id INTEGER, amount REAL);")
db.executemany("INSERT INTO customers VALUES (?, ?, ?)", CUSTOMERS)
db.executemany("INSERT INTO orders VALUES (?, ?, ?)", ORDERS)
db.commit()
db.close()
source_id = await set_source(path)
await engines.invalidate(source_id)
yield path
await engines.invalidate(source_id)
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def chain(*nodes) -> dict:
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
def graph() -> dict:
return chain(
node("start", "input"),
node("top", "tool", tool="db_query__mall", args={"sql": SQL_TOP}),
node("parse", "transform", mode="json", template="{{ nodes.top }}", assign_to="res"),
node("card", "metrics", caliber="商城口径", caliber_version="v1", metrics=METRICS),
node("write", "report", instructions="写一句", on_violation="flag"),
node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}]),
)
async def run_graph(monkeypatch, spec: dict | None = None, text: str = TEXT) -> Run:
from app.providers import mock_model
monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=text))
run_id = (await run_manager.start(graph=spec or graph(), input_payload={})).id
for _ in range(400):
await asyncio.sleep(0.05)
async with SessionLocal() as session:
row = await session.get(Run, run_id)
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
assert row.status == "succeeded", row.error
return row
raise AssertionError(f"运行 {run_id} 没跑完:{row.status}")
async def doc_of(client, run_id: str) -> dict:
[report] = (await client.get(f"/api/runs/{run_id}/evidence")).json()["reports"]
return artifact_store.load(report["doc_artifact"])
async def segment(client, run_id: str, *, text: str | None = None, ref: str | None = None) -> dict:
"""按正文里的字或者引用(v:Q1.r0.spend)找片段,取片段接口的答复。"""
doc = await doc_of(client, run_id)
seg = next(s for s in iter_segments(doc) if (text is not None and s["text"] == text)
or (ref is not None and s.get("ref") == ref))
res = await client.get(f"/api/runs/{run_id}/evidence/segments/{seg['id']}")
assert res.status_code == 200, res.text
return res.json()
def leaks(body) -> list[str]:
raw = json.dumps(body, ensure_ascii=False)
return [p for p in PHONES if p in raw]
def steps(body: dict, kind: str) -> list[dict]:
return [s for s in body["chain"] if s["step"] == kind]
# --------------------------------------------------------------------------
# 查询步骤
# --------------------------------------------------------------------------
async def test_a_renamed_masked_column_is_masked_in_the_query_step(client, monkeypatch, mall):
"""验收原样:点开正文里直接引用的消费额,查询步骤里 c.phone AS tel 的 tel 列遮掉,号码一个都不交出来。"""
await set_source(mall, {"mask_columns": ["phone"]})
row = await run_graph(monkeypatch)
body = await segment(client, row.id, ref="v:Q1.r0.spend")
[query] = steps(body, "query")
assert query["columns"] == ["customer", "tel", "spend"] and query["masked"] == ["tel"], query["masked"]
assert [r[1] for r in query["rows"]] == ["已遮罩"] * len(query["rows"]) and query["rows"][0][2] == 380.0
assert body["redacted"]["columns"] == ["tel"] and body["redacted"]["note"]
assert leaks(body) == []
async def test_the_metric_chain_masks_the_renamed_column_too(client, monkeypatch, mall):
"""口径卡指标 → 输入 → 查询:查询步骤同一套认法,tel 列同样遮掉(验收的 s13)。"""
await set_source(mall, {"mask_columns": ["phone"]})
row = await run_graph(monkeypatch)
body = await segment(client, row.id, text="380.00元")
[query] = steps(body, "query")
assert query["masked"] == ["tel"] and leaks(body) == []
[amount] = steps(body, "input")
assert amount["value"] == 380.0 and "masked" not in amount
async def test_an_input_taken_from_a_renamed_masked_column_is_withheld(client, monkeypatch, mall):
"""cell() 取 tel 那一格:输入不给值、写明取自哪一列,代入式里那一处写「已遮罩」,指标本身的值也不给。报告撰写节点写作时
按同一个判断把它在目录里换成了「已遮罩」,报告上的字也是(tests/test_report_masked_metrics.py),按引用找片段。"""
await set_source(mall, {"mask_columns": ["phone"]})
row = await run_graph(monkeypatch)
body = await segment(client, row.id, ref="m:tel_len")
metric = body["chain"][0]
assert metric["step"] == "metric" and metric["substituted"] == "len(已遮罩)"
assert metric["value"] == "已遮罩" and metric["rendered"] == "已遮罩" and metric["render_ok"] is True
[cell] = steps(body, "input")
assert cell["value"] == "已遮罩" and cell["masked"] == ["tel"] and cell["path"] == TEL_CELL
assert body["redacted"]["columns"] == ["tel"] and leaks(body) == []
async def test_a_scalar_taken_by_position_from_a_renamed_masked_column_is_withheld(client, monkeypatch, mall):
"""vars.res.rows[0][1]:没有定位,顺着取值链认出取自查询结果的 tel 列,tel 又是 phone 改的名字。"""
await set_source(mall, {"mask_columns": ["phone"]})
row = await run_graph(monkeypatch)
body = await segment(client, row.id, ref="m:tel_pos2")
assert body["chain"][0]["substituted"] == "len(已遮罩) * 2" and body["chain"][0]["value"] == "已遮罩"
[pos] = steps(body, "input")
assert pos["path"] == TEL_POS and pos["value"] == "已遮罩" and pos["masked"] == ["tel"]
assert leaks(body) == []
async def test_without_masks_nothing_changes(client, monkeypatch, mall):
row = await run_graph(monkeypatch)
body = await segment(client, row.id, ref="v:Q1.r0.spend")
[query] = steps(body, "query")
assert query["masked"] == [] and query["rows"][0][1] == "199-0000-0102" and body["redacted"] == {"columns": []}
plen = await segment(client, row.id, text="13位")
assert plen["chain"][0]["value"] == 13 and plen["chain"][0]["substituted"] == "len('199-0000-0102')"
async def test_a_mask_set_after_the_run_also_reaches_renamed_columns(client, monkeypatch, mall):
"""事后才给数据源加的遮罩:按数据源现在设的列认 SQL,改了名字的照样遮。"""
row = await run_graph(monkeypatch)
await set_source(mall, {"mask_columns": "phone"})
body = await segment(client, row.id, ref="v:Q1.r0.spend")
assert steps(body, "query")[0]["masked"] == ["tel"] and leaks(body) == []
# --------------------------------------------------------------------------
# 推演:勾行接口、用到 tel 的指标
# --------------------------------------------------------------------------
async def test_the_whatif_masks_the_renamed_column(client, monkeypatch, mall):
await set_source(mall, {"mask_columns": ["phone"]})
row = await run_graph(monkeypatch)
body = await segment(client, row.id, ref="v:Q1.r0.spend")
snapshot = steps(body, "query")[0]["artifact"]
page = (await client.get(f"/api/runs/{row.id}/evidence/whatif/rows", params={"artifact": snapshot})).json()
assert page["masked"] == ["tel"] and all(r[1] == "已遮罩" for r in page["rows"]) and leaks(page) == []
res = await client.post(f"/api/runs/{row.id}/evidence/whatif", json={"card": "card"})
assert res.status_code == 200, res.text
metrics = {m["id"]: m for m in res.json()["metrics"]}
assert metrics["tel_len"]["status"] == "masked" and metrics["tel_pos2"]["status"] == "masked"
assert metrics["top_spend"]["status"] == "unchanged" and metrics["top_spend"]["old"] == 380.0
assert leaks(res.json()) == []
res = await client.post(f"/api/runs/{row.id}/evidence/whatif",
json={"card": "card", "overrides": {"inputs": {TEL_CELL: 1}}})
assert res.status_code == 422 and "遮罩" in res.json()["detail"]
# --------------------------------------------------------------------------
# 合并查询:输入里改了名字的遮罩列,按逐格来历遮到合并结果里
# --------------------------------------------------------------------------
async def test_a_merged_result_masks_a_column_renamed_in_its_input(client, monkeypatch, mall):
await set_source(mall, {"mask_columns": ["phone"]})
spec = chain(
node("start", "input"),
node("q_cust", "tool", tool="db_query__mall", args={"sql": "SELECT id, phone AS tel FROM customers ORDER BY id"}),
node("q_ord", "tool", tool="db_query__mall",
args={"sql": "SELECT customer_id AS id, SUM(amount) AS amount FROM orders GROUP BY customer_id ORDER BY id"}),
node("merged", "merge", inputs={"a": "q_cust", "b": "q_ord"},
sql="SELECT a.id, a.tel, b.amount FROM a JOIN b ON a.id = b.id ORDER BY a.id"),
node("write", "report", instructions="写一句", on_violation="flag"),
node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}]),
)
row = await run_graph(monkeypatch, spec, text="一号客户共消费 [[v:Q3.r0.amount]]。")
body = await segment(client, row.id, ref="v:Q3.r0.amount")
merged = next(s for s in steps(body, "query") if s.get("merge"))
assert "tel" in merged["masked"] and all(r[merged["columns"].index("tel")] == "已遮罩" for r in merged["rows"])
assert leaks(body) == []
# --------------------------------------------------------------------------
# 手搭的快照:SQL 的几种写法,解析不了时的保守规则
# --------------------------------------------------------------------------
#: 库里没有这个数据源:遮罩只有查询当时记在快照里的那一份(phone)
GHOST = "ghost_mall"
async def query_step(sql: str, columns: list[str], rows: list[list]) -> dict:
"""手搭一份封存范围内的查询快照(查询当时记下了遮罩列 phone),直接走查询步骤(evidence._query_step)。"""
from app.api import evidence
snap = await artifact_store.put_json({"columns": columns, "rows": rows, "source": GHOST, "sql": sql,
"mask_columns": ["phone"]}, kind="query")
sealed = evidence._Sealed(run_id="syn-alias", graph={}, events=[],
seal={"sealed": True, "ok": True, "manifest_seq": 1, "legacy": False})
sealed.queries = {snap: {"node_id": "q", "tool": f"db_query__{GHOST}", "source": GHOST}}
return await evidence._query_step(snap, [(0, columns[0])], {"catalog": {}}, sealed, evidence._Masks())
@pytest.mark.parametrize("sql, columns, masked", [
# 函数套一层再经 WITH 转一道
("WITH t AS (SELECT name, UPPER(phone) AS p FROM customers) SELECT name AS customer, p AS contact FROM t",
["customer", "contact"], ["contact"]),
# 子查询里起的别名,外层再选出来
("SELECT x.who, x.num FROM (SELECT name AS who, phone AS num FROM customers) x", ["who", "num"], ["num"]),
# 截一段再拼上星号:值仍取自遮罩列
("SELECT name, substr(phone, 1, 3) || '****' AS short_tel FROM customers", ["name", "short_tel"], ["short_tel"]),
# 只数个数:带不出值,不遮
("SELECT COUNT(DISTINCT phone) AS n FROM customers", ["n"], []),
# 解析不了、又写着遮罩列:整份快照的列都遮
("SELECT name, phone AS p FROM customers WHERE ((", ["name", "p"], ["name", "p"]),
# 每种读法都解析不了的,没写着遮罩列也整份遮:和可验证报告同一个函数(认不出结果里哪一列取自哪一列)
("SELECT name, id AS p FROM customers WHERE ((", ["name", "p"], ["name", "p"]),
])
async def test_the_query_step_reads_the_sql_like_the_bundle_and_the_verify_agent(sql, columns, masked):
rows = [["客户甲" if i == 0 else "199-0000-0101" for i in range(len(columns))]]
step = await query_step(sql, columns, rows)
assert step["masked"] == masked, step["masked"]
assert [v == "已遮罩" for v in step["rows"][0]] == [c in masked for c in columns]