Files
AgentLab/backend/tests/test_whatif_masked_values.py
T
ThearchyHelios 76fa42849c M2 证据面板和推演认得出改了名字的遮罩列,指标本身的原值也不露:查询步骤、指标的输入、推演的勾行与遮罩、合并查询按 SQL 认取自遮罩列的结果列(c.phone AS tel 这类别名、函数、子查询、WITH),和可验证报告、核查 agent 同一套认法;用到遮罩列的指标,推演答复里的 old、rendered_old 和证据面板指标步骤的 value 写「已遮罩」
验收里证据面板对 c.phone AS tel 的快照给出 masked=[]、三个号码原样交出;推演把 phone_raw 判成 masked、输入置空,指标的 old 却还是那串号码。现在:
- 认遮罩列只有一个函数 api/evidence.py 的 _snapshot_masks(查询当时记下的加数据源现在设的,再经 engine/masking.py 的 _derived_hidden 按 SQL 认改了名字的;SQL 解析不了又写着遮罩列的整份遮),查询步骤、_InputMasks、推演的 _mask_of 都调它;_Masks 顺带记数据源类型,按它的方言解析
- 合并查询按逐格来历遮:输入快照里改了名字的遮罩列(输入是合并结果的再往下追)也遮到合并结果里
- 指标的原值按 recompute.hides_value 一个判断:有输入取自遮罩列(含只嵌在下标里的),或者最外层的输入遮没遮核对不了;推演结果标 withheld,报告里引用它的地方 rendered_old 同样写「已遮罩」
- 面板底下的遮罩说明改成「这些列取自数据源中设置了遮罩的列」:遮的可能是别名列
2026-10-07 00:13:50 +08:00

269 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""推演和证据面板不露指标本身的原值:指标的输入取自遮罩格(或者指标的值本身就是遮罩格)时,原值写「已遮罩」。
验收发现:phone_raw = cell(nodes.q_cust, 0, 'phone') 推演时判成 masked,输入里置了空,指标的 old、rendered_old 却还是那串
号码;按位置取的 phone_pos 也一样。同一份答复里输入拿掉了、指标上照给,等于没遮。现在:
- 推演(engine/recompute.py):用到了遮罩列的指标(哪怕只嵌在下标里),old、rendered_old 写「已遮罩」,标 withheld,
原因照旧写在 reason 里;new、rendered_new 本来就是空的(推演不了);报告里引用它的地方 rendered_old 同样写「已遮罩」
- 最外层的输入遮没遮核对不了(快照读不出来)的也一样;只嵌在下标里、遮没遮核对不了的那一格不碍事,原值照给
- 证据面板的指标步骤:value 按同一个判断写「已遮罩」(recompute.hides_value),报告上的字(rendered)照旧
- 没用到遮罩列的指标一字不差
前半是纯函数(手写的虚构卡和快照),后半真跑一遍图(虚构的 SQLite 客户库,号码是编的)。
"""
from __future__ import annotations
import asyncio
import json
import sqlite3
import pytest
from httpx import ASGITransport, AsyncClient
from langchain_core.messages import AIMessage
from sqlalchemy import select
from app.core import artifact_store
from app.data.engine import engines
from app.db.base import SessionLocal
from app.db.models import DataSource, Run
from app.engine import recompute
from app.engine.evidence import iter_segments
from app.engine.recompute import Changes, Sources, whatif
from app.engine.runner import run_manager
from app.main import app
# --------------------------------------------------------------------------
# 纯函数:手写的卡和快照
# --------------------------------------------------------------------------
SNAP, OTHER = "a1" * 32, "b2" * 32
STORE = {
SNAP: {"columns": ["id", "phone", "amount"], "source": "crm", "sql": "SELECT id, phone, amount FROM members",
"rows": [[1, "19900000201", 120.5], [2, "19900000202", 80.0]]},
OTHER: {"columns": ["n"], "source": "crm", "sql": "SELECT 2 AS n", "rows": [[2]]},
}
PHONE = "cell(nodes.crm, 0, 'phone')"
AMOUNT = "cell(nodes.crm, 0, 'amount')"
def cell(row, column, value, artifact=SNAP):
return {"path": f"cell(nodes.crm, {row}, '{column}')", "value": value, "node_id": "crm", "via": "tool_cell",
"locator": {"row": row, "column": column}, "artifact": artifact, "status": "verified"}
def scalar(path, value):
return {"path": path, "value": value, "node_id": "fetch", "via": "transform", "status": "ok"}
def card(mid, expression, substituted, value, inputs, unit=""):
return {"kind": "metric_set", "caliber": "会员口径", "caliber_version": "v1", "metrics": [
{"id": mid, "name": mid, "unit": unit, "value": value, "format": "plain", "expression": expression,
"substituted": substituted, "inputs": inputs, "status": "ok"}]}
def sources(**kw):
base = {"load": STORE.get, "queries": (SNAP, OTHER), "masked": {SNAP: frozenset({"phone"})}}
return Sources(**{**base, **kw})
def text_of(result) -> str:
return json.dumps(result, ensure_ascii=False)
def test_the_value_of_a_metric_that_is_a_masked_cell_is_withheld():
"""验收的 phone_raw:指标就是遮罩列的那一格。old、rendered_old 都写「已遮罩」,号码不出现在答复的任何地方。"""
result = whatif(card("phone_raw", PHONE, "19900000201", 19900000201, [cell(0, "phone", 19900000201)]),
Changes(), sources())
[m] = result["metrics"]
assert (m["status"], m["old"], m["rendered_old"], m["withheld"]) == ("masked", "已遮罩", "已遮罩", True)
assert [m[k] for k in ("new", "rendered_new", "delta", "rendered_delta", "changed", "substituted")] == [None] * 6
assert "遮罩" in m["reason"] and m["inputs"][0]["masked"] is True and m["inputs"][0]["old"] is None
assert "19900000201" not in text_of(result)
def test_a_metric_computed_from_a_masked_cell_withholds_its_value_too():
"""号码位数这类算出来的数同样不给:改一个别的输入、拿指标的值一减,就能算出遮住的那一格(宁可多遮)。"""
[m] = whatif(card("mix", f"len({PHONE}) + {AMOUNT}", "len('19900000201') + 120.5", 131.5,
[cell(0, "phone", "19900000201"), cell(0, "amount", 120.5)]), Changes(), sources())["metrics"]
assert (m["status"], m["old"], m["rendered_old"]) == ("masked", "已遮罩", "已遮罩")
inputs = {i["path"]: i for i in m["inputs"]}
assert inputs[AMOUNT]["old"] == 120.5 and inputs[PHONE]["old"] is None
def test_a_missing_value_stays_missing():
"""本来就没有值的(缺输入)照旧是空的、显示「—」:缺输入还得看得出来。"""
[m] = whatif(card("gone", PHONE, None, None, [cell(0, "phone", None)]), Changes(), sources())["metrics"]
assert m["status"] == "masked" and m["old"] is None and m["rendered_old"] == "—" and m["withheld"] is True
def test_a_top_level_cell_on_an_unreadable_snapshot_withholds_the_value():
"""那一格所在的快照读不出来:遮没遮核对不了,输入不给值,指标的原值同样不给(和代入式一样)。"""
[m] = whatif(card("east", AMOUNT, "120.5", 120.5, [cell(0, "amount", 120.5)]), Changes(),
sources(load=lambda a: None if a == SNAP else STORE.get(a)))["metrics"]
assert m["status"] == "unsupported" and m["old"] == "已遮罩" and m["withheld"] is True
assert "120.5" not in text_of(m)
def test_a_cell_only_choosing_a_row_on_an_unreadable_snapshot_does_not_hide_the_value():
"""只嵌在下标里、遮没遮核对不了的那一格:它只决定取哪一项,取到的值卡上另记着,原值照给、照样推演。"""
pick = f"vars.kpi.list[{AMOUNT.replace('amount', 'id')}] + vars.kpi.k"
top = f"vars.kpi.list[{AMOUNT.replace('amount', 'id')}]"
metric = card("pick", pick, "20 + 2", 22, [scalar(top, 20), cell(0, "id", 1), scalar("vars.kpi.k", 2)])
[m] = whatif(metric, Changes(), sources(load=lambda a: None if a == SNAP else STORE.get(a)))["metrics"]
assert m["status"] == "unchanged" and m["old"] == 22 and "withheld" not in m
[m] = whatif(metric, Changes(inputs={"vars.kpi.k": 5}), sources(load=lambda a: None if a == SNAP else STORE.get(a)))[
"metrics"]
assert (m["status"], m["old"], m["new"]) == ("ok", 22, 25)
def test_an_unmasked_metric_is_unchanged():
result = whatif(card("east", AMOUNT, "120.5", 120.5, [cell(0, "amount", 120.5)]), Changes(), sources())
[m] = result["metrics"]
assert (m["status"], m["old"], m["rendered_old"]) == ("unchanged", 120.5, "120.5") and "withheld" not in m
def test_report_segments_of_a_withheld_metric_do_not_carry_the_value():
"""报告里引用了原值不给的指标:推演答复里那一处的 rendered_old 同样写「已遮罩」。"""
metric = card("phone_raw", PHONE, "19900000201", 19900000201, [cell(0, "phone", 19900000201)])
result = whatif(metric, Changes(), sources())
entry = {"kind": "metric", "node_id": "card", "artifact": "c" * 64, "locator": {"metric": "phone_raw"}}
doc = {"catalog": {"m:phone_raw": entry}, "blocks": [{"units": [{"id": "u0", "segments": [
{"id": "s0", "kind": "number", "text": "19900000201", "state": "deterministic",
"cite": {"alias": "m:phone_raw", "kind": "metric", "status": "resolved"}}]}]}]}
[seg] = recompute.report_segments([("write", doc)], card_node="card", card_artifact="c" * 64,
results=result["metrics"])
assert (seg["rendered_old"], seg["rendered_new"], seg["changed"]) == ("已遮罩", None, None)
@pytest.mark.parametrize("masked, unverified, expected", [
({PHONE}, set(), True), # 取自遮罩列
({"vars.kpi.i"}, set(), True), # 遮罩列的那一格只嵌在下标里:推演整个按遮罩处理,原值也不给
(set(), {AMOUNT}, True), # 最外层的输入遮没遮核对不了
(set(), {"vars.kpi.i"}, False), # 只嵌在下标里、遮没遮核对不了:不碍事
(set(), set(), False),
])
def test_hides_value_is_one_rule_for_the_whatif_and_the_evidence_panel(masked, unverified, expected):
metric = {"expression": f"{AMOUNT} + vars.kpi.list[vars.kpi.i]"}
assert recompute.hides_value(metric, masked, unverified) is expected
# 表达式读不懂:有一个遮没遮核对不了的就不给
assert recompute.hides_value({"expression": "1 +"}, set(), {AMOUNT}) is True
# --------------------------------------------------------------------------
# 真跑:验收原样的口径卡(单格原值、按位置取、按位置取的位数、编号)
# --------------------------------------------------------------------------
CUSTOMERS = [(1, "客户甲", "19900000301"), (2, "客户乙", "19900000302"), (3, "客户丙", "19900000303")]
PHONES = [c[2] for c in CUSTOMERS]
CARD_METRICS = [
{"id": "phone_raw", "name": "首位客户手机号(单格原值)", "format": "plain",
"expression": "cell(nodes.q_cust, 0, 'phone')"},
{"id": "phone_pos", "name": "首位客户手机号(按位置)", "format": "plain", "expression": "vars.cust.rows[0][2]"},
{"id": "phone_pos_len", "name": "首位客户手机号位数", "unit": "位", "format": "plain",
"expression": "len(str(vars.cust.rows[0][2]))"},
{"id": "id0", "name": "首位客户编号", "format": "plain", "expression": "cell(nodes.q_cust, 0, 'id')"},
]
TEXT = "首位客户编号 [[m:id0]],号码位数 [[m:phone_pos_len]]。"
@pytest.fixture
async def engine_up():
await run_manager.setup()
yield
await run_manager.shutdown()
@pytest.fixture
async def client(engine_up):
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
@pytest.fixture
async def members(tmp_path):
path = tmp_path / "members.db"
db = sqlite3.connect(path)
db.executescript("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, phone TEXT);")
db.executemany("INSERT INTO customers VALUES (?, ?, ?)", CUSTOMERS)
db.commit()
db.close()
async with SessionLocal() as session:
row = (await session.execute(select(DataSource).where(DataSource.name == "members"))).scalar_one_or_none()
if row is None:
row = DataSource(name="members", kind="sqlite", database=str(path), readonly=True, enabled=True)
session.add(row)
row.database, row.kind, row.enabled, row.readonly = str(path), "sqlite", True, True
row.options = {"mask_columns": ["phone"]}
await session.commit()
source_id = row.id
await engines.invalidate(source_id)
yield path
await engines.invalidate(source_id)
def node(nid, ntype, **config):
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
def graph() -> dict:
nodes = [
node("start", "input"),
node("q_cust", "tool", tool="db_query__members",
args={"sql": "SELECT id, name, phone FROM customers ORDER BY id LIMIT 5"}),
node("parse_cust", "transform", mode="json", template="{{ nodes.q_cust }}", assign_to="cust"),
node("card_c", "metrics", caliber="遮罩取法对照", caliber_version="v1", metrics=CARD_METRICS),
node("write", "report", instructions="写一句", on_violation="flag"),
node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}]),
]
return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
async def run_graph(monkeypatch) -> Run:
from app.providers import mock_model
monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=TEXT))
run_id = (await run_manager.start(graph=graph(), input_payload={})).id
for _ in range(400):
await asyncio.sleep(0.05)
async with SessionLocal() as session:
row = await session.get(Run, run_id)
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
assert row.status == "succeeded", row.error
return row
raise AssertionError(f"运行 {run_id} 没跑完:{row.status}")
def leaks(raw: str) -> list[str]:
return [p for p in PHONES if p in raw]
async def test_the_whatif_answer_carries_no_masked_value(client, monkeypatch, members):
"""验收原样:推演答复里号码一个都没有(以前在 phone_raw、phone_pos 的 old、rendered_old 里)。"""
row = await run_graph(monkeypatch)
res = await client.post(f"/api/runs/{row.id}/evidence/whatif", json={"card": "card_c"})
assert res.status_code == 200, res.text
assert leaks(res.text) == [], leaks(res.text)
metrics = {m["id"]: m for m in res.json()["metrics"]}
for mid in ("phone_raw", "phone_pos", "phone_pos_len"):
m = metrics[mid]
assert (m["status"], m["old"], m["rendered_old"], m["withheld"]) == ("masked", "已遮罩", "已遮罩", True), m
assert m["new"] is None and m["rendered_new"] is None and "遮罩" in m["reason"]
assert (metrics["id0"]["status"], metrics["id0"]["old"]) == ("unchanged", 1) and "withheld" not in metrics["id0"]
# 报告里引用了号码位数:那一处的原值也写「已遮罩」
[seg] = [s for s in res.json()["segments"] if s["metric"] == "phone_pos_len"]
assert seg["rendered_old"] == "已遮罩" and seg["rendered_new"] is None
async def test_the_evidence_panel_withholds_the_metric_value_by_the_same_rule(client, monkeypatch, members):
"""证据面板的指标步骤:用到遮罩列的指标 value 写「已遮罩」,报告上的字照旧;没用到的照给。"""
row = await run_graph(monkeypatch)
[report] = (await client.get(f"/api/runs/{row.id}/evidence")).json()["reports"]
doc = artifact_store.load(report["doc_artifact"])
by_text = {s["text"]: s["id"] for s in iter_segments(doc) if s.get("ref")}
body = (await client.get(f"/api/runs/{row.id}/evidence/segments/{by_text['11位']}")).json()
metric = body["chain"][0]
assert (metric["step"], metric["value"], metric["rendered"]) == ("metric", "已遮罩", "11位")
assert leaks(json.dumps(body, ensure_ascii=False)) == []
plain = (await client.get(f"/api/runs/{row.id}/evidence/segments/{by_text['1']}")).json()
assert plain["chain"][0]["value"] == 1