mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 18:31:59 +00:00
abb2707904
验收第二轮发现:口径卡 phone_raw = cell(q_cust, 0, 'phone') 被 [[m:phone_raw]] 引用时自查不拦,号码写进正文;写作目录、 文档的证据目录收着上游口径卡的全部指标和原值;可验证报告里口径卡的节点完成事件(预览逐条写着「指标(名称)= 值」)一律 附规范串,没有报告的运行也照样带出号码;另外只改紧挨着被引用数字的普通正文(前面补一位数、加负号、后面接「0 万」)、重算 全部哈希,读者看到的是另一个数,核对页只标无法核对,总览仍是绿的。 报告撰写节点 - 写作之前按证据面板指标步骤的那一套认出值取自遮罩格的指标(api/evidence.py 的 withheld_metrics → _metric_masks, recompute.withheld_value):cell() 取的遮罩格、改了名字的遮罩列、按位置取的一格一行、整份结果里带着遮罩值的。写作 目录、存进文档的证据目录里这些指标的值、渲染、标签都写「已遮罩」并记 masked(evidence.withhold_metrics,没被引用的 也一样),写作目录另说不要引用 - 只按出处认(by_value=False):认不出出处、值碰巧等于遮罩列某一格的普通指标(写死的目标 20000)不算,不然它在报告里 成了「已遮罩」、引用它的正式运行判失败、写作者也改不掉(自查第 3 轮去掉按值认是同一个道理);证据面板展示上照旧多遮 - 引用这类指标照样解析、正文写「已遮罩」(换算不再算),自查记 masked_cell(evidence.masked_metrics),和引用遮罩格同一 套处置:交回重写,仍不改按「违规处置」办;缺口、日志、判失败的下一步说「指标」 - 写成「已遮罩」的别名记进节点产出(masked_metrics),出口契约复核按同一份重建目录(report_catalog),正文里的「已遮罩」 对得上,不说文档被改过;写作时按上游状态建目录,循环里重跑不沿用上一次记下的那份 - 证据面板点开这一处:值、渲染都是「已遮罩」,按文档写的比渲染,不报渲染不一致 可验证报告(bundle/build.py、verifier.js) - 口径卡的节点完成事件:卡只给 id(有输入取自遮罩列,或者有指标的值不给)的只给哈希,和报告引不引用无关;没有台账的 老运行按预览里的 artifact、节点产出工件认,都读不出来的按运行里有没有带遮罩的快照 - 卡只给 id 的判断另按指标认(_Store.metric_word,和写作目录同一个判断):整行、下标不是字面量的一格、整份结果转成的 文字,以前只认字面量下标的一格,卡原样嵌了进去 - 证据目录里还写着这类原值的老文档(升级前生成的,或者生成之后数据源才设的遮罩)整份只给哈希、不附带它引用的工件, 报告节点的完成事件也只给哈希;快照读不出来时按它的数据源现在设的遮罩认,卡读不出来时按这次运行查到过的遮罩值认。 核对页照实说明原因(中性色),这份报告算未能核对,总览不是绿的;文档里写成「已遮罩」的指标所在的卡一律只给 id - 被引用的数字紧挨着(中间没有空白、标点)数字、负号、小数点判不通过,点名「正文里紧挨着这个数的字改变了读者看到的 数」;排在看不见、改变方向的字前面判,同一句里再放一个这种字也降不成无法核对。服务端组装的文档两个数之间补了空格, 原样导出照旧全绿(开发数据里 39 份报告、505 处引用,没有一处紧挨着) 测试 - 新增 tests/test_report_masked_metrics.py(目录、自查、重写、标注 / 失败、出口契约、证据面板、循环)、 tests/test_bundle_masked_metrics.py(没有报告的运行、新文档、老文档连同核对页、普通卡、和写作目录同一个判断、手搭的 几条分支);样本页 masked-card 改成现在的报告撰写节点写的文档,新增 masked-old-doc;check-bundle 补数字前加数字、加 负号、后面接「0 万」三种篡改,各自变红、点名,原有的紧挨着补位、连字符两条改成不通过,报告文档标成遮罩那条改成总览 不是绿的 - test_report_masked_values 的 [[m:phone_raw]] 改成按出处拦、写死的热线号码照常引用;几处按旧的报告正文(「13位」 「11位」「26位」)找片段的测试改成按引用找(test_evidence_metric_masks、test_evidence_masked_aliases、 test_whatif_masked_values),断言照新的正文写
311 lines
16 KiB
Python
311 lines
16 KiB
Python
"""证据面板认得出改了名字的遮罩列:查询步骤、指标的输入、推演的勾行和遮罩,按 SQL 认。
|
|
|
|
验收发现:正式运行的报告没写号码,可验证报告也把 top 查询(c.phone AS tel)的快照只给了哈希,点开正文里的消费额,
|
|
片段接口的查询步骤却是 columns=[customer, tel, spend]、masked=[],三个号码原样交出来。证据面板以前只遮列名恰好等于
|
|
遮罩列的那几列;可验证报告、核查 agent 早已按 SQL 认出别名、函数、子查询、WITH 转一道得来的结果列。现在证据面板
|
|
按同一套认法(engine/masking.py 的 _derived_hidden):
|
|
|
|
- 查询步骤:c.phone AS tel 的 tel 列遮掉,面板底下的遮罩说明带上它
|
|
- 指标的输入:cell() 取 tel 那一格、按位置取的 vars.res.rows[0][1] 都不给值;指标本身的值也不给
|
|
- 推演:勾行接口遮 tel,用到 tel 的指标推演不了、原值不给
|
|
- SQL 解析不了的,整份快照的列都遮(保守规则,和可验证报告同一个函数)
|
|
- 合并查询:输入快照里改了名字的遮罩列,按逐格来历遮到合并结果里
|
|
- 没设遮罩时一切照旧
|
|
|
|
真跑一遍图(虚构的 SQLite 商城库,号码是编的),报告由 MockChatModel 照剧本写。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import sqlite3
|
|
|
|
import pytest
|
|
from httpx import ASGITransport, AsyncClient
|
|
from langchain_core.messages import AIMessage
|
|
from sqlalchemy import select
|
|
|
|
from app.core import artifact_store
|
|
from app.data.engine import engines
|
|
from app.db.base import SessionLocal
|
|
from app.db.models import DataSource, Run
|
|
from app.engine.evidence import iter_segments
|
|
from app.engine.runner import run_manager
|
|
from app.main import app
|
|
|
|
CUSTOMERS = [(1, "客户甲", "199-0000-0101"), (2, "客户乙", "199-0000-0102"), (3, "客户丙", "199-0000-0103"),
|
|
(4, "客户丁", "199-0000-0104")]
|
|
ORDERS = [(1, 1, 120.5), (2, 2, 300.0), (3, 2, 80.0), (4, 3, 210.25), (5, 4, 50.0), (6, 1, 99.5)]
|
|
PHONES = [c[2] for c in CUSTOMERS]
|
|
#: 验收原样的写法:号码起了别名 tel,和消费额排在同一份结果里
|
|
SQL_TOP = ("SELECT c.name AS customer, c.phone AS tel, ROUND(SUM(o.amount), 2) AS spend "
|
|
"FROM orders o JOIN customers c ON c.id = o.customer_id GROUP BY c.id, c.name, c.phone ORDER BY spend DESC")
|
|
TEL_CELL = "cell(nodes.top, 0, 'tel')"
|
|
#: 号码那一格按位置取:vars.res 是数据整形解析好的查询结果,第 1 列是 tel
|
|
TEL_POS = "vars.res.rows[0][1]"
|
|
METRICS = [
|
|
{"id": "top_spend", "name": "最高消费", "unit": "元", "decimals": 2, "format": "thousands",
|
|
"expression": "cell(nodes.top, 0, 'spend')"},
|
|
{"id": "tel_len", "name": "号码位数", "unit": "位", "format": "thousands", "expression": f"len({TEL_CELL})"},
|
|
{"id": "tel_pos2", "name": "号码位数两倍", "unit": "位", "format": "thousands", "expression": f"len({TEL_POS}) * 2"},
|
|
]
|
|
TEXT = "消费最高的客户花了 [[v:Q1.r0.spend]],口径卡记 [[m:top_spend]];号码 [[m:tel_len]],两倍 [[m:tel_pos2]]。"
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
async def engine_up():
|
|
await run_manager.setup()
|
|
yield
|
|
await run_manager.shutdown()
|
|
|
|
|
|
@pytest.fixture
|
|
async def client():
|
|
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
|
yield c
|
|
|
|
|
|
async def set_source(path, options=None):
|
|
async with SessionLocal() as session:
|
|
row = (await session.execute(select(DataSource).where(DataSource.name == "mall"))).scalar_one_or_none()
|
|
if row is None:
|
|
row = DataSource(name="mall", kind="sqlite", database=str(path), readonly=True, enabled=True)
|
|
session.add(row)
|
|
row.database, row.kind, row.enabled, row.readonly = str(path), "sqlite", True, True
|
|
row.options = dict(options or {})
|
|
await session.commit()
|
|
return row.id
|
|
|
|
|
|
@pytest.fixture
|
|
async def mall(tmp_path):
|
|
path = tmp_path / "mall.db"
|
|
db = sqlite3.connect(path)
|
|
db.executescript("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, phone TEXT);"
|
|
"CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_id INTEGER, amount REAL);")
|
|
db.executemany("INSERT INTO customers VALUES (?, ?, ?)", CUSTOMERS)
|
|
db.executemany("INSERT INTO orders VALUES (?, ?, ?)", ORDERS)
|
|
db.commit()
|
|
db.close()
|
|
source_id = await set_source(path)
|
|
await engines.invalidate(source_id)
|
|
yield path
|
|
await engines.invalidate(source_id)
|
|
|
|
|
|
def node(nid, ntype, **config):
|
|
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
|
|
|
|
|
def chain(*nodes) -> dict:
|
|
return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
|
|
|
|
|
def graph() -> dict:
|
|
return chain(
|
|
node("start", "input"),
|
|
node("top", "tool", tool="db_query__mall", args={"sql": SQL_TOP}),
|
|
node("parse", "transform", mode="json", template="{{ nodes.top }}", assign_to="res"),
|
|
node("card", "metrics", caliber="商城口径", caliber_version="v1", metrics=METRICS),
|
|
node("write", "report", instructions="写一句", on_violation="flag"),
|
|
node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}]),
|
|
)
|
|
|
|
|
|
async def run_graph(monkeypatch, spec: dict | None = None, text: str = TEXT) -> Run:
|
|
from app.providers import mock_model
|
|
|
|
monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=text))
|
|
run_id = (await run_manager.start(graph=spec or graph(), input_payload={})).id
|
|
for _ in range(400):
|
|
await asyncio.sleep(0.05)
|
|
async with SessionLocal() as session:
|
|
row = await session.get(Run, run_id)
|
|
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
|
assert row.status == "succeeded", row.error
|
|
return row
|
|
raise AssertionError(f"运行 {run_id} 没跑完:{row.status}")
|
|
|
|
|
|
async def doc_of(client, run_id: str) -> dict:
|
|
[report] = (await client.get(f"/api/runs/{run_id}/evidence")).json()["reports"]
|
|
return artifact_store.load(report["doc_artifact"])
|
|
|
|
|
|
async def segment(client, run_id: str, *, text: str | None = None, ref: str | None = None) -> dict:
|
|
"""按正文里的字或者引用(v:Q1.r0.spend)找片段,取片段接口的答复。"""
|
|
doc = await doc_of(client, run_id)
|
|
seg = next(s for s in iter_segments(doc) if (text is not None and s["text"] == text)
|
|
or (ref is not None and s.get("ref") == ref))
|
|
res = await client.get(f"/api/runs/{run_id}/evidence/segments/{seg['id']}")
|
|
assert res.status_code == 200, res.text
|
|
return res.json()
|
|
|
|
|
|
def leaks(body) -> list[str]:
|
|
raw = json.dumps(body, ensure_ascii=False)
|
|
return [p for p in PHONES if p in raw]
|
|
|
|
|
|
def steps(body: dict, kind: str) -> list[dict]:
|
|
return [s for s in body["chain"] if s["step"] == kind]
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 查询步骤
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_a_renamed_masked_column_is_masked_in_the_query_step(client, monkeypatch, mall):
|
|
"""验收原样:点开正文里直接引用的消费额,查询步骤里 c.phone AS tel 的 tel 列遮掉,号码一个都不交出来。"""
|
|
await set_source(mall, {"mask_columns": ["phone"]})
|
|
row = await run_graph(monkeypatch)
|
|
body = await segment(client, row.id, ref="v:Q1.r0.spend")
|
|
[query] = steps(body, "query")
|
|
assert query["columns"] == ["customer", "tel", "spend"] and query["masked"] == ["tel"], query["masked"]
|
|
assert [r[1] for r in query["rows"]] == ["已遮罩"] * len(query["rows"]) and query["rows"][0][2] == 380.0
|
|
assert body["redacted"]["columns"] == ["tel"] and body["redacted"]["note"]
|
|
assert leaks(body) == []
|
|
|
|
|
|
async def test_the_metric_chain_masks_the_renamed_column_too(client, monkeypatch, mall):
|
|
"""口径卡指标 → 输入 → 查询:查询步骤同一套认法,tel 列同样遮掉(验收的 s13)。"""
|
|
await set_source(mall, {"mask_columns": ["phone"]})
|
|
row = await run_graph(monkeypatch)
|
|
body = await segment(client, row.id, text="380.00元")
|
|
[query] = steps(body, "query")
|
|
assert query["masked"] == ["tel"] and leaks(body) == []
|
|
[amount] = steps(body, "input")
|
|
assert amount["value"] == 380.0 and "masked" not in amount
|
|
|
|
|
|
async def test_an_input_taken_from_a_renamed_masked_column_is_withheld(client, monkeypatch, mall):
|
|
"""cell() 取 tel 那一格:输入不给值、写明取自哪一列,代入式里那一处写「已遮罩」,指标本身的值也不给。报告撰写节点写作时
|
|
按同一个判断把它在目录里换成了「已遮罩」,报告上的字也是(tests/test_report_masked_metrics.py),按引用找片段。"""
|
|
await set_source(mall, {"mask_columns": ["phone"]})
|
|
row = await run_graph(monkeypatch)
|
|
body = await segment(client, row.id, ref="m:tel_len")
|
|
metric = body["chain"][0]
|
|
assert metric["step"] == "metric" and metric["substituted"] == "len(已遮罩)"
|
|
assert metric["value"] == "已遮罩" and metric["rendered"] == "已遮罩" and metric["render_ok"] is True
|
|
[cell] = steps(body, "input")
|
|
assert cell["value"] == "已遮罩" and cell["masked"] == ["tel"] and cell["path"] == TEL_CELL
|
|
assert body["redacted"]["columns"] == ["tel"] and leaks(body) == []
|
|
|
|
|
|
async def test_a_scalar_taken_by_position_from_a_renamed_masked_column_is_withheld(client, monkeypatch, mall):
|
|
"""vars.res.rows[0][1]:没有定位,顺着取值链认出取自查询结果的 tel 列,tel 又是 phone 改的名字。"""
|
|
await set_source(mall, {"mask_columns": ["phone"]})
|
|
row = await run_graph(monkeypatch)
|
|
body = await segment(client, row.id, ref="m:tel_pos2")
|
|
assert body["chain"][0]["substituted"] == "len(已遮罩) * 2" and body["chain"][0]["value"] == "已遮罩"
|
|
[pos] = steps(body, "input")
|
|
assert pos["path"] == TEL_POS and pos["value"] == "已遮罩" and pos["masked"] == ["tel"]
|
|
assert leaks(body) == []
|
|
|
|
|
|
async def test_without_masks_nothing_changes(client, monkeypatch, mall):
|
|
row = await run_graph(monkeypatch)
|
|
body = await segment(client, row.id, ref="v:Q1.r0.spend")
|
|
[query] = steps(body, "query")
|
|
assert query["masked"] == [] and query["rows"][0][1] == "199-0000-0102" and body["redacted"] == {"columns": []}
|
|
plen = await segment(client, row.id, text="13位")
|
|
assert plen["chain"][0]["value"] == 13 and plen["chain"][0]["substituted"] == "len('199-0000-0102')"
|
|
|
|
|
|
async def test_a_mask_set_after_the_run_also_reaches_renamed_columns(client, monkeypatch, mall):
|
|
"""事后才给数据源加的遮罩:按数据源现在设的列认 SQL,改了名字的照样遮。"""
|
|
row = await run_graph(monkeypatch)
|
|
await set_source(mall, {"mask_columns": "phone"})
|
|
body = await segment(client, row.id, ref="v:Q1.r0.spend")
|
|
assert steps(body, "query")[0]["masked"] == ["tel"] and leaks(body) == []
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 推演:勾行接口、用到 tel 的指标
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_the_whatif_masks_the_renamed_column(client, monkeypatch, mall):
|
|
await set_source(mall, {"mask_columns": ["phone"]})
|
|
row = await run_graph(monkeypatch)
|
|
body = await segment(client, row.id, ref="v:Q1.r0.spend")
|
|
snapshot = steps(body, "query")[0]["artifact"]
|
|
page = (await client.get(f"/api/runs/{row.id}/evidence/whatif/rows", params={"artifact": snapshot})).json()
|
|
assert page["masked"] == ["tel"] and all(r[1] == "已遮罩" for r in page["rows"]) and leaks(page) == []
|
|
res = await client.post(f"/api/runs/{row.id}/evidence/whatif", json={"card": "card"})
|
|
assert res.status_code == 200, res.text
|
|
metrics = {m["id"]: m for m in res.json()["metrics"]}
|
|
assert metrics["tel_len"]["status"] == "masked" and metrics["tel_pos2"]["status"] == "masked"
|
|
assert metrics["top_spend"]["status"] == "unchanged" and metrics["top_spend"]["old"] == 380.0
|
|
assert leaks(res.json()) == []
|
|
res = await client.post(f"/api/runs/{row.id}/evidence/whatif",
|
|
json={"card": "card", "overrides": {"inputs": {TEL_CELL: 1}}})
|
|
assert res.status_code == 422 and "遮罩" in res.json()["detail"]
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 合并查询:输入里改了名字的遮罩列,按逐格来历遮到合并结果里
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
async def test_a_merged_result_masks_a_column_renamed_in_its_input(client, monkeypatch, mall):
|
|
await set_source(mall, {"mask_columns": ["phone"]})
|
|
spec = chain(
|
|
node("start", "input"),
|
|
node("q_cust", "tool", tool="db_query__mall", args={"sql": "SELECT id, phone AS tel FROM customers ORDER BY id"}),
|
|
node("q_ord", "tool", tool="db_query__mall",
|
|
args={"sql": "SELECT customer_id AS id, SUM(amount) AS amount FROM orders GROUP BY customer_id ORDER BY id"}),
|
|
node("merged", "merge", inputs={"a": "q_cust", "b": "q_ord"},
|
|
sql="SELECT a.id, a.tel, b.amount FROM a JOIN b ON a.id = b.id ORDER BY a.id"),
|
|
node("write", "report", instructions="写一句", on_violation="flag"),
|
|
node("out", "output", fields=[{"name": "报告", "value": "{{ nodes.write.text }}"}]),
|
|
)
|
|
row = await run_graph(monkeypatch, spec, text="一号客户共消费 [[v:Q3.r0.amount]]。")
|
|
body = await segment(client, row.id, ref="v:Q3.r0.amount")
|
|
merged = next(s for s in steps(body, "query") if s.get("merge"))
|
|
assert "tel" in merged["masked"] and all(r[merged["columns"].index("tel")] == "已遮罩" for r in merged["rows"])
|
|
assert leaks(body) == []
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# 手搭的快照:SQL 的几种写法,解析不了时的保守规则
|
|
# --------------------------------------------------------------------------
|
|
|
|
#: 库里没有这个数据源:遮罩只有查询当时记在快照里的那一份(phone)
|
|
GHOST = "ghost_mall"
|
|
|
|
|
|
async def query_step(sql: str, columns: list[str], rows: list[list]) -> dict:
|
|
"""手搭一份封存范围内的查询快照(查询当时记下了遮罩列 phone),直接走查询步骤(evidence._query_step)。"""
|
|
from app.api import evidence
|
|
|
|
snap = await artifact_store.put_json({"columns": columns, "rows": rows, "source": GHOST, "sql": sql,
|
|
"mask_columns": ["phone"]}, kind="query")
|
|
sealed = evidence._Sealed(run_id="syn-alias", graph={}, events=[],
|
|
seal={"sealed": True, "ok": True, "manifest_seq": 1, "legacy": False})
|
|
sealed.queries = {snap: {"node_id": "q", "tool": f"db_query__{GHOST}", "source": GHOST}}
|
|
return await evidence._query_step(snap, [(0, columns[0])], {"catalog": {}}, sealed, evidence._Masks())
|
|
|
|
|
|
@pytest.mark.parametrize("sql, columns, masked", [
|
|
# 函数套一层再经 WITH 转一道
|
|
("WITH t AS (SELECT name, UPPER(phone) AS p FROM customers) SELECT name AS customer, p AS contact FROM t",
|
|
["customer", "contact"], ["contact"]),
|
|
# 子查询里起的别名,外层再选出来
|
|
("SELECT x.who, x.num FROM (SELECT name AS who, phone AS num FROM customers) x", ["who", "num"], ["num"]),
|
|
# 截一段再拼上星号:值仍取自遮罩列
|
|
("SELECT name, substr(phone, 1, 3) || '****' AS short_tel FROM customers", ["name", "short_tel"], ["short_tel"]),
|
|
# 只数个数:带不出值,不遮
|
|
("SELECT COUNT(DISTINCT phone) AS n FROM customers", ["n"], []),
|
|
# 解析不了、又写着遮罩列:整份快照的列都遮
|
|
("SELECT name, phone AS p FROM customers WHERE ((", ["name", "p"], ["name", "p"]),
|
|
# 每种读法都解析不了的,没写着遮罩列也整份遮:和可验证报告同一个函数(认不出结果里哪一列取自哪一列)
|
|
("SELECT name, id AS p FROM customers WHERE ((", ["name", "p"], ["name", "p"]),
|
|
])
|
|
async def test_the_query_step_reads_the_sql_like_the_bundle_and_the_verify_agent(sql, columns, masked):
|
|
rows = [["客户甲" if i == 0 else "199-0000-0101" for i in range(len(columns))]]
|
|
step = await query_step(sql, columns, rows)
|
|
assert step["masked"] == masked, step["masked"]
|
|
assert [v == "已遮罩" for v in step["rows"][0]] == [c in masked for c in columns]
|