mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 00:11:59 +00:00
ThearchyHelios
1cfa5a524d
第 3 轮的「遮罩表」推断和退回的裁判摘录各开了一个口子，收值时又把按位置认得出的列丢了： - 报告自查去掉「遮罩表」推断（_mask_owners）。数据源的 mask_columns 只认列名、不支持写成「表.列」，按列名在整个数据源上 生效：引用的那一格按 SQL 认出取自任何一张表里叫这个名字的列就记 masked_cell，视图（customer_spend.name）、没有第二个 遮罩列的表（employees.name）、门店名（stores.name）都算，和证据面板的遮法一致。以前经视图引用客户姓名、引用店员姓名， 正式运行照常成功，姓名写进了报告。SQL 血缘只用来点名出处，和排除值确实取自普通表别的列的（SELECT region AS phone FROM stores）；视图、表结构里没有的表、写了别的库名的表、没标 is_view 的老表结构、没有冻结表结构的快照看不出列的出处， 照算（_frozen_schema 只把 is_view 明写为 false 的表当普通表）。违规说明照实写取自哪张表的哪一列：「列「store」按 SQL 认出取自 stores.name，数据源对列名「name」设置了遮罩（按列名对所有表生效）」 - 交给结论句裁判的摘录按快照逐份遮（api/evidence.py 的 _judge_loader）：每份快照里确定取自遮罩列的列（_mask_sets 的 sure）并进交给 judge.prepare 的那一份快照的 mask_columns（工件本身不动），裁判照它自己的办法遮掉，judge.py 不用改。 c.phone AS tel 的号码不再进摘录；不按数据源名合并（另一份快照里同名、取自门店热线的列照给），不用宁可多遮的 hidden （整份遮的快照、只数个数的列照给）。按需裁判和报告撰写节点里当场裁判同一个函数。摘录变了的句子复用键随之变，此前 判过的会重判一次 - 收「这次运行查到过的遮罩值」：_sure 以前凡是要遮的列盖住了全部结果列、名字又对不上，就当认不准、整份算， SELECT UPPER(phone) FROM customers 一个值都收不到，证据面板之后照原样显示 SQL 里写着的号码。现在结果后面垫一列再认一遍 （_whole），垫的那一列也算进来的才是整份算的；按位置认得出的照收，整份遮的照旧只遮显示、不收值 测试：test_report_masked_owners.py 改名 test_report_masked_any_table.py 并按新口径改写（经视图改名 / 不改名、店员、门店名、 客户名真跑判失败且成果里没有值；* 展开、WITH、UNION、视图里改成遮罩列名字的列、没有表结构、没标 is_view、表结构里没有的 表、写了库名的表）；新增 test_judge_excerpt_masks.py（按需、当场两条路的摘录都不给改了名字的号码列，同名的门店热线照给； 没起别名的 UPPER(phone) 摘录也不给）；test_evidence_mask_memo.py 加没起别名的表达式照收值；违规说明的断言跟着改
286 lines
16 KiB
Python
286 lines
16 KiB
Python
"""报告自查只拦能确定取自遮罩列的格子；判失败时的下一步、时间线和缺口的说法照实；只读引用到的快照；审计表单列。
|
||
|
||
复核发现：
|
||
- 自查套用了证据面板展示用的保守认法：SQL 认不准时面板把整份的列都遮，自查跟着把和遮罩列无关的金额也判成「取自遮罩
|
||
列」，正式运行（默认违规即失败）因此判失败，写作者也改不掉；只装着分档标签的列（CASE … THEN '已留'）同样被判。现在自查
|
||
只认 _mask_sets 的 sure：列名就是遮罩列的、SQL 里认得出取自遮罩列的；整份算进来的、只装着标签的不算。按数据源自己的
|
||
方言认（MySQL 里双引号是字符串）
|
||
- 判失败时建议「换用指令遵循能力更强的模型」「先改成标注」：遮罩列多半是写作指令点名要的，换模型没用，改成标注值就随
|
||
报告出去了。现在只说去改写作指令
|
||
- 时间线、缺口说「这些值会随报告出具」「出具已按缺口降档」：探索运行、没有出具契约的成果不出具。现在只说记为缺口
|
||
- 写作之前把目录里每份查询快照都读盘复验一遍：现在组装之后只读正文引用到的那几份
|
||
- 审计表、导出的 CSV 里，引用了遮罩列的格子只是一行普通的「有出处」：现在另列一行
|
||
|
||
真跑的部分用虚构的 SQLite 商城库（号码是编的），写作者是 MockChatModel 照剧本写。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import sqlite3
|
||
|
||
import pytest
|
||
from httpx import ASGITransport, AsyncClient
|
||
from langchain_core.messages import AIMessage
|
||
from sqlalchemy import select
|
||
|
||
from app.core import artifact_store
|
||
from app.data.engine import engines
|
||
from app.db.base import SessionLocal
|
||
from app.db.models import DataSource, Run, RunEvent
|
||
from app.engine.evidence import compose_doc, masked_cells
|
||
from app.engine.runner import run_manager
|
||
from app.main import app
|
||
|
||
CUSTOMERS = [(1, "客户甲", "19900000401"), (2, "客户乙", "19900000402"), (3, "客户丙", "19900000403")]
|
||
ORDERS = [(1, 1, 120.5), (2, 2, 300.0), (3, 2, 80.0), (4, 3, 210.25), (5, 1, 99.5)]
|
||
PHONES = [c[2] for c in CUSTOMERS]
|
||
SQL_TOP = ("SELECT c.name AS customer, c.phone AS tel, ROUND(SUM(o.amount), 2) AS spend "
|
||
"FROM orders o JOIN customers c ON c.id = o.customer_id GROUP BY c.id, c.name, c.phone ORDER BY spend DESC")
|
||
#: 派生来源里有没起别名的 UPPER(c.phone)：遮罩那一套认不准，整份的列都算进要遮的列（面板宁可多遮）
|
||
SQL_UNSURE = ("SELECT name, amount FROM (SELECT c.name, UPPER(c.phone), o.amount FROM customers c "
|
||
"JOIN orders o ON o.customer_id = c.id) t ORDER BY amount DESC")
|
||
#: 只装着分档标签的列：格子里是 SQL 写死的字
|
||
SQL_LABEL = "SELECT id, CASE WHEN phone IS NULL THEN '未留' ELSE '已留' END AS has_phone FROM customers ORDER BY id"
|
||
|
||
|
||
@pytest.fixture
|
||
async def engine_up():
|
||
await run_manager.setup()
|
||
yield
|
||
await run_manager.shutdown()
|
||
|
||
|
||
@pytest.fixture
|
||
async def client(engine_up):
|
||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||
yield c
|
||
|
||
|
||
async def set_source(name: str, kind: str, path=None, options=None) -> str:
|
||
async with SessionLocal() as session:
|
||
row = (await session.execute(select(DataSource).where(DataSource.name == name))).scalar_one_or_none()
|
||
if row is None:
|
||
row = DataSource(name=name, kind=kind, database=str(path or ""), readonly=True, enabled=True)
|
||
session.add(row)
|
||
row.database, row.kind, row.enabled, row.readonly = str(path or ""), kind, True, True
|
||
row.options = dict(options or {})
|
||
await session.commit()
|
||
return row.id
|
||
|
||
|
||
@pytest.fixture
|
||
async def mall(tmp_path, engine_up):
|
||
path = tmp_path / "mall.db"
|
||
db = sqlite3.connect(path)
|
||
db.executescript("CREATE TABLE customers (id INTEGER PRIMARY KEY, name TEXT, phone TEXT);"
|
||
"CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_id INTEGER, amount REAL);")
|
||
db.executemany("INSERT INTO customers VALUES (?, ?, ?)", CUSTOMERS)
|
||
db.executemany("INSERT INTO orders VALUES (?, ?, ?)", ORDERS)
|
||
db.commit()
|
||
db.close()
|
||
source_id = await set_source("mall", "sqlite", path, {"mask_columns": ["phone"]})
|
||
await engines.invalidate(source_id)
|
||
yield path
|
||
await engines.invalidate(source_id)
|
||
|
||
|
||
def node(nid, ntype, **config):
|
||
return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}}
|
||
|
||
|
||
def graph(sql: str, *, contract: dict | None = None, **report) -> dict:
|
||
out = {"fields": [{"name": "报告", "value": "{{ nodes.write.text }}"}]}
|
||
if contract is not None:
|
||
out["contract"] = contract
|
||
nodes = [node("start", "input"), node("top", "tool", tool="db_query__mall", args={"sql": sql}),
|
||
node("write", "report", instructions="写一句", **report), node("out", "output", **out)]
|
||
return {"nodes": nodes, "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]}
|
||
|
||
|
||
async def finish(monkeypatch, spec: dict, text: str) -> Run:
|
||
from app.providers import mock_model
|
||
|
||
monkeypatch.setattr(mock_model.MockChatModel, "_decide", lambda self, messages: AIMessage(content=text))
|
||
run_id = (await run_manager.start(graph=spec, input_payload={})).id
|
||
for _ in range(400):
|
||
await asyncio.sleep(0.05)
|
||
async with SessionLocal() as session:
|
||
row = await session.get(Run, run_id)
|
||
if row.status in ("succeeded", "failed") and row.manifest_seq is not None:
|
||
return row
|
||
raise AssertionError(f"运行 {run_id} 没跑完：{row.status}")
|
||
|
||
|
||
async def events(run_id: str) -> list[RunEvent]:
|
||
async with SessionLocal() as session:
|
||
q = select(RunEvent).where(RunEvent.run_id == run_id).order_by(RunEvent.seq)
|
||
return list((await session.execute(q)).scalars())
|
||
|
||
|
||
def masked_of(violations: list[dict]) -> list[str]:
|
||
return [v["ref"] for v in violations if v["code"] == "masked_cell"]
|
||
|
||
|
||
def leaks(value) -> list[str]:
|
||
"""写着的号码，按千分位写的（19,900,000,101）也算。"""
|
||
raw = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
|
||
return [p for p in PHONES if p in raw or p in raw.replace(",", "")]
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 只拦能确定取自遮罩列的格子
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.parametrize("sql, text", [
|
||
# SQL 认不准、整份算进来的金额：和遮罩列无关
|
||
(SQL_UNSURE, "最大的一笔订单金额 [[v:Q1.r0.amount]]。"),
|
||
# 只装着分档标签的列
|
||
(SQL_LABEL, "第一位客户的手机号[[v:Q1.r0.has_phone]]。"),
|
||
])
|
||
async def test_a_formal_failure_is_not_raised_for_columns_that_are_not_surely_masked(monkeypatch, mall, sql, text):
|
||
"""违规即失败（正式运行的默认值）：认不准、只装着标签的列不判违规，运行照常成功；证据面板照旧多遮（展示）。"""
|
||
row = await finish(monkeypatch, graph(sql, on_violation="fail"), text)
|
||
assert row.status == "succeeded", row.error
|
||
checked = next(e.data for e in await events(row.id) if e.type == "report.checked")
|
||
assert masked_of(checked["violations"]) == [] and checked["ok"] is True, checked["violations"]
|
||
|
||
|
||
async def test_sure_and_hidden_are_told_apart(engine_up, mall):
|
||
"""_mask_sets：hidden 给面板（宁可多遮），sure 给自查（确定的）。"""
|
||
from app.api.evidence import _mask_sets, _Masks
|
||
|
||
async def sets(sql: str, columns: list[str], rows: list[list]):
|
||
return await _mask_sets({"sql": sql, "columns": columns, "rows": rows, "source": "mall"}, "mall", _Masks())
|
||
|
||
top = await sets(SQL_TOP, ["customer", "tel", "spend"], [["客户乙", PHONES[1], 380.0]])
|
||
assert top.sure == {"tel"} and "tel" in top.hidden and top.base == {"phone"}
|
||
unsure = await sets(SQL_UNSURE, ["name", "amount"], [["客户乙", 300.0]])
|
||
assert unsure.hidden >= {"name", "amount"} and unsure.sure == set(), (unsure.hidden, unsure.sure)
|
||
label = await sets(SQL_LABEL, ["id", "has_phone"], [[1, "已留"]])
|
||
assert "has_phone" in label.hidden and label.sure == set(), (label.hidden, label.sure)
|
||
direct = await sets("SELECT name, phone FROM customers", ["name", "phone"], [["客户甲", PHONES[0]]])
|
||
assert direct.sure == {"phone"}
|
||
|
||
|
||
async def test_the_sure_columns_follow_the_dialect_of_the_source(engine_up):
|
||
"""自查按数据源自己的方言认：MySQL 里 "phone" 是字符串，SELECT "phone" AS lit 的 lit 不是遮罩列；Postgres 里是列名，
|
||
lit 就是号码。证据面板的 hidden 按可验证报告的办法把几种读法合起来认，两处都遮（宁可多遮）。"""
|
||
from app.api.evidence import _mask_sets, _Masks
|
||
|
||
await set_source("mysql_mall", "mysql", options={"mask_columns": ["phone"]})
|
||
await set_source("pg_mall", "postgres", options={"mask_columns": ["phone"]})
|
||
snap = {"sql": 'SELECT id, "phone" AS lit FROM customers', "columns": ["id", "lit"], "rows": [[1, "phone"]]}
|
||
mysql = await _mask_sets({**snap, "source": "mysql_mall"}, "mysql_mall", _Masks())
|
||
pg = await _mask_sets({**snap, "source": "pg_mall"}, "pg_mall", _Masks())
|
||
assert "lit" not in mysql.sure and "lit" in pg.sure, (mysql.sure, pg.sure)
|
||
assert "lit" in mysql.hidden and "lit" in pg.hidden
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 说法：判失败的下一步、违规说明、时间线、缺口
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_a_failure_points_at_the_instructions_not_at_the_model(monkeypatch, mall):
|
||
"""遮罩列多半是写作指令点名要的：判失败时只说去掉指令里的引用，不建议换模型、不建议先改成标注。说明点名取自哪张表
|
||
的哪一列（数据源没探查过表结构，SQL 里写着 c.phone、c 是 customers）。"""
|
||
row = await finish(monkeypatch, graph(SQL_TOP, on_violation="fail"), "联系电话 [[v:Q1.r0.tel]]。")
|
||
assert row.status == "failed", row.error
|
||
assert "指令中去掉对这些列的引用" in row.error and "按 SQL 认出取自 customers.phone" in row.error, row.error
|
||
assert "指令遵循能力" not in row.error and "标注" not in row.error and leaks(row.error) == []
|
||
|
||
|
||
def test_mixed_failures_still_point_at_the_instructions_for_the_masked_cells():
|
||
from app.engine.nodes.report import _fail_advice
|
||
|
||
masked, bare = {"code": "masked_cell"}, {"code": "uncited_number"}
|
||
assert "指令中去掉" in _fail_advice([masked]) and "模型" not in _fail_advice([masked])
|
||
both = _fail_advice([masked, bare])
|
||
assert "指令中去掉" in both and "指令遵循能力" in both and "标注" not in both
|
||
assert "标注" in _fail_advice([bare])
|
||
|
||
|
||
async def test_the_timeline_and_the_gap_do_not_foretell_the_issuance(monkeypatch, mall):
|
||
"""标注照常产出：日志、缺口只说记为缺口，不说「随报告出具」「已按缺口降档」——探索运行、没有出具契约的成果不出具。"""
|
||
row = await finish(monkeypatch, graph(SQL_TOP, on_violation="flag"), "联系电话 [[v:Q1.r0.tel]]。")
|
||
assert row.status == "succeeded", row.error
|
||
rows = await events(row.id)
|
||
[log] = [e.data for e in rows if e.type == "log" and e.data.get("code") == "report_masked_cell"]
|
||
finished = next(e.data for e in rows if e.type == "node.finished" and e.node_id == "write")
|
||
[gap] = artifact_store.load(finished["artifact"])["issuance_gaps"]
|
||
for said in (log["message"], gap):
|
||
assert "出具" not in said and "降档" not in said and leaks(said) == [], said
|
||
assert "已记为缺口" in log["message"] and "不应写进报告" in gap
|
||
|
||
|
||
async def test_the_rewrite_request_says_the_value_must_not_be_written_in_any_form():
|
||
"""交回写作者的那句说清：这一列的值不能以任何写法写进报告（只说删掉引用，写作者会把值照抄成文字）。"""
|
||
art = await artifact_store.put_json({"columns": ["customer", "tel"], "rows": [["客户甲", PHONES[0]]],
|
||
"source": "mall", "sql": "SELECT name AS customer, phone AS tel FROM customers"},
|
||
kind="query")
|
||
catalog = {"Q1": {"alias": "Q1", "kind": "query", "artifact": art, "columns": ["customer", "tel"], "rows": 1,
|
||
"node_id": "top", "source": "mall"}}
|
||
doc = compose_doc("电话 [[v:Q1.r0.tel]]。", catalog, node_id="write", run_id="r1")
|
||
[v] = masked_cells(doc, catalog, {art: {"tel": ["phone"]}})
|
||
assert "不能以任何形式写进报告" in v["for_model"] and "照抄的文字" in v["for_model"]
|
||
assert "按 SQL 认出取自设置了遮罩的列「phone」" in v["message"]
|
||
# 列名就是遮罩列的照旧说「在数据源中设置了遮罩」
|
||
[own] = masked_cells(doc, catalog, {art: {"tel": ["tel"]}})
|
||
assert "列「tel」在数据源中设置了遮罩" in own["message"]
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 只读正文引用到的快照
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_only_the_cited_snapshots_are_read_and_each_only_once(engine_up, mall, monkeypatch):
|
||
"""组装好一稿之后才读：只读正文引用到的那几份，每份只读一次，重写的稿子不再读；同一个数据源上没被引用的快照不读
|
||
（第 2 轮按值认时要把设了遮罩的都读一遍，第 3 轮去掉了按值认）。"""
|
||
from app.core import artifact_store as store
|
||
from app.engine.nodes.report import _MaskedCells
|
||
|
||
cited = await store.put_json({"columns": ["customer", "tel"], "rows": [["客户甲", PHONES[0]]], "source": "mall",
|
||
"sql": "SELECT name AS customer, phone AS tel FROM customers"}, kind="query")
|
||
other = await store.put_json({"columns": ["id"], "rows": [[1]], "source": "mall", "sql": "SELECT id FROM customers"},
|
||
kind="query")
|
||
catalog = {"Q1": {"alias": "Q1", "kind": "query", "artifact": cited, "columns": ["customer", "tel"], "rows": 1,
|
||
"node_id": "a", "source": "mall"},
|
||
"Q2": {"alias": "Q2", "kind": "query", "artifact": other, "columns": ["id"], "rows": 1,
|
||
"node_id": "b", "source": "mall"}}
|
||
read: list[str] = []
|
||
real = store.load
|
||
|
||
def counting(artifact: str):
|
||
read.append(artifact)
|
||
return real(artifact)
|
||
|
||
monkeypatch.setattr(store, "load", counting)
|
||
check = _MaskedCells(catalog)
|
||
first = await check(compose_doc("电话 [[v:Q1.r0.tel]]。", catalog, node_id="write", run_id="r1"))
|
||
assert masked_of(first["violations"]) == ["v:Q1.r0.tel"] and read == [cited], read
|
||
second = await check(compose_doc("客户 [[v:Q1.r0.customer]]，电话 [[v:Q1.r0.tel]]。", catalog, node_id="write",
|
||
run_id="r1"))
|
||
assert masked_of(second["violations"]) == ["v:Q1.r0.tel"] and read == [cited], "同一份快照不再读第二遍"
|
||
assert other not in read
|
||
|
||
|
||
# --------------------------------------------------------------------------
|
||
# 审计表、CSV
|
||
# --------------------------------------------------------------------------
|
||
|
||
|
||
async def test_the_audit_lists_a_masked_cell_on_its_own_row(client, monkeypatch, mall):
|
||
"""引用了遮罩列的格子落在一个「有出处」的单元格片段上：审计表、CSV 里另列一行，写明引用了遮罩列，不写值。"""
|
||
row = await finish(monkeypatch, graph(SQL_TOP, on_violation="flag"), "联系电话 [[v:Q1.r0.tel]]。")
|
||
body = (await client.get(f"/api/runs/{row.id}/evidence/audit")).json()
|
||
rows = [r for group in body["groups"] for r in group["rows"]]
|
||
flagged = [r for r in rows if r.get("issue") == "masked_cell"]
|
||
assert len(flagged) == 1 and flagged[0]["ref"] == "v:Q1.r0.tel" and "遮罩" in flagged[0]["note"], flagged
|
||
# 这一行自己不写值（所在的句子是报告原文，标注之后照常产出，读者本来就看得到）
|
||
assert leaks([flagged[0][k] for k in ("text", "ref", "note")]) == []
|
||
csv = (await client.get(f"/api/runs/{row.id}/evidence/audit", params={"format": "csv"})).text
|
||
assert "引用了遮罩列" in csv
|