mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-21 11:41:57 +00:00
M2 证据面板和推演认得出改了名字的遮罩列,指标本身的原值也不露:查询步骤、指标的输入、推演的勾行与遮罩、合并查询按 SQL 认取自遮罩列的结果列(c.phone AS tel 这类别名、函数、子查询、WITH),和可验证报告、核查 agent 同一套认法;用到遮罩列的指标,推演答复里的 old、rendered_old 和证据面板指标步骤的 value 写「已遮罩」
验收里证据面板对 c.phone AS tel 的快照给出 masked=[]、三个号码原样交出;推演把 phone_raw 判成 masked、输入置空,指标的 old 却还是那串号码。现在: - 认遮罩列只有一个函数 api/evidence.py 的 _snapshot_masks(查询当时记下的加数据源现在设的,再经 engine/masking.py 的 _derived_hidden 按 SQL 认改了名字的;SQL 解析不了又写着遮罩列的整份遮),查询步骤、_InputMasks、推演的 _mask_of 都调它;_Masks 顺带记数据源类型,按它的方言解析 - 合并查询按逐格来历遮:输入快照里改了名字的遮罩列(输入是合并结果的再往下追)也遮到合并结果里 - 指标的原值按 recompute.hides_value 一个判断:有输入取自遮罩列(含只嵌在下标里的),或者最外层的输入遮没遮核对不了;推演结果标 withheld,报告里引用它的地方 rendered_old 同样写「已遮罩」 - 面板底下的遮罩说明改成「这些列取自数据源中设置了遮罩的列」:遮的可能是别名列
This commit is contained in:
+90
-27
@@ -127,7 +127,8 @@ NONE_NOTE = "本次运行没有报告文档,也没有出具契约,没有可
|
||||
NO_DOC_NOTE = "本次运行的出具契约采用引用模式核对,但报告没有产出可核对的文档"
|
||||
|
||||
MASKED = "已遮罩"
|
||||
MASK_NOTE = "这些列在数据源中设置了遮罩,面板上不显示原值。遮罩只减少暴露,不是安全边界"
|
||||
#: 面板底下的遮罩说明。遮的列可能是遮罩列本身,也可能是 SQL 里给它起了别名、套了函数得来的结果列(c.phone AS tel)
|
||||
MASK_NOTE = "这些列取自数据源中设置了遮罩的列,面板上不显示原值。遮罩只减少暴露,不是安全边界"
|
||||
#: 查询条目的数据源现在找不到了(改名或删掉了):只能按快照里记下的、查询当时的遮罩来遮
|
||||
SOURCE_GONE = "数据源「{source}」已不存在(可能已改名或删除),按查询时记录的遮罩处理"
|
||||
#: 查询步骤给被引用的行前后各带几行
|
||||
@@ -878,6 +879,7 @@ async def _metric_chain(seg: dict[str, Any], cite: dict[str, Any], entry: dict[s
|
||||
wanted: dict[str, list[tuple[Any, Any]]] = {}
|
||||
hide = _InputMasks(sealed, masks, str(entry.get("node_id") or ""), artifact)
|
||||
withheld: set[str] = set()
|
||||
unverified: set[str] = set()
|
||||
for item in (metric or {}).get("inputs") or []:
|
||||
one = {"step": "input", **item}
|
||||
artifact_of = item.get("artifact")
|
||||
@@ -889,10 +891,16 @@ async def _metric_chain(seg: dict[str, Any], cite: dict[str, Any], entry: dict[s
|
||||
if (hidden := await hide.check(item)) is not None:
|
||||
one = _withheld(one, *hidden)
|
||||
withheld.add(str(item.get("path")))
|
||||
if hidden[1] is not None:
|
||||
unverified.add(str(item.get("path")))
|
||||
inputs.append(one)
|
||||
if withheld:
|
||||
# 代入式里那几处同样写「已遮罩」:不然标签里遮着、代入式里亮着,同一个值换个地方照样看得到
|
||||
step["substituted"] = recompute.substituted_without(metric or {}, withheld, MASKED)
|
||||
if step["value"] is not None and recompute.hides_value(metric or {}, withheld - unverified, unverified):
|
||||
# 指标的值本身也不给,和推演的原值同一个判断:指标可能就是遮罩列的那一格(cell(nodes.q, 0, 'phone')),输入里
|
||||
# 遮着、这里亮着等于没遮。报告上的字(rendered)是读者已经看到的,照旧
|
||||
step["value"] = MASKED
|
||||
queries = [s for a, cells in wanted.items() for s in await _query_steps(a, cells, doc, sealed, masks)]
|
||||
return [step, *inputs, *queries]
|
||||
|
||||
@@ -916,8 +924,8 @@ SNAPSHOT_UNVERIFIED = "查询快照无法读取,核对不了这一格是否取
|
||||
|
||||
|
||||
class _InputMasks:
|
||||
"""一个指标的输入里哪些不能给值。遮的列和查询步骤同一套(_query_step):查询当时记下的加数据源现在设的,合并查询按它
|
||||
合并的那几个数据源算。
|
||||
"""一个指标的输入里哪些不能给值。遮的列和查询步骤同一套(_snapshot_masks):查询当时记下的加数据源现在设的,按 SQL
|
||||
认出改了名字的(c.phone AS tel 的 tel)也算,合并查询按它合并的那几个数据源和逐格来历算。
|
||||
|
||||
取自哪份查询结果的哪几列按 recompute.input_origin 认,推演、可验证报告是同一个认法:cell() 的格、Agent 交来的字段按
|
||||
定位里的列认;vars.res.rows[0][2] 这种按位置取的一格卡上没记定位,顺着取值链认,从写变量的节点在口径卡算这一次之前
|
||||
@@ -975,13 +983,8 @@ class _InputMasks:
|
||||
async def _hidden(self, artifact: str | None, source: str | None, snap: dict[str, Any] | None) -> set[str]:
|
||||
key = (artifact, source)
|
||||
if key not in self._memo:
|
||||
if snap is not None and snap.get("source") == MERGE_SOURCE:
|
||||
hidden, _ = await _merge_masks(snap, self._masks)
|
||||
else:
|
||||
info = self._sealed.queries.get(artifact) if artifact else None
|
||||
current, _ = await self._masks.of((info or {}).get("source") or source or (snap or {}).get("source"))
|
||||
hidden = set(current) | (_recorded_masks(snap) if snap is not None else set())
|
||||
self._memo[key] = hidden
|
||||
info = self._sealed.queries.get(artifact) if artifact else None
|
||||
self._memo[key], _ = await _snapshot_masks(snap, (info or {}).get("source") or source, self._masks)
|
||||
return self._memo[key]
|
||||
|
||||
|
||||
@@ -1283,10 +1286,12 @@ def _challenge_counts(raw: Any, claims: list[dict[str, Any]]) -> dict[str, int]:
|
||||
|
||||
|
||||
class _Masks:
|
||||
"""一次请求里查过的数据源遮罩:数据源名 → (要遮的列(小写), 数据源现在还在不在)。"""
|
||||
"""一次请求里查过的数据源遮罩:数据源名 → (要遮的列(小写), 数据源现在还在不在)。顺带记下数据源的类型(kind),
|
||||
按 SQL 认改了名字的遮罩列时照它的方言解析(_snapshot_masks);数据源不在了的按通用读法。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._memo: dict[str, tuple[set[str], bool]] = {}
|
||||
self._kinds: dict[str, str | None] = {}
|
||||
|
||||
async def of(self, source: Any) -> tuple[set[str], bool]:
|
||||
name = str(source or "")
|
||||
@@ -1296,8 +1301,56 @@ class _Masks:
|
||||
async with SessionLocal() as session:
|
||||
row = (await session.execute(select(DataSource).where(DataSource.name == name))).scalar_one_or_none()
|
||||
self._memo[name] = ({c.lower() for c in masked_columns(row.options if row else None)}, row is not None)
|
||||
self._kinds[name] = row.kind if row is not None else None
|
||||
return self._memo[name]
|
||||
|
||||
async def kind(self, source: Any) -> str | None:
|
||||
name = str(source or "")
|
||||
if not name:
|
||||
return None
|
||||
await self.of(name)
|
||||
return self._kinds.get(name)
|
||||
|
||||
|
||||
def _derived(snap: dict[str, Any], hidden: set[str], kind: str | None) -> set[str]:
|
||||
"""一份查询快照里值取自遮罩列(hidden,小写)的结果列(小写,含 hidden 本身):除了列名对上的,SQL 里给遮罩列起了
|
||||
别名、套了函数、经子查询或 WITH 转一道得来的也算。认法是 engine/masking.py 的 _derived_hidden,核查 agent 在工具边界
|
||||
遮、可验证报告判快照附不附带用的同一套;SQL 解析不了、里面又写着遮罩列(或者 * 这类不写列名也会带出遮罩列的写法)
|
||||
的,整份快照的列都算。合并查询的快照它不看 SQL,原样交回(合并的遮罩由 _merge_masks 按逐格来历算)。
|
||||
|
||||
分析本身出了意外:整份的列都算(宁可多遮)。纯计算,一条 SQL 不到 1 毫秒,在调用方的线程里算:推演在工作线程里
|
||||
等事件循环查库(api/whatif.py 的 _Hidden),这里再开线程会和它争同一个线程池。"""
|
||||
if not hidden:
|
||||
return set()
|
||||
try:
|
||||
# masking 要用这里的说法(MASKED),也是用到时才导入:两边都不在模块顶上互相导入
|
||||
from app.engine.masking import _derived_hidden
|
||||
|
||||
return set(hidden) | _derived_hidden(snap, set(hidden), kind)[0]
|
||||
except Exception: # noqa: BLE001 - 认不准就整份遮掉
|
||||
logger.warning("证据面板:按 SQL 认改了名字的遮罩列时出错,整份快照的列都按遮罩处理", exc_info=True)
|
||||
return set(hidden) | {str(c).lower() for c in snap.get("columns") or []}
|
||||
|
||||
|
||||
async def _snapshot_masks(snap: dict[str, Any] | None, source: Any, masks: _Masks, *,
|
||||
depth: int = 0) -> tuple[set[str], list[str]]:
|
||||
"""一份查询快照要遮的列(小写)和已经不在的数据源:证据面板的查询步骤、指标的输入、推演、报告自查(engine/nodes/
|
||||
report.py)都按这一个函数认。
|
||||
|
||||
- 遮的是「查询当时记下的」和「数据源现在设的」两者之和:事后加的遮罩照样生效,数据源改名、删掉了也不会把当时遮着
|
||||
的列亮出来;数据源不在了的,它的名字进第二项
|
||||
- 列名对上的之外,按 SQL 认出改了名字的遮罩列(_derived)
|
||||
- 合并查询不是数据源:按它合并的那几个数据源和逐格来历算(_merge_masks)
|
||||
- snap 为 None(快照读不出来):只能按 source 现在设的遮罩算,没有 SQL 可认"""
|
||||
if snap is not None and snap.get("source") == MERGE_SOURCE:
|
||||
return await _merge_masks(snap, masks, depth=depth)
|
||||
name = source or (snap or {}).get("source")
|
||||
hidden, found = await masks.of(name)
|
||||
hidden = set(hidden) | (_recorded_masks(snap) if snap is not None else set())
|
||||
if snap is not None and hidden:
|
||||
hidden = _derived(snap, hidden, await masks.kind(name))
|
||||
return hidden, ([] if found or not name else [str(name)])
|
||||
|
||||
|
||||
async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str, Any], sealed: _Sealed,
|
||||
masks: _Masks, *, hint: bool = False) -> dict[str, Any]:
|
||||
@@ -1343,22 +1396,15 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str
|
||||
rows = snap["rows"]
|
||||
step["source"] = step["source"] or snap.get("source")
|
||||
merged = snap.get("source") == MERGE_SOURCE
|
||||
# 遮的列:查询当时记下的加数据源现在设的,按 SQL 认出改了名字的(c.phone AS tel)也遮。合并查询不是数据源,按它
|
||||
# 合并的那几个数据源和逐格来历算,不能拿「合并查询」去找数据源,找不到再说它「已不存在」(_snapshot_masks)
|
||||
hidden, gone = await _snapshot_masks(snap, step["source"], masks)
|
||||
if gone:
|
||||
step["mask_note"] = SOURCE_GONE.format(source="、".join(gone))
|
||||
if merged:
|
||||
# 合并查询不是数据源:遮罩按它合并的那几个数据源算(_merge_masks),不能拿「合并查询」去找数据源,
|
||||
# 找不到再说它「已不存在」
|
||||
hidden, gone = await _merge_masks(snap, masks)
|
||||
if gone:
|
||||
step["mask_note"] = SOURCE_GONE.format(source="、".join(gone))
|
||||
step["merge"] = _merge_view(snap, cells, catalog, sealed)
|
||||
if hint and _leads_to_upload(snap, cells):
|
||||
step["provenance"] = True
|
||||
else:
|
||||
# 遮的是「查询当时记下的」和「数据源现在设的」两者之和:事后加的遮罩照样生效,数据源改名、
|
||||
# 删掉了也不会把当时遮着的列亮出来
|
||||
hidden, found = await masks.of(step["source"])
|
||||
hidden = hidden | _recorded_masks(snap)
|
||||
if not found:
|
||||
step["mask_note"] = SOURCE_GONE.format(source=step["source"])
|
||||
masked = {i for i, c in enumerate(columns) if c.lower() in hidden}
|
||||
index = sorted({i for r in rows_hit for i in range(r - WINDOW, r + WINDOW + 1) if 0 <= i < len(rows)})
|
||||
step.update(sql=snap.get("sql"), columns=columns, total_rows=len(rows), truncated=bool(snap.get("truncated")),
|
||||
@@ -1497,12 +1543,13 @@ def _input_checks(artifact: str) -> list[dict[str, Any]]:
|
||||
return _query_checks(snap) if isinstance(snap, dict) else []
|
||||
|
||||
|
||||
async def _merge_masks(snap: dict[str, Any], masks: _Masks) -> tuple[set[str], list[str]]:
|
||||
async def _merge_masks(snap: dict[str, Any], masks: _Masks, *, depth: int = 0) -> tuple[set[str], list[str]]:
|
||||
"""合并结果要遮的列(小写)和已经不在的数据源:
|
||||
|
||||
- 合并当时记下的(快照的 mask_columns:各输入当时的遮罩列,加上来自遮罩列、换了名字的结果列);
|
||||
- 它合并的每个数据源现在设的(按列名);
|
||||
- 结果列有逐格来历、来自某个数据源现在遮着的列的,按来历遮(换了名字也遮住)。宁可多遮。"""
|
||||
- 结果列有逐格来历的,按来历遮:来自的那一列在输入里要遮(_merge_input_masks:输入快照按查询步骤同一套认,数据源
|
||||
现在设的、SQL 里改了名字的都算,输入本身是合并结果的再往下追),换了名字也遮住。宁可多遮。"""
|
||||
hidden = _recorded_masks(snap)
|
||||
gone: list[str] = []
|
||||
for name in snap.get("sources") or []:
|
||||
@@ -1513,17 +1560,33 @@ async def _merge_masks(snap: dict[str, Any], masks: _Masks) -> tuple[set[str], l
|
||||
lineage = snap.get("lineage") if isinstance(snap.get("lineage"), dict) else {}
|
||||
origins = lineage.get("sources") or []
|
||||
by_alias = {str(i.get("alias")): i for i in snap.get("inputs") or [] if isinstance(i, dict)}
|
||||
inputs: dict[str, set[str]] = {}
|
||||
for name, target in zip(snap.get("columns") or [], lineage.get("columns") or []):
|
||||
if not isinstance(target, dict) or not isinstance(target.get("source"), int) \
|
||||
or not 0 <= target["source"] < len(origins):
|
||||
continue
|
||||
origin = origins[target["source"]] if isinstance(origins[target["source"]], dict) else {}
|
||||
source = (by_alias.get(str(origin.get("input"))) or {}).get("source")
|
||||
if source and source != MERGE_SOURCE and str(target.get("column") or "").lower() in (await masks.of(source))[0]:
|
||||
alias = str(origin.get("input"))
|
||||
if alias not in inputs:
|
||||
inputs[alias] = await _merge_input_masks(by_alias.get(alias) or {}, masks, depth)
|
||||
if str(target.get("column") or "").lower() in inputs[alias]:
|
||||
hidden.add(str(name).lower())
|
||||
return hidden, gone
|
||||
|
||||
|
||||
async def _merge_input_masks(item: dict[str, Any], masks: _Masks, depth: int) -> set[str]:
|
||||
"""合并的一个输入(快照 inputs 里的一项)的结果里要遮的列(小写)。读得出输入快照的按 _snapshot_masks 认(SQL 里给
|
||||
遮罩列起了别名的也算);读不出来、追得太深的,只能按它的数据源现在设的列名算。"""
|
||||
source = item.get("source")
|
||||
artifact = str(item.get("artifact") or "")
|
||||
content, ok = _fetch(artifact) if artifact and depth < MERGE_DEPTH else (None, None)
|
||||
if ok is True and isinstance(content, dict):
|
||||
return (await _snapshot_masks(content, source, masks, depth=depth + 1))[0]
|
||||
if source and source != MERGE_SOURCE:
|
||||
return set((await masks.of(source))[0])
|
||||
return set()
|
||||
|
||||
|
||||
def _leads_to_upload(snap: dict[str, Any], cells: list[tuple[Any, Any]], depth: int = 0) -> bool:
|
||||
"""被引用的格经合并追下去,最后落在上传表格的查询快照上(有 data_version):界面才去请求推断来源。"""
|
||||
if depth >= MERGE_DEPTH:
|
||||
|
||||
+13
-14
@@ -10,7 +10,8 @@ GET /api/runs/{run_id}/evidence/whatif/rows:推演时勾行用,按页取封
|
||||
只认这次运行封存范围内的工件,和证据接口同一套白名单(api/evidence.py 的 _sealed / _sealed_loader):口径卡的指标集
|
||||
从封存的 node.finished 台账找,排除行只认封存范围内交回过的查询快照,nodes.* 的完整值从封存的节点产出工件取,
|
||||
vars.* 的完整值(value_artifact)只认封存的指标集里写着的。还在跑、停在审批的运行也能推演,结果里 sealed 为假:
|
||||
数据可能还会变。遮罩列(查询当时记下的加数据源现在设的)不给值、也不能拿来推演,见 engine/recompute.py。
|
||||
数据可能还会变。遮罩列(查询当时记下的加数据源现在设的,SQL 里改了名字的也算)不给值、也不能拿来推演,用到它的指标
|
||||
原值也不给,见 engine/recompute.py。
|
||||
|
||||
重算本身是纯函数(engine/recompute.py)。这里负责:认请求、找卡、准备工件和遮罩、找报告里引用的地方、标出可能受影响
|
||||
的下游口径卡。
|
||||
@@ -31,17 +32,15 @@ from app.api.evidence import (
|
||||
REPORT_NOT_FOUND,
|
||||
TAMPERED_QUERY,
|
||||
_Masks,
|
||||
_merge_masks,
|
||||
_recorded_masks,
|
||||
_reports,
|
||||
_Sealed,
|
||||
_sealed,
|
||||
_sealed_cards,
|
||||
_sealed_loader,
|
||||
_snapshot_masks,
|
||||
)
|
||||
from app.core.artifact_store import load
|
||||
from app.engine import recompute
|
||||
from app.engine.merge_query import MERGE_SOURCE
|
||||
|
||||
router = APIRouter(prefix="/api/runs", tags=["whatif"])
|
||||
|
||||
@@ -115,7 +114,8 @@ async def evidence_whatif_rows(run_id: str, artifact: str | None = None, offset:
|
||||
"""一页查询快照的行:{artifact, columns, rows, row_offset, row_index, total_rows, masked, hash_ok, sealed, note?}。
|
||||
|
||||
形状和证据面板的查询步骤一致(界面用同一个函数读)。只认这次运行封存范围内交回过的快照;遮罩列(查询当时记下的
|
||||
加数据源现在设的)写「已遮罩」;读不出来、与哈希不一致的不给行,note 照实说。只读:不追加事件、不落工件。
|
||||
加数据源现在设的,SQL 里改了名字的也算)写「已遮罩」;读不出来、与哈希不一致的不给行,note 照实说。只读:不追加
|
||||
事件、不落工件。
|
||||
"""
|
||||
if not artifact or not artifact.strip():
|
||||
raise CodedHTTPException(422, "请求格式有误:artifact 应为查询快照的工件 ID", BAD_REQUEST)
|
||||
@@ -312,9 +312,9 @@ def _loader(sealed: _Sealed, cards: list[tuple[str, str, dict[str, Any]]]) -> An
|
||||
class _Hidden(Mapping[str, frozenset[str]]):
|
||||
"""封存范围内的查询快照 → 要遮的列(小写),推演用到哪份才算哪份。
|
||||
|
||||
遮的列和证据面板查询步骤同一套(api/evidence.py 的 _query_step):查询当时记下的加数据源现在设的,合并查询按它
|
||||
合并的那几个数据源算。快照读不出来、被改过时照样按台账记的数据源算:遮罩不能因为快照坏了就失效(那一格的值另由
|
||||
重算不给)。
|
||||
遮的列和证据面板查询步骤同一套(api/evidence.py 的 _snapshot_masks):查询当时记下的加数据源现在设的,按 SQL 认出
|
||||
改了名字的也算,合并查询按它合并的那几个数据源和逐格来历算。快照读不出来、被改过时照样按台账记的数据源算:遮罩
|
||||
不能因为快照坏了就失效(那一格的值另由重算不给)。
|
||||
|
||||
在重算的工作线程里取用:快照经推演用的同一个 loader 读(重算本来也要读这一份,读过的不再读盘);查数据源现在
|
||||
设的遮罩要访问库,交回事件循环去查(_Masks 是异步的),线程在这里等它。以前每次请求在事件循环里把这次运行的
|
||||
@@ -353,9 +353,8 @@ class _Hidden(Mapping[str, frozenset[str]]):
|
||||
|
||||
|
||||
async def _mask_of(snap: dict[str, Any] | None, info: dict[str, Any], masks: _Masks) -> set[str]:
|
||||
"""一份查询快照要遮的列(小写)。snap 为 None(读不出来)时只能按台账记的数据源现在设的遮罩算。"""
|
||||
if snap is not None and snap.get("source") == MERGE_SOURCE:
|
||||
hidden, _ = await _merge_masks(snap, masks)
|
||||
return hidden
|
||||
hidden, _ = await masks.of(info.get("source") or (snap or {}).get("source"))
|
||||
return hidden | (_recorded_masks(snap) if snap is not None else set())
|
||||
"""一份查询快照要遮的列(小写),和证据面板的查询步骤同一个函数(api/evidence.py 的 _snapshot_masks):查询当时记下的
|
||||
加数据源现在设的,SQL 里给遮罩列起了别名、套了函数的结果列也算。snap 为 None(读不出来)时只能按台账记的数据源现在
|
||||
设的遮罩算。"""
|
||||
hidden, _ = await _snapshot_masks(snap, info.get("source"), masks)
|
||||
return hidden
|
||||
|
||||
Reference in New Issue
Block a user