新设遮罩列时清理目录里的码值、含义与备注

This commit is contained in:
ThearchyHelios
2026-11-03 18:53:39 +08:00
parent 90597d5540
commit 174375c6c7
4 changed files with 304 additions and 1 deletions
+11 -1
View File
@@ -32,7 +32,7 @@ from app.data import introspect_jobs, source_objects
from app.data import table_versions
from app.data.engine import (
CATALOG_DRAFT_GUIDANCE_OPTION, MASK_COLUMNS_OPTION, MAX_QUERY_TIMEOUT_S, QUERY_TIMEOUT_OPTION, SUPPORTED_KINDS,
SnapshotTampered, build_url, engine_args, engines, mask_columns_problem, query_timeout_problem,
SnapshotTampered, build_url, engine_args, engines, mask_columns_problem, masked_columns, query_timeout_problem,
)
from app.db.base import get_session, new_id
from app.db.models import DataSource, ImportStaging, SourceSnapshot, TableImport, TableRecipe
@@ -552,6 +552,16 @@ async def update_source(
if after != before:
# 连的已经不是测过的那个了:旧结果作废,表单里测过这一份的话换成那一次
row.last_check = health.draft_result(after)
if "options" in data:
# 新设了遮罩列:清掉目录里这些列留下的取值类历史(码值、含义、备注、确认依据)。清理是尽力而为——遮罩设置
# 本身必须保存成功,清理失败只记日志。actor 不传:这个接口没有请求上下文,署名留给调用方按需补
was_masked = {c.lower() for c in masked_columns(old_options)}
newly_masked = [c for c in masked_columns(dict(row.options or {})) if c.lower() not in was_masked]
if newly_masked:
try:
await data_catalog.scrub_masked_columns(session, source_id, newly_masked, actor=None, commit=False)
except Exception: # noqa: BLE001 - 清理失败不能挡住遮罩本身的保存
logger.warning("设置遮罩列后清理数据目录失败(数据源 %s)", source_id, exc_info=True)
await session.commit()
await session.refresh(row)
# 连接参数可能变了,旧连接池不能再用——否则改完密码还在用旧连接,很难排查
+86
View File
@@ -45,6 +45,7 @@ import asyncio
import copy
import hashlib
import json
import logging
import re
from collections import Counter
from collections.abc import Callable, Iterable, Mapping
@@ -482,6 +483,91 @@ async def write_entry(session: AsyncSession, source_id: str, table: str, notes:
return _entry(row)
# ==========================================================================
# 遮罩列的历史清理
# ==========================================================================
def _list_text(value: Any) -> list[str]:
"""关系两端的列名列表(旧数据可能不合规,不是列表就当空的)。"""
return [str(c) for c in value] if isinstance(value, list) else []
async def scrub_masked_columns(session: AsyncSession, source_id: str, columns: Iterable[str], *,
actor: str | None = None, commit: bool = True) -> int:
"""新设了遮罩列:把这个数据源目录里这些列留下的取值类历史清掉,返回实际改动的表数。
遮罩列(数据源 options.mask_columns)以前只在读取侧过滤(visible_notes、schema_view),目录里写下的码值、含义、
备注和确认依据照旧带着取值。新设遮罩时清掉这些历史,才是真的不再留着:
- 列级项:这一列的 meaning、codes 整项删掉(码值和含义本身可能就是取值);
- 列级其它项(中文名、单位、度量类型):只删 note、reason(机器说明和确认依据里可能写着样例值),
value、source、status 保留;
- 关系:本表字段或目标字段里有遮罩列的,同样只删 note、reason(关系说明里可能带样例值),两端、基数、
覆盖率、来源、状态都留着。
只清历史,不改「遮罩列只给列名和类型」的读取规则,也不删除整列的目录:中文名、单位、度量类型是列本身的
事实,保留下来照旧可以给模型看。表级说明不动——遮罩的是一个列,不是整张表。
写库走乐观锁(UPDATE … WHERE version = ?),不调 write_entry:write_entry 冲突时会 rollback,会把调用方同一
事务里的其它写入(保存数据源、发布应用)一起回滚。rowcount 不是 1 说明这张表刚被别人改过,跳过这一张,不抛
异常、不回滚——清理是尽力而为。commit=False 只 flush,由调用方随自己的事务提交。
columns 去空白、去重、不分大小写。
"""
wanted = {str(c).strip().lower() for c in columns if str(c).strip()}
if not wanted:
return 0
rows = (await session.execute(select(CatalogNote).where(CatalogNote.source_id == source_id))).scalars().all()
changed = 0
for row in rows:
slots = _slots(row.notes or {})
drop: list[Slot] = []
touched = False
for key, item in slots.items():
if key[0] == "c":
if key[1].lower() not in wanted:
continue
if key[2] in ("meaning", "codes"):
drop.append(key)
touched = True
continue
elif key[0] == "r":
ends = _list_text(item.get("columns")) + _list_text(item.get("to_columns"))
if not any(name.lower() in wanted for name in ends):
continue
else:
continue
for name in ("note", "reason"):
if name in item:
item.pop(name)
touched = True
if not touched:
continue
for key in drop:
del slots[key]
notes = _assemble(slots)
if problems := validate_notes(notes):
logging.getLogger(__name__).warning(
"遮罩列清理跳过了表 %s:清理后的目录不合规(%s)", row.table_name, ";".join(problems[:5]))
continue
current = int(row.version or 0)
result = await session.execute(
update(CatalogNote)
.where(CatalogNote.id == row.id, CatalogNote.version == current)
.values(notes=notes, version=current + 1, updated_by=actor, updated_at=datetime.now(timezone.utc))
.execution_options(synchronize_session=False)
)
if result.rowcount != 1:
continue
changed += 1
if commit:
await session.commit()
else:
await session.flush()
return changed
def relation_id(table: str, columns: list[str], to_table: str, to_columns: list[str]) -> str:
"""关系的稳定编号:由两端的表和列算出来,同一条关系不论哪个来源、第几次起草都得到同一个编号。
+10
View File
@@ -17,6 +17,7 @@
from __future__ import annotations
import asyncio
import logging
import re
from dataclasses import dataclass
from datetime import datetime, timezone
@@ -397,6 +398,15 @@ async def _sync_registration(session: AsyncSession, row: RecordsTable, wanted: l
source.options = {**options, **({MASK_COLUMNS_OPTION: masked} if masked else {})}
source.schema_cache = schema_cache_of(merged, row.name)
source.schema_synced_at = utcnow()
if add:
# 新加的敏感列成了遮罩列:清掉目录里这几列留下的取值类历史(码值、含义、备注、确认依据)。只 flush、不提交,
# 跟随调用方的事务;清理是尽力而为,冲突时 scrub 自己跳过,出错也不能挡住发布或起草
from app.data import catalog as data_catalog
try:
await data_catalog.scrub_masked_columns(session, source.id, add, actor=None, commit=False)
except Exception: # noqa: BLE001 - 清理失败不影响登记本身
logging.getLogger(__name__).warning("设置遮罩列后清理数据目录失败(数据源 %s)", source.id, exc_info=True)
def columns_of(row: RecordsTable) -> list[Column]:
+197
View File
@@ -0,0 +1,197 @@
"""遮罩列的历史清理:新设遮罩时把目录里这些列留下的取值类历史清掉。
遮罩列(数据源 options.mask_columns)以前只在读取侧过滤,目录里写下的码值、含义、备注、确认依据照旧带着取值。
data/catalog.scrub_masked_columns 在「新设遮罩」时清掉这些历史:列级的 meaning、codes 整项删掉,其它列级项和
关系只删 note、reason,列名、中文名、单位、度量类型这些列本身的事实保留。表级说明不动——遮罩的是一个列。
三个入口各验一次:直接调用 scrub(幂等、版本号、不分大小写);PATCH 数据源时新设遮罩;记录表新增敏感列时
(_sync_registration)跟随发布/起草的那个事务清理。
"""
from __future__ import annotations
import uuid
from types import SimpleNamespace
from urllib.parse import quote
import pytest
from httpx import ASGITransport, AsyncClient
from app.data import catalog
from app.data.catalog import make_item
from app.db.base import SessionLocal
from app.db.models import DataSource
from app.main import app as _fastapi_app
from tests.fixtures.sources import drop_source, introspect_now
ACTOR = {"X-Actor": quote("王敏")}
def _notes(**overrides) -> dict:
"""订单表的目录:手机号(要遮的列,每个字段都带着可能泄值的说明和依据)、订单号(不动的列)、
两条关系(一条连手机号、一条连订单号)和一段表级说明。"""
notes = {
"label": make_item("订单", "human", note="一行一个订单", reason="和业务方核对过"),
"columns": {
"phone": {
"label": make_item("手机号", "human", note="样例 13800000000", reason="按样例确认过"),
"meaning": make_item("会员手机号", "llm", note="取值形如 138…"),
"unit": make_item("个", "llm"),
"measure": make_item("attribute", "llm"),
"codes": make_item({"138": "移动"}, "profile", note="前三个号码取的样"),
},
"order_id": {
"label": make_item("订单号", "human", note="业务主键"),
"meaning": make_item("订单的编号", "llm"),
"unit": make_item("个", "llm"),
"measure": make_item("identifier", "llm"),
"codes": make_item({"A": "正常单"}, "human", complete=True),
},
},
"relations": [
{"id": catalog.relation_id("orders", ["phone"], "members", ["id"]), "columns": ["phone"],
"to_table": "members", "to_columns": ["id"], "cardinality": "many_to_one", "coverage": None,
"source": "name", "status": "proposed", "note": "样例会员 13800000000 对得上",
"reason": "抽查了几行"},
{"id": catalog.relation_id("orders", ["order_id"], "order_lines", ["order_id"]), "columns": ["order_id"],
"to_table": "order_lines", "to_columns": ["order_id"], "cardinality": "one_to_many", "coverage": 0.98,
"source": "profile", "status": "verified", "note": "覆盖率 98%"},
],
}
notes.update(overrides)
return notes
async def _source() -> str:
async with SessionLocal() as session:
row = DataSource(name=f"mask_{uuid.uuid4().hex[:10]}", kind="sqlite", database=":memory:")
session.add(row)
await session.commit()
return row.id
async def test_scrub_clears_masked_column_values_but_keeps_the_column():
sid = await _source()
original = _notes()
async with SessionLocal() as session:
await catalog.write_entry(session, sid, "orders", original, if_version=0, actor="起草人")
async with SessionLocal() as session:
changed = await catalog.scrub_masked_columns(session, sid, [" Phone "], actor="王敏")
assert changed == 1
async with SessionLocal() as session:
entry = await catalog.read_entry(session, sid, "orders")
assert entry is not None
phone = entry.notes["columns"]["phone"]
assert "meaning" not in phone and "codes" not in phone, "码值和含义可能带取值,整项删掉"
assert phone["label"]["value"] == "手机号" and "note" not in phone["label"] and "reason" not in phone["label"]
assert phone["unit"]["value"] == "个" and phone["measure"]["value"] == "attribute"
# 不遮的列原样不动
assert entry.notes["columns"]["order_id"] == original["columns"]["order_id"]
# 表级说明不懂:遮的是列,不是整张表
assert entry.notes["label"] == original["label"]
assert entry.version == 2 and entry.updated_by == "王敏"
# 幂等:没有可清的了,不升版本
async with SessionLocal() as session:
assert await catalog.scrub_masked_columns(session, sid, ["PHONE"]) == 0
async with SessionLocal() as session:
again = await catalog.read_entry(session, sid, "orders")
assert again is not None and again.version == 2
async def test_scrub_only_drops_notes_and_reasons_on_relations():
sid = await _source()
original = _notes()
async with SessionLocal() as session:
await catalog.write_entry(session, sid, "orders", original, if_version=0, actor=None)
async with SessionLocal() as session:
await catalog.scrub_masked_columns(session, sid, ["phone"], commit=True)
async with SessionLocal() as session:
entry = await catalog.read_entry(session, sid, "orders")
assert entry is not None
ends_masked, untouched = entry.notes["relations"]
assert "note" not in ends_masked and "reason" not in ends_masked, "关系说明里可能带样例值"
assert {k: v for k, v in ends_masked.items() if k not in ("note", "reason")} == {
k: v for k, v in original["relations"][0].items() if k not in ("note", "reason")}
assert untouched == original["relations"][1], "两端都不是遮罩列的关系不动"
# --------------------------------------------------------------------------
# 两个入口
# --------------------------------------------------------------------------
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c:
yield c
@pytest.fixture
async def source_id(client, scenic_db):
"""接上合成景区库并探查结构(走真实的探查接口),和 test_catalog_api.py 一样。"""
async with SessionLocal() as session:
row = DataSource(name=f"mask_{uuid.uuid4().hex[:10]}", kind="sqlite", database=scenic_db)
session.add(row)
await session.commit()
sid = row.id
await introspect_now(client, sid)
yield sid
await drop_source(sid)
async def test_patch_settings_new_mask_scrubs_the_catalog(client, source_id):
async with SessionLocal() as session:
await catalog.write_entry(session, source_id, "visits", {
"columns": {"ticket_no": {"label": make_item("票号", "human", note="样例 T-0001"),
"meaning": make_item("门票的编号", "llm"),
"measure": make_item("identifier", "llm"),
"codes": make_item({"T": "纸票"}, "human")}},
}, if_version=0, actor=None)
# options 是整份替换:带上原来的 options 再加这一项
listed = (await client.get("/api/datasources")).json()
options = next(item for item in listed if item["id"] == source_id)["options"]
r = await client.patch(f"/api/datasources/{source_id}",
json={"options": {**options, "mask_columns": "ticket_no"}}, headers=ACTOR)
assert r.status_code == 200, r.text
assert r.json()["options"]["mask_columns"] == "ticket_no"
async with SessionLocal() as session:
entry = await catalog.read_entry(session, source_id, "visits")
assert entry is not None
ticket = entry.notes["columns"]["ticket_no"]
assert "meaning" not in ticket and "codes" not in ticket
assert ticket["label"]["value"] == "票号" and "note" not in ticket["label"]
assert entry.version == 2
async def test_records_sensitive_column_scrubs_inside_the_callers_transaction():
"""记录表新增敏感列时由 _sync_registration 清理:只 flush、不提交,跟随 ensure_table 的事务。"""
from app.records import store
from app.records.tables import ensure_table
owner = SimpleNamespace(id=f"mask{uuid.uuid4().hex[:10]}", app_id=None, name="门店报损")
columns = [{"name": "门店", "type": "string"}, {"name": "电话", "type": "string"}]
async with SessionLocal() as session:
row = await ensure_table(session, owner, columns)
await session.commit()
sid = row.source_id
async with SessionLocal() as session:
await catalog.write_entry(session, sid, store.TABLE, {
"columns": {"电话": {"label": make_item("电话", "human"), "meaning": make_item("联系电话", "llm")}},
}, if_version=0, actor=None)
# 同一份「填什么」,电话标成敏感:options 里多了这个遮罩列,目录在同一次提交里清掉
async with SessionLocal() as session:
await ensure_table(session, owner, [columns[0], {**columns[1], "sensitive": True}])
await session.commit()
async with SessionLocal() as session:
source = await session.get(DataSource, sid)
entry = await catalog.read_entry(session, sid, store.TABLE)
assert source is not None and source.options["mask_columns"] == ["电话"]
assert entry is not None and entry.version == 2
assert "meaning" not in entry.notes["columns"]["电话"] and entry.notes["columns"]["电话"]["label"]["value"] == "电话"