From 174375c6c7ec5df821ff851b1fdd673e9408e680 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Tue, 3 Nov 2026 18:53:39 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E8=AE=BE=E9=81=AE=E7=BD=A9=E5=88=97?= =?UTF-8?q?=E6=97=B6=E6=B8=85=E7=90=86=E7=9B=AE=E5=BD=95=E9=87=8C=E7=9A=84?= =?UTF-8?q?=E7=A0=81=E5=80=BC=E3=80=81=E5=90=AB=E4=B9=89=E4=B8=8E=E5=A4=87?= =?UTF-8?q?=E6=B3=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/datasources.py | 12 +- backend/app/data/catalog.py | 86 ++++++++++ backend/app/records/tables.py | 10 ++ backend/tests/test_catalog_mask_scrub.py | 197 +++++++++++++++++++++++ 4 files changed, 304 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_catalog_mask_scrub.py diff --git a/backend/app/api/datasources.py b/backend/app/api/datasources.py index 7a1c2f54..0f75c3dc 100644 --- a/backend/app/api/datasources.py +++ b/backend/app/api/datasources.py @@ -32,7 +32,7 @@ from app.data import introspect_jobs, source_objects from app.data import table_versions from app.data.engine import ( CATALOG_DRAFT_GUIDANCE_OPTION, MASK_COLUMNS_OPTION, MAX_QUERY_TIMEOUT_S, QUERY_TIMEOUT_OPTION, SUPPORTED_KINDS, - SnapshotTampered, build_url, engine_args, engines, mask_columns_problem, query_timeout_problem, + SnapshotTampered, build_url, engine_args, engines, mask_columns_problem, masked_columns, query_timeout_problem, ) from app.db.base import get_session, new_id from app.db.models import DataSource, ImportStaging, SourceSnapshot, TableImport, TableRecipe @@ -552,6 +552,16 @@ async def update_source( if after != before: # 连的已经不是测过的那个了:旧结果作废,表单里测过这一份的话换成那一次 row.last_check = health.draft_result(after) + if "options" in data: + # 新设了遮罩列:清掉目录里这些列留下的取值类历史(码值、含义、备注、确认依据)。清理是尽力而为——遮罩设置 + # 本身必须保存成功,清理失败只记日志。actor 不传:这个接口没有请求上下文,署名留给调用方按需补 + was_masked = {c.lower() for c in masked_columns(old_options)} + newly_masked = [c for c in masked_columns(dict(row.options or {})) if c.lower() not in was_masked] + if newly_masked: + try: + await data_catalog.scrub_masked_columns(session, source_id, newly_masked, actor=None, commit=False) + except Exception: # noqa: BLE001 - 清理失败不能挡住遮罩本身的保存 + logger.warning("设置遮罩列后清理数据目录失败(数据源 %s)", source_id, exc_info=True) await session.commit() await session.refresh(row) # 连接参数可能变了,旧连接池不能再用——否则改完密码还在用旧连接,很难排查 diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index fb39a3e9..71ac0eb1 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -45,6 +45,7 @@ import asyncio import copy import hashlib import json +import logging import re from collections import Counter from collections.abc import Callable, Iterable, Mapping @@ -482,6 +483,91 @@ async def write_entry(session: AsyncSession, source_id: str, table: str, notes: return _entry(row) +# ========================================================================== +# 遮罩列的历史清理 +# ========================================================================== + + +def _list_text(value: Any) -> list[str]: + """关系两端的列名列表(旧数据可能不合规,不是列表就当空的)。""" + return [str(c) for c in value] if isinstance(value, list) else [] + + +async def scrub_masked_columns(session: AsyncSession, source_id: str, columns: Iterable[str], *, + actor: str | None = None, commit: bool = True) -> int: + """新设了遮罩列:把这个数据源目录里这些列留下的取值类历史清掉,返回实际改动的表数。 + + 遮罩列(数据源 options.mask_columns)以前只在读取侧过滤(visible_notes、schema_view),目录里写下的码值、含义、 + 备注和确认依据照旧带着取值。新设遮罩时清掉这些历史,才是真的不再留着: + + - 列级项:这一列的 meaning、codes 整项删掉(码值和含义本身可能就是取值); + - 列级其它项(中文名、单位、度量类型):只删 note、reason(机器说明和确认依据里可能写着样例值), + value、source、status 保留; + - 关系:本表字段或目标字段里有遮罩列的,同样只删 note、reason(关系说明里可能带样例值),两端、基数、 + 覆盖率、来源、状态都留着。 + + 只清历史,不改「遮罩列只给列名和类型」的读取规则,也不删除整列的目录:中文名、单位、度量类型是列本身的 + 事实,保留下来照旧可以给模型看。表级说明不动——遮罩的是一个列,不是整张表。 + + 写库走乐观锁(UPDATE … WHERE version = ?),不调 write_entry:write_entry 冲突时会 rollback,会把调用方同一 + 事务里的其它写入(保存数据源、发布应用)一起回滚。rowcount 不是 1 说明这张表刚被别人改过,跳过这一张,不抛 + 异常、不回滚——清理是尽力而为。commit=False 只 flush,由调用方随自己的事务提交。 + + columns 去空白、去重、不分大小写。 + """ + wanted = {str(c).strip().lower() for c in columns if str(c).strip()} + if not wanted: + return 0 + rows = (await session.execute(select(CatalogNote).where(CatalogNote.source_id == source_id))).scalars().all() + changed = 0 + for row in rows: + slots = _slots(row.notes or {}) + drop: list[Slot] = [] + touched = False + for key, item in slots.items(): + if key[0] == "c": + if key[1].lower() not in wanted: + continue + if key[2] in ("meaning", "codes"): + drop.append(key) + touched = True + continue + elif key[0] == "r": + ends = _list_text(item.get("columns")) + _list_text(item.get("to_columns")) + if not any(name.lower() in wanted for name in ends): + continue + else: + continue + for name in ("note", "reason"): + if name in item: + item.pop(name) + touched = True + if not touched: + continue + for key in drop: + del slots[key] + notes = _assemble(slots) + if problems := validate_notes(notes): + logging.getLogger(__name__).warning( + "遮罩列清理跳过了表 %s:清理后的目录不合规(%s)", row.table_name, ";".join(problems[:5])) + continue + current = int(row.version or 0) + result = await session.execute( + update(CatalogNote) + .where(CatalogNote.id == row.id, CatalogNote.version == current) + .values(notes=notes, version=current + 1, updated_by=actor, updated_at=datetime.now(timezone.utc)) + .execution_options(synchronize_session=False) + ) + if result.rowcount != 1: + continue + changed += 1 + if commit: + await session.commit() + else: + await session.flush() + return changed + + def relation_id(table: str, columns: list[str], to_table: str, to_columns: list[str]) -> str: """关系的稳定编号:由两端的表和列算出来,同一条关系不论哪个来源、第几次起草都得到同一个编号。 diff --git a/backend/app/records/tables.py b/backend/app/records/tables.py index b42c805c..d63d4c28 100644 --- a/backend/app/records/tables.py +++ b/backend/app/records/tables.py @@ -17,6 +17,7 @@ from __future__ import annotations import asyncio +import logging import re from dataclasses import dataclass from datetime import datetime, timezone @@ -397,6 +398,15 @@ async def _sync_registration(session: AsyncSession, row: RecordsTable, wanted: l source.options = {**options, **({MASK_COLUMNS_OPTION: masked} if masked else {})} source.schema_cache = schema_cache_of(merged, row.name) source.schema_synced_at = utcnow() + if add: + # 新加的敏感列成了遮罩列:清掉目录里这几列留下的取值类历史(码值、含义、备注、确认依据)。只 flush、不提交, + # 跟随调用方的事务;清理是尽力而为,冲突时 scrub 自己跳过,出错也不能挡住发布或起草 + from app.data import catalog as data_catalog + + try: + await data_catalog.scrub_masked_columns(session, source.id, add, actor=None, commit=False) + except Exception: # noqa: BLE001 - 清理失败不影响登记本身 + logging.getLogger(__name__).warning("设置遮罩列后清理数据目录失败(数据源 %s)", source.id, exc_info=True) def columns_of(row: RecordsTable) -> list[Column]: diff --git a/backend/tests/test_catalog_mask_scrub.py b/backend/tests/test_catalog_mask_scrub.py new file mode 100644 index 00000000..a175c1f9 --- /dev/null +++ b/backend/tests/test_catalog_mask_scrub.py @@ -0,0 +1,197 @@ +"""遮罩列的历史清理:新设遮罩时把目录里这些列留下的取值类历史清掉。 + +遮罩列(数据源 options.mask_columns)以前只在读取侧过滤,目录里写下的码值、含义、备注、确认依据照旧带着取值。 +data/catalog.scrub_masked_columns 在「新设遮罩」时清掉这些历史:列级的 meaning、codes 整项删掉,其它列级项和 +关系只删 note、reason,列名、中文名、单位、度量类型这些列本身的事实保留。表级说明不动——遮罩的是一个列。 + +三个入口各验一次:直接调用 scrub(幂等、版本号、不分大小写);PATCH 数据源时新设遮罩;记录表新增敏感列时 +(_sync_registration)跟随发布/起草的那个事务清理。 +""" +from __future__ import annotations + +import uuid +from types import SimpleNamespace +from urllib.parse import quote + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.data import catalog +from app.data.catalog import make_item +from app.db.base import SessionLocal +from app.db.models import DataSource +from app.main import app as _fastapi_app +from tests.fixtures.sources import drop_source, introspect_now + +ACTOR = {"X-Actor": quote("王敏")} + + +def _notes(**overrides) -> dict: + """订单表的目录:手机号(要遮的列,每个字段都带着可能泄值的说明和依据)、订单号(不动的列)、 + 两条关系(一条连手机号、一条连订单号)和一段表级说明。""" + notes = { + "label": make_item("订单", "human", note="一行一个订单", reason="和业务方核对过"), + "columns": { + "phone": { + "label": make_item("手机号", "human", note="样例 13800000000", reason="按样例确认过"), + "meaning": make_item("会员手机号", "llm", note="取值形如 138…"), + "unit": make_item("个", "llm"), + "measure": make_item("attribute", "llm"), + "codes": make_item({"138": "移动"}, "profile", note="前三个号码取的样"), + }, + "order_id": { + "label": make_item("订单号", "human", note="业务主键"), + "meaning": make_item("订单的编号", "llm"), + "unit": make_item("个", "llm"), + "measure": make_item("identifier", "llm"), + "codes": make_item({"A": "正常单"}, "human", complete=True), + }, + }, + "relations": [ + {"id": catalog.relation_id("orders", ["phone"], "members", ["id"]), "columns": ["phone"], + "to_table": "members", "to_columns": ["id"], "cardinality": "many_to_one", "coverage": None, + "source": "name", "status": "proposed", "note": "样例会员 13800000000 对得上", + "reason": "抽查了几行"}, + {"id": catalog.relation_id("orders", ["order_id"], "order_lines", ["order_id"]), "columns": ["order_id"], + "to_table": "order_lines", "to_columns": ["order_id"], "cardinality": "one_to_many", "coverage": 0.98, + "source": "profile", "status": "verified", "note": "覆盖率 98%"}, + ], + } + notes.update(overrides) + return notes + + +async def _source() -> str: + async with SessionLocal() as session: + row = DataSource(name=f"mask_{uuid.uuid4().hex[:10]}", kind="sqlite", database=":memory:") + session.add(row) + await session.commit() + return row.id + + +async def test_scrub_clears_masked_column_values_but_keeps_the_column(): + sid = await _source() + original = _notes() + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "orders", original, if_version=0, actor="起草人") + + async with SessionLocal() as session: + changed = await catalog.scrub_masked_columns(session, sid, [" Phone "], actor="王敏") + assert changed == 1 + + async with SessionLocal() as session: + entry = await catalog.read_entry(session, sid, "orders") + assert entry is not None + phone = entry.notes["columns"]["phone"] + assert "meaning" not in phone and "codes" not in phone, "码值和含义可能带取值,整项删掉" + assert phone["label"]["value"] == "手机号" and "note" not in phone["label"] and "reason" not in phone["label"] + assert phone["unit"]["value"] == "个" and phone["measure"]["value"] == "attribute" + # 不遮的列原样不动 + assert entry.notes["columns"]["order_id"] == original["columns"]["order_id"] + # 表级说明不懂:遮的是列,不是整张表 + assert entry.notes["label"] == original["label"] + assert entry.version == 2 and entry.updated_by == "王敏" + + # 幂等:没有可清的了,不升版本 + async with SessionLocal() as session: + assert await catalog.scrub_masked_columns(session, sid, ["PHONE"]) == 0 + async with SessionLocal() as session: + again = await catalog.read_entry(session, sid, "orders") + assert again is not None and again.version == 2 + + +async def test_scrub_only_drops_notes_and_reasons_on_relations(): + sid = await _source() + original = _notes() + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "orders", original, if_version=0, actor=None) + async with SessionLocal() as session: + await catalog.scrub_masked_columns(session, sid, ["phone"], commit=True) + + async with SessionLocal() as session: + entry = await catalog.read_entry(session, sid, "orders") + assert entry is not None + ends_masked, untouched = entry.notes["relations"] + assert "note" not in ends_masked and "reason" not in ends_masked, "关系说明里可能带样例值" + assert {k: v for k, v in ends_masked.items() if k not in ("note", "reason")} == { + k: v for k, v in original["relations"][0].items() if k not in ("note", "reason")} + assert untouched == original["relations"][1], "两端都不是遮罩列的关系不动" + + +# -------------------------------------------------------------------------- +# 两个入口 +# -------------------------------------------------------------------------- + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c: + yield c + + +@pytest.fixture +async def source_id(client, scenic_db): + """接上合成景区库并探查结构(走真实的探查接口),和 test_catalog_api.py 一样。""" + async with SessionLocal() as session: + row = DataSource(name=f"mask_{uuid.uuid4().hex[:10]}", kind="sqlite", database=scenic_db) + session.add(row) + await session.commit() + sid = row.id + await introspect_now(client, sid) + yield sid + await drop_source(sid) + + +async def test_patch_settings_new_mask_scrubs_the_catalog(client, source_id): + async with SessionLocal() as session: + await catalog.write_entry(session, source_id, "visits", { + "columns": {"ticket_no": {"label": make_item("票号", "human", note="样例 T-0001"), + "meaning": make_item("门票的编号", "llm"), + "measure": make_item("identifier", "llm"), + "codes": make_item({"T": "纸票"}, "human")}}, + }, if_version=0, actor=None) + + # options 是整份替换:带上原来的 options 再加这一项 + listed = (await client.get("/api/datasources")).json() + options = next(item for item in listed if item["id"] == source_id)["options"] + r = await client.patch(f"/api/datasources/{source_id}", + json={"options": {**options, "mask_columns": "ticket_no"}}, headers=ACTOR) + assert r.status_code == 200, r.text + assert r.json()["options"]["mask_columns"] == "ticket_no" + + async with SessionLocal() as session: + entry = await catalog.read_entry(session, source_id, "visits") + assert entry is not None + ticket = entry.notes["columns"]["ticket_no"] + assert "meaning" not in ticket and "codes" not in ticket + assert ticket["label"]["value"] == "票号" and "note" not in ticket["label"] + assert entry.version == 2 + + +async def test_records_sensitive_column_scrubs_inside_the_callers_transaction(): + """记录表新增敏感列时由 _sync_registration 清理:只 flush、不提交,跟随 ensure_table 的事务。""" + from app.records import store + from app.records.tables import ensure_table + + owner = SimpleNamespace(id=f"mask{uuid.uuid4().hex[:10]}", app_id=None, name="门店报损") + columns = [{"name": "门店", "type": "string"}, {"name": "电话", "type": "string"}] + async with SessionLocal() as session: + row = await ensure_table(session, owner, columns) + await session.commit() + sid = row.source_id + async with SessionLocal() as session: + await catalog.write_entry(session, sid, store.TABLE, { + "columns": {"电话": {"label": make_item("电话", "human"), "meaning": make_item("联系电话", "llm")}}, + }, if_version=0, actor=None) + + # 同一份「填什么」,电话标成敏感:options 里多了这个遮罩列,目录在同一次提交里清掉 + async with SessionLocal() as session: + await ensure_table(session, owner, [columns[0], {**columns[1], "sensitive": True}]) + await session.commit() + + async with SessionLocal() as session: + source = await session.get(DataSource, sid) + entry = await catalog.read_entry(session, sid, store.TABLE) + assert source is not None and source.options["mask_columns"] == ["电话"] + assert entry is not None and entry.version == 2 + assert "meaning" not in entry.notes["columns"]["电话"] and entry.notes["columns"]["电话"]["label"]["value"] == "电话"