数据目录 F1-A2:在目录页去掉外键约束、命名推断、数据剖析、模型起草得出的关联关系改为驳回、保留编号,只有人工添加的才真正删除;改了两端的旧关系同样驳回。以前删掉的推断关系下次起草又加回来,关系图按表结构现推也会带回来,照样驱动 SQL 检查和助手挑表。表结构推得出的人工关系(人工改过基数的外键关系)去掉时也只驳回。编辑界面上这类关系不给「删除」只给「驳回」(保存时等同单项驳回,可撤销),检查脚本跟上

This commit is contained in:
ThearchyHelios
2026-10-04 01:07:35 +08:00
parent 0fa498ac18
commit fd3ec43662
7 changed files with 243 additions and 28 deletions
+42 -10
View File
@@ -47,7 +47,7 @@ from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.data.names import name_key
from app.db.models import Artifact, CatalogNote, Run
from app.db.models import Artifact, CatalogNote, DataSource, Run
# ==========================================================================
# 取值
@@ -653,14 +653,19 @@ def _normalize_submitted(submitted: Mapping[str, Any], table: str) -> dict[str,
def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str, Any], *, table: str,
at: str | None = None) -> dict[str, Any]:
at: str | None = None, inferable: Iterable[str] = ()) -> dict[str, Any]:
"""人工提交的整份目录 → 要写入的目录(不改入参)。
- 值改过的项、新填的项:记为 human / confirmed。提交里写的来源和状态不作数——客户端不能冒充外键约束
或数据剖析。
- 值没动的项:保持原来的来源和状态;提交里把状态改成确认、驳回或来源的初始状态的,照改(等同单项审阅);
改成别的状态的不认(不能把推断改成「有确证」)。
- 已有、但提交里没有的项:删掉。
- 已有、但提交里没有的表级项、列级项:删掉。
- 已有、但提交里没有的关联关系:外键约束、命名推断、数据剖析、模型起草来的,转为驳回、保留编号;只有人工
新建的才真正删掉。删掉的推断关系下次起草会再提出来,关系图(助手挑表、SQL 检查)也会按表结构现推回来,
照样驱动检查和助手——驳回才留得住。改了两端的关系编号跟着变,旧编号在提交里就「没有了」,同样按这条处理。
inferable 是表结构能推出来的关系编号(外键约束、命名推断,structure_relation_ids):人工改过基数的外键关系
记成了人工填写,删掉照样会被现推回来,也只能驳回。
- 关系编号按两端重算(_normalize_submitted)。
结构不合规抛 CatalogInvalid。
@@ -672,6 +677,7 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str,
raise CatalogInvalid(problems)
at = at or now_iso()
old, new = _slots(existing), _slots(normalized)
inferable = set(inferable)
out: dict[Slot, dict[str, Any]] = {}
for key, item in new.items():
prev = old.get(key)
@@ -692,9 +698,23 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str,
kept.pop("note", None)
kept["updated_at"] = at
out[key] = kept
for key, prev in old.items():
if key in new or key[0] != "r":
continue
if prev.get("source") == "human" and key[1] not in inferable:
continue # 人工新建的:真正删掉
out[key] = prev if prev.get("status") == "rejected" else {**prev, "status": "rejected", "updated_at": at}
return _assemble(out)
def structure_relation_ids(schema_cache: Mapping[str, Any] | None, table: str) -> set[str]:
"""表结构能推出来的这张表的关系编号(外键约束、命名推断)。表结构里没有这张表时为空集。"""
if not isinstance(((schema_cache or {}).get("tables") or {}).get(table), Mapping):
return set()
rels = (fk_relations(schema_cache, table) or []) + infer_name_relations(schema_cache, table)
return {str(r["id"]) for r in rels}
async def review_entry(session: AsyncSession, source_id: str, table: str, path: str, action: str, *,
if_version: int, actor: str | None) -> CatalogEntry:
"""单项审阅并写入(乐观锁同 write_entry)。版本不符抛 CatalogConflict,路径不对抛 CatalogPathError。"""
@@ -707,13 +727,22 @@ async def review_entry(session: AsyncSession, source_id: str, table: str, path:
async def save_human_edit(session: AsyncSession, source_id: str, table: str, submitted: Mapping[str, Any], *,
if_version: int, actor: str | None) -> CatalogEntry:
"""人工提交整份目录并写入(规则见 apply_human_edit,乐观锁同 write_entry)。"""
if_version: int, actor: str | None,
schema_cache: Mapping[str, Any] | None = None) -> CatalogEntry:
"""人工提交整份目录并写入(规则见 apply_human_edit,乐观锁同 write_entry)。
schema_cache 用来判断哪些关系表结构推得出来(去掉时只能驳回);不给就读数据源记录上的,和 SQL 检查建关系图
用的是同一份(sqlcheck.load_checkers)。
"""
entry = await read_entry(session, source_id, table)
current = entry.version if entry else 0
if if_version != current:
raise CatalogConflict(table, current)
notes = apply_human_edit(entry.notes if entry else {}, submitted, table=table)
if schema_cache is None:
row = await session.get(DataSource, source_id)
schema_cache = getattr(row, "schema_cache", None)
notes = apply_human_edit(entry.notes if entry else {}, submitted, table=table,
inferable=structure_relation_ids(schema_cache, table))
return await write_entry(session, source_id, table, notes, if_version=current, actor=actor)
@@ -1885,8 +1914,11 @@ def relation_graph(notes_by_table: Mapping[str, Mapping[str, Any]], *,
schema_cache: Mapping[str, Any] | None = None) -> dict[str, list[JoinEdge]]:
"""邻接表 {表: [从这张表出发的边]},每条关系正反各一条边;被驳回的关系不进图,自己连自己的也不进。
schema_cache 给了的话,没起草过目录的关系按表结构现推(外键约束 + 命名推断)补进来:助手挑表时不必等
每张表都起草过。目录里已有的关系(含驳回的)以目录为准。
schema_cache 给了的话,目录里没有的关系按表结构现推(外键约束 + 命名推断)补进来:助手挑表时不必等每张表
都起草过,只做过部分注释(目录里一条关系都没记)的表也不会丢掉外键关系——所以不能只给「没有目录的表」现推。
目录里已有的关系(含驳回的)以目录为准:编号由两端的表和列算出(relation_id,含本表表名),目录里驳回过的
编号就不再现推。这要求「去掉一条推断出来的关系」落成驳回而不是删除(apply_human_edit),删掉就留不下编号,
下次又被现推回来。
"""
rels: dict[str, tuple[str, Mapping[str, Any]]] = {}
rejected: set[str] = set()
@@ -2052,6 +2084,6 @@ __all__ = [
"initial_status", "join_paths", "make_item", "merge_notes", "now_iso", "parse_path", "plan_patch",
"profile_relation_holds", "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id",
"render_table_index", "render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry",
"review_item", "same_notes", "save_human_edit", "save_patch", "status_counts", "system_notes_source",
"table_usage", "unreviewed_status", "validate_notes", "visible_notes", "write_entry",
"review_item", "same_notes", "save_human_edit", "save_patch", "status_counts", "structure_relation_ids",
"system_notes_source", "table_usage", "unreviewed_status", "validate_notes", "visible_notes", "write_entry",
]
+7 -2
View File
@@ -152,7 +152,7 @@ def test_human_edit_marks_changed_items_as_human_confirmed_and_keeps_the_rest():
"grain": {**existing["grain"], "value": "每张门票每次检票一行"}, # 改了值
"columns": {"amount": {"measure": existing["columns"]["amount"]["measure"], # 没动
"unit": {"value": "元"}}}, # 新填,没写来源和状态
"relations": existing["relations"][:1], # 删掉一条关系
"relations": existing["relations"][:1], # 去掉一条推断出来的关系
"dedup": {"value": "同一票号只计一次", "source": "llm", "status": "verified"}, # 冒充别的来源、状态
}
out = catalog.apply_human_edit(existing, submitted, table="visits", at=AT)
@@ -161,7 +161,9 @@ def test_human_edit_marks_changed_items_as_human_confirmed_and_keeps_the_rest():
assert out["columns"]["amount"]["unit"] == {"value": "元", "source": "human", "status": "confirmed",
"updated_at": AT}
assert out["dedup"]["source"] == "human" and out["dedup"]["status"] == "confirmed"
assert len(out["relations"]) == 1 and out["relations"][0] == existing["relations"][0]
# 去掉的推断关系转为驳回、保留编号(删掉的话下次起草、关系图现推都会把它带回来)
assert out["relations"] == [existing["relations"][0],
{**existing["relations"][1], "status": "rejected", "updated_at": AT}]
assert catalog.validate_notes(out) == []
@@ -181,3 +183,6 @@ def test_human_edit_relation_id_follows_its_ends():
out = catalog.apply_human_edit(existing, {"relations": [moved]}, table="visits", at=AT)
assert out["relations"][0]["id"] == catalog.relation_id("visits", ["gate_id"], "parks", ["id"])
assert out["relations"][0]["source"] == "human"
# 两条原有的推断关系都不在提交里了(一条改了两端):转为驳回
assert [(r["id"], r["status"]) for r in out["relations"][1:]] == [(r["id"], "rejected")
for r in existing["relations"]]
+103
View File
@@ -149,3 +149,106 @@ async def test_confirming_a_profiled_relation_by_patch_keeps_the_fanout_check(cl
assert ("fanout_sum", "error", "gates", "id") in await _found(sid, sql)
# ==========================================================================
# A2:去掉推断或外键来的关系即驳回
# ==========================================================================
def _rel(columns, to_table, to_columns, source="name", status=None, table="visits"):
return {"id": catalog.relation_id(table, columns, to_table, to_columns), "columns": columns,
"to_table": to_table, "to_columns": to_columns, "cardinality": "many_to_one", "coverage": None,
"source": source, "status": status or catalog.initial_status(source)}
def test_human_edit_rejects_removed_inferred_relations_and_deletes_human_ones():
park, gate = _rel(["park_id"], "parks", ["id"], "fk"), _rel(["gate_id"], "gates", ["id"], "name")
mine = _rel(["member_id"], "members", ["id"], "human")
out = catalog.apply_human_edit({"relations": [park, gate, mine]}, {"relations": []}, table="visits", at=AT)
by_id = {r["id"]: r for r in out["relations"]}
assert set(by_id) == {park["id"], gate["id"]} # 人工新建的那条真正删掉
for old in (park, gate):
assert by_id[old["id"]] == {**old, "status": "rejected", "updated_at": AT} # 编号、来源、两端都留着
# 已经驳回的再去掉一次:不变
again = catalog.apply_human_edit(out, {"relations": []}, table="visits", at="2026-10-05T00:00:00+00:00")
assert again == out
def test_human_edit_moving_an_inferred_relation_rejects_the_old_one():
gate = _rel(["gate_id"], "gates", ["id"], "name")
moved = {**gate, "to_table": "parks", "to_columns": ["id"]}
out = catalog.apply_human_edit({"relations": [gate]}, {"relations": [moved]}, table="visits", at=AT)
new_id = catalog.relation_id("visits", ["gate_id"], "parks", ["id"])
by_id = {r["id"]: r for r in out["relations"]}
assert (by_id[new_id]["source"], by_id[new_id]["status"]) == ("human", "confirmed")
assert by_id[gate["id"]]["status"] == "rejected" and by_id[gate["id"]]["source"] == "name"
def test_human_edit_rejects_a_removed_human_relation_that_structure_would_bring_back():
"""人工改过基数的外键关系记成了人工填写;去掉它时若表结构还推得出来,删掉就会被现推回来,只能驳回。"""
park = {**_rel(["park_id"], "parks", ["id"], "fk"), "cardinality": "one_to_one", "source": "human",
"status": "confirmed"}
out = catalog.apply_human_edit({"relations": [park]}, {"relations": []}, table="visits", at=AT,
inferable={park["id"]})
assert [(r["id"], r["status"]) for r in out["relations"]] == [(park["id"], "rejected")]
assert catalog.apply_human_edit({"relations": [park]}, {"relations": []}, table="visits", at=AT) == {}
async def test_removed_or_rejected_relations_stay_out_of_checks_graph_and_redraft(client, make_source, scenic_db):
sid = await make_source(scenic_db, draft=["visits"])
detail = await _notes(client, sid, "visits")
park, gate = _relation(detail["notes"], "park_id"), _relation(detail["notes"], "gate_id")
ticket = _relation(detail["notes"], "ticket_type_id")
assert (park["source"], gate["source"], ticket["source"]) == ("fk", "name", "fk")
gone = {park["id"], gate["id"]}
# 目录页编辑时去掉两条(外键来的、命名推断的):转为驳回
notes = {**detail["notes"], "relations": [r for r in detail["notes"]["relations"] if r["id"] not in gone]}
r = await _put(client, sid, "visits", notes, detail["version"])
assert r.status_code == 200, r.text
saved = {x["id"]: x for x in r.json()["notes"]["relations"]}
assert {saved[i]["status"] for i in gone} == {"rejected"}
# 单项审阅里驳回第三条(界面上非人工来源的关系只给「驳回」)
r = await client.post(f"/api/datasources/{sid}/catalog/visits/review",
json={"path": f"relations.{ticket['id']}", "action": "reject",
"if_version": r.json()["version"]})
assert r.status_code == 200, r.text
gone.add(ticket["id"])
def absent(edges) -> bool:
return not any(e.relation_id in gone for e in edges)
# 检查器:被驳回的关系不当作「对上了」,也不拿来判断基数
checker = await _checker(sid)
for target in ("parks", "gates", "ticket_types"):
assert checker.edges_between("visits", target) == []
joined = await _found(sid, "SELECT COUNT(*) FROM visits v JOIN gates g ON v.gate_id = g.id")
assert ("join_unconfirmed", "warning", "gates", None) in joined
# 关系图(助手挑表也用它):不按表结构现推回来
graph = await _graph(sid)
assert absent(graph.get("visits", [])) and absent(graph.get("parks", [])) and absent(graph.get("gates", []))
# 重新起草:仍是驳回
r = await client.post(f"/api/datasources/{sid}/catalog/draft", json={"tables": ["visits"]})
assert r.status_code == 200, r.text
again = {x["id"]: x for x in (await _notes(client, sid, "visits"))["notes"]["relations"]}
assert {again[i]["status"] for i in gone} == {"rejected"}
assert absent((await _graph(sid)).get("visits", []))
def test_relation_graph_keeps_structure_relations_of_partially_annotated_tables():
"""只做过部分注释的表(目录里没有关系)照样按结构补上外键和命名推断;这张表目录里驳回的不补。"""
cache = {"tables": {
"visits": {"columns": [{"name": "id", "type": "INTEGER"}, {"name": "park_id", "type": "INTEGER"},
{"name": "gate_id", "type": "INTEGER"}],
"primary_key": ["id"], "foreign_keys": [{"columns": ["park_id"], "to_table": "parks",
"to_columns": ["id"]}]},
"parks": {"columns": [{"name": "id", "type": "INTEGER"}], "primary_key": ["id"], "foreign_keys": []},
"gates": {"columns": [{"name": "id", "type": "INTEGER"}], "primary_key": ["id"], "foreign_keys": []},
}}
labelled = {"visits": {"label": catalog.make_item("入园记录", "human")}}
edges = {(e.to_table, e.source) for e in catalog.relation_graph(labelled, schema_cache=cache)["visits"]}
assert edges == {("parks", "fk"), ("gates", "name")}
rejected = {"visits": {"relations": [_rel(["park_id"], "parks", ["id"], "fk", "rejected")]}}
edges = {e.to_table for e in catalog.relation_graph(rejected, schema_cache=cache)["visits"]}
assert edges == {"gates"}