数据目录 F1-A4:码值项加「已列全」标记(complete: true),只有人工明确勾选、或剖析整表统计(没有只看前若干行)得到的码值才有;码值不在码值表中的检查只认标了它的码值表,给模型的话改为提示核对取值,不再叫它照目录里的码值改。以前经对话只补了 status=9 表示作废,之后 status = 1 就被报提醒,模型还可能把对的 SQL 改错。提案补充码值时沿用原有的标记、自己不能标;目录页的码值编辑和填写含义弹窗加「已列出全部取值」勾选,表格里标「已列全」,检查脚本跟上

This commit is contained in:
ThearchyHelios
2026-10-04 01:13:13 +08:00
parent fd3ec43662
commit b078d08ffe
11 changed files with 275 additions and 58 deletions
+41 -10
View File
@@ -15,7 +15,8 @@ notes 结构(每个「项」都是 {value, source, status},可选 note、upd
"valid_filter": 项(str,SQL 条件片段), "dedup": 项(str),
"columns": {列名: {"label": 项, "meaning": 项, "unit": 项,
"measure": 项("flow"|"stock"|"ratio"|"identifier"|"status"|"attribute"),
"codes": 项(dict[码值, 含义];含义可为空串,表示待填写)}},
"codes": 项(dict[码值, 含义];含义可为空串,表示待填写;项上可带 complete: true,
表示已列出全部取值)}},
"relations": [{"id", "columns", "to_table", "to_columns", "cardinality", "coverage",
"source", "status", "note"?}]}
@@ -81,6 +82,9 @@ COLUMN_FIELD_LABEL = {"label": "中文名", "meaning": "含义", "unit": "单位
_INITIAL_STATUS = {"fk": "verified", "human": "confirmed"}
_ITEM_KEYS = frozenset({"value", "source", "status", "note", "updated_at"})
#: 码值项另有一个「已列全」标记(complete: true):码值表列出了这一列的全部取值。只有人工明确确认、或者数据剖析
#: 没截断地看完了全表取值时才有;没有它的码值表只是「已知的一部分」,SQL 检查不能拿它判断某个值「不在码值表里」
_CODES_ITEM_KEYS = _ITEM_KEYS | {"complete"}
_RELATION_KEYS = frozenset({"id", "columns", "to_table", "to_columns", "cardinality", "coverage", "source",
"status", "note", "updated_at"})
_BUSINESS_DATE_KEYS = frozenset({"column", "rule", "timezone"})
@@ -127,16 +131,24 @@ def unreviewed_status(item: Mapping[str, Any]) -> str:
def make_item(value: Any, source: str, status: str | None = None, *, note: str | None = None,
at: str | None = None) -> dict[str, Any]:
"""构造一个项。status 不给时按来源取初始状态(initial_status)。"""
at: str | None = None, complete: bool = False) -> dict[str, Any]:
"""构造一个项。status 不给时按来源取初始状态(initial_status)。complete 只用于码值项(已列出全部取值),
为假时不写这个键:没有标记和标记为否是一回事,写法统一,冻结进快照的哈希才稳定。"""
out: dict[str, Any] = {"value": value, "source": source, "status": status or initial_status(source)}
if note:
out["note"] = note
if at:
out["updated_at"] = at
if complete:
out["complete"] = True
return out
def codes_complete(item: Any) -> bool:
"""码值项是不是标了「已列全」。"""
return isinstance(item, Mapping) and item.get("complete") is True
# ==========================================================================
# 异常
# ==========================================================================
@@ -220,12 +232,15 @@ _COLUMN_VALUE_RULES: dict[str, tuple[Any, str]] = {
}
def _item_problems(where: str, item: Any, rule: tuple[Any, str]) -> list[str]:
def _item_problems(where: str, item: Any, rule: tuple[Any, str], *,
keys: frozenset[str] = _ITEM_KEYS) -> list[str]:
if not isinstance(item, dict) or not {"value", "source", "status"} <= set(item):
return [f"{where}不是有效的目录项:需要值、来源和状态"]
problems: list[str] = []
if extra := sorted(set(item) - _ITEM_KEYS):
if extra := sorted(set(item) - keys):
problems.append(f"{where}含有不认识的字段「{'、'.join(extra)}」")
if "complete" in item and not isinstance(item["complete"], bool):
problems.append(f"{where}是否已列出全部取值应为是或否")
if item["source"] not in ITEM_SOURCES:
problems.append(f"{where}的来源「{item['source']}」不在可选值内")
if item["status"] not in ITEM_STATUSES:
@@ -309,7 +324,8 @@ def validate_notes(notes: Any) -> list[str]:
for field_name in COLUMN_FIELDS:
if field_name in items:
problems += _item_problems(f"列 {col} 的{COLUMN_FIELD_LABEL[field_name]}", items[field_name],
_COLUMN_VALUE_RULES[field_name])
_COLUMN_VALUE_RULES[field_name],
keys=_CODES_ITEM_KEYS if field_name == "codes" else _ITEM_KEYS)
relations = notes.get("relations", [])
if not isinstance(relations, list):
problems.append("关联关系应为列表")
@@ -481,6 +497,14 @@ def _value_of(key: Slot, item: Mapping[str, Any]) -> Any:
return item.get("value")
def _edit_value(key: Slot, item: Mapping[str, Any]) -> Any:
"""人工编辑时比较「改没改」看的部分:值,码值另加「已列全」标记——只勾上这个标记也是人工改动(记为人工填写、
已确认),检查据此认定码值表是完整的。"""
if key[0] == "c" and key[2] == "codes":
return item.get("value"), codes_complete(item)
return _value_of(key, item)
def _core(item: Mapping[str, Any]) -> dict[str, Any]:
"""去掉修改时间的项:判断起草结果和已有的是不是一回事。"""
return {k: v for k, v in item.items() if k != "updated_at"}
@@ -630,6 +654,8 @@ def _normalize_submitted(submitted: Mapping[str, Any], table: str) -> dict[str,
if isinstance(item, dict) and "value" in item:
item.setdefault("source", "human")
item.setdefault("status", "confirmed")
if item.get("complete") is False:
del item["complete"] # 「没列全」就是没有这个标记,写法统一(make_item)
for name in TABLE_FIELDS:
fill(out.get(name))
@@ -657,7 +683,7 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str,
"""人工提交的整份目录 → 要写入的目录(不改入参)。
- 值改过的项、新填的项:记为 human / confirmed。提交里写的来源和状态不作数——客户端不能冒充外键约束
或数据剖析。
或数据剖析。码值的「已列全」标记跟着值走:只勾上或取消它也算改过(_edit_value)。
- 值没动的项:保持原来的来源和状态;提交里把状态改成确认、驳回或来源的初始状态的,照改(等同单项审阅);
改成别的状态的不认(不能把推断改成「有确证」)。
- 已有、但提交里没有的表级项、列级项:删掉。
@@ -681,7 +707,7 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str,
out: dict[Slot, dict[str, Any]] = {}
for key, item in new.items():
prev = old.get(key)
if prev is None or _value_of(key, item) != _value_of(key, prev):
if prev is None or _edit_value(key, item) != _edit_value(key, prev):
fresh = {k: v for k, v in item.items() if k != "updated_at"}
fresh.update(source="human", status="confirmed", updated_at=at)
out[key] = fresh
@@ -1015,7 +1041,11 @@ def apply_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str,
rel["note"] = prev["note"] # 覆盖率沿用了,说明它怎么测的那段备注也留着
slots[key] = rel
else:
slots[key] = {"value": change.after, "source": "human", "status": "confirmed", "updated_at": at}
# 码值是补充、不删已有的码:原来标了「已列全」的,补充之后仍然列全。提案自己不能标「已列全」——
# 用户在对话里说一个码值的意思,不等于确认这一列只有这些取值
done = key[0] == "c" and key[2] == "codes" and prev is not None and prev.get("status") != "rejected" \
and codes_complete(prev)
slots[key] = make_item(change.after, "human", "confirmed", at=at, complete=done)
out = _assemble(slots)
if problems := validate_notes(out):
raise CatalogInvalid(problems)
@@ -2079,7 +2109,8 @@ __all__ = [
"DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL",
"MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_VERIFY_COVERAGE", "PatchChange", "PatchPlan",
"QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft",
"TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "describe_slot",
"TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "codes_complete",
"describe_slot",
"draft_catalog", "draft_structure", "draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations",
"initial_status", "join_paths", "make_item", "merge_notes", "now_iso", "parse_path", "plan_patch",
"profile_relation_holds", "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id",
+8 -4
View File
@@ -528,12 +528,14 @@ class _RelationOutcome:
@dataclass
class _CodesOutcome:
"""一列的取值分布:[(码值, 行数)],按行数从多到少。"""
"""一列的取值分布:[(码值, 行数)],按行数从多到少。complete:看的是全表(没有只看前若干行),取到的就是
这一列的全部取值,码值项标「已列全」,SQL 检查才拿它判断某个值「不在码值表里」。"""
column: str
values: list[tuple[str, int]]
rows: int
note: str
complete: bool = False
@dataclass
@@ -874,7 +876,7 @@ async def _check_codes(cx: _Context, table: str, column: str, out: _TableOutcome
scope = f"按前 {scan_cap} 行统计" if scan_cap else "统计全表"
spread = "、".join(f"{k}({_share(n, rows)})" for k, n in values)
note = f"{NOTE_PREFIX}{cx.day}):{scope},{rows} 行非空值共 {len(values)} 个取值:{spread}。"
out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note))
out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note, complete=scan_cap is None))
def _is_date_column(col: dict[str, Any]) -> bool:
@@ -980,7 +982,8 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *,
- 没被确认、驳回的关系:记为 source=profile,覆盖率够、父键唯一的 verified,否则 proposed;按 merge_notes
的规则并入(剖析顶得掉命名推断、模型起草,顶不掉外键和人工)。
- 人工确认过的关系:只补覆盖率和基数(_patch_confirmed)。
- 码值候选:profile / proposed,含义沿用别的来源写过的(_codes_value);人工确认、驳回过的码值不碰。
- 码值候选:profile / proposed,含义沿用别的来源写过的(_codes_value);整表统计得到的标「已列全」,只看了
前若干行的不标;人工确认、驳回过的码值不碰。
- 业务日期:只有一个日期类列时提议,profile / proposed;已有别的来源定的业务日期不顶掉。
- covered={"profile"}:这次完整检查过、却没再得出的剖析结论删掉(比如取值变多、不再像码值的列,列被遮罩
之后它的码值);没查成的检查(超时、预算用完……)把剖析以前写的结论原样带上,不因为这次没查成就删。
@@ -1015,7 +1018,8 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *,
item = (described.get(c.column) or {}).get("codes") if isinstance(described.get(c.column), dict) else None
if isinstance(item, dict) and item.get("status") in ("confirmed", "rejected"):
continue
columns[c.column] = {"codes": catalog.make_item(_codes_value(c, item), "profile", "proposed", note=c.note)}
columns[c.column] = {"codes": catalog.make_item(_codes_value(c, item), "profile", "proposed", note=c.note,
complete=c.complete)}
findings.append(_codes_finding(c, columns[c.column]["codes"]["value"]))
for column in out.keep_codes:
item = (described.get(column) or {}).get("codes") if isinstance(described.get(column), dict) else None
+11 -7
View File
@@ -12,7 +12,7 @@
- join_unconfirmed:关联条件对不上目录里的任何关系;
- ratio_aggregated:对比率列直接求和或求平均;
- missing_valid_filter:表定义了有效记录条件,查询完全没涉及条件里的列;
- unknown_code:对有码值的列写了 = 或 IN,值不在码值表里;
- unknown_code:对码值表标了「已列全」的列写了 = 或 IN,值不在码值表里(只补了一部分的码值表不拿来判断);
- wrong_date_column:表定义了业务日期,查询却按这张表的另一个时间列分组或筛选。
**级别看依据。** 依据全是 confirmed / verified(人工确认、外键约束、数据剖析)时,fanout_sum、stock_summed 是 error,
@@ -882,7 +882,9 @@ class _ScopeCheck:
ref = self.resolve(column)
item = ref[0].column_item(ref[1], "codes") if ref else None
codes = item.get("value") if item else None
if not isinstance(codes, Mapping) or not codes:
# 只有标了「已列全」的码值表才能说某个值「不在里面」:经对话只补了 status=9 表示作废,之后 status = 1
# 就会被报,模型还会照着改错。没标的码值表只是已知的一部分,不判(宁可漏报)
if not isinstance(codes, Mapping) or not codes or not catalog.codes_complete(item):
continue
for value in values:
literal = _literal(value)
@@ -898,14 +900,16 @@ class _ScopeCheck:
shown = "、".join(f"{k}={v}" for k, v in list(codes.items())[:10]) + ("等" if len(codes) > 10 else "")
told = "、".join(f"「{v}」" for v in values[:5])
listed = "、".join(f"{k}({v})" for k, v in list(codes.items())[:10])
hint = f";拿不准就先用 db_schema__{self.checker.source_name} 查这张表的码值" if self.checker.source_name else ""
hint = f",或者用 db_schema__{self.checker.source_name} 看这张表的码值说明" if self.checker.source_name else ""
# 给模型的话只叫它核对,不叫它「照目录改」:码值表也可能漏了某个取值,照改会把本来对的 SQL 改错
out.append(SqlCheck(
code="unknown_code", level=_level("unknown_code", [item.get("status")]), table=table.name,
column=column, sql_excerpt=self.excerpt(node),
message=(f"「{table.label()}」的「{table.column_label(column)}」没有码值{told}(已知码值:{shown})。"
"请按数据目录中的码值修改筛选条件"),
for_model=(f"{table.name}.{column} 的码值只有 {listed},SQL 里写的 {'、'.join(values[:5])} 不在其中。"
f"照目录里的码值改{hint}")))
message=(f"「{table.label()}」的「{table.column_label(column)}」没有码值{told}(数据目录中已列出的全部码值:"
f"{shown})。请核对筛选条件里的取值"),
for_model=(f"{table.name}.{column} 的码值表已列全,只有 {listed},SQL 里写的 {'、'.join(values[:5])} "
f"不在其中。核对这个取值有没有写错(比如写成了含义、别的列的取值,或者类型不对);拿不准就先查一下"
f"这一列实际有哪些取值{hint}。确认取值存在就保留原写法,并在结果说明里写明数据目录的码值表可能缺了它")))
return out
# ---- wrong_date_column
+9 -4
View File
@@ -24,6 +24,11 @@ def _item(value: Any, status: str) -> dict[str, Any]:
return make_item(value, _SOURCE_OF[status], status)
def _codes(value: dict[str, str], status: str) -> dict[str, Any]:
"""码值项,标了「已列全」:码值不在码值表中的检查只认这样的码值表。"""
return make_item(value, _SOURCE_OF[status], status, complete=True)
def _rel(table: str, columns: list[str], to_table: str, to_columns: list[str], cardinality: str,
status: str) -> dict[str, Any]:
source = {"confirmed": "human", "verified": "fk"}.get(status, "name")
@@ -51,7 +56,7 @@ def notes(status: str = "confirmed", override: dict[tuple[str, str], str] | None
"total_amount": {"label": _item("订单金额", "confirmed"), "unit": _item("元", s),
"measure": _item("flow", s)},
"status": {"label": _item("订单状态", "confirmed"), "measure": _item("status", s),
"codes": _item({"1": "已支付", "2": "已退款"}, s)},
"codes": _codes({"1": "已支付", "2": "已退款"}, s)},
"ordered_at": {"label": _item("下单时间", "confirmed")},
},
},
@@ -72,7 +77,7 @@ def notes(status: str = "confirmed", override: dict[tuple[str, str], str] | None
"valid_filter": _item("status = 1", s),
"columns": {
"visitor_count": {"label": _item("入园人数", "confirmed"), "measure": _item("flow", s)},
"status": {"label": _item("检票状态", "confirmed"), "codes": _item({"1": "有效", "0": "作废"}, s)},
"status": {"label": _item("检票状态", "confirmed"), "codes": _codes({"1": "有效", "0": "作废"}, s)},
"visit_time": {"label": _item("检票时间", "confirmed")},
},
"relations": [_rel("visits", ["park_id"], "parks", ["id"], "many_to_one", s)],
@@ -102,8 +107,8 @@ def notes(status: str = "confirmed", override: dict[tuple[str, str], str] | None
"channels": {
"label": _item("渠道", "confirmed"),
"columns": {"channel_type": {"label": _item("渠道类型", "confirmed"),
"codes": _item({"线上直销": "官网和小程序", "线下": "窗口售票",
"分销": "旅行社和第三方平台"}, s)}},
"codes": _codes({"线上直销": "官网和小程序", "线下": "窗口售票",
"分销": "旅行社和第三方平台"}, s)}},
},
}
for (table, path), wanted in (override or {}).items():
+90
View File
@@ -252,3 +252,93 @@ def test_relation_graph_keeps_structure_relations_of_partially_annotated_tables(
assert edges == {"gates"}
# ==========================================================================
# A4:码值「已列全」
# ==========================================================================
def _codes(value: dict, *, complete: bool | None, status: str = "confirmed") -> dict:
item = catalog.make_item(value, "human" if status == "confirmed" else "profile", status)
if complete is not None:
item["complete"] = complete
return item
async def test_unknown_code_only_for_codes_marked_complete(scenic_db):
cache = await _schema(scenic_db)
sql = "SELECT COUNT(*) FROM visits WHERE status = 3"
def found(item: dict) -> list[str]:
notes = scenic_notes.notes()
notes["visits"]["columns"]["status"]["codes"] = item
checker = sqlcheck.SqlChecker(kind="sqlite", schema_cache=cache, notes=notes, source_name="scenic")
return [c.code for c in checker.check(sql)]
assert "unknown_code" not in found(_codes({"1": "有效", "0": "作废"}, complete=None))
assert "unknown_code" in found(_codes({"1": "有效", "0": "作废"}, complete=True))
async def test_partial_codes_from_a_patch_are_not_treated_as_the_full_list(client, make_source, scenic_db):
"""审查探针 A4:经提案只写了 status=9 表示作废,之后 status = 1 不该被报「码值不在码值表中」。"""
sid = await make_source(scenic_db, draft=["orders"])
detail = await _notes(client, sid, "orders")
r = await client.post(f"/api/datasources/{sid}/catalog/orders/patch", json={
"changes": [{"path": "columns.status.codes", "value": {"9": "作废"}}], "if_version": detail["version"]})
assert r.status_code == 200, r.text
codes = r.json()["notes"]["columns"]["status"]["codes"]
assert codes["value"] == {"9": "作废"} and "complete" not in codes
sql = "SELECT COUNT(*) FROM orders WHERE status = 1"
assert not any(c[0] == "unknown_code" for c in await _found(sid, sql))
# 人工在目录页把码值补全,并勾上「已列出全部取值」:这时才报
notes = r.json()["notes"]
notes["columns"]["status"]["codes"] = {**codes, "value": {"1": "已支付", "2": "已退款", "9": "作废"},
"complete": True}
r = await _put(client, sid, "orders", notes, r.json()["version"])
assert r.status_code == 200, r.text
saved = r.json()["notes"]["columns"]["status"]["codes"]
assert (saved["source"], saved["status"], saved.get("complete")) == ("human", "confirmed", True)
assert not any(c[0] == "unknown_code" for c in await _found(sid, sql))
checker = await _checker(sid)
[check] = [c for c in checker.check("SELECT COUNT(*) FROM orders WHERE status = 5") if c.code == "unknown_code"]
assert check.level == "warning"
# 给模型的话:提示核对取值,不叫它「照目录里的码值改」
assert "照目录里的码值改" not in check.for_model and "核对" in check.for_model
def test_codes_complete_flag_is_validated_and_counts_as_an_edit():
base = {"columns": {"status": {"codes": _codes({"1": "", "0": ""}, complete=None, status="proposed")}}}
assert catalog.validate_notes(base) == []
assert catalog.validate_notes({"columns": {"status": {"codes": _codes({"1": "有效"}, complete=True)}}}) == []
bad = catalog.validate_notes({"columns": {"status": {"codes": {**_codes({"1": "有效"}, complete=None),
"complete": "是"}}}})
assert bad and "码值" in bad[0]
wrong_place = catalog.validate_notes({"label": {**catalog.make_item("入园记录", "human"), "complete": True}})
assert wrong_place and "complete" in wrong_place[0]
# 值没动、只勾上「已列全」:算人工改动,记为人工填写、已确认
ticked = {"columns": {"status": {"codes": {**base["columns"]["status"]["codes"], "complete": True}}}}
out = catalog.apply_human_edit(base, ticked, table="visits", at=AT)
codes = out["columns"]["status"]["codes"]
assert (codes["source"], codes["status"], codes["complete"]) == ("human", "confirmed", True)
# 取消勾选:不留 complete: false,和没有这个标记写法一样
unticked = {"columns": {"status": {"codes": {**codes, "complete": False}}}}
out = catalog.apply_human_edit(out, unticked, table="visits", at=AT)
assert "complete" not in out["columns"]["status"]["codes"]
async def test_profile_marks_codes_complete_only_when_the_whole_table_was_read(client, make_source, scenic_db):
whole = await make_source(scenic_db, options=_on(), draft=["visits"])
r = await client.post(_url(whole), json={"tables": ["visits"]})
assert r.status_code == 200, r.text
codes = (await _notes(client, whole, "visits"))["notes"]["columns"]["status"]["codes"]
assert "统计全表" in codes["note"] and codes.get("complete") is True
# 不做整表统计(只看前若干行):取值可能没看全,不标
capped = await make_source(scenic_db, options=_on(max_scan_rows=0, sample_size=10), draft=["visits"])
r = await client.post(_url(capped), json={"tables": ["visits"]})
assert r.status_code == 200, r.text
codes = (await _notes(client, capped, "visits"))["notes"]["columns"]["status"]["codes"]
assert "按前 10 行统计" in codes["note"] and "complete" not in codes