From b078d08ffeb722c3d8d92d36cb2011d61759b78e Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 01:13:13 +0800 Subject: [PATCH] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95=20F1-A4?= =?UTF-8?q?=EF=BC=9A=E7=A0=81=E5=80=BC=E9=A1=B9=E5=8A=A0=E3=80=8C=E5=B7=B2?= =?UTF-8?q?=E5=88=97=E5=85=A8=E3=80=8D=E6=A0=87=E8=AE=B0=EF=BC=88complete:?= =?UTF-8?q?=20true=EF=BC=89=EF=BC=8C=E5=8F=AA=E6=9C=89=E4=BA=BA=E5=B7=A5?= =?UTF-8?q?=E6=98=8E=E7=A1=AE=E5=8B=BE=E9=80=89=E3=80=81=E6=88=96=E5=89=96?= =?UTF-8?q?=E6=9E=90=E6=95=B4=E8=A1=A8=E7=BB=9F=E8=AE=A1=EF=BC=88=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=E5=8F=AA=E7=9C=8B=E5=89=8D=E8=8B=A5=E5=B9=B2=E8=A1=8C?= =?UTF-8?q?=EF=BC=89=E5=BE=97=E5=88=B0=E7=9A=84=E7=A0=81=E5=80=BC=E6=89=8D?= =?UTF-8?q?=E6=9C=89=EF=BC=9B=E7=A0=81=E5=80=BC=E4=B8=8D=E5=9C=A8=E7=A0=81?= =?UTF-8?q?=E5=80=BC=E8=A1=A8=E4=B8=AD=E7=9A=84=E6=A3=80=E6=9F=A5=E5=8F=AA?= =?UTF-8?q?=E8=AE=A4=E6=A0=87=E4=BA=86=E5=AE=83=E7=9A=84=E7=A0=81=E5=80=BC?= =?UTF-8?q?=E8=A1=A8=EF=BC=8C=E7=BB=99=E6=A8=A1=E5=9E=8B=E7=9A=84=E8=AF=9D?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E6=8F=90=E7=A4=BA=E6=A0=B8=E5=AF=B9=E5=8F=96?= =?UTF-8?q?=E5=80=BC=EF=BC=8C=E4=B8=8D=E5=86=8D=E5=8F=AB=E5=AE=83=E7=85=A7?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E9=87=8C=E7=9A=84=E7=A0=81=E5=80=BC=E6=94=B9?= =?UTF-8?q?=E3=80=82=E4=BB=A5=E5=89=8D=E7=BB=8F=E5=AF=B9=E8=AF=9D=E5=8F=AA?= =?UTF-8?q?=E8=A1=A5=E4=BA=86=20status=3D9=20=E8=A1=A8=E7=A4=BA=E4=BD=9C?= =?UTF-8?q?=E5=BA=9F=EF=BC=8C=E4=B9=8B=E5=90=8E=20status=20=3D=201=20?= =?UTF-8?q?=E5=B0=B1=E8=A2=AB=E6=8A=A5=E6=8F=90=E9=86=92=EF=BC=8C=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E8=BF=98=E5=8F=AF=E8=83=BD=E6=8A=8A=E5=AF=B9=E7=9A=84?= =?UTF-8?q?=20SQL=20=E6=94=B9=E9=94=99=E3=80=82=E6=8F=90=E6=A1=88=E8=A1=A5?= =?UTF-8?q?=E5=85=85=E7=A0=81=E5=80=BC=E6=97=B6=E6=B2=BF=E7=94=A8=E5=8E=9F?= =?UTF-8?q?=E6=9C=89=E7=9A=84=E6=A0=87=E8=AE=B0=E3=80=81=E8=87=AA=E5=B7=B1?= =?UTF-8?q?=E4=B8=8D=E8=83=BD=E6=A0=87=EF=BC=9B=E7=9B=AE=E5=BD=95=E9=A1=B5?= =?UTF-8?q?=E7=9A=84=E7=A0=81=E5=80=BC=E7=BC=96=E8=BE=91=E5=92=8C=E5=A1=AB?= =?UTF-8?q?=E5=86=99=E5=90=AB=E4=B9=89=E5=BC=B9=E7=AA=97=E5=8A=A0=E3=80=8C?= =?UTF-8?q?=E5=B7=B2=E5=88=97=E5=87=BA=E5=85=A8=E9=83=A8=E5=8F=96=E5=80=BC?= =?UTF-8?q?=E3=80=8D=E5=8B=BE=E9=80=89=EF=BC=8C=E8=A1=A8=E6=A0=BC=E9=87=8C?= =?UTF-8?q?=E6=A0=87=E3=80=8C=E5=B7=B2=E5=88=97=E5=85=A8=E3=80=8D=EF=BC=8C?= =?UTF-8?q?=E6=A3=80=E6=9F=A5=E8=84=9A=E6=9C=AC=E8=B7=9F=E4=B8=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog.py | 51 ++++++++--- backend/app/data/catalog_profile.py | 12 ++- backend/app/data/sqlcheck.py | 18 ++-- .../tests/fixtures/catalog/scenic_notes.py | 13 ++- backend/tests/test_catalog_semantics.py | 90 +++++++++++++++++++ frontend/src/lib/terms.ts | 7 +- frontend/src/pages/catalog/CodesDialog.tsx | 19 +++- frontend/src/pages/catalog/ColumnTable.tsx | 52 +++++++---- frontend/src/pages/catalog/TableDetail.tsx | 17 ++-- frontend/src/pages/catalog/model.ts | 34 ++++++- scripts/check-catalog.mjs | 20 ++++- 11 files changed, 275 insertions(+), 58 deletions(-) diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 65952c71..c6e2c8fe 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -15,7 +15,8 @@ notes 结构（每个「项」都是 {value, source, status}，可选 note、upd "valid_filter": 项(str，SQL 条件片段), "dedup": 项(str), "columns": {列名: {"label": 项, "meaning": 项, "unit": 项, "measure": 项("flow"|"stock"|"ratio"|"identifier"|"status"|"attribute"), - "codes": 项(dict[码值, 含义]；含义可为空串，表示待填写)}}, + "codes": 项(dict[码值, 含义]；含义可为空串，表示待填写；项上可带 complete: true， + 表示已列出全部取值)}}, "relations": [{"id", "columns", "to_table", "to_columns", "cardinality", "coverage", "source", "status", "note"?}]} @@ -81,6 +82,9 @@ COLUMN_FIELD_LABEL = {"label": "中文名", "meaning": "含义", "unit": "单位 _INITIAL_STATUS = {"fk": "verified", "human": "confirmed"} _ITEM_KEYS = frozenset({"value", "source", "status", "note", "updated_at"}) +#: 码值项另有一个「已列全」标记（complete: true）：码值表列出了这一列的全部取值。只有人工明确确认、或者数据剖析 +#: 没截断地看完了全表取值时才有；没有它的码值表只是「已知的一部分」，SQL 检查不能拿它判断某个值「不在码值表里」 +_CODES_ITEM_KEYS = _ITEM_KEYS | {"complete"} _RELATION_KEYS = frozenset({"id", "columns", "to_table", "to_columns", "cardinality", "coverage", "source", "status", "note", "updated_at"}) _BUSINESS_DATE_KEYS = frozenset({"column", "rule", "timezone"}) @@ -127,16 +131,24 @@ def unreviewed_status(item: Mapping[str, Any]) -> str: def make_item(value: Any, source: str, status: str | None = None, *, note: str | None = None, - at: str | None = None) -> dict[str, Any]: - """构造一个项。status 不给时按来源取初始状态（initial_status）。""" + at: str | None = None, complete: bool = False) -> dict[str, Any]: + """构造一个项。status 不给时按来源取初始状态（initial_status）。complete 只用于码值项（已列出全部取值）， + 为假时不写这个键：没有标记和标记为否是一回事，写法统一，冻结进快照的哈希才稳定。""" out: dict[str, Any] = {"value": value, "source": source, "status": status or initial_status(source)} if note: out["note"] = note if at: out["updated_at"] = at + if complete: + out["complete"] = True return out +def codes_complete(item: Any) -> bool: + """码值项是不是标了「已列全」。""" + return isinstance(item, Mapping) and item.get("complete") is True + + # ========================================================================== # 异常 # ========================================================================== @@ -220,12 +232,15 @@ _COLUMN_VALUE_RULES: dict[str, tuple[Any, str]] = { } -def _item_problems(where: str, item: Any, rule: tuple[Any, str]) -> list[str]: +def _item_problems(where: str, item: Any, rule: tuple[Any, str], *, + keys: frozenset[str] = _ITEM_KEYS) -> list[str]: if not isinstance(item, dict) or not {"value", "source", "status"} <= set(item): return [f"{where}不是有效的目录项：需要值、来源和状态"] problems: list[str] = [] - if extra := sorted(set(item) - _ITEM_KEYS): + if extra := sorted(set(item) - keys): problems.append(f"{where}含有不认识的字段「{'、'.join(extra)}」") + if "complete" in item and not isinstance(item["complete"], bool): + problems.append(f"{where}是否已列出全部取值应为是或否") if item["source"] not in ITEM_SOURCES: problems.append(f"{where}的来源「{item['source']}」不在可选值内") if item["status"] not in ITEM_STATUSES: @@ -309,7 +324,8 @@ def validate_notes(notes: Any) -> list[str]: for field_name in COLUMN_FIELDS: if field_name in items: problems += _item_problems(f"列 {col} 的{COLUMN_FIELD_LABEL[field_name]}", items[field_name], - _COLUMN_VALUE_RULES[field_name]) + _COLUMN_VALUE_RULES[field_name], + keys=_CODES_ITEM_KEYS if field_name == "codes" else _ITEM_KEYS) relations = notes.get("relations", []) if not isinstance(relations, list): problems.append("关联关系应为列表") @@ -481,6 +497,14 @@ def _value_of(key: Slot, item: Mapping[str, Any]) -> Any: return item.get("value") +def _edit_value(key: Slot, item: Mapping[str, Any]) -> Any: + """人工编辑时比较「改没改」看的部分：值，码值另加「已列全」标记——只勾上这个标记也是人工改动（记为人工填写、 + 已确认），检查据此认定码值表是完整的。""" + if key[0] == "c" and key[2] == "codes": + return item.get("value"), codes_complete(item) + return _value_of(key, item) + + def _core(item: Mapping[str, Any]) -> dict[str, Any]: """去掉修改时间的项：判断起草结果和已有的是不是一回事。""" return {k: v for k, v in item.items() if k != "updated_at"} @@ -630,6 +654,8 @@ def _normalize_submitted(submitted: Mapping[str, Any], table: str) -> dict[str, if isinstance(item, dict) and "value" in item: item.setdefault("source", "human") item.setdefault("status", "confirmed") + if item.get("complete") is False: + del item["complete"] # 「没列全」就是没有这个标记，写法统一（make_item） for name in TABLE_FIELDS: fill(out.get(name)) @@ -657,7 +683,7 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str, """人工提交的整份目录 → 要写入的目录（不改入参）。 - 值改过的项、新填的项：记为 human / confirmed。提交里写的来源和状态不作数——客户端不能冒充外键约束 - 或数据剖析。 + 或数据剖析。码值的「已列全」标记跟着值走：只勾上或取消它也算改过（_edit_value）。 - 值没动的项：保持原来的来源和状态；提交里把状态改成确认、驳回或来源的初始状态的，照改（等同单项审阅）； 改成别的状态的不认（不能把推断改成「有确证」）。 - 已有、但提交里没有的表级项、列级项：删掉。 @@ -681,7 +707,7 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str, out: dict[Slot, dict[str, Any]] = {} for key, item in new.items(): prev = old.get(key) - if prev is None or _value_of(key, item) != _value_of(key, prev): + if prev is None or _edit_value(key, item) != _edit_value(key, prev): fresh = {k: v for k, v in item.items() if k != "updated_at"} fresh.update(source="human", status="confirmed", updated_at=at) out[key] = fresh @@ -1015,7 +1041,11 @@ def apply_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str, rel["note"] = prev["note"] # 覆盖率沿用了，说明它怎么测的那段备注也留着 slots[key] = rel else: - slots[key] = {"value": change.after, "source": "human", "status": "confirmed", "updated_at": at} + # 码值是补充、不删已有的码：原来标了「已列全」的，补充之后仍然列全。提案自己不能标「已列全」—— + # 用户在对话里说一个码值的意思，不等于确认这一列只有这些取值 + done = key[0] == "c" and key[2] == "codes" and prev is not None and prev.get("status") != "rejected" \ + and codes_complete(prev) + slots[key] = make_item(change.after, "human", "confirmed", at=at, complete=done) out = _assemble(slots) if problems := validate_notes(out): raise CatalogInvalid(problems) @@ -2079,7 +2109,8 @@ __all__ = [ "DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL", "MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_VERIFY_COVERAGE", "PatchChange", "PatchPlan", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft", - "TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "describe_slot", + "TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "codes_complete", + "describe_slot", "draft_catalog", "draft_structure", "draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "join_paths", "make_item", "merge_notes", "now_iso", "parse_path", "plan_patch", "profile_relation_holds", "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index c4d51f0a..7b194268 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -528,12 +528,14 @@ class _RelationOutcome: @dataclass class _CodesOutcome: - """一列的取值分布：[(码值, 行数)]，按行数从多到少。""" + """一列的取值分布：[(码值, 行数)]，按行数从多到少。complete：看的是全表（没有只看前若干行），取到的就是 + 这一列的全部取值，码值项标「已列全」，SQL 检查才拿它判断某个值「不在码值表里」。""" column: str values: list[tuple[str, int]] rows: int note: str + complete: bool = False @dataclass @@ -874,7 +876,7 @@ async def _check_codes(cx: _Context, table: str, column: str, out: _TableOutcome scope = f"按前 {scan_cap} 行统计" if scan_cap else "统计全表" spread = "、".join(f"{k}（{_share(n, rows)}）" for k, n in values) note = f"{NOTE_PREFIX}{cx.day}）：{scope}，{rows} 行非空值共 {len(values)} 个取值：{spread}。" - out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note)) + out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note, complete=scan_cap is None)) def _is_date_column(col: dict[str, Any]) -> bool: @@ -980,7 +982,8 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *, - 没被确认、驳回的关系：记为 source=profile，覆盖率够、父键唯一的 verified，否则 proposed；按 merge_notes 的规则并入（剖析顶得掉命名推断、模型起草，顶不掉外键和人工）。 - 人工确认过的关系：只补覆盖率和基数（_patch_confirmed）。 - - 码值候选：profile / proposed，含义沿用别的来源写过的（_codes_value）；人工确认、驳回过的码值不碰。 + - 码值候选：profile / proposed，含义沿用别的来源写过的（_codes_value）；整表统计得到的标「已列全」，只看了 + 前若干行的不标；人工确认、驳回过的码值不碰。 - 业务日期：只有一个日期类列时提议，profile / proposed；已有别的来源定的业务日期不顶掉。 - covered={"profile"}：这次完整检查过、却没再得出的剖析结论删掉（比如取值变多、不再像码值的列，列被遮罩 之后它的码值）；没查成的检查（超时、预算用完……）把剖析以前写的结论原样带上，不因为这次没查成就删。 @@ -1015,7 +1018,8 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *, item = (described.get(c.column) or {}).get("codes") if isinstance(described.get(c.column), dict) else None if isinstance(item, dict) and item.get("status") in ("confirmed", "rejected"): continue - columns[c.column] = {"codes": catalog.make_item(_codes_value(c, item), "profile", "proposed", note=c.note)} + columns[c.column] = {"codes": catalog.make_item(_codes_value(c, item), "profile", "proposed", note=c.note, + complete=c.complete)} findings.append(_codes_finding(c, columns[c.column]["codes"]["value"])) for column in out.keep_codes: item = (described.get(column) or {}).get("codes") if isinstance(described.get(column), dict) else None diff --git a/backend/app/data/sqlcheck.py b/backend/app/data/sqlcheck.py index d5207577..6f234d1a 100644 --- a/backend/app/data/sqlcheck.py +++ b/backend/app/data/sqlcheck.py @@ -12,7 +12,7 @@ - join_unconfirmed：关联条件对不上目录里的任何关系； - ratio_aggregated：对比率列直接求和或求平均； - missing_valid_filter：表定义了有效记录条件，查询完全没涉及条件里的列； -- unknown_code：对有码值的列写了 = 或 IN，值不在码值表里； +- unknown_code：对码值表标了「已列全」的列写了 = 或 IN，值不在码值表里（只补了一部分的码值表不拿来判断）； - wrong_date_column：表定义了业务日期，查询却按这张表的另一个时间列分组或筛选。 **级别看依据。** 依据全是 confirmed / verified（人工确认、外键约束、数据剖析）时，fanout_sum、stock_summed 是 error， @@ -882,7 +882,9 @@ class _ScopeCheck: ref = self.resolve(column) item = ref[0].column_item(ref[1], "codes") if ref else None codes = item.get("value") if item else None - if not isinstance(codes, Mapping) or not codes: + # 只有标了「已列全」的码值表才能说某个值「不在里面」：经对话只补了 status=9 表示作废，之后 status = 1 + # 就会被报，模型还会照着改错。没标的码值表只是已知的一部分，不判（宁可漏报） + if not isinstance(codes, Mapping) or not codes or not catalog.codes_complete(item): continue for value in values: literal = _literal(value) @@ -898,14 +900,16 @@ class _ScopeCheck: shown = "、".join(f"{k}={v}" for k, v in list(codes.items())[:10]) + ("等" if len(codes) > 10 else "") told = "、".join(f"「{v}」" for v in values[:5]) listed = "、".join(f"{k}（{v}）" for k, v in list(codes.items())[:10]) - hint = f"；拿不准就先用 db_schema__{self.checker.source_name} 查这张表的码值" if self.checker.source_name else "" + hint = f"，或者用 db_schema__{self.checker.source_name} 看这张表的码值说明" if self.checker.source_name else "" + # 给模型的话只叫它核对，不叫它「照目录改」：码值表也可能漏了某个取值，照改会把本来对的 SQL 改错 out.append(SqlCheck( code="unknown_code", level=_level("unknown_code", [item.get("status")]), table=table.name, column=column, sql_excerpt=self.excerpt(node), - message=(f"「{table.label()}」的「{table.column_label(column)}」没有码值{told}（已知码值：{shown}）。" - "请按数据目录中的码值修改筛选条件"), - for_model=(f"{table.name}.{column} 的码值只有 {listed}，SQL 里写的 {'、'.join(values[:5])} 不在其中。" - f"照目录里的码值改{hint}"))) + message=(f"「{table.label()}」的「{table.column_label(column)}」没有码值{told}（数据目录中已列出的全部码值：" + f"{shown}）。请核对筛选条件里的取值"), + for_model=(f"{table.name}.{column} 的码值表已列全，只有 {listed}，SQL 里写的 {'、'.join(values[:5])} " + f"不在其中。核对这个取值有没有写错（比如写成了含义、别的列的取值，或者类型不对）；拿不准就先查一下" + f"这一列实际有哪些取值{hint}。确认取值存在就保留原写法，并在结果说明里写明数据目录的码值表可能缺了它"))) return out # ---- wrong_date_column diff --git a/backend/tests/fixtures/catalog/scenic_notes.py b/backend/tests/fixtures/catalog/scenic_notes.py index c17a8af4..4becd655 100644 --- a/backend/tests/fixtures/catalog/scenic_notes.py +++ b/backend/tests/fixtures/catalog/scenic_notes.py @@ -24,6 +24,11 @@ def _item(value: Any, status: str) -> dict[str, Any]: return make_item(value, _SOURCE_OF[status], status) +def _codes(value: dict[str, str], status: str) -> dict[str, Any]: + """码值项，标了「已列全」：码值不在码值表中的检查只认这样的码值表。""" + return make_item(value, _SOURCE_OF[status], status, complete=True) + + def _rel(table: str, columns: list[str], to_table: str, to_columns: list[str], cardinality: str, status: str) -> dict[str, Any]: source = {"confirmed": "human", "verified": "fk"}.get(status, "name") @@ -51,7 +56,7 @@ def notes(status: str = "confirmed", override: dict[tuple[str, str], str] | None "total_amount": {"label": _item("订单金额", "confirmed"), "unit": _item("元", s), "measure": _item("flow", s)}, "status": {"label": _item("订单状态", "confirmed"), "measure": _item("status", s), - "codes": _item({"1": "已支付", "2": "已退款"}, s)}, + "codes": _codes({"1": "已支付", "2": "已退款"}, s)}, "ordered_at": {"label": _item("下单时间", "confirmed")}, }, }, @@ -72,7 +77,7 @@ def notes(status: str = "confirmed", override: dict[tuple[str, str], str] | None "valid_filter": _item("status = 1", s), "columns": { "visitor_count": {"label": _item("入园人数", "confirmed"), "measure": _item("flow", s)}, - "status": {"label": _item("检票状态", "confirmed"), "codes": _item({"1": "有效", "0": "作废"}, s)}, + "status": {"label": _item("检票状态", "confirmed"), "codes": _codes({"1": "有效", "0": "作废"}, s)}, "visit_time": {"label": _item("检票时间", "confirmed")}, }, "relations": [_rel("visits", ["park_id"], "parks", ["id"], "many_to_one", s)], @@ -102,8 +107,8 @@ def notes(status: str = "confirmed", override: dict[tuple[str, str], str] | None "channels": { "label": _item("渠道", "confirmed"), "columns": {"channel_type": {"label": _item("渠道类型", "confirmed"), - "codes": _item({"线上直销": "官网和小程序", "线下": "窗口售票", - "分销": "旅行社和第三方平台"}, s)}}, + "codes": _codes({"线上直销": "官网和小程序", "线下": "窗口售票", + "分销": "旅行社和第三方平台"}, s)}}, }, } for (table, path), wanted in (override or {}).items(): diff --git a/backend/tests/test_catalog_semantics.py b/backend/tests/test_catalog_semantics.py index b7051377..1f2fa07d 100644 --- a/backend/tests/test_catalog_semantics.py +++ b/backend/tests/test_catalog_semantics.py @@ -252,3 +252,93 @@ def test_relation_graph_keeps_structure_relations_of_partially_annotated_tables( assert edges == {"gates"} +# ========================================================================== +# A4：码值「已列全」 +# ========================================================================== + + +def _codes(value: dict, *, complete: bool | None, status: str = "confirmed") -> dict: + item = catalog.make_item(value, "human" if status == "confirmed" else "profile", status) + if complete is not None: + item["complete"] = complete + return item + + +async def test_unknown_code_only_for_codes_marked_complete(scenic_db): + cache = await _schema(scenic_db) + sql = "SELECT COUNT(*) FROM visits WHERE status = 3" + + def found(item: dict) -> list[str]: + notes = scenic_notes.notes() + notes["visits"]["columns"]["status"]["codes"] = item + checker = sqlcheck.SqlChecker(kind="sqlite", schema_cache=cache, notes=notes, source_name="scenic") + return [c.code for c in checker.check(sql)] + + assert "unknown_code" not in found(_codes({"1": "有效", "0": "作废"}, complete=None)) + assert "unknown_code" in found(_codes({"1": "有效", "0": "作废"}, complete=True)) + + +async def test_partial_codes_from_a_patch_are_not_treated_as_the_full_list(client, make_source, scenic_db): + """审查探针 A4：经提案只写了 status=9 表示作废，之后 status = 1 不该被报「码值不在码值表中」。""" + sid = await make_source(scenic_db, draft=["orders"]) + detail = await _notes(client, sid, "orders") + r = await client.post(f"/api/datasources/{sid}/catalog/orders/patch", json={ + "changes": [{"path": "columns.status.codes", "value": {"9": "作废"}}], "if_version": detail["version"]}) + assert r.status_code == 200, r.text + codes = r.json()["notes"]["columns"]["status"]["codes"] + assert codes["value"] == {"9": "作废"} and "complete" not in codes + sql = "SELECT COUNT(*) FROM orders WHERE status = 1" + assert not any(c[0] == "unknown_code" for c in await _found(sid, sql)) + + # 人工在目录页把码值补全，并勾上「已列出全部取值」：这时才报 + notes = r.json()["notes"] + notes["columns"]["status"]["codes"] = {**codes, "value": {"1": "已支付", "2": "已退款", "9": "作废"}, + "complete": True} + r = await _put(client, sid, "orders", notes, r.json()["version"]) + assert r.status_code == 200, r.text + saved = r.json()["notes"]["columns"]["status"]["codes"] + assert (saved["source"], saved["status"], saved.get("complete")) == ("human", "confirmed", True) + assert not any(c[0] == "unknown_code" for c in await _found(sid, sql)) + checker = await _checker(sid) + [check] = [c for c in checker.check("SELECT COUNT(*) FROM orders WHERE status = 5") if c.code == "unknown_code"] + assert check.level == "warning" + # 给模型的话：提示核对取值，不叫它「照目录里的码值改」 + assert "照目录里的码值改" not in check.for_model and "核对" in check.for_model + + +def test_codes_complete_flag_is_validated_and_counts_as_an_edit(): + base = {"columns": {"status": {"codes": _codes({"1": "", "0": ""}, complete=None, status="proposed")}}} + assert catalog.validate_notes(base) == [] + assert catalog.validate_notes({"columns": {"status": {"codes": _codes({"1": "有效"}, complete=True)}}}) == [] + bad = catalog.validate_notes({"columns": {"status": {"codes": {**_codes({"1": "有效"}, complete=None), + "complete": "是"}}}}) + assert bad and "码值" in bad[0] + wrong_place = catalog.validate_notes({"label": {**catalog.make_item("入园记录", "human"), "complete": True}}) + assert wrong_place and "complete" in wrong_place[0] + + # 值没动、只勾上「已列全」：算人工改动，记为人工填写、已确认 + ticked = {"columns": {"status": {"codes": {**base["columns"]["status"]["codes"], "complete": True}}}} + out = catalog.apply_human_edit(base, ticked, table="visits", at=AT) + codes = out["columns"]["status"]["codes"] + assert (codes["source"], codes["status"], codes["complete"]) == ("human", "confirmed", True) + # 取消勾选：不留 complete: false，和没有这个标记写法一样 + unticked = {"columns": {"status": {"codes": {**codes, "complete": False}}}} + out = catalog.apply_human_edit(out, unticked, table="visits", at=AT) + assert "complete" not in out["columns"]["status"]["codes"] + + +async def test_profile_marks_codes_complete_only_when_the_whole_table_was_read(client, make_source, scenic_db): + whole = await make_source(scenic_db, options=_on(), draft=["visits"]) + r = await client.post(_url(whole), json={"tables": ["visits"]}) + assert r.status_code == 200, r.text + codes = (await _notes(client, whole, "visits"))["notes"]["columns"]["status"]["codes"] + assert "统计全表" in codes["note"] and codes.get("complete") is True + + # 不做整表统计（只看前若干行）：取值可能没看全，不标 + capped = await make_source(scenic_db, options=_on(max_scan_rows=0, sample_size=10), draft=["visits"]) + r = await client.post(_url(capped), json={"tables": ["visits"]}) + assert r.status_code == 200, r.text + codes = (await _notes(client, capped, "visits"))["notes"]["columns"]["status"]["codes"] + assert "按前 10 行统计" in codes["note"] and "complete" not in codes + + diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 8248c033..8e67bcc1 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -2465,6 +2465,11 @@ export const CODES_TEXT = { saved: (n: number) => (n ? `已填写 ${formatNumber(n)} 个码值的含义` : '已保存'), save: '保存', cancel: '取消', + // 「已列出全部取值」：只有勾了的码值表，SQL 检查才提醒「码值不在码值表中」；只列了一部分（比如对话里只补了一个码）不勾 + complete: '已列出全部取值', + completeHint: '勾选表示这一列只会出现这些取值，SQL 检查会据此提醒码值表中没有的取值；只列了一部分时不要勾选', + completeMark: '已列全', + completeMarkHint: '已列出全部取值：SQL 检查会提醒码值表中没有的取值', } // =========================================================================== @@ -2492,7 +2497,7 @@ export const SQL_CHECK_RULE_HINT: Record = { join_unconfirmed: '关联条件对不上数据目录中有确证的关系，可能连错了列', ratio_aggregated: '对比率列直接求和或求平均，得不到正确的比率', missing_valid_filter: '表定义了有效记录条件，查询没有按它筛选', - unknown_code: '按码值筛选时用了码值表中没有的值', + unknown_code: '码值表已列出全部取值，按码值筛选时却用了其中没有的值', wrong_date_column: '表定义了业务日期，查询却按另一个时间列分组或筛选', } diff --git a/frontend/src/pages/catalog/CodesDialog.tsx b/frontend/src/pages/catalog/CodesDialog.tsx index 0cf4a601..726ca239 100644 --- a/frontend/src/pages/catalog/CodesDialog.tsx +++ b/frontend/src/pages/catalog/CodesDialog.tsx @@ -3,28 +3,32 @@ import type { CatalogItem } from '../../types' import { isComposing, Modal, Spinner } from '../../components/ui' import { CODES_TEXT as KT } from '../../lib/terms' import { StatusChip } from './parts' +import { codesComplete } from './model' // =========================================================================== // 填写码值的含义。数据剖析只知道列里出现过哪些取值（码值候选的含义是空串），含义要人填：逐个码值一个输入框， // 比在编辑模式里改「码值=含义」的多行文字省事。保存走整份提交，这一项记为人工填写、已确认； // 暂不清楚的含义可以留空，表格里照旧显示「含义待填写」。 +// 底下的「已列出全部取值」写明当前是否已列全，可以勾选或取消：只有勾了的码值表，SQL 检查才会提醒「取值不在码值表中」。 // =========================================================================== export function CodesDialog({ column, item, saving, onSave, onClose }: { column: string item: CatalogItem> saving: boolean - /** 新的码值对照（码值不变，只改含义） */ - onSave: (value: Record) => void + /** 新的码值对照（码值不变，只改含义）和「已列出全部取值」 */ + onSave: (value: Record, complete: boolean) => void onClose: () => void }) { const [initial] = useState(() => ({ ...item.value })) const [value, setValue] = useState>(initial) + const [wasComplete] = useState(() => codesComplete(item)) + const [complete, setComplete] = useState(wasComplete) const codes = Object.keys(initial) - const dirty = codes.some((k) => (value[k] ?? '').trim() !== (initial[k] ?? '').trim()) + const dirty = complete !== wasComplete || codes.some((k) => (value[k] ?? '').trim() !== (initial[k] ?? '').trim()) const save = () => { if (!dirty || saving) return - onSave(Object.fromEntries(codes.map((k) => [k, (value[k] ?? '').trim()]))) + onSave(Object.fromEntries(codes.map((k) => [k, (value[k] ?? '').trim()])), complete) } return ( +

{KT.hint}

 diff --git a/frontend/src/pages/catalog/ColumnTable.tsx b/frontend/src/pages/catalog/ColumnTable.tsx index 706a9090..d07ef1f9 100644 --- a/frontend/src/pages/catalog/ColumnTable.tsx +++ b/frontend/src/pages/catalog/ColumnTable.tsx @@ -11,8 +11,8 @@ import { } from '../../lib/terms' import { ItemMark } from './parts' import type { ReviewTarget } from './parts' -import { COLUMN_FIELDS, columnPath } from './model' -import type { ColumnField } from './model' +import { COLUMN_FIELDS, codesComplete, columnPath, completeOf } from './model' +import type { ColumnDraft, ColumnField } from './model' // =========================================================================== // 单表详情的列表格：列名 / 类型、中文名、含义、单位、度量类型、码值。每格的值旁边是只有图标的状态标识，点开看来源、 @@ -30,9 +30,9 @@ export interface ColumnModel { items: CatalogColumnNotes | undefined } -type ColumnTexts = Partial> +type ColumnTexts = ColumnDraft -export function ColumnTable({ columns, form, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange }: { +export function ColumnTable({ columns, form, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange, onCodesComplete }: { columns: ColumnModel[] /** 编辑中：列名 → 字段 → 文字（form 是当前的，initial 是进入编辑时的）；不在编辑时为 null */ form: Record | null @@ -47,6 +47,8 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote /** 码值里有含义空着的：打开逐个填写含义的弹窗 */ onFillCodes: (col: string) => void onChange: (col: string, field: ColumnField, value: string) => void + /** 编辑中勾选或取消码值的「已列出全部取值」 */ + onCodesComplete: (col: string, value: boolean) => void }) { return (
@@ -68,7 +70,8 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote {columns.map((c) => ( + systemNotes={systemNotes} onReview={onReview} onConfirmColumn={onConfirmColumn} onFillCodes={onFillCodes} onChange={onChange} + onCodesComplete={onCodesComplete} /> ))} @@ -76,7 +79,8 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote ) } -const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange }: { +const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange, + onCodesComplete }: { col: ColumnModel editing: boolean /** 这一列的表单。只有改了这一列时引用才会变，别的行不重画 */ @@ -89,6 +93,7 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble onConfirmColumn: (col: string) => void onFillCodes: (col: string) => void onChange: (col: string, field: ColumnField, value: string) => void + onCodesComplete: (col: string, value: boolean) => void }) { const { name, structure, items } = col const pending = COLUMN_FIELDS.filter((f) => items?.[f]?.status === 'proposed').length @@ -97,7 +102,7 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble const it = items?.[f] as CatalogItem | undefined const where = CT.whereColumn(name, CATALOG_COLUMN_FIELD_LABEL[f]) const problem = problems[`c:${name}:${f}`] - const changed = editing && (draft?.[f] ?? '') !== (initial?.[f] ?? '') + const changed = editing && ((draft?.[f] ?? '') !== (initial?.[f] ?? '') || (f === 'codes' && completeOf(draft) !== completeOf(initial))) return (
@@ -170,15 +175,23 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble {cell( 'codes', ( - } label={KT.fillLabel(name)} + } label={KT.fillLabel(name)} complete={codesComplete(codes)} onFill={codes?.status !== 'rejected' && !busy ? () => onFillCodes(name) : undefined} /> )} />, -