From 661f6a38d76b02df1ef5ee0e01de695d2fa72c7f Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Sun, 4 Oct 2026 01:59:10 +0800 Subject: [PATCH] =?UTF-8?q?A7=20=E5=89=96=E6=9E=90=E5=86=99=E8=BF=9B?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E7=9A=84=E8=AF=B4=E6=98=8E=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E5=86=99=E6=97=A5=E6=9C=9F=EF=BC=9A=E4=BB=A5=E5=89=8D=E5=86=99?= =?UTF-8?q?=E7=9A=84=E6=98=AF=20UTC=20=E6=97=A5=E6=9C=9F=EF=BC=88at[:10]?= =?UTF-8?q?=EF=BC=89=EF=BC=8C=E5=92=8C=E7=95=8C=E9=9D=A2=E4=B8=8A=E6=8C=89?= =?UTF-8?q?=E6=9C=AC=E5=9C=B0=E6=97=B6=E9=97=B4=E6=98=BE=E7=A4=BA=E7=9A=84?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=97=B6=E9=97=B4=E5=AF=B9=E4=B8=8D=E4=B8=8A?= =?UTF-8?q?=EF=BC=9B=E6=94=B9=E4=B8=BA=E3=80=8C=E6=95=B0=E6=8D=AE=E5=89=96?= =?UTF-8?q?=E6=9E=90=EF=BC=9A=E2=80=A6=E3=80=8D=EF=BC=8C=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E4=BB=A5=E8=BF=99=E4=B8=80=E9=A1=B9=E7=9A=84=20updated=5Fat=20?= =?UTF-8?q?=E4=B8=BA=E5=87=86=E3=80=82is=5Fprofile=5Fnote=20=E5=90=8C?= =?UTF-8?q?=E6=97=B6=E8=AE=A4=E5=8D=87=E7=BA=A7=E5=89=8D=E5=B8=A6=E6=97=A5?= =?UTF-8?q?=E6=9C=9F=E7=9A=84=E3=80=8C=E6=95=B0=E6=8D=AE=E5=89=96=E6=9E=90?= =?UTF-8?q?=EF=BC=88=E6=97=A5=E6=9C=9F=EF=BC=89=EF=BC=9A=E3=80=8D=EF=BC=9B?= =?UTF-8?q?same=5Fprofile=5Fnote=20=E7=BB=A7=E7=BB=AD=E5=85=BC=E5=AE=B9?= =?UTF-8?q?=E6=97=A7=E6=95=B0=E6=8D=AE=EF=BC=9A=E5=8F=AA=E5=B7=AE=E6=97=A5?= =?UTF-8?q?=E6=9C=9F=E3=80=81=E6=88=96=E4=B8=80=E6=AE=B5=E5=B8=A6=E6=97=A5?= =?UTF-8?q?=E6=9C=9F=E4=B8=80=E6=AE=B5=E4=B8=8D=E5=B8=A6=E7=9A=84=E5=90=8C?= =?UTF-8?q?=E4=B8=80=E7=BB=93=E8=AE=BA=E9=83=BD=E7=AE=97=E7=9B=B8=E5=90=8C?= =?UTF-8?q?=EF=BC=8C=E5=8D=87=E7=BA=A7=E5=90=8E=E9=87=8D=E6=96=B0=E5=89=96?= =?UTF-8?q?=E6=9E=90=E4=B8=8D=E4=BC=9A=E7=BB=99=E6=AF=8F=E5=BC=A0=E8=A1=A8?= =?UTF-8?q?=E5=8D=87=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog.py | 32 ++++++++++++++++++------- backend/app/data/catalog_profile.py | 24 +++++++++---------- backend/tests/test_catalog_profile.py | 4 ++++ backend/tests/test_catalog_semantics.py | 22 ++++++++++++++++- 4 files changed, 59 insertions(+), 23 deletions(-) diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index a7d559e9..e4d821e6 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -518,22 +518,35 @@ def _core(item: Mapping[str, Any]) -> dict[str, Any]: return {k: v for k, v in item.items() if k != "updated_at"} -#: 数据剖析写的备注都以「数据剖析（日期）：」开头（data/catalog_profile.py 的 NOTE_PREFIX 就是它） -PROFILE_NOTE_PREFIX = "数据剖析（" -_PROFILE_NOTE_DAY = re.compile(r"^数据剖析（\d{4}-\d{2}-\d{2}）") +#: 数据剖析写的备注都以「数据剖析：」开头（data/catalog_profile.py 的 NOTE_PREFIX 就是它）。说明里不写日期：以前写的是 +#: UTC 日期，和界面上按本地时间显示的修改时间对不上；什么时候剖析的，看这一项的 updated_at +PROFILE_NOTE_PREFIX = "数据剖析：" +#: 升级前的剖析备注带着当天的 UTC 日期：「数据剖析（2026-10-01）：」。旧数据里还有，照样认 +_PROFILE_NOTE_DAY = re.compile(r"^数据剖析（\d{4}-\d{2}-\d{2}）(?=：)") + + +def is_profile_note(note: Any) -> bool: + """这段备注是不是数据剖析写的（新写法「数据剖析：」，或者升级前带日期的「数据剖析（日期）：」）。""" + return isinstance(note, str) and (note.startswith(PROFILE_NOTE_PREFIX) or bool(_PROFILE_NOTE_DAY.match(note))) + + +def _undated(note: str) -> str: + """剖析备注去掉升级前写的日期：「数据剖析（2026-10-01）：…」→「数据剖析：…」。别的备注原样返回。""" + return _PROFILE_NOTE_DAY.sub("数据剖析", note, count=1) def same_profile_note(a: Any, b: Any) -> bool: - """两段备注是否相同；数据剖析写的备注只有开头的日期不同，也算相同。 + """两段备注是否相同；数据剖析写的备注只差开头的日期（升级前每次都写当天的日期），或者一段带日期、一段是不带 + 日期的新写法，也算相同。 - 剖析每次都把当天的日期写进备注。隔天重跑、数据没变，结论一字不差、只有日期变了：要是这也算「变了」，目录就 - 升一个版本，发布时记下的目录版本对不上，之后每次正式运行都提醒「目录有变化」。 + 隔天重跑、数据没变，结论一字不差、只有日期变了：要是这也算「变了」，目录就升一个版本，发布时记下的目录版本 + 对不上，之后每次正式运行都提醒「目录有变化」。升级后第一次重新剖析也是一样：结论没变，只是新写法不带日期。 """ if a == b: return True - if not (isinstance(a, str) and isinstance(b, str) and _PROFILE_NOTE_DAY.match(a) and _PROFILE_NOTE_DAY.match(b)): + if not (is_profile_note(a) and is_profile_note(b)): return False - return _PROFILE_NOTE_DAY.sub("", a, count=1) == _PROFILE_NOTE_DAY.sub("", b, count=1) + return _undated(a) == _undated(b) def _same_core(a: Mapping[str, Any], b: Mapping[str, Any]) -> bool: @@ -2239,7 +2252,8 @@ __all__ = [ "PatchChange", "PatchPlan", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft", "TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "codes_complete", "describe_slot", "draft_catalog", "draft_structure", "draft_with_model", - "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "is_date_column", "join_paths", + "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "is_date_column", "is_profile_note", + "join_paths", "make_item", "merge_notes", "name_words", "now_iso", "parse_path", "plan_patch", "profile_relation_holds", "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", "render_table_index", "render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry", "review_item", "same_notes", diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 9d7b14fd..fad61cbe 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -444,8 +444,8 @@ async def estimate_size(run: _Runner, dialect: SqlDialect, meta: dict[str, Any], PROFILE_DEFAULT_TABLES = 10 #: IN 列表每批最多几个值：Oracle 的 IN 列表不能超过 1000 项（ORA-01795），各家统一按它分批 IN_CHUNK = 1000 -#: 剖析写的备注都以它开头：人工确认过的关系，只覆盖剖析自己写的备注，不碰人写的。开头接着剖析的日期， -#: 比较「结论变没变」时日期不算（catalog.same_profile_note） +#: 剖析写的备注都以它开头（「数据剖析：」）：人工确认过的关系，只覆盖剖析自己写的备注，不碰人写的（认的时候连 +#: 升级前带日期的「数据剖析（日期）：」一起认，catalog.is_profile_note）。不写日期，时间看这一项的 updated_at NOTE_PREFIX = catalog.PROFILE_NOTE_PREFIX #: 整次剖析停下的原因 → 剩下没查的项的说明 _STOP_DETAIL = { @@ -575,8 +575,6 @@ class _Context: tables: dict[str, Any] #: 遮罩的列（小写）。遮罩按列名生效、不分大小写、不分表，和证据面板的比法一致 masked: set[str] - #: 写进备注的剖析日期（UTC） - day: str sizes: dict[str, TableSize] = field(default_factory=dict) async def size(self, table: str, result: TableProfile) -> TableSize: @@ -666,10 +664,11 @@ async def _unique(cx: _Context, table: str, meta: dict[str, Any], column: str, return total == distinct, "count" -def _relation_note(day: str, *, sample: int, matched: int, coverage: float, scan_cap: int | None, +def _relation_note(*, sample: int, matched: int, coverage: float, scan_cap: int | None, skipped_values: int, parent: tuple[bool | None, str], child: tuple[bool | None, str] | None, holds: bool) -> str: - """关系的剖析备注：日期、抽样规模、覆盖率、父键和子键唯一的依据。人据此判断这条结论有多可靠。""" + """关系的剖析备注：抽样规模、覆盖率、父键和子键唯一的依据。人据此判断这条结论有多可靠。 + 不写日期：什么时候剖析的看这一项的 updated_at（界面按本地时间显示）。""" parts: list[str] = [] low = coverage < catalog.PROFILE_VERIFY_COVERAGE if low: @@ -701,7 +700,7 @@ def _relation_note(day: str, *, sample: int, matched: int, coverage: float, scan parts.append("未能核对子表一侧是否唯一，按多对一记") if not holds and not low: parts.append("暂不升为有确证") - return f"{NOTE_PREFIX}{day}）：" + "；".join(parts) + "。" + return NOTE_PREFIX + "；".join(parts) + "。" async def _check_relation(cx: _Context, table: str, rel: dict[str, Any], out: _TableOutcome, @@ -752,7 +751,7 @@ async def _check_relation(cx: _Context, table: str, rel: dict[str, Any], out: _T if child_unique is not None: cardinality = "one_to_one" if child_unique[0] is True else "many_to_one" holds = coverage >= catalog.PROFILE_VERIFY_COVERAGE and cardinality is not None - note = _relation_note(cx.day, sample=len(literals), matched=matched, coverage=coverage, scan_cap=scan_cap, + note = _relation_note(sample=len(literals), matched=matched, coverage=coverage, scan_cap=scan_cap, skipped_values=len(values) - len(literals), parent=parent_unique, child=child_unique, holds=holds) checked = child_unique is not None and child_unique[0] is not None @@ -873,7 +872,7 @@ async def _check_codes(cx: _Context, table: str, column: str, out: _TableOutcome rows = sum(n for _, n in values) scope = f"按前 {scan_cap} 行统计" if scan_cap else "统计全表" spread = "、".join(f"{k}（{_share(n, rows)}）" for k, n in values) - note = f"{NOTE_PREFIX}{cx.day}）：{scope}，{rows} 行非空值共 {len(values)} 个取值：{spread}。" + note = f"{NOTE_PREFIX}{scope}，{rows} 行非空值共 {len(values)} 个取值：{spread}。" out.codes.append(_CodesOutcome(column=column, values=values, rows=rows, note=note, complete=scan_cap is None)) @@ -903,7 +902,7 @@ async def _check_dates(cx: _Context, table: str, columns: list[str], total: int, if not result.date_ranges: raise _Skip("no_data", f"{table}.{columns[0]} 没有可识别的日期值") found = result.date_ranges[0] - note = (f"{NOTE_PREFIX}{cx.day}）：表里只有这一个日期类列，取值从 {str(found['min'])[:19]} 到 " + note = (f"{NOTE_PREFIX}表里只有这一个日期类列，取值从 {str(found['min'])[:19]} 到 " f"{str(found['max'])[:19]}，可作为业务日期的参考。") out.business_date = _DateOutcome(column=found["column"], low=found["min"], high=found["max"], note=note) @@ -961,7 +960,7 @@ def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool: rel["cardinality_checked"] = True changed = True note = rel.get("note") - if (not note or str(note).startswith(NOTE_PREFIX)) and not catalog.same_profile_note(note, o.note): + if (not note or catalog.is_profile_note(note)) and not catalog.same_profile_note(note, o.note): rel["note"] = o.note changed = True if changed: @@ -1199,8 +1198,7 @@ async def _profile(session: AsyncSession, row: Any, source: Any, tables: list[st run = _Runner(source, settings, dialect) cx = _Context(source=source, settings=settings, dialect=dialect, run=run, tables=(getattr(source, "schema_cache", None) or {}).get("tables") or {}, - masked={c.lower() for c in data_engine.masked_columns(getattr(row, "options", None))}, - day=at[:10]) + masked={c.lower() for c in data_engine.masked_columns(getattr(row, "options", None))}) entries = await catalog.read_catalog(session, source.id) for table in await _pick_tables(session, source, tables, entries, report): result = TableProfile(table=table) diff --git a/backend/tests/test_catalog_profile.py b/backend/tests/test_catalog_profile.py index aae554d7..fdc605bc 100644 --- a/backend/tests/test_catalog_profile.py +++ b/backend/tests/test_catalog_profile.py @@ -10,6 +10,7 @@ from __future__ import annotations import asyncio +import re import sqlite3 import uuid from pathlib import Path @@ -196,6 +197,9 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d assert "数据剖析" in rel["note"] and "抽样" in rel["note"] # 子表一侧数过 COUNT 和 COUNT(DISTINCT)、确有重复：基数是用数据核实的（A6，一对多关联的检查据此给 error） assert rel["cardinality_checked"] is True + # A7：说明里不写日期，时间看这一项的 updated_at（界面按本地时间显示） + assert rel["note"].startswith("数据剖析：") and not re.search(r"\d{4}-\d{2}-\d{2}", rel["note"]) + assert rel["updated_at"] # 外键约束来的关系不动：外键是外键，剖析是剖析 park = _relation(notes, "park_id") assert (park["source"], park["status"], park["coverage"]) == ("fk", "verified", None) diff --git a/backend/tests/test_catalog_semantics.py b/backend/tests/test_catalog_semantics.py index 3c5d8416..d97a8c1d 100644 --- a/backend/tests/test_catalog_semantics.py +++ b/backend/tests/test_catalog_semantics.py @@ -407,7 +407,9 @@ async def test_rerun_on_another_day_with_same_data_keeps_the_version(client, mak third = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits") assert third["version"] == second["version"] + 1 and third["updated"] >= 1 codes = (await _notes(client, sid, "visits"))["notes"]["columns"]["status"]["codes"] - assert set(codes["value"]) == {"0", "1", "2"} and "2026-10-03" in codes["note"] + assert set(codes["value"]) == {"0", "1", "2"} and codes["updated_at"] == DAY3 + # A7：说明里不写日期（以前写的是 UTC 日期，和界面上的本地时间对不上），时间以这一项的 updated_at 为准 + assert codes["note"].startswith("数据剖析：") and "2026-10-03" not in codes["note"] def test_merge_ignores_a_profile_note_that_only_changed_its_date(): @@ -424,6 +426,24 @@ def test_merge_ignores_a_profile_note_that_only_changed_its_date(): assert stats.updated == 1 and "2026-10-02" in merged["columns"]["status"]["codes"]["note"] +def test_profile_notes_without_date_stay_compatible_with_dated_ones(): + """A7：剖析写的说明不再带日期（「数据剖析：…」）。升级前写的「数据剖析（2026-10-01）：…」照样认得是剖析写的； + 结论一样、只是新的不带日期，不算变化——升级后第一次重新剖析不能因此给每张表升一个版本。""" + dated = "数据剖析（2026-10-01）：统计全表，1500 行非空值共 2 个取值。" + plain = "数据剖析：统计全表，1500 行非空值共 2 个取值。" + assert catalog.is_profile_note(dated) and catalog.is_profile_note(plain) + assert not catalog.is_profile_note("人工备注：数据剖析（2026-10-01）") and not catalog.is_profile_note(None) + assert catalog.same_profile_note(dated, plain) and catalog.same_profile_note(plain, dated) + assert catalog.same_profile_note("数据剖析（2026-10-02）：统计全表，1500 行非空值共 2 个取值。", dated) + assert not catalog.same_profile_note(dated, "数据剖析：统计全表，1600 行非空值共 2 个取值。") + assert not catalog.same_profile_note(plain, "统计全表，1500 行非空值共 2 个取值。") + old = {"columns": {"status": {"codes": catalog.make_item({"1": "", "0": ""}, "profile", "proposed", note=dated, + at=DAY1)}}} + new = {"columns": {"status": {"codes": catalog.make_item({"1": "", "0": ""}, "profile", "proposed", note=plain)}}} + merged, stats = catalog.merge_notes(old, new, covered={"profile"}, at=DAY2) + assert merged == old and stats == catalog.MergeStats() + + # ========================================================================== # B3：公式在人工编辑、模型起草两条路径上也拦下 # ==========================================================================