diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index c6e2c8fe..cace09c0 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -510,6 +510,30 @@ def _core(item: Mapping[str, Any]) -> dict[str, Any]: return {k: v for k, v in item.items() if k != "updated_at"} +#: 数据剖析写的备注都以「数据剖析(日期):」开头(data/catalog_profile.py 的 NOTE_PREFIX 就是它) +PROFILE_NOTE_PREFIX = "数据剖析(" +_PROFILE_NOTE_DAY = re.compile(r"^数据剖析(\d{4}-\d{2}-\d{2})") + + +def same_profile_note(a: Any, b: Any) -> bool: + """两段备注是否相同;数据剖析写的备注只有开头的日期不同,也算相同。 + + 剖析每次都把当天的日期写进备注。隔天重跑、数据没变,结论一字不差、只有日期变了:要是这也算「变了」,目录就 + 升一个版本,发布时记下的目录版本对不上,之后每次正式运行都提醒「目录有变化」。 + """ + if a == b: + return True + if not (isinstance(a, str) and isinstance(b, str) and _PROFILE_NOTE_DAY.match(a) and _PROFILE_NOTE_DAY.match(b)): + return False + return _PROFILE_NOTE_DAY.sub("", a, count=1) == _PROFILE_NOTE_DAY.sub("", b, count=1) + + +def _same_core(a: Mapping[str, Any], b: Mapping[str, Any]) -> bool: + """两项除修改时间外相同(剖析备注里的日期不算,same_profile_note)。""" + ca, cb = _core(a), _core(b) + return same_profile_note(ca.pop("note", None), cb.pop("note", None)) and ca == cb + + def visible_notes(notes: Mapping[str, Any] | None) -> dict[str, Any]: """去掉被驳回的项和关系。给模型看、冻结进证据、将来的检查都只用这一份。""" return _assemble({k: v for k, v in _slots(notes).items() if v.get("status") != "rejected"}) @@ -554,7 +578,7 @@ def merge_notes(existing: Mapping[str, Any] | None, draft: Mapping[str, Any] | N - 已有项是 confirmed 或 rejected:不动。驳回的留着,同一项下次起草就不会再冒出来。 - 起草结果里没有这一项:已有项的来源在 covered 里(这一轮完整算过这个来源)就删掉,否则留着。 外键约束被删掉以后,原来那条「有确证」的关系不能还挂着;而这一轮没调模型,就不能因此删掉模型起草的项。 - - 同一来源:值或状态有变化就更新。 + - 同一来源:值或状态有变化就更新。数据剖析写的备注只是日期变了不算变化(same_profile_note),保留旧的项。 - 不同来源:来源更可信的(_SOURCE_RANK)顶掉不如它的,反过来不行。 新增和更新的项盖上 at(缺省为当前时间)作为 updated_at;内容没变的项保持原样,修改时间也不动。 @@ -575,7 +599,7 @@ def merge_notes(existing: Mapping[str, Any] | None, draft: Mapping[str, Any] | N out[key] = item elif (cand.get("source") == item.get("source") or _SOURCE_RANK.get(cand.get("source"), 0) > _SOURCE_RANK.get(item.get("source"), 0)): - if _core(cand) != _core(item): + if not _same_core(cand, item): out[key] = {**_core(cand), "updated_at": at} updated += 1 else: @@ -2107,7 +2131,8 @@ __all__ = [ "CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict", "CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES", "DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL", - "MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_VERIFY_COVERAGE", "PatchChange", "PatchPlan", + "MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_NOTE_PREFIX", "PROFILE_VERIFY_COVERAGE", + "PatchChange", "PatchPlan", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft", "TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "codes_complete", "describe_slot", diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 7b194268..d76e3ea1 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -443,8 +443,9 @@ async def estimate_size(run: _Runner, dialect: SqlDialect, meta: dict[str, Any], PROFILE_DEFAULT_TABLES = 10 #: IN 列表每批最多几个值:Oracle 的 IN 列表不能超过 1000 项(ORA-01795),各家统一按它分批 IN_CHUNK = 1000 -#: 剖析写的备注都以它开头:人工确认过的关系,只覆盖剖析自己写的备注,不碰人写的 -NOTE_PREFIX = "数据剖析(" +#: 剖析写的备注都以它开头:人工确认过的关系,只覆盖剖析自己写的备注,不碰人写的。开头接着剖析的日期, +#: 比较「结论变没变」时日期不算(catalog.same_profile_note) +NOTE_PREFIX = catalog.PROFILE_NOTE_PREFIX #: 整次剖析停下的原因 → 剩下没查的项的说明 _STOP_DETAIL = { "budget": "本次剖析的查询次数已用完(上限 {max_queries} 条),未检查", @@ -958,7 +959,7 @@ def _codes_finding(c: _CodesOutcome, value: dict[str, str]) -> dict[str, Any]: def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool: """人工确认过的关系只补覆盖率和基数:覆盖率是测量值,照新的写;基数只在原来没有时补,不改人定的; - 备注只在原来为空、或者是剖析自己写的时候换成新的。状态、来源不动。返回改没改。""" + 备注只在原来为空、或者是剖析自己写的时候换成新的(只是日期变了不换)。状态、来源不动。返回改没改。""" changed = False if rel.get("coverage") != o.coverage: rel["coverage"] = o.coverage @@ -967,7 +968,7 @@ def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool: rel["cardinality"] = o.cardinality changed = True note = rel.get("note") - if (not note or str(note).startswith(NOTE_PREFIX)) and note != o.note: + if (not note or str(note).startswith(NOTE_PREFIX)) and not catalog.same_profile_note(note, o.note): rel["note"] = o.note changed = True if changed: diff --git a/backend/tests/test_catalog_semantics.py b/backend/tests/test_catalog_semantics.py index 1f2fa07d..cf0b38d6 100644 --- a/backend/tests/test_catalog_semantics.py +++ b/backend/tests/test_catalog_semantics.py @@ -342,3 +342,55 @@ async def test_profile_marks_codes_complete_only_when_the_whole_table_was_read(c assert "按前 10 行统计" in codes["note"] and "complete" not in codes +# ========================================================================== +# B2:隔天重新剖析、数据没变,不升版本 +# ========================================================================== + + +async def test_rerun_on_another_day_with_same_data_keeps_the_version(client, make_source, tmp_path, monkeypatch): + path = scenic.build(tmp_path / "b2.db") + sid = await make_source(path, options=_on(), draft=["visits"]) + # 人工确认过的关系:剖析只补覆盖率和基数、换自己的备注,同样不能因为日期变了就改 + detail = await _notes(client, sid, "visits") + gate = _relation(detail["notes"], "gate_id") + r = await client.post(f"/api/datasources/{sid}/catalog/visits/review", + json={"path": f"relations.{gate['id']}", "action": "confirm", "if_version": detail["version"]}) + assert r.status_code == 200, r.text + + monkeypatch.setattr(catalog, "now_iso", lambda: DAY1) + first = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits") + notes_day1 = (await _notes(client, sid, "visits"))["notes"] + monkeypatch.setattr(catalog, "now_iso", lambda: DAY2) + second = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits") + assert (second["added"], second["updated"], second["removed"]) == (0, 0, 0) + assert second["version"] == first["version"] + assert (await _notes(client, sid, "visits"))["notes"] == notes_day1 # 备注里还是第一天的日期 + + # 数据变了(多了一种状态),结论跟着变:照常升版本 + db = sqlite3.connect(path) + try: + db.execute("INSERT INTO visits VALUES (1501, 'TK9001501', 1, 1, 1, NULL, '2026-09-30 10:00:00', 1, 2)") + db.commit() + finally: + db.close() + monkeypatch.setattr(catalog, "now_iso", lambda: DAY3) + third = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits") + assert third["version"] == second["version"] + 1 and third["updated"] >= 1 + codes = (await _notes(client, sid, "visits"))["notes"]["columns"]["status"]["codes"] + assert set(codes["value"]) == {"0", "1", "2"} and "2026-10-03" in codes["note"] + + +def test_merge_ignores_a_profile_note_that_only_changed_its_date(): + old = {"columns": {"status": {"codes": catalog.make_item( + {"1": "", "0": ""}, "profile", "proposed", note="数据剖析(2026-10-01):统计全表,1500 行非空值共 2 个取值。", + at=DAY1)}}} + same = {"columns": {"status": {"codes": catalog.make_item( + {"1": "", "0": ""}, "profile", "proposed", note="数据剖析(2026-10-02):统计全表,1500 行非空值共 2 个取值。")}}} + merged, stats = catalog.merge_notes(old, same, covered={"profile"}, at=DAY2) + assert merged == old and stats == catalog.MergeStats() + changed = {"columns": {"status": {"codes": catalog.make_item( + {"1": "", "0": ""}, "profile", "proposed", note="数据剖析(2026-10-02):统计全表,1600 行非空值共 2 个取值。")}}} + merged, stats = catalog.merge_notes(old, changed, covered={"profile"}, at=DAY2) + assert stats.updated == 1 and "2026-10-02" in merged["columns"]["status"]["codes"]["note"] + +