mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 03:41:58 +00:00
数据目录 F1-B2:剖析写的备注只是开头的日期变了、其余都没变时,保留旧的项、不升版本(人工确认过的关系也一样)。以前隔天重新剖析、数据没变,目录版本也加 1,发布时记下的目录版本就对不上,之后每次正式运行都误报目录有变化
This commit is contained in:
@@ -510,6 +510,30 @@ def _core(item: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return {k: v for k, v in item.items() if k != "updated_at"}
|
||||
|
||||
|
||||
#: 数据剖析写的备注都以「数据剖析(日期):」开头(data/catalog_profile.py 的 NOTE_PREFIX 就是它)
|
||||
PROFILE_NOTE_PREFIX = "数据剖析("
|
||||
_PROFILE_NOTE_DAY = re.compile(r"^数据剖析(\d{4}-\d{2}-\d{2})")
|
||||
|
||||
|
||||
def same_profile_note(a: Any, b: Any) -> bool:
|
||||
"""两段备注是否相同;数据剖析写的备注只有开头的日期不同,也算相同。
|
||||
|
||||
剖析每次都把当天的日期写进备注。隔天重跑、数据没变,结论一字不差、只有日期变了:要是这也算「变了」,目录就
|
||||
升一个版本,发布时记下的目录版本对不上,之后每次正式运行都提醒「目录有变化」。
|
||||
"""
|
||||
if a == b:
|
||||
return True
|
||||
if not (isinstance(a, str) and isinstance(b, str) and _PROFILE_NOTE_DAY.match(a) and _PROFILE_NOTE_DAY.match(b)):
|
||||
return False
|
||||
return _PROFILE_NOTE_DAY.sub("", a, count=1) == _PROFILE_NOTE_DAY.sub("", b, count=1)
|
||||
|
||||
|
||||
def _same_core(a: Mapping[str, Any], b: Mapping[str, Any]) -> bool:
|
||||
"""两项除修改时间外相同(剖析备注里的日期不算,same_profile_note)。"""
|
||||
ca, cb = _core(a), _core(b)
|
||||
return same_profile_note(ca.pop("note", None), cb.pop("note", None)) and ca == cb
|
||||
|
||||
|
||||
def visible_notes(notes: Mapping[str, Any] | None) -> dict[str, Any]:
|
||||
"""去掉被驳回的项和关系。给模型看、冻结进证据、将来的检查都只用这一份。"""
|
||||
return _assemble({k: v for k, v in _slots(notes).items() if v.get("status") != "rejected"})
|
||||
@@ -554,7 +578,7 @@ def merge_notes(existing: Mapping[str, Any] | None, draft: Mapping[str, Any] | N
|
||||
- 已有项是 confirmed 或 rejected:不动。驳回的留着,同一项下次起草就不会再冒出来。
|
||||
- 起草结果里没有这一项:已有项的来源在 covered 里(这一轮完整算过这个来源)就删掉,否则留着。
|
||||
外键约束被删掉以后,原来那条「有确证」的关系不能还挂着;而这一轮没调模型,就不能因此删掉模型起草的项。
|
||||
- 同一来源:值或状态有变化就更新。
|
||||
- 同一来源:值或状态有变化就更新。数据剖析写的备注只是日期变了不算变化(same_profile_note),保留旧的项。
|
||||
- 不同来源:来源更可信的(_SOURCE_RANK)顶掉不如它的,反过来不行。
|
||||
|
||||
新增和更新的项盖上 at(缺省为当前时间)作为 updated_at;内容没变的项保持原样,修改时间也不动。
|
||||
@@ -575,7 +599,7 @@ def merge_notes(existing: Mapping[str, Any] | None, draft: Mapping[str, Any] | N
|
||||
out[key] = item
|
||||
elif (cand.get("source") == item.get("source")
|
||||
or _SOURCE_RANK.get(cand.get("source"), 0) > _SOURCE_RANK.get(item.get("source"), 0)):
|
||||
if _core(cand) != _core(item):
|
||||
if not _same_core(cand, item):
|
||||
out[key] = {**_core(cand), "updated_at": at}
|
||||
updated += 1
|
||||
else:
|
||||
@@ -2107,7 +2131,8 @@ __all__ = [
|
||||
"CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict",
|
||||
"CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES",
|
||||
"DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL",
|
||||
"MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_VERIFY_COVERAGE", "PatchChange", "PatchPlan",
|
||||
"MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_NOTE_PREFIX", "PROFILE_VERIFY_COVERAGE",
|
||||
"PatchChange", "PatchPlan",
|
||||
"QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft",
|
||||
"TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", "apply_patch", "codes_complete",
|
||||
"describe_slot",
|
||||
|
||||
@@ -443,8 +443,9 @@ async def estimate_size(run: _Runner, dialect: SqlDialect, meta: dict[str, Any],
|
||||
PROFILE_DEFAULT_TABLES = 10
|
||||
#: IN 列表每批最多几个值:Oracle 的 IN 列表不能超过 1000 项(ORA-01795),各家统一按它分批
|
||||
IN_CHUNK = 1000
|
||||
#: 剖析写的备注都以它开头:人工确认过的关系,只覆盖剖析自己写的备注,不碰人写的
|
||||
NOTE_PREFIX = "数据剖析("
|
||||
#: 剖析写的备注都以它开头:人工确认过的关系,只覆盖剖析自己写的备注,不碰人写的。开头接着剖析的日期,
|
||||
#: 比较「结论变没变」时日期不算(catalog.same_profile_note)
|
||||
NOTE_PREFIX = catalog.PROFILE_NOTE_PREFIX
|
||||
#: 整次剖析停下的原因 → 剩下没查的项的说明
|
||||
_STOP_DETAIL = {
|
||||
"budget": "本次剖析的查询次数已用完(上限 {max_queries} 条),未检查",
|
||||
@@ -958,7 +959,7 @@ def _codes_finding(c: _CodesOutcome, value: dict[str, str]) -> dict[str, Any]:
|
||||
|
||||
def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool:
|
||||
"""人工确认过的关系只补覆盖率和基数:覆盖率是测量值,照新的写;基数只在原来没有时补,不改人定的;
|
||||
备注只在原来为空、或者是剖析自己写的时候换成新的。状态、来源不动。返回改没改。"""
|
||||
备注只在原来为空、或者是剖析自己写的时候换成新的(只是日期变了不换)。状态、来源不动。返回改没改。"""
|
||||
changed = False
|
||||
if rel.get("coverage") != o.coverage:
|
||||
rel["coverage"] = o.coverage
|
||||
@@ -967,7 +968,7 @@ def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool:
|
||||
rel["cardinality"] = o.cardinality
|
||||
changed = True
|
||||
note = rel.get("note")
|
||||
if (not note or str(note).startswith(NOTE_PREFIX)) and note != o.note:
|
||||
if (not note or str(note).startswith(NOTE_PREFIX)) and not catalog.same_profile_note(note, o.note):
|
||||
rel["note"] = o.note
|
||||
changed = True
|
||||
if changed:
|
||||
|
||||
@@ -342,3 +342,55 @@ async def test_profile_marks_codes_complete_only_when_the_whole_table_was_read(c
|
||||
assert "按前 10 行统计" in codes["note"] and "complete" not in codes
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# B2:隔天重新剖析、数据没变,不升版本
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
async def test_rerun_on_another_day_with_same_data_keeps_the_version(client, make_source, tmp_path, monkeypatch):
|
||||
path = scenic.build(tmp_path / "b2.db")
|
||||
sid = await make_source(path, options=_on(), draft=["visits"])
|
||||
# 人工确认过的关系:剖析只补覆盖率和基数、换自己的备注,同样不能因为日期变了就改
|
||||
detail = await _notes(client, sid, "visits")
|
||||
gate = _relation(detail["notes"], "gate_id")
|
||||
r = await client.post(f"/api/datasources/{sid}/catalog/visits/review",
|
||||
json={"path": f"relations.{gate['id']}", "action": "confirm", "if_version": detail["version"]})
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
monkeypatch.setattr(catalog, "now_iso", lambda: DAY1)
|
||||
first = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits")
|
||||
notes_day1 = (await _notes(client, sid, "visits"))["notes"]
|
||||
monkeypatch.setattr(catalog, "now_iso", lambda: DAY2)
|
||||
second = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits")
|
||||
assert (second["added"], second["updated"], second["removed"]) == (0, 0, 0)
|
||||
assert second["version"] == first["version"]
|
||||
assert (await _notes(client, sid, "visits"))["notes"] == notes_day1 # 备注里还是第一天的日期
|
||||
|
||||
# 数据变了(多了一种状态),结论跟着变:照常升版本
|
||||
db = sqlite3.connect(path)
|
||||
try:
|
||||
db.execute("INSERT INTO visits VALUES (1501, 'TK9001501', 1, 1, 1, NULL, '2026-09-30 10:00:00', 1, 2)")
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
monkeypatch.setattr(catalog, "now_iso", lambda: DAY3)
|
||||
third = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits")
|
||||
assert third["version"] == second["version"] + 1 and third["updated"] >= 1
|
||||
codes = (await _notes(client, sid, "visits"))["notes"]["columns"]["status"]["codes"]
|
||||
assert set(codes["value"]) == {"0", "1", "2"} and "2026-10-03" in codes["note"]
|
||||
|
||||
|
||||
def test_merge_ignores_a_profile_note_that_only_changed_its_date():
|
||||
old = {"columns": {"status": {"codes": catalog.make_item(
|
||||
{"1": "", "0": ""}, "profile", "proposed", note="数据剖析(2026-10-01):统计全表,1500 行非空值共 2 个取值。",
|
||||
at=DAY1)}}}
|
||||
same = {"columns": {"status": {"codes": catalog.make_item(
|
||||
{"1": "", "0": ""}, "profile", "proposed", note="数据剖析(2026-10-02):统计全表,1500 行非空值共 2 个取值。")}}}
|
||||
merged, stats = catalog.merge_notes(old, same, covered={"profile"}, at=DAY2)
|
||||
assert merged == old and stats == catalog.MergeStats()
|
||||
changed = {"columns": {"status": {"codes": catalog.make_item(
|
||||
{"1": "", "0": ""}, "profile", "proposed", note="数据剖析(2026-10-02):统计全表,1600 行非空值共 2 个取值。")}}}
|
||||
merged, stats = catalog.merge_notes(old, changed, covered={"profile"}, at=DAY2)
|
||||
assert stats.updated == 1 and "2026-10-02" in merged["columns"]["status"]["codes"]["note"]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user