mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 21:12:01 +00:00
ThearchyHelios
c95e5b4e84
码值出现新取值、比例剧变、业务日期断档、某天没数据，以前都看不到：确认过的码值列不再取分布，大表只看物理顺序最老的 前 10 万行，「已列全」过期以后 SQL 检查还照它提示。现在： - 剖析引擎加四类观察检查（data/catalog_profile_observe.py）：rows（行数，用剖析开头那条估行数的结果）、codes_observe （所有码值列，包括人工确认过、标了已列全的）、freshness（已确认或已验证的业务日期的最新日期）、daily_volume（最近 35 天按天计数）。计入每张表的预算、止损和留痕，每条观察记口径 basis 和它出自哪几条查询；遮罩列不进任何观察查询。 - 大表口径照目录-D7：有确认的业务日期且该列打头有索引看 35 天窗口；PG / Oracle 没有就块采样（比例按表的大小取一档， 不跨档比）；MySQL / SQLite 没有就按存储顺序的前 N 行，照实写口径。最新业务日期、按日行数在大表上只在有索引时才查。 SqlDialect 加 max_date_sql（有效区间里的最大日期），方言金样跟着加了两条。 - data/value_drift.py 纯函数比较器：新码值（已列全的任何新值、没列全的要占 0.5% 以上且有历史）、PSI 剧变（≥0.25 warning、0.1–0.25 info，两次都要 1,000 行）、断档（超过 max(2×中位间隔, 3 天)）、缺天（以前的窗口没有过）、量跌 （不到前 28 天中位数的 30%）、行数跌 30% / 涨 3 倍（估算值要连续两次）。只和同口径的历史比。 - 观察记进 table_observations（每张表每种观察留最近 26 条），比出的取值变化进 source_changes，和这张表的剖析结果、目录在 同一个事务里。机器唯一写进目录的事：数据证明码值表不再列全时撤销「已列全」（目录版本 +1，署名「取值观察」，值、含义、 状态都不动），SQL 检查随之不再按旧码值表提示 unknown_code。 - 剖析任务的范围加 observe（观察到期的有目录的表，只观察）和 maintenance（定时维护：需要剖析的 + 观察到期的）；人点的 「需要剖析 / 全部 / 指定的表」照旧不观察。只观察的那一项不算「上次剖析」（不顶掉表清单的上次剖析和给助手的剖析事实）。 任务的设置快照里记 trigger / fire_at / schedule_id（剖析任务表没有这几列）。 - 观察结论（最新业务日期、更新的行数）经 last_facts → schema_view.profile_lines 和剖析事实同一节进 look、db_schema、 /schema、表助手（X7），遮罩在渲染那一刻再核。 测试：test_value_drift（阈值两侧、口径不同不比）、test_catalog_observe（四类观察与口径、撤销已列全与 SQL 检查、去重、 遮罩不进查询、大表窗口与前 N 行、PG / Oracle 采样 SQL、观察不算上次剖析、进表详情）。
115 lines
6.5 KiB
Python
115 lines
6.5 KiB
Python
"""取值观察的比较器（app/data/value_drift.py，next-round 03 B2 ④）：纯函数。
|
||
|
||
守的是阈值两侧：已列全的码值表出现任何新值都算、要撤销「已列全」；没列全的新值要占 0.5% 以上、第一次只建基线；PSI 刚好落在
|
||
0.1 / 0.25 两侧；断档要超过 max(2 × 中位间隔, 3 天)；缺天要以前的窗口都没有过；量跌按前 28 天的中位数；行数估算要连续两次；
|
||
口径不同不比。取值都是虚构的。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
from datetime import date, datetime, timedelta, timezone
|
||
|
||
from app.data import value_drift as vd
|
||
|
||
AT = datetime(2026, 10, 10, 3, 0, tzinfo=timezone.utc)
|
||
TODAY = date(2026, 10, 10)
|
||
|
||
|
||
def _codes(values: dict[str, int], basis: str = "full", **extra) -> vd.Obs:
|
||
return vd.Obs("codes_observe", basis, {"values": [[k, v] for k, v in values.items()],
|
||
"rows": sum(values.values()), **extra}, AT)
|
||
|
||
|
||
def test_complete_code_table_flags_any_new_value_and_revokes_complete():
|
||
[f] = vd.codes(_codes({"paid": 900, "void": 99, "held": 1}), [], known=["paid", "void"], complete=True)
|
||
assert f.kind == "codes_new" and f.severity == "action" and f.revoke_complete is True
|
||
assert f.after["codes"] == ["held"], "只出现一次也算：它说的是「只有这些取值」"
|
||
assert vd.codes(_codes({"paid": 900, "void": 100}), [], known=["paid", "void"], complete=True) == []
|
||
|
||
|
||
def test_not_complete_needs_history_and_half_a_percent():
|
||
base = _codes({"paid": 900, "void": 100})
|
||
assert vd.codes(_codes({"paid": 900, "void": 95, "held": 5}), [], known=[]) == [], "第一次只建基线"
|
||
[f] = vd.codes(_codes({"paid": 900, "void": 95, "held": 5}), [base], known=[])
|
||
assert f.kind == "codes_new" and f.severity == "warning" and not f.revoke_complete
|
||
assert vd.codes(_codes({"paid": 996, "void": 100, "held": 4}), [base], known=[]) == [], "不到 0.5%：不算"
|
||
seen = _codes({"paid": 900, "void": 90, "held": 10})
|
||
assert [x.kind for x in vd.codes(_codes({"paid": 900, "void": 95, "held": 5}), [seen, base])] == [], "历史里见过的不算新"
|
||
|
||
|
||
def test_psi_on_both_sides_of_the_thresholds():
|
||
before = _codes({"a": 500, "b": 500})
|
||
# PSI(0.5/0.5 → 0.6/0.4) ≈ 0.0405：不报
|
||
assert vd.codes(_codes({"a": 600, "b": 400}), [before]) == []
|
||
# 0.5/0.5 → 0.66/0.34 ≈ 0.1035：info
|
||
[info] = vd.codes(_codes({"a": 660, "b": 340}), [before])
|
||
assert info.kind == "codes_shift" and info.severity == "info" and 0.1 <= info.after["psi"] < 0.25
|
||
# 0.5/0.5 → 0.75/0.25 ≈ 0.2747：warning，列出变化最大的取值
|
||
[warn] = vd.codes(_codes({"a": 750, "b": 250}), [before])
|
||
assert warn.severity == "warning" and warn.after["psi"] >= 0.25 and warn.after["top"][0][0] in ("a", "b")
|
||
small = _codes({"a": 50, "b": 50})
|
||
assert vd.codes(_codes({"a": 90, "b": 10}), [small]) == [], "两次都要至少 1,000 行"
|
||
|
||
|
||
def test_different_basis_is_not_compared():
|
||
before = _codes({"a": 500, "b": 500}, basis="first_rows:1000")
|
||
assert vd.codes(_codes({"a": 900, "b": 100}, basis="window:paid_on:35"), [before]) == []
|
||
same, note = vd.comparable(_codes({}, basis="window:paid_on:35"), [before])
|
||
assert same == [] and "口径不同" in note
|
||
|
||
|
||
def _fresh(day: str) -> vd.Obs:
|
||
return vd.Obs("freshness", "full", {"max": day}, AT)
|
||
|
||
|
||
def test_stalled_date_needs_more_than_twice_the_median_interval_and_three_days():
|
||
history = [_fresh("2026-10-05"), _fresh("2026-10-04"), _fresh("2026-10-03")] # 每天前进一次
|
||
assert vd.freshness(_fresh("2026-10-05"), history, today=date(2026, 10, 8)) == [], "3 天：没超过 max(2, 3)"
|
||
[f] = vd.freshness(_fresh("2026-10-05"), history, today=date(2026, 10, 9))
|
||
assert f.kind == "date_stalled" and f.after["age_days"] == 4 and f.after["threshold_days"] == 3
|
||
assert vd.freshness(_fresh("2026-10-08"), history, today=date(2026, 10, 30)) == [], "前进了就不算断档"
|
||
weekly = [_fresh("2026-09-28"), _fresh("2026-09-21")]
|
||
assert vd.freshness(_fresh("2026-09-28"), weekly, today=date(2026, 10, 10)) == [], "每周一次：14 天内不算"
|
||
assert vd.freshness(_fresh("2026-10-05"), [_fresh("2026-10-05")], today=date(2026, 10, 30)) == [], \
|
||
"没见过前进，不知道它多久前进一次"
|
||
|
||
|
||
def _days(start: date, counts: list[int], basis: str = "window:paid_on:35") -> vd.Obs:
|
||
return vd.Obs("daily_volume", basis, {"days": {(start + timedelta(days=i)).isoformat(): n
|
||
for i, n in enumerate(counts) if n}}, AT)
|
||
|
||
|
||
def test_gap_only_when_previous_windows_had_none():
|
||
full = _days(date(2026, 9, 1), [100] * 30)
|
||
holed = _days(date(2026, 9, 2), [100] * 10 + [0, 0] + [100] * 18)
|
||
[f] = vd.daily(holed, [full], today=TODAY)
|
||
assert f.kind == "date_gap" and f.after["gaps"] == 2
|
||
assert not [x for x in vd.daily(holed, [holed], today=TODAY) if x.kind == "date_gap"], "以前就有缺天：不算新问题"
|
||
assert not [x for x in vd.daily(holed, [], today=TODAY) if x.kind == "date_gap"], "没有历史：只建基线"
|
||
|
||
|
||
def test_volume_drop_against_the_previous_28_days():
|
||
start = TODAY - timedelta(days=31)
|
||
normal = _days(start, [100] * 30) # 最后一天是昨天
|
||
assert vd.daily(normal, [], today=TODAY) == []
|
||
[f] = vd.daily(_days(start, [100] * 29 + [20]), [], today=TODAY)
|
||
assert f.kind == "volume_drop" and f.after["rows"] == 20 and f.before["median"] == 100
|
||
assert vd.daily(_days(start, [100] * 29 + [35]), [], today=TODAY) == [], "35% 不算跌"
|
||
partial = _days(TODAY - timedelta(days=30), [100] * 30 + [5]) # 今天还在写入：往前退一天
|
||
assert vd.daily(partial, [], today=TODAY) == []
|
||
|
||
|
||
def _rows(n: int, method: str = "count") -> vd.Obs:
|
||
return vd.Obs("rows", method, {"rows": n, "method": method}, AT)
|
||
|
||
|
||
def test_rows_drop_and_jump_estimates_need_two_in_a_row():
|
||
assert [f.kind for f in vd.rows(_rows(600), [_rows(1000)])] == ["rows_drop"]
|
||
assert vd.rows(_rows(750), [_rows(1000)]) == [], "跌 25%：不报"
|
||
assert [f.kind for f in vd.rows(_rows(3000), [_rows(1000)])] == ["rows_jump"]
|
||
est = lambda n: _rows(n, "stats") # noqa: E731
|
||
assert vd.rows(est(600), [est(1000), est(1000)]) == [], "估算值跌了一次：先不报"
|
||
assert [f.kind for f in vd.rows(est(600), [est(650), est(1000)])] == ["rows_drop"], "连续两次都跌：报"
|
||
assert vd.rows(_rows(600), [_rows(1000, "stats")]) == [], "数出来的和估算的口径不同：不比"
|
||
assert vd.rows(vd.Obs("rows", "count", {"rows": None, "at_least": 1001}, AT), [_rows(5000)]) == [], \
|
||
"只知道「至少多少行」的不比"
|