Files
AgentLab/backend/tests/test_catalog_profile_units.py
T
ThearchyHelios 7f949cce92 取值观察:业务日期有未来值时不再误报量跌、缺天;「今天」统一按服务端所在时区
验收复现:业务日期最晚到明天、今天只写了几行,记出一条 volume_drop。原因有两层:
- 比较器 value_drift.daily() 取 last = max(days),只在 last >= today 时退一天;max 落在明天时退到的是今天这个
  还没写完的日子。改成最近一个整天 = min(有数据的最后一天, 昨天)。
- 按日行数的查询 day_volume_sql 只有下界,未来的日子也进了窗口。加上界:数到观察当天(含),第二天起不算
  (< 第二天,时间列上当天的时分秒照样算进来,文字存的日期按字符串比同样成立)。签名改成必须传 until。

缺天同样的问题也修了:今天以后的空日子不算缺天;以前那次观察只看到它自己那天(data.until)为止,免得它
窗口里未来的空日子被当成「以前就有过缺天」,挡住这一次真缺的天。断档遇到未来值时离今天是负数,只会报不出、
不会误报,观察本身照实记最新业务日期(目录里也照实给人看),不截未来值,写进了模块说明。

「今天」:原来窗口用 UTC 日期、比较器用 now.date()(也是 UTC),业务日期是本地日期,东八区凌晨 0–8 点会差
一天。统一成服务端所在时区(catalog_profile_observe.local_today,和待办汇总 notify.digest_at 同一个口径),
写进模块说明。

测试:test_value_drift 两条(未来值不算最近整天、未来空日子不算缺天且历史按 until 截);test_catalog_observe
两条(SQLite 真跑剖析任务,明天的行不进窗口、不记量跌;local_today 按 TZ 取本地日期)。去掉修复回跑都挂。
金样(tests/fixtures/dialect/*.sql)的 day_volume 一行随上界更新;scripts/dialect_it.py 加了一条「只数到观察
当天」的真库核对。
2026-10-17 07:59:19 +08:00

230 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""数据剖析(阶段 2A)里不连库的部分:设置的解析与校验、SQL 字面量、各方言的剖析 SQL 都过得了守卫、
目录对剖析结果的容纳(待填含义的码值、撤销审阅回到剖析的结论)。"""
from __future__ import annotations
from types import SimpleNamespace
import pytest
from app.data import catalog, catalog_profile, guard
from app.data.catalog import make_item
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings, profile_settings, profile_settings_problem
from app.data.engine import build_url
AT = "2026-10-03T08:00:00+00:00"
# ---------------------------------------------------------------- 设置
def test_settings_default_off_and_parse_leniently():
s = profile_settings({})
assert s == ProfileSettings()
assert s.enabled is False and s.table_max_queries == 40 and s.table_max_s == 60 and s.query_timeout_s == 10
assert s.sample_size == 2000 and s.max_scan_rows == 100_000
on = profile_settings({PROFILE_OPTION: {"enabled": True, "table_max_queries": "30", "sample_size": 500}})
assert on.enabled is True and on.table_max_queries == 30 and on.sample_size == 500
# 库里存着的值不对(老数据、手改过):那一项用缺省,不因此开启,也不报错;迁移前的旧键不看
odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "table_max_queries": 0, "query_timeout_s": "x",
"max_queries": 5}})
assert odd.enabled is False and odd.table_max_queries == 40 and odd.query_timeout_s == 10
assert profile_settings({PROFILE_OPTION: "on"}) == ProfileSettings()
# 任务里存着的快照读回来是同一份设置
assert ProfileSettings.from_dict(on.to_dict()) == on and ProfileSettings.from_dict(None) == ProfileSettings()
@pytest.mark.parametrize("value, label", [
({"table_max_queries": 0}, "每张表最多查询数"),
({"table_max_queries": 201}, "每张表最多查询数"),
({"table_max_queries": 2.5}, "每张表最多查询数"),
({"table_max_s": 9}, "每张表时长上限"),
({"table_max_s": 601}, "每张表时长上限"),
({"query_timeout_s": 61}, "单条查询时限"),
({"sample_size": 5}, "核对键值数"),
({"max_scan_rows": -1}, "整表统计行数上限"),
({"enabled": "true"}, "开启数据剖析"),
({"table_max_queries": True}, "每张表最多查询数"),
])
def test_settings_problem_names_the_field_in_chinese(value, label):
problem = profile_settings_problem({PROFILE_OPTION: value})
assert problem and label in problem and "数据剖析" in problem
def test_settings_problem_rejects_unknown_keys_and_wrong_shape():
assert "无法识别" in profile_settings_problem({PROFILE_OPTION: {"enabled": True, "rows": 1}})
assert profile_settings_problem({PROFILE_OPTION: ["enabled"]})
assert profile_settings_problem({}) is None
assert profile_settings_problem({PROFILE_OPTION: None}) is None
assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "table_max_queries": 200, "table_max_s": 10.5,
"query_timeout_s": 1.5, "sample_size": 10,
"max_scan_rows": 0}}) is None
def test_legacy_whole_run_budgets_are_stripped_not_rejected():
"""以前按整次剖析算的两项(查询次数上限、总时长上限):保存时只摘掉,不报错;不折算成每张表的预算。"""
old = {"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True, "max_queries": 60, "max_total_s": 120,
"sample_size": 500}}
cleaned = catalog_profile.without_legacy(old)
assert cleaned == {"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True, "sample_size": 500}}
assert old[PROFILE_OPTION]["max_queries"] == 60, "不改调用方的对象"
assert profile_settings_problem(cleaned) is None
plain = {PROFILE_OPTION: {"enabled": True}}
assert catalog_profile.without_legacy(plain) is plain and catalog_profile.without_legacy(None) is None
def test_profile_option_never_reaches_the_connection_url():
"""剖析设置是 AgentLab 自己的配置:拼进连接串,驱动会把它当未知参数拒掉。"""
source = SimpleNamespace(kind="mysql", host="db.example", port=3306, database="shop", username="ro",
password=None, options={"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True}})
url = build_url(source)
assert "charset=utf8mb4" in url and PROFILE_OPTION not in url and "enabled" not in url
# ---------------------------------------------------------------- 字面量
@pytest.mark.parametrize("value, kind, expected", [
(42, "number", "42"),
(-7, None, "-7"),
(3.0, "number", "3"),
("12345678901234567890", "number", "12345678901234567890"), # DECIMAL 经查询层转成了字符串
("A-01", "text", "'A-01'"),
("O'Neil", "text", "'O''Neil'"),
("12:30", "text", "'12:30'"),
])
def test_sql_literal_quotes_safe_values(value, kind, expected):
assert catalog_profile.sql_literal(value, kind) == expected
@pytest.mark.parametrize("value, kind", [
(True, "boolean"), # 布尔不当键
(2.5, "number"), # 非整数的数不当键:浮点相等比较靠不住
("12.50", "number"),
("1e3", "number"),
("2026-07-01", "date"), # 日期的字面量各家写法不同,不猜
("a\\b", "text"), # 反斜杠:MySQL 不同设置下读法不同,守卫会拒
("a :b", "text"), # 「:名字」会被查询层当成绑定参数
(":b", "text"),
("a\nb", "text"), # 控制字符
("x" * 201, "text"),
(None, "text"),
([1], None),
])
def test_sql_literal_refuses_unsafe_or_unsupported_values(value, kind):
assert catalog_profile.sql_literal(value, kind) is None
# ---------------------------------------------------------------- 各方言的剖析 SQL
_META = {"visits": {"schema": None, "columns": [{"name": "memberTagId", "type": "INTEGER"}], "primary_key": ["id"]},
"orders": {"schema": "sales", "columns": [{"name": "status", "type": "INTEGER"}], "primary_key": ["id"]}}
@pytest.mark.parametrize("kind", ["mysql", "mariadb", "postgresql", "oracle", "sqlite"])
def test_profile_sql_passes_readonly_guard_on_every_dialect(kind):
"""剖析的每一种查询(含读统计信息的系统视图)都得过只读守卫——守卫不为剖析放宽。"""
d = catalog_profile.SqlDialect(kind)
sqls = []
for table in ("visits", "orders"):
stats = d.stats_sql(_META[table], table)
if kind == "sqlite":
assert stats is None # SQLite 没有可读的统计信息,改为数到上限
else:
assert stats
sqls.append(stats)
tbl = d.table(_META[table], table)
col = d.quote(_META[table]["columns"][0]["name"])
sqls += [
d.bounded_count_sql(tbl, 100_001),
d.distinct_sample_sql(tbl, col, 2000, scan_cap=None),
d.distinct_sample_sql(tbl, col, 2000, scan_cap=100_000),
d.match_count_sql(tbl, col, ["1", "'A'"]),
d.unique_check_sql(tbl, col),
d.value_counts_sql(tbl, col, 21, scan_cap=None),
d.value_counts_sql(tbl, col, 21, scan_cap=100_000),
d.min_max_sql(tbl, [(col, False)]),
d.min_max_sql(tbl, [(col, True)]),
d.day_volume_sql(tbl, col, "2026-08-01", "2026-09-04"),
d.window_value_counts_sql(tbl, col, col, "2026-08-01", 21),
]
sampled = d.sample_value_counts_sql(tbl, col, 1, 21)
if kind in ("postgresql", "oracle"):
sqls.append(sampled)
else:
assert sampled is None # MySQL、SQLite 没有块采样
for sql in sqls:
assert guard.check(sql, readonly=True, dialect=kind) == sql, sql
def test_dialect_quoting_and_row_limits():
my, ora, pg = (catalog_profile.SqlDialect(k) for k in ("mysql", "oracle", "postgresql"))
assert my.quote("memberTagId") == "`memberTagId`" and pg.quote("memberTagId") == '"memberTagId"'
assert my.table(_META["orders"], "orders") == "sales.orders"
assert pg.table({"schema": "Sales"}, "memberTags") == '"Sales"."memberTags"'
# Oracle 没有 LIMIT(11g 也没有 FETCH FIRST):一律用 ROWNUM
sample = ora.distinct_sample_sql("visits", '"memberTagId"', 50, scan_cap=None)
assert "LIMIT" not in sample and "ROWNUM <= 50" in sample
assert "LIMIT 50" in pg.distinct_sample_sql("visits", "x", 50, scan_cap=None)
# 统计信息按方言读各自的系统视图
assert "information_schema.TABLES" in my.stats_sql(_META["visits"], "visits")
assert "pg_class" in pg.stats_sql(_META["visits"], "visits")
stats = ora.stats_sql(_META["orders"], "orders")
assert "ALL_TABLES" in stats and "'ORDERS'" in stats and "'SALES'" in stats
# ---------------------------------------------------------------- 目录对剖析结果的容纳
def test_codes_with_pending_meanings_are_valid_and_render_as_pending():
"""剖析得到的码值只有取值、含义待人填写:目录要收得下,给模型看时写明含义待填写。"""
notes = {"columns": {"status": {"codes": make_item({"1": "", "0": ""}, "profile", "proposed")},
"channel": {"codes": make_item({"WEB": "官网", "BOX": ""}, "profile", "proposed")}}}
assert catalog.validate_notes(notes) == []
text = catalog.render_table_notes(notes)
assert "1、0(含义待填写)" in text
assert "WEB=官网,BOX 的含义待填写" in text
# 码值仍得是对照表:值不是文字的照样拒收
assert catalog.validate_notes({"columns": {"s": {"codes": make_item({"1": 2}, "profile")}}})
def _profile_rel(coverage, cardinality, status):
return {"id": catalog.relation_id("visits", ["gate_id"], "gates", ["id"]), "columns": ["gate_id"],
"to_table": "gates", "to_columns": ["id"], "cardinality": cardinality, "coverage": coverage,
"source": "profile", "status": status}
def test_reset_returns_profile_relation_to_what_profiling_concluded():
"""撤销审阅回到「来源的初始状态」。剖析的结论随数据而定:覆盖率够、父键唯一(基数有值)就是有确证。"""
verified = {"relations": [_profile_rel(1.0, "many_to_one", "verified")]}
rid = verified["relations"][0]["id"]
out = catalog.review_item(verified, f"relations.{rid}", "confirm", at=AT)
assert out["relations"][0]["status"] == "confirmed"
out = catalog.review_item(out, f"relations.{rid}", "reset", at=AT)
assert out["relations"][0]["status"] == "verified"
out = catalog.review_item(catalog.review_item(out, f"relations.{rid}", "reject", at=AT),
f"relations.{rid}", "reset", at=AT)
assert out["relations"][0]["status"] == "verified"
# 覆盖率低、父键没核实唯一的:撤销后仍是推断
for coverage, cardinality in ((0.62, "many_to_one"), (1.0, None)):
low = {"relations": [_profile_rel(coverage, cardinality, "proposed")]}
out = catalog.review_item(catalog.review_item(low, f"relations.{rid}", "confirm", at=AT),
f"relations.{rid}", "reset", at=AT)
assert out["relations"][0]["status"] == "proposed"
# 剖析提的码值候选、业务日期永远是推断
codes = {"columns": {"status": {"codes": make_item({"1": ""}, "profile", "proposed")}}}
out = catalog.review_item(catalog.review_item(codes, "columns.status.codes", "confirm", at=AT),
"columns.status.codes", "reset", at=AT)
assert out["columns"]["status"]["codes"]["status"] == "proposed"
def test_human_edit_may_restore_profile_conclusion_but_not_invent_one():
rel = _profile_rel(1.0, "many_to_one", "confirmed")
existing = {"relations": [rel]}
back = catalog.apply_human_edit(existing, {"relations": [{**rel, "status": "verified"}]}, table="visits", at=AT)
assert back["relations"][0]["status"] == "verified" and back["relations"][0]["source"] == "profile"
low = _profile_rel(0.4, "many_to_one", "proposed")
kept = catalog.apply_human_edit({"relations": [low]}, {"relations": [{**low, "status": "verified"}]},
table="visits", at=AT)
assert kept["relations"][0]["status"] == "proposed"