mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 02:21:58 +00:00
ThearchyHelios
233 lines
12 KiB
Python
233 lines
12 KiB
Python
"""数据剖析（阶段 2A）里不连库的部分：设置的解析与校验、SQL 字面量、各方言的剖析 SQL 都过得了守卫、
|
||
目录对剖析结果的容纳（待填含义的码值、撤销审阅回到剖析的结论）。"""
|
||
from __future__ import annotations
|
||
|
||
from types import SimpleNamespace
|
||
|
||
import pytest
|
||
|
||
from app.data import catalog, catalog_profile, guard
|
||
from app.data.catalog import make_item
|
||
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings, profile_settings, profile_settings_problem
|
||
from app.data.engine import build_url
|
||
|
||
AT = "2026-10-03T08:00:00+00:00"
|
||
|
||
|
||
# ---------------------------------------------------------------- 设置
|
||
|
||
|
||
def test_settings_default_off_and_parse_leniently():
|
||
s = profile_settings({})
|
||
assert s == ProfileSettings()
|
||
assert s.enabled is False and s.table_max_queries == 40 and s.table_max_s == 60 and s.query_timeout_s == 10
|
||
assert s.sample_size == 2000 and s.max_scan_rows == 100_000 and s.table_concurrency == 1
|
||
on = profile_settings({PROFILE_OPTION: {"enabled": True, "table_max_queries": "30", "sample_size": 500}})
|
||
assert on.enabled is True and on.table_max_queries == 30 and on.sample_size == 500
|
||
# 库里存着的值不对（老数据、手改过）：那一项用缺省，不因此开启，也不报错；迁移前的旧键不看
|
||
odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "table_max_queries": 0, "query_timeout_s": "x",
|
||
"max_queries": 5}})
|
||
assert odd.enabled is False and odd.table_max_queries == 40 and odd.query_timeout_s == 10
|
||
assert profile_settings({PROFILE_OPTION: "on"}) == ProfileSettings()
|
||
# 任务里存着的快照读回来是同一份设置
|
||
assert ProfileSettings.from_dict(on.to_dict()) == on and ProfileSettings.from_dict(None) == ProfileSettings()
|
||
|
||
|
||
@pytest.mark.parametrize("value, label", [
|
||
({"table_max_queries": 0}, "每张表最多查询数"),
|
||
({"table_max_queries": 201}, "每张表最多查询数"),
|
||
({"table_max_queries": 2.5}, "每张表最多查询数"),
|
||
({"table_max_s": 9}, "每张表时长上限"),
|
||
({"table_max_s": 601}, "每张表时长上限"),
|
||
({"query_timeout_s": 61}, "单条查询时限"),
|
||
({"sample_size": 5}, "核对键值数"),
|
||
({"max_scan_rows": -1}, "整表统计行数上限"),
|
||
({"table_concurrency": 0}, "同时剖析的表数"),
|
||
({"table_concurrency": 4}, "同时剖析的表数"),
|
||
({"table_concurrency": 1.5}, "同时剖析的表数"),
|
||
({"enabled": "true"}, "开启数据剖析"),
|
||
({"table_max_queries": True}, "每张表最多查询数"),
|
||
])
|
||
def test_settings_problem_names_the_field_in_chinese(value, label):
|
||
problem = profile_settings_problem({PROFILE_OPTION: value})
|
||
assert problem and label in problem and "数据剖析" in problem
|
||
|
||
|
||
def test_settings_problem_rejects_unknown_keys_and_wrong_shape():
|
||
assert "无法识别" in profile_settings_problem({PROFILE_OPTION: {"enabled": True, "rows": 1}})
|
||
assert profile_settings_problem({PROFILE_OPTION: ["enabled"]})
|
||
assert profile_settings_problem({}) is None
|
||
assert profile_settings_problem({PROFILE_OPTION: None}) is None
|
||
assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "table_max_queries": 200, "table_max_s": 10.5,
|
||
"query_timeout_s": 1.5, "sample_size": 10,
|
||
"max_scan_rows": 0, "table_concurrency": 3}}) is None
|
||
|
||
|
||
def test_legacy_whole_run_budgets_are_stripped_not_rejected():
|
||
"""以前按整次剖析算的两项（查询次数上限、总时长上限）：保存时只摘掉，不报错；不折算成每张表的预算。"""
|
||
old = {"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True, "max_queries": 60, "max_total_s": 120,
|
||
"sample_size": 500}}
|
||
cleaned = catalog_profile.without_legacy(old)
|
||
assert cleaned == {"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True, "sample_size": 500}}
|
||
assert old[PROFILE_OPTION]["max_queries"] == 60, "不改调用方的对象"
|
||
assert profile_settings_problem(cleaned) is None
|
||
plain = {PROFILE_OPTION: {"enabled": True}}
|
||
assert catalog_profile.without_legacy(plain) is plain and catalog_profile.without_legacy(None) is None
|
||
|
||
|
||
def test_profile_option_never_reaches_the_connection_url():
|
||
"""剖析设置是 AgentLab 自己的配置：拼进连接串，驱动会把它当未知参数拒掉。"""
|
||
source = SimpleNamespace(kind="mysql", host="db.example", port=3306, database="shop", username="ro",
|
||
password=None, options={"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True}})
|
||
url = build_url(source)
|
||
assert "charset=utf8mb4" in url and PROFILE_OPTION not in url and "enabled" not in url
|
||
|
||
|
||
# ---------------------------------------------------------------- 字面量
|
||
|
||
|
||
@pytest.mark.parametrize("value, kind, expected", [
|
||
(42, "number", "42"),
|
||
(-7, None, "-7"),
|
||
(3.0, "number", "3"),
|
||
("12345678901234567890", "number", "12345678901234567890"), # DECIMAL 经查询层转成了字符串
|
||
("A-01", "text", "'A-01'"),
|
||
("O'Neil", "text", "'O''Neil'"),
|
||
("12:30", "text", "'12:30'"),
|
||
])
|
||
def test_sql_literal_quotes_safe_values(value, kind, expected):
|
||
assert catalog_profile.sql_literal(value, kind) == expected
|
||
|
||
|
||
@pytest.mark.parametrize("value, kind", [
|
||
(True, "boolean"), # 布尔不当键
|
||
(2.5, "number"), # 非整数的数不当键：浮点相等比较靠不住
|
||
("12.50", "number"),
|
||
("1e3", "number"),
|
||
("2026-07-01", "date"), # 日期的字面量各家写法不同，不猜
|
||
("a\\b", "text"), # 反斜杠：MySQL 不同设置下读法不同，守卫会拒
|
||
("a :b", "text"), # 「:名字」会被查询层当成绑定参数
|
||
(":b", "text"),
|
||
("a\nb", "text"), # 控制字符
|
||
("x" * 201, "text"),
|
||
(None, "text"),
|
||
([1], None),
|
||
])
|
||
def test_sql_literal_refuses_unsafe_or_unsupported_values(value, kind):
|
||
assert catalog_profile.sql_literal(value, kind) is None
|
||
|
||
|
||
# ---------------------------------------------------------------- 各方言的剖析 SQL
|
||
|
||
|
||
_META = {"visits": {"schema": None, "columns": [{"name": "memberTagId", "type": "INTEGER"}], "primary_key": ["id"]},
|
||
"orders": {"schema": "sales", "columns": [{"name": "status", "type": "INTEGER"}], "primary_key": ["id"]}}
|
||
|
||
|
||
@pytest.mark.parametrize("kind", ["mysql", "mariadb", "postgresql", "oracle", "sqlite"])
|
||
def test_profile_sql_passes_readonly_guard_on_every_dialect(kind):
|
||
"""剖析的每一种查询（含读统计信息的系统视图）都得过只读守卫——守卫不为剖析放宽。"""
|
||
d = catalog_profile.SqlDialect(kind)
|
||
sqls = []
|
||
for table in ("visits", "orders"):
|
||
stats = d.stats_sql(_META[table], table)
|
||
if kind == "sqlite":
|
||
assert stats is None # SQLite 没有可读的统计信息，改为数到上限
|
||
else:
|
||
assert stats
|
||
sqls.append(stats)
|
||
tbl = d.table(_META[table], table)
|
||
col = d.quote(_META[table]["columns"][0]["name"])
|
||
sqls += [
|
||
d.bounded_count_sql(tbl, 100_001),
|
||
d.distinct_sample_sql(tbl, col, 2000, scan_cap=None),
|
||
d.distinct_sample_sql(tbl, col, 2000, scan_cap=100_000),
|
||
d.match_count_sql(tbl, col, ["1", "'A'"]),
|
||
d.unique_check_sql(tbl, col),
|
||
d.value_counts_sql(tbl, col, 21, scan_cap=None),
|
||
d.value_counts_sql(tbl, col, 21, scan_cap=100_000),
|
||
d.min_max_sql(tbl, [(col, False)]),
|
||
d.min_max_sql(tbl, [(col, True)]),
|
||
d.day_volume_sql(tbl, col, "2026-08-01", "2026-09-04"),
|
||
d.window_value_counts_sql(tbl, col, col, "2026-08-01", 21),
|
||
]
|
||
sampled = d.sample_value_counts_sql(tbl, col, 1, 21)
|
||
if kind in ("postgresql", "oracle"):
|
||
sqls.append(sampled)
|
||
else:
|
||
assert sampled is None # MySQL、SQLite 没有块采样
|
||
for sql in sqls:
|
||
assert guard.check(sql, readonly=True, dialect=kind) == sql, sql
|
||
|
||
|
||
def test_dialect_quoting_and_row_limits():
|
||
my, ora, pg = (catalog_profile.SqlDialect(k) for k in ("mysql", "oracle", "postgresql"))
|
||
assert my.quote("memberTagId") == "`memberTagId`" and pg.quote("memberTagId") == '"memberTagId"'
|
||
assert my.table(_META["orders"], "orders") == "sales.orders"
|
||
assert pg.table({"schema": "Sales"}, "memberTags") == '"Sales"."memberTags"'
|
||
# Oracle 没有 LIMIT（11g 也没有 FETCH FIRST）：一律用 ROWNUM
|
||
sample = ora.distinct_sample_sql("visits", '"memberTagId"', 50, scan_cap=None)
|
||
assert "LIMIT" not in sample and "ROWNUM <= 50" in sample
|
||
assert "LIMIT 50" in pg.distinct_sample_sql("visits", "x", 50, scan_cap=None)
|
||
# 统计信息按方言读各自的系统视图
|
||
assert "information_schema.TABLES" in my.stats_sql(_META["visits"], "visits")
|
||
assert "pg_class" in pg.stats_sql(_META["visits"], "visits")
|
||
stats = ora.stats_sql(_META["orders"], "orders")
|
||
assert "ALL_TABLES" in stats and "'ORDERS'" in stats and "'SALES'" in stats
|
||
|
||
|
||
# ---------------------------------------------------------------- 目录对剖析结果的容纳
|
||
|
||
|
||
def test_codes_with_pending_meanings_are_valid_and_render_as_pending():
|
||
"""剖析得到的码值只有取值、含义待人填写：目录要收得下，给模型看时写明含义待填写。"""
|
||
notes = {"columns": {"status": {"codes": make_item({"1": "", "0": ""}, "profile", "proposed")},
|
||
"channel": {"codes": make_item({"WEB": "官网", "BOX": ""}, "profile", "proposed")}}}
|
||
assert catalog.validate_notes(notes) == []
|
||
text = catalog.render_table_notes(notes)
|
||
assert "1、0（含义待填写）" in text
|
||
assert "WEB=官网，BOX 的含义待填写" in text
|
||
# 码值仍得是对照表：值不是文字的照样拒收
|
||
assert catalog.validate_notes({"columns": {"s": {"codes": make_item({"1": 2}, "profile")}}})
|
||
|
||
|
||
def _profile_rel(coverage, cardinality, status):
|
||
return {"id": catalog.relation_id("visits", ["gate_id"], "gates", ["id"]), "columns": ["gate_id"],
|
||
"to_table": "gates", "to_columns": ["id"], "cardinality": cardinality, "coverage": coverage,
|
||
"source": "profile", "status": status}
|
||
|
||
|
||
def test_reset_returns_profile_relation_to_what_profiling_concluded():
|
||
"""撤销审阅回到「来源的初始状态」。剖析的结论随数据而定：覆盖率够、父键唯一（基数有值）就是有确证。"""
|
||
verified = {"relations": [_profile_rel(1.0, "many_to_one", "verified")]}
|
||
rid = verified["relations"][0]["id"]
|
||
out = catalog.review_item(verified, f"relations.{rid}", "confirm", at=AT)
|
||
assert out["relations"][0]["status"] == "confirmed"
|
||
out = catalog.review_item(out, f"relations.{rid}", "reset", at=AT)
|
||
assert out["relations"][0]["status"] == "verified"
|
||
out = catalog.review_item(catalog.review_item(out, f"relations.{rid}", "reject", at=AT),
|
||
f"relations.{rid}", "reset", at=AT)
|
||
assert out["relations"][0]["status"] == "verified"
|
||
# 覆盖率低、父键没核实唯一的：撤销后仍是推断
|
||
for coverage, cardinality in ((0.62, "many_to_one"), (1.0, None)):
|
||
low = {"relations": [_profile_rel(coverage, cardinality, "proposed")]}
|
||
out = catalog.review_item(catalog.review_item(low, f"relations.{rid}", "confirm", at=AT),
|
||
f"relations.{rid}", "reset", at=AT)
|
||
assert out["relations"][0]["status"] == "proposed"
|
||
# 剖析提的码值候选、业务日期永远是推断
|
||
codes = {"columns": {"status": {"codes": make_item({"1": ""}, "profile", "proposed")}}}
|
||
out = catalog.review_item(catalog.review_item(codes, "columns.status.codes", "confirm", at=AT),
|
||
"columns.status.codes", "reset", at=AT)
|
||
assert out["columns"]["status"]["codes"]["status"] == "proposed"
|
||
|
||
|
||
def test_human_edit_may_restore_profile_conclusion_but_not_invent_one():
|
||
rel = _profile_rel(1.0, "many_to_one", "confirmed")
|
||
existing = {"relations": [rel]}
|
||
back = catalog.apply_human_edit(existing, {"relations": [{**rel, "status": "verified"}]}, table="visits", at=AT)
|
||
assert back["relations"][0]["status"] == "verified" and back["relations"][0]["source"] == "profile"
|
||
low = _profile_rel(0.4, "many_to_one", "proposed")
|
||
kept = catalog.apply_human_edit({"relations": [low]}, {"relations": [{**low, "status": "verified"}]},
|
||
table="visits", at=AT)
|
||
assert kept["relations"][0]["status"] == "proposed"
|