AgentLab/backend/tests/test_profile_dialect_units.py

"""剖析与探查的方言修正（下一轮 03 细案 C1）：不连库的部分。

- 列类型按方言编译的完整写法：类型族、日期列、码值候选都认得 ENUM / SET、VARCHAR2(20 CHAR)、INTERVAL 这些写法；
- IN 列表的字面量按被指向列的类型族生成（PG 上文字列配整数子列会直接报错，Oracle 会把父列转成数报 ORA-01722）；
- Oracle 小写带引号的对象（quote_name）强制加引号、不转大写；
- 统计信息估出的行数不超过整表统计上限时再数一次确认；
- 日期列的最小值、最大值排除零日期和哨兵日期，照实写排除了几个；
- 取值观察要用的三个方言方法（date_literal / day_trunc / sample_from）；
- 说明文字不再说「抽样」：大表看的是按存储顺序的前 N 行。
"""
from __future__ import annotations

import asyncio
import datetime as _dt
from typing import Any

import pytest
from sqlalchemy import VARCHAR
from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite
from sqlalchemy.sql.elements import quoted_name

from app.data import catalog, catalog_profile, introspect
from app.data.catalog_profile import ProfileSettings, SqlDialect, TableSize
from app.data.engine import QueryResult

DIALECTS = ("mysql", "mariadb", "postgresql", "oracle", "sqlite")


# ---------------------------------------------------------------- 类型的写法


@pytest.mark.parametrize("type_name, family", [
    ("ENUM('paid','refund')", "text"),
    ("ENUM('int','date')", "text"),             # 取值里的词不能让它变成数字或时间
    ("SET('a','b')", "text"),
    ("VARCHAR2(20 CHAR)", "text"),
    ("NVARCHAR2(10)", "text"),
    ("VARCHAR(20) COLLATE utf8mb4_bin", "text"),
    ("NUMBER(10, 0)", "number"),
    ("INTEGER(11)", "number"),
    ("BIGINT UNSIGNED", "number"),
    ("DOUBLE PRECISION", "number"),
    ("TIMESTAMP WITH TIME ZONE", "time"),
    ("TIMESTAMP(3)", "time"),
    ("INTERVAL DAY(2) TO SECOND(6)", None),    # 时长不是数也不是时刻；以前按「INT」认成了数字
    ("order_status", None),
    ("", None),
])
def test_type_family_reads_dialect_full_types(type_name, family):
    assert catalog._type_family(type_name) == family


def test_type_family_of_a_column_uses_its_enum_values():
    """PostgreSQL 的枚举类型编译出来是类型名（order_status），只有取值列表能说明它是文字。"""
    assert catalog.column_family({"name": "status", "type": "order_status", "enum_values": ["a", "b"]}) == "text"
    assert catalog.column_family({"name": "n", "type": "INTEGER"}) == "number"
    assert catalog.column_family({"name": "x"}) is None


def test_enum_values_never_make_a_date_column():
    assert not catalog.is_date_column("status", "ENUM('date','time')")
    assert catalog.is_date_column("paid_at", "TIMESTAMP WITH TIME ZONE")
    assert catalog.is_date_column("biz_dt", "VARCHAR2(10 CHAR)")      # 文字存日期、名字像日期
    assert not catalog.is_date_column("gap", "INTERVAL DAY(2) TO SECOND(6)")


@pytest.mark.parametrize("col, ok", [
    ({"type": "VARCHAR2(20 CHAR)"}, True),
    ({"type": "VARCHAR2(200 CHAR)"}, False),
    ({"type": "CHAR(2)"}, True),
    ({"type": "ENUM('paid','refund')"}, True),
    ({"type": "SET('a','b')"}, True),
    ({"type": "order_status", "enum_values": ["a", "b"]}, True),
    ({"type": "NUMBER(10, 0)"}, True),
    ({"type": "NUMBER(10, 2)"}, False),
    ({"type": "TIMESTAMP(3)"}, False),
    ({"type": "INTERVAL DAY(2) TO SECOND(6)"}, False),
])
def test_code_type_accepts_enum_and_char_semantics(col, ok):
    assert catalog_profile._code_type_ok(col.get("type"), col.get("enum_values")) is ok


def test_enum_columns_are_code_candidates_whatever_their_name():
    """ENUM 的取值列表有限，直接算码值候选：pay_way 这种名字不像码值的列以前被漏掉。"""
    meta = {"columns": [{"name": "id", "type": "INTEGER"}, {"name": "pay_way", "type": "ENUM('cash','card')",
                                                             "enum_values": ["cash", "card"]},
                        {"name": "pay_note", "type": "VARCHAR(20)"}],
            "primary_key": ["id"]}
    assert catalog_profile._code_columns(meta, {}) == ["pay_way"]


# ---------------------------------------------------------------- 探查：类型写法、取值列表、引号、索引首列


@pytest.mark.parametrize("dialect, col_type, text", [
    (mysql.dialect(), mysql.ENUM("paid", "refund"), "ENUM('paid','refund')"),
    (mysql.dialect(), mysql.TINYINT(display_width=1), "TINYINT(1)"),
    (oracle.dialect(), oracle.VARCHAR2(20), "VARCHAR2(20 CHAR)"),
    # Oracle 的 VARCHAR2 反射回来是通用的 VARCHAR（容器集成测试实测）：照数据字典写成 VARCHAR2
    (oracle.dialect(), VARCHAR(10), "VARCHAR2(10 CHAR)"),
    (oracle.dialect(), oracle.NUMBER(10, 0), "NUMBER(10, 0)"),
    (postgresql.dialect(), postgresql.ENUM("a", "b", name="order_status"), "order_status"),
    (sqlite.dialect(), sqlite.DATETIME(), "DATETIME"),
])
def test_reflected_type_is_compiled_for_the_dialect(dialect, col_type, text):
    assert introspect.type_text(col_type, dialect) == text


def test_type_that_cannot_compile_falls_back_to_str():
    from sqlalchemy.types import NullType

    assert introspect.type_text(NullType(), sqlite.dialect()) == "NULL"
    assert introspect.type_text(None, sqlite.dialect()) == ""


def test_column_entry_records_enum_values_and_oracle_quoted_names():
    ora = oracle.dialect()
    entry = introspect.column_entry({"name": quoted_name("level", quote=True), "type": oracle.VARCHAR2(10),
                                     "nullable": True, "comment": None}, ora)
    assert entry == {"name": "level", "type": "VARCHAR2(10 CHAR)", "nullable": True, "comment": None,
                     "quote_name": True}
    plain = introspect.column_entry({"name": "status", "type": mysql.ENUM("a", "b"), "nullable": False}, mysql.dialect())
    assert plain["enum_values"] == ["a", "b"] and "quote_name" not in plain
    members = introspect.column_entry({"name": "tags", "type": mysql.SET("x", "y")}, mysql.dialect())
    assert members["enum_values"] == ["x", "y"] and members["type"] == "SET('x','y')"


def test_index_first_columns_are_recorded():
    indexes = [{"column_names": ["paid_at", "id"]}, {"column_names": [None]}, {"column_names": ["store_id"]},
               {"column_names": ["paid_at"]}]
    assert introspect.index_first_columns(indexes, ["id"]) == ["id", "paid_at", "store_id"]


# ---------------------------------------------------------------- IN 列表的字面量按被指向列的类型族


def test_key_literals_follow_the_parent_family():
    lits, mismatched = catalog_profile.key_literals([12, 7], "number", parent="text", child_type="INTEGER")
    assert lits == ["'12'", "'7'"] and mismatched == 0
    lits, mismatched = catalog_profile.key_literals(["12", "A1", "-3"], "text", parent="number",
                                                    child_type="VARCHAR(10)")
    assert lits == ["12", "-3"] and mismatched == 1
    lits, mismatched = catalog_profile.key_literals([1, 2], "number", parent="time", child_type="INTEGER")
    assert lits == [] and mismatched == 2
    # 父列类型不知道：照子表取回的值的类型写（以前的做法）
    assert catalog_profile.key_literals([3, "x"], None, parent=None, child_type=None) == (["3", "'x'"], 0)


def test_char_keys_lose_their_padding():
    """PG、Oracle 的 CHAR(n) 取回来带尾随空格：拼成 'A ' 去和 VARCHAR 父列比就对不上。"""
    lits, _ = catalog_profile.key_literals(["A ", "BC"], "text", parent="text", child_type="CHAR(2)")
    assert lits == ["'A'", "'BC'"]
    lits, _ = catalog_profile.key_literals(["A "], "text", parent="text", child_type="VARCHAR(2)")
    assert lits == ["'A '"]


# ---------------------------------------------------------------- Oracle 小写带引号的对象


def test_oracle_quote_name_forces_quotes_and_keeps_case():
    ora = SqlDialect("oracle")
    meta = {"schema": "sales", "quote_name": True,
            "columns": [{"name": "level", "type": "VARCHAR2(10 CHAR)", "quote_name": True},
                        {"name": "amount", "type": "NUMBER(10, 2)"}]}
    assert ora.table(meta, "visits") == 'sales."visits"'
    assert ora.column(meta, "level") == '"level"' and ora.column(meta, "amount") == "amount"
    stats = ora.stats_sql(meta, "visits")
    assert "TABLE_NAME = 'visits'" in stats and "OWNER = 'SALES'" in stats
    # 没有 quote_name 的照旧：不加引号，Oracle 自己转大写
    assert ora.table({"schema": None}, "visits") == "visits"
    assert "TABLE_NAME = 'VISITS'" in ora.stats_sql({}, "visits")


# ---------------------------------------------------------------- 统计信息过期：再数一次确认


def _result(value: Any) -> QueryResult:
    return QueryResult(columns=["n"], rows=[[value]], row_count=1, truncated=False, elapsed_ms=1, sql="")


def _runner(answers: list[Any], seen: list[str]):
    async def run(sql: str, **_kw: Any) -> QueryResult:
        seen.append(sql)
        return _result(answers.pop(0))
    return run


@pytest.mark.parametrize("stats, counted, expected", [
    (500, 500, TableSize(500, "count")),                       # 统计信息说小、数过也小：准数
    (500, 100_001, TableSize(None, "count", at_least=100_001)),  # 统计信息过期：其实是大表
    (5_000_000, None, TableSize(5_000_000, "stats")),          # 统计信息说大：不必再数
])
def test_small_stats_are_confirmed_with_a_bounded_count(stats, counted, expected):
    seen: list[str] = []
    answers = [stats] + ([counted] if counted is not None else [])
    size, skips = asyncio.run(catalog_profile.estimate_size(_runner(answers, seen), SqlDialect("postgresql"),
                                                            {"schema": None}, "visits", max_scan_rows=100_000))
    assert size == expected and skips == []
    assert len(seen) == (2 if counted is not None else 1)
    if counted is not None:
        assert "LIMIT 100001" in seen[1]


# ---------------------------------------------------------------- 日期列：排除零日期和哨兵日期


@pytest.mark.parametrize("kind", DIALECTS)
def test_min_max_excludes_invalid_dates_and_counts_them(kind):
    d = SqlDialect(kind)
    sql = d.min_max_sql("t", [("paid_at", False), ("biz_day", True)])
    # 日期类型的列按日期字面量比，文字存的日期按字符串比（PG 上 varchar 和 date 比较直接报错）
    lo, hi = d.date_literal("0001-01-02"), d.date_literal("9999-12-31")
    assert f"paid_at >= {lo}" in sql and f"paid_at < {hi}" in sql
    assert "biz_day >= '0001-01-02'" in sql and "biz_day < '9999-12-31'" in sql
    assert sql.count("SUM(CASE WHEN") == 2


def _dates_cx(row: list[Any], kind: str = "mysql"):
    settings = ProfileSettings(enabled=True)
    source = type("S", (), {"id": "s1", "name": "shop", "kind": kind, "options": {}})()
    job = catalog_profile.ProfileContext(source=source, settings=settings, dialect=SqlDialect(kind),
                                         runner=None, tables={"orders": {"columns": [
                                             {"name": "paid_at", "type": "DATETIME"}]}}, masked=set())
    job.sizes["orders"] = TableSize(10, "count")
    result = catalog_profile.TableResult(table="orders")
    cx = catalog_profile._TableCx(job, "orders", result, catalog_profile.TableHooks())

    async def run(sql: str, **_kw: Any) -> QueryResult:
        return QueryResult(columns=["a", "b", "c"], rows=[row], row_count=1, truncated=False, elapsed_ms=1, sql=sql)
    cx.run = run  # type: ignore[method-assign]
    return cx, result


def test_dates_note_says_how_many_invalid_values_were_left_out():
    cx, result = _dates_cx(["2026-01-03 08:00:00", "2026-09-30 21:00:00", 4])
    out = catalog_profile._TableOutcome()
    asyncio.run(catalog_profile._check_dates(cx, "orders", ["paid_at"], 1, out, result))
    assert result.date_ranges == [{"column": "paid_at", "min": "2026-01-03 08:00:00", "max": "2026-09-30 21:00:00",
                                   "invalid": 4}]
    assert "另有 4 个无效日期" in out.business_date.note and "0000-00-00" in out.business_date.note


def test_only_invalid_dates_is_no_data():
    cx, result = _dates_cx([None, None, 9])
    with pytest.raises(catalog_profile._Skip) as e:
        asyncio.run(catalog_profile._check_dates(cx, "orders", ["paid_at"], 1, catalog_profile._TableOutcome(),
                                                 result))
    assert e.value.reason == "no_data" and "9 个无效日期" in e.value.detail


# ---------------------------------------------------------------- 取值观察要用的方言方法


@pytest.mark.parametrize("kind, literal, day, sample", [
    ("mysql", "DATE '2026-09-01'", "DATE(paid_at)", None),
    ("mariadb", "DATE '2026-09-01'", "DATE(paid_at)", None),
    ("postgresql", "DATE '2026-09-01'", "CAST(paid_at AS DATE)", "orders TABLESAMPLE SYSTEM (1.5)"),
    ("oracle", "DATE '2026-09-01'", "TRUNC(paid_at)", "orders SAMPLE BLOCK (1.5)"),
    ("sqlite", "'2026-09-01'", "substr(paid_at, 1, 10)", None),
])
def test_observation_dialect_methods(kind, literal, day, sample):
    d = SqlDialect(kind)
    assert d.date_literal(_dt.date(2026, 9, 1)) == literal == d.date_literal("2026-09-01")
    assert d.date_literal("2026-09-01", text=True) == "'2026-09-01'"
    assert d.day_trunc("paid_at") == day
    assert d.sample_from("orders", 1.5) == sample


@pytest.mark.parametrize("bad", ["2026-9-1", "2026-09-01'; --", "0000-00-00", 20260901, None])
def test_date_literal_refuses_anything_but_a_real_date(bad):
    with pytest.raises(ValueError):
        SqlDialect("postgresql").date_literal(bad)


@pytest.mark.parametrize("bad", [0, -1, 100, 101, float("nan")])
def test_sample_percent_must_be_between_0_and_100(bad):
    with pytest.raises(ValueError):
        SqlDialect("postgresql").sample_from("orders", bad)


# ---------------------------------------------------------------- 说明文字


def test_notes_say_storage_order_not_sampling():
    note = catalog_profile._relation_note(sample=2000, matched=2000, coverage=1.0, scan_cap=100_000, skipped_values=0,
                                          mismatched=0, parent=(True, "主键"), child=(False, "count"), holds=True,
                                          collation=None)
    assert "按存储顺序只看前 100000 行" in note and "抽样" not in note
    assert catalog_profile.FILTER_SAMPLED.format(rows="100", share="1%").startswith("剖析核对：按存储顺序取前 100 行统计")


def test_mysql_notes_mention_the_collation():
    note = catalog_profile._relation_note(sample=10, matched=10, coverage=1.0, scan_cap=None, skipped_values=0,
                                          mismatched=0, parent=(True, "主键"), child=(False, "count"), holds=True,
                                          collation=catalog_profile.MYSQL_COLLATION_NOTE)
    assert "排序规则" in note and "不分大小写" in note