diff --git a/backend/app/data/introspect.py b/backend/app/data/introspect.py index 9eec9fb2..6824a99b 100644 --- a/backend/app/data/introspect.py +++ b/backend/app/data/introspect.py @@ -118,8 +118,58 @@ def _target_schema(source: Any, explicit: str | None) -> str | None: return str(configured) if configured else None +def _foreign_keys(raw: list[dict[str, Any]], target_schema: str | None, + default_schema: str | None) -> list[dict[str, Any]]: + """反射出来的外键 → [{columns, to_table, to_columns}]。 + + to_table 和 schema_cache["tables"] 的键同一个口径:指向同一个 schema 的只写表名;跨 schema 的写 + 「schema.表名」(那张表不在这次探查的结果里,关系照记,图里是一个悬空的端点)。to_columns 可能为空 + (SQLite 的 REFERENCES t 省略了列),由数据目录按被指向表的主键补。 + """ + home = {s for s in (target_schema, default_schema) if s} + out: list[dict[str, Any]] = [] + for fk in raw: + cols = [c for c in fk.get("constrained_columns") or [] if c] + to_table = fk.get("referred_table") + if not cols or not to_table: + continue + ref_schema = fk.get("referred_schema") + out.append({ + "columns": cols, + "to_table": to_table if not ref_schema or ref_schema in home else f"{ref_schema}.{to_table}", + "to_columns": [c for c in fk.get("referred_columns") or [] if c], + }) + return out + + +def _unique_groups(constraints: list[dict[str, Any]], indexes: list[dict[str, Any]], + pk: list[str]) -> list[list[str]]: + """唯一约束和唯一索引的列组,去重,不含和主键相同的那组。 + + 同一组唯一列在各家可能出现两次:MySQL 的 UNIQUE 约束本身就是唯一索引,PostgreSQL 的唯一约束背后也有 + 一个索引,Oracle 的主键背后还有一个唯一索引。表达式索引(列名里有 None)不算。 + """ + seen = {tuple(c.lower() for c in pk)} if pk else set() + groups: list[list[str]] = [] + candidates = [u.get("column_names") for u in constraints] + candidates += [ix.get("column_names") for ix in indexes if ix.get("unique")] + for cols in candidates: + if not cols or any(c is None for c in cols): + continue + key = tuple(str(c).lower() for c in cols) + if key in seen: + continue + seen.add(key) + groups.append([str(c) for c in cols]) + return groups + + async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any]: - """探查整个库的结构,结果直接存进 DataSource.schema_cache。""" + """探查整个库的结构,结果直接存进 DataSource.schema_cache。 + + 每张表:列(名字、类型、可空、注释)、主键、表注释、是不是视图;读得到的话还有外键约束 + (foreign_keys)和唯一约束 / 唯一索引(unique)——读不到的表不写这两个键,探查照常完成。 + """ engine = await engines.get(source) target = _target_schema(source, schema) @@ -151,9 +201,14 @@ async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any failures: dict[str, str] = {} def _multi(fn: Any, single: Any, names: list[str], *, - errors: dict[str, str] | None = None) -> dict[Any, Any]: + errors: dict[str, str] | None = None, ok: set[str] | None = None) -> dict[Any, Any]: + """批量反射,失败时逐个对象重试。ok 给了的话,记下真正读到了的对象名:批量成功时是全部 + (没有约束的表批量接口也会给空列表,或者干脆不给),逐个重试时只有没报错的那些。""" try: - return dict(fn(schema=target_schema, filter_names=names, kind=ObjectKind.ANY)) + got = dict(fn(schema=target_schema, filter_names=names, kind=ObjectKind.ANY)) + if ok is not None: + ok.update(names) + return got except NotImplementedError: return {} # 表注释这类不是所有方言都支持 except Exception: # noqa: BLE001 - 批量取不到,逐个对象再试 @@ -162,6 +217,8 @@ async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any for name in names: try: out[(target_schema, name)] = single(name, schema=target_schema) + if ok is not None: + ok.add(name) except NotImplementedError: break except Exception as e: # noqa: BLE001 - 只跳过这一个对象 @@ -176,6 +233,20 @@ async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any readable = [n for n in picked if columns_by.get((target_schema, n))] pk_by = _multi(inspector.get_multi_pk_constraint, inspector.get_pk_constraint, readable) comment_by = _multi(inspector.get_multi_table_comment, inspector.get_table_comment, readable) + # 外键和唯一约束给数据目录起草关系用(data/catalog.py)。放在列、主键、注释之后读:PostgreSQL 上一条 + # 语句报错会让整个事务作废,先读的那几样不能被它连累。读不到的表不写这两个键(ok 集合里没有它), + # 下游据此分得清「读过、没有」和「不知道」——升级前探查的缓存里也没有这两个键。 + fk_ok: set[str] = set() + uq_ok: set[str] = set() + ix_ok: set[str] = set() + fk_by = _multi(inspector.get_multi_foreign_keys, inspector.get_foreign_keys, readable, errors={}, ok=fk_ok) + uq_by = _multi(inspector.get_multi_unique_constraints, inspector.get_unique_constraints, readable, + errors={}, ok=uq_ok) + ix_by = _multi(inspector.get_multi_indexes, inspector.get_indexes, readable, errors={}, ok=ix_ok) + try: + default_schema = inspector.default_schema_name + except Exception: # noqa: BLE001 - 只用来判断外键是不是指向别的 schema + default_schema = None tables: dict[str, Any] = {} skipped: dict[str, str] = {} @@ -207,6 +278,11 @@ async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any "comment": comment, "is_view": name in views, } + if name in fk_ok: + tables[name]["foreign_keys"] = _foreign_keys(fk_by.get(key) or [], target_schema, default_schema) + if name in uq_ok: + tables[name]["unique"] = _unique_groups(uq_by.get(key) or [], + (ix_by.get(key) or []) if name in ix_ok else [], pk) payload: dict[str, Any] = {"tables": tables, "truncated": truncated, "total": len(names) + len(views)} if failures and not tables: diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index e6dee138..42722887 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -41,3 +41,14 @@ def _prepare_schema(): def pytest_report_header(config) -> str: return f"测试数据目录:{_TMP}" + + +@pytest.fixture(scope="session") +def scenic_db(tmp_path_factory) -> str: + """合成的景区业务库(tests/fixtures/catalog/scenic.py):50 张表、真外键与只能靠命名推断的关系并存。 + + 每个测试进程建一次,只读使用;要改库内容的测试自己调 scenic.build(tmp_path / "x.db") 另建一个。 + """ + from tests.fixtures.catalog import scenic + + return str(scenic.build(tmp_path_factory.mktemp("catalog") / "scenic.db")) diff --git a/backend/tests/fixtures/catalog/__init__.py b/backend/tests/fixtures/catalog/__init__.py new file mode 100644 index 00000000..da3b7e82 --- /dev/null +++ b/backend/tests/fixtures/catalog/__init__.py @@ -0,0 +1,16 @@ +"""数据目录用的合成业务库夹具。 + +- scenic.py:一个虚构的「景区 + 园内门店」业务库(SQLite,50 张表 + 1 个视图),给数据目录的起草、关系图、 + 表清单测试用,阶段 1B(助手按任务挑表)也用它。表名、数据全部虚构,不对应任何真实部署。 + +用法:: + + from tests.fixtures.catalog import scenic + + path = scenic.build(tmp_path / "scenic.db") # 写一个新库,返回路径;同样的参数造出同样的内容 + scenic.FK_RELATIONS # 库里声明了外键约束的关系 + scenic.NAME_RELATIONS # 没有外键约束、只能靠列名推断出来的关系 + scenic.NOT_INFERRED # 看起来像关联、但命名推断必须放过的列(及原因) + +conftest.py 里有个 scenic_db 夹具(每个测试进程建一次,只读使用),直接要这个参数就行。 +""" diff --git a/backend/tests/fixtures/catalog/scenic.py b/backend/tests/fixtures/catalog/scenic.py new file mode 100644 index 00000000..fd7ffb87 --- /dev/null +++ b/backend/tests/fixtures/catalog/scenic.py @@ -0,0 +1,284 @@ +"""虚构的景区业务库:入园记录、渠道、票种、订单、园内门店、会员、营销、停车、旅行团……共 50 张表和 1 个视图。 + +为数据目录设计的几处特点: + +- **真外键和只能靠命名推断的关系混在一起。** 核心交易表(订单、订单明细、入园记录的票种)声明了外键; + 很多「看名字就知道连哪」的列(visits.gate_id、store_sales.productId、inventory_snapshots.storeID)没有约束, + 只能按列名推断。三种命名都有:xxx_id、xxxId、xxxID;表名有复数、驼峰(memberTags)、y→ies(categories、agencies)。 +- **关联表。** channel_visits 把渠道和入园记录连起来(复合主键,两端都是外键):入园记录本身不带渠道, + 从 visits 到 channels 要经过它,正好是两跳的连接路径。group_visits 同理(旅行团与入园记录)。 +- **命名推断必须放过的列**(NOT_INFERRED):自引用(employees.employee_id 是工号,不是指向自己)、 + 多个候选(guide 和 guides 两张表都在,tour_assignments.guide_id 不知道指哪张)、没有对应的表 + (parking_records.lot_id、operation_logs.operator_id、categories.parentId)、类型对不上 + (feedback_forms.store_id 是文字,stores.id 是整数)。 +- **唯一约束 / 唯一索引。** orders.order_no、channels.channel_code 是列上的 UNIQUE;members.member_no、 + visits.ticket_no 是单独建的唯一索引。 + +数据是固定种子生成的假数,量很小(入园记录 1500 行),只为让查询有结果;名称一律虚构。 +""" +from __future__ import annotations + +import random +import sqlite3 +from pathlib import Path + +DDL = """ +CREATE TABLE parks (id INTEGER PRIMARY KEY, park_code TEXT NOT NULL UNIQUE, name TEXT NOT NULL, city TEXT, + opened_on TEXT); +CREATE TABLE areas (id INTEGER PRIMARY KEY, park_id INTEGER NOT NULL, name TEXT NOT NULL); +CREATE TABLE gates (id INTEGER PRIMARY KEY, park_id INTEGER NOT NULL REFERENCES parks(id), gate_code TEXT NOT NULL, + name TEXT); +CREATE TABLE channels (id INTEGER PRIMARY KEY, channel_code TEXT NOT NULL UNIQUE, name TEXT NOT NULL, + channel_type TEXT NOT NULL); +CREATE TABLE ticket_types (id INTEGER PRIMARY KEY, park_id INTEGER NOT NULL REFERENCES parks(id), + type_code TEXT NOT NULL, name TEXT NOT NULL, base_price REAL NOT NULL); +CREATE TABLE member_levels (id INTEGER PRIMARY KEY, level_name TEXT NOT NULL, discount_rate REAL); +CREATE TABLE members (id INTEGER PRIMARY KEY, member_no TEXT NOT NULL, name TEXT, member_level_id INTEGER, + joined_on TEXT, status INTEGER NOT NULL DEFAULT 1); +CREATE UNIQUE INDEX ux_members_member_no ON members(member_no); +CREATE TABLE "memberTags" (id INTEGER PRIMARY KEY, tag_name TEXT NOT NULL); +CREATE TABLE member_tag_links (id INTEGER PRIMARY KEY, member_id INTEGER NOT NULL REFERENCES members(id), + "memberTagId" INTEGER NOT NULL); +CREATE TABLE orders (id INTEGER PRIMARY KEY, order_no TEXT NOT NULL UNIQUE, member_id INTEGER REFERENCES members(id), + channel_id INTEGER NOT NULL REFERENCES channels(id), ordered_at TEXT NOT NULL, + total_amount REAL NOT NULL, status INTEGER NOT NULL); +CREATE TABLE order_items (id INTEGER PRIMARY KEY, order_id INTEGER NOT NULL REFERENCES orders(id), + ticket_type_id INTEGER NOT NULL REFERENCES ticket_types(id), qty INTEGER NOT NULL, + unit_price REAL NOT NULL, amount REAL NOT NULL); +CREATE TABLE visits (id INTEGER PRIMARY KEY, ticket_no TEXT NOT NULL, park_id INTEGER NOT NULL REFERENCES parks(id), + gate_id INTEGER, ticket_type_id INTEGER NOT NULL REFERENCES ticket_types(id), member_id INTEGER, + visit_time TEXT NOT NULL, visitor_count INTEGER NOT NULL DEFAULT 1, status INTEGER NOT NULL); +CREATE UNIQUE INDEX ux_visits_ticket_no ON visits(ticket_no); +CREATE TABLE channel_visits (channel_id INTEGER NOT NULL REFERENCES channels(id), + visit_id INTEGER NOT NULL REFERENCES visits(id), + attributed_share REAL NOT NULL DEFAULT 1, PRIMARY KEY (channel_id, visit_id)); +CREATE TABLE stores (id INTEGER PRIMARY KEY, store_code TEXT NOT NULL UNIQUE, name TEXT NOT NULL, + park_id INTEGER NOT NULL REFERENCES parks(id), area_id INTEGER, opened_on TEXT); +CREATE TABLE categories (id INTEGER PRIMARY KEY, name TEXT NOT NULL, "parentId" INTEGER); +CREATE TABLE products (id INTEGER PRIMARY KEY, sku TEXT NOT NULL UNIQUE, name TEXT NOT NULL, category_id INTEGER, + list_price REAL); +CREATE TABLE store_sales (id INTEGER PRIMARY KEY, store_id INTEGER NOT NULL REFERENCES stores(id), + "productId" INTEGER NOT NULL, member_id INTEGER, sold_at TEXT NOT NULL, qty INTEGER NOT NULL, + amount REAL NOT NULL); +CREATE TABLE inventory_snapshots (id INTEGER PRIMARY KEY, "storeID" INTEGER NOT NULL, "productId" INTEGER NOT NULL, + snapshot_date TEXT NOT NULL, on_hand INTEGER NOT NULL); +CREATE TABLE campaigns (id INTEGER PRIMARY KEY, channel_id INTEGER REFERENCES channels(id), name TEXT NOT NULL, + starts_on TEXT, ends_on TEXT); +CREATE TABLE coupons (id INTEGER PRIMARY KEY, code TEXT NOT NULL UNIQUE, campaign_id INTEGER, + discount_amount REAL NOT NULL); +CREATE TABLE coupon_redemptions (id INTEGER PRIMARY KEY, coupon_id INTEGER NOT NULL REFERENCES coupons(id), + order_id INTEGER NOT NULL REFERENCES orders(id), redeemed_at TEXT NOT NULL); +CREATE TABLE payments (id INTEGER PRIMARY KEY, order_id INTEGER NOT NULL REFERENCES orders(id), + pay_method TEXT NOT NULL, paid_amount REAL NOT NULL, paid_at TEXT NOT NULL); +CREATE TABLE refunds (id INTEGER PRIMARY KEY, order_id INTEGER NOT NULL REFERENCES orders(id), + refund_amount REAL NOT NULL, reason_code TEXT, refunded_at TEXT NOT NULL); +CREATE TABLE reviews (id INTEGER PRIMARY KEY, visit_id INTEGER NOT NULL, score INTEGER NOT NULL, review_text TEXT, + created_at TEXT NOT NULL); +CREATE TABLE complaints (id INTEGER PRIMARY KEY, visit_id INTEGER, category_code TEXT NOT NULL, + status INTEGER NOT NULL, created_at TEXT NOT NULL); +CREATE TABLE events (id INTEGER PRIMARY KEY, park_id INTEGER NOT NULL REFERENCES parks(id), name TEXT NOT NULL, + event_date TEXT NOT NULL); +CREATE TABLE event_bookings (id INTEGER PRIMARY KEY, event_id INTEGER NOT NULL, member_id INTEGER NOT NULL, + seats INTEGER NOT NULL, booked_at TEXT NOT NULL); +CREATE TABLE parking_lots (id INTEGER PRIMARY KEY, park_id INTEGER NOT NULL REFERENCES parks(id), + name TEXT NOT NULL, capacity INTEGER NOT NULL); +CREATE TABLE parking_records (id INTEGER PRIMARY KEY, lot_id INTEGER NOT NULL, plate_masked TEXT, + entered_at TEXT NOT NULL, exited_at TEXT, fee REAL); +CREATE TABLE weather_daily (park_id INTEGER NOT NULL, biz_date TEXT NOT NULL, weather TEXT, temp_high REAL, + temp_low REAL, PRIMARY KEY (park_id, biz_date)); +CREATE TABLE holidays (biz_date TEXT PRIMARY KEY, holiday_name TEXT NOT NULL, is_holiday INTEGER NOT NULL); +CREATE TABLE dim_dates (date_key INTEGER PRIMARY KEY, biz_date TEXT NOT NULL UNIQUE, weekday INTEGER NOT NULL, + is_weekend INTEGER NOT NULL); +CREATE TABLE price_rules (id INTEGER PRIMARY KEY, ticket_type_id INTEGER NOT NULL REFERENCES ticket_types(id), + valid_from TEXT NOT NULL, valid_to TEXT, price REAL NOT NULL); +CREATE TABLE ticket_inventory (id INTEGER PRIMARY KEY, ticket_type_id INTEGER NOT NULL, biz_date TEXT NOT NULL, + quota INTEGER NOT NULL, sold INTEGER NOT NULL); +CREATE TABLE member_points_log (id INTEGER PRIMARY KEY, member_id INTEGER NOT NULL REFERENCES members(id), + change_points INTEGER NOT NULL, reason TEXT, created_at TEXT NOT NULL); +CREATE TABLE sms_logs (id INTEGER PRIMARY KEY, member_id INTEGER, template_code TEXT NOT NULL, sent_at TEXT NOT NULL); +CREATE TABLE suppliers (id INTEGER PRIMARY KEY, name TEXT NOT NULL); +CREATE TABLE purchase_orders (id INTEGER PRIMARY KEY, supplier_id INTEGER NOT NULL REFERENCES suppliers(id), + store_id INTEGER NOT NULL REFERENCES stores(id), ordered_at TEXT NOT NULL, + amount REAL NOT NULL); +CREATE TABLE purchase_order_lines (id INTEGER PRIMARY KEY, purchase_order_id INTEGER NOT NULL, + product_id INTEGER NOT NULL REFERENCES products(id), qty INTEGER NOT NULL, + cost REAL NOT NULL); +CREATE TABLE employees (id INTEGER PRIMARY KEY, employee_id TEXT NOT NULL, store_id INTEGER, name TEXT NOT NULL, + role TEXT); +CREATE TABLE agencies (id INTEGER PRIMARY KEY, channel_id INTEGER NOT NULL REFERENCES channels(id), + name TEXT NOT NULL); +CREATE TABLE tour_groups (id INTEGER PRIMARY KEY, agency_id INTEGER NOT NULL, group_size INTEGER NOT NULL, + visit_date TEXT NOT NULL); +CREATE TABLE group_visits (tour_group_id INTEGER NOT NULL REFERENCES tour_groups(id), + visit_id INTEGER NOT NULL REFERENCES visits(id), PRIMARY KEY (tour_group_id, visit_id)); +CREATE TABLE guide (id INTEGER PRIMARY KEY, name TEXT); +CREATE TABLE guides (id INTEGER PRIMARY KEY, name TEXT NOT NULL, licensed INTEGER); +CREATE TABLE tour_assignments (id INTEGER PRIMARY KEY, tour_group_id INTEGER NOT NULL, guide_id INTEGER NOT NULL, + assigned_on TEXT NOT NULL); +CREATE TABLE feedback_forms (id INTEGER PRIMARY KEY, store_id TEXT, submitted_at TEXT NOT NULL, rating INTEGER); +CREATE TABLE lost_and_found (id INTEGER PRIMARY KEY, park_id INTEGER NOT NULL, item_desc TEXT NOT NULL, + found_at TEXT NOT NULL); +CREATE TABLE operation_logs (id INTEGER PRIMARY KEY, operator_id INTEGER, action TEXT NOT NULL, + created_at TEXT NOT NULL); +CREATE TABLE sys_config (config_key TEXT PRIMARY KEY, config_value TEXT); +CREATE VIEW v_daily_visits AS + SELECT park_id, substr(visit_time, 1, 10) AS biz_date, SUM(visitor_count) AS visitors + FROM visits WHERE status = 1 GROUP BY park_id, substr(visit_time, 1, 10); +""" + +#: 表的个数(不含视图) +TABLE_COUNT = 50 +VIEWS = ("v_daily_visits",) + +#: 声明了外键约束的关系:(本表, (本表列,), 被指向的表, (被指向的列,)) +FK_RELATIONS = frozenset({ + ("gates", ("park_id",), "parks", ("id",)), + ("ticket_types", ("park_id",), "parks", ("id",)), + ("member_tag_links", ("member_id",), "members", ("id",)), + ("orders", ("member_id",), "members", ("id",)), + ("orders", ("channel_id",), "channels", ("id",)), + ("order_items", ("order_id",), "orders", ("id",)), + ("order_items", ("ticket_type_id",), "ticket_types", ("id",)), + ("visits", ("park_id",), "parks", ("id",)), + ("visits", ("ticket_type_id",), "ticket_types", ("id",)), + ("channel_visits", ("channel_id",), "channels", ("id",)), + ("channel_visits", ("visit_id",), "visits", ("id",)), + ("stores", ("park_id",), "parks", ("id",)), + ("store_sales", ("store_id",), "stores", ("id",)), + ("campaigns", ("channel_id",), "channels", ("id",)), + ("coupon_redemptions", ("coupon_id",), "coupons", ("id",)), + ("coupon_redemptions", ("order_id",), "orders", ("id",)), + ("payments", ("order_id",), "orders", ("id",)), + ("refunds", ("order_id",), "orders", ("id",)), + ("events", ("park_id",), "parks", ("id",)), + ("parking_lots", ("park_id",), "parks", ("id",)), + ("price_rules", ("ticket_type_id",), "ticket_types", ("id",)), + ("member_points_log", ("member_id",), "members", ("id",)), + ("purchase_orders", ("supplier_id",), "suppliers", ("id",)), + ("purchase_orders", ("store_id",), "stores", ("id",)), + ("purchase_order_lines", ("product_id",), "products", ("id",)), + ("agencies", ("channel_id",), "channels", ("id",)), + ("group_visits", ("tour_group_id",), "tour_groups", ("id",)), + ("group_visits", ("visit_id",), "visits", ("id",)), +}) + +#: 没有外键约束、命名推断应当找出来的关系(被指向一端都是单列主键 id) +NAME_RELATIONS = frozenset({ + ("areas", ("park_id",), "parks", ("id",)), + ("members", ("member_level_id",), "member_levels", ("id",)), + ("member_tag_links", ("memberTagId",), "memberTags", ("id",)), + ("visits", ("gate_id",), "gates", ("id",)), + ("visits", ("member_id",), "members", ("id",)), + ("stores", ("area_id",), "areas", ("id",)), + ("products", ("category_id",), "categories", ("id",)), + ("store_sales", ("productId",), "products", ("id",)), + ("store_sales", ("member_id",), "members", ("id",)), + ("inventory_snapshots", ("storeID",), "stores", ("id",)), + ("inventory_snapshots", ("productId",), "products", ("id",)), + ("coupons", ("campaign_id",), "campaigns", ("id",)), + ("reviews", ("visit_id",), "visits", ("id",)), + ("complaints", ("visit_id",), "visits", ("id",)), + ("event_bookings", ("event_id",), "events", ("id",)), + ("event_bookings", ("member_id",), "members", ("id",)), + ("weather_daily", ("park_id",), "parks", ("id",)), + ("ticket_inventory", ("ticket_type_id",), "ticket_types", ("id",)), + ("sms_logs", ("member_id",), "members", ("id",)), + ("purchase_order_lines", ("purchase_order_id",), "purchase_orders", ("id",)), + ("employees", ("store_id",), "stores", ("id",)), + ("tour_groups", ("agency_id",), "agencies", ("id",)), + ("tour_assignments", ("tour_group_id",), "tour_groups", ("id",)), + ("lost_and_found", ("park_id",), "parks", ("id",)), + # 视图没有约束,按列名同样推得出来:按日汇总的入园人数照样能按 park_id 连到景区 + ("v_daily_visits", ("park_id",), "parks", ("id",)), +}) + +#: 看起来像关联、命名推断必须放过的列:(表, 列) → 原因 +NOT_INFERRED = { + ("employees", "employee_id"): "自引用:工号,不是指向本表", + ("tour_assignments", "guide_id"): "多个候选:guide 与 guides", + ("parking_records", "lot_id"): "没有名为 lot / lots 的表", + ("operation_logs", "operator_id"): "没有名为 operator / operators 的表", + ("categories", "parentId"): "没有名为 parent / parents 的表", + ("feedback_forms", "store_id"): "类型对不上:文字列指向整数主键", +} + +#: 唯一约束 / 唯一索引:表 → 列组 +UNIQUE = { + "parks": [["park_code"]], "channels": [["channel_code"]], "orders": [["order_no"]], + "members": [["member_no"]], "visits": [["ticket_no"]], "stores": [["store_code"]], + "products": [["sku"]], "coupons": [["code"]], "dim_dates": [["biz_date"]], +} + +_PARKS = [(1, "P01", "青松谷景区", "甲市", "2015-05-01"), (2, "P02", "白鹭湾景区", "乙市", "2018-04-20"), + (3, "P03", "星河乐园", "甲市", "2021-10-01")] +_CHANNELS = [(1, "WEB", "官网", "线上直销"), (2, "MINI", "小程序", "线上直销"), (3, "BOX", "窗口售票", "线下"), + (4, "AGT", "旅行社分销", "分销"), (5, "OTA1", "第三方平台甲", "分销"), (6, "OTA2", "第三方平台乙", "分销")] +_LEVELS = [(1, "普通会员", 1.0), (2, "银卡会员", 0.95), (3, "金卡会员", 0.9), (4, "年卡会员", 0.0)] + + +def _day(rng: random.Random) -> str: + month = rng.choice((7, 8, 9)) + return f"2026-{month:02d}-{rng.randint(1, 30 if month == 9 else 31):02d}" + + +def build(path: str | Path, *, seed: int = 20261003) -> Path: + """在 path 写一个新的景区业务库并返回路径。path 已存在时先删掉:同样的种子造出同样的内容。""" + path = Path(path) + if path.exists(): + path.unlink() + rng = random.Random(seed) + db = sqlite3.connect(path) + try: + db.executescript(DDL) + db.executemany("INSERT INTO parks VALUES (?,?,?,?,?)", _PARKS) + db.executemany("INSERT INTO areas VALUES (?,?,?)", + [(i, 1 + (i - 1) // 2, f"{'东西南北中'[i % 5]}片区") for i in range(1, 7)]) + db.executemany("INSERT INTO gates VALUES (?,?,?,?)", + [(i, 1 + (i - 1) % 3, f"G{i:02d}", f"{i} 号闸口") for i in range(1, 9)]) + db.executemany("INSERT INTO channels VALUES (?,?,?,?)", _CHANNELS) + types = [(i, 1 + (i - 1) // 3, f"T{i:02d}", name, price) + for i, (name, price) in enumerate([("成人票", 120.0), ("儿童票", 60.0), ("夜场票", 80.0)] * 3, 1)] + db.executemany("INSERT INTO ticket_types VALUES (?,?,?,?,?)", types) + db.executemany("INSERT INTO member_levels VALUES (?,?,?)", _LEVELS) + db.executemany("INSERT INTO members VALUES (?,?,?,?,?,?)", + [(i, f"M{i:05d}", f"会员{i}", rng.randint(1, 4), _day(rng), 1 if rng.random() > 0.05 else 0) + for i in range(1, 201)]) + db.executemany('INSERT INTO "memberTags" VALUES (?,?)', + [(i, t) for i, t in enumerate(["亲子", "夜游", "摄影", "年卡", "团队"], 1)]) + db.executemany("INSERT INTO member_tag_links VALUES (?,?,?)", + [(i, rng.randint(1, 200), rng.randint(1, 5)) for i in range(1, 301)]) + orders, items = [], [] + for i in range(1, 601): + qty = rng.randint(1, 4) + tid = rng.randint(1, 9) + amount = qty * types[tid - 1][4] + orders.append((i, f"OD{i:06d}", rng.choice([None, rng.randint(1, 200)]), rng.randint(1, 6), + f"{_day(rng)} {rng.randint(8, 20):02d}:{rng.randint(0, 59):02d}:00", amount, + 1 if rng.random() > 0.08 else 2)) + items.append((i, i, tid, qty, types[tid - 1][4], amount)) + db.executemany("INSERT INTO orders VALUES (?,?,?,?,?,?,?)", orders) + db.executemany("INSERT INTO order_items VALUES (?,?,?,?,?,?)", items) + visits = [(i, f"TK{i:07d}", 1 + i % 3, rng.randint(1, 8), rng.randint(1, 9), rng.choice([None, rng.randint(1, 200)]), + f"{_day(rng)} {rng.randint(8, 21):02d}:{rng.randint(0, 59):02d}:00", rng.randint(1, 5), + 1 if rng.random() > 0.04 else 0) for i in range(1, 1501)] + db.executemany("INSERT INTO visits VALUES (?,?,?,?,?,?,?,?,?)", visits) + db.executemany("INSERT INTO channel_visits VALUES (?,?,?)", [(rng.randint(1, 6), i, 1.0) for i in range(1, 1501)]) + db.executemany("INSERT INTO stores VALUES (?,?,?,?,?,?)", + [(i, f"S{i:03d}", f"{i} 号门店", 1 + i % 3, 1 + i % 6, "2020-01-01") for i in range(1, 11)]) + db.executemany("INSERT INTO categories VALUES (?,?,?)", + [(1, "餐饮", None), (2, "文创", None), (3, "饮品", 1), (4, "小吃", 1), (5, "纪念品", 2)]) + db.executemany("INSERT INTO products VALUES (?,?,?,?,?)", + [(i, f"SKU{i:04d}", f"商品{i}", rng.randint(1, 5), float(rng.randint(5, 200))) for i in range(1, 41)]) + db.executemany("INSERT INTO store_sales VALUES (?,?,?,?,?,?,?)", + [(i, rng.randint(1, 10), rng.randint(1, 40), rng.choice([None, rng.randint(1, 200)]), + f"{_day(rng)} 12:00:00", rng.randint(1, 3), float(rng.randint(5, 300))) for i in range(1, 801)]) + db.executemany("INSERT INTO dim_dates VALUES (?,?,?,?)", + [(20260700 + d, f"2026-07-{d:02d}", d % 7, int(d % 7 in (5, 6))) for d in range(1, 32)]) + db.executemany("INSERT INTO holidays VALUES (?,?,?)", [("2026-10-01", "国庆节", 1), ("2026-10-02", "国庆节", 1)]) + db.executemany("INSERT INTO sys_config VALUES (?,?)", [("currency", "CNY"), ("timezone", "+08:00")]) + db.executemany("INSERT INTO guide VALUES (?,?)", [(1, "旧导游档案")]) + db.executemany("INSERT INTO guides VALUES (?,?,?)", [(1, "导游甲", 1), (2, "导游乙", 1)]) + db.commit() + finally: + db.close() + return path diff --git a/backend/tests/test_introspect_constraints.py b/backend/tests/test_introspect_constraints.py new file mode 100644 index 00000000..963de20b --- /dev/null +++ b/backend/tests/test_introspect_constraints.py @@ -0,0 +1,119 @@ +"""探查时补读外键约束和唯一约束 / 唯一索引,以及数据目录在合成大库上的起草效果。 + +数据目录的关系起草靠它:外键记成有确证的关系,唯一约束用来判断一对一。读不到(方言不支持、权限不够、 +某个对象反射报错)就跳过那一项,不能毁掉整次探查;跳过的表不写这两个键,下游据此分得清「读过、没有」 +和「不知道」(旧缓存里也没有这两个键)。 +""" +from __future__ import annotations + +from types import SimpleNamespace + +import pytest +from sqlalchemy.engine.reflection import Inspector + +from app.data import catalog +from app.data.engine import engines +from app.data.introspect import introspect +from tests.fixtures.catalog import scenic + + +def _source(path: str, sid: str = "scenic-src") -> SimpleNamespace: + return SimpleNamespace(id=sid, name="scenic", kind="sqlite", database=path, host=None, port=None, username=None, + password=None, options={}, readonly=True, description="景区业务库", schema_cache={}, + origin="manual") + + +@pytest.fixture +async def scenic_cache(scenic_db): + source = _source(scenic_db, "scenic-cache") + try: + cache = await introspect(source) + finally: + await engines.invalidate(source.id) + return cache + + +async def test_fixture_is_large_and_complete(scenic_cache): + tables = scenic_cache["tables"] + assert len(tables) == scenic.TABLE_COUNT + len(scenic.VIEWS) > 40 + assert not scenic_cache.get("truncated") and not scenic_cache.get("skipped") + + +async def test_foreign_keys_are_read(scenic_cache): + found = {(name, tuple(fk["columns"]), fk["to_table"], tuple(fk["to_columns"])) + for name, meta in scenic_cache["tables"].items() for fk in meta["foreign_keys"]} + assert found == scenic.FK_RELATIONS + # 每张表、每个视图都读过:没有外键的是空列表,不是缺键 + assert all(isinstance(m.get("foreign_keys"), list) for m in scenic_cache["tables"].values()) + assert scenic_cache["tables"]["v_daily_visits"]["foreign_keys"] == [] + + +async def test_unique_constraints_and_unique_indexes_are_read(scenic_cache): + tables = scenic_cache["tables"] + for name, groups in scenic.UNIQUE.items(): + for group in groups: + assert group in tables[name]["unique"], (name, group) + # 主键不重复记成唯一约束;没有唯一约束的是空列表 + assert ["id"] not in tables["orders"]["unique"] + assert tables["refunds"]["unique"] == [] + + +async def test_constraint_reflection_failure_does_not_break_introspection(scenic_db, monkeypatch): + def boom(*args, **kwargs): + raise RuntimeError("permission denied for information_schema") + + for name in ("get_multi_foreign_keys", "get_foreign_keys", "get_multi_unique_constraints", + "get_unique_constraints", "get_multi_indexes", "get_indexes"): + monkeypatch.setattr(Inspector, name, boom) + source = _source(scenic_db, "scenic-broken") + try: + cache = await introspect(source) + finally: + await engines.invalidate(source.id) + assert not cache.get("failed") and len(cache["tables"]) == scenic.TABLE_COUNT + len(scenic.VIEWS) + visits = cache["tables"]["visits"] + assert "foreign_keys" not in visits and "unique" not in visits + assert visits["primary_key"] == ["id"] and visits["columns"] + + +async def test_not_implemented_constraints_are_skipped(scenic_db, monkeypatch): + def unsupported(*args, **kwargs): + raise NotImplementedError + + monkeypatch.setattr(Inspector, "get_multi_foreign_keys", unsupported) + monkeypatch.setattr(Inspector, "get_foreign_keys", unsupported) + source = _source(scenic_db, "scenic-nofk") + try: + cache = await introspect(source) + finally: + await engines.invalidate(source.id) + assert "foreign_keys" not in cache["tables"]["visits"] + assert isinstance(cache["tables"]["visits"]["unique"], list) + + +# ---------------------------------------------------------------- 在合成大库上起草 + + +async def test_structure_draft_on_scenic_db(scenic_db, scenic_cache): + source = SimpleNamespace(**{**vars(_source(scenic_db)), "schema_cache": scenic_cache}) + fk, name = set(), set() + for table in scenic_cache["tables"]: + for rel in catalog.draft_structure(source, table).notes.get("relations", []): + entry = (table, tuple(rel["columns"]), rel["to_table"], tuple(rel["to_columns"])) + (fk if rel["source"] == "fk" else name).add(entry) + assert rel["status"] == ("verified" if rel["source"] == "fk" else "proposed") + assert fk == scenic.FK_RELATIONS + assert name == scenic.NAME_RELATIONS + inferred_columns = {(t, c[0]) for t, c, _, _ in name} + assert not inferred_columns & set(scenic.NOT_INFERRED) + + +async def test_join_path_through_link_table(scenic_cache): + graph = catalog.relation_graph({}, schema_cache=scenic_cache) + paths = catalog.join_paths(graph, "visits", "channels") + via = [tuple(e.to_table for e in p) for p in paths] + assert ("channel_visits", "channels") in via + best = paths[0] + assert all(e.status == "verified" for e in best) # 两跳都是外键,排在推断的路径前面 + assert [e.condition() for e in best] == ["visits.id = channel_visits.visit_id", + "channel_visits.channel_id = channels.id"]