From 298e52bd6985411e8c286f038e8b2681467e0989 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 15:36:21 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8C=89=E5=A4=8D=E6=A0=B8=E6=84=8F=E8=A7=81?= =?UTF-8?q?=E4=BF=AE=E5=90=8E=E5=8F=B0=E6=8E=A2=E6=9F=A5=E7=9A=84=E4=B8=89?= =?UTF-8?q?=E5=A4=84=E9=80=80=E5=8C=96=EF=BC=88A-1=E3=80=81A-2=E3=80=81A-3?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A-1 升级后第一次后台探查中途停下,表结构缩成已读完的那几批: schema_cache 只从对象索引物化,老缓存(升级前同步探查写的,没有 scope 摘要)里的表结构却从没进过对象索引。 于是第一次探查只要停止、连接断开、服务重启,或者排队时有人按需加入、探查失败要记一笔,物化出来的就只剩已读完的表, 老缓存里还没重读的表静默消失,skipped 里也没有。先在探查范围面板刷新过对象索引、再探查却连不上时更糟:表结构清成空的。 现在这个 schema 的对象索引第一次建起来时(refresh_index 的 legacy,source_objects.legacy_seed),把老缓存里的表结构 接进对应的行:在范围里、算规则命中,带老结构和它的指纹,结构版本记 0(第一次真读到记 1,不把反射格式的差异当成 结构变化);老缓存里读不到结构的照样记成读不到。缓存的 schema 和配置对不上的不接,对象索引里没有的不接。第一次 任务开始时照规则重圈,没重读到的沿用老结构,和以后每次探查中途停下一样。 A-2 schema 填错、一个对象都没有时不再给候选 schema: PostgreSQL、Oracle 的 schema 写错时字典查询只回 0 行,不抛错,新路径就只物化出 tables={} 并盖上同步时间, available_schemas 没了:卡片上「该服务器上还有……」、助手摘要里「数据可能在这些 schema」都给不出。现在 prepare 拿到 0 个对象时照旧用 list_schemas 拿候选,记在任务的 scope 里,物化时由 rebuild_schema_cache 写进 available_schemas (不置 failed:这不是一次失败);有了对象再探查就不再挂着。 A-3 排队中「加入范围」的表在任务开始时被移出: _apply 只保留建任务时圈好的表和所有 on_demand 的行,目录页显式加入(explicit)的表被当成离开范围、清掉结构。 现在保留 added_at 不早于建任务时刻的行(按需加入、显式加入都算),原因按 pinned > 显式 > 规则 > 按需取高的; 建任务之前就按需加入、又被显式移出的不再一律留着,照规则移出(以前「留着所有 on_demand」会让显式移出不生效)。 测试(tests/test_introspect_jobs.py):60 张表的老缓存,第一批读完后停止 / 第二批连接出错 / 重启标中断 / dry_run 后探查失败,表数都不减少,再探查一次结构版本全是 1;别的 schema 的老缓存不接;空库和替身模拟的 0 对象 schema 带上 候选、DataSourceOut 和助手摘要都看得到;排队中显式加入、按需加入的表任务跑完仍在范围里。去掉三处修复各自回跑, 对应的测试都失败。 --- backend/app/data/introspect_jobs.py | 68 +++++--- backend/app/data/source_objects.py | 75 ++++++++- backend/tests/test_introspect_jobs.py | 215 ++++++++++++++++++++++++++ 3 files changed, 329 insertions(+), 29 deletions(-) diff --git a/backend/app/data/introspect_jobs.py b/backend/app/data/introspect_jobs.py index 37d9205e..244377a9 100644 --- a/backend/app/data/introspect_jobs.py +++ b/backend/app/data/introspect_jobs.py @@ -4,12 +4,14 @@ 进行时,目录和结构会对不上。现在: - **建任务**(prepare,在请求里做完):先刷新对象索引(data/source_objects:一条字典查询,查不了退回 Inspector;连接本身 - 失败抛 IntrospectFailed,失败照实记进 schema_cache,表结构照旧留着),再按探查范围的规则圈表;规则命中超过上限抛 - ScopeTooLarge(说出命中多少张),不建任务、不截断。圈好的范围写进对象索引,任务逐表一行(按名字排)。同一个源同时只有 - 一个排队或进行中的探查任务(IntrospectBusy;部分唯一索引兜底)。 + 失败抛 IntrospectFailed,失败照实记进 schema_cache,表结构照旧留着;升级前探查过的源第一次建索引时接上老缓存里的表结构, + 见 source_objects.legacy_seed;一个对象都没有时记下这台服务器上能选的 schema),再按探查范围的规则圈表;规则命中超过 + 上限抛 ScopeTooLarge(说出命中多少张),不建任务、不截断。任务逐表一行(按名字排)。同一个源同时只有一个排队或进行中的 + 探查任务(IntrospectBusy;部分唯一索引兜底)。 - **在后台反射**(manager.start):照 core/job_runtime 的生命周期,和同源的起草、剖析排同一条队(introspect),轮到了才开始。 - 按批(source_objects.REFLECT_BATCH 张一批)只读反射,每批开始前重读数据源(停用了就停下),读完的表逐表记 ok / failed、 - 写进对象索引(指纹变了结构版本加 1)。全部读完由 rebuild_schema_cache 物化 schema_cache、盖上同步时间。 + 开始时才把圈好的范围写进对象索引(建任务之后才加进范围的表照旧留着)。按批(source_objects.REFLECT_BATCH 张一批)只读 + 反射,每批开始前重读数据源(停用了就停下),读完的表逐表记 ok / failed、写进对象索引(指纹变了结构版本加 1)。全部读完 + 由 rebuild_schema_cache 物化 schema_cache、盖上同步时间。 - **停止**(cancel):排队的直接 cancelled;进行中的停在当前这一批之后(库那边按单条时限截止),没读的表 cancelled;读完的 照常物化进 schema_cache(范围内已反射的行),任务 cancelled。 - **服务重启**(startup):排队、进行中的标 interrupted,没读的表 cancelled、error「服务重启,探查中断」,同样把已读完的物化 @@ -216,15 +218,20 @@ def _refuse(row: Any) -> None: raise IntrospectRefused(RECORDS_REFUSED) -async def _failure_marker(row: Any, cause: BaseException) -> dict[str, Any]: - """对象清单拿不到时记进 schema_cache 的失败标记:原因(驱动的原话第一行)和这台服务器上能选的 schema。""" +async def _other_schemas(row: Any) -> list[str]: + """这台服务器上能选的 schema(introspect.list_schemas:真有对象的在前);拿不到就不给。""" from app.data import introspect try: - candidates = await introspect.list_schemas(row) + return list(await introspect.list_schemas(row)) except Exception: # noqa: BLE001 - 候选也拿不到就不给 - candidates = [] - return {"failed": True, "error": first_line(cause) or describe_exception(cause), "available_schemas": candidates} + return [] + + +async def _failure_marker(row: Any, cause: BaseException) -> dict[str, Any]: + """对象清单拿不到时记进 schema_cache 的失败标记:原因(驱动的原话第一行)和这台服务器上能选的 schema。""" + return {"failed": True, "error": first_line(cause) or describe_exception(cause), + "available_schemas": await _other_schemas(row)} async def _record_failure(session: AsyncSession, row: Any, cause: BaseException) -> None: @@ -263,7 +270,7 @@ async def preview(session: AsyncSession, row: Any, *, schema: str | None = None, out.update(stored=False, objects=[o.name for o in result.objects], scope=plan.summary()) return out async with so.source_lock(row.id): - stats = await so.refresh_index(session, row.id, configured, result) + stats = await so.refresh_index(session, row.id, configured, result, legacy=row.schema_cache) await session.commit() plan = await so.plan_for(session, row, rules) out.update(stored=True, objects=[o.name for o in result.objects], scope=plan.summary(), added=stats.added, @@ -289,8 +296,12 @@ async def prepare(session: AsyncSession, row: Any, *, actor: str | None, except Exception as e: # noqa: BLE001 await _record_failure(session, row, e) raise IntrospectFailed(e) from e + # 一个对象都没有:PostgreSQL、Oracle 的 schema 名写错(大小写、拼写)时字典查询不报错、只回 0 行,最常见的就是这种。 + # 和以前同步探查一样把这台服务器上能选的 schema 记下来(物化时写进 schema_cache 的 available_schemas):卡片上「该 + # 服务器上还有……」、助手摘要里「数据可能在这些 schema」才给得出,不显示成「已同步、0 张表」了事 + candidates = await _other_schemas(row) if not result.objects else [] async with so.source_lock(source_id): - stats = await so.refresh_index(session, source_id, schema, result) + stats = await so.refresh_index(session, source_id, schema, result, legacy=row.schema_cache) await session.flush() rules = so.ScopeRules.from_options(row.options) plan = await so.plan_for(session, row, rules) @@ -298,7 +309,10 @@ async def prepare(session: AsyncSession, row: Any, *, actor: str | None, await session.commit() raise ScopeTooLarge(plan) summary = {**plan.summary(rules), "index_method": stats.method, "index_capped": stats.capped, - "index_note": stats.note, "added": stats.added, "removed": stats.removed} + "index_note": stats.note, "added": stats.added, "removed": stats.removed, + "adopted": stats.adopted} + if candidates: + summary["available_schemas"] = candidates job = IntrospectJob(id=new_id(), source_id=source_id, status="queued", trigger=trigger, schema_name=schema, scope=summary, created_by=actor) session.add(job) @@ -352,8 +366,10 @@ async def _materialize(session: AsyncSession, source_id: str, *, synced: bool, row = await session.get(DataSource, source_id) if row is None: return + scope = (job.scope or {}) if job is not None else {} async with so.source_lock(source_id): - await so.rebuild_schema_cache(session, row, synced=synced, index=_index_stats(job)) + await so.rebuild_schema_cache(session, row, synced=synced, index=_index_stats(job), + candidates=list(scope.get("available_schemas") or [])) if synced: row.schema_synced_at = utcnow() await session.commit() @@ -470,17 +486,27 @@ class _IntrospectRun(runtime.JobRun): return await runtime.claim(session, IntrospectJob, self.job_id) async def _apply(self, session: AsyncSession, source_id: str, schema: str) -> None: - """开始执行时把建任务时圈好的范围写进对象索引:离开范围的清掉结构,范围内的按原因记。建任务之后按需加入的表 - 照旧留在范围里(它们刚读过,不在这次的表单里)。""" + """开始执行时把建任务时圈好的范围写进对象索引:离开范围的清掉结构,范围内的按原因记。 + + 建任务之后才加进范围的表(added_at 不早于建任务的时刻:look、db_schema__ 按需加入的,目录页「加入范围」显式加的) + 照旧留在范围里:它们是加的那一刻当场读过结构的,不在这次的表单里;任务排在剖析后面时,用户刚看到「已加入」,任务 + 一开始就把它移出去、清掉结构,要等再探查一次才回来。原因按圈范围的先后取高的那个(pinned > 显式 > 规则 > 按需)。 + 建任务之前就按需加入的不在此列:圈范围时已经算过(plan_scope),被显式移出的照规则移出。""" + rank = {r: i for i, r in enumerate(so.SCOPE_REASONS)} reasons = {r.table_name: r.reason or "rule" for r in (await session.execute( select(IntrospectTask.table_name, IntrospectTask.reason).where(IntrospectTask.job_id == self.job_id) )).all()} + job = await session.get(IntrospectJob, self.job_id) async with so.source_lock(source_id): - for name in (await session.execute(select(SourceObject.name).where( - SourceObject.source_id == source_id, SourceObject.schema_name == schema, - SourceObject.in_scope.is_(True), SourceObject.scope_reason == "on_demand", - SourceObject.removed_at.is_(None)))).scalars(): - reasons.setdefault(name, "on_demand") + if job is not None: + for name, reason in (await session.execute(select(SourceObject.name, SourceObject.scope_reason).where( + SourceObject.source_id == source_id, SourceObject.schema_name == schema, + SourceObject.in_scope.is_(True), SourceObject.removed_at.is_(None), + SourceObject.added_at >= job.created_at))).all(): + late = reason or "on_demand" + held = reasons.get(name) + if held is None or rank.get(late, len(rank)) < rank.get(held, len(rank)): + reasons[name] = late counts = {r: 0 for r in so.SCOPE_REASONS} for reason in reasons.values(): counts[reason] = counts.get(reason, 0) + 1 diff --git a/backend/app/data/source_objects.py b/backend/app/data/source_objects.py index e6cf78a0..adff0c3a 100644 --- a/backend/app/data/source_objects.py +++ b/backend/app/data/source_objects.py @@ -7,7 +7,8 @@ sqlite_master,MySQL 查 information_schema.TABLES,PostgreSQL 查 pg_class 加 obj_description,Oracle 查 ALL_OBJECTS 加 ALL_TAB_COMMENTS)。字典查询走数据库查询的同一个入口(engine.run_query:只读守卫、单条时限);查不了退回 Inspector 的 get_table_names / get_view_names,只是没有注释。最多收 INDEX_LIMIT 个对象(按名字排序取前面的),超了照实记 capped, - 这时不判删除(没收进来的不等于不在了)。 + 这时不判删除(没收进来的不等于不在了)。升级前同步探查过的源,对象索引第一次建起来时把老缓存里的表结构接进来 + (legacy_seed):不然物化出来的会缩成这一次已经读完的那几张。 - **探查范围**(ScopeRules / plan_scope):数据源 options.introspect_scope 的规则——{mode: all|rules, include: [通配], exclude: [通配], tables: [显式], removed: [显式移出], views: bool, max_tables}。按 names.name_key 不分大小写匹配,排除优先。 有数据目录的表、已发布模板写死的 SQL 用到的表一律在范围里(pinned),规则排不掉,只有显式移出才行。规则命中(含 pinned、 @@ -443,6 +444,8 @@ class RefreshStats: method: str capped: bool note: str | None = None + #: 第一次建对象索引时从升级前的老缓存接进来几张(legacy_seed) + adopted: int = 0 def _chunks(items: list[Any], size: int = 500) -> Iterable[list[Any]]: @@ -450,13 +453,63 @@ def _chunks(items: list[Any], size: int = 500) -> Iterable[list[Any]]: yield items[i:i + size] -async def refresh_index(session: AsyncSession, source_id: str, schema: str, result: IndexResult) -> RefreshStats: +def legacy_seed(cache: Mapping[str, Any] | None, schema: str, + names: Iterable[str]) -> dict[str, dict[str, Any]]: + """升级前同步探查写的老缓存里、对象索引第一次建起来时要接进来的表:{对象索引里的名字: 这一行要带上的列}。 + + 为什么要接:schema_cache 只从对象索引物化(rebuild_schema_cache),老缓存里的表结构却从来没进过对象索引。不接的话, + 升级后第一次探查只要中途停止、连接断开、服务重启,或者排队时有人按需加入、探查失败要记一笔,物化出来的就只剩已经读完 + 的那几张:老缓存里还没重读的表静默消失,助手和问数据看到的表突然变少。接进来以后它们就是那时的探查范围(老缓存是按 + 「前 200 个对象」探的):在范围里、算规则命中,带着老结构和它的指纹;结构版本记 0(还没按对象索引读过),第一次读到 + 记 1,不把反射格式的差异当成一次结构变化。老缓存里读不到结构的(skipped)照样记成范围里读不到结构。第一次探查任务 + 开始执行时照这一次的规则重圈(_IntrospectRun._apply),规则外的照常移出;没重读到的沿用老结构,和以后每一次探查中途 + 停下时一样。 + + 只认老缓存:带 scope 摘要的是从对象索引物化的,不再接;缓存的 schema 和这一次的对不上(以前真探查能临时换 schema) + 也不接,免得把别的 schema 的结构当成这一个的。按名字对,对不上的按 name_key 对(只认唯一的);对象索引里没有的不接: + 库里已经没有这张表了。""" + if not isinstance(cache, Mapping) or "scope" in cache: + return {} + if str(cache.get("schema") or "").strip() != schema: + return {} + tables = cache.get("tables") if isinstance(cache.get("tables"), Mapping) else {} + skipped = cache.get("skipped") if isinstance(cache.get("skipped"), Mapping) else {} + if not tables and not skipped: + return {} + old: dict[str, tuple[str, Any]] = {str(n): ("meta", m) for n, m in tables.items() if isinstance(m, dict) and m} + for n, why in skipped.items(): + old.setdefault(str(n), ("error", why)) + by_key: dict[str, list[str]] = {} + for n in old: + by_key.setdefault(name_key(n), []).append(n) + out: dict[str, dict[str, Any]] = {} + for name in names: + hit = name if name in old else None + if hit is None and len(by_key.get(name_key(name), [])) == 1: + hit = by_key[name_key(name)][0] + if hit is None: + continue + what, value = old[hit] + seed: dict[str, Any] = {"in_scope": True, "scope_reason": "rule"} + if what == "meta": + seed.update(meta=value, fingerprint=fingerprint(value)) + else: + seed["reflect_error"] = str(value or "") or "读不到列" + out[name] = seed + return out + + +async def refresh_index(session: AsyncSession, source_id: str, schema: str, result: IndexResult, *, + legacy: Mapping[str, Any] | None = None) -> RefreshStats: """把字典查询的结果写进对象索引(不提交):新出现的对象加一行;还在的更新类型、注释、行数估算、last_seen_at(删了又 - 出现的清掉 removed_at);不在了的记 removed_at。索引被截断(capped)时不判删除:没收进来的不等于不在了。""" + 出现的清掉 removed_at);不在了的记 removed_at。索引被截断(capped)时不判删除:没收进来的不等于不在了。 + + legacy 是数据源此刻的 schema_cache:这个 schema 的对象索引第一次建起来时,老缓存里的表结构接进对应的行(legacy_seed)。""" now = utcnow() existing = {r.name: r for r in (await session.execute( select(SourceObject.id, SourceObject.name, SourceObject.removed_at, SourceObject.kind, SourceObject.comment, SourceObject.est_rows).where(*_scope_filter(source_id, schema)))).all()} + seeds = legacy_seed(legacy, schema, (o.name for o in result.objects)) if not existing else {} seen: set[str] = set() new_rows: list[dict[str, Any]] = [] changed: list[dict[str, Any]] = [] @@ -470,7 +523,7 @@ async def refresh_index(session: AsyncSession, source_id: str, schema: str, resu new_rows.append({"source_id": source_id, "schema_name": schema, "name": obj.name, "name_key": name_key(obj.name), "kind": obj.kind, "comment": obj.comment, "est_rows": obj.est_rows, "quote_name": obj.quote, "first_seen_at": now, - "last_seen_at": now}) + "last_seen_at": now, **seeds.get(obj.name, {})}) continue if old.removed_at is not None: revived.append(old.id) @@ -502,7 +555,7 @@ async def refresh_index(session: AsyncSession, source_id: str, schema: str, resu await session.execute(update(_SO).where(_SO.id.in_(chunk)).values(removed_at=now, in_scope=False) .execution_options(synchronize_session=False)) return RefreshStats(total=len(seen), added=len(new_rows), removed=len(gone), method=result.method, - capped=result.capped, note=result.note) + capped=result.capped, note=result.note, adopted=len(seeds)) async def live_objects(session: AsyncSession, source_id: str, schema: str) -> list[tuple[str, str]]: @@ -656,14 +709,17 @@ _FAILURE_KEYS = ("failed", "error", "available_schemas") async def rebuild_schema_cache(session: AsyncSession, row: Any, *, failure: Mapping[str, Any] | None = None, keep_failure: bool = False, synced: bool = False, - index: RefreshStats | None = None) -> dict[str, Any]: + index: RefreshStats | None = None, + candidates: list[str] | None = None) -> dict[str, Any]: """从范围内已反射的行物化 schema_cache(唯一写它的地方),写进 row(不提交),返回新的缓存。 形状和以前一样:tables(表在前、视图在后,各按名字)、truncated(有范围外的对象)、total(对象索引里的对象总数)、 schema、synced_at;范围里却读不到结构的进 skipped(同以前的「跳过的对象」);另加 scope 摘要。failure 给了的记上 探查失败(failed、error、available_schemas,表结构照旧留着:助手还能按上一次读到的写 SQL);keep_failure 时沿用缓存里 原有的失败标记(按需加入不改它)。synced:这次是一次探查完成(synced_at 换成现在),否则沿用原来的。index:这次刷新 - 对象索引的结果(字典查询还是 Inspector、有没有被截断),不给沿用缓存里原来记的。""" + 对象索引的结果(字典查询还是 Inspector、有没有被截断),不给沿用缓存里原来记的。candidates:对象索引一个对象都没有时 + 这台服务器上能选的 schema(建任务时拿的,这里不连库),写进 available_schemas——schema 填错多半是这样,不是失败, + failed 不置。""" from app.data.source_objects import ScopeRules as _Rules schema = index_schema(row) @@ -709,6 +765,8 @@ async def rebuild_schema_cache(session: AsyncSession, row: Any, *, failure: Mapp } if skipped: cache["skipped"] = skipped + if candidates and not total: + cache["available_schemas"] = list(candidates) if failure: cache.update({k: failure[k] for k in _FAILURE_KEYS if k in failure}) elif keep_failure: @@ -976,7 +1034,8 @@ __all__ = [ "INDEX_LIMIT", "IndexResult", "IndexedObject", "OutsideObject", "REFLECT_BATCH", "RULES_MAX", "Reflected", "RefreshStats", "SCOPE_DEFAULT", "SCOPE_MAX", "SCOPE_OPTION", "SCOPE_REASONS", "ScopePlan", "ScopeRules", "Trail", "VIA_CATALOG", "VIA_LOOK", "VIA_SCHEMA_TOOL", "absent_reasons", "add_to_scope", "apply_scope", - "count_added_in_run", "fingerprint", "has_index", "index_schema", "index_sql", "list_objects", "live_objects", + "count_added_in_run", "fingerprint", "has_index", "index_schema", "index_sql", "legacy_seed", "list_objects", + "live_objects", "on_demand_keys", "on_demand_list", "outside_names", "outside_objects", "pinned_keys", "plan_for", "plan_scope", "rebuild_schema_cache", "reflect", "refresh_index", "rules_problem", "save_reflected", "search_index", "source_lock", "target_schema", "too_large_message", "trail_dict", diff --git a/backend/tests/test_introspect_jobs.py b/backend/tests/test_introspect_jobs.py index 794d9900..feb03bda 100644 --- a/backend/tests/test_introspect_jobs.py +++ b/backend/tests/test_introspect_jobs.py @@ -449,3 +449,218 @@ async def test_draft_and_profile_jobs_say_they_wait_for_introspect(small): for model, job_id in ((CatalogProfileJob, profile.id), (CatalogDraftJob, draft.id)): await session.delete(await session.get(model, job_id)) await session.commit() + + +# -------------------------------------------------------------------------- +# 升级路径:升级前同步探查写的老缓存(没有 scope,也没有对象索引) +# -------------------------------------------------------------------------- + +LEGACY_TABLES = 60 + + +@pytest.fixture +async def legacy(client, tmp_path): + """60 张表的源,schema_cache 是升级前同步探查写的那一份:没有 scope 摘要,对象索引一行都没有。""" + from app.data import introspect + from app.db.base import utcnow + + path = tmp_path / "legacy.db" + db = sqlite3.connect(path) + for i in range(LEGACY_TABLES): + db.execute(f"CREATE TABLE t{i:02d} (id INTEGER PRIMARY KEY, amount INTEGER)") + db.commit() + db.close() + sid = await _make(client, str(path)) + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + row.schema_cache = await introspect.introspect(row) + row.schema_synced_at = utcnow() + await session.commit() + cache = await _cache(sid) + assert len(cache["tables"]) == LEGACY_TABLES and "scope" not in cache + assert not await _rows(sid) + yield sid + await drop_source(sid) + + +def _second_batch(monkeypatch, *, fail: bool = False) -> None: + """第一批照常读完;第二批起一直卡着(等着被停止),fail 时直接连接出错。""" + real = so.reflect + calls = 0 + + async def gated(*args, **kwargs): + nonlocal calls + calls += 1 + if calls > 1: + if fail: + raise RuntimeError("connection reset by peer") + await asyncio.Event().wait() + return await real(*args, **kwargs) + + monkeypatch.setattr(so, "reflect", gated) + + +async def _until_first_batch(client, sid: str, job_id: str) -> None: + for _ in range(500): + await asyncio.sleep(0.02) + if (await client.get(f"/api/datasources/{sid}/introspect-jobs/{job_id}")).json()["job"]["done"] \ + >= so.REFLECT_BATCH: + return + raise AssertionError("第一批一直没读完") + + +@pytest.mark.parametrize("ending", ["cancel", "fail"]) +async def test_the_first_introspect_after_upgrade_never_shrinks_the_old_cache(client, legacy, monkeypatch, ending): + """升级后第一次后台探查中途停止、连接断开:还没重读的表照旧用老缓存里的那一份,表数不减少。""" + _second_batch(monkeypatch, fail=ending == "fail") + job = (await client.post(f"/api/datasources/{legacy}/introspect")).json()["job"] + assert job["total"] == LEGACY_TABLES + if ending == "cancel": + await _until_first_batch(client, legacy, job["id"]) + r = await client.post(f"/api/datasources/{legacy}/introspect-jobs/{job['id']}/cancel") + assert r.status_code == 200 and r.json()["job"]["status"] == "cancelled", r.text + else: + await introspect_jobs.manager.wait(job["id"], timeout=60) + status = (await client.get(f"/api/datasources/{legacy}/introspect-jobs/{job['id']}")).json()["job"] + assert status["status"] == "failed" and status["counts"]["ok"] == so.REFLECT_BATCH + cache = await _cache(legacy) + assert len(cache["tables"]) == LEGACY_TABLES, f"老缓存里还没重读的表不见了:只剩 {len(cache['tables'])} 张" + assert not cache.get("skipped") + rows = await _rows(legacy) + fresh = [r for r in rows.values() if r.struct_version == 1] + assert len(fresh) == so.REFLECT_BATCH, "这次读到的才有结构版本" + assert all(r.meta and r.in_scope for r in rows.values()), "没重读的沿用老缓存的结构,仍在范围里" + + monkeypatch.undo() + out = await introspect_now(client, legacy) + assert out["table_count"] == LEGACY_TABLES + assert {r.struct_version for r in (await _rows(legacy)).values()} == {1}, "老缓存接进来的不算一次结构变化" + + +async def test_a_dry_run_then_a_failed_introspect_keeps_the_old_cache(client, legacy, monkeypatch): + """先在探查范围面板刷新了对象索引(dry_run),再点探查却连不上:失败照实记上,老缓存的表一张不少。""" + r = await client.post(f"/api/datasources/{legacy}/introspect", params={"dry_run": "true"}) + assert r.status_code == 200 and r.json()["stored"] is True, r.text + + async def down(*_a, **_k): + raise RuntimeError("could not connect to server") + + monkeypatch.setattr(so, "list_objects", down) + r = await client.post(f"/api/datasources/{legacy}/introspect") + assert r.status_code == 400 and r.json()["code"] == "introspect_failed", r.text + cache = await _cache(legacy) + assert cache["failed"] is True and len(cache["tables"]) == LEGACY_TABLES + + +async def test_a_restart_during_the_first_introspect_keeps_the_old_cache(client, legacy): + from app.db.base import utcnow + + async with SessionLocal() as session: + row = await session.get(DataSource, legacy) + queued, n = await introspect_jobs.prepare(session, row, actor="tester") + assert n == LEGACY_TABLES + async with SessionLocal() as session: + await session.execute(update(IntrospectJob).where(IntrospectJob.id == queued.id) + .values(status="running", started_at=utcnow())) + await session.commit() + assert await introspect_jobs.startup() >= 1 + assert len((await _cache(legacy))["tables"]) == LEGACY_TABLES + + +async def test_an_old_cache_from_another_schema_is_not_carried_over(client, legacy): + """老缓存是别的 schema 探出来的(以前真探查能临时换 schema):不接进这个 schema 的对象索引。""" + async with SessionLocal() as session: + row = await session.get(DataSource, legacy) + row.schema_cache = {**row.schema_cache, "schema": "elsewhere"} + await session.commit() + r = await client.post(f"/api/datasources/{legacy}/introspect", params={"dry_run": "true"}) + assert r.status_code == 200, r.text + assert not any(r.in_scope or r.meta for r in (await _rows(legacy)).values()) + + +# -------------------------------------------------------------------------- +# 一张表都没有:说出这台服务器上还有哪些 schema +# -------------------------------------------------------------------------- + + +async def test_an_empty_schema_lists_the_other_schemas(client, tmp_path): + """schema 填错(或者是空的):字典查询不报错、只是 0 行。照旧把候选 schema 记进表结构缓存,卡片和助手摘要都给得出提示, + 不显示成「已同步、0 张表」了事。""" + from app.data import introspect + + path = tmp_path / "empty.db" + sqlite3.connect(path).close() + sid = await _make(client, str(path)) + try: + out = await introspect_now(client, sid) + assert out["table_count"] == 0 and out["schema_error"] == "" + assert "main" in out["available_schemas"], out + cache = await _cache(sid) + assert "main" in cache["available_schemas"] and not cache.get("failed") + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + assert "数据可能在这些 schema:main" in introspect.summary(row) + listed = next(s for s in (await client.get("/api/datasources")).json() if s["id"] == sid) + assert "main" in listed["available_schemas"] + + # 建了表再探查:有对象了,候选不再挂着 + db = sqlite3.connect(path) + db.execute("CREATE TABLE alpha (id INTEGER PRIMARY KEY)") + db.commit() + db.close() + out = await introspect_now(client, sid) + assert out["table_count"] == 1 and out["available_schemas"] == [] + finally: + await drop_source(sid) + + +async def test_an_empty_schema_on_a_server_with_other_schemas(client, small, monkeypatch): + """PostgreSQL、Oracle 的 schema 写错时字典查询只回 0 行:候选照 list_schemas 给(这里用替身模拟服务器上的几个 schema)。""" + sid, _ = small + from app.data import introspect + + async def nothing(_source, _schema): + return so.IndexResult(objects=[], method="dictionary", total=0) + + async def candidates(_source): + return ["sales_prod", "sales_dev"] + + monkeypatch.setattr(so, "list_objects", nothing) + monkeypatch.setattr(introspect, "list_schemas", candidates) + out = await introspect_now(client, sid) + assert out["table_count"] == 0 and out["available_schemas"] == ["sales_prod", "sales_dev"] + + +# -------------------------------------------------------------------------- +# 排队中的变化:任务开始时不把建任务之后加进来的表移出去 +# -------------------------------------------------------------------------- + + +async def test_tables_added_while_the_job_is_queued_stay_in_scope(client, small): + sid, _ = small + await _scope(client, sid, mode="rules", include=["alpha"]) + await introspect_now(client, sid) + # 建任务之前按需加入、后来被显式移出的:这次探查要把它移出去 + got = await so.add_to_scope(sid, ["eta"], trail=so.Trail(via=so.VIA_LOOK, actor="tester")) + assert got.added == ["eta"] + await _scope(client, sid, mode="rules", include=["alpha"], removed=["eta"]) + held = runtime.lanes.enter(sid, "profile", "someone-else") + try: + job = (await client.post(f"/api/datasources/{sid}/introspect")).json()["job"] + assert job["status"] == "queued" and job["total"] == 1 + r = await client.post(f"/api/datasources/{sid}/catalog/scope/add", json={"tables": ["beta"]}) + assert r.status_code == 200 and r.json()["added"] == ["beta"], r.text + got = await so.add_to_scope(sid, ["gamma"], trail=so.Trail(via=so.VIA_LOOK, actor="tester")) + assert got.added == ["gamma"] + # 任务还没开始:范围还是上一次的(eta 还在),新加的两张已经进了表结构 + assert set((await _cache(sid))["tables"]) == {"alpha", "beta", "gamma", "eta"} + finally: + held.release() + await introspect_jobs.manager.wait(job["id"], timeout=30) + assert (await client.get(f"/api/datasources/{sid}/introspect-jobs/{job['id']}")).json()["job"]["status"] \ + == "succeeded" + assert set((await _cache(sid))["tables"]) == {"alpha", "beta", "gamma"}, "排队时加进来的表被任务移出去了" + rows = await _rows(sid) + assert rows["beta"].in_scope and rows["beta"].scope_reason == "explicit" and rows["beta"].meta + assert rows["gamma"].in_scope and rows["gamma"].scope_reason == "on_demand" + assert not rows["eta"].in_scope, "建任务之前按需加入、又被显式移出的,这次探查照规则移出"