From 1a6082f03341f6d70e0f27b70bc16bd6b4d46b1a Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Fri, 16 Oct 2026 14:17:06 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A4=A7=E5=BA=93=E7=9A=84=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=E7=B4=A2=E5=BC=95=E3=80=81=E6=8E=A2=E6=9F=A5=E8=8C=83=E5=9B=B4?= =?UTF-8?q?=E5=92=8C=E5=90=8E=E5=8F=B0=E6=8E=A2=E6=9F=A5=EF=BC=88next-roun?= =?UTF-8?q?d=2003=20A1=EF=BC=89=EF=BC=8Cschema=5Fview=20=E9=A2=84=E7=AD=9B?= =?UTF-8?q?=E4=B8=8E=E6=8C=89=E9=9C=80=E5=8A=A0=E5=85=A5=E3=80=81=E5=89=96?= =?UTF-8?q?=E6=9E=90=E4=BA=8B=E5=AE=9E=E7=9A=84=E5=90=8E=E7=AB=AF=EF=BC=88?= =?UTF-8?q?A2=E3=80=81=E9=81=97=E7=95=99=20e=20=E7=9A=84=E5=BA=95=E5=BA=A7?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为什么:几千张表的库以前用不了——探查每个源最多取 200 个对象、按数据库列出的顺序截断,同步请求、整份替换、不排队; 落在窗口外的表在目录、look、剖析里被说成「表已不在」。 做了什么: - db/models_sources.py:source_objects(对象索引一行一个对象:范围、原因、结构指纹与 struct_version、first_seen / last_seen / removed_at、reflected_at、按需加入的留痕)、introspect_jobs / introspect_tasks(后台探查任务与逐表状态)。 - data/source_objects.py:每种方言一条字典查询建对象索引(失败退回 Inspector,上限 5 万,截断时不判删除);探查范围规则 (options.introspect_scope:通配、按 name_key、排除优先、显式列出 / 移出;有目录的表、已发布模板写死 SQL 用到的表 pinned; 命中超上限拒绝并说出命中数);唯一的 rebuild_schema_cache 从范围内已反射的行物化 schema_cache(形状不变,另加 scope 摘要); 三种「不在」分开说(removed / missing / out_of_scope / reflect_failed);按需加入(当场只读反射、落盘、留痕、进程内按源加锁)。 - data/introspect_jobs.py:探查改成后台任务(core/job_runtime,lane kind introspect,和同源起草、剖析串行;按批反射、逐表记状态、 可停止、读完的照样物化;重启标中断不续跑)。 - api/datasources.py:POST /introspect 返回 202 和任务,dry_run 同步只刷新索引、返回命中数;/introspect-jobs 读任务、停止; /introspect-scope 读、预览、保存规则;/schema?table= 带剖析事实、说清三种「不在」;DataSourceOut 加 schema_scope、introspect_job。 - schema_view:search_tables(BM25 倒排,遮罩列只用列名,范围外只按名字和注释、权重低一档)、profile_lines(剖析事实一节, 附「只供写查询参考」,遮罩按渲染那一刻重滤,上传表格快照不一致时不给)、absent_detail。 - look:候选超过 PREFILTER_LIMIT 时只给预筛结果 + 已用到的表 + 一跳关联;点名或挑中范围外的表按需加入(每次查看最多 4 张), look 记录加 added_to_scope、prefiltered;表详情带剖析事实。 - tools/datasource:db_schema__ 加 keyword、不传 table 时只列前 50 张;点名范围外的表按需加入(每次运行最多 8 张);查询结果 照实列「未按目录检查的表」(unchecked_tables,不新增检查规则);建工具时读剖析事实。 - 剖析:_relation_ends 分清「范围外,可以加入」和「表结构里没有」;任务记上传表格的 snapshot_id;last_facts。表助手、/schema 用同一段剖析事实。 - 目录接口:清单只列范围内,absent_reason 分开说;GET /catalog/search 查对象索引,POST /catalog/scope/add 加入范围。 - 第 1 波留下的一处:check_valid_filter 和 stored_valid_filter 拼有效记录条件时按 quote_name 强制加引号。 - 工具参数纠错:空着的可选参数不止一个时,只认拼写明显接近的那一个(db_schema 多了 keyword 以后 tabel 仍能纠成 table)。 - 测试:3,000 张表的合成库(对象索引、缺省范围超上限拒绝、规则圈 400 张后台读完、停止、重启标中断、和剖析排队、同步起草 409)、删除与结构版本、Inspector 回退、截断不判删除、范围规则、pinned、方言金样;改了同步探查的老测试改为等任务做完。 --- backend/app/api/catalog.py | 112 +- backend/app/api/coded.py | 18 +- backend/app/api/copilot.py | 3 +- backend/app/api/copilot_context.py | 5 +- backend/app/api/datasources.py | 295 +++++- backend/app/data/catalog.py | 2 +- backend/app/data/catalog_assistant.py | 8 + backend/app/data/catalog_jobs.py | 10 +- backend/app/data/catalog_profile.py | 37 +- backend/app/data/catalog_profile_jobs.py | 60 +- backend/app/data/introspect.py | 228 ++-- backend/app/data/introspect_jobs.py | 556 ++++++++++ backend/app/data/schema_view.py | 278 ++++- backend/app/data/source_objects.py | 978 ++++++++++++++++++ backend/app/data/sqlcheck.py | 7 +- backend/app/db/models_sources.py | 134 ++- backend/app/look/base.py | 10 +- backend/app/look/service.py | 5 + backend/app/look/tables.py | 334 +++++- backend/app/main.py | 8 + backend/app/tools/datasource.py | 224 +++- backend/app/tools/registry.py | 7 + backend/tests/fixtures/catalog/large.py | 91 ++ backend/tests/fixtures/sources.py | 18 + backend/tests/test_catalog_api.py | 5 +- backend/tests/test_catalog_assistant.py | 5 +- backend/tests/test_catalog_profile.py | 8 +- .../tests/test_catalog_schema_truncated.py | 89 +- backend/tests/test_datasource_preview.py | 32 +- backend/tests/test_introspect_jobs.py | 418 ++++++++ 30 files changed, 3684 insertions(+), 301 deletions(-) create mode 100644 backend/app/data/introspect_jobs.py create mode 100644 backend/app/data/source_objects.py create mode 100644 backend/tests/fixtures/catalog/large.py create mode 100644 backend/tests/test_introspect_jobs.py diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 2f70d78e..f5a85a6b 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -26,6 +26,8 @@ POST /catalog/import 导入(data/catalog_transfer.py) - POST /catalog/{table}/assistant 表助手:对话完善这张表的目录(text/event-stream),提案不写目录 (data/catalog_assistant.py) +- GET /catalog/search 在对象索引里按名字、注释找表(含探查范围外的);POST /catalog/scope/add 把范围外的表加进 + 探查范围(当场只读读结构、记进规则的显式列出,next-round 03 A2) 规则都在 data/catalog.py,这里只做取源、转换形状和把异常翻成状态码:版本不符 409,结构不合规、审阅路径 不对 422,源或表不存在 404。所有写操作记下署名(api/deps.actor_of:开了登录是认证过的用户名,否则请求头 X-Actor 的自报)。 @@ -44,12 +46,12 @@ from pydantic import BaseModel, Field, ValidationError from sqlalchemy.ext.asyncio import AsyncSession from app.api.coded import ( - CATALOG_DRAFT_BUSY, CATALOG_DRAFT_FINISHED, CATALOG_DRAFT_MODEL_UNAVAILABLE, CATALOG_PROFILE_FINISHED, PROFILE_BUSY, - PROFILE_NOTHING, SCHEMA_MISSING, SNAPSHOT_TAMPERED, CodedHTTPException, + CATALOG_DRAFT_BUSY, CATALOG_DRAFT_FINISHED, CATALOG_DRAFT_MODEL_UNAVAILABLE, CATALOG_PROFILE_FINISHED, + INTROSPECT_BUSY, PROFILE_BUSY, PROFILE_NOTHING, SCHEMA_MISSING, SNAPSHOT_TAMPERED, CodedHTTPException, ) from app.api.deps import actor_of from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, catalog_profile_jobs, introspect -from app.data import table_versions +from app.data import introspect_jobs, source_objects, table_versions from app.data import catalog_assistant, catalog_review, catalog_transfer from app.data.engine import SnapshotTampered from app.db.base import get_session @@ -144,7 +146,7 @@ def _item_value(notes: dict[str, Any], name: str) -> tuple[Any, str | None]: def _summary_row(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogEntry | None, usage: int, last_draft: dict[str, Any] | None = None, - profiling: dict[str, Any] | None = None) -> dict[str, Any]: + profiling: dict[str, Any] | None = None, absent: str | None = None) -> dict[str, Any]: notes = entry.notes if entry else {} label, label_status = _item_value(notes, "label") kind, _ = _item_value(notes, "kind") @@ -153,8 +155,11 @@ def _summary_row(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogE "table_name": name, "qualified": (meta or {}).get("qualified", name), "is_view": bool((meta or {}).get("is_view")), - # 表结构里已经没有这张表(被删了、改名了),目录还留着:界面据此提示清理 + # 表结构里没有这张表,目录还留着:界面据此提示。为什么不在(absent_reason)分开说:removed 表已不在(真删除)、 + # missing 数据源里从来没有、out_of_scope 在探查范围外、reflect_failed 在范围里却读不到结构、unknown 说不清 + # (升级前只探查了一部分的老缓存)。只有 removed 才说「表已不在」 "in_schema": meta is not None, + "absent_reason": None if meta is not None else absent, "label": label, "label_status": label_status, "kind": kind, @@ -173,9 +178,10 @@ def _summary_row(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogE def _profiling(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogEntry | None, *, - tables: dict[str, Any], masked: set[str], last: dict[str, Any] | None) -> dict[str, Any]: + tables: dict[str, Any], masked: set[str], last: dict[str, Any] | None, + outside: frozenset[str] = frozenset()) -> dict[str, Any]: notes = entry.notes if entry else {} - planned = catalog_profile.plan_checks(name, meta, notes, tables=tables, masked=masked) + planned = catalog_profile.plan_checks(name, meta, notes, tables=tables, masked=masked, outside=outside) checks = sum(1 for c in planned if c.skip is None) reason = catalog_profile_jobs.due_reason(meta, notes, last, planned) return {"last_profile": catalog_profile_jobs.public_last(last), "profile_due": reason is not None, @@ -200,9 +206,12 @@ async def _detail(session: AsyncSession, source: Any, table: str, entry: catalog meta = ((source.schema_cache or {}).get("tables") or {}).get(table) if usage is None: usage = (await catalog.table_usage(session, source)).get(table, 0) + absent = None if meta is not None else (await _absent(session, source, [table])).get(table) return { "table_name": table, "in_schema": meta is not None, + # 表结构里没有时为什么(同表清单的 absent_reason) + "absent_reason": absent, "notes": entry.notes if entry else {}, "version": entry.version if entry else 0, "updated_at": _iso(entry.updated_at) if entry else None, @@ -236,11 +245,12 @@ def _invalid(e: catalog.CatalogInvalid) -> HTTPException: @router.get("/{source_id}/catalog") async def list_catalog(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - """表清单:表结构里的每张表各一行(没有目录的也列),使用次数多的在前;目录还在、表已不在的排最后。 + """表清单:探查范围内、读到了结构的每张表各一行(没有目录的也列),使用次数多的在前;目录还在、表结构里没有的排最后, + 各带 absent_reason(表已不在、数据源里没有、在探查范围外、读不到结构,见 _summary_row)。 - 探查结构每个源最多取 200 张表(introspect._MAX_TABLES),多出来的不在表结构里、这里也列不出:schema_truncated - 说明有没有截断,schema_total 是数据库里一共几张表(含视图),界面据此说明「只探查了前 200 张」。老缓存没记 - 总数时按探查到的张数算。 + 几千张表的库只列探查范围内的(next-round 03 A1):范围外的在对象索引里,用 GET /catalog/search 找、POST /catalog/scope/add + 加进来。schema_truncated 是「有范围外的对象」,schema_total 是对象索引里一共几张(含视图);scope 是范围的摘要 + (schema_cache 的 scope:in_scope、reflected、failed、out_of_scope、cap、各原因几张),升级前探查的源为 null。 """ row, source = await _resolved(session, source_id) cache = source.schema_cache or {} @@ -250,16 +260,20 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi drafts = await catalog_jobs.last_drafts(session, source.id) lasts = await catalog_profile_jobs.last_profiles(session, source.id) masked = catalog_profile.masked_of(row.options) + outside = await catalog_profile_jobs.outside_of(session, row) def profiling(name: str, meta: dict[str, Any] | None) -> dict[str, Any]: - return _profiling(name, meta, entries.get(name), tables=tables, masked=masked, last=lasts.get(name)) + return _profiling(name, meta, entries.get(name), tables=tables, masked=masked, last=lasts.get(name), + outside=outside) order = {name: i for i, name in enumerate(tables)} rows = [_summary_row(name, meta, entries.get(name), usage.get(name, 0), drafts.get(name), profiling(name, meta)) for name, meta in tables.items()] rows.sort(key=lambda r: (-r["usage"], order[r["table_name"]])) - rows += [_summary_row(name, None, entry, 0, drafts.get(name), profiling(name, None)) - for name, entry in sorted(entries.items()) if name not in tables] + gone = [name for name in sorted(entries) if name not in tables] + absent = await _absent(session, row, gone) + rows += [_summary_row(name, None, entries[name], 0, drafts.get(name), profiling(name, None), absent.get(name)) + for name in gone] return { "tables": rows, "system_notes": catalog.system_notes_source(source), @@ -267,6 +281,7 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi "schema_note": None if tables else introspect.why_empty(cache), "schema_truncated": bool(cache.get("truncated")), "schema_total": max(_int(cache.get("total")), len(tables)), + "scope": dict(cache["scope"]) if isinstance(cache.get("scope"), dict) else None, # 数据源级的起草要求(没有是 null) "guidance": catalog_jobs.guidance_of(row), # 排队或进行中的起草任务 {id, status, total, done, waiting_for},界面据此显示进度;没有是 null @@ -277,6 +292,71 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi } +async def _absent(session: AsyncSession, row: DataSource, names: list[str]) -> dict[str, str]: + """目录还在、表结构里没有的表为什么不在(source_objects.absent_reasons)。上传表格、记录表的结构随版本走:不在就是那一版 + 里没有这张表(removed)。""" + if not names: + return {} + if (row.origin or "manual") != "manual": + return dict.fromkeys(names, source_objects.ABSENT_REMOVED) + return await source_objects.absent_reasons(session, row, names) + + +# ---- 对象索引里找表、加入探查范围(要登记在 GET /catalog/{table} 前面:不然 search 会被当成表名) + + +@router.get("/{source_id}/catalog/search") +async def search_catalog_tables(source_id: str, q: str = Query("", max_length=200), + limit: int = Query(50, ge=1, le=200), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """在对象索引里按名字、注释找表(不分大小写的包含):{items: [{name, qualified, kind, comment, est_rows, in_scope, + scope_reason, absent_reason}], total, indexed}。清单只列探查范围内的表,范围外的靠这里找到、再「加入范围」。没有对象 + 索引的源(上传表格、升级前探查的)只在表结构里找(indexed=false)。""" + row, source = await _resolved(session, source_id) + cache = source.schema_cache or {} + schema = cache.get("schema") + text = q.strip() + if (row.origin or "manual") == "manual" and await source_objects.has_index( + session, row.id, source_objects.index_schema(row)): + items, total = await source_objects.search_index(session, row, text, limit=limit) + for item in items: + item["qualified"] = f"{schema}.{item['name']}" if schema else item["name"] + return {"items": items, "total": total, "indexed": True} + from app.data.names import name_key + + key = name_key(text) + hits = [(n, m) for n, m in (cache.get("tables") or {}).items() if key and ( + key in name_key(n) or text in str((m or {}).get("comment") or ""))] + return {"items": [{"name": n, "qualified": (m or {}).get("qualified", n), + "kind": "view" if (m or {}).get("is_view") else "table", "comment": (m or {}).get("comment"), + "est_rows": None, "in_scope": True, "scope_reason": None, "absent_reason": None} + for n, m in hits[:limit]], "total": len(hits), "indexed": False} + + +class ScopeAddIn(BaseModel): + #: 要加进探查范围的表(对象索引里的名字),一次最多 ADD_LIMIT 张 + tables: list[str] = Field(min_length=1, max_length=source_objects.ADD_LIMIT) + + +@router.post("/{source_id}/catalog/scope/add") +async def add_catalog_scope(source_id: str, payload: ScopeAddIn, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """把对象索引里探查范围外的表加进探查范围:当场只读读结构、写进表结构(schema_cache),并记进探查范围规则的「显式 + 列出」,以后每次探查都带上;留痕记谁、什么时候。返回 {added, already, failed: [{name, error}], refused: [{name, reason}], + scope}。上传表格、记录表 409(结构随版本走)。""" + row = await _source_row(session, source_id) + if (row.origin or "manual") != "manual": + raise HTTPException(409, "上传的表格、记录表没有探查范围:结构随版本走") + got = await source_objects.add_to_scope(row.id, [str(t) for t in payload.tables], reason="explicit", + trail=source_objects.Trail(via=source_objects.VIA_CATALOG, + actor=actor_of(x_actor))) + cache = got.cache if got.cache is not None else (row.schema_cache or {}) + return {"added": got.added, "already": got.already, + "failed": [{"name": n, "error": e} for n, e in got.failed.items()], + "refused": [{"name": n, "reason": r} for n, r in got.refused.items()], + "scope": dict(cache["scope"]) if isinstance(cache.get("scope"), dict) else None} + + # ---- 审阅队列、审阅进度(要登记在 GET /catalog/{table} 前面:不然 review-queue、review-progress 会被当成表名) @@ -360,6 +440,8 @@ async def import_catalog(source_id: str, payload: CatalogImportIn, x_actor: str raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e except catalog_profile_jobs.ProfileBusy as e: raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e + except introspect_jobs.IntrospectBusy as e: + raise CodedHTTPException(409, str(e), INTROSPECT_BUSY) from e except catalog_transfer.TransferInvalid as e: raise HTTPException(422, str(e)) from e return report.as_dict() @@ -697,6 +779,8 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn, raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e except catalog_profile_jobs.ProfileBusy as e: raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e + except introspect_jobs.IntrospectBusy as e: + raise CodedHTTPException(409, str(e), INTROSPECT_BUSY) from e rows = [{"table_name": r.table, "added": r.added, "updated": r.updated, "removed": r.removed, "version": r.version, "error": r.error, "model_error": r.model_error, "model_items": r.model_items} for r in report.tables] diff --git a/backend/app/api/coded.py b/backend/app/api/coded.py index 4227b648..1f5d6bf6 100644 --- a/backend/app/api/coded.py +++ b/backend/app/api/coded.py @@ -28,6 +28,15 @@ CATALOG_PROFILE_FINISHED = "catalog_profile_finished" CATALOG_DRAFT_BUSY = "catalog_draft_busy" CATALOG_DRAFT_MODEL_UNAVAILABLE = "catalog_draft_model_unavailable" CATALOG_DRAFT_FINISHED = "catalog_draft_finished" +#: 后台探查(api/datasources.py 的 /introspect,next-round 03 A1):同一个源已有排队或进行中的探查任务(409,界面改去 +#: GET /introspect-jobs/active 显示那一个);探查范围的规则命中超过上限(409,另带 hits、cap、pinned,不建任务、不截断); +#: 要停止的探查任务已经结束了(409);对象清单都拿不到(400,连不上、schema 不对:失败照实记进了表结构缓存) +INTROSPECT_BUSY = "introspect_busy" +SCOPE_TOO_LARGE = "scope_too_large" +INTROSPECT_FINISHED = "introspect_finished" +INTROSPECT_FAILED = "introspect_failed" +#: 探查范围的规则不合规(422) +SCOPE_INVALID = "scope_invalid" # ---------------------------------------------------------------- 集成接口(04 细案 A7) # 外部系统用访问密钥调的应用、待办、工件接口,以及角色、运行读权限的拒绝:按 code 认是哪种错,detail 照旧是给人看的那句 @@ -80,17 +89,20 @@ PUBLIC_URL_ENV_LOCKED = "public_url_env_locked" class CodedHTTPException(HTTPException): - """field:出错的是哪一项(表单据此把报错落到那一格)。可选,没有就不出现在返回里。""" + """field:出错的是哪一项(表单据此把报错落到那一格)。可选,没有就不出现在返回里。extra:给程序读的几个数(探查范围 + 超上限时的命中数、上限),并进返回体;不能盖掉 detail、code、field。""" - def __init__(self, status_code: int, detail: str, code: str, *, field: str | None = None) -> None: + def __init__(self, status_code: int, detail: str, code: str, *, field: str | None = None, + extra: dict[str, object] | None = None) -> None: super().__init__(status_code, detail) self.code = code self.field = field + self.extra = {k: v for k, v in (extra or {}).items() if k not in ("detail", "code", "field")} async def coded_handler(_: Request, exc: Exception) -> JSONResponse: assert isinstance(exc, CodedHTTPException) - content: dict[str, str] = {"detail": exc.detail, "code": exc.code} + content: dict[str, object] = {**getattr(exc, "extra", {}), "detail": exc.detail, "code": exc.code} if exc.field: content["field"] = exc.field return JSONResponse(status_code=exc.status_code, content=content, headers=exc.headers) diff --git a/backend/app/api/copilot.py b/backend/app/api/copilot.py index f8b9e200..91022193 100644 --- a/backend/app/api/copilot.py +++ b/backend/app/api/copilot.py @@ -115,7 +115,8 @@ async def _look_service(session: AsyncSession, payload: GenerateIn, sources: lis 里给哪一份数据目录(data/catalog_view.py),不给是当前目录。""" spec = await copilot_model_spec(session, payload, max_tokens=copilot_context.PICK_MAX_TOKENS) return await copilot_context.look_service(sources, spec=spec, graph=graph, kinds=kinds, - answer=payload.intent == "answer", catalog_view=catalog_view) + answer=payload.intent == "answer", catalog_view=catalog_view, + trail={"conversation_id": getattr(payload, "conversation_id", None)}) NODE_REFERENCE = """\ diff --git a/backend/app/api/copilot_context.py b/backend/app/api/copilot_context.py index 7375d5f0..92621802 100644 --- a/backend/app/api/copilot_context.py +++ b/backend/app/api/copilot_context.py @@ -66,16 +66,17 @@ SYSTEM_LOOK_UPGRADE_NEED = "要升级的工作流里,查库节点的 SQL 用 async def look_service(sources: list[Any], *, spec: ModelSpec | None, graph: Mapping[str, Any] | None = None, kinds: tuple[str, ...] = ("table", "capability"), answer: bool = False, - catalog_view: Any = None) -> LookService: + catalog_view: Any = None, trail: Mapping[str, Any] | None = None) -> LookService: """按入口组装这一轮的查看服务。读目录、读能力登记都用各自的短会话,开流前后都能调。 spec 是助手的模型规格:挑选模型第一次要用时才按它拿(关思考、额度按 look/pick.PICK_MAX_TOKENS)。graph 是现有工作流,概览里列出它查过的表。 answer:问数据(意图为回答),能力只列只读的。catalog_view:表详情里给哪一份数据目录(data/catalog_view.py),不给是当前目录。 + trail:按需加入探查范围时的留痕(哪一轮对话:{conversation_id};谁取这次请求的身份)。 """ picker = Picker(spec) made: list[Any] = [] if "table" in kinds: - made.append(await TableKind.load(sources, graph=graph, picker=picker, catalog_view=catalog_view)) + made.append(await TableKind.load(sources, graph=graph, picker=picker, catalog_view=catalog_view, trail=trail)) if "capability" in kinds: made.append(await CapabilityKind.load(picker=picker, read_only=answer)) return LookService(made) diff --git a/backend/app/api/datasources.py b/backend/app/api/datasources.py index 3f8a972f..e386e8fc 100644 --- a/backend/app/api/datasources.py +++ b/backend/app/api/datasources.py @@ -17,14 +17,18 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.api import health -from app.api.coded import PROFILE_SETTINGS_INVALID, CodedHTTPException -from app.api.deps import signer_of, signer_verified +from app.api.coded import ( + INTROSPECT_BUSY, INTROSPECT_FAILED, INTROSPECT_FINISHED, PROFILE_SETTINGS_INVALID, SCOPE_INVALID, SCOPE_TOO_LARGE, + CodedHTTPException, +) +from app.api.deps import actor_of, signer_of, signer_verified from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw from app.core.config import settings from app.core.crypto import encrypt, mask from app.data import catalog as data_catalog from app.data.catalog_profile import profile_settings_rejection, without_legacy from app.data import introspect as introspect_mod +from app.data import introspect_jobs, source_objects from app.data import table_versions from app.data.engine import ( CATALOG_DRAFT_GUIDANCE_OPTION, MASK_COLUMNS_OPTION, MAX_QUERY_TIMEOUT_S, QUERY_TIMEOUT_OPTION, SUPPORTED_KINDS, @@ -113,6 +117,12 @@ class DataSourceOut(BaseModel): current_recipe: dict[str, Any] | None = None #: 这个源最近一个未结束的导入暂存区:{id, kind, status, created_at} open_staging: dict[str, Any] | None = None + #: 探查范围的摘要(schema_cache 的 scope:in_scope、reflected、failed、out_of_scope、cap、index_total、各原因几张); + #: 升级前探查的源、上传表格没有,为 None + schema_scope: dict[str, Any] | None = None + #: 最近一个探查任务(进行中、排队中的,或者最近结束的那个):{id, status, total, done, failed, cancelled, waiting_for, + #: finished_at, error};没探查过(或者是升级前同步探查的)为 None + introspect_job: dict[str, Any] | None = None def _cached_schema(cache: dict[str, Any]) -> str | None: @@ -130,7 +140,8 @@ def _import_mode(row: DataSource) -> str | None: #: options 里由别的接口维护的项:数据源表单不显示、不提交它们,整份回写 options 时原样留着(同模型接入 extra 里的 #: last_check)。数据目录的起草要求只经 PUT /catalog/guidance 改(api/catalog.py):表单把认不得的项放进「高级参数」 #: 的单行输入框,多行的要求会被压成一行 -_MANAGED_OPTIONS = (CATALOG_DRAFT_GUIDANCE_OPTION,) +#: 探查范围(introspect_scope)同理:只经 PUT /introspect-scope 改,表单整份回写 options 时原样留着 +_MANAGED_OPTIONS = (CATALOG_DRAFT_GUIDANCE_OPTION, source_objects.SCOPE_OPTION) def _shown_options(options: dict[str, Any] | None) -> dict[str, Any]: @@ -144,11 +155,13 @@ def _with_managed(options: dict[str, Any], saved: dict[str, Any] | None) -> dict def _to_out(row: DataSource, snapshot: dict[str, Any] | None = None, - recipe_info: dict[str, Any] | None = None) -> DataSourceOut: + recipe_info: dict[str, Any] | None = None, job: dict[str, Any] | None = None) -> DataSourceOut: cache = row.schema_cache or {} tables = cache.get("tables") or {} recipe_info = recipe_info or {} return DataSourceOut( + schema_scope=dict(cache["scope"]) if isinstance(cache.get("scope"), dict) else None, + introspect_job=job, origin=row.origin or "manual", current_snapshot=snapshot, import_mode=_import_mode(row), current_recipe=recipe_info.get("current_recipe"), open_staging=recipe_info.get("open_staging"), @@ -277,7 +290,8 @@ async def _recipe_infos(session: AsyncSession, rows: list[DataSource]) -> dict[s async def _out(session: AsyncSession, row: DataSource) -> DataSourceOut: return _to_out(row, (await _snapshot_infos(session, [row])).get(row.id), - (await _recipe_infos(session, [row])).get(row.id)) + (await _recipe_infos(session, [row])).get(row.id), + await introspect_jobs.job_brief(session, await introspect_jobs.latest_job(session, row.id))) #: 上传源上由系统维护的字段:连接一律从快照解析,database 只作显示 @@ -356,7 +370,8 @@ async def list_sources(session: AsyncSession = Depends(get_session)) -> list[Dat rows = list((await session.execute(select(DataSource).order_by(DataSource.name))).scalars()) snaps = await _snapshot_infos(session, rows) recipes = await _recipe_infos(session, rows) - return [_to_out(r, snaps.get(r.id), recipes.get(r.id)) for r in rows] + jobs = await introspect_jobs.latest_briefs(session, [r.id for r in rows if not _is_upload(r) and not _is_records(r)]) + return [_to_out(r, snaps.get(r.id), recipes.get(r.id), jobs.get(r.id)) for r in rows] #: 每种库都有的「查询时限」。数据库按它自己停下语句,不只是后端不再等 @@ -805,64 +820,238 @@ async def _test_saved(session: AsyncSession, row: DataSource) -> dict[str, Any]: class IntrospectPreview(BaseModel): - """只看不存的探查结果。缓存、同步时间都没动。""" + """只看不存的探查结果(dry_run):刷新对象索引(配置里的 schema)或者只看(别的 schema),按探查范围的规则数命中。 + 不反射、不动表结构缓存和同步时间。""" dry_run: bool = True #: 这次探的是哪个 schema,"" 是默认 schema schema_: str = Field(default="", serialization_alias="schema") + #: 对象索引里有几个对象(表和视图) table_count: int = 0 - #: 带 schema 前缀的对象全名,最多 200 个(和真探查的上限一样) + #: 带 schema 前缀的对象全名,最多 200 个(举例用;全部的在对象索引里) tables: list[str] = Field(default_factory=list) + #: 对象多于上面列出的 truncated: bool = False total: int = 0 schema_error: str = "" available_schemas: list[str] = Field(default_factory=list) + #: 配置里的那个 schema:对象索引存下来了(别的 schema 只看不存) + stored: bool = False + #: 按探查范围的规则数的命中(source_objects.ScopePlan.summary:hits、cap、in_scope、pinned、rule、explicit、on_demand、 + #: out_of_scope、index_total);over 是超没超上限 + scope: dict[str, Any] = Field(default_factory=dict) + #: 对象清单是怎么拿的:dictionary 字典查询 / inspector 退回 Inspector(没有注释);capped 超过了对象索引的上限 + index_method: str = "" + index_capped: bool = False -@router.post("/{source_id}/introspect", response_model=DataSourceOut | IntrospectPreview) +#: 预览时举例列出的对象最多几个 +_PREVIEW_NAMES = 200 + + +def _preview_response(preview: IntrospectPreview) -> JSONResponse: + """dry_run 是同步的只看不存:200(真探查才是 202 建任务)。""" + return JSONResponse(status_code=200, content=preview.model_dump(by_alias=True, mode="json")) + + +def _explained(e: BaseException, row: DataSource) -> str: + reason, hint = _explain_connect(e, (row.kind or "").lower()) + return f"探查表结构失败:{reason}" + (f"。{hint}" if hint else "") + + +def _introspect_refusal(row: DataSource) -> None: + if _is_upload(row): + raise HTTPException(409, introspect_jobs.UPLOAD_REFUSED) + if _is_records(row): + raise HTTPException(409, introspect_jobs.RECORDS_REFUSED) + + +def _too_large(e: introspect_jobs.ScopeTooLarge) -> CodedHTTPException: + plan = e.plan + return CodedHTTPException(409, str(e), SCOPE_TOO_LARGE, extra={ + "hits": plan.capped_count, "cap": plan.cap, "pinned": plan.counts.get("pinned", 0), + "in_scope": len(plan.in_scope), "index_total": plan.index_total}) + + +@router.post("/{source_id}/introspect", status_code=202, response_model=None) async def introspect_source( source_id: str, schema: str | None = None, - dry_run: bool = Query(default=False, description="只返回探到的结构,不写缓存"), + dry_run: bool = Query(default=False, description="只刷新对象索引、数命中,不反射、不写表结构缓存"), + x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session), -) -> DataSourceOut | IntrospectPreview: - """探查结构并缓存。显式动作,不做后台轮询——生产库不该被实验工具定时扫。 +) -> Any: + """探查结构:后台任务(202,返回 {job, source};进度读 GET /introspect-jobs/{job_id})。 - dry_run=true 只看不存:「换个 schema 看看」以前一探就把缓存换掉,用户随后 - 选了「不改」,助手此刻看到的也已经是另一套表了。 + 请求里先刷新对象索引(一条字典查询)、按探查范围的规则圈表、核上限:规则命中超过上限 409 scope_too_large(带命中数, + 不建任务、不截断);同源已有排队或进行中的探查任务 409 introspect_busy;对象清单都拿不到 400 introspect_failed(失败 + 照实记进表结构缓存,卡片上看得到原因和能选的 schema)。任务在后台按批只读反射,和同源的起草、剖析排同一条队。 - 上传的表格不探查:结构在发布那一刻随快照冻结,重新探查只会让缓存和快照对不上。 + dry_run=true 只看不存(同步,200):配置里的 schema 刷新对象索引、返回命中数;schema 写了别的时只看那个 schema 里有 + 什么,对象索引、表结构缓存、配置都不动(「换个 schema 看看」)。 + + 上传的表格不探查:结构在发布那一刻随快照冻结,重新探查只会让缓存和快照对不上。一个数据源一个 schema(目录-D2): + 真探查只按配置里的 schema,要换 schema 先改数据源设置。 """ row = await _get_or_404(session, source_id) - if _is_upload(row): - raise HTTPException(409, "上传的表格无需重新探查,重新上传即可更新") - if _is_records(row): - raise HTTPException(409, "记录表的结构由应用的表单维护,无需重新探查") - try: - cache = await introspect_mod.introspect(row, schema=schema) - except Exception as e: # noqa: BLE001 - reason, hint = _explain_connect(e, (row.kind or "").lower()) - raise HTTPException(400, f"探查表结构失败:{reason}" + (f"。{hint}" if hint else "")) from e + _introspect_refusal(row) if dry_run: - tables = cache.get("tables") or {} - return IntrospectPreview( - schema_=str(cache.get("schema") or ""), - table_count=len(tables), - tables=[m.get("qualified", n) for n, m in tables.items()], - truncated=bool(cache.get("truncated")), - total=int(cache.get("total") or 0), - schema_error=str(cache.get("error") or "") if cache.get("failed") else "", - available_schemas=list(cache.get("available_schemas") or []), - ) - row.schema_cache = cache - # 失败也要把缓存存下来——里面的 available_schemas 正是下一步的线索—— - # 但**不能盖上"已同步"的戳**。以前盖了,于是界面显示同步过、工具却说 - # "还没探查过",两边都不说真话(run 554a0f92) - if not row.schema_cache.get("failed"): - row.schema_synced_at = datetime.now(timezone.utc) + try: + got = await introspect_jobs.preview(session, row, schema=schema) + except introspect_jobs.IntrospectFailed as e: + return _preview_response(IntrospectPreview( + schema_=schema or str(source_objects.index_schema(row)), schema_error=str(e), + available_schemas=await introspect_mod.list_schemas(row))) + names = got["objects"] + prefix = got["schema"] + return _preview_response(IntrospectPreview( + schema_=prefix, table_count=len(names), total=got["index_total"], + tables=[f"{prefix}.{n}" if prefix else n for n in names[:_PREVIEW_NAMES]], + truncated=len(names) > _PREVIEW_NAMES, stored=bool(got.get("stored")), + scope={**got["scope"], "over": got["scope"]["hits"] > got["scope"]["cap"]}, + index_method=got["index_method"], index_capped=bool(got["index_capped"]), + available_schemas=[] if names else await introspect_mod.list_schemas(row), + )) + if schema and schema.strip() and schema.strip() != source_objects.index_schema(row): + raise HTTPException(422, "一个数据源只探查配置里的那个 schema。要换 schema,请先在数据源设置里改「schema」," + "或者用「预览其他 schema」只看不存") + try: + job, _n = await introspect_jobs.prepare(session, row, actor=actor_of(x_actor)) + except introspect_jobs.IntrospectBusy as e: + raise CodedHTTPException(409, str(e), INTROSPECT_BUSY) from e + except introspect_jobs.ScopeTooLarge as e: + raise _too_large(e) from e + except introspect_jobs.IntrospectFailed as e: + raise CodedHTTPException(400, _explained(e.cause, row), INTROSPECT_FAILED) from e + introspect_jobs.manager.start(job.id) + await session.refresh(row) + return JSONResponse(status_code=202, content=jsonable_encoder({ + "job": await introspect_jobs.job_out(session, job), "source": (await _out(session, row)).model_dump()})) + + +async def _job_or_404(session: AsyncSession, source_id: str, job_id: str) -> Any: + job = await introspect_jobs.load(session, source_id, job_id) + if job is None: + raise HTTPException(404, "探查任务不存在,可能已被清理") + return job + + +@router.get("/{source_id}/introspect-jobs/active") +async def active_introspect_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """排队或进行中的探查任务;没有是 {job: null}。""" + await _get_or_404(session, source_id) + return {"job": await introspect_jobs.job_out(session, await introspect_jobs.active_job(session, source_id))} + + +@router.get("/{source_id}/introspect-jobs/latest") +async def latest_introspect_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """最近建的探查任务(任何状态);没有是 {job: null}。""" + await _get_or_404(session, source_id) + return {"job": await introspect_jobs.job_out(session, await introspect_jobs.latest_job(session, source_id))} + + +@router.get("/{source_id}/introspect-jobs/{job_id}") +async def get_introspect_job(source_id: str, job_id: str, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """一个探查任务(进度、读不到的表)和数据源此刻的样子(结束了以后卡片据此换上新的表数)。""" + row = await _get_or_404(session, source_id) + job = await _job_or_404(session, source_id, job_id) + return {"job": await introspect_jobs.job_out(session, job), "source": (await _out(session, row)).model_dump()} + + +@router.post("/{source_id}/introspect-jobs/{job_id}/cancel") +async def cancel_introspect_job(source_id: str, job_id: str, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """停止探查:排队的直接停;进行中的停在当前这一批之后,读完的表照常更新进表结构。已经结束的 409 introspect_finished。""" + await _get_or_404(session, source_id) + job = await _job_or_404(session, source_id, job_id) + try: + await introspect_jobs.cancel(session, job) + except introspect_jobs.IntrospectFinished as e: + raise CodedHTTPException(409, str(e), INTROSPECT_FINISHED) from e + session.expire_all() + row = await _get_or_404(session, source_id) + job = await _job_or_404(session, source_id, job_id) + return {"job": await introspect_jobs.job_out(session, job), "source": (await _out(session, row)).model_dump()} + + +# -------------------------------------------------------------------------- +# 探查范围 +# -------------------------------------------------------------------------- + + +class ScopeIn(BaseModel): + #: {mode, include, exclude, tables, removed, views, max_tables}(source_objects.ScopeRules)。手工读成 dict 核对:说法要 + #: 是中文的一句话(rules_problem) + scope: dict[str, Any] + + +async def _scope_out(session: AsyncSession, row: DataSource, rules: source_objects.ScopeRules | None = None, + ) -> dict[str, Any]: + """探查范围面板要的:规则、缺省和上限、按规则对着存着的对象索引数的命中、对象索引的情况、pinned 几张(举几个例子)、 + 按需加入的表和留痕。没有对象索引的源(升级前探查的、还没探查过的)命中是 null:先「刷新对象索引」或探查一次。""" + stored = source_objects.ScopeRules.from_options(row.options) + rules = rules or stored + schema = source_objects.index_schema(row) + indexed = await source_objects.has_index(session, row.id, schema) + preview: dict[str, Any] | None = None + pinned: list[str] = [] + if indexed: + plan = await source_objects.plan_for(session, row, rules) + preview = {**plan.summary(), "over": plan.over} + pinned = [n for n, r in plan.in_scope.items() if r == "pinned"][:20] + cache = row.schema_cache or {} + scope = cache.get("scope") if isinstance(cache.get("scope"), dict) else {} + return { + "scope": stored.to_dict(), + "defaults": {"max_tables": source_objects.SCOPE_DEFAULT, "max_limit": source_objects.SCOPE_MAX, + "index_limit": source_objects.INDEX_LIMIT, "rules_max": source_objects.RULES_MAX}, + "preview": preview, + "index": {"ready": indexed, "schema": schema, "method": scope.get("index_method"), + "capped": bool(scope.get("index_capped")), "at": scope.get("index_at"), + "total": scope.get("index_total")}, + "pinned_sample": pinned, + "on_demand": await source_objects.on_demand_list(session, row) if indexed else [], + } + + +def _parse_rules(raw: Any) -> source_objects.ScopeRules: + if problem := source_objects.rules_problem(raw): + raise CodedHTTPException(422, problem, SCOPE_INVALID) + return source_objects.ScopeRules.from_dict(raw) + + +@router.get("/{source_id}/introspect-scope") +async def get_introspect_scope(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """探查范围:规则、命中数(对着存着的对象索引数,不连库)、pinned、按需加入的留痕。""" + row = await _get_or_404(session, source_id) + _introspect_refusal(row) + return await _scope_out(session, row) + + +@router.post("/{source_id}/introspect-scope/preview") +async def preview_introspect_scope(source_id: str, payload: ScopeIn, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """按交来的规则数命中(不存、不连库):面板上边改边看命中多少张、超没超上限。""" + row = await _get_or_404(session, source_id) + _introspect_refusal(row) + return await _scope_out(session, row, _parse_rules(payload.scope)) + + +@router.put("/{source_id}/introspect-scope") +async def put_introspect_scope(source_id: str, payload: ScopeIn, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """保存探查范围的规则(记下谁、什么时候改的)。超上限也存得下:下次探查时会被拒绝并说出命中数,面板上提前看得到。 + 规则只在下一次探查时生效:已有的表结构不动。""" + row = await _get_or_404(session, source_id) + _introspect_refusal(row) + rules = _parse_rules(payload.scope) + rules.updated_by = actor_of(x_actor) + rules.updated_at = datetime.now(timezone.utc).isoformat() + row.options = {**(row.options or {}), source_objects.SCOPE_OPTION: rules.to_dict()} await session.commit() await session.refresh(row) - return _to_out(row) + return await _scope_out(session, row) @router.get("/{source_id}/schema") @@ -881,10 +1070,19 @@ async def get_schema(source_id: str, table: str | None = None, # 和 db_schema 工具、助手查看同一个说法(schema_view.table_detail):有数据目录的表,字段清单后面附上目录; # 遮罩列只给列名和类型 entries = await data_catalog.read_catalog(session, found.id) + from app.data import catalog_profile_jobs + + key = data_catalog.resolve_table_name(getattr(row, "schema_cache", None), table) + # 剖析事实(行数、日期范围):四处同一段文字(next-round 遗留 e) + facts = await catalog_profile_jobs.last_facts(session, found.id, [key]) if key else {} + detail = schema_view.table_detail(row, table, catalog_notes={n: e.notes for n, e in entries.items()}, + profile=facts) + if meta is None and (absent := await _absent_text(session, found, row, table)): + detail = absent return { "table": table, # 给模型看的那段文本;界面画表格用下面的结构化字段 - "detail": schema_view.table_detail(row, table, catalog_notes={n: e.notes for n, e in entries.items()}), + "detail": detail, "found": meta is not None, "qualified": meta.get("qualified", table) if meta else None, "kind": ("view" if meta.get("is_view") else "table") if meta else None, @@ -898,6 +1096,19 @@ async def get_schema(source_id: str, table: str | None = None, } +async def _absent_text(session: AsyncSession, found: DataSource, row: Any, table: str) -> str | None: + """/schema?table= 点名的表不在表结构里:有对象索引的手工源说清是哪一种「不在」(表已不在、范围外、读不到结构)。""" + from app.data import schema_view + + if _is_upload(found) or _is_records(found): + return None + reason = (await source_objects.absent_reasons(session, found, [table])).get(table) + if reason == source_objects.ABSENT_UNKNOWN or reason is None: + return None + error = ((row.schema_cache or {}).get("skipped") or {}).get(table) + return schema_view.absent_detail(row, table, reason, error=error) + + # -------------------------------------------------------------------------- # 上传表格 # -------------------------------------------------------------------------- diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 25ac4c8c..68bf3a84 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -1929,7 +1929,7 @@ def _model_facts(entry: Mapping[str, Any], meta: Mapping[str, Any], *, kind: str parsed = sqlcheck.parse_valid_filter(text, meta, kind=kind, table=table) problem = parsed.problem if problem is None: - stored = sqlcheck.stored_valid_filter(parsed, kind) + stored = sqlcheck.stored_valid_filter(parsed, kind, meta) if _text_ok(stored): out["valid_filter"] = _fact_item(stored, reason) else: diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py index 56f2160b..d98f8ed9 100644 --- a/backend/app/data/catalog_assistant.py +++ b/backend/app/data/catalog_assistant.py @@ -508,6 +508,14 @@ async def prepare(session: AsyncSession, row: Any, source: Any, table: str, if guidance := catalog_jobs.guidance_of(row): lines.append(labels["guidance"].format(text=guidance)) lines += ["", *_structure_lines(meta, masked, system=system_notes), ""] + # 剖析事实(行数估算、日期范围):和助手 look、运行时 db_schema、/schema 同一段文字(schema_view.profile_lines, + # next-round 遗留 e),遮罩按此刻的设置重滤,上传表格换了版本的不给 + from app.data import catalog_profile_jobs, schema_view + + facts = (await catalog_profile_jobs.last_facts(session, source.id, [table])).get(table) + if profile := schema_view.profile_lines(facts, masked, schema_view.snapshot_of(source), + upload=schema_view.is_upload(source)): + lines += [*profile, ""] current = _catalog_lines(notes, masked) lines += [labels["catalog"], *current] if current else [labels["catalog_empty"]] incoming = [] diff --git a/backend/app/data/catalog_jobs.py b/backend/app/data/catalog_jobs.py index 557978c4..415b4e5d 100644 --- a/backend/app/data/catalog_jobs.py +++ b/backend/app/data/catalog_jobs.py @@ -248,7 +248,10 @@ async def waiting_for(session: AsyncSession, job: CatalogDraftJob) -> str | None return None ahead = runtime.lanes.ahead(job.source_id, job.id) if ahead is not None: - return "profile" if "profile" in ahead else None + if "profile" in ahead: + return "profile" + # 排在同源的探查任务后面(next-round 03 A1:探查也进同一条队) + return "introspect" if "introspect" in ahead else None earlier = (await session.execute( select(CatalogProfileJob.id).where(CatalogProfileJob.source_id == job.source_id, CatalogProfileJob.status.in_(ACTIVE), @@ -382,6 +385,11 @@ def _lane_busy(row: Any, holder: runtime.LaneTicket, kind: str) -> Exception: then=_then(kind, "profile"))) if holder.kind == "import": return DraftBusy(import_busy_message(row)) + if holder.kind == "introspect": + # 正在探查结构(next-round 03 A1):表结构做到一半,起草、导入要等它读完 + from app.data import introspect_jobs + + return introspect_jobs.IntrospectBusy(introspect_jobs.busy_message(row, None, then=_then(kind, "profile"))) then = _then(kind, "draft") return DraftBusy(sync_busy_message(row, then) if holder.kind == "sync_draft" else busy_message(row, then)) diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 17b59a39..f4976de7 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -969,15 +969,18 @@ class ProfileContext: #: 遮罩的列(小写)。遮罩按列名生效、不分大小写、不分表,和证据面板的比法一致 masked: set[str] sizes: dict[str, TableSize] = field(default_factory=dict) + #: 对象索引里探查范围外的对象(name_key):关系指向它们时说「在探查范围外」,不说「表结构里没有」 + outside: frozenset[str] = frozenset() -def make_context(source: Any, settings: ProfileSettings, *, options: dict[str, Any] | None) -> ProfileContext: +def make_context(source: Any, settings: ProfileSettings, *, options: dict[str, Any] | None, + outside: frozenset[str] = frozenset()) -> ProfileContext: """剖析任务的上下文。options 是数据源记录的 options(遮罩按它);不支持的数据库类型抛 ValueError。""" dialect = SqlDialect(getattr(source, "kind", None)) return ProfileContext(source=source, settings=settings, dialect=dialect, runner=SourceRunner(source, settings, dialect), tables=(getattr(source, "schema_cache", None) or {}).get("tables") or {}, - masked=masked_of(options)) + masked=masked_of(options), outside=outside) def masked_of(options: dict[str, Any] | None) -> set[str]: @@ -1216,7 +1219,7 @@ async def _check_relation(cx: _TableCx, table: str, rel: dict[str, Any], out: _T """从子表取至多 sample_size 个不同的非空键值,去父表里数对上几个,得到覆盖率;再看父键、子键唯不唯一。""" # 两端在表结构里的写法;按规则不查的(多列、不在表结构里、指向视图、碰到遮罩列)plan_checks 已经列成跳过,这里再判 # 一遍只是兜底 - parent, col, pcol = _relation_ends(table, rel, cx.tables, cx.job.masked) + parent, col, pcol = _relation_ends(table, rel, cx.tables, cx.job.masked, cx.job.outside) meta, pmeta = cx.tables[table], cx.tables[parent] if rel.get("source") == "fk": @@ -1677,15 +1680,25 @@ class PlannedCheck: "detail": detail, "queries": 0} -def _relation_ends(table: str, rel: dict[str, Any], tables: dict[str, Any], masked: set[str]) -> tuple[str, str, str]: +#: 被指向的表在探查范围外(对象索引里有):不是「表结构里没有」,重新探查也没用,要把它加进探查范围 +OUT_OF_SCOPE_PARENT = "被指向的表 {table} 在探查范围外,可以在数据源的「探查范围」里加入它(或在数据目录页搜索它、加入范围)" + + +def _relation_ends(table: str, rel: dict[str, Any], tables: dict[str, Any], masked: set[str], + outside: frozenset[str] | set[str] = frozenset()) -> tuple[str, str, str]: """一条关系两端在表结构里的写法:(被指向的表, 子表的列, 被指向的列)。按规则不查的抛 _Skip:多列组成的、表结构里 - 没有的表或列、被指向的是视图、两端有被遮罩的列(遮罩的列不出现在任何一条剖析查询里)。""" + 没有的表或列、被指向的是视图、两端有被遮罩的列(遮罩的列不出现在任何一条剖析查询里)。outside 是对象索引里探查范围外 + 的对象(name_key):被指向的表在里面时说「在探查范围外,可以加入范围」,不说「表结构里没有、请重新探查」。""" + from app.data.names import name_key + cols, to_cols = list(rel.get("columns") or []), list(rel.get("to_columns") or []) to_table = str(rel.get("to_table") or "") if len(cols) != 1 or len(to_cols) != 1: raise _Skip("unsupported", "多列组成的关联关系暂不剖析") meta = tables[table] parent = to_table if to_table in tables else catalog.resolve_table_name({"tables": tables}, to_table) + if parent is None and outside and name_key(to_table.rsplit(".", 1)[-1]) in outside: + raise _Skip("out_of_scope", OUT_OF_SCOPE_PARENT.format(table=to_table)) if parent is None: raise _Skip("missing", f"表结构里没有被指向的表 {to_table},请先重新探查结构") pmeta = tables[parent] @@ -1702,12 +1715,13 @@ def _relation_ends(table: str, rel: dict[str, Any], tables: dict[str, Any], mask def plan_checks(table: str, meta: dict[str, Any] | None, notes: dict[str, Any] | None, *, tables: dict[str, Any], - masked: set[str]) -> list[PlannedCheck]: + masked: set[str], outside: frozenset[str] | set[str] = frozenset()) -> list[PlannedCheck]: """这张表要做的检查,按价值排:关系、码值、日期。范围(profile_checks)、建任务时列出的检查项、执行时用的都是这一份: 先列全再执行,中途停下时没做的每一项都要记下原因。视图、表结构里没有的表返回空列表。 按规则不查的也列出来(带 skip):被遮罩的码值列、日期列,多列的、两端不在表结构里的、指向视图的、碰到遮罩列的关系—— - 要让人看得见为什么它没有结论。masked 是遮罩的列(小写)。""" + 要让人看得见为什么它没有结论。masked 是遮罩的列(小写);outside 是对象索引里探查范围外的对象(name_key,见 + _relation_ends)。""" if not isinstance(meta, dict) or meta.get("is_view"): return [] notes = notes or {} @@ -1715,7 +1729,7 @@ def plan_checks(table: str, meta: dict[str, Any] | None, notes: dict[str, Any] | for rel in _relation_candidates(notes): skip: tuple[str, str, bool] | None = None try: - _relation_ends(table, rel, tables, masked) + _relation_ends(table, rel, tables, masked, outside) except _Skip as e: skip = (e.reason, e.detail, e.carry) checks.append(PlannedCheck("relation", _relation_target(rel), f"relations.{rel['id']}", skip, relation=rel)) @@ -1891,7 +1905,7 @@ async def profile_table(cx: ProfileContext, table: str, *, await db(_record, cx.source.id, table, None, result, actor, hooks.recorded) return result notes = await db(_read_notes, cx.source.id, table) - plan = plan_checks(table, meta, notes, tables=cx.tables, masked=cx.masked) + plan = plan_checks(table, meta, notes, tables=cx.tables, masked=cx.masked, outside=cx.outside) out = _TableOutcome() tcx = _TableCx(cx, table, result, hooks) result.checks = [p.as_check() for p in plan] @@ -2086,8 +2100,9 @@ async def check_valid_filter(source: Any, table: str, meta: dict[str, Any] | Non return _unchecked(str(e)) cap = max(int(settings.sample_size), int(settings.max_scan_rows)) # 和存进目录的是同一条(catalog._model_facts 存的就是这个写法):列换成真名、按方言加引号,别的照原文 - cond = sqlcheck.canonical_valid_filter(parsed, quote=dialect.quote) - cols = ", ".join(dialect.quote(c) for c in parsed.columns) + # 列按表结构里记的 quote_name 强制加引号(Oracle 小写带引号的列:不加引号 Oracle 转成大写,找的是另一列) + cond = sqlcheck.canonical_valid_filter(parsed, quote=lambda c: dialect.column(meta, c)) + cols = ", ".join(dialect.column(meta, c) for c in parsed.columns) inner = dialect.limit(f"SELECT {cols} FROM {dialect.table(meta, table)}", cap) sql = f"SELECT COUNT(*) AS n, SUM(CASE WHEN {cond} THEN 0 ELSE 1 END) AS x FROM ({inner}) s" try: diff --git a/backend/app/data/catalog_profile_jobs.py b/backend/app/data/catalog_profile_jobs.py index 74f9c058..f7160c3a 100644 --- a/backend/app/data/catalog_profile_jobs.py +++ b/backend/app/data/catalog_profile_jobs.py @@ -238,7 +238,8 @@ def job_dict(job: CatalogProfileJob, tasks: list[Any], *, waiting_for: str | Non async def waiting_for(session: AsyncSession, job: CatalogProfileJob) -> str | None: - """排队中的剖析任务在等什么:排在同源的起草任务(或同步起草)后面时是 "draft",否则 None。 + """排队中的剖析任务在等什么:排在同源的起草任务(或同步起草)后面时是 "draft",排在探查任务后面时是 "introspect", + 否则 None。 先看这个源的队(进程内,排在它前面的是谁);任务刚建、后台还没排进队时看库:同源有比它先建的、排队或进行中的起草 任务,或者正在同步起草。""" @@ -246,7 +247,10 @@ async def waiting_for(session: AsyncSession, job: CatalogProfileJob) -> str | No return None ahead = runtime.lanes.ahead(job.source_id, job.id) if ahead is not None: - return "draft" if any(kind in ("draft", "sync_draft") for kind in ahead) else None + if any(kind in ("draft", "sync_draft") for kind in ahead): + return "draft" + # 排在同源的探查任务后面(next-round 03 A1:探查也进同一条队) + return "introspect" if "introspect" in ahead else None from app.data import catalog_jobs if catalog_jobs.sync_drafting(job.source_id): @@ -358,6 +362,33 @@ def public_last(last: dict[str, Any] | None) -> dict[str, Any] | None: return None if last is None else {k: v for k, v in last.items() if k not in _INTERNAL} +async def last_facts(session: AsyncSession, source_id: str, + tables: list[str] | None = None) -> dict[str, dict[str, Any]]: + """每张表「上次剖析」(见 _ranked 的 shown,和表清单、表详情同一项)里给助手看的剖析事实:{表名: {at, job_id, + row_estimate, date_ranges, snapshot_id}},一条查询批量取。没有行数估算也没有日期范围的不给;tables 给了只取这几张。 + + look、运行时 db_schema、/schema、表助手四处都经 schema_view.profile_lines 渲染这一份(next-round 遗留 e):剖析数字 + 是会过期的观测,不写进数据目录;遮罩、快照在渲染那一刻再核。""" + inner = _ranked(source_id, CatalogProfileTask.table_name, CatalogProfileTask.job_id, + CatalogProfileTask.started_at, CatalogProfileTask.finished_at, CatalogProfileTask.row_estimate, + CatalogProfileTask.date_ranges, CatalogProfileTask.snapshot_id) + query = select(inner).where(inner.c.shown == 1) + if tables is not None: + if not tables: + return {} + query = query.where(inner.c.table_name.in_(list(tables))) + out: dict[str, dict[str, Any]] = {} + for row in (await session.execute(query)).mappings(): + estimate = row["row_estimate"] if isinstance(row["row_estimate"], dict) else None + ranges = [r for r in row["date_ranges"] or [] if isinstance(r, dict)] + if not estimate and not ranges: + continue + at = row["finished_at"] or row["started_at"] + out[row["table_name"]] = {"at": _iso(at), "job_id": row["job_id"], "row_estimate": estimate, + "date_ranges": ranges, "snapshot_id": row["snapshot_id"]} + return out + + async def last_detail(session: AsyncSession, source_id: str, table: str) -> dict[str, Any] | None: """这张表「上次剖析」(见 _ranked 的 shown)的明细(表详情的「数据剖析」一节);没剖析过是 None。""" found = (await session.execute( @@ -447,6 +478,18 @@ NOTHING_AT_ALL = ("没有可以剖析的表:表结构里的表都是视图, "或者都被遮罩了)。请先起草数据目录,或指定要剖析的表") +async def outside_of(session: AsyncSession, row: Any) -> frozenset[str]: + """对象索引里探查范围外的对象(name_key):剖析从范围内挑表,关系指向范围外的表时说「在探查范围外,可以加入」 + (catalog_profile._relation_ends)。上传表格、记录表、升级前探查的源没有对象索引,是空集。""" + from app.data import source_objects + + if (getattr(row, "origin", None) or "manual") != "manual": + return frozenset() + if not isinstance((getattr(row, "schema_cache", None) or {}).get("scope"), dict): + return frozenset() + return await source_objects.outside_names(session, row) + + def _skipped_task(job_id: str, seq: int, name: str, reason: str, why: str, checks: list[dict[str, Any]], skipped: list[ProfileSkip], now: Any) -> CatalogProfileTask: entries = [s.to_dict() for s in skipped] + [ProfileSkip("table", name, reason, why).to_dict()] @@ -463,13 +506,15 @@ async def _pick(session: AsyncSession, row: Any, source: Any, *, tables: list[st all_tables = cache.get("tables") or {} masked = catalog_profile.masked_of(getattr(row, "options", None)) entries = await catalog.read_catalog(session, source.id) + outside = await outside_of(session, row) def notes(name: str) -> dict[str, Any]: entry = entries.get(name) return entry.notes if entry else {} def plan(name: str) -> list[Any]: - return catalog_profile.plan_checks(name, all_tables.get(name), notes(name), tables=all_tables, masked=masked) + return catalog_profile.plan_checks(name, all_tables.get(name), notes(name), tables=all_tables, masked=masked, + outside=outside) if scope == "selected": picked: list[tuple[str, str | None, str | None, list[Any]]] = [] @@ -765,6 +810,7 @@ class _ProfileRun(runtime.JobRun): raise runtime.JobFailed(str(e)) from e settings = ProfileSettings.from_dict(job.settings) options = dict(row.options or {}) + outside = await outside_of(session, row) # 要剖析哪些表按开始这一刻的目录定:排在起草后面时,建任务那会儿起草还没提出关系 await self._write(self._rescope, row, source) async with SessionLocal() as session: @@ -772,7 +818,7 @@ class _ProfileRun(runtime.JobRun): pending = [(t.seq, t.table_name) for t in await task_rows(session, job.id) if t.status == "pending"] self.actor = job.created_by try: - self.cx = catalog_profile.make_context(source, settings, options=options) + self.cx = catalog_profile.make_context(source, settings, options=options, outside=outside) except ValueError as e: # 不支持的数据库类型 raise runtime.JobFailed(str(e)) from e hooks = TableHooks(progress=self._progress, query=self._queried, stop_text=self.stop_text) @@ -889,8 +935,10 @@ class _ProfileRun(runtime.JobRun): await session.commit() async def _start_table(self, session: AsyncSession, seq: int) -> None: + # 上传表格源记下这一刻剖析的是哪个快照:换了版本之后,旧版本的剖析事实不再交给助手(schema_view.profile_lines) + snapshot = getattr(self.cx.source, "snapshot_id", None) if self.cx is not None else None await session.execute(self._task(CatalogProfileTask.seq == seq, CatalogProfileTask.status == "pending") - .values(status="running", started_at=utcnow())) + .values(status="running", started_at=utcnow(), snapshot_id=snapshot)) await session.commit() def _elapsed(self) -> int: @@ -960,5 +1008,5 @@ manager = ProfileJobs() __all__ = ["INTERRUPTED_ERROR", "KEEP_JOBS", "MAX_TABLES", "NOTHING_AT_ALL", "NOTHING_NEEDED", "NothingToRetry", "ProfileBusy", "ProfileFinished", "ProfileJobs", "ProfileNothing", "THEN_PROFILE", "active_job", "busy_message", "cancel", "create_job", "due_reason", "ensure_free", "job_brief", "job_dict", "job_out", - "last_detail", "last_profiles", "latest_job", "load", "load_task", "manager", "prune", "public_last", + "last_detail", "last_facts", "last_profiles", "outside_of", "latest_job", "load", "load_task", "manager", "prune", "public_last", "retry_tables", "startup", "task_detail", "task_row", "task_rows", "unsaved_error", "waiting_for"] diff --git a/backend/app/data/introspect.py b/backend/app/data/introspect.py index 7d7f478b..23847c9a 100644 --- a/backend/app/data/introspect.py +++ b/backend/app/data/introspect.py @@ -248,8 +248,128 @@ def index_first_columns(indexes: list[dict[str, Any]], pk: list[str]) -> list[st return sorted(first) +def reflect_names(sync_conn: Any, target_schema: str | None, picked: list[str], + views: set[str] | frozenset[str]) -> dict[str, Any]: + """在一条同步连接上反射给出的这几个对象(名字是 Inspector 列出的写法),返回 {tables, skipped, failures}: + + - tables:{名字: 表结构}——列(名字、按方言编译的类型、可空、注释)、主键、表注释、是不是视图;读得到的话还有外键约束 + (foreign_keys)、唯一约束 / 唯一索引(unique)、打头有索引的列(index_first_columns)——读不到的表不写这几个键; + - skipped:{名字: 原因},反射不出来的对象(读不到列、权限不够); + - failures:{名字: 驱动的原话},批量失败后逐个重试时报错的那些。 + + 整库探查(introspect)和后台探查任务(data/introspect_jobs.py,按批调它)共用这一份:探查范围里的表一批一批读, + 读法、跳过的规矩和整库探查一字不差。""" + inspector = sa_inspect(sync_conn) + + # 批量反射接口(SQLAlchemy 2.0)。 + # + # **别指望它能治超时。** 实测 60 张表:逐表调用 121 次往返、批量 122 次 + # ——SQLAlchemy 的反射缓存本来就把 get_columns / get_pk_constraint 合并 + # 成了同一次底层查询,而 MySQL 方言压根没实现 get_multi_*,落到基类 + # 还是每表一次 SHOW CREATE TABLE。真正能少往返的是 PostgreSQL 这类 + # 原生实现了批量的方言。 + # + # 换成它是因为这是 2.0 的正经接口、少三层 try/except,不是因为它快。 + # shop 那次 "Lost connection to MySQL server during query" + # (run 554a0f92)多半是偶发的服务端超时——所以真正的对策不在这里, + # 在下面:失败要如实记成失败,让工具和界面都说真话。 + # + # 批量是一锤子买卖:其中一个对象反射出错,整批都拿不到。SQLite 连接关了双引号 + # 字符串兼容以后,老库里一个写着 `"yes" AS flag` 的视图就让 PRAGMA 报错;以前这里 + # 吞掉异常返回空,好好的表跟着一起消失,探查却显示「成功、0 张表」。所以批量失败 + # 时逐个对象再取一遍,只跳过真正取不到的那个,名字和原因记进 skipped。 + failures: dict[str, str] = {} + + def _multi(fn: Any, single: Any, names: list[str], *, + errors: dict[str, str] | None = None, ok: set[str] | None = None) -> dict[Any, Any]: + """批量反射,失败时逐个对象重试。ok 给了的话,记下真正读到了的对象名:批量成功时是全部 + (没有约束的表批量接口也会给空列表,或者干脆不给),逐个重试时只有没报错的那些。""" + if not names: + return {} + try: + got = dict(fn(schema=target_schema, filter_names=names, kind=ObjectKind.ANY)) + if ok is not None: + ok.update(names) + return got + except NotImplementedError: + return {} # 表注释这类不是所有方言都支持 + except Exception: # noqa: BLE001 - 批量取不到,逐个对象再试 + pass + out: dict[Any, Any] = {} + for name in names: + try: + out[(target_schema, name)] = single(name, schema=target_schema) + if ok is not None: + ok.add(name) + except NotImplementedError: + break + except Exception as e: # noqa: BLE001 - 只跳过这一个对象 + if errors is not None: + errors[name] = first_line(e) or describe_exception(e) + if not out and len(errors or ()) >= _GIVE_UP_AFTER: + break # 开头几个全取不到,多半是连接本身出了问题,不再挨个耗时间 + return out + + columns_by = _multi(inspector.get_multi_columns, inspector.get_columns, picked, + errors=failures) + readable = [n for n in picked if columns_by.get((target_schema, n))] + pk_by = _multi(inspector.get_multi_pk_constraint, inspector.get_pk_constraint, readable) + comment_by = _multi(inspector.get_multi_table_comment, inspector.get_table_comment, readable) + # 外键和唯一约束给数据目录起草关系用(data/catalog.py)。放在列、主键、注释之后读:PostgreSQL 上一条 + # 语句报错会让整个事务作废,先读的那几样不能被它连累。读不到的表不写这两个键(ok 集合里没有它), + # 下游据此分得清「读过、没有」和「不知道」——升级前探查的缓存里也没有这两个键。 + fk_ok: set[str] = set() + uq_ok: set[str] = set() + ix_ok: set[str] = set() + fk_by = _multi(inspector.get_multi_foreign_keys, inspector.get_foreign_keys, readable, errors={}, ok=fk_ok) + uq_by = _multi(inspector.get_multi_unique_constraints, inspector.get_unique_constraints, readable, + errors={}, ok=uq_ok) + ix_by = _multi(inspector.get_multi_indexes, inspector.get_indexes, readable, errors={}, ok=ix_ok) + try: + default_schema = inspector.default_schema_name + except Exception: # noqa: BLE001 - 只用来判断外键是不是指向别的 schema + default_schema = None + + tables: dict[str, Any] = {} + skipped: dict[str, str] = {} + for name in picked: + key = (target_schema, name) + columns = columns_by.get(key) + if not columns: + # 这个对象反射不出来,跳过,别毁掉整次探查;但要留下名字和原因 + skipped[name] = failures.get(name) or "读不到列" + continue + pk = (pk_by.get(key) or {}).get("constrained_columns") or [] + comment = (comment_by.get(key) or {}).get("text") + + tables[name] = { + # 写 SQL 要用的全名。没有 schema 前缀时 Oracle 直接 ORA-00942—— + # 只读账号名下没有这些对象,全靠跨 schema 授权 + "qualified": f"{target_schema}.{name}" if target_schema else name, + "schema": target_schema, + "columns": [column_entry(c, sync_conn.dialect) for c in columns], + "primary_key": pk, + "comment": comment, + "is_view": name in views, + } + if _forced_quote(name): + tables[name]["quote_name"] = True + if name in fk_ok: + tables[name]["foreign_keys"] = _foreign_keys(fk_by.get(key) or [], target_schema, default_schema) + if name in uq_ok: + tables[name]["unique"] = _unique_groups(uq_by.get(key) or [], + (ix_by.get(key) or []) if name in ix_ok else [], pk) + if name in ix_ok: + tables[name]["index_first_columns"] = index_first_columns(ix_by.get(key) or [], pk) + return {"tables": tables, "skipped": skipped, "failures": failures} + + async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any]: - """探查整个库的结构,结果直接存进 DataSource.schema_cache。 + """一次性探查整个库的结构(最多 _MAX_TABLES 个对象),返回 schema_cache 的形状,不落盘。 + + 上传表格发布新版本时核对表结构(data/table_versions.py)、测试夹具建表结构用它:对象少,一次读完。手工登记的数据库 + 源走对象索引 + 探查范围 + 后台探查任务(data/source_objects.py、data/introspect_jobs.py),schema_cache 由 + rebuild_schema_cache 从范围内已反射的行物化,不再经这里截断。 每张表:列(名字、类型、可空、注释)、主键、表注释、是不是视图;读得到的话还有外键约束 (foreign_keys)和唯一约束 / 唯一索引(unique)——读不到的表不写这两个键,探查照常完成。 @@ -259,110 +379,12 @@ async def introspect(source: Any, *, schema: str | None = None) -> dict[str, Any def _collect(sync_conn: Any) -> dict[str, Any]: inspector = sa_inspect(sync_conn) - target_schema = target - names = inspector.get_table_names(schema=target_schema) - views = inspector.get_view_names(schema=target_schema) + names = inspector.get_table_names(schema=target) + views = inspector.get_view_names(schema=target) truncated = len(names) + len(views) > _MAX_TABLES picked = (names + views)[:_MAX_TABLES] - - # 批量反射接口(SQLAlchemy 2.0)。 - # - # **别指望它能治超时。** 实测 60 张表:逐表调用 121 次往返、批量 122 次 - # ——SQLAlchemy 的反射缓存本来就把 get_columns / get_pk_constraint 合并 - # 成了同一次底层查询,而 MySQL 方言压根没实现 get_multi_*,落到基类 - # 还是每表一次 SHOW CREATE TABLE。真正能少往返的是 PostgreSQL 这类 - # 原生实现了批量的方言。 - # - # 换成它是因为这是 2.0 的正经接口、少三层 try/except,不是因为它快。 - # shop 那次 "Lost connection to MySQL server during query" - # (run 554a0f92)多半是偶发的服务端超时——所以真正的对策不在这里, - # 在下面:失败要如实记成失败,让工具和界面都说真话。 - # - # 批量是一锤子买卖:其中一个对象反射出错,整批都拿不到。SQLite 连接关了双引号 - # 字符串兼容以后,老库里一个写着 `"yes" AS flag` 的视图就让 PRAGMA 报错;以前这里 - # 吞掉异常返回空,好好的表跟着一起消失,探查却显示「成功、0 张表」。所以批量失败 - # 时逐个对象再取一遍,只跳过真正取不到的那个,名字和原因记进 skipped。 - failures: dict[str, str] = {} - - def _multi(fn: Any, single: Any, names: list[str], *, - errors: dict[str, str] | None = None, ok: set[str] | None = None) -> dict[Any, Any]: - """批量反射,失败时逐个对象重试。ok 给了的话,记下真正读到了的对象名:批量成功时是全部 - (没有约束的表批量接口也会给空列表,或者干脆不给),逐个重试时只有没报错的那些。""" - try: - got = dict(fn(schema=target_schema, filter_names=names, kind=ObjectKind.ANY)) - if ok is not None: - ok.update(names) - return got - except NotImplementedError: - return {} # 表注释这类不是所有方言都支持 - except Exception: # noqa: BLE001 - 批量取不到,逐个对象再试 - pass - out: dict[Any, Any] = {} - for name in names: - try: - out[(target_schema, name)] = single(name, schema=target_schema) - if ok is not None: - ok.add(name) - except NotImplementedError: - break - except Exception as e: # noqa: BLE001 - 只跳过这一个对象 - if errors is not None: - errors[name] = first_line(e) or describe_exception(e) - if not out and len(errors or ()) >= _GIVE_UP_AFTER: - break # 开头几个全取不到,多半是连接本身出了问题,不再挨个耗时间 - return out - - columns_by = _multi(inspector.get_multi_columns, inspector.get_columns, picked, - errors=failures) - readable = [n for n in picked if columns_by.get((target_schema, n))] - pk_by = _multi(inspector.get_multi_pk_constraint, inspector.get_pk_constraint, readable) - comment_by = _multi(inspector.get_multi_table_comment, inspector.get_table_comment, readable) - # 外键和唯一约束给数据目录起草关系用(data/catalog.py)。放在列、主键、注释之后读:PostgreSQL 上一条 - # 语句报错会让整个事务作废,先读的那几样不能被它连累。读不到的表不写这两个键(ok 集合里没有它), - # 下游据此分得清「读过、没有」和「不知道」——升级前探查的缓存里也没有这两个键。 - fk_ok: set[str] = set() - uq_ok: set[str] = set() - ix_ok: set[str] = set() - fk_by = _multi(inspector.get_multi_foreign_keys, inspector.get_foreign_keys, readable, errors={}, ok=fk_ok) - uq_by = _multi(inspector.get_multi_unique_constraints, inspector.get_unique_constraints, readable, - errors={}, ok=uq_ok) - ix_by = _multi(inspector.get_multi_indexes, inspector.get_indexes, readable, errors={}, ok=ix_ok) - try: - default_schema = inspector.default_schema_name - except Exception: # noqa: BLE001 - 只用来判断外键是不是指向别的 schema - default_schema = None - - tables: dict[str, Any] = {} - skipped: dict[str, str] = {} - for name in picked: - key = (target_schema, name) - columns = columns_by.get(key) - if not columns: - # 这个对象反射不出来,跳过,别毁掉整次探查;但要留下名字和原因 - skipped[name] = failures.get(name) or "读不到列" - continue - pk = (pk_by.get(key) or {}).get("constrained_columns") or [] - comment = (comment_by.get(key) or {}).get("text") - - tables[name] = { - # 写 SQL 要用的全名。没有 schema 前缀时 Oracle 直接 ORA-00942—— - # 只读账号名下没有这些对象,全靠跨 schema 授权 - "qualified": f"{target_schema}.{name}" if target_schema else name, - "schema": target_schema, - "columns": [column_entry(c, sync_conn.dialect) for c in columns], - "primary_key": pk, - "comment": comment, - "is_view": name in views, - } - if _forced_quote(name): - tables[name]["quote_name"] = True - if name in fk_ok: - tables[name]["foreign_keys"] = _foreign_keys(fk_by.get(key) or [], target_schema, default_schema) - if name in uq_ok: - tables[name]["unique"] = _unique_groups(uq_by.get(key) or [], - (ix_by.get(key) or []) if name in ix_ok else [], pk) - if name in ix_ok: - tables[name]["index_first_columns"] = index_first_columns(ix_by.get(key) or [], pk) + got = reflect_names(sync_conn, target, picked, set(views)) + tables, skipped, failures = got["tables"], got["skipped"], got["failures"] payload: dict[str, Any] = {"tables": tables, "truncated": truncated, "total": len(names) + len(views)} if failures and not tables: diff --git a/backend/app/data/introspect_jobs.py b/backend/app/data/introspect_jobs.py new file mode 100644 index 00000000..0cdf8b64 --- /dev/null +++ b/backend/app/data/introspect_jobs.py @@ -0,0 +1,556 @@ +"""数据源的后台探查任务(POST /api/datasources/{id}/introspect,docs/design/next-round/03-数据接入与目录维护.md A1)。 + +以前探查是一次同步请求、结果整份替换 schema_cache、不排队:MySQL 一张表一次往返,范围一大同步请求就超时;和起草、剖析同时 +进行时,目录和结构会对不上。现在: + +- **建任务**(prepare,在请求里做完):先刷新对象索引(data/source_objects:一条字典查询,查不了退回 Inspector;连接本身 + 失败抛 IntrospectFailed,失败照实记进 schema_cache,表结构照旧留着),再按探查范围的规则圈表;规则命中超过上限抛 + ScopeTooLarge(说出命中多少张),不建任务、不截断。圈好的范围写进对象索引,任务逐表一行(按名字排)。同一个源同时只有 + 一个排队或进行中的探查任务(IntrospectBusy;部分唯一索引兜底)。 +- **在后台反射**(manager.start):照 core/job_runtime 的生命周期,和同源的起草、剖析排同一条队(introspect),轮到了才开始。 + 按批(source_objects.REFLECT_BATCH 张一批)只读反射,每批开始前重读数据源(停用了就停下),读完的表逐表记 ok / failed、 + 写进对象索引(指纹变了结构版本加 1)。全部读完由 rebuild_schema_cache 物化 schema_cache、盖上同步时间。 +- **停止**(cancel):排队的直接 cancelled;进行中的停在当前这一批之后(库那边按单条时限截止),没读的表 cancelled;读完的 + 照常物化进 schema_cache(范围内已反射的行),任务 cancelled。 +- **服务重启**(startup):排队、进行中的标 interrupted,没读的表 cancelled、error「服务重启,探查中断」,同样把已读完的物化 + 进去;不自动续跑:复制出来的数据目录启动时不能自己去查业务库,界面上点「探查结构」重来。 +- **保留**(prune):每个源保留最近 KEEP_JOBS 个结束了的探查任务。 + +任务最后的状态:有表读不到是 partial,否则 succeeded;整体出错(数据源没了、连不上)是 failed。 +""" +from __future__ import annotations + +import logging +from types import SimpleNamespace +from typing import Any + +from sqlalchemy import delete, func, select, update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core import job_runtime as runtime +from app.core.errors import describe_exception, first_line +from app.core.job_runtime import ACTIVE, FINISHED, OPEN, STOP_GRACE_S +from app.data import source_objects as so +from app.db.base import SessionLocal, new_id, utcnow +from app.db.models import DataSource +from app.db.models_sources import IntrospectJob, IntrospectTask, SourceObject + +logger = logging.getLogger(__name__) + +#: 探查任务占同源队(叫法登记在运行时里):和起草、剖析、导入、同步起草串行 +runtime.register_lane_kind("introspect", "探查") + +#: 每个源保留几个结束了的探查任务 +KEEP_JOBS = 20 +#: 任务里列出的读不到的表最多几张(全部的在逐表的行里) +FAILURES_SHOWN = 50 + +INTERRUPTED_ERROR = "服务重启,探查中断" +STOPPED_ERROR = "探查已停止" +SOURCE_GONE = "数据源不存在,可能已被删除" +SOURCE_DISABLED = "数据源「{name}」已停用,探查已停止" +JOB_ERROR = "探查任务出错:{line}" +CONNECT_ERROR = "读表结构时连不上数据库:{line}" +UPLOAD_REFUSED = "上传的表格无需重新探查,重新上传即可更新" +RECORDS_REFUSED = "记录表的结构由应用的表单维护,无需重新探查" + + +class IntrospectBusy(Exception): + """这个数据源已经有排队或进行中的探查任务(message 写明进行到第几张)。""" + + +class IntrospectFinished(Exception): + """要停止的探查任务已经结束了。""" + + +class ScopeTooLarge(Exception): + """规则命中超过上限:不建任务。plan 是圈出的范围(接口回给界面命中数)。""" + + def __init__(self, plan: so.ScopePlan) -> None: + super().__init__(so.too_large_message(plan)) + self.plan = plan + + +class IntrospectFailed(Exception): + """对象清单都拿不到(连不上、schema 不存在)。cause 是原来的异常,接口按它翻译成一句话和下一步。""" + + def __init__(self, cause: BaseException) -> None: + super().__init__(first_line(cause) or describe_exception(cause)) + self.cause = cause + + +class IntrospectRefused(Exception): + """这种源不探查(上传表格、记录表)。""" + + +# ========================================================================== +# 读任务 +# ========================================================================== + + +def _iso(value: Any) -> str | None: + return value.isoformat() if value is not None else None + + +async def active_job(session: AsyncSession, source_id: str) -> IntrospectJob | None: + return (await session.execute( + select(IntrospectJob).where(IntrospectJob.source_id == source_id, IntrospectJob.status.in_(ACTIVE)) + .order_by(IntrospectJob.created_at.desc()).limit(1))).scalar_one_or_none() + + +async def latest_job(session: AsyncSession, source_id: str) -> IntrospectJob | None: + return (await session.execute( + select(IntrospectJob).where(IntrospectJob.source_id == source_id) + .order_by(IntrospectJob.created_at.desc()).limit(1))).scalar_one_or_none() + + +async def load(session: AsyncSession, source_id: str, job_id: str) -> IntrospectJob | None: + job = await session.get(IntrospectJob, job_id) + return job if job is not None and job.source_id == source_id else None + + +async def _counts(session: AsyncSession, job_id: str) -> dict[str, int]: + counts = dict.fromkeys(("pending", "running", "ok", "failed", "cancelled"), 0) + for status, n in (await session.execute( + select(IntrospectTask.status, func.count()).where(IntrospectTask.job_id == job_id) + .group_by(IntrospectTask.status))).all(): + counts[status] = counts.get(status, 0) + n + return counts + + +async def waiting_for(session: AsyncSession, job: IntrospectJob) -> str | None: + """排队中的探查任务在等什么:同源队里排在它前面的第一件事(draft 起草、profile 剖析、import 导入目录、sync_draft 同步 + 起草);任务刚建、后台还没排进队时看库:同源有比它先建的、排队或进行中的起草、剖析任务。""" + if job.status != "queued": + return None + ahead = runtime.lanes.ahead(job.source_id, job.id) + if ahead is not None: + return ahead[0] if ahead else None + from app.db.models import CatalogDraftJob, CatalogProfileJob + + for kind, model in (("draft", CatalogDraftJob), ("profile", CatalogProfileJob)): + earlier = (await session.execute( + select(model.id).where(model.source_id == job.source_id, model.status.in_(ACTIVE), + model.created_at <= job.created_at).limit(1))).first() + if earlier is not None: + return kind + return None + + +async def job_out(session: AsyncSession, job: IntrospectJob | None) -> dict[str, Any] | None: + """接口返回的任务(IntrospectJob):计数按逐表的算,读不到的表列前 FAILURES_SHOWN 张和原因,正在读的那张。""" + if job is None: + return None + counts = await _counts(session, job.id) + failures = [{"name": r.table_name, "error": r.error} for r in (await session.execute( + select(IntrospectTask.table_name, IntrospectTask.error).where( + IntrospectTask.job_id == job.id, IntrospectTask.status == "failed") + .order_by(IntrospectTask.seq).limit(FAILURES_SHOWN))).all()] + current = (await session.execute( + select(IntrospectTask.table_name).where(IntrospectTask.job_id == job.id, IntrospectTask.status == "running") + .order_by(IntrospectTask.seq).limit(1))).scalar_one_or_none() + total = sum(counts.values()) + return { + "id": job.id, "source_id": job.source_id, "status": job.status, "trigger": job.trigger, + "schema": job.schema_name, "created_by": job.created_by, "created_at": _iso(job.created_at), + "started_at": _iso(job.started_at), "finished_at": _iso(job.finished_at), + "total": total, "done": counts["ok"] + counts["failed"] + counts["cancelled"], "counts": counts, + "current": current, "failures": failures, "error": job.error, "scope": dict(job.scope or {}), + "waiting_for": await waiting_for(session, job), + } + + +async def job_brief(session: AsyncSession, job: IntrospectJob | None) -> dict[str, Any] | None: + """数据源卡片上的那一行(DataSourceOut.introspect_job):{id, status, total, done, failed, waiting_for, finished_at, + error}。""" + if job is None: + return None + counts = await _counts(session, job.id) + return {"id": job.id, "status": job.status, "total": sum(counts.values()), + "done": counts["ok"] + counts["failed"] + counts["cancelled"], "failed": counts["failed"], + "cancelled": counts["cancelled"], "waiting_for": await waiting_for(session, job), + "finished_at": _iso(job.finished_at), "error": job.error} + + +async def latest_briefs(session: AsyncSession, source_ids: list[str]) -> dict[str, dict[str, Any]]: + """各源最近一个探查任务的那一行(数据源列表一次取齐:每个源一条查询,源不多)。""" + out: dict[str, dict[str, Any]] = {} + for sid in source_ids: + brief = await job_brief(session, await latest_job(session, sid)) + if brief is not None: + out[sid] = brief + return out + + +#: 撞上探查任务时说法的最后一句缺省是开始新的探查;导入目录、同步起草撞上时各说各的(catalog_jobs._lane_busy) +THEN_INTROSPECT = "再开始新的探查" + + +def busy_message(row: Any, brief: dict[str, Any] | None, *, then: str = THEN_INTROSPECT) -> str: + name = getattr(row, "name", "") or "" + if brief is None: + return f"数据源「{name}」正在探查结构,请等它结束(或先停止它){then}" + if brief["status"] == "queued": + return f"数据源「{name}」已有一个排队中的探查任务(共 {brief['total']:,} 张表),请等它结束(或先停止它){then}" + return (f"数据源「{name}」正在探查结构:第 {min(brief['done'] + 1, brief['total']):,} / {brief['total']:,} 张表," + f"请等它结束(或先停止它){then}") + + +async def ensure_free(session: AsyncSession, row: Any) -> None: + job = await active_job(session, row.id) + if job is not None: + raise IntrospectBusy(busy_message(row, await job_brief(session, job))) + + +# ========================================================================== +# 预览、建任务 +# ========================================================================== + + +def _refuse(row: Any) -> None: + origin = getattr(row, "origin", None) or "manual" + if origin == "upload": + raise IntrospectRefused(UPLOAD_REFUSED) + if origin == "records": + raise IntrospectRefused(RECORDS_REFUSED) + + +async def _failure_marker(row: Any, cause: BaseException) -> dict[str, Any]: + """对象清单拿不到时记进 schema_cache 的失败标记:原因(驱动的原话第一行)和这台服务器上能选的 schema。""" + from app.data import introspect + + try: + candidates = await introspect.list_schemas(row) + except Exception: # noqa: BLE001 - 候选也拿不到就不给 + candidates = [] + return {"failed": True, "error": first_line(cause) or describe_exception(cause), "available_schemas": candidates} + + +async def _record_failure(session: AsyncSession, row: Any, cause: BaseException) -> None: + """失败照实记进 schema_cache:有对象索引的按行物化、带上失败标记;没有的(第一次探查就连不上)只在原来的缓存上加 + 标记。表结构照旧留着。不盖同步时间。""" + marker = await _failure_marker(row, cause) + if await so.has_index(session, row.id, so.index_schema(row)): + await so.rebuild_schema_cache(session, row, failure=marker) + else: + cache = dict(row.schema_cache or {}) + cache.setdefault("tables", {}) + cache.update(marker) + cache["schema"] = so.target_schema(row) + row.schema_cache = cache + await session.commit() + + +async def preview(session: AsyncSession, row: Any, *, schema: str | None = None, + rules: so.ScopeRules | None = None) -> dict[str, Any]: + """只看不存的探查(dry_run):拿对象清单、按规则数命中。schema 是数据源配置的那个(或者没给)时刷新存着的对象索引 + (只刷新索引,不反射、不动 schema_cache);是别的 schema 时只看不存。连不上抛 IntrospectFailed。""" + _refuse(row) + configured = so.index_schema(row) + other = bool(schema) and schema != configured + target = schema if other else so.target_schema(row) + try: + result = await so.list_objects(row, target) + except Exception as e: # noqa: BLE001 + raise IntrospectFailed(e) from e + rules = rules or so.ScopeRules.from_options(row.options) + out: dict[str, Any] = {"schema": target or "", "index_total": result.total, "index_method": result.method, + "index_capped": result.capped, "index_note": result.note} + if other: + live = [(o.name, o.kind) for o in result.objects] + plan = so.plan_scope(live, rules, pinned=set(), on_demand=set()) + out.update(stored=False, objects=[o.name for o in result.objects], scope=plan.summary()) + return out + async with so.source_lock(row.id): + stats = await so.refresh_index(session, row.id, configured, result) + await session.commit() + plan = await so.plan_for(session, row, rules) + out.update(stored=True, objects=[o.name for o in result.objects], scope=plan.summary(), added=stats.added, + removed=stats.removed) + return out + + +async def prepare(session: AsyncSession, row: Any, *, actor: str | None, + trigger: str = "manual") -> tuple[IntrospectJob, int]: + """建一个排队的探查任务(还没开始,调用方接着 manager.start)。返回 (任务, 几张表)。 + + 上传表格、记录表抛 IntrospectRefused;同源已有排队或进行中的探查任务抛 IntrospectBusy;对象清单拿不到抛 + IntrospectFailed(失败照实记进 schema_cache);规则命中超过上限抛 ScopeTooLarge(对象索引照样刷新了)。 + + 这里只刷新对象索引、圈表、核上限、建任务;范围写进对象索引(离开范围的表从 schema_cache 里拿掉)等轮到这个任务、 + 开始执行时再做(_IntrospectRun._apply):排在同源的剖析后面时,剖析看到的表结构不该在它做到一半时变。""" + _refuse(row) + name, source_id = row.name, row.id + await ensure_free(session, row) + schema = so.index_schema(row) + try: + result = await so.list_objects(row, so.target_schema(row)) + except Exception as e: # noqa: BLE001 + await _record_failure(session, row, e) + raise IntrospectFailed(e) from e + async with so.source_lock(source_id): + stats = await so.refresh_index(session, source_id, schema, result) + await session.flush() + rules = so.ScopeRules.from_options(row.options) + plan = await so.plan_for(session, row, rules) + if plan.over: + await session.commit() + raise ScopeTooLarge(plan) + summary = {**plan.summary(rules), "index_method": stats.method, "index_capped": stats.capped, + "index_note": stats.note, "added": stats.added, "removed": stats.removed} + job = IntrospectJob(id=new_id(), source_id=source_id, status="queued", trigger=trigger, schema_name=schema, + scope=summary, created_by=actor) + session.add(job) + try: + await session.flush() + except IntegrityError as e: + await session.rollback() + raise IntrospectBusy(busy_message(SimpleNamespace(name=name), None)) from e + session.add_all([IntrospectTask(job_id=job.id, seq=i, table_name=table, reason=reason) + for i, (table, reason) in enumerate(plan.in_scope.items())]) + await session.commit() + try: + await prune(session, source_id) + except Exception: # noqa: BLE001 - 清旧任务失败不影响这次探查 + logger.exception("清理数据源 %s 的旧探查任务时出错", source_id) + await session.rollback() + return job, len(plan.in_scope) + + +async def prune(session: AsyncSession, source_id: str, *, keep: int = KEEP_JOBS) -> int: + finished = list((await session.execute( + select(IntrospectJob.id).where(IntrospectJob.source_id == source_id, IntrospectJob.status.in_(FINISHED)) + .order_by(IntrospectJob.created_at.desc(), IntrospectJob.id.desc()))).scalars()) + victims = finished[keep:] + if not victims: + return 0 + await session.execute(delete(IntrospectTask).where(IntrospectTask.job_id.in_(victims))) + await session.execute(delete(IntrospectJob).where(IntrospectJob.id.in_(victims))) + await session.commit() + return len(victims) + + +# ========================================================================== +# 收尾、停止、启动时的整理 +# ========================================================================== + + +def _index_stats(job: IntrospectJob | None) -> so.RefreshStats | None: + """建任务时刷新对象索引的结果(记在任务的 scope 里),物化时写进 schema_cache 的 scope 摘要。""" + scope = (job.scope or {}) if job is not None else {} + if "index_method" not in scope: + return None + return so.RefreshStats(total=int(scope.get("index_total") or 0), added=int(scope.get("added") or 0), + removed=int(scope.get("removed") or 0), method=str(scope.get("index_method")), + capped=bool(scope.get("index_capped")), note=scope.get("index_note")) + + +async def _materialize(session: AsyncSession, source_id: str, *, synced: bool, + job: IntrospectJob | None = None) -> None: + """把范围内已反射的行物化进 schema_cache(停止、中断、出错收尾时也做:读完的不白读)。synced:探查做完了,盖同步时间。""" + row = await session.get(DataSource, source_id) + if row is None: + return + async with so.source_lock(source_id): + await so.rebuild_schema_cache(session, row, synced=synced, index=_index_stats(job)) + if synced: + row.schema_synced_at = utcnow() + await session.commit() + + +async def _close(session: AsyncSession, job_id: str, job_status: str, job_error: str | None, + task_error: str | None) -> None: + """收尾:还没读的表记 cancelled(原因 task_error),任务(还没结束的话)记 job_status;读完的物化进 schema_cache。""" + now = utcnow() + job = await session.get(IntrospectJob, job_id) + await session.execute(update(IntrospectTask).where(IntrospectTask.job_id == job_id, + IntrospectTask.status.in_(OPEN)) + .values(status="cancelled", error=task_error, finished_at=now) + .execution_options(synchronize_session=False)) + await session.execute(update(IntrospectJob).where(IntrospectJob.id == job_id, IntrospectJob.status.in_(ACTIVE)) + .values(status=job_status, error=job_error, finished_at=now) + .execution_options(synchronize_session=False)) + await session.commit() + if job is not None and job.started_at is not None: + # 开始过的才物化:没轮到就结束了的,范围还没写进对象索引,schema_cache 照旧 + await _materialize(session, job.source_id, synced=False, job=job) + + +async def cancel(session: AsyncSession, job: IntrospectJob) -> None: + """停止一个探查任务。排队的直接 cancelled;进行中的交给 manager 停(停在当前这一批之后)。已经结束的抛 + IntrospectFinished。""" + job_id, status = job.id, job.status + if status in FINISHED: + raise IntrospectFinished("这次探查已经结束了,不需要停止") + if status == "queued" and await runtime.cancel_queued(session, IntrospectJob, IntrospectTask, job_id): + # 还没开始:范围没写进对象索引、schema_cache 没动,停了就是什么都没发生 + await manager.stop(job_id, "user", grace=STOP_GRACE_S) + return + if await manager.stop(job_id, "user", grace=STOP_GRACE_S): + return + # 库里说在进行,这个进程里却没有在跑它(记录对不上):直接记停止 + await _close(session, job_id, "cancelled", None, STOPPED_ERROR) + + +async def startup() -> int: + """服务启动时:排队、进行中的探查任务标成 interrupted,没读的表 cancelled,读完的物化进去;不自动续跑。""" + async def interrupt(session: AsyncSession, job_id: str) -> None: + await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, INTERRUPTED_ERROR) + + n = await runtime.interrupt_on_startup(IntrospectJob, interrupt, running=manager.running) + if n: + logger.warning("服务重启:%d 个探查任务没有做完,已标为中断,可在数据源卡片上重新探查", n) + return n + + +# ========================================================================== +# 在后台反射 +# ========================================================================== + + +class _IntrospectRun(runtime.JobRun): + """一个探查任务在后台的一次执行(进队、停止、收尾的框架在 runtime.JobRun)。""" + + kind = "introspect" + model = IntrospectJob + + async def _stopped(self) -> None: + if self.stop_reason == "user": + await self._end("cancelled", None, STOPPED_ERROR) + else: + await self._end("interrupted", INTERRUPTED_ERROR, INTERRUPTED_ERROR) + + async def _failed(self, reason: str) -> None: + await self._end("failed", reason, reason) + + async def _crashed(self, e: Exception) -> None: + logger.exception("探查任务 %s 出错", self.job_id) + reason = JOB_ERROR.format(line=first_line(e) or describe_exception(e)) + await self._end("failed", reason, reason) + + async def _end(self, job_status: str, job_error: str | None, task_error: str | None) -> None: + try: + await self._settle() + await self._write(_close, self.job_id, job_status, job_error, task_error) + except Exception: # noqa: BLE001 + logger.exception("探查任务 %s 收尾时写库出错", self.job_id) + + async def _run(self) -> None: + if not await self._write(self._claim): + return # 开始之前就被停止了 + async with SessionLocal() as session: + job = await session.get(IntrospectJob, self.job_id) + if job is None: + raise runtime.JobFailed(SOURCE_GONE) + source_id, schema = job.source_id, job.schema_name + await self._write(self._apply, source_id, schema) + async with SessionLocal() as session: + pending = [(t.seq, t.table_name) for t in (await session.execute( + select(IntrospectTask.seq, IntrospectTask.table_name).where( + IntrospectTask.job_id == self.job_id, IntrospectTask.status == "pending") + .order_by(IntrospectTask.seq))).all()] + for start in range(0, len(pending), so.REFLECT_BATCH): + batch = pending[start:start + so.REFLECT_BATCH] + row = await self._write(self._recheck, source_id) + await self._write(self._mark_running, [seq for seq, _ in batch]) + objs = await self._write(self._objects, source_id, schema, [name for _, name in batch]) + try: + got = await so.reflect(row, [(name, objs.get(name, ("table", False))[1]) for _, name in batch], + views={n for n, (kind, _q) in objs.items() if kind == "view"}) + except Exception as e: # noqa: BLE001 - 连接本身出错:这一批和后面的都读不了 + raise runtime.JobFailed(CONNECT_ERROR.format(line=first_line(e) or describe_exception(e))) from e + await self._write(self._save_batch, source_id, schema, batch, got) + await self._settle() + await self._write(self._finish, source_id) + + # ---------------------------------------------------------------- 库操作 + + async def _claim(self, session: AsyncSession) -> bool: + return await runtime.claim(session, IntrospectJob, self.job_id) + + async def _apply(self, session: AsyncSession, source_id: str, schema: str) -> None: + """开始执行时把建任务时圈好的范围写进对象索引:离开范围的清掉结构,范围内的按原因记。建任务之后按需加入的表 + 照旧留在范围里(它们刚读过,不在这次的表单里)。""" + reasons = {r.table_name: r.reason or "rule" for r in (await session.execute( + select(IntrospectTask.table_name, IntrospectTask.reason).where(IntrospectTask.job_id == self.job_id) + )).all()} + async with so.source_lock(source_id): + for name in (await session.execute(select(SourceObject.name).where( + SourceObject.source_id == source_id, SourceObject.schema_name == schema, + SourceObject.in_scope.is_(True), SourceObject.scope_reason == "on_demand", + SourceObject.removed_at.is_(None)))).scalars(): + reasons.setdefault(name, "on_demand") + counts = {r: 0 for r in so.SCOPE_REASONS} + for reason in reasons.values(): + counts[reason] = counts.get(reason, 0) + 1 + plan = so.ScopePlan(in_scope=reasons, counts=counts, cap=0, index_total=0, out_of_scope=0) + await so.apply_scope(session, source_id, schema, plan) + await session.commit() + + async def _recheck(self, session: AsyncSession, source_id: str) -> Any: + """每批开始前重读数据源:没了抛 JobFailed,停用了也停下(JobFailed,原因写明)。返回会话外可用的数据源对象。""" + row = await session.get(DataSource, source_id) + if row is None: + raise runtime.JobFailed(SOURCE_GONE) + if row.enabled is False: + raise runtime.JobFailed(SOURCE_DISABLED.format(name=row.name)) + session.expunge(row) + return row + + async def _mark_running(self, session: AsyncSession, seqs: list[int]) -> None: + await session.execute(update(IntrospectTask).where(IntrospectTask.job_id == self.job_id, + IntrospectTask.seq.in_(seqs), + IntrospectTask.status == "pending") + .values(status="running").execution_options(synchronize_session=False)) + await session.commit() + + async def _objects(self, session: AsyncSession, source_id: str, schema: str, + names: list[str]) -> dict[str, tuple[str, bool]]: + return {r.name: (r.kind, bool(r.quote_name)) for r in (await session.execute( + select(SourceObject.name, SourceObject.kind, SourceObject.quote_name).where( + SourceObject.source_id == source_id, SourceObject.schema_name == schema, + SourceObject.name.in_(names)))).all()} + + async def _save_batch(self, session: AsyncSession, source_id: str, schema: str, batch: list[tuple[int, str]], + got: so.Reflected) -> None: + """一批读完:读到的结构写进对象索引,逐表记 ok / failed 和结构版本。""" + versions = await so.save_reflected(session, source_id, schema, got) + now = utcnow() + for seq, name in batch: + if name in got.tables: + values: dict[str, Any] = {"status": "ok", "error": None} + else: + values = {"status": "failed", "error": got.skipped.get(name) or "读不到列"} + await session.execute(update(IntrospectTask).where(IntrospectTask.job_id == self.job_id, + IntrospectTask.seq == seq) + .values(**values, struct_version=versions.get(name, 0), finished_at=now) + .execution_options(synchronize_session=False)) + await session.commit() + + async def _finish(self, session: AsyncSession, source_id: str) -> None: + """读完:物化 schema_cache、盖同步时间;有表读不到是 partial,否则 succeeded。""" + failed = int((await session.execute(select(func.count()).select_from(IntrospectTask).where( + IntrospectTask.job_id == self.job_id, IntrospectTask.status == "failed"))).scalar() or 0) + await _materialize(session, source_id, synced=True, job=await session.get(IntrospectJob, self.job_id)) + await session.execute(update(IntrospectJob).where(IntrospectJob.id == self.job_id, + IntrospectJob.status == "running") + .values(status="partial" if failed else "succeeded", finished_at=utcnow()) + .execution_options(synchronize_session=False)) + await session.commit() + + +class IntrospectJobs(runtime.JobManager): + """进程内的探查任务(start / stop / wait / running / shutdown 见 runtime.JobManager)。""" + + def __init__(self) -> None: + super().__init__(_IntrospectRun, label="探查任务", name="introspect") + + +manager = IntrospectJobs() + + +__all__ = ["INTERRUPTED_ERROR", "IntrospectBusy", "IntrospectFailed", "IntrospectFinished", "IntrospectJobs", + "IntrospectRefused", "KEEP_JOBS", "STOPPED_ERROR", "ScopeTooLarge", "active_job", "busy_message", "cancel", + "ensure_free", "job_brief", "job_out", "latest_briefs", "latest_job", "load", "manager", "prepare", + "preview", "prune", "startup", "waiting_for"] diff --git a/backend/app/data/schema_view.py b/backend/app/data/schema_view.py index 80b3cb12..ba7af951 100644 --- a/backend/app/data/schema_view.py +++ b/backend/app/data/schema_view.py @@ -1,4 +1,4 @@ -"""数据源的只读表结构视图:库概览、表目录、表详情(字段、数据目录、连接条件、两跳的连接路径)。 +"""数据源的只读表结构视图:库概览、表目录、表详情(字段、数据目录、剖析事实、连接条件、两跳的连接路径)、大库的检索预筛。 助手查看(app/look/tables.py)、运行时 agent 的 db_schema__(tools/datasource.py)、/api/datasources/{id}/schema?table= 三处共用这一份(look 文档 D6:同一张表只有一种说法,同一套遮罩规矩)。以前三处各拼各的:助手的小库摘要每张表只列 @@ -8,12 +8,25 @@ 这一列的各项(含码值、带具体取值的说明)都不给。和表助手(data/catalog_assistant.py)是同一条规矩。代价是运行时 agent 看到的目录少了遮罩列那部分(README 写明)。 -只读:只看表结构缓存(schema_cache)和调用方读好的数据目录,不碰业务库。这里的文字都是写给模型看的,不上界面。 +**剖析事实**(next-round 遗留 e):剖析记下的行数估算和依据、各日期列的范围(catalog_profile_jobs.last_facts)在表详情里 +成一节(profile_lines),后面跟一句固定提示「只供写查询参考……不能作为回答里的数」——剖析数字是会过期的观测,叙述层无算术靠 +这句和裁判守住。遮罩按渲染这一刻的设置重滤(剖析之后才加遮罩的列,范围不出现);上传表格只在剖析时的快照和这次读的快照 +一致时给,不一致写「剖析早于当前版本,不给」。look、运行时 db_schema、/schema、表助手四处都是这一段文字。 + +**大库的检索预筛**(search_tables,next-round 03 A2):几千张表的库,挑表时不能把整份表目录交给挑选模型。按需求做一次 BM25: +文档是表名、注释、目录里的中文名 / 粒度 / 说明、列名、列的中文名(遮罩列只用列名);探查范围外的对象只按名字和注释参与, +权重低一档。切词用 memory/embeddings._tokens(英文按词、中文按字补二元组)加 catalog.name_words(下划线、驼峰拆词)。 + +**三种「不在」**(absent_detail):表已不在、在探查范围外、在范围里却读不到结构,说法分开;只有真删除才说「表已不在」。 + +只读:只看表结构缓存(schema_cache)和调用方读好的数据目录、剖析事实、对象索引,不碰业务库。这里的文字都是写给模型看的, +不上界面。 """ from __future__ import annotations import copy from collections.abc import Iterable, Mapping +from dataclasses import dataclass from typing import Any from app.data import catalog, introspect @@ -31,6 +44,8 @@ PATH_LINES_LIMIT = 12 OVERVIEW_LINE_PROMPT = "- {name}({kind},{access}):{description}|{state}" OVERVIEW_TABLES_PROMPT = "共 {total} 张表" OVERVIEW_PARTIAL_PROMPT = "共 {total} 张表,只探查了 {explored} 张" +#: 有对象索引的源(next-round 03 A1):范围外的表有名字,点名查看时按需加入 +OVERVIEW_SCOPED_PROMPT = "共 {total} 张表,探查范围内 {explored} 张(范围外的点名查看时按需加入)" OVERVIEW_CATALOG_ALL_PROMPT = "都有数据目录" OVERVIEW_CATALOG_SOME_PROMPT = "{n} 张有数据目录" OVERVIEW_CATALOG_NONE_PROMPT = "还没有数据目录" @@ -43,6 +58,23 @@ JOIN_MORE_PROMPT = "…另有 {n} 条,点名查看相关的表看各表的关 PATH_HEAD_PROMPT = "连接路径:" ROUTE_PROMPT = "{a} 与 {b} 没有直接关联,经 {mid} 连接:FROM {a} JOIN {mid} ON {first} JOIN {b} ON {second}" +#: 剖析事实那一节(profile_lines) +PROFILE_HEAD_PROMPT = "数据剖析({day}):{facts}" +PROFILE_ROWS_PROMPT = {"stats": "约 {n} 行(数据库统计估算)", "count": "{n} 行(逐行数出)", + "at_least": "至少 {n} 行(数到上限为止)"} +PROFILE_RANGE_PROMPT = "{column} 从 {low} 到 {high}" +PROFILE_HINT_PROMPT = "只供写查询参考(定时间范围、估表大小),不是查询结果,不能作为回答里的数" +PROFILE_STALE_PROMPT = "数据剖析:剖析早于当前版本,不给" + +#: 三种「不在」(absent_detail)。{source} 数据源名,{table} 点名的表 +ABSENT_REMOVED_PROMPT = "数据源「{source}」里已经没有 {table}(表已删除或改名)。别拿名字相近的表顶替" +ABSENT_MISSING_PROMPT = "数据源「{source}」里没有 {table}(全库的对象清单里也没有)。别拿名字相近的表顶替,不确定时用 keyword 检索" +ABSENT_OUT_OF_SCOPE_PROMPT = ("{table} 在数据源「{source}」的探查范围外,这次没有它的结构{why}。要用它,请点名查看(会按需加入)," + "或者请管理员把它加进探查范围") +ABSENT_REFLECT_FAILED_PROMPT = "{table} 在数据源「{source}」的探查范围里,但读不到它的结构:{error}。别猜它的字段名" +ABSENT_UNKNOWN_PROMPT = ("数据源「{source}」只探查了一部分表,{table} 不在已探查的表里,无法判断是否存在。请直接查询数据字典" + "(information_schema.tables / columns,SQLite 用 sqlite_master)确认它的结构") + # ========================================================================== # 遮罩 @@ -88,7 +120,8 @@ def overview_line(source: Any, *, catalog_tables: int) -> str: else: total = cache.get("total") total = max(total if isinstance(total, int) and not isinstance(total, bool) else 0, len(tables)) - parts = [OVERVIEW_PARTIAL_PROMPT.format(total=total, explored=len(tables)) if cache.get("truncated") + partial = OVERVIEW_SCOPED_PROMPT if isinstance(cache.get("scope"), dict) else OVERVIEW_PARTIAL_PROMPT + parts = [partial.format(total=total, explored=len(tables)) if cache.get("truncated") and total > len(tables) else OVERVIEW_TABLES_PROMPT.format(total=total)] if catalog_tables >= len(tables): parts.append(OVERVIEW_CATALOG_ALL_PROMPT) @@ -115,11 +148,13 @@ def table_index(source: Any, notes_by_table: Mapping[str, Mapping[str, Any]], *, # ========================================================================== -def table_detail(source: Any, table: str, *, catalog_notes: Mapping[str, Mapping[str, Any]] | None = None) -> str: - """单表的完整字段说明,有可给模型看的目录时附在后面;遮罩列只给列名和类型。 +def table_detail(source: Any, table: str, *, catalog_notes: Mapping[str, Mapping[str, Any]] | None = None, + profile: Mapping[str, Mapping[str, Any]] | None = None) -> str: + """单表的完整字段说明,有可给模型看的目录时附在后面,有剖析事实时再附一节(profile_lines);遮罩列只给列名和类型。 - catalog_notes 是这个源的数据目录 {表名: notes}(原样,含驳回项:render_table_notes 自己去掉)。找不到这张表时 - 交回的几种说法(结构没探查、只探查了一部分、确实没有)和 introspect.describe_table 一样。 + catalog_notes 是这个源的数据目录 {表名: notes}(原样,含驳回项:render_table_notes 自己去掉)。profile 是这个源的剖析 + 事实 {表名: facts}(catalog_profile_jobs.last_facts);不给、这张表没有时输出和以前一字不差。找不到这张表时交回的几种 + 说法(结构没探查、只探查了一部分、确实没有)和 introspect.describe_table 一样。 """ meta = introspect.find_table(source, table) if not meta: @@ -150,9 +185,232 @@ def table_detail(source: Any, table: str, *, catalog_notes: Mapping[str, Mapping meta=shown, system_notes=catalog.system_notes_source(source)) if block: lines.append(block) + if profile: + key = catalog.resolve_table_name(getattr(source, "schema_cache", None), table) + lines += profile_lines(profile.get(key) if key else None, masked, snapshot_of(source), + upload=is_upload(source)) return "\n".join(lines) +# ========================================================================== +# 剖析事实 +# ========================================================================== + + +def is_upload(source: Any) -> bool: + """上传表格的源(含绑定了快照的视图):剖析事实要和快照对得上才给。""" + return (getattr(source, "origin", None) or "manual") == "upload" + + +def snapshot_of(source: Any) -> str | None: + """这次读的是上传表格的哪个快照:绑定了快照的视图(table_versions.SourceView)是它绑定的那个,数据源记录是当前启用的 + 那个;数据库源为 None。""" + if not is_upload(source): + return None + return getattr(source, "snapshot_id", None) or getattr(source, "current_snapshot_id", None) + + +def _day(value: Any) -> str: + return str(value or "")[:10] + + +def _when(value: Any) -> str: + """日期范围的一端:日期只写到日,带时刻的写到秒(剖析记的是数据库原样的文字)。""" + text = str(value or "") + return text[:10] if len(text) <= 10 or text[10:].strip(" T0:.") == "" else text[:19] + + +def profile_lines(facts: Mapping[str, Any] | None, masked: set[str], snapshot_id: str | None, *, + upload: bool = False, indent: str = " ") -> list[str]: + """剖析事实那一节:「数据剖析(YYYY-MM-DD):约 N 行(数据库统计估算);下单日从 … 到 …」,后面一句固定提示。 + + masked 是渲染这一刻的遮罩列(小写):剖析之后才加遮罩的列,范围不出现。upload:上传表格源,剖析时的快照 + (facts.snapshot_id)和这次读的快照对不上时只写「剖析早于当前版本,不给」——升级前的剖析记录没有快照,上传表格在重新剖析 + 之前一律不给。没有剖析事实(或者滤完什么都不剩)返回空列表:表详情和以前一字不差。""" + if not facts: + return [] + if upload and (not snapshot_id or facts.get("snapshot_id") != snapshot_id): + return [indent + PROFILE_STALE_PROMPT] + parts: list[str] = [] + estimate = facts.get("row_estimate") if isinstance(facts.get("row_estimate"), Mapping) else None + if estimate: + method, rows, least = estimate.get("method"), estimate.get("rows"), estimate.get("at_least") + if isinstance(rows, int) and method in ("stats", "count"): + parts.append(PROFILE_ROWS_PROMPT[method].format(n=f"{rows:,}")) + elif isinstance(least, int) and method == "count": + parts.append(PROFILE_ROWS_PROMPT["at_least"].format(n=f"{least:,}")) + for found in facts.get("date_ranges") or []: + if not isinstance(found, Mapping): + continue + column = str(found.get("column") or "") + if not column or column.lower() in masked or found.get("min") is None or found.get("max") is None: + continue + parts.append(PROFILE_RANGE_PROMPT.format(column=column, low=_when(found["min"]), high=_when(found["max"]))) + if not parts: + return [] + return [indent + PROFILE_HEAD_PROMPT.format(day=_day(facts.get("at")), facts=";".join(parts)), + indent + " " + PROFILE_HINT_PROMPT] + + +# ========================================================================== +# 三种「不在」 +# ========================================================================== + + +def absent_detail(source: Any, table: str, reason: str | None, *, error: str | None = None, why: str = "") -> str | None: + """点名的表不在表结构里时的说法(source_objects.absent_reasons 的原因):表已不在、在探查范围外(why 是没能按需加入的 + 原因,比如上限用完了)、在范围里却读不到结构。原因不明(没有对象索引的老缓存)返回 None,由调用方照旧说。""" + from app.data import source_objects as so + + name = getattr(source, "name", "") + if reason == so.ABSENT_REMOVED: + return ABSENT_REMOVED_PROMPT.format(source=name, table=table) + if reason == so.ABSENT_MISSING: + return ABSENT_MISSING_PROMPT.format(source=name, table=table) + if reason == so.ABSENT_OUT_OF_SCOPE: + return ABSENT_OUT_OF_SCOPE_PROMPT.format(source=name, table=table, why=f"({why})" if why else "") + if reason == so.ABSENT_REFLECT_FAILED: + return ABSENT_REFLECT_FAILED_PROMPT.format(source=name, table=table, error=error or "读不到列") + if reason == so.ABSENT_UNKNOWN: + return ABSENT_UNKNOWN_PROMPT.format(source=name, table=table) + return None + + +# ========================================================================== +# 大库的检索预筛 +# ========================================================================== + +#: 范围外的对象在检索里的权重(只有名字和注释,低一档) +OUTSIDE_WEIGHT = 0.6 + + +@dataclass(frozen=True) +class SearchHit: + """检索的一个命中:表名(范围内是 schema_cache 的键,范围外是对象索引里的名字)、得分、在不在探查范围里。""" + + name: str + score: float + in_scope: bool = True + + +def _doc_text(name: str, meta: Mapping[str, Any] | None, notes: Mapping[str, Any] | None, masked: set[str]) -> str: + """一张范围内的表的检索文档:表名(再拆词)、注释、目录里的中文名 / 粒度 / 说明、列名(再拆词)、列的中文名。遮罩列只用 + 列名(注释、目录项不进:里面常写着取值)。""" + from app.data.catalog import name_words + + parts = [name, " ".join(name_words(name))] + meta = meta or {} + if meta.get("comment"): + parts.append(str(meta["comment"])) + visible = catalog.visible_notes(notes or {}) + for key in ("label", "grain", "description", "note"): + item = visible.get(key) + if isinstance(item, Mapping) and isinstance(item.get("value"), str): + parts.append(item["value"]) + columns_notes = visible.get("columns") if isinstance(visible.get("columns"), Mapping) else {} + for col in meta.get("columns") or []: + if not isinstance(col, Mapping) or not col.get("name"): + continue + cname = str(col["name"]) + parts += [cname, " ".join(name_words(cname))] + if cname.lower() in masked: + continue + if col.get("comment"): + parts.append(str(col["comment"])) + items = columns_notes.get(cname) if isinstance(columns_notes, Mapping) else None + label = items.get("label") if isinstance(items, Mapping) else None + if isinstance(label, Mapping) and isinstance(label.get("value"), str): + parts.append(label["value"]) + return " ".join(parts) + + +def _outside_text(name: str, comment: str | None) -> str: + from app.data.catalog import name_words + + return " ".join([name, " ".join(name_words(name)), comment or ""]) + + +class _Index: + """BM25(k1=1.5、b=0.75,和 memory/embeddings.BM25 同一个公式)的倒排实现:几千上万篇文档时只看含这个词的文档, + 不逐篇数。""" + + def __init__(self, docs: list[str]) -> None: + import math + + from app.memory.embeddings import _tokens + + self._tokens = _tokens + self.k1, self.b = 1.5, 0.75 + self.lengths: list[int] = [] + self.postings: dict[str, list[tuple[int, int]]] = {} + for i, doc in enumerate(docs): + toks = _tokens(doc) + self.lengths.append(len(toks)) + counts: dict[str, int] = {} + for t in toks: + counts[t] = counts.get(t, 0) + 1 + for t, n in counts.items(): + self.postings.setdefault(t, []).append((i, n)) + self.n = len(docs) + self.avg = (sum(self.lengths) / self.n) if self.n else 0.0 + self._log = math.log + + def score(self, query: str) -> dict[int, float]: + out: dict[int, float] = {} + for tok in dict.fromkeys(self._tokens(query)): + posting = self.postings.get(tok) + if not posting: + continue + idf = self._log(1 + (self.n - len(posting) + 0.5) / (len(posting) + 0.5)) + for i, tf in posting: + denom = tf + self.k1 * (1 - self.b + self.b * self.lengths[i] / (self.avg or 1)) + out[i] = out.get(i, 0.0) + idf * (tf * (self.k1 + 1)) / denom + return out + + +class TableSearch: + """一个源的检索:范围内的表(schema_cache)和范围外的对象(对象索引里的名字、注释)。建一次(一轮查看里第一次要挑时), + 之后每个需求查一次。""" + + def __init__(self, source: Any, notes_by_table: Mapping[str, Mapping[str, Any]], + outside: Iterable[Any] = ()) -> None: + tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} + masked = masked_set(source) + self.names: list[str] = [] + self.inside: list[bool] = [] + docs: list[str] = [] + for name, meta in tables.items(): + self.names.append(name) + self.inside.append(True) + docs.append(_doc_text(name, meta if isinstance(meta, Mapping) else None, notes_by_table.get(name), masked)) + known = set(self.names) + for obj in outside: + name = str(getattr(obj, "name", obj)) + if name in known: + continue + self.names.append(name) + self.inside.append(False) + docs.append(_outside_text(name, getattr(obj, "comment", None))) + self._index = _Index(docs) + + def __len__(self) -> int: + return len(self.names) + + def search(self, query: str, *, limit: int) -> list[SearchHit]: + """按得分从高到低的前 limit 个(得分为 0 的不算命中)。""" + scored = self._index.score(query) + hits = [SearchHit(self.names[i], s * (1.0 if self.inside[i] else OUTSIDE_WEIGHT), self.inside[i]) + for i, s in scored.items() if s > 0] + hits.sort(key=lambda h: (-h.score, not h.in_scope, h.name)) + return hits[:limit] + + +def search_tables(source: Any, notes_by_table: Mapping[str, Mapping[str, Any]], query: str, *, + outside: Iterable[Any] = (), limit: int = 120) -> list[SearchHit]: + """按需求检索这个源的表(一次性的写法;同一个源要查多次就建一个 TableSearch 复用)。""" + return TableSearch(source, notes_by_table, outside).search(query, limit=limit) + + def table_brief(source: Any, table: str) -> str: """字数超了时的一行:表名和全部字段名(遮罩列的名字照给,名字本身不是值)。""" meta = introspect.find_table(source, table) or {} @@ -253,6 +511,8 @@ def route_lines(source: Any, routes: list[tuple[str, str, tuple[JoinEdge, ...]]] __all__ = [ - "JOIN_LINES_LIMIT", "PATH_LINES_LIMIT", "complete", "join_lines", "masked_notes", "masked_set", "overview_line", - "qualifier", "relation_graph", "route_lines", "table_brief", "table_detail", "table_index", + "JOIN_LINES_LIMIT", "OUTSIDE_WEIGHT", "PATH_LINES_LIMIT", "PROFILE_HINT_PROMPT", "PROFILE_STALE_PROMPT", "SearchHit", + "TableSearch", "absent_detail", "complete", "is_upload", "join_lines", "masked_notes", "masked_set", + "overview_line", "profile_lines", "qualifier", "relation_graph", "route_lines", "search_tables", "snapshot_of", + "table_brief", "table_detail", "table_index", ] diff --git a/backend/app/data/source_objects.py b/backend/app/data/source_objects.py new file mode 100644 index 00000000..e69a3ce4 --- /dev/null +++ b/backend/app/data/source_objects.py @@ -0,0 +1,978 @@ +"""数据源的对象索引、探查范围、表结构的物化与按需加入(docs/design/next-round/03-数据接入与目录维护.md A1、A2)。 + +几千张表的库以前用不了:探查结构每个源最多取 200 个对象,按数据库列出的顺序截断,多出来的表目录、look、剖析、SQL 检查都 +看不到,还会被说成「表已不在」。现在分三层: + +- **对象索引**(list_objects / refresh_index):每种方言一条字典查询,拿全库对象的名字、类型、注释、行数估算(SQLite 查 + sqlite_master,MySQL 查 information_schema.TABLES,PostgreSQL 查 pg_class 加 obj_description,Oracle 查 ALL_OBJECTS 加 + ALL_TAB_COMMENTS)。字典查询走数据库查询的同一个入口(engine.run_query:只读守卫、单条时限);查不了退回 Inspector 的 + get_table_names / get_view_names,只是没有注释。最多收 INDEX_LIMIT 个对象(按名字排序取前面的),超了照实记 capped, + 这时不判删除(没收进来的不等于不在了)。 +- **探查范围**(ScopeRules / plan_scope):数据源 options.introspect_scope 的规则——{mode: all|rules, include: [通配], + exclude: [通配], tables: [显式], removed: [显式移出], views: bool, max_tables}。按 names.name_key 不分大小写匹配,排除优先。 + 有数据目录的表、已发布模板写死的 SQL 用到的表一律在范围里(pinned),规则排不掉,只有显式移出才行。规则命中(含 pinned、 + 显式)超过上限时拒绝开始并说出命中了多少张,不悄悄截断。按需加入的表另算,总数不超过 SCOPE_MAX。 +- **物化**(rebuild_schema_cache):schema_cache 由这一个函数从范围内已反射的行物化出来,形状不变(tables、total、 + truncated、schema、synced_at),另加 scope 摘要(in_scope、cap、out_of_scope……)。total 是对象索引里的对象总数,truncated + 的意思变成「有范围外的对象」:两百多处读 schema_cache 的地方不用改。 + +**按需加入**(add_to_scope):look 点名或挑中范围外的表、运行时 db_schema__ 点名范围外的表、目录页「加入范围」,都当场只读 +反射这一张、落盘(scope_reason on_demand / explicit),记下谁、经哪个入口、哪一轮或哪次运行、需求原文。写 schema_cache 只走 +rebuild_schema_cache,按源加进程内锁(source_lock):和后台探查任务、别的按需加入互斥。前提和回收锁、SourceLanes 一样是 +单进程部署;多进程部署要换成库里的锁。 + +**三种「不在」**(absent_reasons):表已不在(对象索引里没有了,removed_at)、在探查范围外、在范围里却读不到结构,分开说。 +只有第一种说「表已不在」。 + +只读:只发字典查询和反射,不写业务库。 +""" +from __future__ import annotations + +import asyncio +import fnmatch +import hashlib +import json +import logging +import weakref +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any + +from sqlalchemy import bindparam, delete, func, insert, or_, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.errors import describe_exception, first_line +from app.data.names import name_key +from app.db.base import SessionLocal, utcnow +from app.db.models_sources import SourceObject + +logger = logging.getLogger(__name__) + +#: options 里探查范围的键(由 PUT /api/datasources/{id}/introspect-scope 维护,数据源表单不显示、不提交它) +SCOPE_OPTION = "introspect_scope" +#: 对象索引最多收几个对象(目录-D3) +INDEX_LIMIT = 50_000 +#: 探查范围缺省、最多几张(目录-D3):500 张 × 25 列约 1 MB 的 schema_cache,每次读数据源都整份加载 +SCOPE_DEFAULT = 500 +SCOPE_MAX = 2_000 +#: 范围规则里每种最多几条、每条最长几个字;显式列出、显式移出的表最多几张 +RULES_MAX = 100 +RULE_CHARS = 200 +#: 后台探查一批读几张:PostgreSQL 的批量反射一批一次往返,MySQL 每表一次往返;一批也是停止的最小单位 +REFLECT_BATCH = 25 +#: 每次「加入范围」最多几张(目录页手工加) +ADD_LIMIT = 20 + +# 按需加入的入口 +VIA_LOOK = "look" +VIA_SCHEMA_TOOL = "db_schema" +VIA_CATALOG = "catalog" + +# 三种「不在」(目录清单的 absent_reason、look 和 db_schema 的说法都按它),另有「从来没有过」和「说不清」 +ABSENT_REMOVED = "removed" +ABSENT_OUT_OF_SCOPE = "out_of_scope" +ABSENT_REFLECT_FAILED = "reflect_failed" +#: 对象索引里从来没有过(名字写错了,或者是对象索引刷新之后才建的) +ABSENT_MISSING = "missing" +#: 升级前探查的源(还没有对象索引)、被截断的老缓存:说不清是哪一种 +ABSENT_UNKNOWN = "unknown" + +SCOPE_REASONS = ("pinned", "explicit", "rule", "on_demand") + + +# ========================================================================== +# 对象索引:每种方言一条字典查询 +# ========================================================================== + + +@dataclass(frozen=True) +class IndexedObject: + """对象索引里的一个对象:名字是 Inspector 规整过的写法(schema_cache 的键)。quote:Oracle 库里本来就是小写的名字 + (SQLAlchemy 反射回来是 quoted_name(quote=True)),反射时要原样带引号。""" + + name: str + kind: str = "table" + comment: str | None = None + est_rows: int | None = None + quote: bool = False + + +@dataclass +class IndexResult: + objects: list[IndexedObject] + #: dictionary 字典查询 | inspector 退回 Inspector(没有注释) + method: str + #: 数据库里一共几个(超过 INDEX_LIMIT 时 objects 只收了前面的) + total: int + capped: bool = False + #: 字典查询失败、退回 Inspector 时的原因(留痕;不挡探查) + note: str | None = None + + +def _literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +def index_sql(kind: str | None, schema: str | None) -> str | None: + """拿全库对象清单的那条字典查询:四列(名字、类型、注释、行数估算),按名字排序。schema 是要查的 schema(Oracle 已经 + 按数据字典的写法转好大写);为 None 时查连接的当前 schema。不认识的数据库类型返回 None(退回 Inspector)。""" + k = (kind or "").lower() + if k == "sqlite": + # SQLite 一个文件一个 main(写了 schema 就查那个库的 sqlite_master:写错了照实报「没有这个库」);临时表不进来; + # sqlite_ 开头的是内部表(Inspector 也排掉) + master = ('"' + schema.replace('"', '""') + '".sqlite_master') if schema else "sqlite_master" + return (f"SELECT name, type, NULL AS comment, NULL AS est_rows FROM {master} " + "WHERE type IN ('table', 'view') AND substr(name, 1, 7) <> 'sqlite_' ORDER BY name") + if k in ("mysql", "mariadb"): + where = _literal(schema) if schema else "DATABASE()" + return ("SELECT TABLE_NAME, TABLE_TYPE, TABLE_COMMENT, TABLE_ROWS FROM information_schema.TABLES " + f"WHERE TABLE_SCHEMA = {where} ORDER BY TABLE_NAME") + if k in ("postgres", "postgresql"): + where = _literal(schema) if schema else "current_schema()" + # r 普通表、p 分区表的主表、f 外部表、v 视图、m 物化视图;分区的子表不进(relispartition):几千个按月分区的子表 + # 会把真正要找的表淹掉,查主表就能查到子表的数据 + return ("SELECT c.relname, c.relkind, obj_description(c.oid, 'pg_class'), c.reltuples " + "FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace " + f"WHERE n.nspname = {where} AND c.relkind IN ('r', 'p', 'f', 'v', 'm') AND NOT c.relispartition " + "ORDER BY c.relname") + if k == "oracle": + owner = _literal(schema) if schema else "SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')" + # generated = 'N' 排掉系统生成的对象(IOT 溢出段这类),BIN$ 是回收站里的表 + return ("SELECT o.object_name, o.object_type, c.comments, t.num_rows FROM all_objects o " + "LEFT JOIN all_tab_comments c ON c.owner = o.owner AND c.table_name = o.object_name " + "LEFT JOIN all_tables t ON t.owner = o.owner AND t.table_name = o.object_name " + f"WHERE o.owner = {owner} AND o.object_type IN ('TABLE', 'VIEW') AND o.generated = 'N' " + "AND o.object_name NOT LIKE 'BIN$%' ORDER BY o.object_name") + return None + + +_VIEW_TYPES = frozenset({"view", "v", "m", "system view"}) + + +def _object_kind(raw: Any) -> str: + return "view" if str(raw or "").strip().lower() in _VIEW_TYPES else "table" + + +def _est_rows(raw: Any) -> int | None: + """数据字典的行数估算:≤0(PostgreSQL 没分析过的表是 -1)、读不出来都是不知道。""" + try: + n = int(float(str(raw))) + except (TypeError, ValueError): + return None + return n if n > 0 else None + + +def _comment(raw: Any, kind: str) -> str | None: + text = str(raw).strip() if raw is not None else "" + # MySQL 的视图在 TABLE_COMMENT 里写的是字面的「VIEW」 + if not text or (kind == "view" and text.upper() == "VIEW"): + return None + return text + + +async def list_objects(source: Any, schema: str | None) -> IndexResult: + """全库的对象清单(不落盘)。字典查询失败退回 Inspector;两条路都走不通抛出原来的异常(调用方照实记成探查失败)。""" + from app.data import engine as data_engine + from app.data.guard import QueryLimits + + kind = (getattr(source, "kind", None) or "").lower() + engine = await data_engine.engines.get(source) + dialect = engine.dialect + owner = schema + if schema and kind == "oracle": + owner = dialect.denormalize_name(schema) + sql = index_sql(kind, owner) + note: str | None = None + if sql is not None: + limits = QueryLimits(max_rows=INDEX_LIMIT + 1, max_bytes=64 * 1024 * 1024, + timeout_seconds=data_engine.query_timeout(source)) + try: + result = await data_engine.run_query(source, sql, limits=limits) + except Exception as e: # noqa: BLE001 - 字典查询不认(老版本、权限不够):退回 Inspector + note = first_line(e) or describe_exception(e) + logger.info("数据源 %s 的对象清单字典查询失败,退回 Inspector:%s", getattr(source, "name", "?"), note) + else: + objects: list[IndexedObject] = [] + for row in result.rows: + if not row or row[0] is None: + continue + raw_name = str(row[0]) + name: Any = dialect.normalize_name(raw_name) if kind == "oracle" else raw_name + kind_of = _object_kind(row[1] if len(row) > 1 else None) + objects.append(IndexedObject( + name=str(name), kind=kind_of, comment=_comment(row[2] if len(row) > 2 else None, kind_of), + est_rows=_est_rows(row[3] if len(row) > 3 else None), quote=_forced(name))) + capped = len(objects) > INDEX_LIMIT or bool(getattr(result, "truncated", False)) + return IndexResult(objects=objects[:INDEX_LIMIT], method="dictionary", + total=len(objects), capped=capped) + + from sqlalchemy import inspect as sa_inspect + + def _names(sync_conn: Any) -> tuple[list[Any], list[Any]]: + inspector = sa_inspect(sync_conn) + return list(inspector.get_table_names(schema=schema)), list(inspector.get_view_names(schema=schema)) + + async with engine.connect() as conn: + tables, views = await conn.run_sync(_names) + merged = sorted([IndexedObject(str(n), "table", quote=_forced(n)) for n in tables] + + [IndexedObject(str(n), "view", quote=_forced(n)) for n in views], key=lambda o: name_key(o.name)) + return IndexResult(objects=merged[:INDEX_LIMIT], method="inspector", total=len(merged), + capped=len(merged) > INDEX_LIMIT, note=note) + + +def _forced(name: Any) -> bool: + return getattr(name, "quote", None) is True + + +# ========================================================================== +# 探查范围 +# ========================================================================== + + +@dataclass +class ScopeRules: + """数据源的探查范围规则(options.introspect_scope)。缺省 mode=all:全部对象(视图也算),上限 500 张。""" + + mode: str = "all" + include: list[str] = field(default_factory=list) + exclude: list[str] = field(default_factory=list) + tables: list[str] = field(default_factory=list) + removed: list[str] = field(default_factory=list) + views: bool = True + max_tables: int = SCOPE_DEFAULT + updated_by: str | None = None + updated_at: str | None = None + + @classmethod + def from_options(cls, options: Mapping[str, Any] | None) -> ScopeRules: + raw = (options or {}).get(SCOPE_OPTION) + return cls.from_dict(raw if isinstance(raw, Mapping) else {}) + + @classmethod + def from_dict(cls, raw: Mapping[str, Any]) -> ScopeRules: + """存着的规则 → ScopeRules。不合规的项按缺省读(存进去之前已经核对过:这里只防手改过的库)。""" + def words(key: str) -> list[str]: + value = raw.get(key) + items = value if isinstance(value, list) else [] + return list(dict.fromkeys(str(x).strip() for x in items if isinstance(x, (str, int)) and str(x).strip())) + + cap = raw.get("max_tables") + cap = cap if isinstance(cap, int) and not isinstance(cap, bool) and 1 <= cap <= SCOPE_MAX else SCOPE_DEFAULT + return cls(mode="rules" if raw.get("mode") == "rules" else "all", include=words("include"), + exclude=words("exclude"), tables=words("tables"), removed=words("removed"), + views=raw.get("views") is not False, max_tables=cap, + updated_by=raw.get("updated_by") if isinstance(raw.get("updated_by"), str) else None, + updated_at=raw.get("updated_at") if isinstance(raw.get("updated_at"), str) else None) + + def to_dict(self) -> dict[str, Any]: + out: dict[str, Any] = {"mode": self.mode, "include": list(self.include), "exclude": list(self.exclude), + "tables": list(self.tables), "removed": list(self.removed), "views": self.views, + "max_tables": self.max_tables} + if self.updated_by is not None: + out["updated_by"] = self.updated_by + if self.updated_at is not None: + out["updated_at"] = self.updated_at + return out + + def rules_only(self) -> dict[str, Any]: + """留痕用的规则本身(不含谁改的)。""" + return {k: v for k, v in self.to_dict().items() if k not in ("updated_by", "updated_at")} + + def matches(self, name: str, kind: str) -> bool: + """规则命中:mode=all 时全部(包含列表不看),rules 时命中包含里的一条;命中排除的一律不算(排除优先); + views=false 时视图不算。显式列出、pinned、按需加入不经这里。""" + if kind == "view" and not self.views: + return False + key = name_key(name) + if any(_glob(key, p) for p in self.exclude): + return False + if self.mode == "all": + return True + return any(_glob(key, p) for p in self.include) + + +def _glob(key: str, pattern: str) -> bool: + """通配(* ? [...])按 name_key 比;写了 schema 前缀的(sales.ord_*)只看表名那一段。""" + pat = name_key(pattern.rsplit(".", 1)[-1]) + return fnmatch.fnmatchcase(key, pat) + + +def rules_problem(raw: Any) -> str | None: + """PUT /introspect-scope 交来的规则合不合规:不合规返回一句中文,合规返回 None。""" + if not isinstance(raw, Mapping): + return "探查范围要是一个对象:{mode, include, exclude, tables, removed, views, max_tables}" + if raw.get("mode", "all") not in ("all", "rules"): + return "探查范围的方式只能是「全部对象」(all)或「按规则」(rules)" + for key, label, limit in (("include", "包含的规则", RULES_MAX), ("exclude", "排除的规则", RULES_MAX), + ("tables", "显式列出的表", SCOPE_MAX), ("removed", "显式移出的表", SCOPE_MAX)): + value = raw.get(key, []) + if not isinstance(value, list) or not all(isinstance(x, str) for x in value): + return f"{label}要是一组文字" + if len(value) > limit: + return f"{label}最多 {limit} 条,当前 {len(value)} 条" + if any(len(x) > RULE_CHARS for x in value): + return f"{label}每条最长 {RULE_CHARS} 个字" + cap = raw.get("max_tables", SCOPE_DEFAULT) + if not isinstance(cap, int) or isinstance(cap, bool) or not 1 <= cap <= SCOPE_MAX: + return f"探查范围的上限要在 1 到 {SCOPE_MAX:,} 张之间" + if "views" in raw and not isinstance(raw["views"], bool): + return "是否包含视图要是 true 或 false" + return None + + +@dataclass +class ScopePlan: + """按规则圈出的范围。in_scope:{名字: 原因},按名字排序;counts 各原因几张;capped_count 是计上限的那部分(规则、 + pinned、显式;按需加入另算)。""" + + in_scope: dict[str, str] + counts: dict[str, int] + cap: int + index_total: int + out_of_scope: int + + @property + def capped_count(self) -> int: + return sum(n for r, n in self.counts.items() if r != "on_demand") + + @property + def over(self) -> bool: + return self.capped_count > self.cap or len(self.in_scope) > SCOPE_MAX + + def summary(self, rules: ScopeRules | None = None) -> dict[str, Any]: + out: dict[str, Any] = {"in_scope": len(self.in_scope), "hits": self.capped_count, "cap": self.cap, + "out_of_scope": self.out_of_scope, "index_total": self.index_total, + **{r: self.counts.get(r, 0) for r in SCOPE_REASONS}} + if rules is not None: + out["rules"] = rules.rules_only() + return out + + +def plan_scope(objects: Iterable[tuple[str, str]], rules: ScopeRules, *, pinned: set[str], + on_demand: set[str]) -> ScopePlan: + """objects 是对象索引里还在的对象 (名字, 类型);pinned、on_demand 是 name_key 的集合。原因的先后:显式移出 > pinned > + 显式列出 > 规则 > 按需加入。""" + removed = {name_key(n.rsplit(".", 1)[-1]) for n in rules.removed} + explicit = {name_key(n.rsplit(".", 1)[-1]) for n in rules.tables} + picked: dict[str, str] = {} + total = 0 + for name, kind in objects: + total += 1 + key = name_key(name) + if key in removed: + continue + if key in pinned: + picked[name] = "pinned" + elif key in explicit: + picked[name] = "explicit" + elif rules.matches(name, kind): + picked[name] = "rule" + elif key in on_demand: + picked[name] = "on_demand" + counts = {r: 0 for r in SCOPE_REASONS} + for reason in picked.values(): + counts[reason] += 1 + ordered = dict(sorted(picked.items(), key=lambda kv: name_key(kv[0]))) + return ScopePlan(in_scope=ordered, counts=counts, cap=rules.max_tables, index_total=total, + out_of_scope=total - len(ordered)) + + +def too_large_message(plan: ScopePlan) -> str: + """规则命中超过上限时的整句:命中多少张、其中一律在范围里的几张、上限、怎么办。""" + pinned = plan.counts.get("pinned", 0) + held = f"(其中有数据目录或被已发布模板用到、一律在范围内的 {pinned:,} 张)" if pinned else "" + if len(plan.in_scope) > SCOPE_MAX and plan.capped_count <= plan.cap: + return (f"探查范围里已有 {len(plan.in_scope):,} 张表(含按需加入的 {plan.counts.get('on_demand', 0):,} 张)," + f"超过最多 {SCOPE_MAX:,} 张。请在「探查范围」里移出一些按需加入的表,再探查") + return (f"按探查范围的规则命中 {plan.capped_count:,} 张表{held},超过上限 {plan.cap:,} 张,没有开始探查。" + f"请在「探查范围」里收窄规则(加包含、排除的通配),或者把上限调高(最多 {SCOPE_MAX:,} 张)") + + +# ========================================================================== +# 数据源的 schema 与锁 +# ========================================================================== + + +def index_schema(row: Any) -> str: + """对象索引按哪个 schema 记:数据源配置的 schema(一个数据源一个 schema,目录-D2),没配是 ""(连接的默认 schema)。""" + return str((getattr(row, "options", None) or {}).get("schema") or "").strip() + + +def target_schema(row: Any) -> str | None: + """反射、字典查询用的 schema:配置了就是它,没配是 None(当前用户的默认 schema)。""" + return index_schema(row) or None + + +_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, asyncio.Lock]]" = weakref.WeakKeyDictionary() + + +def source_lock(source_id: str) -> asyncio.Lock: + """写一个源的对象索引和 schema_cache 时持的锁(进程内,按事件循环各一份:asyncio.Lock 不能跨循环用)。后台探查的开头 + 和收尾、按需加入、加入范围都持它。""" + loop = asyncio.get_running_loop() + by_source = _locks.get(loop) + if by_source is None: + by_source = _locks[loop] = {} + lock = by_source.get(source_id) + if lock is None: + lock = by_source[source_id] = asyncio.Lock() + return lock + + +def _scope_filter(source_id: str, schema: str) -> tuple[Any, ...]: + return (SourceObject.source_id == source_id, SourceObject.schema_name == schema) + + +async def has_index(session: AsyncSession, source_id: str, schema: str) -> bool: + """这个源在这个 schema 下有没有对象索引(升级前探查的源、上传表格都没有)。""" + return (await session.execute( + select(SourceObject.id).where(*_scope_filter(source_id, schema)).limit(1))).first() is not None + + +# ========================================================================== +# 对象索引落盘 +# ========================================================================== + + +@dataclass +class RefreshStats: + total: int + added: int + removed: int + method: str + capped: bool + note: str | None = None + + +def _chunks(items: list[Any], size: int = 500) -> Iterable[list[Any]]: + for i in range(0, len(items), size): + yield items[i:i + size] + + +async def refresh_index(session: AsyncSession, source_id: str, schema: str, result: IndexResult) -> RefreshStats: + """把字典查询的结果写进对象索引(不提交):新出现的对象加一行;还在的更新类型、注释、行数估算、last_seen_at(删了又 + 出现的清掉 removed_at);不在了的记 removed_at。索引被截断(capped)时不判删除:没收进来的不等于不在了。""" + now = utcnow() + existing = {r.name: r for r in (await session.execute( + select(SourceObject.id, SourceObject.name, SourceObject.removed_at, SourceObject.kind, SourceObject.comment, + SourceObject.est_rows).where(*_scope_filter(source_id, schema)))).all()} + seen: set[str] = set() + new_rows: list[dict[str, Any]] = [] + changed: list[dict[str, Any]] = [] + revived: list[str] = [] + for obj in result.objects: + if obj.name in seen: + continue + seen.add(obj.name) + old = existing.get(obj.name) + if old is None: + new_rows.append({"source_id": source_id, "schema_name": schema, "name": obj.name, + "name_key": name_key(obj.name), "kind": obj.kind, "comment": obj.comment, + "est_rows": obj.est_rows, "quote_name": obj.quote, "first_seen_at": now, + "last_seen_at": now}) + continue + if old.removed_at is not None: + revived.append(old.id) + if (old.kind, old.comment, old.est_rows) != (obj.kind, obj.comment, obj.est_rows): + changed.append({"_id": old.id, "kind": obj.kind, "comment": obj.comment, "est_rows": obj.est_rows}) + _SO = SourceObject + for chunk in _chunks(new_rows): + await session.execute(insert(_SO), chunk) + if changed: + stmt = (update(_SO).where(_SO.id == bindparam("_id")) + .values(kind=bindparam("kind"), comment=bindparam("comment"), est_rows=bindparam("est_rows")) + .execution_options(synchronize_session=False)) + for chunk in _chunks(changed): + await session.execute(stmt, chunk) + still = [r.id for name, r in existing.items() if name in seen] + for chunk in _chunks(still): + await session.execute(update(_SO).where(_SO.id.in_(chunk)).values(last_seen_at=now) + .execution_options(synchronize_session=False)) + for chunk in _chunks(revived): + await session.execute(update(_SO).where(_SO.id.in_(chunk)).values(removed_at=None) + .execution_options(synchronize_session=False)) + gone: list[str] = [] + if not result.capped: + gone = [r.id for name, r in existing.items() if name not in seen and r.removed_at is None] + for chunk in _chunks(gone): + # 真删除:不在范围里了,结构(meta)留着——下一波的结构差异要拿它和以前比 + await session.execute(update(_SO).where(_SO.id.in_(chunk)).values(removed_at=now, in_scope=False) + .execution_options(synchronize_session=False)) + return RefreshStats(total=len(seen), added=len(new_rows), removed=len(gone), method=result.method, + capped=result.capped, note=result.note) + + +async def live_objects(session: AsyncSession, source_id: str, schema: str) -> list[tuple[str, str]]: + """对象索引里还在的对象 (名字, 类型),按名字。""" + return [(r.name, r.kind) for r in (await session.execute( + select(SourceObject.name, SourceObject.kind).where(*_scope_filter(source_id, schema), + SourceObject.removed_at.is_(None)) + .order_by(SourceObject.name_key))).all()] + + +async def pinned_keys(session: AsyncSession, row: Any, names: Iterable[str]) -> set[str]: + """一律在范围里的表(name_key):有数据目录的表,加上已发布模板写死的 SQL 用到的这个源的表(catalog_impact._published + + graph_sql_tables,和影响面同一份)。图解析不了的版本跳过。""" + from app.data.catalog_impact import _published, graph_sql_tables + from app.db.models import CatalogNote + from app.engine.schema import GraphSpec + + keys = {name_key(t) for t in (await session.execute( + select(CatalogNote.table_name).where(CatalogNote.source_id == row.id))).scalars()} + known = list(names) + for _workflow, version in await _published(session): + try: + spec = GraphSpec.model_validate(version.graph or {}) + except Exception: # noqa: BLE001 - 解析不了的版本发起不了正式运行 + continue + hits = graph_sql_tables(spec, {row.name: known}).get(row.name) + if hits: + keys |= {name_key(t) for t in hits} + return keys + + +async def on_demand_keys(session: AsyncSession, source_id: str, schema: str) -> set[str]: + return {name_key(n) for n in (await session.execute( + select(SourceObject.name).where(*_scope_filter(source_id, schema), SourceObject.scope_reason == "on_demand", + SourceObject.in_scope.is_(True)))).scalars()} + + +async def plan_for(session: AsyncSession, row: Any, rules: ScopeRules | None = None) -> ScopePlan: + """按规则(不给是数据源存着的)和这一刻的对象索引圈范围。""" + schema = index_schema(row) + rules = rules or ScopeRules.from_options(getattr(row, "options", None)) + objects = await live_objects(session, row.id, schema) + return plan_scope(objects, rules, pinned=await pinned_keys(session, row, [n for n, _ in objects]), + on_demand=await on_demand_keys(session, row.id, schema)) + + +async def apply_scope(session: AsyncSession, source_id: str, schema: str, plan: ScopePlan) -> None: + """把圈好的范围写进对象索引(不提交):离开范围的清掉结构(指纹和版本留着),范围内的按原因记。""" + current = {r.name: r for r in (await session.execute( + select(SourceObject.id, SourceObject.name, SourceObject.scope_reason).where( + *_scope_filter(source_id, schema), SourceObject.in_scope.is_(True)))).all()} + leaving = [r.id for name, r in current.items() if name not in plan.in_scope] + for chunk in _chunks(leaving): + await session.execute(update(SourceObject).where(SourceObject.id.in_(chunk)) + .values(in_scope=False, scope_reason=None, meta=None, reflect_error=None) + .execution_options(synchronize_session=False)) + by_reason: dict[str, list[str]] = {} + for name, reason in plan.in_scope.items(): + old = current.get(name) + if old is None or old.scope_reason != reason: + by_reason.setdefault(reason, []).append(name) + for reason, names in by_reason.items(): + for chunk in _chunks(names): + await session.execute(update(SourceObject).where(*_scope_filter(source_id, schema), + SourceObject.name.in_(chunk)) + .values(in_scope=True, scope_reason=reason) + .execution_options(synchronize_session=False)) + + +# ========================================================================== +# 反射与结构版本 +# ========================================================================== + + +def fingerprint(meta: Mapping[str, Any]) -> str: + """表结构的指纹:反射到的整份结构的规范 JSON(键排序)的 sha256。""" + text = json.dumps(meta, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str) + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +@dataclass +class Reflected: + tables: dict[str, dict[str, Any]] + skipped: dict[str, str] + + +async def reflect(source: Any, names: list[tuple[str, bool]], *, views: set[str], + timeout: float | None = None) -> Reflected: + """只读反射这几个对象(名字, 要不要强制引号)。连接本身出错抛出原来的异常;单个对象读不到的进 skipped。""" + from sqlalchemy.sql.elements import quoted_name + + from app.data import engine as data_engine + from app.data import introspect + + engine = await data_engine.engines.get(source) + target = target_schema(source) + picked = [quoted_name(n, True) if quote else n for n, quote in names] + + def _go(sync_conn: Any) -> dict[str, Any]: + return introspect.reflect_names(sync_conn, target, picked, views) + + async def _run() -> dict[str, Any]: + async with engine.connect() as conn: + return await conn.run_sync(_go) + + got = await (asyncio.wait_for(_run(), timeout=timeout) if timeout else _run()) + return Reflected(tables={str(k): v for k, v in got["tables"].items()}, + skipped={str(k): v for k, v in got["skipped"].items()}) + + +async def save_reflected(session: AsyncSession, source_id: str, schema: str, got: Reflected) -> dict[str, int]: + """反射结果写进对象索引的行(不提交):读到的换上新结构,指纹变了结构版本加 1(第一次读到是 1);读不到的记原因, + 结构留着上一次读到的。返回 {名字: 结构版本}。""" + names = list(got.tables) + list(got.skipped) + if not names: + return {} + rows = {r.name: r for r in (await session.execute( + select(SourceObject.id, SourceObject.name, SourceObject.fingerprint, SourceObject.struct_version).where( + *_scope_filter(source_id, schema), SourceObject.name.in_(names)))).all()} + now = utcnow() + versions: dict[str, int] = {} + for name, meta in got.tables.items(): + row = rows.get(name) + if row is None: + continue + fp = fingerprint(meta) + version = row.struct_version or 0 + if fp != row.fingerprint or not version: + version += 1 + versions[name] = version + await session.execute(update(SourceObject).where(SourceObject.id == row.id).values( + meta=meta, fingerprint=fp, struct_version=version, reflect_error=None, reflected_at=now) + .execution_options(synchronize_session=False)) + for name, why in got.skipped.items(): + row = rows.get(name) + if row is None: + continue + versions[name] = row.struct_version or 0 + await session.execute(update(SourceObject).where(SourceObject.id == row.id).values(reflect_error=why) + .execution_options(synchronize_session=False)) + return versions + + +# ========================================================================== +# 物化 schema_cache +# ========================================================================== + +#: schema_cache 里由探查失败那一刻写的键:下一次物化时(探查成功)清掉,按需加入时原样留着 +_FAILURE_KEYS = ("failed", "error", "available_schemas") + + +async def rebuild_schema_cache(session: AsyncSession, row: Any, *, failure: Mapping[str, Any] | None = None, + keep_failure: bool = False, synced: bool = False, + index: RefreshStats | None = None) -> dict[str, Any]: + """从范围内已反射的行物化 schema_cache(唯一写它的地方),写进 row(不提交),返回新的缓存。 + + 形状和以前一样:tables(表在前、视图在后,各按名字)、truncated(有范围外的对象)、total(对象索引里的对象总数)、 + schema、synced_at;范围里却读不到结构的进 skipped(同以前的「跳过的对象」);另加 scope 摘要。failure 给了的记上 + 探查失败(failed、error、available_schemas,表结构照旧留着:助手还能按上一次读到的写 SQL);keep_failure 时沿用缓存里 + 原有的失败标记(按需加入不改它)。synced:这次是一次探查完成(synced_at 换成现在),否则沿用原来的。index:这次刷新 + 对象索引的结果(字典查询还是 Inspector、有没有被截断),不给沿用缓存里原来记的。""" + from app.data.source_objects import ScopeRules as _Rules + + schema = index_schema(row) + rows = (await session.execute( + select(SourceObject.name, SourceObject.kind, SourceObject.in_scope, SourceObject.scope_reason, + SourceObject.meta, SourceObject.reflect_error, SourceObject.removed_at) + .where(*_scope_filter(row.id, schema)).order_by(SourceObject.name_key))).all() + old = dict(getattr(row, "schema_cache", None) or {}) + tables: dict[str, Any] = {} + views: dict[str, Any] = {} + skipped: dict[str, str] = {} + counts = {r: 0 for r in SCOPE_REASONS} + total = in_scope = failed = 0 + for r in rows: + if r.removed_at is not None: + continue + total += 1 + if not r.in_scope: + continue + in_scope += 1 + if r.scope_reason in counts: + counts[r.scope_reason] += 1 + if isinstance(r.meta, dict) and r.meta: + (views if r.kind == "view" else tables)[r.name] = r.meta + if r.reflect_error and not (isinstance(r.meta, dict) and r.meta): + skipped[r.name] = r.reflect_error + failed += 1 + rules = _Rules.from_options(getattr(row, "options", None)) + old_scope = old.get("scope") if isinstance(old.get("scope"), dict) else {} + index_info = ({"index_method": index.method, "index_capped": index.capped, "index_at": utcnow().isoformat()} + if index is not None else {k: old_scope[k] for k in ("index_method", "index_capped", "index_at") + if k in old_scope}) + cache: dict[str, Any] = { + "tables": {**tables, **views}, + "truncated": total > in_scope, + "total": total, + "schema": target_schema(row), + "synced_at": datetime.now(timezone.utc).isoformat() if synced or not old.get("synced_at") + else old.get("synced_at"), + "scope": {"in_scope": in_scope, "reflected": len(tables) + len(views), "failed": failed, + "out_of_scope": total - in_scope, "cap": rules.max_tables, "index_total": total, **counts, + **index_info}, + } + if skipped: + cache["skipped"] = skipped + if failure: + cache.update({k: failure[k] for k in _FAILURE_KEYS if k in failure}) + elif keep_failure: + cache.update({k: old[k] for k in _FAILURE_KEYS if k in old}) + row.schema_cache = cache + return cache + + +# ========================================================================== +# 三种「不在」、对象索引里的查找 +# ========================================================================== + + +async def absent_reasons(session: AsyncSession, row: Any, names: Iterable[str]) -> dict[str, str]: + """一批不在 schema_cache 里的表名 → 为什么不在(removed / missing / out_of_scope / reflect_failed / unknown)。 + + 没有对象索引的源(升级前探查的):缓存的跳过清单里有的是读不到结构;老缓存截断过的说不清(unknown);没截断的就是 + 表已不在。有对象索引的:记了 removed_at 的是表已不在(真删除);索引里从来没有的是 missing(索引被截断过的说不清); + 不在范围里的是范围外;在范围里的是读不到结构。""" + wanted = list(dict.fromkeys(str(n) for n in names)) + if not wanted: + return {} + cache = getattr(row, "schema_cache", None) or {} + schema = index_schema(row) + if not await has_index(session, row.id, schema): + skipped = cache.get("skipped") or {} + truncated = bool(cache.get("truncated")) + return {n: ABSENT_REFLECT_FAILED if n in skipped else ABSENT_UNKNOWN if truncated else ABSENT_REMOVED + for n in wanted} + keys = {name_key(n.rsplit(".", 1)[-1]): n for n in wanted} + found = {r.name_key: r for r in (await session.execute( + select(SourceObject.name_key, SourceObject.in_scope, SourceObject.removed_at, SourceObject.meta, + SourceObject.reflect_error) + .where(*_scope_filter(row.id, schema), SourceObject.name_key.in_(list(keys))))).all()} + capped = bool((cache.get("scope") or {}).get("index_capped")) + out: dict[str, str] = {} + for key, name in keys.items(): + r = found.get(key) + if r is None: + out[name] = ABSENT_UNKNOWN if capped else ABSENT_MISSING + elif r.removed_at is not None: + out[name] = ABSENT_REMOVED + elif not r.in_scope: + out[name] = ABSENT_OUT_OF_SCOPE + else: + out[name] = ABSENT_REFLECT_FAILED + return out + + +@dataclass(frozen=True) +class OutsideObject: + """对象索引里在、范围外的一个对象(助手检索、目录搜索用)。""" + + name: str + kind: str + comment: str | None + est_rows: int | None = None + + +async def outside_objects(session: AsyncSession, row: Any) -> list[OutsideObject]: + """范围外、还在的对象,按名字。没有对象索引是空列表。""" + return [OutsideObject(r.name, r.kind, r.comment, r.est_rows) for r in (await session.execute( + select(SourceObject.name, SourceObject.kind, SourceObject.comment, SourceObject.est_rows) + .where(*_scope_filter(row.id, index_schema(row)), SourceObject.removed_at.is_(None), + SourceObject.in_scope.is_(False)).order_by(SourceObject.name_key))).all()] + + +async def outside_names(session: AsyncSession, row: Any) -> frozenset[str]: + """范围外、还在的对象的 name_key(剖析分清「范围外」和「表结构里没有」、SQL 检查列「未按目录检查的表」用)。""" + return frozenset(r for r in (await session.execute( + select(SourceObject.name_key).where(*_scope_filter(row.id, index_schema(row)), + SourceObject.removed_at.is_(None), + SourceObject.in_scope.is_(False)))).scalars()) + + +async def search_index(session: AsyncSession, row: Any, query: str, *, limit: int = 50) -> tuple[list[dict[str, Any]], int]: + """按名字、注释在对象索引里找(不分大小写的包含),名字完全相同的在前、再是开头相同的、再是名字里有的、注释里有的。 + 返回 (前 limit 个 {name, kind, comment, est_rows, in_scope, scope_reason, absent_reason}, 一共命中几个)。""" + text = query.strip() + if not text: + return [], 0 + key = name_key(text.rsplit(".", 1)[-1]) + filt = (*_scope_filter(row.id, index_schema(row)), SourceObject.removed_at.is_(None), + or_(SourceObject.name_key.contains(key, autoescape=True), + SourceObject.comment.contains(text, autoescape=True))) + total = int((await session.execute(select(func.count()).select_from(SourceObject).where(*filt))).scalar() or 0) + rows = (await session.execute( + select(SourceObject.name, SourceObject.name_key, SourceObject.kind, SourceObject.comment, + SourceObject.est_rows, SourceObject.in_scope, SourceObject.scope_reason, SourceObject.meta, + SourceObject.reflect_error).where(*filt))).all() + + def rank(r: Any) -> tuple[int, str]: + if r.name_key == key: + return 0, r.name_key + if r.name_key.startswith(key): + return 1, r.name_key + if key in r.name_key: + return 2, r.name_key + return 3, r.name_key + + out = [] + for r in sorted(rows, key=rank)[:limit]: + reflected = isinstance(r.meta, dict) and bool(r.meta) + out.append({"name": r.name, "kind": r.kind, "comment": r.comment, "est_rows": r.est_rows, + "in_scope": bool(r.in_scope), "scope_reason": r.scope_reason if r.in_scope else None, + "absent_reason": None if r.in_scope and reflected + else ABSENT_REFLECT_FAILED if r.in_scope else ABSENT_OUT_OF_SCOPE}) + return out, total + + +async def on_demand_list(session: AsyncSession, row: Any, *, limit: int = 200) -> list[dict[str, Any]]: + """按需加入、目录页手工加入范围的表和留痕,新的在前。""" + rows = (await session.execute( + select(SourceObject).where(*_scope_filter(row.id, index_schema(row)), SourceObject.in_scope.is_(True), + SourceObject.added_at.is_not(None)) + .order_by(SourceObject.added_at.desc()).limit(limit))).scalars() + return [trail_dict(r) for r in rows] + + +def trail_dict(r: SourceObject) -> dict[str, Any]: + return {"name": r.name, "scope_reason": r.scope_reason, "via": r.added_via, "by": r.added_by, + "run_id": r.added_run_id, "conversation_id": r.added_conversation_id, "ref": r.added_ref, + "need": r.added_need, "at": r.added_at.isoformat() if r.added_at else None} + + +async def count_added_in_run(session: AsyncSession, run_id: str) -> int: + """这次运行里按需加入了几张(各源合计):每次运行最多 ON_DEMAND_PER_RUN 张。""" + return int((await session.execute( + select(func.count()).select_from(SourceObject).where(SourceObject.added_run_id == run_id, + SourceObject.scope_reason == "on_demand") + )).scalar() or 0) + + +# ========================================================================== +# 按需加入、加入范围 +# ========================================================================== + + +@dataclass +class Trail: + """按需加入的留痕:经哪个入口、谁、哪次运行 / 哪一轮对话、哪一下(look 编号、节点 id)、需求原文。""" + + via: str + actor: str | None = None + run_id: str | None = None + conversation_id: str | None = None + ref: str | None = None + need: str | None = None + + +@dataclass +class AddOutcome: + """加入范围的结果。added:加进来、读到了结构的(schema_cache 的键);already:本来就在范围里、读得到的;failed: + {名字: 读不到的原因}(也加进了范围,下次探查再读);refused:{名字: 为什么没加}(对象索引里没有、范围满了); + cache:加完以后的 schema_cache(什么都没加时为 None)。""" + + added: list[str] = field(default_factory=list) + already: list[str] = field(default_factory=list) + failed: dict[str, str] = field(default_factory=dict) + refused: dict[str, str] = field(default_factory=dict) + cache: dict[str, Any] | None = None + + +NOT_IN_INDEX = "对象索引里没有这张表(数据源里没有,或者是对象索引刷新之后才建的:重新探查结构后再试)" +REMOVED_FROM_INDEX = "对象索引里这张表已经不在了(表已删除)" +SCOPE_FULL = f"探查范围已有 {{n:,}} 张表,最多 {SCOPE_MAX:,} 张,不能再加入" +EXPLICITLY_REMOVED = "这张表被显式移出了探查范围:要用它,请在「探查范围」里把它从移出的表里去掉" + + +async def add_to_scope(source_id: str, names: list[str], *, trail: Trail, reason: str = "on_demand") -> AddOutcome: + """把对象索引里范围外的这几张表加进范围:当场只读反射、落盘、记下留痕,再物化 schema_cache。reason:on_demand(助手、 + 运行时按需加入)或 explicit(目录页手工加入,同时记进规则的显式列出)。用自己的会话,不碰调用方会话里加载的对象。 + 反射时限是这个源的单条查询时限(大约 5 条字典查询,见 engine.query_timeout)。""" + from app.data import engine as data_engine + from app.db.models import DataSource + + out = AddOutcome() + wanted = list(dict.fromkeys(n.strip() for n in names if n and n.strip())) + if not wanted: + return out + async with source_lock(source_id): + async with SessionLocal() as session: + row = await session.get(DataSource, source_id) + if row is None: + out.refused = dict.fromkeys(wanted, "数据源不存在,可能已被删除") + return out + schema = index_schema(row) + rules = ScopeRules.from_options(row.options) + removed = {name_key(n.rsplit(".", 1)[-1]) for n in rules.removed} + keys = {name_key(n.rsplit(".", 1)[-1]): n for n in wanted} + found = {r.name_key: r for r in (await session.execute( + select(SourceObject).where(*_scope_filter(source_id, schema), + SourceObject.name_key.in_(list(keys))))).scalars()} + in_scope = int((await session.execute(select(func.count()).select_from(SourceObject).where( + *_scope_filter(source_id, schema), SourceObject.in_scope.is_(True), + SourceObject.removed_at.is_(None)))).scalar() or 0) + todo: list[SourceObject] = [] + for key, asked in keys.items(): + obj = found.get(key) + if obj is None: + out.refused[asked] = NOT_IN_INDEX + elif obj.removed_at is not None: + out.refused[asked] = REMOVED_FROM_INDEX + elif obj.in_scope and isinstance(obj.meta, dict) and obj.meta: + out.already.append(obj.name) + elif key in removed and reason != "explicit": + out.refused[asked] = EXPLICITLY_REMOVED + elif not obj.in_scope and in_scope + len(todo) >= SCOPE_MAX: + out.refused[asked] = SCOPE_FULL.format(n=in_scope + len(todo)) + else: + todo.append(obj) + if not todo: + return out + try: + got = await reflect(row, [(o.name, bool(o.quote_name)) for o in todo], + views={o.name for o in todo if o.kind == "view"}, + timeout=data_engine.query_timeout(row) * 2) + except asyncio.TimeoutError: + for o in todo: + out.failed[o.name] = f"读表结构超过 {data_engine.query_timeout(row) * 2:g} 秒没有完成" + got = Reflected(tables={}, skipped=dict(out.failed)) + except Exception as e: # noqa: BLE001 - 连不上:照实记下,不挡调用方 + why = first_line(e) or describe_exception(e) + for o in todo: + out.failed[o.name] = why + got = Reflected(tables={}, skipped=dict(out.failed)) + now = utcnow() + for o in todo: + was_in = bool(o.in_scope) + o.in_scope = True + if not was_in or reason == "explicit": + o.scope_reason = reason if not was_in or o.scope_reason in (None, "on_demand") else o.scope_reason + o.added_via, o.added_by, o.added_at = trail.via, (trail.actor or None), now + o.added_run_id, o.added_conversation_id = trail.run_id, trail.conversation_id + o.added_ref = (trail.ref or None) and str(trail.ref)[:64] + o.added_need = (trail.need or None) and str(trail.need)[:2000] + await session.flush() + await save_reflected(session, source_id, schema, got) + out.added = [o.name for o in todo if o.name in got.tables] + for name, why in got.skipped.items(): + out.failed.setdefault(name, why) + if reason == "explicit": + listed = {name_key(t) for t in rules.tables} + rules.tables += [o.name for o in todo if name_key(o.name) not in listed] + rules.removed = [n for n in rules.removed + if name_key(n.rsplit(".", 1)[-1]) not in {name_key(o.name) for o in todo}] + rules.updated_by, rules.updated_at = trail.actor, now.isoformat() + row.options = {**(row.options or {}), SCOPE_OPTION: rules.to_dict()} + out.cache = await rebuild_schema_cache(session, row, keep_failure=True) + await session.commit() + logger.info("数据源 %s 按需加入范围:%s(经 %s)", source_id, "、".join(out.added) or "无", trail.via) + return out + + +async def delete_index(session: AsyncSession, source_id: str) -> None: + """删掉一个源的对象索引(测试、清理用;删数据源时随外键级联删除)。""" + await session.execute(delete(SourceObject).where(SourceObject.source_id == source_id)) + + +__all__ = [ + "ABSENT_MISSING", "ABSENT_OUT_OF_SCOPE", "ABSENT_REFLECT_FAILED", "ABSENT_REMOVED", "ABSENT_UNKNOWN", "ADD_LIMIT", "AddOutcome", + "INDEX_LIMIT", "IndexResult", "IndexedObject", "OutsideObject", "REFLECT_BATCH", "RULES_MAX", "Reflected", + "RefreshStats", "SCOPE_DEFAULT", "SCOPE_MAX", "SCOPE_OPTION", "SCOPE_REASONS", "ScopePlan", "ScopeRules", "Trail", + "VIA_CATALOG", "VIA_LOOK", "VIA_SCHEMA_TOOL", "absent_reasons", "add_to_scope", "apply_scope", + "count_added_in_run", "fingerprint", "has_index", "index_schema", "index_sql", "list_objects", "live_objects", + "on_demand_keys", "on_demand_list", "outside_names", "outside_objects", "pinned_keys", "plan_for", "plan_scope", + "rebuild_schema_cache", "reflect", "refresh_index", "rules_problem", "save_reflected", "search_index", + "source_lock", "target_schema", "too_large_message", "trail_dict", +] diff --git a/backend/app/data/sqlcheck.py b/backend/app/data/sqlcheck.py index c3f200f6..0639878c 100644 --- a/backend/app/data/sqlcheck.py +++ b/backend/app/data/sqlcheck.py @@ -1385,16 +1385,19 @@ def _same_columns(text: str, parsed: ValidFilter) -> bool: and not any(c.table for c in columns)) -def stored_valid_filter(parsed: ValidFilter, kind: str | None) -> str: +def stored_valid_filter(parsed: ValidFilter, kind: str | None, meta: Mapping[str, Any] | None = None) -> str: """模型提议的有效行过滤存进目录的写法:canonical_valid_filter,按这个数据源的方言加引号(catalog_profile.SqlDialect: SQLAlchemy 的写法,要加才加——PostgreSQL、SQLite 的 "OrderStatus",MySQL 的 `OrderStatus`,Oracle 探查出来的小写名 - 不加)。""" + 不加)。meta 是这张表的表结构:探查记了 quote_name 的列(Oracle 库里本来就是小写的列)强制加引号,和剖析核对查询 + (catalog_profile.check_valid_filter)同一个写法——不加引号 Oracle 转成大写,找的是另一列。""" from app.data.catalog_profile import SqlDialect try: dialect = SqlDialect(kind) except ValueError: return parsed.text + if meta is not None: + return canonical_valid_filter(parsed, quote=lambda name: dialect.column(dict(meta), name)) return canonical_valid_filter(parsed, quote=dialect.quote) diff --git a/backend/app/db/models_sources.py b/backend/app/db/models_sources.py index 949d5ed1..b684ffed 100644 --- a/backend/app/db/models_sources.py +++ b/backend/app/db/models_sources.py @@ -1,5 +1,137 @@ """数据源的对象索引、探查范围、后台探查任务(docs/design/next-round/03-数据接入与目录维护.md A1)。 -第 2 波 W2-a 填。由 app/db/models.py 末尾整模块导入,create_all 建表。 +由 app/db/models.py 末尾整模块导入,create_all 建表。 + +**为什么要对象索引。** 以前探查结构每个源最多取 200 个对象、按数据库列出的顺序截断,多出来的表哪儿都看不到,目录、look、 +剖析还会把落在窗口外的表说成「表已不在」。现在全库每个对象一行(source_objects,一种方言一条字典查询拿名字、类型、注释, +上限 5 万个),探查范围(数据源 options.introspect_scope 的规则,缺省 500 张、最多 2,000 张)圈出要完整反射的那一部分, +只有范围内的行带着反射到的完整结构(meta)。schema_cache 由 data/source_objects.rebuild_schema_cache 从范围内已反射的行 +物化出来,形状不变:消费它的两百多处不用改。 + +**结构版本。** 每张表一个指纹(反射到的结构的规范 JSON 的哈希)和 struct_version:指纹变了才加 1。第 3 波的结构差异按它 +钉版本、出前后差异;这一波只把版本写好,不出变化记录。 + +**探查任务。** 探查改成后台任务(data/introspect_jobs.py),生命周期是 core/job_runtime 那一套:和同源的起草、剖析排同一条 +队(SourceLanes 的 introspect),逐表记状态,能停止,服务重启标中断、不续跑。表照剖析任务的写法:任务一行、逐表一行。 """ from __future__ import annotations + +from datetime import datetime +from typing import Any + +from sqlalchemy import Boolean, ForeignKey, Index, Integer, String, Text, UniqueConstraint, text +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, UTCDateTime, new_id, utcnow + + +class SourceObject(Base): + """数据源里的一个对象(表或视图)。同一个源同一个 schema 下按名字一行(一个数据源一个 schema,目录-D2):换了 schema + 的源,旧 schema 的行留着不动、不算删除,换回来接着用。 + + - name:schema_cache["tables"] 的键(SQLAlchemy 规整过的写法:Oracle 全大写的名字是小写);name_key 按 names.name_key, + 范围规则、按名字找表都按它比; + - in_scope / scope_reason:在不在探查范围里、为什么(rule 规则命中、explicit 显式列出、pinned 有数据目录或被已发布 + 模板写死的 SQL 用到、on_demand 助手或运行时按需加入); + - meta:反射到的完整结构(schema_cache 里一张表的那份),只有范围内的才有;离开范围时清掉,指纹和版本留着; + - reflect_error:在范围里、却反射不出来(读不到列、权限不够)的原因;这时 meta 是上一次反射成功的那份(有的话); + - removed_at:对象索引里已经没有它了(真删除)。只有它说「表已不在」:范围外、反射失败都不是删除; + - added_*:按需加入、在目录页「加入范围」的留痕:谁、经哪个入口(look / db_schema / catalog)、哪一轮对话或哪次运行、 + 需求原文、什么时候。""" + + __tablename__ = "source_objects" + __table_args__ = ( + UniqueConstraint("source_id", "schema_name", "name"), + Index("ix_source_objects_scope", "source_id", "schema_name", "in_scope"), + Index("ix_source_objects_run", "added_run_id"), + ) + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + source_id: Mapped[str] = mapped_column(ForeignKey("data_sources.id", ondelete="CASCADE")) + #: 这一行是哪个 schema 的("" 是连接的默认 schema) + schema_name: Mapped[str] = mapped_column(String(255), default="") + name: Mapped[str] = mapped_column(String(255)) + name_key: Mapped[str] = mapped_column(String(255)) + #: table | view + kind: Mapped[str] = mapped_column(String(10), default="table") + comment: Mapped[str | None] = mapped_column(Text, default=None) + #: Oracle 库里本来就是小写的名字(SQLAlchemy 反射回来是 quoted_name(quote=True)):存成字符串就丢了这个标记,反射时 + #: 要按它原样加引号,不然 Oracle 转成大写去找,找的是另一个对象 + quote_name: Mapped[bool] = mapped_column(Boolean, default=False) + #: 数据字典里的行数估算(MySQL TABLE_ROWS、PostgreSQL reltuples、Oracle NUM_ROWS);不知道是 NULL + est_rows: Mapped[int | None] = mapped_column(Integer, default=None) + in_scope: Mapped[bool] = mapped_column(Boolean, default=False) + scope_reason: Mapped[str | None] = mapped_column(String(20), default=None) + meta: Mapped[dict[str, Any] | None] = mapped_column(default=None) + fingerprint: Mapped[str | None] = mapped_column(String(64), default=None) + struct_version: Mapped[int] = mapped_column(Integer, default=0) + reflect_error: Mapped[str | None] = mapped_column(Text, default=None) + first_seen_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + last_seen_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + removed_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + reflected_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + #: look | db_schema | catalog + added_via: Mapped[str | None] = mapped_column(String(20), default=None) + added_by: Mapped[str | None] = mapped_column(String(100), default=None) + added_run_id: Mapped[str | None] = mapped_column(String(32), default=None) + added_conversation_id: Mapped[str | None] = mapped_column(String(32), default=None) + #: look 记录的编号(L1…)、运行里的节点 id:同一轮、同一次运行里是哪一下加的 + added_ref: Mapped[str | None] = mapped_column(String(64), default=None) + added_need: Mapped[str | None] = mapped_column(Text, default=None) + added_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + + +class IntrospectJob(Base): + """一次后台探查。建任务的请求里已经刷新了对象索引、按范围规则圈好了表、核过上限(超了 409,不建任务);任务只做反射: + 逐表(按批)读结构、写进对象索引的行,最后由 rebuild_schema_cache 物化 schema_cache。 + + 同一个源同时只有一个排队或进行中的探查任务(接口先查一遍,部分唯一索引兜底);和同源的起草、剖析按建立的先后排队。 + 挂外键、级联删除:任务是数据源的附属记录。""" + + __tablename__ = "introspect_jobs" + __table_args__ = ( + Index("ix_introspect_jobs_source_created", "source_id", "created_at"), + Index("ux_introspect_jobs_active", "source_id", unique=True, + sqlite_where=text("status IN ('queued', 'running')")), + ) + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + source_id: Mapped[str] = mapped_column(ForeignKey("data_sources.id", ondelete="CASCADE")) + # queued | running | succeeded | partial | failed | cancelled | interrupted + status: Mapped[str] = mapped_column(String(20), default="queued") + #: manual 人点的 | schedule 定时维护(第 3 波) + trigger: Mapped[str] = mapped_column(String(20), default="manual") + #: 这次探查的是哪个 schema("" 是默认 schema) + schema_name: Mapped[str] = mapped_column(String(255), default="") + #: 建任务那一刻的范围:{rules(规则原样), in_scope, rule, pinned, explicit, on_demand, out_of_scope, cap, index_total, + #: index_method, index_capped, removed, added}——留痕:这一次按什么规则、圈了多少张 + scope: Mapped[dict[str, Any]] = mapped_column(default=dict) + created_by: Mapped[str | None] = mapped_column(String(100), default=None) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + #: 整个任务出错、数据源中途停用时的整句原因 + error: Mapped[str | None] = mapped_column(Text, default=None) + + +class IntrospectTask(Base): + """探查任务里的一张表。按 seq 排。状态:pending 排队、running 正在读、ok 读到了、failed 读不到(原因在 error)、 + cancelled 停止或服务重启时还没读。""" + + __tablename__ = "introspect_tasks" + __table_args__ = (UniqueConstraint("job_id", "seq"),) + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + job_id: Mapped[str] = mapped_column(ForeignKey("introspect_jobs.id", ondelete="CASCADE"), index=True) + seq: Mapped[int] = mapped_column(Integer) + table_name: Mapped[str] = mapped_column(String(255)) + #: 为什么在范围里(pinned / explicit / rule / on_demand):开始执行时按它把范围写进对象索引 + reason: Mapped[str | None] = mapped_column(String(20), default=None) + status: Mapped[str] = mapped_column(String(20), default="pending") + error: Mapped[str | None] = mapped_column(Text, default=None) + #: 结构版本:读完以后的(指纹没变就和以前一样) + struct_version: Mapped[int] = mapped_column(Integer, default=0) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + + +__all__ = ["IntrospectJob", "IntrospectTask", "SourceObject"] diff --git a/backend/app/look/base.py b/backend/app/look/base.py index 84604fec..bac6ac0a 100644 --- a/backend/app/look/base.py +++ b/backend/app/look/base.py @@ -7,7 +7,8 @@ look 行(模型输出): look 事件(发给前端、存进 meta.looks): {"op":"look","id":"L1","by":"model"|"system","need":"…","status":"ok"|"index"|"failed","reason"?, "elapsed_ms":…,"objects":[{"kind":"table","source":…,"source_id":…,"named":[…],"picked":[…],"joined":[…], - "missing":[…],"total":…,"explored"?:…,"catalog_tables":…,"schema_synced_at":…,"catalog_versions":{…}}, + "missing":[…],"total":…,"explored"?:…,"catalog_tables":…,"schema_synced_at":…,"catalog_versions":{…}, + "added_to_scope"?:[…],"prefiltered"?:…}, {"kind":"capability","named":[…],"missing":[…],"versions":{…}}, {"kind":"workflow_version","named":[…],"missing":[…]}]} """ @@ -27,6 +28,13 @@ CONTEXT_TABLE_LIMIT = 16 # 一次查看最多给几张表的详情(含中 #: 挑选时限。经 OpenAI 兼容接入的推理模型关不掉思考(look/pick.PICK_MAX_TOKENS),先想再写:原来的 20 秒常常想到一半就 #: 掐了,交回目录让主模型自己再看一遍反而更慢 PICK_TIMEOUT_S = 45 +#: 大库的检索预筛(next-round 03 A2):候选超过这么多张时,挑选模型只看检索(schema_view.search_tables)的前这么多个命中、 +#: 工作流已用到的表、命中表的一跳关联表(最多 NEIGHBOR_LIMIT 张),挑选提示词从几万字降到几千字;不超过时照旧给整份目录 +PREFILTER_LIMIT = 120 +NEIGHBOR_LIMIT = 40 +#: 按需加入(目录-D4):点名或挑中探查范围外的表,当场只读反射这一张、落盘、留痕。每次查看最多几张,每次运行最多几张 +ON_DEMAND_PER_LOOK = 4 +ON_DEMAND_PER_RUN = 8 class LookTarget(TypedDict, total=False): diff --git a/backend/app/look/service.py b/backend/app/look/service.py index 769daeee..933f3de6 100644 --- a/backend/app/look/service.py +++ b/backend/app/look/service.py @@ -165,6 +165,11 @@ class LookService: """真正查看:(交回的正文, objects[], 这一次给出详情的对象的身份)。status / reason 写进 record。""" targets = parse_targets(line) sources = _sources(line) + for kind in self.kinds.values(): + # 这一次查看开始:按需加入的上限按次数算、留痕要记需求原文和查看编号(只有表这一类用得着) + begin = getattr(kind, "begin", None) + if callable(begin): + begin(need=need, ref=str(record.get("id") or "")) unknown = [t["kind"] for t in targets if t["kind"] not in self.kinds] named: dict[str, list[LookTarget]] = {} picks: dict[str, list[str] | None] = {} diff --git a/backend/app/look/tables.py b/backend/app/look/tables.py index 0d4e61d4..ff89c3cc 100644 --- a/backend/app/look/tables.py +++ b/backend/app/look/tables.py @@ -2,11 +2,17 @@ - overview:每个库一行(data/schema_view.overview_line),外加「现有工作流用到的表」只列表名; - resolve:点名直取,认全名、只写表名、大小写不同、「库名.表名」(catalog.resolve_table_name);只有一个候选库时可以不写库名; -- pick:候选库的表加起来不超过 PICK_LIMIT 张就全给,否则用挑选模型按需求从表目录(表名|中文名|粒度)里挑; +- pick:候选库的表加起来不超过 PICK_LIMIT 张就全给,否则用挑选模型按需求从表目录(表名|中文名|粒度)里挑;候选超过 + PREFILTER_LIMIT 张(几千张表的库)时先按需求检索一遍(schema_view.search_tables,BM25),挑选模型只看前 PREFILTER_LIMIT + 个命中、工作流已用到的表、命中表的一跳关联表(next-round 03 A2); +- 按需加入:点名或挑中探查范围外的表(对象索引里有、schema_cache 里没有),当场只读反射这一张、落盘、留痕 + (source_objects.add_to_scope:谁、哪一轮、哪一次查看、需求原文),每次查看最多 ON_DEMAND_PER_LOOK 张,超了照实说; + look 记录里记 added_to_scope。读到的结构这一轮接着用(换成带新缓存的源),下一轮、运行时、SQL 检查、目录都看得到; - detail:沿关系图补上两跳以内的中间表(合计不超过 CONTEXT_TABLE_LIMIT),每张表给全部字段和数据目录(遮罩列只给 - 列名和类型),再给表之间的连接条件和连接路径;超过字数上限的表只给字段名。 + 列名和类型)、剖析事实(行数、日期范围,附「只供写查询参考」),再给表之间的连接条件和连接路径;超过字数上限的表只给 + 字段名。 -表结构和目录在开流之前读好(load):目录用单独的会话读,读不到就当没有,不挡生成。 +表结构、目录、剖析事实、范围外的对象清单在开流之前读好(load):用单独的会话读,读不到就当没有,不挡生成。 """ from __future__ import annotations @@ -17,7 +23,10 @@ from typing import Any from app.data import catalog, introspect, schema_view from app.data.catalog import INFERRED_MARK, JoinEdge -from app.look.base import CONTEXT_TABLE_LIMIT, PICK_LIMIT, LookTarget +from app.data.names import name_key +from app.look.base import ( + CONTEXT_TABLE_LIMIT, NEIGHBOR_LIMIT, ON_DEMAND_PER_LOOK, PICK_LIMIT, PREFILTER_LIMIT, LookTarget, +) from app.look.pick import Picker, PickFailed logger = logging.getLogger(__name__) @@ -40,10 +49,21 @@ MISSING_PROMPT = "没找到:{items}(表结构里没有,可能没探查到 MISSING_SOURCE_PROMPT = "没找到数据源「{source}」:不在这一轮可用的数据源里" UNEXPLORED_PROMPT = ("({source} 共 {total} 张表,只探查了 {explored} 张;用户说的表在这里找不到时,先用 db_schema__{source} " "查,也查不到就查数据字典(information_schema.columns,SQLite 用 sqlite_master),别猜)") +#: 有对象索引的源(探查范围以外的表有名字、可以按需加入) +OUT_OF_SCOPE_PROMPT = ("({source} 共 {total} 张表,探查范围内 {explored} 张;用户说的表在这里找不到时,点名 look 它,或者用 " + "db_schema__{source} 按 keyword 查:范围外的表会按需加入,别猜)") +ADDED_PROMPT = "按需加入探查范围:{tables}(在探查范围外,这次当场读了结构)" +ADD_CAPPED_PROMPT = "一次查看最多按需加入 {limit} 张探查范围外的表,{tables} 这次没有加入:下一次 look 点名再看" +ADD_FAILED_PROMPT = "{table} 在探查范围外,按需加入时没能读到结构:{why}" +PREFILTERED_PROMPT = "数据源「{source}」(共 {total} 张表,按需求预筛出 {n} 张,探查范围外的标出来了):" +OUTSIDE_MARK_PROMPT = "(探查范围外,选中后按需加入)" +INDEX_MORE_PROMPT = "…另有 {n} 张没有列出:点名查看,或者换个说法写 need 再 look" INFERRED_PROMPT = ("标「推断,未确认」的说明和连接条件是按列名、数据库注释或模型推断出来的,没有人确认过:照抄前对照字段" "核实") INDEX_HEAD_PROMPT = "数据源「{source}」(共 {total} 张表):" INDEX_PARTIAL_PROMPT = "数据源「{source}」(共 {total} 张表,只探查了 {explored} 张,下面只列探查到的):" +INDEX_SCOPED_PROMPT = ("数据源「{source}」(共 {total} 张表,探查范围内 {explored} 张,下面只列范围内的;范围外的点名查看时" + "按需加入):") PICK_WHAT_PROMPT = "数据表" PICK_LINE_PROMPT = "每行一张表:表名|中文名|粒度" @@ -56,7 +76,13 @@ class SourcePlan: entries: dict[str, catalog.CatalogEntry] = field(default_factory=dict) #: 现有工作流 SQL 里查过的表(schema_cache 的键) seeds: list[str] = field(default_factory=list) + #: 剖析事实 {表名: facts}(catalog_profile_jobs.last_facts):表详情里附一节 + facts: dict[str, dict[str, Any]] = field(default_factory=dict) + #: 对象索引里探查范围外的对象(source_objects.outside_objects):检索、点名时用,点到了按需加入 + outside: list[Any] = field(default_factory=list) _graph: dict[str, list[JoinEdge]] | None = None + _search: Any = None + _outside_keys: dict[str, Any] | None = None @property def name(self) -> str: @@ -82,6 +108,35 @@ class SourcePlan: truncated = (getattr(self.source, "schema_cache", None) or {}).get("truncated") return max(self.total - self.explored, 0) if truncated else 0 + @property + def scoped(self) -> bool: + """有对象索引(按探查范围物化的 schema_cache):范围外的表有名字、可以按需加入。""" + return isinstance((getattr(self.source, "schema_cache", None) or {}).get("scope"), dict) + + def outside_of(self, name: str) -> Any | None: + """对象索引里范围外的这个对象(按 name_key 比,写了 schema 前缀的只看表名)。""" + if self._outside_keys is None: + self._outside_keys = {name_key(o.name): o for o in self.outside} + return self._outside_keys.get(name_key(name.rsplit(".", 1)[-1])) + + def search(self) -> Any: + """这个源的检索(第一次要预筛时才建)。""" + if self._search is None: + self._search = schema_view.TableSearch(self.source, self.notes, self.outside) + return self._search + + def replace_cache(self, cache: dict[str, Any], added: list[str]) -> None: + """按需加入以后:换成带新 schema_cache 的源(不改调用方会话里的数据源对象),关系图、检索重建,加进来的不再算范围外。""" + base = self.source._base if isinstance(self.source, _WithCache) else self.source + self.source = _WithCache(base, cache) + keys = {name_key(n) for n in added} + self.outside = [o for o in self.outside if name_key(o.name) not in keys] + self._graph = self._search = self._outside_keys = None + + def outside_qualified(self, name: str) -> str: + schema = (getattr(self.source, "schema_cache", None) or {}).get("schema") + return f"{schema}.{name}" if schema else name + @property def raw_notes(self) -> dict[str, dict[str, Any]]: return {t: e.notes for t, e in self.entries.items()} @@ -110,6 +165,17 @@ class SourcePlan: return schema_view.qualifier(self.source)(key) +class _WithCache: + """数据源换上一份新的 schema_cache(按需加入以后):别的属性照原来的。不是 ORM 对象,改它不会被调用方的会话写回库。""" + + def __init__(self, base: Any, cache: dict[str, Any]) -> None: + self._base = base + self.schema_cache = cache + + def __getattr__(self, name: str) -> Any: + return getattr(self._base, name) + + @dataclass(frozen=True) class TableRef: """查看结果里的一张表:第几个候选库、schema_cache 的键、怎么来的(named 点名 / picked 按需求挑出)。""" @@ -153,14 +219,27 @@ class TableKind: kind = "table" label = "表" - def __init__(self, plans: list[SourcePlan], *, picker: Picker | None = None) -> None: + def __init__(self, plans: list[SourcePlan], *, picker: Picker | None = None, + trail: Mapping[str, Any] | None = None) -> None: self.plans = plans self.picker = picker + #: 按需加入的留痕:{actor, conversation_id}(开流前定下);这一次查看的需求原文、编号由 begin 记 + self.trail = dict(trail or {}) + self._need = "" + self._ref = "" + #: 这一次查看里已经按需加入了几张、加了哪些({第几个库: [表名]})、没加成的({(库, 点名): 说法}) + self._added_now = 0 + self._added: dict[int, list[str]] = {} + self._notes: list[str] = [] + self._prefiltered: dict[int, int] = {} + self._last_index: dict[tuple[int, ...], str] = {} @classmethod async def load(cls, sources: list[Any], *, graph: Mapping[str, Any] | None = None, - picker: Picker | None = None, catalog_view: Any = None) -> TableKind: - """读各源的数据目录。graph 是现有工作流(改图时的原图、追问时上一轮的图),概览里列出它查过的表。 + picker: Picker | None = None, catalog_view: Any = None, + trail: Mapping[str, Any] | None = None) -> TableKind: + """读各源的数据目录、剖析事实、探查范围外的对象清单。graph 是现有工作流(改图时的原图、追问时上一轮的图),概览里 + 列出它查过的表。trail 是按需加入的留痕(谁、哪一轮对话):不给时谁取这次请求的身份。 读哪一份目录由 catalog_view 定(data/catalog_view.py:当前目录、目录快照、关;只用已核实的项),不给是当前目录。 查看结果里的目录和运行时 db_schema 给模型看的、自查用的是同一份——评测的「目录对照」量的才是整份贡献。""" @@ -170,7 +249,21 @@ class TableKind: seeds = graph_tables(graph, sources) plans = [SourcePlan(source=s, entries=await read(view, s) if introspect.table_names(s) else {}, seeds=seeds.get(s.name, [])) for s in sources] - return cls(plans, picker=picker) + await _load_extras(plans) + trail = dict(trail or {}) + if "actor" not in trail: + from app.api.deps import current_principal + + trail["actor"] = current_principal().username + return cls(plans, picker=picker, trail=trail) + + def begin(self, *, need: str, ref: str) -> None: + """一次查看开始(LookService 调):按需加入的上限按这一次算,留痕记这次的需求和编号。""" + self._need, self._ref = need, ref + self._added_now = 0 + self._added = {} + self._notes = [] + self._prefiltered = {} # ---------------------------------------------------------------- 概览、目录 @@ -193,12 +286,21 @@ class TableKind: return list(dict.fromkeys(found)), [s for s in sources if s not in by_name] async def index(self, sources: list[str] | None) -> str: + """交回给模型点名的表目录。这一次查看预筛过的,交回预筛的那一份(几千张表的整份目录塞不进来);没预筛过、表又太多 + 的只列前 PREFILTER_LIMIT 张,写明另有几张。""" + candidates = self._candidates(sources)[0] + if (cached := self._last_index.get(tuple(candidates))) is not None: + return cached blocks = [] - for i in self._candidates(sources)[0]: + for i in candidates: p = self.plans[i] - head = (INDEX_PARTIAL_PROMPT if p.unexplored else INDEX_HEAD_PROMPT).format( - source=p.name, total=p.total, explored=p.explored) - blocks.append(f"{head}\n{schema_view.table_index(p.source, p.notes)}") + head = (INDEX_HEAD_PROMPT if not p.unexplored else INDEX_SCOPED_PROMPT if p.scoped + else INDEX_PARTIAL_PROMPT).format(source=p.name, total=p.total, explored=p.explored) + names = list(p.tables) + body = schema_view.table_index(p.source, p.notes, tables=names[:PREFILTER_LIMIT]) + if len(names) > PREFILTER_LIMIT: + body += "\n" + INDEX_MORE_PROMPT.format(n=len(names) - PREFILTER_LIMIT) + blocks.append(f"{head}\n{body}") return "\n\n".join(blocks) # ---------------------------------------------------------------- 点名、挑选 @@ -222,20 +324,93 @@ class TableKind: return TableRef(i, key) return None + def _find_outside(self, source: str | None, name: str) -> tuple[int, Any] | None: + """点名的表在哪个候选库的探查范围外(对象索引里有):(第几个库, 对象)。规则同 _find。""" + if source is not None: + candidates = [i for i, p in enumerate(self.plans) if p.name == source] + elif (hit := self._prefixed(name)) is not None: + candidates, name = [hit[0]], hit[1] + else: + candidates = [i for i, p in enumerate(self.plans) if p.tables or p.outside] + for i in candidates: + obj = self.plans[i].outside_of(name) + if obj is not None: + return i, obj + return None + + async def _add(self, wanted: dict[int, list[str]]) -> dict[tuple[int, str], TableRef]: + """按需加入(每次查看最多 ON_DEMAND_PER_LOOK 张,超了的照实说):{(第几个库, 对象名): 加进来以后的 TableRef}。读不到 + 结构、加不进去的记进这一次的说明。""" + from app.data import source_objects as so + + out: dict[tuple[int, str], TableRef] = {} + capped: list[str] = [] + for i, names in wanted.items(): + p = self.plans[i] + room = max(ON_DEMAND_PER_LOOK - self._added_now, 0) + take, rest = names[:room], names[room:] + capped += [f"{p.name} 的 {n}" for n in rest] + if not take: + continue + source_id = getattr(p.source, "id", None) + if not source_id: + continue + try: + got = await so.add_to_scope(source_id, take, trail=so.Trail( + via=so.VIA_LOOK, actor=self.trail.get("actor"), conversation_id=self.trail.get("conversation_id"), + ref=self._ref, need=self._need)) + except Exception as e: # noqa: BLE001 - 按需加入是辅助步骤,失败只是少一张表 + from app.core.errors import explain + + logger.exception("按需加入失败(数据源 %s)", p.name) + self._notes += [ADD_FAILED_PROMPT.format(table=n, why=explain(e)[0]) for n in take] + continue + self._added_now += len(got.added) + for name, why in {**got.failed, **got.refused}.items(): + self._notes.append(ADD_FAILED_PROMPT.format(table=name, why=why)) + if got.cache is not None and got.added: + p.replace_cache(got.cache, got.added) + self._added.setdefault(i, []).extend(got.added) + for name in got.added + got.already: + key = catalog.resolve_table_name(p.source.schema_cache, name) + if key: + out[(i, name_key(name))] = TableRef(i, key) + if capped: + self._notes.append(ADD_CAPPED_PROMPT.format(limit=ON_DEMAND_PER_LOOK, tables="、".join(capped))) + return out + async def resolve(self, targets: list[LookTarget]) -> tuple[list[TableRef], list[LookTarget]]: found: list[TableRef] = [] missing: list[LookTarget] = [] - single = len([p for p in self.plans if p.tables]) == 1 + single = len([p for p in self.plans if p.tables or p.outside]) == 1 + outside: list[tuple[LookTarget, int, Any]] = [] for t in targets: source = t.get("source") or None if source is not None and source not in {p.name for p in self.plans}: missing.append(t) continue - ref = self._find(None if single and source is None else source, str(t.get("name") or "")) + where = None if single and source is None else source + ref = self._find(where, str(t.get("name") or "")) if ref is None: - missing.append(t) + hit = self._find_outside(where, str(t.get("name") or "")) + if hit is not None: + outside.append((t, hit[0], hit[1])) + else: + missing.append(t) elif ref not in found: found.append(ref) + if outside: + wanted: dict[int, list[str]] = {} + for _t, i, obj in outside: + if obj.name not in wanted.setdefault(i, []): + wanted[i].append(obj.name) + added = await self._add(wanted) + for t, i, obj in outside: + ref = added.get((i, name_key(obj.name))) + if ref is None: + missing.append(t) + elif ref not in found: + found.append(ref) return found, missing async def pick(self, need: str, sources: list[str] | None) -> tuple[list[TableRef], str | None]: @@ -247,13 +422,16 @@ class TableKind: return [], ";".join(f"数据源「{p.name}」{introspect.why_empty(p.source.schema_cache or {})}" for p in bare) pool = [TableRef(i, key, "picked") for i in candidates for key in self.plans[i].tables] - if len(pool) <= PICK_LIMIT: + outside = sum(len(self.plans[i].outside) for i in candidates) + if len(pool) + outside <= PICK_LIMIT and not outside: return pool, None if self.picker is None: return [], "没有可用的挑选模型" - index = "\n\n".join( - f"{(INDEX_PARTIAL_PROMPT if p.unexplored else INDEX_HEAD_PROMPT).format(source=p.name, total=p.total, explored=p.explored)}\n" - f"{schema_view.table_index(p.source, p.notes)}" for p in (self.plans[i] for i in candidates)) + if len(pool) + outside > PREFILTER_LIMIT: + index = self._prefiltered_index(need, candidates) + else: + index = "\n\n".join(self._full_index(self.plans[i]) for i in candidates) + self._last_index[tuple(candidates)] = index used = ";".join(f"数据源「{p.name}」的 " + "、".join(p.qualified(t) for t in p.seeds) for p in (self.plans[i] for i in candidates) if p.seeds) try: @@ -263,22 +441,94 @@ class TableKind: return [], str(e) names = {self.plans[i].name for i in candidates} out: list[TableRef] = [] + wanted: dict[int, list[str]] = {} + order: list[tuple[str, Any]] = [] for source, name in picks: if len(candidates) == 1: # 只有一个库要挑时不看模型写的库名(写错了也不要紧) - ref = self._find(self.plans[candidates[0]].name, name) + where: str | None = self.plans[candidates[0]].name elif source in names or source is None: - ref = self._find(source, name) + where = source + elif self._prefixed(name): + where = None else: # 写了一个不在候选里的库名:表名前带着库名的照样认,否则丢掉 - ref = self._find(None, name) if self._prefixed(name) else None - if ref is not None and ref.plan in candidates and not any(r.plan == ref.plan and r.key == ref.key - for r in out): - out.append(TableRef(ref.plan, ref.key, "picked")) + continue + ref = self._find(where, name) + if ref is not None and ref.plan in candidates: + if not any(r.plan == ref.plan and r.key == ref.key for r in out): + out.append(TableRef(ref.plan, ref.key, "picked")) + order.append(("in", out[-1])) + continue + hit = self._find_outside(where, name) + if hit is not None and hit[0] in candidates and hit[1].name not in wanted.setdefault(hit[0], []): + wanted[hit[0]].append(hit[1].name) + order.append(("out", (hit[0], name_key(hit[1].name)))) + if any(wanted.values()): + added = await self._add({i: n for i, n in wanted.items() if n}) + ranked: list[TableRef] = [] + for kind, item in order: + ref = item if kind == "in" else added.get(item) + if ref is not None and not any(r.plan == ref.plan and r.key == ref.key for r in ranked): + ranked.append(TableRef(ref.plan, ref.key, "picked")) + out = ranked if picks and not out: return [], "挑出的表在表结构中都不存在" return out[:PICK_LIMIT], None + def _full_index(self, p: SourcePlan) -> str: + """整份表目录(候选不多时交给挑选模型的那一份):范围内的每张一行,范围外的另起一行、标出来。""" + head = (INDEX_PARTIAL_PROMPT if p.unexplored and not p.outside else INDEX_HEAD_PROMPT).format( + source=p.name, total=p.total, explored=p.explored) + lines = [schema_view.table_index(p.source, p.notes)] + lines += [self._outside_line(p, o) for o in p.outside] + return f"{head}\n" + "\n".join(x for x in lines if x) + + def _outside_line(self, p: SourcePlan, obj: Any) -> str: + comment = f"|{obj.comment}" if getattr(obj, "comment", None) else "" + return f"{p.outside_qualified(obj.name)}{comment}{OUTSIDE_MARK_PROMPT}" + + def _prefiltered_index(self, need: str, candidates: list[int]) -> str: + """几千张表的库:按需求检索一遍,挑选模型只看前 PREFILTER_LIMIT 个命中(各库合起来按得分排)、工作流已用到的表、 + 命中表的一跳关联表(最多 NEIGHBOR_LIMIT 张)。检索什么都没命中时退回各库表结构里的前几张。""" + hits: list[tuple[float, int, Any]] = [] + for i in candidates: + for hit in self.plans[i].search().search(need, limit=PREFILTER_LIMIT): + hits.append((hit.score, i, hit)) + hits.sort(key=lambda h: (-h[0], h[1], h[2].name)) + top = hits[:PREFILTER_LIMIT] + if not top: + per = max(PREFILTER_LIMIT // max(len(candidates), 1), 1) + top = [(0.0, i, schema_view.SearchHit(n, 0.0, True)) for i in candidates + for n in list(self.plans[i].tables)[:per]] + blocks = [] + for i in candidates: + p = self.plans[i] + mine = [h for _s, j, h in top if j == i] + inside = [h.name for h in mine if h.in_scope] + picked = list(dict.fromkeys([*p.seeds, *inside])) + neighbors: list[str] = [] + graph = p.graph() if inside else {} + for name in inside: + for edge in graph.get(name, ()): + if edge.to_table in p.tables and edge.to_table not in picked and edge.to_table not in neighbors: + neighbors.append(edge.to_table) + if len(neighbors) >= NEIGHBOR_LIMIT: + break + shown = picked + neighbors[:NEIGHBOR_LIMIT] + outside = [h.name for h in mine if not h.in_scope] + if not shown and not outside: + continue + self._prefiltered[i] = len(shown) + len(outside) + lines = [schema_view.table_index(p.source, p.notes, tables=shown)] if shown else [] + for name in outside: + obj = p.outside_of(name) + if obj is not None: + lines.append(self._outside_line(p, obj)) + head = PREFILTERED_PROMPT.format(source=p.name, total=p.total, n=len(shown) + len(outside)) + blocks.append(head + "\n" + "\n".join(x for x in lines if x)) + return "\n\n".join(blocks) + def key(self, obj: TableRef) -> str: return f"table:{self.plans[obj.plan].name}:{obj.key}" @@ -319,7 +569,7 @@ class TableKind: lines.append(SOURCE_HEAD_PROMPT.format(source=p.name, kind=p.source.kind, lead=",".join(lead))) notes = p.raw_notes for key in tables: - full = schema_view.table_detail(p.source, key, catalog_notes=notes) + full = schema_view.table_detail(p.source, key, catalog_notes=notes, profile=p.facts) block = full.split("\n") if used + len(full) > budget: block = [schema_view.table_brief(p.source, key)] @@ -330,8 +580,11 @@ class TableKind: lines += [f" {x}" for x in block[1:]] lines += schema_view.join_lines(p.source, p.graph(), tables, indent=" ") lines += schema_view.route_lines(p.source, routes, indent=" ") + if self._added.get(i): + lines.append(" " + ADDED_PROMPT.format(tables="、".join(p.qualified(t) for t in self._added[i]))) if p.unexplored: - lines.append(" " + UNEXPLORED_PROMPT.format(source=p.name, total=p.total, explored=p.explored)) + prompt = OUT_OF_SCOPE_PROMPT if p.scoped else UNEXPLORED_PROMPT + lines.append(" " + prompt.format(source=p.name, total=p.total, explored=p.explored)) blocks.append(self._record(p, named=named, picked=picked, joined=joined, missing=[str(t.get("name") or "") for t in missing if self._missing_in(t) == i])) @@ -339,6 +592,7 @@ class TableKind: lines.append(BRIEF_NOTE_PROMPT) if capped: lines.append(CAPPED_PROMPT.format(limit=CONTEXT_TABLE_LIMIT)) + lines += self._notes lines += self._missing_lines(missing, blocks, set(by_plan)) text = "\n".join(lines) if INFERRED_MARK in text: @@ -391,4 +645,30 @@ class TableKind: } if p.unexplored: out["explored"] = p.explored + index = next(i for i, q in enumerate(self.plans) if q is p) + if self._added.get(index): + out["added_to_scope"] = [p.qualified(t) for t in self._added[index]] + if index in self._prefiltered: + out["prefiltered"] = self._prefiltered[index] return out + + +async def _load_extras(plans: list[SourcePlan]) -> None: + """各源的剖析事实、探查范围外的对象清单:一个短会话读完。读不到就当没有(少一节剖析事实、少几个可按需加入的名字), + 不挡生成。上传表格、记录表没有对象索引;探查范围以内就是全部的源(没有范围外的对象)不读清单。""" + from app.data import catalog_profile_jobs, source_objects + from app.db.base import SessionLocal + + try: + async with SessionLocal() as session: + for p in plans: + source_id = getattr(p.source, "id", None) + if not source_id: + continue + if p.tables: + p.facts = await catalog_profile_jobs.last_facts(session, source_id) + scope = (getattr(p.source, "schema_cache", None) or {}).get("scope") + if isinstance(scope, dict) and scope.get("out_of_scope"): + p.outside = await source_objects.outside_objects(session, p.source) + except Exception: # noqa: BLE001 + logger.exception("读剖析事实、探查范围外的对象清单失败,这一轮查看不给这两样") diff --git a/backend/app/main.py b/backend/app/main.py index 88f6946d..bf12f703 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -108,6 +108,13 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await catalog_profile_jobs.startup() except Exception: # noqa: BLE001 logger.exception("数据剖析任务的启动整理失败,服务照常启动") + # 后台探查任务(next-round 03 A1)同一个规矩:没做完的标中断,读完的表物化进表结构,不自动续跑 + from app.data import introspect_jobs + + try: + await introspect_jobs.startup() + except Exception: # noqa: BLE001 + logger.exception("探查任务的启动整理失败,服务照常启动") await run_manager.setup() # 流程:存量的待审批补建待办(派给任何人,幂等);上次进程没了时已发出、没拿到结果的写入收成结果未知、安排核实 @@ -140,6 +147,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await scheduler.stop() await catalog_jobs.manager.shutdown() await catalog_profile_jobs.manager.shutdown() + await introspect_jobs.manager.shutdown() await run_manager.shutdown() from app.sandbox.manager import sandbox_manager diff --git a/backend/app/tools/datasource.py b/backend/app/tools/datasource.py index 255be841..ecfafc9f 100644 --- a/backend/app/tools/datasource.py +++ b/backend/app/tools/datasource.py @@ -77,7 +77,12 @@ class _SchemaArgs(BaseModel): # 实测一次 8 步的运行里 7 步花在这上面,最后只剩一步来真查数据(run dd9927e6) table: str | list[str] | None = Field( default=None, - description='表名。可以一次给多张:"user" 或 ["user","roles"];留空则列出所有表', + description='表名。可以一次给多张:"user" 或 ["user","roles"];留空则列出表(表多时只列前 50 张)', + ) + # 几千张表的库(next-round 03 A2):不知道表名时按需求的关键词查,和助手 look 用同一个检索 + keyword: str | None = Field( + default=None, + description="不知道表名时写要查的内容的关键词(如「退货 门店」),按表名、注释、数据目录检索相关的表", ) @@ -278,15 +283,56 @@ async def build_datasource_tools( tools.extend(_unavailable_tool(row, prefix, str(e)) for prefix in prefixes) continue entries = await _catalog_of(row, ctx.catalog_view) + facts, outside = await _extras_of(source) + if outside is not None: + # 有对象索引的源:按需加入以后两个工具都要看到新的表结构(查询时的 SQL 检查、冻结的表结构快照) + source = _LiveSource(source) for prefix in prefixes: if prefix == QUERY_PREFIX: tools.append(_make_query_tool(source, ctx, fixed=fixed, catalog=entries, force_readonly=force_readonly, - mask_results=mask_results)) + mask_results=mask_results, outside=outside)) else: - tools.append(_make_schema_tool(source, fixed=fixed, catalog=entries)) + tools.append(_make_schema_tool(source, fixed=fixed, catalog=entries, facts=facts, ctx=ctx, + outside=outside)) return tools +class _LiveSource: + """建工具时的数据源,schema_cache 可以换(运行时按需加入探查范围以后):查询工具、表结构工具拿的是同一个对象,加进来的表 + 两边都看得到。别的属性照原来的。不是 ORM 对象,换它不会写回库(库里由 source_objects.rebuild_schema_cache 写)。""" + + def __init__(self, base: Any) -> None: + self._base = base + self.schema_cache = getattr(base, "schema_cache", None) or {} + + def __getattr__(self, name: str) -> Any: + return getattr(self._base, name) + + +async def _extras_of(source: Any) -> tuple[dict[str, dict[str, Any]], frozenset[str] | None]: + """建工具时读一次(和目录同一时机):这个源的剖析事实(db_schema 的表详情附一节,next-round 遗留 e)、对象索引里探查 + 范围外的对象(name_key;SQL 检查照实列「未按目录检查的表」,点名时按需加入)。没有对象索引的源(上传表格、升级前探查的) + 范围外是 None。读不到就当没有:少一节说明,不挡工具。""" + from app.data import catalog_profile_jobs, source_objects + from app.db.base import SessionLocal + + facts: dict[str, dict[str, Any]] = {} + outside: frozenset[str] | None = None + source_id = getattr(source, "id", None) + if not source_id: + return facts, outside + try: + async with SessionLocal() as session: + if (getattr(source, "schema_cache", None) or {}).get("tables"): + facts = await catalog_profile_jobs.last_facts(session, source_id) + if isinstance((getattr(source, "schema_cache", None) or {}).get("scope"), dict) \ + and (getattr(source, "origin", None) or "manual") == "manual": + outside = await source_objects.outside_names(session, source) + except Exception: # noqa: BLE001 + logger.exception("读数据源 %s 的剖析事实、对象索引失败,这次不给", getattr(source, "name", "?")) + return facts, outside + + async def _catalog_of(row: Any, view: Any = None) -> dict[str, data_catalog.CatalogEntry]: """这个源的数据目录,建工具时读一次:db_schema 给模型看的和查询时冻结进表结构快照的是同一版。 @@ -552,7 +598,8 @@ def _unavailable_tool(row: DataSource, prefix: str, reason: str) -> StructuredTo def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, catalog: Mapping[str, data_catalog.CatalogEntry] | None = None, - force_readonly: bool = False, mask_results: bool = False) -> StructuredTool: + force_readonly: bool = False, mask_results: bool = False, + outside: frozenset[str] | None = None) -> StructuredTool: """source 是手工源的 DataSource,或上传源绑定快照的 SourceView(字段同名)。 fixed:上传源的版本是这次运行固定的(找不到时的说法不同)。 @@ -565,19 +612,22 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, 那里已经有的值(这个 agent 前面的查询遮掉的、调用方交进来的)这一次也按值换,数据源没设遮罩也一样:先查号码、 再查写着号码的备注或别的库,后一份同样遮好。按值换只换文字格,数值格照原样(存下的是证据)。查询失败时报错原话里 的值也换(masking.mask_error_text)。 + outside:对象索引里探查范围外的对象(name_key)。查询用到范围外、表结构里没有的表时,结果里照实列出 + unchecked_tables(「未按目录检查的表」):SQL 检查只按范围内的表结构和目录查,不新增检查规则。 """ # 数据源在 options.query_timeout_s 里配了就用它,否则取缺省。数据库按它停下语句, # 引擎按它告诉界面上限是多少(metadata["timeout_s"]) seconds = query_timeout(source) # 基于数据目录的 SQL 检查器:和冻结进表结构快照的是同一版目录。第一次查询时才建(关系图也是用到才建) - checker: list[SqlChecker] = [] + checker: list[tuple[int, SqlChecker]] = [] def _checker() -> SqlChecker: - if not checker: - checker.append(SqlChecker(kind=source.kind, schema_cache=source.schema_cache, - notes={name: entry.notes for name, entry in (catalog or {}).items()}, - source_name=source.name)) - return checker[0] + # 表结构换过(按需加入了表)就重建:检查按的是此刻的表结构 + if not checker or checker[0][0] != id(source.schema_cache): + checker[:] = [(id(source.schema_cache), SqlChecker( + kind=source.kind, schema_cache=source.schema_cache, + notes={name: entry.notes for name, entry in (catalog or {}).items()}, source_name=source.name))] + return checker[0][1] async def _run(sql: str, limit: int | None = None) -> str: if gone := _vanished(source, fixed): @@ -646,6 +696,8 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, # 快照和以前一字不差 if checks := await _sql_checks(_checker(), sql): payload["checks"] = checks + if outside and (unchecked := _unchecked_tables(sql, source, outside)): + payload["unchecked_tables"] = unchecked # 表结构快照:这次查询时数据源的结构冻结下来,报告里写的表名、字段名按它核对。事后数据源 # 重新探查、改了结构,已经跑完的运行核对的还是当时那一份。上传源冻结的是绑定快照里那份 if schema := await _store_schema(source, ctx, catalog=catalog): @@ -709,6 +761,21 @@ def _make_query_tool(source: Any, ctx: ToolContext, *, fixed: bool = False, ) +#: 查询结果里「未按目录检查的表」那一项交给模型时的说明(键名 unchecked_tables 本身就是给它看的) +def _unchecked_tables(sql: str, source: Any, outside: frozenset[str]) -> list[str]: + """SQL 里用到的、在对象索引里探查范围外、表结构里没有的表(照 SQL 里的写法)。认不出表就当没有。""" + from app.data.names import name_key + from app.engine.evidence import sql_tables + + try: + names = sql_tables(sql) + except Exception: # noqa: BLE001 + return [] + cache = getattr(source, "schema_cache", None) + return [t for t in dict.fromkeys(names) + if name_key(str(t).rsplit(".", 1)[-1]) in outside and data_catalog.resolve_table_name(cache, str(t)) is None] + + async def _sql_checks(checker: SqlChecker, sql: str) -> list[dict[str, Any]]: """一条查询的 SQL 检查结果(as_dict 的形状)。在线程里跑、限时 SQL_CHECK_TIMEOUT_S:解析是纯计算,放在事件循环里 会挡住同一进程里别的运行。出错、超时都返回空列表——检查不能让一次成功的查询变成失败。""" @@ -784,38 +851,73 @@ def _schema_tables(table: str | list[str] | None) -> list[str]: return [t.strip() for one in names for t in one.split(",") if t.strip()] +#: db_schema 不传 table 时最多列几张(几千张表的库整份列出来没法看,也塞不进上下文)、按 keyword 检索给几个 +SCHEMA_LIST_LIMIT = 50 +SCHEMA_KEYWORD_LIMIT = 30 +SCHEMA_MORE_PROMPT = "…另有 {n} 张没有列出:用 keyword 按要查的内容检索,或者 table 直接点名" +SCHEMA_KEYWORD_HEAD = "数据源「{source}」里和「{keyword}」相关的表(按相关程度排):" +SCHEMA_KEYWORD_NONE = "数据源「{source}」里没有检索到和「{keyword}」相关的表:换个说法,或者 table 直接点名" +SCHEMA_OUTSIDE_MARK = "(探查范围外:table 点名它会按需加入)" +SCHEMA_ADDED_PROMPT = "({table} 在探查范围外,这次按需加入了探查范围、当场读了结构)" +SCHEMA_RUN_CAPPED = "这次运行最多按需加入 {limit} 张探查范围外的表,已经用完" + + def _make_schema_tool(source: Any, *, fixed: bool = False, - catalog: Mapping[str, data_catalog.CatalogEntry] | None = None) -> StructuredTool: + catalog: Mapping[str, data_catalog.CatalogEntry] | None = None, + facts: Mapping[str, Mapping[str, Any]] | None = None, ctx: ToolContext | None = None, + outside: frozenset[str] | None = None) -> StructuredTool: """source 同 _make_query_tool:上传源的表结构取绑定快照里冻结的那份。 - catalog:建工具时读到的数据目录,查单表时附在字段清单后面。单表的说法和助手查看、/schema 接口是同一份 - (data/schema_view.table_detail):遮罩列只给列名和类型,它的注释和目录项都不给(look 文档 D6)。 + catalog:建工具时读到的数据目录,查单表时附在字段清单后面;facts:剖析事实,单表详情里附一节。单表的说法和助手查看、 + /schema 接口是同一份(data/schema_view.table_detail):遮罩列只给列名和类型,它的注释和目录项都不给(look 文档 D6)。 + + 几千张表的库(next-round 03 A2):不传 table 时只列范围内前 SCHEMA_LIST_LIMIT 张、提示用 keyword 或 table;keyword 和 + 助手 look 用同一个检索(schema_view.search_tables)。点名探查范围外的表(outside 里有)当场只读反射、加进范围(每次运行 + 最多 ON_DEMAND_PER_RUN 张,留痕记这次运行和节点);表已不在、读不到结构、范围外没加进来的分开说。 """ notes = {name: entry.notes for name, entry in (catalog or {}).items()} - async def _run(table: str | list[str] | None = None) -> str: + async def _run(table: str | list[str] | None = None, keyword: str | None = None) -> str: if gone := _vanished(source, fixed): return f"{DATA_UNAVAILABLE}{gone}" wanted = _schema_tables(table) if wanted: - return "\n\n".join(schema_view.table_detail(source, t, catalog_notes=notes) for t in wanted) + added_lines: list[str] = [] + missing = [t for t in wanted if introspect.find_table(source, t) is None] + reasons: dict[str, str] = {} + if missing and outside is not None: + reasons, added_lines = await _on_demand(source, missing, ctx) + parts = [] + for t in wanted: + if introspect.find_table(source, t) is not None or t not in reasons: + parts.append(schema_view.table_detail(source, t, catalog_notes=notes, profile=facts)) + else: + parts.append(reasons[t]) + return "\n\n".join(parts + added_lines) + if keyword and keyword.strip(): + return await _keyword_list(source, notes, keyword.strip(), outside) tables = introspect.table_names(source) if not tables: # 把实情交出去,而不是让它去点一个它点不到的按钮 return f"数据源「{source.name}」的结构信息不可用。" + _no_schema_note(source) raw = set(unshaped_tables(source)) totals = set(reported_total_tables(source)) - return f"数据源「{source.name}」共 {len(tables)} 张表:\n" + "\n".join( - f" {n}" + ("(按原样导入、未经规整,不能直接对列求和)" if n in raw else "") - + (REPORTED_TOTAL_MARK if n in totals else "") for n in tables - ) + shown = tables[:SCHEMA_LIST_LIMIT] + total = max(len(tables), int((getattr(source, "schema_cache", None) or {}).get("total") or 0)) + head = f"数据源「{source.name}」共 {total} 张表" + (f",探查范围内 {len(tables)} 张" if total > len(tables) else "") + lines = [f" {n}" + ("(按原样导入、未经规整,不能直接对列求和)" if n in raw else "") + + (REPORTED_TOTAL_MARK if n in totals else "") for n in shown] + if total > len(shown): + lines.append(" " + SCHEMA_MORE_PROMPT.format(n=total - len(shown))) + return head + ":\n" + "\n".join(lines) return StructuredTool( name=f"{SCHEMA_PREFIX}{source.name}", description=( f"查看数据源「{source.name}」的表结构。" - "写 SQL 前先用它确认字段名。不传 table 则列出所有表;" - "**要看多张表就一次全传进来**(table 可以是数组),不要一张一张问。" + "写 SQL 前先用它确认字段名。不传 table 则列出表(表多时只列前 50 张);" + "**要看多张表就一次全传进来**(table 可以是数组),不要一张一张问;" + "不知道表名时用 keyword 按要查的内容检索。" ), args_schema=_SchemaArgs, coroutine=_run, @@ -823,6 +925,86 @@ def _make_schema_tool(source: Any, *, fixed: bool = False, ) +async def _keyword_list(source: Any, notes: Mapping[str, Mapping[str, Any]], keyword: str, + outside: frozenset[str] | None) -> str: + """按 keyword 检索(范围内的表、对象索引里范围外的对象),每个命中一行:表名|中文名(范围外的是注释、标出来)。""" + objects: list[Any] = [] + if outside: + from app.data import source_objects + from app.db.base import SessionLocal + + try: + async with SessionLocal() as session: + objects = await source_objects.outside_objects(session, source) + except Exception: # noqa: BLE001 - 读不到范围外的清单就只检索范围内的 + objects = [] + hits = schema_view.search_tables(source, notes, keyword, outside=objects, limit=SCHEMA_KEYWORD_LIMIT) + if not hits: + return SCHEMA_KEYWORD_NONE.format(source=source.name, keyword=keyword) + by_name = {o.name: o for o in objects} + tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} + schema = (getattr(source, "schema_cache", None) or {}).get("schema") + lines = [SCHEMA_KEYWORD_HEAD.format(source=source.name, keyword=keyword)] + for hit in hits: + if hit.in_scope: + lines.append(" " + data_catalog.render_table_index(getattr(source, "schema_cache", None), + {k: data_catalog.visible_notes(v) + for k, v in notes.items()}, tables=[hit.name]) + if hit.name in tables else f" {hit.name}") + else: + obj = by_name.get(hit.name) + comment = f"|{obj.comment}" if obj is not None and obj.comment else "" + lines.append(f" {f'{schema}.{hit.name}' if schema else hit.name}{comment}{SCHEMA_OUTSIDE_MARK}") + return "\n".join(lines) + + +async def _on_demand(source: Any, missing: list[str], ctx: ToolContext | None) -> tuple[dict[str, str], list[str]]: + """db_schema 点名了表结构里没有的表:探查范围外、对象索引里有的当场加进范围(每次运行最多 ON_DEMAND_PER_RUN 张), + 别的说清为什么不在。返回 ({点名: 交回的那句话},加进来的提示行);加进来的表不在前一个字典里(调用方照常给详情)。""" + from app.data import source_objects as so + from app.db.base import SessionLocal + from app.look.base import ON_DEMAND_PER_LOOK, ON_DEMAND_PER_RUN + + out: dict[str, str] = {} + lines: list[str] = [] + try: + async with SessionLocal() as session: + reasons = await so.absent_reasons(session, source, missing) + used = await so.count_added_in_run(session, ctx.run_id) if ctx is not None and ctx.run_id else 0 + except Exception: # noqa: BLE001 - 查不了原因就照旧说 + logger.exception("db_schema 查表不在的原因失败(数据源 %s)", getattr(source, "name", "?")) + return out, lines + limit = ON_DEMAND_PER_RUN if ctx is not None and ctx.run_id else ON_DEMAND_PER_LOOK + room = max(limit - used, 0) + candidates = [t for t in missing if reasons.get(t) == so.ABSENT_OUT_OF_SCOPE] + take = candidates[:room] + for t in missing: + if t in take: + continue + why = SCHEMA_RUN_CAPPED.format(limit=limit) if t in candidates else "" + text = schema_view.absent_detail(source, t, reasons.get(t), why=why, + error=((getattr(source, "schema_cache", None) or {}).get("skipped") or {}) + .get(t)) + if text is not None: + out[t] = text + if take: + got = await so.add_to_scope(getattr(source, "id", ""), take, trail=so.Trail( + via=so.VIA_SCHEMA_TOOL, run_id=(ctx.run_id or None) if ctx else None, + ref=(ctx.node_id or None) if ctx else None, + need=f"运行时 {SCHEMA_PREFIX}{getattr(source, 'name', '')} 点名:{'、'.join(take)}")) + if got.cache is not None and got.added: + source.schema_cache = got.cache + for t in take: + key = data_catalog.resolve_table_name(getattr(source, "schema_cache", None), t) + if key is not None: + lines.append(SCHEMA_ADDED_PROMPT.format(table=key)) + continue + why = got.failed.get(t) or got.refused.get(t) or next( + (v for k, v in {**got.failed, **got.refused}.items() if k.lower() == t.lower()), "") + out[t] = schema_view.absent_detail(source, t, so.ABSENT_OUT_OF_SCOPE, why=why) or why + return out, lines + + def is_dangerous_call(source: Any, sql: str) -> bool: """这次调用要不要走人工审批:非只读源上的写操作要。 diff --git a/backend/app/tools/registry.py b/backend/app/tools/registry.py index c80f0ed4..bf98754c 100644 --- a/backend/app/tools/registry.py +++ b/backend/app/tools/registry.py @@ -363,6 +363,13 @@ def prepare_args( # 少了哪个必填的,就是多出来那个想写的;一个必填都不缺时,退而看还空着哪个可选字段 missing = [k for k in required if k not in args] or [k for k in props if k not in args] + if len(unknown) == 1 and len(missing) > 1: + # 空着的可选字段不止一个(db_schema 有 table、keyword 两个):只有拼写明显接近的那一个才算唯一候选 + import difflib + + close = [k for k in missing if difflib.SequenceMatcher(None, unknown[0].lower(), k.lower()).ratio() >= 0.8] + if len(close) == 1: + missing = close residual: ValidationError | None = None if len(unknown) == 1 and len(missing) == 1: fixed = dict(args) diff --git a/backend/tests/fixtures/catalog/large.py b/backend/tests/fixtures/catalog/large.py new file mode 100644 index 00000000..5a1ed4e7 --- /dev/null +++ b/backend/tests/fixtures/catalog/large.py @@ -0,0 +1,91 @@ +"""虚构的大库:3,000 张表(next-round 03 的大库验收夹具)。名字、注释、需求一律虚构。 + +- 2,990 张「同一种写法」的表:20 个业务域 × 15 种表 × 10 个分区(华东、华南……),表名 {域}_{表}_{分区},注释 + 「{域中文}{表中文}({分区中文})」。它们彼此只差几个字,正好是挑表时最容易混淆的那一类。 +- 10 张目标表(TARGETS):名字和注释各不相同,每张配一句脚本化的需求。前 5 张给数据目录(中文名),有目录的表一律在探查 + 范围里(pinned);后 5 张在探查范围外,只有对象索引里的注释——SQLite 没有表注释,测试里直接写进对象索引的 comment, + 模拟 MySQL 的 TABLE_COMMENT、PostgreSQL 的 obj_description。 + +每张表 5 列(id 主键、业务日期、金额、状态、所属域的编号),没有数据:大库的验收看的是对象索引、范围、挑表,不查数。 +""" +from __future__ import annotations + +import sqlite3 +from dataclasses import dataclass +from pathlib import Path + +DOMAINS = [ + ("ord", "订单"), ("inv", "库存"), ("pur", "采购"), ("sal", "销售"), ("fin", "财务"), ("hrm", "人事"), + ("crm", "客户"), ("lgs", "物流"), ("prd", "商品"), ("mkt", "营销"), ("svc", "客服"), ("whs", "仓储"), + ("pay", "支付"), ("rtn", "退货"), ("sto", "门店"), ("emp", "员工"), ("sup", "供应商"), ("prm", "促销"), + ("mbr", "会员"), ("aud", "审计"), +] +ENTITIES = [ + ("hdr", "表头"), ("line", "明细"), ("log", "日志"), ("snap", "快照"), ("daily", "日汇总"), ("monthly", "月汇总"), + ("cfg", "配置"), ("map", "映射"), ("hist", "历史"), ("tmp", "临时表"), ("bak", "备份"), ("stat", "统计"), + ("dim", "维度"), ("fact", "事实"), ("queue", "队列"), +] +REGIONS = [ + ("east", "华东"), ("south", "华南"), ("north", "华北"), ("west", "西部"), ("central", "华中"), + ("ne", "东北"), ("arch", "历史归档"), ("test", "测试"), ("y2019", "2019年"), ("y2020", "2020年"), +] + + +@dataclass(frozen=True) +class Target: + name: str + comment: str + need: str + #: 有数据目录(中文名)、一律在探查范围里 + cataloged: bool + + +TARGETS = [ + Target("member_points_ledger", "会员积分流水", "会员积分每个月发放了多少、消耗了多少", True), + Target("store_traffic_counter", "门店每日客流计数", "各门店上周每天的客流量", True), + Target("refund_reason_dict", "退货原因字典", "按退货原因统计退货单数", True), + Target("coupon_redemption", "优惠券核销记录", "优惠券的核销率", True), + Target("supplier_delivery_sla", "供应商到货时效", "供应商的到货准时率", True), + Target("warehouse_turnover_snap", "仓库库存周转快照", "各仓库的库存周转天数", False), + Target("shift_roster", "员工排班表", "下周员工的排班情况", False), + Target("complaint_ticket", "客户投诉工单", "客户投诉工单的平均处理时长", False), + Target("online_payment_flow", "线上支付流水", "线上支付的成功率", False), + Target("waybill_track", "物流运单签收轨迹", "物流运单从发货到签收用了多久", False), +] + +TOTAL = len(DOMAINS) * len(ENTITIES) * len(REGIONS) - len(TARGETS) + len(TARGETS) + + +def generic_names() -> list[tuple[str, str]]: + """2,990 张同一种写法的表 (表名, 注释):按域、表、分区的顺序,最后 10 个组合让给目标表。""" + out: list[tuple[str, str]] = [] + for code, cn in DOMAINS: + for ent, ent_cn in ENTITIES: + for reg, reg_cn in REGIONS: + out.append((f"{code}_{ent}_{reg}", f"{cn}{ent_cn}({reg_cn})")) + return out[: len(out) - len(TARGETS)] + + +def comments() -> dict[str, str]: + """全部 3,000 张表的注释(写进对象索引,模拟有表注释的数据库)。""" + out = dict(generic_names()) + out.update({t.name: t.comment for t in TARGETS}) + return out + + +def build(path: Path | str) -> Path: + path = Path(path) + db = sqlite3.connect(path) + for name, _comment in generic_names(): + code = name.split("_", 1)[0] + db.execute(f"CREATE TABLE {name} (id INTEGER PRIMARY KEY, biz_date TEXT, amount REAL, status TEXT, " + f"{code}_id INTEGER)") + for t in TARGETS: + db.execute(f"CREATE TABLE {t.name} (id INTEGER PRIMARY KEY, biz_date TEXT, amount REAL, status TEXT, " + "ref_id INTEGER)") + db.commit() + db.close() + return path + + +__all__ = ["DOMAINS", "ENTITIES", "REGIONS", "TARGETS", "TOTAL", "Target", "build", "comments", "generic_names"] diff --git a/backend/tests/fixtures/sources.py b/backend/tests/fixtures/sources.py index 7b6f6fb3..81d9b6da 100644 --- a/backend/tests/fixtures/sources.py +++ b/backend/tests/fixtures/sources.py @@ -24,3 +24,21 @@ async def drop_source(source_id: str | None = None, *, name: str | None = None) await session.execute(delete(DataSource).where(DataSource.id == source_id)) await session.commit() await engines.invalidate(source_id) + + +async def introspect_now(client, source_id: str, *, expect: int = 202) -> dict: + """探查结构并等它做完(探查是后台任务:POST /introspect 回 202 和任务,next-round 03 A1),返回数据源此刻的样子 + (DataSourceOut)。expect 不是 202 时只发请求、断言状态码,返回响应体。""" + from app.data import introspect_jobs + + r = await client.post(f"/api/datasources/{source_id}/introspect") + assert r.status_code == expect, r.text + if expect != 202: + return r.json() + job_id = r.json()["job"]["id"] + await introspect_jobs.manager.wait(job_id, timeout=120) + r = await client.get(f"/api/datasources/{source_id}/introspect-jobs/{job_id}") + assert r.status_code == 200, r.text + body = r.json() + assert body["job"]["status"] in ("succeeded", "partial"), body["job"] + return body["source"] diff --git a/backend/tests/test_catalog_api.py b/backend/tests/test_catalog_api.py index 1e13e8a3..01a89cfa 100644 --- a/backend/tests/test_catalog_api.py +++ b/backend/tests/test_catalog_api.py @@ -16,7 +16,7 @@ from app.db.models import DataSource from app.main import app as _fastapi_app from tests.fixtures.catalog import scenic from tests.fixtures.catalog.fake_model import FakeModel -from tests.fixtures.sources import drop_source +from tests.fixtures.sources import drop_source, introspect_now ACTOR = {"X-Actor": quote("王敏")} @@ -35,8 +35,7 @@ async def source_id(client, scenic_db): session.add(row) await session.commit() sid = row.id - r = await client.post(f"/api/datasources/{sid}/introspect") - assert r.status_code == 200, r.text + await introspect_now(client, sid) yield sid await drop_source(sid) diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py index c5d0ede4..37aa1087 100644 --- a/backend/tests/test_catalog_assistant.py +++ b/backend/tests/test_catalog_assistant.py @@ -34,7 +34,7 @@ from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION from app.db.base import SessionLocal from app.db.models import CatalogDraftJob, CatalogDraftTask, DataSource from app.main import app as _fastapi_app -from tests.fixtures.sources import drop_source +from tests.fixtures.sources import drop_source, introspect_now PHONES = [f"139000000{i:02d}" for i in range(1, 31)] @@ -154,8 +154,7 @@ async def _source(client, tmp_path, *, profile: bool = True, guidance: str | Non await session.commit() sid = row.id _CREATED.append(sid) - r = await client.post(f"/api/datasources/{sid}/introspect") - assert r.status_code == 200, r.text + await introspect_now(client, sid) async with SessionLocal() as session: await catalog.write_entry(session, sid, "orders", notes if notes is not None else ORDERS_NOTES, if_version=0, actor=None) diff --git a/backend/tests/test_catalog_profile.py b/backend/tests/test_catalog_profile.py index 0a1ef009..0a69de0a 100644 --- a/backend/tests/test_catalog_profile.py +++ b/backend/tests/test_catalog_profile.py @@ -30,7 +30,7 @@ from app.db.base import SessionLocal from app.db.models import DataSource from app.main import app as _fastapi_app from tests.fixtures.catalog import scenic -from tests.fixtures.sources import drop_source +from tests.fixtures.sources import drop_source, introspect_now ACTOR = {"X-Actor": quote("周宁")} @@ -58,8 +58,7 @@ async def make_source(client): await session.commit() sid = row.id made.append(sid) - r = await client.post(f"/api/datasources/{sid}/introspect") - assert r.status_code == 200, r.text + await introspect_now(client, sid) if draft is not None: r = await client.post(f"/api/datasources/{sid}/catalog/draft", json={"tables": draft}) assert r.status_code == 200, r.text @@ -562,8 +561,7 @@ async def test_blocked_profile_says_which_case_by_code(client, make_source, scen r = await _start(client, sid, ["visits"]) assert r.status_code == 409 and r.json()["code"] == "schema_missing" and "探查结构" in r.json()["detail"] - r = await client.post(f"/api/datasources/{sid}/introspect") - assert r.status_code == 200, r.text + await introspect_now(client, sid) async def tampered(*_args, **_kwargs): raise table_versions.SnapshotManifestMismatch("数据版本的哈希登记与快照清单不一致,已拒绝查询") diff --git a/backend/tests/test_catalog_schema_truncated.py b/backend/tests/test_catalog_schema_truncated.py index 7a110c4b..f0c99435 100644 --- a/backend/tests/test_catalog_schema_truncated.py +++ b/backend/tests/test_catalog_schema_truncated.py @@ -1,16 +1,15 @@ -"""没探查到的表要说出来(数据目录完整性审查 F3 / B4)。 +"""探查范围外的表要说出来(数据目录完整性审查 F3 / B4;next-round 03 A1 之后按探查范围说)。 -探查结构每个数据源最多取 200 张表(introspect._MAX_TABLES)。多出来的表不在表结构缓存里:目录页列不出,助手也 -看不到字段。以前这件事哪儿都没说——目录清单只给探查到的 200 行,助手上下文里的「共几张表」是探查到的张数, -提示词里也不提。这组守的是: +以前探查结构每个数据源最多取 200 张表,多出来的不在表结构缓存里,哪儿都没说。现在全库对象进对象索引,探查范围(规则) +圈出要读结构的那部分;范围外的表在对象索引里有名字。这组守的是: -- 目录清单接口返回 schema_truncated、schema_total(数据库里一共几张),界面据此说明「只探查了前 200 张」 -- 助手的概览写「共 205 张表,只探查了 200 张」;查看记录(look 事件)的 total 是表结构缓存里的真实总数,另带 - explored(探查到几张) -- 查看结果里写明还有几张没探查到,遇到不认识的表名先查结构;交回的表目录也写明只列了探查到的 -- 没截断的库照旧,不多说一句 +- 目录清单接口返回 schema_truncated(有范围外的表)、schema_total(对象索引里一共几张),界面据此说明 +- 助手的概览写「共 205 张表,探查范围内 200 张」;查看记录(look 事件)的 total 是对象总数,另带 explored(范围内几张) +- 查看结果里写明范围外的点名查看会按需加入;交回的表目录也写明只列了范围内的 +- 运行时 db_schema 点名范围外的表:按需加入,给出字段 +- 没有范围外的库照旧,不多说一句;升级前只探查了一部分的老缓存照旧说「只探查了」 -合成库:205 张空表,测试里临时建,用完连同数据源一起删掉。 +合成库:205 张空表,探查范围规则只收 t0*、t1*(200 张),测试里临时建,用完连同数据源一起删掉。 """ from __future__ import annotations @@ -23,7 +22,7 @@ from httpx import ASGITransport, AsyncClient from app.db.base import SessionLocal from app.db.models import DataSource from app.main import app -from tests.fixtures.sources import drop_source +from tests.fixtures.sources import drop_source, introspect_now WIDE = 205 @@ -43,20 +42,23 @@ async def client(): yield c -async def _source(client, path: str) -> tuple[str, str]: +async def _source(client, path: str, *, include: list[str] | None = None) -> tuple[str, str]: name = f"wide_{uuid.uuid4().hex[:8]}" r = await client.post("/api/datasources", json={"name": name, "kind": "sqlite", "database": path, "readonly": True}) assert r.status_code in (200, 201), r.text sid = r.json()["id"] - r = await client.post(f"/api/datasources/{sid}/introspect") - assert r.status_code == 200, r.text + if include: + r = await client.put(f"/api/datasources/{sid}/introspect-scope", + json={"scope": {"mode": "rules", "include": include}}) + assert r.status_code == 200, r.text + await introspect_now(client, sid) return sid, name @pytest.fixture async def wide(client, tmp_path): - sid, name = await _source(client, _build(tmp_path / "wide.db", WIDE)) + sid, name = await _source(client, _build(tmp_path / "wide.db", WIDE), include=["t0*", "t1*"]) yield sid, name await drop_source(sid) @@ -97,17 +99,17 @@ async def test_assistant_context_counts_the_real_total(wide): kind = await TableKind.load([await _row(sid)]) looks = LookService([kind]) overview = await looks.overview() - assert f"共 {WIDE} 张表,只探查了 200 张" in overview - # 点名查看一张探查到的表:记录里 total 是真实总数,另带 explored;文字里说清还有多少张没探查到 + assert f"共 {WIDE} 张表,探查范围内 200 张" in overview + # 点名查看一张范围内的表:记录里 total 是对象总数,另带 explored;文字里说清范围外的点名会按需加入 result = await looks.look({"op": "look", "need": "看一下 t000", "targets": [ {"kind": "table", "source": name, "name": "t000"}]}) (row,) = result.record["objects"] - assert row["total"] == WIDE and row["explored"] == 200 - assert f"共 {WIDE} 张表,只探查了 200 张" in result.text - # 遇到不认识的表名先查结构,别猜 - assert f"db_schema__{name}" in result.text and "information_schema" in result.text - # 交回的表目录也写明只列了探查到的 - assert f"共 {WIDE} 张表,只探查了 200 张,下面只列探查到的" in await kind.index(None) + assert row["total"] == WIDE and row["explored"] == 200 and "added_to_scope" not in row + assert f"共 {WIDE} 张表,探查范围内 200 张" in result.text + # 遇到不认识的表名点名 look 或按 keyword 查,别猜 + assert f"db_schema__{name}" in result.text and "keyword" in result.text and "按需加入" in result.text + # 交回的表目录也写明只列了范围内的 + assert f"共 {WIDE} 张表,探查范围内 200 张,下面只列范围内的" in await kind.index(None) async def test_assistant_context_of_a_complete_schema_says_nothing_extra(narrow): @@ -123,18 +125,35 @@ async def test_assistant_context_of_a_complete_schema_says_nothing_extra(narrow) assert "没有探查到" not in result.text and "只探查了" not in result.text -async def test_schema_tool_does_not_deny_a_table_that_was_not_introspected(wide, narrow): - """只探查了一部分表时,查一张没探查到的表不能说「没有」:它可能就在没探查到的那部分里。 - 以前回「数据源中没有 X」,模型据此改用别的表,或者编一个答案。""" - from app.data.introspect import describe_table +async def test_schema_tool_adds_an_out_of_scope_table_instead_of_denying_it(wide, narrow): + """点名一张探查范围外的表不能说「没有」:它在对象索引里,当场读结构、加进范围(每次运行有上限)。以前回「数据源中 + 没有 X」,模型据此改用别的表,或者编一个答案。""" + from app.tools.datasource import build_datasource_tools + from app.tools.registry import ToolContext, Versions - row = await _row(wide[0]) - explored = set((row.schema_cache or {}).get("tables") or {}) - missing = next(f"t{i:03d}" for i in range(WIDE) if f"t{i:03d}" not in explored) - text = describe_table(row, missing) - assert "中没有" not in text - assert "只探查了" in text and "information_schema" in text + sid, name = wide + async with SessionLocal() as session: + (tool,) = await build_datasource_tools([f"db_schema__{name}"], ToolContext(data_versions=Versions.CURRENT), + session) + text = await tool.coroutine(table="t203") + assert "amount" in text and "按需加入了探查范围" in text + row = await _row(sid) + assert "t203" in row.schema_cache["tables"] # 探查完整的库:查不存在的表照旧说没有 - small = await _row(narrow[0]) - assert "中没有 nope" in describe_table(small, "nope") + async with SessionLocal() as session: + (tool,) = await build_datasource_tools([f"db_schema__{(await _row(narrow[0])).name}"], + ToolContext(data_versions=Versions.CURRENT), session) + assert "没有 nope" in await tool.coroutine(table="nope") + + +def test_legacy_truncated_cache_still_does_not_deny(): + """升级前只探查了前 200 张的老缓存(没有对象索引):查一张没探查到的表照旧说「只探查了」、请查数据字典。""" + from types import SimpleNamespace + + from app.data.introspect import describe_table + + row = SimpleNamespace(name="legacy", schema_cache={"tables": {"t000": {"columns": [{"name": "id"}]}}, + "truncated": True, "total": 205}) + text = describe_table(row, "t300") + assert "中没有" not in text and "只探查了" in text and "information_schema" in text diff --git a/backend/tests/test_datasource_preview.py b/backend/tests/test_datasource_preview.py index 9f0f592b..9d3ce2e9 100644 --- a/backend/tests/test_datasource_preview.py +++ b/backend/tests/test_datasource_preview.py @@ -16,6 +16,7 @@ import pytest from httpx import ASGITransport, AsyncClient from app.main import app +from tests.fixtures.sources import introspect_now @pytest.fixture @@ -57,10 +58,13 @@ async def _cache(source_id: str) -> dict: async def test_cached_schema_says_which_schema_the_cache_came_from(client, source): assert source["cached_schema"] is None, "没探过就是没有缓存" - r = await client.post(f"/api/datasources/{source['id']}/introspect") - assert r.json()["cached_schema"] == "", "按默认 schema 探的写空串,和「没探过」分开" - r = await client.post(f"/api/datasources/{source['id']}/introspect", params={"schema": "main"}) - assert r.json()["cached_schema"] == "main" + out = await introspect_now(client, source["id"]) + assert out["cached_schema"] == "", "按默认 schema 探的写空串,和「没探过」分开" + # 一个数据源一个 schema(目录-D2):真探查只按配置里的 schema,要换先改配置 + r = await client.patch(f"/api/datasources/{source['id']}", json={"options": {"schema": "main"}}) + assert r.status_code == 200, r.text + out = await introspect_now(client, source["id"]) + assert out["cached_schema"] == "main" listed = next(s for s in (await client.get("/api/datasources")).json() if s["id"] == source["id"]) assert listed["cached_schema"] == "main" @@ -71,7 +75,7 @@ async def test_cached_schema_says_which_schema_the_cache_came_from(client, sourc async def test_a_dry_run_shows_what_is_there_without_touching_the_cache(client, source): - await client.post(f"/api/datasources/{source['id']}/introspect") + await introspect_now(client, source["id"]) before = await _cache(source["id"]) r = await client.post(f"/api/datasources/{source['id']}/introspect", @@ -89,7 +93,7 @@ async def test_a_dry_run_shows_what_is_there_without_touching_the_cache(client, async def test_a_dry_run_that_finds_nothing_says_why_and_still_leaves_the_cache(client, source): - await client.post(f"/api/datasources/{source['id']}/introspect") + await introspect_now(client, source["id"]) before = await _cache(source["id"]) r = await client.post(f"/api/datasources/{source['id']}/introspect", @@ -104,18 +108,26 @@ async def test_a_dry_run_that_finds_nothing_says_why_and_still_leaves_the_cache( async def test_without_dry_run_the_cache_is_still_replaced(client, source): - r = await client.post(f"/api/datasources/{source['id']}/introspect", params={"schema": "main"}) - assert r.json()["table_count"] == 3 + r = await client.patch(f"/api/datasources/{source['id']}", json={"options": {"schema": "main"}}) + assert r.status_code == 200, r.text + out = await introspect_now(client, source["id"]) + assert out["table_count"] == 3 assert (await _cache(source["id"]))["cache"]["schema"] == "main" +async def test_a_real_introspect_with_another_schema_is_refused(client, source): + """真探查不接受临时换 schema:一个数据源一个 schema,对象索引、探查范围都按配置里的那个记。""" + r = await client.post(f"/api/datasources/{source['id']}/introspect", params={"schema": "other"}) + assert r.status_code == 422 and "schema" in r.json()["detail"] + + # -------------------------------------------------------------------------- # REQ-6 ②:表结构的结构化列 # -------------------------------------------------------------------------- async def test_table_schema_comes_with_structured_columns(client, source): - await client.post(f"/api/datasources/{source['id']}/introspect") + await introspect_now(client, source["id"]) body = (await client.get(f"/api/datasources/{source['id']}/schema", params={"table": "orders"})).json() assert body["found"] is True @@ -133,7 +145,7 @@ async def test_table_schema_comes_with_structured_columns(client, source): async def test_an_unknown_table_has_no_columns_but_keeps_the_explanation(client, source): - await client.post(f"/api/datasources/{source['id']}/introspect") + await introspect_now(client, source["id"]) body = (await client.get(f"/api/datasources/{source['id']}/schema", params={"table": "nope"})).json() assert body["found"] is False and body["columns"] == [] diff --git a/backend/tests/test_introspect_jobs.py b/backend/tests/test_introspect_jobs.py new file mode 100644 index 00000000..32540cc8 --- /dev/null +++ b/backend/tests/test_introspect_jobs.py @@ -0,0 +1,418 @@ +"""对象索引、探查范围、后台探查任务(docs/design/next-round/03-数据接入与目录维护.md A1)。 + +以前探查结构每个源最多取 200 个对象、按数据库列出的顺序截断,同步请求、整份替换、不排队。这组守的是: + +- 对象索引列全库的对象(3,000 张表的合成库:3,000 个),dry_run 只刷新索引、数命中,不反射、不动表结构; +- 缺省范围(全部、500 张)碰到 3,000 张表拒绝开始并说出命中数,不悄悄截断,也不建任务; +- 规则(通配、按 name_key 不分大小写、排除优先、显式列出)圈出 400 张,后台任务逐表读完,schema_cache 形状不变、 + total 是对象总数、truncated 是「有范围外的」,每张表有结构指纹和版本; +- 停止:停在当前一批之后,读完的照样物化进表结构;服务重启:标中断、不续跑; +- 和同源的剖析、起草排同一条队:排在剖析后面时 waiting_for 是 profile,探查进行中同步起草 409 introspect_busy; +- 真删除记 removed_at(目录清单说「表已不在」),范围外、读不到结构分开说;改了结构版本加 1,没改不变; +- 字典查询不认时退回 Inspector;对象索引被截断时不判删除。 + +合成库在临时目录里建,用完连同数据源一起删掉。 +""" +from __future__ import annotations + +import asyncio +import sqlite3 +import uuid + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select, update + +from app.core import job_runtime as runtime +from app.data import introspect_jobs, source_objects as so +from app.db.base import SessionLocal +from app.db.models import DataSource +from app.db.models_sources import IntrospectJob, IntrospectTask, SourceObject +from app.main import app +from tests.fixtures.catalog import large +from tests.fixtures.sources import drop_source, introspect_now + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture(scope="module") +def big_db(tmp_path_factory) -> str: + return str(large.build(tmp_path_factory.mktemp("large") / "large.db")) + + +async def _make(client, path: str) -> str: + r = await client.post("/api/datasources", json={"name": f"big_{uuid.uuid4().hex[:8]}", "kind": "sqlite", + "database": path, "readonly": True}) + assert r.status_code == 201, r.text + return r.json()["id"] + + +@pytest.fixture +async def big(client, big_db): + sid = await _make(client, big_db) + yield sid + await drop_source(sid) + + +@pytest.fixture +async def small(client, tmp_path): + path = tmp_path / "small.db" + db = sqlite3.connect(path) + for name in ("alpha", "beta", "gamma", "Delta_Orders", "eta"): + db.execute(f'CREATE TABLE "{name}" (id INTEGER PRIMARY KEY, amount INTEGER)') + db.execute("CREATE VIEW v_alpha AS SELECT * FROM alpha") + db.commit() + db.close() + sid = await _make(client, str(path)) + yield sid, path + await drop_source(sid) + + +async def _scope(client, sid: str, **rules) -> dict: + r = await client.put(f"/api/datasources/{sid}/introspect-scope", json={"scope": rules}) + assert r.status_code == 200, r.text + return r.json() + + +async def _rows(sid: str) -> dict[str, SourceObject]: + async with SessionLocal() as session: + return {r.name: r for r in (await session.execute( + select(SourceObject).where(SourceObject.source_id == sid))).scalars()} + + +async def _cache(sid: str) -> dict: + async with SessionLocal() as session: + return dict((await session.get(DataSource, sid)).schema_cache or {}) + + +# -------------------------------------------------------------------------- +# 3,000 张表:对象索引、缺省范围超上限、规则圈 400 张 +# -------------------------------------------------------------------------- + + +async def test_index_lists_every_object_and_the_default_scope_refuses_without_truncating(client, big): + r = await client.post(f"/api/datasources/{big}/introspect", params={"dry_run": "true"}) + assert r.status_code == 200, r.text + preview = r.json() + assert preview["table_count"] == large.TOTAL == 3000 and preview["index_method"] == "dictionary" + assert preview["stored"] is True and len(preview["tables"]) == 200 and preview["truncated"] is True + assert preview["scope"]["hits"] == 3000 and preview["scope"]["cap"] == so.SCOPE_DEFAULT + assert preview["scope"]["over"] is True + assert await _cache(big) == {}, "dry_run 只刷新对象索引,不碰表结构缓存" + + r = await client.post(f"/api/datasources/{big}/introspect") + assert r.status_code == 409, r.text + body = r.json() + assert body["code"] == "scope_too_large" and body["hits"] == 3000 and body["cap"] == 500 + assert "3,000" in body["detail"] and "500" in body["detail"], "说出命中了多少张、上限多少" + assert (await client.get(f"/api/datasources/{big}/introspect-jobs/latest")).json()["job"] is None, \ + "超上限不建任务" + rows = await _rows(big) + assert len(rows) == 3000 and not any(r.in_scope for r in rows.values()), "不悄悄截断成前 500 张" + + +async def test_rules_pick_400_tables_and_the_background_job_reflects_them(client, big): + explicit = [t.name for t in large.TARGETS] + await _scope(client, big, mode="rules", include=["ORD_*", "sal_*", "inv_*"], exclude=["*_tmp_*", "*_bak_*"], + tables=explicit) + r = await client.post(f"/api/datasources/{big}/introspect-scope/preview", + json={"scope": {"mode": "rules", "include": ["ord_*"]}}) + assert r.status_code == 200 and r.json()["preview"] is None, "还没有对象索引:命中数要先刷新索引才数得出" + r = await client.post(f"/api/datasources/{big}/introspect") + assert r.status_code == 202, r.text + started = r.json()["job"] + assert started["status"] == "queued" and started["total"] == 400 + assert started["scope"]["rule"] == 390 and started["scope"]["explicit"] == 10 + await introspect_jobs.manager.wait(started["id"], timeout=180) + body = (await client.get(f"/api/datasources/{big}/introspect-jobs/{started['id']}")).json() + job, source = body["job"], body["source"] + assert job["status"] == "succeeded" and job["counts"]["ok"] == 400 and job["done"] == 400 + assert source["table_count"] == 400 and source["schema_scope"]["out_of_scope"] == 2600 + assert source["introspect_job"]["status"] == "succeeded" + + cache = await _cache(big) + assert set(cache) >= {"tables", "truncated", "total", "schema", "synced_at", "scope"}, "形状不变,另加 scope" + assert cache["total"] == 3000 and cache["truncated"] is True and len(cache["tables"]) == 400 + assert "ord_hdr_east" in cache["tables"] and "ord_tmp_east" not in cache["tables"], "排除优先" + assert "member_points_ledger" in cache["tables"], "显式列出的进范围" + meta = cache["tables"]["ord_hdr_east"] + assert {c["name"] for c in meta["columns"]} == {"id", "biz_date", "amount", "status", "ord_id"} + + rows = await _rows(big) + reflected = [r for r in rows.values() if r.in_scope] + assert len(reflected) == 400 and all(r.struct_version == 1 and r.fingerprint for r in reflected) + assert all(r.meta is None for r in rows.values() if not r.in_scope), "范围外的不存结构" + + # 目录清单只列范围内的;没有一张范围外的表被说成「表已不在」 + listed = (await client.get(f"/api/datasources/{big}/catalog")).json() + assert len(listed["tables"]) == 400 and listed["schema_total"] == 3000 and listed["schema_truncated"] is True + assert listed["scope"]["in_scope"] == 400 + + +async def test_stop_keeps_what_was_read_and_restart_marks_interrupted(client, big, monkeypatch): + await _scope(client, big, mode="rules", include=["ord_*"]) + real = so.reflect + gate = asyncio.Event() + + async def slow(*args, **kwargs): + got = await real(*args, **kwargs) + await asyncio.sleep(0.05) + await gate.wait() + return got + + monkeypatch.setattr(so, "reflect", slow) + job = (await client.post(f"/api/datasources/{big}/introspect")).json()["job"] + for _ in range(200): + await asyncio.sleep(0.02) + if (await client.get(f"/api/datasources/{big}/introspect-jobs/active")).json()["job"]["status"] == "running": + break + gate.set() # 放第一批过去,读完、记下了再关上 + for _ in range(300): + await asyncio.sleep(0.02) + if (await client.get(f"/api/datasources/{big}/introspect-jobs/{job['id']}")).json()["job"]["done"]: + break + gate.clear() + r = await client.post(f"/api/datasources/{big}/introspect-jobs/{job['id']}/cancel") + assert r.status_code == 200, r.text + gate.set() + stopped = r.json()["job"] + assert stopped["status"] == "cancelled" + assert 0 < stopped["counts"]["ok"] < 150 and stopped["counts"]["cancelled"] > 0 + cache = await _cache(big) + assert len(cache["tables"]) == stopped["counts"]["ok"], "读完的照样物化进表结构" + r = await client.post(f"/api/datasources/{big}/introspect-jobs/{job['id']}/cancel") + assert r.status_code == 409 and r.json()["code"] == "introspect_finished" + + # 服务重启:排队、进行中的标中断,没读的表 cancelled,不续跑 + async with SessionLocal() as session: + row = await session.get(DataSource, big) + queued, n = await introspect_jobs.prepare(session, row, actor="tester") + assert n == 150 + async with SessionLocal() as session: + await session.execute(update(IntrospectJob).where(IntrospectJob.id == queued.id).values(status="running")) + await session.commit() + assert await introspect_jobs.startup() >= 1 + async with SessionLocal() as session: + again = await session.get(IntrospectJob, queued.id) + tasks = list((await session.execute(select(IntrospectTask).where(IntrospectTask.job_id == queued.id))) + .scalars()) + assert again.status == "interrupted" and again.error == introspect_jobs.INTERRUPTED_ERROR + assert {t.status for t in tasks} == {"cancelled"} and not introspect_jobs.manager.running(queued.id) + + +# -------------------------------------------------------------------------- +# 和同源的起草、剖析排队 +# -------------------------------------------------------------------------- + + +async def test_introspect_queues_behind_profile_and_blocks_sync_writes(client, small, monkeypatch): + sid, _ = small + held = runtime.lanes.enter(sid, "profile", "someone-else") + try: + job = (await client.post(f"/api/datasources/{sid}/introspect")).json()["job"] + await asyncio.sleep(0.1) + now = (await client.get(f"/api/datasources/{sid}/introspect-jobs/{job['id']}")).json()["job"] + assert now["status"] == "queued" and now["waiting_for"] == "profile", "排在同源的剖析后面" + r = await client.post(f"/api/datasources/{sid}/introspect") + assert r.status_code == 409 and r.json()["code"] == "introspect_busy" + finally: + held.release() + await introspect_jobs.manager.wait(job["id"], timeout=30) + assert (await client.get(f"/api/datasources/{sid}/introspect-jobs/{job['id']}")).json()["job"]["status"] \ + == "succeeded" + + # 探查进行中:同步起草、导入目录占不到队,照实说是探查在做 + gate = asyncio.Event() + real = so.reflect + + async def wait(*args, **kwargs): + await gate.wait() + return await real(*args, **kwargs) + + monkeypatch.setattr(so, "reflect", wait) + job = (await client.post(f"/api/datasources/{sid}/introspect")).json()["job"] + for _ in range(100): + await asyncio.sleep(0.02) + if runtime.lanes.holders(sid) and runtime.lanes.holders(sid)[0].kind == "introspect": + break + r = await client.post(f"/api/datasources/{sid}/catalog/draft", json={"tables": ["alpha"]}) + assert r.status_code == 409 and r.json()["code"] == "introspect_busy" and "探查" in r.json()["detail"] + gate.set() + await introspect_jobs.manager.wait(job["id"], timeout=30) + + +# -------------------------------------------------------------------------- +# 删除、结构版本、三种「不在」 +# -------------------------------------------------------------------------- + + +async def test_real_deletion_versions_and_the_three_kinds_of_absent(client, small): + sid, path = small + await introspect_now(client, sid) + rows = await _rows(sid) + assert {r.name: r.struct_version for r in rows.values()} == dict.fromkeys( + ["Delta_Orders", "alpha", "beta", "eta", "gamma", "v_alpha"], 1) + first = {n: r.fingerprint for n, r in rows.items()} + + db = sqlite3.connect(path) + db.execute("ALTER TABLE beta ADD COLUMN note TEXT") + db.execute("DROP TABLE gamma") + db.commit() + db.close() + # 有目录的表:gamma(删掉了)、eta(排除了、显式移出 pinned 才能排掉)、alpha(规则排除也排不掉) + from app.data import catalog + + async with SessionLocal() as session: + for table in ("gamma", "eta", "alpha"): + await catalog.write_entry(session, sid, table, {"label": {"value": table, "source": "human", + "status": "confirmed"}}, + if_version=0, actor=None) + await _scope(client, sid, mode="all", exclude=["alpha", "eta"], removed=["ETA"]) + out = await introspect_now(client, sid) + rows = await _rows(sid) + assert rows["beta"].struct_version == 2 and rows["beta"].fingerprint != first["beta"], "改了结构版本加 1" + assert rows["Delta_Orders"].struct_version == 1 and rows["Delta_Orders"].fingerprint == first["Delta_Orders"] + assert rows["gamma"].removed_at is not None and not rows["gamma"].in_scope, "真删除记 removed_at" + assert rows["alpha"].in_scope and rows["alpha"].scope_reason == "pinned", "有目录的表规则排不掉" + assert not rows["eta"].in_scope, "显式移出才排得掉 pinned" + assert out["table_count"] == 4 + + listed = {t["table_name"]: t for t in (await client.get(f"/api/datasources/{sid}/catalog")).json()["tables"]} + assert listed["gamma"]["absent_reason"] == "removed" and listed["gamma"]["in_schema"] is False + assert listed["eta"]["absent_reason"] == "out_of_scope", "范围外不是「表已不在」" + assert listed["alpha"]["absent_reason"] is None + + # 读不到结构:在范围里、反射失败 + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + await session.execute(update(SourceObject).where(SourceObject.source_id == sid, SourceObject.name == "alpha") + .values(meta=None, reflect_error="权限不够,读不到列")) + await so.rebuild_schema_cache(session, row) + await session.commit() + listed = {t["table_name"]: t for t in (await client.get(f"/api/datasources/{sid}/catalog")).json()["tables"]} + assert listed["alpha"]["absent_reason"] == "reflect_failed" + detail = (await client.get(f"/api/datasources/{sid}/schema", params={"table": "alpha"})).json()["detail"] + assert "读不到它的结构" in detail and "权限不够" in detail + detail = (await client.get(f"/api/datasources/{sid}/schema", params={"table": "eta"})).json()["detail"] + assert "探查范围外" in detail and "已删除" not in detail + detail = (await client.get(f"/api/datasources/{sid}/schema", params={"table": "gamma"})).json()["detail"] + assert "已经没有 gamma" in detail + + # 删了又建回来:清掉 removed_at + db = sqlite3.connect(path) + db.execute("CREATE TABLE gamma (id INTEGER PRIMARY KEY)") + db.commit() + db.close() + await introspect_now(client, sid) + assert (await _rows(sid))["gamma"].removed_at is None + + +async def test_index_falls_back_to_the_inspector_and_a_capped_index_does_not_delete(client, small, monkeypatch): + sid, _ = small + from app.data import engine as data_engine + + async def broken(*_a, **_k): + raise RuntimeError("no such table: sqlite_master") + + monkeypatch.setattr(data_engine, "run_query", broken) + preview = (await client.post(f"/api/datasources/{sid}/introspect", params={"dry_run": "true"})).json() + assert preview["index_method"] == "inspector" and preview["table_count"] == 6 + monkeypatch.undo() + await introspect_now(client, sid) + + monkeypatch.setattr(so, "INDEX_LIMIT", 3) + preview = (await client.post(f"/api/datasources/{sid}/introspect", params={"dry_run": "true"})).json() + assert preview["index_capped"] is True and preview["table_count"] == 3 + rows = await _rows(sid) + assert not any(r.removed_at for r in rows.values()), "对象索引被截断时不判删除" + + +# -------------------------------------------------------------------------- +# 范围规则(纯函数)、published 模板的 pinned +# -------------------------------------------------------------------------- + + +def test_scope_rules(): + objects = [("Orders", "table"), ("order_line", "table"), ("tmp_orders", "table"), ("v_orders", "view"), + ("Customers", "table"), ("audit_log", "table")] + rules = so.ScopeRules(mode="rules", include=["ORDER*", "*orders"], exclude=["tmp_*"], views=False) + plan = so.plan_scope(objects, rules, pinned=set(), on_demand=set()) + assert list(plan.in_scope) == ["order_line", "Orders"], "按 name_key 不分大小写、排除优先、views=false 排掉视图" + plan = so.plan_scope(objects, rules, pinned={"tmp_orders", "audit_log"}, on_demand={"customers"}) + assert plan.in_scope == {"audit_log": "pinned", "Customers": "on_demand", "order_line": "rule", + "Orders": "rule", "tmp_orders": "pinned"} + assert plan.capped_count == 4, "按需加入不计上限" + rules = so.ScopeRules(mode="rules", include=[], tables=["customers"], removed=["TMP_ORDERS"]) + plan = so.plan_scope(objects, rules, pinned={"tmp_orders"}, on_demand=set()) + assert plan.in_scope == {"Customers": "explicit"}, "显式移出压过 pinned;mode=rules 没写包含只剩显式的" + rules = so.ScopeRules(max_tables=2) + plan = so.plan_scope(objects, rules, pinned=set(), on_demand=set()) + assert plan.over and "6 张" in so.too_large_message(plan) and "上限 2 张" in so.too_large_message(plan) + assert so.rules_problem({"mode": "rules", "max_tables": 2001}) and so.rules_problem({"include": "x"}) + assert so.rules_problem({"mode": "rules", "include": ["a*"], "max_tables": 2000}) is None + + +async def test_published_template_tables_are_pinned(client, small, monkeypatch): + sid, _ = small + async with SessionLocal() as session: + name = (await session.get(DataSource, sid)).name + graph = {"nodes": [ + {"id": "start", "type": "input", "data": {"label": "start", "config": {}}}, + {"id": "q", "type": "tool", "data": {"label": "q", "config": {"tool": f"db_query__{name}", + "args": {"sql": "SELECT * FROM beta"}}}}, + {"id": "out", "type": "output", "data": {"label": "out", "config": {"fields": []}}}, + ], "edges": [{"source": "start", "target": "q"}, {"source": "q", "target": "out"}]} + from types import SimpleNamespace + + from app.data import catalog_impact + + async def published(_session): + return [(SimpleNamespace(id="w"), SimpleNamespace(graph=graph))] + + monkeypatch.setattr(catalog_impact, "_published", published) + await _scope(client, sid, mode="rules", include=["alpha"]) + out = await introspect_now(client, sid) + rows = await _rows(sid) + assert rows["beta"].scope_reason == "pinned" and out["table_count"] == 2 + + +# -------------------------------------------------------------------------- +# 每种方言的字典查询(金样) +# -------------------------------------------------------------------------- + +GOLDEN = { + ("sqlite", None): "SELECT name, type, NULL AS comment, NULL AS est_rows FROM sqlite_master WHERE type IN ('table', " + "'view') AND substr(name, 1, 7) <> 'sqlite_' ORDER BY name", + ("mysql", "shop"): "SELECT TABLE_NAME, TABLE_TYPE, TABLE_COMMENT, TABLE_ROWS FROM information_schema.TABLES WHERE " + "TABLE_SCHEMA = 'shop' ORDER BY TABLE_NAME", + ("mysql", None): "SELECT TABLE_NAME, TABLE_TYPE, TABLE_COMMENT, TABLE_ROWS FROM information_schema.TABLES WHERE " + "TABLE_SCHEMA = DATABASE() ORDER BY TABLE_NAME", + ("postgresql", "public"): "SELECT c.relname, c.relkind, obj_description(c.oid, 'pg_class'), c.reltuples FROM " + "pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE " + "n.nspname = 'public' AND c.relkind IN ('r', 'p', 'f', 'v', 'm') AND NOT " + "c.relispartition ORDER BY c.relname", + ("oracle", "ANALYTICS"): "SELECT o.object_name, o.object_type, c.comments, t.num_rows FROM all_objects o LEFT JOIN " + "all_tab_comments c ON c.owner = o.owner AND c.table_name = o.object_name LEFT JOIN " + "all_tables t ON t.owner = o.owner AND t.table_name = o.object_name WHERE o.owner = " + "'ANALYTICS' AND o.object_type IN ('TABLE', 'VIEW') AND o.generated = 'N' AND " + "o.object_name NOT LIKE 'BIN$%' ORDER BY o.object_name", +} + + +@pytest.mark.parametrize("kind,schema", list(GOLDEN)) +def test_index_sql_golden_parses_and_passes_the_guard(kind, schema): + import sqlglot + + from app.data.guard import check + + sql = so.index_sql(kind, schema) + assert sql == GOLDEN[(kind, schema)] + read = {"postgresql": "postgres"}.get(kind, kind) + assert sqlglot.parse_one(sql, read=read) is not None + check(sql, readonly=True, source_name="x", dialect=kind) + assert so.index_sql("mariadb", "shop") == GOLDEN[("mysql", "shop")] + assert so.index_sql("db2", None) is None