"""数据目录接口：/api/datasources/{source_id}/catalog。 - GET /catalog 表清单（没有目录的表也列出来） - GET /catalog/{table} 单表目录 + 表结构 - PUT /catalog/{table} 整份提交（乐观锁），改动过的项记为人工确认 - POST /catalog/{table}/review 单项审阅：确认、驳回、撤销审阅 - POST /catalog/{table}/patch/preview 目录修改提案的预览（只读）：对着当前目录算改前、改后 - POST /catalog/{table}/patch 保存目录修改提案（乐观锁），记为人工确认 - GET /catalog/{table}/impact 影响面：引用这张表的已发布、受管模板（data/catalog_impact.py） - POST /catalog/draft 同步起草（注释、外键、命名推断，可选模型）。界面改用下面的后台起草任务，这个留着； 同一个源有排队或进行中的起草任务（或正在同步起草）时 409 catalog_draft_busy，有剖析任务时 409 profile_busy；同步起草期间建起草任务、重试、导入也是 409，新建的剖析任务排在它后面 - PUT /catalog/guidance 数据源级的起草要求（使用者补充的背景和要求，起草时交给模型） - POST /catalog/draft-jobs 建后台起草任务（202）；GET /draft-jobs/active、/latest、/{job_id} 读任务； POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py）。 同源有剖析任务时照建，排在它后面（waiting_for） - POST /catalog/profile-jobs 建后台剖析任务（202）：对业务库逐表发少量只读查询，核对关系、取码值候选、提议业务日期 （data/catalog_profile_jobs.py，引擎在 data/catalog_profile.py）；GET /profile-jobs/active、 /latest、/{job_id} 读任务，/{job_id}/tasks/{seq} 读一张表的明细（检查项、发现、发过的查询）； POST /profile-jobs/{job_id}/cancel 停止，/retry 只重试没做完的表。同源有起草任务时照建， 排在它后面（waiting_for） - GET /catalog/review-queue 审阅队列：推断的项按使用次数、风险、来源排好；POST /catalog/review/batch 批量审阅， GET /catalog/review-progress 审阅进度，GET /catalog/{table}/usage 用过这张表的运行 （data/catalog_review.py） - GET /catalog/export 导出数据目录（JSON 附件）；POST /catalog/import/preview 导入预览（只读）， POST /catalog/import 导入（data/catalog_transfer.py） - POST /catalog/{table}/assistant 表助手：对话完善这张表的目录（text/event-stream），提案不写目录 （data/catalog_assistant.py） - GET /catalog/search 在对象索引里按名字、注释找表（含探查范围外的）；POST /catalog/scope/add 把范围外的表加进 探查范围（当场只读读结构、记进规则的显式列出，next-round 03 A2） 规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径 不对 422，源或表不存在 404。所有写操作记下署名（api/deps.actor_of：开了登录是认证过的用户名，否则请求头 X-Actor 的自报）。 起草、剖析任务的 409 带机读码（api/coded.py）：同一个源已有排队或进行中的同种任务、起草模型用不了、剖析没开启、按范围 挑不出表、要停止的任务已经结束。 上传表格的源按当前快照的表结构回答（table_versions.resolve_source），和查询实际用的库是同一个版本。 """ from __future__ import annotations from typing import Any, Literal from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query from fastapi.exceptions import RequestValidationError from pydantic import BaseModel, Field, ValidationError from sqlalchemy.ext.asyncio import AsyncSession from app.api.coded import ( CATALOG_DRAFT_BUSY, CATALOG_DRAFT_FINISHED, CATALOG_DRAFT_MODEL_UNAVAILABLE, CATALOG_PROFILE_FINISHED, INTROSPECT_BUSY, PROFILE_BUSY, PROFILE_NOTHING, SCHEMA_MISSING, SNAPSHOT_TAMPERED, CodedHTTPException, ) from app.api.deps import actor_of from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, catalog_profile_jobs, introspect from app.data import introspect_jobs, source_objects, table_versions from app.data import catalog_assistant, catalog_review, catalog_transfer from app.data.engine import SnapshotTampered from app.db.base import get_session from app.db.models import DataSource router = APIRouter(prefix="/api/datasources", tags=["catalog"]) class CatalogPutIn(BaseModel): notes: dict[str, Any] #: 读到的版本；这张表还没有目录时为 0 if_version: int = Field(ge=0) class CatalogReviewIn(BaseModel): #: grain、columns.amount.measure、relations.<编号> path: str = Field(min_length=1, max_length=600) action: Literal["confirm", "reject", "reset"] if_version: int = Field(ge=0) class CatalogPatchChange(BaseModel): #: 审阅接口的路径写法；新增关联关系写 relations.new path: str = Field(min_length=1, max_length=600) value: Any reason: str | None = Field(default=None, max_length=2000) class CatalogPatchPreviewIn(BaseModel): changes: list[CatalogPatchChange] = Field(min_length=1, max_length=catalog.PATCH_MAX_CHANGES) class CatalogPatchIn(CatalogPatchPreviewIn): if_version: int = Field(ge=0) class CatalogDraftIn(BaseModel): #: 要起草的表；不给或为空时按使用次数取前 20 张 tables: list[str] | None = Field(default=None, max_length=200) #: 是否请起草模型（设置里的起草模型，没设沿用助手的模型）写中文名、粒度、列的含义等 use_model: bool = False class DraftJobIn(BaseModel): #: 要起草的表；不给或为空时按使用次数取前 20 张。最多 500 张（超了 422，说明是中文） tables: list[str] | None = None #: 是否请起草模型写中文名、粒度、列的含义等 use_model: bool = False #: 这一次的补充说明，和数据源级的起草要求一起交给模型（最多 1000 字） note: str | None = None #: PUT /catalog/guidance 的请求 {guidance: string}：数据源级的起草要求，空串（或 null）清除，最多 2000 字。必须写 #: 这一项：表名恰好叫 guidance 时，整份提交那张表目录的请求也落到这个路径上，不能把它当成「清除起草要求」。 #: 没写、不是文字都回 422，说明是中文的一句话（同超长） GUIDANCE_MISSING = "请求里要写起草要求（guidance 一项，一段文字；要清除起草要求请传空串）" GUIDANCE_NOT_TEXT = "起草要求要是一段文字（要清除起草要求请传空串）" class ProfileJobIn(BaseModel): #: 要剖析的表（范围 selected，按给的顺序去重）；最多 500 张（超了 422，说明是中文） tables: list[str] | None = None #: 不指定表时的范围：needed 需要剖析的表（缺省）、all 全部表（都不含视图、没有可剖析的项的表）、observe 取值观察（只观察 #: 有数据目录、观察到期的表，不剖析；见 data/catalog_profile_observe）。maintenance 是定时维护用的，这里不收 scope: Literal["needed", "all", "observe"] | None = None async def _resolved(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]: """(数据源记录, 用来读表结构的源)：上传源解析成绑定当前快照的视图。""" row = await session.get(DataSource, source_id) if row is None: raise HTTPException(404, "数据源不存在，可能已被删除") try: return row, await table_versions.resolve_source(session, row) except table_versions.SnapshotMissing as e: raise HTTPException(409, str(e)) from e def _iso(value: Any) -> str | None: return value.isoformat() if value is not None else None def _int(value: Any) -> int: return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else 0 def _item_value(notes: dict[str, Any], name: str) -> tuple[Any, str | None]: item = notes.get(name) if not isinstance(item, dict) or item.get("status") == "rejected": return None, None return item.get("value"), item.get("status") def _summary_row(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogEntry | None, usage: int, last_draft: dict[str, Any] | None = None, profiling: dict[str, Any] | None = None, absent: str | None = None) -> dict[str, Any]: notes = entry.notes if entry else {} label, label_status = _item_value(notes, "label") kind, _ = _item_value(notes, "kind") relations = notes.get("relations") if isinstance(notes.get("relations"), list) else [] return { "table_name": name, "qualified": (meta or {}).get("qualified", name), "is_view": bool((meta or {}).get("is_view")), # 表结构里没有这张表，目录还留着：界面据此提示。为什么不在（absent_reason）分开说：removed 表已不在（真删除）、 # missing 数据源里从来没有、out_of_scope 在探查范围外、reflect_failed 在范围里却读不到结构、unknown 说不清 # （升级前只探查了一部分的老缓存）。只有 removed 才说「表已不在」 "in_schema": meta is not None, "absent_reason": None if meta is not None else absent, "label": label, "label_status": label_status, "kind": kind, "counts": catalog.status_counts(notes), "relations": sum(1 for r in relations if isinstance(r, dict) and r.get("status") != "rejected"), "usage": usage, "version": entry.version if entry else 0, "updated_at": _iso(entry.updated_at) if entry else None, "updated_by": entry.updated_by if entry else None, # 这张表最近一次起草任务里的那一项：{status, job_id, finished_at, error, model_error, blank_columns}，没起草过 # 是 null "last_draft": last_draft, # 剖析：最近一次剖析（CatalogLastProfile，没剖析过是 null）、要不要剖析和为什么、按当前目录和表结构要做几项检查 **(profiling or {"last_profile": None, "profile_due": False, "profile_due_reason": None, "profile_checks": 0}), } def _profiling(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogEntry | None, *, tables: dict[str, Any], masked: set[str], last: dict[str, Any] | None, outside: frozenset[str] = frozenset()) -> dict[str, Any]: notes = entry.notes if entry else {} planned = catalog_profile.plan_checks(name, meta, notes, tables=tables, masked=masked, outside=outside) checks = sum(1 for c in planned if c.skip is None) reason = catalog_profile_jobs.due_reason(meta, notes, last, planned) return {"last_profile": catalog_profile_jobs.public_last(last), "profile_due": reason is not None, "profile_due_reason": reason, "profile_checks": checks} def _structure(meta: dict[str, Any]) -> dict[str, Any]: return { "qualified": meta.get("qualified"), "is_view": bool(meta.get("is_view")), "comment": meta.get("comment"), "columns": introspect.table_columns(meta), "primary_key": list(meta.get("primary_key") or []), # 升级前探查的缓存里没有这两项：为 null 表示「不知道」，和空列表（读过、没有）分开 "foreign_keys": meta.get("foreign_keys"), "unique": meta.get("unique"), } async def _detail(session: AsyncSession, source: Any, table: str, entry: catalog.CatalogEntry | None, usage: int | None = None) -> dict[str, Any]: meta = ((source.schema_cache or {}).get("tables") or {}).get(table) if usage is None: usage = (await catalog.table_usage(session, source)).get(table, 0) absent = None if meta is not None else (await _absent(session, source, [table])).get(table) return { "table_name": table, "in_schema": meta is not None, # 表结构里没有时为什么（同表清单的 absent_reason） "absent_reason": absent, "notes": entry.notes if entry else {}, "version": entry.version if entry else 0, "updated_at": _iso(entry.updated_at) if entry else None, "updated_by": entry.updated_by if entry else None, "structure": _structure(meta) if meta is not None else None, "system_notes": catalog.system_notes_source(source), "usage": usage, # 这张表最近一次剖析（CatalogProfileTaskDetail：检查项、发现、跳过的项、日期范围、发过的查询）；任务还在进行就是 # 进行中的状态；没剖析过是 null "last_profile": await catalog_profile_jobs.last_detail(session, source.id, table), } async def _table_key(session: AsyncSession, source: Any, table: str) -> tuple[str, catalog.CatalogEntry | None]: """路径里的表名 → 目录的键。表结构里有的按表结构的写法；表结构里没有、但还留着目录的也认（便于清理）。都没有时 404， 说清是哪一种「不在」（_missing）。""" key = catalog.resolve_table_name(source.schema_cache, table) entry = await catalog.read_entry(session, source.id, key or table) if key is None and entry is None: raise await _missing(session, source, table) return key or table, entry async def _missing(session: AsyncSession, source: Any, table: str) -> HTTPException: """表结构里没有这张表的 404：表已不在、数据源里没有、在探查范围外、读不到结构、说不清，和 /schema、/catalog/search、 目录清单的 absent_reason 同一套（source_objects.absent_message）。""" return HTTPException(404, await source_objects.absent_message(session, source, table)) def _conflict(e: catalog.CatalogConflict) -> HTTPException: return HTTPException(409, str(e)) def _invalid(e: catalog.CatalogInvalid) -> HTTPException: shown = "；".join(e.problems[:8]) + (f"；另有 {len(e.problems) - 8} 处" if len(e.problems) > 8 else "") return HTTPException(422, f"数据目录的格式不正确：{shown}") @router.get("/{source_id}/catalog") async def list_catalog(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """表清单：探查范围内、读到了结构的每张表各一行（没有目录的也列），使用次数多的在前；目录还在、表结构里没有的排最后， 各带 absent_reason（表已不在、数据源里没有、在探查范围外、读不到结构，见 _summary_row）。 几千张表的库只列探查范围内的（next-round 03 A1）：范围外的在对象索引里，用 GET /catalog/search 找、POST /catalog/scope/add 加进来。schema_truncated 是「有范围外的对象」，schema_total 是对象索引里一共几张（含视图）；scope 是范围的摘要 （schema_cache 的 scope：in_scope、reflected、failed、out_of_scope、cap、各原因几张），升级前探查的源为 null。 """ row, source = await _resolved(session, source_id) cache = source.schema_cache or {} tables = cache.get("tables") or {} entries = await catalog.read_catalog(session, source.id) usage = await catalog.table_usage(session, source) drafts = await catalog_jobs.last_drafts(session, source.id) lasts = await catalog_profile_jobs.last_profiles(session, source.id) masked = catalog_profile.masked_of(row.options) outside = await catalog_profile_jobs.outside_of(session, row) def profiling(name: str, meta: dict[str, Any] | None) -> dict[str, Any]: return _profiling(name, meta, entries.get(name), tables=tables, masked=masked, last=lasts.get(name), outside=outside) order = {name: i for i, name in enumerate(tables)} rows = [_summary_row(name, meta, entries.get(name), usage.get(name, 0), drafts.get(name), profiling(name, meta)) for name, meta in tables.items()] rows.sort(key=lambda r: (-r["usage"], order[r["table_name"]])) gone = [name for name in sorted(entries) if name not in tables] absent = await _absent(session, row, gone) rows += [_summary_row(name, None, entries[name], 0, drafts.get(name), profiling(name, None), absent.get(name)) for name in gone] return { "tables": rows, "system_notes": catalog.system_notes_source(source), # 没有表结构时说明原因（尚未探查 / 探查失败），界面据此引导去探查 "schema_note": None if tables else introspect.why_empty(cache), "schema_truncated": bool(cache.get("truncated")), "schema_total": max(_int(cache.get("total")), len(tables)), "scope": dict(cache["scope"]) if isinstance(cache.get("scope"), dict) else None, # 数据源级的起草要求（没有是 null） "guidance": catalog_jobs.guidance_of(row), # 排队或进行中的起草任务 {id, status, total, done, waiting_for}，界面据此显示进度；没有是 null "active_job": await catalog_jobs.job_brief(session, await catalog_jobs.active_job(session, source.id)), # 排队或进行中的剖析任务 {id, status, total, done, waiting_for}；没有是 null "active_profile_job": await catalog_profile_jobs.job_brief( session, await catalog_profile_jobs.active_job(session, source.id)), } async def _absent(session: AsyncSession, row: DataSource, names: list[str]) -> dict[str, str]: """目录还在、表结构里没有的表为什么不在（source_objects.absent_reasons）。上传表格、记录表的结构随版本走：不在就是那一版 里没有这张表（removed）。""" if not names: return {} if (row.origin or "manual") != "manual": return dict.fromkeys(names, source_objects.ABSENT_REMOVED) return await source_objects.absent_reasons(session, row, names) # ---- 对象索引里找表、加入探查范围（要登记在 GET /catalog/{table} 前面：不然 search 会被当成表名） @router.get("/{source_id}/catalog/search") async def search_catalog_tables(source_id: str, q: str = Query("", max_length=200), limit: int = Query(50, ge=1, le=200), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """在对象索引里按名字、注释找表（不分大小写的包含）：{items: [{name, qualified, kind, comment, est_rows, in_scope, scope_reason, absent_reason}], total, indexed}。清单只列探查范围内的表，范围外的靠这里找到、再「加入范围」。没有对象 索引的源（上传表格、升级前探查的）只在表结构里找（indexed=false）。""" row, source = await _resolved(session, source_id) cache = source.schema_cache or {} schema = cache.get("schema") text = q.strip() if (row.origin or "manual") == "manual" and await source_objects.has_index( session, row.id, source_objects.index_schema(row)): items, total = await source_objects.search_index(session, row, text, limit=limit) for item in items: item["qualified"] = f"{schema}.{item['name']}" if schema else item["name"] return {"items": items, "total": total, "indexed": True} from app.data.names import name_key key = name_key(text) hits = [(n, m) for n, m in (cache.get("tables") or {}).items() if key and ( key in name_key(n) or text in str((m or {}).get("comment") or ""))] return {"items": [{"name": n, "qualified": (m or {}).get("qualified", n), "kind": "view" if (m or {}).get("is_view") else "table", "comment": (m or {}).get("comment"), "est_rows": None, "in_scope": True, "scope_reason": None, "absent_reason": None} for n, m in hits[:limit]], "total": len(hits), "indexed": False} class ScopeAddIn(BaseModel): #: 要加进探查范围的表（对象索引里的名字），一次最多 ADD_LIMIT 张 tables: list[str] = Field(min_length=1, max_length=source_objects.ADD_LIMIT) @router.post("/{source_id}/catalog/scope/add") async def add_catalog_scope(source_id: str, payload: ScopeAddIn, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """把对象索引里探查范围外的表加进探查范围：当场只读读结构、写进表结构（schema_cache），并记进探查范围规则的「显式 列出」，以后每次探查都带上；留痕记谁、什么时候。返回 {added, already, failed: [{name, error}], refused: [{name, reason}], scope}。上传表格、记录表 409（结构随版本走）。""" row = await _source_row(session, source_id) if (row.origin or "manual") != "manual": raise HTTPException(409, "上传的表格、记录表没有探查范围：结构随版本走") got = await source_objects.add_to_scope(row.id, [str(t) for t in payload.tables], reason="explicit", trail=source_objects.Trail(via=source_objects.VIA_CATALOG, actor=actor_of(x_actor))) cache = got.cache if got.cache is not None else (row.schema_cache or {}) return {"added": got.added, "already": got.already, "failed": [{"name": n, "error": e} for n, e in got.failed.items()], "refused": [{"name": n, "reason": r} for n, r in got.refused.items()], "scope": dict(cache["scope"]) if isinstance(cache.get("scope"), dict) else None} # ---- 审阅队列、审阅进度（要登记在 GET /catalog/{table} 前面：不然 review-queue、review-progress 会被当成表名） @router.get("/{source_id}/catalog/review-queue") async def catalog_review_queue(source_id: str, limit: int = Query(50, ge=1, le=catalog_review.QUEUE_LIMIT_MAX), offset: int = Query(0, ge=0), kind: Literal["all", "facts", "names"] = Query("all"), table: str | None = Query(None, max_length=255), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """审阅队列：只列推断的项，按使用次数、风险、来源排好（data/catalog_review.py）。kind 筛影响数字的项（facts）或名称 和说明（names），table 只看一张表。""" _, source = await _resolved(session, source_id) return await catalog_review.review_queue(session, source, kind=kind, table=table or None, limit=limit, offset=offset) @router.get("/{source_id}/catalog/review-progress") async def catalog_review_progress(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """审阅进度：常用 20 张表、全部、影响数字的项各自确认了多少。""" _, source = await _resolved(session, source_id) return await catalog_review.review_progress(session, source) # ---- 导出导入（导出同样要登记在 GET /catalog/{table} 前面） @router.get("/{source_id}/catalog/export") async def export_catalog(source_id: str, session: AsyncSession = Depends(get_session)) -> Any: """导出这个数据源的数据目录：JSON 附件「<数据源名>-数据目录.json」，含全部状态的项（data/catalog_transfer.py）。""" from fastapi.responses import JSONResponse row = await _source_row(session, source_id) body = await catalog_transfer.export_catalog(session, row) name = catalog_transfer.attachment_name(row.name) return JSONResponse(body, headers={"Content-Disposition": catalog_transfer.content_disposition(name, row.id)}) class CatalogImportIn(BaseModel): #: 导出的那个对象（格式在 data/catalog_transfer.read_file 里核对，不对 422，说明是中文） content: Any = None #: 目标还没有起草要求时，是否写入文件里的起草要求 include_guidance: bool = False async def _import_source(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]: row, source = await _resolved(session, source_id) cache = source.schema_cache or {} if not cache.get("tables"): raise CodedHTTPException(409, f"{introspect.why_empty(cache)}，无法导入数据目录。请先在数据源卡片上点「探查结构」", SCHEMA_MISSING) return row, source @router.post("/{source_id}/catalog/import/preview") async def preview_catalog_import(source_id: str, payload: CatalogImportIn, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """导入预览（只读）：每张对上的表新增、更新、保留几项，冲突和跳过的逐条列出。文件格式不对 422；没有表结构 409。""" row, source = await _import_source(session, source_id) try: report = await catalog_transfer.plan_import(session, row, source, payload.content, include_guidance=payload.include_guidance) except catalog_transfer.TransferInvalid as e: raise HTTPException(422, str(e)) from e return report.as_dict() @router.post("/{source_id}/catalog/import") async def import_catalog(source_id: str, payload: CatalogImportIn, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """导入：规则同预览，逐表写入（乐观锁），返回同样形状的报告和每张表写入后的版本。有排队或进行中的起草任务（或正在 同步起草、导入）409 catalog_draft_busy，有剖析任务 409 profile_busy（说法的最后一句说「再导入」）；文件格式不对 422； 没有表结构 409。""" row, source = await _import_source(session, source_id) try: async with catalog_jobs.import_write(session, row): report = await catalog_transfer.run_import(session, row, source, payload.content, include_guidance=payload.include_guidance, actor=actor_of(x_actor)) except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e except catalog_profile_jobs.ProfileBusy as e: raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e except introspect_jobs.IntrospectBusy as e: raise CodedHTTPException(409, str(e), INTROSPECT_BUSY) from e except catalog_transfer.TransferInvalid as e: raise HTTPException(422, str(e)) from e return report.as_dict() # ---- 后台剖析任务（同样登记在 GET /catalog/{table} 前面：active、latest 要先于 /profile-jobs/{job_id}） async def _profile_source(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]: """建剖析任务、重试之前的几道门，409 都带机读码（界面按它分情况给下一步，不从原话里认）：未开启 profile_disabled、 数据源停用 datasource_inactive、快照登记被改动 snapshot_tampered、没有表结构 schema_missing。返回 (数据源记录, 读表结构用的源)。""" row = await _source_row(session, source_id) try: catalog_profile.ensure_enabled(row) except catalog_profile.ProfileDisabled as e: raise CodedHTTPException(409, str(e), e.code) from e try: source = await table_versions.resolve_source(session, row) except SnapshotTampered as e: # 快照清单对不上（SnapshotManifestMismatch 同时是 SnapshotMissing） raise CodedHTTPException(409, str(e), SNAPSHOT_TAMPERED) from e except table_versions.SnapshotMissing as e: raise HTTPException(409, str(e)) from e cache = source.schema_cache or {} if not cache.get("tables"): raise CodedHTTPException(409, f"{introspect.why_empty(cache)}，无法进行数据剖析。请先在数据源卡片上点「探查结构」", SCHEMA_MISSING) return row, source async def _start_profile(session: AsyncSession, row: DataSource, source: Any, *, tables: list[str] | None, scope: str, actor: str | None, retry_of: str | None = None) -> dict[str, Any]: try: job, tasks = await catalog_profile_jobs.create_job( session, row, source, tables=tables, scope=scope, settings=catalog_profile.ensure_enabled(row), actor=actor, retry_of=retry_of) except catalog_profile_jobs.ProfileBusy as e: raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e except catalog_profile_jobs.ProfileNothing as e: raise CodedHTTPException(409, str(e), PROFILE_NOTHING) from e catalog_profile_jobs.manager.start(job.id) return {"job": catalog_profile_jobs.job_dict(job, tasks, waiting_for=await catalog_profile_jobs.waiting_for(session, job))} @router.post("/{source_id}/catalog/profile-jobs", status_code=202) async def create_profile_job(source_id: str, payload: ProfileJobIn, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """建后台剖析任务，马上返回（202，任务是排队中）。tables 非空时范围是 selected（表结构里没有、视图、没有可剖析的项照样 建进任务、状态 skipped），否则按 scope（缺省 needed）挑。同源有排队或进行中的剖析任务 409 profile_busy（说清进行到第几张； 界面改去 GET /profile-jobs/active 显示那一个），按范围挑出 0 张表 409 profile_nothing；同源有起草任务时照建，排在它后面 （waiting_for = "draft"），起草结束后自动开始。""" if payload.tables and len(payload.tables) > catalog_profile_jobs.MAX_TABLES: raise HTTPException(422, f"一次最多剖析 {catalog_profile_jobs.MAX_TABLES} 张表，当前 {len(payload.tables)} 张，" "请分几次剖析") row, source = await _profile_source(session, source_id) tables = [str(t) for t in payload.tables] if payload.tables else None scope = "selected" if tables else (payload.scope or "needed") return await _start_profile(session, row, source, tables=tables, scope=scope, actor=actor_of(x_actor)) @router.get("/{source_id}/catalog/profile-jobs/active") async def active_profile_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """排队或进行中的那个剖析任务，没有是 null。""" row = await _source_row(session, source_id) return {"job": await catalog_profile_jobs.job_out(session, await catalog_profile_jobs.active_job(session, row.id))} @router.get("/{source_id}/catalog/profile-jobs/latest") async def latest_profile_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """最近建的一个剖析任务（任何状态），没有是 null。""" row = await _source_row(session, source_id) return {"job": await catalog_profile_jobs.job_out(session, await catalog_profile_jobs.latest_job(session, row.id))} async def _profile_job_or_404(session: AsyncSession, source_id: str, job_id: str) -> tuple[DataSource, Any]: row = await _source_row(session, source_id) job = await catalog_profile_jobs.load(session, row.id, job_id) if job is None: raise HTTPException(404, "剖析任务不存在，可能属于别的数据源，已随数据源删除，或者超过保留数被清理了") return row, job @router.get("/{source_id}/catalog/profile-jobs/{job_id}") async def get_profile_job(source_id: str, job_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: _, job = await _profile_job_or_404(session, source_id, job_id) return {"job": await catalog_profile_jobs.job_out(session, job)} @router.get("/{source_id}/catalog/profile-jobs/{job_id}/tasks/{seq}") async def get_profile_task(source_id: str, job_id: str, seq: int, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """剖析任务里的一张表（按剖析顺序的序号）：检查项、发现、跳过的项、日期范围、发过的查询（字面量已换成「?」）。""" _, job = await _profile_job_or_404(session, source_id, job_id) task = await catalog_profile_jobs.load_task(session, job, seq) if task is None: raise HTTPException(404, "这次剖析里没有这一张表") return {"task": catalog_profile_jobs.task_detail(job, task)} @router.post("/{source_id}/catalog/profile-jobs/{job_id}/cancel") async def cancel_profile_job(source_id: str, job_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """停止：排队的直接停止；进行中的不再等正在跑的那条查询（数据库按单条时限截止它），这张表做完的项照常写进目录、没做的 保留旧结论，后面的表记停止。已经结束的 409 catalog_profile_finished。返回停止之后的任务。""" row, job = await _profile_job_or_404(session, source_id, job_id) owner = row.id try: await catalog_profile_jobs.cancel(session, job) except catalog_profile_jobs.ProfileFinished as e: raise CodedHTTPException(409, str(e), CATALOG_PROFILE_FINISHED) from e session.expire_all() # 后台在别的会话里收的尾：重新读 return {"job": await catalog_profile_jobs.job_out(session, await catalog_profile_jobs.load(session, owner, job_id))} @router.post("/{source_id}/catalog/profile-jobs/{job_id}/retry", status_code=202) async def retry_profile_job(source_id: str, job_id: str, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """重试：新任务，只含原任务里 failed、partial、cancelled 的表（按原来的顺序），范围 selected，设置取数据源当前的。 同源有排队或进行中的剖析任务 409 profile_busy；没有可重试的表 422；开始不了的几种情况同建任务。""" _, old = await _profile_job_or_404(session, source_id, job_id) row, source = await _profile_source(session, source_id) try: await catalog_profile_jobs.ensure_free(session, row) except catalog_profile_jobs.ProfileBusy as e: raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e try: tables = await catalog_profile_jobs.retry_tables(session, old) except catalog_profile_jobs.NothingToRetry as e: raise HTTPException(422, str(e)) from e return await _start_profile(session, row, source, tables=tables, scope="selected", actor=actor_of(x_actor), retry_of=old.id) @router.get("/{source_id}/catalog/{table}") async def get_catalog_table(source_id: str, table: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: _, source = await _resolved(session, source_id) key, entry = await _table_key(session, source, table) return await _detail(session, source, key, entry) async def _source_row(session: AsyncSession, source_id: str) -> DataSource: row = await session.get(DataSource, source_id) if row is None: raise HTTPException(404, "数据源不存在，可能已被删除") return row @router.put("/{source_id}/catalog/guidance") async def put_catalog_guidance(source_id: str, body: dict[str, Any] = Body(...), x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> Any: """存数据源级的起草要求：去首尾空白，空串清除，最多 2000 字（超了 422）。上传表格的源存在它自己的数据源记录上。 要登记在 PUT /catalog/{table} 前面：不然「guidance」会被当成表名。反过来，表名恰好叫 guidance 时，整份提交那张表 目录的请求（{notes, if_version}，没有 guidance 这一项）也落到这里：照旧当作那张表的整份提交。""" if "guidance" not in body and "notes" in body: try: table_payload = CatalogPutIn.model_validate(body) except ValidationError as e: raise RequestValidationError(e.errors()) from e return await put_catalog_table(source_id, "guidance", table_payload, x_actor=x_actor, session=session) if "guidance" not in body: raise HTTPException(422, GUIDANCE_MISSING) guidance = body["guidance"] if guidance is not None and not isinstance(guidance, str): raise HTTPException(422, GUIDANCE_NOT_TEXT) row = await _source_row(session, source_id) try: value = await catalog_jobs.save_guidance(session, row, guidance) except catalog_jobs.TextTooLong as e: raise HTTPException(422, str(e)) from e return {"guidance": value} async def _ensure_model(session: AsyncSession) -> None: """起草模型用不了（没有接入、停用、只有演示接入……）：409，不建任务。说法同同步起草时的 model_error。""" try: await catalog_jobs.check_model(session) except catalog.CatalogModelUnavailable as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_MODEL_UNAVAILABLE) from e def _no_schema(cache: dict[str, Any]) -> HTTPException: return HTTPException(409, f"{introspect.why_empty(cache)}，无法起草数据目录。请先在数据源卡片上点「探查结构」") @router.post("/{source_id}/catalog/draft-jobs", status_code=202) async def create_draft_job(source_id: str, payload: DraftJobIn, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """建后台起草任务，马上返回（202，任务是排队中）。同一个源已有排队或进行中的任务 409 catalog_draft_busy（界面改去 GET /draft-jobs/active 显示那一个）；用模型而起草模型用不了 409 catalog_draft_model_unavailable；没有表结构 409。 表结构里没有的表照样建进任务、状态 skipped。""" if payload.tables and len(payload.tables) > catalog_jobs.MAX_TABLES: raise HTTPException(422, f"一次最多起草 {catalog_jobs.MAX_TABLES} 张表，当前 {len(payload.tables)} 张，" "请分几次起草") try: note = catalog_jobs.clean_text(payload.note, catalog_jobs.NOTE_MAX, "本次说明") except catalog_jobs.TextTooLong as e: raise HTTPException(422, str(e)) from e row, source = await _resolved(session, source_id) try: await catalog_jobs.ensure_draft_free(session, row) except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e cache = source.schema_cache or {} if not cache.get("tables"): raise _no_schema(cache) if payload.use_model: await _ensure_model(session) try: job, tasks = await catalog_jobs.create_job(session, row, source, tables=payload.tables or None, use_model=payload.use_model, note=note, actor=actor_of(x_actor)) except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e catalog_jobs.manager.start(job.id) return {"job": catalog_jobs.job_dict(job, tasks, waiting_for=await catalog_jobs.waiting_for(session, job))} @router.get("/{source_id}/catalog/draft-jobs/active") async def active_draft_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """排队或进行中的那个起草任务，没有是 null。""" row = await _source_row(session, source_id) return {"job": await catalog_jobs.job_out(session, await catalog_jobs.active_job(session, row.id))} @router.get("/{source_id}/catalog/draft-jobs/latest") async def latest_draft_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """最近建的一个起草任务（任何状态），没有是 null。""" row = await _source_row(session, source_id) return {"job": await catalog_jobs.job_out(session, await catalog_jobs.latest_job(session, row.id))} async def _job_or_404(session: AsyncSession, source_id: str, job_id: str) -> tuple[DataSource, Any]: row = await _source_row(session, source_id) job = await catalog_jobs.load(session, row.id, job_id) if job is None: raise HTTPException(404, "起草任务不存在，可能属于别的数据源，或者已随数据源删除") return row, job @router.get("/{source_id}/catalog/draft-jobs/{job_id}") async def get_draft_job(source_id: str, job_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: _, job = await _job_or_404(session, source_id, job_id) return {"job": await catalog_jobs.job_out(session, job)} @router.post("/{source_id}/catalog/draft-jobs/{job_id}/cancel") async def cancel_draft_job(source_id: str, job_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """停止：排队的直接停止；进行中的停掉正在进行的模型调用，没开始和没写完的表记停止，已经写进目录的不回滚。 已经结束的 409 catalog_draft_finished。返回停止之后的任务。""" row, job = await _job_or_404(session, source_id, job_id) owner = row.id try: await catalog_jobs.cancel(session, job) except catalog_jobs.DraftFinished as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_FINISHED) from e session.expire_all() # 后台在别的会话里收的尾：重新读 return {"job": await catalog_jobs.job_out(session, await catalog_jobs.load(session, owner, job_id))} @router.post("/{source_id}/catalog/draft-jobs/{job_id}/retry", status_code=202) async def retry_draft_job(source_id: str, job_id: str, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """重试：新任务，只含原任务里没写好（failed、partial）和停止了（cancelled）的表；用不用模型、本次说明沿用，起草要求 取数据源当前的。有排队或进行中的任务 409 catalog_draft_busy；没有可重试的表 422；起草模型用不了 409。""" _, old = await _job_or_404(session, source_id, job_id) row, source = await _resolved(session, source_id) try: await catalog_jobs.ensure_draft_free(session, row) except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e try: tables = await catalog_jobs.retry_tables(session, old) except catalog_jobs.NothingToRetry as e: raise HTTPException(422, str(e)) from e cache = source.schema_cache or {} if not cache.get("tables"): raise _no_schema(cache) if old.use_model: await _ensure_model(session) try: job, tasks = await catalog_jobs.create_job(session, row, source, tables=tables, use_model=old.use_model, note=old.note, actor=actor_of(x_actor), retry_of=old.id) except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e catalog_jobs.manager.start(job.id) return {"job": catalog_jobs.job_dict(job, tasks, waiting_for=await catalog_jobs.waiting_for(session, job))} @router.put("/{source_id}/catalog/{table}") async def put_catalog_table(source_id: str, table: str, payload: CatalogPutIn, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """整份提交一张表的目录。改动过的项记为人工确认；版本不符 409。""" _, source = await _resolved(session, source_id) key, _ = await _table_key(session, source, table) try: entry = await catalog.save_human_edit(session, source.id, key, payload.notes, if_version=payload.if_version, actor=actor_of(x_actor)) except catalog.CatalogConflict as e: raise _conflict(e) from e except catalog.CatalogInvalid as e: raise _invalid(e) from e return await _detail(session, source, key, entry) @router.post("/{source_id}/catalog/draft") async def draft_catalog(source_id: str, payload: CatalogDraftIn, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """同步起草（界面改用后台起草任务，这个接口留着，返回形状不变）。引擎同后台任务（分段、按失败原因处理），起草要求 取数据源级的那份。返回每张表新增、更新、删除了几项；模型用不了或某张表的模型起草失败，照实写在结果里。 model_items：模型给了这张表几项可用内容（没用模型、模型这部分失败时为 null）。界面据此区分「模型给了内容、 只是和现有目录一致」（model_items > 0、三个数都是 0）和「模型没给内容」（model_error 有值）。 双向互斥（catalog_jobs.sync_draft，进程内登记）：同一个源有排队或进行中的后台起草任务、或者正在同步起草时 409 catalog_draft_busy（和建任务同一句话）；反过来，同步起草进行期间同一个源建任务、重试、导入目录也是 409。两边同时写 同一个源的目录、抢模型名额。 """ row, source = await _resolved(session, source_id) cache = source.schema_cache or {} try: async with catalog_jobs.sync_draft(session, row): if not cache.get("tables"): raise _no_schema(cache) report = await catalog.draft_catalog(session, source, tables=payload.tables or None, use_model=payload.use_model, actor=actor_of(x_actor), guidance=catalog_jobs.guidance_of(row)) except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e except catalog_profile_jobs.ProfileBusy as e: raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e except introspect_jobs.IntrospectBusy as e: raise CodedHTTPException(409, str(e), INTROSPECT_BUSY) from e rows = [{"table_name": r.table, "added": r.added, "updated": r.updated, "removed": r.removed, "version": r.version, "error": r.error, "model_error": r.model_error, "model_items": r.model_items} for r in report.tables] return { "tables": rows, "total": {k: sum(r[k] for r in rows) for k in ("added", "updated", "removed")}, "model_used": report.model_used, "model": report.model or None, "model_error": report.model_error, } @router.post("/{source_id}/catalog/{table}/review") async def review_catalog_item(source_id: str, table: str, payload: CatalogReviewIn, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """单项审阅。版本不符 409；路径不对、项不存在 422。""" _, source = await _resolved(session, source_id) key, _ = await _table_key(session, source, table) try: entry = await catalog.review_entry(session, source.id, key, payload.path, payload.action, if_version=payload.if_version, actor=actor_of(x_actor)) except catalog.CatalogConflict as e: raise _conflict(e) from e except catalog.CatalogPathError as e: raise HTTPException(422, str(e)) from e return await _detail(session, source, key, entry) class CatalogBatchReviewItem(BaseModel): table_name: str = Field(min_length=1, max_length=255) path: str = Field(min_length=1, max_length=600) action: Literal["confirm", "reject", "reset"] if_version: int = Field(ge=0) class CatalogBatchReviewIn(BaseModel): items: list[CatalogBatchReviewItem] = Field(min_length=1, max_length=catalog_review.BATCH_MAX) @router.post("/{source_id}/catalog/review/batch") async def review_catalog_batch(source_id: str, payload: CatalogBatchReviewIn, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """批量审阅：逐条结果和请求一一对应。逐表按 if_version 乐观锁，同一张表的几条一次写入；一张表冲突、路径不对、表不存在 只记在那几条上（ok 为假，code 是 catalog_conflict / catalog_path_invalid / table_missing），不影响别的。""" _, source = await _resolved(session, source_id) items = [catalog_review.BatchItem(i.table_name, i.path, i.action, i.if_version) for i in payload.items] return {"results": await catalog_review.review_batch(session, source, items, actor=actor_of(x_actor))} @router.get("/{source_id}/catalog/{table}/usage") async def catalog_table_runs(source_id: str, table: str, limit: int = Query(20, ge=1, le=catalog_review.RUNS_LIMIT_MAX), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """用过这张表的运行，新的在前。按运行计：total 是几次运行，一次运行查了这张表几次都只算一次。和表清单的使用次数 （catalog.table_usage）数的是同一批查询快照，但使用次数按查询计（同一次运行里查了两次算两次），所以使用次数可能比 total 大。""" _, source = await _resolved(session, source_id) key, _ = await _table_key(session, source, table) return await catalog_review.table_runs(session, source, key, limit=limit) def _tables_of(source: Any) -> dict[str, Any] | None: """核对提案用的表结构；没有表结构（尚未探查）时不核对列名。""" return ((source.schema_cache or {}).get("tables") or None) if source is not None else None @router.post("/{source_id}/catalog/{table}/patch/preview") async def preview_catalog_patch(source_id: str, table: str, payload: CatalogPatchPreviewIn, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """目录修改提案的预览：对着当前目录逐项算改前、改后，不写库。 助手转出的提案卡片在保存遇到 409（别人刚改过这张表）时用它重新载入：改前的值、码值补充后的样子都按 最新的目录重算，人看过之后再带新版本保存。 """ _, source = await _resolved(session, source_id) key, entry = await _table_key(session, source, table) plan = catalog.plan_patch(entry.notes if entry else {}, [c.model_dump() for c in payload.changes], table=key, tables=_tables_of(source)) return {"table_name": key, "version": entry.version if entry else 0, "changes": [c.as_dict() for c in plan.changes], "problems": plan.problems} @router.post("/{source_id}/catalog/{table}/patch") async def save_catalog_patch(source_id: str, table: str, payload: CatalogPatchIn, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """保存一条目录修改提案：改动过的项记为人工填写、已确认，写了依据的项依据另存成这一项确认时的依据（reason，和备注 分开，规则见 catalog.apply_patch）。版本不符 409；有一项不合法 422，整条不写。""" _, source = await _resolved(session, source_id) key, _ = await _table_key(session, source, table) try: entry = await catalog.save_patch(session, source.id, key, [c.model_dump() for c in payload.changes], if_version=payload.if_version, actor=actor_of(x_actor), tables=_tables_of(source)) except catalog.CatalogConflict as e: raise _conflict(e) from e except catalog.CatalogInvalid as e: raise _invalid(e) from e return await _detail(session, source, key, entry) class AssistantMessageIn(BaseModel): role: Literal["user", "assistant"] content: str = Field(max_length=catalog_assistant.MESSAGE_MAX) class AssistantIn(BaseModel): #: 整段对话（服务端不存）；没有使用者的话时助手先开口 messages: list[AssistantMessageIn] = Field(default_factory=list, max_length=catalog_assistant.MAX_MESSAGES) #: 界面上还没保存的修改，让模型知道。不设上限：宽表上几轮攒下来常常超过一次保存的 200 项，不能因此不让问（背景里只列 #: 最近的 catalog_assistant.PENDING_SHOWN 项，码值照全部合并） pending: list[CatalogPatchChange] = Field(default_factory=list) @router.post("/{source_id}/catalog/{table}/assistant") async def catalog_table_assistant(source_id: str, table: str, payload: AssistantIn, session: AsyncSession = Depends(get_session)) -> Any: """表助手：和使用者对话完善这张表的目录，返回 text/event-stream（事件见 data/catalog_assistant.py）。提案只交给界面， 不写目录；保存走 /catalog/{table}/patch。开始前：表不在 404；对话不合规 422；起草模型用不了 409 catalog_draft_model_unavailable（说法同起草任务）。""" from fastapi.responses import StreamingResponse row, source = await _resolved(session, source_id) key = catalog.resolve_table_name(source.schema_cache, table) if key is None: raise await _missing(session, source, table) messages = [m.model_dump() for m in payload.messages] if problem := catalog_assistant.conversation_problem(messages): raise HTTPException(422, problem) try: model, model_id = await catalog_assistant.resolve_model(session) except catalog.CatalogModelUnavailable as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_MODEL_UNAVAILABLE) from e context = await catalog_assistant.prepare(session, row, source, key, pending=[c.model_dump() for c in payload.pending]) return StreamingResponse(catalog_assistant.stream(model, model_id, context, messages), media_type="text/event-stream", headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) @router.get("/{source_id}/catalog/{table}/impact") async def catalog_table_impact(source_id: str, table: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: """引用这张表的已发布、受管模板，每个模板按当前的已发布版本算：调用工具节点的 SQL 用到了这张表算直接引用， Agent 绑定了这个源的查询工具算可能涉及，合并查询按输入追溯。改目录之前、保存纠正之后都看它。""" row, source = await _resolved(session, source_id) key, _ = await _table_key(session, source, table) tables = _tables_of(source) templates = await catalog_impact.table_impact(session, row.name, key, list(tables) if tables else None) return {"table": key, "templates": templates}