diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 2d7da5b9..865929f0 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -8,12 +8,17 @@ - POST /catalog/{table}/patch 保存目录修改提案（乐观锁），记为人工确认 - GET /catalog/{table}/impact 影响面：引用这张表的已发布、受管模板（data/catalog_impact.py） - POST /catalog/draft 同步起草（注释、外键、命名推断，可选模型）。界面改用下面的后台起草任务，这个留着； - 和后台任务双向互斥：同一个源有排队或进行中的后台任务（或正在同步起草）时 409 - catalog_draft_busy，同步起草期间建任务、重试、导入也是 409 + 同一个源有排队或进行中的起草任务（或正在同步起草）时 409 catalog_draft_busy，有剖析任务时 + 409 profile_busy；同步起草期间建起草任务、重试、导入也是 409，新建的剖析任务排在它后面 - PUT /catalog/guidance 数据源级的起草要求（使用者补充的背景和要求，起草时交给模型） - POST /catalog/draft-jobs 建后台起草任务（202）；GET /draft-jobs/active、/latest、/{job_id} 读任务； - POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py） -- POST /catalog/profile 同步剖析：对业务库发少量只读查询，核对关系、取码值候选（data/catalog_profile.py） + POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py）。 + 同源有剖析任务时照建，排在它后面（waiting_for） +- POST /catalog/profile-jobs 建后台剖析任务（202）：对业务库逐表发少量只读查询，核对关系、取码值候选、提议业务日期 + （data/catalog_profile_jobs.py，引擎在 data/catalog_profile.py）；GET /profile-jobs/active、 + /latest、/{job_id} 读任务，/{job_id}/tasks/{seq} 读一张表的明细（检查项、发现、发过的查询）； + POST /profile-jobs/{job_id}/cancel 停止，/retry 只重试没做完的表。同源有起草任务时照建， + 排在它后面（waiting_for） - GET /catalog/review-queue 审阅队列：推断的项按使用次数、风险、来源排好；POST /catalog/review/batch 批量审阅， GET /catalog/review-progress 审阅进度，GET /catalog/{table}/usage 用过这张表的运行 （data/catalog_review.py） @@ -23,8 +28,9 @@ （data/catalog_assistant.py） 规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径 -不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。 -起草任务的 409 带机读码（api/coded.py）：同一个源已有排队或进行中的任务、起草模型用不了、要停止的任务已经结束。 +不对 422，源或表不存在 404。所有写操作记下署名（api/deps.actor_of：开了登录是认证过的用户名，否则请求头 X-Actor 的自报）。 +起草、剖析任务的 409 带机读码（api/coded.py）：同一个源已有排队或进行中的同种任务、起草模型用不了、剖析没开启、按范围 +挑不出表、要停止的任务已经结束。 上传表格的源按当前快照的表结构回答（table_versions.resolve_source），和查询实际用的库是同一个版本。 """ @@ -38,11 +44,12 @@ from pydantic import BaseModel, Field, ValidationError from sqlalchemy.ext.asyncio import AsyncSession from app.api.coded import ( - CATALOG_DRAFT_BUSY, CATALOG_DRAFT_FINISHED, CATALOG_DRAFT_MODEL_UNAVAILABLE, PROFILE_BUSY, SCHEMA_MISSING, - SNAPSHOT_TAMPERED, CodedHTTPException, + CATALOG_DRAFT_BUSY, CATALOG_DRAFT_FINISHED, CATALOG_DRAFT_MODEL_UNAVAILABLE, CATALOG_PROFILE_FINISHED, PROFILE_BUSY, + PROFILE_NOTHING, SCHEMA_MISSING, SNAPSHOT_TAMPERED, CodedHTTPException, ) -from app.api.runs import actor_of -from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, introspect, table_versions +from app.api.deps import actor_of +from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, catalog_profile_jobs, introspect +from app.data import table_versions from app.data import catalog_assistant, catalog_review, catalog_transfer from app.data.engine import SnapshotTampered from app.db.base import get_session @@ -102,9 +109,11 @@ GUIDANCE_MISSING = "请求里要写起草要求（guidance 一项，一段文字 GUIDANCE_NOT_TEXT = "起草要求要是一段文字（要清除起草要求请传空串）" -class CatalogProfileIn(BaseModel): - #: 要剖析的表；不给或为空时按使用次数取前 10 张目录里有待核实关系的表 - tables: list[str] | None = Field(default=None, max_length=200) +class ProfileJobIn(BaseModel): + #: 要剖析的表（范围 selected，按给的顺序去重）；最多 500 张（超了 422，说明是中文） + tables: list[str] | None = None + #: 不指定表时的范围：needed 需要剖析的表（缺省）、all 全部表（都不含视图、没有可剖析的项的表） + scope: Literal["needed", "all"] | None = None async def _resolved(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]: @@ -134,7 +143,8 @@ def _item_value(notes: dict[str, Any], name: str) -> tuple[Any, str | None]: def _summary_row(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogEntry | None, - usage: int, last_draft: dict[str, Any] | None = None) -> dict[str, Any]: + usage: int, last_draft: dict[str, Any] | None = None, + profiling: dict[str, Any] | None = None) -> dict[str, Any]: notes = entry.notes if entry else {} label, label_status = _item_value(notes, "label") kind, _ = _item_value(notes, "kind") @@ -157,9 +167,20 @@ def _summary_row(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogE # 这张表最近一次起草任务里的那一项：{status, job_id, finished_at, error, model_error, blank_columns}，没起草过 # 是 null "last_draft": last_draft, + # 剖析：最近一次剖析（CatalogLastProfile，没剖析过是 null）、要不要剖析和为什么、按当前目录和表结构要做几项检查 + **(profiling or {"last_profile": None, "profile_due": False, "profile_due_reason": None, "profile_checks": 0}), } +def _profiling(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogEntry | None, *, + tables: dict[str, Any], masked: set[str], last: dict[str, Any] | None) -> dict[str, Any]: + notes = entry.notes if entry else {} + checks = catalog_profile.profile_checks(name, meta, notes, tables=tables, masked=masked) + reason = catalog_profile_jobs.due_reason(meta, notes, last, checks) + return {"last_profile": catalog_profile_jobs.public_last(last), "profile_due": reason is not None, + "profile_due_reason": reason, "profile_checks": checks} + + def _structure(meta: dict[str, Any]) -> dict[str, Any]: return { "qualified": meta.get("qualified"), @@ -188,6 +209,9 @@ async def _detail(session: AsyncSession, source: Any, table: str, entry: catalog "structure": _structure(meta) if meta is not None else None, "system_notes": catalog.system_notes_source(source), "usage": usage, + # 这张表最近一次剖析（CatalogProfileTaskDetail：检查项、发现、跳过的项、日期范围、发过的查询）；任务还在进行就是 + # 进行中的状态；没剖析过是 null + "last_profile": await catalog_profile_jobs.last_detail(session, source.id, table), } @@ -223,11 +247,17 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi entries = await catalog.read_catalog(session, source.id) usage = await catalog.table_usage(session, source) drafts = await catalog_jobs.last_drafts(session, source.id) + lasts = await catalog_profile_jobs.last_profiles(session, source.id) + masked = catalog_profile.masked_of(row.options) + + def profiling(name: str, meta: dict[str, Any] | None) -> dict[str, Any]: + return _profiling(name, meta, entries.get(name), tables=tables, masked=masked, last=lasts.get(name)) + order = {name: i for i, name in enumerate(tables)} - rows = [_summary_row(name, meta, entries.get(name), usage.get(name, 0), drafts.get(name)) + rows = [_summary_row(name, meta, entries.get(name), usage.get(name, 0), drafts.get(name), profiling(name, meta)) for name, meta in tables.items()] rows.sort(key=lambda r: (-r["usage"], order[r["table_name"]])) - rows += [_summary_row(name, None, entry, 0, drafts.get(name)) + rows += [_summary_row(name, None, entry, 0, drafts.get(name), profiling(name, None)) for name, entry in sorted(entries.items()) if name not in tables] return { "tables": rows, @@ -238,8 +268,11 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi "schema_total": max(_int(cache.get("total")), len(tables)), # 数据源级的起草要求（没有是 null） "guidance": catalog_jobs.guidance_of(row), - # 排队或进行中的起草任务 {id, status, total, done}，界面据此显示进度；没有是 null + # 排队或进行中的起草任务 {id, status, total, done, waiting_for}，界面据此显示进度；没有是 null "active_job": await catalog_jobs.job_brief(session, await catalog_jobs.active_job(session, source.id)), + # 排队或进行中的剖析任务 {id, status, total, done, waiting_for}；没有是 null + "active_profile_job": await catalog_profile_jobs.job_brief( + session, await catalog_profile_jobs.active_job(session, source.id)), } @@ -313,22 +346,154 @@ async def preview_catalog_import(source_id: str, payload: CatalogImportIn, @router.post("/{source_id}/catalog/import") async def import_catalog(source_id: str, payload: CatalogImportIn, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - """导入：规则同预览，逐表写入（乐观锁），返回同样形状的报告和每张表写入后的版本。有排队或进行中的起草任务 - 409 catalog_draft_busy；文件格式不对 422；没有表结构 409。""" + """导入：规则同预览，逐表写入（乐观锁），返回同样形状的报告和每张表写入后的版本。有排队或进行中的起草任务（或正在 + 同步起草、导入）409 catalog_draft_busy，有剖析任务 409 profile_busy（说法的最后一句说「再导入」）；文件格式不对 422； + 没有表结构 409。""" row, source = await _import_source(session, source_id) try: - await catalog_jobs.ensure_idle(session, row) + async with catalog_jobs.import_write(session, row): + report = await catalog_transfer.run_import(session, row, source, payload.content, + include_guidance=payload.include_guidance, + actor=actor_of(x_actor)) except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e - try: - report = await catalog_transfer.run_import(session, row, source, payload.content, - include_guidance=payload.include_guidance, - actor=actor_of(x_actor)) + except catalog_profile_jobs.ProfileBusy as e: + raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e except catalog_transfer.TransferInvalid as e: raise HTTPException(422, str(e)) from e return report.as_dict() +# ---- 后台剖析任务（同样登记在 GET /catalog/{table} 前面：active、latest 要先于 /profile-jobs/{job_id}） + + +async def _profile_source(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]: + """建剖析任务、重试之前的几道门，409 都带机读码（界面按它分情况给下一步，不从原话里认）：未开启 profile_disabled、 + 数据源停用 datasource_inactive、快照登记被改动 snapshot_tampered、没有表结构 schema_missing。返回 (数据源记录, + 读表结构用的源)。""" + row = await _source_row(session, source_id) + try: + catalog_profile.ensure_enabled(row) + except catalog_profile.ProfileDisabled as e: + raise CodedHTTPException(409, str(e), e.code) from e + try: + source = await table_versions.resolve_source(session, row) + except SnapshotTampered as e: # 快照清单对不上（SnapshotManifestMismatch 同时是 SnapshotMissing） + raise CodedHTTPException(409, str(e), SNAPSHOT_TAMPERED) from e + except table_versions.SnapshotMissing as e: + raise HTTPException(409, str(e)) from e + cache = source.schema_cache or {} + if not cache.get("tables"): + raise CodedHTTPException(409, f"{introspect.why_empty(cache)}，无法进行数据剖析。请先在数据源卡片上点「探查结构」", + SCHEMA_MISSING) + return row, source + + +async def _start_profile(session: AsyncSession, row: DataSource, source: Any, *, tables: list[str] | None, + scope: str, actor: str | None, retry_of: str | None = None) -> dict[str, Any]: + try: + job, tasks = await catalog_profile_jobs.create_job( + session, row, source, tables=tables, scope=scope, settings=catalog_profile.ensure_enabled(row), + actor=actor, retry_of=retry_of) + except catalog_profile_jobs.ProfileBusy as e: + raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e + except catalog_profile_jobs.ProfileNothing as e: + raise CodedHTTPException(409, str(e), PROFILE_NOTHING) from e + catalog_profile_jobs.manager.start(job.id) + return {"job": catalog_profile_jobs.job_dict(job, tasks, + waiting_for=await catalog_profile_jobs.waiting_for(session, job))} + + +@router.post("/{source_id}/catalog/profile-jobs", status_code=202) +async def create_profile_job(source_id: str, payload: ProfileJobIn, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """建后台剖析任务，马上返回（202，任务是排队中）。tables 非空时范围是 selected（表结构里没有、视图、没有可剖析的项照样 + 建进任务、状态 skipped），否则按 scope（缺省 needed）挑。同源有排队或进行中的剖析任务 409 profile_busy（说清进行到第几张； + 界面改去 GET /profile-jobs/active 显示那一个），按范围挑出 0 张表 409 profile_nothing；同源有起草任务时照建，排在它后面 + （waiting_for = "draft"），起草结束后自动开始。""" + if payload.tables and len(payload.tables) > catalog_profile_jobs.MAX_TABLES: + raise HTTPException(422, f"一次最多剖析 {catalog_profile_jobs.MAX_TABLES} 张表，当前 {len(payload.tables)} 张，" + "请分几次剖析") + row, source = await _profile_source(session, source_id) + tables = [str(t) for t in payload.tables] if payload.tables else None + scope = "selected" if tables else (payload.scope or "needed") + return await _start_profile(session, row, source, tables=tables, scope=scope, actor=actor_of(x_actor)) + + +@router.get("/{source_id}/catalog/profile-jobs/active") +async def active_profile_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """排队或进行中的那个剖析任务，没有是 null。""" + row = await _source_row(session, source_id) + return {"job": await catalog_profile_jobs.job_out(session, await catalog_profile_jobs.active_job(session, row.id))} + + +@router.get("/{source_id}/catalog/profile-jobs/latest") +async def latest_profile_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """最近建的一个剖析任务（任何状态），没有是 null。""" + row = await _source_row(session, source_id) + return {"job": await catalog_profile_jobs.job_out(session, await catalog_profile_jobs.latest_job(session, row.id))} + + +async def _profile_job_or_404(session: AsyncSession, source_id: str, job_id: str) -> tuple[DataSource, Any]: + row = await _source_row(session, source_id) + job = await catalog_profile_jobs.load(session, row.id, job_id) + if job is None: + raise HTTPException(404, "剖析任务不存在，可能属于别的数据源，已随数据源删除，或者超过保留数被清理了") + return row, job + + +@router.get("/{source_id}/catalog/profile-jobs/{job_id}") +async def get_profile_job(source_id: str, job_id: str, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + _, job = await _profile_job_or_404(session, source_id, job_id) + return {"job": await catalog_profile_jobs.job_out(session, job)} + + +@router.get("/{source_id}/catalog/profile-jobs/{job_id}/tasks/{seq}") +async def get_profile_task(source_id: str, job_id: str, seq: int, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """剖析任务里的一张表（按剖析顺序的序号）：检查项、发现、跳过的项、日期范围、发过的查询（字面量已换成「?」）。""" + _, job = await _profile_job_or_404(session, source_id, job_id) + task = await catalog_profile_jobs.load_task(session, job, seq) + if task is None: + raise HTTPException(404, "这次剖析里没有这一张表") + return {"task": catalog_profile_jobs.task_detail(job, task)} + + +@router.post("/{source_id}/catalog/profile-jobs/{job_id}/cancel") +async def cancel_profile_job(source_id: str, job_id: str, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """停止：排队的直接停止；进行中的不再等正在跑的那条查询（数据库按单条时限截止它），这张表做完的项照常写进目录、没做的 + 保留旧结论，后面的表记停止。已经结束的 409 catalog_profile_finished。返回停止之后的任务。""" + row, job = await _profile_job_or_404(session, source_id, job_id) + owner = row.id + try: + await catalog_profile_jobs.cancel(session, job) + except catalog_profile_jobs.ProfileFinished as e: + raise CodedHTTPException(409, str(e), CATALOG_PROFILE_FINISHED) from e + session.expire_all() # 后台在别的会话里收的尾：重新读 + return {"job": await catalog_profile_jobs.job_out(session, await catalog_profile_jobs.load(session, owner, job_id))} + + +@router.post("/{source_id}/catalog/profile-jobs/{job_id}/retry", status_code=202) +async def retry_profile_job(source_id: str, job_id: str, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """重试：新任务，只含原任务里 failed、partial、cancelled 的表（按原来的顺序），范围 selected，设置取数据源当前的。 + 同源有排队或进行中的剖析任务 409 profile_busy；没有可重试的表 422；开始不了的几种情况同建任务。""" + _, old = await _profile_job_or_404(session, source_id, job_id) + row, source = await _profile_source(session, source_id) + try: + await catalog_profile_jobs.ensure_free(session, row) + except catalog_profile_jobs.ProfileBusy as e: + raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e + try: + tables = await catalog_profile_jobs.retry_tables(session, old) + except catalog_profile_jobs.NothingToRetry as e: + raise HTTPException(422, str(e)) from e + return await _start_profile(session, row, source, tables=tables, scope="selected", actor=actor_of(x_actor), + retry_of=old.id) + + @router.get("/{source_id}/catalog/{table}") async def get_catalog_table(source_id: str, table: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: @@ -398,7 +563,7 @@ async def create_draft_job(source_id: str, payload: DraftJobIn, x_actor: str | N raise HTTPException(422, str(e)) from e row, source = await _resolved(session, source_id) try: - await catalog_jobs.ensure_idle(session, row) + await catalog_jobs.ensure_draft_free(session, row) except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e cache = source.schema_cache or {} @@ -412,7 +577,7 @@ async def create_draft_job(source_id: str, payload: DraftJobIn, x_actor: str | N except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e catalog_jobs.manager.start(job.id) - return {"job": catalog_jobs.job_dict(job, tasks)} + return {"job": catalog_jobs.job_dict(job, tasks, waiting_for=await catalog_jobs.waiting_for(session, job))} @router.get("/{source_id}/catalog/draft-jobs/active") @@ -466,7 +631,7 @@ async def retry_draft_job(source_id: str, job_id: str, x_actor: str | None = Hea _, old = await _job_or_404(session, source_id, job_id) row, source = await _resolved(session, source_id) try: - await catalog_jobs.ensure_idle(session, row) + await catalog_jobs.ensure_draft_free(session, row) except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e try: @@ -484,7 +649,7 @@ async def retry_draft_job(source_id: str, job_id: str, x_actor: str | None = Hea except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e catalog_jobs.manager.start(job.id) - return {"job": catalog_jobs.job_dict(job, tasks)} + return {"job": catalog_jobs.job_dict(job, tasks, waiting_for=await catalog_jobs.waiting_for(session, job))} @router.put("/{source_id}/catalog/{table}") @@ -529,6 +694,8 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn, guidance=catalog_jobs.guidance_of(row)) except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e + except catalog_profile_jobs.ProfileBusy as e: + raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e rows = [{"table_name": r.table, "added": r.added, "updated": r.updated, "removed": r.removed, "version": r.version, "error": r.error, "model_error": r.model_error, "model_items": r.model_items} for r in report.tables] @@ -541,71 +708,6 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn, } -def _profile_table_out(t: catalog_profile.TableProfile) -> dict[str, Any]: - return { - "table_name": t.table, - "queries": t.queries, - "row_estimate": t.row_estimate.to_dict() if t.row_estimate else None, - "findings": t.findings, - "skipped": [s.to_dict() for s in t.skipped], - "date_ranges": t.date_ranges, - "added": t.added, - "updated": t.updated, - "removed": t.removed, - "version": t.version, - "error": t.error, - } - - -@router.post("/{source_id}/catalog/profile") -async def profile_catalog_tables(source_id: str, payload: CatalogProfileIn, - x_actor: str | None = Header(default=None), - session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - """同步剖析。返回每张表用了几条查询、写进目录的发现、跳过了什么及原因。 - - 默认关闭：数据源设置里没开启数据剖析、数据源停用，409；同一个数据源已经在剖析，409。 - """ - row, source = await _resolved(session, source_id) - # 409 都带机读码（api/coded.py），界面按它分五种情况给下一步，不从原话里认 - try: - catalog_profile.ensure_enabled(row) - except catalog_profile.ProfileDisabled as e: - raise CodedHTTPException(409, str(e), e.code) from e - cache = source.schema_cache or {} - if not cache.get("tables"): - raise CodedHTTPException(409, f"{introspect.why_empty(cache)}，无法进行数据剖析。请先在数据源卡片上点「探查结构」", - SCHEMA_MISSING) - actor = actor_of(x_actor) - try: - report = await catalog_profile.profile_catalog(session, row, source, tables=payload.tables or None, - actor=actor) - except catalog_profile.ProfileDisabled as e: - raise CodedHTTPException(409, str(e), e.code) from e - except catalog_profile.ProfileBusy as e: - raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e - except SnapshotTampered as e: - raise CodedHTTPException(409, str(e), SNAPSHOT_TAMPERED) from e - rows = [_profile_table_out(t) for t in report.tables] - note = None - if not rows and report.empty_tables: - note = (f"有待核对关联关系的 {len(report.empty_tables)} 张表都是空表，没有可以核实的数据。" - "请在数据有了之后再剖析，或指定要剖析的表") - elif not rows: - note = "目录里还没有待核实的关联关系，请先起草数据目录，或指定要剖析的表" - return { - "profiled_at": report.profiled_at, - "actor": actor, - "settings": report.settings.to_dict(), - "queries_used": report.queries_used, - "stopped": report.stopped, - "tables": rows, - "total": {k: sum(r[k] for r in rows) for k in ("added", "updated", "removed")}, - # 不指定表时挑表途中确知是空表、跳过了的表（指定了表时总是空列表） - "empty_tables": report.empty_tables, - "note": note, - } - - @router.post("/{source_id}/catalog/{table}/review") async def review_catalog_item(source_id: str, table: str, payload: CatalogReviewIn, x_actor: str | None = Header(default=None), diff --git a/backend/app/api/coded.py b/backend/app/api/coded.py index 204b94af..d8fdba23 100644 --- a/backend/app/api/coded.py +++ b/backend/app/api/coded.py @@ -11,14 +11,18 @@ from fastapi.responses import JSONResponse #: 限定的数据源一个都不在了（删了或停用了），见 copilot._sources DATASOURCE_SCOPE_EMPTY = "datasource_scope_empty" -#: 保存数据源时剖析设置不合规（422）：field 是出错的那一项（max_queries……），说不清是哪一项时没有 field +#: 保存数据源时剖析设置不合规（422）：field 是出错的那一项（table_max_queries……），说不清是哪一项时没有 field PROFILE_SETTINGS_INVALID = "profile_settings_invalid" -#: 剖析开始不了（409）的五种情况，见 api/catalog.profile_catalog_tables +#: 剖析任务开始不了（409，api/catalog.create_profile_job）：未开启、数据源停用、没有表结构、快照被改动、这个源已有排队或 +#: 进行中的剖析任务（导入目录、同步起草遇到剖析任务也是它）、按范围挑出 0 张表 PROFILE_DISABLED = "profile_disabled" DATASOURCE_INACTIVE = "datasource_inactive" SCHEMA_MISSING = "schema_missing" PROFILE_BUSY = "profile_busy" SNAPSHOT_TAMPERED = "snapshot_tampered" +PROFILE_NOTHING = "profile_nothing" +#: 要停止的剖析任务已经结束了（409） +CATALOG_PROFILE_FINISHED = "catalog_profile_finished" #: 数据目录的后台起草任务（api/catalog.py 的 /catalog/draft-jobs）：同一个源已有排队或进行中的任务（409，界面改去 #: GET /draft-jobs/active 显示那一个）、起草模型用不了（409，不建任务）、要停止的任务已经结束了（409） CATALOG_DRAFT_BUSY = "catalog_draft_busy" diff --git a/backend/app/api/datasources.py b/backend/app/api/datasources.py index 2680e59b..38bebb9b 100644 --- a/backend/app/api/datasources.py +++ b/backend/app/api/datasources.py @@ -23,7 +23,7 @@ from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw from app.core.config import settings from app.core.crypto import encrypt, mask from app.data import catalog as data_catalog -from app.data.catalog_profile import profile_settings_rejection +from app.data.catalog_profile import profile_settings_rejection, without_legacy from app.data import introspect as introspect_mod from app.data import table_versions from app.data.engine import ( @@ -465,12 +465,13 @@ async def create_source( )).scalar_one_or_none() if exists: raise HTTPException(409, f"已存在名为「{payload.name}」的数据源，请换一个标识") - _refuse_bad_options(payload.options) + options = without_legacy(payload.options) or {} + _refuse_bad_options(options) _refuse_reserved(payload.kind, payload.database) row = DataSource( **payload.model_dump(exclude={"password", "options"}), - options=_with_managed(payload.options, None), + options=_with_managed(options, None), password=encrypt(payload.password), ) _settle_oracle(row, None, None) @@ -519,6 +520,8 @@ async def update_source( # 升级前就指向上传目录的手工源：启动时已强制只读（table_versions.startup），不许再改回可写 raise HTTPException(422, _UPLOAD_DIR_READONLY) if "options" in data: + # 剖析设置里还带着以前按整次剖析算的预算（老页面提交的）：只摘掉，不报错（catalog_profile.LEGACY_KEYS） + data["options"] = without_legacy(data["options"]) _refuse_bad_options(data["options"]) data["options"] = _with_managed(data["options"] or {}, row.options) if "password" in data: diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 9c674cb4..6ca00287 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -421,9 +421,12 @@ async def read_catalog(session: AsyncSession, source_id: str) -> dict[str, Catal async def write_entry(session: AsyncSession, source_id: str, table: str, notes: dict[str, Any], *, - if_version: int, actor: str | None) -> CatalogEntry: + if_version: int, actor: str | None, commit: bool = True) -> CatalogEntry: """整份写入一张表的目录（乐观锁），提交事务，返回写入后的目录。 + commit=False 只写进当前事务、不提交，由调用方和它自己的写入一起提交：剖析一张表结束时，目录和这张表的剖析结果要么 + 都写进去、要么都没写（进程在两次提交之间没了，剖析结果会说「没写进目录」，目录里其实已经写了）。冲突时照样回滚。 + - if_version 是写入方读到的版本，还没有目录时传 0；和库里的不一致抛 CatalogConflict，什么都不写。 比对和写入在同一条 UPDATE … WHERE version = ? 里完成：两个请求同时读到第 3 版，只有一个写得进去。 - 内容和库里的完全相同时不写、不升版本：冻结进表结构快照的是「版本 + 内容」，空写一次也升版本， @@ -446,7 +449,7 @@ async def write_entry(session: AsyncSession, source_id: str, table: str, notes: row = CatalogNote(source_id=source_id, table_name=table, notes=stored, version=1, updated_by=actor) session.add(row) try: - await session.commit() + await (session.commit() if commit else session.flush()) except IntegrityError as e: # 唯一约束 (source_id, table_name)：另一个请求抢先建了这张表的目录 await session.rollback() @@ -461,7 +464,8 @@ async def write_entry(session: AsyncSession, source_id: str, table: str, notes: if result.rowcount != 1: await session.rollback() raise CatalogConflict(table, current + 1) - await session.commit() + if commit: + await session.commit() await session.refresh(row) return _entry(row) diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py index 5d537079..56f2160b 100644 --- a/backend/app/data/catalog_assistant.py +++ b/backend/app/data/catalog_assistant.py @@ -15,7 +15,7 @@ 是这个值又确认过的不提；有效记录条件是模型写的 SQL，另按起草的规矩核对（_Tools.filter_problem：一个条件、只用这张表的 列、不能有子查询……）。核对过的作为 proposal 事件交给界面，**不写目录**。没收下的项和原因交回模型（不带任何取值）。 界面上还没保存的修改（pending）里同一列的码值并进新提的码值：界面上同一路径以后一次为准，不能把先提的码值丢了。 -- run_query：只读查询，只有数据源开了数据剖析才给。经剖析的只读守卫和单条时限（catalog_profile._Runner： +- run_query：只读查询，只有数据源开了数据剖析才给。经剖析的单条查询入口（catalog_profile.run_single_query： guard.check(readonly=True)，单条时限取剖析设置和数据源查询时限中较小的一个），每次请求最多 MAX_QUERIES 条（守卫拒掉的 也算），最多 QUERY_ROWS 行交给模型、SHOWN_ROWS 行交给界面。结果交给模型、界面之前先遮（engine/masking.mask_query_result： 取自遮罩列的结果列整列换成「已遮罩」，改了名字的也认；这一次请求里前面遮掉的值按值换），报错原话里的值也换。没开剖析时 @@ -52,7 +52,6 @@ from __future__ import annotations import asyncio import contextlib -import dataclasses import json import logging from collections.abc import AsyncIterator, Iterable, Mapping @@ -621,15 +620,11 @@ class _Tools: known = known if known.strong else None error: str | None = None result: Any = None - settings = dataclasses.replace(self.ctx.settings, max_queries=1) try: - runner = catalog_profile._Runner(self.ctx.query_source, settings, - catalog_profile.SqlDialect(self.ctx.kind)) - result = await runner(sql, max_rows=QUERY_ROWS) - except catalog_profile._Skip as e: + result = await catalog_profile.run_single_query(self.ctx.query_source, sql, settings=self.ctx.settings, + max_rows=QUERY_ROWS, kind=self.ctx.kind) + except catalog_profile.QueryFailed as e: error = e.detail - except catalog_profile._Stop as e: - error = str(e) except SnapshotTampered as e: error = QUERY_TAMPERED.format(name=self.ctx.source_name, error=e) except ValueError as e: # 不支持的数据库类型 diff --git a/backend/app/data/catalog_job_runtime.py b/backend/app/data/catalog_job_runtime.py new file mode 100644 index 00000000..970159a4 --- /dev/null +++ b/backend/app/data/catalog_job_runtime.py @@ -0,0 +1,400 @@ +"""数据目录后台任务的共用运行时：起草（data/catalog_jobs.py）和剖析（data/catalog_profile_jobs.py）各自建表、各自的步骤， +生命周期是同一套。 + +**为什么抽出来。** 剖析以前是一次同步长请求，和起草改成后台任务之前是同一类问题：关掉页面结果就没了、不能停、不留痕。 +两边要的是同一样东西——进程内跑、能停、重启标中断、写库不被停止打断、同源排队——各写一份，迟早一边修了另一边没修。 +共用的是生命周期代码，不是表：两种任务的进度完全不同（起草按段、块、模型用量，剖析按检查项、查询），合成一张 +「kind + JSON」的通用任务表要丢掉起草那些强类型列，还要迁移刚稳定的起草数据。 + +- **状态词**（ACTIVE、FINISHED、TASK_STATUSES、TASK_DONE、RETRYABLE、OPEN）：任务 queued → running → succeeded / partial + / failed / cancelled / interrupted；逐表 pending → running → ok / partial / failed / skipped / cancelled。两种任务、 + 接口、界面都用这一套词。 +- **SerialWriter**：同一个任务的库写入一条一条来，shield 起来——停止任务时不会打断写到一半的库操作；收尾前 settle 等它们 + 写完。 +- **JobRun / JobManager**：进程内一个 asyncio 任务一件事（照 app/scheduler 的写法）：start / stop(grace) / wait / running + / shutdown。JobRun.run 把「进队、等轮到自己、执行、按停止的原因收尾、出队」写一次，各种任务只写执行和收尾的那几步。 +- **SourceLanes**：每个数据源一条先来先到的队。起草、剖析都打业务库或模型、都写同一个源的目录：同源串行（先建的先跑）， + 不同源并行。任务阻塞地占：建出来是 queued，轮到了才 running。同步起草、导入目录这类同步写操作不阻塞地试占：占不到 + 说明源上正有别的事，接口回 409；占到了，执行期间新建的任务排在它后面。 +- **close_job / interrupt_on_startup**：收尾（还没结束的表记成某个状态，任务记成某个状态），和服务启动时的整理：排队、 + 进行中的标 interrupted，没结束的表记 cancelled。**不自动续跑**：复制出来的数据目录（检查脚本、验收副本）启动时不能 + 自己去调模型、查业务库，界面让人点「重试」接着做。 + +进程内登记、进程内的队：服务端是单进程部署，多进程部署时要换成库里的锁。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import logging +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from typing import Any + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.base import SessionLocal, utcnow + +logger = logging.getLogger(__name__) + +#: 任务还没结束的状态（同一个源同一种任务只能有一个）、结束了的状态 +ACTIVE = ("queued", "running") +FINISHED = ("succeeded", "partial", "failed", "cancelled", "interrupted") +#: 逐表的状态，其中算「已结束」的、重试时要带上的、还没结束的 +TASK_STATUSES = ("pending", "running", "ok", "partial", "failed", "skipped", "cancelled") +TASK_DONE = ("ok", "partial", "failed", "skipped", "cancelled") +RETRYABLE = ("failed", "partial", "cancelled") +OPEN = ("pending", "running") + +#: 停止进行中的任务时最多等它收尾多久（秒）；关停服务时另给（shutdown 的 grace） +STOP_GRACE_S = 10.0 +SHUTDOWN_GRACE_S = 3.0 + + +class JobFailed(Exception): + """任务整体出错（数据源没了、快照没了、模型用不了……）：message 就是任务的 error。""" + + +# ========================================================================== +# 库写入 +# ========================================================================== + + +class SerialWriter: + """一个任务的库写入：一条一条来，每条开一个会话；shield 起来，停止任务时不会打断写到一半的库操作。""" + + def __init__(self) -> None: + self._writes: set[asyncio.Task[Any]] = set() + self._lock = asyncio.Lock() + + async def __call__(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: + async def go() -> Any: + async with self._lock: + async with SessionLocal() as session: + return await fn(session, *args) + + task = asyncio.ensure_future(go()) + self._writes.add(task) + task.add_done_callback(self._writes.discard) + return await asyncio.shield(task) + + async def settle(self) -> None: + """等手上的库写入都写完（停止、出错收尾之前）。""" + while self._writes: + await asyncio.gather(*list(self._writes), return_exceptions=True) + + +# ========================================================================== +# 每个源一条队 +# ========================================================================== + + +class LaneTicket: + """在一个源的队里的一个位置。kind 是占着它的是什么：draft 起草任务、profile 剖析任务、sync_draft 同步起草、 + import 导入目录；job_id 是任务的（同步写操作没有）。轮到它时 granted 置位。""" + + def __init__(self, lanes: SourceLanes, source_id: str, kind: str, job_id: str | None, + loop: asyncio.AbstractEventLoop) -> None: + self._lanes = lanes + self.source_id = source_id + self.kind = kind + self.job_id = job_id + self.loop = loop + self.granted = asyncio.Event() + self.released = False + + async def wait(self) -> None: + """等轮到自己。""" + await self.granted.wait() + + def release(self) -> None: + """出队（轮到了、没轮到都行，重复调用无害）：排在后面的第一个接着开始。""" + self._lanes._release(self) + + +class LaneBusy(Exception): + """不阻塞地试占时这个源的队里已经有人：holder 是排在最前面（正在做）的那一个。""" + + def __init__(self, holder: LaneTicket) -> None: + super().__init__(holder.kind) + self.holder = holder + + +class SourceLanes: + """每个数据源一条先来先到的队。""" + + def __init__(self) -> None: + self._queues: dict[str, list[LaneTicket]] = {} + + def _queue(self, source_id: str) -> list[LaneTicket]: + """这个源的队。别的事件循环留下的（测试每个用例一个循环）去掉：那个循环里没人会再出队了。""" + queue = self._queues.get(source_id) or [] + loop = asyncio.get_running_loop() + alive = [t for t in queue if t.loop is loop and not t.released] + if alive: + self._queues[source_id] = alive + alive[0].granted.set() + else: + self._queues.pop(source_id, None) + return alive + + def enter(self, source_id: str, kind: str, job_id: str | None = None) -> LaneTicket: + """排进队尾，马上返回（不等）。排在最前面的立刻 granted；调用方 await ticket.wait() 等轮到自己。""" + queue = self._queue(source_id) + ticket = LaneTicket(self, source_id, kind, job_id, asyncio.get_running_loop()) + queue.append(ticket) + self._queues[source_id] = queue + if len(queue) == 1: + ticket.granted.set() + return ticket + + def try_enter(self, source_id: str, kind: str) -> LaneTicket: + """队空着才占（马上就轮到）；有人在队里抛 LaneBusy。""" + queue = self._queue(source_id) + if queue: + raise LaneBusy(queue[0]) + return self.enter(source_id, kind) + + def holders(self, source_id: str) -> list[LaneTicket]: + """队里的各位，排在最前面（正在做）的在前。""" + return list(self._queue(source_id)) + + def ahead(self, source_id: str, job_id: str) -> list[str] | None: + """排在这个任务前面的各位是什么（kind，正在做的在前）；这个任务不在队里（刚建、后台还没排进来）返回 None。""" + queue = self._queue(source_id) + for i, ticket in enumerate(queue): + if ticket.job_id == job_id: + return [t.kind for t in queue[:i]] + return None + + def _release(self, ticket: LaneTicket) -> None: + if ticket.released: + return + ticket.released = True + queue = self._queues.get(ticket.source_id) + if not queue or ticket not in queue: + return + head = queue[0] is ticket + queue.remove(ticket) + if not queue: + self._queues.pop(ticket.source_id, None) + elif head: + queue[0].granted.set() + + @contextlib.asynccontextmanager + async def hold(self, source_id: str, kind: str, job_id: str | None = None) -> AsyncIterator[LaneTicket]: + """阻塞地占：排队，轮到了再进去，出来时出队。""" + ticket = self.enter(source_id, kind, job_id) + try: + await ticket.wait() + yield ticket + finally: + ticket.release() + + @contextlib.contextmanager + def try_hold(self, source_id: str, kind: str) -> Iterator[LaneTicket]: + """不阻塞地试占（同步起草、导入目录）：占不到抛 LaneBusy；占到了，执行期间新建的任务排在后面。""" + ticket = self.try_enter(source_id, kind) + try: + yield ticket + finally: + ticket.release() + + +lanes = SourceLanes() + + +# ========================================================================== +# 收尾、启动时的整理 +# ========================================================================== + + +async def close_job(session: AsyncSession, job_model: Any, task_model: Any, job_id: str, job_status: str, + job_error: str | None, task_status: str, task_error: str | None) -> None: + """收尾：还没结束的表记 task_status（原因 task_error），任务（还没结束的话）记 job_status。提交。""" + now = utcnow() + await session.execute( + update(task_model).where(task_model.job_id == job_id, task_model.status.in_(OPEN)) + .values(status=task_status, error=task_error, finished_at=now).execution_options(synchronize_session=False)) + await session.execute( + update(job_model).where(job_model.id == job_id, job_model.status.in_(ACTIVE)) + .values(status=job_status, error=job_error, finished_at=now).execution_options(synchronize_session=False)) + await session.commit() + + +async def cancel_queued(session: AsyncSession, job_model: Any, task_model: Any, job_id: str) -> bool: + """停止一个排队中的任务：任务和没结束的表直接记 cancelled。已经被后台认领（开始了）返回 False，交给调用方按进行中 + 的停。""" + now = utcnow() + claimed = await session.execute( + update(job_model).where(job_model.id == job_id, job_model.status == "queued") + .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) + if claimed.rowcount != 1: + await session.rollback() + return False + await session.execute( + update(task_model).where(task_model.job_id == job_id, task_model.status.in_(OPEN)) + .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) + await session.commit() + return True + + +async def claim(session: AsyncSession, job_model: Any, job_id: str) -> bool: + """认领：排队中 → 进行中。开始之前就被停止了返回 False。""" + result = await session.execute( + update(job_model).where(job_model.id == job_id, job_model.status == "queued") + .values(status="running", started_at=utcnow()).execution_options(synchronize_session=False)) + await session.commit() + return result.rowcount == 1 + + +async def interrupt_on_startup(job_model: Any, close: Callable[[AsyncSession, str], Awaitable[None]], *, + running: Callable[[str], bool]) -> int: + """服务启动时：排队、进行中、而且不在这个进程里跑的任务逐个 close（各自记成 interrupted、没结束的表记 cancelled）。 + 不自动续跑。返回整理了几个。""" + async with SessionLocal() as session: + ids = [job_id for job_id in (await session.execute( + select(job_model.id).where(job_model.status.in_(ACTIVE)))).scalars() if not running(job_id)] + for job_id in ids: + await close(session, job_id) + return len(ids) + + +# ========================================================================== +# 在后台执行 +# ========================================================================== + + +class JobRun: + """一个任务在后台的一次执行。子类给 kind（进队时的身份）、model（任务表，进队前读源和状态），写 _run（执行）和 + _stopped / _failed / _crashed（停止、整体出错、意外出错时的收尾）。""" + + kind = "" + model: Any = None + + def __init__(self, job_id: str) -> None: + self.job_id = job_id + self.task: asyncio.Task[None] | None = None + #: 为什么停：user 使用者点了停止，shutdown 服务关停（标中断） + self.stop_reason: str | None = None + self.writer = SerialWriter() + self.ticket: LaneTicket | None = None + + async def _write(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: + """一条库写入（SerialWriter：一条一条来，停止时不被打断）。""" + return await self.writer(fn, *args) + + async def _settle(self) -> None: + await self.writer.settle() + + async def run(self) -> None: + try: + try: + await self._queue() + await self._run() + except asyncio.CancelledError: + current = asyncio.current_task() + if current is not None: + current.uncancel() + await self._stopped() + except JobFailed as e: + await self._failed(str(e)) + except Exception as e: # noqa: BLE001 - 任务整体出错：照实记下，不留在「进行中」 + await self._crashed(e) + finally: + # 收尾写完才出队：排在后面的任务开始时，这个任务已经记成结束了 + if self.ticket is not None: + self.ticket.release() + + async def _queue(self) -> None: + """进这个源的队，等轮到自己。任务不在了、已经不是排队中（开始之前就被停止了）的不进队：_run 认领不到，直接结束。""" + async with SessionLocal() as session: + job = await session.get(self.model, self.job_id) + source_id = job.source_id if job is not None and job.status == "queued" else None + if source_id is None: + return + self.ticket = lanes.enter(source_id, self.kind, self.job_id) + await self.ticket.wait() + + async def _run(self) -> None: + raise NotImplementedError + + async def _stopped(self) -> None: + raise NotImplementedError + + async def _failed(self, reason: str) -> None: + raise NotImplementedError + + async def _crashed(self, e: Exception) -> None: + raise NotImplementedError + + +class JobManager: + """进程内的一种后台任务：一个 asyncio 任务一件事，停的时候给宽限。factory 按任务编号造一次执行（JobRun）；label 是 + 日志里的叫法（「数据目录起草任务」），name 是 asyncio 任务名的前缀。""" + + def __init__(self, factory: Callable[[str], JobRun], *, label: str, name: str) -> None: + self._runs: dict[str, JobRun] = {} + self._factory = factory + self.label = label + self.name = name + + def _alive(self, job_id: str) -> JobRun | None: + run = self._runs.get(job_id) + if run is None or run.task is None or run.task.done(): + return None + try: + if run.task.get_loop() is not asyncio.get_running_loop(): + self._runs.pop(job_id, None) # 别的事件循环留下的（测试每个用例一个循环） + return None + except RuntimeError: + return None + return run + + def start(self, job_id: str) -> None: + """开始在后台执行这个（排队中的）任务：先进这个源的队，轮到了才开始。""" + run = self._factory(job_id) + run.task = asyncio.get_running_loop().create_task(run.run(), name=f"{self.name}-{job_id}") + self._runs[job_id] = run + + def forget(_: asyncio.Task[None]) -> None: + if self._runs.get(job_id) is run: + self._runs.pop(job_id, None) + + run.task.add_done_callback(forget) + + def running(self, job_id: str) -> bool: + return self._alive(job_id) is not None + + async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S) -> bool: + """停掉这个进程里在跑（或在排队等）的任务，等它收尾（最多 grace 秒）。不在这个进程里返回 False。""" + run = self._alive(job_id) + if run is None: + return False + if run.stop_reason is None: + run.stop_reason = reason + run.task.cancel() # type: ignore[union-attr] + await asyncio.wait([run.task], timeout=grace) # type: ignore[list-item] + return True + + async def wait(self, job_id: str, timeout: float | None = None) -> None: + """等这个任务做完（测试、调试用）。不在跑直接返回。""" + run = self._alive(job_id) + if run is not None: + await asyncio.wait([run.task], timeout=timeout) # type: ignore[list-item] + + async def shutdown(self, *, grace: float = SHUTDOWN_GRACE_S) -> None: + """关停：停掉进行中、排队等着的任务（标中断），最多等 grace 秒收尾。""" + runs = [run for job_id in list(self._runs) if (run := self._alive(job_id)) is not None] + for run in runs: + if run.stop_reason is None: + run.stop_reason = "shutdown" + run.task.cancel() # type: ignore[union-attr] + if runs: + await asyncio.wait([run.task for run in runs], timeout=grace) # type: ignore[misc] + logger.info("已停止 %d 个%s", len(runs), self.label) + + +__all__ = ["ACTIVE", "FINISHED", "JobFailed", "JobManager", "JobRun", "LaneBusy", "LaneTicket", "OPEN", "RETRYABLE", + "SHUTDOWN_GRACE_S", "STOP_GRACE_S", "SerialWriter", "SourceLanes", "TASK_DONE", "TASK_STATUSES", "cancel_queued", + "claim", "close_job", "interrupt_on_startup", "lanes"] diff --git a/backend/app/data/catalog_jobs.py b/backend/app/data/catalog_jobs.py index d63c0313..233338ad 100644 --- a/backend/app/data/catalog_jobs.py +++ b/backend/app/data/catalog_jobs.py @@ -13,15 +13,20 @@ 接口读的是库里的状态。 - **停止**（cancel）：排队的直接 cancelled；进行中的停掉正在进行的模型调用，没开始和没写完的表记 cancelled，已经写进 目录的（含宽表已经写出来的段）不回滚，任务最后是 cancelled。写过段的表 model_error 说清写进了几段、剩几段没起草。 -- **和同步起草互斥**（sync_draft）：同步起草（POST /catalog/draft）进行期间，同一个源建任务、重试、导入（ensure_idle） - 都是 DraftBusy；有排队或进行中的任务时同步起草也是。进程内登记（单进程部署）。 +- **同种任务互斥**（ensure_draft_free）：同一个源同时只有一个排队或进行中的起草任务，同步起草（POST /catalog/draft） + 进行期间也算在起草，建任务、重试都是 DraftBusy。剖析任务不挡起草：同源有剖析任务时起草任务照建，排在它后面 + （waiting_for = "profile"），剖析结束后自动开始（每个源一条队，见 data/catalog_job_runtime.py）。 +- **同步写操作要求源空闲**（ensure_idle；sync_draft、import_write 占队）：同步起草、导入目录要的是这个源上没有别的事——有排队或进行中的 + 起草任务（DraftBusy）、剖析任务（catalog_profile_jobs.ProfileBusy）都不开始；开始了就占着这个源的队，执行期间 + 新建的任务排在它后面。进程内登记（单进程部署）。 - **重试**（retry_tables + create_job）：新任务，只含原任务里 failed、partial、cancelled 的表；用不用模型、本次说明沿用， 起草要求取数据源当前的。 - **服务重启**：启动时（startup）把排队、进行中的任务标成 interrupted，没结束的表记 cancelled、写明「服务重启，起草 中断」。不自动续跑：复制出来的数据目录（检查、验收）启动时不能自己去调模型；界面让人点「重试」续上。关停时 （manager.shutdown）停掉进行中的任务，同样标成 interrupted。 -任务的库写入一律经 _JobRun._write：一条一条来，shield 起来——停止任务时不会打断写到一半的库操作，收尾前等它们写完。 +任务的生命周期（进程内执行、停止、同源排队、库写入一条一条来并 shield、启动时标中断、状态词）在 +data/catalog_job_runtime.py，剖析任务用的是同一套；这里只写起草自己的步骤。 **起草要求**存在数据源 options 的 catalog_draft_guidance 里（上传表格的源存在它自己的 DataSource 行上），一段文字， 去首尾空白、最多 GUIDANCE_MAX 字，空串等于没有。只经 PUT /catalog/guidance 改：数据源接口不显示它、整份回写 options @@ -40,20 +45,15 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from app.data import catalog, catalog_draft +from app.data import catalog_job_runtime as runtime +from app.data.catalog_job_runtime import ACTIVE, FINISHED, RETRYABLE, STOP_GRACE_S, TASK_DONE, TASK_STATUSES from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION from app.db.base import SessionLocal, new_id, utcnow -from app.db.models import CatalogDraftJob, CatalogDraftTask, DataSource +from app.db.models import CatalogDraftJob, CatalogDraftTask, CatalogProfileJob, DataSource logger = logging.getLogger(__name__) -#: 任务还没结束的状态（同一个源只能有一个）、结束了的状态 -ACTIVE = ("queued", "running") -FINISHED = ("succeeded", "partial", "failed", "cancelled", "interrupted") -#: 逐表的状态，以及其中算「已结束」的、重试时要带上的 -TASK_STATUSES = ("pending", "running", "ok", "partial", "failed", "skipped", "cancelled") -TASK_DONE = ("ok", "partial", "failed", "skipped", "cancelled") -RETRYABLE = ("failed", "partial", "cancelled") -_OPEN = ("pending", "running") +_OPEN = runtime.OPEN #: 一个任务最多几张表；起草要求、本次说明最多几个字 MAX_TABLES = 500 @@ -63,9 +63,6 @@ NOTE_MAX = 1000 LAST_DRAFT_JOBS = 100 #: 逐表最多记几个模型留空的列名 BLANK_SAMPLE = 5 -#: 停止进行中的任务时最多等它收尾多久（秒）；关停服务时另给（shutdown 的 grace） -STOP_GRACE_S = 10.0 - INTERRUPTED_ERROR = "服务重启，起草中断" #: 停止、中断时写过段的表（宽表每写出一段就并入目录）：写进了几段、剩几段没起草 STOPPED_WRITTEN = "已停止：已写进目录 {n} 段，剩下 {m} 段没起草" @@ -93,7 +90,7 @@ class TextTooLong(ValueError): """起草要求、本次说明超长。message 给人看。""" -class _JobFailed(Exception): +class _JobFailed(runtime.JobFailed): """任务整体出错（数据源没了、快照没了、模型用不了）：message 就是任务的 error。""" @@ -200,13 +197,15 @@ def _counts(statuses: list[str]) -> dict[str, int]: return counts -def job_dict(job: CatalogDraftJob, tasks: list[CatalogDraftTask]) -> dict[str, Any]: - """接口返回的任务（CatalogDraftJob）：逐表的状态按起草顺序；计数、合计都按逐表的算。""" +def job_dict(job: CatalogDraftJob, tasks: list[CatalogDraftTask], *, waiting_for: str | None = None) -> dict[str, Any]: + """接口返回的任务（CatalogDraftJob）：逐表的状态按起草顺序；计数、合计都按逐表的算。waiting_for：排队中、排在同源的 + 剖析任务后面时是 "profile"（见 waiting_for），否则 null。""" counts = _counts([t.status for t in tasks]) return { "id": job.id, "source_id": job.source_id, "status": job.status, + "waiting_for": waiting_for, "use_model": bool(job.use_model), "model": job.model, "guidance": job.guidance, @@ -235,12 +234,32 @@ def job_dict(job: CatalogDraftJob, tasks: list[CatalogDraftTask]) -> dict[str, A } +async def waiting_for(session: AsyncSession, job: CatalogDraftJob) -> str | None: + """排队中的起草任务在等什么：排在同源的剖析任务后面时是 "profile"，否则 None（进行中、结束了的都是 None）。 + + 先看这个源的队（进程内，排在它前面的是谁）；任务刚建、后台还没排进队时看库：同源有比它先建的、排队或进行中的剖析 + 任务。""" + if job.status != "queued": + return None + ahead = runtime.lanes.ahead(job.source_id, job.id) + if ahead is not None: + return "profile" if "profile" in ahead else None + earlier = (await session.execute( + select(CatalogProfileJob.id).where(CatalogProfileJob.source_id == job.source_id, + CatalogProfileJob.status.in_(ACTIVE), + CatalogProfileJob.created_at <= job.created_at).limit(1) + )).first() + return "profile" if earlier is not None else None + + async def job_out(session: AsyncSession, job: CatalogDraftJob | None) -> dict[str, Any] | None: - return None if job is None else job_dict(job, await tasks_of(session, job.id)) + if job is None: + return None + return job_dict(job, await tasks_of(session, job.id), waiting_for=await waiting_for(session, job)) async def job_brief(session: AsyncSession, job: CatalogDraftJob | None) -> dict[str, Any] | None: - """目录清单顶上的进度：{id, status, total, done}。""" + """目录清单顶上的进度：{id, status, total, done, waiting_for}。""" if job is None: return None rows = (await session.execute( @@ -249,7 +268,7 @@ async def job_brief(session: AsyncSession, job: CatalogDraftJob | None) -> dict[ )).all() counts = {status: n for status, n in rows} return {"id": job.id, "status": job.status, "total": sum(counts.values()), - "done": sum(counts.get(s, 0) for s in TASK_DONE)} + "done": sum(counts.get(s, 0) for s in TASK_DONE), "waiting_for": await waiting_for(session, job)} async def last_drafts(session: AsyncSession, source_id: str) -> dict[str, dict[str, Any]]: @@ -294,12 +313,30 @@ async def last_drafts(session: AsyncSession, source_id: str) -> dict[str, dict[s # ========================================================================== -def busy_message(row: Any) -> str: - return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录，请等这次起草结束（或先停止它）再开始新的起草" +#: 撞上起草时说法的最后一句（「请等这次起草结束{then}」）缺省是建起草任务的；导入目录、同步起草撞上起草、剖析时按这次要做 +#: 的事说（_then） +THEN_DRAFT = "再开始新的起草" -def sync_busy_message(row: Any) -> str: - return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录（同步起草），请等这次起草结束再开始新的起草" +def _then(kind: str, busy: str) -> str: + """同步写操作（kind：import / sync_draft）撞上 busy（draft 起草、profile 剖析）时说法的最后一句：导入撞上什么都是 + 「再导入」；同步起草撞上起草是「再开始新的起草」（和建起草任务同一句话），撞上剖析是「再起草」——那时要做的是起草， + 不是「开始新的剖析」。""" + if kind == "import": + return "再导入" + return THEN_DRAFT if busy == "draft" else "再起草" + + +def busy_message(row: Any, then: str = THEN_DRAFT) -> str: + return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录，请等这次起草结束（或先停止它）{then}" + + +def sync_busy_message(row: Any, then: str = THEN_DRAFT) -> str: + return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录（同步起草），请等这次起草结束{then}" + + +def import_busy_message(row: Any) -> str: + return f"数据源「{getattr(row, 'name', '')}」正在导入数据目录，请等导入完成后再试" #: 正在同步起草（POST /catalog/draft）的数据源。进程内登记：服务端是单进程部署，多进程部署时要换成库里的锁 @@ -311,30 +348,86 @@ def sync_drafting(source_id: str) -> bool: return source_id in _SYNC -async def ensure_idle(session: AsyncSession, row: Any) -> None: - """这个源没有在起草：没有排队或进行中的任务，也没有在同步起草。有就抛 DraftBusy。建任务、重试、导入目录之前都调它。""" +async def ensure_draft_free(session: AsyncSession, row: Any, *, then: str = THEN_DRAFT) -> None: + """同种任务互斥：这个源没有在起草——没有排队或进行中的起草任务，也没有在同步起草。有就抛 DraftBusy。建起草任务、 + 重试之前调它。剖析任务不算：同源有剖析任务时起草任务照建，排在它后面。then 是说法最后一句要做的事。""" if row.id in _SYNC: - raise DraftBusy(sync_busy_message(row)) + raise DraftBusy(sync_busy_message(row, then)) + if await active_job(session, row.id) is not None: + raise DraftBusy(busy_message(row, then)) + + +async def ensure_idle(session: AsyncSession, row: Any, kind: str = "import") -> None: + """同步写操作（kind：import 导入目录、sync_draft 同步起草）要求源空闲：没有在起草（DraftBusy，同 ensure_draft_free）， + 也没有排队或进行中的剖析任务（catalog_profile_jobs.ProfileBusy，说法里写明进行到第几张）。说法最后一句按 kind 说 + 这次要做的事（_then）。""" + from app.data import catalog_profile_jobs + + await ensure_draft_free(session, row, then=_then(kind, "draft")) + await catalog_profile_jobs.ensure_free(session, row, then=_then(kind, "profile")) + + +def _lane_busy(row: Any, holder: runtime.LaneTicket, kind: str) -> Exception: + """库里查过没有任务、这个源的队却被占着（同步起草、导入目录正在做，或者一个任务刚好在收尾）：按占着的是谁说，最后 + 一句按这次要做的事（kind）说。""" + if holder.kind == "profile": + from app.data import catalog_profile_jobs + + return catalog_profile_jobs.ProfileBusy(catalog_profile_jobs.busy_message(row, None, + then=_then(kind, "profile"))) + if holder.kind == "import": + return DraftBusy(import_busy_message(row)) + then = _then(kind, "draft") + return DraftBusy(sync_busy_message(row, then) if holder.kind == "sync_draft" else busy_message(row, then)) + + +@contextlib.asynccontextmanager +async def _sync_write(row: Any, kind: str, check: Callable[[], Awaitable[None]]) -> AsyncIterator[None]: + """同步写操作期间占着这个源的队（不阻塞地试占）：先查库（check：源上有排队或进行中的任务抛 DraftBusy / ProfileBusy）， + 再占队——别的同步写操作正在做时占不到，同样抛忙，什么都不做。占到了，执行期间新建的任务排在它后面，做完才开始。""" + await check() + try: + ticket = runtime.lanes.try_enter(row.id, kind) + except runtime.LaneBusy as e: + raise _lane_busy(row, e.holder, kind) from e + try: + yield + finally: + ticket.release() + + +async def _ensure_idle_but_sync(session: AsyncSession, row: Any) -> None: + """同 ensure_idle(…, "sync_draft")，只是不看 _SYNC：同步起草自己刚登记在里面。""" + from app.data import catalog_profile_jobs + if await active_job(session, row.id) is not None: raise DraftBusy(busy_message(row)) + await catalog_profile_jobs.ensure_free(session, row, then=_then("sync_draft", "profile")) @contextlib.asynccontextmanager async def sync_draft(session: AsyncSession, row: Any) -> AsyncIterator[None]: - """同步起草期间登记这个源：同一个源建任务、重试、导入、再开一次同步起草都抛 DraftBusy；有排队或进行中的任务时 - 这里就抛 DraftBusy。先登记、再查库：查库要让出事件循环，先登记才不会和同时建的任务互相看不见（建任务那边写库之后 - 也再看一次，见 create_job）。""" + """同步起草期间登记这个源、占着它的队：同一个源建起草任务、重试、导入、再开一次同步起草都抛 DraftBusy，期间新建的 + 剖析任务排在它后面；有排队或进行中的起草任务时这里抛 DraftBusy，有剖析任务时抛 ProfileBusy。先登记、再查库：查库要 + 让出事件循环，先登记才不会和同时建的任务互相看不见（建任务那边写库之后也再看一次，见 create_job）。""" if row.id in _SYNC: raise DraftBusy(sync_busy_message(row)) _SYNC.add(row.id) try: - if await active_job(session, row.id) is not None: - raise DraftBusy(busy_message(row)) - yield + async with _sync_write(row, "sync_draft", lambda: _ensure_idle_but_sync(session, row)): + yield finally: _SYNC.discard(row.id) +@contextlib.asynccontextmanager +async def import_write(session: AsyncSession, row: Any) -> AsyncIterator[None]: + """导入目录期间占着这个源的队：源上有排队或进行中的起草、剖析任务（或者正在同步起草、导入）时抛 DraftBusy / + ProfileBusy；导入期间新建的任务排在它后面。""" + async with _sync_write(row, "import", lambda: ensure_idle(session, row, "import")): + yield + + async def check_model(session: AsyncSession) -> None: """起草模型能不能用：不能用抛 catalog.CatalogModelUnavailable（只建模型对象，不发请求）。""" await catalog.resolve_draft_model(session) @@ -344,8 +437,9 @@ async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: li note: str | None, actor: str | None, retry_of: str | None = None) -> tuple[CatalogDraftJob, list[CatalogDraftTask]]: """建一个排队的任务（还没开始，调用方接着 manager.start）。row 是数据源记录（起草要求取它的），source 是读表结构 - 用的源（上传源是绑定当前快照的视图）。有排队或进行中的任务抛 DraftBusy。""" - await ensure_idle(session, row) + 用的源（上传源是绑定当前快照的视图）。有排队或进行中的起草任务（或正在同步起草）抛 DraftBusy；同源有剖析任务时照建， + 排在它后面。""" + await ensure_draft_free(session, row) cache = getattr(source, "schema_cache", None) or {} all_tables = cache.get("tables") or {} wanted = list(tables) if tables else await catalog.default_tables(session, source) @@ -395,22 +489,15 @@ async def retry_tables(session: AsyncSession, job: CatalogDraftJob) -> list[str] async def cancel(session: AsyncSession, job: CatalogDraftJob) -> None: - """停止一个任务。排队的直接 cancelled；进行中的交给 manager 停（停掉模型调用、没写完的表记 cancelled）。 - 已经结束的抛 DraftFinished。""" + """停止一个任务。排队的直接 cancelled（在队里等着的后台执行跟着停掉、出队）；进行中的交给 manager 停（停掉模型调用、 + 没写完的表记 cancelled）。已经结束的抛 DraftFinished。""" if job.status in FINISHED: raise DraftFinished("这次起草已经结束了，不需要停止") if job.status == "queued": - now = utcnow() - claimed = await session.execute( - update(CatalogDraftJob).where(CatalogDraftJob.id == job.id, CatalogDraftJob.status == "queued") - .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) - if claimed.rowcount == 1: - await session.execute( - update(CatalogDraftTask).where(CatalogDraftTask.job_id == job.id, CatalogDraftTask.status.in_(_OPEN)) - .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) - await session.commit() + if await runtime.cancel_queued(session, CatalogDraftJob, CatalogDraftTask, job.id): + await manager.stop(job.id, "user", grace=STOP_GRACE_S) return - await session.rollback() # 刚被后台认领、开始了：按进行中的停 + # 刚被后台认领、开始了：按进行中的停 if await manager.stop(job.id, "user", grace=STOP_GRACE_S): return # 库里说在进行，这个进程里却没有在跑它（记录对不上）：直接记停止 @@ -434,26 +521,19 @@ async def _close(session: AsyncSession, job_id: str, job_status: str, job_error: .values(status=task_status, error=task_error, finished_at=now, model_error=written.format(n=n, m=max(0, int(total or n) - int(n)))) .execution_options(synchronize_session=False)) - await session.execute( - update(CatalogDraftTask).where(CatalogDraftTask.job_id == job_id, CatalogDraftTask.status.in_(_OPEN)) - .values(status=task_status, error=task_error, finished_at=now).execution_options(synchronize_session=False)) - await session.execute( - update(CatalogDraftJob).where(CatalogDraftJob.id == job_id, CatalogDraftJob.status.in_(ACTIVE)) - .values(status=job_status, error=job_error, finished_at=now).execution_options(synchronize_session=False)) - await session.commit() + await runtime.close_job(session, CatalogDraftJob, CatalogDraftTask, job_id, job_status, job_error, task_status, + task_error) async def startup() -> int: """服务启动时：排队、进行中的任务标成 interrupted，没结束的表记 cancelled、写明服务重启。不自动续跑。返回标了几个。""" - async with SessionLocal() as session: - ids = [job_id for job_id in (await session.execute( - select(CatalogDraftJob.id).where(CatalogDraftJob.status.in_(ACTIVE)))).scalars() - if not manager.running(job_id)] - for job_id in ids: - await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) - if ids: - logger.warning("服务重启：%d 个数据目录起草任务没有做完，已标为中断，可在数据目录页重试", len(ids)) - return len(ids) + async def interrupt(session: AsyncSession, job_id: str) -> None: + await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) + + n = await runtime.interrupt_on_startup(CatalogDraftJob, interrupt, running=manager.running) + if n: + logger.warning("服务重启：%d 个数据目录起草任务没有做完，已标为中断，可在数据目录页重试", n) + return n # ========================================================================== @@ -461,60 +541,35 @@ async def startup() -> int: # ========================================================================== -class _JobRun: - """一个任务在后台的一次执行。""" +class _JobRun(runtime.JobRun): + """一个起草任务在后台的一次执行（进队、停止、收尾的框架在 runtime.JobRun）。""" + + kind = "draft" + model = CatalogDraftJob def __init__(self, job_id: str) -> None: - self.job_id = job_id - self.task: asyncio.Task[None] | None = None - #: 为什么停：user 使用者点了停止，shutdown 服务关停（标中断） - self.stop_reason: str | None = None - self._writes: set[asyncio.Task[Any]] = set() - self._lock = asyncio.Lock() + super().__init__(job_id) self._started: set[str] = set() self.source: Any = None self.actor: str | None = None - # ---------------------------------------------------------------- 库写入 + # ---------------------------------------------------------------- 停止、出错时收尾 - async def _write(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: - """一条库写入：同一个任务的写入一条一条来；shield 起来，停止任务时不会打断写到一半的库操作。""" - async def go() -> Any: - async with self._lock: - async with SessionLocal() as session: - return await fn(session, *args) + async def _stopped(self) -> None: + if self.stop_reason == "user": + await self._end("cancelled", None, "cancelled", None) + else: + await self._end("interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) - task = asyncio.ensure_future(go()) - self._writes.add(task) - task.add_done_callback(self._writes.discard) - return await asyncio.shield(task) + async def _failed(self, reason: str) -> None: + await self._end("failed", reason, "failed", reason) - async def _settle(self) -> None: - """等手上的库写入都写完（停止、出错收尾之前）。""" - while self._writes: - await asyncio.gather(*list(self._writes), return_exceptions=True) + async def _crashed(self, e: Exception) -> None: + from app.core.errors import describe_exception, first_line - # ---------------------------------------------------------------- 主流程 - - async def run(self) -> None: - try: - await self._run() - except asyncio.CancelledError: - current = asyncio.current_task() - if current is not None: - current.uncancel() - if self.stop_reason == "user": - await self._end("cancelled", None, "cancelled", None) - else: - await self._end("interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) - except _JobFailed as e: - await self._end("failed", str(e), "failed", str(e)) - except Exception as e: # noqa: BLE001 - 任务整体出错：照实记下，不留在「起草中」 - from app.core.errors import describe_exception, first_line - - logger.exception("数据目录起草任务 %s 出错", self.job_id) - reason = JOB_ERROR.format(line=first_line(e) or describe_exception(e)) - await self._end("failed", reason, "failed", reason) + logger.exception("数据目录起草任务 %s 出错", self.job_id) + reason = JOB_ERROR.format(line=first_line(e) or describe_exception(e)) + await self._end("failed", reason, "failed", reason) async def _end(self, job_status: str, job_error: str | None, task_status: str, task_error: str | None) -> None: """停止、出错时收尾：等手上的库写入写完，再把没结束的表和任务记下来。收尾本身写不进去只记日志（下次启动时 @@ -603,11 +658,7 @@ class _JobRun: .execution_options(synchronize_session=False) async def _claim(self, session: AsyncSession) -> bool: - result = await session.execute( - update(CatalogDraftJob).where(CatalogDraftJob.id == self.job_id, CatalogDraftJob.status == "queued") - .values(status="running", started_at=utcnow()).execution_options(synchronize_session=False)) - await session.commit() - return result.rowcount == 1 + return await runtime.claim(session, CatalogDraftJob, self.job_id) async def _set_model(self, session: AsyncSession, model_id: str) -> None: await session.execute(update(CatalogDraftJob).where(CatalogDraftJob.id == self.job_id) @@ -746,66 +797,11 @@ async def write_part(session: AsyncSession, source: Any, table: str, notes: dict return None -class DraftJobs: - """进程内的起草任务（照 app/scheduler 的写法：一个 asyncio 任务一件事，停的时候给宽限）。""" +class DraftJobs(runtime.JobManager): + """进程内的起草任务（start / stop / wait / running / shutdown 见 runtime.JobManager）。""" def __init__(self) -> None: - self._runs: dict[str, _JobRun] = {} - - def _alive(self, job_id: str) -> _JobRun | None: - run = self._runs.get(job_id) - if run is None or run.task is None or run.task.done(): - return None - try: - if run.task.get_loop() is not asyncio.get_running_loop(): - self._runs.pop(job_id, None) # 别的事件循环留下的（测试每个用例一个循环） - return None - except RuntimeError: - return None - return run - - def start(self, job_id: str) -> None: - """开始在后台起草这个（排队中的）任务。""" - run = _JobRun(job_id) - run.task = asyncio.get_running_loop().create_task(run.run(), name=f"catalog-draft-{job_id}") - self._runs[job_id] = run - - def forget(_: asyncio.Task[None]) -> None: - if self._runs.get(job_id) is run: - self._runs.pop(job_id, None) - - run.task.add_done_callback(forget) - - def running(self, job_id: str) -> bool: - return self._alive(job_id) is not None - - async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S) -> bool: - """停掉这个进程里在跑的任务，等它收尾（最多 grace 秒）。不在这个进程里跑返回 False。""" - run = self._alive(job_id) - if run is None: - return False - if run.stop_reason is None: - run.stop_reason = reason - run.task.cancel() # type: ignore[union-attr] - await asyncio.wait([run.task], timeout=grace) # type: ignore[list-item] - return True - - async def wait(self, job_id: str, timeout: float | None = None) -> None: - """等这个任务做完（测试、调试用）。不在跑直接返回。""" - run = self._alive(job_id) - if run is not None: - await asyncio.wait([run.task], timeout=timeout) # type: ignore[list-item] - - async def shutdown(self, *, grace: float = 3.0) -> None: - """关停：停掉进行中的任务（标中断），最多等 grace 秒收尾。""" - runs = [run for job_id in list(self._runs) if (run := self._alive(job_id)) is not None] - for run in runs: - if run.stop_reason is None: - run.stop_reason = "shutdown" - run.task.cancel() # type: ignore[union-attr] - if runs: - await asyncio.wait([run.task for run in runs], timeout=grace) # type: ignore[misc] - logger.info("已停止 %d 个数据目录起草任务", len(runs)) + super().__init__(_JobRun, label="数据目录起草任务", name="catalog-draft") manager = DraftJobs() @@ -813,7 +809,8 @@ manager = DraftJobs() __all__ = ["ACTIVE", "BLANK_SAMPLE", "DraftBusy", "DraftFinished", "DraftJobs", "FINISHED", "GUIDANCE_MAX", "INTERRUPTED_ERROR", "INTERRUPTED_WRITTEN", "MAX_TABLES", "NOTE_MAX", "NothingToRetry", "RETRYABLE", - "STOPPED_WRITTEN", "TASK_DONE", "TASK_STATUSES", "TextTooLong", - "active_job", "cancel", "check_model", "clean_text", "create_job", "ensure_idle", "guidance_of", "job_brief", - "job_dict", "job_out", "last_drafts", "latest_job", "load", "manager", "retry_tables", "save_guidance", - "startup", "sync_busy_message", "sync_draft", "sync_drafting", "tasks_of", "write_part"] + "STOPPED_WRITTEN", "TASK_DONE", "TASK_STATUSES", "THEN_DRAFT", "TextTooLong", + "active_job", "cancel", "check_model", "clean_text", "create_job", "ensure_draft_free", "ensure_idle", + "guidance_of", "import_busy_message", "import_write", "job_brief", "job_dict", "job_out", "last_drafts", + "latest_job", "load", "manager", "retry_tables", "save_guidance", "startup", "sync_busy_message", "sync_draft", + "sync_drafting", "tasks_of", "waiting_for", "write_part"] diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 59dd6aa5..0636cd70 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -5,28 +5,47 @@ （来源记为 profile，和外键约束的 fk 分开），对不上的保持 proposed 并在备注里写明覆盖率。顺带给状态类的列 取一次取值分布（码值候选，含义留给人填），给只有一个日期列的表提议业务日期。 +**一次剖析一张表。** 入口是 profile_table：先列全这张表要做的检查（plan_checks，纯函数——范围里的 profile_checks、 +建任务时列出的检查项、执行时用的是同一份），第一条查询估行数（确知是空表就整张表跳过），再逐项执行，每项开始、结束经 +hooks 回调（后台任务据此把进度落库），整张表结束把结论并进目录（_apply，乐观锁）。没查成的项，目录里剖析以前对它的 +结论原样留着：这次没查成不等于结论错了。以前是一次同步请求剖析一批表、预算按整次算，选 50 张表只做得完前两三张， +其余整张记「查询次数用完」；现在每张表各有自己的预算，任务、排队、停止、留痕的落库在 data/catalog_profile_jobs.py， +这里不碰任务表。 + **这是数据目录里唯一会对业务库发查询的地方**，约束都是硬的： - 默认关闭，按数据源在 options.catalog_profile 里开启（profile_settings）； - 每条查询先过守卫的只读判定（不论数据源本身可写与否），再交给查询层 run_query：同一套连接、守卫和 - 时限，不另开连接； -- 每条查询有时限（不超过数据源自己的查询时限），每次剖析有查询次数上限和总时长上限； + 时限，不另开连接；查询一条一条发，不并发； +- 单条查询的时限取设置和数据源查询时限中小的，且不超过这张表剩下的时长； +- 每张表有自己的预算（TableBudget：查询次数、时长），用完只停这张表，剩下的项记「这张表的查询次数（时长）用完」，下一张 + 表拿满额。整个任务一个查询器（SourceRunner）管任务级止损：连续 3 条查询失败（多半是连接断了、没有权限）、最近 10 条里 + 7 条超时（库响应慢，再发只是给业务库加压）就停下整个任务；最近 6 条里 4 条超时先提醒； +- 每条发出的查询（含超时、失败、守卫拒掉的）都留痕，SQL 里的字面量换成「?」（sql_semantics.normalize_sql）：抽样到的 + 键值不进 AgentLab 自己的库； - 表有多大先看数据库的统计信息，没有统计信息就数到上限为止，不做全表 COUNT；超过上限的表不做 COUNT(DISTINCT)、MIN/MAX 这类整表统计，抽样和取值分布也只看前若干行； - 数据源设置里遮罩的列不取样，也不出现在任何一条剖析查询里； - 只取低基数列的取值分布（最多 21 个取值），不取明细行。 + +表助手的只读查询、起草时核对有效记录条件走单条查询的入口（run_single_query）：同一个只读守卫、同一个单条时限，一次一条， +不设预算、不留痕。 """ from __future__ import annotations +import asyncio +import collections import copy import logging import re import time from collections.abc import Awaitable, Callable from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone from decimal import Decimal from typing import Any +from sqlalchemy import select, update from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite from sqlalchemy.ext.asyncio import AsyncSession @@ -34,6 +53,8 @@ from app.data import catalog, guard from app.data import engine as data_engine from app.data.engine import CATALOG_PROFILE_OPTION, QueryResult, SnapshotTampered from app.data.guard import QueryLimits, SqlRejected +from app.data.sql_semantics import normalize_sql +from app.db.base import SessionLocal logger = logging.getLogger(__name__) @@ -51,33 +72,42 @@ class ProfileSettings: """一个数据源的剖析设置。缺省即「关闭」：剖析会对业务库发查询，必须有人按源明确打开。""" enabled: bool = False - #: 一次剖析最多发几条查询（读统计信息、抽样、核对都算）。用完就停，剩下的记为「查询次数用完」 - max_queries: int = 60 - #: 单条查询的时限（秒）。实际取它和数据源自己的查询时限中较小的一个 + #: 每张表最多发几条查询（估行数、抽样、核对都算）。用完只停这张表，剩下的项记为「这张表的查询次数用完」 + table_max_queries: int = 40 + #: 每张表的时长上限（秒）。用完同样只停这张表 + table_max_s: float = 60 + #: 单条查询的时限（秒）。实际取它和数据源自己的查询时限中较小的一个，且不超过这张表剩下的时长 query_timeout_s: float = 10 #: 每条关系从子表抽多少个不同的键值去父表里对 sample_size: int = 2000 #: 行数不超过它的表才做 COUNT(DISTINCT)、MIN/MAX 这类整表统计；更大的、或者不知道多大的表只看前若干行。 #: 填 0 表示一律不做整表统计 max_scan_rows: int = 100_000 - #: 一次剖析的总时长上限（秒）。接口是同步的，不能让一个请求无限地挂着 - max_total_s: float = 120 def to_dict(self) -> dict[str, Any]: return asdict(self) + @classmethod + def from_dict(cls, raw: Any) -> ProfileSettings: + """任务里存着的设置快照 → 设置（读法同 profile_settings：哪一项不对就用缺省；开关按快照里的）。""" + return profile_settings({PROFILE_OPTION: raw}) if isinstance(raw, dict) else cls() + #: 数值项：(中文叫法, 下限, 上限, 是否必须是整数)。叫法和数据源设置里剖析预算的标签一致（frontend/src/lib/terms.ts）， #: 报错里只写它 _NUMBER_FIELDS: dict[str, tuple[str, float, float, bool]] = { - "max_queries": ("查询次数上限", 1, 500, True), + "table_max_queries": ("每张表最多查询数", 1, 200, True), + "table_max_s": ("每张表时长上限（秒）", 10, 600, False), "query_timeout_s": ("单条查询时限（秒）", 1, 60, False), "sample_size": ("抽样键值数", 10, 10_000, True), "max_scan_rows": ("整表统计行数上限", 0, 10_000_000, True), - "max_total_s": ("总时长上限（秒）", 10, 600, False), } _ENABLED_LABEL = "开启数据剖析" _KNOWN_KEYS = frozenset({"enabled", *_NUMBER_FIELDS}) +#: 以前的整次预算（一次剖析最多几条查询、总时长）。现在预算按表算，这两项不再有意义：启动时从库里删掉 +#: （migrate_settings），保存时请求里还带着的（老页面）只摘掉、不报错（without_legacy）。不折算成新键：旧值按整次剖析 +#: 算，和「每张表」含义不同，硬折算才是不照实 +LEGACY_KEYS = ("max_queries", "max_total_s") def _fmt(value: float) -> str: @@ -101,7 +131,7 @@ def _number(value: Any, low: float, high: float, integer: bool) -> int | float | def profile_settings(options: dict[str, Any] | None) -> ProfileSettings: """options → 剖析设置。读的时候宽松：库里存着的某一项不对（老数据、手改过），那一项用缺省，不报错； - 开关只认 true，写成别的一律当作没开。""" + 开关只认 true，写成别的一律当作没开。认不出的项（含迁移前的旧键）不看。""" raw = (options or {}).get(PROFILE_OPTION) if not isinstance(raw, dict): return ProfileSettings() @@ -121,7 +151,8 @@ def profile_settings_problem(options: dict[str, Any] | None) -> str | None: def profile_settings_rejection(options: dict[str, Any] | None) -> tuple[str, str | None] | None: """同 profile_settings_problem，另给出错的是哪一项：(给人看的一句, 字段名)。字段名是 enabled 或 _NUMBER_FIELDS - 的键（max_queries……），格式不对、有认不出的项时为 None。接口带着它回 422，表单不必从原话里认是哪一格。""" + 的键（table_max_queries……），格式不对、有认不出的项时为 None。接口带着它回 422，表单不必从原话里认是哪一格。 + 旧键（LEGACY_KEYS）在这之前已经由 without_legacy 摘掉。""" raw = (options or {}).get(PROFILE_OPTION) if raw is None: return None @@ -138,6 +169,36 @@ def profile_settings_rejection(options: dict[str, Any] | None) -> tuple[str, str return None +def without_legacy(options: dict[str, Any] | None) -> dict[str, Any] | None: + """保存数据源时：剖析设置里还带着旧的整次预算（老页面提交的），只摘掉，不报错。没有旧键原样返回。""" + raw = (options or {}).get(PROFILE_OPTION) + if not isinstance(raw, dict) or not any(key in raw for key in LEGACY_KEYS): + return options + return {**(options or {}), PROFILE_OPTION: {k: v for k, v in raw.items() if k not in LEGACY_KEYS}} + + +async def migrate_settings(session: AsyncSession) -> int: + """服务启动时一次性迁移：库里所有数据源 options.catalog_profile 中的旧键（max_queries、max_total_s）删掉，记日志。 + 不折算成新键（见 LEGACY_KEYS）。返回改了几个数据源。只读写设置，不连业务库。""" + from app.db.models import DataSource + + rows = (await session.execute(select(DataSource.id, DataSource.name, DataSource.options))).all() + changed = 0 + for source_id, name, options in rows: + cleaned = without_legacy(options if isinstance(options, dict) else None) + if cleaned is options or cleaned is None: + continue + dropped = {k: (options or {}).get(PROFILE_OPTION, {}).get(k) for k in LEGACY_KEYS + if k in ((options or {}).get(PROFILE_OPTION) or {})} + await session.execute(update(DataSource).where(DataSource.id == source_id).values(options=cleaned) + .execution_options(synchronize_session=False)) + logger.info("数据剖析设置迁移：数据源 %s（%s）去掉了按整次剖析算的旧预算 %s，改按每张表的预算", source_id, name, + dropped) + changed += 1 + if changed: + await session.commit() + return changed + # ========================================================================== # SQL：字面量与方言 # ========================================================================== @@ -279,17 +340,33 @@ class SqlDialect: # ========================================================================== -# 发查询：守卫、查询次数、时限 +# 发查询：守卫、单条时限、每张表的预算、任务级止损、留痕 # ========================================================================== -#: 总时长只剩这么一点时不再发新查询：发出去的时限也只剩零点几秒，白占一次查询次数 +#: 这张表的时长只剩这么一点时不再发新查询：发出去的时限也只剩零点几秒，白占一次查询次数 _MIN_WINDOW_S = 0.5 -#: 连续失败这么多条就停：多半是连接断了、账号没有权限，再发只是一条条重复同一个错误 -_MAX_CONSECUTIVE_ERRORS = 3 +#: 连续失败这么多条就停下整个任务：多半是连接断了、账号没有权限，再发只是一条条重复同一个错误 +MAX_CONSECUTIVE_ERRORS = 3 +#: 最近 SLOW_WINDOW 条查询里 SLOW_STOP 条超时就停下整个任务：库响应慢，再发只是给业务库加压 +SLOW_WINDOW = 10 +SLOW_STOP = 7 +#: 最近 NOTICE_WINDOW 条里 NOTICE_ON 条超时先提醒（任务的 notice），好转后清掉 +NOTICE_WINDOW = 6 +NOTICE_ON = 4 +#: 「运行原因没查成」的跳过原因：计入 unfinished，使这张表成为 partial。其余（遮罩、视图、表太大、取值太多……）是 +#: 按规则不查，不算没做完 +UNFINISHED_REASONS = frozenset({"budget", "deadline", "failed", "timeout", "error", "rejected", "slow", "cancelled"}) + +FAILED_STOP = "连续 {n} 条查询失败，已停止剖析：{line}" +SLOW_STOP_TEXT = f"数据库响应慢：最近 {SLOW_WINDOW} 条查询 {SLOW_STOP} 条超时，已停止，避免给业务库加压" +SLOW_NOTICE = "数据库响应慢：最近 {n} 条查询有 {k} 条超时。超时再多会自动停止，也可以先停止、换个时间再剖析" +#: 停止落在一条查询上：asyncio 这边不再等它的结果，库那边由单条时限截止（run_query 的 _server_deadline） +CANCELLED_QUERY = "剖析已停止，没有等这条查询的结果；数据库按单条时限截止它" class _Stop(Exception): - """整次剖析停下：budget 查询次数用完、deadline 总时长用完、failed 连续失败。剩下的检查都不再发查询。""" + """停下：budget 这张表的查询次数用完、deadline 这张表的时长用完（只停这张表）；failed 连续失败、slow 超时比例 + （整个任务停下）。剩下的检查都不再发查询。""" def __init__(self, reason: str) -> None: super().__init__(reason) @@ -310,6 +387,16 @@ class _Skip(Exception): self.carry = carry +class QueryFailed(Exception): + """单条只读查询（run_single_query）没成：reason 是 rejected 未通过守卫 / timeout 超时 / error 查询失败，detail + 是给人看的整句（同剖析里这一项的说明）。""" + + def __init__(self, reason: str, detail: str) -> None: + super().__init__(detail) + self.reason = reason + self.detail = detail + + def _brief(e: BaseException) -> str: """驱动报错的第一行，去掉 SQLAlchemy 加的「(模块.类名)」前缀和后面附的 SQL、文档链接。""" lines = str(getattr(e, "orig", None) or e).strip().splitlines() @@ -317,13 +404,39 @@ def _brief(e: BaseException) -> str: return first[:160] or type(e).__name__ -class _Runner: - """剖析发的每一条查询都从这里走：先按只读过守卫，再交给查询层 run_query；记次数、管时限。 +class TableBudget: + """一张表的预算：最多 table_max_queries 条查询、table_max_s 秒（从这张表开始算）。用完抛 _Stop（budget / + deadline），只停这张表，下一张表另起一份满额的。""" + + def __init__(self, settings: ProfileSettings) -> None: + self.max_queries = int(settings.table_max_queries) + self.seconds = float(settings.table_max_s) + self.ends = time.monotonic() + self.seconds + self.used = 0 + #: 一条查询的时限被这张表剩下的时长截短后超时了：时长已经用完 + self.out_of_time = False + + def remaining(self) -> float: + """要发一条查询之前：次数、时长都还有，返回这张表剩下的秒数；用完抛 _Stop。不记次数（守卫拒掉的不算）。""" + if self.used >= self.max_queries: + raise _Stop("budget") + left = self.ends - time.monotonic() + if self.out_of_time or left <= _MIN_WINDOW_S: + raise _Stop("deadline") + return left + + +class SourceRunner: + """剖析发的每一条查询都从这里走（整个任务一个）：先按只读过守卫，再交给查询层 run_query；管单条时限、任务级止损， + 每条查询交给 record 留痕。 - 只读：不论数据源本身可写与否，先用 guard.check(readonly=True) 判一遍，再进 run_query——那里还会按 数据源自己的设置判一遍。两遍是同一个守卫，不为剖析放宽，也不另开连接。 - - 查询次数：每发一条记一次（超时、失败的也算），到上限就停。没发出去的（守卫拒了）不算。 - - 时限：单条取设置和数据源查询时限中较小的一个，并且不超过总时长剩下的部分。 + - 单条时限：取设置和数据源查询时限中较小的一个，并且不超过这张表剩下的时长（budget）。 + - 预算：每发一条记一次这张表的次数（超时、失败的也算），守卫拒掉的没发出去，不算。 + - 止损：连续 MAX_CONSECUTIVE_ERRORS 条失败 → stopped = failed；最近 SLOW_WINDOW 条里 SLOW_STOP 条超时 → + stopped = slow。停下之后再发直接抛 _Stop，error 是整句原因。最近 NOTICE_WINDOW 条里 NOTICE_ON 条超时时 notice + 有一句提醒，好转后清掉。 """ def __init__(self, source: Any, settings: ProfileSettings, dialect: SqlDialect) -> None: @@ -331,49 +444,101 @@ class _Runner: self.settings = settings self.kind = dialect.kind self.timeout = min(float(settings.query_timeout_s), float(data_engine.query_timeout(source))) - self.ends = time.monotonic() + float(settings.max_total_s) - self.used = 0 + #: 整个任务发出的查询数、其中超时的、失败的 + self.queries = 0 + self.timeouts = 0 + self.errors = 0 self.stopped: str | None = None - self._errors = 0 + self.error: str | None = None + self.notice: str | None = None + self._consecutive = 0 + self._recent: collections.deque[bool] = collections.deque(maxlen=SLOW_WINDOW) - def _stop(self, reason: str) -> None: - self.stopped = reason - raise _Stop(reason) + def _timed(self, timed_out: bool) -> None: + """记下这条查询超时没有，按比例判提醒和止损。""" + self._recent.append(timed_out) + recent = list(self._recent) + window = recent[-NOTICE_WINDOW:] + hits = sum(window) + self.notice = SLOW_NOTICE.format(n=len(window), k=hits) if hits >= NOTICE_ON else None + if sum(recent) >= SLOW_STOP and self.stopped is None: + self.stopped = "slow" + self.error = SLOW_STOP_TEXT - async def __call__(self, sql: str, *, max_rows: int = 1) -> QueryResult: + async def run(self, sql: str, *, max_rows: int = 1, budget: TableBudget | None = None, + record: Callable[[dict[str, Any]], None] | None = None) -> QueryResult: if self.stopped: raise _Stop(self.stopped) - if self.used >= self.settings.max_queries: - self._stop("budget") - remaining = self.ends - time.monotonic() - if remaining <= _MIN_WINDOW_S: - self._stop("deadline") + left = budget.remaining() if budget is not None else None + at = datetime.now(timezone.utc).isoformat(timespec="milliseconds") + + def log(outcome: str, ms: int, rows: int | None, detail: str | None) -> None: + if record is not None: + record({"at": at, "sql": normalize_sql(sql, self.kind).text, "ms": ms, "rows": rows, + "outcome": outcome, "detail": detail}) + try: guard.check(sql, readonly=True, source_name=getattr(self.source, "name", ""), dialect=self.kind) except SqlRejected as e: - raise _Skip("rejected", f"剖析查询未通过安全守卫：{e}") from e - self.used += 1 - cut = remaining < self.timeout - seconds = round(min(self.timeout, remaining), 3) + detail = f"剖析查询未通过安全守卫：{e}" + log("rejected", 0, None, detail) + raise _Skip("rejected", detail) from e + if budget is not None: + budget.used += 1 + self.queries += 1 + cut = left is not None and left < self.timeout + seconds = round(min(self.timeout, left) if left is not None else self.timeout, 3) limits = QueryLimits(max_rows=max(1, int(max_rows)), timeout_seconds=seconds) + started = time.perf_counter() + + def ms() -> int: + return int((time.perf_counter() - started) * 1000) + try: result = await data_engine.run_query(self.source, sql, limits=limits) except SqlRejected as e: # 上面已经按只读过了同一个守卫，查询层再拒只可能是时限到了 - if cut: - self.stopped = "deadline" - raise _Skip("timeout", f"查询超时：超过 {_fmt(seconds)} 秒被中断，已跳过") from e + self.timeouts += 1 + if cut and budget is not None: + budget.out_of_time = True + detail = f"查询超时：超过 {_fmt(seconds)} 秒被中断，已跳过" + log("timeout", ms(), None, detail) + self._timed(True) + raise _Skip("timeout", detail) from e except SnapshotTampered: raise - except Exception as e: # noqa: BLE001 - 一条查询失败不该拖垮整次剖析，原因照实记下 - self._errors += 1 - if self._errors >= _MAX_CONSECUTIVE_ERRORS: + except asyncio.CancelledError: + log("error", ms(), None, CANCELLED_QUERY) + raise + except Exception as e: # noqa: BLE001 - 一条查询失败不该拖垮整张表，原因照实记下 + self.errors += 1 + self._consecutive += 1 + detail = f"查询失败：{_brief(e)}" + log("error", ms(), None, detail) + self._timed(False) + if self._consecutive >= MAX_CONSECUTIVE_ERRORS and self.stopped is None: self.stopped = "failed" - raise _Skip("error", f"查询失败：{_brief(e)}") from e - self._errors = 0 + self.error = FAILED_STOP.format(n=MAX_CONSECUTIVE_ERRORS, line=_brief(e)) + raise _Skip("error", detail) from e + self._consecutive = 0 + log("ok", ms(), result.row_count, None) + self._timed(False) return result +async def run_single_query(source: Any, sql: str, *, settings: ProfileSettings, max_rows: int = 1, + kind: str | None = None) -> QueryResult: + """单条只读查询（表助手的 run_query、起草时核对有效记录条件）：同剖析一样过只读守卫、交给 run_query，单条时限取剖析 + 设置和数据源查询时限中较小的一个。只有一条：不设预算、不止损、不留痕。 + + 没成抛 QueryFailed（reason、detail）；快照被改动抛 SnapshotTampered；不支持的数据库类型抛 ValueError。""" + dialect = SqlDialect(kind if kind is not None else getattr(source, "kind", None)) + try: + return await SourceRunner(source, settings, dialect).run(sql, max_rows=max_rows) + except _Skip as e: + raise QueryFailed(e.reason, e.detail) from e + + # ========================================================================== # 表有多大 # ========================================================================== @@ -411,12 +576,13 @@ def _as_int(value: Any) -> int | None: return int(number) if number == number and abs(number) != float("inf") else None -async def estimate_size(run: _Runner, dialect: SqlDialect, meta: dict[str, Any], name: str, *, - max_scan_rows: int) -> tuple[TableSize, list[_Skip]]: +async def estimate_size(run: Callable[..., Awaitable[QueryResult]], dialect: SqlDialect, meta: dict[str, Any], + name: str, *, max_scan_rows: int) -> tuple[TableSize, list[_Skip]]: """先读统计信息；读不到（SQLite、没分析过的表）再数到 max_scan_rows + 1 行为止。 统计信息为 0 或负数（PostgreSQL 没分析过的表是 -1，MySQL 的估算值可能是 0）当作不知道：一张其实很大的表 - 要是被当成空表，后面就会对它做整表统计。返回 (行数, 途中没查成的几条)。查询次数、总时长用完时抛 _Stop。 + 要是被当成空表，后面就会对它做整表统计。返回 (行数, 途中没查成的几条)。这张表的查询次数、时长用完（或者任务止损） +时抛 _Stop。 """ skips: list[_Skip] = [] sql = dialect.stats_sql(meta, name) @@ -444,24 +610,33 @@ async def estimate_size(run: _Runner, dialect: SqlDialect, meta: dict[str, Any], # ========================================================================== -# 一次剖析：上下文、结果 +# 一张表的剖析：上下文、结果 # ========================================================================== -#: 不指定表时剖析几张：目录里有要核对的关系的表，还有待核实（推断）关系的在前，再按使用次数；确知是空表的跳过 -PROFILE_DEFAULT_TABLES = 10 -#: 不指定表时，为跳过空表最多先估几张候选表的行数；估行数的查询最多用掉查询次数上限的一半，免得都花在挑表上 -PROFILE_PROBE_TABLES = 30 #: IN 列表每批最多几个值：Oracle 的 IN 列表不能超过 1000 项（ORA-01795），各家统一按它分批 IN_CHUNK = 1000 #: 剖析写的备注都以它开头（「数据剖析：」）：人工确认过的关系，只覆盖剖析自己写的备注，不碰人写的（认的时候连 #: 升级前带日期的「数据剖析（日期）：」一起认，catalog.is_profile_note）。不写日期，时间看这一项的 updated_at NOTE_PREFIX = catalog.PROFILE_NOTE_PREFIX -#: 整次剖析停下的原因 → 剩下没查的项的说明 +#: 停下的原因 → 剩下没查的项的说明。failed、slow 是任务级止损，停的是整个任务；cancelled 的说法由调用方给（停止、 +#: 服务重启各有各的说法，见 TableHooks.stop_text） _STOP_DETAIL = { - "budget": "本次剖析的查询次数已用完（上限 {max_queries} 条），未检查", - "deadline": "本次剖析的总时长已用完（上限 {max_total_s} 秒），未检查", + "budget": "这张表的查询次数已用完（上限 {max_queries} 条），未检查", + "deadline": "这张表的时长已用完（上限 {max_s} 秒），未检查", "failed": "连续多条查询失败，已停止剖析，未检查", + "slow": "数据库响应慢，已停止剖析，未检查", } +#: 整张表的原因（TableResult.error） +VIEW_ERROR = "视图没有统计信息，剖析可能触发整个视图的计算，已跳过" +EMPTY_ERROR = "空表，没有可剖析的数据" +NOTHING_ERROR = "没有可剖析的项：没有要核对的关联关系、像码值的列和日期列，或者都被遮罩了" +CONFLICT_ERROR = "这张表的数据目录正被其他人修改，剖析结果未能写入，请稍后重新剖析" +WRITE_ERROR = "写入数据目录时出错：{line}" +TABLE_ERROR = "剖析这张表时出错：{line}" +#: 停止、服务重启时正在剖析的那张表：做完的项照常写进了目录，剩下的没查 +STOPPED_TABLE = "{why}：已做完的 {done} 项照常写进了数据目录，剩下 {rest} 项没查" +#: 停止的说法（TableHooks.stop_text 没给时） +STOPPED = "剖析已停止" class ProfileDisabled(Exception): @@ -473,18 +648,15 @@ class ProfileDisabled(Exception): self.code = code -class ProfileBusy(Exception): - """这个数据源正在剖析。message 给人看。""" - - @dataclass class ProfileSkip: """没做的一项和原因。 kind：relation 关系、codes 码值、date 日期列、row_estimate 行数估算、table 整张表。 - reason：budget 查询次数用完、deadline 总时长用完、failed 连续失败、timeout 超时、error 查询失败、 - rejected 未通过守卫、masked 字段被遮罩、too_large 表太大、view 视图、unsupported 暂不支持、 - no_data 没有数据、missing 表结构里没有、high_cardinality 取值太多（不像码值）。detail 是给人看的整句。 + reason：budget 这张表的查询次数用完、deadline 这张表的时长用完、failed 连续失败止损、slow 超时比例止损、 + cancelled 停止或服务重启、timeout 超时、error 查询失败、rejected 未通过守卫（以上算「运行原因没查成」，见 + UNFINISHED_REASONS）；masked 字段被遮罩、too_large 表太大、view 视图、unsupported 暂不支持、no_data 没有数据、 + missing 表结构里没有、high_cardinality 取值太多（不像码值）、nothing 没有可剖析的项（按规则不查）。detail 是给人看的整句。 """ kind: str @@ -499,35 +671,57 @@ class ProfileSkip: @dataclass -class TableProfile: - """一张表这次剖析的结果。findings 是写进目录的发现（给界面看，形状见 _finding_*）。""" +class TableResult: + """一张表这次剖析的结果。 + + status：ok 每项都有结论、partial 有项因为运行原因没查成、failed 写入一直冲突或整张表出错、skipped 表结构里没有 / + 视图 / 空表 / 没有可剖析的项。checks 是检查项（[{kind, target, path, state, reason, detail, queries}]，按执行顺序， + 含按规则不查的）；findings 是写进目录的发现（给界面看，形状见 _relation_finding 等）；query_log 是发过的查询 + （字面量已换成「?」）。""" table: str + status: str = "running" + checks: list[dict[str, Any]] = field(default_factory=list) + #: 进行中时正在做哪一项的整句 + current: str | None = None queries: int = 0 + elapsed_ms: int = 0 row_estimate: TableSize | None = None findings: list[dict[str, Any]] = field(default_factory=list) skipped: list[ProfileSkip] = field(default_factory=list) #: 日期类列的取值范围（MIN、MAX）。只有一个日期列时它还写进了业务日期的备注；多个时只在这里 date_ranges: list[dict[str, Any]] = field(default_factory=list) + query_log: list[dict[str, Any]] = field(default_factory=list) added: int = 0 updated: int = 0 removed: int = 0 version: int = 0 - #: 这张表没剖析（表结构里没有）或结果没写进去（写入一直冲突） + #: 整张表的原因：表结构里没有、视图、空表、没有可剖析的项、写入一直冲突、停止 / 中断 / 止损 error: str | None = None + @property + def skipped_count(self) -> int: + return len(self.skipped) -@dataclass -class ProfileReport: - """一次剖析的结果。stopped：整次剖析中途停下的原因（budget / deadline / failed），做完了为 None。""" + @property + def unfinished(self) -> int: + """跳过的项里因为运行原因没查成的（次数、时长、超时、出错、守卫拒、止损、停止）。""" + return sum(1 for s in self.skipped if s.reason in UNFINISHED_REASONS) - profiled_at: str - settings: ProfileSettings - tables: list[TableProfile] = field(default_factory=list) - queries_used: int = 0 - stopped: str | None = None - #: 不指定表时，挑表途中估过行数、确知是空表而跳过的表（表结构顺序）。估行数的查询算在 queries_used 里 - empty_tables: list[str] = field(default_factory=list) + def summary(self) -> dict[str, int]: + return summarize(self.findings) + + +def summarize(findings: list[dict[str, Any]] | None) -> dict[str, int]: + """发现的合计：有结论的关系条数、其中有确证（verified）的、取到码值候选的列数、这些码值里含义还空着的个数、提议了 + 业务日期的表数（一张表时是 0 或 1）。""" + found = [f for f in findings or [] if isinstance(f, dict)] + relations = [f for f in found if f.get("kind") == "relation"] + codes = [f for f in found if f.get("kind") == "codes"] + return {"relations_checked": len(relations), + "relations_verified": sum(1 for f in relations if f.get("status") == "verified"), + "codes": len(codes), "codes_pending": sum(int(f.get("pending") or 0) for f in codes), + "business_dates": sum(1 for f in found if f.get("kind") == "business_date")} @dataclass @@ -584,34 +778,67 @@ class _TableOutcome: @dataclass -class _Context: +class ProfileContext: + """一个剖析任务共用的：源（上传源是绑定当前快照的视图）、设置、方言、查询器（任务级止损在它身上）、表结构、遮罩的 + 列（小写，任务每张表开始前按数据源最新的设置换掉）、已经估过的行数（父表的行数核对关系时会用到，不重复估）。""" + source: Any settings: ProfileSettings dialect: SqlDialect - run: _Runner + runner: SourceRunner tables: dict[str, Any] #: 遮罩的列（小写）。遮罩按列名生效、不分大小写、不分表，和证据面板的比法一致 masked: set[str] sizes: dict[str, TableSize] = field(default_factory=dict) - async def size(self, table: str, result: TableProfile) -> TableSize: - """表有多大，一次剖析里每张表只估一次。途中没查成的记进 result（行数估算这一项）。""" - if table not in self.sizes: - size, skips = await estimate_size(self.run, self.dialect, self.tables[table], table, - max_scan_rows=self.settings.max_scan_rows) - self.sizes[table] = size - result.skipped += [ProfileSkip("row_estimate", table, s.reason, s.detail) for s in skips] - return self.sizes[table] - async def size_with_data(self, table: str, result: TableProfile) -> TableSize: - """同 size，但确知是空表时直接跳过这一项（抛 _Skip），不再为它发查询。""" - size = await self.size(table, result) - if size.rows == 0: - raise _Skip("no_data", f"{table} 是空表，无法剖析") - return size +def make_context(source: Any, settings: ProfileSettings, *, options: dict[str, Any] | None) -> ProfileContext: + """剖析任务的上下文。options 是数据源记录的 options（遮罩按它）；不支持的数据库类型抛 ValueError。""" + dialect = SqlDialect(getattr(source, "kind", None)) + return ProfileContext(source=source, settings=settings, dialect=dialect, + runner=SourceRunner(source, settings, dialect), + tables=(getattr(source, "schema_cache", None) or {}).get("tables") or {}, + masked=masked_of(options)) + + +def masked_of(options: dict[str, Any] | None) -> set[str]: + return {c.lower() for c in data_engine.masked_columns(options)} + + +@dataclass +class TableHooks: + """profile_table 的回调（后台任务据此落库）。都可以不给。 + + - progress：一项检查（含开头的估行数）开始、结束时；result 里的 checks、current、queries、query_log 是这时的样子； + - query：每发出（或被守卫拒掉）一条查询之后； + - recorded：整张表结束，和写目录在同一个事务里（session 是那一次的，shield 起来，停止时也不会被打断；不用自己提交， + 写完由 _record 和目录一起提交）； + - stop_text：停止时的说法（「剖析已停止」「服务重启，剖析中断」），停止之后才调。 + """ + + progress: Callable[[TableResult], Awaitable[None]] | None = None + query: Callable[[TableResult], Awaitable[None]] | None = None + recorded: Callable[[AsyncSession, TableResult], Awaitable[None]] | None = None + stop_text: Callable[[], str] | None = None + + +class _TableCx: + """一张表这次剖析的执行状态：预算、正在做的检查项、发过的查询。检查函数经它发查询、估行数。""" + + def __init__(self, job: ProfileContext, table: str, result: TableResult, hooks: TableHooks) -> None: + self.job = job + self.settings = job.settings + self.dialect = job.dialect + self.tables = job.tables + self.table = table + self.result = result + self.hooks = hooks + self.budget = TableBudget(job.settings) + #: 正在做的检查项（result.checks 里的那一个）；None 是这张表开头的估行数 + self.check: dict[str, Any] | None = None def is_masked(self, column: str) -> bool: - return column.lower() in self.masked + return column.lower() in self.job.masked def scan_cap(self, size: TableSize) -> int | None: """大表（或不知道多大）抽样、取值分布只看前多少行；小表整表看，返回 None。""" @@ -619,6 +846,53 @@ class _Context: return None return max(self.settings.sample_size, self.settings.max_scan_rows) + async def run(self, sql: str, *, max_rows: int = 1, kind: str | None = None, + target: str | None = None) -> QueryResult: + """发一条查询：记进这张表的留痕（kind、target 是它属于哪一项）、计数，再回调 hooks.query。""" + check = self.check + label = (kind or (check["kind"] if check else "row_estimate"), + target or (check["target"] if check else self.table)) + result = self.result + + def record(entry: dict[str, Any]) -> None: + result.query_log.append({"at": entry["at"], "kind": label[0], "target": label[1], "sql": entry["sql"], + "ms": entry["ms"], "rows": entry["rows"], "outcome": entry["outcome"], + "detail": entry["detail"]}) + if entry["outcome"] != "rejected": + result.queries += 1 + if check is not None: + check["queries"] = int(check.get("queries") or 0) + 1 + + try: + found = await self.job.runner.run(sql, max_rows=max_rows, budget=self.budget, record=record) + except _Skip: + await self._queried() + raise + await self._queried() + return found + + async def _queried(self) -> None: + if self.hooks.query is not None: + await self.hooks.query(self.result) + + async def size(self, table: str, result: TableResult) -> TableSize: + """表有多大，一个任务里每张表只估一次。途中没查成的记进 result（行数估算这一项）。""" + if table not in self.job.sizes: + async def run(sql: str, **kw: Any) -> QueryResult: + return await self.run(sql, kind="row_estimate", target=table, **kw) + + size, skips = await estimate_size(run, self.dialect, self.tables[table], table, + max_scan_rows=self.settings.max_scan_rows) + self.job.sizes[table] = size + result.skipped += [ProfileSkip("row_estimate", table, s.reason, s.detail) for s in skips] + return self.job.sizes[table] + + async def size_with_data(self, table: str, result: TableResult) -> TableSize: + """同 size，但确知是空表时直接跳过这一项（抛 _Skip），不再为它发查询。""" + size = await self.size(table, result) + if size.rows == 0: + raise _Skip("no_data", f"{table} 是空表，无法剖析") + return size def _column(meta: dict[str, Any], name: str) -> str | None: """表结构里这一列的写法（目录里的列名大小写可能和表结构不同）；没有返回 None。""" @@ -665,8 +939,8 @@ def _relation_candidates(notes: dict[str, Any] | None) -> list[dict[str, Any]]: if isinstance(r, dict) and r.get("id") and r.get("status") != "rejected"] -async def _unique(cx: _Context, table: str, meta: dict[str, Any], column: str, - result: TableProfile) -> tuple[bool | None, str]: +async def _unique(cx: _TableCx, table: str, meta: dict[str, Any], column: str, + result: TableResult) -> tuple[bool | None, str]: """这一列（非空部分）唯一吗：(True / False / None 不知道, 依据)。有主键或唯一约束直接认；否则只对行数不超过 整表统计上限的表数一次 COUNT 和 COUNT(DISTINCT)，大表不扫。""" how = _constraint(meta, column) @@ -734,27 +1008,13 @@ def _relation_note(*, sample: int, matched: int, coverage: float, scan_cap: int return NOTE_PREFIX + "；".join(parts) + "。" -async def _check_relation(cx: _Context, table: str, rel: dict[str, Any], out: _TableOutcome, - result: TableProfile) -> None: +async def _check_relation(cx: _TableCx, table: str, rel: dict[str, Any], out: _TableOutcome, + result: TableResult) -> None: """从子表取至多 sample_size 个不同的非空键值，去父表里数对上几个，得到覆盖率；再看父键、子键唯不唯一。""" - cols, to_cols = list(rel.get("columns") or []), list(rel.get("to_columns") or []) - to_table = str(rel.get("to_table") or "") - if len(cols) != 1 or len(to_cols) != 1: - raise _Skip("unsupported", "多列组成的关联关系暂不剖析") - meta = cx.tables[table] - parent = to_table if to_table in cx.tables else catalog.resolve_table_name({"tables": cx.tables}, to_table) - if parent is None: - raise _Skip("missing", f"表结构里没有被指向的表 {to_table}，请先重新探查结构") - pmeta = cx.tables[parent] - col, pcol = _column(meta, cols[0]), _column(pmeta, to_cols[0]) - if col is None or pcol is None: - missing = f"{table}.{cols[0]}" if col is None else f"{parent}.{to_cols[0]}" - raise _Skip("missing", f"表结构里没有列 {missing}，请先重新探查结构") - if pmeta.get("is_view"): - raise _Skip("view", f"被指向的 {parent} 是视图，不剖析") - for name, owner in ((col, table), (pcol, parent)): - if cx.is_masked(name): - raise _Skip("masked", f"{owner}.{name} 在数据源设置中被遮罩，不取样", carry=True) + # 两端在表结构里的写法；按规则不查的（多列、不在表结构里、指向视图、碰到遮罩列）plan_checks 已经列成跳过，这里再判 + # 一遍只是兜底 + parent, col, pcol = _relation_ends(table, rel, cx.tables, cx.job.masked) + meta, pmeta = cx.tables[table], cx.tables[parent] if rel.get("source") == "fk": await cx.size_with_data(table, result) @@ -886,7 +1146,7 @@ def _share(count: int, total: int) -> str: return "不足 1%" if 0 < ratio < 0.01 else f"{round(ratio * 100)}%" -async def _check_codes(cx: _Context, table: str, column: str, out: _TableOutcome, result: TableProfile) -> None: +async def _check_codes(cx: _TableCx, table: str, column: str, out: _TableOutcome, result: TableResult) -> None: """GROUP BY 取出现最多的 CODES_MAX + 1 个取值：不超过 CODES_MAX 个就是码值候选（值有了，含义等人填）。""" if cx.is_masked(column): raise _Skip("masked", f"{table}.{column} 在数据源设置中被遮罩，不取值", carry=False) @@ -921,8 +1181,8 @@ def _is_date_column(col: dict[str, Any]) -> bool: return catalog.is_date_column(str(col.get("name") or ""), col.get("type")) -async def _check_dates(cx: _Context, table: str, columns: list[str], total: int, out: _TableOutcome, - result: TableProfile) -> None: +async def _check_dates(cx: _TableCx, table: str, columns: list[str], total: int, out: _TableOutcome, + result: TableResult) -> None: """日期类列的 MIN、MAX，作为业务日期的参考。整表统计，只对行数不超过上限的表做。表里只有一个日期类列 （total 是表结构里日期类列的个数，含被遮罩的）时提议它作为业务日期。""" d, meta = cx.dialect, cx.tables[table] @@ -1131,79 +1391,325 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *, # ========================================================================== -# 入口 +# 一张表要做哪些检查（纯函数） # ========================================================================== -#: 正在剖析的数据源。检查和登记之间没有 await，在一个事件循环里是原子的；服务端是单进程部署， -#: 多进程部署时要换成库里的锁 -_RUNNING: set[str] = set() +@dataclass(frozen=True) +class PlannedCheck: + """一项检查。skip 给了的是按规则不查的（遮罩、多列关系、表结构里没有、被指向的是视图）：(原因, 说明, 要不要原样 + 留着旧结论)，不发查询；relation / column / columns 是执行时要的参数。""" -@dataclass -class _Check: kind: str target: str path: str | None - run: Callable[[], Awaitable[None]] - #: 没查成时调用：把目录里剖析以前对这一项写的结论标记为原样保留 - keep: Callable[[], None] + skip: tuple[str, str, bool] | None = None + relation: dict[str, Any] | None = None + column: str | None = None + columns: tuple[str, ...] = () + #: 日期检查：表结构里日期类列的个数（含被遮罩的）。只有一个时才提议业务日期 + date_total: int = 0 + + def as_check(self) -> dict[str, Any]: + """检查项（TableResult.checks 的一项）：还没开始，或者按规则跳过。""" + state, reason, detail = ("skipped", self.skip[0], self.skip[1]) if self.skip else ("pending", None, None) + return {"kind": self.kind, "target": self.target, "path": self.path, "state": state, "reason": reason, + "detail": detail, "queries": 0} -def _plan(cx: _Context, table: str, notes: dict[str, Any], out: _TableOutcome, - result: TableProfile) -> list[_Check]: - """这张表要做的检查，按价值排：关系、码值、日期。先列全再执行：中途停下时，没做的每一项都要记下原因。""" - meta = cx.tables[table] - checks: list[_Check] = [] +def _relation_ends(table: str, rel: dict[str, Any], tables: dict[str, Any], masked: set[str]) -> tuple[str, str, str]: + """一条关系两端在表结构里的写法：(被指向的表, 子表的列, 被指向的列)。按规则不查的抛 _Skip：多列组成的、表结构里 + 没有的表或列、被指向的是视图、两端有被遮罩的列（遮罩的列不出现在任何一条剖析查询里）。""" + cols, to_cols = list(rel.get("columns") or []), list(rel.get("to_columns") or []) + to_table = str(rel.get("to_table") or "") + if len(cols) != 1 or len(to_cols) != 1: + raise _Skip("unsupported", "多列组成的关联关系暂不剖析") + meta = tables[table] + parent = to_table if to_table in tables else catalog.resolve_table_name({"tables": tables}, to_table) + if parent is None: + raise _Skip("missing", f"表结构里没有被指向的表 {to_table}，请先重新探查结构") + pmeta = tables[parent] + col, pcol = _column(meta, cols[0]), _column(pmeta, to_cols[0]) + if col is None or pcol is None: + missing = f"{table}.{cols[0]}" if col is None else f"{parent}.{to_cols[0]}" + raise _Skip("missing", f"表结构里没有列 {missing}，请先重新探查结构") + if pmeta.get("is_view"): + raise _Skip("view", f"被指向的 {parent} 是视图，不剖析") + for name, owner in ((col, table), (pcol, parent)): + if name.lower() in masked: + raise _Skip("masked", f"{owner}.{name} 在数据源设置中被遮罩，不取样", carry=True) + return parent, col, pcol + + +def plan_checks(table: str, meta: dict[str, Any] | None, notes: dict[str, Any] | None, *, tables: dict[str, Any], + masked: set[str]) -> list[PlannedCheck]: + """这张表要做的检查，按价值排：关系、码值、日期。范围（profile_checks）、建任务时列出的检查项、执行时用的都是这一份： + 先列全再执行，中途停下时没做的每一项都要记下原因。视图、表结构里没有的表返回空列表。 + + 按规则不查的也列出来（带 skip）：被遮罩的码值列、日期列，多列的、两端不在表结构里的、指向视图的、碰到遮罩列的关系—— + 要让人看得见为什么它没有结论。masked 是遮罩的列（小写）。""" + if not isinstance(meta, dict) or meta.get("is_view"): + return [] + notes = notes or {} + checks: list[PlannedCheck] = [] for rel in _relation_candidates(notes): - rid = str(rel["id"]) - checks.append(_Check("relation", _relation_target(rel), f"relations.{rid}", - lambda rel=rel: _check_relation(cx, table, rel, out, result), - lambda rid=rid: out.keep_relations.add(rid))) + skip: tuple[str, str, bool] | None = None + try: + _relation_ends(table, rel, tables, masked) + except _Skip as e: + skip = (e.reason, e.detail, e.carry) + checks.append(PlannedCheck("relation", _relation_target(rel), f"relations.{rel['id']}", skip, relation=rel)) for column in _code_columns(meta, notes): - checks.append(_Check("codes", column, f"columns.{column}.codes", - lambda column=column: _check_codes(cx, table, column, out, result), - lambda column=column: out.keep_codes.add(column))) + skip = (("masked", f"{table}.{column} 在数据源设置中被遮罩，不取值", False) + if column.lower() in masked else None) + checks.append(PlannedCheck("codes", column, f"columns.{column}.codes", skip, column=column)) dates = [str(c["name"]) for c in meta.get("columns") or [] if isinstance(c, dict) and _is_date_column(c)] for column in dates: - if cx.is_masked(column): - result.skipped.append(ProfileSkip("date", column, "masked", - f"{table}.{column} 在数据源设置中被遮罩，不取最小值和最大值")) - usable = [c for c in dates if not cx.is_masked(c)] + if column.lower() in masked: + checks.append(PlannedCheck("date", column, None, + ("masked", f"{table}.{column} 在数据源设置中被遮罩，不取最小值和最大值", False), + column=column)) + usable = [c for c in dates if c.lower() not in masked] if usable: - checks.append(_Check("date", "、".join(usable), "business_date" if len(dates) == 1 else None, - lambda: _check_dates(cx, table, usable, len(dates), out, result), - lambda: setattr(out, "keep_business_date", True))) + checks.append(PlannedCheck("date", "、".join(usable), "business_date" if len(dates) == 1 else None, + columns=tuple(usable), date_total=len(dates))) return checks -async def _profile_table(cx: _Context, table: str, notes: dict[str, Any], result: TableProfile) -> _TableOutcome: - out = _TableOutcome() - if cx.tables[table].get("is_view"): - result.skipped.append(ProfileSkip("table", table, "view", "视图没有统计信息，剖析可能触发整个视图的计算，已跳过")) - return out - checks = _plan(cx, table, notes, out, result) - for i, check in enumerate(checks): +def profile_checks(table: str, meta: dict[str, Any] | None, notes: dict[str, Any] | None, *, tables: dict[str, Any], + masked: set[str]) -> int: + """这张表要做几项检查（按规则不查的不算）：0 表示没有可剖析的项；视图、表结构里没有的表是 0。""" + return sum(1 for c in plan_checks(table, meta, notes, tables=tables, masked=masked) if c.skip is None) + + +def has_pending_relations(notes: dict[str, Any] | None) -> bool: + """这张表还有没有待核实的关系：要核对的关系里有还是推断（proposed）的。挑表时这样的表排在前面。""" + return any(r.get("status") == "proposed" for r in _relation_candidates(notes)) + + +def pending_relation_paths(notes: dict[str, Any] | None) -> set[str]: + """待核实（proposed）关系的路径（relations.<编号>）。上次剖析的检查项里没有它们，就是剖析之后冒出了新的待核实关系。""" + return {f"relations.{r['id']}" for r in _relation_candidates(notes) if r.get("status") == "proposed"} + + +# ========================================================================== +# 入口：剖析一张表 +# ========================================================================== + +ESTIMATING = "正在估算 {table} 的行数" + + +def _current_text(table: str, check: PlannedCheck) -> str: + """进行中时正在做哪一项的整句（逐表一行的最后一栏）。""" + if check.kind == "relation": + return f"正在核对 {table}.{check.target}" + if check.kind == "codes": + return f"正在取 {table}.{check.column} 的取值分布" + if len(check.columns) == 1: + return f"正在取 {table}.{check.columns[0]} 的最小值和最大值" + return f"正在取 {table} 中 {check.target} 的最小值和最大值" + + +def _keep(out: _TableOutcome, check: PlannedCheck) -> None: + """这一项没查成：目录里剖析以前对它写的结论标记为原样保留。""" + if check.kind == "relation" and check.relation is not None: + out.keep_relations.add(str(check.relation["id"])) + elif check.kind == "codes" and check.column: + out.keep_codes.add(check.column) + elif check.kind == "date" and check.columns: + out.keep_business_date = True + + +def _close_open(result: TableResult, plan: list[PlannedCheck], out: _TableOutcome, reason: str, detail: str) -> int: + """还没结束的项（排着的、正在做的）都记为跳过、原因 reason，旧结论原样保留。返回记了几项。""" + n = 0 + for check, planned in zip(result.checks, plan): + if check["state"] in ("pending", "running"): + check.update(state="skipped", reason=reason, detail=detail) + result.skipped.append(ProfileSkip(check["kind"], check["target"], reason, detail, check["path"])) + _keep(out, planned) + n += 1 + return n + + +def _stop_detail(cx: _TableCx, reason: str) -> str: + return _STOP_DETAIL[reason].format(max_queries=cx.settings.table_max_queries, max_s=_fmt(cx.settings.table_max_s)) + + +async def _execute(cx: _TableCx, table: str, check: PlannedCheck, out: _TableOutcome) -> None: + if check.kind == "relation": + await _check_relation(cx, table, check.relation or {}, out, cx.result) + elif check.kind == "codes": + await _check_codes(cx, table, str(check.column), out, cx.result) + else: + await _check_dates(cx, table, list(check.columns), check.date_total, out, cx.result) + + +async def _progress(cx: _TableCx) -> None: + if cx.hooks.progress is not None: + await cx.hooks.progress(cx.result) + + +async def _run_checks(cx: _TableCx, table: str, plan: list[PlannedCheck], runnable: list[int], + out: _TableOutcome) -> None: + """第一条查询估行数（确知是空表就整张表跳过），再逐项执行。预算用完、任务止损时剩下的项记原因、保留旧结论。""" + result = cx.result + result.current = ESTIMATING.format(table=table) + await _progress(cx) + try: + size = await cx.size(table, result) + except _Stop as e: + _close_open(result, plan, out, e.reason, _stop_detail(cx, e.reason)) + if e.reason in ("failed", "slow"): + result.error = cx.job.runner.error + return + finally: + result.current = None + if size.rows == 0: + _close_open(result, plan, out, "no_data", f"{table} 是空表，无法剖析") + result.status, result.error = "skipped", EMPTY_ERROR + return + for i in runnable: + planned, check = plan[i], result.checks[i] + check["state"] = "running" + result.current = _current_text(table, planned) + cx.check = check + await _progress(cx) try: - await check.run() + await _execute(cx, table, planned, out) + check["state"] = "done" except _Skip as e: - result.skipped.append(ProfileSkip(check.kind, check.target, e.reason, e.detail, check.path)) + check.update(state="skipped", reason=e.reason, detail=e.detail) + result.skipped.append(ProfileSkip(planned.kind, planned.target, e.reason, e.detail, planned.path)) if e.carry: - check.keep() + _keep(out, planned) except _Stop as e: - detail = _STOP_DETAIL[e.reason].format(max_queries=cx.settings.max_queries, - max_total_s=_fmt(cx.settings.max_total_s)) - for rest in checks[i:]: - result.skipped.append(ProfileSkip(rest.kind, rest.target, e.reason, detail, rest.path)) - rest.keep() + cx.check = None + _close_open(result, plan, out, e.reason, _stop_detail(cx, e.reason)) + if e.reason in ("failed", "slow"): + result.error = cx.job.runner.error break - if table in cx.sizes: - result.row_estimate = cx.sizes[table] - return out + finally: + cx.check = None + result.current = None + await _progress(cx) -async def _write(session: AsyncSession, source_id: str, table: str, out: _TableOutcome, result: TableProfile, *, +async def _own_session(fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: + async with SessionLocal() as session: + return await fn(session, *args) + + +async def _read_notes(session: AsyncSession, source_id: str, table: str) -> dict[str, Any]: + entry = await catalog.read_entry(session, source_id, table) + return entry.notes if entry else {} + + +async def profile_table(cx: ProfileContext, table: str, *, + db: Callable[..., Awaitable[Any]] | None = None, hooks: TableHooks | None = None, + actor: str | None = None) -> TableResult: + """剖析一张表并把结论写进目录，返回这张表的结果（TableResult）。 + + - 表结构里没有、视图：skipped，不发查询。 + - 先列全检查项（plan_checks，读的是这时目录里的最新内容），按规则不查的直接记跳过；一项可查的都没有：skipped + 「没有可剖析的项」，不发查询。 + - 第一条查询估行数，确知是空表：skipped「空表」。 + - 逐项执行：这张表的查询次数、时长用完（TableBudget）只停这张表，剩下的项记 budget / deadline；任务止损（cx.runner） + 停下时剩下的项记 failed / slow，error 是止损的那一句。 + - 停止（任务被 cancel）：正在跑的那条查询由数据库按单条时限截止；已做完的项照常写进目录，没做的记 cancelled、保留旧 + 结论；写完再把停止传下去（CancelledError）。 + - 整张表结束把结论并进目录（乐观锁，撞上别人刚改过重读重并，三次仍冲突记 failed），和 hooks.recorded 在同一个事务里 + 提交。 + + db 是执行库操作的：db(fn, *args) 开一个会话调 fn(session, *args)。后台任务给它的 SerialWriter（一条一条、shield）， + 不给就各开一个会话。快照被改动（SnapshotTampered）原样抛出，不写目录：整个任务都该停。 + """ + hooks = hooks or TableHooks() + db = db or _own_session + started = time.monotonic() + result = TableResult(table=table) + meta = cx.tables.get(table) + if not isinstance(meta, dict) or meta.get("is_view"): + reason = "view" if isinstance(meta, dict) else "missing" + why = VIEW_ERROR if reason == "view" else catalog.missing_table_error(table) + result.status, result.error = "skipped", why + result.skipped.append(ProfileSkip("table", table, reason, why)) + await db(_record, cx.source.id, table, None, result, actor, hooks.recorded) + return result + notes = await db(_read_notes, cx.source.id, table) + plan = plan_checks(table, meta, notes, tables=cx.tables, masked=cx.masked) + out = _TableOutcome() + tcx = _TableCx(cx, table, result, hooks) + result.checks = [p.as_check() for p in plan] + for planned in plan: + if planned.skip is not None: + result.skipped.append(ProfileSkip(planned.kind, planned.target, planned.skip[0], planned.skip[1], + planned.path)) + if planned.skip[2]: + _keep(out, planned) + runnable = [i for i, p in enumerate(plan) if p.skip is None] + stopped: asyncio.CancelledError | None = None + try: + if not runnable: + result.status, result.error = "skipped", NOTHING_ERROR + result.skipped.append(ProfileSkip("table", table, "nothing", NOTHING_ERROR)) + else: + await _run_checks(tcx, table, plan, runnable, out) + except asyncio.CancelledError as e: + stopped = e + why = hooks.stop_text() if hooks.stop_text is not None else STOPPED + done = sum(1 for c in result.checks if c["state"] == "done") + rest = _close_open(result, plan, out, "cancelled", f"{why}，未检查") + result.error = STOPPED_TABLE.format(why=why, done=done, rest=rest) + except SnapshotTampered: + raise + except Exception as e: # noqa: BLE001 - 一张表出了意外不拖垮别的表：照实记下，做完的项照常写 + logger.exception("剖析表 %s 时出错", table) + line = _brief(e) + _close_open(result, plan, out, "error", TABLE_ERROR.format(line=line)) + result.status, result.error = "failed", TABLE_ERROR.format(line=line) + result.current = None + result.row_estimate = cx.sizes.get(table) + result.elapsed_ms = int((time.monotonic() - started) * 1000) + await db(_record, cx.source.id, table, out, result, actor, hooks.recorded) + if stopped is not None: + raise stopped + return result + + +def _settle_status(result: TableResult) -> None: + """表的状态：skipped、failed 的不变；有运行原因没查成的项是 partial，否则 ok。""" + if result.status in ("skipped", "failed"): + return + result.status = "partial" if result.unfinished else "ok" + + +async def _record(session: AsyncSession, source_id: str, table: str, out: _TableOutcome | None, result: TableResult, + actor: str | None, recorded: Callable[[AsyncSession, TableResult], Awaitable[None]] | None) -> None: + """整张表结束：把结论并进目录（out 为 None 的不写：表结构里没有、视图），定下表的状态，再交给 recorded（同一个会话）。 + 修改时间取这一次写入的时间。 + + 目录和 recorded 写的这张表的结果在同一个事务里提交：进程在中间没了，两样都没写进去，下次启动收尾时这张表做完的项 + 照实记「没写进目录」（catalog_profile_jobs._close_tasks），不会出现目录写了、剖析记录却说没写的情形。""" + if out is not None: + try: + await _write(session, source_id, table, out, result, at=catalog.now_iso(), actor=actor) + except Exception as e: # noqa: BLE001 - 这一张写不进去不影响别的表 + logger.exception("剖析结果写入表 %s 的数据目录时出错", table) + await session.rollback() + result.error = WRITE_ERROR.format(line=_brief(e)) + result.status = "failed" + result.findings = [] + result.added = result.updated = result.removed = 0 + _settle_status(result) + if recorded is not None: + await recorded(session, result) + await session.commit() + + +async def _write(session: AsyncSession, source_id: str, table: str, out: _TableOutcome, result: TableResult, *, at: str, actor: str | None) -> None: - """并入并写入（乐观锁）。撞上别人刚改过就重读重并，最多三次。""" + """并入并写入（乐观锁）。撞上别人刚改过就重读重并，最多三次；三次仍冲突这张表记 failed。写进事务、不提交（见 _record）。""" for _attempt in range(3): entry = await catalog.read_entry(session, source_id, table) version = entry.version if entry else 0 @@ -1214,96 +1720,21 @@ async def _write(session: AsyncSession, source_id: str, table: str, out: _TableO if entry is None and not notes: return # 什么也没得出，不建空目录 try: - written = await catalog.write_entry(session, source_id, table, notes, if_version=version, actor=actor) + written = await catalog.write_entry(session, source_id, table, notes, if_version=version, actor=actor, + commit=False) except catalog.CatalogConflict: continue result.version = written.version return - result.error = "这张表的数据目录正被其他人修改，剖析结果未能写入，请稍后重新剖析" + result.error = CONFLICT_ERROR + result.status = "failed" result.findings = [] result.added = result.updated = result.removed = 0 -async def _pick_tables(session: AsyncSession, source: Any, tables: list[str] | None, - entries: dict[str, catalog.CatalogEntry], report: ProfileReport) -> list[str]: - cache = getattr(source, "schema_cache", None) or {} - all_tables = cache.get("tables") or {} - if tables: - picked: list[str] = [] - for name in dict.fromkeys(str(t) for t in tables): - key = catalog.resolve_table_name(cache, name) - if key is None: - report.tables.append(TableProfile(table=name, error=f"表结构里没有 {name}，请先重新探查结构")) - elif key not in picked: - picked.append(key) - return picked - usage = await catalog.table_usage(session, source) - order = {name: i for i, name in enumerate(all_tables)} - candidates = [name for name, meta in all_tables.items() - if not (meta or {}).get("is_view") and name in entries - and _relation_candidates(entries[name].notes)] - # 还有待核实（推断）关系的表排在前面：剖析就是来核实它们的；关系都核实过、人工确认过的再剖析只是补覆盖率。 - # 同样待核实的按使用次数，再按表结构顺序。返回全部候选，空表由 _skip_empty 估过行数再跳过 - return sorted(candidates, key=lambda n: (not _pending(entries[n].notes), -usage.get(n, 0), order[n])) - - -def _pending(notes: dict[str, Any] | None) -> bool: - """这张表还有没有待核实的关系：要核对的关系里有还是推断（proposed）的。""" - return any(r.get("status") == "proposed" for r in _relation_candidates(notes)) - - -async def _skip_empty(cx: _Context, candidates: list[str], report: ProfileReport) -> list[TableProfile]: - """不指定表时：按顺序估候选表的行数，确知是空表的跳过（记进 report.empty_tables），凑够 PROFILE_DEFAULT_TABLES - 张为止。以前不估，所有表使用次数都是 0 时按表结构顺序取前 10 张，结果大半是空表、一条关系都核实不了。 - - 估行数的查询算进这张表的查询数，行数留在 cx.sizes 里，剖析时不再查。最多估 PROFILE_PROBE_TABLES 张、最多用掉 - 查询次数上限的一半；超过了、或者整次剖析已经停下，剩下的候选不估、照顺序排上（真是空表的，剖析时各项检查会 - 记下「空表，无法剖析」）。 - """ - picked: list[TableProfile] = [] - probed = 0 - for name in candidates: - if len(picked) >= PROFILE_DEFAULT_TABLES: - break - result = TableProfile(table=name) - if (cx.run.stopped is None and probed < PROFILE_PROBE_TABLES - and cx.run.used < cx.settings.max_queries // 2): - probed += 1 - before = cx.run.used - try: - size = await cx.size(name, result) - except _Stop: - size = None - result.queries = cx.run.used - before - if size is not None and size.rows == 0: - report.empty_tables.append(name) - continue - picked.append(result) - return picked - - -async def profile_catalog(session: AsyncSession, row: Any, source: Any, *, tables: list[str] | None = None, - actor: str | None = None, settings: ProfileSettings | None = None) -> ProfileReport: - """剖析这些表并把结论写进数据目录，同步完成。 - - - row：数据源记录（开关、遮罩按它的 options）；source：用来查询和读表结构的源（上传源是绑定当前快照的视图）。 - - settings 不给就按 row.options 读；没开启抛 ProfileDisabled，这个源正在剖析抛 ProfileBusy。 - - tables 为空时按使用次数取前 PROFILE_DEFAULT_TABLES 张目录里有待核实关系的表。 - - 每张表做完就写入，后面的表停下（预算、总时长）不影响前面已经写进去的。 - """ - settings = ensure_enabled(row, settings) - if source.id in _RUNNING: - raise ProfileBusy(f"数据源「{getattr(row, 'name', '')}」正在进行数据剖析，请等待完成后再试") - _RUNNING.add(source.id) - try: - return await _profile(session, row, source, tables, actor, settings) - finally: - _RUNNING.discard(source.id) - - def ensure_enabled(row: Any, settings: ProfileSettings | None = None) -> ProfileSettings: """这个数据源能不能剖析：数据源停用了、或者没开启数据剖析，抛 ProfileDisabled（message 给人看）。 - 返回要用的设置（不给就按 row.options 读）。接口在做别的检查之前先调它：开关是第一道门。""" + 返回要用的设置（不给就按 row.options 读）。建任务之前先调它：开关是第一道门；任务每张表开始前也按它重新看一遍。""" settings = settings or profile_settings(getattr(row, "options", None)) name = getattr(row, "name", "") or "" if getattr(row, "enabled", True) is False: @@ -1313,38 +1744,6 @@ def ensure_enabled(row: Any, settings: ProfileSettings | None = None) -> Profile return settings -async def _profile(session: AsyncSession, row: Any, source: Any, tables: list[str] | None, actor: str | None, - settings: ProfileSettings) -> ProfileReport: - at = catalog.now_iso() - report = ProfileReport(profiled_at=at, settings=settings) - dialect = SqlDialect(source.kind) - run = _Runner(source, settings, dialect) - cx = _Context(source=source, settings=settings, dialect=dialect, run=run, - tables=(getattr(source, "schema_cache", None) or {}).get("tables") or {}, - masked={c.lower() for c in data_engine.masked_columns(getattr(row, "options", None))}) - entries = await catalog.read_catalog(session, source.id) - picked = await _pick_tables(session, source, tables, entries, report) - results = [TableProfile(table=t) for t in picked] if tables else await _skip_empty(cx, picked, report) - for result in results: - table = result.table - report.tables.append(result) - if run.stopped: - detail = _STOP_DETAIL[run.stopped].format(max_queries=settings.max_queries, - max_total_s=_fmt(settings.max_total_s)) - result.skipped.append(ProfileSkip("table", table, run.stopped, detail)) - continue - before = run.used - entry = entries.get(table) - out = await _profile_table(cx, table, entry.notes if entry else {}, result) - result.queries += run.used - before - await _write(session, source.id, table, out, result, at=at, actor=actor) - report.queries_used = run.used - report.stopped = run.stopped - logger.info("数据剖析 源=%s 表=%s 查询=%d 停止=%s 署名=%s", source.id, - ",".join(t.table for t in report.tables), run.used, run.stopped, actor) - return report - - # ========================================================================== # 核对模型提议的有效行过滤：排除了多少行 # ========================================================================== @@ -1399,7 +1798,7 @@ async def check_valid_filter(source: Any, table: str, meta: dict[str, Any] | Non settings: ProfileSettings | None = None) -> FilterCheck: """开了数据剖析的源：数一数这条有效行过滤排除了多少行（模型起草时提议的那条，data/catalog_draft.py 调）。从不抛异常。 - 经剖析的只读守卫（_Runner：guard.check(readonly=True)、单条时限、查询次数上限）发一条查询：只取条件里用到的列， + 经单条查询的入口（run_single_query：guard.check(readonly=True)、单条时限）发一条查询：只取条件里用到的列， 先取前 max(sample_size, max_scan_rows) 行（大表不做整表扫描），再数总行数和不满足条件的行数（条件为 NULL 也算排除， 和 WHERE 一样）。数到的不满前那么多行，就是整张表。 @@ -1437,14 +1836,10 @@ async def check_valid_filter(source: Any, table: str, meta: dict[str, Any] | Non cols = ", ".join(dialect.quote(c) for c in parsed.columns) inner = dialect.limit(f"SELECT {cols} FROM {dialect.table(meta, table)}", cap) sql = f"SELECT COUNT(*) AS n, SUM(CASE WHEN {cond} THEN 0 ELSE 1 END) AS x FROM ({inner}) s" - run = _Runner(source, settings, dialect) try: - res = await run(sql) - except _Skip as e: + res = await run_single_query(source, sql, settings=settings, kind=kind) + except QueryFailed as e: return _unchecked(e.detail.removesuffix("，已跳过")) - except _Stop as e: - return _unchecked(_STOP_DETAIL[e.reason].format(max_queries=settings.max_queries, - max_total_s=_fmt(settings.max_total_s)).removesuffix("，未检查")) except SnapshotTampered as e: return _unchecked(str(e)) row = res.rows[0] if res.rows else [None, None] @@ -1465,9 +1860,12 @@ async def check_valid_filter(source: Any, table: str, meta: dict[str, Any] | Non return FilterCheck(True, note, rows, excluded, sampled) -__all__ = ["CODES_MAX", "FILTER_CHECKED", "FILTER_NOT_ENABLED", "FILTER_SAMPLED", "FILTER_UNCHECKED", "FilterCheck", - "IN_CHUNK", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES", - "PROFILE_OPTION", "PROFILE_PROBE_TABLES", "ProfileBusy", "ProfileDisabled", "ProfileReport", - "ProfileSettings", "ProfileSkip", "SqlDialect", "TableProfile", "TableSize", "check_valid_filter", - "ensure_enabled", "estimate_size", "excluded_share", "profile_catalog", "profile_settings", - "profile_settings_problem", "profile_settings_rejection", "sql_literal"] + +__all__ = ["CODES_MAX", "EMPTY_ERROR", "FILTER_CHECKED", "FILTER_NOT_ENABLED", "FILTER_SAMPLED", "FILTER_UNCHECKED", + "FilterCheck", "IN_CHUNK", "LEGACY_KEYS", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "NOTHING_ERROR", + "PROFILE_OPTION", "PlannedCheck", "ProfileContext", "ProfileDisabled", "ProfileSettings", "ProfileSkip", + "QueryFailed", "SourceRunner", "SqlDialect", "TableBudget", "TableHooks", "TableResult", "TableSize", + "UNFINISHED_REASONS", "VIEW_ERROR", "check_valid_filter", "ensure_enabled", "estimate_size", "excluded_share", + "has_pending_relations", "make_context", "masked_of", "migrate_settings", "pending_relation_paths", + "plan_checks", "profile_checks", "profile_settings", "profile_settings_problem", "profile_settings_rejection", + "profile_table", "run_single_query", "sql_literal", "summarize", "without_legacy"] diff --git a/backend/app/data/catalog_profile_jobs.py b/backend/app/data/catalog_profile_jobs.py new file mode 100644 index 00000000..d1f39f58 --- /dev/null +++ b/backend/app/data/catalog_profile_jobs.py @@ -0,0 +1,890 @@ +"""数据目录的后台剖析任务（/api/datasources/{id}/catalog/profile-jobs）。 + +以前剖析是一次同步长请求（POST /catalog/profile，已去掉）：预算按整次剖析算，选 50 张表只做得完前两三张；进行中只有一条 +按时长走的横条、不能停；关掉页面报告就没了，对业务库查了什么无从查。现在和起草一样是后台任务（生命周期在 +data/catalog_job_runtime.py），逐表剖析（引擎在 data/catalog_profile.py）： + +- **建任务**（create_job）：范围三种——指定的表（selected：按给的顺序去重；表结构里没有、视图、没有可剖析的项照样建进 + 任务、状态 skipped）、需要剖析的表（needed：从没剖析过、上次剖析没做完、剖析之后冒出了新的待核实关系，见 due_reason）、 + 全部表（all）。后两种只挑表结构里、不是视图、有可剖析的项的表，有待核实关系的在前，再按使用次数、表结构顺序；挑出 0 张 + 抛 ProfileNothing。剖析设置随任务记下快照，每张表的检查项按当时的目录先列一遍。同一个源同时只有一个排队或进行中的 + 剖析任务（ProfileBusy，说法里写明进行到第几张；部分唯一索引兜底）；和同源的起草任务按建立的先后排队（每个源一条队）， + 起草在前时 waiting_for 是 "draft"。建任务时顺手清掉超出保留数的旧任务（prune）。 +- **在后台剖析**（manager.start）：轮到了才开始。每张表开始前重读数据源——停用了、关掉了剖析就停下，剩下的表 cancelled、 + error 写明；遮罩按最新的设置——再逐表调 catalog_profile.profile_table（它读的是那时目录里的最新内容：起草刚提的关系也 + 核对）。检查项、正在做哪一项、查询数、发过的查询随时落库；整张表的结果和写目录在同一个事务里，停止时也不会被打断。 +- **任务级止损**（替代以前的整次总量上限）：连续 3 条查询失败、最近 10 条里 7 条超时，整个任务停下（stop_reason、error）， + 正在做的那张表写入已做完的项，后面的表 cancelled、error 带同一句原因；最近 6 条里 4 条超时时 notice 有一句提醒，好转后 + 清掉。任务最后的状态按逐表算：有 failed、partial、cancelled 的表是 partial，否则 succeeded；任务整体出错（数据源没了、 + 快照被改动、意外异常）是 failed。 +- **停止**（cancel）：排队的直接 cancelled；进行中的停掉正在等的那条查询（库那边按单条时限截止），那张表做完的项照常写进 + 目录、没做的保留旧结论，后面的表 cancelled，任务 cancelled。 +- **重试**（retry_tables + create_job）：新任务，只含原任务里 failed、partial、cancelled 的表（按原来的顺序），范围 selected， + retry_of 记原任务；设置取数据源当前的。 +- **服务重启**（startup）：排队、进行中的标 interrupted，没结束的表 cancelled、error「服务重启，剖析中断」（进程没了时正在 + 剖析的那张表，查完的项也没写进目录，照实写明，见文末）。不自动续跑： + 复制出来的数据目录启动时不能自己去查业务库，界面让人点「重试」。关停时（manager.shutdown）停掉进行中的，同样标中断。 + 启动时还一次性迁移剖析设置的旧键（catalog_profile.migrate_settings）。 +- **保留**（prune）：每个源保留最近 KEEP_JOBS 个结束了的剖析任务，更早的删掉；某张表「上次剖析」所在的任务不删，表详情的 + 「上次剖析」一直在。 +- **读**：job_dict（任务和逐表一行）、task_detail（一张表的检查项、发现、跳过的项、发过的查询）、last_profiles / last_detail + （表清单每行、表详情的「上次剖析」）、due_reason（这张表要不要剖析、为什么）。「上次剖析」是真正开始过的最近一次（见 + _ranked）：没开始就停了的、还排着的那一项不挡掉上一次真正剖析的行数、发过的查询。 + +没结束的表收尾时（停止、中断、整体出错）检查项里还排着的、正在做的记为跳过（原因 cancelled 或 error），不留「排队中」的 +检查项在一张已经结束的表上。目录只在一张表结束时和它的结果一起写（同一个事务）：收尾时还没结束的表，查完的项结论也没 +写进目录（进程被杀、快照被改动），同样记为跳过、写明「查完了但结论没有写进数据目录」，算没做完，重试时带上这张表。 +停止、关停这类能收尾的停下不走这条路：正在剖析的那张表做完的项照常写进目录（catalog_profile.profile_table）。 +""" +from __future__ import annotations + +import logging +import time +from typing import Any + +from sqlalchemy import case, delete, func, or_, select, update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import catalog, catalog_profile +from app.data import catalog_job_runtime as runtime +from app.data.catalog_job_runtime import ACTIVE, FINISHED, OPEN, RETRYABLE, STOP_GRACE_S, TASK_DONE, TASK_STATUSES +from app.data.catalog_profile import ( + NOTHING_ERROR, UNFINISHED_REASONS, VIEW_ERROR, ProfileSettings, ProfileSkip, TableHooks, TableResult, +) +from app.data.engine import SnapshotTampered +from app.db.base import SessionLocal, new_id, utcnow +from app.db.models import CatalogDraftJob, CatalogProfileJob, CatalogProfileTask, DataSource + +logger = logging.getLogger(__name__) + +#: 一个任务最多几张表 +MAX_TABLES = 500 +#: 每个源保留几个结束了的剖析任务（某张表最近一次剖析所在的任务另外保留） +KEEP_JOBS = 50 + +INTERRUPTED_ERROR = "服务重启，剖析中断" +STOPPED_ERROR = catalog_profile.STOPPED +UNFINISHED_ERROR = "剖析结束时这张表还没有收尾，请重试这张表" +JOB_ERROR = "剖析任务出错：{line}" +SOURCE_GONE = "数据源不存在，可能已被删除" +SOURCE_DISABLED = "数据源「{name}」已停用，剖析已停止" +PROFILE_TURNED_OFF = "数据源「{name}」已关闭数据剖析，剖析已停止" +#: 没结束的检查项在收尾时的说明（{why} 是停止、中断、出错的原因） +_UNCHECKED = "{why}，未检查" +#: 查完了、结论却没写进目录的检查项：这张表没走到整张表结束、写目录那一步（进程没了、任务整体出错） +_UNSAVED = "{why}，查完了但结论没有写进数据目录" +_CHECK_OPEN = ("pending", "running") + + +def unsaved_error(why: str, done: int, rest: int) -> str: + """没走到写目录那一步的表的整句原因：查完的几项没写进目录，剩下几项没查。""" + text = f"{why}：已查完的 {done} 项没有写进数据目录" + return text + f"，剩下 {rest} 项没查" if rest else text + + +class ProfileBusy(Exception): + """这个数据源已经有排队或进行中的剖析任务。message 给人看（写明进行到第几张）。""" + + +class ProfileFinished(Exception): + """要停止的剖析任务已经结束了。""" + + +class ProfileNothing(Exception): + """按范围挑出 0 张表。message 说明原因、建议怎么办。""" + + +class NothingToRetry(Exception): + """原任务里没有需要重试的表。""" + + +# ========================================================================== +# 读任务 +# ========================================================================== + + +def _iso(value: Any) -> str | None: + return value.isoformat() if value is not None else None + + +def _empty_summary() -> dict[str, int]: + return catalog_profile.summarize([]) + + +async def active_job(session: AsyncSession, source_id: str) -> CatalogProfileJob | None: + """排队或进行中的那个（同一个源最多一个）。""" + return (await session.execute( + select(CatalogProfileJob).where(CatalogProfileJob.source_id == source_id, + CatalogProfileJob.status.in_(ACTIVE)) + .order_by(CatalogProfileJob.created_at.desc()).limit(1) + )).scalar_one_or_none() + + +async def latest_job(session: AsyncSession, source_id: str) -> CatalogProfileJob | None: + """最近建的一个（任何状态）。""" + return (await session.execute( + select(CatalogProfileJob).where(CatalogProfileJob.source_id == source_id) + .order_by(CatalogProfileJob.created_at.desc()).limit(1) + )).scalar_one_or_none() + + +async def load(session: AsyncSession, source_id: str, job_id: str) -> CatalogProfileJob | None: + """这个源的这个任务；不存在、不属于这个源都是 None。""" + job = await session.get(CatalogProfileJob, job_id) + return job if job is not None and job.source_id == source_id else None + + +#: 逐表一行要的列：不读发现、跳过的项、发过的查询（任务进行中界面每秒都要取一次） +_ROW_COLUMNS = (CatalogProfileTask.seq, CatalogProfileTask.table_name, CatalogProfileTask.status, + CatalogProfileTask.checks, CatalogProfileTask.current, CatalogProfileTask.queries, + CatalogProfileTask.elapsed_ms, CatalogProfileTask.row_estimate, CatalogProfileTask.summary, + CatalogProfileTask.skipped_count, CatalogProfileTask.unfinished, CatalogProfileTask.added, + CatalogProfileTask.updated, CatalogProfileTask.removed, CatalogProfileTask.version, + CatalogProfileTask.error, CatalogProfileTask.started_at, CatalogProfileTask.finished_at) + + +async def task_rows(session: AsyncSession, job_id: str) -> list[Any]: + return list((await session.execute( + select(*_ROW_COLUMNS).where(CatalogProfileTask.job_id == job_id).order_by(CatalogProfileTask.seq) + )).all()) + + +def _by_kind(checks: list[Any]) -> dict[str, list[int]]: + """检查项按种类的进度：{relation: [结束了的, 总数], codes: […], date: […]}。结束了的含跳过的。""" + out = {"relation": [0, 0], "codes": [0, 0], "date": [0, 0]} + for check in checks: + if not isinstance(check, dict) or check.get("kind") not in out: + continue + pair = out[check["kind"]] + pair[1] += 1 + if check.get("state") in ("done", "skipped"): + pair[0] += 1 + return out + + +def task_row(task: Any) -> dict[str, Any]: + """逐表一行（CatalogProfileTaskRow）。task 是 CatalogProfileTask，或者 task_rows 取的那几列。""" + checks = [c for c in (task.checks or []) if isinstance(c, dict)] + return { + "seq": task.seq, + "table_name": task.table_name, + "status": task.status, + "checks_total": len(checks), + "checks_done": sum(1 for c in checks if c.get("state") in ("done", "skipped")), + "checks_by_kind": _by_kind(checks), + "current": task.current if task.status == "running" else None, + "queries": task.queries or 0, + "elapsed_ms": task.elapsed_ms or 0, + "row_estimate": task.row_estimate, + "summary": {**_empty_summary(), **(task.summary or {})}, + "skipped_count": task.skipped_count or 0, + "unfinished": task.unfinished or 0, + "added": task.added or 0, + "updated": task.updated or 0, + "removed": task.removed or 0, + "version": task.version or 0, + "error": task.error, + "started_at": _iso(task.started_at), + "finished_at": _iso(task.finished_at), + } + + +def _counts(statuses: list[str]) -> dict[str, int]: + counts = dict.fromkeys(TASK_STATUSES, 0) + for status in statuses: + counts[status] = counts.get(status, 0) + 1 + return counts + + +def job_dict(job: CatalogProfileJob, tasks: list[Any], *, waiting_for: str | None = None) -> dict[str, Any]: + """接口返回的任务（CatalogProfileJob）：逐表按剖析顺序；计数、合计都按逐表的算。""" + rows = [task_row(t) for t in tasks] + counts = _counts([r["status"] for r in rows]) + summary = _empty_summary() + for r in rows: + for key in summary: + summary[key] += int(r["summary"].get(key) or 0) + return { + "id": job.id, + "source_id": job.source_id, + "status": job.status, + "scope": job.scope, + "settings": ProfileSettings.from_dict(job.settings).to_dict(), + "created_at": _iso(job.created_at), + "started_at": _iso(job.started_at), + "finished_at": _iso(job.finished_at), + "created_by": job.created_by, + "retry_of": job.retry_of, + "waiting_for": waiting_for, + "total": len(rows), + "done": sum(counts.get(s, 0) for s in TASK_DONE), + "counts": counts, + "queries": job.queries or 0, + "timeouts": job.timeouts or 0, + "errors": job.errors or 0, + "stop_reason": job.stop_reason, + "notice": job.notice, + "error": job.error, + "added": sum(r["added"] for r in rows), + "updated": sum(r["updated"] for r in rows), + "removed": sum(r["removed"] for r in rows), + "summary": summary, + "tasks": rows, + } + + +async def waiting_for(session: AsyncSession, job: CatalogProfileJob) -> str | None: + """排队中的剖析任务在等什么：排在同源的起草任务（或同步起草）后面时是 "draft"，否则 None。 + + 先看这个源的队（进程内，排在它前面的是谁）；任务刚建、后台还没排进队时看库：同源有比它先建的、排队或进行中的起草 + 任务，或者正在同步起草。""" + if job.status != "queued": + return None + ahead = runtime.lanes.ahead(job.source_id, job.id) + if ahead is not None: + return "draft" if any(kind in ("draft", "sync_draft") for kind in ahead) else None + from app.data import catalog_jobs + + if catalog_jobs.sync_drafting(job.source_id): + return "draft" + earlier = (await session.execute( + select(CatalogDraftJob.id).where(CatalogDraftJob.source_id == job.source_id, + CatalogDraftJob.status.in_(ACTIVE), + CatalogDraftJob.created_at <= job.created_at).limit(1) + )).first() + return "draft" if earlier is not None else None + + +async def job_out(session: AsyncSession, job: CatalogProfileJob | None) -> dict[str, Any] | None: + if job is None: + return None + return job_dict(job, await task_rows(session, job.id), waiting_for=await waiting_for(session, job)) + + +async def job_brief(session: AsyncSession, job: CatalogProfileJob | None) -> dict[str, Any] | None: + """目录清单顶上的进度（CatalogActiveProfileJob）：{id, status, total, done, waiting_for}。""" + if job is None: + return None + rows = (await session.execute( + select(CatalogProfileTask.status, func.count()).where(CatalogProfileTask.job_id == job.id) + .group_by(CatalogProfileTask.status) + )).all() + counts = {status: n for status, n in rows} + return {"id": job.id, "status": job.status, "total": sum(counts.values()), + "done": sum(counts.get(s, 0) for s in TASK_DONE), "waiting_for": await waiting_for(session, job)} + + +def task_detail(job: CatalogProfileJob, task: CatalogProfileTask) -> dict[str, Any]: + """一张表的明细（CatalogProfileTaskDetail）：逐表一行，加上检查项、发现、跳过的项、日期范围、发过的查询。""" + return { + **task_row(task), + "job_id": job.id, + "job_status": job.status, + "created_by": job.created_by, + "checks": [c for c in task.checks or [] if isinstance(c, dict)], + "findings": list(task.findings or []), + "skipped": list(task.skipped or []), + "date_ranges": list(task.date_ranges or []), + "query_log": list(task.query_log or []), + } + + +async def load_task(session: AsyncSession, job: CatalogProfileJob, seq: int) -> CatalogProfileTask | None: + return (await session.execute( + select(CatalogProfileTask).where(CatalogProfileTask.job_id == job.id, CatalogProfileTask.seq == seq) + )).scalar_one_or_none() + + +#: 同一张表的各项谁算「最近」：任务建得晚的在前；同一个任务里同名的取 seq 小的 +_NEWEST = (CatalogProfileJob.created_at.desc(), CatalogProfileJob.id.desc(), CatalogProfileTask.seq) +#: 表清单、表详情显示的「上次剖析」：真正开始过（started_at 有值）的排在没开始过的前面，各自再按 _NEWEST +_SHOWN = (case((CatalogProfileTask.started_at.is_(None), 1), else_=0), *_NEWEST) + + +def _ranked(source_id: str, *columns: Any) -> Any: + """每张表剖析过的各项：子查询，带两个排名。 + + - shown：这张表的「上次剖析」（表清单每行、表详情、要不要剖析都按它；保留也保护它所在的任务）——真正开始过的最近一项， + 一次都没开始过才退回最近一项。没开始就结束了的那一项（停在前面的表上、任务止损或整体出错后没轮到、建任务时就判为 + 跳过）、排队中还没轮到的那一项都不算：它没有行数估算、发过的查询，要是挡掉上一次真正剖析的那一项，表详情就只剩 + 一句「已停止」，那次剖析所在的任务还会被当成不是谁的最近一次给清掉——行数、日期范围、发过的查询只存在剖析任务里。 + - latest：最近一项，不管开始没有。它还排着、在做（pending / running）说明这张表已经在进行中的任务里了。 + """ + shown = func.row_number().over(partition_by=CatalogProfileTask.table_name, order_by=_SHOWN).label("shown") + latest = func.row_number().over(partition_by=CatalogProfileTask.table_name, order_by=_NEWEST).label("latest") + return (select(*columns, shown, latest).join(CatalogProfileJob, CatalogProfileJob.id == CatalogProfileTask.job_id) + .where(CatalogProfileJob.source_id == source_id)).subquery() + + +async def last_profiles(session: AsyncSession, source_id: str) -> dict[str, dict[str, Any]]: + """每张表的「上次剖析」（见 _ranked 的 shown）：{表名: CatalogLastProfile 加上内部用的 paths（那一项检查项的路径，判断 + 有没有新的待核实关系用）、queued（最近一项还排着或在做：这张表已经在进行中的任务里了）}。在做的就是进行中的状态。 + 每张表取哪一项在库里挑好。""" + inner = _ranked(source_id, CatalogProfileTask.table_name, CatalogProfileTask.job_id, + CatalogProfileTask.seq, CatalogProfileTask.status, CatalogProfileTask.started_at, + CatalogProfileTask.finished_at, CatalogProfileTask.summary, + CatalogProfileTask.skipped_count, CatalogProfileTask.unfinished, + CatalogProfileTask.error, CatalogProfileTask.row_estimate, CatalogProfileTask.checks) + rows = list((await session.execute( + select(inner).where(or_(inner.c.shown == 1, inner.c.latest == 1)))).mappings()) + queued = {row["table_name"] for row in rows if row["latest"] == 1 and row["status"] in OPEN} + out: dict[str, dict[str, Any]] = {} + for row in rows: + if row["shown"] != 1: + continue + out[row["table_name"]] = { + "job_id": row["job_id"], "seq": row["seq"], "status": row["status"], + "started_at": _iso(row["started_at"]), "finished_at": _iso(row["finished_at"]), + "summary": {**_empty_summary(), **(row["summary"] or {})}, "skipped_count": row["skipped_count"] or 0, + "unfinished": row["unfinished"] or 0, "error": row["error"], "row_estimate": row["row_estimate"], + "paths": {c.get("path") for c in row["checks"] or [] if isinstance(c, dict) and c.get("path")}, + "queued": row["table_name"] in queued, + } + return out + + +#: last_profiles 里只给内部用的键 +_INTERNAL = ("paths", "queued") + + +def public_last(last: dict[str, Any] | None) -> dict[str, Any] | None: + """接口里的 CatalogLastProfile：去掉内部用的 paths、queued。""" + return None if last is None else {k: v for k, v in last.items() if k not in _INTERNAL} + + +async def last_detail(session: AsyncSession, source_id: str, table: str) -> dict[str, Any] | None: + """这张表「上次剖析」（见 _ranked 的 shown）的明细（表详情的「数据剖析」一节）；没剖析过是 None。""" + found = (await session.execute( + select(CatalogProfileTask, CatalogProfileJob) + .join(CatalogProfileJob, CatalogProfileJob.id == CatalogProfileTask.job_id) + .where(CatalogProfileJob.source_id == source_id, CatalogProfileTask.table_name == table) + .order_by(*_SHOWN).limit(1) + )).first() + return None if found is None else task_detail(found[1], found[0]) + + +def due_reason(meta: dict[str, Any] | None, notes: dict[str, Any] | None, last: dict[str, Any] | None, + checks: int) -> str | None: + """这张表要不要剖析、为什么：never 从没剖析过；unfinished 上次剖析是 partial / failed / cancelled；new_relations + 目录里有待核实的关系不在上次剖析的检查项里。表结构里没有、视图、没有可剖析的项（checks 为 0）、已经在进行中的任务里 + （排着或在做）的不要。 + + 「上次剖析」和表清单显示的是同一项（last_profiles，真正开始过的最近一项）：上次真正剖析完整了，后来某个任务没轮到它 + 就停了，不算没做完——不然表清单一边显示「上次剖析：完成」，一边说「上次剖析没做完」。一次都没开始过的，按最近一项算。""" + if not isinstance(meta, dict) or meta.get("is_view") or checks <= 0: + return None + if last is None: + return "never" + if last.get("queued") or last["status"] in OPEN: + return None + if last["status"] in RETRYABLE: + return "unfinished" + if catalog_profile.pending_relation_paths(notes) - set(last.get("paths") or ()): + return "new_relations" + return None + + +# ========================================================================== +# 建任务、重试、停止 +# ========================================================================== + + +#: 撞上剖析任务时说法的最后一句（「请等这次剖析结束（或先停止它）{then}」）缺省是建剖析任务的；导入目录、同步起草撞上 +#: 时各说各的（catalog_jobs）：那时要做的是导入、起草，不是开始新的剖析 +THEN_PROFILE = "再开始新的剖析" + + +def busy_message(row: Any, brief: dict[str, Any] | None, *, table: str | None = None, + position: int | None = None, then: str = THEN_PROFILE) -> str: + """同源已有剖析任务时的说法：写明进行到第几张；从表详情点「剖析这张表」时再说这张表在不在那个任务里、排第几；最后 + 一句按调用方要做的事说（then）。""" + name = getattr(row, "name", "") or "" + if brief is None: + text = f"数据源「{name}」正在剖析" + elif brief["status"] == "queued": + text = f"数据源「{name}」已有一个排队中的剖析任务（共 {brief['total']} 张表）" + else: + text = f"数据源「{name}」正在剖析：第 {min(brief['done'] + 1, brief['total'])} / {brief['total']} 张表" + if table is not None: + text += f"，{table} 在这次剖析里，排第 {position} 张" if position else f"，{table} 不在这次剖析里" + return text + f"。请等这次剖析结束（或先停止它）{then}" + + +async def ensure_free(session: AsyncSession, row: Any, tables: list[str] | None = None, *, + then: str = THEN_PROFILE) -> None: + """这个源没有排队或进行中的剖析任务；有就抛 ProfileBusy。只指定了一张表时，说法里带上这张表在不在那个任务里；then 是 + 说法的最后一句要做的事（见 busy_message）。""" + job = await active_job(session, row.id) + if job is None: + return + brief = await job_brief(session, job) + table = position = None + if tables and len(tables) == 1: + key = catalog.resolve_table_name(getattr(row, "schema_cache", None), str(tables[0])) or str(tables[0]) + table = key + names = [r.table_name for r in await task_rows(session, job.id)] + position = names.index(key) + 1 if key in names else None + raise ProfileBusy(busy_message(row, brief, table=table, position=position, then=then)) + + +#: 挑出 0 张表时的说法 +NOTHING_NEEDED = ("没有需要剖析的表：能剖析的表都剖析过了，上次剖析也都做完了，之后没有新的待核实关系。" + "要重新剖析，请选「全部表」或指定要剖析的表") +NOTHING_AT_ALL = ("没有可以剖析的表：表结构里的表都是视图，或者没有可剖析的项（要核对的关联关系、像码值的列、日期列都没有，" + "或者都被遮罩了）。请先起草数据目录，或指定要剖析的表") + + +def _skipped_task(job_id: str, seq: int, name: str, reason: str, why: str, checks: list[dict[str, Any]], + skipped: list[ProfileSkip], now: Any) -> CatalogProfileTask: + entries = [s.to_dict() for s in skipped] + [ProfileSkip("table", name, reason, why).to_dict()] + return CatalogProfileTask(job_id=job_id, seq=seq, table_name=name, status="skipped", checks=checks or None, + skipped=entries, skipped_count=len(entries), + unfinished=sum(1 for e in entries if e["reason"] in UNFINISHED_REASONS), + summary=_empty_summary(), error=why, finished_at=now) + + +async def _pick(session: AsyncSession, row: Any, source: Any, *, tables: list[str] | None, + scope: str) -> list[tuple[str, str | None, str | None, list[Any]]]: + """要剖析的表：[(表名, 跳过的原因 / None, 整句, 检查项)]，按剖析顺序。""" + cache = getattr(source, "schema_cache", None) or {} + all_tables = cache.get("tables") or {} + masked = catalog_profile.masked_of(getattr(row, "options", None)) + entries = await catalog.read_catalog(session, source.id) + + def notes(name: str) -> dict[str, Any]: + entry = entries.get(name) + return entry.notes if entry else {} + + def plan(name: str) -> list[Any]: + return catalog_profile.plan_checks(name, all_tables.get(name), notes(name), tables=all_tables, masked=masked) + + if scope == "selected": + picked: list[tuple[str, str | None, str | None, list[Any]]] = [] + seen: set[str] = set() + for name in dict.fromkeys(str(t) for t in tables or []): + key = catalog.resolve_table_name(cache, name) + if key is None: + picked.append((name, "missing", catalog.missing_table_error(name), [])) + continue + if key in seen: + continue + seen.add(key) + if (all_tables.get(key) or {}).get("is_view"): + picked.append((key, "view", VIEW_ERROR, [])) + continue + checks = plan(key) + if not any(c.skip is None for c in checks): + picked.append((key, "nothing", NOTHING_ERROR, checks)) + else: + picked.append((key, None, None, checks)) + return picked + usage = await catalog.table_usage(session, source) + last = await last_profiles(session, source.id) if scope == "needed" else {} + order = {name: i for i, name in enumerate(all_tables)} + candidates: list[tuple[str, list[Any]]] = [] + for name, meta in all_tables.items(): + if not isinstance(meta, dict) or meta.get("is_view"): + continue + checks = plan(name) + runnable = sum(1 for c in checks if c.skip is None) + if runnable == 0: + continue + if scope == "needed" and due_reason(meta, notes(name), last.get(name), runnable) is None: + continue + candidates.append((name, checks)) + if not candidates: + raise ProfileNothing(NOTHING_NEEDED if scope == "needed" else NOTHING_AT_ALL) + # 还有待核实（推断）关系的表排在前面：剖析首先是来核实它们的；再按使用次数、表结构顺序 + candidates.sort(key=lambda c: (not catalog_profile.has_pending_relations(notes(c[0])), -usage.get(c[0], 0), + order[c[0]])) + return [(name, None, None, checks) for name, checks in candidates] + + +async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: list[str] | None, scope: str, + settings: ProfileSettings, actor: str | None, + retry_of: str | None = None) -> tuple[CatalogProfileJob, list[CatalogProfileTask]]: + """建一个排队的剖析任务（还没开始，调用方接着 manager.start）。row 是数据源记录（遮罩按它的设置），source 是读表结构 + 用的源（上传源是绑定当前快照的视图）。同源已有排队或进行中的剖析任务抛 ProfileBusy；挑出 0 张表抛 ProfileNothing。 + 同源有起草任务时照建，排在它后面。建完顺手清掉超出保留数的旧任务。""" + await ensure_free(session, row, tables) + picked = await _pick(session, row, source, tables=tables, scope=scope) + job = CatalogProfileJob(id=new_id(), source_id=row.id, status="queued", scope=scope, + settings=settings.to_dict(), created_by=actor, retry_of=retry_of) + now = utcnow() + tasks: list[CatalogProfileTask] = [] + for seq, (name, reason, why, checks) in enumerate(picked): + shown = [c.as_check() for c in checks] + if reason is not None: + rule = [ProfileSkip(c.kind, c.target, c.skip[0], c.skip[1], c.path) for c in checks if c.skip] + tasks.append(_skipped_task(job.id, seq, name, reason, str(why), shown, rule, now)) + continue + tasks.append(CatalogProfileTask(job_id=job.id, seq=seq, table_name=name, status="pending", checks=shown, + summary=_empty_summary())) + session.add(job) + try: + await session.flush() + session.add_all(tasks) + await session.commit() + except IntegrityError as e: + # 部分唯一索引：另一个请求同时给这个源建了剖析任务 + await session.rollback() + raise ProfileBusy(busy_message(row, None)) from e + try: + await prune(session, row.id) + except Exception: # noqa: BLE001 - 清旧任务失败不影响这次剖析 + logger.exception("清理数据源 %s 的旧剖析任务时出错", row.id) + await session.rollback() + return job, tasks + + +async def retry_tables(session: AsyncSession, job: CatalogProfileJob) -> list[str]: + """重试要带上的表：原任务里 failed、partial、cancelled 的，按原来的顺序。没有抛 NothingToRetry。""" + names = [t.table_name for t in await task_rows(session, job.id) if t.status in RETRYABLE] + if not names: + raise NothingToRetry("这次剖析里没有失败、部分完成或停止了的表，不需要重试") + return names + + +async def prune(session: AsyncSession, source_id: str, *, keep: int | None = None) -> int: + """保留：这个源最近 keep（缺省 KEEP_JOBS）个结束了的剖析任务留着，更早的删掉；某张表「上次剖析」（真正开始过的最近 + 一次，见 _ranked）所在的任务不删——排队中还没轮到、没开始就结束了的那一项不算，不然新建一个任务就能把上一次真正剖析 + 清掉。返回删了几个。""" + keep = KEEP_JOBS if keep is None else keep + finished = list((await session.execute( + select(CatalogProfileJob.id).where(CatalogProfileJob.source_id == source_id, + CatalogProfileJob.status.in_(FINISHED)) + .order_by(CatalogProfileJob.created_at.desc(), CatalogProfileJob.id.desc()) + )).scalars()) + old = finished[keep:] + if not old: + return 0 + inner = _ranked(source_id, CatalogProfileTask.job_id) + protected = set((await session.execute(select(inner.c.job_id).where(inner.c.shown == 1))).scalars()) + victims = [job_id for job_id in old if job_id not in protected] + if not victims: + return 0 + await session.execute(delete(CatalogProfileTask).where(CatalogProfileTask.job_id.in_(victims))) + await session.execute(delete(CatalogProfileJob).where(CatalogProfileJob.id.in_(victims))) + await session.commit() + return len(victims) + + +async def _close_tasks(session: AsyncSession, job_id: str, task_status: str, task_error: str | None, + *, why: str) -> None: + """还没结束的表记 task_status（原因 task_error）；它们的检查项里还排着的、正在做的记为跳过（停止、中断是 cancelled， + 出错是 error，说明是「{why}，未检查」），计进跳过的项和没做完的项。 + + 还没结束的表没走到整张表结束那一步：目录只在那一步和这张表的结果一起写（catalog_profile._record，同一个事务）， + 所以这里已经查完（done）的项结论也没写进目录——进程没了、任务整体出错（快照被改动……）都是这样。它们同样记为跳过、 + 原因同上，说明是「{why}，查完了但结论没有写进数据目录」，算没做完；这张表的原因写明查完的几项没写进目录。不能留着 + 「完成」：检查清单上标着完成、目录里却没有，看的人会以为结论已经在目录里了。""" + now = utcnow() + reason = "cancelled" if task_status == "cancelled" else "error" + detail, unsaved = _UNCHECKED.format(why=why), _UNSAVED.format(why=why) + tasks = (await session.execute( + select(CatalogProfileTask).where(CatalogProfileTask.job_id == job_id, CatalogProfileTask.status.in_(OPEN)) + )).scalars() + for task in tasks: + checks = [dict(c) for c in task.checks or [] if isinstance(c, dict)] + skipped = list(task.skipped or []) + listed = {(s.get("kind"), s.get("target"), s.get("path")) for s in skipped if isinstance(s, dict)} + done = rest = 0 + for check in checks: + state = check.get("state") + if state == "done": + done += 1 + check.update(state="skipped", reason=reason, detail=unsaved) + elif state in _CHECK_OPEN: + rest += 1 + check.update(state="skipped", reason=reason, detail=detail) + elif state != "skipped": + continue + # 跳过的项按检查项列全：进行中的表只随时落了检查项，按规则不查的（遮罩……）也要进跳过的项 + key = (check.get("kind"), check.get("target"), check.get("path")) + if key not in listed: + listed.add(key) + skipped.append(ProfileSkip(check.get("kind") or "", check.get("target") or "", + check.get("reason") or "", check.get("detail") or "", + check.get("path")).to_dict()) + error = unsaved_error(task_error or why, done, rest) if done else task_error + task.status, task.error, task.finished_at, task.current = task_status, error, now, None + task.checks, task.skipped = checks or None, skipped or None + task.skipped_count = len(skipped) + task.unfinished = sum(1 for s in skipped if s.get("reason") in UNFINISHED_REASONS) + + +async def _close(session: AsyncSession, job_id: str, job_status: str, job_error: str | None, task_status: str, + task_error: str | None, *, why: str | None = None) -> None: + """收尾：还没结束的表记 task_status（见 _close_tasks），任务（还没结束的话）记 job_status。提交。""" + await _close_tasks(session, job_id, task_status, task_error, why=why or task_error or STOPPED_ERROR) + await session.execute( + update(CatalogProfileJob).where(CatalogProfileJob.id == job_id, CatalogProfileJob.status.in_(ACTIVE)) + .values(status=job_status, error=job_error, finished_at=utcnow()).execution_options(synchronize_session=False)) + await session.commit() + + +async def cancel(session: AsyncSession, job: CatalogProfileJob) -> None: + """停止一个剖析任务。排队的直接 cancelled（在队里等着的后台执行跟着停掉、出队）；进行中的交给 manager 停：正在等的那条 + 查询不再等（库那边按单条时限截止），那张表做完的项照常写进目录，后面的表 cancelled。已经结束的抛 ProfileFinished。""" + if job.status in FINISHED: + raise ProfileFinished("这次剖析已经结束了，不需要停止") + if job.status == "queued": + claimed = await session.execute( + update(CatalogProfileJob).where(CatalogProfileJob.id == job.id, CatalogProfileJob.status == "queued") + .values(status="cancelled", finished_at=utcnow()).execution_options(synchronize_session=False)) + if claimed.rowcount == 1: + await _close_tasks(session, job.id, "cancelled", None, why=STOPPED_ERROR) + await session.commit() + await manager.stop(job.id, "user", grace=STOP_GRACE_S) + return + await session.rollback() # 刚被后台认领、开始了：按进行中的停 + if await manager.stop(job.id, "user", grace=STOP_GRACE_S): + return + # 库里说在进行，这个进程里却没有在跑它（记录对不上）：直接记停止 + await _close(session, job.id, "cancelled", None, "cancelled", None) + + +async def startup() -> int: + """服务启动时：排队、进行中的剖析任务标成 interrupted，没结束的表记 cancelled、写明服务重启，不自动续跑；再把剖析设置的 + 旧键迁移掉。返回标了几个任务。""" + async def interrupt(session: AsyncSession, job_id: str) -> None: + await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) + + n = await runtime.interrupt_on_startup(CatalogProfileJob, interrupt, running=manager.running) + if n: + logger.warning("服务重启：%d 个数据剖析任务没有做完，已标为中断，可在数据目录页重试", n) + async with SessionLocal() as session: + migrated = await catalog_profile.migrate_settings(session) + if migrated: + logger.info("数据剖析设置：%d 个数据源去掉了按整次剖析算的旧预算", migrated) + return n + + +# ========================================================================== +# 在后台剖析 +# ========================================================================== + + +class _ProfileRun(runtime.JobRun): + """一个剖析任务在后台的一次执行（进队、停止、收尾的框架在 runtime.JobRun）。""" + + kind = "profile" + model = CatalogProfileJob + + def __init__(self, job_id: str) -> None: + super().__init__(job_id) + self.actor: str | None = None + self.cx: catalog_profile.ProfileContext | None = None + #: 正在剖析的那张表：seq 和开始的时刻（进度回调据此落库） + self._seq = -1 + self._table_started = 0.0 + + # ---------------------------------------------------------------- 停止、出错时收尾 + + async def _stopped(self) -> None: + if self.stop_reason == "user": + await self._end("cancelled", None, "cancelled", None) + else: + await self._end("interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) + + async def _failed(self, reason: str) -> None: + await self._end("failed", reason, "failed", reason) + + async def _crashed(self, e: Exception) -> None: + from app.core.errors import describe_exception, first_line + + logger.exception("数据剖析任务 %s 出错", self.job_id) + reason = JOB_ERROR.format(line=first_line(e) or describe_exception(e)) + await self._end("failed", reason, "failed", reason) + + async def _end(self, job_status: str, job_error: str | None, task_status: str, task_error: str | None) -> None: + """停止、出错时收尾：等手上的库写入写完，记下查询计数，再把没结束的表和任务记下来。收尾本身写不进去只记日志（下次 + 启动时 startup 会把它标成中断）。""" + try: + await self._settle() + await self._write(self._save_counters) + await self._write(_close, self.job_id, job_status, job_error, task_status, task_error) + except Exception: # noqa: BLE001 + logger.exception("数据剖析任务 %s 收尾时写库出错", self.job_id) + + def stop_text(self) -> str: + return STOPPED_ERROR if self.stop_reason == "user" else INTERRUPTED_ERROR + + # ---------------------------------------------------------------- 主流程 + + async def _run(self) -> None: + if not await self._write(self._claim): + return # 开始之前就被停止了 + async with SessionLocal() as session: + job = await session.get(CatalogProfileJob, self.job_id) + row = await session.get(DataSource, job.source_id) if job is not None else None + if job is None or row is None: + raise runtime.JobFailed(SOURCE_GONE) + from app.data import table_versions + + try: + source = await table_versions.resolve_source(session, row) + except table_versions.SnapshotMissing as e: + raise runtime.JobFailed(str(e)) from e + pending = [(t.seq, t.table_name) for t in await task_rows(session, job.id) if t.status == "pending"] + settings = ProfileSettings.from_dict(job.settings) + self.actor = job.created_by + options = dict(row.options or {}) + try: + self.cx = catalog_profile.make_context(source, settings, options=options) + except ValueError as e: # 不支持的数据库类型 + raise runtime.JobFailed(str(e)) from e + hooks = TableHooks(progress=self._progress, query=self._queried, stop_text=self.stop_text) + for seq, name in pending: + # 每张表开始前重读数据源：停用了、关掉了剖析就停下；遮罩按最新的设置 + stop = await self._write(self._recheck) + if stop is not None: + await self._write(self._halt, stop, None) + break + self._table_started = time.monotonic() + await self._write(self._start_table, seq) + + async def recorded(session: AsyncSession, result: TableResult, seq: int = seq) -> None: + await self._save_table(session, seq, result) + + hooks.recorded = recorded + self._seq = seq + try: + await catalog_profile.profile_table(self.cx, name, db=self._write, hooks=hooks, actor=self.actor) + except SnapshotTampered as e: + raise runtime.JobFailed(str(e)) from e + if self.cx.runner.stopped: + # 任务级止损：后面的表 cancelled，带同一句原因 + await self._write(self._halt, self.cx.runner.error, self.cx.runner.stopped) + break + await self._settle() + await self._write(self._finish) + + # ---------------------------------------------------------------- 进度回调（引擎调） + + async def _progress(self, result: TableResult) -> None: + await self._write(self._save_progress, self._seq, result) + + async def _queried(self, result: TableResult) -> None: + await self._write(self._save_queries, self._seq, result) + + # ---------------------------------------------------------------- 库操作 + + def _task(self, *where: Any) -> Any: + return update(CatalogProfileTask).where(CatalogProfileTask.job_id == self.job_id, *where) \ + .execution_options(synchronize_session=False) + + def _counters(self) -> dict[str, Any]: + runner = self.cx.runner if self.cx is not None else None + if runner is None: + return {} + return {"queries": runner.queries, "timeouts": runner.timeouts, "errors": runner.errors, + "notice": runner.notice} + + async def _save_counters(self, session: AsyncSession) -> None: + if values := self._counters(): + await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id).values(**values) + .execution_options(synchronize_session=False)) + await session.commit() + + async def _claim(self, session: AsyncSession) -> bool: + return await runtime.claim(session, CatalogProfileJob, self.job_id) + + async def _recheck(self, session: AsyncSession) -> str | None: + """重读数据源：停用了、关掉了剖析返回停下的整句；遮罩换成最新的设置。数据源没了抛 JobFailed。""" + job = await session.get(CatalogProfileJob, self.job_id) + row = await session.get(DataSource, job.source_id) if job is not None else None + if row is None: + raise runtime.JobFailed(SOURCE_GONE) + if row.enabled is False: + return SOURCE_DISABLED.format(name=row.name) + if not catalog_profile.profile_settings(row.options).enabled: + return PROFILE_TURNED_OFF.format(name=row.name) + assert self.cx is not None + self.cx.masked = catalog_profile.masked_of(row.options) + return None + + async def _halt(self, session: AsyncSession, error: str | None, stop_reason: str | None) -> None: + """整个任务停下（止损、数据源停用或关掉了剖析）：后面的表 cancelled、error 是同一句原因；任务记下原因，状态到收尾时 + 按逐表算。""" + await _close_tasks(session, self.job_id, "cancelled", error, why=error or STOPPED_ERROR) + await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id) + .values(error=error, stop_reason=stop_reason, **self._counters()) + .execution_options(synchronize_session=False)) + await session.commit() + + async def _start_table(self, session: AsyncSession, seq: int) -> None: + await session.execute(self._task(CatalogProfileTask.seq == seq, CatalogProfileTask.status == "pending") + .values(status="running", started_at=utcnow())) + await session.commit() + + def _elapsed(self) -> int: + return int((time.monotonic() - self._table_started) * 1000) + + async def _save_progress(self, session: AsyncSession, seq: int, result: TableResult) -> None: + """一项检查开始或结束：这张表的检查项、正在做哪一项、查询数、发过的查询、行数估算，任务的查询计数。""" + await session.execute(self._task(CatalogProfileTask.seq == seq, CatalogProfileTask.status == "running") + .values(checks=[dict(c) for c in result.checks] or None, current=result.current, + queries=result.queries, elapsed_ms=self._elapsed(), + query_log=list(result.query_log) or None, + row_estimate=result.row_estimate.to_dict() if result.row_estimate else None)) + if values := self._counters(): + await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id).values(**values) + .execution_options(synchronize_session=False)) + await session.commit() + + async def _save_queries(self, session: AsyncSession, seq: int, result: TableResult) -> None: + """发出一条查询之后：这张表和任务的查询计数、提醒（发过的查询本身在检查项结束时一起写）。""" + await session.execute(self._task(CatalogProfileTask.seq == seq, CatalogProfileTask.status == "running") + .values(queries=result.queries, elapsed_ms=self._elapsed())) + if values := self._counters(): + await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id).values(**values) + .execution_options(synchronize_session=False)) + await session.commit() + + async def _save_table(self, session: AsyncSession, seq: int, result: TableResult) -> None: + """一张表结束：整张表的结果。和写目录在同一个事务里，由 catalog_profile._record 一起提交，这里不提交。""" + await session.execute( + self._task(CatalogProfileTask.seq == seq, CatalogProfileTask.status.in_(OPEN)) + .values(status=result.status, checks=[dict(c) for c in result.checks] or None, current=None, + queries=result.queries, elapsed_ms=result.elapsed_ms, + row_estimate=result.row_estimate.to_dict() if result.row_estimate else None, + findings=list(result.findings) or None, skipped=[s.to_dict() for s in result.skipped] or None, + date_ranges=list(result.date_ranges) or None, query_log=list(result.query_log) or None, + summary=result.summary(), skipped_count=result.skipped_count, unfinished=result.unfinished, + added=result.added, updated=result.updated, removed=result.removed, version=result.version, + error=result.error, finished_at=utcnow())) + if values := self._counters(): + await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id).values(**values) + .execution_options(synchronize_session=False)) + + async def _finish(self, session: AsyncSession) -> None: + """剖析完：万一还有没收尾的表（进度回调写库失败之类）记失败；有 failed、partial、cancelled 的表是 partial，否则 + succeeded。""" + await _close_tasks(session, self.job_id, "failed", UNFINISHED_ERROR, why=UNFINISHED_ERROR) + statuses = list((await session.execute( + select(CatalogProfileTask.status).where(CatalogProfileTask.job_id == self.job_id))).scalars()) + status = "partial" if any(s in RETRYABLE for s in statuses) else "succeeded" + await session.execute( + update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id, CatalogProfileJob.status == "running") + .values(status=status, finished_at=utcnow(), **self._counters()) + .execution_options(synchronize_session=False)) + await session.commit() + + +class ProfileJobs(runtime.JobManager): + """进程内的剖析任务（start / stop / wait / running / shutdown 见 runtime.JobManager）。""" + + def __init__(self) -> None: + super().__init__(_ProfileRun, label="数据剖析任务", name="catalog-profile") + + +manager = ProfileJobs() + + +__all__ = ["INTERRUPTED_ERROR", "KEEP_JOBS", "MAX_TABLES", "NOTHING_AT_ALL", "NOTHING_NEEDED", "NothingToRetry", + "ProfileBusy", "ProfileFinished", "ProfileJobs", "ProfileNothing", "THEN_PROFILE", "active_job", + "busy_message", "cancel", "create_job", "due_reason", "ensure_free", "job_brief", "job_dict", "job_out", + "last_detail", "last_profiles", "latest_job", "load", "load_task", "manager", "prune", "public_last", + "retry_tables", "startup", "task_detail", "task_row", "task_rows", "unsaved_error", "waiting_for"] diff --git a/backend/app/data/catalog_transfer.py b/backend/app/data/catalog_transfer.py index 39bab51d..d963ca22 100644 --- a/backend/app/data/catalog_transfer.py +++ b/backend/app/data/catalog_transfer.py @@ -28,8 +28,8 @@ tables: {表名: {notes, version}}}，含全部状态的项，表结构里已经 没有是 none。 - 文件格式不对（不是对象、格式标记或版本不对、表的目录不是对象、起草要求不是文字或超长）抛 TransferInvalid，接口回 422。 -有排队或进行中的起草任务时不导入（接口先调 catalog_jobs.ensure_idle，409 catalog_draft_busy）；目标数据源没有表结构时 -接口回 409（什么也对不上）。 +有排队或进行中的起草、剖析任务时不导入（接口经 catalog_jobs.import_write，409 catalog_draft_busy / profile_busy，导入期间 +占着这个源的队）；目标数据源没有表结构时接口回 409（什么也对不上）。 """ from __future__ import annotations diff --git a/backend/app/data/engine.py b/backend/app/data/engine.py index 7cdb4c41..e917df5d 100644 --- a/backend/app/data/engine.py +++ b/backend/app/data/engine.py @@ -62,7 +62,7 @@ MAX_QUERY_TIMEOUT_S = 600 #: 运行事件里有查询结果的预览 MASK_COLUMNS_OPTION = "mask_columns" -#: 数据源 options 里的数据剖析设置（一个对象：开关、查询次数上限、单条时限……）。取值和校验见 +#: 数据源 options 里的数据剖析设置（一个对象：开关、每张表的查询数和时长、单条时限……）。取值和校验见 #: data/catalog_profile.py；放在这里只为让拼连接串时摘掉它 CATALOG_PROFILE_OPTION = "catalog_profile" diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 21349730..7e3ad023 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -435,6 +435,96 @@ class CatalogDraftTask(Base): finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) +class CatalogProfileJob(Base, TimestampMixin): + """数据目录的一次后台剖析任务（data/catalog_profile_jobs.py）。逐表的结果在 catalog_profile_tasks。 + + 以前剖析是一次同步长请求：预算按整次剖析算（选 50 张表只做得完两三张）、不能停、关掉页面报告就没了，对业务库查了什么 + 也无从查。现在建任务后在后台逐表剖析，每张表各有自己的查询次数和时长，进度、发过的查询（字面量去掉）随时落库。生命周期 + 和起草任务是同一套（data/catalog_job_runtime.py），表各建各的：进度字段完全不同。 + + 同一个数据源同时只有一个排队或进行中的剖析任务：接口先查一遍，部分唯一索引兜底。和同源的起草任务按建立的先后排队。 + 服务重启时还没结束的标成 interrupted，不自动续跑。挂外键、级联删除：任务是数据源的附属记录。 + """ + + __tablename__ = "catalog_profile_jobs" + __table_args__ = ( + Index("ix_catalog_profile_jobs_source_created", "source_id", "created_at"), + Index("ux_catalog_profile_jobs_active", "source_id", unique=True, + sqlite_where=text("status IN ('queued', 'running')")), + ) + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + source_id: Mapped[str] = mapped_column(ForeignKey("data_sources.id", ondelete="CASCADE")) + # queued | running | succeeded | partial | failed | cancelled | interrupted + status: Mapped[str] = mapped_column(String(20), default="queued") + # selected 指定的表 | needed 需要剖析的表 | all 全部表 + scope: Mapped[str] = mapped_column(String(20), default="selected") + # 建任务时的剖析设置快照（catalog_profile.ProfileSettings.to_dict） + settings: Mapped[dict[str, Any]] = mapped_column(default=dict) + # 署名（开了登录是认证过的用户名，否则请求头 X-Actor 的自报） + created_by: Mapped[str | None] = mapped_column(String(100), default=None) + started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + # 整个任务发出的查询数、其中超时的、失败的（守卫拒掉的没发出去，不算） + queries: Mapped[int] = mapped_column(Integer, default=0) + timeouts: Mapped[int] = mapped_column(Integer, default=0) + errors: Mapped[int] = mapped_column(Integer, default=0) + # 任务级止损停下的原因：failed 连续失败、slow 超时比例；没有为 NULL + stop_reason: Mapped[str | None] = mapped_column(String(20), default=None) + # 提醒：最近的查询多数超时时的一句话，好转后清掉 + notice: Mapped[str | None] = mapped_column(Text, default=None) + # 任务整体出错、止损、数据源中途停用或关掉剖析时的整句原因 + error: Mapped[str | None] = mapped_column(Text, default=None) + # 重试建的任务记下是重试哪一个 + retry_of: Mapped[str | None] = mapped_column(String(32), default=None) + + +class CatalogProfileTask(Base): + """剖析任务里的一张表。按 seq（剖析顺序）排。状态：pending 排队、running 剖析中、ok 每项都有结论、partial 有项因为 + 运行原因没查成、failed 写入一直冲突或整张表出错、skipped 视图 / 表结构里没有 / 空表 / 没有可剖析的项、cancelled + （停止、止损或服务重启）。 + + 检查项、发现、跳过的项、日期范围、发过的查询都是 JSON：条数受每张表的查询上限约束，大小有界。发过的查询里的 SQL + 字面量都换成了「?」（sql_semantics.normalize_sql）：抽样到的键值不进 AgentLab 自己的库。""" + + __tablename__ = "catalog_profile_tasks" + __table_args__ = (UniqueConstraint("job_id", "seq"),) + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + job_id: Mapped[str] = mapped_column(ForeignKey("catalog_profile_jobs.id", ondelete="CASCADE"), index=True) + seq: Mapped[int] = mapped_column(Integer) + # 表结构里的键；表结构里没有的（skipped）是请求里的原样名字 + table_name: Mapped[str] = mapped_column(String(255)) + status: Mapped[str] = mapped_column(String(20), default="pending") + # 检查项：[{kind, target, path, state, reason, detail, queries}]。建任务时按当时的目录列一遍，开始剖析这张表时按最新的 + # 目录重列 + checks: Mapped[list[Any] | None] = mapped_column(default=None) + queries: Mapped[int] = mapped_column(Integer, default=0) + elapsed_ms: Mapped[int] = mapped_column(Integer, default=0) + # 进行中时正在做哪一项的整句（「正在核对 orders.customer_id → customers.id」） + current: Mapped[str | None] = mapped_column(Text, default=None) + # 行数估算 {rows, method, at_least} + row_estimate: Mapped[dict[str, Any] | None] = mapped_column(default=None) + findings: Mapped[list[Any] | None] = mapped_column(default=None) + skipped: Mapped[list[Any] | None] = mapped_column(default=None) + date_ranges: Mapped[list[Any] | None] = mapped_column(default=None) + # 发过的查询：[{at, kind, target, sql, ms, rows, outcome, detail}] + query_log: Mapped[list[Any] | None] = mapped_column(default=None) + # 发现的合计 {relations_checked, relations_verified, codes, codes_pending, business_dates}；跳过的项数、其中因为运行 + # 原因没查成的（表清单的「上次剖析」读这几列，不必读整份发现） + summary: Mapped[dict[str, Any] | None] = mapped_column(default=None) + skipped_count: Mapped[int] = mapped_column(Integer, default=0) + unfinished: Mapped[int] = mapped_column(Integer, default=0) + added: Mapped[int] = mapped_column(Integer, default=0) + updated: Mapped[int] = mapped_column(Integer, default=0) + removed: Mapped[int] = mapped_column(Integer, default=0) + version: Mapped[int] = mapped_column(Integer, default=0) + # 整张表的原因：表结构里没有、视图、空表、没有可剖析的项、写入一直冲突、停止 / 中断 / 止损 + error: Mapped[str | None] = mapped_column(Text, default=None) + started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + + class Setting(Base, TimestampMixin): """键值形式的用户设置，前端 Settings 页直接读写。""" diff --git a/backend/app/main.py b/backend/app/main.py index c5fa8f46..657c917a 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -93,14 +93,18 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: except Exception: # noqa: BLE001 logger.exception("上传表格的版本整理失败，服务照常启动") - # 上次没做完的数据目录起草任务：标成中断（没结束的表记停止），不自动续跑——拷来的数据目录（检查脚本、验收副本） - # 启动时不能替人去调模型。界面上点「重试」接着起草没写好的表 - from app.data import catalog_jobs + # 上次没做完的数据目录起草、剖析任务：标成中断（没结束的表记停止），不自动续跑——拷来的数据目录（检查脚本、验收副本） + # 启动时不能替人去调模型、查业务库。界面上点「重试」接着做没做完的表。剖析设置里按整次剖析算的旧预算顺带迁移掉 + from app.data import catalog_jobs, catalog_profile_jobs try: await catalog_jobs.startup() except Exception: # noqa: BLE001 logger.exception("数据目录起草任务的启动整理失败，服务照常启动") + try: + await catalog_profile_jobs.startup() + except Exception: # noqa: BLE001 + logger.exception("数据剖析任务的启动整理失败，服务照常启动") await run_manager.setup() # 流程：存量的待审批补建待办（派给任何人，幂等）；上次进程没了时已发出、没拿到结果的写入收成结果未知、安排核实 @@ -129,9 +133,10 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: try: yield finally: - # 先停调度，再停运行：停运行的途中不该再有新运行被发起。起草任务停下来标成中断，下次启动不会再续 + # 先停调度，再停运行：停运行的途中不该再有新运行被发起。起草、剖析任务停下来标成中断，下次启动不会再续 await scheduler.stop() await catalog_jobs.manager.shutdown() + await catalog_profile_jobs.manager.shutdown() await run_manager.shutdown() from app.sandbox.manager import sandbox_manager diff --git a/backend/tests/test_catalog_profile.py b/backend/tests/test_catalog_profile.py index 9c6a5fe9..fee64c5f 100644 --- a/backend/tests/test_catalog_profile.py +++ b/backend/tests/test_catalog_profile.py @@ -1,9 +1,10 @@ -"""数据剖析接口：POST /api/datasources/{id}/catalog/profile（阶段 2A）。 +"""数据剖析：POST /api/datasources/{id}/catalog/profile-jobs 建后台任务，逐表剖析（阶段 2A，后台任务改造之后）。 剖析是数据目录里唯一对业务库发查询的地方，这里逐条钉住方案定的约束：默认关闭、只读、每条查询都过守卫、 -查询次数和单条时限、总时长、遮罩的列不取样、大表不做整表统计、同一个数据源同时只有一次剖析； +每张表的查询次数和时长、单条时限、遮罩的列不取样、大表不做整表统计、同一个数据源同时只有一个剖析任务； 以及剖析的结论怎么进目录：命名推断对上数据的升为有确证（来源记为数据剖析，和外键分开）、对不上的写明 覆盖率、人工确认过的只补覆盖率和基数、状态列得到码值候选、只有一个日期列的表提议业务日期。 +任务本身（排队、停止、重启、重试、止损、范围、留痕、保留、设置迁移）在 test_catalog_profile_jobs.py。 所有测试都在合成的景区库上做（tests/fixtures/catalog/scenic.py），数据和名称全部虚构。 """ @@ -22,7 +23,7 @@ from httpx import ASGITransport, AsyncClient from sqlalchemy import event from sqlalchemy.engine import Engine -from app.data import catalog, catalog_profile +from app.data import catalog_profile, catalog_profile_jobs from app.data import engine as data_engine from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings from app.db.base import SessionLocal @@ -65,6 +66,8 @@ async def make_source(client): return sid yield _make + # 后台任务先收完（停不下来的停掉），再删源：删源会级联删掉任务 + await catalog_profile_jobs.manager.shutdown(grace=5) for sid in made: await drop_source(sid) @@ -119,7 +122,39 @@ def spy(monkeypatch, scenic_db): def _url(sid: str) -> str: - return f"/api/datasources/{sid}/catalog/profile" + return f"/api/datasources/{sid}/catalog/profile-jobs" + + +async def _start(client, sid: str, tables: list[str] | None = None, **body): + payload = {**({"tables": tables} if tables is not None else {}), **body} + return await client.post(_url(sid), json=payload, headers=ACTOR) + + +async def _job(client, sid: str, job_id: str) -> dict: + r = await client.get(f"{_url(sid)}/{job_id}") + assert r.status_code == 200, r.text + return r.json()["job"] + + +async def _detail(client, sid: str, job_id: str, seq: int) -> dict: + r = await client.get(f"{_url(sid)}/{job_id}/tasks/{seq}") + assert r.status_code == 200, r.text + return r.json()["task"] + + +async def _finished(client, sid: str, job_id: str) -> dict: + """等任务做完，返回做完的任务；job["tables"] 是每张表的明细（检查项、发现、跳过的项、发过的查询），按剖析顺序。""" + await catalog_profile_jobs.manager.wait(job_id, timeout=30) + job = await _job(client, sid, job_id) + job["tables"] = [await _detail(client, sid, job_id, t["seq"]) for t in job["tasks"]] + return job + + +async def _profile(client, sid: str, tables: list[str] | None = None, **body) -> dict: + """建剖析任务（202）、等它做完，返回做完的任务（见 _finished）。""" + r = await _start(client, sid, tables, **body) + assert r.status_code == 202, r.text + return await _finished(client, sid, r.json()["job"]["id"]) async def _notes(client, sid: str, table: str) -> dict: @@ -147,31 +182,35 @@ async def test_profile_is_off_by_default_and_says_where_to_turn_it_on(client, ma sid = await make_source(scenic_db, draft=["visits"]) watch = spy(scenic_db) watch.clear() - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) + r = await _start(client, sid, ["visits"]) assert r.status_code == 409 detail = r.json()["detail"] assert "数据源设置" in detail and "开启数据剖析" in detail assert r.json()["code"] == "profile_disabled" # 界面按机读码认是哪一种，不从原话里认 assert watch.executed == [] # 一条查询都没发 + assert (await client.get(f"{_url(sid)}/latest")).json()["job"] is None, "没建任务" async def test_settings_are_saved_and_validated_through_datasource_update(client, make_source, scenic_db): sid = await make_source(scenic_db) - r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=0)}) - assert r.status_code == 422 and "查询次数上限" in r.json()["detail"] + r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(table_max_queries=0)}) + assert r.status_code == 422 and "每张表最多查询数" in r.json()["detail"] # 机读码和出错的那一项：表单按 field 把报错落到那一格 - assert (r.json()["code"], r.json()["field"]) == ("profile_settings_invalid", "max_queries") + assert (r.json()["code"], r.json()["field"]) == ("profile_settings_invalid", "table_max_queries") + r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(table_max_s=5)}) + assert r.status_code == 422 and (r.json()["code"], r.json()["field"]) == ("profile_settings_invalid", "table_max_s") + assert "每张表时长上限" in r.json()["detail"] r = await client.patch(f"/api/datasources/{sid}", json={"options": {PROFILE_OPTION: {"enabled": "是"}}}) assert r.status_code == 422 and (r.json()["code"], r.json()["field"]) == ("profile_settings_invalid", "enabled") r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(rows=5)}) assert r.status_code == 422 and r.json()["code"] == "profile_settings_invalid" and "field" not in r.json() - r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=40, sample_size=500)}) + r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(table_max_queries=30, sample_size=500)}) assert r.status_code == 200, r.text - assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "max_queries": 40, "sample_size": 500} - # 设好以后就能剖析；返回里带着这次实际用的设置 - r = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR) - assert r.status_code == 200, r.text - assert r.json()["settings"]["max_queries"] == 40 and r.json()["settings"]["sample_size"] == 500 + assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "table_max_queries": 30, "sample_size": 500} + # 设好以后就能剖析；任务里记着这次实际用的设置（快照） + job = await _profile(client, sid, ["channels"]) + assert job["settings"] == {"enabled": True, "table_max_queries": 30, "table_max_s": 60, "query_timeout_s": 10, + "sample_size": 500, "max_scan_rows": 100_000} # ---------------------------------------------------------------- 剖析的结论 @@ -182,15 +221,15 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"]) watch = spy(scenic_db) watch.clear() - r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR) - assert r.status_code == 200, r.text - body = r.json() + body = await _profile(client, sid, ["visits", "channels"]) - # 所有发到业务库的语句都经过了守卫 + # 所有发到业务库的语句都经过了守卫；任务记的查询数就是发到库的条数，也是逐表的合计 assert watch.all_guarded(), watch.executed - assert body["queries_used"] == len(watch.executed) <= body["settings"]["max_queries"] - assert body["queries_used"] == sum(t["queries"] for t in body["tables"]) - assert body["stopped"] is None and body["actor"] == "周宁" and body["profiled_at"] + assert body["queries"] == len(watch.executed) == sum(t["queries"] for t in body["tables"]) + assert all(t["queries"] <= body["settings"]["table_max_queries"] for t in body["tables"]) + assert (body["status"], body["stop_reason"], body["error"]) == ("succeeded", None, None) + assert body["created_by"] == "周宁" and body["finished_at"] and body["scope"] == "selected" + assert [t["table_name"] for t in body["tasks"]] == ["visits", "channels"], "按给的顺序" notes = (await _notes(client, sid, "visits")) assert notes["updated_by"] == "周宁" @@ -226,6 +265,7 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d assert "channel_code" not in channels.get("columns", {}) # 唯一约束列不是码值 visits = _table(body, "visits") + assert visits["status"] == "ok" and visits["unfinished"] == 0 kinds = sorted(f["kind"] for f in visits["findings"]) # 两条命名推断的关系（gate_id、member_id），加两条外键关系（park_id、ticket_type_id）：外键的只核实基数 assert kinds == ["business_date", "codes", "relation", "relation", "relation", "relation"] @@ -243,6 +283,19 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d assert visits["row_estimate"] == {"rows": 1500, "method": "count", "at_least": None} # 更新的 4 项：两条命名关系升为有确证，两条外键关系补上用数据核实过的基数 assert visits["added"] == 2 and visits["updated"] == 4 and visits["error"] is None + # 发现的合计：逐表、整个任务各一份 + assert visits["summary"] == {"relations_checked": 4, "relations_verified": 4, "codes": 1, + "codes_pending": blank, "business_dates": 1} + assert body["summary"]["relations_checked"] == 4 and body["summary"]["codes"] == 2 + # 检查项：每项都结束了，按种类的进度对得上 + assert visits["checks_done"] == visits["checks_total"] == len(visits["checks"]) + assert visits["checks_by_kind"]["relation"] == [4, 4] and visits["checks_by_kind"]["date"] == [1, 1] + assert {c["state"] for c in visits["checks"]} == {"done"} + assert sum(c["queries"] for c in visits["checks"]) + sum( + 1 for q in visits["query_log"] if q["kind"] == "row_estimate" and q["target"] == "visits") >= 1 + # 第一条查询是估行数 + assert visits["query_log"][0]["kind"] == "row_estimate" and visits["query_log"][0]["outcome"] == "ok" + assert len(visits["query_log"]) == visits["queries"] async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, make_source, tmp_path): @@ -261,16 +314,15 @@ async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, before = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id") assert (before["to_table"], before["source"], before["status"]) == ("lots", "name", "proposed") - r = await client.post(_url(sid), json={"tables": ["parking_records"]}, headers=ACTOR) - assert r.status_code == 200, r.text + body = await _profile(client, sid, ["parking_records"]) rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id") assert rel["status"] == "proposed" and rel["source"] == "profile" assert rel["coverage"] == 0.15 assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"] - finding = _table(r.json(), "parking_records")["findings"][0] + finding = _table(body, "parking_records")["findings"][0] assert finding["status"] == "proposed" and finding["coverage"] == 0.15 - # 两个日期类列：不提议业务日期，取值范围只在返回里 - ranges = {d["column"] for d in _table(r.json(), "parking_records")["date_ranges"]} + # 两个日期类列：不提议业务日期，取值范围只在剖析结果里 + ranges = {d["column"] for d in _table(body, "parking_records")["date_ranges"]} assert ranges == {"entered_at"} # exited_at 全是空值 assert "business_date" not in (await _notes(client, sid, "parking_records"))["notes"] @@ -284,22 +336,21 @@ async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, mak "if_version": detail["version"]}) assert r.status_code == 200, r.text - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) - assert r.status_code == 200, r.text + body = await _profile(client, sid, ["visits"]) rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id") assert (rel["source"], rel["status"]) == ("name", "confirmed") assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one" # 人定的基数不改；原来就有基数（命名推断记的多对一），剖析也不给它记「用数据核实过」 assert "cardinality_checked" not in rel - finding = next(f for f in _table(r.json(), "visits")["findings"] if f.get("columns") == ["gate_id"]) + finding = next(f for f in _table(body, "visits")["findings"] if f.get("columns") == ["gate_id"]) assert finding["status"] == "confirmed" and finding["confirmed"] is True async def test_rerun_with_same_data_changes_nothing(client, make_source, scenic_db): """同一天、数据没变：第二次剖析不改目录、不升版本（冻结进快照的目录不会无故变）。""" sid = await make_source(scenic_db, options=_on(), draft=["visits"]) - first = (await client.post(_url(sid), json={"tables": ["visits"]})).json() - second = (await client.post(_url(sid), json={"tables": ["visits"]})).json() + first = await _profile(client, sid, ["visits"]) + second = await _profile(client, sid, ["visits"]) a, b = _table(first, "visits"), _table(second, "visits") assert (b["added"], b["updated"], b["removed"]) == (0, 0, 0) assert b["version"] == a["version"] @@ -318,32 +369,38 @@ async def test_masked_columns_are_never_queried(client, make_source, scenic_db, draft=["visits", "channels"]) watch = spy(scenic_db) watch.clear() - r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR) - assert r.status_code == 200, r.text + body = await _profile(client, sid, ["visits", "channels"]) assert watch.all_guarded() assert watch.mentions("member_id") == [] and watch.mentions("channel_type") == [] - visits, channels = _table(r.json(), "visits"), _table(r.json(), "channels") + visits, channels = _table(body, "visits"), _table(body, "channels") masked = [s for s in visits["skipped"] + channels["skipped"] if s["reason"] == "masked"] assert {s["target"] for s in masked} == {"member_id → members.id", "channel_type"} assert all("遮罩" in s["detail"] for s in masked) + # 按规则不查的不算没做完：表照样是完成 + assert visits["status"] == "ok" and visits["unfinished"] == 0 and visits["skipped_count"] == len(visits["skipped"]) + # 留痕里也没有遮罩的列 + assert not any("member_id" in q["sql"].lower() or "channel_type" in q["sql"].lower() + for t in body["tables"] for q in t["query_log"]) notes = (await _notes(client, sid, "visits"))["notes"] assert _relation(notes, "member_id")["source"] == "name" # 没动 assert _relation(notes, "gate_id")["status"] == "verified" # 其余照常 assert "channel_type" not in ((await _notes(client, sid, "channels"))["notes"].get("columns") or {}) -async def test_query_budget_stops_profiling_and_reports_the_rest(client, make_source, scenic_db, spy): - sid = await make_source(scenic_db, options=_on(max_queries=3), draft=["visits"]) +async def test_each_table_has_its_own_query_budget(client, make_source, scenic_db, spy): + """逐表预算：visits 用完自己的 3 条查询记部分完成，剩下的项记「这张表的查询次数用完」、旧结论原样留着；后面的 + channels 照样拿满额，做完。""" + sid = await make_source(scenic_db, options=_on(table_max_queries=3), draft=["visits"]) watch = spy(scenic_db) watch.clear() - r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR) - assert r.status_code == 200, r.text - body = r.json() - assert body["queries_used"] == 3 == len(watch.executed) and body["stopped"] == "budget" + body = await _profile(client, sid, ["visits", "channels"]) + assert body["queries"] == len(watch.executed) and body["stop_reason"] is None visits, channels = _table(body, "visits"), _table(body, "channels") - assert "budget" in _reasons(visits) - assert channels["queries"] == 0 and _reasons(channels) == {"budget"} - assert all("查询次数" in s["detail"] for s in visits["skipped"] + channels["skipped"] if s["reason"] == "budget") + assert visits["status"] == "partial" and visits["queries"] == 3 + assert "budget" in _reasons(visits) and visits["unfinished"] >= 1 + assert all("这张表的查询次数" in s["detail"] for s in visits["skipped"] if s["reason"] == "budget") + assert channels["status"] == "ok" and 0 < channels["queries"] <= 3 and "budget" not in _reasons(channels) + assert body["status"] == "partial", "有部分完成的表" # 没查完的关系不改：原来的命名推断原样留着 rel = _relation((await _notes(client, sid, "visits"))["notes"], "member_id") assert (rel["source"], rel["status"]) == ("name", "proposed") @@ -362,18 +419,20 @@ async def test_single_query_timeout_is_recorded_and_profiling_continues(client, return await real(source, sql, limits=limits) monkeypatch.setattr(data_engine, "run_query", slow_gate) - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) - assert r.status_code == 200, r.text - visits = _table(r.json(), "visits") + body = await _profile(client, sid, ["visits"]) + visits = _table(body, "visits") timeout = [s for s in visits["skipped"] if s["reason"] == "timeout"] assert [s["target"] for s in timeout] == ["gate_id → gates.id"] and "超时" in timeout[0]["detail"] - assert r.json()["stopped"] is None + assert body["stop_reason"] is None and body["timeouts"] == 1 + assert visits["status"] == "partial", "超时算运行原因没查成" + assert [q["outcome"] for q in visits["query_log"]].count("timeout") == 1 notes = (await _notes(client, sid, "visits"))["notes"] assert _relation(notes, "gate_id")["source"] == "name" # 超时的不改 assert _relation(notes, "member_id")["status"] == "verified" # 其余照常 -async def test_total_deadline_stops_profiling(make_source, scenic_db, monkeypatch): +async def test_each_table_has_its_own_deadline(make_source, scenic_db, monkeypatch): + """逐表时长：每条查询慢 0.5 秒，visits 的时长（1.4 秒）用完只停它自己，channels 另拿一份满额的时长。""" sid = await make_source(scenic_db, draft=["visits"]) real = data_engine.run_query @@ -384,13 +443,13 @@ async def test_total_deadline_stops_profiling(make_source, scenic_db, monkeypatc monkeypatch.setattr(data_engine, "run_query", slow) async with SessionLocal() as session: row = await session.get(DataSource, sid) - settings = ProfileSettings(enabled=True, max_total_s=1.4) - report = await catalog_profile.profile_catalog(session, row, row, tables=["visits"], actor=None, - settings=settings) - assert report.stopped == "deadline" - assert 1 <= report.queries_used <= 3 - reasons = {s.reason for t in report.tables for s in t.skipped} - assert "deadline" in reasons + cx = catalog_profile.make_context(row, ProfileSettings(enabled=True, table_max_s=1.4), options=row.options) + visits = await catalog_profile.profile_table(cx, "visits") + assert visits.status == "partial" and 1 <= visits.queries <= 3 + assert "deadline" in {s.reason for s in visits.skipped} + assert any("这张表的时长" in s.detail for s in visits.skipped if s.reason == "deadline") + channels = await catalog_profile.profile_table(cx, "channels") + assert channels.status == "ok" and channels.queries == 2, [s.to_dict() for s in channels.skipped] async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, spy): @@ -398,15 +457,15 @@ async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, sp sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits"]) watch = spy(scenic_db) watch.clear() - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) - assert r.status_code == 200, r.text - visits = _table(r.json(), "visits") + body = await _profile(client, sid, ["visits"]) + visits = _table(body, "visits") assert visits["row_estimate"] == {"rows": None, "method": "count", "at_least": 101} on_visits = [s for s in watch.executed if "from visits" in s.lower()] assert on_visits and all("LIMIT" in s for s in on_visits), on_visits assert not any("MIN(" in s for s in on_visits) assert "too_large" in _reasons(visits) assert any("太大" in s["detail"] or "上限" in s["detail"] for s in visits["skipped"] if s["reason"] == "too_large") + assert visits["status"] == "ok", "表太大是按规则不查，不算没做完" # 父表有主键，覆盖率又够：照样有确证；子表一侧没核对唯一，按多对一记，不算用数据核实过基数 rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id") assert rel["status"] == "verified" and rel["cardinality"] == "many_to_one" @@ -416,7 +475,7 @@ async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, sp # ---------------------------------------------------------------- 接口 -async def test_concurrent_profile_on_same_source_returns_409(client, make_source, scenic_db, monkeypatch): +async def test_second_profile_job_on_same_source_returns_409(client, make_source, scenic_db, monkeypatch): sid = await make_source(scenic_db, options=_on(), draft=["visits"]) real = data_engine.run_query started, release = asyncio.Event(), asyncio.Event() @@ -427,29 +486,39 @@ async def test_concurrent_profile_on_same_source_returns_409(client, make_source return await real(source, sql, limits=limits) monkeypatch.setattr(data_engine, "run_query", held) - first = asyncio.create_task(client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)) + first = await _start(client, sid, ["visits", "channels"]) + assert first.status_code == 202, first.text await asyncio.wait_for(started.wait(), 10) - second = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) - assert second.status_code == 409 and "正在进行数据剖析" in second.json()["detail"] - assert second.json()["code"] == "profile_busy" + second = await _start(client, sid, ["visits"]) + assert second.status_code == 409 and second.json()["code"] == "profile_busy" + # 说清是谁在跑、跑到哪；只点了一张表时说这张表在不在那个任务里、排第几 + assert "正在剖析：第 1 / 2 张表" in second.json()["detail"] and "visits 在这次剖析里，排第 1 张" in second.json()["detail"] + other = await _start(client, sid, ["gates"]) + assert other.status_code == 409 and "gates 不在这次剖析里" in other.json()["detail"] + active = (await client.get(f"{_url(sid)}/active")).json()["job"] + assert active["id"] == first.json()["job"]["id"] and active["status"] == "running" + listed = (await client.get(f"/api/datasources/{sid}/catalog")).json() + assert listed["active_profile_job"] == {"id": active["id"], "status": "running", "total": 2, "done": 0, + "waiting_for": None} release.set() - done = await asyncio.wait_for(first, 30) - assert done.status_code == 200, done.text + await catalog_profile_jobs.manager.wait(active["id"], timeout=30) + assert (await client.get(f"{_url(sid)}/active")).json()["job"] is None # 做完以后可以再来 - again = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR) - assert again.status_code == 200, again.text + again = await _start(client, sid, ["channels"]) + assert again.status_code == 202, again.text + await catalog_profile_jobs.manager.wait(again.json()["job"]["id"], timeout=30) async def test_blocked_profile_says_which_case_by_code(client, make_source, scenic_db, monkeypatch): - """剖析开始不了的 409 都带机读码：数据源停用、没有表结构、快照被改动（未开启、正在剖析见上面两条）。""" - from app.data.engine import SnapshotTampered + """剖析开始不了的 409 都带机读码：数据源停用、没有表结构、快照登记被改动（未开启、正忙见上面两条）。""" + from app.data import table_versions sid = await make_source(scenic_db, options=_on(), draft=["visits"]) async with SessionLocal() as session: row = await session.get(DataSource, sid) row.enabled = False await session.commit() - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) + r = await _start(client, sid, ["visits"]) assert r.status_code == 409 and r.json()["code"] == "datasource_inactive" and "已停用" in r.json()["detail"] async with SessionLocal() as session: @@ -457,103 +526,50 @@ async def test_blocked_profile_says_which_case_by_code(client, make_source, scen row.enabled = True row.schema_cache = {} await session.commit() - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) + r = await _start(client, sid, ["visits"]) assert r.status_code == 409 and r.json()["code"] == "schema_missing" and "探查结构" in r.json()["detail"] r = await client.post(f"/api/datasources/{sid}/introspect") assert r.status_code == 200, r.text + async def tampered(*_args, **_kwargs): + raise table_versions.SnapshotManifestMismatch("数据版本的哈希登记与快照清单不一致，已拒绝查询") + + with monkeypatch.context() as m: + m.setattr(table_versions, "resolve_source", tampered) + r = await _start(client, sid, ["visits"]) + assert r.status_code == 409 and r.json()["code"] == "snapshot_tampered" + assert (await client.get(f"{_url(sid)}/latest")).json()["job"] is None, "开始不了的都没建任务" + + +async def test_a_tampered_snapshot_during_profiling_fails_the_job(client, make_source, scenic_db, monkeypatch): + """剖析途中数据文件和登记的版本对不上：整个任务停下、记失败，原因照实写；没做完的表也记失败，可以重试。""" + from app.data.engine import SnapshotTampered + + sid = await make_source(scenic_db, options=_on(), draft=["visits"]) + async def tampered(*_args, **_kwargs): raise SnapshotTampered("数据文件与登记的版本不一致，可能被修改过，已拒绝查询") - monkeypatch.setattr(catalog_profile, "profile_catalog", tampered) - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) - assert r.status_code == 409 and r.json()["code"] == "snapshot_tampered" - - -async def test_default_tables_are_used_tables_with_relation_candidates(client, make_source, scenic_db): - sid = await make_source(scenic_db, options=_on(), draft=[]) - r = await client.post(_url(sid), json={}, headers=ACTOR) - assert r.status_code == 200, r.text - names = [t["table_name"] for t in r.json()["tables"]] - assert names and len(names) <= catalog_profile.PROFILE_DEFAULT_TABLES - async with SessionLocal() as session: - entries = await catalog.read_catalog(session, sid) - for name in names: - assert catalog_profile._relation_candidates(entries[name].notes), name - assert "parks" not in names and "channels" not in names # 没有待核实关系的表不选 - - -async def _candidates(sid: str) -> tuple[list[str], set[str]]: - """默认挑表的候选（目录里有要核对的关系的表，按表结构顺序）和其中还有待核实（推断）关系的那些。""" - async with SessionLocal() as session: - row = await session.get(DataSource, sid) - entries = await catalog.read_catalog(session, sid) - order = list((row.schema_cache or {}).get("tables") or {}) - rels = {name: catalog_profile._relation_candidates(entries[name].notes) for name in order if name in entries} - names = [name for name in order if rels.get(name)] - return names, {name for name in names if any(r["status"] == "proposed" for r in rels[name])} - - -async def test_default_pick_skips_empty_tables_and_prefers_pending_relations(client, make_source, tmp_path, - monkeypatch): - """B1：表都没被运行查询过（使用次数都是 0）时，以前按表结构顺序取前几张，结果大半是空表（景区库里排在前面的 - 候选表正好多数没有数据）。现在跳过行数估计为 0 的表（记在 empty_tables 里），并且先挑还有待核实关系的表， - 不把刚核实过的再剖析一遍。默认张数调成 3，好在一个小库上看出先后。""" - monkeypatch.setattr(catalog_profile, "PROFILE_DEFAULT_TABLES", 3) - path = scenic.build(tmp_path / "b1.db") - db = sqlite3.connect(path) - try: - tables = [r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY rowid")] - empty = {t for t in tables if db.execute(f'SELECT COUNT(*) FROM "{t}"').fetchone()[0] == 0} - finally: - db.close() - sid = await make_source(str(path), options=_on(), draft=tables) - names, pending = await _candidates(sid) - full = [name for name in names if name not in empty] - # 前提：按表结构顺序的前 3 张候选里有空表（以前就会挑中它们），有数据、还待核实的候选不止 6 张 - assert set(names[:3]) & empty - assert len([name for name in full if name in pending]) >= 6 - - first = (await client.post(_url(sid), json={}, headers=ACTOR)).json() - picked = [t["table_name"] for t in first["tables"]] - assert len(picked) == 3 and not set(picked) & empty, picked - assert first["empty_tables"] and set(first["empty_tables"]) <= empty - assert all(t["queries"] > 0 for t in first["tables"]) - assert first["queries_used"] >= sum(t["queries"] for t in first["tables"]) + len(first["empty_tables"]) - - # 第二次：先挑还有待核实关系的表；第一次核实过的（关系都升为有确证了）排到后面，这次挑不到 - names, pending = await _candidates(sid) - assert not set(picked) & pending # 第一次挑的三张，关系都核实了 - second = (await client.post(_url(sid), json={}, headers=ACTOR)).json() - again = [t["table_name"] for t in second["tables"]] - assert len(again) == 3 and all(name in pending for name in again), (again, pending) - assert not set(again) & set(picked) and not set(again) & empty - - -async def test_profile_note_when_every_candidate_is_empty(client, make_source, tmp_path): - path = scenic.build(tmp_path / "b1-empty.db") - sid = await make_source(str(path), options=_on(), draft=[]) - names, _ = await _candidates(sid) - db = sqlite3.connect(path) - try: - for name in names: - db.execute(f'DELETE FROM "{name}"') - db.commit() - finally: - db.close() - body = (await client.post(_url(sid), json={}, headers=ACTOR)).json() - assert body["tables"] == [] and set(body["empty_tables"]) == set(names) - assert "空表" in body["note"] + monkeypatch.setattr(data_engine, "run_query", tampered) + body = await _profile(client, sid, ["visits", "channels"]) + assert body["status"] == "failed" and body["error"] == "数据文件与登记的版本不一致，可能被修改过，已拒绝查询" + assert {t["status"] for t in body["tables"]} == {"failed"} + assert all(t["error"] == body["error"] for t in body["tables"]) + # 没结束的检查项不留在「排队」：记为跳过，原因写清 + assert all(c["state"] != "pending" for t in body["tables"] for c in t["checks"]) async def test_unknown_table_and_missing_source(client, make_source, scenic_db): sid = await make_source(scenic_db, options=_on(), draft=["visits"]) - r = await client.post(_url(sid), json={"tables": ["nope"]}, headers=ACTOR) - assert r.status_code == 200 - assert _table(r.json(), "nope")["error"] and r.json()["queries_used"] == 0 + body = await _profile(client, sid, ["nope"]) + nope = _table(body, "nope") + assert nope["status"] == "skipped" and nope["error"] == "表结构里没有 nope，请先重新探查结构" + assert body["queries"] == 0 and body["status"] == "succeeded" r = await client.post(_url("0" * 32), json={}, headers=ACTOR) assert r.status_code == 404 + assert (await client.get(f"{_url(sid)}/not-a-job")).status_code == 404 + assert (await client.get(f"{_url(sid)}/{body['id']}/tasks/9")).status_code == 404 def _orders_db(path, *, items_per_order: int) -> str: @@ -583,27 +599,23 @@ async def test_foreign_key_relations_get_their_cardinality_checked_with_data(cli before = _relation((await _notes(client, sid, "order_items"))["notes"], "order_id") assert (before["source"], before["status"]) == ("fk", "verified") - r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR) - assert r.status_code == 200, r.text + body = await _profile(client, sid, ["order_items"]) rel = _relation((await _notes(client, sid, "order_items"))["notes"], "order_id") assert (rel["source"], rel["status"]) == ("fk", "verified") assert rel["cardinality"] == cardinality and rel.get("cardinality_checked") is True assert rel.get("coverage") is None - finding = next(f for f in _table(r.json(), "order_items")["findings"] if f.get("columns") == ["order_id"]) + finding = next(f for f in _table(body, "order_items")["findings"] if f.get("columns") == ["order_id"]) assert finding["cardinality_checked"] is True and finding["cardinality"] == cardinality # 重新剖析、数据没变：版本不升 version = (await _notes(client, sid, "order_items"))["version"] - r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR) - assert r.status_code == 200, r.text + await _profile(client, sid, ["order_items"]) assert (await _notes(client, sid, "order_items"))["version"] == version async def test_fanout_on_a_data_checked_foreign_key_is_an_error_again(client, make_source, tmp_path): """剖析核实过的外键一对多：订单关联明细后对订单金额求和，扇出检查报 error；数据上一对一的不报。""" from app.data.sqlcheck import load_checkers - from app.db.base import SessionLocal - from app.db.models import DataSource sql = "SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id" for items, expected in ((3, "error"), (1, None)): @@ -615,8 +627,7 @@ async def test_fanout_on_a_data_checked_foreign_key_is_an_error_again(client, ma r = await client.put(f"/api/datasources/{sid}/catalog/orders", json={"notes": notes, "if_version": detail["version"]}, headers=ACTOR) assert r.status_code == 200, r.text - r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR) - assert r.status_code == 200, r.text + await _profile(client, sid, ["order_items"]) async with SessionLocal() as session: source = await session.get(DataSource, sid) checker = (await load_checkers(session, [source]))[source.name] diff --git a/backend/tests/test_catalog_profile_jobs.py b/backend/tests/test_catalog_profile_jobs.py new file mode 100644 index 00000000..e9dc2995 --- /dev/null +++ b/backend/tests/test_catalog_profile_jobs.py @@ -0,0 +1,765 @@ +"""数据剖析的后台任务（data/catalog_profile_jobs.py、/api/datasources/{id}/catalog/profile-jobs）。 + +剖析的结论怎么进目录、安全约束（只读、守卫、遮罩、大表）在 test_catalog_profile.py；这里钉住任务本身： + +- 同源排队：起草在跑时剖析排队（waiting_for = "draft"），起草结束自动开始；反过来起草排在剖析后面（不 409）；停掉排队的； + 导入目录、同步起草在剖析时 409 profile_busy；同步起草期间建的剖析任务排在它后面 +- 停止：做完的项写进目录，没做的保留旧结论、记 cancelled，后面的表 cancelled；已结束的再停 409 +- 服务重启标中断、不续跑；关停时同样；重试只带没做完的表 +- 任务级止损：连续 3 条失败、最近 10 条 7 条超时（停下、后面的表带同一句原因），最近 6 条 4 条超时提醒、好转后清掉 +- 每张表开始前重读数据源：停用、关掉剖析就停；遮罩按最新的设置 +- 范围：需要剖析 / 全部 / 指定；空表一条查询记 skipped；视图跳过；没有可剖析的项；挑不出表 409；profile_due 的三种情形 +- 留痕：IN 列表里抽到的键值不进 AgentLab 的库 +- 设置迁移：旧键启动时删掉，老页面带着旧键保存只摘掉 +- 保留：超过保留数删旧的，但不删某张表最近一次剖析所在的任务 + +表名、列名、数据都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import json +import re +import sqlite3 +import uuid +from datetime import timedelta +from types import SimpleNamespace + +import pytest +from sqlalchemy import select + +from app.data import catalog, catalog_jobs, catalog_profile, catalog_profile_jobs, catalog_transfer +from app.data import engine as data_engine +from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings, SourceRunner, SqlDialect +from app.data.guard import SqlRejected +from app.db.base import SessionLocal, utcnow +from app.db.models import CatalogProfileJob, CatalogProfileTask, DataSource +from tests.test_catalog_profile import ( # noqa: F401 + ACTOR, _detail, _finished, _job, _notes, _on, _profile, _relation, _start, _table, _url, client, make_source, +) + + +@pytest.fixture(autouse=True) +async def _stop_jobs(): + yield + await catalog_jobs.manager.shutdown(grace=5) + await catalog_profile_jobs.manager.shutdown(grace=5) + + +async def until(predicate, timeout: float = 10.0) -> None: + for _ in range(int(timeout / 0.01)): + if await predicate(): + return + await asyncio.sleep(0.01) + raise AssertionError("等不到") + + +def _base(sid: str) -> str: + return f"/api/datasources/{sid}/catalog" + + +async def _rows(client, sid: str) -> dict[str, dict]: + r = await client.get(_base(sid)) + assert r.status_code == 200, r.text + return {row["table_name"]: row for row in r.json()["tables"]} + + +class Gate: + """把一个异步函数换成「等放行才执行」：entered 置位说明有调用在等了。""" + + def __init__(self, real, when=lambda *a, **k: True) -> None: + self.real, self.when = real, when + self.entered, self.release = asyncio.Event(), asyncio.Event() + + async def __call__(self, *args, **kwargs): + if self.when(*args, **kwargs): + self.entered.set() + await self.release.wait() + return await self.real(*args, **kwargs) + + +def _hold_queries(monkeypatch, when=lambda source, sql, **kw: True) -> Gate: + gate = Gate(data_engine.run_query, when) + monkeypatch.setattr(data_engine, "run_query", gate) + return gate + + +def _db(path, ddl: str, rows: dict[str, list[tuple]] | None = None) -> str: + db = sqlite3.connect(path) + try: + db.executescript(ddl) + for table, values in (rows or {}).items(): + marks = ", ".join("?" for _ in values[0]) + db.executemany(f"INSERT INTO {table} VALUES ({marks})", values) + db.commit() + finally: + db.close() + return str(path) + + +# -------------------------------------------------------------------------- +# 同源排队 +# -------------------------------------------------------------------------- + + +async def test_a_profile_job_waits_for_the_draft_job_and_then_starts(client, make_source, scenic_db, monkeypatch): + sid = await make_source(scenic_db, options=_on()) + gate = Gate(catalog.write_draft) + monkeypatch.setattr(catalog, "write_draft", gate) + draft = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]})).json()["job"] + await asyncio.wait_for(gate.entered.wait(), 10) + + r = await _start(client, sid, ["channels"]) + assert r.status_code == 202, r.text # 不报错：排在起草后面 + job = r.json()["job"] + assert job["status"] == "queued" and job["waiting_for"] == "draft" + listed = (await client.get(_base(sid))).json() + assert listed["active_profile_job"]["waiting_for"] == "draft" + assert listed["active_job"]["waiting_for"] is None + await asyncio.sleep(0.1) + assert (await _job(client, sid, job["id"]))["status"] == "queued", "起草没结束，剖析不开始" + + gate.release.set() + await catalog_jobs.manager.wait(draft["id"], timeout=10) + done = await _finished(client, sid, job["id"]) + assert done["status"] == "succeeded" and done["waiting_for"] is None + draft_done = (await client.get(f"{_base(sid)}/draft-jobs/{draft['id']}")).json()["job"] + assert done["started_at"] >= draft_done["finished_at"], "起草结束后才开始" + + +async def test_a_draft_job_queues_behind_a_running_profile_job(client, make_source, scenic_db, monkeypatch): + """同种任务互斥、不同种排队：剖析在跑时建起草任务不 409，排在后面（waiting_for = "profile"）。""" + sid = await make_source(scenic_db, options=_on()) + gate = _hold_queries(monkeypatch) + profile = (await _start(client, sid, ["channels"])).json()["job"] + await asyncio.wait_for(gate.entered.wait(), 10) + + r = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]}) + assert r.status_code == 202, r.text + draft = r.json()["job"] + assert draft["status"] == "queued" and draft["waiting_for"] == "profile" + got = (await client.get(f"{_base(sid)}/draft-jobs/{draft['id']}")).json()["job"] + assert got["waiting_for"] == "profile" + assert (await client.get(_base(sid))).json()["active_job"]["waiting_for"] == "profile" + # 同种任务照旧互斥 + again = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]}) + assert again.status_code == 409 and again.json()["code"] == "catalog_draft_busy" + + gate.release.set() + await catalog_profile_jobs.manager.wait(profile["id"], timeout=10) + await catalog_jobs.manager.wait(draft["id"], timeout=10) + got = (await client.get(f"{_base(sid)}/draft-jobs/{draft['id']}")).json()["job"] + profile_done = await _job(client, sid, profile["id"]) + assert got["status"] == "succeeded" and got["started_at"] >= profile_done["finished_at"] + + +async def test_a_queued_profile_job_can_be_cancelled(client, make_source, scenic_db, monkeypatch): + sid = await make_source(scenic_db, options=_on()) + gate = Gate(catalog.write_draft) + monkeypatch.setattr(catalog, "write_draft", gate) + draft = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]})).json()["job"] + await asyncio.wait_for(gate.entered.wait(), 10) + job = (await _start(client, sid, ["channels", "visits"])).json()["job"] + assert job["tasks"][0]["checks_total"] > 0, "排队的表也先列好了检查项" + + r = await client.post(f"{_url(sid)}/{job['id']}/cancel") + assert r.status_code == 200, r.text + stopped = r.json()["job"] + assert stopped["status"] == "cancelled" and stopped["finished_at"] and stopped["queries"] == 0 + assert {t["status"] for t in stopped["tasks"]} == {"cancelled"} + detail = await _detail(client, sid, job["id"], 0) + assert {c["state"] for c in detail["checks"]} == {"skipped"}, "结束了的表上不留排着的检查项" + assert {c["reason"] for c in detail["checks"]} == {"cancelled"} and detail["unfinished"] == len(detail["checks"]) + assert not catalog_profile_jobs.manager.running(job["id"]), "在队里等着的也停掉了" + again = await client.post(f"{_url(sid)}/{job['id']}/cancel") + assert again.status_code == 409 and again.json()["code"] == "catalog_profile_finished" + + gate.release.set() + await catalog_jobs.manager.wait(draft["id"], timeout=10) + assert (await client.get(f"{_base(sid)}/draft-jobs/{draft['id']}")).json()["job"]["status"] == "succeeded" + + +async def test_import_and_sync_draft_are_409_while_profiling(client, make_source, scenic_db, monkeypatch): + sid = await make_source(scenic_db, options=_on(), draft=["gates"]) + exported = (await client.get(f"{_base(sid)}/export")).json() + gate = _hold_queries(monkeypatch) + job = (await _start(client, sid, ["channels"])).json()["job"] + await asyncio.wait_for(gate.entered.wait(), 10) + + # 说法按这次要做的事说下一步：是在导入、起草，不是要开始新的剖析 + r = await client.post(f"{_base(sid)}/import", json={"content": exported}) + assert r.status_code == 409 and r.json()["code"] == "profile_busy", r.text + assert "正在剖析" in r.json()["detail"] and r.json()["detail"].endswith("请等这次剖析结束（或先停止它）再导入") + r = await client.post(f"{_base(sid)}/draft", json={"tables": ["gates"]}) + assert r.status_code == 409 and r.json()["code"] == "profile_busy", r.text + assert r.json()["detail"].endswith("请等这次剖析结束（或先停止它）再起草") + again = await _start(client, sid, ["gates"]) + assert again.json()["detail"].endswith("请等这次剖析结束（或先停止它）再开始新的剖析") + # 库里查过没有任务、队却被剖析占着（任务刚好在收尾）时同样按这次要做的事说 + row, holder = SimpleNamespace(name="x"), SimpleNamespace(kind="profile") + assert str(catalog_jobs._lane_busy(row, holder, "import")).endswith("（或先停止它）再导入") + assert str(catalog_jobs._lane_busy(row, holder, "sync_draft")).endswith("（或先停止它）再起草") + assert str(catalog_jobs._lane_busy(row, SimpleNamespace(kind="draft"), "import")).endswith("（或先停止它）再导入") + + gate.release.set() + await catalog_profile_jobs.manager.wait(job["id"], timeout=10) + r = await client.post(f"{_base(sid)}/import", json={"content": exported}) + assert r.status_code == 200, r.text + + +async def test_a_profile_job_created_during_a_sync_write_runs_after_it(client, make_source, scenic_db, monkeypatch): + """同步起草、导入目录执行期间占着这个源的队：期间新建的剖析任务排在后面，做完才开始。""" + sid = await make_source(scenic_db, options=_on()) + gate = Gate(catalog.draft_catalog) + monkeypatch.setattr(catalog, "draft_catalog", gate) + sync = asyncio.create_task(client.post(f"{_base(sid)}/draft", json={"tables": ["gates"]})) + await asyncio.wait_for(gate.entered.wait(), 10) + job = (await _start(client, sid, ["channels"])).json()["job"] + assert job["status"] == "queued" and job["waiting_for"] == "draft" + await asyncio.sleep(0.1) + assert (await _job(client, sid, job["id"]))["status"] == "queued" + gate.release.set() + assert (await sync).status_code == 200 + assert (await _finished(client, sid, job["id"]))["status"] == "succeeded" + + importing = Gate(catalog_transfer.run_import) + monkeypatch.setattr(catalog_transfer, "run_import", importing) + exported = (await client.get(f"{_base(sid)}/export")).json() + imported = asyncio.create_task(client.post(f"{_base(sid)}/import", json={"content": exported})) + await asyncio.wait_for(importing.entered.wait(), 10) + job = (await _start(client, sid, ["channels"])).json()["job"] + assert job["status"] == "queued" and job["waiting_for"] is None + # 导入期间再导入一次：同步写操作一次一个 + busy = await client.post(f"{_base(sid)}/import", json={"content": exported}) + assert busy.status_code == 409, busy.text + importing.release.set() + assert (await imported).status_code == 200 + assert (await _finished(client, sid, job["id"]))["status"] == "succeeded" + + +# -------------------------------------------------------------------------- +# 停止、服务重启、重试 +# -------------------------------------------------------------------------- + + +def _at_visits_codes(source, sql, **kw) -> bool: + return "GROUP BY status" in sql and "FROM visits" in sql + + +async def test_stopping_writes_what_is_done_and_keeps_the_rest(client, make_source, scenic_db, monkeypatch): + """停在 visits 的码值上：关系都做完了，照常写进目录；码值、日期没做，记停止、目录不动；后面的 orders 记停止。""" + sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"]) + gate = _hold_queries(monkeypatch, _at_visits_codes) + job = (await _start(client, sid, ["channels", "visits", "orders"])).json()["job"] + await asyncio.wait_for(gate.entered.wait(), 10) + running = await _job(client, sid, job["id"]) + row = next(t for t in running["tasks"] if t["table_name"] == "visits") + assert row["status"] == "running" and row["current"] == "正在取 visits.status 的取值分布" + assert row["checks_by_kind"]["relation"] == [4, 4] and row["checks_by_kind"]["codes"] == [0, 1] + + r = await client.post(f"{_url(sid)}/{job['id']}/cancel") + assert r.status_code == 200, r.text + stopped = r.json()["job"] + by = {t["table_name"]: t for t in stopped["tasks"]} + assert stopped["status"] == "cancelled" + assert by["channels"]["status"] == "ok" + assert by["visits"]["status"] == "partial" and by["visits"]["current"] is None + assert by["visits"]["error"] == "剖析已停止：已做完的 4 项照常写进了数据目录，剩下 2 项没查" + assert by["orders"]["status"] == "cancelled" and by["orders"]["error"] is None + + detail = await _detail(client, sid, job["id"], 1) + states = {c["kind"]: (c["state"], c["reason"]) for c in detail["checks"]} + assert states["codes"] == ("skipped", "cancelled") and states["date"] == ("skipped", "cancelled") + assert detail["query_log"][-1]["outcome"] == "error" and "没有等这条查询的结果" in detail["query_log"][-1]["detail"] + notes = (await _notes(client, sid, "visits"))["notes"] + assert _relation(notes, "gate_id")["status"] == "verified", "做完的项写进了目录" + assert "status" not in (notes.get("columns") or {}) and "business_date" not in notes, "没做的不动" + again = await client.post(f"{_url(sid)}/{job['id']}/cancel") + assert again.status_code == 409 and again.json()["code"] == "catalog_profile_finished" + + +async def test_shutdown_marks_the_job_interrupted(client, make_source, scenic_db, monkeypatch): + sid = await make_source(scenic_db, options=_on(), draft=["visits"]) + gate = _hold_queries(monkeypatch, _at_visits_codes) + job = (await _start(client, sid, ["visits", "orders"])).json()["job"] + await asyncio.wait_for(gate.entered.wait(), 10) + await catalog_profile_jobs.manager.shutdown(grace=5) + got = await _job(client, sid, job["id"]) + by = {t["table_name"]: t for t in got["tasks"]} + assert got["status"] == "interrupted" and got["error"] == "服务重启，剖析中断" + assert by["visits"]["status"] == "partial" and by["visits"]["error"].startswith("服务重启，剖析中断：已做完的 4 项") + assert by["orders"]["status"] == "cancelled" and by["orders"]["error"] == "服务重启，剖析中断" + assert _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")["status"] == "verified" + + +async def test_startup_marks_unfinished_jobs_interrupted_and_does_not_resume(client, make_source, scenic_db): + sid = await make_source(scenic_db, options=_on()) + checks = [{"kind": "codes", "target": "status", "path": "columns.status.codes", "state": "done", "reason": None, + "detail": None, "queries": 1}, + {"kind": "date", "target": "visit_time", "path": "business_date", "state": "running", "reason": None, + "detail": None, "queries": 0}] + async with SessionLocal() as session: + job = CatalogProfileJob(source_id=sid, status="running", scope="selected", settings={"enabled": True}) + session.add(job) + await session.flush() + session.add(CatalogProfileTask(job_id=job.id, seq=0, table_name="channels", status="ok")) + session.add(CatalogProfileTask(job_id=job.id, seq=1, table_name="visits", status="running", checks=checks)) + session.add(CatalogProfileTask(job_id=job.id, seq=2, table_name="gates", status="pending")) + await session.commit() + job_id = job.id + assert await catalog_profile_jobs.startup() >= 1 + got = await _job(client, sid, job_id) + by = {t["table_name"]: t for t in got["tasks"]} + assert got["status"] == "interrupted" and got["error"] == "服务重启，剖析中断" and got["finished_at"] + assert by["channels"]["status"] == "ok" and by["channels"]["error"] is None + assert by["visits"]["status"] == by["gates"]["status"] == "cancelled" + assert by["gates"]["error"] == "服务重启，剖析中断" + # 进程没了，visits 没走到写目录那一步：查完的码值也没写进目录，不能还标「完成」 + assert by["visits"]["error"] == "服务重启，剖析中断：已查完的 1 项没有写进数据目录，剩下 1 项没查" + visits = await _detail(client, sid, job_id, 1) + assert [(c["state"], c["reason"]) for c in visits["checks"]] == [("skipped", "cancelled"), ("skipped", "cancelled")] + assert visits["checks"][0]["detail"] == "服务重启，剖析中断，查完了但结论没有写进数据目录" + assert visits["checks"][1]["detail"] == "服务重启，剖析中断，未检查" + assert visits["unfinished"] == visits["skipped_count"] == 2 and visits["findings"] == [] + assert not catalog_profile_jobs.manager.running(job_id), "不自动续跑" + + +async def test_a_job_failing_mid_table_does_not_claim_the_finished_checks(client, make_source, scenic_db, + monkeypatch): + """visits 的 4 条关系核对完、取码值时快照被改动：任务整体失败，visits 没走到写目录那一步。核对完的关系不能还标「完成」： + 记为跳过（error）、写明没写进目录，算没做完；目录里那几条关系原样不动；重试带上 visits。""" + from app.data.engine import SnapshotTampered + + sid = await make_source(scenic_db, options=_on(), draft=["visits"]) + + async def tampered(*_args, **_kwargs): + raise SnapshotTampered("数据文件与登记的版本不一致，可能被修改过，已拒绝查询") + + monkeypatch.setattr(catalog_profile, "_check_codes", tampered) + body = await _profile(client, sid, ["visits"]) + visits = body["tables"][0] + assert body["status"] == "failed" and visits["status"] == "failed" + relations = [c for c in visits["checks"] if c["kind"] == "relation"] + assert len(relations) == 4 and all(c["queries"] > 0 for c in relations), "关系确实查过了" + assert all((c["state"], c["reason"]) == ("skipped", "error") for c in visits["checks"]) + assert relations[0]["detail"] == "数据文件与登记的版本不一致，可能被修改过，已拒绝查询，查完了但结论没有写进数据目录" + assert visits["error"] == "数据文件与登记的版本不一致，可能被修改过，已拒绝查询：已查完的 4 项没有写进数据目录，剩下 2 项没查" + assert visits["unfinished"] == 6 and visits["findings"] == [] and visits["query_log"], "发过的查询照样留痕" + notes = (await _notes(client, sid, "visits"))["notes"] + assert _relation(notes, "gate_id")["status"] == "proposed" and _relation(notes, "gate_id")["source"] == "name" + async with SessionLocal() as session: + failed = await session.get(CatalogProfileJob, body["id"]) + assert await catalog_profile_jobs.retry_tables(session, failed) == ["visits"] + + +async def test_the_catalog_and_the_table_result_commit_together(client, make_source, scenic_db, monkeypatch): + """目录和这张表的剖析结果在同一个事务里：记结果那一步出错，目录也没写进去，收尾时照实说「没写进目录」。""" + sid = await make_source(scenic_db, options=_on(), draft=["visits"]) + real = catalog_profile_jobs._ProfileRun._save_table + + async def broken(self, session, seq, result): + await real(self, session, seq, result) + raise RuntimeError("disk I/O error") + + monkeypatch.setattr(catalog_profile_jobs._ProfileRun, "_save_table", broken) + body = await _profile(client, sid, ["visits"]) + visits = body["tables"][0] + assert body["status"] == "failed" and visits["status"] == "failed" + assert visits["error"].endswith("已查完的 6 项没有写进数据目录") + notes = (await _notes(client, sid, "visits"))["notes"] + assert _relation(notes, "gate_id")["status"] == "proposed", "目录跟着结果一起回滚，没写进去" + + +async def test_retry_takes_only_unfinished_tables(client, make_source, scenic_db, monkeypatch): + sid = await make_source(scenic_db, options=_on(table_max_queries=2), draft=["visits"]) + first = await _profile(client, sid, ["visits", "channels", "nope"]) + by = {t["table_name"]: t["status"] for t in first["tasks"]} + assert by == {"visits": "partial", "channels": "ok", "nope": "skipped"} and first["status"] == "partial" + + r = await client.patch(f"/api/datasources/{sid}", json={"options": _on()}) + assert r.status_code == 200, r.text + r = await client.post(f"{_url(sid)}/{first['id']}/retry", headers=ACTOR) + assert r.status_code == 202, r.text + retry = r.json()["job"] + assert retry["retry_of"] == first["id"] and retry["scope"] == "selected" + assert [t["table_name"] for t in retry["tasks"]] == ["visits"] + assert retry["settings"]["table_max_queries"] == 40, "设置取数据源当前的" + done = await _finished(client, sid, retry["id"]) + assert done["status"] == "succeeded" + r = await client.post(f"{_url(sid)}/{retry['id']}/retry") + assert r.status_code == 422 and "不需要重试" in r.json()["detail"] + + monkeypatch.setattr(catalog_profile_jobs.manager, "start", lambda job_id: None) # 排着、不开始 + queued = (await _start(client, sid, ["gates"])).json()["job"] + r = await client.post(f"{_url(sid)}/{first['id']}/retry") + assert r.status_code == 409 and r.json()["code"] == "profile_busy" + await client.post(f"{_url(sid)}/{queued['id']}/cancel") + + +# -------------------------------------------------------------------------- +# 任务级止损、每张表开始前重读数据源 +# -------------------------------------------------------------------------- + +_MANY = ["visits", "orders", "members", "channels", "ticket_types", "gates", "parks", "stores", "products", + "payments", "refunds", "reviews"] + + +async def test_three_failures_in_a_row_stop_the_job(client, make_source, scenic_db, monkeypatch): + sid = await make_source(scenic_db, options=_on(), draft=["visits"]) + + async def broken(source, sql, *, limits=None): + raise RuntimeError("connection refused by the server\ndetails follow") + + monkeypatch.setattr(data_engine, "run_query", broken) + job = await _profile(client, sid, ["visits", "channels", "orders"]) + assert job["status"] == "partial" and job["stop_reason"] == "failed" + assert job["error"] == "连续 3 条查询失败，已停止剖析：connection refused by the server" + assert (job["queries"], job["errors"], job["timeouts"]) == (3, 3, 0) + visits, channels, orders = (_table(job, n) for n in ("visits", "channels", "orders")) + assert visits["status"] == "partial" and visits["error"] == job["error"] + assert "failed" in {s["reason"] for s in visits["skipped"]} + assert channels["status"] == orders["status"] == "cancelled" + assert channels["error"] == orders["error"] == job["error"] and channels["queries"] == 0 + + +async def test_mostly_timeouts_stop_the_job_as_slow(client, make_source, scenic_db, monkeypatch): + sid = await make_source(scenic_db, options=_on()) + + async def slow(source, sql, *, limits=None): + raise SqlRejected("查询超过 1 秒被中断。请添加 WHERE 条件或 LIMIT 缩小范围。") + + monkeypatch.setattr(data_engine, "run_query", slow) + job = await _profile(client, sid, _MANY) + assert job["stop_reason"] == "slow" and job["status"] == "partial" + assert job["error"] == "数据库响应慢：最近 10 条查询 7 条超时，已停止，避免给业务库加压" + assert job["timeouts"] == job["queries"] == 7 + assert job["notice"] and job["notice"].startswith("数据库响应慢") + cancelled = [t for t in job["tasks"] if t["status"] == "cancelled"] + assert cancelled and all(t["error"] == job["error"] for t in cancelled) + + +def _runner(monkeypatch, outcomes: list[str]) -> SourceRunner: + """SourceRunner 接一个假的查询层：按 outcomes 依次超时（t）、失败（e）或成功（o）。""" + seq = iter(outcomes) + + async def fake(source, sql, *, limits=None): + step = next(seq) + if step == "t": + raise SqlRejected("查询超过 1 秒被中断") + if step == "e": + raise RuntimeError("no such table: orders") + return data_engine.QueryResult(columns=["n"], rows=[[1]], row_count=1, truncated=False, elapsed_ms=1, + sql=sql) + + monkeypatch.setattr(data_engine, "run_query", fake) + source = SimpleNamespace(id="s", name="shop", kind="sqlite", options={}) + return SourceRunner(source, ProfileSettings(enabled=True), SqlDialect("sqlite")) + + +async def _feed(runner: SourceRunner, n: int) -> None: + for _ in range(n): + try: + await runner.run("SELECT 1") + except catalog_profile._Skip: + pass + + +async def test_the_slow_notice_clears_when_queries_recover(monkeypatch): + runner = _runner(monkeypatch, ["t"] * 4 + ["o"] * 3 + ["t"] * 3) + await _feed(runner, 4) + assert runner.notice == "数据库响应慢：最近 4 条查询有 4 条超时。超时再多会自动停止，也可以先停止、换个时间再剖析" + assert runner.stopped is None + await _feed(runner, 3) + assert runner.notice is None, "好转后清掉" + await _feed(runner, 3) + # 最近 10 条里 7 条超时：停下，之后再发直接停 + assert runner.stopped == "slow" and runner.timeouts == 7 and runner.queries == 10 + assert runner.error == "数据库响应慢：最近 10 条查询 7 条超时，已停止，避免给业务库加压" + with pytest.raises(catalog_profile._Stop): + await runner.run("SELECT 1") + + +async def test_a_success_resets_the_failure_count(monkeypatch): + runner = _runner(monkeypatch, ["e", "e", "o", "e", "e", "e"]) + await _feed(runner, 5) + assert runner.stopped is None and runner.errors == 4 + await _feed(runner, 1) + assert runner.stopped == "failed" and runner.error == "连续 3 条查询失败，已停止剖析：no such table: orders" + + +@pytest.mark.parametrize("change, said", [({"enabled": False}, "已停用"), + ({"options": _on(enabled=False)}, "已关闭数据剖析")]) +async def test_disabling_the_source_stops_before_the_next_table(client, make_source, scenic_db, monkeypatch, + change, said): + sid = await make_source(scenic_db, options=_on()) + gate = _hold_queries(monkeypatch, lambda source, sql, **kw: "FROM channels" in sql) + job = (await _start(client, sid, ["channels", "visits", "orders"])).json()["job"] + await asyncio.wait_for(gate.entered.wait(), 10) + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + for key, value in change.items(): + setattr(row, key, value) + await session.commit() + gate.release.set() + done = await _finished(client, sid, job["id"]) + by = {t["table_name"]: t for t in done["tasks"]} + assert by["channels"]["status"] == "ok", "正在做的那张做完" + assert by["visits"]["status"] == by["orders"]["status"] == "cancelled" + assert said in by["visits"]["error"] and by["visits"]["error"] == done["error"] + assert done["status"] == "partial" and done["stop_reason"] is None + + +async def test_masks_follow_the_latest_settings_table_by_table(client, make_source, scenic_db, monkeypatch): + sid = await make_source(scenic_db, options=_on()) + gate = _hold_queries(monkeypatch, lambda source, sql, **kw: "FROM orders" in sql) + job = (await _start(client, sid, ["orders", "channels"])).json()["job"] + await asyncio.wait_for(gate.entered.wait(), 10) + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + row.options = {**_on(), "mask_columns": "channel_type"} + await session.commit() + gate.release.set() + done = await _finished(client, sid, job["id"]) + channels = _table(done, "channels") + assert {(s["target"], s["reason"]) for s in channels["skipped"]} >= {("channel_type", "masked")} + assert not any("channel_type" in q["sql"].lower() for q in channels["query_log"]) + + +# -------------------------------------------------------------------------- +# 范围 +# -------------------------------------------------------------------------- + + +async def test_scopes_needed_all_and_selected(client, make_source, scenic_db, monkeypatch): + sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels", "gates", "suppliers"]) + rows = await _rows(client, sid) + assert rows["v_daily_visits"]["is_view"] and rows["v_daily_visits"]["profile_checks"] == 0 + assert not rows["v_daily_visits"]["profile_due"] + assert rows["suppliers"]["profile_checks"] == 0 and not rows["suppliers"]["profile_due"] + assert (rows["visits"]["profile_due"], rows["visits"]["profile_due_reason"]) == (True, "never") + assert rows["visits"]["last_profile"] is None and rows["visits"]["profile_checks"] == 6 + expected = [name for name, row in rows.items() if row["profile_checks"] > 0 and not row["is_view"]] + + monkeypatch.setattr(catalog_profile_jobs.manager, "start", lambda job_id: None) # 只看挑了哪些表 + everything = (await _start(client, sid, scope="all")).json()["job"] + names = [t["table_name"] for t in everything["tasks"]] + assert everything["scope"] == "all" and sorted(names) == sorted(expected) + # 有待核实关系的表在前（剖析首先是来核实它们的），其余按表结构顺序 + async with SessionLocal() as session: + entries = await catalog.read_catalog(session, sid) + pending = [catalog_profile.has_pending_relations(entries[n].notes if n in entries else {}) for n in names] + assert pending == sorted(pending, reverse=True) and pending[0] is True + await client.post(f"{_url(sid)}/{everything['id']}/cancel") + needed = (await _start(client, sid)).json()["job"] + assert needed["scope"] == "needed" and [t["table_name"] for t in needed["tasks"]] == names, "都没剖析过" + await client.post(f"{_url(sid)}/{needed['id']}/cancel") + + picked = (await _start(client, sid, ["v_daily_visits", "nope", "visits", "VISITS", "suppliers"])).json()["job"] + tasks = {t["table_name"]: t for t in picked["tasks"]} + assert picked["scope"] == "selected" and list(tasks) == ["v_daily_visits", "nope", "visits", "suppliers"] + assert tasks["v_daily_visits"]["status"] == "skipped" and "视图" in tasks["v_daily_visits"]["error"] + assert tasks["nope"]["status"] == "skipped" and "表结构里没有 nope" in tasks["nope"]["error"] + assert tasks["suppliers"]["status"] == "skipped" and tasks["suppliers"]["error"].startswith("没有可剖析的项") + assert tasks["visits"]["status"] == "pending" and tasks["visits"]["checks_total"] == 6 + await client.post(f"{_url(sid)}/{picked['id']}/cancel") + + +async def test_profile_due_never_unfinished_and_new_relations(client, make_source, scenic_db): + sid = await make_source(scenic_db, options=_on(table_max_queries=2), draft=["visits", "gates"]) + await _profile(client, sid, ["visits"]) + rows = await _rows(client, sid) + assert (rows["visits"]["profile_due"], rows["visits"]["profile_due_reason"]) == (True, "unfinished") + assert rows["visits"]["last_profile"]["status"] == "partial" and rows["visits"]["last_profile"]["unfinished"] >= 1 + assert "paths" not in rows["visits"]["last_profile"] + + r = await client.patch(f"/api/datasources/{sid}", json={"options": _on()}) + assert r.status_code == 200, r.text + await _profile(client, sid, ["visits", "gates"]) + rows = await _rows(client, sid) + assert (rows["visits"]["profile_due"], rows["visits"]["profile_due_reason"]) == (False, None) + assert rows["gates"]["last_profile"]["status"] == "ok" and not rows["gates"]["profile_due"] + needed = await _start(client, sid, ["visits"]) + assert needed.status_code == 202, "指定了表照样能剖析" + await catalog_profile_jobs.manager.wait(needed.json()["job"]["id"], timeout=30) + + # 剖析之后目录里冒出了新的待核实关系：这张表又要剖析了 + rel = {"id": catalog.relation_id("gates", ["gate_code"], "channels", ["channel_code"]), "columns": ["gate_code"], + "to_table": "channels", "to_columns": ["channel_code"], "cardinality": "many_to_one", "coverage": None, + "source": "name", "status": "proposed"} + async with SessionLocal() as session: + entry = await catalog.read_entry(session, sid, "gates") + notes = {**entry.notes, "relations": [*entry.notes.get("relations", []), rel]} + await catalog.write_entry(session, sid, "gates", notes, if_version=entry.version, actor=None) + rows = await _rows(client, sid) + assert (rows["gates"]["profile_due"], rows["gates"]["profile_due_reason"]) == (True, "new_relations") + job = (await _start(client, sid)).json()["job"] + assert "gates" in [t["table_name"] for t in job["tasks"]] and "visits" not in [t["table_name"] for t in job["tasks"]] + await catalog_profile_jobs.manager.wait(job["id"], timeout=30) + + +async def test_nothing_to_profile_is_409(client, make_source, tmp_path): + path = _db(tmp_path / "plain.db", "CREATE TABLE suppliers (id INTEGER PRIMARY KEY, name TEXT);", + {"suppliers": [(1, "甲供应商")]}) + sid = await make_source(path, options=_on()) + r = await _start(client, sid) + assert r.status_code == 409 and r.json()["code"] == "profile_nothing" + assert "全部表" in r.json()["detail"] and "指定" in r.json()["detail"] + r = await _start(client, sid, scope="all") + assert r.status_code == 409 and r.json()["code"] == "profile_nothing" and "指定" in r.json()["detail"] + r = await _start(client, sid, [f"t{i}" for i in range(501)]) + assert r.status_code == 422 and "500" in r.json()["detail"] + assert (await client.get(f"{_url(sid)}/latest")).json()["job"] is None + + +async def test_an_empty_table_costs_one_query(client, make_source, tmp_path): + path = _db(tmp_path / "empty.db", "CREATE TABLE orders (id INTEGER PRIMARY KEY, status INTEGER, created_at TEXT);") + sid = await make_source(path, options=_on()) + job = await _profile(client, sid, ["orders"]) + orders = _table(job, "orders") + assert orders["status"] == "skipped" and orders["error"] == "空表，没有可剖析的数据" + assert orders["queries"] == 1 == job["queries"] and orders["query_log"][0]["kind"] == "row_estimate" + assert {c["reason"] for c in orders["checks"]} == {"no_data"} and orders["unfinished"] == 0 + assert job["status"] == "succeeded", "空表不算没做完" + + +# -------------------------------------------------------------------------- +# 留痕、单表接口 +# -------------------------------------------------------------------------- + + +async def test_sampled_keys_never_reach_the_query_log(client, make_source, tmp_path): + """抽样到的键值拼进了 IN 列表，留痕里换成「?」：AgentLab 自己的库里不留这些值。""" + customers = [(f"CUST-{i:04d}", "active" if i % 3 else "closed") for i in range(1, 31)] + orders = [(i, f"CUST-{1 + i % 30:04d}", "paid") for i in range(1, 61)] + path = _db(tmp_path / "shop.db", + "CREATE TABLE customers (code TEXT PRIMARY KEY, status TEXT);" + "CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_code TEXT, status TEXT);", + {"customers": customers, "orders": orders}) + sid = await make_source(path, options=_on()) + detail = await _notes(client, sid, "orders") + rel = {"columns": ["customer_code"], "to_table": "customers", "to_columns": ["code"]} + r = await client.post(f"{_base(sid)}/orders/patch", + json={"changes": [{"path": "relations.new", "value": rel, "reason": "订单的客户编码"}], + "if_version": detail["version"]}) + assert r.status_code == 200, r.text + + job = await _profile(client, sid, ["orders"]) + orders_task = _table(job, "orders") + matched = [q for q in orders_task["query_log"] if " IN (" in q["sql"]] + assert matched and all(re.search(r"IN \((\?(, )?)+\)", q["sql"]) for q in matched), matched + finding = next(f for f in orders_task["findings"] if f["kind"] == "relation") + assert finding["sample"] == 30 and finding["coverage"] == 1.0 + async with SessionLocal() as session: + stored = (await session.execute(select(CatalogProfileTask.query_log, CatalogProfileTask.findings) + .where(CatalogProfileTask.job_id == job["id"]))).all() + assert stored and "CUST-" not in json.dumps([row[0] for row in stored], ensure_ascii=False) + + +async def test_the_table_detail_and_list_carry_the_last_profile(client, make_source, scenic_db, monkeypatch): + sid = await make_source(scenic_db, options=_on(), draft=["visits"]) + gate = _hold_queries(monkeypatch, _at_visits_codes) + job = (await _start(client, sid, ["visits"])).json()["job"] + await asyncio.wait_for(gate.entered.wait(), 10) + detail = await _notes(client, sid, "visits") + assert detail["last_profile"]["status"] == "running" and detail["last_profile"]["job_id"] == job["id"] + assert (await _rows(client, sid))["visits"]["last_profile"]["status"] == "running" + assert not (await _rows(client, sid))["visits"]["profile_due"], "正在剖析的不算要剖析" + gate.release.set() + await catalog_profile_jobs.manager.wait(job["id"], timeout=30) + + detail = await _notes(client, sid, "visits") + last = detail["last_profile"] + assert last["status"] == "ok" and last["created_by"] == "周宁" and last["job_status"] == "succeeded" + assert last["query_log"] and last["findings"] and last["seq"] == 0 + row = (await _rows(client, sid))["visits"]["last_profile"] + assert set(row) == {"job_id", "seq", "status", "started_at", "finished_at", "summary", "skipped_count", + "unfinished", "error", "row_estimate"} + assert row["summary"] == last["summary"] and row["row_estimate"] == {"rows": 1500, "method": "count", + "at_least": None} + assert (await _notes(client, sid, "gates"))["last_profile"] is None + + +async def test_the_last_profile_is_the_last_one_that_actually_ran(client, make_source, scenic_db, monkeypatch): + """channels 剖析完（ok）；再建 [visits, channels]、停在 visits 上：channels 没开始就停了。表详情、表清单的「上次剖析」 + 仍是上一次真正剖析的那一项（行数、发过的查询都在），不算没做完；保留不删它所在的任务，之后再建一个排队的任务也不删。""" + sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"]) + first = await _profile(client, sid, ["channels"]) + gate = _hold_queries(monkeypatch, _at_visits_codes) + second = (await _start(client, sid, ["visits", "channels"])).json()["job"] + await asyncio.wait_for(gate.entered.wait(), 10) + stopped = (await client.post(f"{_url(sid)}/{second['id']}/cancel")).json()["job"] + assert [(t["table_name"], t["status"]) for t in stopped["tasks"]] == [("visits", "partial"), + ("channels", "cancelled")] + + last = (await _notes(client, sid, "channels"))["last_profile"] + assert (last["job_id"], last["status"]) == (first["id"], "ok") + assert last["started_at"] and last["queries"] and last["row_estimate"] and last["query_log"] + rows = await _rows(client, sid) + assert rows["channels"]["last_profile"]["job_id"] == first["id"] and not rows["channels"]["profile_due"] + assert rows["visits"]["last_profile"]["job_id"] == second["id"], "visits 真正剖析过：上次剖析是停下的这一次" + async with SessionLocal() as session: + assert await catalog_profile_jobs.prune(session, sid, keep=1) == 0, "first 是 channels 上次剖析所在的任务" + + # 结束了的任务超出保留数时再建一个排队的任务（建任务时顺手清）：排着的那一项不算 channels 的上次剖析 + monkeypatch.setattr(catalog_profile_jobs, "KEEP_JOBS", 0) + monkeypatch.setattr(catalog_profile_jobs.manager, "start", lambda job_id: None) + queued = (await _start(client, sid, ["channels"])).json()["job"] + async with SessionLocal() as session: + left = set((await session.execute(select(CatalogProfileJob.id) + .where(CatalogProfileJob.source_id == sid))).scalars()) + assert {first["id"], second["id"], queued["id"]} == left + assert (await _notes(client, sid, "channels"))["last_profile"]["job_id"] == first["id"] + row = (await _rows(client, sid))["channels"] + assert row["last_profile"]["job_id"] == first["id"] and not row["profile_due"], "已经排进任务里了" + await client.post(f"{_url(sid)}/{queued['id']}/cancel") + + +# -------------------------------------------------------------------------- +# 设置迁移、保留 +# -------------------------------------------------------------------------- + + +async def test_old_whole_run_budgets_are_removed_on_startup_and_on_save(client, make_source, scenic_db): + sid = await make_source(scenic_db, options={PROFILE_OPTION: {"enabled": True, "max_queries": 60, + "max_total_s": 120, "sample_size": 500}, + "query_timeout_s": "20"}) + assert await catalog_profile_jobs.startup() >= 0 + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + assert row.options == {PROFILE_OPTION: {"enabled": True, "sample_size": 500}, "query_timeout_s": "20"} + # 老页面还带着旧键：只摘掉，不报错 + r = await client.patch(f"/api/datasources/{sid}", + json={"options": _on(max_queries=30, max_total_s=90, table_max_queries=25)}) + assert r.status_code == 200, r.text + assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "table_max_queries": 25} + + +async def test_retention_keeps_recent_jobs_and_each_tables_last_profile(client, make_source, scenic_db, monkeypatch): + sid = await make_source(scenic_db, options=_on()) + start = utcnow() - timedelta(hours=1) + ids: list[str] = [] + async with SessionLocal() as session: + for i in range(5): + job = CatalogProfileJob(id=uuid.uuid4().hex, source_id=sid, status="succeeded", scope="selected", + settings={"enabled": True}, created_at=start + timedelta(minutes=i)) + session.add(job) + await session.flush() + session.add(CatalogProfileTask(job_id=job.id, seq=0, table_name="visits", status="ok")) + if i == 0: + session.add(CatalogProfileTask(job_id=job.id, seq=1, table_name="gates", status="ok")) + ids.append(job.id) + await session.commit() + async with SessionLocal() as session: + assert await catalog_profile_jobs.prune(session, sid, keep=2) == 2 + left = set((await session.execute(select(CatalogProfileJob.id) + .where(CatalogProfileJob.source_id == sid))).scalars()) + assert left == {ids[4], ids[3], ids[0]}, "最早那个是 gates 最近一次剖析所在的任务，不删" + assert (await _notes(client, sid, "gates"))["last_profile"]["job_id"] == ids[0] + + # 建任务时顺手清 + monkeypatch.setattr(catalog_profile_jobs, "KEEP_JOBS", 1) + job = await _profile(client, sid, ["visits"]) + async with SessionLocal() as session: + left = set((await session.execute(select(CatalogProfileJob.id) + .where(CatalogProfileJob.source_id == sid))).scalars()) + assert ids[0] in left and job["id"] in left and ids[3] not in left diff --git a/backend/tests/test_catalog_profile_units.py b/backend/tests/test_catalog_profile_units.py index d41ecf2d..4fa1472f 100644 --- a/backend/tests/test_catalog_profile_units.py +++ b/backend/tests/test_catalog_profile_units.py @@ -20,26 +20,30 @@ AT = "2026-10-03T08:00:00+00:00" def test_settings_default_off_and_parse_leniently(): s = profile_settings({}) assert s == ProfileSettings() - assert s.enabled is False and s.max_queries == 60 and s.query_timeout_s == 10 and s.sample_size == 2000 - assert s.max_scan_rows == 100_000 and s.max_total_s == 120 - on = profile_settings({PROFILE_OPTION: {"enabled": True, "max_queries": "30", "sample_size": 500}}) - assert on.enabled is True and on.max_queries == 30 and on.sample_size == 500 - # 库里存着的值不对（老数据、手改过）：那一项用缺省，不因此开启，也不报错 - odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "max_queries": 0, "query_timeout_s": "x"}}) - assert odd.enabled is False and odd.max_queries == 60 and odd.query_timeout_s == 10 + assert s.enabled is False and s.table_max_queries == 40 and s.table_max_s == 60 and s.query_timeout_s == 10 + assert s.sample_size == 2000 and s.max_scan_rows == 100_000 + on = profile_settings({PROFILE_OPTION: {"enabled": True, "table_max_queries": "30", "sample_size": 500}}) + assert on.enabled is True and on.table_max_queries == 30 and on.sample_size == 500 + # 库里存着的值不对（老数据、手改过）：那一项用缺省，不因此开启，也不报错；迁移前的旧键不看 + odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "table_max_queries": 0, "query_timeout_s": "x", + "max_queries": 5}}) + assert odd.enabled is False and odd.table_max_queries == 40 and odd.query_timeout_s == 10 assert profile_settings({PROFILE_OPTION: "on"}) == ProfileSettings() + # 任务里存着的快照读回来是同一份设置 + assert ProfileSettings.from_dict(on.to_dict()) == on and ProfileSettings.from_dict(None) == ProfileSettings() @pytest.mark.parametrize("value, label", [ - ({"max_queries": 0}, "查询次数上限"), - ({"max_queries": 501}, "查询次数上限"), - ({"max_queries": 2.5}, "查询次数上限"), + ({"table_max_queries": 0}, "每张表最多查询数"), + ({"table_max_queries": 201}, "每张表最多查询数"), + ({"table_max_queries": 2.5}, "每张表最多查询数"), + ({"table_max_s": 9}, "每张表时长上限"), + ({"table_max_s": 601}, "每张表时长上限"), ({"query_timeout_s": 61}, "单条查询时限"), ({"sample_size": 5}, "抽样键值数"), ({"max_scan_rows": -1}, "整表统计行数上限"), - ({"max_total_s": 5}, "总时长上限"), ({"enabled": "true"}, "开启数据剖析"), - ({"max_queries": True}, "查询次数上限"), + ({"table_max_queries": True}, "每张表最多查询数"), ]) def test_settings_problem_names_the_field_in_chinese(value, label): problem = profile_settings_problem({PROFILE_OPTION: value}) @@ -51,9 +55,21 @@ def test_settings_problem_rejects_unknown_keys_and_wrong_shape(): assert profile_settings_problem({PROFILE_OPTION: ["enabled"]}) assert profile_settings_problem({}) is None assert profile_settings_problem({PROFILE_OPTION: None}) is None - assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "max_queries": 500, "query_timeout_s": 1.5, - "sample_size": 10, "max_scan_rows": 0, - "max_total_s": 600}}) is None + assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "table_max_queries": 200, "table_max_s": 10.5, + "query_timeout_s": 1.5, "sample_size": 10, + "max_scan_rows": 0}}) is None + + +def test_legacy_whole_run_budgets_are_stripped_not_rejected(): + """以前按整次剖析算的两项（查询次数上限、总时长上限）：保存时只摘掉，不报错；不折算成每张表的预算。""" + old = {"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True, "max_queries": 60, "max_total_s": 120, + "sample_size": 500}} + cleaned = catalog_profile.without_legacy(old) + assert cleaned == {"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True, "sample_size": 500}} + assert old[PROFILE_OPTION]["max_queries"] == 60, "不改调用方的对象" + assert profile_settings_problem(cleaned) is None + plain = {PROFILE_OPTION: {"enabled": True}} + assert catalog_profile.without_legacy(plain) is plain and catalog_profile.without_legacy(None) is None def test_profile_option_never_reaches_the_connection_url(): diff --git a/backend/tests/test_catalog_semantics.py b/backend/tests/test_catalog_semantics.py index d97a8c1d..25eb9aad 100644 --- a/backend/tests/test_catalog_semantics.py +++ b/backend/tests/test_catalog_semantics.py @@ -24,7 +24,7 @@ from app.db.models import DataSource from tests.fixtures.catalog import scenic, scenic_notes from tests.fixtures.catalog.fake_model import FakeModel from tests.test_catalog_patch import _schema -from tests.test_catalog_profile import _notes, _on, _relation, _table, _url, client, make_source # noqa: F401 +from tests.test_catalog_profile import _notes, _on, _profile, _relation, _table, client, make_source # noqa: F401 AT = "2026-10-04T08:00:00+00:00" DAY1, DAY2, DAY3 = "2026-10-01T08:00:00+00:00", "2026-10-02T08:00:00+00:00", "2026-10-03T08:00:00+00:00" @@ -131,8 +131,7 @@ async def test_confirming_a_profiled_relation_by_patch_keeps_the_fanout_check(cl """审查探针 A1：剖析得出多对一、覆盖率 1.0，经对话提案确认这条关系之后，扇出检查照样报 error。""" sql = "SELECT COUNT(g.id) AS n FROM gates g JOIN visits v ON v.gate_id = g.id" sid = await make_source(scenic_db, options=_on(), draft=["visits", "gates"]) - r = await client.post(_url(sid), json={"tables": ["visits"]}) - assert r.status_code == 200, r.text + await _profile(client, sid, ["visits"]) detail = await _notes(client, sid, "visits") rel = _relation(detail["notes"], "gate_id") assert (rel["source"], rel["status"], rel["cardinality"], rel["coverage"]) == ( @@ -156,8 +155,7 @@ async def test_profiled_relation_without_child_check_only_warns(client, make_sou 核实过：扇出检查只给 warning（「可能重复计算」），不给 error、不让出具降档。""" sql = "SELECT COUNT(g.id) AS n FROM gates g JOIN visits v ON v.gate_id = g.id" sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits", "gates"]) - r = await client.post(_url(sid), json={"tables": ["visits"]}) - assert r.status_code == 200, r.text + await _profile(client, sid, ["visits"]) rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id") assert (rel["source"], rel["status"], rel["cardinality"]) == ("profile", "verified", "many_to_one") assert not rel.get("cardinality_checked") @@ -359,15 +357,13 @@ def test_codes_complete_flag_is_validated_and_counts_as_an_edit(): async def test_profile_marks_codes_complete_only_when_the_whole_table_was_read(client, make_source, scenic_db): whole = await make_source(scenic_db, options=_on(), draft=["visits"]) - r = await client.post(_url(whole), json={"tables": ["visits"]}) - assert r.status_code == 200, r.text + await _profile(client, whole, ["visits"]) codes = (await _notes(client, whole, "visits"))["notes"]["columns"]["status"]["codes"] assert "统计全表" in codes["note"] and codes.get("complete") is True # 不做整表统计（只看前若干行）：取值可能没看全，不标 capped = await make_source(scenic_db, options=_on(max_scan_rows=0, sample_size=10), draft=["visits"]) - r = await client.post(_url(capped), json={"tables": ["visits"]}) - assert r.status_code == 200, r.text + await _profile(client, capped, ["visits"]) codes = (await _notes(client, capped, "visits"))["notes"]["columns"]["status"]["codes"] assert "按前 10 行统计" in codes["note"] and "complete" not in codes @@ -388,10 +384,10 @@ async def test_rerun_on_another_day_with_same_data_keeps_the_version(client, mak assert r.status_code == 200, r.text monkeypatch.setattr(catalog, "now_iso", lambda: DAY1) - first = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits") + first = _table(await _profile(client, sid, ["visits"]), "visits") notes_day1 = (await _notes(client, sid, "visits"))["notes"] monkeypatch.setattr(catalog, "now_iso", lambda: DAY2) - second = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits") + second = _table(await _profile(client, sid, ["visits"]), "visits") assert (second["added"], second["updated"], second["removed"]) == (0, 0, 0) assert second["version"] == first["version"] assert (await _notes(client, sid, "visits"))["notes"] == notes_day1 # 备注里还是第一天的日期 @@ -404,7 +400,7 @@ async def test_rerun_on_another_day_with_same_data_keeps_the_version(client, mak finally: db.close() monkeypatch.setattr(catalog, "now_iso", lambda: DAY3) - third = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits") + third = _table(await _profile(client, sid, ["visits"]), "visits") assert third["version"] == second["version"] + 1 and third["updated"] >= 1 codes = (await _notes(client, sid, "visits"))["notes"]["columns"]["status"]["codes"] assert set(codes["value"]) == {"0", "1", "2"} and codes["updated_at"] == DAY3