From 6b3c737fb89adf94406b03e8b1f4fabde3f45cf1 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Thu, 15 Oct 2026 18:30:48 +0800 Subject: [PATCH] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=89=96=E6=9E=90=E6=94=B9?= =?UTF-8?q?=E6=88=90=E9=80=90=E8=A1=A8=E7=9A=84=E5=90=8E=E5=8F=B0=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=EF=BC=9A=E5=BC=95=E6=93=8E=E6=94=B9=E7=BC=96=E6=8E=92?= =?UTF-8?q?=E3=80=81=E5=89=96=E6=9E=90=E4=BB=BB=E5=8A=A1=E3=80=81=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E3=80=81=E8=AE=BE=E7=BD=AE=E6=8D=A2=E6=96=B0=E9=94=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 以前剖析是一次同步长请求，预算按整次算（max_queries 60、max_total_s 120），选 50 张表只做得完前两三张，其余整张记「查询次数用完」；不能停，关掉页面报告就没了，对业务库查了什么无从查。 - data/catalog_profile.py：检查逻辑和 _apply 并入规则原样保留，只换编排。公开的逐表入口 profile_table：先列全检查项（plan_checks，纯函数，范围的 profile_checks、建任务时列的检查项、执行用同一份；遮罩、多列、不在表结构里、指向视图的关系在这里就列成按规则跳过），第一条查询估行数（确知空表整张表跳过），逐项执行、进度回调，整张表结束写目录并和回调在同一次库操作里；停止时做完的项照常写、没做的记 cancelled 保留旧结论。_Runner 拆成整个任务一个的 SourceRunner（只读守卫、单条时限、任务级止损：连续 3 条失败、最近 10 条 7 条超时；最近 6 条 4 条超时提醒；每条查询留痕，SQL 经 normalize_sql 去掉字面量）和每张表一个的 TableBudget（用完只停这张表）。删掉同步的 profile_catalog、挑表前批量估行数（_skip_empty、PROFILE_DEFAULT_TABLES、PROFILE_PROBE_TABLES）和进程内 _RUNNING。设置换成 table_max_queries（1–200，缺省 40）、table_max_s（10–600，缺省 60），旧键 max_queries、max_total_s 启动时从库里删掉（migrate_settings，不折算：旧值按整次算，含义不同），保存时只摘掉不报错（without_legacy）。表助手的 run_query 和起草时核对有效记录条件改走单条查询入口 run_single_query，行为不变。 - data/catalog_profile_jobs.py：剖析任务的建（selected / needed / all 三种范围，排序同前：有待核实关系的在前、使用次数、表结构顺序；挑出 0 张 ProfileNothing）、排队（和同源起草按建立先后串行，waiting_for）、执行（每张表开始前重读数据源：停用或关掉剖析就停下；遮罩按最新设置）、停止、重试、启动标中断不续跑、保留最近 50 个结束的任务（某张表最近一次剖析所在的不删）、last_profile / profile_due 的查询、任务和逐表的接口形状。 - api/catalog.py：加 /profile-jobs（建、active、latest、{job_id}、tasks/{seq}、cancel、retry），登记在 /{table} 前面；删掉 POST /catalog/profile；清单每行加 last_profile、profile_due、profile_due_reason、profile_checks，顶层加 active_profile_job；单表加 last_profile；导入目录、同步起草遇到剖析任务 409 profile_busy。api/coded.py 加 profile_nothing、catalog_profile_finished；api/datasources.py 保存时摘掉旧键；main.py 启动时标中断加设置迁移，关停时停掉进行中的剖析。 - 测试：test_catalog_profile.py、test_catalog_profile_units.py、test_catalog_semantics.py 里打同步接口和旧设置键的改走任务和新键，检查逻辑的断言保留；同步接口的整次总量上限一条改成逐表次数、逐表时长两条。 --- backend/app/api/catalog.py | 259 ++++-- backend/app/api/coded.py | 8 +- backend/app/api/datasources.py | 9 +- backend/app/data/catalog_assistant.py | 13 +- backend/app/data/catalog_profile.py | 969 ++++++++++++++------ backend/app/data/catalog_profile_jobs.py | 814 +++++++++++++++- backend/app/main.py | 13 +- backend/tests/test_catalog_profile.py | 335 +++---- backend/tests/test_catalog_profile_units.py | 46 +- backend/tests/test_catalog_semantics.py | 20 +- 10 files changed, 1895 insertions(+), 591 deletions(-) diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 7a306d48..41b84944 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -8,12 +8,17 @@ - POST /catalog/{table}/patch 保存目录修改提案（乐观锁），记为人工确认 - GET /catalog/{table}/impact 影响面：引用这张表的已发布、受管模板（data/catalog_impact.py） - POST /catalog/draft 同步起草（注释、外键、命名推断，可选模型）。界面改用下面的后台起草任务，这个留着； - 和后台任务双向互斥：同一个源有排队或进行中的后台任务（或正在同步起草）时 409 - catalog_draft_busy，同步起草期间建任务、重试、导入也是 409 + 同一个源有排队或进行中的起草任务（或正在同步起草）时 409 catalog_draft_busy，有剖析任务时 + 409 profile_busy；同步起草期间建起草任务、重试、导入也是 409，新建的剖析任务排在它后面 - PUT /catalog/guidance 数据源级的起草要求（使用者补充的背景和要求，起草时交给模型） - POST /catalog/draft-jobs 建后台起草任务（202）；GET /draft-jobs/active、/latest、/{job_id} 读任务； - POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py） -- POST /catalog/profile 同步剖析：对业务库发少量只读查询，核对关系、取码值候选（data/catalog_profile.py） + POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py）。 + 同源有剖析任务时照建，排在它后面（waiting_for） +- POST /catalog/profile-jobs 建后台剖析任务（202）：对业务库逐表发少量只读查询，核对关系、取码值候选、提议业务日期 + （data/catalog_profile_jobs.py，引擎在 data/catalog_profile.py）；GET /profile-jobs/active、 + /latest、/{job_id} 读任务，/{job_id}/tasks/{seq} 读一张表的明细（检查项、发现、发过的查询）； + POST /profile-jobs/{job_id}/cancel 停止，/retry 只重试没做完的表。同源有起草任务时照建， + 排在它后面（waiting_for） - GET /catalog/review-queue 审阅队列：推断的项按使用次数、风险、来源排好；POST /catalog/review/batch 批量审阅， GET /catalog/review-progress 审阅进度，GET /catalog/{table}/usage 用过这张表的运行 （data/catalog_review.py） @@ -23,8 +28,9 @@ （data/catalog_assistant.py） 规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径 -不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。 -起草任务的 409 带机读码（api/coded.py）：同一个源已有排队或进行中的任务、起草模型用不了、要停止的任务已经结束。 +不对 422，源或表不存在 404。所有写操作记下署名（api/deps.actor_of：开了登录是认证过的用户名，否则请求头 X-Actor 的自报）。 +起草、剖析任务的 409 带机读码（api/coded.py）：同一个源已有排队或进行中的同种任务、起草模型用不了、剖析没开启、按范围 +挑不出表、要停止的任务已经结束。 上传表格的源按当前快照的表结构回答（table_versions.resolve_source），和查询实际用的库是同一个版本。 """ @@ -38,10 +44,10 @@ from pydantic import BaseModel, Field, ValidationError from sqlalchemy.ext.asyncio import AsyncSession from app.api.coded import ( - CATALOG_DRAFT_BUSY, CATALOG_DRAFT_FINISHED, CATALOG_DRAFT_MODEL_UNAVAILABLE, PROFILE_BUSY, SCHEMA_MISSING, - SNAPSHOT_TAMPERED, CodedHTTPException, + CATALOG_DRAFT_BUSY, CATALOG_DRAFT_FINISHED, CATALOG_DRAFT_MODEL_UNAVAILABLE, CATALOG_PROFILE_FINISHED, PROFILE_BUSY, + PROFILE_NOTHING, SCHEMA_MISSING, SNAPSHOT_TAMPERED, CodedHTTPException, ) -from app.api.runs import actor_of +from app.api.deps import actor_of from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, catalog_profile_jobs, introspect from app.data import table_versions from app.data import catalog_assistant, catalog_review, catalog_transfer @@ -103,9 +109,11 @@ GUIDANCE_MISSING = "请求里要写起草要求（guidance 一项，一段文字 GUIDANCE_NOT_TEXT = "起草要求要是一段文字（要清除起草要求请传空串）" -class CatalogProfileIn(BaseModel): - #: 要剖析的表；不给或为空时按使用次数取前 10 张目录里有待核实关系的表 - tables: list[str] | None = Field(default=None, max_length=200) +class ProfileJobIn(BaseModel): + #: 要剖析的表（范围 selected，按给的顺序去重）；最多 500 张（超了 422，说明是中文） + tables: list[str] | None = None + #: 不指定表时的范围：needed 需要剖析的表（缺省）、all 全部表（都不含视图、没有可剖析的项的表） + scope: Literal["needed", "all"] | None = None async def _resolved(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]: @@ -135,7 +143,8 @@ def _item_value(notes: dict[str, Any], name: str) -> tuple[Any, str | None]: def _summary_row(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogEntry | None, - usage: int, last_draft: dict[str, Any] | None = None) -> dict[str, Any]: + usage: int, last_draft: dict[str, Any] | None = None, + profiling: dict[str, Any] | None = None) -> dict[str, Any]: notes = entry.notes if entry else {} label, label_status = _item_value(notes, "label") kind, _ = _item_value(notes, "kind") @@ -158,9 +167,20 @@ def _summary_row(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogE # 这张表最近一次起草任务里的那一项：{status, job_id, finished_at, error, model_error, blank_columns}，没起草过 # 是 null "last_draft": last_draft, + # 剖析：最近一次剖析（CatalogLastProfile，没剖析过是 null）、要不要剖析和为什么、按当前目录和表结构要做几项检查 + **(profiling or {"last_profile": None, "profile_due": False, "profile_due_reason": None, "profile_checks": 0}), } +def _profiling(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogEntry | None, *, + tables: dict[str, Any], masked: set[str], last: dict[str, Any] | None) -> dict[str, Any]: + notes = entry.notes if entry else {} + checks = catalog_profile.profile_checks(name, meta, notes, tables=tables, masked=masked) + reason = catalog_profile_jobs.due_reason(meta, notes, last, checks) + return {"last_profile": catalog_profile_jobs.public_last(last), "profile_due": reason is not None, + "profile_due_reason": reason, "profile_checks": checks} + + def _structure(meta: dict[str, Any]) -> dict[str, Any]: return { "qualified": meta.get("qualified"), @@ -189,6 +209,9 @@ async def _detail(session: AsyncSession, source: Any, table: str, entry: catalog "structure": _structure(meta) if meta is not None else None, "system_notes": catalog.system_notes_source(source), "usage": usage, + # 这张表最近一次剖析（CatalogProfileTaskDetail：检查项、发现、跳过的项、日期范围、发过的查询）；任务还在进行就是 + # 进行中的状态；没剖析过是 null + "last_profile": await catalog_profile_jobs.last_detail(session, source.id, table), } @@ -224,11 +247,17 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi entries = await catalog.read_catalog(session, source.id) usage = await catalog.table_usage(session, source) drafts = await catalog_jobs.last_drafts(session, source.id) + lasts = await catalog_profile_jobs.last_profiles(session, source.id) + masked = catalog_profile.masked_of(row.options) + + def profiling(name: str, meta: dict[str, Any] | None) -> dict[str, Any]: + return _profiling(name, meta, entries.get(name), tables=tables, masked=masked, last=lasts.get(name)) + order = {name: i for i, name in enumerate(tables)} - rows = [_summary_row(name, meta, entries.get(name), usage.get(name, 0), drafts.get(name)) + rows = [_summary_row(name, meta, entries.get(name), usage.get(name, 0), drafts.get(name), profiling(name, meta)) for name, meta in tables.items()] rows.sort(key=lambda r: (-r["usage"], order[r["table_name"]])) - rows += [_summary_row(name, None, entry, 0, drafts.get(name)) + rows += [_summary_row(name, None, entry, 0, drafts.get(name), profiling(name, None)) for name, entry in sorted(entries.items()) if name not in tables] return { "tables": rows, @@ -239,8 +268,11 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi "schema_total": max(_int(cache.get("total")), len(tables)), # 数据源级的起草要求（没有是 null） "guidance": catalog_jobs.guidance_of(row), - # 排队或进行中的起草任务 {id, status, total, done}，界面据此显示进度；没有是 null + # 排队或进行中的起草任务 {id, status, total, done, waiting_for}，界面据此显示进度；没有是 null "active_job": await catalog_jobs.job_brief(session, await catalog_jobs.active_job(session, source.id)), + # 排队或进行中的剖析任务 {id, status, total, done, waiting_for}；没有是 null + "active_profile_job": await catalog_profile_jobs.job_brief( + session, await catalog_profile_jobs.active_job(session, source.id)), } @@ -331,6 +363,136 @@ async def import_catalog(source_id: str, payload: CatalogImportIn, x_actor: str return report.as_dict() +# ---- 后台剖析任务（同样登记在 GET /catalog/{table} 前面：active、latest 要先于 /profile-jobs/{job_id}） + + +async def _profile_source(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]: + """建剖析任务、重试之前的几道门，409 都带机读码（界面按它分情况给下一步，不从原话里认）：未开启 profile_disabled、 + 数据源停用 datasource_inactive、快照登记被改动 snapshot_tampered、没有表结构 schema_missing。返回 (数据源记录, + 读表结构用的源)。""" + row = await _source_row(session, source_id) + try: + catalog_profile.ensure_enabled(row) + except catalog_profile.ProfileDisabled as e: + raise CodedHTTPException(409, str(e), e.code) from e + try: + source = await table_versions.resolve_source(session, row) + except SnapshotTampered as e: # 快照清单对不上（SnapshotManifestMismatch 同时是 SnapshotMissing） + raise CodedHTTPException(409, str(e), SNAPSHOT_TAMPERED) from e + except table_versions.SnapshotMissing as e: + raise HTTPException(409, str(e)) from e + cache = source.schema_cache or {} + if not cache.get("tables"): + raise CodedHTTPException(409, f"{introspect.why_empty(cache)}，无法进行数据剖析。请先在数据源卡片上点「探查结构」", + SCHEMA_MISSING) + return row, source + + +async def _start_profile(session: AsyncSession, row: DataSource, source: Any, *, tables: list[str] | None, + scope: str, actor: str | None, retry_of: str | None = None) -> dict[str, Any]: + try: + job, tasks = await catalog_profile_jobs.create_job( + session, row, source, tables=tables, scope=scope, settings=catalog_profile.ensure_enabled(row), + actor=actor, retry_of=retry_of) + except catalog_profile_jobs.ProfileBusy as e: + raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e + except catalog_profile_jobs.ProfileNothing as e: + raise CodedHTTPException(409, str(e), PROFILE_NOTHING) from e + catalog_profile_jobs.manager.start(job.id) + return {"job": catalog_profile_jobs.job_dict(job, tasks, + waiting_for=await catalog_profile_jobs.waiting_for(session, job))} + + +@router.post("/{source_id}/catalog/profile-jobs", status_code=202) +async def create_profile_job(source_id: str, payload: ProfileJobIn, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """建后台剖析任务，马上返回（202，任务是排队中）。tables 非空时范围是 selected（表结构里没有、视图、没有可剖析的项照样 + 建进任务、状态 skipped），否则按 scope（缺省 needed）挑。同源有排队或进行中的剖析任务 409 profile_busy（说清进行到第几张； + 界面改去 GET /profile-jobs/active 显示那一个），按范围挑出 0 张表 409 profile_nothing；同源有起草任务时照建，排在它后面 + （waiting_for = "draft"），起草结束后自动开始。""" + if payload.tables and len(payload.tables) > catalog_profile_jobs.MAX_TABLES: + raise HTTPException(422, f"一次最多剖析 {catalog_profile_jobs.MAX_TABLES} 张表，当前 {len(payload.tables)} 张，" + "请分几次剖析") + row, source = await _profile_source(session, source_id) + tables = [str(t) for t in payload.tables] if payload.tables else None + scope = "selected" if tables else (payload.scope or "needed") + return await _start_profile(session, row, source, tables=tables, scope=scope, actor=actor_of(x_actor)) + + +@router.get("/{source_id}/catalog/profile-jobs/active") +async def active_profile_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """排队或进行中的那个剖析任务，没有是 null。""" + row = await _source_row(session, source_id) + return {"job": await catalog_profile_jobs.job_out(session, await catalog_profile_jobs.active_job(session, row.id))} + + +@router.get("/{source_id}/catalog/profile-jobs/latest") +async def latest_profile_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """最近建的一个剖析任务（任何状态），没有是 null。""" + row = await _source_row(session, source_id) + return {"job": await catalog_profile_jobs.job_out(session, await catalog_profile_jobs.latest_job(session, row.id))} + + +async def _profile_job_or_404(session: AsyncSession, source_id: str, job_id: str) -> tuple[DataSource, Any]: + row = await _source_row(session, source_id) + job = await catalog_profile_jobs.load(session, row.id, job_id) + if job is None: + raise HTTPException(404, "剖析任务不存在，可能属于别的数据源，已随数据源删除，或者超过保留数被清理了") + return row, job + + +@router.get("/{source_id}/catalog/profile-jobs/{job_id}") +async def get_profile_job(source_id: str, job_id: str, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + _, job = await _profile_job_or_404(session, source_id, job_id) + return {"job": await catalog_profile_jobs.job_out(session, job)} + + +@router.get("/{source_id}/catalog/profile-jobs/{job_id}/tasks/{seq}") +async def get_profile_task(source_id: str, job_id: str, seq: int, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """剖析任务里的一张表（按剖析顺序的序号）：检查项、发现、跳过的项、日期范围、发过的查询（字面量已换成「?」）。""" + _, job = await _profile_job_or_404(session, source_id, job_id) + task = await catalog_profile_jobs.load_task(session, job, seq) + if task is None: + raise HTTPException(404, "这次剖析里没有这一张表") + return {"task": catalog_profile_jobs.task_detail(job, task)} + + +@router.post("/{source_id}/catalog/profile-jobs/{job_id}/cancel") +async def cancel_profile_job(source_id: str, job_id: str, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """停止：排队的直接停止；进行中的不再等正在跑的那条查询（数据库按单条时限截止它），这张表做完的项照常写进目录、没做的 + 保留旧结论，后面的表记停止。已经结束的 409 catalog_profile_finished。返回停止之后的任务。""" + row, job = await _profile_job_or_404(session, source_id, job_id) + owner = row.id + try: + await catalog_profile_jobs.cancel(session, job) + except catalog_profile_jobs.ProfileFinished as e: + raise CodedHTTPException(409, str(e), CATALOG_PROFILE_FINISHED) from e + session.expire_all() # 后台在别的会话里收的尾：重新读 + return {"job": await catalog_profile_jobs.job_out(session, await catalog_profile_jobs.load(session, owner, job_id))} + + +@router.post("/{source_id}/catalog/profile-jobs/{job_id}/retry", status_code=202) +async def retry_profile_job(source_id: str, job_id: str, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """重试：新任务，只含原任务里 failed、partial、cancelled 的表（按原来的顺序），范围 selected，设置取数据源当前的。 + 同源有排队或进行中的剖析任务 409 profile_busy；没有可重试的表 422；开始不了的几种情况同建任务。""" + _, old = await _profile_job_or_404(session, source_id, job_id) + row, source = await _profile_source(session, source_id) + try: + await catalog_profile_jobs.ensure_free(session, row) + except catalog_profile_jobs.ProfileBusy as e: + raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e + try: + tables = await catalog_profile_jobs.retry_tables(session, old) + except catalog_profile_jobs.NothingToRetry as e: + raise HTTPException(422, str(e)) from e + return await _start_profile(session, row, source, tables=tables, scope="selected", actor=actor_of(x_actor), + retry_of=old.id) + + @router.get("/{source_id}/catalog/{table}") async def get_catalog_table(source_id: str, table: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: @@ -545,71 +707,6 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn, } -def _profile_table_out(t: catalog_profile.TableProfile) -> dict[str, Any]: - return { - "table_name": t.table, - "queries": t.queries, - "row_estimate": t.row_estimate.to_dict() if t.row_estimate else None, - "findings": t.findings, - "skipped": [s.to_dict() for s in t.skipped], - "date_ranges": t.date_ranges, - "added": t.added, - "updated": t.updated, - "removed": t.removed, - "version": t.version, - "error": t.error, - } - - -@router.post("/{source_id}/catalog/profile") -async def profile_catalog_tables(source_id: str, payload: CatalogProfileIn, - x_actor: str | None = Header(default=None), - session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - """同步剖析。返回每张表用了几条查询、写进目录的发现、跳过了什么及原因。 - - 默认关闭：数据源设置里没开启数据剖析、数据源停用，409；同一个数据源已经在剖析，409。 - """ - row, source = await _resolved(session, source_id) - # 409 都带机读码（api/coded.py），界面按它分五种情况给下一步，不从原话里认 - try: - catalog_profile.ensure_enabled(row) - except catalog_profile.ProfileDisabled as e: - raise CodedHTTPException(409, str(e), e.code) from e - cache = source.schema_cache or {} - if not cache.get("tables"): - raise CodedHTTPException(409, f"{introspect.why_empty(cache)}，无法进行数据剖析。请先在数据源卡片上点「探查结构」", - SCHEMA_MISSING) - actor = actor_of(x_actor) - try: - report = await catalog_profile.profile_catalog(session, row, source, tables=payload.tables or None, - actor=actor) - except catalog_profile.ProfileDisabled as e: - raise CodedHTTPException(409, str(e), e.code) from e - except catalog_profile.ProfileBusy as e: - raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e - except SnapshotTampered as e: - raise CodedHTTPException(409, str(e), SNAPSHOT_TAMPERED) from e - rows = [_profile_table_out(t) for t in report.tables] - note = None - if not rows and report.empty_tables: - note = (f"有待核对关联关系的 {len(report.empty_tables)} 张表都是空表，没有可以核实的数据。" - "请在数据有了之后再剖析，或指定要剖析的表") - elif not rows: - note = "目录里还没有待核实的关联关系，请先起草数据目录，或指定要剖析的表" - return { - "profiled_at": report.profiled_at, - "actor": actor, - "settings": report.settings.to_dict(), - "queries_used": report.queries_used, - "stopped": report.stopped, - "tables": rows, - "total": {k: sum(r[k] for r in rows) for k in ("added", "updated", "removed")}, - # 不指定表时挑表途中确知是空表、跳过了的表（指定了表时总是空列表） - "empty_tables": report.empty_tables, - "note": note, - } - - @router.post("/{source_id}/catalog/{table}/review") async def review_catalog_item(source_id: str, table: str, payload: CatalogReviewIn, x_actor: str | None = Header(default=None), diff --git a/backend/app/api/coded.py b/backend/app/api/coded.py index 204b94af..d8fdba23 100644 --- a/backend/app/api/coded.py +++ b/backend/app/api/coded.py @@ -11,14 +11,18 @@ from fastapi.responses import JSONResponse #: 限定的数据源一个都不在了（删了或停用了），见 copilot._sources DATASOURCE_SCOPE_EMPTY = "datasource_scope_empty" -#: 保存数据源时剖析设置不合规（422）：field 是出错的那一项（max_queries……），说不清是哪一项时没有 field +#: 保存数据源时剖析设置不合规（422）：field 是出错的那一项（table_max_queries……），说不清是哪一项时没有 field PROFILE_SETTINGS_INVALID = "profile_settings_invalid" -#: 剖析开始不了（409）的五种情况，见 api/catalog.profile_catalog_tables +#: 剖析任务开始不了（409，api/catalog.create_profile_job）：未开启、数据源停用、没有表结构、快照被改动、这个源已有排队或 +#: 进行中的剖析任务（导入目录、同步起草遇到剖析任务也是它）、按范围挑出 0 张表 PROFILE_DISABLED = "profile_disabled" DATASOURCE_INACTIVE = "datasource_inactive" SCHEMA_MISSING = "schema_missing" PROFILE_BUSY = "profile_busy" SNAPSHOT_TAMPERED = "snapshot_tampered" +PROFILE_NOTHING = "profile_nothing" +#: 要停止的剖析任务已经结束了（409） +CATALOG_PROFILE_FINISHED = "catalog_profile_finished" #: 数据目录的后台起草任务（api/catalog.py 的 /catalog/draft-jobs）：同一个源已有排队或进行中的任务（409，界面改去 #: GET /draft-jobs/active 显示那一个）、起草模型用不了（409，不建任务）、要停止的任务已经结束了（409） CATALOG_DRAFT_BUSY = "catalog_draft_busy" diff --git a/backend/app/api/datasources.py b/backend/app/api/datasources.py index 2680e59b..38bebb9b 100644 --- a/backend/app/api/datasources.py +++ b/backend/app/api/datasources.py @@ -23,7 +23,7 @@ from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw from app.core.config import settings from app.core.crypto import encrypt, mask from app.data import catalog as data_catalog -from app.data.catalog_profile import profile_settings_rejection +from app.data.catalog_profile import profile_settings_rejection, without_legacy from app.data import introspect as introspect_mod from app.data import table_versions from app.data.engine import ( @@ -465,12 +465,13 @@ async def create_source( )).scalar_one_or_none() if exists: raise HTTPException(409, f"已存在名为「{payload.name}」的数据源，请换一个标识") - _refuse_bad_options(payload.options) + options = without_legacy(payload.options) or {} + _refuse_bad_options(options) _refuse_reserved(payload.kind, payload.database) row = DataSource( **payload.model_dump(exclude={"password", "options"}), - options=_with_managed(payload.options, None), + options=_with_managed(options, None), password=encrypt(payload.password), ) _settle_oracle(row, None, None) @@ -519,6 +520,8 @@ async def update_source( # 升级前就指向上传目录的手工源：启动时已强制只读（table_versions.startup），不许再改回可写 raise HTTPException(422, _UPLOAD_DIR_READONLY) if "options" in data: + # 剖析设置里还带着以前按整次剖析算的预算（老页面提交的）：只摘掉，不报错（catalog_profile.LEGACY_KEYS） + data["options"] = without_legacy(data["options"]) _refuse_bad_options(data["options"]) data["options"] = _with_managed(data["options"] or {}, row.options) if "password" in data: diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py index 5d537079..56f2160b 100644 --- a/backend/app/data/catalog_assistant.py +++ b/backend/app/data/catalog_assistant.py @@ -15,7 +15,7 @@ 是这个值又确认过的不提；有效记录条件是模型写的 SQL，另按起草的规矩核对（_Tools.filter_problem：一个条件、只用这张表的 列、不能有子查询……）。核对过的作为 proposal 事件交给界面，**不写目录**。没收下的项和原因交回模型（不带任何取值）。 界面上还没保存的修改（pending）里同一列的码值并进新提的码值：界面上同一路径以后一次为准，不能把先提的码值丢了。 -- run_query：只读查询，只有数据源开了数据剖析才给。经剖析的只读守卫和单条时限（catalog_profile._Runner： +- run_query：只读查询，只有数据源开了数据剖析才给。经剖析的单条查询入口（catalog_profile.run_single_query： guard.check(readonly=True)，单条时限取剖析设置和数据源查询时限中较小的一个），每次请求最多 MAX_QUERIES 条（守卫拒掉的 也算），最多 QUERY_ROWS 行交给模型、SHOWN_ROWS 行交给界面。结果交给模型、界面之前先遮（engine/masking.mask_query_result： 取自遮罩列的结果列整列换成「已遮罩」，改了名字的也认；这一次请求里前面遮掉的值按值换），报错原话里的值也换。没开剖析时 @@ -52,7 +52,6 @@ from __future__ import annotations import asyncio import contextlib -import dataclasses import json import logging from collections.abc import AsyncIterator, Iterable, Mapping @@ -621,15 +620,11 @@ class _Tools: known = known if known.strong else None error: str | None = None result: Any = None - settings = dataclasses.replace(self.ctx.settings, max_queries=1) try: - runner = catalog_profile._Runner(self.ctx.query_source, settings, - catalog_profile.SqlDialect(self.ctx.kind)) - result = await runner(sql, max_rows=QUERY_ROWS) - except catalog_profile._Skip as e: + result = await catalog_profile.run_single_query(self.ctx.query_source, sql, settings=self.ctx.settings, + max_rows=QUERY_ROWS, kind=self.ctx.kind) + except catalog_profile.QueryFailed as e: error = e.detail - except catalog_profile._Stop as e: - error = str(e) except SnapshotTampered as e: error = QUERY_TAMPERED.format(name=self.ctx.source_name, error=e) except ValueError as e: # 不支持的数据库类型 diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 59dd6aa5..2e6f2b7e 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -5,28 +5,47 @@ （来源记为 profile，和外键约束的 fk 分开），对不上的保持 proposed 并在备注里写明覆盖率。顺带给状态类的列 取一次取值分布（码值候选，含义留给人填），给只有一个日期列的表提议业务日期。 +**一次剖析一张表。** 入口是 profile_table：先列全这张表要做的检查（plan_checks，纯函数——范围里的 profile_checks、 +建任务时列出的检查项、执行时用的是同一份），第一条查询估行数（确知是空表就整张表跳过），再逐项执行，每项开始、结束经 +hooks 回调（后台任务据此把进度落库），整张表结束把结论并进目录（_apply，乐观锁）。没查成的项，目录里剖析以前对它的 +结论原样留着：这次没查成不等于结论错了。以前是一次同步请求剖析一批表、预算按整次算，选 50 张表只做得完前两三张， +其余整张记「查询次数用完」；现在每张表各有自己的预算，任务、排队、停止、留痕的落库在 data/catalog_profile_jobs.py， +这里不碰任务表。 + **这是数据目录里唯一会对业务库发查询的地方**，约束都是硬的： - 默认关闭，按数据源在 options.catalog_profile 里开启（profile_settings）； - 每条查询先过守卫的只读判定（不论数据源本身可写与否），再交给查询层 run_query：同一套连接、守卫和 - 时限，不另开连接； -- 每条查询有时限（不超过数据源自己的查询时限），每次剖析有查询次数上限和总时长上限； + 时限，不另开连接；查询一条一条发，不并发； +- 单条查询的时限取设置和数据源查询时限中小的，且不超过这张表剩下的时长； +- 每张表有自己的预算（TableBudget：查询次数、时长），用完只停这张表，剩下的项记「这张表的查询次数（时长）用完」，下一张 + 表拿满额。整个任务一个查询器（SourceRunner）管任务级止损：连续 3 条查询失败（多半是连接断了、没有权限）、最近 10 条里 + 7 条超时（库响应慢，再发只是给业务库加压）就停下整个任务；最近 6 条里 4 条超时先提醒； +- 每条发出的查询（含超时、失败、守卫拒掉的）都留痕，SQL 里的字面量换成「?」（sql_semantics.normalize_sql）：抽样到的 + 键值不进 AgentLab 自己的库； - 表有多大先看数据库的统计信息，没有统计信息就数到上限为止，不做全表 COUNT；超过上限的表不做 COUNT(DISTINCT)、MIN/MAX 这类整表统计，抽样和取值分布也只看前若干行； - 数据源设置里遮罩的列不取样，也不出现在任何一条剖析查询里； - 只取低基数列的取值分布（最多 21 个取值），不取明细行。 + +表助手的只读查询、起草时核对有效记录条件走单条查询的入口（run_single_query）：同一个只读守卫、同一个单条时限，一次一条， +不设预算、不留痕。 """ from __future__ import annotations +import asyncio +import collections import copy import logging import re import time from collections.abc import Awaitable, Callable from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone from decimal import Decimal from typing import Any +from sqlalchemy import select, update from sqlalchemy.dialects import mysql, oracle, postgresql, sqlite from sqlalchemy.ext.asyncio import AsyncSession @@ -34,6 +53,8 @@ from app.data import catalog, guard from app.data import engine as data_engine from app.data.engine import CATALOG_PROFILE_OPTION, QueryResult, SnapshotTampered from app.data.guard import QueryLimits, SqlRejected +from app.data.sql_semantics import normalize_sql +from app.db.base import SessionLocal logger = logging.getLogger(__name__) @@ -51,33 +72,42 @@ class ProfileSettings: """一个数据源的剖析设置。缺省即「关闭」：剖析会对业务库发查询，必须有人按源明确打开。""" enabled: bool = False - #: 一次剖析最多发几条查询（读统计信息、抽样、核对都算）。用完就停，剩下的记为「查询次数用完」 - max_queries: int = 60 - #: 单条查询的时限（秒）。实际取它和数据源自己的查询时限中较小的一个 + #: 每张表最多发几条查询（估行数、抽样、核对都算）。用完只停这张表，剩下的项记为「这张表的查询次数用完」 + table_max_queries: int = 40 + #: 每张表的时长上限（秒）。用完同样只停这张表 + table_max_s: float = 60 + #: 单条查询的时限（秒）。实际取它和数据源自己的查询时限中较小的一个，且不超过这张表剩下的时长 query_timeout_s: float = 10 #: 每条关系从子表抽多少个不同的键值去父表里对 sample_size: int = 2000 #: 行数不超过它的表才做 COUNT(DISTINCT)、MIN/MAX 这类整表统计；更大的、或者不知道多大的表只看前若干行。 #: 填 0 表示一律不做整表统计 max_scan_rows: int = 100_000 - #: 一次剖析的总时长上限（秒）。接口是同步的，不能让一个请求无限地挂着 - max_total_s: float = 120 def to_dict(self) -> dict[str, Any]: return asdict(self) + @classmethod + def from_dict(cls, raw: Any) -> ProfileSettings: + """任务里存着的设置快照 → 设置（读法同 profile_settings：哪一项不对就用缺省；开关按快照里的）。""" + return profile_settings({PROFILE_OPTION: raw}) if isinstance(raw, dict) else cls() + #: 数值项：(中文叫法, 下限, 上限, 是否必须是整数)。叫法和数据源设置里剖析预算的标签一致（frontend/src/lib/terms.ts）， #: 报错里只写它 _NUMBER_FIELDS: dict[str, tuple[str, float, float, bool]] = { - "max_queries": ("查询次数上限", 1, 500, True), + "table_max_queries": ("每张表最多查询数", 1, 200, True), + "table_max_s": ("每张表时长上限（秒）", 10, 600, False), "query_timeout_s": ("单条查询时限（秒）", 1, 60, False), "sample_size": ("抽样键值数", 10, 10_000, True), "max_scan_rows": ("整表统计行数上限", 0, 10_000_000, True), - "max_total_s": ("总时长上限（秒）", 10, 600, False), } _ENABLED_LABEL = "开启数据剖析" _KNOWN_KEYS = frozenset({"enabled", *_NUMBER_FIELDS}) +#: 以前的整次预算（一次剖析最多几条查询、总时长）。现在预算按表算，这两项不再有意义：启动时从库里删掉 +#: （migrate_settings），保存时请求里还带着的（老页面）只摘掉、不报错（without_legacy）。不折算成新键：旧值按整次剖析 +#: 算，和「每张表」含义不同，硬折算才是不照实 +LEGACY_KEYS = ("max_queries", "max_total_s") def _fmt(value: float) -> str: @@ -101,7 +131,7 @@ def _number(value: Any, low: float, high: float, integer: bool) -> int | float | def profile_settings(options: dict[str, Any] | None) -> ProfileSettings: """options → 剖析设置。读的时候宽松：库里存着的某一项不对（老数据、手改过），那一项用缺省，不报错； - 开关只认 true，写成别的一律当作没开。""" + 开关只认 true，写成别的一律当作没开。认不出的项（含迁移前的旧键）不看。""" raw = (options or {}).get(PROFILE_OPTION) if not isinstance(raw, dict): return ProfileSettings() @@ -121,7 +151,8 @@ def profile_settings_problem(options: dict[str, Any] | None) -> str | None: def profile_settings_rejection(options: dict[str, Any] | None) -> tuple[str, str | None] | None: """同 profile_settings_problem，另给出错的是哪一项：(给人看的一句, 字段名)。字段名是 enabled 或 _NUMBER_FIELDS - 的键（max_queries……），格式不对、有认不出的项时为 None。接口带着它回 422，表单不必从原话里认是哪一格。""" + 的键（table_max_queries……），格式不对、有认不出的项时为 None。接口带着它回 422，表单不必从原话里认是哪一格。 + 旧键（LEGACY_KEYS）在这之前已经由 without_legacy 摘掉。""" raw = (options or {}).get(PROFILE_OPTION) if raw is None: return None @@ -138,6 +169,36 @@ def profile_settings_rejection(options: dict[str, Any] | None) -> tuple[str, str return None +def without_legacy(options: dict[str, Any] | None) -> dict[str, Any] | None: + """保存数据源时：剖析设置里还带着旧的整次预算（老页面提交的），只摘掉，不报错。没有旧键原样返回。""" + raw = (options or {}).get(PROFILE_OPTION) + if not isinstance(raw, dict) or not any(key in raw for key in LEGACY_KEYS): + return options + return {**(options or {}), PROFILE_OPTION: {k: v for k, v in raw.items() if k not in LEGACY_KEYS}} + + +async def migrate_settings(session: AsyncSession) -> int: + """服务启动时一次性迁移：库里所有数据源 options.catalog_profile 中的旧键（max_queries、max_total_s）删掉，记日志。 + 不折算成新键（见 LEGACY_KEYS）。返回改了几个数据源。只读写设置，不连业务库。""" + from app.db.models import DataSource + + rows = (await session.execute(select(DataSource.id, DataSource.name, DataSource.options))).all() + changed = 0 + for source_id, name, options in rows: + cleaned = without_legacy(options if isinstance(options, dict) else None) + if cleaned is options or cleaned is None: + continue + dropped = {k: (options or {}).get(PROFILE_OPTION, {}).get(k) for k in LEGACY_KEYS + if k in ((options or {}).get(PROFILE_OPTION) or {})} + await session.execute(update(DataSource).where(DataSource.id == source_id).values(options=cleaned) + .execution_options(synchronize_session=False)) + logger.info("数据剖析设置迁移：数据源 %s（%s）去掉了按整次剖析算的旧预算 %s，改按每张表的预算", source_id, name, + dropped) + changed += 1 + if changed: + await session.commit() + return changed + # ========================================================================== # SQL：字面量与方言 # ========================================================================== @@ -279,17 +340,33 @@ class SqlDialect: # ========================================================================== -# 发查询：守卫、查询次数、时限 +# 发查询：守卫、单条时限、每张表的预算、任务级止损、留痕 # ========================================================================== -#: 总时长只剩这么一点时不再发新查询：发出去的时限也只剩零点几秒，白占一次查询次数 +#: 这张表的时长只剩这么一点时不再发新查询：发出去的时限也只剩零点几秒，白占一次查询次数 _MIN_WINDOW_S = 0.5 -#: 连续失败这么多条就停：多半是连接断了、账号没有权限，再发只是一条条重复同一个错误 -_MAX_CONSECUTIVE_ERRORS = 3 +#: 连续失败这么多条就停下整个任务：多半是连接断了、账号没有权限，再发只是一条条重复同一个错误 +MAX_CONSECUTIVE_ERRORS = 3 +#: 最近 SLOW_WINDOW 条查询里 SLOW_STOP 条超时就停下整个任务：库响应慢，再发只是给业务库加压 +SLOW_WINDOW = 10 +SLOW_STOP = 7 +#: 最近 NOTICE_WINDOW 条里 NOTICE_ON 条超时先提醒（任务的 notice），好转后清掉 +NOTICE_WINDOW = 6 +NOTICE_ON = 4 +#: 「运行原因没查成」的跳过原因：计入 unfinished，使这张表成为 partial。其余（遮罩、视图、表太大、取值太多……）是 +#: 按规则不查，不算没做完 +UNFINISHED_REASONS = frozenset({"budget", "deadline", "failed", "timeout", "error", "rejected", "slow", "cancelled"}) + +FAILED_STOP = "连续 {n} 条查询失败，已停止剖析：{line}" +SLOW_STOP_TEXT = f"数据库响应慢：最近 {SLOW_WINDOW} 条查询 {SLOW_STOP} 条超时，已停止，避免给业务库加压" +SLOW_NOTICE = "数据库响应慢：最近 {n} 条查询有 {k} 条超时。超时再多会自动停止，也可以先停止、换个时间再剖析" +#: 停止落在一条查询上：asyncio 这边不再等它的结果，库那边由单条时限截止（run_query 的 _server_deadline） +CANCELLED_QUERY = "剖析已停止，没有等这条查询的结果；数据库按单条时限截止它" class _Stop(Exception): - """整次剖析停下：budget 查询次数用完、deadline 总时长用完、failed 连续失败。剩下的检查都不再发查询。""" + """停下：budget 这张表的查询次数用完、deadline 这张表的时长用完（只停这张表）；failed 连续失败、slow 超时比例 + （整个任务停下）。剩下的检查都不再发查询。""" def __init__(self, reason: str) -> None: super().__init__(reason) @@ -310,6 +387,16 @@ class _Skip(Exception): self.carry = carry +class QueryFailed(Exception): + """单条只读查询（run_single_query）没成：reason 是 rejected 未通过守卫 / timeout 超时 / error 查询失败，detail + 是给人看的整句（同剖析里这一项的说明）。""" + + def __init__(self, reason: str, detail: str) -> None: + super().__init__(detail) + self.reason = reason + self.detail = detail + + def _brief(e: BaseException) -> str: """驱动报错的第一行，去掉 SQLAlchemy 加的「(模块.类名)」前缀和后面附的 SQL、文档链接。""" lines = str(getattr(e, "orig", None) or e).strip().splitlines() @@ -317,13 +404,39 @@ def _brief(e: BaseException) -> str: return first[:160] or type(e).__name__ -class _Runner: - """剖析发的每一条查询都从这里走：先按只读过守卫，再交给查询层 run_query；记次数、管时限。 +class TableBudget: + """一张表的预算：最多 table_max_queries 条查询、table_max_s 秒（从这张表开始算）。用完抛 _Stop（budget / + deadline），只停这张表，下一张表另起一份满额的。""" + + def __init__(self, settings: ProfileSettings) -> None: + self.max_queries = int(settings.table_max_queries) + self.seconds = float(settings.table_max_s) + self.ends = time.monotonic() + self.seconds + self.used = 0 + #: 一条查询的时限被这张表剩下的时长截短后超时了：时长已经用完 + self.out_of_time = False + + def remaining(self) -> float: + """要发一条查询之前：次数、时长都还有，返回这张表剩下的秒数；用完抛 _Stop。不记次数（守卫拒掉的不算）。""" + if self.used >= self.max_queries: + raise _Stop("budget") + left = self.ends - time.monotonic() + if self.out_of_time or left <= _MIN_WINDOW_S: + raise _Stop("deadline") + return left + + +class SourceRunner: + """剖析发的每一条查询都从这里走（整个任务一个）：先按只读过守卫，再交给查询层 run_query；管单条时限、任务级止损， + 每条查询交给 record 留痕。 - 只读：不论数据源本身可写与否，先用 guard.check(readonly=True) 判一遍，再进 run_query——那里还会按 数据源自己的设置判一遍。两遍是同一个守卫，不为剖析放宽，也不另开连接。 - - 查询次数：每发一条记一次（超时、失败的也算），到上限就停。没发出去的（守卫拒了）不算。 - - 时限：单条取设置和数据源查询时限中较小的一个，并且不超过总时长剩下的部分。 + - 单条时限：取设置和数据源查询时限中较小的一个，并且不超过这张表剩下的时长（budget）。 + - 预算：每发一条记一次这张表的次数（超时、失败的也算），守卫拒掉的没发出去，不算。 + - 止损：连续 MAX_CONSECUTIVE_ERRORS 条失败 → stopped = failed；最近 SLOW_WINDOW 条里 SLOW_STOP 条超时 → + stopped = slow。停下之后再发直接抛 _Stop，error 是整句原因。最近 NOTICE_WINDOW 条里 NOTICE_ON 条超时时 notice + 有一句提醒，好转后清掉。 """ def __init__(self, source: Any, settings: ProfileSettings, dialect: SqlDialect) -> None: @@ -331,49 +444,101 @@ class _Runner: self.settings = settings self.kind = dialect.kind self.timeout = min(float(settings.query_timeout_s), float(data_engine.query_timeout(source))) - self.ends = time.monotonic() + float(settings.max_total_s) - self.used = 0 + #: 整个任务发出的查询数、其中超时的、失败的 + self.queries = 0 + self.timeouts = 0 + self.errors = 0 self.stopped: str | None = None - self._errors = 0 + self.error: str | None = None + self.notice: str | None = None + self._consecutive = 0 + self._recent: collections.deque[bool] = collections.deque(maxlen=SLOW_WINDOW) - def _stop(self, reason: str) -> None: - self.stopped = reason - raise _Stop(reason) + def _timed(self, timed_out: bool) -> None: + """记下这条查询超时没有，按比例判提醒和止损。""" + self._recent.append(timed_out) + recent = list(self._recent) + window = recent[-NOTICE_WINDOW:] + hits = sum(window) + self.notice = SLOW_NOTICE.format(n=len(window), k=hits) if hits >= NOTICE_ON else None + if sum(recent) >= SLOW_STOP and self.stopped is None: + self.stopped = "slow" + self.error = SLOW_STOP_TEXT - async def __call__(self, sql: str, *, max_rows: int = 1) -> QueryResult: + async def run(self, sql: str, *, max_rows: int = 1, budget: TableBudget | None = None, + record: Callable[[dict[str, Any]], None] | None = None) -> QueryResult: if self.stopped: raise _Stop(self.stopped) - if self.used >= self.settings.max_queries: - self._stop("budget") - remaining = self.ends - time.monotonic() - if remaining <= _MIN_WINDOW_S: - self._stop("deadline") + left = budget.remaining() if budget is not None else None + at = datetime.now(timezone.utc).isoformat(timespec="milliseconds") + + def log(outcome: str, ms: int, rows: int | None, detail: str | None) -> None: + if record is not None: + record({"at": at, "sql": normalize_sql(sql, self.kind).text, "ms": ms, "rows": rows, + "outcome": outcome, "detail": detail}) + try: guard.check(sql, readonly=True, source_name=getattr(self.source, "name", ""), dialect=self.kind) except SqlRejected as e: - raise _Skip("rejected", f"剖析查询未通过安全守卫：{e}") from e - self.used += 1 - cut = remaining < self.timeout - seconds = round(min(self.timeout, remaining), 3) + detail = f"剖析查询未通过安全守卫：{e}" + log("rejected", 0, None, detail) + raise _Skip("rejected", detail) from e + if budget is not None: + budget.used += 1 + self.queries += 1 + cut = left is not None and left < self.timeout + seconds = round(min(self.timeout, left) if left is not None else self.timeout, 3) limits = QueryLimits(max_rows=max(1, int(max_rows)), timeout_seconds=seconds) + started = time.perf_counter() + + def ms() -> int: + return int((time.perf_counter() - started) * 1000) + try: result = await data_engine.run_query(self.source, sql, limits=limits) except SqlRejected as e: # 上面已经按只读过了同一个守卫，查询层再拒只可能是时限到了 - if cut: - self.stopped = "deadline" - raise _Skip("timeout", f"查询超时：超过 {_fmt(seconds)} 秒被中断，已跳过") from e + self.timeouts += 1 + if cut and budget is not None: + budget.out_of_time = True + detail = f"查询超时：超过 {_fmt(seconds)} 秒被中断，已跳过" + log("timeout", ms(), None, detail) + self._timed(True) + raise _Skip("timeout", detail) from e except SnapshotTampered: raise - except Exception as e: # noqa: BLE001 - 一条查询失败不该拖垮整次剖析，原因照实记下 - self._errors += 1 - if self._errors >= _MAX_CONSECUTIVE_ERRORS: + except asyncio.CancelledError: + log("error", ms(), None, CANCELLED_QUERY) + raise + except Exception as e: # noqa: BLE001 - 一条查询失败不该拖垮整张表，原因照实记下 + self.errors += 1 + self._consecutive += 1 + detail = f"查询失败：{_brief(e)}" + log("error", ms(), None, detail) + self._timed(False) + if self._consecutive >= MAX_CONSECUTIVE_ERRORS and self.stopped is None: self.stopped = "failed" - raise _Skip("error", f"查询失败：{_brief(e)}") from e - self._errors = 0 + self.error = FAILED_STOP.format(n=MAX_CONSECUTIVE_ERRORS, line=_brief(e)) + raise _Skip("error", detail) from e + self._consecutive = 0 + log("ok", ms(), result.row_count, None) + self._timed(False) return result +async def run_single_query(source: Any, sql: str, *, settings: ProfileSettings, max_rows: int = 1, + kind: str | None = None) -> QueryResult: + """单条只读查询（表助手的 run_query、起草时核对有效记录条件）：同剖析一样过只读守卫、交给 run_query，单条时限取剖析 + 设置和数据源查询时限中较小的一个。只有一条：不设预算、不止损、不留痕。 + + 没成抛 QueryFailed（reason、detail）；快照被改动抛 SnapshotTampered；不支持的数据库类型抛 ValueError。""" + dialect = SqlDialect(kind if kind is not None else getattr(source, "kind", None)) + try: + return await SourceRunner(source, settings, dialect).run(sql, max_rows=max_rows) + except _Skip as e: + raise QueryFailed(e.reason, e.detail) from e + + # ========================================================================== # 表有多大 # ========================================================================== @@ -411,12 +576,13 @@ def _as_int(value: Any) -> int | None: return int(number) if number == number and abs(number) != float("inf") else None -async def estimate_size(run: _Runner, dialect: SqlDialect, meta: dict[str, Any], name: str, *, - max_scan_rows: int) -> tuple[TableSize, list[_Skip]]: +async def estimate_size(run: Callable[..., Awaitable[QueryResult]], dialect: SqlDialect, meta: dict[str, Any], + name: str, *, max_scan_rows: int) -> tuple[TableSize, list[_Skip]]: """先读统计信息；读不到（SQLite、没分析过的表）再数到 max_scan_rows + 1 行为止。 统计信息为 0 或负数（PostgreSQL 没分析过的表是 -1，MySQL 的估算值可能是 0）当作不知道：一张其实很大的表 - 要是被当成空表，后面就会对它做整表统计。返回 (行数, 途中没查成的几条)。查询次数、总时长用完时抛 _Stop。 + 要是被当成空表，后面就会对它做整表统计。返回 (行数, 途中没查成的几条)。这张表的查询次数、时长用完（或者任务止损） +时抛 _Stop。 """ skips: list[_Skip] = [] sql = dialect.stats_sql(meta, name) @@ -444,24 +610,33 @@ async def estimate_size(run: _Runner, dialect: SqlDialect, meta: dict[str, Any], # ========================================================================== -# 一次剖析：上下文、结果 +# 一张表的剖析：上下文、结果 # ========================================================================== -#: 不指定表时剖析几张：目录里有要核对的关系的表，还有待核实（推断）关系的在前，再按使用次数；确知是空表的跳过 -PROFILE_DEFAULT_TABLES = 10 -#: 不指定表时，为跳过空表最多先估几张候选表的行数；估行数的查询最多用掉查询次数上限的一半，免得都花在挑表上 -PROFILE_PROBE_TABLES = 30 #: IN 列表每批最多几个值：Oracle 的 IN 列表不能超过 1000 项（ORA-01795），各家统一按它分批 IN_CHUNK = 1000 #: 剖析写的备注都以它开头（「数据剖析：」）：人工确认过的关系，只覆盖剖析自己写的备注，不碰人写的（认的时候连 #: 升级前带日期的「数据剖析（日期）：」一起认，catalog.is_profile_note）。不写日期，时间看这一项的 updated_at NOTE_PREFIX = catalog.PROFILE_NOTE_PREFIX -#: 整次剖析停下的原因 → 剩下没查的项的说明 +#: 停下的原因 → 剩下没查的项的说明。failed、slow 是任务级止损，停的是整个任务；cancelled 的说法由调用方给（停止、 +#: 服务重启各有各的说法，见 TableHooks.stop_text） _STOP_DETAIL = { - "budget": "本次剖析的查询次数已用完（上限 {max_queries} 条），未检查", - "deadline": "本次剖析的总时长已用完（上限 {max_total_s} 秒），未检查", + "budget": "这张表的查询次数已用完（上限 {max_queries} 条），未检查", + "deadline": "这张表的时长已用完（上限 {max_s} 秒），未检查", "failed": "连续多条查询失败，已停止剖析，未检查", + "slow": "数据库响应慢，已停止剖析，未检查", } +#: 整张表的原因（TableResult.error） +VIEW_ERROR = "视图没有统计信息，剖析可能触发整个视图的计算，已跳过" +EMPTY_ERROR = "空表，没有可剖析的数据" +NOTHING_ERROR = "没有可剖析的项：没有要核对的关联关系、像码值的列和日期列，或者都被遮罩了" +CONFLICT_ERROR = "这张表的数据目录正被其他人修改，剖析结果未能写入，请稍后重新剖析" +WRITE_ERROR = "写入数据目录时出错：{line}" +TABLE_ERROR = "剖析这张表时出错：{line}" +#: 停止、服务重启时正在剖析的那张表：做完的项照常写进了目录，剩下的没查 +STOPPED_TABLE = "{why}：已做完的 {done} 项已写进数据目录，剩下 {rest} 项没查" +#: 停止的说法（TableHooks.stop_text 没给时） +STOPPED = "剖析已停止" class ProfileDisabled(Exception): @@ -473,18 +648,15 @@ class ProfileDisabled(Exception): self.code = code -class ProfileBusy(Exception): - """这个数据源正在剖析。message 给人看。""" - - @dataclass class ProfileSkip: """没做的一项和原因。 kind：relation 关系、codes 码值、date 日期列、row_estimate 行数估算、table 整张表。 - reason：budget 查询次数用完、deadline 总时长用完、failed 连续失败、timeout 超时、error 查询失败、 - rejected 未通过守卫、masked 字段被遮罩、too_large 表太大、view 视图、unsupported 暂不支持、 - no_data 没有数据、missing 表结构里没有、high_cardinality 取值太多（不像码值）。detail 是给人看的整句。 + reason：budget 这张表的查询次数用完、deadline 这张表的时长用完、failed 连续失败止损、slow 超时比例止损、 + cancelled 停止或服务重启、timeout 超时、error 查询失败、rejected 未通过守卫（以上算「运行原因没查成」，见 + UNFINISHED_REASONS）；masked 字段被遮罩、too_large 表太大、view 视图、unsupported 暂不支持、no_data 没有数据、 + missing 表结构里没有、high_cardinality 取值太多（不像码值）、nothing 没有可剖析的项（按规则不查）。detail 是给人看的整句。 """ kind: str @@ -499,35 +671,57 @@ class ProfileSkip: @dataclass -class TableProfile: - """一张表这次剖析的结果。findings 是写进目录的发现（给界面看，形状见 _finding_*）。""" +class TableResult: + """一张表这次剖析的结果。 + + status：ok 每项都有结论、partial 有项因为运行原因没查成、failed 写入一直冲突或整张表出错、skipped 表结构里没有 / + 视图 / 空表 / 没有可剖析的项。checks 是检查项（[{kind, target, path, state, reason, detail, queries}]，按执行顺序， + 含按规则不查的）；findings 是写进目录的发现（给界面看，形状见 _relation_finding 等）；query_log 是发过的查询 + （字面量已换成「?」）。""" table: str + status: str = "running" + checks: list[dict[str, Any]] = field(default_factory=list) + #: 进行中时正在做哪一项的整句 + current: str | None = None queries: int = 0 + elapsed_ms: int = 0 row_estimate: TableSize | None = None findings: list[dict[str, Any]] = field(default_factory=list) skipped: list[ProfileSkip] = field(default_factory=list) #: 日期类列的取值范围（MIN、MAX）。只有一个日期列时它还写进了业务日期的备注；多个时只在这里 date_ranges: list[dict[str, Any]] = field(default_factory=list) + query_log: list[dict[str, Any]] = field(default_factory=list) added: int = 0 updated: int = 0 removed: int = 0 version: int = 0 - #: 这张表没剖析（表结构里没有）或结果没写进去（写入一直冲突） + #: 整张表的原因：表结构里没有、视图、空表、没有可剖析的项、写入一直冲突、停止 / 中断 / 止损 error: str | None = None + @property + def skipped_count(self) -> int: + return len(self.skipped) -@dataclass -class ProfileReport: - """一次剖析的结果。stopped：整次剖析中途停下的原因（budget / deadline / failed），做完了为 None。""" + @property + def unfinished(self) -> int: + """跳过的项里因为运行原因没查成的（次数、时长、超时、出错、守卫拒、止损、停止）。""" + return sum(1 for s in self.skipped if s.reason in UNFINISHED_REASONS) - profiled_at: str - settings: ProfileSettings - tables: list[TableProfile] = field(default_factory=list) - queries_used: int = 0 - stopped: str | None = None - #: 不指定表时，挑表途中估过行数、确知是空表而跳过的表（表结构顺序）。估行数的查询算在 queries_used 里 - empty_tables: list[str] = field(default_factory=list) + def summary(self) -> dict[str, int]: + return summarize(self.findings) + + +def summarize(findings: list[dict[str, Any]] | None) -> dict[str, int]: + """发现的合计：有结论的关系条数、其中有确证（verified）的、取到码值候选的列数、这些码值里含义还空着的个数、提议了 + 业务日期的表数（一张表时是 0 或 1）。""" + found = [f for f in findings or [] if isinstance(f, dict)] + relations = [f for f in found if f.get("kind") == "relation"] + codes = [f for f in found if f.get("kind") == "codes"] + return {"relations_checked": len(relations), + "relations_verified": sum(1 for f in relations if f.get("status") == "verified"), + "codes": len(codes), "codes_pending": sum(int(f.get("pending") or 0) for f in codes), + "business_dates": sum(1 for f in found if f.get("kind") == "business_date")} @dataclass @@ -584,34 +778,66 @@ class _TableOutcome: @dataclass -class _Context: +class ProfileContext: + """一个剖析任务共用的：源（上传源是绑定当前快照的视图）、设置、方言、查询器（任务级止损在它身上）、表结构、遮罩的 + 列（小写，任务每张表开始前按数据源最新的设置换掉）、已经估过的行数（父表的行数核对关系时会用到，不重复估）。""" + source: Any settings: ProfileSettings dialect: SqlDialect - run: _Runner + runner: SourceRunner tables: dict[str, Any] #: 遮罩的列（小写）。遮罩按列名生效、不分大小写、不分表，和证据面板的比法一致 masked: set[str] sizes: dict[str, TableSize] = field(default_factory=dict) - async def size(self, table: str, result: TableProfile) -> TableSize: - """表有多大，一次剖析里每张表只估一次。途中没查成的记进 result（行数估算这一项）。""" - if table not in self.sizes: - size, skips = await estimate_size(self.run, self.dialect, self.tables[table], table, - max_scan_rows=self.settings.max_scan_rows) - self.sizes[table] = size - result.skipped += [ProfileSkip("row_estimate", table, s.reason, s.detail) for s in skips] - return self.sizes[table] - async def size_with_data(self, table: str, result: TableProfile) -> TableSize: - """同 size，但确知是空表时直接跳过这一项（抛 _Skip），不再为它发查询。""" - size = await self.size(table, result) - if size.rows == 0: - raise _Skip("no_data", f"{table} 是空表，无法剖析") - return size +def make_context(source: Any, settings: ProfileSettings, *, options: dict[str, Any] | None) -> ProfileContext: + """剖析任务的上下文。options 是数据源记录的 options（遮罩按它）；不支持的数据库类型抛 ValueError。""" + dialect = SqlDialect(getattr(source, "kind", None)) + return ProfileContext(source=source, settings=settings, dialect=dialect, + runner=SourceRunner(source, settings, dialect), + tables=(getattr(source, "schema_cache", None) or {}).get("tables") or {}, + masked=masked_of(options)) + + +def masked_of(options: dict[str, Any] | None) -> set[str]: + return {c.lower() for c in data_engine.masked_columns(options)} + + +@dataclass +class TableHooks: + """profile_table 的回调（后台任务据此落库）。都可以不给。 + + - progress：一项检查（含开头的估行数）开始、结束时；result 里的 checks、current、queries、query_log 是这时的样子； + - query：每发出（或被守卫拒掉）一条查询之后； + - recorded：整张表结束，和写目录在同一次库操作里（session 是那一次的，shield 起来，停止时也不会被打断）； + - stop_text：停止时的说法（「剖析已停止」「服务重启，剖析中断」），停止之后才调。 + """ + + progress: Callable[[TableResult], Awaitable[None]] | None = None + query: Callable[[TableResult], Awaitable[None]] | None = None + recorded: Callable[[AsyncSession, TableResult], Awaitable[None]] | None = None + stop_text: Callable[[], str] | None = None + + +class _TableCx: + """一张表这次剖析的执行状态：预算、正在做的检查项、发过的查询。检查函数经它发查询、估行数。""" + + def __init__(self, job: ProfileContext, table: str, result: TableResult, hooks: TableHooks) -> None: + self.job = job + self.settings = job.settings + self.dialect = job.dialect + self.tables = job.tables + self.table = table + self.result = result + self.hooks = hooks + self.budget = TableBudget(job.settings) + #: 正在做的检查项（result.checks 里的那一个）；None 是这张表开头的估行数 + self.check: dict[str, Any] | None = None def is_masked(self, column: str) -> bool: - return column.lower() in self.masked + return column.lower() in self.job.masked def scan_cap(self, size: TableSize) -> int | None: """大表（或不知道多大）抽样、取值分布只看前多少行；小表整表看，返回 None。""" @@ -619,6 +845,53 @@ class _Context: return None return max(self.settings.sample_size, self.settings.max_scan_rows) + async def run(self, sql: str, *, max_rows: int = 1, kind: str | None = None, + target: str | None = None) -> QueryResult: + """发一条查询：记进这张表的留痕（kind、target 是它属于哪一项）、计数，再回调 hooks.query。""" + check = self.check + label = (kind or (check["kind"] if check else "row_estimate"), + target or (check["target"] if check else self.table)) + result = self.result + + def record(entry: dict[str, Any]) -> None: + result.query_log.append({"at": entry["at"], "kind": label[0], "target": label[1], "sql": entry["sql"], + "ms": entry["ms"], "rows": entry["rows"], "outcome": entry["outcome"], + "detail": entry["detail"]}) + if entry["outcome"] != "rejected": + result.queries += 1 + if check is not None: + check["queries"] = int(check.get("queries") or 0) + 1 + + try: + found = await self.job.runner.run(sql, max_rows=max_rows, budget=self.budget, record=record) + except _Skip: + await self._queried() + raise + await self._queried() + return found + + async def _queried(self) -> None: + if self.hooks.query is not None: + await self.hooks.query(self.result) + + async def size(self, table: str, result: TableResult) -> TableSize: + """表有多大，一个任务里每张表只估一次。途中没查成的记进 result（行数估算这一项）。""" + if table not in self.job.sizes: + async def run(sql: str, **kw: Any) -> QueryResult: + return await self.run(sql, kind="row_estimate", target=table, **kw) + + size, skips = await estimate_size(run, self.dialect, self.tables[table], table, + max_scan_rows=self.settings.max_scan_rows) + self.job.sizes[table] = size + result.skipped += [ProfileSkip("row_estimate", table, s.reason, s.detail) for s in skips] + return self.job.sizes[table] + + async def size_with_data(self, table: str, result: TableResult) -> TableSize: + """同 size，但确知是空表时直接跳过这一项（抛 _Skip），不再为它发查询。""" + size = await self.size(table, result) + if size.rows == 0: + raise _Skip("no_data", f"{table} 是空表，无法剖析") + return size def _column(meta: dict[str, Any], name: str) -> str | None: """表结构里这一列的写法（目录里的列名大小写可能和表结构不同）；没有返回 None。""" @@ -665,8 +938,8 @@ def _relation_candidates(notes: dict[str, Any] | None) -> list[dict[str, Any]]: if isinstance(r, dict) and r.get("id") and r.get("status") != "rejected"] -async def _unique(cx: _Context, table: str, meta: dict[str, Any], column: str, - result: TableProfile) -> tuple[bool | None, str]: +async def _unique(cx: _TableCx, table: str, meta: dict[str, Any], column: str, + result: TableResult) -> tuple[bool | None, str]: """这一列（非空部分）唯一吗：(True / False / None 不知道, 依据)。有主键或唯一约束直接认；否则只对行数不超过 整表统计上限的表数一次 COUNT 和 COUNT(DISTINCT)，大表不扫。""" how = _constraint(meta, column) @@ -734,27 +1007,13 @@ def _relation_note(*, sample: int, matched: int, coverage: float, scan_cap: int return NOTE_PREFIX + "；".join(parts) + "。" -async def _check_relation(cx: _Context, table: str, rel: dict[str, Any], out: _TableOutcome, - result: TableProfile) -> None: +async def _check_relation(cx: _TableCx, table: str, rel: dict[str, Any], out: _TableOutcome, + result: TableResult) -> None: """从子表取至多 sample_size 个不同的非空键值，去父表里数对上几个，得到覆盖率；再看父键、子键唯不唯一。""" - cols, to_cols = list(rel.get("columns") or []), list(rel.get("to_columns") or []) - to_table = str(rel.get("to_table") or "") - if len(cols) != 1 or len(to_cols) != 1: - raise _Skip("unsupported", "多列组成的关联关系暂不剖析") - meta = cx.tables[table] - parent = to_table if to_table in cx.tables else catalog.resolve_table_name({"tables": cx.tables}, to_table) - if parent is None: - raise _Skip("missing", f"表结构里没有被指向的表 {to_table}，请先重新探查结构") - pmeta = cx.tables[parent] - col, pcol = _column(meta, cols[0]), _column(pmeta, to_cols[0]) - if col is None or pcol is None: - missing = f"{table}.{cols[0]}" if col is None else f"{parent}.{to_cols[0]}" - raise _Skip("missing", f"表结构里没有列 {missing}，请先重新探查结构") - if pmeta.get("is_view"): - raise _Skip("view", f"被指向的 {parent} 是视图，不剖析") - for name, owner in ((col, table), (pcol, parent)): - if cx.is_masked(name): - raise _Skip("masked", f"{owner}.{name} 在数据源设置中被遮罩，不取样", carry=True) + # 两端在表结构里的写法；按规则不查的（多列、不在表结构里、指向视图、碰到遮罩列）plan_checks 已经列成跳过，这里再判 + # 一遍只是兜底 + parent, col, pcol = _relation_ends(table, rel, cx.tables, cx.job.masked) + meta, pmeta = cx.tables[table], cx.tables[parent] if rel.get("source") == "fk": await cx.size_with_data(table, result) @@ -886,7 +1145,7 @@ def _share(count: int, total: int) -> str: return "不足 1%" if 0 < ratio < 0.01 else f"{round(ratio * 100)}%" -async def _check_codes(cx: _Context, table: str, column: str, out: _TableOutcome, result: TableProfile) -> None: +async def _check_codes(cx: _TableCx, table: str, column: str, out: _TableOutcome, result: TableResult) -> None: """GROUP BY 取出现最多的 CODES_MAX + 1 个取值：不超过 CODES_MAX 个就是码值候选（值有了，含义等人填）。""" if cx.is_masked(column): raise _Skip("masked", f"{table}.{column} 在数据源设置中被遮罩，不取值", carry=False) @@ -921,8 +1180,8 @@ def _is_date_column(col: dict[str, Any]) -> bool: return catalog.is_date_column(str(col.get("name") or ""), col.get("type")) -async def _check_dates(cx: _Context, table: str, columns: list[str], total: int, out: _TableOutcome, - result: TableProfile) -> None: +async def _check_dates(cx: _TableCx, table: str, columns: list[str], total: int, out: _TableOutcome, + result: TableResult) -> None: """日期类列的 MIN、MAX，作为业务日期的参考。整表统计，只对行数不超过上限的表做。表里只有一个日期类列 （total 是表结构里日期类列的个数，含被遮罩的）时提议它作为业务日期。""" d, meta = cx.dialect, cx.tables[table] @@ -1131,79 +1390,320 @@ def _apply(existing: dict[str, Any] | None, out: _TableOutcome, *, # ========================================================================== -# 入口 +# 一张表要做哪些检查（纯函数） # ========================================================================== -#: 正在剖析的数据源。检查和登记之间没有 await，在一个事件循环里是原子的；服务端是单进程部署， -#: 多进程部署时要换成库里的锁 -_RUNNING: set[str] = set() +@dataclass(frozen=True) +class PlannedCheck: + """一项检查。skip 给了的是按规则不查的（遮罩、多列关系、表结构里没有、被指向的是视图）：(原因, 说明, 要不要原样 + 留着旧结论)，不发查询；relation / column / columns 是执行时要的参数。""" -@dataclass -class _Check: kind: str target: str path: str | None - run: Callable[[], Awaitable[None]] - #: 没查成时调用：把目录里剖析以前对这一项写的结论标记为原样保留 - keep: Callable[[], None] + skip: tuple[str, str, bool] | None = None + relation: dict[str, Any] | None = None + column: str | None = None + columns: tuple[str, ...] = () + #: 日期检查：表结构里日期类列的个数（含被遮罩的）。只有一个时才提议业务日期 + date_total: int = 0 + + def as_check(self) -> dict[str, Any]: + """检查项（TableResult.checks 的一项）：还没开始，或者按规则跳过。""" + state, reason, detail = ("skipped", self.skip[0], self.skip[1]) if self.skip else ("pending", None, None) + return {"kind": self.kind, "target": self.target, "path": self.path, "state": state, "reason": reason, + "detail": detail, "queries": 0} -def _plan(cx: _Context, table: str, notes: dict[str, Any], out: _TableOutcome, - result: TableProfile) -> list[_Check]: - """这张表要做的检查，按价值排：关系、码值、日期。先列全再执行：中途停下时，没做的每一项都要记下原因。""" - meta = cx.tables[table] - checks: list[_Check] = [] +def _relation_ends(table: str, rel: dict[str, Any], tables: dict[str, Any], masked: set[str]) -> tuple[str, str, str]: + """一条关系两端在表结构里的写法：(被指向的表, 子表的列, 被指向的列)。按规则不查的抛 _Skip：多列组成的、表结构里 + 没有的表或列、被指向的是视图、两端有被遮罩的列（遮罩的列不出现在任何一条剖析查询里）。""" + cols, to_cols = list(rel.get("columns") or []), list(rel.get("to_columns") or []) + to_table = str(rel.get("to_table") or "") + if len(cols) != 1 or len(to_cols) != 1: + raise _Skip("unsupported", "多列组成的关联关系暂不剖析") + meta = tables[table] + parent = to_table if to_table in tables else catalog.resolve_table_name({"tables": tables}, to_table) + if parent is None: + raise _Skip("missing", f"表结构里没有被指向的表 {to_table}，请先重新探查结构") + pmeta = tables[parent] + col, pcol = _column(meta, cols[0]), _column(pmeta, to_cols[0]) + if col is None or pcol is None: + missing = f"{table}.{cols[0]}" if col is None else f"{parent}.{to_cols[0]}" + raise _Skip("missing", f"表结构里没有列 {missing}，请先重新探查结构") + if pmeta.get("is_view"): + raise _Skip("view", f"被指向的 {parent} 是视图，不剖析") + for name, owner in ((col, table), (pcol, parent)): + if name.lower() in masked: + raise _Skip("masked", f"{owner}.{name} 在数据源设置中被遮罩，不取样", carry=True) + return parent, col, pcol + + +def plan_checks(table: str, meta: dict[str, Any] | None, notes: dict[str, Any] | None, *, tables: dict[str, Any], + masked: set[str]) -> list[PlannedCheck]: + """这张表要做的检查，按价值排：关系、码值、日期。范围（profile_checks）、建任务时列出的检查项、执行时用的都是这一份： + 先列全再执行，中途停下时没做的每一项都要记下原因。视图、表结构里没有的表返回空列表。 + + 按规则不查的也列出来（带 skip）：被遮罩的码值列、日期列，多列的、两端不在表结构里的、指向视图的、碰到遮罩列的关系—— + 要让人看得见为什么它没有结论。masked 是遮罩的列（小写）。""" + if not isinstance(meta, dict) or meta.get("is_view"): + return [] + notes = notes or {} + checks: list[PlannedCheck] = [] for rel in _relation_candidates(notes): - rid = str(rel["id"]) - checks.append(_Check("relation", _relation_target(rel), f"relations.{rid}", - lambda rel=rel: _check_relation(cx, table, rel, out, result), - lambda rid=rid: out.keep_relations.add(rid))) + skip: tuple[str, str, bool] | None = None + try: + _relation_ends(table, rel, tables, masked) + except _Skip as e: + skip = (e.reason, e.detail, e.carry) + checks.append(PlannedCheck("relation", _relation_target(rel), f"relations.{rel['id']}", skip, relation=rel)) for column in _code_columns(meta, notes): - checks.append(_Check("codes", column, f"columns.{column}.codes", - lambda column=column: _check_codes(cx, table, column, out, result), - lambda column=column: out.keep_codes.add(column))) + skip = (("masked", f"{table}.{column} 在数据源设置中被遮罩，不取值", False) + if column.lower() in masked else None) + checks.append(PlannedCheck("codes", column, f"columns.{column}.codes", skip, column=column)) dates = [str(c["name"]) for c in meta.get("columns") or [] if isinstance(c, dict) and _is_date_column(c)] for column in dates: - if cx.is_masked(column): - result.skipped.append(ProfileSkip("date", column, "masked", - f"{table}.{column} 在数据源设置中被遮罩，不取最小值和最大值")) - usable = [c for c in dates if not cx.is_masked(c)] + if column.lower() in masked: + checks.append(PlannedCheck("date", column, None, + ("masked", f"{table}.{column} 在数据源设置中被遮罩，不取最小值和最大值", False), + column=column)) + usable = [c for c in dates if c.lower() not in masked] if usable: - checks.append(_Check("date", "、".join(usable), "business_date" if len(dates) == 1 else None, - lambda: _check_dates(cx, table, usable, len(dates), out, result), - lambda: setattr(out, "keep_business_date", True))) + checks.append(PlannedCheck("date", "、".join(usable), "business_date" if len(dates) == 1 else None, + columns=tuple(usable), date_total=len(dates))) return checks -async def _profile_table(cx: _Context, table: str, notes: dict[str, Any], result: TableProfile) -> _TableOutcome: - out = _TableOutcome() - if cx.tables[table].get("is_view"): - result.skipped.append(ProfileSkip("table", table, "view", "视图没有统计信息，剖析可能触发整个视图的计算，已跳过")) - return out - checks = _plan(cx, table, notes, out, result) - for i, check in enumerate(checks): +def profile_checks(table: str, meta: dict[str, Any] | None, notes: dict[str, Any] | None, *, tables: dict[str, Any], + masked: set[str]) -> int: + """这张表要做几项检查（按规则不查的不算）：0 表示没有可剖析的项；视图、表结构里没有的表是 0。""" + return sum(1 for c in plan_checks(table, meta, notes, tables=tables, masked=masked) if c.skip is None) + + +def has_pending_relations(notes: dict[str, Any] | None) -> bool: + """这张表还有没有待核实的关系：要核对的关系里有还是推断（proposed）的。挑表时这样的表排在前面。""" + return any(r.get("status") == "proposed" for r in _relation_candidates(notes)) + + +def pending_relation_paths(notes: dict[str, Any] | None) -> set[str]: + """待核实（proposed）关系的路径（relations.<编号>）。上次剖析的检查项里没有它们，就是剖析之后冒出了新的待核实关系。""" + return {f"relations.{r['id']}" for r in _relation_candidates(notes) if r.get("status") == "proposed"} + + +# ========================================================================== +# 入口：剖析一张表 +# ========================================================================== + +ESTIMATING = "正在估算 {table} 的行数" + + +def _current_text(table: str, check: PlannedCheck) -> str: + """进行中时正在做哪一项的整句（逐表一行的最后一栏）。""" + if check.kind == "relation": + return f"正在核对 {table}.{check.target}" + if check.kind == "codes": + return f"正在取 {table}.{check.column} 的取值分布" + if len(check.columns) == 1: + return f"正在取 {table}.{check.columns[0]} 的最小值和最大值" + return f"正在取 {table} 中 {check.target} 的最小值和最大值" + + +def _keep(out: _TableOutcome, check: PlannedCheck) -> None: + """这一项没查成：目录里剖析以前对它写的结论标记为原样保留。""" + if check.kind == "relation" and check.relation is not None: + out.keep_relations.add(str(check.relation["id"])) + elif check.kind == "codes" and check.column: + out.keep_codes.add(check.column) + elif check.kind == "date" and check.columns: + out.keep_business_date = True + + +def _close_open(result: TableResult, plan: list[PlannedCheck], out: _TableOutcome, reason: str, detail: str) -> int: + """还没结束的项（排着的、正在做的）都记为跳过、原因 reason，旧结论原样保留。返回记了几项。""" + n = 0 + for check, planned in zip(result.checks, plan): + if check["state"] in ("pending", "running"): + check.update(state="skipped", reason=reason, detail=detail) + result.skipped.append(ProfileSkip(check["kind"], check["target"], reason, detail, check["path"])) + _keep(out, planned) + n += 1 + return n + + +def _stop_detail(cx: _TableCx, reason: str) -> str: + return _STOP_DETAIL[reason].format(max_queries=cx.settings.table_max_queries, max_s=_fmt(cx.settings.table_max_s)) + + +async def _execute(cx: _TableCx, table: str, check: PlannedCheck, out: _TableOutcome) -> None: + if check.kind == "relation": + await _check_relation(cx, table, check.relation or {}, out, cx.result) + elif check.kind == "codes": + await _check_codes(cx, table, str(check.column), out, cx.result) + else: + await _check_dates(cx, table, list(check.columns), check.date_total, out, cx.result) + + +async def _progress(cx: _TableCx) -> None: + if cx.hooks.progress is not None: + await cx.hooks.progress(cx.result) + + +async def _run_checks(cx: _TableCx, table: str, plan: list[PlannedCheck], runnable: list[int], + out: _TableOutcome) -> None: + """第一条查询估行数（确知是空表就整张表跳过），再逐项执行。预算用完、任务止损时剩下的项记原因、保留旧结论。""" + result = cx.result + result.current = ESTIMATING.format(table=table) + await _progress(cx) + try: + size = await cx.size(table, result) + except _Stop as e: + _close_open(result, plan, out, e.reason, _stop_detail(cx, e.reason)) + if e.reason in ("failed", "slow"): + result.error = cx.job.runner.error + return + finally: + result.current = None + if size.rows == 0: + _close_open(result, plan, out, "no_data", f"{table} 是空表，无法剖析") + result.status, result.error = "skipped", EMPTY_ERROR + return + for i in runnable: + planned, check = plan[i], result.checks[i] + check["state"] = "running" + result.current = _current_text(table, planned) + cx.check = check + await _progress(cx) try: - await check.run() + await _execute(cx, table, planned, out) + check["state"] = "done" except _Skip as e: - result.skipped.append(ProfileSkip(check.kind, check.target, e.reason, e.detail, check.path)) + check.update(state="skipped", reason=e.reason, detail=e.detail) + result.skipped.append(ProfileSkip(planned.kind, planned.target, e.reason, e.detail, planned.path)) if e.carry: - check.keep() + _keep(out, planned) except _Stop as e: - detail = _STOP_DETAIL[e.reason].format(max_queries=cx.settings.max_queries, - max_total_s=_fmt(cx.settings.max_total_s)) - for rest in checks[i:]: - result.skipped.append(ProfileSkip(rest.kind, rest.target, e.reason, detail, rest.path)) - rest.keep() + cx.check = None + _close_open(result, plan, out, e.reason, _stop_detail(cx, e.reason)) + if e.reason in ("failed", "slow"): + result.error = cx.job.runner.error break - if table in cx.sizes: - result.row_estimate = cx.sizes[table] - return out + finally: + cx.check = None + result.current = None + await _progress(cx) -async def _write(session: AsyncSession, source_id: str, table: str, out: _TableOutcome, result: TableProfile, *, +async def _own_session(fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: + async with SessionLocal() as session: + return await fn(session, *args) + + +async def _read_notes(session: AsyncSession, source_id: str, table: str) -> dict[str, Any]: + entry = await catalog.read_entry(session, source_id, table) + return entry.notes if entry else {} + + +async def profile_table(cx: ProfileContext, table: str, *, + db: Callable[..., Awaitable[Any]] | None = None, hooks: TableHooks | None = None, + actor: str | None = None) -> TableResult: + """剖析一张表并把结论写进目录，返回这张表的结果（TableResult）。 + + - 表结构里没有、视图：skipped，不发查询。 + - 先列全检查项（plan_checks，读的是这时目录里的最新内容），按规则不查的直接记跳过；一项可查的都没有：skipped + 「没有可剖析的项」，不发查询。 + - 第一条查询估行数，确知是空表：skipped「空表」。 + - 逐项执行：这张表的查询次数、时长用完（TableBudget）只停这张表，剩下的项记 budget / deadline；任务止损（cx.runner） + 停下时剩下的项记 failed / slow，error 是止损的那一句。 + - 停止（任务被 cancel）：正在跑的那条查询由数据库按单条时限截止；已做完的项照常写进目录，没做的记 cancelled、保留旧 + 结论；写完再把停止传下去（CancelledError）。 + - 整张表结束把结论并进目录（乐观锁，撞上别人刚改过重读重并，三次仍冲突记 failed），和 hooks.recorded 在同一次库操作里。 + + db 是执行库操作的：db(fn, *args) 开一个会话调 fn(session, *args)。后台任务给它的 SerialWriter（一条一条、shield）， + 不给就各开一个会话。快照被改动（SnapshotTampered）原样抛出，不写目录：整个任务都该停。 + """ + hooks = hooks or TableHooks() + db = db or _own_session + started = time.monotonic() + result = TableResult(table=table) + meta = cx.tables.get(table) + if not isinstance(meta, dict) or meta.get("is_view"): + reason = "view" if isinstance(meta, dict) else "missing" + why = VIEW_ERROR if reason == "view" else catalog.missing_table_error(table) + result.status, result.error = "skipped", why + result.skipped.append(ProfileSkip("table", table, reason, why)) + await db(_record, cx.source.id, table, None, result, actor, hooks.recorded) + return result + notes = await db(_read_notes, cx.source.id, table) + plan = plan_checks(table, meta, notes, tables=cx.tables, masked=cx.masked) + out = _TableOutcome() + tcx = _TableCx(cx, table, result, hooks) + result.checks = [p.as_check() for p in plan] + for planned in plan: + if planned.skip is not None: + result.skipped.append(ProfileSkip(planned.kind, planned.target, planned.skip[0], planned.skip[1], + planned.path)) + if planned.skip[2]: + _keep(out, planned) + runnable = [i for i, p in enumerate(plan) if p.skip is None] + stopped: asyncio.CancelledError | None = None + try: + if not runnable: + result.status, result.error = "skipped", NOTHING_ERROR + result.skipped.append(ProfileSkip("table", table, "nothing", NOTHING_ERROR)) + else: + await _run_checks(tcx, table, plan, runnable, out) + except asyncio.CancelledError as e: + stopped = e + why = hooks.stop_text() if hooks.stop_text is not None else STOPPED + done = sum(1 for c in result.checks if c["state"] == "done") + rest = _close_open(result, plan, out, "cancelled", f"{why}，未检查") + result.error = STOPPED_TABLE.format(why=why, done=done, rest=rest) + except SnapshotTampered: + raise + except Exception as e: # noqa: BLE001 - 一张表出了意外不拖垮别的表：照实记下，做完的项照常写 + logger.exception("剖析表 %s 时出错", table) + line = _brief(e) + _close_open(result, plan, out, "error", TABLE_ERROR.format(line=line)) + result.status, result.error = "failed", TABLE_ERROR.format(line=line) + result.current = None + result.row_estimate = cx.sizes.get(table) + result.elapsed_ms = int((time.monotonic() - started) * 1000) + await db(_record, cx.source.id, table, out, result, actor, hooks.recorded) + if stopped is not None: + raise stopped + return result + + +def _settle_status(result: TableResult) -> None: + """表的状态：skipped、failed 的不变；有运行原因没查成的项是 partial，否则 ok。""" + if result.status in ("skipped", "failed"): + return + result.status = "partial" if result.unfinished else "ok" + + +async def _record(session: AsyncSession, source_id: str, table: str, out: _TableOutcome | None, result: TableResult, + actor: str | None, recorded: Callable[[AsyncSession, TableResult], Awaitable[None]] | None) -> None: + """整张表结束：把结论并进目录（out 为 None 的不写：表结构里没有、视图），定下表的状态，再交给 recorded（同一个会话）。 + 修改时间取这一次写入的时间。""" + if out is not None: + try: + await _write(session, source_id, table, out, result, at=catalog.now_iso(), actor=actor) + except Exception as e: # noqa: BLE001 - 这一张写不进去不影响别的表 + logger.exception("剖析结果写入表 %s 的数据目录时出错", table) + await session.rollback() + result.error = WRITE_ERROR.format(line=_brief(e)) + result.status = "failed" + result.findings = [] + result.added = result.updated = result.removed = 0 + _settle_status(result) + if recorded is not None: + await recorded(session, result) + + +async def _write(session: AsyncSession, source_id: str, table: str, out: _TableOutcome, result: TableResult, *, at: str, actor: str | None) -> None: - """并入并写入（乐观锁）。撞上别人刚改过就重读重并，最多三次。""" + """并入并写入（乐观锁）。撞上别人刚改过就重读重并，最多三次；三次仍冲突这张表记 failed。""" for _attempt in range(3): entry = await catalog.read_entry(session, source_id, table) version = entry.version if entry else 0 @@ -1219,91 +1719,15 @@ async def _write(session: AsyncSession, source_id: str, table: str, out: _TableO continue result.version = written.version return - result.error = "这张表的数据目录正被其他人修改，剖析结果未能写入，请稍后重新剖析" + result.error = CONFLICT_ERROR + result.status = "failed" result.findings = [] result.added = result.updated = result.removed = 0 -async def _pick_tables(session: AsyncSession, source: Any, tables: list[str] | None, - entries: dict[str, catalog.CatalogEntry], report: ProfileReport) -> list[str]: - cache = getattr(source, "schema_cache", None) or {} - all_tables = cache.get("tables") or {} - if tables: - picked: list[str] = [] - for name in dict.fromkeys(str(t) for t in tables): - key = catalog.resolve_table_name(cache, name) - if key is None: - report.tables.append(TableProfile(table=name, error=f"表结构里没有 {name}，请先重新探查结构")) - elif key not in picked: - picked.append(key) - return picked - usage = await catalog.table_usage(session, source) - order = {name: i for i, name in enumerate(all_tables)} - candidates = [name for name, meta in all_tables.items() - if not (meta or {}).get("is_view") and name in entries - and _relation_candidates(entries[name].notes)] - # 还有待核实（推断）关系的表排在前面：剖析就是来核实它们的；关系都核实过、人工确认过的再剖析只是补覆盖率。 - # 同样待核实的按使用次数，再按表结构顺序。返回全部候选，空表由 _skip_empty 估过行数再跳过 - return sorted(candidates, key=lambda n: (not _pending(entries[n].notes), -usage.get(n, 0), order[n])) - - -def _pending(notes: dict[str, Any] | None) -> bool: - """这张表还有没有待核实的关系：要核对的关系里有还是推断（proposed）的。""" - return any(r.get("status") == "proposed" for r in _relation_candidates(notes)) - - -async def _skip_empty(cx: _Context, candidates: list[str], report: ProfileReport) -> list[TableProfile]: - """不指定表时：按顺序估候选表的行数，确知是空表的跳过（记进 report.empty_tables），凑够 PROFILE_DEFAULT_TABLES - 张为止。以前不估，所有表使用次数都是 0 时按表结构顺序取前 10 张，结果大半是空表、一条关系都核实不了。 - - 估行数的查询算进这张表的查询数，行数留在 cx.sizes 里，剖析时不再查。最多估 PROFILE_PROBE_TABLES 张、最多用掉 - 查询次数上限的一半；超过了、或者整次剖析已经停下，剩下的候选不估、照顺序排上（真是空表的，剖析时各项检查会 - 记下「空表，无法剖析」）。 - """ - picked: list[TableProfile] = [] - probed = 0 - for name in candidates: - if len(picked) >= PROFILE_DEFAULT_TABLES: - break - result = TableProfile(table=name) - if (cx.run.stopped is None and probed < PROFILE_PROBE_TABLES - and cx.run.used < cx.settings.max_queries // 2): - probed += 1 - before = cx.run.used - try: - size = await cx.size(name, result) - except _Stop: - size = None - result.queries = cx.run.used - before - if size is not None and size.rows == 0: - report.empty_tables.append(name) - continue - picked.append(result) - return picked - - -async def profile_catalog(session: AsyncSession, row: Any, source: Any, *, tables: list[str] | None = None, - actor: str | None = None, settings: ProfileSettings | None = None) -> ProfileReport: - """剖析这些表并把结论写进数据目录，同步完成。 - - - row：数据源记录（开关、遮罩按它的 options）；source：用来查询和读表结构的源（上传源是绑定当前快照的视图）。 - - settings 不给就按 row.options 读；没开启抛 ProfileDisabled，这个源正在剖析抛 ProfileBusy。 - - tables 为空时按使用次数取前 PROFILE_DEFAULT_TABLES 张目录里有待核实关系的表。 - - 每张表做完就写入，后面的表停下（预算、总时长）不影响前面已经写进去的。 - """ - settings = ensure_enabled(row, settings) - if source.id in _RUNNING: - raise ProfileBusy(f"数据源「{getattr(row, 'name', '')}」正在进行数据剖析，请等待完成后再试") - _RUNNING.add(source.id) - try: - return await _profile(session, row, source, tables, actor, settings) - finally: - _RUNNING.discard(source.id) - - def ensure_enabled(row: Any, settings: ProfileSettings | None = None) -> ProfileSettings: """这个数据源能不能剖析：数据源停用了、或者没开启数据剖析，抛 ProfileDisabled（message 给人看）。 - 返回要用的设置（不给就按 row.options 读）。接口在做别的检查之前先调它：开关是第一道门。""" + 返回要用的设置（不给就按 row.options 读）。建任务之前先调它：开关是第一道门；任务每张表开始前也按它重新看一遍。""" settings = settings or profile_settings(getattr(row, "options", None)) name = getattr(row, "name", "") or "" if getattr(row, "enabled", True) is False: @@ -1313,38 +1737,6 @@ def ensure_enabled(row: Any, settings: ProfileSettings | None = None) -> Profile return settings -async def _profile(session: AsyncSession, row: Any, source: Any, tables: list[str] | None, actor: str | None, - settings: ProfileSettings) -> ProfileReport: - at = catalog.now_iso() - report = ProfileReport(profiled_at=at, settings=settings) - dialect = SqlDialect(source.kind) - run = _Runner(source, settings, dialect) - cx = _Context(source=source, settings=settings, dialect=dialect, run=run, - tables=(getattr(source, "schema_cache", None) or {}).get("tables") or {}, - masked={c.lower() for c in data_engine.masked_columns(getattr(row, "options", None))}) - entries = await catalog.read_catalog(session, source.id) - picked = await _pick_tables(session, source, tables, entries, report) - results = [TableProfile(table=t) for t in picked] if tables else await _skip_empty(cx, picked, report) - for result in results: - table = result.table - report.tables.append(result) - if run.stopped: - detail = _STOP_DETAIL[run.stopped].format(max_queries=settings.max_queries, - max_total_s=_fmt(settings.max_total_s)) - result.skipped.append(ProfileSkip("table", table, run.stopped, detail)) - continue - before = run.used - entry = entries.get(table) - out = await _profile_table(cx, table, entry.notes if entry else {}, result) - result.queries += run.used - before - await _write(session, source.id, table, out, result, at=at, actor=actor) - report.queries_used = run.used - report.stopped = run.stopped - logger.info("数据剖析 源=%s 表=%s 查询=%d 停止=%s 署名=%s", source.id, - ",".join(t.table for t in report.tables), run.used, run.stopped, actor) - return report - - # ========================================================================== # 核对模型提议的有效行过滤：排除了多少行 # ========================================================================== @@ -1399,7 +1791,7 @@ async def check_valid_filter(source: Any, table: str, meta: dict[str, Any] | Non settings: ProfileSettings | None = None) -> FilterCheck: """开了数据剖析的源：数一数这条有效行过滤排除了多少行（模型起草时提议的那条，data/catalog_draft.py 调）。从不抛异常。 - 经剖析的只读守卫（_Runner：guard.check(readonly=True)、单条时限、查询次数上限）发一条查询：只取条件里用到的列， + 经单条查询的入口（run_single_query：guard.check(readonly=True)、单条时限）发一条查询：只取条件里用到的列， 先取前 max(sample_size, max_scan_rows) 行（大表不做整表扫描），再数总行数和不满足条件的行数（条件为 NULL 也算排除， 和 WHERE 一样）。数到的不满前那么多行，就是整张表。 @@ -1437,14 +1829,10 @@ async def check_valid_filter(source: Any, table: str, meta: dict[str, Any] | Non cols = ", ".join(dialect.quote(c) for c in parsed.columns) inner = dialect.limit(f"SELECT {cols} FROM {dialect.table(meta, table)}", cap) sql = f"SELECT COUNT(*) AS n, SUM(CASE WHEN {cond} THEN 0 ELSE 1 END) AS x FROM ({inner}) s" - run = _Runner(source, settings, dialect) try: - res = await run(sql) - except _Skip as e: + res = await run_single_query(source, sql, settings=settings, kind=kind) + except QueryFailed as e: return _unchecked(e.detail.removesuffix("，已跳过")) - except _Stop as e: - return _unchecked(_STOP_DETAIL[e.reason].format(max_queries=settings.max_queries, - max_total_s=_fmt(settings.max_total_s)).removesuffix("，未检查")) except SnapshotTampered as e: return _unchecked(str(e)) row = res.rows[0] if res.rows else [None, None] @@ -1465,9 +1853,12 @@ async def check_valid_filter(source: Any, table: str, meta: dict[str, Any] | Non return FilterCheck(True, note, rows, excluded, sampled) -__all__ = ["CODES_MAX", "FILTER_CHECKED", "FILTER_NOT_ENABLED", "FILTER_SAMPLED", "FILTER_UNCHECKED", "FilterCheck", - "IN_CHUNK", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES", - "PROFILE_OPTION", "PROFILE_PROBE_TABLES", "ProfileBusy", "ProfileDisabled", "ProfileReport", - "ProfileSettings", "ProfileSkip", "SqlDialect", "TableProfile", "TableSize", "check_valid_filter", - "ensure_enabled", "estimate_size", "excluded_share", "profile_catalog", "profile_settings", - "profile_settings_problem", "profile_settings_rejection", "sql_literal"] + +__all__ = ["CODES_MAX", "EMPTY_ERROR", "FILTER_CHECKED", "FILTER_NOT_ENABLED", "FILTER_SAMPLED", "FILTER_UNCHECKED", + "FilterCheck", "IN_CHUNK", "LEGACY_KEYS", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "NOTHING_ERROR", + "PROFILE_OPTION", "PlannedCheck", "ProfileContext", "ProfileDisabled", "ProfileSettings", "ProfileSkip", + "QueryFailed", "SourceRunner", "SqlDialect", "TableBudget", "TableHooks", "TableResult", "TableSize", + "UNFINISHED_REASONS", "VIEW_ERROR", "check_valid_filter", "ensure_enabled", "estimate_size", "excluded_share", + "has_pending_relations", "make_context", "masked_of", "migrate_settings", "pending_relation_paths", + "plan_checks", "profile_checks", "profile_settings", "profile_settings_problem", "profile_settings_rejection", + "profile_table", "run_single_query", "sql_literal", "summarize", "without_legacy"] diff --git a/backend/app/data/catalog_profile_jobs.py b/backend/app/data/catalog_profile_jobs.py index 6a97c3a7..78b9511e 100644 --- a/backend/app/data/catalog_profile_jobs.py +++ b/backend/app/data/catalog_profile_jobs.py @@ -1,20 +1,107 @@ -"""数据目录的后台剖析任务（草稿，下一步补全）。""" +"""数据目录的后台剖析任务（/api/datasources/{id}/catalog/profile-jobs）。 + +以前剖析是一次同步长请求（POST /catalog/profile，已去掉）：预算按整次剖析算，选 50 张表只做得完前两三张；进行中只有一条 +按时长走的横条、不能停；关掉页面报告就没了，对业务库查了什么无从查。现在和起草一样是后台任务（生命周期在 +data/catalog_job_runtime.py），逐表剖析（引擎在 data/catalog_profile.py）： + +- **建任务**（create_job）：范围三种——指定的表（selected：按给的顺序去重；表结构里没有、视图、没有可剖析的项照样建进 + 任务、状态 skipped）、需要剖析的表（needed：从没剖析过、上次剖析没做完、剖析之后冒出了新的待核实关系，见 due_reason）、 + 全部表（all）。后两种只挑表结构里、不是视图、有可剖析的项的表，有待核实关系的在前，再按使用次数、表结构顺序；挑出 0 张 + 抛 ProfileNothing。剖析设置随任务记下快照，每张表的检查项按当时的目录先列一遍。同一个源同时只有一个排队或进行中的 + 剖析任务（ProfileBusy，说法里写明进行到第几张；部分唯一索引兜底）；和同源的起草任务按建立的先后排队（每个源一条队）， + 起草在前时 waiting_for 是 "draft"。建任务时顺手清掉超出保留数的旧任务（prune）。 +- **在后台剖析**（manager.start）：轮到了才开始。每张表开始前重读数据源——停用了、关掉了剖析就停下，剩下的表 cancelled、 + error 写明；遮罩按最新的设置——再逐表调 catalog_profile.profile_table（它读的是那时目录里的最新内容：起草刚提的关系也 + 核对）。检查项、正在做哪一项、查询数、发过的查询随时落库；整张表的结果和写目录在同一次库操作里，停止时也不会被打断。 +- **任务级止损**（替代以前的整次总量上限）：连续 3 条查询失败、最近 10 条里 7 条超时，整个任务停下（stop_reason、error）， + 正在做的那张表写入已做完的项，后面的表 cancelled、error 带同一句原因；最近 6 条里 4 条超时时 notice 有一句提醒，好转后 + 清掉。任务最后的状态按逐表算：有 failed、partial、cancelled 的表是 partial，否则 succeeded；任务整体出错（数据源没了、 + 快照被改动、意外异常）是 failed。 +- **停止**（cancel）：排队的直接 cancelled；进行中的停掉正在等的那条查询（库那边按单条时限截止），那张表做完的项照常写进 + 目录、没做的保留旧结论，后面的表 cancelled，任务 cancelled。 +- **重试**（retry_tables + create_job）：新任务，只含原任务里 failed、partial、cancelled 的表（按原来的顺序），范围 selected， + retry_of 记原任务；设置取数据源当前的。 +- **服务重启**（startup）：排队、进行中的标 interrupted，没结束的表 cancelled、error「服务重启，剖析中断」。不自动续跑： + 复制出来的数据目录启动时不能自己去查业务库，界面让人点「重试」。关停时（manager.shutdown）停掉进行中的，同样标中断。 + 启动时还一次性迁移剖析设置的旧键（catalog_profile.migrate_settings）。 +- **保留**（prune）：每个源保留最近 KEEP_JOBS 个结束了的剖析任务，更早的删掉；某张表最近一次剖析所在的任务不删，表详情的 + 「上次剖析」一直在。 +- **读**：job_dict（任务和逐表一行）、task_detail（一张表的检查项、发现、跳过的项、发过的查询）、last_profiles（表清单每行 + 的「上次剖析」）、due_reason（这张表要不要剖析、为什么）。 + +没结束的表收尾时（停止、中断、整体出错）检查项里还排着的、正在做的记为跳过（原因 cancelled 或 error），不留「排队中」的 +检查项在一张已经结束的表上。 +""" from __future__ import annotations +import logging +import time from typing import Any -from sqlalchemy import func, select +from sqlalchemy import delete, func, select, update +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession -from app.data.catalog_job_runtime import ACTIVE, TASK_DONE -from app.db.models import CatalogProfileJob, CatalogProfileTask +from app.data import catalog, catalog_profile +from app.data import catalog_job_runtime as runtime +from app.data.catalog_job_runtime import ACTIVE, FINISHED, OPEN, RETRYABLE, STOP_GRACE_S, TASK_DONE, TASK_STATUSES +from app.data.catalog_profile import ( + NOTHING_ERROR, UNFINISHED_REASONS, VIEW_ERROR, ProfileSettings, ProfileSkip, TableHooks, TableResult, +) +from app.data.engine import SnapshotTampered +from app.db.base import SessionLocal, new_id, utcnow +from app.db.models import CatalogDraftJob, CatalogProfileJob, CatalogProfileTask, DataSource + +logger = logging.getLogger(__name__) + +#: 一个任务最多几张表 +MAX_TABLES = 500 +#: 每个源保留几个结束了的剖析任务（某张表最近一次剖析所在的任务另外保留） +KEEP_JOBS = 50 + +INTERRUPTED_ERROR = "服务重启，剖析中断" +STOPPED_ERROR = catalog_profile.STOPPED +UNFINISHED_ERROR = "剖析结束时这张表还没有收尾，请重试这张表" +JOB_ERROR = "剖析任务出错：{line}" +SOURCE_GONE = "数据源不存在，可能已被删除" +SOURCE_DISABLED = "数据源「{name}」已停用，剖析已停止" +PROFILE_TURNED_OFF = "数据源「{name}」已关闭数据剖析，剖析已停止" +#: 没结束的检查项在收尾时的说明（{why} 是停止、中断、出错的原因） +_UNCHECKED = "{why}，未检查" +_CHECK_OPEN = ("pending", "running") class ProfileBusy(Exception): - """这个数据源已经有排队或进行中的剖析任务。message 给人看。""" + """这个数据源已经有排队或进行中的剖析任务。message 给人看（写明进行到第几张）。""" + + +class ProfileFinished(Exception): + """要停止的剖析任务已经结束了。""" + + +class ProfileNothing(Exception): + """按范围挑出 0 张表。message 说明原因、建议怎么办。""" + + +class NothingToRetry(Exception): + """原任务里没有需要重试的表。""" + + +# ========================================================================== +# 读任务 +# ========================================================================== + + +def _iso(value: Any) -> str | None: + return value.isoformat() if value is not None else None + + +def _empty_summary() -> dict[str, int]: + return catalog_profile.summarize([]) async def active_job(session: AsyncSession, source_id: str) -> CatalogProfileJob | None: + """排队或进行中的那个（同一个源最多一个）。""" return (await session.execute( select(CatalogProfileJob).where(CatalogProfileJob.source_id == source_id, CatalogProfileJob.status.in_(ACTIVE)) @@ -22,16 +109,715 @@ async def active_job(session: AsyncSession, source_id: str) -> CatalogProfileJob )).scalar_one_or_none() -def busy_message(row: Any, progress: tuple[int, int] | None) -> str: +async def latest_job(session: AsyncSession, source_id: str) -> CatalogProfileJob | None: + """最近建的一个（任何状态）。""" + return (await session.execute( + select(CatalogProfileJob).where(CatalogProfileJob.source_id == source_id) + .order_by(CatalogProfileJob.created_at.desc()).limit(1) + )).scalar_one_or_none() + + +async def load(session: AsyncSession, source_id: str, job_id: str) -> CatalogProfileJob | None: + """这个源的这个任务；不存在、不属于这个源都是 None。""" + job = await session.get(CatalogProfileJob, job_id) + return job if job is not None and job.source_id == source_id else None + + +#: 逐表一行要的列：不读发现、跳过的项、发过的查询（任务进行中界面每秒都要取一次） +_ROW_COLUMNS = (CatalogProfileTask.seq, CatalogProfileTask.table_name, CatalogProfileTask.status, + CatalogProfileTask.checks, CatalogProfileTask.current, CatalogProfileTask.queries, + CatalogProfileTask.elapsed_ms, CatalogProfileTask.row_estimate, CatalogProfileTask.summary, + CatalogProfileTask.skipped_count, CatalogProfileTask.unfinished, CatalogProfileTask.added, + CatalogProfileTask.updated, CatalogProfileTask.removed, CatalogProfileTask.version, + CatalogProfileTask.error, CatalogProfileTask.started_at, CatalogProfileTask.finished_at) + + +async def task_rows(session: AsyncSession, job_id: str) -> list[Any]: + return list((await session.execute( + select(*_ROW_COLUMNS).where(CatalogProfileTask.job_id == job_id).order_by(CatalogProfileTask.seq) + )).all()) + + +def _by_kind(checks: list[Any]) -> dict[str, list[int]]: + """检查项按种类的进度：{relation: [结束了的, 总数], codes: […], date: […]}。结束了的含跳过的。""" + out = {"relation": [0, 0], "codes": [0, 0], "date": [0, 0]} + for check in checks: + if not isinstance(check, dict) or check.get("kind") not in out: + continue + pair = out[check["kind"]] + pair[1] += 1 + if check.get("state") in ("done", "skipped"): + pair[0] += 1 + return out + + +def task_row(task: Any) -> dict[str, Any]: + """逐表一行（CatalogProfileTaskRow）。task 是 CatalogProfileTask，或者 task_rows 取的那几列。""" + checks = [c for c in (task.checks or []) if isinstance(c, dict)] + return { + "seq": task.seq, + "table_name": task.table_name, + "status": task.status, + "checks_total": len(checks), + "checks_done": sum(1 for c in checks if c.get("state") in ("done", "skipped")), + "checks_by_kind": _by_kind(checks), + "current": task.current if task.status == "running" else None, + "queries": task.queries or 0, + "elapsed_ms": task.elapsed_ms or 0, + "row_estimate": task.row_estimate, + "summary": {**_empty_summary(), **(task.summary or {})}, + "skipped_count": task.skipped_count or 0, + "unfinished": task.unfinished or 0, + "added": task.added or 0, + "updated": task.updated or 0, + "removed": task.removed or 0, + "version": task.version or 0, + "error": task.error, + "started_at": _iso(task.started_at), + "finished_at": _iso(task.finished_at), + } + + +def _counts(statuses: list[str]) -> dict[str, int]: + counts = dict.fromkeys(TASK_STATUSES, 0) + for status in statuses: + counts[status] = counts.get(status, 0) + 1 + return counts + + +def job_dict(job: CatalogProfileJob, tasks: list[Any], *, waiting_for: str | None = None) -> dict[str, Any]: + """接口返回的任务（CatalogProfileJob）：逐表按剖析顺序；计数、合计都按逐表的算。""" + rows = [task_row(t) for t in tasks] + counts = _counts([r["status"] for r in rows]) + summary = _empty_summary() + for r in rows: + for key in summary: + summary[key] += int(r["summary"].get(key) or 0) + return { + "id": job.id, + "source_id": job.source_id, + "status": job.status, + "scope": job.scope, + "settings": ProfileSettings.from_dict(job.settings).to_dict(), + "created_at": _iso(job.created_at), + "started_at": _iso(job.started_at), + "finished_at": _iso(job.finished_at), + "created_by": job.created_by, + "retry_of": job.retry_of, + "waiting_for": waiting_for, + "total": len(rows), + "done": sum(counts.get(s, 0) for s in TASK_DONE), + "counts": counts, + "queries": job.queries or 0, + "timeouts": job.timeouts or 0, + "errors": job.errors or 0, + "stop_reason": job.stop_reason, + "notice": job.notice, + "error": job.error, + "added": sum(r["added"] for r in rows), + "updated": sum(r["updated"] for r in rows), + "removed": sum(r["removed"] for r in rows), + "summary": summary, + "tasks": rows, + } + + +async def waiting_for(session: AsyncSession, job: CatalogProfileJob) -> str | None: + """排队中的剖析任务在等什么：排在同源的起草任务（或同步起草）后面时是 "draft"，否则 None。 + + 先看这个源的队（进程内，排在它前面的是谁）；任务刚建、后台还没排进队时看库：同源有比它先建的、排队或进行中的起草 + 任务，或者正在同步起草。""" + if job.status != "queued": + return None + ahead = runtime.lanes.ahead(job.source_id, job.id) + if ahead is not None: + return "draft" if any(kind in ("draft", "sync_draft") for kind in ahead) else None + from app.data import catalog_jobs + + if catalog_jobs.sync_drafting(job.source_id): + return "draft" + earlier = (await session.execute( + select(CatalogDraftJob.id).where(CatalogDraftJob.source_id == job.source_id, + CatalogDraftJob.status.in_(ACTIVE), + CatalogDraftJob.created_at <= job.created_at).limit(1) + )).first() + return "draft" if earlier is not None else None + + +async def job_out(session: AsyncSession, job: CatalogProfileJob | None) -> dict[str, Any] | None: + if job is None: + return None + return job_dict(job, await task_rows(session, job.id), waiting_for=await waiting_for(session, job)) + + +async def job_brief(session: AsyncSession, job: CatalogProfileJob | None) -> dict[str, Any] | None: + """目录清单顶上的进度（CatalogActiveProfileJob）：{id, status, total, done, waiting_for}。""" + if job is None: + return None + rows = (await session.execute( + select(CatalogProfileTask.status, func.count()).where(CatalogProfileTask.job_id == job.id) + .group_by(CatalogProfileTask.status) + )).all() + counts = {status: n for status, n in rows} + return {"id": job.id, "status": job.status, "total": sum(counts.values()), + "done": sum(counts.get(s, 0) for s in TASK_DONE), "waiting_for": await waiting_for(session, job)} + + +def task_detail(job: CatalogProfileJob, task: CatalogProfileTask) -> dict[str, Any]: + """一张表的明细（CatalogProfileTaskDetail）：逐表一行，加上检查项、发现、跳过的项、日期范围、发过的查询。""" + return { + **task_row(task), + "job_id": job.id, + "job_status": job.status, + "created_by": job.created_by, + "checks": [c for c in task.checks or [] if isinstance(c, dict)], + "findings": list(task.findings or []), + "skipped": list(task.skipped or []), + "date_ranges": list(task.date_ranges or []), + "query_log": list(task.query_log or []), + } + + +async def load_task(session: AsyncSession, job: CatalogProfileJob, seq: int) -> CatalogProfileTask | None: + return (await session.execute( + select(CatalogProfileTask).where(CatalogProfileTask.job_id == job.id, CatalogProfileTask.seq == seq) + )).scalar_one_or_none() + + +def _latest_per_table(source_id: str, *columns: Any) -> Any: + """每张表最近一次剖析的那一项（最近建的任务里的；同一个任务里同名的取 seq 小的）：子查询，带 rank 列。""" + rank = func.row_number().over(partition_by=CatalogProfileTask.table_name, + order_by=(CatalogProfileJob.created_at.desc(), CatalogProfileTask.seq)).label("rank") + return (select(*columns, rank).join(CatalogProfileJob, CatalogProfileJob.id == CatalogProfileTask.job_id) + .where(CatalogProfileJob.source_id == source_id)).subquery() + + +async def last_profiles(session: AsyncSession, source_id: str) -> dict[str, dict[str, Any]]: + """每张表最近一次剖析里的那一项：{表名: CatalogLastProfile 加上 paths（检查项的路径，判断有没有新的待核实关系用）}。 + 任务还在进行，就是进行中的状态。每张表取哪一项在库里挑好。""" + inner = _latest_per_table(source_id, CatalogProfileTask.table_name, CatalogProfileTask.job_id, + CatalogProfileTask.seq, CatalogProfileTask.status, CatalogProfileTask.started_at, + CatalogProfileTask.finished_at, CatalogProfileTask.summary, + CatalogProfileTask.skipped_count, CatalogProfileTask.unfinished, + CatalogProfileTask.error, CatalogProfileTask.row_estimate, CatalogProfileTask.checks) + rows = (await session.execute(select(inner).where(inner.c.rank == 1))).mappings() + out: dict[str, dict[str, Any]] = {} + for row in rows: + out[row["table_name"]] = { + "job_id": row["job_id"], "seq": row["seq"], "status": row["status"], + "started_at": _iso(row["started_at"]), "finished_at": _iso(row["finished_at"]), + "summary": {**_empty_summary(), **(row["summary"] or {})}, "skipped_count": row["skipped_count"] or 0, + "unfinished": row["unfinished"] or 0, "error": row["error"], "row_estimate": row["row_estimate"], + "paths": {c.get("path") for c in row["checks"] or [] if isinstance(c, dict) and c.get("path")}, + } + return out + + +def public_last(last: dict[str, Any] | None) -> dict[str, Any] | None: + """接口里的 CatalogLastProfile：去掉内部用的 paths。""" + return None if last is None else {k: v for k, v in last.items() if k != "paths"} + + +async def last_detail(session: AsyncSession, source_id: str, table: str) -> dict[str, Any] | None: + """这张表最近一次剖析的明细（表详情的「数据剖析」一节）；没剖析过是 None。""" + found = (await session.execute( + select(CatalogProfileTask, CatalogProfileJob) + .join(CatalogProfileJob, CatalogProfileJob.id == CatalogProfileTask.job_id) + .where(CatalogProfileJob.source_id == source_id, CatalogProfileTask.table_name == table) + .order_by(CatalogProfileJob.created_at.desc(), CatalogProfileTask.seq).limit(1) + )).first() + return None if found is None else task_detail(found[1], found[0]) + + +def due_reason(meta: dict[str, Any] | None, notes: dict[str, Any] | None, last: dict[str, Any] | None, + checks: int) -> str | None: + """这张表要不要剖析、为什么：never 从没剖析过；unfinished 最近一次剖析是 partial / failed / cancelled；new_relations + 目录里有待核实的关系不在最近一次剖析的检查项里。表结构里没有、视图、没有可剖析的项（checks 为 0）、正在剖析的不要。""" + if not isinstance(meta, dict) or meta.get("is_view") or checks <= 0: + return None + if last is None: + return "never" + if last["status"] in OPEN: + return None + if last["status"] in RETRYABLE: + return "unfinished" + if catalog_profile.pending_relation_paths(notes) - set(last.get("paths") or ()): + return "new_relations" + return None + + +# ========================================================================== +# 建任务、重试、停止 +# ========================================================================== + + +def busy_message(row: Any, brief: dict[str, Any] | None, *, table: str | None = None, + position: int | None = None) -> str: + """同源已有剖析任务时的说法：写明进行到第几张；从表详情点「剖析这张表」时再说这张表在不在那个任务里、排第几。""" name = getattr(row, "name", "") or "" - return f"数据源「{name}」正在剖析，请等这次剖析结束（或先停止它）再开始新的剖析" + if brief is None: + text = f"数据源「{name}」正在剖析" + elif brief["status"] == "queued": + text = f"数据源「{name}」已有一个排队中的剖析任务（共 {brief['total']} 张表）" + else: + text = f"数据源「{name}」正在剖析：第 {min(brief['done'] + 1, brief['total'])} / {brief['total']} 张表" + if table is not None: + text += f"，{table} 在这次剖析里，排第 {position} 张" if position else f"，{table} 不在这次剖析里" + return text + "。请等这次剖析结束（或先停止它）再开始新的剖析" -async def ensure_free(session: AsyncSession, row: Any) -> None: +async def ensure_free(session: AsyncSession, row: Any, tables: list[str] | None = None) -> None: + """这个源没有排队或进行中的剖析任务；有就抛 ProfileBusy。只指定了一张表时，说法里带上这张表在不在那个任务里。""" job = await active_job(session, row.id) - if job is not None: - rows = (await session.execute( - select(CatalogProfileTask.status, func.count()).where(CatalogProfileTask.job_id == job.id) - .group_by(CatalogProfileTask.status))).all() - counts = dict(rows) - raise ProfileBusy(busy_message(row, (sum(counts.get(s, 0) for s in TASK_DONE), sum(counts.values())))) + if job is None: + return + brief = await job_brief(session, job) + table = position = None + if tables and len(tables) == 1: + key = catalog.resolve_table_name(getattr(row, "schema_cache", None), str(tables[0])) or str(tables[0]) + table = key + names = [r.table_name for r in await task_rows(session, job.id)] + position = names.index(key) + 1 if key in names else None + raise ProfileBusy(busy_message(row, brief, table=table, position=position)) + + +#: 挑出 0 张表时的说法 +NOTHING_NEEDED = ("没有需要剖析的表：能剖析的表都剖析过了，上次剖析也都做完了，之后没有新的待核实关系。" + "要重新剖析，请选「全部表」或指定要剖析的表") +NOTHING_AT_ALL = ("没有可以剖析的表：表结构里的表都是视图，或者没有可剖析的项（要核对的关联关系、像码值的列、日期列都没有，" + "或者都被遮罩了）。请先起草数据目录，或指定要剖析的表") + + +def _skipped_task(job_id: str, seq: int, name: str, reason: str, why: str, checks: list[dict[str, Any]], + skipped: list[ProfileSkip], now: Any) -> CatalogProfileTask: + entries = [s.to_dict() for s in skipped] + [ProfileSkip("table", name, reason, why).to_dict()] + return CatalogProfileTask(job_id=job_id, seq=seq, table_name=name, status="skipped", checks=checks or None, + skipped=entries, skipped_count=len(entries), + unfinished=sum(1 for e in entries if e["reason"] in UNFINISHED_REASONS), + summary=_empty_summary(), error=why, finished_at=now) + + +async def _pick(session: AsyncSession, row: Any, source: Any, *, tables: list[str] | None, + scope: str) -> list[tuple[str, str | None, str | None, list[Any]]]: + """要剖析的表：[(表名, 跳过的原因 / None, 整句, 检查项)]，按剖析顺序。""" + cache = getattr(source, "schema_cache", None) or {} + all_tables = cache.get("tables") or {} + masked = catalog_profile.masked_of(getattr(row, "options", None)) + entries = await catalog.read_catalog(session, source.id) + + def notes(name: str) -> dict[str, Any]: + entry = entries.get(name) + return entry.notes if entry else {} + + def plan(name: str) -> list[Any]: + return catalog_profile.plan_checks(name, all_tables.get(name), notes(name), tables=all_tables, masked=masked) + + if scope == "selected": + picked: list[tuple[str, str | None, str | None, list[Any]]] = [] + seen: set[str] = set() + for name in dict.fromkeys(str(t) for t in tables or []): + key = catalog.resolve_table_name(cache, name) + if key is None: + picked.append((name, "missing", catalog.missing_table_error(name), [])) + continue + if key in seen: + continue + seen.add(key) + if (all_tables.get(key) or {}).get("is_view"): + picked.append((key, "view", VIEW_ERROR, [])) + continue + checks = plan(key) + if not any(c.skip is None for c in checks): + picked.append((key, "nothing", NOTHING_ERROR, checks)) + else: + picked.append((key, None, None, checks)) + return picked + usage = await catalog.table_usage(session, source) + last = await last_profiles(session, source.id) if scope == "needed" else {} + order = {name: i for i, name in enumerate(all_tables)} + candidates: list[tuple[str, list[Any]]] = [] + for name, meta in all_tables.items(): + if not isinstance(meta, dict) or meta.get("is_view"): + continue + checks = plan(name) + runnable = sum(1 for c in checks if c.skip is None) + if runnable == 0: + continue + if scope == "needed" and due_reason(meta, notes(name), last.get(name), runnable) is None: + continue + candidates.append((name, checks)) + if not candidates: + raise ProfileNothing(NOTHING_NEEDED if scope == "needed" else NOTHING_AT_ALL) + # 还有待核实（推断）关系的表排在前面：剖析首先是来核实它们的；再按使用次数、表结构顺序 + candidates.sort(key=lambda c: (not catalog_profile.has_pending_relations(notes(c[0])), -usage.get(c[0], 0), + order[c[0]])) + return [(name, None, None, checks) for name, checks in candidates] + + +async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: list[str] | None, scope: str, + settings: ProfileSettings, actor: str | None, + retry_of: str | None = None) -> tuple[CatalogProfileJob, list[CatalogProfileTask]]: + """建一个排队的剖析任务（还没开始，调用方接着 manager.start）。row 是数据源记录（遮罩按它的设置），source 是读表结构 + 用的源（上传源是绑定当前快照的视图）。同源已有排队或进行中的剖析任务抛 ProfileBusy；挑出 0 张表抛 ProfileNothing。 + 同源有起草任务时照建，排在它后面。建完顺手清掉超出保留数的旧任务。""" + await ensure_free(session, row, tables) + picked = await _pick(session, row, source, tables=tables, scope=scope) + job = CatalogProfileJob(id=new_id(), source_id=row.id, status="queued", scope=scope, + settings=settings.to_dict(), created_by=actor, retry_of=retry_of) + now = utcnow() + tasks: list[CatalogProfileTask] = [] + for seq, (name, reason, why, checks) in enumerate(picked): + shown = [c.as_check() for c in checks] + if reason is not None: + rule = [ProfileSkip(c.kind, c.target, c.skip[0], c.skip[1], c.path) for c in checks if c.skip] + tasks.append(_skipped_task(job.id, seq, name, reason, str(why), shown, rule, now)) + continue + tasks.append(CatalogProfileTask(job_id=job.id, seq=seq, table_name=name, status="pending", checks=shown, + summary=_empty_summary())) + session.add(job) + try: + await session.flush() + session.add_all(tasks) + await session.commit() + except IntegrityError as e: + # 部分唯一索引：另一个请求同时给这个源建了剖析任务 + await session.rollback() + raise ProfileBusy(busy_message(row, None)) from e + try: + await prune(session, row.id) + except Exception: # noqa: BLE001 - 清旧任务失败不影响这次剖析 + logger.exception("清理数据源 %s 的旧剖析任务时出错", row.id) + await session.rollback() + return job, tasks + + +async def retry_tables(session: AsyncSession, job: CatalogProfileJob) -> list[str]: + """重试要带上的表：原任务里 failed、partial、cancelled 的，按原来的顺序。没有抛 NothingToRetry。""" + names = [t.table_name for t in await task_rows(session, job.id) if t.status in RETRYABLE] + if not names: + raise NothingToRetry("这次剖析里没有失败、部分完成或停止了的表，不需要重试") + return names + + +async def prune(session: AsyncSession, source_id: str, *, keep: int | None = None) -> int: + """保留：这个源最近 keep（缺省 KEEP_JOBS）个结束了的剖析任务留着，更早的删掉；某张表最近一次剖析所在的任务不删。 + 返回删了几个。""" + keep = KEEP_JOBS if keep is None else keep + finished = list((await session.execute( + select(CatalogProfileJob.id).where(CatalogProfileJob.source_id == source_id, + CatalogProfileJob.status.in_(FINISHED)) + .order_by(CatalogProfileJob.created_at.desc(), CatalogProfileJob.id.desc()) + )).scalars()) + old = finished[keep:] + if not old: + return 0 + inner = _latest_per_table(source_id, CatalogProfileTask.job_id) + protected = set((await session.execute(select(inner.c.job_id).where(inner.c.rank == 1))).scalars()) + victims = [job_id for job_id in old if job_id not in protected] + if not victims: + return 0 + await session.execute(delete(CatalogProfileTask).where(CatalogProfileTask.job_id.in_(victims))) + await session.execute(delete(CatalogProfileJob).where(CatalogProfileJob.id.in_(victims))) + await session.commit() + return len(victims) + + +async def _close_tasks(session: AsyncSession, job_id: str, task_status: str, task_error: str | None, + *, why: str) -> None: + """还没结束的表记 task_status（原因 task_error）；它们的检查项里还排着的、正在做的记为跳过（停止、中断是 cancelled， + 出错是 error，说明是「{why}，未检查」），计进跳过的项和没做完的项。""" + now = utcnow() + reason = "cancelled" if task_status == "cancelled" else "error" + detail = _UNCHECKED.format(why=why) + tasks = (await session.execute( + select(CatalogProfileTask).where(CatalogProfileTask.job_id == job_id, CatalogProfileTask.status.in_(OPEN)) + )).scalars() + for task in tasks: + checks = [dict(c) for c in task.checks or [] if isinstance(c, dict)] + skipped = list(task.skipped or []) + for check in checks: + if check.get("state") in _CHECK_OPEN: + check.update(state="skipped", reason=reason, detail=detail) + skipped.append(ProfileSkip(check.get("kind") or "", check.get("target") or "", reason, detail, + check.get("path")).to_dict()) + task.status, task.error, task.finished_at, task.current = task_status, task_error, now, None + task.checks, task.skipped = checks or None, skipped or None + task.skipped_count = len(skipped) + task.unfinished = sum(1 for s in skipped if s.get("reason") in UNFINISHED_REASONS) + + +async def _close(session: AsyncSession, job_id: str, job_status: str, job_error: str | None, task_status: str, + task_error: str | None, *, why: str | None = None) -> None: + """收尾：还没结束的表记 task_status（见 _close_tasks），任务（还没结束的话）记 job_status。提交。""" + await _close_tasks(session, job_id, task_status, task_error, why=why or task_error or STOPPED_ERROR) + await session.execute( + update(CatalogProfileJob).where(CatalogProfileJob.id == job_id, CatalogProfileJob.status.in_(ACTIVE)) + .values(status=job_status, error=job_error, finished_at=utcnow()).execution_options(synchronize_session=False)) + await session.commit() + + +async def cancel(session: AsyncSession, job: CatalogProfileJob) -> None: + """停止一个剖析任务。排队的直接 cancelled（在队里等着的后台执行跟着停掉、出队）；进行中的交给 manager 停：正在等的那条 + 查询不再等（库那边按单条时限截止），那张表做完的项照常写进目录，后面的表 cancelled。已经结束的抛 ProfileFinished。""" + if job.status in FINISHED: + raise ProfileFinished("这次剖析已经结束了，不需要停止") + if job.status == "queued": + claimed = await session.execute( + update(CatalogProfileJob).where(CatalogProfileJob.id == job.id, CatalogProfileJob.status == "queued") + .values(status="cancelled", finished_at=utcnow()).execution_options(synchronize_session=False)) + if claimed.rowcount == 1: + await _close_tasks(session, job.id, "cancelled", None, why=STOPPED_ERROR) + await session.commit() + await manager.stop(job.id, "user", grace=STOP_GRACE_S) + return + await session.rollback() # 刚被后台认领、开始了：按进行中的停 + if await manager.stop(job.id, "user", grace=STOP_GRACE_S): + return + # 库里说在进行，这个进程里却没有在跑它（记录对不上）：直接记停止 + await _close(session, job.id, "cancelled", None, "cancelled", None) + + +async def startup() -> int: + """服务启动时：排队、进行中的剖析任务标成 interrupted，没结束的表记 cancelled、写明服务重启，不自动续跑；再把剖析设置的 + 旧键迁移掉。返回标了几个任务。""" + async def interrupt(session: AsyncSession, job_id: str) -> None: + await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) + + n = await runtime.interrupt_on_startup(CatalogProfileJob, interrupt, running=manager.running) + if n: + logger.warning("服务重启：%d 个数据剖析任务没有做完，已标为中断，可在数据目录页重试", n) + async with SessionLocal() as session: + migrated = await catalog_profile.migrate_settings(session) + if migrated: + logger.info("数据剖析设置：%d 个数据源去掉了按整次剖析算的旧预算", migrated) + return n + + +# ========================================================================== +# 在后台剖析 +# ========================================================================== + + +class _ProfileRun(runtime.JobRun): + """一个剖析任务在后台的一次执行（进队、停止、收尾的框架在 runtime.JobRun）。""" + + kind = "profile" + model = CatalogProfileJob + + def __init__(self, job_id: str) -> None: + super().__init__(job_id) + self.actor: str | None = None + self.cx: catalog_profile.ProfileContext | None = None + #: 正在剖析的那张表：seq 和开始的时刻（进度回调据此落库） + self._seq = -1 + self._table_started = 0.0 + + # ---------------------------------------------------------------- 停止、出错时收尾 + + async def _stopped(self) -> None: + if self.stop_reason == "user": + await self._end("cancelled", None, "cancelled", None) + else: + await self._end("interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) + + async def _failed(self, reason: str) -> None: + await self._end("failed", reason, "failed", reason) + + async def _crashed(self, e: Exception) -> None: + from app.core.errors import describe_exception, first_line + + logger.exception("数据剖析任务 %s 出错", self.job_id) + reason = JOB_ERROR.format(line=first_line(e) or describe_exception(e)) + await self._end("failed", reason, "failed", reason) + + async def _end(self, job_status: str, job_error: str | None, task_status: str, task_error: str | None) -> None: + """停止、出错时收尾：等手上的库写入写完，记下查询计数，再把没结束的表和任务记下来。收尾本身写不进去只记日志（下次 + 启动时 startup 会把它标成中断）。""" + try: + await self._settle() + await self._write(self._save_counters) + await self._write(_close, self.job_id, job_status, job_error, task_status, task_error) + except Exception: # noqa: BLE001 + logger.exception("数据剖析任务 %s 收尾时写库出错", self.job_id) + + def stop_text(self) -> str: + return STOPPED_ERROR if self.stop_reason == "user" else INTERRUPTED_ERROR + + # ---------------------------------------------------------------- 主流程 + + async def _run(self) -> None: + if not await self._write(self._claim): + return # 开始之前就被停止了 + async with SessionLocal() as session: + job = await session.get(CatalogProfileJob, self.job_id) + row = await session.get(DataSource, job.source_id) if job is not None else None + if job is None or row is None: + raise runtime.JobFailed(SOURCE_GONE) + from app.data import table_versions + + try: + source = await table_versions.resolve_source(session, row) + except table_versions.SnapshotMissing as e: + raise runtime.JobFailed(str(e)) from e + pending = [(t.seq, t.table_name) for t in await task_rows(session, job.id) if t.status == "pending"] + settings = ProfileSettings.from_dict(job.settings) + self.actor = job.created_by + options = dict(row.options or {}) + try: + self.cx = catalog_profile.make_context(source, settings, options=options) + except ValueError as e: # 不支持的数据库类型 + raise runtime.JobFailed(str(e)) from e + hooks = TableHooks(progress=self._progress, query=self._queried, stop_text=self.stop_text) + for seq, name in pending: + # 每张表开始前重读数据源：停用了、关掉了剖析就停下；遮罩按最新的设置 + stop = await self._write(self._recheck) + if stop is not None: + await self._write(self._halt, stop, None) + break + self._table_started = time.monotonic() + await self._write(self._start_table, seq) + + async def recorded(session: AsyncSession, result: TableResult, seq: int = seq) -> None: + await self._save_table(session, seq, result) + + hooks.recorded = recorded + self._seq = seq + try: + await catalog_profile.profile_table(self.cx, name, db=self._write, hooks=hooks, actor=self.actor) + except SnapshotTampered as e: + raise runtime.JobFailed(str(e)) from e + if self.cx.runner.stopped: + # 任务级止损：后面的表 cancelled，带同一句原因 + await self._write(self._halt, self.cx.runner.error, self.cx.runner.stopped) + break + await self._settle() + await self._write(self._finish) + + # ---------------------------------------------------------------- 进度回调（引擎调） + + async def _progress(self, result: TableResult) -> None: + await self._write(self._save_progress, self._seq, result) + + async def _queried(self, result: TableResult) -> None: + await self._write(self._save_queries, self._seq, result) + + # ---------------------------------------------------------------- 库操作 + + def _task(self, *where: Any) -> Any: + return update(CatalogProfileTask).where(CatalogProfileTask.job_id == self.job_id, *where) \ + .execution_options(synchronize_session=False) + + def _counters(self) -> dict[str, Any]: + runner = self.cx.runner if self.cx is not None else None + if runner is None: + return {} + return {"queries": runner.queries, "timeouts": runner.timeouts, "errors": runner.errors, + "notice": runner.notice} + + async def _save_counters(self, session: AsyncSession) -> None: + if values := self._counters(): + await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id).values(**values) + .execution_options(synchronize_session=False)) + await session.commit() + + async def _claim(self, session: AsyncSession) -> bool: + return await runtime.claim(session, CatalogProfileJob, self.job_id) + + async def _recheck(self, session: AsyncSession) -> str | None: + """重读数据源：停用了、关掉了剖析返回停下的整句；遮罩换成最新的设置。数据源没了抛 JobFailed。""" + job = await session.get(CatalogProfileJob, self.job_id) + row = await session.get(DataSource, job.source_id) if job is not None else None + if row is None: + raise runtime.JobFailed(SOURCE_GONE) + if row.enabled is False: + return SOURCE_DISABLED.format(name=row.name) + if not catalog_profile.profile_settings(row.options).enabled: + return PROFILE_TURNED_OFF.format(name=row.name) + assert self.cx is not None + self.cx.masked = catalog_profile.masked_of(row.options) + return None + + async def _halt(self, session: AsyncSession, error: str | None, stop_reason: str | None) -> None: + """整个任务停下（止损、数据源停用或关掉了剖析）：后面的表 cancelled、error 是同一句原因；任务记下原因，状态到收尾时 + 按逐表算。""" + await _close_tasks(session, self.job_id, "cancelled", error, why=error or STOPPED_ERROR) + await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id) + .values(error=error, stop_reason=stop_reason, **self._counters()) + .execution_options(synchronize_session=False)) + await session.commit() + + async def _start_table(self, session: AsyncSession, seq: int) -> None: + await session.execute(self._task(CatalogProfileTask.seq == seq, CatalogProfileTask.status == "pending") + .values(status="running", started_at=utcnow())) + await session.commit() + + def _elapsed(self) -> int: + return int((time.monotonic() - self._table_started) * 1000) + + async def _save_progress(self, session: AsyncSession, seq: int, result: TableResult) -> None: + """一项检查开始或结束：这张表的检查项、正在做哪一项、查询数、发过的查询、行数估算，任务的查询计数。""" + await session.execute(self._task(CatalogProfileTask.seq == seq, CatalogProfileTask.status == "running") + .values(checks=[dict(c) for c in result.checks] or None, current=result.current, + queries=result.queries, elapsed_ms=self._elapsed(), + query_log=list(result.query_log) or None, + row_estimate=result.row_estimate.to_dict() if result.row_estimate else None)) + if values := self._counters(): + await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id).values(**values) + .execution_options(synchronize_session=False)) + await session.commit() + + async def _save_queries(self, session: AsyncSession, seq: int, result: TableResult) -> None: + """发出一条查询之后：这张表和任务的查询计数、提醒（发过的查询本身在检查项结束时一起写）。""" + await session.execute(self._task(CatalogProfileTask.seq == seq, CatalogProfileTask.status == "running") + .values(queries=result.queries, elapsed_ms=self._elapsed())) + if values := self._counters(): + await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id).values(**values) + .execution_options(synchronize_session=False)) + await session.commit() + + async def _save_table(self, session: AsyncSession, seq: int, result: TableResult) -> None: + """一张表结束（和写目录在同一次库操作里）：整张表的结果。""" + await session.execute( + self._task(CatalogProfileTask.seq == seq, CatalogProfileTask.status.in_(OPEN)) + .values(status=result.status, checks=[dict(c) for c in result.checks] or None, current=None, + queries=result.queries, elapsed_ms=result.elapsed_ms, + row_estimate=result.row_estimate.to_dict() if result.row_estimate else None, + findings=list(result.findings) or None, skipped=[s.to_dict() for s in result.skipped] or None, + date_ranges=list(result.date_ranges) or None, query_log=list(result.query_log) or None, + summary=result.summary(), skipped_count=result.skipped_count, unfinished=result.unfinished, + added=result.added, updated=result.updated, removed=result.removed, version=result.version, + error=result.error, finished_at=utcnow())) + if values := self._counters(): + await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id).values(**values) + .execution_options(synchronize_session=False)) + await session.commit() + + async def _finish(self, session: AsyncSession) -> None: + """剖析完：万一还有没收尾的表（进度回调写库失败之类）记失败；有 failed、partial、cancelled 的表是 partial，否则 + succeeded。""" + await _close_tasks(session, self.job_id, "failed", UNFINISHED_ERROR, why=UNFINISHED_ERROR) + statuses = list((await session.execute( + select(CatalogProfileTask.status).where(CatalogProfileTask.job_id == self.job_id))).scalars()) + status = "partial" if any(s in RETRYABLE for s in statuses) else "succeeded" + await session.execute( + update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id, CatalogProfileJob.status == "running") + .values(status=status, finished_at=utcnow(), **self._counters()) + .execution_options(synchronize_session=False)) + await session.commit() + + +class ProfileJobs(runtime.JobManager): + """进程内的剖析任务（start / stop / wait / running / shutdown 见 runtime.JobManager）。""" + + def __init__(self) -> None: + super().__init__(_ProfileRun, label="数据剖析任务", name="catalog-profile") + + +manager = ProfileJobs() + + +__all__ = ["INTERRUPTED_ERROR", "KEEP_JOBS", "MAX_TABLES", "NOTHING_AT_ALL", "NOTHING_NEEDED", "NothingToRetry", + "ProfileBusy", "ProfileFinished", "ProfileJobs", "ProfileNothing", "active_job", "busy_message", "cancel", + "create_job", "due_reason", "ensure_free", "job_brief", "job_dict", "job_out", "last_detail", + "last_profiles", "latest_job", "load", "load_task", "manager", "prune", "public_last", "retry_tables", + "startup", "task_detail", "task_row", "task_rows", "waiting_for"] diff --git a/backend/app/main.py b/backend/app/main.py index c5fa8f46..657c917a 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -93,14 +93,18 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: except Exception: # noqa: BLE001 logger.exception("上传表格的版本整理失败，服务照常启动") - # 上次没做完的数据目录起草任务：标成中断（没结束的表记停止），不自动续跑——拷来的数据目录（检查脚本、验收副本） - # 启动时不能替人去调模型。界面上点「重试」接着起草没写好的表 - from app.data import catalog_jobs + # 上次没做完的数据目录起草、剖析任务：标成中断（没结束的表记停止），不自动续跑——拷来的数据目录（检查脚本、验收副本） + # 启动时不能替人去调模型、查业务库。界面上点「重试」接着做没做完的表。剖析设置里按整次剖析算的旧预算顺带迁移掉 + from app.data import catalog_jobs, catalog_profile_jobs try: await catalog_jobs.startup() except Exception: # noqa: BLE001 logger.exception("数据目录起草任务的启动整理失败，服务照常启动") + try: + await catalog_profile_jobs.startup() + except Exception: # noqa: BLE001 + logger.exception("数据剖析任务的启动整理失败，服务照常启动") await run_manager.setup() # 流程：存量的待审批补建待办（派给任何人，幂等）；上次进程没了时已发出、没拿到结果的写入收成结果未知、安排核实 @@ -129,9 +133,10 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: try: yield finally: - # 先停调度，再停运行：停运行的途中不该再有新运行被发起。起草任务停下来标成中断，下次启动不会再续 + # 先停调度，再停运行：停运行的途中不该再有新运行被发起。起草、剖析任务停下来标成中断，下次启动不会再续 await scheduler.stop() await catalog_jobs.manager.shutdown() + await catalog_profile_jobs.manager.shutdown() await run_manager.shutdown() from app.sandbox.manager import sandbox_manager diff --git a/backend/tests/test_catalog_profile.py b/backend/tests/test_catalog_profile.py index 9c6a5fe9..fee64c5f 100644 --- a/backend/tests/test_catalog_profile.py +++ b/backend/tests/test_catalog_profile.py @@ -1,9 +1,10 @@ -"""数据剖析接口：POST /api/datasources/{id}/catalog/profile（阶段 2A）。 +"""数据剖析：POST /api/datasources/{id}/catalog/profile-jobs 建后台任务，逐表剖析（阶段 2A，后台任务改造之后）。 剖析是数据目录里唯一对业务库发查询的地方，这里逐条钉住方案定的约束：默认关闭、只读、每条查询都过守卫、 -查询次数和单条时限、总时长、遮罩的列不取样、大表不做整表统计、同一个数据源同时只有一次剖析； +每张表的查询次数和时长、单条时限、遮罩的列不取样、大表不做整表统计、同一个数据源同时只有一个剖析任务； 以及剖析的结论怎么进目录：命名推断对上数据的升为有确证（来源记为数据剖析，和外键分开）、对不上的写明 覆盖率、人工确认过的只补覆盖率和基数、状态列得到码值候选、只有一个日期列的表提议业务日期。 +任务本身（排队、停止、重启、重试、止损、范围、留痕、保留、设置迁移）在 test_catalog_profile_jobs.py。 所有测试都在合成的景区库上做（tests/fixtures/catalog/scenic.py），数据和名称全部虚构。 """ @@ -22,7 +23,7 @@ from httpx import ASGITransport, AsyncClient from sqlalchemy import event from sqlalchemy.engine import Engine -from app.data import catalog, catalog_profile +from app.data import catalog_profile, catalog_profile_jobs from app.data import engine as data_engine from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings from app.db.base import SessionLocal @@ -65,6 +66,8 @@ async def make_source(client): return sid yield _make + # 后台任务先收完（停不下来的停掉），再删源：删源会级联删掉任务 + await catalog_profile_jobs.manager.shutdown(grace=5) for sid in made: await drop_source(sid) @@ -119,7 +122,39 @@ def spy(monkeypatch, scenic_db): def _url(sid: str) -> str: - return f"/api/datasources/{sid}/catalog/profile" + return f"/api/datasources/{sid}/catalog/profile-jobs" + + +async def _start(client, sid: str, tables: list[str] | None = None, **body): + payload = {**({"tables": tables} if tables is not None else {}), **body} + return await client.post(_url(sid), json=payload, headers=ACTOR) + + +async def _job(client, sid: str, job_id: str) -> dict: + r = await client.get(f"{_url(sid)}/{job_id}") + assert r.status_code == 200, r.text + return r.json()["job"] + + +async def _detail(client, sid: str, job_id: str, seq: int) -> dict: + r = await client.get(f"{_url(sid)}/{job_id}/tasks/{seq}") + assert r.status_code == 200, r.text + return r.json()["task"] + + +async def _finished(client, sid: str, job_id: str) -> dict: + """等任务做完，返回做完的任务；job["tables"] 是每张表的明细（检查项、发现、跳过的项、发过的查询），按剖析顺序。""" + await catalog_profile_jobs.manager.wait(job_id, timeout=30) + job = await _job(client, sid, job_id) + job["tables"] = [await _detail(client, sid, job_id, t["seq"]) for t in job["tasks"]] + return job + + +async def _profile(client, sid: str, tables: list[str] | None = None, **body) -> dict: + """建剖析任务（202）、等它做完，返回做完的任务（见 _finished）。""" + r = await _start(client, sid, tables, **body) + assert r.status_code == 202, r.text + return await _finished(client, sid, r.json()["job"]["id"]) async def _notes(client, sid: str, table: str) -> dict: @@ -147,31 +182,35 @@ async def test_profile_is_off_by_default_and_says_where_to_turn_it_on(client, ma sid = await make_source(scenic_db, draft=["visits"]) watch = spy(scenic_db) watch.clear() - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) + r = await _start(client, sid, ["visits"]) assert r.status_code == 409 detail = r.json()["detail"] assert "数据源设置" in detail and "开启数据剖析" in detail assert r.json()["code"] == "profile_disabled" # 界面按机读码认是哪一种，不从原话里认 assert watch.executed == [] # 一条查询都没发 + assert (await client.get(f"{_url(sid)}/latest")).json()["job"] is None, "没建任务" async def test_settings_are_saved_and_validated_through_datasource_update(client, make_source, scenic_db): sid = await make_source(scenic_db) - r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=0)}) - assert r.status_code == 422 and "查询次数上限" in r.json()["detail"] + r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(table_max_queries=0)}) + assert r.status_code == 422 and "每张表最多查询数" in r.json()["detail"] # 机读码和出错的那一项：表单按 field 把报错落到那一格 - assert (r.json()["code"], r.json()["field"]) == ("profile_settings_invalid", "max_queries") + assert (r.json()["code"], r.json()["field"]) == ("profile_settings_invalid", "table_max_queries") + r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(table_max_s=5)}) + assert r.status_code == 422 and (r.json()["code"], r.json()["field"]) == ("profile_settings_invalid", "table_max_s") + assert "每张表时长上限" in r.json()["detail"] r = await client.patch(f"/api/datasources/{sid}", json={"options": {PROFILE_OPTION: {"enabled": "是"}}}) assert r.status_code == 422 and (r.json()["code"], r.json()["field"]) == ("profile_settings_invalid", "enabled") r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(rows=5)}) assert r.status_code == 422 and r.json()["code"] == "profile_settings_invalid" and "field" not in r.json() - r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=40, sample_size=500)}) + r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(table_max_queries=30, sample_size=500)}) assert r.status_code == 200, r.text - assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "max_queries": 40, "sample_size": 500} - # 设好以后就能剖析；返回里带着这次实际用的设置 - r = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR) - assert r.status_code == 200, r.text - assert r.json()["settings"]["max_queries"] == 40 and r.json()["settings"]["sample_size"] == 500 + assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "table_max_queries": 30, "sample_size": 500} + # 设好以后就能剖析；任务里记着这次实际用的设置（快照） + job = await _profile(client, sid, ["channels"]) + assert job["settings"] == {"enabled": True, "table_max_queries": 30, "table_max_s": 60, "query_timeout_s": 10, + "sample_size": 500, "max_scan_rows": 100_000} # ---------------------------------------------------------------- 剖析的结论 @@ -182,15 +221,15 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"]) watch = spy(scenic_db) watch.clear() - r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR) - assert r.status_code == 200, r.text - body = r.json() + body = await _profile(client, sid, ["visits", "channels"]) - # 所有发到业务库的语句都经过了守卫 + # 所有发到业务库的语句都经过了守卫；任务记的查询数就是发到库的条数，也是逐表的合计 assert watch.all_guarded(), watch.executed - assert body["queries_used"] == len(watch.executed) <= body["settings"]["max_queries"] - assert body["queries_used"] == sum(t["queries"] for t in body["tables"]) - assert body["stopped"] is None and body["actor"] == "周宁" and body["profiled_at"] + assert body["queries"] == len(watch.executed) == sum(t["queries"] for t in body["tables"]) + assert all(t["queries"] <= body["settings"]["table_max_queries"] for t in body["tables"]) + assert (body["status"], body["stop_reason"], body["error"]) == ("succeeded", None, None) + assert body["created_by"] == "周宁" and body["finished_at"] and body["scope"] == "selected" + assert [t["table_name"] for t in body["tasks"]] == ["visits", "channels"], "按给的顺序" notes = (await _notes(client, sid, "visits")) assert notes["updated_by"] == "周宁" @@ -226,6 +265,7 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d assert "channel_code" not in channels.get("columns", {}) # 唯一约束列不是码值 visits = _table(body, "visits") + assert visits["status"] == "ok" and visits["unfinished"] == 0 kinds = sorted(f["kind"] for f in visits["findings"]) # 两条命名推断的关系（gate_id、member_id），加两条外键关系（park_id、ticket_type_id）：外键的只核实基数 assert kinds == ["business_date", "codes", "relation", "relation", "relation", "relation"] @@ -243,6 +283,19 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d assert visits["row_estimate"] == {"rows": 1500, "method": "count", "at_least": None} # 更新的 4 项：两条命名关系升为有确证，两条外键关系补上用数据核实过的基数 assert visits["added"] == 2 and visits["updated"] == 4 and visits["error"] is None + # 发现的合计：逐表、整个任务各一份 + assert visits["summary"] == {"relations_checked": 4, "relations_verified": 4, "codes": 1, + "codes_pending": blank, "business_dates": 1} + assert body["summary"]["relations_checked"] == 4 and body["summary"]["codes"] == 2 + # 检查项：每项都结束了，按种类的进度对得上 + assert visits["checks_done"] == visits["checks_total"] == len(visits["checks"]) + assert visits["checks_by_kind"]["relation"] == [4, 4] and visits["checks_by_kind"]["date"] == [1, 1] + assert {c["state"] for c in visits["checks"]} == {"done"} + assert sum(c["queries"] for c in visits["checks"]) + sum( + 1 for q in visits["query_log"] if q["kind"] == "row_estimate" and q["target"] == "visits") >= 1 + # 第一条查询是估行数 + assert visits["query_log"][0]["kind"] == "row_estimate" and visits["query_log"][0]["outcome"] == "ok" + assert len(visits["query_log"]) == visits["queries"] async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, make_source, tmp_path): @@ -261,16 +314,15 @@ async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, before = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id") assert (before["to_table"], before["source"], before["status"]) == ("lots", "name", "proposed") - r = await client.post(_url(sid), json={"tables": ["parking_records"]}, headers=ACTOR) - assert r.status_code == 200, r.text + body = await _profile(client, sid, ["parking_records"]) rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id") assert rel["status"] == "proposed" and rel["source"] == "profile" assert rel["coverage"] == 0.15 assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"] - finding = _table(r.json(), "parking_records")["findings"][0] + finding = _table(body, "parking_records")["findings"][0] assert finding["status"] == "proposed" and finding["coverage"] == 0.15 - # 两个日期类列：不提议业务日期，取值范围只在返回里 - ranges = {d["column"] for d in _table(r.json(), "parking_records")["date_ranges"]} + # 两个日期类列：不提议业务日期，取值范围只在剖析结果里 + ranges = {d["column"] for d in _table(body, "parking_records")["date_ranges"]} assert ranges == {"entered_at"} # exited_at 全是空值 assert "business_date" not in (await _notes(client, sid, "parking_records"))["notes"] @@ -284,22 +336,21 @@ async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, mak "if_version": detail["version"]}) assert r.status_code == 200, r.text - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) - assert r.status_code == 200, r.text + body = await _profile(client, sid, ["visits"]) rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id") assert (rel["source"], rel["status"]) == ("name", "confirmed") assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one" # 人定的基数不改；原来就有基数（命名推断记的多对一），剖析也不给它记「用数据核实过」 assert "cardinality_checked" not in rel - finding = next(f for f in _table(r.json(), "visits")["findings"] if f.get("columns") == ["gate_id"]) + finding = next(f for f in _table(body, "visits")["findings"] if f.get("columns") == ["gate_id"]) assert finding["status"] == "confirmed" and finding["confirmed"] is True async def test_rerun_with_same_data_changes_nothing(client, make_source, scenic_db): """同一天、数据没变：第二次剖析不改目录、不升版本（冻结进快照的目录不会无故变）。""" sid = await make_source(scenic_db, options=_on(), draft=["visits"]) - first = (await client.post(_url(sid), json={"tables": ["visits"]})).json() - second = (await client.post(_url(sid), json={"tables": ["visits"]})).json() + first = await _profile(client, sid, ["visits"]) + second = await _profile(client, sid, ["visits"]) a, b = _table(first, "visits"), _table(second, "visits") assert (b["added"], b["updated"], b["removed"]) == (0, 0, 0) assert b["version"] == a["version"] @@ -318,32 +369,38 @@ async def test_masked_columns_are_never_queried(client, make_source, scenic_db, draft=["visits", "channels"]) watch = spy(scenic_db) watch.clear() - r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR) - assert r.status_code == 200, r.text + body = await _profile(client, sid, ["visits", "channels"]) assert watch.all_guarded() assert watch.mentions("member_id") == [] and watch.mentions("channel_type") == [] - visits, channels = _table(r.json(), "visits"), _table(r.json(), "channels") + visits, channels = _table(body, "visits"), _table(body, "channels") masked = [s for s in visits["skipped"] + channels["skipped"] if s["reason"] == "masked"] assert {s["target"] for s in masked} == {"member_id → members.id", "channel_type"} assert all("遮罩" in s["detail"] for s in masked) + # 按规则不查的不算没做完：表照样是完成 + assert visits["status"] == "ok" and visits["unfinished"] == 0 and visits["skipped_count"] == len(visits["skipped"]) + # 留痕里也没有遮罩的列 + assert not any("member_id" in q["sql"].lower() or "channel_type" in q["sql"].lower() + for t in body["tables"] for q in t["query_log"]) notes = (await _notes(client, sid, "visits"))["notes"] assert _relation(notes, "member_id")["source"] == "name" # 没动 assert _relation(notes, "gate_id")["status"] == "verified" # 其余照常 assert "channel_type" not in ((await _notes(client, sid, "channels"))["notes"].get("columns") or {}) -async def test_query_budget_stops_profiling_and_reports_the_rest(client, make_source, scenic_db, spy): - sid = await make_source(scenic_db, options=_on(max_queries=3), draft=["visits"]) +async def test_each_table_has_its_own_query_budget(client, make_source, scenic_db, spy): + """逐表预算：visits 用完自己的 3 条查询记部分完成，剩下的项记「这张表的查询次数用完」、旧结论原样留着；后面的 + channels 照样拿满额，做完。""" + sid = await make_source(scenic_db, options=_on(table_max_queries=3), draft=["visits"]) watch = spy(scenic_db) watch.clear() - r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR) - assert r.status_code == 200, r.text - body = r.json() - assert body["queries_used"] == 3 == len(watch.executed) and body["stopped"] == "budget" + body = await _profile(client, sid, ["visits", "channels"]) + assert body["queries"] == len(watch.executed) and body["stop_reason"] is None visits, channels = _table(body, "visits"), _table(body, "channels") - assert "budget" in _reasons(visits) - assert channels["queries"] == 0 and _reasons(channels) == {"budget"} - assert all("查询次数" in s["detail"] for s in visits["skipped"] + channels["skipped"] if s["reason"] == "budget") + assert visits["status"] == "partial" and visits["queries"] == 3 + assert "budget" in _reasons(visits) and visits["unfinished"] >= 1 + assert all("这张表的查询次数" in s["detail"] for s in visits["skipped"] if s["reason"] == "budget") + assert channels["status"] == "ok" and 0 < channels["queries"] <= 3 and "budget" not in _reasons(channels) + assert body["status"] == "partial", "有部分完成的表" # 没查完的关系不改：原来的命名推断原样留着 rel = _relation((await _notes(client, sid, "visits"))["notes"], "member_id") assert (rel["source"], rel["status"]) == ("name", "proposed") @@ -362,18 +419,20 @@ async def test_single_query_timeout_is_recorded_and_profiling_continues(client, return await real(source, sql, limits=limits) monkeypatch.setattr(data_engine, "run_query", slow_gate) - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) - assert r.status_code == 200, r.text - visits = _table(r.json(), "visits") + body = await _profile(client, sid, ["visits"]) + visits = _table(body, "visits") timeout = [s for s in visits["skipped"] if s["reason"] == "timeout"] assert [s["target"] for s in timeout] == ["gate_id → gates.id"] and "超时" in timeout[0]["detail"] - assert r.json()["stopped"] is None + assert body["stop_reason"] is None and body["timeouts"] == 1 + assert visits["status"] == "partial", "超时算运行原因没查成" + assert [q["outcome"] for q in visits["query_log"]].count("timeout") == 1 notes = (await _notes(client, sid, "visits"))["notes"] assert _relation(notes, "gate_id")["source"] == "name" # 超时的不改 assert _relation(notes, "member_id")["status"] == "verified" # 其余照常 -async def test_total_deadline_stops_profiling(make_source, scenic_db, monkeypatch): +async def test_each_table_has_its_own_deadline(make_source, scenic_db, monkeypatch): + """逐表时长：每条查询慢 0.5 秒，visits 的时长（1.4 秒）用完只停它自己，channels 另拿一份满额的时长。""" sid = await make_source(scenic_db, draft=["visits"]) real = data_engine.run_query @@ -384,13 +443,13 @@ async def test_total_deadline_stops_profiling(make_source, scenic_db, monkeypatc monkeypatch.setattr(data_engine, "run_query", slow) async with SessionLocal() as session: row = await session.get(DataSource, sid) - settings = ProfileSettings(enabled=True, max_total_s=1.4) - report = await catalog_profile.profile_catalog(session, row, row, tables=["visits"], actor=None, - settings=settings) - assert report.stopped == "deadline" - assert 1 <= report.queries_used <= 3 - reasons = {s.reason for t in report.tables for s in t.skipped} - assert "deadline" in reasons + cx = catalog_profile.make_context(row, ProfileSettings(enabled=True, table_max_s=1.4), options=row.options) + visits = await catalog_profile.profile_table(cx, "visits") + assert visits.status == "partial" and 1 <= visits.queries <= 3 + assert "deadline" in {s.reason for s in visits.skipped} + assert any("这张表的时长" in s.detail for s in visits.skipped if s.reason == "deadline") + channels = await catalog_profile.profile_table(cx, "channels") + assert channels.status == "ok" and channels.queries == 2, [s.to_dict() for s in channels.skipped] async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, spy): @@ -398,15 +457,15 @@ async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, sp sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits"]) watch = spy(scenic_db) watch.clear() - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) - assert r.status_code == 200, r.text - visits = _table(r.json(), "visits") + body = await _profile(client, sid, ["visits"]) + visits = _table(body, "visits") assert visits["row_estimate"] == {"rows": None, "method": "count", "at_least": 101} on_visits = [s for s in watch.executed if "from visits" in s.lower()] assert on_visits and all("LIMIT" in s for s in on_visits), on_visits assert not any("MIN(" in s for s in on_visits) assert "too_large" in _reasons(visits) assert any("太大" in s["detail"] or "上限" in s["detail"] for s in visits["skipped"] if s["reason"] == "too_large") + assert visits["status"] == "ok", "表太大是按规则不查，不算没做完" # 父表有主键，覆盖率又够：照样有确证；子表一侧没核对唯一，按多对一记，不算用数据核实过基数 rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id") assert rel["status"] == "verified" and rel["cardinality"] == "many_to_one" @@ -416,7 +475,7 @@ async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, sp # ---------------------------------------------------------------- 接口 -async def test_concurrent_profile_on_same_source_returns_409(client, make_source, scenic_db, monkeypatch): +async def test_second_profile_job_on_same_source_returns_409(client, make_source, scenic_db, monkeypatch): sid = await make_source(scenic_db, options=_on(), draft=["visits"]) real = data_engine.run_query started, release = asyncio.Event(), asyncio.Event() @@ -427,29 +486,39 @@ async def test_concurrent_profile_on_same_source_returns_409(client, make_source return await real(source, sql, limits=limits) monkeypatch.setattr(data_engine, "run_query", held) - first = asyncio.create_task(client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)) + first = await _start(client, sid, ["visits", "channels"]) + assert first.status_code == 202, first.text await asyncio.wait_for(started.wait(), 10) - second = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) - assert second.status_code == 409 and "正在进行数据剖析" in second.json()["detail"] - assert second.json()["code"] == "profile_busy" + second = await _start(client, sid, ["visits"]) + assert second.status_code == 409 and second.json()["code"] == "profile_busy" + # 说清是谁在跑、跑到哪；只点了一张表时说这张表在不在那个任务里、排第几 + assert "正在剖析：第 1 / 2 张表" in second.json()["detail"] and "visits 在这次剖析里，排第 1 张" in second.json()["detail"] + other = await _start(client, sid, ["gates"]) + assert other.status_code == 409 and "gates 不在这次剖析里" in other.json()["detail"] + active = (await client.get(f"{_url(sid)}/active")).json()["job"] + assert active["id"] == first.json()["job"]["id"] and active["status"] == "running" + listed = (await client.get(f"/api/datasources/{sid}/catalog")).json() + assert listed["active_profile_job"] == {"id": active["id"], "status": "running", "total": 2, "done": 0, + "waiting_for": None} release.set() - done = await asyncio.wait_for(first, 30) - assert done.status_code == 200, done.text + await catalog_profile_jobs.manager.wait(active["id"], timeout=30) + assert (await client.get(f"{_url(sid)}/active")).json()["job"] is None # 做完以后可以再来 - again = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR) - assert again.status_code == 200, again.text + again = await _start(client, sid, ["channels"]) + assert again.status_code == 202, again.text + await catalog_profile_jobs.manager.wait(again.json()["job"]["id"], timeout=30) async def test_blocked_profile_says_which_case_by_code(client, make_source, scenic_db, monkeypatch): - """剖析开始不了的 409 都带机读码：数据源停用、没有表结构、快照被改动（未开启、正在剖析见上面两条）。""" - from app.data.engine import SnapshotTampered + """剖析开始不了的 409 都带机读码：数据源停用、没有表结构、快照登记被改动（未开启、正忙见上面两条）。""" + from app.data import table_versions sid = await make_source(scenic_db, options=_on(), draft=["visits"]) async with SessionLocal() as session: row = await session.get(DataSource, sid) row.enabled = False await session.commit() - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) + r = await _start(client, sid, ["visits"]) assert r.status_code == 409 and r.json()["code"] == "datasource_inactive" and "已停用" in r.json()["detail"] async with SessionLocal() as session: @@ -457,103 +526,50 @@ async def test_blocked_profile_says_which_case_by_code(client, make_source, scen row.enabled = True row.schema_cache = {} await session.commit() - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) + r = await _start(client, sid, ["visits"]) assert r.status_code == 409 and r.json()["code"] == "schema_missing" and "探查结构" in r.json()["detail"] r = await client.post(f"/api/datasources/{sid}/introspect") assert r.status_code == 200, r.text + async def tampered(*_args, **_kwargs): + raise table_versions.SnapshotManifestMismatch("数据版本的哈希登记与快照清单不一致，已拒绝查询") + + with monkeypatch.context() as m: + m.setattr(table_versions, "resolve_source", tampered) + r = await _start(client, sid, ["visits"]) + assert r.status_code == 409 and r.json()["code"] == "snapshot_tampered" + assert (await client.get(f"{_url(sid)}/latest")).json()["job"] is None, "开始不了的都没建任务" + + +async def test_a_tampered_snapshot_during_profiling_fails_the_job(client, make_source, scenic_db, monkeypatch): + """剖析途中数据文件和登记的版本对不上：整个任务停下、记失败，原因照实写；没做完的表也记失败，可以重试。""" + from app.data.engine import SnapshotTampered + + sid = await make_source(scenic_db, options=_on(), draft=["visits"]) + async def tampered(*_args, **_kwargs): raise SnapshotTampered("数据文件与登记的版本不一致，可能被修改过，已拒绝查询") - monkeypatch.setattr(catalog_profile, "profile_catalog", tampered) - r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR) - assert r.status_code == 409 and r.json()["code"] == "snapshot_tampered" - - -async def test_default_tables_are_used_tables_with_relation_candidates(client, make_source, scenic_db): - sid = await make_source(scenic_db, options=_on(), draft=[]) - r = await client.post(_url(sid), json={}, headers=ACTOR) - assert r.status_code == 200, r.text - names = [t["table_name"] for t in r.json()["tables"]] - assert names and len(names) <= catalog_profile.PROFILE_DEFAULT_TABLES - async with SessionLocal() as session: - entries = await catalog.read_catalog(session, sid) - for name in names: - assert catalog_profile._relation_candidates(entries[name].notes), name - assert "parks" not in names and "channels" not in names # 没有待核实关系的表不选 - - -async def _candidates(sid: str) -> tuple[list[str], set[str]]: - """默认挑表的候选（目录里有要核对的关系的表，按表结构顺序）和其中还有待核实（推断）关系的那些。""" - async with SessionLocal() as session: - row = await session.get(DataSource, sid) - entries = await catalog.read_catalog(session, sid) - order = list((row.schema_cache or {}).get("tables") or {}) - rels = {name: catalog_profile._relation_candidates(entries[name].notes) for name in order if name in entries} - names = [name for name in order if rels.get(name)] - return names, {name for name in names if any(r["status"] == "proposed" for r in rels[name])} - - -async def test_default_pick_skips_empty_tables_and_prefers_pending_relations(client, make_source, tmp_path, - monkeypatch): - """B1：表都没被运行查询过（使用次数都是 0）时，以前按表结构顺序取前几张，结果大半是空表（景区库里排在前面的 - 候选表正好多数没有数据）。现在跳过行数估计为 0 的表（记在 empty_tables 里），并且先挑还有待核实关系的表， - 不把刚核实过的再剖析一遍。默认张数调成 3，好在一个小库上看出先后。""" - monkeypatch.setattr(catalog_profile, "PROFILE_DEFAULT_TABLES", 3) - path = scenic.build(tmp_path / "b1.db") - db = sqlite3.connect(path) - try: - tables = [r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY rowid")] - empty = {t for t in tables if db.execute(f'SELECT COUNT(*) FROM "{t}"').fetchone()[0] == 0} - finally: - db.close() - sid = await make_source(str(path), options=_on(), draft=tables) - names, pending = await _candidates(sid) - full = [name for name in names if name not in empty] - # 前提：按表结构顺序的前 3 张候选里有空表（以前就会挑中它们），有数据、还待核实的候选不止 6 张 - assert set(names[:3]) & empty - assert len([name for name in full if name in pending]) >= 6 - - first = (await client.post(_url(sid), json={}, headers=ACTOR)).json() - picked = [t["table_name"] for t in first["tables"]] - assert len(picked) == 3 and not set(picked) & empty, picked - assert first["empty_tables"] and set(first["empty_tables"]) <= empty - assert all(t["queries"] > 0 for t in first["tables"]) - assert first["queries_used"] >= sum(t["queries"] for t in first["tables"]) + len(first["empty_tables"]) - - # 第二次：先挑还有待核实关系的表；第一次核实过的（关系都升为有确证了）排到后面，这次挑不到 - names, pending = await _candidates(sid) - assert not set(picked) & pending # 第一次挑的三张，关系都核实了 - second = (await client.post(_url(sid), json={}, headers=ACTOR)).json() - again = [t["table_name"] for t in second["tables"]] - assert len(again) == 3 and all(name in pending for name in again), (again, pending) - assert not set(again) & set(picked) and not set(again) & empty - - -async def test_profile_note_when_every_candidate_is_empty(client, make_source, tmp_path): - path = scenic.build(tmp_path / "b1-empty.db") - sid = await make_source(str(path), options=_on(), draft=[]) - names, _ = await _candidates(sid) - db = sqlite3.connect(path) - try: - for name in names: - db.execute(f'DELETE FROM "{name}"') - db.commit() - finally: - db.close() - body = (await client.post(_url(sid), json={}, headers=ACTOR)).json() - assert body["tables"] == [] and set(body["empty_tables"]) == set(names) - assert "空表" in body["note"] + monkeypatch.setattr(data_engine, "run_query", tampered) + body = await _profile(client, sid, ["visits", "channels"]) + assert body["status"] == "failed" and body["error"] == "数据文件与登记的版本不一致，可能被修改过，已拒绝查询" + assert {t["status"] for t in body["tables"]} == {"failed"} + assert all(t["error"] == body["error"] for t in body["tables"]) + # 没结束的检查项不留在「排队」：记为跳过，原因写清 + assert all(c["state"] != "pending" for t in body["tables"] for c in t["checks"]) async def test_unknown_table_and_missing_source(client, make_source, scenic_db): sid = await make_source(scenic_db, options=_on(), draft=["visits"]) - r = await client.post(_url(sid), json={"tables": ["nope"]}, headers=ACTOR) - assert r.status_code == 200 - assert _table(r.json(), "nope")["error"] and r.json()["queries_used"] == 0 + body = await _profile(client, sid, ["nope"]) + nope = _table(body, "nope") + assert nope["status"] == "skipped" and nope["error"] == "表结构里没有 nope，请先重新探查结构" + assert body["queries"] == 0 and body["status"] == "succeeded" r = await client.post(_url("0" * 32), json={}, headers=ACTOR) assert r.status_code == 404 + assert (await client.get(f"{_url(sid)}/not-a-job")).status_code == 404 + assert (await client.get(f"{_url(sid)}/{body['id']}/tasks/9")).status_code == 404 def _orders_db(path, *, items_per_order: int) -> str: @@ -583,27 +599,23 @@ async def test_foreign_key_relations_get_their_cardinality_checked_with_data(cli before = _relation((await _notes(client, sid, "order_items"))["notes"], "order_id") assert (before["source"], before["status"]) == ("fk", "verified") - r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR) - assert r.status_code == 200, r.text + body = await _profile(client, sid, ["order_items"]) rel = _relation((await _notes(client, sid, "order_items"))["notes"], "order_id") assert (rel["source"], rel["status"]) == ("fk", "verified") assert rel["cardinality"] == cardinality and rel.get("cardinality_checked") is True assert rel.get("coverage") is None - finding = next(f for f in _table(r.json(), "order_items")["findings"] if f.get("columns") == ["order_id"]) + finding = next(f for f in _table(body, "order_items")["findings"] if f.get("columns") == ["order_id"]) assert finding["cardinality_checked"] is True and finding["cardinality"] == cardinality # 重新剖析、数据没变：版本不升 version = (await _notes(client, sid, "order_items"))["version"] - r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR) - assert r.status_code == 200, r.text + await _profile(client, sid, ["order_items"]) assert (await _notes(client, sid, "order_items"))["version"] == version async def test_fanout_on_a_data_checked_foreign_key_is_an_error_again(client, make_source, tmp_path): """剖析核实过的外键一对多：订单关联明细后对订单金额求和，扇出检查报 error；数据上一对一的不报。""" from app.data.sqlcheck import load_checkers - from app.db.base import SessionLocal - from app.db.models import DataSource sql = "SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id" for items, expected in ((3, "error"), (1, None)): @@ -615,8 +627,7 @@ async def test_fanout_on_a_data_checked_foreign_key_is_an_error_again(client, ma r = await client.put(f"/api/datasources/{sid}/catalog/orders", json={"notes": notes, "if_version": detail["version"]}, headers=ACTOR) assert r.status_code == 200, r.text - r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR) - assert r.status_code == 200, r.text + await _profile(client, sid, ["order_items"]) async with SessionLocal() as session: source = await session.get(DataSource, sid) checker = (await load_checkers(session, [source]))[source.name] diff --git a/backend/tests/test_catalog_profile_units.py b/backend/tests/test_catalog_profile_units.py index d41ecf2d..4fa1472f 100644 --- a/backend/tests/test_catalog_profile_units.py +++ b/backend/tests/test_catalog_profile_units.py @@ -20,26 +20,30 @@ AT = "2026-10-03T08:00:00+00:00" def test_settings_default_off_and_parse_leniently(): s = profile_settings({}) assert s == ProfileSettings() - assert s.enabled is False and s.max_queries == 60 and s.query_timeout_s == 10 and s.sample_size == 2000 - assert s.max_scan_rows == 100_000 and s.max_total_s == 120 - on = profile_settings({PROFILE_OPTION: {"enabled": True, "max_queries": "30", "sample_size": 500}}) - assert on.enabled is True and on.max_queries == 30 and on.sample_size == 500 - # 库里存着的值不对（老数据、手改过）：那一项用缺省，不因此开启，也不报错 - odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "max_queries": 0, "query_timeout_s": "x"}}) - assert odd.enabled is False and odd.max_queries == 60 and odd.query_timeout_s == 10 + assert s.enabled is False and s.table_max_queries == 40 and s.table_max_s == 60 and s.query_timeout_s == 10 + assert s.sample_size == 2000 and s.max_scan_rows == 100_000 + on = profile_settings({PROFILE_OPTION: {"enabled": True, "table_max_queries": "30", "sample_size": 500}}) + assert on.enabled is True and on.table_max_queries == 30 and on.sample_size == 500 + # 库里存着的值不对（老数据、手改过）：那一项用缺省，不因此开启，也不报错；迁移前的旧键不看 + odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "table_max_queries": 0, "query_timeout_s": "x", + "max_queries": 5}}) + assert odd.enabled is False and odd.table_max_queries == 40 and odd.query_timeout_s == 10 assert profile_settings({PROFILE_OPTION: "on"}) == ProfileSettings() + # 任务里存着的快照读回来是同一份设置 + assert ProfileSettings.from_dict(on.to_dict()) == on and ProfileSettings.from_dict(None) == ProfileSettings() @pytest.mark.parametrize("value, label", [ - ({"max_queries": 0}, "查询次数上限"), - ({"max_queries": 501}, "查询次数上限"), - ({"max_queries": 2.5}, "查询次数上限"), + ({"table_max_queries": 0}, "每张表最多查询数"), + ({"table_max_queries": 201}, "每张表最多查询数"), + ({"table_max_queries": 2.5}, "每张表最多查询数"), + ({"table_max_s": 9}, "每张表时长上限"), + ({"table_max_s": 601}, "每张表时长上限"), ({"query_timeout_s": 61}, "单条查询时限"), ({"sample_size": 5}, "抽样键值数"), ({"max_scan_rows": -1}, "整表统计行数上限"), - ({"max_total_s": 5}, "总时长上限"), ({"enabled": "true"}, "开启数据剖析"), - ({"max_queries": True}, "查询次数上限"), + ({"table_max_queries": True}, "每张表最多查询数"), ]) def test_settings_problem_names_the_field_in_chinese(value, label): problem = profile_settings_problem({PROFILE_OPTION: value}) @@ -51,9 +55,21 @@ def test_settings_problem_rejects_unknown_keys_and_wrong_shape(): assert profile_settings_problem({PROFILE_OPTION: ["enabled"]}) assert profile_settings_problem({}) is None assert profile_settings_problem({PROFILE_OPTION: None}) is None - assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "max_queries": 500, "query_timeout_s": 1.5, - "sample_size": 10, "max_scan_rows": 0, - "max_total_s": 600}}) is None + assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "table_max_queries": 200, "table_max_s": 10.5, + "query_timeout_s": 1.5, "sample_size": 10, + "max_scan_rows": 0}}) is None + + +def test_legacy_whole_run_budgets_are_stripped_not_rejected(): + """以前按整次剖析算的两项（查询次数上限、总时长上限）：保存时只摘掉，不报错；不折算成每张表的预算。""" + old = {"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True, "max_queries": 60, "max_total_s": 120, + "sample_size": 500}} + cleaned = catalog_profile.without_legacy(old) + assert cleaned == {"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True, "sample_size": 500}} + assert old[PROFILE_OPTION]["max_queries"] == 60, "不改调用方的对象" + assert profile_settings_problem(cleaned) is None + plain = {PROFILE_OPTION: {"enabled": True}} + assert catalog_profile.without_legacy(plain) is plain and catalog_profile.without_legacy(None) is None def test_profile_option_never_reaches_the_connection_url(): diff --git a/backend/tests/test_catalog_semantics.py b/backend/tests/test_catalog_semantics.py index d97a8c1d..25eb9aad 100644 --- a/backend/tests/test_catalog_semantics.py +++ b/backend/tests/test_catalog_semantics.py @@ -24,7 +24,7 @@ from app.db.models import DataSource from tests.fixtures.catalog import scenic, scenic_notes from tests.fixtures.catalog.fake_model import FakeModel from tests.test_catalog_patch import _schema -from tests.test_catalog_profile import _notes, _on, _relation, _table, _url, client, make_source # noqa: F401 +from tests.test_catalog_profile import _notes, _on, _profile, _relation, _table, client, make_source # noqa: F401 AT = "2026-10-04T08:00:00+00:00" DAY1, DAY2, DAY3 = "2026-10-01T08:00:00+00:00", "2026-10-02T08:00:00+00:00", "2026-10-03T08:00:00+00:00" @@ -131,8 +131,7 @@ async def test_confirming_a_profiled_relation_by_patch_keeps_the_fanout_check(cl """审查探针 A1：剖析得出多对一、覆盖率 1.0，经对话提案确认这条关系之后，扇出检查照样报 error。""" sql = "SELECT COUNT(g.id) AS n FROM gates g JOIN visits v ON v.gate_id = g.id" sid = await make_source(scenic_db, options=_on(), draft=["visits", "gates"]) - r = await client.post(_url(sid), json={"tables": ["visits"]}) - assert r.status_code == 200, r.text + await _profile(client, sid, ["visits"]) detail = await _notes(client, sid, "visits") rel = _relation(detail["notes"], "gate_id") assert (rel["source"], rel["status"], rel["cardinality"], rel["coverage"]) == ( @@ -156,8 +155,7 @@ async def test_profiled_relation_without_child_check_only_warns(client, make_sou 核实过：扇出检查只给 warning（「可能重复计算」），不给 error、不让出具降档。""" sql = "SELECT COUNT(g.id) AS n FROM gates g JOIN visits v ON v.gate_id = g.id" sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits", "gates"]) - r = await client.post(_url(sid), json={"tables": ["visits"]}) - assert r.status_code == 200, r.text + await _profile(client, sid, ["visits"]) rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id") assert (rel["source"], rel["status"], rel["cardinality"]) == ("profile", "verified", "many_to_one") assert not rel.get("cardinality_checked") @@ -359,15 +357,13 @@ def test_codes_complete_flag_is_validated_and_counts_as_an_edit(): async def test_profile_marks_codes_complete_only_when_the_whole_table_was_read(client, make_source, scenic_db): whole = await make_source(scenic_db, options=_on(), draft=["visits"]) - r = await client.post(_url(whole), json={"tables": ["visits"]}) - assert r.status_code == 200, r.text + await _profile(client, whole, ["visits"]) codes = (await _notes(client, whole, "visits"))["notes"]["columns"]["status"]["codes"] assert "统计全表" in codes["note"] and codes.get("complete") is True # 不做整表统计（只看前若干行）：取值可能没看全，不标 capped = await make_source(scenic_db, options=_on(max_scan_rows=0, sample_size=10), draft=["visits"]) - r = await client.post(_url(capped), json={"tables": ["visits"]}) - assert r.status_code == 200, r.text + await _profile(client, capped, ["visits"]) codes = (await _notes(client, capped, "visits"))["notes"]["columns"]["status"]["codes"] assert "按前 10 行统计" in codes["note"] and "complete" not in codes @@ -388,10 +384,10 @@ async def test_rerun_on_another_day_with_same_data_keeps_the_version(client, mak assert r.status_code == 200, r.text monkeypatch.setattr(catalog, "now_iso", lambda: DAY1) - first = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits") + first = _table(await _profile(client, sid, ["visits"]), "visits") notes_day1 = (await _notes(client, sid, "visits"))["notes"] monkeypatch.setattr(catalog, "now_iso", lambda: DAY2) - second = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits") + second = _table(await _profile(client, sid, ["visits"]), "visits") assert (second["added"], second["updated"], second["removed"]) == (0, 0, 0) assert second["version"] == first["version"] assert (await _notes(client, sid, "visits"))["notes"] == notes_day1 # 备注里还是第一天的日期 @@ -404,7 +400,7 @@ async def test_rerun_on_another_day_with_same_data_keeps_the_version(client, mak finally: db.close() monkeypatch.setattr(catalog, "now_iso", lambda: DAY3) - third = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits") + third = _table(await _profile(client, sid, ["visits"]), "visits") assert third["version"] == second["version"] + 1 and third["updated"] >= 1 codes = (await _notes(client, sid, "visits"))["notes"]["columns"]["status"]["codes"] assert set(codes["value"]) == {"0", "1", "2"} and codes["updated_at"] == DAY3