mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 04:01:58 +00:00
合入剖析后端：抽出数据目录后台任务的共用运行时（起草任务改用它，行为不变），数据剖析改成逐表的后台任务（逐表预算、任务级止损、排队、停止、重启标中断、重试、查询留痕、保留），设置换成按表的两项，去掉同步剖析接口
This commit is contained in:
ThearchyHelios
+195
-93
@@ -8,12 +8,17 @@
|
||||
- POST /catalog/{table}/patch 保存目录修改提案（乐观锁），记为人工确认
|
||||
- GET /catalog/{table}/impact 影响面：引用这张表的已发布、受管模板（data/catalog_impact.py）
|
||||
- POST /catalog/draft 同步起草（注释、外键、命名推断，可选模型）。界面改用下面的后台起草任务，这个留着；
|
||||
和后台任务双向互斥：同一个源有排队或进行中的后台任务（或正在同步起草）时 409
|
||||
catalog_draft_busy，同步起草期间建任务、重试、导入也是 409
|
||||
同一个源有排队或进行中的起草任务（或正在同步起草）时 409 catalog_draft_busy，有剖析任务时
|
||||
409 profile_busy；同步起草期间建起草任务、重试、导入也是 409，新建的剖析任务排在它后面
|
||||
- PUT /catalog/guidance 数据源级的起草要求（使用者补充的背景和要求，起草时交给模型）
|
||||
- POST /catalog/draft-jobs 建后台起草任务（202）；GET /draft-jobs/active、/latest、/{job_id} 读任务；
|
||||
POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py）
|
||||
- POST /catalog/profile 同步剖析：对业务库发少量只读查询，核对关系、取码值候选（data/catalog_profile.py）
|
||||
POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py）。
|
||||
同源有剖析任务时照建，排在它后面（waiting_for）
|
||||
- POST /catalog/profile-jobs 建后台剖析任务（202）：对业务库逐表发少量只读查询，核对关系、取码值候选、提议业务日期
|
||||
（data/catalog_profile_jobs.py，引擎在 data/catalog_profile.py）；GET /profile-jobs/active、
|
||||
/latest、/{job_id} 读任务，/{job_id}/tasks/{seq} 读一张表的明细（检查项、发现、发过的查询）；
|
||||
POST /profile-jobs/{job_id}/cancel 停止，/retry 只重试没做完的表。同源有起草任务时照建，
|
||||
排在它后面（waiting_for）
|
||||
- GET /catalog/review-queue 审阅队列：推断的项按使用次数、风险、来源排好；POST /catalog/review/batch 批量审阅，
|
||||
GET /catalog/review-progress 审阅进度，GET /catalog/{table}/usage 用过这张表的运行
|
||||
（data/catalog_review.py）
|
||||
@@ -23,8 +28,9 @@
|
||||
（data/catalog_assistant.py）
|
||||
|
||||
规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径
|
||||
不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。
|
||||
起草任务的 409 带机读码（api/coded.py）：同一个源已有排队或进行中的任务、起草模型用不了、要停止的任务已经结束。
|
||||
不对 422，源或表不存在 404。所有写操作记下署名（api/deps.actor_of：开了登录是认证过的用户名，否则请求头 X-Actor 的自报）。
|
||||
起草、剖析任务的 409 带机读码（api/coded.py）：同一个源已有排队或进行中的同种任务、起草模型用不了、剖析没开启、按范围
|
||||
挑不出表、要停止的任务已经结束。
|
||||
|
||||
上传表格的源按当前快照的表结构回答（table_versions.resolve_source），和查询实际用的库是同一个版本。
|
||||
"""
|
||||
@@ -38,11 +44,12 @@ from pydantic import BaseModel, Field, ValidationError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.coded import (
|
||||
CATALOG_DRAFT_BUSY, CATALOG_DRAFT_FINISHED, CATALOG_DRAFT_MODEL_UNAVAILABLE, PROFILE_BUSY, SCHEMA_MISSING,
|
||||
SNAPSHOT_TAMPERED, CodedHTTPException,
|
||||
CATALOG_DRAFT_BUSY, CATALOG_DRAFT_FINISHED, CATALOG_DRAFT_MODEL_UNAVAILABLE, CATALOG_PROFILE_FINISHED, PROFILE_BUSY,
|
||||
PROFILE_NOTHING, SCHEMA_MISSING, SNAPSHOT_TAMPERED, CodedHTTPException,
|
||||
)
|
||||
from app.api.runs import actor_of
|
||||
from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, introspect, table_versions
|
||||
from app.api.deps import actor_of
|
||||
from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, catalog_profile_jobs, introspect
|
||||
from app.data import table_versions
|
||||
from app.data import catalog_assistant, catalog_review, catalog_transfer
|
||||
from app.data.engine import SnapshotTampered
|
||||
from app.db.base import get_session
|
||||
@@ -102,9 +109,11 @@ GUIDANCE_MISSING = "请求里要写起草要求（guidance 一项，一段文字
|
||||
GUIDANCE_NOT_TEXT = "起草要求要是一段文字（要清除起草要求请传空串）"
|
||||
|
||||
|
||||
class CatalogProfileIn(BaseModel):
|
||||
#: 要剖析的表；不给或为空时按使用次数取前 10 张目录里有待核实关系的表
|
||||
tables: list[str] | None = Field(default=None, max_length=200)
|
||||
class ProfileJobIn(BaseModel):
|
||||
#: 要剖析的表（范围 selected，按给的顺序去重）；最多 500 张（超了 422，说明是中文）
|
||||
tables: list[str] | None = None
|
||||
#: 不指定表时的范围：needed 需要剖析的表（缺省）、all 全部表（都不含视图、没有可剖析的项的表）
|
||||
scope: Literal["needed", "all"] | None = None
|
||||
|
||||
|
||||
async def _resolved(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]:
|
||||
@@ -134,7 +143,8 @@ def _item_value(notes: dict[str, Any], name: str) -> tuple[Any, str | None]:
|
||||
|
||||
|
||||
def _summary_row(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogEntry | None,
|
||||
usage: int, last_draft: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
usage: int, last_draft: dict[str, Any] | None = None,
|
||||
profiling: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
notes = entry.notes if entry else {}
|
||||
label, label_status = _item_value(notes, "label")
|
||||
kind, _ = _item_value(notes, "kind")
|
||||
@@ -157,9 +167,20 @@ def _summary_row(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogE
|
||||
# 这张表最近一次起草任务里的那一项：{status, job_id, finished_at, error, model_error, blank_columns}，没起草过
|
||||
# 是 null
|
||||
"last_draft": last_draft,
|
||||
# 剖析：最近一次剖析（CatalogLastProfile，没剖析过是 null）、要不要剖析和为什么、按当前目录和表结构要做几项检查
|
||||
**(profiling or {"last_profile": None, "profile_due": False, "profile_due_reason": None, "profile_checks": 0}),
|
||||
}
|
||||
|
||||
|
||||
def _profiling(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogEntry | None, *,
|
||||
tables: dict[str, Any], masked: set[str], last: dict[str, Any] | None) -> dict[str, Any]:
|
||||
notes = entry.notes if entry else {}
|
||||
checks = catalog_profile.profile_checks(name, meta, notes, tables=tables, masked=masked)
|
||||
reason = catalog_profile_jobs.due_reason(meta, notes, last, checks)
|
||||
return {"last_profile": catalog_profile_jobs.public_last(last), "profile_due": reason is not None,
|
||||
"profile_due_reason": reason, "profile_checks": checks}
|
||||
|
||||
|
||||
def _structure(meta: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"qualified": meta.get("qualified"),
|
||||
@@ -188,6 +209,9 @@ async def _detail(session: AsyncSession, source: Any, table: str, entry: catalog
|
||||
"structure": _structure(meta) if meta is not None else None,
|
||||
"system_notes": catalog.system_notes_source(source),
|
||||
"usage": usage,
|
||||
# 这张表最近一次剖析（CatalogProfileTaskDetail：检查项、发现、跳过的项、日期范围、发过的查询）；任务还在进行就是
|
||||
# 进行中的状态；没剖析过是 null
|
||||
"last_profile": await catalog_profile_jobs.last_detail(session, source.id, table),
|
||||
}
|
||||
|
||||
|
||||
@@ -223,11 +247,17 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi
|
||||
entries = await catalog.read_catalog(session, source.id)
|
||||
usage = await catalog.table_usage(session, source)
|
||||
drafts = await catalog_jobs.last_drafts(session, source.id)
|
||||
lasts = await catalog_profile_jobs.last_profiles(session, source.id)
|
||||
masked = catalog_profile.masked_of(row.options)
|
||||
|
||||
def profiling(name: str, meta: dict[str, Any] | None) -> dict[str, Any]:
|
||||
return _profiling(name, meta, entries.get(name), tables=tables, masked=masked, last=lasts.get(name))
|
||||
|
||||
order = {name: i for i, name in enumerate(tables)}
|
||||
rows = [_summary_row(name, meta, entries.get(name), usage.get(name, 0), drafts.get(name))
|
||||
rows = [_summary_row(name, meta, entries.get(name), usage.get(name, 0), drafts.get(name), profiling(name, meta))
|
||||
for name, meta in tables.items()]
|
||||
rows.sort(key=lambda r: (-r["usage"], order[r["table_name"]]))
|
||||
rows += [_summary_row(name, None, entry, 0, drafts.get(name))
|
||||
rows += [_summary_row(name, None, entry, 0, drafts.get(name), profiling(name, None))
|
||||
for name, entry in sorted(entries.items()) if name not in tables]
|
||||
return {
|
||||
"tables": rows,
|
||||
@@ -238,8 +268,11 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi
|
||||
"schema_total": max(_int(cache.get("total")), len(tables)),
|
||||
# 数据源级的起草要求（没有是 null）
|
||||
"guidance": catalog_jobs.guidance_of(row),
|
||||
# 排队或进行中的起草任务 {id, status, total, done}，界面据此显示进度；没有是 null
|
||||
# 排队或进行中的起草任务 {id, status, total, done, waiting_for}，界面据此显示进度；没有是 null
|
||||
"active_job": await catalog_jobs.job_brief(session, await catalog_jobs.active_job(session, source.id)),
|
||||
# 排队或进行中的剖析任务 {id, status, total, done, waiting_for}；没有是 null
|
||||
"active_profile_job": await catalog_profile_jobs.job_brief(
|
||||
session, await catalog_profile_jobs.active_job(session, source.id)),
|
||||
}
|
||||
|
||||
|
||||
@@ -313,22 +346,154 @@ async def preview_catalog_import(source_id: str, payload: CatalogImportIn,
|
||||
@router.post("/{source_id}/catalog/import")
|
||||
async def import_catalog(source_id: str, payload: CatalogImportIn, x_actor: str | None = Header(default=None),
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""导入：规则同预览，逐表写入（乐观锁），返回同样形状的报告和每张表写入后的版本。有排队或进行中的起草任务
|
||||
409 catalog_draft_busy；文件格式不对 422；没有表结构 409。"""
|
||||
"""导入：规则同预览，逐表写入（乐观锁），返回同样形状的报告和每张表写入后的版本。有排队或进行中的起草任务（或正在
|
||||
同步起草、导入）409 catalog_draft_busy，有剖析任务 409 profile_busy（说法的最后一句说「再导入」）；文件格式不对 422；
|
||||
没有表结构 409。"""
|
||||
row, source = await _import_source(session, source_id)
|
||||
try:
|
||||
await catalog_jobs.ensure_idle(session, row)
|
||||
async with catalog_jobs.import_write(session, row):
|
||||
report = await catalog_transfer.run_import(session, row, source, payload.content,
|
||||
include_guidance=payload.include_guidance,
|
||||
actor=actor_of(x_actor))
|
||||
except catalog_jobs.DraftBusy as e:
|
||||
raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e
|
||||
try:
|
||||
report = await catalog_transfer.run_import(session, row, source, payload.content,
|
||||
include_guidance=payload.include_guidance,
|
||||
actor=actor_of(x_actor))
|
||||
except catalog_profile_jobs.ProfileBusy as e:
|
||||
raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e
|
||||
except catalog_transfer.TransferInvalid as e:
|
||||
raise HTTPException(422, str(e)) from e
|
||||
return report.as_dict()
|
||||
|
||||
|
||||
# ---- 后台剖析任务（同样登记在 GET /catalog/{table} 前面：active、latest 要先于 /profile-jobs/{job_id}）
|
||||
|
||||
|
||||
async def _profile_source(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]:
|
||||
"""建剖析任务、重试之前的几道门，409 都带机读码（界面按它分情况给下一步，不从原话里认）：未开启 profile_disabled、
|
||||
数据源停用 datasource_inactive、快照登记被改动 snapshot_tampered、没有表结构 schema_missing。返回 (数据源记录,
|
||||
读表结构用的源)。"""
|
||||
row = await _source_row(session, source_id)
|
||||
try:
|
||||
catalog_profile.ensure_enabled(row)
|
||||
except catalog_profile.ProfileDisabled as e:
|
||||
raise CodedHTTPException(409, str(e), e.code) from e
|
||||
try:
|
||||
source = await table_versions.resolve_source(session, row)
|
||||
except SnapshotTampered as e: # 快照清单对不上（SnapshotManifestMismatch 同时是 SnapshotMissing）
|
||||
raise CodedHTTPException(409, str(e), SNAPSHOT_TAMPERED) from e
|
||||
except table_versions.SnapshotMissing as e:
|
||||
raise HTTPException(409, str(e)) from e
|
||||
cache = source.schema_cache or {}
|
||||
if not cache.get("tables"):
|
||||
raise CodedHTTPException(409, f"{introspect.why_empty(cache)}，无法进行数据剖析。请先在数据源卡片上点「探查结构」",
|
||||
SCHEMA_MISSING)
|
||||
return row, source
|
||||
|
||||
|
||||
async def _start_profile(session: AsyncSession, row: DataSource, source: Any, *, tables: list[str] | None,
|
||||
scope: str, actor: str | None, retry_of: str | None = None) -> dict[str, Any]:
|
||||
try:
|
||||
job, tasks = await catalog_profile_jobs.create_job(
|
||||
session, row, source, tables=tables, scope=scope, settings=catalog_profile.ensure_enabled(row),
|
||||
actor=actor, retry_of=retry_of)
|
||||
except catalog_profile_jobs.ProfileBusy as e:
|
||||
raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e
|
||||
except catalog_profile_jobs.ProfileNothing as e:
|
||||
raise CodedHTTPException(409, str(e), PROFILE_NOTHING) from e
|
||||
catalog_profile_jobs.manager.start(job.id)
|
||||
return {"job": catalog_profile_jobs.job_dict(job, tasks,
|
||||
waiting_for=await catalog_profile_jobs.waiting_for(session, job))}
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/profile-jobs", status_code=202)
|
||||
async def create_profile_job(source_id: str, payload: ProfileJobIn, x_actor: str | None = Header(default=None),
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""建后台剖析任务，马上返回（202，任务是排队中）。tables 非空时范围是 selected（表结构里没有、视图、没有可剖析的项照样
|
||||
建进任务、状态 skipped），否则按 scope（缺省 needed）挑。同源有排队或进行中的剖析任务 409 profile_busy（说清进行到第几张；
|
||||
界面改去 GET /profile-jobs/active 显示那一个），按范围挑出 0 张表 409 profile_nothing；同源有起草任务时照建，排在它后面
|
||||
（waiting_for = "draft"），起草结束后自动开始。"""
|
||||
if payload.tables and len(payload.tables) > catalog_profile_jobs.MAX_TABLES:
|
||||
raise HTTPException(422, f"一次最多剖析 {catalog_profile_jobs.MAX_TABLES} 张表，当前 {len(payload.tables)} 张，"
|
||||
"请分几次剖析")
|
||||
row, source = await _profile_source(session, source_id)
|
||||
tables = [str(t) for t in payload.tables] if payload.tables else None
|
||||
scope = "selected" if tables else (payload.scope or "needed")
|
||||
return await _start_profile(session, row, source, tables=tables, scope=scope, actor=actor_of(x_actor))
|
||||
|
||||
|
||||
@router.get("/{source_id}/catalog/profile-jobs/active")
|
||||
async def active_profile_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""排队或进行中的那个剖析任务，没有是 null。"""
|
||||
row = await _source_row(session, source_id)
|
||||
return {"job": await catalog_profile_jobs.job_out(session, await catalog_profile_jobs.active_job(session, row.id))}
|
||||
|
||||
|
||||
@router.get("/{source_id}/catalog/profile-jobs/latest")
|
||||
async def latest_profile_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""最近建的一个剖析任务（任何状态），没有是 null。"""
|
||||
row = await _source_row(session, source_id)
|
||||
return {"job": await catalog_profile_jobs.job_out(session, await catalog_profile_jobs.latest_job(session, row.id))}
|
||||
|
||||
|
||||
async def _profile_job_or_404(session: AsyncSession, source_id: str, job_id: str) -> tuple[DataSource, Any]:
|
||||
row = await _source_row(session, source_id)
|
||||
job = await catalog_profile_jobs.load(session, row.id, job_id)
|
||||
if job is None:
|
||||
raise HTTPException(404, "剖析任务不存在，可能属于别的数据源，已随数据源删除，或者超过保留数被清理了")
|
||||
return row, job
|
||||
|
||||
|
||||
@router.get("/{source_id}/catalog/profile-jobs/{job_id}")
|
||||
async def get_profile_job(source_id: str, job_id: str,
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
_, job = await _profile_job_or_404(session, source_id, job_id)
|
||||
return {"job": await catalog_profile_jobs.job_out(session, job)}
|
||||
|
||||
|
||||
@router.get("/{source_id}/catalog/profile-jobs/{job_id}/tasks/{seq}")
|
||||
async def get_profile_task(source_id: str, job_id: str, seq: int,
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""剖析任务里的一张表（按剖析顺序的序号）：检查项、发现、跳过的项、日期范围、发过的查询（字面量已换成「?」）。"""
|
||||
_, job = await _profile_job_or_404(session, source_id, job_id)
|
||||
task = await catalog_profile_jobs.load_task(session, job, seq)
|
||||
if task is None:
|
||||
raise HTTPException(404, "这次剖析里没有这一张表")
|
||||
return {"task": catalog_profile_jobs.task_detail(job, task)}
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/profile-jobs/{job_id}/cancel")
|
||||
async def cancel_profile_job(source_id: str, job_id: str,
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""停止：排队的直接停止；进行中的不再等正在跑的那条查询（数据库按单条时限截止它），这张表做完的项照常写进目录、没做的
|
||||
保留旧结论，后面的表记停止。已经结束的 409 catalog_profile_finished。返回停止之后的任务。"""
|
||||
row, job = await _profile_job_or_404(session, source_id, job_id)
|
||||
owner = row.id
|
||||
try:
|
||||
await catalog_profile_jobs.cancel(session, job)
|
||||
except catalog_profile_jobs.ProfileFinished as e:
|
||||
raise CodedHTTPException(409, str(e), CATALOG_PROFILE_FINISHED) from e
|
||||
session.expire_all() # 后台在别的会话里收的尾：重新读
|
||||
return {"job": await catalog_profile_jobs.job_out(session, await catalog_profile_jobs.load(session, owner, job_id))}
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/profile-jobs/{job_id}/retry", status_code=202)
|
||||
async def retry_profile_job(source_id: str, job_id: str, x_actor: str | None = Header(default=None),
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""重试：新任务，只含原任务里 failed、partial、cancelled 的表（按原来的顺序），范围 selected，设置取数据源当前的。
|
||||
同源有排队或进行中的剖析任务 409 profile_busy；没有可重试的表 422；开始不了的几种情况同建任务。"""
|
||||
_, old = await _profile_job_or_404(session, source_id, job_id)
|
||||
row, source = await _profile_source(session, source_id)
|
||||
try:
|
||||
await catalog_profile_jobs.ensure_free(session, row)
|
||||
except catalog_profile_jobs.ProfileBusy as e:
|
||||
raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e
|
||||
try:
|
||||
tables = await catalog_profile_jobs.retry_tables(session, old)
|
||||
except catalog_profile_jobs.NothingToRetry as e:
|
||||
raise HTTPException(422, str(e)) from e
|
||||
return await _start_profile(session, row, source, tables=tables, scope="selected", actor=actor_of(x_actor),
|
||||
retry_of=old.id)
|
||||
|
||||
|
||||
@router.get("/{source_id}/catalog/{table}")
|
||||
async def get_catalog_table(source_id: str, table: str,
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
@@ -398,7 +563,7 @@ async def create_draft_job(source_id: str, payload: DraftJobIn, x_actor: str | N
|
||||
raise HTTPException(422, str(e)) from e
|
||||
row, source = await _resolved(session, source_id)
|
||||
try:
|
||||
await catalog_jobs.ensure_idle(session, row)
|
||||
await catalog_jobs.ensure_draft_free(session, row)
|
||||
except catalog_jobs.DraftBusy as e:
|
||||
raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e
|
||||
cache = source.schema_cache or {}
|
||||
@@ -412,7 +577,7 @@ async def create_draft_job(source_id: str, payload: DraftJobIn, x_actor: str | N
|
||||
except catalog_jobs.DraftBusy as e:
|
||||
raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e
|
||||
catalog_jobs.manager.start(job.id)
|
||||
return {"job": catalog_jobs.job_dict(job, tasks)}
|
||||
return {"job": catalog_jobs.job_dict(job, tasks, waiting_for=await catalog_jobs.waiting_for(session, job))}
|
||||
|
||||
|
||||
@router.get("/{source_id}/catalog/draft-jobs/active")
|
||||
@@ -466,7 +631,7 @@ async def retry_draft_job(source_id: str, job_id: str, x_actor: str | None = Hea
|
||||
_, old = await _job_or_404(session, source_id, job_id)
|
||||
row, source = await _resolved(session, source_id)
|
||||
try:
|
||||
await catalog_jobs.ensure_idle(session, row)
|
||||
await catalog_jobs.ensure_draft_free(session, row)
|
||||
except catalog_jobs.DraftBusy as e:
|
||||
raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e
|
||||
try:
|
||||
@@ -484,7 +649,7 @@ async def retry_draft_job(source_id: str, job_id: str, x_actor: str | None = Hea
|
||||
except catalog_jobs.DraftBusy as e:
|
||||
raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e
|
||||
catalog_jobs.manager.start(job.id)
|
||||
return {"job": catalog_jobs.job_dict(job, tasks)}
|
||||
return {"job": catalog_jobs.job_dict(job, tasks, waiting_for=await catalog_jobs.waiting_for(session, job))}
|
||||
|
||||
|
||||
@router.put("/{source_id}/catalog/{table}")
|
||||
@@ -529,6 +694,8 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn,
|
||||
guidance=catalog_jobs.guidance_of(row))
|
||||
except catalog_jobs.DraftBusy as e:
|
||||
raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e
|
||||
except catalog_profile_jobs.ProfileBusy as e:
|
||||
raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e
|
||||
rows = [{"table_name": r.table, "added": r.added, "updated": r.updated, "removed": r.removed,
|
||||
"version": r.version, "error": r.error, "model_error": r.model_error, "model_items": r.model_items}
|
||||
for r in report.tables]
|
||||
@@ -541,71 +708,6 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn,
|
||||
}
|
||||
|
||||
|
||||
def _profile_table_out(t: catalog_profile.TableProfile) -> dict[str, Any]:
|
||||
return {
|
||||
"table_name": t.table,
|
||||
"queries": t.queries,
|
||||
"row_estimate": t.row_estimate.to_dict() if t.row_estimate else None,
|
||||
"findings": t.findings,
|
||||
"skipped": [s.to_dict() for s in t.skipped],
|
||||
"date_ranges": t.date_ranges,
|
||||
"added": t.added,
|
||||
"updated": t.updated,
|
||||
"removed": t.removed,
|
||||
"version": t.version,
|
||||
"error": t.error,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/profile")
|
||||
async def profile_catalog_tables(source_id: str, payload: CatalogProfileIn,
|
||||
x_actor: str | None = Header(default=None),
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""同步剖析。返回每张表用了几条查询、写进目录的发现、跳过了什么及原因。
|
||||
|
||||
默认关闭：数据源设置里没开启数据剖析、数据源停用，409；同一个数据源已经在剖析，409。
|
||||
"""
|
||||
row, source = await _resolved(session, source_id)
|
||||
# 409 都带机读码（api/coded.py），界面按它分五种情况给下一步，不从原话里认
|
||||
try:
|
||||
catalog_profile.ensure_enabled(row)
|
||||
except catalog_profile.ProfileDisabled as e:
|
||||
raise CodedHTTPException(409, str(e), e.code) from e
|
||||
cache = source.schema_cache or {}
|
||||
if not cache.get("tables"):
|
||||
raise CodedHTTPException(409, f"{introspect.why_empty(cache)}，无法进行数据剖析。请先在数据源卡片上点「探查结构」",
|
||||
SCHEMA_MISSING)
|
||||
actor = actor_of(x_actor)
|
||||
try:
|
||||
report = await catalog_profile.profile_catalog(session, row, source, tables=payload.tables or None,
|
||||
actor=actor)
|
||||
except catalog_profile.ProfileDisabled as e:
|
||||
raise CodedHTTPException(409, str(e), e.code) from e
|
||||
except catalog_profile.ProfileBusy as e:
|
||||
raise CodedHTTPException(409, str(e), PROFILE_BUSY) from e
|
||||
except SnapshotTampered as e:
|
||||
raise CodedHTTPException(409, str(e), SNAPSHOT_TAMPERED) from e
|
||||
rows = [_profile_table_out(t) for t in report.tables]
|
||||
note = None
|
||||
if not rows and report.empty_tables:
|
||||
note = (f"有待核对关联关系的 {len(report.empty_tables)} 张表都是空表，没有可以核实的数据。"
|
||||
"请在数据有了之后再剖析，或指定要剖析的表")
|
||||
elif not rows:
|
||||
note = "目录里还没有待核实的关联关系，请先起草数据目录，或指定要剖析的表"
|
||||
return {
|
||||
"profiled_at": report.profiled_at,
|
||||
"actor": actor,
|
||||
"settings": report.settings.to_dict(),
|
||||
"queries_used": report.queries_used,
|
||||
"stopped": report.stopped,
|
||||
"tables": rows,
|
||||
"total": {k: sum(r[k] for r in rows) for k in ("added", "updated", "removed")},
|
||||
# 不指定表时挑表途中确知是空表、跳过了的表（指定了表时总是空列表）
|
||||
"empty_tables": report.empty_tables,
|
||||
"note": note,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/{table}/review")
|
||||
async def review_catalog_item(source_id: str, table: str, payload: CatalogReviewIn,
|
||||
x_actor: str | None = Header(default=None),
|
||||
|
||||
@@ -11,14 +11,18 @@ from fastapi.responses import JSONResponse
|
||||
#: 限定的数据源一个都不在了（删了或停用了），见 copilot._sources
|
||||
DATASOURCE_SCOPE_EMPTY = "datasource_scope_empty"
|
||||
|
||||
#: 保存数据源时剖析设置不合规（422）：field 是出错的那一项（max_queries……），说不清是哪一项时没有 field
|
||||
#: 保存数据源时剖析设置不合规（422）：field 是出错的那一项（table_max_queries……），说不清是哪一项时没有 field
|
||||
PROFILE_SETTINGS_INVALID = "profile_settings_invalid"
|
||||
#: 剖析开始不了（409）的五种情况，见 api/catalog.profile_catalog_tables
|
||||
#: 剖析任务开始不了（409，api/catalog.create_profile_job）：未开启、数据源停用、没有表结构、快照被改动、这个源已有排队或
|
||||
#: 进行中的剖析任务（导入目录、同步起草遇到剖析任务也是它）、按范围挑出 0 张表
|
||||
PROFILE_DISABLED = "profile_disabled"
|
||||
DATASOURCE_INACTIVE = "datasource_inactive"
|
||||
SCHEMA_MISSING = "schema_missing"
|
||||
PROFILE_BUSY = "profile_busy"
|
||||
SNAPSHOT_TAMPERED = "snapshot_tampered"
|
||||
PROFILE_NOTHING = "profile_nothing"
|
||||
#: 要停止的剖析任务已经结束了（409）
|
||||
CATALOG_PROFILE_FINISHED = "catalog_profile_finished"
|
||||
#: 数据目录的后台起草任务（api/catalog.py 的 /catalog/draft-jobs）：同一个源已有排队或进行中的任务（409，界面改去
|
||||
#: GET /draft-jobs/active 显示那一个）、起草模型用不了（409，不建任务）、要停止的任务已经结束了（409）
|
||||
CATALOG_DRAFT_BUSY = "catalog_draft_busy"
|
||||
|
||||
@@ -23,7 +23,7 @@ from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw
|
||||
from app.core.config import settings
|
||||
from app.core.crypto import encrypt, mask
|
||||
from app.data import catalog as data_catalog
|
||||
from app.data.catalog_profile import profile_settings_rejection
|
||||
from app.data.catalog_profile import profile_settings_rejection, without_legacy
|
||||
from app.data import introspect as introspect_mod
|
||||
from app.data import table_versions
|
||||
from app.data.engine import (
|
||||
@@ -465,12 +465,13 @@ async def create_source(
|
||||
)).scalar_one_or_none()
|
||||
if exists:
|
||||
raise HTTPException(409, f"已存在名为「{payload.name}」的数据源，请换一个标识")
|
||||
_refuse_bad_options(payload.options)
|
||||
options = without_legacy(payload.options) or {}
|
||||
_refuse_bad_options(options)
|
||||
_refuse_reserved(payload.kind, payload.database)
|
||||
|
||||
row = DataSource(
|
||||
**payload.model_dump(exclude={"password", "options"}),
|
||||
options=_with_managed(payload.options, None),
|
||||
options=_with_managed(options, None),
|
||||
password=encrypt(payload.password),
|
||||
)
|
||||
_settle_oracle(row, None, None)
|
||||
@@ -519,6 +520,8 @@ async def update_source(
|
||||
# 升级前就指向上传目录的手工源：启动时已强制只读（table_versions.startup），不许再改回可写
|
||||
raise HTTPException(422, _UPLOAD_DIR_READONLY)
|
||||
if "options" in data:
|
||||
# 剖析设置里还带着以前按整次剖析算的预算（老页面提交的）：只摘掉，不报错（catalog_profile.LEGACY_KEYS）
|
||||
data["options"] = without_legacy(data["options"])
|
||||
_refuse_bad_options(data["options"])
|
||||
data["options"] = _with_managed(data["options"] or {}, row.options)
|
||||
if "password" in data:
|
||||
|
||||
@@ -421,9 +421,12 @@ async def read_catalog(session: AsyncSession, source_id: str) -> dict[str, Catal
|
||||
|
||||
|
||||
async def write_entry(session: AsyncSession, source_id: str, table: str, notes: dict[str, Any], *,
|
||||
if_version: int, actor: str | None) -> CatalogEntry:
|
||||
if_version: int, actor: str | None, commit: bool = True) -> CatalogEntry:
|
||||
"""整份写入一张表的目录（乐观锁），提交事务，返回写入后的目录。
|
||||
|
||||
commit=False 只写进当前事务、不提交，由调用方和它自己的写入一起提交：剖析一张表结束时，目录和这张表的剖析结果要么
|
||||
都写进去、要么都没写（进程在两次提交之间没了，剖析结果会说「没写进目录」，目录里其实已经写了）。冲突时照样回滚。
|
||||
|
||||
- if_version 是写入方读到的版本，还没有目录时传 0；和库里的不一致抛 CatalogConflict，什么都不写。
|
||||
比对和写入在同一条 UPDATE … WHERE version = ? 里完成：两个请求同时读到第 3 版，只有一个写得进去。
|
||||
- 内容和库里的完全相同时不写、不升版本：冻结进表结构快照的是「版本 + 内容」，空写一次也升版本，
|
||||
@@ -446,7 +449,7 @@ async def write_entry(session: AsyncSession, source_id: str, table: str, notes:
|
||||
row = CatalogNote(source_id=source_id, table_name=table, notes=stored, version=1, updated_by=actor)
|
||||
session.add(row)
|
||||
try:
|
||||
await session.commit()
|
||||
await (session.commit() if commit else session.flush())
|
||||
except IntegrityError as e:
|
||||
# 唯一约束 (source_id, table_name)：另一个请求抢先建了这张表的目录
|
||||
await session.rollback()
|
||||
@@ -461,7 +464,8 @@ async def write_entry(session: AsyncSession, source_id: str, table: str, notes:
|
||||
if result.rowcount != 1:
|
||||
await session.rollback()
|
||||
raise CatalogConflict(table, current + 1)
|
||||
await session.commit()
|
||||
if commit:
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return _entry(row)
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
是这个值又确认过的不提；有效记录条件是模型写的 SQL，另按起草的规矩核对（_Tools.filter_problem：一个条件、只用这张表的
|
||||
列、不能有子查询……）。核对过的作为 proposal 事件交给界面，**不写目录**。没收下的项和原因交回模型（不带任何取值）。
|
||||
界面上还没保存的修改（pending）里同一列的码值并进新提的码值：界面上同一路径以后一次为准，不能把先提的码值丢了。
|
||||
- run_query：只读查询，只有数据源开了数据剖析才给。经剖析的只读守卫和单条时限（catalog_profile._Runner：
|
||||
- run_query：只读查询，只有数据源开了数据剖析才给。经剖析的单条查询入口（catalog_profile.run_single_query：
|
||||
guard.check(readonly=True)，单条时限取剖析设置和数据源查询时限中较小的一个），每次请求最多 MAX_QUERIES 条（守卫拒掉的
|
||||
也算），最多 QUERY_ROWS 行交给模型、SHOWN_ROWS 行交给界面。结果交给模型、界面之前先遮（engine/masking.mask_query_result：
|
||||
取自遮罩列的结果列整列换成「已遮罩」，改了名字的也认；这一次请求里前面遮掉的值按值换），报错原话里的值也换。没开剖析时
|
||||
@@ -52,7 +52,6 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import AsyncIterator, Iterable, Mapping
|
||||
@@ -621,15 +620,11 @@ class _Tools:
|
||||
known = known if known.strong else None
|
||||
error: str | None = None
|
||||
result: Any = None
|
||||
settings = dataclasses.replace(self.ctx.settings, max_queries=1)
|
||||
try:
|
||||
runner = catalog_profile._Runner(self.ctx.query_source, settings,
|
||||
catalog_profile.SqlDialect(self.ctx.kind))
|
||||
result = await runner(sql, max_rows=QUERY_ROWS)
|
||||
except catalog_profile._Skip as e:
|
||||
result = await catalog_profile.run_single_query(self.ctx.query_source, sql, settings=self.ctx.settings,
|
||||
max_rows=QUERY_ROWS, kind=self.ctx.kind)
|
||||
except catalog_profile.QueryFailed as e:
|
||||
error = e.detail
|
||||
except catalog_profile._Stop as e:
|
||||
error = str(e)
|
||||
except SnapshotTampered as e:
|
||||
error = QUERY_TAMPERED.format(name=self.ctx.source_name, error=e)
|
||||
except ValueError as e: # 不支持的数据库类型
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
"""数据目录后台任务的共用运行时：起草（data/catalog_jobs.py）和剖析（data/catalog_profile_jobs.py）各自建表、各自的步骤，
|
||||
生命周期是同一套。
|
||||
|
||||
**为什么抽出来。** 剖析以前是一次同步长请求，和起草改成后台任务之前是同一类问题：关掉页面结果就没了、不能停、不留痕。
|
||||
两边要的是同一样东西——进程内跑、能停、重启标中断、写库不被停止打断、同源排队——各写一份，迟早一边修了另一边没修。
|
||||
共用的是生命周期代码，不是表：两种任务的进度完全不同（起草按段、块、模型用量，剖析按检查项、查询），合成一张
|
||||
「kind + JSON」的通用任务表要丢掉起草那些强类型列，还要迁移刚稳定的起草数据。
|
||||
|
||||
- **状态词**（ACTIVE、FINISHED、TASK_STATUSES、TASK_DONE、RETRYABLE、OPEN）：任务 queued → running → succeeded / partial
|
||||
/ failed / cancelled / interrupted；逐表 pending → running → ok / partial / failed / skipped / cancelled。两种任务、
|
||||
接口、界面都用这一套词。
|
||||
- **SerialWriter**：同一个任务的库写入一条一条来，shield 起来——停止任务时不会打断写到一半的库操作；收尾前 settle 等它们
|
||||
写完。
|
||||
- **JobRun / JobManager**：进程内一个 asyncio 任务一件事（照 app/scheduler 的写法）：start / stop(grace) / wait / running
|
||||
/ shutdown。JobRun.run 把「进队、等轮到自己、执行、按停止的原因收尾、出队」写一次，各种任务只写执行和收尾的那几步。
|
||||
- **SourceLanes**：每个数据源一条先来先到的队。起草、剖析都打业务库或模型、都写同一个源的目录：同源串行（先建的先跑），
|
||||
不同源并行。任务阻塞地占：建出来是 queued，轮到了才 running。同步起草、导入目录这类同步写操作不阻塞地试占：占不到
|
||||
说明源上正有别的事，接口回 409；占到了，执行期间新建的任务排在它后面。
|
||||
- **close_job / interrupt_on_startup**：收尾（还没结束的表记成某个状态，任务记成某个状态），和服务启动时的整理：排队、
|
||||
进行中的标 interrupted，没结束的表记 cancelled。**不自动续跑**：复制出来的数据目录（检查脚本、验收副本）启动时不能
|
||||
自己去调模型、查业务库，界面让人点「重试」接着做。
|
||||
|
||||
进程内登记、进程内的队：服务端是单进程部署，多进程部署时要换成库里的锁。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.base import SessionLocal, utcnow
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: 任务还没结束的状态（同一个源同一种任务只能有一个）、结束了的状态
|
||||
ACTIVE = ("queued", "running")
|
||||
FINISHED = ("succeeded", "partial", "failed", "cancelled", "interrupted")
|
||||
#: 逐表的状态，其中算「已结束」的、重试时要带上的、还没结束的
|
||||
TASK_STATUSES = ("pending", "running", "ok", "partial", "failed", "skipped", "cancelled")
|
||||
TASK_DONE = ("ok", "partial", "failed", "skipped", "cancelled")
|
||||
RETRYABLE = ("failed", "partial", "cancelled")
|
||||
OPEN = ("pending", "running")
|
||||
|
||||
#: 停止进行中的任务时最多等它收尾多久（秒）；关停服务时另给（shutdown 的 grace）
|
||||
STOP_GRACE_S = 10.0
|
||||
SHUTDOWN_GRACE_S = 3.0
|
||||
|
||||
|
||||
class JobFailed(Exception):
|
||||
"""任务整体出错（数据源没了、快照没了、模型用不了……）：message 就是任务的 error。"""
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 库写入
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
class SerialWriter:
|
||||
"""一个任务的库写入：一条一条来，每条开一个会话；shield 起来，停止任务时不会打断写到一半的库操作。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._writes: set[asyncio.Task[Any]] = set()
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def __call__(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any:
|
||||
async def go() -> Any:
|
||||
async with self._lock:
|
||||
async with SessionLocal() as session:
|
||||
return await fn(session, *args)
|
||||
|
||||
task = asyncio.ensure_future(go())
|
||||
self._writes.add(task)
|
||||
task.add_done_callback(self._writes.discard)
|
||||
return await asyncio.shield(task)
|
||||
|
||||
async def settle(self) -> None:
|
||||
"""等手上的库写入都写完（停止、出错收尾之前）。"""
|
||||
while self._writes:
|
||||
await asyncio.gather(*list(self._writes), return_exceptions=True)
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 每个源一条队
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
class LaneTicket:
|
||||
"""在一个源的队里的一个位置。kind 是占着它的是什么：draft 起草任务、profile 剖析任务、sync_draft 同步起草、
|
||||
import 导入目录；job_id 是任务的（同步写操作没有）。轮到它时 granted 置位。"""
|
||||
|
||||
def __init__(self, lanes: SourceLanes, source_id: str, kind: str, job_id: str | None,
|
||||
loop: asyncio.AbstractEventLoop) -> None:
|
||||
self._lanes = lanes
|
||||
self.source_id = source_id
|
||||
self.kind = kind
|
||||
self.job_id = job_id
|
||||
self.loop = loop
|
||||
self.granted = asyncio.Event()
|
||||
self.released = False
|
||||
|
||||
async def wait(self) -> None:
|
||||
"""等轮到自己。"""
|
||||
await self.granted.wait()
|
||||
|
||||
def release(self) -> None:
|
||||
"""出队（轮到了、没轮到都行，重复调用无害）：排在后面的第一个接着开始。"""
|
||||
self._lanes._release(self)
|
||||
|
||||
|
||||
class LaneBusy(Exception):
|
||||
"""不阻塞地试占时这个源的队里已经有人：holder 是排在最前面（正在做）的那一个。"""
|
||||
|
||||
def __init__(self, holder: LaneTicket) -> None:
|
||||
super().__init__(holder.kind)
|
||||
self.holder = holder
|
||||
|
||||
|
||||
class SourceLanes:
|
||||
"""每个数据源一条先来先到的队。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._queues: dict[str, list[LaneTicket]] = {}
|
||||
|
||||
def _queue(self, source_id: str) -> list[LaneTicket]:
|
||||
"""这个源的队。别的事件循环留下的（测试每个用例一个循环）去掉：那个循环里没人会再出队了。"""
|
||||
queue = self._queues.get(source_id) or []
|
||||
loop = asyncio.get_running_loop()
|
||||
alive = [t for t in queue if t.loop is loop and not t.released]
|
||||
if alive:
|
||||
self._queues[source_id] = alive
|
||||
alive[0].granted.set()
|
||||
else:
|
||||
self._queues.pop(source_id, None)
|
||||
return alive
|
||||
|
||||
def enter(self, source_id: str, kind: str, job_id: str | None = None) -> LaneTicket:
|
||||
"""排进队尾，马上返回（不等）。排在最前面的立刻 granted；调用方 await ticket.wait() 等轮到自己。"""
|
||||
queue = self._queue(source_id)
|
||||
ticket = LaneTicket(self, source_id, kind, job_id, asyncio.get_running_loop())
|
||||
queue.append(ticket)
|
||||
self._queues[source_id] = queue
|
||||
if len(queue) == 1:
|
||||
ticket.granted.set()
|
||||
return ticket
|
||||
|
||||
def try_enter(self, source_id: str, kind: str) -> LaneTicket:
|
||||
"""队空着才占（马上就轮到）；有人在队里抛 LaneBusy。"""
|
||||
queue = self._queue(source_id)
|
||||
if queue:
|
||||
raise LaneBusy(queue[0])
|
||||
return self.enter(source_id, kind)
|
||||
|
||||
def holders(self, source_id: str) -> list[LaneTicket]:
|
||||
"""队里的各位，排在最前面（正在做）的在前。"""
|
||||
return list(self._queue(source_id))
|
||||
|
||||
def ahead(self, source_id: str, job_id: str) -> list[str] | None:
|
||||
"""排在这个任务前面的各位是什么（kind，正在做的在前）；这个任务不在队里（刚建、后台还没排进来）返回 None。"""
|
||||
queue = self._queue(source_id)
|
||||
for i, ticket in enumerate(queue):
|
||||
if ticket.job_id == job_id:
|
||||
return [t.kind for t in queue[:i]]
|
||||
return None
|
||||
|
||||
def _release(self, ticket: LaneTicket) -> None:
|
||||
if ticket.released:
|
||||
return
|
||||
ticket.released = True
|
||||
queue = self._queues.get(ticket.source_id)
|
||||
if not queue or ticket not in queue:
|
||||
return
|
||||
head = queue[0] is ticket
|
||||
queue.remove(ticket)
|
||||
if not queue:
|
||||
self._queues.pop(ticket.source_id, None)
|
||||
elif head:
|
||||
queue[0].granted.set()
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def hold(self, source_id: str, kind: str, job_id: str | None = None) -> AsyncIterator[LaneTicket]:
|
||||
"""阻塞地占：排队，轮到了再进去，出来时出队。"""
|
||||
ticket = self.enter(source_id, kind, job_id)
|
||||
try:
|
||||
await ticket.wait()
|
||||
yield ticket
|
||||
finally:
|
||||
ticket.release()
|
||||
|
||||
@contextlib.contextmanager
|
||||
def try_hold(self, source_id: str, kind: str) -> Iterator[LaneTicket]:
|
||||
"""不阻塞地试占（同步起草、导入目录）：占不到抛 LaneBusy；占到了，执行期间新建的任务排在后面。"""
|
||||
ticket = self.try_enter(source_id, kind)
|
||||
try:
|
||||
yield ticket
|
||||
finally:
|
||||
ticket.release()
|
||||
|
||||
|
||||
lanes = SourceLanes()
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 收尾、启动时的整理
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
async def close_job(session: AsyncSession, job_model: Any, task_model: Any, job_id: str, job_status: str,
|
||||
job_error: str | None, task_status: str, task_error: str | None) -> None:
|
||||
"""收尾：还没结束的表记 task_status（原因 task_error），任务（还没结束的话）记 job_status。提交。"""
|
||||
now = utcnow()
|
||||
await session.execute(
|
||||
update(task_model).where(task_model.job_id == job_id, task_model.status.in_(OPEN))
|
||||
.values(status=task_status, error=task_error, finished_at=now).execution_options(synchronize_session=False))
|
||||
await session.execute(
|
||||
update(job_model).where(job_model.id == job_id, job_model.status.in_(ACTIVE))
|
||||
.values(status=job_status, error=job_error, finished_at=now).execution_options(synchronize_session=False))
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def cancel_queued(session: AsyncSession, job_model: Any, task_model: Any, job_id: str) -> bool:
|
||||
"""停止一个排队中的任务：任务和没结束的表直接记 cancelled。已经被后台认领（开始了）返回 False，交给调用方按进行中
|
||||
的停。"""
|
||||
now = utcnow()
|
||||
claimed = await session.execute(
|
||||
update(job_model).where(job_model.id == job_id, job_model.status == "queued")
|
||||
.values(status="cancelled", finished_at=now).execution_options(synchronize_session=False))
|
||||
if claimed.rowcount != 1:
|
||||
await session.rollback()
|
||||
return False
|
||||
await session.execute(
|
||||
update(task_model).where(task_model.job_id == job_id, task_model.status.in_(OPEN))
|
||||
.values(status="cancelled", finished_at=now).execution_options(synchronize_session=False))
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def claim(session: AsyncSession, job_model: Any, job_id: str) -> bool:
|
||||
"""认领：排队中 → 进行中。开始之前就被停止了返回 False。"""
|
||||
result = await session.execute(
|
||||
update(job_model).where(job_model.id == job_id, job_model.status == "queued")
|
||||
.values(status="running", started_at=utcnow()).execution_options(synchronize_session=False))
|
||||
await session.commit()
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
async def interrupt_on_startup(job_model: Any, close: Callable[[AsyncSession, str], Awaitable[None]], *,
|
||||
running: Callable[[str], bool]) -> int:
|
||||
"""服务启动时：排队、进行中、而且不在这个进程里跑的任务逐个 close（各自记成 interrupted、没结束的表记 cancelled）。
|
||||
不自动续跑。返回整理了几个。"""
|
||||
async with SessionLocal() as session:
|
||||
ids = [job_id for job_id in (await session.execute(
|
||||
select(job_model.id).where(job_model.status.in_(ACTIVE)))).scalars() if not running(job_id)]
|
||||
for job_id in ids:
|
||||
await close(session, job_id)
|
||||
return len(ids)
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 在后台执行
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
class JobRun:
|
||||
"""一个任务在后台的一次执行。子类给 kind（进队时的身份）、model（任务表，进队前读源和状态），写 _run（执行）和
|
||||
_stopped / _failed / _crashed（停止、整体出错、意外出错时的收尾）。"""
|
||||
|
||||
kind = ""
|
||||
model: Any = None
|
||||
|
||||
def __init__(self, job_id: str) -> None:
|
||||
self.job_id = job_id
|
||||
self.task: asyncio.Task[None] | None = None
|
||||
#: 为什么停：user 使用者点了停止，shutdown 服务关停（标中断）
|
||||
self.stop_reason: str | None = None
|
||||
self.writer = SerialWriter()
|
||||
self.ticket: LaneTicket | None = None
|
||||
|
||||
async def _write(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any:
|
||||
"""一条库写入（SerialWriter：一条一条来，停止时不被打断）。"""
|
||||
return await self.writer(fn, *args)
|
||||
|
||||
async def _settle(self) -> None:
|
||||
await self.writer.settle()
|
||||
|
||||
async def run(self) -> None:
|
||||
try:
|
||||
try:
|
||||
await self._queue()
|
||||
await self._run()
|
||||
except asyncio.CancelledError:
|
||||
current = asyncio.current_task()
|
||||
if current is not None:
|
||||
current.uncancel()
|
||||
await self._stopped()
|
||||
except JobFailed as e:
|
||||
await self._failed(str(e))
|
||||
except Exception as e: # noqa: BLE001 - 任务整体出错：照实记下，不留在「进行中」
|
||||
await self._crashed(e)
|
||||
finally:
|
||||
# 收尾写完才出队：排在后面的任务开始时，这个任务已经记成结束了
|
||||
if self.ticket is not None:
|
||||
self.ticket.release()
|
||||
|
||||
async def _queue(self) -> None:
|
||||
"""进这个源的队，等轮到自己。任务不在了、已经不是排队中（开始之前就被停止了）的不进队：_run 认领不到，直接结束。"""
|
||||
async with SessionLocal() as session:
|
||||
job = await session.get(self.model, self.job_id)
|
||||
source_id = job.source_id if job is not None and job.status == "queued" else None
|
||||
if source_id is None:
|
||||
return
|
||||
self.ticket = lanes.enter(source_id, self.kind, self.job_id)
|
||||
await self.ticket.wait()
|
||||
|
||||
async def _run(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def _stopped(self) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def _failed(self, reason: str) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def _crashed(self, e: Exception) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class JobManager:
|
||||
"""进程内的一种后台任务：一个 asyncio 任务一件事，停的时候给宽限。factory 按任务编号造一次执行（JobRun）；label 是
|
||||
日志里的叫法（「数据目录起草任务」），name 是 asyncio 任务名的前缀。"""
|
||||
|
||||
def __init__(self, factory: Callable[[str], JobRun], *, label: str, name: str) -> None:
|
||||
self._runs: dict[str, JobRun] = {}
|
||||
self._factory = factory
|
||||
self.label = label
|
||||
self.name = name
|
||||
|
||||
def _alive(self, job_id: str) -> JobRun | None:
|
||||
run = self._runs.get(job_id)
|
||||
if run is None or run.task is None or run.task.done():
|
||||
return None
|
||||
try:
|
||||
if run.task.get_loop() is not asyncio.get_running_loop():
|
||||
self._runs.pop(job_id, None) # 别的事件循环留下的（测试每个用例一个循环）
|
||||
return None
|
||||
except RuntimeError:
|
||||
return None
|
||||
return run
|
||||
|
||||
def start(self, job_id: str) -> None:
|
||||
"""开始在后台执行这个（排队中的）任务：先进这个源的队，轮到了才开始。"""
|
||||
run = self._factory(job_id)
|
||||
run.task = asyncio.get_running_loop().create_task(run.run(), name=f"{self.name}-{job_id}")
|
||||
self._runs[job_id] = run
|
||||
|
||||
def forget(_: asyncio.Task[None]) -> None:
|
||||
if self._runs.get(job_id) is run:
|
||||
self._runs.pop(job_id, None)
|
||||
|
||||
run.task.add_done_callback(forget)
|
||||
|
||||
def running(self, job_id: str) -> bool:
|
||||
return self._alive(job_id) is not None
|
||||
|
||||
async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S) -> bool:
|
||||
"""停掉这个进程里在跑（或在排队等）的任务，等它收尾（最多 grace 秒）。不在这个进程里返回 False。"""
|
||||
run = self._alive(job_id)
|
||||
if run is None:
|
||||
return False
|
||||
if run.stop_reason is None:
|
||||
run.stop_reason = reason
|
||||
run.task.cancel() # type: ignore[union-attr]
|
||||
await asyncio.wait([run.task], timeout=grace) # type: ignore[list-item]
|
||||
return True
|
||||
|
||||
async def wait(self, job_id: str, timeout: float | None = None) -> None:
|
||||
"""等这个任务做完（测试、调试用）。不在跑直接返回。"""
|
||||
run = self._alive(job_id)
|
||||
if run is not None:
|
||||
await asyncio.wait([run.task], timeout=timeout) # type: ignore[list-item]
|
||||
|
||||
async def shutdown(self, *, grace: float = SHUTDOWN_GRACE_S) -> None:
|
||||
"""关停：停掉进行中、排队等着的任务（标中断），最多等 grace 秒收尾。"""
|
||||
runs = [run for job_id in list(self._runs) if (run := self._alive(job_id)) is not None]
|
||||
for run in runs:
|
||||
if run.stop_reason is None:
|
||||
run.stop_reason = "shutdown"
|
||||
run.task.cancel() # type: ignore[union-attr]
|
||||
if runs:
|
||||
await asyncio.wait([run.task for run in runs], timeout=grace) # type: ignore[misc]
|
||||
logger.info("已停止 %d 个%s", len(runs), self.label)
|
||||
|
||||
|
||||
__all__ = ["ACTIVE", "FINISHED", "JobFailed", "JobManager", "JobRun", "LaneBusy", "LaneTicket", "OPEN", "RETRYABLE",
|
||||
"SHUTDOWN_GRACE_S", "STOP_GRACE_S", "SerialWriter", "SourceLanes", "TASK_DONE", "TASK_STATUSES", "cancel_queued",
|
||||
"claim", "close_job", "interrupt_on_startup", "lanes"]
|
||||
+172
-175
@@ -13,15 +13,20 @@
|
||||
接口读的是库里的状态。
|
||||
- **停止**（cancel）：排队的直接 cancelled；进行中的停掉正在进行的模型调用，没开始和没写完的表记 cancelled，已经写进
|
||||
目录的（含宽表已经写出来的段）不回滚，任务最后是 cancelled。写过段的表 model_error 说清写进了几段、剩几段没起草。
|
||||
- **和同步起草互斥**（sync_draft）：同步起草（POST /catalog/draft）进行期间，同一个源建任务、重试、导入（ensure_idle）
|
||||
都是 DraftBusy；有排队或进行中的任务时同步起草也是。进程内登记（单进程部署）。
|
||||
- **同种任务互斥**（ensure_draft_free）：同一个源同时只有一个排队或进行中的起草任务，同步起草（POST /catalog/draft）
|
||||
进行期间也算在起草，建任务、重试都是 DraftBusy。剖析任务不挡起草：同源有剖析任务时起草任务照建，排在它后面
|
||||
（waiting_for = "profile"），剖析结束后自动开始（每个源一条队，见 data/catalog_job_runtime.py）。
|
||||
- **同步写操作要求源空闲**（ensure_idle；sync_draft、import_write 占队）：同步起草、导入目录要的是这个源上没有别的事——有排队或进行中的
|
||||
起草任务（DraftBusy）、剖析任务（catalog_profile_jobs.ProfileBusy）都不开始；开始了就占着这个源的队，执行期间
|
||||
新建的任务排在它后面。进程内登记（单进程部署）。
|
||||
- **重试**（retry_tables + create_job）：新任务，只含原任务里 failed、partial、cancelled 的表；用不用模型、本次说明沿用，
|
||||
起草要求取数据源当前的。
|
||||
- **服务重启**：启动时（startup）把排队、进行中的任务标成 interrupted，没结束的表记 cancelled、写明「服务重启，起草
|
||||
中断」。不自动续跑：复制出来的数据目录（检查、验收）启动时不能自己去调模型；界面让人点「重试」续上。关停时
|
||||
（manager.shutdown）停掉进行中的任务，同样标成 interrupted。
|
||||
|
||||
任务的库写入一律经 _JobRun._write：一条一条来，shield 起来——停止任务时不会打断写到一半的库操作，收尾前等它们写完。
|
||||
任务的生命周期（进程内执行、停止、同源排队、库写入一条一条来并 shield、启动时标中断、状态词）在
|
||||
data/catalog_job_runtime.py，剖析任务用的是同一套；这里只写起草自己的步骤。
|
||||
|
||||
**起草要求**存在数据源 options 的 catalog_draft_guidance 里（上传表格的源存在它自己的 DataSource 行上），一段文字，
|
||||
去首尾空白、最多 GUIDANCE_MAX 字，空串等于没有。只经 PUT /catalog/guidance 改：数据源接口不显示它、整份回写 options
|
||||
@@ -40,20 +45,15 @@ from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.data import catalog, catalog_draft
|
||||
from app.data import catalog_job_runtime as runtime
|
||||
from app.data.catalog_job_runtime import ACTIVE, FINISHED, RETRYABLE, STOP_GRACE_S, TASK_DONE, TASK_STATUSES
|
||||
from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION
|
||||
from app.db.base import SessionLocal, new_id, utcnow
|
||||
from app.db.models import CatalogDraftJob, CatalogDraftTask, DataSource
|
||||
from app.db.models import CatalogDraftJob, CatalogDraftTask, CatalogProfileJob, DataSource
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: 任务还没结束的状态（同一个源只能有一个）、结束了的状态
|
||||
ACTIVE = ("queued", "running")
|
||||
FINISHED = ("succeeded", "partial", "failed", "cancelled", "interrupted")
|
||||
#: 逐表的状态，以及其中算「已结束」的、重试时要带上的
|
||||
TASK_STATUSES = ("pending", "running", "ok", "partial", "failed", "skipped", "cancelled")
|
||||
TASK_DONE = ("ok", "partial", "failed", "skipped", "cancelled")
|
||||
RETRYABLE = ("failed", "partial", "cancelled")
|
||||
_OPEN = ("pending", "running")
|
||||
_OPEN = runtime.OPEN
|
||||
|
||||
#: 一个任务最多几张表；起草要求、本次说明最多几个字
|
||||
MAX_TABLES = 500
|
||||
@@ -63,9 +63,6 @@ NOTE_MAX = 1000
|
||||
LAST_DRAFT_JOBS = 100
|
||||
#: 逐表最多记几个模型留空的列名
|
||||
BLANK_SAMPLE = 5
|
||||
#: 停止进行中的任务时最多等它收尾多久（秒）；关停服务时另给（shutdown 的 grace）
|
||||
STOP_GRACE_S = 10.0
|
||||
|
||||
INTERRUPTED_ERROR = "服务重启，起草中断"
|
||||
#: 停止、中断时写过段的表（宽表每写出一段就并入目录）：写进了几段、剩几段没起草
|
||||
STOPPED_WRITTEN = "已停止：已写进目录 {n} 段，剩下 {m} 段没起草"
|
||||
@@ -93,7 +90,7 @@ class TextTooLong(ValueError):
|
||||
"""起草要求、本次说明超长。message 给人看。"""
|
||||
|
||||
|
||||
class _JobFailed(Exception):
|
||||
class _JobFailed(runtime.JobFailed):
|
||||
"""任务整体出错（数据源没了、快照没了、模型用不了）：message 就是任务的 error。"""
|
||||
|
||||
|
||||
@@ -200,13 +197,15 @@ def _counts(statuses: list[str]) -> dict[str, int]:
|
||||
return counts
|
||||
|
||||
|
||||
def job_dict(job: CatalogDraftJob, tasks: list[CatalogDraftTask]) -> dict[str, Any]:
|
||||
"""接口返回的任务（CatalogDraftJob）：逐表的状态按起草顺序；计数、合计都按逐表的算。"""
|
||||
def job_dict(job: CatalogDraftJob, tasks: list[CatalogDraftTask], *, waiting_for: str | None = None) -> dict[str, Any]:
|
||||
"""接口返回的任务（CatalogDraftJob）：逐表的状态按起草顺序；计数、合计都按逐表的算。waiting_for：排队中、排在同源的
|
||||
剖析任务后面时是 "profile"（见 waiting_for），否则 null。"""
|
||||
counts = _counts([t.status for t in tasks])
|
||||
return {
|
||||
"id": job.id,
|
||||
"source_id": job.source_id,
|
||||
"status": job.status,
|
||||
"waiting_for": waiting_for,
|
||||
"use_model": bool(job.use_model),
|
||||
"model": job.model,
|
||||
"guidance": job.guidance,
|
||||
@@ -235,12 +234,32 @@ def job_dict(job: CatalogDraftJob, tasks: list[CatalogDraftTask]) -> dict[str, A
|
||||
}
|
||||
|
||||
|
||||
async def waiting_for(session: AsyncSession, job: CatalogDraftJob) -> str | None:
|
||||
"""排队中的起草任务在等什么：排在同源的剖析任务后面时是 "profile"，否则 None（进行中、结束了的都是 None）。
|
||||
|
||||
先看这个源的队（进程内，排在它前面的是谁）；任务刚建、后台还没排进队时看库：同源有比它先建的、排队或进行中的剖析
|
||||
任务。"""
|
||||
if job.status != "queued":
|
||||
return None
|
||||
ahead = runtime.lanes.ahead(job.source_id, job.id)
|
||||
if ahead is not None:
|
||||
return "profile" if "profile" in ahead else None
|
||||
earlier = (await session.execute(
|
||||
select(CatalogProfileJob.id).where(CatalogProfileJob.source_id == job.source_id,
|
||||
CatalogProfileJob.status.in_(ACTIVE),
|
||||
CatalogProfileJob.created_at <= job.created_at).limit(1)
|
||||
)).first()
|
||||
return "profile" if earlier is not None else None
|
||||
|
||||
|
||||
async def job_out(session: AsyncSession, job: CatalogDraftJob | None) -> dict[str, Any] | None:
|
||||
return None if job is None else job_dict(job, await tasks_of(session, job.id))
|
||||
if job is None:
|
||||
return None
|
||||
return job_dict(job, await tasks_of(session, job.id), waiting_for=await waiting_for(session, job))
|
||||
|
||||
|
||||
async def job_brief(session: AsyncSession, job: CatalogDraftJob | None) -> dict[str, Any] | None:
|
||||
"""目录清单顶上的进度：{id, status, total, done}。"""
|
||||
"""目录清单顶上的进度：{id, status, total, done, waiting_for}。"""
|
||||
if job is None:
|
||||
return None
|
||||
rows = (await session.execute(
|
||||
@@ -249,7 +268,7 @@ async def job_brief(session: AsyncSession, job: CatalogDraftJob | None) -> dict[
|
||||
)).all()
|
||||
counts = {status: n for status, n in rows}
|
||||
return {"id": job.id, "status": job.status, "total": sum(counts.values()),
|
||||
"done": sum(counts.get(s, 0) for s in TASK_DONE)}
|
||||
"done": sum(counts.get(s, 0) for s in TASK_DONE), "waiting_for": await waiting_for(session, job)}
|
||||
|
||||
|
||||
async def last_drafts(session: AsyncSession, source_id: str) -> dict[str, dict[str, Any]]:
|
||||
@@ -294,12 +313,30 @@ async def last_drafts(session: AsyncSession, source_id: str) -> dict[str, dict[s
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def busy_message(row: Any) -> str:
|
||||
return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录，请等这次起草结束（或先停止它）再开始新的起草"
|
||||
#: 撞上起草时说法的最后一句（「请等这次起草结束{then}」）缺省是建起草任务的；导入目录、同步起草撞上起草、剖析时按这次要做
|
||||
#: 的事说（_then）
|
||||
THEN_DRAFT = "再开始新的起草"
|
||||
|
||||
|
||||
def sync_busy_message(row: Any) -> str:
|
||||
return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录（同步起草），请等这次起草结束再开始新的起草"
|
||||
def _then(kind: str, busy: str) -> str:
|
||||
"""同步写操作（kind：import / sync_draft）撞上 busy（draft 起草、profile 剖析）时说法的最后一句：导入撞上什么都是
|
||||
「再导入」；同步起草撞上起草是「再开始新的起草」（和建起草任务同一句话），撞上剖析是「再起草」——那时要做的是起草，
|
||||
不是「开始新的剖析」。"""
|
||||
if kind == "import":
|
||||
return "再导入"
|
||||
return THEN_DRAFT if busy == "draft" else "再起草"
|
||||
|
||||
|
||||
def busy_message(row: Any, then: str = THEN_DRAFT) -> str:
|
||||
return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录，请等这次起草结束（或先停止它）{then}"
|
||||
|
||||
|
||||
def sync_busy_message(row: Any, then: str = THEN_DRAFT) -> str:
|
||||
return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录（同步起草），请等这次起草结束{then}"
|
||||
|
||||
|
||||
def import_busy_message(row: Any) -> str:
|
||||
return f"数据源「{getattr(row, 'name', '')}」正在导入数据目录，请等导入完成后再试"
|
||||
|
||||
|
||||
#: 正在同步起草（POST /catalog/draft）的数据源。进程内登记：服务端是单进程部署，多进程部署时要换成库里的锁
|
||||
@@ -311,30 +348,86 @@ def sync_drafting(source_id: str) -> bool:
|
||||
return source_id in _SYNC
|
||||
|
||||
|
||||
async def ensure_idle(session: AsyncSession, row: Any) -> None:
|
||||
"""这个源没有在起草：没有排队或进行中的任务，也没有在同步起草。有就抛 DraftBusy。建任务、重试、导入目录之前都调它。"""
|
||||
async def ensure_draft_free(session: AsyncSession, row: Any, *, then: str = THEN_DRAFT) -> None:
|
||||
"""同种任务互斥：这个源没有在起草——没有排队或进行中的起草任务，也没有在同步起草。有就抛 DraftBusy。建起草任务、
|
||||
重试之前调它。剖析任务不算：同源有剖析任务时起草任务照建，排在它后面。then 是说法最后一句要做的事。"""
|
||||
if row.id in _SYNC:
|
||||
raise DraftBusy(sync_busy_message(row))
|
||||
raise DraftBusy(sync_busy_message(row, then))
|
||||
if await active_job(session, row.id) is not None:
|
||||
raise DraftBusy(busy_message(row, then))
|
||||
|
||||
|
||||
async def ensure_idle(session: AsyncSession, row: Any, kind: str = "import") -> None:
|
||||
"""同步写操作（kind：import 导入目录、sync_draft 同步起草）要求源空闲：没有在起草（DraftBusy，同 ensure_draft_free），
|
||||
也没有排队或进行中的剖析任务（catalog_profile_jobs.ProfileBusy，说法里写明进行到第几张）。说法最后一句按 kind 说
|
||||
这次要做的事（_then）。"""
|
||||
from app.data import catalog_profile_jobs
|
||||
|
||||
await ensure_draft_free(session, row, then=_then(kind, "draft"))
|
||||
await catalog_profile_jobs.ensure_free(session, row, then=_then(kind, "profile"))
|
||||
|
||||
|
||||
def _lane_busy(row: Any, holder: runtime.LaneTicket, kind: str) -> Exception:
|
||||
"""库里查过没有任务、这个源的队却被占着（同步起草、导入目录正在做，或者一个任务刚好在收尾）：按占着的是谁说，最后
|
||||
一句按这次要做的事（kind）说。"""
|
||||
if holder.kind == "profile":
|
||||
from app.data import catalog_profile_jobs
|
||||
|
||||
return catalog_profile_jobs.ProfileBusy(catalog_profile_jobs.busy_message(row, None,
|
||||
then=_then(kind, "profile")))
|
||||
if holder.kind == "import":
|
||||
return DraftBusy(import_busy_message(row))
|
||||
then = _then(kind, "draft")
|
||||
return DraftBusy(sync_busy_message(row, then) if holder.kind == "sync_draft" else busy_message(row, then))
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _sync_write(row: Any, kind: str, check: Callable[[], Awaitable[None]]) -> AsyncIterator[None]:
|
||||
"""同步写操作期间占着这个源的队（不阻塞地试占）：先查库（check：源上有排队或进行中的任务抛 DraftBusy / ProfileBusy），
|
||||
再占队——别的同步写操作正在做时占不到，同样抛忙，什么都不做。占到了，执行期间新建的任务排在它后面，做完才开始。"""
|
||||
await check()
|
||||
try:
|
||||
ticket = runtime.lanes.try_enter(row.id, kind)
|
||||
except runtime.LaneBusy as e:
|
||||
raise _lane_busy(row, e.holder, kind) from e
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
ticket.release()
|
||||
|
||||
|
||||
async def _ensure_idle_but_sync(session: AsyncSession, row: Any) -> None:
|
||||
"""同 ensure_idle(…, "sync_draft")，只是不看 _SYNC：同步起草自己刚登记在里面。"""
|
||||
from app.data import catalog_profile_jobs
|
||||
|
||||
if await active_job(session, row.id) is not None:
|
||||
raise DraftBusy(busy_message(row))
|
||||
await catalog_profile_jobs.ensure_free(session, row, then=_then("sync_draft", "profile"))
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def sync_draft(session: AsyncSession, row: Any) -> AsyncIterator[None]:
|
||||
"""同步起草期间登记这个源：同一个源建任务、重试、导入、再开一次同步起草都抛 DraftBusy；有排队或进行中的任务时
|
||||
这里就抛 DraftBusy。先登记、再查库：查库要让出事件循环，先登记才不会和同时建的任务互相看不见（建任务那边写库之后
|
||||
也再看一次，见 create_job）。"""
|
||||
"""同步起草期间登记这个源、占着它的队：同一个源建起草任务、重试、导入、再开一次同步起草都抛 DraftBusy，期间新建的
|
||||
剖析任务排在它后面；有排队或进行中的起草任务时这里抛 DraftBusy，有剖析任务时抛 ProfileBusy。先登记、再查库：查库要
|
||||
让出事件循环，先登记才不会和同时建的任务互相看不见（建任务那边写库之后也再看一次，见 create_job）。"""
|
||||
if row.id in _SYNC:
|
||||
raise DraftBusy(sync_busy_message(row))
|
||||
_SYNC.add(row.id)
|
||||
try:
|
||||
if await active_job(session, row.id) is not None:
|
||||
raise DraftBusy(busy_message(row))
|
||||
yield
|
||||
async with _sync_write(row, "sync_draft", lambda: _ensure_idle_but_sync(session, row)):
|
||||
yield
|
||||
finally:
|
||||
_SYNC.discard(row.id)
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def import_write(session: AsyncSession, row: Any) -> AsyncIterator[None]:
|
||||
"""导入目录期间占着这个源的队：源上有排队或进行中的起草、剖析任务（或者正在同步起草、导入）时抛 DraftBusy /
|
||||
ProfileBusy；导入期间新建的任务排在它后面。"""
|
||||
async with _sync_write(row, "import", lambda: ensure_idle(session, row, "import")):
|
||||
yield
|
||||
|
||||
|
||||
async def check_model(session: AsyncSession) -> None:
|
||||
"""起草模型能不能用：不能用抛 catalog.CatalogModelUnavailable（只建模型对象，不发请求）。"""
|
||||
await catalog.resolve_draft_model(session)
|
||||
@@ -344,8 +437,9 @@ async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: li
|
||||
note: str | None, actor: str | None,
|
||||
retry_of: str | None = None) -> tuple[CatalogDraftJob, list[CatalogDraftTask]]:
|
||||
"""建一个排队的任务（还没开始，调用方接着 manager.start）。row 是数据源记录（起草要求取它的），source 是读表结构
|
||||
用的源（上传源是绑定当前快照的视图）。有排队或进行中的任务抛 DraftBusy。"""
|
||||
await ensure_idle(session, row)
|
||||
用的源（上传源是绑定当前快照的视图）。有排队或进行中的起草任务（或正在同步起草）抛 DraftBusy；同源有剖析任务时照建，
|
||||
排在它后面。"""
|
||||
await ensure_draft_free(session, row)
|
||||
cache = getattr(source, "schema_cache", None) or {}
|
||||
all_tables = cache.get("tables") or {}
|
||||
wanted = list(tables) if tables else await catalog.default_tables(session, source)
|
||||
@@ -395,22 +489,15 @@ async def retry_tables(session: AsyncSession, job: CatalogDraftJob) -> list[str]
|
||||
|
||||
|
||||
async def cancel(session: AsyncSession, job: CatalogDraftJob) -> None:
|
||||
"""停止一个任务。排队的直接 cancelled；进行中的交给 manager 停（停掉模型调用、没写完的表记 cancelled）。
|
||||
已经结束的抛 DraftFinished。"""
|
||||
"""停止一个任务。排队的直接 cancelled（在队里等着的后台执行跟着停掉、出队）；进行中的交给 manager 停（停掉模型调用、
|
||||
没写完的表记 cancelled）。已经结束的抛 DraftFinished。"""
|
||||
if job.status in FINISHED:
|
||||
raise DraftFinished("这次起草已经结束了，不需要停止")
|
||||
if job.status == "queued":
|
||||
now = utcnow()
|
||||
claimed = await session.execute(
|
||||
update(CatalogDraftJob).where(CatalogDraftJob.id == job.id, CatalogDraftJob.status == "queued")
|
||||
.values(status="cancelled", finished_at=now).execution_options(synchronize_session=False))
|
||||
if claimed.rowcount == 1:
|
||||
await session.execute(
|
||||
update(CatalogDraftTask).where(CatalogDraftTask.job_id == job.id, CatalogDraftTask.status.in_(_OPEN))
|
||||
.values(status="cancelled", finished_at=now).execution_options(synchronize_session=False))
|
||||
await session.commit()
|
||||
if await runtime.cancel_queued(session, CatalogDraftJob, CatalogDraftTask, job.id):
|
||||
await manager.stop(job.id, "user", grace=STOP_GRACE_S)
|
||||
return
|
||||
await session.rollback() # 刚被后台认领、开始了：按进行中的停
|
||||
# 刚被后台认领、开始了：按进行中的停
|
||||
if await manager.stop(job.id, "user", grace=STOP_GRACE_S):
|
||||
return
|
||||
# 库里说在进行，这个进程里却没有在跑它（记录对不上）：直接记停止
|
||||
@@ -434,26 +521,19 @@ async def _close(session: AsyncSession, job_id: str, job_status: str, job_error:
|
||||
.values(status=task_status, error=task_error, finished_at=now,
|
||||
model_error=written.format(n=n, m=max(0, int(total or n) - int(n))))
|
||||
.execution_options(synchronize_session=False))
|
||||
await session.execute(
|
||||
update(CatalogDraftTask).where(CatalogDraftTask.job_id == job_id, CatalogDraftTask.status.in_(_OPEN))
|
||||
.values(status=task_status, error=task_error, finished_at=now).execution_options(synchronize_session=False))
|
||||
await session.execute(
|
||||
update(CatalogDraftJob).where(CatalogDraftJob.id == job_id, CatalogDraftJob.status.in_(ACTIVE))
|
||||
.values(status=job_status, error=job_error, finished_at=now).execution_options(synchronize_session=False))
|
||||
await session.commit()
|
||||
await runtime.close_job(session, CatalogDraftJob, CatalogDraftTask, job_id, job_status, job_error, task_status,
|
||||
task_error)
|
||||
|
||||
|
||||
async def startup() -> int:
|
||||
"""服务启动时：排队、进行中的任务标成 interrupted，没结束的表记 cancelled、写明服务重启。不自动续跑。返回标了几个。"""
|
||||
async with SessionLocal() as session:
|
||||
ids = [job_id for job_id in (await session.execute(
|
||||
select(CatalogDraftJob.id).where(CatalogDraftJob.status.in_(ACTIVE)))).scalars()
|
||||
if not manager.running(job_id)]
|
||||
for job_id in ids:
|
||||
await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR)
|
||||
if ids:
|
||||
logger.warning("服务重启：%d 个数据目录起草任务没有做完，已标为中断，可在数据目录页重试", len(ids))
|
||||
return len(ids)
|
||||
async def interrupt(session: AsyncSession, job_id: str) -> None:
|
||||
await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR)
|
||||
|
||||
n = await runtime.interrupt_on_startup(CatalogDraftJob, interrupt, running=manager.running)
|
||||
if n:
|
||||
logger.warning("服务重启：%d 个数据目录起草任务没有做完，已标为中断，可在数据目录页重试", n)
|
||||
return n
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
@@ -461,60 +541,35 @@ async def startup() -> int:
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
class _JobRun:
|
||||
"""一个任务在后台的一次执行。"""
|
||||
class _JobRun(runtime.JobRun):
|
||||
"""一个起草任务在后台的一次执行（进队、停止、收尾的框架在 runtime.JobRun）。"""
|
||||
|
||||
kind = "draft"
|
||||
model = CatalogDraftJob
|
||||
|
||||
def __init__(self, job_id: str) -> None:
|
||||
self.job_id = job_id
|
||||
self.task: asyncio.Task[None] | None = None
|
||||
#: 为什么停：user 使用者点了停止，shutdown 服务关停（标中断）
|
||||
self.stop_reason: str | None = None
|
||||
self._writes: set[asyncio.Task[Any]] = set()
|
||||
self._lock = asyncio.Lock()
|
||||
super().__init__(job_id)
|
||||
self._started: set[str] = set()
|
||||
self.source: Any = None
|
||||
self.actor: str | None = None
|
||||
|
||||
# ---------------------------------------------------------------- 库写入
|
||||
# ---------------------------------------------------------------- 停止、出错时收尾
|
||||
|
||||
async def _write(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any:
|
||||
"""一条库写入：同一个任务的写入一条一条来；shield 起来，停止任务时不会打断写到一半的库操作。"""
|
||||
async def go() -> Any:
|
||||
async with self._lock:
|
||||
async with SessionLocal() as session:
|
||||
return await fn(session, *args)
|
||||
async def _stopped(self) -> None:
|
||||
if self.stop_reason == "user":
|
||||
await self._end("cancelled", None, "cancelled", None)
|
||||
else:
|
||||
await self._end("interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR)
|
||||
|
||||
task = asyncio.ensure_future(go())
|
||||
self._writes.add(task)
|
||||
task.add_done_callback(self._writes.discard)
|
||||
return await asyncio.shield(task)
|
||||
async def _failed(self, reason: str) -> None:
|
||||
await self._end("failed", reason, "failed", reason)
|
||||
|
||||
async def _settle(self) -> None:
|
||||
"""等手上的库写入都写完（停止、出错收尾之前）。"""
|
||||
while self._writes:
|
||||
await asyncio.gather(*list(self._writes), return_exceptions=True)
|
||||
async def _crashed(self, e: Exception) -> None:
|
||||
from app.core.errors import describe_exception, first_line
|
||||
|
||||
# ---------------------------------------------------------------- 主流程
|
||||
|
||||
async def run(self) -> None:
|
||||
try:
|
||||
await self._run()
|
||||
except asyncio.CancelledError:
|
||||
current = asyncio.current_task()
|
||||
if current is not None:
|
||||
current.uncancel()
|
||||
if self.stop_reason == "user":
|
||||
await self._end("cancelled", None, "cancelled", None)
|
||||
else:
|
||||
await self._end("interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR)
|
||||
except _JobFailed as e:
|
||||
await self._end("failed", str(e), "failed", str(e))
|
||||
except Exception as e: # noqa: BLE001 - 任务整体出错：照实记下，不留在「起草中」
|
||||
from app.core.errors import describe_exception, first_line
|
||||
|
||||
logger.exception("数据目录起草任务 %s 出错", self.job_id)
|
||||
reason = JOB_ERROR.format(line=first_line(e) or describe_exception(e))
|
||||
await self._end("failed", reason, "failed", reason)
|
||||
logger.exception("数据目录起草任务 %s 出错", self.job_id)
|
||||
reason = JOB_ERROR.format(line=first_line(e) or describe_exception(e))
|
||||
await self._end("failed", reason, "failed", reason)
|
||||
|
||||
async def _end(self, job_status: str, job_error: str | None, task_status: str, task_error: str | None) -> None:
|
||||
"""停止、出错时收尾：等手上的库写入写完，再把没结束的表和任务记下来。收尾本身写不进去只记日志（下次启动时
|
||||
@@ -603,11 +658,7 @@ class _JobRun:
|
||||
.execution_options(synchronize_session=False)
|
||||
|
||||
async def _claim(self, session: AsyncSession) -> bool:
|
||||
result = await session.execute(
|
||||
update(CatalogDraftJob).where(CatalogDraftJob.id == self.job_id, CatalogDraftJob.status == "queued")
|
||||
.values(status="running", started_at=utcnow()).execution_options(synchronize_session=False))
|
||||
await session.commit()
|
||||
return result.rowcount == 1
|
||||
return await runtime.claim(session, CatalogDraftJob, self.job_id)
|
||||
|
||||
async def _set_model(self, session: AsyncSession, model_id: str) -> None:
|
||||
await session.execute(update(CatalogDraftJob).where(CatalogDraftJob.id == self.job_id)
|
||||
@@ -746,66 +797,11 @@ async def write_part(session: AsyncSession, source: Any, table: str, notes: dict
|
||||
return None
|
||||
|
||||
|
||||
class DraftJobs:
|
||||
"""进程内的起草任务（照 app/scheduler 的写法：一个 asyncio 任务一件事，停的时候给宽限）。"""
|
||||
class DraftJobs(runtime.JobManager):
|
||||
"""进程内的起草任务（start / stop / wait / running / shutdown 见 runtime.JobManager）。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._runs: dict[str, _JobRun] = {}
|
||||
|
||||
def _alive(self, job_id: str) -> _JobRun | None:
|
||||
run = self._runs.get(job_id)
|
||||
if run is None or run.task is None or run.task.done():
|
||||
return None
|
||||
try:
|
||||
if run.task.get_loop() is not asyncio.get_running_loop():
|
||||
self._runs.pop(job_id, None) # 别的事件循环留下的（测试每个用例一个循环）
|
||||
return None
|
||||
except RuntimeError:
|
||||
return None
|
||||
return run
|
||||
|
||||
def start(self, job_id: str) -> None:
|
||||
"""开始在后台起草这个（排队中的）任务。"""
|
||||
run = _JobRun(job_id)
|
||||
run.task = asyncio.get_running_loop().create_task(run.run(), name=f"catalog-draft-{job_id}")
|
||||
self._runs[job_id] = run
|
||||
|
||||
def forget(_: asyncio.Task[None]) -> None:
|
||||
if self._runs.get(job_id) is run:
|
||||
self._runs.pop(job_id, None)
|
||||
|
||||
run.task.add_done_callback(forget)
|
||||
|
||||
def running(self, job_id: str) -> bool:
|
||||
return self._alive(job_id) is not None
|
||||
|
||||
async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S) -> bool:
|
||||
"""停掉这个进程里在跑的任务，等它收尾（最多 grace 秒）。不在这个进程里跑返回 False。"""
|
||||
run = self._alive(job_id)
|
||||
if run is None:
|
||||
return False
|
||||
if run.stop_reason is None:
|
||||
run.stop_reason = reason
|
||||
run.task.cancel() # type: ignore[union-attr]
|
||||
await asyncio.wait([run.task], timeout=grace) # type: ignore[list-item]
|
||||
return True
|
||||
|
||||
async def wait(self, job_id: str, timeout: float | None = None) -> None:
|
||||
"""等这个任务做完（测试、调试用）。不在跑直接返回。"""
|
||||
run = self._alive(job_id)
|
||||
if run is not None:
|
||||
await asyncio.wait([run.task], timeout=timeout) # type: ignore[list-item]
|
||||
|
||||
async def shutdown(self, *, grace: float = 3.0) -> None:
|
||||
"""关停：停掉进行中的任务（标中断），最多等 grace 秒收尾。"""
|
||||
runs = [run for job_id in list(self._runs) if (run := self._alive(job_id)) is not None]
|
||||
for run in runs:
|
||||
if run.stop_reason is None:
|
||||
run.stop_reason = "shutdown"
|
||||
run.task.cancel() # type: ignore[union-attr]
|
||||
if runs:
|
||||
await asyncio.wait([run.task for run in runs], timeout=grace) # type: ignore[misc]
|
||||
logger.info("已停止 %d 个数据目录起草任务", len(runs))
|
||||
super().__init__(_JobRun, label="数据目录起草任务", name="catalog-draft")
|
||||
|
||||
|
||||
manager = DraftJobs()
|
||||
@@ -813,7 +809,8 @@ manager = DraftJobs()
|
||||
|
||||
__all__ = ["ACTIVE", "BLANK_SAMPLE", "DraftBusy", "DraftFinished", "DraftJobs", "FINISHED", "GUIDANCE_MAX",
|
||||
"INTERRUPTED_ERROR", "INTERRUPTED_WRITTEN", "MAX_TABLES", "NOTE_MAX", "NothingToRetry", "RETRYABLE",
|
||||
"STOPPED_WRITTEN", "TASK_DONE", "TASK_STATUSES", "TextTooLong",
|
||||
"active_job", "cancel", "check_model", "clean_text", "create_job", "ensure_idle", "guidance_of", "job_brief",
|
||||
"job_dict", "job_out", "last_drafts", "latest_job", "load", "manager", "retry_tables", "save_guidance",
|
||||
"startup", "sync_busy_message", "sync_draft", "sync_drafting", "tasks_of", "write_part"]
|
||||
"STOPPED_WRITTEN", "TASK_DONE", "TASK_STATUSES", "THEN_DRAFT", "TextTooLong",
|
||||
"active_job", "cancel", "check_model", "clean_text", "create_job", "ensure_draft_free", "ensure_idle",
|
||||
"guidance_of", "import_busy_message", "import_write", "job_brief", "job_dict", "job_out", "last_drafts",
|
||||
"latest_job", "load", "manager", "retry_tables", "save_guidance", "startup", "sync_busy_message", "sync_draft",
|
||||
"sync_drafting", "tasks_of", "waiting_for", "write_part"]
|
||||
|
||||
+688
-290
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,890 @@
|
||||
"""数据目录的后台剖析任务（/api/datasources/{id}/catalog/profile-jobs）。
|
||||
|
||||
以前剖析是一次同步长请求（POST /catalog/profile，已去掉）：预算按整次剖析算，选 50 张表只做得完前两三张；进行中只有一条
|
||||
按时长走的横条、不能停；关掉页面报告就没了，对业务库查了什么无从查。现在和起草一样是后台任务（生命周期在
|
||||
data/catalog_job_runtime.py），逐表剖析（引擎在 data/catalog_profile.py）：
|
||||
|
||||
- **建任务**（create_job）：范围三种——指定的表（selected：按给的顺序去重；表结构里没有、视图、没有可剖析的项照样建进
|
||||
任务、状态 skipped）、需要剖析的表（needed：从没剖析过、上次剖析没做完、剖析之后冒出了新的待核实关系，见 due_reason）、
|
||||
全部表（all）。后两种只挑表结构里、不是视图、有可剖析的项的表，有待核实关系的在前，再按使用次数、表结构顺序；挑出 0 张
|
||||
抛 ProfileNothing。剖析设置随任务记下快照，每张表的检查项按当时的目录先列一遍。同一个源同时只有一个排队或进行中的
|
||||
剖析任务（ProfileBusy，说法里写明进行到第几张；部分唯一索引兜底）；和同源的起草任务按建立的先后排队（每个源一条队），
|
||||
起草在前时 waiting_for 是 "draft"。建任务时顺手清掉超出保留数的旧任务（prune）。
|
||||
- **在后台剖析**（manager.start）：轮到了才开始。每张表开始前重读数据源——停用了、关掉了剖析就停下，剩下的表 cancelled、
|
||||
error 写明；遮罩按最新的设置——再逐表调 catalog_profile.profile_table（它读的是那时目录里的最新内容：起草刚提的关系也
|
||||
核对）。检查项、正在做哪一项、查询数、发过的查询随时落库；整张表的结果和写目录在同一个事务里，停止时也不会被打断。
|
||||
- **任务级止损**（替代以前的整次总量上限）：连续 3 条查询失败、最近 10 条里 7 条超时，整个任务停下（stop_reason、error），
|
||||
正在做的那张表写入已做完的项，后面的表 cancelled、error 带同一句原因；最近 6 条里 4 条超时时 notice 有一句提醒，好转后
|
||||
清掉。任务最后的状态按逐表算：有 failed、partial、cancelled 的表是 partial，否则 succeeded；任务整体出错（数据源没了、
|
||||
快照被改动、意外异常）是 failed。
|
||||
- **停止**（cancel）：排队的直接 cancelled；进行中的停掉正在等的那条查询（库那边按单条时限截止），那张表做完的项照常写进
|
||||
目录、没做的保留旧结论，后面的表 cancelled，任务 cancelled。
|
||||
- **重试**（retry_tables + create_job）：新任务，只含原任务里 failed、partial、cancelled 的表（按原来的顺序），范围 selected，
|
||||
retry_of 记原任务；设置取数据源当前的。
|
||||
- **服务重启**（startup）：排队、进行中的标 interrupted，没结束的表 cancelled、error「服务重启，剖析中断」（进程没了时正在
|
||||
剖析的那张表，查完的项也没写进目录，照实写明，见文末）。不自动续跑：
|
||||
复制出来的数据目录启动时不能自己去查业务库，界面让人点「重试」。关停时（manager.shutdown）停掉进行中的，同样标中断。
|
||||
启动时还一次性迁移剖析设置的旧键（catalog_profile.migrate_settings）。
|
||||
- **保留**（prune）：每个源保留最近 KEEP_JOBS 个结束了的剖析任务，更早的删掉；某张表「上次剖析」所在的任务不删，表详情的
|
||||
「上次剖析」一直在。
|
||||
- **读**：job_dict（任务和逐表一行）、task_detail（一张表的检查项、发现、跳过的项、发过的查询）、last_profiles / last_detail
|
||||
（表清单每行、表详情的「上次剖析」）、due_reason（这张表要不要剖析、为什么）。「上次剖析」是真正开始过的最近一次（见
|
||||
_ranked）：没开始就停了的、还排着的那一项不挡掉上一次真正剖析的行数、发过的查询。
|
||||
|
||||
没结束的表收尾时（停止、中断、整体出错）检查项里还排着的、正在做的记为跳过（原因 cancelled 或 error），不留「排队中」的
|
||||
检查项在一张已经结束的表上。目录只在一张表结束时和它的结果一起写（同一个事务）：收尾时还没结束的表，查完的项结论也没
|
||||
写进目录（进程被杀、快照被改动），同样记为跳过、写明「查完了但结论没有写进数据目录」，算没做完，重试时带上这张表。
|
||||
停止、关停这类能收尾的停下不走这条路：正在剖析的那张表做完的项照常写进目录（catalog_profile.profile_table）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import case, delete, func, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.data import catalog, catalog_profile
|
||||
from app.data import catalog_job_runtime as runtime
|
||||
from app.data.catalog_job_runtime import ACTIVE, FINISHED, OPEN, RETRYABLE, STOP_GRACE_S, TASK_DONE, TASK_STATUSES
|
||||
from app.data.catalog_profile import (
|
||||
NOTHING_ERROR, UNFINISHED_REASONS, VIEW_ERROR, ProfileSettings, ProfileSkip, TableHooks, TableResult,
|
||||
)
|
||||
from app.data.engine import SnapshotTampered
|
||||
from app.db.base import SessionLocal, new_id, utcnow
|
||||
from app.db.models import CatalogDraftJob, CatalogProfileJob, CatalogProfileTask, DataSource
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: 一个任务最多几张表
|
||||
MAX_TABLES = 500
|
||||
#: 每个源保留几个结束了的剖析任务（某张表最近一次剖析所在的任务另外保留）
|
||||
KEEP_JOBS = 50
|
||||
|
||||
INTERRUPTED_ERROR = "服务重启，剖析中断"
|
||||
STOPPED_ERROR = catalog_profile.STOPPED
|
||||
UNFINISHED_ERROR = "剖析结束时这张表还没有收尾，请重试这张表"
|
||||
JOB_ERROR = "剖析任务出错：{line}"
|
||||
SOURCE_GONE = "数据源不存在，可能已被删除"
|
||||
SOURCE_DISABLED = "数据源「{name}」已停用，剖析已停止"
|
||||
PROFILE_TURNED_OFF = "数据源「{name}」已关闭数据剖析，剖析已停止"
|
||||
#: 没结束的检查项在收尾时的说明（{why} 是停止、中断、出错的原因）
|
||||
_UNCHECKED = "{why}，未检查"
|
||||
#: 查完了、结论却没写进目录的检查项：这张表没走到整张表结束、写目录那一步（进程没了、任务整体出错）
|
||||
_UNSAVED = "{why}，查完了但结论没有写进数据目录"
|
||||
_CHECK_OPEN = ("pending", "running")
|
||||
|
||||
|
||||
def unsaved_error(why: str, done: int, rest: int) -> str:
|
||||
"""没走到写目录那一步的表的整句原因：查完的几项没写进目录，剩下几项没查。"""
|
||||
text = f"{why}：已查完的 {done} 项没有写进数据目录"
|
||||
return text + f"，剩下 {rest} 项没查" if rest else text
|
||||
|
||||
|
||||
class ProfileBusy(Exception):
|
||||
"""这个数据源已经有排队或进行中的剖析任务。message 给人看（写明进行到第几张）。"""
|
||||
|
||||
|
||||
class ProfileFinished(Exception):
|
||||
"""要停止的剖析任务已经结束了。"""
|
||||
|
||||
|
||||
class ProfileNothing(Exception):
|
||||
"""按范围挑出 0 张表。message 说明原因、建议怎么办。"""
|
||||
|
||||
|
||||
class NothingToRetry(Exception):
|
||||
"""原任务里没有需要重试的表。"""
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 读任务
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def _iso(value: Any) -> str | None:
|
||||
return value.isoformat() if value is not None else None
|
||||
|
||||
|
||||
def _empty_summary() -> dict[str, int]:
|
||||
return catalog_profile.summarize([])
|
||||
|
||||
|
||||
async def active_job(session: AsyncSession, source_id: str) -> CatalogProfileJob | None:
|
||||
"""排队或进行中的那个（同一个源最多一个）。"""
|
||||
return (await session.execute(
|
||||
select(CatalogProfileJob).where(CatalogProfileJob.source_id == source_id,
|
||||
CatalogProfileJob.status.in_(ACTIVE))
|
||||
.order_by(CatalogProfileJob.created_at.desc()).limit(1)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
|
||||
async def latest_job(session: AsyncSession, source_id: str) -> CatalogProfileJob | None:
|
||||
"""最近建的一个（任何状态）。"""
|
||||
return (await session.execute(
|
||||
select(CatalogProfileJob).where(CatalogProfileJob.source_id == source_id)
|
||||
.order_by(CatalogProfileJob.created_at.desc()).limit(1)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
|
||||
async def load(session: AsyncSession, source_id: str, job_id: str) -> CatalogProfileJob | None:
|
||||
"""这个源的这个任务；不存在、不属于这个源都是 None。"""
|
||||
job = await session.get(CatalogProfileJob, job_id)
|
||||
return job if job is not None and job.source_id == source_id else None
|
||||
|
||||
|
||||
#: 逐表一行要的列：不读发现、跳过的项、发过的查询（任务进行中界面每秒都要取一次）
|
||||
_ROW_COLUMNS = (CatalogProfileTask.seq, CatalogProfileTask.table_name, CatalogProfileTask.status,
|
||||
CatalogProfileTask.checks, CatalogProfileTask.current, CatalogProfileTask.queries,
|
||||
CatalogProfileTask.elapsed_ms, CatalogProfileTask.row_estimate, CatalogProfileTask.summary,
|
||||
CatalogProfileTask.skipped_count, CatalogProfileTask.unfinished, CatalogProfileTask.added,
|
||||
CatalogProfileTask.updated, CatalogProfileTask.removed, CatalogProfileTask.version,
|
||||
CatalogProfileTask.error, CatalogProfileTask.started_at, CatalogProfileTask.finished_at)
|
||||
|
||||
|
||||
async def task_rows(session: AsyncSession, job_id: str) -> list[Any]:
|
||||
return list((await session.execute(
|
||||
select(*_ROW_COLUMNS).where(CatalogProfileTask.job_id == job_id).order_by(CatalogProfileTask.seq)
|
||||
)).all())
|
||||
|
||||
|
||||
def _by_kind(checks: list[Any]) -> dict[str, list[int]]:
|
||||
"""检查项按种类的进度：{relation: [结束了的, 总数], codes: […], date: […]}。结束了的含跳过的。"""
|
||||
out = {"relation": [0, 0], "codes": [0, 0], "date": [0, 0]}
|
||||
for check in checks:
|
||||
if not isinstance(check, dict) or check.get("kind") not in out:
|
||||
continue
|
||||
pair = out[check["kind"]]
|
||||
pair[1] += 1
|
||||
if check.get("state") in ("done", "skipped"):
|
||||
pair[0] += 1
|
||||
return out
|
||||
|
||||
|
||||
def task_row(task: Any) -> dict[str, Any]:
|
||||
"""逐表一行（CatalogProfileTaskRow）。task 是 CatalogProfileTask，或者 task_rows 取的那几列。"""
|
||||
checks = [c for c in (task.checks or []) if isinstance(c, dict)]
|
||||
return {
|
||||
"seq": task.seq,
|
||||
"table_name": task.table_name,
|
||||
"status": task.status,
|
||||
"checks_total": len(checks),
|
||||
"checks_done": sum(1 for c in checks if c.get("state") in ("done", "skipped")),
|
||||
"checks_by_kind": _by_kind(checks),
|
||||
"current": task.current if task.status == "running" else None,
|
||||
"queries": task.queries or 0,
|
||||
"elapsed_ms": task.elapsed_ms or 0,
|
||||
"row_estimate": task.row_estimate,
|
||||
"summary": {**_empty_summary(), **(task.summary or {})},
|
||||
"skipped_count": task.skipped_count or 0,
|
||||
"unfinished": task.unfinished or 0,
|
||||
"added": task.added or 0,
|
||||
"updated": task.updated or 0,
|
||||
"removed": task.removed or 0,
|
||||
"version": task.version or 0,
|
||||
"error": task.error,
|
||||
"started_at": _iso(task.started_at),
|
||||
"finished_at": _iso(task.finished_at),
|
||||
}
|
||||
|
||||
|
||||
def _counts(statuses: list[str]) -> dict[str, int]:
|
||||
counts = dict.fromkeys(TASK_STATUSES, 0)
|
||||
for status in statuses:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
return counts
|
||||
|
||||
|
||||
def job_dict(job: CatalogProfileJob, tasks: list[Any], *, waiting_for: str | None = None) -> dict[str, Any]:
|
||||
"""接口返回的任务（CatalogProfileJob）：逐表按剖析顺序；计数、合计都按逐表的算。"""
|
||||
rows = [task_row(t) for t in tasks]
|
||||
counts = _counts([r["status"] for r in rows])
|
||||
summary = _empty_summary()
|
||||
for r in rows:
|
||||
for key in summary:
|
||||
summary[key] += int(r["summary"].get(key) or 0)
|
||||
return {
|
||||
"id": job.id,
|
||||
"source_id": job.source_id,
|
||||
"status": job.status,
|
||||
"scope": job.scope,
|
||||
"settings": ProfileSettings.from_dict(job.settings).to_dict(),
|
||||
"created_at": _iso(job.created_at),
|
||||
"started_at": _iso(job.started_at),
|
||||
"finished_at": _iso(job.finished_at),
|
||||
"created_by": job.created_by,
|
||||
"retry_of": job.retry_of,
|
||||
"waiting_for": waiting_for,
|
||||
"total": len(rows),
|
||||
"done": sum(counts.get(s, 0) for s in TASK_DONE),
|
||||
"counts": counts,
|
||||
"queries": job.queries or 0,
|
||||
"timeouts": job.timeouts or 0,
|
||||
"errors": job.errors or 0,
|
||||
"stop_reason": job.stop_reason,
|
||||
"notice": job.notice,
|
||||
"error": job.error,
|
||||
"added": sum(r["added"] for r in rows),
|
||||
"updated": sum(r["updated"] for r in rows),
|
||||
"removed": sum(r["removed"] for r in rows),
|
||||
"summary": summary,
|
||||
"tasks": rows,
|
||||
}
|
||||
|
||||
|
||||
async def waiting_for(session: AsyncSession, job: CatalogProfileJob) -> str | None:
|
||||
"""排队中的剖析任务在等什么：排在同源的起草任务（或同步起草）后面时是 "draft"，否则 None。
|
||||
|
||||
先看这个源的队（进程内，排在它前面的是谁）；任务刚建、后台还没排进队时看库：同源有比它先建的、排队或进行中的起草
|
||||
任务，或者正在同步起草。"""
|
||||
if job.status != "queued":
|
||||
return None
|
||||
ahead = runtime.lanes.ahead(job.source_id, job.id)
|
||||
if ahead is not None:
|
||||
return "draft" if any(kind in ("draft", "sync_draft") for kind in ahead) else None
|
||||
from app.data import catalog_jobs
|
||||
|
||||
if catalog_jobs.sync_drafting(job.source_id):
|
||||
return "draft"
|
||||
earlier = (await session.execute(
|
||||
select(CatalogDraftJob.id).where(CatalogDraftJob.source_id == job.source_id,
|
||||
CatalogDraftJob.status.in_(ACTIVE),
|
||||
CatalogDraftJob.created_at <= job.created_at).limit(1)
|
||||
)).first()
|
||||
return "draft" if earlier is not None else None
|
||||
|
||||
|
||||
async def job_out(session: AsyncSession, job: CatalogProfileJob | None) -> dict[str, Any] | None:
|
||||
if job is None:
|
||||
return None
|
||||
return job_dict(job, await task_rows(session, job.id), waiting_for=await waiting_for(session, job))
|
||||
|
||||
|
||||
async def job_brief(session: AsyncSession, job: CatalogProfileJob | None) -> dict[str, Any] | None:
|
||||
"""目录清单顶上的进度（CatalogActiveProfileJob）：{id, status, total, done, waiting_for}。"""
|
||||
if job is None:
|
||||
return None
|
||||
rows = (await session.execute(
|
||||
select(CatalogProfileTask.status, func.count()).where(CatalogProfileTask.job_id == job.id)
|
||||
.group_by(CatalogProfileTask.status)
|
||||
)).all()
|
||||
counts = {status: n for status, n in rows}
|
||||
return {"id": job.id, "status": job.status, "total": sum(counts.values()),
|
||||
"done": sum(counts.get(s, 0) for s in TASK_DONE), "waiting_for": await waiting_for(session, job)}
|
||||
|
||||
|
||||
def task_detail(job: CatalogProfileJob, task: CatalogProfileTask) -> dict[str, Any]:
|
||||
"""一张表的明细（CatalogProfileTaskDetail）：逐表一行，加上检查项、发现、跳过的项、日期范围、发过的查询。"""
|
||||
return {
|
||||
**task_row(task),
|
||||
"job_id": job.id,
|
||||
"job_status": job.status,
|
||||
"created_by": job.created_by,
|
||||
"checks": [c for c in task.checks or [] if isinstance(c, dict)],
|
||||
"findings": list(task.findings or []),
|
||||
"skipped": list(task.skipped or []),
|
||||
"date_ranges": list(task.date_ranges or []),
|
||||
"query_log": list(task.query_log or []),
|
||||
}
|
||||
|
||||
|
||||
async def load_task(session: AsyncSession, job: CatalogProfileJob, seq: int) -> CatalogProfileTask | None:
|
||||
return (await session.execute(
|
||||
select(CatalogProfileTask).where(CatalogProfileTask.job_id == job.id, CatalogProfileTask.seq == seq)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
|
||||
#: 同一张表的各项谁算「最近」：任务建得晚的在前；同一个任务里同名的取 seq 小的
|
||||
_NEWEST = (CatalogProfileJob.created_at.desc(), CatalogProfileJob.id.desc(), CatalogProfileTask.seq)
|
||||
#: 表清单、表详情显示的「上次剖析」：真正开始过（started_at 有值）的排在没开始过的前面，各自再按 _NEWEST
|
||||
_SHOWN = (case((CatalogProfileTask.started_at.is_(None), 1), else_=0), *_NEWEST)
|
||||
|
||||
|
||||
def _ranked(source_id: str, *columns: Any) -> Any:
|
||||
"""每张表剖析过的各项：子查询，带两个排名。
|
||||
|
||||
- shown：这张表的「上次剖析」（表清单每行、表详情、要不要剖析都按它；保留也保护它所在的任务）——真正开始过的最近一项，
|
||||
一次都没开始过才退回最近一项。没开始就结束了的那一项（停在前面的表上、任务止损或整体出错后没轮到、建任务时就判为
|
||||
跳过）、排队中还没轮到的那一项都不算：它没有行数估算、发过的查询，要是挡掉上一次真正剖析的那一项，表详情就只剩
|
||||
一句「已停止」，那次剖析所在的任务还会被当成不是谁的最近一次给清掉——行数、日期范围、发过的查询只存在剖析任务里。
|
||||
- latest：最近一项，不管开始没有。它还排着、在做（pending / running）说明这张表已经在进行中的任务里了。
|
||||
"""
|
||||
shown = func.row_number().over(partition_by=CatalogProfileTask.table_name, order_by=_SHOWN).label("shown")
|
||||
latest = func.row_number().over(partition_by=CatalogProfileTask.table_name, order_by=_NEWEST).label("latest")
|
||||
return (select(*columns, shown, latest).join(CatalogProfileJob, CatalogProfileJob.id == CatalogProfileTask.job_id)
|
||||
.where(CatalogProfileJob.source_id == source_id)).subquery()
|
||||
|
||||
|
||||
async def last_profiles(session: AsyncSession, source_id: str) -> dict[str, dict[str, Any]]:
|
||||
"""每张表的「上次剖析」（见 _ranked 的 shown）：{表名: CatalogLastProfile 加上内部用的 paths（那一项检查项的路径，判断
|
||||
有没有新的待核实关系用）、queued（最近一项还排着或在做：这张表已经在进行中的任务里了）}。在做的就是进行中的状态。
|
||||
每张表取哪一项在库里挑好。"""
|
||||
inner = _ranked(source_id, CatalogProfileTask.table_name, CatalogProfileTask.job_id,
|
||||
CatalogProfileTask.seq, CatalogProfileTask.status, CatalogProfileTask.started_at,
|
||||
CatalogProfileTask.finished_at, CatalogProfileTask.summary,
|
||||
CatalogProfileTask.skipped_count, CatalogProfileTask.unfinished,
|
||||
CatalogProfileTask.error, CatalogProfileTask.row_estimate, CatalogProfileTask.checks)
|
||||
rows = list((await session.execute(
|
||||
select(inner).where(or_(inner.c.shown == 1, inner.c.latest == 1)))).mappings())
|
||||
queued = {row["table_name"] for row in rows if row["latest"] == 1 and row["status"] in OPEN}
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for row in rows:
|
||||
if row["shown"] != 1:
|
||||
continue
|
||||
out[row["table_name"]] = {
|
||||
"job_id": row["job_id"], "seq": row["seq"], "status": row["status"],
|
||||
"started_at": _iso(row["started_at"]), "finished_at": _iso(row["finished_at"]),
|
||||
"summary": {**_empty_summary(), **(row["summary"] or {})}, "skipped_count": row["skipped_count"] or 0,
|
||||
"unfinished": row["unfinished"] or 0, "error": row["error"], "row_estimate": row["row_estimate"],
|
||||
"paths": {c.get("path") for c in row["checks"] or [] if isinstance(c, dict) and c.get("path")},
|
||||
"queued": row["table_name"] in queued,
|
||||
}
|
||||
return out
|
||||
|
||||
|
||||
#: last_profiles 里只给内部用的键
|
||||
_INTERNAL = ("paths", "queued")
|
||||
|
||||
|
||||
def public_last(last: dict[str, Any] | None) -> dict[str, Any] | None:
|
||||
"""接口里的 CatalogLastProfile：去掉内部用的 paths、queued。"""
|
||||
return None if last is None else {k: v for k, v in last.items() if k not in _INTERNAL}
|
||||
|
||||
|
||||
async def last_detail(session: AsyncSession, source_id: str, table: str) -> dict[str, Any] | None:
|
||||
"""这张表「上次剖析」（见 _ranked 的 shown）的明细（表详情的「数据剖析」一节）；没剖析过是 None。"""
|
||||
found = (await session.execute(
|
||||
select(CatalogProfileTask, CatalogProfileJob)
|
||||
.join(CatalogProfileJob, CatalogProfileJob.id == CatalogProfileTask.job_id)
|
||||
.where(CatalogProfileJob.source_id == source_id, CatalogProfileTask.table_name == table)
|
||||
.order_by(*_SHOWN).limit(1)
|
||||
)).first()
|
||||
return None if found is None else task_detail(found[1], found[0])
|
||||
|
||||
|
||||
def due_reason(meta: dict[str, Any] | None, notes: dict[str, Any] | None, last: dict[str, Any] | None,
|
||||
checks: int) -> str | None:
|
||||
"""这张表要不要剖析、为什么：never 从没剖析过；unfinished 上次剖析是 partial / failed / cancelled；new_relations
|
||||
目录里有待核实的关系不在上次剖析的检查项里。表结构里没有、视图、没有可剖析的项（checks 为 0）、已经在进行中的任务里
|
||||
（排着或在做）的不要。
|
||||
|
||||
「上次剖析」和表清单显示的是同一项（last_profiles，真正开始过的最近一项）：上次真正剖析完整了，后来某个任务没轮到它
|
||||
就停了，不算没做完——不然表清单一边显示「上次剖析：完成」，一边说「上次剖析没做完」。一次都没开始过的，按最近一项算。"""
|
||||
if not isinstance(meta, dict) or meta.get("is_view") or checks <= 0:
|
||||
return None
|
||||
if last is None:
|
||||
return "never"
|
||||
if last.get("queued") or last["status"] in OPEN:
|
||||
return None
|
||||
if last["status"] in RETRYABLE:
|
||||
return "unfinished"
|
||||
if catalog_profile.pending_relation_paths(notes) - set(last.get("paths") or ()):
|
||||
return "new_relations"
|
||||
return None
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 建任务、重试、停止
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
#: 撞上剖析任务时说法的最后一句（「请等这次剖析结束（或先停止它）{then}」）缺省是建剖析任务的；导入目录、同步起草撞上
|
||||
#: 时各说各的（catalog_jobs）：那时要做的是导入、起草，不是开始新的剖析
|
||||
THEN_PROFILE = "再开始新的剖析"
|
||||
|
||||
|
||||
def busy_message(row: Any, brief: dict[str, Any] | None, *, table: str | None = None,
|
||||
position: int | None = None, then: str = THEN_PROFILE) -> str:
|
||||
"""同源已有剖析任务时的说法：写明进行到第几张；从表详情点「剖析这张表」时再说这张表在不在那个任务里、排第几；最后
|
||||
一句按调用方要做的事说（then）。"""
|
||||
name = getattr(row, "name", "") or ""
|
||||
if brief is None:
|
||||
text = f"数据源「{name}」正在剖析"
|
||||
elif brief["status"] == "queued":
|
||||
text = f"数据源「{name}」已有一个排队中的剖析任务（共 {brief['total']} 张表）"
|
||||
else:
|
||||
text = f"数据源「{name}」正在剖析：第 {min(brief['done'] + 1, brief['total'])} / {brief['total']} 张表"
|
||||
if table is not None:
|
||||
text += f"，{table} 在这次剖析里，排第 {position} 张" if position else f"，{table} 不在这次剖析里"
|
||||
return text + f"。请等这次剖析结束（或先停止它）{then}"
|
||||
|
||||
|
||||
async def ensure_free(session: AsyncSession, row: Any, tables: list[str] | None = None, *,
|
||||
then: str = THEN_PROFILE) -> None:
|
||||
"""这个源没有排队或进行中的剖析任务；有就抛 ProfileBusy。只指定了一张表时，说法里带上这张表在不在那个任务里；then 是
|
||||
说法的最后一句要做的事（见 busy_message）。"""
|
||||
job = await active_job(session, row.id)
|
||||
if job is None:
|
||||
return
|
||||
brief = await job_brief(session, job)
|
||||
table = position = None
|
||||
if tables and len(tables) == 1:
|
||||
key = catalog.resolve_table_name(getattr(row, "schema_cache", None), str(tables[0])) or str(tables[0])
|
||||
table = key
|
||||
names = [r.table_name for r in await task_rows(session, job.id)]
|
||||
position = names.index(key) + 1 if key in names else None
|
||||
raise ProfileBusy(busy_message(row, brief, table=table, position=position, then=then))
|
||||
|
||||
|
||||
#: 挑出 0 张表时的说法
|
||||
NOTHING_NEEDED = ("没有需要剖析的表：能剖析的表都剖析过了，上次剖析也都做完了，之后没有新的待核实关系。"
|
||||
"要重新剖析，请选「全部表」或指定要剖析的表")
|
||||
NOTHING_AT_ALL = ("没有可以剖析的表：表结构里的表都是视图，或者没有可剖析的项（要核对的关联关系、像码值的列、日期列都没有，"
|
||||
"或者都被遮罩了）。请先起草数据目录，或指定要剖析的表")
|
||||
|
||||
|
||||
def _skipped_task(job_id: str, seq: int, name: str, reason: str, why: str, checks: list[dict[str, Any]],
|
||||
skipped: list[ProfileSkip], now: Any) -> CatalogProfileTask:
|
||||
entries = [s.to_dict() for s in skipped] + [ProfileSkip("table", name, reason, why).to_dict()]
|
||||
return CatalogProfileTask(job_id=job_id, seq=seq, table_name=name, status="skipped", checks=checks or None,
|
||||
skipped=entries, skipped_count=len(entries),
|
||||
unfinished=sum(1 for e in entries if e["reason"] in UNFINISHED_REASONS),
|
||||
summary=_empty_summary(), error=why, finished_at=now)
|
||||
|
||||
|
||||
async def _pick(session: AsyncSession, row: Any, source: Any, *, tables: list[str] | None,
|
||||
scope: str) -> list[tuple[str, str | None, str | None, list[Any]]]:
|
||||
"""要剖析的表：[(表名, 跳过的原因 / None, 整句, 检查项)]，按剖析顺序。"""
|
||||
cache = getattr(source, "schema_cache", None) or {}
|
||||
all_tables = cache.get("tables") or {}
|
||||
masked = catalog_profile.masked_of(getattr(row, "options", None))
|
||||
entries = await catalog.read_catalog(session, source.id)
|
||||
|
||||
def notes(name: str) -> dict[str, Any]:
|
||||
entry = entries.get(name)
|
||||
return entry.notes if entry else {}
|
||||
|
||||
def plan(name: str) -> list[Any]:
|
||||
return catalog_profile.plan_checks(name, all_tables.get(name), notes(name), tables=all_tables, masked=masked)
|
||||
|
||||
if scope == "selected":
|
||||
picked: list[tuple[str, str | None, str | None, list[Any]]] = []
|
||||
seen: set[str] = set()
|
||||
for name in dict.fromkeys(str(t) for t in tables or []):
|
||||
key = catalog.resolve_table_name(cache, name)
|
||||
if key is None:
|
||||
picked.append((name, "missing", catalog.missing_table_error(name), []))
|
||||
continue
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
if (all_tables.get(key) or {}).get("is_view"):
|
||||
picked.append((key, "view", VIEW_ERROR, []))
|
||||
continue
|
||||
checks = plan(key)
|
||||
if not any(c.skip is None for c in checks):
|
||||
picked.append((key, "nothing", NOTHING_ERROR, checks))
|
||||
else:
|
||||
picked.append((key, None, None, checks))
|
||||
return picked
|
||||
usage = await catalog.table_usage(session, source)
|
||||
last = await last_profiles(session, source.id) if scope == "needed" else {}
|
||||
order = {name: i for i, name in enumerate(all_tables)}
|
||||
candidates: list[tuple[str, list[Any]]] = []
|
||||
for name, meta in all_tables.items():
|
||||
if not isinstance(meta, dict) or meta.get("is_view"):
|
||||
continue
|
||||
checks = plan(name)
|
||||
runnable = sum(1 for c in checks if c.skip is None)
|
||||
if runnable == 0:
|
||||
continue
|
||||
if scope == "needed" and due_reason(meta, notes(name), last.get(name), runnable) is None:
|
||||
continue
|
||||
candidates.append((name, checks))
|
||||
if not candidates:
|
||||
raise ProfileNothing(NOTHING_NEEDED if scope == "needed" else NOTHING_AT_ALL)
|
||||
# 还有待核实（推断）关系的表排在前面：剖析首先是来核实它们的；再按使用次数、表结构顺序
|
||||
candidates.sort(key=lambda c: (not catalog_profile.has_pending_relations(notes(c[0])), -usage.get(c[0], 0),
|
||||
order[c[0]]))
|
||||
return [(name, None, None, checks) for name, checks in candidates]
|
||||
|
||||
|
||||
async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: list[str] | None, scope: str,
|
||||
settings: ProfileSettings, actor: str | None,
|
||||
retry_of: str | None = None) -> tuple[CatalogProfileJob, list[CatalogProfileTask]]:
|
||||
"""建一个排队的剖析任务（还没开始，调用方接着 manager.start）。row 是数据源记录（遮罩按它的设置），source 是读表结构
|
||||
用的源（上传源是绑定当前快照的视图）。同源已有排队或进行中的剖析任务抛 ProfileBusy；挑出 0 张表抛 ProfileNothing。
|
||||
同源有起草任务时照建，排在它后面。建完顺手清掉超出保留数的旧任务。"""
|
||||
await ensure_free(session, row, tables)
|
||||
picked = await _pick(session, row, source, tables=tables, scope=scope)
|
||||
job = CatalogProfileJob(id=new_id(), source_id=row.id, status="queued", scope=scope,
|
||||
settings=settings.to_dict(), created_by=actor, retry_of=retry_of)
|
||||
now = utcnow()
|
||||
tasks: list[CatalogProfileTask] = []
|
||||
for seq, (name, reason, why, checks) in enumerate(picked):
|
||||
shown = [c.as_check() for c in checks]
|
||||
if reason is not None:
|
||||
rule = [ProfileSkip(c.kind, c.target, c.skip[0], c.skip[1], c.path) for c in checks if c.skip]
|
||||
tasks.append(_skipped_task(job.id, seq, name, reason, str(why), shown, rule, now))
|
||||
continue
|
||||
tasks.append(CatalogProfileTask(job_id=job.id, seq=seq, table_name=name, status="pending", checks=shown,
|
||||
summary=_empty_summary()))
|
||||
session.add(job)
|
||||
try:
|
||||
await session.flush()
|
||||
session.add_all(tasks)
|
||||
await session.commit()
|
||||
except IntegrityError as e:
|
||||
# 部分唯一索引：另一个请求同时给这个源建了剖析任务
|
||||
await session.rollback()
|
||||
raise ProfileBusy(busy_message(row, None)) from e
|
||||
try:
|
||||
await prune(session, row.id)
|
||||
except Exception: # noqa: BLE001 - 清旧任务失败不影响这次剖析
|
||||
logger.exception("清理数据源 %s 的旧剖析任务时出错", row.id)
|
||||
await session.rollback()
|
||||
return job, tasks
|
||||
|
||||
|
||||
async def retry_tables(session: AsyncSession, job: CatalogProfileJob) -> list[str]:
|
||||
"""重试要带上的表：原任务里 failed、partial、cancelled 的，按原来的顺序。没有抛 NothingToRetry。"""
|
||||
names = [t.table_name for t in await task_rows(session, job.id) if t.status in RETRYABLE]
|
||||
if not names:
|
||||
raise NothingToRetry("这次剖析里没有失败、部分完成或停止了的表，不需要重试")
|
||||
return names
|
||||
|
||||
|
||||
async def prune(session: AsyncSession, source_id: str, *, keep: int | None = None) -> int:
|
||||
"""保留：这个源最近 keep（缺省 KEEP_JOBS）个结束了的剖析任务留着，更早的删掉；某张表「上次剖析」（真正开始过的最近
|
||||
一次，见 _ranked）所在的任务不删——排队中还没轮到、没开始就结束了的那一项不算，不然新建一个任务就能把上一次真正剖析
|
||||
清掉。返回删了几个。"""
|
||||
keep = KEEP_JOBS if keep is None else keep
|
||||
finished = list((await session.execute(
|
||||
select(CatalogProfileJob.id).where(CatalogProfileJob.source_id == source_id,
|
||||
CatalogProfileJob.status.in_(FINISHED))
|
||||
.order_by(CatalogProfileJob.created_at.desc(), CatalogProfileJob.id.desc())
|
||||
)).scalars())
|
||||
old = finished[keep:]
|
||||
if not old:
|
||||
return 0
|
||||
inner = _ranked(source_id, CatalogProfileTask.job_id)
|
||||
protected = set((await session.execute(select(inner.c.job_id).where(inner.c.shown == 1))).scalars())
|
||||
victims = [job_id for job_id in old if job_id not in protected]
|
||||
if not victims:
|
||||
return 0
|
||||
await session.execute(delete(CatalogProfileTask).where(CatalogProfileTask.job_id.in_(victims)))
|
||||
await session.execute(delete(CatalogProfileJob).where(CatalogProfileJob.id.in_(victims)))
|
||||
await session.commit()
|
||||
return len(victims)
|
||||
|
||||
|
||||
async def _close_tasks(session: AsyncSession, job_id: str, task_status: str, task_error: str | None,
|
||||
*, why: str) -> None:
|
||||
"""还没结束的表记 task_status（原因 task_error）；它们的检查项里还排着的、正在做的记为跳过（停止、中断是 cancelled，
|
||||
出错是 error，说明是「{why}，未检查」），计进跳过的项和没做完的项。
|
||||
|
||||
还没结束的表没走到整张表结束那一步：目录只在那一步和这张表的结果一起写（catalog_profile._record，同一个事务），
|
||||
所以这里已经查完（done）的项结论也没写进目录——进程没了、任务整体出错（快照被改动……）都是这样。它们同样记为跳过、
|
||||
原因同上，说明是「{why}，查完了但结论没有写进数据目录」，算没做完；这张表的原因写明查完的几项没写进目录。不能留着
|
||||
「完成」：检查清单上标着完成、目录里却没有，看的人会以为结论已经在目录里了。"""
|
||||
now = utcnow()
|
||||
reason = "cancelled" if task_status == "cancelled" else "error"
|
||||
detail, unsaved = _UNCHECKED.format(why=why), _UNSAVED.format(why=why)
|
||||
tasks = (await session.execute(
|
||||
select(CatalogProfileTask).where(CatalogProfileTask.job_id == job_id, CatalogProfileTask.status.in_(OPEN))
|
||||
)).scalars()
|
||||
for task in tasks:
|
||||
checks = [dict(c) for c in task.checks or [] if isinstance(c, dict)]
|
||||
skipped = list(task.skipped or [])
|
||||
listed = {(s.get("kind"), s.get("target"), s.get("path")) for s in skipped if isinstance(s, dict)}
|
||||
done = rest = 0
|
||||
for check in checks:
|
||||
state = check.get("state")
|
||||
if state == "done":
|
||||
done += 1
|
||||
check.update(state="skipped", reason=reason, detail=unsaved)
|
||||
elif state in _CHECK_OPEN:
|
||||
rest += 1
|
||||
check.update(state="skipped", reason=reason, detail=detail)
|
||||
elif state != "skipped":
|
||||
continue
|
||||
# 跳过的项按检查项列全：进行中的表只随时落了检查项，按规则不查的（遮罩……）也要进跳过的项
|
||||
key = (check.get("kind"), check.get("target"), check.get("path"))
|
||||
if key not in listed:
|
||||
listed.add(key)
|
||||
skipped.append(ProfileSkip(check.get("kind") or "", check.get("target") or "",
|
||||
check.get("reason") or "", check.get("detail") or "",
|
||||
check.get("path")).to_dict())
|
||||
error = unsaved_error(task_error or why, done, rest) if done else task_error
|
||||
task.status, task.error, task.finished_at, task.current = task_status, error, now, None
|
||||
task.checks, task.skipped = checks or None, skipped or None
|
||||
task.skipped_count = len(skipped)
|
||||
task.unfinished = sum(1 for s in skipped if s.get("reason") in UNFINISHED_REASONS)
|
||||
|
||||
|
||||
async def _close(session: AsyncSession, job_id: str, job_status: str, job_error: str | None, task_status: str,
|
||||
task_error: str | None, *, why: str | None = None) -> None:
|
||||
"""收尾：还没结束的表记 task_status（见 _close_tasks），任务（还没结束的话）记 job_status。提交。"""
|
||||
await _close_tasks(session, job_id, task_status, task_error, why=why or task_error or STOPPED_ERROR)
|
||||
await session.execute(
|
||||
update(CatalogProfileJob).where(CatalogProfileJob.id == job_id, CatalogProfileJob.status.in_(ACTIVE))
|
||||
.values(status=job_status, error=job_error, finished_at=utcnow()).execution_options(synchronize_session=False))
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def cancel(session: AsyncSession, job: CatalogProfileJob) -> None:
|
||||
"""停止一个剖析任务。排队的直接 cancelled（在队里等着的后台执行跟着停掉、出队）；进行中的交给 manager 停：正在等的那条
|
||||
查询不再等（库那边按单条时限截止），那张表做完的项照常写进目录，后面的表 cancelled。已经结束的抛 ProfileFinished。"""
|
||||
if job.status in FINISHED:
|
||||
raise ProfileFinished("这次剖析已经结束了，不需要停止")
|
||||
if job.status == "queued":
|
||||
claimed = await session.execute(
|
||||
update(CatalogProfileJob).where(CatalogProfileJob.id == job.id, CatalogProfileJob.status == "queued")
|
||||
.values(status="cancelled", finished_at=utcnow()).execution_options(synchronize_session=False))
|
||||
if claimed.rowcount == 1:
|
||||
await _close_tasks(session, job.id, "cancelled", None, why=STOPPED_ERROR)
|
||||
await session.commit()
|
||||
await manager.stop(job.id, "user", grace=STOP_GRACE_S)
|
||||
return
|
||||
await session.rollback() # 刚被后台认领、开始了：按进行中的停
|
||||
if await manager.stop(job.id, "user", grace=STOP_GRACE_S):
|
||||
return
|
||||
# 库里说在进行，这个进程里却没有在跑它（记录对不上）：直接记停止
|
||||
await _close(session, job.id, "cancelled", None, "cancelled", None)
|
||||
|
||||
|
||||
async def startup() -> int:
|
||||
"""服务启动时：排队、进行中的剖析任务标成 interrupted，没结束的表记 cancelled、写明服务重启，不自动续跑；再把剖析设置的
|
||||
旧键迁移掉。返回标了几个任务。"""
|
||||
async def interrupt(session: AsyncSession, job_id: str) -> None:
|
||||
await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR)
|
||||
|
||||
n = await runtime.interrupt_on_startup(CatalogProfileJob, interrupt, running=manager.running)
|
||||
if n:
|
||||
logger.warning("服务重启：%d 个数据剖析任务没有做完，已标为中断，可在数据目录页重试", n)
|
||||
async with SessionLocal() as session:
|
||||
migrated = await catalog_profile.migrate_settings(session)
|
||||
if migrated:
|
||||
logger.info("数据剖析设置：%d 个数据源去掉了按整次剖析算的旧预算", migrated)
|
||||
return n
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 在后台剖析
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
class _ProfileRun(runtime.JobRun):
|
||||
"""一个剖析任务在后台的一次执行（进队、停止、收尾的框架在 runtime.JobRun）。"""
|
||||
|
||||
kind = "profile"
|
||||
model = CatalogProfileJob
|
||||
|
||||
def __init__(self, job_id: str) -> None:
|
||||
super().__init__(job_id)
|
||||
self.actor: str | None = None
|
||||
self.cx: catalog_profile.ProfileContext | None = None
|
||||
#: 正在剖析的那张表：seq 和开始的时刻（进度回调据此落库）
|
||||
self._seq = -1
|
||||
self._table_started = 0.0
|
||||
|
||||
# ---------------------------------------------------------------- 停止、出错时收尾
|
||||
|
||||
async def _stopped(self) -> None:
|
||||
if self.stop_reason == "user":
|
||||
await self._end("cancelled", None, "cancelled", None)
|
||||
else:
|
||||
await self._end("interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR)
|
||||
|
||||
async def _failed(self, reason: str) -> None:
|
||||
await self._end("failed", reason, "failed", reason)
|
||||
|
||||
async def _crashed(self, e: Exception) -> None:
|
||||
from app.core.errors import describe_exception, first_line
|
||||
|
||||
logger.exception("数据剖析任务 %s 出错", self.job_id)
|
||||
reason = JOB_ERROR.format(line=first_line(e) or describe_exception(e))
|
||||
await self._end("failed", reason, "failed", reason)
|
||||
|
||||
async def _end(self, job_status: str, job_error: str | None, task_status: str, task_error: str | None) -> None:
|
||||
"""停止、出错时收尾：等手上的库写入写完，记下查询计数，再把没结束的表和任务记下来。收尾本身写不进去只记日志（下次
|
||||
启动时 startup 会把它标成中断）。"""
|
||||
try:
|
||||
await self._settle()
|
||||
await self._write(self._save_counters)
|
||||
await self._write(_close, self.job_id, job_status, job_error, task_status, task_error)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("数据剖析任务 %s 收尾时写库出错", self.job_id)
|
||||
|
||||
def stop_text(self) -> str:
|
||||
return STOPPED_ERROR if self.stop_reason == "user" else INTERRUPTED_ERROR
|
||||
|
||||
# ---------------------------------------------------------------- 主流程
|
||||
|
||||
async def _run(self) -> None:
|
||||
if not await self._write(self._claim):
|
||||
return # 开始之前就被停止了
|
||||
async with SessionLocal() as session:
|
||||
job = await session.get(CatalogProfileJob, self.job_id)
|
||||
row = await session.get(DataSource, job.source_id) if job is not None else None
|
||||
if job is None or row is None:
|
||||
raise runtime.JobFailed(SOURCE_GONE)
|
||||
from app.data import table_versions
|
||||
|
||||
try:
|
||||
source = await table_versions.resolve_source(session, row)
|
||||
except table_versions.SnapshotMissing as e:
|
||||
raise runtime.JobFailed(str(e)) from e
|
||||
pending = [(t.seq, t.table_name) for t in await task_rows(session, job.id) if t.status == "pending"]
|
||||
settings = ProfileSettings.from_dict(job.settings)
|
||||
self.actor = job.created_by
|
||||
options = dict(row.options or {})
|
||||
try:
|
||||
self.cx = catalog_profile.make_context(source, settings, options=options)
|
||||
except ValueError as e: # 不支持的数据库类型
|
||||
raise runtime.JobFailed(str(e)) from e
|
||||
hooks = TableHooks(progress=self._progress, query=self._queried, stop_text=self.stop_text)
|
||||
for seq, name in pending:
|
||||
# 每张表开始前重读数据源：停用了、关掉了剖析就停下；遮罩按最新的设置
|
||||
stop = await self._write(self._recheck)
|
||||
if stop is not None:
|
||||
await self._write(self._halt, stop, None)
|
||||
break
|
||||
self._table_started = time.monotonic()
|
||||
await self._write(self._start_table, seq)
|
||||
|
||||
async def recorded(session: AsyncSession, result: TableResult, seq: int = seq) -> None:
|
||||
await self._save_table(session, seq, result)
|
||||
|
||||
hooks.recorded = recorded
|
||||
self._seq = seq
|
||||
try:
|
||||
await catalog_profile.profile_table(self.cx, name, db=self._write, hooks=hooks, actor=self.actor)
|
||||
except SnapshotTampered as e:
|
||||
raise runtime.JobFailed(str(e)) from e
|
||||
if self.cx.runner.stopped:
|
||||
# 任务级止损：后面的表 cancelled，带同一句原因
|
||||
await self._write(self._halt, self.cx.runner.error, self.cx.runner.stopped)
|
||||
break
|
||||
await self._settle()
|
||||
await self._write(self._finish)
|
||||
|
||||
# ---------------------------------------------------------------- 进度回调（引擎调）
|
||||
|
||||
async def _progress(self, result: TableResult) -> None:
|
||||
await self._write(self._save_progress, self._seq, result)
|
||||
|
||||
async def _queried(self, result: TableResult) -> None:
|
||||
await self._write(self._save_queries, self._seq, result)
|
||||
|
||||
# ---------------------------------------------------------------- 库操作
|
||||
|
||||
def _task(self, *where: Any) -> Any:
|
||||
return update(CatalogProfileTask).where(CatalogProfileTask.job_id == self.job_id, *where) \
|
||||
.execution_options(synchronize_session=False)
|
||||
|
||||
def _counters(self) -> dict[str, Any]:
|
||||
runner = self.cx.runner if self.cx is not None else None
|
||||
if runner is None:
|
||||
return {}
|
||||
return {"queries": runner.queries, "timeouts": runner.timeouts, "errors": runner.errors,
|
||||
"notice": runner.notice}
|
||||
|
||||
async def _save_counters(self, session: AsyncSession) -> None:
|
||||
if values := self._counters():
|
||||
await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id).values(**values)
|
||||
.execution_options(synchronize_session=False))
|
||||
await session.commit()
|
||||
|
||||
async def _claim(self, session: AsyncSession) -> bool:
|
||||
return await runtime.claim(session, CatalogProfileJob, self.job_id)
|
||||
|
||||
async def _recheck(self, session: AsyncSession) -> str | None:
|
||||
"""重读数据源：停用了、关掉了剖析返回停下的整句；遮罩换成最新的设置。数据源没了抛 JobFailed。"""
|
||||
job = await session.get(CatalogProfileJob, self.job_id)
|
||||
row = await session.get(DataSource, job.source_id) if job is not None else None
|
||||
if row is None:
|
||||
raise runtime.JobFailed(SOURCE_GONE)
|
||||
if row.enabled is False:
|
||||
return SOURCE_DISABLED.format(name=row.name)
|
||||
if not catalog_profile.profile_settings(row.options).enabled:
|
||||
return PROFILE_TURNED_OFF.format(name=row.name)
|
||||
assert self.cx is not None
|
||||
self.cx.masked = catalog_profile.masked_of(row.options)
|
||||
return None
|
||||
|
||||
async def _halt(self, session: AsyncSession, error: str | None, stop_reason: str | None) -> None:
|
||||
"""整个任务停下（止损、数据源停用或关掉了剖析）：后面的表 cancelled、error 是同一句原因；任务记下原因，状态到收尾时
|
||||
按逐表算。"""
|
||||
await _close_tasks(session, self.job_id, "cancelled", error, why=error or STOPPED_ERROR)
|
||||
await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id)
|
||||
.values(error=error, stop_reason=stop_reason, **self._counters())
|
||||
.execution_options(synchronize_session=False))
|
||||
await session.commit()
|
||||
|
||||
async def _start_table(self, session: AsyncSession, seq: int) -> None:
|
||||
await session.execute(self._task(CatalogProfileTask.seq == seq, CatalogProfileTask.status == "pending")
|
||||
.values(status="running", started_at=utcnow()))
|
||||
await session.commit()
|
||||
|
||||
def _elapsed(self) -> int:
|
||||
return int((time.monotonic() - self._table_started) * 1000)
|
||||
|
||||
async def _save_progress(self, session: AsyncSession, seq: int, result: TableResult) -> None:
|
||||
"""一项检查开始或结束：这张表的检查项、正在做哪一项、查询数、发过的查询、行数估算，任务的查询计数。"""
|
||||
await session.execute(self._task(CatalogProfileTask.seq == seq, CatalogProfileTask.status == "running")
|
||||
.values(checks=[dict(c) for c in result.checks] or None, current=result.current,
|
||||
queries=result.queries, elapsed_ms=self._elapsed(),
|
||||
query_log=list(result.query_log) or None,
|
||||
row_estimate=result.row_estimate.to_dict() if result.row_estimate else None))
|
||||
if values := self._counters():
|
||||
await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id).values(**values)
|
||||
.execution_options(synchronize_session=False))
|
||||
await session.commit()
|
||||
|
||||
async def _save_queries(self, session: AsyncSession, seq: int, result: TableResult) -> None:
|
||||
"""发出一条查询之后：这张表和任务的查询计数、提醒（发过的查询本身在检查项结束时一起写）。"""
|
||||
await session.execute(self._task(CatalogProfileTask.seq == seq, CatalogProfileTask.status == "running")
|
||||
.values(queries=result.queries, elapsed_ms=self._elapsed()))
|
||||
if values := self._counters():
|
||||
await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id).values(**values)
|
||||
.execution_options(synchronize_session=False))
|
||||
await session.commit()
|
||||
|
||||
async def _save_table(self, session: AsyncSession, seq: int, result: TableResult) -> None:
|
||||
"""一张表结束：整张表的结果。和写目录在同一个事务里，由 catalog_profile._record 一起提交，这里不提交。"""
|
||||
await session.execute(
|
||||
self._task(CatalogProfileTask.seq == seq, CatalogProfileTask.status.in_(OPEN))
|
||||
.values(status=result.status, checks=[dict(c) for c in result.checks] or None, current=None,
|
||||
queries=result.queries, elapsed_ms=result.elapsed_ms,
|
||||
row_estimate=result.row_estimate.to_dict() if result.row_estimate else None,
|
||||
findings=list(result.findings) or None, skipped=[s.to_dict() for s in result.skipped] or None,
|
||||
date_ranges=list(result.date_ranges) or None, query_log=list(result.query_log) or None,
|
||||
summary=result.summary(), skipped_count=result.skipped_count, unfinished=result.unfinished,
|
||||
added=result.added, updated=result.updated, removed=result.removed, version=result.version,
|
||||
error=result.error, finished_at=utcnow()))
|
||||
if values := self._counters():
|
||||
await session.execute(update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id).values(**values)
|
||||
.execution_options(synchronize_session=False))
|
||||
|
||||
async def _finish(self, session: AsyncSession) -> None:
|
||||
"""剖析完：万一还有没收尾的表（进度回调写库失败之类）记失败；有 failed、partial、cancelled 的表是 partial，否则
|
||||
succeeded。"""
|
||||
await _close_tasks(session, self.job_id, "failed", UNFINISHED_ERROR, why=UNFINISHED_ERROR)
|
||||
statuses = list((await session.execute(
|
||||
select(CatalogProfileTask.status).where(CatalogProfileTask.job_id == self.job_id))).scalars())
|
||||
status = "partial" if any(s in RETRYABLE for s in statuses) else "succeeded"
|
||||
await session.execute(
|
||||
update(CatalogProfileJob).where(CatalogProfileJob.id == self.job_id, CatalogProfileJob.status == "running")
|
||||
.values(status=status, finished_at=utcnow(), **self._counters())
|
||||
.execution_options(synchronize_session=False))
|
||||
await session.commit()
|
||||
|
||||
|
||||
class ProfileJobs(runtime.JobManager):
|
||||
"""进程内的剖析任务（start / stop / wait / running / shutdown 见 runtime.JobManager）。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(_ProfileRun, label="数据剖析任务", name="catalog-profile")
|
||||
|
||||
|
||||
manager = ProfileJobs()
|
||||
|
||||
|
||||
__all__ = ["INTERRUPTED_ERROR", "KEEP_JOBS", "MAX_TABLES", "NOTHING_AT_ALL", "NOTHING_NEEDED", "NothingToRetry",
|
||||
"ProfileBusy", "ProfileFinished", "ProfileJobs", "ProfileNothing", "THEN_PROFILE", "active_job",
|
||||
"busy_message", "cancel", "create_job", "due_reason", "ensure_free", "job_brief", "job_dict", "job_out",
|
||||
"last_detail", "last_profiles", "latest_job", "load", "load_task", "manager", "prune", "public_last",
|
||||
"retry_tables", "startup", "task_detail", "task_row", "task_rows", "unsaved_error", "waiting_for"]
|
||||
@@ -28,8 +28,8 @@ tables: {表名: {notes, version}}}，含全部状态的项，表结构里已经
|
||||
没有是 none。
|
||||
- 文件格式不对（不是对象、格式标记或版本不对、表的目录不是对象、起草要求不是文字或超长）抛 TransferInvalid，接口回 422。
|
||||
|
||||
有排队或进行中的起草任务时不导入（接口先调 catalog_jobs.ensure_idle，409 catalog_draft_busy）；目标数据源没有表结构时
|
||||
接口回 409（什么也对不上）。
|
||||
有排队或进行中的起草、剖析任务时不导入（接口经 catalog_jobs.import_write，409 catalog_draft_busy / profile_busy，导入期间
|
||||
占着这个源的队）；目标数据源没有表结构时接口回 409（什么也对不上）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ MAX_QUERY_TIMEOUT_S = 600
|
||||
#: 运行事件里有查询结果的预览
|
||||
MASK_COLUMNS_OPTION = "mask_columns"
|
||||
|
||||
#: 数据源 options 里的数据剖析设置（一个对象：开关、查询次数上限、单条时限……）。取值和校验见
|
||||
#: 数据源 options 里的数据剖析设置（一个对象：开关、每张表的查询数和时长、单条时限……）。取值和校验见
|
||||
#: data/catalog_profile.py；放在这里只为让拼连接串时摘掉它
|
||||
CATALOG_PROFILE_OPTION = "catalog_profile"
|
||||
|
||||
|
||||
@@ -435,6 +435,96 @@ class CatalogDraftTask(Base):
|
||||
finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None)
|
||||
|
||||
|
||||
class CatalogProfileJob(Base, TimestampMixin):
|
||||
"""数据目录的一次后台剖析任务（data/catalog_profile_jobs.py）。逐表的结果在 catalog_profile_tasks。
|
||||
|
||||
以前剖析是一次同步长请求：预算按整次剖析算（选 50 张表只做得完两三张）、不能停、关掉页面报告就没了，对业务库查了什么
|
||||
也无从查。现在建任务后在后台逐表剖析，每张表各有自己的查询次数和时长，进度、发过的查询（字面量去掉）随时落库。生命周期
|
||||
和起草任务是同一套（data/catalog_job_runtime.py），表各建各的：进度字段完全不同。
|
||||
|
||||
同一个数据源同时只有一个排队或进行中的剖析任务：接口先查一遍，部分唯一索引兜底。和同源的起草任务按建立的先后排队。
|
||||
服务重启时还没结束的标成 interrupted，不自动续跑。挂外键、级联删除：任务是数据源的附属记录。
|
||||
"""
|
||||
|
||||
__tablename__ = "catalog_profile_jobs"
|
||||
__table_args__ = (
|
||||
Index("ix_catalog_profile_jobs_source_created", "source_id", "created_at"),
|
||||
Index("ux_catalog_profile_jobs_active", "source_id", unique=True,
|
||||
sqlite_where=text("status IN ('queued', 'running')")),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id)
|
||||
source_id: Mapped[str] = mapped_column(ForeignKey("data_sources.id", ondelete="CASCADE"))
|
||||
# queued | running | succeeded | partial | failed | cancelled | interrupted
|
||||
status: Mapped[str] = mapped_column(String(20), default="queued")
|
||||
# selected 指定的表 | needed 需要剖析的表 | all 全部表
|
||||
scope: Mapped[str] = mapped_column(String(20), default="selected")
|
||||
# 建任务时的剖析设置快照（catalog_profile.ProfileSettings.to_dict）
|
||||
settings: Mapped[dict[str, Any]] = mapped_column(default=dict)
|
||||
# 署名（开了登录是认证过的用户名，否则请求头 X-Actor 的自报）
|
||||
created_by: Mapped[str | None] = mapped_column(String(100), default=None)
|
||||
started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None)
|
||||
# 整个任务发出的查询数、其中超时的、失败的（守卫拒掉的没发出去，不算）
|
||||
queries: Mapped[int] = mapped_column(Integer, default=0)
|
||||
timeouts: Mapped[int] = mapped_column(Integer, default=0)
|
||||
errors: Mapped[int] = mapped_column(Integer, default=0)
|
||||
# 任务级止损停下的原因：failed 连续失败、slow 超时比例；没有为 NULL
|
||||
stop_reason: Mapped[str | None] = mapped_column(String(20), default=None)
|
||||
# 提醒：最近的查询多数超时时的一句话，好转后清掉
|
||||
notice: Mapped[str | None] = mapped_column(Text, default=None)
|
||||
# 任务整体出错、止损、数据源中途停用或关掉剖析时的整句原因
|
||||
error: Mapped[str | None] = mapped_column(Text, default=None)
|
||||
# 重试建的任务记下是重试哪一个
|
||||
retry_of: Mapped[str | None] = mapped_column(String(32), default=None)
|
||||
|
||||
|
||||
class CatalogProfileTask(Base):
|
||||
"""剖析任务里的一张表。按 seq（剖析顺序）排。状态：pending 排队、running 剖析中、ok 每项都有结论、partial 有项因为
|
||||
运行原因没查成、failed 写入一直冲突或整张表出错、skipped 视图 / 表结构里没有 / 空表 / 没有可剖析的项、cancelled
|
||||
（停止、止损或服务重启）。
|
||||
|
||||
检查项、发现、跳过的项、日期范围、发过的查询都是 JSON：条数受每张表的查询上限约束，大小有界。发过的查询里的 SQL
|
||||
字面量都换成了「?」（sql_semantics.normalize_sql）：抽样到的键值不进 AgentLab 自己的库。"""
|
||||
|
||||
__tablename__ = "catalog_profile_tasks"
|
||||
__table_args__ = (UniqueConstraint("job_id", "seq"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id)
|
||||
job_id: Mapped[str] = mapped_column(ForeignKey("catalog_profile_jobs.id", ondelete="CASCADE"), index=True)
|
||||
seq: Mapped[int] = mapped_column(Integer)
|
||||
# 表结构里的键；表结构里没有的（skipped）是请求里的原样名字
|
||||
table_name: Mapped[str] = mapped_column(String(255))
|
||||
status: Mapped[str] = mapped_column(String(20), default="pending")
|
||||
# 检查项：[{kind, target, path, state, reason, detail, queries}]。建任务时按当时的目录列一遍，开始剖析这张表时按最新的
|
||||
# 目录重列
|
||||
checks: Mapped[list[Any] | None] = mapped_column(default=None)
|
||||
queries: Mapped[int] = mapped_column(Integer, default=0)
|
||||
elapsed_ms: Mapped[int] = mapped_column(Integer, default=0)
|
||||
# 进行中时正在做哪一项的整句（「正在核对 orders.customer_id → customers.id」）
|
||||
current: Mapped[str | None] = mapped_column(Text, default=None)
|
||||
# 行数估算 {rows, method, at_least}
|
||||
row_estimate: Mapped[dict[str, Any] | None] = mapped_column(default=None)
|
||||
findings: Mapped[list[Any] | None] = mapped_column(default=None)
|
||||
skipped: Mapped[list[Any] | None] = mapped_column(default=None)
|
||||
date_ranges: Mapped[list[Any] | None] = mapped_column(default=None)
|
||||
# 发过的查询：[{at, kind, target, sql, ms, rows, outcome, detail}]
|
||||
query_log: Mapped[list[Any] | None] = mapped_column(default=None)
|
||||
# 发现的合计 {relations_checked, relations_verified, codes, codes_pending, business_dates}；跳过的项数、其中因为运行
|
||||
# 原因没查成的（表清单的「上次剖析」读这几列，不必读整份发现）
|
||||
summary: Mapped[dict[str, Any] | None] = mapped_column(default=None)
|
||||
skipped_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
unfinished: Mapped[int] = mapped_column(Integer, default=0)
|
||||
added: Mapped[int] = mapped_column(Integer, default=0)
|
||||
updated: Mapped[int] = mapped_column(Integer, default=0)
|
||||
removed: Mapped[int] = mapped_column(Integer, default=0)
|
||||
version: Mapped[int] = mapped_column(Integer, default=0)
|
||||
# 整张表的原因：表结构里没有、视图、空表、没有可剖析的项、写入一直冲突、停止 / 中断 / 止损
|
||||
error: Mapped[str | None] = mapped_column(Text, default=None)
|
||||
started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None)
|
||||
|
||||
|
||||
class Setting(Base, TimestampMixin):
|
||||
"""键值形式的用户设置，前端 Settings 页直接读写。"""
|
||||
|
||||
|
||||
+9
-4
@@ -93,14 +93,18 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("上传表格的版本整理失败，服务照常启动")
|
||||
|
||||
# 上次没做完的数据目录起草任务：标成中断（没结束的表记停止），不自动续跑——拷来的数据目录（检查脚本、验收副本）
|
||||
# 启动时不能替人去调模型。界面上点「重试」接着起草没写好的表
|
||||
from app.data import catalog_jobs
|
||||
# 上次没做完的数据目录起草、剖析任务：标成中断（没结束的表记停止），不自动续跑——拷来的数据目录（检查脚本、验收副本）
|
||||
# 启动时不能替人去调模型、查业务库。界面上点「重试」接着做没做完的表。剖析设置里按整次剖析算的旧预算顺带迁移掉
|
||||
from app.data import catalog_jobs, catalog_profile_jobs
|
||||
|
||||
try:
|
||||
await catalog_jobs.startup()
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("数据目录起草任务的启动整理失败，服务照常启动")
|
||||
try:
|
||||
await catalog_profile_jobs.startup()
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("数据剖析任务的启动整理失败，服务照常启动")
|
||||
|
||||
await run_manager.setup()
|
||||
# 流程：存量的待审批补建待办（派给任何人，幂等）；上次进程没了时已发出、没拿到结果的写入收成结果未知、安排核实
|
||||
@@ -129,9 +133,10 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
# 先停调度，再停运行：停运行的途中不该再有新运行被发起。起草任务停下来标成中断，下次启动不会再续
|
||||
# 先停调度，再停运行：停运行的途中不该再有新运行被发起。起草、剖析任务停下来标成中断，下次启动不会再续
|
||||
await scheduler.stop()
|
||||
await catalog_jobs.manager.shutdown()
|
||||
await catalog_profile_jobs.manager.shutdown()
|
||||
await run_manager.shutdown()
|
||||
from app.sandbox.manager import sandbox_manager
|
||||
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
"""数据剖析接口：POST /api/datasources/{id}/catalog/profile（阶段 2A）。
|
||||
"""数据剖析：POST /api/datasources/{id}/catalog/profile-jobs 建后台任务，逐表剖析（阶段 2A，后台任务改造之后）。
|
||||
|
||||
剖析是数据目录里唯一对业务库发查询的地方，这里逐条钉住方案定的约束：默认关闭、只读、每条查询都过守卫、
|
||||
查询次数和单条时限、总时长、遮罩的列不取样、大表不做整表统计、同一个数据源同时只有一次剖析；
|
||||
每张表的查询次数和时长、单条时限、遮罩的列不取样、大表不做整表统计、同一个数据源同时只有一个剖析任务；
|
||||
以及剖析的结论怎么进目录：命名推断对上数据的升为有确证（来源记为数据剖析，和外键分开）、对不上的写明
|
||||
覆盖率、人工确认过的只补覆盖率和基数、状态列得到码值候选、只有一个日期列的表提议业务日期。
|
||||
任务本身（排队、停止、重启、重试、止损、范围、留痕、保留、设置迁移）在 test_catalog_profile_jobs.py。
|
||||
|
||||
所有测试都在合成的景区库上做（tests/fixtures/catalog/scenic.py），数据和名称全部虚构。
|
||||
"""
|
||||
@@ -22,7 +23,7 @@ from httpx import ASGITransport, AsyncClient
|
||||
from sqlalchemy import event
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
from app.data import catalog, catalog_profile
|
||||
from app.data import catalog_profile, catalog_profile_jobs
|
||||
from app.data import engine as data_engine
|
||||
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings
|
||||
from app.db.base import SessionLocal
|
||||
@@ -65,6 +66,8 @@ async def make_source(client):
|
||||
return sid
|
||||
|
||||
yield _make
|
||||
# 后台任务先收完（停不下来的停掉），再删源：删源会级联删掉任务
|
||||
await catalog_profile_jobs.manager.shutdown(grace=5)
|
||||
for sid in made:
|
||||
await drop_source(sid)
|
||||
|
||||
@@ -119,7 +122,39 @@ def spy(monkeypatch, scenic_db):
|
||||
|
||||
|
||||
def _url(sid: str) -> str:
|
||||
return f"/api/datasources/{sid}/catalog/profile"
|
||||
return f"/api/datasources/{sid}/catalog/profile-jobs"
|
||||
|
||||
|
||||
async def _start(client, sid: str, tables: list[str] | None = None, **body):
|
||||
payload = {**({"tables": tables} if tables is not None else {}), **body}
|
||||
return await client.post(_url(sid), json=payload, headers=ACTOR)
|
||||
|
||||
|
||||
async def _job(client, sid: str, job_id: str) -> dict:
|
||||
r = await client.get(f"{_url(sid)}/{job_id}")
|
||||
assert r.status_code == 200, r.text
|
||||
return r.json()["job"]
|
||||
|
||||
|
||||
async def _detail(client, sid: str, job_id: str, seq: int) -> dict:
|
||||
r = await client.get(f"{_url(sid)}/{job_id}/tasks/{seq}")
|
||||
assert r.status_code == 200, r.text
|
||||
return r.json()["task"]
|
||||
|
||||
|
||||
async def _finished(client, sid: str, job_id: str) -> dict:
|
||||
"""等任务做完，返回做完的任务；job["tables"] 是每张表的明细（检查项、发现、跳过的项、发过的查询），按剖析顺序。"""
|
||||
await catalog_profile_jobs.manager.wait(job_id, timeout=30)
|
||||
job = await _job(client, sid, job_id)
|
||||
job["tables"] = [await _detail(client, sid, job_id, t["seq"]) for t in job["tasks"]]
|
||||
return job
|
||||
|
||||
|
||||
async def _profile(client, sid: str, tables: list[str] | None = None, **body) -> dict:
|
||||
"""建剖析任务（202）、等它做完，返回做完的任务（见 _finished）。"""
|
||||
r = await _start(client, sid, tables, **body)
|
||||
assert r.status_code == 202, r.text
|
||||
return await _finished(client, sid, r.json()["job"]["id"])
|
||||
|
||||
|
||||
async def _notes(client, sid: str, table: str) -> dict:
|
||||
@@ -147,31 +182,35 @@ async def test_profile_is_off_by_default_and_says_where_to_turn_it_on(client, ma
|
||||
sid = await make_source(scenic_db, draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
r = await _start(client, sid, ["visits"])
|
||||
assert r.status_code == 409
|
||||
detail = r.json()["detail"]
|
||||
assert "数据源设置" in detail and "开启数据剖析" in detail
|
||||
assert r.json()["code"] == "profile_disabled" # 界面按机读码认是哪一种，不从原话里认
|
||||
assert watch.executed == [] # 一条查询都没发
|
||||
assert (await client.get(f"{_url(sid)}/latest")).json()["job"] is None, "没建任务"
|
||||
|
||||
|
||||
async def test_settings_are_saved_and_validated_through_datasource_update(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db)
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=0)})
|
||||
assert r.status_code == 422 and "查询次数上限" in r.json()["detail"]
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(table_max_queries=0)})
|
||||
assert r.status_code == 422 and "每张表最多查询数" in r.json()["detail"]
|
||||
# 机读码和出错的那一项：表单按 field 把报错落到那一格
|
||||
assert (r.json()["code"], r.json()["field"]) == ("profile_settings_invalid", "max_queries")
|
||||
assert (r.json()["code"], r.json()["field"]) == ("profile_settings_invalid", "table_max_queries")
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(table_max_s=5)})
|
||||
assert r.status_code == 422 and (r.json()["code"], r.json()["field"]) == ("profile_settings_invalid", "table_max_s")
|
||||
assert "每张表时长上限" in r.json()["detail"]
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": {PROFILE_OPTION: {"enabled": "是"}}})
|
||||
assert r.status_code == 422 and (r.json()["code"], r.json()["field"]) == ("profile_settings_invalid", "enabled")
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(rows=5)})
|
||||
assert r.status_code == 422 and r.json()["code"] == "profile_settings_invalid" and "field" not in r.json()
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(max_queries=40, sample_size=500)})
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on(table_max_queries=30, sample_size=500)})
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "max_queries": 40, "sample_size": 500}
|
||||
# 设好以后就能剖析；返回里带着这次实际用的设置
|
||||
r = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["settings"]["max_queries"] == 40 and r.json()["settings"]["sample_size"] == 500
|
||||
assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "table_max_queries": 30, "sample_size": 500}
|
||||
# 设好以后就能剖析；任务里记着这次实际用的设置（快照）
|
||||
job = await _profile(client, sid, ["channels"])
|
||||
assert job["settings"] == {"enabled": True, "table_max_queries": 30, "table_max_s": 60, "query_timeout_s": 10,
|
||||
"sample_size": 500, "max_scan_rows": 100_000}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 剖析的结论
|
||||
@@ -182,15 +221,15 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
body = await _profile(client, sid, ["visits", "channels"])
|
||||
|
||||
# 所有发到业务库的语句都经过了守卫
|
||||
# 所有发到业务库的语句都经过了守卫；任务记的查询数就是发到库的条数，也是逐表的合计
|
||||
assert watch.all_guarded(), watch.executed
|
||||
assert body["queries_used"] == len(watch.executed) <= body["settings"]["max_queries"]
|
||||
assert body["queries_used"] == sum(t["queries"] for t in body["tables"])
|
||||
assert body["stopped"] is None and body["actor"] == "周宁" and body["profiled_at"]
|
||||
assert body["queries"] == len(watch.executed) == sum(t["queries"] for t in body["tables"])
|
||||
assert all(t["queries"] <= body["settings"]["table_max_queries"] for t in body["tables"])
|
||||
assert (body["status"], body["stop_reason"], body["error"]) == ("succeeded", None, None)
|
||||
assert body["created_by"] == "周宁" and body["finished_at"] and body["scope"] == "selected"
|
||||
assert [t["table_name"] for t in body["tasks"]] == ["visits", "channels"], "按给的顺序"
|
||||
|
||||
notes = (await _notes(client, sid, "visits"))
|
||||
assert notes["updated_by"] == "周宁"
|
||||
@@ -226,6 +265,7 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d
|
||||
assert "channel_code" not in channels.get("columns", {}) # 唯一约束列不是码值
|
||||
|
||||
visits = _table(body, "visits")
|
||||
assert visits["status"] == "ok" and visits["unfinished"] == 0
|
||||
kinds = sorted(f["kind"] for f in visits["findings"])
|
||||
# 两条命名推断的关系（gate_id、member_id），加两条外键关系（park_id、ticket_type_id）：外键的只核实基数
|
||||
assert kinds == ["business_date", "codes", "relation", "relation", "relation", "relation"]
|
||||
@@ -243,6 +283,19 @@ async def test_profile_verifies_name_relations_and_proposes_codes_and_business_d
|
||||
assert visits["row_estimate"] == {"rows": 1500, "method": "count", "at_least": None}
|
||||
# 更新的 4 项：两条命名关系升为有确证，两条外键关系补上用数据核实过的基数
|
||||
assert visits["added"] == 2 and visits["updated"] == 4 and visits["error"] is None
|
||||
# 发现的合计：逐表、整个任务各一份
|
||||
assert visits["summary"] == {"relations_checked": 4, "relations_verified": 4, "codes": 1,
|
||||
"codes_pending": blank, "business_dates": 1}
|
||||
assert body["summary"]["relations_checked"] == 4 and body["summary"]["codes"] == 2
|
||||
# 检查项：每项都结束了，按种类的进度对得上
|
||||
assert visits["checks_done"] == visits["checks_total"] == len(visits["checks"])
|
||||
assert visits["checks_by_kind"]["relation"] == [4, 4] and visits["checks_by_kind"]["date"] == [1, 1]
|
||||
assert {c["state"] for c in visits["checks"]} == {"done"}
|
||||
assert sum(c["queries"] for c in visits["checks"]) + sum(
|
||||
1 for q in visits["query_log"] if q["kind"] == "row_estimate" and q["target"] == "visits") >= 1
|
||||
# 第一条查询是估行数
|
||||
assert visits["query_log"][0]["kind"] == "row_estimate" and visits["query_log"][0]["outcome"] == "ok"
|
||||
assert len(visits["query_log"]) == visits["queries"]
|
||||
|
||||
|
||||
async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client, make_source, tmp_path):
|
||||
@@ -261,16 +314,15 @@ async def test_wrong_name_relation_stays_proposed_with_coverage_in_note(client,
|
||||
before = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
|
||||
assert (before["to_table"], before["source"], before["status"]) == ("lots", "name", "proposed")
|
||||
|
||||
r = await client.post(_url(sid), json={"tables": ["parking_records"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = await _profile(client, sid, ["parking_records"])
|
||||
rel = _relation((await _notes(client, sid, "parking_records"))["notes"], "lot_id")
|
||||
assert rel["status"] == "proposed" and rel["source"] == "profile"
|
||||
assert rel["coverage"] == 0.15
|
||||
assert "抽样覆盖率 15%" in rel["note"] and "可能不是这条关系" in rel["note"] and "20 个" in rel["note"]
|
||||
finding = _table(r.json(), "parking_records")["findings"][0]
|
||||
finding = _table(body, "parking_records")["findings"][0]
|
||||
assert finding["status"] == "proposed" and finding["coverage"] == 0.15
|
||||
# 两个日期类列：不提议业务日期，取值范围只在返回里
|
||||
ranges = {d["column"] for d in _table(r.json(), "parking_records")["date_ranges"]}
|
||||
# 两个日期类列：不提议业务日期，取值范围只在剖析结果里
|
||||
ranges = {d["column"] for d in _table(body, "parking_records")["date_ranges"]}
|
||||
assert ranges == {"entered_at"} # exited_at 全是空值
|
||||
assert "business_date" not in (await _notes(client, sid, "parking_records"))["notes"]
|
||||
|
||||
@@ -284,22 +336,21 @@ async def test_confirmed_relation_only_gets_coverage_and_cardinality(client, mak
|
||||
"if_version": detail["version"]})
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = await _profile(client, sid, ["visits"])
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert (rel["source"], rel["status"]) == ("name", "confirmed")
|
||||
assert rel["coverage"] == 1.0 and rel["cardinality"] == "many_to_one"
|
||||
# 人定的基数不改；原来就有基数（命名推断记的多对一），剖析也不给它记「用数据核实过」
|
||||
assert "cardinality_checked" not in rel
|
||||
finding = next(f for f in _table(r.json(), "visits")["findings"] if f.get("columns") == ["gate_id"])
|
||||
finding = next(f for f in _table(body, "visits")["findings"] if f.get("columns") == ["gate_id"])
|
||||
assert finding["status"] == "confirmed" and finding["confirmed"] is True
|
||||
|
||||
|
||||
async def test_rerun_with_same_data_changes_nothing(client, make_source, scenic_db):
|
||||
"""同一天、数据没变：第二次剖析不改目录、不升版本（冻结进快照的目录不会无故变）。"""
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
first = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
|
||||
second = (await client.post(_url(sid), json={"tables": ["visits"]})).json()
|
||||
first = await _profile(client, sid, ["visits"])
|
||||
second = await _profile(client, sid, ["visits"])
|
||||
a, b = _table(first, "visits"), _table(second, "visits")
|
||||
assert (b["added"], b["updated"], b["removed"]) == (0, 0, 0)
|
||||
assert b["version"] == a["version"]
|
||||
@@ -318,32 +369,38 @@ async def test_masked_columns_are_never_queried(client, make_source, scenic_db,
|
||||
draft=["visits", "channels"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = await _profile(client, sid, ["visits", "channels"])
|
||||
assert watch.all_guarded()
|
||||
assert watch.mentions("member_id") == [] and watch.mentions("channel_type") == []
|
||||
visits, channels = _table(r.json(), "visits"), _table(r.json(), "channels")
|
||||
visits, channels = _table(body, "visits"), _table(body, "channels")
|
||||
masked = [s for s in visits["skipped"] + channels["skipped"] if s["reason"] == "masked"]
|
||||
assert {s["target"] for s in masked} == {"member_id → members.id", "channel_type"}
|
||||
assert all("遮罩" in s["detail"] for s in masked)
|
||||
# 按规则不查的不算没做完：表照样是完成
|
||||
assert visits["status"] == "ok" and visits["unfinished"] == 0 and visits["skipped_count"] == len(visits["skipped"])
|
||||
# 留痕里也没有遮罩的列
|
||||
assert not any("member_id" in q["sql"].lower() or "channel_type" in q["sql"].lower()
|
||||
for t in body["tables"] for q in t["query_log"])
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "member_id")["source"] == "name" # 没动
|
||||
assert _relation(notes, "gate_id")["status"] == "verified" # 其余照常
|
||||
assert "channel_type" not in ((await _notes(client, sid, "channels"))["notes"].get("columns") or {})
|
||||
|
||||
|
||||
async def test_query_budget_stops_profiling_and_reports_the_rest(client, make_source, scenic_db, spy):
|
||||
sid = await make_source(scenic_db, options=_on(max_queries=3), draft=["visits"])
|
||||
async def test_each_table_has_its_own_query_budget(client, make_source, scenic_db, spy):
|
||||
"""逐表预算：visits 用完自己的 3 条查询记部分完成，剩下的项记「这张表的查询次数用完」、旧结论原样留着；后面的
|
||||
channels 照样拿满额，做完。"""
|
||||
sid = await make_source(scenic_db, options=_on(table_max_queries=3), draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits", "channels"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
assert body["queries_used"] == 3 == len(watch.executed) and body["stopped"] == "budget"
|
||||
body = await _profile(client, sid, ["visits", "channels"])
|
||||
assert body["queries"] == len(watch.executed) and body["stop_reason"] is None
|
||||
visits, channels = _table(body, "visits"), _table(body, "channels")
|
||||
assert "budget" in _reasons(visits)
|
||||
assert channels["queries"] == 0 and _reasons(channels) == {"budget"}
|
||||
assert all("查询次数" in s["detail"] for s in visits["skipped"] + channels["skipped"] if s["reason"] == "budget")
|
||||
assert visits["status"] == "partial" and visits["queries"] == 3
|
||||
assert "budget" in _reasons(visits) and visits["unfinished"] >= 1
|
||||
assert all("这张表的查询次数" in s["detail"] for s in visits["skipped"] if s["reason"] == "budget")
|
||||
assert channels["status"] == "ok" and 0 < channels["queries"] <= 3 and "budget" not in _reasons(channels)
|
||||
assert body["status"] == "partial", "有部分完成的表"
|
||||
# 没查完的关系不改：原来的命名推断原样留着
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "member_id")
|
||||
assert (rel["source"], rel["status"]) == ("name", "proposed")
|
||||
@@ -362,18 +419,20 @@ async def test_single_query_timeout_is_recorded_and_profiling_continues(client,
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", slow_gate)
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
visits = _table(r.json(), "visits")
|
||||
body = await _profile(client, sid, ["visits"])
|
||||
visits = _table(body, "visits")
|
||||
timeout = [s for s in visits["skipped"] if s["reason"] == "timeout"]
|
||||
assert [s["target"] for s in timeout] == ["gate_id → gates.id"] and "超时" in timeout[0]["detail"]
|
||||
assert r.json()["stopped"] is None
|
||||
assert body["stop_reason"] is None and body["timeouts"] == 1
|
||||
assert visits["status"] == "partial", "超时算运行原因没查成"
|
||||
assert [q["outcome"] for q in visits["query_log"]].count("timeout") == 1
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "gate_id")["source"] == "name" # 超时的不改
|
||||
assert _relation(notes, "member_id")["status"] == "verified" # 其余照常
|
||||
|
||||
|
||||
async def test_total_deadline_stops_profiling(make_source, scenic_db, monkeypatch):
|
||||
async def test_each_table_has_its_own_deadline(make_source, scenic_db, monkeypatch):
|
||||
"""逐表时长：每条查询慢 0.5 秒，visits 的时长（1.4 秒）用完只停它自己，channels 另拿一份满额的时长。"""
|
||||
sid = await make_source(scenic_db, draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
|
||||
@@ -384,13 +443,13 @@ async def test_total_deadline_stops_profiling(make_source, scenic_db, monkeypatc
|
||||
monkeypatch.setattr(data_engine, "run_query", slow)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
settings = ProfileSettings(enabled=True, max_total_s=1.4)
|
||||
report = await catalog_profile.profile_catalog(session, row, row, tables=["visits"], actor=None,
|
||||
settings=settings)
|
||||
assert report.stopped == "deadline"
|
||||
assert 1 <= report.queries_used <= 3
|
||||
reasons = {s.reason for t in report.tables for s in t.skipped}
|
||||
assert "deadline" in reasons
|
||||
cx = catalog_profile.make_context(row, ProfileSettings(enabled=True, table_max_s=1.4), options=row.options)
|
||||
visits = await catalog_profile.profile_table(cx, "visits")
|
||||
assert visits.status == "partial" and 1 <= visits.queries <= 3
|
||||
assert "deadline" in {s.reason for s in visits.skipped}
|
||||
assert any("这张表的时长" in s.detail for s in visits.skipped if s.reason == "deadline")
|
||||
channels = await catalog_profile.profile_table(cx, "channels")
|
||||
assert channels.status == "ok" and channels.queries == 2, [s.to_dict() for s in channels.skipped]
|
||||
|
||||
|
||||
async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, spy):
|
||||
@@ -398,15 +457,15 @@ async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, sp
|
||||
sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits"])
|
||||
watch = spy(scenic_db)
|
||||
watch.clear()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
visits = _table(r.json(), "visits")
|
||||
body = await _profile(client, sid, ["visits"])
|
||||
visits = _table(body, "visits")
|
||||
assert visits["row_estimate"] == {"rows": None, "method": "count", "at_least": 101}
|
||||
on_visits = [s for s in watch.executed if "from visits" in s.lower()]
|
||||
assert on_visits and all("LIMIT" in s for s in on_visits), on_visits
|
||||
assert not any("MIN(" in s for s in on_visits)
|
||||
assert "too_large" in _reasons(visits)
|
||||
assert any("太大" in s["detail"] or "上限" in s["detail"] for s in visits["skipped"] if s["reason"] == "too_large")
|
||||
assert visits["status"] == "ok", "表太大是按规则不查，不算没做完"
|
||||
# 父表有主键，覆盖率又够：照样有确证；子表一侧没核对唯一，按多对一记，不算用数据核实过基数
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert rel["status"] == "verified" and rel["cardinality"] == "many_to_one"
|
||||
@@ -416,7 +475,7 @@ async def test_large_tables_get_no_full_scans(client, make_source, scenic_db, sp
|
||||
# ---------------------------------------------------------------- 接口
|
||||
|
||||
|
||||
async def test_concurrent_profile_on_same_source_returns_409(client, make_source, scenic_db, monkeypatch):
|
||||
async def test_second_profile_job_on_same_source_returns_409(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
real = data_engine.run_query
|
||||
started, release = asyncio.Event(), asyncio.Event()
|
||||
@@ -427,29 +486,39 @@ async def test_concurrent_profile_on_same_source_returns_409(client, make_source
|
||||
return await real(source, sql, limits=limits)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", held)
|
||||
first = asyncio.create_task(client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR))
|
||||
first = await _start(client, sid, ["visits", "channels"])
|
||||
assert first.status_code == 202, first.text
|
||||
await asyncio.wait_for(started.wait(), 10)
|
||||
second = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert second.status_code == 409 and "正在进行数据剖析" in second.json()["detail"]
|
||||
assert second.json()["code"] == "profile_busy"
|
||||
second = await _start(client, sid, ["visits"])
|
||||
assert second.status_code == 409 and second.json()["code"] == "profile_busy"
|
||||
# 说清是谁在跑、跑到哪；只点了一张表时说这张表在不在那个任务里、排第几
|
||||
assert "正在剖析：第 1 / 2 张表" in second.json()["detail"] and "visits 在这次剖析里，排第 1 张" in second.json()["detail"]
|
||||
other = await _start(client, sid, ["gates"])
|
||||
assert other.status_code == 409 and "gates 不在这次剖析里" in other.json()["detail"]
|
||||
active = (await client.get(f"{_url(sid)}/active")).json()["job"]
|
||||
assert active["id"] == first.json()["job"]["id"] and active["status"] == "running"
|
||||
listed = (await client.get(f"/api/datasources/{sid}/catalog")).json()
|
||||
assert listed["active_profile_job"] == {"id": active["id"], "status": "running", "total": 2, "done": 0,
|
||||
"waiting_for": None}
|
||||
release.set()
|
||||
done = await asyncio.wait_for(first, 30)
|
||||
assert done.status_code == 200, done.text
|
||||
await catalog_profile_jobs.manager.wait(active["id"], timeout=30)
|
||||
assert (await client.get(f"{_url(sid)}/active")).json()["job"] is None
|
||||
# 做完以后可以再来
|
||||
again = await client.post(_url(sid), json={"tables": ["channels"]}, headers=ACTOR)
|
||||
assert again.status_code == 200, again.text
|
||||
again = await _start(client, sid, ["channels"])
|
||||
assert again.status_code == 202, again.text
|
||||
await catalog_profile_jobs.manager.wait(again.json()["job"]["id"], timeout=30)
|
||||
|
||||
|
||||
async def test_blocked_profile_says_which_case_by_code(client, make_source, scenic_db, monkeypatch):
|
||||
"""剖析开始不了的 409 都带机读码：数据源停用、没有表结构、快照被改动（未开启、正在剖析见上面两条）。"""
|
||||
from app.data.engine import SnapshotTampered
|
||||
"""剖析开始不了的 409 都带机读码：数据源停用、没有表结构、快照登记被改动（未开启、正忙见上面两条）。"""
|
||||
from app.data import table_versions
|
||||
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
row.enabled = False
|
||||
await session.commit()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
r = await _start(client, sid, ["visits"])
|
||||
assert r.status_code == 409 and r.json()["code"] == "datasource_inactive" and "已停用" in r.json()["detail"]
|
||||
|
||||
async with SessionLocal() as session:
|
||||
@@ -457,103 +526,50 @@ async def test_blocked_profile_says_which_case_by_code(client, make_source, scen
|
||||
row.enabled = True
|
||||
row.schema_cache = {}
|
||||
await session.commit()
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
r = await _start(client, sid, ["visits"])
|
||||
assert r.status_code == 409 and r.json()["code"] == "schema_missing" and "探查结构" in r.json()["detail"]
|
||||
|
||||
r = await client.post(f"/api/datasources/{sid}/introspect")
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
async def tampered(*_args, **_kwargs):
|
||||
raise table_versions.SnapshotManifestMismatch("数据版本的哈希登记与快照清单不一致，已拒绝查询")
|
||||
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(table_versions, "resolve_source", tampered)
|
||||
r = await _start(client, sid, ["visits"])
|
||||
assert r.status_code == 409 and r.json()["code"] == "snapshot_tampered"
|
||||
assert (await client.get(f"{_url(sid)}/latest")).json()["job"] is None, "开始不了的都没建任务"
|
||||
|
||||
|
||||
async def test_a_tampered_snapshot_during_profiling_fails_the_job(client, make_source, scenic_db, monkeypatch):
|
||||
"""剖析途中数据文件和登记的版本对不上：整个任务停下、记失败，原因照实写；没做完的表也记失败，可以重试。"""
|
||||
from app.data.engine import SnapshotTampered
|
||||
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
|
||||
async def tampered(*_args, **_kwargs):
|
||||
raise SnapshotTampered("数据文件与登记的版本不一致，可能被修改过，已拒绝查询")
|
||||
|
||||
monkeypatch.setattr(catalog_profile, "profile_catalog", tampered)
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]}, headers=ACTOR)
|
||||
assert r.status_code == 409 and r.json()["code"] == "snapshot_tampered"
|
||||
|
||||
|
||||
async def test_default_tables_are_used_tables_with_relation_candidates(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=[])
|
||||
r = await client.post(_url(sid), json={}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
names = [t["table_name"] for t in r.json()["tables"]]
|
||||
assert names and len(names) <= catalog_profile.PROFILE_DEFAULT_TABLES
|
||||
async with SessionLocal() as session:
|
||||
entries = await catalog.read_catalog(session, sid)
|
||||
for name in names:
|
||||
assert catalog_profile._relation_candidates(entries[name].notes), name
|
||||
assert "parks" not in names and "channels" not in names # 没有待核实关系的表不选
|
||||
|
||||
|
||||
async def _candidates(sid: str) -> tuple[list[str], set[str]]:
|
||||
"""默认挑表的候选（目录里有要核对的关系的表，按表结构顺序）和其中还有待核实（推断）关系的那些。"""
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
entries = await catalog.read_catalog(session, sid)
|
||||
order = list((row.schema_cache or {}).get("tables") or {})
|
||||
rels = {name: catalog_profile._relation_candidates(entries[name].notes) for name in order if name in entries}
|
||||
names = [name for name in order if rels.get(name)]
|
||||
return names, {name for name in names if any(r["status"] == "proposed" for r in rels[name])}
|
||||
|
||||
|
||||
async def test_default_pick_skips_empty_tables_and_prefers_pending_relations(client, make_source, tmp_path,
|
||||
monkeypatch):
|
||||
"""B1：表都没被运行查询过（使用次数都是 0）时，以前按表结构顺序取前几张，结果大半是空表（景区库里排在前面的
|
||||
候选表正好多数没有数据）。现在跳过行数估计为 0 的表（记在 empty_tables 里），并且先挑还有待核实关系的表，
|
||||
不把刚核实过的再剖析一遍。默认张数调成 3，好在一个小库上看出先后。"""
|
||||
monkeypatch.setattr(catalog_profile, "PROFILE_DEFAULT_TABLES", 3)
|
||||
path = scenic.build(tmp_path / "b1.db")
|
||||
db = sqlite3.connect(path)
|
||||
try:
|
||||
tables = [r[0] for r in db.execute("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY rowid")]
|
||||
empty = {t for t in tables if db.execute(f'SELECT COUNT(*) FROM "{t}"').fetchone()[0] == 0}
|
||||
finally:
|
||||
db.close()
|
||||
sid = await make_source(str(path), options=_on(), draft=tables)
|
||||
names, pending = await _candidates(sid)
|
||||
full = [name for name in names if name not in empty]
|
||||
# 前提：按表结构顺序的前 3 张候选里有空表（以前就会挑中它们），有数据、还待核实的候选不止 6 张
|
||||
assert set(names[:3]) & empty
|
||||
assert len([name for name in full if name in pending]) >= 6
|
||||
|
||||
first = (await client.post(_url(sid), json={}, headers=ACTOR)).json()
|
||||
picked = [t["table_name"] for t in first["tables"]]
|
||||
assert len(picked) == 3 and not set(picked) & empty, picked
|
||||
assert first["empty_tables"] and set(first["empty_tables"]) <= empty
|
||||
assert all(t["queries"] > 0 for t in first["tables"])
|
||||
assert first["queries_used"] >= sum(t["queries"] for t in first["tables"]) + len(first["empty_tables"])
|
||||
|
||||
# 第二次：先挑还有待核实关系的表；第一次核实过的（关系都升为有确证了）排到后面，这次挑不到
|
||||
names, pending = await _candidates(sid)
|
||||
assert not set(picked) & pending # 第一次挑的三张，关系都核实了
|
||||
second = (await client.post(_url(sid), json={}, headers=ACTOR)).json()
|
||||
again = [t["table_name"] for t in second["tables"]]
|
||||
assert len(again) == 3 and all(name in pending for name in again), (again, pending)
|
||||
assert not set(again) & set(picked) and not set(again) & empty
|
||||
|
||||
|
||||
async def test_profile_note_when_every_candidate_is_empty(client, make_source, tmp_path):
|
||||
path = scenic.build(tmp_path / "b1-empty.db")
|
||||
sid = await make_source(str(path), options=_on(), draft=[])
|
||||
names, _ = await _candidates(sid)
|
||||
db = sqlite3.connect(path)
|
||||
try:
|
||||
for name in names:
|
||||
db.execute(f'DELETE FROM "{name}"')
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
body = (await client.post(_url(sid), json={}, headers=ACTOR)).json()
|
||||
assert body["tables"] == [] and set(body["empty_tables"]) == set(names)
|
||||
assert "空表" in body["note"]
|
||||
monkeypatch.setattr(data_engine, "run_query", tampered)
|
||||
body = await _profile(client, sid, ["visits", "channels"])
|
||||
assert body["status"] == "failed" and body["error"] == "数据文件与登记的版本不一致，可能被修改过，已拒绝查询"
|
||||
assert {t["status"] for t in body["tables"]} == {"failed"}
|
||||
assert all(t["error"] == body["error"] for t in body["tables"])
|
||||
# 没结束的检查项不留在「排队」：记为跳过，原因写清
|
||||
assert all(c["state"] != "pending" for t in body["tables"] for c in t["checks"])
|
||||
|
||||
|
||||
async def test_unknown_table_and_missing_source(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
r = await client.post(_url(sid), json={"tables": ["nope"]}, headers=ACTOR)
|
||||
assert r.status_code == 200
|
||||
assert _table(r.json(), "nope")["error"] and r.json()["queries_used"] == 0
|
||||
body = await _profile(client, sid, ["nope"])
|
||||
nope = _table(body, "nope")
|
||||
assert nope["status"] == "skipped" and nope["error"] == "表结构里没有 nope，请先重新探查结构"
|
||||
assert body["queries"] == 0 and body["status"] == "succeeded"
|
||||
r = await client.post(_url("0" * 32), json={}, headers=ACTOR)
|
||||
assert r.status_code == 404
|
||||
assert (await client.get(f"{_url(sid)}/not-a-job")).status_code == 404
|
||||
assert (await client.get(f"{_url(sid)}/{body['id']}/tasks/9")).status_code == 404
|
||||
|
||||
|
||||
def _orders_db(path, *, items_per_order: int) -> str:
|
||||
@@ -583,27 +599,23 @@ async def test_foreign_key_relations_get_their_cardinality_checked_with_data(cli
|
||||
before = _relation((await _notes(client, sid, "order_items"))["notes"], "order_id")
|
||||
assert (before["source"], before["status"]) == ("fk", "verified")
|
||||
|
||||
r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
body = await _profile(client, sid, ["order_items"])
|
||||
rel = _relation((await _notes(client, sid, "order_items"))["notes"], "order_id")
|
||||
assert (rel["source"], rel["status"]) == ("fk", "verified")
|
||||
assert rel["cardinality"] == cardinality and rel.get("cardinality_checked") is True
|
||||
assert rel.get("coverage") is None
|
||||
finding = next(f for f in _table(r.json(), "order_items")["findings"] if f.get("columns") == ["order_id"])
|
||||
finding = next(f for f in _table(body, "order_items")["findings"] if f.get("columns") == ["order_id"])
|
||||
assert finding["cardinality_checked"] is True and finding["cardinality"] == cardinality
|
||||
|
||||
# 重新剖析、数据没变：版本不升
|
||||
version = (await _notes(client, sid, "order_items"))["version"]
|
||||
r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
await _profile(client, sid, ["order_items"])
|
||||
assert (await _notes(client, sid, "order_items"))["version"] == version
|
||||
|
||||
|
||||
async def test_fanout_on_a_data_checked_foreign_key_is_an_error_again(client, make_source, tmp_path):
|
||||
"""剖析核实过的外键一对多：订单关联明细后对订单金额求和，扇出检查报 error；数据上一对一的不报。"""
|
||||
from app.data.sqlcheck import load_checkers
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource
|
||||
|
||||
sql = "SELECT SUM(o.total_amount) AS gmv FROM orders o JOIN order_items i ON i.order_id = o.id"
|
||||
for items, expected in ((3, "error"), (1, None)):
|
||||
@@ -615,8 +627,7 @@ async def test_fanout_on_a_data_checked_foreign_key_is_an_error_again(client, ma
|
||||
r = await client.put(f"/api/datasources/{sid}/catalog/orders",
|
||||
json={"notes": notes, "if_version": detail["version"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
r = await client.post(_url(sid), json={"tables": ["order_items"]}, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
await _profile(client, sid, ["order_items"])
|
||||
async with SessionLocal() as session:
|
||||
source = await session.get(DataSource, sid)
|
||||
checker = (await load_checkers(session, [source]))[source.name]
|
||||
|
||||
@@ -0,0 +1,765 @@
|
||||
"""数据剖析的后台任务（data/catalog_profile_jobs.py、/api/datasources/{id}/catalog/profile-jobs）。
|
||||
|
||||
剖析的结论怎么进目录、安全约束（只读、守卫、遮罩、大表）在 test_catalog_profile.py；这里钉住任务本身：
|
||||
|
||||
- 同源排队：起草在跑时剖析排队（waiting_for = "draft"），起草结束自动开始；反过来起草排在剖析后面（不 409）；停掉排队的；
|
||||
导入目录、同步起草在剖析时 409 profile_busy；同步起草期间建的剖析任务排在它后面
|
||||
- 停止：做完的项写进目录，没做的保留旧结论、记 cancelled，后面的表 cancelled；已结束的再停 409
|
||||
- 服务重启标中断、不续跑；关停时同样；重试只带没做完的表
|
||||
- 任务级止损：连续 3 条失败、最近 10 条 7 条超时（停下、后面的表带同一句原因），最近 6 条 4 条超时提醒、好转后清掉
|
||||
- 每张表开始前重读数据源：停用、关掉剖析就停；遮罩按最新的设置
|
||||
- 范围：需要剖析 / 全部 / 指定；空表一条查询记 skipped；视图跳过；没有可剖析的项；挑不出表 409；profile_due 的三种情形
|
||||
- 留痕：IN 列表里抽到的键值不进 AgentLab 的库
|
||||
- 设置迁移：旧键启动时删掉，老页面带着旧键保存只摘掉
|
||||
- 保留：超过保留数删旧的，但不删某张表最近一次剖析所在的任务
|
||||
|
||||
表名、列名、数据都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
import uuid
|
||||
from datetime import timedelta
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.data import catalog, catalog_jobs, catalog_profile, catalog_profile_jobs, catalog_transfer
|
||||
from app.data import engine as data_engine
|
||||
from app.data.catalog_profile import PROFILE_OPTION, ProfileSettings, SourceRunner, SqlDialect
|
||||
from app.data.guard import SqlRejected
|
||||
from app.db.base import SessionLocal, utcnow
|
||||
from app.db.models import CatalogProfileJob, CatalogProfileTask, DataSource
|
||||
from tests.test_catalog_profile import ( # noqa: F401
|
||||
ACTOR, _detail, _finished, _job, _notes, _on, _profile, _relation, _start, _table, _url, client, make_source,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def _stop_jobs():
|
||||
yield
|
||||
await catalog_jobs.manager.shutdown(grace=5)
|
||||
await catalog_profile_jobs.manager.shutdown(grace=5)
|
||||
|
||||
|
||||
async def until(predicate, timeout: float = 10.0) -> None:
|
||||
for _ in range(int(timeout / 0.01)):
|
||||
if await predicate():
|
||||
return
|
||||
await asyncio.sleep(0.01)
|
||||
raise AssertionError("等不到")
|
||||
|
||||
|
||||
def _base(sid: str) -> str:
|
||||
return f"/api/datasources/{sid}/catalog"
|
||||
|
||||
|
||||
async def _rows(client, sid: str) -> dict[str, dict]:
|
||||
r = await client.get(_base(sid))
|
||||
assert r.status_code == 200, r.text
|
||||
return {row["table_name"]: row for row in r.json()["tables"]}
|
||||
|
||||
|
||||
class Gate:
|
||||
"""把一个异步函数换成「等放行才执行」：entered 置位说明有调用在等了。"""
|
||||
|
||||
def __init__(self, real, when=lambda *a, **k: True) -> None:
|
||||
self.real, self.when = real, when
|
||||
self.entered, self.release = asyncio.Event(), asyncio.Event()
|
||||
|
||||
async def __call__(self, *args, **kwargs):
|
||||
if self.when(*args, **kwargs):
|
||||
self.entered.set()
|
||||
await self.release.wait()
|
||||
return await self.real(*args, **kwargs)
|
||||
|
||||
|
||||
def _hold_queries(monkeypatch, when=lambda source, sql, **kw: True) -> Gate:
|
||||
gate = Gate(data_engine.run_query, when)
|
||||
monkeypatch.setattr(data_engine, "run_query", gate)
|
||||
return gate
|
||||
|
||||
|
||||
def _db(path, ddl: str, rows: dict[str, list[tuple]] | None = None) -> str:
|
||||
db = sqlite3.connect(path)
|
||||
try:
|
||||
db.executescript(ddl)
|
||||
for table, values in (rows or {}).items():
|
||||
marks = ", ".join("?" for _ in values[0])
|
||||
db.executemany(f"INSERT INTO {table} VALUES ({marks})", values)
|
||||
db.commit()
|
||||
finally:
|
||||
db.close()
|
||||
return str(path)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 同源排队
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_a_profile_job_waits_for_the_draft_job_and_then_starts(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
gate = Gate(catalog.write_draft)
|
||||
monkeypatch.setattr(catalog, "write_draft", gate)
|
||||
draft = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]})).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
|
||||
r = await _start(client, sid, ["channels"])
|
||||
assert r.status_code == 202, r.text # 不报错：排在起草后面
|
||||
job = r.json()["job"]
|
||||
assert job["status"] == "queued" and job["waiting_for"] == "draft"
|
||||
listed = (await client.get(_base(sid))).json()
|
||||
assert listed["active_profile_job"]["waiting_for"] == "draft"
|
||||
assert listed["active_job"]["waiting_for"] is None
|
||||
await asyncio.sleep(0.1)
|
||||
assert (await _job(client, sid, job["id"]))["status"] == "queued", "起草没结束，剖析不开始"
|
||||
|
||||
gate.release.set()
|
||||
await catalog_jobs.manager.wait(draft["id"], timeout=10)
|
||||
done = await _finished(client, sid, job["id"])
|
||||
assert done["status"] == "succeeded" and done["waiting_for"] is None
|
||||
draft_done = (await client.get(f"{_base(sid)}/draft-jobs/{draft['id']}")).json()["job"]
|
||||
assert done["started_at"] >= draft_done["finished_at"], "起草结束后才开始"
|
||||
|
||||
|
||||
async def test_a_draft_job_queues_behind_a_running_profile_job(client, make_source, scenic_db, monkeypatch):
|
||||
"""同种任务互斥、不同种排队：剖析在跑时建起草任务不 409，排在后面（waiting_for = "profile"）。"""
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
gate = _hold_queries(monkeypatch)
|
||||
profile = (await _start(client, sid, ["channels"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
|
||||
r = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]})
|
||||
assert r.status_code == 202, r.text
|
||||
draft = r.json()["job"]
|
||||
assert draft["status"] == "queued" and draft["waiting_for"] == "profile"
|
||||
got = (await client.get(f"{_base(sid)}/draft-jobs/{draft['id']}")).json()["job"]
|
||||
assert got["waiting_for"] == "profile"
|
||||
assert (await client.get(_base(sid))).json()["active_job"]["waiting_for"] == "profile"
|
||||
# 同种任务照旧互斥
|
||||
again = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]})
|
||||
assert again.status_code == 409 and again.json()["code"] == "catalog_draft_busy"
|
||||
|
||||
gate.release.set()
|
||||
await catalog_profile_jobs.manager.wait(profile["id"], timeout=10)
|
||||
await catalog_jobs.manager.wait(draft["id"], timeout=10)
|
||||
got = (await client.get(f"{_base(sid)}/draft-jobs/{draft['id']}")).json()["job"]
|
||||
profile_done = await _job(client, sid, profile["id"])
|
||||
assert got["status"] == "succeeded" and got["started_at"] >= profile_done["finished_at"]
|
||||
|
||||
|
||||
async def test_a_queued_profile_job_can_be_cancelled(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
gate = Gate(catalog.write_draft)
|
||||
monkeypatch.setattr(catalog, "write_draft", gate)
|
||||
draft = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]})).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
job = (await _start(client, sid, ["channels", "visits"])).json()["job"]
|
||||
assert job["tasks"][0]["checks_total"] > 0, "排队的表也先列好了检查项"
|
||||
|
||||
r = await client.post(f"{_url(sid)}/{job['id']}/cancel")
|
||||
assert r.status_code == 200, r.text
|
||||
stopped = r.json()["job"]
|
||||
assert stopped["status"] == "cancelled" and stopped["finished_at"] and stopped["queries"] == 0
|
||||
assert {t["status"] for t in stopped["tasks"]} == {"cancelled"}
|
||||
detail = await _detail(client, sid, job["id"], 0)
|
||||
assert {c["state"] for c in detail["checks"]} == {"skipped"}, "结束了的表上不留排着的检查项"
|
||||
assert {c["reason"] for c in detail["checks"]} == {"cancelled"} and detail["unfinished"] == len(detail["checks"])
|
||||
assert not catalog_profile_jobs.manager.running(job["id"]), "在队里等着的也停掉了"
|
||||
again = await client.post(f"{_url(sid)}/{job['id']}/cancel")
|
||||
assert again.status_code == 409 and again.json()["code"] == "catalog_profile_finished"
|
||||
|
||||
gate.release.set()
|
||||
await catalog_jobs.manager.wait(draft["id"], timeout=10)
|
||||
assert (await client.get(f"{_base(sid)}/draft-jobs/{draft['id']}")).json()["job"]["status"] == "succeeded"
|
||||
|
||||
|
||||
async def test_import_and_sync_draft_are_409_while_profiling(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["gates"])
|
||||
exported = (await client.get(f"{_base(sid)}/export")).json()
|
||||
gate = _hold_queries(monkeypatch)
|
||||
job = (await _start(client, sid, ["channels"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
|
||||
# 说法按这次要做的事说下一步：是在导入、起草，不是要开始新的剖析
|
||||
r = await client.post(f"{_base(sid)}/import", json={"content": exported})
|
||||
assert r.status_code == 409 and r.json()["code"] == "profile_busy", r.text
|
||||
assert "正在剖析" in r.json()["detail"] and r.json()["detail"].endswith("请等这次剖析结束（或先停止它）再导入")
|
||||
r = await client.post(f"{_base(sid)}/draft", json={"tables": ["gates"]})
|
||||
assert r.status_code == 409 and r.json()["code"] == "profile_busy", r.text
|
||||
assert r.json()["detail"].endswith("请等这次剖析结束（或先停止它）再起草")
|
||||
again = await _start(client, sid, ["gates"])
|
||||
assert again.json()["detail"].endswith("请等这次剖析结束（或先停止它）再开始新的剖析")
|
||||
# 库里查过没有任务、队却被剖析占着（任务刚好在收尾）时同样按这次要做的事说
|
||||
row, holder = SimpleNamespace(name="x"), SimpleNamespace(kind="profile")
|
||||
assert str(catalog_jobs._lane_busy(row, holder, "import")).endswith("（或先停止它）再导入")
|
||||
assert str(catalog_jobs._lane_busy(row, holder, "sync_draft")).endswith("（或先停止它）再起草")
|
||||
assert str(catalog_jobs._lane_busy(row, SimpleNamespace(kind="draft"), "import")).endswith("（或先停止它）再导入")
|
||||
|
||||
gate.release.set()
|
||||
await catalog_profile_jobs.manager.wait(job["id"], timeout=10)
|
||||
r = await client.post(f"{_base(sid)}/import", json={"content": exported})
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
|
||||
async def test_a_profile_job_created_during_a_sync_write_runs_after_it(client, make_source, scenic_db, monkeypatch):
|
||||
"""同步起草、导入目录执行期间占着这个源的队：期间新建的剖析任务排在后面，做完才开始。"""
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
gate = Gate(catalog.draft_catalog)
|
||||
monkeypatch.setattr(catalog, "draft_catalog", gate)
|
||||
sync = asyncio.create_task(client.post(f"{_base(sid)}/draft", json={"tables": ["gates"]}))
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
job = (await _start(client, sid, ["channels"])).json()["job"]
|
||||
assert job["status"] == "queued" and job["waiting_for"] == "draft"
|
||||
await asyncio.sleep(0.1)
|
||||
assert (await _job(client, sid, job["id"]))["status"] == "queued"
|
||||
gate.release.set()
|
||||
assert (await sync).status_code == 200
|
||||
assert (await _finished(client, sid, job["id"]))["status"] == "succeeded"
|
||||
|
||||
importing = Gate(catalog_transfer.run_import)
|
||||
monkeypatch.setattr(catalog_transfer, "run_import", importing)
|
||||
exported = (await client.get(f"{_base(sid)}/export")).json()
|
||||
imported = asyncio.create_task(client.post(f"{_base(sid)}/import", json={"content": exported}))
|
||||
await asyncio.wait_for(importing.entered.wait(), 10)
|
||||
job = (await _start(client, sid, ["channels"])).json()["job"]
|
||||
assert job["status"] == "queued" and job["waiting_for"] is None
|
||||
# 导入期间再导入一次：同步写操作一次一个
|
||||
busy = await client.post(f"{_base(sid)}/import", json={"content": exported})
|
||||
assert busy.status_code == 409, busy.text
|
||||
importing.release.set()
|
||||
assert (await imported).status_code == 200
|
||||
assert (await _finished(client, sid, job["id"]))["status"] == "succeeded"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 停止、服务重启、重试
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _at_visits_codes(source, sql, **kw) -> bool:
|
||||
return "GROUP BY status" in sql and "FROM visits" in sql
|
||||
|
||||
|
||||
async def test_stopping_writes_what_is_done_and_keeps_the_rest(client, make_source, scenic_db, monkeypatch):
|
||||
"""停在 visits 的码值上：关系都做完了，照常写进目录；码值、日期没做，记停止、目录不动；后面的 orders 记停止。"""
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"])
|
||||
gate = _hold_queries(monkeypatch, _at_visits_codes)
|
||||
job = (await _start(client, sid, ["channels", "visits", "orders"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
running = await _job(client, sid, job["id"])
|
||||
row = next(t for t in running["tasks"] if t["table_name"] == "visits")
|
||||
assert row["status"] == "running" and row["current"] == "正在取 visits.status 的取值分布"
|
||||
assert row["checks_by_kind"]["relation"] == [4, 4] and row["checks_by_kind"]["codes"] == [0, 1]
|
||||
|
||||
r = await client.post(f"{_url(sid)}/{job['id']}/cancel")
|
||||
assert r.status_code == 200, r.text
|
||||
stopped = r.json()["job"]
|
||||
by = {t["table_name"]: t for t in stopped["tasks"]}
|
||||
assert stopped["status"] == "cancelled"
|
||||
assert by["channels"]["status"] == "ok"
|
||||
assert by["visits"]["status"] == "partial" and by["visits"]["current"] is None
|
||||
assert by["visits"]["error"] == "剖析已停止：已做完的 4 项照常写进了数据目录，剩下 2 项没查"
|
||||
assert by["orders"]["status"] == "cancelled" and by["orders"]["error"] is None
|
||||
|
||||
detail = await _detail(client, sid, job["id"], 1)
|
||||
states = {c["kind"]: (c["state"], c["reason"]) for c in detail["checks"]}
|
||||
assert states["codes"] == ("skipped", "cancelled") and states["date"] == ("skipped", "cancelled")
|
||||
assert detail["query_log"][-1]["outcome"] == "error" and "没有等这条查询的结果" in detail["query_log"][-1]["detail"]
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "gate_id")["status"] == "verified", "做完的项写进了目录"
|
||||
assert "status" not in (notes.get("columns") or {}) and "business_date" not in notes, "没做的不动"
|
||||
again = await client.post(f"{_url(sid)}/{job['id']}/cancel")
|
||||
assert again.status_code == 409 and again.json()["code"] == "catalog_profile_finished"
|
||||
|
||||
|
||||
async def test_shutdown_marks_the_job_interrupted(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
gate = _hold_queries(monkeypatch, _at_visits_codes)
|
||||
job = (await _start(client, sid, ["visits", "orders"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
await catalog_profile_jobs.manager.shutdown(grace=5)
|
||||
got = await _job(client, sid, job["id"])
|
||||
by = {t["table_name"]: t for t in got["tasks"]}
|
||||
assert got["status"] == "interrupted" and got["error"] == "服务重启，剖析中断"
|
||||
assert by["visits"]["status"] == "partial" and by["visits"]["error"].startswith("服务重启，剖析中断：已做完的 4 项")
|
||||
assert by["orders"]["status"] == "cancelled" and by["orders"]["error"] == "服务重启，剖析中断"
|
||||
assert _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")["status"] == "verified"
|
||||
|
||||
|
||||
async def test_startup_marks_unfinished_jobs_interrupted_and_does_not_resume(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
checks = [{"kind": "codes", "target": "status", "path": "columns.status.codes", "state": "done", "reason": None,
|
||||
"detail": None, "queries": 1},
|
||||
{"kind": "date", "target": "visit_time", "path": "business_date", "state": "running", "reason": None,
|
||||
"detail": None, "queries": 0}]
|
||||
async with SessionLocal() as session:
|
||||
job = CatalogProfileJob(source_id=sid, status="running", scope="selected", settings={"enabled": True})
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
session.add(CatalogProfileTask(job_id=job.id, seq=0, table_name="channels", status="ok"))
|
||||
session.add(CatalogProfileTask(job_id=job.id, seq=1, table_name="visits", status="running", checks=checks))
|
||||
session.add(CatalogProfileTask(job_id=job.id, seq=2, table_name="gates", status="pending"))
|
||||
await session.commit()
|
||||
job_id = job.id
|
||||
assert await catalog_profile_jobs.startup() >= 1
|
||||
got = await _job(client, sid, job_id)
|
||||
by = {t["table_name"]: t for t in got["tasks"]}
|
||||
assert got["status"] == "interrupted" and got["error"] == "服务重启，剖析中断" and got["finished_at"]
|
||||
assert by["channels"]["status"] == "ok" and by["channels"]["error"] is None
|
||||
assert by["visits"]["status"] == by["gates"]["status"] == "cancelled"
|
||||
assert by["gates"]["error"] == "服务重启，剖析中断"
|
||||
# 进程没了，visits 没走到写目录那一步：查完的码值也没写进目录，不能还标「完成」
|
||||
assert by["visits"]["error"] == "服务重启，剖析中断：已查完的 1 项没有写进数据目录，剩下 1 项没查"
|
||||
visits = await _detail(client, sid, job_id, 1)
|
||||
assert [(c["state"], c["reason"]) for c in visits["checks"]] == [("skipped", "cancelled"), ("skipped", "cancelled")]
|
||||
assert visits["checks"][0]["detail"] == "服务重启，剖析中断，查完了但结论没有写进数据目录"
|
||||
assert visits["checks"][1]["detail"] == "服务重启，剖析中断，未检查"
|
||||
assert visits["unfinished"] == visits["skipped_count"] == 2 and visits["findings"] == []
|
||||
assert not catalog_profile_jobs.manager.running(job_id), "不自动续跑"
|
||||
|
||||
|
||||
async def test_a_job_failing_mid_table_does_not_claim_the_finished_checks(client, make_source, scenic_db,
|
||||
monkeypatch):
|
||||
"""visits 的 4 条关系核对完、取码值时快照被改动：任务整体失败，visits 没走到写目录那一步。核对完的关系不能还标「完成」：
|
||||
记为跳过（error）、写明没写进目录，算没做完；目录里那几条关系原样不动；重试带上 visits。"""
|
||||
from app.data.engine import SnapshotTampered
|
||||
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
|
||||
async def tampered(*_args, **_kwargs):
|
||||
raise SnapshotTampered("数据文件与登记的版本不一致，可能被修改过，已拒绝查询")
|
||||
|
||||
monkeypatch.setattr(catalog_profile, "_check_codes", tampered)
|
||||
body = await _profile(client, sid, ["visits"])
|
||||
visits = body["tables"][0]
|
||||
assert body["status"] == "failed" and visits["status"] == "failed"
|
||||
relations = [c for c in visits["checks"] if c["kind"] == "relation"]
|
||||
assert len(relations) == 4 and all(c["queries"] > 0 for c in relations), "关系确实查过了"
|
||||
assert all((c["state"], c["reason"]) == ("skipped", "error") for c in visits["checks"])
|
||||
assert relations[0]["detail"] == "数据文件与登记的版本不一致，可能被修改过，已拒绝查询，查完了但结论没有写进数据目录"
|
||||
assert visits["error"] == "数据文件与登记的版本不一致，可能被修改过，已拒绝查询：已查完的 4 项没有写进数据目录，剩下 2 项没查"
|
||||
assert visits["unfinished"] == 6 and visits["findings"] == [] and visits["query_log"], "发过的查询照样留痕"
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "gate_id")["status"] == "proposed" and _relation(notes, "gate_id")["source"] == "name"
|
||||
async with SessionLocal() as session:
|
||||
failed = await session.get(CatalogProfileJob, body["id"])
|
||||
assert await catalog_profile_jobs.retry_tables(session, failed) == ["visits"]
|
||||
|
||||
|
||||
async def test_the_catalog_and_the_table_result_commit_together(client, make_source, scenic_db, monkeypatch):
|
||||
"""目录和这张表的剖析结果在同一个事务里：记结果那一步出错，目录也没写进去，收尾时照实说「没写进目录」。"""
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
real = catalog_profile_jobs._ProfileRun._save_table
|
||||
|
||||
async def broken(self, session, seq, result):
|
||||
await real(self, session, seq, result)
|
||||
raise RuntimeError("disk I/O error")
|
||||
|
||||
monkeypatch.setattr(catalog_profile_jobs._ProfileRun, "_save_table", broken)
|
||||
body = await _profile(client, sid, ["visits"])
|
||||
visits = body["tables"][0]
|
||||
assert body["status"] == "failed" and visits["status"] == "failed"
|
||||
assert visits["error"].endswith("已查完的 6 项没有写进数据目录")
|
||||
notes = (await _notes(client, sid, "visits"))["notes"]
|
||||
assert _relation(notes, "gate_id")["status"] == "proposed", "目录跟着结果一起回滚，没写进去"
|
||||
|
||||
|
||||
async def test_retry_takes_only_unfinished_tables(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(table_max_queries=2), draft=["visits"])
|
||||
first = await _profile(client, sid, ["visits", "channels", "nope"])
|
||||
by = {t["table_name"]: t["status"] for t in first["tasks"]}
|
||||
assert by == {"visits": "partial", "channels": "ok", "nope": "skipped"} and first["status"] == "partial"
|
||||
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on()})
|
||||
assert r.status_code == 200, r.text
|
||||
r = await client.post(f"{_url(sid)}/{first['id']}/retry", headers=ACTOR)
|
||||
assert r.status_code == 202, r.text
|
||||
retry = r.json()["job"]
|
||||
assert retry["retry_of"] == first["id"] and retry["scope"] == "selected"
|
||||
assert [t["table_name"] for t in retry["tasks"]] == ["visits"]
|
||||
assert retry["settings"]["table_max_queries"] == 40, "设置取数据源当前的"
|
||||
done = await _finished(client, sid, retry["id"])
|
||||
assert done["status"] == "succeeded"
|
||||
r = await client.post(f"{_url(sid)}/{retry['id']}/retry")
|
||||
assert r.status_code == 422 and "不需要重试" in r.json()["detail"]
|
||||
|
||||
monkeypatch.setattr(catalog_profile_jobs.manager, "start", lambda job_id: None) # 排着、不开始
|
||||
queued = (await _start(client, sid, ["gates"])).json()["job"]
|
||||
r = await client.post(f"{_url(sid)}/{first['id']}/retry")
|
||||
assert r.status_code == 409 and r.json()["code"] == "profile_busy"
|
||||
await client.post(f"{_url(sid)}/{queued['id']}/cancel")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 任务级止损、每张表开始前重读数据源
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
_MANY = ["visits", "orders", "members", "channels", "ticket_types", "gates", "parks", "stores", "products",
|
||||
"payments", "refunds", "reviews"]
|
||||
|
||||
|
||||
async def test_three_failures_in_a_row_stop_the_job(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
|
||||
async def broken(source, sql, *, limits=None):
|
||||
raise RuntimeError("connection refused by the server\ndetails follow")
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", broken)
|
||||
job = await _profile(client, sid, ["visits", "channels", "orders"])
|
||||
assert job["status"] == "partial" and job["stop_reason"] == "failed"
|
||||
assert job["error"] == "连续 3 条查询失败，已停止剖析：connection refused by the server"
|
||||
assert (job["queries"], job["errors"], job["timeouts"]) == (3, 3, 0)
|
||||
visits, channels, orders = (_table(job, n) for n in ("visits", "channels", "orders"))
|
||||
assert visits["status"] == "partial" and visits["error"] == job["error"]
|
||||
assert "failed" in {s["reason"] for s in visits["skipped"]}
|
||||
assert channels["status"] == orders["status"] == "cancelled"
|
||||
assert channels["error"] == orders["error"] == job["error"] and channels["queries"] == 0
|
||||
|
||||
|
||||
async def test_mostly_timeouts_stop_the_job_as_slow(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
|
||||
async def slow(source, sql, *, limits=None):
|
||||
raise SqlRejected("查询超过 1 秒被中断。请添加 WHERE 条件或 LIMIT 缩小范围。")
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", slow)
|
||||
job = await _profile(client, sid, _MANY)
|
||||
assert job["stop_reason"] == "slow" and job["status"] == "partial"
|
||||
assert job["error"] == "数据库响应慢：最近 10 条查询 7 条超时，已停止，避免给业务库加压"
|
||||
assert job["timeouts"] == job["queries"] == 7
|
||||
assert job["notice"] and job["notice"].startswith("数据库响应慢")
|
||||
cancelled = [t for t in job["tasks"] if t["status"] == "cancelled"]
|
||||
assert cancelled and all(t["error"] == job["error"] for t in cancelled)
|
||||
|
||||
|
||||
def _runner(monkeypatch, outcomes: list[str]) -> SourceRunner:
|
||||
"""SourceRunner 接一个假的查询层：按 outcomes 依次超时（t）、失败（e）或成功（o）。"""
|
||||
seq = iter(outcomes)
|
||||
|
||||
async def fake(source, sql, *, limits=None):
|
||||
step = next(seq)
|
||||
if step == "t":
|
||||
raise SqlRejected("查询超过 1 秒被中断")
|
||||
if step == "e":
|
||||
raise RuntimeError("no such table: orders")
|
||||
return data_engine.QueryResult(columns=["n"], rows=[[1]], row_count=1, truncated=False, elapsed_ms=1,
|
||||
sql=sql)
|
||||
|
||||
monkeypatch.setattr(data_engine, "run_query", fake)
|
||||
source = SimpleNamespace(id="s", name="shop", kind="sqlite", options={})
|
||||
return SourceRunner(source, ProfileSettings(enabled=True), SqlDialect("sqlite"))
|
||||
|
||||
|
||||
async def _feed(runner: SourceRunner, n: int) -> None:
|
||||
for _ in range(n):
|
||||
try:
|
||||
await runner.run("SELECT 1")
|
||||
except catalog_profile._Skip:
|
||||
pass
|
||||
|
||||
|
||||
async def test_the_slow_notice_clears_when_queries_recover(monkeypatch):
|
||||
runner = _runner(monkeypatch, ["t"] * 4 + ["o"] * 3 + ["t"] * 3)
|
||||
await _feed(runner, 4)
|
||||
assert runner.notice == "数据库响应慢：最近 4 条查询有 4 条超时。超时再多会自动停止，也可以先停止、换个时间再剖析"
|
||||
assert runner.stopped is None
|
||||
await _feed(runner, 3)
|
||||
assert runner.notice is None, "好转后清掉"
|
||||
await _feed(runner, 3)
|
||||
# 最近 10 条里 7 条超时：停下，之后再发直接停
|
||||
assert runner.stopped == "slow" and runner.timeouts == 7 and runner.queries == 10
|
||||
assert runner.error == "数据库响应慢：最近 10 条查询 7 条超时，已停止，避免给业务库加压"
|
||||
with pytest.raises(catalog_profile._Stop):
|
||||
await runner.run("SELECT 1")
|
||||
|
||||
|
||||
async def test_a_success_resets_the_failure_count(monkeypatch):
|
||||
runner = _runner(monkeypatch, ["e", "e", "o", "e", "e", "e"])
|
||||
await _feed(runner, 5)
|
||||
assert runner.stopped is None and runner.errors == 4
|
||||
await _feed(runner, 1)
|
||||
assert runner.stopped == "failed" and runner.error == "连续 3 条查询失败，已停止剖析：no such table: orders"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change, said", [({"enabled": False}, "已停用"),
|
||||
({"options": _on(enabled=False)}, "已关闭数据剖析")])
|
||||
async def test_disabling_the_source_stops_before_the_next_table(client, make_source, scenic_db, monkeypatch,
|
||||
change, said):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
gate = _hold_queries(monkeypatch, lambda source, sql, **kw: "FROM channels" in sql)
|
||||
job = (await _start(client, sid, ["channels", "visits", "orders"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
for key, value in change.items():
|
||||
setattr(row, key, value)
|
||||
await session.commit()
|
||||
gate.release.set()
|
||||
done = await _finished(client, sid, job["id"])
|
||||
by = {t["table_name"]: t for t in done["tasks"]}
|
||||
assert by["channels"]["status"] == "ok", "正在做的那张做完"
|
||||
assert by["visits"]["status"] == by["orders"]["status"] == "cancelled"
|
||||
assert said in by["visits"]["error"] and by["visits"]["error"] == done["error"]
|
||||
assert done["status"] == "partial" and done["stop_reason"] is None
|
||||
|
||||
|
||||
async def test_masks_follow_the_latest_settings_table_by_table(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
gate = _hold_queries(monkeypatch, lambda source, sql, **kw: "FROM orders" in sql)
|
||||
job = (await _start(client, sid, ["orders", "channels"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
row.options = {**_on(), "mask_columns": "channel_type"}
|
||||
await session.commit()
|
||||
gate.release.set()
|
||||
done = await _finished(client, sid, job["id"])
|
||||
channels = _table(done, "channels")
|
||||
assert {(s["target"], s["reason"]) for s in channels["skipped"]} >= {("channel_type", "masked")}
|
||||
assert not any("channel_type" in q["sql"].lower() for q in channels["query_log"])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 范围
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_scopes_needed_all_and_selected(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels", "gates", "suppliers"])
|
||||
rows = await _rows(client, sid)
|
||||
assert rows["v_daily_visits"]["is_view"] and rows["v_daily_visits"]["profile_checks"] == 0
|
||||
assert not rows["v_daily_visits"]["profile_due"]
|
||||
assert rows["suppliers"]["profile_checks"] == 0 and not rows["suppliers"]["profile_due"]
|
||||
assert (rows["visits"]["profile_due"], rows["visits"]["profile_due_reason"]) == (True, "never")
|
||||
assert rows["visits"]["last_profile"] is None and rows["visits"]["profile_checks"] == 6
|
||||
expected = [name for name, row in rows.items() if row["profile_checks"] > 0 and not row["is_view"]]
|
||||
|
||||
monkeypatch.setattr(catalog_profile_jobs.manager, "start", lambda job_id: None) # 只看挑了哪些表
|
||||
everything = (await _start(client, sid, scope="all")).json()["job"]
|
||||
names = [t["table_name"] for t in everything["tasks"]]
|
||||
assert everything["scope"] == "all" and sorted(names) == sorted(expected)
|
||||
# 有待核实关系的表在前（剖析首先是来核实它们的），其余按表结构顺序
|
||||
async with SessionLocal() as session:
|
||||
entries = await catalog.read_catalog(session, sid)
|
||||
pending = [catalog_profile.has_pending_relations(entries[n].notes if n in entries else {}) for n in names]
|
||||
assert pending == sorted(pending, reverse=True) and pending[0] is True
|
||||
await client.post(f"{_url(sid)}/{everything['id']}/cancel")
|
||||
needed = (await _start(client, sid)).json()["job"]
|
||||
assert needed["scope"] == "needed" and [t["table_name"] for t in needed["tasks"]] == names, "都没剖析过"
|
||||
await client.post(f"{_url(sid)}/{needed['id']}/cancel")
|
||||
|
||||
picked = (await _start(client, sid, ["v_daily_visits", "nope", "visits", "VISITS", "suppliers"])).json()["job"]
|
||||
tasks = {t["table_name"]: t for t in picked["tasks"]}
|
||||
assert picked["scope"] == "selected" and list(tasks) == ["v_daily_visits", "nope", "visits", "suppliers"]
|
||||
assert tasks["v_daily_visits"]["status"] == "skipped" and "视图" in tasks["v_daily_visits"]["error"]
|
||||
assert tasks["nope"]["status"] == "skipped" and "表结构里没有 nope" in tasks["nope"]["error"]
|
||||
assert tasks["suppliers"]["status"] == "skipped" and tasks["suppliers"]["error"].startswith("没有可剖析的项")
|
||||
assert tasks["visits"]["status"] == "pending" and tasks["visits"]["checks_total"] == 6
|
||||
await client.post(f"{_url(sid)}/{picked['id']}/cancel")
|
||||
|
||||
|
||||
async def test_profile_due_never_unfinished_and_new_relations(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options=_on(table_max_queries=2), draft=["visits", "gates"])
|
||||
await _profile(client, sid, ["visits"])
|
||||
rows = await _rows(client, sid)
|
||||
assert (rows["visits"]["profile_due"], rows["visits"]["profile_due_reason"]) == (True, "unfinished")
|
||||
assert rows["visits"]["last_profile"]["status"] == "partial" and rows["visits"]["last_profile"]["unfinished"] >= 1
|
||||
assert "paths" not in rows["visits"]["last_profile"]
|
||||
|
||||
r = await client.patch(f"/api/datasources/{sid}", json={"options": _on()})
|
||||
assert r.status_code == 200, r.text
|
||||
await _profile(client, sid, ["visits", "gates"])
|
||||
rows = await _rows(client, sid)
|
||||
assert (rows["visits"]["profile_due"], rows["visits"]["profile_due_reason"]) == (False, None)
|
||||
assert rows["gates"]["last_profile"]["status"] == "ok" and not rows["gates"]["profile_due"]
|
||||
needed = await _start(client, sid, ["visits"])
|
||||
assert needed.status_code == 202, "指定了表照样能剖析"
|
||||
await catalog_profile_jobs.manager.wait(needed.json()["job"]["id"], timeout=30)
|
||||
|
||||
# 剖析之后目录里冒出了新的待核实关系：这张表又要剖析了
|
||||
rel = {"id": catalog.relation_id("gates", ["gate_code"], "channels", ["channel_code"]), "columns": ["gate_code"],
|
||||
"to_table": "channels", "to_columns": ["channel_code"], "cardinality": "many_to_one", "coverage": None,
|
||||
"source": "name", "status": "proposed"}
|
||||
async with SessionLocal() as session:
|
||||
entry = await catalog.read_entry(session, sid, "gates")
|
||||
notes = {**entry.notes, "relations": [*entry.notes.get("relations", []), rel]}
|
||||
await catalog.write_entry(session, sid, "gates", notes, if_version=entry.version, actor=None)
|
||||
rows = await _rows(client, sid)
|
||||
assert (rows["gates"]["profile_due"], rows["gates"]["profile_due_reason"]) == (True, "new_relations")
|
||||
job = (await _start(client, sid)).json()["job"]
|
||||
assert "gates" in [t["table_name"] for t in job["tasks"]] and "visits" not in [t["table_name"] for t in job["tasks"]]
|
||||
await catalog_profile_jobs.manager.wait(job["id"], timeout=30)
|
||||
|
||||
|
||||
async def test_nothing_to_profile_is_409(client, make_source, tmp_path):
|
||||
path = _db(tmp_path / "plain.db", "CREATE TABLE suppliers (id INTEGER PRIMARY KEY, name TEXT);",
|
||||
{"suppliers": [(1, "甲供应商")]})
|
||||
sid = await make_source(path, options=_on())
|
||||
r = await _start(client, sid)
|
||||
assert r.status_code == 409 and r.json()["code"] == "profile_nothing"
|
||||
assert "全部表" in r.json()["detail"] and "指定" in r.json()["detail"]
|
||||
r = await _start(client, sid, scope="all")
|
||||
assert r.status_code == 409 and r.json()["code"] == "profile_nothing" and "指定" in r.json()["detail"]
|
||||
r = await _start(client, sid, [f"t{i}" for i in range(501)])
|
||||
assert r.status_code == 422 and "500" in r.json()["detail"]
|
||||
assert (await client.get(f"{_url(sid)}/latest")).json()["job"] is None
|
||||
|
||||
|
||||
async def test_an_empty_table_costs_one_query(client, make_source, tmp_path):
|
||||
path = _db(tmp_path / "empty.db", "CREATE TABLE orders (id INTEGER PRIMARY KEY, status INTEGER, created_at TEXT);")
|
||||
sid = await make_source(path, options=_on())
|
||||
job = await _profile(client, sid, ["orders"])
|
||||
orders = _table(job, "orders")
|
||||
assert orders["status"] == "skipped" and orders["error"] == "空表，没有可剖析的数据"
|
||||
assert orders["queries"] == 1 == job["queries"] and orders["query_log"][0]["kind"] == "row_estimate"
|
||||
assert {c["reason"] for c in orders["checks"]} == {"no_data"} and orders["unfinished"] == 0
|
||||
assert job["status"] == "succeeded", "空表不算没做完"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 留痕、单表接口
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_sampled_keys_never_reach_the_query_log(client, make_source, tmp_path):
|
||||
"""抽样到的键值拼进了 IN 列表，留痕里换成「?」：AgentLab 自己的库里不留这些值。"""
|
||||
customers = [(f"CUST-{i:04d}", "active" if i % 3 else "closed") for i in range(1, 31)]
|
||||
orders = [(i, f"CUST-{1 + i % 30:04d}", "paid") for i in range(1, 61)]
|
||||
path = _db(tmp_path / "shop.db",
|
||||
"CREATE TABLE customers (code TEXT PRIMARY KEY, status TEXT);"
|
||||
"CREATE TABLE orders (id INTEGER PRIMARY KEY, customer_code TEXT, status TEXT);",
|
||||
{"customers": customers, "orders": orders})
|
||||
sid = await make_source(path, options=_on())
|
||||
detail = await _notes(client, sid, "orders")
|
||||
rel = {"columns": ["customer_code"], "to_table": "customers", "to_columns": ["code"]}
|
||||
r = await client.post(f"{_base(sid)}/orders/patch",
|
||||
json={"changes": [{"path": "relations.new", "value": rel, "reason": "订单的客户编码"}],
|
||||
"if_version": detail["version"]})
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
job = await _profile(client, sid, ["orders"])
|
||||
orders_task = _table(job, "orders")
|
||||
matched = [q for q in orders_task["query_log"] if " IN (" in q["sql"]]
|
||||
assert matched and all(re.search(r"IN \((\?(, )?)+\)", q["sql"]) for q in matched), matched
|
||||
finding = next(f for f in orders_task["findings"] if f["kind"] == "relation")
|
||||
assert finding["sample"] == 30 and finding["coverage"] == 1.0
|
||||
async with SessionLocal() as session:
|
||||
stored = (await session.execute(select(CatalogProfileTask.query_log, CatalogProfileTask.findings)
|
||||
.where(CatalogProfileTask.job_id == job["id"]))).all()
|
||||
assert stored and "CUST-" not in json.dumps([row[0] for row in stored], ensure_ascii=False)
|
||||
|
||||
|
||||
async def test_the_table_detail_and_list_carry_the_last_profile(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
gate = _hold_queries(monkeypatch, _at_visits_codes)
|
||||
job = (await _start(client, sid, ["visits"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
detail = await _notes(client, sid, "visits")
|
||||
assert detail["last_profile"]["status"] == "running" and detail["last_profile"]["job_id"] == job["id"]
|
||||
assert (await _rows(client, sid))["visits"]["last_profile"]["status"] == "running"
|
||||
assert not (await _rows(client, sid))["visits"]["profile_due"], "正在剖析的不算要剖析"
|
||||
gate.release.set()
|
||||
await catalog_profile_jobs.manager.wait(job["id"], timeout=30)
|
||||
|
||||
detail = await _notes(client, sid, "visits")
|
||||
last = detail["last_profile"]
|
||||
assert last["status"] == "ok" and last["created_by"] == "周宁" and last["job_status"] == "succeeded"
|
||||
assert last["query_log"] and last["findings"] and last["seq"] == 0
|
||||
row = (await _rows(client, sid))["visits"]["last_profile"]
|
||||
assert set(row) == {"job_id", "seq", "status", "started_at", "finished_at", "summary", "skipped_count",
|
||||
"unfinished", "error", "row_estimate"}
|
||||
assert row["summary"] == last["summary"] and row["row_estimate"] == {"rows": 1500, "method": "count",
|
||||
"at_least": None}
|
||||
assert (await _notes(client, sid, "gates"))["last_profile"] is None
|
||||
|
||||
|
||||
async def test_the_last_profile_is_the_last_one_that_actually_ran(client, make_source, scenic_db, monkeypatch):
|
||||
"""channels 剖析完（ok）；再建 [visits, channels]、停在 visits 上：channels 没开始就停了。表详情、表清单的「上次剖析」
|
||||
仍是上一次真正剖析的那一项（行数、发过的查询都在），不算没做完；保留不删它所在的任务，之后再建一个排队的任务也不删。"""
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits", "channels"])
|
||||
first = await _profile(client, sid, ["channels"])
|
||||
gate = _hold_queries(monkeypatch, _at_visits_codes)
|
||||
second = (await _start(client, sid, ["visits", "channels"])).json()["job"]
|
||||
await asyncio.wait_for(gate.entered.wait(), 10)
|
||||
stopped = (await client.post(f"{_url(sid)}/{second['id']}/cancel")).json()["job"]
|
||||
assert [(t["table_name"], t["status"]) for t in stopped["tasks"]] == [("visits", "partial"),
|
||||
("channels", "cancelled")]
|
||||
|
||||
last = (await _notes(client, sid, "channels"))["last_profile"]
|
||||
assert (last["job_id"], last["status"]) == (first["id"], "ok")
|
||||
assert last["started_at"] and last["queries"] and last["row_estimate"] and last["query_log"]
|
||||
rows = await _rows(client, sid)
|
||||
assert rows["channels"]["last_profile"]["job_id"] == first["id"] and not rows["channels"]["profile_due"]
|
||||
assert rows["visits"]["last_profile"]["job_id"] == second["id"], "visits 真正剖析过：上次剖析是停下的这一次"
|
||||
async with SessionLocal() as session:
|
||||
assert await catalog_profile_jobs.prune(session, sid, keep=1) == 0, "first 是 channels 上次剖析所在的任务"
|
||||
|
||||
# 结束了的任务超出保留数时再建一个排队的任务（建任务时顺手清）：排着的那一项不算 channels 的上次剖析
|
||||
monkeypatch.setattr(catalog_profile_jobs, "KEEP_JOBS", 0)
|
||||
monkeypatch.setattr(catalog_profile_jobs.manager, "start", lambda job_id: None)
|
||||
queued = (await _start(client, sid, ["channels"])).json()["job"]
|
||||
async with SessionLocal() as session:
|
||||
left = set((await session.execute(select(CatalogProfileJob.id)
|
||||
.where(CatalogProfileJob.source_id == sid))).scalars())
|
||||
assert {first["id"], second["id"], queued["id"]} == left
|
||||
assert (await _notes(client, sid, "channels"))["last_profile"]["job_id"] == first["id"]
|
||||
row = (await _rows(client, sid))["channels"]
|
||||
assert row["last_profile"]["job_id"] == first["id"] and not row["profile_due"], "已经排进任务里了"
|
||||
await client.post(f"{_url(sid)}/{queued['id']}/cancel")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 设置迁移、保留
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_old_whole_run_budgets_are_removed_on_startup_and_on_save(client, make_source, scenic_db):
|
||||
sid = await make_source(scenic_db, options={PROFILE_OPTION: {"enabled": True, "max_queries": 60,
|
||||
"max_total_s": 120, "sample_size": 500},
|
||||
"query_timeout_s": "20"})
|
||||
assert await catalog_profile_jobs.startup() >= 0
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, sid)
|
||||
assert row.options == {PROFILE_OPTION: {"enabled": True, "sample_size": 500}, "query_timeout_s": "20"}
|
||||
# 老页面还带着旧键：只摘掉，不报错
|
||||
r = await client.patch(f"/api/datasources/{sid}",
|
||||
json={"options": _on(max_queries=30, max_total_s=90, table_max_queries=25)})
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.json()["options"][PROFILE_OPTION] == {"enabled": True, "table_max_queries": 25}
|
||||
|
||||
|
||||
async def test_retention_keeps_recent_jobs_and_each_tables_last_profile(client, make_source, scenic_db, monkeypatch):
|
||||
sid = await make_source(scenic_db, options=_on())
|
||||
start = utcnow() - timedelta(hours=1)
|
||||
ids: list[str] = []
|
||||
async with SessionLocal() as session:
|
||||
for i in range(5):
|
||||
job = CatalogProfileJob(id=uuid.uuid4().hex, source_id=sid, status="succeeded", scope="selected",
|
||||
settings={"enabled": True}, created_at=start + timedelta(minutes=i))
|
||||
session.add(job)
|
||||
await session.flush()
|
||||
session.add(CatalogProfileTask(job_id=job.id, seq=0, table_name="visits", status="ok"))
|
||||
if i == 0:
|
||||
session.add(CatalogProfileTask(job_id=job.id, seq=1, table_name="gates", status="ok"))
|
||||
ids.append(job.id)
|
||||
await session.commit()
|
||||
async with SessionLocal() as session:
|
||||
assert await catalog_profile_jobs.prune(session, sid, keep=2) == 2
|
||||
left = set((await session.execute(select(CatalogProfileJob.id)
|
||||
.where(CatalogProfileJob.source_id == sid))).scalars())
|
||||
assert left == {ids[4], ids[3], ids[0]}, "最早那个是 gates 最近一次剖析所在的任务，不删"
|
||||
assert (await _notes(client, sid, "gates"))["last_profile"]["job_id"] == ids[0]
|
||||
|
||||
# 建任务时顺手清
|
||||
monkeypatch.setattr(catalog_profile_jobs, "KEEP_JOBS", 1)
|
||||
job = await _profile(client, sid, ["visits"])
|
||||
async with SessionLocal() as session:
|
||||
left = set((await session.execute(select(CatalogProfileJob.id)
|
||||
.where(CatalogProfileJob.source_id == sid))).scalars())
|
||||
assert ids[0] in left and job["id"] in left and ids[3] not in left
|
||||
@@ -20,26 +20,30 @@ AT = "2026-10-03T08:00:00+00:00"
|
||||
def test_settings_default_off_and_parse_leniently():
|
||||
s = profile_settings({})
|
||||
assert s == ProfileSettings()
|
||||
assert s.enabled is False and s.max_queries == 60 and s.query_timeout_s == 10 and s.sample_size == 2000
|
||||
assert s.max_scan_rows == 100_000 and s.max_total_s == 120
|
||||
on = profile_settings({PROFILE_OPTION: {"enabled": True, "max_queries": "30", "sample_size": 500}})
|
||||
assert on.enabled is True and on.max_queries == 30 and on.sample_size == 500
|
||||
# 库里存着的值不对（老数据、手改过）：那一项用缺省，不因此开启，也不报错
|
||||
odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "max_queries": 0, "query_timeout_s": "x"}})
|
||||
assert odd.enabled is False and odd.max_queries == 60 and odd.query_timeout_s == 10
|
||||
assert s.enabled is False and s.table_max_queries == 40 and s.table_max_s == 60 and s.query_timeout_s == 10
|
||||
assert s.sample_size == 2000 and s.max_scan_rows == 100_000
|
||||
on = profile_settings({PROFILE_OPTION: {"enabled": True, "table_max_queries": "30", "sample_size": 500}})
|
||||
assert on.enabled is True and on.table_max_queries == 30 and on.sample_size == 500
|
||||
# 库里存着的值不对（老数据、手改过）：那一项用缺省，不因此开启，也不报错；迁移前的旧键不看
|
||||
odd = profile_settings({PROFILE_OPTION: {"enabled": "yes", "table_max_queries": 0, "query_timeout_s": "x",
|
||||
"max_queries": 5}})
|
||||
assert odd.enabled is False and odd.table_max_queries == 40 and odd.query_timeout_s == 10
|
||||
assert profile_settings({PROFILE_OPTION: "on"}) == ProfileSettings()
|
||||
# 任务里存着的快照读回来是同一份设置
|
||||
assert ProfileSettings.from_dict(on.to_dict()) == on and ProfileSettings.from_dict(None) == ProfileSettings()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value, label", [
|
||||
({"max_queries": 0}, "查询次数上限"),
|
||||
({"max_queries": 501}, "查询次数上限"),
|
||||
({"max_queries": 2.5}, "查询次数上限"),
|
||||
({"table_max_queries": 0}, "每张表最多查询数"),
|
||||
({"table_max_queries": 201}, "每张表最多查询数"),
|
||||
({"table_max_queries": 2.5}, "每张表最多查询数"),
|
||||
({"table_max_s": 9}, "每张表时长上限"),
|
||||
({"table_max_s": 601}, "每张表时长上限"),
|
||||
({"query_timeout_s": 61}, "单条查询时限"),
|
||||
({"sample_size": 5}, "抽样键值数"),
|
||||
({"max_scan_rows": -1}, "整表统计行数上限"),
|
||||
({"max_total_s": 5}, "总时长上限"),
|
||||
({"enabled": "true"}, "开启数据剖析"),
|
||||
({"max_queries": True}, "查询次数上限"),
|
||||
({"table_max_queries": True}, "每张表最多查询数"),
|
||||
])
|
||||
def test_settings_problem_names_the_field_in_chinese(value, label):
|
||||
problem = profile_settings_problem({PROFILE_OPTION: value})
|
||||
@@ -51,9 +55,21 @@ def test_settings_problem_rejects_unknown_keys_and_wrong_shape():
|
||||
assert profile_settings_problem({PROFILE_OPTION: ["enabled"]})
|
||||
assert profile_settings_problem({}) is None
|
||||
assert profile_settings_problem({PROFILE_OPTION: None}) is None
|
||||
assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "max_queries": 500, "query_timeout_s": 1.5,
|
||||
"sample_size": 10, "max_scan_rows": 0,
|
||||
"max_total_s": 600}}) is None
|
||||
assert profile_settings_problem({PROFILE_OPTION: {"enabled": True, "table_max_queries": 200, "table_max_s": 10.5,
|
||||
"query_timeout_s": 1.5, "sample_size": 10,
|
||||
"max_scan_rows": 0}}) is None
|
||||
|
||||
|
||||
def test_legacy_whole_run_budgets_are_stripped_not_rejected():
|
||||
"""以前按整次剖析算的两项（查询次数上限、总时长上限）：保存时只摘掉，不报错；不折算成每张表的预算。"""
|
||||
old = {"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True, "max_queries": 60, "max_total_s": 120,
|
||||
"sample_size": 500}}
|
||||
cleaned = catalog_profile.without_legacy(old)
|
||||
assert cleaned == {"charset": "utf8mb4", PROFILE_OPTION: {"enabled": True, "sample_size": 500}}
|
||||
assert old[PROFILE_OPTION]["max_queries"] == 60, "不改调用方的对象"
|
||||
assert profile_settings_problem(cleaned) is None
|
||||
plain = {PROFILE_OPTION: {"enabled": True}}
|
||||
assert catalog_profile.without_legacy(plain) is plain and catalog_profile.without_legacy(None) is None
|
||||
|
||||
|
||||
def test_profile_option_never_reaches_the_connection_url():
|
||||
|
||||
@@ -24,7 +24,7 @@ from app.db.models import DataSource
|
||||
from tests.fixtures.catalog import scenic, scenic_notes
|
||||
from tests.fixtures.catalog.fake_model import FakeModel
|
||||
from tests.test_catalog_patch import _schema
|
||||
from tests.test_catalog_profile import _notes, _on, _relation, _table, _url, client, make_source # noqa: F401
|
||||
from tests.test_catalog_profile import _notes, _on, _profile, _relation, _table, client, make_source # noqa: F401
|
||||
|
||||
AT = "2026-10-04T08:00:00+00:00"
|
||||
DAY1, DAY2, DAY3 = "2026-10-01T08:00:00+00:00", "2026-10-02T08:00:00+00:00", "2026-10-03T08:00:00+00:00"
|
||||
@@ -131,8 +131,7 @@ async def test_confirming_a_profiled_relation_by_patch_keeps_the_fanout_check(cl
|
||||
"""审查探针 A1：剖析得出多对一、覆盖率 1.0，经对话提案确认这条关系之后，扇出检查照样报 error。"""
|
||||
sql = "SELECT COUNT(g.id) AS n FROM gates g JOIN visits v ON v.gate_id = g.id"
|
||||
sid = await make_source(scenic_db, options=_on(), draft=["visits", "gates"])
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]})
|
||||
assert r.status_code == 200, r.text
|
||||
await _profile(client, sid, ["visits"])
|
||||
detail = await _notes(client, sid, "visits")
|
||||
rel = _relation(detail["notes"], "gate_id")
|
||||
assert (rel["source"], rel["status"], rel["cardinality"], rel["coverage"]) == (
|
||||
@@ -156,8 +155,7 @@ async def test_profiled_relation_without_child_check_only_warns(client, make_sou
|
||||
核实过：扇出检查只给 warning（「可能重复计算」），不给 error、不让出具降档。"""
|
||||
sql = "SELECT COUNT(g.id) AS n FROM gates g JOIN visits v ON v.gate_id = g.id"
|
||||
sid = await make_source(scenic_db, options=_on(max_scan_rows=100), draft=["visits", "gates"])
|
||||
r = await client.post(_url(sid), json={"tables": ["visits"]})
|
||||
assert r.status_code == 200, r.text
|
||||
await _profile(client, sid, ["visits"])
|
||||
rel = _relation((await _notes(client, sid, "visits"))["notes"], "gate_id")
|
||||
assert (rel["source"], rel["status"], rel["cardinality"]) == ("profile", "verified", "many_to_one")
|
||||
assert not rel.get("cardinality_checked")
|
||||
@@ -359,15 +357,13 @@ def test_codes_complete_flag_is_validated_and_counts_as_an_edit():
|
||||
|
||||
async def test_profile_marks_codes_complete_only_when_the_whole_table_was_read(client, make_source, scenic_db):
|
||||
whole = await make_source(scenic_db, options=_on(), draft=["visits"])
|
||||
r = await client.post(_url(whole), json={"tables": ["visits"]})
|
||||
assert r.status_code == 200, r.text
|
||||
await _profile(client, whole, ["visits"])
|
||||
codes = (await _notes(client, whole, "visits"))["notes"]["columns"]["status"]["codes"]
|
||||
assert "统计全表" in codes["note"] and codes.get("complete") is True
|
||||
|
||||
# 不做整表统计（只看前若干行）：取值可能没看全，不标
|
||||
capped = await make_source(scenic_db, options=_on(max_scan_rows=0, sample_size=10), draft=["visits"])
|
||||
r = await client.post(_url(capped), json={"tables": ["visits"]})
|
||||
assert r.status_code == 200, r.text
|
||||
await _profile(client, capped, ["visits"])
|
||||
codes = (await _notes(client, capped, "visits"))["notes"]["columns"]["status"]["codes"]
|
||||
assert "按前 10 行统计" in codes["note"] and "complete" not in codes
|
||||
|
||||
@@ -388,10 +384,10 @@ async def test_rerun_on_another_day_with_same_data_keeps_the_version(client, mak
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
monkeypatch.setattr(catalog, "now_iso", lambda: DAY1)
|
||||
first = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits")
|
||||
first = _table(await _profile(client, sid, ["visits"]), "visits")
|
||||
notes_day1 = (await _notes(client, sid, "visits"))["notes"]
|
||||
monkeypatch.setattr(catalog, "now_iso", lambda: DAY2)
|
||||
second = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits")
|
||||
second = _table(await _profile(client, sid, ["visits"]), "visits")
|
||||
assert (second["added"], second["updated"], second["removed"]) == (0, 0, 0)
|
||||
assert second["version"] == first["version"]
|
||||
assert (await _notes(client, sid, "visits"))["notes"] == notes_day1 # 备注里还是第一天的日期
|
||||
@@ -404,7 +400,7 @@ async def test_rerun_on_another_day_with_same_data_keeps_the_version(client, mak
|
||||
finally:
|
||||
db.close()
|
||||
monkeypatch.setattr(catalog, "now_iso", lambda: DAY3)
|
||||
third = _table((await client.post(_url(sid), json={"tables": ["visits"]})).json(), "visits")
|
||||
third = _table(await _profile(client, sid, ["visits"]), "visits")
|
||||
assert third["version"] == second["version"] + 1 and third["updated"] >= 1
|
||||
codes = (await _notes(client, sid, "visits"))["notes"]["columns"]["status"]["codes"]
|
||||
assert set(codes["value"]) == {"0", "1", "2"} and codes["updated_at"] == DAY3
|
||||
|
||||
Reference in New Issue
Block a user