mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-18 12:21:59 +00:00
820 lines
44 KiB
Python
820 lines
44 KiB
Python
"""数据目录的后台起草任务,以及数据源级的起草要求。
|
||
|
||
以前起草是一次同步长请求(POST /catalog/draft,界面每 12 张表一个请求):一个请求出错整个对话框就停,关掉对话框
|
||
之后没有任何记录哪些表失败、为什么。现在:
|
||
|
||
- **建任务**(create_job):同一个数据源同时只有一个排队或进行中的任务(DraftBusy;部分唯一索引兜底)。不指定表时
|
||
按使用次数取前 20 张;表结构里没有的表照样建进任务、状态 skipped。起草要求(数据源级)和本次说明随任务记下快照。
|
||
- **在后台起草**(manager.start):进程内的 asyncio 任务,引擎是 data/catalog_draft.py(分段、按失败原因处理)。
|
||
宽表每写出一段就并入目录(write_part:这一次合并不删任何东西),整张表结束(不管写全没写全)再按完整覆盖的规则
|
||
收尾一次(catalog.write_draft,规则和同步起草一样:人工确认、驳回过的不动,模型这次没给的项不删以前起草的,只删
|
||
已经不在表结构里的列的模型项,乐观锁冲突重读重并);新增、更新、删除的计数是各次写入的合计。逐表的状态、到第几段、
|
||
写进目录的段数、块进度、计数、模型留空的列,任务的用量(含超时、没写出来的调用次数)、当前的列数上限、提醒随时落库,
|
||
接口读的是库里的状态。
|
||
- **停止**(cancel):排队的直接 cancelled;进行中的停掉正在进行的模型调用,没开始和没写完的表记 cancelled,已经写进
|
||
目录的(含宽表已经写出来的段)不回滚,任务最后是 cancelled。写过段的表 model_error 说清写进了几段、剩几段没起草。
|
||
- **和同步起草互斥**(sync_draft):同步起草(POST /catalog/draft)进行期间,同一个源建任务、重试、导入(ensure_idle)
|
||
都是 DraftBusy;有排队或进行中的任务时同步起草也是。进程内登记(单进程部署)。
|
||
- **重试**(retry_tables + create_job):新任务,只含原任务里 failed、partial、cancelled 的表;用不用模型、本次说明沿用,
|
||
起草要求取数据源当前的。
|
||
- **服务重启**:启动时(startup)把排队、进行中的任务标成 interrupted,没结束的表记 cancelled、写明「服务重启,起草
|
||
中断」。不自动续跑:复制出来的数据目录(检查、验收)启动时不能自己去调模型;界面让人点「重试」续上。关停时
|
||
(manager.shutdown)停掉进行中的任务,同样标成 interrupted。
|
||
|
||
任务的库写入一律经 _JobRun._write:一条一条来,shield 起来——停止任务时不会打断写到一半的库操作,收尾前等它们写完。
|
||
|
||
**起草要求**存在数据源 options 的 catalog_draft_guidance 里(上传表格的源存在它自己的 DataSource 行上),一段文字,
|
||
去首尾空白、最多 GUIDANCE_MAX 字,空串等于没有。只经 PUT /catalog/guidance 改:数据源接口不显示它、整份回写 options
|
||
时留着(api/datasources.py),拼连接串时摘掉(data/engine.py)。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import contextlib
|
||
import logging
|
||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||
from typing import Any
|
||
|
||
from sqlalchemy import delete, func, or_, select, update
|
||
from sqlalchemy.exc import IntegrityError
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from app.data import catalog, catalog_draft
|
||
from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION
|
||
from app.db.base import SessionLocal, new_id, utcnow
|
||
from app.db.models import CatalogDraftJob, CatalogDraftTask, DataSource
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
#: 任务还没结束的状态(同一个源只能有一个)、结束了的状态
|
||
ACTIVE = ("queued", "running")
|
||
FINISHED = ("succeeded", "partial", "failed", "cancelled", "interrupted")
|
||
#: 逐表的状态,以及其中算「已结束」的、重试时要带上的
|
||
TASK_STATUSES = ("pending", "running", "ok", "partial", "failed", "skipped", "cancelled")
|
||
TASK_DONE = ("ok", "partial", "failed", "skipped", "cancelled")
|
||
RETRYABLE = ("failed", "partial", "cancelled")
|
||
_OPEN = ("pending", "running")
|
||
|
||
#: 一个任务最多几张表;起草要求、本次说明最多几个字
|
||
MAX_TABLES = 500
|
||
GUIDANCE_MAX = 2000
|
||
NOTE_MAX = 1000
|
||
#: 目录清单上「这张表最近一次起草」只看最近这么多个任务
|
||
LAST_DRAFT_JOBS = 100
|
||
#: 逐表最多记几个模型留空的列名
|
||
BLANK_SAMPLE = 5
|
||
#: 停止进行中的任务时最多等它收尾多久(秒);关停服务时另给(shutdown 的 grace)
|
||
STOP_GRACE_S = 10.0
|
||
|
||
INTERRUPTED_ERROR = "服务重启,起草中断"
|
||
#: 停止、中断时写过段的表(宽表每写出一段就并入目录):写进了几段、剩几段没起草
|
||
STOPPED_WRITTEN = "已停止:已写进目录 {n} 段,剩下 {m} 段没起草"
|
||
INTERRUPTED_WRITTEN = "已中断:已写进目录 {n} 段,剩下 {m} 段没起草"
|
||
_WRITTEN = {"cancelled": STOPPED_WRITTEN, "interrupted": INTERRUPTED_WRITTEN}
|
||
UNFINISHED_ERROR = "起草结束时这张表还没有写完,请重试这张表"
|
||
WRITE_ERROR = "写入数据目录时出错:{line}"
|
||
JOB_ERROR = "起草任务出错:{line}"
|
||
SOURCE_GONE = "数据源不存在,可能已被删除"
|
||
|
||
|
||
class DraftBusy(Exception):
|
||
"""这个数据源已经有排队或进行中的起草任务。message 给人看。"""
|
||
|
||
|
||
class DraftFinished(Exception):
|
||
"""要停止的任务已经结束了。"""
|
||
|
||
|
||
class NothingToRetry(Exception):
|
||
"""原任务里没有需要重试的表。"""
|
||
|
||
|
||
class TextTooLong(ValueError):
|
||
"""起草要求、本次说明超长。message 给人看。"""
|
||
|
||
|
||
class _JobFailed(Exception):
|
||
"""任务整体出错(数据源没了、快照没了、模型用不了):message 就是任务的 error。"""
|
||
|
||
|
||
# ==========================================================================
|
||
# 起草要求、本次说明
|
||
# ==========================================================================
|
||
|
||
|
||
def clean_text(value: Any, limit: int, label: str) -> str | None:
|
||
"""去首尾空白;空串(和不是字符串的)等于没有;超过 limit 个字抛 TextTooLong。"""
|
||
text = value.strip() if isinstance(value, str) else ""
|
||
if len(text) > limit:
|
||
raise TextTooLong(f"{label}最多 {limit} 字,当前 {len(text)} 字,请删减后再保存")
|
||
return text or None
|
||
|
||
|
||
def guidance_of(row: Any) -> str | None:
|
||
"""数据源级的起草要求(没有是 None)。"""
|
||
raw = (getattr(row, "options", None) or {}).get(CATALOG_DRAFT_GUIDANCE_OPTION)
|
||
return raw.strip() or None if isinstance(raw, str) else None
|
||
|
||
|
||
async def save_guidance(session: AsyncSession, row: DataSource, value: Any) -> str | None:
|
||
"""存起草要求,返回存下的(清除了是 None)。超长抛 TextTooLong,什么都不改。"""
|
||
text = clean_text(value, GUIDANCE_MAX, "起草要求")
|
||
options = {k: v for k, v in (row.options or {}).items() if k != CATALOG_DRAFT_GUIDANCE_OPTION}
|
||
if text:
|
||
options[CATALOG_DRAFT_GUIDANCE_OPTION] = text
|
||
row.options = options
|
||
await session.commit()
|
||
return text
|
||
|
||
|
||
# ==========================================================================
|
||
# 读任务
|
||
# ==========================================================================
|
||
|
||
|
||
def _iso(value: Any) -> str | None:
|
||
return value.isoformat() if value is not None else None
|
||
|
||
|
||
async def active_job(session: AsyncSession, source_id: str) -> CatalogDraftJob | None:
|
||
"""排队或进行中的那个(同一个源最多一个)。"""
|
||
return (await session.execute(
|
||
select(CatalogDraftJob).where(CatalogDraftJob.source_id == source_id, CatalogDraftJob.status.in_(ACTIVE))
|
||
.order_by(CatalogDraftJob.created_at.desc()).limit(1)
|
||
)).scalar_one_or_none()
|
||
|
||
|
||
async def latest_job(session: AsyncSession, source_id: str) -> CatalogDraftJob | None:
|
||
"""最近建的一个(任何状态)。"""
|
||
return (await session.execute(
|
||
select(CatalogDraftJob).where(CatalogDraftJob.source_id == source_id)
|
||
.order_by(CatalogDraftJob.created_at.desc()).limit(1)
|
||
)).scalar_one_or_none()
|
||
|
||
|
||
async def load(session: AsyncSession, source_id: str, job_id: str) -> CatalogDraftJob | None:
|
||
"""这个源的这个任务;不存在、不属于这个源都是 None。"""
|
||
job = await session.get(CatalogDraftJob, job_id)
|
||
return job if job is not None and job.source_id == source_id else None
|
||
|
||
|
||
async def tasks_of(session: AsyncSession, job_id: str) -> list[CatalogDraftTask]:
|
||
return list((await session.execute(
|
||
select(CatalogDraftTask).where(CatalogDraftTask.job_id == job_id).order_by(CatalogDraftTask.seq)
|
||
)).scalars())
|
||
|
||
|
||
def task_dict(task: CatalogDraftTask) -> dict[str, Any]:
|
||
return {
|
||
"table_name": task.table_name,
|
||
"status": task.status,
|
||
"columns": task.column_count,
|
||
"parts_total": task.parts_total,
|
||
"parts_done": task.parts_done,
|
||
# 宽表每写出一段就并入目录:已经写进目录的段数(停止、服务重启时这些段留着)
|
||
"parts_written": task.parts_written or 0,
|
||
# 块进度:这张表实际分成的块数和已结束的块数(段被缩小、拆成两半后会变多);不用模型时为 null
|
||
"chunks_total": task.chunks_total,
|
||
"chunks_done": task.chunks_done,
|
||
"added": task.added,
|
||
"updated": task.updated,
|
||
"removed": task.removed,
|
||
"version": task.version,
|
||
"model_items": task.model_items,
|
||
# 模型拿不准、留空了的列(中文名和含义都没给):几列、最多 5 个列名(按表结构的顺序)。表的状态不因此变
|
||
"blank_columns": task.blank_columns or 0,
|
||
"blank_sample": list(task.blank_sample or []),
|
||
# 起草时记下的几句话,不算失败(模型提议的有效记录条件没有收下的原因:没通过校验、剖析核对排除 0% 或 100%)
|
||
"log": list(task.log or []),
|
||
"error": task.error,
|
||
"model_error": task.model_error,
|
||
"started_at": _iso(task.started_at),
|
||
"finished_at": _iso(task.finished_at),
|
||
}
|
||
|
||
|
||
def _counts(statuses: list[str]) -> dict[str, int]:
|
||
counts = dict.fromkeys(TASK_STATUSES, 0)
|
||
for status in statuses:
|
||
counts[status] = counts.get(status, 0) + 1
|
||
return counts
|
||
|
||
|
||
def job_dict(job: CatalogDraftJob, tasks: list[CatalogDraftTask]) -> dict[str, Any]:
|
||
"""接口返回的任务(CatalogDraftJob):逐表的状态按起草顺序;计数、合计都按逐表的算。"""
|
||
counts = _counts([t.status for t in tasks])
|
||
return {
|
||
"id": job.id,
|
||
"source_id": job.source_id,
|
||
"status": job.status,
|
||
"use_model": bool(job.use_model),
|
||
"model": job.model,
|
||
"guidance": job.guidance,
|
||
"note": job.note,
|
||
"created_at": _iso(job.created_at),
|
||
"started_at": _iso(job.started_at),
|
||
"finished_at": _iso(job.finished_at),
|
||
"created_by": job.created_by,
|
||
"retry_of": job.retry_of,
|
||
"total": len(tasks),
|
||
"done": sum(counts.get(s, 0) for s in TASK_DONE),
|
||
"counts": counts,
|
||
"added": sum(t.added or 0 for t in tasks),
|
||
"updated": sum(t.updated or 0 for t in tasks),
|
||
"removed": sum(t.removed or 0 for t in tasks),
|
||
"blank_columns": sum(t.blank_columns or 0 for t in tasks),
|
||
# 超时的调用拿不到用量,token 按 0 算(网关那边可能已经生成了);failed_calls 含额度用完、只给空壳、超时、出错
|
||
"usage": {"calls": job.calls or 0, "input_tokens": job.input_tokens or 0,
|
||
"output_tokens": job.output_tokens or 0, "timeouts": job.timeouts or 0,
|
||
"failed_calls": job.failed_calls or 0},
|
||
# 新开始的段每段最多几列(不用模型、还没开始时为 null);提醒:最近几次调用多数没写出来时的一句话,没有为 null
|
||
"part_columns": job.part_columns,
|
||
"notice": job.notice,
|
||
"error": job.error,
|
||
"tasks": [task_dict(t) for t in tasks],
|
||
}
|
||
|
||
|
||
async def job_out(session: AsyncSession, job: CatalogDraftJob | None) -> dict[str, Any] | None:
|
||
return None if job is None else job_dict(job, await tasks_of(session, job.id))
|
||
|
||
|
||
async def job_brief(session: AsyncSession, job: CatalogDraftJob | None) -> dict[str, Any] | None:
|
||
"""目录清单顶上的进度:{id, status, total, done}。"""
|
||
if job is None:
|
||
return None
|
||
rows = (await session.execute(
|
||
select(CatalogDraftTask.status, func.count()).where(CatalogDraftTask.job_id == job.id)
|
||
.group_by(CatalogDraftTask.status)
|
||
)).all()
|
||
counts = {status: n for status, n in rows}
|
||
return {"id": job.id, "status": job.status, "total": sum(counts.values()),
|
||
"done": sum(counts.get(s, 0) for s in TASK_DONE)}
|
||
|
||
|
||
async def last_drafts(session: AsyncSession, source_id: str) -> dict[str, dict[str, Any]]:
|
||
"""每张表最近一次起草任务里的那一项:{表名: {status, job_id, finished_at, error, model_error, blank_columns,
|
||
parts_written, blank_columns_from}}。任务还在进行,就是进行中的状态。只看最近 LAST_DRAFT_JOBS 个任务;每张表取哪一项
|
||
在库里挑好(清单页起草时会反复取)。
|
||
|
||
blank_columns_from:最近一次结束于 ok / partial 的用模型起草记下的留空列数(数字),没有是 null。之后的起草停止了、
|
||
中断了、而且没有写进新内容(parts_written 为 0)时照样带着,界面据此继续提示留空的列;之后有起草写进了新内容(停止
|
||
前已经写了几段、或者正在写)就是 null:以前的留空列数可能已经不对了。不用模型的起草不改模型留空的列,不算。"""
|
||
recent = (select(CatalogDraftJob.id).where(CatalogDraftJob.source_id == source_id)
|
||
.order_by(CatalogDraftJob.created_at.desc()).limit(LAST_DRAFT_JOBS)).scalar_subquery()
|
||
|
||
def ranked() -> Any:
|
||
return func.row_number().over(partition_by=CatalogDraftTask.table_name,
|
||
order_by=(CatalogDraftJob.created_at.desc(), CatalogDraftTask.seq)).label("rank")
|
||
|
||
inner = (select(CatalogDraftTask.table_name, CatalogDraftTask.status, CatalogDraftTask.job_id,
|
||
CatalogDraftTask.finished_at, CatalogDraftTask.error, CatalogDraftTask.model_error,
|
||
CatalogDraftTask.blank_columns, CatalogDraftTask.parts_written, ranked())
|
||
.join(CatalogDraftJob, CatalogDraftJob.id == CatalogDraftTask.job_id)
|
||
.where(CatalogDraftJob.id.in_(recent))).subquery()
|
||
rows = (await session.execute(select(inner).where(inner.c.rank == 1))).mappings()
|
||
# 每张表最近一次「写进了模型起草的内容」的用模型起草:结束于 ok / partial 的,或者写过段的(停止、中断、进行中)
|
||
wrote = (select(CatalogDraftTask.table_name, CatalogDraftTask.status, CatalogDraftTask.blank_columns, ranked())
|
||
.join(CatalogDraftJob, CatalogDraftJob.id == CatalogDraftTask.job_id)
|
||
.where(CatalogDraftJob.id.in_(recent), CatalogDraftTask.parts_total.is_not(None),
|
||
or_(CatalogDraftTask.status.in_(("ok", "partial")),
|
||
func.coalesce(CatalogDraftTask.parts_written, 0) > 0))).subquery()
|
||
blank_from = {row["table_name"]: (row["blank_columns"] or 0) if row["status"] in ("ok", "partial") else None
|
||
for row in (await session.execute(select(wrote).where(wrote.c.rank == 1))).mappings()}
|
||
return {row["table_name"]: {"status": row["status"], "job_id": row["job_id"],
|
||
"finished_at": _iso(row["finished_at"]), "error": row["error"],
|
||
"model_error": row["model_error"], "blank_columns": row["blank_columns"] or 0,
|
||
"parts_written": row["parts_written"] or 0,
|
||
"blank_columns_from": blank_from.get(row["table_name"])}
|
||
for row in rows}
|
||
|
||
|
||
# ==========================================================================
|
||
# 建任务、重试、停止
|
||
# ==========================================================================
|
||
|
||
|
||
def busy_message(row: Any) -> str:
|
||
return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录,请等这次起草结束(或先停止它)再开始新的起草"
|
||
|
||
|
||
def sync_busy_message(row: Any) -> str:
|
||
return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录(同步起草),请等这次起草结束再开始新的起草"
|
||
|
||
|
||
#: 正在同步起草(POST /catalog/draft)的数据源。进程内登记:服务端是单进程部署,多进程部署时要换成库里的锁
|
||
_SYNC: set[str] = set()
|
||
|
||
|
||
def sync_drafting(source_id: str) -> bool:
|
||
"""这个源正在同步起草。"""
|
||
return source_id in _SYNC
|
||
|
||
|
||
async def ensure_idle(session: AsyncSession, row: Any) -> None:
|
||
"""这个源没有在起草:没有排队或进行中的任务,也没有在同步起草。有就抛 DraftBusy。建任务、重试、导入目录之前都调它。"""
|
||
if row.id in _SYNC:
|
||
raise DraftBusy(sync_busy_message(row))
|
||
if await active_job(session, row.id) is not None:
|
||
raise DraftBusy(busy_message(row))
|
||
|
||
|
||
@contextlib.asynccontextmanager
|
||
async def sync_draft(session: AsyncSession, row: Any) -> AsyncIterator[None]:
|
||
"""同步起草期间登记这个源:同一个源建任务、重试、导入、再开一次同步起草都抛 DraftBusy;有排队或进行中的任务时
|
||
这里就抛 DraftBusy。先登记、再查库:查库要让出事件循环,先登记才不会和同时建的任务互相看不见(建任务那边写库之后
|
||
也再看一次,见 create_job)。"""
|
||
if row.id in _SYNC:
|
||
raise DraftBusy(sync_busy_message(row))
|
||
_SYNC.add(row.id)
|
||
try:
|
||
if await active_job(session, row.id) is not None:
|
||
raise DraftBusy(busy_message(row))
|
||
yield
|
||
finally:
|
||
_SYNC.discard(row.id)
|
||
|
||
|
||
async def check_model(session: AsyncSession) -> None:
|
||
"""起草模型能不能用:不能用抛 catalog.CatalogModelUnavailable(只建模型对象,不发请求)。"""
|
||
await catalog.resolve_draft_model(session)
|
||
|
||
|
||
async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: list[str] | None, use_model: bool,
|
||
note: str | None, actor: str | None,
|
||
retry_of: str | None = None) -> tuple[CatalogDraftJob, list[CatalogDraftTask]]:
|
||
"""建一个排队的任务(还没开始,调用方接着 manager.start)。row 是数据源记录(起草要求取它的),source 是读表结构
|
||
用的源(上传源是绑定当前快照的视图)。有排队或进行中的任务抛 DraftBusy。"""
|
||
await ensure_idle(session, row)
|
||
cache = getattr(source, "schema_cache", None) or {}
|
||
all_tables = cache.get("tables") or {}
|
||
wanted = list(tables) if tables else await catalog.default_tables(session, source)
|
||
job = CatalogDraftJob(id=new_id(), source_id=row.id, status="queued", use_model=bool(use_model),
|
||
guidance=guidance_of(row), note=note, created_by=actor, retry_of=retry_of)
|
||
tasks: list[CatalogDraftTask] = []
|
||
seen: set[str] = set()
|
||
now = utcnow()
|
||
for name in dict.fromkeys(str(t) for t in wanted):
|
||
key = catalog.resolve_table_name(cache, name)
|
||
if key is None:
|
||
tasks.append(CatalogDraftTask(job_id=job.id, seq=len(tasks), table_name=name, status="skipped",
|
||
error=catalog.missing_table_error(name), finished_at=now))
|
||
continue
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
columns = len((all_tables.get(key) or {}).get("columns") or [])
|
||
parts = catalog_draft.parts_for(columns) if use_model else None
|
||
tasks.append(CatalogDraftTask(job_id=job.id, seq=len(tasks), table_name=key, status="pending",
|
||
column_count=columns, parts_total=parts, parts_written=0,
|
||
chunks_total=parts, chunks_done=0 if use_model else None))
|
||
session.add(job)
|
||
try:
|
||
await session.flush()
|
||
session.add_all(tasks)
|
||
await session.commit()
|
||
except IntegrityError as e:
|
||
# 部分唯一索引:另一个请求同时给这个源建了任务
|
||
await session.rollback()
|
||
raise DraftBusy(busy_message(row)) from e
|
||
if row.id in _SYNC:
|
||
# 写库的时候同一个源开始了同步起草(它查库时还没看到这个任务):这个任务不要了
|
||
await session.execute(delete(CatalogDraftTask).where(CatalogDraftTask.job_id == job.id))
|
||
await session.execute(delete(CatalogDraftJob).where(CatalogDraftJob.id == job.id))
|
||
await session.commit()
|
||
raise DraftBusy(sync_busy_message(row))
|
||
return job, tasks
|
||
|
||
|
||
async def retry_tables(session: AsyncSession, job: CatalogDraftJob) -> list[str]:
|
||
"""重试要带上的表:原任务里 failed、partial、cancelled 的,按原来的顺序。没有抛 NothingToRetry。"""
|
||
names = [t.table_name for t in await tasks_of(session, job.id) if t.status in RETRYABLE]
|
||
if not names:
|
||
raise NothingToRetry("这次起草里没有失败、只写出一部分或停止了的表,不需要重试")
|
||
return names
|
||
|
||
|
||
async def cancel(session: AsyncSession, job: CatalogDraftJob) -> None:
|
||
"""停止一个任务。排队的直接 cancelled;进行中的交给 manager 停(停掉模型调用、没写完的表记 cancelled)。
|
||
已经结束的抛 DraftFinished。"""
|
||
if job.status in FINISHED:
|
||
raise DraftFinished("这次起草已经结束了,不需要停止")
|
||
if job.status == "queued":
|
||
now = utcnow()
|
||
claimed = await session.execute(
|
||
update(CatalogDraftJob).where(CatalogDraftJob.id == job.id, CatalogDraftJob.status == "queued")
|
||
.values(status="cancelled", finished_at=now).execution_options(synchronize_session=False))
|
||
if claimed.rowcount == 1:
|
||
await session.execute(
|
||
update(CatalogDraftTask).where(CatalogDraftTask.job_id == job.id, CatalogDraftTask.status.in_(_OPEN))
|
||
.values(status="cancelled", finished_at=now).execution_options(synchronize_session=False))
|
||
await session.commit()
|
||
return
|
||
await session.rollback() # 刚被后台认领、开始了:按进行中的停
|
||
if await manager.stop(job.id, "user", grace=STOP_GRACE_S):
|
||
return
|
||
# 库里说在进行,这个进程里却没有在跑它(记录对不上):直接记停止
|
||
await _close(session, job.id, "cancelled", None, "cancelled", None)
|
||
|
||
|
||
async def _close(session: AsyncSession, job_id: str, job_status: str, job_error: str | None, task_status: str,
|
||
task_error: str | None) -> None:
|
||
"""收尾:还没结束的表记 task_status,任务(还没结束的话)记 job_status。停止、中断时写过段的表 model_error 说清写进了
|
||
几段、剩几段没起草(已经写进去的段留在目录里)。"""
|
||
now = utcnow()
|
||
written = _WRITTEN.get(job_status)
|
||
if written is not None:
|
||
rows = (await session.execute(
|
||
select(CatalogDraftTask.id, CatalogDraftTask.parts_written, CatalogDraftTask.parts_total)
|
||
.where(CatalogDraftTask.job_id == job_id, CatalogDraftTask.status.in_(_OPEN),
|
||
func.coalesce(CatalogDraftTask.parts_written, 0) > 0))).all()
|
||
for task_id, n, total in rows:
|
||
await session.execute(
|
||
update(CatalogDraftTask).where(CatalogDraftTask.id == task_id, CatalogDraftTask.status.in_(_OPEN))
|
||
.values(status=task_status, error=task_error, finished_at=now,
|
||
model_error=written.format(n=n, m=max(0, int(total or n) - int(n))))
|
||
.execution_options(synchronize_session=False))
|
||
await session.execute(
|
||
update(CatalogDraftTask).where(CatalogDraftTask.job_id == job_id, CatalogDraftTask.status.in_(_OPEN))
|
||
.values(status=task_status, error=task_error, finished_at=now).execution_options(synchronize_session=False))
|
||
await session.execute(
|
||
update(CatalogDraftJob).where(CatalogDraftJob.id == job_id, CatalogDraftJob.status.in_(ACTIVE))
|
||
.values(status=job_status, error=job_error, finished_at=now).execution_options(synchronize_session=False))
|
||
await session.commit()
|
||
|
||
|
||
async def startup() -> int:
|
||
"""服务启动时:排队、进行中的任务标成 interrupted,没结束的表记 cancelled、写明服务重启。不自动续跑。返回标了几个。"""
|
||
async with SessionLocal() as session:
|
||
ids = [job_id for job_id in (await session.execute(
|
||
select(CatalogDraftJob.id).where(CatalogDraftJob.status.in_(ACTIVE)))).scalars()
|
||
if not manager.running(job_id)]
|
||
for job_id in ids:
|
||
await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR)
|
||
if ids:
|
||
logger.warning("服务重启:%d 个数据目录起草任务没有做完,已标为中断,可在数据目录页重试", len(ids))
|
||
return len(ids)
|
||
|
||
|
||
# ==========================================================================
|
||
# 在后台起草
|
||
# ==========================================================================
|
||
|
||
|
||
class _JobRun:
|
||
"""一个任务在后台的一次执行。"""
|
||
|
||
def __init__(self, job_id: str) -> None:
|
||
self.job_id = job_id
|
||
self.task: asyncio.Task[None] | None = None
|
||
#: 为什么停:user 使用者点了停止,shutdown 服务关停(标中断)
|
||
self.stop_reason: str | None = None
|
||
self._writes: set[asyncio.Task[Any]] = set()
|
||
self._lock = asyncio.Lock()
|
||
self._started: set[str] = set()
|
||
self.source: Any = None
|
||
self.actor: str | None = None
|
||
|
||
# ---------------------------------------------------------------- 库写入
|
||
|
||
async def _write(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any:
|
||
"""一条库写入:同一个任务的写入一条一条来;shield 起来,停止任务时不会打断写到一半的库操作。"""
|
||
async def go() -> Any:
|
||
async with self._lock:
|
||
async with SessionLocal() as session:
|
||
return await fn(session, *args)
|
||
|
||
task = asyncio.ensure_future(go())
|
||
self._writes.add(task)
|
||
task.add_done_callback(self._writes.discard)
|
||
return await asyncio.shield(task)
|
||
|
||
async def _settle(self) -> None:
|
||
"""等手上的库写入都写完(停止、出错收尾之前)。"""
|
||
while self._writes:
|
||
await asyncio.gather(*list(self._writes), return_exceptions=True)
|
||
|
||
# ---------------------------------------------------------------- 主流程
|
||
|
||
async def run(self) -> None:
|
||
try:
|
||
await self._run()
|
||
except asyncio.CancelledError:
|
||
current = asyncio.current_task()
|
||
if current is not None:
|
||
current.uncancel()
|
||
if self.stop_reason == "user":
|
||
await self._end("cancelled", None, "cancelled", None)
|
||
else:
|
||
await self._end("interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR)
|
||
except _JobFailed as e:
|
||
await self._end("failed", str(e), "failed", str(e))
|
||
except Exception as e: # noqa: BLE001 - 任务整体出错:照实记下,不留在「起草中」
|
||
from app.core.errors import describe_exception, first_line
|
||
|
||
logger.exception("数据目录起草任务 %s 出错", self.job_id)
|
||
reason = JOB_ERROR.format(line=first_line(e) or describe_exception(e))
|
||
await self._end("failed", reason, "failed", reason)
|
||
|
||
async def _end(self, job_status: str, job_error: str | None, task_status: str, task_error: str | None) -> None:
|
||
"""停止、出错时收尾:等手上的库写入写完,再把没结束的表和任务记下来。收尾本身写不进去只记日志(下次启动时
|
||
startup 会把它标成中断)。"""
|
||
try:
|
||
await self._settle()
|
||
await self._write(_close, self.job_id, job_status, job_error, task_status, task_error)
|
||
except asyncio.CancelledError:
|
||
raise
|
||
except Exception: # noqa: BLE001
|
||
logger.exception("数据目录起草任务 %s 收尾时写库出错", self.job_id)
|
||
|
||
async def _run(self) -> None:
|
||
if not await self._write(self._claim):
|
||
return # 开始之前就被停止了
|
||
async with SessionLocal() as session:
|
||
job = await session.get(CatalogDraftJob, self.job_id)
|
||
row = await session.get(DataSource, job.source_id) if job is not None else None
|
||
if job is None or row is None:
|
||
raise _JobFailed(SOURCE_GONE)
|
||
from app.data import table_versions
|
||
|
||
try:
|
||
source = await table_versions.resolve_source(session, row)
|
||
except table_versions.SnapshotMissing as e:
|
||
raise _JobFailed(str(e)) from e
|
||
pending = [t.table_name for t in await tasks_of(session, job.id) if t.status == "pending"]
|
||
use_model, background = bool(job.use_model), catalog_draft.Background(guidance=job.guidance, note=job.note)
|
||
self.actor = job.created_by
|
||
model: Any = None
|
||
existing: dict[str, Any] = {}
|
||
if use_model and pending:
|
||
try:
|
||
model, model_id = await catalog.resolve_draft_model(session)
|
||
except catalog.CatalogModelUnavailable as e:
|
||
raise _JobFailed(str(e)) from e
|
||
existing = {t: e.notes for t, e in (await catalog.read_catalog(session, source.id)).items()}
|
||
self.source = source
|
||
all_tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {}
|
||
names = [n for n in pending if isinstance(all_tables.get(n), dict)]
|
||
gone = [n for n in pending if n not in names]
|
||
if gone:
|
||
# 建任务之后表结构变了(重新探查过):这几张表跳过
|
||
await self._write(self._skip, gone)
|
||
if use_model and names:
|
||
await self._write(self._set_model, model_id)
|
||
hooks = catalog_draft.Hooks(call_started=self._call_started, call_done=self._call_done,
|
||
part_done=self._part_done, table_done=self._table_done,
|
||
part_written=self._part_written, chunks_changed=self._chunks_changed)
|
||
await catalog_draft.run_model_draft(model, source, names, existing=existing, background=background,
|
||
hooks=hooks)
|
||
else:
|
||
for name in names:
|
||
await self._call_started([name])
|
||
await self._table_done(name, None)
|
||
await self._settle()
|
||
await self._write(self._finish)
|
||
|
||
# ---------------------------------------------------------------- 进度回调(引擎调)
|
||
|
||
async def _call_started(self, tables: list[str]) -> None:
|
||
fresh = [t for t in dict.fromkeys(tables) if t not in self._started]
|
||
if fresh:
|
||
self._started.update(fresh)
|
||
await self._write(self._mark_running, fresh)
|
||
|
||
async def _call_done(self, report: catalog_draft.CallReport) -> None:
|
||
await self._write(self._add_call, report)
|
||
|
||
async def _part_done(self, table: str) -> None:
|
||
await self._write(self._bump_part, table)
|
||
|
||
async def _table_done(self, table: str, outcome: catalog_draft.ModelOutcome | None) -> None:
|
||
await self._write(self._write_table, table, outcome)
|
||
|
||
async def _part_written(self, table: str, written: catalog_draft.PartWrite) -> None:
|
||
await self._write(self._write_part, table, written)
|
||
|
||
async def _chunks_changed(self, table: str, total: int, done: int) -> None:
|
||
await self._write(self._set_chunks, table, total, done)
|
||
|
||
# ---------------------------------------------------------------- 库操作
|
||
|
||
def _task(self, *where: Any) -> Any:
|
||
return update(CatalogDraftTask).where(CatalogDraftTask.job_id == self.job_id, *where) \
|
||
.execution_options(synchronize_session=False)
|
||
|
||
async def _claim(self, session: AsyncSession) -> bool:
|
||
result = await session.execute(
|
||
update(CatalogDraftJob).where(CatalogDraftJob.id == self.job_id, CatalogDraftJob.status == "queued")
|
||
.values(status="running", started_at=utcnow()).execution_options(synchronize_session=False))
|
||
await session.commit()
|
||
return result.rowcount == 1
|
||
|
||
async def _set_model(self, session: AsyncSession, model_id: str) -> None:
|
||
await session.execute(update(CatalogDraftJob).where(CatalogDraftJob.id == self.job_id)
|
||
.values(model=model_id or None, part_columns=max(1, int(catalog.MODEL_BATCH_COLUMNS)))
|
||
.execution_options(synchronize_session=False))
|
||
await session.commit()
|
||
|
||
async def _skip(self, session: AsyncSession, names: list[str]) -> None:
|
||
now = utcnow()
|
||
for name in names:
|
||
await session.execute(self._task(CatalogDraftTask.table_name == name, CatalogDraftTask.status == "pending")
|
||
.values(status="skipped", error=catalog.missing_table_error(name), finished_at=now))
|
||
await session.commit()
|
||
|
||
async def _mark_running(self, session: AsyncSession, names: list[str]) -> None:
|
||
await session.execute(self._task(CatalogDraftTask.table_name.in_(names), CatalogDraftTask.status == "pending")
|
||
.values(status="running", started_at=utcnow()))
|
||
await session.commit()
|
||
|
||
async def _add_call(self, session: AsyncSession, report: catalog_draft.CallReport) -> None:
|
||
"""一次调用:用量、超时、没写出来的次数累加,当前的列数上限和提醒照这次调用之后的写(按调用的先后写进来)。"""
|
||
await session.execute(
|
||
update(CatalogDraftJob).where(CatalogDraftJob.id == self.job_id)
|
||
.values(calls=CatalogDraftJob.calls + 1,
|
||
input_tokens=CatalogDraftJob.input_tokens + int(report.input_tokens),
|
||
output_tokens=CatalogDraftJob.output_tokens + int(report.output_tokens),
|
||
timeouts=func.coalesce(CatalogDraftJob.timeouts, 0) + int(report.timeout),
|
||
failed_calls=func.coalesce(CatalogDraftJob.failed_calls, 0) + int(report.failure is not None),
|
||
part_columns=report.part_columns, notice=report.notice)
|
||
.execution_options(synchronize_session=False))
|
||
await session.commit()
|
||
|
||
async def _bump_part(self, session: AsyncSession, table: str) -> None:
|
||
await session.execute(self._task(CatalogDraftTask.table_name == table)
|
||
.values(parts_done=CatalogDraftTask.parts_done + 1))
|
||
await session.commit()
|
||
|
||
async def _set_chunks(self, session: AsyncSession, table: str, total: int, done: int) -> None:
|
||
await session.execute(self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN))
|
||
.values(chunks_total=int(total), chunks_done=int(done)))
|
||
await session.commit()
|
||
|
||
async def _write_part(self, session: AsyncSession, table: str, written: catalog_draft.PartWrite) -> None:
|
||
"""宽表写出了一段:马上并入目录(这一次不删任何东西),记下写进去的段数、累加计数。写不进去只记日志:整张表结束
|
||
时收尾那次还会把它带上。
|
||
|
||
新增、更新的项的修改时间是这一次写入的时间(每次写入各取一次):以前整个任务只在开始时取一次,起草了几十分钟的宽表,
|
||
每一段写进去的项都记成任务开始那一刻,比任务的开始时间还早。"""
|
||
try:
|
||
found = await write_part(session, self.source, table, written.notes, at=catalog.now_iso(), actor=self.actor)
|
||
except Exception: # noqa: BLE001 - 这一段写不进去不影响起草
|
||
logger.exception("起草任务 %s 把表 %s 写出来的一段并入数据目录时出错", self.job_id, table)
|
||
await session.rollback()
|
||
return
|
||
if found is None:
|
||
logger.warning("起草任务 %s:表 %s 的数据目录正被修改,这一段等整张表写完时再并入", self.job_id, table)
|
||
return
|
||
stats, version = found
|
||
await session.execute(
|
||
self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN))
|
||
.values(parts_written=int(written.parts_written), version=version,
|
||
added=CatalogDraftTask.added + stats.added, updated=CatalogDraftTask.updated + stats.updated,
|
||
removed=CatalogDraftTask.removed + stats.removed))
|
||
await session.commit()
|
||
|
||
async def _write_table(self, session: AsyncSession, table: str,
|
||
outcome: catalog_draft.ModelOutcome | None) -> None:
|
||
"""一张表结束:并入目录、写入(宽表写过的段已经并入过,这里是整张表的收尾),记下这张表的结果。计数加在逐段写入
|
||
的计数上。修改时间取这一次写入的时间(同 _write_part):逐段写进去、内容没变的项留着那一段的时间。"""
|
||
try:
|
||
result = await catalog.write_draft(session, self.source, table, outcome, at=catalog.now_iso(),
|
||
actor=self.actor)
|
||
except Exception as e: # noqa: BLE001 - 这一张写不进去不影响别的表
|
||
from app.core.errors import describe_exception, first_line
|
||
|
||
logger.exception("起草任务 %s 写入表 %s 的数据目录时出错", self.job_id, table)
|
||
await session.rollback()
|
||
result = catalog.TableDraftResult(table=table, status="failed",
|
||
error=WRITE_ERROR.format(line=first_line(e) or describe_exception(e)),
|
||
model_error=outcome.error if outcome is not None else None)
|
||
values: dict[str, Any] = {
|
||
"status": result.status, "added": CatalogDraftTask.added + result.added,
|
||
"updated": CatalogDraftTask.updated + result.updated, "removed": CatalogDraftTask.removed + result.removed,
|
||
"model_items": result.model_items, "error": result.error,
|
||
"model_error": result.model_error, "finished_at": utcnow(),
|
||
"blank_columns": len(result.blank), "blank_sample": result.blank[:BLANK_SAMPLE] or None,
|
||
}
|
||
if result.version:
|
||
values["version"] = result.version
|
||
if outcome is not None:
|
||
values.update(parts_done=outcome.parts_done, chunks_total=outcome.chunks_total,
|
||
chunks_done=outcome.chunks_done, log=list(outcome.log) or None)
|
||
if not result.error:
|
||
# 收尾写进去了:写出内容的段都在目录里了。没写进去(一直撞上别人刚改过)的照库里逐段写进去的段数
|
||
values["parts_written"] = outcome.parts_written
|
||
await session.execute(self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN))
|
||
.values(**values))
|
||
await session.commit()
|
||
|
||
async def _finish(self, session: AsyncSession) -> None:
|
||
"""起草完:万一还有没收尾的表(进度回调写库失败之类)记失败;有 failed、partial、cancelled 的表是 partial,
|
||
否则 succeeded。"""
|
||
now = utcnow()
|
||
await session.execute(self._task(CatalogDraftTask.status.in_(_OPEN))
|
||
.values(status="failed", error=UNFINISHED_ERROR, finished_at=now))
|
||
statuses = list((await session.execute(
|
||
select(CatalogDraftTask.status).where(CatalogDraftTask.job_id == self.job_id))).scalars())
|
||
status = "partial" if any(s in RETRYABLE for s in statuses) else "succeeded"
|
||
await session.execute(
|
||
update(CatalogDraftJob).where(CatalogDraftJob.id == self.job_id, CatalogDraftJob.status == "running")
|
||
.values(status=status, finished_at=now).execution_options(synchronize_session=False))
|
||
await session.commit()
|
||
|
||
|
||
async def write_part(session: AsyncSession, source: Any, table: str, notes: dict[str, Any], *, at: str,
|
||
actor: str | None) -> tuple[catalog.MergeStats, int] | None:
|
||
"""宽表写出来的一段并入目录:(计数, 写入后的版本);撞上别人刚改过、重读重并三次仍冲突返回 None。
|
||
|
||
数据库注释、外键、命名推断和这张表到这时拿到的模型起草,按 merge_notes 的规则并入(人工确认、驳回过的不动),
|
||
**这一次不删任何东西**(covered 为空):删掉表结构里已经没有的、这一轮没再得出的,是整张表写完时 catalog.write_draft
|
||
收尾那一次的事。内容没变不写、不升版本。"""
|
||
structure = catalog.draft_structure(source, table)
|
||
for _attempt in range(3):
|
||
entry = await catalog.read_entry(session, source.id, table)
|
||
version = entry.version if entry else 0
|
||
merged, stats = catalog.merge_notes(entry.notes if entry else {}, structure.notes, at=at)
|
||
merged, more = catalog.merge_notes(merged, notes, at=at)
|
||
stats += more
|
||
if entry is None and not merged:
|
||
return stats, 0
|
||
try:
|
||
written = await catalog.write_entry(session, source.id, table, merged, if_version=version, actor=actor)
|
||
except catalog.CatalogConflict:
|
||
continue
|
||
return stats, written.version
|
||
return None
|
||
|
||
|
||
class DraftJobs:
|
||
"""进程内的起草任务(照 app/scheduler 的写法:一个 asyncio 任务一件事,停的时候给宽限)。"""
|
||
|
||
def __init__(self) -> None:
|
||
self._runs: dict[str, _JobRun] = {}
|
||
|
||
def _alive(self, job_id: str) -> _JobRun | None:
|
||
run = self._runs.get(job_id)
|
||
if run is None or run.task is None or run.task.done():
|
||
return None
|
||
try:
|
||
if run.task.get_loop() is not asyncio.get_running_loop():
|
||
self._runs.pop(job_id, None) # 别的事件循环留下的(测试每个用例一个循环)
|
||
return None
|
||
except RuntimeError:
|
||
return None
|
||
return run
|
||
|
||
def start(self, job_id: str) -> None:
|
||
"""开始在后台起草这个(排队中的)任务。"""
|
||
run = _JobRun(job_id)
|
||
run.task = asyncio.get_running_loop().create_task(run.run(), name=f"catalog-draft-{job_id}")
|
||
self._runs[job_id] = run
|
||
|
||
def forget(_: asyncio.Task[None]) -> None:
|
||
if self._runs.get(job_id) is run:
|
||
self._runs.pop(job_id, None)
|
||
|
||
run.task.add_done_callback(forget)
|
||
|
||
def running(self, job_id: str) -> bool:
|
||
return self._alive(job_id) is not None
|
||
|
||
async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S) -> bool:
|
||
"""停掉这个进程里在跑的任务,等它收尾(最多 grace 秒)。不在这个进程里跑返回 False。"""
|
||
run = self._alive(job_id)
|
||
if run is None:
|
||
return False
|
||
if run.stop_reason is None:
|
||
run.stop_reason = reason
|
||
run.task.cancel() # type: ignore[union-attr]
|
||
await asyncio.wait([run.task], timeout=grace) # type: ignore[list-item]
|
||
return True
|
||
|
||
async def wait(self, job_id: str, timeout: float | None = None) -> None:
|
||
"""等这个任务做完(测试、调试用)。不在跑直接返回。"""
|
||
run = self._alive(job_id)
|
||
if run is not None:
|
||
await asyncio.wait([run.task], timeout=timeout) # type: ignore[list-item]
|
||
|
||
async def shutdown(self, *, grace: float = 3.0) -> None:
|
||
"""关停:停掉进行中的任务(标中断),最多等 grace 秒收尾。"""
|
||
runs = [run for job_id in list(self._runs) if (run := self._alive(job_id)) is not None]
|
||
for run in runs:
|
||
if run.stop_reason is None:
|
||
run.stop_reason = "shutdown"
|
||
run.task.cancel() # type: ignore[union-attr]
|
||
if runs:
|
||
await asyncio.wait([run.task for run in runs], timeout=grace) # type: ignore[misc]
|
||
logger.info("已停止 %d 个数据目录起草任务", len(runs))
|
||
|
||
|
||
manager = DraftJobs()
|
||
|
||
|
||
__all__ = ["ACTIVE", "BLANK_SAMPLE", "DraftBusy", "DraftFinished", "DraftJobs", "FINISHED", "GUIDANCE_MAX",
|
||
"INTERRUPTED_ERROR", "INTERRUPTED_WRITTEN", "MAX_TABLES", "NOTE_MAX", "NothingToRetry", "RETRYABLE",
|
||
"STOPPED_WRITTEN", "TASK_DONE", "TASK_STATUSES", "TextTooLong",
|
||
"active_job", "cancel", "check_model", "clean_text", "create_job", "ensure_idle", "guidance_of", "job_brief",
|
||
"job_dict", "job_out", "last_drafts", "latest_job", "load", "manager", "retry_tables", "save_guidance",
|
||
"startup", "sync_busy_message", "sync_draft", "sync_drafting", "tasks_of", "write_part"]
|