diff --git a/README.md b/README.md index eff62429..431e6e76 100644 --- a/README.md +++ b/README.md @@ -276,7 +276,8 @@ flowchart LR | 环节 | 做法 | |---|---| -| 起草 | 探查时读外键和唯一约束；按 `xxx_id`、`xxxId` 推断关联，表名带业务前缀的也认；可选让模型起草中文名、粒度、度量类型。导入表格的源已有系统按核对结果生成的说明，目录不重复、不改写它 | +| 起草 | 探查时读外键和唯一约束；按 `xxx_id`、`xxxId` 推断关联，表名带业务前缀的也认；可选让模型起草中文名、粒度、表类型、业务主键、业务日期和每列的含义、单位、度量类型。导入表格的源已有系统按核对结果生成的说明，目录不重复、不改写它 | +| 模型起草 | 作为后台任务逐表进行，每张表记下状态和原因，写完一张就并入目录；关掉对话框不停，可以停止、只重试失败的表，清单上标出上次起草失败或只完成一部分的表。服务重启时没跑完的任务标成中断，不会自己续跑，点重试接上。保留模型的思考，输出额度 32768（服务方不收时退回 8192）；每次调用最多 40 列，宽表先起草表级的项、再把列均分成几段起草，按表的先后排队，不插到窄表前面；额度用完或超时就拆小再问（超时的两半同时问，拆到每段 10 列仍接连超时就不再拆），累计两次后新开始的段减半（最低 10 列），每段连同它分开、拆小的各块最多调用 8 次，缺的表和列只补问缺的，被截断的回复先留下已经写完整的部分；最近 6 次调用有 4 次没写出来时提醒换模型。没写出来的原因写清是第几段、哪几列。重新起草时模型这次留空或没通过校验的项不删以前起草的；模型拿不准、留空的列（目录里还没有中文名和含义的）逐表记下，可以补充起草要求后重起草。库注释就是表名、列名本身或者是标识符写法的，不记进目录（不当中文名，也不当说明、含义）。数据源上存的「起草要求」和这次的补充说明一起交给模型，帮它理解业务、统一用词，不能推翻「只记数据事实」的规矩，交回的内容照样逐项校验。用哪个模型在「设置 → 偏好设置」的「起草模型」里选，空着跟随助手的模型；按配方导入表格的 AI 起草也用它 | | 数据剖析 | 默认关闭，按数据源开启。只读、经过守卫，单条查询和整次剖析都有上限，大表用统计信息估算行数，遮罩的列不取样。抽样核实推断出的关联，补上覆盖率和基数，站得住的升为已验证；状态列给出码值候选，含义等人填写 | | 对话里纠正 | 用户说「status=9 表示作废」，助手给出一条目录修改提案，列出改前、改后和理由，人确认后才保存；公式不收，留给口径卡 | | 版本与提醒 | 目录随表结构快照冻结进证据，每次运行用的是哪一版查得到。发布时记下模板引用的表的目录版本，正式运行时目录有变化就提醒；agent 运行时才写 SQL 的，按「可能涉及」单独提醒。只提醒，不拦运行 | diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index d52ff971..0405a0f7 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -7,11 +7,16 @@ - POST /catalog/{table}/patch/preview 目录修改提案的预览（只读）：对着当前目录算改前、改后 - POST /catalog/{table}/patch 保存目录修改提案（乐观锁），记为人工确认 - GET /catalog/{table}/impact 影响面：引用这张表的已发布、受管模板（data/catalog_impact.py） -- POST /catalog/draft 同步起草（注释、外键、命名推断，可选模型） +- POST /catalog/draft 同步起草（注释、外键、命名推断，可选模型）。界面改用下面的后台起草任务，这个留着； + 同一个源有排队或进行中的后台任务时 409 catalog_draft_busy +- PUT /catalog/guidance 数据源级的起草要求（使用者补充的背景和要求，起草时交给模型） +- POST /catalog/draft-jobs 建后台起草任务（202）；GET /draft-jobs/active、/latest、/{job_id} 读任务； + POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py） - POST /catalog/profile 同步剖析：对业务库发少量只读查询，核对关系、取码值候选（data/catalog_profile.py） 规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径 不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。 +起草任务的 409 带机读码（api/coded.py）：同一个源已有排队或进行中的任务、起草模型用不了、要停止的任务已经结束。 上传表格的源按当前快照的表结构回答（table_versions.resolve_source），和查询实际用的库是同一个版本。 """ @@ -19,13 +24,17 @@ from __future__ import annotations from typing import Any, Literal -from fastapi import APIRouter, Depends, Header, HTTPException -from pydantic import BaseModel, Field +from fastapi import APIRouter, Body, Depends, Header, HTTPException +from fastapi.exceptions import RequestValidationError +from pydantic import BaseModel, Field, ValidationError from sqlalchemy.ext.asyncio import AsyncSession -from app.api.coded import PROFILE_BUSY, SCHEMA_MISSING, SNAPSHOT_TAMPERED, CodedHTTPException +from app.api.coded import ( + CATALOG_DRAFT_BUSY, CATALOG_DRAFT_FINISHED, CATALOG_DRAFT_MODEL_UNAVAILABLE, PROFILE_BUSY, SCHEMA_MISSING, + SNAPSHOT_TAMPERED, CodedHTTPException, +) from app.api.runs import actor_of -from app.data import catalog, catalog_impact, catalog_profile, introspect, table_versions +from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, introspect, table_versions from app.data.engine import SnapshotTampered from app.db.base import get_session from app.db.models import DataSource @@ -64,10 +73,26 @@ class CatalogPatchIn(CatalogPatchPreviewIn): class CatalogDraftIn(BaseModel): #: 要起草的表；不给或为空时按使用次数取前 20 张 tables: list[str] | None = Field(default=None, max_length=200) - #: 是否请助手的模型起草中文名、粒度、列的含义等 + #: 是否请起草模型（设置里的起草模型，没设沿用助手的模型）写中文名、粒度、列的含义等 use_model: bool = False +class DraftJobIn(BaseModel): + #: 要起草的表；不给或为空时按使用次数取前 20 张。最多 500 张（超了 422，说明是中文） + tables: list[str] | None = None + #: 是否请起草模型写中文名、粒度、列的含义等 + use_model: bool = False + #: 这一次的补充说明，和数据源级的起草要求一起交给模型（最多 1000 字） + note: str | None = None + + +#: PUT /catalog/guidance 的请求 {guidance: string}：数据源级的起草要求，空串（或 null）清除，最多 2000 字。必须写 +#: 这一项：表名恰好叫 guidance 时，整份提交那张表目录的请求也落到这个路径上，不能把它当成「清除起草要求」。 +#: 没写、不是文字都回 422，说明是中文的一句话（同超长） +GUIDANCE_MISSING = "请求里要写起草要求（guidance 一项，一段文字；要清除起草要求请传空串）" +GUIDANCE_NOT_TEXT = "起草要求要是一段文字（要清除起草要求请传空串）" + + class CatalogProfileIn(BaseModel): #: 要剖析的表；不给或为空时按使用次数取前 10 张目录里有待核实关系的表 tables: list[str] | None = Field(default=None, max_length=200) @@ -100,7 +125,7 @@ def _item_value(notes: dict[str, Any], name: str) -> tuple[Any, str | None]: def _summary_row(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogEntry | None, - usage: int) -> dict[str, Any]: + usage: int, last_draft: dict[str, Any] | None = None) -> dict[str, Any]: notes = entry.notes if entry else {} label, label_status = _item_value(notes, "label") kind, _ = _item_value(notes, "kind") @@ -120,6 +145,9 @@ def _summary_row(name: str, meta: dict[str, Any] | None, entry: catalog.CatalogE "version": entry.version if entry else 0, "updated_at": _iso(entry.updated_at) if entry else None, "updated_by": entry.updated_by if entry else None, + # 这张表最近一次起草任务里的那一项：{status, job_id, finished_at, error, model_error, blank_columns}，没起草过 + # 是 null + "last_draft": last_draft, } @@ -180,15 +208,18 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi 说明有没有截断，schema_total 是数据库里一共几张表（含视图），界面据此说明「只探查了前 200 张」。老缓存没记 总数时按探查到的张数算。 """ - _, source = await _resolved(session, source_id) + row, source = await _resolved(session, source_id) cache = source.schema_cache or {} tables = cache.get("tables") or {} entries = await catalog.read_catalog(session, source.id) usage = await catalog.table_usage(session, source) + drafts = await catalog_jobs.last_drafts(session, source.id) order = {name: i for i, name in enumerate(tables)} - rows = [_summary_row(name, meta, entries.get(name), usage.get(name, 0)) for name, meta in tables.items()] + rows = [_summary_row(name, meta, entries.get(name), usage.get(name, 0), drafts.get(name)) + for name, meta in tables.items()] rows.sort(key=lambda r: (-r["usage"], order[r["table_name"]])) - rows += [_summary_row(name, None, entry, 0) for name, entry in sorted(entries.items()) if name not in tables] + rows += [_summary_row(name, None, entry, 0, drafts.get(name)) + for name, entry in sorted(entries.items()) if name not in tables] return { "tables": rows, "system_notes": catalog.system_notes_source(source), @@ -196,6 +227,10 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi "schema_note": None if tables else introspect.why_empty(cache), "schema_truncated": bool(cache.get("truncated")), "schema_total": max(_int(cache.get("total")), len(tables)), + # 数据源级的起草要求（没有是 null） + "guidance": catalog_jobs.guidance_of(row), + # 排队或进行中的起草任务 {id, status, total, done}，界面据此显示进度；没有是 null + "active_job": await catalog_jobs.job_brief(session, await catalog_jobs.active_job(session, source.id)), } @@ -207,6 +242,156 @@ async def get_catalog_table(source_id: str, table: str, return await _detail(session, source, key, entry) +async def _source_row(session: AsyncSession, source_id: str) -> DataSource: + row = await session.get(DataSource, source_id) + if row is None: + raise HTTPException(404, "数据源不存在，可能已被删除") + return row + + +@router.put("/{source_id}/catalog/guidance") +async def put_catalog_guidance(source_id: str, body: dict[str, Any] = Body(...), + x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> Any: + """存数据源级的起草要求：去首尾空白，空串清除，最多 2000 字（超了 422）。上传表格的源存在它自己的数据源记录上。 + + 要登记在 PUT /catalog/{table} 前面：不然「guidance」会被当成表名。反过来，表名恰好叫 guidance 时，整份提交那张表 + 目录的请求（{notes, if_version}，没有 guidance 这一项）也落到这里：照旧当作那张表的整份提交。""" + if "guidance" not in body and "notes" in body: + try: + table_payload = CatalogPutIn.model_validate(body) + except ValidationError as e: + raise RequestValidationError(e.errors()) from e + return await put_catalog_table(source_id, "guidance", table_payload, x_actor=x_actor, session=session) + if "guidance" not in body: + raise HTTPException(422, GUIDANCE_MISSING) + guidance = body["guidance"] + if guidance is not None and not isinstance(guidance, str): + raise HTTPException(422, GUIDANCE_NOT_TEXT) + row = await _source_row(session, source_id) + try: + value = await catalog_jobs.save_guidance(session, row, guidance) + except catalog_jobs.TextTooLong as e: + raise HTTPException(422, str(e)) from e + return {"guidance": value} + + +async def _ensure_model(session: AsyncSession) -> None: + """起草模型用不了（没有接入、停用、只有演示接入……）：409，不建任务。说法同同步起草时的 model_error。""" + try: + await catalog_jobs.check_model(session) + except catalog.CatalogModelUnavailable as e: + raise CodedHTTPException(409, str(e), CATALOG_DRAFT_MODEL_UNAVAILABLE) from e + + +def _no_schema(cache: dict[str, Any]) -> HTTPException: + return HTTPException(409, f"{introspect.why_empty(cache)}，无法起草数据目录。请先在数据源卡片上点「探查结构」") + + +@router.post("/{source_id}/catalog/draft-jobs", status_code=202) +async def create_draft_job(source_id: str, payload: DraftJobIn, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """建后台起草任务，马上返回（202，任务是排队中）。同一个源已有排队或进行中的任务 409 catalog_draft_busy（界面改去 + GET /draft-jobs/active 显示那一个）；用模型而起草模型用不了 409 catalog_draft_model_unavailable；没有表结构 409。 + 表结构里没有的表照样建进任务、状态 skipped。""" + if payload.tables and len(payload.tables) > catalog_jobs.MAX_TABLES: + raise HTTPException(422, f"一次最多起草 {catalog_jobs.MAX_TABLES} 张表，当前 {len(payload.tables)} 张，" + "请分几次起草") + try: + note = catalog_jobs.clean_text(payload.note, catalog_jobs.NOTE_MAX, "本次说明") + except catalog_jobs.TextTooLong as e: + raise HTTPException(422, str(e)) from e + row, source = await _resolved(session, source_id) + try: + await catalog_jobs.ensure_idle(session, row) + except catalog_jobs.DraftBusy as e: + raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e + cache = source.schema_cache or {} + if not cache.get("tables"): + raise _no_schema(cache) + if payload.use_model: + await _ensure_model(session) + try: + job, tasks = await catalog_jobs.create_job(session, row, source, tables=payload.tables or None, + use_model=payload.use_model, note=note, actor=actor_of(x_actor)) + except catalog_jobs.DraftBusy as e: + raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e + catalog_jobs.manager.start(job.id) + return {"job": catalog_jobs.job_dict(job, tasks)} + + +@router.get("/{source_id}/catalog/draft-jobs/active") +async def active_draft_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """排队或进行中的那个起草任务，没有是 null。""" + row = await _source_row(session, source_id) + return {"job": await catalog_jobs.job_out(session, await catalog_jobs.active_job(session, row.id))} + + +@router.get("/{source_id}/catalog/draft-jobs/latest") +async def latest_draft_job(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """最近建的一个起草任务（任何状态），没有是 null。""" + row = await _source_row(session, source_id) + return {"job": await catalog_jobs.job_out(session, await catalog_jobs.latest_job(session, row.id))} + + +async def _job_or_404(session: AsyncSession, source_id: str, job_id: str) -> tuple[DataSource, Any]: + row = await _source_row(session, source_id) + job = await catalog_jobs.load(session, row.id, job_id) + if job is None: + raise HTTPException(404, "起草任务不存在，可能属于别的数据源，或者已随数据源删除") + return row, job + + +@router.get("/{source_id}/catalog/draft-jobs/{job_id}") +async def get_draft_job(source_id: str, job_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + _, job = await _job_or_404(session, source_id, job_id) + return {"job": await catalog_jobs.job_out(session, job)} + + +@router.post("/{source_id}/catalog/draft-jobs/{job_id}/cancel") +async def cancel_draft_job(source_id: str, job_id: str, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """停止：排队的直接停止；进行中的停掉正在进行的模型调用，没开始和没写完的表记停止，已经写进目录的不回滚。 + 已经结束的 409 catalog_draft_finished。返回停止之后的任务。""" + row, job = await _job_or_404(session, source_id, job_id) + owner = row.id + try: + await catalog_jobs.cancel(session, job) + except catalog_jobs.DraftFinished as e: + raise CodedHTTPException(409, str(e), CATALOG_DRAFT_FINISHED) from e + session.expire_all() # 后台在别的会话里收的尾：重新读 + return {"job": await catalog_jobs.job_out(session, await catalog_jobs.load(session, owner, job_id))} + + +@router.post("/{source_id}/catalog/draft-jobs/{job_id}/retry", status_code=202) +async def retry_draft_job(source_id: str, job_id: str, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """重试：新任务，只含原任务里没写好（failed、partial）和停止了（cancelled）的表；用不用模型、本次说明沿用，起草要求 + 取数据源当前的。有排队或进行中的任务 409 catalog_draft_busy；没有可重试的表 422；起草模型用不了 409。""" + _, old = await _job_or_404(session, source_id, job_id) + row, source = await _resolved(session, source_id) + try: + await catalog_jobs.ensure_idle(session, row) + except catalog_jobs.DraftBusy as e: + raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e + try: + tables = await catalog_jobs.retry_tables(session, old) + except catalog_jobs.NothingToRetry as e: + raise HTTPException(422, str(e)) from e + cache = source.schema_cache or {} + if not cache.get("tables"): + raise _no_schema(cache) + if old.use_model: + await _ensure_model(session) + try: + job, tasks = await catalog_jobs.create_job(session, row, source, tables=tables, use_model=old.use_model, + note=old.note, actor=actor_of(x_actor), retry_of=old.id) + except catalog_jobs.DraftBusy as e: + raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e + catalog_jobs.manager.start(job.id) + return {"job": catalog_jobs.job_dict(job, tasks)} + + @router.put("/{source_id}/catalog/{table}") async def put_catalog_table(source_id: str, table: str, payload: CatalogPutIn, x_actor: str | None = Header(default=None), @@ -228,17 +413,26 @@ async def put_catalog_table(source_id: str, table: str, payload: CatalogPutIn, async def draft_catalog(source_id: str, payload: CatalogDraftIn, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - """同步起草。返回每张表新增、更新、删除了几项；模型用不了或某张表的模型起草失败，照实写在结果里。 + """同步起草（界面改用后台起草任务，这个接口留着，返回形状不变）。引擎同后台任务（分段、按失败原因处理），起草要求 + 取数据源级的那份。返回每张表新增、更新、删除了几项；模型用不了或某张表的模型起草失败，照实写在结果里。 model_items：模型给了这张表几项可用内容（没用模型、模型这部分失败时为 null）。界面据此区分「模型给了内容、 只是和现有目录一致」（model_items > 0、三个数都是 0）和「模型没给内容」（model_error 有值）。 + + 同一个源有排队或进行中的后台起草任务时 409 catalog_draft_busy（和建任务同一句话）：两边同时写同一个源的目录、 + 抢模型名额。 """ - _, source = await _resolved(session, source_id) + row, source = await _resolved(session, source_id) + try: + await catalog_jobs.ensure_idle(session, row) + except catalog_jobs.DraftBusy as e: + raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e cache = source.schema_cache or {} if not cache.get("tables"): - raise HTTPException(409, f"{introspect.why_empty(cache)}，无法起草数据目录。请先在数据源卡片上点「探查结构」") + raise _no_schema(cache) report = await catalog.draft_catalog(session, source, tables=payload.tables or None, - use_model=payload.use_model, actor=actor_of(x_actor)) + use_model=payload.use_model, actor=actor_of(x_actor), + guidance=catalog_jobs.guidance_of(row)) rows = [{"table_name": r.table, "added": r.added, "updated": r.updated, "removed": r.removed, "version": r.version, "error": r.error, "model_error": r.model_error, "model_items": r.model_items} for r in report.tables] diff --git a/backend/app/api/coded.py b/backend/app/api/coded.py index 4cf6da90..204b94af 100644 --- a/backend/app/api/coded.py +++ b/backend/app/api/coded.py @@ -19,6 +19,11 @@ DATASOURCE_INACTIVE = "datasource_inactive" SCHEMA_MISSING = "schema_missing" PROFILE_BUSY = "profile_busy" SNAPSHOT_TAMPERED = "snapshot_tampered" +#: 数据目录的后台起草任务（api/catalog.py 的 /catalog/draft-jobs）：同一个源已有排队或进行中的任务（409，界面改去 +#: GET /draft-jobs/active 显示那一个）、起草模型用不了（409，不建任务）、要停止的任务已经结束了（409） +CATALOG_DRAFT_BUSY = "catalog_draft_busy" +CATALOG_DRAFT_MODEL_UNAVAILABLE = "catalog_draft_model_unavailable" +CATALOG_DRAFT_FINISHED = "catalog_draft_finished" class CodedHTTPException(HTTPException): diff --git a/backend/app/api/datasources.py b/backend/app/api/datasources.py index c4be16a8..f2a9e008 100644 --- a/backend/app/api/datasources.py +++ b/backend/app/api/datasources.py @@ -27,8 +27,8 @@ from app.data.catalog_profile import profile_settings_rejection from app.data import introspect as introspect_mod from app.data import table_versions from app.data.engine import ( - MASK_COLUMNS_OPTION, MAX_QUERY_TIMEOUT_S, QUERY_TIMEOUT_OPTION, SUPPORTED_KINDS, SnapshotTampered, build_url, - engine_args, engines, mask_columns_problem, query_timeout_problem, + CATALOG_DRAFT_GUIDANCE_OPTION, MASK_COLUMNS_OPTION, MAX_QUERY_TIMEOUT_S, QUERY_TIMEOUT_OPTION, SUPPORTED_KINDS, + SnapshotTampered, build_url, engine_args, engines, mask_columns_problem, query_timeout_problem, ) from app.db.base import get_session, new_id from app.db.models import DataSource, ImportStaging, SourceSnapshot, TableImport, TableRecipe @@ -124,6 +124,22 @@ def _import_mode(row: DataSource) -> str | None: return "simple" if _is_upload(row) else None +#: options 里由别的接口维护的项：数据源表单不显示、不提交它们，整份回写 options 时原样留着（同模型接入 extra 里的 +#: last_check）。数据目录的起草要求只经 PUT /catalog/guidance 改（api/catalog.py）：表单把认不得的项放进「高级参数」 +#: 的单行输入框，多行的要求会被压成一行 +_MANAGED_OPTIONS = (CATALOG_DRAFT_GUIDANCE_OPTION,) + + +def _shown_options(options: dict[str, Any] | None) -> dict[str, Any]: + return {k: v for k, v in (options or {}).items() if k not in _MANAGED_OPTIONS} + + +def _with_managed(options: dict[str, Any], saved: dict[str, Any] | None) -> dict[str, Any]: + """表单提交的 options + 存着的、由别的接口维护的项（提交里带了也不认）。""" + kept = {k: v for k, v in (saved or {}).items() if k in _MANAGED_OPTIONS} + return {**_shown_options(options), **kept} + + def _to_out(row: DataSource, snapshot: dict[str, Any] | None = None, recipe_info: dict[str, Any] | None = None) -> DataSourceOut: cache = row.schema_cache or {} @@ -137,7 +153,7 @@ def _to_out(row: DataSource, snapshot: dict[str, Any] | None = None, schema_error=str(cache.get("error") or "") if cache.get("failed") else "", available_schemas=list(cache.get("available_schemas") or []), id=row.id, name=row.name, kind=row.kind, host=row.host, port=row.port, - database=row.database, username=row.username, options=row.options or {}, + database=row.database, username=row.username, options=_shown_options(row.options), readonly=row.readonly, description=row.description, enabled=row.enabled, password_masked=mask(row.password), has_password=bool(row.password), @@ -429,7 +445,8 @@ async def create_source( _refuse_reserved(payload.kind, payload.database) row = DataSource( - **payload.model_dump(exclude={"password"}), + **payload.model_dump(exclude={"password", "options"}), + options=_with_managed(payload.options, None), password=encrypt(payload.password), ) _settle_oracle(row, None, None) @@ -472,6 +489,7 @@ async def update_source( raise HTTPException(422, _UPLOAD_DIR_READONLY) if "options" in data: _refuse_bad_options(data["options"]) + data["options"] = _with_managed(data["options"] or {}, row.options) if "password" in data: # 空串表示清空，非空表示换新的 data["password"] = encrypt(data["password"]) if data["password"] else None diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 9a4ee8ad..e1fdcafb 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -14,6 +14,7 @@ from app.api.coded import CodedHTTPException from app.core.errors import explain, not_configured, raw from app.core.config import settings as app_settings from app.core.crypto import encrypt, mask +from app.data.draft_model import CATALOG_DRAFT_SETTING_KEY, setting_pair from app.db.base import get_session from app.db.models import Provider, Setting from app.engine.judge import JUDGE_DEFAULTS, MAX_TOKENS as JUDGE_MAX_TOKENS @@ -569,6 +570,39 @@ async def verify_model_spec(session: AsyncSession, node_config: dict[str, Any] | return ModelSpec(provider=provider, model=model, max_tokens=JUDGE_MAX_TOKENS, thinking="off") +# -------------------------------------------------------------------------- +# 起草模型：数据目录的「用模型起草」和按配方导入的 AI 起草用哪个模型 +# -------------------------------------------------------------------------- + +#: 设置里 catalog_draft 这一组：{provider, model}。都留空时沿用助手的模型，再没有就是默认接入（取值顺序见 +#: data/draft_model.py，两处起草共用）。只放接入和模型：额度、思考由两处起草各自定 +CATALOG_DRAFT_GROUP = CATALOG_DRAFT_SETTING_KEY +CATALOG_DRAFT_DEFAULTS: dict[str, Any] = {"provider": None, "model": None} +#: 设置页「起草模型」一节各项的叫法：报错时写它，不写键名 +CATALOG_DRAFT_LABEL = {"provider": "起草模型 · 接入", "model": "起草模型 · 模型"} + + +def catalog_draft_problem(value: Any) -> str | None: + """设置页写进来的 catalog_draft 这一组有什么问题，没问题返回 None。点名的接入在不在由 _provider_problem 查。""" + if not isinstance(value, dict): + return f"「起草模型」设置格式有误，需要是一个对象，当前为「{value}」" + for key, item in value.items(): + if key not in CATALOG_DRAFT_DEFAULTS: + return (f"「起草模型」设置中不支持「{key}」，可设置的项：" + + "、".join(f"「{CATALOG_DRAFT_LABEL[k]}」" for k in CATALOG_DRAFT_DEFAULTS)) + if item is not None and not isinstance(item, str): + # 「跟随助手的模型」是设置页接入下拉框空选项的原话 + what = "模型接入的名称" if key == "provider" else "模型 ID" + return f"「{CATALOG_DRAFT_LABEL[key]}」需要填写{what}，留空表示跟随助手的模型，当前为「{item}」" + return None + + +def catalog_draft_values(stored: Any) -> dict[str, Any]: + """库里存的 catalog_draft 一组 → 补齐默认、规范化（去掉首尾空白，空串和存坏了的当没填）。""" + provider, model = setting_pair(stored) + return {"provider": provider, "model": model} + + # -------------------------------------------------------------------------- # 通知：定时任务推送 Webhook 时拼运行链接用的对外访问地址 # -------------------------------------------------------------------------- @@ -661,6 +695,8 @@ DEFAULT_SETTINGS: dict[str, Any] = { VERIFY_GROUP: verify_values(None), # 通知：定时任务的 Webhook 里拼运行链接用的对外访问地址 NOTIFY_GROUP: notify_values(None), + # 起草模型：数据目录起草、按配方导入的 AI 起草用的模型，留空沿用助手的模型 + CATALOG_DRAFT_GROUP: catalog_draft_values(None), } @@ -717,6 +753,7 @@ async def get_settings(session: AsyncSession = Depends(get_session)) -> dict[str merged[JUDGE_GROUP] = judge_values(stored.get(JUDGE_GROUP)) merged[VERIFY_GROUP] = verify_values(stored.get(VERIFY_GROUP)) merged[NOTIFY_GROUP] = notify_values(stored.get(NOTIFY_GROUP)) + merged[CATALOG_DRAFT_GROUP] = catalog_draft_values(stored.get(CATALOG_DRAFT_GROUP)) for key, value in stored.items(): merged.setdefault(key, value) return merged @@ -731,8 +768,8 @@ PROVIDER_INVALID = "settings_provider_invalid" def _provider_fields() -> dict[tuple[str, str], tuple[str, str | None]]: - """设置里点名模型接入的项：(组, 键) → (这一项在界面上的叫法, 同一组模型那一项的叫法)。门控、裁判、核查的叫法和 - 设置页一致；助手的模型由「编排」页助手栏的「助手使用的模型」单独设定（PUT /api/copilot/model），只有这一个下拉框 + """设置里点名模型接入的项：(组, 键) → (这一项在界面上的叫法, 同一组模型那一项的叫法)。门控、裁判、核查、起草模型的 + 叫法和设置页一致；助手的模型由「编排」页助手栏的「助手使用的模型」单独设定（PUT /api/copilot/model），只有这一个下拉框 （模型那一项为 None），这里只防经设置接口写进去的那一份。""" from app.api.copilot import COPILOT_SETTING_KEY # 在这里导入：copilot 模块很大，设置模块被引擎各处导入 from app.providers.factory import ASSISTANT_MODEL_FIELD @@ -741,6 +778,7 @@ def _provider_fields() -> dict[tuple[str, str], tuple[str, str | None]]: ("run", "tool_gate_provider"): ("门控模型 · 接入", "门控模型 · 模型"), (JUDGE_GROUP, "provider"): (JUDGE_LABEL["provider"], JUDGE_LABEL["model"]), (VERIFY_GROUP, "provider"): (VERIFY_SETTING_LABEL["provider"], VERIFY_SETTING_LABEL["model"]), + (CATALOG_DRAFT_GROUP, "provider"): (CATALOG_DRAFT_LABEL["provider"], CATALOG_DRAFT_LABEL["model"]), (COPILOT_SETTING_KEY, "provider"): (ASSISTANT_MODEL_FIELD, None), } @@ -798,6 +836,9 @@ async def put_settings( raise HTTPException(422, problem) if VERIFY_GROUP in payload.values and (problem := verify_problem(payload.values[VERIFY_GROUP])): raise HTTPException(422, problem) + if CATALOG_DRAFT_GROUP in payload.values and ( + problem := catalog_draft_problem(payload.values[CATALOG_DRAFT_GROUP])): + raise HTTPException(422, problem) if NOTIFY_GROUP in payload.values: if problem := notify_problem(payload.values[NOTIFY_GROUP]): raise HTTPException(422, problem) diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index fb7e7063..3dd0bb39 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -591,21 +591,41 @@ class MergeStats: return MergeStats(self.added + other.added, self.updated + other.updated, self.removed + other.removed) +def _about_gone_columns(key: Slot, item: Mapping[str, Any], columns: set[str]) -> bool: + """这一项说的列已经不在表结构里了：列级的项看列名，业务主键、业务日期看它们指着的列。""" + if key[0] == "c": + return key[1] not in columns + value = item.get("value") + if key == ("t", "keys") and isinstance(value, list): + return any(str(col) not in columns for col in value) + if key == ("t", "business_date") and isinstance(value, Mapping) and value.get("column") is not None: + return str(value["column"]) not in columns + return False + + def merge_notes(existing: Mapping[str, Any] | None, draft: Mapping[str, Any] | None, *, - covered: Iterable[str] = (), at: str | None = None) -> tuple[dict[str, Any], MergeStats]: + covered: Iterable[str] = (), at: str | None = None, + columns: Iterable[str] | None = None) -> tuple[dict[str, Any], MergeStats]: """把起草结果并入已有目录，返回 (新目录, 计数)。不改入参。 规则（逐个槽位）： - 已有项是 confirmed 或 rejected：不动。驳回的留着，同一项下次起草就不会再冒出来。 - - 起草结果里没有这一项：已有项的来源在 covered 里（这一轮完整算过这个来源）就删掉，否则留着。 - 外键约束被删掉以后，原来那条「有确证」的关系不能还挂着；而这一轮没调模型，就不能因此删掉模型起草的项。 + - 起草结果里没有这一项： + - 模型起草的项（llm）留着。模型这次留空、给的值没通过校验（_from_model 里就丢掉了），都不能把以前起草好的 + 删掉——实测整库重跑一次，模型这次没给的几百项全被删掉，有的表覆盖率从八成多掉到五成。只有这一轮完整起草过 + （covered 含 llm）、而它说的列已经不在表结构里（columns：表结构里的列名；业务主键、业务日期指着的列也算） + 才删；不知道表结构（columns 为 None）时不删。 + - 别的来源：来源在 covered 里（这一轮完整算过这个来源）就删掉，否则留着。外键约束被删掉以后，原来那条「有确证」 + 的关系不能还挂着；库注释改了，旧的那条注释也不能还挂着。 - 同一来源：值或状态有变化就更新。数据剖析写的备注只是日期变了不算变化（same_profile_note），保留旧的项。 - 不同来源：来源更可信的（_SOURCE_RANK）顶掉不如它的，反过来不行。 新增和更新的项盖上 at（缺省为当前时间）作为 updated_at；内容没变的项保持原样，修改时间也不动。 + removed 只数真删掉的项。 """ at = at or now_iso() covered = set(covered) + known = {str(c) for c in columns} if columns is not None else None old, new = _slots(existing), _slots(draft) out: dict[Slot, dict[str, Any]] = {} added = updated = removed = 0 @@ -614,7 +634,11 @@ def merge_notes(existing: Mapping[str, Any] | None, draft: Mapping[str, Any] | N if item.get("status") in _LOCKED: out[key] = item elif cand is None: - if item.get("source") in covered: + if item.get("source") == "llm": + gone = "llm" in covered and known is not None and _about_gone_columns(key, item, known) + else: + gone = item.get("source") in covered + if gone: removed += 1 else: out[key] = item @@ -1130,10 +1154,15 @@ async def save_patch(session: AsyncSession, source_id: str, table: str, changes: @dataclass(frozen=True) class TableDraft: - """一张表的起草结果。covered：这一轮完整算过的来源（merge_notes 据此删掉这些来源里已经没有的旧项）。""" + """一张表的起草结果。covered：这一轮完整算过的来源（merge_notes 据此删掉这些来源里已经没有的旧项；模型起草的项 + 只在列已经不在表结构里时才删）。 + + error：模型起草只写出了一部分（有的段没写出来）时，没写出来的是哪部分、为什么。这时 covered 不含 llm：没写全 + 就不算完整起草过。""" notes: dict[str, Any] covered: frozenset[str] = frozenset() + error: str | None = None def system_notes_source(source: Any) -> bool: @@ -1176,6 +1205,34 @@ def _comment_is_label(text: str) -> bool: return len(text) <= _LABEL_MAX and not _SENTENCE_MARKS.search(text) +#: 标识符写法的注释：只有字母、数字、下划线、点，不带空格（「member_card」「ord.status」「GMV」） +_IDENTIFIER = re.compile(r"[A-Za-z0-9_.]+") +#: 比较注释和名字时忽略的字符：下划线、连字符、空白 +_NAME_FILLER = re.compile(r"[\s_\-]+") + + +def _bare_name(text: str) -> str: + return _NAME_FILLER.sub("", text).lower() + + +def comment_role(text: str, name: str) -> str | None: + """数据库注释在目录里记成什么：label 中文名；note 一段解释（表记进说明，列记进含义）；None 不记。 + + - 注释就是名字本身（和表名、列名相同，忽略大小写、下划线、连字符和空白；表名带 schema 前缀的也比不带前缀的）， + 或者是标识符写法（只有字母、数字、下划线、点，不带空格）：说明不了什么，不记。以前这种注释短、不带句读，就当 + 中文名记下了；库注释的级别又比模型起草高，模型起草的中文名永远进不了目录。交给模型的表结构里照旧有这条注释； + - 短、不带句读的是名字（「实收金额」）； + - 否则是一段解释（「状态：1 表示有效…」）。 + + 只影响以后的起草：已有的目录不另做迁移，起草到那张表时按这一轮的注释重新算（旧的那条随之删掉）。 + """ + bare = _bare_name(text) + names = {name, name.rsplit(".", 1)[-1]} + if not bare or any(bare == _bare_name(n) for n in names) or _IDENTIFIER.fullmatch(text): + return None + return "label" if _comment_is_label(text) else "note" + + def _is_key(meta: Mapping[str, Any], columns: list[str]) -> bool: """这几列恰好是这张表的主键或某个唯一约束（不分大小写、不计顺序）。""" wanted = {str(c).lower() for c in columns} @@ -1367,7 +1424,8 @@ def draft_structure(source: Any, table: str) -> TableDraft: """只看表结构就能起草的部分：数据库注释、外键约束、命名推断。table 必须是 schema_cache["tables"] 的键。 - 数据库注释 → 中文名或含义（comment / proposed）：短而不带句读的是名字，记进表或列的中文名；否则是 - 一段解释，记进表的说明或列的含义。导入表格的源例外（system_notes_source）：说明是系统生成的，不拷。 + 一段解释，记进表的说明或列的含义；就是名字本身、标识符写法的不记（comment_role）。导入表格的源例外 + （system_notes_source）：说明是系统生成的，不拷。 - 外键约束 → 关系（fk / verified）。 - 命名推断 → 关系（name / proposed）。 """ @@ -1378,13 +1436,15 @@ def draft_structure(source: Any, table: str) -> TableDraft: notes: dict[str, Any] = {} if not system_notes_source(source): comment = (meta.get("comment") or "").strip() - if comment: - notes["label" if _comment_is_label(comment) else "description"] = make_item(comment, "comment") + role = comment_role(comment, table) if comment else None + if role is not None: + notes["label" if role == "label" else "description"] = make_item(comment, "comment") columns: dict[str, dict[str, Any]] = {} for col in meta.get("columns") or []: text = (col.get("comment") or "").strip() - if text: - columns[col["name"]] = {("label" if _comment_is_label(text) else "meaning"): make_item(text, "comment")} + role = comment_role(text, str(col["name"])) if text else None + if role is not None: + columns[col["name"]] = {("label" if role == "label" else "meaning"): make_item(text, "comment")} if columns: notes["columns"] = columns fks = fk_relations(cache, table) @@ -1401,15 +1461,39 @@ def draft_structure(source: Any, table: str) -> TableDraft: # 起草：模型（可选，按表分批，结构化输出） # ========================================================================== -#: 每批最多几张表、合计多少列。一批太大模型容易漏表、输出被截断；太小则调用次数多 +#: 每次调用最多几张表、合计多少列（data/catalog_draft.py 按它分段）。实测每列约 50 个输出 token（不算思考）， +#: 会思考的模型思考也算在输出额度里：一段太大，想完就没额度写正文。列数超过 MODEL_BATCH_COLUMNS 的宽表拆成 +#: 「表级一段 + 列分段」，列分段数是列数除以它向上取整，各段列数平均（相差不超过 1），不留很小的尾段 MODEL_BATCH_TABLES = 4 -MODEL_BATCH_COLUMNS = 120 -#: 同时发出的批数 +MODEL_BATCH_COLUMNS = 40 +#: 一次起草里额度用完或超时累计这么多次，之后新开始的段列数上限减半（MODEL_BATCH_COLUMNS → 一半 → …）， +#: 最低 MODEL_MIN_PART_COLUMNS 列 +MODEL_SHRINK_AFTER = 2 +MODEL_MIN_PART_COLUMNS = 10 +#: 一段（连同它拆出来的、补问的、重试的）最多调用几次模型，到了次数还没写完的记没写出来 +MODEL_PART_MAX_CALLS = 8 +#: 提醒：最近 MODEL_NOTICE_WINDOW 次调用里有 MODEL_NOTICE_ON 次以上没写出来时提醒换模型；之后少于 MODEL_NOTICE_OFF +#: 次时清掉 +MODEL_NOTICE_WINDOW = 6 +MODEL_NOTICE_ON = 4 +MODEL_NOTICE_OFF = 2 +#: 全进程同时进行的模型调用最多几个（所有起草任务、同步起草共用） MODEL_CONCURRENCY = 3 -#: 一批最多等多久 -MODEL_TIMEOUT_S = 120 -#: 输出额度。一批四张表的目录远用不完；给太大的额度有的服务方会直接拒收 -MODEL_MAX_TOKENS = 8192 +#: 单次调用最多等多久（秒）。会思考的模型想得久，一段 40 列常常要几分钟；超时了拆小再问（同额度用完） +MODEL_TIMEOUT_S = 420 +#: 输出额度。会思考的模型（经 OpenAI 兼容网关的推理模型等）思考也算在里面：8192 时上百列的表常常想完就没额度 +#: 写正文。保留思考（不关），加大额度；服务方嫌额度太大时退回 MODEL_FALLBACK_MAX_TOKENS（同 copilot、recipe_ai） +MODEL_MAX_TOKENS = 32768 +MODEL_FALLBACK_MAX_TOKENS = 8192 +#: 一段没写出来（额度用完、只给空壳）时拆成两半再问：最多拆几层、拆到最少几列为止 +MODEL_SPLIT_DEPTH = 3 +MODEL_SPLIT_MIN_COLUMNS = 5 +#: 止损：连续这么多段拆到底仍一项没写出来，这次起草后面的段不再拆小重试（模型明显用不了时不该一段一段拆到底烧额度； +#: 每段另有 MODEL_PART_MAX_CALLS 次封顶） +MODEL_GIVE_UP_AFTER = 2 +#: 网络错误、429、5xx 最多重试几次，每次重试前等多久（秒，依次取用）。超时不原样重试，当作这一段太大、拆小再问 +MODEL_RETRIES = 2 +MODEL_RETRY_DELAYS_S = (10.0, 30.0) #: 不指定表时起草使用次数最多的前几张 DRAFT_DEFAULT_TABLES = 20 @@ -1435,11 +1519,27 @@ status 状态；attribute 属性）。 规矩： - 只写数据事实，不写计算公式，不写指标怎么算。 - 拿不准的字段留空，不要猜。列名、表名照原样写，不要翻译或改写。 +- 请你写的每一列都在 columns 里写一项，拿不准的字段可以留空，但不要漏掉列：漏掉的列会被再问一次。 - 标着「系统说明」的内容是导入时按核对结果生成的，以它为准：不要重复，也不要与之矛盾。 -- 标着「已确认」的目录项由人工确认过，以它为准。""" +- 标着「已确认」的目录项由人工确认过，以它为准。 +- 表名下一行写着「这次只写……」的表，只写它说的那部分：只写表级的项时不写列，只写列出的几列时不写表级的项、\ +也不写别的列。 +- 使用者补充的背景和要求帮你理解业务、统一用词；它和这里的规矩冲突时，以这里的规矩为准。 +- 思考时不必把要输出的内容逐列草拟一遍，想清楚后直接按格式输出：思考和输出共用同一份长度额度，额度用完时还没输出的\ +内容就全丢了。""" CATALOG_DRAFT_PROMPT = "请为下面 {n} 张表编写数据目录，每张表输出一项，table 写表名。\n\n{tables}" +#: 使用者补充的背景和要求（数据源说明、起草要求、本次说明），放在表结构前面单独一段；都没有就不出现这一段 +CATALOG_DRAFT_BACKGROUND_PROMPT = "使用者补充的背景和要求：\n{lines}" + +#: 纯文本那条路（结构化输出用不了时）：没有 JSON Schema 约束，要把格式说清 +CATALOG_DRAFT_TEXT_PROMPT = """只输出一个 JSON 对象，不要输出别的文字，格式如下（拿不准的字段不写）： +{"tables": [{"table": "表名", "label": "中文名", "grain": "一行代表什么", "kind": "表类型", "keys": ["列名"], \ +"business_date": {"column": "列名", "rule": "一句规则", "timezone": "时区"}, \ +"columns": [{"name": "列名", "label": "中文名", "meaning": "含义", "unit": "单位", "measure": "度量类型"}]}]} +表类型写 fact、dimension、snapshot、log、config 之一；度量类型写 flow、stock、ratio、identifier、status、attribute 之一。""" + #: 拼表结构描述用的固定片段（写给模型） CATALOG_DRAFT_PROMPT_LABELS = { "table": "表 {name}", @@ -1455,6 +1555,16 @@ CATALOG_DRAFT_PROMPT_LABELS = { "col_system": "系统说明：{text}", "unique": "唯一约束：{cols}", "confirmed": "已确认的目录项：", + # 分段起草：表名下一行说这次只写哪部分（宽表的表级一段、列分段，拆小重问时的半段） + "only_table": "这次只写表级的项（中文名、粒度、表类型、业务主键、业务日期），不写列", + "only_columns": "这次只写下面列出的 {n} 列（这张表共 {total} 列），不写表级的项", + "table_and_columns": "这次只写表级的项和这 {n} 列：{cols}", + "brief_columns": "列（共 {n} 列，这里只列名字和类型）：", + "drafted": "已起草的表级内容：", + # 使用者补充的背景和要求 + "description": "数据源说明：{text}", + "guidance": "起草要求：{text}", + "note": "本次说明：{text}", } CATALOG_DRAFT_PROMPT_SCHEMA: dict[str, Any] = { @@ -1499,99 +1609,69 @@ CATALOG_DRAFT_PROMPT_SCHEMA: dict[str, Any] = { "required": ["tables"], } -_P = CATALOG_DRAFT_PROMPT_LABELS +class DraftModel: + """起草用的模型：先按 MODEL_MAX_TOKENS 的额度调；服务方嫌额度太大（budget_rejected）时换成 MODEL_FALLBACK_MAX_TOKENS + 的那一个，之后的调用都用它（shrink）。一次起草（一个起草任务、一次同步起草）共用一个，退回之后不再先失败一次。 + + 只转发起草用到的两个方法：with_structured_output 每次都取当前那个模型的，ainvoke 同理。""" + + def __init__(self, big: Any, small: Any | None, model_name: str = "") -> None: + self.model, self._small = big, small + self.model_name = model_name + self.budget = MODEL_MAX_TOKENS + + def shrink(self) -> bool: + """换成小额度的模型。已经换过、没有小的，返回 False。""" + if self._small is None or self.model is self._small: + return False + self.model, self.budget = self._small, MODEL_FALLBACK_MAX_TOKENS + return True + + def with_structured_output(self, schema: Any, **kwargs: Any) -> Any: + return self.model.with_structured_output(schema, **kwargs) + + async def ainvoke(self, messages: list[Any]) -> Any: + return await self.model.ainvoke(messages) async def resolve_draft_model(session: AsyncSession) -> tuple[Any, str]: - """(模型, 模型 id)：用设置里助手的模型，关掉思考、温度 0。没有可用的模型抛 CatalogModelUnavailable。 + """(模型, 模型 id)：用哪个接入、哪个模型按 data/draft_model.py 的顺序（设置里的起草模型 > 助手的模型 > 默认接入）， + 和按配方导入的 AI 起草共用。没有可用的模型抛 CatalogModelUnavailable。 + + 不关思考（spec 里不传 thinking，按接入默认）、温度 0、额度 MODEL_MAX_TOKENS，服务方嫌大时退回 + MODEL_FALLBACK_MAX_TOKENS（DraftModel）。单次调用的时限 MODEL_TIMEOUT_S 也交给 SDK，免得它先按全局的 5 分钟断开。 + SDK 自己不重试（max_retries=0）：429、5xx、网络错误由起草引擎退避重试（最多 MODEL_RETRIES 次）；超时当作这一段 + 太大，拆小再问（data/catalog_draft.py），不原样重试。不然两层叠起来一段最多发 9 次请求，报错却写「已重试 2 次」， + 网关正在限流时还会放大流量。 + 只建模型对象，不发请求。 和 recipe_ai.resolve_draft_model 一样不用 get_chat_model：它在没有接入时静默退回演示模型，拿演示模型 - 起草出来的目录会被当成真的写进库里。数据层不导入接口层，设置键取 recipe_ai 里那份（与助手的同值， - 那边有测试核对）。 + 起草出来的目录会被当成真的写进库里。 """ - from app.data.recipe_ai import COPILOT_SETTING_KEY - from app.db.models import Setting - from app.providers.factory import ( - ModelSpec, ProviderNotConfigured, assistant_unavailable, build_chat_model, resolve_provider, - ) + from app.data.draft_model import DraftModelUnavailable, draft_unavailable, resolve_draft_provider + from app.providers.factory import ModelSpec, ProviderNotConfigured, build_chat_model - row = await session.get(Setting, COPILOT_SETTING_KEY) - saved = (row.value if row else None) or {} - spec = ModelSpec(provider=saved.get("provider") or None, model=saved.get("model") or None, - max_tokens=MODEL_MAX_TOKENS, temperature=0, thinking="off") try: - # 存的接入名不存在、已停用，只存了模型名、提供它的接入已停用：resolve_provider 都会报出来 - provider = await resolve_provider(session, spec.provider, spec.model) - except ProviderNotConfigured as e: - raise CatalogModelUnavailable(assistant_unavailable(e)) from e - if provider is None: - raise CatalogModelUnavailable("未配置模型接入，无法由助手起草数据目录。请到「设置 → 模型接入」添加") - if provider.kind == "mock": - raise CatalogModelUnavailable("当前只有演示用的模型接入，无法用于起草数据目录。" - "请到「设置 → 模型接入」添加真实的模型接入") + provider, choice = await resolve_draft_provider( + session, none_reason="未配置模型接入，无法由模型起草数据目录。请到「设置 → 模型接入」添加", + mock_reason="当前只有演示用的模型接入，无法用于起草数据目录。请到「设置 → 模型接入」添加真实的模型接入") + except DraftModelUnavailable as e: + raise CatalogModelUnavailable(str(e)) from e + spec = ModelSpec(provider=choice.provider, model=choice.model, max_tokens=MODEL_MAX_TOKENS, temperature=0, + timeout=MODEL_TIMEOUT_S, max_retries=0) try: - model = build_chat_model(provider, spec) + big = build_chat_model(provider, spec) + small = build_chat_model(provider, spec.model_copy(update={"max_tokens": MODEL_FALLBACK_MAX_TOKENS})) except ProviderNotConfigured as e: - raise CatalogModelUnavailable(assistant_unavailable(e)) from e - return model, spec.model or provider.default_model or "" + raise CatalogModelUnavailable(draft_unavailable(e, choice.origin)) from e + model_id = spec.model or provider.default_model or "" + return DraftModel(big, small, model_id), model_id def _confirmed_only(notes: Mapping[str, Any] | None) -> dict[str, Any]: return _assemble({k: v for k, v in _slots(notes).items() if v.get("status") in ("confirmed", "verified")}) -def _describe_for_model(name: str, meta: Mapping[str, Any], *, system: bool, existing: Mapping[str, Any] | None) -> str: - """一张表交给模型的结构描述：列、类型、主键、外键、唯一约束、注释；不含任何数据值。""" - lines = [(_P["view"] if meta.get("is_view") else _P["table"]).format(name=name)] - comment = (meta.get("comment") or "").strip() - if comment: - lines.append((_P["system"] if system else _P["comment"]).format(text=comment)) - pk = set(meta.get("primary_key") or []) - fk_of: dict[str, str] = {} - for fk in meta.get("foreign_keys") or []: - if isinstance(fk, dict) and len(fk.get("columns") or []) == 1: - to_cols = fk.get("to_columns") or [] - fk_of[fk["columns"][0]] = f"{fk.get('to_table')}.{to_cols[0]}" if to_cols else str(fk.get("to_table")) - lines.append(_P["columns"]) - for col in meta.get("columns") or []: - marks = [] - if col["name"] in pk: - marks.append(_P["pk"]) - if not col.get("nullable", True): - marks.append(_P["not_null"]) - if col["name"] in fk_of: - marks.append(_P["fk"].format(target=fk_of[col["name"]])) - if text := (col.get("comment") or "").strip(): - marks.append((_P["col_system"] if system else _P["col_comment"]).format(text=text)) - line = _P["column"].format(name=col["name"], type=col.get("type") or "") - lines.append(line + ("，" + "，".join(marks) if marks else "")) - for group in meta.get("unique") or []: - if isinstance(group, list) and group: - lines.append(_P["unique"].format(cols="、".join(group))) - confirmed = _notes_lines(_confirmed_only(existing), meta=meta, system_notes=False) - if confirmed: - lines.append(_P["confirmed"]) - lines += confirmed - return "\n".join(lines) - - -def _batches(names: list[str], tables: Mapping[str, Any]) -> list[list[str]]: - """按表分批：每批不超过 MODEL_BATCH_TABLES 张、合计不超过 MODEL_BATCH_COLUMNS 列（一张表超了就自成一批）。""" - out: list[list[str]] = [] - batch: list[str] = [] - width = 0 - for name in names: - n = len((tables.get(name) or {}).get("columns") or []) - if batch and (len(batch) >= MODEL_BATCH_TABLES or width + n > MODEL_BATCH_COLUMNS): - out.append(batch) - batch, width = [], 0 - batch.append(name) - width += n - if batch: - out.append(batch) - return out - - #: 模型给了这张表、但一项可用内容都没有（只有表名的空壳，或者给的值全不合规）：这张表记模型失败，不算起草成功。 #: 不然界面只看到「新增 0 项」，会说成「没有变化」 MODEL_EMPTY_REASON = "模型没有给出这张表的可用内容" @@ -1621,39 +1701,6 @@ def _model_tables(raw: Any) -> list[dict[str, Any]] | None: return [t for t in raw if isinstance(t, dict)] -async def _ask_model(model: Any, messages: list[Any], - usable: Callable[[list[dict[str, Any]]], bool] | None = None) -> list[dict[str, Any]]: - """问一批。先走结构化输出；模型不支持、调用报错、给回来的不是约定的结构，或者 usable 说它不够用（实测见过 - 结构化输出只给 {"tables":[{"table":"x"}]} 这样的空壳），再退回纯文本、从正文里取 JSON。 - - 结构化输出报错也退回（同 judge._ask）：有的网关不支持工具调用，一调就报错，纯文本却能用。代价是真的 - 网络故障会多试一次，两次加起来仍受 MODEL_TIMEOUT_S 约束。 - - 两条路都拿到了东西时，结构化的在前、纯文本的在后一起交回（同一张表各有一份，调用方挑能用的那份）；纯文本那次 - 失败了就只交回结构化的那份（调用方据此给空壳的表记失败）。 - """ - try: - parsed = _model_tables(await model.with_structured_output(CATALOG_DRAFT_PROMPT_SCHEMA).ainvoke(messages)) - except Exception: # noqa: BLE001 - 退回纯文本 - parsed = None - if parsed is not None and (usable is None or usable(parsed)): - return parsed - from app.engine.state import message_text - - try: - reply = await model.ainvoke(messages) - except Exception: # noqa: BLE001 - 结构化那份还在：交回它，空壳的表由调用方记失败 - if parsed is not None: - return parsed - raise - text = _model_tables(message_text(reply)) - if text is None: - if parsed is not None: - return parsed - raise ValueError("模型的回复不是约定的格式") - return [*(parsed or []), *text] - - def _clean_text(value: Any, limit: int = _TEXT_MAX) -> str | None: if not isinstance(value, str): return None @@ -1664,7 +1711,8 @@ def _clean_text(value: Any, limit: int = _TEXT_MAX) -> str | None: def _from_model(entry: Mapping[str, Any], meta: Mapping[str, Any], *, system: bool) -> dict[str, Any]: """模型给的一张表 → 目录项（llm / proposed）。不合规的值整项丢掉，不硬塞：列名对不上表结构的、 表类型和度量类型不在取值里的、业务主键里有不存在的列的、看起来是计算公式的（_formula_like，公式进口径卡）。 - 导入表格的源，说明已经覆盖的字段也丢掉。""" + 该是文字的地方写成了数组、对象（列名、表类型、业务日期的列、度量类型……）同样丢掉，不抛异常：实测见过把列名 + 写成数组的，以前在这里抛 TypeError，整张表记出错。导入表格的源，说明已经覆盖的字段也丢掉。""" columns = {c["name"]: c for c in meta.get("columns") or []} notes: dict[str, Any] = {} table_has_system_note = system and bool((meta.get("comment") or "").strip()) @@ -1673,21 +1721,23 @@ def _from_model(entry: Mapping[str, Any], meta: Mapping[str, Any], *, system: bo continue if (text := _clean_text(entry.get(name))) is not None and not _formula_like(("t", name), text): notes[name] = make_item(text, "llm") - if entry.get("kind") in TABLE_KINDS: - notes["kind"] = make_item(entry["kind"], "llm") + kind = entry.get("kind") + if isinstance(kind, str) and kind in TABLE_KINDS: + notes["kind"] = make_item(kind, "llm") keys = entry.get("keys") if _str_list_ok(keys) and all(k in columns for k in keys): notes["keys"] = make_item(list(dict.fromkeys(keys)), "llm") date = entry.get("business_date") - if isinstance(date, dict) and date.get("column") in columns: + if isinstance(date, dict) and isinstance(date.get("column"), str) and date["column"] in columns: value = {"column": date["column"]} for k in ("rule", "timezone"): if (text := _clean_text(date.get(k))) is not None: value[k] = text notes["business_date"] = make_item(value, "llm") out_cols: dict[str, dict[str, Any]] = {} - for col in entry.get("columns") or []: - if not isinstance(col, dict) or col.get("name") not in columns: + listed = entry.get("columns") + for col in listed if isinstance(listed, list) else []: + if not isinstance(col, dict) or not isinstance(col.get("name"), str) or col["name"] not in columns: continue name = col["name"] has_system_note = system and bool((columns[name].get("comment") or "").strip()) @@ -1698,8 +1748,9 @@ def _from_model(entry: Mapping[str, Any], meta: Mapping[str, Any], *, system: bo text = _clean_text(col.get(field_name), limit) if text is not None and not _formula_like(("c", name, field_name), text): items[field_name] = make_item(text, "llm") - if col.get("measure") in COLUMN_MEASURES: - items["measure"] = make_item(col["measure"], "llm") + measure = col.get("measure") + if isinstance(measure, str) and measure in COLUMN_MEASURES: + items["measure"] = make_item(measure, "llm") if items: out_cols[name] = items if out_cols: @@ -1708,65 +1759,30 @@ def _from_model(entry: Mapping[str, Any], meta: Mapping[str, Any], *, system: bo async def draft_with_model(model: Any, source: Any, tables: list[str], *, - existing: Mapping[str, Mapping[str, Any]] | None = None) -> dict[str, TableDraft | str]: + existing: Mapping[str, Mapping[str, Any]] | None = None, + guidance: str | None = None, note: str | None = None) -> dict[str, TableDraft | str]: """用模型起草这些表的中文名、粒度、表类型、业务主键、业务日期和每列的中文名、含义、单位、度量类型。 - model 要有 with_structured_output(schema) 或 ainvoke(messages)（测试注入假模型）。按表分批（_batches）， - 最多 MODEL_CONCURRENCY 批同时发。返回 {表名: TableDraft 或失败原因}：一批失败只影响这一批的表， - 其余照常；失败的表不算 covered，已有的模型起草项不会因此被删。 + 实现在 data/catalog_draft.py（分段、按失败原因处理、捡回被截断的部分）；这里是给老调用方的形状：返回 + {表名: TableDraft 或失败原因}。全部写出来的 covered 是 {"llm"}；只写出一部分的仍是 TableDraft，covered 为空、 + error 写明哪部分没写出来；一项都没写出来的是失败原因。表结构里没有的表不在结果里。 - 交给模型的只有表结构（列、类型、主键、外键、唯一约束、注释），不发任何数据值；existing 里已确认、 - 有确证的项作为「以此为准」的背景一起交过去。 + model 要有 with_structured_output(schema) 或 ainvoke(messages)（测试注入假模型）。交给模型的只有表结构和使用者 + 补充的背景（数据源说明、guidance 起草要求、note 本次说明），不发任何数据值；existing 里已确认、有确证的项作为 + 「以此为准」的背景一起交过去。 """ - from langchain_core.messages import HumanMessage, SystemMessage + from app.data import catalog_draft - all_tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} - system = system_notes_source(source) - names = [t for t in tables if isinstance(all_tables.get(t), dict)] - results: dict[str, TableDraft | str] = {} - gate = asyncio.Semaphore(MODEL_CONCURRENCY) - - async def run(batch: list[str]) -> None: - described = "\n\n".join(_describe_for_model(n, all_tables[n], system=system, - existing=(existing or {}).get(n)) for n in batch) - messages = [SystemMessage(content=CATALOG_DRAFT_SYSTEM), - HumanMessage(content=CATALOG_DRAFT_PROMPT.format(n=len(batch), tables=described))] - def drafted(entries: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: - """这批里每张表能用的那份：同一张表给了几份（结构化、纯文本各一份）时取第一份有可用内容的。""" - out: dict[str, dict[str, Any]] = {} - for entry in entries: - name = str(entry.get("table")) - if name in batch and name not in out: - notes = _from_model(entry, all_tables[name], system=system) - if notes: - out[name] = notes - return out - - try: - async with gate: - got = await asyncio.wait_for( - _ask_model(model, messages, usable=lambda entries: len(drafted(entries)) == len(batch)), - timeout=MODEL_TIMEOUT_S) - except asyncio.TimeoutError: - for n in batch: - results[n] = f"模型起草超时：{MODEL_TIMEOUT_S} 秒内没有收到回复" - return - except Exception as e: # noqa: BLE001 - 一批失败不影响其他批和其他来源 - from app.core.errors import describe_exception, first_line - - for n in batch: - results[n] = f"模型起草失败：{first_line(e) or describe_exception(e)}" - return - named = {str(t.get("table")) for t in got} - usable = drafted(got) - for n in batch: - if n in usable: - results[n] = TableDraft(notes=usable[n], covered=frozenset({"llm"})) - else: - results[n] = MODEL_EMPTY_REASON if n in named else "模型的回复里没有这张表" - - await asyncio.gather(*(run(b) for b in _batches(names, all_tables))) - return {n: results[n] for n in names if n in results} + outcomes = await catalog_draft.run_model_draft( + model, source, tables, existing=existing, background=catalog_draft.Background(guidance=guidance, note=note)) + out: dict[str, TableDraft | str] = {} + for name, outcome in outcomes.items(): + if outcome.status == "failed": + out[name] = outcome.error or MODEL_EMPTY_REASON + else: + out[name] = TableDraft(notes=outcome.notes, covered=frozenset({"llm"}) if outcome.status == "ok" + else frozenset(), error=outcome.error) + return out # ========================================================================== @@ -1776,7 +1792,8 @@ async def draft_with_model(model: Any, source: Any, tables: list[str], *, @dataclass class TableDraftResult: - """一张表这次起草的结果。error：这张表没起草（表结构里没有、写入一直冲突）；model_error：只是模型那部分失败。""" + """一张表这次起草的结果。error：这张表没起草（表结构里没有、写入一直冲突）；model_error：只是模型那部分失败， + 或者只写出了一部分（哪部分没写出来、为什么）。""" table: str added: int = 0 @@ -1785,9 +1802,17 @@ class TableDraftResult: version: int = 0 error: str | None = None model_error: str | None = None - #: 模型给了这张表几项可用内容（并入目录之前，按项数）。没用模型、模型这部分失败时为 None。 + #: 模型给了这张表几项可用内容（并入目录之前，按项数）。没用模型、模型这部分一项都没写出来时为 None。 #: 大于 0 而新增、更新、删除都是 0，才是「模型给了内容、只是和现有目录一致」 model_items: int | None = None + #: ok 写入了（用模型时各段都写出来了）、partial 模型只写出一部分、failed 没写入或模型一段都没写出来、 + #: skipped 表结构里没有这张表 + status: str = "ok" + #: 模型分几段起草（不用模型时为 None） + parts_total: int | None = None + #: 模型拿不准、留空了的列（catalog_draft.ModelOutcome.blank，按表结构的顺序），只留并入目录之后仍然没有中文名和 + #: 含义的；表的状态不因此变 + blank: list[str] = field(default_factory=list) @dataclass @@ -1802,35 +1827,113 @@ class DraftReport: model_error: str | None = None -async def draft_catalog(session: AsyncSession, source: Any, *, tables: list[str] | None = None, - use_model: bool = False, model: Any = None, actor: str | None = None) -> DraftReport: - """起草这些表的目录并写入，同步完成。 +#: 表结构里没有要起草的表（被删了、改名了，或者名字写错了） +def missing_table_error(name: str) -> str: + return f"表结构里没有 {name}，请先重新探查结构" + + +#: 写入一直撞上别人刚改过（乐观锁重读重并三次仍冲突） +WRITE_CONFLICT_ERROR = "这张表的数据目录正被其他人修改，请稍后再起草" + + +async def write_draft(session: AsyncSession, source: Any, table: str, modeled: Any = None, *, at: str, + actor: str | None) -> TableDraftResult: + """一张表的起草并入目录并写入：数据库注释、外键约束、命名推断照做，modeled（catalog_draft.ModelOutcome， + 没用模型时为 None）有可用内容的再并上。同步起草和后台起草任务（每张表一结束就调）共用这一处。 - - source：手工源的 DataSource，或上传源绑定当前快照的视图（table_versions.resolve_source）；按 source.id 存。 - - tables 为空时按使用次数（table_usage）取前 DRAFT_DEFAULT_TABLES 张，次数相同按表结构里的顺序。 - - 每张表：数据库注释、外键约束、命名推断照做；use_model 时再加模型起草。model 可注入（测试用假模型）， - 不给就按设置解析（resolve_draft_model）；模型用不了、某一批失败，都不影响其余来源照常写入。 - 并入已有目录按 merge_notes 的规则：人工确认、驳回过的项不动。内容没变不写、不升版本。 + - 模型这次没给的项（留空、没通过校验）不删以前起草的；列已经不在表结构里的，模型各段都写出来时（covered 含 llm） + 照旧删掉。模型只写出一部分的不算完整起草过（covered 不含 llm）。 + - 模型留空的列（blank）按并入之后的目录再筛一遍：已经有中文名或含义的（以前起草的、库注释、起草进行中有人填的） + 不算留空。 - 写入撞上别人刚改过（乐观锁）就重读重并，最多三次。 """ - cache = getattr(source, "schema_cache", None) or {} - all_tables = cache.get("tables") or {} - report = DraftReport() - if tables: - wanted = list(dict.fromkeys(str(t) for t in tables)) + result = TableDraftResult(table=table) + structure = draft_structure(source, table) + meta = ((getattr(source, "schema_cache", None) or {}).get("tables") or {}).get(table) or {} + columns = [str(c["name"]) for c in meta.get("columns") or [] if isinstance(c, dict) and c.get("name") is not None] + model_draft: TableDraft | None = None + if modeled is not None: + result.parts_total = modeled.parts_total + result.model_error = modeled.error + if modeled.items: + result.model_items = modeled.items + result.blank = list(getattr(modeled, "blank", None) or []) + model_draft = TableDraft(notes=modeled.notes, covered=frozenset({"llm"}) if modeled.status == "ok" + else frozenset()) + for _attempt in range(3): + entry = await read_entry(session, source.id, table) + version = entry.version if entry else 0 + merged, stats = merge_notes(entry.notes if entry else {}, structure.notes, covered=structure.covered, at=at, + columns=columns) + if model_draft is not None: + merged, more = merge_notes(merged, model_draft.notes, covered=model_draft.covered, at=at, columns=columns) + stats += more + result.added, result.updated, result.removed = stats.added, stats.updated, stats.removed + result.version = version + if entry is None and not merged: + break # 什么也没起草出来，不建空目录 + try: + written = await write_entry(session, source.id, table, merged, if_version=version, actor=actor) + except CatalogConflict: + continue + result.version = written.version + break else: - usage = await table_usage(session, source) - order = {name: i for i, name in enumerate(all_tables)} - wanted = sorted(all_tables, key=lambda n: (-usage.get(n, 0), order[n]))[:DRAFT_DEFAULT_TABLES] + result.error = WRITE_CONFLICT_ERROR + result.added = result.updated = result.removed = 0 + result.blank = [] + if result.blank: + named = visible_notes(merged).get("columns") or {} + result.blank = [c for c in result.blank if not any(f in (named.get(c) or {}) for f in ("label", "meaning"))] + if result.error or (modeled is not None and modeled.status == "failed"): + result.status = "failed" + elif modeled is not None and modeled.status == "partial": + result.status = "partial" + return result + + +def pick_tables(cache: Mapping[str, Any], wanted: Iterable[str]) -> tuple[list[str], list[str]]: + """要起草的表 → (表结构里的键，去重、按给的顺序；表结构里没有的原样名字)。""" picked: list[str] = [] - for name in wanted: + unknown: list[str] = [] + for name in dict.fromkeys(str(t) for t in wanted): key = resolve_table_name(cache, name) if key is None: - report.tables.append(TableDraftResult(table=name, error=f"表结构里没有 {name}，请先重新探查结构")) + unknown.append(name) elif key not in picked: picked.append(key) + return picked, unknown - llm: dict[str, TableDraft | str] = {} + +async def default_tables(session: AsyncSession, source: Any) -> list[str]: + """不指定表时起草哪些：按使用次数（table_usage）取前 DRAFT_DEFAULT_TABLES 张，次数相同按表结构里的顺序。""" + all_tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} + usage = await table_usage(session, source) + order = {name: i for i, name in enumerate(all_tables)} + return sorted(all_tables, key=lambda n: (-usage.get(n, 0), order[n]))[:DRAFT_DEFAULT_TABLES] + + +async def draft_catalog(session: AsyncSession, source: Any, *, tables: list[str] | None = None, + use_model: bool = False, model: Any = None, actor: str | None = None, + guidance: str | None = None, note: str | None = None) -> DraftReport: + """起草这些表的目录并写入，同步完成（POST /catalog/draft；界面改用后台起草任务，见 data/catalog_jobs.py）。 + + - source：手工源的 DataSource，或上传源绑定当前快照的视图（table_versions.resolve_source）；按 source.id 存。 + - tables 为空时按使用次数取前 DRAFT_DEFAULT_TABLES 张（default_tables）。 + - 每张表：数据库注释、外键约束、命名推断照做；use_model 时再加模型起草（data/catalog_draft.py：分段、按失败 + 原因处理）。model 可注入（测试用假模型），不给就按设置解析（resolve_draft_model）；模型用不了、某一段失败， + 都不影响其余来源照常写入。guidance 是数据源级的起草要求，note 是本次说明，和数据源说明一起交给模型。 + - 并入、写入见 write_draft。 + """ + cache = getattr(source, "schema_cache", None) or {} + report = DraftReport() + wanted = list(tables) if tables else await default_tables(session, source) + picked, unknown = pick_tables(cache, wanted) + for name in unknown: + report.tables.append(TableDraftResult(table=name, error=missing_table_error(name), status="skipped")) + + outcomes: dict[str, Any] = {} if use_model and picked: try: if model is None: @@ -1840,41 +1943,17 @@ async def draft_catalog(session: AsyncSession, source: Any, *, tables: list[str] except CatalogModelUnavailable as e: report.model_error = str(e) else: + from app.data import catalog_draft + report.model_used = True existing = await read_catalog(session, source.id) - llm = await draft_with_model(model, source, picked, existing={t: e.notes for t, e in existing.items()}) + outcomes = await catalog_draft.run_model_draft( + model, source, picked, existing={t: e.notes for t, e in existing.items()}, + background=catalog_draft.Background(guidance=guidance, note=note)) at = now_iso() for table in picked: - result = TableDraftResult(table=table) - structure = draft_structure(source, table) - modeled = llm.get(table) - if isinstance(modeled, str): - result.model_error = modeled - elif isinstance(modeled, TableDraft): - result.model_items = len(_slots(modeled.notes)) - for _attempt in range(3): - entry = await read_entry(session, source.id, table) - version = entry.version if entry else 0 - merged, stats = merge_notes(entry.notes if entry else {}, structure.notes, covered=structure.covered, - at=at) - if isinstance(modeled, TableDraft): - merged, more = merge_notes(merged, modeled.notes, covered=modeled.covered, at=at) - stats += more - result.added, result.updated, result.removed = stats.added, stats.updated, stats.removed - result.version = version - if entry is None and not merged: - break # 什么也没起草出来，不建空目录 - try: - written = await write_entry(session, source.id, table, merged, if_version=version, actor=actor) - except CatalogConflict: - continue - result.version = written.version - break - else: - result.error = "这张表的数据目录正被其他人修改，请稍后再起草" - result.added = result.updated = result.removed = 0 - report.tables.append(result) + report.tables.append(await write_draft(session, source, table, outcomes.get(table), at=at, actor=actor)) return report @@ -2239,15 +2318,22 @@ def frozen_catalog(entries: Mapping[str, CatalogEntry], tables: Iterable[str]) - __all__ = [ - "CARDINALITIES", "CARDINALITY_LABEL", "CATALOG_DRAFT_PROMPT", "CATALOG_DRAFT_PROMPT_SCHEMA", - "CATALOG_DRAFT_SYSTEM", "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict", + "CARDINALITIES", "CARDINALITY_LABEL", "CATALOG_DRAFT_BACKGROUND_PROMPT", "CATALOG_DRAFT_PROMPT", + "CATALOG_DRAFT_PROMPT_LABELS", "CATALOG_DRAFT_PROMPT_SCHEMA", "CATALOG_DRAFT_SYSTEM", "CATALOG_DRAFT_TEXT_PROMPT", + "COLUMN_FIELDS", "COLUMN_FIELD_LABEL", "COLUMN_MEASURES", "CatalogConflict", "CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES", - "DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL", - "MODEL_EMPTY_REASON", "MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_NOTE_PREFIX", - "PROFILE_VERIFY_COVERAGE", + "DraftModel", "DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", + "MEASURE_LABEL", "MODEL_BATCH_COLUMNS", "MODEL_BATCH_TABLES", "MODEL_CONCURRENCY", "MODEL_EMPTY_REASON", + "MODEL_FALLBACK_MAX_TOKENS", "MODEL_MAX_TOKENS", "MODEL_MIN_PART_COLUMNS", "MODEL_NOTICE_OFF", "MODEL_NOTICE_ON", + "MODEL_NOTICE_WINDOW", "MODEL_PART_MAX_CALLS", "MODEL_RETRIES", "MODEL_RETRY_DELAYS_S", "MODEL_SHRINK_AFTER", + "MODEL_SPLIT_DEPTH", "MODEL_SPLIT_MIN_COLUMNS", "MODEL_TIMEOUT_S", "MergeStats", "NEW_RELATION_PATH", + "PATCH_MAX_CHANGES", + "PROFILE_NOTE_PREFIX", "PROFILE_VERIFY_COVERAGE", "WRITE_CONFLICT_ERROR", "default_tables", + "missing_table_error", "pick_tables", "write_draft", "PatchChange", "PatchPlan", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", "TABLE_KINDS", "TableDraft", "TableDraftResult", "UI_KIND_LABEL", "UI_MEASURE_LABEL", "apply_human_edit", - "apply_patch", "codes_complete", "describe_slot", "draft_catalog", "draft_structure", "draft_with_model", + "apply_patch", "codes_complete", "comment_role", "describe_slot", "draft_catalog", "draft_structure", + "draft_with_model", "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "is_date_column", "is_profile_note", "join_paths", "make_item", "merge_notes", "name_words", "now_iso", "parse_path", "plan_patch", "profile_relation_holds", diff --git a/backend/app/data/catalog_draft.py b/backend/app/data/catalog_draft.py new file mode 100644 index 00000000..4183bff1 --- /dev/null +++ b/backend/app/data/catalog_draft.py @@ -0,0 +1,1540 @@ +"""数据目录的模型起草：分段、按失败原因处理、捡回被截断的部分、记用量。 + +对外的名字留在 data/catalog.py（draft_with_model、draft_catalog、write_draft、CATALOG_DRAFT_SYSTEM、各个 MODEL_* +常量……），这里是实现。常量一律在调用时按 catalog.X 读：测试改 catalog 上的常量就能生效。 + +**为什么要分段、按原因处理。** 实测（经 OpenAI 兼容网关的会思考的模型）：思考算在输出额度里，一段上百列时思考就把 +额度全用掉，正文 0 字或只给一个空壳；同一批表有时成功有时失败，看模型这次想多久。以前失败了退回纯文本、发同一个提示词， +同样失败、耗时翻倍；超时把第一次拿到的部分也丢掉；一批里有一张表是空的就整批重问。 + +**分段。** 每次调用最多 MODEL_BATCH_TABLES 张表、合计 MODEL_BATCH_COLUMNS 列：列数不超过它的表按顺序拼在一起 +（整表段）；超过它的宽表拆成「表级一段 + 列分段」——表级那段给全部列名和类型做背景、只要表级的项（中文名、粒度、 +表类型、业务主键、业务日期）；列分段数是列数除以 MODEL_BATCH_COLUMNS 向上取整、各段列数平均（相差不超过 1，以前 +按 40 列定长切，留下几列的尾段，实测会思考的模型常对这种小段只给空壳），带着表名、表注释和表级起草结果做背景，只要 +这几列。最后合并成这张表的一份，照旧经 catalog._from_model 逐项校验（起草要求不能让模型绕过校验）。 + +**段大小随模型调整。** 一次起草里额度用完或超时累计 MODEL_SHRINK_AFTER 次，之后新开始的段列数上限减半（40 → 20 +→ 10，最低 MODEL_MIN_PART_COLUMNS）：新开始的段（整表段、列分段）超过上限就先按上限均分开再问。已经开始的段照旧 +按失败原因拆小。 + +**按失败原因处理，不原样重发。** + +- 思考加输出用完额度（finish_reason=length、LengthFinishReasonError、正文为空而有思考）、只给空壳、超时（当作这一段 + 太大，不原样重试同样大小的段）：把这一段拆成两半再问（多表的段按表拆，单表的按列拆），最小 + MODEL_SPLIT_MIN_COLUMNS 列，最多拆 MODEL_SPLIT_DEPTH 层。同一张表按列拆时带表级项的前一半先写、后一半拿它做背景； + 超时的两半同时问（一次超时就是一个时限，串着问 40 → 20 → 10 要好几个时限）。拆不开的空壳再问一次：结构化输出给的 + 换纯文本（有的网关的结构化输出只给空壳、纯文本却能写——写出来了，这次起草后面都走纯文本），已经走纯文本的原样再问； + 仍是空壳才记没写出来。 +- 结构化输出回了一段说明文字（不是 JSON、也没停在额度上）：网关接受了结构化输出的参数却没照做（例如开了丢弃不认识 + 的参数），提示词里又没有格式说明，模型就写成了说明文字。这次起草后面改走纯文本，这一段换纯文本问一次。有思考 + 不等于额度用完：只有停在额度上、或者正文为空而有思考才算。 +- 被截断的 JSON：先把已经完整的表、列对象捡回来，只补剩下的。 +- 一段里缺了的表、列只补缺的，补一次。补问之后：模型列出了、字段留空的列是拿不准（补问时才列出来的也一样），不算 + 没写出来（补问的那次调用把请写的都列出来了，用量和提醒里也不算没写出来）；整列没列出来的算没写出来（这张表是 + partial）；整张表还一项都没有的，照只给空壳处理（拆小、再问一次）。 +- 结构化输出这个接口不支持（400 / 422 提到 response_format、json_schema、tool……，或者模型类不支持）：这次起草 + 后面改走纯文本、从正文取 JSON（纯文本那条路另外说清 JSON 的格式）。 +- 服务方嫌额度太大：退回 MODEL_FALLBACK_MAX_TOKENS 重来，这次起草后面都用它（catalog.DraftModel.shrink）。 +- 网络错误、429、5xx：等一会儿重试（最多 MODEL_RETRIES 次，退避）。单次调用的时限是 MODEL_TIMEOUT_S。SDK + 自己不再重试（catalog.resolve_draft_model 建模型时 max_retries=0），实际重试次数和报错写的一致。 +- 别的错误：这一段没写出来，原因照实写。 +- 读回复时出了意料之外的错：这一块记没写出来（「起草这张表时出错」），这次调用照样记进用量、算一次出错。该是文字的 + 地方写成了数组、对象的值在校验（catalog._from_model）里丢掉，不算出错。 +- 封顶：每个原始段（计划里的整表段、表级那段或一个列分段，连同按缩小后的上限分开的各块、拆出来的、补问的、重试的） + 一共最多调用 MODEL_PART_MAX_CALLS 次，到了次数还没写完的记没写出来。 +- 止损：连续 MODEL_GIVE_UP_AFTER 段拆到底仍一项没写出来（模型明显用不了，例如思考总是吃满额度），这次起草后面的 + 段不再拆小重试、没写出来就直接记下，报错建议更换起草模型；有一段写出来了就恢复。超时另有一条：拆到每段 + MODEL_MIN_PART_COLUMNS 列的调用接连超时 MODEL_GIVE_UP_AFTER 次（超时和段大小无关，再拆小也是一次一个时限），后面 + 超时的不再拆小，超时拆出来、还没问的块也不再问；有一次调用写出来了就恢复。不然一直超时的模型首轮同时起草的几段 + 要各自拆满次数，一段 40 列串着问约 7 个时限（420 秒时约 49 分钟）。 +- 提醒：最近 MODEL_NOTICE_WINDOW 次调用里有 MODEL_NOTICE_ON 次以上没写出来（额度用完、超时、空壳、出错），给一句 + 提醒（次数和原因，建议换起草模型、缩小起草范围或停止），经进度回调交给任务；之后少于 MODEL_NOTICE_OFF 次时清掉。 + 以前看不出调用在反复失败：一个任务 55 分钟只结束 1 张表，界面上只有段进度和用量。 + +**原因。** 没写出来的原因只写进这张表的 model_error：几段因为同一个原因没写出来时说「这几段」，分段起草的表（只写 +出一部分的、一项都没写出来的都一样）每段写上列的范围（「第 5 段（列名 到 列名，共 33 列）」，列名照表结构原样）， +真拆小重试过的才补「已拆小重试仍没写出」。只有一段的表，这一段就是整张表：只写出一部分的写清是哪几列，一项都没写 +出来的只写原因。 + +**留空的列。** 请模型写了、模型也列出了，中文名和含义却都没给（或者给的都没通过校验）的列是模型拿不准，不补问， +记在 ModelOutcome.blank 里（表的状态不因此变）。目录里已经有中文名或含义的列不算（不论来源，驳回的不算有：以前模型 +起草的留着，照「补充起草要求再重新起草」做一遍，模型这次没再给的老列不该又说成拿不准），库注释已经说清的不算 +（提示词里它们以此为准）；写入时（catalog.write_draft）再按并入目录之后的结果筛一遍。 + +**并发。** 全进程最多 MODEL_CONCURRENCY 个调用同时进行（model_gate：按事件循环各一个信号量，所有起草任务和同步 +起草共用，先来先放）。一次起草里同时只起草 MODEL_CONCURRENCY 段（一段拆小、补缺都做完再起草下一段），同时在等 +名额的调用也最多这么多个：不把几百段一下子都排上去，几次起草（不同数据源的任务、同步起草）的调用交替排队，后开始 +的不会等前一个排着的段全部调完。一次起草里按表的先后排队：宽表的表级那段写完，它的列分段排到已经排着的段后面 +（先来先服务），不插到窄表前面。 + +**进度。** 调用方给 Hooks：开始调用、每次调用的结果（用量、没写出来的原因、当前的提醒和列数上限）、一段结束、一张表 +的模型部分全部结束。后台起草任务据此落库，每张表一结束就并入目录（data/catalog_jobs.py）。 + +交给模型的只有表结构（列、类型、主键、外键、唯一约束、注释）和使用者补充的背景，不发任何数据值。写给模型的文字都在 +catalog.py 名字带 _SYSTEM / _PROMPT 的常量里；这里的 REASON_* 是给人看的（拼进这张表的 model_error）。 +""" +from __future__ import annotations + +import asyncio +import collections +import copy +import json +import logging +import re +import weakref +from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any + +from app.data import catalog as C +from app.data.draft_model import DRAFT_MODEL_PLACE +from app.data.names import name_key + +logger = logging.getLogger(__name__) + +#: 退避等待用的 sleep（测试里换掉，不真等） +_sleep = asyncio.sleep + +#: 服务方嫌额度太大时报错里会出现的字样（同 copilot._BUDGET_WORDS、recipe_ai._BUDGET_WORDS，有测试核对） +BUDGET_WORDS = ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length") +#: 结构化输出这个接口不支持时报错里会出现的字样：json_schema 的 response_format（有的服务只认 json_object）、 +#: 工具调用（有的网关不支持）、Anthropic 的 output_config +STRUCTURED_WORDS = ("response_format", "json_schema", "json_object", "tool", "function", "output_config", + "output_format", "structured output") +#: 一张表的表级的项在「没写出来」记录里的键（不会和列名撞上） +_TABLE = "\0table" + +# ---- 一段没写出来的原因（给人看，拼进这张表的 model_error）。{seg} 在拼成一句时换成「这一段」或「这几段」：同一个 +#: 原因落在几段上（宽表的几个列分段）说「这几段」，同一段拆出来的几块仍是「这一段」 +SEG = "{seg}" +SEG_ONE = "这一段" +SEG_MANY = "这几段" +REASON_EXHAUSTED = "模型的思考和输出用完了额度（{budget} 个 token），{seg}没写出来" +REASON_SPLIT = "；已拆小重试仍没写出" +REASON_HOLLOW = "模型只给了空壳，没有写出可用的内容" +REASON_TRUNCATED = "模型的输出被截断，{seg}没写完" +REASON_FORMAT = "模型的回复不是约定的格式" +REASON_MISSING = "模型的回复里没有这张表" +REASON_UNANSWERED = "模型的回复里漏了这些列，补问一次仍没有给出" +REASON_TIMEOUT = "模型 {seconds} 秒内没有写完{seg}" +REASON_CALL = "调用模型出错：{line}" +REASON_RETRIED = "（已重试 {n} 次）" +REASON_CRASH = "起草这张表时出错：{line}" +#: 一段到了调用次数的上限（MODEL_PART_MAX_CALLS）还没写完 +REASON_CAPPED = "每段最多调用 {n} 次模型，{seg}到了次数仍没写完，已停止重试" +#: 止损（连续几段拆到底仍一项没写出来，后面的段不再拆小重试）时补在原因后面 +REASON_HALTED = "；模型连续 {n} 段都没写出来，已停止拆小重试，建议更换起草模型" +#: 拆到每段 MODEL_MIN_PART_COLUMNS 列仍接连超时（超时和段大小无关，后面超时的不再拆小）时补在原因后面 +REASON_SLOW = "；拆到每段 {n} 列仍然超时，已停止拆小重试，建议更换起草模型" + +# ---- 提醒（最近几次调用多数没写出来），经进度回调交给任务（CatalogDraftJob.notice） +NOTICE = ("最近 {calls} 次调用有 {failed} 次没写出来（{kinds}）。建议在{place}换一个模型、缩小起草范围，" + "或者停止这次起草") +#: 一次调用没写出来的原因种类 → 提醒里的说法（按这个顺序列） +FAILURE_LABEL = {"exhausted": "额度用完", "timeout": "超时", "hollow": "只给空壳", "error": "出错"} + + +# ========================================================================== +# 对外的形状 +# ========================================================================== + + +@dataclass(frozen=True) +class Background: + """使用者补充的背景和要求：guidance 是数据源级的起草要求，note 是这一次的补充说明。数据源说明（description） + 从源上读。都没有就不出现那一段。""" + + guidance: str | None = None + note: str | None = None + + +@dataclass +class ModelOutcome: + """一张表的模型起草结果。 + + status：ok 各段都写出来了；partial 有的段写出来、有的没有（error 写清哪部分、为什么）；failed 一项都没写出来 + （error 写原因）。notes 是写出来的那些（已逐项校验，llm / proposed）；items 是项数。parts_total 分几段起草 + （整表段是 1，宽表是 1 + 列分段数），parts_done 已经结束了几段。blank：模型拿不准、留空了的列（按表结构的顺序； + 表的状态不因此变）。""" + + status: str + notes: dict[str, Any] + items: int + error: str | None + parts_total: int + parts_done: int + blank: list[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class CallReport: + """一次调用的结果（进度回调 call_done 收到的）。 + + failure：没写出来的原因种类，写出来了是 None——exhausted 额度用完（一项都没捡回来）、timeout 超时、hollow 只给空壳 + （回复里没有可用的内容）、error 出错（会重试的那次也算一次）。超时的调用拿不到用量，token 记 0（网关那边可能已经 + 生成了）。notice、part_columns 是这次调用之后的：提醒（没有为 None）、新开始的段每段最多几列。""" + + input_tokens: int + output_tokens: int + failure: str | None + notice: str | None + part_columns: int + + @property + def timeout(self) -> bool: + return self.failure == "timeout" + + +@dataclass +class Hooks: + """进度回调，都是 async，出错只记日志、不影响起草。 + + - call_started(表名列表)：一次调用拿到并发名额、马上要发出去了； + - call_done(CallReport)：每次调用一回（条数就是调用次数），带着用量、没写出来的原因、当前的提醒和列数上限； + - part_done(表名)：这张表的一段结束了（写没写出来都算）； + - table_done(表名, ModelOutcome)：这张表的模型部分全部结束。""" + + call_started: Callable[[list[str]], Awaitable[None]] | None = None + call_done: Callable[[CallReport], Awaitable[None]] | None = None + part_done: Callable[[str], Awaitable[None]] | None = None + table_done: Callable[[str, ModelOutcome], Awaitable[None]] | None = None + + +def parts_for(columns: int) -> int: + """一张有这么多列的表分几段起草：不超过 MODEL_BATCH_COLUMNS 列是 1 段，否则是表级一段加列分段（列数除以 + MODEL_BATCH_COLUMNS 向上取整）。""" + width = max(1, int(C.MODEL_BATCH_COLUMNS)) + return 1 if columns <= width else 1 + -(-columns // width) + + +def _even(columns: Sequence[str], parts: int) -> list[tuple[str, ...]]: + """按顺序切成 parts 段，各段列数平均（相差不超过 1，多出来的放在前面几段）。""" + columns = tuple(columns) + parts = max(1, min(int(parts), len(columns) or 1)) + size, extra = divmod(len(columns), parts) + out: list[tuple[str, ...]] = [] + start = 0 + for i in range(parts): + n = size + (1 if i < extra else 0) + out.append(columns[start:start + n]) + start += n + return out + + +async def run_model_draft(model: Any, source: Any, tables: Sequence[str], *, + existing: Mapping[str, Mapping[str, Any]] | None = None, + background: Background | None = None, + hooks: Hooks | None = None) -> dict[str, ModelOutcome]: + """用模型起草这些表（schema_cache["tables"] 的键；表结构里没有的跳过，不在结果里）。 + + model 要有 with_structured_output(schema, include_raw=…) 和 ainvoke(messages)；有 shrink() 的（catalog.DraftModel） + 在服务方嫌额度太大时换小额度。existing 是 {表名: 目录}：已确认、有确证的项作为「以此为准」的背景交给模型。 + """ + run = _Run(model, source, tables, existing or {}, background or Background(), hooks or Hooks()) + return await run.run() + + +# ========================================================================== +# 并发名额 +# ========================================================================== + +_GATES: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, tuple[int, asyncio.Semaphore]]" = \ + weakref.WeakKeyDictionary() + + +def model_gate() -> asyncio.Semaphore: + """全进程的模型调用名额（MODEL_CONCURRENCY 个）。按事件循环各一个：asyncio 的信号量一旦在某个循环里等过， + 就不能换循环用（测试每个用例一个循环）。""" + loop = asyncio.get_running_loop() + limit = max(1, int(C.MODEL_CONCURRENCY)) + held = _GATES.get(loop) + if held is None or held[0] != limit: + held = (limit, asyncio.Semaphore(limit)) + _GATES[loop] = held + return held[1] + + +# ========================================================================== +# 失败的种类 +# ========================================================================== + + +def _status_of(e: BaseException) -> int | None: + for obj in (e, getattr(e, "response", None)): + code = getattr(obj, "status_code", None) + if isinstance(code, int): + return code + return None + + +def classify_failure(e: BaseException) -> str: + """调用模型抛出来的异常属于哪一种：length 额度用完（LengthFinishReasonError）、budget 服务方嫌额度太大、 + unsupported 结构化输出不支持、timeout 超时、transient 网络错误 / 429 / 5xx、error 别的。""" + from app.core.errors import NETWORK, RATE_LIMIT, SERVER, TIMEOUT, classify + + names = {c.__name__ for c in type(e).__mro__} + if "LengthFinishReasonError" in names: + return "length" + status = _status_of(e) + text = str(e).lower() + if status in (400, 422) and any(w in text for w in BUDGET_WORDS): + return "budget" + # thinking 打开时 Anthropic 不让强制调用工具，langchain 改成不强制，模型没调用工具就抛 OutputParserException + if isinstance(e, NotImplementedError) or "OutputParserException" in names: + return "unsupported" + if status in (400, 422) and any(w in text for w in STRUCTURED_WORDS): + return "unsupported" + if isinstance(e, TimeoutError): + return "timeout" + kind = classify(e) + if kind == TIMEOUT: + return "timeout" + if kind in (RATE_LIMIT, SERVER, NETWORK): + return "transient" + return "error" + + +def _seconds(value: Any) -> Any: + number = float(value) + return int(number) if number.is_integer() else number + + +# ========================================================================== +# 读回复：结构化输出、正文里的 JSON、被截断的 JSON +# ========================================================================== + + +@dataclass +class _Reply: + #: 回复里的表对象（截断的只含已经完整的字段和列） + entries: list[dict[str, Any]] = field(default_factory=list) + #: JSON 没写完（捡回了一部分） + truncated: bool = False + #: 思考加输出用完了额度 + exhausted: bool = False + #: 这一段最终没写出来的原因（重试用完、别的错误、不是约定的格式） + error: str | None = None + #: 结构化输出回了一段说明文字（error 是「不是约定的格式」）：换纯文本还可能写出来 + prose: bool = False + #: 这次调用超时了（当作这一段太大：拆小再问，不原样重试） + timeout: bool = False + #: 这一段的调用次数用完了，这次没有调用 + capped: bool = False + #: 超时拆出来的块，轮到它时已经止损：这次没有调用（见 _Run._ask 的 optional） + skipped: bool = False + #: 这次调用的用量 (输入, 输出)，还没报给进度回调；没有调用（capped）的是 None + usage: tuple[int, int] | None = None + + +@dataclass(frozen=True) +class _Why: + """一部分没写出来的原因。base 给人看的原因（可以带 SEG，拼成一句时换成「这一段」「这几段」）；split 真拆小重试过； + halted 本来还能拆小、因为止损才没拆时止损的说明（REASON_HALTED：前面连续几段都没写出来；REASON_SLOW：拆到每段 + MODEL_MIN_PART_COLUMNS 列仍接连超时），没有是空串。""" + + base: str + split: bool = False + halted: str = "" + + +def _join(failures: Iterable[tuple[_Why, int]]) -> str: + """几处没写出来的原因拼成一句。failures 是 (原因, 属于这张表的第几段)。 + + 同样的原因只说一次（最多说两种）；同一个原因落在几段上说「这几段」，同一段拆出来的几块仍是「这一段」；拆小重试过、 + 止损的说明各在最后补一次（只看说出来的那几种；两种止损各说一次）。不然宽表的表级那段拆不开、列分段拆过时，同一句 + 原因会说两遍。""" + failures = list(failures) + parts: dict[str, set[int]] = {} + for why, part in failures: + parts.setdefault(why.base, set()).add(part) + shown = list(parts)[:2] + told = [w for w, _ in failures if w.base in shown] + text = "；".join(base.replace(SEG, SEG_MANY if len(parts[base]) > 1 else SEG_ONE) for base in shown) + if any(w.split for w in told): + text += REASON_SPLIT + for note in dict.fromkeys(w.halted for w in told if w.halted): + text += note + return text + + +def _columns_text(order: Sequence[str], columns: Iterable[str], *, count: bool) -> str: + """说清是哪几列（列名照表结构原样，按表结构的顺序）：一到三列逐个写；更多的连着就写「头 到 尾」，不连着写前三个加 + 「等」。count：后面带上列数（「，共 33 列」）。""" + pos = {name: i for i, name in enumerate(order)} + cols = sorted(dict.fromkeys(columns), key=lambda c: pos.get(c, len(pos))) + n = len(cols) + if n <= 3: + text = "、".join(cols) + elif pos.get(cols[-1], -1) - pos.get(cols[0], 0) + 1 == n: + text = f"{cols[0]} 到 {cols[-1]}" + else: + head = "、".join(cols[:3]) + return f"{head} 等 {n} 列" if count else f"{head} 等" + return f"{text}，共 {n} 列" if count else text + + +class _Cut: + """被截断的容器（对象或数组）：value 里只有已经完整的成员，最后一个没写完的容器成员也包成 _Cut 留着，好往里捡。""" + + __slots__ = ("value",) + + def __init__(self, value: Any) -> None: + self.value = value + + +class _Bad(ValueError): + """不是 JSON（不是写到一半，是写错了）。""" + + +_NUMBER = re.compile(r"-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?") +_LITERALS = (("true", True), ("false", False), ("null", None)) +_MAX_DEPTH = 64 + + +def _skip(text: str, i: int) -> int: + n = len(text) + while i < n and text[i] in " \t\r\n": + i += 1 + return i + + +def _string(text: str, i: int) -> tuple[Any, int, bool]: + n = len(text) + j = i + 1 + while j < n: + ch = text[j] + if ch == "\\": + j += 2 + continue + if ch == '"': + try: + return json.loads(text[i:j + 1]), j + 1, True + except ValueError as e: + raise _Bad from e + j += 1 + return None, n, False + + +def _value(text: str, i: int, depth: int) -> tuple[Any, int, bool]: + """从 text[i] 起读一个 JSON 值：(值, 读到哪, 写完了没有)。读到结尾还没写完的容器返回 _Cut。""" + if depth > _MAX_DEPTH: + raise _Bad + n = len(text) + i = _skip(text, i) + if i >= n: + return None, i, False + ch = text[i] + if ch == "{": + return _object(text, i + 1, depth + 1) + if ch == "[": + return _array(text, i + 1, depth + 1) + if ch == '"': + return _string(text, i) + for word, value in _LITERALS: + if text.startswith(word, i): + return value, i + len(word), True + if word.startswith(text[i:]): + return None, n, False + m = _NUMBER.match(text, i) + if not m: + raise _Bad + j = m.end() + if j >= n: + return None, j, False # 数字写到了结尾，可能还没写完 + number = m.group() + return (float(number) if any(c in number for c in ".eE") else int(number)), j, True + + +def _object(text: str, i: int, depth: int) -> tuple[Any, int, bool]: + out: dict[str, Any] = {} + n = len(text) + while True: + i = _skip(text, i) + if i >= n: + return _Cut(out), i, False + ch = text[i] + if ch == "}": + return out, i + 1, True + if ch == ",": + i += 1 + continue + if ch != '"': + raise _Bad + key, i, ok = _string(text, i) + if not ok: + return _Cut(out), i, False + i = _skip(text, i) + if i >= n: + return _Cut(out), i, False + if text[i] != ":": + raise _Bad + value, i, ok = _value(text, i + 1, depth) + if not ok: + if isinstance(value, _Cut): + out[key] = value + return _Cut(out), i, False + out[key] = value + + +def _array(text: str, i: int, depth: int) -> tuple[Any, int, bool]: + out: list[Any] = [] + n = len(text) + while True: + i = _skip(text, i) + if i >= n: + return _Cut(out), i, False + ch = text[i] + if ch == "]": + return out, i + 1, True + if ch == ",": + i += 1 + continue + value, i, ok = _value(text, i, depth) + if not ok: + if isinstance(value, _Cut): + out.append(value) + return _Cut(out), i, False + out.append(value) + + +#: 正文里最多试几个 { 或 [ 作为 JSON 的开头（前面的说明文字里也可能有括号） +_MAX_STARTS = 16 + + +def _starts(text: str) -> list[int]: + out: list[int] = [] + for i, ch in enumerate(text): + if ch in "{[": + out.append(i) + if len(out) >= _MAX_STARTS: + break + return out + + +def _entry(item: Any) -> dict[str, Any] | None: + """一个表对象：只留写完了的字段；columns 留写完了的列对象。没有表名的不要。""" + if isinstance(item, _Cut): + item = item.value + if not isinstance(item, dict) or not isinstance(item.get("table"), str): + return None + out: dict[str, Any] = {} + for key, value in item.items(): + if key == "columns": + cols = value.value if isinstance(value, _Cut) else value + if isinstance(cols, list): + out["columns"] = [c for c in cols if isinstance(c, dict)] + elif not isinstance(value, _Cut): + out[key] = value + return out + + +def _tables_of(root: Any) -> list[dict[str, Any]] | None: + """解析出来的根 → 表对象列表。{"tables": […]}、[…]、单个表对象（实测见过不带 tables 外壳的）都认。""" + if isinstance(root, _Cut): + root = root.value + if isinstance(root, dict): + if "tables" in root: + items = root["tables"] + elif isinstance(root.get("table"), str): + items = [root] + elif not root: + return [] + else: + return None + else: + items = root + if isinstance(items, _Cut): + items = items.value + if not isinstance(items, list): + return None + return [e for e in (_entry(item) for item in items) if e is not None] + + +def _parse_text(text: str, cut: bool) -> _Reply | None: + """正文里的表对象。前面可以有一句话、围栏，后面可以有别的字；JSON 没写完的（截断）只收已经写完的部分。 + 认不出约定的格式返回 None。""" + empty: _Reply | None = None + for start in _starts(text): + try: + root, _, complete = _value(text, start, 0) + except (_Bad, RecursionError): + continue + entries = _tables_of(root) + if entries is None: + continue + reply = _Reply(entries=entries) if complete else _Reply(entries=entries, truncated=True, exhausted=cut) + if entries: + return reply + empty = empty or reply # {} 或 {"tables": []}：后面还可能有真的那一份 + return empty + + +def _cut_off(raw: Any) -> bool: + """额度用完停下的：OpenAI 系是 finish_reason=length，Anthropic 是 stop_reason=max_tokens。""" + meta = getattr(raw, "response_metadata", None) or {} + return (meta.get("finish_reason") or meta.get("stop_reason")) in ("length", "max_tokens") + + +def _thought(raw: Any) -> bool: + """这次回复有没有思考：思考内容（Anthropic 的 thinking 块、OpenAI 兼容系的 reasoning_content），或者用量里有推理 token。""" + from app.engine.state import thinking_text + + if raw is None: + return False + if thinking_text(raw): + return True + details = (getattr(raw, "usage_metadata", None) or {}).get("output_token_details") or {} + return bool(details.get("reasoning")) + + +def _text_of(raw: Any) -> str: + from app.engine.state import message_text + + return message_text(raw) if raw is not None else "" + + +def _raw_texts(raw: Any) -> list[str]: + """结构化输出没解析出来时，原始回复里可能还有 JSON 的地方：工具调用的参数（截断的在 invalid_tool_calls 里，是一段 + 字符串）、正文。""" + out: list[str] = [] + for call in getattr(raw, "tool_calls", None) or []: + args = call.get("args") if isinstance(call, dict) else None + if isinstance(args, dict) and args: + out.append(json.dumps(args, ensure_ascii=False)) + for call in getattr(raw, "invalid_tool_calls", None) or []: + args = call.get("args") if isinstance(call, dict) else None + if isinstance(args, str) and args.strip(): + out.append(args) + text = _text_of(raw) + if text.strip(): + out.append(text) + return out + + +def _entries_of(parsed: Any) -> list[dict[str, Any]] | None: + if parsed is None: + return None + if isinstance(parsed, dict) and not parsed: + return [] + return C._model_tables(parsed) + + +def _structured_reply(raw: Any, parsed: Any) -> _Reply: + entries = _entries_of(parsed) + if entries is not None: + return _Reply(entries=entries) + cut = _cut_off(raw) + texts = _raw_texts(raw) + for text in texts: + reply = _parse_text(text, cut) + if reply is not None: + return reply + if cut or (not texts and _thought(raw)): + return _Reply(exhausted=True) # 停在额度上，或者正文为空而有思考（同 _text_reply） + if not texts: + return _Reply() # 什么都没写：按空壳处理 + # 写了字却不是约定的格式、也没停在额度上：网关接受了结构化输出的参数却没照做，模型写成了说明文字。有没有思考都 + # 一样，不是额度的问题；换纯文本（另带格式说明）还能写出来 + return _Reply(error=REASON_FORMAT, prose=True) + + +def _text_reply(raw: Any) -> _Reply: + text = _text_of(raw) + cut = _cut_off(raw) + if text.strip(): + reply = _parse_text(text, cut) + if reply is not None: + return reply + if cut: + return _Reply(exhausted=True) # 写了字、额度就用完了，一个 JSON 对象都没捡回来 + return _Reply(error=REASON_FORMAT) + return _Reply(exhausted=cut or _thought(raw)) + + +def _usage_of(raw: Any) -> tuple[int, int]: + """一次调用的用量 (输入, 输出)：拿不到的是 0（测试的假模型没有原始回复）。""" + meta = getattr(raw, "usage_metadata", None) or {} + return int(meta.get("input_tokens") or 0), int(meta.get("output_tokens") or 0) + + +def _unpack(out: Any) -> tuple[Any, Any]: + """with_structured_output(include_raw=True) 交回 {raw, parsed, parsing_error}；测试的假模型直接交回解析好的对象。""" + if isinstance(out, dict) and "raw" in out and ("parsed" in out or "parsing_error" in out): + return out.get("raw"), out.get("parsed") + return None, out + + +# ========================================================================== +# 一次起草 +# ========================================================================== + + +@dataclass(frozen=True) +class _Piece: + """一段里的一张表：要写哪些列、要不要表级的项、属于这张表的第几段（从 1 起）。""" + + table: str + columns: tuple[str, ...] + table_level: bool + part: int + + +@dataclass +class _Gain: + """一次回复里某张表拿到了什么。""" + + named: bool = False + items: int = 0 + table_items: int = 0 + answered: set[str] = field(default_factory=set) + + +@dataclass +class _Table: + name: str + meta: Mapping[str, Any] + columns: list[str] + existing: Mapping[str, Any] | None + parts: list[_Piece] = field(default_factory=list) + #: 已经拿到的模型起草（逐段合并） + notes: dict[str, Any] = field(default_factory=dict) + #: 模型的回复里出现过这张表 + named: bool = False + got_table: bool = False + got_columns: set[str] = field(default_factory=set) + #: 没写出来的列（表级的项记在 _TABLE）→ 原因 + failed: dict[str, _Why] = field(default_factory=dict) + parts_done: int = 0 + finished: bool = False + + def whole(self, piece: _Piece) -> bool: + return piece.table_level and len(piece.columns) == len(self.columns) + + def has_any(self) -> bool: + """这张表已经拿到了模型起草的内容（表级的项或任何一列）。""" + return self.got_table or bool(self.got_columns) + + def wrote(self, piece: _Piece) -> bool: + """这一段请写的部分拿到了内容。""" + return (piece.table_level and self.got_table) or any(c in self.got_columns for c in piece.columns) + + def part_of(self) -> dict[str, int]: + """每一列（表级的项是 _TABLE）属于这张表的第几段。""" + out = {col: piece.part for piece in self.parts for col in piece.columns} + out[_TABLE] = 1 + return out + + +def _restrict(notes: Mapping[str, Any], piece: _Piece) -> dict[str, Any]: + """只留这一段请模型写的部分：没请写表级的项就不要表级的项，列只要请写的那几列。""" + out: dict[str, Any] = {} + if piece.table_level: + out.update({k: v for k, v in notes.items() if k != "columns"}) + wanted = set(piece.columns) + cols = {c: items for c, items in (notes.get("columns") or {}).items() if c in wanted} + if cols: + out["columns"] = cols + return out + + +@dataclass +class _Work: + """排队起草的一段（顶层的段：窄表拼成的整表段、宽表的表级那段或一个列分段）。then：这一段写完才能起草的段（宽表 + 的表级那段写完，才轮到它的列分段）。""" + + pieces: list[_Piece] + then: list[_Work] = field(default_factory=list) + + +@dataclass +class _Budget: + """一段（新开始的一段，连同它拆出来的、补问的、重试的）还能调用几次模型。""" + + left: int + + +def _pack(pieces: list[_Piece], cap: int) -> list[list[_Piece]]: + """多表的段按顺序重新拼成每段不超过 cap 列（超过 cap 的表自成一段）。""" + groups: list[list[_Piece]] = [] + used = 0 + for piece in pieces: + n = len(piece.columns) + if not groups or used + n > cap: + groups.append([]) + used = 0 + groups[-1].append(piece) + used += n + return groups + + +def _halves(pieces: list[_Piece]) -> tuple[list[_Piece], list[_Piece]] | None: + """拆成两半：多表的段按表拆；单表的按列拆，两半都至少 MODEL_SPLIT_MIN_COLUMNS 列（表级的项跟着前一半）。拆不开返回 None。""" + if len(pieces) > 1: + k = (len(pieces) + 1) // 2 + return pieces[:k], pieces[k:] + piece = pieces[0] + n = len(piece.columns) + if n < 2 * max(1, int(C.MODEL_SPLIT_MIN_COLUMNS)): + return None + h = (n + 1) // 2 + return ([_Piece(piece.table, piece.columns[:h], piece.table_level, piece.part)], + [_Piece(piece.table, piece.columns[h:], False, piece.part)]) + + +class _Run: + def __init__(self, model: Any, source: Any, tables: Sequence[str], existing: Mapping[str, Mapping[str, Any]], + background: Background, hooks: Hooks) -> None: + self.model = model + self.hooks = hooks + #: structured 结构化输出；text 纯文本、从正文取 JSON。结构化输出用不了时这次起草后面都改走纯文本 + self.mode = "structured" + self.system = C.system_notes_source(source) + all_tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} + self.order = [t for t in dict.fromkeys(str(t) for t in tables) if isinstance(all_tables.get(t), dict)] + self.tables = { + name: _Table(name=name, meta=all_tables[name], existing=existing.get(name), + columns=[str(c["name"]) for c in all_tables[name].get("columns") or [] + if isinstance(c, dict) and c.get("name") is not None]) + for name in self.order + } + self.background = self._background(getattr(source, "description", None), background) + #: 这次起草同时在等名额或正在进行的调用最多几个（全进程的名额另算，见 model_gate） + self.gate = asyncio.Semaphore(max(1, int(C.MODEL_CONCURRENCY))) + #: 连续几段拆到底仍一项没写出来（止损用，有一段写出来就清零） + self.dead_streak = 0 + #: 拆到每段 MODEL_MIN_PART_COLUMNS 列（或更小）的调用接连超时了几次（有一次调用写出来就清零）：超时和段大小 + #: 无关，再拆小也是一次一个时限，后面超时的不再拆小 + self.slow_streak = 0 + #: 新开始的段每段最多几列：额度用完或超时（strikes）累计 MODEL_SHRINK_AFTER 次减半一次 + self.part_columns = max(1, int(C.MODEL_BATCH_COLUMNS)) + self.strikes = 0 + #: 最近几次调用没写出来的原因种类（写出来了是 None），提醒看它 + self.recent: collections.deque[str | None] = collections.deque(maxlen=max(1, int(C.MODEL_NOTICE_WINDOW))) + self.notice: str | None = None + + def _halt_note(self, kind: str | None = None) -> str: + """止损的说明（拼在原因后面），没止损是空串。 + + - 连续 MODEL_GIVE_UP_AFTER 段一项都没写出来：后面的段不再拆小重试（不论原因）； + - 超时的（kind 是 timeout）另看：拆到每段 MODEL_MIN_PART_COLUMNS 列的调用接连超时 MODEL_GIVE_UP_AFTER 次， + 超时就和段大小无关了，再拆小也是一次一个时限，后面超时的不再拆小，超时拆出来还没问的块也不再问。只管超时： + 额度用完、只给空壳照旧拆小（段小了思考和输出才放得下）。""" + give_up = max(1, int(C.MODEL_GIVE_UP_AFTER)) + if self.dead_streak >= give_up: + return REASON_HALTED.format(n=give_up) + if kind == "timeout" and self.slow_streak >= give_up: + return REASON_SLOW.format(n=self._floor()) + return "" + + @staticmethod + def _floor() -> int: + """段大小减半的下限（每段最少几列）。""" + return min(max(1, int(C.MODEL_BATCH_COLUMNS)), max(1, int(C.MODEL_MIN_PART_COLUMNS))) + + # ---------------------------------------------------------------- 总流程 + + async def run(self) -> dict[str, ModelOutcome]: + """按计划的先后起草：同时只起草 MODEL_CONCURRENCY 段，一段（连同拆小、补缺）做完再取下一段；宽表的表级那段 + 写完，它的列分段排到队尾（先来先服务：排在已经排着的窄表段后面，不插队）。不一下子把所有段（包括宽表的全部列 + 分段）都排上去：那样后开始的另一次起草要等这次排着的段全部调完；拆小重试也排在所有段的第一次调用后面，一段要 + 等几乎所有调用都做完才结束，止损形同虚设。""" + queue: collections.deque[_Work] = collections.deque() + for unit in self._plan(): + if isinstance(unit, list): + queue.append(_Work([self.tables[n].parts[0] for n in unit])) + else: + head, *rest = self.tables[unit].parts + # 表级那段先写：列分段拿它做背景 + queue.append(_Work([head], [_Work([p]) for p in rest])) + left = {name: len(st.parts) for name, st in self.tables.items()} + busy = 0 + changed = asyncio.Event() + + async def worker() -> None: + nonlocal busy + while True: + if queue: + work = queue.popleft() + busy += 1 + try: + await self._work(work, queue, left) + finally: + busy -= 1 + changed.set() + continue + if not busy: + return + changed.clear() # 队列空着、别的段还在起草（宽表的表级那段写完会排上列分段）：等它们 + await changed.wait() + + await asyncio.gather(*(worker() for _ in range(max(1, int(C.MODEL_CONCURRENCY))))) + return {name: self._outcome(t) for name, t in self.tables.items()} + + def _plan(self) -> list[list[str] | str]: + """分段：窄表按顺序拼成整表段（列表），宽表一个一项（表名），列分段均分（_even）。各项按第一张表在 tables 里的 + 位置排。""" + width = max(1, int(C.MODEL_BATCH_COLUMNS)) + most = max(1, int(C.MODEL_BATCH_TABLES)) + units: list[list[str] | str] = [] + pack: list[str] | None = None + used = 0 + for name in self.order: + st = self.tables[name] + n = len(st.columns) + if n > width: + st.parts = [_Piece(name, (), True, 1)] + [ + _Piece(name, cols, False, k + 2) for k, cols in enumerate(_even(st.columns, -(-n // width)))] + units.append(name) + continue + st.parts = [_Piece(name, tuple(st.columns), True, 1)] + if pack is None or len(pack) >= most or used + n > width: + pack, used = [], 0 + units.append(pack) + pack.append(name) + used += n + return units + + async def _work(self, work: _Work, queue: collections.deque[_Work], left: dict[str, int]) -> None: + """起草一段；它之后的段（宽表的列分段）排到队尾；一张表的段都结束了，这张表的模型部分就结束了。 + + 这一段（原始段：计划里的整表段、表级那段或一个列分段）连同按缩小后的上限分开的各块、拆出来的、补问的、重试的， + 一共最多调用 MODEL_PART_MAX_CALLS 次（各块共用一个 _Budget）；止损按原始段数（_tally）。""" + await self._start(work.pieces, _Budget(max(1, int(C.MODEL_PART_MAX_CALLS)))) + self._tally(work.pieces) + for piece in work.pieces: + await self._part_done(piece.table) + queue.extend(work.then) + for piece in work.pieces: + left[piece.table] -= 1 + if not left[piece.table]: + await self._table_done(piece.table) + + async def _start(self, pieces: list[_Piece], budget: _Budget) -> None: + """开始起草一段。超过当前的列数上限（额度用完、超时多了会减半）的先按上限分开：多表的段按表重新拼，单张表的 + 按列均分（表级的项跟着第一块，先写它，后面几块拿它做背景）。分开的各块同时起草，共用这一段的调用次数（budget）： + 以前每块各有 MODEL_PART_MAX_CALLS 次，上限缩到 10 列以后一段 40 列最多能调用 32 次。""" + cap = max(1, self.part_columns) + if sum(len(p.columns) for p in pieces) <= cap: + await self._guarded(pieces, budget) + return + if len(pieces) > 1: + await asyncio.gather(*(self._start(group, budget) for group in _pack(pieces, cap))) + return + piece = pieces[0] + chunks = _even(piece.columns, -(-len(piece.columns) // cap)) + first = [_Piece(piece.table, chunks[0], piece.table_level, piece.part)] + rest = [[_Piece(piece.table, cols, False, piece.part)] for cols in chunks[1:]] + if piece.table_level: + await self._guarded(first, budget) + await asyncio.gather(*(self._guarded(group, budget) for group in rest)) + else: + await asyncio.gather(*(self._guarded(group, budget) for group in [first, *rest])) + + async def _guarded(self, pieces: list[_Piece], budget: _Budget) -> None: + """起草一段（或它按上限分开的一块）；意料之外的错误只记这一块没写出来，不影响别的段和别的表。""" + try: + await self._segment(pieces, 0, False, budget) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 + from app.core.errors import describe_exception, first_line + + logger.exception("起草数据目录时出错：%s", "、".join(p.table for p in pieces)) + self._fail(pieces, REASON_CRASH.format(line=first_line(e) or describe_exception(e))) + + def _tally(self, pieces: list[_Piece]) -> None: + """一段（原始段，连同它分开的各块、拆小、补缺）结束：一项都没写出来的接着数，写出来了清零。""" + if any(self.tables[p.table].wrote(p) for p in pieces): + self.dead_streak = 0 + return + self.dead_streak += 1 + if self.dead_streak == max(1, int(C.MODEL_GIVE_UP_AFTER)): + logger.warning("起草数据目录：模型连续 %d 段都没写出来，这次起草后面的段不再拆小重试", self.dead_streak) + + # ---------------------------------------------------------------- 一段 + + async def _segment(self, pieces: list[_Piece], depth: int, followup: bool, budget: _Budget, *, + halved: bool = False, mode: str | None = None, last: bool = False, + optional: bool = False) -> None: + """问一段，拿到的并进各表；按失败原因拆开、补缺、记下没写出来的。 + + depth：已经拆了几层（截断后只补剩下的也算一层）。followup：这一段是在补缺（补过就不再补，见 _after_followup）。 + budget：这一段（原始段）还能调用几次。halved：这几块是真拆小（_split）出来的——原因里「已拆小重试」只在这时写。 + mode / last：换纯文本问的那一次——拆不开的空壳、结构化输出回了说明文字（再不行就记没写出来）。 + optional：超时拆出来的块，轮到它时已经止损就不问了（_ask）。""" + reply = await self._ask(pieces, budget, mode, optional=optional) + if reply.capped: + self._fail(pieces, self._capped(halved)) + return + if reply.skipped: + note = self._halt_note("timeout") + self._fail(pieces, lambda p: self._reason("timeout", split=True, piece=p, halted=note)) + return + if reply.timeout: + # 先记下再报用量：报用量要写库、会让出事件循环，等名额的块醒来时要看到这一次 + if sum(len(p.columns) for p in pieces) <= self._floor(): + self.slow_streak += 1 + if self.slow_streak == max(1, int(C.MODEL_GIVE_UP_AFTER)): + logger.warning("起草数据目录：拆到每段 %d 列仍接连超时 %d 次，后面超时的不再拆小", + self._floor(), self.slow_streak) + await self._report(reply, wrote=False) + await self._split(pieces, depth, "timeout", followup, budget, halved) + return + if reply.error is not None: + await self._report(reply, wrote=False) + if reply.prose and not last: + await self._prose_to_text(pieces, depth, followup, budget, halved) + return + self._fail(pieces, _Why(reply.error, split=halved)) + return + try: + gains = self._take(pieces, reply.entries) + except Exception: + # 读回复时出了意料之外的错（_guarded 记这一块没写出来）：这次调用照样记进用量，算一次出错 + await self._called(reply.usage or (0, 0), "error") + raise + anything = any(g.items for g in gains) + complete = not (reply.exhausted or reply.truncated) + rest = self._remainder(pieces, gains, truncated=False, followup=followup) if complete else [] + # 补问时请写的都列出来了、字段留空：是模型拿不准（记留空），这次调用不算没写出来（不进提醒的窗口） + await self._report(reply, wrote=anything or (followup and complete and not rest)) + if last and anything and self.mode != "text": + # 结构化输出只给空壳、纯文本写出来了：这个模型的结构化输出靠不住，这次起草后面都走纯文本 + self.mode = "text" + logger.info("起草数据目录：结构化输出只给了空壳，纯文本写出来了，后面改走纯文本") + if not complete: + rest = self._remainder(pieces, gains, truncated=True, followup=followup) + if not rest: + return + kind = "exhausted" if reply.exhausted else "truncated" + if not anything: + await self._split(rest, depth, kind, followup, budget, halved) + elif depth < int(C.MODEL_SPLIT_DEPTH): + # 捡回了一部分：只补剩下的（不是拆小） + await self._segment(rest, depth + 1, followup, budget, halved=halved) + else: + self._fail(rest, self._reason(kind, split=halved)) + return + if not anything and not followup: + if last: + halted = self._stopped(pieces, depth) + self._fail(pieces, lambda p: self._reason("hollow", split=halved, piece=p, halted=halted)) + return + await self._split(pieces, depth, "hollow", followup, budget, halved) + return + if not rest: + return + if followup: + await self._after_followup(rest, depth, budget, halved, last=last) + return + await self._segment(rest, depth, True, budget, halved=halved) + + async def _after_followup(self, rest: list[_Piece], depth: int, budget: _Budget, halved: bool, *, + last: bool) -> None: + """补问过仍然没给的部分（补过就不再补）。 + + - 这张表已经拿到了内容：补问时仍没列出来的列记没写出来（这张表是 partial）。补问时列出了、字段留空的列是拿不准 + （记留空），不在这里（_remainder 只把没列出来的列交过来）； + - 这张表到现在还一项都没有（回复里没有这张表、只给了空壳）：照第一次只给空壳的规矩，拆小再问，拆不开再问一次。 + 拆出来的半段是新的一段（还能补缺一次），层数照算。""" + given = [p for p in rest if self.tables[p.table].has_any()] + empty = [p for p in rest if not self.tables[p.table].has_any()] + if given: + self._fail(given, _Why(REASON_UNANSWERED, split=halved)) + if not empty: + return + if last: + halted = self._stopped(empty, depth) + self._fail(empty, lambda p: self._reason("hollow", split=halved, piece=p, halted=halted)) + return + await self._split(empty, depth, "hollow", False, budget, halved) + + async def _prose_to_text(self, pieces: list[_Piece], depth: int, followup: bool, budget: _Budget, + halved: bool) -> None: + """结构化输出回了一段说明文字：网关接受了结构化输出的参数却没照做。这次起草后面都走纯文本（另带格式说明）， + 这一段换纯文本问一次。纯文本额度用完了还能照常拆小（拆出来的半段也走纯文本，不再先问一遍结构化输出）。""" + if self.mode != "text": + self.mode = "text" + logger.info("起草数据目录：结构化输出回了说明文字，后面改走纯文本") + await self._segment(pieces, depth, followup, budget, halved=halved, mode="text", last=True) + + def _stopped(self, pieces: list[_Piece], depth: int) -> str: + """这几段本来还能拆小、因为止损才没拆时，止损的说明；不是的话是空串。""" + if depth < int(C.MODEL_SPLIT_DEPTH) and _halves(pieces) is not None: + return self._halt_note() + return "" + + async def _split(self, pieces: list[_Piece], depth: int, kind: str, followup: bool, budget: _Budget, + halved: bool) -> None: + """拆成两半再问（额度用完、超时、只给空壳、截断后一项都没捡回来）。 + + - 同一张表按列拆、前一半带着表级的项：先写前一半，后一半拿它做背景。超时的不等：一次超时就是一个时限，40 → 20 + → 10 一层一层串着问要好几个时限，两半同时问；拆出来的块轮到它时已经止损的不再问（optional）。 + - 拆不开的空壳再问一次：结构化输出给的换纯文本，已经走纯文本的原样再问。止损时本来能拆的不拆：结构化输出给的 + 照旧换纯文本问一次（看看是不是结构化输出的问题），已经走纯文本的不再问。 + - 别的原因拆不开、止损了，就记没写出来。""" + halves = _halves(pieces) if depth < int(C.MODEL_SPLIT_DEPTH) else None + halted = self._halt_note(kind) if halves is not None else "" + if halves is not None and not halted: + first, second = halves + timeout = kind == "timeout" + if (not timeout and len(first) == len(second) == 1 and first[0].table == second[0].table + and first[0].table_level): + await self._segment(first, depth + 1, followup, budget, halved=True) + await self._segment(second, depth + 1, followup, budget, halved=True) + return + results = await asyncio.gather( + self._segment(first, depth + 1, followup, budget, halved=True, optional=timeout), + self._segment(second, depth + 1, followup, budget, halved=True, optional=timeout), + return_exceptions=True) + for result in results: + if isinstance(result, BaseException): + raise result + return + if kind == "hollow" and (not halted or self.mode == "structured"): + await self._segment(pieces, depth, followup, budget, halved=halved, mode="text", last=True) + return + self._fail(pieces, lambda p: self._reason(kind, split=halved, piece=p, halted=halted)) + + def _remainder(self, pieces: list[_Piece], gains: list[_Gain], *, truncated: bool, + followup: bool) -> list[_Piece]: + """还没写出来的部分：回复里没有这张表、或者这张表一项可用内容都没有的整段重问；有内容的只问没列出来的列 + （列出了、字段留空的算模型拿不准，不再问）。截断的回复里表级的项也没拿到的，表级的项一起重问（没截断时表级 + 空着算模型拿不准：这张表在回复里，表级的字段留空和列的字段留空一样）。 + + 补问（followup）时这张表已经有内容了：补问的列列出来了就算答过（字段留空是拿不准），只交回没列出来的列。""" + out: list[_Piece] = [] + for piece, gain in zip(pieces, gains): + answered_blank = followup and gain.named and self.tables[piece.table].has_any() + if not gain.named or (not gain.items and not answered_blank): + out.append(piece) + continue + cols = tuple(c for c in piece.columns if c not in gain.answered) + table_level = piece.table_level and truncated and not gain.table_items + if cols or table_level: + out.append(_Piece(piece.table, cols, table_level, piece.part)) + return out + + # ---------------------------------------------------------------- 调用 + + async def _ask(self, pieces: list[_Piece], budget: _Budget, mode: str | None = None, *, + optional: bool = False) -> _Reply: + """调一次模型（按原因重来：额度被拒、结构化输出不支持、网络错误……），交回读好的回复。 + + 这一段的调用次数用完了交回 capped（不调用）。optional（超时拆出来的块）：拿到名额时已经止损，交回 skipped（不调用， + 次数退回）——等名额的时候前面的块接连超时，再问也是一个时限。超时不原样重试：交回 timeout，由调用方当作这一段 + 太大、拆小再问。重来之前的那次调用在这里就报给进度回调；交回的那次由调用方读完内容再报（写没写出来要看校验之后 + 的内容）。""" + mode = mode or self.mode + retried = 0 + while True: + if budget.left <= 0: + return _Reply(capped=True) + budget.left -= 1 # 先占上：各块共用这一段的次数，等名额的时候别的块不能把它用掉 + used = getattr(self.model, "budget", None) + try: + # 先占这次起草自己的名额，再去排全进程的：一次起草排在全进程名额上的调用最多 MODEL_CONCURRENCY 个 + async with self.gate, model_gate(): + if optional and self._halt_note("timeout"): + budget.left += 1 + return _Reply(skipped=True) + await self._hook(self.hooks.call_started, [p.table for p in pieces]) + reply, raw = await asyncio.wait_for(self._call(pieces, mode), timeout=C.MODEL_TIMEOUT_S) + reply.usage = _usage_of(raw) + return reply + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 - 按原因分别处理 + kind = classify_failure(e) + if kind == "length": + return self._length_reply(e) + if kind == "timeout": + return _Reply(timeout=True, usage=(0, 0)) + if kind == "budget" and (getattr(self.model, "budget", None) != used or self._shrink()): + await self._called((0, 0), "error") + continue + if kind == "unsupported" and mode == "structured": + if self.mode == "structured": + logger.info("起草数据目录：结构化输出用不了（%s），后面改走纯文本", str(e)[:200]) + self.mode = mode = "text" + await self._called((0, 0), "error") + continue + if kind == "transient" and retried < int(C.MODEL_RETRIES): + await self._called((0, 0), "error") + delays = tuple(C.MODEL_RETRY_DELAYS_S) or (0.0,) + await _sleep(delays[min(retried, len(delays) - 1)]) + retried += 1 + continue + return _Reply(error=self._call_error(e, retried), usage=(0, 0)) + + async def _call(self, pieces: list[_Piece], mode: str) -> tuple[_Reply, Any]: + """一次调用：(读好的回复, 原始回复)。原始回复给用量用（测试的假模型没有，是 None）。""" + messages = self._messages(pieces, mode) + if mode == "structured": + runnable = self.model.with_structured_output(C.CATALOG_DRAFT_PROMPT_SCHEMA, include_raw=True) + raw, parsed = _unpack(await runnable.ainvoke(messages)) + return _structured_reply(raw, parsed), raw + raw = await self.model.ainvoke(messages) + return _text_reply(raw), raw + + @staticmethod + def _length_reply(e: BaseException) -> _Reply: + """LengthFinishReasonError：额度用完了。异常里带着那次的回复（和用量），能捡回来的照捡。""" + completion = getattr(e, "completion", None) + usage = getattr(completion, "usage", None) + tokens = (int(getattr(usage, "prompt_tokens", 0) or 0), int(getattr(usage, "completion_tokens", 0) or 0)) + try: + text = completion.choices[0].message.content or "" # type: ignore[union-attr] + except (AttributeError, IndexError, TypeError): + text = "" + if text.strip(): + reply = _parse_text(text, True) + if reply is not None and reply.entries: + return _Reply(entries=reply.entries, truncated=True, exhausted=True, usage=tokens) + return _Reply(exhausted=True, usage=tokens) + + def _shrink(self) -> bool: + shrink = getattr(self.model, "shrink", None) + if callable(shrink) and shrink(): + logger.info("起草数据目录：服务方不接受这么大的输出额度，这次起草改用 %s", getattr(self.model, "budget", "")) + return True + return False + + def _budget(self) -> int: + return int(getattr(self.model, "budget", None) or C.MODEL_MAX_TOKENS) + + @staticmethod + def _call_error(e: BaseException, retried: int) -> str: + from app.core.errors import describe_exception, first_line + + tail = REASON_RETRIED.format(n=retried) if retried else "" + return REASON_CALL.format(line=first_line(e) or describe_exception(e)) + tail + + # ---------------------------------------------------------------- 每次调用之后：用量、段大小、提醒 + + async def _report(self, reply: _Reply, *, wrote: bool) -> None: + """交回的那次调用读完内容之后报给进度回调。""" + if reply.usage is None: + return + if reply.timeout: + failure: str | None = "timeout" + elif reply.error is not None: + failure = "error" + elif wrote: + failure = None + else: + failure = "exhausted" if reply.exhausted else "hollow" + await self._called(reply.usage, failure, strike=reply.timeout or reply.exhausted) + + async def _called(self, usage: tuple[int, int], failure: str | None, *, strike: bool = False) -> None: + """记下一次调用：额度用完或超时（strike）累计 MODEL_SHRINK_AFTER 次，新开始的段列数上限减半；最近几次调用多数 + 没写出来时提醒，少了清掉。写出来了（failure 为 None）就清掉接连超时的计数。报给进度回调。""" + self.recent.append(failure) + if failure is None: + self.slow_streak = 0 + if strike: + self.strikes += 1 + width = max(1, int(C.MODEL_BATCH_COLUMNS)) + cap = max(self._floor(), width >> (self.strikes // max(1, int(C.MODEL_SHRINK_AFTER)))) + if cap < self.part_columns: + self.part_columns = cap + logger.info("起草数据目录：额度用完或超时已有 %d 次，之后新开始的段每段最多 %d 列", self.strikes, cap) + failed = [f for f in self.recent if f is not None] + if len(failed) >= int(C.MODEL_NOTICE_ON) or (self.notice is not None and len(failed) >= int(C.MODEL_NOTICE_OFF)): + counts = collections.Counter(failed) + kinds = "、".join(f"{label} {counts[kind]} 次" for kind, label in FAILURE_LABEL.items() if counts[kind]) + if self.notice is None: + logger.warning("起草数据目录：最近 %d 次调用有 %d 次没写出来（%s）", len(self.recent), len(failed), kinds) + self.notice = NOTICE.format(calls=len(self.recent), failed=len(failed), kinds=kinds, place=DRAFT_MODEL_PLACE) + else: + self.notice = None + await self._hook(self.hooks.call_done, CallReport(int(usage[0]), int(usage[1]), failure, self.notice, + self.part_columns)) + + # ---------------------------------------------------------------- 拿到的内容 + + def _match(self, pieces: list[_Piece], entries: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]: + """回复里的表对象按表名对到这一段的表上。表名照原样写的直接对上；带了 schema 前缀、大小写不一致的，只在恰好 + 对上一张时认。""" + names = [p.table for p in pieces] + out: dict[str, list[dict[str, Any]]] = {} + for entry in entries: + raw = entry.get("table") + if not isinstance(raw, str): + continue + name = raw if raw in names else None + if name is None: + short = raw.strip().rsplit(".", 1)[-1].strip('"`[] ') + found = [n for n in names if n == short or name_key(n) == name_key(short)] + name = found[0] if len(found) == 1 else None + if name is not None: + out.setdefault(name, []).append(entry) + return out + + def _take(self, pieces: list[_Piece], entries: list[dict[str, Any]]) -> list[_Gain]: + """回复里能用的并进各表（逐项校验，只留这一段请写的部分），返回每张表这次拿到了什么。""" + by_table = self._match(pieces, entries) + gains: list[_Gain] = [] + for piece in pieces: + st = self.tables[piece.table] + gain = _Gain() + for entry in by_table.get(piece.table, []): + gain.named = st.named = True + notes = _restrict(C._from_model(entry, st.meta, system=self.system), piece) + cols = entry.get("columns") if isinstance(entry.get("columns"), list) else [] + gain.answered |= {c.get("name") for c in cols if isinstance(c, dict) and c.get("name") in piece.columns} + slots = C._slots(notes) + gain.items += len(slots) + gain.table_items += sum(1 for key in slots if key[0] == "t") + self._merge(st, notes) + gains.append(gain) + return gains + + @staticmethod + def _merge(st: _Table, notes: Mapping[str, Any]) -> None: + for key, value in notes.items(): + if key == "columns": + cols = st.notes.setdefault("columns", {}) + for col, items in value.items(): + target = cols.setdefault(col, {}) + for name, item in items.items(): + target.setdefault(name, item) + st.got_columns.add(col) + else: + st.notes.setdefault(key, value) + st.got_table = True + + def _fail(self, pieces: list[_Piece], reason: _Why | str | Callable[[_Piece], _Why]) -> None: + """这几段没写出来：还没拿到内容的列（和表级的项）记下原因。""" + for piece in pieces: + st = self.tables[piece.table] + why = reason(piece) if callable(reason) else reason + if isinstance(why, str): + why = _Why(why) + for col in piece.columns: + if col not in st.got_columns: + st.failed.setdefault(col, why) + if piece.table_level and not st.got_table: + st.failed.setdefault(_TABLE, why) + + def _reason(self, kind: str, *, split: bool, piece: _Piece | None = None, halted: str = "") -> _Why: + if kind == "exhausted": + base = REASON_EXHAUSTED.format(budget=self._budget(), seg=SEG) + elif kind == "truncated": + base = REASON_TRUNCATED.format(seg=SEG) + elif kind == "timeout": + base = REASON_TIMEOUT.format(seconds=_seconds(C.MODEL_TIMEOUT_S), seg=SEG) + else: + st = self.tables[piece.table] if piece is not None else None + if st is not None and st.whole(piece): + base = C.MODEL_EMPTY_REASON if st.named else REASON_MISSING + else: + base = REASON_HOLLOW + return _Why(base, split=split, halted=halted) + + @staticmethod + def _capped(halved: bool) -> _Why: + return _Why(REASON_CAPPED.format(n=max(1, int(C.MODEL_PART_MAX_CALLS)), seg=SEG), split=halved) + + # ---------------------------------------------------------------- 收尾 + + def _outcome(self, st: _Table) -> ModelOutcome: + items = len(C._slots(st.notes)) + failed_cols = [c for c in st.columns if c in st.failed and c not in st.got_columns] + failed_table = _TABLE in st.failed and not st.got_table + total = len(st.parts) or 1 + part_of = st.part_of() + if not items: + failures = [(why, part_of.get(key, 1)) for key, why in st.failed.items()] + reason = _join(failures) if failures else (C.MODEL_EMPTY_REASON if st.named else REASON_MISSING) + if failures and len(st.parts) > 1: + # 宽表一项都没写出来：同样逐段写上列的范围 + said = self._parts_said(st, failed_cols, failed_table) + every = "都" if len(said) == len(st.parts) else "" + reason = f"这张表 {len(st.columns)} 列，分 {len(st.parts)} 段起草，{'、'.join(said)}{every}没写出来：{reason}" + return ModelOutcome("failed", {}, 0, reason, total, st.parts_done) + notes = copy.deepcopy(st.notes) + blank = self._blank(st, set(failed_cols)) + if failed_cols or failed_table: + return ModelOutcome("partial", notes, items, self._partial_text(st, failed_cols, failed_table), total, + st.parts_done, blank) + return ModelOutcome("ok", notes, items, None, total, st.parts_done, blank) + + def _blank(self, st: _Table, failed: set[str]) -> list[str]: + """模型拿不准、留空了的列：请它写了（不在没写出来的里面），中文名和含义却都没给（或者给的都没通过校验）。 + + 目录里已经有中文名或含义的不算（不论来源：以前模型起草的——重新起草时模型这次没给的留着、不删；人工填写、确认的； + 库注释；驳回的不算有）：照「补充起草要求再重新起草」做一遍，模型这次没再给的老列不该又说成拿不准。库注释已经说清 + 的（catalog.comment_role 记进目录的那种；导入表格的源是系统说明）同样不算：提示词里它们以此为准，模型不写也不缺。 + 写入时（catalog.write_draft）再按并入目录之后的结果筛一遍。""" + drafted = st.notes.get("columns") or {} + known = C.visible_notes(st.existing).get("columns") or {} + comments = {str(c.get("name")): (c.get("comment") or "").strip() + for c in st.meta.get("columns") or [] if isinstance(c, dict)} + out: list[str] = [] + for col in st.columns: + if col in failed: + continue + if any(f in (drafted.get(col) or {}) or f in (known.get(col) or {}) for f in ("label", "meaning")): + continue + text = comments.get(col) + if text and (self.system or C.comment_role(text, col) is not None): + continue + out.append(col) + return out + + @staticmethod + def _parts_said(st: _Table, failed_cols: list[str], failed_table: bool) -> list[str]: + """分段起草的表没写出来的是哪几段，每段写上列的范围（列名照表结构原样）：「第 1 段（表级的项）」「第 3 段（列名 到 + 列名，共 32 列）」。""" + part_of = st.part_of() + bad: dict[int, list[str]] = {} + for col in failed_cols: + bad.setdefault(part_of.get(col, 1), []).append(col) + said = ["第 1 段（表级的项）"] if failed_table else [] + return said + [f"第 {k} 段（{_columns_text(st.columns, cols, count=True)}）" for k, cols in sorted(bad.items())] + + @classmethod + def _partial_text(cls, st: _Table, failed_cols: list[str], failed_table: bool) -> str: + """只写出一部分时说清哪部分没写出来（第几段、哪几列）、为什么、怎么办。""" + part_of = st.part_of() + keys = ([_TABLE] if failed_table else []) + failed_cols + reason = _join((st.failed[k], part_of.get(k, 1)) for k in keys) + n = len(st.columns) + if len(st.parts) > 1: + said = cls._parts_said(st, failed_cols, failed_table) + head = f"这张表 {n} 列，分 {len(st.parts)} 段起草，{'、'.join(said)}没写出来" + elif failed_cols and failed_table: + head = (f"这张表 {n} 列，有表级的项和 {len(failed_cols)} 列（{_columns_text(st.columns, failed_cols, count=False)}）" + "没写出来") + elif failed_cols: + head = f"这张表 {n} 列，有 {len(failed_cols)} 列（{_columns_text(st.columns, failed_cols, count=False)}）没写出来" + else: + head = f"这张表 {n} 列，表级的项没写出来" + return f"{head}：{reason}。可以只重试这张表" + + async def _part_done(self, name: str) -> None: + self.tables[name].parts_done += 1 + await self._hook(self.hooks.part_done, name) + + async def _table_done(self, name: str) -> None: + st = self.tables[name] + if st.finished: + return + st.finished = True + await self._hook(self.hooks.table_done, name, self._outcome(st)) + + @staticmethod + async def _hook(fn: Callable[..., Awaitable[None]] | None, *args: Any) -> None: + if fn is None: + return + try: + await fn(*args) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 - 进度回调出错不影响起草 + logger.exception("起草数据目录的进度回调出错") + + # ---------------------------------------------------------------- 提示词 + + @staticmethod + def _background(description: Any, background: Background) -> str: + labels = C.CATALOG_DRAFT_PROMPT_LABELS + lines = [] + for key, text in (("description", description), ("guidance", background.guidance), ("note", background.note)): + text = text.strip() if isinstance(text, str) else "" + if text: + lines.append(labels[key].format(text=text)) + return C.CATALOG_DRAFT_BACKGROUND_PROMPT.format(lines="\n".join(lines)) if lines else "" + + def _messages(self, pieces: list[_Piece], mode: str) -> list[Any]: + from langchain_core.messages import HumanMessage, SystemMessage + + text = C.CATALOG_DRAFT_PROMPT.format(n=len(pieces), tables="\n\n".join(self._describe(p) for p in pieces)) + if self.background: + text = f"{self.background}\n\n{text}" + if mode == "text": + text = f"{text}\n\n{C.CATALOG_DRAFT_TEXT_PROMPT}" + return [SystemMessage(content=C.CATALOG_DRAFT_SYSTEM), HumanMessage(content=text)] + + def _describe(self, piece: _Piece) -> str: + """一段里一张表的结构描述：列、类型、主键、外键、唯一约束、注释；不含任何数据值。 + + 整表：照旧的完整描述。只写表级的项：全部列只给名字和类型。表级的项加几列：完整描述，表名下一行点名这几列。 + 只写几列：表注释、已起草的表级内容做背景，只列这几列。""" + labels = C.CATALOG_DRAFT_PROMPT_LABELS + st = self.tables[piece.table] + meta = st.meta + lines = [(labels["view"] if meta.get("is_view") else labels["table"]).format(name=st.name)] + brief = piece.table_level and not piece.columns + if not st.whole(piece): + if brief: + lines.append(labels["only_table"]) + elif piece.table_level: + lines.append(labels["table_and_columns"].format(n=len(piece.columns), cols="、".join(piece.columns))) + else: + lines.append(labels["only_columns"].format(n=len(piece.columns), total=len(st.columns))) + comment = (meta.get("comment") or "").strip() + if comment: + lines.append((labels["system"] if self.system else labels["comment"]).format(text=comment)) + pk = set(meta.get("primary_key") or []) + fk_of: dict[str, str] = {} + for fk in meta.get("foreign_keys") or []: + if isinstance(fk, dict) and len(fk.get("columns") or []) == 1: + to_cols = fk.get("to_columns") or [] + fk_of[fk["columns"][0]] = f"{fk.get('to_table')}.{to_cols[0]}" if to_cols else str(fk.get("to_table")) + if brief: + lines.append(labels["brief_columns"].format(n=len(st.columns))) + wanted = set(st.columns) + else: + if not piece.table_level: + drafted = C._notes_lines({k: v for k, v in st.notes.items() if k in C.TABLE_FIELDS}, meta=meta, + system_notes=False) + if drafted: + lines.append(labels["drafted"]) + lines += drafted + lines.append(labels["columns"]) + wanted = set(st.columns) if piece.table_level else set(piece.columns) + for col in meta.get("columns") or []: + if isinstance(col, dict) and col.get("name") in wanted: + lines.append(self._column_line(col, pk, fk_of, brief=brief)) + if piece.table_level: + for group in meta.get("unique") or []: + if isinstance(group, list) and group: + lines.append(labels["unique"].format(cols="、".join(group))) + confirmed = self._confirmed_lines(st, piece) + if confirmed: + lines.append(labels["confirmed"]) + lines += confirmed + return "\n".join(lines) + + def _column_line(self, col: Mapping[str, Any], pk: set[str], fk_of: Mapping[str, str], *, brief: bool) -> str: + labels = C.CATALOG_DRAFT_PROMPT_LABELS + marks = [] + if col["name"] in pk: + marks.append(labels["pk"]) + if not brief and not col.get("nullable", True): + marks.append(labels["not_null"]) + if col["name"] in fk_of: + marks.append(labels["fk"].format(target=fk_of[col["name"]])) + if not brief and (text := (col.get("comment") or "").strip()): + marks.append((labels["col_system"] if self.system else labels["col_comment"]).format(text=text)) + line = labels["column"].format(name=col["name"], type=col.get("type") or "") + return line + ("，" + "，".join(marks) if marks else "") + + @staticmethod + def _confirmed_lines(st: _Table, piece: _Piece) -> list[str]: + """已确认、有确证的项（以此为准的背景）：整表全给；其余给表级的项和这一段的列，关系只在要写表级的项时给。""" + confirmed = C._confirmed_only(st.existing) + if not st.whole(piece): + wanted = set(piece.columns) + kept = {key: item for key, item in C._slots(confirmed).items() + if key[0] == "t" or (key[0] == "c" and key[1] in wanted) or (key[0] == "r" and piece.table_level)} + confirmed = C._assemble(kept) + return C._notes_lines(confirmed, meta=st.meta, system_notes=False) + + +__all__ = ["BUDGET_WORDS", "Background", "CallReport", "FAILURE_LABEL", "Hooks", "ModelOutcome", "NOTICE", "REASON_CALL", + "REASON_CAPPED", "REASON_EXHAUSTED", "REASON_FORMAT", "REASON_HALTED", "REASON_HOLLOW", "REASON_MISSING", + "REASON_RETRIED", "REASON_SLOW", "REASON_SPLIT", "REASON_TIMEOUT", "REASON_TRUNCATED", "REASON_UNANSWERED", + "SEG", "classify_failure", "model_gate", "parts_for", "run_model_draft"] diff --git a/backend/app/data/catalog_jobs.py b/backend/app/data/catalog_jobs.py new file mode 100644 index 00000000..d708ed50 --- /dev/null +++ b/backend/app/data/catalog_jobs.py @@ -0,0 +1,667 @@ +"""数据目录的后台起草任务，以及数据源级的起草要求。 + +以前起草是一次同步长请求（POST /catalog/draft，界面每 12 张表一个请求）：一个请求出错整个对话框就停，关掉对话框 +之后没有任何记录哪些表失败、为什么。现在： + +- **建任务**（create_job）：同一个数据源同时只有一个排队或进行中的任务（DraftBusy；部分唯一索引兜底）。不指定表时 + 按使用次数取前 20 张；表结构里没有的表照样建进任务、状态 skipped。起草要求（数据源级）和本次说明随任务记下快照。 +- **在后台起草**（manager.start）：进程内的 asyncio 任务，引擎是 data/catalog_draft.py（分段、按失败原因处理）。 + 每张表一结束就并入目录（catalog.write_draft，规则和同步起草一样：人工确认、驳回过的不动，模型这次没给的项不删 + 以前起草的，乐观锁冲突重读重并）；逐表的状态、到第几段、计数、模型留空的列，任务的用量（含超时、没写出来的调用 + 次数）、当前的列数上限、提醒随时落库，接口读的是库里的状态。 +- **停止**（cancel）：排队的直接 cancelled；进行中的停掉正在进行的模型调用，没开始和没写完的表记 cancelled，已经写进 + 目录的不回滚，任务最后是 cancelled。 +- **重试**（retry_tables + create_job）：新任务，只含原任务里 failed、partial、cancelled 的表；用不用模型、本次说明沿用， + 起草要求取数据源当前的。 +- **服务重启**：启动时（startup）把排队、进行中的任务标成 interrupted，没结束的表记 cancelled、写明「服务重启，起草 + 中断」。不自动续跑：复制出来的数据目录（检查、验收）启动时不能自己去调模型；界面让人点「重试」续上。关停时 + （manager.shutdown）停掉进行中的任务，同样标成 interrupted。 + +任务的库写入一律经 _JobRun._write：一条一条来，shield 起来——停止任务时不会打断写到一半的库操作，收尾前等它们写完。 + +**起草要求**存在数据源 options 的 catalog_draft_guidance 里（上传表格的源存在它自己的 DataSource 行上），一段文字， +去首尾空白、最多 GUIDANCE_MAX 字，空串等于没有。只经 PUT /catalog/guidance 改：数据源接口不显示它、整份回写 options +时留着（api/datasources.py），拼连接串时摘掉（data/engine.py）。 +""" +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from typing import Any + +from sqlalchemy import func, select, update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import catalog, catalog_draft +from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION +from app.db.base import SessionLocal, new_id, utcnow +from app.db.models import CatalogDraftJob, CatalogDraftTask, DataSource + +logger = logging.getLogger(__name__) + +#: 任务还没结束的状态（同一个源只能有一个）、结束了的状态 +ACTIVE = ("queued", "running") +FINISHED = ("succeeded", "partial", "failed", "cancelled", "interrupted") +#: 逐表的状态，以及其中算「已结束」的、重试时要带上的 +TASK_STATUSES = ("pending", "running", "ok", "partial", "failed", "skipped", "cancelled") +TASK_DONE = ("ok", "partial", "failed", "skipped", "cancelled") +RETRYABLE = ("failed", "partial", "cancelled") +_OPEN = ("pending", "running") + +#: 一个任务最多几张表；起草要求、本次说明最多几个字 +MAX_TABLES = 500 +GUIDANCE_MAX = 2000 +NOTE_MAX = 1000 +#: 目录清单上「这张表最近一次起草」只看最近这么多个任务 +LAST_DRAFT_JOBS = 100 +#: 逐表最多记几个模型留空的列名 +BLANK_SAMPLE = 5 +#: 停止进行中的任务时最多等它收尾多久（秒）；关停服务时另给（shutdown 的 grace） +STOP_GRACE_S = 10.0 + +INTERRUPTED_ERROR = "服务重启，起草中断" +UNFINISHED_ERROR = "起草结束时这张表还没有写完，请重试这张表" +WRITE_ERROR = "写入数据目录时出错：{line}" +JOB_ERROR = "起草任务出错：{line}" +SOURCE_GONE = "数据源不存在，可能已被删除" + + +class DraftBusy(Exception): + """这个数据源已经有排队或进行中的起草任务。message 给人看。""" + + +class DraftFinished(Exception): + """要停止的任务已经结束了。""" + + +class NothingToRetry(Exception): + """原任务里没有需要重试的表。""" + + +class TextTooLong(ValueError): + """起草要求、本次说明超长。message 给人看。""" + + +class _JobFailed(Exception): + """任务整体出错（数据源没了、快照没了、模型用不了）：message 就是任务的 error。""" + + +# ========================================================================== +# 起草要求、本次说明 +# ========================================================================== + + +def clean_text(value: Any, limit: int, label: str) -> str | None: + """去首尾空白；空串（和不是字符串的）等于没有；超过 limit 个字抛 TextTooLong。""" + text = value.strip() if isinstance(value, str) else "" + if len(text) > limit: + raise TextTooLong(f"{label}最多 {limit} 字，当前 {len(text)} 字，请删减后再保存") + return text or None + + +def guidance_of(row: Any) -> str | None: + """数据源级的起草要求（没有是 None）。""" + raw = (getattr(row, "options", None) or {}).get(CATALOG_DRAFT_GUIDANCE_OPTION) + return raw.strip() or None if isinstance(raw, str) else None + + +async def save_guidance(session: AsyncSession, row: DataSource, value: Any) -> str | None: + """存起草要求，返回存下的（清除了是 None）。超长抛 TextTooLong，什么都不改。""" + text = clean_text(value, GUIDANCE_MAX, "起草要求") + options = {k: v for k, v in (row.options or {}).items() if k != CATALOG_DRAFT_GUIDANCE_OPTION} + if text: + options[CATALOG_DRAFT_GUIDANCE_OPTION] = text + row.options = options + await session.commit() + return text + + +# ========================================================================== +# 读任务 +# ========================================================================== + + +def _iso(value: Any) -> str | None: + return value.isoformat() if value is not None else None + + +async def active_job(session: AsyncSession, source_id: str) -> CatalogDraftJob | None: + """排队或进行中的那个（同一个源最多一个）。""" + return (await session.execute( + select(CatalogDraftJob).where(CatalogDraftJob.source_id == source_id, CatalogDraftJob.status.in_(ACTIVE)) + .order_by(CatalogDraftJob.created_at.desc()).limit(1) + )).scalar_one_or_none() + + +async def latest_job(session: AsyncSession, source_id: str) -> CatalogDraftJob | None: + """最近建的一个（任何状态）。""" + return (await session.execute( + select(CatalogDraftJob).where(CatalogDraftJob.source_id == source_id) + .order_by(CatalogDraftJob.created_at.desc()).limit(1) + )).scalar_one_or_none() + + +async def load(session: AsyncSession, source_id: str, job_id: str) -> CatalogDraftJob | None: + """这个源的这个任务；不存在、不属于这个源都是 None。""" + job = await session.get(CatalogDraftJob, job_id) + return job if job is not None and job.source_id == source_id else None + + +async def tasks_of(session: AsyncSession, job_id: str) -> list[CatalogDraftTask]: + return list((await session.execute( + select(CatalogDraftTask).where(CatalogDraftTask.job_id == job_id).order_by(CatalogDraftTask.seq) + )).scalars()) + + +def task_dict(task: CatalogDraftTask) -> dict[str, Any]: + return { + "table_name": task.table_name, + "status": task.status, + "columns": task.column_count, + "parts_total": task.parts_total, + "parts_done": task.parts_done, + "added": task.added, + "updated": task.updated, + "removed": task.removed, + "version": task.version, + "model_items": task.model_items, + # 模型拿不准、留空了的列（中文名和含义都没给）：几列、最多 5 个列名（按表结构的顺序）。表的状态不因此变 + "blank_columns": task.blank_columns or 0, + "blank_sample": list(task.blank_sample or []), + "error": task.error, + "model_error": task.model_error, + "started_at": _iso(task.started_at), + "finished_at": _iso(task.finished_at), + } + + +def _counts(statuses: list[str]) -> dict[str, int]: + counts = dict.fromkeys(TASK_STATUSES, 0) + for status in statuses: + counts[status] = counts.get(status, 0) + 1 + return counts + + +def job_dict(job: CatalogDraftJob, tasks: list[CatalogDraftTask]) -> dict[str, Any]: + """接口返回的任务（CatalogDraftJob）：逐表的状态按起草顺序；计数、合计都按逐表的算。""" + counts = _counts([t.status for t in tasks]) + return { + "id": job.id, + "source_id": job.source_id, + "status": job.status, + "use_model": bool(job.use_model), + "model": job.model, + "guidance": job.guidance, + "note": job.note, + "created_at": _iso(job.created_at), + "started_at": _iso(job.started_at), + "finished_at": _iso(job.finished_at), + "created_by": job.created_by, + "retry_of": job.retry_of, + "total": len(tasks), + "done": sum(counts.get(s, 0) for s in TASK_DONE), + "counts": counts, + "added": sum(t.added or 0 for t in tasks), + "updated": sum(t.updated or 0 for t in tasks), + "removed": sum(t.removed or 0 for t in tasks), + "blank_columns": sum(t.blank_columns or 0 for t in tasks), + # 超时的调用拿不到用量，token 按 0 算（网关那边可能已经生成了）；failed_calls 含额度用完、只给空壳、超时、出错 + "usage": {"calls": job.calls or 0, "input_tokens": job.input_tokens or 0, + "output_tokens": job.output_tokens or 0, "timeouts": job.timeouts or 0, + "failed_calls": job.failed_calls or 0}, + # 新开始的段每段最多几列（不用模型、还没开始时为 null）；提醒：最近几次调用多数没写出来时的一句话，没有为 null + "part_columns": job.part_columns, + "notice": job.notice, + "error": job.error, + "tasks": [task_dict(t) for t in tasks], + } + + +async def job_out(session: AsyncSession, job: CatalogDraftJob | None) -> dict[str, Any] | None: + return None if job is None else job_dict(job, await tasks_of(session, job.id)) + + +async def job_brief(session: AsyncSession, job: CatalogDraftJob | None) -> dict[str, Any] | None: + """目录清单顶上的进度：{id, status, total, done}。""" + if job is None: + return None + rows = (await session.execute( + select(CatalogDraftTask.status, func.count()).where(CatalogDraftTask.job_id == job.id) + .group_by(CatalogDraftTask.status) + )).all() + counts = {status: n for status, n in rows} + return {"id": job.id, "status": job.status, "total": sum(counts.values()), + "done": sum(counts.get(s, 0) for s in TASK_DONE)} + + +async def last_drafts(session: AsyncSession, source_id: str) -> dict[str, dict[str, Any]]: + """每张表最近一次起草任务里的那一项：{表名: {status, job_id, finished_at, error, model_error, blank_columns}}。 + 任务还在进行，就是进行中的状态。只看最近 LAST_DRAFT_JOBS 个任务；每张表取哪一项在库里挑好（清单页起草时会反复取）。""" + recent = (select(CatalogDraftJob.id).where(CatalogDraftJob.source_id == source_id) + .order_by(CatalogDraftJob.created_at.desc()).limit(LAST_DRAFT_JOBS)).scalar_subquery() + rank = func.row_number().over(partition_by=CatalogDraftTask.table_name, + order_by=(CatalogDraftJob.created_at.desc(), CatalogDraftTask.seq)).label("rank") + inner = (select(CatalogDraftTask.table_name, CatalogDraftTask.status, CatalogDraftTask.job_id, + CatalogDraftTask.finished_at, CatalogDraftTask.error, CatalogDraftTask.model_error, + CatalogDraftTask.blank_columns, rank) + .join(CatalogDraftJob, CatalogDraftJob.id == CatalogDraftTask.job_id) + .where(CatalogDraftJob.id.in_(recent))).subquery() + rows = (await session.execute(select(inner).where(inner.c.rank == 1))).mappings() + return {row["table_name"]: {"status": row["status"], "job_id": row["job_id"], + "finished_at": _iso(row["finished_at"]), "error": row["error"], + "model_error": row["model_error"], "blank_columns": row["blank_columns"] or 0} + for row in rows} + + +# ========================================================================== +# 建任务、重试、停止 +# ========================================================================== + + +def busy_message(row: Any) -> str: + return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录，请等这次起草结束（或先停止它）再开始新的起草" + + +async def ensure_idle(session: AsyncSession, row: Any) -> None: + """这个源没有排队或进行中的任务；有就抛 DraftBusy。""" + if await active_job(session, row.id) is not None: + raise DraftBusy(busy_message(row)) + + +async def check_model(session: AsyncSession) -> None: + """起草模型能不能用：不能用抛 catalog.CatalogModelUnavailable（只建模型对象，不发请求）。""" + await catalog.resolve_draft_model(session) + + +async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: list[str] | None, use_model: bool, + note: str | None, actor: str | None, + retry_of: str | None = None) -> tuple[CatalogDraftJob, list[CatalogDraftTask]]: + """建一个排队的任务（还没开始，调用方接着 manager.start）。row 是数据源记录（起草要求取它的），source 是读表结构 + 用的源（上传源是绑定当前快照的视图）。有排队或进行中的任务抛 DraftBusy。""" + await ensure_idle(session, row) + cache = getattr(source, "schema_cache", None) or {} + all_tables = cache.get("tables") or {} + wanted = list(tables) if tables else await catalog.default_tables(session, source) + job = CatalogDraftJob(id=new_id(), source_id=row.id, status="queued", use_model=bool(use_model), + guidance=guidance_of(row), note=note, created_by=actor, retry_of=retry_of) + tasks: list[CatalogDraftTask] = [] + seen: set[str] = set() + now = utcnow() + for name in dict.fromkeys(str(t) for t in wanted): + key = catalog.resolve_table_name(cache, name) + if key is None: + tasks.append(CatalogDraftTask(job_id=job.id, seq=len(tasks), table_name=name, status="skipped", + error=catalog.missing_table_error(name), finished_at=now)) + continue + if key in seen: + continue + seen.add(key) + columns = len((all_tables.get(key) or {}).get("columns") or []) + tasks.append(CatalogDraftTask(job_id=job.id, seq=len(tasks), table_name=key, status="pending", + column_count=columns, + parts_total=catalog_draft.parts_for(columns) if use_model else None)) + session.add(job) + try: + await session.flush() + session.add_all(tasks) + await session.commit() + except IntegrityError as e: + # 部分唯一索引：另一个请求同时给这个源建了任务 + await session.rollback() + raise DraftBusy(busy_message(row)) from e + return job, tasks + + +async def retry_tables(session: AsyncSession, job: CatalogDraftJob) -> list[str]: + """重试要带上的表：原任务里 failed、partial、cancelled 的，按原来的顺序。没有抛 NothingToRetry。""" + names = [t.table_name for t in await tasks_of(session, job.id) if t.status in RETRYABLE] + if not names: + raise NothingToRetry("这次起草里没有失败、只写出一部分或停止了的表，不需要重试") + return names + + +async def cancel(session: AsyncSession, job: CatalogDraftJob) -> None: + """停止一个任务。排队的直接 cancelled；进行中的交给 manager 停（停掉模型调用、没写完的表记 cancelled）。 + 已经结束的抛 DraftFinished。""" + if job.status in FINISHED: + raise DraftFinished("这次起草已经结束了，不需要停止") + if job.status == "queued": + now = utcnow() + claimed = await session.execute( + update(CatalogDraftJob).where(CatalogDraftJob.id == job.id, CatalogDraftJob.status == "queued") + .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) + if claimed.rowcount == 1: + await session.execute( + update(CatalogDraftTask).where(CatalogDraftTask.job_id == job.id, CatalogDraftTask.status.in_(_OPEN)) + .values(status="cancelled", finished_at=now).execution_options(synchronize_session=False)) + await session.commit() + return + await session.rollback() # 刚被后台认领、开始了：按进行中的停 + if await manager.stop(job.id, "user", grace=STOP_GRACE_S): + return + # 库里说在进行，这个进程里却没有在跑它（记录对不上）：直接记停止 + await _close(session, job.id, "cancelled", None, "cancelled", None) + + +async def _close(session: AsyncSession, job_id: str, job_status: str, job_error: str | None, task_status: str, + task_error: str | None) -> None: + """收尾：还没结束的表记 task_status，任务（还没结束的话）记 job_status。""" + now = utcnow() + await session.execute( + update(CatalogDraftTask).where(CatalogDraftTask.job_id == job_id, CatalogDraftTask.status.in_(_OPEN)) + .values(status=task_status, error=task_error, finished_at=now).execution_options(synchronize_session=False)) + await session.execute( + update(CatalogDraftJob).where(CatalogDraftJob.id == job_id, CatalogDraftJob.status.in_(ACTIVE)) + .values(status=job_status, error=job_error, finished_at=now).execution_options(synchronize_session=False)) + await session.commit() + + +async def startup() -> int: + """服务启动时：排队、进行中的任务标成 interrupted，没结束的表记 cancelled、写明服务重启。不自动续跑。返回标了几个。""" + async with SessionLocal() as session: + ids = [job_id for job_id in (await session.execute( + select(CatalogDraftJob.id).where(CatalogDraftJob.status.in_(ACTIVE)))).scalars() + if not manager.running(job_id)] + for job_id in ids: + await _close(session, job_id, "interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) + if ids: + logger.warning("服务重启：%d 个数据目录起草任务没有做完，已标为中断，可在数据目录页重试", len(ids)) + return len(ids) + + +# ========================================================================== +# 在后台起草 +# ========================================================================== + + +class _JobRun: + """一个任务在后台的一次执行。""" + + def __init__(self, job_id: str) -> None: + self.job_id = job_id + self.task: asyncio.Task[None] | None = None + #: 为什么停：user 使用者点了停止，shutdown 服务关停（标中断） + self.stop_reason: str | None = None + self._writes: set[asyncio.Task[Any]] = set() + self._lock = asyncio.Lock() + self._started: set[str] = set() + self.source: Any = None + self.actor: str | None = None + self.at = "" + + # ---------------------------------------------------------------- 库写入 + + async def _write(self, fn: Callable[..., Awaitable[Any]], *args: Any) -> Any: + """一条库写入：同一个任务的写入一条一条来；shield 起来，停止任务时不会打断写到一半的库操作。""" + async def go() -> Any: + async with self._lock: + async with SessionLocal() as session: + return await fn(session, *args) + + task = asyncio.ensure_future(go()) + self._writes.add(task) + task.add_done_callback(self._writes.discard) + return await asyncio.shield(task) + + async def _settle(self) -> None: + """等手上的库写入都写完（停止、出错收尾之前）。""" + while self._writes: + await asyncio.gather(*list(self._writes), return_exceptions=True) + + # ---------------------------------------------------------------- 主流程 + + async def run(self) -> None: + try: + await self._run() + except asyncio.CancelledError: + current = asyncio.current_task() + if current is not None: + current.uncancel() + if self.stop_reason == "user": + await self._end("cancelled", None, "cancelled", None) + else: + await self._end("interrupted", INTERRUPTED_ERROR, "cancelled", INTERRUPTED_ERROR) + except _JobFailed as e: + await self._end("failed", str(e), "failed", str(e)) + except Exception as e: # noqa: BLE001 - 任务整体出错：照实记下，不留在「起草中」 + from app.core.errors import describe_exception, first_line + + logger.exception("数据目录起草任务 %s 出错", self.job_id) + reason = JOB_ERROR.format(line=first_line(e) or describe_exception(e)) + await self._end("failed", reason, "failed", reason) + + async def _end(self, job_status: str, job_error: str | None, task_status: str, task_error: str | None) -> None: + """停止、出错时收尾：等手上的库写入写完，再把没结束的表和任务记下来。收尾本身写不进去只记日志（下次启动时 + startup 会把它标成中断）。""" + try: + await self._settle() + await self._write(_close, self.job_id, job_status, job_error, task_status, task_error) + except asyncio.CancelledError: + raise + except Exception: # noqa: BLE001 + logger.exception("数据目录起草任务 %s 收尾时写库出错", self.job_id) + + async def _run(self) -> None: + if not await self._write(self._claim): + return # 开始之前就被停止了 + async with SessionLocal() as session: + job = await session.get(CatalogDraftJob, self.job_id) + row = await session.get(DataSource, job.source_id) if job is not None else None + if job is None or row is None: + raise _JobFailed(SOURCE_GONE) + from app.data import table_versions + + try: + source = await table_versions.resolve_source(session, row) + except table_versions.SnapshotMissing as e: + raise _JobFailed(str(e)) from e + pending = [t.table_name for t in await tasks_of(session, job.id) if t.status == "pending"] + use_model, background = bool(job.use_model), catalog_draft.Background(guidance=job.guidance, note=job.note) + self.actor = job.created_by + model: Any = None + existing: dict[str, Any] = {} + if use_model and pending: + try: + model, model_id = await catalog.resolve_draft_model(session) + except catalog.CatalogModelUnavailable as e: + raise _JobFailed(str(e)) from e + existing = {t: e.notes for t, e in (await catalog.read_catalog(session, source.id)).items()} + self.source, self.at = source, catalog.now_iso() + all_tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} + names = [n for n in pending if isinstance(all_tables.get(n), dict)] + gone = [n for n in pending if n not in names] + if gone: + # 建任务之后表结构变了（重新探查过）：这几张表跳过 + await self._write(self._skip, gone) + if use_model and names: + await self._write(self._set_model, model_id) + hooks = catalog_draft.Hooks(call_started=self._call_started, call_done=self._call_done, + part_done=self._part_done, table_done=self._table_done) + await catalog_draft.run_model_draft(model, source, names, existing=existing, background=background, + hooks=hooks) + else: + for name in names: + await self._call_started([name]) + await self._table_done(name, None) + await self._settle() + await self._write(self._finish) + + # ---------------------------------------------------------------- 进度回调（引擎调） + + async def _call_started(self, tables: list[str]) -> None: + fresh = [t for t in dict.fromkeys(tables) if t not in self._started] + if fresh: + self._started.update(fresh) + await self._write(self._mark_running, fresh) + + async def _call_done(self, report: catalog_draft.CallReport) -> None: + await self._write(self._add_call, report) + + async def _part_done(self, table: str) -> None: + await self._write(self._bump_part, table) + + async def _table_done(self, table: str, outcome: catalog_draft.ModelOutcome | None) -> None: + await self._write(self._write_table, table, outcome) + + # ---------------------------------------------------------------- 库操作 + + def _task(self, *where: Any) -> Any: + return update(CatalogDraftTask).where(CatalogDraftTask.job_id == self.job_id, *where) \ + .execution_options(synchronize_session=False) + + async def _claim(self, session: AsyncSession) -> bool: + result = await session.execute( + update(CatalogDraftJob).where(CatalogDraftJob.id == self.job_id, CatalogDraftJob.status == "queued") + .values(status="running", started_at=utcnow()).execution_options(synchronize_session=False)) + await session.commit() + return result.rowcount == 1 + + async def _set_model(self, session: AsyncSession, model_id: str) -> None: + await session.execute(update(CatalogDraftJob).where(CatalogDraftJob.id == self.job_id) + .values(model=model_id or None, part_columns=max(1, int(catalog.MODEL_BATCH_COLUMNS))) + .execution_options(synchronize_session=False)) + await session.commit() + + async def _skip(self, session: AsyncSession, names: list[str]) -> None: + now = utcnow() + for name in names: + await session.execute(self._task(CatalogDraftTask.table_name == name, CatalogDraftTask.status == "pending") + .values(status="skipped", error=catalog.missing_table_error(name), finished_at=now)) + await session.commit() + + async def _mark_running(self, session: AsyncSession, names: list[str]) -> None: + await session.execute(self._task(CatalogDraftTask.table_name.in_(names), CatalogDraftTask.status == "pending") + .values(status="running", started_at=utcnow())) + await session.commit() + + async def _add_call(self, session: AsyncSession, report: catalog_draft.CallReport) -> None: + """一次调用：用量、超时、没写出来的次数累加，当前的列数上限和提醒照这次调用之后的写（按调用的先后写进来）。""" + await session.execute( + update(CatalogDraftJob).where(CatalogDraftJob.id == self.job_id) + .values(calls=CatalogDraftJob.calls + 1, + input_tokens=CatalogDraftJob.input_tokens + int(report.input_tokens), + output_tokens=CatalogDraftJob.output_tokens + int(report.output_tokens), + timeouts=func.coalesce(CatalogDraftJob.timeouts, 0) + int(report.timeout), + failed_calls=func.coalesce(CatalogDraftJob.failed_calls, 0) + int(report.failure is not None), + part_columns=report.part_columns, notice=report.notice) + .execution_options(synchronize_session=False)) + await session.commit() + + async def _bump_part(self, session: AsyncSession, table: str) -> None: + await session.execute(self._task(CatalogDraftTask.table_name == table) + .values(parts_done=CatalogDraftTask.parts_done + 1)) + await session.commit() + + async def _write_table(self, session: AsyncSession, table: str, + outcome: catalog_draft.ModelOutcome | None) -> None: + """一张表结束：并入目录、写入，记下这张表的结果。""" + try: + result = await catalog.write_draft(session, self.source, table, outcome, at=self.at, actor=self.actor) + except Exception as e: # noqa: BLE001 - 这一张写不进去不影响别的表 + from app.core.errors import describe_exception, first_line + + logger.exception("起草任务 %s 写入表 %s 的数据目录时出错", self.job_id, table) + await session.rollback() + result = catalog.TableDraftResult(table=table, status="failed", + error=WRITE_ERROR.format(line=first_line(e) or describe_exception(e)), + model_error=outcome.error if outcome is not None else None) + values: dict[str, Any] = { + "status": result.status, "added": result.added, "updated": result.updated, "removed": result.removed, + "version": result.version, "model_items": result.model_items, "error": result.error, + "model_error": result.model_error, "finished_at": utcnow(), + "blank_columns": len(result.blank), "blank_sample": result.blank[:BLANK_SAMPLE] or None, + } + if outcome is not None: + values["parts_done"] = outcome.parts_done + await session.execute(self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN)) + .values(**values)) + await session.commit() + + async def _finish(self, session: AsyncSession) -> None: + """起草完：万一还有没收尾的表（进度回调写库失败之类）记失败；有 failed、partial、cancelled 的表是 partial， + 否则 succeeded。""" + now = utcnow() + await session.execute(self._task(CatalogDraftTask.status.in_(_OPEN)) + .values(status="failed", error=UNFINISHED_ERROR, finished_at=now)) + statuses = list((await session.execute( + select(CatalogDraftTask.status).where(CatalogDraftTask.job_id == self.job_id))).scalars()) + status = "partial" if any(s in RETRYABLE for s in statuses) else "succeeded" + await session.execute( + update(CatalogDraftJob).where(CatalogDraftJob.id == self.job_id, CatalogDraftJob.status == "running") + .values(status=status, finished_at=now).execution_options(synchronize_session=False)) + await session.commit() + + +class DraftJobs: + """进程内的起草任务（照 app/scheduler 的写法：一个 asyncio 任务一件事，停的时候给宽限）。""" + + def __init__(self) -> None: + self._runs: dict[str, _JobRun] = {} + + def _alive(self, job_id: str) -> _JobRun | None: + run = self._runs.get(job_id) + if run is None or run.task is None or run.task.done(): + return None + try: + if run.task.get_loop() is not asyncio.get_running_loop(): + self._runs.pop(job_id, None) # 别的事件循环留下的（测试每个用例一个循环） + return None + except RuntimeError: + return None + return run + + def start(self, job_id: str) -> None: + """开始在后台起草这个（排队中的）任务。""" + run = _JobRun(job_id) + run.task = asyncio.get_running_loop().create_task(run.run(), name=f"catalog-draft-{job_id}") + self._runs[job_id] = run + + def forget(_: asyncio.Task[None]) -> None: + if self._runs.get(job_id) is run: + self._runs.pop(job_id, None) + + run.task.add_done_callback(forget) + + def running(self, job_id: str) -> bool: + return self._alive(job_id) is not None + + async def stop(self, job_id: str, reason: str, *, grace: float = STOP_GRACE_S) -> bool: + """停掉这个进程里在跑的任务，等它收尾（最多 grace 秒）。不在这个进程里跑返回 False。""" + run = self._alive(job_id) + if run is None: + return False + if run.stop_reason is None: + run.stop_reason = reason + run.task.cancel() # type: ignore[union-attr] + await asyncio.wait([run.task], timeout=grace) # type: ignore[list-item] + return True + + async def wait(self, job_id: str, timeout: float | None = None) -> None: + """等这个任务做完（测试、调试用）。不在跑直接返回。""" + run = self._alive(job_id) + if run is not None: + await asyncio.wait([run.task], timeout=timeout) # type: ignore[list-item] + + async def shutdown(self, *, grace: float = 3.0) -> None: + """关停：停掉进行中的任务（标中断），最多等 grace 秒收尾。""" + runs = [run for job_id in list(self._runs) if (run := self._alive(job_id)) is not None] + for run in runs: + if run.stop_reason is None: + run.stop_reason = "shutdown" + run.task.cancel() # type: ignore[union-attr] + if runs: + await asyncio.wait([run.task for run in runs], timeout=grace) # type: ignore[misc] + logger.info("已停止 %d 个数据目录起草任务", len(runs)) + + +manager = DraftJobs() + + +__all__ = ["ACTIVE", "BLANK_SAMPLE", "DraftBusy", "DraftFinished", "DraftJobs", "FINISHED", "GUIDANCE_MAX", + "INTERRUPTED_ERROR", "MAX_TABLES", "NOTE_MAX", "NothingToRetry", "RETRYABLE", "TASK_DONE", "TASK_STATUSES", + "TextTooLong", + "active_job", "cancel", "check_model", "clean_text", "create_job", "ensure_idle", "guidance_of", "job_brief", + "job_dict", "job_out", "last_drafts", "latest_job", "load", "manager", "retry_tables", "save_guidance", + "startup", "tasks_of"] diff --git a/backend/app/data/draft_model.py b/backend/app/data/draft_model.py new file mode 100644 index 00000000..da3be8be --- /dev/null +++ b/backend/app/data/draft_model.py @@ -0,0 +1,114 @@ +"""起草用哪个模型：数据目录的「用模型起草」和按配方导入的 AI 起草共用这一处取值。 + +顺序（哪一级写了就整组用哪一级，不跨级拼接入和模型）： + +1. 设置里的起草模型（catalog_draft 一组：{provider, model}），填了接入或模型就用它； +2. 否则沿用助手的模型设置（copilot 一组，「编排」页助手栏的「助手使用的模型」）； +3. 再没有就是默认接入（providers/factory.resolve_provider 的回退）。 + +这里只决定「用哪个接入、哪个模型」。额度、思考、温度由两边各自定：数据目录起草保留思考、额度 32768 +（data/catalog.resolve_draft_model），配方 AI 起草照旧关思考（data/recipe_ai.resolve_draft_model）。 + +**为什么不用 get_chat_model。** 它在没有接入时静默退回演示模型，拿演示模型起草出来的东西会被当成真的。 +两边都自己解析接入：没有、停用、只有演示接入、缺 key，都在调用模型之前判定为不可用，说清是哪一级的设置出了问题。 + +数据层不导入接口层：设置键在这里定义，api/settings.py 从这里取（助手那一项和 api/copilot.py 同值，有测试核对）。 +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +#: 设置里起草模型那一组的键 +CATALOG_DRAFT_SETTING_KEY = "catalog_draft" +#: 助手的模型设置键（与 api/copilot.py 的 COPILOT_SETTING_KEY 同值） +COPILOT_SETTING_KEY = "copilot" +#: 起草模型在设置页上的位置。报错里让人去这里改；和前端设置页放这一组的地方一致 +DRAFT_MODEL_PLACE = "「设置 → 偏好设置」的「起草模型」" + +#: 取自哪一级：draft 设置里的起草模型、assistant 助手的模型、default 默认接入 +ORIGINS = ("draft", "assistant", "default") + + +@dataclass(frozen=True) +class DraftModelChoice: + """起草用的接入和模型（都可以为 None：接入为 None 时按模型名找、再退回默认接入；模型为 None 时用接入的默认模型）。""" + + provider: str | None + model: str | None + origin: str + + +def _text(value: Any) -> str | None: + """设置里的一项：字符串去掉首尾空白，空串和别的类型（存坏了的）都当没填。""" + return value.strip() or None if isinstance(value, str) else None + + +def setting_pair(stored: Any) -> tuple[str | None, str | None]: + """库里存的一组 → (接入, 模型)，规范化同 _text。""" + stored = stored if isinstance(stored, dict) else {} + return _text(stored.get("provider")), _text(stored.get("model")) + + +async def draft_model_choice(session: AsyncSession) -> DraftModelChoice: + """按上面的顺序取。只读设置，不查接入在不在（resolve_provider 去查）。""" + from app.db.models import Setting + + for key, origin in ((CATALOG_DRAFT_SETTING_KEY, "draft"), (COPILOT_SETTING_KEY, "assistant")): + row = await session.get(Setting, key) + provider, model = setting_pair(row.value if row is not None else None) + if provider or model: + return DraftModelChoice(provider, model, origin) + return DraftModelChoice(None, None, "default") + + +def draft_unavailable(e: BaseException, origin: str) -> str: + """取到的接入用不了（点名的不存在、已停用，缺 key、没有模型……）时给人看的那句话。 + + 沿用助手的模型（或默认接入）时，和助手取不到模型时同一句（factory.assistant_unavailable）；用的是设置里的起草 + 模型时，照实说是起草模型那一项、去设置页哪里改。""" + from app.core.errors import not_configured + from app.providers.factory import NAMED_FIX, assistant_unavailable, model_fact + + if origin != "draft": + return assistant_unavailable(e) + code = getattr(e, "code", "") + if code in NAMED_FIX: + said = f"起草模型使用的{getattr(e, 'reason', '') or e}。" + if owner := getattr(e, "model_owner", ""): + return (f"{said}{NAMED_FIX[code]}；{model_fact(getattr(e, 'name', ''), owner)}，" + f"也可在{DRAFT_MODEL_PLACE}中把它填进模型一栏") + return f"{said}{NAMED_FIX[code]}，或在{DRAFT_MODEL_PLACE}中重新选择" + return (f"起草模型尚未配置完成：{not_configured(e, with_hint=False)}。请到「设置 → 模型接入」检查，" + f"或在{DRAFT_MODEL_PLACE}中重新选择") + + +class DraftModelUnavailable(RuntimeError): + """没有可用于起草的模型。message 给人看；在调用模型之前判定。""" + + +async def resolve_draft_provider(session: AsyncSession, *, none_reason: str, mock_reason: str) -> tuple[Any, DraftModelChoice]: + """(接入, 取值)。用不了时抛 DraftModelUnavailable：没有任何启用的接入说 none_reason，只有演示接入说 mock_reason + （两边各有各的说法），点名的接入不存在、已停用按取自哪一级说（draft_unavailable）。 + + 只解析接入，不建模型对象：建的时候缺 key 之类的问题由调用方经 draft_unavailable 说。""" + from app.providers.factory import ProviderNotConfigured, resolve_provider + + choice = await draft_model_choice(session) + try: + # 存的接入名不存在、已停用，只存了模型名、提供它的接入已停用：resolve_provider 都会报出来 + provider = await resolve_provider(session, choice.provider, choice.model) + except ProviderNotConfigured as e: + raise DraftModelUnavailable(draft_unavailable(e, choice.origin)) from e + if provider is None: + raise DraftModelUnavailable(none_reason) + if provider.kind == "mock": + raise DraftModelUnavailable(mock_reason) + return provider, choice + + +__all__ = ["CATALOG_DRAFT_SETTING_KEY", "COPILOT_SETTING_KEY", "DRAFT_MODEL_PLACE", "DraftModelChoice", + "DraftModelUnavailable", "draft_model_choice", "draft_unavailable", "resolve_draft_provider", + "setting_pair"] diff --git a/backend/app/data/engine.py b/backend/app/data/engine.py index 0f4cbf3d..59315118 100644 --- a/backend/app/data/engine.py +++ b/backend/app/data/engine.py @@ -62,12 +62,18 @@ MASK_COLUMNS_OPTION = "mask_columns" #: data/catalog_profile.py；放在这里只为让拼连接串时摘掉它 CATALOG_PROFILE_OPTION = "catalog_profile" +#: 数据源 options 里的数据目录起草要求：使用者补充的背景和要求，起草时交给模型（一段文字，最多 2000 字）。 +#: 只经 PUT /api/datasources/{id}/catalog/guidance 维护，见 data/catalog_jobs.py；放在这里只为让拼连接串时摘掉它 +CATALOG_DRAFT_GUIDANCE_OPTION = "catalog_draft_guidance" + # options 里这些 key 是 AgentLab 自己的配置，不是驱动参数，拼 URL 时要摘掉。 # schema：探查哪个 schema（企业库里只读账号名下常常什么都没有，数据在别处） # query_timeout_s：查询时限，由数据层按语句下发给数据库，见 _server_deadline # mask_columns：证据面板遮罩的列，驱动不认识它，拼进连接串会被当成未知参数拒掉 # catalog_profile：数据剖析设置，同上；它还是个对象，拼进去连 URL 都不成形 -_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION, CATALOG_PROFILE_OPTION}) +# catalog_draft_guidance：数据目录的起草要求，一段中文，拼进去连接直接失败 +_NON_DRIVER_OPTIONS = frozenset({"schema", QUERY_TIMEOUT_OPTION, MASK_COLUMNS_OPTION, CATALOG_PROFILE_OPTION, + CATALOG_DRAFT_GUIDANCE_OPTION}) _MASK_SPLIT = re.compile(r"[,，、;；\n]+") diff --git a/backend/app/data/recipe_ai.py b/backend/app/data/recipe_ai.py index 44ec1f92..5cb8b42f 100644 --- a/backend/app/data/recipe_ai.py +++ b/backend/app/data/recipe_ai.py @@ -6,7 +6,8 @@ 必须人工确认之后才能启用。每月重传按已确认的配方重放，不再调用模型。 **为什么不用 get_chat_model。** 它在没有接入时会静默退回演示模型；拿演示模型起草等于假装起草了。这里 -自己按助手的模型设置解析接入，没有、停用、只有演示接入、缺 key 都在调用模型之前判定为不可用。 +自己解析接入（取值顺序和数据目录起草共用 data/draft_model.py：设置里的起草模型 > 助手的模型 > 默认接入）， +没有、停用、只有演示接入、缺 key 都在调用模型之前判定为不可用。 **调用参数照 c837cdb 修过的问题来**：关思考（thinking="off"）、额度 32768、服务方嫌额度大时退回 8192； 正文一律经 message_text 抽取（回复可能是块列表）；输出被截断单独记一种失败，不当成「不是 JSON」去修订。 @@ -42,6 +43,7 @@ from app.data.recipe_types import ( GridCell, Recipe, ) +from app.data.draft_model import COPILOT_SETTING_KEY as _DRAFT_COPILOT_KEY from app.data.tabular import EXCEL_ERRORS from app.data.xlsx_scan import WorkbookScan, cell_ref, col_letter @@ -58,8 +60,9 @@ class AiTooLarge(ValueError): # 没有 AiFailed：调用之后的任何失败都体现在返回值里（draft=None、error、usage 照记） -#: 助手的模型设置键（与 app/api/copilot.py 的 COPILOT_SETTING_KEY 同值；数据层不导入接口层，测试断言两者相等） -COPILOT_SETTING_KEY = "copilot" +#: 助手的模型设置键（与 app/api/copilot.py 的 COPILOT_SETTING_KEY 同值；数据层不导入接口层，测试断言两者相等）。 +#: 取模型时由 data/draft_model.py 读，这里留着这个名字给测试和老代码用 +COPILOT_SETTING_KEY = _DRAFT_COPILOT_KEY #: 1 次起草 + 最多 2 次修订 AI_MAX_ATTEMPTS = 3 #: 单次调用的超时（秒）；测试 monkeypatch 调小 @@ -252,37 +255,27 @@ class _BudgetFallback: return await self.model.ainvoke(messages) -def _unconfigured(e: BaseException) -> str: - """和 copilot._unconfigured 同一句（providers/factory.assistant_unavailable）。""" - from app.providers.factory import assistant_unavailable - - return assistant_unavailable(e) - - async def resolve_draft_model(session: AsyncSession) -> tuple[Any | None, str, AiAvailability]: - """(可调用的模型, 模型 id, 可用性)；不可用时模型为 None。只建模型对象，不发请求。""" - from app.db.models import Setting - from app.providers.factory import ModelSpec, ProviderNotConfigured, build_chat_model, resolve_provider + """(可调用的模型, 模型 id, 可用性)；不可用时模型为 None。只建模型对象，不发请求。 + + 用哪个接入、哪个模型和数据目录起草共用一处取值（data/draft_model.py：起草模型 > 助手的模型 > 默认接入）； + 额度、关思考照旧是这里自己的。""" + from app.data.draft_model import DraftModelUnavailable, draft_unavailable, resolve_draft_provider + from app.providers.factory import ModelSpec, ProviderNotConfigured, build_chat_model - row = await session.get(Setting, COPILOT_SETTING_KEY) - saved = (row.value if row else None) or {} - spec = ModelSpec(provider=saved.get("provider") or None, model=saved.get("model") or None, - max_tokens=AI_MAX_TOKENS, temperature=0, thinking="off") try: - provider = await resolve_provider(session, spec.provider, spec.model) - except ProviderNotConfigured as e: - # 存的接入名不存在、已停用，只存了模型名、提供它的接入已停用：resolve_provider 都会报出来 - return None, "", AiAvailability(False, reason=_unconfigured(e)) - if provider is None: - return None, "", AiAvailability(False, reason="未配置模型接入，无法使用 AI 起草。请到「设置 → 模型接入」添加") - if provider.kind == "mock": - return None, "", AiAvailability(False, reason="当前只有演示用的模型接入，无法用于 AI 起草。" - "请到「设置 → 模型接入」添加真实的模型接入") + provider, choice = await resolve_draft_provider( + session, none_reason="未配置模型接入，无法使用 AI 起草。请到「设置 → 模型接入」添加", + mock_reason="当前只有演示用的模型接入，无法用于 AI 起草。请到「设置 → 模型接入」添加真实的模型接入") + except DraftModelUnavailable as e: + return None, "", AiAvailability(False, reason=str(e)) + spec = ModelSpec(provider=choice.provider, model=choice.model, max_tokens=AI_MAX_TOKENS, temperature=0, + thinking="off") try: big = build_chat_model(provider, spec) small = build_chat_model(provider, spec.model_copy(update={"max_tokens": AI_FALLBACK_MAX_TOKENS})) except ProviderNotConfigured as e: - return None, "", AiAvailability(False, reason=_unconfigured(e)) + return None, "", AiAvailability(False, reason=draft_unavailable(e, choice.origin)) model_id = spec.model or provider.default_model or "" return (_BudgetFallback(big, small, model_id), model_id, AiAvailability(True, model=model_id, provider=provider.name)) diff --git a/backend/app/db/base.py b/backend/app/db/base.py index 68ee478b..f0da4ac6 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -204,6 +204,14 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [ # 没有选项，也不是定时任务发起的 ("runs", "options", "JSON DEFAULT '{}'"), ("runs", "schedule_id", "VARCHAR(32)"), + # 数据目录起草任务（验收后修正）：超时、没写出来的调用次数，当前的列数上限，提醒；逐表的模型留空的列。 + # 存量任务回填 0 和 NULL：升级前没记这些 + ("catalog_draft_jobs", "timeouts", "INTEGER DEFAULT 0"), + ("catalog_draft_jobs", "failed_calls", "INTEGER DEFAULT 0"), + ("catalog_draft_jobs", "part_columns", "INTEGER"), + ("catalog_draft_jobs", "notice", "TEXT"), + ("catalog_draft_tasks", "blank_columns", "INTEGER DEFAULT 0"), + ("catalog_draft_tasks", "blank_sample", "JSON"), ] #: 迁移补上的列要建的索引：(索引名, 表, 列)。create_all 不碰已有的表，表上新列的索引也就不会建。 diff --git a/backend/app/db/models.py b/backend/app/db/models.py index db3e6dac..a5438d84 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -4,7 +4,7 @@ from datetime import datetime from typing import Any from sqlalchemy import ( - Float, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, + Float, ForeignKey, Index, Integer, LargeBinary, String, Text, UniqueConstraint, text, ) from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -348,6 +348,86 @@ class CatalogNote(Base, TimestampMixin): updated_by: Mapped[str | None] = mapped_column(String(100), default=None) +class CatalogDraftJob(Base, TimestampMixin): + """数据目录的一次后台起草任务（data/catalog_jobs.py）。逐表的状态在 catalog_draft_tasks。 + + 以前起草是一次同步长请求，出错整个对话框就停、关掉之后没有任何记录哪些表失败、为什么；现在建任务后在后台起草， + 进度（每张表到第几段、计数、用量）随时落库，接口读的是库里的状态。 + + 同一个数据源同时只有一个排队或进行中的任务：接口先查一遍，部分唯一索引兜底（两个请求同时建，后提交的撞上）。 + 服务重启时还没结束的标成 interrupted，不自动续跑。挂外键、级联删除：任务是数据源的附属记录，源删掉了留着没有意义。 + """ + + __tablename__ = "catalog_draft_jobs" + __table_args__ = ( + Index("ix_catalog_draft_jobs_source_created", "source_id", "created_at"), + Index("ux_catalog_draft_jobs_active", "source_id", unique=True, + sqlite_where=text("status IN ('queued', 'running')")), + ) + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + source_id: Mapped[str] = mapped_column(ForeignKey("data_sources.id", ondelete="CASCADE")) + # queued | running | succeeded | partial | failed | cancelled | interrupted + status: Mapped[str] = mapped_column(String(20), default="queued") + use_model: Mapped[bool] = mapped_column(default=False) + # 实际用的模型 id（用模型时开始起草后写上） + model: Mapped[str | None] = mapped_column(String(200), default=None) + # 建任务时的快照：数据源级的起草要求、这一次的补充说明 + guidance: Mapped[str | None] = mapped_column(Text, default=None) + note: Mapped[str | None] = mapped_column(Text, default=None) + # 请求头 X-Actor 的署名（自报、未认证） + created_by: Mapped[str | None] = mapped_column(String(100), default=None) + started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + # 用量：调用次数、输入和输出 token（拿不到用量的调用按 0 算，超时的调用也是 0）、超时几次、没写出来几次（额度用完、 + # 只给空壳、超时、出错） + calls: Mapped[int] = mapped_column(Integer, default=0) + input_tokens: Mapped[int] = mapped_column(Integer, default=0) + output_tokens: Mapped[int] = mapped_column(Integer, default=0) + timeouts: Mapped[int] = mapped_column(Integer, default=0) + failed_calls: Mapped[int] = mapped_column(Integer, default=0) + # 新开始的段每段最多几列（额度用完、超时多了会减半；用模型时开始起草后写上） + part_columns: Mapped[int | None] = mapped_column(Integer, default=None) + # 提醒：最近几次调用多数没写出来时的一句话（建议换起草模型、减少表或停止），少了清掉 + notice: Mapped[str | None] = mapped_column(Text, default=None) + # 任务整体出错的原因（逐表的原因在 catalog_draft_tasks） + error: Mapped[str | None] = mapped_column(Text, default=None) + # 重试建的任务记下是重试哪一个 + retry_of: Mapped[str | None] = mapped_column(String(32), default=None) + + +class CatalogDraftTask(Base): + """起草任务里的一张表。按 seq（起草顺序）排。状态：pending 排队、running 起草中、ok、partial 模型只写出一部分、 + failed、skipped 表结构里没有、cancelled（停止或服务重启）。""" + + __tablename__ = "catalog_draft_tasks" + __table_args__ = (UniqueConstraint("job_id", "seq"),) + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + job_id: Mapped[str] = mapped_column(ForeignKey("catalog_draft_jobs.id", ondelete="CASCADE"), index=True) + seq: Mapped[int] = mapped_column(Integer) + # 表结构里的键；表结构里没有的（skipped）是请求里的原样名字 + table_name: Mapped[str] = mapped_column(String(255)) + status: Mapped[str] = mapped_column(String(20), default="pending") + column_count: Mapped[int] = mapped_column(Integer, default=0) + # 模型分几段起草（不用模型时为 NULL）、已经结束了几段 + parts_total: Mapped[int | None] = mapped_column(Integer, default=None) + parts_done: Mapped[int] = mapped_column(Integer, default=0) + added: Mapped[int] = mapped_column(Integer, default=0) + updated: Mapped[int] = mapped_column(Integer, default=0) + removed: Mapped[int] = mapped_column(Integer, default=0) + version: Mapped[int] = mapped_column(Integer, default=0) + model_items: Mapped[int | None] = mapped_column(Integer, default=None) + # 模型拿不准、留空了的列（中文名和含义都没给）：几列，最多记 5 个列名。表的状态不因此变 + blank_columns: Mapped[int] = mapped_column(Integer, default=0) + blank_sample: Mapped[list[Any] | None] = mapped_column(default=None) + # error：整张表没起草（表结构里没有、写入一直冲突、服务重启中断）；model_error：模型那部分没写出来的原因和办法 + error: Mapped[str | None] = mapped_column(Text, default=None) + model_error: Mapped[str | None] = mapped_column(Text, default=None) + started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + + class Setting(Base, TimestampMixin): """键值形式的用户设置，前端 Settings 页直接读写。""" diff --git a/backend/app/main.py b/backend/app/main.py index 8a1feb7a..ba7dade7 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -70,6 +70,15 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: except Exception: # noqa: BLE001 logger.exception("上传表格的版本整理失败，服务照常启动") + # 上次没做完的数据目录起草任务：标成中断（没结束的表记停止），不自动续跑——拷来的数据目录（检查脚本、验收副本） + # 启动时不能替人去调模型。界面上点「重试」接着起草没写好的表 + from app.data import catalog_jobs + + try: + await catalog_jobs.startup() + except Exception: # noqa: BLE001 + logger.exception("数据目录起草任务的启动整理失败，服务照常启动") + await run_manager.setup() from app.sandbox.manager import sandbox_manager as _sbm @@ -84,8 +93,9 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: try: yield finally: - # 先停调度，再停运行：停运行的途中不该再有新运行被发起 + # 先停调度，再停运行：停运行的途中不该再有新运行被发起。起草任务停下来标成中断，下次启动不会再续 await scheduler.stop() + await catalog_jobs.manager.shutdown() await run_manager.shutdown() from app.sandbox.manager import sandbox_manager diff --git a/backend/app/providers/factory.py b/backend/app/providers/factory.py index b94ae6d6..e7fbb29b 100644 --- a/backend/app/providers/factory.py +++ b/backend/app/providers/factory.py @@ -28,6 +28,9 @@ class ModelSpec(BaseModel): thinking: str | None = None # off | adaptive effort: str | None = None # low | medium | high | xhigh | max timeout: int | None = None + # SDK 自己遇到 429、5xx、连接错误、超时时重试几次；None 照 SDK 默认（openai、anthropic 都是 2 次）。 + # 自己按失败原因重试的调用方（数据目录起草）设 0，免得两层重试叠在一起、实际次数和报错写的对不上 + max_retries: int | None = None extra: dict[str, Any] = Field(default_factory=dict) @@ -227,6 +230,8 @@ def build_chat_model(provider: Provider, spec: ModelSpec) -> BaseChatModel: # 以前只有节点显式配了才传，而没有任何地方给 spec.timeout 赋过值——所有调用 # 都走 SDK 默认的 10 分钟 kwargs["timeout"] = spec.timeout or settings.model_timeout_seconds + if spec.max_retries is not None: + kwargs["max_retries"] = spec.max_retries # 采样参数：能力允许才传 if spec.temperature is not None and catalog.supports_sampling(model): @@ -314,7 +319,8 @@ def build_chat_model(provider: Provider, spec: ModelSpec) -> BaseChatModel: # 且 api_key=None 时不会发出 x-api-key。 import anthropic - client_args = {"base_url": base_url, "api_key": None, "auth_token": api_key} + client_args = {"base_url": base_url, "api_key": None, "auth_token": api_key, + "max_retries": chat.max_retries} chat._client = anthropic.Anthropic(**client_args) chat._async_client = anthropic.AsyncAnthropic(**client_args) return chat diff --git a/backend/tests/fixtures/catalog/fake_model.py b/backend/tests/fixtures/catalog/fake_model.py index 42bfdec5..1574076e 100644 --- a/backend/tests/fixtures/catalog/fake_model.py +++ b/backend/tests/fixtures/catalog/fake_model.py @@ -3,10 +3,14 @@ model = FakeModel({"visits": {"label": "入园记录", "kind": "fact"}}, fail={"orders"}) await catalog.draft_with_model(model, source, ["visits"]) -- 结构化输出（with_structured_output(...).ainvoke）从提示词里认出这一批有哪些表（「表 <表名>」）， - 返回 {"tables": [{"table": 表名, **replies[表名]}, …]}； +- 结构化输出（with_structured_output(...).ainvoke）从提示词里认出这一段请它写哪些表、每张表写哪部分（requested： + 整张表、只写表级的项、只写列出的几列、表级的项加几列），返回 {"tables": [{"table": 表名, …}, …]}，只给请它写的 + 那部分（整张表给 replies[表名] 的全部；只写几列时只给 replies 里这几列）； +- 照提示词的要求，请它写的每一列都列出来：replies 里没写的列只给列名（{"name": 列名}，字段留空，是拿不准）， + 不整列漏掉。list_all=False 时只给 replies 里有的列（模拟漏列的模型：漏掉的列会被补问一次，补问后仍没给的算没写出来）； - 直接 ainvoke 是纯文本那条路：同样的内容写成一段正文（AIMessage），前面带一句话； -- fail 里的表所在的那一批抛异常，用来验证「一批失败只影响这一批」； +- fail 里的表所在的那一段抛异常，用来验证「一段失败只影响这一段」； +- hang=N：头 N 次调用一直不回（等起草引擎按时限掐掉，当作超时）； - calls 记下每次收到的提示词全文，用来核对交给模型的内容（只有结构，没有数据）；modes 记下每次走的是 structured 还是 text。 @@ -19,15 +23,95 @@ structured、text 模拟实测见过的几种回复（两条路各自设）： """ from __future__ import annotations +import asyncio import json from typing import Any +from app.data.catalog import CATALOG_DRAFT_PROMPT_LABELS as _P + +#: 表名下一行写着「这次只写…」时，请模型写的是哪一部分（data/catalog_draft.py 按这几句写提示词） +_ONLY_TABLE = _P["only_table"] +_ONLY_COLUMNS = _P["only_columns"].split("{")[0] +_TABLE_AND = _P["table_and_columns"].split("{")[0] + + +def requested(text: str) -> dict[str, tuple[str, list[str] | None]]: + """提示词里请模型写哪些表、每张表写哪部分：{表名: (哪部分, 列)}。哪部分是 full（整张表）、table（只写表级的项）、 + columns（只写列出的列，列是表结构描述里列出的那几列）、table+columns（表级的项加「这次只写」那一行点名的列）。""" + out: dict[str, tuple[str, list[str] | None]] = {} + for chunk in text.split("\n\n"): + lines = chunk.split("\n") + head = lines[0] + if not head.startswith("表 "): + continue + name = head[2:] + if name.endswith("（视图）"): + name = name[:-len("（视图）")] + second = lines[1] if len(lines) > 1 else "" + listed = [line[2:].split(" ")[0] for line in lines if line.startswith("- ")] + if second.startswith(_ONLY_TABLE): + out[name] = ("table", None) + elif second.startswith(_ONLY_COLUMNS): + out[name] = ("columns", listed) + elif second.startswith(_TABLE_AND): + out[name] = ("table+columns", second.split("：", 1)[1].split("、")) + else: + out[name] = ("full", None) + return out + + +def answer(reply: dict, name: str, part: str, columns: list[str] | None) -> dict: + """按请它写的那部分，从 replies[表名] 里取。""" + if part == "full": + return {"table": name, **reply} + out: dict[str, Any] = {"table": name} + if part in ("table", "table+columns"): + out.update({k: v for k, v in reply.items() if k != "columns"}) + if part in ("columns", "table+columns"): + wanted = set(columns or []) + out["columns"] = [c for c in reply.get("columns") or [] if isinstance(c, dict) and c.get("name") in wanted] + return out + + +def listed(text: str) -> dict[str, list[str]]: + """提示词里每张表的结构描述列出了哪些列（「- 列名 类型」那几行）。""" + out: dict[str, list[str]] = {} + for chunk in text.split("\n\n"): + lines = chunk.split("\n") + if not lines[0].startswith("表 "): + continue + name = lines[0][2:] + if name.endswith("（视图）"): + name = name[:-len("（视图）")] + out[name] = [line[2:].split(" ")[0] for line in lines if line.startswith("- ")] + return out + + +def wanted_columns(text: str, name: str, part: str, columns: list[str] | None) -> list[str]: + """这一段请它写这张表的哪几列：整张表是结构描述里的全部列；只写表级的项时没有。""" + if part == "full": + return listed(text).get(name, []) + if part in ("columns", "table+columns"): + return list(columns or []) + return [] + + +def with_every_column(entry: dict, wanted: list[str]) -> dict: + """请写的列都列出来：没写的只给列名（字段留空）。""" + have = {c.get("name") for c in entry.get("columns") or [] if isinstance(c, dict)} + missing = [{"name": c} for c in wanted if c not in have] + if not missing: + return entry + return {**entry, "columns": list(entry.get("columns") or []) + missing} + class FakeModel: def __init__(self, replies: dict[str, dict], fail: set[str] = frozenset(), *, - structured: str = "tables", text: str = "tables") -> None: + structured: str = "tables", text: str = "tables", list_all: bool = True, hang: int = 0) -> None: self.replies, self.fail, self.calls = replies, set(fail), [] self.structured, self.text = structured, text + self.list_all = list_all + self.hang = hang self.modes: list[str] = [] def with_structured_output(self, schema: Any, **_: Any) -> "_Structured": @@ -37,24 +121,36 @@ class FakeModel: async def ainvoke(self, messages: list[Any]) -> Any: from langchain_core.messages import AIMessage - reply = self._reply(messages, "text") + reply = await self._answer(messages, "text") return AIMessage(content="草稿如下：" + json.dumps(reply, ensure_ascii=False)) + async def _answer(self, messages: list[Any], mode: str) -> dict: + if self.hang > 0: + self.hang -= 1 + self.calls.append("\n".join(str(getattr(m, "content", m)) for m in messages)) + self.modes.append(mode) + await asyncio.sleep(3600) + return self._reply(messages, mode) + def _reply(self, messages: list[Any], mode: str) -> dict: text = "\n".join(str(getattr(m, "content", m)) for m in messages) self.calls.append(text) self.modes.append(mode) - names = [t for t in self.replies if f"表 {t}\n" in text or f"表 {t}（" in text] - if any(t in self.fail for t in names): + asked = requested(text) + names = [t for t in asked if t in self.replies] + if any(t in self.fail for t in asked): raise RuntimeError("网关超时") shape = self.structured if mode == "structured" else self.text if shape == "error": raise NotImplementedError("这条路用不了") if shape == "hollow": return {"tables": [{"table": t} for t in names]} + answers = [answer(self.replies[t], t, *asked[t]) for t in names] + if self.list_all: + answers = [with_every_column(a, wanted_columns(text, t, *asked[t])) for a, t in zip(answers, names)] if shape == "single": - return {"table": names[0], **self.replies[names[0]]} if names else {} - return {"tables": [{"table": t, **self.replies[t]} for t in names]} + return answers[0] if answers else {} + return {"tables": answers} class _Structured: @@ -62,4 +158,4 @@ class _Structured: self.owner = owner async def ainvoke(self, messages: list[Any]) -> dict: - return self.owner._reply(messages, "structured") + return await self.owner._answer(messages, "structured") diff --git a/backend/tests/test_catalog_draft.py b/backend/tests/test_catalog_draft.py index 96162564..81a5629b 100644 --- a/backend/tests/test_catalog_draft.py +++ b/backend/tests/test_catalog_draft.py @@ -67,6 +67,48 @@ def test_long_table_comment_goes_to_description(): assert draft.notes["description"]["value"].startswith("游客每次检票入园") +@pytest.mark.parametrize("comment, role", [ + ("实收金额", "label"), + ("Paid amount", "label"), # 带空格的英文短语照旧当名字 + ("状态：1 表示有效，0 表示作废", "note"), + ("paid_amt", None), ("PAID-AMT", None), ("Paid Amt", None), # 和列名相同（忽略大小写、下划线、连字符、空白） + ("member_level_code", None), ("ord.status", None), ("GMV", None), ("v2", None), # 标识符写法 + ("member-level", "label"), # 带连字符、和名字不同：不是标识符写法 +]) +def test_comment_role_by_text_and_name(comment, role): + assert catalog.comment_role(comment, "paid_amt") == role + + +def test_a_comment_that_is_just_the_name_or_an_identifier_is_not_a_label(): + """注释就是表名本身（列注释就是列名）、或者是标识符写法：不当中文名，也不当说明（不然它比模型起草可信，模型起草的 + 中文名、含义永远进不了目录）；交给模型的表结构里照旧有这条注释。""" + src = _source({"member_card": _table( + _col("id"), _col("card_no", "TEXT", comment="CARD-NO"), _col("open_dt", "TEXT", comment="open_date"), + _col("bal", "REAL", comment="卡内余额"), comment="MEMBER CARD")}) + draft = catalog.draft_structure(src, "member_card") + assert "label" not in draft.notes and "description" not in draft.notes + assert draft.notes["columns"] == {"bal": {"label": make_item("卡内余额", "comment")}} + assert "comment" in draft.covered + # schema 前缀不算：注释写的是不带前缀的表名，同样不当中文名 + src = _source({"core.member_card": _table(_col("id"), comment="MEMBER_CARD")}) + assert "label" not in catalog.draft_structure(src, "core.member_card").notes + + +async def test_a_redraft_drops_an_old_label_that_was_just_the_name_and_takes_the_model_one(): + """以前按老规矩把「注释就是表名」记成了中文名（注释的级别高于模型起草）：以后起草时这条注释不再当中文名，旧的那一项 + 随这一轮的注释一起删掉，模型起草的中文名就能进目录。已有目录不另做迁移，起草到这张表时才变。""" + tables = {"member_card": _table(_col("id"), _col("bal", "REAL"), comment="MEMBER_CARD", fks=[])} + row = await _db_source(tables) + async with SessionLocal() as session: + await catalog.write_entry(session, row.id, "member_card", {"label": make_item("MEMBER_CARD", "comment")}, + if_version=0, actor=None) + report = await catalog.draft_catalog(session, row, tables=["member_card"], use_model=True, + model=FakeModel({"member_card": {"label": "会员卡"}})) + entry = await catalog.read_entry(session, row.id, "member_card") + assert entry.notes["label"]["value"] == "会员卡" and entry.notes["label"]["source"] == "llm" + assert (report.tables[0].added, report.tables[0].removed) == (1, 1) + + @pytest.mark.parametrize("kw", [{"origin": "upload"}, {"import_mode": "recipe"}]) def test_imported_table_comments_are_not_copied(kw): """导入表格的说明是按核对结果生成的（recipe_notes），数据目录不重复它们。""" @@ -274,11 +316,15 @@ async def test_model_single_table_object_without_wrapper_is_recognized(): async def test_hollow_structured_output_retries_with_text(): - """结构化输出只给了 {"tables":[{"table":"visits"}]} 这样的空壳：改走纯文本再问一次，用纯文本给的内容。""" - model = FakeModel({"visits": {"label": "入园记录", "kind": "fact"}}, structured="hollow", text="tables") + """结构化输出只给了 {"tables":[{"table":"visits"}]} 这样的空壳、这一段又拆不开（不到 10 列）：改走纯文本再问一次， + 用纯文本给的内容。纯文本只给了表级的项、一列都没列出来：再补问一次没给的列（只问缺的，data/catalog_draft.py）； + 补问后仍没给，这几列算没写出来，表是只写出一部分（旧的模型起草项不删）。""" + model = FakeModel({"visits": {"label": "入园记录", "kind": "fact"}}, structured="hollow", text="tables", + list_all=False) got = await catalog.draft_with_model(model, _source(_scenic_tables()), ["visits"]) - assert model.modes == ["structured", "text"] + assert model.modes == ["structured", "text", "text"] assert got["visits"].notes == {"label": make_item("入园记录", "llm"), "kind": make_item("fact", "llm")} + assert got["visits"].covered == frozenset() and "有 5 列（id 到 status）没写出来" in got["visits"].error async def test_hollow_structured_then_single_text_object(): diff --git a/backend/tests/test_catalog_draft_engine.py b/backend/tests/test_catalog_draft_engine.py new file mode 100644 index 00000000..4a1a94aa --- /dev/null +++ b/backend/tests/test_catalog_draft_engine.py @@ -0,0 +1,1301 @@ +"""数据目录模型起草的引擎（data/catalog_draft.py）：分段、按失败原因处理、捡回被截断的部分、起草要求、用量。 + +用假模型复刻实测查明的几种失败（不调用真实模型）： +- 思考加输出用完额度：finish_reason=length 且正文为空（只有思考）、LengthFinishReasonError → 拆成两半再问； +- 只给空壳 → 拆开再问；拆不开的换纯文本问一次，写出来了这次起草后面都走纯文本； +- 被截断的 JSON：已经完整的表、列对象捡回来，只补剩下的； +- 只回一部分表：只补缺的，补一次； +- 服务方嫌额度太大 → 退回 8192，后面都用它；结构化输出不支持 → 后面改走纯文本； +- 网络错误、5xx → 退避重试，最多 2 次，报错写明重试过几次；超时当作这一段太大，拆小再问，不原样重试。 + +验收后修正的几条（用假模型复现）：宽表的列分段均分、没有很小的尾段；拆不开的空壳纯文本也再问一次；额度用完或超时 +累计 2 次后新开始的段列数减半；一段最多调用 8 次；最近 6 次调用 4 次没写出来时提醒、少于 2 次时清掉；记超时和没写 +出来的调用；模型留空的列计数；窄表不被宽表的列分段插队；原因写清是哪几段、哪几列。 + +表名、列名、说明文字都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import inspect +import json +from dataclasses import dataclass, field +from types import SimpleNamespace +from typing import Any, Callable + +import pytest +from langchain_core.messages import AIMessage + +from app.data import catalog +from app.data import catalog_draft as D +from tests.fixtures.catalog.fake_model import answer, requested + + +# -------------------------------------------------------------------------- +# 夹具 +# -------------------------------------------------------------------------- + + +def _col(name: str, type_: str = "INTEGER") -> dict: + return {"name": name, "type": type_, "nullable": True, "comment": None} + + +def _table(n: int, prefix: str = "c", comment: str | None = None) -> dict: + return {"qualified": "", "schema": None, "columns": [_col(f"{prefix}{i:03d}") for i in range(1, n + 1)], + "primary_key": [f"{prefix}001"], "comment": comment, "is_view": False, "foreign_keys": []} + + +def _source(tables: dict, **kw) -> SimpleNamespace: + for name, meta in tables.items(): + meta["qualified"] = name + return SimpleNamespace(id="src", name="shop", kind="sqlite", origin="manual", + description=kw.pop("description", ""), schema_cache={"tables": tables}) + + +def _reply(table: str, meta: dict) -> dict: + """一张表的完整草稿：表级三项 + 每列中文名和度量类型。""" + return {"label": f"{table}的中文名", "grain": "每条记录一行", "kind": "fact", + "columns": [{"name": c["name"], "label": f"{c['name']}的中文名", "measure": "attribute"} + for c in meta["columns"]]} + + +@dataclass +class Reply: + """一次回复。obj 是正常的 JSON 对象；text 给了就原样作正文（截断的 JSON 之类）。""" + + obj: Any = None + text: str | None = None + finish: str = "stop" + thinking: str = "" + usage: tuple[int, int] = (0, 0) + + def render(self, mode: str, include_raw: bool) -> Any: + if self.text is not None: + content = self.text + else: + content = json.dumps(self.obj, ensure_ascii=False) if self.obj is not None else "" + msg = AIMessage( + content=content, + additional_kwargs={"reasoning_content": self.thinking} if self.thinking else {}, + response_metadata={"finish_reason": self.finish}, + usage_metadata={"input_tokens": self.usage[0], "output_tokens": self.usage[1], + "total_tokens": sum(self.usage)}, + ) + if mode == "text": + return msg + parsed = self.obj if self.text is None else None + if not include_raw: + return parsed + return {"raw": msg, "parsed": parsed, "parsing_error": None if parsed is not None else ValueError("坏的")} + + +@dataclass +class Call: + mode: str + text: str + asked: dict + n: int + + +class Scripted: + """每次调用交给 handler(call) 决定怎么回：返回 Reply，或者抛异常；handler 可以是协程。""" + + def __init__(self, handler: Callable[[Call], Any], name: str = "scripted-model") -> None: + self.handler, self.model_name = handler, name + self.calls: list[Call] = [] + + def with_structured_output(self, schema: Any, include_raw: bool = False, **_: Any) -> Any: + owner = self + + class _S: + async def ainvoke(self, messages): + return await owner._run(messages, "structured", include_raw) + + return _S() + + async def ainvoke(self, messages: list[Any]) -> Any: + return await self._run(messages, "text", False) + + async def _run(self, messages: list[Any], mode: str, include_raw: bool) -> Any: + text = "\n".join(str(getattr(m, "content", m)) for m in messages) + call = Call(mode=mode, text=text, asked=requested(text), n=len(self.calls)) + self.calls.append(call) + out = self.handler(call) + if inspect.isawaitable(out): + out = await out + return out.render(mode, include_raw) + + +def answering(tables: dict, **kw) -> Callable[[Call], Reply]: + """照请求写：每张表按请它写的那部分给出 _reply 的内容。""" + def handle(call: Call) -> Reply: + return Reply(obj={"tables": [answer(_reply(t, tables[t]), t, *call.asked[t]) for t in call.asked]}, **kw) + return handle + + +class _Status(Exception): + def __init__(self, status: int, text: str) -> None: + super().__init__(text) + self.status_code = status + + +class LengthFinishReasonError(Exception): + """和 openai 的同名异常一样：解析结构化输出时发现 finish_reason=length。""" + + def __init__(self) -> None: + super().__init__("Could not parse response content as the length limit was reached") + self.completion = None + + +class ConnectError(Exception): + pass + + +@pytest.fixture(autouse=True) +def _no_wait(monkeypatch): + """退避不真等：记下要等多久。""" + waited: list[float] = [] + + async def fake_sleep(seconds: float) -> None: + waited.append(seconds) + + monkeypatch.setattr(D, "_sleep", fake_sleep) + return waited + + +def _status(outcomes: dict) -> dict: + return {t: o.status for t, o in outcomes.items()} + + +def _hollow(call: Call) -> Reply: + return Reply(obj={"tables": [{"table": t} for t in call.asked]}) + + +def _exhausted(seg: str = "这一段", budget: int = 32768) -> str: + return D.REASON_EXHAUSTED.format(budget=budget, seg=seg) + + +EXHAUSTED = Reply(text="", finish="length", thinking="……") + + +class Reports: + """进度回调 call_done 收到的每次调用的结果。""" + + def __init__(self) -> None: + self.items: list[D.CallReport] = [] + + async def __call__(self, report: D.CallReport) -> None: + self.items.append(report) + + def hooks(self) -> D.Hooks: + return D.Hooks(call_done=self) + + @property + def failures(self) -> list[str | None]: + return [r.failure for r in self.items] + + +# -------------------------------------------------------------------------- +# 思考用完额度、空壳：拆开再问 +# -------------------------------------------------------------------------- + + +async def test_thinking_used_up_the_budget_splits_the_segment(): + tables = {"orders_x": _table(24)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + if call.n == 0: # 只想不写：正文为空、只有思考，额度用完 + return Reply(text="", finish="length", thinking="先逐列想一遍……") + return ok(call) + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["orders_x"]) + o = got["orders_x"] + assert o.status == "ok" and o.error is None and o.items == 3 + 24 * 2 + assert len(model.calls) == 3 + cols = [f"c{i:03d}" for i in range(1, 25)] + assert model.calls[1].asked == {"orders_x": ("table+columns", cols[:12])} + assert model.calls[2].asked == {"orders_x": ("columns", cols[12:])} + # 后半段在前半段之后问，带着刚起草的表级内容做背景 + assert catalog.CATALOG_DRAFT_PROMPT_LABELS["drafted"] in model.calls[2].text + assert "orders_x的中文名" in model.calls[2].text + + +async def test_length_finish_reason_error_also_splits(): + tables = {"orders_x": _table(24)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + if call.n == 0: + raise LengthFinishReasonError() + return ok(call) + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["orders_x"]) + assert got["orders_x"].status == "ok" and len(model.calls) == 3 + + +async def test_hollow_shell_splits_the_pack_by_table(): + tables = {"a_tab": _table(6), "b_tab": _table(6)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + if call.n == 0: + return Reply(obj={"tables": [{"table": "a_tab"}, {"table": "b_tab"}]}) + return ok(call) + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["a_tab", "b_tab"]) + assert _status(got) == {"a_tab": "ok", "b_tab": "ok"} + assert [set(c.asked) for c in model.calls] == [{"a_tab", "b_tab"}, {"a_tab"}, {"b_tab"}] + assert all(c.asked[t][0] == "full" for c in model.calls[1:] for t in c.asked) + + +async def test_splitting_stops_at_three_levels_and_five_columns(monkeypatch): + monkeypatch.setattr(catalog, "MODEL_PART_MAX_CALLS", 99) + tables = {"wide40": _table(40)} + model = Scripted(lambda call: EXHAUSTED) + got = await D.run_model_draft(model, _source(tables), ["wide40"]) + o = got["wide40"] + assert len(model.calls) == 1 + 2 + 4 + 8, "40 → 20 → 10 → 5 列，最多拆三层" + assert min(len(c.asked["wide40"][1] or []) for c in model.calls[1:]) == 5 + assert o.status == "failed" and o.items == 0 + assert o.error == "模型的思考和输出用完了额度（32768 个 token），这一段没写出来；已拆小重试仍没写出" + + small = Scripted(lambda call: Reply(text="", finish="length")) + got = await D.run_model_draft(small, _source({"t9": _table(9)}), ["t9"]) + assert len(small.calls) == 1, "不到 10 列拆不出两段 5 列" + assert got["t9"].error == "模型的思考和输出用完了额度（32768 个 token），这一段没写出来" + + +async def test_one_part_makes_at_most_eight_calls(monkeypatch): + """一段（连同它拆出来的、补问的、重试的）最多调用 8 次：40 列一直用完额度，拆到第 8 次就停，剩下没问的记没写出来、 + 原因写清。以前一段 40 列拆到底要 15 次。""" + monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99) + assert catalog.MODEL_PART_MAX_CALLS == 8 + model = Scripted(lambda call: EXHAUSTED) + got = await D.run_model_draft(model, _source({"t40": _table(40)}), ["t40"]) + assert len(model.calls) == 8 + o = got["t40"] + assert o.status == "failed" + assert o.error == (_exhausted() + "；每段最多调用 8 次模型，这一段到了次数仍没写完，已停止重试" + + D.REASON_SPLIT) + + # 网络错误的重试也算在这 8 次里 + def flaky(call: Call) -> Reply: + raise ConnectError("All connection attempts failed") + + monkeypatch.setattr(catalog, "MODEL_PART_MAX_CALLS", 2) + model = Scripted(flaky) + got = await D.run_model_draft(model, _source({"t4": _table(4)}), ["t4"]) + assert len(model.calls) == 2 + assert got["t4"].error == "每段最多调用 2 次模型，这一段到了次数仍没写完，已停止重试" + + +async def test_an_unsplittable_hollow_shell_tries_text_once_then_stays_on_text(): + """有的网关的结构化输出只给空壳：拆不开时换纯文本问一次，写出来了，这次起草后面直接走纯文本。 + + 用宽表看先后：表级那段（拆不开）写完之后才问列分段。""" + tables = {"wide45": _table(45)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + if call.mode == "structured": + return Reply(obj={"tables": [{"table": t} for t in call.asked]}) + return ok(call) + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["wide45"]) + assert got["wide45"].status == "ok" and len(got["wide45"].notes["columns"]) == 45 + assert [c.mode for c in model.calls] == ["structured", "text", "text", "text"] + assert catalog.CATALOG_DRAFT_TEXT_PROMPT in model.calls[1].text, "纯文本那条路要说清 JSON 的格式" + + +def _text_only(handle_text: Callable[[Call], Reply]) -> Callable[[Call], Reply]: + """结构化输出不支持（这次起草一开始就改走纯文本），纯文本交给 handle_text。""" + def handle(call: Call) -> Reply: + if call.mode == "structured": + raise _Status(400, "This response_format type is unavailable now") + return handle_text(call) + return handle + + +async def test_an_unsplittable_hollow_shell_is_asked_again_in_text_mode_too(): + """已经走纯文本时，拆不开的段（不到 10 列）只给空壳：原样再问一次，写出来就用；仍是空壳才记没写出来。 + 以前纯文本模式下不补问，整张表直接记没写出来。""" + tables = {"t8": _table(8)} + ok = answering(tables) + model = Scripted(_text_only(lambda call: _hollow(call) if call.n == 1 else ok(call))) + got = await D.run_model_draft(model, _source(tables), ["t8"]) + assert [c.mode for c in model.calls] == ["structured", "text", "text"] + assert got["t8"].status == "ok" and len(got["t8"].notes["columns"]) == 8 + + model = Scripted(_text_only(_hollow)) + got = await D.run_model_draft(model, _source(tables), ["t8"]) + assert [c.mode for c in model.calls] == ["structured", "text", "text"], "再问一次，不多问" + assert got["t8"].status == "failed" and got["t8"].error == catalog.MODEL_EMPTY_REASON + + +# -------------------------------------------------------------------------- +# 截断、缺表、缺列 +# -------------------------------------------------------------------------- + + +async def test_truncated_json_keeps_complete_columns_and_asks_only_the_rest(): + tables = {"t12": _table(12)} + full = json.dumps({"tables": [{"table": "t12", **_reply("t12", tables["t12"])}]}, ensure_ascii=False) + cut = full[:full.index('"c006"') + 8] # 第 6 列写到一半 + ok = answering(tables) + + def handle(call: Call) -> Reply: + if call.n == 0: + return Reply(text=cut, finish="length") + return ok(call) + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["t12"]) + o = got["t12"] + assert o.status == "ok" and set(o.notes["columns"]) == {f"c{i:03d}" for i in range(1, 13)} + assert o.notes["label"]["value"] == "t12的中文名" + assert len(model.calls) == 2 + assert model.calls[1].asked == {"t12": ("columns", [f"c{i:03d}" for i in range(6, 13)])} + + +async def test_truncated_tool_call_arguments_are_salvaged_too(): + """结构化输出走工具调用时，截断的参数在 invalid_tool_calls 里（一段字符串）。""" + tables = {"t12": _table(12)} + full = json.dumps({"tables": [{"table": "t12", **_reply("t12", tables["t12"])}]}, ensure_ascii=False) + cut = full[:full.index('"c004"') + 3] # 第 4 列的列名写到一半 + ok = answering(tables) + + class ToolModel(Scripted): + async def _run(self, messages, mode, include_raw): + if not self.calls: + self.calls.append(Call(mode, "", {}, 0)) + raw = AIMessage(content="", invalid_tool_calls=[{"name": "catalog_draft", "args": cut, "id": "x", + "error": None, "type": "invalid_tool_call"}], + response_metadata={"finish_reason": "length"}) + return {"raw": raw, "parsed": None, "parsing_error": ValueError("截断")} + return await super()._run(messages, mode, include_raw) + + model = ToolModel(ok) + got = await D.run_model_draft(model, _source(tables), ["t12"]) + assert got["t12"].status == "ok" + assert model.calls[1].asked == {"t12": ("columns", [f"c{i:03d}" for i in range(4, 13)])} + + +async def test_missing_tables_are_asked_again_once(): + """缺了的表只补缺的、补一次；补问时整张表还是一项都没有，按只给空壳处理（拆不开就换纯文本问一次）。""" + tables = {"a_tab": _table(4), "b_tab": _table(4), "c_tab": _table(4)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + reply = ok(call) + keep = {0: {"a_tab"}, 1: {"b_tab"}}.get(call.n, set()) + reply.obj["tables"] = [e for e in reply.obj["tables"] if e["table"] in keep] + return reply + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["a_tab", "b_tab", "c_tab"]) + assert len(model.calls) == 3 + assert model.calls[1].asked == {"b_tab": ("full", None), "c_tab": ("full", None)} + assert model.calls[2].asked == {"c_tab": ("full", None)} and model.calls[2].mode == "text" + assert _status(got) == {"a_tab": "ok", "b_tab": "ok", "c_tab": "failed"} + assert got["c_tab"].error == "模型的回复里没有这张表" + + +async def test_a_table_still_missing_after_the_followup_is_split_like_a_hollow_shell(): + """补问时整张表还是一项都没有：照第一次的规则拆小（这里按列拆成两半），不直接记失败。""" + tables = {"a_tab": _table(4), "b_tab": _table(12)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + reply = ok(call) + if call.n < 2: + reply.obj["tables"] = [e for e in reply.obj["tables"] if e["table"] == "a_tab"] + return reply + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["a_tab", "b_tab"]) + assert _status(got) == {"a_tab": "ok", "b_tab": "ok"} + cols = [f"c{i:03d}" for i in range(1, 13)] + assert [c.asked for c in model.calls[2:]] == [{"b_tab": ("table+columns", cols[:6])}, + {"b_tab": ("columns", cols[6:])}] + + +async def test_columns_still_missing_after_the_followup_are_not_written(): + """补问过仍然没列出来的列算没写出来：表是 partial，说清哪部分没写出来（不算拿不准，旧的模型起草项不删）。""" + tables = {"t8": _table(8)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + reply = ok(call) + for entry in reply.obj["tables"]: + entry["columns"] = [c for c in entry.get("columns", []) if c["name"] not in ("c007", "c008")] + return reply + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["t8"]) + assert len(model.calls) == 2 and model.calls[1].asked == {"t8": ("columns", ["c007", "c008"])} + o = got["t8"] + assert o.status == "partial" + assert o.error == ("这张表 8 列，有 2 列（c007、c008）没写出来：模型的回复里漏了这些列，补问一次仍没有给出。" + "可以只重试这张表") + assert set(o.notes["columns"]) == {f"c{i:03d}" for i in range(1, 7)} + assert o.blank == [], "没写出来的列不算留空" + compat = (await catalog.draft_with_model(Scripted(handle), _source(tables), ["t8"]))["t8"] + assert compat.covered == frozenset(), "没写全的不算完整起草过" + + +async def test_columns_listed_blank_in_the_followup_are_left_blank_not_failed(): + """补问时模型把漏的列列出来了、字段却留空：是拿不准（记留空），不算「漏了这些列」。""" + tables = {"t8": _table(8)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + reply = ok(call) + entry = reply.obj["tables"][0] + if call.n == 0: + entry["columns"] = [c for c in entry["columns"] if c["name"] not in ("c003", "c007")] + else: + entry["columns"] = [{"name": c["name"]} for c in entry["columns"]] + return reply + + model, reports = Scripted(handle), Reports() + got = await D.run_model_draft(model, _source(tables), ["t8"], hooks=reports.hooks()) + assert model.calls[1].asked == {"t8": ("columns", ["c003", "c007"])} + o = got["t8"] + assert o.status == "ok" and o.error is None and o.blank == ["c003", "c007"] + # 补问的那次把请写的列都列出来了（字段留空是拿不准）：用量和提醒里也不算一次「只给空壳」 + assert reports.failures == [None, None] + + +async def test_a_followup_that_comes_back_as_an_empty_shell_leaves_the_table_partial(): + """40 列的表第一次只回了 20 列，补问只给一个空壳：没写出来的 20 列照实记下，表是 partial。""" + tables = {"t40": _table(40)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + if call.n == 0: + reply = ok(call) + reply.obj["tables"][0]["columns"] = reply.obj["tables"][0]["columns"][:20] + return reply + return Reply(obj={"tables": [{"table": "t40"}]}) + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["t40"]) + o = got["t40"] + assert len(model.calls) == 2 + assert model.calls[1].asked == {"t40": ("columns", [f"c{i:03d}" for i in range(21, 41)])} + assert o.status == "partial" and len(o.notes["columns"]) == 20 + assert o.error == ("这张表 40 列，有 20 列（c021 到 c040）没写出来：模型的回复里漏了这些列，补问一次仍没有给出。" + "可以只重试这张表") + + +async def test_columns_listed_with_empty_fields_are_left_to_the_model(): + """列出了、字段留空的列是模型拿不准：不补问，也不算没写出来；记成留空（表的状态不因此变）。""" + tables = {"t8": _table(8)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + reply = ok(call) + for entry in reply.obj["tables"]: + entry["columns"] = [c if c["name"] not in ("c007", "c008") else {"name": c["name"]} + for c in entry.get("columns", [])] + return reply + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["t8"]) + assert len(model.calls) == 1 + assert got["t8"].status == "ok" and got["t8"].error is None + assert set(got["t8"].notes["columns"]) == {f"c{i:03d}" for i in range(1, 7)} + assert got["t8"].blank == ["c007", "c008"] + + +async def test_blank_columns_are_the_ones_without_a_label_or_meaning(): + """留空的列：请模型写了、中文名和含义却都没给（只给了度量类型、单位，或者给的都没通过校验）。已确认的、库注释已经说清 + 的不算：给模型的提示词里它们以此为准，模型不写也不缺。""" + meta = _table(9) + meta["columns"][5]["comment"] = "会员等级" # c006：库注释就是中文名 + meta["columns"][8]["comment"] = "C009" # c009：注释就是列名，说明不了什么 + tables = {"t9": meta} + existing = {"t9": {"columns": {"c005": {"label": catalog.make_item("已确认的名字", "human")}}}} + + def handle(call: Call) -> Reply: + return Reply(obj={"tables": [{"table": "t9", "label": "某表", "columns": [ + {"name": "c001", "label": "第一列"}, + {"name": "c002", "meaning": "第二列的含义"}, + {"name": "c003", "measure": "attribute", "unit": "元"}, # 中文名、含义都没给 + {"name": "c004", "label": "=c001+c002"}, # 像公式：校验丢掉 + {"name": "c005"}, {"name": "c006"}, {"name": "c007"}, {"name": "c008"}, {"name": "c009"}]}]}) + + got = await D.run_model_draft(Scripted(handle), _source(tables), ["t9"], existing=existing) + o = got["t9"] + assert o.status == "ok" and o.error is None + assert o.blank == ["c003", "c004", "c007", "c008", "c009"] + + +# -------------------------------------------------------------------------- +# 额度被拒、结构化输出不支持、网络错误、超时 +# -------------------------------------------------------------------------- + + +async def test_budget_rejection_falls_back_to_8192_for_the_rest(): + tables = {"wide45": _table(45)} + + def refuse(call: Call) -> Reply: + raise _Status(400, "max_tokens: 32768 > 8192, which is the maximum allowed number of output tokens") + + big, small = Scripted(refuse), Scripted(answering(tables)) + model = catalog.DraftModel(big, small, "m") + got = await D.run_model_draft(model, _source(tables), ["wide45"]) + assert got["wide45"].status == "ok" + assert len(big.calls) == 1 and len(small.calls) == 3, "退回之后，表级那段重来、两个列分段都用小额度" + assert model.budget == 8192 + + +async def test_exhausted_message_names_the_budget_actually_used(): + def refuse(call: Call) -> Reply: + raise _Status(400, "max_tokens is too large") + + model = catalog.DraftModel(Scripted(refuse), Scripted(lambda c: Reply(text="", finish="length")), "m") + got = await D.run_model_draft(model, _source({"t4": _table(4)}), ["t4"]) + assert got["t4"].error == "模型的思考和输出用完了额度（8192 个 token），这一段没写出来" + + +async def test_structured_output_unsupported_switches_to_text_for_the_rest(): + tables = {"wide45": _table(45)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + if call.mode == "structured": + raise _Status(400, "This response_format type is unavailable now") + return ok(call) + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["wide45"]) + assert got["wide45"].status == "ok" + assert [c.mode for c in model.calls] == ["structured", "text", "text", "text"] + + +@pytest.mark.parametrize("thinking", ["", "先想想每一列是什么……"], ids=["plain", "with-thinking"]) +async def test_structured_output_that_comes_back_as_prose_falls_back_to_text(thinking): + """网关接受了结构化输出的参数却没照做（模型写成了一段说明文字）：换纯文本问，不当成额度用完去拆，也不直接判失败。""" + tables = {"t40": _table(40)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + if call.mode == "structured": + return Reply(text="好的。这张表记录的是一笔笔业务，各列的含义如下：第一列是编号……", thinking=thinking) + return ok(call) + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["t40"]) + assert got["t40"].status == "ok" and len(got["t40"].notes["columns"]) == 40 + assert [c.mode for c in model.calls] == ["structured", "text"] + assert catalog.CATALOG_DRAFT_TEXT_PROMPT in model.calls[1].text + + +async def test_prose_on_both_paths_is_a_format_failure_not_a_budget_one(): + def handle(call: Call) -> Reply: + return Reply(text="这张表看起来是业务明细，具体含义需要结合业务确认。", thinking="……") + + model = Scripted(handle) + got = await D.run_model_draft(model, _source({"t40": _table(40)}), ["t40"]) + assert [c.mode for c in model.calls] == ["structured", "text"], "格式不对不拆小重试" + assert got["t40"].status == "failed" and got["t40"].error == "模型的回复不是约定的格式" + + +async def test_a_model_without_structured_output_goes_to_text(): + class TextOnly(Scripted): + def with_structured_output(self, schema, **_): + raise NotImplementedError + + tables = {"t4": _table(4)} + model = TextOnly(answering(tables)) + got = await D.run_model_draft(model, _source(tables), ["t4"]) + assert got["t4"].status == "ok" and [c.mode for c in model.calls] == ["text"] + + +async def test_network_errors_are_retried_with_backoff(_no_wait): + tables = {"t4": _table(4)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + if call.n < 2: + raise ConnectError("All connection attempts failed") + return ok(call) + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["t4"]) + assert got["t4"].status == "ok" and len(model.calls) == 3 + assert _no_wait == list(catalog.MODEL_RETRY_DELAYS_S) + + +async def test_server_errors_give_up_after_two_retries(): + def handle(call: Call) -> Reply: + raise _Status(503, "Service Unavailable") + + model = Scripted(handle) + got = await D.run_model_draft(model, _source({"t4": _table(4)}), ["t4"]) + assert len(model.calls) == 3 + assert got["t4"].status == "failed" + assert got["t4"].error == "调用模型出错：Service Unavailable（已重试 2 次）" + + +async def test_other_errors_are_not_retried(): + def handle(call: Call) -> Reply: + raise RuntimeError("网关拒绝了这次请求") + + model = Scripted(handle) + got = await D.run_model_draft(model, _source({"t4": _table(4)}), ["t4"]) + assert len(model.calls) == 1 and got["t4"].error == "调用模型出错：网关拒绝了这次请求" + + +async def test_a_call_past_the_time_limit_is_split_not_retried(monkeypatch, _no_wait): + """超时当作这一段太大：不原样重试同样大小的段，拆成两半再问（同额度用完）。超时的调用拿不到用量，记 0。""" + monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.05) + tables = {"t24": _table(24)} + ok = answering(tables, usage=(100, 50)) + + async def handle(call: Call) -> Reply: + if call.n == 0: + await asyncio.sleep(1) + return ok(call) + + model, reports = Scripted(handle), Reports() + got = await D.run_model_draft(model, _source(tables), ["t24"], hooks=reports.hooks()) + cols = [f"c{i:03d}" for i in range(1, 25)] + assert got["t24"].status == "ok" + assert [c.asked for c in model.calls] == [{"t24": ("full", None)}, {"t24": ("table+columns", cols[:12])}, + {"t24": ("columns", cols[12:])}] + assert reports.failures == ["timeout", None, None] + assert [(r.input_tokens, r.output_tokens) for r in reports.items] == [(0, 0), (100, 50), (100, 50)] + assert _no_wait == [], "不退避重试" + + +async def test_a_small_part_past_the_time_limit_is_not_retried(monkeypatch): + monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.05) + + async def slow(call: Call) -> Reply: + await asyncio.sleep(1) + return Reply(obj={"tables": []}) + + model = Scripted(slow) + got = await D.run_model_draft(model, _source({"t4": _table(4)}), ["t4"]) + assert len(model.calls) == 1, "拆不开、也不原样重试" + assert got["t4"].status == "failed" and got["t4"].error == "模型 0.05 秒内没有写完这一段" + + +# -------------------------------------------------------------------------- +# 分段：宽表拆成表级一段 + 列分段，窄表拼段 +# -------------------------------------------------------------------------- + + +async def test_a_wide_table_is_split_into_a_table_part_and_column_parts(): + tables = {"wide95": _table(95, comment="虚构的宽表")} + model = Scripted(answering(tables)) + done: list[str] = [] + + async def part_done(table: str) -> None: + done.append(table) + + got = await D.run_model_draft(model, _source(tables), ["wide95"], hooks=D.Hooks(part_done=part_done)) + o = got["wide95"] + assert o.status == "ok" and o.parts_total == 4 and o.parts_done == 4 and done == ["wide95"] * 4 + assert len(o.notes["columns"]) == 95 and o.notes["label"]["value"] == "wide95的中文名" + first, *rest = model.calls + assert first.asked == {"wide95": ("table", None)} + assert catalog.CATALOG_DRAFT_PROMPT_LABELS["brief_columns"].format(n=95) in first.text + assert sorted(len(c.asked["wide95"][1]) for c in rest) == [31, 32, 32] + for call in rest: + assert call.asked["wide95"][0] == "columns" + assert "wide95的中文名" in call.text and "虚构的宽表" in call.text, "列分段带着表注释和表级起草结果" + assert {c for call in rest for c in call.asked["wide95"][1]} == {f"c{i:03d}" for i in range(1, 96)} + + +@pytest.mark.parametrize("n, sizes", [(164, [33, 33, 33, 33, 32]), (95, [32, 32, 31]), (41, [21, 20]), + (80, [40, 40]), (81, [27, 27, 27])]) +async def test_column_parts_are_even_without_a_small_tail(monkeypatch, n, sizes): + """列分段数 = 列数 / 40 向上取整，各段列数平均（相差不超过 1）：不再留下几列的尾段（实测会思考的模型常对这种小段只给 + 空壳）。表级那段照旧在最前面。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + tables = {"wide": _table(n)} + model = Scripted(answering(tables)) + got = await D.run_model_draft(model, _source(tables), ["wide"]) + o = got["wide"] + assert o.status == "ok" and o.parts_total == 1 + len(sizes) == D.parts_for(n) + assert model.calls[0].asked == {"wide": ("table", None)} + parts = [c.asked["wide"][1] for c in model.calls[1:]] + assert [len(p) for p in parts] == sizes + assert [c for p in parts for c in p] == [f"c{i:03d}" for i in range(1, n + 1)], "按表结构的顺序连续切开" + + +async def test_a_wide_table_with_one_failed_part_is_partial_and_says_which(): + tables = {"wide95": _table(95)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + part, cols = call.asked["wide95"] + if part == "columns" and "c041" in cols: + raise RuntimeError("网关拒绝") + return ok(call) + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["wide95"]) + o = got["wide95"] + assert o.status == "partial" and len(o.notes["columns"]) == 63 + assert o.error == ("这张表 95 列，分 4 段起草，第 3 段（c033 到 c064，共 32 列）没写出来：调用模型出错：网关拒绝。" + "可以只重试这张表") + compat = (await catalog.draft_with_model(Scripted(handle), _source(tables), ["wide95"]))["wide95"] + assert compat.covered == frozenset() and compat.error == o.error, "没写全的不算完整起草过，旧的模型项不删" + + +async def test_reasons_name_each_part_and_say_these_parts(monkeypatch): + """几段因为同一个原因没写出来：原因里写「这几段」；每段写上列的范围（列名照表结构原样）；拆过的才补「已拆小重试」。""" + monkeypatch.setattr(catalog, "MODEL_SHRINK_AFTER", 99) + tables = {"wide95": _table(95)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + part, cols = call.asked["wide95"] + if part == "columns" and cols[0] >= "c033": # 第 3、4 段（c033 起）一直用完额度 + return EXHAUSTED + return ok(call) + + got = await D.run_model_draft(Scripted(handle), _source(tables), ["wide95"]) + o = got["wide95"] + assert o.status == "partial" and len(o.notes["columns"]) == 32 + assert o.error == ("这张表 95 列，分 4 段起草，第 3 段（c033 到 c064，共 32 列）、第 4 段（c065 到 c095，共 31 列）没写出来：" + + _exhausted("这几段") + D.REASON_SPLIT + "。可以只重试这张表") + + +async def test_a_part_that_was_never_halved_does_not_say_it_was(): + """被截断后只补剩下的（不是拆小）：补的那次也用完了额度、又拆不开，原因里不写「已拆小重试」。""" + tables = {"t9": _table(9)} + full = json.dumps({"tables": [{"table": "t9", **_reply("t9", tables["t9"])}]}, ensure_ascii=False) + cut = full[:full.index('"c005"') + 8] + + def handle(call: Call) -> Reply: + return Reply(text=cut, finish="length") if call.n == 0 else EXHAUSTED + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["t9"]) + o = got["t9"] + assert len(model.calls) == 2 and model.calls[1].asked == {"t9": ("columns", [f"c{i:03d}" for i in range(5, 10)])} + assert o.status == "partial" + assert o.error == "这张表 9 列，有 5 列（c005 到 c009）没写出来：" + _exhausted() + "。可以只重试这张表" + + +@pytest.mark.parametrize("missing, said", [ + (["c004"], "有 1 列（c004）"), + (["c003", "c007"], "有 2 列（c003、c007）"), + (["c002", "c004", "c006", "c008"], "有 4 列（c002、c004、c006 等）"), + (["c003", "c004", "c005", "c006"], "有 4 列（c003 到 c006）"), +]) +async def test_columns_not_written_are_named(missing, said): + tables = {"t8": _table(8)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + reply = ok(call) + for entry in reply.obj["tables"]: + entry["columns"] = [c for c in entry.get("columns", []) if c["name"] not in missing] + return reply + + got = await D.run_model_draft(Scripted(handle), _source(tables), ["t8"]) + assert got["t8"].error == f"这张表 8 列，{said}没写出来：{D.REASON_UNANSWERED}。可以只重试这张表" + + +async def test_tables_are_packed_up_to_40_columns_and_4_tables(): + widths = {"ta": 15, "tb": 15, "tc": 15, "td": 5, "te": 5, "tf": 5, "tg": 5, "th": 41} + tables = {name: _table(n) for name, n in widths.items()} + model = Scripted(answering(tables)) + got = await D.run_model_draft(model, _source(tables), list(widths)) + assert set(_status(got).values()) == {"ok"} + asked = sorted((tuple(sorted(c.asked)), c.asked[next(iter(c.asked))][0]) for c in model.calls) + assert asked == sorted([ + (("ta", "tb"), "full"), (("tc", "td", "te", "tf"), "full"), (("tg",), "full"), + (("th",), "table"), (("th",), "columns"), (("th",), "columns"), + ]) + + +# -------------------------------------------------------------------------- +# 报错去重、止损、几次起草轮流用模型 +# -------------------------------------------------------------------------- + + +async def test_the_same_reason_is_said_once_even_when_some_parts_were_split(monkeypatch): + """宽表的表级那段拆不开、列分段拆过：原因只说一次，拆过的在最后补一句。""" + monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99) + monkeypatch.setattr(catalog, "MODEL_PART_MAX_CALLS", 99) + model = Scripted(_hollow) + got = await D.run_model_draft(model, _source({"wide45": _table(45)}), ["wide45"]) + o = got["wide45"] + assert o.status == "failed" + assert o.error == ("这张表 45 列，分 3 段起草，第 1 段（表级的项）、第 2 段（c001 到 c023，共 23 列）、" + "第 3 段（c024 到 c045，共 22 列）都没写出来：模型只给了空壳，没有写出可用的内容；已拆小重试仍没写出") + + +def test_reasons_are_joined_by_their_base(): + """(原因, 第几段)：同样的原因只说一次；同一个原因落在几段上写「这几段」，同一段拆出来的几块仍是「这一段」。""" + W = D._Why + ex = D.REASON_EXHAUSTED.format(budget=32768, seg=D.SEG) + assert D._join([(W(D.REASON_HOLLOW), 1), (W(D.REASON_HOLLOW, split=True), 2)]) == \ + D.REASON_HOLLOW + D.REASON_SPLIT + assert D._join([(W(ex, split=True), 2), (W(D.REASON_HOLLOW), 1)]) == \ + f"{_exhausted()}；{D.REASON_HOLLOW}{D.REASON_SPLIT}" + assert D._join([(W(ex), 2), (W(ex), 3)]) == _exhausted("这几段") + assert D._join([(W(ex), 2), (W(ex, split=True), 2)]) == _exhausted() + D.REASON_SPLIT + assert D._join([(W("甲"), 1), (W("乙"), 1), (W("丙", split=True), 1)]) == "甲；乙", "没说出来的原因不补后缀" + halted = D.REASON_HALTED.format(n=catalog.MODEL_GIVE_UP_AFTER) + assert D._join([(W(D.REASON_HOLLOW, halted=halted), 1), (W(D.REASON_HOLLOW), 2)]) == D.REASON_HOLLOW + halted + slow = D.REASON_SLOW.format(n=catalog.MODEL_MIN_PART_COLUMNS) + assert D._join([(W(D.REASON_HOLLOW, halted=halted), 1), (W(D.REASON_HOLLOW, halted=slow), 2), + (W(D.REASON_HOLLOW, halted=halted), 3)]) == D.REASON_HOLLOW + halted + slow, "两种止损各说一次" + assert D._join([(W("调用模型出错：{坏的}"), 1)]) == "调用模型出错：{坏的}", "报错原文里的花括号原样留着" + + +def _by_table(model: Scripted) -> dict[str, int]: + out: dict[str, int] = {} + for call in model.calls: + for name in call.asked: + out[name] = out.get(name, 0) + 1 + return out + + +async def test_after_two_segments_in_a_row_come_to_nothing_the_rest_are_not_split(monkeypatch): + """模型明显用不了（思考总是吃满额度）：连续 2 段拆到底仍一项没写出，后面的段只问一次、不再拆，报错说明已停止拆小重试。 + 有一段写出来了就恢复拆小。一次只起草一段（并发 1），先后是确定的。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + assert catalog.MODEL_GIVE_UP_AFTER == 2 + tables = {f"t{i}": _table(10) for i in range(1, 5)} + exhausted = Reply(text="", finish="length", thinking="……") + + model = Scripted(lambda call: exhausted) + got = await D.run_model_draft(model, _source(tables), list(tables)) + assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 1} + budget = _exhausted() + assert got["t1"].error == budget + D.REASON_SPLIT + assert got["t3"].error == budget + "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型" + + ok = answering(tables) + model = Scripted(lambda call: ok(call) if "t3" in call.asked else exhausted) + got = await D.run_model_draft(model, _source(tables), list(tables)) + assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 3}, "t3 写出来了，t4 照常拆小" + assert got["t3"].status == "ok" and got["t4"].error == budget + D.REASON_SPLIT + + +async def test_the_column_parts_of_a_wide_table_stop_splitting_too(monkeypatch): + """宽表的列分段一段一段排队起草（一段拆到底再起草下一段），止损在宽表里同样起作用。 + + 95 列：表级那段拆不开（结构化、纯文本各一次）；第 2 段 32 列拆小、到 8 次封顶；这时已经连续 2 段没写出来，第 3、4 段 + 只问一次、换纯文本一次。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + model = Scripted(_hollow) + got = await D.run_model_draft(model, _source({"wide95": _table(95)}), ["wide95"]) + assert len(model.calls) == 2 + 8 + 2 + 2 + assert got["wide95"].error == (WIDE95_PARTS + "都没写出来：" + D.REASON_HOLLOW + + "；每段最多调用 8 次模型，这一段到了次数仍没写完，已停止重试" + + D.REASON_SPLIT + "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型") + + +async def test_a_wide_table_that_never_comes_back_is_bounded_by_the_first_wave(): + """默认并发（3 段同时起草）：294 列（8 个列分段）一直只给空壳，最多是表级那段 2 次、头 3 个列分段各 8 次（封顶）、 + 其余 5 段各 2 次。以前所有列分段一下子都排上去、一层一层地拆，要 168 次。""" + model = Scripted(_hollow) + got = await D.run_model_draft(model, _source({"wide294": _table(294)}), ["wide294"]) + assert got["wide294"].status == "failed" and got["wide294"].parts_total == 9 + assert len(model.calls) <= 2 + 3 * 8 + 5 * 2 + + +async def test_a_big_draft_does_not_hold_the_model_ahead_of_a_later_one(monkeypatch): + """全进程的调用名额先来先放：一次起草不把它的所有段一下子都排上去，后开始的那次起草（另一个数据源）能轮上。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + big = {f"a{i}": _table(4) for i in range(1, 5)} + small = {"b1": _table(4)} + order: list[str] = [] + release = asyncio.Event() + + def handler(tables: dict) -> Callable[[Call], Any]: + ok = answering(tables) + + async def handle(call: Call) -> Reply: + order.append(next(iter(call.asked))) + if len(order) == 1: + await release.wait() + return ok(call) + return handle + + first = asyncio.create_task(D.run_model_draft(Scripted(handler(big)), _source(big), list(big))) + while not order: + await asyncio.sleep(0) + second = asyncio.create_task(D.run_model_draft(Scripted(handler(small)), _source(small), ["b1"])) + for _ in range(20): + await asyncio.sleep(0) + release.set() + got_big, got_small = await asyncio.gather(first, second) + assert set(_status(got_big).values()) == {"ok"} and _status(got_small) == {"b1": "ok"} + assert order == ["a1", "b1", "a2", "a3", "a4"] + + +#: 95 列的宽表分 4 段起草，逐段写上列的范围 +WIDE95_PARTS = ("这张表 95 列，分 4 段起草，第 1 段（表级的项）、第 2 段（c001 到 c032，共 32 列）、" + "第 3 段（c033 到 c064，共 32 列）、第 4 段（c065 到 c095，共 31 列）") + + +def _order(model: Scripted) -> list[tuple[str, str]]: + return [(name, part) for call in model.calls for name, (part, _) in call.asked.items()] + + +async def test_narrow_tables_are_not_cut_in_by_the_column_parts_of_a_wide_one(monkeypatch): + """一个任务里按表的先后排队：宽表的表级那段写完，它的列分段排到已经排着的窄表段后面（先来先服务），不插队。 + 以前插到队首，窄表要等宽表的列分段全部派完。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + tables = {"wide": _table(45), "na": _table(4), "nb": _table(4)} + model = Scripted(answering(tables)) + got = await D.run_model_draft(model, _source(tables), ["wide", "na", "nb"]) + assert set(_status(got).values()) == {"ok"} + assert _order(model) == [("wide", "table"), ("na", "full"), ("nb", "full"), ("wide", "columns"), + ("wide", "columns")] + + +async def test_after_two_exhausted_or_timed_out_calls_new_parts_are_halved(monkeypatch): + """额度用完或超时累计 2 次，之后新开始的段列数上限减半（40 → 20 → 10，最低 10）；已经开始的段照旧（按原因拆小）。 + 每次调用的结果里带着当前的上限（part_columns）。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + tables = {f"t{i}": _table(40) for i in range(1, 4)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + (_, (part, cols)), = call.asked.items() + width = 40 if part == "full" else len(cols) + return EXHAUSTED if width > 10 else ok(call) # 这个模型一次只写得出 10 列 + + model, reports = Scripted(handle), Reports() + got = await D.run_model_draft(model, _source(tables), list(tables), hooks=reports.hooks()) + assert set(_status(got).values()) == {"ok"} + asked = {name: [(part, len(cols or [])) for c in model.calls for n, (part, cols) in c.asked.items() if n == name] + for name in tables} + # t1：整张 40 列开始，用完额度拆成 20、20，又各拆成 10、10 + assert asked["t1"][0] == ("full", 0) and len(asked["t1"]) == 7 + # 第 2 次用完额度后上限是 20：t2 一开始就按 20 列切成两段（各自再拆一次） + assert asked["t2"][0] == ("table+columns", 20) and ("columns", 20) in asked["t2"] + # 第 4 次之后上限是 10：t3 一开始就切成 4 段 10 列，一次都没用完额度 + assert asked["t3"] == [("table+columns", 10), ("columns", 10), ("columns", 10), ("columns", 10)] + caps = [r.part_columns for r in reports.items] + assert list(dict.fromkeys(caps)) == [40, 20, 10] + assert caps[:2] == [40, 20], "第 2 次用完额度的那次调用之后就是 20" + assert min(caps) == 10 == catalog.MODEL_MIN_PART_COLUMNS + + +async def test_the_notice_comes_when_four_of_six_calls_fail_and_goes_when_fewer_than_two(monkeypatch): + """最近 6 次调用有 4 次以上没写出来：提醒（写明次数和原因，建议换起草模型、减少表或停止）；之后最近 6 次里没写出来 + 的少于 2 次就清掉。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + tables = {f"t{i}": _table(4) for i in range(1, 10)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + name = next(iter(call.asked)) + if name in ("t1", "t2"): + raise RuntimeError("网关拒绝了这次请求") + if name == "t3": + return EXHAUSTED + if name == "t4": + return _hollow(call) + return ok(call) + + reports = Reports() + got = await D.run_model_draft(Scripted(handle), _source(tables), list(tables), hooks=reports.hooks()) + assert reports.failures == ["error", "error", "exhausted", "hollow", "hollow"] + [None] * 5 + notices = [r.notice for r in reports.items] + assert notices[:3] == [None] * 3 + assert notices[3] == ("最近 4 次调用有 4 次没写出来（额度用完 1 次、只给空壳 1 次、出错 2 次）。建议在「设置 → 偏好设置」" + "的「起草模型」换一个模型、缩小起草范围，或者停止这次起草") + assert notices[4].startswith("最近 5 次调用有 5 次没写出来") + assert notices[7].startswith("最近 6 次调用有 3 次没写出来"), "还有 2 次以上没写出来：留着，次数照实写" + assert notices[8].startswith("最近 6 次调用有 2 次没写出来") + assert notices[9] is None, "最近 6 次只有 1 次没写出来：清掉" + assert got["t9"].status == "ok" + + +# -------------------------------------------------------------------------- +# 复核后的修正：每个原始段一共 8 次、一直超时的模型几轮就停、读回复出错也记用量、目录里已有的不算留空、补问列出了不算 +# 没写出来、一项都没写出来的宽表逐段写范围、止损后纯文本不再原样再问 +# -------------------------------------------------------------------------- + + +def _width(call: Call, tables: dict) -> int: + """这次调用请模型写几列（只写表级的项是 0）。""" + return sum(len(tables[name]["columns"]) if part == "full" else len(cols or []) + for name, (part, cols) in call.asked.items()) + + +async def test_the_chunks_of_a_part_cut_by_the_smaller_cap_share_its_eight_calls(monkeypatch): + """每个原始段（连同段大小减半后按上限分开的各块、拆出来的、补问的、重试的）一共最多调用 8 次。以前分开的每一块各有 + 8 次：上限缩到 10 列以后一直只给空壳，4 张窄表拼成的 40 列段调用了 20 次，单张 40 列的表也是 20 次。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99) # 不让止损先停下来 + tables = {"w40": _table(40), **{f"n{i}": _table(10) for i in range(1, 5)}, "x40": _table(40)} + + def handle(call: Call) -> Reply: + return EXHAUSTED if "w40" in call.asked else _hollow(call) + + model, reports = Scripted(handle), Reports() + got = await D.run_model_draft(model, _source(tables), list(tables), hooks=reports.hooks()) + first = sum(1 for c in model.calls if "w40" in c.asked) + assert first == 8 and reports.items[first - 1].part_columns == 10, "w40 一直用完额度，上限缩到 10 列" + packed = [c for c in model.calls if set(c.asked) & {"n1", "n2", "n3", "n4"}] + assert len(packed) == 8, "拼成的 40 列段按 10 列分开成 4 块，4 块一共 8 次" + assert all(len(c.asked) == 1 and _width(c, tables) <= 10 for c in packed) + assert {got[f"n{i}"].status for i in range(1, 5)} == {"failed"} + assert "每段最多调用 8 次模型，这一段到了次数仍没写完，已停止重试" in {got[f"n{i}"].error for i in range(1, 5)}, \ + "次数用完时还没问的块记「每段最多调用 8 次」" + wide = [c for c in model.calls if "x40" in c.asked] + assert len(wide) == 8 and _width(wide[0], tables) == 10, "单张 40 列的表按 10 列分开，各块一共 8 次" + assert len(reports.items) == len(model.calls) + + +class AlwaysSlow: + """每次调用都超时（立刻抛 TimeoutError，和到了单次调用的时限走同一条路），fits(列数) 为真的照常写。 + + 记下每次调用在第几轮：每次调用一样长（一个时限），一次调用开始时，已经结束的调用里最晚的那一轮的下一轮—— + 轮数乘以时限就是用掉的时间。""" + + def __init__(self, tables: dict, fits: Callable[[int], bool] = lambda n: False) -> None: + self.tables, self.fits, self.ok = tables, fits, answering(tables) + self.rounds: list[int] = [] + self._ended: list[int] = [] + + async def __call__(self, call: Call) -> Reply: + r = 1 + max(self._ended, default=0) + self.rounds.append(r) + for _ in range(5): + await asyncio.sleep(0) + self._ended.append(r) + if self.fits(_width(call, self.tables)): + return self.ok(call) + raise TimeoutError() + + +SLOW_STOP = "；拆到每段 10 列仍然超时，已停止拆小重试，建议更换起草模型" + + +async def test_a_model_that_always_times_out_stops_after_three_rounds_not_seven(): + """每次调用都超时、和段大小无关：超时拆出来的两半同时问（不再等带表级项的前一半写完才问后一半），拆到每段 10 列的 + 调用接连超时 2 次就停止拆小，拆出来还没问的块不再问。40 列的表 3 轮、6 次调用就结束（时限 420 秒时约 21 分钟）； + 以前 40→20→10→5 一层一层串着问，8 次调用约 7 轮（约 49 分钟）。""" + tables = {"t40": _table(40)} + slow = AlwaysSlow(tables) + model, reports = Scripted(slow), Reports() + got = await D.run_model_draft(model, _source(tables), ["t40"], hooks=reports.hooks()) + assert [_width(c, tables) for c in model.calls] == [40, 20, 20, 10, 10, 10], "拆到每段 10 列仍超时就不再往下拆" + assert slow.rounds == [1, 2, 2, 3, 3, 3], "两半同时问" + assert reports.failures == ["timeout"] * 6 and len(reports.items) == len(model.calls) + o = got["t40"] + assert o.status == "failed" and o.error == "模型 420 秒内没有写完这一段" + D.REASON_SPLIT + SLOW_STOP + + +async def test_three_parts_that_always_time_out_end_in_four_rounds(): + """首轮同时起草的 3 段都一直超时：拆到每段 10 列仍超时就一起停下，不再各自拆满 8 次。12 次调用、4 轮；以前 24 次 + 调用、约 8 轮（止损要等一整段结束才计数，首轮的 3 段都跑满）。""" + tables = {f"t{i}": _table(40) for i in range(1, 4)} + slow = AlwaysSlow(tables) + model = Scripted(slow) + got = await D.run_model_draft(model, _source(tables), list(tables)) + assert len(model.calls) == 12 and max(slow.rounds) == 4 + assert {o.status for o in got.values()} == {"failed"} + assert all(o.error.endswith(SLOW_STOP) for o in got.values()) + + +async def test_timeouts_that_go_away_at_ten_columns_are_still_split_down_and_written(): + """超时和段大小有关（40、20 列超时，10 列写得完）：照旧拆到 10 列写出来，不因为前面超时多就停下。""" + tables = {"t40": _table(40)} + slow = AlwaysSlow(tables, fits=lambda n: n <= 10) + model = Scripted(slow) + got = await D.run_model_draft(model, _source(tables), ["t40"]) + assert got["t40"].status == "ok" and len(got["t40"].notes["columns"]) == 40 + assert len(model.calls) == 7 and max(slow.rounds) == 4 + + +async def test_odd_values_in_a_reply_are_dropped_not_a_crash(): + """模型把列名、表类型、业务主键、业务日期的列、度量类型写成了数组或对象：这几项丢掉，别的照收；不当成「起草这张表 + 时出错」。以前列名是数组时校验里抛 TypeError，整张表记出错，这次调用也没有记进用量。""" + tables = {"t4": _table(4)} + + def handle(call: Call) -> Reply: + return Reply(obj={"tables": [{ + "table": "t4", "label": "某表", "kind": ["fact"], "keys": [["c001"]], + "business_date": {"column": ["c002"], "rule": "按下单时间"}, + "columns": [{"name": ["c001"], "label": "数组列名"}, {"name": {"x": 1}, "label": "对象列名"}, + {"name": "c002", "label": "第二列", "measure": ["flow"]}, + {"name": "c003", "label": "第三列", "measure": {"x": "flow"}}, + {"name": "c004", "label": "第四列"}]}]}) + + model, reports = Scripted(handle), Reports() + got = await D.run_model_draft(model, _source(tables), ["t4"], hooks=reports.hooks()) + o = got["t4"] + assert len(reports.items) == len(model.calls) == 2, "c001 没有按列名列出来：补问一次" + assert o.status == "partial" and "c001" in o.error and "出错" not in o.error + assert set(o.notes) == {"label", "columns"} and set(o.notes["columns"]) == {"c002", "c003", "c004"} + assert all(set(items) == {"label"} for items in o.notes["columns"].values()) + + +async def test_a_call_whose_reply_cannot_be_read_still_counts(monkeypatch): + """读回复时出了意料之外的错：这张表记出错，这次调用照样记进用量（算一次出错），调用次数和用量对得上。""" + tables = {"t4": _table(4)} + + def broken(entry, meta, *, system): + raise RuntimeError("校验时出了意外") + + monkeypatch.setattr(catalog, "_from_model", broken) + model, reports = Scripted(answering(tables, usage=(100, 50))), Reports() + got = await D.run_model_draft(model, _source(tables), ["t4"], hooks=reports.hooks()) + assert len(model.calls) == 1 and reports.failures == ["error"] + assert [(r.input_tokens, r.output_tokens) for r in reports.items] == [(100, 50)] + assert got["t4"].status == "failed" and got["t4"].error == "起草这张表时出错：校验时出了意外" + + +async def test_columns_the_catalog_already_names_are_not_counted_blank(): + """模型这次留空、目录里已经有中文名或含义的列（不论来源：以前模型起草的、库注释、人工；驳回的不算）不算留空：照界面 + 的建议补了起草要求再起草，以前起草好的留着（重新起草不丢内容），这一列不该再说成「模型拿不准、留空了」。""" + tables = {"t6": _table(6)} + item = catalog.make_item + existing = {"t6": {"columns": { + "c002": {"label": item("以前起草的名字", "llm")}, + "c003": {"meaning": item("库注释说清的含义", "comment")}, + "c004": {"label": item("驳回的名字", "llm", "rejected")}, + "c005": {"unit": item("元", "llm")}, + }}} + + def handle(call: Call) -> Reply: + cols = [{"name": "c001", "label": "第一列"}] + [{"name": f"c00{i}"} for i in range(2, 7)] + return Reply(obj={"tables": [{"table": "t6", "label": "某表", "columns": cols}]}) + + got = await D.run_model_draft(Scripted(handle), _source(tables), ["t6"], existing=existing) + assert got["t6"].status == "ok" and got["t6"].blank == ["c004", "c005", "c006"] + + +async def test_a_wide_table_that_wrote_nothing_names_every_part(): + """一项都没写出来的宽表同样逐段写上列的范围（以前只写原因和「这一段」「这几段」，看不出是哪几段）。""" + def handle(call: Call) -> Reply: + raise RuntimeError("网关拒绝") + + got = await D.run_model_draft(Scripted(handle), _source({"wide95": _table(95)}), ["wide95"]) + o = got["wide95"] + assert o.status == "failed" and o.error == WIDE95_PARTS + "都没写出来：调用模型出错：网关拒绝" + + narrow = await D.run_model_draft(Scripted(handle), _source({"t8": _table(8)}), ["t8"]) + assert narrow["t8"].error == "调用模型出错：网关拒绝", "只有一段的表：这一段就是整张表，照旧只写原因" + + +async def test_a_halted_part_that_could_be_split_is_not_asked_again_in_text_mode(monkeypatch): + """止损以后本来能拆小的段只给空壳：已经走纯文本的不再原样再问（约定只对拆不开的段再问一次）；结构化输出的照旧换 + 纯文本问一次，看看是不是结构化输出的问题（见 test_the_column_parts_of_a_wide_table_stop_splitting_too）。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + tables = {f"t{i}": _table(10) for i in range(1, 5)} + model = Scripted(_text_only(_hollow)) + got = await D.run_model_draft(model, _source(tables), list(tables)) + # t1：结构化不支持 1 次、纯文本 1 次、拆成两半各问一次再原样再问一次；t2 同样（少了结构化那次）；之后止损 + assert _by_table(model) == {"t1": 6, "t2": 5, "t3": 1, "t4": 1} + assert got["t3"].error == (catalog.MODEL_EMPTY_REASON + + "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型") + + +# -------------------------------------------------------------------------- +# 起草要求、用量 +# -------------------------------------------------------------------------- + + +async def test_guidance_goes_into_the_prompt_after_the_rules(): + tables = {"t4": _table(4)} + model = Scripted(answering(tables)) + src = _source(tables, description="虚构的景区票务库") + await D.run_model_draft(model, src, ["t4"], + background=D.Background(guidance="金额一律按元写单位", note="这次重点看会员相关的列")) + text = model.calls[0].text + block = ("使用者补充的背景和要求：\n数据源说明：虚构的景区票务库\n起草要求：金额一律按元写单位\n" + "本次说明：这次重点看会员相关的列") + assert block in text + rule = "它和这里的规矩冲突时，以这里的规矩为准" + assert rule in catalog.CATALOG_DRAFT_SYSTEM + assert text.index(rule) < text.index(block) < text.index("请为下面") < text.index("表 t4") + + plain = Scripted(answering(tables)) + await D.run_model_draft(plain, _source(tables), ["t4"]) + assert "使用者补充的背景和要求：" not in plain.calls[0].text, "都没有就不出现这一段（系统提示词里那条规矩照旧在）" + only_note = Scripted(answering(tables)) + await D.run_model_draft(only_note, _source(tables), ["t4"], background=D.Background(note="只看金额")) + assert "使用者补充的背景和要求：\n本次说明：只看金额\n\n请为下面" in only_note.calls[0].text + + +async def test_the_prompt_asks_not_to_draft_every_column_while_thinking(): + assert "思考时不必把要输出的内容逐列草拟一遍" in catalog.CATALOG_DRAFT_SYSTEM + + +async def test_usage_is_added_up_through_the_hooks(): + tables = {"orders_x": _table(24)} + ok = answering(tables, usage=(120, 60)) + + def handle(call: Call) -> Reply: + if call.n == 0: + return Reply(text="", finish="length", thinking="……", usage=(100, 32768)) + return ok(call) + + reports = Reports() + await D.run_model_draft(Scripted(handle), _source(tables), ["orders_x"], hooks=reports.hooks()) + assert [(r.input_tokens, r.output_tokens, r.failure) for r in reports.items] == \ + [(100, 32768, "exhausted"), (120, 60, None), (120, 60, None)] + assert {r.notice for r in reports.items} == {None} and {r.part_columns for r in reports.items} == {40} + + +def test_budget_words_match_the_other_drafts(): + from app.data import recipe_ai + + assert D.BUDGET_WORDS == recipe_ai._BUDGET_WORDS + + +# -------------------------------------------------------------------------- +# 从正文里取 JSON +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("text, tables, truncated", [ + # 前面有一句话、带括号的说明、围栏 + ('按 {table} 的格式写好了：\n```json\n{"tables": [{"table": "a", "label": "甲"}]}\n```', ["a"], False), + ('[说明] 草稿：{"tables": [{"table": "a"}]} 以上', ["a"], False), + # 外面多包了一层 + ('{"result": {"tables": [{"table": "a"}, {"table": "b"}]}}', ["a", "b"], False), + # 截断在字符串、数字、字面量、键名中间：只收写完了的 + ('{"tables": [{"table": "a", "label": "甲"}, {"table": "b", "label": "乙', ["a", "b"], True), + ('{"tables": [{"table": "a", "columns": [{"name": "x", "label": "某"}, {"name": "y", "len": 12', ["a"], True), + ('{"tables": [{"table": "a", "keys": ["x", "y"', ["a"], True), + ('{"tables": [{"table": "a"}, {"tab', ["a"], True), + ('{"tables": [{"table": "a", "flag": tr', ["a"], True), +]) +def test_json_is_taken_from_the_text(text, tables, truncated): + reply = D._parse_text(text, cut=False) + assert reply is not None and [e["table"] for e in reply.entries] == tables + assert reply.truncated is truncated + + +def test_cut_values_are_dropped_and_complete_ones_kept(): + text = '{"tables": [{"table": "a", "label": "甲", "keys": ["x", "y"], "columns": [{"name": "x", "label": "某"}, ' \ + '{"name": "y", "label": "半' + entry = D._parse_text(text, cut=True).entries[0] + assert entry == {"table": "a", "label": "甲", "keys": ["x", "y"], "columns": [{"name": "x", "label": "某"}]} + cut_keys = D._parse_text('{"tables": [{"table": "a", "label": "甲", "keys": ["x", "y"', cut=True).entries[0] + assert cut_keys == {"table": "a", "label": "甲"}, "没写完的业务主键整项不要，不能只留前半" + + +@pytest.mark.parametrize("text", ["好的，我来写", "{坏的 json}", ""]) +def test_not_json_is_not_a_reply(text): + assert D._parse_text(text, cut=False) is None diff --git a/backend/tests/test_catalog_draft_jobs.py b/backend/tests/test_catalog_draft_jobs.py new file mode 100644 index 00000000..bc366876 --- /dev/null +++ b/backend/tests/test_catalog_draft_jobs.py @@ -0,0 +1,781 @@ +"""数据目录的后台起草任务（data/catalog_jobs.py、/api/datasources/{id}/catalog/draft-jobs）与起草要求。 + +- 建任务 202；同一个源已有排队或进行中的任务 409 catalog_draft_busy；起草模型用不了 409 + catalog_draft_model_unavailable（不建任务）；没有表结构 409；表结构里没有的表照样建进任务、状态 skipped +- 在后台起草，每张表一结束就并入目录；进度、计数、用量落库，接口读的是库里的状态 +- 停止：排队的直接 cancelled；进行中的停掉正在进行的模型调用，没写完的表记 cancelled，已经写进目录的不回滚；已结束的 409 +- 服务启动时把没结束的任务标成 interrupted（没结束的表记 cancelled、写明服务重启），不自动续跑；关停时同样 +- 重试：只含原任务里 failed、partial、cancelled 的表，本次说明沿用、起草要求取数据源当前的；没有可重试的 422 +- 目录清单：每行 last_draft（带模型留空的列数），顶层 guidance、active_job +- 重新起草时模型这次留空、给的值没通过校验：以前起草好的模型项留着，删除只在列已经不在表结构里时才有 +- 用量记超时次数、没写出来的调用次数；任务记当前的列数上限、提醒（最近 6 次调用 4 次没写出来）；逐表记模型留空的列 +- 旧的同步起草：同一个源有排队或进行中的任务时 409 catalog_draft_busy +- 起草要求存在数据源 options 里（上传表格的源存在它自己那一行），数据源接口不显示、整份回写 options 时留着，拼连接串时摘掉 + +表名、列名、说明文字都是虚构的。 +""" +from __future__ import annotations + +import asyncio +import uuid +from types import SimpleNamespace +from urllib.parse import quote + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select + +from app.data import catalog, catalog_jobs +from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION, build_url +from app.db.base import SessionLocal +from app.db.models import CatalogDraftJob, CatalogDraftTask, DataSource +from app.main import app as _fastapi_app +from tests.fixtures.catalog.fake_model import FakeModel, requested +from tests.fixtures.sources import drop_source + +ACTOR = {"X-Actor": quote("王敏")} + + +# -------------------------------------------------------------------------- +# 夹具 +# -------------------------------------------------------------------------- + + +def _col(name: str, type_: str = "INTEGER", comment: str | None = None) -> dict: + return {"name": name, "type": type_, "nullable": True, "comment": comment} + + +def _tables() -> dict: + return { + "visits": {"qualified": "visits", "columns": [_col("id"), _col("ticket_no", "TEXT"), _col("visit_time", "TEXT"), + _col("channel_id")], + "primary_key": ["id"], "comment": None, "is_view": False, + "foreign_keys": [{"columns": ["channel_id"], "to_table": "channels", "to_columns": ["id"]}]}, + "channels": {"qualified": "channels", "columns": [_col("id"), _col("name", "TEXT")], "primary_key": ["id"], + "comment": "售票渠道", "is_view": False, "foreign_keys": []}, + "gates": {"qualified": "gates", "columns": [_col("id"), _col("gate_code", "TEXT")], "primary_key": ["id"], + "comment": None, "is_view": False, "foreign_keys": []}, + } + + +REPLIES = { + "visits": {"label": "入园记录", "kind": "fact", "columns": [{"name": "ticket_no", "label": "票号"}]}, + "channels": {"label": "渠道", "kind": "dimension"}, + "gates": {"label": "闸口", "kind": "dimension"}, +} + +_CREATED: list[str] = [] + + +@pytest.fixture(autouse=True) +async def _cleanup(): + yield + # 后台任务先收完（停不下来的停掉），再删源：删源会级联删掉任务 + await catalog_jobs.manager.shutdown(grace=5) + while _CREATED: + await drop_source(_CREATED.pop()) + + +async def _source(tables: dict | None = None, **kw) -> str: + async with SessionLocal() as session: + row = DataSource(name=f"cdj_{uuid.uuid4().hex[:10]}", kind="sqlite", database=":memory:", + schema_cache={"tables": tables if tables is not None else _tables()}, **kw) + session.add(row) + await session.commit() + _CREATED.append(row.id) + return row.id + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c: + yield c + + +def _base(sid: str) -> str: + return f"/api/datasources/{sid}/catalog" + + +def use(monkeypatch, model) -> None: + """起草模型换成假模型（建任务时查一次能不能用，开始起草时再取一次）。""" + async def resolve(session): + return model, "fake-model" + + monkeypatch.setattr(catalog, "resolve_draft_model", resolve) + + +async def finished(job_id: str) -> None: + await catalog_jobs.manager.wait(job_id, timeout=10) + + +async def get_job(client, sid: str, job_id: str) -> dict: + r = await client.get(f"{_base(sid)}/draft-jobs/{job_id}") + assert r.status_code == 200, r.text + return r.json()["job"] + + +class Held: + """假模型：请求里有 hold 里的表时，等 release 才回；被停止时记下这次调用被取消了。""" + + def __init__(self, replies: dict, hold: set[str]) -> None: + self.inner, self.hold = FakeModel(replies), set(hold) + self.release = asyncio.Event() + self.cancelled = 0 + self.model_name = "held-model" + + def with_structured_output(self, schema, **_): + outer = self + + class _S: + async def ainvoke(self, messages): + text = "\n".join(str(m.content) for m in messages) + if outer.hold & set(requested(text)): + try: + await outer.release.wait() + except asyncio.CancelledError: + outer.cancelled += 1 + raise + return outer.inner._reply(messages, "structured") + + return _S() + + async def ainvoke(self, messages): + return await self.inner.ainvoke(messages) + + +async def until(predicate, timeout: float = 5.0) -> None: + for _ in range(int(timeout / 0.01)): + if await predicate(): + return + await asyncio.sleep(0.01) + raise AssertionError("等不到") + + +# -------------------------------------------------------------------------- +# 建任务、在后台起草 +# -------------------------------------------------------------------------- + + +async def test_a_job_drafts_in_the_background_and_writes_each_table(client, monkeypatch): + sid = await _source() + model = FakeModel(REPLIES) + use(monkeypatch, model) + await client.put(f"{_base(sid)}/guidance", json={"guidance": "金额一律按元写单位"}) + r = await client.post(f"{_base(sid)}/draft-jobs", headers=ACTOR, + json={"tables": ["visits", "nope", "channels"], "use_model": True, "note": " 看渠道 "}) + assert r.status_code == 202, r.text + job = r.json()["job"] + assert job["status"] in ("queued", "running") and job["source_id"] == sid + assert job["use_model"] is True and job["guidance"] == "金额一律按元写单位" and job["note"] == "看渠道" + assert job["created_by"] == "王敏" and job["total"] == 3 + assert [t["table_name"] for t in job["tasks"]] == ["visits", "nope", "channels"], "按起草顺序" + await finished(job["id"]) + + job = await get_job(client, sid, job["id"]) + assert job["status"] == "succeeded" and job["error"] is None and job["model"] == "fake-model" + assert job["done"] == 3 and job["counts"] == {"pending": 0, "running": 0, "ok": 2, "partial": 0, "failed": 0, + "skipped": 1, "cancelled": 0} + assert job["started_at"] and job["finished_at"] and job["usage"]["calls"] >= 1 + by = {t["table_name"]: t for t in job["tasks"]} + assert by["nope"]["status"] == "skipped" and by["nope"]["error"] == "表结构里没有 nope，请先重新探查结构" + visits = by["visits"] + assert visits["status"] == "ok" and visits["parts_total"] == 1 and visits["parts_done"] == 1 + assert visits["model_items"] == 3 and visits["added"] == 4 and visits["version"] == 1 and visits["columns"] == 4 + assert visits["started_at"] and visits["finished_at"] and visits["error"] is None + assert job["added"] == sum(t["added"] for t in job["tasks"]) + # 起草要求和本次说明都交给了模型 + assert "起草要求：金额一律按元写单位" in model.calls[0] and "本次说明：看渠道" in model.calls[0] + + detail = (await client.get(f"{_base(sid)}/visits")).json() + assert detail["notes"]["label"]["value"] == "入园记录" and detail["updated_by"] == "王敏" + + +async def test_a_job_without_the_model_writes_structure_only(client): + sid = await _source() + r = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": [], "use_model": False}) + assert r.status_code == 202, r.text + job = r.json()["job"] + assert job["total"] == 3, "不给表时按使用次数取前 20 张" + await finished(job["id"]) + job = await get_job(client, sid, job["id"]) + assert job["status"] == "succeeded" and job["model"] is None and job["usage"]["calls"] == 0 + assert all(t["parts_total"] is None and t["model_items"] is None for t in job["tasks"]) + assert {t["table_name"]: t["added"] for t in job["tasks"]}["visits"] == 1 # 外键一条 + + +async def test_a_table_whose_model_part_failed_is_failed_but_structure_is_written(client, monkeypatch): + sid = await _source() + use(monkeypatch, FakeModel(REPLIES, fail={"visits"})) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["visits", "channels"], + "use_model": True})).json()["job"] + await finished(job["id"]) + job = await get_job(client, sid, job["id"]) + by = {t["table_name"]: t for t in job["tasks"]} + assert job["status"] == "partial" + assert by["visits"]["status"] == "failed" and "网关超时" in by["visits"]["model_error"] + assert by["visits"]["error"] is None and by["visits"]["added"] == 1, "外键照常写入" + assert by["channels"]["status"] == "ok" + + +class HalfThenHollow: + """假模型：第一次只给前一半列，补问时只给一个空壳（表名对上、一项都没有）。""" + + model_name = "half-model" + + def __init__(self, table: str, reply: dict) -> None: + self.table, self.reply, self.calls = table, reply, 0 + + def with_structured_output(self, schema, **_): + outer = self + + class _S: + async def ainvoke(self, messages): + outer.calls += 1 + if outer.calls == 1: + cols = outer.reply["columns"] + return {"tables": [{"table": outer.table, **outer.reply, "columns": cols[:len(cols) // 2]}]} + return {"tables": [{"table": outer.table}]} + + return _S() + + async def ainvoke(self, messages): + raise AssertionError("这里不该走纯文本") + + +async def test_a_followup_that_comes_back_empty_keeps_the_earlier_model_items(client, monkeypatch): + """补问只给空壳：表是 partial、任务是 partial，以前起草过的列（待确认的模型起草项）不因这次没写全被删掉。""" + cols = [_col(f"c{i:02d}") for i in range(1, 9)] + sid = await _source({"tickets": {"qualified": "tickets", "columns": cols, "primary_key": ["c01"], "comment": None, + "is_view": False, "foreign_keys": []}}) + full = {"label": "门票", "kind": "fact", + "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} for i in range(1, 9)]} + use(monkeypatch, FakeModel({"tickets": full})) + first = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["tickets"], "use_model": True})).json() + await finished(first["job"]["id"]) + before = (await client.get(f"{_base(sid)}/tickets")).json()["notes"]["columns"] + assert {c for c, items in before.items() if "label" in items} == {f"c{i:02d}" for i in range(1, 9)} + + model = HalfThenHollow("tickets", full) + use(monkeypatch, model) + second = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["tickets"], "use_model": True})).json() + await finished(second["job"]["id"]) + job = await get_job(client, sid, second["job"]["id"]) + task = job["tasks"][0] + assert model.calls == 2 + assert job["status"] == "partial" and task["status"] == "partial" and task["removed"] == 0 + assert "有 4 列（c05 到 c08）没写出来" in task["model_error"] and "可以只重试这张表" in task["model_error"] + after = (await client.get(f"{_base(sid)}/tickets")).json()["notes"]["columns"] + assert {c for c, items in after.items() if "label" in items} == {f"c{i:02d}" for i in range(1, 9)} + + +def _tickets(n: int = 6) -> dict: + return {"tickets": {"qualified": "tickets", "columns": [_col(f"c{i:02d}") for i in range(1, n + 1)], + "primary_key": ["c01"], "comment": None, "is_view": False, "foreign_keys": []}} + + +async def _draft(client, sid: str, monkeypatch, model, tables: list[str]) -> dict: + use(monkeypatch, model) + r = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": tables, "use_model": True}) + assert r.status_code == 202, r.text + await finished(r.json()["job"]["id"]) + return await get_job(client, sid, r.json()["job"]["id"]) + + +async def test_a_redraft_keeps_what_the_model_left_blank_this_time(client, monkeypatch): + """重新起草（比如补了起草要求再起草一次）时模型这次留空、或者给的值没通过校验：以前起草好的模型项留着，不算删除； + 给了合格新值的才替换。以前整库重跑一次删掉几百项，表仍记「已完成」。""" + sid = await _source(_tickets()) + full = {"label": "门票", "kind": "fact", + "columns": [{"name": f"c{i:02d}", "label": f"第{i}列", "meaning": f"第{i}列的含义"} for i in range(1, 6)]} + first = await _draft(client, sid, monkeypatch, FakeModel({"tickets": full}), ["tickets"]) + assert first["tasks"][0]["status"] == "ok" and first["blank_columns"] == 1, "c06 第一次就留空了" + before = (await client.get(f"{_base(sid)}/tickets")).json()["notes"] + + # 这一次：表的中文名换了；c03 的中文名像公式（校验丢掉）、含义换了；c04 只给了中文名；其余列列出了但都留空 + again = {"label": "门票明细", "columns": [{"name": "c03", "label": "=c01+c02", "meaning": "新的含义"}, + {"name": "c04", "label": "第4列"}]} + second = await _draft(client, sid, monkeypatch, FakeModel({"tickets": again}), ["tickets"]) + task = second["tasks"][0] + assert task["status"] == "ok" and task["removed"] == 0 and second["removed"] == 0 + assert (task["added"], task["updated"]) == (0, 2) + after = (await client.get(f"{_base(sid)}/tickets")).json()["notes"] + assert after["label"]["value"] == "门票明细" and after["kind"] == before["kind"] + assert after["columns"]["c03"]["label"] == before["columns"]["c03"]["label"] + assert after["columns"]["c03"]["meaning"]["value"] == "新的含义" + for col in ("c01", "c02", "c05"): + assert after["columns"][col] == before["columns"][col], col + # 留空的列只记目录里还是没有中文名、含义的（c06）：c01、c02、c05 这次留空，以前起草好的还在，不算留空——不然照 + # 提示补了起草要求再起草，提示指着的却是目录里有中文名的列，一直消不掉 + assert task["blank_columns"] == 1 and task["blank_sample"] == ["c06"] + + +async def test_model_items_of_a_column_gone_from_the_schema_are_removed(client, monkeypatch): + sid = await _source(_tickets()) + full = {"label": "门票", "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} for i in range(1, 7)]} + await _draft(client, sid, monkeypatch, FakeModel({"tickets": full}), ["tickets"]) + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + row.schema_cache = {"tables": _tickets(5)} # 重新探查过：c06 已经没有了 + await session.commit() + job = await _draft(client, sid, monkeypatch, FakeModel({"tickets": {"label": "门票"}}), ["tickets"]) + assert job["tasks"][0]["removed"] == 1 + after = (await client.get(f"{_base(sid)}/tickets")).json()["notes"] + assert set(after["columns"]) == {"c01", "c02", "c03", "c04", "c05"} + + +async def test_usage_counts_timeouts_and_failed_calls(client, monkeypatch): + """超时的调用拆小再问，用量里记超时几次、没写出来几次（超时的调用拿不到用量，token 记 0）；额度用完或超时累计 2 次， + 新开始的段列数上限减半，任务上记着当前的上限。""" + monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.05) + sid = await _source(_tickets(24)) + replies = {"tickets": {"label": "门票", "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} + for i in range(1, 25)]}} + model = FakeModel(replies, hang=2) + job = await _draft(client, sid, monkeypatch, model, ["tickets"]) + assert job["status"] == "succeeded" and job["tasks"][0]["status"] == "ok" + assert job["usage"] == {"calls": 5, "input_tokens": 0, "output_tokens": 0, "timeouts": 2, "failed_calls": 2} + assert job["part_columns"] == 20 and job["notice"] is None + # 24 列整张问超时 → 两半同时问：前 12 列（带表级的项）超时、后 12 列写出 → 前 12 列拆成 6、6 + assert [len(requested(c)["tickets"][1] or []) for c in model.calls] == [0, 12, 12, 6, 6] + + plain = await _draft(client, sid, monkeypatch, FakeModel(replies), ["tickets"]) + assert plain["part_columns"] == 40 and plain["usage"]["timeouts"] == plain["usage"]["failed_calls"] == 0 + + +async def test_the_notice_is_kept_on_the_job(client, monkeypatch): + """最近 6 次调用有 4 次以上没写出来：任务上写一句提醒（次数、原因、建议换起草模型）；不用模型的任务没有。""" + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + tables = {f"t{i}": {"qualified": f"t{i}", "columns": [_col("id"), _col("v")], "primary_key": ["id"], + "comment": None, "is_view": False, "foreign_keys": []} for i in range(1, 6)} + sid = await _source(tables) + replies = {name: {"label": f"表{name}"} for name in tables} + job = await _draft(client, sid, monkeypatch, FakeModel(replies, fail={"t1", "t2", "t3", "t4"}), list(tables)) + assert job["usage"]["calls"] == 5 and job["usage"]["failed_calls"] == 4 + assert job["notice"] == ("最近 5 次调用有 4 次没写出来（出错 4 次）。建议在「设置 → 偏好设置」的「起草模型」换一个模型、" + "缩小起草范围，或者停止这次起草") + plain = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["t1"]})).json()["job"] + await finished(plain["id"]) + plain = await get_job(client, sid, plain["id"]) + assert plain["notice"] is None and plain["part_columns"] is None and plain["blank_columns"] == 0 + + +async def test_blank_columns_reach_the_task_the_job_and_the_list(client, monkeypatch): + """模型列出了、中文名和含义都没给的列：逐表记列数和最多 5 个列名，任务上是合计，目录清单的 last_draft 也带列数。""" + tables = {**_tickets(8), "gates": _tables()["gates"]} + sid = await _source(tables) + replies = {"tickets": {"label": "门票", "columns": [{"name": "c01", "label": "编号"}]}, + "gates": {"label": "闸口", "columns": [{"name": "id", "label": "编号"}]}} + job = await _draft(client, sid, monkeypatch, FakeModel(replies), ["tickets", "gates"]) + by = {t["table_name"]: t for t in job["tasks"]} + assert by["tickets"]["status"] == "ok" and by["tickets"]["blank_columns"] == 7 + assert by["tickets"]["blank_sample"] == ["c02", "c03", "c04", "c05", "c06"] + assert by["gates"]["blank_columns"] == 1 and by["gates"]["blank_sample"] == ["gate_code"] + assert job["blank_columns"] == 8 + rows = {r["table_name"]: r for r in (await client.get(_base(sid))).json()["tables"]} + assert rows["tickets"]["last_draft"]["blank_columns"] == 7 and rows["gates"]["last_draft"]["blank_columns"] == 1 + + +async def test_blank_columns_are_counted_against_the_catalog_as_written(client, monkeypatch): + """留空的列按并入目录之后的结果算：起草进行中有人给某一列填了中文名，模型这次留空，这一列不算留空。""" + sid = await _source(_tickets(4)) + model = Held({"tickets": {"label": "门票", "columns": [{"name": "c01", "label": "编号"}]}}, hold={"tickets"}) + use(monkeypatch, model) + r = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["tickets"], "use_model": True}) + job_id = r.json()["job"]["id"] + + async def asking() -> bool: + return (await get_job(client, sid, job_id))["tasks"][0]["status"] == "running" + + await until(asking) + saved = await client.put(f"{_base(sid)}/tickets", json={"notes": {"columns": {"c02": {"label": {"value": "闸机号"}}}}, + "if_version": 0}) + assert saved.status_code == 200, saved.text + model.release.set() + await finished(job_id) + task = (await get_job(client, sid, job_id))["tasks"][0] + assert task["status"] == "ok" and task["blank_columns"] == 2 and task["blank_sample"] == ["c03", "c04"] + after = (await client.get(f"{_base(sid)}/tickets")).json()["notes"]["columns"] + assert after["c02"]["label"]["value"] == "闸机号" and after["c01"]["label"]["value"] == "编号" + + +async def test_the_sync_draft_is_409_while_a_job_is_queued_or_running(client, monkeypatch): + """旧的同步起草不和后台任务同时写同一个源的目录、抢模型名额：有排队或进行中的任务时 409（和建任务同一句话）。""" + sid = await _source() + start = catalog_jobs.manager.start + monkeypatch.setattr(catalog_jobs.manager, "start", lambda job_id: None) # 排着、不开始 + queued = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]})).json()["job"] + r = await client.post(f"{_base(sid)}/draft", json={"tables": ["channels"]}) + assert r.status_code == 409 and r.json()["code"] == "catalog_draft_busy", r.text + busy = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]}) + assert busy.status_code == 409 and r.json()["detail"] == busy.json()["detail"] + assert (await client.get(f"{_base(sid)}/channels")).json()["version"] == 0, "什么都没写" + await client.post(f"{_base(sid)}/draft-jobs/{queued['id']}/cancel") + + monkeypatch.setattr(catalog_jobs.manager, "start", start) + model = Held(REPLIES, hold={"visits"}) + use(monkeypatch, model) + running = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["visits"], "use_model": True})).json() + r = await client.post(f"{_base(sid)}/draft", json={"tables": ["channels"]}) + assert r.status_code == 409 and r.json()["code"] == "catalog_draft_busy", r.text + model.release.set() + await finished(running["job"]["id"]) + r = await client.post(f"{_base(sid)}/draft", json={"tables": ["channels"]}) + assert r.status_code == 200, r.text + + +def test_old_job_tables_get_the_new_columns(): + """升级前建的起草任务表：新列补上，存量行的计数是 0、提醒和列数上限为空。""" + from sqlalchemy import create_engine, text + + from app.db.base import _migrate + + db = create_engine("sqlite://") + with db.begin() as conn: + conn.execute(text("CREATE TABLE catalog_draft_jobs (id VARCHAR(32) PRIMARY KEY, calls INTEGER)")) + conn.execute(text("CREATE TABLE catalog_draft_tasks (id VARCHAR(32) PRIMARY KEY, removed INTEGER)")) + conn.execute(text("INSERT INTO catalog_draft_jobs (id, calls) VALUES ('j1', 3)")) + conn.execute(text("INSERT INTO catalog_draft_tasks (id, removed) VALUES ('t1', 0)")) + _migrate(conn) + _migrate(conn) + assert conn.execute(text("SELECT timeouts, failed_calls, part_columns, notice FROM catalog_draft_jobs")) \ + .one() == (0, 0, None, None) + assert conn.execute(text("SELECT blank_columns, blank_sample FROM catalog_draft_tasks")).one() == (0, None) + + +async def test_a_job_that_cannot_start_is_failed_with_the_reason(client, monkeypatch): + """建任务时起草模型还能用，开始起草时已经用不了（接入刚被停用）：任务整体 failed，error 写原因，没开始的表也记 failed。""" + sid = await _source() + calls: list[int] = [] + + async def flaky(session): + calls.append(1) + if len(calls) > 1: + raise catalog.CatalogModelUnavailable("起草模型使用的模型接入「虚构接入」已停用") + return FakeModel(REPLIES), "fake-model" + + monkeypatch.setattr(catalog, "resolve_draft_model", flaky) + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["visits", "gates"], + "use_model": True})).json()["job"] + await finished(job["id"]) + got = await get_job(client, sid, job["id"]) + assert got["status"] == "failed" and got["error"] == "起草模型使用的模型接入「虚构接入」已停用" + assert {t["status"] for t in got["tasks"]} == {"failed"} and got["finished_at"] + assert all(t["error"] == got["error"] for t in got["tasks"]) + retry = await client.post(f"{_base(sid)}/draft-jobs/{job['id']}/retry") + assert retry.status_code == 409 and retry.json()["code"] == "catalog_draft_model_unavailable" + + +async def test_busy_source_is_409_and_the_active_job_is_reachable(client, monkeypatch): + sid = await _source() + model = Held(REPLIES, hold={"visits"}) + use(monkeypatch, model) + first = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["visits"], "use_model": True})).json() + r = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"], "use_model": False}) + assert r.status_code == 409 and r.json()["code"] == "catalog_draft_busy", r.text + active = (await client.get(f"{_base(sid)}/draft-jobs/active")).json()["job"] + assert active["id"] == first["job"]["id"] + listed = (await client.get(_base(sid))).json() + assert listed["active_job"]["id"] == first["job"]["id"] and listed["active_job"]["total"] == 1 + assert listed["active_job"]["status"] in ("queued", "running") and listed["active_job"]["done"] == 0 + model.release.set() + await finished(first["job"]["id"]) + assert (await client.get(f"{_base(sid)}/draft-jobs/active")).json()["job"] is None + latest = (await client.get(f"{_base(sid)}/draft-jobs/latest")).json()["job"] + assert latest["id"] == first["job"]["id"] and latest["status"] == "succeeded" + assert (await client.get(_base(sid))).json()["active_job"] is None + + +async def test_unavailable_model_is_409_and_no_job_is_created(client, monkeypatch): + sid = await _source() + + async def unavailable(session): + raise catalog.CatalogModelUnavailable("未配置模型接入，无法由模型起草数据目录。请到「设置 → 模型接入」添加") + + monkeypatch.setattr(catalog, "resolve_draft_model", unavailable) + r = await client.post(f"{_base(sid)}/draft-jobs", json={"use_model": True}) + assert r.status_code == 409, r.text + assert r.json()["code"] == "catalog_draft_model_unavailable" and "模型接入" in r.json()["detail"] + assert (await client.get(f"{_base(sid)}/draft-jobs/latest")).json()["job"] is None + + +async def test_without_schema_or_with_bad_input_nothing_is_created(client): + sid = await _source(tables={}) + r = await client.post(f"{_base(sid)}/draft-jobs", json={"use_model": False}) + assert r.status_code == 409 and "探查结构" in r.json()["detail"] + sid = await _source() + r = await client.post(f"{_base(sid)}/draft-jobs", json={"use_model": False, "note": "说" * 1001}) + assert r.status_code == 422 and "1000" in r.json()["detail"] + r = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": [f"t{i}" for i in range(501)]}) + assert r.status_code == 422 and "500" in r.json()["detail"] + assert (await client.get(f"{_base(sid)}/draft-jobs/latest")).json()["job"] is None + assert (await client.get(f"{_base('missing-source')}/draft-jobs/latest")).status_code == 404 + assert (await client.get(f"{_base(sid)}/draft-jobs/not-a-job")).status_code == 404 + + +async def test_a_job_of_another_source_is_404(client): + a, b = await _source(), await _source() + job = (await client.post(f"{_base(a)}/draft-jobs", json={"tables": ["gates"]})).json()["job"] + await finished(job["id"]) + assert (await client.get(f"{_base(b)}/draft-jobs/{job['id']}")).status_code == 404 + assert (await client.post(f"{_base(b)}/draft-jobs/{job['id']}/cancel")).status_code == 404 + assert (await client.post(f"{_base(b)}/draft-jobs/{job['id']}/retry")).status_code == 404 + + +# -------------------------------------------------------------------------- +# 停止、服务重启 +# -------------------------------------------------------------------------- + + +async def test_cancelling_a_queued_job(client, monkeypatch): + sid = await _source() + monkeypatch.setattr(catalog_jobs.manager, "start", lambda job_id: None) # 不让它开始 + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates", "channels"]})).json()["job"] + assert job["status"] == "queued" + r = await client.post(f"{_base(sid)}/draft-jobs/{job['id']}/cancel") + assert r.status_code == 200, r.text + job = r.json()["job"] + assert job["status"] == "cancelled" and job["finished_at"] + assert {t["status"] for t in job["tasks"]} == {"cancelled"} + again = await client.post(f"{_base(sid)}/draft-jobs/{job['id']}/cancel") + assert again.status_code == 409 and again.json()["code"] == "catalog_draft_finished" + + +async def test_cancelling_a_running_job_stops_the_call_and_keeps_what_is_written(client, monkeypatch): + sid = await _source() + model = Held(REPLIES, hold={"visits"}) + use(monkeypatch, model) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + job = (await client.post(f"{_base(sid)}/draft-jobs", + json={"tables": ["channels", "visits"], "use_model": True})).json()["job"] + + async def channels_written() -> bool: + tasks = {t["table_name"]: t for t in (await get_job(client, sid, job["id"]))["tasks"]} + return tasks["channels"]["status"] == "ok" and tasks["visits"]["status"] == "running" + + await until(channels_written) + r = await client.post(f"{_base(sid)}/draft-jobs/{job['id']}/cancel") + assert r.status_code == 200, r.text + after = r.json()["job"] + by = {t["table_name"]: t for t in after["tasks"]} + assert after["status"] == "cancelled" and after["finished_at"] + assert by["channels"]["status"] == "ok" and by["visits"]["status"] == "cancelled" + assert model.cancelled == 1, "正在进行的模型调用被停掉" + detail = (await client.get(f"{_base(sid)}/channels")).json() + assert detail["notes"]["kind"]["value"] == "dimension", "已经写进目录的不回滚" + assert (await client.get(f"{_base(sid)}/draft-jobs/active")).json()["job"] is None + + +async def test_startup_marks_unfinished_jobs_interrupted_and_does_not_resume(client): + sid = await _source() + async with SessionLocal() as session: + job = CatalogDraftJob(source_id=sid, status="running", use_model=True) + session.add(job) + await session.flush() + for seq, (name, status) in enumerate([("visits", "ok"), ("channels", "running"), ("gates", "pending")]): + session.add(CatalogDraftTask(job_id=job.id, seq=seq, table_name=name, status=status)) + await session.commit() + job_id = job.id + assert await catalog_jobs.startup() >= 1 + got = await get_job(client, sid, job_id) + by = {t["table_name"]: t for t in got["tasks"]} + assert got["status"] == "interrupted" and got["error"] == "服务重启，起草中断" and got["finished_at"] + assert by["visits"]["status"] == "ok" and by["visits"]["error"] is None + assert by["channels"]["status"] == by["gates"]["status"] == "cancelled" + assert by["channels"]["error"] == by["gates"]["error"] == "服务重启，起草中断" + assert not catalog_jobs.manager.running(job_id), "不自动续跑" + + +async def test_shutdown_stops_running_jobs_as_interrupted(client, monkeypatch): + sid = await _source() + model = Held(REPLIES, hold={"visits"}) + use(monkeypatch, model) + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["visits"], "use_model": True})).json()["job"] + + async def running() -> bool: + return (await get_job(client, sid, job["id"]))["tasks"][0]["status"] == "running" + + await until(running) + await catalog_jobs.manager.shutdown(grace=5) + got = await get_job(client, sid, job["id"]) + assert got["status"] == "interrupted" and got["tasks"][0]["status"] == "cancelled" + assert got["tasks"][0]["error"] == "服务重启，起草中断" + + +# -------------------------------------------------------------------------- +# 重试 +# -------------------------------------------------------------------------- + + +async def test_retry_takes_only_failed_partial_and_cancelled_tables(client, monkeypatch): + sid = await _source() + use(monkeypatch, FakeModel(REPLIES, fail={"visits"})) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + await client.put(f"{_base(sid)}/guidance", json={"guidance": "旧的要求"}) + first = (await client.post(f"{_base(sid)}/draft-jobs", json={ + "tables": ["visits", "channels", "nope"], "use_model": True, "note": "第一次"})).json()["job"] + await finished(first["id"]) + await client.put(f"{_base(sid)}/guidance", json={"guidance": "新的要求"}) + assert (await get_job(client, sid, first["id"]))["guidance"] == "旧的要求", "任务记下的是当时的快照" + + use(monkeypatch, FakeModel(REPLIES)) + r = await client.post(f"{_base(sid)}/draft-jobs/{first['id']}/retry", headers=ACTOR) + assert r.status_code == 202, r.text + retry = r.json()["job"] + assert [t["table_name"] for t in retry["tasks"]] == ["visits"] + assert retry["use_model"] is True and retry["note"] == "第一次" and retry["guidance"] == "新的要求" + assert retry["retry_of"] == first["id"] and retry["created_by"] == "王敏" + await finished(retry["id"]) + assert (await get_job(client, sid, retry["id"]))["status"] == "succeeded" + + nothing = await client.post(f"{_base(sid)}/draft-jobs/{retry['id']}/retry") + assert nothing.status_code == 422 and "没有" in nothing.json()["detail"] + + +async def test_retry_while_busy_is_409(client, monkeypatch): + sid = await _source() + monkeypatch.setattr(catalog_jobs.manager, "start", lambda job_id: None) + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]})).json()["job"] + await client.post(f"{_base(sid)}/draft-jobs/{job['id']}/cancel") + other = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["channels"]})).json()["job"] + assert other["status"] == "queued" + r = await client.post(f"{_base(sid)}/draft-jobs/{job['id']}/retry") + assert r.status_code == 409 and r.json()["code"] == "catalog_draft_busy" + + +async def test_retry_with_an_unavailable_model_is_409(client, monkeypatch): + sid = await _source() + monkeypatch.setattr(catalog_jobs.manager, "start", lambda job_id: None) + use(monkeypatch, FakeModel(REPLIES)) + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"], "use_model": True})).json()["job"] + await client.post(f"{_base(sid)}/draft-jobs/{job['id']}/cancel") + + async def unavailable(session): + raise catalog.CatalogModelUnavailable("当前只有演示用的模型接入，无法用于起草数据目录") + + monkeypatch.setattr(catalog, "resolve_draft_model", unavailable) + r = await client.post(f"{_base(sid)}/draft-jobs/{job['id']}/retry") + assert r.status_code == 409 and r.json()["code"] == "catalog_draft_model_unavailable" + + +# -------------------------------------------------------------------------- +# 目录清单上的起草状态 +# -------------------------------------------------------------------------- + + +async def test_the_list_carries_last_draft_per_table(client, monkeypatch): + sid = await _source() + use(monkeypatch, FakeModel(REPLIES, fail={"visits"})) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + first = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["visits", "channels"], + "use_model": True})).json()["job"] + await finished(first["id"]) + second = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["channels"]})).json()["job"] + await finished(second["id"]) + body = (await client.get(_base(sid))).json() + rows = {r["table_name"]: r for r in body["tables"]} + assert rows["gates"]["last_draft"] is None + visits = rows["visits"]["last_draft"] + assert visits["status"] == "failed" and visits["job_id"] == first["id"] and visits["finished_at"] + assert "网关超时" in visits["model_error"] and visits["error"] is None + assert rows["channels"]["last_draft"]["job_id"] == second["id"], "取最近一次任务里的那一项" + assert body["guidance"] is None and body["active_job"] is None + + +# -------------------------------------------------------------------------- +# 起草要求 +# -------------------------------------------------------------------------- + + +async def test_guidance_is_saved_trimmed_cleared_and_limited(client): + sid = await _source() + r = await client.put(f"{_base(sid)}/guidance", json={"guidance": " 票务库，金额按元。\n状态 9 是作废 "}) + assert r.status_code == 200, r.text + assert r.json() == {"guidance": "票务库，金额按元。\n状态 9 是作废"} + assert (await client.get(_base(sid))).json()["guidance"] == "票务库，金额按元。\n状态 9 是作废" + too_long = await client.put(f"{_base(sid)}/guidance", json={"guidance": "字" * 2001}) + assert too_long.status_code == 422 and "2000" in too_long.json()["detail"] + assert (await client.get(_base(sid))).json()["guidance"] == "票务库，金额按元。\n状态 9 是作废" + assert (await client.put(f"{_base(sid)}/guidance", json={"guidance": "字" * 2000})).status_code == 200 + cleared = await client.put(f"{_base(sid)}/guidance", json={"guidance": " "}) + assert cleared.json() == {"guidance": None} + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + assert CATALOG_DRAFT_GUIDANCE_OPTION not in (row.options or {}) + assert (await client.put(f"{_base('missing-source')}/guidance", json={"guidance": "x"})).status_code == 404 + + +async def test_a_table_named_guidance_can_still_be_saved(client): + """起草要求的路径和「整份提交一张表」撞名：表名恰好叫 guidance 时，{notes, if_version} 照旧是那张表的提交， + 不会被当成清除起草要求；两样都没写的 422。""" + tables = _tables() + tables["guidance"] = {"qualified": "guidance", "columns": [_col("id"), _col("body", "TEXT")], + "primary_key": ["id"], "comment": None, "is_view": False, "foreign_keys": []} + sid = await _source(tables) + await client.put(f"{_base(sid)}/guidance", json={"guidance": "金额按元"}) + r = await client.put(f"{_base(sid)}/guidance", json={"notes": {"label": {"value": "操作指引"}}, "if_version": 0}, + headers=ACTOR) + assert r.status_code == 200, r.text + assert r.json()["table_name"] == "guidance" and r.json()["notes"]["label"]["value"] == "操作指引" + assert (await client.get(_base(sid))).json()["guidance"] == "金额按元" + for body in ({}, {"guidance": 3}, {"guidance": ["金额按元"]}): + r = await client.put(f"{_base(sid)}/guidance", json=body) + assert r.status_code == 422, body + assert isinstance(r.json()["detail"], str) and "起草要求" in r.json()["detail"], "说明是中文的一句话" + assert (await client.get(_base(sid))).json()["guidance"] == "金额按元" + + +async def test_guidance_of_an_uploaded_source_lives_on_its_own_row(client): + sid = await _source(origin="upload") + r = await client.put(f"{_base(sid)}/guidance", json={"guidance": "按期上传的客流日报"}) + assert r.status_code == 200, r.text + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + assert row.options[CATALOG_DRAFT_GUIDANCE_OPTION] == "按期上传的客流日报" + + +async def test_the_datasource_form_neither_shows_nor_drops_the_guidance(client): + sid = await _source(options={"schema": "main"}) + await client.put(f"{_base(sid)}/guidance", json={"guidance": "金额按元"}) + shown = (await client.get("/api/datasources")).json() + mine = next(s for s in shown if s["id"] == sid) + assert CATALOG_DRAFT_GUIDANCE_OPTION not in mine["options"] and mine["options"]["schema"] == "main" + # 表单整份回写 options（不带起草要求，或者带着一份别的）：起草要求留着，只认 PUT /catalog/guidance + r = await client.patch(f"/api/datasources/{sid}", json={"options": {"schema": "other", + CATALOG_DRAFT_GUIDANCE_OPTION: "篡改"}}) + assert r.status_code == 200, r.text + assert CATALOG_DRAFT_GUIDANCE_OPTION not in r.json()["options"] + assert (await client.get(_base(sid))).json()["guidance"] == "金额按元" + async with SessionLocal() as session: + assert (await session.get(DataSource, sid)).options == {"schema": "other", + CATALOG_DRAFT_GUIDANCE_OPTION: "金额按元"} + + +def test_the_guidance_never_reaches_the_driver(): + src = SimpleNamespace(kind="mysql", host="db.example", port=3306, database="shop", username="reader", + password=None, options={"charset": "utf8mb4", CATALOG_DRAFT_GUIDANCE_OPTION: "金额按元"}) + url = build_url(src) + assert "charset=utf8mb4" in url and CATALOG_DRAFT_GUIDANCE_OPTION not in url + + +async def test_the_sync_draft_uses_the_source_guidance(client, monkeypatch): + sid = await _source(description="虚构的景区票务库") + model = FakeModel(REPLIES) + use(monkeypatch, model) + await client.put(f"{_base(sid)}/guidance", json={"guidance": "渠道表的名称是对外展示名"}) + r = await client.post(f"{_base(sid)}/draft", json={"tables": ["channels"], "use_model": True}) + assert r.status_code == 200, r.text + assert "数据源说明：虚构的景区票务库\n起草要求：渠道表的名称是对外展示名" in model.calls[0] + assert set(r.json()) == {"tables", "total", "model_used", "model", "model_error"}, "旧的同步接口返回形状不变" + + +async def test_jobs_go_away_with_their_source(client): + sid = await _source() + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]})).json()["job"] + await finished(job["id"]) + await drop_source(sid) + _CREATED.remove(sid) + async with SessionLocal() as session: + assert (await session.execute(select(CatalogDraftJob).where(CatalogDraftJob.id == job["id"]))).first() is None + assert (await session.execute(select(CatalogDraftTask).where(CatalogDraftTask.job_id == job["id"]))).first() \ + is None diff --git a/backend/tests/test_catalog_draft_settings.py b/backend/tests/test_catalog_draft_settings.py new file mode 100644 index 00000000..d36e73e1 --- /dev/null +++ b/backend/tests/test_catalog_draft_settings.py @@ -0,0 +1,270 @@ +"""起草模型（设置里的 catalog_draft 一组）与起草取模型的顺序。 + +- 设置组 catalog_draft：{provider, model}，经 PUT /api/settings 写、GET 读，没存过补 {provider: null, model: null}； + provider 不是字符串、点名的接入不存在或已停用 → 422，说法写设置页上的叫法「起草模型 · 接入」「起草模型 · 模型」 +- 取模型的顺序（数据目录起草和按配方导入的 AI 起草共用 draft_model.draft_model_choice）：起草模型填了接入或模型就整组 + 用它；否则沿用助手的模型设置；再没有就是默认接入。不跨级拼接入和模型 +- 数据目录起草不关思考（按接入默认）、额度 32768，服务方嫌大退回 8192；配方 AI 起草只改取模型的顺序，额度、思考不动 + +接入名、模型名都是虚构的。 +""" +from __future__ import annotations + +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete + +from app.data import catalog, draft_model +from app.data import recipe_ai as A +from app.db.base import SessionLocal +from app.db.models import Provider, Setting +from app.main import app + +LIVE, OFF, GONE = "起草用接入", "起草用停用接入", "起草用不存在的接入" + + +async def _clean() -> None: + async with SessionLocal() as session: + await session.execute(delete(Setting).where(Setting.key.in_([draft_model.CATALOG_DRAFT_SETTING_KEY]))) + await session.execute(delete(Provider).where(Provider.name.in_([LIVE, OFF]))) + await session.commit() + + +@pytest.fixture +async def providers(): + """一个启用、一个停用的演示类接入（不联网）；用完删掉。""" + await _clean() + async with SessionLocal() as session: + session.add_all([ + Provider(name=LIVE, kind="mock", models=[{"id": "起草模型甲"}], default_model="起草模型甲", enabled=True, + extra={}), + Provider(name=OFF, kind="mock", models=[{"id": "起草模型乙"}], default_model="起草模型乙", enabled=False, + extra={}), + ]) + await session.commit() + yield + await _clean() + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +# -------------------------------------------------------------------------- +# 设置组的读写与校验 +# -------------------------------------------------------------------------- + + +async def test_the_group_is_exposed_with_empty_defaults(client, providers): + got = (await client.get("/api/settings")).json() + assert got["catalog_draft"] == {"provider": None, "model": None} + + +async def test_the_group_round_trips(client, providers): + r = await client.put("/api/settings", json={"values": {"catalog_draft": {"provider": LIVE, "model": "起草模型甲"}}}) + assert r.status_code == 200, r.text + assert r.json()["catalog_draft"] == {"provider": LIVE, "model": "起草模型甲"} + got = (await client.get("/api/settings")).json()["catalog_draft"] + assert got == {"provider": LIVE, "model": "起草模型甲"} + # 只写模型、接入留空：照存，接入是 null + r = await client.put("/api/settings", json={"values": {"catalog_draft": {"provider": " ", "model": "x"}}}) + assert r.status_code == 200, r.text + assert r.json()["catalog_draft"] == {"provider": None, "model": "x"} + + +@pytest.mark.parametrize("value,label", [ + ({"provider": 3}, "起草模型 · 接入"), + ({"model": ["a"]}, "起草模型 · 模型"), + ({"temperature": 0}, "temperature"), + ("起草模型甲", "起草模型"), +]) +async def test_bad_values_are_refused_with_the_settings_page_wording(client, providers, value, label): + r = await client.put("/api/settings", json={"values": {"catalog_draft": value, "ui": {"theme": "dark"}}}) + assert r.status_code == 422, r.text + assert label in r.json()["detail"] + got = (await client.get("/api/settings")).json() + assert got["catalog_draft"] == {"provider": None, "model": None} and got["ui"]["theme"] != "dark" + + +@pytest.mark.parametrize("name,reason,fix", [ + (GONE, f"模型接入「{GONE}」不存在", "请在「设置 → 模型接入」中核对名称"), + (OFF, f"模型接入「{OFF}」已停用", "请在「设置 → 模型接入」中启用它"), +]) +async def test_a_missing_or_disabled_provider_is_refused_with_the_field(client, providers, name, reason, fix): + r = await client.put("/api/settings", json={"values": {"catalog_draft": {"provider": name}}}) + assert r.status_code == 422, r.text + body = r.json() + assert body["code"] == "settings_provider_invalid" and body["field"] == "catalog_draft.provider" + assert body["detail"] == f"「起草模型 · 接入」指定的{reason}。{fix}，或换一个接入" + async with SessionLocal() as session: + assert await session.get(Setting, "catalog_draft") is None + + +# -------------------------------------------------------------------------- +# 取模型的顺序 +# -------------------------------------------------------------------------- + + +class _MemDB: + """独立的内存库：取模型只看这里的接入和设置，不受别的测试建的接入影响。""" + + async def __aenter__(self): + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + from sqlalchemy.pool import StaticPool + + import app.db.models # noqa: F401 + from app.db.base import Base + + self.engine = create_async_engine("sqlite+aiosqlite:///:memory:", poolclass=StaticPool) + async with self.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + self.session = async_sessionmaker(self.engine, expire_on_commit=False, class_=AsyncSession)() + return self.session + + async def __aexit__(self, *exc): + await self.session.close() + await self.engine.dispose() + + +@pytest.fixture +async def mem(): + async with _MemDB() as session: + yield session + + +@pytest.fixture +def no_env_keys(monkeypatch): + for k in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "OPENAI_API_KEY", "ANTHROPIC_BASE_URL"): + monkeypatch.delenv(k, raising=False) + + +async def _setup(session, *, draft: dict | None = None, copilot: dict | None = None) -> None: + from app.core.crypto import encrypt + + for name, model in (("甲接入", "模型甲"), ("乙接入", "模型乙"), ("丙接入", "模型丙")): + session.add(Provider(name=name, kind="anthropic", api_key=encrypt("sk-test"), models=[{"id": model}], + default_model=model, enabled=True, extra={})) + if draft is not None: + session.add(Setting(key=draft_model.CATALOG_DRAFT_SETTING_KEY, value=draft)) + if copilot is not None: + session.add(Setting(key=draft_model.COPILOT_SETTING_KEY, value=copilot)) + await session.commit() + + +@pytest.mark.parametrize("draft,copilot,expected", [ + (None, None, (None, None, "default")), + (None, {"provider": "乙接入", "model": ""}, ("乙接入", None, "assistant")), + ({"provider": None, "model": None}, {"provider": "", "model": "模型乙"}, (None, "模型乙", "assistant")), + ({"provider": "", "model": "模型丙"}, {"provider": "乙接入", "model": "模型乙"}, (None, "模型丙", "draft")), + ({"provider": "甲接入", "model": None}, {"provider": "乙接入", "model": "模型乙"}, ("甲接入", None, "draft")), + ({"provider": 7}, {"provider": "乙接入"}, ("乙接入", None, "assistant")), # 存坏了的按没填 +]) +async def test_choice_order(mem, draft, copilot, expected): + await _setup(mem, draft=draft, copilot=copilot) + got = await draft_model.draft_model_choice(mem) + assert (got.provider, got.model, got.origin) == expected + + +def _spy(monkeypatch) -> list[Any]: + from app.providers import factory + + specs: list[Any] = [] + + class Built: + def __init__(self, provider, spec): + self.provider, self.spec = provider.name, spec + + def fake_build(provider, spec): + specs.append((provider.name, spec)) + return Built(provider, spec) + + monkeypatch.setattr(factory, "build_chat_model", fake_build) + return specs + + +async def test_catalog_draft_uses_the_draft_setting_keeps_thinking_and_a_big_budget(mem, no_env_keys, monkeypatch): + await _setup(mem, draft={"provider": "甲接入", "model": None}, copilot={"provider": "乙接入", "model": "模型乙"}) + specs = _spy(monkeypatch) + model, model_id = await catalog.resolve_draft_model(mem) + assert model_id == "模型甲" + assert [p for p, _ in specs] == ["甲接入", "甲接入"] + assert [s.max_tokens for _, s in specs] == [catalog.MODEL_MAX_TOKENS, catalog.MODEL_FALLBACK_MAX_TOKENS] + assert catalog.MODEL_MAX_TOKENS == 32768 and catalog.MODEL_FALLBACK_MAX_TOKENS == 8192 + assert all(s.thinking is None and s.temperature == 0 for _, s in specs), "不关思考，按接入默认" + assert all(s.timeout == catalog.MODEL_TIMEOUT_S == 420 for _, s in specs) + assert all(s.max_retries == 0 for _, s in specs), "重试只由起草引擎按失败原因来做，SDK 自己不再重试" + assert model.budget == 32768 and model.model_name == "模型甲" + + +@pytest.mark.parametrize("kind,extra", [ + ("openai_compatible", {}), + ("anthropic", {}), + ("anthropic", {"auth_style": "bearer"}), +], ids=["openai-compatible", "anthropic", "anthropic-bearer"]) +def test_the_built_model_does_not_retry_on_its_own_when_asked(kind, extra): + """ModelSpec.max_retries 一直传到 SDK 客户端（Bearer 鉴权换上的客户端也是）；不写就照 SDK 默认。""" + from app.core.crypto import encrypt + from app.providers.factory import ModelSpec, build_chat_model + + provider = Provider(name="虚构接入", kind=kind, api_key=encrypt("sk-test"), base_url="https://gateway.example/v1", + models=[], default_model="model-x", extra=extra) + + def client_retries(chat: Any) -> int: + client = chat.root_async_client if kind == "openai_compatible" else chat._async_client + return client.max_retries + + assert client_retries(build_chat_model(provider, ModelSpec(model="model-x", max_retries=0))) == 0 + assert client_retries(build_chat_model(provider, ModelSpec(model="model-x"))) == 2 + + +async def test_catalog_draft_falls_back_to_the_assistant_model(mem, no_env_keys, monkeypatch): + await _setup(mem, copilot={"provider": "", "model": "模型乙"}) + specs = _spy(monkeypatch) + _, model_id = await catalog.resolve_draft_model(mem) + assert model_id == "模型乙" and specs[0][0] == "乙接入" + + +async def test_recipe_ai_draft_follows_the_same_order_but_keeps_its_own_spec(mem, no_env_keys, monkeypatch): + await _setup(mem, draft={"provider": "", "model": "模型丙"}, copilot={"provider": "乙接入", "model": "模型乙"}) + specs = _spy(monkeypatch) + model, model_id, avail = await A.resolve_draft_model(mem) + assert avail.available and model_id == "模型丙" and avail.provider == "丙接入" + assert [s.max_tokens for _, s in specs] == [A.AI_MAX_TOKENS, A.AI_FALLBACK_MAX_TOKENS] + assert all(s.thinking == "off" and s.temperature == 0 for _, s in specs), "配方 AI 起草的思考设置不动" + assert all(s.max_retries is None for _, s in specs), "配方 AI 起草的重试照旧（SDK 默认）" + + +async def test_a_broken_draft_setting_says_where_to_fix_it(mem, no_env_keys): + """起草模型点名的接入不存在：报错说的是起草模型，不是助手的模型。沿用助手模型时照旧说助手。""" + from app.core.crypto import encrypt + + mem.add(Provider(name="停用的接入", kind="anthropic", api_key=encrypt("k"), models=[{"id": "模型丁"}], + default_model="模型丁", enabled=False, extra={})) + mem.add(Setting(key=draft_model.CATALOG_DRAFT_SETTING_KEY, value={"provider": "停用的接入"})) + await mem.commit() + with pytest.raises(catalog.CatalogModelUnavailable) as e: + await catalog.resolve_draft_model(mem) + assert "起草模型" in str(e.value) and "停用的接入" in str(e.value) and "已停用" in str(e.value) + assert "助手" not in str(e.value) + _, _, avail = await A.resolve_draft_model(mem) + assert not avail.available and "起草模型" in avail.reason and "助手" not in avail.reason + + await mem.execute(delete(Setting)) + mem.add(Setting(key=draft_model.COPILOT_SETTING_KEY, value={"provider": "停用的接入"})) + await mem.commit() + with pytest.raises(catalog.CatalogModelUnavailable) as e: + await catalog.resolve_draft_model(mem) + assert "助手使用的" in str(e.value) + + +async def test_only_a_demo_provider_is_refused(mem, no_env_keys): + mem.add(Provider(name="演示", kind="mock", models=[{"id": "mock-fast"}], default_model="mock-fast", enabled=True, + extra={})) + await mem.commit() + with pytest.raises(catalog.CatalogModelUnavailable) as e: + await catalog.resolve_draft_model(mem) + assert "演示" in str(e.value) diff --git a/backend/tests/test_catalog_merge.py b/backend/tests/test_catalog_merge.py index f7584664..260e9bf5 100644 --- a/backend/tests/test_catalog_merge.py +++ b/backend/tests/test_catalog_merge.py @@ -87,11 +87,58 @@ def test_merge_removes_stale_items_only_for_covered_sources(): def test_merge_keeps_column_entries_tidy(): - existing = {"columns": {"amount": {"unit": make_item("元", "llm")}}} - merged, stats = catalog.merge_notes(existing, {}, covered={"llm"}, at=AT) + existing = {"columns": {"amount": {"unit": make_item("元", "comment")}}} + merged, stats = catalog.merge_notes(existing, {}, covered={"comment"}, at=AT) assert merged == {} and stats.removed == 1 +COLUMNS = ["id", "amount", "visit_date"] + + +def test_a_redraft_keeps_model_items_the_model_left_blank_this_time(): + """模型这次对某一项没给值（留空，或者给的值没通过校验、在 _from_model 里就丢掉了）：以前模型起草的项留着，不算删除。 + 只有给了合格的新值才替换。""" + existing = {"label": make_item("入园记录", "llm"), "kind": make_item("fact", "llm"), + "keys": make_item(["id"], "llm"), "business_date": make_item({"column": "visit_date"}, "llm"), + "columns": {"amount": {"label": make_item("实收金额", "llm"), "unit": make_item("元", "llm")}, + "visit_date": {"meaning": make_item("游客检票入园的日期", "llm")}}} + draft = {"label": make_item("入园明细", "llm"), "columns": {"amount": {"unit": make_item("元", "llm")}}} + merged, stats = catalog.merge_notes(existing, draft, covered={"llm"}, at=AT, columns=COLUMNS) + assert stats == catalog.MergeStats(added=0, updated=1, removed=0) + assert merged["label"]["value"] == "入园明细" and merged["label"]["updated_at"] == AT + for key in ("kind", "keys", "business_date"): + assert merged[key] == existing[key], key + assert merged["columns"] == existing["columns"] + # 不知道表结构里有哪些列时更不删 + merged, stats = catalog.merge_notes(existing, {}, covered={"llm"}, at=AT) + assert merged == existing and stats.removed == 0 + + +def test_model_items_of_columns_gone_from_the_schema_are_still_removed(): + """表结构里已经没有的列：这一轮完整起草过（covered 含 llm）时，模型起草的这一列的项照旧删掉；业务主键、业务日期 + 指着这一列的也删。没完整起草过（covered 不含 llm）的不删。""" + existing = {"keys": make_item(["id", "old_no"], "llm"), "business_date": make_item({"column": "old_day"}, "llm"), + "grain": make_item("每次入园一行", "llm"), + "columns": {"old_no": {"label": make_item("旧编号", "llm"), "meaning": make_item("已经删掉的列", "llm")}, + "amount": {"label": make_item("实收金额", "llm")}}} + merged, stats = catalog.merge_notes(existing, {}, covered={"llm"}, at=AT, columns=COLUMNS) + assert stats.removed == 4 + assert set(merged) == {"grain", "columns"} and set(merged["columns"]) == {"amount"} + merged, stats = catalog.merge_notes(existing, {}, covered=set(), at=AT, columns=COLUMNS) + assert merged == existing and stats.removed == 0 + + +def test_other_sources_keep_their_covered_rule(): + """只改模型起草的「覆盖即删」：数据库注释、外键、命名推断、剖析照旧——这一轮完整算过、却没有了的删掉。""" + existing = {"label": make_item("入园记录", "comment"), + "columns": {"amount": {"label": make_item("实收金额", "comment"), + "meaning": make_item("模型写的含义", "llm")}}, + "relations": [_rel(["gate_id"], "gates", ["id"], source="name")]} + merged, stats = catalog.merge_notes(existing, {}, covered={"comment", "name", "llm"}, at=AT, columns=COLUMNS) + assert stats.removed == 3 + assert merged == {"columns": {"amount": {"meaning": make_item("模型写的含义", "llm")}}} + + # ---------------------------------------------------------------- 单项审阅 diff --git a/backend/tests/test_masking_tool_boundary.py b/backend/tests/test_masking_tool_boundary.py index 9f2797ad..2323c2af 100644 --- a/backend/tests/test_masking_tool_boundary.py +++ b/backend/tests/test_masking_tool_boundary.py @@ -12,6 +12,7 @@ from __future__ import annotations import json +import re import pytest from httpx import ASGITransport, AsyncClient @@ -86,7 +87,8 @@ async def test_the_model_and_the_snapshot_see_only_masks_when_the_analysis_fails assert _clean(json.dumps(snapshot, ensure_ascii=False)) async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: got = await client.get(f"/api/artifacts/{out.queries[0]['artifact']}") - assert got.status_code == 200 and _clean(got.text) and "1200" not in got.text + # 金额原值不出现。先去掉哈希再查：工件 id、快照里记的哈希是随机的十六进制，偶尔碰巧含「1200」 + assert got.status_code == 200 and _clean(got.text) and "1200" not in re.sub(r"[0-9a-f]{32,}", "", got.text) assert out.result["value"] is None assert out.cells[0]["reason"] == "Q1 的写法认不全，整份结果查询时已遮罩，不采信" diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 00edd8da..eb0f3abf 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,5 +1,5 @@ import type { - ActivateSnapshotBody, ActivateSnapshotOut, AiPreview, CatalogDetail, CatalogDraftOut, CatalogImpact, CatalogList, CatalogNotes, + ActivateSnapshotBody, ActivateSnapshotOut, AiPreview, CatalogDetail, CatalogDraftJob, CatalogImpact, CatalogList, CatalogNotes, CatalogPatchPreview, CatalogPatchSubmit, CatalogProfileOut, CatalogReviewAction, CommitOut, CurrentRecipe, EditPreview, EditRequest, ImportRecord, ManifestOut, PurgeRawBody, PurgeRawOut, QuestionAnswer, Recipe, RedraftRulesOut, RemovePeriodBody, RemovePeriodOut, @@ -31,6 +31,9 @@ export type { ApiErrorKind, RequestOptions, UploadOptions, UploadProgress } from const isObj = (v: unknown): v is Record => !!v && typeof v === 'object' && !Array.isArray(v) +/** 数据目录的接口前缀：/datasources/{id}/catalog */ +const catalogBase = (sourceId: string) => `/datasources/${encodeURIComponent(sourceId)}/catalog` + /** * 上传表格被退回、要用户先选怎么处理（422 带 decision）时，取出那份决定；别的错误返回 null。 * @@ -409,12 +412,38 @@ export const api = { post(`/datasources/${encodeURIComponent(sourceId)}/catalog/${encodeURIComponent(table)}/patch`, { changes, if_version: ifVersion }), /** - * 同步起草（注释、外键、命名推断；use_model 时再请助手的模型起草）。不给 tables 时按使用次数取前 20 张。 - * 人工确认、驳回过的项不动。没有表结构时回 409 + * 建一个起草任务（后台进行：注释、外键、命名推断；use_model 时再请起草模型起草，带上数据源的起草要求和这次的 note）。 + * 不给 tables 时按使用次数取前 20 张；表结构里没有的表照样建进任务、记 skipped。人工确认、驳回过的项不动。 + * 409：同一个数据源已有排队、进行中的任务（code 为 catalog_draft_busy，改去取 activeDraftJob）；起草模型用不了 + * （catalog_draft_model_unavailable，不建任务）；没有表结构 */ - draft: (sourceId: string, body: { tables?: string[]; use_model: boolean }, opts?: RequestOptions) => - request(`/datasources/${encodeURIComponent(sourceId)}/catalog/draft`, - { method: 'POST', body: JSON.stringify(body), ...opts }), + createDraftJob: (sourceId: string, body: { tables?: string[]; use_model: boolean; note?: string }) => + post<{ job: CatalogDraftJob }>(`${catalogBase(sourceId)}/draft-jobs`, body), + /** 排队或进行中的那个起草任务；没有为 null */ + activeDraftJob: (sourceId: string, opts?: RequestOptions) => + get<{ job: CatalogDraftJob | null }>(`${catalogBase(sourceId)}/draft-jobs/active`, opts), + /** 最近建的一个起草任务（任何状态）；没有为 null */ + latestDraftJob: (sourceId: string, opts?: RequestOptions) => + get<{ job: CatalogDraftJob | null }>(`${catalogBase(sourceId)}/draft-jobs/latest`, opts), + /** 一个起草任务的当前状态；不存在或不属于这个数据源回 404 */ + draftJob: (sourceId: string, jobId: string, opts?: RequestOptions) => + get<{ job: CatalogDraftJob }>(`${catalogBase(sourceId)}/draft-jobs/${encodeURIComponent(jobId)}`, opts), + /** + * 停止起草：排队的直接停；进行中的停掉正在进行的模型调用，没开始和没写完的表记已停止，已经写进目录的不回滚。 + * 已经结束的回 409（catalog_draft_finished） + */ + cancelDraftJob: (sourceId: string, jobId: string) => + post<{ job: CatalogDraftJob }>(`${catalogBase(sourceId)}/draft-jobs/${encodeURIComponent(jobId)}/cancel`), + /** + * 只重试失败的表：新任务只含原任务里失败、部分完成、已停止的表，用模型和补充说明沿用，起草要求取数据源现在的。 + * 没有可重试的表 422；有进行中的任务 409（catalog_draft_busy）；和建任务一样，起草模型用不了（原任务用模型时） + * 409 catalog_draft_model_unavailable、没有表结构 409 + */ + retryDraftJob: (sourceId: string, jobId: string) => + post<{ job: CatalogDraftJob }>(`${catalogBase(sourceId)}/draft-jobs/${encodeURIComponent(jobId)}/retry`), + /** 存数据源的起草要求（去首尾空白，最多 2000 字，空串清除）。上传表格建的源存在它自己的数据源上 */ + putGuidance: (sourceId: string, guidance: string) => + put<{ guidance: string | null }>(`${catalogBase(sourceId)}/guidance`, { guidance }), /** * 同步剖析：对业务库发少量只读查询，核对关系、取码值候选，结论写进目录。不给 tables 时按使用次数取前 10 张 * 有待核实关系的表。没开启、数据源停用、没有表结构、同一个数据源正在剖析、快照被改动时回 409（detail 是一句中文） diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 61073e35..663b6329 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -43,10 +43,11 @@ */ import type { - CatalogCardinality, CatalogMeasure, CatalogProfileNumberKey, CatalogSource, CatalogStatus, CatalogTableKind, NodeType, SnapshotReasonCode, + CatalogCardinality, CatalogDraftJobStatus, CatalogDraftTaskStatus, CatalogMeasure, CatalogProfileNumberKey, CatalogSource, CatalogStatus, + CatalogTableKind, NodeType, SnapshotReasonCode, SqlCheckCode, SqlCheckLevel, ToolTrust, } from '../types' -import { formatCost, formatNumber } from './format' +import { formatCost, formatNumber, formatTokens } from './format' export const TERMS = { workflow: '工作流', @@ -963,6 +964,8 @@ export const PROVIDER_REF_TEXT = { judgeBroken: '裁判调用将失败，所有结论句记为未裁判', // 和核查一组「不存在」那句（VERIFY_SETTING_TEXT.section.providerGone）同一个后果 verifyBroken: '核查调用将失败', + // 起草模型（数据目录的用模型起草、按配方导入的 AI 起草）：开始起草时就报错，不建任务 + draftBroken: '用模型起草时将报错，无法起草', /** * 存着的接入名恰好是某个启用接入的模型（多半是把模型名填进了接入那一栏）：照「启用它」去做，启用的是那条旧接入。 * 和服务端保存时 422 的那一句（factory.model_note）一字不差；两栏写这一组的叫法（「门控模型 · 模型」「门控模型 · 接入」） @@ -2198,7 +2201,7 @@ export const CATALOG_TEXT = { noSchemaBody: (note: string) => `${note.replace(/[。.]$/, '')}。数据目录按表结构逐表记录，请先回到数据源卡片点击「探查结构」。`, noSchemaUploadBody: '这个表格还没有可用的表结构，请重新上传后再编写数据目录。', emptyTitle: '还没有数据目录', - emptyBody: '先起草，再逐表确认。起草按数据库注释、外键约束和列名推断，也可以请助手的模型起草中文名和含义。', + emptyBody: '先起草，再逐表确认。起草按数据库注释、外键约束和列名推断，也可以用模型起草中文名和含义。', // ---- 导入表格的说明 systemTitle: '导入时生成的说明', systemBody: '这个数据源由导入表格生成，表和列的说明由系统按核对结果生成，只读，不能在数据目录中改写。', @@ -2260,29 +2263,17 @@ export const CATALOG_TEXT = { scopeAll: (n: number) => `全部 ${formatNumber(n)} 张表`, draftBase: '按数据库注释、外键约束和列名推断起草。已确认、已驳回的项不会被改动；起草出的项都是推断，需要逐项确认。', useModel: '用模型起草', - useModelHint: '请助手的模型按表结构起草中文名、说明、粒度和列的含义。只发送表结构（表名、列名、类型、约束、注释），不发送数据；耗时较长，并产生模型费用。', + useModelHint: '请起草模型按表结构起草中文名、说明、粒度和列的含义。只发送表结构（表名、列名、类型、约束、注释）和起草要求，不发送数据；在后台进行，耗时较长，并产生模型费用。', draftStart: (n: number) => `起草 ${formatNumber(n)} 张表`, - drafting: '正在起草…', - draftProgress: (done: number, total: number) => `已完成 ${formatNumber(done)} / ${formatNumber(total)} 张表`, - draftStop: '停止', - draftStopping: '当前这一批完成后停止…', - draftDone: '起草完成', - draftStopped: (done: number, total: number) => `已停止：完成 ${formatNumber(done)} / ${formatNumber(total)} 张表，其余未起草`, - draftFailed: '起草中断', draftSummary: (added: number, updated: number, removed: number) => `新增 ${formatNumber(added)} 项、更新 ${formatNumber(updated)} 项${removed ? `、删除 ${formatNumber(removed)} 项` : ''}`, - // 只在起草确实给出了内容、只是和现有目录一致时用：有表的模型起草失败或没给出可用内容（model_error）时 - // 不能说「没有变化」，改用 draftNothingWritten + // 只在起草确实给出了内容、只是和现有目录一致时用：模型没给出可用内容时说 draftNothingWritten，有表没写全时说 + // draftNothingWrittenFailed，不能说「没有变化」 draftNoChange: '没有变化：起草出的内容与现有目录一致', draftNothingWritten: '目录没有更新：模型没有给出可用的内容，原因见下方', - draftModelSkipped: (reason: string) => `模型未参与起草：${reason.replace(/[。.]$/, '')}。已按注释、外键和命名起草。`, - draftModelUsed: (model: string) => `模型：${model}`, - draftTableErrors: (n: number) => `${formatNumber(n)} 张表未能起草`, - draftModelErrors: (n: number) => `${formatNumber(n)} 张表的模型起草失败，已按注释、外键和命名起草`, + draftNothingWrittenFailed: '目录没有更新：部分表未写全，原因见下方', draftClose: '完成', - moreTables: (n: number) => `…另有 ${formatNumber(n)} 张`, cancel: '取消', - draftBackground: '起草仍在进行，当前这一批完成后停止', // ---- 表详情 backToList: '返回表清单', prev: '上一张', @@ -2389,6 +2380,201 @@ export const CATALOG_TEXT = { reloadDiscardAction: '放弃修改并重新载入', } +// =========================================================================== +// 数据目录的起草任务（/api/datasources/{id}/catalog/draft-jobs）：起草在服务端后台进行，关掉对话框不停；逐表起草， +// 每张表一结束就写进目录。 +// +// 术语： +// - 「起草任务」的状态：排队中、正在起草、起草完成、部分表未写全、起草失败、已停止、服务重启中断。 +// 一张表：等待、起草中、已完成、部分完成、失败、已跳过（表结构里没有）、已停止。 +// - 「起草要求」：使用者补充给模型的业务背景和用词约定，存在数据源上，以后每次用模型起草都带上； +// 「本次补充说明」只用于这一次（重试时沿用）。两者都只交给模型，不改起草规则：冲突时以规则为准。 +// - 「起草模型」：设置里单独的一项（偏好设置 → 起草模型），留空跟随助手的模型。 +// - 宽表的「段」：超过 40 列的表拆成表级一段和若干列分段，逐段起草；写的是「已完成 3 / 9 段」。模型多次用完额度或 +// 超时后，之后的段缩小（每段最多 20 列、10 列）。 +// - 「只重试失败的表」：重新起草失败、部分完成和已停止的表；已经写进目录的内容不回滚、不受影响。 +// - 「没写出来」：一次调用没有写出内容（额度用完、只给空壳、超时、出错）。接连没写出来时服务端给一句「提醒」， +// 好转后收起；提醒是提醒的样式，不是错误。超时的调用不计用量。 +// - 「留空」：模型列出了一列（或这一段请它写了），中文名和含义却都没给，写「N 列模型拿不准、留空了」。表的状态不因此 +// 变，不算没写全；办法是在起草要求里补充缩写和术语的含义，再重新起草，已有的内容不会丢。 +// =========================================================================== + +/** 起草要求、本次补充说明的字数上限（服务端同一个数：超长 422） */ +export const DRAFT_GUIDANCE_MAX = 2000 +export const DRAFT_NOTE_MAX = 1000 + +export const CATALOG_DRAFT_TEXT = { + // ---- 设置阶段：起草要求、本次补充说明、起草模型 + guidance: '起草要求', + guidanceHint: '写给模型的业务背景和用词约定，保存在这个数据源上，以后每次用模型起草都会带上。例如：「核销」指检票入园；金额单位是元', + guidanceRule: '起草要求只帮助模型理解业务、统一用词；与起草规则冲突时以规则为准，起草出的项仍需逐项确认', + guidancePlaceholder: '业务背景、常用说法、单位约定……', + note: '本次补充说明', + noteHint: '只用于这一次起草，例如「这一批是会员相关的表」；重试时沿用', + notePlaceholder: '可不填', + count: (n: number, max: number) => `${formatNumber(n)} / ${formatNumber(max)} 字`, + tooLong: (max: number) => `不超过 ${formatNumber(max)} 字`, + sourceDescription: (text: string) => `数据源说明也会一并交给模型：${text}`, + model: '起草模型', + /** 设置里没配起草模型：跟随助手的模型，写出当前是哪个（取不到就不写） */ + modelFollow: (current: string | null) => (current ? `跟随助手的模型（当前：${current}）` : '跟随助手的模型'), + /** 设置里配了：模型（没填模型就是那个接入的默认模型）和接入 */ + modelNamed: (model: string | null, provider: string | null) => (model + ? (provider ? `${model}（模型接入「${provider}」）` : model) + : `模型接入「${provider}」的默认模型`), + modelWhere: '可在「设置 → 偏好设置 → 起草模型」中更换', + tooMany: (max: number) => `一次最多起草 ${formatNumber(max)} 张表，请缩小范围`, + saveGuidanceFailed: (reason: string) => `起草要求未能保存，未开始起草：${reason}`, + startFailed: (reason: string) => `未能开始起草：${reason}`, + // ---- 上一次起草（设置阶段，没写全、有留空列的才提） + last: (state: string, at: string) => `上一次起草（${at}）：${state}`, + /** 上一次的结果：状态，有留空的列时接着写几列 */ + lastState: (title: string, blank: number) => (blank ? `${title}，${formatNumber(blank)} 列模型拿不准、留空了` : title), + lastOpen: '查看', + // ---- 从结果、表详情回到设置：范围就是那几张表 + scopeBlank: (n: number) => `有留空列的 ${formatNumber(n)} 张表`, + scopeTable: (name: string) => `这张表：${name}`, + /** + * 预先选好的范围里有表已经不在表结构里（打开表详情以后别处重新探查了结构）：写在那一项下面。一张都不剩时那一项 + * 照旧选着、不能开始，不退回别的范围 + */ + presetMissing: (gone: number, total: number) => (gone < total + ? `其中 ${formatNumber(gone)} 张在表结构中已没有，不会起草` + : total === 1 ? '表结构中已没有这张表，不能起草' : `表结构中已没有这 ${formatNumber(total)} 张表，不能起草`), + // ---- 进行中 + queued: '排队中，等待开始…', + running: '正在起草…', + // 服务端点了停止就取消进行中的模型调用（不等它答完，一次调用可能要几分钟），只等手上正在写入目录的那张表写完 + stopping: '正在停止：取消进行中的模型调用，正在写入目录的表写完后停止…', + progress: (done: number, total: number) => `已完成 ${formatNumber(done)} / ${formatNumber(total)} 张表`, + /** 结束以后：全部写出来的表有几张（停止、失败、部分完成的不算） */ + progressDone: (ok: number, total: number) => `写完 ${formatNumber(ok)} / ${formatNumber(total)} 张表`, + parts: (done: number, total: number) => `${formatNumber(done)} / ${formatNumber(total)} 段`, + partsHint: '超过 40 列的表拆成表级一段和若干列分段，逐段起草', + hide: '在后台继续', + hideHint: '关闭对话框不会停止起草，进度显示在目录页上方；每张表起草完就写进目录', + stop: '停止', + busy: '这个数据源已有进行中的起草任务，下面显示的是它的进度', + modelUsed: (model: string) => `模型：${model}`, + usage: (calls: number, input: number, output: number) => + `调用模型 ${formatNumber(calls)} 次，输入 ${formatTokens(input, { compact: true })}、输出 ${formatTokens(output, { compact: true })}`, + /** + * 用量那一行接着写的：没写出来、超时的调用各几次（没有的不写）。没写出来的次数里含超时的，写成「其中」，免得 + * 读成两个数相加 + */ + usageFailures: (timeouts: number, failed: number) => { + if (failed && timeouts && timeouts <= failed) return `没写出来 ${formatNumber(failed)} 次（其中超时 ${formatNumber(timeouts)} 次）` + return [failed ? `没写出来 ${formatNumber(failed)} 次` : '', timeouts ? `超时 ${formatNumber(timeouts)} 次` : ''].filter(Boolean).join('、') + }, + usageFailuresHint: '没写出来：调用没有写出内容，包括额度用完、只给空壳、超时和出错', + // 超时的调用服务端照实记 0 个 token，服务方却可能已经生成了内容 + timeoutNote: '超时的调用不计用量，实际用量可能更多', + /** 额度用完、超时多了，服务端把之后的段缩小了 */ + partColumns: (n: number) => `模型多次用完额度或超时，之后的段每段最多 ${formatNumber(n)} 列`, + // ---- 留空的列：模型列出了、中文名和含义却都没给。表的状态不因此变 + blank: (n: number) => `${formatNumber(n)} 列模型拿不准、留空了`, + /** 留空的列名（服务端最多给 5 个）；没列全时写一共几列 */ + blankSample: (sample: string[], n: number) => + `留空的列：${sample.join('、')}${n > sample.length ? ` 等 ${formatNumber(n)} 列` : ''}`, + blankTotal: (columns: number, tables: number) => `共 ${formatNumber(columns)} 列模型拿不准、留空了（${formatNumber(tables)} 张表）`, + blankAdvice: '可以在起草要求里补充缩写和术语的含义，再重新起草这几张表，已有的内容不会丢', + blankRedraft: (n: number) => `重新起草这 ${formatNumber(n)} 张表`, + blankRedraftHint: '回到起草设置，范围是这几张表：先在起草要求里补充缩写和术语的含义，再开始起草', + // ---- 结束 + title: { + queued: '排队中，等待开始…', + running: '正在起草…', + succeeded: '起草完成', + partial: '起草完成，部分表未写全', + failed: '起草失败', + cancelled: '已停止', + interrupted: '服务重启，起草中断', + } as Record, + interrupted: '服务重启时这次起草还没有完成，没写完的表未起草。已写进目录的内容保留；可以只重试这些表，接着起草', + cancelled: '已写进目录的内容保留，没写完的表未起草', + // 任务整体出错的原因（job.error）照服务端的原话写在标题「起草失败」下面，不再加前缀：服务端的原因有的已经是整句 + // （「数据源不存在……」），有的自带「起草任务出错：」开头。没结束的表记失败、原因同任务的，逐表不再重复 + /** 没写全的表：失败、部分完成、已停止各几张；跳过的（表结构里没有）另说 */ + problems: (c: { failed: number; partial: number; cancelled: number; skipped: number }) => { + const parts = [ + c.failed ? `失败 ${formatNumber(c.failed)} 张` : '', + c.partial ? `部分完成 ${formatNumber(c.partial)} 张` : '', + c.cancelled ? `已停止 ${formatNumber(c.cancelled)} 张` : '', + ].filter(Boolean) + const bad = parts.length ? `${formatNumber(c.failed + c.partial + c.cancelled)} 张表未写全：${parts.join('、')}` : '' + const skipped = c.skipped ? `${formatNumber(c.skipped)} 张表已跳过（表结构里没有）` : '' + return [bad, skipped].filter(Boolean).join('；') + }, + retry: (n: number) => `只重试失败的表（${formatNumber(n)} 张）`, + retryHint: '重新起草失败、部分完成和已停止的表；已写进目录的内容不受影响', + showAll: (n: number) => `显示全部 ${formatNumber(n)} 张表`, + showProblems: '只看未写全的表', + showProblemsBlank: '只看未写全、有留空列的表', + tasks: '逐表进度', + taskStatus: { + pending: '等待', running: '起草中', ok: '已完成', partial: '部分完成', failed: '失败', skipped: '已跳过', cancelled: '已停止', + } as Record, + taskAdded: (added: number, updated: number) => (added || updated + ? `+${formatNumber(added)}${updated ? ` · 改 ${formatNumber(updated)}` : ''}` : ''), + taskAddedHint: (added: number, updated: number, removed: number) => + `新增 ${formatNumber(added)} 项、更新 ${formatNumber(updated)} 项${removed ? `、删除 ${formatNumber(removed)} 项` : ''}`, + // ---- 目录页上方的进度、结束提示 + strip: (done: number, total: number) => `正在起草：已完成 ${formatNumber(done)} / ${formatNumber(total)} 张表`, + stripQueued: (total: number) => `起草任务排队中，共 ${formatNumber(total)} 张表`, + stripOpen: '查看进度', + toast: { + succeeded: (summary: string) => `数据目录起草完成：${summary}`, + partial: (n: number) => `数据目录起草完成，${formatNumber(n)} 张表未写全`, + /** 完成、部分完成的提示后面接着写留空的列（没有就不写） */ + withBlank: (text: string, blank: number) => (blank ? `${text}；${formatNumber(blank)} 列模型拿不准、留空了` : text), + failed: (reason: string) => (reason ? `数据目录起草失败：${reason}` : '数据目录起草失败'), + cancelled: '数据目录起草已停止', + interrupted: '服务重启，数据目录起草中断', + }, + viewResult: '查看结果', + // ---- 清单上的标记、表详情里的说明 + lastFailed: '上次起草失败', + lastPartial: '上次起草部分完成', + lastWith: (label: string, reason: string) => `${label}：${reason}`, + /** 上次起草有留空的列：清单上是轻提示（不是失败标记），表详情顶部同一句 */ + lastBlank: (n: number) => `上次起草有 ${formatNumber(n)} 列模型拿不准、留空了`, + lastBlankAdvice: '可以在起草要求里补充缩写和术语的含义，再重新起草这张表，已有的内容不会丢', + draftGuidance: '补充起草要求', + draftGuidanceHint: '打开起草设置，范围是这张表：先在起草要求里补充缩写和术语的含义，再开始起草', + redraft: '重新起草这张表', + redraftHint: '用模型重新起草这张表（不带补充说明，带上数据源的起草要求）；人工确认、驳回过的项不会被改动', + redraftBusy: '已有起草任务在进行，结束后再重新起草这张表', + redraftEditing: '正在编辑这张表，保存或取消后再重新起草', + /** 表结构里已经没有的表（只剩目录）：上次起草的结果照写，不给重新起草（服务端只会把它记成已跳过） */ + redraftMissing: '表结构中已没有这张表，不能重新起草', +} + +/** 设置页「偏好设置」里「起草模型」一节（服务端 api/settings.py 的 catalog_draft 一组） */ +export const DRAFT_MODEL_SETTING_TEXT = { + title: '起草模型', + hint: '数据目录的「用模型起草」和按配方导入表格时的 AI 起草使用这个模型，留空则跟随助手的模型。' + + '起草一次要写出几十到上百列的说明，建议选择输出额度大、能稳定按格式输出的模型；会思考的模型，思考也计入输出额度。', + /** 和服务端 _provider_fields 里的叫法一字不差：保存被拒时的报错写的是它 */ + provider: '起草模型 · 接入', + model: '起草模型 · 模型', + notSet: '未单独配置，跟随助手的模型', + providerDefault: '跟随助手的模型', + providerFromModel: '按模型名找接入', + providerOff: '（已停用）', + providerGoneOption: (name: string) => `${name}（不存在）`, + providerGone: (name: string) => `模型接入「${name}」不存在：${PROVIDER_REF_TEXT.draftBroken}`, + modelDefault: '留空：用接入的默认模型', + modelFollow: '留空：跟随助手的模型', + modelOf: (model: string) => `留空则使用 ${model}`, + /** 只填了起草模型、接入留空时实际用哪个接入（同核查一组的说法，只是上一级是助手的模型） */ + byModel: (model: string, found: { name: string; enabled: boolean } | null | undefined, fallback: string | null | undefined) => { + if (found && !found.enabled) return `只填了模型：「${model}」属于已停用的接入「${found.name}」，${PROVIDER_REF_TEXT.draftBroken}。请启用该接入或选择其他接入` + if (found) return `只填了模型：将按模型名使用接入「${found.name}」。「跟随助手的模型」仅在接入和模型都留空时生效` + return `只填了模型：所有接入的模型列表中都没有「${model}」，将使用默认接入${fallback ? `「${fallback}」` : ''}调用。` + + '如该模型不属于默认接入，请选择对应的接入。「跟随助手的模型」仅在接入和模型都留空时生效' + }, +} + // =========================================================================== // 目录修改提案：用户在助手（画布）或问数据页说出一条数据事实时，助手提出的数据目录修改。 // diff --git a/frontend/src/pages/DraftModelSettings.tsx b/frontend/src/pages/DraftModelSettings.tsx new file mode 100644 index 00000000..0dfddd47 --- /dev/null +++ b/frontend/src/pages/DraftModelSettings.tsx @@ -0,0 +1,84 @@ +import { modelOptions, providerOfModel, useCatalog } from '../store/catalog' +import { Field, SectionBar } from '../components/ui' +import { DRAFT_MODEL_SETTING_TEXT as T, PROVIDER_REF_TEXT } from '../lib/terms' +import { ProviderOptions } from './VerifySettings' + +/* + * 设置页「偏好设置」里「起草模型」一节：数据目录的「用模型起草」和按配方导入表格时的 AI 起草用哪个模型 + * （服务端 api/settings.py 的 catalog_draft 一组：{provider, model}）。 + * + * 取模型的顺序（服务端）：这一组填了接入或模型就用它（只填模型时按模型名找接入，不跨级拼）；两项都留空跟随助手的 + * 模型；再没有就是默认接入。草稿由 PrefsTab 持有、和其余设置共用一条保存条，这里只画；两栏的叫法和服务端 + * _provider_fields 一字不差，保存被拒（422 带 field：catalog_draft.provider）时报错写的就是它 + */ + +export function DraftModelSection({ provider, model, error, onEdit }: { + provider: string + model: string + /** 服务端按这一项拒收时的整句（设置页从 422 的 field 落过来） */ + error?: string + onEdit: (patch: { draftProvider?: string; draftModel?: string }) => void +}) { + const providers = useCatalog((s) => s.providers) + const enabledProviders = providers.filter((p) => p.enabled) + // 只填了模型、按模型名找不到接入时服务端用默认接入（第一个启用的真实接入，没有就第一个启用的） + const defaultProvider = enabledProviders.find((p) => p.kind !== 'mock') ?? enabledProviders[0] + const found = provider ? providers.find((p) => p.name === provider) : undefined + const missing = !!provider && providers.length > 0 && !found + // 存着的接入不存在、已停用：开始起草时就会报错（不建任务），在这一格下面说。名字恰好是某个启用接入的模型时 + // 补一句怎么用它（同设置页别的几组，和服务端保存时那句一致） + const trap = found?.enabled === false || missing ? providerOfModel(providers, provider) : undefined + const trapNote = trap?.enabled ? `。${PROVIDER_REF_TEXT.modelNote(provider, trap.provider.name, T.model, T.provider)}` : '' + const providerError = error ?? (missing ? `${T.providerGone(provider)}${trapNote}` + : found && !found.enabled ? `${PROVIDER_REF_TEXT.disabled(provider)}：${PROVIDER_REF_TEXT.draftBroken}${trapNote}` : undefined) + const modelName = model.trim() + const byModel = !provider && !!modelName + const notSet = !provider && !modelName + const owner = byModel ? providerOfModel(providers, modelName) : undefined + const byModelHint = byModel && providers.length > 0 + ? T.byModel(modelName, owner && { name: owner.provider.name, enabled: owner.enabled }, defaultProvider?.name) : '' + // 接入留空时按模型名找接入：所有启用接入的模型都能选；选了接入只列这一家的 + const models = found ? (found.models ?? []).map((m) => m.id) : [...new Set(modelOptions(providers).map((o) => o.value))] + const pickProvider = (name: string) => { + // 换了接入，原来填的模型不在新接入的清单里就清掉：拿 A 家的模型名去调 B 家必然失败 + const next = name ? providers.find((p) => p.name === name) : undefined + const keep = !name || !model || (next?.models ?? []).some((m) => m.id === model) || next?.default_model === model + onEdit({ draftProvider: name, ...(keep ? {} : { draftModel: '' }) }) + } + + return ( +
+ +
+
{T.title}
+ {notSet &&

{T.notSet}

} +
+ {byModelHint} + ) : undefined}> + {(p) => ( + + )} + + + {(p) => ( + <> + onEdit({ draftModel: e.target.value })} /> + {models.map((m) => + + )} + +
+
+
+ ) +} diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx index e58951b5..2f96a130 100644 --- a/frontend/src/pages/SettingsPage.tsx +++ b/frontend/src/pages/SettingsPage.tsx @@ -23,12 +23,14 @@ import { normalizeTheme } from '../lib/theme' import { useLeaveGuard } from '../lib/leave' import { localActor, setLocalActor } from '../lib/actor' import { - AGENT_GUARD_TEXT, JUDGE_SETTING_TEXT, PROVIDER_REF_TEXT, TOOL_GATE_TEXT, VERIFY_SETTING_TEXT, judgeByModelText, judgeUnlimitedText, + AGENT_GUARD_TEXT, DRAFT_MODEL_SETTING_TEXT, JUDGE_SETTING_TEXT, PROVIDER_REF_TEXT, TOOL_GATE_TEXT, VERIFY_SETTING_TEXT, judgeByModelText, + judgeUnlimitedText, } from '../lib/terms' import type { ThemePref } from '../lib/theme' import { ProviderOptions, VerifySection, verifyChanged, verifyDraftOf, verifyErrors, verifyLabels, verifyPayload, type VerifyDraft, } from './VerifySettings' +import { DraftModelSection } from './DraftModelSettings' // 提到模块级：tab 名同时是 URL 的最后一段，两处各写一份迟早对不上。 // 数据源搬去了顶级的「数据」页（/data），/settings/datasources 跳过去 @@ -626,6 +628,8 @@ function PrefsTab() { judgeProvider: string; judgeModel: string // 证据裁判的五项上限：null 是不限（存 null），字符串是正在填的数 jClaims: string | null; jCost: string | null; jTimeout: string | null; jClick: string | null; jDaily: string | null + // 起草模型（数据目录的用模型起草、按配方导入的 AI 起草）：空串 = 跟随助手的模型（存 null） + draftProvider: string; draftModel: string } | null>(null) const [spend, setSpend] = useState<{ usd?: number; calls?: number; unpriced_calls?: number } | null>(null) // 核查一组（双盲复算、反方审稿、审批质询）的草稿另放一份，和上面的共用保存条；verifySaves 是存过几次，今日核查花费跟着再取 @@ -673,6 +677,9 @@ function PrefsTab() { jTimeout: limitText(values?.judge, 'report_timeout_s'), jClick: limitText(values?.judge, 'click_max_cost_usd'), jDaily: limitText(values?.judge, 'daily_max_usd'), + // 起草模型（服务端 catalog_draft 一组；老服务端没有这一组时两项都是空，这一节也不显示） + draftProvider: typeof values?.catalog_draft?.provider === 'string' ? values.catalog_draft.provider : '', + draftModel: typeof values?.catalog_draft?.model === 'string' ? values.catalog_draft.model : '', } const cur = draft ?? saved const stepCap = Number(values?.limits?.max_agent_steps) || 100 @@ -748,7 +755,8 @@ function PrefsTab() { savedActor.current = actor } const judgeChanged = changed.some((k) => JUDGE_DRAFT.has(k)) - const runChanged = changed.some((k) => k !== 'actor' && !JUDGE_DRAFT.has(k)) + const draftChanged = changed.some((k) => DRAFT_MODEL_DRAFT.has(k)) + const runChanged = changed.some((k) => k !== 'actor' && !JUDGE_DRAFT.has(k) && !DRAFT_MODEL_DRAFT.has(k)) try { // 裁判一组整组写：后端对没给的键回落到默认值，只发改过的那一项会把别的项冲回默认 const judge = judgeChanged ? { @@ -762,7 +770,9 @@ function PrefsTab() { } : null // 核查一组同样整组写（VerifySettings 的 verifyPayload） const verify = verifyKeys.length ? verifyPayload(verifyCur) : null - if (runChanged || judge || verify) { + // 起草模型一组：两项一起写，空着的存 null（跟随助手的模型） + const catalogDraft = draftChanged ? { provider: cur.draftProvider || null, model: cur.draftModel.trim() || null } : null + if (runChanged || judge || verify || catalogDraft) { const run = { ...(values.run ?? {}), default_memory_scope: cur.scope, @@ -774,12 +784,16 @@ function PrefsTab() { agent_budget_tokens: cur.budgetTokens.trim() === '' ? null : Number(cur.budgetTokens), agent_budget_usd: cur.budgetUsd.trim() === '' ? null : Number(cur.budgetUsd), } - const out = await api.settings.put({ ...(runChanged ? { run } : {}), ...(judge ? { judge } : {}), ...(verify ? { verify } : {}) }) + const out = await api.settings.put({ + ...(runChanged ? { run } : {}), ...(judge ? { judge } : {}), ...(verify ? { verify } : {}), + ...(catalogDraft ? { catalog_draft: catalogDraft } : {}), + }) setValues((v) => ({ ...(v ?? {}), ...(runChanged ? { run: out?.run ?? run } : {}), ...(judge ? { judge: out?.judge ?? judge } : {}), ...(verify ? { verify: out?.verify ?? verify } : {}), + ...(catalogDraft ? { catalog_draft: out?.catalog_draft ?? catalogDraft } : {}), })) // 每日上限改了：旁边那句「今天已花」的分母跟着变，顺手再取一次 if (judge) void loadSpend() @@ -1090,6 +1104,11 @@ function PrefsTab() { {values.verify && } + + {values.catalog_draft && ( + + )} )} @@ -1113,6 +1132,7 @@ function PrefsTab() { judgeProvider: '裁判模型的接入', judgeModel: '裁判模型', jClaims: '每份报告最多裁判句数', jCost: '每份报告的金额上限', jTimeout: '每份报告的时长上限', jClick: '每次点击的金额上限', jDaily: '每日金额上限', + draftProvider: DRAFT_MODEL_SETTING_TEXT.provider, draftModel: DRAFT_MODEL_SETTING_TEXT.model, })[k]).concat(verifyLabels(verifyKeys)).join('、')} @@ -1147,10 +1167,13 @@ function PrefsTab() { /** 服务端按项拒收时（422 带 field）那一项在页面上是哪个控件：点名的模型接入的三格 */ const PROVIDER_FIELD_IDS: Record = { 'run.tool_gate_provider': 'pref-gate-provider', 'judge.provider': 'pref-judge-provider', 'verify.provider': 'pref-verify-provider', + 'catalog_draft.provider': 'pref-draft-provider', } /** 裁判那一组在草稿里的键：它们改了才写 judge 一组 */ const JUDGE_DRAFT = new Set(['judgeProvider', 'judgeModel', 'jClaims', 'jCost', 'jTimeout', 'jClick', 'jDaily']) +/** 起草模型那一组在草稿里的键：它们改了才写 catalog_draft 一组 */ +const DRAFT_MODEL_DRAFT = new Set(['draftProvider', 'draftModel']) /** 勾掉「不限」时填回的数：同后端 JUDGE_DEFAULTS */ const JUDGE_FALLBACK = { diff --git a/frontend/src/pages/catalog/CatalogPage.tsx b/frontend/src/pages/catalog/CatalogPage.tsx index 48442f3a..3257ec32 100644 --- a/frontend/src/pages/catalog/CatalogPage.tsx +++ b/frontend/src/pages/catalog/CatalogPage.tsx @@ -1,15 +1,19 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import type { ReactNode } from 'react' import { useNavigate, useParams, useSearchParams } from 'react-router-dom' -import { ArrowLeft, BookMarked, Database, ListFilter as ListFilterIcon, ScanSearch, Sparkles } from 'lucide-react' +import { ArrowLeft, BookMarked, Database, ListFilter as ListFilterIcon, ScanSearch, Sparkles, TriangleAlert } from 'lucide-react' import clsx from 'clsx' import { ApiError, api } from '../../api/client' import type { CatalogDetail, CatalogList, CatalogTableRow } from '../../types' import { EmptyState, ErrorState, PageHeader, Skeleton, Spinner, toast, useTicker } from '../../components/ui' import { useCatalog, useDatasources, useOnReconnect } from '../../store/catalog' -import { CATALOG_TEXT as CT, PROFILE_STOP_LABEL, PROFILE_TEXT as PT, SCHEMA_PARTIAL_TEXT as SPT } from '../../lib/terms' +import { + CATALOG_DRAFT_TEXT as DT, CATALOG_TEXT as CT, PROFILE_STOP_LABEL, PROFILE_TEXT as PT, SCHEMA_PARTIAL_TEXT as SPT, +} from '../../lib/terms' import { StatusLegend, SystemNotesNotice } from './parts' import { DraftDialog } from './DraftDialog' +import { isJobActive, jobBlank, retryableCount, useDraftJob } from './draftJob' +import type { DraftPreset, DraftStartBody } from './draftJob' import { ProfileDialog } from './ProfileDialog' import type { ProfileJob } from './ProfileDialog' import { ProfileSettingsDialog } from './ProfileSettings' @@ -99,7 +103,11 @@ export function CatalogPage() { const search = writeListParams(params, list).toString() searchRef.current = search ? `?${search}` : '' const [selected, setSelected] = useState>(() => new Set()) - const [drafting, setDrafting] = useState(false) + /** + * 起草对话框开着没有、打开时显示什么：setup 选范围开始；job 显示页面跟着的任务（进行中的进度、刚结束的结果）。 + * busy：是撞上已有任务（409）转过去显示的；preset：设置阶段预先选好的范围（表详情里「补充起草要求」） + */ + const [draftOpen, setDraftOpen] = useState<{ view: 'setup' | 'job'; busy?: boolean; preset?: DraftPreset } | null>(null) /** 起草写进了这些表：详情据此重新载入（编辑中的不动，保存时服务端会用版本号拦下） */ const [drafted, setDrafted] = useState<{ seq: number; tables: Set }>({ seq: 0, tables: new Set() }) const [profiling, setProfiling] = useState(false) @@ -142,11 +150,67 @@ export function CatalogPage() { })) }, []) - const onDrafted = useCallback((tables: string[]) => { - setDrafted((cur) => ({ seq: cur.seq + 1, tables: new Set(tables) })) - setSelected(new Set()) - void load() - }, [load]) + /** + * 起草任务（后台进行，状态在页面上）：每张表一结束就重取清单、正在看的表在里面就重新载入（编辑中的不动，保存时服务端 + * 会用版本号拦下）；结束时重取一次清单。对话框关着时结束的，弹提示并给「查看结果」 + */ + const draftOpenRef = useRef(draftOpen) + draftOpenRef.current = draftOpen + const draft = useDraftJob(sourceId, data?.active_job, { + onTablesEnded: (tables) => { + setDrafted((cur) => ({ seq: cur.seq + 1, tables: new Set(tables) })) + void load() + }, + onFinished: (job) => { + void load() + if (draftOpenRef.current) return + const src = job.source_id + const opts = { + key: `catalog-draft:${src}`, duration: 10000, + action: { label: DT.viewResult, onClick: () => { if (sourceRef.current === src) setDraftOpen({ view: 'job' }) } }, + } + // 留空的列不算没写全，提示里接着写一句，点「查看结果」看是哪几张表、怎么办 + const blank = jobBlank(job) + if (job.status === 'succeeded') toast.ok(DT.toast.withBlank(DT.toast.succeeded(CT.draftSummary(job.added, job.updated, job.removed)), blank), opts) + else if (job.status === 'partial') toast.warn(DT.toast.withBlank(DT.toast.partial(retryableCount(job)), blank), opts) + else if (job.status === 'failed') toast.error(DT.toast.failed(job.error ?? ''), opts) + else if (job.status === 'interrupted') toast.warn(DT.toast.interrupted, opts) + else toast.warn(DT.toast.cancelled, opts) + }, + }) + const draftJob = draft.job + const draftActive = isJobActive(draftJob) + /** 打开起草对话框：有进行中的任务就直接显示它；上一次的结果看过了，再点就是重新开始 */ + const openDraft = useCallback(() => setDraftOpen({ view: draftActive ? 'job' : 'setup' }), [draftActive]) + const startDraft = useCallback(async (body: DraftStartBody) => { + const r = await draft.start(body) + // 选中的表已经交给起草任务了：清空，免得接着点「起草所选」又起草一遍 + if (!r.busy) setSelected(new Set()) + return r + }, [draft.start]) + /** 详情里「重新起草这张表」：只含这张表、用模型、不带补充说明。模型用不了等开始不了的，直接提示原因 */ + const redraftTable = useCallback(async (t: string) => { + try { + const r = await draft.start({ tables: [t], use_model: true }) + setDraftOpen({ view: 'job', busy: r.busy }) + } catch (e) { + toast.error(e) + } + }, [draft.start]) + /** 详情里「补充起草要求」（上次起草有留空的列）：打开起草设置，范围只有这张表，先补起草要求再开始 */ + const draftWithGuidance = useCallback((t: string) => { + setDraftOpen({ view: 'setup', preset: { tables: [t], label: DT.scopeTable(t) } }) + }, []) + const onGuidanceSaved = useCallback((g: string | null) => { + setData((cur) => cur && ({ ...cur, guidance: g })) + }, []) + // 上方的进度：跟着的任务在进行就按它；跟着的就是清单说的那一个、已经结束了，就不信清单（清单还没重取回来） + const listActive = isJobActive(data?.active_job) ? data?.active_job ?? null : null + const stripJob: { done: number; total: number; status: string } | null = draftJob && (draftActive || draftJob.id === listActive?.id) + ? (draftActive ? draftJob : null) + : listActive + // 调用接连没写出来的提醒：关着对话框时也要看得到（上方的进度带着它） + const stripNotice = draftActive ? draftJob?.notice?.trim() || null : null const options = savedOptions?.sourceId === sourceId ? savedOptions.options : source?.options const profileSettings = useMemo(() => profileSettingsOf(options), [options]) @@ -262,7 +326,7 @@ export function CatalogPage() { const empty = rows.length > 0 && rows.every((r) => progressOf(r.counts) === 'none') const hasTables = rows.length > 0 const draftButton = (primary = false) => ( - ) @@ -281,7 +345,7 @@ export function CatalogPage() { ) // 页头的「起草」窄屏只留图标，标题才放得下 const headerDraft = ( - ) @@ -335,7 +399,7 @@ export function CatalogPage() { onSelected={setSelected} active={table} onOpen={openTable} - onDraft={() => setDrafting(true)} + onDraft={openDraft} onProfile={openProfile} /> @@ -343,7 +407,8 @@ export function CatalogPage() { {table ? ( + onFillDone={onFillDone} onOpen={openTable} onBack={backToList} onDetail={onDetail} + draftBusy={!!stripJob} onRedraft={redraftTable} onDraftGuidance={draftWithGuidance} /> ) : } @@ -373,16 +438,28 @@ export function CatalogPage() { onGoSource={() => navigate(back)} /> )} {hasTables && usage.used > 0 && !empty && } + {hasTables && stripJob && !draftOpen && } {body} - {drafting && data && ( + {draftOpen && data && ( setDrafting(false)} - onDrafted={onDrafted} + job={draftJob} + initial={draftOpen.view} + initialBusy={!!draftOpen.busy} + preset={draftOpen.preset ?? null} + stopping={draft.stopping} + onStart={startDraft} + onCancel={draft.cancel} + onRetry={draft.retry} + onTrack={draft.track} + onGuidanceSaved={onGuidanceSaved} + onClose={() => setDraftOpen(null)} /> )} {profiling && data && ( @@ -412,6 +489,41 @@ export function CatalogPage() { ) } +/** + * 起草任务在进行：目录页上方一条进度（关掉对话框任务照常进行，每张表起草完就写进目录）。点「查看进度」回到对话框。 + * notice：调用接连没写出来时服务端给的提醒，跟着写一行（全文在悬停里，对话框里也有） + */ +function DraftStrip({ job, notice, onOpen }: { + job: { done: number; total: number; status: string } + notice: string | null + onOpen: () => void +}) { + const text = job.status === 'queued' ? DT.stripQueued(job.total) : DT.strip(job.done, job.total) + return ( +
+ +
+

{text}

+ {notice && ( +

+ + {notice} +

+ )} +
+
+
+
+ +
+ ) +} + /** 媒体查询的当前结果，跟着窗口大小变 */ function useMedia(query: string): boolean { const [on, setOn] = useState(() => typeof matchMedia !== 'function' || matchMedia(query).matches) diff --git a/frontend/src/pages/catalog/DraftDialog.tsx b/frontend/src/pages/catalog/DraftDialog.tsx index c9fba7fc..3a7547fc 100644 --- a/frontend/src/pages/catalog/DraftDialog.tsx +++ b/frontend/src/pages/catalog/DraftDialog.tsx @@ -1,66 +1,58 @@ -import { useMemo, useRef, useState } from 'react' -import { Sparkles } from 'lucide-react' +import { useEffect, useMemo, useRef, useState } from 'react' +import { CircleCheck, CircleDashed, CircleMinus, CircleSlash, CircleX, Info, RotateCcw, Sparkles, TriangleAlert } from 'lucide-react' +import type { LucideIcon } from 'lucide-react' import clsx from 'clsx' -import { api } from '../../api/client' -import type { CatalogDraftRow, CatalogTableRow } from '../../types' -import { Modal, Notice, Spinner } from '../../components/ui' +import { ApiError, api } from '../../api/client' +import type { CatalogDraftJob, CatalogDraftTask, CatalogDraftTaskStatus, CatalogTableRow } from '../../types' +import { Field, Modal, Notice, Spinner } from '../../components/ui' import { errorMessage } from '../../lib/errors' -import { CATALOG_TEXT as CT } from '../../lib/terms' +import { formatTime } from '../../lib/format' +import { CATALOG_DRAFT_TEXT as DT, CATALOG_TEXT as CT, DRAFT_GUIDANCE_MAX, DRAFT_NOTE_MAX } from '../../lib/terms' +import { + DRAFT_MAX_TABLES, DRAFT_MODEL_UNAVAILABLE, DRAFT_PART_COLUMNS, DRAFT_TOP_N, RETRYABLE, blankOf, isJobActive, jobBlank, needsLook, + nothingWritten, retryableCount, taskReason, +} from './draftJob' +import type { DraftPreset, DraftStartBody } from './draftJob' // =========================================================================== -// 起草数据目录。接口是同步的，一次请求起草完才返回；表多时分批调用，每批回来就更新进度和累计的新增、更新、删除， -// 中途可以停止（当前这一批做完后不再发下一批）。模型整体用不了时第一批就会知道：之后的批次不再请模型，结果里 -// 写明原因。已经写进去的批次不回滚——起草只增改推断项，人工确认、驳回过的项服务端不会动。 +// 起草数据目录。起草是服务端的后台任务：这里选范围、写起草要求和本次补充说明，建任务；之后显示页面跟着的那个任务 +// （进行中的进度、结束后的结果），任务本身的状态在页面上（draftJob.ts），关掉对话框照常进行。 +// +// 三段：设置（范围、用不用模型；用模型时写起草要求、补充说明，写明用哪个起草模型）→ 进行中（总进度、合计、调用接连 +// 没写出来时的提醒、逐表的状态和宽表的段进度、没写全的原因、留空的列；可以停止、在后台继续）→ 结束（汇总、没写全的表 +// 和原因、留空的列合计、只重试失败的表）。 +// 已经写进目录的内容不回滚——起草只增改推断项，人工确认、驳回过的项服务端不会动。 +// +// 留空的列（模型列出了、中文名和含义都没给）不算没写全，表照样是已完成；结果里写合计和办法——在起草要求里补充缩写和 +// 术语的含义，再重新起草这几张表。「重新起草这几张表」回到设置：范围就是那几张、勾着用模型、焦点在起草要求， +// 补完再开始。表详情里的「补充起草要求」也是这样打开（范围是那一张，见 preset）。预先选好的范围里的表已经不在表结构 +// 里时不退回别的范围：那一项照旧选着、写明原因，不能开始。 // =========================================================================== -/** 每批几张表。只按注释、外键和命名起草时只看探查缓存，很快；请模型时服务端每 4 张一批、最多 3 批并发 */ -const BATCH_PLAIN = 40 -const BATCH_MODEL = 12 -/** 服务端不给 tables 时取前 20 张，这里给出同样的范围，进度才算得出来 */ -const TOP_N = 20 -/** 结果里逐张列出的表最多几张，其余合成一句 */ -const LIST_MAX = 8 +type Scope = 'preset' | 'selected' | 'filtered' | 'top' | 'all' -type Scope = 'selected' | 'filtered' | 'top' | 'all' -type Phase = 'setup' | 'running' | 'done' +/** 字数按服务端的算法：去掉首尾空白后的字符数（按码位，不按 UTF-16 的长度） */ +const charCount = (s: string) => [...s.trim()].length -interface Outcome { - done: number - total: number - added: number - updated: number - removed: number - tableErrors: CatalogDraftRow[] - modelErrors: CatalogDraftRow[] - modelSkipped: string | null - model: string | null - failure: string | null - stopped: boolean - /** 这一次勾了「用模型起草」 */ - modelAsked: boolean - /** 模型给了可用内容（model_items > 0）的表有几张；服务端没给 model_items（老服务端）时 modelItemsKnown 为假 */ - modelItems: number - modelItemsKnown: boolean -} - -const fresh = (total: number, modelAsked = false): Outcome => ({ - done: 0, total, added: 0, updated: 0, removed: 0, tableErrors: [], modelErrors: [], modelSkipped: null, model: null, - failure: null, stopped: false, modelAsked, modelItems: 0, modelItemsKnown: false, -}) +/** 起草要求输入框的 id：预先选好范围回到设置时焦点放在这里 */ +const GUIDANCE_ID = 'catalog-draft-guidance' /** - * 一项都没写进去时怎么说。「没有变化」只在确实是「起草出了内容、和现有目录一致」时说：有表没起草、模型在某些表上 - * 没给出内容（model_error），或者勾了用模型、却没有一张表拿到模型的可用内容（模型整体用不了、model_items 都是 0）， - * 都说「目录没有更新」，原因写在下面 + * 建任务（开始、只重试失败的表）没建成怎么说。起草模型用不了（409，建任务和重试都会回）：服务端那句话已经写了原因和 + * 去处，照原样，长句也不拆成「标题：原因」；别的错误前面写明没有开始 */ -function nothingWrittenText(out: Outcome): string { - const failed = out.modelErrors.length > 0 || out.tableErrors.length > 0 - const modelEmpty = out.modelAsked && (!!out.modelSkipped || (out.modelItemsKnown && out.modelItems === 0)) - return failed || modelEmpty ? CT.draftNothingWritten : CT.draftNoChange -} +const startErrorText = (e: unknown) => + (e instanceof ApiError && e.code === DRAFT_MODEL_UNAVAILABLE ? e.message : DT.startFailed(errorMessage(e))) -export function DraftDialog({ sourceId, rows, visible, selected, filtered, onClose, onDrafted }: { +export function DraftDialog({ + sourceId, sourceDescription, guidance, rows, visible, selected, filtered, job, initial, initialBusy = false, preset: initialPreset = null, + stopping, onStart, onCancel, onRetry, onTrack, onGuidanceSaved, onClose, +}: { sourceId: string + /** 数据源说明：用模型起草时连同起草要求一起交给模型 */ + sourceDescription: string + /** 数据源存着的起草要求（没有为空串） */ + guidance: string /** 全部表（服务端的顺序：使用次数多的在前） */ rows: CatalogTableRow[] /** 当前筛选结果 */ @@ -68,93 +60,233 @@ export function DraftDialog({ sourceId, rows, visible, selected, filtered, onClo selected: string[] /** 清单上有没有生效的搜索或筛选：有时才给「当前筛选结果」这一项 */ filtered: boolean + /** 页面跟着的起草任务（进行中的、刚结束的） */ + job: CatalogDraftJob | null + /** 打开时显示什么：setup 选范围、开始；job 显示页面跟着的那个任务 */ + initial: 'setup' | 'job' + /** 打开时显示的任务是撞上已有任务（409）转过来的：写明 */ + initialBusy?: boolean + /** + * 打开设置时预先选好的范围（表详情里「补充起草要求」：只有那张表）。给了就选它；里面有可起草的表时勾上用模型、 + * 焦点放在起草要求上（打开它就是为了先补起草要求再起草）。一张可起草的都没有（已不在表结构里）时照旧选着它、 + * 写明原因、不能开始，也不勾用模型——不退回别的范围 + */ + preset?: DraftPreset | null + /** 点了停止、服务端还没停下来 */ + stopping: boolean + /** 建任务；409 已有任务时页面改取进行中的那一个（busy 为真） */ + onStart: (body: DraftStartBody) => Promise<{ job: CatalogDraftJob; busy: boolean }> + onCancel: () => Promise + onRetry: (from: CatalogDraftJob) => Promise<{ job: CatalogDraftJob; busy: boolean }> + /** 设置阶段点开了上一次起草的结果（或者发现别处建的进行中的任务）：页面改跟这个任务 */ + onTrack: (job: CatalogDraftJob) => void + /** 起草要求存好了：页面记下新的这一份 */ + onGuidanceSaved: (guidance: string | null) => void onClose: () => void - /** 有批次写进去了：重取清单、刷新正在看的表 */ - onDrafted: (tables: string[]) => void }) { - // 表结构里已经没有的表起草不了（服务端会回「表结构里没有」），不算进范围 + // 表结构里已经没有的表起草不了（服务端会记「已跳过」），不算进范围 const live = useMemo(() => rows.filter((r) => r.in_schema).map((r) => r.table_name), [rows]) const liveSet = useMemo(() => new Set(live), [live]) + /** 预先选好的范围：打开时给的，或者结果里点了「重新起草这几张表」 */ + const [preset, setPreset] = useState(initialPreset) const scopes = useMemo(() => { - const out: { key: Scope; label: string; tables: string[] }[] = [] + const out: { key: Scope; label: string; tables: string[]; gone?: number }[] = [] + // 预先选好的范围总在第一项：里面的表已经不在表结构里的（打开表详情以后别处重新探查了结构）不起草、写明有几张； + // 一张都不剩也留着这一项，不退回别的范围——以为只起草那一张，实际按别的范围用模型起草了 + if (preset) { + const pre = preset.tables.filter((t) => liveSet.has(t)) + out.push({ key: 'preset', label: preset.label, tables: pre, gone: preset.tables.length - pre.length }) + } const sel = selected.filter((t) => liveSet.has(t)) if (sel.length) out.push({ key: 'selected', label: CT.scopeSelected(sel.length), tables: sel }) const vis = visible.filter((r) => r.in_schema).map((r) => r.table_name) if (filtered && vis.length && vis.length !== live.length) out.push({ key: 'filtered', label: CT.scopeFiltered(vis.length), tables: vis }) - if (live.length > TOP_N) out.push({ key: 'top', label: CT.scopeTop(TOP_N), tables: live.slice(0, TOP_N) }) + if (live.length > DRAFT_TOP_N) out.push({ key: 'top', label: CT.scopeTop(DRAFT_TOP_N), tables: live.slice(0, DRAFT_TOP_N) }) out.push({ key: 'all', label: CT.scopeAll(live.length), tables: live }) return out - }, [selected, visible, filtered, live, liveSet]) + }, [preset, selected, visible, filtered, live, liveSet]) const [scope, setScope] = useState(() => scopes[0].key) - const [useModel, setUseModel] = useState(false) - const [phase, setPhase] = useState('setup') - const [out, setOut] = useState(() => fresh(0)) - const [stopping, setStopping] = useState(false) - const stopRef = useRef(false) - const tables = scopes.find((s) => s.key === scope)?.tables ?? scopes[0].tables + /** 起草的就是这一项：选的那一项不在了（所选、筛选变了）就是第一项，单选框也照它勾，看得见起草的是哪些 */ + const current = scopes.find((s) => s.key === scope) ?? scopes[0] + const tables = current.tables + const tooMany = tables.length > DRAFT_MAX_TABLES + /** 打开时给的范围里有可起草的表：勾上用模型、焦点放进起草要求（打开它就是为了先补起草要求）；一张都没有就都不做 */ + const presetLive = !!initialPreset && initialPreset.tables.some((t) => liveSet.has(t)) + const [useModel, setUseModel] = useState(presetLive) + /** 回到设置以后把焦点放进起草要求（预先选好范围的两种入口都是为了先补起草要求） */ + const [focusGuidance, setFocusGuidance] = useState(presetLive) + const [guidanceText, setGuidanceText] = useState(guidance) + const [note, setNote] = useState('') + const guidanceLen = charCount(guidanceText) + const noteLen = charCount(note) + const guidanceOver = guidanceLen > DRAFT_GUIDANCE_MAX + const noteOver = noteLen > DRAFT_NOTE_MAX + const guidanceChanged = guidanceText.trim() !== guidance.trim() - const run = async () => { - stopRef.current = false - setStopping(false) - setPhase('running') - const acc = fresh(tables.length, useModel) - setOut({ ...acc }) - const touched: string[] = [] - let withModel = useModel - let i = 0 - while (i < tables.length) { - if (stopRef.current) { acc.stopped = true; break } - const batch = tables.slice(i, i + (withModel ? BATCH_MODEL : BATCH_PLAIN)) - try { - const r = await api.dataCatalog.draft(sourceId, { tables: batch, use_model: withModel }) - acc.added += r.total.added - acc.updated += r.total.updated - acc.removed += r.total.removed - acc.tableErrors.push(...r.tables.filter((t) => t.error)) - acc.modelErrors.push(...r.tables.filter((t) => !t.error && t.model_error)) - acc.modelItems += r.tables.filter((t) => !t.error && (t.model_items ?? 0) > 0).length - if (r.tables.some((t) => t.model_items !== undefined)) acc.modelItemsKnown = true - if (r.model) acc.model = r.model - // 模型整体用不了（没配置、已停用……）：后面的批次不再请模型，免得每批都撞一次同样的错 - if (withModel && r.model_error && !r.model_used) { acc.modelSkipped = r.model_error; withModel = false } - touched.push(...r.tables.filter((t) => !t.error).map((t) => t.table_name)) - } catch (e) { - acc.failure = errorMessage(e) - break + const [mode, setMode] = useState<'setup' | 'job'>(initial === 'job' && job ? 'job' : 'setup') + const [starting, setStarting] = useState(false) + const [startError, setStartError] = useState(null) + /** 是 409（已有任务）转过来显示的那一个 */ + const [busy, setBusy] = useState(initialBusy) + const [acting, setActing] = useState(false) + const [actionError, setActionError] = useState(null) + /** 上一次起草（没写全、有留空列的才提），设置阶段写一行、可以点开 */ + const [last, setLast] = useState(null) + /** 起草模型的说法：设置里的起草模型，没配就是跟随助手的模型 */ + const [modelLine, setModelLine] = useState<{ text: string; warn: string | null } | null>(null) + const onTrackRef = useRef(onTrack) + onTrackRef.current = onTrack + + const inSetup = mode === 'setup' || !job + // 设置阶段：取上一次起草。别处（另一个标签页）建的任务还在进行：直接显示它 + useEffect(() => { + if (!inSetup) return + let alive = true + api.dataCatalog.latestDraftJob(sourceId).then(({ job: latest }) => { + if (!alive || !latest) return + if (isJobActive(latest)) { + onTrackRef.current(latest) + setBusy(true) + setMode('job') + } else if (latest.status !== 'succeeded' || jobBlank(latest) > 0) { + setLast(latest) } - i += batch.length - acc.done = i - setOut({ ...acc }) + }, () => {}) + return () => { alive = false } + }, [inSetup, sourceId]) + + // 预先选好范围回到设置（打开时给的、结果里点的）：焦点放进起草要求 + useEffect(() => { + if (!focusGuidance || mode !== 'setup' || !useModel) return + const el = document.getElementById(GUIDANCE_ID) + if (!el) return + el.focus() + setFocusGuidance(false) + }, [focusGuidance, mode, useModel]) + + // 起草模型：设置里的 catalog_draft 填了就是它；没填跟随助手的模型（写出当前是哪个，用不了时写原因） + useEffect(() => { + let alive = true + void Promise.allSettled([api.settings.get(), api.copilot.getModel()]).then(([s, c]) => { + if (!alive) return + const cfg = s.status === 'fulfilled' ? s.value?.catalog_draft : null + const provider = typeof cfg?.provider === 'string' && cfg.provider.trim() ? cfg.provider.trim() : null + const model = typeof cfg?.model === 'string' && cfg.model.trim() ? cfg.model.trim() : null + if (provider || model) { + setModelLine({ text: DT.modelNamed(model, provider), warn: null }) + return + } + const assistant = c.status === 'fulfilled' ? c.value : null + setModelLine({ text: DT.modelFollow(assistant?.effective_model ?? null), warn: assistant?.error ?? null }) + }) + return () => { alive = false } + }, []) + + const begin = async () => { + if (starting || !tables.length || tooMany || (useModel && (guidanceOver || noteOver))) return + setStarting(true) + setStartError(null) + try { + // 起草要求改过：先存（存不上就不开始，免得模型拿着旧的那份起草） + if (useModel && guidanceChanged) { + try { + const saved = await api.dataCatalog.putGuidance(sourceId, guidanceText.trim()) + onGuidanceSaved(saved.guidance) + } catch (e) { + setStartError(DT.saveGuidanceFailed(errorMessage(e))) + return + } + } + const extra = useModel && note.trim() ? { note: note.trim() } : {} + const r = await onStart({ tables, use_model: useModel, ...extra }) + setBusy(r.busy) + setMode('job') + } catch (e) { + // 起草模型用不了（没有接入、已停用、只有演示接入）时留在这一步，写明原因 + setStartError(startErrorText(e)) + } finally { + setStarting(false) } - setOut({ ...acc }) - setPhase('done') - if (touched.length) onDrafted(touched) } - const stop = () => { - stopRef.current = true - setStopping(true) + const act = async (fn: () => Promise, describe: (e: unknown) => string = errorMessage) => { + setActing(true) + setActionError(null) + try { + await fn() + } catch (e) { + setActionError(describe(e)) + } finally { + setActing(false) + } } + const stop = () => act(onCancel) + // 重试是新建一个任务：没建成（起草模型用不了、没有表结构……）和开始时同一个说法，留在结果里 + const retry = (from: CatalogDraftJob) => act(async () => { + const r = await onRetry(from) + setBusy(r.busy) + }, startErrorText) + const openLast = (j: CatalogDraftJob) => { + onTrack(j) + setBusy(false) + setMode('job') + } + /** + * 结果里「重新起草这几张表」（有留空列的表）：回到设置，范围就是这几张，勾上用模型，焦点放进起草要求——先补上 + * 缩写和术语的含义再开始。任务本身不动（页面照旧跟着它，开始新的起草才换） + */ + const redraftBlank = (names: string[]) => { + setPreset({ tables: names, label: DT.scopeBlank(names.length) }) + setScope('preset') + setUseModel(true) + setFocusGuidance(true) + setStartError(null) + setActionError(null) + setBusy(false) + setLast(null) + setMode('setup') + } + /** 结果里有留空列、还在表结构里的表（「重新起草这几张表」的范围） */ + const blankTables = useMemo( + () => (job ? job.tasks.filter((t) => blankOf(t) > 0 && liveSet.has(t.table_name)).map((t) => t.table_name) : []), + [job, liveSet], + ) - // 进行中关窗 = 停止：当前这一批做完再关，结果照常显示 - const requestClose = () => (phase === 'running' ? stop() : onClose()) + const phase: 'setup' | 'running' | 'done' = inSetup ? 'setup' : isJobActive(job) ? 'running' : 'done' + const canStart = !!tables.length && !tooMany && !(useModel && (guidanceOver || noteOver)) && !starting + const retryable = job && phase === 'done' ? retryableCount(job) : 0 const footer = phase === 'setup' ? ( <> - ) : phase === 'running' - ? - : + ? ( + <> + + + + ) + : ( + <> + {retryable > 0 && job && ( + + )} + + + ) return ( - -
+ +
{phase === 'setup' && ( <>
@@ -162,77 +294,268 @@ export function DraftDialog({ sourceId, rows, visible, selected, filtered, onClo
{scopes.map((s) => ( ))}
+ {tooMany &&

{DT.tooMany(DRAFT_MAX_TABLES)}

}

{CT.draftBase}

- +
+ + {useModel && ( +
+
+ {DT.model}： + {modelLine ? modelLine.text : '…'} + {DT.modelWhere} + {modelLine?.warn && {modelLine.warn}} +
+
+ + {(p) => ( +