diff --git a/README.md b/README.md index 431e6e76..21f51124 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,11 @@ flowchart LR | 环节 | 做法 | |---|---| | 起草 | 探查时读外键和唯一约束；按 `xxx_id`、`xxxId` 推断关联，表名带业务前缀的也认；可选让模型起草中文名、粒度、表类型、业务主键、业务日期和每列的含义、单位、度量类型。导入表格的源已有系统按核对结果生成的说明，目录不重复、不改写它 | -| 模型起草 | 作为后台任务逐表进行，每张表记下状态和原因，写完一张就并入目录；关掉对话框不停，可以停止、只重试失败的表，清单上标出上次起草失败或只完成一部分的表。服务重启时没跑完的任务标成中断，不会自己续跑，点重试接上。保留模型的思考，输出额度 32768（服务方不收时退回 8192）；每次调用最多 40 列，宽表先起草表级的项、再把列均分成几段起草，按表的先后排队，不插到窄表前面；额度用完或超时就拆小再问（超时的两半同时问，拆到每段 10 列仍接连超时就不再拆），累计两次后新开始的段减半（最低 10 列），每段连同它分开、拆小的各块最多调用 8 次，缺的表和列只补问缺的，被截断的回复先留下已经写完整的部分；最近 6 次调用有 4 次没写出来时提醒换模型。没写出来的原因写清是第几段、哪几列。重新起草时模型这次留空或没通过校验的项不删以前起草的；模型拿不准、留空的列（目录里还没有中文名和含义的）逐表记下，可以补充起草要求后重起草。库注释就是表名、列名本身或者是标识符写法的，不记进目录（不当中文名，也不当说明、含义）。数据源上存的「起草要求」和这次的补充说明一起交给模型，帮它理解业务、统一用词，不能推翻「只记数据事实」的规矩，交回的内容照样逐项校验。用哪个模型在「设置 → 偏好设置」的「起草模型」里选，空着跟随助手的模型；按配方导入表格的 AI 起草也用它 | +| 模型起草 | 作为后台任务逐表进行，每张表记下状态和原因，写完一张就并入目录，宽表每写完一段就并入；关掉对话框不停，可以停止、只重试失败的表，清单上标出上次起草失败或只完成一部分的表。停止、服务重启时已经写进目录的段保留；服务重启时没跑完的任务标成中断，不会自己续跑，点重试重新起草这些表（已写进的不会丢）。不设总时长上限，靠止损和手动停止：最近 10 次调用有 7 次没写出来时后面的段不再拆小，同一张表接连 3 段都没写出来就跳过它剩下的段。保留模型的思考，输出额度 32768（服务方不收时退回 8192）；每次调用最多 40 列，宽表先起草表级的项、再把列均分成几段起草，按表的先后排队，不插到窄表前面；额度用完或超时就拆小再问（超时的两半同时问，拆到每段 10 列仍接连超时就不再拆），累计两次后新开始的段减半（最低 10 列），每段连同它分开、拆小的各块最多调用 8 次，进度上看得到段被分成了几块，缺的表和列只补问缺的，被截断的回复先留下已经写完整的部分；最近 6 次调用有 4 次没写出来时提醒换模型。没写出来的原因写清是第几段、哪几列，「已拆小重试」只跟在真拆过的段后面。同步起草和后台任务互斥。重新起草时模型这次留空或没通过校验的项不删以前起草的；模型拿不准、留空的列（目录里还没有中文名和含义的）逐表记下，可以补充起草要求后重起草。库注释就是表名、列名本身或者是标识符写法的，不记进目录（不当中文名，也不当说明、含义）。数据源上存的「起草要求」和这次的补充说明一起交给模型，帮它理解业务、统一用词，不能推翻「只记数据事实」的规矩，交回的内容照样逐项校验。用哪个模型在「设置 → 偏好设置」的「起草模型」里选，空着跟随助手的模型；按配方导入表格的 AI 起草也用它 | +| 防错事实的提议 | 模型起草时另提议「有效记录条件」（一段 SQL 条件）和「去重规则」，各带依据；开了剖析的源把目录里的码值一起交给模型（遮罩列不给）。条件按数据源方言用 sqlglot 解析，只能引用这张表的列，不能有子查询、聚合、窗口函数，只允许几个常用函数，不过就不收（逐表说明里写原因）；开了剖析的源上经只读守卫数出排除比例，排除 0% 或 100% 的不收，收下的在说明里写「剖析核对：排除 x% 的行」。这些都记为推断，SQL 检查的错误级规则只认人确认过的，提议在审阅队列里排在前面 | +| 审阅队列 | 只列推断的项，按「表被运行用到的次数 × 这一项多影响数字 × 来源」排序：有效记录条件、去重规则、业务日期、关联关系最前，业务主键、度量类型次之，中文名、含义、单位靠后，每项写明排在这里的理由；按类别和表筛选，多选批量确认、驳回或撤销审阅（逐表乐观锁，冲突逐条标出）；看得到一张表被哪些运行用过，以及常用表、全部和影响数字的项各确认了多少 | +| 表助手 | 表详情页里和助手对话完善这一张表：它看得到表结构、目录现状（每项的来源和状态）、关联关系、剖析结果、起草要求和上次起草留空的列（遮罩列只给列名），按你的指示成批提出修改，每项带改前、改后和依据（照目录的校验：公式不收、列名要在表结构里），拿不准的会反过来问你；只在开了剖析的源上经只读守卫跑查询核实（每次请求最多 5 条，结果先遮）。提案攒在「待保存的修改」里，确认后一次保存（一次最多 200 项），记为人工确认，每项的依据另存为这一项「确认时的依据」（和备注分开，表详情里看得到；撤销审阅回到推断时去掉）。对话不存服务端，只改目录、不碰业务库 | +| 导出与导入 | 导出成一个 JSON 文件（全部表的目录、各项状态和起草要求），在另一个实例或数据源上导入：先预览哪些表、列对得上，每一项会新增、更新还是保留，冲突和跳过的逐条列出；人确认、驳回过的项照带，目标里已经确认过而值不同的保留目标、记成冲突，文件里推断的项只填空缺，表结构里没有的列跳过并写原因；有起草在进行时不能导入。起草要求只在目标没有时写入 | | 数据剖析 | 默认关闭，按数据源开启。只读、经过守卫，单条查询和整次剖析都有上限，大表用统计信息估算行数，遮罩的列不取样。抽样核实推断出的关联，补上覆盖率和基数，站得住的升为已验证；状态列给出码值候选，含义等人填写 | | 对话里纠正 | 用户说「status=9 表示作废」，助手给出一条目录修改提案，列出改前、改后和理由，人确认后才保存；公式不收，留给口径卡 | | 版本与提醒 | 目录随表结构快照冻结进证据，每次运行用的是哪一版查得到。发布时记下模板引用的表的目录版本，正式运行时目录有变化就提醒；agent 运行时才写 SQL 的，按「可能涉及」单独提醒。只提醒，不拦运行 | diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 0405a0f7..2d7da5b9 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -8,11 +8,19 @@ - POST /catalog/{table}/patch 保存目录修改提案（乐观锁），记为人工确认 - GET /catalog/{table}/impact 影响面：引用这张表的已发布、受管模板（data/catalog_impact.py） - POST /catalog/draft 同步起草（注释、外键、命名推断，可选模型）。界面改用下面的后台起草任务，这个留着； - 同一个源有排队或进行中的后台任务时 409 catalog_draft_busy + 和后台任务双向互斥：同一个源有排队或进行中的后台任务（或正在同步起草）时 409 + catalog_draft_busy，同步起草期间建任务、重试、导入也是 409 - PUT /catalog/guidance 数据源级的起草要求（使用者补充的背景和要求，起草时交给模型） - POST /catalog/draft-jobs 建后台起草任务（202）；GET /draft-jobs/active、/latest、/{job_id} 读任务； POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py） - POST /catalog/profile 同步剖析：对业务库发少量只读查询，核对关系、取码值候选（data/catalog_profile.py） +- GET /catalog/review-queue 审阅队列：推断的项按使用次数、风险、来源排好；POST /catalog/review/batch 批量审阅， + GET /catalog/review-progress 审阅进度，GET /catalog/{table}/usage 用过这张表的运行 + （data/catalog_review.py） +- GET /catalog/export 导出数据目录（JSON 附件）；POST /catalog/import/preview 导入预览（只读）， + POST /catalog/import 导入（data/catalog_transfer.py） +- POST /catalog/{table}/assistant 表助手：对话完善这张表的目录（text/event-stream），提案不写目录 + （data/catalog_assistant.py） 规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径 不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。 @@ -24,7 +32,7 @@ from __future__ import annotations from typing import Any, Literal -from fastapi import APIRouter, Body, Depends, Header, HTTPException +from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query from fastapi.exceptions import RequestValidationError from pydantic import BaseModel, Field, ValidationError from sqlalchemy.ext.asyncio import AsyncSession @@ -35,6 +43,7 @@ from app.api.coded import ( ) from app.api.runs import actor_of from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, introspect, table_versions +from app.data import catalog_assistant, catalog_review, catalog_transfer from app.data.engine import SnapshotTampered from app.db.base import get_session from app.db.models import DataSource @@ -234,6 +243,92 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi } +# ---- 审阅队列、审阅进度（要登记在 GET /catalog/{table} 前面：不然 review-queue、review-progress 会被当成表名） + + +@router.get("/{source_id}/catalog/review-queue") +async def catalog_review_queue(source_id: str, + limit: int = Query(50, ge=1, le=catalog_review.QUEUE_LIMIT_MAX), + offset: int = Query(0, ge=0), + kind: Literal["all", "facts", "names"] = Query("all"), + table: str | None = Query(None, max_length=255), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """审阅队列：只列推断的项，按使用次数、风险、来源排好（data/catalog_review.py）。kind 筛影响数字的项（facts）或名称 + 和说明（names），table 只看一张表。""" + _, source = await _resolved(session, source_id) + return await catalog_review.review_queue(session, source, kind=kind, table=table or None, limit=limit, + offset=offset) + + +@router.get("/{source_id}/catalog/review-progress") +async def catalog_review_progress(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """审阅进度：常用 20 张表、全部、影响数字的项各自确认了多少。""" + _, source = await _resolved(session, source_id) + return await catalog_review.review_progress(session, source) + + +# ---- 导出导入（导出同样要登记在 GET /catalog/{table} 前面） + + +@router.get("/{source_id}/catalog/export") +async def export_catalog(source_id: str, session: AsyncSession = Depends(get_session)) -> Any: + """导出这个数据源的数据目录：JSON 附件「<数据源名>-数据目录.json」，含全部状态的项（data/catalog_transfer.py）。""" + from fastapi.responses import JSONResponse + + row = await _source_row(session, source_id) + body = await catalog_transfer.export_catalog(session, row) + name = catalog_transfer.attachment_name(row.name) + return JSONResponse(body, headers={"Content-Disposition": catalog_transfer.content_disposition(name, row.id)}) + + +class CatalogImportIn(BaseModel): + #: 导出的那个对象（格式在 data/catalog_transfer.read_file 里核对，不对 422，说明是中文） + content: Any = None + #: 目标还没有起草要求时，是否写入文件里的起草要求 + include_guidance: bool = False + + +async def _import_source(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]: + row, source = await _resolved(session, source_id) + cache = source.schema_cache or {} + if not cache.get("tables"): + raise CodedHTTPException(409, f"{introspect.why_empty(cache)}，无法导入数据目录。请先在数据源卡片上点「探查结构」", + SCHEMA_MISSING) + return row, source + + +@router.post("/{source_id}/catalog/import/preview") +async def preview_catalog_import(source_id: str, payload: CatalogImportIn, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """导入预览（只读）：每张对上的表新增、更新、保留几项，冲突和跳过的逐条列出。文件格式不对 422；没有表结构 409。""" + row, source = await _import_source(session, source_id) + try: + report = await catalog_transfer.plan_import(session, row, source, payload.content, + include_guidance=payload.include_guidance) + except catalog_transfer.TransferInvalid as e: + raise HTTPException(422, str(e)) from e + return report.as_dict() + + +@router.post("/{source_id}/catalog/import") +async def import_catalog(source_id: str, payload: CatalogImportIn, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """导入：规则同预览，逐表写入（乐观锁），返回同样形状的报告和每张表写入后的版本。有排队或进行中的起草任务 + 409 catalog_draft_busy；文件格式不对 422；没有表结构 409。""" + row, source = await _import_source(session, source_id) + try: + await catalog_jobs.ensure_idle(session, row) + except catalog_jobs.DraftBusy as e: + raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e + try: + report = await catalog_transfer.run_import(session, row, source, payload.content, + include_guidance=payload.include_guidance, + actor=actor_of(x_actor)) + except catalog_transfer.TransferInvalid as e: + raise HTTPException(422, str(e)) from e + return report.as_dict() + + @router.get("/{source_id}/catalog/{table}") async def get_catalog_table(source_id: str, table: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: @@ -419,20 +514,21 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn, model_items：模型给了这张表几项可用内容（没用模型、模型这部分失败时为 null）。界面据此区分「模型给了内容、 只是和现有目录一致」（model_items > 0、三个数都是 0）和「模型没给内容」（model_error 有值）。 - 同一个源有排队或进行中的后台起草任务时 409 catalog_draft_busy（和建任务同一句话）：两边同时写同一个源的目录、 - 抢模型名额。 + 双向互斥（catalog_jobs.sync_draft，进程内登记）：同一个源有排队或进行中的后台起草任务、或者正在同步起草时 409 + catalog_draft_busy（和建任务同一句话）；反过来，同步起草进行期间同一个源建任务、重试、导入目录也是 409。两边同时写 + 同一个源的目录、抢模型名额。 """ row, source = await _resolved(session, source_id) + cache = source.schema_cache or {} try: - await catalog_jobs.ensure_idle(session, row) + async with catalog_jobs.sync_draft(session, row): + if not cache.get("tables"): + raise _no_schema(cache) + report = await catalog.draft_catalog(session, source, tables=payload.tables or None, + use_model=payload.use_model, actor=actor_of(x_actor), + guidance=catalog_jobs.guidance_of(row)) except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e - cache = source.schema_cache or {} - if not cache.get("tables"): - raise _no_schema(cache) - report = await catalog.draft_catalog(session, source, tables=payload.tables or None, - use_model=payload.use_model, actor=actor_of(x_actor), - guidance=catalog_jobs.guidance_of(row)) rows = [{"table_name": r.table, "added": r.added, "updated": r.updated, "removed": r.removed, "version": r.version, "error": r.error, "model_error": r.model_error, "model_items": r.model_items} for r in report.tables] @@ -527,6 +623,38 @@ async def review_catalog_item(source_id: str, table: str, payload: CatalogReview return await _detail(session, source, key, entry) +class CatalogBatchReviewItem(BaseModel): + table_name: str = Field(min_length=1, max_length=255) + path: str = Field(min_length=1, max_length=600) + action: Literal["confirm", "reject", "reset"] + if_version: int = Field(ge=0) + + +class CatalogBatchReviewIn(BaseModel): + items: list[CatalogBatchReviewItem] = Field(min_length=1, max_length=catalog_review.BATCH_MAX) + + +@router.post("/{source_id}/catalog/review/batch") +async def review_catalog_batch(source_id: str, payload: CatalogBatchReviewIn, + x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """批量审阅：逐条结果和请求一一对应。逐表按 if_version 乐观锁，同一张表的几条一次写入；一张表冲突、路径不对、表不存在 + 只记在那几条上（ok 为假，code 是 catalog_conflict / catalog_path_invalid / table_missing），不影响别的。""" + _, source = await _resolved(session, source_id) + items = [catalog_review.BatchItem(i.table_name, i.path, i.action, i.if_version) for i in payload.items] + return {"results": await catalog_review.review_batch(session, source, items, actor=actor_of(x_actor))} + + +@router.get("/{source_id}/catalog/{table}/usage") +async def catalog_table_runs(source_id: str, table: str, + limit: int = Query(20, ge=1, le=catalog_review.RUNS_LIMIT_MAX), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """用过这张表的运行（total 是几次运行），新的在前；取法同使用次数（catalog.table_usage）。""" + _, source = await _resolved(session, source_id) + key, _ = await _table_key(session, source, table) + return await catalog_review.table_runs(session, source, key, limit=limit) + + def _tables_of(source: Any) -> dict[str, Any] | None: """核对提案用的表结构；没有表结构（尚未探查）时不核对列名。""" return ((source.schema_cache or {}).get("tables") or None) if source is not None else None @@ -552,7 +680,8 @@ async def preview_catalog_patch(source_id: str, table: str, payload: CatalogPatc async def save_catalog_patch(source_id: str, table: str, payload: CatalogPatchIn, x_actor: str | None = Header(default=None), session: AsyncSession = Depends(get_session)) -> dict[str, Any]: - """保存一条目录修改提案：改动过的项记为人工填写、已确认。版本不符 409；有一项不合法 422，整条不写。""" + """保存一条目录修改提案：改动过的项记为人工填写、已确认，写了依据的项依据另存成这一项确认时的依据（reason，和备注 + 分开，规则见 catalog.apply_patch）。版本不符 409；有一项不合法 422，整条不写。""" _, source = await _resolved(session, source_id) key, _ = await _table_key(session, source, table) try: @@ -566,6 +695,45 @@ async def save_catalog_patch(source_id: str, table: str, payload: CatalogPatchIn return await _detail(session, source, key, entry) +class AssistantMessageIn(BaseModel): + role: Literal["user", "assistant"] + content: str = Field(max_length=catalog_assistant.MESSAGE_MAX) + + +class AssistantIn(BaseModel): + #: 整段对话（服务端不存）；没有使用者的话时助手先开口 + messages: list[AssistantMessageIn] = Field(default_factory=list, max_length=catalog_assistant.MAX_MESSAGES) + #: 界面上还没保存的修改，让模型知道。不设上限：宽表上几轮攒下来常常超过一次保存的 200 项，不能因此不让问（背景里只列 + #: 最近的 catalog_assistant.PENDING_SHOWN 项，码值照全部合并） + pending: list[CatalogPatchChange] = Field(default_factory=list) + + +@router.post("/{source_id}/catalog/{table}/assistant") +async def catalog_table_assistant(source_id: str, table: str, payload: AssistantIn, + session: AsyncSession = Depends(get_session)) -> Any: + """表助手：和使用者对话完善这张表的目录，返回 text/event-stream（事件见 data/catalog_assistant.py）。提案只交给界面， + 不写目录；保存走 /catalog/{table}/patch。开始前：表不在 404；对话不合规 422；起草模型用不了 409 + catalog_draft_model_unavailable（说法同起草任务）。""" + from fastapi.responses import StreamingResponse + + row, source = await _resolved(session, source_id) + key = catalog.resolve_table_name(source.schema_cache, table) + if key is None: + raise HTTPException(404, f"数据源中没有表 {table}，可能已被删除或尚未探查结构") + messages = [m.model_dump() for m in payload.messages] + if problem := catalog_assistant.conversation_problem(messages): + raise HTTPException(422, problem) + try: + model, model_id = await catalog_assistant.resolve_model(session) + except catalog.CatalogModelUnavailable as e: + raise CodedHTTPException(409, str(e), CATALOG_DRAFT_MODEL_UNAVAILABLE) from e + context = await catalog_assistant.prepare(session, row, source, key, + pending=[c.model_dump() for c in payload.pending]) + return StreamingResponse(catalog_assistant.stream(model, model_id, context, messages), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + + @router.get("/{source_id}/catalog/{table}/impact") async def catalog_table_impact(source_id: str, table: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 3dd0bb39..9c674cb4 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -7,7 +7,7 @@ **只记数据事实，不放计算公式。** 公式（指标怎么算）在口径卡里；这里只描述数据本身。所以 notes 的 字段是固定的一组，不认识的字段一律拒收（validate_notes）。 -notes 结构（每个「项」都是 {value, source, status}，可选 note、updated_at）:: +notes 结构（每个「项」都是 {value, source, status}，可选 note、reason、updated_at）:: {"label": 项(str), "description": 项(str), "grain": 项(str), "keys": 项(list[str]), "kind": 项("fact"|"dimension"|"snapshot"|"log"|"config"), @@ -18,7 +18,10 @@ notes 结构（每个「项」都是 {value, source, status}，可选 note、upd "codes": 项(dict[码值, 含义]；含义可为空串，表示待填写；项上可带 complete: true， 表示已列出全部取值)}}, "relations": [{"id", "columns", "to_table", "to_columns", "cardinality", "coverage", - "source", "status", "note"?, "cardinality_checked"?}]} + "source", "status", "note"?, "reason"?, "cardinality_checked"?}]} + +note 是机器写的说明（模型起草的依据、数据剖析的测量、双盲复算的出处……）；reason 是保存修改提案时使用者确认的依据 +（界面上叫「确认时的依据」，最多 500 字），只由 apply_patch 写，两者分开存，规矩见 apply_patch。 关系上的 cardinality_checked: true 只由数据剖析写：基数是用数据核实的（子表一侧数过 COUNT 和 COUNT(DISTINCT)， 或者有主键、唯一约束）。外键约束、命名推断给的基数是按表结构推的，子表一侧可能每个键恰好一行；SQL 检查只在 @@ -87,16 +90,18 @@ COLUMN_FIELD_LABEL = {"label": "中文名", "meaning": "含义", "unit": "单位 #: 各来源起草出来的项的初始状态：外键约束是数据库强制的，算有确证；人工填写即确认；其余都是推断 _INITIAL_STATUS = {"fk": "verified", "human": "confirmed"} -_ITEM_KEYS = frozenset({"value", "source", "status", "note", "updated_at"}) +_ITEM_KEYS = frozenset({"value", "source", "status", "note", "reason", "updated_at"}) #: 码值项另有一个「已列全」标记（complete: true）：码值表列出了这一列的全部取值。只有人工明确确认、或者数据剖析 #: 没截断地看完了全表取值时才有；没有它的码值表只是「已知的一部分」，SQL 检查不能拿它判断某个值「不在码值表里」 _CODES_ITEM_KEYS = _ITEM_KEYS | {"complete"} _RELATION_KEYS = frozenset({"id", "columns", "to_table", "to_columns", "cardinality", "coverage", "source", - "status", "note", "updated_at", "cardinality_checked"}) + "status", "note", "reason", "updated_at", "cardinality_checked"}) _BUSINESS_DATE_KEYS = frozenset({"column", "rule", "timezone"}) #: 文本项的长度上限。目录是给模型和人读的短说明，长篇大论放口径卡或知识库 _TEXT_MAX = 500 _LONG_TEXT_MAX = 2000 +#: 确认时的依据（项上的 reason）的长度上限：同文本项 +REASON_MAX = _TEXT_MAX def now_iso() -> str: @@ -137,12 +142,15 @@ def unreviewed_status(item: Mapping[str, Any]) -> str: def make_item(value: Any, source: str, status: str | None = None, *, note: str | None = None, - at: str | None = None, complete: bool = False) -> dict[str, Any]: + at: str | None = None, complete: bool = False, reason: str | None = None) -> dict[str, Any]: """构造一个项。status 不给时按来源取初始状态（initial_status）。complete 只用于码值项（已列出全部取值）， - 为假时不写这个键：没有标记和标记为否是一回事，写法统一，冻结进快照的哈希才稳定。""" + 为假时不写这个键：没有标记和标记为否是一回事，写法统一，冻结进快照的哈希才稳定。reason 是确认时的依据 + （apply_patch 写），没有时不写这个键。""" out: dict[str, Any] = {"value": value, "source": source, "status": status or initial_status(source)} if note: out["note"] = note + if reason: + out["reason"] = reason if at: out["updated_at"] = at if complete: @@ -150,6 +158,15 @@ def make_item(value: Any, source: str, status: str | None = None, *, note: str | return out +def clip_reason(reason: Any) -> str | None: + """提案里的依据 → 存进项里的 reason：去掉首尾空白，超过 REASON_MAX 字截成 REASON_MAX - 1 字加「…」。没有（空的、 + 不是文字）为 None。预览（plan_patch）和保存（apply_patch）用的是同一句。""" + text = reason.strip() if isinstance(reason, str) else "" + if not text: + return None + return text if len(text) <= REASON_MAX else text[:REASON_MAX - 1] + "…" + + def codes_complete(item: Any) -> bool: """码值项是不是标了「已列全」。""" return isinstance(item, Mapping) and item.get("complete") is True @@ -256,11 +273,20 @@ def _item_problems(where: str, item: Any, rule: tuple[Any, str], *, problems.append(f"{where}{hint}") if "note" in item and not isinstance(item["note"], str): problems.append(f"{where}的备注应为文字") + if "reason" in item and not _reason_ok(item["reason"]): + problems.append(f"{where}{_REASON_HINT}") if "updated_at" in item and not isinstance(item["updated_at"], str): problems.append(f"{where}的修改时间应为文字") return problems +def _reason_ok(value: Any) -> bool: + return isinstance(value, str) and len(value) <= REASON_MAX + + +_REASON_HINT = f"确认时的依据应为不超过 {REASON_MAX} 字的文字" + + def _relation_problems(rel: Any, seen: set[str]) -> list[str]: if not isinstance(rel, dict): return ["关联关系应为对象"] @@ -297,6 +323,8 @@ def _relation_problems(rel: Any, seen: set[str]) -> list[str]: problems.append(f"{where}的状态「{rel.get('status')}」不在可选值内") if "note" in rel and not isinstance(rel["note"], str): problems.append(f"{where}的备注应为文字") + if "reason" in rel and not _reason_ok(rel["reason"]): + problems.append(f"{where}{_REASON_HINT}") if "cardinality_checked" in rel and not isinstance(rel["cardinality_checked"], bool): problems.append(f"{where}的「基数已用数据核实」应为是或否") if "updated_at" in rel and not isinstance(rel["updated_at"], str): @@ -609,7 +637,7 @@ def merge_notes(existing: Mapping[str, Any] | None, draft: Mapping[str, Any] | N """把起草结果并入已有目录，返回 (新目录, 计数)。不改入参。 规则（逐个槽位）： - - 已有项是 confirmed 或 rejected：不动。驳回的留着，同一项下次起草就不会再冒出来。 + - 已有项是 confirmed 或 rejected：不动（确认时的依据 reason 跟着留着）。驳回的留着，同一项下次起草就不会再冒出来。 - 起草结果里没有这一项： - 模型起草的项（llm）留着。模型这次留空、给的值没通过校验（_from_model 里就丢掉了），都不能把以前起草好的 删掉——实测整库重跑一次，模型这次没给的几百项全被删掉，有的表覆盖率从八成多掉到五成。只有这一轮完整起草过 @@ -698,7 +726,10 @@ def describe_slot(key: Slot) -> str: def review_item(notes: Mapping[str, Any] | None, path: str, action: str, *, at: str | None = None) -> dict[str, Any]: - """对一项做审阅，返回新目录（不改入参）。路径不对、项不存在、操作不认识都抛 CatalogPathError。""" + """对一项做审阅，返回新目录（不改入参）。路径不对、项不存在、操作不认识都抛 CatalogPathError。 + + 确认时的依据（reason）：确认、驳回不带依据，原有的留着；撤销审阅回到推断，去掉——推断项上挂着「使用者说：……」， + 看起来像使用者确认过它。""" if action not in REVIEW_ACTIONS: raise CatalogPathError("不支持这种审阅操作，只能确认、驳回或撤销审阅") key = parse_path(path) @@ -711,6 +742,8 @@ def review_item(notes: Mapping[str, Any] | None, path: str, action: str, *, at: del slots[key] return _assemble(slots) item["status"] = {"confirm": "confirmed", "reject": "rejected"}.get(action) or unreviewed_status(item) + if action == "reset": + item.pop("reason", None) item["updated_at"] = at or now_iso() return _assemble(slots) @@ -755,6 +788,8 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str, 或数据剖析。码值的「已列全」标记跟着值走：只勾上或取消它也算改过（_edit_value）。 - 值没动的项：保持原来的来源和状态；提交里把状态改成确认、驳回或来源的初始状态的，照改（等同单项审阅）； 改成别的状态的不认（不能把推断改成「有确证」）。 + - 确认时的依据（reason）只由保存修改提案写（apply_patch），提交里带的不作数：值改过、新填的项去掉（原来的依据说的 + 是旧值，界面原样带回来的也不认）；值没动的留着库里的；状态改回来源的初始状态（等同撤销审阅）的去掉。 - 已有、但提交里没有的表级项、列级项：删掉。 - 已有、但提交里没有的关联关系：外键约束、命名推断、数据剖析、模型起草来的，转为驳回、保留编号；只有人工 新建的才真正删掉。删掉的推断关系下次起草会再提出来，关系图（助手挑表、SQL 检查）也会按表结构现推回来， @@ -785,8 +820,9 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str, for key, item in new.items(): prev = old.get(key) if prev is None or _edit_value(key, item) != _edit_value(key, prev): - # 「基数已用数据核实」只由数据剖析写：人工改过的关系记成人工确认，提交里带的这个标记不作数 - fresh = {k: v for k, v in item.items() if k not in ("updated_at", "cardinality_checked")} + # 「基数已用数据核实」只由数据剖析写：人工改过的关系记成人工确认，提交里带的这个标记不作数；确认时的依据只由 + # 保存修改提案写，说的是旧值，不留 + fresh = {k: v for k, v in item.items() if k not in ("updated_at", "cardinality_checked", "reason")} fresh.update(source="human", status="confirmed", updated_at=at) out[key] = fresh continue @@ -795,6 +831,8 @@ def apply_human_edit(existing: Mapping[str, Any] | None, submitted: Mapping[str, if status != prev.get("status") and status in ("confirmed", "rejected", unreviewed_status(prev)): kept["status"] = status kept["updated_at"] = at + if status not in ("confirmed", "rejected"): + kept.pop("reason", None) # 等同撤销审阅：回到推断，确认时的依据不留 if item.get("note") != prev.get("note"): if item.get("note"): kept["note"] = item["note"] @@ -860,9 +898,9 @@ async def save_human_edit(session: AsyncSession, source_id: str, table: str, sub #: 新增关联关系的路径：relations.new，值写两端；编号按两端算（relation_id），同一条关系不论谁提都是同一个编号 NEW_RELATION_PATH = "relations.new" -#: 一条提案最多几项：用户说的是一两条事实，几十项的「提案」多半是模型在重写整张表 -PATCH_MAX_CHANGES = 20 -_REASON_MAX = 500 +#: 一条提案最多几项（预览、保存都按它）。对话里的纠正通常是一两条事实；表助手（data/catalog_assistant.py）会逐列补中文名、 +#: 含义，界面上跨几轮攒下的待保存修改一次保存，宽表上常常几十上百项，所以放到 200 +PATCH_MAX_CHANGES = 200 #: 看起来像计算公式的文字：「转化率=下单数/访问数」「= SUM(amount)」。公式进口径卡，目录只记数据事实。 #: 不认减号：「上线日期=2024-01-01」不是公式。有效记录条件本来就是 SQL 条件，不查它。 #: 对话提案（plan_patch）、人工整份提交（apply_human_edit，只查改动过的项）、模型起草（_from_model，丢掉这一项） @@ -877,9 +915,11 @@ class PatchChange: """提案里的一项，核对过的。 - path：规范化后的路径（新增关系写成 relations.<编号>）； - - before / before_status：当前目录里这一项的值和状态（被驳回的和没有的都算没有，为 None）； + - before / before_status：当前目录里这一项的值和状态，没有这一项为 None。被驳回的照样给出原值、状态写 rejected + （同导入预览「目标里这一项已经驳回」）：只是给人看的，算改后、算 state 都按没有这一项，保存时换成人工确认的新值； - after：保存后的值（码值是补充后的完整对照）；value：要交回保存的原样取值（码值只有补充的那几个， 409 之后在最新的目录上重新补）； + - reason：提案的依据，按保存时的样子截过（clip_reason），没有为空串； - state：change 值有变化；confirm 值相同、但还不是已确认（保存即确认）；same 已经是这个值且已确认。 - note：服务端对这一项的说明，给人看；没有时为 None，as_dict 里也不出现。目前只有一种：新增的关联关系没写 基数、又推算不出来（_relation_after）。 @@ -1013,6 +1053,8 @@ def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str, 没有这个码时加进去，不会把已有的含义抹掉（_merge_codes）。 - 关联关系不抹掉测量结果：覆盖率沿用目录里的，基数没写时沿用或推算（_relation_after）。两端、基数、覆盖率都和 目录里一样时只是确认，来源不变。 + - 被驳回的项算没有：码值不在驳回的码值上补，和驳回的值一样也是 change（保存时换成人工确认的新值）。改前照样写出 + 驳回的原值、before_status 为 rejected，不然界面上只看到「未填写」，看不出原来有一个被驳回的值。 - 看起来像计算公式的文字不收（_FORMULA）：公式进口径卡。 - 同一个路径出现两次只认第一次。 """ @@ -1033,8 +1075,7 @@ def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str, continue path = raw["path"].strip() value = copy.deepcopy(raw["value"]) - reason = raw.get("reason") - reason = reason.strip()[:_REASON_MAX] if isinstance(reason, str) else "" + reason = clip_reason(raw.get("reason")) or "" if path == NEW_RELATION_PATH: rel, problem = _patch_relation(table, value, None, tables, columns) if problem: @@ -1093,8 +1134,10 @@ def plan_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str, state = "same" if live.get("status") == "confirmed" else "confirm" else: state = "change" - out.append(PatchChange(path=norm, before=copy.deepcopy(before), - before_status=live.get("status") if live is not None else None, + # 改前给人看：被驳回的也写出原值（上面算改后、算 state 照旧按没有这一项） + shown = live if live is not None else prev + out.append(PatchChange(path=norm, before=copy.deepcopy(_value_of(key, shown)) if shown is not None else None, + before_status=shown.get("status") if shown is not None else None, after=after, value=value, reason=reason, state=state, note=note)) return PatchPlan(out, problems) @@ -1105,6 +1148,11 @@ def apply_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str, 值有变化的项记为人工填写、已确认（和人工编辑同一条规矩：改动出自人的确认，不是模型起草）；值没变的只把 状态改成已确认、来源不变（等同单项确认）；已经确认过的原样不动。 + + 提案里的依据（reason，clip_reason 截过）记进这一项的 reason（确认时的依据），和备注 note 分开：note 是起草的依据、 + 剖析的测量这类机器写的说明，照旧不拼依据——以前拼进 note，再剖析时测量的说法换不了新（catalog_profile._patch_confirmed + 只换剖析自己写的备注）、旧依据叠在新依据后面、撤销审阅后还挂在推断项上。依据每次保存整句换掉、不叠加：值改了的项 + （关联关系改了值也一样）是这一次的依据，没写就没有（原来的说的是旧值）；值不变、保存即确认的也换成这一次的。 """ plan = plan_patch(notes, changes, table=table, tables=tables) if plan.problems: @@ -1116,20 +1164,25 @@ def apply_patch(notes: Mapping[str, Any] | None, changes: Any, *, table: str, if change.state == "same": continue prev = slots.get(key) + reason = change.reason or None if change.state == "confirm" and prev is not None: - slots[key] = {**prev, "status": "confirmed", "updated_at": at} + item = {k: v for k, v in prev.items() if k != "reason"} + item.update(status="confirmed", updated_at=at, **({"reason": reason} if reason else {})) + slots[key] = item continue if key[0] == "r": rel = {"id": key[1], **change.after, "source": "human", "status": "confirmed", "updated_at": at} if prev is not None and prev.get("note"): rel["note"] = prev["note"] # 覆盖率沿用了，说明它怎么测的那段备注也留着 + if reason: + rel["reason"] = reason slots[key] = rel else: # 码值是补充、不删已有的码：原来标了「已列全」的，补充之后仍然列全。提案自己不能标「已列全」—— # 用户在对话里说一个码值的意思，不等于确认这一列只有这些取值 done = key[0] == "c" and key[2] == "codes" and prev is not None and prev.get("status") != "rejected" \ and codes_complete(prev) - slots[key] = make_item(change.after, "human", "confirmed", at=at, complete=done) + slots[key] = make_item(change.after, "human", "confirmed", at=at, complete=done, reason=reason) out = _assemble(slots) if problems := validate_notes(out): raise CatalogInvalid(problems) @@ -1488,9 +1541,13 @@ MODEL_FALLBACK_MAX_TOKENS = 8192 #: 一段没写出来（额度用完、只给空壳）时拆成两半再问：最多拆几层、拆到最少几列为止 MODEL_SPLIT_DEPTH = 3 MODEL_SPLIT_MIN_COLUMNS = 5 -#: 止损：连续这么多段拆到底仍一项没写出来，这次起草后面的段不再拆小重试（模型明显用不了时不该一段一段拆到底烧额度； -#: 每段另有 MODEL_PART_MAX_CALLS 次封顶） -MODEL_GIVE_UP_AFTER = 2 +#: 止损（按比例）：这次起草最近 MODEL_HALT_WINDOW 次调用里有 MODEL_HALT_ON 次以上没写出来（额度用完、超时、空壳、出错）， +#: 之后的段不再拆小、每块只问一次，直到比例降下去。模型明显用不了时不该一段一段拆到底烧额度；以前按「连续几段」算， +#: 偶尔一次写出来就清零，会思考的模型在宽表上 90 分钟停不下来。每段另有 MODEL_PART_MAX_CALLS 次封顶 +MODEL_HALT_WINDOW = 10 +MODEL_HALT_ON = 7 +#: 同一张表连续这么多个原始段都一项没写出来，这张表剩下的段跳过不问（可以换一个起草模型后只重试这张表） +MODEL_TABLE_GIVE_UP_AFTER = 3 #: 网络错误、429、5xx 最多重试几次，每次重试前等多久（秒，依次取用）。超时不原样重试，当作这一段太大、拆小再问 MODEL_RETRIES = 2 MODEL_RETRY_DELAYS_S = (10.0, 30.0) @@ -1512,6 +1569,10 @@ CATALOG_DRAFT_SYSTEM = """你在为一个业务数据库编写数据目录：说 log（日志表）、config（配置表）之一。 - keys：业务主键，即在业务上唯一确定一行的列（可以和数据库主键不同），写列名。 - business_date：业务日期按哪一列算，rule 写一句规则，timezone 写时区（不确定就留空）。 +- valid_filter：有效行过滤。condition 写一段 SQL 条件，筛出业务上算数的行（例如 STATUS NOT IN ('C', 'X')，去掉作废、\ +测试的行）：只用这张表自己的列、不带表名，只写比较、IN、BETWEEN、LIKE、IS NULL，用 AND、OR、NOT 连接，函数只用 UPPER、\ +LOWER、TRIM、COALESCE，不写子查询、聚合、窗口函数和分号；reason 写一句依据（例如哪条注释、哪些码值说明了这一点）。 +- dedup：去重规则，rule 写一句话（例如「同一订单号有多行时只取最后修改的那一行」），reason 写一句依据。 - columns：每一列的 label（中文名）、meaning（含义，一句话）、unit（单位，例如「元」「人」，没有单位就留空）、\ measure（度量类型：flow 流量，可跨期加总；stock 存量，时点值，不能跨期加总；ratio 比率；identifier 标识；\ status 状态；attribute 属性）。 @@ -1519,6 +1580,8 @@ status 状态；attribute 属性）。 规矩： - 只写数据事实，不写计算公式，不写指标怎么算。 - 拿不准的字段留空，不要猜。列名、表名照原样写，不要翻译或改写。 +- 有效行过滤、去重规则只在表结构、注释、码值能看出来时才写（例如状态列的注释写明了哪些取值是作废、测试），拿不准就不写，\ +没有就不写：写错了会把算数的行筛掉、把不该合并的行合并。 - 请你写的每一列都在 columns 里写一项，拿不准的字段可以留空，但不要漏掉列：漏掉的列会被再问一次。 - 标着「系统说明」的内容是导入时按核对结果生成的，以它为准：不要重复，也不要与之矛盾。 - 标着「已确认」的目录项由人工确认过，以它为准。 @@ -1537,6 +1600,7 @@ CATALOG_DRAFT_BACKGROUND_PROMPT = "使用者补充的背景和要求：\n{lines} CATALOG_DRAFT_TEXT_PROMPT = """只输出一个 JSON 对象，不要输出别的文字，格式如下（拿不准的字段不写）： {"tables": [{"table": "表名", "label": "中文名", "grain": "一行代表什么", "kind": "表类型", "keys": ["列名"], \ "business_date": {"column": "列名", "rule": "一句规则", "timezone": "时区"}, \ +"valid_filter": {"condition": "SQL 条件", "reason": "一句依据"}, "dedup": {"rule": "一句话", "reason": "一句依据"}, \ "columns": [{"name": "列名", "label": "中文名", "meaning": "含义", "unit": "单位", "measure": "度量类型"}]}]} 表类型写 fact、dimension、snapshot、log、config 之一；度量类型写 flow、stock、ratio、identifier、status、attribute 之一。""" @@ -1556,11 +1620,14 @@ CATALOG_DRAFT_PROMPT_LABELS = { "unique": "唯一约束：{cols}", "confirmed": "已确认的目录项：", # 分段起草：表名下一行说这次只写哪部分（宽表的表级一段、列分段，拆小重问时的半段） - "only_table": "这次只写表级的项（中文名、粒度、表类型、业务主键、业务日期），不写列", + "only_table": "这次只写表级的项（中文名、粒度、表类型、业务主键、业务日期、有效行过滤、去重规则），不写列", "only_columns": "这次只写下面列出的 {n} 列（这张表共 {total} 列），不写表级的项", "table_and_columns": "这次只写表级的项和这 {n} 列：{cols}", "brief_columns": "列（共 {n} 列，这里只列名字和类型）：", "drafted": "已起草的表级内容：", + # 开了数据剖析的源：目录里记下的码值（剖析看到的取值和已知含义）交给模型，判断有效行过滤用；遮罩的列不给 + "codes": "目录里记下的码值（数据剖析看到的取值和已知含义，可据此判断有效行过滤）：", + "code": "- {name}：{codes}", # 使用者补充的背景和要求 "description": "数据源说明：{text}", "guidance": "起草要求：{text}", @@ -1588,6 +1655,20 @@ CATALOG_DRAFT_PROMPT_SCHEMA: dict[str, Any] = { "timezone": {"type": "string"}}, "required": ["column"], }, + "valid_filter": { + "type": "object", + "description": "有效行过滤：只在表结构、注释、码值能看出来时才写", + "properties": {"condition": {"type": "string", "description": "SQL 条件，只用这张表的列"}, + "reason": {"type": "string", "description": "一句依据"}}, + "required": ["condition"], + }, + "dedup": { + "type": "object", + "description": "去重规则：只在表结构、注释能看出来时才写", + "properties": {"rule": {"type": "string", "description": "一句话"}, + "reason": {"type": "string", "description": "一句依据"}}, + "required": ["rule"], + }, "columns": { "type": "array", "items": { @@ -1708,11 +1789,77 @@ def _clean_text(value: Any, limit: int = _TEXT_MAX) -> str | None: return text if text and len(text) <= limit else None -def _from_model(entry: Mapping[str, Any], meta: Mapping[str, Any], *, system: bool) -> dict[str, Any]: +#: 模型给的依据（有效行过滤、去重规则的 reason）写进这一项的 note +FACT_REASON_NOTE = "依据：{reason}" +#: 有效行过滤按数据源的写法整理（列名加了引号）以后超过目录里这一项的长度上限 +FACT_FILTER_TOO_LONG = "{label}按数据源的写法整理后超过 {limit} 字" + + +def _model_fact(raw: Any, key: str) -> tuple[str, str | None] | None: + """模型给的有效行过滤、去重规则 → (取值, 依据)。约定是对象（{condition 或 rule, reason}），只写了一段文字的也认； + 取值不是非空文字的不要。""" + if isinstance(raw, str): + text, reason = raw, None + elif isinstance(raw, Mapping): + text, reason = raw.get(key), raw.get("reason") + if text is None: + text = raw.get("value") + else: + return None + if not isinstance(text, str) or not text.strip(): + return None + return text.strip(), _clean_text(reason) + + +def _fact_item(value: str, reason: str | None) -> dict[str, Any]: + return make_item(value, "llm", note=FACT_REASON_NOTE.format(reason=reason) if reason else None) + + +def _model_facts(entry: Mapping[str, Any], meta: Mapping[str, Any], *, kind: str | None, table: str | None, + dropped: list[tuple[str, str]] | None = None) -> dict[str, Any]: + """模型提议的有效行过滤、去重规则（llm / proposed，依据写进 note）。 + + 有效行过滤按数据源的方言解析成一个条件、只能用这张表的列（sqlcheck.parse_valid_filter），不过就丢掉、不算这张表 + 失败：服务端日志记一条，dropped 给了就记下 (原因, 模型写的条件)，起草引擎把它写进这张表的日志（起草任务逐表的 + log）。存的是 sqlcheck.stored_valid_filter 整理过的写法：原文照旧，只把列换成表结构里的真名、不带表名，按数据源的 + 需要加引号——剖析核对的、SQL 检查让模型照着加的都是这一条。去重规则是一句话，同文字项的规矩。""" + from app.data import sqlcheck + + out: dict[str, Any] = {} + found = _model_fact(entry.get("valid_filter"), "condition") + if found is not None: + text, reason = found + parsed = sqlcheck.parse_valid_filter(text, meta, kind=kind, table=table) + problem = parsed.problem + if problem is None: + stored = sqlcheck.stored_valid_filter(parsed, kind) + if _text_ok(stored): + out["valid_filter"] = _fact_item(stored, reason) + else: + # 加了引号以后超过上限：写进去会让整张表写入失败（validate_notes），不收 + problem = FACT_FILTER_TOO_LONG.format(label=TABLE_FIELD_LABEL["valid_filter"], limit=_TEXT_MAX) + if problem is not None: + import logging + + logging.getLogger(__name__).info("起草数据目录：表 %s 的有效行过滤没有通过校验，已丢掉：%s（%s）", + table or meta.get("qualified") or entry.get("table"), problem, text[:200]) + if dropped is not None: + dropped.append((problem, text)) + found = _model_fact(entry.get("dedup"), "rule") + if found is not None and (text := _clean_text(found[0])) is not None: + out["dedup"] = _fact_item(text, found[1]) + return out + + +def _from_model(entry: Mapping[str, Any], meta: Mapping[str, Any], *, system: bool, source_kind: str | None = None, + table: str | None = None, dropped: list[tuple[str, str]] | None = None) -> dict[str, Any]: """模型给的一张表 → 目录项（llm / proposed）。不合规的值整项丢掉，不硬塞：列名对不上表结构的、 表类型和度量类型不在取值里的、业务主键里有不存在的列的、看起来是计算公式的（_formula_like，公式进口径卡）。 该是文字的地方写成了数组、对象（列名、表类型、业务日期的列、度量类型……）同样丢掉，不抛异常：实测见过把列名 - 写成数组的，以前在这里抛 TypeError，整张表记出错。导入表格的源，说明已经覆盖的字段也丢掉。""" + 写成数组的，以前在这里抛 TypeError，整张表记出错。导入表格的源，说明已经覆盖的字段也丢掉。 + + 有效行过滤、去重规则见 _model_facts：source_kind 是数据源类型（按它的方言解析条件，不认识的类型不收有效行过滤）， + table 是表名（条件里的列可以带这张表的表名），dropped 收没通过校验的有效行过滤 (原因, 模型写的条件)。""" columns = {c["name"]: c for c in meta.get("columns") or []} notes: dict[str, Any] = {} table_has_system_note = system and bool((meta.get("comment") or "").strip()) @@ -1734,6 +1881,7 @@ def _from_model(entry: Mapping[str, Any], meta: Mapping[str, Any], *, system: bo if (text := _clean_text(date.get(k))) is not None: value[k] = text notes["business_date"] = make_item(value, "llm") + notes.update(_model_facts(entry, meta, kind=source_kind, table=table, dropped=dropped)) out_cols: dict[str, dict[str, Any]] = {} listed = entry.get("columns") for col in listed if isinstance(listed, list) else []: @@ -1842,8 +1990,10 @@ async def write_draft(session: AsyncSession, source: Any, table: str, modeled: A 没用模型时为 None）有可用内容的再并上。同步起草和后台起草任务（每张表一结束就调）共用这一处。 - 并入已有目录按 merge_notes 的规则：人工确认、驳回过的项不动。内容没变不写、不升版本。 - - 模型这次没给的项（留空、没通过校验）不删以前起草的；列已经不在表结构里的，模型各段都写出来时（covered 含 llm） - 照旧删掉。模型只写出一部分的不算完整起草过（covered 不含 llm）。 + - 模型这次没给的项（留空、没通过校验）不删以前起草的；列已经不在表结构里的（业务主键、业务日期指着的列也算）， + 整张表起草完（各段都写出来的 ok、只写出一部分的 partial 都一样）就按完整覆盖的规则删掉：删的只是说已经不存在的 + 列的模型起草项，和这次写没写全无关。后台任务里宽表每写出一段就先并入一次（catalog_jobs.write_part，那一次不删 + 任何东西），这里是整张表的收尾。 - 模型留空的列（blank）按并入之后的目录再筛一遍：已经有中文名或含义的（以前起草的、库注释、起草进行中有人填的） 不算留空。 - 写入撞上别人刚改过（乐观锁）就重读重并，最多三次。 @@ -1859,8 +2009,8 @@ async def write_draft(session: AsyncSession, source: Any, table: str, modeled: A if modeled.items: result.model_items = modeled.items result.blank = list(getattr(modeled, "blank", None) or []) - model_draft = TableDraft(notes=modeled.notes, covered=frozenset({"llm"}) if modeled.status == "ok" - else frozenset()) + model_draft = TableDraft(notes=modeled.notes, covered=frozenset({"llm"}) + if modeled.status in ("ok", "partial") else frozenset()) for _attempt in range(3): entry = await read_entry(session, source.id, table) version = entry.version if entry else 0 @@ -2249,27 +2399,27 @@ def query_snapshot_meta(source: Any, sql: str) -> dict[str, Any]: return {"source_id": getattr(source, "id", None), "source": getattr(source, "name", None), "tables": tables} -async def table_usage(session: AsyncSession, source: Any) -> dict[str, int]: - """每张表被运行查询过的次数：{schema_cache 的表名: 次数}，没被查过的表不出现。 +async def query_snapshot_tables(session: AsyncSession, source: Any) -> list[tuple[str, frozenset[str]]]: + """这个源上属于运行的每一行查询快照引用：(运行 id, 这条查询用到的表——schema_cache 的表名)，没对上任何表的不收。 - 取法：数据源查询工具每次成功查询都存一份查询快照工件（tools/datasource.py），工件引用表 artifacts 里 - 按 (内容哈希, 运行) 各记一行。数「属于某次运行（runs 里有这次运行）、属于这个源、SQL 里用到了这张表」的 - 行：同一次运行里结果一模一样的重复查询只算一次，工具库里的试用（不属于任何运行）不算。 + 使用次数（table_usage）和「这张表被哪些运行用过」（data/catalog_review.py）共用这一份取法：数据源查询工具每次成功 + 查询都存一份查询快照工件（tools/datasource.py），工件引用表 artifacts 里按 (内容哈希, 运行) 各记一行；只收属于某次运行 + （runs 里有这次运行）、属于这个源的行，工具库里的试用（不属于任何运行）不算。 - 表名在查询当时就记进了引用的 meta（query_snapshot_meta），这里只读 artifacts 一张表；升级前的老引用 - 没有 meta，退回读工件内容里的源名和 SQL，读过的按工件 id 缓存（内容永不改变）。SQL 里写的表名去掉 - schema 前缀后按 name_key（只把 ASCII 字母转小写，和 SQLite 比较标识符的口径一致）对到表结构上，对不上的不计。 + 表名在查询当时就记进了引用的 meta（query_snapshot_meta），这里只读 artifacts 一张表；升级前的老引用没有 meta，退回读 + 工件内容里的源名和 SQL，读过的按工件 id 缓存（内容永不改变）。SQL 里写的表名去掉 schema 前缀后按 name_key（只把 + ASCII 字母转小写，和 SQLite 比较标识符的口径一致）对到表结构上，对不上的不计。 """ rows = (await session.execute( - select(Artifact.id, Artifact.meta).join(Run, Run.id == Artifact.run_id) + select(Artifact.id, Artifact.meta, Artifact.run_id).join(Run, Run.id == Artifact.run_id) .where(Artifact.kind == QUERY_SNAPSHOT_KIND) )).all() tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} lookup: dict[str, str] = {} for name in tables: lookup.setdefault(name_key(name), name) - counts: Counter[str] = Counter() - for artifact_id, meta in rows: + out: list[tuple[str, frozenset[str]]] = [] + for artifact_id, meta, run_id in rows: meta = meta if isinstance(meta, dict) else {} if isinstance(meta.get("tables"), list): source_id, source_name, names = meta.get("source_id"), meta.get("source"), meta["tables"] @@ -2281,8 +2431,21 @@ async def table_usage(session: AsyncSession, source: Any) -> dict[str, int]: continue elif source_name != source.name: continue - hit = {lookup.get(name_key(str(n).rsplit(".", 1)[-1])) for n in names} - counts.update(t for t in hit if t is not None) + hit = frozenset(t for t in (lookup.get(name_key(str(n).rsplit(".", 1)[-1])) for n in names) if t is not None) + if hit: + out.append((str(run_id), hit)) + return out + + +async def table_usage(session: AsyncSession, source: Any) -> dict[str, int]: + """每张表被运行查询过的次数：{schema_cache 的表名: 次数}，没被查过的表不出现。 + + 按查询快照引用的行数（query_snapshot_tables）：同一次运行里结果一模一样的重复查询只算一次，同一次运行里不同的查询 + 各算一次，工具库里的试用不算。 + """ + counts: Counter[str] = Counter() + for _run, hit in await query_snapshot_tables(session, source): + counts.update(hit) return dict(counts) @@ -2324,10 +2487,11 @@ __all__ = [ "CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES", "DraftModel", "DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL", "MODEL_BATCH_COLUMNS", "MODEL_BATCH_TABLES", "MODEL_CONCURRENCY", "MODEL_EMPTY_REASON", - "MODEL_FALLBACK_MAX_TOKENS", "MODEL_MAX_TOKENS", "MODEL_MIN_PART_COLUMNS", "MODEL_NOTICE_OFF", "MODEL_NOTICE_ON", - "MODEL_NOTICE_WINDOW", "MODEL_PART_MAX_CALLS", "MODEL_RETRIES", "MODEL_RETRY_DELAYS_S", "MODEL_SHRINK_AFTER", - "MODEL_SPLIT_DEPTH", "MODEL_SPLIT_MIN_COLUMNS", "MODEL_TIMEOUT_S", "MergeStats", "NEW_RELATION_PATH", - "PATCH_MAX_CHANGES", + "FACT_REASON_NOTE", "MODEL_FALLBACK_MAX_TOKENS", "MODEL_HALT_ON", "MODEL_HALT_WINDOW", "MODEL_MAX_TOKENS", + "MODEL_MIN_PART_COLUMNS", "MODEL_NOTICE_OFF", "MODEL_NOTICE_ON", "MODEL_NOTICE_WINDOW", "MODEL_PART_MAX_CALLS", + "MODEL_RETRIES", "MODEL_RETRY_DELAYS_S", "MODEL_SHRINK_AFTER", "MODEL_SPLIT_DEPTH", "MODEL_SPLIT_MIN_COLUMNS", + "MODEL_TABLE_GIVE_UP_AFTER", "MODEL_TIMEOUT_S", "MergeStats", "NEW_RELATION_PATH", + "PATCH_MAX_CHANGES", "REASON_MAX", "clip_reason", "PROFILE_NOTE_PREFIX", "PROFILE_VERIFY_COVERAGE", "WRITE_CONFLICT_ERROR", "default_tables", "missing_table_error", "pick_tables", "write_draft", "PatchChange", "PatchPlan", "QUERY_SNAPSHOT_KIND", "REVIEW_ACTIONS", "TABLE_FIELDS", "TABLE_FIELD_LABEL", diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py new file mode 100644 index 00000000..5d537079 --- /dev/null +++ b/backend/app/data/catalog_assistant.py @@ -0,0 +1,927 @@ +"""表助手：在表详情页和使用者对话，一起完善一张表的数据目录（POST /catalog/{table}/assistant，SSE）。 + +**为什么要有它。** 模型起草只看表结构，缩写列名、状态码的意思常常拿不准而留空；审阅时人知道答案，却要一项一项填。表助手 +把这张表的结构、目录现状（每项的来源和状态）、关联关系、剖析结果、起草要求、使用次数、上次起草留空的列交给模型，让它主动 +问拿不准的地方；使用者说出一条业务事实，它整理成成批的修改提案（形状同助手流里的 catalog_patch），使用者在界面上逐项看过、 +一次保存（PATCH /catalog/{table}/patch，记为人工确认，每项的 reason 另存成这一项的确认时的依据，见 catalog.apply_patch）。 + +**依据只引用这次对话。** 数据源的起草要求是以前写在数据源上的背景，背景里标明「不是这次对话里使用者说的话」；目录项上以前 +保存时记下的依据也一样，背景里写成「以前确认时的依据：……」，和备注分开。提示词要求 reason 里的原话只引用这次对话里使用者的 +消息，依据来自起草要求的写「起草要求写着：……」，来自以前的依据的写「以前确认时的依据写着：……」。验收时模型把起草要求写成了 +「使用者说：……」，保存之后看起来像是使用者亲口确认过。 + +**只改目录、不改业务库。** 模型只有两个工具： +- propose_changes：一批修改提案。按 catalog.plan_patch 核对（取值格式、列名必须在表结构里、公式不收），遮罩列不给码值，已经 + 是这个值又确认过的不提；有效记录条件是模型写的 SQL，另按起草的规矩核对（_Tools.filter_problem：一个条件、只用这张表的 + 列、不能有子查询……）。核对过的作为 proposal 事件交给界面，**不写目录**。没收下的项和原因交回模型（不带任何取值）。 + 界面上还没保存的修改（pending）里同一列的码值并进新提的码值：界面上同一路径以后一次为准，不能把先提的码值丢了。 +- run_query：只读查询，只有数据源开了数据剖析才给。经剖析的只读守卫和单条时限（catalog_profile._Runner： + guard.check(readonly=True)，单条时限取剖析设置和数据源查询时限中较小的一个），每次请求最多 MAX_QUERIES 条（守卫拒掉的 + 也算），最多 QUERY_ROWS 行交给模型、SHOWN_ROWS 行交给界面。结果交给模型、界面之前先遮（engine/masking.mask_query_result： + 取自遮罩列的结果列整列换成「已遮罩」，改了名字的也认；这一次请求里前面遮掉的值按值换），报错原话里的值也换。没开剖析时 + 提示词里写明只能根据表结构和目录推断。 + +**遮罩列只给列名。** 表结构里遮罩的列只写列名和类型（不给注释），目录里这些列的码值、带着取值的备注和确认时的依据一律不给 +（剖析的备注里有取值范围，使用者的原话里也可能有）；界面上还没保存的修改里这些列的码值也不给。 + +**模型。** 取模型的顺序同起草（data/draft_model.py：起草模型 → 助手的模型 → 默认接入），用不了时的说法也同起草 +（先过 catalog.resolve_draft_model，接口回 409 catalog_draft_model_unavailable）。保留思考、温度 0，额度 +catalog.MODEL_MAX_TOKENS，服务方嫌大时退回 catalog.MODEL_FALLBACK_MAX_TOKENS（providers/budget.BudgetFallback）；单次调用 +最多等 catalog.MODEL_TIMEOUT_S（同起草）。网络错误、429、5xx 在还没输出任何东西时重试（最多 len(RETRY_DELAYS_S) 次）。 +一次请求最多调用 MAX_ROUNDS 次模型（查询、提案之后接着答）；倒数第二轮的工具结果里提醒它直接回答。 + +**事件**（每个事件一行 data: ）： +- {"type": "text", "delta"}：回答正文的增量。几轮之间补一个空行； +- {"type": "query", "sql", "columns", "rows"（前 SHOWN_ROWS 行）, "row_count", "truncated", "error"}：跑了一条只读查询； +- {"type": "proposal", "source", "source_id", "table", "table_label", "version", "changes": [{path, label, before, + before_status, after, value, reason, state, note?}]}：一批核对过的修改提案； +- {"type": "done", "usage": {input_tokens, output_tokens}, "queries"}； +- {"type": "error", "message", "code"}：中途出错，流到此为止（之后没有 done）。code：assistant_model_error 调用模型出错、 + assistant_truncated 输出到了额度上限被截断、assistant_timeout 单次调用超过时限、assistant_rounds 调用次数用完还没答完、 + assistant_failed 意料之外的错误。 +等模型的时候每 HEARTBEAT_S 秒发一行 SSE 注释（「: keep-alive」，不是事件），免得中间的代理断开。 + +**对话不存服务端**：每次请求带上整段对话（最多 MAX_MESSAGES 条、每条最多 MESSAGE_MAX 字）。查询结果只在这一次请求里 +看得到，提示词里让模型把要用的结论写进回答。界面上待保存的修改不设上限（宽表上逐列补中文名、含义，几轮下来常常超过一次 +保存的 200 项，不能因此不让问）：背景里只列最近的 PENDING_SHOWN 项（界面上新的提案接在后面），写明另有几项；码值照全部 +待保存的修改合并。 + +写给模型的文字都在名字带 _SYSTEM / _PROMPT 的常量里；这里的 ERROR_* 是给人看的。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +import json +import logging +from collections.abc import AsyncIterator, Iterable, Mapping +from dataclasses import dataclass, field +from datetime import date, datetime +from decimal import Decimal +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import catalog, catalog_jobs, catalog_profile, sqlcheck +from app.data.catalog_review import SOURCE_LABEL, item_label, slot_path, table_usage_counts +from app.data.catalog_transfer import filter_columns_problem +from app.data.draft_model import DRAFT_MODEL_PLACE +from app.data.engine import SnapshotTampered, masked_columns + +logger = logging.getLogger(__name__) + +#: 对话最多几条、每条最多几个字；界面上待保存的修改在背景里最多列几项（最近的；请求里交多少都收） +MAX_MESSAGES = 40 +MESSAGE_MAX = 4000 +PENDING_SHOWN = 200 +#: 每次请求最多调用几次模型、最多几条只读查询 +MAX_ROUNDS = 8 +MAX_QUERIES = 5 +#: 一条查询最多几行交给模型、几行交给界面；交给模型时每格最多几个字、一条结果最多几个字 +QUERY_ROWS = 50 +SHOWN_ROWS = 10 +CELL_MAX = 300 +TOOL_CHARS = 12_000 +#: 网络错误、429、5xx 重试前等几秒（依次取用，个数就是最多重试几次）。在对话里等，比起草（10、30 秒）短 +RETRY_DELAYS_S = (3.0, 10.0) +#: 等模型的时候每隔几秒发一行 SSE 注释 +HEARTBEAT_S = 10.0 +#: 背景里「目录里还没有中文名和含义的列」最多列几个；别的表指过来的关联关系最多列几条；这张表的目录最多列几项 +#: （几百列的宽表全部起草过有上千项，全列出来提示词就太长了：表级的项、关联关系在前，列的项在后） +UNNAMED_SHOWN = 30 +INCOMING_SHOWN = 30 +CATALOG_ITEMS_SHOWN = 600 + +QUERY_TOOL = "run_query" +PROPOSE_TOOL = "propose_changes" + +#: 错误事件的机读码 +CODE_MODEL = "assistant_model_error" +CODE_TRUNCATED = "assistant_truncated" +CODE_TIMEOUT = "assistant_timeout" +CODE_ROUNDS = "assistant_rounds" +CODE_FAILED = "assistant_failed" + +# ---- 给人看的 +ERROR_MODEL = ("调用模型「{model}」失败：{reason}。请稍后重试；如持续失败，请在{place}中换一个模型，" + "或到「设置 → 模型接入」测试该模型") +ERROR_TRUNCATED = "{cut}，回答没有写完。通常是思考占满了输出额度，可以把问题拆小一点再问，或在{place}中换一个模型" +ERROR_TIMEOUT = "模型 {seconds} 秒内没有回答完，已停止这一轮。可以把问题拆小一点再问，或在{place}中换一个模型" +ERROR_ROUNDS = "这一轮已经调用了 {n} 次模型仍没有答完，已停止。可以换个问法，或者分几次问" +ERROR_FAILED = "表助手出错：{line}" +#: 对话的最后一条是助手说的（接口 422） +LAST_NOT_USER = "对话的最后一条应当是使用者的话，请重新发送" +#: 起草模型刚才还能用、建流式模型时却用不了（期间设置被改了）时的说法 +ERROR_NO_MODEL = "起草模型暂时无法使用，请稍后重试，或到「设置 → 模型接入」检查" +#: 查询事件里的报错 +QUERY_TAMPERED = "数据源「{name}」的{error}。请联系管理员检查数据目录" + +# ---- 写给模型的 +ASSISTANT_SYSTEM = """你是 AgentLab 的「表助手」，和使用者一起完善一张表的数据目录：表和列的中文名、含义、单位、度量类型、码值、\ +粒度、表类型、业务主键、业务日期、有效记录条件、去重规则、关联关系。 + +规矩： +- 你只改数据目录，不改业务库：不能写数据、建表、改结构。 +- 只记数据事实，不写计算公式：「转化率=下单数/访问数」这类指标怎么算的说法属于口径卡，不进数据目录。 +- 拿不准就问，不要猜。一次只问一两件最要紧的事，问得具体（例如「status 为 9 的行是不是作废的单子？」）。目录里留空的列、\ +标着「推断」的项是最值得问的。 +- 使用者说出一条业务事实时，用 propose_changes 把它整理成修改提案，reason 里引用使用者的原话（例如「使用者说：9 是作废」）。\ +reason 里引用使用者原话只能引用这次对话里使用者的消息；依据来自下面背景里的数据源起草要求时写「起草要求写着：……」，\ +来自背景里目录项下面的「以前确认时的依据」时写「以前确认时的依据写着：……」（那是以前保存时记下的，不是这次对话里\ +使用者说的话），都不要写成「使用者说」。一次可以提多项。提案只是建议：使用者在界面上看过、保存之后才写进数据目录，\ +记为人工确认，reason 会作为这一项确认时的依据一起保存。 +- 表名、列名照原样写，只能用下面表结构里有的列。 +- 标着「已确认」的项由人工确认过，使用者没有明确要改就不要提修改；标着「已驳回」的项使用者驳回过，不要再提。 +- 「界面上还没保存的修改」是使用者已经接受、还没保存的提案：不要重复提；要改其中一项就再提一次，写完整的新值。 +- 遮罩的列只给了列名：不要猜它的取值，也不要给它写码值。 +- 回答用中文，简短直接。查询结果只在这一轮看得到，后面要用的结论请写进回答里。 + +修改提案的写法（propose_changes 的 changes，每项 path、value、reason）： +- path：表级项直接写字段名（label 中文名、description 说明、grain 粒度、keys 业务主键的列名列表、kind 表类型 \ +fact/dimension/snapshot/log/config、business_date {"column","rule","timezone"}、valid_filter 有效记录的 SQL 条件、dedup 去重\ +规则）；列级项写 columns.<列名>.<字段>（label、meaning 含义、unit 单位、measure 度量类型 \ +flow/stock/ratio/identifier/status/attribute、codes 码值 {"码":"含义"}）；已有的关联关系写 relations.<编号>；新增关联关系\ +写 relations.new，value 写 {"columns":[本表列],"to_table":"目标表","to_columns":[目标列],\ +"cardinality":"many_to_one/one_to_one/one_to_many"}。 +- codes 只写要补充或更正的码，原有的码会保留；valid_filter 要写成完整的条件（原有条件也要带上）。 +- valid_filter 只能用这张表的列（不带表名），由比较、IN、BETWEEN、LIKE、IS NULL 用 AND / OR / NOT 连起来，例如 \ +status NOT IN ('C', 'X')；不能有子查询、聚合、窗口函数、分号，函数只能用 UPPER、LOWER、TRIM、COALESCE。""" + +#: 查询工具的说明（开了剖析时接在系统提示后面）、没开剖析时的那句 +ASSISTANT_QUERY_PROMPT = """只读查询：run_query 在这个数据源上执行一条只读 SQL（{dialect} 的写法），最多返回 {rows} 行，\ +这一轮最多 {limit} 条。用它核对码值的取值和分布、日期范围、关联对不对得上；不要取明细的个人信息。遮罩列的值显示为「已遮罩」。""" +ASSISTANT_NO_QUERY_PROMPT = "这个数据源没开剖析，不能查数据，只能根据表结构和目录推断。" +ASSISTANT_DISABLED_PROMPT = "这个数据源已停用，不能查数据，只能根据表结构和目录推断。" +#: 使用者还没说话（刚打开表助手）时补的那句 +ASSISTANT_OPENING_PROMPT = ("（使用者刚打开表助手，还没有说话。）请用一两句话说明这张表的目录现在最缺什么，然后就最要紧的一两项" + "向使用者提问；先不要提修改提案。") +#: 对话从助手的开场白开始时，补在最前面的一句（各家的接口都要求从使用者这一边开始） +ASSISTANT_OPENED_PROMPT = "（使用者打开了表助手。）" + +#: 背景各段的写法 +ASSISTANT_PROMPT_LABELS = { + "head": "下面是要完善的表。", + "table": "数据源「{source}」（{kind}），表 {table}{view}", + "view": "（视图）", + "comment": "数据库注释：{text}", + "system": "系统说明：{text}", + "usage": "这张表被 {runs} 次运行用到（查询 {n} 次）。", + "usage_none": "这张表还没有被运行用到过。", + "description": "数据源说明：{text}", + "guidance": "数据源的起草要求（不是这次对话里使用者说的话）：{text}", + "structure": "表结构：", + "column": "- {name} {type}{marks}", + "pk": "主键", + "not_null": "非空", + "col_comment": "注释：{text}", + "masked": "遮罩列：只给列名，不给任何取值", + "unique": "唯一约束：{cols}", + "fk": "外键：{cols} → {target}", + "catalog": "这张表现在的数据目录（每项写明来源和状态；路径就是修改提案里的 path）：", + "catalog_empty": "这张表还没有数据目录。", + "item": "- {path}（{label}）：{value}〔{source}，{status}〕", + "item_note": " 备注：{text}", + "item_reason": " 以前确认时的依据：{text}", + "catalog_more": "（还有 {n} 项没有列出：这张表的目录太长。要改的项照上面的路径写法提，核对时按当前目录算）", + "relation": "{columns} → {to_table}.{to_columns}{extra}", + "relation_extra": "（{text}）", + "cardinality": "基数 {value}", + "coverage": "覆盖率 {value}", + "incoming": "别的表指向这张表的关联关系：", + "incoming_item": "- {table}.{columns} → {target}.{to_columns}〔{source}，{status}〕", + "blank": "最近一次起草时模型拿不准、留空的列：{cols}（共 {n} 列）", + "unnamed": "目录里还没有中文名和含义的列：{cols}（共 {n} 列{more}）", + "unnamed_more": "，这里列出前 {n} 个", + "pending": "界面上还没保存的修改（使用者已经接受、还没保存）：", + "pending_more": "（另有 {n} 项更早的待保存修改没有列出：太多了。它们同样已经接受，不要重复提）", + "pending_item": "- {path}：{value}（依据：{reason}）", + "status_proposed": "推断", + "status_verified": "有确证", + "status_confirmed": "已确认", + "status_rejected": "已驳回", + "field_relation": "关联关系", +} + +#: 工具结果的写法（交回模型） +ASSISTANT_TOOL_PROMPT = { + "proposed": "已把 {n} 项修改交给使用者确认（使用者保存后才写进数据目录）：{paths}", + "nothing": "没有可以交给使用者的修改。", + "dropped": "没有收下的项：{reasons}", + "masked_codes": "列 {column} 在数据源设置中被遮罩，不能给它写码值", + "same": "{where}已经是这个值、也确认过了，不用再提", + "no_changes": "提案里没有要修改的项", + "query_failed": "查询失败：{error}", + "query_capped": "未执行：这一轮最多查 {n} 条，不能再查了。请根据已经查到的结果回答", + "query_disabled": "未执行：这个数据源没开数据剖析，不能查数据", + "unknown_tool": "没有名为 {name} 的工具，可用的工具：{tools}", + "last_round": "（这一轮只剩最后一步了：请直接回答使用者，不要再调用工具。）", + "empty_sql": "未执行：没有给出 SQL", +} + +QUERY_TOOL_PROMPT: dict[str, Any] = { + "type": "function", + "function": { + "name": QUERY_TOOL, + "description": "在这个数据源上执行一条只读 SQL 查询，返回列名和前几行（遮罩列的值显示为「已遮罩」）。", + "parameters": { + "type": "object", + "properties": {"sql": {"type": "string", "description": "一条只读的 SELECT 语句"}}, + "required": ["sql"], + }, + }, +} +PROPOSE_TOOL_PROMPT: dict[str, Any] = { + "type": "function", + "function": { + "name": PROPOSE_TOOL, + "description": "把使用者说出的业务事实整理成一批数据目录修改提案，交给使用者确认后保存。不会直接写进数据目录。", + "parameters": { + "type": "object", + "properties": { + "changes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "path": {"type": "string", "description": "要改的项，写法见系统提示"}, + "value": {"description": "新的值：文字、列名列表、码值对照或关联关系"}, + "reason": {"type": "string", + "description": "一句话依据：引用这次对话里使用者的原话（「使用者说：……」）；" + "来自数据源起草要求的写「起草要求写着：……」，来自以前确认时的依据" + "的写「以前确认时的依据写着：……」"}, + }, + "required": ["path", "value", "reason"], + }, + }, + }, + "required": ["changes"], + }, + }, +} + + +# ========================================================================== +# 模型 +# ========================================================================== + + +async def resolve_model(session: AsyncSession) -> tuple[Any, str]: + """(模型, 模型 id)。能不能用、用不了时的说法和起草完全一致：先过 catalog.resolve_draft_model（起草任务建任务时也是这样 + 查的），用不了抛 catalog.CatalogModelUnavailable。再按同一个接入、模型建流式对话用的模型：保留思考、温度 0，大小两份额度 + （BudgetFallback），单次时限同起草。只建模型对象，不发请求。""" + from app.data.draft_model import DraftModelUnavailable, draft_unavailable, resolve_draft_provider + from app.providers.budget import BudgetFallback + from app.providers.factory import ModelSpec, ProviderNotConfigured, build_chat_model + + await catalog.resolve_draft_model(session) + try: + provider, choice = await resolve_draft_provider(session, none_reason=ERROR_NO_MODEL, mock_reason=ERROR_NO_MODEL) + except DraftModelUnavailable as e: + raise catalog.CatalogModelUnavailable(str(e)) from e + # SDK 自己不重试（同起草）：网络错误、429、5xx 由这里在还没输出时重试，两层叠起来实际次数就对不上了 + spec = ModelSpec(provider=choice.provider, model=choice.model, max_tokens=catalog.MODEL_MAX_TOKENS, temperature=0, + timeout=int(catalog.MODEL_TIMEOUT_S), max_retries=0) + try: + big = build_chat_model(provider, spec) + small = build_chat_model(provider, spec.model_copy(update={"max_tokens": catalog.MODEL_FALLBACK_MAX_TOKENS})) + except ProviderNotConfigured as e: + raise catalog.CatalogModelUnavailable(draft_unavailable(e, choice.origin)) from e + model_id = spec.model or provider.default_model or "" + return BudgetFallback(big, small, limits=(catalog.MODEL_MAX_TOKENS, catalog.MODEL_FALLBACK_MAX_TOKENS)), model_id + + +# ========================================================================== +# 对话 +# ========================================================================== + + +def conversation_problem(messages: Iterable[Mapping[str, Any]]) -> str | None: + """对话本身的问题（接口 422）：使用者说过话、最后一条却是助手说的。没有使用者的话是「刚打开表助手」，不算问题。""" + said = [m for m in messages if str(m.get("content") or "").strip()] + if any(m.get("role") == "user" for m in said) and said[-1].get("role") != "user": + return LAST_NOT_USER + return None + + +def _history(messages: Iterable[Mapping[str, Any]]) -> list[Any]: + """对话 → 模型的消息：空的不要，同一方接连说的并成一条；从助手的开场白开始的，前面补一句使用者打开了表助手；使用者还没 + 说话的，补一句开场的请求。""" + from langchain_core.messages import AIMessage, HumanMessage + + merged: list[tuple[str, str]] = [] + for m in messages: + text = str(m.get("content") or "").strip() + if not text: + continue + role = "user" if m.get("role") == "user" else "assistant" + if merged and merged[-1][0] == role: + merged[-1] = (role, f"{merged[-1][1]}\n\n{text}") + else: + merged.append((role, text)) + out: list[Any] = [] + if merged and merged[0][0] == "assistant": + out.append(HumanMessage(content=ASSISTANT_OPENED_PROMPT)) + for role, text in merged: + out.append(HumanMessage(content=text) if role == "user" else AIMessage(content=text)) + if not any(role == "user" for role, _ in merged): + out.append(HumanMessage(content=ASSISTANT_OPENING_PROMPT)) + return out + + +# ========================================================================== +# 背景 +# ========================================================================== + + +def _value_text(value: Any) -> str: + return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str) + + +def _status_text(status: Any) -> str: + return ASSISTANT_PROMPT_LABELS.get(f"status_{status}", str(status)) + + +def _source_text(source: Any) -> str: + return SOURCE_LABEL.get(str(source), str(source)) + + +def _relation_line(rel: Mapping[str, Any]) -> str: + labels = ASSISTANT_PROMPT_LABELS + extra = [] + if rel.get("cardinality"): + extra.append(labels["cardinality"].format(value=rel["cardinality"])) + coverage = rel.get("coverage") + if isinstance(coverage, (int, float)) and not isinstance(coverage, bool): + extra.append(labels["coverage"].format(value=f"{coverage:.0%}")) + return labels["relation"].format( + columns=", ".join(str(c) for c in rel.get("columns") or []), to_table=rel.get("to_table"), + to_columns=", ".join(str(c) for c in rel.get("to_columns") or []), + extra=labels["relation_extra"].format(text="，".join(extra)) if extra else "") + + +def _touches_masked(key: catalog.Slot, item: Mapping[str, Any], masked: set[str]) -> bool: + """这一项说的是不是遮罩列：列级的项看列名，业务主键、业务日期看指着的列，关联关系看本表这一端。""" + if key[0] == "c": + return key[1].lower() in masked + if key[0] == "r": + return any(str(c).lower() in masked for c in item.get("columns") or []) + value = item.get("value") + if key == ("t", "business_date") and isinstance(value, Mapping): + return str(value.get("column") or "").lower() in masked + if key == ("t", "keys") and isinstance(value, list): + return any(str(c).lower() in masked for c in value) + return False + + +def _catalog_lines(notes: Mapping[str, Any] | None, masked: set[str]) -> list[str]: + """这张表的目录，每项一行：路径、字段、值、来源和状态，有备注、有确认时的依据的各另起一行。依据是以前保存时记下的， + 写成「以前确认时的依据：……」，不让模型当成这次对话里使用者说的话。遮罩列上的码值不给，说到遮罩列的项不给备注和 + 依据（剖析的备注里有取值范围，使用者的原话里也可能有）。""" + labels = ASSISTANT_PROMPT_LABELS + lines: list[str] = [] + # 表级的项、关联关系在前，列的项在后（稳定排序，各自照目录里的顺序）；遮罩列上的码值不算 + slots = [(key, item) for key, item in sorted(catalog._slots(notes).items(), key=lambda kv: kv[0][0] == "c") + if not (key[0] == "c" and key[2] == "codes" and key[1].lower() in masked)] + for shown, (key, item) in enumerate(slots): + if shown >= CATALOG_ITEMS_SHOWN: + lines.append(labels["catalog_more"].format(n=len(slots) - shown)) + break + hidden = _touches_masked(key, item, masked) + if key[0] == "r": + label, value = labels["field_relation"], _relation_line(item) + else: + label = (catalog.TABLE_FIELD_LABEL if key[0] == "t" else catalog.COLUMN_FIELD_LABEL).get(key[-1], key[-1]) + value = _value_text(item.get("value")) + lines.append(labels["item"].format(path=slot_path(key), label=label, value=value, + source=_source_text(item.get("source")), + status=_status_text(item.get("status")))) + note = item.get("note") + if isinstance(note, str) and note.strip() and not hidden: + lines.append(labels["item_note"].format(text=note.strip())) + reason = item.get("reason") + if isinstance(reason, str) and reason.strip() and not hidden: + lines.append(labels["item_reason"].format(text=reason.strip())) + return lines + + +def _structure_lines(meta: Mapping[str, Any], masked: set[str], *, system: bool) -> list[str]: + labels = ASSISTANT_PROMPT_LABELS + pk = {str(c) for c in meta.get("primary_key") or []} + lines = [labels["structure"]] + for col in meta.get("columns") or []: + if not isinstance(col, Mapping) or col.get("name") is None: + continue + name = str(col["name"]) + marks: list[str] = [] + if name.lower() in masked: + marks.append(labels["masked"]) + else: + if name in pk: + marks.append(labels["pk"]) + if not col.get("nullable", True): + marks.append(labels["not_null"]) + text = str(col.get("comment") or "").strip() + if text: + marks.append((labels["system"] if system else labels["col_comment"]).format(text=text)) + lines.append(labels["column"].format(name=name, type=col.get("type") or "", + marks=("，" + "，".join(marks)) if marks else "")) + for group in meta.get("unique") or []: + if isinstance(group, list) and group: + lines.append(labels["unique"].format(cols="、".join(str(c) for c in group))) + for fk in meta.get("foreign_keys") or []: + if isinstance(fk, Mapping) and fk.get("columns") and fk.get("to_table"): + target = ", ".join(f"{fk['to_table']}.{c}" for c in fk.get("to_columns") or []) or str(fk["to_table"]) + lines.append(labels["fk"].format(cols=", ".join(str(c) for c in fk["columns"]), target=target)) + return lines + + +async def _last_blank(session: AsyncSession, source_id: str, table: str) -> tuple[int, list[str]]: + """最近一次结束于 ok / partial 的起草里这张表模型留空的列：(几列, 最多 5 个列名)。停止、中断的那次不算（那次没起草完， + 留空数是 0）。没有是 (0, [])。""" + from app.db.models import CatalogDraftJob, CatalogDraftTask + + row = (await session.execute( + select(CatalogDraftTask.blank_columns, CatalogDraftTask.blank_sample) + .join(CatalogDraftJob, CatalogDraftJob.id == CatalogDraftTask.job_id) + .where(CatalogDraftJob.source_id == source_id, CatalogDraftTask.table_name == table, + CatalogDraftTask.status.in_(("ok", "partial"))) + .order_by(CatalogDraftJob.created_at.desc(), CatalogDraftTask.seq).limit(1) + )).first() + if row is None: + return 0, [] + return int(row[0] or 0), [str(c) for c in row[1] or []] + + +@dataclass +class AssistantContext: + """一次请求要用到的东西，开流之前读好（流里不再用请求的会话）。""" + + source_id: str + source_name: str + kind: str + table: str + tables: dict[str, Any] + #: 读表结构、发查询用的源（上传源是绑定当前快照的视图） + query_source: Any + options: dict[str, Any] + masked: set[str] + can_query: bool + settings: catalog_profile.ProfileSettings + system: str + pending_codes: dict[str, dict[str, Any]] = field(default_factory=dict) + + def tools(self) -> list[dict[str, Any]]: + return [QUERY_TOOL_PROMPT, PROPOSE_TOOL_PROMPT] if self.can_query else [PROPOSE_TOOL_PROMPT] + + +async def prepare(session: AsyncSession, row: Any, source: Any, table: str, + pending: Iterable[Mapping[str, Any]] = ()) -> AssistantContext: + """读好这次请求要用的背景：表结构、目录、别的表指过来的关联关系、使用次数、起草要求、留空的列、待保存的修改，拼成系统 + 提示。row 是数据源记录（遮罩、剖析开关、起草要求按它），source 是读表结构和发查询用的源。table 是表结构里的键。""" + labels = ASSISTANT_PROMPT_LABELS + cache = getattr(source, "schema_cache", None) or {} + tables = cache.get("tables") or {} + meta = tables.get(table) or {} + options = dict(getattr(row, "options", None) or {}) + masked = {c.lower() for c in masked_columns(options)} + settings = catalog_profile.profile_settings(options) + enabled = getattr(row, "enabled", True) is not False + can_query = settings.enabled and enabled + system_notes = catalog.system_notes_source(source) + entries = await catalog.read_catalog(session, source.id) + entry = entries.get(table) + notes = entry.notes if entry else {} + usage = await table_usage_counts(session, source) + used, runs = usage.queries.get(table, 0), usage.runs.get(table, 0) + blank_n, blank_cols = await _last_blank(session, source.id, table) + + lines = [labels["head"], labels["table"].format(source=row.name, kind=row.kind, table=table, + view=labels["view"] if meta.get("is_view") else "")] + comment = str(meta.get("comment") or "").strip() + if comment: + lines.append((labels["system"] if system_notes else labels["comment"]).format(text=comment)) + lines.append(labels["usage"].format(runs=runs, n=used) if used else labels["usage_none"]) + description = str(getattr(row, "description", "") or "").strip() + if description: + lines.append(labels["description"].format(text=description)) + if guidance := catalog_jobs.guidance_of(row): + lines.append(labels["guidance"].format(text=guidance)) + lines += ["", *_structure_lines(meta, masked, system=system_notes), ""] + current = _catalog_lines(notes, masked) + lines += [labels["catalog"], *current] if current else [labels["catalog_empty"]] + incoming = [] + for other, other_entry in sorted(entries.items()): + if other == table: + continue + for key, rel in catalog._slots(other_entry.notes).items(): + if key[0] == "r" and rel.get("to_table") == table: + incoming.append(labels["incoming_item"].format( + table=other, columns=", ".join(str(c) for c in rel.get("columns") or []), target=table, + to_columns=", ".join(str(c) for c in rel.get("to_columns") or []), + source=_source_text(rel.get("source")), status=_status_text(rel.get("status")))) + if incoming: + lines += ["", labels["incoming"], *incoming[:INCOMING_SHOWN]] + named = catalog.visible_notes(notes).get("columns") or {} + columns = [str(c["name"]) for c in meta.get("columns") or [] if isinstance(c, Mapping) and c.get("name") is not None] + unnamed = [c for c in columns if not any(f in (named.get(c) or {}) for f in ("label", "meaning"))] + blank = [c for c in blank_cols if c in unnamed] + if blank_n and blank: + lines += ["", labels["blank"].format(cols="、".join(blank), n=blank_n)] + if unnamed: + more = labels["unnamed_more"].format(n=UNNAMED_SHOWN) if len(unnamed) > UNNAMED_SHOWN else "" + lines.append(labels["unnamed"].format(cols="、".join(unnamed[:UNNAMED_SHOWN]), n=len(unnamed), more=more)) + pending_codes: dict[str, dict[str, Any]] = {} + shown_pending: list[str] = [] + for change in pending: + path = str(change.get("path") or "").strip() + value = change.get("value") + try: + key = catalog.parse_path(path) + except catalog.CatalogPathError: + continue + if key[0] == "c" and key[2] == "codes": + if key[1].lower() in masked: + continue + if isinstance(value, Mapping): + pending_codes[path] = dict(value) + reason = str(change.get("reason") or "").strip() + shown_pending.append(labels["pending_item"].format(path=path, value=_value_text(value), reason=reason or "—")) + if shown_pending: + lines += ["", labels["pending"]] + if len(shown_pending) > PENDING_SHOWN: + lines.append(labels["pending_more"].format(n=len(shown_pending) - PENDING_SHOWN)) + lines += shown_pending[-PENDING_SHOWN:] + if can_query: + tools = ASSISTANT_QUERY_PROMPT.format(dialect=row.kind, rows=QUERY_ROWS, limit=MAX_QUERIES) + else: + tools = ASSISTANT_NO_QUERY_PROMPT if enabled or not settings.enabled else ASSISTANT_DISABLED_PROMPT + system = "\n\n".join([ASSISTANT_SYSTEM, tools, "\n".join(lines)]) + return AssistantContext(source_id=source.id, source_name=row.name, kind=str(row.kind or ""), table=table, + tables=tables, query_source=source, options=options, masked=masked, can_query=can_query, + settings=settings, system=system, pending_codes=pending_codes) + + +# ========================================================================== +# 工具 +# ========================================================================== + + +def _cell(value: Any, limit: int | None = None) -> Any: + """一格交出去的样子：数、布尔、空照原样，别的写成文字（日期、Decimal……），长的截断。""" + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, (datetime, date)): + text = value.isoformat() + elif isinstance(value, Decimal): + text = str(value) + elif isinstance(value, (bytes, bytearray)): + text = f"<{len(value)} 字节>" + else: + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str) + if limit is not None and len(text) > limit: + return text[:limit] + "…" + return text + + +def _rows(rows: Any, n: int, limit: int | None) -> list[list[Any]]: + out: list[list[Any]] = [] + for row in (rows or [])[:n]: + cells = row if isinstance(row, list) else list(row.values()) if isinstance(row, dict) else [row] + out.append([_cell(v, limit) for v in cells]) + return out + + +class _Tools: + """一次请求里的工具：查询次数、这一次遮掉的值（后面的查询按值换）。""" + + def __init__(self, ctx: AssistantContext) -> None: + from app.engine.masking import SeenValues + + self.ctx = ctx + self.queries = 0 + self.seen = SeenValues() + + # ---------------------------------------------------------------- 查询 + + async def query(self, sql: Any) -> tuple[dict[str, Any] | None, str]: + """(给界面的 query 事件, 交回模型的结果)。""" + prompts = ASSISTANT_TOOL_PROMPT + if not self.ctx.can_query: + return None, prompts["query_disabled"] + if self.queries >= MAX_QUERIES: + return None, prompts["query_capped"].format(n=MAX_QUERIES) + sql = sql.strip() if isinstance(sql, str) else "" + if not sql: + return None, prompts["empty_sql"] + self.queries += 1 + mask = masked_columns(self.ctx.options) + known = await asyncio.to_thread(self.seen.pooled) + known = known if known.strong else None + error: str | None = None + result: Any = None + settings = dataclasses.replace(self.ctx.settings, max_queries=1) + try: + runner = catalog_profile._Runner(self.ctx.query_source, settings, + catalog_profile.SqlDialect(self.ctx.kind)) + result = await runner(sql, max_rows=QUERY_ROWS) + except catalog_profile._Skip as e: + error = e.detail + except catalog_profile._Stop as e: + error = str(e) + except SnapshotTampered as e: + error = QUERY_TAMPERED.format(name=self.ctx.source_name, error=e) + except ValueError as e: # 不支持的数据库类型 + error = str(e) + if error is not None: + if mask or known is not None: + from app.engine.masking import mask_error_text + + error = mask_error_text(error, sql, known) + event = {"type": "query", "sql": sql, "columns": [], "rows": [], "row_count": 0, "truncated": False, + "error": error} + return event, prompts["query_failed"].format(error=error) + payload = result.to_payload() + if mask or known is not None: + from app.engine.masking import mask_query_result + + schema = getattr(self.ctx.query_source, "schema_cache", None) + done = await asyncio.to_thread(mask_query_result, payload, mask, self.ctx.kind, known, schema=schema) + payload = done.payload + await asyncio.to_thread(self.seen.add, done.secrets) + columns = [str(c) for c in payload.get("columns") or []] + rows = payload.get("rows") or [] + event = {"type": "query", "sql": str(payload.get("sql") or sql), "columns": columns, + "rows": _rows(rows, SHOWN_ROWS, None), "row_count": int(payload.get("row_count") or len(rows)), + "truncated": bool(payload.get("truncated")), "error": None} + given = {"columns": columns, "rows": _rows(rows, QUERY_ROWS, CELL_MAX), "row_count": event["row_count"], + "truncated": event["truncated"]} + text = json.dumps(given, ensure_ascii=False, default=str) + while len(text) > TOOL_CHARS and given["rows"]: + given["rows"] = given["rows"][:max(1, len(given["rows"]) // 2)] if len(given["rows"]) > 1 else [] + given["rows_shown"] = len(given["rows"]) + text = json.dumps(given, ensure_ascii=False, default=str) + return event, text + + # ---------------------------------------------------------------- 提案 + + def _with_pending(self, change: Any) -> Any: + """界面上还没保存的修改里同一列的码值，并进新提的码值（新提的同一个码为准）。""" + if not isinstance(change, Mapping): + return change + path = str(change.get("path") or "").strip() + before = self.ctx.pending_codes.get(path) + value = change.get("value") + if before is None or not isinstance(value, Mapping): + return change + return {**change, "value": {**before, **value}} + + def filter_problem(self, condition: Any) -> str | None: + """模型提的有效记录条件哪里不合规（交回模型的一句话），合规是 None。 + + 按模型起草时的规矩核对（sqlcheck.valid_filter_problem：一个条件、只用这张表的列、不能有子查询、聚合、窗口、分号， + 只能用比较、IN、BETWEEN、LIKE、IS NULL 和几个函数）；sqlcheck 里还没有这条规矩时，至少核对用到的列都是这张表的、 + 都在表结构里、没有子查询（catalog_transfer.filter_columns_problem，导入时核对人工写的条件也用它）。""" + meta = self.ctx.tables.get(self.ctx.table) or {} + strict = getattr(sqlcheck, "valid_filter_problem", None) + if strict is not None: + return strict(condition, meta, kind=self.ctx.kind, table=self.ctx.table) + return filter_columns_problem(condition, meta, kind=self.ctx.kind, table=self.ctx.table) + + async def propose(self, changes: Any) -> tuple[dict[str, Any] | None, str]: + """(给界面的 proposal 事件, 交回模型的结果)。提案不写目录。""" + from app.db.base import SessionLocal + + prompts = ASSISTANT_TOOL_PROMPT + if not isinstance(changes, list) or not changes: + return None, prompts["no_changes"] + changes = [self._with_pending(c) for c in changes] + async with SessionLocal() as session: + entry = await catalog.read_entry(session, self.ctx.source_id, self.ctx.table) + notes = entry.notes if entry else {} + plan = catalog.plan_patch(notes, changes, table=self.ctx.table, tables=self.ctx.tables) + dropped = list(plan.problems) + kept: list[catalog.PatchChange] = [] + for change in plan.changes: + key = catalog.parse_path(change.path) + if key[0] == "c" and key[2] == "codes" and key[1].lower() in self.ctx.masked: + dropped.append(prompts["masked_codes"].format(column=key[1])) + continue + if change.state == "same": + dropped.append(prompts["same"].format(where=_where(key, change))) + continue + if key == ("t", "valid_filter") and (problem := self.filter_problem(change.after)) is not None: + dropped.append(problem) + continue + kept.append(change) + said: list[str] = [] + event: dict[str, Any] | None = None + if kept: + label = notes.get("label") if isinstance(notes.get("label"), Mapping) else {} + event = { + "type": "proposal", "source": self.ctx.source_name, "source_id": self.ctx.source_id, + "table": self.ctx.table, + "table_label": label.get("value") if label and label.get("status") != "rejected" else None, + "version": entry.version if entry else 0, + "changes": [{**c.as_dict(), "label": _where(catalog.parse_path(c.path), c)} for c in kept], + } + said.append(prompts["proposed"].format(n=len(kept), paths="、".join(c.path for c in kept))) + else: + said.append(prompts["nothing"]) + if dropped: + said.append(prompts["dropped"].format(reasons="；".join(dropped))) + return event, "\n".join(said) + + +def _where(key: catalog.Slot, change: catalog.PatchChange) -> str: + """提案里一项的中文说法：「列 status 的码值」「表的有效记录条件」「指向 members 的关联关系」（同审阅队列的 field_label）。""" + value = change.after if isinstance(change.after, Mapping) else change.before + return item_label(key, value if isinstance(value, Mapping) else None) + + +# ========================================================================== +# 流 +# ========================================================================== + + +#: 上一轮的正文以这些结尾（说完了一句话）时，下一轮另起一段 +_SENTENCE_END = ("。", "！", "？", "；", "：", ".", "!", "?", ";", ":") + + +def _sse(obj: Mapping[str, Any]) -> str: + return f"data: {json.dumps(obj, ensure_ascii=False, default=str)}\n\n" + + +class _CallFailed(Exception): + """一次模型调用没成：code 是错误事件的机读码，message 给人看。""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +def _seconds(value: Any) -> str: + number = float(value) + return str(int(number)) if number.is_integer() else f"{number:g}" + + +def _transient(e: BaseException) -> bool: + from app.core.errors import NETWORK, RATE_LIMIT, SERVER, classify + + return classify(e) in (RATE_LIMIT, SERVER, NETWORK) + + +async def _pump(runner: Any, history: list[Any], queue: asyncio.Queue[tuple[str, Any]]) -> None: + """在单独的任务里读模型的流，一块一块放进队列；读完放 end，出错放 error。""" + try: + async for chunk in runner.astream(history): + await queue.put(("chunk", chunk)) + await queue.put(("end", None)) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 - 交给读队列的一方按原因处理 + await queue.put(("error", e)) + + +async def _round(runner: Any, history: list[Any], model_id: str) -> AsyncIterator[tuple[str, Any]]: + """调一次模型：交出 ("text", 增量)、("beat", None)（等得久了，发心跳）、最后 ("message", (合起来的消息, 截断了没有))。 + 网络错误、429、5xx 在还没输出任何正文时重试；超时、别的错误抛 _CallFailed。""" + from app.core.errors import TIMEOUT, classify, describe_exception + from app.engine.state import message_text + from app.providers.budget import cut_off + + loop = asyncio.get_running_loop() + attempt = 0 + while True: + queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue() + task = asyncio.ensure_future(_pump(runner, history, queue)) + deadline = loop.time() + float(catalog.MODEL_TIMEOUT_S) + message: Any = None + cut = False + started = False + failure: BaseException | None = None + try: + while True: + remaining = deadline - loop.time() + if remaining <= 0: + raise _CallFailed(CODE_TIMEOUT, ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S), + place=DRAFT_MODEL_PLACE)) + try: + kind, value = await asyncio.wait_for(queue.get(), timeout=min(remaining, HEARTBEAT_S)) + except asyncio.TimeoutError: + yield "beat", None + continue + if kind == "end": + break + if kind == "error": + failure = value + break + message = value if message is None else message + value + cut = cut or bool(cut_off(value)) + text = message_text(value) + if text: + started = True + yield "text", text + finally: + if not task.done(): + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + if failure is None: + yield "message", (message, cut) + return + if not started and attempt < len(RETRY_DELAYS_S) and _transient(failure): + logger.info("表助手：调用模型出错（%s），%s 秒后重试", type(failure).__name__, RETRY_DELAYS_S[attempt]) + await asyncio.sleep(RETRY_DELAYS_S[attempt]) + attempt += 1 + continue + if isinstance(failure, TimeoutError) or classify(failure) == TIMEOUT: + raise _CallFailed(CODE_TIMEOUT, ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S), + place=DRAFT_MODEL_PLACE)) from failure + raise _CallFailed(CODE_MODEL, ERROR_MODEL.format(model=model_id or "", reason=describe_exception(failure), + place=DRAFT_MODEL_PLACE)) from failure + + +async def stream(model: Any, model_id: str, ctx: AssistantContext, + messages: Iterable[Mapping[str, Any]]) -> AsyncIterator[str]: + """表助手的一次请求：交出 SSE 的每一行（事件、心跳注释）。""" + from langchain_core.messages import SystemMessage, ToolMessage, message_chunk_to_message + + from app.providers.budget import cut_words + + history: list[Any] = [SystemMessage(content=ctx.system), *_history(messages)] + tools = _Tools(ctx) + usage = {"input_tokens": 0, "output_tokens": 0} + wrote = "" + names = [t["function"]["name"] for t in ctx.tools()] + try: + # 绑工具也放在这里面：响应头已经发出去了，出了错要照实交一条错误事件，不能让流半路断掉 + runner = model.bind_tools(ctx.tools()) + for round_no in range(MAX_ROUNDS): + fresh = True + message: Any = None + cut = False + # 流在半路被关掉（使用者点了停止、断开）时，正在读的那次调用跟着停：aclosing 让 _round 的收尾马上执行 + async with contextlib.aclosing(_round(runner, history, model_id)) as rounds: + async for kind, value in rounds: + if kind == "beat": + yield ": keep-alive\n\n" + elif kind == "text": + if fresh and wrote.endswith(_SENTENCE_END): + # 上一轮（查询、提案之前）说完了一句话：这一轮另起一段 + yield _sse({"type": "text", "delta": "\n\n"}) + fresh = False + wrote = value + yield _sse({"type": "text", "delta": value}) + else: + message, cut = value + meta = getattr(message, "usage_metadata", None) or {} + usage["input_tokens"] += int(meta.get("input_tokens") or 0) + usage["output_tokens"] += int(meta.get("output_tokens") or 0) + if cut: + limit = getattr(model, "max_tokens", None) + yield _sse({"type": "error", "code": CODE_TRUNCATED, + "message": ERROR_TRUNCATED.format(cut=cut_words("表助手", limit), place=DRAFT_MODEL_PLACE)}) + return + calls = list(getattr(message, "tool_calls", None) or []) if message is not None else [] + if not calls: + yield _sse({"type": "done", "usage": usage, "queries": tools.queries}) + return + if round_no == MAX_ROUNDS - 1: + yield _sse({"type": "error", "code": CODE_ROUNDS, "message": ERROR_ROUNDS.format(n=MAX_ROUNDS)}) + return + # 合起来的流式块换成普通消息再交回模型（不带半截的工具调用块） + history.append(message_chunk_to_message(message)) + for call in calls: + name = str(call.get("name") or "") + args = call.get("args") if isinstance(call.get("args"), Mapping) else {} + call_id = str(call.get("id") or f"{name}-{round_no}-{len(history)}") + event: dict[str, Any] | None = None + if name == QUERY_TOOL: + event, content = await tools.query(args.get("sql")) + elif name == PROPOSE_TOOL: + event, content = await tools.propose(args.get("changes")) + else: + content = ASSISTANT_TOOL_PROMPT["unknown_tool"].format(name=name, tools="、".join(names)) + if event is not None: + yield _sse(event) + if round_no == MAX_ROUNDS - 2: + content = f"{content}\n{ASSISTANT_TOOL_PROMPT['last_round']}" + history.append(ToolMessage(content=content, tool_call_id=call_id)) + except _CallFailed as e: + yield _sse({"type": "error", "code": e.code, "message": e.message}) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 - 流已经开了，照实交一条错误事件 + from app.core.errors import describe_exception, first_line + + logger.exception("表助手出错：数据源 %s 表 %s", ctx.source_id, ctx.table) + yield _sse({"type": "error", "code": CODE_FAILED, + "message": ERROR_FAILED.format(line=first_line(e) or describe_exception(e))}) + + +__all__ = ["AssistantContext", "CODE_FAILED", "CODE_MODEL", "CODE_ROUNDS", "CODE_TIMEOUT", "CODE_TRUNCATED", + "HEARTBEAT_S", "MAX_MESSAGES", "MAX_QUERIES", "MAX_ROUNDS", "MESSAGE_MAX", "PENDING_SHOWN", "PROPOSE_TOOL", + "QUERY_ROWS", "QUERY_TOOL", "SHOWN_ROWS", "conversation_problem", "prepare", "resolve_model", "stream"] diff --git a/backend/app/data/catalog_draft.py b/backend/app/data/catalog_draft.py index 4183bff1..d972c7fb 100644 --- a/backend/app/data/catalog_draft.py +++ b/backend/app/data/catalog_draft.py @@ -42,19 +42,25 @@ 地方写成了数组、对象的值在校验（catalog._from_model）里丢掉，不算出错。 - 封顶：每个原始段（计划里的整表段、表级那段或一个列分段，连同按缩小后的上限分开的各块、拆出来的、补问的、重试的） 一共最多调用 MODEL_PART_MAX_CALLS 次，到了次数还没写完的记没写出来。 -- 止损：连续 MODEL_GIVE_UP_AFTER 段拆到底仍一项没写出来（模型明显用不了，例如思考总是吃满额度），这次起草后面的 - 段不再拆小重试、没写出来就直接记下，报错建议更换起草模型；有一段写出来了就恢复。超时另有一条：拆到每段 - MODEL_MIN_PART_COLUMNS 列的调用接连超时 MODEL_GIVE_UP_AFTER 次（超时和段大小无关，再拆小也是一次一个时限），后面 - 超时的不再拆小，超时拆出来、还没问的块也不再问；有一次调用写出来了就恢复。不然一直超时的模型首轮同时起草的几段 - 要各自拆满次数，一段 40 列串着问约 7 个时限（420 秒时约 49 分钟）。 +- 止损按比例：这次起草最近 MODEL_HALT_WINDOW 次调用里有 MODEL_HALT_ON 次以上没写出来（额度用完、超时、空壳、出错）， + 之后的段不再拆小、每块只问一次（不拆小、不补问漏掉的列、截断后不再补剩下的、空壳不再换纯文本原样再问；拆出来还没问 + 的块也不再问），报错建议更换起草模型，直到比例降下去。以前按「连续几段拆到底」「接连超时」算，偶尔一次写出来就清零： + 实测会思考的模型在宽表上单次 1 到 7 分钟、四分之一超时，90 分钟停不下来。几处只管通信的调整照旧：服务方嫌额度 + 太大退回小额度、结构化输出不支持或回了说明文字改走纯文本、网络错误退避重试；结构化输出只给空壳的，这次起草还没换 + 纯文本问过的照旧问一次（看看是不是结构化输出的问题）。止损的窗口只记每次调用的最终结局：上面这几种重来之前的那次 + 不花额度、也说明不了模型写不出来，照旧报给进度回调（用量、提醒照算），不进止损的窗口——以前都记成没写出来，额度上限 + 8192、不支持 json_schema 的 OpenAI 兼容网关上，并发的头几次调用全是这种重来，第一次额度用完就凑满 10 次里 7 次， + 那一段直接记没写出来、不再拆小。重试用完仍出错的那一次是最终结局，照常算。 +- 同一张表连续 MODEL_TABLE_GIVE_UP_AFTER 个原始段都一项没写出来：这张表剩下还没开始的段跳过不问（宽表一段一段问下去 + 只是烧额度），原因写清跳过了几段、可以换一个起草模型后只重试这张表。中间有一段写出来了就不算「接连」。 - 提醒：最近 MODEL_NOTICE_WINDOW 次调用里有 MODEL_NOTICE_ON 次以上没写出来（额度用完、超时、空壳、出错），给一句 提醒（次数和原因，建议换起草模型、缩小起草范围或停止），经进度回调交给任务；之后少于 MODEL_NOTICE_OFF 次时清掉。 以前看不出调用在反复失败：一个任务 55 分钟只结束 1 张表，界面上只有段进度和用量。 **原因。** 没写出来的原因只写进这张表的 model_error：几段因为同一个原因没写出来时说「这几段」，分段起草的表（只写 出一部分的、一项都没写出来的都一样）每段写上列的范围（「第 5 段（列名 到 列名，共 33 列）」，列名照表结构原样）， -真拆小重试过的才补「已拆小重试仍没写出」。只有一段的表，这一段就是整张表：只写出一部分的写清是哪几列，一项都没写 -出来的只写原因。 +真拆小重试过的才补「已拆小重试仍没写出」，而且只跟在拆过的那几段后面说（说到的段只有一部分拆过时写「其中第 3 段」）。 +只有一段的表，这一段就是整张表：只写出一部分的写清是哪几列，一项都没写出来的只写原因。 **留空的列。** 请模型写了、模型也列出了，中文名和含义却都没给（或者给的都没通过校验）的列是模型拿不准，不补问， 记在 ModelOutcome.blank 里（表的状态不因此变）。目录里已经有中文名或含义的列不算（不论来源，驳回的不算有：以前模型 @@ -67,11 +73,22 @@ 的不会等前一个排着的段全部调完。一次起草里按表的先后排队：宽表的表级那段写完，它的列分段排到已经排着的段后面 （先来先服务），不插到窄表前面。 -**进度。** 调用方给 Hooks：开始调用、每次调用的结果（用量、没写出来的原因、当前的提醒和列数上限）、一段结束、一张表 -的模型部分全部结束。后台起草任务据此落库，每张表一结束就并入目录（data/catalog_jobs.py）。 +**进度。** 调用方给 Hooks：开始调用、每次调用的结果（用量、没写出来的原因、当前的提醒和列数上限）、一段结束、宽表的 +一段写出来了（交出这张表到这时拿到的全部内容，调用方马上并入目录：停止、服务重启时已经写出来的段不丢）、块进度变了、 +一张表的模型部分全部结束。后台起草任务据此落库（data/catalog_jobs.py）。块：这张表实际分成几块来问——一开始是段数， +一段被缩小的上限分开成几块、被拆成两半，块数就多几块；一块问完（写没写出来都算）算结束一块。 -交给模型的只有表结构（列、类型、主键、外键、唯一约束、注释）和使用者补充的背景，不发任何数据值。写给模型的文字都在 -catalog.py 名字带 _SYSTEM / _PROMPT 的常量里；这里的 REASON_* 是给人看的（拼进这张表的 model_error）。 +**防错事实。** 表级的项里模型可以提议有效行过滤（valid_filter）和去重规则（dedup），各附一句依据（写进这一项的 note）， +校验见 catalog._from_model（有效行过滤按方言解析，不过就丢掉；存的是整理过的写法：列换成表结构里的真名、按方言加引号， +剖析核对的也是这一条）。开了数据剖析的源（catalog_profile.profile_settings）：目录里记下的码值交给模型做背景（遮罩的列 +不给，「已确认的目录项」里遮罩列的码值同样不给），表级那段写完后经剖析的只读守卫数一数有效行过滤排除了多少行 +（catalog_profile.check_valid_filter），排除 0% 或 100% 的不收，收下的 note 写上排除比例；没开剖析、连不上的照收， +note 写没有核对的原因。没收下的（没通过校验、排除 0% 或 100%）不算失败，在这张表的日志里记一句（ModelOutcome.log， +起草任务逐表的 log）。 + +交给模型的是表结构（列、类型、主键、外键、唯一约束、注释）和使用者补充的背景；开了数据剖析的源另给目录里记下的码值 +（剖析取过的取值和含义，遮罩的列不给），别的数据值一概不发。写给模型的文字都在 catalog.py 名字带 _SYSTEM / _PROMPT +的常量里；这里的 REASON_* 是给人看的（拼进这张表的 model_error）。 """ from __future__ import annotations @@ -111,21 +128,36 @@ SEG_ONE = "这一段" SEG_MANY = "这几段" REASON_EXHAUSTED = "模型的思考和输出用完了额度（{budget} 个 token），{seg}没写出来" REASON_SPLIT = "；已拆小重试仍没写出" +#: 说到的几段里只有一部分真拆小重试过：只说那几段 +REASON_SPLIT_PARTS = "；其中第 {parts} 段已拆小重试仍没写出" REASON_HOLLOW = "模型只给了空壳，没有写出可用的内容" REASON_TRUNCATED = "模型的输出被截断，{seg}没写完" REASON_FORMAT = "模型的回复不是约定的格式" REASON_MISSING = "模型的回复里没有这张表" REASON_UNANSWERED = "模型的回复里漏了这些列，补问一次仍没有给出" +#: 止损期间每块只问一次：漏了的列不补问 +REASON_OMITTED = "模型的回复里漏了这些列" REASON_TIMEOUT = "模型 {seconds} 秒内没有写完{seg}" REASON_CALL = "调用模型出错：{line}" REASON_RETRIED = "（已重试 {n} 次）" REASON_CRASH = "起草这张表时出错：{line}" #: 一段到了调用次数的上限（MODEL_PART_MAX_CALLS）还没写完 REASON_CAPPED = "每段最多调用 {n} 次模型，{seg}到了次数仍没写完，已停止重试" -#: 止损（连续几段拆到底仍一项没写出来，后面的段不再拆小重试）时补在原因后面 -REASON_HALTED = "；模型连续 {n} 段都没写出来，已停止拆小重试，建议更换起草模型" -#: 拆到每段 MODEL_MIN_PART_COLUMNS 列仍接连超时（超时和段大小无关，后面超时的不再拆小）时补在原因后面 -REASON_SLOW = "；拆到每段 {n} 列仍然超时，已停止拆小重试，建议更换起草模型" +#: 止损（最近几次调用多数没写出来，之后的段不再拆小、每块只问一次）时补在原因后面 +REASON_HALTED = "；最近 {window} 次调用有 {on} 次以上没写出来，已停止拆小重试，建议更换起草模型" +#: 同一张表接连几段都没写出来，剩下的段跳过不问 +REASON_ABANDONED = "这张表接连 {k} 段都没写出来，剩下的 {n} 段已跳过，可以换一个起草模型后只重试这张表" +#: 跳过不问的段在「没写出来」记录里的原因（不拼进原因，见 _Run._outcome） +_SKIPPED = "\0skipped" + +# ---- 这张表起草时记下的几句话（不算失败），经 ModelOutcome.log 交给起草任务逐表的 log（界面看得到） +#: 模型提议的有效记录条件没有收下：why 是原因（没通过校验的说法，或者剖析核对的结果） +LOG_FILTER_DROPPED = "{why}，模型提议的这一条没有收下：{condition}" +#: 剖析核对排除了 0% 或 100% 的行 +LOG_FILTER_COUNTED = "{label}按剖析核对{what}（共 {rows} 行）" +#: 一张表最多记几句；日志里的条件最多写几个字 +LOG_MAX = 10 +_LOG_CONDITION_MAX = 200 # ---- 提醒（最近几次调用多数没写出来），经进度回调交给任务（CatalogDraftJob.notice） NOTICE = ("最近 {calls} 次调用有 {failed} 次没写出来（{kinds}）。建议在{place}换一个模型、缩小起草范围，" @@ -154,8 +186,10 @@ class ModelOutcome: status：ok 各段都写出来了；partial 有的段写出来、有的没有（error 写清哪部分、为什么）；failed 一项都没写出来 （error 写原因）。notes 是写出来的那些（已逐项校验，llm / proposed）；items 是项数。parts_total 分几段起草 - （整表段是 1，宽表是 1 + 列分段数），parts_done 已经结束了几段。blank：模型拿不准、留空了的列（按表结构的顺序； - 表的状态不因此变）。""" + （整表段是 1，宽表是 1 + 列分段数），parts_done 已经结束了几段，parts_written 其中写出了内容的段数。blank：模型拿 + 不准、留空了的列（按表结构的顺序；表的状态不因此变）。chunks_total、chunks_done：这张表实际分成的块数和已结束的 + 块数（段被缩小的上限分开、拆成两半后会变多）。log：起草时记下的几句话，不算失败（模型提议的有效记录条件没有收下的 + 原因），最多 LOG_MAX 句。""" status: str notes: dict[str, Any] @@ -164,6 +198,19 @@ class ModelOutcome: parts_total: int parts_done: int blank: list[str] = field(default_factory=list) + parts_written: int = 0 + chunks_total: int = 0 + chunks_done: int = 0 + log: list[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class PartWrite: + """宽表写出了一段（进度回调 part_written 收到的）：notes 是这张表到这时拿到的全部模型起草（已校验、已核对，深拷贝）， + parts_written 是其中写出了内容的段数。调用方马上并入目录（这一次不删任何东西），整张表结束时另有 table_done 收尾。""" + + notes: dict[str, Any] + parts_written: int @dataclass(frozen=True) @@ -192,12 +239,16 @@ class Hooks: - call_started(表名列表)：一次调用拿到并发名额、马上要发出去了； - call_done(CallReport)：每次调用一回（条数就是调用次数），带着用量、没写出来的原因、当前的提醒和列数上限； - part_done(表名)：这张表的一段结束了（写没写出来都算）； + - part_written(表名, PartWrite)：宽表写出了一段、后面还有段没结束（最后一段由 table_done 收尾；只有一段的表不调）； + - chunks_changed(表名, 块数, 已结束的块数)：块进度变了； - table_done(表名, ModelOutcome)：这张表的模型部分全部结束。""" call_started: Callable[[list[str]], Awaitable[None]] | None = None call_done: Callable[[CallReport], Awaitable[None]] | None = None part_done: Callable[[str], Awaitable[None]] | None = None table_done: Callable[[str, ModelOutcome], Awaitable[None]] | None = None + part_written: Callable[[str, PartWrite], Awaitable[None]] | None = None + chunks_changed: Callable[[str, int, int], Awaitable[None]] | None = None def parts_for(columns: int) -> int: @@ -328,9 +379,9 @@ class _Reply: @dataclass(frozen=True) class _Why: - """一部分没写出来的原因。base 给人看的原因（可以带 SEG，拼成一句时换成「这一段」「这几段」）；split 真拆小重试过； - halted 本来还能拆小、因为止损才没拆时止损的说明（REASON_HALTED：前面连续几段都没写出来；REASON_SLOW：拆到每段 - MODEL_MIN_PART_COLUMNS 列仍接连超时），没有是空串。""" + """一部分没写出来的原因。base 给人看的原因（可以带 SEG，拼成一句时换成「这一段」「这几段」；跳过不问的段是 _SKIPPED， + 不拼进原因）；split 真拆小重试过；halted 本来还能拆小、补问或再问、因为止损才没问时止损的说明（REASON_HALTED：最近 + 几次调用多数没写出来），没有是空串。""" base: str split: bool = False @@ -341,18 +392,23 @@ def _join(failures: Iterable[tuple[_Why, int]]) -> str: """几处没写出来的原因拼成一句。failures 是 (原因, 属于这张表的第几段)。 同样的原因只说一次（最多说两种）；同一个原因落在几段上说「这几段」，同一段拆出来的几块仍是「这一段」；拆小重试过、 - 止损的说明各在最后补一次（只看说出来的那几种；两种止损各说一次）。不然宽表的表级那段拆不开、列分段拆过时，同一句 - 原因会说两遍。""" + 止损的说明各在最后补一次（只看说出来的那几种）。不然宽表的表级那段拆不开、列分段拆过时，同一句原因会说两遍。 + + 「已拆小重试仍没写出」只跟在真拆过的那几段后面说：说到的段都拆过（只有一段的表就是这一段）时不点名，只有一部分拆过 + 时写「其中第 3、4 段已拆小重试仍没写出」。以前一律补在最后，读起来像每一段都拆过。""" failures = list(failures) parts: dict[str, set[int]] = {} for why, part in failures: parts.setdefault(why.base, set()).add(part) shown = list(parts)[:2] - told = [w for w, _ in failures if w.base in shown] + told = [(w, part) for w, part in failures if w.base in shown] text = "；".join(base.replace(SEG, SEG_MANY if len(parts[base]) > 1 else SEG_ONE) for base in shown) - if any(w.split for w in told): - text += REASON_SPLIT - for note in dict.fromkeys(w.halted for w in told if w.halted): + split = sorted({part for w, part in told if w.split}) + if split: + everywhere = {part for _, part in told} + text += REASON_SPLIT if set(split) == everywhere else \ + REASON_SPLIT_PARTS.format(parts="、".join(str(part) for part in split)) + for note in dict.fromkeys(w.halted for w, _ in told if w.halted): text += note return text @@ -706,6 +762,20 @@ class _Table: failed: dict[str, _Why] = field(default_factory=dict) parts_done: int = 0 finished: bool = False + #: 写出了内容的段（第几段） + written_parts: set[int] = field(default_factory=set) + #: 接连几个原始段一项都没写出来（写出来就清零）；到了 MODEL_TABLE_GIVE_UP_AFTER 段，剩下的段跳过不问 + dead_parts: int = 0 + abandoned: bool = False + #: 跳过不问的段（第几段） + skipped_parts: list[int] = field(default_factory=list) + #: 块进度：实际分成几块、结束了几块 + chunks_total: int = 0 + chunks_done: int = 0 + #: 有效行过滤核对过了（表级那段写完核对一次） + vetted: bool = False + #: 起草时记下的几句话（不算失败）：模型提议的有效记录条件没有收下的原因 + log: list[str] = field(default_factory=list) def whole(self, piece: _Piece) -> bool: return piece.table_level and len(piece.columns) == len(self.columns) @@ -753,6 +823,15 @@ class _Budget: left: int +@dataclass +class _Chunk: + """一块：一次拿来问的几张表（多表拼成的段每张表各是一块）。补问、截断后补剩下的、换纯文本再问都还是这一块；拆成两半 + （单张表按列拆，多一块；多表的段按表拆，各张表仍是一块）后，这一块里被拆的表交给新的块，split 记下它们。""" + + tables: tuple[str, ...] + split: set[str] = field(default_factory=set) + + def _pack(pieces: list[_Piece], cap: int) -> list[list[_Piece]]: """多表的段按顺序重新拼成每段不超过 cap 列（超过 cap 的表自成一段）。""" groups: list[list[_Piece]] = [] @@ -800,31 +879,37 @@ class _Run: self.background = self._background(getattr(source, "description", None), background) #: 这次起草同时在等名额或正在进行的调用最多几个（全进程的名额另算，见 model_gate） self.gate = asyncio.Semaphore(max(1, int(C.MODEL_CONCURRENCY))) - #: 连续几段拆到底仍一项没写出来（止损用，有一段写出来就清零） - self.dead_streak = 0 - #: 拆到每段 MODEL_MIN_PART_COLUMNS 列（或更小）的调用接连超时了几次（有一次调用写出来就清零）：超时和段大小 - #: 无关，再拆小也是一次一个时限，后面超时的不再拆小 - self.slow_streak = 0 #: 新开始的段每段最多几列：额度用完或超时（strikes）累计 MODEL_SHRINK_AFTER 次减半一次 self.part_columns = max(1, int(C.MODEL_BATCH_COLUMNS)) self.strikes = 0 #: 最近几次调用没写出来的原因种类（写出来了是 None），提醒看它 self.recent: collections.deque[str | None] = collections.deque(maxlen=max(1, int(C.MODEL_NOTICE_WINDOW))) self.notice: str | None = None + #: 最近 MODEL_HALT_WINDOW 次调用没写出来没有（止损按比例看它） + self.window: collections.deque[bool] = collections.deque(maxlen=max(1, int(C.MODEL_HALT_WINDOW))) + self.was_halted = False + #: 这次起草里结构化输出只给空壳时换纯文本问过没有（止损期间只问这一次） + self.text_probed = False + #: 数据源：开了数据剖析的源把码值交给模型、核对有效行过滤的排除比例（遮罩的列都不碰） + from app.data import catalog_profile + from app.data.engine import masked_columns - def _halt_note(self, kind: str | None = None) -> str: - """止损的说明（拼在原因后面），没止损是空串。 + self.source = source + self.kind = getattr(source, "kind", None) + options = getattr(source, "options", None) + self.profiling = catalog_profile.profile_settings(options if isinstance(options, dict) else None) + self.masked = {c.lower() for c in masked_columns(options if isinstance(options, dict) else None)} - - 连续 MODEL_GIVE_UP_AFTER 段一项都没写出来：后面的段不再拆小重试（不论原因）； - - 超时的（kind 是 timeout）另看：拆到每段 MODEL_MIN_PART_COLUMNS 列的调用接连超时 MODEL_GIVE_UP_AFTER 次， - 超时就和段大小无关了，再拆小也是一次一个时限，后面超时的不再拆小，超时拆出来还没问的块也不再问。只管超时： - 额度用完、只给空壳照旧拆小（段小了思考和输出才放得下）。""" - give_up = max(1, int(C.MODEL_GIVE_UP_AFTER)) - if self.dead_streak >= give_up: - return REASON_HALTED.format(n=give_up) - if kind == "timeout" and self.slow_streak >= give_up: - return REASON_SLOW.format(n=self._floor()) - return "" + def _halted(self) -> bool: + """止损：最近 MODEL_HALT_WINDOW 次调用里有 MODEL_HALT_ON 次以上没写出来。之后的段不再拆小、每块只问一次，直到 + 比例降下去（偶尔一次写出来不会清零）。""" + return sum(self.window) >= max(1, int(C.MODEL_HALT_ON)) + + def _halt_note(self) -> str: + """止损的说明（拼在原因后面），没止损是空串。""" + if not self._halted(): + return "" + return REASON_HALTED.format(window=max(1, int(C.MODEL_HALT_WINDOW)), on=max(1, int(C.MODEL_HALT_ON))) @staticmethod def _floor() -> int: @@ -846,6 +931,8 @@ class _Run: head, *rest = self.tables[unit].parts # 表级那段先写：列分段拿它做背景 queue.append(_Work([head], [_Work([p]) for p in rest])) + for st in self.tables.values(): + st.chunks_total = len(st.parts) left = {name: len(st.parts) for name, st in self.tables.items()} busy = 0 changed = asyncio.Event() @@ -898,9 +985,19 @@ class _Run: """起草一段；它之后的段（宽表的列分段）排到队尾；一张表的段都结束了，这张表的模型部分就结束了。 这一段（原始段：计划里的整表段、表级那段或一个列分段）连同按缩小后的上限分开的各块、拆出来的、补问的、重试的， - 一共最多调用 MODEL_PART_MAX_CALLS 次（各块共用一个 _Budget）；止损按原始段数（_tally）。""" - await self._start(work.pieces, _Budget(max(1, int(C.MODEL_PART_MAX_CALLS)))) - self._tally(work.pieces) + 一共最多调用 MODEL_PART_MAX_CALLS 次（各块共用一个 _Budget）。表级那段写完先核对模型提议的有效行过滤（_vet）， + 再交出去。接连几段没写出来的表（_tally），剩下的段跳过不问。宽表写出了一段、后面还有段没结束，就把这张表到这时 + 拿到的内容交给调用方并入目录（part_written）；最后一段结束由 table_done 收尾。""" + asked = [p for p in work.pieces if not self.tables[p.table].abandoned] + for piece in work.pieces: + if self.tables[piece.table].abandoned: + await self._skip_part(piece) + if asked: + await self._start(asked, _Budget(max(1, int(C.MODEL_PART_MAX_CALLS)))) + for piece in asked: + if piece.table_level: + await self._vet(self.tables[piece.table]) + self._tally(asked) for piece in work.pieces: await self._part_done(piece.table) queue.extend(work.then) @@ -908,6 +1005,8 @@ class _Run: left[piece.table] -= 1 if not left[piece.table]: await self._table_done(piece.table) + elif piece.part in self.tables[piece.table].written_parts: + await self._part_written(piece.table) async def _start(self, pieces: list[_Piece], budget: _Budget) -> None: """开始起草一段。超过当前的列数上限（额度用完、超时多了会减半）的先按上限分开：多表的段按表重新拼，单张表的 @@ -922,6 +1021,7 @@ class _Run: return piece = pieces[0] chunks = _even(piece.columns, -(-len(piece.columns) // cap)) + await self._more_chunks(piece.table, len(chunks) - 1) first = [_Piece(piece.table, chunks[0], piece.table_level, piece.part)] rest = [[_Piece(piece.table, cols, False, piece.part)] for cols in chunks[1:]] if piece.table_level: @@ -931,9 +1031,10 @@ class _Run: await asyncio.gather(*(self._guarded(group, budget) for group in [first, *rest])) async def _guarded(self, pieces: list[_Piece], budget: _Budget) -> None: - """起草一段（或它按上限分开的一块）；意料之外的错误只记这一块没写出来，不影响别的段和别的表。""" + """起草一块（一段，或它按上限分开的一块）；意料之外的错误只记这一块没写出来，不影响别的段和别的表。""" + chunk = _Chunk(tuple(dict.fromkeys(p.table for p in pieces))) try: - await self._segment(pieces, 0, False, budget) + await self._segment(pieces, 0, False, budget, chunk=chunk) except asyncio.CancelledError: raise except Exception as e: # noqa: BLE001 @@ -941,49 +1042,80 @@ class _Run: logger.exception("起草数据目录时出错：%s", "、".join(p.table for p in pieces)) self._fail(pieces, REASON_CRASH.format(line=first_line(e) or describe_exception(e))) + await self._chunk_done(chunk) def _tally(self, pieces: list[_Piece]) -> None: - """一段（原始段，连同它分开的各块、拆小、补缺）结束：一项都没写出来的接着数，写出来了清零。""" - if any(self.tables[p.table].wrote(p) for p in pieces): - self.dead_streak = 0 + """一段（原始段，连同它分开的各块、拆小、补缺）结束：记下这一段写出内容没有；同一张表接连 + MODEL_TABLE_GIVE_UP_AFTER 段一项都没写出来，这张表剩下的段跳过不问（写出来了清零）。""" + give_up = max(1, int(C.MODEL_TABLE_GIVE_UP_AFTER)) + for piece in pieces: + st = self.tables[piece.table] + if st.wrote(piece): + st.written_parts.add(piece.part) + st.dead_parts = 0 + continue + st.dead_parts += 1 + if st.dead_parts >= give_up and not st.abandoned and len(st.parts) > 1: + st.abandoned = True + logger.warning("起草数据目录：表 %s 接连 %d 段都没写出来，剩下的段跳过", st.name, st.dead_parts) + + async def _skip_part(self, piece: _Piece) -> None: + """这张表接连几段都没写出来：这一段跳过不问（算一块结束）。""" + st = self.tables[piece.table] + st.skipped_parts.append(piece.part) + self._fail([piece], _Why(_SKIPPED)) + st.chunks_done += 1 + await self._hook(self.hooks.chunks_changed, st.name, st.chunks_total, st.chunks_done) + + # ---------------------------------------------------------------- 块进度 + + async def _more_chunks(self, table: str, n: int) -> None: + if n <= 0: return - self.dead_streak += 1 - if self.dead_streak == max(1, int(C.MODEL_GIVE_UP_AFTER)): - logger.warning("起草数据目录：模型连续 %d 段都没写出来，这次起草后面的段不再拆小重试", self.dead_streak) + st = self.tables[table] + st.chunks_total += n + await self._hook(self.hooks.chunks_changed, table, st.chunks_total, st.chunks_done) + + async def _chunk_done(self, chunk: _Chunk) -> None: + """一块问完了：这一块里没被拆给新块的表，各结束一块。""" + for table in chunk.tables: + if table in chunk.split: + continue + st = self.tables[table] + st.chunks_done += 1 + await self._hook(self.hooks.chunks_changed, table, st.chunks_total, st.chunks_done) # ---------------------------------------------------------------- 一段 async def _segment(self, pieces: list[_Piece], depth: int, followup: bool, budget: _Budget, *, - halved: bool = False, mode: str | None = None, last: bool = False, - optional: bool = False) -> None: + chunk: _Chunk, halved: bool = False, mode: str | None = None, last: bool = False, + optional: str | None = None) -> None: """问一段，拿到的并进各表；按失败原因拆开、补缺、记下没写出来的。 depth：已经拆了几层（截断后只补剩下的也算一层）。followup：这一段是在补缺（补过就不再补，见 _after_followup）。 - budget：这一段（原始段）还能调用几次。halved：这几块是真拆小（_split）出来的——原因里「已拆小重试」只在这时写。 - mode / last：换纯文本问的那一次——拆不开的空壳、结构化输出回了说明文字（再不行就记没写出来）。 - optional：超时拆出来的块，轮到它时已经止损就不问了（_ask）。""" + budget：这一段（原始段）还能调用几次。chunk：这是哪一块（块进度）。halved：这几块是真拆小（_split）出来的—— + 原因里「已拆小重试」只在这时写。mode / last：换纯文本问的那一次——拆不开的空壳、结构化输出回了说明文字（再不行 + 就记没写出来）。optional：拆出来的块（值是拆它的原因），轮到它时已经止损就不问了（_ask）。 + + 止损期间（_halted）每块只问一次：不拆小（_split）、不补问漏掉的列、截断后不再补剩下的。""" reply = await self._ask(pieces, budget, mode, optional=optional) if reply.capped: self._fail(pieces, self._capped(halved)) return if reply.skipped: - note = self._halt_note("timeout") - self._fail(pieces, lambda p: self._reason("timeout", split=True, piece=p, halted=note)) + note = self._halt_note() + self._fail(pieces, lambda p: self._reason(optional or "hollow", split=True, piece=p, halted=note)) return if reply.timeout: - # 先记下再报用量：报用量要写库、会让出事件循环，等名额的块醒来时要看到这一次 - if sum(len(p.columns) for p in pieces) <= self._floor(): - self.slow_streak += 1 - if self.slow_streak == max(1, int(C.MODEL_GIVE_UP_AFTER)): - logger.warning("起草数据目录：拆到每段 %d 列仍接连超时 %d 次，后面超时的不再拆小", - self._floor(), self.slow_streak) await self._report(reply, wrote=False) - await self._split(pieces, depth, "timeout", followup, budget, halved) + await self._split(pieces, depth, "timeout", followup, budget, halved, chunk) return if reply.error is not None: - await self._report(reply, wrote=False) - if reply.prose and not last: - await self._prose_to_text(pieces, depth, followup, budget, halved) + # 结构化输出回了说明文字、这一块马上改走纯文本再问：只管通信的重来，不进止损的窗口 + redo = reply.prose and not last + await self._report(reply, wrote=False, halting=not redo) + if redo: + await self._prose_to_text(pieces, depth, followup, budget, halved, chunk) return self._fail(pieces, _Why(reply.error, split=halved)) return @@ -1008,28 +1140,36 @@ class _Run: return kind = "exhausted" if reply.exhausted else "truncated" if not anything: - await self._split(rest, depth, kind, followup, budget, halved) - elif depth < int(C.MODEL_SPLIT_DEPTH): + await self._split(rest, depth, kind, followup, budget, halved, chunk) + elif depth < int(C.MODEL_SPLIT_DEPTH) and not self._halted(): # 捡回了一部分：只补剩下的（不是拆小） - await self._segment(rest, depth + 1, followup, budget, halved=halved) + await self._segment(rest, depth + 1, followup, budget, chunk=chunk, halved=halved) else: - self._fail(rest, self._reason(kind, split=halved)) + note = self._halt_note() if depth < int(C.MODEL_SPLIT_DEPTH) else "" + self._fail(rest, self._reason(kind, split=halved, halted=note)) return if not anything and not followup: if last: halted = self._stopped(pieces, depth) self._fail(pieces, lambda p: self._reason("hollow", split=halved, piece=p, halted=halted)) return - await self._split(pieces, depth, "hollow", followup, budget, halved) + await self._split(pieces, depth, "hollow", followup, budget, halved, chunk) return if not rest: return if followup: - await self._after_followup(rest, depth, budget, halved, last=last) + await self._after_followup(rest, depth, budget, halved, chunk, last=last) return - await self._segment(rest, depth, True, budget, halved=halved) + if self._halted(): + # 止损期间每块只问一次：漏了的不补问 + note = self._halt_note() + self._fail(rest, lambda p: _Why(REASON_OMITTED, split=halved, halted=note) + if self.tables[p.table].has_any() else self._reason("hollow", split=halved, piece=p, + halted=note)) + return + await self._segment(rest, depth, True, budget, chunk=chunk, halved=halved) - async def _after_followup(self, rest: list[_Piece], depth: int, budget: _Budget, halved: bool, *, + async def _after_followup(self, rest: list[_Piece], depth: int, budget: _Budget, halved: bool, chunk: _Chunk, *, last: bool) -> None: """补问过仍然没给的部分（补过就不再补）。 @@ -1047,16 +1187,17 @@ class _Run: halted = self._stopped(empty, depth) self._fail(empty, lambda p: self._reason("hollow", split=halved, piece=p, halted=halted)) return - await self._split(empty, depth, "hollow", False, budget, halved) + await self._split(empty, depth, "hollow", False, budget, halved, chunk) async def _prose_to_text(self, pieces: list[_Piece], depth: int, followup: bool, budget: _Budget, - halved: bool) -> None: + halved: bool, chunk: _Chunk) -> None: """结构化输出回了一段说明文字：网关接受了结构化输出的参数却没照做。这次起草后面都走纯文本（另带格式说明）， - 这一段换纯文本问一次。纯文本额度用完了还能照常拆小（拆出来的半段也走纯文本，不再先问一遍结构化输出）。""" + 这一段换纯文本问一次（止损期间也问：这是通信方式的问题，不是模型写不出来）。纯文本额度用完了还能照常拆小 + （拆出来的半段也走纯文本，不再先问一遍结构化输出）。""" if self.mode != "text": self.mode = "text" logger.info("起草数据目录：结构化输出回了说明文字，后面改走纯文本") - await self._segment(pieces, depth, followup, budget, halved=halved, mode="text", last=True) + await self._segment(pieces, depth, followup, budget, chunk=chunk, halved=halved, mode="text", last=True) def _stopped(self, pieces: list[_Piece], depth: int) -> str: """这几段本来还能拆小、因为止损才没拆时，止损的说明；不是的话是空串。""" @@ -1065,36 +1206,57 @@ class _Run: return "" async def _split(self, pieces: list[_Piece], depth: int, kind: str, followup: bool, budget: _Budget, - halved: bool) -> None: + halved: bool, chunk: _Chunk) -> None: """拆成两半再问（额度用完、超时、只给空壳、截断后一项都没捡回来）。 - 同一张表按列拆、前一半带着表级的项：先写前一半，后一半拿它做背景。超时的不等：一次超时就是一个时限，40 → 20 - → 10 一层一层串着问要好几个时限，两半同时问；拆出来的块轮到它时已经止损的不再问（optional）。 - - 拆不开的空壳再问一次：结构化输出给的换纯文本，已经走纯文本的原样再问。止损时本来能拆的不拆：结构化输出给的 - 照旧换纯文本问一次（看看是不是结构化输出的问题），已经走纯文本的不再问。 + → 10 一层一层串着问要好几个时限，两半同时问。拆出来的两半是新的块（单张表多一块）；轮到它时已经止损的不再问 + （optional）。 + - 拆不开的空壳再问一次：结构化输出给的换纯文本，已经走纯文本的原样再问。 + - 止损期间（_halted）不拆小、每块只问一次：本来能拆的不拆；空壳不再原样再问，只有结构化输出给的、这次起草还没 + 换纯文本问过的照旧问一次（看看是不是结构化输出的问题）。 - 别的原因拆不开、止损了，就记没写出来。""" halves = _halves(pieces) if depth < int(C.MODEL_SPLIT_DEPTH) else None - halted = self._halt_note(kind) if halves is not None else "" + halted = self._halted() if halves is not None and not halted: first, second = halves - timeout = kind == "timeout" - if (not timeout and len(first) == len(second) == 1 and first[0].table == second[0].table + await self._split_chunk(chunk, pieces) + if (kind != "timeout" and len(first) == len(second) == 1 and first[0].table == second[0].table and first[0].table_level): - await self._segment(first, depth + 1, followup, budget, halved=True) - await self._segment(second, depth + 1, followup, budget, halved=True) + await self._half(first, depth + 1, followup, budget, kind) + await self._half(second, depth + 1, followup, budget, kind) return - results = await asyncio.gather( - self._segment(first, depth + 1, followup, budget, halved=True, optional=timeout), - self._segment(second, depth + 1, followup, budget, halved=True, optional=timeout), - return_exceptions=True) + results = await asyncio.gather(self._half(first, depth + 1, followup, budget, kind), + self._half(second, depth + 1, followup, budget, kind), + return_exceptions=True) for result in results: if isinstance(result, BaseException): raise result return - if kind == "hollow" and (not halted or self.mode == "structured"): - await self._segment(pieces, depth, followup, budget, halved=halved, mode="text", last=True) + if kind == "hollow" and (not halted or (self.mode == "structured" and not self.text_probed)): + self.text_probed = True + await self._segment(pieces, depth, followup, budget, chunk=chunk, halved=halved, mode="text", last=True) return - self._fail(pieces, lambda p: self._reason(kind, split=halved, piece=p, halted=halted)) + note = self._halt_note() if halted and (halves is not None or kind == "hollow") else "" + self._fail(pieces, lambda p: self._reason(kind, split=halved, piece=p, halted=note)) + + async def _half(self, pieces: list[_Piece], depth: int, followup: bool, budget: _Budget, kind: str) -> None: + """拆出来的一半：新的一块，问完算结束一块。""" + chunk = _Chunk(tuple(dict.fromkeys(p.table for p in pieces))) + try: + await self._segment(pieces, depth, followup, budget, chunk=chunk, halved=True, optional=kind) + except asyncio.CancelledError: + raise + except Exception: + await self._chunk_done(chunk) # 出错也算问完（_guarded 记没写出来），块进度对得上 + raise + await self._chunk_done(chunk) + + async def _split_chunk(self, chunk: _Chunk, pieces: list[_Piece]) -> None: + """这一块里的这几张表拆给了新的块：单张表按列拆成两半，多一块；多表的段按表拆，各张表仍是一块。""" + chunk.split.update(dict.fromkeys(p.table for p in pieces)) + if len(pieces) == 1: + await self._more_chunks(pieces[0].table, 1) def _remainder(self, pieces: list[_Piece], gains: list[_Gain], *, truncated: bool, followup: bool) -> list[_Piece]: @@ -1118,13 +1280,14 @@ class _Run: # ---------------------------------------------------------------- 调用 async def _ask(self, pieces: list[_Piece], budget: _Budget, mode: str | None = None, *, - optional: bool = False) -> _Reply: + optional: str | None = None) -> _Reply: """调一次模型（按原因重来：额度被拒、结构化输出不支持、网络错误……），交回读好的回复。 - 这一段的调用次数用完了交回 capped（不调用）。optional（超时拆出来的块）：拿到名额时已经止损，交回 skipped（不调用， - 次数退回）——等名额的时候前面的块接连超时，再问也是一个时限。超时不原样重试：交回 timeout，由调用方当作这一段 - 太大、拆小再问。重来之前的那次调用在这里就报给进度回调；交回的那次由调用方读完内容再报（写没写出来要看校验之后 - 的内容）。""" + 这一段的调用次数用完了交回 capped（不调用）。optional（拆出来的块）：拿到名额时已经止损，交回 skipped（不调用， + 次数退回）——等名额的时候前面的调用多数没写出来，不再拆小重试。超时不原样重试：交回 timeout，由调用方当作这一段 + 太大、拆小再问。重来之前的那次调用在这里就报给进度回调（记进用量和提醒，不进止损的窗口：halting=False）；交回的 + 那次由调用方读完内容再报（写没写出来要看校验之后的内容）。服务方嫌额度太大、结构化输出不支持、网络错误的重来止损 + 期间照旧（只管通信，不是模型写不出来）。""" mode = mode or self.mode retried = 0 while True: @@ -1135,7 +1298,7 @@ class _Run: try: # 先占这次起草自己的名额，再去排全进程的：一次起草排在全进程名额上的调用最多 MODEL_CONCURRENCY 个 async with self.gate, model_gate(): - if optional and self._halt_note("timeout"): + if optional and self._halted(): budget.left += 1 return _Reply(skipped=True) await self._hook(self.hooks.call_started, [p.table for p in pieces]) @@ -1151,16 +1314,16 @@ class _Run: if kind == "timeout": return _Reply(timeout=True, usage=(0, 0)) if kind == "budget" and (getattr(self.model, "budget", None) != used or self._shrink()): - await self._called((0, 0), "error") + await self._called((0, 0), "error", halting=False) continue if kind == "unsupported" and mode == "structured": if self.mode == "structured": logger.info("起草数据目录：结构化输出用不了（%s），后面改走纯文本", str(e)[:200]) self.mode = mode = "text" - await self._called((0, 0), "error") + await self._called((0, 0), "error", halting=False) continue if kind == "transient" and retried < int(C.MODEL_RETRIES): - await self._called((0, 0), "error") + await self._called((0, 0), "error", halting=False) delays = tuple(C.MODEL_RETRY_DELAYS_S) or (0.0,) await _sleep(delays[min(retried, len(delays) - 1)]) retried += 1 @@ -1212,8 +1375,9 @@ class _Run: # ---------------------------------------------------------------- 每次调用之后：用量、段大小、提醒 - async def _report(self, reply: _Reply, *, wrote: bool) -> None: - """交回的那次调用读完内容之后报给进度回调。""" + async def _report(self, reply: _Reply, *, wrote: bool, halting: bool = True) -> None: + """交回的那次调用读完内容之后报给进度回调。halting：这次是不是这一块这次调用的最终结局（进不进止损的窗口，见 + _called）。""" if reply.usage is None: return if reply.timeout: @@ -1224,14 +1388,26 @@ class _Run: failure = None else: failure = "exhausted" if reply.exhausted else "hollow" - await self._called(reply.usage, failure, strike=reply.timeout or reply.exhausted) + await self._called(reply.usage, failure, strike=reply.timeout or reply.exhausted, halting=halting) - async def _called(self, usage: tuple[int, int], failure: str | None, *, strike: bool = False) -> None: + async def _called(self, usage: tuple[int, int], failure: str | None, *, strike: bool = False, + halting: bool = True) -> None: """记下一次调用：额度用完或超时（strike）累计 MODEL_SHRINK_AFTER 次，新开始的段列数上限减半；最近几次调用多数 - 没写出来时提醒，少了清掉。写出来了（failure 为 None）就清掉接连超时的计数。报给进度回调。""" + 没写出来时提醒，少了清掉；止损按最近 MODEL_HALT_WINDOW 次调用的比例（_halted）。报给进度回调。 + + halting=False：只管通信的重来之前的那次（服务方嫌额度太大、结构化输出不支持或回了说明文字、429 / 5xx 退避重试）。 + 照旧记进用量和提醒，不进止损的窗口：它不花额度，也说明不了模型写不出来；止损只看每次调用的最终结局。""" self.recent.append(failure) - if failure is None: - self.slow_streak = 0 + if halting: + self.window.append(failure is not None) + halted = self._halted() + if halted != self.was_halted: + self.was_halted = halted + if halted: + logger.warning("起草数据目录：最近 %d 次调用有 %d 次没写出来，之后的段不再拆小、每块只问一次", + len(self.window), sum(self.window)) + else: + logger.info("起草数据目录：最近的调用多数写出来了，恢复拆小重试") if strike: self.strikes += 1 width = max(1, int(C.MODEL_BATCH_COLUMNS)) @@ -1280,7 +1456,14 @@ class _Run: gain = _Gain() for entry in by_table.get(piece.table, []): gain.named = st.named = True - notes = _restrict(C._from_model(entry, st.meta, system=self.system), piece) + dropped: list[tuple[str, str]] = [] + drafted = C._from_model(entry, st.meta, system=self.system, source_kind=self.kind, table=st.name, + dropped=dropped) + if piece.table_level and "valid_filter" not in st.notes: + # 只在请它写表级的项时记：只写几列的那几段里多给的表级的项本来就不要 + for why, condition in dropped: + self._log(st, why, condition) + notes = _restrict(drafted, piece) cols = entry.get("columns") if isinstance(entry.get("columns"), list) else [] gain.answered |= {c.get("name") for c in cols if isinstance(c, dict) and c.get("name") in piece.columns} slots = C._slots(notes) @@ -1344,21 +1527,33 @@ class _Run: failed_table = _TABLE in st.failed and not st.got_table total = len(st.parts) or 1 part_of = st.part_of() + progress = {"parts_written": len(st.written_parts), "chunks_total": st.chunks_total, + "chunks_done": st.chunks_done, "log": list(dict.fromkeys(st.log))[:LOG_MAX]} + abandoned = self._abandoned_text(st) if not items: - failures = [(why, part_of.get(key, 1)) for key, why in st.failed.items()] + failures = [(why, part_of.get(key, 1)) for key, why in st.failed.items() if why.base != _SKIPPED] reason = _join(failures) if failures else (C.MODEL_EMPTY_REASON if st.named else REASON_MISSING) - if failures and len(st.parts) > 1: + if abandoned: + reason = f"{reason}；{abandoned}" + if st.failed and len(st.parts) > 1: # 宽表一项都没写出来：同样逐段写上列的范围 said = self._parts_said(st, failed_cols, failed_table) every = "都" if len(said) == len(st.parts) else "" reason = f"这张表 {len(st.columns)} 列，分 {len(st.parts)} 段起草，{'、'.join(said)}{every}没写出来：{reason}" - return ModelOutcome("failed", {}, 0, reason, total, st.parts_done) + return ModelOutcome("failed", {}, 0, reason, total, st.parts_done, **progress) notes = copy.deepcopy(st.notes) blank = self._blank(st, set(failed_cols)) if failed_cols or failed_table: - return ModelOutcome("partial", notes, items, self._partial_text(st, failed_cols, failed_table), total, - st.parts_done, blank) - return ModelOutcome("ok", notes, items, None, total, st.parts_done, blank) + return ModelOutcome("partial", notes, items, self._partial_text(st, failed_cols, failed_table, abandoned), + total, st.parts_done, blank, **progress) + return ModelOutcome("ok", notes, items, None, total, st.parts_done, blank, **progress) + + @staticmethod + def _abandoned_text(st: _Table) -> str: + """这张表接连几段没写出来、剩下的段跳过了：说清跳过了几段、怎么办。没跳过是空串。""" + if not st.skipped_parts: + return "" + return REASON_ABANDONED.format(k=max(1, int(C.MODEL_TABLE_GIVE_UP_AFTER)), n=len(st.skipped_parts)) def _blank(self, st: _Table, failed: set[str]) -> list[str]: """模型拿不准、留空了的列：请它写了（不在没写出来的里面），中文名和含义却都没给（或者给的都没通过校验）。 @@ -1395,11 +1590,14 @@ class _Run: return said + [f"第 {k} 段（{_columns_text(st.columns, cols, count=True)}）" for k, cols in sorted(bad.items())] @classmethod - def _partial_text(cls, st: _Table, failed_cols: list[str], failed_table: bool) -> str: - """只写出一部分时说清哪部分没写出来（第几段、哪几列）、为什么、怎么办。""" + def _partial_text(cls, st: _Table, failed_cols: list[str], failed_table: bool, abandoned: str = "") -> str: + """只写出一部分时说清哪部分没写出来（第几段、哪几列）、为什么、怎么办。剩下的段跳过了的，跳过的说明里已经说了 + 怎么办（换一个起草模型后只重试这张表），不再另说「可以只重试这张表」。""" part_of = st.part_of() keys = ([_TABLE] if failed_table else []) + failed_cols - reason = _join((st.failed[k], part_of.get(k, 1)) for k in keys) + reason = _join((st.failed[k], part_of.get(k, 1)) for k in keys if st.failed[k].base != _SKIPPED) + if abandoned: + reason = f"{reason}；{abandoned}" if reason else abandoned n = len(st.columns) if len(st.parts) > 1: said = cls._parts_said(st, failed_cols, failed_table) @@ -1411,7 +1609,7 @@ class _Run: head = f"这张表 {n} 列，有 {len(failed_cols)} 列（{_columns_text(st.columns, failed_cols, count=False)}）没写出来" else: head = f"这张表 {n} 列，表级的项没写出来" - return f"{head}：{reason}。可以只重试这张表" + return f"{head}：{reason}" if abandoned else f"{head}：{reason}。可以只重试这张表" async def _part_done(self, name: str) -> None: self.tables[name].parts_done += 1 @@ -1424,6 +1622,65 @@ class _Run: st.finished = True await self._hook(self.hooks.table_done, name, self._outcome(st)) + async def _part_written(self, name: str) -> None: + """宽表写出了一段、后面还有段没结束：把这张表到这时拿到的全部内容交给调用方并入目录（停止、服务重启时不丢）。""" + st = self.tables[name] + await self._hook(self.hooks.part_written, name, PartWrite(copy.deepcopy(st.notes), len(st.written_parts))) + + # ---------------------------------------------------------------- 防错事实：核对有效行过滤 + + async def _vet(self, st: _Table) -> None: + """表级那段写完，核对模型提议的有效行过滤（只核对一次）。 + + - 目录里人工确认、驳回过的有效行过滤：并入时不会采用这次的提议，不必再查库； + - 没开数据剖析：照收，note 写「没有核对排除比例（数据源没有开数据剖析）」； + - 开了：经剖析的只读守卫数排除比例（catalog_profile.check_valid_filter）。排除 0% 或 100% 的不收（这张表的日志 + 记一句，不算失败）；收下的 note 在依据后面写上剖析核对的结果，或者没有核对的原因。""" + if st.vetted: + return + st.vetted = True + item = st.notes.get("valid_filter") + if not isinstance(item, dict): + return + held = (st.existing or {}).get("valid_filter") + if isinstance(held, Mapping) and held.get("status") in ("confirmed", "rejected"): + return + from app.data import catalog_profile + + if not self.profiling.enabled: + found = catalog_profile.FilterCheck(True, catalog_profile.FILTER_UNCHECKED.format( + why=catalog_profile.FILTER_NOT_ENABLED)) + else: + try: + found = await catalog_profile.check_valid_filter(self.source, st.name, dict(st.meta), + str(item.get("value") or ""), settings=self.profiling) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 - 核对是附加的：出了意外照收，写明没有核对 + from app.core.errors import describe_exception, first_line + + logger.exception("起草数据目录：核对表 %s 的有效行过滤时出错", st.name) + found = catalog_profile.FilterCheck(True, catalog_profile.FILTER_UNCHECKED.format( + why=f"核对时出错：{first_line(e) or describe_exception(e)}")) + if not found.keep: + logger.info("起草数据目录：表 %s 的有效行过滤 %s 没有收下：%s", st.name, item.get("value"), found.dropped) + st.notes.pop("valid_filter", None) + why = found.dropped or found.note + if found.rows is not None: + why = LOG_FILTER_COUNTED.format(label=C.TABLE_FIELD_LABEL["valid_filter"], what=found.dropped, + rows=f"{found.rows:,}") + self._log(st, why, str(item.get("value") or "")) + return + item["note"] = _sentences(item.get("note"), found.note) + + @staticmethod + def _log(st: _Table, why: str, condition: str) -> None: + """这张表的日志记一句：模型提议的有效记录条件没有收下（不算失败）。""" + condition = " ".join(condition.split()) + if len(condition) > _LOG_CONDITION_MAX: + condition = condition[:_LOG_CONDITION_MAX] + "…" + st.log.append(LOG_FILTER_DROPPED.format(why=why, condition=condition)) + @staticmethod async def _hook(fn: Callable[..., Awaitable[None]] | None, *args: Any) -> None: if fn is None: @@ -1502,7 +1759,12 @@ class _Run: for group in meta.get("unique") or []: if isinstance(group, list) and group: lines.append(labels["unique"].format(cols="、".join(group))) - confirmed = self._confirmed_lines(st, piece) + shown = self._confirmed_for(st, piece) + codes = self._codes_lines(st, piece, shown) + if codes: + lines.append(labels["codes"]) + lines += codes + confirmed = C._notes_lines(shown, meta=st.meta, system_notes=False) if confirmed: lines.append(labels["confirmed"]) lines += confirmed @@ -1522,19 +1784,48 @@ class _Run: line = labels["column"].format(name=col["name"], type=col.get("type") or "") return line + ("，" + "，".join(marks) if marks else "") - @staticmethod - def _confirmed_lines(st: _Table, piece: _Piece) -> list[str]: - """已确认、有确证的项（以此为准的背景）：整表全给；其余给表级的项和这一段的列，关系只在要写表级的项时给。""" - confirmed = C._confirmed_only(st.existing) - if not st.whole(piece): - wanted = set(piece.columns) - kept = {key: item for key, item in C._slots(confirmed).items() - if key[0] == "t" or (key[0] == "c" and key[1] in wanted) or (key[0] == "r" and piece.table_level)} - confirmed = C._assemble(kept) - return C._notes_lines(confirmed, meta=st.meta, system_notes=False) + def _confirmed_for(self, st: _Table, piece: _Piece) -> dict[str, Any]: + """已确认、有确证的项（以此为准的背景）：整表全给；其余给表级的项和这一段的列，关系只在要写表级的项时给。 + + 遮罩的列上的码值不给，人工确认过的也一样：码值就是这一列的取值（同 _codes_lines，也同表助手的背景、导入目录的 + 规矩）。以前只有新加的码值背景挡住了，这里照给。这一列别的确认过的项（中文名、含义……）照给。""" + whole = st.whole(piece) + wanted = set(piece.columns) + kept = {key: item for key, item in C._slots(C._confirmed_only(st.existing)).items() + if not (key[0] == "c" and key[2] == "codes" and key[1].lower() in self.masked) + and (whole or key[0] == "t" or (key[0] == "c" and key[1] in wanted) + or (key[0] == "r" and piece.table_level))} + return C._assemble(kept) + + def _codes_lines(self, st: _Table, piece: _Piece, shown: Mapping[str, Any]) -> list[str]: + """开了数据剖析的源：目录里记下的码值（剖析取过的取值和已知含义）交给模型，判断有效行过滤、列的含义用。要写表级 + 的项的给全部列的，只写几列的给这几列的；遮罩的列、驳回的码值不给；「已确认的目录项」里已经有的不重复。""" + if not self.profiling.enabled: + return [] + labels = C.CATALOG_DRAFT_PROMPT_LABELS + wanted = set(st.columns) if piece.table_level else set(piece.columns) + known = C.visible_notes(st.existing).get("columns") or {} + given = (shown.get("columns") or {}) if isinstance(shown.get("columns"), Mapping) else {} + out: list[str] = [] + for col in st.columns: + item = (known.get(col) or {}).get("codes") + if col not in wanted or col.lower() in self.masked or not isinstance(item, Mapping): + continue + if "codes" in (given.get(col) or {}) or not isinstance(item.get("value"), Mapping) or not item["value"]: + continue + out.append(labels["code"].format(name=col, codes=C._codes_text(item["value"]))) + return out -__all__ = ["BUDGET_WORDS", "Background", "CallReport", "FAILURE_LABEL", "Hooks", "ModelOutcome", "NOTICE", "REASON_CALL", - "REASON_CAPPED", "REASON_EXHAUSTED", "REASON_FORMAT", "REASON_HALTED", "REASON_HOLLOW", "REASON_MISSING", - "REASON_RETRIED", "REASON_SLOW", "REASON_SPLIT", "REASON_TIMEOUT", "REASON_TRUNCATED", "REASON_UNANSWERED", - "SEG", "classify_failure", "model_gate", "parts_for", "run_model_draft"] +def _sentences(*parts: Any) -> str: + """几句话连成一段：各句去掉句末的句号，用句号隔开（「依据：……。剖析核对：……」）。""" + texts = [str(p).strip().rstrip("。") for p in parts if isinstance(p, str) and p.strip()] + return "。".join(texts) + + +__all__ = ["BUDGET_WORDS", "Background", "CallReport", "FAILURE_LABEL", "Hooks", "LOG_FILTER_COUNTED", + "LOG_FILTER_DROPPED", "LOG_MAX", "ModelOutcome", "NOTICE", "PartWrite", + "REASON_ABANDONED", "REASON_CALL", "REASON_CAPPED", "REASON_EXHAUSTED", "REASON_FORMAT", "REASON_HALTED", + "REASON_HOLLOW", "REASON_MISSING", "REASON_OMITTED", "REASON_RETRIED", "REASON_SPLIT", "REASON_SPLIT_PARTS", + "REASON_TIMEOUT", "REASON_TRUNCATED", "REASON_UNANSWERED", "SEG", "classify_failure", "model_gate", + "parts_for", "run_model_draft"] diff --git a/backend/app/data/catalog_jobs.py b/backend/app/data/catalog_jobs.py index d708ed50..d63c0313 100644 --- a/backend/app/data/catalog_jobs.py +++ b/backend/app/data/catalog_jobs.py @@ -6,11 +6,15 @@ - **建任务**（create_job）：同一个数据源同时只有一个排队或进行中的任务（DraftBusy；部分唯一索引兜底）。不指定表时 按使用次数取前 20 张；表结构里没有的表照样建进任务、状态 skipped。起草要求（数据源级）和本次说明随任务记下快照。 - **在后台起草**（manager.start）：进程内的 asyncio 任务，引擎是 data/catalog_draft.py（分段、按失败原因处理）。 - 每张表一结束就并入目录（catalog.write_draft，规则和同步起草一样：人工确认、驳回过的不动，模型这次没给的项不删 - 以前起草的，乐观锁冲突重读重并）；逐表的状态、到第几段、计数、模型留空的列，任务的用量（含超时、没写出来的调用 - 次数）、当前的列数上限、提醒随时落库，接口读的是库里的状态。 + 宽表每写出一段就并入目录（write_part：这一次合并不删任何东西），整张表结束（不管写全没写全）再按完整覆盖的规则 + 收尾一次（catalog.write_draft，规则和同步起草一样：人工确认、驳回过的不动，模型这次没给的项不删以前起草的，只删 + 已经不在表结构里的列的模型项，乐观锁冲突重读重并）；新增、更新、删除的计数是各次写入的合计。逐表的状态、到第几段、 + 写进目录的段数、块进度、计数、模型留空的列，任务的用量（含超时、没写出来的调用次数）、当前的列数上限、提醒随时落库， + 接口读的是库里的状态。 - **停止**（cancel）：排队的直接 cancelled；进行中的停掉正在进行的模型调用，没开始和没写完的表记 cancelled，已经写进 - 目录的不回滚，任务最后是 cancelled。 + 目录的（含宽表已经写出来的段）不回滚，任务最后是 cancelled。写过段的表 model_error 说清写进了几段、剩几段没起草。 +- **和同步起草互斥**（sync_draft）：同步起草（POST /catalog/draft）进行期间，同一个源建任务、重试、导入（ensure_idle） + 都是 DraftBusy；有排队或进行中的任务时同步起草也是。进程内登记（单进程部署）。 - **重试**（retry_tables + create_job）：新任务，只含原任务里 failed、partial、cancelled 的表；用不用模型、本次说明沿用， 起草要求取数据源当前的。 - **服务重启**：启动时（startup）把排队、进行中的任务标成 interrupted，没结束的表记 cancelled、写明「服务重启，起草 @@ -26,11 +30,12 @@ from __future__ import annotations import asyncio +import contextlib import logging -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from typing import Any -from sqlalchemy import func, select, update +from sqlalchemy import delete, func, or_, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession @@ -62,6 +67,10 @@ BLANK_SAMPLE = 5 STOP_GRACE_S = 10.0 INTERRUPTED_ERROR = "服务重启，起草中断" +#: 停止、中断时写过段的表（宽表每写出一段就并入目录）：写进了几段、剩几段没起草 +STOPPED_WRITTEN = "已停止：已写进目录 {n} 段，剩下 {m} 段没起草" +INTERRUPTED_WRITTEN = "已中断：已写进目录 {n} 段，剩下 {m} 段没起草" +_WRITTEN = {"cancelled": STOPPED_WRITTEN, "interrupted": INTERRUPTED_WRITTEN} UNFINISHED_ERROR = "起草结束时这张表还没有写完，请重试这张表" WRITE_ERROR = "写入数据目录时出错：{line}" JOB_ERROR = "起草任务出错：{line}" @@ -162,6 +171,11 @@ def task_dict(task: CatalogDraftTask) -> dict[str, Any]: "columns": task.column_count, "parts_total": task.parts_total, "parts_done": task.parts_done, + # 宽表每写出一段就并入目录：已经写进目录的段数（停止、服务重启时这些段留着） + "parts_written": task.parts_written or 0, + # 块进度：这张表实际分成的块数和已结束的块数（段被缩小、拆成两半后会变多）；不用模型时为 null + "chunks_total": task.chunks_total, + "chunks_done": task.chunks_done, "added": task.added, "updated": task.updated, "removed": task.removed, @@ -170,6 +184,8 @@ def task_dict(task: CatalogDraftTask) -> dict[str, Any]: # 模型拿不准、留空了的列（中文名和含义都没给）：几列、最多 5 个列名（按表结构的顺序）。表的状态不因此变 "blank_columns": task.blank_columns or 0, "blank_sample": list(task.blank_sample or []), + # 起草时记下的几句话，不算失败（模型提议的有效记录条件没有收下的原因：没通过校验、剖析核对排除 0% 或 100%） + "log": list(task.log or []), "error": task.error, "model_error": task.model_error, "started_at": _iso(task.started_at), @@ -237,21 +253,39 @@ async def job_brief(session: AsyncSession, job: CatalogDraftJob | None) -> dict[ async def last_drafts(session: AsyncSession, source_id: str) -> dict[str, dict[str, Any]]: - """每张表最近一次起草任务里的那一项：{表名: {status, job_id, finished_at, error, model_error, blank_columns}}。 - 任务还在进行，就是进行中的状态。只看最近 LAST_DRAFT_JOBS 个任务；每张表取哪一项在库里挑好（清单页起草时会反复取）。""" + """每张表最近一次起草任务里的那一项：{表名: {status, job_id, finished_at, error, model_error, blank_columns, + parts_written, blank_columns_from}}。任务还在进行，就是进行中的状态。只看最近 LAST_DRAFT_JOBS 个任务；每张表取哪一项 + 在库里挑好（清单页起草时会反复取）。 + + blank_columns_from：最近一次结束于 ok / partial 的用模型起草记下的留空列数（数字），没有是 null。之后的起草停止了、 + 中断了、而且没有写进新内容（parts_written 为 0）时照样带着，界面据此继续提示留空的列；之后有起草写进了新内容（停止 + 前已经写了几段、或者正在写）就是 null：以前的留空列数可能已经不对了。不用模型的起草不改模型留空的列，不算。""" recent = (select(CatalogDraftJob.id).where(CatalogDraftJob.source_id == source_id) .order_by(CatalogDraftJob.created_at.desc()).limit(LAST_DRAFT_JOBS)).scalar_subquery() - rank = func.row_number().over(partition_by=CatalogDraftTask.table_name, - order_by=(CatalogDraftJob.created_at.desc(), CatalogDraftTask.seq)).label("rank") + + def ranked() -> Any: + return func.row_number().over(partition_by=CatalogDraftTask.table_name, + order_by=(CatalogDraftJob.created_at.desc(), CatalogDraftTask.seq)).label("rank") + inner = (select(CatalogDraftTask.table_name, CatalogDraftTask.status, CatalogDraftTask.job_id, CatalogDraftTask.finished_at, CatalogDraftTask.error, CatalogDraftTask.model_error, - CatalogDraftTask.blank_columns, rank) + CatalogDraftTask.blank_columns, CatalogDraftTask.parts_written, ranked()) .join(CatalogDraftJob, CatalogDraftJob.id == CatalogDraftTask.job_id) .where(CatalogDraftJob.id.in_(recent))).subquery() rows = (await session.execute(select(inner).where(inner.c.rank == 1))).mappings() + # 每张表最近一次「写进了模型起草的内容」的用模型起草：结束于 ok / partial 的，或者写过段的（停止、中断、进行中） + wrote = (select(CatalogDraftTask.table_name, CatalogDraftTask.status, CatalogDraftTask.blank_columns, ranked()) + .join(CatalogDraftJob, CatalogDraftJob.id == CatalogDraftTask.job_id) + .where(CatalogDraftJob.id.in_(recent), CatalogDraftTask.parts_total.is_not(None), + or_(CatalogDraftTask.status.in_(("ok", "partial")), + func.coalesce(CatalogDraftTask.parts_written, 0) > 0))).subquery() + blank_from = {row["table_name"]: (row["blank_columns"] or 0) if row["status"] in ("ok", "partial") else None + for row in (await session.execute(select(wrote).where(wrote.c.rank == 1))).mappings()} return {row["table_name"]: {"status": row["status"], "job_id": row["job_id"], "finished_at": _iso(row["finished_at"]), "error": row["error"], - "model_error": row["model_error"], "blank_columns": row["blank_columns"] or 0} + "model_error": row["model_error"], "blank_columns": row["blank_columns"] or 0, + "parts_written": row["parts_written"] or 0, + "blank_columns_from": blank_from.get(row["table_name"])} for row in rows} @@ -264,12 +298,43 @@ def busy_message(row: Any) -> str: return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录，请等这次起草结束（或先停止它）再开始新的起草" +def sync_busy_message(row: Any) -> str: + return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录（同步起草），请等这次起草结束再开始新的起草" + + +#: 正在同步起草（POST /catalog/draft）的数据源。进程内登记：服务端是单进程部署，多进程部署时要换成库里的锁 +_SYNC: set[str] = set() + + +def sync_drafting(source_id: str) -> bool: + """这个源正在同步起草。""" + return source_id in _SYNC + + async def ensure_idle(session: AsyncSession, row: Any) -> None: - """这个源没有排队或进行中的任务；有就抛 DraftBusy。""" + """这个源没有在起草：没有排队或进行中的任务，也没有在同步起草。有就抛 DraftBusy。建任务、重试、导入目录之前都调它。""" + if row.id in _SYNC: + raise DraftBusy(sync_busy_message(row)) if await active_job(session, row.id) is not None: raise DraftBusy(busy_message(row)) +@contextlib.asynccontextmanager +async def sync_draft(session: AsyncSession, row: Any) -> AsyncIterator[None]: + """同步起草期间登记这个源：同一个源建任务、重试、导入、再开一次同步起草都抛 DraftBusy；有排队或进行中的任务时 + 这里就抛 DraftBusy。先登记、再查库：查库要让出事件循环，先登记才不会和同时建的任务互相看不见（建任务那边写库之后 + 也再看一次，见 create_job）。""" + if row.id in _SYNC: + raise DraftBusy(sync_busy_message(row)) + _SYNC.add(row.id) + try: + if await active_job(session, row.id) is not None: + raise DraftBusy(busy_message(row)) + yield + finally: + _SYNC.discard(row.id) + + async def check_model(session: AsyncSession) -> None: """起草模型能不能用：不能用抛 catalog.CatalogModelUnavailable（只建模型对象，不发请求）。""" await catalog.resolve_draft_model(session) @@ -299,9 +364,10 @@ async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: li continue seen.add(key) columns = len((all_tables.get(key) or {}).get("columns") or []) + parts = catalog_draft.parts_for(columns) if use_model else None tasks.append(CatalogDraftTask(job_id=job.id, seq=len(tasks), table_name=key, status="pending", - column_count=columns, - parts_total=catalog_draft.parts_for(columns) if use_model else None)) + column_count=columns, parts_total=parts, parts_written=0, + chunks_total=parts, chunks_done=0 if use_model else None)) session.add(job) try: await session.flush() @@ -311,6 +377,12 @@ async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: li # 部分唯一索引：另一个请求同时给这个源建了任务 await session.rollback() raise DraftBusy(busy_message(row)) from e + if row.id in _SYNC: + # 写库的时候同一个源开始了同步起草（它查库时还没看到这个任务）：这个任务不要了 + await session.execute(delete(CatalogDraftTask).where(CatalogDraftTask.job_id == job.id)) + await session.execute(delete(CatalogDraftJob).where(CatalogDraftJob.id == job.id)) + await session.commit() + raise DraftBusy(sync_busy_message(row)) return job, tasks @@ -347,8 +419,21 @@ async def cancel(session: AsyncSession, job: CatalogDraftJob) -> None: async def _close(session: AsyncSession, job_id: str, job_status: str, job_error: str | None, task_status: str, task_error: str | None) -> None: - """收尾：还没结束的表记 task_status，任务（还没结束的话）记 job_status。""" + """收尾：还没结束的表记 task_status，任务（还没结束的话）记 job_status。停止、中断时写过段的表 model_error 说清写进了 + 几段、剩几段没起草（已经写进去的段留在目录里）。""" now = utcnow() + written = _WRITTEN.get(job_status) + if written is not None: + rows = (await session.execute( + select(CatalogDraftTask.id, CatalogDraftTask.parts_written, CatalogDraftTask.parts_total) + .where(CatalogDraftTask.job_id == job_id, CatalogDraftTask.status.in_(_OPEN), + func.coalesce(CatalogDraftTask.parts_written, 0) > 0))).all() + for task_id, n, total in rows: + await session.execute( + update(CatalogDraftTask).where(CatalogDraftTask.id == task_id, CatalogDraftTask.status.in_(_OPEN)) + .values(status=task_status, error=task_error, finished_at=now, + model_error=written.format(n=n, m=max(0, int(total or n) - int(n)))) + .execution_options(synchronize_session=False)) await session.execute( update(CatalogDraftTask).where(CatalogDraftTask.job_id == job_id, CatalogDraftTask.status.in_(_OPEN)) .values(status=task_status, error=task_error, finished_at=now).execution_options(synchronize_session=False)) @@ -389,7 +474,6 @@ class _JobRun: self._started: set[str] = set() self.source: Any = None self.actor: str | None = None - self.at = "" # ---------------------------------------------------------------- 库写入 @@ -468,7 +552,7 @@ class _JobRun: except catalog.CatalogModelUnavailable as e: raise _JobFailed(str(e)) from e existing = {t: e.notes for t, e in (await catalog.read_catalog(session, source.id)).items()} - self.source, self.at = source, catalog.now_iso() + self.source = source all_tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} names = [n for n in pending if isinstance(all_tables.get(n), dict)] gone = [n for n in pending if n not in names] @@ -478,7 +562,8 @@ class _JobRun: if use_model and names: await self._write(self._set_model, model_id) hooks = catalog_draft.Hooks(call_started=self._call_started, call_done=self._call_done, - part_done=self._part_done, table_done=self._table_done) + part_done=self._part_done, table_done=self._table_done, + part_written=self._part_written, chunks_changed=self._chunks_changed) await catalog_draft.run_model_draft(model, source, names, existing=existing, background=background, hooks=hooks) else: @@ -505,6 +590,12 @@ class _JobRun: async def _table_done(self, table: str, outcome: catalog_draft.ModelOutcome | None) -> None: await self._write(self._write_table, table, outcome) + async def _part_written(self, table: str, written: catalog_draft.PartWrite) -> None: + await self._write(self._write_part, table, written) + + async def _chunks_changed(self, table: str, total: int, done: int) -> None: + await self._write(self._set_chunks, table, total, done) + # ---------------------------------------------------------------- 库操作 def _task(self, *where: Any) -> Any: @@ -554,11 +645,41 @@ class _JobRun: .values(parts_done=CatalogDraftTask.parts_done + 1)) await session.commit() + async def _set_chunks(self, session: AsyncSession, table: str, total: int, done: int) -> None: + await session.execute(self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN)) + .values(chunks_total=int(total), chunks_done=int(done))) + await session.commit() + + async def _write_part(self, session: AsyncSession, table: str, written: catalog_draft.PartWrite) -> None: + """宽表写出了一段：马上并入目录（这一次不删任何东西），记下写进去的段数、累加计数。写不进去只记日志：整张表结束 + 时收尾那次还会把它带上。 + + 新增、更新的项的修改时间是这一次写入的时间（每次写入各取一次）：以前整个任务只在开始时取一次，起草了几十分钟的宽表， + 每一段写进去的项都记成任务开始那一刻，比任务的开始时间还早。""" + try: + found = await write_part(session, self.source, table, written.notes, at=catalog.now_iso(), actor=self.actor) + except Exception: # noqa: BLE001 - 这一段写不进去不影响起草 + logger.exception("起草任务 %s 把表 %s 写出来的一段并入数据目录时出错", self.job_id, table) + await session.rollback() + return + if found is None: + logger.warning("起草任务 %s：表 %s 的数据目录正被修改，这一段等整张表写完时再并入", self.job_id, table) + return + stats, version = found + await session.execute( + self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN)) + .values(parts_written=int(written.parts_written), version=version, + added=CatalogDraftTask.added + stats.added, updated=CatalogDraftTask.updated + stats.updated, + removed=CatalogDraftTask.removed + stats.removed)) + await session.commit() + async def _write_table(self, session: AsyncSession, table: str, outcome: catalog_draft.ModelOutcome | None) -> None: - """一张表结束：并入目录、写入，记下这张表的结果。""" + """一张表结束：并入目录、写入（宽表写过的段已经并入过，这里是整张表的收尾），记下这张表的结果。计数加在逐段写入 + 的计数上。修改时间取这一次写入的时间（同 _write_part）：逐段写进去、内容没变的项留着那一段的时间。""" try: - result = await catalog.write_draft(session, self.source, table, outcome, at=self.at, actor=self.actor) + result = await catalog.write_draft(session, self.source, table, outcome, at=catalog.now_iso(), + actor=self.actor) except Exception as e: # noqa: BLE001 - 这一张写不进去不影响别的表 from app.core.errors import describe_exception, first_line @@ -568,13 +689,20 @@ class _JobRun: error=WRITE_ERROR.format(line=first_line(e) or describe_exception(e)), model_error=outcome.error if outcome is not None else None) values: dict[str, Any] = { - "status": result.status, "added": result.added, "updated": result.updated, "removed": result.removed, - "version": result.version, "model_items": result.model_items, "error": result.error, + "status": result.status, "added": CatalogDraftTask.added + result.added, + "updated": CatalogDraftTask.updated + result.updated, "removed": CatalogDraftTask.removed + result.removed, + "model_items": result.model_items, "error": result.error, "model_error": result.model_error, "finished_at": utcnow(), "blank_columns": len(result.blank), "blank_sample": result.blank[:BLANK_SAMPLE] or None, } + if result.version: + values["version"] = result.version if outcome is not None: - values["parts_done"] = outcome.parts_done + values.update(parts_done=outcome.parts_done, chunks_total=outcome.chunks_total, + chunks_done=outcome.chunks_done, log=list(outcome.log) or None) + if not result.error: + # 收尾写进去了：写出内容的段都在目录里了。没写进去（一直撞上别人刚改过）的照库里逐段写进去的段数 + values["parts_written"] = outcome.parts_written await session.execute(self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN)) .values(**values)) await session.commit() @@ -594,6 +722,30 @@ class _JobRun: await session.commit() +async def write_part(session: AsyncSession, source: Any, table: str, notes: dict[str, Any], *, at: str, + actor: str | None) -> tuple[catalog.MergeStats, int] | None: + """宽表写出来的一段并入目录：(计数, 写入后的版本)；撞上别人刚改过、重读重并三次仍冲突返回 None。 + + 数据库注释、外键、命名推断和这张表到这时拿到的模型起草，按 merge_notes 的规则并入（人工确认、驳回过的不动）， + **这一次不删任何东西**（covered 为空）：删掉表结构里已经没有的、这一轮没再得出的，是整张表写完时 catalog.write_draft + 收尾那一次的事。内容没变不写、不升版本。""" + structure = catalog.draft_structure(source, table) + for _attempt in range(3): + entry = await catalog.read_entry(session, source.id, table) + version = entry.version if entry else 0 + merged, stats = catalog.merge_notes(entry.notes if entry else {}, structure.notes, at=at) + merged, more = catalog.merge_notes(merged, notes, at=at) + stats += more + if entry is None and not merged: + return stats, 0 + try: + written = await catalog.write_entry(session, source.id, table, merged, if_version=version, actor=actor) + except catalog.CatalogConflict: + continue + return stats, written.version + return None + + class DraftJobs: """进程内的起草任务（照 app/scheduler 的写法：一个 asyncio 任务一件事，停的时候给宽限）。""" @@ -660,8 +812,8 @@ manager = DraftJobs() __all__ = ["ACTIVE", "BLANK_SAMPLE", "DraftBusy", "DraftFinished", "DraftJobs", "FINISHED", "GUIDANCE_MAX", - "INTERRUPTED_ERROR", "MAX_TABLES", "NOTE_MAX", "NothingToRetry", "RETRYABLE", "TASK_DONE", "TASK_STATUSES", - "TextTooLong", + "INTERRUPTED_ERROR", "INTERRUPTED_WRITTEN", "MAX_TABLES", "NOTE_MAX", "NothingToRetry", "RETRYABLE", + "STOPPED_WRITTEN", "TASK_DONE", "TASK_STATUSES", "TextTooLong", "active_job", "cancel", "check_model", "clean_text", "create_job", "ensure_idle", "guidance_of", "job_brief", "job_dict", "job_out", "last_drafts", "latest_job", "load", "manager", "retry_tables", "save_guidance", - "startup", "tasks_of"] + "startup", "sync_busy_message", "sync_draft", "sync_drafting", "tasks_of", "write_part"] diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 6966b26d..59dd6aa5 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -999,7 +999,8 @@ def _codes_finding(c: _CodesOutcome, value: dict[str, str]) -> dict[str, Any]: def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool: """人工确认过的关系只补覆盖率和基数：覆盖率是测量值，照新的写；基数只在原来没有时补，不改人定的； - 备注只在原来为空、或者是剖析自己写的时候换成新的（只是日期变了不换）。状态、来源不动。返回改没改。""" + 备注只在原来为空、或者是剖析自己写的时候换成新的（只是日期变了不换）。状态、来源、确认时的依据（reason，保存提案时 + 另存，不在备注里）不动。返回改没改。""" changed = False if rel.get("coverage") != o.coverage: rel["coverage"] = o.coverage @@ -1020,9 +1021,12 @@ def _patch_confirmed(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool: def _patch_fk(rel: dict[str, Any], o: _RelationOutcome, at: str) -> bool: """外键关系只补基数和「用数据核实过」：外键结构推的多对一可能其实是一对一，数据说了算。来源、状态、 - 覆盖率都不动；备注同 _patch_confirmed（只是旧备注的日期不同不换）。返回改没改。""" + 覆盖率都不动；备注同 _patch_confirmed（只是旧备注的日期不同不换）。人工确认过的（值不变、保存即确认的外键关系， + 来源仍是 fk）同 _patch_confirmed：不改人定的基数，数据核实出的不一样时也不标「用数据核实过」，测量的说法照常写进 + 备注。返回改没改。""" changed = False - if o.cardinality_checked: + overruled = rel.get("status") == "confirmed" and rel.get("cardinality") not in (None, o.cardinality) + if o.cardinality_checked and not overruled: if rel.get("cardinality") != o.cardinality: rel["cardinality"] = o.cardinality changed = True @@ -1341,8 +1345,129 @@ async def _profile(session: AsyncSession, row: Any, source: Any, tables: list[st return report -__all__ = ["CODES_MAX", "IN_CHUNK", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES", +# ========================================================================== +# 核对模型提议的有效行过滤：排除了多少行 +# ========================================================================== + +#: 有效行过滤核对的说法（写进这一项的 note；前面是模型给的依据） +FILTER_CHECKED = "剖析核对：排除 {share} 的行（共 {rows} 行）" +FILTER_SAMPLED = "剖析核对：按前 {rows} 行统计，排除 {share} 的行" +FILTER_SAMPLED_EDGE = "剖析核对：按前 {rows} 行统计，{what}，请人工核对" +FILTER_UNCHECKED = "没有核对排除比例（{why}）" +FILTER_NOT_ENABLED = "数据源没有开数据剖析" + + +@dataclass(frozen=True) +class FilterCheck: + """核对一条有效行过滤的结果。 + + keep：收不收（排除 0% 或 100% 的不收：前者什么都没筛掉，后者把所有行都筛掉了，多半写错了）；note：写进这一项 note + 的那句（剖析核对的排除比例，或者没有核对的原因）；rows / excluded：数到的行数和其中被排除的行数（没核对是 None）； + sampled：表太大或不知道多大，只数了前若干行；dropped：不收的原因（记日志用）。""" + + keep: bool + note: str + rows: int | None = None + excluded: int | None = None + sampled: bool = False + dropped: str | None = None + + +def excluded_share(excluded: int, rows: int) -> str: + """排除比例写成百分数：留一位小数（整数不带小数）；不到 0.1% 的写「不足 0.1%」，不是全部的不四舍五入成 100%。""" + if rows <= 0: + return "0%" + pct = excluded * 100 / rows + if 0 < pct < 0.1: + return "不足 0.1%" + if 99.9 < pct < 100: + return "超过 99.9%" + return f"{pct:.1f}".rstrip("0").rstrip(".") + "%" + + +def _unchecked(why: str) -> FilterCheck: + return FilterCheck(True, FILTER_UNCHECKED.format(why=why)) + + +def _unsafe_literal(value: str) -> bool: + """拼不进核对查询的字符串取值：查询层用 SQLAlchemy 的 text() 执行，它会把字符串里的「:名字」当成绑定参数； + 反斜杠在 MySQL 上读法不定、守卫会整条拒掉；控制字符不该出现（同 sql_literal 的规矩）。""" + return "\\" in value or bool(_CONTROL.search(value)) or bool(_BIND_LIKE.search(value)) + + +async def check_valid_filter(source: Any, table: str, meta: dict[str, Any] | None, condition: str, *, + settings: ProfileSettings | None = None) -> FilterCheck: + """开了数据剖析的源：数一数这条有效行过滤排除了多少行（模型起草时提议的那条，data/catalog_draft.py 调）。从不抛异常。 + + 经剖析的只读守卫（_Runner：guard.check(readonly=True)、单条时限、查询次数上限）发一条查询：只取条件里用到的列， + 先取前 max(sample_size, max_scan_rows) 行（大表不做整表扫描），再数总行数和不满足条件的行数（条件为 NULL 也算排除， + 和 WHERE 一样）。数到的不满前那么多行，就是整张表。 + + - 排除 0% 或 100% 的不收（keep 为假）；只数了前若干行的照收，写明「请人工核对」； + - 没开剖析、数据源停用、条件里有遮罩的列、视图、取值拼不进查询、查询失败或超时：照收，note 写没有核对的原因。 + """ + from app.data import sqlcheck + + options = getattr(source, "options", None) + settings = settings or profile_settings(options) + if not settings.enabled: + return _unchecked(FILTER_NOT_ENABLED) + if getattr(source, "enabled", True) is False: + return _unchecked("数据源已停用") + kind = getattr(source, "kind", None) + parsed = sqlcheck.parse_valid_filter(condition, meta, kind=kind, table=table) + if parsed.problem: + return FilterCheck(False, parsed.problem, dropped=parsed.problem) + masked = {c.lower() for c in data_engine.masked_columns(options)} + if hit := [c for c in parsed.columns if c.lower() in masked]: + return _unchecked(f"条件里的 {'、'.join(hit)} 在数据源设置中被遮罩，不取数") + if (meta or {}).get("is_view"): + return _unchecked("这张表是视图，剖析可能触发整个视图的计算") + from sqlglot import exp + + if any(n.is_string and _unsafe_literal(str(n.this)) for n in parsed.expression.find_all(exp.Literal)): + return _unchecked("条件里的取值含有冒号或反斜杠，拼不进核对查询") + try: + dialect = SqlDialect(kind) + except ValueError as e: + return _unchecked(str(e)) + cap = max(int(settings.sample_size), int(settings.max_scan_rows)) + # 和存进目录的是同一条（catalog._model_facts 存的就是这个写法）：列换成真名、按方言加引号，别的照原文 + cond = sqlcheck.canonical_valid_filter(parsed, quote=dialect.quote) + cols = ", ".join(dialect.quote(c) for c in parsed.columns) + inner = dialect.limit(f"SELECT {cols} FROM {dialect.table(meta, table)}", cap) + sql = f"SELECT COUNT(*) AS n, SUM(CASE WHEN {cond} THEN 0 ELSE 1 END) AS x FROM ({inner}) s" + run = _Runner(source, settings, dialect) + try: + res = await run(sql) + except _Skip as e: + return _unchecked(e.detail.removesuffix("，已跳过")) + except _Stop as e: + return _unchecked(_STOP_DETAIL[e.reason].format(max_queries=settings.max_queries, + max_total_s=_fmt(settings.max_total_s)).removesuffix("，未检查")) + except SnapshotTampered as e: + return _unchecked(str(e)) + row = res.rows[0] if res.rows else [None, None] + rows, excluded = _as_int(row[0]), _as_int(row[1]) if len(row) > 1 else None + if not rows: + return _unchecked("表里还没有数据") + excluded = excluded or 0 + sampled = rows >= cap + counted = f"{rows:,}" + if excluded in (0, rows): + what = "没有排除任何一行" if excluded == 0 else "所有行都被排除了" + if sampled: + return FilterCheck(True, FILTER_SAMPLED_EDGE.format(rows=counted, what=what), rows, excluded, True) + return FilterCheck(False, FILTER_CHECKED.format(share=excluded_share(excluded, rows), rows=counted), rows, + excluded, False, dropped=what) + share = excluded_share(excluded, rows) + note = (FILTER_SAMPLED if sampled else FILTER_CHECKED).format(share=share, rows=counted) + return FilterCheck(True, note, rows, excluded, sampled) + + +__all__ = ["CODES_MAX", "FILTER_CHECKED", "FILTER_NOT_ENABLED", "FILTER_SAMPLED", "FILTER_UNCHECKED", "FilterCheck", + "IN_CHUNK", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES", "PROFILE_OPTION", "PROFILE_PROBE_TABLES", "ProfileBusy", "ProfileDisabled", "ProfileReport", - "ProfileSettings", "ProfileSkip", "SqlDialect", "TableProfile", "TableSize", "ensure_enabled", - "estimate_size", "profile_catalog", "profile_settings", "profile_settings_problem", "profile_settings_rejection", - "sql_literal"] + "ProfileSettings", "ProfileSkip", "SqlDialect", "TableProfile", "TableSize", "check_valid_filter", + "ensure_enabled", "estimate_size", "excluded_share", "profile_catalog", "profile_settings", + "profile_settings_problem", "profile_settings_rejection", "sql_literal"] diff --git a/backend/app/data/catalog_review.py b/backend/app/data/catalog_review.py new file mode 100644 index 00000000..5e7fba68 --- /dev/null +++ b/backend/app/data/catalog_review.py @@ -0,0 +1,447 @@ +"""数据目录的审阅队列、批量审阅、表被哪些运行用过、审阅进度。 + +**为什么要有它。** 起草（数据库注释、外键、命名推断、数据剖析、模型）一次写出成百上千项，绝大多数停在「推断」：SQL 检查的 +错误级规则只认已确认、有确证的项，推断的项只作提示。逐表点开确认太慢，也看不出先确认哪几项。队列把推断的项摊平，按 +「这张表常不常用 × 这一项错了影响多大 × 这一项从哪来」排好，最该先看的在前面；勾几项一次确认、驳回或撤销审阅。 + +- **队列**（review_queue，GET /catalog/review-queue）：只列推断（proposed）的项。priority = 使用次数权重 × 风险权重 × + 来源权重，从高到低；同分按表名、路径排，结果稳定。 + - 使用次数权重：1 + log2(1 + 使用次数)。使用次数同目录清单（catalog.table_usage，清单的「使用 N 次」和排序、起草的默认 + 范围都按它）：按对数增长，常用表在前，但不至于让常用表上的中文名压过所有表上影响数字的项。 + - 风险权重（RISK_WEIGHT）：有效记录条件、去重规则、业务日期、关联关系最高（直接改变数字），业务主键、度量类型次之 + （计数和能不能加总），码值、粒度、表类型再次，中文名、说明、含义、单位最低。 + - 来源权重（SOURCE_WEIGHT）：模型起草、命名推断是猜的，比数据库注释优先；数据剖析（码值候选、业务日期、没核实的关系） + 居中。 + - 两组（kind）：facts 影响数字的项（有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型）；names 名称和说明 + （中文名、说明、含义、单位、码值、粒度、表类型）。表的「说明」和中文名、含义一样只是说明文字，归 names。 + - 表结构里已经没有的表、列上的项不列（确认了也没有用，目录页上另有清理的提示）。 + - field_label 是这一项给人看的位置：「表的有效记录条件」「列 status 的码值」「指向 members 的关联关系」（item_label， + 和表助手提案、导入冲突里的说法一致）；列级的项另给 column。 + - table_usage 是所在表的使用次数（同清单）。reasons 用中文说为什么排在这里：常用表（被 N 次运行用到）、影响数字的项 + （有效记录条件）、模型起草，未确认……「被 N 次运行用到」数的是运行（同 /usage 的 total）；同一次运行里查了几次、 + 使用次数比运行次数多时两个都写（「被 2 次运行用到，使用 5 次」），和清单、运行列表上的数都对得上。 +- **批量审阅**（review_batch，POST /catalog/review/batch）：逐表按 if_version 乐观锁，同一张表的几条在一次写入里完成， + 一张表冲突、写不进去不影响别的表；路径不对、表不存在逐条报（results 和请求一一对应）。审阅的规则同单项审阅 + （catalog.review_item）。 +- **被哪些运行用过**（table_runs，GET /catalog/{table}/usage）：取法同使用次数（catalog.query_snapshot_tables），按运行 + 去重，新的在前；total 是运行次数。 +- **进度**（review_progress，GET /catalog/review-progress）：常用表（使用次数前 TOP_N 张，和起草对话框「使用次数最多的 + 20 张表」是同一批：catalog.default_tables 的取法，用到的不够 TOP_N 张时按表结构的顺序补齐）、全部、影响数字的项各自的 + 确认情况。项的范围和队列一样：表结构里没有的表、列不算。 +""" +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import catalog +from app.db.models import Run + +#: 队列的筛选：全部、影响数字的项、名称和说明 +KINDS = ("all", "facts", "names") +#: 队列一页最多几项；批量审阅一次最多几条；「被哪些运行用过」一次最多几次运行 +QUEUE_LIMIT_MAX = 200 +BATCH_MAX = 200 +RUNS_LIMIT_MAX = 100 +#: 进度里的「常用表」取使用次数前几张（同起草的默认范围 catalog.DRAFT_DEFAULT_TABLES） +TOP_N = 20 + +#: 每一项属于哪一组、风险几档：键是 (t, 字段) / (c, 字段) / (r, "")，档 4 最高 +FIELD_GROUP: dict[tuple[str, str], tuple[str, int]] = { + ("t", "valid_filter"): ("facts", 4), ("t", "dedup"): ("facts", 4), ("t", "business_date"): ("facts", 4), + ("r", ""): ("facts", 4), + ("t", "keys"): ("facts", 3), ("c", "measure"): ("facts", 3), + ("c", "codes"): ("names", 2), ("t", "grain"): ("names", 2), ("t", "kind"): ("names", 2), + ("t", "label"): ("names", 1), ("t", "description"): ("names", 1), ("c", "label"): ("names", 1), + ("c", "meaning"): ("names", 1), ("c", "unit"): ("names", 1), +} +RISK_WEIGHT = {4: 4.0, 3: 3.0, 2: 2.0, 1: 1.0} +#: 来源权重：没写的（外键约束、人工填写，它们本来就不是推断）按 1 +SOURCE_WEIGHT = {"llm": 1.5, "name": 1.5, "profile": 1.25, "comment": 1.0} + +#: 来源的界面叫法（和 frontend/src/lib/terms.ts 的 CATALOG_SOURCE_LABEL 一致，有测试核对） +SOURCE_LABEL = {"comment": "数据库注释", "fk": "外键约束", "name": "命名推断", "profile": "数据剖析", "llm": "模型起草", + "human": "人工填写"} +RELATION_LABEL = "关联关系" +#: reasons 的说法：常用表（被几次运行用到）；使用次数比运行次数多（同一次运行里查了几次）时两个都写 +REASON_TOP = "常用表（{used}）" +USED_RUNS = "被 {runs} 次运行用到" +USED_RUNS_AND_QUERIES = "被 {runs} 次运行用到，使用 {n} 次" +REASON_RISK = {4: "影响数字的项（{label}）", 3: "影响计数和汇总的项（{label}）", 2: "帮助理解取值的项（{label}）", + 1: "名称和说明（{label}）"} +REASON_PENDING_CODES = "{n} 个码值的含义待填写" +REASON_SOURCE = "{source}，未确认" +#: 码值含义为空（数据剖析写进来的候选）时的说法，同界面的 CODES_TEXT.pending +CODE_PENDING = "含义待填写" + +#: 批量审阅逐条结果的机读码 +CODE_CONFLICT = "catalog_conflict" +CODE_PATH = "catalog_path_invalid" +CODE_INVALID = "catalog_invalid" +CODE_TABLE_MISSING = "table_missing" + +Slot = catalog.Slot + + +# ========================================================================== +# 一项的路径、叫法、显示 +# ========================================================================== + + +def slot_path(key: Slot) -> str: + """槽位 → 审阅接口的路径写法（catalog.parse_path 的反过来）。""" + if key[0] == "t": + return key[1] + if key[0] == "c": + return f"columns.{key[1]}.{key[2]}" + return f"relations.{key[1]}" + + +def field_of(key: Slot) -> tuple[str, str]: + """槽位 → FIELD_GROUP 的键。""" + if key[0] == "r": + return ("r", "") + return (key[0], key[-1]) + + +def field_name(key: Slot) -> str: + """这一项的字段名：表级、列级的项是字段名，关联关系是 relation。""" + return "relation" if key[0] == "r" else key[-1] + + +def field_text(key: Slot) -> str: + """字段的界面叫法（不带列名）：中文名、码值、有效记录条件、关联关系……""" + if key[0] == "t": + return catalog.TABLE_FIELD_LABEL.get(key[1], key[1]) + if key[0] == "c": + return catalog.COLUMN_FIELD_LABEL.get(key[2], key[2]) + return RELATION_LABEL + + +def item_label(key: Slot, item: Mapping[str, Any] | None = None) -> str: + """一项给人看的位置：「表的有效记录条件」「列 status 的码值」「指向 members 的关联关系」（不知道指向哪张表时写编号）。""" + if key[0] == "r" and isinstance(item, Mapping) and item.get("to_table"): + return f"指向 {item['to_table']} 的{RELATION_LABEL}" + return catalog.describe_slot(key) + + +def item_value(key: Slot, item: Mapping[str, Any]) -> Any: + """一项的值：关联关系是两端、基数和覆盖率，别的是 value。""" + if key[0] == "r": + return {k: item.get(k) for k in ("columns", "to_table", "to_columns", "cardinality", "coverage")} + return item.get("value") + + +def _percent(ratio: float) -> str: + """覆盖率写成百分数：整数不带小数，其余留一位；不到 100% 的不写成 100%。""" + text = f"{ratio * 100:.1f}".rstrip("0").rstrip(".") + return "99.9%" if text == "100" and ratio < 1 else f"{text}%" + + +def relation_text(rel: Mapping[str, Any]) -> str: + """关联关系写成一行：member_id → members.id（多对一，覆盖率 98%）。""" + left = "、".join(str(c) for c in rel.get("columns") or []) + right = "、".join(f"{rel.get('to_table')}.{c}" for c in rel.get("to_columns") or []) + extra = [] + if rel.get("cardinality") in catalog.CARDINALITY_LABEL: + extra.append(catalog.CARDINALITY_LABEL[rel["cardinality"]]) + coverage = rel.get("coverage") + if isinstance(coverage, (int, float)) and not isinstance(coverage, bool): + extra.append(f"覆盖率 {_percent(float(coverage))}") + return f"{left} → {right}" + (f"（{'，'.join(extra)}）" if extra else "") + + +def display(key: Slot, value: Any) -> str: + """一项的值写成一行字，用界面上的说法（明细表、可累加、多对一……）。""" + if key[0] == "r": + return relation_text(value if isinstance(value, Mapping) else {}) + field_ = key[-1] + if field_ == "kind": + return catalog.UI_KIND_LABEL.get(value, str(value)) + if field_ == "measure": + return catalog.UI_MEASURE_LABEL.get(value, str(value)) + if field_ == "keys" and isinstance(value, list): + return "、".join(str(v) for v in value) + if field_ == "codes" and isinstance(value, Mapping): + return "、".join(f"{k}={v if str(v).strip() else CODE_PENDING}" for k, v in value.items()) + if field_ == "business_date" and isinstance(value, Mapping): + parts = [str(value.get("column") or "")] + if value.get("rule"): + parts.append(f"规则：{value['rule']}") + if value.get("timezone"): + parts.append(f"时区：{value['timezone']}") + return "；".join(parts) + return value if isinstance(value, str) else str(value) + + +# ========================================================================== +# 排序 +# ========================================================================== + + +def usage_weight(usage: int) -> float: + return 1.0 + math.log2(1 + max(0, int(usage or 0))) + + +def priority(usage: int, key: Slot, source: Any) -> float: + """一项的优先级：使用次数权重 × 风险权重 × 来源权重（不取整；排序和交给界面时取四位小数）。""" + _, tier = FIELD_GROUP[field_of(key)] + return usage_weight(usage) * RISK_WEIGHT[tier] * SOURCE_WEIGHT.get(str(source), 1.0) + + +def top_tables(tables: Mapping[str, Any], usage: Mapping[str, int]) -> list[str]: + """使用次数前 TOP_N 张表（同 catalog.default_tables，起草对话框的「使用次数最多的 20 张表」）：使用次数从多到少， + 一样多的（包括都没用过的）按表结构里的顺序。表结构里不到 TOP_N 张时就是全部。""" + order = {name: i for i, name in enumerate(tables)} + return sorted(tables, key=lambda n: (-usage.get(n, 0), order[n]))[:TOP_N] + + +@dataclass(frozen=True) +class Usage: + """每张表用得多不多（没被用过的表不出现）。 + + queries：使用次数，同 catalog.table_usage（目录清单的「使用 N 次」、清单的排序、起草的默认范围）； + runs：用到它的运行有几次，同「被哪些运行用过」（table_runs）的 total。同一次运行里查了几次这张表，queries 比 runs 多。 + """ + + queries: dict[str, int] + runs: dict[str, int] + + +async def table_usage_counts(session: AsyncSession, source: Any) -> Usage: + """一次读出每张表的使用次数和运行次数（同一份查询快照引用：catalog.query_snapshot_tables）。""" + queries: Counter[str] = Counter() + runs: dict[str, set[str]] = {} + for run, hit in await catalog.query_snapshot_tables(session, source): + queries.update(hit) + for name in hit: + runs.setdefault(name, set()).add(run) + return Usage(dict(queries), {name: len(ids) for name, ids in runs.items()}) + + +def usage_text(queries: int, runs: int) -> str: + """「被 2 次运行用到」；使用次数比运行次数多时「被 2 次运行用到，使用 5 次」。""" + if queries > runs: + return USED_RUNS_AND_QUERIES.format(runs=f"{runs:,}", n=f"{queries:,}") + return USED_RUNS.format(runs=f"{runs:,}") + + +def _reasons(key: Slot, item: Mapping[str, Any], usage: int, runs: int, top: bool) -> list[str]: + out: list[str] = [] + if usage > 0: + used = usage_text(usage, runs) + out.append(REASON_TOP.format(used=used) if top else used) + _, tier = FIELD_GROUP[field_of(key)] + out.append(REASON_RISK[tier].format(label=field_text(key))) + if key[0] == "c" and key[2] == "codes" and isinstance(item.get("value"), Mapping): + pending = sum(1 for v in item["value"].values() if not str(v).strip()) + if pending: + out.append(REASON_PENDING_CODES.format(n=pending)) + source = str(item.get("source") or "") + out.append(REASON_SOURCE.format(source=SOURCE_LABEL.get(source, source))) + return out + + +def live_slots(notes: Mapping[str, Any] | None, meta: Mapping[str, Any] | None) -> list[tuple[Slot, dict[str, Any]]]: + """一张表目录里还算数的项：表结构里已经没有的列上的项不算（不知道表结构时都算）。""" + columns = catalog._column_names(meta) + return [(key, item) for key, item in catalog._slots(notes).items() + if not (key[0] == "c" and columns is not None and key[1] not in columns)] + + +# ========================================================================== +# 队列 +# ========================================================================== + + +async def review_queue(session: AsyncSession, source: Any, *, kind: str = "all", table: str | None = None, + limit: int = 50, offset: int = 0) -> dict[str, Any]: + """审阅队列：{total, items}。source 是读表结构用的源（上传源是绑定当前快照的视图）。""" + cache = getattr(source, "schema_cache", None) or {} + tables = cache.get("tables") or {} + wanted = None + if table: + wanted = catalog.resolve_table_name(cache, table) or table + entries = await catalog.read_catalog(session, source.id) + usage = await table_usage_counts(session, source) + top = set(top_tables(tables, usage.queries)) + items: list[dict[str, Any]] = [] + for name, entry in entries.items(): + meta = tables.get(name) + if meta is None or (wanted is not None and name != wanted): + continue + used = usage.queries.get(name, 0) + for key, item in live_slots(entry.notes, meta): + if item.get("status") != "proposed": + continue + group, _ = FIELD_GROUP[field_of(key)] + if kind != "all" and group != kind: + continue + value = item_value(key, item) + items.append({ + "table_name": name, + "path": slot_path(key), + "field": field_name(key), + "field_label": item_label(key, item), + # 列级的项是哪一列（表级的项、关联关系为 null） + "column": key[1] if key[0] == "c" else None, + "value": value, + "display": display(key, value), + "source": item.get("source"), + "status": item.get("status"), + "note": item.get("note") if isinstance(item.get("note"), str) and item.get("note") else None, + "priority": round(priority(used, key, item.get("source")), 4), + "reasons": _reasons(key, item, used, usage.runs.get(name, 0), name in top), + "table_usage": used, + "version": entry.version, + }) + items.sort(key=lambda i: (-i["priority"], i["table_name"], i["path"])) + return {"total": len(items), "items": items[offset:offset + limit]} + + +# ========================================================================== +# 批量审阅 +# ========================================================================== + + +@dataclass(frozen=True) +class BatchItem: + table_name: str + path: str + action: str + if_version: int + + +def _result(item: BatchItem, *, ok: bool, version: int | None = None, error: str | None = None, + code: str | None = None) -> dict[str, Any]: + return {"table_name": item.table_name, "path": item.path, "ok": ok, "version": version if ok else None, + "error": error, "code": code} + + +async def review_batch(session: AsyncSession, source: Any, items: Iterable[BatchItem], *, + actor: str | None) -> list[dict[str, Any]]: + """批量审阅，返回逐条结果（和请求一一对应）。 + + 表名按表结构认（全名、大小写不一致都认）；表结构里没有、但还留着目录的也认（同单项审阅）。同一张表的几条按请求里的 + 顺序在当前目录上依次审阅，一次写入：版本不符（if_version 和库里的不一样）、路径不对的那几条不做，其余照做；写入时撞上 + 别人刚改过，这张表的几条都记冲突。""" + items = list(items) + cache = getattr(source, "schema_cache", None) or {} + results: list[dict[str, Any] | None] = [None] * len(items) + groups: dict[str, list[int]] = {} + known: dict[str, str | None] = {} + for i, item in enumerate(items): + if item.table_name not in known: + key = catalog.resolve_table_name(cache, item.table_name) + if key is None and await catalog.read_entry(session, source.id, item.table_name) is not None: + key = item.table_name + known[item.table_name] = key + key = known[item.table_name] + if key is None: + results[i] = _result(item, ok=False, code=CODE_TABLE_MISSING, + error=f"数据源中没有表 {item.table_name}，可能已被删除或尚未探查结构") + continue + groups.setdefault(key, []).append(i) + at = catalog.now_iso() + for key, indexes in groups.items(): + entry = await catalog.read_entry(session, source.id, key) + current = entry.version if entry else 0 + notes: dict[str, Any] = entry.notes if entry else {} + applied: list[int] = [] + for i in indexes: + item = items[i] + if item.if_version != current: + results[i] = _result(item, ok=False, code=CODE_CONFLICT, error=str(catalog.CatalogConflict(key, current))) + continue + try: + notes = catalog.review_item(notes, item.path, item.action, at=at) + except catalog.CatalogPathError as e: + results[i] = _result(item, ok=False, code=CODE_PATH, error=str(e)) + continue + applied.append(i) + if not applied: + continue + try: + written = await catalog.write_entry(session, source.id, key, notes, if_version=current, actor=actor) + except catalog.CatalogConflict as e: + for i in applied: + results[i] = _result(items[i], ok=False, code=CODE_CONFLICT, error=str(e)) + continue + except catalog.CatalogInvalid as e: + for i in applied: + results[i] = _result(items[i], ok=False, code=CODE_INVALID, error=str(e)) + continue + for i in applied: + results[i] = _result(items[i], ok=True, version=written.version) + return [r for r in results if r is not None] + + +# ========================================================================== +# 被哪些运行用过 +# ========================================================================== + + +async def table_runs(session: AsyncSession, source: Any, table: str, *, limit: int = 20) -> dict[str, Any]: + """用过这张表的运行：{total, runs}，新的在前（按开始时间，没开始的按建立时间）。取法同使用次数。""" + run_ids = {run for run, hit in await catalog.query_snapshot_tables(session, source) if table in hit} + if not run_ids: + return {"total": 0, "runs": []} + started = func.coalesce(Run.started_at, Run.created_at) + rows = (await session.execute( + select(Run.id, Run.workflow_id, Run.workflow_name, Run.run_class, Run.status, Run.started_at) + .where(Run.id.in_(run_ids)).order_by(started.desc(), Run.id).limit(limit) + )).all() + return { + "total": len(run_ids), + "runs": [{"run_id": r.id, "workflow_id": r.workflow_id, "workflow_name": r.workflow_name or "", + "run_class": r.run_class, "status": r.status, + "started_at": r.started_at.isoformat() if r.started_at else None} for r in rows], + } + + +# ========================================================================== +# 进度 +# ========================================================================== + + +async def review_progress(session: AsyncSession, source: Any) -> dict[str, Any]: + """审阅进度：常用 TOP_N 张表（top_tables）、全部、影响数字的项，项按状态计数。全部的 items_total 里还有有确证的项 + （不单列）。""" + tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} + entries = await catalog.read_catalog(session, source.id) + usage = await catalog.table_usage(session, source) + top = set(top_tables(tables, usage)) + every: Counter[str] = Counter() + head: Counter[str] = Counter() + facts: Counter[str] = Counter() + for name, entry in entries.items(): + meta = tables.get(name) + if meta is None: + continue + for key, item in live_slots(entry.notes, meta): + status = str(item.get("status")) + every[status] += 1 + if name in top: + head[status] += 1 + if FIELD_GROUP[field_of(key)][0] == "facts": + facts[status] += 1 + return { + "top_n": TOP_N, + "top": {"tables": len(top), "items_total": sum(head.values()), "items_confirmed": head["confirmed"]}, + "all": {"items_total": sum(every.values()), "items_confirmed": every["confirmed"], + "items_proposed": every["proposed"], "items_rejected": every["rejected"]}, + "facts": {"proposed": facts["proposed"], "confirmed": facts["confirmed"]}, + } + + +__all__ = ["BATCH_MAX", "BatchItem", "CODE_CONFLICT", "CODE_INVALID", "CODE_PATH", "CODE_TABLE_MISSING", + "FIELD_GROUP", "KINDS", "QUEUE_LIMIT_MAX", "RISK_WEIGHT", "RUNS_LIMIT_MAX", "SOURCE_LABEL", "SOURCE_WEIGHT", + "TOP_N", "Usage", "display", "field_name", "field_of", "field_text", "item_label", "item_value", + "live_slots", "priority", "relation_text", "review_batch", "review_progress", "review_queue", "slot_path", + "table_runs", "table_usage_counts", "top_tables", "usage_text", "usage_weight"] diff --git a/backend/app/data/catalog_transfer.py b/backend/app/data/catalog_transfer.py new file mode 100644 index 00000000..39bab51d --- /dev/null +++ b/backend/app/data/catalog_transfer.py @@ -0,0 +1,545 @@ +"""数据目录的导出导入：本机整理好的目录带到部署实例上（GET /catalog/export、POST /catalog/import/preview、/import）。 + +**为什么要有它。** 目录（尤其是人工确认、驳回过的项）是在本机对着开发库一项项审出来的；部署实例连的是同一套业务库， +却没有办法把这些结论带过去，只能在那边从头再审一遍。导出是一个 JSON 文件，导入先预览、再确认。 + +**文件**（export_catalog）：{format: "agentlab-catalog", version: 1, exported_at, source: {name, kind}, guidance, +tables: {表名: {notes, version}}}，含全部状态的项，表结构里已经没有的表也带上（目录里有什么就导出什么）。 + +**导入的规矩**（plan_import，预览和导入共用这一份）： + +- 表按名字对：表结构里的写法、忽略大小写（name_key）、带 schema 前缀的对不上时去掉前缀再对（catalog.resolve_table_name）。 + 只对目标数据源表结构里的表；对不上的列在 unmatched_tables。文件里两张表对上目标的同一张时，写法完全一样的那张优先， + 另一张算对不上。列名同样按表结构的写法、再忽略大小写对；业务主键、业务日期、关联关系两端的列和指向的表都换成目标的写法， + 关联关系的编号按目标的表名、列名重算（catalog.relation_id）。 +- 文件里已确认、已驳回的项照带：来源、状态、备注、确认时的依据（reason）、修改时间都不变。目标那一格空着就新增；目标是 + 推断、有确证的就更新；目标也是已确认、已驳回的，和文件一样就保留，不一样（值或状态不同）就保留目标的、记一条冲突。关联 + 关系的「一样」看两端和基数，覆盖率是测量值、不算。 +- 文件里推断、有确证的项只填目标的空缺，目标那一格有东西（哪怕是驳回）就保留目标的，不算冲突。这样的项不带确认时的依据 + （文件被手改过才会有）：推断项上挂着依据，看起来像使用者确认过它。 +- 跳过并写原因：表结构里没有的列、业务主键和业务日期指着的列没有的、关联关系两端对不上的（指向的表、两端的列）、 + 目标数据源遮罩的列上的码值（码值就是这一列的取值，遮罩了就不该进目录）、不认识的字段、取值不合规的项（说法同 + catalog.validate_notes）。 +- 有效记录条件是表级的一项、却写着列名，按列跳过的规矩拦不住它：按目标数据源的方言解析出用到的列 + （filter_columns_problem），不在目标的表结构里、是别的表的列、解析不了或不是一个条件、带子查询（没法核对子查询用到的表 + 和列）的跳过。人工写的条件不限函数（模型起草时的那套白名单只管模型写的条件）；列名大小写和目标不同、带本表表名的照带。 +- 逐表写（乐观锁，撞上别人刚改过就重读重算，最多三次），一张表写不进去只记在那张表上，不影响别的表。 +- 起草要求：勾了 include_guidance、文件里有、目标还没有时才写（action: replace）；目标已有的不覆盖（keep）；没勾或文件里 + 没有是 none。 +- 文件格式不对（不是对象、格式标记或版本不对、表的目录不是对象、起草要求不是文字或超长）抛 TransferInvalid，接口回 422。 + +有排队或进行中的起草任务时不导入（接口先调 catalog_jobs.ensure_idle，409 catalog_draft_busy）；目标数据源没有表结构时 +接口回 409（什么也对不上）。 +""" +from __future__ import annotations + +import copy +import json +import re +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import catalog, catalog_jobs +from app.data.catalog_review import item_label, slot_path +from app.data.engine import masked_columns +from app.data.names import name_key +from app.data.sqlcheck import dialect_of + +FORMAT = "agentlab-catalog" +VERSION = 1 +FILE_SUFFIX = "-数据目录.json" +_BAD_FILENAME = re.compile(r'[\\/:*?"<>|\x00-\x1f\x7f]') +_NAME_MAX = 80 +_UNNAMED = "数据源" + +#: 状态：照带的、只填空缺的 +_LOCKED = ("confirmed", "rejected") +#: 状态的界面叫法（同 frontend/src/lib/terms.ts 的 CATALOG_STATUS_LABEL） +_STATUS_LABEL = {"proposed": "推断", "verified": "已验证", "confirmed": "已确认", "rejected": "已驳回"} + +#: 写入一直撞上别人刚改过（重读重算三次仍冲突） +WRITE_CONFLICT_ERROR = "这张表的数据目录正被其他人修改，没有导入，请稍后重新导入" +WRITE_ERROR = "写入这张表的数据目录时出错，没有导入：{line}" + + +class TransferInvalid(ValueError): + """文件格式不对。message 给人看（接口回 422）。""" + + +# ========================================================================== +# 导出 +# ========================================================================== + + +def attachment_name(source_name: str) -> str: + """附件名「<数据源名>-数据目录.json」。文件名里不能有的字换成下划线。""" + name = _BAD_FILENAME.sub("_", source_name or "").strip(" .")[:_NAME_MAX].strip(" .") + return f"{name or _UNNAMED}{FILE_SUFFIX}" + + +def content_disposition(filename: str, source_id: str) -> str: + """附件头：中文名按 RFC 5987 写在 filename* 里；filename 给不认 filename* 的老客户端一个纯 ASCII 的名字。""" + from urllib.parse import quote + + ascii_id = re.sub(r"[^A-Za-z0-9]", "", source_id)[:8] + return f"attachment; filename=\"agentlab-catalog-{ascii_id}.json\"; filename*=UTF-8''{quote(filename, safe='')}" + + +async def export_catalog(session: AsyncSession, row: Any) -> dict[str, Any]: + """导出这个数据源的数据目录（row 是数据源记录，目录按它的 id 存）。""" + entries = await catalog.read_catalog(session, row.id) + return { + "format": FORMAT, + "version": VERSION, + "exported_at": catalog.now_iso(), + "source": {"name": row.name, "kind": row.kind}, + "guidance": catalog_jobs.guidance_of(row), + "tables": {name: {"notes": entry.notes, "version": entry.version} for name, entry in sorted(entries.items())}, + } + + +# ========================================================================== +# 读文件 +# ========================================================================== + + +@dataclass(frozen=True) +class ImportFile: + tables: dict[str, dict[str, Any]] + guidance: str | None + + +def read_file(content: Any, *, include_guidance: bool) -> ImportFile: + """导出的那个对象 → ImportFile。界面直接交来文件的原文（一段 JSON 文字）也认。格式不对抛 TransferInvalid（说法给人看）。""" + if isinstance(content, str): + try: + content = json.loads(content) + except ValueError as e: + raise TransferInvalid("文件内容不是合法的 JSON，请选择从 AgentLab 导出的数据目录文件") from e + if not isinstance(content, Mapping): + raise TransferInvalid("文件内容不是数据目录导出的格式，请选择从 AgentLab 导出的数据目录文件") + if content.get("format") != FORMAT: + raise TransferInvalid("这个文件不是 AgentLab 导出的数据目录（文件里的格式标记不对），请重新选择文件") + version = content.get("version") + if version != VERSION: + raise TransferInvalid(f"这个数据目录文件的格式版本是「{version}」，当前只能导入第 {VERSION} 版的文件，" + "请用同一版本的 AgentLab 重新导出") + tables = content.get("tables") + if not isinstance(tables, Mapping): + raise TransferInvalid("数据目录文件里没有表的目录，文件可能不完整，请重新导出") + out: dict[str, dict[str, Any]] = {} + for name, body in tables.items(): + notes = body.get("notes") if isinstance(body, Mapping) else None + if not isinstance(name, str) or not name.strip() or not isinstance(notes, Mapping): + raise TransferInvalid(f"数据目录文件里表「{str(name)[:60]}」的目录格式不正确，文件可能被改动过，请重新导出") + out[name] = copy.deepcopy(dict(notes)) + guidance = content.get("guidance") + if guidance is not None and not isinstance(guidance, str): + raise TransferInvalid("数据目录文件里的起草要求格式不正确，文件可能被改动过，请重新导出") + text = guidance.strip() if isinstance(guidance, str) else "" + if include_guidance and len(text) > catalog_jobs.GUIDANCE_MAX: + raise TransferInvalid(f"数据目录文件里的起草要求有 {len(text)} 字，超过了 {catalog_jobs.GUIDANCE_MAX} 字的上限，" + "请删减后再导入，或者不导入起草要求") + return ImportFile(tables=out, guidance=text or None) + + +# ========================================================================== +# 对表、对列 +# ========================================================================== + + +def match_tables(file_tables: list[str], schema_cache: Mapping[str, Any]) -> tuple[dict[str, str], list[str]]: + """文件里的表 → 目标表结构里的键。返回 ({文件里的表名: 目标表名}, 对不上的表名)。写法完全一样的先认领。""" + tables = (schema_cache or {}).get("tables") or {} + matched: dict[str, str] = {} + claimed: set[str] = set() + ordered = sorted(file_tables, key=lambda n: n not in tables) + for name in ordered: + key = catalog.resolve_table_name(schema_cache, name) + if key is None or key in claimed: + continue + matched[name] = key + claimed.add(key) + unmatched = [name for name in file_tables if name not in matched] + return {name: matched[name] for name in file_tables if name in matched}, unmatched + + +class _Columns: + """一张目标表的列：按表结构的写法认，再忽略大小写认（只认唯一对上的）。""" + + def __init__(self, meta: Mapping[str, Any] | None) -> None: + names = [str(c.get("name")) for c in (meta or {}).get("columns") or [] if isinstance(c, Mapping) + and c.get("name") is not None] + self.names = set(names) + self._loose: dict[str, list[str]] = {} + for name in names: + self._loose.setdefault(name_key(name), []).append(name) + + def find(self, name: Any) -> str | None: + if not isinstance(name, str): + return None + if name in self.names: + return name + found = self._loose.get(name_key(name)) or [] + return found[0] if len(found) == 1 else None + + +def filter_columns_problem(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None, table: str, + also: Iterable[str] = (), where: str = "表结构") -> str | None: + """有效记录条件用到的列是不是都在这张表的表结构里（给人看的一句话；没问题返回 None）。 + + 按数据源的方言（sqlcheck.dialect_of）解析成一个条件：解析不了、不止一句、是一条查询、带子查询（没法核对子查询用到的 + 表和列）的不收；列不带表名，或带这张表的表名（table、表结构里的 qualified、also 里的写法，比如文件里的表名；忽略大小写、 + 去掉 schema 前缀），列名忽略大小写在表结构里。不限函数、运算。不是文字的交给合规检查说（返回 None）；数据源类型 + 没有对应的方言时没法解析，不核对。where 是说法里「表结构」的叫法（导入时是「目标的表结构」）。""" + if not isinstance(condition, str) or not condition.strip(): + return None + dialect = dialect_of(kind) + if dialect is None: + return None + import sqlglot + from sqlglot import exp + + try: + parsed = [s for s in sqlglot.parse(condition.strip(), read=dialect) if s is not None] + except Exception: # noqa: BLE001 - 语法错、方言不支持的写法 + parsed = [] + if len(parsed) != 1 or isinstance(parsed[0], exp.Query) or not isinstance(parsed[0], exp.Condition): + return f"有效记录条件不是一个能解析的 SQL 条件（按 {kind} 的写法），没法核对它用到的列" + expression = parsed[0] + if any(isinstance(node, (exp.Query, exp.Subquery)) for node in expression.walk()): + return "有效记录条件里有子查询，没法核对子查询用到的表和列" + own = {name_key(str(n).rsplit(".", 1)[-1]) for n in (table, (meta or {}).get("qualified"), *also) if n} + columns = {str(c.get("name")).lower() for c in (meta or {}).get("columns") or [] + if isinstance(c, Mapping) and c.get("name") is not None} + foreign: list[str] = [] + missing: list[str] = [] + for column in expression.find_all(exp.Column): + if column.is_star: + continue + if column.table and name_key(column.table) not in own: + foreign.append(column.sql(dialect=dialect)) + elif column.name.lower() not in columns: + missing.append(column.name) + if foreign: + return f"有效记录条件引用了别的表的列（{'、'.join(dict.fromkeys(foreign))}），只能用这张表的列" + if missing: + return f"有效记录条件里的列「{'、'.join(dict.fromkeys(missing))}」不在{where}里" + return None + + +# ========================================================================== +# 一张表怎么并 +# ========================================================================== + + +@dataclass +class TablePlan: + """文件里一张表并进目标的计划。writes 是要写进目标的项（目标的槽位 → 项）。""" + + table_name: str + matched_as: str + add: int = 0 + update: int = 0 + keep: int = 0 + conflicts: list[dict[str, Any]] = field(default_factory=list) + skipped: list[dict[str, Any]] = field(default_factory=list) + writes: dict[catalog.Slot, dict[str, Any]] = field(default_factory=dict) + version: int = 0 + error: str | None = None + + def as_dict(self) -> dict[str, Any]: + return {"table_name": self.table_name, "matched_as": self.matched_as, "add": self.add, "update": self.update, + "keep": self.keep, "conflicts": self.conflicts, "skipped": self.skipped, "version": self.version, + "error": self.error} + + +def _raw_items(notes: Mapping[str, Any]) -> list[tuple[str, catalog.Slot | None, Any, str | None]]: + """文件里一张表的每一项：(文件里的路径, 槽位, 项, 不认的原因)。不认识的字段、不是对象的列也逐个列出来（槽位为 None）。""" + out: list[tuple[str, catalog.Slot | None, Any, str | None]] = [] + for name, item in notes.items(): + if name in catalog.TABLE_FIELDS: + out.append((name, ("t", name), item, None)) + elif name not in ("columns", "relations"): + out.append((str(name), None, item, f"数据目录里不认识的字段「{name}」")) + columns = notes.get("columns", {}) + if not isinstance(columns, Mapping): + out.append(("columns", None, columns, "列的目录应为以列名为键的对象")) + columns = {} + for col, items in columns.items(): + if not isinstance(items, Mapping): + out.append((f"columns.{col}", None, items, f"列 {col} 的目录应为对象")) + continue + for name, item in items.items(): + path = f"columns.{col}.{name}" + if name in catalog.COLUMN_FIELDS: + out.append((path, ("c", str(col), name), item, None)) + else: + out.append((path, None, item, f"列 {col} 含有不认识的字段「{name}」")) + relations = notes.get("relations", []) + if not isinstance(relations, list): + out.append(("relations", None, relations, "关联关系应为列表")) + relations = [] + for rel in relations: + rid = rel.get("id") if isinstance(rel, Mapping) else None + out.append((f"relations.{rid}", ("r", str(rid)), rel, None)) + return out + + +def _item_problem(key: catalog.Slot, item: Any) -> str | None: + """这一项本身合不合规（说法同 catalog.validate_notes）。""" + if key[0] == "t": + probe: dict[str, Any] = {key[1]: item} + elif key[0] == "c": + probe = {"columns": {key[1]: {key[2]: item}}} + else: + probe = {"relations": [item]} + problems = catalog.validate_notes(probe) + return problems[0] if problems else None + + +def _decision(key: catalog.Slot, item: Mapping[str, Any]) -> Any: + """判断「一样不一样」看的部分：状态，加上值（码值另看「已列全」）。关联关系的两端由编号定了，再看基数；覆盖率不算。""" + if key[0] == "r": + return item.get("status"), item.get("cardinality") + if key[0] == "c" and key[2] == "codes": + return item.get("status"), item.get("value"), catalog.codes_complete(item) + return item.get("status"), item.get("value") + + +def _conflict_why(file_status: str, target_status: str) -> str: + if target_status == "rejected": + return f"目标里这一项已经驳回，文件里是{_STATUS_LABEL.get(file_status, file_status)}，保留目标的" + if file_status == "rejected": + return "目标里这一项已经人工确认过，文件里是已驳回，保留目标的" + return "目标里这一项已经人工确认过，和文件里的不一样，保留目标的" + + +class _Mapper: + """把文件里的一项换成目标的写法：列名、业务主键和业务日期的列、关联关系两端和编号；有效记录条件核对用到的列。对不上 + 返回原因。name 是文件里的表名（条件里带它当表名也认），kind 是目标数据源的类型（解析条件用它的方言）。""" + + def __init__(self, name: str, target: str, schema_cache: Mapping[str, Any], masked: set[str], + kind: str | None) -> None: + self.name = name + self.target = target + self.cache = schema_cache + self.tables = (schema_cache or {}).get("tables") or {} + self.columns = _Columns(self.tables.get(target)) + self.masked = masked + self.kind = kind + + def __call__(self, key: catalog.Slot, item: Any) -> tuple[catalog.Slot | None, Any, str | None]: + if not isinstance(item, Mapping): + return None, item, None # 交给合规检查说 + item = copy.deepcopy(dict(item)) + if key[0] == "c": + col = self.columns.find(key[1]) + if col is None: + return None, item, f"目标的表结构里没有列 {key[1]}，跳过" + if key[2] == "codes" and col.lower() in self.masked: + return None, item, f"列 {col} 在目标数据源的设置中被遮罩，码值是这一列的取值，不导入" + return ("c", col, key[2]), item, None + if key[0] == "t": + value = item.get("value") + if key[1] == "keys" and isinstance(value, list): + mapped = [self.columns.find(c) for c in value] + if missing := [str(c) for c, m in zip(value, mapped) if m is None]: + return None, item, f"业务主键里的列「{'、'.join(missing)}」不在目标的表结构里，跳过" + item["value"] = mapped + elif key[1] == "business_date" and isinstance(value, Mapping) and isinstance(value.get("column"), str): + col = self.columns.find(value["column"]) + if col is None: + return None, item, f"业务日期的列「{value['column']}」不在目标的表结构里，跳过" + item["value"] = {**value, "column": col} + elif key[1] == "valid_filter": + problem = filter_columns_problem(value, self.tables.get(self.target), kind=self.kind, + table=self.target, also=(self.name,), where="目标的表结构") + if problem is not None: + return None, item, f"{problem}，跳过" + return key, item, None + return self._relation(item) + + def _relation(self, rel: dict[str, Any]) -> tuple[catalog.Slot | None, Any, str | None]: + cols, to_table, to_cols = rel.get("columns"), rel.get("to_table"), rel.get("to_columns") + if not (isinstance(cols, list) and isinstance(to_cols, list) and isinstance(to_table, str)): + return None, rel, None + target_table = catalog.resolve_table_name(self.cache, to_table) + if target_table is None: + return None, rel, f"关联关系指向的表 {to_table} 不在目标数据源的表结构里，跳过" + mine = [self.columns.find(c) for c in cols] + if missing := [str(c) for c, m in zip(cols, mine) if m is None]: + return None, rel, f"关联关系的本表字段「{'、'.join(missing)}」不在目标的表结构里，跳过" + theirs_columns = _Columns(self.tables.get(target_table)) + theirs = [theirs_columns.find(c) for c in to_cols] + if missing := [str(c) for c, m in zip(to_cols, theirs) if m is None]: + return None, rel, f"关联关系的目标字段「{'、'.join(missing)}」不在 {target_table} 的表结构里，跳过" + rid = catalog.relation_id(self.target, mine, target_table, theirs) + return ("r", rid), {**rel, "id": rid, "columns": mine, "to_table": target_table, "to_columns": theirs}, None + + +def plan_table(table_name: str, target: str, file_notes: Mapping[str, Any], target_notes: Mapping[str, Any] | None, + schema_cache: Mapping[str, Any], masked: set[str], kind: str | None = None) -> TablePlan: + """文件里一张表并进目标那张表的计划（纯函数，不写库）。kind 是目标数据源的类型（解析有效记录条件用）。""" + plan = TablePlan(table_name=table_name, matched_as=target) + existing = catalog._slots(target_notes) + mapper = _Mapper(table_name, target, schema_cache, masked, kind) + seen: set[catalog.Slot] = set() + for path, key, item, why in _raw_items(file_notes): + if why is not None or key is None: + plan.skipped.append({"path": path, "why": why or "无法识别这一项"}) + continue + mapped, item, why = mapper(key, item) + if why is not None: + plan.skipped.append({"path": path, "why": why}) + continue + if isinstance(item, dict) and item.get("status") not in _LOCKED and "reason" in item: + # 推断、有确证的项不带确认时的依据：在校验之前去掉，手改的文件里依据不合规也不把整项跳过 + item = {k: v for k, v in item.items() if k != "reason"} + problem = _item_problem(mapped or key, item) + if problem is not None or mapped is None: + plan.skipped.append({"path": path, "why": problem or "无法识别这一项"}) + continue + if mapped in seen: + plan.skipped.append({"path": path, "why": f"和文件里前面的{catalog.describe_slot(mapped)}是同一项，只导入第一项"}) + continue + seen.add(mapped) + status = item.get("status") + have = existing.get(mapped) + if have is None: + plan.add += 1 + plan.writes[mapped] = item + elif status not in _LOCKED: + plan.keep += 1 # 推断、有确证的只填空缺 + elif have.get("status") in _LOCKED: + if _decision(mapped, have) == _decision(mapped, item): + plan.keep += 1 + else: + plan.conflicts.append({ + "path": slot_path(mapped), "label": item_label(mapped, item), + "file_value": _shown_value(mapped, item), "target_value": _shown_value(mapped, have), + "file_status": status, "target_status": have.get("status"), + "why": _conflict_why(str(status), str(have.get("status"))), + }) + elif _decision(mapped, have) == _decision(mapped, item): + plan.keep += 1 + else: + plan.update += 1 + plan.writes[mapped] = item + return plan + + +def _shown_value(key: catalog.Slot, item: Mapping[str, Any]) -> Any: + if key[0] == "r": + return {k: item.get(k) for k in ("columns", "to_table", "to_columns", "cardinality", "coverage")} + return item.get("value") + + +# ========================================================================== +# 预览、导入 +# ========================================================================== + + +@dataclass +class ImportReport: + tables: list[TablePlan] + unmatched: list[str] + guidance: dict[str, Any] + + def as_dict(self) -> dict[str, Any]: + return { + "tables": [t.as_dict() for t in self.tables], + "unmatched_tables": self.unmatched, + "totals": {"tables": len(self.tables), "add": sum(t.add for t in self.tables), + "update": sum(t.update for t in self.tables), "keep": sum(t.keep for t in self.tables), + "conflicts": sum(len(t.conflicts) for t in self.tables), + "skipped": sum(len(t.skipped) for t in self.tables)}, + "guidance": self.guidance, + } + + +def _guidance(file: ImportFile, row: Any, include: bool) -> dict[str, Any]: + target = catalog_jobs.guidance_of(row) + if not include or file.guidance is None: + action = "none" + elif target: + action = "keep" + else: + action = "replace" + return {"file": file.guidance, "target": target, "action": action} + + +async def plan_import(session: AsyncSession, row: Any, source: Any, content: Any, *, + include_guidance: bool) -> ImportReport: + """预览：逐表算出要新增、更新、保留什么，冲突和跳过的项及原因。不写库。row 是数据源记录（遮罩、起草要求按它）， + source 是读表结构用的源（上传源是绑定当前快照的视图）。格式不对抛 TransferInvalid。""" + file = read_file(content, include_guidance=include_guidance) + cache = getattr(source, "schema_cache", None) or {} + matched, unmatched = match_tables(list(file.tables), cache) + masked = {c.lower() for c in masked_columns(getattr(row, "options", None))} + plans: list[TablePlan] = [] + for name, target in matched.items(): + entry = await catalog.read_entry(session, source.id, target) + plan = plan_table(name, target, file.tables[name], entry.notes if entry else None, cache, masked, row.kind) + plan.version = entry.version if entry else 0 + plans.append(plan) + return ImportReport(plans, unmatched, _guidance(file, row, include_guidance)) + + +async def run_import(session: AsyncSession, row: Any, source: Any, content: Any, *, include_guidance: bool, + actor: str | None) -> ImportReport: + """导入：按 plan_import 的规矩逐表写入（乐观锁，撞上别人刚改过就重读重算，最多三次），再按需写起草要求。 + 一张表写不进去只记在它的 error 上（新增、更新记 0），不影响别的表。""" + file = read_file(content, include_guidance=include_guidance) + cache = getattr(source, "schema_cache", None) or {} + matched, unmatched = match_tables(list(file.tables), cache) + masked = {c.lower() for c in masked_columns(getattr(row, "options", None))} + plans: list[TablePlan] = [] + for name, target in matched.items(): + plans.append(await _import_table(session, source, name, target, file.tables[name], cache, masked, row.kind, + actor)) + guidance = _guidance(file, row, include_guidance) + if guidance["action"] == "replace": + await catalog_jobs.save_guidance(session, row, file.guidance) + return ImportReport(plans, unmatched, guidance) + + +async def _import_table(session: AsyncSession, source: Any, name: str, target: str, notes: Mapping[str, Any], + cache: Mapping[str, Any], masked: set[str], kind: str | None, actor: str | None) -> TablePlan: + plan = TablePlan(table_name=name, matched_as=target) + try: + for _attempt in range(3): + entry = await catalog.read_entry(session, source.id, target) + version = entry.version if entry else 0 + plan = plan_table(name, target, notes, entry.notes if entry else None, cache, masked, kind) + plan.version = version + if not plan.writes: + return plan + slots = catalog._slots(entry.notes if entry else None) + slots.update(plan.writes) + try: + written = await catalog.write_entry(session, source.id, target, catalog._assemble(slots), + if_version=version, actor=actor) + except catalog.CatalogConflict: + continue + plan.version = written.version + return plan + plan.error = WRITE_CONFLICT_ERROR + except catalog.CatalogInvalid as e: + await session.rollback() + plan.error = WRITE_ERROR.format(line=str(e)) + except Exception as e: # noqa: BLE001 - 这一张写不进去不影响别的表 + from app.core.errors import describe_exception, first_line + + await session.rollback() + plan.error = WRITE_ERROR.format(line=first_line(e) or describe_exception(e)) + plan.add = plan.update = 0 + return plan + + +__all__ = ["FORMAT", "ImportFile", "ImportReport", "TablePlan", "TransferInvalid", "VERSION", "attachment_name", + "content_disposition", "export_catalog", "filter_columns_problem", "match_tables", "plan_import", + "plan_table", "read_file", "run_import"] diff --git a/backend/app/data/sqlcheck.py b/backend/app/data/sqlcheck.py index c233d430..2a300acd 100644 --- a/backend/app/data/sqlcheck.py +++ b/backend/app/data/sqlcheck.py @@ -1131,5 +1131,274 @@ async def load_checkers_by_name(session: Any, names: Iterable[str]) -> dict[str, return await load_checkers(session, rows) -__all__ = ["CHECK_CODES", "LEVELS", "MAX_CHECKS", "MAX_SQL_CHARS", "SqlCheck", "SqlChecker", "check_sql", - "dialect_of", "graph_checks", "graph_sources", "load_checkers", "load_checkers_by_name", "tool_query"] +# ========================================================================== +# 有效行过滤（valid_filter）的校验：模型起草的条件要能进目录、能拼进剖析的核对查询 +# ========================================================================== + +#: 有效行过滤最多几个字（同目录里文本项的上限） +VALID_FILTER_MAX = 500 +#: 有效行过滤里能用的函数（其余函数、聚合、窗口一律不收） +VALID_FILTER_FUNCTIONS = ("UPPER", "LOWER", "TRIM", "COALESCE") +#: 模板占位：条件里不该有，有了也说不清最终拼出来的是什么（注释在解析之后按语法树认，字符串里的 -- 不算） +_FILTER_TEMPLATE = re.compile(r"\{\{|\{%") + +#: 不合规的说法以这一项在界面上的叫法开头（catalog.TABLE_FIELD_LABEL：「有效记录条件」）：没收下的原因会记进起草任务 +#: 逐表的日志，界面上看得到 +_FILTER_PREFIX = catalog.TABLE_FIELD_LABEL["valid_filter"] + + +@dataclass(frozen=True) +class ValidFilter: + """一段有效行过滤解析的结果。 + + expression：sqlglot 的条件（不合规时为 None）；columns：条件里用到的列（表结构里的真名，按出现的先后去重）； + names：条件里写的列名 → 表结构里的真名；problem：不合规时给人看的一句话，合规是 None；dialect：解析用的 sqlglot + 方言。text：解析的原文（去掉首尾空白）；spans：每处列引用（连同表名前缀）在原文里的位置 (起, 止（含）, 真名)，按 + 书写的先后，有一处拿不到位置时为 None（canonical_valid_filter 退回按语法树重写）。 + """ + + expression: Any = None + columns: tuple[str, ...] = () + names: Mapping[str, str] = field(default_factory=dict) + problem: str | None = None + dialect: str | None = None + text: str = "" + spans: tuple[tuple[int, int, str], ...] | None = None + + +def _bad_filter(problem: str, dialect: str | None = None) -> ValidFilter: + return ValidFilter(problem=f"{_FILTER_PREFIX}{problem}", dialect=dialect) + + +def _filter_node_problem(node: Any, *, predicate: bool) -> str | None: + """一个不在允许范围里的节点为什么不收（没有问题返回 None 的调用方不会走到这里）。""" + from sqlglot import exp + + if isinstance(node, exp.Query): + return "应为一个条件，不能是一条查询" + if any(isinstance(n, (exp.Query, exp.Subquery)) for n in node.walk()): + return "里不能有子查询" + if any(isinstance(n, exp.Window) for n in node.walk()): + return "里不能有窗口函数" + if any(isinstance(n, exp.AggFunc) for n in node.walk()): + return "里不能有聚合函数" + if isinstance(node, exp.Func): + name = str(node.this) if isinstance(node, exp.Anonymous) else node.sql_name() + return f"里只能用 {'、'.join(VALID_FILTER_FUNCTIONS)} 这几个函数，不能用 {name.upper()}" + if predicate: + return "应为一个条件（比较、IN、BETWEEN、LIKE、IS NULL，用 AND、OR、NOT 连接）" + text = node.sql() + return f"里有不支持的写法（{text[:40]}{'…' if len(text) > 40 else ''}）" + + +def _filter_value_problem(node: Any) -> str | None: + """比较两边的一个值：列、字面量、负数、允许的几个函数（参数同样是值）。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, (exp.Column, exp.Literal, exp.Null, exp.Boolean)): + return None + if isinstance(node, exp.Neg) and isinstance(_unparen(node.this), exp.Literal) and not _unparen(node.this).is_string: + return None + if isinstance(node, (exp.Upper, exp.Lower, exp.Trim, exp.Coalesce)): + for arg in node.iter_expressions(): + if problem := _filter_value_problem(arg): + return problem + return None + return _filter_node_problem(node, predicate=False) + + +def _filter_predicate_problem(node: Any) -> str | None: + """一个条件：比较、IN（只能是值的列表）、BETWEEN、LIKE、IS NULL，或者用 AND / OR / NOT 连起来的条件。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, (exp.And, exp.Or)): + return _filter_predicate_problem(node.left) or _filter_predicate_problem(node.right) + if isinstance(node, exp.Not): + return _filter_predicate_problem(node.this) + if isinstance(node, (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE)): + return _filter_value_problem(node.left) or _filter_value_problem(node.right) + if isinstance(node, exp.In): + if any(node.args.get(k) for k in ("query", "unnest", "field")): + return "里不能有子查询" + if not node.expressions: + return _filter_node_problem(node, predicate=True) + return _filter_value_problem(node.this) or next( + (p for p in (_filter_value_problem(e) for e in node.expressions) if p), None) + if isinstance(node, exp.Between): + return next((p for p in (_filter_value_problem(node.args.get(k)) for k in ("this", "low", "high")) if p), None) + if isinstance(node, exp.Like): + return _filter_value_problem(node.this) or _filter_value_problem(node.expression) + if isinstance(node, exp.Is): + if not isinstance(_unparen(node.expression), exp.Null): + return "里的 IS 只能写 IS NULL 或 IS NOT NULL" + return _filter_value_problem(node.this) + return _filter_node_problem(node, predicate=True) + + +def parse_valid_filter(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None, + table: str | None = None) -> ValidFilter: + """按数据源的方言把有效行过滤解析成一个条件，并核对它只用了这张表的列。从不抛异常，不合规写在 problem 里。 + + 规矩：一段文字、不超过 VALID_FILTER_MAX 字；不能有分号、模板占位、注释；解析出来恰好是一个条件，只由比较、IN（值的 + 列表）、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE 组成（不能有子查询、聚合、窗口、 + 别的函数、算术、参数占位）；列不带表名或带这张表的表名（table），都在表结构里，至少用到一列。 + + 列名先照原样对表结构，对不上再不分大小写对；表结构里有只差大小写的几列（PostgreSQL 上 "Status" 和 status 可以 + 并存）、写法又和哪一列都不一样时说不清是哪一列，不收。 + """ + from app.data.names import name_key as _key + + dialect = dialect_of(kind) + if not isinstance(condition, str) or not condition.strip(): + return _bad_filter("应为一段 SQL 条件", dialect) + text = condition.strip() + if len(text) > VALID_FILTER_MAX: + return _bad_filter(f"超过 {VALID_FILTER_MAX} 字", dialect) + if ";" in text: + return _bad_filter("里不能有分号", dialect) + if _FILTER_TEMPLATE.search(text): + return _bad_filter("里不能有模板占位", dialect) + if dialect is None: + return _bad_filter(f"没法校验：数据源类型「{kind}」不支持", dialect) + try: + import sqlglot + + statements = [s for s in sqlglot.parse(text, read=dialect) if s is not None] + except Exception: # noqa: BLE001 - 语法错、方言不支持的写法：一律不收 + return _bad_filter("不是一段能解析的 SQL 条件", dialect) + if len(statements) != 1: + return _bad_filter("不是一段能解析的 SQL 条件", dialect) + expression = statements[0] + if any(node.comments for node in expression.walk()): + return _bad_filter("里不能有注释", dialect) + if problem := _filter_predicate_problem(expression): + return _bad_filter(problem, dialect) + from sqlglot import exp + + real_names = [str(c.get("name")) for c in (meta or {}).get("columns") or [] + if isinstance(c, Mapping) and c.get("name") is not None] + exact = set(real_names) + by_lower: dict[str, list[str]] = {} + for name in real_names: + by_lower.setdefault(name.lower(), []).append(name) + own = {_key(str(n).rsplit(".", 1)[-1]) for n in (table, (meta or {}).get("qualified")) if n} + used: dict[str, str] = {} + spans: list[tuple[int, int, str]] | None = [] + for column in expression.find_all(exp.Column, bfs=False): # 按书写的先后 + if column.is_star: + return _bad_filter("里不能写 *", dialect) + qualifier = column.table + if column.args.get("db") or column.args.get("catalog") or (qualifier and _key(qualifier) not in own): + return _bad_filter(f"只能引用这张表的列，不能引用 {column.sql(dialect=dialect)}", dialect) + written = column.name + if written in exact: + real = written + else: + found = by_lower.get(written.lower()) or [] + if not found: + return _bad_filter(f"里的列「{written}」不在表结构里", dialect) + if len(found) > 1: + return _bad_filter(f"里的列「{written}」在表结构里有只差大小写的几列（{'、'.join(found)}），" + "要照表结构的写法写清是哪一列", dialect) + real = found[0] + used.setdefault(written, real) + span = _column_span(column, text) + spans = None if spans is None or span is None else [*spans, (*span, real)] + if not used: + return _bad_filter("没有用到这张表的任何列", dialect) + return ValidFilter(expression=expression, columns=tuple(dict.fromkeys(used.values())), names=used, + dialect=dialect, text=text, spans=tuple(spans) if spans is not None else None) + + +def _column_span(column: Any, text: str) -> tuple[int, int] | None: + """一处列引用（连同表名前缀）在原文里的位置 (起, 止（含）)：sqlglot 在标识符的 meta 里记下了它在原文里的起止。 + 拿不到、或者那一截原文里认不出这个列名时为 None。""" + ident = column.this + meta = getattr(ident, "meta", None) or {} + start, end = meta.get("start"), meta.get("end") + prefix = column.args.get("table") + if prefix is not None: + start = (getattr(prefix, "meta", None) or {}).get("start") + if not (isinstance(start, int) and isinstance(end, int) and 0 <= start <= end < len(text)): + return None + if not text[start:end + 1].rstrip("\"`]").lower().endswith(str(column.name).lower()): + return None + return start, end + + +def valid_filter_problem(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None, + table: str | None = None) -> str | None: + """有效行过滤哪里不合规（给人看的一句话）；合规返回 None。规矩见 parse_valid_filter。""" + return parse_valid_filter(condition, meta, kind=kind, table=table).problem + + +def render_valid_filter(parsed: ValidFilter, *, quoted: Any) -> str: + """把解析好的条件按数据源的方言重新写出来，列换成表结构里的真名、不带表名；quoted(真名) 为真的加引号（和剖析查询 + 其余部分的写法一致：SqlDialect.quote 要加引号的才加）。按语法树重写，NOT IN、IS NOT NULL 会写成 NOT x IN、 + NOT x IS NULL：canonical_valid_filter 拿不到原文里的位置时才退回用它。""" + from sqlglot import exp + + if parsed.expression is None: + raise ValueError(parsed.problem or "有效行过滤不合规") + + def swap(node: Any) -> Any: + if isinstance(node, exp.Column): + real = parsed.names.get(node.name, node.name) + return exp.Column(this=exp.Identifier(this=real, quoted=bool(quoted(real)))) + return node + + return parsed.expression.copy().transform(swap).sql(dialect=parsed.dialect) + + +def canonical_valid_filter(parsed: ValidFilter, *, quote: Any) -> str: + """有效行过滤存进目录、拼进剖析核对查询的写法（两处用的是同一条）：原文照旧，只把每处列引用换成 quote(真名)——表结构 + 里的真名、不带表名，按数据源的需要加引号（quote 是 catalog_profile.SqlDialect.quote，和剖析查询其余部分的写法一致）。 + + 以前目录里存的是模型的原文，核对时却按真名加了引号：PostgreSQL 上混合大小写的列（"OrderStatus"）不加引号会被折成 + 小写，存下的 orderstatus <> 'X' 一跑就报列不存在，核对过的却能跑通，SQL 检查还会让模型照抄存下的那一句。 + + 拿不到原文里的位置、换完以后认不出原来那几列时，退回按语法树重写（render_valid_filter）。""" + if parsed.expression is None: + raise ValueError(parsed.problem or "有效行过滤不合规") + if parsed.spans: + text = parsed.text + for start, end, real in sorted(parsed.spans, reverse=True): + text = text[:start] + quote(real) + text[end + 1:] + if _same_columns(text, parsed): + return text + return render_valid_filter(parsed, quoted=lambda name: quote(name) != name) + + +def _same_columns(text: str, parsed: ValidFilter) -> bool: + """换过列名的原文还是一个条件，列依次是表结构里的那几列、都不带表名。""" + import sqlglot + from sqlglot import exp + + try: + tree = sqlglot.parse_one(text, read=parsed.dialect) + except Exception: # noqa: BLE001 - 换坏了就退回按语法树重写 + return False + columns = list(tree.find_all(exp.Column, bfs=False)) + return ([c.name for c in columns] == [real for _, _, real in parsed.spans or ()] + and not any(c.table for c in columns)) + + +def stored_valid_filter(parsed: ValidFilter, kind: str | None) -> str: + """模型提议的有效行过滤存进目录的写法：canonical_valid_filter，按这个数据源的方言加引号（catalog_profile.SqlDialect： + SQLAlchemy 的写法，要加才加——PostgreSQL、SQLite 的 "OrderStatus"，MySQL 的 `OrderStatus`，Oracle 探查出来的小写名 + 不加）。""" + from app.data.catalog_profile import SqlDialect + + try: + dialect = SqlDialect(kind) + except ValueError: + return parsed.text + return canonical_valid_filter(parsed, quote=dialect.quote) + + +__all__ = ["CHECK_CODES", "LEVELS", "MAX_CHECKS", "MAX_SQL_CHARS", "SqlCheck", "SqlChecker", "VALID_FILTER_FUNCTIONS", + "VALID_FILTER_MAX", "ValidFilter", "canonical_valid_filter", "check_sql", "dialect_of", "graph_checks", + "graph_sources", "load_checkers", "load_checkers_by_name", "parse_valid_filter", "render_valid_filter", + "stored_valid_filter", "tool_query", "valid_filter_problem"] diff --git a/backend/app/db/base.py b/backend/app/db/base.py index f0da4ac6..3bc178cc 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -212,6 +212,12 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [ ("catalog_draft_jobs", "notice", "TEXT"), ("catalog_draft_tasks", "blank_columns", "INTEGER DEFAULT 0"), ("catalog_draft_tasks", "blank_sample", "JSON"), + # 数据目录闭环第一步：宽表逐段落盘的段数、块进度。存量任务回填 0 和 NULL：升级前没记这些 + ("catalog_draft_tasks", "parts_written", "INTEGER DEFAULT 0"), + ("catalog_draft_tasks", "chunks_total", "INTEGER"), + ("catalog_draft_tasks", "chunks_done", "INTEGER"), + # 复核后：逐表的日志（模型提议的有效记录条件没有收下的原因）。存量任务回填 NULL，接口给空列表 + ("catalog_draft_tasks", "log", "JSON"), ] #: 迁移补上的列要建的索引：(索引名, 表, 列)。create_all 不碰已有的表，表上新列的索引也就不会建。 diff --git a/backend/app/db/models.py b/backend/app/db/models.py index a5438d84..f341e89a 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -410,9 +410,14 @@ class CatalogDraftTask(Base): table_name: Mapped[str] = mapped_column(String(255)) status: Mapped[str] = mapped_column(String(20), default="pending") column_count: Mapped[int] = mapped_column(Integer, default=0) - # 模型分几段起草（不用模型时为 NULL）、已经结束了几段 + # 模型分几段起草（不用模型时为 NULL）、已经结束了几段、其中写出了内容并进了目录的段数（宽表每写出一段就并入目录， + # 停止、服务重启时已经写进去的留着） parts_total: Mapped[int | None] = mapped_column(Integer, default=None) parts_done: Mapped[int] = mapped_column(Integer, default=0) + parts_written: Mapped[int] = mapped_column(Integer, default=0) + # 块进度：这张表实际分成几块来问、已经结束了几块（段被缩小、拆成两半后会变多）；不用模型时为 NULL + chunks_total: Mapped[int | None] = mapped_column(Integer, default=None) + chunks_done: Mapped[int | None] = mapped_column(Integer, default=None) added: Mapped[int] = mapped_column(Integer, default=0) updated: Mapped[int] = mapped_column(Integer, default=0) removed: Mapped[int] = mapped_column(Integer, default=0) @@ -421,6 +426,8 @@ class CatalogDraftTask(Base): # 模型拿不准、留空了的列（中文名和含义都没给）：几列，最多记 5 个列名。表的状态不因此变 blank_columns: Mapped[int] = mapped_column(Integer, default=0) blank_sample: Mapped[list[Any] | None] = mapped_column(default=None) + # 起草时记下的几句话，不算失败（模型提议的有效记录条件没有收下的原因），最多 10 句 + log: Mapped[list[Any] | None] = mapped_column(default=None) # error：整张表没起草（表结构里没有、写入一直冲突、服务重启中断）；model_error：模型那部分没写出来的原因和办法 error: Mapped[str | None] = mapped_column(Text, default=None) model_error: Mapped[str | None] = mapped_column(Text, default=None) diff --git a/backend/app/engine/dual_check_facts.py b/backend/app/engine/dual_check_facts.py index da1d3d4e..6b9e077f 100644 --- a/backend/app/engine/dual_check_facts.py +++ b/backend/app/engine/dual_check_facts.py @@ -50,7 +50,9 @@ is_test = 0、is_valid = 1、deleted_at IS NULL）；只涉及一张表。拿不 原因，只有审批停下期间目录被人改了、数据源换了版本时才会和最后写的不一样：真写按写入时的目录、表结构和导入时认作 日期的列判断。组合太多的（JOINT_LIMIT）那一项不预告，卡上有条件地说。 -catalog.save_patch 不收备注，这里用它的同一套原料（read_entry、apply_patch、write_entry 的乐观锁）写，只多把备注放进项里。 +catalog.save_patch 把提案的依据另存成这一项确认时的依据（reason），这里要写的是备注里的出处（哪次运行、哪个节点）， +所以用它的同一套原料（read_entry、apply_patch、write_entry 的乐观锁）写，再把这一项整个换成带出处备注的人工确认项： +以前确认时记下的依据说的是旧值，不留。 """ from __future__ import annotations @@ -1051,11 +1053,11 @@ async def _write(session: Any, row: DataSource, fact: Fact, *, run_id: str, node note = note_of(kept, run_id=run_id, node_title=node_title) at = catalog.now_iso() try: - merged = catalog.apply_patch(notes, [{"path": kept.item, "value": value, "reason": note}], + merged = catalog.apply_patch(notes, [{"path": kept.item, "value": value}], table=kept.table, tables=tables, at=at) except catalog.CatalogInvalid as e: return None, [*reasons, f"「{kept.table}」的{label}未写入数据目录：{e}"] - # 不论原来是谁提的：这一项现在出自人的选择，记为人工填写、已确认，备注写明出处 + # 不论原来是谁提的：这一项现在出自人的选择，记为人工填写、已确认，备注写明出处（整项换掉：以前确认时的依据不留） merged[kept.item] = catalog.make_item(value, "human", "confirmed", note=note, at=at) try: await catalog.write_entry(session, row.id, kept.table, merged, if_version=version, actor=actor) diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py new file mode 100644 index 00000000..c5d0ede4 --- /dev/null +++ b/backend/tests/test_catalog_assistant.py @@ -0,0 +1,778 @@ +"""表助手（data/catalog_assistant.py，POST /api/datasources/{id}/catalog/{table}/assistant）。 + +- 开始前：表不在 404；起草模型用不了 409 catalog_draft_model_unavailable（说法同起草任务）；对话超过 40 条、一条超过 + 4000 字 422。界面上待保存的修改不设上限（宽表上攒到 200 项以上也照常对话）：背景里只列最近的 200 项、写明另有几项， + 码值照全部待保存的修改合并 +- 流：text 增量、query（跑过的只读查询，前 10 行，遮罩列写「已遮罩」）、proposal（按 plan_patch 核对过的提案，形状同助手流 + 的 catalog_patch：公式不收、列名必须在表结构里、遮罩列不给码值；有效记录条件要是只用这张表的列的一个条件，有起草的 + 校验规矩 sqlcheck.valid_filter_problem 时按它）、done（用量、查询条数）、error（中途出错，之后没有 done） +- 交给模型的背景：表结构、这张表的目录（每项的来源和状态）、关联关系（含别的表指过来的）、使用次数、数据源的起草要求、 + 最近一次起草留空的列、界面上还没保存的修改；遮罩列只给列名、不给任何值 +- 只读查询：只有开了数据剖析的源才给这个工具；经剖析的只读守卫和单条时限；每次请求最多 5 条；最多 50 行交给模型；结果 + 先遮再交给模型；没开剖析时提示词里写明不能查数据 +- 对话不存服务端；提案不写目录（保存走 PATCH，最多 200 项） + +表名、列名、业务数字都是虚构的；号码是 139000000xx 这样的虚构号码。 +""" +from __future__ import annotations + +import asyncio +import json +import sqlite3 +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessageChunk, HumanMessage, SystemMessage, ToolMessage + +from app.data import catalog, catalog_assistant +from app.data.catalog import make_item +from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION +from app.db.base import SessionLocal +from app.db.models import CatalogDraftJob, CatalogDraftTask, DataSource +from app.main import app as _fastapi_app +from tests.fixtures.sources import drop_source + +PHONES = [f"139000000{i:02d}" for i in range(1, 31)] + + +# -------------------------------------------------------------------------- +# 假模型：按剧本一轮一轮回 +# -------------------------------------------------------------------------- + + +@dataclass +class Turn: + text: list[str] = field(default_factory=list) + calls: list[tuple[str, dict]] = field(default_factory=list) + cut: bool = False + usage: tuple[int, int] = (100, 20) + error: BaseException | None = None + hang: bool = False + + +class Scripted: + """bind_tools 记下交给它的工具；astream 每调一次按剧本回一轮，记下这一轮收到的消息。""" + + def __init__(self, turns: list[Turn]) -> None: + self.turns = list(turns) + self.seen: list[list[Any]] = [] + self.tools: list[Any] | None = None + + def bind_tools(self, tools: list[Any], **_: Any) -> "Scripted": + self.tools = tools + return self + + async def astream(self, messages: list[Any]): + self.seen.append(list(messages)) + turn = self.turns.pop(0) + if turn.error is not None: + raise turn.error + if turn.hang: + await asyncio.sleep(3600) + for piece in turn.text: + yield AIMessageChunk(content=piece) + for i, (name, args) in enumerate(turn.calls): + yield AIMessageChunk(content="", tool_call_chunks=[ + {"name": name, "args": json.dumps(args, ensure_ascii=False), "id": f"c{len(self.seen)}_{i}", + "index": i}]) + yield AIMessageChunk(content="", usage_metadata={"input_tokens": turn.usage[0], "output_tokens": turn.usage[1], + "total_tokens": sum(turn.usage)}, + response_metadata={"finish_reason": "length" if turn.cut else "stop"}) + + def tool_names(self) -> list[str]: + return [t["function"]["name"] for t in self.tools or []] + + +class _Unavailable(Exception): + status_code = 503 + + +# -------------------------------------------------------------------------- +# 夹具 +# -------------------------------------------------------------------------- + + +def _shop_db(path) -> str: + conn = sqlite3.connect(path) + conn.executescript(""" + CREATE TABLE members (id INTEGER PRIMARY KEY, nick TEXT); + CREATE TABLE orders (id INTEGER PRIMARY KEY, order_no TEXT NOT NULL, status INTEGER NOT NULL, + amount REAL, buyer_phone TEXT, created_at TEXT, + member_id INTEGER REFERENCES members(id)); + """) + conn.executemany("INSERT INTO members VALUES (?, ?)", [(i, f"会员{i}") for i in range(1, 6)]) + conn.executemany("INSERT INTO orders VALUES (?, ?, ?, ?, ?, ?, ?)", [ + (i, f"NO{i:04d}", 9 if i % 10 == 0 else 1, 10.0 * i, PHONES[i - 1], f"2026-09-{i:02d}", i % 5 + 1) + for i in range(1, 31)]) + conn.commit() + conn.close() + return str(path) + + +ORDERS_NOTES = { + "label": make_item("订单", "llm"), + "grain": make_item("每笔订单一行", "human", "confirmed"), + "kind": make_item("fact", "llm", "rejected"), + "columns": { + "status": {"codes": make_item({"1": "已支付"}, "profile", note="数据剖析：全表 30 行，2 个取值")}, + "buyer_phone": {"label": make_item("联系电话", "llm"), + "codes": make_item({PHONES[0]: "测试号"}, "human", "confirmed")}, + }, +} + +_CREATED: list[str] = [] + + +@pytest.fixture(autouse=True) +async def _cleanup(): + yield + while _CREATED: + await drop_source(_CREATED.pop()) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c: + yield c + + +async def _source(client, tmp_path, *, profile: bool = True, guidance: str | None = "金额一律按元", + notes: dict | None = None) -> str: + options: dict[str, Any] = {"mask_columns": "buyer_phone"} + if profile: + options["catalog_profile"] = {"enabled": True} + if guidance: + options[CATALOG_DRAFT_GUIDANCE_OPTION] = guidance + async with SessionLocal() as session: + row = DataSource(name=f"shop_{uuid.uuid4().hex[:8]}", kind="sqlite", database=_shop_db(tmp_path / "shop.db"), + options=options, description="虚构的网店库") + session.add(row) + await session.commit() + sid = row.id + _CREATED.append(sid) + r = await client.post(f"/api/datasources/{sid}/introspect") + assert r.status_code == 200, r.text + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "orders", notes if notes is not None else ORDERS_NOTES, + if_version=0, actor=None) + return sid + + +def use(monkeypatch, model: Scripted) -> None: + async def resolve(session): + return model, "fake-chat" + + monkeypatch.setattr(catalog_assistant, "resolve_model", resolve) + + +def _url(sid: str, table: str = "orders") -> str: + return f"/api/datasources/{sid}/catalog/{table}/assistant" + + +async def _talk(client, sid: str, body: dict, table: str = "orders") -> tuple[list[dict], list[str]]: + """发一轮，返回 (事件, 原始行)。""" + events: list[dict] = [] + lines: list[str] = [] + async with client.stream("POST", _url(sid, table), json=body) as r: + assert r.status_code == 200, await r.aread() + assert r.headers["content-type"].startswith("text/event-stream") + async for line in r.aiter_lines(): + lines.append(line) + if line.startswith("data: "): + events.append(json.loads(line[6:])) + return events, lines + + +def _text(events: list[dict]) -> str: + return "".join(e["delta"] for e in events if e["type"] == "text") + + +def _ask(text: str, **extra) -> dict: + return {"messages": [{"role": "user", "content": text}], **extra} + + +# -------------------------------------------------------------------------- +# 开始前的检查 +# -------------------------------------------------------------------------- + + +async def test_refusals_before_the_stream(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + use(monkeypatch, Scripted([])) + assert (await client.post(_url(sid, "ghost"), json=_ask("你好"))).status_code == 404 + assert (await client.post("/api/datasources/nope/catalog/orders/assistant", json=_ask("x"))).status_code == 404 + too_many = {"messages": [{"role": "user", "content": "x"}] * 41} + assert (await client.post(_url(sid), json=too_many)).status_code == 422 + too_long = _ask("长" * 4001) + assert (await client.post(_url(sid), json=too_long)).status_code == 422 + wrong_role = {"messages": [{"role": "system", "content": "x"}]} + assert (await client.post(_url(sid), json=wrong_role)).status_code == 422 + last_is_answer = {"messages": [{"role": "user", "content": "x"}, {"role": "assistant", "content": "y"}]} + r = await client.post(_url(sid), json=last_is_answer) + assert r.status_code == 422 and "最后一条" in r.json()["detail"] + + +async def test_unavailable_draft_model_is_409_with_the_draft_wording(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + + async def unavailable(session): + raise catalog.CatalogModelUnavailable("未配置模型接入，无法由模型起草数据目录。请到「设置 → 模型接入」添加") + + monkeypatch.setattr(catalog, "resolve_draft_model", unavailable) + r = await client.post(_url(sid), json=_ask("你好")) + assert r.status_code == 409 + assert r.json()["code"] == "catalog_draft_model_unavailable" and "模型接入" in r.json()["detail"] + + +@pytest.fixture +async def mem(monkeypatch): + """单独的内存库：接入和设置只在这个用例里。""" + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + from sqlalchemy.pool import StaticPool + + from app.db.base import Base + + for key in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "OPENAI_API_KEY", "ANTHROPIC_BASE_URL"): + monkeypatch.delenv(key, raising=False) + engine = create_async_engine("sqlite+aiosqlite:///:memory:", poolclass=StaticPool) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + session = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)() + yield session + await session.close() + await engine.dispose() + + +async def _provider(session, kind: str) -> None: + from app.core.crypto import encrypt + from app.data import draft_model + from app.db.models import Provider, Setting + + session.add(Provider(name="起草网关", kind=kind, base_url="http://gw.example.test/v1" if kind != "anthropic" else None, + api_key=encrypt("sk-test"), models=[{"id": "模型甲"}], default_model="模型甲", enabled=True, + extra={})) + session.add(Setting(key=draft_model.CATALOG_DRAFT_SETTING_KEY, value={"provider": "起草网关", "model": None})) + await session.commit() + + +async def test_model_follows_the_draft_setting_keeps_thinking_and_a_big_budget(mem, monkeypatch): + from app.providers import factory + + await _provider(mem, "openai_compatible") + specs: list[Any] = [] + real = factory.build_chat_model + + def spy(provider, spec): + specs.append(spec) + return real(provider, spec) + + monkeypatch.setattr(factory, "build_chat_model", spy) + model, model_id = await catalog_assistant.resolve_model(mem) + assert model_id == "模型甲" and model.max_tokens == catalog.MODEL_MAX_TOKENS == 32768 + mine = specs[-2:] # 前两份是 catalog.resolve_draft_model 查能不能用时建的 + assert [s.max_tokens for s in mine] == [catalog.MODEL_MAX_TOKENS, catalog.MODEL_FALLBACK_MAX_TOKENS] + assert all(s.thinking is None and s.temperature == 0 and s.max_retries == 0 for s in mine) + assert all(s.timeout == catalog.MODEL_TIMEOUT_S for s in mine) + + +@pytest.mark.parametrize("kind", ["openai_compatible", "anthropic"]) +async def test_tools_bind_on_real_chat_models(mem, kind): + """两个工具的定义各家的模型都收（只绑定，不发请求）。""" + await _provider(mem, kind) + model, _ = await catalog_assistant.resolve_model(mem) + tools = [catalog_assistant.QUERY_TOOL_PROMPT, catalog_assistant.PROPOSE_TOOL_PROMPT] + bound = model.bind_tools(tools) + given = bound.model.kwargs["tools"] + names = [t.get("name") or t["function"]["name"] for t in given] + assert names == ["run_query", "propose_changes"] + + +# -------------------------------------------------------------------------- +# 背景 +# -------------------------------------------------------------------------- + + +async def _blank_draft(sid: str) -> None: + """最近一次结束于 ok 的起草任务记下留空的列；之后一次停止了的不算。""" + async with SessionLocal() as session: + done = CatalogDraftJob(id=uuid.uuid4().hex, source_id=sid, status="succeeded", + created_at=datetime(2026, 10, 1, tzinfo=timezone.utc)) + stopped = CatalogDraftJob(id=uuid.uuid4().hex, source_id=sid, status="cancelled", + created_at=datetime(2026, 10, 2, tzinfo=timezone.utc)) + session.add_all([done, stopped]) + await session.flush() + session.add_all([ + CatalogDraftTask(job_id=done.id, seq=0, table_name="orders", status="ok", blank_columns=2, + blank_sample=["order_no", "created_at"]), + CatalogDraftTask(job_id=stopped.id, seq=0, table_name="orders", status="cancelled", blank_columns=0), + ]) + await session.commit() + + +async def test_background_has_structure_catalog_and_hides_masked_values(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + await _blank_draft(sid) + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "members", {"relations": [ + {"id": catalog.relation_id("members", ["id"], "orders", ["member_id"]), "columns": ["id"], + "to_table": "orders", "to_columns": ["member_id"], "cardinality": "one_to_many", "coverage": None, + "source": "human", "status": "confirmed"}]}, if_version=0, actor=None) + model = Scripted([Turn(text=["这张表还缺 order_no 的含义。"])]) + use(monkeypatch, model) + pending = [{"path": "columns.amount.unit", "value": "元", "reason": "用户说：金额是元"}] + events, _ = await _talk(client, sid, _ask("看看还缺什么", pending=pending)) + assert _text(events) == "这张表还缺 order_no 的含义。" + assert events[-1] == {"type": "done", "usage": {"input_tokens": 100, "output_tokens": 20}, "queries": 0} + [messages] = model.seen + assert isinstance(messages[0], SystemMessage) and isinstance(messages[-1], HumanMessage) + system = messages[0].content + # 规矩 + for rule in ("只改数据目录", "不改业务库", "公式", "拿不准", "原话"): + assert rule in system + # 有效记录条件的写法（同起草时的校验规矩），免得提了再被退回 + assert "只能用这张表的列" in system and "不能有子查询" in system and "UPPER、LOWER、TRIM、COALESCE" in system + # 表结构、目录（来源和状态）、使用次数、起草要求、数据源说明 + for fact in ("order_no", "status", "amount", "member_id → members.id", "主键", "每笔订单一行", "人工填写", "已确认", + "模型起草", "已驳回", "还没有被运行用到过", "金额一律按元", "虚构的网店库", "数据剖析：全表 30 行"): + assert fact in system, fact + # 别的表指过来的关联关系 + assert "members.id → orders.member_id" in system + # 最近一次结束于 ok 的起草留空的列（之后停止了的那次不算） + assert "order_no、created_at" in system + # 界面上还没保存的修改 + assert "columns.amount.unit" in system and "用户说：金额是元" in system + # 遮罩列只给列名：码值、取值一个都不给 + assert "buyer_phone" in system + assert not any(p in system for p in PHONES) and "测试号" not in system + # 没开剖析的源才写不能查数据；这里开了，给了查询工具 + assert model.tool_names() == ["run_query", "propose_changes"] + assert "没开剖析" not in system + + +async def test_the_draft_guidance_is_not_what_the_user_said(client, tmp_path, monkeypatch): + """数据源的起草要求是以前写在数据源上的背景，不是这次对话里使用者说的话：背景里标明白；系统提示词写清 reason 里引用的 + 原话只能是这次对话里使用者的消息，依据来自起草要求时写「起草要求写着：……」。验收时模型把起草要求写成了 + 「使用者说：……」，保存之后看起来像是使用者亲口确认过。""" + sid = await _source(client, tmp_path, guidance="pay_type：1=现金，2=刷卡") + model = Scripted([Turn(text=["好。"])]) + use(monkeypatch, model) + await _talk(client, sid, _ask("看看还缺什么")) + system = model.seen[0][0].content + assert "数据源的起草要求（不是这次对话里使用者说的话）：pay_type：1=现金，2=刷卡" in system + assert "使用者补充的背景和要求" not in system + assert "只能引用这次对话里使用者的消息" in system and "起草要求写着：" in system + reason = (catalog_assistant.PROPOSE_TOOL_PROMPT["function"]["parameters"]["properties"]["changes"]["items"] + ["properties"]["reason"]["description"]) + assert "这次对话里使用者" in reason and "起草要求写着" in reason + + +async def test_reasons_saved_earlier_are_not_what_the_user_said(client, tmp_path, monkeypatch): + """目录项上以前保存时记下的依据（reason）不是这次对话里使用者说的话：背景里在这一项下面标成「以前确认时的依据：……」， + 和备注分开；系统提示词、propose_changes 的说明写清 reason 里引用的原话只能是这次对话里使用者的消息，来自以前的依据时写 + 「以前确认时的依据写着：……」。说到遮罩列的项不给依据（同备注：里面可能有取值）。""" + notes = {**ORDERS_NOTES, + "grain": make_item("每笔订单一行", "human", "confirmed", reason="使用者说：一笔订单一行"), + "columns": {**ORDERS_NOTES["columns"], + "buyer_phone": {"label": make_item("联系电话", "human", "confirmed", + reason=f"使用者说：{PHONES[1]} 是客服电话")}}} + sid = await _source(client, tmp_path, notes=notes) + model = Scripted([Turn(text=["好。"])]) + use(monkeypatch, model) + await _talk(client, sid, _ask("看看还缺什么")) + system = model.seen[0][0].content + assert "- grain（粒度）：每笔订单一行〔人工填写，已确认〕\n 以前确认时的依据：使用者说：一笔订单一行" in system + # 备注照旧另起一行，不和依据混在一起 + assert " 备注：数据剖析：全表 30 行，2 个取值" in system and "备注：使用者说" not in system + assert PHONES[1] not in system and "客服电话" not in system + assert "只能引用这次对话里使用者的消息" in system and "以前确认时的依据写着：" in system + reason = (catalog_assistant.PROPOSE_TOOL_PROMPT["function"]["parameters"]["properties"]["changes"]["items"] + ["properties"]["reason"]["description"]) + assert "以前确认时的依据写着" in reason + + +async def test_background_says_how_many_runs_used_the_table(client, tmp_path, monkeypatch): + """使用次数的口径同审阅队列：被几次运行用到（同「用过这张表的运行」的 total），查询几次（同目录清单的「使用 N 次」）。""" + from app.db.models import Artifact, Run + + sid = await _source(client, tmp_path) + async with SessionLocal() as session: + name = (await session.get(DataSource, sid)).name + for i, n in enumerate((2, 1)): + rid = uuid.uuid4().hex + session.add(Run(id=rid, graph={}, workflow_name=f"周报{i}", status="succeeded")) + for _ in range(n): + session.add(Artifact(id=uuid.uuid4().hex, run_id=rid, kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": ["orders"]})) + await session.commit() + model = Scripted([Turn(text=["好。"])]) + use(monkeypatch, model) + await _talk(client, sid, _ask("看看")) + assert "这张表被 2 次运行用到（查询 3 次）。" in model.seen[0][0].content + + +async def test_long_catalogs_are_cut_with_a_note(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + monkeypatch.setattr(catalog_assistant, "CATALOG_ITEMS_SHOWN", 2) + model = Scripted([Turn(text=["好。"])]) + use(monkeypatch, model) + await _talk(client, sid, _ask("看看")) + system = model.seen[0][0].content + # 表级的项在前；遮罩列的码值本来就不算；剩下的说明还有几项 + assert "- label（中文名）：订单" in system and "- grain（粒度）" in system and "- kind（表类型）" not in system + assert "还有 3 项没有列出" in system + + +async def test_without_profiling_there_is_no_query_tool(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path, profile=False) + model = Scripted([Turn(text=["只能根据表结构推断。"])]) + use(monkeypatch, model) + await _talk(client, sid, _ask("状态 9 是什么")) + assert model.tool_names() == ["propose_changes"] + assert "这个数据源没开剖析，不能查数据，只能根据表结构和目录推断" in model.seen[0][0].content + + +async def test_opening_without_user_messages(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + model = Scripted([Turn(text=["order_no 是什么意思？"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, {"messages": []}) + assert _text(events) == "order_no 是什么意思？" + assert isinstance(model.seen[0][-1], HumanMessage) + # 界面先显示了一句开场白（助手说的）：照样从使用者这一边开始 + model = Scripted([Turn(text=["好的。"])]) + use(monkeypatch, model) + await _talk(client, sid, {"messages": [{"role": "assistant", "content": "有两列留空了"}, + {"role": "user", "content": "order_no 是订单号"}]}) + kinds = [type(m).__name__ for m in model.seen[0]] + assert kinds == ["SystemMessage", "HumanMessage", "AIMessage", "HumanMessage"] + + +# -------------------------------------------------------------------------- +# 提案 +# -------------------------------------------------------------------------- + + +async def test_proposals_are_checked_and_never_written(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + changes = [ + {"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说：9 是作废"}, + {"path": "valid_filter", "value": "status <> 9", "reason": "用户说：统计时排除作废"}, + {"path": "columns.amount.meaning", "value": "实收=单价*数量", "reason": "x"}, # 公式：不收 + {"path": "columns.ghost.label", "value": "幽灵", "reason": "x"}, # 列不存在：不收 + {"path": "columns.buyer_phone.codes", "value": {PHONES[1]: "客服"}, "reason": "x"}, # 遮罩列：不给码值 + {"path": "grain", "value": "每笔订单一行", "reason": "x"}, # 已经是这个值：不用提 + ] + model = Scripted([Turn(text=["好的，"], calls=[("propose_changes", {"changes": changes})]), + Turn(text=["已经整理成提案。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("9 是作废，统计时排除")) + assert _text(events) == "好的，已经整理成提案。" + [proposal] = [e for e in events if e["type"] == "proposal"] + assert proposal["source_id"] == sid and proposal["table"] == "orders" and proposal["version"] == 1 + assert proposal["source"].startswith("shop_") and proposal["table_label"] == "订单" + by = {c["path"]: c for c in proposal["changes"]} + assert set(by) == {"columns.status.codes", "valid_filter"} + codes = by["columns.status.codes"] + assert codes["label"] == "列 status 的码值" and codes["reason"] == "用户说：9 是作废" + assert codes["before"] == {"1": "已支付"} and codes["after"] == {"1": "已支付", "9": "作废"} + assert codes["value"] == {"9": "作废"} and codes["state"] == "change" + assert by["valid_filter"]["label"] == "表的有效记录条件" and by["valid_filter"]["before"] is None + # 没收的几项照实告诉模型（不带遮罩列的任何取值） + tool = model.seen[1][-1] + assert isinstance(tool, ToolMessage) + for said in ("2 项", "公式", "ghost", "遮罩", "已经是这个值"): + assert said in tool.content, said + assert not any(p in tool.content for p in PHONES) + assert events[-1]["type"] == "done" and events[-1]["usage"] == {"input_tokens": 200, "output_tokens": 40} + async with SessionLocal() as session: + entry = await catalog.read_entry(session, sid, "orders") + assert entry.version == 1 and "valid_filter" not in entry.notes + + +async def test_proposal_codes_build_on_the_pending_ones(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + model = Scripted([Turn(calls=[("propose_changes", {"changes": [ + {"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说：9 是作废"}]})]), + Turn(text=["好。"])]) + use(monkeypatch, model) + pending = [{"path": "columns.status.codes", "value": {"2": "已退款"}, "reason": "用户说：2 是退款"}] + events, _ = await _talk(client, sid, _ask("9 是作废", pending=pending)) + [proposal] = [e for e in events if e["type"] == "proposal"] + [change] = proposal["changes"] + assert change["value"] == {"2": "已退款", "9": "作废"} + assert change["after"] == {"1": "已支付", "2": "已退款", "9": "作废"} + + +async def test_many_pending_changes_do_not_block_the_conversation(client, tmp_path, monkeypatch): + """宽表上逐列补中文名、含义，待保存的修改攒到 200 项以上：照常对话（不 422）。背景里只列最近的 200 项、写明另有几项； + 码值照全部待保存的修改合并（最早那条码值没列进背景，新提的码值也不能把它丢了）。""" + sid = await _source(client, tmp_path) + oldest = {"path": "columns.status.codes", "value": {"2": "已退款"}, "reason": "用户说：2 是退款"} + labels = [{"path": f"columns.c{i:03d}.label", "value": f"第 {i} 列", "reason": f"用户说：第 {i} 列"} + for i in range(201)] + model = Scripted([Turn(calls=[("propose_changes", {"changes": [ + {"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说：9 是作废"}]})]), + Turn(text=["好。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("9 是作废", pending=[oldest, *labels])) + assert events[-1]["type"] == "done" + system = model.seen[0][0].content + # 最近的 200 项列出来了，最早的两项没列，写明另有 2 项 + assert "columns.c200.label" in system and "columns.c001.label" in system + assert "columns.c000.label" not in system and "用户说：2 是退款" not in system + assert "另有 2 项" in system + [proposal] = [e for e in events if e["type"] == "proposal"] + [change] = proposal["changes"] + assert change["value"] == {"2": "已退款", "9": "作废"} + # 再多也收 + model = Scripted([Turn(text=["好。"])]) + use(monkeypatch, model) + many = [{"path": f"columns.c{i:04d}.meaning", "value": "说明", "reason": "x"} for i in range(1200)] + events, _ = await _talk(client, sid, _ask("接着看", pending=many)) + assert events[-1]["type"] == "done" and "另有 1000 项" in model.seen[0][0].content + + +async def _propose_filters(client, sid: str, monkeypatch, filters: list[str]) -> tuple[list[dict], Scripted]: + """模型一轮提一条有效记录条件（同一条提案里同一个路径只认第一次），最后一轮回答。""" + turns = [Turn(calls=[("propose_changes", {"changes": [ + {"path": "valid_filter", "value": f, "reason": "用户说：统计时排除作废"}]})]) for f in filters] + model = Scripted([*turns, Turn(text=["好。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("统计时排除作废的单子")) + return events, model + + +async def test_valid_filter_proposals_must_be_a_condition_on_this_table(client, tmp_path, monkeypatch): + """有效记录条件是模型写的 SQL：引用表结构里没有的列、别的表的列、带子查询的不交给界面，原因交回模型；合规的照常提。""" + sid = await _source(client, tmp_path) + events, model = await _propose_filters(client, sid, monkeypatch, [ + "status <> 9 AND ghost_col = 1", + "members.nick IS NOT NULL", + "member_id IN (SELECT id FROM members)", + "status <> 9", + ]) + proposals = [e for e in events if e["type"] == "proposal"] + assert [[c["value"] for c in p["changes"]] for p in proposals] == [["status <> 9"]] + said = [[m for m in seen if isinstance(m, ToolMessage)][-1].content for seen in model.seen[1:5]] + assert "ghost_col" in said[0] and "不在表结构里" in said[0] and "没有可以交给使用者的修改" in said[0] + assert "members.nick" in said[1] + assert "子查询" in said[2] + assert "已把 1 项修改交给使用者确认" in said[3] + + +async def test_valid_filter_proposals_follow_the_drafting_rules(client, tmp_path, monkeypatch): + """有起草的校验规矩（sqlcheck.valid_filter_problem：只能用几个函数、不能有聚合……）时按它核对，说法照它的。""" + from app.data import sqlcheck + + sid = await _source(client, tmp_path) + calls: list[tuple] = [] + + def strict(condition, meta, *, kind, table=None): + calls.append((condition, [c["name"] for c in meta["columns"]], kind, table)) + return "有效行过滤里只能用 UPPER、LOWER、TRIM、COALESCE 这几个函数，不能用 ABS" if "ABS" in condition else None + + monkeypatch.setattr(sqlcheck, "valid_filter_problem", strict, raising=False) + events, model = await _propose_filters(client, sid, monkeypatch, ["ABS(amount) > 0", "status <> 9"]) + proposals = [e for e in events if e["type"] == "proposal"] + assert [[c["value"] for c in p["changes"]] for p in proposals] == [["status <> 9"]] + assert "不能用 ABS" in [m for m in model.seen[1] if isinstance(m, ToolMessage)][-1].content + assert calls[0][0] == "ABS(amount) > 0" and calls[0][2:] == ("sqlite", "orders") + assert "status" in calls[0][1] and "member_id" in calls[0][1] + + +async def test_nothing_acceptable_means_no_proposal_event(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + model = Scripted([Turn(calls=[("propose_changes", {"changes": [{"path": "columns.ghost.label", "value": "x"}]})]), + Turn(text=["那一列不存在。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("ghost 列叫什么")) + assert not [e for e in events if e["type"] == "proposal"] + assert "没有可以交给使用者的修改" in model.seen[1][-1].content + + +# -------------------------------------------------------------------------- +# 只读查询 +# -------------------------------------------------------------------------- + + +async def test_queries_are_read_only_masked_and_capped(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + queries = [ + ("run_query", {"sql": "SELECT status, COUNT(*) AS n FROM orders GROUP BY status ORDER BY status"}), + ("run_query", {"sql": "SELECT id, buyer_phone AS phone, amount FROM orders ORDER BY id"}), + ("run_query", {"sql": "DELETE FROM orders"}), + ] + model = Scripted([Turn(text=["我查一下。"], calls=queries), + Turn(calls=[("run_query", {"sql": "SELECT 1"})] * 3), + Turn(text=["9 占一成。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("状态分布")) + shown = [e for e in events if e["type"] == "query"] + assert len(shown) == 5 # 每次请求最多 5 条 + first, second, third = shown[:3] + assert first["columns"] == ["status", "n"] and first["rows"] == [[1, 27], [9, 3]] + assert first["row_count"] == 2 and first["error"] is None + # 遮罩列改了名字也认得出：值写「已遮罩」；界面只给前 10 行 + assert second["columns"] == ["id", "phone", "amount"] and len(second["rows"]) == 10 + assert {row[1] for row in second["rows"]} == {"已遮罩"} and second["row_count"] == 30 + # 只读守卫：写语句不执行 + assert third["error"] and "守卫" in third["error"] and third["rows"] == [] + # 交给模型的：最多 50 行、遮罩值看不到；第 6 条没执行 + second_round = model.seen[1] + results = [m for m in second_round if isinstance(m, ToolMessage)] + assert len(results) == 3 + assert json.loads(results[1].content)["rows"][29][1] == "已遮罩" + assert not any(p in m.content for m in second_round for p in PHONES) + capped = [m for m in model.seen[2] if isinstance(m, ToolMessage)][-1] + assert "未执行" in capped.content and "5 条" in capped.content + assert events[-1] == {"type": "done", "usage": {"input_tokens": 300, "output_tokens": 60}, "queries": 5} + with sqlite3.connect(tmp_path / "shop.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM orders").fetchone()[0] == 30 + + +async def test_query_results_given_to_the_model_stop_at_50_rows(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + sql = "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 80) SELECT i FROM n" + model = Scripted([Turn(calls=[("run_query", {"sql": sql})]), Turn(text=["好。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("数一下")) + [query] = [e for e in events if e["type"] == "query"] + assert query["row_count"] == 50 and query["truncated"] is True and len(query["rows"]) == 10 + result = json.loads([m for m in model.seen[1] if isinstance(m, ToolMessage)][0].content) + assert len(result["rows"]) == 50 and result["truncated"] is True + + +# -------------------------------------------------------------------------- +# 中途出错 +# -------------------------------------------------------------------------- + + +async def test_errors_end_the_stream_without_done(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + use(monkeypatch, Scripted([Turn(error=RuntimeError("接口不认这个模型"))])) + events, _ = await _talk(client, sid, _ask("你好")) + assert events[-1]["type"] == "error" and events[-1]["code"] == "assistant_model_error" + assert "fake-chat" in events[-1]["message"] and not [e for e in events if e["type"] == "done"] + use(monkeypatch, Scripted([Turn(text=["写到一半"], cut=True)])) + events, _ = await _talk(client, sid, _ask("你好")) + assert _text(events) == "写到一半" + assert events[-1]["type"] == "error" and events[-1]["code"] == "assistant_truncated" + assert "上限" in events[-1]["message"] + + +async def test_a_model_without_tools_reports_an_error_event(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + + class NoTools(Scripted): + def bind_tools(self, tools, **_): + raise NotImplementedError("这个模型不支持工具调用") + + use(monkeypatch, NoTools([])) + events, _ = await _talk(client, sid, _ask("你好")) + assert events == [{"type": "error", "code": "assistant_failed", "message": events[0]["message"]}] + assert "工具调用" in events[0]["message"] + + +async def test_transient_errors_are_retried_before_any_output(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + monkeypatch.setattr(catalog_assistant, "RETRY_DELAYS_S", (0.0, 0.0)) + model = Scripted([Turn(error=_Unavailable("服务暂时不可用")), Turn(text=["好了。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("你好")) + assert _text(events) == "好了。" and events[-1]["type"] == "done" + assert len(model.seen) == 2 + + +async def test_a_call_over_the_time_limit_is_stopped(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.3) + monkeypatch.setattr(catalog_assistant, "HEARTBEAT_S", 0.1) + use(monkeypatch, Scripted([Turn(hang=True)])) + events, lines = await _talk(client, sid, _ask("你好")) + assert events == [events[-1]] and events[-1]["code"] == "assistant_timeout" + assert "0.3 秒" in events[-1]["message"] + # 等模型的时候按拍发 SSE 注释（不是事件），免得中间的代理断开 + assert any(line.startswith(":") for line in lines) + + +async def test_steps_are_bounded(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + monkeypatch.setattr(catalog_assistant, "MAX_ROUNDS", 3) + again = Turn(calls=[("propose_changes", {"changes": [{"path": "columns.ghost.label", "value": "x"}]})]) + model = Scripted([again, again, again]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("你好")) + assert len(model.seen) == 3 + assert events[-1]["type"] == "error" and events[-1]["code"] == "assistant_rounds" + # 倒数第二轮的工具结果里提醒模型直接回答 + assert "直接回答" in [m for m in model.seen[2] if isinstance(m, ToolMessage)][-1].content + + +class _Slow(Scripted): + """说了半句就一直想：记下这次调用有没有被停掉。""" + + def __init__(self) -> None: + super().__init__([]) + self.cancelled = False + + async def astream(self, messages: list[Any]): + self.seen.append(list(messages)) + yield AIMessageChunk(content="我想一想") + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + self.cancelled = True + raise + yield AIMessageChunk(content="……") + + +async def test_stopping_the_stream_stops_the_model_call(): + """使用者点停止（连接断开、流被关掉）：正在进行的模型调用跟着停，不在后台接着花额度。""" + from app.data import catalog_profile + + model = _Slow() + ctx = catalog_assistant.AssistantContext( + source_id="s", source_name="虚构库", kind="sqlite", table="orders", tables={}, query_source=None, options={}, + masked=set(), can_query=False, settings=catalog_profile.ProfileSettings(), system="系统提示") + events = catalog_assistant.stream(model, "fake-chat", ctx, [{"role": "user", "content": "你好"}]) + first = await events.__anext__() + assert json.loads(first[len("data: "):]) == {"type": "text", "delta": "我想一想"} + await events.aclose() + assert model.cancelled + + +# -------------------------------------------------------------------------- +# 保存：PATCH 一次最多 200 项 +# -------------------------------------------------------------------------- + + +async def test_patch_accepts_up_to_200_changes(client): + """表助手一次能提很多项（逐列补中文名），保存不再卡在 20 项：预览、保存都收 200 项，201 项 422。""" + assert catalog.PATCH_MAX_CHANGES == 200 + columns = [f"c{i:03d}" for i in range(200)] + table = {"qualified": "wide", "primary_key": [], "comment": None, "is_view": False, "foreign_keys": [], + "unique": [], "columns": [{"name": c, "type": "TEXT", "nullable": True, "comment": None} for c in columns]} + async with SessionLocal() as session: + row = DataSource(name=f"wide_{uuid.uuid4().hex[:8]}", kind="sqlite", database=":memory:", + schema_cache={"tables": {"wide": table}}) + session.add(row) + await session.commit() + sid = row.id + _CREATED.append(sid) + base = f"/api/datasources/{sid}/catalog/wide" + changes = [{"path": f"columns.{c}.label", "value": f"第 {i} 列", "reason": "用户逐列说明"} + for i, c in enumerate(columns)] + preview = await client.post(f"{base}/patch/preview", json={"changes": changes}) + assert preview.status_code == 200, preview.text + assert len(preview.json()["changes"]) == 200 and preview.json()["problems"] == [] + saved = await client.post(f"{base}/patch", json={"changes": changes, "if_version": 0}) + assert saved.status_code == 200, saved.text + assert len(saved.json()["notes"]["columns"]) == 200 + assert saved.json()["notes"]["columns"]["c199"]["label"]["value"] == "第 199 列" + too_many = changes + [{"path": "label", "value": "宽表"}] + assert (await client.post(f"{base}/patch/preview", json={"changes": too_many})).status_code == 422 + assert (await client.post(f"{base}/patch", json={"changes": too_many, "if_version": 1})).status_code == 422 diff --git a/backend/tests/test_catalog_draft_engine.py b/backend/tests/test_catalog_draft_engine.py index 4a1a94aa..39438de0 100644 --- a/backend/tests/test_catalog_draft_engine.py +++ b/backend/tests/test_catalog_draft_engine.py @@ -254,6 +254,7 @@ async def test_hollow_shell_splits_the_pack_by_table(): async def test_splitting_stops_at_three_levels_and_five_columns(monkeypatch): monkeypatch.setattr(catalog, "MODEL_PART_MAX_CALLS", 99) + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) # 不让按比例止损先停下来 tables = {"wide40": _table(40)} model = Scripted(lambda call: EXHAUSTED) got = await D.run_model_draft(model, _source(tables), ["wide40"]) @@ -272,7 +273,7 @@ async def test_splitting_stops_at_three_levels_and_five_columns(monkeypatch): async def test_one_part_makes_at_most_eight_calls(monkeypatch): """一段（连同它拆出来的、补问的、重试的）最多调用 8 次：40 列一直用完额度，拆到第 8 次就停，剩下没问的记没写出来、 原因写清。以前一段 40 列拆到底要 15 次。""" - monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99) + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) assert catalog.MODEL_PART_MAX_CALLS == 8 model = Scripted(lambda call: EXHAUSTED) got = await D.run_model_draft(model, _source({"t40": _table(40)}), ["t40"]) @@ -755,6 +756,7 @@ async def test_a_wide_table_with_one_failed_part_is_partial_and_says_which(): async def test_reasons_name_each_part_and_say_these_parts(monkeypatch): """几段因为同一个原因没写出来：原因里写「这几段」；每段写上列的范围（列名照表结构原样）；拆过的才补「已拆小重试」。""" monkeypatch.setattr(catalog, "MODEL_SHRINK_AFTER", 99) + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) tables = {"wide95": _table(95)} ok = answering(tables) @@ -827,33 +829,38 @@ async def test_tables_are_packed_up_to_40_columns_and_4_tables(): async def test_the_same_reason_is_said_once_even_when_some_parts_were_split(monkeypatch): - """宽表的表级那段拆不开、列分段拆过：原因只说一次，拆过的在最后补一句。""" - monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99) + """宽表的表级那段拆不开、列分段拆过：原因只说一次，「已拆小重试」只跟在真拆过的那几段后面说。""" + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) monkeypatch.setattr(catalog, "MODEL_PART_MAX_CALLS", 99) model = Scripted(_hollow) got = await D.run_model_draft(model, _source({"wide45": _table(45)}), ["wide45"]) o = got["wide45"] assert o.status == "failed" assert o.error == ("这张表 45 列，分 3 段起草，第 1 段（表级的项）、第 2 段（c001 到 c023，共 23 列）、" - "第 3 段（c024 到 c045，共 22 列）都没写出来：模型只给了空壳，没有写出可用的内容；已拆小重试仍没写出") + "第 3 段（c024 到 c045，共 22 列）都没写出来：模型只给了空壳，没有写出可用的内容；" + "其中第 2、3 段已拆小重试仍没写出") def test_reasons_are_joined_by_their_base(): - """(原因, 第几段)：同样的原因只说一次；同一个原因落在几段上写「这几段」，同一段拆出来的几块仍是「这一段」。""" + """(原因, 第几段)：同样的原因只说一次；同一个原因落在几段上写「这几段」，同一段拆出来的几块仍是「这一段」。 + 「已拆小重试仍没写出」只跟在真拆过的那几段后面说：说到的段都拆过时不点名，只有一部分拆过时写「其中第几段」。""" W = D._Why ex = D.REASON_EXHAUSTED.format(budget=32768, seg=D.SEG) assert D._join([(W(D.REASON_HOLLOW), 1), (W(D.REASON_HOLLOW, split=True), 2)]) == \ - D.REASON_HOLLOW + D.REASON_SPLIT + D.REASON_HOLLOW + "；其中第 2 段已拆小重试仍没写出" assert D._join([(W(ex, split=True), 2), (W(D.REASON_HOLLOW), 1)]) == \ - f"{_exhausted()}；{D.REASON_HOLLOW}{D.REASON_SPLIT}" + f"{_exhausted()}；{D.REASON_HOLLOW}；其中第 2 段已拆小重试仍没写出" + assert D._join([(W(ex, split=True), 2), (W(ex, split=True), 3)]) == _exhausted("这几段") + D.REASON_SPLIT + assert D._join([(W(ex, split=True), 3), (W(ex), 4), (W(ex, split=True), 2)]) == \ + _exhausted("这几段") + "；其中第 2、3 段已拆小重试仍没写出" assert D._join([(W(ex), 2), (W(ex), 3)]) == _exhausted("这几段") - assert D._join([(W(ex), 2), (W(ex, split=True), 2)]) == _exhausted() + D.REASON_SPLIT + assert D._join([(W(ex), 2), (W(ex, split=True), 2)]) == _exhausted() + D.REASON_SPLIT, "同一段里有一块拆过就算拆过" assert D._join([(W("甲"), 1), (W("乙"), 1), (W("丙", split=True), 1)]) == "甲；乙", "没说出来的原因不补后缀" - halted = D.REASON_HALTED.format(n=catalog.MODEL_GIVE_UP_AFTER) + halted = D.REASON_HALTED.format(window=catalog.MODEL_HALT_WINDOW, on=catalog.MODEL_HALT_ON) + assert halted == "；最近 10 次调用有 7 次以上没写出来，已停止拆小重试，建议更换起草模型" assert D._join([(W(D.REASON_HOLLOW, halted=halted), 1), (W(D.REASON_HOLLOW), 2)]) == D.REASON_HOLLOW + halted - slow = D.REASON_SLOW.format(n=catalog.MODEL_MIN_PART_COLUMNS) - assert D._join([(W(D.REASON_HOLLOW, halted=halted), 1), (W(D.REASON_HOLLOW, halted=slow), 2), - (W(D.REASON_HOLLOW, halted=halted), 3)]) == D.REASON_HOLLOW + halted + slow, "两种止损各说一次" + assert D._join([(W(D.REASON_HOLLOW, halted=halted), 1), (W(D.REASON_HOLLOW, halted=halted), 3)]) == \ + D.REASON_HOLLOW + halted, "止损只说一次" assert D._join([(W("调用模型出错：{坏的}"), 1)]) == "调用模型出错：{坏的}", "报错原文里的花括号原样留着" @@ -865,50 +872,133 @@ def _by_table(model: Scripted) -> dict[str, int]: return out -async def test_after_two_segments_in_a_row_come_to_nothing_the_rest_are_not_split(monkeypatch): - """模型明显用不了（思考总是吃满额度）：连续 2 段拆到底仍一项没写出，后面的段只问一次、不再拆，报错说明已停止拆小重试。 - 有一段写出来了就恢复拆小。一次只起草一段（并发 1），先后是确定的。""" +HALTED = "；最近 10 次调用有 7 次以上没写出来，已停止拆小重试，建议更换起草模型" + + +async def test_when_most_recent_calls_fail_the_rest_are_asked_once_until_the_ratio_drops(monkeypatch): + """按比例止损：这次起草最近 10 次调用有 7 次以上没写出来，之后的段不再拆小、每块只问一次，报错说明已停止拆小重试； + 比例降下去（最近 10 次里少于 7 次）就恢复拆小。偶尔一次写出来不会清零：t4 到 t6 写出来了，最近 10 次里仍有 7 次没写 + 出来，照旧不拆；t7 写出来以后才降下去，t8 照常拆小。一次只起草一段（并发 1），先后是确定的。""" monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) - assert catalog.MODEL_GIVE_UP_AFTER == 2 - tables = {f"t{i}": _table(10) for i in range(1, 5)} - exhausted = Reply(text="", finish="length", thinking="……") - - model = Scripted(lambda call: exhausted) - got = await D.run_model_draft(model, _source(tables), list(tables)) - assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 1} - budget = _exhausted() - assert got["t1"].error == budget + D.REASON_SPLIT - assert got["t3"].error == budget + "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型" - + assert (catalog.MODEL_HALT_WINDOW, catalog.MODEL_HALT_ON) == (10, 7) + tables = {f"t{i}": _table(10) for i in range(1, 9)} ok = answering(tables) - model = Scripted(lambda call: ok(call) if "t3" in call.asked else exhausted) + bad = {"t1", "t2", "t3", "t8"} + + model, reports = Scripted(lambda call: EXHAUSTED if set(call.asked) & bad else ok(call)), Reports() + got = await D.run_model_draft(model, _source(tables), list(tables), hooks=reports.hooks()) + assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 1, "t5": 1, "t6": 1, "t7": 1, "t8": 3} + assert got["t1"].error == _exhausted() + D.REASON_SPLIT + assert got["t3"].error == _exhausted() + HALTED + assert {got[f"t{i}"].status for i in range(4, 8)} == {"ok"} + assert got["t8"].error == _exhausted() + D.REASON_SPLIT, "比例降下去以后照常拆小" + assert len(reports.items) == len(model.calls) + + +async def test_while_halted_a_chunk_is_not_asked_again_for_missing_columns(monkeypatch): + """止损期间每块只问一次：回复里漏了几列不补问，漏的列记没写出来、说明已停止拆小重试。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + tables = {**{f"x{i}": _table(4) for i in range(1, 8)}, "t8": _table(8)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + if "t8" not in call.asked: + raise RuntimeError("网关拒绝了这次请求") + reply = ok(call) + reply.obj["tables"][0]["columns"] = reply.obj["tables"][0]["columns"][:6] + return reply + + model = Scripted(handle) got = await D.run_model_draft(model, _source(tables), list(tables)) - assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 3}, "t3 写出来了，t4 照常拆小" - assert got["t3"].status == "ok" and got["t4"].error == budget + D.REASON_SPLIT + assert _by_table(model)["t8"] == 1, "漏了 c007、c008 也不补问" + o = got["t8"] + assert o.status == "partial" + assert o.error == ("这张表 8 列，有 2 列（c007、c008）没写出来：模型的回复里漏了这些列" + HALTED + + "。可以只重试这张表") + + +#: 200 列的宽表分 6 段起草（表级一段、列分段 5 段），逐段写上列的范围 +WIDE200_PARTS = ("这张表 200 列，分 6 段起草，第 1 段（表级的项）、第 2 段（c001 到 c040，共 40 列）、" + "第 3 段（c041 到 c080，共 40 列）、第 4 段（c081 到 c120，共 40 列）、第 5 段（c121 到 c160，共 40 列）、" + "第 6 段（c161 到 c200，共 40 列）") + + +async def test_a_wide_table_skips_its_remaining_parts_after_three_dead_parts_in_a_row(monkeypatch): + """同一张表连续 3 个原始段都一项没写出来：这张表剩下的段跳过不问，原因写清跳过了几段、可以换一个起草模型后只重试 + 这张表。200 列分 6 段（表级一段、列分段 5 段）。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + assert catalog.MODEL_TABLE_GIVE_UP_AFTER == 3 + tables = {"wide200": _table(200)} + + def refuse(call: Call) -> Reply: + raise RuntimeError("网关拒绝") + + model = Scripted(refuse) + got = await D.run_model_draft(model, _source(tables), ["wide200"]) + o = got["wide200"] + assert len(model.calls) == 3, "第 4 到 6 段跳过不问" + assert (o.status, o.parts_total, o.parts_done) == ("failed", 6, 6) + assert o.error == (WIDE200_PARTS + "都没写出来：调用模型出错：网关拒绝；" + "这张表接连 3 段都没写出来，剩下的 3 段已跳过，可以换一个起草模型后只重试这张表") + + # 表级那段写出来了、后面接连 3 段没写出来：只写出一部分，剩下的 2 段跳过；不再另说「可以只重试这张表」 + ok = answering(tables) + model = Scripted(lambda call: ok(call) if call.n == 0 else refuse(call)) + got = await D.run_model_draft(model, _source(tables), ["wide200"]) + o = got["wide200"] + assert len(model.calls) == 4 and o.status == "partial" + assert o.error == ("这张表 200 列，分 6 段起草，第 2 段（c001 到 c040，共 40 列）、第 3 段（c041 到 c080，共 40 列）、" + "第 4 段（c081 到 c120，共 40 列）、第 5 段（c121 到 c160，共 40 列）、第 6 段（c161 到 c200，共 40 列）" + "没写出来：调用模型出错：网关拒绝；这张表接连 3 段都没写出来，剩下的 2 段已跳过，" + "可以换一个起草模型后只重试这张表") + + +async def test_a_part_that_writes_resets_the_dead_parts_of_its_table(monkeypatch): + """中间有一段写出来了，就不算「接连」：200 列的第 2、3、5、6 段没写出来、第 4 段写出来了，每段都照常问。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + tables = {"wide200": _table(200)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + part, cols = call.asked["wide200"] + if part == "table" or "c081" in (cols or []): + return ok(call) + raise RuntimeError("网关拒绝") + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["wide200"]) + assert len(model.calls) == 6 and got["wide200"].status == "partial" + assert "已跳过" not in got["wide200"].error async def test_the_column_parts_of_a_wide_table_stop_splitting_too(monkeypatch): - """宽表的列分段一段一段排队起草（一段拆到底再起草下一段），止损在宽表里同样起作用。 - - 95 列：表级那段拆不开（结构化、纯文本各一次）；第 2 段 32 列拆小、到 8 次封顶；这时已经连续 2 段没写出来，第 3、4 段 - 只问一次、换纯文本一次。""" + """按比例止损在宽表里同样起作用，接连没写出来的宽表剩下的段跳过：95 列一直只给空壳（已经走纯文本），第 2 段拆小到 + 止损；之后的第 3 段只问一次；接连 3 段没写出来，第 4 段跳过。""" monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) - model = Scripted(_hollow) + model = Scripted(_text_only(_hollow)) got = await D.run_model_draft(model, _source({"wide95": _table(95)}), ["wide95"]) - assert len(model.calls) == 2 + 8 + 2 + 2 - assert got["wide95"].error == (WIDE95_PARTS + "都没写出来：" + D.REASON_HOLLOW - + "；每段最多调用 8 次模型，这一段到了次数仍没写完，已停止重试" - + D.REASON_SPLIT + "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型") + o = got["wide95"] + asked = [c for c in model.calls if c.mode == "text"] + assert len(asked) <= 2 + 8 + 1 + assert not any("c065" in (c.asked["wide95"][1] or []) for c in asked), "第 4 段没问" + assert o.status == "failed" and o.error.startswith(WIDE95_PARTS + "都没写出来：") + assert HALTED in o.error + assert o.error.endswith("这张表接连 3 段都没写出来，剩下的 1 段已跳过，可以换一个起草模型后只重试这张表") async def test_a_wide_table_that_never_comes_back_is_bounded_by_the_first_wave(): - """默认并发（3 段同时起草）：294 列（8 个列分段）一直只给空壳，最多是表级那段 2 次、头 3 个列分段各 8 次（封顶）、 - 其余 5 段各 2 次。以前所有列分段一下子都排上去、一层一层地拆，要 168 次。""" + """默认并发（3 段同时起草）：294 列（8 个列分段）一直只给空壳：表级那段 2 次，头一批同时起草的列分段拆小到止损或 + 封顶；接连 3 段没写出来以后，没开始的段跳过不问。以前所有列分段一下子都排上去、一层一层地拆，要 168 次。""" model = Scripted(_hollow) got = await D.run_model_draft(model, _source({"wide294": _table(294)}), ["wide294"]) - assert got["wide294"].status == "failed" and got["wide294"].parts_total == 9 - assert len(model.calls) <= 2 + 3 * 8 + 5 * 2 + o = got["wide294"] + assert o.status == "failed" and o.parts_total == 9 and o.parts_done == 9 + assert len(model.calls) <= 2 + 4 * 8 + assert "已跳过" in o.error + asked = {c for call in model.calls for c in call.asked["wide294"][1] or []} + assert "c294" not in asked, "最后一段没问" async def test_a_big_draft_does_not_hold_the_model_ahead_of_a_later_one(monkeypatch): @@ -1042,7 +1132,7 @@ async def test_the_chunks_of_a_part_cut_by_the_smaller_cap_share_its_eight_calls """每个原始段（连同段大小减半后按上限分开的各块、拆出来的、补问的、重试的）一共最多调用 8 次。以前分开的每一块各有 8 次：上限缩到 10 列以后一直只给空壳，4 张窄表拼成的 40 列段调用了 20 次，单张 40 列的表也是 20 次。""" monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) - monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99) # 不让止损先停下来 + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) # 不让止损先停下来 tables = {"w40": _table(40), **{f"n{i}": _table(10) for i in range(1, 5)}, "x40": _table(40)} def handle(call: Call) -> Reply: @@ -1085,34 +1175,31 @@ class AlwaysSlow: raise TimeoutError() -SLOW_STOP = "；拆到每段 10 列仍然超时，已停止拆小重试，建议更换起草模型" - - -async def test_a_model_that_always_times_out_stops_after_three_rounds_not_seven(): - """每次调用都超时、和段大小无关：超时拆出来的两半同时问（不再等带表级项的前一半写完才问后一半），拆到每段 10 列的 - 调用接连超时 2 次就停止拆小，拆出来还没问的块不再问。40 列的表 3 轮、6 次调用就结束（时限 420 秒时约 21 分钟）； - 以前 40→20→10→5 一层一层串着问，8 次调用约 7 轮（约 49 分钟）。""" +async def test_a_model_that_always_times_out_is_bounded_by_the_part_cap(): + """每次调用都超时：超时拆出来的两半同时问，最多拆到这一段的调用次数用完（8 次）或止损。以前 40→20→10→5 一层一层串着问， + 8 次调用约 7 轮；现在两半同时问，至多 4 轮。""" tables = {"t40": _table(40)} slow = AlwaysSlow(tables) model, reports = Scripted(slow), Reports() got = await D.run_model_draft(model, _source(tables), ["t40"], hooks=reports.hooks()) - assert [_width(c, tables) for c in model.calls] == [40, 20, 20, 10, 10, 10], "拆到每段 10 列仍超时就不再往下拆" - assert slow.rounds == [1, 2, 2, 3, 3, 3], "两半同时问" - assert reports.failures == ["timeout"] * 6 and len(reports.items) == len(model.calls) + assert len(model.calls) <= catalog.MODEL_PART_MAX_CALLS and max(slow.rounds) <= 4 + assert [_width(c, tables) for c in model.calls[:3]] == [40, 20, 20], "两半同时问" + assert set(reports.failures) == {"timeout"} and len(reports.items) == len(model.calls) o = got["t40"] - assert o.status == "failed" and o.error == "模型 420 秒内没有写完这一段" + D.REASON_SPLIT + SLOW_STOP + assert o.status == "failed" and "模型 420 秒内没有写完这一段" + D.REASON_SPLIT in o.error + assert o.chunks_done == o.chunks_total > 1, "拆出来的块都结束了" -async def test_three_parts_that_always_time_out_end_in_four_rounds(): - """首轮同时起草的 3 段都一直超时：拆到每段 10 列仍超时就一起停下，不再各自拆满 8 次。12 次调用、4 轮；以前 24 次 - 调用、约 8 轮（止损要等一整段结束才计数，首轮的 3 段都跑满）。""" +async def test_three_parts_that_always_time_out_stop_together(): + """首轮同时起草的 3 段都一直超时：最近 10 次调用 7 次以上没写出来以后，拆出来还没问的块不再问，不再各自拆满 8 次。 + 以前（接连超时的止损）12 次调用；更早要 24 次。""" tables = {f"t{i}": _table(40) for i in range(1, 4)} slow = AlwaysSlow(tables) model = Scripted(slow) got = await D.run_model_draft(model, _source(tables), list(tables)) - assert len(model.calls) == 12 and max(slow.rounds) == 4 + assert len(model.calls) <= 9 and max(slow.rounds) <= 4 assert {o.status for o in got.values()} == {"failed"} - assert all(o.error.endswith(SLOW_STOP) for o in got.values()) + assert any(HALTED in o.error for o in got.values()) async def test_timeouts_that_go_away_at_ten_columns_are_still_split_down_and_written(): @@ -1152,7 +1239,7 @@ async def test_a_call_whose_reply_cannot_be_read_still_counts(monkeypatch): """读回复时出了意料之外的错：这张表记出错，这次调用照样记进用量（算一次出错），调用次数和用量对得上。""" tables = {"t4": _table(4)} - def broken(entry, meta, *, system): + def broken(entry, meta, *, system, **_): raise RuntimeError("校验时出了意外") monkeypatch.setattr(catalog, "_from_model", broken) @@ -1190,24 +1277,54 @@ async def test_a_wide_table_that_wrote_nothing_names_every_part(): got = await D.run_model_draft(Scripted(handle), _source({"wide95": _table(95)}), ["wide95"]) o = got["wide95"] - assert o.status == "failed" and o.error == WIDE95_PARTS + "都没写出来：调用模型出错：网关拒绝" + assert o.status == "failed" and o.error == (WIDE95_PARTS + "都没写出来：调用模型出错：网关拒绝；" + "这张表接连 3 段都没写出来，剩下的 1 段已跳过，可以换一个起草模型后只重试这张表") narrow = await D.run_model_draft(Scripted(handle), _source({"t8": _table(8)}), ["t8"]) assert narrow["t8"].error == "调用模型出错：网关拒绝", "只有一段的表：这一段就是整张表，照旧只写原因" async def test_a_halted_part_that_could_be_split_is_not_asked_again_in_text_mode(monkeypatch): - """止损以后本来能拆小的段只给空壳：已经走纯文本的不再原样再问（约定只对拆不开的段再问一次）；结构化输出的照旧换 - 纯文本问一次，看看是不是结构化输出的问题（见 test_the_column_parts_of_a_wide_table_stop_splitting_too）。""" + """止损以后每块只问一次：已经走纯文本的空壳不再原样再问、也不拆小。t1：结构化不支持 1 次（只管通信的重来，不进 + 止损的窗口）、纯文本 1 次、拆成两半各问一次再原样再问一次（5 次没写出来）；t2 的第一次是第 6 次，照常拆小，前一半 + 是第 7 次没写出来，之后止损：后一半不再问；t3 只问一次、不拆小。""" monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) tables = {f"t{i}": _table(10) for i in range(1, 5)} model = Scripted(_text_only(_hollow)) got = await D.run_model_draft(model, _source(tables), list(tables)) - # t1：结构化不支持 1 次、纯文本 1 次、拆成两半各问一次再原样再问一次；t2 同样（少了结构化那次）；之后止损 - assert _by_table(model) == {"t1": 6, "t2": 5, "t3": 1, "t4": 1} - assert got["t3"].error == (catalog.MODEL_EMPTY_REASON - + "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型") + assert _by_table(model) == {"t1": 6, "t2": 2, "t3": 1, "t4": 1} + assert got["t2"].error == D.REASON_HOLLOW + D.REASON_SPLIT + HALTED + assert got["t3"].error == catalog.MODEL_EMPTY_REASON + HALTED + + +async def test_while_halted_structured_output_still_gets_one_probe_in_text(monkeypatch): + """止损期间结构化输出只给空壳：这次起草还没换纯文本问过的，照旧换纯文本问一次（看看是不是结构化输出的问题）；纯文本 + 写出来了，后面都走纯文本。问过一次（纯文本也只给空壳）就不再问。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + tables = {**{f"x{i}": _table(4) for i in range(1, 8)}, "t8": _table(8), "t9": _table(8)} + ok = answering(tables) + + def handle(text_works: bool) -> Callable[[Call], Reply]: + def go(call: Call) -> Reply: + if next(iter(call.asked)).startswith("x"): + raise RuntimeError("网关拒绝了这次请求") + if call.mode == "structured": + return _hollow(call) + return ok(call) if text_works else _hollow(call) + return go + + model = Scripted(handle(True)) + got = await D.run_model_draft(model, _source(tables), list(tables)) + assert [c.mode for c in model.calls if "x1" not in c.asked and not next(iter(c.asked)).startswith("x")] == \ + ["structured", "text", "text"] + assert got["t8"].status == got["t9"].status == "ok" + + model = Scripted(handle(False)) + got = await D.run_model_draft(model, _source(tables), list(tables)) + assert _by_table(model)["t8"] == 2 and _by_table(model)["t9"] == 1, "纯文本问过一次，不再问" + assert got["t9"].error == catalog.MODEL_EMPTY_REASON + HALTED # -------------------------------------------------------------------------- @@ -1299,3 +1416,224 @@ def test_cut_values_are_dropped_and_complete_ones_kept(): @pytest.mark.parametrize("text", ["好的，我来写", "{坏的 json}", ""]) def test_not_json_is_not_a_reply(text): assert D._parse_text(text, cut=False) is None + + +# -------------------------------------------------------------------------- +# 闭环第一步的起草遗留：逐段交出去落盘、块进度、「已拆小重试」只说拆过的段 +# -------------------------------------------------------------------------- + + +class Written: + """进度回调 part_written 收到的：(表名, 已写出内容的段数, 这时这张表拿到的列)；chunks_changed 收到的块进度。""" + + def __init__(self) -> None: + self.parts: list[tuple[str, int, set[str], bool]] = [] + self.chunks: dict[str, list[tuple[int, int]]] = {} + + async def part_written(self, table: str, written: D.PartWrite) -> None: + cols = set((written.notes.get("columns") or {})) + self.parts.append((table, written.parts_written, cols, "label" in written.notes)) + + async def chunks_changed(self, table: str, total: int, done: int) -> None: + self.chunks.setdefault(table, []).append((total, done)) + + def hooks(self) -> D.Hooks: + return D.Hooks(part_written=self.part_written, chunks_changed=self.chunks_changed) + + +async def test_each_part_of_a_wide_table_is_handed_over_as_soon_as_it_is_written(monkeypatch): + """宽表每写出一段就交给调用方并入目录（最后一段由 table_done 收尾）：交出去的是这张表到这时拿到的全部内容。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + tables = {"wide95": _table(95)} + seen = Written() + got = await D.run_model_draft(Scripted(answering(tables)), _source(tables), ["wide95"], hooks=seen.hooks()) + o = got["wide95"] + assert o.status == "ok" and o.parts_written == 4 + assert [(t, n, len(cols), label) for t, n, cols, label in seen.parts] == [ + ("wide95", 1, 0, True), ("wide95", 2, 32, True), ("wide95", 3, 64, True)] + + def handle(call: Call) -> Reply: + part, cols = call.asked["wide95"] + if part == "columns" and "c033" in cols: + raise RuntimeError("网关拒绝") + return answering(tables)(call) + + seen = Written() + got = await D.run_model_draft(Scripted(handle), _source(tables), ["wide95"], hooks=seen.hooks()) + assert got["wide95"].status == "partial" and got["wide95"].parts_written == 3 + assert [n for _, n, _, _ in seen.parts] == [1, 2], "没写出来的那段不交" + + +async def test_a_narrow_table_is_handed_over_only_once_at_the_end(): + tables = {"t8": _table(8)} + seen = Written() + got = await D.run_model_draft(Scripted(answering(tables)), _source(tables), ["t8"], hooks=seen.hooks()) + assert seen.parts == [] and got["t8"].parts_written == 1 + failed = await D.run_model_draft(Scripted(lambda call: EXHAUSTED), _source({"t4": _table(4)}), ["t4"]) + assert failed["t4"].parts_written == 0 + + +async def test_chunks_count_what_the_table_is_actually_asked_in(monkeypatch): + """块进度：一开始是这张表分的段数；一段被拆成两半多一块、被缩小的上限分开成几块就多几块；一块结束算一块。 + 多表拼成的段按表拆开时，每张表还是一块。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + monkeypatch.setattr(catalog, "MODEL_SHRINK_AFTER", 1) + tables = {"a12": _table(12), "b40": _table(40)} + ok = answering(tables) + seen = Written() + got = await D.run_model_draft(Scripted(lambda call: EXHAUSTED if call.n == 0 else ok(call)), _source(tables), + ["a12", "b40"], hooks=seen.hooks()) + a, b = got["a12"], got["b40"] + assert a.status == b.status == "ok" + assert (a.chunks_total, a.chunks_done) == (2, 2), "额度用完拆成两半" + assert (b.chunks_total, b.chunks_done) == (2, 2), "上限缩到 20 列，40 列的段分开成两块" + assert seen.chunks["a12"][-1] == (2, 2) and seen.chunks["b40"][-1] == (2, 2) + assert all(done <= total for pairs in seen.chunks.values() for total, done in pairs) + + wide = {"wide95": _table(95)} + got = await D.run_model_draft(Scripted(answering(wide)), _source(wide), ["wide95"]) + assert (got["wide95"].chunks_total, got["wide95"].chunks_done) == (4, 4) + + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 4) + packed = {"p6": _table(6), "q6": _table(6)} + okp = answering(packed) + model = Scripted(lambda call: _hollow(call) if call.n == 0 else okp(call)) + got = await D.run_model_draft(model, _source(packed), ["p6", "q6"]) + assert [set(c.asked) for c in model.calls] == [{"p6", "q6"}, {"p6"}, {"q6"}], "按表拆开" + assert {(o.chunks_total, o.chunks_done) for o in got.values()} == {(1, 1)} + + +async def test_only_the_parts_that_were_split_say_so(monkeypatch): + """一张表几段没写出来、只有一部分真拆小重试过：「已拆小重试仍没写出」只跟在拆过的那几段后面说。95 列：第 3 段一直用完 + 额度（拆小重试过），第 4 段调用出错（没拆过）。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_SHRINK_AFTER", 99) + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) + tables = {"wide95": _table(95)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + part, cols = call.asked["wide95"] + if part == "columns" and cols[0] >= "c065": + raise RuntimeError("网关拒绝") + if part == "columns" and cols[0] >= "c033": + return EXHAUSTED + return ok(call) + + got = await D.run_model_draft(Scripted(handle), _source(tables), ["wide95"]) + o = got["wide95"] + assert o.status == "partial" + assert o.error == ("这张表 95 列，分 4 段起草，第 3 段（c033 到 c064，共 32 列）、第 4 段（c065 到 c095，共 31 列）没写出来：" + + _exhausted() + "；调用模型出错：网关拒绝；其中第 3 段已拆小重试仍没写出。可以只重试这张表") + + +# -------------------------------------------------------------------------- +# 复核后的修正：止损的窗口只记每次调用的最终结局。通信层的重来（服务方嫌额度太大退回 8192、结构化输出不支持或回了 +# 说明文字改走纯文本、429 / 5xx 退避重试）不花额度，也说明不了模型写不出来：照旧报给进度回调（用量、提醒照算）， +# 不进止损的窗口。以前都记成没写出来，OpenAI 兼容网关（额度上限 8192、不支持 json_schema）上并发的头几次调用全是 +# 重来，第一次额度用完就凑满 10 次里 7 次，那一段直接记没写出来、不再拆小。 +# -------------------------------------------------------------------------- + + +def _six_tables() -> dict: + """6 张 40 列的表：每张自成一段，默认并发 3 段同时起草。""" + return {f"t{i}": _table(40, prefix=f"t{i}_") for i in range(6)} + + +def _halved(model: Scripted, table: str) -> bool: + """这张表被拆成两半问过（表级的项加前 20 列那一半）。""" + return any(part == "table+columns" and len(cols or []) == 20 + for call in model.calls for name, (part, cols) in call.asked.items() if name == table) + + +async def test_budget_and_structured_output_fallbacks_do_not_halt_a_concurrent_draft(): + tables = _six_tables() + ok = answering(tables) + spent: set[str] = set() + + async def refuse(call: Call) -> Reply: + await asyncio.sleep(0.01) + raise _Status(400, "max_tokens: 32768 > 8192, which is the maximum allowed number of output tokens") + + async def small(call: Call) -> Reply: + await asyncio.sleep(0.01) + if call.mode == "structured": + raise _Status(400, "This response_format type is unavailable now") + if "t0" in call.asked and "t0" not in spent: + spent.add("t0") + return EXHAUSTED + return ok(call) + + big, sm, reports = Scripted(refuse), Scripted(small), Reports() + got = await D.run_model_draft(catalog.DraftModel(big, sm, "m"), _source(tables), list(tables), + hooks=reports.hooks()) + assert _status(got) == dict.fromkeys(tables, "ok"), {t: o.error for t, o in got.items()} + assert _halved(sm, "t0"), "额度用完的那一段照常拆小重试" + assert len(big.calls) == 3 and reports.failures.count("error") == 6, "退回小额度、改走纯文本的重来照旧报给进度回调" + assert reports.failures.count("exhausted") == 1 + + +async def test_rate_limit_retries_do_not_halt_the_draft(): + tables = _six_tables() + ok = answering(tables) + limited: dict[str, int] = {} + spent: set[str] = set() + + async def handle(call: Call) -> Reply: + await asyncio.sleep(0.01) + name = next(iter(call.asked)) + if name in ("t0", "t1", "t2") and limited.get(name, 0) < 2: + limited[name] = limited.get(name, 0) + 1 + raise _Status(429, "Too Many Requests") + if name == "t0" and not spent: + spent.add(name) + return EXHAUSTED + return ok(call) + + model, reports = Scripted(handle), Reports() + got = await D.run_model_draft(model, _source(tables), list(tables), hooks=reports.hooks()) + assert _status(got) == dict.fromkeys(tables, "ok"), {t: o.error for t, o in got.items()} + assert _halved(model, "t0"), "额度用完的那一段照常拆小重试" + assert reports.failures.count("error") == 6, "429 之后重试成功的那几次照旧报给进度回调" + + +async def test_structured_output_that_comes_back_as_prose_does_not_count_toward_halting(monkeypatch): + """结构化输出回了说明文字、这一块改走纯文本再问：也是通信方式的问题，那一次不进止损的窗口（止损门槛调低到 3 次， + 并发的 3 块都回说明文字也不止损）；纯文本写出来的、没写出来的照常算。""" + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 3) + tables = _six_tables() + ok = answering(tables) + spent: set[str] = set() + + async def handle(call: Call) -> Reply: + await asyncio.sleep(0.01) + if call.mode == "structured": + return Reply(text="好的。这张表记录的是一笔笔业务，各列的含义如下：第一列是编号……") + if "t0" in call.asked and "t0" not in spent: + spent.add("t0") + return EXHAUSTED + return ok(call) + + model, reports = Scripted(handle), Reports() + got = await D.run_model_draft(model, _source(tables), list(tables), hooks=reports.hooks()) + assert _status(got) == dict.fromkeys(tables, "ok"), {t: o.error for t, o in got.items()} + assert _halved(model, "t0") + assert reports.failures.count("error") == 3, "回了说明文字的那几次照旧报给进度回调" + + +async def test_a_call_that_still_fails_after_its_retries_counts_toward_halting(monkeypatch): + """重试用完仍然出错的那一次是这次调用的最终结局，照常进止损的窗口：一直 503 的网关，止损照样起作用。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + tables = {f"t{i}": _table(10) for i in range(1, 9)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + if set(call.asked) & {"t1", "t2", "t3", "t4", "t5", "t6", "t7"}: + raise _Status(503, "Service Unavailable") + return EXHAUSTED if "t8" in call.asked else ok(call) + + got = await D.run_model_draft(Scripted(handle), _source(tables), list(tables)) + assert got["t7"].error == "调用模型出错：Service Unavailable（已重试 2 次）" + assert got["t8"].error == _exhausted() + HALTED, "最近 10 次的最终结局有 7 次以上没写出来：t8 不再拆小" diff --git a/backend/tests/test_catalog_draft_facts.py b/backend/tests/test_catalog_draft_facts.py new file mode 100644 index 00000000..37f3705b --- /dev/null +++ b/backend/tests/test_catalog_draft_facts.py @@ -0,0 +1,498 @@ +"""防错事实的提议：模型起草时提议有效行过滤（valid_filter）、去重规则（dedup），附一句依据。 + +- 提示词写清两项只在表结构、注释、码值能看出来时才写；结构化输出、纯文本两条路都认这两项； +- 有效行过滤按数据源的方言用 sqlglot 解析成一个条件：只能引用这张表的列，不能有子查询、聚合、窗口函数、分号，只允许 + 比较、IN、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE，不超过 500 字。不过就丢掉； + 存的是整理过的写法：原文照旧，列换成表结构里的真名、不带表名、按方言加引号（剖析核对的也是这一条）； +- 开了数据剖析、能连上的源：经剖析的只读守卫数出排除比例，排除 0% 或 100% 的不收，收下的在 note 里写剖析核对的结果； + 没开剖析、连不上的照收，note 写没有核对的原因； +- 没收下的（没通过校验、排除 0% 或 100%）不算失败，在这张表的日志里记一句（起草任务逐表的 log）； +- 开了剖析的源，目录里记下的码值交给模型做背景（遮罩的列不给；「已确认的目录项」里遮罩列的码值也不给）； +- 这两项照旧记模型起草 / 推断；SQL 检查只在人工确认、有确证时才把漏了有效行过滤报成警告，推断的只是提示。 + +用合成的景区业务库（tests/fixtures/catalog/scenic.py）和虚构的表结构，不调用真实模型。 +""" +from __future__ import annotations + +import logging +import sqlite3 +from types import SimpleNamespace + +import pytest + +from app.data import catalog, catalog_profile, sqlcheck +from app.data import catalog_draft as D +from app.data.catalog import make_item +from app.data.catalog_profile import PROFILE_OPTION +from app.data.engine import MASK_COLUMNS_OPTION, engines +from app.data.introspect import introspect +from tests.fixtures.catalog.fake_model import FakeModel, answer, requested + +# -------------------------------------------------------------------------- +# 夹具 +# -------------------------------------------------------------------------- + + +def _col(name: str, type_: str = "INTEGER", comment: str | None = None) -> dict: + return {"name": name, "type": type_, "nullable": True, "comment": comment} + + +def _orders() -> dict: + return {"qualified": "orders", "schema": None, "is_view": False, "primary_key": ["id"], "foreign_keys": [], + "comment": None, + "columns": [_col("id"), _col("order_no", "TEXT"), _col("status", "TEXT", comment="C 作废，X 测试单"), + _col("amount", "REAL"), _col("updated_at", "TEXT")]} + + +def _plain(tables: dict, **kw) -> SimpleNamespace: + """不连库的源（没有 options：没开数据剖析）。""" + return SimpleNamespace(id="facts-src", name="facts", kind=kw.pop("kind", "sqlite"), origin="manual", + description="", schema_cache={"tables": tables}, **kw) + + +_CACHE: dict[str, dict] = {} + + +async def _scenic_cache(path: str) -> dict: + if path not in _CACHE: + probe = SimpleNamespace(id="facts-scenic-probe", name="scenic", kind="sqlite", database=path, host=None, + port=None, username=None, password=None, options={}, readonly=True, description="", + schema_cache={}, origin="manual") + try: + _CACHE[path] = await introspect(probe) + finally: + await engines.invalidate(probe.id) + return _CACHE[path] + + +@pytest.fixture +async def scenic(scenic_db): + """接上景区库的源（开了数据剖析）。用完作废连接池。""" + made: list[str] = [] + + async def make(*, options: dict | None = None, database: str | None = None, enabled: bool = True): + sid = f"facts-{len(made)}-{id(made)}" + made.append(sid) + opts = {PROFILE_OPTION: {"enabled": True}} if options is None else options + return SimpleNamespace(id=sid, name="scenic", kind="sqlite", database=database or scenic_db, host=None, + port=None, username=None, password=None, options=opts, readonly=True, description="", + schema_cache=await _scenic_cache(scenic_db), origin="manual", enabled=enabled) + + yield make + for sid in made: + await engines.invalidate(sid) + + +def _count(path: str, where: str) -> tuple[int, int]: + """(全表行数, 不满足条件的行数)：直接用 sqlite3 数，核对剖析算出来的数。""" + db = sqlite3.connect(path) + try: + total = db.execute("SELECT COUNT(*) FROM visits").fetchone()[0] + kept = db.execute(f"SELECT COUNT(*) FROM visits WHERE {where}").fetchone()[0] + finally: + db.close() + return total, total - kept + + +# -------------------------------------------------------------------------- +# 校验有效行过滤 +# -------------------------------------------------------------------------- + +META = _orders() + + +@pytest.mark.parametrize("condition, columns", [ + ("status NOT IN ('C', 'X')", ["status"]), + ("STATUS NOT IN ('C', 'X')", ["status"]), # 列名不分大小写，记表结构里的写法 + ("orders.status <> 'C'", ["status"]), # 带这张表的表名 + ("status IS NOT NULL AND amount > 0", ["status", "amount"]), + ("NOT (status = 'C' OR status = 'X')", ["status"]), + ("amount BETWEEN 0 AND 100000", ["amount"]), + ("order_no NOT LIKE 'TEST%'", ["order_no"]), + ("UPPER(TRIM(status)) <> 'C'", ["status"]), + ("COALESCE(status, '') <> 'X' AND amount >= -1", ["status", "amount"]), + ("status = 'C' OR updated_at IS NULL", ["status", "updated_at"]), +]) +def test_valid_filters_that_pass(condition, columns): + parsed = sqlcheck.parse_valid_filter(condition, META, kind="sqlite", table="orders") + assert parsed.problem is None, parsed.problem + assert list(parsed.columns) == columns + assert sqlcheck.valid_filter_problem(condition, META, kind="mysql", table="orders") is None + + +@pytest.mark.parametrize("condition, said", [ + ("status = 'C'; DROP TABLE orders", "分号"), + ("status IN (SELECT code FROM codes)", "子查询"), + ("EXISTS (SELECT 1 FROM refunds)", "子查询"), + ("COUNT(status) > 1", "聚合"), + ("ROW_NUMBER() OVER (ORDER BY id) = 1", "窗口"), + ("LENGTH(status) > 0", "LENGTH"), + ("amount * 2 > 10", "不支持的写法"), + ("payments.status = 1", "只能引用这张表的列"), + ("nope = 1", "nope"), + ("1 = 1", "没有用到这张表的任何列"), + ("status", "一个条件"), + ("status = ?", "不支持的写法"), + ("status = 'C' -- 注释", "注释"), + ("status = '{{ input.code }}'", "模板"), + ("status = '" + "C" * 600 + "'", "500"), + ("status = (", "解析"), + ("status = 'C' UNION SELECT 1", "一个条件"), +]) +def test_valid_filters_that_are_dropped(condition, said): + problem = sqlcheck.valid_filter_problem(condition, META, kind="sqlite", table="orders") + assert problem is not None and said in problem, problem + + +def test_a_source_kind_without_a_dialect_cannot_be_checked(): + assert "不支持" in sqlcheck.valid_filter_problem("status = 'C'", META, kind="mssql", table="orders") + + +def test_the_condition_is_rendered_with_the_columns_quoted_like_the_source(): + meta = {**META, "columns": [*META["columns"], _col("isTest")]} + parsed = sqlcheck.parse_valid_filter("istest = 0 AND STATUS <> 'C'", meta, kind="mysql", table="orders") + assert parsed.problem is None + rendered = sqlcheck.render_valid_filter(parsed, quoted=lambda name: name != name.lower()) + assert rendered == "`isTest` = 0 AND status <> 'C'" + + +#: 带一列混合大小写列名的订单表（ORM 建的 PostgreSQL 库里常见） +MIXED = {**META, "columns": [*META["columns"], _col("OrderStatus", "TEXT")]} + + +@pytest.mark.parametrize("kind, condition, stored", [ + # PostgreSQL 把没加引号的名字折成小写：混合大小写的列一律加引号，不论模型写的大小写对不对 + ("postgresql", "orderstatus <> 'X' AND STATUS NOT IN ('C', 'X')", + "\"OrderStatus\" <> 'X' AND status NOT IN ('C', 'X')"), + ("postgresql", "OrderStatus <> 'X'", "\"OrderStatus\" <> 'X'"), + ("postgresql", "orders.\"OrderStatus\" IS NOT NULL", "\"OrderStatus\" IS NOT NULL"), + ("mysql", "orderstatus <> 'X'", "`OrderStatus` <> 'X'"), + ("oracle", "ORDERS.STATUS <> 'C'", "status <> 'C'"), + # 不用改的照原文：NOT IN、IS NOT NULL 这些写法不被改写 + ("sqlite", "status NOT IN ('C', 'X') AND updated_at IS NOT NULL", + "status NOT IN ('C', 'X') AND updated_at IS NOT NULL"), +]) +def test_the_stored_filter_names_its_columns_the_way_the_source_needs(kind, condition, stored): + """存进目录的有效记录条件：原文照旧，只把列换成表结构里的真名、不带表名，按数据源的需要加引号（和剖析核对查询的写法 + 一致）。以前存的是模型的原文：PostgreSQL 上 orderstatus 会被当成另一列，核对时却按真名加了引号，存下的和核对过的 + 不是同一条，SQL 检查还会让模型照抄这一句。""" + entry = {"table": "orders", "valid_filter": {"condition": condition, "reason": "注释写明"}} + notes = catalog._from_model(entry, MIXED, system=False, source_kind=kind, table="orders") + assert notes["valid_filter"]["value"] == stored + again = sqlcheck.parse_valid_filter(stored, MIXED, kind=kind, table="orders") + assert again.problem is None and sqlcheck.stored_valid_filter(again, kind) == stored, "存下的再整理一遍不变" + + +def test_a_filter_that_grows_past_the_limit_once_quoted_is_dropped_not_written(): + """整理时加了引号、超过 500 字的不收（记下原因）：目录里这一项最多 500 字，写进去会让整张表写入失败。""" + condition = "orderstatus IN (" + ", ".join(f"'{i:03d}'" for i in range(66)) + ") OR updated_at IS NULL" + assert len(condition) == 499 + dropped: list[tuple[str, str]] = [] + notes = catalog._from_model({"table": "orders", "label": "订单", "valid_filter": condition}, MIXED, system=False, + source_kind="postgresql", table="orders", dropped=dropped) + assert "valid_filter" not in notes and notes["label"]["value"] == "订单" + assert dropped == [("有效记录条件按数据源的写法整理后超过 500 字", condition)] + assert not catalog.validate_notes(notes) + + +def test_columns_that_differ_only_in_case_must_be_named_exactly(): + """表结构里有只差大小写的两列：写得和其中一列一模一样的认那一列，否则说不清是哪一列，不收。""" + meta = {**META, "columns": [*META["columns"], _col("Status", "TEXT")]} + problem = sqlcheck.valid_filter_problem("STATUS = 'C'", meta, kind="postgresql", table="orders") + assert problem == ("有效记录条件里的列「STATUS」在表结构里有只差大小写的几列（status、Status），" + "要照表结构的写法写清是哪一列") + for condition, stored in (("\"Status\" = 'C'", "\"Status\" = 'C'"), ("status = 'C'", "status = 'C'")): + parsed = sqlcheck.parse_valid_filter(condition, meta, kind="postgresql", table="orders") + assert parsed.problem is None and sqlcheck.stored_valid_filter(parsed, "postgresql") == stored + + +# -------------------------------------------------------------------------- +# 模型给的两项：校验、依据 +# -------------------------------------------------------------------------- + + +def test_the_model_fields_become_llm_items_with_their_reason(): + entry = {"table": "orders", "label": "订单", + "valid_filter": {"condition": "status NOT IN ('C', 'X')", "reason": "状态注释写明 C 作废、X 测试单"}, + "dedup": {"rule": "同一订单号有多行时只取最后修改的一行", "reason": "订单号可能重复导出"}} + notes = catalog._from_model(entry, META, system=False, source_kind="sqlite", table="orders") + vf, dd = notes["valid_filter"], notes["dedup"] + assert (vf["value"], vf["source"], vf["status"]) == ("status NOT IN ('C', 'X')", "llm", "proposed") + assert vf["note"] == "依据：状态注释写明 C 作废、X 测试单" + assert (dd["value"], dd["source"], dd["status"]) == ("同一订单号有多行时只取最后修改的一行", "llm", "proposed") + assert dd["note"] == "依据：订单号可能重复导出" + assert not catalog.validate_notes(notes) + + +def test_plain_strings_are_accepted_and_bad_filters_dropped(caplog): + entry = {"table": "orders", "label": "订单", "valid_filter": "status <> 'C'", "dedup": "按订单号去重"} + notes = catalog._from_model(entry, META, system=False, source_kind="sqlite", table="orders") + assert notes["valid_filter"]["value"] == "status <> 'C'" and "note" not in notes["valid_filter"] + assert notes["dedup"]["value"] == "按订单号去重" + + with caplog.at_level(logging.INFO, logger="app.data.catalog"): + notes = catalog._from_model({"table": "orders", "label": "订单", + "valid_filter": {"condition": "status IN (SELECT 1)", "reason": "猜的"}}, + META, system=False, source_kind="sqlite", table="orders") + assert "valid_filter" not in notes and notes["label"]["value"] == "订单", "丢掉这一项，别的照收" + assert any("有效行过滤" in r.getMessage() and "orders" in r.getMessage() for r in caplog.records), "日志记一条" + for junk in ({"condition": ""}, {"reason": "只有依据"}, ["status = 1"], 3): + notes = catalog._from_model({"table": "orders", "valid_filter": junk, "dedup": junk}, META, system=False, + source_kind="sqlite", table="orders") + assert "valid_filter" not in notes and "dedup" not in notes, junk + + +def test_the_prompt_asks_for_the_two_facts_only_when_evident(): + system = catalog.CATALOG_DRAFT_SYSTEM + assert "valid_filter" in system and "dedup" in system + assert "拿不准就不写" in system and "reason" in system + assert "UPPER、LOWER、TRIM、COALESCE" in system + props = catalog.CATALOG_DRAFT_PROMPT_SCHEMA["properties"]["tables"]["items"]["properties"] + assert set(props["valid_filter"]["properties"]) == {"condition", "reason"} + assert set(props["dedup"]["properties"]) == {"rule", "reason"} + assert '"valid_filter": {"condition"' in catalog.CATALOG_DRAFT_TEXT_PROMPT + assert '"dedup": {"rule"' in catalog.CATALOG_DRAFT_TEXT_PROMPT + assert "有效行过滤" in catalog.CATALOG_DRAFT_PROMPT_LABELS["only_table"] + + +# -------------------------------------------------------------------------- +# 引擎：没开剖析照收、写明没有核对；开了剖析的源把码值交给模型 +# -------------------------------------------------------------------------- + +REPLY = {"label": "订单", "grain": "每笔订单一行", + "valid_filter": {"condition": "status NOT IN ('C', 'X')", "reason": "状态注释写明 C 作废、X 测试单"}, + "dedup": {"rule": "同一订单号只取最后一行", "reason": "订单号可能重复"}, + "columns": [{"name": "status", "label": "订单状态"}]} + + +async def test_without_profiling_the_filter_is_kept_and_says_it_was_not_checked(): + model = FakeModel({"orders": REPLY}) + got = await D.run_model_draft(model, _plain({"orders": _orders()}), ["orders"]) + o = got["orders"] + assert o.status == "ok" + vf = o.notes["valid_filter"] + assert (vf["source"], vf["status"]) == ("llm", "proposed") + assert vf["note"] == "依据：状态注释写明 C 作废、X 测试单。没有核对排除比例（数据源没有开数据剖析）" + assert o.notes["dedup"]["note"] == "依据：订单号可能重复" + + +async def test_an_invalid_filter_from_the_model_is_dropped_not_failed(): + reply = {**REPLY, "valid_filter": {"condition": "status IN (SELECT code FROM bad_codes)", "reason": "猜的"}} + got = await D.run_model_draft(FakeModel({"orders": reply}), _plain({"orders": _orders()}), ["orders"]) + o = got["orders"] + assert o.status == "ok" and o.error is None, "不算失败" + assert "valid_filter" not in o.notes and o.notes["dedup"]["value"] == "同一订单号只取最后一行" + + +def _codes_existing() -> dict: + return {"orders": {"columns": { + "status": {"codes": make_item({"C": "作废", "X": "", "P": "已支付"}, "profile", "proposed")}, + "order_no": {"codes": make_item({"A1": "甲"}, "profile", "proposed")}, + "amount": {"codes": make_item({"0": "零"}, "profile", "rejected")}, + }}} + + +async def test_codes_go_to_the_model_when_profiling_is_on_but_not_masked_ones(): + on = {PROFILE_OPTION: {"enabled": True}, MASK_COLUMNS_OPTION: ["ORDER_NO"]} + model = FakeModel({"orders": REPLY}) + await D.run_model_draft(model, _plain({"orders": _orders()}, options=on), ["orders"], existing=_codes_existing()) + text = model.calls[0] + head = catalog.CATALOG_DRAFT_PROMPT_LABELS["codes"] + assert head in text + block = text[text.index(head):] + assert "- status：C=作废、P=已支付，X 的含义待填写" in block + assert "A1" not in text, "遮罩的列不给码值" + assert "零" not in text, "驳回的码值不给" + + plain = FakeModel({"orders": REPLY}) + await D.run_model_draft(plain, _plain({"orders": _orders()}), ["orders"], existing=_codes_existing()) + assert head not in plain.calls[0] and "已支付" not in plain.calls[0], "没开剖析不给码值" + + +@pytest.mark.parametrize("options", [ + {PROFILE_OPTION: {"enabled": True}, MASK_COLUMNS_OPTION: ["ORDER_NO"]}, + {MASK_COLUMNS_OPTION: "order_no"}, +], ids=["profiled", "not-profiled"]) +async def test_confirmed_codes_of_a_masked_column_do_not_reach_the_model_either(options): + """遮罩的列不给码值，人工确认过的也一样：码值就是这一列的取值。以前经「已确认的目录项」照给（开没开剖析都给）； + 没遮罩的列确认过的码值照给（以此为准），遮罩列别的确认过的项（中文名）也照给。""" + existing = {"orders": {"columns": { + "order_no": {"codes": make_item({"ZX-0417": "内部测试单"}, "human", "confirmed"), + "label": make_item("订单号", "human", "confirmed")}, + "status": {"codes": make_item({"C": "作废"}, "human", "confirmed")}, + }}} + model = FakeModel({"orders": REPLY}) + await D.run_model_draft(model, _plain({"orders": _orders()}, options=options), ["orders"], existing=existing) + text = model.calls[0] + assert "ZX-0417" not in text and "内部测试单" not in text + confirmed = text[text.index(catalog.CATALOG_DRAFT_PROMPT_LABELS["confirmed"]):] + assert "status：码值：C=作废" in confirmed and "order_no：订单号" in confirmed + + +async def test_codes_reach_the_table_part_of_a_wide_table_and_only_their_column_part(): + cols = [_col(f"c{i:03d}") for i in range(1, 46)] + [_col("state", "TEXT")] + wide = {"qualified": "wide", "schema": None, "is_view": False, "primary_key": ["c001"], "foreign_keys": [], + "comment": None, "columns": cols} + existing = {"wide": {"columns": {"state": {"codes": make_item({"9": "作废"}, "profile", "proposed")}}}} + on = {PROFILE_OPTION: {"enabled": True}} + model = FakeModel({"wide": {"label": "宽表"}}) + await D.run_model_draft(model, _plain({"wide": wide}, options=on), ["wide"], existing=existing) + head = catalog.CATALOG_DRAFT_PROMPT_LABELS["codes"] + with_codes = [requested(c)["wide"] for c in model.calls if "9=作废" in c] + assert ("table", None) in with_codes, "表级那段要提议有效行过滤：给全部码值" + assert all(part == "table" or "state" in (cols or []) for part, cols in with_codes) + assert any(head not in c for c in model.calls), "不含这一列的列分段不给" + + +# -------------------------------------------------------------------------- +# 剖析核对排除比例 +# -------------------------------------------------------------------------- + + +async def test_the_check_counts_excluded_rows_through_the_read_only_guard(scenic, scenic_db, monkeypatch): + seen: list[tuple[str, bool]] = [] + real = catalog_profile.guard.check + + def spy(sql, **kw): + seen.append((sql, kw.get("readonly"))) + return real(sql, **kw) + + monkeypatch.setattr(catalog_profile.guard, "check", spy) + source = await scenic() + meta = source.schema_cache["tables"]["visits"] + found = await catalog_profile.check_valid_filter(source, "visits", meta, "status = 1") + total, excluded = _count(scenic_db, "status = 1") + assert 0 < excluded < total + assert found.keep and (found.rows, found.excluded) == (total, excluded) and not found.sampled + share = catalog_profile.excluded_share(excluded, total) + assert found.note == f"剖析核对：排除 {share} 的行（共 {total:,} 行）" + assert len(seen) == 1 and seen[0][1] is True, "一条查询，过只读守卫" + assert "COUNT(*)" in seen[0][0] and "status = 1" in seen[0][0] + + +async def test_the_check_runs_the_condition_as_it_is_stored(scenic, monkeypatch): + """剖析核对的就是存进目录的那一条：表名去掉、列换成表结构里的真名，别的照原文（NOT IN 不被改写成 NOT x IN）。""" + seen: list[str] = [] + real = catalog_profile.guard.check + + def spy(sql, **kw): + seen.append(sql) + return real(sql, **kw) + + monkeypatch.setattr(catalog_profile.guard, "check", spy) + source = await scenic() + reply = {"label": "入园记录", "valid_filter": {"condition": "VISITS.STATUS NOT IN (0)", "reason": "0 是作废的检票"}} + got = await D.run_model_draft(FakeModel({"visits": reply}), source, ["visits"]) + vf = got["visits"].notes["valid_filter"] + assert vf["value"] == "status NOT IN (0)" + assert len(seen) == 1 and f"CASE WHEN {vf['value']} THEN 0 ELSE 1 END" in seen[0] + assert "剖析核对：排除" in vf["note"] + + +@pytest.mark.parametrize("condition", ["status IN (0, 1)", "status = 7"]) +async def test_a_filter_that_excludes_nothing_or_everything_is_dropped(scenic, condition): + source = await scenic() + found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"], + condition) + assert not found.keep and found.dropped + + +async def test_a_big_table_is_checked_on_its_first_rows(scenic): + source = await scenic(options={PROFILE_OPTION: {"enabled": True, "max_scan_rows": 100, "sample_size": 10}}) + found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"], + "status = 1") + assert found.sampled and found.rows == 100 + assert found.note.startswith("剖析核对：按前 100 行统计，") + + +async def test_unchecked_filters_are_kept_with_the_reason(scenic, tmp_path): + meta_of = lambda s: s.schema_cache["tables"]["visits"] # noqa: E731 + off = await scenic(options={}) + found = await catalog_profile.check_valid_filter(off, "visits", meta_of(off), "status = 1") + assert found.keep and found.note == "没有核对排除比例（数据源没有开数据剖析）" + + masked = await scenic(options={PROFILE_OPTION: {"enabled": True}, MASK_COLUMNS_OPTION: "Status"}) + found = await catalog_profile.check_valid_filter(masked, "visits", meta_of(masked), "status = 1") + assert found.keep and found.note == "没有核对排除比例（条件里的 status 在数据源设置中被遮罩，不取数）" + + stopped = await scenic(enabled=False) + found = await catalog_profile.check_valid_filter(stopped, "visits", meta_of(stopped), "status = 1") + assert found.keep and found.note == "没有核对排除比例（数据源已停用）" + + gone = await scenic(database=str(tmp_path / "missing" / "nope.db")) + found = await catalog_profile.check_valid_filter(gone, "visits", meta_of(gone), "status = 1") + assert found.keep and found.note.startswith("没有核对排除比例（查询失败：") + + +async def test_the_engine_checks_the_filter_on_a_profiled_source(scenic, scenic_db): + source = await scenic() + good = {"label": "入园记录", "valid_filter": {"condition": "status = 1", "reason": "0 是作废的检票"}} + got = await D.run_model_draft(FakeModel({"visits": good}), source, ["visits"]) + vf = got["visits"].notes["valid_filter"] + total, excluded = _count(scenic_db, "status = 1") + share = catalog_profile.excluded_share(excluded, total) + assert vf["note"] == f"依据：0 是作废的检票。剖析核对：排除 {share} 的行（共 {total:,} 行）" + + useless = {"label": "入园记录", "valid_filter": {"condition": "status IN (0, 1)", "reason": "猜的"}} + got = await D.run_model_draft(FakeModel({"visits": useless}), source, ["visits"]) + assert got["visits"].status == "ok" and "valid_filter" not in got["visits"].notes, "排除 0% 的不收" + assert got["visits"].notes["label"]["value"] == "入园记录" + + +async def test_a_dropped_filter_is_written_into_the_table_log(scenic, scenic_db): + """没收下的有效记录条件不算失败，在这张表的日志里记一条（起草任务逐表的 log，界面看得到）：没通过校验的写原因， + 剖析核对排除 0% 或 100% 的写数到了多少行。收下的、模型没提的不记。""" + bad = {**REPLY, "valid_filter": {"condition": "status IN (SELECT code FROM bad_codes)", "reason": "猜的"}} + got = await D.run_model_draft(FakeModel({"orders": bad}), _plain({"orders": _orders()}), ["orders"]) + assert got["orders"].status == "ok" and got["orders"].error is None + assert got["orders"].log == [ + "有效记录条件里不能有子查询，模型提议的这一条没有收下：status IN (SELECT code FROM bad_codes)"] + + source = await scenic() + total, _ = _count(scenic_db, "1 = 1") + for condition, what in (("status IN (0, 1)", "没有排除任何一行"), ("status = 7", "所有行都被排除了")): + useless = {"label": "入园记录", "valid_filter": {"condition": condition, "reason": "猜的"}} + got = await D.run_model_draft(FakeModel({"visits": useless}), source, ["visits"]) + assert got["visits"].log == [ + f"有效记录条件按剖析核对{what}（共 {total:,} 行），模型提议的这一条没有收下：{condition}"] + + for reply in (REPLY, {"label": "订单"}): + got = await D.run_model_draft(FakeModel({"orders": reply}), _plain({"orders": _orders()}), ["orders"]) + assert got["orders"].log == [] + + +async def test_a_confirmed_filter_is_not_checked_again(scenic, monkeypatch): + source = await scenic() + called: list[str] = [] + + async def counting(*args, **kw): + called.append("x") + return catalog_profile.FilterCheck(True, "不该用到") + + monkeypatch.setattr(catalog_profile, "check_valid_filter", counting) + existing = {"visits": {"valid_filter": make_item("status = 1", "human", "confirmed")}} + reply = {"label": "入园记录", "valid_filter": {"condition": "status <> 0", "reason": "0 是作废"}} + await D.run_model_draft(FakeModel({"visits": reply}), source, ["visits"], existing=existing) + assert called == [], "人工确认过的有效行过滤，起草的提议并入时不会采用，不必再查库" + + +# -------------------------------------------------------------------------- +# SQL 检查：推断的有效行过滤只是提示，确认后才是警告 +# -------------------------------------------------------------------------- + + +async def test_a_drafted_filter_only_hints_until_someone_confirms_it(scenic): + source = await scenic(options={}) + entry = {"table": "visits", "valid_filter": {"condition": "status = 1", "reason": "0 是作废"}} + meta = source.schema_cache["tables"]["visits"] + notes = catalog._from_model(entry, meta, system=False, source_kind="sqlite", table="visits") + sql = "SELECT COUNT(*) FROM visits" + + def levels(table_notes: dict) -> list[tuple[str, str]]: + checker = sqlcheck.SqlChecker(kind="sqlite", schema_cache=source.schema_cache, notes={"visits": table_notes}) + return [(c.code, c.level) for c in checker.check(sql)] + + assert levels(notes) == [("missing_valid_filter", "info")] + confirmed = catalog.review_item(notes, "valid_filter", "confirm") + assert levels(confirmed) == [("missing_valid_filter", "warning")] + assert levels(catalog.review_item(notes, "valid_filter", "reject")) == [] diff --git a/backend/tests/test_catalog_draft_jobs.py b/backend/tests/test_catalog_draft_jobs.py index bc366876..2ce4aef9 100644 --- a/backend/tests/test_catalog_draft_jobs.py +++ b/backend/tests/test_catalog_draft_jobs.py @@ -17,7 +17,9 @@ from __future__ import annotations import asyncio +import itertools import uuid +from datetime import datetime, timedelta, timezone from types import SimpleNamespace from urllib.parse import quote @@ -441,6 +443,11 @@ def test_old_job_tables_get_the_new_columns(): assert conn.execute(text("SELECT timeouts, failed_calls, part_columns, notice FROM catalog_draft_jobs")) \ .one() == (0, 0, None, None) assert conn.execute(text("SELECT blank_columns, blank_sample FROM catalog_draft_tasks")).one() == (0, None) + # 闭环第一步：逐段落盘的段数、块进度。存量行是 0 和 NULL：升级前没记这些 + assert conn.execute(text("SELECT parts_written, chunks_total, chunks_done FROM catalog_draft_tasks")) \ + .one() == (0, None, None) + # 复核后：逐表的日志（没收下的有效记录条件）。存量行是 NULL，接口给空列表 + assert conn.execute(text("SELECT log FROM catalog_draft_tasks")).one() == (None,) async def test_a_job_that_cannot_start_is_failed_with_the_reason(client, monkeypatch): @@ -779,3 +786,295 @@ async def test_jobs_go_away_with_their_source(client): assert (await session.execute(select(CatalogDraftJob).where(CatalogDraftJob.id == job["id"]))).first() is None assert (await session.execute(select(CatalogDraftTask).where(CatalogDraftTask.job_id == job["id"]))).first() \ is None + + +# -------------------------------------------------------------------------- +# 闭环第一步：宽表逐段落盘、停止后的说法、块进度、停止后的留空提示、同步起草与任务互斥 +# -------------------------------------------------------------------------- + + +def _wide(n: int = 95) -> dict: + return {"wide": {"qualified": "wide", "columns": [_col(f"c{i:03d}") for i in range(1, n + 1)], + "primary_key": ["c001"], "comment": None, "is_view": False, "foreign_keys": []}} + + +def _wide_reply(blank: set[str] = frozenset()) -> dict: + """95 列的宽表每列都写中文名；blank 里的列只列出、不写（模型拿不准）。""" + return {"wide": {"label": "宽表", "kind": "fact", + "columns": [{"name": f"c{i:03d}", "label": f"第{i}列"} for i in range(1, 96) + if f"c{i:03d}" not in blank]}} + + +class Hold: + """假模型：请写的表在 tables 里、或请写的列里有 columns 里的列时，等 release 才回（停止时这次调用被取消）；别的照 + FakeModel 回。waiting：有调用在等了。""" + + def __init__(self, replies: dict, *, tables: set[str] = frozenset(), columns: set[str] = frozenset()) -> None: + self.inner = FakeModel(replies) + self.tables, self.columns = set(tables), set(columns) + self.release, self.waiting = asyncio.Event(), asyncio.Event() + self.cancelled = 0 + self.model_name = "hold-model" + + def with_structured_output(self, schema, **_): + outer = self + + class _S: + async def ainvoke(self, messages): + asked = requested("\n".join(str(m.content) for m in messages)) + cols = {c for _, listed in asked.values() for c in listed or []} + if outer.tables & set(asked) or outer.columns & cols: + outer.waiting.set() + try: + await outer.release.wait() + except asyncio.CancelledError: + outer.cancelled += 1 + raise + return outer.inner._reply(messages, "structured") + + return _S() + + async def ainvoke(self, messages): + return await self.inner.ainvoke(messages) + + +async def _task(client, sid: str, job_id: str, table: str) -> dict: + return next(t for t in (await get_job(client, sid, job_id))["tasks"] if t["table_name"] == table) + + +async def _start_held_wide(client, sid: str, monkeypatch) -> tuple[str, Hold]: + """95 列的宽表分 4 段：表级那段和第 2 段照常写，第 3、4 段一直等着。等到前两段都写进目录才返回。""" + model = Hold(_wide_reply(), columns={"c033", "c065"}) + use(monkeypatch, model) + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["wide"], "use_model": True})).json()["job"] + + async def two_written() -> bool: + return (await _task(client, sid, job["id"], "wide"))["parts_written"] == 2 + + await until(two_written) + return job["id"], model + + +async def test_a_stopped_wide_table_keeps_the_parts_already_written(client, monkeypatch): + """宽表每写出一段就并入目录：停止时已经写进去的段留着，这张表照旧记停止，model_error 说清写进了几段、剩几段没起草。""" + sid = await _source(_wide()) + job_id, model = await _start_held_wide(client, sid, monkeypatch) + task = await _task(client, sid, job_id, "wide") + assert task["status"] == "running" and task["parts_total"] == 4 and task["parts_done"] == 2 + assert (task["chunks_total"], task["chunks_done"]) == (4, 2) + assert task["added"] == 2 + 32 and task["version"] >= 1, "表级两项 + 第 2 段 32 列" + + r = await client.post(f"{_base(sid)}/draft-jobs/{job_id}/cancel") + assert r.status_code == 200, r.text + task = next(t for t in r.json()["job"]["tasks"] if t["table_name"] == "wide") + assert task["status"] == "cancelled" and task["error"] is None and task["parts_written"] == 2 + assert task["model_error"] == "已停止：已写进目录 2 段，剩下 2 段没起草" + assert model.cancelled == 2, "第 3、4 段正在进行的调用被停掉" + notes = (await client.get(f"{_base(sid)}/wide")).json()["notes"] + assert notes["label"]["value"] == "宽表" and notes["kind"]["value"] == "fact" + assert set(notes["columns"]) == {f"c{i:03d}" for i in range(1, 33)}, "第 2 段写进去的留着" + row = next(r for r in (await client.get(_base(sid))).json()["tables"] if r["table_name"] == "wide") + assert row["last_draft"]["status"] == "cancelled" and row["last_draft"]["parts_written"] == 2 + assert row["last_draft"]["blank_columns_from"] is None, "停止前写进了新内容：以前的留空列数不再沿用" + + +async def test_an_interrupted_wide_table_keeps_its_written_parts_too(client, monkeypatch): + sid = await _source(_wide()) + job_id, _ = await _start_held_wide(client, sid, monkeypatch) + await catalog_jobs.manager.shutdown(grace=5) + task = await _task(client, sid, job_id, "wide") + assert task["status"] == "cancelled" and task["error"] == "服务重启，起草中断" + assert task["model_error"] == "已中断：已写进目录 2 段，剩下 2 段没起草" + assert len((await client.get(f"{_base(sid)}/wide")).json()["notes"]["columns"]) == 32 + + # 进程没来得及收尾（直接退出）：下次启动时同样写清 + async with SessionLocal() as session: + job = CatalogDraftJob(source_id=sid, status="running", use_model=True) + session.add(job) + await session.flush() + session.add(CatalogDraftTask(job_id=job.id, seq=0, table_name="wide", status="running", parts_total=3, + parts_written=1)) + session.add(CatalogDraftTask(job_id=job.id, seq=1, table_name="gates", status="pending", parts_total=1)) + await session.commit() + crashed = job.id + await catalog_jobs.startup() + by = {t["table_name"]: t for t in (await get_job(client, sid, crashed))["tasks"]} + assert by["wide"]["model_error"] == "已中断：已写进目录 1 段，剩下 2 段没起草" + assert by["gates"]["model_error"] is None and by["gates"]["error"] == "服务重启，起草中断" + + +async def test_a_finished_wide_table_counts_every_write_once(client, monkeypatch): + """逐段落盘之后整张表收尾一次：新增、更新的计数是各次写入的合计（每一项只数一次），块进度和写进去的段数都记下。""" + sid = await _source(_wide()) + job = await _draft(client, sid, monkeypatch, FakeModel(_wide_reply({"c010", "c040", "c070"})), ["wide"]) + task = job["tasks"][0] + assert task["status"] == "ok" and task["parts_written"] == 4 and task["parts_done"] == 4 + assert (task["chunks_total"], task["chunks_done"]) == (4, 4) + assert (task["added"], task["updated"], task["removed"]) == (2 + 92, 0, 0) + assert task["blank_columns"] == 3 and task["blank_sample"] == ["c010", "c040", "c070"] + notes = (await client.get(f"{_base(sid)}/wide")).json()["notes"] + assert len(notes["columns"]) == 92 + + plain = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["wide"]})).json()["job"] + await finished(plain["id"]) + task = (await get_job(client, sid, plain["id"]))["tasks"][0] + assert (task["parts_written"], task["chunks_total"], task["chunks_done"]) == (0, None, None), "不用模型时为 null" + + +async def test_chunk_progress_shows_a_part_that_was_split(client, monkeypatch): + """块进度：这张表实际分成的块数和已结束的块数，一段超时拆成两半后变多。""" + monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.05) + sid = await _source(_tickets(24)) + replies = {"tickets": {"label": "门票", "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} + for i in range(1, 25)]}} + job = await _draft(client, sid, monkeypatch, FakeModel(replies, hang=1), ["tickets"]) + task = job["tasks"][0] + assert task["status"] == "ok" and (task["chunks_total"], task["chunks_done"]) == (2, 2) + assert task["parts_written"] == 1 and task["parts_total"] == 1 + + +async def test_a_partial_table_also_drops_model_items_of_columns_gone_from_the_schema(client, monkeypatch): + """整张表写完时按完整覆盖的规则收尾，只写出一部分的也一样：只删已经不在表结构里的列的模型起草项。""" + sid = await _source(_tickets(6)) + full = {"label": "门票", "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} for i in range(1, 7)]} + await _draft(client, sid, monkeypatch, FakeModel({"tickets": full}), ["tickets"]) + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + row.schema_cache = {"tables": _tickets(5)} # 重新探查过：c06 已经没有了 + await session.commit() + some = {"label": "门票", "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} for i in range(1, 5)]} + job = await _draft(client, sid, monkeypatch, FakeModel({"tickets": some}, list_all=False), ["tickets"]) + task = job["tasks"][0] + assert task["status"] == "partial" and task["removed"] == 1 + after = (await client.get(f"{_base(sid)}/tickets")).json()["notes"]["columns"] + assert set(after) == {"c01", "c02", "c03", "c04", "c05"}, "c05 是以前起草的，这次没写出来也留着" + + +async def test_blank_columns_from_carries_the_last_finished_draft_over_a_stop(client, monkeypatch): + """目录清单的 last_draft.blank_columns_from：最近一次结束于完成、部分完成的起草记下的留空列数。之后的起草停止了、 + 没写进新内容时照样带着，界面据此继续提示；不用模型的起草不算。""" + sid = await _source(_tickets(4)) + replies = {"tickets": {"label": "门票", "columns": [{"name": "c01", "label": "编号"}]}} + await _draft(client, sid, monkeypatch, FakeModel(replies), ["tickets"]) + + async def last() -> dict: + rows = (await client.get(_base(sid))).json()["tables"] + return next(r for r in rows if r["table_name"] == "tickets")["last_draft"] + + first = await last() + assert first["status"] == "ok" and first["blank_columns"] == 3 and first["blank_columns_from"] == 3 + assert first["parts_written"] == 1 + + plain = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["tickets"]})).json()["job"] + await finished(plain["id"]) + assert (await last())["blank_columns_from"] == 3, "不用模型的起草不改模型留空的列" + + monkeypatch.setattr(catalog_jobs.manager, "start", lambda job_id: None) + queued = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["tickets"], "use_model": True})).json() + await client.post(f"{_base(sid)}/draft-jobs/{queued['job']['id']}/cancel") + stopped = await last() + assert stopped["status"] == "cancelled" and stopped["blank_columns"] == 0 and stopped["parts_written"] == 0 + assert stopped["blank_columns_from"] == 3 + + other = await _source(_tickets(4)) + rows = (await client.get(_base(other))).json()["tables"] + assert rows[0]["last_draft"] is None + + +async def test_a_sync_draft_and_draft_jobs_exclude_each_other(client, monkeypatch): + """同步起草进行期间，同一个源建任务、重试、再开一次同步起草都 409 catalog_draft_busy（导入照 ensure_idle 认）； + 同步起草结束后照常。""" + sid = await _source() + use(monkeypatch, FakeModel(REPLIES)) + old = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"], "use_model": True})).json()["job"] + await finished(old["id"]) + async with SessionLocal() as session: + await session.execute( + CatalogDraftTask.__table__.update().where(CatalogDraftTask.job_id == old["id"]).values(status="failed")) + await session.commit() + + model = Hold(REPLIES, tables={"visits"}) + use(monkeypatch, model) + sync = asyncio.create_task(client.post(f"{_base(sid)}/draft", json={"tables": ["visits"], "use_model": True})) + await asyncio.wait_for(model.waiting.wait(), 5) + + busy = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]}) + assert busy.status_code == 409 and busy.json()["code"] == "catalog_draft_busy", busy.text + assert "正在起草" in busy.json()["detail"] + retry = await client.post(f"{_base(sid)}/draft-jobs/{old['id']}/retry") + assert retry.status_code == 409 and retry.json()["code"] == "catalog_draft_busy", retry.text + again = await client.post(f"{_base(sid)}/draft", json={"tables": ["gates"]}) + assert again.status_code == 409 and again.json()["code"] == "catalog_draft_busy", again.text + async with SessionLocal() as session: + with pytest.raises(catalog_jobs.DraftBusy): + await catalog_jobs.ensure_idle(session, await session.get(DataSource, sid)) + assert (await client.get(f"{_base(sid)}/draft-jobs/latest")).json()["job"]["id"] == old["id"], "没建新任务" + + model.release.set() + done = await sync + assert done.status_code == 200, done.text + after = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]}) + assert after.status_code == 202, after.text + await finished(after.json()["job"]["id"]) + + +async def test_a_wide_table_whose_closing_write_fails_keeps_the_count_of_parts_on_disk(client, monkeypatch): + """整张表收尾那一次写不进去（一直撞上别人刚改过）：已经逐段写进去的段留着，写进目录的段数照库里的记，不按引擎写出来 + 的段数虚报。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) # 一段一段来：最后一段只在收尾那次写 + sid = await _source(_wide()) + use(monkeypatch, FakeModel(_wide_reply())) + real = catalog.write_draft + + async def conflicted(session, source, table, modeled=None, *, at, actor): + result = await real(session, source, table, None, at=at, actor=actor) + result.error, result.status = catalog.WRITE_CONFLICT_ERROR, "failed" + result.added = result.updated = result.removed = 0 + return result + + monkeypatch.setattr(catalog, "write_draft", conflicted) + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["wide"], "use_model": True})).json()["job"] + await finished(job["id"]) + task = await _task(client, sid, job["id"], "wide") + assert task["status"] == "failed" and task["error"] == catalog.WRITE_CONFLICT_ERROR + assert task["parts_written"] == 3, "最后一段没写进去" + assert len((await client.get(f"{_base(sid)}/wide")).json()["notes"]["columns"]) == 64 + + +async def test_a_dropped_filter_is_logged_on_the_task_not_failed(client, monkeypatch): + """模型提议的有效记录条件没通过校验：丢掉、不算失败（表照样完成），逐表的 log 里记一条，接口看得到；没有就是 + 空列表。""" + sid = await _source() + replies = {**REPLIES, "gates": {"label": "闸口", + "valid_filter": {"condition": "gate_code IN (SELECT 1)", "reason": "猜的"}}} + job = await _draft(client, sid, monkeypatch, FakeModel(replies), ["gates", "channels"]) + by = {t["table_name"]: t for t in job["tasks"]} + assert by["gates"]["status"] == "ok" and by["gates"]["model_error"] is None + assert by["gates"]["log"] == ["有效记录条件里不能有子查询，模型提议的这一条没有收下：gate_code IN (SELECT 1)"] + assert by["channels"]["log"] == [] + notes = (await client.get(f"{_base(sid)}/gates")).json()["notes"] + assert "valid_filter" not in notes and notes["label"]["value"] == "闸口" + + +async def test_each_write_of_a_job_stamps_its_own_time(client, monkeypatch): + """起草任务写进目录的项，修改时间是那一次写入的时间：宽表逐段落盘的每一段、整张表收尾各取一次当时的时间。以前只在任务 + 开始时取一次（截到秒），整个任务写进去的项都是开始那一刻，比任务的开始时间还早，也看不出哪一段是什么时候写的。 + + 时钟换成每取一次走一分钟的假时钟；同时只起草一段，各段的先后是定的：表级那段、第 2、3 段逐段落盘，第 4 段由收尾写进去。""" + ticks = itertools.count() + start = datetime(2026, 10, 14, 3, tzinfo=timezone.utc) + monkeypatch.setattr(catalog, "now_iso", + lambda: (start + timedelta(minutes=next(ticks))).isoformat(timespec="seconds")) + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + sid = await _source(_wide()) + job = await _draft(client, sid, monkeypatch, FakeModel(_wide_reply()), ["wide"]) + task = job["tasks"][0] + assert task["status"] == "ok" and task["parts_written"] == 4 + notes = (await client.get(f"{_base(sid)}/wide")).json()["notes"] + cols = notes["columns"] + parts = [{cols[f"c{i:03d}"]["label"]["updated_at"] for i in range(lo, hi + 1)} + for lo, hi in ((1, 32), (33, 64), (65, 95))] + assert all(len(p) == 1 for p in parts), "同一段的项是同一次写进去的" + assert notes["kind"]["updated_at"] == notes["label"]["updated_at"], "表级那段的两项是同一次写进去的" + stamps = [notes["label"]["updated_at"], *(p.pop() for p in parts)] + assert len(set(stamps)) == 4 and stamps == sorted(stamps), stamps diff --git a/backend/tests/test_catalog_patch.py b/backend/tests/test_catalog_patch.py index 5520313b..e9c1a49d 100644 --- a/backend/tests/test_catalog_patch.py +++ b/backend/tests/test_catalog_patch.py @@ -169,6 +169,39 @@ async def test_apply_patch_records_human_confirmed(scenic_db): catalog.apply_patch(proposed, [{"path": "kind", "value": "nope"}], table="visits", tables=tables) +_PARKS = catalog.relation_id("visits", ["park_id"], "parks", ["id"]) + + +async def test_rejected_items_show_their_value_before(scenic_db): + """提案落在已驳回的项上：改前带上被驳回的原值、状态写已驳回（同导入预览「目标里这一项已经驳回」）。只改预览给出的 + 信息，保存的规矩不变：照旧换成人工确认的新值；码值不在驳回的码值上补；和驳回的值一样的也不算「值不变」，照样存成 + 人工确认。""" + tables = (await _schema(scenic_db))["tables"] + rejected = scenic_notes.notes("rejected")["visits"] + rel_value = {"columns": ["park_id"], "to_table": "parks", "to_columns": ["id"], "cardinality": "many_to_one"} + changes = [ + {"path": "columns.visitor_count.measure", "value": "stock"}, + {"path": "columns.status.codes", "value": {"9": "作废"}}, + {"path": "valid_filter", "value": "status = 1"}, # 和驳回的值一样 + {"path": f"relations.{_PARKS}", "value": rel_value}, # 两端、基数都和驳回的那条一样 + ] + plan = catalog.plan_patch(rejected, changes, table="visits", tables=tables) + assert plan.problems == [] + measure, codes, valid, rel = plan.changes + assert (measure.before, measure.before_status, measure.state) == ("flow", "rejected", "change") + assert (codes.before, codes.before_status, codes.state) == ({"1": "有效", "0": "作废"}, "rejected", "change") + assert codes.after == {"9": "作废"}, "码值不在驳回的码值上补" + assert (valid.before, valid.before_status, valid.state) == ("status = 1", "rejected", "change") + assert (rel.before, rel.before_status, rel.state) == ({**rel_value, "coverage": None}, "rejected", "change") + assert rel.as_dict()["before_status"] == "rejected" and rel.as_dict()["before"]["to_table"] == "parks" + out = catalog.apply_patch(rejected, changes, table="visits", tables=tables) + saved = [out["columns"]["visitor_count"]["measure"], out["columns"]["status"]["codes"], out["valid_filter"], + next(r for r in out["relations"] if r["id"] == _PARKS)] + assert all((item["source"], item["status"]) == ("human", "confirmed") for item in saved) + codes = out["columns"]["status"]["codes"] + assert codes["value"] == {"9": "作废"} and "complete" not in codes + + # -------------------------------------------------------------------------- # 协议 # -------------------------------------------------------------------------- @@ -337,6 +370,8 @@ async def test_preview_and_save(scenic_source): assert detail["version"] == version + 1 and detail["updated_by"] == "王敏" codes = detail["notes"]["columns"]["status"]["codes"] assert codes["value"]["9"] == "作废" and (codes["source"], codes["status"]) == ("human", "confirmed") + # 依据随项另存（确认时的依据，规则见 test_catalog_reason.py），不拼进备注 + assert codes["reason"] == "作废单" and "note" not in codes # 拿旧版本再存：409，什么都不写 r = await client.post(f"{base}/patch", json={"changes": changes, "if_version": version}) diff --git a/backend/tests/test_catalog_reason.py b/backend/tests/test_catalog_reason.py new file mode 100644 index 00000000..0a51d3c8 --- /dev/null +++ b/backend/tests/test_catalog_reason.py @@ -0,0 +1,260 @@ +"""保存修改提案时的依据：目录项上单独的一个字段 reason（界面上叫「确认时的依据」），和备注 note 分开。 + +表助手、助手流里「建议更新数据目录」的提案经 /catalog/{table}/patch 保存，带了依据的项把依据记进这一项的 reason（最多 +500 字，超了截成 499 字加「…」）；note 照旧是起草的依据、剖析的测量说明、双盲复算的出处这类机器写的话，不拼依据。 + +- 去留：值改了（关联关系改了值也一样）换成这次的依据，没有依据就去掉旧的；值不变、保存即确认的换成这次的（不叠加）； + 单项审阅的确认、驳回留着，撤销审阅（回到推断）去掉；剖析、起草、导入以外的来源改写这一项（值变了）时去掉；剖析刷新 + 关联关系的测量只动 note。 +- 校验认它（文字、最多 500 字）；起草、剖析的合并不碰人工确认的项，依据留着；导入时已确认、已驳回的项连依据一起带， + 推断、有确证的项不带。 + +以前把依据拼进 note（「依据：……。数据剖析：……」）：再剖析时测量的说法换不了新，和覆盖率那一栏对不上；改了值的关联关系 +旧依据叠在新依据后面；撤销审阅回到推断，「依据：使用者说：……」还挂在推断项上，看起来像使用者确认过。 + +表名、列名、数据全部虚构。 +""" +from __future__ import annotations + +from app.data import catalog, catalog_profile, catalog_transfer +from app.data.catalog import make_item + +AT = "2026-10-14T08:00:00+00:00" +LATER = "2026-10-15T08:00:00+00:00" + +#: 提案核对用的表结构（只要列名和主键） +TABLES = { + "visits": {"qualified": "visits", "primary_key": ["id"], + "columns": [{"name": c} for c in ("id", "ticket_no", "park_id", "visitor_count", "status")]}, + "parks": {"qualified": "parks", "primary_key": ["id"], "columns": [{"name": "id"}, {"name": "name"}]}, +} +PARKS = catalog.relation_id("visits", ["park_id"], "parks", ["id"]) +PARK_ENDS = {"columns": ["park_id"], "to_table": "parks", "to_columns": ["id"]} +PROFILE_NOTE = "数据剖析：抽样 1,000 行，覆盖率 97%。" +FILTER_NOTE = "依据：码值 0 是作废。剖析核对：排除 3.2% 的行（共 1,000 行）" + + +def _rel(**extra) -> dict: + """数据剖析核实过的入园记录 → 景区：覆盖率 97%，备注是测量的说法。""" + return {"id": PARKS, **PARK_ENDS, "cardinality": "many_to_one", "coverage": 0.97, "source": "profile", + "status": "verified", "note": PROFILE_NOTE, **extra} + + +def _visits() -> dict: + return { + "grain": make_item("一张门票一行", "llm"), + "valid_filter": make_item("status = 1", "llm", note=FILTER_NOTE), + "columns": {"visitor_count": {"measure": make_item("flow", "llm")}}, + "relations": [_rel()], + } + + +def _patch(notes: dict, changes: list[dict], at: str = AT) -> dict: + return catalog.apply_patch(notes, changes, table="visits", tables=TABLES, at=at) + + +# -------------------------------------------------------------------------- +# 保存提案 +# -------------------------------------------------------------------------- + + +def test_saving_keeps_the_reason_apart_from_the_note(): + out = _patch(_visits(), [ + {"path": "columns.visitor_count.measure", "value": "stock", "reason": "使用者说：在园人数是存量"}, + {"path": "columns.visitor_count.unit", "value": "人"}, # 没写依据 + {"path": "valid_filter", "value": "status = 1", "reason": "使用者说：只算有效的"}, # 值不变：保存即确认 + {"path": f"relations.{PARKS}", "value": {**PARK_ENDS, "cardinality": "one_to_one"}, + "reason": "使用者说：一个景区一条"}, + {"path": "grain", "value": "一张门票的一次入园", "reason": "长" * 600}, + ]) + measure = out["columns"]["visitor_count"]["measure"] + assert measure["reason"] == "使用者说：在园人数是存量" and "note" not in measure + assert (measure["value"], measure["source"], measure["status"]) == ("stock", "human", "confirmed") + unit = out["columns"]["visitor_count"]["unit"] + assert "reason" not in unit and "note" not in unit + # 值不变的确认：来源、备注都不变，依据另记 + valid = out["valid_filter"] + assert valid["note"] == FILTER_NOTE and valid["reason"] == "使用者说：只算有效的" + assert (valid["source"], valid["status"]) == ("llm", "confirmed") + # 关联关系改了基数：覆盖率和说明它怎么测的备注照旧沿用，备注里不拼依据 + rel = out["relations"][0] + assert rel["note"] == PROFILE_NOTE and rel["reason"] == "使用者说：一个景区一条" + assert (rel["cardinality"], rel["coverage"], rel["source"], rel["status"]) == ( + "one_to_one", 0.97, "human", "confirmed") + # 超过 500 字：截成 499 字加「…」 + grain = out["grain"] + assert grain["reason"] == "长" * 499 + "…" and "note" not in grain + assert catalog.validate_notes(out) == [] + # 正好 500 字的不截；预览给出的依据和保存的是同一句 + exact = _patch(_visits(), [{"path": "grain", "value": "一行一张票", "reason": "短" * 500}]) + assert exact["grain"]["reason"] == "短" * 500 + plan = catalog.plan_patch(_visits(), [{"path": "grain", "value": "一行一张票", "reason": "长" * 600}], + table="visits", tables=TABLES) + assert plan.changes[0].reason == "长" * 499 + "…" + + +def test_profiling_again_refreshes_the_measurement_and_keeps_the_reason(): + """带依据保存的关联关系再剖析：覆盖率和备注里测量的说法一起换新（以前备注成了「依据：……。数据剖析：……」，剖析 + 认不出是自己写的，覆盖率变了、备注却停在保存那一刻），依据不动，人定的基数不改。""" + saved = _patch({"relations": [_rel()]}, [ + {"path": f"relations.{PARKS}", "value": {**PARK_ENDS, "cardinality": "one_to_one"}, + "reason": "使用者说：一个景区一条"}, + ]) + outcome = catalog_profile._TableOutcome(relations=[catalog_profile._RelationOutcome( + rid=PARKS, target="parks.id", coverage=0.99, cardinality="many_to_one", holds=True, sample=1000, matched=990, + note="数据剖析：抽样 1,000 行，覆盖率 99%。")]) + merged, findings, stats = catalog_profile._apply(saved, outcome, at=LATER) + rel = merged["relations"][0] + assert rel["coverage"] == 0.99 and rel["note"] == "数据剖析：抽样 1,000 行，覆盖率 99%。" + assert rel["reason"] == "使用者说：一个景区一条" and rel["cardinality"] == "one_to_one" + assert (rel["source"], rel["status"], rel["updated_at"]) == ("human", "confirmed", LATER) + assert stats.updated == 1 and findings[0]["confirmed"] is True + + +def test_saving_again_replaces_the_reason_instead_of_piling_it_up(): + first = _patch(_visits(), [ + {"path": f"relations.{PARKS}", "value": {**PARK_ENDS, "cardinality": "one_to_one"}, + "reason": "使用者说：一个景区一条"}, + {"path": "columns.visitor_count.measure", "value": "stock", "reason": "使用者说：在园人数是存量"}, + {"path": "valid_filter", "value": "status = 1", "reason": "使用者说：只算有效的"}, + ]) + again = _patch(first, [ + # 关联关系又改了值：换成这次的依据，旧的不叠在后面；备注照旧只是测量的说法 + {"path": f"relations.{PARKS}", "value": {**PARK_ENDS, "cardinality": "many_to_one"}, + "reason": "使用者说：一个景区有多条入园记录"}, + # 改了值、这次没写依据：旧的依据说的是旧值，去掉 + {"path": "columns.visitor_count.measure", "value": "ratio"}, + ]) + rel = again["relations"][0] + assert rel["reason"] == "使用者说：一个景区有多条入园记录" and rel["note"] == PROFILE_NOTE + assert "reason" not in again["columns"]["visitor_count"]["measure"] + # 撤销审阅回到推断：依据去掉；再一次保存即确认，只有这一次的依据，备注还是起草时的那句 + reset = catalog.review_item(again, "valid_filter", "reset", at=AT) + assert reset["valid_filter"]["status"] == "proposed" and "reason" not in reset["valid_filter"] + confirmed = _patch(reset, [{"path": "valid_filter", "value": "status = 1", "reason": "使用者说：确认无误"}]) + assert confirmed["valid_filter"]["reason"] == "使用者说：确认无误" + assert confirmed["valid_filter"]["note"] == FILTER_NOTE + # 推断项上不知怎么留着一句依据（手改过的数据）：保存即确认时换成这一次的，不叠 + stray = {"grain": {**make_item("一张门票一行", "llm"), "reason": "旧的依据"}} + assert _patch(stray, [{"path": "grain", "value": "一张门票一行", "reason": "新的依据"}])["grain"]["reason"] == "新的依据" + assert "reason" not in _patch(stray, [{"path": "grain", "value": "一张门票一行"}])["grain"] + + +# -------------------------------------------------------------------------- +# 审阅、人工编辑 +# -------------------------------------------------------------------------- + + +def test_review_keeps_the_reason_until_the_review_is_undone(): + notes = {"grain": make_item("一张门票一行", "llm", "confirmed", reason="使用者说：一张票一行"), + "label": make_item("入园记录", "human", "confirmed", reason="使用者说：就叫入园记录")} + rejected = catalog.review_item(notes, "grain", "reject", at=AT) + assert rejected["grain"]["status"] == "rejected" and rejected["grain"]["reason"] == "使用者说：一张票一行" + again = catalog.review_item(rejected, "grain", "confirm", at=AT) + assert again["grain"]["status"] == "confirmed" and again["grain"]["reason"] == "使用者说：一张票一行" + reset = catalog.review_item(again, "grain", "reset", at=AT) + assert reset["grain"]["status"] == "proposed" and "reason" not in reset["grain"] + # 人工填写的项撤销就是删掉，依据一起没了 + assert "label" not in catalog.review_item(notes, "label", "reset", at=AT) + + +def test_editing_by_hand_drops_the_reason_only_when_the_value_changes(): + """整份提交（表详情里的编辑）不写依据：改了值的项去掉旧的依据（界面原样带回来的也不认），值没动的照旧留着库里的， + 状态改回推断等同撤销审阅、去掉；去掉的推断关系转为驳回，依据留着（同单项驳回）。""" + rel = {"id": PARKS, **PARK_ENDS, "cardinality": "many_to_one", "coverage": None, "source": "name", + "status": "confirmed", "reason": "使用者说：一个景区一条"} + existing = { + "label": make_item("入园记录", "human", "confirmed", reason="使用者说：就叫入园记录"), + "grain": make_item("一张门票一行", "llm", "confirmed", reason="使用者说：一张票一行"), + "kind": make_item("fact", "llm", "confirmed", reason="使用者说：是明细"), + "relations": [rel], + } + submitted = { + "label": {**existing["label"], "value": "入园流水"}, + "grain": {**existing["grain"], "reason": "界面上写不了依据"}, + "kind": {**existing["kind"], "status": "proposed"}, + "columns": {"status": {"codes": {"value": {"1": "有效"}, "reason": "新填的也不收"}}}, + } + out = catalog.apply_human_edit(existing, submitted, table="visits", at=AT) + assert out["label"]["value"] == "入园流水" and "reason" not in out["label"] + assert out["grain"]["reason"] == "使用者说：一张票一行" + assert out["kind"]["status"] == "proposed" and "reason" not in out["kind"] + codes = out["columns"]["status"]["codes"] + assert (codes["source"], codes["status"]) == ("human", "confirmed") and "reason" not in codes + [kept] = out["relations"] + assert kept["status"] == "rejected" and kept["reason"] == "使用者说：一个景区一条" + assert catalog.validate_notes(out) == [] + + +# -------------------------------------------------------------------------- +# 校验、合并、导入 +# -------------------------------------------------------------------------- + + +def test_the_reason_is_text_of_at_most_500_characters(): + ok = {"label": make_item("入园记录", "human", "confirmed", reason="短" * 500), + "columns": {"status": {"codes": make_item({"1": "有效"}, "human", "confirmed", complete=True, + reason="使用者说：1 是有效")}}, + "relations": [_rel(status="confirmed", reason="使用者说：一个景区一条")]} + assert catalog.validate_notes(ok) == [] + bad = catalog.validate_notes({"label": {**make_item("入园记录", "human", "confirmed"), "reason": "长" * 501}, + "grain": {**make_item("一行", "human", "confirmed"), "reason": 3}, + "relations": [_rel(reason=["使用者说"])]}) + assert len(bad) == 3 and all("确认时的依据" in p for p in bad), bad + + +def test_drafting_and_profiling_leave_the_reason_of_reviewed_items_alone(): + existing = {"label": make_item("入园记录", "llm", "confirmed", reason="使用者说：就叫入园记录"), + "grain": make_item("一张门票一行", "llm", "rejected", reason="使用者说：一张票一行"), + "columns": {"status": {"codes": make_item({"1": "有效"}, "profile", "confirmed", + reason="使用者说：1 是有效")}}} + draft = {"label": make_item("入园流水", "llm"), "grain": make_item("每次刷闸一行", "llm"), + "kind": make_item("fact", "llm")} + merged, _ = catalog.merge_notes(existing, draft, covered={"llm"}, at=AT, columns=["id", "status"]) + assert merged["label"] == existing["label"] and merged["grain"] == existing["grain"] + assert "reason" not in merged["kind"] + outcome = catalog_profile._TableOutcome(codes=[catalog_profile._CodesOutcome( + column="status", values=[("1", 900), ("0", 100)], rows=1000, note="数据剖析：全表 1,000 行，2 个取值。", + complete=True)]) + profiled, _, _ = catalog_profile._apply(merged, outcome, at=LATER) + assert profiled["columns"]["status"]["codes"] == existing["columns"]["status"]["codes"] + + +def test_importing_carries_the_reason_of_confirmed_and_rejected_items_only(): + cache = {"tables": {"visits": TABLES["visits"]}} + file_notes = { + "label": make_item("入园记录", "human", "confirmed", reason="使用者说：就叫入园记录"), + "grain": make_item("一张门票一行", "llm", "rejected", reason="使用者说：一张票一行"), + # 推断项上带着依据（文件被手改过）：只填空缺，不带确认时的依据 + "description": {**make_item("每次刷闸记一行", "llm"), "reason": "手改的文件"}, + } + plan = catalog_transfer.plan_table("visits", "visits", file_notes, {}, cache, set(), "sqlite") + assert plan.skipped == [] and plan.add == 3 + writes = {key[1]: item for key, item in plan.writes.items()} + assert writes["label"]["reason"] == "使用者说：就叫入园记录" + assert writes["grain"]["reason"] == "使用者说：一张票一行" + assert writes["description"]["value"] == "每次刷闸记一行" and "reason" not in writes["description"] + + +def test_profiling_keeps_the_cardinality_of_a_confirmed_foreign_key(): + """外键来的关联关系经「值不变、保存即确认」带依据确认之后，来源仍是 fk。再剖析时数据核实出的基数不同：以前 _patch_fk + 不看状态，直接改写人确认过的基数，依据还挂着，前后矛盾。现在同 _patch_confirmed，不改人定的基数、也不标「用数据核实过」， + 依据留着；测量的说法照常写进备注。没确认过的外键关系照旧按数据改基数、标核实过。""" + fk = {k: v for k, v in _rel(source="fk", status="verified").items() if k not in ("coverage", "note")} + saved = _patch({"relations": [fk]}, [ + {"path": f"relations.{PARKS}", "value": {**PARK_ENDS, "cardinality": "many_to_one"}, + "reason": "使用者说：一个景区有多条入园记录"}, + ]) + rel = saved["relations"][0] + assert (rel["source"], rel["status"], rel["reason"]) == ("fk", "confirmed", "使用者说：一个景区有多条入园记录") + outcome = catalog_profile._TableOutcome(relations=[catalog_profile._RelationOutcome( + rid=PARKS, target="parks.id", coverage=None, cardinality="one_to_one", cardinality_checked=True, holds=True, + sample=1000, matched=1000, note="数据剖析：子表一侧每个键都只有一行。")]) + rel = catalog_profile._apply(saved, outcome, at=LATER)[0]["relations"][0] + assert rel["cardinality"] == "many_to_one" and rel.get("cardinality_checked") is not True + assert (rel["source"], rel["status"], rel["reason"]) == ("fk", "confirmed", "使用者说：一个景区有多条入园记录") + assert rel["note"] == "数据剖析：子表一侧每个键都只有一行。" + + plain = catalog_profile._apply({"relations": [fk]}, outcome, at=LATER)[0]["relations"][0] + assert plain["cardinality"] == "one_to_one" and plain["cardinality_checked"] is True + assert (plain["source"], plain["status"]) == ("fk", "verified") diff --git a/backend/tests/test_catalog_review_queue.py b/backend/tests/test_catalog_review_queue.py new file mode 100644 index 00000000..239694d2 --- /dev/null +++ b/backend/tests/test_catalog_review_queue.py @@ -0,0 +1,397 @@ +"""数据目录的审阅队列、批量审阅、表被哪些运行用过、审阅进度（data/catalog_review.py）。 + +- GET /catalog/review-queue：只列推断（proposed）的项，按「使用次数权重 × 风险权重 × 来源权重」从高到低，同分按表名、 + 路径；facts（有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型）/ names（中文名、说明、含义、单位、码值、 + 粒度、表类型）两组；按表筛选；分页；field_label 是给人看的位置（「列 status 的码值」）；reasons 写为什么排在这里 + （被几次运行用到，和 /usage 的 total 一致）；table_usage 是使用次数（同目录清单）；表结构里已经没有的表、列不列 +- POST /catalog/review/batch：逐表按 if_version 乐观锁，一张表冲突不影响别的表；同一张表的几条一次写入；路径不对、表不存在 + 逐条报 +- GET /catalog/{table}/usage：用过这张表的运行，新的在前，取法同 catalog.table_usage +- GET /catalog/review-progress：常用 20 张表（使用次数前 20 张，同起草的默认范围：用到的不够 20 张时按表结构的顺序 + 补齐）、全部、影响数字的项各自的确认情况 + +表名、列名、业务数字都是虚构的。 +""" +from __future__ import annotations + +import re +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from urllib.parse import quote + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.data import catalog, catalog_review +from app.data.catalog import make_item +from app.db.base import SessionLocal +from app.db.models import Artifact, DataSource, Run +from app.main import app as _fastapi_app +from tests.fixtures.sources import drop_source + +ACTOR = {"X-Actor": quote("林晓")} +TERMS = Path(__file__).resolve().parents[2] / "frontend" / "src" / "lib" / "terms.ts" + + +def _col(name: str, type_: str = "INTEGER") -> dict: + return {"name": name, "type": type_, "nullable": True, "comment": None} + + +def _tables() -> dict: + return { + "orders": {"qualified": "orders", "primary_key": ["id"], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], + "columns": [_col("id"), _col("status"), _col("amount", "REAL"), _col("member_id"), + _col("paid_at", "TEXT")]}, + "members": {"qualified": "members", "primary_key": ["id"], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], + "columns": [_col("id"), _col("nick", "TEXT"), _col("tier")]}, + "coupons": {"qualified": "coupons", "primary_key": ["id"], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], "columns": [_col("id"), _col("face_value", "REAL")]}, + } + + +def _member_rel(status: str = "proposed", source: str = "name") -> dict: + return {"id": catalog.relation_id("orders", ["member_id"], "members", ["id"]), "columns": ["member_id"], + "to_table": "members", "to_columns": ["id"], "cardinality": "many_to_one", "coverage": 0.98, + "source": source, "status": status} + + +ORDERS = { + "label": make_item("订单", "llm"), + "grain": make_item("每笔订单一行", "llm", "confirmed"), + "valid_filter": make_item("status <> 9", "llm", note="剖析核对：排除 3.2% 的行（共 12,345 行）"), + "kind": make_item("fact", "llm"), + "columns": { + "status": {"label": make_item("状态", "comment"), "codes": make_item({"1": "已支付", "9": ""}, "profile")}, + "amount": {"measure": make_item("flow", "llm"), "unit": make_item("元", "llm", "rejected")}, + # 表结构里已经没有这一列：不进队列 + "legacy_flag": {"label": make_item("旧标记", "llm")}, + }, + "relations": [_member_rel()], +} +MEMBERS = {"label": make_item("会员", "llm"), "keys": make_item(["id"], "name"), + "columns": {"nick": {"label": make_item("昵称", "comment")}}} +COUPONS = {"label": make_item("优惠券", "comment")} +#: 表结构里已经没有这张表：不进队列 +GONE = {"label": make_item("旧表", "llm")} + +_CREATED: list[str] = [] + + +@pytest.fixture(autouse=True) +async def _cleanup(): + yield + while _CREATED: + await drop_source(_CREATED.pop()) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c: + yield c + + +async def _write(sid: str, table: str, notes: dict) -> None: + async with SessionLocal() as session: + entry = await catalog.read_entry(session, sid, table) + await catalog.write_entry(session, sid, table, notes, if_version=entry.version if entry else 0, actor=None) + + +async def _use(sid: str, name: str, table_runs: dict[str, int]) -> list[str]: + """给每张表造几次运行查询它（每次运行一份查询快照），返回运行 id（先建的在前、开始得早）。""" + runs: list[str] = [] + base = datetime(2026, 9, 1, tzinfo=timezone.utc) + async with SessionLocal() as session: + for table, n in table_runs.items(): + for _ in range(n): + rid = uuid.uuid4().hex + session.add(Run(id=rid, graph={}, workflow_name=f"周报-{table}", run_class="formal", + status="succeeded", started_at=base + timedelta(hours=len(runs)))) + session.add(Artifact(id=uuid.uuid4().hex, run_id=rid, kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": [table]})) + runs.append(rid) + await session.commit() + return runs + + +async def _source(*, usage: dict[str, int] | None = None) -> str: + name = f"rq_{uuid.uuid4().hex[:10]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=":memory:", schema_cache={"tables": _tables()}) + session.add(row) + await session.commit() + sid = row.id + _CREATED.append(sid) + for table, notes in (("orders", ORDERS), ("members", MEMBERS), ("coupons", COUPONS), ("gone_table", GONE)): + await _write(sid, table, notes) + if usage: + await _use(sid, name, usage) + return sid + + +def _base(sid: str) -> str: + return f"/api/datasources/{sid}/catalog" + + +# -------------------------------------------------------------------------- +# 排序规则（纯函数） +# -------------------------------------------------------------------------- + + +def test_priority_multiplies_usage_risk_and_source_weights(): + p = catalog_review.priority + # 风险：有效记录条件 > 业务主键 > 码值 > 中文名 + assert p(0, ("t", "valid_filter"), "llm") > p(0, ("t", "keys"), "llm") > p(0, ("c", "x", "codes"), "llm") \ + > p(0, ("c", "x", "label"), "llm") + assert p(0, ("r", "r1"), "name") == p(0, ("t", "dedup"), "name") == p(0, ("t", "business_date"), "name") + assert p(0, ("c", "x", "measure"), "llm") == p(0, ("t", "keys"), "llm") + assert p(0, ("t", "grain"), "llm") == p(0, ("t", "kind"), "llm") == p(0, ("c", "x", "codes"), "llm") + assert p(0, ("c", "x", "meaning"), "llm") == p(0, ("c", "x", "unit"), "llm") == p(0, ("t", "label"), "llm") + # 来源：模型起草、命名推断比数据库注释优先 + assert p(0, ("t", "label"), "llm") == p(0, ("t", "label"), "name") > p(0, ("t", "label"), "comment") + # 使用次数：用得多的在前，但按对数增长，不压过风险 + assert p(37, ("t", "label"), "llm") > p(1, ("t", "label"), "llm") > p(0, ("t", "label"), "llm") + assert p(37, ("t", "label"), "llm") == pytest.approx( + p(0, ("t", "label"), "llm") * p(37, ("t", "label"), "comment") / p(0, ("t", "label"), "comment")) + + +def test_source_labels_follow_the_ui(): + text = TERMS.read_text(encoding="utf-8") + block = re.search(r"export const CATALOG_SOURCE_LABEL\b[^=]*=\s*\{(.*?)\n\}", text, re.S) + assert block + ui = dict(re.findall(r"(\w+):\s*'([^']+)'", block.group(1))) + assert ui == catalog_review.SOURCE_LABEL + + +# -------------------------------------------------------------------------- +# 队列 +# -------------------------------------------------------------------------- + + +async def test_queue_lists_only_proposed_items_by_priority(client): + sid = await _source(usage={"orders": 37, "members": 2}) + r = await client.get(f"{_base(sid)}/review-queue") + assert r.status_code == 200, r.text + body = r.json() + items = body["items"] + paths = [(i["table_name"], i["path"]) for i in items] + # 已确认、已驳回的不列；表结构里没有的表、列不列 + assert ("orders", "grain") not in paths and ("orders", "columns.amount.unit") not in paths + assert not any(t == "gone_table" for t, _ in paths) + assert ("orders", "columns.legacy_flag.label") not in paths + assert body["total"] == len(items) == 11 + assert all(i["status"] == "proposed" for i in items) + # 从高到低，同分按表名、路径 + keys = [(-i["priority"], i["table_name"], i["path"]) for i in items] + assert keys == sorted(keys) + first = items[0] + rel_id = _member_rel()["id"] + assert (first["table_name"], first["path"]) in {("orders", "valid_filter"), ("orders", f"relations.{rel_id}")} + vf = next(i for i in items if i["path"] == "valid_filter") + assert vf["field"] == "valid_filter" and vf["field_label"] == "表的有效记录条件" + assert vf["value"] == "status <> 9" and vf["display"] == "status <> 9" + assert vf["source"] == "llm" and vf["note"] == "剖析核对：排除 3.2% 的行（共 12,345 行）" + assert vf["table_usage"] == 37 and vf["version"] == 1 + assert vf["reasons"] == ["常用表（被 37 次运行用到）", "影响数字的项（有效记录条件）", "模型起草，未确认"] + # 常用表上的中文名排在没人用的表上的中文名前面；同一张表上影响数字的项排在中文名前面 + order = [(i["table_name"], i["path"]) for i in items] + assert order.index(("orders", "label")) < order.index(("coupons", "label")) + assert order.index(("members", "keys")) < order.index(("members", "label")) + + +async def test_queue_values_are_displayed_in_ui_words(client): + sid = await _source() + items = {(i["table_name"], i["path"]): i for i in (await client.get(f"{_base(sid)}/review-queue")).json()["items"]} + rel = items[("orders", f"relations.{_member_rel()['id']}")] + assert rel["field"] == "relation" and rel["field_label"] == "指向 members 的关联关系" + assert rel["value"] == {"columns": ["member_id"], "to_table": "members", "to_columns": ["id"], + "cardinality": "many_to_one", "coverage": 0.98} + assert rel["display"] == "member_id → members.id（多对一，覆盖率 98%）" + assert rel["reasons"][-1] == "命名推断，未确认" + kind = items[("orders", "kind")] + assert kind["display"] == "明细表" and kind["value"] == "fact" + codes = items[("orders", "columns.status.codes")] + assert codes["field"] == "codes" and codes["field_label"] == "列 status 的码值" and codes["column"] == "status" + assert codes["display"] == "1=已支付、9=含义待填写" + assert "1 个码值的含义待填写" in codes["reasons"] and "数据剖析，未确认" in codes["reasons"] + assert items[("orders", "columns.amount.measure")]["display"] == "可累加" + assert items[("members", "keys")]["display"] == "id" and items[("members", "keys")]["field_label"] == "表的业务主键" + assert items[("orders", "columns.amount.measure")]["field_label"] == "列 amount 的度量类型" + assert items[("members", "label")]["reasons"] == ["名称和说明（中文名）", "模型起草，未确认"] + assert items[("coupons", "label")]["reasons"] == ["名称和说明（中文名）", "数据库注释，未确认"] + + +async def test_queue_reasons_count_runs_like_the_usage_list(client): + """同一次运行里查了几次这张表：原因写「被几次运行用到」（和 /usage 的 total 一样），table_usage 是使用次数（和目录清单的 + 「使用 N 次」一样），两个数不一样时原因里两个都写；优先级按使用次数（同清单的排序）。""" + sid = await _source() + async with SessionLocal() as session: + name = (await session.get(DataSource, sid)).name + runs = await _use(sid, name, {"orders": 2, "members": 1}) + async with SessionLocal() as session: + for _ in range(3): + session.add(Artifact(id=uuid.uuid4().hex, run_id=runs[0], kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": ["orders"]})) + await session.commit() + usage = await catalog.table_usage(session, await session.get(DataSource, sid)) + assert usage["orders"] == 5 + items = {(i["table_name"], i["path"]): i for i in (await client.get(f"{_base(sid)}/review-queue")).json()["items"]} + vf = items[("orders", "valid_filter")] + assert vf["table_usage"] == 5 + assert vf["reasons"][0] == "常用表（被 2 次运行用到，使用 5 次）" + assert items[("members", "label")]["reasons"][0] == "常用表（被 1 次运行用到）" + assert vf["priority"] == round(catalog_review.priority(5, ("t", "valid_filter"), "llm"), 4) + assert (await client.get(f"{_base(sid)}/orders/usage")).json()["total"] == 2 + + +async def test_queue_filters_by_kind_and_table_and_pages(client): + sid = await _source(usage={"orders": 3}) + facts = (await client.get(f"{_base(sid)}/review-queue", params={"kind": "facts"})).json() + assert {i["field"] for i in facts["items"]} == {"valid_filter", "relation", "measure", "keys"} + names = (await client.get(f"{_base(sid)}/review-queue", params={"kind": "names"})).json() + assert {i["field"] for i in names["items"]} == {"label", "codes", "kind"} + assert facts["total"] + names["total"] == 11 + only = (await client.get(f"{_base(sid)}/review-queue", params={"table": "MEMBERS"})).json() + assert {i["table_name"] for i in only["items"]} == {"members"} and only["total"] == 3 + everything = (await client.get(f"{_base(sid)}/review-queue")).json()["items"] + page = (await client.get(f"{_base(sid)}/review-queue", params={"limit": 4, "offset": 4})).json() + assert page["total"] == 11 and page["items"] == everything[4:8] + bad = await client.get(f"{_base(sid)}/review-queue", params={"kind": "other"}) + assert bad.status_code == 422 + assert (await client.get(f"{_base(sid)}/review-queue", params={"limit": 0})).status_code == 422 + missing = await client.get("/api/datasources/nope/catalog/review-queue") + assert missing.status_code == 404 + + +# -------------------------------------------------------------------------- +# 批量审阅 +# -------------------------------------------------------------------------- + + +async def test_batch_review_writes_each_table_once_and_isolates_conflicts(client): + sid = await _source() + rel_path = f"relations.{_member_rel()['id']}" + body = {"items": [ + {"table_name": "orders", "path": "valid_filter", "action": "confirm", "if_version": 1}, + {"table_name": "orders", "path": rel_path, "action": "reject", "if_version": 1}, + {"table_name": "orders", "path": "columns.nothing.label", "action": "confirm", "if_version": 1}, + {"table_name": "members", "path": "label", "action": "confirm", "if_version": 7}, # 版本不对 + {"table_name": "coupons", "path": "label", "action": "reject", "if_version": 1}, + {"table_name": "ghost", "path": "label", "action": "confirm", "if_version": 0}, + ]} + r = await client.post(f"{_base(sid)}/review/batch", json=body, headers=ACTOR) + assert r.status_code == 200, r.text + results = r.json()["results"] + assert [(x["table_name"], x["path"], x["ok"]) for x in results] == [ + ("orders", "valid_filter", True), ("orders", rel_path, True), ("orders", "columns.nothing.label", False), + ("members", "label", False), ("coupons", "label", True), ("ghost", "label", False)] + # 同一张表的两条一次写进去：版本只加 1 + assert results[0]["version"] == results[1]["version"] == 2 + assert results[2]["code"] == "catalog_path_invalid" and "找不到" in results[2]["error"] + assert results[2]["version"] is None + assert results[3]["code"] == "catalog_conflict" and "重新载入" in results[3]["error"] + assert results[4]["version"] == 2 and results[4]["error"] is None and results[4]["code"] is None + assert results[5]["code"] == "table_missing" + async with SessionLocal() as session: + orders = await catalog.read_entry(session, sid, "orders") + members = await catalog.read_entry(session, sid, "members") + coupons = await catalog.read_entry(session, sid, "coupons") + assert orders.version == 2 and orders.updated_by == "林晓" + assert orders.notes["valid_filter"]["status"] == "confirmed" + assert orders.notes["relations"][0]["status"] == "rejected" + assert members.version == 1 and members.notes["label"]["status"] == "proposed" + assert coupons.notes["label"]["status"] == "rejected" + # 撤销审阅：回到来源的初始状态 + again = await client.post(f"{_base(sid)}/review/batch", json={"items": [ + {"table_name": "orders", "path": "valid_filter", "action": "reset", "if_version": 2}]}) + assert again.json()["results"][0]["ok"] is True + async with SessionLocal() as session: + assert (await catalog.read_entry(session, sid, "orders")).notes["valid_filter"]["status"] == "proposed" + + +async def test_batch_review_limits(client): + sid = await _source() + many = [{"table_name": "orders", "path": "label", "action": "confirm", "if_version": 1}] * 201 + assert (await client.post(f"{_base(sid)}/review/batch", json={"items": many})).status_code == 422 + assert (await client.post(f"{_base(sid)}/review/batch", json={"items": []})).status_code == 422 + wrong = [{"table_name": "orders", "path": "label", "action": "approve", "if_version": 1}] + assert (await client.post(f"{_base(sid)}/review/batch", json={"items": wrong})).status_code == 422 + ok = [{"table_name": "orders", "path": "label", "action": "confirm", "if_version": 1}] * 200 + r = await client.post(f"{_base(sid)}/review/batch", json={"items": ok}) + assert r.status_code == 200 and all(x["ok"] for x in r.json()["results"]) + + +# -------------------------------------------------------------------------- +# 被哪些运行用过 +# -------------------------------------------------------------------------- + + +async def test_table_usage_lists_runs_newest_first(client): + sid = await _source() + async with SessionLocal() as session: + name = (await session.get(DataSource, sid)).name + runs = await _use(sid, name, {"orders": 3, "members": 1}) + # 同一次运行查了两次 orders：只算一次运行 + async with SessionLocal() as session: + session.add(Artifact(id=uuid.uuid4().hex, run_id=runs[0], kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": ["orders", "members"]})) + # 别的数据源的同名表不算 + other = uuid.uuid4().hex + session.add(Run(id=other, graph={}, workflow_name="别处", status="succeeded")) + session.add(Artifact(id=uuid.uuid4().hex, run_id=other, kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": "elsewhere", "source": "elsewhere", "tables": ["orders"]})) + await session.commit() + r = await client.get(f"{_base(sid)}/orders/usage") + assert r.status_code == 200, r.text + body = r.json() + assert body["total"] == 3 + assert [x["run_id"] for x in body["runs"]] == [runs[2], runs[1], runs[0]] + first = body["runs"][0] + assert first["workflow_name"] == "周报-orders" and first["run_class"] == "formal" + assert first["status"] == "succeeded" and first["started_at"].startswith("2026-09-01T02:00") + assert set(first) == {"run_id", "workflow_id", "workflow_name", "run_class", "status", "started_at"} + limited = (await client.get(f"{_base(sid)}/orders/usage", params={"limit": 1})).json() + assert limited["total"] == 3 and len(limited["runs"]) == 1 + members = (await client.get(f"{_base(sid)}/members/usage")).json() + assert members["total"] == 2 + assert (await client.get(f"{_base(sid)}/ghost/usage")).status_code == 404 + + +# -------------------------------------------------------------------------- +# 进度 +# -------------------------------------------------------------------------- + + +async def test_review_progress_counts_top_tables_all_and_facts(client, monkeypatch): + sid = await _source(usage={"orders": 5}) + r = await client.get(f"{_base(sid)}/review-progress") + assert r.status_code == 200, r.text + body = r.json() + assert body["top_n"] == 20 + # 表结构里只有 3 张表：都算常用表（使用次数前 20 张）；表结构里没有的表、列不算 + assert body["top"] == {"tables": 3, "items_total": 13, "items_confirmed": 1} + assert body["all"] == {"items_total": 13, "items_confirmed": 1, "items_proposed": 11, "items_rejected": 1} + # 影响数字的项：有效记录条件、关联关系、度量类型、业务主键 + assert body["facts"] == {"proposed": 4, "confirmed": 0} + await client.post(f"{_base(sid)}/review/batch", json={"items": [ + {"table_name": "orders", "path": "valid_filter", "action": "confirm", "if_version": 1}]}) + after = (await client.get(f"{_base(sid)}/review-progress")).json() + assert after["top"]["items_confirmed"] == 2 and after["facts"] == {"proposed": 3, "confirmed": 1} + # 只取前 2 张：用得多的在前，没人用的按表结构的顺序补（同起草的默认范围） + monkeypatch.setattr(catalog_review, "TOP_N", 2) + used = await _source(usage={"coupons": 1}) + top = (await client.get(f"{_base(used)}/review-progress")).json()["top"] + assert top == {"tables": 2, "items_total": 1 + 9, "items_confirmed": 1} + + +def test_top_tables_are_the_draft_default_scope(): + """常用表和起草对话框「使用次数最多的 20 张表」是同一批：使用次数从多到少，一样多（包括都没用过）的按表结构里的顺序。""" + tables = {f"t{i:02d}": {} for i in range(30)} + usage = {"t29": 4, "t05": 4, "t17": 9} + top = catalog_review.top_tables(tables, usage) + assert catalog_review.TOP_N == catalog.DRAFT_DEFAULT_TABLES == 20 + assert top[:3] == ["t17", "t05", "t29"] and len(top) == 20 + assert top[3:] == [n for n in tables if n not in usage][:17] + assert catalog_review.top_tables({"a": {}, "b": {}}, {}) == ["a", "b"] diff --git a/backend/tests/test_catalog_transfer.py b/backend/tests/test_catalog_transfer.py new file mode 100644 index 00000000..61a210cb --- /dev/null +++ b/backend/tests/test_catalog_transfer.py @@ -0,0 +1,432 @@ +"""数据目录的导出导入（data/catalog_transfer.py）：本机整理好的目录带到部署实例上。 + +- GET /catalog/export：附件「<数据源名>-数据目录.json」，{format: agentlab-catalog, version: 1, exported_at, + source: {name, kind}, guidance, tables: {表名: {notes, version}}}，含全部状态的项（表结构里已经没有的表也带） +- POST /catalog/import/preview：只读，报告每张对上的表新增、更新、保留、冲突、跳过了什么 +- POST /catalog/import：同样的报告，加每张表写入后的版本。规则： + - 表按名字对（忽略大小写；带 schema 前缀的对不上时去掉前缀再对），对不上的列在 unmatched_tables + - 文件里已确认、已驳回的项照带（来源、状态不变）；目标里已确认、已驳回的项和文件不一样时保留目标的、记冲突 + - 文件里推断、有确证的项只填目标的空缺，不覆盖目标已有的 + - 表结构里没有的列、关系两端对不上的跳过并写原因；目标数据源遮罩的列，码值不导入；有效记录条件用到的列（按目标的方言 + 解析）不在目标的表结构里、引用别的表的列、带子查询的跳过（人工写的条件不限函数） + - 逐表乐观锁，一张表写不进去不影响别的；有排队或进行中的起草任务时 409 catalog_draft_busy；文件格式不对 422 + - 起草要求：勾了导入、目标还没有时才写；目标已有的不覆盖 + +表名、列名、业务数字都是虚构的。 +""" +from __future__ import annotations + +import copy +import json +import uuid +from urllib.parse import quote, unquote + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.data import catalog, catalog_jobs, catalog_transfer +from app.data.catalog import make_item +from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION +from app.db.base import SessionLocal +from app.db.models import CatalogDraftJob, DataSource +from app.main import app as _fastapi_app +from tests.fixtures.sources import drop_source + +ACTOR = {"X-Actor": quote("沈岚")} + + +def _col(name: str, type_: str = "INTEGER") -> dict: + return {"name": name, "type": type_, "nullable": True, "comment": None} + + +def _tables(*, upper: bool = False) -> dict: + """两边的表结构：目标那边（upper）表名、列名写法不同，还少了一列 tier。""" + def n(name: str) -> str: + return name.upper() if upper else name + + members_cols = [_col(n("id")), _col(n("nick"), "TEXT")] + ([] if upper else [_col("tier")]) + return { + n("orders"): {"qualified": n("orders"), "primary_key": [n("id")], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], + "columns": [_col(n("id")), _col(n("status")), _col(n("amount"), "REAL"), _col(n("member_id")), + _col(n("buyer_phone"), "TEXT")]}, + n("members"): {"qualified": n("members"), "primary_key": [n("id")], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], "columns": members_cols}, + n("refunds"): {"qualified": n("refunds"), "primary_key": [n("id")], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], "columns": [_col(n("id")), _col(n("order_id"))]}, + } + + +def _rel(to_table: str, columns: list[str], to_columns: list[str], *, status: str, source: str = "name", + table: str = "orders") -> dict: + return {"id": catalog.relation_id(table, columns, to_table, to_columns), "columns": columns, + "to_table": to_table, "to_columns": to_columns, "cardinality": "many_to_one", "coverage": None, + "source": source, "status": status} + + +#: 本机（源）的目录 +SOURCE_ORDERS = { + "label": make_item("订单", "human", "confirmed", at="2026-10-01T08:00:00+00:00"), + "grain": make_item("每笔订单一行", "llm"), + # 确认、驳回过的项带着保存提案时记下的依据：导出、导入照带 + "valid_filter": make_item("status <> 9", "human", "confirmed", reason="使用者说：9 是作废，统计时排除"), + "dedup": make_item("同一订单号取最后一次", "llm", "rejected", reason="使用者说：同一订单号只有一行"), + "kind": make_item("fact", "llm", "confirmed"), + "columns": { + "status": {"codes": make_item({"1": "已支付", "9": "作废"}, "human", "confirmed", complete=True), + "label": make_item("状态", "llm")}, + "amount": {"unit": make_item("元", "human", "confirmed"), "measure": make_item("flow", "llm")}, + "buyer_phone": {"codes": make_item({"13900000001": "测试号"}, "human", "confirmed"), + "label": make_item("联系电话", "human", "confirmed")}, + # 目标表结构里没有这一列 + "gift_flag": {"label": make_item("赠品标记", "human", "confirmed")}, + }, + "relations": [ + _rel("members", ["member_id"], ["id"], status="confirmed"), + # 目标那边没有这张表 + _rel("stores", ["member_id"], ["id"], status="rejected"), + ], +} +SOURCE_MEMBERS = {"label": make_item("会员", "llm"), "keys": make_item(["tier"], "human", "confirmed"), + "columns": {"nick": {"label": make_item("昵称", "llm", "verified")}}} +#: 目标那边没有的表 +SOURCE_GHOST = {"label": make_item("旧表", "human", "confirmed")} + +#: 目标（部署实例）上已有的目录 +TARGET_ORDERS = { + "label": make_item("订单表", "human", "confirmed"), # 和文件不一样、都确认过：冲突，保留目标 + "grain": make_item("一行一单", "llm"), # 文件里是推断：保留目标 + "valid_filter": make_item("STATUS <> 9", "llm"), # 目标是推断、文件确认过：更新 + "kind": make_item("fact", "human", "confirmed"), # 一样：保留 + "columns": {"STATUS": {"label": make_item("状态码", "comment")}}, +} + +_CREATED: list[str] = [] + + +@pytest.fixture(autouse=True) +async def _cleanup(): + yield + while _CREATED: + await drop_source(_CREATED.pop()) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c: + yield c + + +async def _write(sid: str, table: str, notes: dict) -> None: + async with SessionLocal() as session: + entry = await catalog.read_entry(session, sid, table) + await catalog.write_entry(session, sid, table, notes, if_version=entry.version if entry else 0, actor=None) + + +async def _source(name: str, tables: dict, entries: dict[str, dict], **kw) -> str: + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=":memory:", schema_cache={"tables": tables}, **kw) + session.add(row) + await session.commit() + sid = row.id + _CREATED.append(sid) + for table, notes in entries.items(): + await _write(sid, table, notes) + return sid + + +async def _pair() -> tuple[str, str]: + local = await _source(f"本机库_{uuid.uuid4().hex[:6]}", _tables(), + {"orders": SOURCE_ORDERS, "members": SOURCE_MEMBERS, "ghost_table": SOURCE_GHOST}, + options={CATALOG_DRAFT_GUIDANCE_OPTION: "金额一律按元"}) + remote = await _source(f"deploy_{uuid.uuid4().hex[:6]}", _tables(upper=True), {"ORDERS": TARGET_ORDERS}, + options={"mask_columns": "buyer_phone"}) + return local, remote + + +def _base(sid: str) -> str: + return f"/api/datasources/{sid}/catalog" + + +async def _export(client, sid: str) -> dict: + r = await client.get(f"{_base(sid)}/export") + assert r.status_code == 200, r.text + return r.json() + + +# -------------------------------------------------------------------------- +# 导出 +# -------------------------------------------------------------------------- + + +async def test_export_is_a_json_attachment_with_every_item(client): + local, _ = await _pair() + r = await client.get(f"{_base(local)}/export") + assert r.status_code == 200, r.text + assert r.headers["content-type"].startswith("application/json") + disposition = r.headers["content-disposition"] + assert disposition.startswith("attachment;") and "filename=\"agentlab-catalog-" in disposition + async with SessionLocal() as session: + name = (await session.get(DataSource, local)).name + assert unquote(disposition.split("filename*=UTF-8''", 1)[1]) == f"{name}-数据目录.json" + body = r.json() + assert body["format"] == "agentlab-catalog" and body["version"] == 1 + assert body["exported_at"] and body["source"] == {"name": name, "kind": "sqlite"} + assert body["guidance"] == "金额一律按元" + assert set(body["tables"]) == {"orders", "members", "ghost_table"} + assert body["tables"]["orders"]["version"] == 1 + # 含全部状态的项：驳回的也带 + assert body["tables"]["orders"]["notes"]["dedup"]["status"] == "rejected" + assert body["tables"]["orders"]["notes"] == SOURCE_ORDERS + assert body["tables"]["ghost_table"] == {"notes": SOURCE_GHOST, "version": 1} + + +def test_attachment_name_drops_characters_files_cannot_have(): + assert catalog_transfer.attachment_name('景区/客流:库?') == "景区_客流_库_-数据目录.json" + assert catalog_transfer.attachment_name(" ") == "数据源-数据目录.json" + + +# -------------------------------------------------------------------------- +# 预览与导入 +# -------------------------------------------------------------------------- + + +def _by(report: dict) -> dict[str, dict]: + return {t["matched_as"]: t for t in report["tables"]} + + +async def test_preview_reports_and_writes_nothing(client): + local, remote = await _pair() + content = await _export(client, local) + r = await client.post(f"{_base(remote)}/import/preview", json={"content": content, "include_guidance": True}) + assert r.status_code == 200, r.text + report = r.json() + assert report["unmatched_tables"] == ["ghost_table"] + tables = _by(report) + assert set(tables) == {"ORDERS", "MEMBERS"} + orders = tables["ORDERS"] + assert orders["table_name"] == "orders" + conflicts = {c["path"]: c for c in orders["conflicts"]} + assert set(conflicts) == {"label"} + label = conflicts["label"] + assert label["label"] == "表的中文名" and label["file_value"] == "订单" and label["target_value"] == "订单表" + assert "保留" in label["why"] + skipped = {s["path"]: s["why"] for s in orders["skipped"]} + assert "columns.gift_flag.label" in skipped and "表结构里没有列 gift_flag" in skipped["columns.gift_flag.label"] + assert "遮罩" in skipped["columns.buyer_phone.codes"] + stores = next(p for p in skipped if p.startswith("relations.")) + assert "stores" in skipped[stores] + # 新增：dedup（驳回的照带）、status 码值、amount 两项、联系电话的中文名、关联关系；更新：有效记录条件 + # 保留：粒度（文件里是推断，目标已有）、表类型（一样）、STATUS 的中文名（文件里是推断，目标已有） + assert (orders["add"], orders["update"], orders["keep"]) == (6, 1, 3) + members = tables["MEMBERS"] + # 业务主键指着的列 tier 目标那边没有：跳过；会员的中文名和昵称填空缺 + assert (members["add"], members["update"], members["keep"]) == (2, 0, 0) + assert [s["path"] for s in members["skipped"]] == ["keys"] + assert report["totals"] == {"tables": 2, "add": 8, "update": 1, "keep": 3, "conflicts": 1, + "skipped": len(orders["skipped"]) + 1} + assert report["guidance"] == {"file": "金额一律按元", "target": None, "action": "replace"} + async with SessionLocal() as session: + assert (await catalog.read_entry(session, remote, "ORDERS")).version == 1 + assert await catalog.read_entry(session, remote, "MEMBERS") is None + assert catalog_jobs.guidance_of(await session.get(DataSource, remote)) is None + + +async def test_import_writes_by_the_rules_and_reports_versions(client): + local, remote = await _pair() + content = await _export(client, local) + r = await client.post(f"{_base(remote)}/import", json={"content": content, "include_guidance": True}, + headers=ACTOR) + assert r.status_code == 200, r.text + report = r.json() + tables = _by(report) + assert tables["ORDERS"]["version"] == 2 and tables["MEMBERS"]["version"] == 1 + assert tables["ORDERS"]["error"] is None + assert report["guidance"]["action"] == "replace" + async with SessionLocal() as session: + orders = await catalog.read_entry(session, remote, "ORDERS") + members = await catalog.read_entry(session, remote, "MEMBERS") + row = await session.get(DataSource, remote) + notes = orders.notes + assert orders.updated_by == "沈岚" + assert notes["label"]["value"] == "订单表" # 冲突：保留目标的 + assert notes["grain"]["value"] == "一行一单" # 推断只填空缺 + assert notes["valid_filter"] == {**SOURCE_ORDERS["valid_filter"]} # 照带：来源、状态、依据不变 + assert notes["dedup"]["status"] == "rejected" and notes["dedup"]["source"] == "llm" + assert notes["dedup"]["reason"] == "使用者说：同一订单号只有一行" + codes = notes["columns"]["STATUS"]["codes"] + assert codes["value"] == {"1": "已支付", "9": "作废"} and codes["complete"] is True + assert notes["columns"]["STATUS"]["label"]["value"] == "状态码" + assert notes["columns"]["AMOUNT"]["measure"] == SOURCE_ORDERS["columns"]["amount"]["measure"] + # 遮罩列：中文名照带，码值不导入 + assert notes["columns"]["BUYER_PHONE"] == {"label": SOURCE_ORDERS["columns"]["buyer_phone"]["label"]} + assert "GIFT_FLAG" not in notes["columns"] and "gift_flag" not in notes["columns"] + [rel] = notes["relations"] + assert rel["id"] == catalog.relation_id("ORDERS", ["MEMBER_ID"], "MEMBERS", ["ID"]) + assert (rel["columns"], rel["to_table"], rel["to_columns"]) == (["MEMBER_ID"], "MEMBERS", ["ID"]) + assert rel["status"] == "confirmed" and rel["source"] == "name" + assert catalog.validate_notes(notes) == [] + assert members.notes["label"]["status"] == "proposed" and "keys" not in members.notes + assert members.notes["columns"]["NICK"]["label"]["status"] == "verified" + assert catalog_jobs.guidance_of(row) == "金额一律按元" + # 再导一次：都对上了，什么都不写；目标已经有起草要求，不覆盖 + again = (await client.post(f"{_base(remote)}/import", json={"content": content, "include_guidance": True})).json() + assert again["totals"]["add"] == again["totals"]["update"] == 0 + assert _by(again)["ORDERS"]["version"] == 2 + assert again["guidance"] == {"file": "金额一律按元", "target": "金额一律按元", "action": "keep"} + + +async def test_target_rejection_beats_file_confirmation_and_guidance_is_optional(client): + local, remote = await _pair() + await _write(remote, "MEMBERS", {"label": make_item("会员", "llm", "rejected")}) + async with SessionLocal() as session: + row = await session.get(DataSource, remote) + await catalog_jobs.save_guidance(session, row, "部署实例自己的要求") + content = await _export(client, local) + content["tables"]["members"]["notes"]["label"] = make_item("会员", "human", "confirmed") + report = (await client.post(f"{_base(remote)}/import", + json={"content": content, "include_guidance": False})).json() + members = _by(report)["MEMBERS"] + [conflict] = members["conflicts"] + assert conflict["path"] == "label" and conflict["target_status"] == "rejected" + assert conflict["file_status"] == "confirmed" and "驳回" in conflict["why"] + assert report["guidance"] == {"file": "金额一律按元", "target": "部署实例自己的要求", "action": "none"} + async with SessionLocal() as session: + entry = await catalog.read_entry(session, remote, "MEMBERS") + assert entry.notes["label"]["status"] == "rejected" + assert catalog_jobs.guidance_of(await session.get(DataSource, remote)) == "部署实例自己的要求" + + +async def test_one_table_failing_does_not_stop_the_others(client, monkeypatch): + local, remote = await _pair() + content = await _export(client, local) + real = catalog.write_entry + + async def flaky(session, source_id, table, notes, **kw): + if table == "ORDERS": + raise catalog.CatalogConflict(table, 99) + return await real(session, source_id, table, notes, **kw) + + monkeypatch.setattr(catalog, "write_entry", flaky) + report = (await client.post(f"{_base(remote)}/import", json={"content": content})).json() + tables = _by(report) + assert tables["ORDERS"]["error"] and "稍后" in tables["ORDERS"]["error"] + assert tables["ORDERS"]["add"] == tables["ORDERS"]["update"] == 0 + assert tables["MEMBERS"]["error"] is None and tables["MEMBERS"]["version"] == 1 + + +async def test_import_waits_for_draft_jobs_and_rejects_bad_files(client): + local, remote = await _pair() + content = await _export(client, local) + async with SessionLocal() as session: + session.add(CatalogDraftJob(id=uuid.uuid4().hex, source_id=remote, status="running")) + await session.commit() + busy = await client.post(f"{_base(remote)}/import", json={"content": content}) + assert busy.status_code == 409 and busy.json()["code"] == "catalog_draft_busy" + # 预览是只读的，照常给 + assert (await client.post(f"{_base(remote)}/import/preview", json={"content": content})).status_code == 200 + bad = [ + "不是对象", + {**content, "format": "something-else"}, + {**content, "version": 2}, + {**content, "tables": []}, + {**content, "tables": {"orders": {"notes": "x"}}}, + {**content, "guidance": 3}, + ] + for body in bad: + r = await client.post(f"{_base(local)}/import/preview", json={"content": body}) + assert r.status_code == 422, body + assert any("一" <= ch <= "鿿" for ch in r.json()["detail"]) + # 界面直接交来文件原文也认；不是 JSON 的 422 + as_text = await client.post(f"{_base(local)}/import/preview", json={"content": json.dumps(content)}) + assert as_text.status_code == 200 and as_text.json()["totals"]["tables"] == 2 + not_json = await client.post(f"{_base(local)}/import/preview", json={"content": "{不是 JSON"}) + assert not_json.status_code == 422 and "JSON" in not_json.json()["detail"] + too_long = {**content, "guidance": "长" * (catalog_jobs.GUIDANCE_MAX + 1)} + r = await client.post(f"{_base(local)}/import/preview", json={"content": too_long, "include_guidance": True}) + assert r.status_code == 422 and "起草要求" in r.json()["detail"] + + +async def test_invalid_items_are_skipped_not_fatal(client): + local, remote = await _pair() + content = await _export(client, local) + notes = content["tables"]["orders"]["notes"] + notes["kind"] = {"value": "event", "source": "human", "status": "confirmed"} # 取值不在可选值内 + notes["formula"] = {"value": "sum(amount)", "source": "human", "status": "confirmed"} # 不认识的字段 + notes["columns"]["amount"]["colour"] = {"value": "红", "source": "human", "status": "confirmed"} + report = (await client.post(f"{_base(remote)}/import/preview", json={"content": content})).json() + skipped = {s["path"]: s["why"] for s in _by(report)["ORDERS"]["skipped"]} + assert "明细表" in skipped["kind"] + assert "formula" in skipped["formula"] and "columns.amount.colour" in skipped + + +async def test_valid_filter_columns_must_be_in_the_target_table(client): + """有效记录条件是表级的一项，按列跳过的规矩拦不住它：解析出用到的列，不在目标的表结构里、是别的表的列、带子查询的都跳过 + 并写原因。人工写的条件不限函数；带本表表名（文件里的写法、目标的写法都认）、大小写不同的照带。""" + local, remote = await _pair() + content = await _export(client, local) + + async def outcome(condition: str) -> tuple[str | None, int]: + body = copy.deepcopy(content) + body["tables"]["orders"]["notes"]["valid_filter"] = make_item(condition, "human", "confirmed") + report = (await client.post(f"{_base(remote)}/import/preview", json={"content": body})).json() + orders = _by(report)["ORDERS"] + why = {s["path"]: s["why"] for s in orders["skipped"]}.get("valid_filter") + return why, orders["update"] + + why, update = await outcome("gift_flag = 0 AND status <> 9") + assert why and "gift_flag" in why and "目标的表结构" in why and update == 0 + why, _ = await outcome("members.tier = 1") + assert why and "members.tier" in why and "别的表" in why + why, _ = await outcome("member_id IN (SELECT id FROM members)") + assert why and "子查询" in why + why, _ = await outcome("status <> 9; DELETE FROM orders") + assert why and "条件" in why + for fine in ("orders.status <> 9", "ORDERS.STATUS <> 9 AND ABS(amount) > 0", "buyer_phone IS NOT NULL"): + why, update = await outcome(fine) + assert why is None and update == 1, fine + + +async def test_import_needs_a_schema(client): + local, _ = await _pair() + content = await _export(client, local) + empty = await _source(f"empty_{uuid.uuid4().hex[:6]}", {}, {}) + r = await client.post(f"{_base(empty)}/import/preview", json={"content": content}) + assert r.status_code == 409 and r.json()["code"] == "schema_missing" + + +async def test_round_trip_into_an_empty_copy_reproduces_the_catalog(client): + local, _ = await _pair() + content = await _export(client, local) + copy_sid = await _source(f"copy_{uuid.uuid4().hex[:6]}", _tables(), {}) + report = (await client.post(f"{_base(copy_sid)}/import", json={"content": json.loads(json.dumps(content)), + "include_guidance": True})).json() + assert report["unmatched_tables"] == ["ghost_table"] + assert all(not t["conflicts"] for t in report["tables"]) + async with SessionLocal() as session: + mine = await catalog.read_entry(session, local, "orders") + theirs = await catalog.read_entry(session, copy_sid, "orders") + # 关系两端都对得上的照带；指向目标那边没有的表的跳过 + expected = copy.deepcopy(mine.notes) + expected["relations"] = [r for r in mine.notes["relations"] if r["to_table"] != "stores"] + del expected["columns"]["gift_flag"] # 表结构里没有这一列 + assert theirs.notes == expected + + +async def test_a_bad_reason_on_an_inferred_item_is_dropped_not_the_item(client): + """手改过的文件里，推断的项带着不合规的确认时的依据（推断项本来就不该有依据）：去掉依据、照常填空缺，不把整项跳过。 + 已确认的项依据不合规照旧跳过（依据是它的一部分，照带就得合规）。""" + local, remote = await _pair() + content = await _export(client, local) + notes = content["tables"]["orders"]["notes"] + notes["grain"] = {"value": "每笔订单一行", "source": "llm", "status": "proposed", "reason": "依" * 600} + notes["description"] = {"value": "订单头", "source": "human", "status": "confirmed", "reason": 42} + report = (await client.post(f"{_base(remote)}/import/preview", json={"content": content})).json() + skipped = {s["path"]: s["why"] for s in _by(report)["ORDERS"]["skipped"]} + assert "grain" not in skipped + assert "description" in skipped and "依据" in skipped["description"] + r = await client.post(f"{_base(remote)}/import", json={"content": content}) + assert r.status_code == 200, r.text + assert "grain" not in {s["path"] for s in _by(r.json())["ORDERS"]["skipped"]} + got = (await client.get(f"{_base(remote)}/ORDERS")).json()["notes"] + assert "reason" not in got["grain"] # 目标原有的推断粒度保留（推断只填空缺），也没被挂上依据 diff --git a/backend/tests/test_dual_check_facts.py b/backend/tests/test_dual_check_facts.py index 0e27e978..66fdf79b 100644 --- a/backend/tests/test_dual_check_facts.py +++ b/backend/tests/test_dual_check_facts.py @@ -631,6 +631,20 @@ async def test_a_rejected_item_is_not_brought_back_but_a_rejected_other_value_do assert (await entry_of(source_id)).notes["business_date"]["value"] == {"column": "order_date"} +async def test_writing_over_an_item_drops_the_reason_it_was_confirmed_with(shop): + """双盲复算的选择改写这一项（值变了）：以前确认时记下的依据说的是旧值，不留；出处照旧写在备注里。""" + name, source_id = shop + await seed(source_id, {"business_date": catalog.make_item({"column": "paid_date"}, "llm", "rejected", + reason="使用者说：按支付日算")}) + out = await record_choices([candidate(BY_ORDER_DATE, RIGHT, "a", source=name)], run_id="run-7", + node_title="复核", actor=None) + assert out["written"] == [{"source": name, "table": "orders", "item": "business_date", + "value": {"column": "order_date"}}] + item = (await entry_of(source_id)).notes["business_date"] + assert item["value"] == {"column": "order_date"} and (item["source"], item["status"]) == ("human", "confirmed") + assert "reason" not in item and "run-7" in item["note"] + + async def test_contradictory_choices_in_one_decision_write_nothing(shop): """同一次裁决里，一个指标采纳了按下单日、另一个采纳了按支付日：一张表只有一个业务日期，写哪个都是替人拿主意， 都不写。""" diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index eb0f3abf..6827f719 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,6 +1,8 @@ import type { - ActivateSnapshotBody, ActivateSnapshotOut, AiPreview, CatalogDetail, CatalogDraftJob, CatalogImpact, CatalogList, CatalogNotes, - CatalogPatchPreview, CatalogPatchSubmit, CatalogProfileOut, CatalogReviewAction, CommitOut, CurrentRecipe, EditPreview, + ActivateSnapshotBody, ActivateSnapshotOut, AiPreview, CatalogAssistantEvent, CatalogAssistantRequest, CatalogDetail, CatalogDraftJob, + CatalogImpact, CatalogImportReport, CatalogList, CatalogNotes, CatalogPatchPreview, CatalogPatchSubmit, CatalogProfileOut, + CatalogReviewAction, CatalogReviewBatchItem, CatalogReviewBatchResult, CatalogReviewKind, CatalogReviewProgress, CatalogReviewQueue, + CatalogTableUsage, CommitOut, CurrentRecipe, EditPreview, EditRequest, ImportRecord, ManifestOut, PurgeRawBody, PurgeRawOut, QuestionAnswer, Recipe, RedraftRulesOut, RemovePeriodBody, RemovePeriodOut, RevokeAcceptanceBody, RevokeAcceptanceOut, SnapshotOut, Staging, @@ -15,7 +17,7 @@ import { ApiError, BASE, NETWORK_MESSAGE, actorHeader, codeOf, del, describeDetail, failure, get, isGatewayFailure, parseBody, patch, post, put, qs, report, request, upload, type RequestOptions, type UploadOptions, } from './http' -import { bundle } from './bundle' +import { bundle, dispositionName } from './bundle' import { compare } from './compare' import { inquiry } from './inquiry' import { schedules } from './schedules' @@ -451,6 +453,51 @@ export const api = { profile: (sourceId: string, body: { tables?: string[] }, opts?: RequestOptions) => request(`/datasources/${encodeURIComponent(sourceId)}/catalog/profile`, { method: 'POST', body: JSON.stringify(body), ...opts }), + // ---- 审阅队列（只列推断项，按表的使用次数、这一项对数字的影响和来源排好） + /** 一页待确认项。kind：全部 / 影响数字的项 / 名称和含义；table 只看这一张表 */ + reviewQueue: (sourceId: string, params: { limit?: number; offset?: number; kind?: CatalogReviewKind; table?: string | null }, + opts?: RequestOptions) => + get(`${catalogBase(sourceId)}/review-queue${qs({ limit: 50, offset: 0, kind: 'all', ...params })}`, opts), + /** + * 成批确认、驳回、撤销审阅（最多 200 条）：逐表按 if_version 乐观锁，一张表冲突不影响别的表，同一张表的几条一次写入。 + * 逐条回结果：成功的带这张表写入后的版本，冲突、不合法的带原因和机读码 + */ + reviewBatch: (sourceId: string, items: CatalogReviewBatchItem[]) => + post<{ results: CatalogReviewBatchResult[] }>(`${catalogBase(sourceId)}/review/batch`, { items }), + /** 审阅进度：使用次数前 top_n 张表、全部表、影响数字的项 */ + reviewProgress: (sourceId: string, opts?: RequestOptions) => + get(`${catalogBase(sourceId)}/review-progress`, opts), + /** 用过这张表的运行，新的在前 */ + usage: (sourceId: string, table: string, limit = 20, opts?: RequestOptions) => + get(`${catalogBase(sourceId)}/${encodeURIComponent(table)}/usage${qs({ limit })}`, opts), + // ---- 导出导入 + /** + * 导出这个数据源的数据目录（GET /catalog/export，附件）：回来的是文件本身和服务端给的文件名，没给就是 null，由调用方 + * 按数据源名补。原样存下，不解析：导入时交回的就是这份 + */ + exportFile: async (sourceId: string): Promise<{ blob: Blob; name: string | null }> => { + const path = `${catalogBase(sourceId)}/export` + let res: Response + try { + res = await fetch(BASE + path, { headers: actorHeader() }) + } catch (e: any) { + const err = new ApiError(0, NETWORK_MESSAGE, { kind: 'network', raw: `${e?.name ?? 'Error'}: ${e?.message ?? e}` }) + report(false, err) + throw err + } + if (!res.ok) throw failure(res.status, res.statusText, await res.text().catch(() => ''), path) + report(true) + return { blob: await res.blob(), name: dispositionName(res.headers.get('Content-Disposition')) } + }, + /** 导入的预览（只读）：哪些表对上了，各有几项新增、更新、保留、冲突、跳过，冲突逐条列出 */ + importPreview: (sourceId: string, body: { content: unknown; include_guidance: boolean }) => + post(`${catalogBase(sourceId)}/import/preview`, body), + /** + * 导入：规则同预览，逐表乐观锁，一张表失败不影响别的。有排队、进行中的起草任务时 409（catalog_draft_busy）； + * 文件格式不对 422（detail 是一句中文） + */ + importCatalog: (sourceId: string, body: { content: unknown; include_guidance: boolean }) => + post(`${catalogBase(sourceId)}/import`, body), }, runs: { @@ -867,6 +914,83 @@ export function streamCopilot( return () => controller.abort() } +/** 表助手一轮结束时的情况：error 是给人看的那句话（开流前被拒时带状态码和机读码）；aborted 是点了停止 */ +export interface CatalogAssistantEnd { + error?: string + status?: number + code?: string + aborted?: boolean +} + +/** + * 表助手的一轮（POST /datasources/{id}/catalog/{table}/assistant）：回答是事件流，每个事件一行 data: ，到一个回调 + * 一个（text 增量、query 只读查询、proposal 修改提案、done、error）。返回取消函数，就是「停止回答」。 + * + * 开流之前就被拒（表不在 404、起草模型用不了 409 catalog_draft_model_unavailable、请求太长 422）时 onEnd 带服务端那句话、 + * 状态码和机读码；流到一半断了和一开始就连不上一样，说无法连接服务。认不出的行（心跳、半截）跳过 + */ +export function streamCatalogAssistant( + sourceId: string, + table: string, + body: CatalogAssistantRequest, + onEvent: (event: CatalogAssistantEvent) => void, + onEnd: (end: CatalogAssistantEnd) => void, +): () => void { + const controller = new AbortController() + const path = `${catalogBase(sourceId)}/${encodeURIComponent(table)}/assistant` + const emit = (line: string) => { + const text = line.replace(/\r$/, '') + if (!text.startsWith('data:')) return + const payload = text.slice(5).trim() + if (!payload) return + try { + const event = parseBody(payload) + if (event && typeof event === 'object' && typeof event.type === 'string') onEvent(event as CatalogAssistantEvent) + } catch { /* 认不出的一行，跳过 */ } + } + void (async () => { + try { + const res = await fetch(BASE + path, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'text/event-stream', ...actorHeader() }, + body: JSON.stringify(body), + signal: controller.signal, + }) + if (!res.ok || !res.body) { + const err = failure(res.status, res.statusText, await res.text().catch(() => ''), path) + onEnd(err.kind === 'network' ? { error: NETWORK_MESSAGE } : { error: err.message, status: res.status, code: err.code }) + return + } + report(true) + const reader = res.body.getReader() + const decoder = new TextDecoder() + let buffer = '' + for (;;) { + const { done, value } = await reader.read() + if (done) break + buffer += decoder.decode(value, { stream: true }) + let idx: number + while ((idx = buffer.indexOf('\n')) >= 0) { + emit(buffer.slice(0, idx)) + buffer = buffer.slice(idx + 1) + } + } + buffer += decoder.decode() + if (buffer) emit(buffer) + onEnd({}) + } catch (e: any) { + if (e?.name === 'AbortError') { onEnd({ aborted: true }); return } + if (e instanceof TypeError) { + report(false, new ApiError(0, NETWORK_MESSAGE, { kind: 'network', raw: String(e) })) + onEnd({ error: NETWORK_MESSAGE }) + return + } + onEnd({ error: e?.message ?? NETWORK_MESSAGE }) + } + })() + return () => controller.abort() +} + /** * 订阅一次运行的事件流。 * diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 663b6329..2d6d030b 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -2445,14 +2445,24 @@ export const CATALOG_DRAFT_TEXT = { queued: '排队中，等待开始…', running: '正在起草…', // 服务端点了停止就取消进行中的模型调用（不等它答完，一次调用可能要几分钟），只等手上正在写入目录的那张表写完 - stopping: '正在停止：取消进行中的模型调用，正在写入目录的表写完后停止…', + // 宽表每写完一段就写进目录：停下来时已写进的段保留，只停掉进行中的调用 + stopping: '正在停止：取消进行中的模型调用，已写进目录的段保留…', progress: (done: number, total: number) => `已完成 ${formatNumber(done)} / ${formatNumber(total)} 张表`, /** 结束以后：全部写出来的表有几张（停止、失败、部分完成的不算） */ progressDone: (ok: number, total: number) => `写完 ${formatNumber(ok)} / ${formatNumber(total)} 张表`, parts: (done: number, total: number) => `${formatNumber(done)} / ${formatNumber(total)} 段`, - partsHint: '超过 40 列的表拆成表级一段和若干列分段，逐段起草', + partsHint: '超过 40 列的表拆成表级一段和若干列分段，逐段起草，每写完一段就写进目录', + /** 段被缩小、拆半后实际分成的块（比段多时才写） */ + chunks: (done: number, total: number) => `${formatNumber(done)} / ${formatNumber(total)} 块`, + chunksHint: (total: number) => `模型用完额度或超时后，段被缩小、拆半：这张表实际分成 ${formatNumber(total)} 块起草`, + /** 已经写进目录的段（停止、服务重启时保留） */ + partsWritten: (n: number) => `已写进目录 ${formatNumber(n)} 段`, + partsWrittenHint: '每写完一段就写进目录，停止或服务重启时已写进的段保留', + /** 停下来的宽表：服务端的原因里没写时，界面补这一句 */ + partsWrittenStopped: (n: number, rest: number) => (rest > 0 + ? `已写进目录 ${formatNumber(n)} 段，剩下 ${formatNumber(rest)} 段没起草` : `已写进目录 ${formatNumber(n)} 段`), hide: '在后台继续', - hideHint: '关闭对话框不会停止起草，进度显示在目录页上方；每张表起草完就写进目录', + hideHint: '关闭对话框不会停止起草，进度显示在目录页上方；每写完一段就写进目录', stop: '停止', busy: '这个数据源已有进行中的起草任务，下面显示的是它的进度', modelUsed: (model: string) => `模型：${model}`, @@ -2490,8 +2500,13 @@ export const CATALOG_DRAFT_TEXT = { cancelled: '已停止', interrupted: '服务重启，起草中断', } as Record, - interrupted: '服务重启时这次起草还没有完成，没写完的表未起草。已写进目录的内容保留；可以只重试这些表，接着起草', - cancelled: '已写进目录的内容保留，没写完的表未起草', + // 重试不是续跑：服务端把这些表整张重新分段起草（不跳过已写进目录的段），已写进目录的内容不会丢。别写成「接着起草 + // 剩下的段」——宽表重试花的时间和额度和第一次差不多 + interrupted: '服务重启时这次起草还没有完成，没写完的部分未起草。已写进目录的内容保留（每写完一段就写进目录）；可以只重试这些表：会重新起草整张表，已写进目录的内容不会丢', + cancelled: '已写进目录的内容保留（每写完一段就写进目录），没写完的部分未起草', + /** 停下来时有表已经写进了一部分段 */ + cancelledParts: (tables: number, parts: number) => + `其中 ${formatNumber(tables)} 张表已写进目录 ${formatNumber(parts)} 段；只重试这些表时会重新起草整张表，已写进的不会丢`, // 任务整体出错的原因（job.error）照服务端的原话写在标题「起草失败」下面，不再加前缀：服务端的原因有的已经是整句 // （「数据源不存在……」），有的自带「起草任务出错：」开头。没结束的表记失败、原因同任务的，逐表不再重复 /** 没写全的表：失败、部分完成、已停止各几张；跳过的（表结构里没有）另说 */ @@ -2539,6 +2554,8 @@ export const CATALOG_DRAFT_TEXT = { /** 上次起草有留空的列：清单上是轻提示（不是失败标记），表详情顶部同一句 */ lastBlank: (n: number) => `上次起草有 ${formatNumber(n)} 列模型拿不准、留空了`, lastBlankAdvice: '可以在起草要求里补充缩写和术语的含义，再重新起草这张表，已有的内容不会丢', + /** 最近一次起草停止了、没有新写入：留空的列是再之前那次写完的起草记下的 */ + lastBlankEarlier: '最近一次起草已停止，留空的列按再之前那次起草记下的', draftGuidance: '补充起草要求', draftGuidanceHint: '打开起草设置，范围是这张表：先在起草要求里补充缩写和术语的含义，再开始起草', redraft: '重新起草这张表', @@ -2593,6 +2610,8 @@ export const CATALOG_PATCH_TEXT = { after: '改后', reason: '理由', empty: '未填写', + /** 改前是一个被驳回的值（保存后换成人工确认的新值）：同导入预览「目标里这一项已经驳回」 */ + beforeRejected: (value: string) => `已驳回：${value}`, stateConfirm: '值不变，保存即确认', stateSame: '已是这个值', save: '保存到数据目录', @@ -2953,6 +2972,344 @@ export const CATALOG_UI_TEXT = { cardinalityCheckedShort: '已核实', cardinalityCheckedHint: '基数由数据剖析用数据核实（子表一侧确有重复或确实唯一），一对多关联后重复计算的检查据此报错误', emptyTables: (n: number) => `挑表时跳过 ${formatNumber(n)} 张空表`, + // ---- 表详情里有效记录条件、去重规则的说明（模型起草时附的依据、数据剖析核对的排除比例） + fieldNote: '说明', + // ---- 保存修改提案时记下的依据（目录项的 reason）：和上面的说明分开写，表详情、状态弹层同一个叫法 + fieldReason: '确认时的依据', +} + +// =========================================================================== +// 表助手（表详情里「和助手一起完善」）：和助手对话完善一张表的数据目录。 +// +// 术语： +// - 面板叫「表助手」，入口写「和助手一起完善」。助手只改数据目录、不改业务库；只记数据事实，不写公式。 +// - 助手提出的修改叫「修改提案」，每一项写「改前 / 改后 / 依据」，可以逐项「去掉」「加回」。 +// - 「待保存的修改」：跨多轮累积，同一项以最后一次提案为准；「保存 N 项修改」一次写入，改动的项记为人工填写、已确认。 +// - 只读查询只在开了数据剖析的源上有：写「查询了数据」，SQL 和前几行可以展开；遮罩列的值写「已遮罩」（服务端给的）。 +// - 对话不保存在服务端：关闭表助手、换表、离开页面后就没有了。 +// =========================================================================== + +export const CATALOG_ASSISTANT_TEXT = { + open: '和助手一起完善', + openHint: '和表助手对话完善这张表的数据目录：说出业务事实，助手整理成修改提案，确认后再保存', + openMissing: '表结构中已没有这张表，不能和助手一起完善', + title: '表助手', + subtitle: (table: string) => `完善「${table}」的数据目录`, + close: '关闭表助手', + closeTitle: '关闭表助手？', + closeConsequences: ['待保存的修改和这段对话都会丢失'], + closeConfirm: '放弃并关闭', + leaveTitle: '表助手有未保存的修改', + leaveConsequences: ['离开后待保存的修改和这段对话都会丢失'], + leaveConfirm: '放弃并离开', + noProfile: '这个数据源没开数据剖析，助手只能根据表结构和目录推断，不能查数据', + profileOn: '这个数据源开了数据剖析：助手可以发少量只读查询核对数据，遮罩列不取值', + notStored: '对话不保存在服务端，关闭表助手或离开页面后就没有了', + greeting: '说出你知道的业务事实（某一列的含义、码值、有效记录条件……），助手整理成修改提案，你确认后才写进数据目录。助手只记数据事实，不写计算公式。', + /** + * 上次起草有留空的列：开场引导先问这几列（assistant.ts 的 blankLeadOf）。例子只举起草时记下的留空列里还没补上的； + * 拿不到起草时记下的列名时，改说目录里还没有中文名和含义的列——那里面还有没写出来的列，不能说成留空的列 + */ + blankLead: (b: { from: 'draft' | 'catalog'; n: number; examples: string[]; remaining: number | null; unnamed: number }) => { + const head = `上次起草有 ${formatNumber(b.n)} 列模型拿不准、留空了` + const cols = b.examples.join('、') + if (b.from === 'catalog') { + return b.unnamed > 0 + ? `${head}。目录里还有 ${formatNumber(b.unnamed)} 列没有中文名和含义${cols ? `，例如 ${cols}` : ''}，可以先从这些列开始。` + : `${head}，现在目录里的列都已有中文名或含义。` + } + if (b.remaining === 0) return `${head}，现在都已补上中文名或含义。` + if (!cols) return `${head}。` + if (b.remaining == null) return `${head}，例如 ${cols}。可以先从这几列开始。` + return b.remaining < b.n + ? `${head}，还有 ${formatNumber(b.remaining)} 列没补上：${cols}。可以先从这几列开始。` + : `${head}：${cols}。可以先从这几列开始。` + }, + suggestLabel: '可以这样开始', + /** 「补上……」按钮：上面的数字就是点名的列数（不知道一共几列时不写数字） */ + suggestBlank: (from: 'draft' | 'catalog', total: number | null) => (from === 'catalog' + ? `补上没有中文名和含义的 ${formatNumber(total ?? 0)} 列` + : total == null ? '补上留空的列' : `补上留空的 ${formatNumber(total)} 列`), + suggestBlankText: (from: 'draft' | 'catalog', cols: string[], total: number | null) => { + const named = cols.join('、') + if (from === 'draft' && total == null) return `请帮我补上上次起草时模型留空的列的中文名和含义，例如 ${named}。拿不准的地方先问我。` + const more = total != null && total > cols.length ? ` 等 ${formatNumber(total)} 列` : '' + return `请帮我补上这几列的中文名和含义：${named}${more}。拿不准的地方先问我。` + }, + suggestFacts: '核对有效记录条件和去重规则', + suggestFactsText: '这张表的有效记录条件和去重规则应该怎么写？拿不准的地方先问我。', + suggestPending: '看看哪些推断可以确认', + suggestPendingText: '这张表还没确认的推断项里，哪些可以直接确认、哪些需要我补充？', + you: '你', + assistant: '助手', + logLabel: '和表助手的对话', + inputLabel: '给表助手的消息', + inputPlaceholder: '例如：status 为 9 表示作废，统计时要排除', + inputHint: 'Enter 发送，Shift+Enter 换行', + count: (n: number, max: number) => `${formatNumber(n)} / ${formatNumber(max)} 字`, + tooLong: (max: number) => `一条消息不超过 ${formatNumber(max)} 字`, + send: '发送', + stop: '停止回答', + answering: '助手正在回答…', + answered: '助手已回答', + stopped: '已停止回答', + interrupted: '连接中断，这一轮没有答完', + failed: (reason: string) => `这一轮没有完成：${reason}`, + usage: (input: number, output: number) => + `输入 ${formatTokens(input, { compact: true })}、输出 ${formatTokens(output, { compact: true })}`, + queries: (n: number) => `查询 ${formatNumber(n)} 次`, + // ---- 只读查询（rows 是取回的行数；truncated：后面还有行，取回的只是前一截，不是结果的总行数） + query: (rows: number, truncated = false) => (truncated + ? `查询了数据（取回前 ${formatNumber(rows)} 行，后面还有）` : `查询了数据（${formatNumber(rows)} 行）`), + queryFailed: '查询没有成功', + queryRows: (shown: number, total: number, truncated = false) => { + const head = truncated ? `取回前 ${formatNumber(total)} 行（后面还有）` : `共 ${formatNumber(total)} 行` + return total > shown ? `${head}，显示前 ${formatNumber(shown)} 行` : head + }, + queryEmpty: '没有返回行', + // ---- 修改提案 + proposal: (n: number) => `修改提案（${formatNumber(n)} 项）`, + before: '改前', + after: '改后', + reason: '依据', + drop: '去掉', + dropLabel: (where: string) => `去掉：${where}`, + restore: '加回', + restoreLabel: (where: string) => `加回：${where}`, + state: { + pending: '待保存', + replaced: '已改用另一条提案的值', + dropped: '已去掉', + saved: '已保存', + } as Record<'pending' | 'replaced' | 'dropped' | 'saved', string>, + stateConfirm: '值不变，保存即确认', + // ---- 待保存的修改 + pending: (n: number) => `待保存的修改（${formatNumber(n)} 项）`, + pendingEmpty: '还没有待保存的修改', + pendingHint: '跨多轮累积，同一项以最后一次提案为准。保存后改动的项记为人工填写、已确认', + pendingShow: '展开待保存的修改', + pendingHide: '收起待保存的修改', + save: (n: number) => `保存 ${formatNumber(n)} 项修改`, + saving: '正在保存…', + clear: '全部去掉', + clearTitle: (n: number) => `去掉全部 ${formatNumber(n)} 项待保存的修改？`, + clearConfirm: '全部去掉', + tooMany: (n: number, max: number) => `有 ${formatNumber(n)} 项，一次最多保存 ${formatNumber(max)} 项，请先去掉一些`, + editing: '正在编辑这张表，保存或取消后再保存助手的修改', + saved: (n: number, version: number) => `已保存 ${formatNumber(n)} 项修改（第 ${formatNumber(version)} 版）`, + savedToast: (n: number, table: string) => `已保存 ${formatNumber(n)} 项修改到「${table}」的数据目录`, + conflictTitle: '这张表刚被修改过', + conflictBody: '已按最新内容重算改前和改后，请核对后再保存。', + conflictFailed: '无法按最新内容重算改前和改后，请稍后重试', + problems: '以下几项已无法保存，已从待保存的修改中去掉：', +} + +// =========================================================================== +// 审阅队列（数据目录页的「审阅队列」视图）：跨表列出推断项，按表的使用次数、这一项对数字的影响和来源排好， +// 可以成批确认、驳回、撤销审阅。 +// +// 术语： +// - 「影响数字的项」：有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型——写错了查出来的数就不对，排在最前； +// 「名称和含义」：中文名、含义、单位、码值、粒度、表类型。 +// - 审阅过的项原地标出结果（已确认、已驳回、未生效），刷新队列后才移除；「撤销审阅」回到起草时的状态（推断）。 +// - 每一项的位置写「列 status 的码值」「表的有效记录条件」「指向 parks 的关联关系」（同提案卡片），不只写字段名。 +// - 「常用 N 张表」：被运行用过、使用次数最多的前几张（最多 20 张），N 写实际张数；没有运行用过的表时写明。 +// - 进度写「已确认 42%」，三行同一个口径：已确认 ÷（已确认 + 待确认）；已驳回的、有确证的（外键约束）不计入。 +// - 「使用 N 次」是所在表被运行查询过的次数（按查询算，同表清单）；「用过这张表的运行」写「共 N 次运行」（按运行算）， +// 两个数不一样，各自写明单位。 +// =========================================================================== + +export const CATALOG_REVIEW_TEXT = { + viewLabel: '数据目录的视图', + view: { list: '表清单', review: '审阅队列' } as Record<'list' | 'review', string>, + viewHint: { + list: '逐表查看和编辑数据目录', + review: '跨表列出待确认的推断项，按使用次数和对数字的影响排好，可以成批确认、驳回', + } as Record<'list' | 'review', string>, + queueLabel: '待确认的推断项', + kindLabel: '按项筛选', + kind: { all: '全部', facts: '影响数字的项', names: '名称和含义' } as Record<'all' | 'facts' | 'names', string>, + kindHint: { + all: '全部待确认的推断项', + facts: '有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型：写错了查出来的数就不对', + names: '中文名、含义、单位、码值、粒度、表类型', + } as Record<'all' | 'facts' | 'names', string>, + tableLabel: '按表筛选', + tableAll: '全部表', + tableOption: (label: string | null, table: string, n: number) => `${label ? `${label}（${table}）` : table} · ${formatNumber(n)} 项`, + total: (n: number) => `${formatNumber(n)} 项待确认`, + shown: (shown: number, total: number) => `显示 ${formatNumber(shown)} / ${formatNumber(total)} 项`, + empty: '没有待确认的推断项', + emptyBody: '推断的项都审阅过了。之后起草出的新推断会出现在这里。', + emptyFiltered: '没有符合条件的待确认项', + emptyFilteredBody: '请换个筛选条件。', + loadFailed: '无法读取审阅队列', + retry: '重试', + refresh: '刷新队列', + refreshHint: '审阅过的项刷新后从队列中移除', + more: (n: number) => `再载入 ${formatNumber(n)} 项`, + loading: '正在载入…', + selectAll: '全选当前列表', + select: (where: string, table: string) => `选择：${table} ${where}`, + selected: (n: number) => `已选 ${formatNumber(n)} 项`, + clearSelection: '取消选择', + usage: (n: number) => `使用 ${formatNumber(n)} 次`, + usageHint: '所在表被运行查询过的次数，同一次运行中结果相同的重复查询只算一次', + open: (where: string, table: string) => `打开「${table}」，查看${where}`, + reasonsLabel: '排在这里的原因', + // ---- 成批审阅 + confirm: (n: number) => `确认（${formatNumber(n)}）`, + reject: (n: number) => `驳回（${formatNumber(n)}）`, + reset: (n: number) => `撤销审阅（${formatNumber(n)}）`, + resetHint: '把这一页上审阅过的项回到起草时的状态（推断）', + confirmTitle: (n: number) => `确认 ${formatNumber(n)} 项推断？`, + confirmConsequences: ['确认后这些项参与 SQL 检查', '来源保持不变，状态改为已确认', '之后仍可撤销审阅'], + confirmAction: (n: number) => `确认 ${formatNumber(n)} 项`, + rejectTitle: (n: number) => `驳回 ${formatNumber(n)} 项推断？`, + rejectConsequences: ['驳回后不提供给助手、不参与 SQL 检查', '之后起草也不会再提出', '之后仍可撤销审阅'], + rejectAction: (n: number) => `驳回 ${formatNumber(n)} 项`, + tooMany: (max: number) => `一次最多审阅 ${formatNumber(max)} 项，请少选一些`, + working: '正在提交…', + done: { confirm: '已确认', reject: '已驳回', reset: '已撤销审阅' } as Record<'confirm' | 'reject' | 'reset', string>, + conflict: '未生效：这张表刚被修改过，请刷新队列后再审阅', + failed: '未生效', + /** 一批的结果：成功几项、冲突几项、别的原因没成功几项 */ + summary: (action: 'confirm' | 'reject' | 'reset', ok: number, conflicts: number, failed: number) => { + const head = `${{ confirm: '已确认', reject: '已驳回', reset: '已撤销审阅' }[action]} ${formatNumber(ok)} 项` + const bad = [ + conflicts ? `${formatNumber(conflicts)} 项未生效（所在表刚被修改过）` : '', + failed ? `${formatNumber(failed)} 项未生效` : '', + ].filter(Boolean) + return bad.length ? `${head}；${bad.join('、')}，请刷新队列后再审阅` : head + }, + failedAll: (reason: string) => `未能提交：${reason}`, + // ---- 进度 + progressTitle: '审阅进度', + /** 常用表：被运行用过、使用次数最多的前几张（最多 topN 张），n 是实际张数 */ + progressTop: (n: number) => `常用 ${formatNumber(n)} 张表`, + progressTopHint: (n: number, topN: number) => (n > 0 + ? `被运行用过、使用次数最多的 ${formatNumber(n)} 张表里的项（最多取 ${formatNumber(topN)} 张）` + : `被运行用过、使用次数最多的表里的项（最多取 ${formatNumber(topN)} 张）`), + progressTopNone: '常用表', + progressTopEmpty: '还没有运行用过的表', + progressAll: '全部表', + progressAllHint: '这个数据源全部表里的项', + progressFacts: '影响数字的项', + progressFactsHint: '有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型', + progressRule: '已确认 ÷（已确认 + 待确认）：已驳回的、有确证的（外键约束）不用再确认，不计入', + /** 一行进度：已确认百分比和待确认几项；不知道待确认几项时只写已确认几项 */ + progressText: (pct: number | null, confirmed: number, proposed: number | null, total: number | null) => { + if (total === 0) return '还没有目录项' + if (proposed == null) return `已确认 ${formatNumber(confirmed)} 项` + if (pct == null) return '没有待确认的项' + return `已确认 ${formatNumber(pct)}% · 待确认 ${formatNumber(proposed)} 项` + }, + progressFailed: '无法读取审阅进度', + // ---- 没选中项时的右栏 + overviewTitle: '从左侧队列挑一项开始审阅', + overviewBody: '队列按所在表的使用次数、这一项对数字的影响和来源排序：常用表里影响数字的推断排在最前，模型起草、命名推断的比数据库注释的靠前。点开一项可以看它所在表的详情和用过这张表的运行。', + // ---- 从队列打开的项 + focusMark: '从审阅队列打开', + // ---- 表清单视图概览里的入口 + openQueue: '打开审阅队列', + openQueueHint: '跨表按使用次数和对数字的影响排好，可以成批确认、驳回', +} + +/** 表详情里「用过这张表的运行」（GET /catalog/{table}/usage） */ +export const CATALOG_USAGE_TEXT = { + title: '用过这张表的运行', + hint: '新的在前', + show: '查看用过这张表的运行', + hide: '收起', + none: '还没有运行查询过这张表', + /** 按运行算（一次运行查询了几次这张表都只算一次），和表头按查询算的「使用 N 次」不是一个数 */ + count: (shown: number, total: number) => (total > shown + ? `共 ${formatNumber(total)} 次运行，显示最近 ${formatNumber(shown)} 次` : `共 ${formatNumber(total)} 次运行`), + countHint: '按运行算：一次运行查询了几次这张表都只算一次；表头的使用次数按查询算，两个数可能不一样', + error: '无法读取用过这张表的运行', + retry: '重试', + unnamed: '未保存的工作流', + open: (name: string) => `打开运行记录：${name}`, +} + +// =========================================================================== +// 导出导入数据目录（数据目录页的「导出目录」「导入目录」）：本机整理好的目录带到另一个实例。 +// +// 术语： +// - 导出的是一个 .json 文件，含全部状态的项和起草要求。导入先「预览」（只读），确认后才写入。 +// - 预览逐表写「新增 / 更新 / 保留 / 冲突 / 跳过」：冲突是这里已确认、已驳回的项和文件里不一样，以这里为准； +// 跳过是表结构里没有的列、两端对不上的关联关系。 +// =========================================================================== + +export const CATALOG_TRANSFER_TEXT = { + export: '导出目录', + exportHint: '把这个数据源的数据目录导出成文件（含全部状态的项和起草要求），可以导入到另一个实例', + exporting: '正在导出…', + exported: (name: string) => `已导出「${name}」`, + exportFailed: (reason: string) => `未能导出数据目录：${reason}`, + fileName: (source: string) => `${source}-数据目录.json`, + import: '导入目录', + importHint: '从导出的文件导入数据目录：先预览，确认后再写入', + title: '导入数据目录', + chooseLabel: '导出的数据目录文件', + chooseHint: '「导出目录」得到的 .json 文件', + choose: '选择文件', + chosen: (name: string) => `已选择：${name}`, + includeGuidance: '同时导入起草要求', + includeGuidanceHint: '这个数据源还没有起草要求时才写入，已有的不覆盖', + rulesTitle: '导入的规矩', + rules: [ + '表按名字对上：不分大小写；带 schema 前缀的对不上时去掉前缀再对', + '文件中已确认、已驳回的项照原样带入，来源和状态不变', + '这里已确认、已驳回的项和文件中的不一样时以这里为准，记为冲突', + '文件中推断、已验证的项只填补这里的空缺，不覆盖已有的', + '表结构里没有的列、两端对不上的关联关系跳过，写明原因', + ], + preview: '预览', + previewing: '正在预览…', + notJson: '文件不是有效的 JSON，请选择「导出目录」得到的文件', + readFailed: '无法读取这个文件', + fileMeta: (source: string, at: string) => `导出自「${source}」 · ${at}`, + totals: (t: { tables: number; add: number; update: number; keep: number; conflicts: number; skipped: number }) => + `${formatNumber(t.tables)} 张表：新增 ${formatNumber(t.add)} 项、更新 ${formatNumber(t.update)} 项、保留 ${formatNumber(t.keep)} 项、` + + `冲突 ${formatNumber(t.conflicts)} 项、跳过 ${formatNumber(t.skipped)} 项`, + head: { table: '表', add: '新增', update: '更新', keep: '保留', conflicts: '冲突', skipped: '跳过' }, + matchedAs: (name: string) => `对上「${name}」`, + unmatched: (n: number) => `${formatNumber(n)} 张表在这个数据源里对不上，不导入：`, + conflictsTitle: (n: number) => `冲突 ${formatNumber(n)} 项：以这里为准，不导入`, + conflictFile: '文件中', + conflictTarget: '这里', + skippedTitle: (n: number) => `跳过 ${formatNumber(n)} 项`, + guidance: { + replace: '起草要求：写入文件中的起草要求（这个数据源还没有）', + keep: '起草要求：保留这个数据源现有的，不覆盖', + none: '起草要求：不导入', + } as Record<'keep' | 'replace' | 'none', string>, + confirm: (n: number) => `导入 ${formatNumber(n)} 张表`, + importing: '正在导入…', + nothing: '没有可以导入的内容：文件中的表在这个数据源里都对不上', + done: '导入完成', + /** + * 导入结果的合计。written 只数真正写进去的表（有新增或更新的项、版本前进了），没有变化的表不算（服务端的 totals.tables + * 是对上的表数）；一张都没写进去时说清没有需要写入的内容。failed 是写入出错、没有导入的表；guidance 为真时起草要求写进去了 + */ + doneTotals: (written: number, t: { add: number; update: number; conflicts: number }, failed: number, guidance: boolean) => { + const head = written + ? `写入 ${formatNumber(written)} 张表：新增 ${formatNumber(t.add)} 项、更新 ${formatNumber(t.update)} 项` + : failed ? '没有写入任何表' : guidance ? '目录里没有需要写入的内容，只写入了起草要求' : '没有需要写入的内容' + const tail = [ + t.conflicts ? `冲突 ${formatNumber(t.conflicts)} 项以这里为准` : '', + failed ? `${formatNumber(failed)} 张表没有导入，原因见下` : '', + ].filter(Boolean) + return [head, ...tail].join('；') + }, + version: (v: number) => `第 ${formatNumber(v)} 版`, + /** 导入结果里没有写进任何内容的表 */ + unchanged: '没有变化', + tableFailed: (reason: string) => `这张表未导入：${reason}`, + back: '重新选择文件', + cancel: '取消', + close: '完成', } // =========================================================================== diff --git a/frontend/src/pages/catalog/CatalogPage.tsx b/frontend/src/pages/catalog/CatalogPage.tsx index 3257ec32..09a1ddc0 100644 --- a/frontend/src/pages/catalog/CatalogPage.tsx +++ b/frontend/src/pages/catalog/CatalogPage.tsx @@ -1,14 +1,19 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' -import type { ReactNode } from 'react' +import type { KeyboardEvent as ReactKeyboardEvent, ReactNode } from 'react' import { useNavigate, useParams, useSearchParams } from 'react-router-dom' -import { ArrowLeft, BookMarked, Database, ListFilter as ListFilterIcon, ScanSearch, Sparkles, TriangleAlert } from 'lucide-react' +import { + ArrowLeft, BookMarked, Database, Download, ListFilter as ListFilterIcon, ScanSearch, Sparkles, TriangleAlert, Upload, +} from 'lucide-react' import clsx from 'clsx' import { ApiError, api } from '../../api/client' -import type { CatalogDetail, CatalogList, CatalogTableRow } from '../../types' -import { EmptyState, ErrorState, PageHeader, Skeleton, Spinner, toast, useTicker } from '../../components/ui' +import type { CatalogDetail, CatalogList, CatalogReviewItem, CatalogReviewKind, CatalogTableRow } from '../../types' +import { EmptyState, ErrorState, PageHeader, Skeleton, Spinner, rovingTarget, toast, useTicker } from '../../components/ui' import { useCatalog, useDatasources, useOnReconnect } from '../../store/catalog' +import { errorMessage } from '../../lib/errors' +import { formatNumber } from '../../lib/format' import { - CATALOG_DRAFT_TEXT as DT, CATALOG_TEXT as CT, PROFILE_STOP_LABEL, PROFILE_TEXT as PT, SCHEMA_PARTIAL_TEXT as SPT, + CATALOG_DRAFT_TEXT as DT, CATALOG_REVIEW_TEXT as RT, CATALOG_TEXT as CT, CATALOG_TRANSFER_TEXT as XT, PROFILE_STOP_LABEL, + PROFILE_TEXT as PT, SCHEMA_PARTIAL_TEXT as SPT, } from '../../lib/terms' import { StatusLegend, SystemNotesNotice } from './parts' import { DraftDialog } from './DraftDialog' @@ -20,6 +25,10 @@ import { ProfileSettingsDialog } from './ProfileSettings' import { fillKey, profileBlockOf, profileSettingsOf, secondsText } from './profile' import { TableDetail } from './TableDetail' import { TableIndex } from './TableIndex' +import { ReviewOverview, ReviewQueue } from './ReviewQueue' +import { ImportDialog } from './TransferDialog' +import { itemKey, reviewParamsOf, writeReviewParams } from './review' +import type { CatalogView } from './review' import { filterCounts, listParamsOf, progressOf, rowFromDetail, usageSummary, visibleRows, writeListParams } from './model' import type { ListFilter, ListParams, ListSort } from './model' @@ -34,6 +43,10 @@ import type { ListFilter, ListParams, ListSort } from './model' // // 清单的搜索词、筛选、排序和「只看运行中查询过的表」记在地址的查询参数里（q / filter / sort / used），刷新、前进后退 // 都不丢；在清单和表详情之间跳转时带着它们走。 +// +// 左栏有两个视图：表清单（逐表看）和审阅队列（view=review：跨表按优先级列出推断项，成批确认、驳回，筛选记在 kind / +// rtable 里）。从队列点开一项，右栏是所在表的详情，地址带着那一项（item），详情滚到那一项、列出用过这张表的运行。 +// 页头的「导出目录」「导入目录」把整理好的目录带到另一个实例（导入先预览，确认后才写）。 // =========================================================================== export function CatalogPage() { @@ -98,10 +111,20 @@ export function CatalogPage() { const setFilter = useCallback((f: ListFilter) => setList((c) => ({ ...c, filter: f })), []) const setSort = useCallback((v: ListSort) => setList((c) => ({ ...c, sort: v })), []) const setUsedOnly = useCallback((v: boolean) => setList((c) => ({ ...c, used: v })), []) - /** 跳到别的表、回到清单时带上清单的查询参数（按页面上的算：搜索词可能还没写进地址） */ + /** + * 跳到别的表、回到清单时带上清单的查询参数（按页面上的算：搜索词可能还没写进地址）。审阅视图的参数（view、kind、rtable） + * 照旧带着，从审阅队列打开的那一项（item）不带：换了表就不是那一项了 + */ const searchRef = useRef('') - const search = writeListParams(params, list).toString() + const carried = writeListParams(params, list) + carried.delete('item') + const search = carried.toString() searchRef.current = search ? `?${search}` : '' + const carriedRef = useRef(carried) + carriedRef.current = carried + /** 审阅视图：view=review 时左栏是审阅队列；kind、rtable 是队列的筛选；item 是右栏从队列打开的那一项 */ + const review = useMemo(() => reviewParamsOf(params), [params]) + const inReview = review.view === 'review' const [selected, setSelected] = useState>(() => new Set()) /** * 起草对话框开着没有、打开时显示什么：setup 选范围开始；job 显示页面跟着的任务（进行中的进度、刚结束的结果）。 @@ -142,12 +165,15 @@ export function CatalogPage() { const backToList = useCallback(() => navigate(`${base}${searchRef.current}`), [navigate, base]) const back = source?.origin === 'upload' ? '/data/tables' : '/data/databases' + /** 右栏的表详情载入、改过之后的目录：审阅队列里这张表的项跟着换状态 */ + const [detailSync, setDetailSync] = useState<{ seq: number; detail: CatalogDetail } | null>(null) /** 写入、审阅之后按返回的单表目录更新清单里那一行 */ const onDetail = useCallback((d: CatalogDetail) => { setData((cur) => cur && ({ ...cur, tables: cur.tables.map((r) => (r.table_name === d.table_name ? rowFromDetail(d, r) : r)), })) + setDetailSync((cur) => ({ seq: (cur?.seq ?? 0) + 1, detail: d })) }, []) /** @@ -306,6 +332,52 @@ export function CatalogPage() { return [list[i - 1]?.table_name ?? null, list[i + 1]?.table_name ?? null] }, [table, visible, rows, sort]) + /** + * 改地址的查询参数，按地址栏现在的样子改：setSearchParams 的函数式写法拿的是上一次渲染时的参数，连着改两样（先点 + * 「全部」、紧接着换表）时后一次会把前一次盖掉 + */ + const patchSearch = useCallback((fn: (p: URLSearchParams) => URLSearchParams, opts?: { replace?: boolean }) => { + const next = fn(new URLSearchParams(window.location.search)).toString() + navigate({ search: next ? `?${next}` : '' }, opts) + }, [navigate]) + /** 换视图（表清单 / 审阅队列）：记进地址（后退回到上一个视图），右栏正在看的表不动 */ + const setView = useCallback((v: CatalogView) => { + patchSearch((prev) => writeReviewParams(prev, { view: v, item: '' })) + }, [patchSearch]) + /** 审阅队列的筛选：替换当前这条历史（筛选不该占满后退键） */ + const setReviewParams = useCallback((p: { kind?: CatalogReviewKind; table?: string }) => { + patchSearch((prev) => writeReviewParams(prev, p), { replace: true }) + }, [patchSearch]) + /** 从审阅队列打开一项：右栏是所在表的详情，地址带着这一项 */ + const openReviewItem = useCallback((i: CatalogReviewItem) => { + const q = writeReviewParams(carriedRef.current, { view: 'review', item: i.path }).toString() + navigate(`${base}/${encodeURIComponent(i.table_name)}${q ? `?${q}` : ''}`) + }, [navigate, base]) + /** 成批审阅、导入写进了这些表：重取清单，正在看的表重新载入（编辑中的不动，保存时服务端会用版本号拦下） */ + const onTablesWritten = useCallback((tables: string[]) => { + setDrafted((cur) => ({ seq: cur.seq + 1, tables: new Set(tables) })) + void load() + }, [load]) + const activeReviewKey = inReview && table && review.item ? itemKey({ table_name: table, path: review.item }) : null + const proposedTotal = useMemo(() => rows.reduce((n, r) => n + (r.in_schema ? r.counts.proposed : 0), 0), [rows]) + + /** 导出目录：下载服务端给的文件，文件名按服务端的，没给就按数据源名补 */ + const [exporting, setExporting] = useState(false) + const [importing, setImporting] = useState(false) + const exportCatalog = useCallback(async () => { + setExporting(true) + try { + const { blob, name } = await api.dataCatalog.exportFile(sourceId) + const file = name ?? XT.fileName(source?.name ?? sourceId) + saveFile(file, blob) + toast.ok(XT.exported(file)) + } catch (e) { + toast.error(XT.exportFailed(errorMessage(e))) + } finally { + setExporting(false) + } + }, [sourceId, source?.name]) + const usage = useMemo(() => usageSummary(rows), [rows]) const wideHeader = useMedia('(min-width: 640px)') /** @@ -375,42 +447,58 @@ export function CatalogPage() { body = (
{table ? ( + draftBusy={!!stripJob} onRedraft={redraftTable} onDraftGuidance={draftWithGuidance} + profileEnabled={profileSettings.enabled} focusPath={inReview ? review.item || null : null} usageOpen={inReview} /> ) - : } + : inReview + ? + : setView('review')} />}
) @@ -427,6 +515,18 @@ export function CatalogPage() {
{hasTables && headerProfile} {hasTables && headerDraft} + {hasTables && ( + + )} + {hasTables && ( + + )} @@ -481,6 +581,9 @@ export function CatalogPage() { onReset={() => setJob(null)} /> )} + {importing && ( + setImporting(false)} onImported={onTablesWritten} /> + )} {profileSettingsOpen && source && ( setProfileSettingsOpen(false)} onSaved={onProfileSettingsSaved} /> @@ -524,6 +627,51 @@ function DraftStrip({ job, notice, onOpen }: { ) } +/** 存成文件（同运行记录里的下载）：浏览器下载，文件名照给的 */ +function saveFile(name: string, blob: Blob) { + const url = URL.createObjectURL(blob) + const a = document.createElement('a') + a.href = url + a.download = name + a.click() + setTimeout(() => URL.revokeObjectURL(url), 1000) +} + +const VIEWS: CatalogView[] = ['list', 'review'] + +/** + * 左栏上方的两个视图：表清单（逐表看）、审阅队列（跨表按优先级看推断项，带着还有几项待确认）。标签页语义， + * ←→ Home End 切换 + */ +function ViewTabs({ view, proposed, onChange }: { view: CatalogView; proposed: number; onChange: (v: CatalogView) => void }) { + const refs = useRef>({}) + const move = (e: ReactKeyboardEvent, i: number) => { + const j = rovingTarget(e, i, VIEWS.length, 'horizontal') + if (j < 0) return + e.preventDefault() + onChange(VIEWS[j]) + refs.current[VIEWS[j]]?.focus() + } + return ( +
+ {VIEWS.map((v, i) => { + const selected = view === v + return ( + + ) + })} +
+ ) +} + /** 媒体查询的当前结果，跟着窗口大小变 */ function useMedia(query: string): boolean { const [on, setOn] = useState(() => typeof matchMedia !== 'function' || matchMedia(query).matches) @@ -601,13 +749,14 @@ function UsageSummary({ used, pending, none, active, onFocus }: { ) } -/** 没有选中表时的右栏：空目录时引导起草；否则写审阅进度，给出下一张该审的表 */ -function Overview({ rows, empty, systemNotes, draft, onOpen }: { +/** 没有选中表时的右栏：空目录时引导起草；否则写审阅进度，给出下一张该审的表，也可以去审阅队列跨表成批审阅 */ +function Overview({ rows, empty, systemNotes, draft, onOpen, onReview }: { rows: CatalogTableRow[] empty: boolean systemNotes: boolean draft: ReactNode onOpen: (table: string) => void + onReview: () => void }) { const pendingNext = rows.find((r) => r.in_schema && r.counts.proposed > 0) const stats = { pending: 0, done: 0, none: 0 } @@ -634,6 +783,12 @@ function Overview({ rows, empty, systemNotes, draft, onOpen }: { {CT.reviewNextHint}
)} + {pendingNext && ( +
+ + {RT.openQueueHint} +
+ )} )} {systemNotes && } diff --git a/frontend/src/pages/catalog/ColumnTable.tsx b/frontend/src/pages/catalog/ColumnTable.tsx index cec5a1ba..12b043d3 100644 --- a/frontend/src/pages/catalog/ColumnTable.tsx +++ b/frontend/src/pages/catalog/ColumnTable.tsx @@ -9,7 +9,7 @@ import { IconButton } from '../../components/ui' import { CATALOG_COLUMN_FIELD_LABEL, CATALOG_MEASURE_HINT, CATALOG_MEASURE_LABEL, CATALOG_TEXT as CT, CODES_TEXT as KT, } from '../../lib/terms' -import { ItemMark } from './parts' +import { FOCUS_RING, FOCUS_STYLE, ItemMark, NoteHint, ReasonHint, confirmedNote, confirmedReason } from './parts' import type { ReviewTarget } from './parts' import { COLUMN_FIELDS, codesComplete, columnPath, completeOf } from './model' import type { ColumnDraft, ColumnField } from './model' @@ -50,7 +50,16 @@ export interface ColumnModel { type ColumnTexts = ColumnDraft -export function ColumnTable({ columns, form, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange, onCodesComplete }: { +/** 审阅路径 columns.<列名>.<字段> → [列名, 字段]（列名里可以有句点，按最后一个句点切）；不是列级项为 null */ +function columnOfPath(path: string | null | undefined): [string, string] | null { + if (!path?.startsWith('columns.')) return null + const rest = path.slice('columns.'.length) + const at = rest.lastIndexOf('.') + return at > 0 ? [rest.slice(0, at), rest.slice(at + 1)] : null +} + +export function ColumnTable({ columns, form, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange, onCodesComplete, + focusPath = null }: { columns: ColumnModel[] /** 编辑中：列名 → 字段 → 文字（form 是当前的，initial 是进入编辑时的）；不在编辑时为 null */ form: Record | null @@ -67,7 +76,10 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote onChange: (col: string, field: ColumnField, value: string) => void /** 编辑中勾选或取消码值的「已列出全部取值」 */ onCodesComplete: (col: string, value: boolean) => void + /** 从审阅队列打开的那一项（列级项的审阅路径）：那一格标出来 */ + focusPath?: string | null }) { + const focus = columnOfPath(focusPath) return (
@@ -89,7 +101,7 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote {columns.map((c) => ( + onCodesComplete={onCodesComplete} focusField={focus && focus[0] === c.name ? focus[1] : null} /> ))}
@@ -98,7 +110,7 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote } const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange, - onCodesComplete }: { + onCodesComplete, focusField }: { col: ColumnModel editing: boolean /** 这一列的表单。只有改了这一列时引用才会变，别的行不重画 */ @@ -112,6 +124,8 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble onFillCodes: (col: string) => void onChange: (col: string, field: ColumnField, value: string) => void onCodesComplete: (col: string, value: boolean) => void + /** 从审阅队列打开的是这一列的哪一项 */ + focusField: string | null }) { const { name, structure, items } = col const pending = COLUMN_FIELDS.filter((f) => items?.[f]?.status === 'proposed').length @@ -121,16 +135,25 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble const where = CT.whereColumn(name, CATALOG_COLUMN_FIELD_LABEL[f]) const problem = problems[`c:${name}:${f}`] const changed = editing && ((draft?.[f] ?? '') !== (initial?.[f] ?? '') || (f === 'codes' && completeOf(draft) !== completeOf(initial))) + const focused = focusField === f + // 人工确认的项：确认时的依据、备注（剖析的说明……）各一个图标，悬停看 + const reason = editing ? '' : confirmedReason(it) + const note = editing ? '' : confirmedNote(it) return ( // 窄框下没填的项不占行（编辑时照常列出来好填） - + {CT.columnHead[f]}
{editing ? input : view}
+ + {note && } {it && !changed && ( onReview({ path: columnPath(name, f), where, source: it.source, status: it.status }, a)} /> )} @@ -223,6 +246,7 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble ) }, (a, b) => a.col === b.col && a.editing === b.editing && a.draft === b.draft && a.initial === b.initial && a.busy === b.busy + && a.focusField === b.focusField && a.systemNotes === b.systemNotes && a.onReview === b.onReview && a.onConfirmColumn === b.onConfirmColumn && a.onFillCodes === b.onFillCodes && a.onChange === b.onChange && a.onCodesComplete === b.onCodesComplete // 格式问题每次都是新对象：只比这一列的几格 diff --git a/frontend/src/pages/catalog/DraftDialog.tsx b/frontend/src/pages/catalog/DraftDialog.tsx index 3a7547fc..2e5b71c2 100644 --- a/frontend/src/pages/catalog/DraftDialog.tsx +++ b/frontend/src/pages/catalog/DraftDialog.tsx @@ -9,8 +9,8 @@ import { errorMessage } from '../../lib/errors' import { formatTime } from '../../lib/format' import { CATALOG_DRAFT_TEXT as DT, CATALOG_TEXT as CT, DRAFT_GUIDANCE_MAX, DRAFT_NOTE_MAX } from '../../lib/terms' import { - DRAFT_MAX_TABLES, DRAFT_MODEL_UNAVAILABLE, DRAFT_PART_COLUMNS, DRAFT_TOP_N, RETRYABLE, blankOf, isJobActive, jobBlank, needsLook, - nothingWritten, retryableCount, taskReason, + DRAFT_MAX_TABLES, DRAFT_MODEL_UNAVAILABLE, DRAFT_PART_COLUMNS, DRAFT_TOP_N, RETRYABLE, blankOf, chunksOf, isJobActive, jobBlank, needsLook, + nothingWritten, retryableCount, taskReason, writtenOf, } from './draftJob' import type { DraftPreset, DraftStartBody } from './draftJob' @@ -435,6 +435,9 @@ function JobView({ job, running, stopping, busy, blankTables, onRedraftBlank }: // 额度用完、超时多了，之后的段缩小了：进行中写一句，段数为什么变多看得明白 const partColumns = running && job.use_model && job.part_columns != null && job.part_columns < DRAFT_PART_COLUMNS ? job.part_columns : null const blankTotal = jobBlank(job) + // 停下来时已经写进了一部分段的表（宽表每写完一段就写进目录）：几张、合计几段 + const stopped = job.tasks.filter((t) => t.status === 'cancelled' && writtenOf(t) > 0) + const stoppedParts = { tables: stopped.length, parts: stopped.reduce((n, t) => n + writtenOf(t), 0) } return (
{busy && {DT.busy}} @@ -468,7 +471,12 @@ function JobView({ job, running, stopping, busy, blankTables, onRedraftBlank }: {running &&

{DT.hideHint}

} {job.status === 'failed' && job.error && {job.error}} {job.status === 'interrupted' && {DT.interrupted}} - {job.status === 'cancelled' &&

{DT.cancelled}

} + {job.status === 'cancelled' && ( +

+ {DT.cancelled} + {stoppedParts.tables > 0 && {DT.cancelledParts(stoppedParts.tables, stoppedParts.parts)}} +

+ )} {problemText && (

{problemText}

)} @@ -523,10 +531,20 @@ const TASK_ICON: Record 1 && t.status !== 'pending' && t.status !== 'skipped' + const started = t.status !== 'pending' && t.status !== 'skipped' + const parts = t.parts_total != null && t.parts_total > 1 && started + // 段被缩小、拆半后实际的块（比段多时才写）；已经写进目录的段：起草中、部分完成时写着（停止也不丢，部分完成的 + // 段进度是「都结束了」，看不出写进了几段），停下来时补一句剩下几段没起草。服务端的原因里已经写了的不重复，写完了的不写 + const chunks = started ? chunksOf(t) : null + const written = writtenOf(t) + const writtenText = written <= 0 || reason.includes(DT.partsWritten(written)) ? '' + : t.status === 'running' || t.status === 'partial' ? DT.partsWritten(written) + : t.status === 'cancelled' ? DT.partsWrittenStopped(written, Math.max(0, (t.parts_total ?? written) - written)) : '' const changes = DT.taskAdded(t.added, t.updated) const blank = blankOf(t) const sample = t.blank_sample?.filter(Boolean) ?? [] + // 起草时记下的几句话（服务端原话，不算失败）：没收下的有效记录条件和原因，照原样列在这张表下面 + const notes = t.log?.filter(Boolean) ?? [] return (
  • @@ -537,6 +555,11 @@ function TaskRow({ task: t, jobError }: { task: CatalogDraftTask; jobError: stri {DT.parts(t.parts_done, t.parts_total!)} )} + {chunks && ( + + {DT.chunks(chunks.done, chunks.total)} + + )} {changes && ( {changes} )} @@ -545,6 +568,11 @@ function TaskRow({ task: t, jobError }: { task: CatalogDraftTask; jobError: stri {DT.taskStatus[t.status]}
    + {writtenText && ( +

    + {writtenText} +

    + )} {reason && (

    {reason}

    )} @@ -556,6 +584,9 @@ function TaskRow({ task: t, jobError }: { task: CatalogDraftTask; jobError: stri ) :

    {DT.blank(blank)}

    )} + {notes.map((line, i) => ( +

    {line}

    + ))}
    • ) } diff --git a/frontend/src/pages/catalog/ReviewQueue.tsx b/frontend/src/pages/catalog/ReviewQueue.tsx new file mode 100644 index 00000000..fd82abec --- /dev/null +++ b/frontend/src/pages/catalog/ReviewQueue.tsx @@ -0,0 +1,439 @@ +import { memo, useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { AlertTriangle, CheckCheck, RotateCcw, RotateCw, X } from 'lucide-react' +import clsx from 'clsx' +import { api } from '../../api/client' +import type { + CatalogDetail, CatalogReviewAction, CatalogReviewItem, CatalogReviewKind, CatalogReviewProgress, CatalogTableRow, +} from '../../types' +import { EmptyState, ErrorState, Notice, Skeleton, Spinner, confirmDialog, toast, useRadioGroup } from '../../components/ui' +import { errorMessage } from '../../lib/errors' +import { CATALOG_REVIEW_TEXT as RT } from '../../lib/terms' +import { StatusChip } from './parts' +import { + REVIEW_BATCH_MAX, REVIEW_KINDS, REVIEW_PAGE, applyResults, batchTargets, confirmProgress, itemKey, pendingTotal, reviewWhere, + reviewedCount, statusOf, syncFromDetail, +} from './review' +import type { ItemOutcome } from './review' + +// =========================================================================== +// 审阅队列（数据目录页的「审阅队列」视图，左栏）：跨表列出推断项，顺序由服务端排好（所在表的使用次数 × 这一项对数字的 +// 影响 × 来源），每一项写为什么排在这里。可以按项（影响数字的项 / 名称和含义）和表筛选；多选后成批确认、驳回，审阅过的 +// 原地标出结果（冲突的标出来），选中可以撤销审阅；刷新队列后审阅过的才移除，列表头的「N 项待确认」审阅成功一项就少一项 +// （pendingTotal，和视图标签、按表筛选的计数同一个口径）。顶上是审阅进度：常用表（使用次数最多、 +// 被运行用过的前 20 张）、全部表、影响数字的项各确认了多少，三行同一个口径。点开一项，右栏是所在表的详情（滚到那一项、 +// 标出来）和用过这张表的运行。 +// =========================================================================== + +export function ReviewQueue({ sourceId, rows, kind, table, activeKey, detailSync, onParams, onOpenItem, onChanged }: { + sourceId: string + /** 目录清单的各行：按表筛选的候选（有推断项的表）、表的中文名 */ + rows: CatalogTableRow[] + kind: CatalogReviewKind + /** 只看这一张表；空串是全部表 */ + table: string + /** 右栏正打开的那一项（itemKey） */ + activeKey: string | null + /** 右栏的表详情载入、改过之后的目录：队列里这张表的项跟着换状态 */ + detailSync: { seq: number; detail: CatalogDetail } | null + onParams: (p: { kind?: CatalogReviewKind; table?: string }) => void + onOpenItem: (item: CatalogReviewItem) => void + /** 成批审阅写进了这些表：页面重取清单、正在看的表重新载入 */ + onChanged: (tables: string[]) => void +}) { + const [items, setItems] = useState(null) + const [total, setTotal] = useState(0) + const [loadError, setLoadError] = useState(null) + const [loadingMore, setLoadingMore] = useState(false) + const [outcomes, setOutcomes] = useState>(() => new Map()) + const [versions, setVersions] = useState>(() => new Map()) + const [selected, setSelected] = useState>(() => new Set()) + const [acting, setActing] = useState(null) + const [summary, setSummary] = useState<{ action: CatalogReviewAction; ok: number; conflicts: number; failed: number } | null>(null) + const [batchError, setBatchError] = useState(null) + const [progress, setProgress] = useState(null) + const [progressError, setProgressError] = useState(false) + const [reloadSeq, setReloadSeq] = useState(0) + const loadSeq = useRef(0) + const itemsRef = useRef(items) + itemsRef.current = items + const outcomesRef = useRef(outcomes) + outcomesRef.current = outcomes + const versionsRef = useRef(versions) + versionsRef.current = versions + const allRef = useRef(null) + const radio = useRadioGroup(REVIEW_KINDS, kind, (k) => onParams({ kind: k })) + + const loadProgress = useCallback(() => { + api.dataCatalog.reviewProgress(sourceId).then((p) => { setProgress(p); setProgressError(false) }, () => setProgressError(true)) + }, [sourceId]) + useEffect(() => { loadProgress() }, [loadProgress]) + + // 第一页：筛选变了、点了刷新就从头取，审阅过的标记、选择、汇总都清掉 + useEffect(() => { + const mine = ++loadSeq.current + setItems(null) + setLoadError(null) + api.dataCatalog.reviewQueue(sourceId, { limit: REVIEW_PAGE, offset: 0, kind, table: table || null }).then((q) => { + if (mine !== loadSeq.current) return + setItems(q.items) + setTotal(q.total) + setOutcomes(new Map()) + setVersions(new Map()) + setSelected(new Set()) + setSummary(null) + setBatchError(null) + }, (e) => { if (mine === loadSeq.current) setLoadError(e) }) + }, [sourceId, kind, table, reloadSeq]) + + const refresh = () => { + setReloadSeq((n) => n + 1) + loadProgress() + } + + /** + * 再载入一页。服务端的队列里已经没有审阅成功、不再是推断的项：偏移按「这一页上还在服务端队列里的项」数， + * 不会跳过没载入的项 + */ + const loadMore = async () => { + const list = itemsRef.current + if (!list || loadingMore) return + const gone = reviewedCount(list, outcomes) + setLoadingMore(true) + try { + const q = await api.dataCatalog.reviewQueue(sourceId, { limit: REVIEW_PAGE, offset: list.length - gone, kind, table: table || null }) + setItems((cur) => { + const have = new Set((cur ?? []).map(itemKey)) + return [...(cur ?? []), ...q.items.filter((i) => !have.has(itemKey(i)))] + }) + setTotal(q.total + gone) + } catch (e) { + toast.error(e) + } finally { + setLoadingMore(false) + } + } + + // 右栏的表详情载入、改过：这张表的项按最新的目录换状态 + const syncSeen = useRef(detailSync?.seq ?? 0) + useEffect(() => { + if (!detailSync || detailSync.seq === syncSeen.current) return + syncSeen.current = detailSync.seq + const list = itemsRef.current + if (!list) return + const d = detailSync.detail + if (!list.some((i) => i.table_name === d.table_name)) return + const next = syncFromDetail(list, d.table_name, d.notes, d.version, outcomesRef.current, versionsRef.current) + setOutcomes(next.outcomes) + setVersions(next.versions) + loadProgress() + }, [detailSync, loadProgress]) + + const act = async (action: CatalogReviewAction) => { + const list = itemsRef.current ?? [] + const targets = batchTargets(list, selected, action, outcomes) + if (!targets.length || acting) return + if (targets.length > REVIEW_BATCH_MAX) { + setBatchError(RT.tooMany(REVIEW_BATCH_MAX)) + return + } + if (action !== 'reset') { + const ok = await confirmDialog(action === 'confirm' + ? { title: RT.confirmTitle(targets.length), consequences: RT.confirmConsequences, confirmLabel: RT.confirmAction(targets.length) } + : { title: RT.rejectTitle(targets.length), consequences: RT.rejectConsequences, confirmLabel: RT.rejectAction(targets.length), danger: true }) + if (!ok) return + } + setActing(action) + setBatchError(null) + try { + const { results } = await api.dataCatalog.reviewBatch(sourceId, targets.map((i) => ({ + table_name: i.table_name, path: i.path, action, if_version: versions.get(i.table_name) ?? i.version, + }))) + const next = applyResults(results, action, outcomes, versions) + setOutcomes(next.outcomes) + setVersions(next.versions) + setSelected(new Set()) + const ok = results.filter((r) => r.ok) + const conflicts = results.filter((r) => !r.ok && next.outcomes.get(itemKey(r))?.kind === 'conflict').length + setSummary({ action, ok: ok.length, conflicts, failed: results.length - ok.length - conflicts }) + if (ok.length) onChanged([...new Set(ok.map((r) => r.table_name))]) + loadProgress() + } catch (e) { + setBatchError(RT.failedAll(errorMessage(e))) + } finally { + setActing(null) + } + } + + // 按表筛选的候选：有推断项的表（使用次数多的在前），正在筛的那张不在里面也列上 + const tableOptions = useMemo(() => { + const list = rows.filter((r) => r.counts.proposed > 0) + if (table && !list.some((r) => r.table_name === table)) { + const row = rows.find((r) => r.table_name === table) + return [...list, row ?? ({ table_name: table, label: null, counts: { proposed: 0 } } as unknown as CatalogTableRow)] + } + return list + }, [rows, table]) + const labelOf = useMemo(() => new Map(rows.map((r) => [r.table_name, r.label])), [rows]) + + const loaded = items ?? [] + const picked = loaded.filter((i) => selected.has(itemKey(i))).length + const all = loaded.length > 0 && picked === loaded.length + useEffect(() => { + if (allRef.current) allRef.current.indeterminate = picked > 0 && !all + }, [picked, all]) + const toggleAll = () => setSelected(all ? new Set() : new Set(loaded.map(itemKey))) + const toggle = useCallback((key: string) => setSelected((cur) => { + const next = new Set(cur) + if (next.has(key)) next.delete(key) + else next.add(key) + return next + }), []) + const counts = { + confirm: batchTargets(loaded, selected, 'confirm', outcomes).length, + reject: batchTargets(loaded, selected, 'reject', outcomes).length, + reset: batchTargets(loaded, selected, 'reset', outcomes).length, + } + const filtered = kind !== 'all' || !!table + // 还没载入的项（total 里含列表里审阅过、刷新前还留着的项）；列表头写的是还待确认的（审阅过的减掉） + const remaining = Math.max(0, total - loaded.length) + const pending = pendingTotal(total, loaded, outcomes) + + return ( +
    + +
    +
    + {REVIEW_KINDS.map((k) => ( + + ))} +
    +
    + + + +
    +
    + + + {selected.size ? RT.selected(selected.size) : items ? RT.total(pending) : RT.loading} + + {selected.size > 0 && ( + + )} +
    +
    + + {summary && ( +

    + {summary.conflicts || summary.failed + ? + : } + {RT.summary(summary.action, summary.ok, summary.conflicts, summary.failed)} +

    + )} + {batchError &&
    {batchError}
    } + +
    + {loadError + ?
    + : !items + ?
    + : !items.length + ? ( + } title={filtered ? RT.emptyFiltered : RT.empty} + body={filtered ? RT.emptyFilteredBody : RT.emptyBody} offline={false} className="!py-10" /> + ) + : ( +
      + {items.map((i) => { + const key = itemKey(i) + return ( + + ) + })} +
    + )} + {items && remaining > 0 && ( +
    + +
    + )} +
    + + {selected.size > 0 && ( +
    + {RT.selected(selected.size)} + + + + +
    + )} +
    + ) +} + +/** 队列里的一行：复选框、位置和所在表、值、说明、为什么排在这里、来源和状态（审阅过的写结果）、所在表的使用次数 */ +const QueueRow = memo(function QueueRow({ item: i, itemKeyText, tableLabel, outcome, checked, active, onToggle, onOpen }: { + item: CatalogReviewItem + itemKeyText: string + tableLabel: string | null + outcome: ItemOutcome | undefined + checked: boolean + active: boolean + onToggle: (key: string) => void + onOpen: (item: CatalogReviewItem) => void +}) { + const status = statusOf(i, outcome) + // 位置按 path 说（「列 status 的码值」）：服务端的 field_label 只是字段的叫法，列级的项只写它分不清是哪一列 + const where = reviewWhere(i) + const tableText = tableLabel ? `${tableLabel}（${i.table_name}）` : i.table_name + const mono = i.field === 'valid_filter' || i.field === 'keys' + return ( +
  • + {active && } + + onToggle(itemKeyText)} aria-label={RT.select(where, i.table_name)} + data-review-select="" /> + + +
    • + ) +}) + +/** + * 审阅进度：常用表、全部表、影响数字的项，三行同一个口径——已确认 ÷（已确认 + 待确认），已驳回的、有确证的不计入， + * 审完了就是 100%。常用表只算被运行用过的（最多 top_n 张），标签写实际几张，一张都没有时写明还没有运行用过的表。 + * 数字写在字里，色条只给眼睛看 + */ +function ProgressBlock({ progress, failed }: { progress: CatalogReviewProgress | null; failed: boolean }) { + if (!progress) { + return ( +
    + {failed ? RT.progressFailed : } +
    + ) + } + const { top, all, facts } = progress + const topTables = Math.max(0, top.tables ?? 0) + const topLine = confirmProgress(top.items_confirmed, top.items_proposed) + const allLine = confirmProgress(all.items_confirmed, all.items_proposed) + const factsLine = confirmProgress(facts.confirmed, facts.proposed) + return ( +
    + {topTables > 0 + ? ( + + ) + : ( +
    + {RT.progressTopNone} + {RT.progressTopEmpty} +
    + )} + + 0} /> +
    + ) +} + +function ProgressLine({ label, hint, line, total, attr, warn = false }: { + label: string + hint: string + line: ReturnType + /** 这一行一共几项（全部状态）：0 时写还没有目录项；不知道时为 null */ + total: number | null + attr: string + warn?: boolean +}) { + const text = RT.progressText(line.pct, line.confirmed, line.proposed, total) + return ( +
    +
    + {label} + {text} +
    + {line.pct != null && ( +
    +
    +
    + )} +
    + ) +} + +/** 审阅视图里没打开项时的右栏：怎么排的序、点开能看到什么 */ +export function ReviewOverview() { + return ( +
    +
    +

    {RT.overviewTitle}

    +

    {RT.overviewBody}

    +

    {RT.kindHint.facts}

    +
    +
    + ) +} diff --git a/frontend/src/pages/catalog/TableAssistant.tsx b/frontend/src/pages/catalog/TableAssistant.tsx new file mode 100644 index 00000000..ea398543 --- /dev/null +++ b/frontend/src/pages/catalog/TableAssistant.tsx @@ -0,0 +1,617 @@ +import { memo, useCallback, useEffect, useId, useMemo, useRef, useState } from 'react' +import type { KeyboardEvent as ReactKeyboardEvent } from 'react' +import { ArrowUp, BookMarked, Database, Info, MessagesSquare, Minus, Plus, Square, Trash2, X } from 'lucide-react' +import clsx from 'clsx' +import { ApiError, api, streamCatalogAssistant } from '../../api/client' +import type { CatalogAssistantEnd } from '../../api/client' +import type { CatalogDetail } from '../../types' +import { Notice, Spinner, confirmDialog, isComposing, toast } from '../../components/ui' +import { useLeaveGuard } from '../../lib/leave' +import { CATALOG_ASSISTANT_TEXT as T, CATALOG_STATUS_LABEL } from '../../lib/terms' +import { Markdown } from '../../run/Markdown' +import { beforeStatusShown, patchBeforeText, patchValueText } from '../../run/catalogPatch' +import { + ASSISTANT_MAX_CHARS, ASSISTANT_MAX_SAVE, QUERY_ROWS_SHOWN, applyEvent, applyPreview, assistantChangesOf, blankColumnsOf, blankLeadOf, + changeStateOf, changeWhere, dropPending, historyOf, mergeProposal, pendingForAssistant, pendingSubmit, pendingVersion, savedKey, +} from './assistant' +import type { AssistantTurn, ChangeState, PendingChange, Turn, TurnPart } from './assistant' + +// =========================================================================== +// 表助手：在表详情里和助手对话完善这一张表的数据目录（POST /catalog/{table}/assistant，事件流）。 +// +// 回答流式显示，能停止；助手跑过的只读查询（只有开了数据剖析的源才有）写 SQL 和前几行，默认收起；修改提案逐项写改前、 +// 改后、依据，可以逐项去掉、加回。提案并进「待保存的修改」：跨多轮累积，同一项以最后一次提案为准，一键保存（最多 200 项）。 +// 保存带待保存的修改里最早那条提案对照的版本：别人在那之后改过就 409，按最新内容重算（预览接口，只读）后停下来让人再看 +// 一眼；只重算交去保存的那几项，保存期间回答里又到的提案原样留着。对话不存服务端：每一轮把前面的对话和待保存的修改 +// （最多 200 项）一起交上去；关闭、换表、离开页面就没有了，有待保存的修改时先问。 +// 开场：上次起草有留空的列时，引导先问这几列——列名取那次起草记下的（任务里这张表的 blank_sample），拿不到时改说目录里 +// 还没有中文名和含义的列。 +// =========================================================================== + +type SaveState = + | { phase: 'idle'; error?: string | null; saved?: { n: number; version: number } | null } + | { phase: 'saving' } + | { phase: 'reloading' } + | { phase: 'conflict'; problems: string[] } + +const charCount = (s: string) => [...s.trim()].length + +export function TableAssistant({ sourceId, table, tableLabel, detail, profileEnabled, lastBlank, editing, onSaved, onClose }: { + sourceId: string + table: string + /** 表的中文名（没有就是表名）：标题、提示里用 */ + tableLabel: string + /** 页面上这张表最新的目录：留空的列、保存后更新 */ + detail: CatalogDetail + /** 数据源开了数据剖析：助手才能发只读查询 */ + profileEnabled: boolean + /** + * 上次起草模型拿不准、留空的列数（n）和出处：开场引导先问这几列。jobId 是那次起草的任务（从中取起草时记下的列名）； + * earlier：最近一次停止了，n 是再之前那次起草记下的（不知道是哪一次，取不到列名） + */ + lastBlank: { n: number; earlier: boolean; jobId: string | null } + /** 正在编辑这张表：先不让保存（同一张表两处同时改，保存会撞上版本冲突） */ + editing: boolean + /** 保存成功：页面换上返回的目录 */ + onSaved: (d: CatalogDetail) => void + /** 关上面板（有待保存的修改时这里先问过了） */ + onClose: () => void +}) { + const [turns, setTurns] = useState([]) + const [pending, setPending] = useState([]) + const [saved, setSaved] = useState>(() => new Set()) + const [text, setText] = useState('') + const [save, setSave] = useState({ phase: 'idle' }) + const [trayOpen, setTrayOpen] = useState(true) + const turnsRef = useRef(turns) + turnsRef.current = turns + const pendingRef = useRef(pending) + pendingRef.current = pending + const stopRef = useRef<(() => void) | null>(null) + const seq = useRef(0) + const inputRef = useRef(null) + const logRef = useRef(null) + const titleId = useId() + const streaming = turns.some((t) => t.role === 'assistant' && t.state === 'streaming') + const dirty = pending.length > 0 || streaming + + // 打开时焦点给输入框；卸下时停掉进行中的回答 + useEffect(() => { inputRef.current?.focus() }, []) + useEffect(() => () => { stopRef.current?.() }, []) + + useLeaveGuard(pending.length > 0, () => confirmDialog({ + title: T.leaveTitle, consequences: T.leaveConsequences, confirmLabel: T.leaveConfirm, danger: true, + })) + + // 新的内容到了：滚到底（人往上翻着看的时候不打扰） + const lastLen = useRef(0) + useEffect(() => { + const el = logRef.current + if (!el) return + const size = turns.reduce((n, t) => n + (t.role === 'user' ? 1 : t.parts.length + 1), 0) + const near = el.scrollHeight - el.scrollTop - el.clientHeight < 160 + if (size !== lastLen.current || near) el.scrollTop = el.scrollHeight + lastLen.current = size + }, [turns]) + + // 上次起草记下的留空列名：从那次起草的任务里取（null 是还没取回来；取不到是空数组，改说目录里还没有中文名和含义的列） + const blankN = lastBlank.n + const blankJob = blankN > 0 && !lastBlank.earlier ? lastBlank.jobId : null + const [blankSample, setBlankSample] = useState(null) + useEffect(() => { + if (!blankJob) return + let live = true + setBlankSample(null) + api.dataCatalog.draftJob(sourceId, blankJob) + .then(({ job }) => job.tasks.find((t) => t.table_name === table)?.blank_sample?.filter(Boolean) ?? []) + .catch(() => [] as string[]) + .then((cols) => { if (live) setBlankSample(cols) }) + return () => { live = false } + }, [sourceId, table, blankJob]) + // 没有留空的列、还在取那次起草记下的列名时不写 + const blankLead = useMemo(() => (blankN <= 0 || (blankJob && blankSample == null) + ? null : blankLeadOf(blankN, blankJob ? blankSample : null, blankColumnsOf(detail))), [blankN, blankJob, blankSample, detail]) + const hasProposed = useMemo(() => { + const n = detail.notes + const items = [n.label, n.description, n.grain, n.keys, n.kind, n.business_date, n.valid_filter, n.dedup, + ...Object.values(n.columns ?? {}).flatMap((c) => Object.values(c ?? {})), ...(n.relations ?? [])] + return items.some((it) => !!it && (it as { status?: string }).status === 'proposed') + }, [detail]) + + const updateTurn = useCallback((id: string, fn: (t: AssistantTurn) => AssistantTurn) => { + setTurns((ts) => ts.map((t) => (t.id === id && t.role === 'assistant' ? fn(t) : t))) + }, []) + + const send = useCallback((raw: string) => { + const question = raw.trim() + if (!question || charCount(question) > ASSISTANT_MAX_CHARS) return + if (turnsRef.current.some((t) => t.role === 'assistant' && t.state === 'streaming')) return + const n = ++seq.current + const userTurn: Turn = { id: `u${n}`, role: 'user', text: question } + const reply: AssistantTurn = { id: `a${n}`, role: 'assistant', parts: [], state: 'streaming' } + const messages = historyOf(turnsRef.current, question) + setTurns((ts) => [...ts, userTurn, reply]) + setText('') + let proposals = 0 + // 这一轮已经收尾（done、error）：之后再来的事件不认 + let ended = false + stopRef.current = streamCatalogAssistant(sourceId, table, { messages, pending: pendingForAssistant(pendingRef.current) }, (ev) => { + if (ended) return + if (ev.type === 'done' || ev.type === 'error') ended = true + if (ev.type === 'proposal') { + const pid = `${reply.id}.p${++proposals}` + const changes = assistantChangesOf(ev.changes) + updateTurn(reply.id, (t) => applyEvent(t, ev, pid)) + if (changes.length) { + const version = typeof ev.version === 'number' && Number.isFinite(ev.version) ? ev.version : detail.version + setPending((cur) => mergeProposal(cur, changes, pid, version)) + // 待保存的修改变了：上一次保存的结果、冲突的说明都不再对。正在保存、重算的不动（按钮照旧不能点，结果出来再说） + setSave((s) => (s.phase === 'saving' || s.phase === 'reloading' ? s : { phase: 'idle' })) + } + return + } + updateTurn(reply.id, (t) => applyEvent(t, ev, '')) + }, (end: CatalogAssistantEnd) => { + stopRef.current = null + updateTurn(reply.id, (t) => { + if (t.state !== 'streaming') return t + if (end.aborted) return { ...t, state: 'stopped' } + if (end.error) return { ...t, state: 'error', error: end.error, code: end.code ?? null } + // 流结束了却没有 done：这一轮没答完 + return { ...t, state: 'error', error: T.interrupted } + }) + }) + }, [sourceId, table, detail.version, updateTurn]) + + const stop = () => stopRef.current?.() + + const onKeyDown = (e: ReactKeyboardEvent) => { + if (e.key !== 'Enter' || e.shiftKey || isComposing(e)) return + e.preventDefault() + if (!streaming) send(text) + } + + // 两个回调保持不变：流式的正文每来一段只重画正在回答的那一轮 + const drop = useCallback((path: string) => setPending((cur) => dropPending(cur, path)), []) + const restore = useCallback((part: Extract, path: string) => { + const change = part.changes.find((c) => c.path === path) + if (change) setPending((cur) => mergeProposal(cur, [change], part.id, part.version)) + }, []) + const clearAll = async () => { + if (!pendingRef.current.length) return + if (!(await confirmDialog({ title: T.clearTitle(pendingRef.current.length), confirmLabel: T.clearConfirm, danger: true }))) return + setPending([]) + setSave({ phase: 'idle' }) + } + + const doSave = async () => { + const list = pendingRef.current + if (!list.length || list.length > ASSISTANT_MAX_SAVE || editing) return + setSave({ phase: 'saving' }) + try { + const d = await api.dataCatalog.patch(sourceId, table, pendingSubmit(list), pendingVersion(list)) + setSaved((cur) => new Set([...cur, ...list.map((p) => savedKey(p.origin, p.path))])) + setPending((cur) => cur.filter((p) => !list.includes(p))) + setSave({ phase: 'idle', saved: { n: list.length, version: d.version } }) + onSaved(d) + toast.ok(T.savedToast(list.length, tableLabel)) + } catch (e) { + if (e instanceof ApiError && e.status === 409) { + // 别人刚改过：按最新内容重算（只读），停下来让人再看一眼 + setSave({ phase: 'reloading' }) + try { + const p = await api.dataCatalog.patchPreview(sourceId, table, pendingSubmit(list)) + // 只重算交去保存的那几项：保存、重算期间回答里又到的提案不在预览里，原样留着 + setPending((cur) => applyPreview(cur, p, list)) + setSave({ phase: 'conflict', problems: p.problems }) + } catch { + setSave({ phase: 'idle', error: T.conflictFailed }) + } + } else if (e instanceof ApiError && e.kind === 'http' && (e.status === 422 || e.status === 404)) { + setSave({ phase: 'idle', error: e.message }) + } else { + setSave({ phase: 'idle' }) + toast.error(e) + } + } + } + + const close = async () => { + if (dirty && !(await confirmDialog({ title: T.closeTitle, consequences: T.closeConsequences, confirmLabel: T.closeConfirm, danger: true }))) return + stopRef.current?.() + onClose() + } + + const onPanelKey = (e: ReactKeyboardEvent) => { + if (e.key !== 'Escape' || isComposing(e) || e.defaultPrevented) return + e.preventDefault() + void close() + } + + const fresh = turns.length === 0 + const len = charCount(text) + const over = len > ASSISTANT_MAX_CHARS + const lastAssistant = [...turns].reverse().find((t): t is AssistantTurn => t.role === 'assistant') + const status = streaming ? T.answering : lastAssistant?.state === 'stopped' ? T.stopped : lastAssistant ? T.answered : '' + + return ( +