From adc56da6cffff274a553f1224e694fa655930c72 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 07:59:51 +0800 Subject: [PATCH 01/39] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E5=AE=A1=E9=98=85=E9=98=9F=E5=88=97=EF=BC=9A=E6=8E=A8=E6=96=AD?= =?UTF-8?q?=E7=9A=84=E9=A1=B9=E6=8C=89=E3=80=8C=E4=BD=BF=E7=94=A8=E6=AC=A1?= =?UTF-8?q?=E6=95=B0=E6=9D=83=E9=87=8D=20=C3=97=20=E9=A3=8E=E9=99=A9?= =?UTF-8?q?=E6=9D=83=E9=87=8D=20=C3=97=20=E6=9D=A5=E6=BA=90=E6=9D=83?= =?UTF-8?q?=E9=87=8D=E3=80=8D=E6=8E=92=E5=A5=BD=EF=BC=88=E6=9C=89=E6=95=88?= =?UTF-8?q?=E8=AE=B0=E5=BD=95=E6=9D=A1=E4=BB=B6=E3=80=81=E5=8E=BB=E9=87=8D?= =?UTF-8?q?=E8=A7=84=E5=88=99=E3=80=81=E4=B8=9A=E5=8A=A1=E6=97=A5=E6=9C=9F?= =?UTF-8?q?=E3=80=81=E5=85=B3=E8=81=94=E5=85=B3=E7=B3=BB=E6=9C=80=E9=AB=98?= =?UTF-8?q?=EF=BC=8C=E6=A8=A1=E5=9E=8B=E8=B5=B7=E8=8D=89=E3=80=81=E5=91=BD?= =?UTF-8?q?=E5=90=8D=E6=8E=A8=E6=96=AD=E6=AF=94=E5=BA=93=E6=B3=A8=E9=87=8A?= =?UTF-8?q?=E4=BC=98=E5=85=88=EF=BC=89=EF=BC=8C=E5=B8=A6=E6=8E=92=E5=9C=A8?= =?UTF-8?q?=E8=BF=99=E9=87=8C=E7=9A=84=E7=90=86=E7=94=B1=EF=BC=8C=E5=8F=AF?= =?UTF-8?q?=E6=8C=89=E5=BD=B1=E5=93=8D=E6=95=B0=E5=AD=97=E7=9A=84=E9=A1=B9?= =?UTF-8?q?=20/=20=E5=90=8D=E7=A7=B0=E5=92=8C=E8=AF=B4=E6=98=8E=E3=80=81?= =?UTF-8?q?=E6=8C=89=E8=A1=A8=E7=AD=9B=E9=80=89=E5=92=8C=E5=88=86=E9=A1=B5?= =?UTF-8?q?=EF=BC=9B=E6=89=B9=E9=87=8F=E7=A1=AE=E8=AE=A4=E3=80=81=E9=A9=B3?= =?UTF-8?q?=E5=9B=9E=E3=80=81=E6=92=A4=E9=94=80=E5=AE=A1=E9=98=85=E9=80=90?= =?UTF-8?q?=E8=A1=A8=E6=8C=89=E7=89=88=E6=9C=AC=E4=B9=90=E8=A7=82=E9=94=81?= =?UTF-8?q?=EF=BC=8C=E5=90=8C=E4=B8=80=E5=BC=A0=E8=A1=A8=E7=9A=84=E5=87=A0?= =?UTF-8?q?=E6=9D=A1=E4=B8=80=E6=AC=A1=E5=86=99=E5=85=A5=EF=BC=8C=E4=B8=80?= =?UTF-8?q?=E5=BC=A0=E8=A1=A8=E5=86=B2=E7=AA=81=E4=B8=8D=E5=BD=B1=E5=93=8D?= =?UTF-8?q?=E5=88=AB=E7=9A=84=E8=A1=A8=EF=BC=9B=E8=A1=A8=E8=A2=AB=E5=93=AA?= =?UTF-8?q?=E4=BA=9B=E8=BF=90=E8=A1=8C=E7=94=A8=E8=BF=87=EF=BC=88=E6=96=B0?= =?UTF-8?q?=E7=9A=84=E5=9C=A8=E5=89=8D=EF=BC=8C=E5=8F=96=E6=B3=95=E5=90=8C?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=AC=A1=E6=95=B0=EF=BC=8C=E6=8A=BD=E5=87=BA?= =?UTF-8?q?=20query=5Fsnapshot=5Ftables=20=E5=85=B1=E7=94=A8=EF=BC=89?= =?UTF-8?q?=EF=BC=9B=E5=AE=A1=E9=98=85=E8=BF=9B=E5=BA=A6=EF=BC=88=E5=B8=B8?= =?UTF-8?q?=E7=94=A8=2020=20=E5=BC=A0=E8=A1=A8=E3=80=81=E5=85=A8=E9=83=A8?= =?UTF-8?q?=E3=80=81=E5=BD=B1=E5=93=8D=E6=95=B0=E5=AD=97=E7=9A=84=E9=A1=B9?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/catalog.py | 62 +++- backend/app/data/catalog.py | 42 ++- backend/app/data/catalog_review.py | 404 +++++++++++++++++++++ backend/tests/test_catalog_review_queue.py | 357 ++++++++++++++++++ 4 files changed, 850 insertions(+), 15 deletions(-) create mode 100644 backend/app/data/catalog_review.py create mode 100644 backend/tests/test_catalog_review_queue.py diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 0405a0f7..7680eb54 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -13,6 +13,9 @@ - POST /catalog/draft-jobs 建后台起草任务（202）；GET /draft-jobs/active、/latest、/{job_id} 读任务； POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py） - POST /catalog/profile 同步剖析：对业务库发少量只读查询，核对关系、取码值候选（data/catalog_profile.py） +- GET /catalog/review-queue 审阅队列：推断的项按使用次数、风险、来源排好；POST /catalog/review/batch 批量审阅， + GET /catalog/review-progress 审阅进度，GET /catalog/{table}/usage 用过这张表的运行 + （data/catalog_review.py） 规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径 不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。 @@ -24,7 +27,7 @@ from __future__ import annotations from typing import Any, Literal -from fastapi import APIRouter, Body, Depends, Header, HTTPException +from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query from fastapi.exceptions import RequestValidationError from pydantic import BaseModel, Field, ValidationError from sqlalchemy.ext.asyncio import AsyncSession @@ -35,6 +38,7 @@ from app.api.coded import ( ) from app.api.runs import actor_of from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, introspect, table_versions +from app.data import catalog_review from app.data.engine import SnapshotTampered from app.db.base import get_session from app.db.models import DataSource @@ -234,6 +238,30 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi } +# ---- 审阅队列、审阅进度（要登记在 GET /catalog/{table} 前面：不然 review-queue、review-progress 会被当成表名） + + +@router.get("/{source_id}/catalog/review-queue") +async def catalog_review_queue(source_id: str, + limit: int = Query(50, ge=1, le=catalog_review.QUEUE_LIMIT_MAX), + offset: int = Query(0, ge=0), + kind: Literal["all", "facts", "names"] = Query("all"), + table: str | None = Query(None, max_length=255), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """审阅队列：只列推断的项，按使用次数、风险、来源排好（data/catalog_review.py）。kind 筛影响数字的项（facts）或名称 + 和说明（names），table 只看一张表。""" + _, source = await _resolved(session, source_id) + return await catalog_review.review_queue(session, source, kind=kind, table=table or None, limit=limit, + offset=offset) + + +@router.get("/{source_id}/catalog/review-progress") +async def catalog_review_progress(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """审阅进度：常用 20 张表、全部、影响数字的项各自确认了多少。""" + _, source = await _resolved(session, source_id) + return await catalog_review.review_progress(session, source) + + @router.get("/{source_id}/catalog/{table}") async def get_catalog_table(source_id: str, table: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: @@ -527,6 +555,38 @@ async def review_catalog_item(source_id: str, table: str, payload: CatalogReview return await _detail(session, source, key, entry) +class CatalogBatchReviewItem(BaseModel): + table_name: str = Field(min_length=1, max_length=255) + path: str = Field(min_length=1, max_length=600) + action: Literal["confirm", "reject", "reset"] + if_version: int = Field(ge=0) + + +class CatalogBatchReviewIn(BaseModel): + items: list[CatalogBatchReviewItem] = Field(min_length=1, max_length=catalog_review.BATCH_MAX) + + +@router.post("/{source_id}/catalog/review/batch") +async def review_catalog_batch(source_id: str, payload: CatalogBatchReviewIn, + x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """批量审阅：逐条结果和请求一一对应。逐表按 if_version 乐观锁，同一张表的几条一次写入；一张表冲突、路径不对、表不存在 + 只记在那几条上（ok 为假，code 是 catalog_conflict / catalog_path_invalid / table_missing），不影响别的。""" + _, source = await _resolved(session, source_id) + items = [catalog_review.BatchItem(i.table_name, i.path, i.action, i.if_version) for i in payload.items] + return {"results": await catalog_review.review_batch(session, source, items, actor=actor_of(x_actor))} + + +@router.get("/{source_id}/catalog/{table}/usage") +async def catalog_table_runs(source_id: str, table: str, + limit: int = Query(20, ge=1, le=catalog_review.RUNS_LIMIT_MAX), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """用过这张表的运行（total 是几次运行），新的在前；取法同使用次数（catalog.table_usage）。""" + _, source = await _resolved(session, source_id) + key, _ = await _table_key(session, source, table) + return await catalog_review.table_runs(session, source, key, limit=limit) + + def _tables_of(source: Any) -> dict[str, Any] | None: """核对提案用的表结构；没有表结构（尚未探查）时不核对列名。""" return ((source.schema_cache or {}).get("tables") or None) if source is not None else None diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 3dd0bb39..cdef28cf 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -2249,27 +2249,27 @@ def query_snapshot_meta(source: Any, sql: str) -> dict[str, Any]: return {"source_id": getattr(source, "id", None), "source": getattr(source, "name", None), "tables": tables} -async def table_usage(session: AsyncSession, source: Any) -> dict[str, int]: - """每张表被运行查询过的次数：{schema_cache 的表名: 次数}，没被查过的表不出现。 +async def query_snapshot_tables(session: AsyncSession, source: Any) -> list[tuple[str, frozenset[str]]]: + """这个源上属于运行的每一行查询快照引用：(运行 id, 这条查询用到的表——schema_cache 的表名)，没对上任何表的不收。 - 取法：数据源查询工具每次成功查询都存一份查询快照工件（tools/datasource.py），工件引用表 artifacts 里 - 按 (内容哈希, 运行) 各记一行。数「属于某次运行（runs 里有这次运行）、属于这个源、SQL 里用到了这张表」的 - 行：同一次运行里结果一模一样的重复查询只算一次，工具库里的试用（不属于任何运行）不算。 + 使用次数（table_usage）和「这张表被哪些运行用过」（data/catalog_review.py）共用这一份取法：数据源查询工具每次成功 + 查询都存一份查询快照工件（tools/datasource.py），工件引用表 artifacts 里按 (内容哈希, 运行) 各记一行；只收属于某次运行 + （runs 里有这次运行）、属于这个源的行，工具库里的试用（不属于任何运行）不算。 - 表名在查询当时就记进了引用的 meta（query_snapshot_meta），这里只读 artifacts 一张表；升级前的老引用 - 没有 meta，退回读工件内容里的源名和 SQL，读过的按工件 id 缓存（内容永不改变）。SQL 里写的表名去掉 - schema 前缀后按 name_key（只把 ASCII 字母转小写，和 SQLite 比较标识符的口径一致）对到表结构上，对不上的不计。 + 表名在查询当时就记进了引用的 meta（query_snapshot_meta），这里只读 artifacts 一张表；升级前的老引用没有 meta，退回读 + 工件内容里的源名和 SQL，读过的按工件 id 缓存（内容永不改变）。SQL 里写的表名去掉 schema 前缀后按 name_key（只把 + ASCII 字母转小写，和 SQLite 比较标识符的口径一致）对到表结构上，对不上的不计。 """ rows = (await session.execute( - select(Artifact.id, Artifact.meta).join(Run, Run.id == Artifact.run_id) + select(Artifact.id, Artifact.meta, Artifact.run_id).join(Run, Run.id == Artifact.run_id) .where(Artifact.kind == QUERY_SNAPSHOT_KIND) )).all() tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} lookup: dict[str, str] = {} for name in tables: lookup.setdefault(name_key(name), name) - counts: Counter[str] = Counter() - for artifact_id, meta in rows: + out: list[tuple[str, frozenset[str]]] = [] + for artifact_id, meta, run_id in rows: meta = meta if isinstance(meta, dict) else {} if isinstance(meta.get("tables"), list): source_id, source_name, names = meta.get("source_id"), meta.get("source"), meta["tables"] @@ -2281,8 +2281,21 @@ async def table_usage(session: AsyncSession, source: Any) -> dict[str, int]: continue elif source_name != source.name: continue - hit = {lookup.get(name_key(str(n).rsplit(".", 1)[-1])) for n in names} - counts.update(t for t in hit if t is not None) + hit = frozenset(t for t in (lookup.get(name_key(str(n).rsplit(".", 1)[-1])) for n in names) if t is not None) + if hit: + out.append((str(run_id), hit)) + return out + + +async def table_usage(session: AsyncSession, source: Any) -> dict[str, int]: + """每张表被运行查询过的次数：{schema_cache 的表名: 次数}，没被查过的表不出现。 + + 按查询快照引用的行数（query_snapshot_tables）：同一次运行里结果一模一样的重复查询只算一次，同一次运行里不同的查询 + 各算一次，工具库里的试用不算。 + """ + counts: Counter[str] = Counter() + for _run, hit in await query_snapshot_tables(session, source): + counts.update(hit) return dict(counts) @@ -2337,7 +2350,8 @@ __all__ = [ "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "is_date_column", "is_profile_note", "join_paths", "make_item", "merge_notes", "name_words", "now_iso", "parse_path", "plan_patch", "profile_relation_holds", - "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", "render_table_index", + "query_snapshot_meta", "query_snapshot_tables", "read_catalog", "read_entry", "relation_graph", "relation_id", + "render_table_index", "render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry", "review_item", "same_notes", "same_profile_note", "save_human_edit", "save_patch", "status_counts", "structure_relation_ids", "system_notes_source", "table_usage", "unreviewed_status", "validate_notes", "visible_notes", "write_entry", diff --git a/backend/app/data/catalog_review.py b/backend/app/data/catalog_review.py new file mode 100644 index 00000000..b03c9e31 --- /dev/null +++ b/backend/app/data/catalog_review.py @@ -0,0 +1,404 @@ +"""数据目录的审阅队列、批量审阅、表被哪些运行用过、审阅进度。 + +**为什么要有它。** 起草（数据库注释、外键、命名推断、数据剖析、模型）一次写出成百上千项，绝大多数停在「推断」：SQL 检查的 +错误级规则只认已确认、有确证的项，推断的项只作提示。逐表点开确认太慢，也看不出先确认哪几项。队列把推断的项摊平，按 +「这张表常不常用 × 这一项错了影响多大 × 这一项从哪来」排好，最该先看的在前面；勾几项一次确认、驳回或撤销审阅。 + +- **队列**（review_queue，GET /catalog/review-queue）：只列推断（proposed）的项。priority = 使用次数权重 × 风险权重 × + 来源权重，从高到低；同分按表名、路径排，结果稳定。 + - 使用次数权重：1 + log2(1 + 使用次数)。使用次数同目录清单（catalog.table_usage）：按对数增长，常用表在前，但不至于 + 让常用表上的中文名压过所有表上影响数字的项。 + - 风险权重（RISK_WEIGHT）：有效记录条件、去重规则、业务日期、关联关系最高（直接改变数字），业务主键、度量类型次之 + （计数和能不能加总），码值、粒度、表类型再次，中文名、说明、含义、单位最低。 + - 来源权重（SOURCE_WEIGHT）：模型起草、命名推断是猜的，比数据库注释优先；数据剖析（码值候选、业务日期、没核实的关系） + 居中。 + - 两组（kind）：facts 影响数字的项（有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型）；names 名称和说明 + （中文名、说明、含义、单位、码值、粒度、表类型）。表的「说明」和中文名、含义一样只是说明文字，归 names。 + - 表结构里已经没有的表、列上的项不列（确认了也没有用，目录页上另有清理的提示）。 + - reasons 用中文说为什么排在这里：常用表（使用 N 次）、影响数字的项（有效记录条件）、模型起草，未确认…… +- **批量审阅**（review_batch，POST /catalog/review/batch）：逐表按 if_version 乐观锁，同一张表的几条在一次写入里完成， + 一张表冲突、写不进去不影响别的表；路径不对、表不存在逐条报（results 和请求一一对应）。审阅的规则同单项审阅 + （catalog.review_item）。 +- **被哪些运行用过**（table_runs，GET /catalog/{table}/usage）：取法同使用次数（catalog.query_snapshot_tables），按运行 + 去重，新的在前。 +- **进度**（review_progress，GET /catalog/review-progress）：使用次数前 TOP_N 张表（只算被运行用到过的）、全部、影响 + 数字的项各自的确认情况。项的范围和队列一样：表结构里没有的表、列不算。 +""" +from __future__ import annotations + +import math +from collections import Counter +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import catalog +from app.db.models import Run + +#: 队列的筛选：全部、影响数字的项、名称和说明 +KINDS = ("all", "facts", "names") +#: 队列一页最多几项；批量审阅一次最多几条；「被哪些运行用过」一次最多几次运行 +QUEUE_LIMIT_MAX = 200 +BATCH_MAX = 200 +RUNS_LIMIT_MAX = 100 +#: 进度里的「常用表」取使用次数前几张 +TOP_N = 20 + +#: 每一项属于哪一组、风险几档：键是 (t, 字段) / (c, 字段) / (r, "")，档 4 最高 +FIELD_GROUP: dict[tuple[str, str], tuple[str, int]] = { + ("t", "valid_filter"): ("facts", 4), ("t", "dedup"): ("facts", 4), ("t", "business_date"): ("facts", 4), + ("r", ""): ("facts", 4), + ("t", "keys"): ("facts", 3), ("c", "measure"): ("facts", 3), + ("c", "codes"): ("names", 2), ("t", "grain"): ("names", 2), ("t", "kind"): ("names", 2), + ("t", "label"): ("names", 1), ("t", "description"): ("names", 1), ("c", "label"): ("names", 1), + ("c", "meaning"): ("names", 1), ("c", "unit"): ("names", 1), +} +RISK_WEIGHT = {4: 4.0, 3: 3.0, 2: 2.0, 1: 1.0} +#: 来源权重：没写的（外键约束、人工填写，它们本来就不是推断）按 1 +SOURCE_WEIGHT = {"llm": 1.5, "name": 1.5, "profile": 1.25, "comment": 1.0} + +#: 来源的界面叫法（和 frontend/src/lib/terms.ts 的 CATALOG_SOURCE_LABEL 一致，有测试核对） +SOURCE_LABEL = {"comment": "数据库注释", "fk": "外键约束", "name": "命名推断", "profile": "数据剖析", "llm": "模型起草", + "human": "人工填写"} +RELATION_LABEL = "关联关系" +#: reasons 的说法 +REASON_TOP = "常用表（使用 {n} 次）" +REASON_USED = "使用 {n} 次" +REASON_RISK = {4: "影响数字的项（{label}）", 3: "影响计数和汇总的项（{label}）", 2: "帮助理解取值的项（{label}）", + 1: "名称和说明（{label}）"} +REASON_PENDING_CODES = "{n} 个码值的含义待填写" +REASON_SOURCE = "{source}，未确认" +#: 码值含义为空（数据剖析写进来的候选）时的说法，同界面的 CODES_TEXT.pending +CODE_PENDING = "含义待填写" + +#: 批量审阅逐条结果的机读码 +CODE_CONFLICT = "catalog_conflict" +CODE_PATH = "catalog_path_invalid" +CODE_INVALID = "catalog_invalid" +CODE_TABLE_MISSING = "table_missing" + +Slot = catalog.Slot + + +# ========================================================================== +# 一项的路径、叫法、显示 +# ========================================================================== + + +def slot_path(key: Slot) -> str: + """槽位 → 审阅接口的路径写法（catalog.parse_path 的反过来）。""" + if key[0] == "t": + return key[1] + if key[0] == "c": + return f"columns.{key[1]}.{key[2]}" + return f"relations.{key[1]}" + + +def field_of(key: Slot) -> tuple[str, str]: + """槽位 → FIELD_GROUP 的键。""" + if key[0] == "r": + return ("r", "") + return (key[0], key[-1]) + + +def field_name(key: Slot) -> str: + """这一项的字段名：表级、列级的项是字段名，关联关系是 relation。""" + return "relation" if key[0] == "r" else key[-1] + + +def field_label(key: Slot) -> str: + """字段的界面叫法：中文名、码值、有效记录条件、关联关系……""" + if key[0] == "t": + return catalog.TABLE_FIELD_LABEL.get(key[1], key[1]) + if key[0] == "c": + return catalog.COLUMN_FIELD_LABEL.get(key[2], key[2]) + return RELATION_LABEL + + +def item_value(key: Slot, item: Mapping[str, Any]) -> Any: + """一项的值：关联关系是两端、基数和覆盖率，别的是 value。""" + if key[0] == "r": + return {k: item.get(k) for k in ("columns", "to_table", "to_columns", "cardinality", "coverage")} + return item.get("value") + + +def _percent(ratio: float) -> str: + """覆盖率写成百分数：整数不带小数，其余留一位；不到 100% 的不写成 100%。""" + text = f"{ratio * 100:.1f}".rstrip("0").rstrip(".") + return "99.9%" if text == "100" and ratio < 1 else f"{text}%" + + +def relation_text(rel: Mapping[str, Any]) -> str: + """关联关系写成一行：member_id → members.id（多对一，覆盖率 98%）。""" + left = "、".join(str(c) for c in rel.get("columns") or []) + right = "、".join(f"{rel.get('to_table')}.{c}" for c in rel.get("to_columns") or []) + extra = [] + if rel.get("cardinality") in catalog.CARDINALITY_LABEL: + extra.append(catalog.CARDINALITY_LABEL[rel["cardinality"]]) + coverage = rel.get("coverage") + if isinstance(coverage, (int, float)) and not isinstance(coverage, bool): + extra.append(f"覆盖率 {_percent(float(coverage))}") + return f"{left} → {right}" + (f"（{'，'.join(extra)}）" if extra else "") + + +def display(key: Slot, value: Any) -> str: + """一项的值写成一行字，用界面上的说法（明细表、可累加、多对一……）。""" + if key[0] == "r": + return relation_text(value if isinstance(value, Mapping) else {}) + field_ = key[-1] + if field_ == "kind": + return catalog.UI_KIND_LABEL.get(value, str(value)) + if field_ == "measure": + return catalog.UI_MEASURE_LABEL.get(value, str(value)) + if field_ == "keys" and isinstance(value, list): + return "、".join(str(v) for v in value) + if field_ == "codes" and isinstance(value, Mapping): + return "、".join(f"{k}={v if str(v).strip() else CODE_PENDING}" for k, v in value.items()) + if field_ == "business_date" and isinstance(value, Mapping): + parts = [str(value.get("column") or "")] + if value.get("rule"): + parts.append(f"规则：{value['rule']}") + if value.get("timezone"): + parts.append(f"时区：{value['timezone']}") + return "；".join(parts) + return value if isinstance(value, str) else str(value) + + +# ========================================================================== +# 排序 +# ========================================================================== + + +def usage_weight(usage: int) -> float: + return 1.0 + math.log2(1 + max(0, int(usage or 0))) + + +def priority(usage: int, key: Slot, source: Any) -> float: + """一项的优先级：使用次数权重 × 风险权重 × 来源权重（不取整；排序和交给界面时取四位小数）。""" + _, tier = FIELD_GROUP[field_of(key)] + return usage_weight(usage) * RISK_WEIGHT[tier] * SOURCE_WEIGHT.get(str(source), 1.0) + + +def top_tables(tables: Mapping[str, Any], usage: Mapping[str, int]) -> list[str]: + """使用次数前 TOP_N 张表：只算被运行用到过的，同样多的按表结构里的顺序。""" + order = {name: i for i, name in enumerate(tables)} + used = [name for name in tables if usage.get(name, 0) > 0] + return sorted(used, key=lambda n: (-usage[n], order[n]))[:TOP_N] + + +def _reasons(key: Slot, item: Mapping[str, Any], usage: int, top: bool) -> list[str]: + out: list[str] = [] + if usage > 0: + out.append((REASON_TOP if top else REASON_USED).format(n=f"{usage:,}")) + _, tier = FIELD_GROUP[field_of(key)] + out.append(REASON_RISK[tier].format(label=field_label(key))) + if key[0] == "c" and key[2] == "codes" and isinstance(item.get("value"), Mapping): + pending = sum(1 for v in item["value"].values() if not str(v).strip()) + if pending: + out.append(REASON_PENDING_CODES.format(n=pending)) + source = str(item.get("source") or "") + out.append(REASON_SOURCE.format(source=SOURCE_LABEL.get(source, source))) + return out + + +def live_slots(notes: Mapping[str, Any] | None, meta: Mapping[str, Any] | None) -> list[tuple[Slot, dict[str, Any]]]: + """一张表目录里还算数的项：表结构里已经没有的列上的项不算（不知道表结构时都算）。""" + columns = catalog._column_names(meta) + return [(key, item) for key, item in catalog._slots(notes).items() + if not (key[0] == "c" and columns is not None and key[1] not in columns)] + + +# ========================================================================== +# 队列 +# ========================================================================== + + +async def review_queue(session: AsyncSession, source: Any, *, kind: str = "all", table: str | None = None, + limit: int = 50, offset: int = 0) -> dict[str, Any]: + """审阅队列：{total, items}。source 是读表结构用的源（上传源是绑定当前快照的视图）。""" + cache = getattr(source, "schema_cache", None) or {} + tables = cache.get("tables") or {} + wanted = None + if table: + wanted = catalog.resolve_table_name(cache, table) or table + entries = await catalog.read_catalog(session, source.id) + usage = await catalog.table_usage(session, source) + top = set(top_tables(tables, usage)) + items: list[dict[str, Any]] = [] + for name, entry in entries.items(): + meta = tables.get(name) + if meta is None or (wanted is not None and name != wanted): + continue + used = usage.get(name, 0) + for key, item in live_slots(entry.notes, meta): + if item.get("status") != "proposed": + continue + group, _ = FIELD_GROUP[field_of(key)] + if kind != "all" and group != kind: + continue + value = item_value(key, item) + items.append({ + "table_name": name, + "path": slot_path(key), + "field": field_name(key), + "field_label": field_label(key), + # 列级的项是哪一列（表级的项、关联关系为 null） + "column": key[1] if key[0] == "c" else None, + "value": value, + "display": display(key, value), + "source": item.get("source"), + "status": item.get("status"), + "note": item.get("note") if isinstance(item.get("note"), str) and item.get("note") else None, + "priority": round(priority(used, key, item.get("source")), 4), + "reasons": _reasons(key, item, used, name in top), + "table_usage": used, + "version": entry.version, + }) + items.sort(key=lambda i: (-i["priority"], i["table_name"], i["path"])) + return {"total": len(items), "items": items[offset:offset + limit]} + + +# ========================================================================== +# 批量审阅 +# ========================================================================== + + +@dataclass(frozen=True) +class BatchItem: + table_name: str + path: str + action: str + if_version: int + + +def _result(item: BatchItem, *, ok: bool, version: int | None = None, error: str | None = None, + code: str | None = None) -> dict[str, Any]: + return {"table_name": item.table_name, "path": item.path, "ok": ok, "version": version if ok else None, + "error": error, "code": code} + + +async def review_batch(session: AsyncSession, source: Any, items: Iterable[BatchItem], *, + actor: str | None) -> list[dict[str, Any]]: + """批量审阅，返回逐条结果（和请求一一对应）。 + + 表名按表结构认（全名、大小写不一致都认）；表结构里没有、但还留着目录的也认（同单项审阅）。同一张表的几条按请求里的 + 顺序在当前目录上依次审阅，一次写入：版本不符（if_version 和库里的不一样）、路径不对的那几条不做，其余照做；写入时撞上 + 别人刚改过，这张表的几条都记冲突。""" + items = list(items) + cache = getattr(source, "schema_cache", None) or {} + results: list[dict[str, Any] | None] = [None] * len(items) + groups: dict[str, list[int]] = {} + known: dict[str, str | None] = {} + for i, item in enumerate(items): + if item.table_name not in known: + key = catalog.resolve_table_name(cache, item.table_name) + if key is None and await catalog.read_entry(session, source.id, item.table_name) is not None: + key = item.table_name + known[item.table_name] = key + key = known[item.table_name] + if key is None: + results[i] = _result(item, ok=False, code=CODE_TABLE_MISSING, + error=f"数据源中没有表 {item.table_name}，可能已被删除或尚未探查结构") + continue + groups.setdefault(key, []).append(i) + at = catalog.now_iso() + for key, indexes in groups.items(): + entry = await catalog.read_entry(session, source.id, key) + current = entry.version if entry else 0 + notes: dict[str, Any] = entry.notes if entry else {} + applied: list[int] = [] + for i in indexes: + item = items[i] + if item.if_version != current: + results[i] = _result(item, ok=False, code=CODE_CONFLICT, error=str(catalog.CatalogConflict(key, current))) + continue + try: + notes = catalog.review_item(notes, item.path, item.action, at=at) + except catalog.CatalogPathError as e: + results[i] = _result(item, ok=False, code=CODE_PATH, error=str(e)) + continue + applied.append(i) + if not applied: + continue + try: + written = await catalog.write_entry(session, source.id, key, notes, if_version=current, actor=actor) + except catalog.CatalogConflict as e: + for i in applied: + results[i] = _result(items[i], ok=False, code=CODE_CONFLICT, error=str(e)) + continue + except catalog.CatalogInvalid as e: + for i in applied: + results[i] = _result(items[i], ok=False, code=CODE_INVALID, error=str(e)) + continue + for i in applied: + results[i] = _result(items[i], ok=True, version=written.version) + return [r for r in results if r is not None] + + +# ========================================================================== +# 被哪些运行用过 +# ========================================================================== + + +async def table_runs(session: AsyncSession, source: Any, table: str, *, limit: int = 20) -> dict[str, Any]: + """用过这张表的运行：{total, runs}，新的在前（按开始时间，没开始的按建立时间）。取法同使用次数。""" + run_ids = {run for run, hit in await catalog.query_snapshot_tables(session, source) if table in hit} + if not run_ids: + return {"total": 0, "runs": []} + started = func.coalesce(Run.started_at, Run.created_at) + rows = (await session.execute( + select(Run.id, Run.workflow_id, Run.workflow_name, Run.run_class, Run.status, Run.started_at) + .where(Run.id.in_(run_ids)).order_by(started.desc(), Run.id).limit(limit) + )).all() + return { + "total": len(run_ids), + "runs": [{"run_id": r.id, "workflow_id": r.workflow_id, "workflow_name": r.workflow_name or "", + "run_class": r.run_class, "status": r.status, + "started_at": r.started_at.isoformat() if r.started_at else None} for r in rows], + } + + +# ========================================================================== +# 进度 +# ========================================================================== + + +async def review_progress(session: AsyncSession, source: Any) -> dict[str, Any]: + """审阅进度：常用 TOP_N 张表（只算被运行用到过的）、全部、影响数字的项。项按状态计数；verified（有确证）另给一项。""" + tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} + entries = await catalog.read_catalog(session, source.id) + usage = await catalog.table_usage(session, source) + top = set(top_tables(tables, usage)) + every: Counter[str] = Counter() + head: Counter[str] = Counter() + facts: Counter[str] = Counter() + for name, entry in entries.items(): + meta = tables.get(name) + if meta is None: + continue + for key, item in live_slots(entry.notes, meta): + status = str(item.get("status")) + every[status] += 1 + if name in top: + head[status] += 1 + if FIELD_GROUP[field_of(key)][0] == "facts": + facts[status] += 1 + return { + "top_n": TOP_N, + "top": {"tables": len(top), "items_total": sum(head.values()), "items_confirmed": head["confirmed"], + "items_proposed": head["proposed"], "items_verified": head["verified"]}, + "all": {"items_total": sum(every.values()), "items_confirmed": every["confirmed"], + "items_proposed": every["proposed"], "items_rejected": every["rejected"], + "items_verified": every["verified"]}, + "facts": {"proposed": facts["proposed"], "confirmed": facts["confirmed"]}, + } + + +__all__ = ["BATCH_MAX", "BatchItem", "CODE_CONFLICT", "CODE_INVALID", "CODE_PATH", "CODE_TABLE_MISSING", + "FIELD_GROUP", "KINDS", "QUEUE_LIMIT_MAX", "RISK_WEIGHT", "RUNS_LIMIT_MAX", "SOURCE_LABEL", "SOURCE_WEIGHT", + "TOP_N", "display", "field_label", "field_name", "field_of", "item_value", "live_slots", "priority", + "relation_text", "review_batch", "review_progress", "review_queue", "slot_path", "table_runs", "top_tables", + "usage_weight"] diff --git a/backend/tests/test_catalog_review_queue.py b/backend/tests/test_catalog_review_queue.py new file mode 100644 index 00000000..1598103c --- /dev/null +++ b/backend/tests/test_catalog_review_queue.py @@ -0,0 +1,357 @@ +"""数据目录的审阅队列、批量审阅、表被哪些运行用过、审阅进度（data/catalog_review.py）。 + +- GET /catalog/review-queue：只列推断（proposed）的项，按「使用次数权重 × 风险权重 × 来源权重」从高到低，同分按表名、 + 路径；facts（有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型）/ names（中文名、说明、含义、单位、码值、 + 粒度、表类型）两组；按表筛选；分页；reasons 写为什么排在这里；表结构里已经没有的表、列不列 +- POST /catalog/review/batch：逐表按 if_version 乐观锁，一张表冲突不影响别的表；同一张表的几条一次写入；路径不对、表不存在 + 逐条报 +- GET /catalog/{table}/usage：用过这张表的运行，新的在前，取法同 catalog.table_usage +- GET /catalog/review-progress：常用 20 张表、全部、影响数字的项各自的确认情况 + +表名、列名、业务数字都是虚构的。 +""" +from __future__ import annotations + +import re +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from urllib.parse import quote + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.data import catalog, catalog_review +from app.data.catalog import make_item +from app.db.base import SessionLocal +from app.db.models import Artifact, DataSource, Run +from app.main import app as _fastapi_app +from tests.fixtures.sources import drop_source + +ACTOR = {"X-Actor": quote("林晓")} +TERMS = Path(__file__).resolve().parents[2] / "frontend" / "src" / "lib" / "terms.ts" + + +def _col(name: str, type_: str = "INTEGER") -> dict: + return {"name": name, "type": type_, "nullable": True, "comment": None} + + +def _tables() -> dict: + return { + "orders": {"qualified": "orders", "primary_key": ["id"], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], + "columns": [_col("id"), _col("status"), _col("amount", "REAL"), _col("member_id"), + _col("paid_at", "TEXT")]}, + "members": {"qualified": "members", "primary_key": ["id"], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], + "columns": [_col("id"), _col("nick", "TEXT"), _col("tier")]}, + "coupons": {"qualified": "coupons", "primary_key": ["id"], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], "columns": [_col("id"), _col("face_value", "REAL")]}, + } + + +def _member_rel(status: str = "proposed", source: str = "name") -> dict: + return {"id": catalog.relation_id("orders", ["member_id"], "members", ["id"]), "columns": ["member_id"], + "to_table": "members", "to_columns": ["id"], "cardinality": "many_to_one", "coverage": 0.98, + "source": source, "status": status} + + +ORDERS = { + "label": make_item("订单", "llm"), + "grain": make_item("每笔订单一行", "llm", "confirmed"), + "valid_filter": make_item("status <> 9", "llm", note="剖析核对：排除 3.2% 的行（共 12,345 行）"), + "kind": make_item("fact", "llm"), + "columns": { + "status": {"label": make_item("状态", "comment"), "codes": make_item({"1": "已支付", "9": ""}, "profile")}, + "amount": {"measure": make_item("flow", "llm"), "unit": make_item("元", "llm", "rejected")}, + # 表结构里已经没有这一列：不进队列 + "legacy_flag": {"label": make_item("旧标记", "llm")}, + }, + "relations": [_member_rel()], +} +MEMBERS = {"label": make_item("会员", "llm"), "keys": make_item(["id"], "name"), + "columns": {"nick": {"label": make_item("昵称", "comment")}}} +COUPONS = {"label": make_item("优惠券", "comment")} +#: 表结构里已经没有这张表：不进队列 +GONE = {"label": make_item("旧表", "llm")} + +_CREATED: list[str] = [] + + +@pytest.fixture(autouse=True) +async def _cleanup(): + yield + while _CREATED: + await drop_source(_CREATED.pop()) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c: + yield c + + +async def _write(sid: str, table: str, notes: dict) -> None: + async with SessionLocal() as session: + entry = await catalog.read_entry(session, sid, table) + await catalog.write_entry(session, sid, table, notes, if_version=entry.version if entry else 0, actor=None) + + +async def _use(sid: str, name: str, table_runs: dict[str, int]) -> list[str]: + """给每张表造几次运行查询它（每次运行一份查询快照），返回运行 id（先建的在前、开始得早）。""" + runs: list[str] = [] + base = datetime(2026, 9, 1, tzinfo=timezone.utc) + async with SessionLocal() as session: + for table, n in table_runs.items(): + for _ in range(n): + rid = uuid.uuid4().hex + session.add(Run(id=rid, graph={}, workflow_name=f"周报-{table}", run_class="formal", + status="succeeded", started_at=base + timedelta(hours=len(runs)))) + session.add(Artifact(id=uuid.uuid4().hex, run_id=rid, kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": [table]})) + runs.append(rid) + await session.commit() + return runs + + +async def _source(*, usage: dict[str, int] | None = None) -> str: + name = f"rq_{uuid.uuid4().hex[:10]}" + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=":memory:", schema_cache={"tables": _tables()}) + session.add(row) + await session.commit() + sid = row.id + _CREATED.append(sid) + for table, notes in (("orders", ORDERS), ("members", MEMBERS), ("coupons", COUPONS), ("gone_table", GONE)): + await _write(sid, table, notes) + if usage: + await _use(sid, name, usage) + return sid + + +def _base(sid: str) -> str: + return f"/api/datasources/{sid}/catalog" + + +# -------------------------------------------------------------------------- +# 排序规则（纯函数） +# -------------------------------------------------------------------------- + + +def test_priority_multiplies_usage_risk_and_source_weights(): + p = catalog_review.priority + # 风险：有效记录条件 > 业务主键 > 码值 > 中文名 + assert p(0, ("t", "valid_filter"), "llm") > p(0, ("t", "keys"), "llm") > p(0, ("c", "x", "codes"), "llm") \ + > p(0, ("c", "x", "label"), "llm") + assert p(0, ("r", "r1"), "name") == p(0, ("t", "dedup"), "name") == p(0, ("t", "business_date"), "name") + assert p(0, ("c", "x", "measure"), "llm") == p(0, ("t", "keys"), "llm") + assert p(0, ("t", "grain"), "llm") == p(0, ("t", "kind"), "llm") == p(0, ("c", "x", "codes"), "llm") + assert p(0, ("c", "x", "meaning"), "llm") == p(0, ("c", "x", "unit"), "llm") == p(0, ("t", "label"), "llm") + # 来源：模型起草、命名推断比数据库注释优先 + assert p(0, ("t", "label"), "llm") == p(0, ("t", "label"), "name") > p(0, ("t", "label"), "comment") + # 使用次数：用得多的在前，但按对数增长，不压过风险 + assert p(37, ("t", "label"), "llm") > p(1, ("t", "label"), "llm") > p(0, ("t", "label"), "llm") + assert p(37, ("t", "label"), "llm") == pytest.approx( + p(0, ("t", "label"), "llm") * p(37, ("t", "label"), "comment") / p(0, ("t", "label"), "comment")) + + +def test_source_labels_follow_the_ui(): + text = TERMS.read_text(encoding="utf-8") + block = re.search(r"export const CATALOG_SOURCE_LABEL\b[^=]*=\s*\{(.*?)\n\}", text, re.S) + assert block + ui = dict(re.findall(r"(\w+):\s*'([^']+)'", block.group(1))) + assert ui == catalog_review.SOURCE_LABEL + + +# -------------------------------------------------------------------------- +# 队列 +# -------------------------------------------------------------------------- + + +async def test_queue_lists_only_proposed_items_by_priority(client): + sid = await _source(usage={"orders": 37, "members": 2}) + r = await client.get(f"{_base(sid)}/review-queue") + assert r.status_code == 200, r.text + body = r.json() + items = body["items"] + paths = [(i["table_name"], i["path"]) for i in items] + # 已确认、已驳回的不列；表结构里没有的表、列不列 + assert ("orders", "grain") not in paths and ("orders", "columns.amount.unit") not in paths + assert not any(t == "gone_table" for t, _ in paths) + assert ("orders", "columns.legacy_flag.label") not in paths + assert body["total"] == len(items) == 11 + assert all(i["status"] == "proposed" for i in items) + # 从高到低，同分按表名、路径 + keys = [(-i["priority"], i["table_name"], i["path"]) for i in items] + assert keys == sorted(keys) + first = items[0] + rel_id = _member_rel()["id"] + assert (first["table_name"], first["path"]) in {("orders", "valid_filter"), ("orders", f"relations.{rel_id}")} + vf = next(i for i in items if i["path"] == "valid_filter") + assert vf["field"] == "valid_filter" and vf["field_label"] == "有效记录条件" + assert vf["value"] == "status <> 9" and vf["display"] == "status <> 9" + assert vf["source"] == "llm" and vf["note"] == "剖析核对：排除 3.2% 的行（共 12,345 行）" + assert vf["table_usage"] == 37 and vf["version"] == 1 + assert vf["reasons"] == ["常用表（使用 37 次）", "影响数字的项（有效记录条件）", "模型起草，未确认"] + # 常用表上的中文名排在没人用的表上的中文名前面；同一张表上影响数字的项排在中文名前面 + order = [(i["table_name"], i["path"]) for i in items] + assert order.index(("orders", "label")) < order.index(("coupons", "label")) + assert order.index(("members", "keys")) < order.index(("members", "label")) + + +async def test_queue_values_are_displayed_in_ui_words(client): + sid = await _source() + items = {(i["table_name"], i["path"]): i for i in (await client.get(f"{_base(sid)}/review-queue")).json()["items"]} + rel = items[("orders", f"relations.{_member_rel()['id']}")] + assert rel["field"] == "relation" and rel["field_label"] == "关联关系" + assert rel["value"] == {"columns": ["member_id"], "to_table": "members", "to_columns": ["id"], + "cardinality": "many_to_one", "coverage": 0.98} + assert rel["display"] == "member_id → members.id（多对一，覆盖率 98%）" + assert rel["reasons"][-1] == "命名推断，未确认" + kind = items[("orders", "kind")] + assert kind["display"] == "明细表" and kind["value"] == "fact" + codes = items[("orders", "columns.status.codes")] + assert codes["field"] == "codes" and codes["field_label"] == "码值" and codes["column"] == "status" + assert codes["display"] == "1=已支付、9=含义待填写" + assert "1 个码值的含义待填写" in codes["reasons"] and "数据剖析，未确认" in codes["reasons"] + assert items[("orders", "columns.amount.measure")]["display"] == "可累加" + assert items[("members", "keys")]["display"] == "id" + assert items[("members", "label")]["reasons"] == ["名称和说明（中文名）", "模型起草，未确认"] + assert items[("coupons", "label")]["reasons"] == ["名称和说明（中文名）", "数据库注释，未确认"] + + +async def test_queue_filters_by_kind_and_table_and_pages(client): + sid = await _source(usage={"orders": 3}) + facts = (await client.get(f"{_base(sid)}/review-queue", params={"kind": "facts"})).json() + assert {i["field"] for i in facts["items"]} == {"valid_filter", "relation", "measure", "keys"} + names = (await client.get(f"{_base(sid)}/review-queue", params={"kind": "names"})).json() + assert {i["field"] for i in names["items"]} == {"label", "codes", "kind"} + assert facts["total"] + names["total"] == 11 + only = (await client.get(f"{_base(sid)}/review-queue", params={"table": "MEMBERS"})).json() + assert {i["table_name"] for i in only["items"]} == {"members"} and only["total"] == 3 + everything = (await client.get(f"{_base(sid)}/review-queue")).json()["items"] + page = (await client.get(f"{_base(sid)}/review-queue", params={"limit": 4, "offset": 4})).json() + assert page["total"] == 11 and page["items"] == everything[4:8] + bad = await client.get(f"{_base(sid)}/review-queue", params={"kind": "other"}) + assert bad.status_code == 422 + assert (await client.get(f"{_base(sid)}/review-queue", params={"limit": 0})).status_code == 422 + missing = await client.get("/api/datasources/nope/catalog/review-queue") + assert missing.status_code == 404 + + +# -------------------------------------------------------------------------- +# 批量审阅 +# -------------------------------------------------------------------------- + + +async def test_batch_review_writes_each_table_once_and_isolates_conflicts(client): + sid = await _source() + rel_path = f"relations.{_member_rel()['id']}" + body = {"items": [ + {"table_name": "orders", "path": "valid_filter", "action": "confirm", "if_version": 1}, + {"table_name": "orders", "path": rel_path, "action": "reject", "if_version": 1}, + {"table_name": "orders", "path": "columns.nothing.label", "action": "confirm", "if_version": 1}, + {"table_name": "members", "path": "label", "action": "confirm", "if_version": 7}, # 版本不对 + {"table_name": "coupons", "path": "label", "action": "reject", "if_version": 1}, + {"table_name": "ghost", "path": "label", "action": "confirm", "if_version": 0}, + ]} + r = await client.post(f"{_base(sid)}/review/batch", json=body, headers=ACTOR) + assert r.status_code == 200, r.text + results = r.json()["results"] + assert [(x["table_name"], x["path"], x["ok"]) for x in results] == [ + ("orders", "valid_filter", True), ("orders", rel_path, True), ("orders", "columns.nothing.label", False), + ("members", "label", False), ("coupons", "label", True), ("ghost", "label", False)] + # 同一张表的两条一次写进去：版本只加 1 + assert results[0]["version"] == results[1]["version"] == 2 + assert results[2]["code"] == "catalog_path_invalid" and "找不到" in results[2]["error"] + assert results[2]["version"] is None + assert results[3]["code"] == "catalog_conflict" and "重新载入" in results[3]["error"] + assert results[4]["version"] == 2 and results[4]["error"] is None and results[4]["code"] is None + assert results[5]["code"] == "table_missing" + async with SessionLocal() as session: + orders = await catalog.read_entry(session, sid, "orders") + members = await catalog.read_entry(session, sid, "members") + coupons = await catalog.read_entry(session, sid, "coupons") + assert orders.version == 2 and orders.updated_by == "林晓" + assert orders.notes["valid_filter"]["status"] == "confirmed" + assert orders.notes["relations"][0]["status"] == "rejected" + assert members.version == 1 and members.notes["label"]["status"] == "proposed" + assert coupons.notes["label"]["status"] == "rejected" + # 撤销审阅：回到来源的初始状态 + again = await client.post(f"{_base(sid)}/review/batch", json={"items": [ + {"table_name": "orders", "path": "valid_filter", "action": "reset", "if_version": 2}]}) + assert again.json()["results"][0]["ok"] is True + async with SessionLocal() as session: + assert (await catalog.read_entry(session, sid, "orders")).notes["valid_filter"]["status"] == "proposed" + + +async def test_batch_review_limits(client): + sid = await _source() + many = [{"table_name": "orders", "path": "label", "action": "confirm", "if_version": 1}] * 201 + assert (await client.post(f"{_base(sid)}/review/batch", json={"items": many})).status_code == 422 + assert (await client.post(f"{_base(sid)}/review/batch", json={"items": []})).status_code == 422 + wrong = [{"table_name": "orders", "path": "label", "action": "approve", "if_version": 1}] + assert (await client.post(f"{_base(sid)}/review/batch", json={"items": wrong})).status_code == 422 + ok = [{"table_name": "orders", "path": "label", "action": "confirm", "if_version": 1}] * 200 + r = await client.post(f"{_base(sid)}/review/batch", json={"items": ok}) + assert r.status_code == 200 and all(x["ok"] for x in r.json()["results"]) + + +# -------------------------------------------------------------------------- +# 被哪些运行用过 +# -------------------------------------------------------------------------- + + +async def test_table_usage_lists_runs_newest_first(client): + sid = await _source() + async with SessionLocal() as session: + name = (await session.get(DataSource, sid)).name + runs = await _use(sid, name, {"orders": 3, "members": 1}) + # 同一次运行查了两次 orders：只算一次运行 + async with SessionLocal() as session: + session.add(Artifact(id=uuid.uuid4().hex, run_id=runs[0], kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": sid, "source": name, "tables": ["orders", "members"]})) + # 别的数据源的同名表不算 + other = uuid.uuid4().hex + session.add(Run(id=other, graph={}, workflow_name="别处", status="succeeded")) + session.add(Artifact(id=uuid.uuid4().hex, run_id=other, kind=catalog.QUERY_SNAPSHOT_KIND, + meta={"source_id": "elsewhere", "source": "elsewhere", "tables": ["orders"]})) + await session.commit() + r = await client.get(f"{_base(sid)}/orders/usage") + assert r.status_code == 200, r.text + body = r.json() + assert body["total"] == 3 + assert [x["run_id"] for x in body["runs"]] == [runs[2], runs[1], runs[0]] + first = body["runs"][0] + assert first["workflow_name"] == "周报-orders" and first["run_class"] == "formal" + assert first["status"] == "succeeded" and first["started_at"].startswith("2026-09-01T02:00") + assert set(first) == {"run_id", "workflow_id", "workflow_name", "run_class", "status", "started_at"} + limited = (await client.get(f"{_base(sid)}/orders/usage", params={"limit": 1})).json() + assert limited["total"] == 3 and len(limited["runs"]) == 1 + members = (await client.get(f"{_base(sid)}/members/usage")).json() + assert members["total"] == 2 + assert (await client.get(f"{_base(sid)}/ghost/usage")).status_code == 404 + + +# -------------------------------------------------------------------------- +# 进度 +# -------------------------------------------------------------------------- + + +async def test_review_progress_counts_top_tables_all_and_facts(client): + sid = await _source(usage={"orders": 5}) + r = await client.get(f"{_base(sid)}/review-progress") + assert r.status_code == 200, r.text + body = r.json() + assert body["top_n"] == 20 + # 常用表只算被运行用到过的表：这里只有 orders（9 项：表结构里没有的列不算） + assert body["top"] == {"tables": 1, "items_total": 9, "items_confirmed": 1, "items_proposed": 7, + "items_verified": 0} + assert body["all"] == {"items_total": 13, "items_confirmed": 1, "items_proposed": 11, "items_rejected": 1, + "items_verified": 0} + # 影响数字的项：有效记录条件、关联关系、度量类型、业务主键 + assert body["facts"] == {"proposed": 4, "confirmed": 0} + await client.post(f"{_base(sid)}/review/batch", json={"items": [ + {"table_name": "orders", "path": "valid_filter", "action": "confirm", "if_version": 1}]}) + after = (await client.get(f"{_base(sid)}/review-progress")).json() + assert after["top"]["items_confirmed"] == 2 and after["facts"] == {"proposed": 3, "confirmed": 1} From 2298cf848aa255f76076ac4895140ae2ef5e9e69 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:06:45 +0800 Subject: [PATCH 02/39] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E5=AF=BC=E5=87=BA=E5=AF=BC=E5=85=A5=EF=BC=9A=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E4=B8=BA=E3=80=8C<=E6=95=B0=E6=8D=AE=E6=BA=90=E5=90=8D>-?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95.json=E3=80=8D=E9=99=84?= =?UTF-8?q?=E4=BB=B6=EF=BC=88=E5=85=A8=E9=83=A8=E7=8A=B6=E6=80=81=E7=9A=84?= =?UTF-8?q?=E9=A1=B9=E3=80=81=E8=B5=B7=E8=8D=89=E8=A6=81=E6=B1=82=EF=BC=89?= =?UTF-8?q?=EF=BC=9B=E5=AF=BC=E5=85=A5=E5=85=88=E9=A2=84=E8=A7=88=E5=86=8D?= =?UTF-8?q?=E5=86=99=E5=85=A5=EF=BC=8C=E8=A1=A8=E5=90=8D=E3=80=81=E5=88=97?= =?UTF-8?q?=E5=90=8D=E6=8C=89=E8=A1=A8=E7=BB=93=E6=9E=84=E7=9A=84=E5=86=99?= =?UTF-8?q?=E6=B3=95=E5=86=8D=E5=BF=BD=E7=95=A5=E5=A4=A7=E5=B0=8F=E5=86=99?= =?UTF-8?q?=E5=AF=B9=EF=BC=88=E5=B8=A6=20schema=20=E5=89=8D=E7=BC=80?= =?UTF-8?q?=E7=9A=84=E5=8E=BB=E6=8E=89=E5=89=8D=E7=BC=80=E5=86=8D=E5=AF=B9?= =?UTF-8?q?=EF=BC=89=EF=BC=8C=E5=85=B3=E8=81=94=E5=85=B3=E7=B3=BB=E4=B8=A4?= =?UTF-8?q?=E7=AB=AF=E6=8D=A2=E6=88=90=E7=9B=AE=E6=A0=87=E7=9A=84=E5=86=99?= =?UTF-8?q?=E6=B3=95=E5=B9=B6=E9=87=8D=E7=AE=97=E7=BC=96=E5=8F=B7=EF=BC=9B?= =?UTF-8?q?=E5=B7=B2=E7=A1=AE=E8=AE=A4=E3=80=81=E5=B7=B2=E9=A9=B3=E5=9B=9E?= =?UTF-8?q?=E7=9A=84=E9=A1=B9=E7=85=A7=E5=B8=A6=EF=BC=88=E6=9D=A5=E6=BA=90?= =?UTF-8?q?=E3=80=81=E7=8A=B6=E6=80=81=E4=B8=8D=E5=8F=98=EF=BC=89=EF=BC=8C?= =?UTF-8?q?=E7=9B=AE=E6=A0=87=E9=87=8C=E5=B7=B2=E7=A1=AE=E8=AE=A4=E3=80=81?= =?UTF-8?q?=E5=B7=B2=E9=A9=B3=E5=9B=9E=E8=80=8C=E4=B8=8D=E4=B8=80=E6=A0=B7?= =?UTF-8?q?=E7=9A=84=E4=BF=9D=E7=95=99=E7=9B=AE=E6=A0=87=E7=9A=84=E5=B9=B6?= =?UTF-8?q?=E8=AE=B0=E5=86=B2=E7=AA=81=EF=BC=8C=E6=8E=A8=E6=96=AD=E3=80=81?= =?UTF-8?q?=E6=9C=89=E7=A1=AE=E8=AF=81=E7=9A=84=E5=8F=AA=E5=A1=AB=E7=A9=BA?= =?UTF-8?q?=E7=BC=BA=EF=BC=9B=E8=A1=A8=E7=BB=93=E6=9E=84=E9=87=8C=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=E7=9A=84=E5=88=97=E3=80=81=E4=B8=A4=E7=AB=AF=E5=AF=B9?= =?UTF-8?q?=E4=B8=8D=E4=B8=8A=E7=9A=84=E5=85=B3=E7=B3=BB=E3=80=81=E7=9B=AE?= =?UTF-8?q?=E6=A0=87=E9=81=AE=E7=BD=A9=E5=88=97=E7=9A=84=E7=A0=81=E5=80=BC?= =?UTF-8?q?=E3=80=81=E4=B8=8D=E5=90=88=E8=A7=84=E7=9A=84=E9=A1=B9=E8=B7=B3?= =?UTF-8?q?=E8=BF=87=E5=B9=B6=E5=86=99=E5=8E=9F=E5=9B=A0=EF=BC=9B=E9=80=90?= =?UTF-8?q?=E8=A1=A8=E4=B9=90=E8=A7=82=E9=94=81=E3=80=81=E4=B8=80=E5=BC=A0?= =?UTF-8?q?=E8=A1=A8=E5=86=99=E4=B8=8D=E8=BF=9B=E5=8E=BB=E4=B8=8D=E5=BD=B1?= =?UTF-8?q?=E5=93=8D=E5=88=AB=E7=9A=84=EF=BC=9B=E6=9C=89=E8=B5=B7=E8=8D=89?= =?UTF-8?q?=E4=BB=BB=E5=8A=A1=E6=97=B6=20409=EF=BC=8C=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E4=B8=8D=E5=AF=B9=20422=EF=BC=8C=E6=B2=A1?= =?UTF-8?q?=E6=9C=89=E8=A1=A8=E7=BB=93=E6=9E=84=20409=EF=BC=9B=E8=B5=B7?= =?UTF-8?q?=E8=8D=89=E8=A6=81=E6=B1=82=E5=8F=AA=E5=9C=A8=E5=8B=BE=E9=80=89?= =?UTF-8?q?=E4=B8=94=E7=9B=AE=E6=A0=87=E8=BF=98=E6=B2=A1=E6=9C=89=E6=97=B6?= =?UTF-8?q?=E5=86=99=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/catalog.py | 66 +++- backend/app/data/catalog_transfer.py | 484 +++++++++++++++++++++++++ backend/tests/test_catalog_transfer.py | 378 +++++++++++++++++++ 3 files changed, 927 insertions(+), 1 deletion(-) create mode 100644 backend/app/data/catalog_transfer.py create mode 100644 backend/tests/test_catalog_transfer.py diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 7680eb54..b3554756 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -16,6 +16,8 @@ - GET /catalog/review-queue 审阅队列：推断的项按使用次数、风险、来源排好；POST /catalog/review/batch 批量审阅， GET /catalog/review-progress 审阅进度，GET /catalog/{table}/usage 用过这张表的运行 （data/catalog_review.py） +- GET /catalog/export 导出数据目录（JSON 附件）；POST /catalog/import/preview 导入预览（只读）， + POST /catalog/import 导入（data/catalog_transfer.py） 规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径 不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。 @@ -38,7 +40,7 @@ from app.api.coded import ( ) from app.api.runs import actor_of from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, introspect, table_versions -from app.data import catalog_review +from app.data import catalog_review, catalog_transfer from app.data.engine import SnapshotTampered from app.db.base import get_session from app.db.models import DataSource @@ -262,6 +264,68 @@ async def catalog_review_progress(source_id: str, session: AsyncSession = Depend return await catalog_review.review_progress(session, source) +# ---- 导出导入（导出同样要登记在 GET /catalog/{table} 前面） + + +@router.get("/{source_id}/catalog/export") +async def export_catalog(source_id: str, session: AsyncSession = Depends(get_session)) -> Any: + """导出这个数据源的数据目录：JSON 附件「<数据源名>-数据目录.json」，含全部状态的项（data/catalog_transfer.py）。""" + from fastapi.responses import JSONResponse + + row = await _source_row(session, source_id) + body = await catalog_transfer.export_catalog(session, row) + name = catalog_transfer.attachment_name(row.name) + return JSONResponse(body, headers={"Content-Disposition": catalog_transfer.content_disposition(name, row.id)}) + + +class CatalogImportIn(BaseModel): + #: 导出的那个对象（格式在 data/catalog_transfer.read_file 里核对，不对 422，说明是中文） + content: Any = None + #: 目标还没有起草要求时，是否写入文件里的起草要求 + include_guidance: bool = False + + +async def _import_source(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]: + row, source = await _resolved(session, source_id) + cache = source.schema_cache or {} + if not cache.get("tables"): + raise CodedHTTPException(409, f"{introspect.why_empty(cache)}，无法导入数据目录。请先在数据源卡片上点「探查结构」", + SCHEMA_MISSING) + return row, source + + +@router.post("/{source_id}/catalog/import/preview") +async def preview_catalog_import(source_id: str, payload: CatalogImportIn, + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """导入预览（只读）：每张对上的表新增、更新、保留几项，冲突和跳过的逐条列出。文件格式不对 422；没有表结构 409。""" + row, source = await _import_source(session, source_id) + try: + report = await catalog_transfer.plan_import(session, row, source, payload.content, + include_guidance=payload.include_guidance) + except catalog_transfer.TransferInvalid as e: + raise HTTPException(422, str(e)) from e + return report.as_dict() + + +@router.post("/{source_id}/catalog/import") +async def import_catalog(source_id: str, payload: CatalogImportIn, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> dict[str, Any]: + """导入：规则同预览，逐表写入（乐观锁），返回同样形状的报告和每张表写入后的版本。有排队或进行中的起草任务 + 409 catalog_draft_busy；文件格式不对 422；没有表结构 409。""" + row, source = await _import_source(session, source_id) + try: + await catalog_jobs.ensure_idle(session, row) + except catalog_jobs.DraftBusy as e: + raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e + try: + report = await catalog_transfer.run_import(session, row, source, payload.content, + include_guidance=payload.include_guidance, + actor=actor_of(x_actor)) + except catalog_transfer.TransferInvalid as e: + raise HTTPException(422, str(e)) from e + return report.as_dict() + + @router.get("/{source_id}/catalog/{table}") async def get_catalog_table(source_id: str, table: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: diff --git a/backend/app/data/catalog_transfer.py b/backend/app/data/catalog_transfer.py new file mode 100644 index 00000000..34954f8d --- /dev/null +++ b/backend/app/data/catalog_transfer.py @@ -0,0 +1,484 @@ +"""数据目录的导出导入：本机整理好的目录带到部署实例上（GET /catalog/export、POST /catalog/import/preview、/import）。 + +**为什么要有它。** 目录（尤其是人工确认、驳回过的项）是在本机对着开发库一项项审出来的；部署实例连的是同一套业务库， +却没有办法把这些结论带过去，只能在那边从头再审一遍。导出是一个 JSON 文件，导入先预览、再确认。 + +**文件**（export_catalog）：{format: "agentlab-catalog", version: 1, exported_at, source: {name, kind}, guidance, +tables: {表名: {notes, version}}}，含全部状态的项，表结构里已经没有的表也带上（目录里有什么就导出什么）。 + +**导入的规矩**（plan_import，预览和导入共用这一份）： + +- 表按名字对：表结构里的写法、忽略大小写（name_key）、带 schema 前缀的对不上时去掉前缀再对（catalog.resolve_table_name）。 + 只对目标数据源表结构里的表；对不上的列在 unmatched_tables。文件里两张表对上目标的同一张时，写法完全一样的那张优先， + 另一张算对不上。列名同样按表结构的写法、再忽略大小写对；业务主键、业务日期、关联关系两端的列和指向的表都换成目标的写法， + 关联关系的编号按目标的表名、列名重算（catalog.relation_id）。 +- 文件里已确认、已驳回的项照带：来源、状态、备注、修改时间都不变。目标那一格空着就新增；目标是推断、有确证的就更新； + 目标也是已确认、已驳回的，和文件一样就保留，不一样（值或状态不同）就保留目标的、记一条冲突。关联关系的「一样」看两端和 + 基数，覆盖率是测量值、不算。 +- 文件里推断、有确证的项只填目标的空缺，目标那一格有东西（哪怕是驳回）就保留目标的，不算冲突。 +- 跳过并写原因：表结构里没有的列、业务主键和业务日期指着的列没有的、关联关系两端对不上的（指向的表、两端的列）、 + 目标数据源遮罩的列上的码值（码值就是这一列的取值，遮罩了就不该进目录）、不认识的字段、取值不合规的项（说法同 + catalog.validate_notes）。 +- 逐表写（乐观锁，撞上别人刚改过就重读重算，最多三次），一张表写不进去只记在那张表上，不影响别的表。 +- 起草要求：勾了 include_guidance、文件里有、目标还没有时才写（action: replace）；目标已有的不覆盖（keep）；没勾或文件里 + 没有是 none。 +- 文件格式不对（不是对象、格式标记或版本不对、表的目录不是对象、起草要求不是文字或超长）抛 TransferInvalid，接口回 422。 + +有排队或进行中的起草任务时不导入（接口先调 catalog_jobs.ensure_idle，409 catalog_draft_busy）；目标数据源没有表结构时 +接口回 409（什么也对不上）。 +""" +from __future__ import annotations + +import copy +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import catalog, catalog_jobs +from app.data.catalog_review import field_label, slot_path +from app.data.engine import masked_columns +from app.data.names import name_key + +FORMAT = "agentlab-catalog" +VERSION = 1 +FILE_SUFFIX = "-数据目录.json" +_BAD_FILENAME = re.compile(r'[\\/:*?"<>|\x00-\x1f\x7f]') +_NAME_MAX = 80 +_UNNAMED = "数据源" + +#: 状态：照带的、只填空缺的 +_LOCKED = ("confirmed", "rejected") +_STATUS_LABEL = {"proposed": "推断", "verified": "有确证", "confirmed": "已确认", "rejected": "已驳回"} + +#: 写入一直撞上别人刚改过（重读重算三次仍冲突） +WRITE_CONFLICT_ERROR = "这张表的数据目录正被其他人修改，没有导入，请稍后重新导入" +WRITE_ERROR = "写入这张表的数据目录时出错，没有导入：{line}" + + +class TransferInvalid(ValueError): + """文件格式不对。message 给人看（接口回 422）。""" + + +# ========================================================================== +# 导出 +# ========================================================================== + + +def attachment_name(source_name: str) -> str: + """附件名「<数据源名>-数据目录.json」。文件名里不能有的字换成下划线。""" + name = _BAD_FILENAME.sub("_", source_name or "").strip(" .")[:_NAME_MAX].strip(" .") + return f"{name or _UNNAMED}{FILE_SUFFIX}" + + +def content_disposition(filename: str, source_id: str) -> str: + """附件头：中文名按 RFC 5987 写在 filename* 里；filename 给不认 filename* 的老客户端一个纯 ASCII 的名字。""" + from urllib.parse import quote + + ascii_id = re.sub(r"[^A-Za-z0-9]", "", source_id)[:8] + return f"attachment; filename=\"agentlab-catalog-{ascii_id}.json\"; filename*=UTF-8''{quote(filename, safe='')}" + + +async def export_catalog(session: AsyncSession, row: Any) -> dict[str, Any]: + """导出这个数据源的数据目录（row 是数据源记录，目录按它的 id 存）。""" + entries = await catalog.read_catalog(session, row.id) + return { + "format": FORMAT, + "version": VERSION, + "exported_at": catalog.now_iso(), + "source": {"name": row.name, "kind": row.kind}, + "guidance": catalog_jobs.guidance_of(row), + "tables": {name: {"notes": entry.notes, "version": entry.version} for name, entry in sorted(entries.items())}, + } + + +# ========================================================================== +# 读文件 +# ========================================================================== + + +@dataclass(frozen=True) +class ImportFile: + tables: dict[str, dict[str, Any]] + guidance: str | None + + +def read_file(content: Any, *, include_guidance: bool) -> ImportFile: + """导出的那个对象 → ImportFile。格式不对抛 TransferInvalid（说法给人看）。""" + if not isinstance(content, Mapping): + raise TransferInvalid("文件内容不是数据目录导出的格式，请选择从 AgentLab 导出的数据目录文件") + if content.get("format") != FORMAT: + raise TransferInvalid("这个文件不是 AgentLab 导出的数据目录（文件里的格式标记不对），请重新选择文件") + version = content.get("version") + if version != VERSION: + raise TransferInvalid(f"这个数据目录文件的格式版本是「{version}」，当前只能导入第 {VERSION} 版的文件，" + "请用同一版本的 AgentLab 重新导出") + tables = content.get("tables") + if not isinstance(tables, Mapping): + raise TransferInvalid("数据目录文件里没有表的目录，文件可能不完整，请重新导出") + out: dict[str, dict[str, Any]] = {} + for name, body in tables.items(): + notes = body.get("notes") if isinstance(body, Mapping) else None + if not isinstance(name, str) or not name.strip() or not isinstance(notes, Mapping): + raise TransferInvalid(f"数据目录文件里表「{str(name)[:60]}」的目录格式不正确，文件可能被改动过，请重新导出") + out[name] = copy.deepcopy(dict(notes)) + guidance = content.get("guidance") + if guidance is not None and not isinstance(guidance, str): + raise TransferInvalid("数据目录文件里的起草要求格式不正确，文件可能被改动过，请重新导出") + text = guidance.strip() if isinstance(guidance, str) else "" + if include_guidance and len(text) > catalog_jobs.GUIDANCE_MAX: + raise TransferInvalid(f"数据目录文件里的起草要求有 {len(text)} 字，超过了 {catalog_jobs.GUIDANCE_MAX} 字的上限，" + "请删减后再导入，或者不导入起草要求") + return ImportFile(tables=out, guidance=text or None) + + +# ========================================================================== +# 对表、对列 +# ========================================================================== + + +def match_tables(file_tables: list[str], schema_cache: Mapping[str, Any]) -> tuple[dict[str, str], list[str]]: + """文件里的表 → 目标表结构里的键。返回 ({文件里的表名: 目标表名}, 对不上的表名)。写法完全一样的先认领。""" + tables = (schema_cache or {}).get("tables") or {} + matched: dict[str, str] = {} + claimed: set[str] = set() + ordered = sorted(file_tables, key=lambda n: n not in tables) + for name in ordered: + key = catalog.resolve_table_name(schema_cache, name) + if key is None or key in claimed: + continue + matched[name] = key + claimed.add(key) + unmatched = [name for name in file_tables if name not in matched] + return {name: matched[name] for name in file_tables if name in matched}, unmatched + + +class _Columns: + """一张目标表的列：按表结构的写法认，再忽略大小写认（只认唯一对上的）。""" + + def __init__(self, meta: Mapping[str, Any] | None) -> None: + names = [str(c.get("name")) for c in (meta or {}).get("columns") or [] if isinstance(c, Mapping) + and c.get("name") is not None] + self.names = set(names) + self._loose: dict[str, list[str]] = {} + for name in names: + self._loose.setdefault(name_key(name), []).append(name) + + def find(self, name: Any) -> str | None: + if not isinstance(name, str): + return None + if name in self.names: + return name + found = self._loose.get(name_key(name)) or [] + return found[0] if len(found) == 1 else None + + +# ========================================================================== +# 一张表怎么并 +# ========================================================================== + + +@dataclass +class TablePlan: + """文件里一张表并进目标的计划。writes 是要写进目标的项（目标的槽位 → 项）。""" + + table_name: str + matched_as: str + add: int = 0 + update: int = 0 + keep: int = 0 + conflicts: list[dict[str, Any]] = field(default_factory=list) + skipped: list[dict[str, Any]] = field(default_factory=list) + writes: dict[catalog.Slot, dict[str, Any]] = field(default_factory=dict) + version: int = 0 + error: str | None = None + + def as_dict(self) -> dict[str, Any]: + return {"table_name": self.table_name, "matched_as": self.matched_as, "add": self.add, "update": self.update, + "keep": self.keep, "conflicts": self.conflicts, "skipped": self.skipped, "version": self.version, + "error": self.error} + + +def _raw_items(notes: Mapping[str, Any]) -> list[tuple[str, catalog.Slot | None, Any, str | None]]: + """文件里一张表的每一项：(文件里的路径, 槽位, 项, 不认的原因)。不认识的字段、不是对象的列也逐个列出来（槽位为 None）。""" + out: list[tuple[str, catalog.Slot | None, Any, str | None]] = [] + for name, item in notes.items(): + if name in catalog.TABLE_FIELDS: + out.append((name, ("t", name), item, None)) + elif name not in ("columns", "relations"): + out.append((str(name), None, item, f"数据目录里不认识的字段「{name}」")) + columns = notes.get("columns", {}) + if not isinstance(columns, Mapping): + out.append(("columns", None, columns, "列的目录应为以列名为键的对象")) + columns = {} + for col, items in columns.items(): + if not isinstance(items, Mapping): + out.append((f"columns.{col}", None, items, f"列 {col} 的目录应为对象")) + continue + for name, item in items.items(): + path = f"columns.{col}.{name}" + if name in catalog.COLUMN_FIELDS: + out.append((path, ("c", str(col), name), item, None)) + else: + out.append((path, None, item, f"列 {col} 含有不认识的字段「{name}」")) + relations = notes.get("relations", []) + if not isinstance(relations, list): + out.append(("relations", None, relations, "关联关系应为列表")) + relations = [] + for rel in relations: + rid = rel.get("id") if isinstance(rel, Mapping) else None + out.append((f"relations.{rid}", ("r", str(rid)), rel, None)) + return out + + +def _item_problem(key: catalog.Slot, item: Any) -> str | None: + """这一项本身合不合规（说法同 catalog.validate_notes）。""" + if key[0] == "t": + probe: dict[str, Any] = {key[1]: item} + elif key[0] == "c": + probe = {"columns": {key[1]: {key[2]: item}}} + else: + probe = {"relations": [item]} + problems = catalog.validate_notes(probe) + return problems[0] if problems else None + + +def _decision(key: catalog.Slot, item: Mapping[str, Any]) -> Any: + """判断「一样不一样」看的部分：状态，加上值（码值另看「已列全」）。关联关系的两端由编号定了，再看基数；覆盖率不算。""" + if key[0] == "r": + return item.get("status"), item.get("cardinality") + if key[0] == "c" and key[2] == "codes": + return item.get("status"), item.get("value"), catalog.codes_complete(item) + return item.get("status"), item.get("value") + + +def _conflict_why(file_status: str, target_status: str) -> str: + if target_status == "rejected": + return f"目标里这一项已经驳回，文件里是{_STATUS_LABEL.get(file_status, file_status)}，保留目标的" + if file_status == "rejected": + return "目标里这一项已经人工确认过，文件里是已驳回，保留目标的" + return "目标里这一项已经人工确认过，和文件里的不一样，保留目标的" + + +class _Mapper: + """把文件里的一项换成目标的写法：列名、业务主键和业务日期的列、关联关系两端和编号。对不上返回原因。""" + + def __init__(self, target: str, schema_cache: Mapping[str, Any], masked: set[str]) -> None: + self.target = target + self.cache = schema_cache + self.tables = (schema_cache or {}).get("tables") or {} + self.columns = _Columns(self.tables.get(target)) + self.masked = masked + + def __call__(self, key: catalog.Slot, item: Any) -> tuple[catalog.Slot | None, Any, str | None]: + if not isinstance(item, Mapping): + return None, item, None # 交给合规检查说 + item = copy.deepcopy(dict(item)) + if key[0] == "c": + col = self.columns.find(key[1]) + if col is None: + return None, item, f"目标的表结构里没有列 {key[1]}，跳过" + if key[2] == "codes" and col.lower() in self.masked: + return None, item, f"列 {col} 在目标数据源的设置中被遮罩，码值是这一列的取值，不导入" + return ("c", col, key[2]), item, None + if key[0] == "t": + value = item.get("value") + if key[1] == "keys" and isinstance(value, list): + mapped = [self.columns.find(c) for c in value] + if missing := [str(c) for c, m in zip(value, mapped) if m is None]: + return None, item, f"业务主键里的列「{'、'.join(missing)}」不在目标的表结构里，跳过" + item["value"] = mapped + elif key[1] == "business_date" and isinstance(value, Mapping) and isinstance(value.get("column"), str): + col = self.columns.find(value["column"]) + if col is None: + return None, item, f"业务日期的列「{value['column']}」不在目标的表结构里，跳过" + item["value"] = {**value, "column": col} + return key, item, None + return self._relation(item) + + def _relation(self, rel: dict[str, Any]) -> tuple[catalog.Slot | None, Any, str | None]: + cols, to_table, to_cols = rel.get("columns"), rel.get("to_table"), rel.get("to_columns") + if not (isinstance(cols, list) and isinstance(to_cols, list) and isinstance(to_table, str)): + return None, rel, None + target_table = catalog.resolve_table_name(self.cache, to_table) + if target_table is None: + return None, rel, f"关联关系指向的表 {to_table} 不在目标数据源的表结构里，跳过" + mine = [self.columns.find(c) for c in cols] + if missing := [str(c) for c, m in zip(cols, mine) if m is None]: + return None, rel, f"关联关系的本表字段「{'、'.join(missing)}」不在目标的表结构里，跳过" + theirs_columns = _Columns(self.tables.get(target_table)) + theirs = [theirs_columns.find(c) for c in to_cols] + if missing := [str(c) for c, m in zip(to_cols, theirs) if m is None]: + return None, rel, f"关联关系的目标字段「{'、'.join(missing)}」不在 {target_table} 的表结构里，跳过" + rid = catalog.relation_id(self.target, mine, target_table, theirs) + return ("r", rid), {**rel, "id": rid, "columns": mine, "to_table": target_table, "to_columns": theirs}, None + + +def plan_table(table_name: str, target: str, file_notes: Mapping[str, Any], target_notes: Mapping[str, Any] | None, + schema_cache: Mapping[str, Any], masked: set[str]) -> TablePlan: + """文件里一张表并进目标那张表的计划（纯函数，不写库）。""" + plan = TablePlan(table_name=table_name, matched_as=target) + existing = catalog._slots(target_notes) + mapper = _Mapper(target, schema_cache, masked) + seen: set[catalog.Slot] = set() + for path, key, item, why in _raw_items(file_notes): + if why is not None or key is None: + plan.skipped.append({"path": path, "why": why or "无法识别这一项"}) + continue + mapped, item, why = mapper(key, item) + if why is not None: + plan.skipped.append({"path": path, "why": why}) + continue + problem = _item_problem(mapped or key, item) + if problem is not None or mapped is None: + plan.skipped.append({"path": path, "why": problem or "无法识别这一项"}) + continue + if mapped in seen: + plan.skipped.append({"path": path, "why": f"和文件里前面的{catalog.describe_slot(mapped)}是同一项，只导入第一项"}) + continue + seen.add(mapped) + status = item.get("status") + have = existing.get(mapped) + if have is None: + plan.add += 1 + plan.writes[mapped] = item + elif status not in _LOCKED: + plan.keep += 1 # 推断、有确证的只填空缺 + elif have.get("status") in _LOCKED: + if _decision(mapped, have) == _decision(mapped, item): + plan.keep += 1 + else: + plan.conflicts.append({ + "path": slot_path(mapped), "label": _label(mapped, item), + "file_value": _shown_value(mapped, item), "target_value": _shown_value(mapped, have), + "file_status": status, "target_status": have.get("status"), + "why": _conflict_why(str(status), str(have.get("status"))), + }) + elif _decision(mapped, have) == _decision(mapped, item): + plan.keep += 1 + else: + plan.update += 1 + plan.writes[mapped] = item + return plan + + +def _label(key: catalog.Slot, item: Mapping[str, Any]) -> str: + if key[0] == "t": + return f"表的{field_label(key)}" + if key[0] == "c": + return f"列 {key[1]} 的{field_label(key)}" + return f"指向 {item.get('to_table')} 的关联关系" + + +def _shown_value(key: catalog.Slot, item: Mapping[str, Any]) -> Any: + if key[0] == "r": + return {k: item.get(k) for k in ("columns", "to_table", "to_columns", "cardinality", "coverage")} + return item.get("value") + + +# ========================================================================== +# 预览、导入 +# ========================================================================== + + +@dataclass +class ImportReport: + tables: list[TablePlan] + unmatched: list[str] + guidance: dict[str, Any] + + def as_dict(self) -> dict[str, Any]: + return { + "tables": [t.as_dict() for t in self.tables], + "unmatched_tables": self.unmatched, + "totals": {"tables": len(self.tables), "add": sum(t.add for t in self.tables), + "update": sum(t.update for t in self.tables), "keep": sum(t.keep for t in self.tables), + "conflicts": sum(len(t.conflicts) for t in self.tables), + "skipped": sum(len(t.skipped) for t in self.tables)}, + "guidance": self.guidance, + } + + +def _guidance(file: ImportFile, row: Any, include: bool) -> dict[str, Any]: + target = catalog_jobs.guidance_of(row) + if not include or file.guidance is None: + action = "none" + elif target: + action = "keep" + else: + action = "replace" + return {"file": file.guidance, "target": target, "action": action} + + +async def plan_import(session: AsyncSession, row: Any, source: Any, content: Any, *, + include_guidance: bool) -> ImportReport: + """预览：逐表算出要新增、更新、保留什么，冲突和跳过的项及原因。不写库。row 是数据源记录（遮罩、起草要求按它）， + source 是读表结构用的源（上传源是绑定当前快照的视图）。格式不对抛 TransferInvalid。""" + file = read_file(content, include_guidance=include_guidance) + cache = getattr(source, "schema_cache", None) or {} + matched, unmatched = match_tables(list(file.tables), cache) + masked = {c.lower() for c in masked_columns(getattr(row, "options", None))} + plans: list[TablePlan] = [] + for name, target in matched.items(): + entry = await catalog.read_entry(session, source.id, target) + plan = plan_table(name, target, file.tables[name], entry.notes if entry else None, cache, masked) + plan.version = entry.version if entry else 0 + plans.append(plan) + return ImportReport(plans, unmatched, _guidance(file, row, include_guidance)) + + +async def run_import(session: AsyncSession, row: Any, source: Any, content: Any, *, include_guidance: bool, + actor: str | None) -> ImportReport: + """导入：按 plan_import 的规矩逐表写入（乐观锁，撞上别人刚改过就重读重算，最多三次），再按需写起草要求。 + 一张表写不进去只记在它的 error 上（新增、更新记 0），不影响别的表。""" + file = read_file(content, include_guidance=include_guidance) + cache = getattr(source, "schema_cache", None) or {} + matched, unmatched = match_tables(list(file.tables), cache) + masked = {c.lower() for c in masked_columns(getattr(row, "options", None))} + plans: list[TablePlan] = [] + for name, target in matched.items(): + plans.append(await _import_table(session, source, name, target, file.tables[name], cache, masked, actor)) + guidance = _guidance(file, row, include_guidance) + if guidance["action"] == "replace": + await catalog_jobs.save_guidance(session, row, file.guidance) + return ImportReport(plans, unmatched, guidance) + + +async def _import_table(session: AsyncSession, source: Any, name: str, target: str, notes: Mapping[str, Any], + cache: Mapping[str, Any], masked: set[str], actor: str | None) -> TablePlan: + plan = TablePlan(table_name=name, matched_as=target) + try: + for _attempt in range(3): + entry = await catalog.read_entry(session, source.id, target) + version = entry.version if entry else 0 + plan = plan_table(name, target, notes, entry.notes if entry else None, cache, masked) + plan.version = version + if not plan.writes: + return plan + slots = catalog._slots(entry.notes if entry else None) + slots.update(plan.writes) + try: + written = await catalog.write_entry(session, source.id, target, catalog._assemble(slots), + if_version=version, actor=actor) + except catalog.CatalogConflict: + continue + plan.version = written.version + return plan + plan.error = WRITE_CONFLICT_ERROR + except catalog.CatalogInvalid as e: + await session.rollback() + plan.error = WRITE_ERROR.format(line=str(e)) + except Exception as e: # noqa: BLE001 - 这一张写不进去不影响别的表 + from app.core.errors import describe_exception, first_line + + await session.rollback() + plan.error = WRITE_ERROR.format(line=first_line(e) or describe_exception(e)) + plan.add = plan.update = 0 + return plan + + +__all__ = ["FORMAT", "ImportFile", "ImportReport", "TablePlan", "TransferInvalid", "VERSION", "attachment_name", + "content_disposition", "export_catalog", "match_tables", "plan_import", "plan_table", "read_file", + "run_import"] diff --git a/backend/tests/test_catalog_transfer.py b/backend/tests/test_catalog_transfer.py new file mode 100644 index 00000000..de5164b5 --- /dev/null +++ b/backend/tests/test_catalog_transfer.py @@ -0,0 +1,378 @@ +"""数据目录的导出导入（data/catalog_transfer.py）：本机整理好的目录带到部署实例上。 + +- GET /catalog/export：附件「<数据源名>-数据目录.json」，{format: agentlab-catalog, version: 1, exported_at, + source: {name, kind}, guidance, tables: {表名: {notes, version}}}，含全部状态的项（表结构里已经没有的表也带） +- POST /catalog/import/preview：只读，报告每张对上的表新增、更新、保留、冲突、跳过了什么 +- POST /catalog/import：同样的报告，加每张表写入后的版本。规则： + - 表按名字对（忽略大小写；带 schema 前缀的对不上时去掉前缀再对），对不上的列在 unmatched_tables + - 文件里已确认、已驳回的项照带（来源、状态不变）；目标里已确认、已驳回的项和文件不一样时保留目标的、记冲突 + - 文件里推断、有确证的项只填目标的空缺，不覆盖目标已有的 + - 表结构里没有的列、关系两端对不上的跳过并写原因；目标数据源遮罩的列，码值不导入 + - 逐表乐观锁，一张表写不进去不影响别的；有排队或进行中的起草任务时 409 catalog_draft_busy；文件格式不对 422 + - 起草要求：勾了导入、目标还没有时才写；目标已有的不覆盖 + +表名、列名、业务数字都是虚构的。 +""" +from __future__ import annotations + +import copy +import json +import uuid +from urllib.parse import quote, unquote + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.data import catalog, catalog_jobs, catalog_transfer +from app.data.catalog import make_item +from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION +from app.db.base import SessionLocal +from app.db.models import CatalogDraftJob, DataSource +from app.main import app as _fastapi_app +from tests.fixtures.sources import drop_source + +ACTOR = {"X-Actor": quote("沈岚")} + + +def _col(name: str, type_: str = "INTEGER") -> dict: + return {"name": name, "type": type_, "nullable": True, "comment": None} + + +def _tables(*, upper: bool = False) -> dict: + """两边的表结构：目标那边（upper）表名、列名写法不同，还少了一列 tier。""" + def n(name: str) -> str: + return name.upper() if upper else name + + members_cols = [_col(n("id")), _col(n("nick"), "TEXT")] + ([] if upper else [_col("tier")]) + return { + n("orders"): {"qualified": n("orders"), "primary_key": [n("id")], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], + "columns": [_col(n("id")), _col(n("status")), _col(n("amount"), "REAL"), _col(n("member_id")), + _col(n("contact_phone"), "TEXT")]}, + n("members"): {"qualified": n("members"), "primary_key": [n("id")], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], "columns": members_cols}, + n("refunds"): {"qualified": n("refunds"), "primary_key": [n("id")], "comment": None, "is_view": False, + "foreign_keys": [], "unique": [], "columns": [_col(n("id")), _col(n("order_id"))]}, + } + + +def _rel(to_table: str, columns: list[str], to_columns: list[str], *, status: str, source: str = "name", + table: str = "orders") -> dict: + return {"id": catalog.relation_id(table, columns, to_table, to_columns), "columns": columns, + "to_table": to_table, "to_columns": to_columns, "cardinality": "many_to_one", "coverage": None, + "source": source, "status": status} + + +#: 本机（源）的目录 +SOURCE_ORDERS = { + "label": make_item("订单", "human", "confirmed", at="2026-10-01T08:00:00+00:00"), + "grain": make_item("每笔订单一行", "llm"), + "valid_filter": make_item("status <> 9", "human", "confirmed"), + "dedup": make_item("同一订单号取最后一次", "llm", "rejected"), + "kind": make_item("fact", "llm", "confirmed"), + "columns": { + "status": {"codes": make_item({"1": "已支付", "9": "作废"}, "human", "confirmed", complete=True), + "label": make_item("状态", "llm")}, + "amount": {"unit": make_item("元", "human", "confirmed"), "measure": make_item("flow", "llm")}, + "contact_phone": {"codes": make_item({"13900000001": "测试号"}, "human", "confirmed"), + "label": make_item("联系电话", "human", "confirmed")}, + # 目标表结构里没有这一列 + "gift_flag": {"label": make_item("赠品标记", "human", "confirmed")}, + }, + "relations": [ + _rel("members", ["member_id"], ["id"], status="confirmed"), + # 目标那边没有这张表 + _rel("stores", ["member_id"], ["id"], status="rejected"), + ], +} +SOURCE_MEMBERS = {"label": make_item("会员", "llm"), "keys": make_item(["tier"], "human", "confirmed"), + "columns": {"nick": {"label": make_item("昵称", "llm", "verified")}}} +#: 目标那边没有的表 +SOURCE_GHOST = {"label": make_item("旧表", "human", "confirmed")} + +#: 目标（部署实例）上已有的目录 +TARGET_ORDERS = { + "label": make_item("订单表", "human", "confirmed"), # 和文件不一样、都确认过：冲突，保留目标 + "grain": make_item("一行一单", "llm"), # 文件里是推断：保留目标 + "valid_filter": make_item("STATUS <> 9", "llm"), # 目标是推断、文件确认过：更新 + "kind": make_item("fact", "human", "confirmed"), # 一样：保留 + "columns": {"STATUS": {"label": make_item("状态码", "comment")}}, +} + +_CREATED: list[str] = [] + + +@pytest.fixture(autouse=True) +async def _cleanup(): + yield + while _CREATED: + await drop_source(_CREATED.pop()) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c: + yield c + + +async def _write(sid: str, table: str, notes: dict) -> None: + async with SessionLocal() as session: + entry = await catalog.read_entry(session, sid, table) + await catalog.write_entry(session, sid, table, notes, if_version=entry.version if entry else 0, actor=None) + + +async def _source(name: str, tables: dict, entries: dict[str, dict], **kw) -> str: + async with SessionLocal() as session: + row = DataSource(name=name, kind="sqlite", database=":memory:", schema_cache={"tables": tables}, **kw) + session.add(row) + await session.commit() + sid = row.id + _CREATED.append(sid) + for table, notes in entries.items(): + await _write(sid, table, notes) + return sid + + +async def _pair() -> tuple[str, str]: + local = await _source(f"本机库_{uuid.uuid4().hex[:6]}", _tables(), + {"orders": SOURCE_ORDERS, "members": SOURCE_MEMBERS, "ghost_table": SOURCE_GHOST}, + options={CATALOG_DRAFT_GUIDANCE_OPTION: "金额一律按元"}) + remote = await _source(f"deploy_{uuid.uuid4().hex[:6]}", _tables(upper=True), {"ORDERS": TARGET_ORDERS}, + options={"mask_columns": "contact_phone"}) + return local, remote + + +def _base(sid: str) -> str: + return f"/api/datasources/{sid}/catalog" + + +async def _export(client, sid: str) -> dict: + r = await client.get(f"{_base(sid)}/export") + assert r.status_code == 200, r.text + return r.json() + + +# -------------------------------------------------------------------------- +# 导出 +# -------------------------------------------------------------------------- + + +async def test_export_is_a_json_attachment_with_every_item(client): + local, _ = await _pair() + r = await client.get(f"{_base(local)}/export") + assert r.status_code == 200, r.text + assert r.headers["content-type"].startswith("application/json") + disposition = r.headers["content-disposition"] + assert disposition.startswith("attachment;") and "filename=\"agentlab-catalog-" in disposition + async with SessionLocal() as session: + name = (await session.get(DataSource, local)).name + assert unquote(disposition.split("filename*=UTF-8''", 1)[1]) == f"{name}-数据目录.json" + body = r.json() + assert body["format"] == "agentlab-catalog" and body["version"] == 1 + assert body["exported_at"] and body["source"] == {"name": name, "kind": "sqlite"} + assert body["guidance"] == "金额一律按元" + assert set(body["tables"]) == {"orders", "members", "ghost_table"} + assert body["tables"]["orders"]["version"] == 1 + # 含全部状态的项：驳回的也带 + assert body["tables"]["orders"]["notes"]["dedup"]["status"] == "rejected" + assert body["tables"]["orders"]["notes"] == SOURCE_ORDERS + assert body["tables"]["ghost_table"] == {"notes": SOURCE_GHOST, "version": 1} + + +def test_attachment_name_drops_characters_files_cannot_have(): + assert catalog_transfer.attachment_name('景区/客流:库?') == "景区_客流_库_-数据目录.json" + assert catalog_transfer.attachment_name(" ") == "数据源-数据目录.json" + + +# -------------------------------------------------------------------------- +# 预览与导入 +# -------------------------------------------------------------------------- + + +def _by(report: dict) -> dict[str, dict]: + return {t["matched_as"]: t for t in report["tables"]} + + +async def test_preview_reports_and_writes_nothing(client): + local, remote = await _pair() + content = await _export(client, local) + r = await client.post(f"{_base(remote)}/import/preview", json={"content": content, "include_guidance": True}) + assert r.status_code == 200, r.text + report = r.json() + assert report["unmatched_tables"] == ["ghost_table"] + tables = _by(report) + assert set(tables) == {"ORDERS", "MEMBERS"} + orders = tables["ORDERS"] + assert orders["table_name"] == "orders" + conflicts = {c["path"]: c for c in orders["conflicts"]} + assert set(conflicts) == {"label"} + label = conflicts["label"] + assert label["label"] == "表的中文名" and label["file_value"] == "订单" and label["target_value"] == "订单表" + assert "保留" in label["why"] + skipped = {s["path"]: s["why"] for s in orders["skipped"]} + assert "columns.gift_flag.label" in skipped and "表结构里没有列 gift_flag" in skipped["columns.gift_flag.label"] + assert "遮罩" in skipped["columns.contact_phone.codes"] + stores = next(p for p in skipped if p.startswith("relations.")) + assert "stores" in skipped[stores] + # 新增：dedup（驳回的照带）、status 码值、amount 两项、联系电话的中文名、关联关系；更新：有效记录条件 + # 保留：粒度（文件里是推断，目标已有）、表类型（一样）、STATUS 的中文名（文件里是推断，目标已有） + assert (orders["add"], orders["update"], orders["keep"]) == (6, 1, 3) + members = tables["MEMBERS"] + # 业务主键指着的列 tier 目标那边没有：跳过；会员的中文名和昵称填空缺 + assert (members["add"], members["update"], members["keep"]) == (2, 0, 0) + assert [s["path"] for s in members["skipped"]] == ["keys"] + assert report["totals"] == {"tables": 2, "add": 8, "update": 1, "keep": 3, "conflicts": 1, + "skipped": len(orders["skipped"]) + 1} + assert report["guidance"] == {"file": "金额一律按元", "target": None, "action": "replace"} + async with SessionLocal() as session: + assert (await catalog.read_entry(session, remote, "ORDERS")).version == 1 + assert await catalog.read_entry(session, remote, "MEMBERS") is None + assert catalog_jobs.guidance_of(await session.get(DataSource, remote)) is None + + +async def test_import_writes_by_the_rules_and_reports_versions(client): + local, remote = await _pair() + content = await _export(client, local) + r = await client.post(f"{_base(remote)}/import", json={"content": content, "include_guidance": True}, + headers=ACTOR) + assert r.status_code == 200, r.text + report = r.json() + tables = _by(report) + assert tables["ORDERS"]["version"] == 2 and tables["MEMBERS"]["version"] == 1 + assert tables["ORDERS"]["error"] is None + assert report["guidance"]["action"] == "replace" + async with SessionLocal() as session: + orders = await catalog.read_entry(session, remote, "ORDERS") + members = await catalog.read_entry(session, remote, "MEMBERS") + row = await session.get(DataSource, remote) + notes = orders.notes + assert orders.updated_by == "沈岚" + assert notes["label"]["value"] == "订单表" # 冲突：保留目标的 + assert notes["grain"]["value"] == "一行一单" # 推断只填空缺 + assert notes["valid_filter"] == {**SOURCE_ORDERS["valid_filter"]} # 照带：来源、状态不变 + assert notes["dedup"]["status"] == "rejected" and notes["dedup"]["source"] == "llm" + codes = notes["columns"]["STATUS"]["codes"] + assert codes["value"] == {"1": "已支付", "9": "作废"} and codes["complete"] is True + assert notes["columns"]["STATUS"]["label"]["value"] == "状态码" + assert notes["columns"]["AMOUNT"]["measure"] == SOURCE_ORDERS["columns"]["amount"]["measure"] + # 遮罩列：中文名照带，码值不导入 + assert notes["columns"]["CONTACT_PHONE"] == {"label": SOURCE_ORDERS["columns"]["contact_phone"]["label"]} + assert "GIFT_FLAG" not in notes["columns"] and "gift_flag" not in notes["columns"] + [rel] = notes["relations"] + assert rel["id"] == catalog.relation_id("ORDERS", ["MEMBER_ID"], "MEMBERS", ["ID"]) + assert (rel["columns"], rel["to_table"], rel["to_columns"]) == (["MEMBER_ID"], "MEMBERS", ["ID"]) + assert rel["status"] == "confirmed" and rel["source"] == "name" + assert catalog.validate_notes(notes) == [] + assert members.notes["label"]["status"] == "proposed" and "keys" not in members.notes + assert members.notes["columns"]["NICK"]["label"]["status"] == "verified" + assert catalog_jobs.guidance_of(row) == "金额一律按元" + # 再导一次：都对上了，什么都不写；目标已经有起草要求，不覆盖 + again = (await client.post(f"{_base(remote)}/import", json={"content": content, "include_guidance": True})).json() + assert again["totals"]["add"] == again["totals"]["update"] == 0 + assert _by(again)["ORDERS"]["version"] == 2 + assert again["guidance"] == {"file": "金额一律按元", "target": "金额一律按元", "action": "keep"} + + +async def test_target_rejection_beats_file_confirmation_and_guidance_is_optional(client): + local, remote = await _pair() + await _write(remote, "MEMBERS", {"label": make_item("会员", "llm", "rejected")}) + async with SessionLocal() as session: + row = await session.get(DataSource, remote) + await catalog_jobs.save_guidance(session, row, "部署实例自己的要求") + content = await _export(client, local) + content["tables"]["members"]["notes"]["label"] = make_item("会员", "human", "confirmed") + report = (await client.post(f"{_base(remote)}/import", + json={"content": content, "include_guidance": False})).json() + members = _by(report)["MEMBERS"] + [conflict] = members["conflicts"] + assert conflict["path"] == "label" and conflict["target_status"] == "rejected" + assert conflict["file_status"] == "confirmed" and "驳回" in conflict["why"] + assert report["guidance"] == {"file": "金额一律按元", "target": "部署实例自己的要求", "action": "none"} + async with SessionLocal() as session: + entry = await catalog.read_entry(session, remote, "MEMBERS") + assert entry.notes["label"]["status"] == "rejected" + assert catalog_jobs.guidance_of(await session.get(DataSource, remote)) == "部署实例自己的要求" + + +async def test_one_table_failing_does_not_stop_the_others(client, monkeypatch): + local, remote = await _pair() + content = await _export(client, local) + real = catalog.write_entry + + async def flaky(session, source_id, table, notes, **kw): + if table == "ORDERS": + raise catalog.CatalogConflict(table, 99) + return await real(session, source_id, table, notes, **kw) + + monkeypatch.setattr(catalog, "write_entry", flaky) + report = (await client.post(f"{_base(remote)}/import", json={"content": content})).json() + tables = _by(report) + assert tables["ORDERS"]["error"] and "稍后" in tables["ORDERS"]["error"] + assert tables["ORDERS"]["add"] == tables["ORDERS"]["update"] == 0 + assert tables["MEMBERS"]["error"] is None and tables["MEMBERS"]["version"] == 1 + + +async def test_import_waits_for_draft_jobs_and_rejects_bad_files(client): + local, remote = await _pair() + content = await _export(client, local) + async with SessionLocal() as session: + session.add(CatalogDraftJob(id=uuid.uuid4().hex, source_id=remote, status="running")) + await session.commit() + busy = await client.post(f"{_base(remote)}/import", json={"content": content}) + assert busy.status_code == 409 and busy.json()["code"] == "catalog_draft_busy" + # 预览是只读的，照常给 + assert (await client.post(f"{_base(remote)}/import/preview", json={"content": content})).status_code == 200 + bad = [ + "不是对象", + {**content, "format": "something-else"}, + {**content, "version": 2}, + {**content, "tables": []}, + {**content, "tables": {"orders": {"notes": "x"}}}, + {**content, "guidance": 3}, + ] + for body in bad: + r = await client.post(f"{_base(local)}/import/preview", json={"content": body}) + assert r.status_code == 422, body + assert any("一" <= ch <= "鿿" for ch in r.json()["detail"]) + too_long = {**content, "guidance": "长" * (catalog_jobs.GUIDANCE_MAX + 1)} + r = await client.post(f"{_base(local)}/import/preview", json={"content": too_long, "include_guidance": True}) + assert r.status_code == 422 and "起草要求" in r.json()["detail"] + + +async def test_invalid_items_are_skipped_not_fatal(client): + local, remote = await _pair() + content = await _export(client, local) + notes = content["tables"]["orders"]["notes"] + notes["kind"] = {"value": "event", "source": "human", "status": "confirmed"} # 取值不在可选值内 + notes["formula"] = {"value": "sum(amount)", "source": "human", "status": "confirmed"} # 不认识的字段 + notes["columns"]["amount"]["colour"] = {"value": "红", "source": "human", "status": "confirmed"} + report = (await client.post(f"{_base(remote)}/import/preview", json={"content": content})).json() + skipped = {s["path"]: s["why"] for s in _by(report)["ORDERS"]["skipped"]} + assert "明细表" in skipped["kind"] + assert "formula" in skipped["formula"] and "columns.amount.colour" in skipped + + +async def test_import_needs_a_schema(client): + local, _ = await _pair() + content = await _export(client, local) + empty = await _source(f"empty_{uuid.uuid4().hex[:6]}", {}, {}) + r = await client.post(f"{_base(empty)}/import/preview", json={"content": content}) + assert r.status_code == 409 and r.json()["code"] == "schema_missing" + + +async def test_round_trip_into_an_empty_copy_reproduces_the_catalog(client): + local, _ = await _pair() + content = await _export(client, local) + copy_sid = await _source(f"copy_{uuid.uuid4().hex[:6]}", _tables(), {}) + report = (await client.post(f"{_base(copy_sid)}/import", json={"content": json.loads(json.dumps(content)), + "include_guidance": True})).json() + assert report["unmatched_tables"] == ["ghost_table"] + assert all(not t["conflicts"] for t in report["tables"]) + async with SessionLocal() as session: + mine = await catalog.read_entry(session, local, "orders") + theirs = await catalog.read_entry(session, copy_sid, "orders") + # 关系两端都对得上的照带；指向目标那边没有的表的跳过 + expected = copy.deepcopy(mine.notes) + expected["relations"] = [r for r in mine.notes["relations"] if r["to_table"] != "stores"] + del expected["columns"]["gift_flag"] # 表结构里没有这一列 + assert theirs.notes == expected From a5d9ee0a16af9c9ed231b79f7c74380fc37702f7 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:20:02 +0800 Subject: [PATCH 03/39] =?UTF-8?q?=E8=A1=A8=E5=8A=A9=E6=89=8B=EF=BC=9A?= =?UTF-8?q?=E8=A1=A8=E8=AF=A6=E6=83=85=E9=A1=B5=E9=87=8C=E5=92=8C=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E8=80=85=E5=AF=B9=E8=AF=9D=E5=AE=8C=E5=96=84=E4=B8=80?= =?UTF-8?q?=E5=BC=A0=E8=A1=A8=E7=9A=84=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=EF=BC=88POST=20/catalog/{table}/assistant=EF=BC=8CSSE=EF=BC=9A?= =?UTF-8?q?=E6=AD=A3=E6=96=87=E5=A2=9E=E9=87=8F=E3=80=81=E5=8F=AA=E8=AF=BB?= =?UTF-8?q?=E6=9F=A5=E8=AF=A2=E3=80=81=E4=BF=AE=E6=94=B9=E6=8F=90=E6=A1=88?= =?UTF-8?q?=E3=80=81=E5=AE=8C=E6=88=90=E3=80=81=E5=87=BA=E9=94=99=EF=BC=89?= =?UTF-8?q?=EF=BC=9B=E4=BA=A4=E7=BB=99=E6=A8=A1=E5=9E=8B=E8=A1=A8=E7=BB=93?= =?UTF-8?q?=E6=9E=84=E3=80=81=E7=9B=AE=E5=BD=95=E7=8E=B0=E7=8A=B6=EF=BC=88?= =?UTF-8?q?=E6=AF=8F=E9=A1=B9=E7=9A=84=E6=9D=A5=E6=BA=90=E5=92=8C=E7=8A=B6?= =?UTF-8?q?=E6=80=81=EF=BC=89=E3=80=81=E5=85=B3=E8=81=94=E5=85=B3=E7=B3=BB?= =?UTF-8?q?=EF=BC=88=E5=90=AB=E5=88=AB=E7=9A=84=E8=A1=A8=E6=8C=87=E8=BF=87?= =?UTF-8?q?=E6=9D=A5=E7=9A=84=EF=BC=89=E3=80=81=E5=89=96=E6=9E=90=E7=BB=93?= =?UTF-8?q?=E8=AE=BA=E3=80=81=E8=B5=B7=E8=8D=89=E8=A6=81=E6=B1=82=E3=80=81?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E6=AC=A1=E6=95=B0=E3=80=81=E6=9C=80=E8=BF=91?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E8=B5=B7=E8=8D=89=E7=95=99=E7=A9=BA=E7=9A=84?= =?UTF-8?q?=E5=88=97=E5=92=8C=E7=95=8C=E9=9D=A2=E4=B8=8A=E5=BE=85=E4=BF=9D?= =?UTF-8?q?=E5=AD=98=E7=9A=84=E4=BF=AE=E6=94=B9=EF=BC=8C=E9=81=AE=E7=BD=A9?= =?UTF-8?q?=E5=88=97=E5=8F=AA=E7=BB=99=E5=88=97=E5=90=8D=EF=BC=9B=E6=8F=90?= =?UTF-8?q?=E6=A1=88=E6=8C=89=20plan=5Fpatch=20=E6=A0=B8=E5=AF=B9=EF=BC=88?= =?UTF-8?q?=E5=85=AC=E5=BC=8F=E4=B8=8D=E6=94=B6=E3=80=81=E5=88=97=E5=90=8D?= =?UTF-8?q?=E5=BF=85=E9=A1=BB=E5=9C=A8=E8=A1=A8=E7=BB=93=E6=9E=84=E9=87=8C?= =?UTF-8?q?=E3=80=81=E9=81=AE=E7=BD=A9=E5=88=97=E4=B8=8D=E7=BB=99=E7=A0=81?= =?UTF-8?q?=E5=80=BC=E3=80=81=E5=B7=B2=E7=A1=AE=E8=AE=A4=E7=9A=84=E5=90=8C?= =?UTF-8?q?=E5=80=BC=E4=B8=8D=E6=8F=90=EF=BC=89=EF=BC=8C=E5=BE=85=E4=BF=9D?= =?UTF-8?q?=E5=AD=98=E4=BF=AE=E6=94=B9=E9=87=8C=E7=9A=84=E7=A0=81=E5=80=BC?= =?UTF-8?q?=E5=B9=B6=E8=BF=9B=E6=96=B0=E6=8F=90=E7=9A=84=E7=A0=81=E5=80=BC?= =?UTF-8?q?=EF=BC=8C=E5=8F=AA=E4=BA=A4=E7=BB=99=E7=95=8C=E9=9D=A2=E4=B8=8D?= =?UTF-8?q?=E5=86=99=E7=9B=AE=E5=BD=95=EF=BC=9B=E5=8F=AA=E8=AF=BB=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E5=8F=AA=E5=9C=A8=E5=BC=80=E4=BA=86=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=89=96=E6=9E=90=E7=9A=84=E6=BA=90=E4=B8=8A=E7=BB=99=EF=BC=8C?= =?UTF-8?q?=E7=BB=8F=E5=89=96=E6=9E=90=E7=9A=84=E5=8F=AA=E8=AF=BB=E5=AE=88?= =?UTF-8?q?=E5=8D=AB=E5=92=8C=E5=8D=95=E6=9D=A1=E6=97=B6=E9=99=90=E3=80=81?= =?UTF-8?q?=E6=AF=8F=E6=AC=A1=E8=AF=B7=E6=B1=82=E6=9C=80=E5=A4=9A=205=20?= =?UTF-8?q?=E6=9D=A1=E3=80=8150=20=E8=A1=8C=E4=BA=A4=E7=BB=99=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=EF=BC=8C=E7=BB=93=E6=9E=9C=E5=92=8C=E6=8A=A5=E9=94=99?= =?UTF-8?q?=E5=85=88=E9=81=AE=E5=86=8D=E4=BA=A4=EF=BC=9B=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E5=8F=96=E6=B3=95=E3=80=81=E8=AF=B4=E6=B3=95=E5=90=8C=E8=B5=B7?= =?UTF-8?q?=E8=8D=89=EF=BC=88=E7=94=A8=E4=B8=8D=E4=BA=86=20409=EF=BC=89?= =?UTF-8?q?=EF=BC=8C=E4=BF=9D=E7=95=99=E6=80=9D=E8=80=83=E3=80=81=E9=A2=9D?= =?UTF-8?q?=E5=BA=A6=2032768=20=E5=8F=AF=E9=80=80=E5=9B=9E=208192=E3=80=81?= =?UTF-8?q?=E5=8D=95=E6=AC=A1=E6=97=B6=E9=99=90=E5=90=8C=E8=B5=B7=E8=8D=89?= =?UTF-8?q?=EF=BC=8C=E8=BF=98=E6=B2=A1=E8=BE=93=E5=87=BA=E6=97=B6=E7=BD=91?= =?UTF-8?q?=E7=BB=9C=E9=94=99=E8=AF=AF=E5=92=8C=20429=E3=80=815xx=20?= =?UTF-8?q?=E9=87=8D=E8=AF=95=EF=BC=8C=E7=AD=89=E5=BE=85=E6=97=B6=E5=8F=91?= =?UTF-8?q?=20SSE=20=E6=B3=A8=E9=87=8A=E4=BF=9D=E6=B4=BB=EF=BC=8C=E5=81=9C?= =?UTF-8?q?=E6=AD=A2=E6=97=B6=E6=AD=A3=E5=9C=A8=E8=BF=9B=E8=A1=8C=E7=9A=84?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E8=B7=9F=E7=9D=80=E5=81=9C=EF=BC=9BPATCH=20?= =?UTF-8?q?=E4=B8=80=E6=AC=A1=E6=9C=80=E5=A4=9A=20200=20=E9=A1=B9=EF=BC=88?= =?UTF-8?q?=E9=A2=84=E8=A7=88=E5=92=8C=E4=BF=9D=E5=AD=98=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/api/catalog.py | 42 +- backend/app/data/catalog.py | 5 +- backend/app/data/catalog_assistant.py | 878 ++++++++++++++++++++++++ backend/tests/test_catalog_assistant.py | 550 +++++++++++++++ 4 files changed, 1472 insertions(+), 3 deletions(-) create mode 100644 backend/app/data/catalog_assistant.py create mode 100644 backend/tests/test_catalog_assistant.py diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index b3554756..370c412e 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -18,6 +18,8 @@ （data/catalog_review.py） - GET /catalog/export 导出数据目录（JSON 附件）；POST /catalog/import/preview 导入预览（只读）， POST /catalog/import 导入（data/catalog_transfer.py） +- POST /catalog/{table}/assistant 表助手：对话完善这张表的目录（text/event-stream），提案不写目录 + （data/catalog_assistant.py） 规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径 不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。 @@ -40,7 +42,7 @@ from app.api.coded import ( ) from app.api.runs import actor_of from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, introspect, table_versions -from app.data import catalog_review, catalog_transfer +from app.data import catalog_assistant, catalog_review, catalog_transfer from app.data.engine import SnapshotTampered from app.db.base import get_session from app.db.models import DataSource @@ -690,6 +692,44 @@ async def save_catalog_patch(source_id: str, table: str, payload: CatalogPatchIn return await _detail(session, source, key, entry) +class AssistantMessageIn(BaseModel): + role: Literal["user", "assistant"] + content: str = Field(max_length=catalog_assistant.MESSAGE_MAX) + + +class AssistantIn(BaseModel): + #: 整段对话（服务端不存）；没有使用者的话时助手先开口 + messages: list[AssistantMessageIn] = Field(default_factory=list, max_length=catalog_assistant.MAX_MESSAGES) + #: 界面上还没保存的修改，让模型知道 + pending: list[CatalogPatchChange] = Field(default_factory=list, max_length=catalog_assistant.MAX_PENDING) + + +@router.post("/{source_id}/catalog/{table}/assistant") +async def catalog_table_assistant(source_id: str, table: str, payload: AssistantIn, + session: AsyncSession = Depends(get_session)) -> Any: + """表助手：和使用者对话完善这张表的目录，返回 text/event-stream（事件见 data/catalog_assistant.py）。提案只交给界面， + 不写目录；保存走 /catalog/{table}/patch。开始前：表不在 404；对话不合规 422；起草模型用不了 409 + catalog_draft_model_unavailable（说法同起草任务）。""" + from fastapi.responses import StreamingResponse + + row, source = await _resolved(session, source_id) + key = catalog.resolve_table_name(source.schema_cache, table) + if key is None: + raise HTTPException(404, f"数据源中没有表 {table}，可能已被删除或尚未探查结构") + messages = [m.model_dump() for m in payload.messages] + if problem := catalog_assistant.conversation_problem(messages): + raise HTTPException(422, problem) + try: + model, model_id = await catalog_assistant.resolve_model(session) + except catalog.CatalogModelUnavailable as e: + raise CodedHTTPException(409, str(e), CATALOG_DRAFT_MODEL_UNAVAILABLE) from e + context = await catalog_assistant.prepare(session, row, source, key, + pending=[c.model_dump() for c in payload.pending]) + return StreamingResponse(catalog_assistant.stream(model, model_id, context, messages), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}) + + @router.get("/{source_id}/catalog/{table}/impact") async def catalog_table_impact(source_id: str, table: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]: diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index cdef28cf..40bdaab8 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -860,8 +860,9 @@ async def save_human_edit(session: AsyncSession, source_id: str, table: str, sub #: 新增关联关系的路径：relations.new，值写两端；编号按两端算（relation_id），同一条关系不论谁提都是同一个编号 NEW_RELATION_PATH = "relations.new" -#: 一条提案最多几项：用户说的是一两条事实，几十项的「提案」多半是模型在重写整张表 -PATCH_MAX_CHANGES = 20 +#: 一条提案最多几项（预览、保存都按它）。对话里的纠正通常是一两条事实；表助手（data/catalog_assistant.py）会逐列补中文名、 +#: 含义，界面上跨几轮攒下的待保存修改一次保存，宽表上常常几十上百项，所以放到 200 +PATCH_MAX_CHANGES = 200 _REASON_MAX = 500 #: 看起来像计算公式的文字：「转化率=下单数/访问数」「= SUM(amount)」。公式进口径卡，目录只记数据事实。 #: 不认减号：「上线日期=2024-01-01」不是公式。有效记录条件本来就是 SQL 条件，不查它。 diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py new file mode 100644 index 00000000..475a6a5e --- /dev/null +++ b/backend/app/data/catalog_assistant.py @@ -0,0 +1,878 @@ +"""表助手：在表详情页和使用者对话，一起完善一张表的数据目录（POST /catalog/{table}/assistant，SSE）。 + +**为什么要有它。** 模型起草只看表结构，缩写列名、状态码的意思常常拿不准而留空；审阅时人知道答案，却要一项一项填。表助手 +把这张表的结构、目录现状（每项的来源和状态）、关联关系、剖析结果、起草要求、使用次数、上次起草留空的列交给模型，让它主动 +问拿不准的地方；使用者说出一条业务事实，它整理成成批的修改提案（形状同助手流里的 catalog_patch），使用者在界面上逐项看过、 +一次保存（PATCH /catalog/{table}/patch，记为人工确认）。 + +**只改目录、不改业务库。** 模型只有两个工具： +- propose_changes：一批修改提案。按 catalog.plan_patch 核对（取值格式、列名必须在表结构里、公式不收），遮罩列不给码值，已经 + 是这个值又确认过的不提；核对过的作为 proposal 事件交给界面，**不写目录**。没收下的项和原因交回模型（不带任何取值）。 + 界面上还没保存的修改（pending）里同一列的码值并进新提的码值：界面上同一路径以后一次为准，不能把先提的码值丢了。 +- run_query：只读查询，只有数据源开了数据剖析才给。经剖析的只读守卫和单条时限（catalog_profile._Runner： + guard.check(readonly=True)，单条时限取剖析设置和数据源查询时限中较小的一个），每次请求最多 MAX_QUERIES 条（守卫拒掉的 + 也算），最多 QUERY_ROWS 行交给模型、SHOWN_ROWS 行交给界面。结果交给模型、界面之前先遮（engine/masking.mask_query_result： + 取自遮罩列的结果列整列换成「已遮罩」，改了名字的也认；这一次请求里前面遮掉的值按值换），报错原话里的值也换。没开剖析时 + 提示词里写明只能根据表结构和目录推断。 + +**遮罩列只给列名。** 表结构里遮罩的列只写列名和类型（不给注释），目录里这些列的码值、带着取值的备注一律不给（剖析的备注里 +有取值范围）；界面上还没保存的修改里这些列的码值也不给。 + +**模型。** 取模型的顺序同起草（data/draft_model.py：起草模型 → 助手的模型 → 默认接入），用不了时的说法也同起草 +（先过 catalog.resolve_draft_model，接口回 409 catalog_draft_model_unavailable）。保留思考、温度 0，额度 +catalog.MODEL_MAX_TOKENS，服务方嫌大时退回 catalog.MODEL_FALLBACK_MAX_TOKENS（providers/budget.BudgetFallback）；单次调用 +最多等 catalog.MODEL_TIMEOUT_S（同起草）。网络错误、429、5xx 在还没输出任何东西时重试（最多 len(RETRY_DELAYS_S) 次）。 +一次请求最多调用 MAX_ROUNDS 次模型（查询、提案之后接着答）；倒数第二轮的工具结果里提醒它直接回答。 + +**事件**（每个事件一行 data: ）： +- {"type": "text", "delta"}：回答正文的增量。几轮之间补一个空行； +- {"type": "query", "sql", "columns", "rows"（前 SHOWN_ROWS 行）, "row_count", "truncated", "error"}：跑了一条只读查询； +- {"type": "proposal", "source", "source_id", "table", "table_label", "version", "changes": [{path, label, before, + before_status, after, value, reason, state, note?}]}：一批核对过的修改提案； +- {"type": "done", "usage": {input_tokens, output_tokens}, "queries"}； +- {"type": "error", "message", "code"}：中途出错，流到此为止（之后没有 done）。code：assistant_model_error 调用模型出错、 + assistant_truncated 输出到了额度上限被截断、assistant_timeout 单次调用超过时限、assistant_rounds 调用次数用完还没答完、 + assistant_failed 意料之外的错误。 +等模型的时候每 HEARTBEAT_S 秒发一行 SSE 注释（「: keep-alive」，不是事件），免得中间的代理断开。 + +**对话不存服务端**：每次请求带上整段对话（最多 MAX_MESSAGES 条、每条最多 MESSAGE_MAX 字）。查询结果只在这一次请求里 +看得到，提示词里让模型把要用的结论写进回答。 + +写给模型的文字都在名字带 _SYSTEM / _PROMPT 的常量里；这里的 ERROR_* 是给人看的。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +import json +import logging +from collections.abc import AsyncIterator, Iterable, Mapping +from dataclasses import dataclass, field +from datetime import date, datetime +from decimal import Decimal +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import catalog, catalog_jobs, catalog_profile +from app.data.catalog_review import SOURCE_LABEL, slot_path +from app.data.draft_model import DRAFT_MODEL_PLACE +from app.data.engine import SnapshotTampered, masked_columns + +logger = logging.getLogger(__name__) + +#: 对话最多几条、每条最多几个字；界面上待保存的修改最多几项（同 PATCH 的上限） +MAX_MESSAGES = 40 +MESSAGE_MAX = 4000 +MAX_PENDING = catalog.PATCH_MAX_CHANGES +#: 每次请求最多调用几次模型、最多几条只读查询 +MAX_ROUNDS = 8 +MAX_QUERIES = 5 +#: 一条查询最多几行交给模型、几行交给界面；交给模型时每格最多几个字、一条结果最多几个字 +QUERY_ROWS = 50 +SHOWN_ROWS = 10 +CELL_MAX = 300 +TOOL_CHARS = 12_000 +#: 网络错误、429、5xx 重试前等几秒（依次取用，个数就是最多重试几次） +RETRY_DELAYS_S = (2.0, 5.0) +#: 等模型的时候每隔几秒发一行 SSE 注释 +HEARTBEAT_S = 10.0 +#: 背景里「目录里还没有中文名和含义的列」最多列几个；别的表指过来的关联关系最多列几条 +UNNAMED_SHOWN = 30 +INCOMING_SHOWN = 30 + +QUERY_TOOL = "run_query" +PROPOSE_TOOL = "propose_changes" + +#: 错误事件的机读码 +CODE_MODEL = "assistant_model_error" +CODE_TRUNCATED = "assistant_truncated" +CODE_TIMEOUT = "assistant_timeout" +CODE_ROUNDS = "assistant_rounds" +CODE_FAILED = "assistant_failed" + +# ---- 给人看的 +ERROR_MODEL = ("调用模型「{model}」失败：{reason}。请稍后重试；如持续失败，请在{place}中换一个模型，" + "或到「设置 → 模型接入」测试该模型") +ERROR_TRUNCATED = "{cut}，回答没有写完。通常是思考占满了输出额度，可以把问题拆小一点再问，或在{place}中换一个模型" +ERROR_TIMEOUT = "模型 {seconds} 秒内没有回答完，已停止这一轮。可以把问题拆小一点再问，或在{place}中换一个模型" +ERROR_ROUNDS = "这一轮已经调用了 {n} 次模型仍没有答完，已停止。可以换个问法，或者分几次问" +ERROR_FAILED = "表助手出错：{line}" +#: 对话的最后一条是助手说的（接口 422） +LAST_NOT_USER = "对话的最后一条应当是使用者的话，请重新发送" +#: 起草模型刚才还能用、建流式模型时却用不了（期间设置被改了）时的说法 +ERROR_NO_MODEL = "起草模型暂时无法使用，请稍后重试，或到「设置 → 模型接入」检查" +#: 查询事件里的报错 +QUERY_TAMPERED = "数据源「{name}」的{error}。请联系管理员检查数据目录" + +# ---- 写给模型的 +ASSISTANT_SYSTEM = """你是 AgentLab 的「表助手」，和使用者一起完善一张表的数据目录：表和列的中文名、含义、单位、度量类型、码值、\ +粒度、表类型、业务主键、业务日期、有效记录条件、去重规则、关联关系。 + +规矩： +- 你只改数据目录，不改业务库：不能写数据、建表、改结构。 +- 只记数据事实，不写计算公式：「转化率=下单数/访问数」这类指标怎么算的说法属于口径卡，不进数据目录。 +- 拿不准就问，不要猜。一次只问一两件最要紧的事，问得具体（例如「status 为 9 的行是不是作废的单子？」）。目录里留空的列、\ +标着「推断」的项是最值得问的。 +- 使用者说出一条业务事实时，用 propose_changes 把它整理成修改提案，reason 里引用使用者的原话（例如「使用者说：9 是作废」）。\ +一次可以提多项。提案只是建议：使用者在界面上看过、保存之后才写进数据目录，记为人工确认。 +- 表名、列名照原样写，只能用下面表结构里有的列。 +- 标着「已确认」的项由人工确认过，使用者没有明确要改就不要提修改；标着「已驳回」的项使用者驳回过，不要再提。 +- 「界面上还没保存的修改」是使用者已经接受、还没保存的提案：不要重复提；要改其中一项就再提一次，写完整的新值。 +- 遮罩的列只给了列名：不要猜它的取值，也不要给它写码值。 +- 回答用中文，简短直接。查询结果只在这一轮看得到，后面要用的结论请写进回答里。 + +修改提案的写法（propose_changes 的 changes，每项 path、value、reason）： +- path：表级项直接写字段名（label 中文名、description 说明、grain 粒度、keys 业务主键的列名列表、kind 表类型 \ +fact/dimension/snapshot/log/config、business_date {"column","rule","timezone"}、valid_filter 有效记录的 SQL 条件、dedup 去重\ +规则）；列级项写 columns.<列名>.<字段>（label、meaning 含义、unit 单位、measure 度量类型 \ +flow/stock/ratio/identifier/status/attribute、codes 码值 {"码":"含义"}）；已有的关联关系写 relations.<编号>；新增关联关系\ +写 relations.new，value 写 {"columns":[本表列],"to_table":"目标表","to_columns":[目标列],\ +"cardinality":"many_to_one/one_to_one/one_to_many"}。 +- codes 只写要补充或更正的码，原有的码会保留；valid_filter 要写成完整的条件（原有条件也要带上）。""" + +#: 查询工具的说明（开了剖析时接在系统提示后面）、没开剖析时的那句 +ASSISTANT_QUERY_PROMPT = """只读查询：run_query 在这个数据源上执行一条只读 SQL（{dialect} 的写法），最多返回 {rows} 行，\ +这一轮最多 {limit} 条。用它核对码值的取值和分布、日期范围、关联对不对得上；不要取明细的个人信息。遮罩列的值显示为「已遮罩」。""" +ASSISTANT_NO_QUERY_PROMPT = "这个数据源没开剖析，不能查数据，只能根据表结构和目录推断。" +ASSISTANT_DISABLED_PROMPT = "这个数据源已停用，不能查数据，只能根据表结构和目录推断。" +#: 使用者还没说话（刚打开表助手）时补的那句 +ASSISTANT_OPENING_PROMPT = ("（使用者刚打开表助手，还没有说话。）请用一两句话说明这张表的目录现在最缺什么，然后就最要紧的一两项" + "向使用者提问；先不要提修改提案。") +#: 对话从助手的开场白开始时，补在最前面的一句（各家的接口都要求从使用者这一边开始） +ASSISTANT_OPENED_PROMPT = "（使用者打开了表助手。）" + +#: 背景各段的写法 +ASSISTANT_PROMPT_LABELS = { + "head": "下面是要完善的表。", + "table": "数据源「{source}」（{kind}），表 {table}{view}", + "view": "（视图）", + "comment": "数据库注释：{text}", + "system": "系统说明：{text}", + "usage": "这张表被运行用过 {n} 次。", + "description": "数据源说明：{text}", + "guidance": "数据源的起草要求（使用者补充的背景和要求）：{text}", + "structure": "表结构：", + "column": "- {name} {type}{marks}", + "pk": "主键", + "not_null": "非空", + "col_comment": "注释：{text}", + "masked": "遮罩列：只给列名，不给任何取值", + "unique": "唯一约束：{cols}", + "fk": "外键：{cols} → {target}", + "catalog": "这张表现在的数据目录（每项写明来源和状态；路径就是修改提案里的 path）：", + "catalog_empty": "这张表还没有数据目录。", + "item": "- {path}（{label}）：{value}〔{source}，{status}〕", + "item_note": " 备注：{text}", + "relation": "{columns} → {to_table}.{to_columns}{extra}", + "relation_extra": "（{text}）", + "cardinality": "基数 {value}", + "coverage": "覆盖率 {value}", + "incoming": "别的表指向这张表的关联关系：", + "incoming_item": "- {table}.{columns} → {target}.{to_columns}〔{source}，{status}〕", + "blank": "最近一次起草时模型拿不准、留空的列：{cols}（共 {n} 列）", + "unnamed": "目录里还没有中文名和含义的列：{cols}（共 {n} 列{more}）", + "unnamed_more": "，这里列出前 {n} 个", + "pending": "界面上还没保存的修改（使用者已经接受、还没保存）：", + "pending_item": "- {path}：{value}（依据：{reason}）", + "status_proposed": "推断", + "status_verified": "有确证", + "status_confirmed": "已确认", + "status_rejected": "已驳回", + "field_relation": "关联关系", +} + +#: 工具结果的写法（交回模型） +ASSISTANT_TOOL_PROMPT = { + "proposed": "已把 {n} 项修改交给使用者确认（使用者保存后才写进数据目录）：{paths}", + "nothing": "没有可以交给使用者的修改。", + "dropped": "没有收下的项：{reasons}", + "masked_codes": "列 {column} 在数据源设置中被遮罩，不能给它写码值", + "same": "{where}已经是这个值、也确认过了，不用再提", + "no_changes": "提案里没有要修改的项", + "query_failed": "查询失败：{error}", + "query_capped": "未执行：这一轮最多查 {n} 条，不能再查了。请根据已经查到的结果回答", + "query_disabled": "未执行：这个数据源没开数据剖析，不能查数据", + "unknown_tool": "没有名为 {name} 的工具，可用的工具：{tools}", + "last_round": "（这一轮只剩最后一步了：请直接回答使用者，不要再调用工具。）", + "empty_sql": "未执行：没有给出 SQL", +} + +QUERY_TOOL_PROMPT: dict[str, Any] = { + "type": "function", + "function": { + "name": QUERY_TOOL, + "description": "在这个数据源上执行一条只读 SQL 查询，返回列名和前几行（遮罩列的值显示为「已遮罩」）。", + "parameters": { + "type": "object", + "properties": {"sql": {"type": "string", "description": "一条只读的 SELECT 语句"}}, + "required": ["sql"], + }, + }, +} +PROPOSE_TOOL_PROMPT: dict[str, Any] = { + "type": "function", + "function": { + "name": PROPOSE_TOOL, + "description": "把使用者说出的业务事实整理成一批数据目录修改提案，交给使用者确认后保存。不会直接写进数据目录。", + "parameters": { + "type": "object", + "properties": { + "changes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "path": {"type": "string", "description": "要改的项，写法见系统提示"}, + "value": {"description": "新的值：文字、列名列表、码值对照或关联关系"}, + "reason": {"type": "string", "description": "一句话依据，引用使用者的原话"}, + }, + "required": ["path", "value", "reason"], + }, + }, + }, + "required": ["changes"], + }, + }, +} + + +# ========================================================================== +# 模型 +# ========================================================================== + + +async def resolve_model(session: AsyncSession) -> tuple[Any, str]: + """(模型, 模型 id)。能不能用、用不了时的说法和起草完全一致：先过 catalog.resolve_draft_model（起草任务建任务时也是这样 + 查的），用不了抛 catalog.CatalogModelUnavailable。再按同一个接入、模型建流式对话用的模型：保留思考、温度 0，大小两份额度 + （BudgetFallback），单次时限同起草。只建模型对象，不发请求。""" + from app.data.draft_model import DraftModelUnavailable, draft_unavailable, resolve_draft_provider + from app.providers.budget import BudgetFallback + from app.providers.factory import ModelSpec, ProviderNotConfigured, build_chat_model + + await catalog.resolve_draft_model(session) + try: + provider, choice = await resolve_draft_provider(session, none_reason=ERROR_NO_MODEL, mock_reason=ERROR_NO_MODEL) + except DraftModelUnavailable as e: + raise catalog.CatalogModelUnavailable(str(e)) from e + spec = ModelSpec(provider=choice.provider, model=choice.model, max_tokens=catalog.MODEL_MAX_TOKENS, temperature=0, + timeout=int(catalog.MODEL_TIMEOUT_S)) + try: + big = build_chat_model(provider, spec) + small = build_chat_model(provider, spec.model_copy(update={"max_tokens": catalog.MODEL_FALLBACK_MAX_TOKENS})) + except ProviderNotConfigured as e: + raise catalog.CatalogModelUnavailable(draft_unavailable(e, choice.origin)) from e + model_id = spec.model or provider.default_model or "" + return BudgetFallback(big, small, limits=(catalog.MODEL_MAX_TOKENS, catalog.MODEL_FALLBACK_MAX_TOKENS)), model_id + + +# ========================================================================== +# 对话 +# ========================================================================== + + +def conversation_problem(messages: Iterable[Mapping[str, Any]]) -> str | None: + """对话本身的问题（接口 422）：使用者说过话、最后一条却是助手说的。没有使用者的话是「刚打开表助手」，不算问题。""" + said = [m for m in messages if str(m.get("content") or "").strip()] + if any(m.get("role") == "user" for m in said) and said[-1].get("role") != "user": + return LAST_NOT_USER + return None + + +def _history(messages: Iterable[Mapping[str, Any]]) -> list[Any]: + """对话 → 模型的消息：空的不要，同一方接连说的并成一条；从助手的开场白开始的，前面补一句使用者打开了表助手；使用者还没 + 说话的，补一句开场的请求。""" + from langchain_core.messages import AIMessage, HumanMessage + + merged: list[tuple[str, str]] = [] + for m in messages: + text = str(m.get("content") or "").strip() + if not text: + continue + role = "user" if m.get("role") == "user" else "assistant" + if merged and merged[-1][0] == role: + merged[-1] = (role, f"{merged[-1][1]}\n\n{text}") + else: + merged.append((role, text)) + out: list[Any] = [] + if merged and merged[0][0] == "assistant": + out.append(HumanMessage(content=ASSISTANT_OPENED_PROMPT)) + for role, text in merged: + out.append(HumanMessage(content=text) if role == "user" else AIMessage(content=text)) + if not any(role == "user" for role, _ in merged): + out.append(HumanMessage(content=ASSISTANT_OPENING_PROMPT)) + return out + + +# ========================================================================== +# 背景 +# ========================================================================== + + +def _value_text(value: Any) -> str: + return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str) + + +def _status_text(status: Any) -> str: + return ASSISTANT_PROMPT_LABELS.get(f"status_{status}", str(status)) + + +def _source_text(source: Any) -> str: + return SOURCE_LABEL.get(str(source), str(source)) + + +def _relation_line(rel: Mapping[str, Any]) -> str: + labels = ASSISTANT_PROMPT_LABELS + extra = [] + if rel.get("cardinality"): + extra.append(labels["cardinality"].format(value=rel["cardinality"])) + coverage = rel.get("coverage") + if isinstance(coverage, (int, float)) and not isinstance(coverage, bool): + extra.append(labels["coverage"].format(value=f"{coverage:.0%}")) + return labels["relation"].format( + columns=", ".join(str(c) for c in rel.get("columns") or []), to_table=rel.get("to_table"), + to_columns=", ".join(str(c) for c in rel.get("to_columns") or []), + extra=labels["relation_extra"].format(text="，".join(extra)) if extra else "") + + +def _touches_masked(key: catalog.Slot, item: Mapping[str, Any], masked: set[str]) -> bool: + """这一项说的是不是遮罩列：列级的项看列名，业务主键、业务日期看指着的列，关联关系看本表这一端。""" + if key[0] == "c": + return key[1].lower() in masked + if key[0] == "r": + return any(str(c).lower() in masked for c in item.get("columns") or []) + value = item.get("value") + if key == ("t", "business_date") and isinstance(value, Mapping): + return str(value.get("column") or "").lower() in masked + if key == ("t", "keys") and isinstance(value, list): + return any(str(c).lower() in masked for c in value) + return False + + +def _catalog_lines(notes: Mapping[str, Any] | None, masked: set[str]) -> list[str]: + """这张表的目录，每项一行：路径、字段、值、来源和状态，有备注的另起一行。遮罩列上的码值不给，说到遮罩列的项不给备注 + （剖析的备注里有取值范围）。""" + labels = ASSISTANT_PROMPT_LABELS + lines: list[str] = [] + for key, item in catalog._slots(notes).items(): + hidden = _touches_masked(key, item, masked) + if hidden and key[0] == "c" and key[2] == "codes": + continue + if key[0] == "r": + label, value = labels["field_relation"], _relation_line(item) + else: + label = (catalog.TABLE_FIELD_LABEL if key[0] == "t" else catalog.COLUMN_FIELD_LABEL).get(key[-1], key[-1]) + value = _value_text(item.get("value")) + lines.append(labels["item"].format(path=slot_path(key), label=label, value=value, + source=_source_text(item.get("source")), + status=_status_text(item.get("status")))) + note = item.get("note") + if isinstance(note, str) and note.strip() and not hidden: + lines.append(labels["item_note"].format(text=note.strip())) + return lines + + +def _structure_lines(meta: Mapping[str, Any], masked: set[str], *, system: bool) -> list[str]: + labels = ASSISTANT_PROMPT_LABELS + pk = {str(c) for c in meta.get("primary_key") or []} + lines = [labels["structure"]] + for col in meta.get("columns") or []: + if not isinstance(col, Mapping) or col.get("name") is None: + continue + name = str(col["name"]) + marks: list[str] = [] + if name.lower() in masked: + marks.append(labels["masked"]) + else: + if name in pk: + marks.append(labels["pk"]) + if not col.get("nullable", True): + marks.append(labels["not_null"]) + text = str(col.get("comment") or "").strip() + if text: + marks.append((labels["system"] if system else labels["col_comment"]).format(text=text)) + lines.append(labels["column"].format(name=name, type=col.get("type") or "", + marks=("，" + "，".join(marks)) if marks else "")) + for group in meta.get("unique") or []: + if isinstance(group, list) and group: + lines.append(labels["unique"].format(cols="、".join(str(c) for c in group))) + for fk in meta.get("foreign_keys") or []: + if isinstance(fk, Mapping) and fk.get("columns") and fk.get("to_table"): + target = ", ".join(f"{fk['to_table']}.{c}" for c in fk.get("to_columns") or []) or str(fk["to_table"]) + lines.append(labels["fk"].format(cols=", ".join(str(c) for c in fk["columns"]), target=target)) + return lines + + +async def _last_blank(session: AsyncSession, source_id: str, table: str) -> tuple[int, list[str]]: + """最近一次结束于 ok / partial 的起草里这张表模型留空的列：(几列, 最多 5 个列名)。停止、中断的那次不算（那次没起草完， + 留空数是 0）。没有是 (0, [])。""" + from app.db.models import CatalogDraftJob, CatalogDraftTask + + row = (await session.execute( + select(CatalogDraftTask.blank_columns, CatalogDraftTask.blank_sample) + .join(CatalogDraftJob, CatalogDraftJob.id == CatalogDraftTask.job_id) + .where(CatalogDraftJob.source_id == source_id, CatalogDraftTask.table_name == table, + CatalogDraftTask.status.in_(("ok", "partial"))) + .order_by(CatalogDraftJob.created_at.desc(), CatalogDraftTask.seq).limit(1) + )).first() + if row is None: + return 0, [] + return int(row[0] or 0), [str(c) for c in row[1] or []] + + +@dataclass +class AssistantContext: + """一次请求要用到的东西，开流之前读好（流里不再用请求的会话）。""" + + source_id: str + source_name: str + kind: str + table: str + tables: dict[str, Any] + #: 读表结构、发查询用的源（上传源是绑定当前快照的视图） + query_source: Any + options: dict[str, Any] + masked: set[str] + can_query: bool + settings: catalog_profile.ProfileSettings + system: str + pending_codes: dict[str, dict[str, Any]] = field(default_factory=dict) + + def tools(self) -> list[dict[str, Any]]: + return [QUERY_TOOL_PROMPT, PROPOSE_TOOL_PROMPT] if self.can_query else [PROPOSE_TOOL_PROMPT] + + +async def prepare(session: AsyncSession, row: Any, source: Any, table: str, + pending: Iterable[Mapping[str, Any]] = ()) -> AssistantContext: + """读好这次请求要用的背景：表结构、目录、别的表指过来的关联关系、使用次数、起草要求、留空的列、待保存的修改，拼成系统 + 提示。row 是数据源记录（遮罩、剖析开关、起草要求按它），source 是读表结构和发查询用的源。table 是表结构里的键。""" + labels = ASSISTANT_PROMPT_LABELS + cache = getattr(source, "schema_cache", None) or {} + tables = cache.get("tables") or {} + meta = tables.get(table) or {} + options = dict(getattr(row, "options", None) or {}) + masked = {c.lower() for c in masked_columns(options)} + settings = catalog_profile.profile_settings(options) + enabled = getattr(row, "enabled", True) is not False + can_query = settings.enabled and enabled + system_notes = catalog.system_notes_source(source) + entries = await catalog.read_catalog(session, source.id) + entry = entries.get(table) + notes = entry.notes if entry else {} + usage = (await catalog.table_usage(session, source)).get(table, 0) + blank_n, blank_cols = await _last_blank(session, source.id, table) + + lines = [labels["head"], labels["table"].format(source=row.name, kind=row.kind, table=table, + view=labels["view"] if meta.get("is_view") else "")] + comment = str(meta.get("comment") or "").strip() + if comment: + lines.append((labels["system"] if system_notes else labels["comment"]).format(text=comment)) + lines.append(labels["usage"].format(n=usage)) + description = str(getattr(row, "description", "") or "").strip() + if description: + lines.append(labels["description"].format(text=description)) + if guidance := catalog_jobs.guidance_of(row): + lines.append(labels["guidance"].format(text=guidance)) + lines += ["", *_structure_lines(meta, masked, system=system_notes), ""] + current = _catalog_lines(notes, masked) + lines += [labels["catalog"], *current] if current else [labels["catalog_empty"]] + incoming = [] + for other, other_entry in sorted(entries.items()): + if other == table: + continue + for key, rel in catalog._slots(other_entry.notes).items(): + if key[0] == "r" and rel.get("to_table") == table: + incoming.append(labels["incoming_item"].format( + table=other, columns=", ".join(str(c) for c in rel.get("columns") or []), target=table, + to_columns=", ".join(str(c) for c in rel.get("to_columns") or []), + source=_source_text(rel.get("source")), status=_status_text(rel.get("status")))) + if incoming: + lines += ["", labels["incoming"], *incoming[:INCOMING_SHOWN]] + named = catalog.visible_notes(notes).get("columns") or {} + columns = [str(c["name"]) for c in meta.get("columns") or [] if isinstance(c, Mapping) and c.get("name") is not None] + unnamed = [c for c in columns if not any(f in (named.get(c) or {}) for f in ("label", "meaning"))] + blank = [c for c in blank_cols if c in unnamed] + if blank_n and blank: + lines += ["", labels["blank"].format(cols="、".join(blank), n=blank_n)] + if unnamed: + more = labels["unnamed_more"].format(n=UNNAMED_SHOWN) if len(unnamed) > UNNAMED_SHOWN else "" + lines.append(labels["unnamed"].format(cols="、".join(unnamed[:UNNAMED_SHOWN]), n=len(unnamed), more=more)) + pending = list(pending) + pending_codes: dict[str, dict[str, Any]] = {} + shown_pending: list[str] = [] + for change in pending: + path = str(change.get("path") or "").strip() + value = change.get("value") + try: + key = catalog.parse_path(path) + except catalog.CatalogPathError: + continue + if key[0] == "c" and key[2] == "codes": + if key[1].lower() in masked: + continue + if isinstance(value, Mapping): + pending_codes[path] = dict(value) + reason = str(change.get("reason") or "").strip() + shown_pending.append(labels["pending_item"].format(path=path, value=_value_text(value), reason=reason or "—")) + if shown_pending: + lines += ["", labels["pending"], *shown_pending] + if can_query: + tools = ASSISTANT_QUERY_PROMPT.format(dialect=row.kind, rows=QUERY_ROWS, limit=MAX_QUERIES) + else: + tools = ASSISTANT_NO_QUERY_PROMPT if enabled or not settings.enabled else ASSISTANT_DISABLED_PROMPT + system = "\n\n".join([ASSISTANT_SYSTEM, tools, "\n".join(lines)]) + return AssistantContext(source_id=source.id, source_name=row.name, kind=str(row.kind or ""), table=table, + tables=tables, query_source=source, options=options, masked=masked, can_query=can_query, + settings=settings, system=system, pending_codes=pending_codes) + + +# ========================================================================== +# 工具 +# ========================================================================== + + +def _cell(value: Any, limit: int | None = None) -> Any: + """一格交出去的样子：数、布尔、空照原样，别的写成文字（日期、Decimal……），长的截断。""" + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, (datetime, date)): + text = value.isoformat() + elif isinstance(value, Decimal): + text = str(value) + elif isinstance(value, (bytes, bytearray)): + text = f"<{len(value)} 字节>" + else: + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str) + if limit is not None and len(text) > limit: + return text[:limit] + "…" + return text + + +def _rows(rows: Any, n: int, limit: int | None) -> list[list[Any]]: + out: list[list[Any]] = [] + for row in (rows or [])[:n]: + cells = row if isinstance(row, list) else list(row.values()) if isinstance(row, dict) else [row] + out.append([_cell(v, limit) for v in cells]) + return out + + +class _Tools: + """一次请求里的工具：查询次数、这一次遮掉的值（后面的查询按值换）。""" + + def __init__(self, ctx: AssistantContext) -> None: + from app.engine.masking import SeenValues + + self.ctx = ctx + self.queries = 0 + self.seen = SeenValues() + + # ---------------------------------------------------------------- 查询 + + async def query(self, sql: Any) -> tuple[dict[str, Any] | None, str]: + """(给界面的 query 事件, 交回模型的结果)。""" + prompts = ASSISTANT_TOOL_PROMPT + if not self.ctx.can_query: + return None, prompts["query_disabled"] + if self.queries >= MAX_QUERIES: + return None, prompts["query_capped"].format(n=MAX_QUERIES) + sql = sql.strip() if isinstance(sql, str) else "" + if not sql: + return None, prompts["empty_sql"] + self.queries += 1 + mask = masked_columns(self.ctx.options) + known = await asyncio.to_thread(self.seen.pooled) + known = known if known.strong else None + error: str | None = None + result: Any = None + settings = dataclasses.replace(self.ctx.settings, max_queries=1) + try: + runner = catalog_profile._Runner(self.ctx.query_source, settings, + catalog_profile.SqlDialect(self.ctx.kind)) + result = await runner(sql, max_rows=QUERY_ROWS) + except catalog_profile._Skip as e: + error = e.detail + except catalog_profile._Stop as e: + error = str(e) + except SnapshotTampered as e: + error = QUERY_TAMPERED.format(name=self.ctx.source_name, error=e) + except ValueError as e: # 不支持的数据库类型 + error = str(e) + if error is not None: + if mask or known is not None: + from app.engine.masking import mask_error_text + + error = mask_error_text(error, sql, known) + event = {"type": "query", "sql": sql, "columns": [], "rows": [], "row_count": 0, "truncated": False, + "error": error} + return event, prompts["query_failed"].format(error=error) + payload = result.to_payload() + if mask or known is not None: + from app.engine.masking import mask_query_result + + schema = getattr(self.ctx.query_source, "schema_cache", None) + done = await asyncio.to_thread(mask_query_result, payload, mask, self.ctx.kind, known, schema=schema) + payload = done.payload + await asyncio.to_thread(self.seen.add, done.secrets) + columns = [str(c) for c in payload.get("columns") or []] + rows = payload.get("rows") or [] + event = {"type": "query", "sql": str(payload.get("sql") or sql), "columns": columns, + "rows": _rows(rows, SHOWN_ROWS, None), "row_count": int(payload.get("row_count") or len(rows)), + "truncated": bool(payload.get("truncated")), "error": None} + given = {"columns": columns, "rows": _rows(rows, QUERY_ROWS, CELL_MAX), "row_count": event["row_count"], + "truncated": event["truncated"]} + text = json.dumps(given, ensure_ascii=False, default=str) + while len(text) > TOOL_CHARS and given["rows"]: + given["rows"] = given["rows"][:max(1, len(given["rows"]) // 2)] if len(given["rows"]) > 1 else [] + given["rows_shown"] = len(given["rows"]) + text = json.dumps(given, ensure_ascii=False, default=str) + return event, text + + # ---------------------------------------------------------------- 提案 + + def _with_pending(self, change: Any) -> Any: + """界面上还没保存的修改里同一列的码值，并进新提的码值（新提的同一个码为准）。""" + if not isinstance(change, Mapping): + return change + path = str(change.get("path") or "").strip() + before = self.ctx.pending_codes.get(path) + value = change.get("value") + if before is None or not isinstance(value, Mapping): + return change + return {**change, "value": {**before, **value}} + + async def propose(self, changes: Any) -> tuple[dict[str, Any] | None, str]: + """(给界面的 proposal 事件, 交回模型的结果)。提案不写目录。""" + from app.db.base import SessionLocal + + prompts = ASSISTANT_TOOL_PROMPT + if not isinstance(changes, list) or not changes: + return None, prompts["no_changes"] + changes = [self._with_pending(c) for c in changes] + async with SessionLocal() as session: + entry = await catalog.read_entry(session, self.ctx.source_id, self.ctx.table) + notes = entry.notes if entry else {} + plan = catalog.plan_patch(notes, changes, table=self.ctx.table, tables=self.ctx.tables) + dropped = list(plan.problems) + kept: list[catalog.PatchChange] = [] + for change in plan.changes: + key = catalog.parse_path(change.path) + if key[0] == "c" and key[2] == "codes" and key[1].lower() in self.ctx.masked: + dropped.append(prompts["masked_codes"].format(column=key[1])) + continue + if change.state == "same": + dropped.append(prompts["same"].format(where=_where(key, change))) + continue + kept.append(change) + said: list[str] = [] + event: dict[str, Any] | None = None + if kept: + label = notes.get("label") if isinstance(notes.get("label"), Mapping) else {} + event = { + "type": "proposal", "source": self.ctx.source_name, "source_id": self.ctx.source_id, + "table": self.ctx.table, + "table_label": label.get("value") if label and label.get("status") != "rejected" else None, + "version": entry.version if entry else 0, + "changes": [{**c.as_dict(), "label": _where(catalog.parse_path(c.path), c)} for c in kept], + } + said.append(prompts["proposed"].format(n=len(kept), paths="、".join(c.path for c in kept))) + else: + said.append(prompts["nothing"]) + if dropped: + said.append(prompts["dropped"].format(reasons="；".join(dropped))) + return event, "\n".join(said) + + +def _where(key: catalog.Slot, change: catalog.PatchChange) -> str: + """提案里一项的中文说法：「列 status 的码值」「表的有效记录条件」「指向 members 的关联关系」。""" + value = change.after if isinstance(change.after, Mapping) else change.before + if key[0] == "r" and isinstance(value, Mapping) and value.get("to_table"): + return f"指向 {value['to_table']} 的关联关系" + return catalog.describe_slot(key) + + +# ========================================================================== +# 流 +# ========================================================================== + + +#: 上一轮的正文以这些结尾（说完了一句话）时，下一轮另起一段 +_SENTENCE_END = ("。", "！", "？", "；", "：", ".", "!", "?", ";", ":") + + +def _sse(obj: Mapping[str, Any]) -> str: + return f"data: {json.dumps(obj, ensure_ascii=False, default=str)}\n\n" + + +class _CallFailed(Exception): + """一次模型调用没成：code 是错误事件的机读码，message 给人看。""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +def _seconds(value: Any) -> str: + number = float(value) + return str(int(number)) if number.is_integer() else f"{number:g}" + + +def _transient(e: BaseException) -> bool: + from app.core.errors import NETWORK, RATE_LIMIT, SERVER, classify + + return classify(e) in (RATE_LIMIT, SERVER, NETWORK) + + +async def _pump(runner: Any, history: list[Any], queue: asyncio.Queue[tuple[str, Any]]) -> None: + """在单独的任务里读模型的流，一块一块放进队列；读完放 end，出错放 error。""" + try: + async for chunk in runner.astream(history): + await queue.put(("chunk", chunk)) + await queue.put(("end", None)) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 - 交给读队列的一方按原因处理 + await queue.put(("error", e)) + + +async def _round(runner: Any, history: list[Any], model_id: str) -> AsyncIterator[tuple[str, Any]]: + """调一次模型：交出 ("text", 增量)、("beat", None)（等得久了，发心跳）、最后 ("message", (合起来的消息, 截断了没有))。 + 网络错误、429、5xx 在还没输出任何正文时重试；超时、别的错误抛 _CallFailed。""" + from app.core.errors import TIMEOUT, classify, describe_exception + from app.engine.state import message_text + from app.providers.budget import cut_off + + loop = asyncio.get_running_loop() + attempt = 0 + while True: + queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue() + task = asyncio.ensure_future(_pump(runner, history, queue)) + deadline = loop.time() + float(catalog.MODEL_TIMEOUT_S) + message: Any = None + cut = False + started = False + failure: BaseException | None = None + try: + while True: + remaining = deadline - loop.time() + if remaining <= 0: + raise _CallFailed(CODE_TIMEOUT, ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S), + place=DRAFT_MODEL_PLACE)) + try: + kind, value = await asyncio.wait_for(queue.get(), timeout=min(remaining, HEARTBEAT_S)) + except asyncio.TimeoutError: + yield "beat", None + continue + if kind == "end": + break + if kind == "error": + failure = value + break + message = value if message is None else message + value + cut = cut or bool(cut_off(value)) + text = message_text(value) + if text: + started = True + yield "text", text + finally: + if not task.done(): + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + if failure is None: + yield "message", (message, cut) + return + if not started and attempt < len(RETRY_DELAYS_S) and _transient(failure): + logger.info("表助手：调用模型出错（%s），%s 秒后重试", type(failure).__name__, RETRY_DELAYS_S[attempt]) + await asyncio.sleep(RETRY_DELAYS_S[attempt]) + attempt += 1 + continue + if isinstance(failure, TimeoutError) or classify(failure) == TIMEOUT: + raise _CallFailed(CODE_TIMEOUT, ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S), + place=DRAFT_MODEL_PLACE)) from failure + raise _CallFailed(CODE_MODEL, ERROR_MODEL.format(model=model_id or "", reason=describe_exception(failure), + place=DRAFT_MODEL_PLACE)) from failure + + +async def stream(model: Any, model_id: str, ctx: AssistantContext, + messages: Iterable[Mapping[str, Any]]) -> AsyncIterator[str]: + """表助手的一次请求：交出 SSE 的每一行（事件、心跳注释）。""" + from langchain_core.messages import SystemMessage, ToolMessage, message_chunk_to_message + + from app.providers.budget import cut_words + + history: list[Any] = [SystemMessage(content=ctx.system), *_history(messages)] + tools = _Tools(ctx) + runner = model.bind_tools(ctx.tools()) + usage = {"input_tokens": 0, "output_tokens": 0} + wrote = "" + names = [t["function"]["name"] for t in ctx.tools()] + try: + for round_no in range(MAX_ROUNDS): + fresh = True + message: Any = None + cut = False + # 流在半路被关掉（使用者点了停止、断开）时，正在读的那次调用跟着停：aclosing 让 _round 的收尾马上执行 + async with contextlib.aclosing(_round(runner, history, model_id)) as rounds: + async for kind, value in rounds: + if kind == "beat": + yield ": keep-alive\n\n" + elif kind == "text": + if fresh and wrote.endswith(_SENTENCE_END): + # 上一轮（查询、提案之前）说完了一句话：这一轮另起一段 + yield _sse({"type": "text", "delta": "\n\n"}) + fresh = False + wrote = value + yield _sse({"type": "text", "delta": value}) + else: + message, cut = value + meta = getattr(message, "usage_metadata", None) or {} + usage["input_tokens"] += int(meta.get("input_tokens") or 0) + usage["output_tokens"] += int(meta.get("output_tokens") or 0) + if cut: + limit = getattr(model, "max_tokens", None) + yield _sse({"type": "error", "code": CODE_TRUNCATED, + "message": ERROR_TRUNCATED.format(cut=cut_words("表助手", limit), place=DRAFT_MODEL_PLACE)}) + return + calls = list(getattr(message, "tool_calls", None) or []) if message is not None else [] + if not calls: + yield _sse({"type": "done", "usage": usage, "queries": tools.queries}) + return + if round_no == MAX_ROUNDS - 1: + yield _sse({"type": "error", "code": CODE_ROUNDS, "message": ERROR_ROUNDS.format(n=MAX_ROUNDS)}) + return + # 合起来的流式块换成普通消息再交回模型（不带半截的工具调用块） + history.append(message_chunk_to_message(message)) + for call in calls: + name = str(call.get("name") or "") + args = call.get("args") if isinstance(call.get("args"), Mapping) else {} + call_id = str(call.get("id") or f"{name}-{round_no}-{len(history)}") + event: dict[str, Any] | None = None + if name == QUERY_TOOL: + event, content = await tools.query(args.get("sql")) + elif name == PROPOSE_TOOL: + event, content = await tools.propose(args.get("changes")) + else: + content = ASSISTANT_TOOL_PROMPT["unknown_tool"].format(name=name, tools="、".join(names)) + if event is not None: + yield _sse(event) + if round_no == MAX_ROUNDS - 2: + content = f"{content}\n{ASSISTANT_TOOL_PROMPT['last_round']}" + history.append(ToolMessage(content=content, tool_call_id=call_id)) + except _CallFailed as e: + yield _sse({"type": "error", "code": e.code, "message": e.message}) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 - 流已经开了，照实交一条错误事件 + from app.core.errors import describe_exception, first_line + + logger.exception("表助手出错：数据源 %s 表 %s", ctx.source_id, ctx.table) + yield _sse({"type": "error", "code": CODE_FAILED, + "message": ERROR_FAILED.format(line=first_line(e) or describe_exception(e))}) + + +__all__ = ["AssistantContext", "CODE_FAILED", "CODE_MODEL", "CODE_ROUNDS", "CODE_TIMEOUT", "CODE_TRUNCATED", + "HEARTBEAT_S", "MAX_MESSAGES", "MAX_PENDING", "MAX_QUERIES", "MAX_ROUNDS", "MESSAGE_MAX", "PROPOSE_TOOL", + "QUERY_ROWS", "QUERY_TOOL", "SHOWN_ROWS", "conversation_problem", "prepare", "resolve_model", "stream"] diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py new file mode 100644 index 00000000..2d268bb0 --- /dev/null +++ b/backend/tests/test_catalog_assistant.py @@ -0,0 +1,550 @@ +"""表助手（data/catalog_assistant.py，POST /api/datasources/{id}/catalog/{table}/assistant）。 + +- 开始前：表不在 404；起草模型用不了 409 catalog_draft_model_unavailable（说法同起草任务）；对话超过 40 条、一条超过 + 4000 字、待保存的修改超过 200 项 422 +- 流：text 增量、query（跑过的只读查询，前 10 行，遮罩列写「已遮罩」）、proposal（按 plan_patch 核对过的提案，形状同助手流 + 的 catalog_patch：公式不收、列名必须在表结构里、遮罩列不给码值）、done（用量、查询条数）、error（中途出错，之后没有 done） +- 交给模型的背景：表结构、这张表的目录（每项的来源和状态）、关联关系（含别的表指过来的）、使用次数、数据源的起草要求、 + 最近一次起草留空的列、界面上还没保存的修改；遮罩列只给列名、不给任何值 +- 只读查询：只有开了数据剖析的源才给这个工具；经剖析的只读守卫和单条时限；每次请求最多 5 条；最多 50 行交给模型；结果 + 先遮再交给模型；没开剖析时提示词里写明不能查数据 +- 对话不存服务端；提案不写目录（保存走 PATCH，最多 200 项） + +表名、列名、业务数字都是虚构的；号码是 139000000xx 这样的虚构号码。 +""" +from __future__ import annotations + +import asyncio +import json +import sqlite3 +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessageChunk, HumanMessage, SystemMessage, ToolMessage + +from app.data import catalog, catalog_assistant +from app.data.catalog import make_item +from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION +from app.db.base import SessionLocal +from app.db.models import CatalogDraftJob, CatalogDraftTask, DataSource +from app.main import app as _fastapi_app +from tests.fixtures.sources import drop_source + +PHONES = [f"139000000{i:02d}" for i in range(1, 31)] + + +# -------------------------------------------------------------------------- +# 假模型：按剧本一轮一轮回 +# -------------------------------------------------------------------------- + + +@dataclass +class Turn: + text: list[str] = field(default_factory=list) + calls: list[tuple[str, dict]] = field(default_factory=list) + cut: bool = False + usage: tuple[int, int] = (100, 20) + error: BaseException | None = None + hang: bool = False + + +class Scripted: + """bind_tools 记下交给它的工具；astream 每调一次按剧本回一轮，记下这一轮收到的消息。""" + + def __init__(self, turns: list[Turn]) -> None: + self.turns = list(turns) + self.seen: list[list[Any]] = [] + self.tools: list[Any] | None = None + + def bind_tools(self, tools: list[Any], **_: Any) -> "Scripted": + self.tools = tools + return self + + async def astream(self, messages: list[Any]): + self.seen.append(list(messages)) + turn = self.turns.pop(0) + if turn.error is not None: + raise turn.error + if turn.hang: + await asyncio.sleep(3600) + for piece in turn.text: + yield AIMessageChunk(content=piece) + for i, (name, args) in enumerate(turn.calls): + yield AIMessageChunk(content="", tool_call_chunks=[ + {"name": name, "args": json.dumps(args, ensure_ascii=False), "id": f"c{len(self.seen)}_{i}", + "index": i}]) + yield AIMessageChunk(content="", usage_metadata={"input_tokens": turn.usage[0], "output_tokens": turn.usage[1], + "total_tokens": sum(turn.usage)}, + response_metadata={"finish_reason": "length" if turn.cut else "stop"}) + + def tool_names(self) -> list[str]: + return [t["function"]["name"] for t in self.tools or []] + + +class _Unavailable(Exception): + status_code = 503 + + +# -------------------------------------------------------------------------- +# 夹具 +# -------------------------------------------------------------------------- + + +def _shop_db(path) -> str: + conn = sqlite3.connect(path) + conn.executescript(""" + CREATE TABLE members (id INTEGER PRIMARY KEY, nick TEXT); + CREATE TABLE orders (id INTEGER PRIMARY KEY, order_no TEXT NOT NULL, status INTEGER NOT NULL, + amount REAL, contact_phone TEXT, created_at TEXT, + member_id INTEGER REFERENCES members(id)); + """) + conn.executemany("INSERT INTO members VALUES (?, ?)", [(i, f"会员{i}") for i in range(1, 6)]) + conn.executemany("INSERT INTO orders VALUES (?, ?, ?, ?, ?, ?, ?)", [ + (i, f"NO{i:04d}", 9 if i % 10 == 0 else 1, 10.0 * i, PHONES[i - 1], f"2026-09-{i:02d}", i % 5 + 1) + for i in range(1, 31)]) + conn.commit() + conn.close() + return str(path) + + +ORDERS_NOTES = { + "label": make_item("订单", "llm"), + "grain": make_item("每笔订单一行", "human", "confirmed"), + "kind": make_item("fact", "llm", "rejected"), + "columns": { + "status": {"codes": make_item({"1": "已支付"}, "profile", note="数据剖析：全表 30 行，2 个取值")}, + "contact_phone": {"label": make_item("联系电话", "llm"), + "codes": make_item({PHONES[0]: "测试号"}, "human", "confirmed")}, + }, +} + +_CREATED: list[str] = [] + + +@pytest.fixture(autouse=True) +async def _cleanup(): + yield + while _CREATED: + await drop_source(_CREATED.pop()) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c: + yield c + + +async def _source(client, tmp_path, *, profile: bool = True, guidance: str | None = "金额一律按元", + notes: dict | None = None) -> str: + options: dict[str, Any] = {"mask_columns": "contact_phone"} + if profile: + options["catalog_profile"] = {"enabled": True} + if guidance: + options[CATALOG_DRAFT_GUIDANCE_OPTION] = guidance + async with SessionLocal() as session: + row = DataSource(name=f"shop_{uuid.uuid4().hex[:8]}", kind="sqlite", database=_shop_db(tmp_path / "shop.db"), + options=options, description="虚构的网店库") + session.add(row) + await session.commit() + sid = row.id + _CREATED.append(sid) + r = await client.post(f"/api/datasources/{sid}/introspect") + assert r.status_code == 200, r.text + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "orders", notes if notes is not None else ORDERS_NOTES, + if_version=0, actor=None) + return sid + + +def use(monkeypatch, model: Scripted) -> None: + async def resolve(session): + return model, "fake-chat" + + monkeypatch.setattr(catalog_assistant, "resolve_model", resolve) + + +def _url(sid: str, table: str = "orders") -> str: + return f"/api/datasources/{sid}/catalog/{table}/assistant" + + +async def _talk(client, sid: str, body: dict, table: str = "orders") -> tuple[list[dict], list[str]]: + """发一轮，返回 (事件, 原始行)。""" + events: list[dict] = [] + lines: list[str] = [] + async with client.stream("POST", _url(sid, table), json=body) as r: + assert r.status_code == 200, await r.aread() + assert r.headers["content-type"].startswith("text/event-stream") + async for line in r.aiter_lines(): + lines.append(line) + if line.startswith("data: "): + events.append(json.loads(line[6:])) + return events, lines + + +def _text(events: list[dict]) -> str: + return "".join(e["delta"] for e in events if e["type"] == "text") + + +def _ask(text: str, **extra) -> dict: + return {"messages": [{"role": "user", "content": text}], **extra} + + +# -------------------------------------------------------------------------- +# 开始前的检查 +# -------------------------------------------------------------------------- + + +async def test_refusals_before_the_stream(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + use(monkeypatch, Scripted([])) + assert (await client.post(_url(sid, "ghost"), json=_ask("你好"))).status_code == 404 + assert (await client.post("/api/datasources/nope/catalog/orders/assistant", json=_ask("x"))).status_code == 404 + too_many = {"messages": [{"role": "user", "content": "x"}] * 41} + assert (await client.post(_url(sid), json=too_many)).status_code == 422 + too_long = _ask("长" * 4001) + assert (await client.post(_url(sid), json=too_long)).status_code == 422 + pending = [{"path": "label", "value": "订单", "reason": "x"}] * 201 + assert (await client.post(_url(sid), json=_ask("x", pending=pending))).status_code == 422 + wrong_role = {"messages": [{"role": "system", "content": "x"}]} + assert (await client.post(_url(sid), json=wrong_role)).status_code == 422 + last_is_answer = {"messages": [{"role": "user", "content": "x"}, {"role": "assistant", "content": "y"}]} + r = await client.post(_url(sid), json=last_is_answer) + assert r.status_code == 422 and "最后一条" in r.json()["detail"] + + +async def test_unavailable_draft_model_is_409_with_the_draft_wording(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + + async def unavailable(session): + raise catalog.CatalogModelUnavailable("未配置模型接入，无法由模型起草数据目录。请到「设置 → 模型接入」添加") + + monkeypatch.setattr(catalog, "resolve_draft_model", unavailable) + r = await client.post(_url(sid), json=_ask("你好")) + assert r.status_code == 409 + assert r.json()["code"] == "catalog_draft_model_unavailable" and "模型接入" in r.json()["detail"] + + +# -------------------------------------------------------------------------- +# 背景 +# -------------------------------------------------------------------------- + + +async def _blank_draft(sid: str) -> None: + """最近一次结束于 ok 的起草任务记下留空的列；之后一次停止了的不算。""" + async with SessionLocal() as session: + done = CatalogDraftJob(id=uuid.uuid4().hex, source_id=sid, status="succeeded", + created_at=datetime(2026, 10, 1, tzinfo=timezone.utc)) + stopped = CatalogDraftJob(id=uuid.uuid4().hex, source_id=sid, status="cancelled", + created_at=datetime(2026, 10, 2, tzinfo=timezone.utc)) + session.add_all([done, stopped]) + await session.flush() + session.add_all([ + CatalogDraftTask(job_id=done.id, seq=0, table_name="orders", status="ok", blank_columns=2, + blank_sample=["order_no", "created_at"]), + CatalogDraftTask(job_id=stopped.id, seq=0, table_name="orders", status="cancelled", blank_columns=0), + ]) + await session.commit() + + +async def test_background_has_structure_catalog_and_hides_masked_values(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + await _blank_draft(sid) + async with SessionLocal() as session: + await catalog.write_entry(session, sid, "members", {"relations": [ + {"id": catalog.relation_id("members", ["id"], "orders", ["member_id"]), "columns": ["id"], + "to_table": "orders", "to_columns": ["member_id"], "cardinality": "one_to_many", "coverage": None, + "source": "human", "status": "confirmed"}]}, if_version=0, actor=None) + model = Scripted([Turn(text=["这张表还缺 order_no 的含义。"])]) + use(monkeypatch, model) + pending = [{"path": "columns.amount.unit", "value": "元", "reason": "用户说：金额是元"}] + events, _ = await _talk(client, sid, _ask("看看还缺什么", pending=pending)) + assert _text(events) == "这张表还缺 order_no 的含义。" + assert events[-1] == {"type": "done", "usage": {"input_tokens": 100, "output_tokens": 20}, "queries": 0} + [messages] = model.seen + assert isinstance(messages[0], SystemMessage) and isinstance(messages[-1], HumanMessage) + system = messages[0].content + # 规矩 + for rule in ("只改数据目录", "不改业务库", "公式", "拿不准", "原话"): + assert rule in system + # 表结构、目录（来源和状态）、使用次数、起草要求、数据源说明 + for fact in ("order_no", "status", "amount", "member_id → members.id", "主键", "每笔订单一行", "人工填写", "已确认", + "模型起草", "已驳回", "被运行用过 0 次", "金额一律按元", "虚构的网店库", "数据剖析：全表 30 行"): + assert fact in system, fact + # 别的表指过来的关联关系 + assert "members.id → orders.member_id" in system + # 最近一次结束于 ok 的起草留空的列（之后停止了的那次不算） + assert "order_no、created_at" in system + # 界面上还没保存的修改 + assert "columns.amount.unit" in system and "用户说：金额是元" in system + # 遮罩列只给列名：码值、取值一个都不给 + assert "contact_phone" in system + assert not any(p in system for p in PHONES) and "测试号" not in system + # 没开剖析的源才写不能查数据；这里开了，给了查询工具 + assert model.tool_names() == ["run_query", "propose_changes"] + assert "没开剖析" not in system + + +async def test_without_profiling_there_is_no_query_tool(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path, profile=False) + model = Scripted([Turn(text=["只能根据表结构推断。"])]) + use(monkeypatch, model) + await _talk(client, sid, _ask("状态 9 是什么")) + assert model.tool_names() == ["propose_changes"] + assert "这个数据源没开剖析，不能查数据，只能根据表结构和目录推断" in model.seen[0][0].content + + +async def test_opening_without_user_messages(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + model = Scripted([Turn(text=["order_no 是什么意思？"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, {"messages": []}) + assert _text(events) == "order_no 是什么意思？" + assert isinstance(model.seen[0][-1], HumanMessage) + # 界面先显示了一句开场白（助手说的）：照样从使用者这一边开始 + model = Scripted([Turn(text=["好的。"])]) + use(monkeypatch, model) + await _talk(client, sid, {"messages": [{"role": "assistant", "content": "有两列留空了"}, + {"role": "user", "content": "order_no 是订单号"}]}) + kinds = [type(m).__name__ for m in model.seen[0]] + assert kinds == ["SystemMessage", "HumanMessage", "AIMessage", "HumanMessage"] + + +# -------------------------------------------------------------------------- +# 提案 +# -------------------------------------------------------------------------- + + +async def test_proposals_are_checked_and_never_written(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + changes = [ + {"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说：9 是作废"}, + {"path": "valid_filter", "value": "status <> 9", "reason": "用户说：统计时排除作废"}, + {"path": "columns.amount.meaning", "value": "实收=单价*数量", "reason": "x"}, # 公式：不收 + {"path": "columns.ghost.label", "value": "幽灵", "reason": "x"}, # 列不存在：不收 + {"path": "columns.contact_phone.codes", "value": {PHONES[1]: "客服"}, "reason": "x"}, # 遮罩列：不给码值 + {"path": "grain", "value": "每笔订单一行", "reason": "x"}, # 已经是这个值：不用提 + ] + model = Scripted([Turn(text=["好的，"], calls=[("propose_changes", {"changes": changes})]), + Turn(text=["已经整理成提案。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("9 是作废，统计时排除")) + assert _text(events) == "好的，已经整理成提案。" + [proposal] = [e for e in events if e["type"] == "proposal"] + assert proposal["source_id"] == sid and proposal["table"] == "orders" and proposal["version"] == 1 + assert proposal["source"].startswith("shop_") and proposal["table_label"] == "订单" + by = {c["path"]: c for c in proposal["changes"]} + assert set(by) == {"columns.status.codes", "valid_filter"} + codes = by["columns.status.codes"] + assert codes["label"] == "列 status 的码值" and codes["reason"] == "用户说：9 是作废" + assert codes["before"] == {"1": "已支付"} and codes["after"] == {"1": "已支付", "9": "作废"} + assert codes["value"] == {"9": "作废"} and codes["state"] == "change" + assert by["valid_filter"]["label"] == "表的有效记录条件" and by["valid_filter"]["before"] is None + # 没收的几项照实告诉模型（不带遮罩列的任何取值） + tool = model.seen[1][-1] + assert isinstance(tool, ToolMessage) + for said in ("2 项", "公式", "ghost", "遮罩", "已经是这个值"): + assert said in tool.content, said + assert not any(p in tool.content for p in PHONES) + assert events[-1]["type"] == "done" and events[-1]["usage"] == {"input_tokens": 200, "output_tokens": 40} + async with SessionLocal() as session: + entry = await catalog.read_entry(session, sid, "orders") + assert entry.version == 1 and "valid_filter" not in entry.notes + + +async def test_proposal_codes_build_on_the_pending_ones(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + model = Scripted([Turn(calls=[("propose_changes", {"changes": [ + {"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说：9 是作废"}]})]), + Turn(text=["好。"])]) + use(monkeypatch, model) + pending = [{"path": "columns.status.codes", "value": {"2": "已退款"}, "reason": "用户说：2 是退款"}] + events, _ = await _talk(client, sid, _ask("9 是作废", pending=pending)) + [proposal] = [e for e in events if e["type"] == "proposal"] + [change] = proposal["changes"] + assert change["value"] == {"2": "已退款", "9": "作废"} + assert change["after"] == {"1": "已支付", "2": "已退款", "9": "作废"} + + +async def test_nothing_acceptable_means_no_proposal_event(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + model = Scripted([Turn(calls=[("propose_changes", {"changes": [{"path": "columns.ghost.label", "value": "x"}]})]), + Turn(text=["那一列不存在。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("ghost 列叫什么")) + assert not [e for e in events if e["type"] == "proposal"] + assert "没有可以交给使用者的修改" in model.seen[1][-1].content + + +# -------------------------------------------------------------------------- +# 只读查询 +# -------------------------------------------------------------------------- + + +async def test_queries_are_read_only_masked_and_capped(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + queries = [ + ("run_query", {"sql": "SELECT status, COUNT(*) AS n FROM orders GROUP BY status ORDER BY status"}), + ("run_query", {"sql": "SELECT id, contact_phone AS phone, amount FROM orders ORDER BY id"}), + ("run_query", {"sql": "DELETE FROM orders"}), + ] + model = Scripted([Turn(text=["我查一下。"], calls=queries), + Turn(calls=[("run_query", {"sql": "SELECT 1"})] * 3), + Turn(text=["9 占一成。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("状态分布")) + shown = [e for e in events if e["type"] == "query"] + assert len(shown) == 5 # 每次请求最多 5 条 + first, second, third = shown[:3] + assert first["columns"] == ["status", "n"] and first["rows"] == [[1, 27], [9, 3]] + assert first["row_count"] == 2 and first["error"] is None + # 遮罩列改了名字也认得出：值写「已遮罩」；界面只给前 10 行 + assert second["columns"] == ["id", "phone", "amount"] and len(second["rows"]) == 10 + assert {row[1] for row in second["rows"]} == {"已遮罩"} and second["row_count"] == 30 + # 只读守卫：写语句不执行 + assert third["error"] and "守卫" in third["error"] and third["rows"] == [] + # 交给模型的：最多 50 行、遮罩值看不到；第 6 条没执行 + second_round = model.seen[1] + results = [m for m in second_round if isinstance(m, ToolMessage)] + assert len(results) == 3 + assert json.loads(results[1].content)["rows"][29][1] == "已遮罩" + assert not any(p in m.content for m in second_round for p in PHONES) + capped = [m for m in model.seen[2] if isinstance(m, ToolMessage)][-1] + assert "未执行" in capped.content and "5 条" in capped.content + assert events[-1] == {"type": "done", "usage": {"input_tokens": 300, "output_tokens": 60}, "queries": 5} + with sqlite3.connect(tmp_path / "shop.db") as conn: + assert conn.execute("SELECT COUNT(*) FROM orders").fetchone()[0] == 30 + + +async def test_query_results_given_to_the_model_stop_at_50_rows(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + sql = "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 80) SELECT i FROM n" + model = Scripted([Turn(calls=[("run_query", {"sql": sql})]), Turn(text=["好。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("数一下")) + [query] = [e for e in events if e["type"] == "query"] + assert query["row_count"] == 50 and query["truncated"] is True and len(query["rows"]) == 10 + result = json.loads([m for m in model.seen[1] if isinstance(m, ToolMessage)][0].content) + assert len(result["rows"]) == 50 and result["truncated"] is True + + +# -------------------------------------------------------------------------- +# 中途出错 +# -------------------------------------------------------------------------- + + +async def test_errors_end_the_stream_without_done(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + use(monkeypatch, Scripted([Turn(error=RuntimeError("接口不认这个模型"))])) + events, _ = await _talk(client, sid, _ask("你好")) + assert events[-1]["type"] == "error" and events[-1]["code"] == "assistant_model_error" + assert "fake-chat" in events[-1]["message"] and not [e for e in events if e["type"] == "done"] + use(monkeypatch, Scripted([Turn(text=["写到一半"], cut=True)])) + events, _ = await _talk(client, sid, _ask("你好")) + assert _text(events) == "写到一半" + assert events[-1]["type"] == "error" and events[-1]["code"] == "assistant_truncated" + assert "上限" in events[-1]["message"] + + +async def test_transient_errors_are_retried_before_any_output(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + monkeypatch.setattr(catalog_assistant, "RETRY_DELAYS_S", (0.0, 0.0)) + model = Scripted([Turn(error=_Unavailable("服务暂时不可用")), Turn(text=["好了。"])]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("你好")) + assert _text(events) == "好了。" and events[-1]["type"] == "done" + assert len(model.seen) == 2 + + +async def test_a_call_over_the_time_limit_is_stopped(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.3) + monkeypatch.setattr(catalog_assistant, "HEARTBEAT_S", 0.1) + use(monkeypatch, Scripted([Turn(hang=True)])) + events, lines = await _talk(client, sid, _ask("你好")) + assert events == [events[-1]] and events[-1]["code"] == "assistant_timeout" + assert "0.3 秒" in events[-1]["message"] + # 等模型的时候按拍发 SSE 注释（不是事件），免得中间的代理断开 + assert any(line.startswith(":") for line in lines) + + +async def test_steps_are_bounded(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + monkeypatch.setattr(catalog_assistant, "MAX_ROUNDS", 3) + again = Turn(calls=[("propose_changes", {"changes": [{"path": "columns.ghost.label", "value": "x"}]})]) + model = Scripted([again, again, again]) + use(monkeypatch, model) + events, _ = await _talk(client, sid, _ask("你好")) + assert len(model.seen) == 3 + assert events[-1]["type"] == "error" and events[-1]["code"] == "assistant_rounds" + # 倒数第二轮的工具结果里提醒模型直接回答 + assert "直接回答" in [m for m in model.seen[2] if isinstance(m, ToolMessage)][-1].content + + +class _Slow(Scripted): + """说了半句就一直想：记下这次调用有没有被停掉。""" + + def __init__(self) -> None: + super().__init__([]) + self.cancelled = False + + async def astream(self, messages: list[Any]): + self.seen.append(list(messages)) + yield AIMessageChunk(content="我想一想") + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + self.cancelled = True + raise + yield AIMessageChunk(content="……") + + +async def test_stopping_the_stream_stops_the_model_call(): + """使用者点停止（连接断开、流被关掉）：正在进行的模型调用跟着停，不在后台接着花额度。""" + from app.data import catalog_profile + + model = _Slow() + ctx = catalog_assistant.AssistantContext( + source_id="s", source_name="虚构库", kind="sqlite", table="orders", tables={}, query_source=None, options={}, + masked=set(), can_query=False, settings=catalog_profile.ProfileSettings(), system="系统提示") + events = catalog_assistant.stream(model, "fake-chat", ctx, [{"role": "user", "content": "你好"}]) + first = await events.__anext__() + assert json.loads(first[len("data: "):]) == {"type": "text", "delta": "我想一想"} + await events.aclose() + assert model.cancelled + + +# -------------------------------------------------------------------------- +# 保存：PATCH 一次最多 200 项 +# -------------------------------------------------------------------------- + + +async def test_patch_accepts_up_to_200_changes(client): + """表助手一次能提很多项（逐列补中文名），保存不再卡在 20 项：预览、保存都收 200 项，201 项 422。""" + assert catalog.PATCH_MAX_CHANGES == 200 + columns = [f"c{i:03d}" for i in range(200)] + table = {"qualified": "wide", "primary_key": [], "comment": None, "is_view": False, "foreign_keys": [], + "unique": [], "columns": [{"name": c, "type": "TEXT", "nullable": True, "comment": None} for c in columns]} + async with SessionLocal() as session: + row = DataSource(name=f"wide_{uuid.uuid4().hex[:8]}", kind="sqlite", database=":memory:", + schema_cache={"tables": {"wide": table}}) + session.add(row) + await session.commit() + sid = row.id + _CREATED.append(sid) + base = f"/api/datasources/{sid}/catalog/wide" + changes = [{"path": f"columns.{c}.label", "value": f"第 {i} 列", "reason": "用户逐列说明"} + for i, c in enumerate(columns)] + preview = await client.post(f"{base}/patch/preview", json={"changes": changes}) + assert preview.status_code == 200, preview.text + assert len(preview.json()["changes"]) == 200 and preview.json()["problems"] == [] + saved = await client.post(f"{base}/patch", json={"changes": changes, "if_version": 0}) + assert saved.status_code == 200, saved.text + assert len(saved.json()["notes"]["columns"]) == 200 + assert saved.json()["notes"]["columns"]["c199"]["label"]["value"] == "第 199 列" + too_many = changes + [{"path": "label", "value": "宽表"}] + assert (await client.post(f"{base}/patch/preview", json={"changes": too_many})).status_code == 422 + assert (await client.post(f"{base}/patch", json={"changes": too_many, "if_version": 1})).status_code == 422 From 65369b8a6e9366cc3caab25c29a6f12c39a3164e Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:23:08 +0800 Subject: [PATCH 04/39] =?UTF-8?q?=E8=A1=A8=E5=8A=A9=E6=89=8B=EF=BC=9ASDK?= =?UTF-8?q?=20=E4=B8=8D=E5=86=8D=E8=87=AA=E5=B7=B1=E9=87=8D=E8=AF=95?= =?UTF-8?q?=EF=BC=88=E5=90=8C=E8=B5=B7=E8=8D=89=EF=BC=8C=E7=BD=91=E7=BB=9C?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E3=80=81429=E3=80=815xx=20=E5=8F=AA=E7=94=B1?= =?UTF-8?q?=E8=A1=A8=E5=8A=A9=E6=89=8B=E5=9C=A8=E8=BF=98=E6=B2=A1=E8=BE=93?= =?UTF-8?q?=E5=87=BA=E6=97=B6=E9=87=8D=E8=AF=95=EF=BC=8C=E9=80=80=E9=81=BF?= =?UTF-8?q?=203=E3=80=8110=20=E7=A7=92=EF=BC=89=EF=BC=9B=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E8=A1=A5=E4=B8=8A=E5=8F=96=E6=A8=A1=E5=9E=8B=E7=9A=84=E8=A7=84?= =?UTF-8?q?=E6=A0=BC=EF=BC=88=E8=B5=B7=E8=8D=89=E8=AE=BE=E7=BD=AE=E3=80=81?= =?UTF-8?q?=E4=BF=9D=E7=95=99=E6=80=9D=E8=80=83=E3=80=81=E9=A2=9D=E5=BA=A6?= =?UTF-8?q?=2032768=20/=208192=E3=80=81=E5=8D=95=E6=AC=A1=E6=97=B6?= =?UTF-8?q?=E9=99=90=E3=80=81=E4=B8=8D=E9=87=8D=E8=AF=95=EF=BC=89=E5=92=8C?= =?UTF-8?q?=E4=B8=A4=E4=B8=AA=E5=B7=A5=E5=85=B7=E5=9C=A8=20OpenAI=20?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E3=80=81Anthropic=20=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E4=B8=8A=E9=83=BD=E7=BB=91=E5=BE=97=E4=B8=8A=EF=BC=9Bquery=5Fs?= =?UTF-8?q?napshot=5Ftables=20=E4=B8=8D=E8=BF=9B=20catalog=20=E7=9A=84=20?= =?UTF-8?q?=5F=5Fall=5F=5F=EF=BC=88=E6=8C=89=E6=A8=A1=E5=9D=97=E5=B1=9E?= =?UTF-8?q?=E6=80=A7=E5=8F=96=E7=94=A8=EF=BC=8C=E5=B0=91=E5=92=8C=E8=B5=B7?= =?UTF-8?q?=E8=8D=89=E9=82=A3=E8=BE=B9=E7=9A=84=E6=94=B9=E5=8A=A8=E6=92=9E?= =?UTF-8?q?=E5=9C=A8=E4=B8=80=E8=B5=B7=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog.py | 3 +- backend/app/data/catalog_assistant.py | 7 +-- backend/tests/test_catalog_assistant.py | 63 +++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 40bdaab8..36d481d3 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -2351,8 +2351,7 @@ __all__ = [ "fk_relations", "frozen_catalog", "infer_name_relations", "initial_status", "is_date_column", "is_profile_note", "join_paths", "make_item", "merge_notes", "name_words", "now_iso", "parse_path", "plan_patch", "profile_relation_holds", - "query_snapshot_meta", "query_snapshot_tables", "read_catalog", "read_entry", "relation_graph", "relation_id", - "render_table_index", + "query_snapshot_meta", "read_catalog", "read_entry", "relation_graph", "relation_id", "render_table_index", "render_table_notes", "resolve_draft_model", "resolve_table_name", "review_entry", "review_item", "same_notes", "same_profile_note", "save_human_edit", "save_patch", "status_counts", "structure_relation_ids", "system_notes_source", "table_usage", "unreviewed_status", "validate_notes", "visible_notes", "write_entry", diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py index 475a6a5e..f8ac5e4e 100644 --- a/backend/app/data/catalog_assistant.py +++ b/backend/app/data/catalog_assistant.py @@ -75,8 +75,8 @@ QUERY_ROWS = 50 SHOWN_ROWS = 10 CELL_MAX = 300 TOOL_CHARS = 12_000 -#: 网络错误、429、5xx 重试前等几秒（依次取用，个数就是最多重试几次） -RETRY_DELAYS_S = (2.0, 5.0) +#: 网络错误、429、5xx 重试前等几秒（依次取用，个数就是最多重试几次）。在对话里等，比起草（10、30 秒）短 +RETRY_DELAYS_S = (3.0, 10.0) #: 等模型的时候每隔几秒发一行 SSE 注释 HEARTBEAT_S = 10.0 #: 背景里「目录里还没有中文名和含义的列」最多列几个；别的表指过来的关联关系最多列几条 @@ -257,8 +257,9 @@ async def resolve_model(session: AsyncSession) -> tuple[Any, str]: provider, choice = await resolve_draft_provider(session, none_reason=ERROR_NO_MODEL, mock_reason=ERROR_NO_MODEL) except DraftModelUnavailable as e: raise catalog.CatalogModelUnavailable(str(e)) from e + # SDK 自己不重试（同起草）：网络错误、429、5xx 由这里在还没输出时重试，两层叠起来实际次数就对不上了 spec = ModelSpec(provider=choice.provider, model=choice.model, max_tokens=catalog.MODEL_MAX_TOKENS, temperature=0, - timeout=int(catalog.MODEL_TIMEOUT_S)) + timeout=int(catalog.MODEL_TIMEOUT_S), max_retries=0) try: big = build_chat_model(provider, spec) small = build_chat_model(provider, spec.model_copy(update={"max_tokens": catalog.MODEL_FALLBACK_MAX_TOKENS})) diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py index 2d268bb0..89f29bd8 100644 --- a/backend/tests/test_catalog_assistant.py +++ b/backend/tests/test_catalog_assistant.py @@ -228,6 +228,69 @@ async def test_unavailable_draft_model_is_409_with_the_draft_wording(client, tmp assert r.json()["code"] == "catalog_draft_model_unavailable" and "模型接入" in r.json()["detail"] +@pytest.fixture +async def mem(monkeypatch): + """单独的内存库：接入和设置只在这个用例里。""" + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + from sqlalchemy.pool import StaticPool + + from app.db.base import Base + + for key in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "OPENAI_API_KEY", "ANTHROPIC_BASE_URL"): + monkeypatch.delenv(key, raising=False) + engine = create_async_engine("sqlite+aiosqlite:///:memory:", poolclass=StaticPool) + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + session = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)() + yield session + await session.close() + await engine.dispose() + + +async def _provider(session, kind: str) -> None: + from app.core.crypto import encrypt + from app.data import draft_model + from app.db.models import Provider, Setting + + session.add(Provider(name="起草网关", kind=kind, base_url="http://gw.example.test/v1" if kind != "anthropic" else None, + api_key=encrypt("sk-test"), models=[{"id": "模型甲"}], default_model="模型甲", enabled=True, + extra={})) + session.add(Setting(key=draft_model.CATALOG_DRAFT_SETTING_KEY, value={"provider": "起草网关", "model": None})) + await session.commit() + + +async def test_model_follows_the_draft_setting_keeps_thinking_and_a_big_budget(mem, monkeypatch): + from app.providers import factory + + await _provider(mem, "openai_compatible") + specs: list[Any] = [] + real = factory.build_chat_model + + def spy(provider, spec): + specs.append(spec) + return real(provider, spec) + + monkeypatch.setattr(factory, "build_chat_model", spy) + model, model_id = await catalog_assistant.resolve_model(mem) + assert model_id == "模型甲" and model.max_tokens == catalog.MODEL_MAX_TOKENS == 32768 + mine = specs[-2:] # 前两份是 catalog.resolve_draft_model 查能不能用时建的 + assert [s.max_tokens for s in mine] == [catalog.MODEL_MAX_TOKENS, catalog.MODEL_FALLBACK_MAX_TOKENS] + assert all(s.thinking is None and s.temperature == 0 and s.max_retries == 0 for s in mine) + assert all(s.timeout == catalog.MODEL_TIMEOUT_S for s in mine) + + +@pytest.mark.parametrize("kind", ["openai_compatible", "anthropic"]) +async def test_tools_bind_on_real_chat_models(mem, kind): + """两个工具的定义各家的模型都收（只绑定，不发请求）。""" + await _provider(mem, kind) + model, _ = await catalog_assistant.resolve_model(mem) + tools = [catalog_assistant.QUERY_TOOL_PROMPT, catalog_assistant.PROPOSE_TOOL_PROMPT] + bound = model.bind_tools(tools) + given = bound.model.kwargs["tools"] + names = [t.get("name") or t["function"]["name"] for t in given] + assert names == ["run_query", "propose_changes"] + + # -------------------------------------------------------------------------- # 背景 # -------------------------------------------------------------------------- From 77ddbe96ea303b2b86e0bc8f7b35c05f2279f965 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:26:05 +0800 Subject: [PATCH 05/39] =?UTF-8?q?=E8=A1=A8=E5=8A=A9=E6=89=8B=E7=9A=84?= =?UTF-8?q?=E8=83=8C=E6=99=AF=E5=9C=A8=E5=AE=BD=E8=A1=A8=E4=B8=8A=E4=B8=8D?= =?UTF-8?q?=E8=87=B3=E4=BA=8E=E5=A4=AA=E9=95=BF=EF=BC=9A=E8=BF=99=E5=BC=A0?= =?UTF-8?q?=E8=A1=A8=E7=9A=84=E7=9B=AE=E5=BD=95=E6=9C=80=E5=A4=9A=E5=88=97?= =?UTF-8?q?=20600=20=E9=A1=B9=EF=BC=88=E8=A1=A8=E7=BA=A7=E7=9A=84=E9=A1=B9?= =?UTF-8?q?=E3=80=81=E5=85=B3=E8=81=94=E5=85=B3=E7=B3=BB=E5=9C=A8=E5=89=8D?= =?UTF-8?q?=EF=BC=89=EF=BC=8C=E5=85=B6=E4=BD=99=E5=86=99=E6=98=8E=E8=BF=98?= =?UTF-8?q?=E6=9C=89=E5=87=A0=E9=A1=B9=E6=B2=A1=E5=88=97=E5=87=BA=EF=BC=9B?= =?UTF-8?q?=E5=AF=BC=E5=85=A5=E6=97=B6=E7=95=8C=E9=9D=A2=E7=9B=B4=E6=8E=A5?= =?UTF-8?q?=E4=BA=A4=E6=9D=A5=E7=9A=84=E6=96=87=E4=BB=B6=E5=8E=9F=E6=96=87?= =?UTF-8?q?=EF=BC=88JSON=20=E6=96=87=E5=AD=97=EF=BC=89=E4=B9=9F=E8=AE=A4?= =?UTF-8?q?=EF=BC=8C=E4=B8=8D=E6=98=AF=20JSON=20=E7=9A=84=20422=EF=BC=9B?= =?UTF-8?q?=E5=AF=BC=E5=85=A5=E5=86=B2=E7=AA=81=E8=AF=B4=E6=98=8E=E9=87=8C?= =?UTF-8?q?=E7=9A=84=E7=8A=B6=E6=80=81=E7=94=A8=E7=95=8C=E9=9D=A2=E7=9A=84?= =?UTF-8?q?=E5=8F=AB=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog_assistant.py | 15 +++++++++++---- backend/app/data/catalog_transfer.py | 11 +++++++++-- backend/tests/test_catalog_assistant.py | 12 ++++++++++++ backend/tests/test_catalog_transfer.py | 5 +++++ 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py index f8ac5e4e..9dadbd98 100644 --- a/backend/app/data/catalog_assistant.py +++ b/backend/app/data/catalog_assistant.py @@ -79,9 +79,11 @@ TOOL_CHARS = 12_000 RETRY_DELAYS_S = (3.0, 10.0) #: 等模型的时候每隔几秒发一行 SSE 注释 HEARTBEAT_S = 10.0 -#: 背景里「目录里还没有中文名和含义的列」最多列几个；别的表指过来的关联关系最多列几条 +#: 背景里「目录里还没有中文名和含义的列」最多列几个；别的表指过来的关联关系最多列几条；这张表的目录最多列几项 +#: （几百列的宽表全部起草过有上千项，全列出来提示词就太长了：表级的项、关联关系在前，列的项在后） UNNAMED_SHOWN = 30 INCOMING_SHOWN = 30 +CATALOG_ITEMS_SHOWN = 600 QUERY_TOOL = "run_query" PROPOSE_TOOL = "propose_changes" @@ -166,6 +168,7 @@ ASSISTANT_PROMPT_LABELS = { "catalog_empty": "这张表还没有数据目录。", "item": "- {path}（{label}）：{value}〔{source}，{status}〕", "item_note": " 备注：{text}", + "catalog_more": "（还有 {n} 项没有列出：这张表的目录太长。要改的项照上面的路径写法提，核对时按当前目录算）", "relation": "{columns} → {to_table}.{to_columns}{extra}", "relation_extra": "（{text}）", "cardinality": "基数 {value}", @@ -357,10 +360,14 @@ def _catalog_lines(notes: Mapping[str, Any] | None, masked: set[str]) -> list[st （剖析的备注里有取值范围）。""" labels = ASSISTANT_PROMPT_LABELS lines: list[str] = [] - for key, item in catalog._slots(notes).items(): + # 表级的项、关联关系在前，列的项在后（稳定排序，各自照目录里的顺序）；遮罩列上的码值不算 + slots = [(key, item) for key, item in sorted(catalog._slots(notes).items(), key=lambda kv: kv[0][0] == "c") + if not (key[0] == "c" and key[2] == "codes" and key[1].lower() in masked)] + for shown, (key, item) in enumerate(slots): + if shown >= CATALOG_ITEMS_SHOWN: + lines.append(labels["catalog_more"].format(n=len(slots) - shown)) + break hidden = _touches_masked(key, item, masked) - if hidden and key[0] == "c" and key[2] == "codes": - continue if key[0] == "r": label, value = labels["field_relation"], _relation_line(item) else: diff --git a/backend/app/data/catalog_transfer.py b/backend/app/data/catalog_transfer.py index 34954f8d..6dccd7ec 100644 --- a/backend/app/data/catalog_transfer.py +++ b/backend/app/data/catalog_transfer.py @@ -30,6 +30,7 @@ tables: {表名: {notes, version}}}，含全部状态的项，表结构里已经 from __future__ import annotations import copy +import json import re from collections.abc import Mapping from dataclasses import dataclass, field @@ -51,7 +52,8 @@ _UNNAMED = "数据源" #: 状态：照带的、只填空缺的 _LOCKED = ("confirmed", "rejected") -_STATUS_LABEL = {"proposed": "推断", "verified": "有确证", "confirmed": "已确认", "rejected": "已驳回"} +#: 状态的界面叫法（同 frontend/src/lib/terms.ts 的 CATALOG_STATUS_LABEL） +_STATUS_LABEL = {"proposed": "推断", "verified": "已验证", "confirmed": "已确认", "rejected": "已驳回"} #: 写入一直撞上别人刚改过（重读重算三次仍冲突） WRITE_CONFLICT_ERROR = "这张表的数据目录正被其他人修改，没有导入，请稍后重新导入" @@ -106,7 +108,12 @@ class ImportFile: def read_file(content: Any, *, include_guidance: bool) -> ImportFile: - """导出的那个对象 → ImportFile。格式不对抛 TransferInvalid（说法给人看）。""" + """导出的那个对象 → ImportFile。界面直接交来文件的原文（一段 JSON 文字）也认。格式不对抛 TransferInvalid（说法给人看）。""" + if isinstance(content, str): + try: + content = json.loads(content) + except ValueError as e: + raise TransferInvalid("文件内容不是合法的 JSON，请选择从 AgentLab 导出的数据目录文件") from e if not isinstance(content, Mapping): raise TransferInvalid("文件内容不是数据目录导出的格式，请选择从 AgentLab 导出的数据目录文件") if content.get("format") != FORMAT: diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py index 89f29bd8..f8f90b6f 100644 --- a/backend/tests/test_catalog_assistant.py +++ b/backend/tests/test_catalog_assistant.py @@ -351,6 +351,18 @@ async def test_background_has_structure_catalog_and_hides_masked_values(client, assert "没开剖析" not in system +async def test_long_catalogs_are_cut_with_a_note(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + monkeypatch.setattr(catalog_assistant, "CATALOG_ITEMS_SHOWN", 2) + model = Scripted([Turn(text=["好。"])]) + use(monkeypatch, model) + await _talk(client, sid, _ask("看看")) + system = model.seen[0][0].content + # 表级的项在前；遮罩列的码值本来就不算；剩下的说明还有几项 + assert "- label（中文名）：订单" in system and "- grain（粒度）" in system and "- kind（表类型）" not in system + assert "还有 3 项没有列出" in system + + async def test_without_profiling_there_is_no_query_tool(client, tmp_path, monkeypatch): sid = await _source(client, tmp_path, profile=False) model = Scripted([Turn(text=["只能根据表结构推断。"])]) diff --git a/backend/tests/test_catalog_transfer.py b/backend/tests/test_catalog_transfer.py index de5164b5..e9e50b98 100644 --- a/backend/tests/test_catalog_transfer.py +++ b/backend/tests/test_catalog_transfer.py @@ -334,6 +334,11 @@ async def test_import_waits_for_draft_jobs_and_rejects_bad_files(client): r = await client.post(f"{_base(local)}/import/preview", json={"content": body}) assert r.status_code == 422, body assert any("一" <= ch <= "鿿" for ch in r.json()["detail"]) + # 界面直接交来文件原文也认；不是 JSON 的 422 + as_text = await client.post(f"{_base(local)}/import/preview", json={"content": json.dumps(content)}) + assert as_text.status_code == 200 and as_text.json()["totals"]["tables"] == 2 + not_json = await client.post(f"{_base(local)}/import/preview", json={"content": "{不是 JSON"}) + assert not_json.status_code == 422 and "JSON" in not_json.json()["detail"] too_long = {**content, "guidance": "长" * (catalog_jobs.GUIDANCE_MAX + 1)} r = await client.post(f"{_base(local)}/import/preview", json={"content": too_long, "include_guidance": True}) assert r.status_code == 422 and "起草要求" in r.json()["detail"] From 8c270987bd489afeace001349f83bb13366ede55 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:28:30 +0800 Subject: [PATCH 06/39] =?UTF-8?q?=E8=A1=A8=E5=8A=A9=E6=89=8B=EF=BC=9A?= =?UTF-8?q?=E7=BB=91=E5=AE=9A=E5=B7=A5=E5=85=B7=E5=87=BA=E9=94=99=EF=BC=88?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E4=B8=8D=E6=94=AF=E6=8C=81=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E4=B9=8B=E7=B1=BB=EF=BC=89=E4=B9=9F=E5=9C=A8?= =?UTF-8?q?=E6=B5=81=E9=87=8C=E7=85=A7=E5=AE=9E=E4=BA=A4=E4=B8=80=E6=9D=A1?= =?UTF-8?q?=E9=94=99=E8=AF=AF=E4=BA=8B=E4=BB=B6=EF=BC=8C=E4=B8=8D=E8=AE=A9?= =?UTF-8?q?=E6=B5=81=E5=8D=8A=E8=B7=AF=E6=96=AD=E6=8E=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/data/catalog_assistant.py | 3 ++- backend/tests/test_catalog_assistant.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/backend/app/data/catalog_assistant.py b/backend/app/data/catalog_assistant.py index 9dadbd98..c6e5d9fa 100644 --- a/backend/app/data/catalog_assistant.py +++ b/backend/app/data/catalog_assistant.py @@ -813,11 +813,12 @@ async def stream(model: Any, model_id: str, ctx: AssistantContext, history: list[Any] = [SystemMessage(content=ctx.system), *_history(messages)] tools = _Tools(ctx) - runner = model.bind_tools(ctx.tools()) usage = {"input_tokens": 0, "output_tokens": 0} wrote = "" names = [t["function"]["name"] for t in ctx.tools()] try: + # 绑工具也放在这里面：响应头已经发出去了，出了错要照实交一条错误事件，不能让流半路断掉 + runner = model.bind_tools(ctx.tools()) for round_no in range(MAX_ROUNDS): fresh = True message: Any = None diff --git a/backend/tests/test_catalog_assistant.py b/backend/tests/test_catalog_assistant.py index f8f90b6f..8013dd75 100644 --- a/backend/tests/test_catalog_assistant.py +++ b/backend/tests/test_catalog_assistant.py @@ -524,6 +524,19 @@ async def test_errors_end_the_stream_without_done(client, tmp_path, monkeypatch) assert "上限" in events[-1]["message"] +async def test_a_model_without_tools_reports_an_error_event(client, tmp_path, monkeypatch): + sid = await _source(client, tmp_path) + + class NoTools(Scripted): + def bind_tools(self, tools, **_): + raise NotImplementedError("这个模型不支持工具调用") + + use(monkeypatch, NoTools([])) + events, _ = await _talk(client, sid, _ask("你好")) + assert events == [{"type": "error", "code": "assistant_failed", "message": events[0]["message"]}] + assert "工具调用" in events[0]["message"] + + async def test_transient_errors_are_retried_before_any_output(client, tmp_path, monkeypatch): sid = await _source(client, tmp_path) monkeypatch.setattr(catalog_assistant, "RETRY_DELAYS_S", (0.0, 0.0)) From 447c66649c9d2bd7e2619545423820f06e8c21bd Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:30:14 +0800 Subject: [PATCH 07/39] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E9=97=AD=E7=8E=AF=E7=AC=AC=E4=B8=80=E6=AD=A5=EF=BC=88=E5=90=8E?= =?UTF-8?q?=E7=AB=AF=20A=EF=BC=89=EF=BC=9A=E6=9C=89=E6=95=88=E8=A1=8C?= =?UTF-8?q?=E8=BF=87=E6=BB=A4=E7=9A=84=E6=A0=A1=E9=AA=8C=E5=92=8C=E5=89=96?= =?UTF-8?q?=E6=9E=90=E6=A0=B8=E5=AF=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - sqlcheck.parse_valid_filter / valid_filter_problem：按数据源的方言用 sqlglot 把有效行过滤解析成一个条件，只能引用这张表的列（不带表名或带这张表的表名，不分大小写），不能有子查询、聚合、窗口函数、分号、注释、模板占位、参数占位、算术，只允许比较、IN（值的列表）、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE，不超过 500 字，至少用到一列；不认识的数据源类型不收 - sqlcheck.render_valid_filter：把条件按方言重新写出来，列换成表结构里的真名、按剖析查询的规矩加引号，拼进核对查询用 - catalog_profile.check_valid_filter：开了数据剖析的源经剖析的只读守卫（guard.check(readonly=True)、单条时限、查询次数上限）发一条查询，只取条件里的列、先取前 max(抽样键值数, 整表统计行数上限) 行，数出总行数和被排除的行数；排除 0% 或 100% 的不收，收下的写「剖析核对：排除 3.2% 的行（共 12,345 行）」，只数了前若干行的写明按前几行统计；没开剖析、数据源停用、条件里有遮罩的列、视图、取值拼不进查询、查询失败或超时照收，写明没有核对的原因 --- backend/app/data/catalog_profile.py | 128 ++++++++++++- backend/app/data/sqlcheck.py | 186 ++++++++++++++++++- backend/tests/test_catalog_draft_facts.py | 215 ++++++++++++++++++++++ 3 files changed, 523 insertions(+), 6 deletions(-) create mode 100644 backend/tests/test_catalog_draft_facts.py diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 6966b26d..1e9ec5bb 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -1341,8 +1341,128 @@ async def _profile(session: AsyncSession, row: Any, source: Any, tables: list[st return report -__all__ = ["CODES_MAX", "IN_CHUNK", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES", +# ========================================================================== +# 核对模型提议的有效行过滤：排除了多少行 +# ========================================================================== + +#: 有效行过滤核对的说法（写进这一项的 note；前面是模型给的依据） +FILTER_CHECKED = "剖析核对：排除 {share} 的行（共 {rows} 行）" +FILTER_SAMPLED = "剖析核对：按前 {rows} 行统计，排除 {share} 的行" +FILTER_SAMPLED_EDGE = "剖析核对：按前 {rows} 行统计，{what}，请人工核对" +FILTER_UNCHECKED = "没有核对排除比例（{why}）" +FILTER_NOT_ENABLED = "数据源没有开数据剖析" + + +@dataclass(frozen=True) +class FilterCheck: + """核对一条有效行过滤的结果。 + + keep：收不收（排除 0% 或 100% 的不收：前者什么都没筛掉，后者把所有行都筛掉了，多半写错了）；note：写进这一项 note + 的那句（剖析核对的排除比例，或者没有核对的原因）；rows / excluded：数到的行数和其中被排除的行数（没核对是 None）； + sampled：表太大或不知道多大，只数了前若干行；dropped：不收的原因（记日志用）。""" + + keep: bool + note: str + rows: int | None = None + excluded: int | None = None + sampled: bool = False + dropped: str | None = None + + +def excluded_share(excluded: int, rows: int) -> str: + """排除比例写成百分数：留一位小数（整数不带小数）；不到 0.1% 的写「不足 0.1%」，不是全部的不四舍五入成 100%。""" + if rows <= 0: + return "0%" + pct = excluded * 100 / rows + if 0 < pct < 0.1: + return "不足 0.1%" + if 99.9 < pct < 100: + return "超过 99.9%" + return f"{pct:.1f}".rstrip("0").rstrip(".") + "%" + + +def _unchecked(why: str) -> FilterCheck: + return FilterCheck(True, FILTER_UNCHECKED.format(why=why)) + + +def _unsafe_literal(value: str) -> bool: + """拼不进核对查询的字符串取值：查询层用 SQLAlchemy 的 text() 执行，它会把字符串里的「:名字」当成绑定参数； + 反斜杠在 MySQL 上读法不定、守卫会整条拒掉；控制字符不该出现（同 sql_literal 的规矩）。""" + return "\\" in value or bool(_CONTROL.search(value)) or bool(_BIND_LIKE.search(value)) + + +async def check_valid_filter(source: Any, table: str, meta: dict[str, Any] | None, condition: str, *, + settings: ProfileSettings | None = None) -> FilterCheck: + """开了数据剖析的源：数一数这条有效行过滤排除了多少行（模型起草时提议的那条，data/catalog_draft.py 调）。从不抛异常。 + + 经剖析的只读守卫（_Runner：guard.check(readonly=True)、单条时限、查询次数上限）发一条查询：只取条件里用到的列， + 先取前 max(sample_size, max_scan_rows) 行（大表不做整表扫描），再数总行数和不满足条件的行数（条件为 NULL 也算排除， + 和 WHERE 一样）。数到的不满前那么多行，就是整张表。 + + - 排除 0% 或 100% 的不收（keep 为假）；只数了前若干行的照收，写明「请人工核对」； + - 没开剖析、数据源停用、条件里有遮罩的列、视图、取值拼不进查询、查询失败或超时：照收，note 写没有核对的原因。 + """ + from app.data import sqlcheck + + options = getattr(source, "options", None) + settings = settings or profile_settings(options) + if not settings.enabled: + return _unchecked(FILTER_NOT_ENABLED) + if getattr(source, "enabled", True) is False: + return _unchecked("数据源已停用") + kind = getattr(source, "kind", None) + parsed = sqlcheck.parse_valid_filter(condition, meta, kind=kind, table=table) + if parsed.problem: + return FilterCheck(False, parsed.problem, dropped=parsed.problem) + masked = {c.lower() for c in data_engine.masked_columns(options)} + if hit := [c for c in parsed.columns if c.lower() in masked]: + return _unchecked(f"条件里的 {'、'.join(hit)} 在数据源设置中被遮罩，不取数") + if (meta or {}).get("is_view"): + return _unchecked("这张表是视图，剖析可能触发整个视图的计算") + from sqlglot import exp + + if any(n.is_string and _unsafe_literal(str(n.this)) for n in parsed.expression.find_all(exp.Literal)): + return _unchecked("条件里的取值含有冒号或反斜杠，拼不进核对查询") + try: + dialect = SqlDialect(kind) + except ValueError as e: + return _unchecked(str(e)) + cap = max(int(settings.sample_size), int(settings.max_scan_rows)) + cond = sqlcheck.render_valid_filter(parsed, quoted=lambda name: dialect.quote(name) != name) + cols = ", ".join(dialect.quote(c) for c in parsed.columns) + inner = dialect.limit(f"SELECT {cols} FROM {dialect.table(meta, table)}", cap) + sql = f"SELECT COUNT(*) AS n, SUM(CASE WHEN {cond} THEN 0 ELSE 1 END) AS x FROM ({inner}) s" + run = _Runner(source, settings, dialect) + try: + res = await run(sql) + except _Skip as e: + return _unchecked(e.detail.removesuffix("，已跳过")) + except _Stop as e: + return _unchecked(_STOP_DETAIL[e.reason].format(max_queries=settings.max_queries, + max_total_s=_fmt(settings.max_total_s)).removesuffix("，未检查")) + except SnapshotTampered as e: + return _unchecked(str(e)) + row = res.rows[0] if res.rows else [None, None] + rows, excluded = _as_int(row[0]), _as_int(row[1]) if len(row) > 1 else None + if not rows: + return _unchecked("表里还没有数据") + excluded = excluded or 0 + sampled = rows >= cap + counted = f"{rows:,}" + if excluded in (0, rows): + what = "没有排除任何一行" if excluded == 0 else "所有行都被排除了" + if sampled: + return FilterCheck(True, FILTER_SAMPLED_EDGE.format(rows=counted, what=what), rows, excluded, True) + return FilterCheck(False, FILTER_CHECKED.format(share=excluded_share(excluded, rows), rows=counted), rows, + excluded, False, dropped=what) + share = excluded_share(excluded, rows) + note = (FILTER_SAMPLED if sampled else FILTER_CHECKED).format(share=share, rows=counted) + return FilterCheck(True, note, rows, excluded, sampled) + + +__all__ = ["CODES_MAX", "FILTER_CHECKED", "FILTER_NOT_ENABLED", "FILTER_SAMPLED", "FILTER_UNCHECKED", "FilterCheck", + "IN_CHUNK", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES", "PROFILE_OPTION", "PROFILE_PROBE_TABLES", "ProfileBusy", "ProfileDisabled", "ProfileReport", - "ProfileSettings", "ProfileSkip", "SqlDialect", "TableProfile", "TableSize", "ensure_enabled", - "estimate_size", "profile_catalog", "profile_settings", "profile_settings_problem", "profile_settings_rejection", - "sql_literal"] + "ProfileSettings", "ProfileSkip", "SqlDialect", "TableProfile", "TableSize", "check_valid_filter", + "ensure_enabled", "estimate_size", "excluded_share", "profile_catalog", "profile_settings", + "profile_settings_problem", "profile_settings_rejection", "sql_literal"] diff --git a/backend/app/data/sqlcheck.py b/backend/app/data/sqlcheck.py index c233d430..8cd6e440 100644 --- a/backend/app/data/sqlcheck.py +++ b/backend/app/data/sqlcheck.py @@ -1131,5 +1131,187 @@ async def load_checkers_by_name(session: Any, names: Iterable[str]) -> dict[str, return await load_checkers(session, rows) -__all__ = ["CHECK_CODES", "LEVELS", "MAX_CHECKS", "MAX_SQL_CHARS", "SqlCheck", "SqlChecker", "check_sql", - "dialect_of", "graph_checks", "graph_sources", "load_checkers", "load_checkers_by_name", "tool_query"] +# ========================================================================== +# 有效行过滤（valid_filter）的校验：模型起草的条件要能进目录、能拼进剖析的核对查询 +# ========================================================================== + +#: 有效行过滤最多几个字（同目录里文本项的上限） +VALID_FILTER_MAX = 500 +#: 有效行过滤里能用的函数（其余函数、聚合、窗口一律不收） +VALID_FILTER_FUNCTIONS = ("UPPER", "LOWER", "TRIM", "COALESCE") +#: 模板占位：条件里不该有，有了也说不清最终拼出来的是什么（注释在解析之后按语法树认，字符串里的 -- 不算） +_FILTER_TEMPLATE = re.compile(r"\{\{|\{%") + +_FILTER_PREFIX = "有效行过滤" + + +@dataclass(frozen=True) +class ValidFilter: + """一段有效行过滤解析的结果。 + + expression：sqlglot 的条件（不合规时为 None）；columns：条件里用到的列（表结构里的真名，按出现的先后去重）； + names：小写列名 → 表结构里的真名；problem：不合规时给人看的一句话，合规是 None；dialect：解析用的 sqlglot 方言。 + """ + + expression: Any = None + columns: tuple[str, ...] = () + names: Mapping[str, str] = field(default_factory=dict) + problem: str | None = None + dialect: str | None = None + + +def _bad_filter(problem: str, dialect: str | None = None) -> ValidFilter: + return ValidFilter(problem=f"{_FILTER_PREFIX}{problem}", dialect=dialect) + + +def _filter_node_problem(node: Any, *, predicate: bool) -> str | None: + """一个不在允许范围里的节点为什么不收（没有问题返回 None 的调用方不会走到这里）。""" + from sqlglot import exp + + if isinstance(node, exp.Query): + return "应为一个条件，不能是一条查询" + if any(isinstance(n, (exp.Query, exp.Subquery)) for n in node.walk()): + return "里不能有子查询" + if any(isinstance(n, exp.Window) for n in node.walk()): + return "里不能有窗口函数" + if any(isinstance(n, exp.AggFunc) for n in node.walk()): + return "里不能有聚合函数" + if isinstance(node, exp.Func): + name = str(node.this) if isinstance(node, exp.Anonymous) else node.sql_name() + return f"里只能用 {'、'.join(VALID_FILTER_FUNCTIONS)} 这几个函数，不能用 {name.upper()}" + if predicate: + return "应为一个条件（比较、IN、BETWEEN、LIKE、IS NULL，用 AND、OR、NOT 连接）" + text = node.sql() + return f"里有不支持的写法（{text[:40]}{'…' if len(text) > 40 else ''}）" + + +def _filter_value_problem(node: Any) -> str | None: + """比较两边的一个值：列、字面量、负数、允许的几个函数（参数同样是值）。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, (exp.Column, exp.Literal, exp.Null, exp.Boolean)): + return None + if isinstance(node, exp.Neg) and isinstance(_unparen(node.this), exp.Literal) and not _unparen(node.this).is_string: + return None + if isinstance(node, (exp.Upper, exp.Lower, exp.Trim, exp.Coalesce)): + for arg in node.iter_expressions(): + if problem := _filter_value_problem(arg): + return problem + return None + return _filter_node_problem(node, predicate=False) + + +def _filter_predicate_problem(node: Any) -> str | None: + """一个条件：比较、IN（只能是值的列表）、BETWEEN、LIKE、IS NULL，或者用 AND / OR / NOT 连起来的条件。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, (exp.And, exp.Or)): + return _filter_predicate_problem(node.left) or _filter_predicate_problem(node.right) + if isinstance(node, exp.Not): + return _filter_predicate_problem(node.this) + if isinstance(node, (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE)): + return _filter_value_problem(node.left) or _filter_value_problem(node.right) + if isinstance(node, exp.In): + if any(node.args.get(k) for k in ("query", "unnest", "field")): + return "里不能有子查询" + if not node.expressions: + return _filter_node_problem(node, predicate=True) + return _filter_value_problem(node.this) or next( + (p for p in (_filter_value_problem(e) for e in node.expressions) if p), None) + if isinstance(node, exp.Between): + return next((p for p in (_filter_value_problem(node.args.get(k)) for k in ("this", "low", "high")) if p), None) + if isinstance(node, exp.Like): + return _filter_value_problem(node.this) or _filter_value_problem(node.expression) + if isinstance(node, exp.Is): + if not isinstance(_unparen(node.expression), exp.Null): + return "里的 IS 只能写 IS NULL 或 IS NOT NULL" + return _filter_value_problem(node.this) + return _filter_node_problem(node, predicate=True) + + +def parse_valid_filter(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None, + table: str | None = None) -> ValidFilter: + """按数据源的方言把有效行过滤解析成一个条件，并核对它只用了这张表的列。从不抛异常，不合规写在 problem 里。 + + 规矩：一段文字、不超过 VALID_FILTER_MAX 字；不能有分号、模板占位、注释；解析出来恰好是一个条件，只由比较、IN（值的 + 列表）、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE 组成（不能有子查询、聚合、窗口、 + 别的函数、算术、参数占位）；列不带表名或带这张表的表名（table），都在表结构里（不分大小写），至少用到一列。 + """ + from app.data.names import name_key as _key + + dialect = dialect_of(kind) + if not isinstance(condition, str) or not condition.strip(): + return _bad_filter("应为一段 SQL 条件", dialect) + text = condition.strip() + if len(text) > VALID_FILTER_MAX: + return _bad_filter(f"超过 {VALID_FILTER_MAX} 字", dialect) + if ";" in text: + return _bad_filter("里不能有分号", dialect) + if _FILTER_TEMPLATE.search(text): + return _bad_filter("里不能有模板占位", dialect) + if dialect is None: + return _bad_filter(f"没法校验：数据源类型「{kind}」不支持", dialect) + try: + import sqlglot + + statements = [s for s in sqlglot.parse(text, read=dialect) if s is not None] + except Exception: # noqa: BLE001 - 语法错、方言不支持的写法：一律不收 + return _bad_filter("不是一段能解析的 SQL 条件", dialect) + if len(statements) != 1: + return _bad_filter("不是一段能解析的 SQL 条件", dialect) + expression = statements[0] + if any(node.comments for node in expression.walk()): + return _bad_filter("里不能有注释", dialect) + if problem := _filter_predicate_problem(expression): + return _bad_filter(problem, dialect) + from sqlglot import exp + + lookup = {str(c.get("name")).lower(): str(c.get("name")) for c in (meta or {}).get("columns") or [] + if isinstance(c, Mapping) and c.get("name") is not None} + own = {_key(str(n).rsplit(".", 1)[-1]) for n in (table, (meta or {}).get("qualified")) if n} + used: dict[str, str] = {} + for column in expression.find_all(exp.Column, bfs=False): # 按书写的先后 + if column.is_star: + return _bad_filter("里不能写 *", dialect) + qualifier = column.table + if column.args.get("db") or column.args.get("catalog") or (qualifier and _key(qualifier) not in own): + return _bad_filter(f"只能引用这张表的列，不能引用 {column.sql(dialect=dialect)}", dialect) + real = lookup.get(column.name.lower()) + if real is None: + return _bad_filter(f"里的列「{column.name}」不在表结构里", dialect) + used.setdefault(column.name.lower(), real) + if not used: + return _bad_filter("没有用到这张表的任何列", dialect) + return ValidFilter(expression=expression, columns=tuple(dict.fromkeys(used.values())), names=used, + dialect=dialect) + + +def valid_filter_problem(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None, + table: str | None = None) -> str | None: + """有效行过滤哪里不合规（给人看的一句话）；合规返回 None。规矩见 parse_valid_filter。""" + return parse_valid_filter(condition, meta, kind=kind, table=table).problem + + +def render_valid_filter(parsed: ValidFilter, *, quoted: Any) -> str: + """把解析好的条件按数据源的方言重新写出来，列换成表结构里的真名、不带表名；quoted(真名) 为真的加引号（和剖析查询 + 其余部分的写法一致：SqlDialect.quote 要加引号的才加）。拼进剖析的核对查询用，不改目录里存的原文。""" + from sqlglot import exp + + if parsed.expression is None: + raise ValueError(parsed.problem or "有效行过滤不合规") + + def swap(node: Any) -> Any: + if isinstance(node, exp.Column): + real = parsed.names.get(node.name.lower(), node.name) + return exp.Column(this=exp.Identifier(this=real, quoted=bool(quoted(real)))) + return node + + return parsed.expression.copy().transform(swap).sql(dialect=parsed.dialect) + + +__all__ = ["CHECK_CODES", "LEVELS", "MAX_CHECKS", "MAX_SQL_CHARS", "SqlCheck", "SqlChecker", "VALID_FILTER_FUNCTIONS", + "VALID_FILTER_MAX", "ValidFilter", "check_sql", "dialect_of", "graph_checks", "graph_sources", + "load_checkers", "load_checkers_by_name", "parse_valid_filter", "render_valid_filter", "tool_query", + "valid_filter_problem"] diff --git a/backend/tests/test_catalog_draft_facts.py b/backend/tests/test_catalog_draft_facts.py new file mode 100644 index 00000000..dba6db48 --- /dev/null +++ b/backend/tests/test_catalog_draft_facts.py @@ -0,0 +1,215 @@ +"""防错事实的提议：模型起草时提议有效行过滤（valid_filter）、去重规则（dedup），附一句依据。 + +- 提示词写清两项只在表结构、注释、码值能看出来时才写；结构化输出、纯文本两条路都认这两项； +- 有效行过滤按数据源的方言用 sqlglot 解析成一个条件：只能引用这张表的列，不能有子查询、聚合、窗口函数、分号，只允许 + 比较、IN、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE，不超过 500 字。不过就丢掉； +- 开了数据剖析、能连上的源：经剖析的只读守卫数出排除比例，排除 0% 或 100% 的不收，收下的在 note 里写剖析核对的结果； + 没开剖析、连不上的照收，note 写没有核对的原因； +- 开了剖析的源，目录里记下的码值交给模型做背景（遮罩的列不给）； +- 这两项照旧记模型起草 / 推断；SQL 检查只在人工确认、有确证时才把漏了有效行过滤报成警告，推断的只是提示。 + +用合成的景区业务库（tests/fixtures/catalog/scenic.py）和虚构的表结构，不调用真实模型。 +""" +from __future__ import annotations + +import logging +import sqlite3 +from types import SimpleNamespace + +import pytest + +from app.data import catalog, catalog_profile, sqlcheck +from app.data import catalog_draft as D +from app.data.catalog import make_item +from app.data.catalog_profile import PROFILE_OPTION +from app.data.engine import MASK_COLUMNS_OPTION, engines +from app.data.introspect import introspect +from tests.fixtures.catalog.fake_model import FakeModel, answer, requested + +# -------------------------------------------------------------------------- +# 夹具 +# -------------------------------------------------------------------------- + + +def _col(name: str, type_: str = "INTEGER", comment: str | None = None) -> dict: + return {"name": name, "type": type_, "nullable": True, "comment": comment} + + +def _orders() -> dict: + return {"qualified": "orders", "schema": None, "is_view": False, "primary_key": ["id"], "foreign_keys": [], + "comment": None, + "columns": [_col("id"), _col("order_no", "TEXT"), _col("status", "TEXT", comment="C 作废，X 测试单"), + _col("amount", "REAL"), _col("updated_at", "TEXT")]} + + +def _plain(tables: dict, **kw) -> SimpleNamespace: + """不连库的源（没有 options：没开数据剖析）。""" + return SimpleNamespace(id="facts-src", name="facts", kind=kw.pop("kind", "sqlite"), origin="manual", + description="", schema_cache={"tables": tables}, **kw) + + +_CACHE: dict[str, dict] = {} + + +async def _scenic_cache(path: str) -> dict: + if path not in _CACHE: + probe = SimpleNamespace(id="facts-scenic-probe", name="scenic", kind="sqlite", database=path, host=None, + port=None, username=None, password=None, options={}, readonly=True, description="", + schema_cache={}, origin="manual") + try: + _CACHE[path] = await introspect(probe) + finally: + await engines.invalidate(probe.id) + return _CACHE[path] + + +@pytest.fixture +async def scenic(scenic_db): + """接上景区库的源（开了数据剖析）。用完作废连接池。""" + made: list[str] = [] + + async def make(*, options: dict | None = None, database: str | None = None, enabled: bool = True): + sid = f"facts-{len(made)}-{id(made)}" + made.append(sid) + opts = {PROFILE_OPTION: {"enabled": True}} if options is None else options + return SimpleNamespace(id=sid, name="scenic", kind="sqlite", database=database or scenic_db, host=None, + port=None, username=None, password=None, options=opts, readonly=True, description="", + schema_cache=await _scenic_cache(scenic_db), origin="manual", enabled=enabled) + + yield make + for sid in made: + await engines.invalidate(sid) + + +def _count(path: str, where: str) -> tuple[int, int]: + """(全表行数, 不满足条件的行数)：直接用 sqlite3 数，核对剖析算出来的数。""" + db = sqlite3.connect(path) + try: + total = db.execute("SELECT COUNT(*) FROM visits").fetchone()[0] + kept = db.execute(f"SELECT COUNT(*) FROM visits WHERE {where}").fetchone()[0] + finally: + db.close() + return total, total - kept + + +# -------------------------------------------------------------------------- +# 校验有效行过滤 +# -------------------------------------------------------------------------- + +META = _orders() + + +@pytest.mark.parametrize("condition, columns", [ + ("status NOT IN ('C', 'X')", ["status"]), + ("STATUS NOT IN ('C', 'X')", ["status"]), # 列名不分大小写，记表结构里的写法 + ("orders.status <> 'C'", ["status"]), # 带这张表的表名 + ("status IS NOT NULL AND amount > 0", ["status", "amount"]), + ("NOT (status = 'C' OR status = 'X')", ["status"]), + ("amount BETWEEN 0 AND 100000", ["amount"]), + ("order_no NOT LIKE 'TEST%'", ["order_no"]), + ("UPPER(TRIM(status)) <> 'C'", ["status"]), + ("COALESCE(status, '') <> 'X' AND amount >= -1", ["status", "amount"]), + ("status = 'C' OR updated_at IS NULL", ["status", "updated_at"]), +]) +def test_valid_filters_that_pass(condition, columns): + parsed = sqlcheck.parse_valid_filter(condition, META, kind="sqlite", table="orders") + assert parsed.problem is None, parsed.problem + assert list(parsed.columns) == columns + assert sqlcheck.valid_filter_problem(condition, META, kind="mysql", table="orders") is None + + +@pytest.mark.parametrize("condition, said", [ + ("status = 'C'; DROP TABLE orders", "分号"), + ("status IN (SELECT code FROM codes)", "子查询"), + ("EXISTS (SELECT 1 FROM refunds)", "子查询"), + ("COUNT(status) > 1", "聚合"), + ("ROW_NUMBER() OVER (ORDER BY id) = 1", "窗口"), + ("LENGTH(status) > 0", "LENGTH"), + ("amount * 2 > 10", "不支持的写法"), + ("payments.status = 1", "只能引用这张表的列"), + ("nope = 1", "nope"), + ("1 = 1", "没有用到这张表的任何列"), + ("status", "一个条件"), + ("status = ?", "不支持的写法"), + ("status = 'C' -- 注释", "注释"), + ("status = '{{ input.code }}'", "模板"), + ("status = '" + "C" * 600 + "'", "500"), + ("status = (", "解析"), + ("status = 'C' UNION SELECT 1", "一个条件"), +]) +def test_valid_filters_that_are_dropped(condition, said): + problem = sqlcheck.valid_filter_problem(condition, META, kind="sqlite", table="orders") + assert problem is not None and said in problem, problem + + +def test_a_source_kind_without_a_dialect_cannot_be_checked(): + assert "不支持" in sqlcheck.valid_filter_problem("status = 'C'", META, kind="mssql", table="orders") + + +def test_the_condition_is_rendered_with_the_columns_quoted_like_the_source(): + meta = {**META, "columns": [*META["columns"], _col("isTest")]} + parsed = sqlcheck.parse_valid_filter("istest = 0 AND STATUS <> 'C'", meta, kind="mysql", table="orders") + assert parsed.problem is None + rendered = sqlcheck.render_valid_filter(parsed, quoted=lambda name: name != name.lower()) + assert rendered == "`isTest` = 0 AND status <> 'C'" + + +# -------------------------------------------------------------------------- +# 剖析核对排除比例 +# -------------------------------------------------------------------------- + + +async def test_the_check_counts_excluded_rows_through_the_read_only_guard(scenic, scenic_db, monkeypatch): + seen: list[tuple[str, bool]] = [] + real = catalog_profile.guard.check + + def spy(sql, **kw): + seen.append((sql, kw.get("readonly"))) + return real(sql, **kw) + + monkeypatch.setattr(catalog_profile.guard, "check", spy) + source = await scenic() + meta = source.schema_cache["tables"]["visits"] + found = await catalog_profile.check_valid_filter(source, "visits", meta, "status = 1") + total, excluded = _count(scenic_db, "status = 1") + assert 0 < excluded < total + assert found.keep and (found.rows, found.excluded) == (total, excluded) and not found.sampled + share = catalog_profile.excluded_share(excluded, total) + assert found.note == f"剖析核对：排除 {share} 的行（共 {total:,} 行）" + assert len(seen) == 1 and seen[0][1] is True, "一条查询，过只读守卫" + assert "COUNT(*)" in seen[0][0] and "status = 1" in seen[0][0] + + +@pytest.mark.parametrize("condition", ["status IN (0, 1)", "status = 7"]) +async def test_a_filter_that_excludes_nothing_or_everything_is_dropped(scenic, condition): + source = await scenic() + found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"], + condition) + assert not found.keep and found.dropped + + +async def test_a_big_table_is_checked_on_its_first_rows(scenic): + source = await scenic(options={PROFILE_OPTION: {"enabled": True, "max_scan_rows": 100, "sample_size": 10}}) + found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"], + "status = 1") + assert found.sampled and found.rows == 100 + assert found.note.startswith("剖析核对：按前 100 行统计，") + + +async def test_unchecked_filters_are_kept_with_the_reason(scenic, tmp_path): + meta_of = lambda s: s.schema_cache["tables"]["visits"] # noqa: E731 + off = await scenic(options={}) + found = await catalog_profile.check_valid_filter(off, "visits", meta_of(off), "status = 1") + assert found.keep and found.note == "没有核对排除比例（数据源没有开数据剖析）" + + masked = await scenic(options={PROFILE_OPTION: {"enabled": True}, MASK_COLUMNS_OPTION: "Status"}) + found = await catalog_profile.check_valid_filter(masked, "visits", meta_of(masked), "status = 1") + assert found.keep and found.note == "没有核对排除比例（条件里的 status 在数据源设置中被遮罩，不取数）" + + stopped = await scenic(enabled=False) + found = await catalog_profile.check_valid_filter(stopped, "visits", meta_of(stopped), "status = 1") + assert found.keep and found.note == "没有核对排除比例（数据源已停用）" + + gone = await scenic(database=str(tmp_path / "missing" / "nope.db")) + found = await catalog_profile.check_valid_filter(gone, "visits", meta_of(gone), "status = 1") + assert found.keep and found.note.startswith("没有核对排除比例（查询失败：") From 7c3484adba4eacccc7f23c3125023836a6d5c51b Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:31:35 +0800 Subject: [PATCH 08/39] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E9=97=AD=E7=8E=AF=E7=AC=AC=E4=B8=80=E6=AD=A5=EF=BC=88=E5=90=8E?= =?UTF-8?q?=E7=AB=AF=20A=EF=BC=89=EF=BC=9A=E8=B5=B7=E8=8D=89=E9=81=97?= =?UTF-8?q?=E7=95=99=E2=80=94=E2=80=94=E5=AE=BD=E8=A1=A8=E9=80=90=E6=AE=B5?= =?UTF-8?q?=E8=90=BD=E7=9B=98=E3=80=81=E6=8C=89=E6=AF=94=E4=BE=8B=E6=AD=A2?= =?UTF-8?q?=E6=8D=9F=E3=80=81=E5=9D=97=E8=BF=9B=E5=BA=A6=E3=80=81=E5=81=9C?= =?UTF-8?q?=E6=AD=A2=E5=90=8E=E7=9A=84=E8=AF=B4=E6=B3=95=E3=80=81=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E8=B5=B7=E8=8D=89=E4=B8=8E=E4=BB=BB=E5=8A=A1=E4=BA=92?= =?UTF-8?q?=E6=96=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 宽表逐段落盘：每写出一段（表级那段、每个列分段）就把这张表到这时拿到的内容并入目录（catalog_jobs.write_part，这一次合并不删任何东西）；整张表写完（ok、partial 都一样）再按完整覆盖的规则收尾一次，只删已经不在表结构里的列的模型项（write_draft 对 partial 也照删）。停止、服务重启时已经写进去的段留着；写过段的表照旧记 cancelled，model_error 写「已停止：已写进目录 N 段，剩下 M 段没起草」（中断写「已中断：…」）。新增、更新、删除的计数是各次写入的合计 - CatalogDraftTask 加 parts_written（已写进目录的段数）、chunks_total / chunks_done（这张表实际分成的块数和已结束的块数，段被缩小的上限分开、拆成两半后会变多；不用模型时为 null），升级时补列 - 止损按比例：这次起草最近 10 次调用有 7 次以上没写出来时，之后的段不再拆小、每块只问一次（不补问漏掉的列、截断后不再补剩下的、空壳不再原样再问，拆出来还没问的块也不再问），直到比例降下去；去掉会被偶尔一次成功清零的「连续 2 段」「接连超时」两条。结构化输出只给空壳的，这次起草还没换纯文本问过的照旧问一次；额度被拒、结构化输出不支持、网络错误这些通信上的重来照旧 - 同一张表接连 3 个原始段都一项没写出来：剩下没开始的段跳过不问，原因写「这张表接连 3 段都没写出来，剩下的 N 段已跳过，可以换一个起草模型后只重试这张表」 - 原因里「已拆小重试仍没写出」只跟在真拆过的那几段后面说：说到的段只有一部分拆过时写「其中第 3 段已拆小重试仍没写出」 - 目录清单的 last_draft 加 blank_columns_from（最近一次结束于 ok / partial 的用模型起草记下的留空列数；之后有起草写进了新内容时为 null）和 parts_written - 同步起草和后台任务双向互斥：同步起草进行期间同一个源建任务、重试、导入（ensure_idle）、再开一次同步起草都 409 catalog_draft_busy（进程内登记） --- backend/app/api/catalog.py | 22 +- backend/app/data/catalog.py | 27 +- backend/app/data/catalog_draft.py | 383 ++++++++++++++------- backend/app/data/catalog_jobs.py | 192 +++++++++-- backend/app/db/base.py | 4 + backend/app/db/models.py | 7 +- backend/tests/test_catalog_draft_engine.py | 359 +++++++++++++++---- backend/tests/test_catalog_draft_jobs.py | 233 +++++++++++++ 8 files changed, 998 insertions(+), 229 deletions(-) diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 0405a0f7..c8c68126 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -8,7 +8,8 @@ - POST /catalog/{table}/patch 保存目录修改提案（乐观锁），记为人工确认 - GET /catalog/{table}/impact 影响面：引用这张表的已发布、受管模板（data/catalog_impact.py） - POST /catalog/draft 同步起草（注释、外键、命名推断，可选模型）。界面改用下面的后台起草任务，这个留着； - 同一个源有排队或进行中的后台任务时 409 catalog_draft_busy + 和后台任务双向互斥：同一个源有排队或进行中的后台任务（或正在同步起草）时 409 + catalog_draft_busy，同步起草期间建任务、重试、导入也是 409 - PUT /catalog/guidance 数据源级的起草要求（使用者补充的背景和要求，起草时交给模型） - POST /catalog/draft-jobs 建后台起草任务（202）；GET /draft-jobs/active、/latest、/{job_id} 读任务； POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py） @@ -419,20 +420,21 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn, model_items：模型给了这张表几项可用内容（没用模型、模型这部分失败时为 null）。界面据此区分「模型给了内容、 只是和现有目录一致」（model_items > 0、三个数都是 0）和「模型没给内容」（model_error 有值）。 - 同一个源有排队或进行中的后台起草任务时 409 catalog_draft_busy（和建任务同一句话）：两边同时写同一个源的目录、 - 抢模型名额。 + 双向互斥（catalog_jobs.sync_draft，进程内登记）：同一个源有排队或进行中的后台起草任务、或者正在同步起草时 409 + catalog_draft_busy（和建任务同一句话）；反过来，同步起草进行期间同一个源建任务、重试、导入目录也是 409。两边同时写 + 同一个源的目录、抢模型名额。 """ row, source = await _resolved(session, source_id) + cache = source.schema_cache or {} try: - await catalog_jobs.ensure_idle(session, row) + async with catalog_jobs.sync_draft(session, row): + if not cache.get("tables"): + raise _no_schema(cache) + report = await catalog.draft_catalog(session, source, tables=payload.tables or None, + use_model=payload.use_model, actor=actor_of(x_actor), + guidance=catalog_jobs.guidance_of(row)) except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e - cache = source.schema_cache or {} - if not cache.get("tables"): - raise _no_schema(cache) - report = await catalog.draft_catalog(session, source, tables=payload.tables or None, - use_model=payload.use_model, actor=actor_of(x_actor), - guidance=catalog_jobs.guidance_of(row)) rows = [{"table_name": r.table, "added": r.added, "updated": r.updated, "removed": r.removed, "version": r.version, "error": r.error, "model_error": r.model_error, "model_items": r.model_items} for r in report.tables] diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 3dd0bb39..01eb87c1 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -1488,9 +1488,13 @@ MODEL_FALLBACK_MAX_TOKENS = 8192 #: 一段没写出来（额度用完、只给空壳）时拆成两半再问：最多拆几层、拆到最少几列为止 MODEL_SPLIT_DEPTH = 3 MODEL_SPLIT_MIN_COLUMNS = 5 -#: 止损：连续这么多段拆到底仍一项没写出来，这次起草后面的段不再拆小重试（模型明显用不了时不该一段一段拆到底烧额度； -#: 每段另有 MODEL_PART_MAX_CALLS 次封顶） -MODEL_GIVE_UP_AFTER = 2 +#: 止损（按比例）：这次起草最近 MODEL_HALT_WINDOW 次调用里有 MODEL_HALT_ON 次以上没写出来（额度用完、超时、空壳、出错）， +#: 之后的段不再拆小、每块只问一次，直到比例降下去。模型明显用不了时不该一段一段拆到底烧额度；以前按「连续几段」算， +#: 偶尔一次写出来就清零，会思考的模型在宽表上 90 分钟停不下来。每段另有 MODEL_PART_MAX_CALLS 次封顶 +MODEL_HALT_WINDOW = 10 +MODEL_HALT_ON = 7 +#: 同一张表连续这么多个原始段都一项没写出来，这张表剩下的段跳过不问（可以换一个起草模型后只重试这张表） +MODEL_TABLE_GIVE_UP_AFTER = 3 #: 网络错误、429、5xx 最多重试几次，每次重试前等多久（秒，依次取用）。超时不原样重试，当作这一段太大、拆小再问 MODEL_RETRIES = 2 MODEL_RETRY_DELAYS_S = (10.0, 30.0) @@ -1842,8 +1846,10 @@ async def write_draft(session: AsyncSession, source: Any, table: str, modeled: A 没用模型时为 None）有可用内容的再并上。同步起草和后台起草任务（每张表一结束就调）共用这一处。 - 并入已有目录按 merge_notes 的规则：人工确认、驳回过的项不动。内容没变不写、不升版本。 - - 模型这次没给的项（留空、没通过校验）不删以前起草的；列已经不在表结构里的，模型各段都写出来时（covered 含 llm） - 照旧删掉。模型只写出一部分的不算完整起草过（covered 不含 llm）。 + - 模型这次没给的项（留空、没通过校验）不删以前起草的；列已经不在表结构里的（业务主键、业务日期指着的列也算）， + 整张表起草完（各段都写出来的 ok、只写出一部分的 partial 都一样）就按完整覆盖的规则删掉：删的只是说已经不存在的 + 列的模型起草项，和这次写没写全无关。后台任务里宽表每写出一段就先并入一次（catalog_jobs.write_part，那一次不删 + 任何东西），这里是整张表的收尾。 - 模型留空的列（blank）按并入之后的目录再筛一遍：已经有中文名或含义的（以前起草的、库注释、起草进行中有人填的） 不算留空。 - 写入撞上别人刚改过（乐观锁）就重读重并，最多三次。 @@ -1859,8 +1865,8 @@ async def write_draft(session: AsyncSession, source: Any, table: str, modeled: A if modeled.items: result.model_items = modeled.items result.blank = list(getattr(modeled, "blank", None) or []) - model_draft = TableDraft(notes=modeled.notes, covered=frozenset({"llm"}) if modeled.status == "ok" - else frozenset()) + model_draft = TableDraft(notes=modeled.notes, covered=frozenset({"llm"}) + if modeled.status in ("ok", "partial") else frozenset()) for _attempt in range(3): entry = await read_entry(session, source.id, table) version = entry.version if entry else 0 @@ -2324,9 +2330,10 @@ __all__ = [ "CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES", "DraftModel", "DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL", "MODEL_BATCH_COLUMNS", "MODEL_BATCH_TABLES", "MODEL_CONCURRENCY", "MODEL_EMPTY_REASON", - "MODEL_FALLBACK_MAX_TOKENS", "MODEL_MAX_TOKENS", "MODEL_MIN_PART_COLUMNS", "MODEL_NOTICE_OFF", "MODEL_NOTICE_ON", - "MODEL_NOTICE_WINDOW", "MODEL_PART_MAX_CALLS", "MODEL_RETRIES", "MODEL_RETRY_DELAYS_S", "MODEL_SHRINK_AFTER", - "MODEL_SPLIT_DEPTH", "MODEL_SPLIT_MIN_COLUMNS", "MODEL_TIMEOUT_S", "MergeStats", "NEW_RELATION_PATH", + "MODEL_FALLBACK_MAX_TOKENS", "MODEL_HALT_ON", "MODEL_HALT_WINDOW", "MODEL_MAX_TOKENS", + "MODEL_MIN_PART_COLUMNS", "MODEL_NOTICE_OFF", "MODEL_NOTICE_ON", "MODEL_NOTICE_WINDOW", "MODEL_PART_MAX_CALLS", + "MODEL_RETRIES", "MODEL_RETRY_DELAYS_S", "MODEL_SHRINK_AFTER", "MODEL_SPLIT_DEPTH", "MODEL_SPLIT_MIN_COLUMNS", + "MODEL_TABLE_GIVE_UP_AFTER", "MODEL_TIMEOUT_S", "MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_NOTE_PREFIX", "PROFILE_VERIFY_COVERAGE", "WRITE_CONFLICT_ERROR", "default_tables", "missing_table_error", "pick_tables", "write_draft", diff --git a/backend/app/data/catalog_draft.py b/backend/app/data/catalog_draft.py index 4183bff1..49e059b1 100644 --- a/backend/app/data/catalog_draft.py +++ b/backend/app/data/catalog_draft.py @@ -42,19 +42,22 @@ 地方写成了数组、对象的值在校验（catalog._from_model）里丢掉，不算出错。 - 封顶：每个原始段（计划里的整表段、表级那段或一个列分段，连同按缩小后的上限分开的各块、拆出来的、补问的、重试的） 一共最多调用 MODEL_PART_MAX_CALLS 次，到了次数还没写完的记没写出来。 -- 止损：连续 MODEL_GIVE_UP_AFTER 段拆到底仍一项没写出来（模型明显用不了，例如思考总是吃满额度），这次起草后面的 - 段不再拆小重试、没写出来就直接记下，报错建议更换起草模型；有一段写出来了就恢复。超时另有一条：拆到每段 - MODEL_MIN_PART_COLUMNS 列的调用接连超时 MODEL_GIVE_UP_AFTER 次（超时和段大小无关，再拆小也是一次一个时限），后面 - 超时的不再拆小，超时拆出来、还没问的块也不再问；有一次调用写出来了就恢复。不然一直超时的模型首轮同时起草的几段 - 要各自拆满次数，一段 40 列串着问约 7 个时限（420 秒时约 49 分钟）。 +- 止损按比例：这次起草最近 MODEL_HALT_WINDOW 次调用里有 MODEL_HALT_ON 次以上没写出来（额度用完、超时、空壳、出错）， + 之后的段不再拆小、每块只问一次（不拆小、不补问漏掉的列、截断后不再补剩下的、空壳不再换纯文本原样再问；拆出来还没问 + 的块也不再问），报错建议更换起草模型，直到比例降下去。以前按「连续几段拆到底」「接连超时」算，偶尔一次写出来就清零： + 实测会思考的模型在宽表上单次 1 到 7 分钟、四分之一超时，90 分钟停不下来。几处只管通信的调整照旧：服务方嫌额度 + 太大退回小额度、结构化输出不支持或回了说明文字改走纯文本、网络错误退避重试；结构化输出只给空壳的，这次起草还没换 + 纯文本问过的照旧问一次（看看是不是结构化输出的问题）。 +- 同一张表连续 MODEL_TABLE_GIVE_UP_AFTER 个原始段都一项没写出来：这张表剩下还没开始的段跳过不问（宽表一段一段问下去 + 只是烧额度），原因写清跳过了几段、可以换一个起草模型后只重试这张表。中间有一段写出来了就不算「接连」。 - 提醒：最近 MODEL_NOTICE_WINDOW 次调用里有 MODEL_NOTICE_ON 次以上没写出来（额度用完、超时、空壳、出错），给一句 提醒（次数和原因，建议换起草模型、缩小起草范围或停止），经进度回调交给任务；之后少于 MODEL_NOTICE_OFF 次时清掉。 以前看不出调用在反复失败：一个任务 55 分钟只结束 1 张表，界面上只有段进度和用量。 **原因。** 没写出来的原因只写进这张表的 model_error：几段因为同一个原因没写出来时说「这几段」，分段起草的表（只写 出一部分的、一项都没写出来的都一样）每段写上列的范围（「第 5 段（列名 到 列名，共 33 列）」，列名照表结构原样）， -真拆小重试过的才补「已拆小重试仍没写出」。只有一段的表，这一段就是整张表：只写出一部分的写清是哪几列，一项都没写 -出来的只写原因。 +真拆小重试过的才补「已拆小重试仍没写出」，而且只跟在拆过的那几段后面说（说到的段只有一部分拆过时写「其中第 3 段」）。 +只有一段的表，这一段就是整张表：只写出一部分的写清是哪几列，一项都没写出来的只写原因。 **留空的列。** 请模型写了、模型也列出了，中文名和含义却都没给（或者给的都没通过校验）的列是模型拿不准，不补问， 记在 ModelOutcome.blank 里（表的状态不因此变）。目录里已经有中文名或含义的列不算（不论来源，驳回的不算有：以前模型 @@ -67,8 +70,10 @@ 的不会等前一个排着的段全部调完。一次起草里按表的先后排队：宽表的表级那段写完，它的列分段排到已经排着的段后面 （先来先服务），不插到窄表前面。 -**进度。** 调用方给 Hooks：开始调用、每次调用的结果（用量、没写出来的原因、当前的提醒和列数上限）、一段结束、一张表 -的模型部分全部结束。后台起草任务据此落库，每张表一结束就并入目录（data/catalog_jobs.py）。 +**进度。** 调用方给 Hooks：开始调用、每次调用的结果（用量、没写出来的原因、当前的提醒和列数上限）、一段结束、宽表的 +一段写出来了（交出这张表到这时拿到的全部内容，调用方马上并入目录：停止、服务重启时已经写出来的段不丢）、块进度变了、 +一张表的模型部分全部结束。后台起草任务据此落库（data/catalog_jobs.py）。块：这张表实际分成几块来问——一开始是段数， +一段被缩小的上限分开成几块、被拆成两半，块数就多几块；一块问完（写没写出来都算）算结束一块。 交给模型的只有表结构（列、类型、主键、外键、唯一约束、注释）和使用者补充的背景，不发任何数据值。写给模型的文字都在 catalog.py 名字带 _SYSTEM / _PROMPT 的常量里；这里的 REASON_* 是给人看的（拼进这张表的 model_error）。 @@ -111,21 +116,27 @@ SEG_ONE = "这一段" SEG_MANY = "这几段" REASON_EXHAUSTED = "模型的思考和输出用完了额度（{budget} 个 token），{seg}没写出来" REASON_SPLIT = "；已拆小重试仍没写出" +#: 说到的几段里只有一部分真拆小重试过：只说那几段 +REASON_SPLIT_PARTS = "；其中第 {parts} 段已拆小重试仍没写出" REASON_HOLLOW = "模型只给了空壳，没有写出可用的内容" REASON_TRUNCATED = "模型的输出被截断，{seg}没写完" REASON_FORMAT = "模型的回复不是约定的格式" REASON_MISSING = "模型的回复里没有这张表" REASON_UNANSWERED = "模型的回复里漏了这些列，补问一次仍没有给出" +#: 止损期间每块只问一次：漏了的列不补问 +REASON_OMITTED = "模型的回复里漏了这些列" REASON_TIMEOUT = "模型 {seconds} 秒内没有写完{seg}" REASON_CALL = "调用模型出错：{line}" REASON_RETRIED = "（已重试 {n} 次）" REASON_CRASH = "起草这张表时出错：{line}" #: 一段到了调用次数的上限（MODEL_PART_MAX_CALLS）还没写完 REASON_CAPPED = "每段最多调用 {n} 次模型，{seg}到了次数仍没写完，已停止重试" -#: 止损（连续几段拆到底仍一项没写出来，后面的段不再拆小重试）时补在原因后面 -REASON_HALTED = "；模型连续 {n} 段都没写出来，已停止拆小重试，建议更换起草模型" -#: 拆到每段 MODEL_MIN_PART_COLUMNS 列仍接连超时（超时和段大小无关，后面超时的不再拆小）时补在原因后面 -REASON_SLOW = "；拆到每段 {n} 列仍然超时，已停止拆小重试，建议更换起草模型" +#: 止损（最近几次调用多数没写出来，之后的段不再拆小、每块只问一次）时补在原因后面 +REASON_HALTED = "；最近 {window} 次调用有 {on} 次以上没写出来，已停止拆小重试，建议更换起草模型" +#: 同一张表接连几段都没写出来，剩下的段跳过不问 +REASON_ABANDONED = "这张表接连 {k} 段都没写出来，剩下的 {n} 段已跳过，可以换一个起草模型后只重试这张表" +#: 跳过不问的段在「没写出来」记录里的原因（不拼进原因，见 _Run._outcome） +_SKIPPED = "\0skipped" # ---- 提醒（最近几次调用多数没写出来），经进度回调交给任务（CatalogDraftJob.notice） NOTICE = ("最近 {calls} 次调用有 {failed} 次没写出来（{kinds}）。建议在{place}换一个模型、缩小起草范围，" @@ -154,8 +165,9 @@ class ModelOutcome: status：ok 各段都写出来了；partial 有的段写出来、有的没有（error 写清哪部分、为什么）；failed 一项都没写出来 （error 写原因）。notes 是写出来的那些（已逐项校验，llm / proposed）；items 是项数。parts_total 分几段起草 - （整表段是 1，宽表是 1 + 列分段数），parts_done 已经结束了几段。blank：模型拿不准、留空了的列（按表结构的顺序； - 表的状态不因此变）。""" + （整表段是 1，宽表是 1 + 列分段数），parts_done 已经结束了几段，parts_written 其中写出了内容的段数。blank：模型拿 + 不准、留空了的列（按表结构的顺序；表的状态不因此变）。chunks_total、chunks_done：这张表实际分成的块数和已结束的 + 块数（段被缩小的上限分开、拆成两半后会变多）。""" status: str notes: dict[str, Any] @@ -164,6 +176,18 @@ class ModelOutcome: parts_total: int parts_done: int blank: list[str] = field(default_factory=list) + parts_written: int = 0 + chunks_total: int = 0 + chunks_done: int = 0 + + +@dataclass(frozen=True) +class PartWrite: + """宽表写出了一段（进度回调 part_written 收到的）：notes 是这张表到这时拿到的全部模型起草（已校验、已核对，深拷贝）， + parts_written 是其中写出了内容的段数。调用方马上并入目录（这一次不删任何东西），整张表结束时另有 table_done 收尾。""" + + notes: dict[str, Any] + parts_written: int @dataclass(frozen=True) @@ -192,12 +216,16 @@ class Hooks: - call_started(表名列表)：一次调用拿到并发名额、马上要发出去了； - call_done(CallReport)：每次调用一回（条数就是调用次数），带着用量、没写出来的原因、当前的提醒和列数上限； - part_done(表名)：这张表的一段结束了（写没写出来都算）； + - part_written(表名, PartWrite)：宽表写出了一段、后面还有段没结束（最后一段由 table_done 收尾；只有一段的表不调）； + - chunks_changed(表名, 块数, 已结束的块数)：块进度变了； - table_done(表名, ModelOutcome)：这张表的模型部分全部结束。""" call_started: Callable[[list[str]], Awaitable[None]] | None = None call_done: Callable[[CallReport], Awaitable[None]] | None = None part_done: Callable[[str], Awaitable[None]] | None = None table_done: Callable[[str, ModelOutcome], Awaitable[None]] | None = None + part_written: Callable[[str, PartWrite], Awaitable[None]] | None = None + chunks_changed: Callable[[str, int, int], Awaitable[None]] | None = None def parts_for(columns: int) -> int: @@ -328,9 +356,9 @@ class _Reply: @dataclass(frozen=True) class _Why: - """一部分没写出来的原因。base 给人看的原因（可以带 SEG，拼成一句时换成「这一段」「这几段」）；split 真拆小重试过； - halted 本来还能拆小、因为止损才没拆时止损的说明（REASON_HALTED：前面连续几段都没写出来；REASON_SLOW：拆到每段 - MODEL_MIN_PART_COLUMNS 列仍接连超时），没有是空串。""" + """一部分没写出来的原因。base 给人看的原因（可以带 SEG，拼成一句时换成「这一段」「这几段」；跳过不问的段是 _SKIPPED， + 不拼进原因）；split 真拆小重试过；halted 本来还能拆小、补问或再问、因为止损才没问时止损的说明（REASON_HALTED：最近 + 几次调用多数没写出来），没有是空串。""" base: str split: bool = False @@ -341,18 +369,23 @@ def _join(failures: Iterable[tuple[_Why, int]]) -> str: """几处没写出来的原因拼成一句。failures 是 (原因, 属于这张表的第几段)。 同样的原因只说一次（最多说两种）；同一个原因落在几段上说「这几段」，同一段拆出来的几块仍是「这一段」；拆小重试过、 - 止损的说明各在最后补一次（只看说出来的那几种；两种止损各说一次）。不然宽表的表级那段拆不开、列分段拆过时，同一句 - 原因会说两遍。""" + 止损的说明各在最后补一次（只看说出来的那几种）。不然宽表的表级那段拆不开、列分段拆过时，同一句原因会说两遍。 + + 「已拆小重试仍没写出」只跟在真拆过的那几段后面说：说到的段都拆过（只有一段的表就是这一段）时不点名，只有一部分拆过 + 时写「其中第 3、4 段已拆小重试仍没写出」。以前一律补在最后，读起来像每一段都拆过。""" failures = list(failures) parts: dict[str, set[int]] = {} for why, part in failures: parts.setdefault(why.base, set()).add(part) shown = list(parts)[:2] - told = [w for w, _ in failures if w.base in shown] + told = [(w, part) for w, part in failures if w.base in shown] text = "；".join(base.replace(SEG, SEG_MANY if len(parts[base]) > 1 else SEG_ONE) for base in shown) - if any(w.split for w in told): - text += REASON_SPLIT - for note in dict.fromkeys(w.halted for w in told if w.halted): + split = sorted({part for w, part in told if w.split}) + if split: + everywhere = {part for _, part in told} + text += REASON_SPLIT if set(split) == everywhere else \ + REASON_SPLIT_PARTS.format(parts="、".join(str(part) for part in split)) + for note in dict.fromkeys(w.halted for w, _ in told if w.halted): text += note return text @@ -706,6 +739,16 @@ class _Table: failed: dict[str, _Why] = field(default_factory=dict) parts_done: int = 0 finished: bool = False + #: 写出了内容的段（第几段） + written_parts: set[int] = field(default_factory=set) + #: 接连几个原始段一项都没写出来（写出来就清零）；到了 MODEL_TABLE_GIVE_UP_AFTER 段，剩下的段跳过不问 + dead_parts: int = 0 + abandoned: bool = False + #: 跳过不问的段（第几段） + skipped_parts: list[int] = field(default_factory=list) + #: 块进度：实际分成几块、结束了几块 + chunks_total: int = 0 + chunks_done: int = 0 def whole(self, piece: _Piece) -> bool: return piece.table_level and len(piece.columns) == len(self.columns) @@ -753,6 +796,15 @@ class _Budget: left: int +@dataclass +class _Chunk: + """一块：一次拿来问的几张表（多表拼成的段每张表各是一块）。补问、截断后补剩下的、换纯文本再问都还是这一块；拆成两半 + （单张表按列拆，多一块；多表的段按表拆，各张表仍是一块）后，这一块里被拆的表交给新的块，split 记下它们。""" + + tables: tuple[str, ...] + split: set[str] = field(default_factory=set) + + def _pack(pieces: list[_Piece], cap: int) -> list[list[_Piece]]: """多表的段按顺序重新拼成每段不超过 cap 列（超过 cap 的表自成一段）。""" groups: list[list[_Piece]] = [] @@ -800,31 +852,28 @@ class _Run: self.background = self._background(getattr(source, "description", None), background) #: 这次起草同时在等名额或正在进行的调用最多几个（全进程的名额另算，见 model_gate） self.gate = asyncio.Semaphore(max(1, int(C.MODEL_CONCURRENCY))) - #: 连续几段拆到底仍一项没写出来（止损用，有一段写出来就清零） - self.dead_streak = 0 - #: 拆到每段 MODEL_MIN_PART_COLUMNS 列（或更小）的调用接连超时了几次（有一次调用写出来就清零）：超时和段大小 - #: 无关，再拆小也是一次一个时限，后面超时的不再拆小 - self.slow_streak = 0 #: 新开始的段每段最多几列：额度用完或超时（strikes）累计 MODEL_SHRINK_AFTER 次减半一次 self.part_columns = max(1, int(C.MODEL_BATCH_COLUMNS)) self.strikes = 0 #: 最近几次调用没写出来的原因种类（写出来了是 None），提醒看它 self.recent: collections.deque[str | None] = collections.deque(maxlen=max(1, int(C.MODEL_NOTICE_WINDOW))) self.notice: str | None = None + #: 最近 MODEL_HALT_WINDOW 次调用没写出来没有（止损按比例看它） + self.window: collections.deque[bool] = collections.deque(maxlen=max(1, int(C.MODEL_HALT_WINDOW))) + self.was_halted = False + #: 这次起草里结构化输出只给空壳时换纯文本问过没有（止损期间只问这一次） + self.text_probed = False - def _halt_note(self, kind: str | None = None) -> str: - """止损的说明（拼在原因后面），没止损是空串。 + def _halted(self) -> bool: + """止损：最近 MODEL_HALT_WINDOW 次调用里有 MODEL_HALT_ON 次以上没写出来。之后的段不再拆小、每块只问一次，直到 + 比例降下去（偶尔一次写出来不会清零）。""" + return sum(self.window) >= max(1, int(C.MODEL_HALT_ON)) - - 连续 MODEL_GIVE_UP_AFTER 段一项都没写出来：后面的段不再拆小重试（不论原因）； - - 超时的（kind 是 timeout）另看：拆到每段 MODEL_MIN_PART_COLUMNS 列的调用接连超时 MODEL_GIVE_UP_AFTER 次， - 超时就和段大小无关了，再拆小也是一次一个时限，后面超时的不再拆小，超时拆出来还没问的块也不再问。只管超时： - 额度用完、只给空壳照旧拆小（段小了思考和输出才放得下）。""" - give_up = max(1, int(C.MODEL_GIVE_UP_AFTER)) - if self.dead_streak >= give_up: - return REASON_HALTED.format(n=give_up) - if kind == "timeout" and self.slow_streak >= give_up: - return REASON_SLOW.format(n=self._floor()) - return "" + def _halt_note(self) -> str: + """止损的说明（拼在原因后面），没止损是空串。""" + if not self._halted(): + return "" + return REASON_HALTED.format(window=max(1, int(C.MODEL_HALT_WINDOW)), on=max(1, int(C.MODEL_HALT_ON))) @staticmethod def _floor() -> int: @@ -846,6 +895,8 @@ class _Run: head, *rest = self.tables[unit].parts # 表级那段先写：列分段拿它做背景 queue.append(_Work([head], [_Work([p]) for p in rest])) + for st in self.tables.values(): + st.chunks_total = len(st.parts) left = {name: len(st.parts) for name, st in self.tables.items()} busy = 0 changed = asyncio.Event() @@ -898,9 +949,16 @@ class _Run: """起草一段；它之后的段（宽表的列分段）排到队尾；一张表的段都结束了，这张表的模型部分就结束了。 这一段（原始段：计划里的整表段、表级那段或一个列分段）连同按缩小后的上限分开的各块、拆出来的、补问的、重试的， - 一共最多调用 MODEL_PART_MAX_CALLS 次（各块共用一个 _Budget）；止损按原始段数（_tally）。""" - await self._start(work.pieces, _Budget(max(1, int(C.MODEL_PART_MAX_CALLS)))) - self._tally(work.pieces) + 一共最多调用 MODEL_PART_MAX_CALLS 次（各块共用一个 _Budget）。接连几段没写出来的表（_tally），剩下的段跳过不问。 + 宽表写出了一段、后面还有段没结束，就把这张表到这时拿到的内容交给调用方并入目录（part_written）；最后一段结束由 + table_done 收尾。""" + asked = [p for p in work.pieces if not self.tables[p.table].abandoned] + for piece in work.pieces: + if self.tables[piece.table].abandoned: + await self._skip_part(piece) + if asked: + await self._start(asked, _Budget(max(1, int(C.MODEL_PART_MAX_CALLS)))) + self._tally(asked) for piece in work.pieces: await self._part_done(piece.table) queue.extend(work.then) @@ -908,6 +966,8 @@ class _Run: left[piece.table] -= 1 if not left[piece.table]: await self._table_done(piece.table) + elif piece.part in self.tables[piece.table].written_parts: + await self._part_written(piece.table) async def _start(self, pieces: list[_Piece], budget: _Budget) -> None: """开始起草一段。超过当前的列数上限（额度用完、超时多了会减半）的先按上限分开：多表的段按表重新拼，单张表的 @@ -922,6 +982,7 @@ class _Run: return piece = pieces[0] chunks = _even(piece.columns, -(-len(piece.columns) // cap)) + await self._more_chunks(piece.table, len(chunks) - 1) first = [_Piece(piece.table, chunks[0], piece.table_level, piece.part)] rest = [[_Piece(piece.table, cols, False, piece.part)] for cols in chunks[1:]] if piece.table_level: @@ -931,9 +992,10 @@ class _Run: await asyncio.gather(*(self._guarded(group, budget) for group in [first, *rest])) async def _guarded(self, pieces: list[_Piece], budget: _Budget) -> None: - """起草一段（或它按上限分开的一块）；意料之外的错误只记这一块没写出来，不影响别的段和别的表。""" + """起草一块（一段，或它按上限分开的一块）；意料之外的错误只记这一块没写出来，不影响别的段和别的表。""" + chunk = _Chunk(tuple(dict.fromkeys(p.table for p in pieces))) try: - await self._segment(pieces, 0, False, budget) + await self._segment(pieces, 0, False, budget, chunk=chunk) except asyncio.CancelledError: raise except Exception as e: # noqa: BLE001 @@ -941,49 +1003,78 @@ class _Run: logger.exception("起草数据目录时出错：%s", "、".join(p.table for p in pieces)) self._fail(pieces, REASON_CRASH.format(line=first_line(e) or describe_exception(e))) + await self._chunk_done(chunk) def _tally(self, pieces: list[_Piece]) -> None: - """一段（原始段，连同它分开的各块、拆小、补缺）结束：一项都没写出来的接着数，写出来了清零。""" - if any(self.tables[p.table].wrote(p) for p in pieces): - self.dead_streak = 0 + """一段（原始段，连同它分开的各块、拆小、补缺）结束：记下这一段写出内容没有；同一张表接连 + MODEL_TABLE_GIVE_UP_AFTER 段一项都没写出来，这张表剩下的段跳过不问（写出来了清零）。""" + give_up = max(1, int(C.MODEL_TABLE_GIVE_UP_AFTER)) + for piece in pieces: + st = self.tables[piece.table] + if st.wrote(piece): + st.written_parts.add(piece.part) + st.dead_parts = 0 + continue + st.dead_parts += 1 + if st.dead_parts >= give_up and not st.abandoned and len(st.parts) > 1: + st.abandoned = True + logger.warning("起草数据目录：表 %s 接连 %d 段都没写出来，剩下的段跳过", st.name, st.dead_parts) + + async def _skip_part(self, piece: _Piece) -> None: + """这张表接连几段都没写出来：这一段跳过不问（算一块结束）。""" + st = self.tables[piece.table] + st.skipped_parts.append(piece.part) + self._fail([piece], _Why(_SKIPPED)) + st.chunks_done += 1 + await self._hook(self.hooks.chunks_changed, st.name, st.chunks_total, st.chunks_done) + + # ---------------------------------------------------------------- 块进度 + + async def _more_chunks(self, table: str, n: int) -> None: + if n <= 0: return - self.dead_streak += 1 - if self.dead_streak == max(1, int(C.MODEL_GIVE_UP_AFTER)): - logger.warning("起草数据目录：模型连续 %d 段都没写出来，这次起草后面的段不再拆小重试", self.dead_streak) + st = self.tables[table] + st.chunks_total += n + await self._hook(self.hooks.chunks_changed, table, st.chunks_total, st.chunks_done) + + async def _chunk_done(self, chunk: _Chunk) -> None: + """一块问完了：这一块里没被拆给新块的表，各结束一块。""" + for table in chunk.tables: + if table in chunk.split: + continue + st = self.tables[table] + st.chunks_done += 1 + await self._hook(self.hooks.chunks_changed, table, st.chunks_total, st.chunks_done) # ---------------------------------------------------------------- 一段 async def _segment(self, pieces: list[_Piece], depth: int, followup: bool, budget: _Budget, *, - halved: bool = False, mode: str | None = None, last: bool = False, - optional: bool = False) -> None: + chunk: _Chunk, halved: bool = False, mode: str | None = None, last: bool = False, + optional: str | None = None) -> None: """问一段，拿到的并进各表；按失败原因拆开、补缺、记下没写出来的。 depth：已经拆了几层（截断后只补剩下的也算一层）。followup：这一段是在补缺（补过就不再补，见 _after_followup）。 - budget：这一段（原始段）还能调用几次。halved：这几块是真拆小（_split）出来的——原因里「已拆小重试」只在这时写。 - mode / last：换纯文本问的那一次——拆不开的空壳、结构化输出回了说明文字（再不行就记没写出来）。 - optional：超时拆出来的块，轮到它时已经止损就不问了（_ask）。""" + budget：这一段（原始段）还能调用几次。chunk：这是哪一块（块进度）。halved：这几块是真拆小（_split）出来的—— + 原因里「已拆小重试」只在这时写。mode / last：换纯文本问的那一次——拆不开的空壳、结构化输出回了说明文字（再不行 + 就记没写出来）。optional：拆出来的块（值是拆它的原因），轮到它时已经止损就不问了（_ask）。 + + 止损期间（_halted）每块只问一次：不拆小（_split）、不补问漏掉的列、截断后不再补剩下的。""" reply = await self._ask(pieces, budget, mode, optional=optional) if reply.capped: self._fail(pieces, self._capped(halved)) return if reply.skipped: - note = self._halt_note("timeout") - self._fail(pieces, lambda p: self._reason("timeout", split=True, piece=p, halted=note)) + note = self._halt_note() + self._fail(pieces, lambda p: self._reason(optional or "hollow", split=True, piece=p, halted=note)) return if reply.timeout: - # 先记下再报用量：报用量要写库、会让出事件循环，等名额的块醒来时要看到这一次 - if sum(len(p.columns) for p in pieces) <= self._floor(): - self.slow_streak += 1 - if self.slow_streak == max(1, int(C.MODEL_GIVE_UP_AFTER)): - logger.warning("起草数据目录：拆到每段 %d 列仍接连超时 %d 次，后面超时的不再拆小", - self._floor(), self.slow_streak) await self._report(reply, wrote=False) - await self._split(pieces, depth, "timeout", followup, budget, halved) + await self._split(pieces, depth, "timeout", followup, budget, halved, chunk) return if reply.error is not None: await self._report(reply, wrote=False) if reply.prose and not last: - await self._prose_to_text(pieces, depth, followup, budget, halved) + await self._prose_to_text(pieces, depth, followup, budget, halved, chunk) return self._fail(pieces, _Why(reply.error, split=halved)) return @@ -1008,28 +1099,36 @@ class _Run: return kind = "exhausted" if reply.exhausted else "truncated" if not anything: - await self._split(rest, depth, kind, followup, budget, halved) - elif depth < int(C.MODEL_SPLIT_DEPTH): + await self._split(rest, depth, kind, followup, budget, halved, chunk) + elif depth < int(C.MODEL_SPLIT_DEPTH) and not self._halted(): # 捡回了一部分：只补剩下的（不是拆小） - await self._segment(rest, depth + 1, followup, budget, halved=halved) + await self._segment(rest, depth + 1, followup, budget, chunk=chunk, halved=halved) else: - self._fail(rest, self._reason(kind, split=halved)) + note = self._halt_note() if depth < int(C.MODEL_SPLIT_DEPTH) else "" + self._fail(rest, self._reason(kind, split=halved, halted=note)) return if not anything and not followup: if last: halted = self._stopped(pieces, depth) self._fail(pieces, lambda p: self._reason("hollow", split=halved, piece=p, halted=halted)) return - await self._split(pieces, depth, "hollow", followup, budget, halved) + await self._split(pieces, depth, "hollow", followup, budget, halved, chunk) return if not rest: return if followup: - await self._after_followup(rest, depth, budget, halved, last=last) + await self._after_followup(rest, depth, budget, halved, chunk, last=last) return - await self._segment(rest, depth, True, budget, halved=halved) + if self._halted(): + # 止损期间每块只问一次：漏了的不补问 + note = self._halt_note() + self._fail(rest, lambda p: _Why(REASON_OMITTED, split=halved, halted=note) + if self.tables[p.table].has_any() else self._reason("hollow", split=halved, piece=p, + halted=note)) + return + await self._segment(rest, depth, True, budget, chunk=chunk, halved=halved) - async def _after_followup(self, rest: list[_Piece], depth: int, budget: _Budget, halved: bool, *, + async def _after_followup(self, rest: list[_Piece], depth: int, budget: _Budget, halved: bool, chunk: _Chunk, *, last: bool) -> None: """补问过仍然没给的部分（补过就不再补）。 @@ -1047,16 +1146,17 @@ class _Run: halted = self._stopped(empty, depth) self._fail(empty, lambda p: self._reason("hollow", split=halved, piece=p, halted=halted)) return - await self._split(empty, depth, "hollow", False, budget, halved) + await self._split(empty, depth, "hollow", False, budget, halved, chunk) async def _prose_to_text(self, pieces: list[_Piece], depth: int, followup: bool, budget: _Budget, - halved: bool) -> None: + halved: bool, chunk: _Chunk) -> None: """结构化输出回了一段说明文字：网关接受了结构化输出的参数却没照做。这次起草后面都走纯文本（另带格式说明）， - 这一段换纯文本问一次。纯文本额度用完了还能照常拆小（拆出来的半段也走纯文本，不再先问一遍结构化输出）。""" + 这一段换纯文本问一次（止损期间也问：这是通信方式的问题，不是模型写不出来）。纯文本额度用完了还能照常拆小 + （拆出来的半段也走纯文本，不再先问一遍结构化输出）。""" if self.mode != "text": self.mode = "text" logger.info("起草数据目录：结构化输出回了说明文字，后面改走纯文本") - await self._segment(pieces, depth, followup, budget, halved=halved, mode="text", last=True) + await self._segment(pieces, depth, followup, budget, chunk=chunk, halved=halved, mode="text", last=True) def _stopped(self, pieces: list[_Piece], depth: int) -> str: """这几段本来还能拆小、因为止损才没拆时，止损的说明；不是的话是空串。""" @@ -1065,36 +1165,57 @@ class _Run: return "" async def _split(self, pieces: list[_Piece], depth: int, kind: str, followup: bool, budget: _Budget, - halved: bool) -> None: + halved: bool, chunk: _Chunk) -> None: """拆成两半再问（额度用完、超时、只给空壳、截断后一项都没捡回来）。 - 同一张表按列拆、前一半带着表级的项：先写前一半，后一半拿它做背景。超时的不等：一次超时就是一个时限，40 → 20 - → 10 一层一层串着问要好几个时限，两半同时问；拆出来的块轮到它时已经止损的不再问（optional）。 - - 拆不开的空壳再问一次：结构化输出给的换纯文本，已经走纯文本的原样再问。止损时本来能拆的不拆：结构化输出给的 - 照旧换纯文本问一次（看看是不是结构化输出的问题），已经走纯文本的不再问。 + → 10 一层一层串着问要好几个时限，两半同时问。拆出来的两半是新的块（单张表多一块）；轮到它时已经止损的不再问 + （optional）。 + - 拆不开的空壳再问一次：结构化输出给的换纯文本，已经走纯文本的原样再问。 + - 止损期间（_halted）不拆小、每块只问一次：本来能拆的不拆；空壳不再原样再问，只有结构化输出给的、这次起草还没 + 换纯文本问过的照旧问一次（看看是不是结构化输出的问题）。 - 别的原因拆不开、止损了，就记没写出来。""" halves = _halves(pieces) if depth < int(C.MODEL_SPLIT_DEPTH) else None - halted = self._halt_note(kind) if halves is not None else "" + halted = self._halted() if halves is not None and not halted: first, second = halves - timeout = kind == "timeout" - if (not timeout and len(first) == len(second) == 1 and first[0].table == second[0].table + await self._split_chunk(chunk, pieces) + if (kind != "timeout" and len(first) == len(second) == 1 and first[0].table == second[0].table and first[0].table_level): - await self._segment(first, depth + 1, followup, budget, halved=True) - await self._segment(second, depth + 1, followup, budget, halved=True) + await self._half(first, depth + 1, followup, budget, kind) + await self._half(second, depth + 1, followup, budget, kind) return - results = await asyncio.gather( - self._segment(first, depth + 1, followup, budget, halved=True, optional=timeout), - self._segment(second, depth + 1, followup, budget, halved=True, optional=timeout), - return_exceptions=True) + results = await asyncio.gather(self._half(first, depth + 1, followup, budget, kind), + self._half(second, depth + 1, followup, budget, kind), + return_exceptions=True) for result in results: if isinstance(result, BaseException): raise result return - if kind == "hollow" and (not halted or self.mode == "structured"): - await self._segment(pieces, depth, followup, budget, halved=halved, mode="text", last=True) + if kind == "hollow" and (not halted or (self.mode == "structured" and not self.text_probed)): + self.text_probed = True + await self._segment(pieces, depth, followup, budget, chunk=chunk, halved=halved, mode="text", last=True) return - self._fail(pieces, lambda p: self._reason(kind, split=halved, piece=p, halted=halted)) + note = self._halt_note() if halted and (halves is not None or kind == "hollow") else "" + self._fail(pieces, lambda p: self._reason(kind, split=halved, piece=p, halted=note)) + + async def _half(self, pieces: list[_Piece], depth: int, followup: bool, budget: _Budget, kind: str) -> None: + """拆出来的一半：新的一块，问完算结束一块。""" + chunk = _Chunk(tuple(dict.fromkeys(p.table for p in pieces))) + try: + await self._segment(pieces, depth, followup, budget, chunk=chunk, halved=True, optional=kind) + except asyncio.CancelledError: + raise + except Exception: + await self._chunk_done(chunk) # 出错也算问完（_guarded 记没写出来），块进度对得上 + raise + await self._chunk_done(chunk) + + async def _split_chunk(self, chunk: _Chunk, pieces: list[_Piece]) -> None: + """这一块里的这几张表拆给了新的块：单张表按列拆成两半，多一块；多表的段按表拆，各张表仍是一块。""" + chunk.split.update(dict.fromkeys(p.table for p in pieces)) + if len(pieces) == 1: + await self._more_chunks(pieces[0].table, 1) def _remainder(self, pieces: list[_Piece], gains: list[_Gain], *, truncated: bool, followup: bool) -> list[_Piece]: @@ -1118,13 +1239,13 @@ class _Run: # ---------------------------------------------------------------- 调用 async def _ask(self, pieces: list[_Piece], budget: _Budget, mode: str | None = None, *, - optional: bool = False) -> _Reply: + optional: str | None = None) -> _Reply: """调一次模型（按原因重来：额度被拒、结构化输出不支持、网络错误……），交回读好的回复。 - 这一段的调用次数用完了交回 capped（不调用）。optional（超时拆出来的块）：拿到名额时已经止损，交回 skipped（不调用， - 次数退回）——等名额的时候前面的块接连超时，再问也是一个时限。超时不原样重试：交回 timeout，由调用方当作这一段 + 这一段的调用次数用完了交回 capped（不调用）。optional（拆出来的块）：拿到名额时已经止损，交回 skipped（不调用， + 次数退回）——等名额的时候前面的调用多数没写出来，不再拆小重试。超时不原样重试：交回 timeout，由调用方当作这一段 太大、拆小再问。重来之前的那次调用在这里就报给进度回调；交回的那次由调用方读完内容再报（写没写出来要看校验之后 - 的内容）。""" + 的内容）。服务方嫌额度太大、结构化输出不支持、网络错误的重来止损期间照旧（只管通信，不是模型写不出来）。""" mode = mode or self.mode retried = 0 while True: @@ -1135,7 +1256,7 @@ class _Run: try: # 先占这次起草自己的名额，再去排全进程的：一次起草排在全进程名额上的调用最多 MODEL_CONCURRENCY 个 async with self.gate, model_gate(): - if optional and self._halt_note("timeout"): + if optional and self._halted(): budget.left += 1 return _Reply(skipped=True) await self._hook(self.hooks.call_started, [p.table for p in pieces]) @@ -1228,10 +1349,17 @@ class _Run: async def _called(self, usage: tuple[int, int], failure: str | None, *, strike: bool = False) -> None: """记下一次调用：额度用完或超时（strike）累计 MODEL_SHRINK_AFTER 次，新开始的段列数上限减半；最近几次调用多数 - 没写出来时提醒，少了清掉。写出来了（failure 为 None）就清掉接连超时的计数。报给进度回调。""" + 没写出来时提醒，少了清掉；止损按最近 MODEL_HALT_WINDOW 次调用的比例（_halted）。报给进度回调。""" self.recent.append(failure) - if failure is None: - self.slow_streak = 0 + self.window.append(failure is not None) + halted = self._halted() + if halted != self.was_halted: + self.was_halted = halted + if halted: + logger.warning("起草数据目录：最近 %d 次调用有 %d 次没写出来，之后的段不再拆小、每块只问一次", + len(self.window), sum(self.window)) + else: + logger.info("起草数据目录：最近的调用多数写出来了，恢复拆小重试") if strike: self.strikes += 1 width = max(1, int(C.MODEL_BATCH_COLUMNS)) @@ -1344,21 +1472,33 @@ class _Run: failed_table = _TABLE in st.failed and not st.got_table total = len(st.parts) or 1 part_of = st.part_of() + progress = {"parts_written": len(st.written_parts), "chunks_total": st.chunks_total, + "chunks_done": st.chunks_done} + abandoned = self._abandoned_text(st) if not items: - failures = [(why, part_of.get(key, 1)) for key, why in st.failed.items()] + failures = [(why, part_of.get(key, 1)) for key, why in st.failed.items() if why.base != _SKIPPED] reason = _join(failures) if failures else (C.MODEL_EMPTY_REASON if st.named else REASON_MISSING) - if failures and len(st.parts) > 1: + if abandoned: + reason = f"{reason}；{abandoned}" + if st.failed and len(st.parts) > 1: # 宽表一项都没写出来：同样逐段写上列的范围 said = self._parts_said(st, failed_cols, failed_table) every = "都" if len(said) == len(st.parts) else "" reason = f"这张表 {len(st.columns)} 列，分 {len(st.parts)} 段起草，{'、'.join(said)}{every}没写出来：{reason}" - return ModelOutcome("failed", {}, 0, reason, total, st.parts_done) + return ModelOutcome("failed", {}, 0, reason, total, st.parts_done, **progress) notes = copy.deepcopy(st.notes) blank = self._blank(st, set(failed_cols)) if failed_cols or failed_table: - return ModelOutcome("partial", notes, items, self._partial_text(st, failed_cols, failed_table), total, - st.parts_done, blank) - return ModelOutcome("ok", notes, items, None, total, st.parts_done, blank) + return ModelOutcome("partial", notes, items, self._partial_text(st, failed_cols, failed_table, abandoned), + total, st.parts_done, blank, **progress) + return ModelOutcome("ok", notes, items, None, total, st.parts_done, blank, **progress) + + @staticmethod + def _abandoned_text(st: _Table) -> str: + """这张表接连几段没写出来、剩下的段跳过了：说清跳过了几段、怎么办。没跳过是空串。""" + if not st.skipped_parts: + return "" + return REASON_ABANDONED.format(k=max(1, int(C.MODEL_TABLE_GIVE_UP_AFTER)), n=len(st.skipped_parts)) def _blank(self, st: _Table, failed: set[str]) -> list[str]: """模型拿不准、留空了的列：请它写了（不在没写出来的里面），中文名和含义却都没给（或者给的都没通过校验）。 @@ -1395,11 +1535,14 @@ class _Run: return said + [f"第 {k} 段（{_columns_text(st.columns, cols, count=True)}）" for k, cols in sorted(bad.items())] @classmethod - def _partial_text(cls, st: _Table, failed_cols: list[str], failed_table: bool) -> str: - """只写出一部分时说清哪部分没写出来（第几段、哪几列）、为什么、怎么办。""" + def _partial_text(cls, st: _Table, failed_cols: list[str], failed_table: bool, abandoned: str = "") -> str: + """只写出一部分时说清哪部分没写出来（第几段、哪几列）、为什么、怎么办。剩下的段跳过了的，跳过的说明里已经说了 + 怎么办（换一个起草模型后只重试这张表），不再另说「可以只重试这张表」。""" part_of = st.part_of() keys = ([_TABLE] if failed_table else []) + failed_cols - reason = _join((st.failed[k], part_of.get(k, 1)) for k in keys) + reason = _join((st.failed[k], part_of.get(k, 1)) for k in keys if st.failed[k].base != _SKIPPED) + if abandoned: + reason = f"{reason}；{abandoned}" if reason else abandoned n = len(st.columns) if len(st.parts) > 1: said = cls._parts_said(st, failed_cols, failed_table) @@ -1411,7 +1554,7 @@ class _Run: head = f"这张表 {n} 列，有 {len(failed_cols)} 列（{_columns_text(st.columns, failed_cols, count=False)}）没写出来" else: head = f"这张表 {n} 列，表级的项没写出来" - return f"{head}：{reason}。可以只重试这张表" + return f"{head}：{reason}" if abandoned else f"{head}：{reason}。可以只重试这张表" async def _part_done(self, name: str) -> None: self.tables[name].parts_done += 1 @@ -1424,6 +1567,11 @@ class _Run: st.finished = True await self._hook(self.hooks.table_done, name, self._outcome(st)) + async def _part_written(self, name: str) -> None: + """宽表写出了一段、后面还有段没结束：把这张表到这时拿到的全部内容交给调用方并入目录（停止、服务重启时不丢）。""" + st = self.tables[name] + await self._hook(self.hooks.part_written, name, PartWrite(copy.deepcopy(st.notes), len(st.written_parts))) + @staticmethod async def _hook(fn: Callable[..., Awaitable[None]] | None, *args: Any) -> None: if fn is None: @@ -1534,7 +1682,8 @@ class _Run: return C._notes_lines(confirmed, meta=st.meta, system_notes=False) -__all__ = ["BUDGET_WORDS", "Background", "CallReport", "FAILURE_LABEL", "Hooks", "ModelOutcome", "NOTICE", "REASON_CALL", - "REASON_CAPPED", "REASON_EXHAUSTED", "REASON_FORMAT", "REASON_HALTED", "REASON_HOLLOW", "REASON_MISSING", - "REASON_RETRIED", "REASON_SLOW", "REASON_SPLIT", "REASON_TIMEOUT", "REASON_TRUNCATED", "REASON_UNANSWERED", - "SEG", "classify_failure", "model_gate", "parts_for", "run_model_draft"] +__all__ = ["BUDGET_WORDS", "Background", "CallReport", "FAILURE_LABEL", "Hooks", "ModelOutcome", "NOTICE", "PartWrite", + "REASON_ABANDONED", "REASON_CALL", "REASON_CAPPED", "REASON_EXHAUSTED", "REASON_FORMAT", "REASON_HALTED", + "REASON_HOLLOW", "REASON_MISSING", "REASON_OMITTED", "REASON_RETRIED", "REASON_SPLIT", "REASON_SPLIT_PARTS", + "REASON_TIMEOUT", "REASON_TRUNCATED", "REASON_UNANSWERED", "SEG", "classify_failure", "model_gate", + "parts_for", "run_model_draft"] diff --git a/backend/app/data/catalog_jobs.py b/backend/app/data/catalog_jobs.py index d708ed50..555c5256 100644 --- a/backend/app/data/catalog_jobs.py +++ b/backend/app/data/catalog_jobs.py @@ -6,11 +6,15 @@ - **建任务**（create_job）：同一个数据源同时只有一个排队或进行中的任务（DraftBusy；部分唯一索引兜底）。不指定表时 按使用次数取前 20 张；表结构里没有的表照样建进任务、状态 skipped。起草要求（数据源级）和本次说明随任务记下快照。 - **在后台起草**（manager.start）：进程内的 asyncio 任务，引擎是 data/catalog_draft.py（分段、按失败原因处理）。 - 每张表一结束就并入目录（catalog.write_draft，规则和同步起草一样：人工确认、驳回过的不动，模型这次没给的项不删 - 以前起草的，乐观锁冲突重读重并）；逐表的状态、到第几段、计数、模型留空的列，任务的用量（含超时、没写出来的调用 - 次数）、当前的列数上限、提醒随时落库，接口读的是库里的状态。 + 宽表每写出一段就并入目录（write_part：这一次合并不删任何东西），整张表结束（不管写全没写全）再按完整覆盖的规则 + 收尾一次（catalog.write_draft，规则和同步起草一样：人工确认、驳回过的不动，模型这次没给的项不删以前起草的，只删 + 已经不在表结构里的列的模型项，乐观锁冲突重读重并）；新增、更新、删除的计数是各次写入的合计。逐表的状态、到第几段、 + 写进目录的段数、块进度、计数、模型留空的列，任务的用量（含超时、没写出来的调用次数）、当前的列数上限、提醒随时落库， + 接口读的是库里的状态。 - **停止**（cancel）：排队的直接 cancelled；进行中的停掉正在进行的模型调用，没开始和没写完的表记 cancelled，已经写进 - 目录的不回滚，任务最后是 cancelled。 + 目录的（含宽表已经写出来的段）不回滚，任务最后是 cancelled。写过段的表 model_error 说清写进了几段、剩几段没起草。 +- **和同步起草互斥**（sync_draft）：同步起草（POST /catalog/draft）进行期间，同一个源建任务、重试、导入（ensure_idle） + 都是 DraftBusy；有排队或进行中的任务时同步起草也是。进程内登记（单进程部署）。 - **重试**（retry_tables + create_job）：新任务，只含原任务里 failed、partial、cancelled 的表；用不用模型、本次说明沿用， 起草要求取数据源当前的。 - **服务重启**：启动时（startup）把排队、进行中的任务标成 interrupted，没结束的表记 cancelled、写明「服务重启，起草 @@ -26,11 +30,12 @@ from __future__ import annotations import asyncio +import contextlib import logging -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from typing import Any -from sqlalchemy import func, select, update +from sqlalchemy import delete, func, or_, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession @@ -62,6 +67,10 @@ BLANK_SAMPLE = 5 STOP_GRACE_S = 10.0 INTERRUPTED_ERROR = "服务重启，起草中断" +#: 停止、中断时写过段的表（宽表每写出一段就并入目录）：写进了几段、剩几段没起草 +STOPPED_WRITTEN = "已停止：已写进目录 {n} 段，剩下 {m} 段没起草" +INTERRUPTED_WRITTEN = "已中断：已写进目录 {n} 段，剩下 {m} 段没起草" +_WRITTEN = {"cancelled": STOPPED_WRITTEN, "interrupted": INTERRUPTED_WRITTEN} UNFINISHED_ERROR = "起草结束时这张表还没有写完，请重试这张表" WRITE_ERROR = "写入数据目录时出错：{line}" JOB_ERROR = "起草任务出错：{line}" @@ -162,6 +171,11 @@ def task_dict(task: CatalogDraftTask) -> dict[str, Any]: "columns": task.column_count, "parts_total": task.parts_total, "parts_done": task.parts_done, + # 宽表每写出一段就并入目录：已经写进目录的段数（停止、服务重启时这些段留着） + "parts_written": task.parts_written or 0, + # 块进度：这张表实际分成的块数和已结束的块数（段被缩小、拆成两半后会变多）；不用模型时为 null + "chunks_total": task.chunks_total, + "chunks_done": task.chunks_done, "added": task.added, "updated": task.updated, "removed": task.removed, @@ -237,21 +251,39 @@ async def job_brief(session: AsyncSession, job: CatalogDraftJob | None) -> dict[ async def last_drafts(session: AsyncSession, source_id: str) -> dict[str, dict[str, Any]]: - """每张表最近一次起草任务里的那一项：{表名: {status, job_id, finished_at, error, model_error, blank_columns}}。 - 任务还在进行，就是进行中的状态。只看最近 LAST_DRAFT_JOBS 个任务；每张表取哪一项在库里挑好（清单页起草时会反复取）。""" + """每张表最近一次起草任务里的那一项：{表名: {status, job_id, finished_at, error, model_error, blank_columns, + parts_written, blank_columns_from}}。任务还在进行，就是进行中的状态。只看最近 LAST_DRAFT_JOBS 个任务；每张表取哪一项 + 在库里挑好（清单页起草时会反复取）。 + + blank_columns_from：最近一次结束于 ok / partial 的用模型起草记下的留空列数（数字），没有是 null。之后的起草停止了、 + 中断了、而且没有写进新内容（parts_written 为 0）时照样带着，界面据此继续提示留空的列；之后有起草写进了新内容（停止 + 前已经写了几段、或者正在写）就是 null：以前的留空列数可能已经不对了。不用模型的起草不改模型留空的列，不算。""" recent = (select(CatalogDraftJob.id).where(CatalogDraftJob.source_id == source_id) .order_by(CatalogDraftJob.created_at.desc()).limit(LAST_DRAFT_JOBS)).scalar_subquery() - rank = func.row_number().over(partition_by=CatalogDraftTask.table_name, - order_by=(CatalogDraftJob.created_at.desc(), CatalogDraftTask.seq)).label("rank") + + def ranked() -> Any: + return func.row_number().over(partition_by=CatalogDraftTask.table_name, + order_by=(CatalogDraftJob.created_at.desc(), CatalogDraftTask.seq)).label("rank") + inner = (select(CatalogDraftTask.table_name, CatalogDraftTask.status, CatalogDraftTask.job_id, CatalogDraftTask.finished_at, CatalogDraftTask.error, CatalogDraftTask.model_error, - CatalogDraftTask.blank_columns, rank) + CatalogDraftTask.blank_columns, CatalogDraftTask.parts_written, ranked()) .join(CatalogDraftJob, CatalogDraftJob.id == CatalogDraftTask.job_id) .where(CatalogDraftJob.id.in_(recent))).subquery() rows = (await session.execute(select(inner).where(inner.c.rank == 1))).mappings() + # 每张表最近一次「写进了模型起草的内容」的用模型起草：结束于 ok / partial 的，或者写过段的（停止、中断、进行中） + wrote = (select(CatalogDraftTask.table_name, CatalogDraftTask.status, CatalogDraftTask.blank_columns, ranked()) + .join(CatalogDraftJob, CatalogDraftJob.id == CatalogDraftTask.job_id) + .where(CatalogDraftJob.id.in_(recent), CatalogDraftTask.parts_total.is_not(None), + or_(CatalogDraftTask.status.in_(("ok", "partial")), + func.coalesce(CatalogDraftTask.parts_written, 0) > 0))).subquery() + blank_from = {row["table_name"]: (row["blank_columns"] or 0) if row["status"] in ("ok", "partial") else None + for row in (await session.execute(select(wrote).where(wrote.c.rank == 1))).mappings()} return {row["table_name"]: {"status": row["status"], "job_id": row["job_id"], "finished_at": _iso(row["finished_at"]), "error": row["error"], - "model_error": row["model_error"], "blank_columns": row["blank_columns"] or 0} + "model_error": row["model_error"], "blank_columns": row["blank_columns"] or 0, + "parts_written": row["parts_written"] or 0, + "blank_columns_from": blank_from.get(row["table_name"])} for row in rows} @@ -264,12 +296,43 @@ def busy_message(row: Any) -> str: return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录，请等这次起草结束（或先停止它）再开始新的起草" +def sync_busy_message(row: Any) -> str: + return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录（同步起草），请等这次起草结束再开始新的起草" + + +#: 正在同步起草（POST /catalog/draft）的数据源。进程内登记：服务端是单进程部署，多进程部署时要换成库里的锁 +_SYNC: set[str] = set() + + +def sync_drafting(source_id: str) -> bool: + """这个源正在同步起草。""" + return source_id in _SYNC + + async def ensure_idle(session: AsyncSession, row: Any) -> None: - """这个源没有排队或进行中的任务；有就抛 DraftBusy。""" + """这个源没有在起草：没有排队或进行中的任务，也没有在同步起草。有就抛 DraftBusy。建任务、重试、导入目录之前都调它。""" + if row.id in _SYNC: + raise DraftBusy(sync_busy_message(row)) if await active_job(session, row.id) is not None: raise DraftBusy(busy_message(row)) +@contextlib.asynccontextmanager +async def sync_draft(session: AsyncSession, row: Any) -> AsyncIterator[None]: + """同步起草期间登记这个源：同一个源建任务、重试、导入、再开一次同步起草都抛 DraftBusy；有排队或进行中的任务时 + 这里就抛 DraftBusy。先登记、再查库：查库要让出事件循环，先登记才不会和同时建的任务互相看不见（建任务那边写库之后 + 也再看一次，见 create_job）。""" + if row.id in _SYNC: + raise DraftBusy(sync_busy_message(row)) + _SYNC.add(row.id) + try: + if await active_job(session, row.id) is not None: + raise DraftBusy(busy_message(row)) + yield + finally: + _SYNC.discard(row.id) + + async def check_model(session: AsyncSession) -> None: """起草模型能不能用：不能用抛 catalog.CatalogModelUnavailable（只建模型对象，不发请求）。""" await catalog.resolve_draft_model(session) @@ -299,9 +362,10 @@ async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: li continue seen.add(key) columns = len((all_tables.get(key) or {}).get("columns") or []) + parts = catalog_draft.parts_for(columns) if use_model else None tasks.append(CatalogDraftTask(job_id=job.id, seq=len(tasks), table_name=key, status="pending", - column_count=columns, - parts_total=catalog_draft.parts_for(columns) if use_model else None)) + column_count=columns, parts_total=parts, parts_written=0, + chunks_total=parts, chunks_done=0 if use_model else None)) session.add(job) try: await session.flush() @@ -311,6 +375,12 @@ async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: li # 部分唯一索引：另一个请求同时给这个源建了任务 await session.rollback() raise DraftBusy(busy_message(row)) from e + if row.id in _SYNC: + # 写库的时候同一个源开始了同步起草（它查库时还没看到这个任务）：这个任务不要了 + await session.execute(delete(CatalogDraftTask).where(CatalogDraftTask.job_id == job.id)) + await session.execute(delete(CatalogDraftJob).where(CatalogDraftJob.id == job.id)) + await session.commit() + raise DraftBusy(sync_busy_message(row)) return job, tasks @@ -347,8 +417,21 @@ async def cancel(session: AsyncSession, job: CatalogDraftJob) -> None: async def _close(session: AsyncSession, job_id: str, job_status: str, job_error: str | None, task_status: str, task_error: str | None) -> None: - """收尾：还没结束的表记 task_status，任务（还没结束的话）记 job_status。""" + """收尾：还没结束的表记 task_status，任务（还没结束的话）记 job_status。停止、中断时写过段的表 model_error 说清写进了 + 几段、剩几段没起草（已经写进去的段留在目录里）。""" now = utcnow() + written = _WRITTEN.get(job_status) + if written is not None: + rows = (await session.execute( + select(CatalogDraftTask.id, CatalogDraftTask.parts_written, CatalogDraftTask.parts_total) + .where(CatalogDraftTask.job_id == job_id, CatalogDraftTask.status.in_(_OPEN), + func.coalesce(CatalogDraftTask.parts_written, 0) > 0))).all() + for task_id, n, total in rows: + await session.execute( + update(CatalogDraftTask).where(CatalogDraftTask.id == task_id, CatalogDraftTask.status.in_(_OPEN)) + .values(status=task_status, error=task_error, finished_at=now, + model_error=written.format(n=n, m=max(0, int(total or n) - int(n)))) + .execution_options(synchronize_session=False)) await session.execute( update(CatalogDraftTask).where(CatalogDraftTask.job_id == job_id, CatalogDraftTask.status.in_(_OPEN)) .values(status=task_status, error=task_error, finished_at=now).execution_options(synchronize_session=False)) @@ -478,7 +561,8 @@ class _JobRun: if use_model and names: await self._write(self._set_model, model_id) hooks = catalog_draft.Hooks(call_started=self._call_started, call_done=self._call_done, - part_done=self._part_done, table_done=self._table_done) + part_done=self._part_done, table_done=self._table_done, + part_written=self._part_written, chunks_changed=self._chunks_changed) await catalog_draft.run_model_draft(model, source, names, existing=existing, background=background, hooks=hooks) else: @@ -505,6 +589,12 @@ class _JobRun: async def _table_done(self, table: str, outcome: catalog_draft.ModelOutcome | None) -> None: await self._write(self._write_table, table, outcome) + async def _part_written(self, table: str, written: catalog_draft.PartWrite) -> None: + await self._write(self._write_part, table, written) + + async def _chunks_changed(self, table: str, total: int, done: int) -> None: + await self._write(self._set_chunks, table, total, done) + # ---------------------------------------------------------------- 库操作 def _task(self, *where: Any) -> Any: @@ -554,9 +644,35 @@ class _JobRun: .values(parts_done=CatalogDraftTask.parts_done + 1)) await session.commit() + async def _set_chunks(self, session: AsyncSession, table: str, total: int, done: int) -> None: + await session.execute(self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN)) + .values(chunks_total=int(total), chunks_done=int(done))) + await session.commit() + + async def _write_part(self, session: AsyncSession, table: str, written: catalog_draft.PartWrite) -> None: + """宽表写出了一段：马上并入目录（这一次不删任何东西），记下写进去的段数、累加计数。写不进去只记日志：整张表结束 + 时收尾那次还会把它带上。""" + try: + found = await write_part(session, self.source, table, written.notes, at=self.at, actor=self.actor) + except Exception: # noqa: BLE001 - 这一段写不进去不影响起草 + logger.exception("起草任务 %s 把表 %s 写出来的一段并入数据目录时出错", self.job_id, table) + await session.rollback() + return + if found is None: + logger.warning("起草任务 %s：表 %s 的数据目录正被修改，这一段等整张表写完时再并入", self.job_id, table) + return + stats, version = found + await session.execute( + self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN)) + .values(parts_written=int(written.parts_written), version=version, + added=CatalogDraftTask.added + stats.added, updated=CatalogDraftTask.updated + stats.updated, + removed=CatalogDraftTask.removed + stats.removed)) + await session.commit() + async def _write_table(self, session: AsyncSession, table: str, outcome: catalog_draft.ModelOutcome | None) -> None: - """一张表结束：并入目录、写入，记下这张表的结果。""" + """一张表结束：并入目录、写入（宽表写过的段已经并入过，这里是整张表的收尾），记下这张表的结果。计数加在逐段写入 + 的计数上。""" try: result = await catalog.write_draft(session, self.source, table, outcome, at=self.at, actor=self.actor) except Exception as e: # noqa: BLE001 - 这一张写不进去不影响别的表 @@ -568,13 +684,17 @@ class _JobRun: error=WRITE_ERROR.format(line=first_line(e) or describe_exception(e)), model_error=outcome.error if outcome is not None else None) values: dict[str, Any] = { - "status": result.status, "added": result.added, "updated": result.updated, "removed": result.removed, - "version": result.version, "model_items": result.model_items, "error": result.error, + "status": result.status, "added": CatalogDraftTask.added + result.added, + "updated": CatalogDraftTask.updated + result.updated, "removed": CatalogDraftTask.removed + result.removed, + "model_items": result.model_items, "error": result.error, "model_error": result.model_error, "finished_at": utcnow(), "blank_columns": len(result.blank), "blank_sample": result.blank[:BLANK_SAMPLE] or None, } + if result.version: + values["version"] = result.version if outcome is not None: - values["parts_done"] = outcome.parts_done + values.update(parts_done=outcome.parts_done, parts_written=outcome.parts_written, + chunks_total=outcome.chunks_total, chunks_done=outcome.chunks_done) await session.execute(self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN)) .values(**values)) await session.commit() @@ -594,6 +714,30 @@ class _JobRun: await session.commit() +async def write_part(session: AsyncSession, source: Any, table: str, notes: dict[str, Any], *, at: str, + actor: str | None) -> tuple[catalog.MergeStats, int] | None: + """宽表写出来的一段并入目录：(计数, 写入后的版本)；撞上别人刚改过、重读重并三次仍冲突返回 None。 + + 数据库注释、外键、命名推断和这张表到这时拿到的模型起草，按 merge_notes 的规则并入（人工确认、驳回过的不动）， + **这一次不删任何东西**（covered 为空）：删掉表结构里已经没有的、这一轮没再得出的，是整张表写完时 catalog.write_draft + 收尾那一次的事。内容没变不写、不升版本。""" + structure = catalog.draft_structure(source, table) + for _attempt in range(3): + entry = await catalog.read_entry(session, source.id, table) + version = entry.version if entry else 0 + merged, stats = catalog.merge_notes(entry.notes if entry else {}, structure.notes, at=at) + merged, more = catalog.merge_notes(merged, notes, at=at) + stats += more + if entry is None and not merged: + return stats, 0 + try: + written = await catalog.write_entry(session, source.id, table, merged, if_version=version, actor=actor) + except catalog.CatalogConflict: + continue + return stats, written.version + return None + + class DraftJobs: """进程内的起草任务（照 app/scheduler 的写法：一个 asyncio 任务一件事，停的时候给宽限）。""" @@ -660,8 +804,8 @@ manager = DraftJobs() __all__ = ["ACTIVE", "BLANK_SAMPLE", "DraftBusy", "DraftFinished", "DraftJobs", "FINISHED", "GUIDANCE_MAX", - "INTERRUPTED_ERROR", "MAX_TABLES", "NOTE_MAX", "NothingToRetry", "RETRYABLE", "TASK_DONE", "TASK_STATUSES", - "TextTooLong", + "INTERRUPTED_ERROR", "INTERRUPTED_WRITTEN", "MAX_TABLES", "NOTE_MAX", "NothingToRetry", "RETRYABLE", + "STOPPED_WRITTEN", "TASK_DONE", "TASK_STATUSES", "TextTooLong", "active_job", "cancel", "check_model", "clean_text", "create_job", "ensure_idle", "guidance_of", "job_brief", "job_dict", "job_out", "last_drafts", "latest_job", "load", "manager", "retry_tables", "save_guidance", - "startup", "tasks_of"] + "startup", "sync_busy_message", "sync_draft", "sync_drafting", "tasks_of", "write_part"] diff --git a/backend/app/db/base.py b/backend/app/db/base.py index f0da4ac6..6cd9298b 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -212,6 +212,10 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [ ("catalog_draft_jobs", "notice", "TEXT"), ("catalog_draft_tasks", "blank_columns", "INTEGER DEFAULT 0"), ("catalog_draft_tasks", "blank_sample", "JSON"), + # 数据目录闭环第一步：宽表逐段落盘的段数、块进度。存量任务回填 0 和 NULL：升级前没记这些 + ("catalog_draft_tasks", "parts_written", "INTEGER DEFAULT 0"), + ("catalog_draft_tasks", "chunks_total", "INTEGER"), + ("catalog_draft_tasks", "chunks_done", "INTEGER"), ] #: 迁移补上的列要建的索引：(索引名, 表, 列)。create_all 不碰已有的表，表上新列的索引也就不会建。 diff --git a/backend/app/db/models.py b/backend/app/db/models.py index a5438d84..0671e33b 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -410,9 +410,14 @@ class CatalogDraftTask(Base): table_name: Mapped[str] = mapped_column(String(255)) status: Mapped[str] = mapped_column(String(20), default="pending") column_count: Mapped[int] = mapped_column(Integer, default=0) - # 模型分几段起草（不用模型时为 NULL）、已经结束了几段 + # 模型分几段起草（不用模型时为 NULL）、已经结束了几段、其中写出了内容并进了目录的段数（宽表每写出一段就并入目录， + # 停止、服务重启时已经写进去的留着） parts_total: Mapped[int | None] = mapped_column(Integer, default=None) parts_done: Mapped[int] = mapped_column(Integer, default=0) + parts_written: Mapped[int] = mapped_column(Integer, default=0) + # 块进度：这张表实际分成几块来问、已经结束了几块（段被缩小、拆成两半后会变多）；不用模型时为 NULL + chunks_total: Mapped[int | None] = mapped_column(Integer, default=None) + chunks_done: Mapped[int | None] = mapped_column(Integer, default=None) added: Mapped[int] = mapped_column(Integer, default=0) updated: Mapped[int] = mapped_column(Integer, default=0) removed: Mapped[int] = mapped_column(Integer, default=0) diff --git a/backend/tests/test_catalog_draft_engine.py b/backend/tests/test_catalog_draft_engine.py index 4a1a94aa..aadb8ced 100644 --- a/backend/tests/test_catalog_draft_engine.py +++ b/backend/tests/test_catalog_draft_engine.py @@ -254,6 +254,7 @@ async def test_hollow_shell_splits_the_pack_by_table(): async def test_splitting_stops_at_three_levels_and_five_columns(monkeypatch): monkeypatch.setattr(catalog, "MODEL_PART_MAX_CALLS", 99) + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) # 不让按比例止损先停下来 tables = {"wide40": _table(40)} model = Scripted(lambda call: EXHAUSTED) got = await D.run_model_draft(model, _source(tables), ["wide40"]) @@ -272,7 +273,7 @@ async def test_splitting_stops_at_three_levels_and_five_columns(monkeypatch): async def test_one_part_makes_at_most_eight_calls(monkeypatch): """一段（连同它拆出来的、补问的、重试的）最多调用 8 次：40 列一直用完额度，拆到第 8 次就停，剩下没问的记没写出来、 原因写清。以前一段 40 列拆到底要 15 次。""" - monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99) + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) assert catalog.MODEL_PART_MAX_CALLS == 8 model = Scripted(lambda call: EXHAUSTED) got = await D.run_model_draft(model, _source({"t40": _table(40)}), ["t40"]) @@ -755,6 +756,7 @@ async def test_a_wide_table_with_one_failed_part_is_partial_and_says_which(): async def test_reasons_name_each_part_and_say_these_parts(monkeypatch): """几段因为同一个原因没写出来：原因里写「这几段」；每段写上列的范围（列名照表结构原样）；拆过的才补「已拆小重试」。""" monkeypatch.setattr(catalog, "MODEL_SHRINK_AFTER", 99) + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) tables = {"wide95": _table(95)} ok = answering(tables) @@ -827,33 +829,38 @@ async def test_tables_are_packed_up_to_40_columns_and_4_tables(): async def test_the_same_reason_is_said_once_even_when_some_parts_were_split(monkeypatch): - """宽表的表级那段拆不开、列分段拆过：原因只说一次，拆过的在最后补一句。""" - monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99) + """宽表的表级那段拆不开、列分段拆过：原因只说一次，「已拆小重试」只跟在真拆过的那几段后面说。""" + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) monkeypatch.setattr(catalog, "MODEL_PART_MAX_CALLS", 99) model = Scripted(_hollow) got = await D.run_model_draft(model, _source({"wide45": _table(45)}), ["wide45"]) o = got["wide45"] assert o.status == "failed" assert o.error == ("这张表 45 列，分 3 段起草，第 1 段（表级的项）、第 2 段（c001 到 c023，共 23 列）、" - "第 3 段（c024 到 c045，共 22 列）都没写出来：模型只给了空壳，没有写出可用的内容；已拆小重试仍没写出") + "第 3 段（c024 到 c045，共 22 列）都没写出来：模型只给了空壳，没有写出可用的内容；" + "其中第 2、3 段已拆小重试仍没写出") def test_reasons_are_joined_by_their_base(): - """(原因, 第几段)：同样的原因只说一次；同一个原因落在几段上写「这几段」，同一段拆出来的几块仍是「这一段」。""" + """(原因, 第几段)：同样的原因只说一次；同一个原因落在几段上写「这几段」，同一段拆出来的几块仍是「这一段」。 + 「已拆小重试仍没写出」只跟在真拆过的那几段后面说：说到的段都拆过时不点名，只有一部分拆过时写「其中第几段」。""" W = D._Why ex = D.REASON_EXHAUSTED.format(budget=32768, seg=D.SEG) assert D._join([(W(D.REASON_HOLLOW), 1), (W(D.REASON_HOLLOW, split=True), 2)]) == \ - D.REASON_HOLLOW + D.REASON_SPLIT + D.REASON_HOLLOW + "；其中第 2 段已拆小重试仍没写出" assert D._join([(W(ex, split=True), 2), (W(D.REASON_HOLLOW), 1)]) == \ - f"{_exhausted()}；{D.REASON_HOLLOW}{D.REASON_SPLIT}" + f"{_exhausted()}；{D.REASON_HOLLOW}；其中第 2 段已拆小重试仍没写出" + assert D._join([(W(ex, split=True), 2), (W(ex, split=True), 3)]) == _exhausted("这几段") + D.REASON_SPLIT + assert D._join([(W(ex, split=True), 3), (W(ex), 4), (W(ex, split=True), 2)]) == \ + _exhausted("这几段") + "；其中第 2、3 段已拆小重试仍没写出" assert D._join([(W(ex), 2), (W(ex), 3)]) == _exhausted("这几段") - assert D._join([(W(ex), 2), (W(ex, split=True), 2)]) == _exhausted() + D.REASON_SPLIT + assert D._join([(W(ex), 2), (W(ex, split=True), 2)]) == _exhausted() + D.REASON_SPLIT, "同一段里有一块拆过就算拆过" assert D._join([(W("甲"), 1), (W("乙"), 1), (W("丙", split=True), 1)]) == "甲；乙", "没说出来的原因不补后缀" - halted = D.REASON_HALTED.format(n=catalog.MODEL_GIVE_UP_AFTER) + halted = D.REASON_HALTED.format(window=catalog.MODEL_HALT_WINDOW, on=catalog.MODEL_HALT_ON) + assert halted == "；最近 10 次调用有 7 次以上没写出来，已停止拆小重试，建议更换起草模型" assert D._join([(W(D.REASON_HOLLOW, halted=halted), 1), (W(D.REASON_HOLLOW), 2)]) == D.REASON_HOLLOW + halted - slow = D.REASON_SLOW.format(n=catalog.MODEL_MIN_PART_COLUMNS) - assert D._join([(W(D.REASON_HOLLOW, halted=halted), 1), (W(D.REASON_HOLLOW, halted=slow), 2), - (W(D.REASON_HOLLOW, halted=halted), 3)]) == D.REASON_HOLLOW + halted + slow, "两种止损各说一次" + assert D._join([(W(D.REASON_HOLLOW, halted=halted), 1), (W(D.REASON_HOLLOW, halted=halted), 3)]) == \ + D.REASON_HOLLOW + halted, "止损只说一次" assert D._join([(W("调用模型出错：{坏的}"), 1)]) == "调用模型出错：{坏的}", "报错原文里的花括号原样留着" @@ -865,50 +872,133 @@ def _by_table(model: Scripted) -> dict[str, int]: return out -async def test_after_two_segments_in_a_row_come_to_nothing_the_rest_are_not_split(monkeypatch): - """模型明显用不了（思考总是吃满额度）：连续 2 段拆到底仍一项没写出，后面的段只问一次、不再拆，报错说明已停止拆小重试。 - 有一段写出来了就恢复拆小。一次只起草一段（并发 1），先后是确定的。""" +HALTED = "；最近 10 次调用有 7 次以上没写出来，已停止拆小重试，建议更换起草模型" + + +async def test_when_most_recent_calls_fail_the_rest_are_asked_once_until_the_ratio_drops(monkeypatch): + """按比例止损：这次起草最近 10 次调用有 7 次以上没写出来，之后的段不再拆小、每块只问一次，报错说明已停止拆小重试； + 比例降下去（最近 10 次里少于 7 次）就恢复拆小。偶尔一次写出来不会清零：t4 到 t6 写出来了，最近 10 次里仍有 7 次没写 + 出来，照旧不拆；t7 写出来以后才降下去，t8 照常拆小。一次只起草一段（并发 1），先后是确定的。""" monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) - assert catalog.MODEL_GIVE_UP_AFTER == 2 - tables = {f"t{i}": _table(10) for i in range(1, 5)} - exhausted = Reply(text="", finish="length", thinking="……") - - model = Scripted(lambda call: exhausted) - got = await D.run_model_draft(model, _source(tables), list(tables)) - assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 1} - budget = _exhausted() - assert got["t1"].error == budget + D.REASON_SPLIT - assert got["t3"].error == budget + "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型" - + assert (catalog.MODEL_HALT_WINDOW, catalog.MODEL_HALT_ON) == (10, 7) + tables = {f"t{i}": _table(10) for i in range(1, 9)} ok = answering(tables) - model = Scripted(lambda call: ok(call) if "t3" in call.asked else exhausted) + bad = {"t1", "t2", "t3", "t8"} + + model, reports = Scripted(lambda call: EXHAUSTED if set(call.asked) & bad else ok(call)), Reports() + got = await D.run_model_draft(model, _source(tables), list(tables), hooks=reports.hooks()) + assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 1, "t5": 1, "t6": 1, "t7": 1, "t8": 3} + assert got["t1"].error == _exhausted() + D.REASON_SPLIT + assert got["t3"].error == _exhausted() + HALTED + assert {got[f"t{i}"].status for i in range(4, 8)} == {"ok"} + assert got["t8"].error == _exhausted() + D.REASON_SPLIT, "比例降下去以后照常拆小" + assert len(reports.items) == len(model.calls) + + +async def test_while_halted_a_chunk_is_not_asked_again_for_missing_columns(monkeypatch): + """止损期间每块只问一次：回复里漏了几列不补问，漏的列记没写出来、说明已停止拆小重试。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + tables = {**{f"x{i}": _table(4) for i in range(1, 8)}, "t8": _table(8)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + if "t8" not in call.asked: + raise RuntimeError("网关拒绝了这次请求") + reply = ok(call) + reply.obj["tables"][0]["columns"] = reply.obj["tables"][0]["columns"][:6] + return reply + + model = Scripted(handle) got = await D.run_model_draft(model, _source(tables), list(tables)) - assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 3}, "t3 写出来了，t4 照常拆小" - assert got["t3"].status == "ok" and got["t4"].error == budget + D.REASON_SPLIT + assert _by_table(model)["t8"] == 1, "漏了 c007、c008 也不补问" + o = got["t8"] + assert o.status == "partial" + assert o.error == ("这张表 8 列，有 2 列（c007、c008）没写出来：模型的回复里漏了这些列" + HALTED + + "。可以只重试这张表") + + +#: 200 列的宽表分 6 段起草（表级一段、列分段 5 段），逐段写上列的范围 +WIDE200_PARTS = ("这张表 200 列，分 6 段起草，第 1 段（表级的项）、第 2 段（c001 到 c040，共 40 列）、" + "第 3 段（c041 到 c080，共 40 列）、第 4 段（c081 到 c120，共 40 列）、第 5 段（c121 到 c160，共 40 列）、" + "第 6 段（c161 到 c200，共 40 列）") + + +async def test_a_wide_table_skips_its_remaining_parts_after_three_dead_parts_in_a_row(monkeypatch): + """同一张表连续 3 个原始段都一项没写出来：这张表剩下的段跳过不问，原因写清跳过了几段、可以换一个起草模型后只重试 + 这张表。200 列分 6 段（表级一段、列分段 5 段）。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + assert catalog.MODEL_TABLE_GIVE_UP_AFTER == 3 + tables = {"wide200": _table(200)} + + def refuse(call: Call) -> Reply: + raise RuntimeError("网关拒绝") + + model = Scripted(refuse) + got = await D.run_model_draft(model, _source(tables), ["wide200"]) + o = got["wide200"] + assert len(model.calls) == 3, "第 4 到 6 段跳过不问" + assert (o.status, o.parts_total, o.parts_done) == ("failed", 6, 6) + assert o.error == (WIDE200_PARTS + "都没写出来：调用模型出错：网关拒绝；" + "这张表接连 3 段都没写出来，剩下的 3 段已跳过，可以换一个起草模型后只重试这张表") + + # 表级那段写出来了、后面接连 3 段没写出来：只写出一部分，剩下的 2 段跳过；不再另说「可以只重试这张表」 + ok = answering(tables) + model = Scripted(lambda call: ok(call) if call.n == 0 else refuse(call)) + got = await D.run_model_draft(model, _source(tables), ["wide200"]) + o = got["wide200"] + assert len(model.calls) == 4 and o.status == "partial" + assert o.error == ("这张表 200 列，分 6 段起草，第 2 段（c001 到 c040，共 40 列）、第 3 段（c041 到 c080，共 40 列）、" + "第 4 段（c081 到 c120，共 40 列）、第 5 段（c121 到 c160，共 40 列）、第 6 段（c161 到 c200，共 40 列）" + "没写出来：调用模型出错：网关拒绝；这张表接连 3 段都没写出来，剩下的 2 段已跳过，" + "可以换一个起草模型后只重试这张表") + + +async def test_a_part_that_writes_resets_the_dead_parts_of_its_table(monkeypatch): + """中间有一段写出来了，就不算「接连」：200 列的第 2、3、5、6 段没写出来、第 4 段写出来了，每段都照常问。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + tables = {"wide200": _table(200)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + part, cols = call.asked["wide200"] + if part == "table" or "c081" in (cols or []): + return ok(call) + raise RuntimeError("网关拒绝") + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["wide200"]) + assert len(model.calls) == 6 and got["wide200"].status == "partial" + assert "已跳过" not in got["wide200"].error async def test_the_column_parts_of_a_wide_table_stop_splitting_too(monkeypatch): - """宽表的列分段一段一段排队起草（一段拆到底再起草下一段），止损在宽表里同样起作用。 - - 95 列：表级那段拆不开（结构化、纯文本各一次）；第 2 段 32 列拆小、到 8 次封顶；这时已经连续 2 段没写出来，第 3、4 段 - 只问一次、换纯文本一次。""" + """按比例止损在宽表里同样起作用，接连没写出来的宽表剩下的段跳过：95 列一直只给空壳（已经走纯文本），第 2 段拆小到 + 止损；之后的第 3 段只问一次；接连 3 段没写出来，第 4 段跳过。""" monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) - model = Scripted(_hollow) + model = Scripted(_text_only(_hollow)) got = await D.run_model_draft(model, _source({"wide95": _table(95)}), ["wide95"]) - assert len(model.calls) == 2 + 8 + 2 + 2 - assert got["wide95"].error == (WIDE95_PARTS + "都没写出来：" + D.REASON_HOLLOW - + "；每段最多调用 8 次模型，这一段到了次数仍没写完，已停止重试" - + D.REASON_SPLIT + "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型") + o = got["wide95"] + asked = [c for c in model.calls if c.mode == "text"] + assert len(asked) <= 2 + 8 + 1 + assert not any("c065" in (c.asked["wide95"][1] or []) for c in asked), "第 4 段没问" + assert o.status == "failed" and o.error.startswith(WIDE95_PARTS + "都没写出来：") + assert HALTED in o.error + assert o.error.endswith("这张表接连 3 段都没写出来，剩下的 1 段已跳过，可以换一个起草模型后只重试这张表") async def test_a_wide_table_that_never_comes_back_is_bounded_by_the_first_wave(): - """默认并发（3 段同时起草）：294 列（8 个列分段）一直只给空壳，最多是表级那段 2 次、头 3 个列分段各 8 次（封顶）、 - 其余 5 段各 2 次。以前所有列分段一下子都排上去、一层一层地拆，要 168 次。""" + """默认并发（3 段同时起草）：294 列（8 个列分段）一直只给空壳：表级那段 2 次，头一批同时起草的列分段拆小到止损或 + 封顶；接连 3 段没写出来以后，没开始的段跳过不问。以前所有列分段一下子都排上去、一层一层地拆，要 168 次。""" model = Scripted(_hollow) got = await D.run_model_draft(model, _source({"wide294": _table(294)}), ["wide294"]) - assert got["wide294"].status == "failed" and got["wide294"].parts_total == 9 - assert len(model.calls) <= 2 + 3 * 8 + 5 * 2 + o = got["wide294"] + assert o.status == "failed" and o.parts_total == 9 and o.parts_done == 9 + assert len(model.calls) <= 2 + 4 * 8 + assert "已跳过" in o.error + asked = {c for call in model.calls for c in call.asked["wide294"][1] or []} + assert "c294" not in asked, "最后一段没问" async def test_a_big_draft_does_not_hold_the_model_ahead_of_a_later_one(monkeypatch): @@ -1042,7 +1132,7 @@ async def test_the_chunks_of_a_part_cut_by_the_smaller_cap_share_its_eight_calls """每个原始段（连同段大小减半后按上限分开的各块、拆出来的、补问的、重试的）一共最多调用 8 次。以前分开的每一块各有 8 次：上限缩到 10 列以后一直只给空壳，4 张窄表拼成的 40 列段调用了 20 次，单张 40 列的表也是 20 次。""" monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) - monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99) # 不让止损先停下来 + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) # 不让止损先停下来 tables = {"w40": _table(40), **{f"n{i}": _table(10) for i in range(1, 5)}, "x40": _table(40)} def handle(call: Call) -> Reply: @@ -1085,34 +1175,31 @@ class AlwaysSlow: raise TimeoutError() -SLOW_STOP = "；拆到每段 10 列仍然超时，已停止拆小重试，建议更换起草模型" - - -async def test_a_model_that_always_times_out_stops_after_three_rounds_not_seven(): - """每次调用都超时、和段大小无关：超时拆出来的两半同时问（不再等带表级项的前一半写完才问后一半），拆到每段 10 列的 - 调用接连超时 2 次就停止拆小，拆出来还没问的块不再问。40 列的表 3 轮、6 次调用就结束（时限 420 秒时约 21 分钟）； - 以前 40→20→10→5 一层一层串着问，8 次调用约 7 轮（约 49 分钟）。""" +async def test_a_model_that_always_times_out_is_bounded_by_the_part_cap(): + """每次调用都超时：超时拆出来的两半同时问，最多拆到这一段的调用次数用完（8 次）或止损。以前 40→20→10→5 一层一层串着问， + 8 次调用约 7 轮；现在两半同时问，至多 4 轮。""" tables = {"t40": _table(40)} slow = AlwaysSlow(tables) model, reports = Scripted(slow), Reports() got = await D.run_model_draft(model, _source(tables), ["t40"], hooks=reports.hooks()) - assert [_width(c, tables) for c in model.calls] == [40, 20, 20, 10, 10, 10], "拆到每段 10 列仍超时就不再往下拆" - assert slow.rounds == [1, 2, 2, 3, 3, 3], "两半同时问" - assert reports.failures == ["timeout"] * 6 and len(reports.items) == len(model.calls) + assert len(model.calls) <= catalog.MODEL_PART_MAX_CALLS and max(slow.rounds) <= 4 + assert [_width(c, tables) for c in model.calls[:3]] == [40, 20, 20], "两半同时问" + assert set(reports.failures) == {"timeout"} and len(reports.items) == len(model.calls) o = got["t40"] - assert o.status == "failed" and o.error == "模型 420 秒内没有写完这一段" + D.REASON_SPLIT + SLOW_STOP + assert o.status == "failed" and "模型 420 秒内没有写完这一段" + D.REASON_SPLIT in o.error + assert o.chunks_done == o.chunks_total > 1, "拆出来的块都结束了" -async def test_three_parts_that_always_time_out_end_in_four_rounds(): - """首轮同时起草的 3 段都一直超时：拆到每段 10 列仍超时就一起停下，不再各自拆满 8 次。12 次调用、4 轮；以前 24 次 - 调用、约 8 轮（止损要等一整段结束才计数，首轮的 3 段都跑满）。""" +async def test_three_parts_that_always_time_out_stop_together(): + """首轮同时起草的 3 段都一直超时：最近 10 次调用 7 次以上没写出来以后，拆出来还没问的块不再问，不再各自拆满 8 次。 + 以前（接连超时的止损）12 次调用；更早要 24 次。""" tables = {f"t{i}": _table(40) for i in range(1, 4)} slow = AlwaysSlow(tables) model = Scripted(slow) got = await D.run_model_draft(model, _source(tables), list(tables)) - assert len(model.calls) == 12 and max(slow.rounds) == 4 + assert len(model.calls) <= 9 and max(slow.rounds) <= 4 assert {o.status for o in got.values()} == {"failed"} - assert all(o.error.endswith(SLOW_STOP) for o in got.values()) + assert any(HALTED in o.error for o in got.values()) async def test_timeouts_that_go_away_at_ten_columns_are_still_split_down_and_written(): @@ -1152,7 +1239,7 @@ async def test_a_call_whose_reply_cannot_be_read_still_counts(monkeypatch): """读回复时出了意料之外的错：这张表记出错，这次调用照样记进用量（算一次出错），调用次数和用量对得上。""" tables = {"t4": _table(4)} - def broken(entry, meta, *, system): + def broken(entry, meta, *, system, **_): raise RuntimeError("校验时出了意外") monkeypatch.setattr(catalog, "_from_model", broken) @@ -1190,24 +1277,52 @@ async def test_a_wide_table_that_wrote_nothing_names_every_part(): got = await D.run_model_draft(Scripted(handle), _source({"wide95": _table(95)}), ["wide95"]) o = got["wide95"] - assert o.status == "failed" and o.error == WIDE95_PARTS + "都没写出来：调用模型出错：网关拒绝" + assert o.status == "failed" and o.error == (WIDE95_PARTS + "都没写出来：调用模型出错：网关拒绝；" + "这张表接连 3 段都没写出来，剩下的 1 段已跳过，可以换一个起草模型后只重试这张表") narrow = await D.run_model_draft(Scripted(handle), _source({"t8": _table(8)}), ["t8"]) assert narrow["t8"].error == "调用模型出错：网关拒绝", "只有一段的表：这一段就是整张表，照旧只写原因" async def test_a_halted_part_that_could_be_split_is_not_asked_again_in_text_mode(monkeypatch): - """止损以后本来能拆小的段只给空壳：已经走纯文本的不再原样再问（约定只对拆不开的段再问一次）；结构化输出的照旧换 - 纯文本问一次，看看是不是结构化输出的问题（见 test_the_column_parts_of_a_wide_table_stop_splitting_too）。""" + """止损以后每块只问一次：已经走纯文本的空壳不再原样再问、也不拆小。t1：结构化不支持 1 次、纯文本 1 次、拆成两半各问 + 一次再原样再问一次（6 次都没写出来）；t2 的第一次是第 7 次没写出来，之后止损。""" monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) tables = {f"t{i}": _table(10) for i in range(1, 5)} model = Scripted(_text_only(_hollow)) got = await D.run_model_draft(model, _source(tables), list(tables)) - # t1：结构化不支持 1 次、纯文本 1 次、拆成两半各问一次再原样再问一次；t2 同样（少了结构化那次）；之后止损 - assert _by_table(model) == {"t1": 6, "t2": 5, "t3": 1, "t4": 1} - assert got["t3"].error == (catalog.MODEL_EMPTY_REASON - + "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型") + assert _by_table(model) == {"t1": 6, "t2": 1, "t3": 1, "t4": 1} + assert got["t2"].error == catalog.MODEL_EMPTY_REASON + HALTED + + +async def test_while_halted_structured_output_still_gets_one_probe_in_text(monkeypatch): + """止损期间结构化输出只给空壳：这次起草还没换纯文本问过的，照旧换纯文本问一次（看看是不是结构化输出的问题）；纯文本 + 写出来了，后面都走纯文本。问过一次（纯文本也只给空壳）就不再问。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + tables = {**{f"x{i}": _table(4) for i in range(1, 8)}, "t8": _table(8), "t9": _table(8)} + ok = answering(tables) + + def handle(text_works: bool) -> Callable[[Call], Reply]: + def go(call: Call) -> Reply: + if next(iter(call.asked)).startswith("x"): + raise RuntimeError("网关拒绝了这次请求") + if call.mode == "structured": + return _hollow(call) + return ok(call) if text_works else _hollow(call) + return go + + model = Scripted(handle(True)) + got = await D.run_model_draft(model, _source(tables), list(tables)) + assert [c.mode for c in model.calls if "x1" not in c.asked and not next(iter(c.asked)).startswith("x")] == \ + ["structured", "text", "text"] + assert got["t8"].status == got["t9"].status == "ok" + + model = Scripted(handle(False)) + got = await D.run_model_draft(model, _source(tables), list(tables)) + assert _by_table(model)["t8"] == 2 and _by_table(model)["t9"] == 1, "纯文本问过一次，不再问" + assert got["t9"].error == catalog.MODEL_EMPTY_REASON + HALTED # -------------------------------------------------------------------------- @@ -1299,3 +1414,113 @@ def test_cut_values_are_dropped_and_complete_ones_kept(): @pytest.mark.parametrize("text", ["好的，我来写", "{坏的 json}", ""]) def test_not_json_is_not_a_reply(text): assert D._parse_text(text, cut=False) is None + + +# -------------------------------------------------------------------------- +# 闭环第一步的起草遗留：逐段交出去落盘、块进度、「已拆小重试」只说拆过的段 +# -------------------------------------------------------------------------- + + +class Written: + """进度回调 part_written 收到的：(表名, 已写出内容的段数, 这时这张表拿到的列)；chunks_changed 收到的块进度。""" + + def __init__(self) -> None: + self.parts: list[tuple[str, int, set[str], bool]] = [] + self.chunks: dict[str, list[tuple[int, int]]] = {} + + async def part_written(self, table: str, written: D.PartWrite) -> None: + cols = set((written.notes.get("columns") or {})) + self.parts.append((table, written.parts_written, cols, "label" in written.notes)) + + async def chunks_changed(self, table: str, total: int, done: int) -> None: + self.chunks.setdefault(table, []).append((total, done)) + + def hooks(self) -> D.Hooks: + return D.Hooks(part_written=self.part_written, chunks_changed=self.chunks_changed) + + +async def test_each_part_of_a_wide_table_is_handed_over_as_soon_as_it_is_written(monkeypatch): + """宽表每写出一段就交给调用方并入目录（最后一段由 table_done 收尾）：交出去的是这张表到这时拿到的全部内容。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + tables = {"wide95": _table(95)} + seen = Written() + got = await D.run_model_draft(Scripted(answering(tables)), _source(tables), ["wide95"], hooks=seen.hooks()) + o = got["wide95"] + assert o.status == "ok" and o.parts_written == 4 + assert [(t, n, len(cols), label) for t, n, cols, label in seen.parts] == [ + ("wide95", 1, 0, True), ("wide95", 2, 32, True), ("wide95", 3, 64, True)] + + def handle(call: Call) -> Reply: + part, cols = call.asked["wide95"] + if part == "columns" and "c033" in cols: + raise RuntimeError("网关拒绝") + return answering(tables)(call) + + seen = Written() + got = await D.run_model_draft(Scripted(handle), _source(tables), ["wide95"], hooks=seen.hooks()) + assert got["wide95"].status == "partial" and got["wide95"].parts_written == 3 + assert [n for _, n, _, _ in seen.parts] == [1, 2], "没写出来的那段不交" + + +async def test_a_narrow_table_is_handed_over_only_once_at_the_end(): + tables = {"t8": _table(8)} + seen = Written() + got = await D.run_model_draft(Scripted(answering(tables)), _source(tables), ["t8"], hooks=seen.hooks()) + assert seen.parts == [] and got["t8"].parts_written == 1 + failed = await D.run_model_draft(Scripted(lambda call: EXHAUSTED), _source({"t4": _table(4)}), ["t4"]) + assert failed["t4"].parts_written == 0 + + +async def test_chunks_count_what_the_table_is_actually_asked_in(monkeypatch): + """块进度：一开始是这张表分的段数；一段被拆成两半多一块、被缩小的上限分开成几块就多几块；一块结束算一块。 + 多表拼成的段按表拆开时，每张表还是一块。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + monkeypatch.setattr(catalog, "MODEL_SHRINK_AFTER", 1) + tables = {"a12": _table(12), "b40": _table(40)} + ok = answering(tables) + seen = Written() + got = await D.run_model_draft(Scripted(lambda call: EXHAUSTED if call.n == 0 else ok(call)), _source(tables), + ["a12", "b40"], hooks=seen.hooks()) + a, b = got["a12"], got["b40"] + assert a.status == b.status == "ok" + assert (a.chunks_total, a.chunks_done) == (2, 2), "额度用完拆成两半" + assert (b.chunks_total, b.chunks_done) == (2, 2), "上限缩到 20 列，40 列的段分开成两块" + assert seen.chunks["a12"][-1] == (2, 2) and seen.chunks["b40"][-1] == (2, 2) + assert all(done <= total for pairs in seen.chunks.values() for total, done in pairs) + + wide = {"wide95": _table(95)} + got = await D.run_model_draft(Scripted(answering(wide)), _source(wide), ["wide95"]) + assert (got["wide95"].chunks_total, got["wide95"].chunks_done) == (4, 4) + + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 4) + packed = {"p6": _table(6), "q6": _table(6)} + okp = answering(packed) + model = Scripted(lambda call: _hollow(call) if call.n == 0 else okp(call)) + got = await D.run_model_draft(model, _source(packed), ["p6", "q6"]) + assert [set(c.asked) for c in model.calls] == [{"p6", "q6"}, {"p6"}, {"q6"}], "按表拆开" + assert {(o.chunks_total, o.chunks_done) for o in got.values()} == {(1, 1)} + + +async def test_only_the_parts_that_were_split_say_so(monkeypatch): + """一张表几段没写出来、只有一部分真拆小重试过：「已拆小重试仍没写出」只跟在拆过的那几段后面说。95 列：第 3 段一直用完 + 额度（拆小重试过），第 4 段调用出错（没拆过）。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_SHRINK_AFTER", 99) + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) + tables = {"wide95": _table(95)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + part, cols = call.asked["wide95"] + if part == "columns" and cols[0] >= "c065": + raise RuntimeError("网关拒绝") + if part == "columns" and cols[0] >= "c033": + return EXHAUSTED + return ok(call) + + got = await D.run_model_draft(Scripted(handle), _source(tables), ["wide95"]) + o = got["wide95"] + assert o.status == "partial" + assert o.error == ("这张表 95 列，分 4 段起草，第 3 段（c033 到 c064，共 32 列）、第 4 段（c065 到 c095，共 31 列）没写出来：" + + _exhausted() + "；调用模型出错：网关拒绝；其中第 3 段已拆小重试仍没写出。可以只重试这张表") diff --git a/backend/tests/test_catalog_draft_jobs.py b/backend/tests/test_catalog_draft_jobs.py index bc366876..d69704b7 100644 --- a/backend/tests/test_catalog_draft_jobs.py +++ b/backend/tests/test_catalog_draft_jobs.py @@ -441,6 +441,9 @@ def test_old_job_tables_get_the_new_columns(): assert conn.execute(text("SELECT timeouts, failed_calls, part_columns, notice FROM catalog_draft_jobs")) \ .one() == (0, 0, None, None) assert conn.execute(text("SELECT blank_columns, blank_sample FROM catalog_draft_tasks")).one() == (0, None) + # 闭环第一步：逐段落盘的段数、块进度。存量行是 0 和 NULL：升级前没记这些 + assert conn.execute(text("SELECT parts_written, chunks_total, chunks_done FROM catalog_draft_tasks")) \ + .one() == (0, None, None) async def test_a_job_that_cannot_start_is_failed_with_the_reason(client, monkeypatch): @@ -779,3 +782,233 @@ async def test_jobs_go_away_with_their_source(client): assert (await session.execute(select(CatalogDraftJob).where(CatalogDraftJob.id == job["id"]))).first() is None assert (await session.execute(select(CatalogDraftTask).where(CatalogDraftTask.job_id == job["id"]))).first() \ is None + + +# -------------------------------------------------------------------------- +# 闭环第一步：宽表逐段落盘、停止后的说法、块进度、停止后的留空提示、同步起草与任务互斥 +# -------------------------------------------------------------------------- + + +def _wide(n: int = 95) -> dict: + return {"wide": {"qualified": "wide", "columns": [_col(f"c{i:03d}") for i in range(1, n + 1)], + "primary_key": ["c001"], "comment": None, "is_view": False, "foreign_keys": []}} + + +def _wide_reply(blank: set[str] = frozenset()) -> dict: + """95 列的宽表每列都写中文名；blank 里的列只列出、不写（模型拿不准）。""" + return {"wide": {"label": "宽表", "kind": "fact", + "columns": [{"name": f"c{i:03d}", "label": f"第{i}列"} for i in range(1, 96) + if f"c{i:03d}" not in blank]}} + + +class Hold: + """假模型：请写的表在 tables 里、或请写的列里有 columns 里的列时，等 release 才回（停止时这次调用被取消）；别的照 + FakeModel 回。waiting：有调用在等了。""" + + def __init__(self, replies: dict, *, tables: set[str] = frozenset(), columns: set[str] = frozenset()) -> None: + self.inner = FakeModel(replies) + self.tables, self.columns = set(tables), set(columns) + self.release, self.waiting = asyncio.Event(), asyncio.Event() + self.cancelled = 0 + self.model_name = "hold-model" + + def with_structured_output(self, schema, **_): + outer = self + + class _S: + async def ainvoke(self, messages): + asked = requested("\n".join(str(m.content) for m in messages)) + cols = {c for _, listed in asked.values() for c in listed or []} + if outer.tables & set(asked) or outer.columns & cols: + outer.waiting.set() + try: + await outer.release.wait() + except asyncio.CancelledError: + outer.cancelled += 1 + raise + return outer.inner._reply(messages, "structured") + + return _S() + + async def ainvoke(self, messages): + return await self.inner.ainvoke(messages) + + +async def _task(client, sid: str, job_id: str, table: str) -> dict: + return next(t for t in (await get_job(client, sid, job_id))["tasks"] if t["table_name"] == table) + + +async def _start_held_wide(client, sid: str, monkeypatch) -> tuple[str, Hold]: + """95 列的宽表分 4 段：表级那段和第 2 段照常写，第 3、4 段一直等着。等到前两段都写进目录才返回。""" + model = Hold(_wide_reply(), columns={"c033", "c065"}) + use(monkeypatch, model) + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["wide"], "use_model": True})).json()["job"] + + async def two_written() -> bool: + return (await _task(client, sid, job["id"], "wide"))["parts_written"] == 2 + + await until(two_written) + return job["id"], model + + +async def test_a_stopped_wide_table_keeps_the_parts_already_written(client, monkeypatch): + """宽表每写出一段就并入目录：停止时已经写进去的段留着，这张表照旧记停止，model_error 说清写进了几段、剩几段没起草。""" + sid = await _source(_wide()) + job_id, model = await _start_held_wide(client, sid, monkeypatch) + task = await _task(client, sid, job_id, "wide") + assert task["status"] == "running" and task["parts_total"] == 4 and task["parts_done"] == 2 + assert (task["chunks_total"], task["chunks_done"]) == (4, 2) + assert task["added"] == 2 + 32 and task["version"] >= 1, "表级两项 + 第 2 段 32 列" + + r = await client.post(f"{_base(sid)}/draft-jobs/{job_id}/cancel") + assert r.status_code == 200, r.text + task = next(t for t in r.json()["job"]["tasks"] if t["table_name"] == "wide") + assert task["status"] == "cancelled" and task["error"] is None and task["parts_written"] == 2 + assert task["model_error"] == "已停止：已写进目录 2 段，剩下 2 段没起草" + assert model.cancelled == 2, "第 3、4 段正在进行的调用被停掉" + notes = (await client.get(f"{_base(sid)}/wide")).json()["notes"] + assert notes["label"]["value"] == "宽表" and notes["kind"]["value"] == "fact" + assert set(notes["columns"]) == {f"c{i:03d}" for i in range(1, 33)}, "第 2 段写进去的留着" + row = next(r for r in (await client.get(_base(sid))).json()["tables"] if r["table_name"] == "wide") + assert row["last_draft"]["status"] == "cancelled" and row["last_draft"]["parts_written"] == 2 + assert row["last_draft"]["blank_columns_from"] is None, "停止前写进了新内容：以前的留空列数不再沿用" + + +async def test_an_interrupted_wide_table_keeps_its_written_parts_too(client, monkeypatch): + sid = await _source(_wide()) + job_id, _ = await _start_held_wide(client, sid, monkeypatch) + await catalog_jobs.manager.shutdown(grace=5) + task = await _task(client, sid, job_id, "wide") + assert task["status"] == "cancelled" and task["error"] == "服务重启，起草中断" + assert task["model_error"] == "已中断：已写进目录 2 段，剩下 2 段没起草" + assert len((await client.get(f"{_base(sid)}/wide")).json()["notes"]["columns"]) == 32 + + # 进程没来得及收尾（直接退出）：下次启动时同样写清 + async with SessionLocal() as session: + job = CatalogDraftJob(source_id=sid, status="running", use_model=True) + session.add(job) + await session.flush() + session.add(CatalogDraftTask(job_id=job.id, seq=0, table_name="wide", status="running", parts_total=3, + parts_written=1)) + session.add(CatalogDraftTask(job_id=job.id, seq=1, table_name="gates", status="pending", parts_total=1)) + await session.commit() + crashed = job.id + await catalog_jobs.startup() + by = {t["table_name"]: t for t in (await get_job(client, sid, crashed))["tasks"]} + assert by["wide"]["model_error"] == "已中断：已写进目录 1 段，剩下 2 段没起草" + assert by["gates"]["model_error"] is None and by["gates"]["error"] == "服务重启，起草中断" + + +async def test_a_finished_wide_table_counts_every_write_once(client, monkeypatch): + """逐段落盘之后整张表收尾一次：新增、更新的计数是各次写入的合计（每一项只数一次），块进度和写进去的段数都记下。""" + sid = await _source(_wide()) + job = await _draft(client, sid, monkeypatch, FakeModel(_wide_reply({"c010", "c040", "c070"})), ["wide"]) + task = job["tasks"][0] + assert task["status"] == "ok" and task["parts_written"] == 4 and task["parts_done"] == 4 + assert (task["chunks_total"], task["chunks_done"]) == (4, 4) + assert (task["added"], task["updated"], task["removed"]) == (2 + 92, 0, 0) + assert task["blank_columns"] == 3 and task["blank_sample"] == ["c010", "c040", "c070"] + notes = (await client.get(f"{_base(sid)}/wide")).json()["notes"] + assert len(notes["columns"]) == 92 + + plain = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["wide"]})).json()["job"] + await finished(plain["id"]) + task = (await get_job(client, sid, plain["id"]))["tasks"][0] + assert (task["parts_written"], task["chunks_total"], task["chunks_done"]) == (0, None, None), "不用模型时为 null" + + +async def test_chunk_progress_shows_a_part_that_was_split(client, monkeypatch): + """块进度：这张表实际分成的块数和已结束的块数，一段超时拆成两半后变多。""" + monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.05) + sid = await _source(_tickets(24)) + replies = {"tickets": {"label": "门票", "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} + for i in range(1, 25)]}} + job = await _draft(client, sid, monkeypatch, FakeModel(replies, hang=1), ["tickets"]) + task = job["tasks"][0] + assert task["status"] == "ok" and (task["chunks_total"], task["chunks_done"]) == (2, 2) + assert task["parts_written"] == 1 and task["parts_total"] == 1 + + +async def test_a_partial_table_also_drops_model_items_of_columns_gone_from_the_schema(client, monkeypatch): + """整张表写完时按完整覆盖的规则收尾，只写出一部分的也一样：只删已经不在表结构里的列的模型起草项。""" + sid = await _source(_tickets(6)) + full = {"label": "门票", "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} for i in range(1, 7)]} + await _draft(client, sid, monkeypatch, FakeModel({"tickets": full}), ["tickets"]) + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + row.schema_cache = {"tables": _tickets(5)} # 重新探查过：c06 已经没有了 + await session.commit() + some = {"label": "门票", "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} for i in range(1, 5)]} + job = await _draft(client, sid, monkeypatch, FakeModel({"tickets": some}, list_all=False), ["tickets"]) + task = job["tasks"][0] + assert task["status"] == "partial" and task["removed"] == 1 + after = (await client.get(f"{_base(sid)}/tickets")).json()["notes"]["columns"] + assert set(after) == {"c01", "c02", "c03", "c04", "c05"}, "c05 是以前起草的，这次没写出来也留着" + + +async def test_blank_columns_from_carries_the_last_finished_draft_over_a_stop(client, monkeypatch): + """目录清单的 last_draft.blank_columns_from：最近一次结束于完成、部分完成的起草记下的留空列数。之后的起草停止了、 + 没写进新内容时照样带着，界面据此继续提示；不用模型的起草不算。""" + sid = await _source(_tickets(4)) + replies = {"tickets": {"label": "门票", "columns": [{"name": "c01", "label": "编号"}]}} + await _draft(client, sid, monkeypatch, FakeModel(replies), ["tickets"]) + + async def last() -> dict: + rows = (await client.get(_base(sid))).json()["tables"] + return next(r for r in rows if r["table_name"] == "tickets")["last_draft"] + + first = await last() + assert first["status"] == "ok" and first["blank_columns"] == 3 and first["blank_columns_from"] == 3 + assert first["parts_written"] == 1 + + plain = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["tickets"]})).json()["job"] + await finished(plain["id"]) + assert (await last())["blank_columns_from"] == 3, "不用模型的起草不改模型留空的列" + + monkeypatch.setattr(catalog_jobs.manager, "start", lambda job_id: None) + queued = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["tickets"], "use_model": True})).json() + await client.post(f"{_base(sid)}/draft-jobs/{queued['job']['id']}/cancel") + stopped = await last() + assert stopped["status"] == "cancelled" and stopped["blank_columns"] == 0 and stopped["parts_written"] == 0 + assert stopped["blank_columns_from"] == 3 + + other = await _source(_tickets(4)) + rows = (await client.get(_base(other))).json()["tables"] + assert rows[0]["last_draft"] is None + + +async def test_a_sync_draft_and_draft_jobs_exclude_each_other(client, monkeypatch): + """同步起草进行期间，同一个源建任务、重试、再开一次同步起草都 409 catalog_draft_busy（导入照 ensure_idle 认）； + 同步起草结束后照常。""" + sid = await _source() + use(monkeypatch, FakeModel(REPLIES)) + old = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"], "use_model": True})).json()["job"] + await finished(old["id"]) + async with SessionLocal() as session: + await session.execute( + CatalogDraftTask.__table__.update().where(CatalogDraftTask.job_id == old["id"]).values(status="failed")) + await session.commit() + + model = Hold(REPLIES, tables={"visits"}) + use(monkeypatch, model) + sync = asyncio.create_task(client.post(f"{_base(sid)}/draft", json={"tables": ["visits"], "use_model": True})) + await asyncio.wait_for(model.waiting.wait(), 5) + + busy = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]}) + assert busy.status_code == 409 and busy.json()["code"] == "catalog_draft_busy", busy.text + assert "正在起草" in busy.json()["detail"] + retry = await client.post(f"{_base(sid)}/draft-jobs/{old['id']}/retry") + assert retry.status_code == 409 and retry.json()["code"] == "catalog_draft_busy", retry.text + again = await client.post(f"{_base(sid)}/draft", json={"tables": ["gates"]}) + assert again.status_code == 409 and again.json()["code"] == "catalog_draft_busy", again.text + async with SessionLocal() as session: + with pytest.raises(catalog_jobs.DraftBusy): + await catalog_jobs.ensure_idle(session, await session.get(DataSource, sid)) + assert (await client.get(f"{_base(sid)}/draft-jobs/latest")).json()["job"]["id"] == old["id"], "没建新任务" + + model.release.set() + done = await sync + assert done.status_code == 200, done.text + after = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]}) + assert after.status_code == 202, after.text + await finished(after.json()["job"]["id"]) From 4647fc9ff3c77cfe89e6827e18dbc876d3c66061 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:32:44 +0800 Subject: [PATCH 09/39] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E9=97=AD=E7=8E=AF=E7=AC=AC=E4=B8=80=E6=AD=A5=EF=BC=88=E5=90=8E?= =?UTF-8?q?=E7=AB=AF=20A=EF=BC=89=EF=BC=9A=E9=98=B2=E9=94=99=E4=BA=8B?= =?UTF-8?q?=E5=AE=9E=E7=9A=84=E6=8F=90=E8=AE=AE=E2=80=94=E2=80=94=E8=B5=B7?= =?UTF-8?q?=E8=8D=89=E6=97=B6=E6=8F=90=E8=AE=AE=E6=9C=89=E6=95=88=E8=A1=8C?= =?UTF-8?q?=E8=BF=87=E6=BB=A4=E5=92=8C=E5=8E=BB=E9=87=8D=E8=A7=84=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 提示词：表级的项多要两项，valid_filter（有效行过滤，condition 写一段 SQL 条件、只用这张表的列，函数只用 UPPER、LOWER、TRIM、COALESCE）和 dedup（去重规则，rule 写一句话），各附一句 reason；写清只在表结构、注释、码值能看出来时才写，拿不准、没有就不写；业务日期照旧。结构化输出的约定、纯文本那条路的格式说明、只写表级的项那一行都带上这两项 - catalog._from_model：两项记模型起草、推断，依据写进 note（「依据：……」）；有效行过滤按数据源的方言解析（sqlcheck.parse_valid_filter），不过就丢掉、日志记一条，不算这张表失败；只写了一段文字的也认 - 起草引擎：表级那段写完核对有效行过滤。开了数据剖析的源经剖析的只读守卫数排除比例（catalog_profile.check_valid_filter），排除 0% 或 100% 的不收，收下的 note 写上「剖析核对：排除 3.2% 的行（共 12,345 行）」；没开剖析、连不上的照收，写明没有核对的原因；目录里人工确认、驳回过的不再查库 - 开了数据剖析的源：目录里记下的码值（剖析取过的取值和已知含义）交给模型做背景，要写表级的项的给全部列的、只写几列的给这几列的，遮罩的列、驳回的码值不给，已确认的不重复 - 测试核一遍 SQL 检查的规矩：起草来的有效行过滤是推断，漏了它只是提示；人工确认以后才是警告，驳回以后不报 --- backend/app/data/catalog.py | 87 ++++++++++- backend/app/data/catalog_draft.py | 111 ++++++++++++-- backend/tests/test_catalog_draft_facts.py | 170 ++++++++++++++++++++++ 3 files changed, 355 insertions(+), 13 deletions(-) diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 01eb87c1..d6016713 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -1516,6 +1516,10 @@ CATALOG_DRAFT_SYSTEM = """你在为一个业务数据库编写数据目录：说 log（日志表）、config（配置表）之一。 - keys：业务主键，即在业务上唯一确定一行的列（可以和数据库主键不同），写列名。 - business_date：业务日期按哪一列算，rule 写一句规则，timezone 写时区（不确定就留空）。 +- valid_filter：有效行过滤。condition 写一段 SQL 条件，筛出业务上算数的行（例如 STATUS NOT IN ('C', 'X')，去掉作废、\ +测试的行）：只用这张表自己的列、不带表名，只写比较、IN、BETWEEN、LIKE、IS NULL，用 AND、OR、NOT 连接，函数只用 UPPER、\ +LOWER、TRIM、COALESCE，不写子查询、聚合、窗口函数和分号；reason 写一句依据（例如哪条注释、哪些码值说明了这一点）。 +- dedup：去重规则，rule 写一句话（例如「同一订单号有多行时只取最后修改的那一行」），reason 写一句依据。 - columns：每一列的 label（中文名）、meaning（含义，一句话）、unit（单位，例如「元」「人」，没有单位就留空）、\ measure（度量类型：flow 流量，可跨期加总；stock 存量，时点值，不能跨期加总；ratio 比率；identifier 标识；\ status 状态；attribute 属性）。 @@ -1523,6 +1527,8 @@ status 状态；attribute 属性）。 规矩： - 只写数据事实，不写计算公式，不写指标怎么算。 - 拿不准的字段留空，不要猜。列名、表名照原样写，不要翻译或改写。 +- 有效行过滤、去重规则只在表结构、注释、码值能看出来时才写（例如状态列的注释写明了哪些取值是作废、测试），拿不准就不写，\ +没有就不写：写错了会把算数的行筛掉、把不该合并的行合并。 - 请你写的每一列都在 columns 里写一项，拿不准的字段可以留空，但不要漏掉列：漏掉的列会被再问一次。 - 标着「系统说明」的内容是导入时按核对结果生成的，以它为准：不要重复，也不要与之矛盾。 - 标着「已确认」的目录项由人工确认过，以它为准。 @@ -1541,6 +1547,7 @@ CATALOG_DRAFT_BACKGROUND_PROMPT = "使用者补充的背景和要求：\n{lines} CATALOG_DRAFT_TEXT_PROMPT = """只输出一个 JSON 对象，不要输出别的文字，格式如下（拿不准的字段不写）： {"tables": [{"table": "表名", "label": "中文名", "grain": "一行代表什么", "kind": "表类型", "keys": ["列名"], \ "business_date": {"column": "列名", "rule": "一句规则", "timezone": "时区"}, \ +"valid_filter": {"condition": "SQL 条件", "reason": "一句依据"}, "dedup": {"rule": "一句话", "reason": "一句依据"}, \ "columns": [{"name": "列名", "label": "中文名", "meaning": "含义", "unit": "单位", "measure": "度量类型"}]}]} 表类型写 fact、dimension、snapshot、log、config 之一；度量类型写 flow、stock、ratio、identifier、status、attribute 之一。""" @@ -1560,11 +1567,14 @@ CATALOG_DRAFT_PROMPT_LABELS = { "unique": "唯一约束：{cols}", "confirmed": "已确认的目录项：", # 分段起草：表名下一行说这次只写哪部分（宽表的表级一段、列分段，拆小重问时的半段） - "only_table": "这次只写表级的项（中文名、粒度、表类型、业务主键、业务日期），不写列", + "only_table": "这次只写表级的项（中文名、粒度、表类型、业务主键、业务日期、有效行过滤、去重规则），不写列", "only_columns": "这次只写下面列出的 {n} 列（这张表共 {total} 列），不写表级的项", "table_and_columns": "这次只写表级的项和这 {n} 列：{cols}", "brief_columns": "列（共 {n} 列，这里只列名字和类型）：", "drafted": "已起草的表级内容：", + # 开了数据剖析的源：目录里记下的码值（剖析看到的取值和已知含义）交给模型，判断有效行过滤用；遮罩的列不给 + "codes": "目录里记下的码值（数据剖析看到的取值和已知含义，可据此判断有效行过滤）：", + "code": "- {name}：{codes}", # 使用者补充的背景和要求 "description": "数据源说明：{text}", "guidance": "起草要求：{text}", @@ -1592,6 +1602,20 @@ CATALOG_DRAFT_PROMPT_SCHEMA: dict[str, Any] = { "timezone": {"type": "string"}}, "required": ["column"], }, + "valid_filter": { + "type": "object", + "description": "有效行过滤：只在表结构、注释、码值能看出来时才写", + "properties": {"condition": {"type": "string", "description": "SQL 条件，只用这张表的列"}, + "reason": {"type": "string", "description": "一句依据"}}, + "required": ["condition"], + }, + "dedup": { + "type": "object", + "description": "去重规则：只在表结构、注释能看出来时才写", + "properties": {"rule": {"type": "string", "description": "一句话"}, + "reason": {"type": "string", "description": "一句依据"}}, + "required": ["rule"], + }, "columns": { "type": "array", "items": { @@ -1712,11 +1736,65 @@ def _clean_text(value: Any, limit: int = _TEXT_MAX) -> str | None: return text if text and len(text) <= limit else None -def _from_model(entry: Mapping[str, Any], meta: Mapping[str, Any], *, system: bool) -> dict[str, Any]: +#: 模型给的依据（有效行过滤、去重规则的 reason）写进这一项的 note +FACT_REASON_NOTE = "依据：{reason}" + + +def _model_fact(raw: Any, key: str) -> tuple[str, str | None] | None: + """模型给的有效行过滤、去重规则 → (取值, 依据)。约定是对象（{condition 或 rule, reason}），只写了一段文字的也认； + 取值不是非空文字的不要。""" + if isinstance(raw, str): + text, reason = raw, None + elif isinstance(raw, Mapping): + text, reason = raw.get(key), raw.get("reason") + if text is None: + text = raw.get("value") + else: + return None + if not isinstance(text, str) or not text.strip(): + return None + return text.strip(), _clean_text(reason) + + +def _fact_item(value: str, reason: str | None) -> dict[str, Any]: + return make_item(value, "llm", note=FACT_REASON_NOTE.format(reason=reason) if reason else None) + + +def _model_facts(entry: Mapping[str, Any], meta: Mapping[str, Any], *, kind: str | None, + table: str | None) -> dict[str, Any]: + """模型提议的有效行过滤、去重规则（llm / proposed，依据写进 note）。 + + 有效行过滤按数据源的方言解析成一个条件、只能用这张表的列（sqlcheck.parse_valid_filter），不过就丢掉、日志记一条， + 不算这张表失败；存的是模型写的原文（只去掉首尾空白，字符串里的空格不动）。去重规则是一句话，同文字项的规矩。""" + from app.data import sqlcheck + + out: dict[str, Any] = {} + found = _model_fact(entry.get("valid_filter"), "condition") + if found is not None: + text, reason = found + problem = sqlcheck.valid_filter_problem(text, meta, kind=kind, table=table) + if problem is None: + out["valid_filter"] = _fact_item(text, reason) + else: + import logging + + logging.getLogger(__name__).info("起草数据目录：表 %s 的有效行过滤没有通过校验，已丢掉：%s（%s）", + table or meta.get("qualified") or entry.get("table"), problem, text[:200]) + found = _model_fact(entry.get("dedup"), "rule") + if found is not None and (text := _clean_text(found[0])) is not None: + out["dedup"] = _fact_item(text, found[1]) + return out + + +def _from_model(entry: Mapping[str, Any], meta: Mapping[str, Any], *, system: bool, source_kind: str | None = None, + table: str | None = None) -> dict[str, Any]: """模型给的一张表 → 目录项（llm / proposed）。不合规的值整项丢掉，不硬塞：列名对不上表结构的、 表类型和度量类型不在取值里的、业务主键里有不存在的列的、看起来是计算公式的（_formula_like，公式进口径卡）。 该是文字的地方写成了数组、对象（列名、表类型、业务日期的列、度量类型……）同样丢掉，不抛异常：实测见过把列名 - 写成数组的，以前在这里抛 TypeError，整张表记出错。导入表格的源，说明已经覆盖的字段也丢掉。""" + 写成数组的，以前在这里抛 TypeError，整张表记出错。导入表格的源，说明已经覆盖的字段也丢掉。 + + 有效行过滤、去重规则见 _model_facts：source_kind 是数据源类型（按它的方言解析条件，不认识的类型不收有效行过滤）， + table 是表名（条件里的列可以带这张表的表名）。""" columns = {c["name"]: c for c in meta.get("columns") or []} notes: dict[str, Any] = {} table_has_system_note = system and bool((meta.get("comment") or "").strip()) @@ -1738,6 +1816,7 @@ def _from_model(entry: Mapping[str, Any], meta: Mapping[str, Any], *, system: bo if (text := _clean_text(date.get(k))) is not None: value[k] = text notes["business_date"] = make_item(value, "llm") + notes.update(_model_facts(entry, meta, kind=source_kind, table=table)) out_cols: dict[str, dict[str, Any]] = {} listed = entry.get("columns") for col in listed if isinstance(listed, list) else []: @@ -2330,7 +2409,7 @@ __all__ = [ "CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES", "DraftModel", "DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL", "MODEL_BATCH_COLUMNS", "MODEL_BATCH_TABLES", "MODEL_CONCURRENCY", "MODEL_EMPTY_REASON", - "MODEL_FALLBACK_MAX_TOKENS", "MODEL_HALT_ON", "MODEL_HALT_WINDOW", "MODEL_MAX_TOKENS", + "FACT_REASON_NOTE", "MODEL_FALLBACK_MAX_TOKENS", "MODEL_HALT_ON", "MODEL_HALT_WINDOW", "MODEL_MAX_TOKENS", "MODEL_MIN_PART_COLUMNS", "MODEL_NOTICE_OFF", "MODEL_NOTICE_ON", "MODEL_NOTICE_WINDOW", "MODEL_PART_MAX_CALLS", "MODEL_RETRIES", "MODEL_RETRY_DELAYS_S", "MODEL_SHRINK_AFTER", "MODEL_SPLIT_DEPTH", "MODEL_SPLIT_MIN_COLUMNS", "MODEL_TABLE_GIVE_UP_AFTER", "MODEL_TIMEOUT_S", "MergeStats", "NEW_RELATION_PATH", diff --git a/backend/app/data/catalog_draft.py b/backend/app/data/catalog_draft.py index 49e059b1..07e83877 100644 --- a/backend/app/data/catalog_draft.py +++ b/backend/app/data/catalog_draft.py @@ -75,8 +75,15 @@ 一张表的模型部分全部结束。后台起草任务据此落库（data/catalog_jobs.py）。块：这张表实际分成几块来问——一开始是段数， 一段被缩小的上限分开成几块、被拆成两半，块数就多几块；一块问完（写没写出来都算）算结束一块。 -交给模型的只有表结构（列、类型、主键、外键、唯一约束、注释）和使用者补充的背景，不发任何数据值。写给模型的文字都在 -catalog.py 名字带 _SYSTEM / _PROMPT 的常量里；这里的 REASON_* 是给人看的（拼进这张表的 model_error）。 +**防错事实。** 表级的项里模型可以提议有效行过滤（valid_filter）和去重规则（dedup），各附一句依据（写进这一项的 note）， +校验见 catalog._from_model（有效行过滤按方言解析，不过就丢掉）。开了数据剖析的源（catalog_profile.profile_settings）： +目录里记下的码值交给模型做背景（遮罩的列不给），表级那段写完后经剖析的只读守卫数一数有效行过滤排除了多少行 +（catalog_profile.check_valid_filter），排除 0% 或 100% 的不收，收下的 note 写上排除比例；没开剖析、连不上的照收， +note 写没有核对的原因。 + +交给模型的是表结构（列、类型、主键、外键、唯一约束、注释）和使用者补充的背景；开了数据剖析的源另给目录里记下的码值 +（剖析取过的取值和含义，遮罩的列不给），别的数据值一概不发。写给模型的文字都在 catalog.py 名字带 _SYSTEM / _PROMPT +的常量里；这里的 REASON_* 是给人看的（拼进这张表的 model_error）。 """ from __future__ import annotations @@ -749,6 +756,8 @@ class _Table: #: 块进度：实际分成几块、结束了几块 chunks_total: int = 0 chunks_done: int = 0 + #: 有效行过滤核对过了（表级那段写完核对一次） + vetted: bool = False def whole(self, piece: _Piece) -> bool: return piece.table_level and len(piece.columns) == len(self.columns) @@ -863,6 +872,15 @@ class _Run: self.was_halted = False #: 这次起草里结构化输出只给空壳时换纯文本问过没有（止损期间只问这一次） self.text_probed = False + #: 数据源：开了数据剖析的源把码值交给模型、核对有效行过滤的排除比例（遮罩的列都不碰） + from app.data import catalog_profile + from app.data.engine import masked_columns + + self.source = source + self.kind = getattr(source, "kind", None) + options = getattr(source, "options", None) + self.profiling = catalog_profile.profile_settings(options if isinstance(options, dict) else None) + self.masked = {c.lower() for c in masked_columns(options if isinstance(options, dict) else None)} def _halted(self) -> bool: """止损：最近 MODEL_HALT_WINDOW 次调用里有 MODEL_HALT_ON 次以上没写出来。之后的段不再拆小、每块只问一次，直到 @@ -949,15 +967,18 @@ class _Run: """起草一段；它之后的段（宽表的列分段）排到队尾；一张表的段都结束了，这张表的模型部分就结束了。 这一段（原始段：计划里的整表段、表级那段或一个列分段）连同按缩小后的上限分开的各块、拆出来的、补问的、重试的， - 一共最多调用 MODEL_PART_MAX_CALLS 次（各块共用一个 _Budget）。接连几段没写出来的表（_tally），剩下的段跳过不问。 - 宽表写出了一段、后面还有段没结束，就把这张表到这时拿到的内容交给调用方并入目录（part_written）；最后一段结束由 - table_done 收尾。""" + 一共最多调用 MODEL_PART_MAX_CALLS 次（各块共用一个 _Budget）。表级那段写完先核对模型提议的有效行过滤（_vet）， + 再交出去。接连几段没写出来的表（_tally），剩下的段跳过不问。宽表写出了一段、后面还有段没结束，就把这张表到这时 + 拿到的内容交给调用方并入目录（part_written）；最后一段结束由 table_done 收尾。""" asked = [p for p in work.pieces if not self.tables[p.table].abandoned] for piece in work.pieces: if self.tables[piece.table].abandoned: await self._skip_part(piece) if asked: await self._start(asked, _Budget(max(1, int(C.MODEL_PART_MAX_CALLS)))) + for piece in asked: + if piece.table_level: + await self._vet(self.tables[piece.table]) self._tally(asked) for piece in work.pieces: await self._part_done(piece.table) @@ -1408,7 +1429,8 @@ class _Run: gain = _Gain() for entry in by_table.get(piece.table, []): gain.named = st.named = True - notes = _restrict(C._from_model(entry, st.meta, system=self.system), piece) + drafted = C._from_model(entry, st.meta, system=self.system, source_kind=self.kind, table=st.name) + notes = _restrict(drafted, piece) cols = entry.get("columns") if isinstance(entry.get("columns"), list) else [] gain.answered |= {c.get("name") for c in cols if isinstance(c, dict) and c.get("name") in piece.columns} slots = C._slots(notes) @@ -1572,6 +1594,47 @@ class _Run: st = self.tables[name] await self._hook(self.hooks.part_written, name, PartWrite(copy.deepcopy(st.notes), len(st.written_parts))) + # ---------------------------------------------------------------- 防错事实：核对有效行过滤 + + async def _vet(self, st: _Table) -> None: + """表级那段写完，核对模型提议的有效行过滤（只核对一次）。 + + - 目录里人工确认、驳回过的有效行过滤：并入时不会采用这次的提议，不必再查库； + - 没开数据剖析：照收，note 写「没有核对排除比例（数据源没有开数据剖析）」； + - 开了：经剖析的只读守卫数排除比例（catalog_profile.check_valid_filter）。排除 0% 或 100% 的不收（日志记一条， + 不算失败）；收下的 note 在依据后面写上剖析核对的结果，或者没有核对的原因。""" + if st.vetted: + return + st.vetted = True + item = st.notes.get("valid_filter") + if not isinstance(item, dict): + return + held = (st.existing or {}).get("valid_filter") + if isinstance(held, Mapping) and held.get("status") in ("confirmed", "rejected"): + return + from app.data import catalog_profile + + if not self.profiling.enabled: + found = catalog_profile.FilterCheck(True, catalog_profile.FILTER_UNCHECKED.format( + why=catalog_profile.FILTER_NOT_ENABLED)) + else: + try: + found = await catalog_profile.check_valid_filter(self.source, st.name, dict(st.meta), + str(item.get("value") or ""), settings=self.profiling) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 - 核对是附加的：出了意外照收，写明没有核对 + from app.core.errors import describe_exception, first_line + + logger.exception("起草数据目录：核对表 %s 的有效行过滤时出错", st.name) + found = catalog_profile.FilterCheck(True, catalog_profile.FILTER_UNCHECKED.format( + why=f"核对时出错：{first_line(e) or describe_exception(e)}")) + if not found.keep: + logger.info("起草数据目录：表 %s 的有效行过滤 %s 没有收下：%s", st.name, item.get("value"), found.dropped) + st.notes.pop("valid_filter", None) + return + item["note"] = _sentences(item.get("note"), found.note) + @staticmethod async def _hook(fn: Callable[..., Awaitable[None]] | None, *args: Any) -> None: if fn is None: @@ -1650,7 +1713,12 @@ class _Run: for group in meta.get("unique") or []: if isinstance(group, list) and group: lines.append(labels["unique"].format(cols="、".join(group))) - confirmed = self._confirmed_lines(st, piece) + shown = self._confirmed_for(st, piece) + codes = self._codes_lines(st, piece, shown) + if codes: + lines.append(labels["codes"]) + lines += codes + confirmed = C._notes_lines(shown, meta=st.meta, system_notes=False) if confirmed: lines.append(labels["confirmed"]) lines += confirmed @@ -1671,7 +1739,7 @@ class _Run: return line + ("，" + "，".join(marks) if marks else "") @staticmethod - def _confirmed_lines(st: _Table, piece: _Piece) -> list[str]: + def _confirmed_for(st: _Table, piece: _Piece) -> dict[str, Any]: """已确认、有确证的项（以此为准的背景）：整表全给；其余给表级的项和这一段的列，关系只在要写表级的项时给。""" confirmed = C._confirmed_only(st.existing) if not st.whole(piece): @@ -1679,7 +1747,32 @@ class _Run: kept = {key: item for key, item in C._slots(confirmed).items() if key[0] == "t" or (key[0] == "c" and key[1] in wanted) or (key[0] == "r" and piece.table_level)} confirmed = C._assemble(kept) - return C._notes_lines(confirmed, meta=st.meta, system_notes=False) + return confirmed + + def _codes_lines(self, st: _Table, piece: _Piece, shown: Mapping[str, Any]) -> list[str]: + """开了数据剖析的源：目录里记下的码值（剖析取过的取值和已知含义）交给模型，判断有效行过滤、列的含义用。要写表级 + 的项的给全部列的，只写几列的给这几列的；遮罩的列、驳回的码值不给；「已确认的目录项」里已经有的不重复。""" + if not self.profiling.enabled: + return [] + labels = C.CATALOG_DRAFT_PROMPT_LABELS + wanted = set(st.columns) if piece.table_level else set(piece.columns) + known = C.visible_notes(st.existing).get("columns") or {} + given = (shown.get("columns") or {}) if isinstance(shown.get("columns"), Mapping) else {} + out: list[str] = [] + for col in st.columns: + item = (known.get(col) or {}).get("codes") + if col not in wanted or col.lower() in self.masked or not isinstance(item, Mapping): + continue + if "codes" in (given.get(col) or {}) or not isinstance(item.get("value"), Mapping) or not item["value"]: + continue + out.append(labels["code"].format(name=col, codes=C._codes_text(item["value"]))) + return out + + +def _sentences(*parts: Any) -> str: + """几句话连成一段：各句去掉句末的句号，用句号隔开（「依据：……。剖析核对：……」）。""" + texts = [str(p).strip().rstrip("。") for p in parts if isinstance(p, str) and p.strip()] + return "。".join(texts) __all__ = ["BUDGET_WORDS", "Background", "CallReport", "FAILURE_LABEL", "Hooks", "ModelOutcome", "NOTICE", "PartWrite", diff --git a/backend/tests/test_catalog_draft_facts.py b/backend/tests/test_catalog_draft_facts.py index dba6db48..c7c2cabd 100644 --- a/backend/tests/test_catalog_draft_facts.py +++ b/backend/tests/test_catalog_draft_facts.py @@ -154,6 +154,124 @@ def test_the_condition_is_rendered_with_the_columns_quoted_like_the_source(): assert rendered == "`isTest` = 0 AND status <> 'C'" +# -------------------------------------------------------------------------- +# 模型给的两项：校验、依据 +# -------------------------------------------------------------------------- + + +def test_the_model_fields_become_llm_items_with_their_reason(): + entry = {"table": "orders", "label": "订单", + "valid_filter": {"condition": "status NOT IN ('C', 'X')", "reason": "状态注释写明 C 作废、X 测试单"}, + "dedup": {"rule": "同一订单号有多行时只取最后修改的一行", "reason": "订单号可能重复导出"}} + notes = catalog._from_model(entry, META, system=False, source_kind="sqlite", table="orders") + vf, dd = notes["valid_filter"], notes["dedup"] + assert (vf["value"], vf["source"], vf["status"]) == ("status NOT IN ('C', 'X')", "llm", "proposed") + assert vf["note"] == "依据：状态注释写明 C 作废、X 测试单" + assert (dd["value"], dd["source"], dd["status"]) == ("同一订单号有多行时只取最后修改的一行", "llm", "proposed") + assert dd["note"] == "依据：订单号可能重复导出" + assert not catalog.validate_notes(notes) + + +def test_plain_strings_are_accepted_and_bad_filters_dropped(caplog): + entry = {"table": "orders", "label": "订单", "valid_filter": "status <> 'C'", "dedup": "按订单号去重"} + notes = catalog._from_model(entry, META, system=False, source_kind="sqlite", table="orders") + assert notes["valid_filter"]["value"] == "status <> 'C'" and "note" not in notes["valid_filter"] + assert notes["dedup"]["value"] == "按订单号去重" + + with caplog.at_level(logging.INFO, logger="app.data.catalog"): + notes = catalog._from_model({"table": "orders", "label": "订单", + "valid_filter": {"condition": "status IN (SELECT 1)", "reason": "猜的"}}, + META, system=False, source_kind="sqlite", table="orders") + assert "valid_filter" not in notes and notes["label"]["value"] == "订单", "丢掉这一项，别的照收" + assert any("有效行过滤" in r.getMessage() and "orders" in r.getMessage() for r in caplog.records), "日志记一条" + for junk in ({"condition": ""}, {"reason": "只有依据"}, ["status = 1"], 3): + notes = catalog._from_model({"table": "orders", "valid_filter": junk, "dedup": junk}, META, system=False, + source_kind="sqlite", table="orders") + assert "valid_filter" not in notes and "dedup" not in notes, junk + + +def test_the_prompt_asks_for_the_two_facts_only_when_evident(): + system = catalog.CATALOG_DRAFT_SYSTEM + assert "valid_filter" in system and "dedup" in system + assert "拿不准就不写" in system and "reason" in system + assert "UPPER、LOWER、TRIM、COALESCE" in system + props = catalog.CATALOG_DRAFT_PROMPT_SCHEMA["properties"]["tables"]["items"]["properties"] + assert set(props["valid_filter"]["properties"]) == {"condition", "reason"} + assert set(props["dedup"]["properties"]) == {"rule", "reason"} + assert '"valid_filter": {"condition"' in catalog.CATALOG_DRAFT_TEXT_PROMPT + assert '"dedup": {"rule"' in catalog.CATALOG_DRAFT_TEXT_PROMPT + assert "有效行过滤" in catalog.CATALOG_DRAFT_PROMPT_LABELS["only_table"] + + +# -------------------------------------------------------------------------- +# 引擎：没开剖析照收、写明没有核对；开了剖析的源把码值交给模型 +# -------------------------------------------------------------------------- + +REPLY = {"label": "订单", "grain": "每笔订单一行", + "valid_filter": {"condition": "status NOT IN ('C', 'X')", "reason": "状态注释写明 C 作废、X 测试单"}, + "dedup": {"rule": "同一订单号只取最后一行", "reason": "订单号可能重复"}, + "columns": [{"name": "status", "label": "订单状态"}]} + + +async def test_without_profiling_the_filter_is_kept_and_says_it_was_not_checked(): + model = FakeModel({"orders": REPLY}) + got = await D.run_model_draft(model, _plain({"orders": _orders()}), ["orders"]) + o = got["orders"] + assert o.status == "ok" + vf = o.notes["valid_filter"] + assert (vf["source"], vf["status"]) == ("llm", "proposed") + assert vf["note"] == "依据：状态注释写明 C 作废、X 测试单。没有核对排除比例（数据源没有开数据剖析）" + assert o.notes["dedup"]["note"] == "依据：订单号可能重复" + + +async def test_an_invalid_filter_from_the_model_is_dropped_not_failed(): + reply = {**REPLY, "valid_filter": {"condition": "status IN (SELECT code FROM bad_codes)", "reason": "猜的"}} + got = await D.run_model_draft(FakeModel({"orders": reply}), _plain({"orders": _orders()}), ["orders"]) + o = got["orders"] + assert o.status == "ok" and o.error is None, "不算失败" + assert "valid_filter" not in o.notes and o.notes["dedup"]["value"] == "同一订单号只取最后一行" + + +def _codes_existing() -> dict: + return {"orders": {"columns": { + "status": {"codes": make_item({"C": "作废", "X": "", "P": "已支付"}, "profile", "proposed")}, + "order_no": {"codes": make_item({"A1": "甲"}, "profile", "proposed")}, + "amount": {"codes": make_item({"0": "零"}, "profile", "rejected")}, + }}} + + +async def test_codes_go_to_the_model_when_profiling_is_on_but_not_masked_ones(): + on = {PROFILE_OPTION: {"enabled": True}, MASK_COLUMNS_OPTION: ["ORDER_NO"]} + model = FakeModel({"orders": REPLY}) + await D.run_model_draft(model, _plain({"orders": _orders()}, options=on), ["orders"], existing=_codes_existing()) + text = model.calls[0] + head = catalog.CATALOG_DRAFT_PROMPT_LABELS["codes"] + assert head in text + block = text[text.index(head):] + assert "- status：C=作废、P=已支付，X 的含义待填写" in block + assert "A1" not in text, "遮罩的列不给码值" + assert "零" not in text, "驳回的码值不给" + + plain = FakeModel({"orders": REPLY}) + await D.run_model_draft(plain, _plain({"orders": _orders()}), ["orders"], existing=_codes_existing()) + assert head not in plain.calls[0] and "已支付" not in plain.calls[0], "没开剖析不给码值" + + +async def test_codes_reach_the_table_part_of_a_wide_table_and_only_their_column_part(): + cols = [_col(f"c{i:03d}") for i in range(1, 46)] + [_col("state", "TEXT")] + wide = {"qualified": "wide", "schema": None, "is_view": False, "primary_key": ["c001"], "foreign_keys": [], + "comment": None, "columns": cols} + existing = {"wide": {"columns": {"state": {"codes": make_item({"9": "作废"}, "profile", "proposed")}}}} + on = {PROFILE_OPTION: {"enabled": True}} + model = FakeModel({"wide": {"label": "宽表"}}) + await D.run_model_draft(model, _plain({"wide": wide}, options=on), ["wide"], existing=existing) + head = catalog.CATALOG_DRAFT_PROMPT_LABELS["codes"] + with_codes = [requested(c)["wide"] for c in model.calls if "9=作废" in c] + assert ("table", None) in with_codes, "表级那段要提议有效行过滤：给全部码值" + assert all(part == "table" or "state" in (cols or []) for part, cols in with_codes) + assert any(head not in c for c in model.calls), "不含这一列的列分段不给" + + # -------------------------------------------------------------------------- # 剖析核对排除比例 # -------------------------------------------------------------------------- @@ -213,3 +331,55 @@ async def test_unchecked_filters_are_kept_with_the_reason(scenic, tmp_path): gone = await scenic(database=str(tmp_path / "missing" / "nope.db")) found = await catalog_profile.check_valid_filter(gone, "visits", meta_of(gone), "status = 1") assert found.keep and found.note.startswith("没有核对排除比例（查询失败：") + + +async def test_the_engine_checks_the_filter_on_a_profiled_source(scenic, scenic_db): + source = await scenic() + good = {"label": "入园记录", "valid_filter": {"condition": "status = 1", "reason": "0 是作废的检票"}} + got = await D.run_model_draft(FakeModel({"visits": good}), source, ["visits"]) + vf = got["visits"].notes["valid_filter"] + total, excluded = _count(scenic_db, "status = 1") + share = catalog_profile.excluded_share(excluded, total) + assert vf["note"] == f"依据：0 是作废的检票。剖析核对：排除 {share} 的行（共 {total:,} 行）" + + useless = {"label": "入园记录", "valid_filter": {"condition": "status IN (0, 1)", "reason": "猜的"}} + got = await D.run_model_draft(FakeModel({"visits": useless}), source, ["visits"]) + assert got["visits"].status == "ok" and "valid_filter" not in got["visits"].notes, "排除 0% 的不收" + assert got["visits"].notes["label"]["value"] == "入园记录" + + +async def test_a_confirmed_filter_is_not_checked_again(scenic, monkeypatch): + source = await scenic() + called: list[str] = [] + + async def counting(*args, **kw): + called.append("x") + return catalog_profile.FilterCheck(True, "不该用到") + + monkeypatch.setattr(catalog_profile, "check_valid_filter", counting) + existing = {"visits": {"valid_filter": make_item("status = 1", "human", "confirmed")}} + reply = {"label": "入园记录", "valid_filter": {"condition": "status <> 0", "reason": "0 是作废"}} + await D.run_model_draft(FakeModel({"visits": reply}), source, ["visits"], existing=existing) + assert called == [], "人工确认过的有效行过滤，起草的提议并入时不会采用，不必再查库" + + +# -------------------------------------------------------------------------- +# SQL 检查：推断的有效行过滤只是提示，确认后才是警告 +# -------------------------------------------------------------------------- + + +async def test_a_drafted_filter_only_hints_until_someone_confirms_it(scenic): + source = await scenic(options={}) + entry = {"table": "visits", "valid_filter": {"condition": "status = 1", "reason": "0 是作废"}} + meta = source.schema_cache["tables"]["visits"] + notes = catalog._from_model(entry, meta, system=False, source_kind="sqlite", table="visits") + sql = "SELECT COUNT(*) FROM visits" + + def levels(table_notes: dict) -> list[tuple[str, str]]: + checker = sqlcheck.SqlChecker(kind="sqlite", schema_cache=source.schema_cache, notes={"visits": table_notes}) + return [(c.code, c.level) for c in checker.check(sql)] + + assert levels(notes) == [("missing_valid_filter", "info")] + confirmed = catalog.review_item(notes, "valid_filter", "confirm") + assert levels(confirmed) == [("missing_valid_filter", "warning")] + assert levels(catalog.review_item(notes, "valid_filter", "reject")) == [] From 87b6cb4090ff35274c5be8acfd18db9b633358bc Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:35:44 +0800 Subject: [PATCH 10/39] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E9=97=AD=E7=8E=AF=E7=AC=AC=E4=B8=80=E6=AD=A5=EF=BC=88=E5=90=8E?= =?UTF-8?q?=E7=AB=AF=20A=EF=BC=89=EF=BC=9A=E5=AE=BD=E8=A1=A8=E6=94=B6?= =?UTF-8?q?=E5=B0=BE=E9=82=A3=E6=AC=A1=E5=86=99=E4=B8=8D=E8=BF=9B=E5=8E=BB?= =?UTF-8?q?=E6=97=B6=EF=BC=8C=E5=86=99=E8=BF=9B=E7=9B=AE=E5=BD=95=E7=9A=84?= =?UTF-8?q?=E6=AE=B5=E6=95=B0=E7=85=A7=E5=BA=93=E9=87=8C=E9=80=90=E6=AE=B5?= =?UTF-8?q?=E5=86=99=E8=BF=9B=E5=8E=BB=E7=9A=84=E8=AE=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 整张表收尾的写入一直撞上别人刚改过（写入冲突）时，最后一段没进目录：parts_written 不再按引擎写出来的段数虚报，留着逐段写入时记下的数 --- backend/app/data/catalog_jobs.py | 7 +++++-- backend/tests/test_catalog_draft_jobs.py | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/backend/app/data/catalog_jobs.py b/backend/app/data/catalog_jobs.py index 555c5256..a721cf58 100644 --- a/backend/app/data/catalog_jobs.py +++ b/backend/app/data/catalog_jobs.py @@ -693,8 +693,11 @@ class _JobRun: if result.version: values["version"] = result.version if outcome is not None: - values.update(parts_done=outcome.parts_done, parts_written=outcome.parts_written, - chunks_total=outcome.chunks_total, chunks_done=outcome.chunks_done) + values.update(parts_done=outcome.parts_done, chunks_total=outcome.chunks_total, + chunks_done=outcome.chunks_done) + if not result.error: + # 收尾写进去了：写出内容的段都在目录里了。没写进去（一直撞上别人刚改过）的照库里逐段写进去的段数 + values["parts_written"] = outcome.parts_written await session.execute(self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN)) .values(**values)) await session.commit() diff --git a/backend/tests/test_catalog_draft_jobs.py b/backend/tests/test_catalog_draft_jobs.py index d69704b7..29bbee8d 100644 --- a/backend/tests/test_catalog_draft_jobs.py +++ b/backend/tests/test_catalog_draft_jobs.py @@ -1012,3 +1012,26 @@ async def test_a_sync_draft_and_draft_jobs_exclude_each_other(client, monkeypatc after = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]}) assert after.status_code == 202, after.text await finished(after.json()["job"]["id"]) + + +async def test_a_wide_table_whose_closing_write_fails_keeps_the_count_of_parts_on_disk(client, monkeypatch): + """整张表收尾那一次写不进去（一直撞上别人刚改过）：已经逐段写进去的段留着，写进目录的段数照库里的记，不按引擎写出来 + 的段数虚报。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) # 一段一段来：最后一段只在收尾那次写 + sid = await _source(_wide()) + use(monkeypatch, FakeModel(_wide_reply())) + real = catalog.write_draft + + async def conflicted(session, source, table, modeled=None, *, at, actor): + result = await real(session, source, table, None, at=at, actor=actor) + result.error, result.status = catalog.WRITE_CONFLICT_ERROR, "failed" + result.added = result.updated = result.removed = 0 + return result + + monkeypatch.setattr(catalog, "write_draft", conflicted) + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["wide"], "use_model": True})).json()["job"] + await finished(job["id"]) + task = await _task(client, sid, job["id"], "wide") + assert task["status"] == "failed" and task["error"] == catalog.WRITE_CONFLICT_ERROR + assert task["parts_written"] == 3, "最后一段没写进去" + assert len((await client.get(f"{_base(sid)}/wide")).json()["notes"]["columns"]) == 64 From a9ffbcfa86f1ea1e4db6ca4e92dbc76b9b5ea0cf Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:44:14 +0800 Subject: [PATCH 11/39] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E9=97=AD=E7=8E=AF=EF=BC=88=E5=89=8D=E7=AB=AF=EF=BC=89=EF=BC=9A?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E3=80=81=E6=8E=A5=E5=8F=A3=E3=80=81=E6=96=87?= =?UTF-8?q?=E6=A1=88=E5=92=8C=E7=BA=AF=E5=87=BD=E6=95=B0=E2=80=94=E2=80=94?= =?UTF-8?q?=E8=B5=B7=E8=8D=89=E4=BB=BB=E5=8A=A1=E7=9A=84=E9=80=90=E6=AE=B5?= =?UTF-8?q?=E8=90=BD=E7=9B=98=E3=80=81=E5=9D=97=E8=BF=9B=E5=BA=A6=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=E5=92=8C=E6=B8=85=E5=8D=95=E7=9A=84=20blank=5Fcolumns?= =?UTF-8?q?=5Ffrom=EF=BC=9B=E5=AE=A1=E9=98=85=E9=98=9F=E5=88=97=E3=80=81?= =?UTF-8?q?=E6=88=90=E6=89=B9=E5=AE=A1=E9=98=85=E3=80=81=E5=AE=A1=E9=98=85?= =?UTF-8?q?=E8=BF=9B=E5=BA=A6=E3=80=81=E7=94=A8=E8=BF=87=E8=BF=99=E5=BC=A0?= =?UTF-8?q?=E8=A1=A8=E7=9A=84=E8=BF=90=E8=A1=8C=EF=BC=9B=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E5=AF=BC=E5=85=A5=EF=BC=88=E4=B8=8B=E8=BD=BD=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E3=80=81=E9=A2=84=E8=A7=88=E3=80=81=E5=AF=BC=E5=85=A5=EF=BC=89?= =?UTF-8?q?=EF=BC=9B=E8=A1=A8=E5=8A=A9=E6=89=8B=E7=9A=84=E4=BA=8B=E4=BB=B6?= =?UTF-8?q?=E6=B5=81=EF=BC=88=E6=8C=89=E8=A1=8C=E8=AF=BB=20data:=20?= =?UTF-8?q?=EF=BC=8C=E5=BC=80=E6=B5=81=E5=89=8D=E8=A2=AB=E6=8B=92=E5=B8=A6?= =?UTF-8?q?=E7=8A=B6=E6=80=81=E7=A0=81=E5=92=8C=E6=9C=BA=E8=AF=BB=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E8=83=BD=E5=81=9C=E6=AD=A2=EF=BC=89=EF=BC=9B=E8=A1=A8?= =?UTF-8?q?=E5=8A=A9=E6=89=8B=E7=9A=84=E5=AF=B9=E8=AF=9D=E6=80=8E=E4=B9=88?= =?UTF-8?q?=E6=88=AA=EF=BC=88=E6=9C=80=E5=A4=9A=2040=20=E6=9D=A1=E3=80=81?= =?UTF-8?q?=E6=AF=8F=E6=9D=A1=204000=20=E5=AD=97=E3=80=81=E4=BB=8E?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E7=9A=84=E8=AF=9D=E5=BC=80=E5=A7=8B=EF=BC=89?= =?UTF-8?q?=E3=80=81=E5=BE=85=E4=BF=9D=E5=AD=98=E7=9A=84=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E6=80=8E=E4=B9=88=E7=B4=AF=E7=A7=AF=EF=BC=88=E5=90=8C=E4=B8=80?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E4=BB=A5=E6=9C=80=E5=90=8E=E4=B8=80=E6=AC=A1?= =?UTF-8?q?=E6=8F=90=E6=A1=88=E4=B8=BA=E5=87=86=E3=80=81=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E5=B8=A6=E6=9C=80=E6=97=A9=E9=82=A3=E6=9D=A1=E6=8F=90=E6=A1=88?= =?UTF-8?q?=E7=9A=84=E7=89=88=E6=9C=AC=E3=80=81409=20=E5=90=8E=E6=8C=89?= =?UTF-8?q?=E9=A2=84=E8=A7=88=E9=87=8D=E7=AE=97=EF=BC=89=EF=BC=9B=E5=AE=A1?= =?UTF-8?q?=E9=98=85=E8=A7=86=E5=9B=BE=E7=9A=84=E5=9C=B0=E5=9D=80=E5=8F=82?= =?UTF-8?q?=E6=95=B0=E3=80=81=E6=88=90=E6=89=B9=E7=BB=93=E6=9E=9C=E5=92=8C?= =?UTF-8?q?=E8=A1=A8=E8=AF=A6=E6=83=85=E6=94=B9=E8=BF=87=E4=B9=8B=E5=90=8E?= =?UTF-8?q?=E7=9A=84=E7=8A=B6=E6=80=81=E5=90=8C=E6=AD=A5=EF=BC=9B=E6=96=87?= =?UTF-8?q?=E6=A1=88=E8=BF=9B=20lib/terms.ts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/api/client.ts | 130 +++++++++- frontend/src/lib/terms.ts | 304 +++++++++++++++++++++++- frontend/src/pages/catalog/assistant.ts | 213 +++++++++++++++++ frontend/src/pages/catalog/review.ts | 132 ++++++++++ frontend/src/run/catalogPatch.ts | 7 +- frontend/src/types.ts | 182 ++++++++++++++ 6 files changed, 959 insertions(+), 9 deletions(-) create mode 100644 frontend/src/pages/catalog/assistant.ts create mode 100644 frontend/src/pages/catalog/review.ts diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index eb0f3abf..6827f719 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,6 +1,8 @@ import type { - ActivateSnapshotBody, ActivateSnapshotOut, AiPreview, CatalogDetail, CatalogDraftJob, CatalogImpact, CatalogList, CatalogNotes, - CatalogPatchPreview, CatalogPatchSubmit, CatalogProfileOut, CatalogReviewAction, CommitOut, CurrentRecipe, EditPreview, + ActivateSnapshotBody, ActivateSnapshotOut, AiPreview, CatalogAssistantEvent, CatalogAssistantRequest, CatalogDetail, CatalogDraftJob, + CatalogImpact, CatalogImportReport, CatalogList, CatalogNotes, CatalogPatchPreview, CatalogPatchSubmit, CatalogProfileOut, + CatalogReviewAction, CatalogReviewBatchItem, CatalogReviewBatchResult, CatalogReviewKind, CatalogReviewProgress, CatalogReviewQueue, + CatalogTableUsage, CommitOut, CurrentRecipe, EditPreview, EditRequest, ImportRecord, ManifestOut, PurgeRawBody, PurgeRawOut, QuestionAnswer, Recipe, RedraftRulesOut, RemovePeriodBody, RemovePeriodOut, RevokeAcceptanceBody, RevokeAcceptanceOut, SnapshotOut, Staging, @@ -15,7 +17,7 @@ import { ApiError, BASE, NETWORK_MESSAGE, actorHeader, codeOf, del, describeDetail, failure, get, isGatewayFailure, parseBody, patch, post, put, qs, report, request, upload, type RequestOptions, type UploadOptions, } from './http' -import { bundle } from './bundle' +import { bundle, dispositionName } from './bundle' import { compare } from './compare' import { inquiry } from './inquiry' import { schedules } from './schedules' @@ -451,6 +453,51 @@ export const api = { profile: (sourceId: string, body: { tables?: string[] }, opts?: RequestOptions) => request(`/datasources/${encodeURIComponent(sourceId)}/catalog/profile`, { method: 'POST', body: JSON.stringify(body), ...opts }), + // ---- 审阅队列（只列推断项，按表的使用次数、这一项对数字的影响和来源排好） + /** 一页待确认项。kind：全部 / 影响数字的项 / 名称和含义；table 只看这一张表 */ + reviewQueue: (sourceId: string, params: { limit?: number; offset?: number; kind?: CatalogReviewKind; table?: string | null }, + opts?: RequestOptions) => + get(`${catalogBase(sourceId)}/review-queue${qs({ limit: 50, offset: 0, kind: 'all', ...params })}`, opts), + /** + * 成批确认、驳回、撤销审阅（最多 200 条）：逐表按 if_version 乐观锁，一张表冲突不影响别的表，同一张表的几条一次写入。 + * 逐条回结果：成功的带这张表写入后的版本，冲突、不合法的带原因和机读码 + */ + reviewBatch: (sourceId: string, items: CatalogReviewBatchItem[]) => + post<{ results: CatalogReviewBatchResult[] }>(`${catalogBase(sourceId)}/review/batch`, { items }), + /** 审阅进度：使用次数前 top_n 张表、全部表、影响数字的项 */ + reviewProgress: (sourceId: string, opts?: RequestOptions) => + get(`${catalogBase(sourceId)}/review-progress`, opts), + /** 用过这张表的运行，新的在前 */ + usage: (sourceId: string, table: string, limit = 20, opts?: RequestOptions) => + get(`${catalogBase(sourceId)}/${encodeURIComponent(table)}/usage${qs({ limit })}`, opts), + // ---- 导出导入 + /** + * 导出这个数据源的数据目录（GET /catalog/export，附件）：回来的是文件本身和服务端给的文件名，没给就是 null，由调用方 + * 按数据源名补。原样存下，不解析：导入时交回的就是这份 + */ + exportFile: async (sourceId: string): Promise<{ blob: Blob; name: string | null }> => { + const path = `${catalogBase(sourceId)}/export` + let res: Response + try { + res = await fetch(BASE + path, { headers: actorHeader() }) + } catch (e: any) { + const err = new ApiError(0, NETWORK_MESSAGE, { kind: 'network', raw: `${e?.name ?? 'Error'}: ${e?.message ?? e}` }) + report(false, err) + throw err + } + if (!res.ok) throw failure(res.status, res.statusText, await res.text().catch(() => ''), path) + report(true) + return { blob: await res.blob(), name: dispositionName(res.headers.get('Content-Disposition')) } + }, + /** 导入的预览（只读）：哪些表对上了，各有几项新增、更新、保留、冲突、跳过，冲突逐条列出 */ + importPreview: (sourceId: string, body: { content: unknown; include_guidance: boolean }) => + post(`${catalogBase(sourceId)}/import/preview`, body), + /** + * 导入：规则同预览，逐表乐观锁，一张表失败不影响别的。有排队、进行中的起草任务时 409（catalog_draft_busy）； + * 文件格式不对 422（detail 是一句中文） + */ + importCatalog: (sourceId: string, body: { content: unknown; include_guidance: boolean }) => + post(`${catalogBase(sourceId)}/import`, body), }, runs: { @@ -867,6 +914,83 @@ export function streamCopilot( return () => controller.abort() } +/** 表助手一轮结束时的情况：error 是给人看的那句话（开流前被拒时带状态码和机读码）；aborted 是点了停止 */ +export interface CatalogAssistantEnd { + error?: string + status?: number + code?: string + aborted?: boolean +} + +/** + * 表助手的一轮（POST /datasources/{id}/catalog/{table}/assistant）：回答是事件流，每个事件一行 data: ，到一个回调 + * 一个（text 增量、query 只读查询、proposal 修改提案、done、error）。返回取消函数，就是「停止回答」。 + * + * 开流之前就被拒（表不在 404、起草模型用不了 409 catalog_draft_model_unavailable、请求太长 422）时 onEnd 带服务端那句话、 + * 状态码和机读码；流到一半断了和一开始就连不上一样，说无法连接服务。认不出的行（心跳、半截）跳过 + */ +export function streamCatalogAssistant( + sourceId: string, + table: string, + body: CatalogAssistantRequest, + onEvent: (event: CatalogAssistantEvent) => void, + onEnd: (end: CatalogAssistantEnd) => void, +): () => void { + const controller = new AbortController() + const path = `${catalogBase(sourceId)}/${encodeURIComponent(table)}/assistant` + const emit = (line: string) => { + const text = line.replace(/\r$/, '') + if (!text.startsWith('data:')) return + const payload = text.slice(5).trim() + if (!payload) return + try { + const event = parseBody(payload) + if (event && typeof event === 'object' && typeof event.type === 'string') onEvent(event as CatalogAssistantEvent) + } catch { /* 认不出的一行，跳过 */ } + } + void (async () => { + try { + const res = await fetch(BASE + path, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'text/event-stream', ...actorHeader() }, + body: JSON.stringify(body), + signal: controller.signal, + }) + if (!res.ok || !res.body) { + const err = failure(res.status, res.statusText, await res.text().catch(() => ''), path) + onEnd(err.kind === 'network' ? { error: NETWORK_MESSAGE } : { error: err.message, status: res.status, code: err.code }) + return + } + report(true) + const reader = res.body.getReader() + const decoder = new TextDecoder() + let buffer = '' + for (;;) { + const { done, value } = await reader.read() + if (done) break + buffer += decoder.decode(value, { stream: true }) + let idx: number + while ((idx = buffer.indexOf('\n')) >= 0) { + emit(buffer.slice(0, idx)) + buffer = buffer.slice(idx + 1) + } + } + buffer += decoder.decode() + if (buffer) emit(buffer) + onEnd({}) + } catch (e: any) { + if (e?.name === 'AbortError') { onEnd({ aborted: true }); return } + if (e instanceof TypeError) { + report(false, new ApiError(0, NETWORK_MESSAGE, { kind: 'network', raw: String(e) })) + onEnd({ error: NETWORK_MESSAGE }) + return + } + onEnd({ error: e?.message ?? NETWORK_MESSAGE }) + } + })() + return () => controller.abort() +} + /** * 订阅一次运行的事件流。 * diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index 663b6329..148bc7bb 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -2445,14 +2445,24 @@ export const CATALOG_DRAFT_TEXT = { queued: '排队中，等待开始…', running: '正在起草…', // 服务端点了停止就取消进行中的模型调用（不等它答完，一次调用可能要几分钟），只等手上正在写入目录的那张表写完 - stopping: '正在停止：取消进行中的模型调用，正在写入目录的表写完后停止…', + // 宽表每写完一段就写进目录：停下来时已写进的段保留，只停掉进行中的调用 + stopping: '正在停止：取消进行中的模型调用，已写进目录的段保留…', progress: (done: number, total: number) => `已完成 ${formatNumber(done)} / ${formatNumber(total)} 张表`, /** 结束以后：全部写出来的表有几张（停止、失败、部分完成的不算） */ progressDone: (ok: number, total: number) => `写完 ${formatNumber(ok)} / ${formatNumber(total)} 张表`, parts: (done: number, total: number) => `${formatNumber(done)} / ${formatNumber(total)} 段`, - partsHint: '超过 40 列的表拆成表级一段和若干列分段，逐段起草', + partsHint: '超过 40 列的表拆成表级一段和若干列分段，逐段起草，每写完一段就写进目录', + /** 段被缩小、拆半后实际分成的块（比段多时才写） */ + chunks: (done: number, total: number) => `${formatNumber(done)} / ${formatNumber(total)} 块`, + chunksHint: (total: number) => `模型用完额度或超时后，段被缩小、拆半：这张表实际分成 ${formatNumber(total)} 块起草`, + /** 已经写进目录的段（停止、服务重启时保留） */ + partsWritten: (n: number) => `已写进目录 ${formatNumber(n)} 段`, + partsWrittenHint: '每写完一段就写进目录，停止或服务重启时已写进的段保留', + /** 停下来的宽表：服务端的原因里没写时，界面补这一句 */ + partsWrittenStopped: (n: number, rest: number) => (rest > 0 + ? `已写进目录 ${formatNumber(n)} 段，剩下 ${formatNumber(rest)} 段没起草` : `已写进目录 ${formatNumber(n)} 段`), hide: '在后台继续', - hideHint: '关闭对话框不会停止起草，进度显示在目录页上方；每张表起草完就写进目录', + hideHint: '关闭对话框不会停止起草，进度显示在目录页上方；每写完一段就写进目录', stop: '停止', busy: '这个数据源已有进行中的起草任务，下面显示的是它的进度', modelUsed: (model: string) => `模型：${model}`, @@ -2490,8 +2500,11 @@ export const CATALOG_DRAFT_TEXT = { cancelled: '已停止', interrupted: '服务重启，起草中断', } as Record, - interrupted: '服务重启时这次起草还没有完成，没写完的表未起草。已写进目录的内容保留；可以只重试这些表，接着起草', - cancelled: '已写进目录的内容保留，没写完的表未起草', + interrupted: '服务重启时这次起草还没有完成，没写完的部分未起草。已写进目录的内容保留（每写完一段就写进目录）；可以只重试这些表，接着起草', + cancelled: '已写进目录的内容保留（每写完一段就写进目录），没写完的部分未起草', + /** 停下来时有表已经写进了一部分段 */ + cancelledParts: (tables: number, parts: number) => + `其中 ${formatNumber(tables)} 张表已写进目录 ${formatNumber(parts)} 段；只重试这些表时接着起草剩下的段，已写进的不会丢`, // 任务整体出错的原因（job.error）照服务端的原话写在标题「起草失败」下面，不再加前缀：服务端的原因有的已经是整句 // （「数据源不存在……」），有的自带「起草任务出错：」开头。没结束的表记失败、原因同任务的，逐表不再重复 /** 没写全的表：失败、部分完成、已停止各几张；跳过的（表结构里没有）另说 */ @@ -2539,6 +2552,8 @@ export const CATALOG_DRAFT_TEXT = { /** 上次起草有留空的列：清单上是轻提示（不是失败标记），表详情顶部同一句 */ lastBlank: (n: number) => `上次起草有 ${formatNumber(n)} 列模型拿不准、留空了`, lastBlankAdvice: '可以在起草要求里补充缩写和术语的含义，再重新起草这张表，已有的内容不会丢', + /** 最近一次起草停止了、没有新写入：留空的列是再之前那次写完的起草记下的 */ + lastBlankEarlier: '最近一次起草已停止，留空的列按再之前那次起草记下的', draftGuidance: '补充起草要求', draftGuidanceHint: '打开起草设置，范围是这张表：先在起草要求里补充缩写和术语的含义，再开始起草', redraft: '重新起草这张表', @@ -2953,6 +2968,285 @@ export const CATALOG_UI_TEXT = { cardinalityCheckedShort: '已核实', cardinalityCheckedHint: '基数由数据剖析用数据核实（子表一侧确有重复或确实唯一），一对多关联后重复计算的检查据此报错误', emptyTables: (n: number) => `挑表时跳过 ${formatNumber(n)} 张空表`, + // ---- 表详情里有效记录条件、去重规则的说明（模型起草时附的依据、数据剖析核对的排除比例） + fieldNote: '说明', +} + +// =========================================================================== +// 表助手（表详情里「和助手一起完善」）：和助手对话完善一张表的数据目录。 +// +// 术语： +// - 面板叫「表助手」，入口写「和助手一起完善」。助手只改数据目录、不改业务库；只记数据事实，不写公式。 +// - 助手提出的修改叫「修改提案」，每一项写「改前 / 改后 / 依据」，可以逐项「去掉」「加回」。 +// - 「待保存的修改」：跨多轮累积，同一项以最后一次提案为准；「保存 N 项修改」一次写入，改动的项记为人工填写、已确认。 +// - 只读查询只在开了数据剖析的源上有：写「查询了数据」，SQL 和前几行可以展开；遮罩列的值写「已遮罩」（服务端给的）。 +// - 对话不保存在服务端：关闭表助手、换表、离开页面后就没有了。 +// =========================================================================== + +export const CATALOG_ASSISTANT_TEXT = { + open: '和助手一起完善', + openHint: '和表助手对话完善这张表的数据目录：说出业务事实，助手整理成修改提案，确认后再保存', + openMissing: '表结构中已没有这张表，不能和助手一起完善', + title: '表助手', + subtitle: (table: string) => `完善「${table}」的数据目录`, + close: '关闭表助手', + closeTitle: '关闭表助手？', + closeConsequences: ['待保存的修改和这段对话都会丢失'], + closeConfirm: '放弃并关闭', + leaveTitle: '表助手有未保存的修改', + leaveConsequences: ['离开后待保存的修改和这段对话都会丢失'], + leaveConfirm: '放弃并离开', + noProfile: '这个数据源没开数据剖析，助手只能根据表结构和目录推断，不能查数据', + profileOn: '这个数据源开了数据剖析：助手可以发少量只读查询核对数据，遮罩列不取值', + notStored: '对话不保存在服务端，关闭表助手或离开页面后就没有了', + greeting: '说出你知道的业务事实（某一列的含义、码值、有效记录条件……），助手整理成修改提案，你确认后才写进数据目录。助手只记数据事实，不写计算公式。', + /** 上次起草有留空的列：开场引导先问这几列 */ + blankLead: (n: number, cols: string[]) => + `上次起草有 ${formatNumber(n)} 列模型拿不准、留空了${cols.length ? `，例如 ${cols.join('、')}` : ''}。可以先从这几列开始。`, + suggestLabel: '可以这样开始', + suggestBlank: (n: number) => `补上留空的 ${formatNumber(n)} 列`, + suggestBlankText: (cols: string[]) => `请帮我补上这几列的中文名和含义：${cols.join('、')}。拿不准的地方先问我。`, + suggestFacts: '核对有效记录条件和去重规则', + suggestFactsText: '这张表的有效记录条件和去重规则应该怎么写？拿不准的地方先问我。', + suggestPending: '看看哪些推断可以确认', + suggestPendingText: '这张表还没确认的推断项里，哪些可以直接确认、哪些需要我补充？', + you: '你', + assistant: '助手', + logLabel: '和表助手的对话', + inputLabel: '给表助手的消息', + inputPlaceholder: '例如：status 为 9 表示作废，统计时要排除', + inputHint: 'Enter 发送，Shift+Enter 换行', + count: (n: number, max: number) => `${formatNumber(n)} / ${formatNumber(max)} 字`, + tooLong: (max: number) => `一条消息不超过 ${formatNumber(max)} 字`, + send: '发送', + stop: '停止回答', + answering: '助手正在回答…', + answered: '助手已回答', + stopped: '已停止回答', + interrupted: '连接中断，这一轮没有答完', + failed: (reason: string) => `这一轮没有完成：${reason}`, + usage: (input: number, output: number) => + `输入 ${formatTokens(input, { compact: true })}、输出 ${formatTokens(output, { compact: true })}`, + queries: (n: number) => `查询 ${formatNumber(n)} 次`, + // ---- 只读查询 + query: (rows: number) => `查询了数据（${formatNumber(rows)} 行）`, + queryFailed: '查询没有成功', + queryRows: (shown: number, total: number) => (total > shown + ? `共 ${formatNumber(total)} 行，显示前 ${formatNumber(shown)} 行` : `共 ${formatNumber(total)} 行`), + queryEmpty: '没有返回行', + // ---- 修改提案 + proposal: (n: number) => `修改提案（${formatNumber(n)} 项）`, + before: '改前', + after: '改后', + reason: '依据', + drop: '去掉', + dropLabel: (where: string) => `去掉：${where}`, + restore: '加回', + restoreLabel: (where: string) => `加回：${where}`, + state: { + pending: '待保存', + replaced: '已改用另一条提案的值', + dropped: '已去掉', + saved: '已保存', + } as Record<'pending' | 'replaced' | 'dropped' | 'saved', string>, + stateConfirm: '值不变，保存即确认', + // ---- 待保存的修改 + pending: (n: number) => `待保存的修改（${formatNumber(n)} 项）`, + pendingEmpty: '还没有待保存的修改', + pendingHint: '跨多轮累积，同一项以最后一次提案为准。保存后改动的项记为人工填写、已确认', + pendingShow: '展开待保存的修改', + pendingHide: '收起待保存的修改', + save: (n: number) => `保存 ${formatNumber(n)} 项修改`, + saving: '正在保存…', + clear: '全部去掉', + clearTitle: (n: number) => `去掉全部 ${formatNumber(n)} 项待保存的修改？`, + clearConfirm: '全部去掉', + tooMany: (n: number, max: number) => `有 ${formatNumber(n)} 项，一次最多保存 ${formatNumber(max)} 项，请先去掉一些`, + editing: '正在编辑这张表，保存或取消后再保存助手的修改', + saved: (n: number, version: number) => `已保存 ${formatNumber(n)} 项修改（第 ${formatNumber(version)} 版）`, + savedToast: (n: number, table: string) => `已保存 ${formatNumber(n)} 项修改到「${table}」的数据目录`, + conflictTitle: '这张表刚被修改过', + conflictBody: '已按最新内容重算改前和改后，请核对后再保存。', + conflictFailed: '无法按最新内容重算改前和改后，请稍后重试', + problems: '以下几项已无法保存，已从待保存的修改中去掉：', +} + +// =========================================================================== +// 审阅队列（数据目录页的「审阅队列」视图）：跨表列出推断项，按表的使用次数、这一项对数字的影响和来源排好， +// 可以成批确认、驳回、撤销审阅。 +// +// 术语： +// - 「影响数字的项」：有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型——写错了查出来的数就不对，排在最前； +// 「名称和含义」：中文名、含义、单位、码值、粒度、表类型。 +// - 审阅过的项原地标出结果（已确认、已驳回、未生效），刷新队列后才移除；「撤销审阅」回到起草时的状态（推断）。 +// - 「常用 20 张表」：使用次数最多的 20 张。进度写「已确认 42%」。 +// =========================================================================== + +export const CATALOG_REVIEW_TEXT = { + viewLabel: '数据目录的视图', + view: { list: '表清单', review: '审阅队列' } as Record<'list' | 'review', string>, + viewHint: { + list: '逐表查看和编辑数据目录', + review: '跨表列出待确认的推断项，按使用次数和对数字的影响排好，可以成批确认、驳回', + } as Record<'list' | 'review', string>, + queueLabel: '待确认的推断项', + kindLabel: '按项筛选', + kind: { all: '全部', facts: '影响数字的项', names: '名称和含义' } as Record<'all' | 'facts' | 'names', string>, + kindHint: { + all: '全部待确认的推断项', + facts: '有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型：写错了查出来的数就不对', + names: '中文名、含义、单位、码值、粒度、表类型', + } as Record<'all' | 'facts' | 'names', string>, + tableLabel: '按表筛选', + tableAll: '全部表', + tableOption: (label: string | null, table: string, n: number) => `${label ? `${label}（${table}）` : table} · ${formatNumber(n)} 项`, + total: (n: number) => `${formatNumber(n)} 项待确认`, + shown: (shown: number, total: number) => `显示 ${formatNumber(shown)} / ${formatNumber(total)} 项`, + empty: '没有待确认的推断项', + emptyBody: '推断的项都审阅过了。之后起草出的新推断会出现在这里。', + emptyFiltered: '没有符合条件的待确认项', + emptyFilteredBody: '请换个筛选条件。', + loadFailed: '无法读取审阅队列', + retry: '重试', + refresh: '刷新队列', + refreshHint: '审阅过的项刷新后从队列中移除', + more: (n: number) => `再载入 ${formatNumber(n)} 项`, + loading: '正在载入…', + selectAll: '全选当前列表', + select: (where: string, table: string) => `选择：${table} ${where}`, + selected: (n: number) => `已选 ${formatNumber(n)} 项`, + clearSelection: '取消选择', + usage: (n: number) => `使用 ${formatNumber(n)} 次`, + usageHint: '所在表被运行查询过的次数', + open: (where: string, table: string) => `打开「${table}」，查看${where}`, + reasonsLabel: '排在这里的原因', + // ---- 成批审阅 + confirm: (n: number) => `确认（${formatNumber(n)}）`, + reject: (n: number) => `驳回（${formatNumber(n)}）`, + reset: (n: number) => `撤销审阅（${formatNumber(n)}）`, + resetHint: '把这一页上审阅过的项回到起草时的状态（推断）', + confirmTitle: (n: number) => `确认 ${formatNumber(n)} 项推断？`, + confirmConsequences: ['确认后这些项参与 SQL 检查', '来源保持不变，状态改为已确认', '之后仍可撤销审阅'], + confirmAction: (n: number) => `确认 ${formatNumber(n)} 项`, + rejectTitle: (n: number) => `驳回 ${formatNumber(n)} 项推断？`, + rejectConsequences: ['驳回后不提供给助手、不参与 SQL 检查', '之后起草也不会再提出', '之后仍可撤销审阅'], + rejectAction: (n: number) => `驳回 ${formatNumber(n)} 项`, + tooMany: (max: number) => `一次最多审阅 ${formatNumber(max)} 项，请少选一些`, + working: '正在提交…', + done: { confirm: '已确认', reject: '已驳回', reset: '已撤销审阅' } as Record<'confirm' | 'reject' | 'reset', string>, + conflict: '未生效：这张表刚被修改过，请刷新队列后再审阅', + failed: '未生效', + /** 一批的结果：成功几项、冲突几项、别的原因没成功几项 */ + summary: (action: 'confirm' | 'reject' | 'reset', ok: number, conflicts: number, failed: number) => { + const head = `${{ confirm: '已确认', reject: '已驳回', reset: '已撤销审阅' }[action]} ${formatNumber(ok)} 项` + const bad = [ + conflicts ? `${formatNumber(conflicts)} 项未生效（所在表刚被修改过）` : '', + failed ? `${formatNumber(failed)} 项未生效` : '', + ].filter(Boolean) + return bad.length ? `${head}；${bad.join('、')}，请刷新队列后再审阅` : head + }, + failedAll: (reason: string) => `未能提交：${reason}`, + // ---- 进度 + progressTitle: '审阅进度', + progressTop: (n: number) => `常用 ${formatNumber(n)} 张表`, + progressTopHint: (n: number) => `使用次数最多的 ${formatNumber(n)} 张表里的项`, + progressAll: '全部表', + progressFacts: '影响数字的项', + progressFactsHint: '有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型', + confirmedPct: (pct: number | null) => (pct == null ? '还没有目录项' : `已确认 ${formatNumber(pct)}%`), + itemsOf: (confirmed: number, total: number) => `${formatNumber(confirmed)} / ${formatNumber(total)} 项`, + allDetail: (proposed: number, rejected: number) => `推断 ${formatNumber(proposed)} 项、已驳回 ${formatNumber(rejected)} 项`, + factsDetail: (proposed: number, confirmed: number) => `待确认 ${formatNumber(proposed)} 项、已确认 ${formatNumber(confirmed)} 项`, + progressFailed: '无法读取审阅进度', + // ---- 没选中项时的右栏 + overviewTitle: '从左侧队列挑一项开始审阅', + overviewBody: '队列按所在表的使用次数、这一项对数字的影响和来源排序：常用表里影响数字的推断排在最前，模型起草、命名推断的比数据库注释的靠前。点开一项可以看它所在表的详情和用过这张表的运行。', + // ---- 从队列打开的项 + focusMark: '从审阅队列打开', + // ---- 表清单视图概览里的入口 + openQueue: '打开审阅队列', + openQueueHint: '跨表按使用次数和对数字的影响排好，可以成批确认、驳回', +} + +/** 表详情里「用过这张表的运行」（GET /catalog/{table}/usage） */ +export const CATALOG_USAGE_TEXT = { + title: '用过这张表的运行', + hint: '新的在前', + show: '查看用过这张表的运行', + hide: '收起', + none: '还没有运行查询过这张表', + count: (shown: number, total: number) => (total > shown + ? `最近 ${formatNumber(shown)} 次，共 ${formatNumber(total)} 次` : `共 ${formatNumber(total)} 次`), + error: '无法读取用过这张表的运行', + retry: '重试', + unnamed: '未保存的工作流', + open: (name: string) => `打开运行记录：${name}`, +} + +// =========================================================================== +// 导出导入数据目录（数据目录页的「导出目录」「导入目录」）：本机整理好的目录带到另一个实例。 +// +// 术语： +// - 导出的是一个 .json 文件，含全部状态的项和起草要求。导入先「预览」（只读），确认后才写入。 +// - 预览逐表写「新增 / 更新 / 保留 / 冲突 / 跳过」：冲突是这里已确认、已驳回的项和文件里不一样，以这里为准； +// 跳过是表结构里没有的列、两端对不上的关联关系。 +// =========================================================================== + +export const CATALOG_TRANSFER_TEXT = { + export: '导出目录', + exportHint: '把这个数据源的数据目录导出成文件（含全部状态的项和起草要求），可以导入到另一个实例', + exporting: '正在导出…', + exported: (name: string) => `已导出「${name}」`, + exportFailed: (reason: string) => `未能导出数据目录：${reason}`, + fileName: (source: string) => `${source}-数据目录.json`, + import: '导入目录', + importHint: '从导出的文件导入数据目录：先预览，确认后再写入', + title: '导入数据目录', + chooseLabel: '导出的数据目录文件', + chooseHint: '「导出目录」得到的 .json 文件', + choose: '选择文件', + chosen: (name: string) => `已选择：${name}`, + includeGuidance: '同时导入起草要求', + includeGuidanceHint: '这个数据源还没有起草要求时才写入，已有的不覆盖', + rulesTitle: '导入的规矩', + rules: [ + '表按名字对上：不分大小写；带 schema 前缀的对不上时去掉前缀再对', + '文件中已确认、已驳回的项照原样带入，来源和状态不变', + '这里已确认、已驳回的项和文件中的不一样时以这里为准，记为冲突', + '文件中推断、已验证的项只填补这里的空缺，不覆盖已有的', + '表结构里没有的列、两端对不上的关联关系跳过，写明原因', + ], + preview: '预览', + previewing: '正在预览…', + notJson: '文件不是有效的 JSON，请选择「导出目录」得到的文件', + readFailed: '无法读取这个文件', + fileMeta: (source: string, at: string) => `导出自「${source}」 · ${at}`, + totals: (t: { tables: number; add: number; update: number; keep: number; conflicts: number; skipped: number }) => + `${formatNumber(t.tables)} 张表：新增 ${formatNumber(t.add)} 项、更新 ${formatNumber(t.update)} 项、保留 ${formatNumber(t.keep)} 项、` + + `冲突 ${formatNumber(t.conflicts)} 项、跳过 ${formatNumber(t.skipped)} 项`, + head: { table: '表', add: '新增', update: '更新', keep: '保留', conflicts: '冲突', skipped: '跳过' }, + matchedAs: (name: string) => `对上「${name}」`, + unmatched: (n: number) => `${formatNumber(n)} 张表在这个数据源里对不上，不导入：`, + conflictsTitle: (n: number) => `冲突 ${formatNumber(n)} 项：以这里为准，不导入`, + conflictFile: '文件中', + conflictTarget: '这里', + skippedTitle: (n: number) => `跳过 ${formatNumber(n)} 项`, + guidance: { + replace: '起草要求：写入文件中的起草要求（这个数据源还没有）', + keep: '起草要求：保留这个数据源现有的，不覆盖', + none: '起草要求：不导入', + } as Record<'keep' | 'replace' | 'none', string>, + confirm: (n: number) => `导入 ${formatNumber(n)} 张表`, + importing: '正在导入…', + nothing: '没有可以导入的内容：文件中的表在这个数据源里都对不上', + done: '导入完成', + doneTotals: (t: { tables: number; add: number; update: number; conflicts: number }) => + `写入 ${formatNumber(t.tables)} 张表：新增 ${formatNumber(t.add)} 项、更新 ${formatNumber(t.update)} 项；冲突 ${formatNumber(t.conflicts)} 项以这里为准`, + version: (v: number) => `第 ${formatNumber(v)} 版`, + tableFailed: (reason: string) => `这张表未导入：${reason}`, + back: '重新选择文件', + cancel: '取消', + close: '完成', } // =========================================================================== diff --git a/frontend/src/pages/catalog/assistant.ts b/frontend/src/pages/catalog/assistant.ts new file mode 100644 index 00000000..20840bb3 --- /dev/null +++ b/frontend/src/pages/catalog/assistant.ts @@ -0,0 +1,213 @@ +import type { + CatalogAssistantChange, CatalogAssistantEvent, CatalogAssistantRequest, CatalogDetail, CatalogPatchChange, CatalogPatchPreview, + CatalogPatchSubmit, +} from '../../types' +import { patchChangesOf, patchWhere } from '../../run/catalogPatch' + +// =========================================================================== +// 表助手的纯函数：一轮回答怎么由事件拼起来、交给服务端的对话怎么截、待保存的修改怎么累积（同一项以最后一次提案为准）、 +// 保存遇到 409 后怎么按预览重算。界面在 TableAssistant.tsx。 +// +// 对话不存服务端：每一轮把前面的对话（最多 40 条、每条最多 4000 字）和界面上还没保存的修改一起交上去，服务端据此 +// 知道已经提过什么、用户说过什么。 +// =========================================================================== + +/** 每一轮最多带几条对话（服务端同一个上限） */ +export const ASSISTANT_MAX_MESSAGES = 40 +/** 每条对话最多几个字（服务端同一个上限） */ +export const ASSISTANT_MAX_CHARS = 4000 +/** 一次最多保存几项（服务端 PATCH_MAX_CHANGES） */ +export const ASSISTANT_MAX_SAVE = 200 +/** 查询结果在界面上显示前几行（服务端最多给 10 行） */ +export const QUERY_ROWS_SHOWN = 10 + +/** 一轮回答里按先后到来的几块：正文、它跑的只读查询、修改提案 */ +export type TurnPart = + | { kind: 'text'; text: string } + | { kind: 'query'; sql: string; columns: string[]; rows: unknown[][]; rowCount: number; error: string | null } + | { kind: 'proposal'; id: string; version: number; changes: CatalogAssistantChange[] } + +export interface UserTurn { id: string; role: 'user'; text: string } + +export interface AssistantTurn { + id: string + role: 'assistant' + parts: TurnPart[] + /** streaming 还在回答；done 答完；stopped 点了停止；error 出错（开流前被拒、流里的 error 事件、连接中断） */ + state: 'streaming' | 'done' | 'stopped' | 'error' + error?: string | null + code?: string | null + usage?: { input_tokens: number; output_tokens: number } | null + queries?: number +} + +export type Turn = UserTurn | AssistantTurn + +const isObj = (v: unknown): v is Record => !!v && typeof v === 'object' && !Array.isArray(v) +const num = (v: unknown, fallback = 0): number => (typeof v === 'number' && Number.isFinite(v) ? v : fallback) + +/** 提案里的各项：形状同助手流里的 catalog_patch（认不出的项跳过），多一个给人看的位置 */ +export function assistantChangesOf(raw: unknown): CatalogAssistantChange[] { + const list = Array.isArray(raw) ? raw : [] + const parsed = patchChangesOf(list) + return parsed.map((c) => { + const src = list.find((x) => isObj(x) && x.path === c.path) as Record | undefined + const label = typeof src?.label === 'string' && src.label.trim() ? src.label.trim() : undefined + return label ? { ...c, label } : c + }) +} + +/** 一项给人看的位置：服务端写了就用它，没写按路径说 */ +export const changeWhere = (c: CatalogAssistantChange): string => c.label ?? patchWhere(c) + +/** 一个事件并进这一轮回答（返回新的一轮，不改入参）。proposalId：提案的编号（卡片、待保存的修改按它认出处） */ +export function applyEvent(turn: AssistantTurn, ev: CatalogAssistantEvent, proposalId: string): AssistantTurn { + const parts = [...turn.parts] + switch (ev.type) { + case 'text': { + const delta = typeof ev.delta === 'string' ? ev.delta : '' + if (!delta) return turn + const last = parts.at(-1) + if (last?.kind === 'text') parts[parts.length - 1] = { kind: 'text', text: last.text + delta } + else parts.push({ kind: 'text', text: delta }) + return { ...turn, parts } + } + case 'query': { + const columns = Array.isArray(ev.columns) ? ev.columns.map(String) : [] + const rows = Array.isArray(ev.rows) ? ev.rows.filter(Array.isArray).slice(0, QUERY_ROWS_SHOWN) as unknown[][] : [] + parts.push({ + kind: 'query', sql: typeof ev.sql === 'string' ? ev.sql : '', columns, rows, rowCount: num(ev.row_count, rows.length), + error: typeof ev.error === 'string' && ev.error.trim() ? ev.error : null, + }) + return { ...turn, parts } + } + case 'proposal': { + const changes = assistantChangesOf(ev.changes) + if (!changes.length) return turn + parts.push({ kind: 'proposal', id: proposalId, version: num(ev.version), changes }) + return { ...turn, parts } + } + case 'done': { + const usage = isObj(ev.usage) ? { input_tokens: num(ev.usage.input_tokens), output_tokens: num(ev.usage.output_tokens) } : null + return { ...turn, state: 'done', usage, queries: num(ev.queries) } + } + case 'error': + return { ...turn, state: 'error', error: typeof ev.message === 'string' && ev.message ? ev.message : null, code: ev.code ?? null } + default: + return turn + } +} + +/** 这一轮回答的正文（各段正文接起来） */ +export const turnText = (turn: AssistantTurn): string => + turn.parts.filter((p): p is Extract => p.kind === 'text').map((p) => p.text).join('') + +/** 这一轮里的提案 */ +export const turnProposals = (turn: AssistantTurn) => + turn.parts.filter((p): p is Extract => p.kind === 'proposal') + +/** + * 没有正文的一轮回答交给服务端时写成什么（只提了修改、只查了数据、没答完就停了）：保持一问一答交替。这几句只给模型看 + */ +function placeholderOf(turn: AssistantTurn): string { + const proposals = turnProposals(turn).reduce((n, p) => n + p.changes.length, 0) + if (proposals) return `（提出了 ${proposals} 项修改提案）` + if (turn.parts.some((p) => p.kind === 'query')) return '（查询了数据，没有写出回答）' + return '（没有回答）' +} + +/** 截到上限：每条最多 4000 字 */ +const clip = (s: string) => ([...s].length > ASSISTANT_MAX_CHARS ? [...s].slice(0, ASSISTANT_MAX_CHARS).join('') : s) + +/** + * 交给服务端的对话：前面的对话加上这一条新问题，最多 40 条、每条最多 4000 字。超出时去掉最早的几条，并且总从用户的话 + * 开始（去掉打头的助手回答） + */ +export function historyOf(turns: Turn[], next: string): CatalogAssistantRequest['messages'] { + const out: CatalogAssistantRequest['messages'] = [] + for (const t of turns) { + if (t.role === 'user') out.push({ role: 'user', content: clip(t.text) }) + else out.push({ role: 'assistant', content: clip(turnText(t).trim() || placeholderOf(t)) }) + } + out.push({ role: 'user', content: clip(next) }) + let tail = out.slice(-ASSISTANT_MAX_MESSAGES) + while (tail.length > 1 && tail[0].role !== 'user') tail = tail.slice(1) + return tail +} + +// --------------------------------------------------------------------------- +// 待保存的修改：跨多轮累积，同一项（路径）以最后一次提案为准；可以逐项去掉、加回。保存成功后清空 +// --------------------------------------------------------------------------- + +export interface PendingChange extends CatalogAssistantChange { + /** 来自哪一条提案 */ + origin: string + /** 提案对照的目录版本：保存时带最早的那个，别人在那之后改过就 409，按最新内容重算后再保存 */ + version: number +} + +/** 一条提案并进待保存的修改：同一路径的换成这一条，位置不动；已经是这个值且已确认的（same）不收 */ +export function mergeProposal(pending: PendingChange[], changes: CatalogAssistantChange[], origin: string, version: number): PendingChange[] { + const out = [...pending] + for (const c of changes) { + if (c.state === 'same') continue + const next: PendingChange = { ...c, origin, version } + const i = out.findIndex((p) => p.path === c.path) + if (i >= 0) out[i] = next + else out.push(next) + } + return out +} + +export const dropPending = (pending: PendingChange[], path: string): PendingChange[] => pending.filter((p) => p.path !== path) + +/** 交回保存、交给助手的那一份：路径、原样取值、理由 */ +export function pendingSubmit(pending: PendingChange[]): CatalogPatchSubmit[] { + return pending.map((p) => ({ path: p.path, value: p.value, ...(p.reason ? { reason: p.reason } : {}) })) +} + +/** 保存时带的版本：待保存的修改里最早的那条提案对照的版本 */ +export const pendingVersion = (pending: PendingChange[]): number => + (pending.length ? Math.min(...pending.map((p) => p.version)) : 0) + +/** 提案卡片上一项的处境：待保存（就是这一条）、改用了另一条提案的值、去掉了、已保存 */ +export type ChangeState = 'pending' | 'replaced' | 'dropped' | 'saved' + +export function changeStateOf(pending: PendingChange[], saved: ReadonlySet, origin: string, path: string): ChangeState { + if (saved.has(savedKey(origin, path))) return 'saved' + const p = pending.find((x) => x.path === path) + if (!p) return 'dropped' + return p.origin === origin ? 'pending' : 'replaced' +} + +export const savedKey = (origin: string, path: string) => `${origin}\u0000${path}` + +/** + * 保存遇到 409（别人刚改过这张表）后按预览重算：预览里有的项换上最新的改前、改后和状态，版本记成预览的； + * 预览里没有的（已经不合法了，原因在 problems 里）去掉；已经是这个值且已确认的（same）也去掉，不用再存 + */ +export function applyPreview(pending: PendingChange[], preview: CatalogPatchPreview): PendingChange[] { + const byPath = new Map(preview.changes.map((c) => [c.path, c])) + const out: PendingChange[] = [] + for (const p of pending) { + const c: CatalogPatchChange | undefined = byPath.get(p.path) + if (!c || c.state === 'same') continue + out.push({ ...p, before: c.before, before_status: c.before_status, after: c.after, value: c.value, state: c.state, + ...(c.note ? { note: c.note } : {}), version: preview.version }) + } + return out +} + +// --------------------------------------------------------------------------- +// 开场：上次起草有留空的列时，引导先问这几列 +// --------------------------------------------------------------------------- + +/** 还没有中文名和含义的列（表结构里的顺序；被驳回的不算有） */ +export function blankColumnsOf(detail: CatalogDetail): string[] { + const live = (it: { status: string } | undefined) => !!it && it.status !== 'rejected' + return (detail.structure?.columns ?? []).map((c) => c.name) + .filter((name) => { + const items = detail.notes.columns?.[name] + return !live(items?.label) && !live(items?.meaning) + }) +} diff --git a/frontend/src/pages/catalog/review.ts b/frontend/src/pages/catalog/review.ts new file mode 100644 index 00000000..7aa844da --- /dev/null +++ b/frontend/src/pages/catalog/review.ts @@ -0,0 +1,132 @@ +import type { + CatalogNotes, CatalogReviewAction, CatalogReviewBatchResult, CatalogReviewItem, CatalogReviewKind, CatalogStatus, +} from '../../types' +import { slotsOf } from './model' + +// =========================================================================== +// 审阅队列的纯函数：地址里的视图和筛选、队列一项的键、成批审阅之后逐条的结果、表详情里改过之后队列跟着变、进度的百分比。 +// 界面在 ReviewQueue.tsx。 +// +// 队列只列推断项，顺序由服务端排好（表的使用次数 × 风险 × 来源）。审阅过的项不马上从列表里拿掉：原地标出结果 +// （已确认、已驳回、冲突），可以选中撤销；刷新队列后才移除。 +// =========================================================================== + +/** 数据目录页的两个视图：表清单（逐表看）、审阅队列（跨表按优先级看推断项） */ +export type CatalogView = 'list' | 'review' + +export const REVIEW_KINDS: readonly CatalogReviewKind[] = ['all', 'facts', 'names'] +/** 一页几项、一次成批审阅最多几条（服务端同一个上限） */ +export const REVIEW_PAGE = 50 +export const REVIEW_BATCH_MAX = 200 + +/** + * 审阅视图记在地址的查询参数里：view=review；kind 筛选（默认全部不写）；rtable 只看一张表；item 从队列打开的那一项 + * （表详情据此滚到那一项、标出来）。和清单的 q / filter / sort / used 互不相干，各自只动自己的键 + */ +export interface ReviewParams { view: CatalogView; kind: CatalogReviewKind; table: string; item: string } + +export function reviewParamsOf(p: URLSearchParams): ReviewParams { + const kind = p.get('kind') as CatalogReviewKind | null + return { + view: p.get('view') === 'review' ? 'review' : 'list', + kind: kind && REVIEW_KINDS.includes(kind) ? kind : 'all', + table: p.get('rtable') ?? '', + item: p.get('item') ?? '', + } +} + +export function writeReviewParams(p: URLSearchParams, s: Partial): URLSearchParams { + const next = new URLSearchParams(p) + const put = (k: string, v: string | null | undefined) => (v ? next.set(k, v) : next.delete(k)) + if ('view' in s) put('view', s.view === 'review' ? 'review' : null) + if ('kind' in s) put('kind', s.kind === 'all' ? null : s.kind) + if ('table' in s) put('rtable', s.table) + if ('item' in s) put('item', s.item) + return next +} + +/** 队列一项的键（表名里、路径里都可能有句点，中间用一个不会出现的字符隔开） */ +export const itemKey = (i: { table_name: string; path: string }) => `${i.table_name}\u0000${i.path}` + +/** 一项审阅过之后的处境：成功（现在的状态、这张表写入后的版本）、冲突（这张表刚被改过）、没成功（服务端的原因） */ +export type ItemOutcome = + | { kind: 'ok'; action: CatalogReviewAction; status: CatalogStatus } + | { kind: 'conflict'; message: string | null } + | { kind: 'error'; message: string | null } + +/** 成批审阅之后这一项现在的状态：确认 → 已确认；驳回 → 已驳回；撤销审阅 → 回到来源的初始状态（队列里都是推断） */ +const STATUS_AFTER: Record = { confirm: 'confirmed', reject: 'rejected', reset: 'proposed' } + +/** 服务端的机读码说的是不是版本冲突 */ +export const isConflict = (r: Pick): boolean => !!r.code && /conflict|version/i.test(r.code) + +/** + * 逐条的结果并进去：outcomes 按项的键记处境，versions 按表记最新的版本（之后再审阅这张表的项带它）。返回新的两份，不改入参 + */ +export function applyResults( + results: CatalogReviewBatchResult[], action: CatalogReviewAction, + outcomes: ReadonlyMap, versions: ReadonlyMap, +): { outcomes: Map; versions: Map } { + const o = new Map(outcomes) + const v = new Map(versions) + for (const r of results) { + const key = itemKey(r) + if (r.ok) { + o.set(key, { kind: 'ok', action, status: STATUS_AFTER[action] }) + if (typeof r.version === 'number') v.set(r.table_name, Math.max(v.get(r.table_name) ?? 0, r.version)) + } else { + o.set(key, isConflict(r) ? { kind: 'conflict', message: r.error } : { kind: 'error', message: r.error }) + } + } + return { outcomes: o, versions: v } +} + +/** 一项现在显示的状态：审阅成功过的按结果，否则是队列里给的 */ +export function statusOf(item: CatalogReviewItem, outcome: ItemOutcome | undefined): CatalogStatus { + return outcome?.kind === 'ok' ? outcome.status : item.status +} + +/** + * 一批选中的项里，这个动作要发哪几条：确认发还不是已确认的，驳回发还不是已驳回的，撤销审阅只发这一页上审阅成功过的 + * （队列里本来都是推断，没审阅过的撤销了也没有变化）。冲突过的照旧可以再发 + */ +export function batchTargets(items: CatalogReviewItem[], selected: ReadonlySet, action: CatalogReviewAction, + outcomes: ReadonlyMap): CatalogReviewItem[] { + return items.filter((i) => { + if (!selected.has(itemKey(i))) return false + const o = outcomes.get(itemKey(i)) + const status = statusOf(i, o) + if (action === 'reset') return o?.kind === 'ok' && o.action !== 'reset' + return action === 'confirm' ? status !== 'confirmed' : status !== 'rejected' + }) +} + +/** + * 表详情里改过这张表（单项审阅、编辑、助手的修改）：队列里这张表的项按最新的目录换状态——不再是推断的记成审阅成功， + * 版本换成最新的。目录里已经没有的项不动（刷新后服务端不会再列） + */ +export function syncFromDetail(items: CatalogReviewItem[], table: string, notes: CatalogNotes, version: number, + outcomes: ReadonlyMap, versions: ReadonlyMap, +): { outcomes: Map; versions: Map } { + const o = new Map(outcomes) + const v = new Map(versions) + v.set(table, version) + const slots = new Map(slotsOf(notes).map((s) => [s.path, s.item.status])) + for (const i of items) { + if (i.table_name !== table) continue + const status = slots.get(i.path) + if (!status) continue + const key = itemKey(i) + if (status === 'proposed') { + // 回到推断：之前审阅成功（确认、驳回）过的记成撤销审阅；冲突的照旧标着 + const cur = o.get(key) + if (cur?.kind === 'ok' && cur.status !== 'proposed') o.set(key, { kind: 'ok', action: 'reset', status: 'proposed' }) + } else { + o.set(key, { kind: 'ok', action: status === 'rejected' ? 'reject' : 'confirm', status }) + } + } + return { outcomes: o, versions: v } +} + +/** 百分比（四舍五入到整数）；分母为 0 时为 null */ +export const percentOf = (n: number, total: number): number | null => (total > 0 ? Math.round((n / total) * 100) : null) diff --git a/frontend/src/run/catalogPatch.ts b/frontend/src/run/catalogPatch.ts index 900ae164..29821bc3 100644 --- a/frontend/src/run/catalogPatch.ts +++ b/frontend/src/run/catalogPatch.ts @@ -65,10 +65,15 @@ function changeOf(x: unknown): CatalogPatchChange | null { } } +/** 提案里的各项（catalog_patch、表助手的 proposal 事件同一个形状）：认不出的项跳过 */ +export function patchChangesOf(raw: unknown): CatalogPatchChange[] { + return (Array.isArray(raw) ? raw : []).map(changeOf).filter((c): c is CatalogPatchChange => !!c) +} + /** catalog_patch 操作 → 提案。认不出的形状返回 null */ export function catalogPatchOf(op: Record, key = ''): CatalogPatch | null { if (typeof op.source !== 'string' || typeof op.source_id !== 'string' || typeof op.table !== 'string') return null - const changes = (Array.isArray(op.changes) ? op.changes : []).map(changeOf).filter((c): c is CatalogPatchChange => !!c) + const changes = patchChangesOf(op.changes) if (!changes.length) return null const version = typeof op.version === 'number' && Number.isFinite(op.version) ? op.version : 0 return { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 4fd72ba3..e9ea3dbf 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -2884,6 +2884,16 @@ export interface CatalogDraftTask { /** 模型分几段起草（超过 40 列的宽表拆成表级一段加列分段）；不用模型时为 null */ parts_total: number | null parts_done: number + /** + * 已经写进目录的段数：每写完一段就并入目录，停止、服务重启时已写进的段保留。老服务端没有这个键 + */ + parts_written?: number + /** + * 这张表实际分成的块数和已结束的块数：额度用完、超时后段被缩小、拆半，块会比段多。不用模型时为 null； + * 老服务端没有这两个键 + */ + chunks_total?: number | null + chunks_done?: number | null added: number updated: number removed: number @@ -2984,6 +2994,13 @@ export interface CatalogLastDraft { model_error: string | null /** 那次起草模型拿不准、留空的列数（同 CatalogDraftTask.blank_columns）。老服务端没有这个键 */ blank_columns?: number + /** + * 最近一次这张表结束于完成、部分完成的起草记下的留空列数（没有这样的起草为 null）。最近一次是停止、服务重启中断、 + * 又没有新写入时，界面用它接着提示留空的列。老服务端没有这个键 + */ + blank_columns_from?: number | null + /** 那次起草已写进目录的段数（同 CatalogDraftTask.parts_written）。接口约定里没有这一项，服务端给了才用 */ + parts_written?: number } /** @@ -3046,6 +3063,171 @@ export interface CatalogImpactTemplate { nodes: CatalogImpactNode[] } +// --------------------------------------------------------------------------- +// 审阅队列（/api/datasources/{id}/catalog/review-queue、review/batch、review-progress、{table}/usage，后端 +// data/catalog_review.py）：跨表列出推断项，按表的使用次数、这一项对数字的影响和来源排好，可以成批确认、驳回。 +// --------------------------------------------------------------------------- + +/** 队列的筛选：全部、影响数字的项（有效记录条件、去重、业务日期、关联关系、业务主键、度量类型）、名称和含义 */ +export type CatalogReviewKind = 'all' | 'facts' | 'names' + +/** 队列里的一项（只列推断状态的项） */ +export interface CatalogReviewItem { + table_name: string + /** 审阅接口的路径写法 */ + path: string + /** 字段名（valid_filter、label……） */ + field: string + /** 给人看的位置（服务端写好的，比如「列 status 的码值」） */ + field_label: string + value: unknown + /** 值写成一行字（服务端写好的） */ + display: string + source: CatalogSource + status: CatalogStatus + note: string | null + /** 越大越靠前：表的使用次数权重 × 风险权重 × 来源权重 */ + priority: number + /** 为什么排在这里（中文，服务端写好的） */ + reasons: string[] + /** 所在表被运行用过的次数 */ + table_usage: number + /** 所在表的目录版本：成批审阅时逐表带上 */ + version: number +} + +/** GET /catalog/review-queue */ +export interface CatalogReviewQueue { + total: number + items: CatalogReviewItem[] +} + +/** POST /catalog/review/batch 的一项 */ +export interface CatalogReviewBatchItem { + table_name: string + path: string + action: CatalogReviewAction + if_version: number +} + +/** 成批审阅逐条的结果：一张表冲突不影响别的表；成功的写明这张表写入后的版本 */ +export interface CatalogReviewBatchResult { + table_name: string + path: string + ok: boolean + version: number | null + error: string | null + code: string | null +} + +/** 用过这张表的一次运行（新的在前） */ +export interface CatalogUsageRun { + run_id: string + workflow_id: string | null + workflow_name: string | null + run_class: 'formal' | 'exploratory' | null + status: string + started_at: string | null +} + +/** GET /catalog/{table}/usage */ +export interface CatalogTableUsage { + total: number + runs: CatalogUsageRun[] +} + +/** GET /catalog/review-progress：常用的前 top_n 张表、全部表、影响数字的项各审阅到哪儿 */ +export interface CatalogReviewProgress { + top_n: number + top: { tables: number; items_total: number; items_confirmed: number } + all: { items_total: number; items_confirmed: number; items_proposed: number; items_rejected: number } + facts: { proposed: number; confirmed: number } +} + +// --------------------------------------------------------------------------- +// 导出导入（/api/datasources/{id}/catalog/export、import/preview、import，后端 data/catalog_transfer.py）： +// 本机整理好的目录带到另一个实例。先预览（只读），确认后逐表写入。 +// --------------------------------------------------------------------------- + +/** 导出的文件：含全部状态的项 */ +export interface CatalogExportFile { + format: 'agentlab-catalog' + version: 1 + exported_at: string + source: { name: string; kind: string } + guidance: string | null + tables: Record +} + +/** 一处冲突：这里已确认、已驳回的项和文件中的不一样，以这里为准 */ +export interface CatalogImportConflict { + path: string + label: string + file_value: unknown + target_value: unknown + why: string +} + +/** 预览、导入报告里的一张表 */ +export interface CatalogImportTable { + /** 文件中的表名 */ + table_name: string + /** 对上的这个数据源里的表名（名字不一样时才有意思）；对不上为 null */ + matched_as: string | null + add: number + update: number + keep: number + conflicts: CatalogImportConflict[] + skipped: { path: string; why: string }[] + /** 导入后这张表的目录版本（只有导入的报告有） */ + version?: number | null + /** 这张表没导入的原因（逐表写入，一张表失败不影响别的）。接口约定里没有写明，服务端给了才显示 */ + error?: string | null +} + +export interface CatalogImportTotals { + tables: number + add: number + update: number + keep: number + conflicts: number + skipped: number +} + +/** POST /catalog/import/preview、/catalog/import */ +export interface CatalogImportReport { + tables: CatalogImportTable[] + /** 文件中有、这个数据源里对不上的表 */ + unmatched_tables: string[] + totals: CatalogImportTotals + /** 起草要求：写入文件中的（这里还没有）、保留这里的、不导入 */ + guidance: { file: string | null; target: string | null; action: 'keep' | 'replace' | 'none' } +} + +// --------------------------------------------------------------------------- +// 表助手（POST /api/datasources/{id}/catalog/{table}/assistant，后端 data/catalog_assistant.py）：对话完善一张表的 +// 目录。回答是事件流，每个事件一行 data: 。对话不存服务端，每一轮都把前面的对话和还没保存的修改带上。 +// --------------------------------------------------------------------------- + +/** 提案里的一项：和助手流里的 catalog_patch 一样，多一个给人看的位置 */ +export interface CatalogAssistantChange extends CatalogPatchChange { + label?: string +} + +export type CatalogAssistantEvent = + | { type: 'text'; delta: string } + | { type: 'query'; sql: string; columns: string[]; rows: unknown[][]; row_count: number; error: string | null } + | { type: 'proposal'; source_id: string; table: string; version: number; changes: CatalogAssistantChange[] } + | { type: 'done'; usage: { input_tokens: number; output_tokens: number }; queries: number } + | { type: 'error'; message: string; code?: string | null } + +/** 交给表助手的一轮请求 */ +export interface CatalogAssistantRequest { + messages: { role: 'user' | 'assistant'; content: string }[] + /** 界面上还没保存的修改 */ + pending: CatalogPatchSubmit[] +} + /** GET /datasources/{id}/catalog/{table}/impact */ export interface CatalogImpact { table: string From 1cc9fe1a3dfa8e6ba78752c1898244bbfa593783 Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Wed, 14 Oct 2026 08:44:37 +0800 Subject: [PATCH 12/39] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E9=97=AD=E7=8E=AF=EF=BC=88=E5=89=8D=E7=AB=AF=EF=BC=89=EF=BC=9A?= =?UTF-8?q?=E8=A1=A8=E8=AF=A6=E6=83=85=E5=8A=A0=E3=80=8C=E8=A1=A8=E5=8A=A9?= =?UTF-8?q?=E6=89=8B=E3=80=8D=E2=80=94=E2=80=94=E9=A1=B6=E9=83=A8=E5=92=8C?= =?UTF-8?q?=E4=B8=8A=E6=AC=A1=E8=B5=B7=E8=8D=89=E7=9A=84=E6=8F=90=E7=A4=BA?= =?UTF-8?q?=E9=87=8C=E9=83=BD=E6=9C=89=E3=80=8C=E5=92=8C=E5=8A=A9=E6=89=8B?= =?UTF-8?q?=E4=B8=80=E8=B5=B7=E5=AE=8C=E5=96=84=E3=80=8D=E5=85=A5=E5=8F=A3?= =?UTF-8?q?=EF=BC=8C=E5=AF=B9=E8=AF=9D=E6=B5=81=E5=BC=8F=E6=98=BE=E7=A4=BA?= =?UTF-8?q?=E3=80=81=E8=83=BD=E5=81=9C=E6=AD=A2=EF=BC=8C=E5=8A=A9=E6=89=8B?= =?UTF-8?q?=E8=B7=91=E8=BF=87=E7=9A=84=E5=8F=AA=E8=AF=BB=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E5=86=99=20SQL=20=E5=92=8C=E5=89=8D=E5=87=A0=E8=A1=8C=EF=BC=88?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E6=94=B6=E8=B5=B7=EF=BC=89=EF=BC=8C=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E6=8F=90=E6=A1=88=E9=80=90=E9=A1=B9=E5=86=99=E6=94=B9?= =?UTF-8?q?=E5=89=8D=E3=80=81=E6=94=B9=E5=90=8E=E3=80=81=E4=BE=9D=E6=8D=AE?= =?UTF-8?q?=E5=B9=B6=E8=83=BD=E9=80=90=E9=A1=B9=E5=8E=BB=E6=8E=89=E3=80=81?= =?UTF-8?q?=E5=8A=A0=E5=9B=9E=EF=BC=8C=E3=80=8C=E5=BE=85=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=E7=9A=84=E4=BF=AE=E6=94=B9=E3=80=8D=E8=B7=A8=E5=A4=9A=E8=BD=AE?= =?UTF-8?q?=E7=B4=AF=E7=A7=AF=EF=BC=88=E5=90=8C=E4=B8=80=E9=A1=B9=E4=BB=A5?= =?UTF-8?q?=E6=9C=80=E5=90=8E=E4=B8=80=E6=AC=A1=E6=8F=90=E6=A1=88=E4=B8=BA?= =?UTF-8?q?=E5=87=86=EF=BC=89=EF=BC=8C=E4=B8=80=E9=94=AE=E4=BF=9D=E5=AD=98?= =?UTF-8?q?=EF=BC=88=E6=9C=80=E5=A4=9A=20200=20=E9=A1=B9=EF=BC=8C409=20?= =?UTF-8?q?=E6=97=B6=E6=8C=89=E6=9C=80=E6=96=B0=E5=86=85=E5=AE=B9=E9=87=8D?= =?UTF-8?q?=E7=AE=97=E5=90=8E=E5=81=9C=E4=B8=8B=E6=9D=A5=E5=86=8D=E7=9C=8B?= =?UTF-8?q?=E4=B8=80=E7=9C=BC=EF=BC=89=EF=BC=8C=E6=B2=A1=E5=BC=80=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=89=96=E6=9E=90=E7=9A=84=E6=BA=90=E5=86=99=E6=98=8E?= =?UTF-8?q?=E5=8F=AA=E8=83=BD=E6=A0=B9=E6=8D=AE=E8=A1=A8=E7=BB=93=E6=9E=84?= =?UTF-8?q?=E5=92=8C=E7=9B=AE=E5=BD=95=E6=8E=A8=E6=96=AD=E3=80=81=E4=B8=8D?= =?UTF-8?q?=E8=83=BD=E6=9F=A5=E6=95=B0=E6=8D=AE=EF=BC=8C=E6=9C=89=E7=95=99?= =?UTF-8?q?=E7=A9=BA=E5=88=97=E6=97=B6=E5=BC=80=E5=9C=BA=E5=BC=95=E5=AF=BC?= =?UTF-8?q?=E5=85=88=E9=97=AE=E8=BF=99=E5=87=A0=E5=88=97=EF=BC=8C=E6=9C=89?= =?UTF-8?q?=E5=BE=85=E4=BF=9D=E5=AD=98=E7=9A=84=E4=BF=AE=E6=94=B9=E6=97=B6?= =?UTF-8?q?=E6=8D=A2=E8=A1=A8=E3=80=81=E5=85=B3=E9=97=AD=E5=85=88=E9=97=AE?= =?UTF-8?q?=EF=BC=9B=E8=A1=A8=E8=AF=A6=E6=83=85=E5=8A=A0=E3=80=8C=E7=94=A8?= =?UTF-8?q?=E8=BF=87=E8=BF=99=E5=BC=A0=E8=A1=A8=E7=9A=84=E8=BF=90=E8=A1=8C?= =?UTF-8?q?=E3=80=8D=E3=80=81=E6=9C=89=E6=95=88=E8=AE=B0=E5=BD=95=E6=9D=A1?= =?UTF-8?q?=E4=BB=B6=E5=92=8C=E5=8E=BB=E9=87=8D=E8=A7=84=E5=88=99=E7=9A=84?= =?UTF-8?q?=E6=8E=A8=E6=96=AD=E9=A1=B9=E5=9C=A8=E5=80=BC=E4=B8=8B=E9=9D=A2?= =?UTF-8?q?=E5=86=99=E8=AF=B4=E6=98=8E=EF=BC=88=E5=89=96=E6=9E=90=E6=A0=B8?= =?UTF-8?q?=E5=AF=B9=E7=9A=84=E6=8E=92=E9=99=A4=E6=AF=94=E4=BE=8B=E6=88=96?= =?UTF-8?q?=E6=B2=A1=E6=A0=B8=E5=AF=B9=E7=9A=84=E5=8E=9F=E5=9B=A0=EF=BC=89?= =?UTF-8?q?=E3=80=81=E4=BB=8E=E5=AE=A1=E9=98=85=E9=98=9F=E5=88=97=E6=89=93?= =?UTF-8?q?=E5=BC=80=E7=9A=84=E9=82=A3=E4=B8=80=E9=A1=B9=E6=A0=87=E5=87=BA?= =?UTF-8?q?=E6=9D=A5=EF=BC=9B=E8=B5=B7=E8=8D=89=E4=BB=BB=E5=8A=A1=E9=80=90?= =?UTF-8?q?=E8=A1=A8=E6=98=BE=E7=A4=BA=E5=9D=97=E8=BF=9B=E5=BA=A6=EF=BC=88?= =?UTF-8?q?=E6=AE=B5=E8=A2=AB=E6=8B=86=E5=B0=8F=E5=90=8E=E7=9C=8B=E5=BE=97?= =?UTF-8?q?=E5=87=BA=E6=9D=A5=EF=BC=89=E5=92=8C=E5=B7=B2=E5=86=99=E8=BF=9B?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E7=9A=84=E6=AE=B5=EF=BC=8C=E5=81=9C=E6=AD=A2?= =?UTF-8?q?=E5=90=8E=E5=86=99=E6=98=8E=E5=B7=B2=E5=86=99=E8=BF=9B=E5=87=A0?= =?UTF-8?q?=E6=AE=B5=E3=80=81=E5=8F=AA=E9=87=8D=E8=AF=95=E6=97=B6=E6=8E=A5?= =?UTF-8?q?=E7=9D=80=E8=B5=B7=E8=8D=89=EF=BC=8C=E6=B8=85=E5=8D=95=E5=92=8C?= =?UTF-8?q?=E8=A1=A8=E8=AF=A6=E6=83=85=E5=9C=A8=E6=9C=80=E8=BF=91=E4=B8=80?= =?UTF-8?q?=E6=AC=A1=E5=81=9C=E6=AD=A2=E3=80=81=E4=B8=AD=E6=96=AD=E5=8F=88?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E6=96=B0=E5=86=99=E5=85=A5=E6=97=B6=E6=8E=A5?= =?UTF-8?q?=E7=9D=80=E7=94=A8=E5=86=8D=E4=B9=8B=E5=89=8D=E9=82=A3=E6=AC=A1?= =?UTF-8?q?=E8=B5=B7=E8=8D=89=E8=AE=B0=E4=B8=8B=E7=9A=84=E7=95=99=E7=A9=BA?= =?UTF-8?q?=E5=88=97=E6=95=B0=E6=8F=90=E7=A4=BA=EF=BC=9Bcheck-catalog=20?= =?UTF-8?q?=E5=8A=A0=E4=B8=80=E8=8A=82=E6=8C=89=E7=BA=A6=E5=AE=9A=E7=9A=84?= =?UTF-8?q?=E6=96=B0=E5=AD=97=E6=AE=B5=E6=8E=A8=E8=BF=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- frontend/src/pages/catalog/ColumnTable.tsx | 28 +- frontend/src/pages/catalog/DraftDialog.tsx | 34 +- frontend/src/pages/catalog/TableAssistant.tsx | 591 ++++++++++++++++++ frontend/src/pages/catalog/TableDetail.tsx | 117 +++- frontend/src/pages/catalog/TableIndex.tsx | 12 +- frontend/src/pages/catalog/TableUsage.tsx | 90 +++ frontend/src/pages/catalog/draftJob.ts | 37 +- frontend/src/pages/catalog/parts.tsx | 4 + scripts/check-catalog.mjs | 109 +++- 9 files changed, 983 insertions(+), 39 deletions(-) create mode 100644 frontend/src/pages/catalog/TableAssistant.tsx create mode 100644 frontend/src/pages/catalog/TableUsage.tsx diff --git a/frontend/src/pages/catalog/ColumnTable.tsx b/frontend/src/pages/catalog/ColumnTable.tsx index cec5a1ba..56e64f27 100644 --- a/frontend/src/pages/catalog/ColumnTable.tsx +++ b/frontend/src/pages/catalog/ColumnTable.tsx @@ -9,7 +9,7 @@ import { IconButton } from '../../components/ui' import { CATALOG_COLUMN_FIELD_LABEL, CATALOG_MEASURE_HINT, CATALOG_MEASURE_LABEL, CATALOG_TEXT as CT, CODES_TEXT as KT, } from '../../lib/terms' -import { ItemMark } from './parts' +import { FOCUS_RING, FOCUS_STYLE, ItemMark } from './parts' import type { ReviewTarget } from './parts' import { COLUMN_FIELDS, codesComplete, columnPath, completeOf } from './model' import type { ColumnDraft, ColumnField } from './model' @@ -50,7 +50,16 @@ export interface ColumnModel { type ColumnTexts = ColumnDraft -export function ColumnTable({ columns, form, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange, onCodesComplete }: { +/** 审阅路径 columns.<列名>.<字段> → [列名, 字段]（列名里可以有句点，按最后一个句点切）；不是列级项为 null */ +function columnOfPath(path: string | null | undefined): [string, string] | null { + if (!path?.startsWith('columns.')) return null + const rest = path.slice('columns.'.length) + const at = rest.lastIndexOf('.') + return at > 0 ? [rest.slice(0, at), rest.slice(at + 1)] : null +} + +export function ColumnTable({ columns, form, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange, onCodesComplete, + focusPath = null }: { columns: ColumnModel[] /** 编辑中：列名 → 字段 → 文字（form 是当前的，initial 是进入编辑时的）；不在编辑时为 null */ form: Record | null @@ -67,7 +76,10 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote onChange: (col: string, field: ColumnField, value: string) => void /** 编辑中勾选或取消码值的「已列出全部取值」 */ onCodesComplete: (col: string, value: boolean) => void + /** 从审阅队列打开的那一项（列级项的审阅路径）：那一格标出来 */ + focusPath?: string | null }) { + const focus = columnOfPath(focusPath) return (
@@ -89,7 +101,7 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote {columns.map((c) => ( + onCodesComplete={onCodesComplete} focusField={focus && focus[0] === c.name ? focus[1] : null} /> ))}
@@ -98,7 +110,7 @@ export function ColumnTable({ columns, form, initial, problems, busy, systemNote } const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, problems, busy, systemNotes, onReview, onConfirmColumn, onFillCodes, onChange, - onCodesComplete }: { + onCodesComplete, focusField }: { col: ColumnModel editing: boolean /** 这一列的表单。只有改了这一列时引用才会变，别的行不重画 */ @@ -112,6 +124,8 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble onFillCodes: (col: string) => void onChange: (col: string, field: ColumnField, value: string) => void onCodesComplete: (col: string, value: boolean) => void + /** 从审阅队列打开的是这一列的哪一项 */ + focusField: string | null }) { const { name, structure, items } = col const pending = COLUMN_FIELDS.filter((f) => items?.[f]?.status === 'proposed').length @@ -121,9 +135,12 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble const where = CT.whereColumn(name, CATALOG_COLUMN_FIELD_LABEL[f]) const problem = problems[`c:${name}:${f}`] const changed = editing && ((draft?.[f] ?? '') !== (initial?.[f] ?? '') || (f === 'codes' && completeOf(draft) !== completeOf(initial))) + const focused = focusField === f return ( // 窄框下没填的项不占行（编辑时照常列出来好填） - + {CT.columnHead[f]}
{editing ? input : view}
@@ -223,6 +240,7 @@ const ColumnRow = memo(function ColumnRow({ col, editing, draft, initial, proble ) }, (a, b) => a.col === b.col && a.editing === b.editing && a.draft === b.draft && a.initial === b.initial && a.busy === b.busy + && a.focusField === b.focusField && a.systemNotes === b.systemNotes && a.onReview === b.onReview && a.onConfirmColumn === b.onConfirmColumn && a.onFillCodes === b.onFillCodes && a.onChange === b.onChange && a.onCodesComplete === b.onCodesComplete // 格式问题每次都是新对象：只比这一列的几格 diff --git a/frontend/src/pages/catalog/DraftDialog.tsx b/frontend/src/pages/catalog/DraftDialog.tsx index 3a7547fc..06371021 100644 --- a/frontend/src/pages/catalog/DraftDialog.tsx +++ b/frontend/src/pages/catalog/DraftDialog.tsx @@ -9,8 +9,8 @@ import { errorMessage } from '../../lib/errors' import { formatTime } from '../../lib/format' import { CATALOG_DRAFT_TEXT as DT, CATALOG_TEXT as CT, DRAFT_GUIDANCE_MAX, DRAFT_NOTE_MAX } from '../../lib/terms' import { - DRAFT_MAX_TABLES, DRAFT_MODEL_UNAVAILABLE, DRAFT_PART_COLUMNS, DRAFT_TOP_N, RETRYABLE, blankOf, isJobActive, jobBlank, needsLook, - nothingWritten, retryableCount, taskReason, + DRAFT_MAX_TABLES, DRAFT_MODEL_UNAVAILABLE, DRAFT_PART_COLUMNS, DRAFT_TOP_N, RETRYABLE, blankOf, chunksOf, isJobActive, jobBlank, needsLook, + nothingWritten, retryableCount, taskReason, writtenOf, } from './draftJob' import type { DraftPreset, DraftStartBody } from './draftJob' @@ -435,6 +435,9 @@ function JobView({ job, running, stopping, busy, blankTables, onRedraftBlank }: // 额度用完、超时多了，之后的段缩小了：进行中写一句，段数为什么变多看得明白 const partColumns = running && job.use_model && job.part_columns != null && job.part_columns < DRAFT_PART_COLUMNS ? job.part_columns : null const blankTotal = jobBlank(job) + // 停下来时已经写进了一部分段的表（宽表每写完一段就写进目录）：几张、合计几段 + const stopped = job.tasks.filter((t) => t.status === 'cancelled' && writtenOf(t) > 0) + const stoppedParts = { tables: stopped.length, parts: stopped.reduce((n, t) => n + writtenOf(t), 0) } return (
{busy && {DT.busy}} @@ -468,7 +471,12 @@ function JobView({ job, running, stopping, busy, blankTables, onRedraftBlank }: {running &&

{DT.hideHint}

} {job.status === 'failed' && job.error && {job.error}} {job.status === 'interrupted' && {DT.interrupted}} - {job.status === 'cancelled' &&

{DT.cancelled}

} + {job.status === 'cancelled' && ( +

+ {DT.cancelled} + {stoppedParts.tables > 0 && {DT.cancelledParts(stoppedParts.tables, stoppedParts.parts)}} +

+ )} {problemText && (

{problemText}

)} @@ -523,7 +531,15 @@ const TASK_ICON: Record 1 && t.status !== 'pending' && t.status !== 'skipped' + const started = t.status !== 'pending' && t.status !== 'skipped' + const parts = t.parts_total != null && t.parts_total > 1 && started + // 段被缩小、拆半后实际的块（比段多时才写）；已经写进目录的段：起草中写着（停止也不丢），停下来时服务端的原因里 + // 没说的才补一句（没写全的段为什么没写出来，服务端的原因里有），写完了的不写 + const chunks = started ? chunksOf(t) : null + const written = writtenOf(t) + const writtenText = written > 0 && t.status === 'running' ? DT.partsWritten(written) + : written > 0 && t.status === 'cancelled' && !reason.includes(DT.partsWritten(written)) + ? DT.partsWrittenStopped(written, Math.max(0, (t.parts_total ?? written) - written)) : '' const changes = DT.taskAdded(t.added, t.updated) const blank = blankOf(t) const sample = t.blank_sample?.filter(Boolean) ?? [] @@ -537,6 +553,11 @@ function TaskRow({ task: t, jobError }: { task: CatalogDraftTask; jobError: stri {DT.parts(t.parts_done, t.parts_total!)} )} + {chunks && ( + + {DT.chunks(chunks.done, chunks.total)} + + )} {changes && ( {changes} )} @@ -545,6 +566,11 @@ function TaskRow({ task: t, jobError }: { task: CatalogDraftTask; jobError: stri {DT.taskStatus[t.status]}
+ {writtenText && ( +

+ {writtenText} +

+ )} {reason && (

{reason}

)} diff --git a/frontend/src/pages/catalog/TableAssistant.tsx b/frontend/src/pages/catalog/TableAssistant.tsx new file mode 100644 index 00000000..db116014 --- /dev/null +++ b/frontend/src/pages/catalog/TableAssistant.tsx @@ -0,0 +1,591 @@ +import { useCallback, useEffect, useId, useMemo, useRef, useState } from 'react' +import type { KeyboardEvent as ReactKeyboardEvent } from 'react' +import { ArrowUp, BookMarked, Database, Info, MessagesSquare, Minus, Plus, Square, Trash2, X } from 'lucide-react' +import clsx from 'clsx' +import { ApiError, api, streamCatalogAssistant } from '../../api/client' +import type { CatalogAssistantEnd } from '../../api/client' +import type { CatalogDetail } from '../../types' +import { Notice, Spinner, confirmDialog, isComposing, toast } from '../../components/ui' +import { useLeaveGuard } from '../../lib/leave' +import { CATALOG_ASSISTANT_TEXT as T, CATALOG_STATUS_LABEL } from '../../lib/terms' +import { Markdown } from '../../run/Markdown' +import { patchValueText } from '../../run/catalogPatch' +import { + ASSISTANT_MAX_CHARS, ASSISTANT_MAX_SAVE, QUERY_ROWS_SHOWN, applyEvent, applyPreview, assistantChangesOf, blankColumnsOf, changeStateOf, + changeWhere, dropPending, historyOf, mergeProposal, pendingSubmit, pendingVersion, savedKey, +} from './assistant' +import type { AssistantTurn, ChangeState, PendingChange, Turn, TurnPart } from './assistant' + +// =========================================================================== +// 表助手：在表详情里和助手对话完善这一张表的数据目录（POST /catalog/{table}/assistant，事件流）。 +// +// 回答流式显示，能停止；助手跑过的只读查询（只有开了数据剖析的源才有）写 SQL 和前几行，默认收起；修改提案逐项写改前、 +// 改后、依据，可以逐项去掉、加回。提案并进「待保存的修改」：跨多轮累积，同一项以最后一次提案为准，一键保存（最多 200 项）。 +// 保存带待保存的修改里最早那条提案对照的版本：别人在那之后改过就 409，按最新内容重算（预览接口，只读）后停下来让人再看 +// 一眼。对话不存服务端：每一轮把前面的对话和待保存的修改一起交上去；关闭、换表、离开页面就没有了，有待保存的修改时先问。 +// =========================================================================== + +type SaveState = + | { phase: 'idle'; error?: string | null; saved?: { n: number; version: number } | null } + | { phase: 'saving' } + | { phase: 'reloading' } + | { phase: 'conflict'; problems: string[] } + +const charCount = (s: string) => [...s.trim()].length + +export function TableAssistant({ sourceId, table, tableLabel, detail, profileEnabled, lastBlank, editing, onSaved, onClose }: { + sourceId: string + table: string + /** 表的中文名（没有就是表名）：标题、提示里用 */ + tableLabel: string + /** 页面上这张表最新的目录：留空的列、保存后更新 */ + detail: CatalogDetail + /** 数据源开了数据剖析：助手才能发只读查询 */ + profileEnabled: boolean + /** 上次起草模型拿不准、留空的列数：开场引导先问这几列 */ + lastBlank: number + /** 正在编辑这张表：先不让保存（同一张表两处同时改，保存会撞上版本冲突） */ + editing: boolean + /** 保存成功：页面换上返回的目录 */ + onSaved: (d: CatalogDetail) => void + /** 关上面板（有待保存的修改时这里先问过了） */ + onClose: () => void +}) { + const [turns, setTurns] = useState([]) + const [pending, setPending] = useState([]) + const [saved, setSaved] = useState>(() => new Set()) + const [text, setText] = useState('') + const [save, setSave] = useState({ phase: 'idle' }) + const [trayOpen, setTrayOpen] = useState(true) + const turnsRef = useRef(turns) + turnsRef.current = turns + const pendingRef = useRef(pending) + pendingRef.current = pending + const stopRef = useRef<(() => void) | null>(null) + const seq = useRef(0) + const inputRef = useRef(null) + const logRef = useRef(null) + const titleId = useId() + const streaming = turns.some((t) => t.role === 'assistant' && t.state === 'streaming') + const dirty = pending.length > 0 || streaming + + // 打开时焦点给输入框；卸下时停掉进行中的回答 + useEffect(() => { inputRef.current?.focus() }, []) + useEffect(() => () => { stopRef.current?.() }, []) + + useLeaveGuard(pending.length > 0, () => confirmDialog({ + title: T.leaveTitle, consequences: T.leaveConsequences, confirmLabel: T.leaveConfirm, danger: true, + })) + + // 新的内容到了：滚到底（人往上翻着看的时候不打扰） + const lastLen = useRef(0) + useEffect(() => { + const el = logRef.current + if (!el) return + const size = turns.reduce((n, t) => n + (t.role === 'user' ? 1 : t.parts.length + 1), 0) + const near = el.scrollHeight - el.scrollTop - el.clientHeight < 160 + if (size !== lastLen.current || near) el.scrollTop = el.scrollHeight + lastLen.current = size + }, [turns]) + + const blankCols = useMemo(() => (lastBlank > 0 ? blankColumnsOf(detail) : []), [lastBlank, detail]) + const hasProposed = useMemo(() => { + const n = detail.notes + const items = [n.label, n.description, n.grain, n.keys, n.kind, n.business_date, n.valid_filter, n.dedup, + ...Object.values(n.columns ?? {}).flatMap((c) => Object.values(c ?? {})), ...(n.relations ?? [])] + return items.some((it) => !!it && (it as { status?: string }).status === 'proposed') + }, [detail]) + + const updateTurn = useCallback((id: string, fn: (t: AssistantTurn) => AssistantTurn) => { + setTurns((ts) => ts.map((t) => (t.id === id && t.role === 'assistant' ? fn(t) : t))) + }, []) + + const send = useCallback((raw: string) => { + const question = raw.trim() + if (!question || charCount(question) > ASSISTANT_MAX_CHARS) return + if (turnsRef.current.some((t) => t.role === 'assistant' && t.state === 'streaming')) return + const n = ++seq.current + const userTurn: Turn = { id: `u${n}`, role: 'user', text: question } + const reply: AssistantTurn = { id: `a${n}`, role: 'assistant', parts: [], state: 'streaming' } + const messages = historyOf(turnsRef.current, question) + setTurns((ts) => [...ts, userTurn, reply]) + setText('') + let proposals = 0 + // 这一轮已经收尾（done、error）：之后再来的事件不认 + let ended = false + stopRef.current = streamCatalogAssistant(sourceId, table, { messages, pending: pendingSubmit(pendingRef.current) }, (ev) => { + if (ended) return + if (ev.type === 'done' || ev.type === 'error') ended = true + if (ev.type === 'proposal') { + const pid = `${reply.id}.p${++proposals}` + const changes = assistantChangesOf(ev.changes) + updateTurn(reply.id, (t) => applyEvent(t, ev, pid)) + if (changes.length) { + const version = typeof ev.version === 'number' && Number.isFinite(ev.version) ? ev.version : detail.version + setPending((cur) => mergeProposal(cur, changes, pid, version)) + // 待保存的修改变了：上一次保存的结果、冲突的说明都不再对 + setSave({ phase: 'idle' }) + } + return + } + updateTurn(reply.id, (t) => applyEvent(t, ev, '')) + }, (end: CatalogAssistantEnd) => { + stopRef.current = null + updateTurn(reply.id, (t) => { + if (t.state !== 'streaming') return t + if (end.aborted) return { ...t, state: 'stopped' } + if (end.error) return { ...t, state: 'error', error: end.error, code: end.code ?? null } + // 流结束了却没有 done：这一轮没答完 + return { ...t, state: 'error', error: T.interrupted } + }) + }) + }, [sourceId, table, detail.version, updateTurn]) + + const stop = () => stopRef.current?.() + + const onKeyDown = (e: ReactKeyboardEvent) => { + if (e.key !== 'Enter' || e.shiftKey || isComposing(e)) return + e.preventDefault() + if (!streaming) send(text) + } + + const drop = (path: string) => setPending((cur) => dropPending(cur, path)) + const restore = (part: Extract, path: string) => { + const change = part.changes.find((c) => c.path === path) + if (change) setPending((cur) => mergeProposal(cur, [change], part.id, part.version)) + } + const clearAll = async () => { + if (!pendingRef.current.length) return + if (!(await confirmDialog({ title: T.clearTitle(pendingRef.current.length), confirmLabel: T.clearConfirm, danger: true }))) return + setPending([]) + setSave({ phase: 'idle' }) + } + + const doSave = async () => { + const list = pendingRef.current + if (!list.length || list.length > ASSISTANT_MAX_SAVE || editing) return + setSave({ phase: 'saving' }) + try { + const d = await api.dataCatalog.patch(sourceId, table, pendingSubmit(list), pendingVersion(list)) + setSaved((cur) => new Set([...cur, ...list.map((p) => savedKey(p.origin, p.path))])) + setPending((cur) => cur.filter((p) => !list.includes(p))) + setSave({ phase: 'idle', saved: { n: list.length, version: d.version } }) + onSaved(d) + toast.ok(T.savedToast(list.length, tableLabel)) + } catch (e) { + if (e instanceof ApiError && e.status === 409) { + // 别人刚改过：按最新内容重算（只读），停下来让人再看一眼 + setSave({ phase: 'reloading' }) + try { + const p = await api.dataCatalog.patchPreview(sourceId, table, pendingSubmit(list)) + setPending((cur) => applyPreview(cur, p)) + setSave({ phase: 'conflict', problems: p.problems }) + } catch { + setSave({ phase: 'idle', error: T.conflictFailed }) + } + } else if (e instanceof ApiError && e.kind === 'http' && (e.status === 422 || e.status === 404)) { + setSave({ phase: 'idle', error: e.message }) + } else { + setSave({ phase: 'idle' }) + toast.error(e) + } + } + } + + const close = async () => { + if (dirty && !(await confirmDialog({ title: T.closeTitle, consequences: T.closeConsequences, confirmLabel: T.closeConfirm, danger: true }))) return + stopRef.current?.() + onClose() + } + + const onPanelKey = (e: ReactKeyboardEvent) => { + if (e.key !== 'Escape' || isComposing(e) || e.defaultPrevented) return + e.preventDefault() + void close() + } + + const fresh = turns.length === 0 + const len = charCount(text) + const over = len > ASSISTANT_MAX_CHARS + const lastAssistant = [...turns].reverse().find((t): t is AssistantTurn => t.role === 'assistant') + const status = streaming ? T.answering : lastAssistant?.state === 'stopped' ? T.stopped : lastAssistant ? T.answered : '' + const sampleCols = blankCols.slice(0, Math.min(Math.max(lastBlank, 1), 5)) + + return ( +