mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-15 06:11:58 +00:00
合入数据目录闭环第一步（后端 B）：表助手（表详情页对话，流式返回，成批提出经校验的修改提案，只在开了剖析的源上经只读守卫跑查询、结果先遮）；审阅队列（按使用次数、风险、来源排序，批量确认驳回逐表乐观锁，表被哪些运行用过，审阅进度）；数据目录导出导入（先预览，确认和驳回的照带、冲突保留目标、推断只填空缺、对不上的跳过并写原因）；一次修改上限提到 200 项
This commit is contained in:
ThearchyHelios
+166
-1
@@ -14,6 +14,13 @@
|
||||
- POST /catalog/draft-jobs 建后台起草任务（202）；GET /draft-jobs/active、/latest、/{job_id} 读任务；
|
||||
POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py）
|
||||
- POST /catalog/profile 同步剖析：对业务库发少量只读查询，核对关系、取码值候选（data/catalog_profile.py）
|
||||
- GET /catalog/review-queue 审阅队列：推断的项按使用次数、风险、来源排好；POST /catalog/review/batch 批量审阅，
|
||||
GET /catalog/review-progress 审阅进度，GET /catalog/{table}/usage 用过这张表的运行
|
||||
（data/catalog_review.py）
|
||||
- GET /catalog/export 导出数据目录（JSON 附件）；POST /catalog/import/preview 导入预览（只读），
|
||||
POST /catalog/import 导入（data/catalog_transfer.py）
|
||||
- POST /catalog/{table}/assistant 表助手：对话完善这张表的目录（text/event-stream），提案不写目录
|
||||
（data/catalog_assistant.py）
|
||||
|
||||
规则都在 data/catalog.py，这里只做取源、转换形状和把异常翻成状态码：版本不符 409，结构不合规、审阅路径
|
||||
不对 422，源或表不存在 404。所有写操作记录请求头 X-Actor 的署名（自报、未认证）。剖析没开启、正在剖析 409。
|
||||
@@ -25,7 +32,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, Header, HTTPException
|
||||
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
@@ -36,6 +43,7 @@ from app.api.coded import (
|
||||
)
|
||||
from app.api.runs import actor_of
|
||||
from app.data import catalog, catalog_impact, catalog_jobs, catalog_profile, introspect, table_versions
|
||||
from app.data import catalog_assistant, catalog_review, catalog_transfer
|
||||
from app.data.engine import SnapshotTampered
|
||||
from app.db.base import get_session
|
||||
from app.db.models import DataSource
|
||||
@@ -235,6 +243,92 @@ async def list_catalog(source_id: str, session: AsyncSession = Depends(get_sessi
|
||||
}
|
||||
|
||||
|
||||
# ---- 审阅队列、审阅进度（要登记在 GET /catalog/{table} 前面：不然 review-queue、review-progress 会被当成表名）
|
||||
|
||||
|
||||
@router.get("/{source_id}/catalog/review-queue")
|
||||
async def catalog_review_queue(source_id: str,
|
||||
limit: int = Query(50, ge=1, le=catalog_review.QUEUE_LIMIT_MAX),
|
||||
offset: int = Query(0, ge=0),
|
||||
kind: Literal["all", "facts", "names"] = Query("all"),
|
||||
table: str | None = Query(None, max_length=255),
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""审阅队列：只列推断的项，按使用次数、风险、来源排好（data/catalog_review.py）。kind 筛影响数字的项（facts）或名称
|
||||
和说明（names），table 只看一张表。"""
|
||||
_, source = await _resolved(session, source_id)
|
||||
return await catalog_review.review_queue(session, source, kind=kind, table=table or None, limit=limit,
|
||||
offset=offset)
|
||||
|
||||
|
||||
@router.get("/{source_id}/catalog/review-progress")
|
||||
async def catalog_review_progress(source_id: str, session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""审阅进度：常用 20 张表、全部、影响数字的项各自确认了多少。"""
|
||||
_, source = await _resolved(session, source_id)
|
||||
return await catalog_review.review_progress(session, source)
|
||||
|
||||
|
||||
# ---- 导出导入（导出同样要登记在 GET /catalog/{table} 前面）
|
||||
|
||||
|
||||
@router.get("/{source_id}/catalog/export")
|
||||
async def export_catalog(source_id: str, session: AsyncSession = Depends(get_session)) -> Any:
|
||||
"""导出这个数据源的数据目录：JSON 附件「<数据源名>-数据目录.json」，含全部状态的项（data/catalog_transfer.py）。"""
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
row = await _source_row(session, source_id)
|
||||
body = await catalog_transfer.export_catalog(session, row)
|
||||
name = catalog_transfer.attachment_name(row.name)
|
||||
return JSONResponse(body, headers={"Content-Disposition": catalog_transfer.content_disposition(name, row.id)})
|
||||
|
||||
|
||||
class CatalogImportIn(BaseModel):
|
||||
#: 导出的那个对象（格式在 data/catalog_transfer.read_file 里核对，不对 422，说明是中文）
|
||||
content: Any = None
|
||||
#: 目标还没有起草要求时，是否写入文件里的起草要求
|
||||
include_guidance: bool = False
|
||||
|
||||
|
||||
async def _import_source(session: AsyncSession, source_id: str) -> tuple[DataSource, Any]:
|
||||
row, source = await _resolved(session, source_id)
|
||||
cache = source.schema_cache or {}
|
||||
if not cache.get("tables"):
|
||||
raise CodedHTTPException(409, f"{introspect.why_empty(cache)}，无法导入数据目录。请先在数据源卡片上点「探查结构」",
|
||||
SCHEMA_MISSING)
|
||||
return row, source
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/import/preview")
|
||||
async def preview_catalog_import(source_id: str, payload: CatalogImportIn,
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""导入预览（只读）：每张对上的表新增、更新、保留几项，冲突和跳过的逐条列出。文件格式不对 422；没有表结构 409。"""
|
||||
row, source = await _import_source(session, source_id)
|
||||
try:
|
||||
report = await catalog_transfer.plan_import(session, row, source, payload.content,
|
||||
include_guidance=payload.include_guidance)
|
||||
except catalog_transfer.TransferInvalid as e:
|
||||
raise HTTPException(422, str(e)) from e
|
||||
return report.as_dict()
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/import")
|
||||
async def import_catalog(source_id: str, payload: CatalogImportIn, x_actor: str | None = Header(default=None),
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""导入：规则同预览，逐表写入（乐观锁），返回同样形状的报告和每张表写入后的版本。有排队或进行中的起草任务
|
||||
409 catalog_draft_busy；文件格式不对 422；没有表结构 409。"""
|
||||
row, source = await _import_source(session, source_id)
|
||||
try:
|
||||
await catalog_jobs.ensure_idle(session, row)
|
||||
except catalog_jobs.DraftBusy as e:
|
||||
raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e
|
||||
try:
|
||||
report = await catalog_transfer.run_import(session, row, source, payload.content,
|
||||
include_guidance=payload.include_guidance,
|
||||
actor=actor_of(x_actor))
|
||||
except catalog_transfer.TransferInvalid as e:
|
||||
raise HTTPException(422, str(e)) from e
|
||||
return report.as_dict()
|
||||
|
||||
|
||||
@router.get("/{source_id}/catalog/{table}")
|
||||
async def get_catalog_table(source_id: str, table: str,
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
@@ -529,6 +623,38 @@ async def review_catalog_item(source_id: str, table: str, payload: CatalogReview
|
||||
return await _detail(session, source, key, entry)
|
||||
|
||||
|
||||
class CatalogBatchReviewItem(BaseModel):
|
||||
table_name: str = Field(min_length=1, max_length=255)
|
||||
path: str = Field(min_length=1, max_length=600)
|
||||
action: Literal["confirm", "reject", "reset"]
|
||||
if_version: int = Field(ge=0)
|
||||
|
||||
|
||||
class CatalogBatchReviewIn(BaseModel):
|
||||
items: list[CatalogBatchReviewItem] = Field(min_length=1, max_length=catalog_review.BATCH_MAX)
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/review/batch")
|
||||
async def review_catalog_batch(source_id: str, payload: CatalogBatchReviewIn,
|
||||
x_actor: str | None = Header(default=None),
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""批量审阅：逐条结果和请求一一对应。逐表按 if_version 乐观锁，同一张表的几条一次写入；一张表冲突、路径不对、表不存在
|
||||
只记在那几条上（ok 为假，code 是 catalog_conflict / catalog_path_invalid / table_missing），不影响别的。"""
|
||||
_, source = await _resolved(session, source_id)
|
||||
items = [catalog_review.BatchItem(i.table_name, i.path, i.action, i.if_version) for i in payload.items]
|
||||
return {"results": await catalog_review.review_batch(session, source, items, actor=actor_of(x_actor))}
|
||||
|
||||
|
||||
@router.get("/{source_id}/catalog/{table}/usage")
|
||||
async def catalog_table_runs(source_id: str, table: str,
|
||||
limit: int = Query(20, ge=1, le=catalog_review.RUNS_LIMIT_MAX),
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
"""用过这张表的运行（total 是几次运行），新的在前；取法同使用次数（catalog.table_usage）。"""
|
||||
_, source = await _resolved(session, source_id)
|
||||
key, _ = await _table_key(session, source, table)
|
||||
return await catalog_review.table_runs(session, source, key, limit=limit)
|
||||
|
||||
|
||||
def _tables_of(source: Any) -> dict[str, Any] | None:
|
||||
"""核对提案用的表结构；没有表结构（尚未探查）时不核对列名。"""
|
||||
return ((source.schema_cache or {}).get("tables") or None) if source is not None else None
|
||||
@@ -568,6 +694,45 @@ async def save_catalog_patch(source_id: str, table: str, payload: CatalogPatchIn
|
||||
return await _detail(session, source, key, entry)
|
||||
|
||||
|
||||
class AssistantMessageIn(BaseModel):
|
||||
role: Literal["user", "assistant"]
|
||||
content: str = Field(max_length=catalog_assistant.MESSAGE_MAX)
|
||||
|
||||
|
||||
class AssistantIn(BaseModel):
|
||||
#: 整段对话（服务端不存）；没有使用者的话时助手先开口
|
||||
messages: list[AssistantMessageIn] = Field(default_factory=list, max_length=catalog_assistant.MAX_MESSAGES)
|
||||
#: 界面上还没保存的修改，让模型知道。不设上限：宽表上几轮攒下来常常超过一次保存的 200 项，不能因此不让问（背景里只列
|
||||
#: 最近的 catalog_assistant.PENDING_SHOWN 项，码值照全部合并）
|
||||
pending: list[CatalogPatchChange] = Field(default_factory=list)
|
||||
|
||||
|
||||
@router.post("/{source_id}/catalog/{table}/assistant")
|
||||
async def catalog_table_assistant(source_id: str, table: str, payload: AssistantIn,
|
||||
session: AsyncSession = Depends(get_session)) -> Any:
|
||||
"""表助手：和使用者对话完善这张表的目录，返回 text/event-stream（事件见 data/catalog_assistant.py）。提案只交给界面，
|
||||
不写目录；保存走 /catalog/{table}/patch。开始前：表不在 404；对话不合规 422；起草模型用不了 409
|
||||
catalog_draft_model_unavailable（说法同起草任务）。"""
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
row, source = await _resolved(session, source_id)
|
||||
key = catalog.resolve_table_name(source.schema_cache, table)
|
||||
if key is None:
|
||||
raise HTTPException(404, f"数据源中没有表 {table}，可能已被删除或尚未探查结构")
|
||||
messages = [m.model_dump() for m in payload.messages]
|
||||
if problem := catalog_assistant.conversation_problem(messages):
|
||||
raise HTTPException(422, problem)
|
||||
try:
|
||||
model, model_id = await catalog_assistant.resolve_model(session)
|
||||
except catalog.CatalogModelUnavailable as e:
|
||||
raise CodedHTTPException(409, str(e), CATALOG_DRAFT_MODEL_UNAVAILABLE) from e
|
||||
context = await catalog_assistant.prepare(session, row, source, key,
|
||||
pending=[c.model_dump() for c in payload.pending])
|
||||
return StreamingResponse(catalog_assistant.stream(model, model_id, context, messages),
|
||||
media_type="text/event-stream",
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"})
|
||||
|
||||
|
||||
@router.get("/{source_id}/catalog/{table}/impact")
|
||||
async def catalog_table_impact(source_id: str, table: str,
|
||||
session: AsyncSession = Depends(get_session)) -> dict[str, Any]:
|
||||
|
||||
+29
-15
@@ -860,8 +860,9 @@ async def save_human_edit(session: AsyncSession, source_id: str, table: str, sub
|
||||
|
||||
#: 新增关联关系的路径：relations.new，值写两端；编号按两端算（relation_id），同一条关系不论谁提都是同一个编号
|
||||
NEW_RELATION_PATH = "relations.new"
|
||||
#: 一条提案最多几项：用户说的是一两条事实，几十项的「提案」多半是模型在重写整张表
|
||||
PATCH_MAX_CHANGES = 20
|
||||
#: 一条提案最多几项（预览、保存都按它）。对话里的纠正通常是一两条事实；表助手（data/catalog_assistant.py）会逐列补中文名、
|
||||
#: 含义，界面上跨几轮攒下的待保存修改一次保存，宽表上常常几十上百项，所以放到 200
|
||||
PATCH_MAX_CHANGES = 200
|
||||
_REASON_MAX = 500
|
||||
#: 看起来像计算公式的文字：「转化率=下单数/访问数」「= SUM(amount)」。公式进口径卡，目录只记数据事实。
|
||||
#: 不认减号：「上线日期=2024-01-01」不是公式。有效记录条件本来就是 SQL 条件，不查它。
|
||||
@@ -2346,27 +2347,27 @@ def query_snapshot_meta(source: Any, sql: str) -> dict[str, Any]:
|
||||
return {"source_id": getattr(source, "id", None), "source": getattr(source, "name", None), "tables": tables}
|
||||
|
||||
|
||||
async def table_usage(session: AsyncSession, source: Any) -> dict[str, int]:
|
||||
"""每张表被运行查询过的次数：{schema_cache 的表名: 次数}，没被查过的表不出现。
|
||||
async def query_snapshot_tables(session: AsyncSession, source: Any) -> list[tuple[str, frozenset[str]]]:
|
||||
"""这个源上属于运行的每一行查询快照引用：(运行 id, 这条查询用到的表——schema_cache 的表名)，没对上任何表的不收。
|
||||
|
||||
取法：数据源查询工具每次成功查询都存一份查询快照工件（tools/datasource.py），工件引用表 artifacts 里
|
||||
按 (内容哈希, 运行) 各记一行。数「属于某次运行（runs 里有这次运行）、属于这个源、SQL 里用到了这张表」的
|
||||
行：同一次运行里结果一模一样的重复查询只算一次，工具库里的试用（不属于任何运行）不算。
|
||||
使用次数（table_usage）和「这张表被哪些运行用过」（data/catalog_review.py）共用这一份取法：数据源查询工具每次成功
|
||||
查询都存一份查询快照工件（tools/datasource.py），工件引用表 artifacts 里按 (内容哈希, 运行) 各记一行；只收属于某次运行
|
||||
（runs 里有这次运行）、属于这个源的行，工具库里的试用（不属于任何运行）不算。
|
||||
|
||||
表名在查询当时就记进了引用的 meta（query_snapshot_meta），这里只读 artifacts 一张表；升级前的老引用
|
||||
没有 meta，退回读工件内容里的源名和 SQL，读过的按工件 id 缓存（内容永不改变）。SQL 里写的表名去掉
|
||||
schema 前缀后按 name_key（只把 ASCII 字母转小写，和 SQLite 比较标识符的口径一致）对到表结构上，对不上的不计。
|
||||
表名在查询当时就记进了引用的 meta（query_snapshot_meta），这里只读 artifacts 一张表；升级前的老引用没有 meta，退回读
|
||||
工件内容里的源名和 SQL，读过的按工件 id 缓存（内容永不改变）。SQL 里写的表名去掉 schema 前缀后按 name_key（只把
|
||||
ASCII 字母转小写，和 SQLite 比较标识符的口径一致）对到表结构上，对不上的不计。
|
||||
"""
|
||||
rows = (await session.execute(
|
||||
select(Artifact.id, Artifact.meta).join(Run, Run.id == Artifact.run_id)
|
||||
select(Artifact.id, Artifact.meta, Artifact.run_id).join(Run, Run.id == Artifact.run_id)
|
||||
.where(Artifact.kind == QUERY_SNAPSHOT_KIND)
|
||||
)).all()
|
||||
tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {}
|
||||
lookup: dict[str, str] = {}
|
||||
for name in tables:
|
||||
lookup.setdefault(name_key(name), name)
|
||||
counts: Counter[str] = Counter()
|
||||
for artifact_id, meta in rows:
|
||||
out: list[tuple[str, frozenset[str]]] = []
|
||||
for artifact_id, meta, run_id in rows:
|
||||
meta = meta if isinstance(meta, dict) else {}
|
||||
if isinstance(meta.get("tables"), list):
|
||||
source_id, source_name, names = meta.get("source_id"), meta.get("source"), meta["tables"]
|
||||
@@ -2378,8 +2379,21 @@ async def table_usage(session: AsyncSession, source: Any) -> dict[str, int]:
|
||||
continue
|
||||
elif source_name != source.name:
|
||||
continue
|
||||
hit = {lookup.get(name_key(str(n).rsplit(".", 1)[-1])) for n in names}
|
||||
counts.update(t for t in hit if t is not None)
|
||||
hit = frozenset(t for t in (lookup.get(name_key(str(n).rsplit(".", 1)[-1])) for n in names) if t is not None)
|
||||
if hit:
|
||||
out.append((str(run_id), hit))
|
||||
return out
|
||||
|
||||
|
||||
async def table_usage(session: AsyncSession, source: Any) -> dict[str, int]:
|
||||
"""每张表被运行查询过的次数：{schema_cache 的表名: 次数}，没被查过的表不出现。
|
||||
|
||||
按查询快照引用的行数（query_snapshot_tables）：同一次运行里结果一模一样的重复查询只算一次，同一次运行里不同的查询
|
||||
各算一次，工具库里的试用不算。
|
||||
"""
|
||||
counts: Counter[str] = Counter()
|
||||
for _run, hit in await query_snapshot_tables(session, source):
|
||||
counts.update(hit)
|
||||
return dict(counts)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,911 @@
|
||||
"""表助手：在表详情页和使用者对话，一起完善一张表的数据目录（POST /catalog/{table}/assistant，SSE）。
|
||||
|
||||
**为什么要有它。** 模型起草只看表结构，缩写列名、状态码的意思常常拿不准而留空；审阅时人知道答案，却要一项一项填。表助手
|
||||
把这张表的结构、目录现状（每项的来源和状态）、关联关系、剖析结果、起草要求、使用次数、上次起草留空的列交给模型，让它主动
|
||||
问拿不准的地方；使用者说出一条业务事实，它整理成成批的修改提案（形状同助手流里的 catalog_patch），使用者在界面上逐项看过、
|
||||
一次保存（PATCH /catalog/{table}/patch，记为人工确认）。
|
||||
|
||||
**只改目录、不改业务库。** 模型只有两个工具：
|
||||
- propose_changes：一批修改提案。按 catalog.plan_patch 核对（取值格式、列名必须在表结构里、公式不收），遮罩列不给码值，已经
|
||||
是这个值又确认过的不提；有效记录条件是模型写的 SQL，另按起草的规矩核对（_Tools.filter_problem：一个条件、只用这张表的
|
||||
列、不能有子查询……）。核对过的作为 proposal 事件交给界面，**不写目录**。没收下的项和原因交回模型（不带任何取值）。
|
||||
界面上还没保存的修改（pending）里同一列的码值并进新提的码值：界面上同一路径以后一次为准，不能把先提的码值丢了。
|
||||
- run_query：只读查询，只有数据源开了数据剖析才给。经剖析的只读守卫和单条时限（catalog_profile._Runner：
|
||||
guard.check(readonly=True)，单条时限取剖析设置和数据源查询时限中较小的一个），每次请求最多 MAX_QUERIES 条（守卫拒掉的
|
||||
也算），最多 QUERY_ROWS 行交给模型、SHOWN_ROWS 行交给界面。结果交给模型、界面之前先遮（engine/masking.mask_query_result：
|
||||
取自遮罩列的结果列整列换成「已遮罩」，改了名字的也认；这一次请求里前面遮掉的值按值换），报错原话里的值也换。没开剖析时
|
||||
提示词里写明只能根据表结构和目录推断。
|
||||
|
||||
**遮罩列只给列名。** 表结构里遮罩的列只写列名和类型（不给注释），目录里这些列的码值、带着取值的备注一律不给（剖析的备注里
|
||||
有取值范围）；界面上还没保存的修改里这些列的码值也不给。
|
||||
|
||||
**模型。** 取模型的顺序同起草（data/draft_model.py：起草模型 → 助手的模型 → 默认接入），用不了时的说法也同起草
|
||||
（先过 catalog.resolve_draft_model，接口回 409 catalog_draft_model_unavailable）。保留思考、温度 0，额度
|
||||
catalog.MODEL_MAX_TOKENS，服务方嫌大时退回 catalog.MODEL_FALLBACK_MAX_TOKENS（providers/budget.BudgetFallback）；单次调用
|
||||
最多等 catalog.MODEL_TIMEOUT_S（同起草）。网络错误、429、5xx 在还没输出任何东西时重试（最多 len(RETRY_DELAYS_S) 次）。
|
||||
一次请求最多调用 MAX_ROUNDS 次模型（查询、提案之后接着答）；倒数第二轮的工具结果里提醒它直接回答。
|
||||
|
||||
**事件**（每个事件一行 data: <JSON>）：
|
||||
- {"type": "text", "delta"}：回答正文的增量。几轮之间补一个空行；
|
||||
- {"type": "query", "sql", "columns", "rows"（前 SHOWN_ROWS 行）, "row_count", "truncated", "error"}：跑了一条只读查询；
|
||||
- {"type": "proposal", "source", "source_id", "table", "table_label", "version", "changes": [{path, label, before,
|
||||
before_status, after, value, reason, state, note?}]}：一批核对过的修改提案；
|
||||
- {"type": "done", "usage": {input_tokens, output_tokens}, "queries"}；
|
||||
- {"type": "error", "message", "code"}：中途出错，流到此为止（之后没有 done）。code：assistant_model_error 调用模型出错、
|
||||
assistant_truncated 输出到了额度上限被截断、assistant_timeout 单次调用超过时限、assistant_rounds 调用次数用完还没答完、
|
||||
assistant_failed 意料之外的错误。
|
||||
等模型的时候每 HEARTBEAT_S 秒发一行 SSE 注释（「: keep-alive」，不是事件），免得中间的代理断开。
|
||||
|
||||
**对话不存服务端**：每次请求带上整段对话（最多 MAX_MESSAGES 条、每条最多 MESSAGE_MAX 字）。查询结果只在这一次请求里
|
||||
看得到，提示词里让模型把要用的结论写进回答。界面上待保存的修改不设上限（宽表上逐列补中文名、含义，几轮下来常常超过一次
|
||||
保存的 200 项，不能因此不让问）：背景里只列最近的 PENDING_SHOWN 项（界面上新的提案接在后面），写明另有几项；码值照全部
|
||||
待保存的修改合并。
|
||||
|
||||
写给模型的文字都在名字带 _SYSTEM / _PROMPT 的常量里；这里的 ERROR_* 是给人看的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import AsyncIterator, Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.data import catalog, catalog_jobs, catalog_profile, sqlcheck
|
||||
from app.data.catalog_review import SOURCE_LABEL, item_label, slot_path, table_usage_counts
|
||||
from app.data.catalog_transfer import filter_columns_problem
|
||||
from app.data.draft_model import DRAFT_MODEL_PLACE
|
||||
from app.data.engine import SnapshotTampered, masked_columns
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: 对话最多几条、每条最多几个字；界面上待保存的修改在背景里最多列几项（最近的；请求里交多少都收）
|
||||
MAX_MESSAGES = 40
|
||||
MESSAGE_MAX = 4000
|
||||
PENDING_SHOWN = 200
|
||||
#: 每次请求最多调用几次模型、最多几条只读查询
|
||||
MAX_ROUNDS = 8
|
||||
MAX_QUERIES = 5
|
||||
#: 一条查询最多几行交给模型、几行交给界面；交给模型时每格最多几个字、一条结果最多几个字
|
||||
QUERY_ROWS = 50
|
||||
SHOWN_ROWS = 10
|
||||
CELL_MAX = 300
|
||||
TOOL_CHARS = 12_000
|
||||
#: 网络错误、429、5xx 重试前等几秒（依次取用，个数就是最多重试几次）。在对话里等，比起草（10、30 秒）短
|
||||
RETRY_DELAYS_S = (3.0, 10.0)
|
||||
#: 等模型的时候每隔几秒发一行 SSE 注释
|
||||
HEARTBEAT_S = 10.0
|
||||
#: 背景里「目录里还没有中文名和含义的列」最多列几个；别的表指过来的关联关系最多列几条；这张表的目录最多列几项
|
||||
#: （几百列的宽表全部起草过有上千项，全列出来提示词就太长了：表级的项、关联关系在前，列的项在后）
|
||||
UNNAMED_SHOWN = 30
|
||||
INCOMING_SHOWN = 30
|
||||
CATALOG_ITEMS_SHOWN = 600
|
||||
|
||||
QUERY_TOOL = "run_query"
|
||||
PROPOSE_TOOL = "propose_changes"
|
||||
|
||||
#: 错误事件的机读码
|
||||
CODE_MODEL = "assistant_model_error"
|
||||
CODE_TRUNCATED = "assistant_truncated"
|
||||
CODE_TIMEOUT = "assistant_timeout"
|
||||
CODE_ROUNDS = "assistant_rounds"
|
||||
CODE_FAILED = "assistant_failed"
|
||||
|
||||
# ---- 给人看的
|
||||
ERROR_MODEL = ("调用模型「{model}」失败：{reason}。请稍后重试；如持续失败，请在{place}中换一个模型，"
|
||||
"或到「设置 → 模型接入」测试该模型")
|
||||
ERROR_TRUNCATED = "{cut}，回答没有写完。通常是思考占满了输出额度，可以把问题拆小一点再问，或在{place}中换一个模型"
|
||||
ERROR_TIMEOUT = "模型 {seconds} 秒内没有回答完，已停止这一轮。可以把问题拆小一点再问，或在{place}中换一个模型"
|
||||
ERROR_ROUNDS = "这一轮已经调用了 {n} 次模型仍没有答完，已停止。可以换个问法，或者分几次问"
|
||||
ERROR_FAILED = "表助手出错：{line}"
|
||||
#: 对话的最后一条是助手说的（接口 422）
|
||||
LAST_NOT_USER = "对话的最后一条应当是使用者的话，请重新发送"
|
||||
#: 起草模型刚才还能用、建流式模型时却用不了（期间设置被改了）时的说法
|
||||
ERROR_NO_MODEL = "起草模型暂时无法使用，请稍后重试，或到「设置 → 模型接入」检查"
|
||||
#: 查询事件里的报错
|
||||
QUERY_TAMPERED = "数据源「{name}」的{error}。请联系管理员检查数据目录"
|
||||
|
||||
# ---- 写给模型的
|
||||
ASSISTANT_SYSTEM = """你是 AgentLab 的「表助手」，和使用者一起完善一张表的数据目录：表和列的中文名、含义、单位、度量类型、码值、\
|
||||
粒度、表类型、业务主键、业务日期、有效记录条件、去重规则、关联关系。
|
||||
|
||||
规矩：
|
||||
- 你只改数据目录，不改业务库：不能写数据、建表、改结构。
|
||||
- 只记数据事实，不写计算公式：「转化率=下单数/访问数」这类指标怎么算的说法属于口径卡，不进数据目录。
|
||||
- 拿不准就问，不要猜。一次只问一两件最要紧的事，问得具体（例如「status 为 9 的行是不是作废的单子？」）。目录里留空的列、\
|
||||
标着「推断」的项是最值得问的。
|
||||
- 使用者说出一条业务事实时，用 propose_changes 把它整理成修改提案，reason 里引用使用者的原话（例如「使用者说：9 是作废」）。\
|
||||
一次可以提多项。提案只是建议：使用者在界面上看过、保存之后才写进数据目录，记为人工确认。
|
||||
- 表名、列名照原样写，只能用下面表结构里有的列。
|
||||
- 标着「已确认」的项由人工确认过，使用者没有明确要改就不要提修改；标着「已驳回」的项使用者驳回过，不要再提。
|
||||
- 「界面上还没保存的修改」是使用者已经接受、还没保存的提案：不要重复提；要改其中一项就再提一次，写完整的新值。
|
||||
- 遮罩的列只给了列名：不要猜它的取值，也不要给它写码值。
|
||||
- 回答用中文，简短直接。查询结果只在这一轮看得到，后面要用的结论请写进回答里。
|
||||
|
||||
修改提案的写法（propose_changes 的 changes，每项 path、value、reason）：
|
||||
- path：表级项直接写字段名（label 中文名、description 说明、grain 粒度、keys 业务主键的列名列表、kind 表类型 \
|
||||
fact/dimension/snapshot/log/config、business_date {"column","rule","timezone"}、valid_filter 有效记录的 SQL 条件、dedup 去重\
|
||||
规则）；列级项写 columns.<列名>.<字段>（label、meaning 含义、unit 单位、measure 度量类型 \
|
||||
flow/stock/ratio/identifier/status/attribute、codes 码值 {"码":"含义"}）；已有的关联关系写 relations.<编号>；新增关联关系\
|
||||
写 relations.new，value 写 {"columns":[本表列],"to_table":"目标表","to_columns":[目标列],\
|
||||
"cardinality":"many_to_one/one_to_one/one_to_many"}。
|
||||
- codes 只写要补充或更正的码，原有的码会保留；valid_filter 要写成完整的条件（原有条件也要带上）。
|
||||
- valid_filter 只能用这张表的列（不带表名），由比较、IN、BETWEEN、LIKE、IS NULL 用 AND / OR / NOT 连起来，例如 \
|
||||
status NOT IN ('C', 'X')；不能有子查询、聚合、窗口函数、分号，函数只能用 UPPER、LOWER、TRIM、COALESCE。"""
|
||||
|
||||
#: 查询工具的说明（开了剖析时接在系统提示后面）、没开剖析时的那句
|
||||
ASSISTANT_QUERY_PROMPT = """只读查询：run_query 在这个数据源上执行一条只读 SQL（{dialect} 的写法），最多返回 {rows} 行，\
|
||||
这一轮最多 {limit} 条。用它核对码值的取值和分布、日期范围、关联对不对得上；不要取明细的个人信息。遮罩列的值显示为「已遮罩」。"""
|
||||
ASSISTANT_NO_QUERY_PROMPT = "这个数据源没开剖析，不能查数据，只能根据表结构和目录推断。"
|
||||
ASSISTANT_DISABLED_PROMPT = "这个数据源已停用，不能查数据，只能根据表结构和目录推断。"
|
||||
#: 使用者还没说话（刚打开表助手）时补的那句
|
||||
ASSISTANT_OPENING_PROMPT = ("（使用者刚打开表助手，还没有说话。）请用一两句话说明这张表的目录现在最缺什么，然后就最要紧的一两项"
|
||||
"向使用者提问；先不要提修改提案。")
|
||||
#: 对话从助手的开场白开始时，补在最前面的一句（各家的接口都要求从使用者这一边开始）
|
||||
ASSISTANT_OPENED_PROMPT = "（使用者打开了表助手。）"
|
||||
|
||||
#: 背景各段的写法
|
||||
ASSISTANT_PROMPT_LABELS = {
|
||||
"head": "下面是要完善的表。",
|
||||
"table": "数据源「{source}」（{kind}），表 {table}{view}",
|
||||
"view": "（视图）",
|
||||
"comment": "数据库注释：{text}",
|
||||
"system": "系统说明：{text}",
|
||||
"usage": "这张表被 {runs} 次运行用到（查询 {n} 次）。",
|
||||
"usage_none": "这张表还没有被运行用到过。",
|
||||
"description": "数据源说明：{text}",
|
||||
"guidance": "数据源的起草要求（使用者补充的背景和要求）：{text}",
|
||||
"structure": "表结构：",
|
||||
"column": "- {name} {type}{marks}",
|
||||
"pk": "主键",
|
||||
"not_null": "非空",
|
||||
"col_comment": "注释：{text}",
|
||||
"masked": "遮罩列：只给列名，不给任何取值",
|
||||
"unique": "唯一约束：{cols}",
|
||||
"fk": "外键：{cols} → {target}",
|
||||
"catalog": "这张表现在的数据目录（每项写明来源和状态；路径就是修改提案里的 path）：",
|
||||
"catalog_empty": "这张表还没有数据目录。",
|
||||
"item": "- {path}（{label}）：{value}〔{source}，{status}〕",
|
||||
"item_note": " 备注：{text}",
|
||||
"catalog_more": "（还有 {n} 项没有列出：这张表的目录太长。要改的项照上面的路径写法提，核对时按当前目录算）",
|
||||
"relation": "{columns} → {to_table}.{to_columns}{extra}",
|
||||
"relation_extra": "（{text}）",
|
||||
"cardinality": "基数 {value}",
|
||||
"coverage": "覆盖率 {value}",
|
||||
"incoming": "别的表指向这张表的关联关系：",
|
||||
"incoming_item": "- {table}.{columns} → {target}.{to_columns}〔{source}，{status}〕",
|
||||
"blank": "最近一次起草时模型拿不准、留空的列：{cols}（共 {n} 列）",
|
||||
"unnamed": "目录里还没有中文名和含义的列：{cols}（共 {n} 列{more}）",
|
||||
"unnamed_more": "，这里列出前 {n} 个",
|
||||
"pending": "界面上还没保存的修改（使用者已经接受、还没保存）：",
|
||||
"pending_more": "（另有 {n} 项更早的待保存修改没有列出：太多了。它们同样已经接受，不要重复提）",
|
||||
"pending_item": "- {path}：{value}（依据：{reason}）",
|
||||
"status_proposed": "推断",
|
||||
"status_verified": "有确证",
|
||||
"status_confirmed": "已确认",
|
||||
"status_rejected": "已驳回",
|
||||
"field_relation": "关联关系",
|
||||
}
|
||||
|
||||
#: 工具结果的写法（交回模型）
|
||||
ASSISTANT_TOOL_PROMPT = {
|
||||
"proposed": "已把 {n} 项修改交给使用者确认（使用者保存后才写进数据目录）：{paths}",
|
||||
"nothing": "没有可以交给使用者的修改。",
|
||||
"dropped": "没有收下的项：{reasons}",
|
||||
"masked_codes": "列 {column} 在数据源设置中被遮罩，不能给它写码值",
|
||||
"same": "{where}已经是这个值、也确认过了，不用再提",
|
||||
"no_changes": "提案里没有要修改的项",
|
||||
"query_failed": "查询失败：{error}",
|
||||
"query_capped": "未执行：这一轮最多查 {n} 条，不能再查了。请根据已经查到的结果回答",
|
||||
"query_disabled": "未执行：这个数据源没开数据剖析，不能查数据",
|
||||
"unknown_tool": "没有名为 {name} 的工具，可用的工具：{tools}",
|
||||
"last_round": "（这一轮只剩最后一步了：请直接回答使用者，不要再调用工具。）",
|
||||
"empty_sql": "未执行：没有给出 SQL",
|
||||
}
|
||||
|
||||
QUERY_TOOL_PROMPT: dict[str, Any] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": QUERY_TOOL,
|
||||
"description": "在这个数据源上执行一条只读 SQL 查询，返回列名和前几行（遮罩列的值显示为「已遮罩」）。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"sql": {"type": "string", "description": "一条只读的 SELECT 语句"}},
|
||||
"required": ["sql"],
|
||||
},
|
||||
},
|
||||
}
|
||||
PROPOSE_TOOL_PROMPT: dict[str, Any] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": PROPOSE_TOOL,
|
||||
"description": "把使用者说出的业务事实整理成一批数据目录修改提案，交给使用者确认后保存。不会直接写进数据目录。",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"changes": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {"type": "string", "description": "要改的项，写法见系统提示"},
|
||||
"value": {"description": "新的值：文字、列名列表、码值对照或关联关系"},
|
||||
"reason": {"type": "string", "description": "一句话依据，引用使用者的原话"},
|
||||
},
|
||||
"required": ["path", "value", "reason"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["changes"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 模型
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
async def resolve_model(session: AsyncSession) -> tuple[Any, str]:
|
||||
"""(模型, 模型 id)。能不能用、用不了时的说法和起草完全一致：先过 catalog.resolve_draft_model（起草任务建任务时也是这样
|
||||
查的），用不了抛 catalog.CatalogModelUnavailable。再按同一个接入、模型建流式对话用的模型：保留思考、温度 0，大小两份额度
|
||||
（BudgetFallback），单次时限同起草。只建模型对象，不发请求。"""
|
||||
from app.data.draft_model import DraftModelUnavailable, draft_unavailable, resolve_draft_provider
|
||||
from app.providers.budget import BudgetFallback
|
||||
from app.providers.factory import ModelSpec, ProviderNotConfigured, build_chat_model
|
||||
|
||||
await catalog.resolve_draft_model(session)
|
||||
try:
|
||||
provider, choice = await resolve_draft_provider(session, none_reason=ERROR_NO_MODEL, mock_reason=ERROR_NO_MODEL)
|
||||
except DraftModelUnavailable as e:
|
||||
raise catalog.CatalogModelUnavailable(str(e)) from e
|
||||
# SDK 自己不重试（同起草）：网络错误、429、5xx 由这里在还没输出时重试，两层叠起来实际次数就对不上了
|
||||
spec = ModelSpec(provider=choice.provider, model=choice.model, max_tokens=catalog.MODEL_MAX_TOKENS, temperature=0,
|
||||
timeout=int(catalog.MODEL_TIMEOUT_S), max_retries=0)
|
||||
try:
|
||||
big = build_chat_model(provider, spec)
|
||||
small = build_chat_model(provider, spec.model_copy(update={"max_tokens": catalog.MODEL_FALLBACK_MAX_TOKENS}))
|
||||
except ProviderNotConfigured as e:
|
||||
raise catalog.CatalogModelUnavailable(draft_unavailable(e, choice.origin)) from e
|
||||
model_id = spec.model or provider.default_model or ""
|
||||
return BudgetFallback(big, small, limits=(catalog.MODEL_MAX_TOKENS, catalog.MODEL_FALLBACK_MAX_TOKENS)), model_id
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 对话
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def conversation_problem(messages: Iterable[Mapping[str, Any]]) -> str | None:
|
||||
"""对话本身的问题（接口 422）：使用者说过话、最后一条却是助手说的。没有使用者的话是「刚打开表助手」，不算问题。"""
|
||||
said = [m for m in messages if str(m.get("content") or "").strip()]
|
||||
if any(m.get("role") == "user" for m in said) and said[-1].get("role") != "user":
|
||||
return LAST_NOT_USER
|
||||
return None
|
||||
|
||||
|
||||
def _history(messages: Iterable[Mapping[str, Any]]) -> list[Any]:
|
||||
"""对话 → 模型的消息：空的不要，同一方接连说的并成一条；从助手的开场白开始的，前面补一句使用者打开了表助手；使用者还没
|
||||
说话的，补一句开场的请求。"""
|
||||
from langchain_core.messages import AIMessage, HumanMessage
|
||||
|
||||
merged: list[tuple[str, str]] = []
|
||||
for m in messages:
|
||||
text = str(m.get("content") or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
role = "user" if m.get("role") == "user" else "assistant"
|
||||
if merged and merged[-1][0] == role:
|
||||
merged[-1] = (role, f"{merged[-1][1]}\n\n{text}")
|
||||
else:
|
||||
merged.append((role, text))
|
||||
out: list[Any] = []
|
||||
if merged and merged[0][0] == "assistant":
|
||||
out.append(HumanMessage(content=ASSISTANT_OPENED_PROMPT))
|
||||
for role, text in merged:
|
||||
out.append(HumanMessage(content=text) if role == "user" else AIMessage(content=text))
|
||||
if not any(role == "user" for role, _ in merged):
|
||||
out.append(HumanMessage(content=ASSISTANT_OPENING_PROMPT))
|
||||
return out
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 背景
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def _value_text(value: Any) -> str:
|
||||
return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def _status_text(status: Any) -> str:
|
||||
return ASSISTANT_PROMPT_LABELS.get(f"status_{status}", str(status))
|
||||
|
||||
|
||||
def _source_text(source: Any) -> str:
|
||||
return SOURCE_LABEL.get(str(source), str(source))
|
||||
|
||||
|
||||
def _relation_line(rel: Mapping[str, Any]) -> str:
|
||||
labels = ASSISTANT_PROMPT_LABELS
|
||||
extra = []
|
||||
if rel.get("cardinality"):
|
||||
extra.append(labels["cardinality"].format(value=rel["cardinality"]))
|
||||
coverage = rel.get("coverage")
|
||||
if isinstance(coverage, (int, float)) and not isinstance(coverage, bool):
|
||||
extra.append(labels["coverage"].format(value=f"{coverage:.0%}"))
|
||||
return labels["relation"].format(
|
||||
columns=", ".join(str(c) for c in rel.get("columns") or []), to_table=rel.get("to_table"),
|
||||
to_columns=", ".join(str(c) for c in rel.get("to_columns") or []),
|
||||
extra=labels["relation_extra"].format(text="，".join(extra)) if extra else "")
|
||||
|
||||
|
||||
def _touches_masked(key: catalog.Slot, item: Mapping[str, Any], masked: set[str]) -> bool:
|
||||
"""这一项说的是不是遮罩列：列级的项看列名，业务主键、业务日期看指着的列，关联关系看本表这一端。"""
|
||||
if key[0] == "c":
|
||||
return key[1].lower() in masked
|
||||
if key[0] == "r":
|
||||
return any(str(c).lower() in masked for c in item.get("columns") or [])
|
||||
value = item.get("value")
|
||||
if key == ("t", "business_date") and isinstance(value, Mapping):
|
||||
return str(value.get("column") or "").lower() in masked
|
||||
if key == ("t", "keys") and isinstance(value, list):
|
||||
return any(str(c).lower() in masked for c in value)
|
||||
return False
|
||||
|
||||
|
||||
def _catalog_lines(notes: Mapping[str, Any] | None, masked: set[str]) -> list[str]:
|
||||
"""这张表的目录，每项一行：路径、字段、值、来源和状态，有备注的另起一行。遮罩列上的码值不给，说到遮罩列的项不给备注
|
||||
（剖析的备注里有取值范围）。"""
|
||||
labels = ASSISTANT_PROMPT_LABELS
|
||||
lines: list[str] = []
|
||||
# 表级的项、关联关系在前，列的项在后（稳定排序，各自照目录里的顺序）；遮罩列上的码值不算
|
||||
slots = [(key, item) for key, item in sorted(catalog._slots(notes).items(), key=lambda kv: kv[0][0] == "c")
|
||||
if not (key[0] == "c" and key[2] == "codes" and key[1].lower() in masked)]
|
||||
for shown, (key, item) in enumerate(slots):
|
||||
if shown >= CATALOG_ITEMS_SHOWN:
|
||||
lines.append(labels["catalog_more"].format(n=len(slots) - shown))
|
||||
break
|
||||
hidden = _touches_masked(key, item, masked)
|
||||
if key[0] == "r":
|
||||
label, value = labels["field_relation"], _relation_line(item)
|
||||
else:
|
||||
label = (catalog.TABLE_FIELD_LABEL if key[0] == "t" else catalog.COLUMN_FIELD_LABEL).get(key[-1], key[-1])
|
||||
value = _value_text(item.get("value"))
|
||||
lines.append(labels["item"].format(path=slot_path(key), label=label, value=value,
|
||||
source=_source_text(item.get("source")),
|
||||
status=_status_text(item.get("status"))))
|
||||
note = item.get("note")
|
||||
if isinstance(note, str) and note.strip() and not hidden:
|
||||
lines.append(labels["item_note"].format(text=note.strip()))
|
||||
return lines
|
||||
|
||||
|
||||
def _structure_lines(meta: Mapping[str, Any], masked: set[str], *, system: bool) -> list[str]:
|
||||
labels = ASSISTANT_PROMPT_LABELS
|
||||
pk = {str(c) for c in meta.get("primary_key") or []}
|
||||
lines = [labels["structure"]]
|
||||
for col in meta.get("columns") or []:
|
||||
if not isinstance(col, Mapping) or col.get("name") is None:
|
||||
continue
|
||||
name = str(col["name"])
|
||||
marks: list[str] = []
|
||||
if name.lower() in masked:
|
||||
marks.append(labels["masked"])
|
||||
else:
|
||||
if name in pk:
|
||||
marks.append(labels["pk"])
|
||||
if not col.get("nullable", True):
|
||||
marks.append(labels["not_null"])
|
||||
text = str(col.get("comment") or "").strip()
|
||||
if text:
|
||||
marks.append((labels["system"] if system else labels["col_comment"]).format(text=text))
|
||||
lines.append(labels["column"].format(name=name, type=col.get("type") or "",
|
||||
marks=("，" + "，".join(marks)) if marks else ""))
|
||||
for group in meta.get("unique") or []:
|
||||
if isinstance(group, list) and group:
|
||||
lines.append(labels["unique"].format(cols="、".join(str(c) for c in group)))
|
||||
for fk in meta.get("foreign_keys") or []:
|
||||
if isinstance(fk, Mapping) and fk.get("columns") and fk.get("to_table"):
|
||||
target = ", ".join(f"{fk['to_table']}.{c}" for c in fk.get("to_columns") or []) or str(fk["to_table"])
|
||||
lines.append(labels["fk"].format(cols=", ".join(str(c) for c in fk["columns"]), target=target))
|
||||
return lines
|
||||
|
||||
|
||||
async def _last_blank(session: AsyncSession, source_id: str, table: str) -> tuple[int, list[str]]:
|
||||
"""最近一次结束于 ok / partial 的起草里这张表模型留空的列：(几列, 最多 5 个列名)。停止、中断的那次不算（那次没起草完，
|
||||
留空数是 0）。没有是 (0, [])。"""
|
||||
from app.db.models import CatalogDraftJob, CatalogDraftTask
|
||||
|
||||
row = (await session.execute(
|
||||
select(CatalogDraftTask.blank_columns, CatalogDraftTask.blank_sample)
|
||||
.join(CatalogDraftJob, CatalogDraftJob.id == CatalogDraftTask.job_id)
|
||||
.where(CatalogDraftJob.source_id == source_id, CatalogDraftTask.table_name == table,
|
||||
CatalogDraftTask.status.in_(("ok", "partial")))
|
||||
.order_by(CatalogDraftJob.created_at.desc(), CatalogDraftTask.seq).limit(1)
|
||||
)).first()
|
||||
if row is None:
|
||||
return 0, []
|
||||
return int(row[0] or 0), [str(c) for c in row[1] or []]
|
||||
|
||||
|
||||
@dataclass
|
||||
class AssistantContext:
|
||||
"""一次请求要用到的东西，开流之前读好（流里不再用请求的会话）。"""
|
||||
|
||||
source_id: str
|
||||
source_name: str
|
||||
kind: str
|
||||
table: str
|
||||
tables: dict[str, Any]
|
||||
#: 读表结构、发查询用的源（上传源是绑定当前快照的视图）
|
||||
query_source: Any
|
||||
options: dict[str, Any]
|
||||
masked: set[str]
|
||||
can_query: bool
|
||||
settings: catalog_profile.ProfileSettings
|
||||
system: str
|
||||
pending_codes: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
|
||||
def tools(self) -> list[dict[str, Any]]:
|
||||
return [QUERY_TOOL_PROMPT, PROPOSE_TOOL_PROMPT] if self.can_query else [PROPOSE_TOOL_PROMPT]
|
||||
|
||||
|
||||
async def prepare(session: AsyncSession, row: Any, source: Any, table: str,
|
||||
pending: Iterable[Mapping[str, Any]] = ()) -> AssistantContext:
|
||||
"""读好这次请求要用的背景：表结构、目录、别的表指过来的关联关系、使用次数、起草要求、留空的列、待保存的修改，拼成系统
|
||||
提示。row 是数据源记录（遮罩、剖析开关、起草要求按它），source 是读表结构和发查询用的源。table 是表结构里的键。"""
|
||||
labels = ASSISTANT_PROMPT_LABELS
|
||||
cache = getattr(source, "schema_cache", None) or {}
|
||||
tables = cache.get("tables") or {}
|
||||
meta = tables.get(table) or {}
|
||||
options = dict(getattr(row, "options", None) or {})
|
||||
masked = {c.lower() for c in masked_columns(options)}
|
||||
settings = catalog_profile.profile_settings(options)
|
||||
enabled = getattr(row, "enabled", True) is not False
|
||||
can_query = settings.enabled and enabled
|
||||
system_notes = catalog.system_notes_source(source)
|
||||
entries = await catalog.read_catalog(session, source.id)
|
||||
entry = entries.get(table)
|
||||
notes = entry.notes if entry else {}
|
||||
usage = await table_usage_counts(session, source)
|
||||
used, runs = usage.queries.get(table, 0), usage.runs.get(table, 0)
|
||||
blank_n, blank_cols = await _last_blank(session, source.id, table)
|
||||
|
||||
lines = [labels["head"], labels["table"].format(source=row.name, kind=row.kind, table=table,
|
||||
view=labels["view"] if meta.get("is_view") else "")]
|
||||
comment = str(meta.get("comment") or "").strip()
|
||||
if comment:
|
||||
lines.append((labels["system"] if system_notes else labels["comment"]).format(text=comment))
|
||||
lines.append(labels["usage"].format(runs=runs, n=used) if used else labels["usage_none"])
|
||||
description = str(getattr(row, "description", "") or "").strip()
|
||||
if description:
|
||||
lines.append(labels["description"].format(text=description))
|
||||
if guidance := catalog_jobs.guidance_of(row):
|
||||
lines.append(labels["guidance"].format(text=guidance))
|
||||
lines += ["", *_structure_lines(meta, masked, system=system_notes), ""]
|
||||
current = _catalog_lines(notes, masked)
|
||||
lines += [labels["catalog"], *current] if current else [labels["catalog_empty"]]
|
||||
incoming = []
|
||||
for other, other_entry in sorted(entries.items()):
|
||||
if other == table:
|
||||
continue
|
||||
for key, rel in catalog._slots(other_entry.notes).items():
|
||||
if key[0] == "r" and rel.get("to_table") == table:
|
||||
incoming.append(labels["incoming_item"].format(
|
||||
table=other, columns=", ".join(str(c) for c in rel.get("columns") or []), target=table,
|
||||
to_columns=", ".join(str(c) for c in rel.get("to_columns") or []),
|
||||
source=_source_text(rel.get("source")), status=_status_text(rel.get("status"))))
|
||||
if incoming:
|
||||
lines += ["", labels["incoming"], *incoming[:INCOMING_SHOWN]]
|
||||
named = catalog.visible_notes(notes).get("columns") or {}
|
||||
columns = [str(c["name"]) for c in meta.get("columns") or [] if isinstance(c, Mapping) and c.get("name") is not None]
|
||||
unnamed = [c for c in columns if not any(f in (named.get(c) or {}) for f in ("label", "meaning"))]
|
||||
blank = [c for c in blank_cols if c in unnamed]
|
||||
if blank_n and blank:
|
||||
lines += ["", labels["blank"].format(cols="、".join(blank), n=blank_n)]
|
||||
if unnamed:
|
||||
more = labels["unnamed_more"].format(n=UNNAMED_SHOWN) if len(unnamed) > UNNAMED_SHOWN else ""
|
||||
lines.append(labels["unnamed"].format(cols="、".join(unnamed[:UNNAMED_SHOWN]), n=len(unnamed), more=more))
|
||||
pending_codes: dict[str, dict[str, Any]] = {}
|
||||
shown_pending: list[str] = []
|
||||
for change in pending:
|
||||
path = str(change.get("path") or "").strip()
|
||||
value = change.get("value")
|
||||
try:
|
||||
key = catalog.parse_path(path)
|
||||
except catalog.CatalogPathError:
|
||||
continue
|
||||
if key[0] == "c" and key[2] == "codes":
|
||||
if key[1].lower() in masked:
|
||||
continue
|
||||
if isinstance(value, Mapping):
|
||||
pending_codes[path] = dict(value)
|
||||
reason = str(change.get("reason") or "").strip()
|
||||
shown_pending.append(labels["pending_item"].format(path=path, value=_value_text(value), reason=reason or "—"))
|
||||
if shown_pending:
|
||||
lines += ["", labels["pending"]]
|
||||
if len(shown_pending) > PENDING_SHOWN:
|
||||
lines.append(labels["pending_more"].format(n=len(shown_pending) - PENDING_SHOWN))
|
||||
lines += shown_pending[-PENDING_SHOWN:]
|
||||
if can_query:
|
||||
tools = ASSISTANT_QUERY_PROMPT.format(dialect=row.kind, rows=QUERY_ROWS, limit=MAX_QUERIES)
|
||||
else:
|
||||
tools = ASSISTANT_NO_QUERY_PROMPT if enabled or not settings.enabled else ASSISTANT_DISABLED_PROMPT
|
||||
system = "\n\n".join([ASSISTANT_SYSTEM, tools, "\n".join(lines)])
|
||||
return AssistantContext(source_id=source.id, source_name=row.name, kind=str(row.kind or ""), table=table,
|
||||
tables=tables, query_source=source, options=options, masked=masked, can_query=can_query,
|
||||
settings=settings, system=system, pending_codes=pending_codes)
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 工具
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def _cell(value: Any, limit: int | None = None) -> Any:
|
||||
"""一格交出去的样子：数、布尔、空照原样，别的写成文字（日期、Decimal……），长的截断。"""
|
||||
if value is None or isinstance(value, (bool, int, float)):
|
||||
return value
|
||||
if isinstance(value, (datetime, date)):
|
||||
text = value.isoformat()
|
||||
elif isinstance(value, Decimal):
|
||||
text = str(value)
|
||||
elif isinstance(value, (bytes, bytearray)):
|
||||
text = f"<{len(value)} 字节>"
|
||||
else:
|
||||
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str)
|
||||
if limit is not None and len(text) > limit:
|
||||
return text[:limit] + "…"
|
||||
return text
|
||||
|
||||
|
||||
def _rows(rows: Any, n: int, limit: int | None) -> list[list[Any]]:
|
||||
out: list[list[Any]] = []
|
||||
for row in (rows or [])[:n]:
|
||||
cells = row if isinstance(row, list) else list(row.values()) if isinstance(row, dict) else [row]
|
||||
out.append([_cell(v, limit) for v in cells])
|
||||
return out
|
||||
|
||||
|
||||
class _Tools:
|
||||
"""一次请求里的工具：查询次数、这一次遮掉的值（后面的查询按值换）。"""
|
||||
|
||||
def __init__(self, ctx: AssistantContext) -> None:
|
||||
from app.engine.masking import SeenValues
|
||||
|
||||
self.ctx = ctx
|
||||
self.queries = 0
|
||||
self.seen = SeenValues()
|
||||
|
||||
# ---------------------------------------------------------------- 查询
|
||||
|
||||
async def query(self, sql: Any) -> tuple[dict[str, Any] | None, str]:
|
||||
"""(给界面的 query 事件, 交回模型的结果)。"""
|
||||
prompts = ASSISTANT_TOOL_PROMPT
|
||||
if not self.ctx.can_query:
|
||||
return None, prompts["query_disabled"]
|
||||
if self.queries >= MAX_QUERIES:
|
||||
return None, prompts["query_capped"].format(n=MAX_QUERIES)
|
||||
sql = sql.strip() if isinstance(sql, str) else ""
|
||||
if not sql:
|
||||
return None, prompts["empty_sql"]
|
||||
self.queries += 1
|
||||
mask = masked_columns(self.ctx.options)
|
||||
known = await asyncio.to_thread(self.seen.pooled)
|
||||
known = known if known.strong else None
|
||||
error: str | None = None
|
||||
result: Any = None
|
||||
settings = dataclasses.replace(self.ctx.settings, max_queries=1)
|
||||
try:
|
||||
runner = catalog_profile._Runner(self.ctx.query_source, settings,
|
||||
catalog_profile.SqlDialect(self.ctx.kind))
|
||||
result = await runner(sql, max_rows=QUERY_ROWS)
|
||||
except catalog_profile._Skip as e:
|
||||
error = e.detail
|
||||
except catalog_profile._Stop as e:
|
||||
error = str(e)
|
||||
except SnapshotTampered as e:
|
||||
error = QUERY_TAMPERED.format(name=self.ctx.source_name, error=e)
|
||||
except ValueError as e: # 不支持的数据库类型
|
||||
error = str(e)
|
||||
if error is not None:
|
||||
if mask or known is not None:
|
||||
from app.engine.masking import mask_error_text
|
||||
|
||||
error = mask_error_text(error, sql, known)
|
||||
event = {"type": "query", "sql": sql, "columns": [], "rows": [], "row_count": 0, "truncated": False,
|
||||
"error": error}
|
||||
return event, prompts["query_failed"].format(error=error)
|
||||
payload = result.to_payload()
|
||||
if mask or known is not None:
|
||||
from app.engine.masking import mask_query_result
|
||||
|
||||
schema = getattr(self.ctx.query_source, "schema_cache", None)
|
||||
done = await asyncio.to_thread(mask_query_result, payload, mask, self.ctx.kind, known, schema=schema)
|
||||
payload = done.payload
|
||||
await asyncio.to_thread(self.seen.add, done.secrets)
|
||||
columns = [str(c) for c in payload.get("columns") or []]
|
||||
rows = payload.get("rows") or []
|
||||
event = {"type": "query", "sql": str(payload.get("sql") or sql), "columns": columns,
|
||||
"rows": _rows(rows, SHOWN_ROWS, None), "row_count": int(payload.get("row_count") or len(rows)),
|
||||
"truncated": bool(payload.get("truncated")), "error": None}
|
||||
given = {"columns": columns, "rows": _rows(rows, QUERY_ROWS, CELL_MAX), "row_count": event["row_count"],
|
||||
"truncated": event["truncated"]}
|
||||
text = json.dumps(given, ensure_ascii=False, default=str)
|
||||
while len(text) > TOOL_CHARS and given["rows"]:
|
||||
given["rows"] = given["rows"][:max(1, len(given["rows"]) // 2)] if len(given["rows"]) > 1 else []
|
||||
given["rows_shown"] = len(given["rows"])
|
||||
text = json.dumps(given, ensure_ascii=False, default=str)
|
||||
return event, text
|
||||
|
||||
# ---------------------------------------------------------------- 提案
|
||||
|
||||
def _with_pending(self, change: Any) -> Any:
|
||||
"""界面上还没保存的修改里同一列的码值，并进新提的码值（新提的同一个码为准）。"""
|
||||
if not isinstance(change, Mapping):
|
||||
return change
|
||||
path = str(change.get("path") or "").strip()
|
||||
before = self.ctx.pending_codes.get(path)
|
||||
value = change.get("value")
|
||||
if before is None or not isinstance(value, Mapping):
|
||||
return change
|
||||
return {**change, "value": {**before, **value}}
|
||||
|
||||
def filter_problem(self, condition: Any) -> str | None:
|
||||
"""模型提的有效记录条件哪里不合规（交回模型的一句话），合规是 None。
|
||||
|
||||
按模型起草时的规矩核对（sqlcheck.valid_filter_problem：一个条件、只用这张表的列、不能有子查询、聚合、窗口、分号，
|
||||
只能用比较、IN、BETWEEN、LIKE、IS NULL 和几个函数）；sqlcheck 里还没有这条规矩时，至少核对用到的列都是这张表的、
|
||||
都在表结构里、没有子查询（catalog_transfer.filter_columns_problem，导入时核对人工写的条件也用它）。"""
|
||||
meta = self.ctx.tables.get(self.ctx.table) or {}
|
||||
strict = getattr(sqlcheck, "valid_filter_problem", None)
|
||||
if strict is not None:
|
||||
return strict(condition, meta, kind=self.ctx.kind, table=self.ctx.table)
|
||||
return filter_columns_problem(condition, meta, kind=self.ctx.kind, table=self.ctx.table)
|
||||
|
||||
async def propose(self, changes: Any) -> tuple[dict[str, Any] | None, str]:
|
||||
"""(给界面的 proposal 事件, 交回模型的结果)。提案不写目录。"""
|
||||
from app.db.base import SessionLocal
|
||||
|
||||
prompts = ASSISTANT_TOOL_PROMPT
|
||||
if not isinstance(changes, list) or not changes:
|
||||
return None, prompts["no_changes"]
|
||||
changes = [self._with_pending(c) for c in changes]
|
||||
async with SessionLocal() as session:
|
||||
entry = await catalog.read_entry(session, self.ctx.source_id, self.ctx.table)
|
||||
notes = entry.notes if entry else {}
|
||||
plan = catalog.plan_patch(notes, changes, table=self.ctx.table, tables=self.ctx.tables)
|
||||
dropped = list(plan.problems)
|
||||
kept: list[catalog.PatchChange] = []
|
||||
for change in plan.changes:
|
||||
key = catalog.parse_path(change.path)
|
||||
if key[0] == "c" and key[2] == "codes" and key[1].lower() in self.ctx.masked:
|
||||
dropped.append(prompts["masked_codes"].format(column=key[1]))
|
||||
continue
|
||||
if change.state == "same":
|
||||
dropped.append(prompts["same"].format(where=_where(key, change)))
|
||||
continue
|
||||
if key == ("t", "valid_filter") and (problem := self.filter_problem(change.after)) is not None:
|
||||
dropped.append(problem)
|
||||
continue
|
||||
kept.append(change)
|
||||
said: list[str] = []
|
||||
event: dict[str, Any] | None = None
|
||||
if kept:
|
||||
label = notes.get("label") if isinstance(notes.get("label"), Mapping) else {}
|
||||
event = {
|
||||
"type": "proposal", "source": self.ctx.source_name, "source_id": self.ctx.source_id,
|
||||
"table": self.ctx.table,
|
||||
"table_label": label.get("value") if label and label.get("status") != "rejected" else None,
|
||||
"version": entry.version if entry else 0,
|
||||
"changes": [{**c.as_dict(), "label": _where(catalog.parse_path(c.path), c)} for c in kept],
|
||||
}
|
||||
said.append(prompts["proposed"].format(n=len(kept), paths="、".join(c.path for c in kept)))
|
||||
else:
|
||||
said.append(prompts["nothing"])
|
||||
if dropped:
|
||||
said.append(prompts["dropped"].format(reasons="；".join(dropped)))
|
||||
return event, "\n".join(said)
|
||||
|
||||
|
||||
def _where(key: catalog.Slot, change: catalog.PatchChange) -> str:
|
||||
"""提案里一项的中文说法：「列 status 的码值」「表的有效记录条件」「指向 members 的关联关系」（同审阅队列的 field_label）。"""
|
||||
value = change.after if isinstance(change.after, Mapping) else change.before
|
||||
return item_label(key, value if isinstance(value, Mapping) else None)
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 流
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
#: 上一轮的正文以这些结尾（说完了一句话）时，下一轮另起一段
|
||||
_SENTENCE_END = ("。", "！", "？", "；", "：", ".", "!", "?", ";", ":")
|
||||
|
||||
|
||||
def _sse(obj: Mapping[str, Any]) -> str:
|
||||
return f"data: {json.dumps(obj, ensure_ascii=False, default=str)}\n\n"
|
||||
|
||||
|
||||
class _CallFailed(Exception):
|
||||
"""一次模型调用没成：code 是错误事件的机读码，message 给人看。"""
|
||||
|
||||
def __init__(self, code: str, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
def _seconds(value: Any) -> str:
|
||||
number = float(value)
|
||||
return str(int(number)) if number.is_integer() else f"{number:g}"
|
||||
|
||||
|
||||
def _transient(e: BaseException) -> bool:
|
||||
from app.core.errors import NETWORK, RATE_LIMIT, SERVER, classify
|
||||
|
||||
return classify(e) in (RATE_LIMIT, SERVER, NETWORK)
|
||||
|
||||
|
||||
async def _pump(runner: Any, history: list[Any], queue: asyncio.Queue[tuple[str, Any]]) -> None:
|
||||
"""在单独的任务里读模型的流，一块一块放进队列；读完放 end，出错放 error。"""
|
||||
try:
|
||||
async for chunk in runner.astream(history):
|
||||
await queue.put(("chunk", chunk))
|
||||
await queue.put(("end", None))
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e: # noqa: BLE001 - 交给读队列的一方按原因处理
|
||||
await queue.put(("error", e))
|
||||
|
||||
|
||||
async def _round(runner: Any, history: list[Any], model_id: str) -> AsyncIterator[tuple[str, Any]]:
|
||||
"""调一次模型：交出 ("text", 增量)、("beat", None)（等得久了，发心跳）、最后 ("message", (合起来的消息, 截断了没有))。
|
||||
网络错误、429、5xx 在还没输出任何正文时重试；超时、别的错误抛 _CallFailed。"""
|
||||
from app.core.errors import TIMEOUT, classify, describe_exception
|
||||
from app.engine.state import message_text
|
||||
from app.providers.budget import cut_off
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
attempt = 0
|
||||
while True:
|
||||
queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue()
|
||||
task = asyncio.ensure_future(_pump(runner, history, queue))
|
||||
deadline = loop.time() + float(catalog.MODEL_TIMEOUT_S)
|
||||
message: Any = None
|
||||
cut = False
|
||||
started = False
|
||||
failure: BaseException | None = None
|
||||
try:
|
||||
while True:
|
||||
remaining = deadline - loop.time()
|
||||
if remaining <= 0:
|
||||
raise _CallFailed(CODE_TIMEOUT, ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S),
|
||||
place=DRAFT_MODEL_PLACE))
|
||||
try:
|
||||
kind, value = await asyncio.wait_for(queue.get(), timeout=min(remaining, HEARTBEAT_S))
|
||||
except asyncio.TimeoutError:
|
||||
yield "beat", None
|
||||
continue
|
||||
if kind == "end":
|
||||
break
|
||||
if kind == "error":
|
||||
failure = value
|
||||
break
|
||||
message = value if message is None else message + value
|
||||
cut = cut or bool(cut_off(value))
|
||||
text = message_text(value)
|
||||
if text:
|
||||
started = True
|
||||
yield "text", text
|
||||
finally:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
with contextlib.suppress(asyncio.CancelledError, Exception):
|
||||
await task
|
||||
if failure is None:
|
||||
yield "message", (message, cut)
|
||||
return
|
||||
if not started and attempt < len(RETRY_DELAYS_S) and _transient(failure):
|
||||
logger.info("表助手：调用模型出错（%s），%s 秒后重试", type(failure).__name__, RETRY_DELAYS_S[attempt])
|
||||
await asyncio.sleep(RETRY_DELAYS_S[attempt])
|
||||
attempt += 1
|
||||
continue
|
||||
if isinstance(failure, TimeoutError) or classify(failure) == TIMEOUT:
|
||||
raise _CallFailed(CODE_TIMEOUT, ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S),
|
||||
place=DRAFT_MODEL_PLACE)) from failure
|
||||
raise _CallFailed(CODE_MODEL, ERROR_MODEL.format(model=model_id or "", reason=describe_exception(failure),
|
||||
place=DRAFT_MODEL_PLACE)) from failure
|
||||
|
||||
|
||||
async def stream(model: Any, model_id: str, ctx: AssistantContext,
|
||||
messages: Iterable[Mapping[str, Any]]) -> AsyncIterator[str]:
|
||||
"""表助手的一次请求：交出 SSE 的每一行（事件、心跳注释）。"""
|
||||
from langchain_core.messages import SystemMessage, ToolMessage, message_chunk_to_message
|
||||
|
||||
from app.providers.budget import cut_words
|
||||
|
||||
history: list[Any] = [SystemMessage(content=ctx.system), *_history(messages)]
|
||||
tools = _Tools(ctx)
|
||||
usage = {"input_tokens": 0, "output_tokens": 0}
|
||||
wrote = ""
|
||||
names = [t["function"]["name"] for t in ctx.tools()]
|
||||
try:
|
||||
# 绑工具也放在这里面：响应头已经发出去了，出了错要照实交一条错误事件，不能让流半路断掉
|
||||
runner = model.bind_tools(ctx.tools())
|
||||
for round_no in range(MAX_ROUNDS):
|
||||
fresh = True
|
||||
message: Any = None
|
||||
cut = False
|
||||
# 流在半路被关掉（使用者点了停止、断开）时，正在读的那次调用跟着停：aclosing 让 _round 的收尾马上执行
|
||||
async with contextlib.aclosing(_round(runner, history, model_id)) as rounds:
|
||||
async for kind, value in rounds:
|
||||
if kind == "beat":
|
||||
yield ": keep-alive\n\n"
|
||||
elif kind == "text":
|
||||
if fresh and wrote.endswith(_SENTENCE_END):
|
||||
# 上一轮（查询、提案之前）说完了一句话：这一轮另起一段
|
||||
yield _sse({"type": "text", "delta": "\n\n"})
|
||||
fresh = False
|
||||
wrote = value
|
||||
yield _sse({"type": "text", "delta": value})
|
||||
else:
|
||||
message, cut = value
|
||||
meta = getattr(message, "usage_metadata", None) or {}
|
||||
usage["input_tokens"] += int(meta.get("input_tokens") or 0)
|
||||
usage["output_tokens"] += int(meta.get("output_tokens") or 0)
|
||||
if cut:
|
||||
limit = getattr(model, "max_tokens", None)
|
||||
yield _sse({"type": "error", "code": CODE_TRUNCATED,
|
||||
"message": ERROR_TRUNCATED.format(cut=cut_words("表助手", limit), place=DRAFT_MODEL_PLACE)})
|
||||
return
|
||||
calls = list(getattr(message, "tool_calls", None) or []) if message is not None else []
|
||||
if not calls:
|
||||
yield _sse({"type": "done", "usage": usage, "queries": tools.queries})
|
||||
return
|
||||
if round_no == MAX_ROUNDS - 1:
|
||||
yield _sse({"type": "error", "code": CODE_ROUNDS, "message": ERROR_ROUNDS.format(n=MAX_ROUNDS)})
|
||||
return
|
||||
# 合起来的流式块换成普通消息再交回模型（不带半截的工具调用块）
|
||||
history.append(message_chunk_to_message(message))
|
||||
for call in calls:
|
||||
name = str(call.get("name") or "")
|
||||
args = call.get("args") if isinstance(call.get("args"), Mapping) else {}
|
||||
call_id = str(call.get("id") or f"{name}-{round_no}-{len(history)}")
|
||||
event: dict[str, Any] | None = None
|
||||
if name == QUERY_TOOL:
|
||||
event, content = await tools.query(args.get("sql"))
|
||||
elif name == PROPOSE_TOOL:
|
||||
event, content = await tools.propose(args.get("changes"))
|
||||
else:
|
||||
content = ASSISTANT_TOOL_PROMPT["unknown_tool"].format(name=name, tools="、".join(names))
|
||||
if event is not None:
|
||||
yield _sse(event)
|
||||
if round_no == MAX_ROUNDS - 2:
|
||||
content = f"{content}\n{ASSISTANT_TOOL_PROMPT['last_round']}"
|
||||
history.append(ToolMessage(content=content, tool_call_id=call_id))
|
||||
except _CallFailed as e:
|
||||
yield _sse({"type": "error", "code": e.code, "message": e.message})
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e: # noqa: BLE001 - 流已经开了，照实交一条错误事件
|
||||
from app.core.errors import describe_exception, first_line
|
||||
|
||||
logger.exception("表助手出错：数据源 %s 表 %s", ctx.source_id, ctx.table)
|
||||
yield _sse({"type": "error", "code": CODE_FAILED,
|
||||
"message": ERROR_FAILED.format(line=first_line(e) or describe_exception(e))})
|
||||
|
||||
|
||||
__all__ = ["AssistantContext", "CODE_FAILED", "CODE_MODEL", "CODE_ROUNDS", "CODE_TIMEOUT", "CODE_TRUNCATED",
|
||||
"HEARTBEAT_S", "MAX_MESSAGES", "MAX_QUERIES", "MAX_ROUNDS", "MESSAGE_MAX", "PENDING_SHOWN", "PROPOSE_TOOL",
|
||||
"QUERY_ROWS", "QUERY_TOOL", "SHOWN_ROWS", "conversation_problem", "prepare", "resolve_model", "stream"]
|
||||
@@ -0,0 +1,447 @@
|
||||
"""数据目录的审阅队列、批量审阅、表被哪些运行用过、审阅进度。
|
||||
|
||||
**为什么要有它。** 起草（数据库注释、外键、命名推断、数据剖析、模型）一次写出成百上千项，绝大多数停在「推断」：SQL 检查的
|
||||
错误级规则只认已确认、有确证的项，推断的项只作提示。逐表点开确认太慢，也看不出先确认哪几项。队列把推断的项摊平，按
|
||||
「这张表常不常用 × 这一项错了影响多大 × 这一项从哪来」排好，最该先看的在前面；勾几项一次确认、驳回或撤销审阅。
|
||||
|
||||
- **队列**（review_queue，GET /catalog/review-queue）：只列推断（proposed）的项。priority = 使用次数权重 × 风险权重 ×
|
||||
来源权重，从高到低；同分按表名、路径排，结果稳定。
|
||||
- 使用次数权重：1 + log2(1 + 使用次数)。使用次数同目录清单（catalog.table_usage，清单的「使用 N 次」和排序、起草的默认
|
||||
范围都按它）：按对数增长，常用表在前，但不至于让常用表上的中文名压过所有表上影响数字的项。
|
||||
- 风险权重（RISK_WEIGHT）：有效记录条件、去重规则、业务日期、关联关系最高（直接改变数字），业务主键、度量类型次之
|
||||
（计数和能不能加总），码值、粒度、表类型再次，中文名、说明、含义、单位最低。
|
||||
- 来源权重（SOURCE_WEIGHT）：模型起草、命名推断是猜的，比数据库注释优先；数据剖析（码值候选、业务日期、没核实的关系）
|
||||
居中。
|
||||
- 两组（kind）：facts 影响数字的项（有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型）；names 名称和说明
|
||||
（中文名、说明、含义、单位、码值、粒度、表类型）。表的「说明」和中文名、含义一样只是说明文字，归 names。
|
||||
- 表结构里已经没有的表、列上的项不列（确认了也没有用，目录页上另有清理的提示）。
|
||||
- field_label 是这一项给人看的位置：「表的有效记录条件」「列 status 的码值」「指向 members 的关联关系」（item_label，
|
||||
和表助手提案、导入冲突里的说法一致）；列级的项另给 column。
|
||||
- table_usage 是所在表的使用次数（同清单）。reasons 用中文说为什么排在这里：常用表（被 N 次运行用到）、影响数字的项
|
||||
（有效记录条件）、模型起草，未确认……「被 N 次运行用到」数的是运行（同 /usage 的 total）；同一次运行里查了几次、
|
||||
使用次数比运行次数多时两个都写（「被 2 次运行用到，使用 5 次」），和清单、运行列表上的数都对得上。
|
||||
- **批量审阅**（review_batch，POST /catalog/review/batch）：逐表按 if_version 乐观锁，同一张表的几条在一次写入里完成，
|
||||
一张表冲突、写不进去不影响别的表；路径不对、表不存在逐条报（results 和请求一一对应）。审阅的规则同单项审阅
|
||||
（catalog.review_item）。
|
||||
- **被哪些运行用过**（table_runs，GET /catalog/{table}/usage）：取法同使用次数（catalog.query_snapshot_tables），按运行
|
||||
去重，新的在前；total 是运行次数。
|
||||
- **进度**（review_progress，GET /catalog/review-progress）：常用表（使用次数前 TOP_N 张，和起草对话框「使用次数最多的
|
||||
20 张表」是同一批：catalog.default_tables 的取法，用到的不够 TOP_N 张时按表结构的顺序补齐）、全部、影响数字的项各自的
|
||||
确认情况。项的范围和队列一样：表结构里没有的表、列不算。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.data import catalog
|
||||
from app.db.models import Run
|
||||
|
||||
#: 队列的筛选：全部、影响数字的项、名称和说明
|
||||
KINDS = ("all", "facts", "names")
|
||||
#: 队列一页最多几项；批量审阅一次最多几条；「被哪些运行用过」一次最多几次运行
|
||||
QUEUE_LIMIT_MAX = 200
|
||||
BATCH_MAX = 200
|
||||
RUNS_LIMIT_MAX = 100
|
||||
#: 进度里的「常用表」取使用次数前几张（同起草的默认范围 catalog.DRAFT_DEFAULT_TABLES）
|
||||
TOP_N = 20
|
||||
|
||||
#: 每一项属于哪一组、风险几档：键是 (t, 字段) / (c, 字段) / (r, "")，档 4 最高
|
||||
FIELD_GROUP: dict[tuple[str, str], tuple[str, int]] = {
|
||||
("t", "valid_filter"): ("facts", 4), ("t", "dedup"): ("facts", 4), ("t", "business_date"): ("facts", 4),
|
||||
("r", ""): ("facts", 4),
|
||||
("t", "keys"): ("facts", 3), ("c", "measure"): ("facts", 3),
|
||||
("c", "codes"): ("names", 2), ("t", "grain"): ("names", 2), ("t", "kind"): ("names", 2),
|
||||
("t", "label"): ("names", 1), ("t", "description"): ("names", 1), ("c", "label"): ("names", 1),
|
||||
("c", "meaning"): ("names", 1), ("c", "unit"): ("names", 1),
|
||||
}
|
||||
RISK_WEIGHT = {4: 4.0, 3: 3.0, 2: 2.0, 1: 1.0}
|
||||
#: 来源权重：没写的（外键约束、人工填写，它们本来就不是推断）按 1
|
||||
SOURCE_WEIGHT = {"llm": 1.5, "name": 1.5, "profile": 1.25, "comment": 1.0}
|
||||
|
||||
#: 来源的界面叫法（和 frontend/src/lib/terms.ts 的 CATALOG_SOURCE_LABEL 一致，有测试核对）
|
||||
SOURCE_LABEL = {"comment": "数据库注释", "fk": "外键约束", "name": "命名推断", "profile": "数据剖析", "llm": "模型起草",
|
||||
"human": "人工填写"}
|
||||
RELATION_LABEL = "关联关系"
|
||||
#: reasons 的说法：常用表（被几次运行用到）；使用次数比运行次数多（同一次运行里查了几次）时两个都写
|
||||
REASON_TOP = "常用表（{used}）"
|
||||
USED_RUNS = "被 {runs} 次运行用到"
|
||||
USED_RUNS_AND_QUERIES = "被 {runs} 次运行用到，使用 {n} 次"
|
||||
REASON_RISK = {4: "影响数字的项（{label}）", 3: "影响计数和汇总的项（{label}）", 2: "帮助理解取值的项（{label}）",
|
||||
1: "名称和说明（{label}）"}
|
||||
REASON_PENDING_CODES = "{n} 个码值的含义待填写"
|
||||
REASON_SOURCE = "{source}，未确认"
|
||||
#: 码值含义为空（数据剖析写进来的候选）时的说法，同界面的 CODES_TEXT.pending
|
||||
CODE_PENDING = "含义待填写"
|
||||
|
||||
#: 批量审阅逐条结果的机读码
|
||||
CODE_CONFLICT = "catalog_conflict"
|
||||
CODE_PATH = "catalog_path_invalid"
|
||||
CODE_INVALID = "catalog_invalid"
|
||||
CODE_TABLE_MISSING = "table_missing"
|
||||
|
||||
Slot = catalog.Slot
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 一项的路径、叫法、显示
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def slot_path(key: Slot) -> str:
|
||||
"""槽位 → 审阅接口的路径写法（catalog.parse_path 的反过来）。"""
|
||||
if key[0] == "t":
|
||||
return key[1]
|
||||
if key[0] == "c":
|
||||
return f"columns.{key[1]}.{key[2]}"
|
||||
return f"relations.{key[1]}"
|
||||
|
||||
|
||||
def field_of(key: Slot) -> tuple[str, str]:
|
||||
"""槽位 → FIELD_GROUP 的键。"""
|
||||
if key[0] == "r":
|
||||
return ("r", "")
|
||||
return (key[0], key[-1])
|
||||
|
||||
|
||||
def field_name(key: Slot) -> str:
|
||||
"""这一项的字段名：表级、列级的项是字段名，关联关系是 relation。"""
|
||||
return "relation" if key[0] == "r" else key[-1]
|
||||
|
||||
|
||||
def field_text(key: Slot) -> str:
|
||||
"""字段的界面叫法（不带列名）：中文名、码值、有效记录条件、关联关系……"""
|
||||
if key[0] == "t":
|
||||
return catalog.TABLE_FIELD_LABEL.get(key[1], key[1])
|
||||
if key[0] == "c":
|
||||
return catalog.COLUMN_FIELD_LABEL.get(key[2], key[2])
|
||||
return RELATION_LABEL
|
||||
|
||||
|
||||
def item_label(key: Slot, item: Mapping[str, Any] | None = None) -> str:
|
||||
"""一项给人看的位置：「表的有效记录条件」「列 status 的码值」「指向 members 的关联关系」（不知道指向哪张表时写编号）。"""
|
||||
if key[0] == "r" and isinstance(item, Mapping) and item.get("to_table"):
|
||||
return f"指向 {item['to_table']} 的{RELATION_LABEL}"
|
||||
return catalog.describe_slot(key)
|
||||
|
||||
|
||||
def item_value(key: Slot, item: Mapping[str, Any]) -> Any:
|
||||
"""一项的值：关联关系是两端、基数和覆盖率，别的是 value。"""
|
||||
if key[0] == "r":
|
||||
return {k: item.get(k) for k in ("columns", "to_table", "to_columns", "cardinality", "coverage")}
|
||||
return item.get("value")
|
||||
|
||||
|
||||
def _percent(ratio: float) -> str:
|
||||
"""覆盖率写成百分数：整数不带小数，其余留一位；不到 100% 的不写成 100%。"""
|
||||
text = f"{ratio * 100:.1f}".rstrip("0").rstrip(".")
|
||||
return "99.9%" if text == "100" and ratio < 1 else f"{text}%"
|
||||
|
||||
|
||||
def relation_text(rel: Mapping[str, Any]) -> str:
|
||||
"""关联关系写成一行：member_id → members.id（多对一，覆盖率 98%）。"""
|
||||
left = "、".join(str(c) for c in rel.get("columns") or [])
|
||||
right = "、".join(f"{rel.get('to_table')}.{c}" for c in rel.get("to_columns") or [])
|
||||
extra = []
|
||||
if rel.get("cardinality") in catalog.CARDINALITY_LABEL:
|
||||
extra.append(catalog.CARDINALITY_LABEL[rel["cardinality"]])
|
||||
coverage = rel.get("coverage")
|
||||
if isinstance(coverage, (int, float)) and not isinstance(coverage, bool):
|
||||
extra.append(f"覆盖率 {_percent(float(coverage))}")
|
||||
return f"{left} → {right}" + (f"（{'，'.join(extra)}）" if extra else "")
|
||||
|
||||
|
||||
def display(key: Slot, value: Any) -> str:
|
||||
"""一项的值写成一行字，用界面上的说法（明细表、可累加、多对一……）。"""
|
||||
if key[0] == "r":
|
||||
return relation_text(value if isinstance(value, Mapping) else {})
|
||||
field_ = key[-1]
|
||||
if field_ == "kind":
|
||||
return catalog.UI_KIND_LABEL.get(value, str(value))
|
||||
if field_ == "measure":
|
||||
return catalog.UI_MEASURE_LABEL.get(value, str(value))
|
||||
if field_ == "keys" and isinstance(value, list):
|
||||
return "、".join(str(v) for v in value)
|
||||
if field_ == "codes" and isinstance(value, Mapping):
|
||||
return "、".join(f"{k}={v if str(v).strip() else CODE_PENDING}" for k, v in value.items())
|
||||
if field_ == "business_date" and isinstance(value, Mapping):
|
||||
parts = [str(value.get("column") or "")]
|
||||
if value.get("rule"):
|
||||
parts.append(f"规则：{value['rule']}")
|
||||
if value.get("timezone"):
|
||||
parts.append(f"时区：{value['timezone']}")
|
||||
return "；".join(parts)
|
||||
return value if isinstance(value, str) else str(value)
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 排序
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def usage_weight(usage: int) -> float:
|
||||
return 1.0 + math.log2(1 + max(0, int(usage or 0)))
|
||||
|
||||
|
||||
def priority(usage: int, key: Slot, source: Any) -> float:
|
||||
"""一项的优先级：使用次数权重 × 风险权重 × 来源权重（不取整；排序和交给界面时取四位小数）。"""
|
||||
_, tier = FIELD_GROUP[field_of(key)]
|
||||
return usage_weight(usage) * RISK_WEIGHT[tier] * SOURCE_WEIGHT.get(str(source), 1.0)
|
||||
|
||||
|
||||
def top_tables(tables: Mapping[str, Any], usage: Mapping[str, int]) -> list[str]:
|
||||
"""使用次数前 TOP_N 张表（同 catalog.default_tables，起草对话框的「使用次数最多的 20 张表」）：使用次数从多到少，
|
||||
一样多的（包括都没用过的）按表结构里的顺序。表结构里不到 TOP_N 张时就是全部。"""
|
||||
order = {name: i for i, name in enumerate(tables)}
|
||||
return sorted(tables, key=lambda n: (-usage.get(n, 0), order[n]))[:TOP_N]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Usage:
|
||||
"""每张表用得多不多（没被用过的表不出现）。
|
||||
|
||||
queries：使用次数，同 catalog.table_usage（目录清单的「使用 N 次」、清单的排序、起草的默认范围）；
|
||||
runs：用到它的运行有几次，同「被哪些运行用过」（table_runs）的 total。同一次运行里查了几次这张表，queries 比 runs 多。
|
||||
"""
|
||||
|
||||
queries: dict[str, int]
|
||||
runs: dict[str, int]
|
||||
|
||||
|
||||
async def table_usage_counts(session: AsyncSession, source: Any) -> Usage:
|
||||
"""一次读出每张表的使用次数和运行次数（同一份查询快照引用：catalog.query_snapshot_tables）。"""
|
||||
queries: Counter[str] = Counter()
|
||||
runs: dict[str, set[str]] = {}
|
||||
for run, hit in await catalog.query_snapshot_tables(session, source):
|
||||
queries.update(hit)
|
||||
for name in hit:
|
||||
runs.setdefault(name, set()).add(run)
|
||||
return Usage(dict(queries), {name: len(ids) for name, ids in runs.items()})
|
||||
|
||||
|
||||
def usage_text(queries: int, runs: int) -> str:
|
||||
"""「被 2 次运行用到」；使用次数比运行次数多时「被 2 次运行用到，使用 5 次」。"""
|
||||
if queries > runs:
|
||||
return USED_RUNS_AND_QUERIES.format(runs=f"{runs:,}", n=f"{queries:,}")
|
||||
return USED_RUNS.format(runs=f"{runs:,}")
|
||||
|
||||
|
||||
def _reasons(key: Slot, item: Mapping[str, Any], usage: int, runs: int, top: bool) -> list[str]:
|
||||
out: list[str] = []
|
||||
if usage > 0:
|
||||
used = usage_text(usage, runs)
|
||||
out.append(REASON_TOP.format(used=used) if top else used)
|
||||
_, tier = FIELD_GROUP[field_of(key)]
|
||||
out.append(REASON_RISK[tier].format(label=field_text(key)))
|
||||
if key[0] == "c" and key[2] == "codes" and isinstance(item.get("value"), Mapping):
|
||||
pending = sum(1 for v in item["value"].values() if not str(v).strip())
|
||||
if pending:
|
||||
out.append(REASON_PENDING_CODES.format(n=pending))
|
||||
source = str(item.get("source") or "")
|
||||
out.append(REASON_SOURCE.format(source=SOURCE_LABEL.get(source, source)))
|
||||
return out
|
||||
|
||||
|
||||
def live_slots(notes: Mapping[str, Any] | None, meta: Mapping[str, Any] | None) -> list[tuple[Slot, dict[str, Any]]]:
|
||||
"""一张表目录里还算数的项：表结构里已经没有的列上的项不算（不知道表结构时都算）。"""
|
||||
columns = catalog._column_names(meta)
|
||||
return [(key, item) for key, item in catalog._slots(notes).items()
|
||||
if not (key[0] == "c" and columns is not None and key[1] not in columns)]
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 队列
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
async def review_queue(session: AsyncSession, source: Any, *, kind: str = "all", table: str | None = None,
|
||||
limit: int = 50, offset: int = 0) -> dict[str, Any]:
|
||||
"""审阅队列：{total, items}。source 是读表结构用的源（上传源是绑定当前快照的视图）。"""
|
||||
cache = getattr(source, "schema_cache", None) or {}
|
||||
tables = cache.get("tables") or {}
|
||||
wanted = None
|
||||
if table:
|
||||
wanted = catalog.resolve_table_name(cache, table) or table
|
||||
entries = await catalog.read_catalog(session, source.id)
|
||||
usage = await table_usage_counts(session, source)
|
||||
top = set(top_tables(tables, usage.queries))
|
||||
items: list[dict[str, Any]] = []
|
||||
for name, entry in entries.items():
|
||||
meta = tables.get(name)
|
||||
if meta is None or (wanted is not None and name != wanted):
|
||||
continue
|
||||
used = usage.queries.get(name, 0)
|
||||
for key, item in live_slots(entry.notes, meta):
|
||||
if item.get("status") != "proposed":
|
||||
continue
|
||||
group, _ = FIELD_GROUP[field_of(key)]
|
||||
if kind != "all" and group != kind:
|
||||
continue
|
||||
value = item_value(key, item)
|
||||
items.append({
|
||||
"table_name": name,
|
||||
"path": slot_path(key),
|
||||
"field": field_name(key),
|
||||
"field_label": item_label(key, item),
|
||||
# 列级的项是哪一列（表级的项、关联关系为 null）
|
||||
"column": key[1] if key[0] == "c" else None,
|
||||
"value": value,
|
||||
"display": display(key, value),
|
||||
"source": item.get("source"),
|
||||
"status": item.get("status"),
|
||||
"note": item.get("note") if isinstance(item.get("note"), str) and item.get("note") else None,
|
||||
"priority": round(priority(used, key, item.get("source")), 4),
|
||||
"reasons": _reasons(key, item, used, usage.runs.get(name, 0), name in top),
|
||||
"table_usage": used,
|
||||
"version": entry.version,
|
||||
})
|
||||
items.sort(key=lambda i: (-i["priority"], i["table_name"], i["path"]))
|
||||
return {"total": len(items), "items": items[offset:offset + limit]}
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 批量审阅
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BatchItem:
|
||||
table_name: str
|
||||
path: str
|
||||
action: str
|
||||
if_version: int
|
||||
|
||||
|
||||
def _result(item: BatchItem, *, ok: bool, version: int | None = None, error: str | None = None,
|
||||
code: str | None = None) -> dict[str, Any]:
|
||||
return {"table_name": item.table_name, "path": item.path, "ok": ok, "version": version if ok else None,
|
||||
"error": error, "code": code}
|
||||
|
||||
|
||||
async def review_batch(session: AsyncSession, source: Any, items: Iterable[BatchItem], *,
|
||||
actor: str | None) -> list[dict[str, Any]]:
|
||||
"""批量审阅，返回逐条结果（和请求一一对应）。
|
||||
|
||||
表名按表结构认（全名、大小写不一致都认）；表结构里没有、但还留着目录的也认（同单项审阅）。同一张表的几条按请求里的
|
||||
顺序在当前目录上依次审阅，一次写入：版本不符（if_version 和库里的不一样）、路径不对的那几条不做，其余照做；写入时撞上
|
||||
别人刚改过，这张表的几条都记冲突。"""
|
||||
items = list(items)
|
||||
cache = getattr(source, "schema_cache", None) or {}
|
||||
results: list[dict[str, Any] | None] = [None] * len(items)
|
||||
groups: dict[str, list[int]] = {}
|
||||
known: dict[str, str | None] = {}
|
||||
for i, item in enumerate(items):
|
||||
if item.table_name not in known:
|
||||
key = catalog.resolve_table_name(cache, item.table_name)
|
||||
if key is None and await catalog.read_entry(session, source.id, item.table_name) is not None:
|
||||
key = item.table_name
|
||||
known[item.table_name] = key
|
||||
key = known[item.table_name]
|
||||
if key is None:
|
||||
results[i] = _result(item, ok=False, code=CODE_TABLE_MISSING,
|
||||
error=f"数据源中没有表 {item.table_name}，可能已被删除或尚未探查结构")
|
||||
continue
|
||||
groups.setdefault(key, []).append(i)
|
||||
at = catalog.now_iso()
|
||||
for key, indexes in groups.items():
|
||||
entry = await catalog.read_entry(session, source.id, key)
|
||||
current = entry.version if entry else 0
|
||||
notes: dict[str, Any] = entry.notes if entry else {}
|
||||
applied: list[int] = []
|
||||
for i in indexes:
|
||||
item = items[i]
|
||||
if item.if_version != current:
|
||||
results[i] = _result(item, ok=False, code=CODE_CONFLICT, error=str(catalog.CatalogConflict(key, current)))
|
||||
continue
|
||||
try:
|
||||
notes = catalog.review_item(notes, item.path, item.action, at=at)
|
||||
except catalog.CatalogPathError as e:
|
||||
results[i] = _result(item, ok=False, code=CODE_PATH, error=str(e))
|
||||
continue
|
||||
applied.append(i)
|
||||
if not applied:
|
||||
continue
|
||||
try:
|
||||
written = await catalog.write_entry(session, source.id, key, notes, if_version=current, actor=actor)
|
||||
except catalog.CatalogConflict as e:
|
||||
for i in applied:
|
||||
results[i] = _result(items[i], ok=False, code=CODE_CONFLICT, error=str(e))
|
||||
continue
|
||||
except catalog.CatalogInvalid as e:
|
||||
for i in applied:
|
||||
results[i] = _result(items[i], ok=False, code=CODE_INVALID, error=str(e))
|
||||
continue
|
||||
for i in applied:
|
||||
results[i] = _result(items[i], ok=True, version=written.version)
|
||||
return [r for r in results if r is not None]
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 被哪些运行用过
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
async def table_runs(session: AsyncSession, source: Any, table: str, *, limit: int = 20) -> dict[str, Any]:
|
||||
"""用过这张表的运行：{total, runs}，新的在前（按开始时间，没开始的按建立时间）。取法同使用次数。"""
|
||||
run_ids = {run for run, hit in await catalog.query_snapshot_tables(session, source) if table in hit}
|
||||
if not run_ids:
|
||||
return {"total": 0, "runs": []}
|
||||
started = func.coalesce(Run.started_at, Run.created_at)
|
||||
rows = (await session.execute(
|
||||
select(Run.id, Run.workflow_id, Run.workflow_name, Run.run_class, Run.status, Run.started_at)
|
||||
.where(Run.id.in_(run_ids)).order_by(started.desc(), Run.id).limit(limit)
|
||||
)).all()
|
||||
return {
|
||||
"total": len(run_ids),
|
||||
"runs": [{"run_id": r.id, "workflow_id": r.workflow_id, "workflow_name": r.workflow_name or "",
|
||||
"run_class": r.run_class, "status": r.status,
|
||||
"started_at": r.started_at.isoformat() if r.started_at else None} for r in rows],
|
||||
}
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 进度
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
async def review_progress(session: AsyncSession, source: Any) -> dict[str, Any]:
|
||||
"""审阅进度：常用 TOP_N 张表（top_tables）、全部、影响数字的项，项按状态计数。全部的 items_total 里还有有确证的项
|
||||
（不单列）。"""
|
||||
tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {}
|
||||
entries = await catalog.read_catalog(session, source.id)
|
||||
usage = await catalog.table_usage(session, source)
|
||||
top = set(top_tables(tables, usage))
|
||||
every: Counter[str] = Counter()
|
||||
head: Counter[str] = Counter()
|
||||
facts: Counter[str] = Counter()
|
||||
for name, entry in entries.items():
|
||||
meta = tables.get(name)
|
||||
if meta is None:
|
||||
continue
|
||||
for key, item in live_slots(entry.notes, meta):
|
||||
status = str(item.get("status"))
|
||||
every[status] += 1
|
||||
if name in top:
|
||||
head[status] += 1
|
||||
if FIELD_GROUP[field_of(key)][0] == "facts":
|
||||
facts[status] += 1
|
||||
return {
|
||||
"top_n": TOP_N,
|
||||
"top": {"tables": len(top), "items_total": sum(head.values()), "items_confirmed": head["confirmed"]},
|
||||
"all": {"items_total": sum(every.values()), "items_confirmed": every["confirmed"],
|
||||
"items_proposed": every["proposed"], "items_rejected": every["rejected"]},
|
||||
"facts": {"proposed": facts["proposed"], "confirmed": facts["confirmed"]},
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["BATCH_MAX", "BatchItem", "CODE_CONFLICT", "CODE_INVALID", "CODE_PATH", "CODE_TABLE_MISSING",
|
||||
"FIELD_GROUP", "KINDS", "QUEUE_LIMIT_MAX", "RISK_WEIGHT", "RUNS_LIMIT_MAX", "SOURCE_LABEL", "SOURCE_WEIGHT",
|
||||
"TOP_N", "Usage", "display", "field_name", "field_of", "field_text", "item_label", "item_value",
|
||||
"live_slots", "priority", "relation_text", "review_batch", "review_progress", "review_queue", "slot_path",
|
||||
"table_runs", "table_usage_counts", "top_tables", "usage_text", "usage_weight"]
|
||||
@@ -0,0 +1,541 @@
|
||||
"""数据目录的导出导入：本机整理好的目录带到部署实例上（GET /catalog/export、POST /catalog/import/preview、/import）。
|
||||
|
||||
**为什么要有它。** 目录（尤其是人工确认、驳回过的项）是在本机对着开发库一项项审出来的；部署实例连的是同一套业务库，
|
||||
却没有办法把这些结论带过去，只能在那边从头再审一遍。导出是一个 JSON 文件，导入先预览、再确认。
|
||||
|
||||
**文件**（export_catalog）：{format: "agentlab-catalog", version: 1, exported_at, source: {name, kind}, guidance,
|
||||
tables: {表名: {notes, version}}}，含全部状态的项，表结构里已经没有的表也带上（目录里有什么就导出什么）。
|
||||
|
||||
**导入的规矩**（plan_import，预览和导入共用这一份）：
|
||||
|
||||
- 表按名字对：表结构里的写法、忽略大小写（name_key）、带 schema 前缀的对不上时去掉前缀再对（catalog.resolve_table_name）。
|
||||
只对目标数据源表结构里的表；对不上的列在 unmatched_tables。文件里两张表对上目标的同一张时，写法完全一样的那张优先，
|
||||
另一张算对不上。列名同样按表结构的写法、再忽略大小写对；业务主键、业务日期、关联关系两端的列和指向的表都换成目标的写法，
|
||||
关联关系的编号按目标的表名、列名重算（catalog.relation_id）。
|
||||
- 文件里已确认、已驳回的项照带：来源、状态、备注、修改时间都不变。目标那一格空着就新增；目标是推断、有确证的就更新；
|
||||
目标也是已确认、已驳回的，和文件一样就保留，不一样（值或状态不同）就保留目标的、记一条冲突。关联关系的「一样」看两端和
|
||||
基数，覆盖率是测量值、不算。
|
||||
- 文件里推断、有确证的项只填目标的空缺，目标那一格有东西（哪怕是驳回）就保留目标的，不算冲突。
|
||||
- 跳过并写原因：表结构里没有的列、业务主键和业务日期指着的列没有的、关联关系两端对不上的（指向的表、两端的列）、
|
||||
目标数据源遮罩的列上的码值（码值就是这一列的取值，遮罩了就不该进目录）、不认识的字段、取值不合规的项（说法同
|
||||
catalog.validate_notes）。
|
||||
- 有效记录条件是表级的一项、却写着列名，按列跳过的规矩拦不住它：按目标数据源的方言解析出用到的列
|
||||
（filter_columns_problem），不在目标的表结构里、是别的表的列、解析不了或不是一个条件、带子查询（没法核对子查询用到的表
|
||||
和列）的跳过。人工写的条件不限函数（模型起草时的那套白名单只管模型写的条件）；列名大小写和目标不同、带本表表名的照带。
|
||||
- 逐表写（乐观锁，撞上别人刚改过就重读重算，最多三次），一张表写不进去只记在那张表上，不影响别的表。
|
||||
- 起草要求：勾了 include_guidance、文件里有、目标还没有时才写（action: replace）；目标已有的不覆盖（keep）；没勾或文件里
|
||||
没有是 none。
|
||||
- 文件格式不对（不是对象、格式标记或版本不对、表的目录不是对象、起草要求不是文字或超长）抛 TransferInvalid，接口回 422。
|
||||
|
||||
有排队或进行中的起草任务时不导入（接口先调 catalog_jobs.ensure_idle，409 catalog_draft_busy）；目标数据源没有表结构时
|
||||
接口回 409（什么也对不上）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.data import catalog, catalog_jobs
|
||||
from app.data.catalog_review import item_label, slot_path
|
||||
from app.data.engine import masked_columns
|
||||
from app.data.names import name_key
|
||||
from app.data.sqlcheck import dialect_of
|
||||
|
||||
FORMAT = "agentlab-catalog"
|
||||
VERSION = 1
|
||||
FILE_SUFFIX = "-数据目录.json"
|
||||
_BAD_FILENAME = re.compile(r'[\\/:*?"<>|\x00-\x1f\x7f]')
|
||||
_NAME_MAX = 80
|
||||
_UNNAMED = "数据源"
|
||||
|
||||
#: 状态：照带的、只填空缺的
|
||||
_LOCKED = ("confirmed", "rejected")
|
||||
#: 状态的界面叫法（同 frontend/src/lib/terms.ts 的 CATALOG_STATUS_LABEL）
|
||||
_STATUS_LABEL = {"proposed": "推断", "verified": "已验证", "confirmed": "已确认", "rejected": "已驳回"}
|
||||
|
||||
#: 写入一直撞上别人刚改过（重读重算三次仍冲突）
|
||||
WRITE_CONFLICT_ERROR = "这张表的数据目录正被其他人修改，没有导入，请稍后重新导入"
|
||||
WRITE_ERROR = "写入这张表的数据目录时出错，没有导入：{line}"
|
||||
|
||||
|
||||
class TransferInvalid(ValueError):
|
||||
"""文件格式不对。message 给人看（接口回 422）。"""
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 导出
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def attachment_name(source_name: str) -> str:
|
||||
"""附件名「<数据源名>-数据目录.json」。文件名里不能有的字换成下划线。"""
|
||||
name = _BAD_FILENAME.sub("_", source_name or "").strip(" .")[:_NAME_MAX].strip(" .")
|
||||
return f"{name or _UNNAMED}{FILE_SUFFIX}"
|
||||
|
||||
|
||||
def content_disposition(filename: str, source_id: str) -> str:
|
||||
"""附件头：中文名按 RFC 5987 写在 filename* 里；filename 给不认 filename* 的老客户端一个纯 ASCII 的名字。"""
|
||||
from urllib.parse import quote
|
||||
|
||||
ascii_id = re.sub(r"[^A-Za-z0-9]", "", source_id)[:8]
|
||||
return f"attachment; filename=\"agentlab-catalog-{ascii_id}.json\"; filename*=UTF-8''{quote(filename, safe='')}"
|
||||
|
||||
|
||||
async def export_catalog(session: AsyncSession, row: Any) -> dict[str, Any]:
|
||||
"""导出这个数据源的数据目录（row 是数据源记录，目录按它的 id 存）。"""
|
||||
entries = await catalog.read_catalog(session, row.id)
|
||||
return {
|
||||
"format": FORMAT,
|
||||
"version": VERSION,
|
||||
"exported_at": catalog.now_iso(),
|
||||
"source": {"name": row.name, "kind": row.kind},
|
||||
"guidance": catalog_jobs.guidance_of(row),
|
||||
"tables": {name: {"notes": entry.notes, "version": entry.version} for name, entry in sorted(entries.items())},
|
||||
}
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 读文件
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportFile:
|
||||
tables: dict[str, dict[str, Any]]
|
||||
guidance: str | None
|
||||
|
||||
|
||||
def read_file(content: Any, *, include_guidance: bool) -> ImportFile:
|
||||
"""导出的那个对象 → ImportFile。界面直接交来文件的原文（一段 JSON 文字）也认。格式不对抛 TransferInvalid（说法给人看）。"""
|
||||
if isinstance(content, str):
|
||||
try:
|
||||
content = json.loads(content)
|
||||
except ValueError as e:
|
||||
raise TransferInvalid("文件内容不是合法的 JSON，请选择从 AgentLab 导出的数据目录文件") from e
|
||||
if not isinstance(content, Mapping):
|
||||
raise TransferInvalid("文件内容不是数据目录导出的格式，请选择从 AgentLab 导出的数据目录文件")
|
||||
if content.get("format") != FORMAT:
|
||||
raise TransferInvalid("这个文件不是 AgentLab 导出的数据目录（文件里的格式标记不对），请重新选择文件")
|
||||
version = content.get("version")
|
||||
if version != VERSION:
|
||||
raise TransferInvalid(f"这个数据目录文件的格式版本是「{version}」，当前只能导入第 {VERSION} 版的文件，"
|
||||
"请用同一版本的 AgentLab 重新导出")
|
||||
tables = content.get("tables")
|
||||
if not isinstance(tables, Mapping):
|
||||
raise TransferInvalid("数据目录文件里没有表的目录，文件可能不完整，请重新导出")
|
||||
out: dict[str, dict[str, Any]] = {}
|
||||
for name, body in tables.items():
|
||||
notes = body.get("notes") if isinstance(body, Mapping) else None
|
||||
if not isinstance(name, str) or not name.strip() or not isinstance(notes, Mapping):
|
||||
raise TransferInvalid(f"数据目录文件里表「{str(name)[:60]}」的目录格式不正确，文件可能被改动过，请重新导出")
|
||||
out[name] = copy.deepcopy(dict(notes))
|
||||
guidance = content.get("guidance")
|
||||
if guidance is not None and not isinstance(guidance, str):
|
||||
raise TransferInvalid("数据目录文件里的起草要求格式不正确，文件可能被改动过，请重新导出")
|
||||
text = guidance.strip() if isinstance(guidance, str) else ""
|
||||
if include_guidance and len(text) > catalog_jobs.GUIDANCE_MAX:
|
||||
raise TransferInvalid(f"数据目录文件里的起草要求有 {len(text)} 字，超过了 {catalog_jobs.GUIDANCE_MAX} 字的上限，"
|
||||
"请删减后再导入，或者不导入起草要求")
|
||||
return ImportFile(tables=out, guidance=text or None)
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 对表、对列
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def match_tables(file_tables: list[str], schema_cache: Mapping[str, Any]) -> tuple[dict[str, str], list[str]]:
|
||||
"""文件里的表 → 目标表结构里的键。返回 ({文件里的表名: 目标表名}, 对不上的表名)。写法完全一样的先认领。"""
|
||||
tables = (schema_cache or {}).get("tables") or {}
|
||||
matched: dict[str, str] = {}
|
||||
claimed: set[str] = set()
|
||||
ordered = sorted(file_tables, key=lambda n: n not in tables)
|
||||
for name in ordered:
|
||||
key = catalog.resolve_table_name(schema_cache, name)
|
||||
if key is None or key in claimed:
|
||||
continue
|
||||
matched[name] = key
|
||||
claimed.add(key)
|
||||
unmatched = [name for name in file_tables if name not in matched]
|
||||
return {name: matched[name] for name in file_tables if name in matched}, unmatched
|
||||
|
||||
|
||||
class _Columns:
|
||||
"""一张目标表的列：按表结构的写法认，再忽略大小写认（只认唯一对上的）。"""
|
||||
|
||||
def __init__(self, meta: Mapping[str, Any] | None) -> None:
|
||||
names = [str(c.get("name")) for c in (meta or {}).get("columns") or [] if isinstance(c, Mapping)
|
||||
and c.get("name") is not None]
|
||||
self.names = set(names)
|
||||
self._loose: dict[str, list[str]] = {}
|
||||
for name in names:
|
||||
self._loose.setdefault(name_key(name), []).append(name)
|
||||
|
||||
def find(self, name: Any) -> str | None:
|
||||
if not isinstance(name, str):
|
||||
return None
|
||||
if name in self.names:
|
||||
return name
|
||||
found = self._loose.get(name_key(name)) or []
|
||||
return found[0] if len(found) == 1 else None
|
||||
|
||||
|
||||
def filter_columns_problem(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None, table: str,
|
||||
also: Iterable[str] = (), where: str = "表结构") -> str | None:
|
||||
"""有效记录条件用到的列是不是都在这张表的表结构里（给人看的一句话；没问题返回 None）。
|
||||
|
||||
按数据源的方言（sqlcheck.dialect_of）解析成一个条件：解析不了、不止一句、是一条查询、带子查询（没法核对子查询用到的
|
||||
表和列）的不收；列不带表名，或带这张表的表名（table、表结构里的 qualified、also 里的写法，比如文件里的表名；忽略大小写、
|
||||
去掉 schema 前缀），列名忽略大小写在表结构里。不限函数、运算。不是文字的交给合规检查说（返回 None）；数据源类型
|
||||
没有对应的方言时没法解析，不核对。where 是说法里「表结构」的叫法（导入时是「目标的表结构」）。"""
|
||||
if not isinstance(condition, str) or not condition.strip():
|
||||
return None
|
||||
dialect = dialect_of(kind)
|
||||
if dialect is None:
|
||||
return None
|
||||
import sqlglot
|
||||
from sqlglot import exp
|
||||
|
||||
try:
|
||||
parsed = [s for s in sqlglot.parse(condition.strip(), read=dialect) if s is not None]
|
||||
except Exception: # noqa: BLE001 - 语法错、方言不支持的写法
|
||||
parsed = []
|
||||
if len(parsed) != 1 or isinstance(parsed[0], exp.Query) or not isinstance(parsed[0], exp.Condition):
|
||||
return f"有效记录条件不是一个能解析的 SQL 条件（按 {kind} 的写法），没法核对它用到的列"
|
||||
expression = parsed[0]
|
||||
if any(isinstance(node, (exp.Query, exp.Subquery)) for node in expression.walk()):
|
||||
return "有效记录条件里有子查询，没法核对子查询用到的表和列"
|
||||
own = {name_key(str(n).rsplit(".", 1)[-1]) for n in (table, (meta or {}).get("qualified"), *also) if n}
|
||||
columns = {str(c.get("name")).lower() for c in (meta or {}).get("columns") or []
|
||||
if isinstance(c, Mapping) and c.get("name") is not None}
|
||||
foreign: list[str] = []
|
||||
missing: list[str] = []
|
||||
for column in expression.find_all(exp.Column):
|
||||
if column.is_star:
|
||||
continue
|
||||
if column.table and name_key(column.table) not in own:
|
||||
foreign.append(column.sql(dialect=dialect))
|
||||
elif column.name.lower() not in columns:
|
||||
missing.append(column.name)
|
||||
if foreign:
|
||||
return f"有效记录条件引用了别的表的列（{'、'.join(dict.fromkeys(foreign))}），只能用这张表的列"
|
||||
if missing:
|
||||
return f"有效记录条件里的列「{'、'.join(dict.fromkeys(missing))}」不在{where}里"
|
||||
return None
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 一张表怎么并
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class TablePlan:
|
||||
"""文件里一张表并进目标的计划。writes 是要写进目标的项（目标的槽位 → 项）。"""
|
||||
|
||||
table_name: str
|
||||
matched_as: str
|
||||
add: int = 0
|
||||
update: int = 0
|
||||
keep: int = 0
|
||||
conflicts: list[dict[str, Any]] = field(default_factory=list)
|
||||
skipped: list[dict[str, Any]] = field(default_factory=list)
|
||||
writes: dict[catalog.Slot, dict[str, Any]] = field(default_factory=dict)
|
||||
version: int = 0
|
||||
error: str | None = None
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {"table_name": self.table_name, "matched_as": self.matched_as, "add": self.add, "update": self.update,
|
||||
"keep": self.keep, "conflicts": self.conflicts, "skipped": self.skipped, "version": self.version,
|
||||
"error": self.error}
|
||||
|
||||
|
||||
def _raw_items(notes: Mapping[str, Any]) -> list[tuple[str, catalog.Slot | None, Any, str | None]]:
|
||||
"""文件里一张表的每一项：(文件里的路径, 槽位, 项, 不认的原因)。不认识的字段、不是对象的列也逐个列出来（槽位为 None）。"""
|
||||
out: list[tuple[str, catalog.Slot | None, Any, str | None]] = []
|
||||
for name, item in notes.items():
|
||||
if name in catalog.TABLE_FIELDS:
|
||||
out.append((name, ("t", name), item, None))
|
||||
elif name not in ("columns", "relations"):
|
||||
out.append((str(name), None, item, f"数据目录里不认识的字段「{name}」"))
|
||||
columns = notes.get("columns", {})
|
||||
if not isinstance(columns, Mapping):
|
||||
out.append(("columns", None, columns, "列的目录应为以列名为键的对象"))
|
||||
columns = {}
|
||||
for col, items in columns.items():
|
||||
if not isinstance(items, Mapping):
|
||||
out.append((f"columns.{col}", None, items, f"列 {col} 的目录应为对象"))
|
||||
continue
|
||||
for name, item in items.items():
|
||||
path = f"columns.{col}.{name}"
|
||||
if name in catalog.COLUMN_FIELDS:
|
||||
out.append((path, ("c", str(col), name), item, None))
|
||||
else:
|
||||
out.append((path, None, item, f"列 {col} 含有不认识的字段「{name}」"))
|
||||
relations = notes.get("relations", [])
|
||||
if not isinstance(relations, list):
|
||||
out.append(("relations", None, relations, "关联关系应为列表"))
|
||||
relations = []
|
||||
for rel in relations:
|
||||
rid = rel.get("id") if isinstance(rel, Mapping) else None
|
||||
out.append((f"relations.{rid}", ("r", str(rid)), rel, None))
|
||||
return out
|
||||
|
||||
|
||||
def _item_problem(key: catalog.Slot, item: Any) -> str | None:
|
||||
"""这一项本身合不合规（说法同 catalog.validate_notes）。"""
|
||||
if key[0] == "t":
|
||||
probe: dict[str, Any] = {key[1]: item}
|
||||
elif key[0] == "c":
|
||||
probe = {"columns": {key[1]: {key[2]: item}}}
|
||||
else:
|
||||
probe = {"relations": [item]}
|
||||
problems = catalog.validate_notes(probe)
|
||||
return problems[0] if problems else None
|
||||
|
||||
|
||||
def _decision(key: catalog.Slot, item: Mapping[str, Any]) -> Any:
|
||||
"""判断「一样不一样」看的部分：状态，加上值（码值另看「已列全」）。关联关系的两端由编号定了，再看基数；覆盖率不算。"""
|
||||
if key[0] == "r":
|
||||
return item.get("status"), item.get("cardinality")
|
||||
if key[0] == "c" and key[2] == "codes":
|
||||
return item.get("status"), item.get("value"), catalog.codes_complete(item)
|
||||
return item.get("status"), item.get("value")
|
||||
|
||||
|
||||
def _conflict_why(file_status: str, target_status: str) -> str:
|
||||
if target_status == "rejected":
|
||||
return f"目标里这一项已经驳回，文件里是{_STATUS_LABEL.get(file_status, file_status)}，保留目标的"
|
||||
if file_status == "rejected":
|
||||
return "目标里这一项已经人工确认过，文件里是已驳回，保留目标的"
|
||||
return "目标里这一项已经人工确认过，和文件里的不一样，保留目标的"
|
||||
|
||||
|
||||
class _Mapper:
|
||||
"""把文件里的一项换成目标的写法：列名、业务主键和业务日期的列、关联关系两端和编号；有效记录条件核对用到的列。对不上
|
||||
返回原因。name 是文件里的表名（条件里带它当表名也认），kind 是目标数据源的类型（解析条件用它的方言）。"""
|
||||
|
||||
def __init__(self, name: str, target: str, schema_cache: Mapping[str, Any], masked: set[str],
|
||||
kind: str | None) -> None:
|
||||
self.name = name
|
||||
self.target = target
|
||||
self.cache = schema_cache
|
||||
self.tables = (schema_cache or {}).get("tables") or {}
|
||||
self.columns = _Columns(self.tables.get(target))
|
||||
self.masked = masked
|
||||
self.kind = kind
|
||||
|
||||
def __call__(self, key: catalog.Slot, item: Any) -> tuple[catalog.Slot | None, Any, str | None]:
|
||||
if not isinstance(item, Mapping):
|
||||
return None, item, None # 交给合规检查说
|
||||
item = copy.deepcopy(dict(item))
|
||||
if key[0] == "c":
|
||||
col = self.columns.find(key[1])
|
||||
if col is None:
|
||||
return None, item, f"目标的表结构里没有列 {key[1]}，跳过"
|
||||
if key[2] == "codes" and col.lower() in self.masked:
|
||||
return None, item, f"列 {col} 在目标数据源的设置中被遮罩，码值是这一列的取值，不导入"
|
||||
return ("c", col, key[2]), item, None
|
||||
if key[0] == "t":
|
||||
value = item.get("value")
|
||||
if key[1] == "keys" and isinstance(value, list):
|
||||
mapped = [self.columns.find(c) for c in value]
|
||||
if missing := [str(c) for c, m in zip(value, mapped) if m is None]:
|
||||
return None, item, f"业务主键里的列「{'、'.join(missing)}」不在目标的表结构里，跳过"
|
||||
item["value"] = mapped
|
||||
elif key[1] == "business_date" and isinstance(value, Mapping) and isinstance(value.get("column"), str):
|
||||
col = self.columns.find(value["column"])
|
||||
if col is None:
|
||||
return None, item, f"业务日期的列「{value['column']}」不在目标的表结构里，跳过"
|
||||
item["value"] = {**value, "column": col}
|
||||
elif key[1] == "valid_filter":
|
||||
problem = filter_columns_problem(value, self.tables.get(self.target), kind=self.kind,
|
||||
table=self.target, also=(self.name,), where="目标的表结构")
|
||||
if problem is not None:
|
||||
return None, item, f"{problem}，跳过"
|
||||
return key, item, None
|
||||
return self._relation(item)
|
||||
|
||||
def _relation(self, rel: dict[str, Any]) -> tuple[catalog.Slot | None, Any, str | None]:
|
||||
cols, to_table, to_cols = rel.get("columns"), rel.get("to_table"), rel.get("to_columns")
|
||||
if not (isinstance(cols, list) and isinstance(to_cols, list) and isinstance(to_table, str)):
|
||||
return None, rel, None
|
||||
target_table = catalog.resolve_table_name(self.cache, to_table)
|
||||
if target_table is None:
|
||||
return None, rel, f"关联关系指向的表 {to_table} 不在目标数据源的表结构里，跳过"
|
||||
mine = [self.columns.find(c) for c in cols]
|
||||
if missing := [str(c) for c, m in zip(cols, mine) if m is None]:
|
||||
return None, rel, f"关联关系的本表字段「{'、'.join(missing)}」不在目标的表结构里，跳过"
|
||||
theirs_columns = _Columns(self.tables.get(target_table))
|
||||
theirs = [theirs_columns.find(c) for c in to_cols]
|
||||
if missing := [str(c) for c, m in zip(to_cols, theirs) if m is None]:
|
||||
return None, rel, f"关联关系的目标字段「{'、'.join(missing)}」不在 {target_table} 的表结构里，跳过"
|
||||
rid = catalog.relation_id(self.target, mine, target_table, theirs)
|
||||
return ("r", rid), {**rel, "id": rid, "columns": mine, "to_table": target_table, "to_columns": theirs}, None
|
||||
|
||||
|
||||
def plan_table(table_name: str, target: str, file_notes: Mapping[str, Any], target_notes: Mapping[str, Any] | None,
|
||||
schema_cache: Mapping[str, Any], masked: set[str], kind: str | None = None) -> TablePlan:
|
||||
"""文件里一张表并进目标那张表的计划（纯函数，不写库）。kind 是目标数据源的类型（解析有效记录条件用）。"""
|
||||
plan = TablePlan(table_name=table_name, matched_as=target)
|
||||
existing = catalog._slots(target_notes)
|
||||
mapper = _Mapper(table_name, target, schema_cache, masked, kind)
|
||||
seen: set[catalog.Slot] = set()
|
||||
for path, key, item, why in _raw_items(file_notes):
|
||||
if why is not None or key is None:
|
||||
plan.skipped.append({"path": path, "why": why or "无法识别这一项"})
|
||||
continue
|
||||
mapped, item, why = mapper(key, item)
|
||||
if why is not None:
|
||||
plan.skipped.append({"path": path, "why": why})
|
||||
continue
|
||||
problem = _item_problem(mapped or key, item)
|
||||
if problem is not None or mapped is None:
|
||||
plan.skipped.append({"path": path, "why": problem or "无法识别这一项"})
|
||||
continue
|
||||
if mapped in seen:
|
||||
plan.skipped.append({"path": path, "why": f"和文件里前面的{catalog.describe_slot(mapped)}是同一项，只导入第一项"})
|
||||
continue
|
||||
seen.add(mapped)
|
||||
status = item.get("status")
|
||||
have = existing.get(mapped)
|
||||
if have is None:
|
||||
plan.add += 1
|
||||
plan.writes[mapped] = item
|
||||
elif status not in _LOCKED:
|
||||
plan.keep += 1 # 推断、有确证的只填空缺
|
||||
elif have.get("status") in _LOCKED:
|
||||
if _decision(mapped, have) == _decision(mapped, item):
|
||||
plan.keep += 1
|
||||
else:
|
||||
plan.conflicts.append({
|
||||
"path": slot_path(mapped), "label": item_label(mapped, item),
|
||||
"file_value": _shown_value(mapped, item), "target_value": _shown_value(mapped, have),
|
||||
"file_status": status, "target_status": have.get("status"),
|
||||
"why": _conflict_why(str(status), str(have.get("status"))),
|
||||
})
|
||||
elif _decision(mapped, have) == _decision(mapped, item):
|
||||
plan.keep += 1
|
||||
else:
|
||||
plan.update += 1
|
||||
plan.writes[mapped] = item
|
||||
return plan
|
||||
|
||||
|
||||
def _shown_value(key: catalog.Slot, item: Mapping[str, Any]) -> Any:
|
||||
if key[0] == "r":
|
||||
return {k: item.get(k) for k in ("columns", "to_table", "to_columns", "cardinality", "coverage")}
|
||||
return item.get("value")
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 预览、导入
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportReport:
|
||||
tables: list[TablePlan]
|
||||
unmatched: list[str]
|
||||
guidance: dict[str, Any]
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"tables": [t.as_dict() for t in self.tables],
|
||||
"unmatched_tables": self.unmatched,
|
||||
"totals": {"tables": len(self.tables), "add": sum(t.add for t in self.tables),
|
||||
"update": sum(t.update for t in self.tables), "keep": sum(t.keep for t in self.tables),
|
||||
"conflicts": sum(len(t.conflicts) for t in self.tables),
|
||||
"skipped": sum(len(t.skipped) for t in self.tables)},
|
||||
"guidance": self.guidance,
|
||||
}
|
||||
|
||||
|
||||
def _guidance(file: ImportFile, row: Any, include: bool) -> dict[str, Any]:
|
||||
target = catalog_jobs.guidance_of(row)
|
||||
if not include or file.guidance is None:
|
||||
action = "none"
|
||||
elif target:
|
||||
action = "keep"
|
||||
else:
|
||||
action = "replace"
|
||||
return {"file": file.guidance, "target": target, "action": action}
|
||||
|
||||
|
||||
async def plan_import(session: AsyncSession, row: Any, source: Any, content: Any, *,
|
||||
include_guidance: bool) -> ImportReport:
|
||||
"""预览：逐表算出要新增、更新、保留什么，冲突和跳过的项及原因。不写库。row 是数据源记录（遮罩、起草要求按它），
|
||||
source 是读表结构用的源（上传源是绑定当前快照的视图）。格式不对抛 TransferInvalid。"""
|
||||
file = read_file(content, include_guidance=include_guidance)
|
||||
cache = getattr(source, "schema_cache", None) or {}
|
||||
matched, unmatched = match_tables(list(file.tables), cache)
|
||||
masked = {c.lower() for c in masked_columns(getattr(row, "options", None))}
|
||||
plans: list[TablePlan] = []
|
||||
for name, target in matched.items():
|
||||
entry = await catalog.read_entry(session, source.id, target)
|
||||
plan = plan_table(name, target, file.tables[name], entry.notes if entry else None, cache, masked, row.kind)
|
||||
plan.version = entry.version if entry else 0
|
||||
plans.append(plan)
|
||||
return ImportReport(plans, unmatched, _guidance(file, row, include_guidance))
|
||||
|
||||
|
||||
async def run_import(session: AsyncSession, row: Any, source: Any, content: Any, *, include_guidance: bool,
|
||||
actor: str | None) -> ImportReport:
|
||||
"""导入：按 plan_import 的规矩逐表写入（乐观锁，撞上别人刚改过就重读重算，最多三次），再按需写起草要求。
|
||||
一张表写不进去只记在它的 error 上（新增、更新记 0），不影响别的表。"""
|
||||
file = read_file(content, include_guidance=include_guidance)
|
||||
cache = getattr(source, "schema_cache", None) or {}
|
||||
matched, unmatched = match_tables(list(file.tables), cache)
|
||||
masked = {c.lower() for c in masked_columns(getattr(row, "options", None))}
|
||||
plans: list[TablePlan] = []
|
||||
for name, target in matched.items():
|
||||
plans.append(await _import_table(session, source, name, target, file.tables[name], cache, masked, row.kind,
|
||||
actor))
|
||||
guidance = _guidance(file, row, include_guidance)
|
||||
if guidance["action"] == "replace":
|
||||
await catalog_jobs.save_guidance(session, row, file.guidance)
|
||||
return ImportReport(plans, unmatched, guidance)
|
||||
|
||||
|
||||
async def _import_table(session: AsyncSession, source: Any, name: str, target: str, notes: Mapping[str, Any],
|
||||
cache: Mapping[str, Any], masked: set[str], kind: str | None, actor: str | None) -> TablePlan:
|
||||
plan = TablePlan(table_name=name, matched_as=target)
|
||||
try:
|
||||
for _attempt in range(3):
|
||||
entry = await catalog.read_entry(session, source.id, target)
|
||||
version = entry.version if entry else 0
|
||||
plan = plan_table(name, target, notes, entry.notes if entry else None, cache, masked, kind)
|
||||
plan.version = version
|
||||
if not plan.writes:
|
||||
return plan
|
||||
slots = catalog._slots(entry.notes if entry else None)
|
||||
slots.update(plan.writes)
|
||||
try:
|
||||
written = await catalog.write_entry(session, source.id, target, catalog._assemble(slots),
|
||||
if_version=version, actor=actor)
|
||||
except catalog.CatalogConflict:
|
||||
continue
|
||||
plan.version = written.version
|
||||
return plan
|
||||
plan.error = WRITE_CONFLICT_ERROR
|
||||
except catalog.CatalogInvalid as e:
|
||||
await session.rollback()
|
||||
plan.error = WRITE_ERROR.format(line=str(e))
|
||||
except Exception as e: # noqa: BLE001 - 这一张写不进去不影响别的表
|
||||
from app.core.errors import describe_exception, first_line
|
||||
|
||||
await session.rollback()
|
||||
plan.error = WRITE_ERROR.format(line=first_line(e) or describe_exception(e))
|
||||
plan.add = plan.update = 0
|
||||
return plan
|
||||
|
||||
|
||||
__all__ = ["FORMAT", "ImportFile", "ImportReport", "TablePlan", "TransferInvalid", "VERSION", "attachment_name",
|
||||
"content_disposition", "export_catalog", "filter_columns_problem", "match_tables", "plan_import",
|
||||
"plan_table", "read_file", "run_import"]
|
||||
@@ -0,0 +1,737 @@
|
||||
"""表助手（data/catalog_assistant.py，POST /api/datasources/{id}/catalog/{table}/assistant）。
|
||||
|
||||
- 开始前：表不在 404；起草模型用不了 409 catalog_draft_model_unavailable（说法同起草任务）；对话超过 40 条、一条超过
|
||||
4000 字 422。界面上待保存的修改不设上限（宽表上攒到 200 项以上也照常对话）：背景里只列最近的 200 项、写明另有几项，
|
||||
码值照全部待保存的修改合并
|
||||
- 流：text 增量、query（跑过的只读查询，前 10 行，遮罩列写「已遮罩」）、proposal（按 plan_patch 核对过的提案，形状同助手流
|
||||
的 catalog_patch：公式不收、列名必须在表结构里、遮罩列不给码值；有效记录条件要是只用这张表的列的一个条件，有起草的
|
||||
校验规矩 sqlcheck.valid_filter_problem 时按它）、done（用量、查询条数）、error（中途出错，之后没有 done）
|
||||
- 交给模型的背景：表结构、这张表的目录（每项的来源和状态）、关联关系（含别的表指过来的）、使用次数、数据源的起草要求、
|
||||
最近一次起草留空的列、界面上还没保存的修改；遮罩列只给列名、不给任何值
|
||||
- 只读查询：只有开了数据剖析的源才给这个工具；经剖析的只读守卫和单条时限；每次请求最多 5 条；最多 50 行交给模型；结果
|
||||
先遮再交给模型；没开剖析时提示词里写明不能查数据
|
||||
- 对话不存服务端；提案不写目录（保存走 PATCH，最多 200 项）
|
||||
|
||||
表名、列名、业务数字都是虚构的；号码是 139000000xx 这样的虚构号码。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sqlite3
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from langchain_core.messages import AIMessageChunk, HumanMessage, SystemMessage, ToolMessage
|
||||
|
||||
from app.data import catalog, catalog_assistant
|
||||
from app.data.catalog import make_item
|
||||
from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import CatalogDraftJob, CatalogDraftTask, DataSource
|
||||
from app.main import app as _fastapi_app
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
PHONES = [f"139000000{i:02d}" for i in range(1, 31)]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 假模型：按剧本一轮一轮回
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class Turn:
|
||||
text: list[str] = field(default_factory=list)
|
||||
calls: list[tuple[str, dict]] = field(default_factory=list)
|
||||
cut: bool = False
|
||||
usage: tuple[int, int] = (100, 20)
|
||||
error: BaseException | None = None
|
||||
hang: bool = False
|
||||
|
||||
|
||||
class Scripted:
|
||||
"""bind_tools 记下交给它的工具；astream 每调一次按剧本回一轮，记下这一轮收到的消息。"""
|
||||
|
||||
def __init__(self, turns: list[Turn]) -> None:
|
||||
self.turns = list(turns)
|
||||
self.seen: list[list[Any]] = []
|
||||
self.tools: list[Any] | None = None
|
||||
|
||||
def bind_tools(self, tools: list[Any], **_: Any) -> "Scripted":
|
||||
self.tools = tools
|
||||
return self
|
||||
|
||||
async def astream(self, messages: list[Any]):
|
||||
self.seen.append(list(messages))
|
||||
turn = self.turns.pop(0)
|
||||
if turn.error is not None:
|
||||
raise turn.error
|
||||
if turn.hang:
|
||||
await asyncio.sleep(3600)
|
||||
for piece in turn.text:
|
||||
yield AIMessageChunk(content=piece)
|
||||
for i, (name, args) in enumerate(turn.calls):
|
||||
yield AIMessageChunk(content="", tool_call_chunks=[
|
||||
{"name": name, "args": json.dumps(args, ensure_ascii=False), "id": f"c{len(self.seen)}_{i}",
|
||||
"index": i}])
|
||||
yield AIMessageChunk(content="", usage_metadata={"input_tokens": turn.usage[0], "output_tokens": turn.usage[1],
|
||||
"total_tokens": sum(turn.usage)},
|
||||
response_metadata={"finish_reason": "length" if turn.cut else "stop"})
|
||||
|
||||
def tool_names(self) -> list[str]:
|
||||
return [t["function"]["name"] for t in self.tools or []]
|
||||
|
||||
|
||||
class _Unavailable(Exception):
|
||||
status_code = 503
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 夹具
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _shop_db(path) -> str:
|
||||
conn = sqlite3.connect(path)
|
||||
conn.executescript("""
|
||||
CREATE TABLE members (id INTEGER PRIMARY KEY, nick TEXT);
|
||||
CREATE TABLE orders (id INTEGER PRIMARY KEY, order_no TEXT NOT NULL, status INTEGER NOT NULL,
|
||||
amount REAL, contact_phone TEXT, created_at TEXT,
|
||||
member_id INTEGER REFERENCES members(id));
|
||||
""")
|
||||
conn.executemany("INSERT INTO members VALUES (?, ?)", [(i, f"会员{i}") for i in range(1, 6)])
|
||||
conn.executemany("INSERT INTO orders VALUES (?, ?, ?, ?, ?, ?, ?)", [
|
||||
(i, f"NO{i:04d}", 9 if i % 10 == 0 else 1, 10.0 * i, PHONES[i - 1], f"2026-09-{i:02d}", i % 5 + 1)
|
||||
for i in range(1, 31)])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return str(path)
|
||||
|
||||
|
||||
ORDERS_NOTES = {
|
||||
"label": make_item("订单", "llm"),
|
||||
"grain": make_item("每笔订单一行", "human", "confirmed"),
|
||||
"kind": make_item("fact", "llm", "rejected"),
|
||||
"columns": {
|
||||
"status": {"codes": make_item({"1": "已支付"}, "profile", note="数据剖析：全表 30 行，2 个取值")},
|
||||
"contact_phone": {"label": make_item("联系电话", "llm"),
|
||||
"codes": make_item({PHONES[0]: "测试号"}, "human", "confirmed")},
|
||||
},
|
||||
}
|
||||
|
||||
_CREATED: list[str] = []
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def _cleanup():
|
||||
yield
|
||||
while _CREATED:
|
||||
await drop_source(_CREATED.pop())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
async def _source(client, tmp_path, *, profile: bool = True, guidance: str | None = "金额一律按元",
|
||||
notes: dict | None = None) -> str:
|
||||
options: dict[str, Any] = {"mask_columns": "contact_phone"}
|
||||
if profile:
|
||||
options["catalog_profile"] = {"enabled": True}
|
||||
if guidance:
|
||||
options[CATALOG_DRAFT_GUIDANCE_OPTION] = guidance
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=f"shop_{uuid.uuid4().hex[:8]}", kind="sqlite", database=_shop_db(tmp_path / "shop.db"),
|
||||
options=options, description="虚构的网店库")
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
sid = row.id
|
||||
_CREATED.append(sid)
|
||||
r = await client.post(f"/api/datasources/{sid}/introspect")
|
||||
assert r.status_code == 200, r.text
|
||||
async with SessionLocal() as session:
|
||||
await catalog.write_entry(session, sid, "orders", notes if notes is not None else ORDERS_NOTES,
|
||||
if_version=0, actor=None)
|
||||
return sid
|
||||
|
||||
|
||||
def use(monkeypatch, model: Scripted) -> None:
|
||||
async def resolve(session):
|
||||
return model, "fake-chat"
|
||||
|
||||
monkeypatch.setattr(catalog_assistant, "resolve_model", resolve)
|
||||
|
||||
|
||||
def _url(sid: str, table: str = "orders") -> str:
|
||||
return f"/api/datasources/{sid}/catalog/{table}/assistant"
|
||||
|
||||
|
||||
async def _talk(client, sid: str, body: dict, table: str = "orders") -> tuple[list[dict], list[str]]:
|
||||
"""发一轮，返回 (事件, 原始行)。"""
|
||||
events: list[dict] = []
|
||||
lines: list[str] = []
|
||||
async with client.stream("POST", _url(sid, table), json=body) as r:
|
||||
assert r.status_code == 200, await r.aread()
|
||||
assert r.headers["content-type"].startswith("text/event-stream")
|
||||
async for line in r.aiter_lines():
|
||||
lines.append(line)
|
||||
if line.startswith("data: "):
|
||||
events.append(json.loads(line[6:]))
|
||||
return events, lines
|
||||
|
||||
|
||||
def _text(events: list[dict]) -> str:
|
||||
return "".join(e["delta"] for e in events if e["type"] == "text")
|
||||
|
||||
|
||||
def _ask(text: str, **extra) -> dict:
|
||||
return {"messages": [{"role": "user", "content": text}], **extra}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 开始前的检查
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_refusals_before_the_stream(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
use(monkeypatch, Scripted([]))
|
||||
assert (await client.post(_url(sid, "ghost"), json=_ask("你好"))).status_code == 404
|
||||
assert (await client.post("/api/datasources/nope/catalog/orders/assistant", json=_ask("x"))).status_code == 404
|
||||
too_many = {"messages": [{"role": "user", "content": "x"}] * 41}
|
||||
assert (await client.post(_url(sid), json=too_many)).status_code == 422
|
||||
too_long = _ask("长" * 4001)
|
||||
assert (await client.post(_url(sid), json=too_long)).status_code == 422
|
||||
wrong_role = {"messages": [{"role": "system", "content": "x"}]}
|
||||
assert (await client.post(_url(sid), json=wrong_role)).status_code == 422
|
||||
last_is_answer = {"messages": [{"role": "user", "content": "x"}, {"role": "assistant", "content": "y"}]}
|
||||
r = await client.post(_url(sid), json=last_is_answer)
|
||||
assert r.status_code == 422 and "最后一条" in r.json()["detail"]
|
||||
|
||||
|
||||
async def test_unavailable_draft_model_is_409_with_the_draft_wording(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
|
||||
async def unavailable(session):
|
||||
raise catalog.CatalogModelUnavailable("未配置模型接入，无法由模型起草数据目录。请到「设置 → 模型接入」添加")
|
||||
|
||||
monkeypatch.setattr(catalog, "resolve_draft_model", unavailable)
|
||||
r = await client.post(_url(sid), json=_ask("你好"))
|
||||
assert r.status_code == 409
|
||||
assert r.json()["code"] == "catalog_draft_model_unavailable" and "模型接入" in r.json()["detail"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def mem(monkeypatch):
|
||||
"""单独的内存库：接入和设置只在这个用例里。"""
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.pool import StaticPool
|
||||
|
||||
from app.db.base import Base
|
||||
|
||||
for key in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "OPENAI_API_KEY", "ANTHROPIC_BASE_URL"):
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
engine = create_async_engine("sqlite+aiosqlite:///:memory:", poolclass=StaticPool)
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
session = async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)()
|
||||
yield session
|
||||
await session.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def _provider(session, kind: str) -> None:
|
||||
from app.core.crypto import encrypt
|
||||
from app.data import draft_model
|
||||
from app.db.models import Provider, Setting
|
||||
|
||||
session.add(Provider(name="起草网关", kind=kind, base_url="http://gw.example.test/v1" if kind != "anthropic" else None,
|
||||
api_key=encrypt("sk-test"), models=[{"id": "模型甲"}], default_model="模型甲", enabled=True,
|
||||
extra={}))
|
||||
session.add(Setting(key=draft_model.CATALOG_DRAFT_SETTING_KEY, value={"provider": "起草网关", "model": None}))
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def test_model_follows_the_draft_setting_keeps_thinking_and_a_big_budget(mem, monkeypatch):
|
||||
from app.providers import factory
|
||||
|
||||
await _provider(mem, "openai_compatible")
|
||||
specs: list[Any] = []
|
||||
real = factory.build_chat_model
|
||||
|
||||
def spy(provider, spec):
|
||||
specs.append(spec)
|
||||
return real(provider, spec)
|
||||
|
||||
monkeypatch.setattr(factory, "build_chat_model", spy)
|
||||
model, model_id = await catalog_assistant.resolve_model(mem)
|
||||
assert model_id == "模型甲" and model.max_tokens == catalog.MODEL_MAX_TOKENS == 32768
|
||||
mine = specs[-2:] # 前两份是 catalog.resolve_draft_model 查能不能用时建的
|
||||
assert [s.max_tokens for s in mine] == [catalog.MODEL_MAX_TOKENS, catalog.MODEL_FALLBACK_MAX_TOKENS]
|
||||
assert all(s.thinking is None and s.temperature == 0 and s.max_retries == 0 for s in mine)
|
||||
assert all(s.timeout == catalog.MODEL_TIMEOUT_S for s in mine)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["openai_compatible", "anthropic"])
|
||||
async def test_tools_bind_on_real_chat_models(mem, kind):
|
||||
"""两个工具的定义各家的模型都收（只绑定，不发请求）。"""
|
||||
await _provider(mem, kind)
|
||||
model, _ = await catalog_assistant.resolve_model(mem)
|
||||
tools = [catalog_assistant.QUERY_TOOL_PROMPT, catalog_assistant.PROPOSE_TOOL_PROMPT]
|
||||
bound = model.bind_tools(tools)
|
||||
given = bound.model.kwargs["tools"]
|
||||
names = [t.get("name") or t["function"]["name"] for t in given]
|
||||
assert names == ["run_query", "propose_changes"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 背景
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _blank_draft(sid: str) -> None:
|
||||
"""最近一次结束于 ok 的起草任务记下留空的列；之后一次停止了的不算。"""
|
||||
async with SessionLocal() as session:
|
||||
done = CatalogDraftJob(id=uuid.uuid4().hex, source_id=sid, status="succeeded",
|
||||
created_at=datetime(2026, 10, 1, tzinfo=timezone.utc))
|
||||
stopped = CatalogDraftJob(id=uuid.uuid4().hex, source_id=sid, status="cancelled",
|
||||
created_at=datetime(2026, 10, 2, tzinfo=timezone.utc))
|
||||
session.add_all([done, stopped])
|
||||
await session.flush()
|
||||
session.add_all([
|
||||
CatalogDraftTask(job_id=done.id, seq=0, table_name="orders", status="ok", blank_columns=2,
|
||||
blank_sample=["order_no", "created_at"]),
|
||||
CatalogDraftTask(job_id=stopped.id, seq=0, table_name="orders", status="cancelled", blank_columns=0),
|
||||
])
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def test_background_has_structure_catalog_and_hides_masked_values(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
await _blank_draft(sid)
|
||||
async with SessionLocal() as session:
|
||||
await catalog.write_entry(session, sid, "members", {"relations": [
|
||||
{"id": catalog.relation_id("members", ["id"], "orders", ["member_id"]), "columns": ["id"],
|
||||
"to_table": "orders", "to_columns": ["member_id"], "cardinality": "one_to_many", "coverage": None,
|
||||
"source": "human", "status": "confirmed"}]}, if_version=0, actor=None)
|
||||
model = Scripted([Turn(text=["这张表还缺 order_no 的含义。"])])
|
||||
use(monkeypatch, model)
|
||||
pending = [{"path": "columns.amount.unit", "value": "元", "reason": "用户说：金额是元"}]
|
||||
events, _ = await _talk(client, sid, _ask("看看还缺什么", pending=pending))
|
||||
assert _text(events) == "这张表还缺 order_no 的含义。"
|
||||
assert events[-1] == {"type": "done", "usage": {"input_tokens": 100, "output_tokens": 20}, "queries": 0}
|
||||
[messages] = model.seen
|
||||
assert isinstance(messages[0], SystemMessage) and isinstance(messages[-1], HumanMessage)
|
||||
system = messages[0].content
|
||||
# 规矩
|
||||
for rule in ("只改数据目录", "不改业务库", "公式", "拿不准", "原话"):
|
||||
assert rule in system
|
||||
# 有效记录条件的写法（同起草时的校验规矩），免得提了再被退回
|
||||
assert "只能用这张表的列" in system and "不能有子查询" in system and "UPPER、LOWER、TRIM、COALESCE" in system
|
||||
# 表结构、目录（来源和状态）、使用次数、起草要求、数据源说明
|
||||
for fact in ("order_no", "status", "amount", "member_id → members.id", "主键", "每笔订单一行", "人工填写", "已确认",
|
||||
"模型起草", "已驳回", "还没有被运行用到过", "金额一律按元", "虚构的网店库", "数据剖析：全表 30 行"):
|
||||
assert fact in system, fact
|
||||
# 别的表指过来的关联关系
|
||||
assert "members.id → orders.member_id" in system
|
||||
# 最近一次结束于 ok 的起草留空的列（之后停止了的那次不算）
|
||||
assert "order_no、created_at" in system
|
||||
# 界面上还没保存的修改
|
||||
assert "columns.amount.unit" in system and "用户说：金额是元" in system
|
||||
# 遮罩列只给列名：码值、取值一个都不给
|
||||
assert "contact_phone" in system
|
||||
assert not any(p in system for p in PHONES) and "测试号" not in system
|
||||
# 没开剖析的源才写不能查数据；这里开了，给了查询工具
|
||||
assert model.tool_names() == ["run_query", "propose_changes"]
|
||||
assert "没开剖析" not in system
|
||||
|
||||
|
||||
async def test_background_says_how_many_runs_used_the_table(client, tmp_path, monkeypatch):
|
||||
"""使用次数的口径同审阅队列：被几次运行用到（同「用过这张表的运行」的 total），查询几次（同目录清单的「使用 N 次」）。"""
|
||||
from app.db.models import Artifact, Run
|
||||
|
||||
sid = await _source(client, tmp_path)
|
||||
async with SessionLocal() as session:
|
||||
name = (await session.get(DataSource, sid)).name
|
||||
for i, n in enumerate((2, 1)):
|
||||
rid = uuid.uuid4().hex
|
||||
session.add(Run(id=rid, graph={}, workflow_name=f"周报{i}", status="succeeded"))
|
||||
for _ in range(n):
|
||||
session.add(Artifact(id=uuid.uuid4().hex, run_id=rid, kind=catalog.QUERY_SNAPSHOT_KIND,
|
||||
meta={"source_id": sid, "source": name, "tables": ["orders"]}))
|
||||
await session.commit()
|
||||
model = Scripted([Turn(text=["好。"])])
|
||||
use(monkeypatch, model)
|
||||
await _talk(client, sid, _ask("看看"))
|
||||
assert "这张表被 2 次运行用到（查询 3 次）。" in model.seen[0][0].content
|
||||
|
||||
|
||||
async def test_long_catalogs_are_cut_with_a_note(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
monkeypatch.setattr(catalog_assistant, "CATALOG_ITEMS_SHOWN", 2)
|
||||
model = Scripted([Turn(text=["好。"])])
|
||||
use(monkeypatch, model)
|
||||
await _talk(client, sid, _ask("看看"))
|
||||
system = model.seen[0][0].content
|
||||
# 表级的项在前；遮罩列的码值本来就不算；剩下的说明还有几项
|
||||
assert "- label（中文名）：订单" in system and "- grain（粒度）" in system and "- kind（表类型）" not in system
|
||||
assert "还有 3 项没有列出" in system
|
||||
|
||||
|
||||
async def test_without_profiling_there_is_no_query_tool(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path, profile=False)
|
||||
model = Scripted([Turn(text=["只能根据表结构推断。"])])
|
||||
use(monkeypatch, model)
|
||||
await _talk(client, sid, _ask("状态 9 是什么"))
|
||||
assert model.tool_names() == ["propose_changes"]
|
||||
assert "这个数据源没开剖析，不能查数据，只能根据表结构和目录推断" in model.seen[0][0].content
|
||||
|
||||
|
||||
async def test_opening_without_user_messages(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
model = Scripted([Turn(text=["order_no 是什么意思？"])])
|
||||
use(monkeypatch, model)
|
||||
events, _ = await _talk(client, sid, {"messages": []})
|
||||
assert _text(events) == "order_no 是什么意思？"
|
||||
assert isinstance(model.seen[0][-1], HumanMessage)
|
||||
# 界面先显示了一句开场白（助手说的）：照样从使用者这一边开始
|
||||
model = Scripted([Turn(text=["好的。"])])
|
||||
use(monkeypatch, model)
|
||||
await _talk(client, sid, {"messages": [{"role": "assistant", "content": "有两列留空了"},
|
||||
{"role": "user", "content": "order_no 是订单号"}]})
|
||||
kinds = [type(m).__name__ for m in model.seen[0]]
|
||||
assert kinds == ["SystemMessage", "HumanMessage", "AIMessage", "HumanMessage"]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 提案
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_proposals_are_checked_and_never_written(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
changes = [
|
||||
{"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说：9 是作废"},
|
||||
{"path": "valid_filter", "value": "status <> 9", "reason": "用户说：统计时排除作废"},
|
||||
{"path": "columns.amount.meaning", "value": "实收=单价*数量", "reason": "x"}, # 公式：不收
|
||||
{"path": "columns.ghost.label", "value": "幽灵", "reason": "x"}, # 列不存在：不收
|
||||
{"path": "columns.contact_phone.codes", "value": {PHONES[1]: "客服"}, "reason": "x"}, # 遮罩列：不给码值
|
||||
{"path": "grain", "value": "每笔订单一行", "reason": "x"}, # 已经是这个值：不用提
|
||||
]
|
||||
model = Scripted([Turn(text=["好的，"], calls=[("propose_changes", {"changes": changes})]),
|
||||
Turn(text=["已经整理成提案。"])])
|
||||
use(monkeypatch, model)
|
||||
events, _ = await _talk(client, sid, _ask("9 是作废，统计时排除"))
|
||||
assert _text(events) == "好的，已经整理成提案。"
|
||||
[proposal] = [e for e in events if e["type"] == "proposal"]
|
||||
assert proposal["source_id"] == sid and proposal["table"] == "orders" and proposal["version"] == 1
|
||||
assert proposal["source"].startswith("shop_") and proposal["table_label"] == "订单"
|
||||
by = {c["path"]: c for c in proposal["changes"]}
|
||||
assert set(by) == {"columns.status.codes", "valid_filter"}
|
||||
codes = by["columns.status.codes"]
|
||||
assert codes["label"] == "列 status 的码值" and codes["reason"] == "用户说：9 是作废"
|
||||
assert codes["before"] == {"1": "已支付"} and codes["after"] == {"1": "已支付", "9": "作废"}
|
||||
assert codes["value"] == {"9": "作废"} and codes["state"] == "change"
|
||||
assert by["valid_filter"]["label"] == "表的有效记录条件" and by["valid_filter"]["before"] is None
|
||||
# 没收的几项照实告诉模型（不带遮罩列的任何取值）
|
||||
tool = model.seen[1][-1]
|
||||
assert isinstance(tool, ToolMessage)
|
||||
for said in ("2 项", "公式", "ghost", "遮罩", "已经是这个值"):
|
||||
assert said in tool.content, said
|
||||
assert not any(p in tool.content for p in PHONES)
|
||||
assert events[-1]["type"] == "done" and events[-1]["usage"] == {"input_tokens": 200, "output_tokens": 40}
|
||||
async with SessionLocal() as session:
|
||||
entry = await catalog.read_entry(session, sid, "orders")
|
||||
assert entry.version == 1 and "valid_filter" not in entry.notes
|
||||
|
||||
|
||||
async def test_proposal_codes_build_on_the_pending_ones(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
model = Scripted([Turn(calls=[("propose_changes", {"changes": [
|
||||
{"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说：9 是作废"}]})]),
|
||||
Turn(text=["好。"])])
|
||||
use(monkeypatch, model)
|
||||
pending = [{"path": "columns.status.codes", "value": {"2": "已退款"}, "reason": "用户说：2 是退款"}]
|
||||
events, _ = await _talk(client, sid, _ask("9 是作废", pending=pending))
|
||||
[proposal] = [e for e in events if e["type"] == "proposal"]
|
||||
[change] = proposal["changes"]
|
||||
assert change["value"] == {"2": "已退款", "9": "作废"}
|
||||
assert change["after"] == {"1": "已支付", "2": "已退款", "9": "作废"}
|
||||
|
||||
|
||||
async def test_many_pending_changes_do_not_block_the_conversation(client, tmp_path, monkeypatch):
|
||||
"""宽表上逐列补中文名、含义，待保存的修改攒到 200 项以上：照常对话（不 422）。背景里只列最近的 200 项、写明另有几项；
|
||||
码值照全部待保存的修改合并（最早那条码值没列进背景，新提的码值也不能把它丢了）。"""
|
||||
sid = await _source(client, tmp_path)
|
||||
oldest = {"path": "columns.status.codes", "value": {"2": "已退款"}, "reason": "用户说：2 是退款"}
|
||||
labels = [{"path": f"columns.c{i:03d}.label", "value": f"第 {i} 列", "reason": f"用户说：第 {i} 列"}
|
||||
for i in range(201)]
|
||||
model = Scripted([Turn(calls=[("propose_changes", {"changes": [
|
||||
{"path": "columns.status.codes", "value": {"9": "作废"}, "reason": "用户说：9 是作废"}]})]),
|
||||
Turn(text=["好。"])])
|
||||
use(monkeypatch, model)
|
||||
events, _ = await _talk(client, sid, _ask("9 是作废", pending=[oldest, *labels]))
|
||||
assert events[-1]["type"] == "done"
|
||||
system = model.seen[0][0].content
|
||||
# 最近的 200 项列出来了，最早的两项没列，写明另有 2 项
|
||||
assert "columns.c200.label" in system and "columns.c001.label" in system
|
||||
assert "columns.c000.label" not in system and "用户说：2 是退款" not in system
|
||||
assert "另有 2 项" in system
|
||||
[proposal] = [e for e in events if e["type"] == "proposal"]
|
||||
[change] = proposal["changes"]
|
||||
assert change["value"] == {"2": "已退款", "9": "作废"}
|
||||
# 再多也收
|
||||
model = Scripted([Turn(text=["好。"])])
|
||||
use(monkeypatch, model)
|
||||
many = [{"path": f"columns.c{i:04d}.meaning", "value": "说明", "reason": "x"} for i in range(1200)]
|
||||
events, _ = await _talk(client, sid, _ask("接着看", pending=many))
|
||||
assert events[-1]["type"] == "done" and "另有 1000 项" in model.seen[0][0].content
|
||||
|
||||
|
||||
async def _propose_filters(client, sid: str, monkeypatch, filters: list[str]) -> tuple[list[dict], Scripted]:
|
||||
"""模型一轮提一条有效记录条件（同一条提案里同一个路径只认第一次），最后一轮回答。"""
|
||||
turns = [Turn(calls=[("propose_changes", {"changes": [
|
||||
{"path": "valid_filter", "value": f, "reason": "用户说：统计时排除作废"}]})]) for f in filters]
|
||||
model = Scripted([*turns, Turn(text=["好。"])])
|
||||
use(monkeypatch, model)
|
||||
events, _ = await _talk(client, sid, _ask("统计时排除作废的单子"))
|
||||
return events, model
|
||||
|
||||
|
||||
async def test_valid_filter_proposals_must_be_a_condition_on_this_table(client, tmp_path, monkeypatch):
|
||||
"""有效记录条件是模型写的 SQL：引用表结构里没有的列、别的表的列、带子查询的不交给界面，原因交回模型；合规的照常提。"""
|
||||
sid = await _source(client, tmp_path)
|
||||
events, model = await _propose_filters(client, sid, monkeypatch, [
|
||||
"status <> 9 AND ghost_col = 1",
|
||||
"members.nick IS NOT NULL",
|
||||
"member_id IN (SELECT id FROM members)",
|
||||
"status <> 9",
|
||||
])
|
||||
proposals = [e for e in events if e["type"] == "proposal"]
|
||||
assert [[c["value"] for c in p["changes"]] for p in proposals] == [["status <> 9"]]
|
||||
said = [[m for m in seen if isinstance(m, ToolMessage)][-1].content for seen in model.seen[1:5]]
|
||||
assert "ghost_col" in said[0] and "不在表结构里" in said[0] and "没有可以交给使用者的修改" in said[0]
|
||||
assert "members.nick" in said[1]
|
||||
assert "子查询" in said[2]
|
||||
assert "已把 1 项修改交给使用者确认" in said[3]
|
||||
|
||||
|
||||
async def test_valid_filter_proposals_follow_the_drafting_rules(client, tmp_path, monkeypatch):
|
||||
"""有起草的校验规矩（sqlcheck.valid_filter_problem：只能用几个函数、不能有聚合……）时按它核对，说法照它的。"""
|
||||
from app.data import sqlcheck
|
||||
|
||||
sid = await _source(client, tmp_path)
|
||||
calls: list[tuple] = []
|
||||
|
||||
def strict(condition, meta, *, kind, table=None):
|
||||
calls.append((condition, [c["name"] for c in meta["columns"]], kind, table))
|
||||
return "有效行过滤里只能用 UPPER、LOWER、TRIM、COALESCE 这几个函数，不能用 ABS" if "ABS" in condition else None
|
||||
|
||||
monkeypatch.setattr(sqlcheck, "valid_filter_problem", strict, raising=False)
|
||||
events, model = await _propose_filters(client, sid, monkeypatch, ["ABS(amount) > 0", "status <> 9"])
|
||||
proposals = [e for e in events if e["type"] == "proposal"]
|
||||
assert [[c["value"] for c in p["changes"]] for p in proposals] == [["status <> 9"]]
|
||||
assert "不能用 ABS" in [m for m in model.seen[1] if isinstance(m, ToolMessage)][-1].content
|
||||
assert calls[0][0] == "ABS(amount) > 0" and calls[0][2:] == ("sqlite", "orders")
|
||||
assert "status" in calls[0][1] and "member_id" in calls[0][1]
|
||||
|
||||
|
||||
async def test_nothing_acceptable_means_no_proposal_event(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
model = Scripted([Turn(calls=[("propose_changes", {"changes": [{"path": "columns.ghost.label", "value": "x"}]})]),
|
||||
Turn(text=["那一列不存在。"])])
|
||||
use(monkeypatch, model)
|
||||
events, _ = await _talk(client, sid, _ask("ghost 列叫什么"))
|
||||
assert not [e for e in events if e["type"] == "proposal"]
|
||||
assert "没有可以交给使用者的修改" in model.seen[1][-1].content
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 只读查询
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_queries_are_read_only_masked_and_capped(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
queries = [
|
||||
("run_query", {"sql": "SELECT status, COUNT(*) AS n FROM orders GROUP BY status ORDER BY status"}),
|
||||
("run_query", {"sql": "SELECT id, contact_phone AS phone, amount FROM orders ORDER BY id"}),
|
||||
("run_query", {"sql": "DELETE FROM orders"}),
|
||||
]
|
||||
model = Scripted([Turn(text=["我查一下。"], calls=queries),
|
||||
Turn(calls=[("run_query", {"sql": "SELECT 1"})] * 3),
|
||||
Turn(text=["9 占一成。"])])
|
||||
use(monkeypatch, model)
|
||||
events, _ = await _talk(client, sid, _ask("状态分布"))
|
||||
shown = [e for e in events if e["type"] == "query"]
|
||||
assert len(shown) == 5 # 每次请求最多 5 条
|
||||
first, second, third = shown[:3]
|
||||
assert first["columns"] == ["status", "n"] and first["rows"] == [[1, 27], [9, 3]]
|
||||
assert first["row_count"] == 2 and first["error"] is None
|
||||
# 遮罩列改了名字也认得出：值写「已遮罩」；界面只给前 10 行
|
||||
assert second["columns"] == ["id", "phone", "amount"] and len(second["rows"]) == 10
|
||||
assert {row[1] for row in second["rows"]} == {"已遮罩"} and second["row_count"] == 30
|
||||
# 只读守卫：写语句不执行
|
||||
assert third["error"] and "守卫" in third["error"] and third["rows"] == []
|
||||
# 交给模型的：最多 50 行、遮罩值看不到；第 6 条没执行
|
||||
second_round = model.seen[1]
|
||||
results = [m for m in second_round if isinstance(m, ToolMessage)]
|
||||
assert len(results) == 3
|
||||
assert json.loads(results[1].content)["rows"][29][1] == "已遮罩"
|
||||
assert not any(p in m.content for m in second_round for p in PHONES)
|
||||
capped = [m for m in model.seen[2] if isinstance(m, ToolMessage)][-1]
|
||||
assert "未执行" in capped.content and "5 条" in capped.content
|
||||
assert events[-1] == {"type": "done", "usage": {"input_tokens": 300, "output_tokens": 60}, "queries": 5}
|
||||
with sqlite3.connect(tmp_path / "shop.db") as conn:
|
||||
assert conn.execute("SELECT COUNT(*) FROM orders").fetchone()[0] == 30
|
||||
|
||||
|
||||
async def test_query_results_given_to_the_model_stop_at_50_rows(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
sql = "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 80) SELECT i FROM n"
|
||||
model = Scripted([Turn(calls=[("run_query", {"sql": sql})]), Turn(text=["好。"])])
|
||||
use(monkeypatch, model)
|
||||
events, _ = await _talk(client, sid, _ask("数一下"))
|
||||
[query] = [e for e in events if e["type"] == "query"]
|
||||
assert query["row_count"] == 50 and query["truncated"] is True and len(query["rows"]) == 10
|
||||
result = json.loads([m for m in model.seen[1] if isinstance(m, ToolMessage)][0].content)
|
||||
assert len(result["rows"]) == 50 and result["truncated"] is True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 中途出错
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_errors_end_the_stream_without_done(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
use(monkeypatch, Scripted([Turn(error=RuntimeError("接口不认这个模型"))]))
|
||||
events, _ = await _talk(client, sid, _ask("你好"))
|
||||
assert events[-1]["type"] == "error" and events[-1]["code"] == "assistant_model_error"
|
||||
assert "fake-chat" in events[-1]["message"] and not [e for e in events if e["type"] == "done"]
|
||||
use(monkeypatch, Scripted([Turn(text=["写到一半"], cut=True)]))
|
||||
events, _ = await _talk(client, sid, _ask("你好"))
|
||||
assert _text(events) == "写到一半"
|
||||
assert events[-1]["type"] == "error" and events[-1]["code"] == "assistant_truncated"
|
||||
assert "上限" in events[-1]["message"]
|
||||
|
||||
|
||||
async def test_a_model_without_tools_reports_an_error_event(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
|
||||
class NoTools(Scripted):
|
||||
def bind_tools(self, tools, **_):
|
||||
raise NotImplementedError("这个模型不支持工具调用")
|
||||
|
||||
use(monkeypatch, NoTools([]))
|
||||
events, _ = await _talk(client, sid, _ask("你好"))
|
||||
assert events == [{"type": "error", "code": "assistant_failed", "message": events[0]["message"]}]
|
||||
assert "工具调用" in events[0]["message"]
|
||||
|
||||
|
||||
async def test_transient_errors_are_retried_before_any_output(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
monkeypatch.setattr(catalog_assistant, "RETRY_DELAYS_S", (0.0, 0.0))
|
||||
model = Scripted([Turn(error=_Unavailable("服务暂时不可用")), Turn(text=["好了。"])])
|
||||
use(monkeypatch, model)
|
||||
events, _ = await _talk(client, sid, _ask("你好"))
|
||||
assert _text(events) == "好了。" and events[-1]["type"] == "done"
|
||||
assert len(model.seen) == 2
|
||||
|
||||
|
||||
async def test_a_call_over_the_time_limit_is_stopped(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.3)
|
||||
monkeypatch.setattr(catalog_assistant, "HEARTBEAT_S", 0.1)
|
||||
use(monkeypatch, Scripted([Turn(hang=True)]))
|
||||
events, lines = await _talk(client, sid, _ask("你好"))
|
||||
assert events == [events[-1]] and events[-1]["code"] == "assistant_timeout"
|
||||
assert "0.3 秒" in events[-1]["message"]
|
||||
# 等模型的时候按拍发 SSE 注释（不是事件），免得中间的代理断开
|
||||
assert any(line.startswith(":") for line in lines)
|
||||
|
||||
|
||||
async def test_steps_are_bounded(client, tmp_path, monkeypatch):
|
||||
sid = await _source(client, tmp_path)
|
||||
monkeypatch.setattr(catalog_assistant, "MAX_ROUNDS", 3)
|
||||
again = Turn(calls=[("propose_changes", {"changes": [{"path": "columns.ghost.label", "value": "x"}]})])
|
||||
model = Scripted([again, again, again])
|
||||
use(monkeypatch, model)
|
||||
events, _ = await _talk(client, sid, _ask("你好"))
|
||||
assert len(model.seen) == 3
|
||||
assert events[-1]["type"] == "error" and events[-1]["code"] == "assistant_rounds"
|
||||
# 倒数第二轮的工具结果里提醒模型直接回答
|
||||
assert "直接回答" in [m for m in model.seen[2] if isinstance(m, ToolMessage)][-1].content
|
||||
|
||||
|
||||
class _Slow(Scripted):
|
||||
"""说了半句就一直想：记下这次调用有没有被停掉。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__([])
|
||||
self.cancelled = False
|
||||
|
||||
async def astream(self, messages: list[Any]):
|
||||
self.seen.append(list(messages))
|
||||
yield AIMessageChunk(content="我想一想")
|
||||
try:
|
||||
await asyncio.sleep(3600)
|
||||
except asyncio.CancelledError:
|
||||
self.cancelled = True
|
||||
raise
|
||||
yield AIMessageChunk(content="……")
|
||||
|
||||
|
||||
async def test_stopping_the_stream_stops_the_model_call():
|
||||
"""使用者点停止（连接断开、流被关掉）：正在进行的模型调用跟着停，不在后台接着花额度。"""
|
||||
from app.data import catalog_profile
|
||||
|
||||
model = _Slow()
|
||||
ctx = catalog_assistant.AssistantContext(
|
||||
source_id="s", source_name="虚构库", kind="sqlite", table="orders", tables={}, query_source=None, options={},
|
||||
masked=set(), can_query=False, settings=catalog_profile.ProfileSettings(), system="系统提示")
|
||||
events = catalog_assistant.stream(model, "fake-chat", ctx, [{"role": "user", "content": "你好"}])
|
||||
first = await events.__anext__()
|
||||
assert json.loads(first[len("data: "):]) == {"type": "text", "delta": "我想一想"}
|
||||
await events.aclose()
|
||||
assert model.cancelled
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 保存：PATCH 一次最多 200 项
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_patch_accepts_up_to_200_changes(client):
|
||||
"""表助手一次能提很多项（逐列补中文名），保存不再卡在 20 项：预览、保存都收 200 项，201 项 422。"""
|
||||
assert catalog.PATCH_MAX_CHANGES == 200
|
||||
columns = [f"c{i:03d}" for i in range(200)]
|
||||
table = {"qualified": "wide", "primary_key": [], "comment": None, "is_view": False, "foreign_keys": [],
|
||||
"unique": [], "columns": [{"name": c, "type": "TEXT", "nullable": True, "comment": None} for c in columns]}
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=f"wide_{uuid.uuid4().hex[:8]}", kind="sqlite", database=":memory:",
|
||||
schema_cache={"tables": {"wide": table}})
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
sid = row.id
|
||||
_CREATED.append(sid)
|
||||
base = f"/api/datasources/{sid}/catalog/wide"
|
||||
changes = [{"path": f"columns.{c}.label", "value": f"第 {i} 列", "reason": "用户逐列说明"}
|
||||
for i, c in enumerate(columns)]
|
||||
preview = await client.post(f"{base}/patch/preview", json={"changes": changes})
|
||||
assert preview.status_code == 200, preview.text
|
||||
assert len(preview.json()["changes"]) == 200 and preview.json()["problems"] == []
|
||||
saved = await client.post(f"{base}/patch", json={"changes": changes, "if_version": 0})
|
||||
assert saved.status_code == 200, saved.text
|
||||
assert len(saved.json()["notes"]["columns"]) == 200
|
||||
assert saved.json()["notes"]["columns"]["c199"]["label"]["value"] == "第 199 列"
|
||||
too_many = changes + [{"path": "label", "value": "宽表"}]
|
||||
assert (await client.post(f"{base}/patch/preview", json={"changes": too_many})).status_code == 422
|
||||
assert (await client.post(f"{base}/patch", json={"changes": too_many, "if_version": 1})).status_code == 422
|
||||
@@ -0,0 +1,397 @@
|
||||
"""数据目录的审阅队列、批量审阅、表被哪些运行用过、审阅进度（data/catalog_review.py）。
|
||||
|
||||
- GET /catalog/review-queue：只列推断（proposed）的项，按「使用次数权重 × 风险权重 × 来源权重」从高到低，同分按表名、
|
||||
路径；facts（有效记录条件、去重规则、业务日期、关联关系、业务主键、度量类型）/ names（中文名、说明、含义、单位、码值、
|
||||
粒度、表类型）两组；按表筛选；分页；field_label 是给人看的位置（「列 status 的码值」）；reasons 写为什么排在这里
|
||||
（被几次运行用到，和 /usage 的 total 一致）；table_usage 是使用次数（同目录清单）；表结构里已经没有的表、列不列
|
||||
- POST /catalog/review/batch：逐表按 if_version 乐观锁，一张表冲突不影响别的表；同一张表的几条一次写入；路径不对、表不存在
|
||||
逐条报
|
||||
- GET /catalog/{table}/usage：用过这张表的运行，新的在前，取法同 catalog.table_usage
|
||||
- GET /catalog/review-progress：常用 20 张表（使用次数前 20 张，同起草的默认范围：用到的不够 20 张时按表结构的顺序
|
||||
补齐）、全部、影响数字的项各自的确认情况
|
||||
|
||||
表名、列名、业务数字都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.data import catalog, catalog_review
|
||||
from app.data.catalog import make_item
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact, DataSource, Run
|
||||
from app.main import app as _fastapi_app
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
ACTOR = {"X-Actor": quote("林晓")}
|
||||
TERMS = Path(__file__).resolve().parents[2] / "frontend" / "src" / "lib" / "terms.ts"
|
||||
|
||||
|
||||
def _col(name: str, type_: str = "INTEGER") -> dict:
|
||||
return {"name": name, "type": type_, "nullable": True, "comment": None}
|
||||
|
||||
|
||||
def _tables() -> dict:
|
||||
return {
|
||||
"orders": {"qualified": "orders", "primary_key": ["id"], "comment": None, "is_view": False,
|
||||
"foreign_keys": [], "unique": [],
|
||||
"columns": [_col("id"), _col("status"), _col("amount", "REAL"), _col("member_id"),
|
||||
_col("paid_at", "TEXT")]},
|
||||
"members": {"qualified": "members", "primary_key": ["id"], "comment": None, "is_view": False,
|
||||
"foreign_keys": [], "unique": [],
|
||||
"columns": [_col("id"), _col("nick", "TEXT"), _col("tier")]},
|
||||
"coupons": {"qualified": "coupons", "primary_key": ["id"], "comment": None, "is_view": False,
|
||||
"foreign_keys": [], "unique": [], "columns": [_col("id"), _col("face_value", "REAL")]},
|
||||
}
|
||||
|
||||
|
||||
def _member_rel(status: str = "proposed", source: str = "name") -> dict:
|
||||
return {"id": catalog.relation_id("orders", ["member_id"], "members", ["id"]), "columns": ["member_id"],
|
||||
"to_table": "members", "to_columns": ["id"], "cardinality": "many_to_one", "coverage": 0.98,
|
||||
"source": source, "status": status}
|
||||
|
||||
|
||||
ORDERS = {
|
||||
"label": make_item("订单", "llm"),
|
||||
"grain": make_item("每笔订单一行", "llm", "confirmed"),
|
||||
"valid_filter": make_item("status <> 9", "llm", note="剖析核对：排除 3.2% 的行（共 12,345 行）"),
|
||||
"kind": make_item("fact", "llm"),
|
||||
"columns": {
|
||||
"status": {"label": make_item("状态", "comment"), "codes": make_item({"1": "已支付", "9": ""}, "profile")},
|
||||
"amount": {"measure": make_item("flow", "llm"), "unit": make_item("元", "llm", "rejected")},
|
||||
# 表结构里已经没有这一列：不进队列
|
||||
"legacy_flag": {"label": make_item("旧标记", "llm")},
|
||||
},
|
||||
"relations": [_member_rel()],
|
||||
}
|
||||
MEMBERS = {"label": make_item("会员", "llm"), "keys": make_item(["id"], "name"),
|
||||
"columns": {"nick": {"label": make_item("昵称", "comment")}}}
|
||||
COUPONS = {"label": make_item("优惠券", "comment")}
|
||||
#: 表结构里已经没有这张表：不进队列
|
||||
GONE = {"label": make_item("旧表", "llm")}
|
||||
|
||||
_CREATED: list[str] = []
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def _cleanup():
|
||||
yield
|
||||
while _CREATED:
|
||||
await drop_source(_CREATED.pop())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
async def _write(sid: str, table: str, notes: dict) -> None:
|
||||
async with SessionLocal() as session:
|
||||
entry = await catalog.read_entry(session, sid, table)
|
||||
await catalog.write_entry(session, sid, table, notes, if_version=entry.version if entry else 0, actor=None)
|
||||
|
||||
|
||||
async def _use(sid: str, name: str, table_runs: dict[str, int]) -> list[str]:
|
||||
"""给每张表造几次运行查询它（每次运行一份查询快照），返回运行 id（先建的在前、开始得早）。"""
|
||||
runs: list[str] = []
|
||||
base = datetime(2026, 9, 1, tzinfo=timezone.utc)
|
||||
async with SessionLocal() as session:
|
||||
for table, n in table_runs.items():
|
||||
for _ in range(n):
|
||||
rid = uuid.uuid4().hex
|
||||
session.add(Run(id=rid, graph={}, workflow_name=f"周报-{table}", run_class="formal",
|
||||
status="succeeded", started_at=base + timedelta(hours=len(runs))))
|
||||
session.add(Artifact(id=uuid.uuid4().hex, run_id=rid, kind=catalog.QUERY_SNAPSHOT_KIND,
|
||||
meta={"source_id": sid, "source": name, "tables": [table]}))
|
||||
runs.append(rid)
|
||||
await session.commit()
|
||||
return runs
|
||||
|
||||
|
||||
async def _source(*, usage: dict[str, int] | None = None) -> str:
|
||||
name = f"rq_{uuid.uuid4().hex[:10]}"
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=name, kind="sqlite", database=":memory:", schema_cache={"tables": _tables()})
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
sid = row.id
|
||||
_CREATED.append(sid)
|
||||
for table, notes in (("orders", ORDERS), ("members", MEMBERS), ("coupons", COUPONS), ("gone_table", GONE)):
|
||||
await _write(sid, table, notes)
|
||||
if usage:
|
||||
await _use(sid, name, usage)
|
||||
return sid
|
||||
|
||||
|
||||
def _base(sid: str) -> str:
|
||||
return f"/api/datasources/{sid}/catalog"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 排序规则（纯函数）
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_priority_multiplies_usage_risk_and_source_weights():
|
||||
p = catalog_review.priority
|
||||
# 风险：有效记录条件 > 业务主键 > 码值 > 中文名
|
||||
assert p(0, ("t", "valid_filter"), "llm") > p(0, ("t", "keys"), "llm") > p(0, ("c", "x", "codes"), "llm") \
|
||||
> p(0, ("c", "x", "label"), "llm")
|
||||
assert p(0, ("r", "r1"), "name") == p(0, ("t", "dedup"), "name") == p(0, ("t", "business_date"), "name")
|
||||
assert p(0, ("c", "x", "measure"), "llm") == p(0, ("t", "keys"), "llm")
|
||||
assert p(0, ("t", "grain"), "llm") == p(0, ("t", "kind"), "llm") == p(0, ("c", "x", "codes"), "llm")
|
||||
assert p(0, ("c", "x", "meaning"), "llm") == p(0, ("c", "x", "unit"), "llm") == p(0, ("t", "label"), "llm")
|
||||
# 来源：模型起草、命名推断比数据库注释优先
|
||||
assert p(0, ("t", "label"), "llm") == p(0, ("t", "label"), "name") > p(0, ("t", "label"), "comment")
|
||||
# 使用次数：用得多的在前，但按对数增长，不压过风险
|
||||
assert p(37, ("t", "label"), "llm") > p(1, ("t", "label"), "llm") > p(0, ("t", "label"), "llm")
|
||||
assert p(37, ("t", "label"), "llm") == pytest.approx(
|
||||
p(0, ("t", "label"), "llm") * p(37, ("t", "label"), "comment") / p(0, ("t", "label"), "comment"))
|
||||
|
||||
|
||||
def test_source_labels_follow_the_ui():
|
||||
text = TERMS.read_text(encoding="utf-8")
|
||||
block = re.search(r"export const CATALOG_SOURCE_LABEL\b[^=]*=\s*\{(.*?)\n\}", text, re.S)
|
||||
assert block
|
||||
ui = dict(re.findall(r"(\w+):\s*'([^']+)'", block.group(1)))
|
||||
assert ui == catalog_review.SOURCE_LABEL
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 队列
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_queue_lists_only_proposed_items_by_priority(client):
|
||||
sid = await _source(usage={"orders": 37, "members": 2})
|
||||
r = await client.get(f"{_base(sid)}/review-queue")
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
items = body["items"]
|
||||
paths = [(i["table_name"], i["path"]) for i in items]
|
||||
# 已确认、已驳回的不列；表结构里没有的表、列不列
|
||||
assert ("orders", "grain") not in paths and ("orders", "columns.amount.unit") not in paths
|
||||
assert not any(t == "gone_table" for t, _ in paths)
|
||||
assert ("orders", "columns.legacy_flag.label") not in paths
|
||||
assert body["total"] == len(items) == 11
|
||||
assert all(i["status"] == "proposed" for i in items)
|
||||
# 从高到低，同分按表名、路径
|
||||
keys = [(-i["priority"], i["table_name"], i["path"]) for i in items]
|
||||
assert keys == sorted(keys)
|
||||
first = items[0]
|
||||
rel_id = _member_rel()["id"]
|
||||
assert (first["table_name"], first["path"]) in {("orders", "valid_filter"), ("orders", f"relations.{rel_id}")}
|
||||
vf = next(i for i in items if i["path"] == "valid_filter")
|
||||
assert vf["field"] == "valid_filter" and vf["field_label"] == "表的有效记录条件"
|
||||
assert vf["value"] == "status <> 9" and vf["display"] == "status <> 9"
|
||||
assert vf["source"] == "llm" and vf["note"] == "剖析核对：排除 3.2% 的行（共 12,345 行）"
|
||||
assert vf["table_usage"] == 37 and vf["version"] == 1
|
||||
assert vf["reasons"] == ["常用表（被 37 次运行用到）", "影响数字的项（有效记录条件）", "模型起草，未确认"]
|
||||
# 常用表上的中文名排在没人用的表上的中文名前面；同一张表上影响数字的项排在中文名前面
|
||||
order = [(i["table_name"], i["path"]) for i in items]
|
||||
assert order.index(("orders", "label")) < order.index(("coupons", "label"))
|
||||
assert order.index(("members", "keys")) < order.index(("members", "label"))
|
||||
|
||||
|
||||
async def test_queue_values_are_displayed_in_ui_words(client):
|
||||
sid = await _source()
|
||||
items = {(i["table_name"], i["path"]): i for i in (await client.get(f"{_base(sid)}/review-queue")).json()["items"]}
|
||||
rel = items[("orders", f"relations.{_member_rel()['id']}")]
|
||||
assert rel["field"] == "relation" and rel["field_label"] == "指向 members 的关联关系"
|
||||
assert rel["value"] == {"columns": ["member_id"], "to_table": "members", "to_columns": ["id"],
|
||||
"cardinality": "many_to_one", "coverage": 0.98}
|
||||
assert rel["display"] == "member_id → members.id（多对一，覆盖率 98%）"
|
||||
assert rel["reasons"][-1] == "命名推断，未确认"
|
||||
kind = items[("orders", "kind")]
|
||||
assert kind["display"] == "明细表" and kind["value"] == "fact"
|
||||
codes = items[("orders", "columns.status.codes")]
|
||||
assert codes["field"] == "codes" and codes["field_label"] == "列 status 的码值" and codes["column"] == "status"
|
||||
assert codes["display"] == "1=已支付、9=含义待填写"
|
||||
assert "1 个码值的含义待填写" in codes["reasons"] and "数据剖析，未确认" in codes["reasons"]
|
||||
assert items[("orders", "columns.amount.measure")]["display"] == "可累加"
|
||||
assert items[("members", "keys")]["display"] == "id" and items[("members", "keys")]["field_label"] == "表的业务主键"
|
||||
assert items[("orders", "columns.amount.measure")]["field_label"] == "列 amount 的度量类型"
|
||||
assert items[("members", "label")]["reasons"] == ["名称和说明（中文名）", "模型起草，未确认"]
|
||||
assert items[("coupons", "label")]["reasons"] == ["名称和说明（中文名）", "数据库注释，未确认"]
|
||||
|
||||
|
||||
async def test_queue_reasons_count_runs_like_the_usage_list(client):
|
||||
"""同一次运行里查了几次这张表：原因写「被几次运行用到」（和 /usage 的 total 一样），table_usage 是使用次数（和目录清单的
|
||||
「使用 N 次」一样），两个数不一样时原因里两个都写；优先级按使用次数（同清单的排序）。"""
|
||||
sid = await _source()
|
||||
async with SessionLocal() as session:
|
||||
name = (await session.get(DataSource, sid)).name
|
||||
runs = await _use(sid, name, {"orders": 2, "members": 1})
|
||||
async with SessionLocal() as session:
|
||||
for _ in range(3):
|
||||
session.add(Artifact(id=uuid.uuid4().hex, run_id=runs[0], kind=catalog.QUERY_SNAPSHOT_KIND,
|
||||
meta={"source_id": sid, "source": name, "tables": ["orders"]}))
|
||||
await session.commit()
|
||||
usage = await catalog.table_usage(session, await session.get(DataSource, sid))
|
||||
assert usage["orders"] == 5
|
||||
items = {(i["table_name"], i["path"]): i for i in (await client.get(f"{_base(sid)}/review-queue")).json()["items"]}
|
||||
vf = items[("orders", "valid_filter")]
|
||||
assert vf["table_usage"] == 5
|
||||
assert vf["reasons"][0] == "常用表（被 2 次运行用到，使用 5 次）"
|
||||
assert items[("members", "label")]["reasons"][0] == "常用表（被 1 次运行用到）"
|
||||
assert vf["priority"] == round(catalog_review.priority(5, ("t", "valid_filter"), "llm"), 4)
|
||||
assert (await client.get(f"{_base(sid)}/orders/usage")).json()["total"] == 2
|
||||
|
||||
|
||||
async def test_queue_filters_by_kind_and_table_and_pages(client):
|
||||
sid = await _source(usage={"orders": 3})
|
||||
facts = (await client.get(f"{_base(sid)}/review-queue", params={"kind": "facts"})).json()
|
||||
assert {i["field"] for i in facts["items"]} == {"valid_filter", "relation", "measure", "keys"}
|
||||
names = (await client.get(f"{_base(sid)}/review-queue", params={"kind": "names"})).json()
|
||||
assert {i["field"] for i in names["items"]} == {"label", "codes", "kind"}
|
||||
assert facts["total"] + names["total"] == 11
|
||||
only = (await client.get(f"{_base(sid)}/review-queue", params={"table": "MEMBERS"})).json()
|
||||
assert {i["table_name"] for i in only["items"]} == {"members"} and only["total"] == 3
|
||||
everything = (await client.get(f"{_base(sid)}/review-queue")).json()["items"]
|
||||
page = (await client.get(f"{_base(sid)}/review-queue", params={"limit": 4, "offset": 4})).json()
|
||||
assert page["total"] == 11 and page["items"] == everything[4:8]
|
||||
bad = await client.get(f"{_base(sid)}/review-queue", params={"kind": "other"})
|
||||
assert bad.status_code == 422
|
||||
assert (await client.get(f"{_base(sid)}/review-queue", params={"limit": 0})).status_code == 422
|
||||
missing = await client.get("/api/datasources/nope/catalog/review-queue")
|
||||
assert missing.status_code == 404
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 批量审阅
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_batch_review_writes_each_table_once_and_isolates_conflicts(client):
|
||||
sid = await _source()
|
||||
rel_path = f"relations.{_member_rel()['id']}"
|
||||
body = {"items": [
|
||||
{"table_name": "orders", "path": "valid_filter", "action": "confirm", "if_version": 1},
|
||||
{"table_name": "orders", "path": rel_path, "action": "reject", "if_version": 1},
|
||||
{"table_name": "orders", "path": "columns.nothing.label", "action": "confirm", "if_version": 1},
|
||||
{"table_name": "members", "path": "label", "action": "confirm", "if_version": 7}, # 版本不对
|
||||
{"table_name": "coupons", "path": "label", "action": "reject", "if_version": 1},
|
||||
{"table_name": "ghost", "path": "label", "action": "confirm", "if_version": 0},
|
||||
]}
|
||||
r = await client.post(f"{_base(sid)}/review/batch", json=body, headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
results = r.json()["results"]
|
||||
assert [(x["table_name"], x["path"], x["ok"]) for x in results] == [
|
||||
("orders", "valid_filter", True), ("orders", rel_path, True), ("orders", "columns.nothing.label", False),
|
||||
("members", "label", False), ("coupons", "label", True), ("ghost", "label", False)]
|
||||
# 同一张表的两条一次写进去：版本只加 1
|
||||
assert results[0]["version"] == results[1]["version"] == 2
|
||||
assert results[2]["code"] == "catalog_path_invalid" and "找不到" in results[2]["error"]
|
||||
assert results[2]["version"] is None
|
||||
assert results[3]["code"] == "catalog_conflict" and "重新载入" in results[3]["error"]
|
||||
assert results[4]["version"] == 2 and results[4]["error"] is None and results[4]["code"] is None
|
||||
assert results[5]["code"] == "table_missing"
|
||||
async with SessionLocal() as session:
|
||||
orders = await catalog.read_entry(session, sid, "orders")
|
||||
members = await catalog.read_entry(session, sid, "members")
|
||||
coupons = await catalog.read_entry(session, sid, "coupons")
|
||||
assert orders.version == 2 and orders.updated_by == "林晓"
|
||||
assert orders.notes["valid_filter"]["status"] == "confirmed"
|
||||
assert orders.notes["relations"][0]["status"] == "rejected"
|
||||
assert members.version == 1 and members.notes["label"]["status"] == "proposed"
|
||||
assert coupons.notes["label"]["status"] == "rejected"
|
||||
# 撤销审阅：回到来源的初始状态
|
||||
again = await client.post(f"{_base(sid)}/review/batch", json={"items": [
|
||||
{"table_name": "orders", "path": "valid_filter", "action": "reset", "if_version": 2}]})
|
||||
assert again.json()["results"][0]["ok"] is True
|
||||
async with SessionLocal() as session:
|
||||
assert (await catalog.read_entry(session, sid, "orders")).notes["valid_filter"]["status"] == "proposed"
|
||||
|
||||
|
||||
async def test_batch_review_limits(client):
|
||||
sid = await _source()
|
||||
many = [{"table_name": "orders", "path": "label", "action": "confirm", "if_version": 1}] * 201
|
||||
assert (await client.post(f"{_base(sid)}/review/batch", json={"items": many})).status_code == 422
|
||||
assert (await client.post(f"{_base(sid)}/review/batch", json={"items": []})).status_code == 422
|
||||
wrong = [{"table_name": "orders", "path": "label", "action": "approve", "if_version": 1}]
|
||||
assert (await client.post(f"{_base(sid)}/review/batch", json={"items": wrong})).status_code == 422
|
||||
ok = [{"table_name": "orders", "path": "label", "action": "confirm", "if_version": 1}] * 200
|
||||
r = await client.post(f"{_base(sid)}/review/batch", json={"items": ok})
|
||||
assert r.status_code == 200 and all(x["ok"] for x in r.json()["results"])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 被哪些运行用过
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_table_usage_lists_runs_newest_first(client):
|
||||
sid = await _source()
|
||||
async with SessionLocal() as session:
|
||||
name = (await session.get(DataSource, sid)).name
|
||||
runs = await _use(sid, name, {"orders": 3, "members": 1})
|
||||
# 同一次运行查了两次 orders：只算一次运行
|
||||
async with SessionLocal() as session:
|
||||
session.add(Artifact(id=uuid.uuid4().hex, run_id=runs[0], kind=catalog.QUERY_SNAPSHOT_KIND,
|
||||
meta={"source_id": sid, "source": name, "tables": ["orders", "members"]}))
|
||||
# 别的数据源的同名表不算
|
||||
other = uuid.uuid4().hex
|
||||
session.add(Run(id=other, graph={}, workflow_name="别处", status="succeeded"))
|
||||
session.add(Artifact(id=uuid.uuid4().hex, run_id=other, kind=catalog.QUERY_SNAPSHOT_KIND,
|
||||
meta={"source_id": "elsewhere", "source": "elsewhere", "tables": ["orders"]}))
|
||||
await session.commit()
|
||||
r = await client.get(f"{_base(sid)}/orders/usage")
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
assert body["total"] == 3
|
||||
assert [x["run_id"] for x in body["runs"]] == [runs[2], runs[1], runs[0]]
|
||||
first = body["runs"][0]
|
||||
assert first["workflow_name"] == "周报-orders" and first["run_class"] == "formal"
|
||||
assert first["status"] == "succeeded" and first["started_at"].startswith("2026-09-01T02:00")
|
||||
assert set(first) == {"run_id", "workflow_id", "workflow_name", "run_class", "status", "started_at"}
|
||||
limited = (await client.get(f"{_base(sid)}/orders/usage", params={"limit": 1})).json()
|
||||
assert limited["total"] == 3 and len(limited["runs"]) == 1
|
||||
members = (await client.get(f"{_base(sid)}/members/usage")).json()
|
||||
assert members["total"] == 2
|
||||
assert (await client.get(f"{_base(sid)}/ghost/usage")).status_code == 404
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 进度
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_review_progress_counts_top_tables_all_and_facts(client, monkeypatch):
|
||||
sid = await _source(usage={"orders": 5})
|
||||
r = await client.get(f"{_base(sid)}/review-progress")
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
assert body["top_n"] == 20
|
||||
# 表结构里只有 3 张表：都算常用表（使用次数前 20 张）；表结构里没有的表、列不算
|
||||
assert body["top"] == {"tables": 3, "items_total": 13, "items_confirmed": 1}
|
||||
assert body["all"] == {"items_total": 13, "items_confirmed": 1, "items_proposed": 11, "items_rejected": 1}
|
||||
# 影响数字的项：有效记录条件、关联关系、度量类型、业务主键
|
||||
assert body["facts"] == {"proposed": 4, "confirmed": 0}
|
||||
await client.post(f"{_base(sid)}/review/batch", json={"items": [
|
||||
{"table_name": "orders", "path": "valid_filter", "action": "confirm", "if_version": 1}]})
|
||||
after = (await client.get(f"{_base(sid)}/review-progress")).json()
|
||||
assert after["top"]["items_confirmed"] == 2 and after["facts"] == {"proposed": 3, "confirmed": 1}
|
||||
# 只取前 2 张：用得多的在前，没人用的按表结构的顺序补（同起草的默认范围）
|
||||
monkeypatch.setattr(catalog_review, "TOP_N", 2)
|
||||
used = await _source(usage={"coupons": 1})
|
||||
top = (await client.get(f"{_base(used)}/review-progress")).json()["top"]
|
||||
assert top == {"tables": 2, "items_total": 1 + 9, "items_confirmed": 1}
|
||||
|
||||
|
||||
def test_top_tables_are_the_draft_default_scope():
|
||||
"""常用表和起草对话框「使用次数最多的 20 张表」是同一批：使用次数从多到少，一样多（包括都没用过）的按表结构里的顺序。"""
|
||||
tables = {f"t{i:02d}": {} for i in range(30)}
|
||||
usage = {"t29": 4, "t05": 4, "t17": 9}
|
||||
top = catalog_review.top_tables(tables, usage)
|
||||
assert catalog_review.TOP_N == catalog.DRAFT_DEFAULT_TABLES == 20
|
||||
assert top[:3] == ["t17", "t05", "t29"] and len(top) == 20
|
||||
assert top[3:] == [n for n in tables if n not in usage][:17]
|
||||
assert catalog_review.top_tables({"a": {}, "b": {}}, {}) == ["a", "b"]
|
||||
@@ -0,0 +1,411 @@
|
||||
"""数据目录的导出导入（data/catalog_transfer.py）：本机整理好的目录带到部署实例上。
|
||||
|
||||
- GET /catalog/export：附件「<数据源名>-数据目录.json」，{format: agentlab-catalog, version: 1, exported_at,
|
||||
source: {name, kind}, guidance, tables: {表名: {notes, version}}}，含全部状态的项（表结构里已经没有的表也带）
|
||||
- POST /catalog/import/preview：只读，报告每张对上的表新增、更新、保留、冲突、跳过了什么
|
||||
- POST /catalog/import：同样的报告，加每张表写入后的版本。规则：
|
||||
- 表按名字对（忽略大小写；带 schema 前缀的对不上时去掉前缀再对），对不上的列在 unmatched_tables
|
||||
- 文件里已确认、已驳回的项照带（来源、状态不变）；目标里已确认、已驳回的项和文件不一样时保留目标的、记冲突
|
||||
- 文件里推断、有确证的项只填目标的空缺，不覆盖目标已有的
|
||||
- 表结构里没有的列、关系两端对不上的跳过并写原因；目标数据源遮罩的列，码值不导入；有效记录条件用到的列（按目标的方言
|
||||
解析）不在目标的表结构里、引用别的表的列、带子查询的跳过（人工写的条件不限函数）
|
||||
- 逐表乐观锁，一张表写不进去不影响别的；有排队或进行中的起草任务时 409 catalog_draft_busy；文件格式不对 422
|
||||
- 起草要求：勾了导入、目标还没有时才写；目标已有的不覆盖
|
||||
|
||||
表名、列名、业务数字都是虚构的。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import uuid
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.data import catalog, catalog_jobs, catalog_transfer
|
||||
from app.data.catalog import make_item
|
||||
from app.data.engine import CATALOG_DRAFT_GUIDANCE_OPTION
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import CatalogDraftJob, DataSource
|
||||
from app.main import app as _fastapi_app
|
||||
from tests.fixtures.sources import drop_source
|
||||
|
||||
ACTOR = {"X-Actor": quote("沈岚")}
|
||||
|
||||
|
||||
def _col(name: str, type_: str = "INTEGER") -> dict:
|
||||
return {"name": name, "type": type_, "nullable": True, "comment": None}
|
||||
|
||||
|
||||
def _tables(*, upper: bool = False) -> dict:
|
||||
"""两边的表结构：目标那边（upper）表名、列名写法不同，还少了一列 tier。"""
|
||||
def n(name: str) -> str:
|
||||
return name.upper() if upper else name
|
||||
|
||||
members_cols = [_col(n("id")), _col(n("nick"), "TEXT")] + ([] if upper else [_col("tier")])
|
||||
return {
|
||||
n("orders"): {"qualified": n("orders"), "primary_key": [n("id")], "comment": None, "is_view": False,
|
||||
"foreign_keys": [], "unique": [],
|
||||
"columns": [_col(n("id")), _col(n("status")), _col(n("amount"), "REAL"), _col(n("member_id")),
|
||||
_col(n("contact_phone"), "TEXT")]},
|
||||
n("members"): {"qualified": n("members"), "primary_key": [n("id")], "comment": None, "is_view": False,
|
||||
"foreign_keys": [], "unique": [], "columns": members_cols},
|
||||
n("refunds"): {"qualified": n("refunds"), "primary_key": [n("id")], "comment": None, "is_view": False,
|
||||
"foreign_keys": [], "unique": [], "columns": [_col(n("id")), _col(n("order_id"))]},
|
||||
}
|
||||
|
||||
|
||||
def _rel(to_table: str, columns: list[str], to_columns: list[str], *, status: str, source: str = "name",
|
||||
table: str = "orders") -> dict:
|
||||
return {"id": catalog.relation_id(table, columns, to_table, to_columns), "columns": columns,
|
||||
"to_table": to_table, "to_columns": to_columns, "cardinality": "many_to_one", "coverage": None,
|
||||
"source": source, "status": status}
|
||||
|
||||
|
||||
#: 本机（源）的目录
|
||||
SOURCE_ORDERS = {
|
||||
"label": make_item("订单", "human", "confirmed", at="2026-10-01T08:00:00+00:00"),
|
||||
"grain": make_item("每笔订单一行", "llm"),
|
||||
"valid_filter": make_item("status <> 9", "human", "confirmed"),
|
||||
"dedup": make_item("同一订单号取最后一次", "llm", "rejected"),
|
||||
"kind": make_item("fact", "llm", "confirmed"),
|
||||
"columns": {
|
||||
"status": {"codes": make_item({"1": "已支付", "9": "作废"}, "human", "confirmed", complete=True),
|
||||
"label": make_item("状态", "llm")},
|
||||
"amount": {"unit": make_item("元", "human", "confirmed"), "measure": make_item("flow", "llm")},
|
||||
"contact_phone": {"codes": make_item({"13900000001": "测试号"}, "human", "confirmed"),
|
||||
"label": make_item("联系电话", "human", "confirmed")},
|
||||
# 目标表结构里没有这一列
|
||||
"gift_flag": {"label": make_item("赠品标记", "human", "confirmed")},
|
||||
},
|
||||
"relations": [
|
||||
_rel("members", ["member_id"], ["id"], status="confirmed"),
|
||||
# 目标那边没有这张表
|
||||
_rel("stores", ["member_id"], ["id"], status="rejected"),
|
||||
],
|
||||
}
|
||||
SOURCE_MEMBERS = {"label": make_item("会员", "llm"), "keys": make_item(["tier"], "human", "confirmed"),
|
||||
"columns": {"nick": {"label": make_item("昵称", "llm", "verified")}}}
|
||||
#: 目标那边没有的表
|
||||
SOURCE_GHOST = {"label": make_item("旧表", "human", "confirmed")}
|
||||
|
||||
#: 目标（部署实例）上已有的目录
|
||||
TARGET_ORDERS = {
|
||||
"label": make_item("订单表", "human", "confirmed"), # 和文件不一样、都确认过：冲突，保留目标
|
||||
"grain": make_item("一行一单", "llm"), # 文件里是推断：保留目标
|
||||
"valid_filter": make_item("STATUS <> 9", "llm"), # 目标是推断、文件确认过：更新
|
||||
"kind": make_item("fact", "human", "confirmed"), # 一样：保留
|
||||
"columns": {"STATUS": {"label": make_item("状态码", "comment")}},
|
||||
}
|
||||
|
||||
_CREATED: list[str] = []
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def _cleanup():
|
||||
yield
|
||||
while _CREATED:
|
||||
await drop_source(_CREATED.pop())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=_fastapi_app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
async def _write(sid: str, table: str, notes: dict) -> None:
|
||||
async with SessionLocal() as session:
|
||||
entry = await catalog.read_entry(session, sid, table)
|
||||
await catalog.write_entry(session, sid, table, notes, if_version=entry.version if entry else 0, actor=None)
|
||||
|
||||
|
||||
async def _source(name: str, tables: dict, entries: dict[str, dict], **kw) -> str:
|
||||
async with SessionLocal() as session:
|
||||
row = DataSource(name=name, kind="sqlite", database=":memory:", schema_cache={"tables": tables}, **kw)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
sid = row.id
|
||||
_CREATED.append(sid)
|
||||
for table, notes in entries.items():
|
||||
await _write(sid, table, notes)
|
||||
return sid
|
||||
|
||||
|
||||
async def _pair() -> tuple[str, str]:
|
||||
local = await _source(f"本机库_{uuid.uuid4().hex[:6]}", _tables(),
|
||||
{"orders": SOURCE_ORDERS, "members": SOURCE_MEMBERS, "ghost_table": SOURCE_GHOST},
|
||||
options={CATALOG_DRAFT_GUIDANCE_OPTION: "金额一律按元"})
|
||||
remote = await _source(f"deploy_{uuid.uuid4().hex[:6]}", _tables(upper=True), {"ORDERS": TARGET_ORDERS},
|
||||
options={"mask_columns": "contact_phone"})
|
||||
return local, remote
|
||||
|
||||
|
||||
def _base(sid: str) -> str:
|
||||
return f"/api/datasources/{sid}/catalog"
|
||||
|
||||
|
||||
async def _export(client, sid: str) -> dict:
|
||||
r = await client.get(f"{_base(sid)}/export")
|
||||
assert r.status_code == 200, r.text
|
||||
return r.json()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 导出
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_export_is_a_json_attachment_with_every_item(client):
|
||||
local, _ = await _pair()
|
||||
r = await client.get(f"{_base(local)}/export")
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.headers["content-type"].startswith("application/json")
|
||||
disposition = r.headers["content-disposition"]
|
||||
assert disposition.startswith("attachment;") and "filename=\"agentlab-catalog-" in disposition
|
||||
async with SessionLocal() as session:
|
||||
name = (await session.get(DataSource, local)).name
|
||||
assert unquote(disposition.split("filename*=UTF-8''", 1)[1]) == f"{name}-数据目录.json"
|
||||
body = r.json()
|
||||
assert body["format"] == "agentlab-catalog" and body["version"] == 1
|
||||
assert body["exported_at"] and body["source"] == {"name": name, "kind": "sqlite"}
|
||||
assert body["guidance"] == "金额一律按元"
|
||||
assert set(body["tables"]) == {"orders", "members", "ghost_table"}
|
||||
assert body["tables"]["orders"]["version"] == 1
|
||||
# 含全部状态的项：驳回的也带
|
||||
assert body["tables"]["orders"]["notes"]["dedup"]["status"] == "rejected"
|
||||
assert body["tables"]["orders"]["notes"] == SOURCE_ORDERS
|
||||
assert body["tables"]["ghost_table"] == {"notes": SOURCE_GHOST, "version": 1}
|
||||
|
||||
|
||||
def test_attachment_name_drops_characters_files_cannot_have():
|
||||
assert catalog_transfer.attachment_name('景区/客流:库?') == "景区_客流_库_-数据目录.json"
|
||||
assert catalog_transfer.attachment_name(" ") == "数据源-数据目录.json"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 预览与导入
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _by(report: dict) -> dict[str, dict]:
|
||||
return {t["matched_as"]: t for t in report["tables"]}
|
||||
|
||||
|
||||
async def test_preview_reports_and_writes_nothing(client):
|
||||
local, remote = await _pair()
|
||||
content = await _export(client, local)
|
||||
r = await client.post(f"{_base(remote)}/import/preview", json={"content": content, "include_guidance": True})
|
||||
assert r.status_code == 200, r.text
|
||||
report = r.json()
|
||||
assert report["unmatched_tables"] == ["ghost_table"]
|
||||
tables = _by(report)
|
||||
assert set(tables) == {"ORDERS", "MEMBERS"}
|
||||
orders = tables["ORDERS"]
|
||||
assert orders["table_name"] == "orders"
|
||||
conflicts = {c["path"]: c for c in orders["conflicts"]}
|
||||
assert set(conflicts) == {"label"}
|
||||
label = conflicts["label"]
|
||||
assert label["label"] == "表的中文名" and label["file_value"] == "订单" and label["target_value"] == "订单表"
|
||||
assert "保留" in label["why"]
|
||||
skipped = {s["path"]: s["why"] for s in orders["skipped"]}
|
||||
assert "columns.gift_flag.label" in skipped and "表结构里没有列 gift_flag" in skipped["columns.gift_flag.label"]
|
||||
assert "遮罩" in skipped["columns.contact_phone.codes"]
|
||||
stores = next(p for p in skipped if p.startswith("relations."))
|
||||
assert "stores" in skipped[stores]
|
||||
# 新增：dedup（驳回的照带）、status 码值、amount 两项、联系电话的中文名、关联关系；更新：有效记录条件
|
||||
# 保留：粒度（文件里是推断，目标已有）、表类型（一样）、STATUS 的中文名（文件里是推断，目标已有）
|
||||
assert (orders["add"], orders["update"], orders["keep"]) == (6, 1, 3)
|
||||
members = tables["MEMBERS"]
|
||||
# 业务主键指着的列 tier 目标那边没有：跳过；会员的中文名和昵称填空缺
|
||||
assert (members["add"], members["update"], members["keep"]) == (2, 0, 0)
|
||||
assert [s["path"] for s in members["skipped"]] == ["keys"]
|
||||
assert report["totals"] == {"tables": 2, "add": 8, "update": 1, "keep": 3, "conflicts": 1,
|
||||
"skipped": len(orders["skipped"]) + 1}
|
||||
assert report["guidance"] == {"file": "金额一律按元", "target": None, "action": "replace"}
|
||||
async with SessionLocal() as session:
|
||||
assert (await catalog.read_entry(session, remote, "ORDERS")).version == 1
|
||||
assert await catalog.read_entry(session, remote, "MEMBERS") is None
|
||||
assert catalog_jobs.guidance_of(await session.get(DataSource, remote)) is None
|
||||
|
||||
|
||||
async def test_import_writes_by_the_rules_and_reports_versions(client):
|
||||
local, remote = await _pair()
|
||||
content = await _export(client, local)
|
||||
r = await client.post(f"{_base(remote)}/import", json={"content": content, "include_guidance": True},
|
||||
headers=ACTOR)
|
||||
assert r.status_code == 200, r.text
|
||||
report = r.json()
|
||||
tables = _by(report)
|
||||
assert tables["ORDERS"]["version"] == 2 and tables["MEMBERS"]["version"] == 1
|
||||
assert tables["ORDERS"]["error"] is None
|
||||
assert report["guidance"]["action"] == "replace"
|
||||
async with SessionLocal() as session:
|
||||
orders = await catalog.read_entry(session, remote, "ORDERS")
|
||||
members = await catalog.read_entry(session, remote, "MEMBERS")
|
||||
row = await session.get(DataSource, remote)
|
||||
notes = orders.notes
|
||||
assert orders.updated_by == "沈岚"
|
||||
assert notes["label"]["value"] == "订单表" # 冲突：保留目标的
|
||||
assert notes["grain"]["value"] == "一行一单" # 推断只填空缺
|
||||
assert notes["valid_filter"] == {**SOURCE_ORDERS["valid_filter"]} # 照带：来源、状态不变
|
||||
assert notes["dedup"]["status"] == "rejected" and notes["dedup"]["source"] == "llm"
|
||||
codes = notes["columns"]["STATUS"]["codes"]
|
||||
assert codes["value"] == {"1": "已支付", "9": "作废"} and codes["complete"] is True
|
||||
assert notes["columns"]["STATUS"]["label"]["value"] == "状态码"
|
||||
assert notes["columns"]["AMOUNT"]["measure"] == SOURCE_ORDERS["columns"]["amount"]["measure"]
|
||||
# 遮罩列：中文名照带，码值不导入
|
||||
assert notes["columns"]["CONTACT_PHONE"] == {"label": SOURCE_ORDERS["columns"]["contact_phone"]["label"]}
|
||||
assert "GIFT_FLAG" not in notes["columns"] and "gift_flag" not in notes["columns"]
|
||||
[rel] = notes["relations"]
|
||||
assert rel["id"] == catalog.relation_id("ORDERS", ["MEMBER_ID"], "MEMBERS", ["ID"])
|
||||
assert (rel["columns"], rel["to_table"], rel["to_columns"]) == (["MEMBER_ID"], "MEMBERS", ["ID"])
|
||||
assert rel["status"] == "confirmed" and rel["source"] == "name"
|
||||
assert catalog.validate_notes(notes) == []
|
||||
assert members.notes["label"]["status"] == "proposed" and "keys" not in members.notes
|
||||
assert members.notes["columns"]["NICK"]["label"]["status"] == "verified"
|
||||
assert catalog_jobs.guidance_of(row) == "金额一律按元"
|
||||
# 再导一次：都对上了，什么都不写；目标已经有起草要求，不覆盖
|
||||
again = (await client.post(f"{_base(remote)}/import", json={"content": content, "include_guidance": True})).json()
|
||||
assert again["totals"]["add"] == again["totals"]["update"] == 0
|
||||
assert _by(again)["ORDERS"]["version"] == 2
|
||||
assert again["guidance"] == {"file": "金额一律按元", "target": "金额一律按元", "action": "keep"}
|
||||
|
||||
|
||||
async def test_target_rejection_beats_file_confirmation_and_guidance_is_optional(client):
|
||||
local, remote = await _pair()
|
||||
await _write(remote, "MEMBERS", {"label": make_item("会员", "llm", "rejected")})
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(DataSource, remote)
|
||||
await catalog_jobs.save_guidance(session, row, "部署实例自己的要求")
|
||||
content = await _export(client, local)
|
||||
content["tables"]["members"]["notes"]["label"] = make_item("会员", "human", "confirmed")
|
||||
report = (await client.post(f"{_base(remote)}/import",
|
||||
json={"content": content, "include_guidance": False})).json()
|
||||
members = _by(report)["MEMBERS"]
|
||||
[conflict] = members["conflicts"]
|
||||
assert conflict["path"] == "label" and conflict["target_status"] == "rejected"
|
||||
assert conflict["file_status"] == "confirmed" and "驳回" in conflict["why"]
|
||||
assert report["guidance"] == {"file": "金额一律按元", "target": "部署实例自己的要求", "action": "none"}
|
||||
async with SessionLocal() as session:
|
||||
entry = await catalog.read_entry(session, remote, "MEMBERS")
|
||||
assert entry.notes["label"]["status"] == "rejected"
|
||||
assert catalog_jobs.guidance_of(await session.get(DataSource, remote)) == "部署实例自己的要求"
|
||||
|
||||
|
||||
async def test_one_table_failing_does_not_stop_the_others(client, monkeypatch):
|
||||
local, remote = await _pair()
|
||||
content = await _export(client, local)
|
||||
real = catalog.write_entry
|
||||
|
||||
async def flaky(session, source_id, table, notes, **kw):
|
||||
if table == "ORDERS":
|
||||
raise catalog.CatalogConflict(table, 99)
|
||||
return await real(session, source_id, table, notes, **kw)
|
||||
|
||||
monkeypatch.setattr(catalog, "write_entry", flaky)
|
||||
report = (await client.post(f"{_base(remote)}/import", json={"content": content})).json()
|
||||
tables = _by(report)
|
||||
assert tables["ORDERS"]["error"] and "稍后" in tables["ORDERS"]["error"]
|
||||
assert tables["ORDERS"]["add"] == tables["ORDERS"]["update"] == 0
|
||||
assert tables["MEMBERS"]["error"] is None and tables["MEMBERS"]["version"] == 1
|
||||
|
||||
|
||||
async def test_import_waits_for_draft_jobs_and_rejects_bad_files(client):
|
||||
local, remote = await _pair()
|
||||
content = await _export(client, local)
|
||||
async with SessionLocal() as session:
|
||||
session.add(CatalogDraftJob(id=uuid.uuid4().hex, source_id=remote, status="running"))
|
||||
await session.commit()
|
||||
busy = await client.post(f"{_base(remote)}/import", json={"content": content})
|
||||
assert busy.status_code == 409 and busy.json()["code"] == "catalog_draft_busy"
|
||||
# 预览是只读的，照常给
|
||||
assert (await client.post(f"{_base(remote)}/import/preview", json={"content": content})).status_code == 200
|
||||
bad = [
|
||||
"不是对象",
|
||||
{**content, "format": "something-else"},
|
||||
{**content, "version": 2},
|
||||
{**content, "tables": []},
|
||||
{**content, "tables": {"orders": {"notes": "x"}}},
|
||||
{**content, "guidance": 3},
|
||||
]
|
||||
for body in bad:
|
||||
r = await client.post(f"{_base(local)}/import/preview", json={"content": body})
|
||||
assert r.status_code == 422, body
|
||||
assert any("一" <= ch <= "鿿" for ch in r.json()["detail"])
|
||||
# 界面直接交来文件原文也认；不是 JSON 的 422
|
||||
as_text = await client.post(f"{_base(local)}/import/preview", json={"content": json.dumps(content)})
|
||||
assert as_text.status_code == 200 and as_text.json()["totals"]["tables"] == 2
|
||||
not_json = await client.post(f"{_base(local)}/import/preview", json={"content": "{不是 JSON"})
|
||||
assert not_json.status_code == 422 and "JSON" in not_json.json()["detail"]
|
||||
too_long = {**content, "guidance": "长" * (catalog_jobs.GUIDANCE_MAX + 1)}
|
||||
r = await client.post(f"{_base(local)}/import/preview", json={"content": too_long, "include_guidance": True})
|
||||
assert r.status_code == 422 and "起草要求" in r.json()["detail"]
|
||||
|
||||
|
||||
async def test_invalid_items_are_skipped_not_fatal(client):
|
||||
local, remote = await _pair()
|
||||
content = await _export(client, local)
|
||||
notes = content["tables"]["orders"]["notes"]
|
||||
notes["kind"] = {"value": "event", "source": "human", "status": "confirmed"} # 取值不在可选值内
|
||||
notes["formula"] = {"value": "sum(amount)", "source": "human", "status": "confirmed"} # 不认识的字段
|
||||
notes["columns"]["amount"]["colour"] = {"value": "红", "source": "human", "status": "confirmed"}
|
||||
report = (await client.post(f"{_base(remote)}/import/preview", json={"content": content})).json()
|
||||
skipped = {s["path"]: s["why"] for s in _by(report)["ORDERS"]["skipped"]}
|
||||
assert "明细表" in skipped["kind"]
|
||||
assert "formula" in skipped["formula"] and "columns.amount.colour" in skipped
|
||||
|
||||
|
||||
async def test_valid_filter_columns_must_be_in_the_target_table(client):
|
||||
"""有效记录条件是表级的一项，按列跳过的规矩拦不住它：解析出用到的列，不在目标的表结构里、是别的表的列、带子查询的都跳过
|
||||
并写原因。人工写的条件不限函数；带本表表名（文件里的写法、目标的写法都认）、大小写不同的照带。"""
|
||||
local, remote = await _pair()
|
||||
content = await _export(client, local)
|
||||
|
||||
async def outcome(condition: str) -> tuple[str | None, int]:
|
||||
body = copy.deepcopy(content)
|
||||
body["tables"]["orders"]["notes"]["valid_filter"] = make_item(condition, "human", "confirmed")
|
||||
report = (await client.post(f"{_base(remote)}/import/preview", json={"content": body})).json()
|
||||
orders = _by(report)["ORDERS"]
|
||||
why = {s["path"]: s["why"] for s in orders["skipped"]}.get("valid_filter")
|
||||
return why, orders["update"]
|
||||
|
||||
why, update = await outcome("gift_flag = 0 AND status <> 9")
|
||||
assert why and "gift_flag" in why and "目标的表结构" in why and update == 0
|
||||
why, _ = await outcome("members.tier = 1")
|
||||
assert why and "members.tier" in why and "别的表" in why
|
||||
why, _ = await outcome("member_id IN (SELECT id FROM members)")
|
||||
assert why and "子查询" in why
|
||||
why, _ = await outcome("status <> 9; DELETE FROM orders")
|
||||
assert why and "条件" in why
|
||||
for fine in ("orders.status <> 9", "ORDERS.STATUS <> 9 AND ABS(amount) > 0", "contact_phone IS NOT NULL"):
|
||||
why, update = await outcome(fine)
|
||||
assert why is None and update == 1, fine
|
||||
|
||||
|
||||
async def test_import_needs_a_schema(client):
|
||||
local, _ = await _pair()
|
||||
content = await _export(client, local)
|
||||
empty = await _source(f"empty_{uuid.uuid4().hex[:6]}", {}, {})
|
||||
r = await client.post(f"{_base(empty)}/import/preview", json={"content": content})
|
||||
assert r.status_code == 409 and r.json()["code"] == "schema_missing"
|
||||
|
||||
|
||||
async def test_round_trip_into_an_empty_copy_reproduces_the_catalog(client):
|
||||
local, _ = await _pair()
|
||||
content = await _export(client, local)
|
||||
copy_sid = await _source(f"copy_{uuid.uuid4().hex[:6]}", _tables(), {})
|
||||
report = (await client.post(f"{_base(copy_sid)}/import", json={"content": json.loads(json.dumps(content)),
|
||||
"include_guidance": True})).json()
|
||||
assert report["unmatched_tables"] == ["ghost_table"]
|
||||
assert all(not t["conflicts"] for t in report["tables"])
|
||||
async with SessionLocal() as session:
|
||||
mine = await catalog.read_entry(session, local, "orders")
|
||||
theirs = await catalog.read_entry(session, copy_sid, "orders")
|
||||
# 关系两端都对得上的照带；指向目标那边没有的表的跳过
|
||||
expected = copy.deepcopy(mine.notes)
|
||||
expected["relations"] = [r for r in mine.notes["relations"] if r["to_table"] != "stores"]
|
||||
del expected["columns"]["gift_flag"] # 表结构里没有这一列
|
||||
assert theirs.notes == expected
|
||||
Reference in New Issue
Block a user