diff --git a/backend/app/api/catalog.py b/backend/app/api/catalog.py index 0405a0f7..c8c68126 100644 --- a/backend/app/api/catalog.py +++ b/backend/app/api/catalog.py @@ -8,7 +8,8 @@ - POST /catalog/{table}/patch 保存目录修改提案（乐观锁），记为人工确认 - GET /catalog/{table}/impact 影响面：引用这张表的已发布、受管模板（data/catalog_impact.py） - POST /catalog/draft 同步起草（注释、外键、命名推断，可选模型）。界面改用下面的后台起草任务，这个留着； - 同一个源有排队或进行中的后台任务时 409 catalog_draft_busy + 和后台任务双向互斥：同一个源有排队或进行中的后台任务（或正在同步起草）时 409 + catalog_draft_busy，同步起草期间建任务、重试、导入也是 409 - PUT /catalog/guidance 数据源级的起草要求（使用者补充的背景和要求，起草时交给模型） - POST /catalog/draft-jobs 建后台起草任务（202）；GET /draft-jobs/active、/latest、/{job_id} 读任务； POST /draft-jobs/{job_id}/cancel 停止，/retry 只重试没写好的表（data/catalog_jobs.py） @@ -419,20 +420,21 @@ async def draft_catalog(source_id: str, payload: CatalogDraftIn, model_items：模型给了这张表几项可用内容（没用模型、模型这部分失败时为 null）。界面据此区分「模型给了内容、 只是和现有目录一致」（model_items > 0、三个数都是 0）和「模型没给内容」（model_error 有值）。 - 同一个源有排队或进行中的后台起草任务时 409 catalog_draft_busy（和建任务同一句话）：两边同时写同一个源的目录、 - 抢模型名额。 + 双向互斥（catalog_jobs.sync_draft，进程内登记）：同一个源有排队或进行中的后台起草任务、或者正在同步起草时 409 + catalog_draft_busy（和建任务同一句话）；反过来，同步起草进行期间同一个源建任务、重试、导入目录也是 409。两边同时写 + 同一个源的目录、抢模型名额。 """ row, source = await _resolved(session, source_id) + cache = source.schema_cache or {} try: - await catalog_jobs.ensure_idle(session, row) + async with catalog_jobs.sync_draft(session, row): + if not cache.get("tables"): + raise _no_schema(cache) + report = await catalog.draft_catalog(session, source, tables=payload.tables or None, + use_model=payload.use_model, actor=actor_of(x_actor), + guidance=catalog_jobs.guidance_of(row)) except catalog_jobs.DraftBusy as e: raise CodedHTTPException(409, str(e), CATALOG_DRAFT_BUSY) from e - cache = source.schema_cache or {} - if not cache.get("tables"): - raise _no_schema(cache) - report = await catalog.draft_catalog(session, source, tables=payload.tables or None, - use_model=payload.use_model, actor=actor_of(x_actor), - guidance=catalog_jobs.guidance_of(row)) rows = [{"table_name": r.table, "added": r.added, "updated": r.updated, "removed": r.removed, "version": r.version, "error": r.error, "model_error": r.model_error, "model_items": r.model_items} for r in report.tables] diff --git a/backend/app/data/catalog.py b/backend/app/data/catalog.py index 3dd0bb39..a16446ab 100644 --- a/backend/app/data/catalog.py +++ b/backend/app/data/catalog.py @@ -1488,9 +1488,13 @@ MODEL_FALLBACK_MAX_TOKENS = 8192 #: 一段没写出来（额度用完、只给空壳）时拆成两半再问：最多拆几层、拆到最少几列为止 MODEL_SPLIT_DEPTH = 3 MODEL_SPLIT_MIN_COLUMNS = 5 -#: 止损：连续这么多段拆到底仍一项没写出来，这次起草后面的段不再拆小重试（模型明显用不了时不该一段一段拆到底烧额度； -#: 每段另有 MODEL_PART_MAX_CALLS 次封顶） -MODEL_GIVE_UP_AFTER = 2 +#: 止损（按比例）：这次起草最近 MODEL_HALT_WINDOW 次调用里有 MODEL_HALT_ON 次以上没写出来（额度用完、超时、空壳、出错）， +#: 之后的段不再拆小、每块只问一次，直到比例降下去。模型明显用不了时不该一段一段拆到底烧额度；以前按「连续几段」算， +#: 偶尔一次写出来就清零，会思考的模型在宽表上 90 分钟停不下来。每段另有 MODEL_PART_MAX_CALLS 次封顶 +MODEL_HALT_WINDOW = 10 +MODEL_HALT_ON = 7 +#: 同一张表连续这么多个原始段都一项没写出来，这张表剩下的段跳过不问（可以换一个起草模型后只重试这张表） +MODEL_TABLE_GIVE_UP_AFTER = 3 #: 网络错误、429、5xx 最多重试几次，每次重试前等多久（秒，依次取用）。超时不原样重试，当作这一段太大、拆小再问 MODEL_RETRIES = 2 MODEL_RETRY_DELAYS_S = (10.0, 30.0) @@ -1512,6 +1516,10 @@ CATALOG_DRAFT_SYSTEM = """你在为一个业务数据库编写数据目录：说 log（日志表）、config（配置表）之一。 - keys：业务主键，即在业务上唯一确定一行的列（可以和数据库主键不同），写列名。 - business_date：业务日期按哪一列算，rule 写一句规则，timezone 写时区（不确定就留空）。 +- valid_filter：有效行过滤。condition 写一段 SQL 条件，筛出业务上算数的行（例如 STATUS NOT IN ('C', 'X')，去掉作废、\ +测试的行）：只用这张表自己的列、不带表名，只写比较、IN、BETWEEN、LIKE、IS NULL，用 AND、OR、NOT 连接，函数只用 UPPER、\ +LOWER、TRIM、COALESCE，不写子查询、聚合、窗口函数和分号；reason 写一句依据（例如哪条注释、哪些码值说明了这一点）。 +- dedup：去重规则，rule 写一句话（例如「同一订单号有多行时只取最后修改的那一行」），reason 写一句依据。 - columns：每一列的 label（中文名）、meaning（含义，一句话）、unit（单位，例如「元」「人」，没有单位就留空）、\ measure（度量类型：flow 流量，可跨期加总；stock 存量，时点值，不能跨期加总；ratio 比率；identifier 标识；\ status 状态；attribute 属性）。 @@ -1519,6 +1527,8 @@ status 状态；attribute 属性）。 规矩： - 只写数据事实，不写计算公式，不写指标怎么算。 - 拿不准的字段留空，不要猜。列名、表名照原样写，不要翻译或改写。 +- 有效行过滤、去重规则只在表结构、注释、码值能看出来时才写（例如状态列的注释写明了哪些取值是作废、测试），拿不准就不写，\ +没有就不写：写错了会把算数的行筛掉、把不该合并的行合并。 - 请你写的每一列都在 columns 里写一项，拿不准的字段可以留空，但不要漏掉列：漏掉的列会被再问一次。 - 标着「系统说明」的内容是导入时按核对结果生成的，以它为准：不要重复，也不要与之矛盾。 - 标着「已确认」的目录项由人工确认过，以它为准。 @@ -1537,6 +1547,7 @@ CATALOG_DRAFT_BACKGROUND_PROMPT = "使用者补充的背景和要求：\n{lines} CATALOG_DRAFT_TEXT_PROMPT = """只输出一个 JSON 对象，不要输出别的文字，格式如下（拿不准的字段不写）： {"tables": [{"table": "表名", "label": "中文名", "grain": "一行代表什么", "kind": "表类型", "keys": ["列名"], \ "business_date": {"column": "列名", "rule": "一句规则", "timezone": "时区"}, \ +"valid_filter": {"condition": "SQL 条件", "reason": "一句依据"}, "dedup": {"rule": "一句话", "reason": "一句依据"}, \ "columns": [{"name": "列名", "label": "中文名", "meaning": "含义", "unit": "单位", "measure": "度量类型"}]}]} 表类型写 fact、dimension、snapshot、log、config 之一；度量类型写 flow、stock、ratio、identifier、status、attribute 之一。""" @@ -1556,11 +1567,14 @@ CATALOG_DRAFT_PROMPT_LABELS = { "unique": "唯一约束：{cols}", "confirmed": "已确认的目录项：", # 分段起草：表名下一行说这次只写哪部分（宽表的表级一段、列分段，拆小重问时的半段） - "only_table": "这次只写表级的项（中文名、粒度、表类型、业务主键、业务日期），不写列", + "only_table": "这次只写表级的项（中文名、粒度、表类型、业务主键、业务日期、有效行过滤、去重规则），不写列", "only_columns": "这次只写下面列出的 {n} 列（这张表共 {total} 列），不写表级的项", "table_and_columns": "这次只写表级的项和这 {n} 列：{cols}", "brief_columns": "列（共 {n} 列，这里只列名字和类型）：", "drafted": "已起草的表级内容：", + # 开了数据剖析的源：目录里记下的码值（剖析看到的取值和已知含义）交给模型，判断有效行过滤用；遮罩的列不给 + "codes": "目录里记下的码值（数据剖析看到的取值和已知含义，可据此判断有效行过滤）：", + "code": "- {name}：{codes}", # 使用者补充的背景和要求 "description": "数据源说明：{text}", "guidance": "起草要求：{text}", @@ -1588,6 +1602,20 @@ CATALOG_DRAFT_PROMPT_SCHEMA: dict[str, Any] = { "timezone": {"type": "string"}}, "required": ["column"], }, + "valid_filter": { + "type": "object", + "description": "有效行过滤：只在表结构、注释、码值能看出来时才写", + "properties": {"condition": {"type": "string", "description": "SQL 条件，只用这张表的列"}, + "reason": {"type": "string", "description": "一句依据"}}, + "required": ["condition"], + }, + "dedup": { + "type": "object", + "description": "去重规则：只在表结构、注释能看出来时才写", + "properties": {"rule": {"type": "string", "description": "一句话"}, + "reason": {"type": "string", "description": "一句依据"}}, + "required": ["rule"], + }, "columns": { "type": "array", "items": { @@ -1708,11 +1736,77 @@ def _clean_text(value: Any, limit: int = _TEXT_MAX) -> str | None: return text if text and len(text) <= limit else None -def _from_model(entry: Mapping[str, Any], meta: Mapping[str, Any], *, system: bool) -> dict[str, Any]: +#: 模型给的依据（有效行过滤、去重规则的 reason）写进这一项的 note +FACT_REASON_NOTE = "依据：{reason}" +#: 有效行过滤按数据源的写法整理（列名加了引号）以后超过目录里这一项的长度上限 +FACT_FILTER_TOO_LONG = "{label}按数据源的写法整理后超过 {limit} 字" + + +def _model_fact(raw: Any, key: str) -> tuple[str, str | None] | None: + """模型给的有效行过滤、去重规则 → (取值, 依据)。约定是对象（{condition 或 rule, reason}），只写了一段文字的也认； + 取值不是非空文字的不要。""" + if isinstance(raw, str): + text, reason = raw, None + elif isinstance(raw, Mapping): + text, reason = raw.get(key), raw.get("reason") + if text is None: + text = raw.get("value") + else: + return None + if not isinstance(text, str) or not text.strip(): + return None + return text.strip(), _clean_text(reason) + + +def _fact_item(value: str, reason: str | None) -> dict[str, Any]: + return make_item(value, "llm", note=FACT_REASON_NOTE.format(reason=reason) if reason else None) + + +def _model_facts(entry: Mapping[str, Any], meta: Mapping[str, Any], *, kind: str | None, table: str | None, + dropped: list[tuple[str, str]] | None = None) -> dict[str, Any]: + """模型提议的有效行过滤、去重规则（llm / proposed，依据写进 note）。 + + 有效行过滤按数据源的方言解析成一个条件、只能用这张表的列（sqlcheck.parse_valid_filter），不过就丢掉、不算这张表 + 失败：服务端日志记一条，dropped 给了就记下 (原因, 模型写的条件)，起草引擎把它写进这张表的日志（起草任务逐表的 + log）。存的是 sqlcheck.stored_valid_filter 整理过的写法：原文照旧，只把列换成表结构里的真名、不带表名，按数据源的 + 需要加引号——剖析核对的、SQL 检查让模型照着加的都是这一条。去重规则是一句话，同文字项的规矩。""" + from app.data import sqlcheck + + out: dict[str, Any] = {} + found = _model_fact(entry.get("valid_filter"), "condition") + if found is not None: + text, reason = found + parsed = sqlcheck.parse_valid_filter(text, meta, kind=kind, table=table) + problem = parsed.problem + if problem is None: + stored = sqlcheck.stored_valid_filter(parsed, kind) + if _text_ok(stored): + out["valid_filter"] = _fact_item(stored, reason) + else: + # 加了引号以后超过上限：写进去会让整张表写入失败（validate_notes），不收 + problem = FACT_FILTER_TOO_LONG.format(label=TABLE_FIELD_LABEL["valid_filter"], limit=_TEXT_MAX) + if problem is not None: + import logging + + logging.getLogger(__name__).info("起草数据目录：表 %s 的有效行过滤没有通过校验，已丢掉：%s（%s）", + table or meta.get("qualified") or entry.get("table"), problem, text[:200]) + if dropped is not None: + dropped.append((problem, text)) + found = _model_fact(entry.get("dedup"), "rule") + if found is not None and (text := _clean_text(found[0])) is not None: + out["dedup"] = _fact_item(text, found[1]) + return out + + +def _from_model(entry: Mapping[str, Any], meta: Mapping[str, Any], *, system: bool, source_kind: str | None = None, + table: str | None = None, dropped: list[tuple[str, str]] | None = None) -> dict[str, Any]: """模型给的一张表 → 目录项（llm / proposed）。不合规的值整项丢掉，不硬塞：列名对不上表结构的、 表类型和度量类型不在取值里的、业务主键里有不存在的列的、看起来是计算公式的（_formula_like，公式进口径卡）。 该是文字的地方写成了数组、对象（列名、表类型、业务日期的列、度量类型……）同样丢掉，不抛异常：实测见过把列名 - 写成数组的，以前在这里抛 TypeError，整张表记出错。导入表格的源，说明已经覆盖的字段也丢掉。""" + 写成数组的，以前在这里抛 TypeError，整张表记出错。导入表格的源，说明已经覆盖的字段也丢掉。 + + 有效行过滤、去重规则见 _model_facts：source_kind 是数据源类型（按它的方言解析条件，不认识的类型不收有效行过滤）， + table 是表名（条件里的列可以带这张表的表名），dropped 收没通过校验的有效行过滤 (原因, 模型写的条件)。""" columns = {c["name"]: c for c in meta.get("columns") or []} notes: dict[str, Any] = {} table_has_system_note = system and bool((meta.get("comment") or "").strip()) @@ -1734,6 +1828,7 @@ def _from_model(entry: Mapping[str, Any], meta: Mapping[str, Any], *, system: bo if (text := _clean_text(date.get(k))) is not None: value[k] = text notes["business_date"] = make_item(value, "llm") + notes.update(_model_facts(entry, meta, kind=source_kind, table=table, dropped=dropped)) out_cols: dict[str, dict[str, Any]] = {} listed = entry.get("columns") for col in listed if isinstance(listed, list) else []: @@ -1842,8 +1937,10 @@ async def write_draft(session: AsyncSession, source: Any, table: str, modeled: A 没用模型时为 None）有可用内容的再并上。同步起草和后台起草任务（每张表一结束就调）共用这一处。 - 并入已有目录按 merge_notes 的规则：人工确认、驳回过的项不动。内容没变不写、不升版本。 - - 模型这次没给的项（留空、没通过校验）不删以前起草的；列已经不在表结构里的，模型各段都写出来时（covered 含 llm） - 照旧删掉。模型只写出一部分的不算完整起草过（covered 不含 llm）。 + - 模型这次没给的项（留空、没通过校验）不删以前起草的；列已经不在表结构里的（业务主键、业务日期指着的列也算）， + 整张表起草完（各段都写出来的 ok、只写出一部分的 partial 都一样）就按完整覆盖的规则删掉：删的只是说已经不存在的 + 列的模型起草项，和这次写没写全无关。后台任务里宽表每写出一段就先并入一次（catalog_jobs.write_part，那一次不删 + 任何东西），这里是整张表的收尾。 - 模型留空的列（blank）按并入之后的目录再筛一遍：已经有中文名或含义的（以前起草的、库注释、起草进行中有人填的） 不算留空。 - 写入撞上别人刚改过（乐观锁）就重读重并，最多三次。 @@ -1859,8 +1956,8 @@ async def write_draft(session: AsyncSession, source: Any, table: str, modeled: A if modeled.items: result.model_items = modeled.items result.blank = list(getattr(modeled, "blank", None) or []) - model_draft = TableDraft(notes=modeled.notes, covered=frozenset({"llm"}) if modeled.status == "ok" - else frozenset()) + model_draft = TableDraft(notes=modeled.notes, covered=frozenset({"llm"}) + if modeled.status in ("ok", "partial") else frozenset()) for _attempt in range(3): entry = await read_entry(session, source.id, table) version = entry.version if entry else 0 @@ -2324,9 +2421,10 @@ __all__ = [ "CatalogEntry", "CatalogInvalid", "CatalogModelUnavailable", "CatalogPathError", "DRAFT_DEFAULT_TABLES", "DraftModel", "DraftReport", "INFERRED_MARK", "ITEM_SOURCES", "ITEM_STATUSES", "JoinEdge", "KIND_LABEL", "MEASURE_LABEL", "MODEL_BATCH_COLUMNS", "MODEL_BATCH_TABLES", "MODEL_CONCURRENCY", "MODEL_EMPTY_REASON", - "MODEL_FALLBACK_MAX_TOKENS", "MODEL_MAX_TOKENS", "MODEL_MIN_PART_COLUMNS", "MODEL_NOTICE_OFF", "MODEL_NOTICE_ON", - "MODEL_NOTICE_WINDOW", "MODEL_PART_MAX_CALLS", "MODEL_RETRIES", "MODEL_RETRY_DELAYS_S", "MODEL_SHRINK_AFTER", - "MODEL_SPLIT_DEPTH", "MODEL_SPLIT_MIN_COLUMNS", "MODEL_TIMEOUT_S", "MergeStats", "NEW_RELATION_PATH", + "FACT_REASON_NOTE", "MODEL_FALLBACK_MAX_TOKENS", "MODEL_HALT_ON", "MODEL_HALT_WINDOW", "MODEL_MAX_TOKENS", + "MODEL_MIN_PART_COLUMNS", "MODEL_NOTICE_OFF", "MODEL_NOTICE_ON", "MODEL_NOTICE_WINDOW", "MODEL_PART_MAX_CALLS", + "MODEL_RETRIES", "MODEL_RETRY_DELAYS_S", "MODEL_SHRINK_AFTER", "MODEL_SPLIT_DEPTH", "MODEL_SPLIT_MIN_COLUMNS", + "MODEL_TABLE_GIVE_UP_AFTER", "MODEL_TIMEOUT_S", "MergeStats", "NEW_RELATION_PATH", "PATCH_MAX_CHANGES", "PROFILE_NOTE_PREFIX", "PROFILE_VERIFY_COVERAGE", "WRITE_CONFLICT_ERROR", "default_tables", "missing_table_error", "pick_tables", "write_draft", diff --git a/backend/app/data/catalog_draft.py b/backend/app/data/catalog_draft.py index 4183bff1..d972c7fb 100644 --- a/backend/app/data/catalog_draft.py +++ b/backend/app/data/catalog_draft.py @@ -42,19 +42,25 @@ 地方写成了数组、对象的值在校验（catalog._from_model）里丢掉，不算出错。 - 封顶：每个原始段（计划里的整表段、表级那段或一个列分段，连同按缩小后的上限分开的各块、拆出来的、补问的、重试的） 一共最多调用 MODEL_PART_MAX_CALLS 次，到了次数还没写完的记没写出来。 -- 止损：连续 MODEL_GIVE_UP_AFTER 段拆到底仍一项没写出来（模型明显用不了，例如思考总是吃满额度），这次起草后面的 - 段不再拆小重试、没写出来就直接记下，报错建议更换起草模型；有一段写出来了就恢复。超时另有一条：拆到每段 - MODEL_MIN_PART_COLUMNS 列的调用接连超时 MODEL_GIVE_UP_AFTER 次（超时和段大小无关，再拆小也是一次一个时限），后面 - 超时的不再拆小，超时拆出来、还没问的块也不再问；有一次调用写出来了就恢复。不然一直超时的模型首轮同时起草的几段 - 要各自拆满次数，一段 40 列串着问约 7 个时限（420 秒时约 49 分钟）。 +- 止损按比例：这次起草最近 MODEL_HALT_WINDOW 次调用里有 MODEL_HALT_ON 次以上没写出来（额度用完、超时、空壳、出错）， + 之后的段不再拆小、每块只问一次（不拆小、不补问漏掉的列、截断后不再补剩下的、空壳不再换纯文本原样再问；拆出来还没问 + 的块也不再问），报错建议更换起草模型，直到比例降下去。以前按「连续几段拆到底」「接连超时」算，偶尔一次写出来就清零： + 实测会思考的模型在宽表上单次 1 到 7 分钟、四分之一超时，90 分钟停不下来。几处只管通信的调整照旧：服务方嫌额度 + 太大退回小额度、结构化输出不支持或回了说明文字改走纯文本、网络错误退避重试；结构化输出只给空壳的，这次起草还没换 + 纯文本问过的照旧问一次（看看是不是结构化输出的问题）。止损的窗口只记每次调用的最终结局：上面这几种重来之前的那次 + 不花额度、也说明不了模型写不出来，照旧报给进度回调（用量、提醒照算），不进止损的窗口——以前都记成没写出来，额度上限 + 8192、不支持 json_schema 的 OpenAI 兼容网关上，并发的头几次调用全是这种重来，第一次额度用完就凑满 10 次里 7 次， + 那一段直接记没写出来、不再拆小。重试用完仍出错的那一次是最终结局，照常算。 +- 同一张表连续 MODEL_TABLE_GIVE_UP_AFTER 个原始段都一项没写出来：这张表剩下还没开始的段跳过不问（宽表一段一段问下去 + 只是烧额度），原因写清跳过了几段、可以换一个起草模型后只重试这张表。中间有一段写出来了就不算「接连」。 - 提醒：最近 MODEL_NOTICE_WINDOW 次调用里有 MODEL_NOTICE_ON 次以上没写出来（额度用完、超时、空壳、出错），给一句 提醒（次数和原因，建议换起草模型、缩小起草范围或停止），经进度回调交给任务；之后少于 MODEL_NOTICE_OFF 次时清掉。 以前看不出调用在反复失败：一个任务 55 分钟只结束 1 张表，界面上只有段进度和用量。 **原因。** 没写出来的原因只写进这张表的 model_error：几段因为同一个原因没写出来时说「这几段」，分段起草的表（只写 出一部分的、一项都没写出来的都一样）每段写上列的范围（「第 5 段（列名 到 列名，共 33 列）」，列名照表结构原样）， -真拆小重试过的才补「已拆小重试仍没写出」。只有一段的表，这一段就是整张表：只写出一部分的写清是哪几列，一项都没写 -出来的只写原因。 +真拆小重试过的才补「已拆小重试仍没写出」，而且只跟在拆过的那几段后面说（说到的段只有一部分拆过时写「其中第 3 段」）。 +只有一段的表，这一段就是整张表：只写出一部分的写清是哪几列，一项都没写出来的只写原因。 **留空的列。** 请模型写了、模型也列出了，中文名和含义却都没给（或者给的都没通过校验）的列是模型拿不准，不补问， 记在 ModelOutcome.blank 里（表的状态不因此变）。目录里已经有中文名或含义的列不算（不论来源，驳回的不算有：以前模型 @@ -67,11 +73,22 @@ 的不会等前一个排着的段全部调完。一次起草里按表的先后排队：宽表的表级那段写完，它的列分段排到已经排着的段后面 （先来先服务），不插到窄表前面。 -**进度。** 调用方给 Hooks：开始调用、每次调用的结果（用量、没写出来的原因、当前的提醒和列数上限）、一段结束、一张表 -的模型部分全部结束。后台起草任务据此落库，每张表一结束就并入目录（data/catalog_jobs.py）。 +**进度。** 调用方给 Hooks：开始调用、每次调用的结果（用量、没写出来的原因、当前的提醒和列数上限）、一段结束、宽表的 +一段写出来了（交出这张表到这时拿到的全部内容，调用方马上并入目录：停止、服务重启时已经写出来的段不丢）、块进度变了、 +一张表的模型部分全部结束。后台起草任务据此落库（data/catalog_jobs.py）。块：这张表实际分成几块来问——一开始是段数， +一段被缩小的上限分开成几块、被拆成两半，块数就多几块；一块问完（写没写出来都算）算结束一块。 -交给模型的只有表结构（列、类型、主键、外键、唯一约束、注释）和使用者补充的背景，不发任何数据值。写给模型的文字都在 -catalog.py 名字带 _SYSTEM / _PROMPT 的常量里；这里的 REASON_* 是给人看的（拼进这张表的 model_error）。 +**防错事实。** 表级的项里模型可以提议有效行过滤（valid_filter）和去重规则（dedup），各附一句依据（写进这一项的 note）， +校验见 catalog._from_model（有效行过滤按方言解析，不过就丢掉；存的是整理过的写法：列换成表结构里的真名、按方言加引号， +剖析核对的也是这一条）。开了数据剖析的源（catalog_profile.profile_settings）：目录里记下的码值交给模型做背景（遮罩的列 +不给，「已确认的目录项」里遮罩列的码值同样不给），表级那段写完后经剖析的只读守卫数一数有效行过滤排除了多少行 +（catalog_profile.check_valid_filter），排除 0% 或 100% 的不收，收下的 note 写上排除比例；没开剖析、连不上的照收， +note 写没有核对的原因。没收下的（没通过校验、排除 0% 或 100%）不算失败，在这张表的日志里记一句（ModelOutcome.log， +起草任务逐表的 log）。 + +交给模型的是表结构（列、类型、主键、外键、唯一约束、注释）和使用者补充的背景；开了数据剖析的源另给目录里记下的码值 +（剖析取过的取值和含义，遮罩的列不给），别的数据值一概不发。写给模型的文字都在 catalog.py 名字带 _SYSTEM / _PROMPT +的常量里；这里的 REASON_* 是给人看的（拼进这张表的 model_error）。 """ from __future__ import annotations @@ -111,21 +128,36 @@ SEG_ONE = "这一段" SEG_MANY = "这几段" REASON_EXHAUSTED = "模型的思考和输出用完了额度（{budget} 个 token），{seg}没写出来" REASON_SPLIT = "；已拆小重试仍没写出" +#: 说到的几段里只有一部分真拆小重试过：只说那几段 +REASON_SPLIT_PARTS = "；其中第 {parts} 段已拆小重试仍没写出" REASON_HOLLOW = "模型只给了空壳，没有写出可用的内容" REASON_TRUNCATED = "模型的输出被截断，{seg}没写完" REASON_FORMAT = "模型的回复不是约定的格式" REASON_MISSING = "模型的回复里没有这张表" REASON_UNANSWERED = "模型的回复里漏了这些列，补问一次仍没有给出" +#: 止损期间每块只问一次：漏了的列不补问 +REASON_OMITTED = "模型的回复里漏了这些列" REASON_TIMEOUT = "模型 {seconds} 秒内没有写完{seg}" REASON_CALL = "调用模型出错：{line}" REASON_RETRIED = "（已重试 {n} 次）" REASON_CRASH = "起草这张表时出错：{line}" #: 一段到了调用次数的上限（MODEL_PART_MAX_CALLS）还没写完 REASON_CAPPED = "每段最多调用 {n} 次模型，{seg}到了次数仍没写完，已停止重试" -#: 止损（连续几段拆到底仍一项没写出来，后面的段不再拆小重试）时补在原因后面 -REASON_HALTED = "；模型连续 {n} 段都没写出来，已停止拆小重试，建议更换起草模型" -#: 拆到每段 MODEL_MIN_PART_COLUMNS 列仍接连超时（超时和段大小无关，后面超时的不再拆小）时补在原因后面 -REASON_SLOW = "；拆到每段 {n} 列仍然超时，已停止拆小重试，建议更换起草模型" +#: 止损（最近几次调用多数没写出来，之后的段不再拆小、每块只问一次）时补在原因后面 +REASON_HALTED = "；最近 {window} 次调用有 {on} 次以上没写出来，已停止拆小重试，建议更换起草模型" +#: 同一张表接连几段都没写出来，剩下的段跳过不问 +REASON_ABANDONED = "这张表接连 {k} 段都没写出来，剩下的 {n} 段已跳过，可以换一个起草模型后只重试这张表" +#: 跳过不问的段在「没写出来」记录里的原因（不拼进原因，见 _Run._outcome） +_SKIPPED = "\0skipped" + +# ---- 这张表起草时记下的几句话（不算失败），经 ModelOutcome.log 交给起草任务逐表的 log（界面看得到） +#: 模型提议的有效记录条件没有收下：why 是原因（没通过校验的说法，或者剖析核对的结果） +LOG_FILTER_DROPPED = "{why}，模型提议的这一条没有收下：{condition}" +#: 剖析核对排除了 0% 或 100% 的行 +LOG_FILTER_COUNTED = "{label}按剖析核对{what}（共 {rows} 行）" +#: 一张表最多记几句；日志里的条件最多写几个字 +LOG_MAX = 10 +_LOG_CONDITION_MAX = 200 # ---- 提醒（最近几次调用多数没写出来），经进度回调交给任务（CatalogDraftJob.notice） NOTICE = ("最近 {calls} 次调用有 {failed} 次没写出来（{kinds}）。建议在{place}换一个模型、缩小起草范围，" @@ -154,8 +186,10 @@ class ModelOutcome: status：ok 各段都写出来了；partial 有的段写出来、有的没有（error 写清哪部分、为什么）；failed 一项都没写出来 （error 写原因）。notes 是写出来的那些（已逐项校验，llm / proposed）；items 是项数。parts_total 分几段起草 - （整表段是 1，宽表是 1 + 列分段数），parts_done 已经结束了几段。blank：模型拿不准、留空了的列（按表结构的顺序； - 表的状态不因此变）。""" + （整表段是 1，宽表是 1 + 列分段数），parts_done 已经结束了几段，parts_written 其中写出了内容的段数。blank：模型拿 + 不准、留空了的列（按表结构的顺序；表的状态不因此变）。chunks_total、chunks_done：这张表实际分成的块数和已结束的 + 块数（段被缩小的上限分开、拆成两半后会变多）。log：起草时记下的几句话，不算失败（模型提议的有效记录条件没有收下的 + 原因），最多 LOG_MAX 句。""" status: str notes: dict[str, Any] @@ -164,6 +198,19 @@ class ModelOutcome: parts_total: int parts_done: int blank: list[str] = field(default_factory=list) + parts_written: int = 0 + chunks_total: int = 0 + chunks_done: int = 0 + log: list[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class PartWrite: + """宽表写出了一段（进度回调 part_written 收到的）：notes 是这张表到这时拿到的全部模型起草（已校验、已核对，深拷贝）， + parts_written 是其中写出了内容的段数。调用方马上并入目录（这一次不删任何东西），整张表结束时另有 table_done 收尾。""" + + notes: dict[str, Any] + parts_written: int @dataclass(frozen=True) @@ -192,12 +239,16 @@ class Hooks: - call_started(表名列表)：一次调用拿到并发名额、马上要发出去了； - call_done(CallReport)：每次调用一回（条数就是调用次数），带着用量、没写出来的原因、当前的提醒和列数上限； - part_done(表名)：这张表的一段结束了（写没写出来都算）； + - part_written(表名, PartWrite)：宽表写出了一段、后面还有段没结束（最后一段由 table_done 收尾；只有一段的表不调）； + - chunks_changed(表名, 块数, 已结束的块数)：块进度变了； - table_done(表名, ModelOutcome)：这张表的模型部分全部结束。""" call_started: Callable[[list[str]], Awaitable[None]] | None = None call_done: Callable[[CallReport], Awaitable[None]] | None = None part_done: Callable[[str], Awaitable[None]] | None = None table_done: Callable[[str, ModelOutcome], Awaitable[None]] | None = None + part_written: Callable[[str, PartWrite], Awaitable[None]] | None = None + chunks_changed: Callable[[str, int, int], Awaitable[None]] | None = None def parts_for(columns: int) -> int: @@ -328,9 +379,9 @@ class _Reply: @dataclass(frozen=True) class _Why: - """一部分没写出来的原因。base 给人看的原因（可以带 SEG，拼成一句时换成「这一段」「这几段」）；split 真拆小重试过； - halted 本来还能拆小、因为止损才没拆时止损的说明（REASON_HALTED：前面连续几段都没写出来；REASON_SLOW：拆到每段 - MODEL_MIN_PART_COLUMNS 列仍接连超时），没有是空串。""" + """一部分没写出来的原因。base 给人看的原因（可以带 SEG，拼成一句时换成「这一段」「这几段」；跳过不问的段是 _SKIPPED， + 不拼进原因）；split 真拆小重试过；halted 本来还能拆小、补问或再问、因为止损才没问时止损的说明（REASON_HALTED：最近 + 几次调用多数没写出来），没有是空串。""" base: str split: bool = False @@ -341,18 +392,23 @@ def _join(failures: Iterable[tuple[_Why, int]]) -> str: """几处没写出来的原因拼成一句。failures 是 (原因, 属于这张表的第几段)。 同样的原因只说一次（最多说两种）；同一个原因落在几段上说「这几段」，同一段拆出来的几块仍是「这一段」；拆小重试过、 - 止损的说明各在最后补一次（只看说出来的那几种；两种止损各说一次）。不然宽表的表级那段拆不开、列分段拆过时，同一句 - 原因会说两遍。""" + 止损的说明各在最后补一次（只看说出来的那几种）。不然宽表的表级那段拆不开、列分段拆过时，同一句原因会说两遍。 + + 「已拆小重试仍没写出」只跟在真拆过的那几段后面说：说到的段都拆过（只有一段的表就是这一段）时不点名，只有一部分拆过 + 时写「其中第 3、4 段已拆小重试仍没写出」。以前一律补在最后，读起来像每一段都拆过。""" failures = list(failures) parts: dict[str, set[int]] = {} for why, part in failures: parts.setdefault(why.base, set()).add(part) shown = list(parts)[:2] - told = [w for w, _ in failures if w.base in shown] + told = [(w, part) for w, part in failures if w.base in shown] text = "；".join(base.replace(SEG, SEG_MANY if len(parts[base]) > 1 else SEG_ONE) for base in shown) - if any(w.split for w in told): - text += REASON_SPLIT - for note in dict.fromkeys(w.halted for w in told if w.halted): + split = sorted({part for w, part in told if w.split}) + if split: + everywhere = {part for _, part in told} + text += REASON_SPLIT if set(split) == everywhere else \ + REASON_SPLIT_PARTS.format(parts="、".join(str(part) for part in split)) + for note in dict.fromkeys(w.halted for w, _ in told if w.halted): text += note return text @@ -706,6 +762,20 @@ class _Table: failed: dict[str, _Why] = field(default_factory=dict) parts_done: int = 0 finished: bool = False + #: 写出了内容的段（第几段） + written_parts: set[int] = field(default_factory=set) + #: 接连几个原始段一项都没写出来（写出来就清零）；到了 MODEL_TABLE_GIVE_UP_AFTER 段，剩下的段跳过不问 + dead_parts: int = 0 + abandoned: bool = False + #: 跳过不问的段（第几段） + skipped_parts: list[int] = field(default_factory=list) + #: 块进度：实际分成几块、结束了几块 + chunks_total: int = 0 + chunks_done: int = 0 + #: 有效行过滤核对过了（表级那段写完核对一次） + vetted: bool = False + #: 起草时记下的几句话（不算失败）：模型提议的有效记录条件没有收下的原因 + log: list[str] = field(default_factory=list) def whole(self, piece: _Piece) -> bool: return piece.table_level and len(piece.columns) == len(self.columns) @@ -753,6 +823,15 @@ class _Budget: left: int +@dataclass +class _Chunk: + """一块：一次拿来问的几张表（多表拼成的段每张表各是一块）。补问、截断后补剩下的、换纯文本再问都还是这一块；拆成两半 + （单张表按列拆，多一块；多表的段按表拆，各张表仍是一块）后，这一块里被拆的表交给新的块，split 记下它们。""" + + tables: tuple[str, ...] + split: set[str] = field(default_factory=set) + + def _pack(pieces: list[_Piece], cap: int) -> list[list[_Piece]]: """多表的段按顺序重新拼成每段不超过 cap 列（超过 cap 的表自成一段）。""" groups: list[list[_Piece]] = [] @@ -800,31 +879,37 @@ class _Run: self.background = self._background(getattr(source, "description", None), background) #: 这次起草同时在等名额或正在进行的调用最多几个（全进程的名额另算，见 model_gate） self.gate = asyncio.Semaphore(max(1, int(C.MODEL_CONCURRENCY))) - #: 连续几段拆到底仍一项没写出来（止损用，有一段写出来就清零） - self.dead_streak = 0 - #: 拆到每段 MODEL_MIN_PART_COLUMNS 列（或更小）的调用接连超时了几次（有一次调用写出来就清零）：超时和段大小 - #: 无关，再拆小也是一次一个时限，后面超时的不再拆小 - self.slow_streak = 0 #: 新开始的段每段最多几列：额度用完或超时（strikes）累计 MODEL_SHRINK_AFTER 次减半一次 self.part_columns = max(1, int(C.MODEL_BATCH_COLUMNS)) self.strikes = 0 #: 最近几次调用没写出来的原因种类（写出来了是 None），提醒看它 self.recent: collections.deque[str | None] = collections.deque(maxlen=max(1, int(C.MODEL_NOTICE_WINDOW))) self.notice: str | None = None + #: 最近 MODEL_HALT_WINDOW 次调用没写出来没有（止损按比例看它） + self.window: collections.deque[bool] = collections.deque(maxlen=max(1, int(C.MODEL_HALT_WINDOW))) + self.was_halted = False + #: 这次起草里结构化输出只给空壳时换纯文本问过没有（止损期间只问这一次） + self.text_probed = False + #: 数据源：开了数据剖析的源把码值交给模型、核对有效行过滤的排除比例（遮罩的列都不碰） + from app.data import catalog_profile + from app.data.engine import masked_columns - def _halt_note(self, kind: str | None = None) -> str: - """止损的说明（拼在原因后面），没止损是空串。 + self.source = source + self.kind = getattr(source, "kind", None) + options = getattr(source, "options", None) + self.profiling = catalog_profile.profile_settings(options if isinstance(options, dict) else None) + self.masked = {c.lower() for c in masked_columns(options if isinstance(options, dict) else None)} - - 连续 MODEL_GIVE_UP_AFTER 段一项都没写出来：后面的段不再拆小重试（不论原因）； - - 超时的（kind 是 timeout）另看：拆到每段 MODEL_MIN_PART_COLUMNS 列的调用接连超时 MODEL_GIVE_UP_AFTER 次， - 超时就和段大小无关了，再拆小也是一次一个时限，后面超时的不再拆小，超时拆出来还没问的块也不再问。只管超时： - 额度用完、只给空壳照旧拆小（段小了思考和输出才放得下）。""" - give_up = max(1, int(C.MODEL_GIVE_UP_AFTER)) - if self.dead_streak >= give_up: - return REASON_HALTED.format(n=give_up) - if kind == "timeout" and self.slow_streak >= give_up: - return REASON_SLOW.format(n=self._floor()) - return "" + def _halted(self) -> bool: + """止损：最近 MODEL_HALT_WINDOW 次调用里有 MODEL_HALT_ON 次以上没写出来。之后的段不再拆小、每块只问一次，直到 + 比例降下去（偶尔一次写出来不会清零）。""" + return sum(self.window) >= max(1, int(C.MODEL_HALT_ON)) + + def _halt_note(self) -> str: + """止损的说明（拼在原因后面），没止损是空串。""" + if not self._halted(): + return "" + return REASON_HALTED.format(window=max(1, int(C.MODEL_HALT_WINDOW)), on=max(1, int(C.MODEL_HALT_ON))) @staticmethod def _floor() -> int: @@ -846,6 +931,8 @@ class _Run: head, *rest = self.tables[unit].parts # 表级那段先写：列分段拿它做背景 queue.append(_Work([head], [_Work([p]) for p in rest])) + for st in self.tables.values(): + st.chunks_total = len(st.parts) left = {name: len(st.parts) for name, st in self.tables.items()} busy = 0 changed = asyncio.Event() @@ -898,9 +985,19 @@ class _Run: """起草一段；它之后的段（宽表的列分段）排到队尾；一张表的段都结束了，这张表的模型部分就结束了。 这一段（原始段：计划里的整表段、表级那段或一个列分段）连同按缩小后的上限分开的各块、拆出来的、补问的、重试的， - 一共最多调用 MODEL_PART_MAX_CALLS 次（各块共用一个 _Budget）；止损按原始段数（_tally）。""" - await self._start(work.pieces, _Budget(max(1, int(C.MODEL_PART_MAX_CALLS)))) - self._tally(work.pieces) + 一共最多调用 MODEL_PART_MAX_CALLS 次（各块共用一个 _Budget）。表级那段写完先核对模型提议的有效行过滤（_vet）， + 再交出去。接连几段没写出来的表（_tally），剩下的段跳过不问。宽表写出了一段、后面还有段没结束，就把这张表到这时 + 拿到的内容交给调用方并入目录（part_written）；最后一段结束由 table_done 收尾。""" + asked = [p for p in work.pieces if not self.tables[p.table].abandoned] + for piece in work.pieces: + if self.tables[piece.table].abandoned: + await self._skip_part(piece) + if asked: + await self._start(asked, _Budget(max(1, int(C.MODEL_PART_MAX_CALLS)))) + for piece in asked: + if piece.table_level: + await self._vet(self.tables[piece.table]) + self._tally(asked) for piece in work.pieces: await self._part_done(piece.table) queue.extend(work.then) @@ -908,6 +1005,8 @@ class _Run: left[piece.table] -= 1 if not left[piece.table]: await self._table_done(piece.table) + elif piece.part in self.tables[piece.table].written_parts: + await self._part_written(piece.table) async def _start(self, pieces: list[_Piece], budget: _Budget) -> None: """开始起草一段。超过当前的列数上限（额度用完、超时多了会减半）的先按上限分开：多表的段按表重新拼，单张表的 @@ -922,6 +1021,7 @@ class _Run: return piece = pieces[0] chunks = _even(piece.columns, -(-len(piece.columns) // cap)) + await self._more_chunks(piece.table, len(chunks) - 1) first = [_Piece(piece.table, chunks[0], piece.table_level, piece.part)] rest = [[_Piece(piece.table, cols, False, piece.part)] for cols in chunks[1:]] if piece.table_level: @@ -931,9 +1031,10 @@ class _Run: await asyncio.gather(*(self._guarded(group, budget) for group in [first, *rest])) async def _guarded(self, pieces: list[_Piece], budget: _Budget) -> None: - """起草一段（或它按上限分开的一块）；意料之外的错误只记这一块没写出来，不影响别的段和别的表。""" + """起草一块（一段，或它按上限分开的一块）；意料之外的错误只记这一块没写出来，不影响别的段和别的表。""" + chunk = _Chunk(tuple(dict.fromkeys(p.table for p in pieces))) try: - await self._segment(pieces, 0, False, budget) + await self._segment(pieces, 0, False, budget, chunk=chunk) except asyncio.CancelledError: raise except Exception as e: # noqa: BLE001 @@ -941,49 +1042,80 @@ class _Run: logger.exception("起草数据目录时出错：%s", "、".join(p.table for p in pieces)) self._fail(pieces, REASON_CRASH.format(line=first_line(e) or describe_exception(e))) + await self._chunk_done(chunk) def _tally(self, pieces: list[_Piece]) -> None: - """一段（原始段，连同它分开的各块、拆小、补缺）结束：一项都没写出来的接着数，写出来了清零。""" - if any(self.tables[p.table].wrote(p) for p in pieces): - self.dead_streak = 0 + """一段（原始段，连同它分开的各块、拆小、补缺）结束：记下这一段写出内容没有；同一张表接连 + MODEL_TABLE_GIVE_UP_AFTER 段一项都没写出来，这张表剩下的段跳过不问（写出来了清零）。""" + give_up = max(1, int(C.MODEL_TABLE_GIVE_UP_AFTER)) + for piece in pieces: + st = self.tables[piece.table] + if st.wrote(piece): + st.written_parts.add(piece.part) + st.dead_parts = 0 + continue + st.dead_parts += 1 + if st.dead_parts >= give_up and not st.abandoned and len(st.parts) > 1: + st.abandoned = True + logger.warning("起草数据目录：表 %s 接连 %d 段都没写出来，剩下的段跳过", st.name, st.dead_parts) + + async def _skip_part(self, piece: _Piece) -> None: + """这张表接连几段都没写出来：这一段跳过不问（算一块结束）。""" + st = self.tables[piece.table] + st.skipped_parts.append(piece.part) + self._fail([piece], _Why(_SKIPPED)) + st.chunks_done += 1 + await self._hook(self.hooks.chunks_changed, st.name, st.chunks_total, st.chunks_done) + + # ---------------------------------------------------------------- 块进度 + + async def _more_chunks(self, table: str, n: int) -> None: + if n <= 0: return - self.dead_streak += 1 - if self.dead_streak == max(1, int(C.MODEL_GIVE_UP_AFTER)): - logger.warning("起草数据目录：模型连续 %d 段都没写出来，这次起草后面的段不再拆小重试", self.dead_streak) + st = self.tables[table] + st.chunks_total += n + await self._hook(self.hooks.chunks_changed, table, st.chunks_total, st.chunks_done) + + async def _chunk_done(self, chunk: _Chunk) -> None: + """一块问完了：这一块里没被拆给新块的表，各结束一块。""" + for table in chunk.tables: + if table in chunk.split: + continue + st = self.tables[table] + st.chunks_done += 1 + await self._hook(self.hooks.chunks_changed, table, st.chunks_total, st.chunks_done) # ---------------------------------------------------------------- 一段 async def _segment(self, pieces: list[_Piece], depth: int, followup: bool, budget: _Budget, *, - halved: bool = False, mode: str | None = None, last: bool = False, - optional: bool = False) -> None: + chunk: _Chunk, halved: bool = False, mode: str | None = None, last: bool = False, + optional: str | None = None) -> None: """问一段，拿到的并进各表；按失败原因拆开、补缺、记下没写出来的。 depth：已经拆了几层（截断后只补剩下的也算一层）。followup：这一段是在补缺（补过就不再补，见 _after_followup）。 - budget：这一段（原始段）还能调用几次。halved：这几块是真拆小（_split）出来的——原因里「已拆小重试」只在这时写。 - mode / last：换纯文本问的那一次——拆不开的空壳、结构化输出回了说明文字（再不行就记没写出来）。 - optional：超时拆出来的块，轮到它时已经止损就不问了（_ask）。""" + budget：这一段（原始段）还能调用几次。chunk：这是哪一块（块进度）。halved：这几块是真拆小（_split）出来的—— + 原因里「已拆小重试」只在这时写。mode / last：换纯文本问的那一次——拆不开的空壳、结构化输出回了说明文字（再不行 + 就记没写出来）。optional：拆出来的块（值是拆它的原因），轮到它时已经止损就不问了（_ask）。 + + 止损期间（_halted）每块只问一次：不拆小（_split）、不补问漏掉的列、截断后不再补剩下的。""" reply = await self._ask(pieces, budget, mode, optional=optional) if reply.capped: self._fail(pieces, self._capped(halved)) return if reply.skipped: - note = self._halt_note("timeout") - self._fail(pieces, lambda p: self._reason("timeout", split=True, piece=p, halted=note)) + note = self._halt_note() + self._fail(pieces, lambda p: self._reason(optional or "hollow", split=True, piece=p, halted=note)) return if reply.timeout: - # 先记下再报用量：报用量要写库、会让出事件循环，等名额的块醒来时要看到这一次 - if sum(len(p.columns) for p in pieces) <= self._floor(): - self.slow_streak += 1 - if self.slow_streak == max(1, int(C.MODEL_GIVE_UP_AFTER)): - logger.warning("起草数据目录：拆到每段 %d 列仍接连超时 %d 次，后面超时的不再拆小", - self._floor(), self.slow_streak) await self._report(reply, wrote=False) - await self._split(pieces, depth, "timeout", followup, budget, halved) + await self._split(pieces, depth, "timeout", followup, budget, halved, chunk) return if reply.error is not None: - await self._report(reply, wrote=False) - if reply.prose and not last: - await self._prose_to_text(pieces, depth, followup, budget, halved) + # 结构化输出回了说明文字、这一块马上改走纯文本再问：只管通信的重来，不进止损的窗口 + redo = reply.prose and not last + await self._report(reply, wrote=False, halting=not redo) + if redo: + await self._prose_to_text(pieces, depth, followup, budget, halved, chunk) return self._fail(pieces, _Why(reply.error, split=halved)) return @@ -1008,28 +1140,36 @@ class _Run: return kind = "exhausted" if reply.exhausted else "truncated" if not anything: - await self._split(rest, depth, kind, followup, budget, halved) - elif depth < int(C.MODEL_SPLIT_DEPTH): + await self._split(rest, depth, kind, followup, budget, halved, chunk) + elif depth < int(C.MODEL_SPLIT_DEPTH) and not self._halted(): # 捡回了一部分：只补剩下的（不是拆小） - await self._segment(rest, depth + 1, followup, budget, halved=halved) + await self._segment(rest, depth + 1, followup, budget, chunk=chunk, halved=halved) else: - self._fail(rest, self._reason(kind, split=halved)) + note = self._halt_note() if depth < int(C.MODEL_SPLIT_DEPTH) else "" + self._fail(rest, self._reason(kind, split=halved, halted=note)) return if not anything and not followup: if last: halted = self._stopped(pieces, depth) self._fail(pieces, lambda p: self._reason("hollow", split=halved, piece=p, halted=halted)) return - await self._split(pieces, depth, "hollow", followup, budget, halved) + await self._split(pieces, depth, "hollow", followup, budget, halved, chunk) return if not rest: return if followup: - await self._after_followup(rest, depth, budget, halved, last=last) + await self._after_followup(rest, depth, budget, halved, chunk, last=last) return - await self._segment(rest, depth, True, budget, halved=halved) + if self._halted(): + # 止损期间每块只问一次：漏了的不补问 + note = self._halt_note() + self._fail(rest, lambda p: _Why(REASON_OMITTED, split=halved, halted=note) + if self.tables[p.table].has_any() else self._reason("hollow", split=halved, piece=p, + halted=note)) + return + await self._segment(rest, depth, True, budget, chunk=chunk, halved=halved) - async def _after_followup(self, rest: list[_Piece], depth: int, budget: _Budget, halved: bool, *, + async def _after_followup(self, rest: list[_Piece], depth: int, budget: _Budget, halved: bool, chunk: _Chunk, *, last: bool) -> None: """补问过仍然没给的部分（补过就不再补）。 @@ -1047,16 +1187,17 @@ class _Run: halted = self._stopped(empty, depth) self._fail(empty, lambda p: self._reason("hollow", split=halved, piece=p, halted=halted)) return - await self._split(empty, depth, "hollow", False, budget, halved) + await self._split(empty, depth, "hollow", False, budget, halved, chunk) async def _prose_to_text(self, pieces: list[_Piece], depth: int, followup: bool, budget: _Budget, - halved: bool) -> None: + halved: bool, chunk: _Chunk) -> None: """结构化输出回了一段说明文字：网关接受了结构化输出的参数却没照做。这次起草后面都走纯文本（另带格式说明）， - 这一段换纯文本问一次。纯文本额度用完了还能照常拆小（拆出来的半段也走纯文本，不再先问一遍结构化输出）。""" + 这一段换纯文本问一次（止损期间也问：这是通信方式的问题，不是模型写不出来）。纯文本额度用完了还能照常拆小 + （拆出来的半段也走纯文本，不再先问一遍结构化输出）。""" if self.mode != "text": self.mode = "text" logger.info("起草数据目录：结构化输出回了说明文字，后面改走纯文本") - await self._segment(pieces, depth, followup, budget, halved=halved, mode="text", last=True) + await self._segment(pieces, depth, followup, budget, chunk=chunk, halved=halved, mode="text", last=True) def _stopped(self, pieces: list[_Piece], depth: int) -> str: """这几段本来还能拆小、因为止损才没拆时，止损的说明；不是的话是空串。""" @@ -1065,36 +1206,57 @@ class _Run: return "" async def _split(self, pieces: list[_Piece], depth: int, kind: str, followup: bool, budget: _Budget, - halved: bool) -> None: + halved: bool, chunk: _Chunk) -> None: """拆成两半再问（额度用完、超时、只给空壳、截断后一项都没捡回来）。 - 同一张表按列拆、前一半带着表级的项：先写前一半，后一半拿它做背景。超时的不等：一次超时就是一个时限，40 → 20 - → 10 一层一层串着问要好几个时限，两半同时问；拆出来的块轮到它时已经止损的不再问（optional）。 - - 拆不开的空壳再问一次：结构化输出给的换纯文本，已经走纯文本的原样再问。止损时本来能拆的不拆：结构化输出给的 - 照旧换纯文本问一次（看看是不是结构化输出的问题），已经走纯文本的不再问。 + → 10 一层一层串着问要好几个时限，两半同时问。拆出来的两半是新的块（单张表多一块）；轮到它时已经止损的不再问 + （optional）。 + - 拆不开的空壳再问一次：结构化输出给的换纯文本，已经走纯文本的原样再问。 + - 止损期间（_halted）不拆小、每块只问一次：本来能拆的不拆；空壳不再原样再问，只有结构化输出给的、这次起草还没 + 换纯文本问过的照旧问一次（看看是不是结构化输出的问题）。 - 别的原因拆不开、止损了，就记没写出来。""" halves = _halves(pieces) if depth < int(C.MODEL_SPLIT_DEPTH) else None - halted = self._halt_note(kind) if halves is not None else "" + halted = self._halted() if halves is not None and not halted: first, second = halves - timeout = kind == "timeout" - if (not timeout and len(first) == len(second) == 1 and first[0].table == second[0].table + await self._split_chunk(chunk, pieces) + if (kind != "timeout" and len(first) == len(second) == 1 and first[0].table == second[0].table and first[0].table_level): - await self._segment(first, depth + 1, followup, budget, halved=True) - await self._segment(second, depth + 1, followup, budget, halved=True) + await self._half(first, depth + 1, followup, budget, kind) + await self._half(second, depth + 1, followup, budget, kind) return - results = await asyncio.gather( - self._segment(first, depth + 1, followup, budget, halved=True, optional=timeout), - self._segment(second, depth + 1, followup, budget, halved=True, optional=timeout), - return_exceptions=True) + results = await asyncio.gather(self._half(first, depth + 1, followup, budget, kind), + self._half(second, depth + 1, followup, budget, kind), + return_exceptions=True) for result in results: if isinstance(result, BaseException): raise result return - if kind == "hollow" and (not halted or self.mode == "structured"): - await self._segment(pieces, depth, followup, budget, halved=halved, mode="text", last=True) + if kind == "hollow" and (not halted or (self.mode == "structured" and not self.text_probed)): + self.text_probed = True + await self._segment(pieces, depth, followup, budget, chunk=chunk, halved=halved, mode="text", last=True) return - self._fail(pieces, lambda p: self._reason(kind, split=halved, piece=p, halted=halted)) + note = self._halt_note() if halted and (halves is not None or kind == "hollow") else "" + self._fail(pieces, lambda p: self._reason(kind, split=halved, piece=p, halted=note)) + + async def _half(self, pieces: list[_Piece], depth: int, followup: bool, budget: _Budget, kind: str) -> None: + """拆出来的一半：新的一块，问完算结束一块。""" + chunk = _Chunk(tuple(dict.fromkeys(p.table for p in pieces))) + try: + await self._segment(pieces, depth, followup, budget, chunk=chunk, halved=True, optional=kind) + except asyncio.CancelledError: + raise + except Exception: + await self._chunk_done(chunk) # 出错也算问完（_guarded 记没写出来），块进度对得上 + raise + await self._chunk_done(chunk) + + async def _split_chunk(self, chunk: _Chunk, pieces: list[_Piece]) -> None: + """这一块里的这几张表拆给了新的块：单张表按列拆成两半，多一块；多表的段按表拆，各张表仍是一块。""" + chunk.split.update(dict.fromkeys(p.table for p in pieces)) + if len(pieces) == 1: + await self._more_chunks(pieces[0].table, 1) def _remainder(self, pieces: list[_Piece], gains: list[_Gain], *, truncated: bool, followup: bool) -> list[_Piece]: @@ -1118,13 +1280,14 @@ class _Run: # ---------------------------------------------------------------- 调用 async def _ask(self, pieces: list[_Piece], budget: _Budget, mode: str | None = None, *, - optional: bool = False) -> _Reply: + optional: str | None = None) -> _Reply: """调一次模型（按原因重来：额度被拒、结构化输出不支持、网络错误……），交回读好的回复。 - 这一段的调用次数用完了交回 capped（不调用）。optional（超时拆出来的块）：拿到名额时已经止损，交回 skipped（不调用， - 次数退回）——等名额的时候前面的块接连超时，再问也是一个时限。超时不原样重试：交回 timeout，由调用方当作这一段 - 太大、拆小再问。重来之前的那次调用在这里就报给进度回调；交回的那次由调用方读完内容再报（写没写出来要看校验之后 - 的内容）。""" + 这一段的调用次数用完了交回 capped（不调用）。optional（拆出来的块）：拿到名额时已经止损，交回 skipped（不调用， + 次数退回）——等名额的时候前面的调用多数没写出来，不再拆小重试。超时不原样重试：交回 timeout，由调用方当作这一段 + 太大、拆小再问。重来之前的那次调用在这里就报给进度回调（记进用量和提醒，不进止损的窗口：halting=False）；交回的 + 那次由调用方读完内容再报（写没写出来要看校验之后的内容）。服务方嫌额度太大、结构化输出不支持、网络错误的重来止损 + 期间照旧（只管通信，不是模型写不出来）。""" mode = mode or self.mode retried = 0 while True: @@ -1135,7 +1298,7 @@ class _Run: try: # 先占这次起草自己的名额，再去排全进程的：一次起草排在全进程名额上的调用最多 MODEL_CONCURRENCY 个 async with self.gate, model_gate(): - if optional and self._halt_note("timeout"): + if optional and self._halted(): budget.left += 1 return _Reply(skipped=True) await self._hook(self.hooks.call_started, [p.table for p in pieces]) @@ -1151,16 +1314,16 @@ class _Run: if kind == "timeout": return _Reply(timeout=True, usage=(0, 0)) if kind == "budget" and (getattr(self.model, "budget", None) != used or self._shrink()): - await self._called((0, 0), "error") + await self._called((0, 0), "error", halting=False) continue if kind == "unsupported" and mode == "structured": if self.mode == "structured": logger.info("起草数据目录：结构化输出用不了（%s），后面改走纯文本", str(e)[:200]) self.mode = mode = "text" - await self._called((0, 0), "error") + await self._called((0, 0), "error", halting=False) continue if kind == "transient" and retried < int(C.MODEL_RETRIES): - await self._called((0, 0), "error") + await self._called((0, 0), "error", halting=False) delays = tuple(C.MODEL_RETRY_DELAYS_S) or (0.0,) await _sleep(delays[min(retried, len(delays) - 1)]) retried += 1 @@ -1212,8 +1375,9 @@ class _Run: # ---------------------------------------------------------------- 每次调用之后：用量、段大小、提醒 - async def _report(self, reply: _Reply, *, wrote: bool) -> None: - """交回的那次调用读完内容之后报给进度回调。""" + async def _report(self, reply: _Reply, *, wrote: bool, halting: bool = True) -> None: + """交回的那次调用读完内容之后报给进度回调。halting：这次是不是这一块这次调用的最终结局（进不进止损的窗口，见 + _called）。""" if reply.usage is None: return if reply.timeout: @@ -1224,14 +1388,26 @@ class _Run: failure = None else: failure = "exhausted" if reply.exhausted else "hollow" - await self._called(reply.usage, failure, strike=reply.timeout or reply.exhausted) + await self._called(reply.usage, failure, strike=reply.timeout or reply.exhausted, halting=halting) - async def _called(self, usage: tuple[int, int], failure: str | None, *, strike: bool = False) -> None: + async def _called(self, usage: tuple[int, int], failure: str | None, *, strike: bool = False, + halting: bool = True) -> None: """记下一次调用：额度用完或超时（strike）累计 MODEL_SHRINK_AFTER 次，新开始的段列数上限减半；最近几次调用多数 - 没写出来时提醒，少了清掉。写出来了（failure 为 None）就清掉接连超时的计数。报给进度回调。""" + 没写出来时提醒，少了清掉；止损按最近 MODEL_HALT_WINDOW 次调用的比例（_halted）。报给进度回调。 + + halting=False：只管通信的重来之前的那次（服务方嫌额度太大、结构化输出不支持或回了说明文字、429 / 5xx 退避重试）。 + 照旧记进用量和提醒，不进止损的窗口：它不花额度，也说明不了模型写不出来；止损只看每次调用的最终结局。""" self.recent.append(failure) - if failure is None: - self.slow_streak = 0 + if halting: + self.window.append(failure is not None) + halted = self._halted() + if halted != self.was_halted: + self.was_halted = halted + if halted: + logger.warning("起草数据目录：最近 %d 次调用有 %d 次没写出来，之后的段不再拆小、每块只问一次", + len(self.window), sum(self.window)) + else: + logger.info("起草数据目录：最近的调用多数写出来了，恢复拆小重试") if strike: self.strikes += 1 width = max(1, int(C.MODEL_BATCH_COLUMNS)) @@ -1280,7 +1456,14 @@ class _Run: gain = _Gain() for entry in by_table.get(piece.table, []): gain.named = st.named = True - notes = _restrict(C._from_model(entry, st.meta, system=self.system), piece) + dropped: list[tuple[str, str]] = [] + drafted = C._from_model(entry, st.meta, system=self.system, source_kind=self.kind, table=st.name, + dropped=dropped) + if piece.table_level and "valid_filter" not in st.notes: + # 只在请它写表级的项时记：只写几列的那几段里多给的表级的项本来就不要 + for why, condition in dropped: + self._log(st, why, condition) + notes = _restrict(drafted, piece) cols = entry.get("columns") if isinstance(entry.get("columns"), list) else [] gain.answered |= {c.get("name") for c in cols if isinstance(c, dict) and c.get("name") in piece.columns} slots = C._slots(notes) @@ -1344,21 +1527,33 @@ class _Run: failed_table = _TABLE in st.failed and not st.got_table total = len(st.parts) or 1 part_of = st.part_of() + progress = {"parts_written": len(st.written_parts), "chunks_total": st.chunks_total, + "chunks_done": st.chunks_done, "log": list(dict.fromkeys(st.log))[:LOG_MAX]} + abandoned = self._abandoned_text(st) if not items: - failures = [(why, part_of.get(key, 1)) for key, why in st.failed.items()] + failures = [(why, part_of.get(key, 1)) for key, why in st.failed.items() if why.base != _SKIPPED] reason = _join(failures) if failures else (C.MODEL_EMPTY_REASON if st.named else REASON_MISSING) - if failures and len(st.parts) > 1: + if abandoned: + reason = f"{reason}；{abandoned}" + if st.failed and len(st.parts) > 1: # 宽表一项都没写出来：同样逐段写上列的范围 said = self._parts_said(st, failed_cols, failed_table) every = "都" if len(said) == len(st.parts) else "" reason = f"这张表 {len(st.columns)} 列，分 {len(st.parts)} 段起草，{'、'.join(said)}{every}没写出来：{reason}" - return ModelOutcome("failed", {}, 0, reason, total, st.parts_done) + return ModelOutcome("failed", {}, 0, reason, total, st.parts_done, **progress) notes = copy.deepcopy(st.notes) blank = self._blank(st, set(failed_cols)) if failed_cols or failed_table: - return ModelOutcome("partial", notes, items, self._partial_text(st, failed_cols, failed_table), total, - st.parts_done, blank) - return ModelOutcome("ok", notes, items, None, total, st.parts_done, blank) + return ModelOutcome("partial", notes, items, self._partial_text(st, failed_cols, failed_table, abandoned), + total, st.parts_done, blank, **progress) + return ModelOutcome("ok", notes, items, None, total, st.parts_done, blank, **progress) + + @staticmethod + def _abandoned_text(st: _Table) -> str: + """这张表接连几段没写出来、剩下的段跳过了：说清跳过了几段、怎么办。没跳过是空串。""" + if not st.skipped_parts: + return "" + return REASON_ABANDONED.format(k=max(1, int(C.MODEL_TABLE_GIVE_UP_AFTER)), n=len(st.skipped_parts)) def _blank(self, st: _Table, failed: set[str]) -> list[str]: """模型拿不准、留空了的列：请它写了（不在没写出来的里面），中文名和含义却都没给（或者给的都没通过校验）。 @@ -1395,11 +1590,14 @@ class _Run: return said + [f"第 {k} 段（{_columns_text(st.columns, cols, count=True)}）" for k, cols in sorted(bad.items())] @classmethod - def _partial_text(cls, st: _Table, failed_cols: list[str], failed_table: bool) -> str: - """只写出一部分时说清哪部分没写出来（第几段、哪几列）、为什么、怎么办。""" + def _partial_text(cls, st: _Table, failed_cols: list[str], failed_table: bool, abandoned: str = "") -> str: + """只写出一部分时说清哪部分没写出来（第几段、哪几列）、为什么、怎么办。剩下的段跳过了的，跳过的说明里已经说了 + 怎么办（换一个起草模型后只重试这张表），不再另说「可以只重试这张表」。""" part_of = st.part_of() keys = ([_TABLE] if failed_table else []) + failed_cols - reason = _join((st.failed[k], part_of.get(k, 1)) for k in keys) + reason = _join((st.failed[k], part_of.get(k, 1)) for k in keys if st.failed[k].base != _SKIPPED) + if abandoned: + reason = f"{reason}；{abandoned}" if reason else abandoned n = len(st.columns) if len(st.parts) > 1: said = cls._parts_said(st, failed_cols, failed_table) @@ -1411,7 +1609,7 @@ class _Run: head = f"这张表 {n} 列，有 {len(failed_cols)} 列（{_columns_text(st.columns, failed_cols, count=False)}）没写出来" else: head = f"这张表 {n} 列，表级的项没写出来" - return f"{head}：{reason}。可以只重试这张表" + return f"{head}：{reason}" if abandoned else f"{head}：{reason}。可以只重试这张表" async def _part_done(self, name: str) -> None: self.tables[name].parts_done += 1 @@ -1424,6 +1622,65 @@ class _Run: st.finished = True await self._hook(self.hooks.table_done, name, self._outcome(st)) + async def _part_written(self, name: str) -> None: + """宽表写出了一段、后面还有段没结束：把这张表到这时拿到的全部内容交给调用方并入目录（停止、服务重启时不丢）。""" + st = self.tables[name] + await self._hook(self.hooks.part_written, name, PartWrite(copy.deepcopy(st.notes), len(st.written_parts))) + + # ---------------------------------------------------------------- 防错事实：核对有效行过滤 + + async def _vet(self, st: _Table) -> None: + """表级那段写完，核对模型提议的有效行过滤（只核对一次）。 + + - 目录里人工确认、驳回过的有效行过滤：并入时不会采用这次的提议，不必再查库； + - 没开数据剖析：照收，note 写「没有核对排除比例（数据源没有开数据剖析）」； + - 开了：经剖析的只读守卫数排除比例（catalog_profile.check_valid_filter）。排除 0% 或 100% 的不收（这张表的日志 + 记一句，不算失败）；收下的 note 在依据后面写上剖析核对的结果，或者没有核对的原因。""" + if st.vetted: + return + st.vetted = True + item = st.notes.get("valid_filter") + if not isinstance(item, dict): + return + held = (st.existing or {}).get("valid_filter") + if isinstance(held, Mapping) and held.get("status") in ("confirmed", "rejected"): + return + from app.data import catalog_profile + + if not self.profiling.enabled: + found = catalog_profile.FilterCheck(True, catalog_profile.FILTER_UNCHECKED.format( + why=catalog_profile.FILTER_NOT_ENABLED)) + else: + try: + found = await catalog_profile.check_valid_filter(self.source, st.name, dict(st.meta), + str(item.get("value") or ""), settings=self.profiling) + except asyncio.CancelledError: + raise + except Exception as e: # noqa: BLE001 - 核对是附加的：出了意外照收，写明没有核对 + from app.core.errors import describe_exception, first_line + + logger.exception("起草数据目录：核对表 %s 的有效行过滤时出错", st.name) + found = catalog_profile.FilterCheck(True, catalog_profile.FILTER_UNCHECKED.format( + why=f"核对时出错：{first_line(e) or describe_exception(e)}")) + if not found.keep: + logger.info("起草数据目录：表 %s 的有效行过滤 %s 没有收下：%s", st.name, item.get("value"), found.dropped) + st.notes.pop("valid_filter", None) + why = found.dropped or found.note + if found.rows is not None: + why = LOG_FILTER_COUNTED.format(label=C.TABLE_FIELD_LABEL["valid_filter"], what=found.dropped, + rows=f"{found.rows:,}") + self._log(st, why, str(item.get("value") or "")) + return + item["note"] = _sentences(item.get("note"), found.note) + + @staticmethod + def _log(st: _Table, why: str, condition: str) -> None: + """这张表的日志记一句：模型提议的有效记录条件没有收下（不算失败）。""" + condition = " ".join(condition.split()) + if len(condition) > _LOG_CONDITION_MAX: + condition = condition[:_LOG_CONDITION_MAX] + "…" + st.log.append(LOG_FILTER_DROPPED.format(why=why, condition=condition)) + @staticmethod async def _hook(fn: Callable[..., Awaitable[None]] | None, *args: Any) -> None: if fn is None: @@ -1502,7 +1759,12 @@ class _Run: for group in meta.get("unique") or []: if isinstance(group, list) and group: lines.append(labels["unique"].format(cols="、".join(group))) - confirmed = self._confirmed_lines(st, piece) + shown = self._confirmed_for(st, piece) + codes = self._codes_lines(st, piece, shown) + if codes: + lines.append(labels["codes"]) + lines += codes + confirmed = C._notes_lines(shown, meta=st.meta, system_notes=False) if confirmed: lines.append(labels["confirmed"]) lines += confirmed @@ -1522,19 +1784,48 @@ class _Run: line = labels["column"].format(name=col["name"], type=col.get("type") or "") return line + ("，" + "，".join(marks) if marks else "") - @staticmethod - def _confirmed_lines(st: _Table, piece: _Piece) -> list[str]: - """已确认、有确证的项（以此为准的背景）：整表全给；其余给表级的项和这一段的列，关系只在要写表级的项时给。""" - confirmed = C._confirmed_only(st.existing) - if not st.whole(piece): - wanted = set(piece.columns) - kept = {key: item for key, item in C._slots(confirmed).items() - if key[0] == "t" or (key[0] == "c" and key[1] in wanted) or (key[0] == "r" and piece.table_level)} - confirmed = C._assemble(kept) - return C._notes_lines(confirmed, meta=st.meta, system_notes=False) + def _confirmed_for(self, st: _Table, piece: _Piece) -> dict[str, Any]: + """已确认、有确证的项（以此为准的背景）：整表全给；其余给表级的项和这一段的列，关系只在要写表级的项时给。 + + 遮罩的列上的码值不给，人工确认过的也一样：码值就是这一列的取值（同 _codes_lines，也同表助手的背景、导入目录的 + 规矩）。以前只有新加的码值背景挡住了，这里照给。这一列别的确认过的项（中文名、含义……）照给。""" + whole = st.whole(piece) + wanted = set(piece.columns) + kept = {key: item for key, item in C._slots(C._confirmed_only(st.existing)).items() + if not (key[0] == "c" and key[2] == "codes" and key[1].lower() in self.masked) + and (whole or key[0] == "t" or (key[0] == "c" and key[1] in wanted) + or (key[0] == "r" and piece.table_level))} + return C._assemble(kept) + + def _codes_lines(self, st: _Table, piece: _Piece, shown: Mapping[str, Any]) -> list[str]: + """开了数据剖析的源：目录里记下的码值（剖析取过的取值和已知含义）交给模型，判断有效行过滤、列的含义用。要写表级 + 的项的给全部列的，只写几列的给这几列的；遮罩的列、驳回的码值不给；「已确认的目录项」里已经有的不重复。""" + if not self.profiling.enabled: + return [] + labels = C.CATALOG_DRAFT_PROMPT_LABELS + wanted = set(st.columns) if piece.table_level else set(piece.columns) + known = C.visible_notes(st.existing).get("columns") or {} + given = (shown.get("columns") or {}) if isinstance(shown.get("columns"), Mapping) else {} + out: list[str] = [] + for col in st.columns: + item = (known.get(col) or {}).get("codes") + if col not in wanted or col.lower() in self.masked or not isinstance(item, Mapping): + continue + if "codes" in (given.get(col) or {}) or not isinstance(item.get("value"), Mapping) or not item["value"]: + continue + out.append(labels["code"].format(name=col, codes=C._codes_text(item["value"]))) + return out -__all__ = ["BUDGET_WORDS", "Background", "CallReport", "FAILURE_LABEL", "Hooks", "ModelOutcome", "NOTICE", "REASON_CALL", - "REASON_CAPPED", "REASON_EXHAUSTED", "REASON_FORMAT", "REASON_HALTED", "REASON_HOLLOW", "REASON_MISSING", - "REASON_RETRIED", "REASON_SLOW", "REASON_SPLIT", "REASON_TIMEOUT", "REASON_TRUNCATED", "REASON_UNANSWERED", - "SEG", "classify_failure", "model_gate", "parts_for", "run_model_draft"] +def _sentences(*parts: Any) -> str: + """几句话连成一段：各句去掉句末的句号，用句号隔开（「依据：……。剖析核对：……」）。""" + texts = [str(p).strip().rstrip("。") for p in parts if isinstance(p, str) and p.strip()] + return "。".join(texts) + + +__all__ = ["BUDGET_WORDS", "Background", "CallReport", "FAILURE_LABEL", "Hooks", "LOG_FILTER_COUNTED", + "LOG_FILTER_DROPPED", "LOG_MAX", "ModelOutcome", "NOTICE", "PartWrite", + "REASON_ABANDONED", "REASON_CALL", "REASON_CAPPED", "REASON_EXHAUSTED", "REASON_FORMAT", "REASON_HALTED", + "REASON_HOLLOW", "REASON_MISSING", "REASON_OMITTED", "REASON_RETRIED", "REASON_SPLIT", "REASON_SPLIT_PARTS", + "REASON_TIMEOUT", "REASON_TRUNCATED", "REASON_UNANSWERED", "SEG", "classify_failure", "model_gate", + "parts_for", "run_model_draft"] diff --git a/backend/app/data/catalog_jobs.py b/backend/app/data/catalog_jobs.py index d708ed50..48c37056 100644 --- a/backend/app/data/catalog_jobs.py +++ b/backend/app/data/catalog_jobs.py @@ -6,11 +6,15 @@ - **建任务**（create_job）：同一个数据源同时只有一个排队或进行中的任务（DraftBusy；部分唯一索引兜底）。不指定表时 按使用次数取前 20 张；表结构里没有的表照样建进任务、状态 skipped。起草要求（数据源级）和本次说明随任务记下快照。 - **在后台起草**（manager.start）：进程内的 asyncio 任务，引擎是 data/catalog_draft.py（分段、按失败原因处理）。 - 每张表一结束就并入目录（catalog.write_draft，规则和同步起草一样：人工确认、驳回过的不动，模型这次没给的项不删 - 以前起草的，乐观锁冲突重读重并）；逐表的状态、到第几段、计数、模型留空的列，任务的用量（含超时、没写出来的调用 - 次数）、当前的列数上限、提醒随时落库，接口读的是库里的状态。 + 宽表每写出一段就并入目录（write_part：这一次合并不删任何东西），整张表结束（不管写全没写全）再按完整覆盖的规则 + 收尾一次（catalog.write_draft，规则和同步起草一样：人工确认、驳回过的不动，模型这次没给的项不删以前起草的，只删 + 已经不在表结构里的列的模型项，乐观锁冲突重读重并）；新增、更新、删除的计数是各次写入的合计。逐表的状态、到第几段、 + 写进目录的段数、块进度、计数、模型留空的列，任务的用量（含超时、没写出来的调用次数）、当前的列数上限、提醒随时落库， + 接口读的是库里的状态。 - **停止**（cancel）：排队的直接 cancelled；进行中的停掉正在进行的模型调用，没开始和没写完的表记 cancelled，已经写进 - 目录的不回滚，任务最后是 cancelled。 + 目录的（含宽表已经写出来的段）不回滚，任务最后是 cancelled。写过段的表 model_error 说清写进了几段、剩几段没起草。 +- **和同步起草互斥**（sync_draft）：同步起草（POST /catalog/draft）进行期间，同一个源建任务、重试、导入（ensure_idle） + 都是 DraftBusy；有排队或进行中的任务时同步起草也是。进程内登记（单进程部署）。 - **重试**（retry_tables + create_job）：新任务，只含原任务里 failed、partial、cancelled 的表；用不用模型、本次说明沿用， 起草要求取数据源当前的。 - **服务重启**：启动时（startup）把排队、进行中的任务标成 interrupted，没结束的表记 cancelled、写明「服务重启，起草 @@ -26,11 +30,12 @@ from __future__ import annotations import asyncio +import contextlib import logging -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from typing import Any -from sqlalchemy import func, select, update +from sqlalchemy import delete, func, or_, select, update from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession @@ -62,6 +67,10 @@ BLANK_SAMPLE = 5 STOP_GRACE_S = 10.0 INTERRUPTED_ERROR = "服务重启，起草中断" +#: 停止、中断时写过段的表（宽表每写出一段就并入目录）：写进了几段、剩几段没起草 +STOPPED_WRITTEN = "已停止：已写进目录 {n} 段，剩下 {m} 段没起草" +INTERRUPTED_WRITTEN = "已中断：已写进目录 {n} 段，剩下 {m} 段没起草" +_WRITTEN = {"cancelled": STOPPED_WRITTEN, "interrupted": INTERRUPTED_WRITTEN} UNFINISHED_ERROR = "起草结束时这张表还没有写完，请重试这张表" WRITE_ERROR = "写入数据目录时出错：{line}" JOB_ERROR = "起草任务出错：{line}" @@ -162,6 +171,11 @@ def task_dict(task: CatalogDraftTask) -> dict[str, Any]: "columns": task.column_count, "parts_total": task.parts_total, "parts_done": task.parts_done, + # 宽表每写出一段就并入目录：已经写进目录的段数（停止、服务重启时这些段留着） + "parts_written": task.parts_written or 0, + # 块进度：这张表实际分成的块数和已结束的块数（段被缩小、拆成两半后会变多）；不用模型时为 null + "chunks_total": task.chunks_total, + "chunks_done": task.chunks_done, "added": task.added, "updated": task.updated, "removed": task.removed, @@ -170,6 +184,8 @@ def task_dict(task: CatalogDraftTask) -> dict[str, Any]: # 模型拿不准、留空了的列（中文名和含义都没给）：几列、最多 5 个列名（按表结构的顺序）。表的状态不因此变 "blank_columns": task.blank_columns or 0, "blank_sample": list(task.blank_sample or []), + # 起草时记下的几句话，不算失败（模型提议的有效记录条件没有收下的原因：没通过校验、剖析核对排除 0% 或 100%） + "log": list(task.log or []), "error": task.error, "model_error": task.model_error, "started_at": _iso(task.started_at), @@ -237,21 +253,39 @@ async def job_brief(session: AsyncSession, job: CatalogDraftJob | None) -> dict[ async def last_drafts(session: AsyncSession, source_id: str) -> dict[str, dict[str, Any]]: - """每张表最近一次起草任务里的那一项：{表名: {status, job_id, finished_at, error, model_error, blank_columns}}。 - 任务还在进行，就是进行中的状态。只看最近 LAST_DRAFT_JOBS 个任务；每张表取哪一项在库里挑好（清单页起草时会反复取）。""" + """每张表最近一次起草任务里的那一项：{表名: {status, job_id, finished_at, error, model_error, blank_columns, + parts_written, blank_columns_from}}。任务还在进行，就是进行中的状态。只看最近 LAST_DRAFT_JOBS 个任务；每张表取哪一项 + 在库里挑好（清单页起草时会反复取）。 + + blank_columns_from：最近一次结束于 ok / partial 的用模型起草记下的留空列数（数字），没有是 null。之后的起草停止了、 + 中断了、而且没有写进新内容（parts_written 为 0）时照样带着，界面据此继续提示留空的列；之后有起草写进了新内容（停止 + 前已经写了几段、或者正在写）就是 null：以前的留空列数可能已经不对了。不用模型的起草不改模型留空的列，不算。""" recent = (select(CatalogDraftJob.id).where(CatalogDraftJob.source_id == source_id) .order_by(CatalogDraftJob.created_at.desc()).limit(LAST_DRAFT_JOBS)).scalar_subquery() - rank = func.row_number().over(partition_by=CatalogDraftTask.table_name, - order_by=(CatalogDraftJob.created_at.desc(), CatalogDraftTask.seq)).label("rank") + + def ranked() -> Any: + return func.row_number().over(partition_by=CatalogDraftTask.table_name, + order_by=(CatalogDraftJob.created_at.desc(), CatalogDraftTask.seq)).label("rank") + inner = (select(CatalogDraftTask.table_name, CatalogDraftTask.status, CatalogDraftTask.job_id, CatalogDraftTask.finished_at, CatalogDraftTask.error, CatalogDraftTask.model_error, - CatalogDraftTask.blank_columns, rank) + CatalogDraftTask.blank_columns, CatalogDraftTask.parts_written, ranked()) .join(CatalogDraftJob, CatalogDraftJob.id == CatalogDraftTask.job_id) .where(CatalogDraftJob.id.in_(recent))).subquery() rows = (await session.execute(select(inner).where(inner.c.rank == 1))).mappings() + # 每张表最近一次「写进了模型起草的内容」的用模型起草：结束于 ok / partial 的，或者写过段的（停止、中断、进行中） + wrote = (select(CatalogDraftTask.table_name, CatalogDraftTask.status, CatalogDraftTask.blank_columns, ranked()) + .join(CatalogDraftJob, CatalogDraftJob.id == CatalogDraftTask.job_id) + .where(CatalogDraftJob.id.in_(recent), CatalogDraftTask.parts_total.is_not(None), + or_(CatalogDraftTask.status.in_(("ok", "partial")), + func.coalesce(CatalogDraftTask.parts_written, 0) > 0))).subquery() + blank_from = {row["table_name"]: (row["blank_columns"] or 0) if row["status"] in ("ok", "partial") else None + for row in (await session.execute(select(wrote).where(wrote.c.rank == 1))).mappings()} return {row["table_name"]: {"status": row["status"], "job_id": row["job_id"], "finished_at": _iso(row["finished_at"]), "error": row["error"], - "model_error": row["model_error"], "blank_columns": row["blank_columns"] or 0} + "model_error": row["model_error"], "blank_columns": row["blank_columns"] or 0, + "parts_written": row["parts_written"] or 0, + "blank_columns_from": blank_from.get(row["table_name"])} for row in rows} @@ -264,12 +298,43 @@ def busy_message(row: Any) -> str: return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录，请等这次起草结束（或先停止它）再开始新的起草" +def sync_busy_message(row: Any) -> str: + return f"数据源「{getattr(row, 'name', '')}」正在起草数据目录（同步起草），请等这次起草结束再开始新的起草" + + +#: 正在同步起草（POST /catalog/draft）的数据源。进程内登记：服务端是单进程部署，多进程部署时要换成库里的锁 +_SYNC: set[str] = set() + + +def sync_drafting(source_id: str) -> bool: + """这个源正在同步起草。""" + return source_id in _SYNC + + async def ensure_idle(session: AsyncSession, row: Any) -> None: - """这个源没有排队或进行中的任务；有就抛 DraftBusy。""" + """这个源没有在起草：没有排队或进行中的任务，也没有在同步起草。有就抛 DraftBusy。建任务、重试、导入目录之前都调它。""" + if row.id in _SYNC: + raise DraftBusy(sync_busy_message(row)) if await active_job(session, row.id) is not None: raise DraftBusy(busy_message(row)) +@contextlib.asynccontextmanager +async def sync_draft(session: AsyncSession, row: Any) -> AsyncIterator[None]: + """同步起草期间登记这个源：同一个源建任务、重试、导入、再开一次同步起草都抛 DraftBusy；有排队或进行中的任务时 + 这里就抛 DraftBusy。先登记、再查库：查库要让出事件循环，先登记才不会和同时建的任务互相看不见（建任务那边写库之后 + 也再看一次，见 create_job）。""" + if row.id in _SYNC: + raise DraftBusy(sync_busy_message(row)) + _SYNC.add(row.id) + try: + if await active_job(session, row.id) is not None: + raise DraftBusy(busy_message(row)) + yield + finally: + _SYNC.discard(row.id) + + async def check_model(session: AsyncSession) -> None: """起草模型能不能用：不能用抛 catalog.CatalogModelUnavailable（只建模型对象，不发请求）。""" await catalog.resolve_draft_model(session) @@ -299,9 +364,10 @@ async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: li continue seen.add(key) columns = len((all_tables.get(key) or {}).get("columns") or []) + parts = catalog_draft.parts_for(columns) if use_model else None tasks.append(CatalogDraftTask(job_id=job.id, seq=len(tasks), table_name=key, status="pending", - column_count=columns, - parts_total=catalog_draft.parts_for(columns) if use_model else None)) + column_count=columns, parts_total=parts, parts_written=0, + chunks_total=parts, chunks_done=0 if use_model else None)) session.add(job) try: await session.flush() @@ -311,6 +377,12 @@ async def create_job(session: AsyncSession, row: Any, source: Any, *, tables: li # 部分唯一索引：另一个请求同时给这个源建了任务 await session.rollback() raise DraftBusy(busy_message(row)) from e + if row.id in _SYNC: + # 写库的时候同一个源开始了同步起草（它查库时还没看到这个任务）：这个任务不要了 + await session.execute(delete(CatalogDraftTask).where(CatalogDraftTask.job_id == job.id)) + await session.execute(delete(CatalogDraftJob).where(CatalogDraftJob.id == job.id)) + await session.commit() + raise DraftBusy(sync_busy_message(row)) return job, tasks @@ -347,8 +419,21 @@ async def cancel(session: AsyncSession, job: CatalogDraftJob) -> None: async def _close(session: AsyncSession, job_id: str, job_status: str, job_error: str | None, task_status: str, task_error: str | None) -> None: - """收尾：还没结束的表记 task_status，任务（还没结束的话）记 job_status。""" + """收尾：还没结束的表记 task_status，任务（还没结束的话）记 job_status。停止、中断时写过段的表 model_error 说清写进了 + 几段、剩几段没起草（已经写进去的段留在目录里）。""" now = utcnow() + written = _WRITTEN.get(job_status) + if written is not None: + rows = (await session.execute( + select(CatalogDraftTask.id, CatalogDraftTask.parts_written, CatalogDraftTask.parts_total) + .where(CatalogDraftTask.job_id == job_id, CatalogDraftTask.status.in_(_OPEN), + func.coalesce(CatalogDraftTask.parts_written, 0) > 0))).all() + for task_id, n, total in rows: + await session.execute( + update(CatalogDraftTask).where(CatalogDraftTask.id == task_id, CatalogDraftTask.status.in_(_OPEN)) + .values(status=task_status, error=task_error, finished_at=now, + model_error=written.format(n=n, m=max(0, int(total or n) - int(n)))) + .execution_options(synchronize_session=False)) await session.execute( update(CatalogDraftTask).where(CatalogDraftTask.job_id == job_id, CatalogDraftTask.status.in_(_OPEN)) .values(status=task_status, error=task_error, finished_at=now).execution_options(synchronize_session=False)) @@ -478,7 +563,8 @@ class _JobRun: if use_model and names: await self._write(self._set_model, model_id) hooks = catalog_draft.Hooks(call_started=self._call_started, call_done=self._call_done, - part_done=self._part_done, table_done=self._table_done) + part_done=self._part_done, table_done=self._table_done, + part_written=self._part_written, chunks_changed=self._chunks_changed) await catalog_draft.run_model_draft(model, source, names, existing=existing, background=background, hooks=hooks) else: @@ -505,6 +591,12 @@ class _JobRun: async def _table_done(self, table: str, outcome: catalog_draft.ModelOutcome | None) -> None: await self._write(self._write_table, table, outcome) + async def _part_written(self, table: str, written: catalog_draft.PartWrite) -> None: + await self._write(self._write_part, table, written) + + async def _chunks_changed(self, table: str, total: int, done: int) -> None: + await self._write(self._set_chunks, table, total, done) + # ---------------------------------------------------------------- 库操作 def _task(self, *where: Any) -> Any: @@ -554,9 +646,35 @@ class _JobRun: .values(parts_done=CatalogDraftTask.parts_done + 1)) await session.commit() + async def _set_chunks(self, session: AsyncSession, table: str, total: int, done: int) -> None: + await session.execute(self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN)) + .values(chunks_total=int(total), chunks_done=int(done))) + await session.commit() + + async def _write_part(self, session: AsyncSession, table: str, written: catalog_draft.PartWrite) -> None: + """宽表写出了一段：马上并入目录（这一次不删任何东西），记下写进去的段数、累加计数。写不进去只记日志：整张表结束 + 时收尾那次还会把它带上。""" + try: + found = await write_part(session, self.source, table, written.notes, at=self.at, actor=self.actor) + except Exception: # noqa: BLE001 - 这一段写不进去不影响起草 + logger.exception("起草任务 %s 把表 %s 写出来的一段并入数据目录时出错", self.job_id, table) + await session.rollback() + return + if found is None: + logger.warning("起草任务 %s：表 %s 的数据目录正被修改，这一段等整张表写完时再并入", self.job_id, table) + return + stats, version = found + await session.execute( + self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN)) + .values(parts_written=int(written.parts_written), version=version, + added=CatalogDraftTask.added + stats.added, updated=CatalogDraftTask.updated + stats.updated, + removed=CatalogDraftTask.removed + stats.removed)) + await session.commit() + async def _write_table(self, session: AsyncSession, table: str, outcome: catalog_draft.ModelOutcome | None) -> None: - """一张表结束：并入目录、写入，记下这张表的结果。""" + """一张表结束：并入目录、写入（宽表写过的段已经并入过，这里是整张表的收尾），记下这张表的结果。计数加在逐段写入 + 的计数上。""" try: result = await catalog.write_draft(session, self.source, table, outcome, at=self.at, actor=self.actor) except Exception as e: # noqa: BLE001 - 这一张写不进去不影响别的表 @@ -568,13 +686,20 @@ class _JobRun: error=WRITE_ERROR.format(line=first_line(e) or describe_exception(e)), model_error=outcome.error if outcome is not None else None) values: dict[str, Any] = { - "status": result.status, "added": result.added, "updated": result.updated, "removed": result.removed, - "version": result.version, "model_items": result.model_items, "error": result.error, + "status": result.status, "added": CatalogDraftTask.added + result.added, + "updated": CatalogDraftTask.updated + result.updated, "removed": CatalogDraftTask.removed + result.removed, + "model_items": result.model_items, "error": result.error, "model_error": result.model_error, "finished_at": utcnow(), "blank_columns": len(result.blank), "blank_sample": result.blank[:BLANK_SAMPLE] or None, } + if result.version: + values["version"] = result.version if outcome is not None: - values["parts_done"] = outcome.parts_done + values.update(parts_done=outcome.parts_done, chunks_total=outcome.chunks_total, + chunks_done=outcome.chunks_done, log=list(outcome.log) or None) + if not result.error: + # 收尾写进去了：写出内容的段都在目录里了。没写进去（一直撞上别人刚改过）的照库里逐段写进去的段数 + values["parts_written"] = outcome.parts_written await session.execute(self._task(CatalogDraftTask.table_name == table, CatalogDraftTask.status.in_(_OPEN)) .values(**values)) await session.commit() @@ -594,6 +719,30 @@ class _JobRun: await session.commit() +async def write_part(session: AsyncSession, source: Any, table: str, notes: dict[str, Any], *, at: str, + actor: str | None) -> tuple[catalog.MergeStats, int] | None: + """宽表写出来的一段并入目录：(计数, 写入后的版本)；撞上别人刚改过、重读重并三次仍冲突返回 None。 + + 数据库注释、外键、命名推断和这张表到这时拿到的模型起草，按 merge_notes 的规则并入（人工确认、驳回过的不动）， + **这一次不删任何东西**（covered 为空）：删掉表结构里已经没有的、这一轮没再得出的，是整张表写完时 catalog.write_draft + 收尾那一次的事。内容没变不写、不升版本。""" + structure = catalog.draft_structure(source, table) + for _attempt in range(3): + entry = await catalog.read_entry(session, source.id, table) + version = entry.version if entry else 0 + merged, stats = catalog.merge_notes(entry.notes if entry else {}, structure.notes, at=at) + merged, more = catalog.merge_notes(merged, notes, at=at) + stats += more + if entry is None and not merged: + return stats, 0 + try: + written = await catalog.write_entry(session, source.id, table, merged, if_version=version, actor=actor) + except catalog.CatalogConflict: + continue + return stats, written.version + return None + + class DraftJobs: """进程内的起草任务（照 app/scheduler 的写法：一个 asyncio 任务一件事，停的时候给宽限）。""" @@ -660,8 +809,8 @@ manager = DraftJobs() __all__ = ["ACTIVE", "BLANK_SAMPLE", "DraftBusy", "DraftFinished", "DraftJobs", "FINISHED", "GUIDANCE_MAX", - "INTERRUPTED_ERROR", "MAX_TABLES", "NOTE_MAX", "NothingToRetry", "RETRYABLE", "TASK_DONE", "TASK_STATUSES", - "TextTooLong", + "INTERRUPTED_ERROR", "INTERRUPTED_WRITTEN", "MAX_TABLES", "NOTE_MAX", "NothingToRetry", "RETRYABLE", + "STOPPED_WRITTEN", "TASK_DONE", "TASK_STATUSES", "TextTooLong", "active_job", "cancel", "check_model", "clean_text", "create_job", "ensure_idle", "guidance_of", "job_brief", "job_dict", "job_out", "last_drafts", "latest_job", "load", "manager", "retry_tables", "save_guidance", - "startup", "tasks_of"] + "startup", "sync_busy_message", "sync_draft", "sync_drafting", "tasks_of", "write_part"] diff --git a/backend/app/data/catalog_profile.py b/backend/app/data/catalog_profile.py index 6966b26d..f905945c 100644 --- a/backend/app/data/catalog_profile.py +++ b/backend/app/data/catalog_profile.py @@ -1341,8 +1341,129 @@ async def _profile(session: AsyncSession, row: Any, source: Any, tables: list[st return report -__all__ = ["CODES_MAX", "IN_CHUNK", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES", +# ========================================================================== +# 核对模型提议的有效行过滤：排除了多少行 +# ========================================================================== + +#: 有效行过滤核对的说法（写进这一项的 note；前面是模型给的依据） +FILTER_CHECKED = "剖析核对：排除 {share} 的行（共 {rows} 行）" +FILTER_SAMPLED = "剖析核对：按前 {rows} 行统计，排除 {share} 的行" +FILTER_SAMPLED_EDGE = "剖析核对：按前 {rows} 行统计，{what}，请人工核对" +FILTER_UNCHECKED = "没有核对排除比例（{why}）" +FILTER_NOT_ENABLED = "数据源没有开数据剖析" + + +@dataclass(frozen=True) +class FilterCheck: + """核对一条有效行过滤的结果。 + + keep：收不收（排除 0% 或 100% 的不收：前者什么都没筛掉，后者把所有行都筛掉了，多半写错了）；note：写进这一项 note + 的那句（剖析核对的排除比例，或者没有核对的原因）；rows / excluded：数到的行数和其中被排除的行数（没核对是 None）； + sampled：表太大或不知道多大，只数了前若干行；dropped：不收的原因（记日志用）。""" + + keep: bool + note: str + rows: int | None = None + excluded: int | None = None + sampled: bool = False + dropped: str | None = None + + +def excluded_share(excluded: int, rows: int) -> str: + """排除比例写成百分数：留一位小数（整数不带小数）；不到 0.1% 的写「不足 0.1%」，不是全部的不四舍五入成 100%。""" + if rows <= 0: + return "0%" + pct = excluded * 100 / rows + if 0 < pct < 0.1: + return "不足 0.1%" + if 99.9 < pct < 100: + return "超过 99.9%" + return f"{pct:.1f}".rstrip("0").rstrip(".") + "%" + + +def _unchecked(why: str) -> FilterCheck: + return FilterCheck(True, FILTER_UNCHECKED.format(why=why)) + + +def _unsafe_literal(value: str) -> bool: + """拼不进核对查询的字符串取值：查询层用 SQLAlchemy 的 text() 执行，它会把字符串里的「:名字」当成绑定参数； + 反斜杠在 MySQL 上读法不定、守卫会整条拒掉；控制字符不该出现（同 sql_literal 的规矩）。""" + return "\\" in value or bool(_CONTROL.search(value)) or bool(_BIND_LIKE.search(value)) + + +async def check_valid_filter(source: Any, table: str, meta: dict[str, Any] | None, condition: str, *, + settings: ProfileSettings | None = None) -> FilterCheck: + """开了数据剖析的源：数一数这条有效行过滤排除了多少行（模型起草时提议的那条，data/catalog_draft.py 调）。从不抛异常。 + + 经剖析的只读守卫（_Runner：guard.check(readonly=True)、单条时限、查询次数上限）发一条查询：只取条件里用到的列， + 先取前 max(sample_size, max_scan_rows) 行（大表不做整表扫描），再数总行数和不满足条件的行数（条件为 NULL 也算排除， + 和 WHERE 一样）。数到的不满前那么多行，就是整张表。 + + - 排除 0% 或 100% 的不收（keep 为假）；只数了前若干行的照收，写明「请人工核对」； + - 没开剖析、数据源停用、条件里有遮罩的列、视图、取值拼不进查询、查询失败或超时：照收，note 写没有核对的原因。 + """ + from app.data import sqlcheck + + options = getattr(source, "options", None) + settings = settings or profile_settings(options) + if not settings.enabled: + return _unchecked(FILTER_NOT_ENABLED) + if getattr(source, "enabled", True) is False: + return _unchecked("数据源已停用") + kind = getattr(source, "kind", None) + parsed = sqlcheck.parse_valid_filter(condition, meta, kind=kind, table=table) + if parsed.problem: + return FilterCheck(False, parsed.problem, dropped=parsed.problem) + masked = {c.lower() for c in data_engine.masked_columns(options)} + if hit := [c for c in parsed.columns if c.lower() in masked]: + return _unchecked(f"条件里的 {'、'.join(hit)} 在数据源设置中被遮罩，不取数") + if (meta or {}).get("is_view"): + return _unchecked("这张表是视图，剖析可能触发整个视图的计算") + from sqlglot import exp + + if any(n.is_string and _unsafe_literal(str(n.this)) for n in parsed.expression.find_all(exp.Literal)): + return _unchecked("条件里的取值含有冒号或反斜杠，拼不进核对查询") + try: + dialect = SqlDialect(kind) + except ValueError as e: + return _unchecked(str(e)) + cap = max(int(settings.sample_size), int(settings.max_scan_rows)) + # 和存进目录的是同一条（catalog._model_facts 存的就是这个写法）：列换成真名、按方言加引号，别的照原文 + cond = sqlcheck.canonical_valid_filter(parsed, quote=dialect.quote) + cols = ", ".join(dialect.quote(c) for c in parsed.columns) + inner = dialect.limit(f"SELECT {cols} FROM {dialect.table(meta, table)}", cap) + sql = f"SELECT COUNT(*) AS n, SUM(CASE WHEN {cond} THEN 0 ELSE 1 END) AS x FROM ({inner}) s" + run = _Runner(source, settings, dialect) + try: + res = await run(sql) + except _Skip as e: + return _unchecked(e.detail.removesuffix("，已跳过")) + except _Stop as e: + return _unchecked(_STOP_DETAIL[e.reason].format(max_queries=settings.max_queries, + max_total_s=_fmt(settings.max_total_s)).removesuffix("，未检查")) + except SnapshotTampered as e: + return _unchecked(str(e)) + row = res.rows[0] if res.rows else [None, None] + rows, excluded = _as_int(row[0]), _as_int(row[1]) if len(row) > 1 else None + if not rows: + return _unchecked("表里还没有数据") + excluded = excluded or 0 + sampled = rows >= cap + counted = f"{rows:,}" + if excluded in (0, rows): + what = "没有排除任何一行" if excluded == 0 else "所有行都被排除了" + if sampled: + return FilterCheck(True, FILTER_SAMPLED_EDGE.format(rows=counted, what=what), rows, excluded, True) + return FilterCheck(False, FILTER_CHECKED.format(share=excluded_share(excluded, rows), rows=counted), rows, + excluded, False, dropped=what) + share = excluded_share(excluded, rows) + note = (FILTER_SAMPLED if sampled else FILTER_CHECKED).format(share=share, rows=counted) + return FilterCheck(True, note, rows, excluded, sampled) + + +__all__ = ["CODES_MAX", "FILTER_CHECKED", "FILTER_NOT_ENABLED", "FILTER_SAMPLED", "FILTER_UNCHECKED", "FilterCheck", + "IN_CHUNK", "MAX_CODE_LEN", "MAX_LITERAL_LEN", "NOTE_PREFIX", "PROFILE_DEFAULT_TABLES", "PROFILE_OPTION", "PROFILE_PROBE_TABLES", "ProfileBusy", "ProfileDisabled", "ProfileReport", - "ProfileSettings", "ProfileSkip", "SqlDialect", "TableProfile", "TableSize", "ensure_enabled", - "estimate_size", "profile_catalog", "profile_settings", "profile_settings_problem", "profile_settings_rejection", - "sql_literal"] + "ProfileSettings", "ProfileSkip", "SqlDialect", "TableProfile", "TableSize", "check_valid_filter", + "ensure_enabled", "estimate_size", "excluded_share", "profile_catalog", "profile_settings", + "profile_settings_problem", "profile_settings_rejection", "sql_literal"] diff --git a/backend/app/data/sqlcheck.py b/backend/app/data/sqlcheck.py index c233d430..2a300acd 100644 --- a/backend/app/data/sqlcheck.py +++ b/backend/app/data/sqlcheck.py @@ -1131,5 +1131,274 @@ async def load_checkers_by_name(session: Any, names: Iterable[str]) -> dict[str, return await load_checkers(session, rows) -__all__ = ["CHECK_CODES", "LEVELS", "MAX_CHECKS", "MAX_SQL_CHARS", "SqlCheck", "SqlChecker", "check_sql", - "dialect_of", "graph_checks", "graph_sources", "load_checkers", "load_checkers_by_name", "tool_query"] +# ========================================================================== +# 有效行过滤（valid_filter）的校验：模型起草的条件要能进目录、能拼进剖析的核对查询 +# ========================================================================== + +#: 有效行过滤最多几个字（同目录里文本项的上限） +VALID_FILTER_MAX = 500 +#: 有效行过滤里能用的函数（其余函数、聚合、窗口一律不收） +VALID_FILTER_FUNCTIONS = ("UPPER", "LOWER", "TRIM", "COALESCE") +#: 模板占位：条件里不该有，有了也说不清最终拼出来的是什么（注释在解析之后按语法树认，字符串里的 -- 不算） +_FILTER_TEMPLATE = re.compile(r"\{\{|\{%") + +#: 不合规的说法以这一项在界面上的叫法开头（catalog.TABLE_FIELD_LABEL：「有效记录条件」）：没收下的原因会记进起草任务 +#: 逐表的日志，界面上看得到 +_FILTER_PREFIX = catalog.TABLE_FIELD_LABEL["valid_filter"] + + +@dataclass(frozen=True) +class ValidFilter: + """一段有效行过滤解析的结果。 + + expression：sqlglot 的条件（不合规时为 None）；columns：条件里用到的列（表结构里的真名，按出现的先后去重）； + names：条件里写的列名 → 表结构里的真名；problem：不合规时给人看的一句话，合规是 None；dialect：解析用的 sqlglot + 方言。text：解析的原文（去掉首尾空白）；spans：每处列引用（连同表名前缀）在原文里的位置 (起, 止（含）, 真名)，按 + 书写的先后，有一处拿不到位置时为 None（canonical_valid_filter 退回按语法树重写）。 + """ + + expression: Any = None + columns: tuple[str, ...] = () + names: Mapping[str, str] = field(default_factory=dict) + problem: str | None = None + dialect: str | None = None + text: str = "" + spans: tuple[tuple[int, int, str], ...] | None = None + + +def _bad_filter(problem: str, dialect: str | None = None) -> ValidFilter: + return ValidFilter(problem=f"{_FILTER_PREFIX}{problem}", dialect=dialect) + + +def _filter_node_problem(node: Any, *, predicate: bool) -> str | None: + """一个不在允许范围里的节点为什么不收（没有问题返回 None 的调用方不会走到这里）。""" + from sqlglot import exp + + if isinstance(node, exp.Query): + return "应为一个条件，不能是一条查询" + if any(isinstance(n, (exp.Query, exp.Subquery)) for n in node.walk()): + return "里不能有子查询" + if any(isinstance(n, exp.Window) for n in node.walk()): + return "里不能有窗口函数" + if any(isinstance(n, exp.AggFunc) for n in node.walk()): + return "里不能有聚合函数" + if isinstance(node, exp.Func): + name = str(node.this) if isinstance(node, exp.Anonymous) else node.sql_name() + return f"里只能用 {'、'.join(VALID_FILTER_FUNCTIONS)} 这几个函数，不能用 {name.upper()}" + if predicate: + return "应为一个条件（比较、IN、BETWEEN、LIKE、IS NULL，用 AND、OR、NOT 连接）" + text = node.sql() + return f"里有不支持的写法（{text[:40]}{'…' if len(text) > 40 else ''}）" + + +def _filter_value_problem(node: Any) -> str | None: + """比较两边的一个值：列、字面量、负数、允许的几个函数（参数同样是值）。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, (exp.Column, exp.Literal, exp.Null, exp.Boolean)): + return None + if isinstance(node, exp.Neg) and isinstance(_unparen(node.this), exp.Literal) and not _unparen(node.this).is_string: + return None + if isinstance(node, (exp.Upper, exp.Lower, exp.Trim, exp.Coalesce)): + for arg in node.iter_expressions(): + if problem := _filter_value_problem(arg): + return problem + return None + return _filter_node_problem(node, predicate=False) + + +def _filter_predicate_problem(node: Any) -> str | None: + """一个条件：比较、IN（只能是值的列表）、BETWEEN、LIKE、IS NULL，或者用 AND / OR / NOT 连起来的条件。""" + from sqlglot import exp + + node = _unparen(node) + if isinstance(node, (exp.And, exp.Or)): + return _filter_predicate_problem(node.left) or _filter_predicate_problem(node.right) + if isinstance(node, exp.Not): + return _filter_predicate_problem(node.this) + if isinstance(node, (exp.EQ, exp.NEQ, exp.GT, exp.GTE, exp.LT, exp.LTE)): + return _filter_value_problem(node.left) or _filter_value_problem(node.right) + if isinstance(node, exp.In): + if any(node.args.get(k) for k in ("query", "unnest", "field")): + return "里不能有子查询" + if not node.expressions: + return _filter_node_problem(node, predicate=True) + return _filter_value_problem(node.this) or next( + (p for p in (_filter_value_problem(e) for e in node.expressions) if p), None) + if isinstance(node, exp.Between): + return next((p for p in (_filter_value_problem(node.args.get(k)) for k in ("this", "low", "high")) if p), None) + if isinstance(node, exp.Like): + return _filter_value_problem(node.this) or _filter_value_problem(node.expression) + if isinstance(node, exp.Is): + if not isinstance(_unparen(node.expression), exp.Null): + return "里的 IS 只能写 IS NULL 或 IS NOT NULL" + return _filter_value_problem(node.this) + return _filter_node_problem(node, predicate=True) + + +def parse_valid_filter(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None, + table: str | None = None) -> ValidFilter: + """按数据源的方言把有效行过滤解析成一个条件，并核对它只用了这张表的列。从不抛异常，不合规写在 problem 里。 + + 规矩：一段文字、不超过 VALID_FILTER_MAX 字；不能有分号、模板占位、注释；解析出来恰好是一个条件，只由比较、IN（值的 + 列表）、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE 组成（不能有子查询、聚合、窗口、 + 别的函数、算术、参数占位）；列不带表名或带这张表的表名（table），都在表结构里，至少用到一列。 + + 列名先照原样对表结构，对不上再不分大小写对；表结构里有只差大小写的几列（PostgreSQL 上 "Status" 和 status 可以 + 并存）、写法又和哪一列都不一样时说不清是哪一列，不收。 + """ + from app.data.names import name_key as _key + + dialect = dialect_of(kind) + if not isinstance(condition, str) or not condition.strip(): + return _bad_filter("应为一段 SQL 条件", dialect) + text = condition.strip() + if len(text) > VALID_FILTER_MAX: + return _bad_filter(f"超过 {VALID_FILTER_MAX} 字", dialect) + if ";" in text: + return _bad_filter("里不能有分号", dialect) + if _FILTER_TEMPLATE.search(text): + return _bad_filter("里不能有模板占位", dialect) + if dialect is None: + return _bad_filter(f"没法校验：数据源类型「{kind}」不支持", dialect) + try: + import sqlglot + + statements = [s for s in sqlglot.parse(text, read=dialect) if s is not None] + except Exception: # noqa: BLE001 - 语法错、方言不支持的写法：一律不收 + return _bad_filter("不是一段能解析的 SQL 条件", dialect) + if len(statements) != 1: + return _bad_filter("不是一段能解析的 SQL 条件", dialect) + expression = statements[0] + if any(node.comments for node in expression.walk()): + return _bad_filter("里不能有注释", dialect) + if problem := _filter_predicate_problem(expression): + return _bad_filter(problem, dialect) + from sqlglot import exp + + real_names = [str(c.get("name")) for c in (meta or {}).get("columns") or [] + if isinstance(c, Mapping) and c.get("name") is not None] + exact = set(real_names) + by_lower: dict[str, list[str]] = {} + for name in real_names: + by_lower.setdefault(name.lower(), []).append(name) + own = {_key(str(n).rsplit(".", 1)[-1]) for n in (table, (meta or {}).get("qualified")) if n} + used: dict[str, str] = {} + spans: list[tuple[int, int, str]] | None = [] + for column in expression.find_all(exp.Column, bfs=False): # 按书写的先后 + if column.is_star: + return _bad_filter("里不能写 *", dialect) + qualifier = column.table + if column.args.get("db") or column.args.get("catalog") or (qualifier and _key(qualifier) not in own): + return _bad_filter(f"只能引用这张表的列，不能引用 {column.sql(dialect=dialect)}", dialect) + written = column.name + if written in exact: + real = written + else: + found = by_lower.get(written.lower()) or [] + if not found: + return _bad_filter(f"里的列「{written}」不在表结构里", dialect) + if len(found) > 1: + return _bad_filter(f"里的列「{written}」在表结构里有只差大小写的几列（{'、'.join(found)}），" + "要照表结构的写法写清是哪一列", dialect) + real = found[0] + used.setdefault(written, real) + span = _column_span(column, text) + spans = None if spans is None or span is None else [*spans, (*span, real)] + if not used: + return _bad_filter("没有用到这张表的任何列", dialect) + return ValidFilter(expression=expression, columns=tuple(dict.fromkeys(used.values())), names=used, + dialect=dialect, text=text, spans=tuple(spans) if spans is not None else None) + + +def _column_span(column: Any, text: str) -> tuple[int, int] | None: + """一处列引用（连同表名前缀）在原文里的位置 (起, 止（含）)：sqlglot 在标识符的 meta 里记下了它在原文里的起止。 + 拿不到、或者那一截原文里认不出这个列名时为 None。""" + ident = column.this + meta = getattr(ident, "meta", None) or {} + start, end = meta.get("start"), meta.get("end") + prefix = column.args.get("table") + if prefix is not None: + start = (getattr(prefix, "meta", None) or {}).get("start") + if not (isinstance(start, int) and isinstance(end, int) and 0 <= start <= end < len(text)): + return None + if not text[start:end + 1].rstrip("\"`]").lower().endswith(str(column.name).lower()): + return None + return start, end + + +def valid_filter_problem(condition: Any, meta: Mapping[str, Any] | None, *, kind: str | None, + table: str | None = None) -> str | None: + """有效行过滤哪里不合规（给人看的一句话）；合规返回 None。规矩见 parse_valid_filter。""" + return parse_valid_filter(condition, meta, kind=kind, table=table).problem + + +def render_valid_filter(parsed: ValidFilter, *, quoted: Any) -> str: + """把解析好的条件按数据源的方言重新写出来，列换成表结构里的真名、不带表名；quoted(真名) 为真的加引号（和剖析查询 + 其余部分的写法一致：SqlDialect.quote 要加引号的才加）。按语法树重写，NOT IN、IS NOT NULL 会写成 NOT x IN、 + NOT x IS NULL：canonical_valid_filter 拿不到原文里的位置时才退回用它。""" + from sqlglot import exp + + if parsed.expression is None: + raise ValueError(parsed.problem or "有效行过滤不合规") + + def swap(node: Any) -> Any: + if isinstance(node, exp.Column): + real = parsed.names.get(node.name, node.name) + return exp.Column(this=exp.Identifier(this=real, quoted=bool(quoted(real)))) + return node + + return parsed.expression.copy().transform(swap).sql(dialect=parsed.dialect) + + +def canonical_valid_filter(parsed: ValidFilter, *, quote: Any) -> str: + """有效行过滤存进目录、拼进剖析核对查询的写法（两处用的是同一条）：原文照旧，只把每处列引用换成 quote(真名)——表结构 + 里的真名、不带表名，按数据源的需要加引号（quote 是 catalog_profile.SqlDialect.quote，和剖析查询其余部分的写法一致）。 + + 以前目录里存的是模型的原文，核对时却按真名加了引号：PostgreSQL 上混合大小写的列（"OrderStatus"）不加引号会被折成 + 小写，存下的 orderstatus <> 'X' 一跑就报列不存在，核对过的却能跑通，SQL 检查还会让模型照抄存下的那一句。 + + 拿不到原文里的位置、换完以后认不出原来那几列时，退回按语法树重写（render_valid_filter）。""" + if parsed.expression is None: + raise ValueError(parsed.problem or "有效行过滤不合规") + if parsed.spans: + text = parsed.text + for start, end, real in sorted(parsed.spans, reverse=True): + text = text[:start] + quote(real) + text[end + 1:] + if _same_columns(text, parsed): + return text + return render_valid_filter(parsed, quoted=lambda name: quote(name) != name) + + +def _same_columns(text: str, parsed: ValidFilter) -> bool: + """换过列名的原文还是一个条件，列依次是表结构里的那几列、都不带表名。""" + import sqlglot + from sqlglot import exp + + try: + tree = sqlglot.parse_one(text, read=parsed.dialect) + except Exception: # noqa: BLE001 - 换坏了就退回按语法树重写 + return False + columns = list(tree.find_all(exp.Column, bfs=False)) + return ([c.name for c in columns] == [real for _, _, real in parsed.spans or ()] + and not any(c.table for c in columns)) + + +def stored_valid_filter(parsed: ValidFilter, kind: str | None) -> str: + """模型提议的有效行过滤存进目录的写法：canonical_valid_filter，按这个数据源的方言加引号（catalog_profile.SqlDialect： + SQLAlchemy 的写法，要加才加——PostgreSQL、SQLite 的 "OrderStatus"，MySQL 的 `OrderStatus`，Oracle 探查出来的小写名 + 不加）。""" + from app.data.catalog_profile import SqlDialect + + try: + dialect = SqlDialect(kind) + except ValueError: + return parsed.text + return canonical_valid_filter(parsed, quote=dialect.quote) + + +__all__ = ["CHECK_CODES", "LEVELS", "MAX_CHECKS", "MAX_SQL_CHARS", "SqlCheck", "SqlChecker", "VALID_FILTER_FUNCTIONS", + "VALID_FILTER_MAX", "ValidFilter", "canonical_valid_filter", "check_sql", "dialect_of", "graph_checks", + "graph_sources", "load_checkers", "load_checkers_by_name", "parse_valid_filter", "render_valid_filter", + "stored_valid_filter", "tool_query", "valid_filter_problem"] diff --git a/backend/app/db/base.py b/backend/app/db/base.py index f0da4ac6..3bc178cc 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -212,6 +212,12 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [ ("catalog_draft_jobs", "notice", "TEXT"), ("catalog_draft_tasks", "blank_columns", "INTEGER DEFAULT 0"), ("catalog_draft_tasks", "blank_sample", "JSON"), + # 数据目录闭环第一步：宽表逐段落盘的段数、块进度。存量任务回填 0 和 NULL：升级前没记这些 + ("catalog_draft_tasks", "parts_written", "INTEGER DEFAULT 0"), + ("catalog_draft_tasks", "chunks_total", "INTEGER"), + ("catalog_draft_tasks", "chunks_done", "INTEGER"), + # 复核后：逐表的日志（模型提议的有效记录条件没有收下的原因）。存量任务回填 NULL，接口给空列表 + ("catalog_draft_tasks", "log", "JSON"), ] #: 迁移补上的列要建的索引：(索引名, 表, 列)。create_all 不碰已有的表，表上新列的索引也就不会建。 diff --git a/backend/app/db/models.py b/backend/app/db/models.py index a5438d84..f341e89a 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -410,9 +410,14 @@ class CatalogDraftTask(Base): table_name: Mapped[str] = mapped_column(String(255)) status: Mapped[str] = mapped_column(String(20), default="pending") column_count: Mapped[int] = mapped_column(Integer, default=0) - # 模型分几段起草（不用模型时为 NULL）、已经结束了几段 + # 模型分几段起草（不用模型时为 NULL）、已经结束了几段、其中写出了内容并进了目录的段数（宽表每写出一段就并入目录， + # 停止、服务重启时已经写进去的留着） parts_total: Mapped[int | None] = mapped_column(Integer, default=None) parts_done: Mapped[int] = mapped_column(Integer, default=0) + parts_written: Mapped[int] = mapped_column(Integer, default=0) + # 块进度：这张表实际分成几块来问、已经结束了几块（段被缩小、拆成两半后会变多）；不用模型时为 NULL + chunks_total: Mapped[int | None] = mapped_column(Integer, default=None) + chunks_done: Mapped[int | None] = mapped_column(Integer, default=None) added: Mapped[int] = mapped_column(Integer, default=0) updated: Mapped[int] = mapped_column(Integer, default=0) removed: Mapped[int] = mapped_column(Integer, default=0) @@ -421,6 +426,8 @@ class CatalogDraftTask(Base): # 模型拿不准、留空了的列（中文名和含义都没给）：几列，最多记 5 个列名。表的状态不因此变 blank_columns: Mapped[int] = mapped_column(Integer, default=0) blank_sample: Mapped[list[Any] | None] = mapped_column(default=None) + # 起草时记下的几句话，不算失败（模型提议的有效记录条件没有收下的原因），最多 10 句 + log: Mapped[list[Any] | None] = mapped_column(default=None) # error：整张表没起草（表结构里没有、写入一直冲突、服务重启中断）；model_error：模型那部分没写出来的原因和办法 error: Mapped[str | None] = mapped_column(Text, default=None) model_error: Mapped[str | None] = mapped_column(Text, default=None) diff --git a/backend/tests/test_catalog_draft_engine.py b/backend/tests/test_catalog_draft_engine.py index 4a1a94aa..39438de0 100644 --- a/backend/tests/test_catalog_draft_engine.py +++ b/backend/tests/test_catalog_draft_engine.py @@ -254,6 +254,7 @@ async def test_hollow_shell_splits_the_pack_by_table(): async def test_splitting_stops_at_three_levels_and_five_columns(monkeypatch): monkeypatch.setattr(catalog, "MODEL_PART_MAX_CALLS", 99) + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) # 不让按比例止损先停下来 tables = {"wide40": _table(40)} model = Scripted(lambda call: EXHAUSTED) got = await D.run_model_draft(model, _source(tables), ["wide40"]) @@ -272,7 +273,7 @@ async def test_splitting_stops_at_three_levels_and_five_columns(monkeypatch): async def test_one_part_makes_at_most_eight_calls(monkeypatch): """一段（连同它拆出来的、补问的、重试的）最多调用 8 次：40 列一直用完额度，拆到第 8 次就停，剩下没问的记没写出来、 原因写清。以前一段 40 列拆到底要 15 次。""" - monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99) + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) assert catalog.MODEL_PART_MAX_CALLS == 8 model = Scripted(lambda call: EXHAUSTED) got = await D.run_model_draft(model, _source({"t40": _table(40)}), ["t40"]) @@ -755,6 +756,7 @@ async def test_a_wide_table_with_one_failed_part_is_partial_and_says_which(): async def test_reasons_name_each_part_and_say_these_parts(monkeypatch): """几段因为同一个原因没写出来：原因里写「这几段」；每段写上列的范围（列名照表结构原样）；拆过的才补「已拆小重试」。""" monkeypatch.setattr(catalog, "MODEL_SHRINK_AFTER", 99) + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) tables = {"wide95": _table(95)} ok = answering(tables) @@ -827,33 +829,38 @@ async def test_tables_are_packed_up_to_40_columns_and_4_tables(): async def test_the_same_reason_is_said_once_even_when_some_parts_were_split(monkeypatch): - """宽表的表级那段拆不开、列分段拆过：原因只说一次，拆过的在最后补一句。""" - monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99) + """宽表的表级那段拆不开、列分段拆过：原因只说一次，「已拆小重试」只跟在真拆过的那几段后面说。""" + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) monkeypatch.setattr(catalog, "MODEL_PART_MAX_CALLS", 99) model = Scripted(_hollow) got = await D.run_model_draft(model, _source({"wide45": _table(45)}), ["wide45"]) o = got["wide45"] assert o.status == "failed" assert o.error == ("这张表 45 列，分 3 段起草，第 1 段（表级的项）、第 2 段（c001 到 c023，共 23 列）、" - "第 3 段（c024 到 c045，共 22 列）都没写出来：模型只给了空壳，没有写出可用的内容；已拆小重试仍没写出") + "第 3 段（c024 到 c045，共 22 列）都没写出来：模型只给了空壳，没有写出可用的内容；" + "其中第 2、3 段已拆小重试仍没写出") def test_reasons_are_joined_by_their_base(): - """(原因, 第几段)：同样的原因只说一次；同一个原因落在几段上写「这几段」，同一段拆出来的几块仍是「这一段」。""" + """(原因, 第几段)：同样的原因只说一次；同一个原因落在几段上写「这几段」，同一段拆出来的几块仍是「这一段」。 + 「已拆小重试仍没写出」只跟在真拆过的那几段后面说：说到的段都拆过时不点名，只有一部分拆过时写「其中第几段」。""" W = D._Why ex = D.REASON_EXHAUSTED.format(budget=32768, seg=D.SEG) assert D._join([(W(D.REASON_HOLLOW), 1), (W(D.REASON_HOLLOW, split=True), 2)]) == \ - D.REASON_HOLLOW + D.REASON_SPLIT + D.REASON_HOLLOW + "；其中第 2 段已拆小重试仍没写出" assert D._join([(W(ex, split=True), 2), (W(D.REASON_HOLLOW), 1)]) == \ - f"{_exhausted()}；{D.REASON_HOLLOW}{D.REASON_SPLIT}" + f"{_exhausted()}；{D.REASON_HOLLOW}；其中第 2 段已拆小重试仍没写出" + assert D._join([(W(ex, split=True), 2), (W(ex, split=True), 3)]) == _exhausted("这几段") + D.REASON_SPLIT + assert D._join([(W(ex, split=True), 3), (W(ex), 4), (W(ex, split=True), 2)]) == \ + _exhausted("这几段") + "；其中第 2、3 段已拆小重试仍没写出" assert D._join([(W(ex), 2), (W(ex), 3)]) == _exhausted("这几段") - assert D._join([(W(ex), 2), (W(ex, split=True), 2)]) == _exhausted() + D.REASON_SPLIT + assert D._join([(W(ex), 2), (W(ex, split=True), 2)]) == _exhausted() + D.REASON_SPLIT, "同一段里有一块拆过就算拆过" assert D._join([(W("甲"), 1), (W("乙"), 1), (W("丙", split=True), 1)]) == "甲；乙", "没说出来的原因不补后缀" - halted = D.REASON_HALTED.format(n=catalog.MODEL_GIVE_UP_AFTER) + halted = D.REASON_HALTED.format(window=catalog.MODEL_HALT_WINDOW, on=catalog.MODEL_HALT_ON) + assert halted == "；最近 10 次调用有 7 次以上没写出来，已停止拆小重试，建议更换起草模型" assert D._join([(W(D.REASON_HOLLOW, halted=halted), 1), (W(D.REASON_HOLLOW), 2)]) == D.REASON_HOLLOW + halted - slow = D.REASON_SLOW.format(n=catalog.MODEL_MIN_PART_COLUMNS) - assert D._join([(W(D.REASON_HOLLOW, halted=halted), 1), (W(D.REASON_HOLLOW, halted=slow), 2), - (W(D.REASON_HOLLOW, halted=halted), 3)]) == D.REASON_HOLLOW + halted + slow, "两种止损各说一次" + assert D._join([(W(D.REASON_HOLLOW, halted=halted), 1), (W(D.REASON_HOLLOW, halted=halted), 3)]) == \ + D.REASON_HOLLOW + halted, "止损只说一次" assert D._join([(W("调用模型出错：{坏的}"), 1)]) == "调用模型出错：{坏的}", "报错原文里的花括号原样留着" @@ -865,50 +872,133 @@ def _by_table(model: Scripted) -> dict[str, int]: return out -async def test_after_two_segments_in_a_row_come_to_nothing_the_rest_are_not_split(monkeypatch): - """模型明显用不了（思考总是吃满额度）：连续 2 段拆到底仍一项没写出，后面的段只问一次、不再拆，报错说明已停止拆小重试。 - 有一段写出来了就恢复拆小。一次只起草一段（并发 1），先后是确定的。""" +HALTED = "；最近 10 次调用有 7 次以上没写出来，已停止拆小重试，建议更换起草模型" + + +async def test_when_most_recent_calls_fail_the_rest_are_asked_once_until_the_ratio_drops(monkeypatch): + """按比例止损：这次起草最近 10 次调用有 7 次以上没写出来，之后的段不再拆小、每块只问一次，报错说明已停止拆小重试； + 比例降下去（最近 10 次里少于 7 次）就恢复拆小。偶尔一次写出来不会清零：t4 到 t6 写出来了，最近 10 次里仍有 7 次没写 + 出来，照旧不拆；t7 写出来以后才降下去，t8 照常拆小。一次只起草一段（并发 1），先后是确定的。""" monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) - assert catalog.MODEL_GIVE_UP_AFTER == 2 - tables = {f"t{i}": _table(10) for i in range(1, 5)} - exhausted = Reply(text="", finish="length", thinking="……") - - model = Scripted(lambda call: exhausted) - got = await D.run_model_draft(model, _source(tables), list(tables)) - assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 1} - budget = _exhausted() - assert got["t1"].error == budget + D.REASON_SPLIT - assert got["t3"].error == budget + "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型" - + assert (catalog.MODEL_HALT_WINDOW, catalog.MODEL_HALT_ON) == (10, 7) + tables = {f"t{i}": _table(10) for i in range(1, 9)} ok = answering(tables) - model = Scripted(lambda call: ok(call) if "t3" in call.asked else exhausted) + bad = {"t1", "t2", "t3", "t8"} + + model, reports = Scripted(lambda call: EXHAUSTED if set(call.asked) & bad else ok(call)), Reports() + got = await D.run_model_draft(model, _source(tables), list(tables), hooks=reports.hooks()) + assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 1, "t5": 1, "t6": 1, "t7": 1, "t8": 3} + assert got["t1"].error == _exhausted() + D.REASON_SPLIT + assert got["t3"].error == _exhausted() + HALTED + assert {got[f"t{i}"].status for i in range(4, 8)} == {"ok"} + assert got["t8"].error == _exhausted() + D.REASON_SPLIT, "比例降下去以后照常拆小" + assert len(reports.items) == len(model.calls) + + +async def test_while_halted_a_chunk_is_not_asked_again_for_missing_columns(monkeypatch): + """止损期间每块只问一次：回复里漏了几列不补问，漏的列记没写出来、说明已停止拆小重试。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + tables = {**{f"x{i}": _table(4) for i in range(1, 8)}, "t8": _table(8)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + if "t8" not in call.asked: + raise RuntimeError("网关拒绝了这次请求") + reply = ok(call) + reply.obj["tables"][0]["columns"] = reply.obj["tables"][0]["columns"][:6] + return reply + + model = Scripted(handle) got = await D.run_model_draft(model, _source(tables), list(tables)) - assert _by_table(model) == {"t1": 3, "t2": 3, "t3": 1, "t4": 3}, "t3 写出来了，t4 照常拆小" - assert got["t3"].status == "ok" and got["t4"].error == budget + D.REASON_SPLIT + assert _by_table(model)["t8"] == 1, "漏了 c007、c008 也不补问" + o = got["t8"] + assert o.status == "partial" + assert o.error == ("这张表 8 列，有 2 列（c007、c008）没写出来：模型的回复里漏了这些列" + HALTED + + "。可以只重试这张表") + + +#: 200 列的宽表分 6 段起草（表级一段、列分段 5 段），逐段写上列的范围 +WIDE200_PARTS = ("这张表 200 列，分 6 段起草，第 1 段（表级的项）、第 2 段（c001 到 c040，共 40 列）、" + "第 3 段（c041 到 c080，共 40 列）、第 4 段（c081 到 c120，共 40 列）、第 5 段（c121 到 c160，共 40 列）、" + "第 6 段（c161 到 c200，共 40 列）") + + +async def test_a_wide_table_skips_its_remaining_parts_after_three_dead_parts_in_a_row(monkeypatch): + """同一张表连续 3 个原始段都一项没写出来：这张表剩下的段跳过不问，原因写清跳过了几段、可以换一个起草模型后只重试 + 这张表。200 列分 6 段（表级一段、列分段 5 段）。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + assert catalog.MODEL_TABLE_GIVE_UP_AFTER == 3 + tables = {"wide200": _table(200)} + + def refuse(call: Call) -> Reply: + raise RuntimeError("网关拒绝") + + model = Scripted(refuse) + got = await D.run_model_draft(model, _source(tables), ["wide200"]) + o = got["wide200"] + assert len(model.calls) == 3, "第 4 到 6 段跳过不问" + assert (o.status, o.parts_total, o.parts_done) == ("failed", 6, 6) + assert o.error == (WIDE200_PARTS + "都没写出来：调用模型出错：网关拒绝；" + "这张表接连 3 段都没写出来，剩下的 3 段已跳过，可以换一个起草模型后只重试这张表") + + # 表级那段写出来了、后面接连 3 段没写出来：只写出一部分，剩下的 2 段跳过；不再另说「可以只重试这张表」 + ok = answering(tables) + model = Scripted(lambda call: ok(call) if call.n == 0 else refuse(call)) + got = await D.run_model_draft(model, _source(tables), ["wide200"]) + o = got["wide200"] + assert len(model.calls) == 4 and o.status == "partial" + assert o.error == ("这张表 200 列，分 6 段起草，第 2 段（c001 到 c040，共 40 列）、第 3 段（c041 到 c080，共 40 列）、" + "第 4 段（c081 到 c120，共 40 列）、第 5 段（c121 到 c160，共 40 列）、第 6 段（c161 到 c200，共 40 列）" + "没写出来：调用模型出错：网关拒绝；这张表接连 3 段都没写出来，剩下的 2 段已跳过，" + "可以换一个起草模型后只重试这张表") + + +async def test_a_part_that_writes_resets_the_dead_parts_of_its_table(monkeypatch): + """中间有一段写出来了，就不算「接连」：200 列的第 2、3、5、6 段没写出来、第 4 段写出来了，每段都照常问。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + tables = {"wide200": _table(200)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + part, cols = call.asked["wide200"] + if part == "table" or "c081" in (cols or []): + return ok(call) + raise RuntimeError("网关拒绝") + + model = Scripted(handle) + got = await D.run_model_draft(model, _source(tables), ["wide200"]) + assert len(model.calls) == 6 and got["wide200"].status == "partial" + assert "已跳过" not in got["wide200"].error async def test_the_column_parts_of_a_wide_table_stop_splitting_too(monkeypatch): - """宽表的列分段一段一段排队起草（一段拆到底再起草下一段），止损在宽表里同样起作用。 - - 95 列：表级那段拆不开（结构化、纯文本各一次）；第 2 段 32 列拆小、到 8 次封顶；这时已经连续 2 段没写出来，第 3、4 段 - 只问一次、换纯文本一次。""" + """按比例止损在宽表里同样起作用，接连没写出来的宽表剩下的段跳过：95 列一直只给空壳（已经走纯文本），第 2 段拆小到 + 止损；之后的第 3 段只问一次；接连 3 段没写出来，第 4 段跳过。""" monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) - model = Scripted(_hollow) + model = Scripted(_text_only(_hollow)) got = await D.run_model_draft(model, _source({"wide95": _table(95)}), ["wide95"]) - assert len(model.calls) == 2 + 8 + 2 + 2 - assert got["wide95"].error == (WIDE95_PARTS + "都没写出来：" + D.REASON_HOLLOW - + "；每段最多调用 8 次模型，这一段到了次数仍没写完，已停止重试" - + D.REASON_SPLIT + "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型") + o = got["wide95"] + asked = [c for c in model.calls if c.mode == "text"] + assert len(asked) <= 2 + 8 + 1 + assert not any("c065" in (c.asked["wide95"][1] or []) for c in asked), "第 4 段没问" + assert o.status == "failed" and o.error.startswith(WIDE95_PARTS + "都没写出来：") + assert HALTED in o.error + assert o.error.endswith("这张表接连 3 段都没写出来，剩下的 1 段已跳过，可以换一个起草模型后只重试这张表") async def test_a_wide_table_that_never_comes_back_is_bounded_by_the_first_wave(): - """默认并发（3 段同时起草）：294 列（8 个列分段）一直只给空壳，最多是表级那段 2 次、头 3 个列分段各 8 次（封顶）、 - 其余 5 段各 2 次。以前所有列分段一下子都排上去、一层一层地拆，要 168 次。""" + """默认并发（3 段同时起草）：294 列（8 个列分段）一直只给空壳：表级那段 2 次，头一批同时起草的列分段拆小到止损或 + 封顶；接连 3 段没写出来以后，没开始的段跳过不问。以前所有列分段一下子都排上去、一层一层地拆，要 168 次。""" model = Scripted(_hollow) got = await D.run_model_draft(model, _source({"wide294": _table(294)}), ["wide294"]) - assert got["wide294"].status == "failed" and got["wide294"].parts_total == 9 - assert len(model.calls) <= 2 + 3 * 8 + 5 * 2 + o = got["wide294"] + assert o.status == "failed" and o.parts_total == 9 and o.parts_done == 9 + assert len(model.calls) <= 2 + 4 * 8 + assert "已跳过" in o.error + asked = {c for call in model.calls for c in call.asked["wide294"][1] or []} + assert "c294" not in asked, "最后一段没问" async def test_a_big_draft_does_not_hold_the_model_ahead_of_a_later_one(monkeypatch): @@ -1042,7 +1132,7 @@ async def test_the_chunks_of_a_part_cut_by_the_smaller_cap_share_its_eight_calls """每个原始段（连同段大小减半后按上限分开的各块、拆出来的、补问的、重试的）一共最多调用 8 次。以前分开的每一块各有 8 次：上限缩到 10 列以后一直只给空壳，4 张窄表拼成的 40 列段调用了 20 次，单张 40 列的表也是 20 次。""" monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) - monkeypatch.setattr(catalog, "MODEL_GIVE_UP_AFTER", 99) # 不让止损先停下来 + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) # 不让止损先停下来 tables = {"w40": _table(40), **{f"n{i}": _table(10) for i in range(1, 5)}, "x40": _table(40)} def handle(call: Call) -> Reply: @@ -1085,34 +1175,31 @@ class AlwaysSlow: raise TimeoutError() -SLOW_STOP = "；拆到每段 10 列仍然超时，已停止拆小重试，建议更换起草模型" - - -async def test_a_model_that_always_times_out_stops_after_three_rounds_not_seven(): - """每次调用都超时、和段大小无关：超时拆出来的两半同时问（不再等带表级项的前一半写完才问后一半），拆到每段 10 列的 - 调用接连超时 2 次就停止拆小，拆出来还没问的块不再问。40 列的表 3 轮、6 次调用就结束（时限 420 秒时约 21 分钟）； - 以前 40→20→10→5 一层一层串着问，8 次调用约 7 轮（约 49 分钟）。""" +async def test_a_model_that_always_times_out_is_bounded_by_the_part_cap(): + """每次调用都超时：超时拆出来的两半同时问，最多拆到这一段的调用次数用完（8 次）或止损。以前 40→20→10→5 一层一层串着问， + 8 次调用约 7 轮；现在两半同时问，至多 4 轮。""" tables = {"t40": _table(40)} slow = AlwaysSlow(tables) model, reports = Scripted(slow), Reports() got = await D.run_model_draft(model, _source(tables), ["t40"], hooks=reports.hooks()) - assert [_width(c, tables) for c in model.calls] == [40, 20, 20, 10, 10, 10], "拆到每段 10 列仍超时就不再往下拆" - assert slow.rounds == [1, 2, 2, 3, 3, 3], "两半同时问" - assert reports.failures == ["timeout"] * 6 and len(reports.items) == len(model.calls) + assert len(model.calls) <= catalog.MODEL_PART_MAX_CALLS and max(slow.rounds) <= 4 + assert [_width(c, tables) for c in model.calls[:3]] == [40, 20, 20], "两半同时问" + assert set(reports.failures) == {"timeout"} and len(reports.items) == len(model.calls) o = got["t40"] - assert o.status == "failed" and o.error == "模型 420 秒内没有写完这一段" + D.REASON_SPLIT + SLOW_STOP + assert o.status == "failed" and "模型 420 秒内没有写完这一段" + D.REASON_SPLIT in o.error + assert o.chunks_done == o.chunks_total > 1, "拆出来的块都结束了" -async def test_three_parts_that_always_time_out_end_in_four_rounds(): - """首轮同时起草的 3 段都一直超时：拆到每段 10 列仍超时就一起停下，不再各自拆满 8 次。12 次调用、4 轮；以前 24 次 - 调用、约 8 轮（止损要等一整段结束才计数，首轮的 3 段都跑满）。""" +async def test_three_parts_that_always_time_out_stop_together(): + """首轮同时起草的 3 段都一直超时：最近 10 次调用 7 次以上没写出来以后，拆出来还没问的块不再问，不再各自拆满 8 次。 + 以前（接连超时的止损）12 次调用；更早要 24 次。""" tables = {f"t{i}": _table(40) for i in range(1, 4)} slow = AlwaysSlow(tables) model = Scripted(slow) got = await D.run_model_draft(model, _source(tables), list(tables)) - assert len(model.calls) == 12 and max(slow.rounds) == 4 + assert len(model.calls) <= 9 and max(slow.rounds) <= 4 assert {o.status for o in got.values()} == {"failed"} - assert all(o.error.endswith(SLOW_STOP) for o in got.values()) + assert any(HALTED in o.error for o in got.values()) async def test_timeouts_that_go_away_at_ten_columns_are_still_split_down_and_written(): @@ -1152,7 +1239,7 @@ async def test_a_call_whose_reply_cannot_be_read_still_counts(monkeypatch): """读回复时出了意料之外的错：这张表记出错，这次调用照样记进用量（算一次出错），调用次数和用量对得上。""" tables = {"t4": _table(4)} - def broken(entry, meta, *, system): + def broken(entry, meta, *, system, **_): raise RuntimeError("校验时出了意外") monkeypatch.setattr(catalog, "_from_model", broken) @@ -1190,24 +1277,54 @@ async def test_a_wide_table_that_wrote_nothing_names_every_part(): got = await D.run_model_draft(Scripted(handle), _source({"wide95": _table(95)}), ["wide95"]) o = got["wide95"] - assert o.status == "failed" and o.error == WIDE95_PARTS + "都没写出来：调用模型出错：网关拒绝" + assert o.status == "failed" and o.error == (WIDE95_PARTS + "都没写出来：调用模型出错：网关拒绝；" + "这张表接连 3 段都没写出来，剩下的 1 段已跳过，可以换一个起草模型后只重试这张表") narrow = await D.run_model_draft(Scripted(handle), _source({"t8": _table(8)}), ["t8"]) assert narrow["t8"].error == "调用模型出错：网关拒绝", "只有一段的表：这一段就是整张表，照旧只写原因" async def test_a_halted_part_that_could_be_split_is_not_asked_again_in_text_mode(monkeypatch): - """止损以后本来能拆小的段只给空壳：已经走纯文本的不再原样再问（约定只对拆不开的段再问一次）；结构化输出的照旧换 - 纯文本问一次，看看是不是结构化输出的问题（见 test_the_column_parts_of_a_wide_table_stop_splitting_too）。""" + """止损以后每块只问一次：已经走纯文本的空壳不再原样再问、也不拆小。t1：结构化不支持 1 次（只管通信的重来，不进 + 止损的窗口）、纯文本 1 次、拆成两半各问一次再原样再问一次（5 次没写出来）；t2 的第一次是第 6 次，照常拆小，前一半 + 是第 7 次没写出来，之后止损：后一半不再问；t3 只问一次、不拆小。""" monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) tables = {f"t{i}": _table(10) for i in range(1, 5)} model = Scripted(_text_only(_hollow)) got = await D.run_model_draft(model, _source(tables), list(tables)) - # t1：结构化不支持 1 次、纯文本 1 次、拆成两半各问一次再原样再问一次；t2 同样（少了结构化那次）；之后止损 - assert _by_table(model) == {"t1": 6, "t2": 5, "t3": 1, "t4": 1} - assert got["t3"].error == (catalog.MODEL_EMPTY_REASON - + "；模型连续 2 段都没写出来，已停止拆小重试，建议更换起草模型") + assert _by_table(model) == {"t1": 6, "t2": 2, "t3": 1, "t4": 1} + assert got["t2"].error == D.REASON_HOLLOW + D.REASON_SPLIT + HALTED + assert got["t3"].error == catalog.MODEL_EMPTY_REASON + HALTED + + +async def test_while_halted_structured_output_still_gets_one_probe_in_text(monkeypatch): + """止损期间结构化输出只给空壳：这次起草还没换纯文本问过的，照旧换纯文本问一次（看看是不是结构化输出的问题）；纯文本 + 写出来了，后面都走纯文本。问过一次（纯文本也只给空壳）就不再问。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + tables = {**{f"x{i}": _table(4) for i in range(1, 8)}, "t8": _table(8), "t9": _table(8)} + ok = answering(tables) + + def handle(text_works: bool) -> Callable[[Call], Reply]: + def go(call: Call) -> Reply: + if next(iter(call.asked)).startswith("x"): + raise RuntimeError("网关拒绝了这次请求") + if call.mode == "structured": + return _hollow(call) + return ok(call) if text_works else _hollow(call) + return go + + model = Scripted(handle(True)) + got = await D.run_model_draft(model, _source(tables), list(tables)) + assert [c.mode for c in model.calls if "x1" not in c.asked and not next(iter(c.asked)).startswith("x")] == \ + ["structured", "text", "text"] + assert got["t8"].status == got["t9"].status == "ok" + + model = Scripted(handle(False)) + got = await D.run_model_draft(model, _source(tables), list(tables)) + assert _by_table(model)["t8"] == 2 and _by_table(model)["t9"] == 1, "纯文本问过一次，不再问" + assert got["t9"].error == catalog.MODEL_EMPTY_REASON + HALTED # -------------------------------------------------------------------------- @@ -1299,3 +1416,224 @@ def test_cut_values_are_dropped_and_complete_ones_kept(): @pytest.mark.parametrize("text", ["好的，我来写", "{坏的 json}", ""]) def test_not_json_is_not_a_reply(text): assert D._parse_text(text, cut=False) is None + + +# -------------------------------------------------------------------------- +# 闭环第一步的起草遗留：逐段交出去落盘、块进度、「已拆小重试」只说拆过的段 +# -------------------------------------------------------------------------- + + +class Written: + """进度回调 part_written 收到的：(表名, 已写出内容的段数, 这时这张表拿到的列)；chunks_changed 收到的块进度。""" + + def __init__(self) -> None: + self.parts: list[tuple[str, int, set[str], bool]] = [] + self.chunks: dict[str, list[tuple[int, int]]] = {} + + async def part_written(self, table: str, written: D.PartWrite) -> None: + cols = set((written.notes.get("columns") or {})) + self.parts.append((table, written.parts_written, cols, "label" in written.notes)) + + async def chunks_changed(self, table: str, total: int, done: int) -> None: + self.chunks.setdefault(table, []).append((total, done)) + + def hooks(self) -> D.Hooks: + return D.Hooks(part_written=self.part_written, chunks_changed=self.chunks_changed) + + +async def test_each_part_of_a_wide_table_is_handed_over_as_soon_as_it_is_written(monkeypatch): + """宽表每写出一段就交给调用方并入目录（最后一段由 table_done 收尾）：交出去的是这张表到这时拿到的全部内容。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + tables = {"wide95": _table(95)} + seen = Written() + got = await D.run_model_draft(Scripted(answering(tables)), _source(tables), ["wide95"], hooks=seen.hooks()) + o = got["wide95"] + assert o.status == "ok" and o.parts_written == 4 + assert [(t, n, len(cols), label) for t, n, cols, label in seen.parts] == [ + ("wide95", 1, 0, True), ("wide95", 2, 32, True), ("wide95", 3, 64, True)] + + def handle(call: Call) -> Reply: + part, cols = call.asked["wide95"] + if part == "columns" and "c033" in cols: + raise RuntimeError("网关拒绝") + return answering(tables)(call) + + seen = Written() + got = await D.run_model_draft(Scripted(handle), _source(tables), ["wide95"], hooks=seen.hooks()) + assert got["wide95"].status == "partial" and got["wide95"].parts_written == 3 + assert [n for _, n, _, _ in seen.parts] == [1, 2], "没写出来的那段不交" + + +async def test_a_narrow_table_is_handed_over_only_once_at_the_end(): + tables = {"t8": _table(8)} + seen = Written() + got = await D.run_model_draft(Scripted(answering(tables)), _source(tables), ["t8"], hooks=seen.hooks()) + assert seen.parts == [] and got["t8"].parts_written == 1 + failed = await D.run_model_draft(Scripted(lambda call: EXHAUSTED), _source({"t4": _table(4)}), ["t4"]) + assert failed["t4"].parts_written == 0 + + +async def test_chunks_count_what_the_table_is_actually_asked_in(monkeypatch): + """块进度：一开始是这张表分的段数；一段被拆成两半多一块、被缩小的上限分开成几块就多几块；一块结束算一块。 + 多表拼成的段按表拆开时，每张表还是一块。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + monkeypatch.setattr(catalog, "MODEL_SHRINK_AFTER", 1) + tables = {"a12": _table(12), "b40": _table(40)} + ok = answering(tables) + seen = Written() + got = await D.run_model_draft(Scripted(lambda call: EXHAUSTED if call.n == 0 else ok(call)), _source(tables), + ["a12", "b40"], hooks=seen.hooks()) + a, b = got["a12"], got["b40"] + assert a.status == b.status == "ok" + assert (a.chunks_total, a.chunks_done) == (2, 2), "额度用完拆成两半" + assert (b.chunks_total, b.chunks_done) == (2, 2), "上限缩到 20 列，40 列的段分开成两块" + assert seen.chunks["a12"][-1] == (2, 2) and seen.chunks["b40"][-1] == (2, 2) + assert all(done <= total for pairs in seen.chunks.values() for total, done in pairs) + + wide = {"wide95": _table(95)} + got = await D.run_model_draft(Scripted(answering(wide)), _source(wide), ["wide95"]) + assert (got["wide95"].chunks_total, got["wide95"].chunks_done) == (4, 4) + + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 4) + packed = {"p6": _table(6), "q6": _table(6)} + okp = answering(packed) + model = Scripted(lambda call: _hollow(call) if call.n == 0 else okp(call)) + got = await D.run_model_draft(model, _source(packed), ["p6", "q6"]) + assert [set(c.asked) for c in model.calls] == [{"p6", "q6"}, {"p6"}, {"q6"}], "按表拆开" + assert {(o.chunks_total, o.chunks_done) for o in got.values()} == {(1, 1)} + + +async def test_only_the_parts_that_were_split_say_so(monkeypatch): + """一张表几段没写出来、只有一部分真拆小重试过：「已拆小重试仍没写出」只跟在拆过的那几段后面说。95 列：第 3 段一直用完 + 额度（拆小重试过），第 4 段调用出错（没拆过）。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_SHRINK_AFTER", 99) + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 99) + tables = {"wide95": _table(95)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + part, cols = call.asked["wide95"] + if part == "columns" and cols[0] >= "c065": + raise RuntimeError("网关拒绝") + if part == "columns" and cols[0] >= "c033": + return EXHAUSTED + return ok(call) + + got = await D.run_model_draft(Scripted(handle), _source(tables), ["wide95"]) + o = got["wide95"] + assert o.status == "partial" + assert o.error == ("这张表 95 列，分 4 段起草，第 3 段（c033 到 c064，共 32 列）、第 4 段（c065 到 c095，共 31 列）没写出来：" + + _exhausted() + "；调用模型出错：网关拒绝；其中第 3 段已拆小重试仍没写出。可以只重试这张表") + + +# -------------------------------------------------------------------------- +# 复核后的修正：止损的窗口只记每次调用的最终结局。通信层的重来（服务方嫌额度太大退回 8192、结构化输出不支持或回了 +# 说明文字改走纯文本、429 / 5xx 退避重试）不花额度，也说明不了模型写不出来：照旧报给进度回调（用量、提醒照算）， +# 不进止损的窗口。以前都记成没写出来，OpenAI 兼容网关（额度上限 8192、不支持 json_schema）上并发的头几次调用全是 +# 重来，第一次额度用完就凑满 10 次里 7 次，那一段直接记没写出来、不再拆小。 +# -------------------------------------------------------------------------- + + +def _six_tables() -> dict: + """6 张 40 列的表：每张自成一段，默认并发 3 段同时起草。""" + return {f"t{i}": _table(40, prefix=f"t{i}_") for i in range(6)} + + +def _halved(model: Scripted, table: str) -> bool: + """这张表被拆成两半问过（表级的项加前 20 列那一半）。""" + return any(part == "table+columns" and len(cols or []) == 20 + for call in model.calls for name, (part, cols) in call.asked.items() if name == table) + + +async def test_budget_and_structured_output_fallbacks_do_not_halt_a_concurrent_draft(): + tables = _six_tables() + ok = answering(tables) + spent: set[str] = set() + + async def refuse(call: Call) -> Reply: + await asyncio.sleep(0.01) + raise _Status(400, "max_tokens: 32768 > 8192, which is the maximum allowed number of output tokens") + + async def small(call: Call) -> Reply: + await asyncio.sleep(0.01) + if call.mode == "structured": + raise _Status(400, "This response_format type is unavailable now") + if "t0" in call.asked and "t0" not in spent: + spent.add("t0") + return EXHAUSTED + return ok(call) + + big, sm, reports = Scripted(refuse), Scripted(small), Reports() + got = await D.run_model_draft(catalog.DraftModel(big, sm, "m"), _source(tables), list(tables), + hooks=reports.hooks()) + assert _status(got) == dict.fromkeys(tables, "ok"), {t: o.error for t, o in got.items()} + assert _halved(sm, "t0"), "额度用完的那一段照常拆小重试" + assert len(big.calls) == 3 and reports.failures.count("error") == 6, "退回小额度、改走纯文本的重来照旧报给进度回调" + assert reports.failures.count("exhausted") == 1 + + +async def test_rate_limit_retries_do_not_halt_the_draft(): + tables = _six_tables() + ok = answering(tables) + limited: dict[str, int] = {} + spent: set[str] = set() + + async def handle(call: Call) -> Reply: + await asyncio.sleep(0.01) + name = next(iter(call.asked)) + if name in ("t0", "t1", "t2") and limited.get(name, 0) < 2: + limited[name] = limited.get(name, 0) + 1 + raise _Status(429, "Too Many Requests") + if name == "t0" and not spent: + spent.add(name) + return EXHAUSTED + return ok(call) + + model, reports = Scripted(handle), Reports() + got = await D.run_model_draft(model, _source(tables), list(tables), hooks=reports.hooks()) + assert _status(got) == dict.fromkeys(tables, "ok"), {t: o.error for t, o in got.items()} + assert _halved(model, "t0"), "额度用完的那一段照常拆小重试" + assert reports.failures.count("error") == 6, "429 之后重试成功的那几次照旧报给进度回调" + + +async def test_structured_output_that_comes_back_as_prose_does_not_count_toward_halting(monkeypatch): + """结构化输出回了说明文字、这一块改走纯文本再问：也是通信方式的问题，那一次不进止损的窗口（止损门槛调低到 3 次， + 并发的 3 块都回说明文字也不止损）；纯文本写出来的、没写出来的照常算。""" + monkeypatch.setattr(catalog, "MODEL_HALT_ON", 3) + tables = _six_tables() + ok = answering(tables) + spent: set[str] = set() + + async def handle(call: Call) -> Reply: + await asyncio.sleep(0.01) + if call.mode == "structured": + return Reply(text="好的。这张表记录的是一笔笔业务，各列的含义如下：第一列是编号……") + if "t0" in call.asked and "t0" not in spent: + spent.add("t0") + return EXHAUSTED + return ok(call) + + model, reports = Scripted(handle), Reports() + got = await D.run_model_draft(model, _source(tables), list(tables), hooks=reports.hooks()) + assert _status(got) == dict.fromkeys(tables, "ok"), {t: o.error for t, o in got.items()} + assert _halved(model, "t0") + assert reports.failures.count("error") == 3, "回了说明文字的那几次照旧报给进度回调" + + +async def test_a_call_that_still_fails_after_its_retries_counts_toward_halting(monkeypatch): + """重试用完仍然出错的那一次是这次调用的最终结局，照常进止损的窗口：一直 503 的网关，止损照样起作用。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) + monkeypatch.setattr(catalog, "MODEL_BATCH_TABLES", 1) + tables = {f"t{i}": _table(10) for i in range(1, 9)} + ok = answering(tables) + + def handle(call: Call) -> Reply: + if set(call.asked) & {"t1", "t2", "t3", "t4", "t5", "t6", "t7"}: + raise _Status(503, "Service Unavailable") + return EXHAUSTED if "t8" in call.asked else ok(call) + + got = await D.run_model_draft(Scripted(handle), _source(tables), list(tables)) + assert got["t7"].error == "调用模型出错：Service Unavailable（已重试 2 次）" + assert got["t8"].error == _exhausted() + HALTED, "最近 10 次的最终结局有 7 次以上没写出来：t8 不再拆小" diff --git a/backend/tests/test_catalog_draft_facts.py b/backend/tests/test_catalog_draft_facts.py new file mode 100644 index 00000000..37f3705b --- /dev/null +++ b/backend/tests/test_catalog_draft_facts.py @@ -0,0 +1,498 @@ +"""防错事实的提议：模型起草时提议有效行过滤（valid_filter）、去重规则（dedup），附一句依据。 + +- 提示词写清两项只在表结构、注释、码值能看出来时才写；结构化输出、纯文本两条路都认这两项； +- 有效行过滤按数据源的方言用 sqlglot 解析成一个条件：只能引用这张表的列，不能有子查询、聚合、窗口函数、分号，只允许 + 比较、IN、BETWEEN、LIKE、IS NULL、AND / OR / NOT 和 UPPER / LOWER / TRIM / COALESCE，不超过 500 字。不过就丢掉； + 存的是整理过的写法：原文照旧，列换成表结构里的真名、不带表名、按方言加引号（剖析核对的也是这一条）； +- 开了数据剖析、能连上的源：经剖析的只读守卫数出排除比例，排除 0% 或 100% 的不收，收下的在 note 里写剖析核对的结果； + 没开剖析、连不上的照收，note 写没有核对的原因； +- 没收下的（没通过校验、排除 0% 或 100%）不算失败，在这张表的日志里记一句（起草任务逐表的 log）； +- 开了剖析的源，目录里记下的码值交给模型做背景（遮罩的列不给；「已确认的目录项」里遮罩列的码值也不给）； +- 这两项照旧记模型起草 / 推断；SQL 检查只在人工确认、有确证时才把漏了有效行过滤报成警告，推断的只是提示。 + +用合成的景区业务库（tests/fixtures/catalog/scenic.py）和虚构的表结构，不调用真实模型。 +""" +from __future__ import annotations + +import logging +import sqlite3 +from types import SimpleNamespace + +import pytest + +from app.data import catalog, catalog_profile, sqlcheck +from app.data import catalog_draft as D +from app.data.catalog import make_item +from app.data.catalog_profile import PROFILE_OPTION +from app.data.engine import MASK_COLUMNS_OPTION, engines +from app.data.introspect import introspect +from tests.fixtures.catalog.fake_model import FakeModel, answer, requested + +# -------------------------------------------------------------------------- +# 夹具 +# -------------------------------------------------------------------------- + + +def _col(name: str, type_: str = "INTEGER", comment: str | None = None) -> dict: + return {"name": name, "type": type_, "nullable": True, "comment": comment} + + +def _orders() -> dict: + return {"qualified": "orders", "schema": None, "is_view": False, "primary_key": ["id"], "foreign_keys": [], + "comment": None, + "columns": [_col("id"), _col("order_no", "TEXT"), _col("status", "TEXT", comment="C 作废，X 测试单"), + _col("amount", "REAL"), _col("updated_at", "TEXT")]} + + +def _plain(tables: dict, **kw) -> SimpleNamespace: + """不连库的源（没有 options：没开数据剖析）。""" + return SimpleNamespace(id="facts-src", name="facts", kind=kw.pop("kind", "sqlite"), origin="manual", + description="", schema_cache={"tables": tables}, **kw) + + +_CACHE: dict[str, dict] = {} + + +async def _scenic_cache(path: str) -> dict: + if path not in _CACHE: + probe = SimpleNamespace(id="facts-scenic-probe", name="scenic", kind="sqlite", database=path, host=None, + port=None, username=None, password=None, options={}, readonly=True, description="", + schema_cache={}, origin="manual") + try: + _CACHE[path] = await introspect(probe) + finally: + await engines.invalidate(probe.id) + return _CACHE[path] + + +@pytest.fixture +async def scenic(scenic_db): + """接上景区库的源（开了数据剖析）。用完作废连接池。""" + made: list[str] = [] + + async def make(*, options: dict | None = None, database: str | None = None, enabled: bool = True): + sid = f"facts-{len(made)}-{id(made)}" + made.append(sid) + opts = {PROFILE_OPTION: {"enabled": True}} if options is None else options + return SimpleNamespace(id=sid, name="scenic", kind="sqlite", database=database or scenic_db, host=None, + port=None, username=None, password=None, options=opts, readonly=True, description="", + schema_cache=await _scenic_cache(scenic_db), origin="manual", enabled=enabled) + + yield make + for sid in made: + await engines.invalidate(sid) + + +def _count(path: str, where: str) -> tuple[int, int]: + """(全表行数, 不满足条件的行数)：直接用 sqlite3 数，核对剖析算出来的数。""" + db = sqlite3.connect(path) + try: + total = db.execute("SELECT COUNT(*) FROM visits").fetchone()[0] + kept = db.execute(f"SELECT COUNT(*) FROM visits WHERE {where}").fetchone()[0] + finally: + db.close() + return total, total - kept + + +# -------------------------------------------------------------------------- +# 校验有效行过滤 +# -------------------------------------------------------------------------- + +META = _orders() + + +@pytest.mark.parametrize("condition, columns", [ + ("status NOT IN ('C', 'X')", ["status"]), + ("STATUS NOT IN ('C', 'X')", ["status"]), # 列名不分大小写，记表结构里的写法 + ("orders.status <> 'C'", ["status"]), # 带这张表的表名 + ("status IS NOT NULL AND amount > 0", ["status", "amount"]), + ("NOT (status = 'C' OR status = 'X')", ["status"]), + ("amount BETWEEN 0 AND 100000", ["amount"]), + ("order_no NOT LIKE 'TEST%'", ["order_no"]), + ("UPPER(TRIM(status)) <> 'C'", ["status"]), + ("COALESCE(status, '') <> 'X' AND amount >= -1", ["status", "amount"]), + ("status = 'C' OR updated_at IS NULL", ["status", "updated_at"]), +]) +def test_valid_filters_that_pass(condition, columns): + parsed = sqlcheck.parse_valid_filter(condition, META, kind="sqlite", table="orders") + assert parsed.problem is None, parsed.problem + assert list(parsed.columns) == columns + assert sqlcheck.valid_filter_problem(condition, META, kind="mysql", table="orders") is None + + +@pytest.mark.parametrize("condition, said", [ + ("status = 'C'; DROP TABLE orders", "分号"), + ("status IN (SELECT code FROM codes)", "子查询"), + ("EXISTS (SELECT 1 FROM refunds)", "子查询"), + ("COUNT(status) > 1", "聚合"), + ("ROW_NUMBER() OVER (ORDER BY id) = 1", "窗口"), + ("LENGTH(status) > 0", "LENGTH"), + ("amount * 2 > 10", "不支持的写法"), + ("payments.status = 1", "只能引用这张表的列"), + ("nope = 1", "nope"), + ("1 = 1", "没有用到这张表的任何列"), + ("status", "一个条件"), + ("status = ?", "不支持的写法"), + ("status = 'C' -- 注释", "注释"), + ("status = '{{ input.code }}'", "模板"), + ("status = '" + "C" * 600 + "'", "500"), + ("status = (", "解析"), + ("status = 'C' UNION SELECT 1", "一个条件"), +]) +def test_valid_filters_that_are_dropped(condition, said): + problem = sqlcheck.valid_filter_problem(condition, META, kind="sqlite", table="orders") + assert problem is not None and said in problem, problem + + +def test_a_source_kind_without_a_dialect_cannot_be_checked(): + assert "不支持" in sqlcheck.valid_filter_problem("status = 'C'", META, kind="mssql", table="orders") + + +def test_the_condition_is_rendered_with_the_columns_quoted_like_the_source(): + meta = {**META, "columns": [*META["columns"], _col("isTest")]} + parsed = sqlcheck.parse_valid_filter("istest = 0 AND STATUS <> 'C'", meta, kind="mysql", table="orders") + assert parsed.problem is None + rendered = sqlcheck.render_valid_filter(parsed, quoted=lambda name: name != name.lower()) + assert rendered == "`isTest` = 0 AND status <> 'C'" + + +#: 带一列混合大小写列名的订单表（ORM 建的 PostgreSQL 库里常见） +MIXED = {**META, "columns": [*META["columns"], _col("OrderStatus", "TEXT")]} + + +@pytest.mark.parametrize("kind, condition, stored", [ + # PostgreSQL 把没加引号的名字折成小写：混合大小写的列一律加引号，不论模型写的大小写对不对 + ("postgresql", "orderstatus <> 'X' AND STATUS NOT IN ('C', 'X')", + "\"OrderStatus\" <> 'X' AND status NOT IN ('C', 'X')"), + ("postgresql", "OrderStatus <> 'X'", "\"OrderStatus\" <> 'X'"), + ("postgresql", "orders.\"OrderStatus\" IS NOT NULL", "\"OrderStatus\" IS NOT NULL"), + ("mysql", "orderstatus <> 'X'", "`OrderStatus` <> 'X'"), + ("oracle", "ORDERS.STATUS <> 'C'", "status <> 'C'"), + # 不用改的照原文：NOT IN、IS NOT NULL 这些写法不被改写 + ("sqlite", "status NOT IN ('C', 'X') AND updated_at IS NOT NULL", + "status NOT IN ('C', 'X') AND updated_at IS NOT NULL"), +]) +def test_the_stored_filter_names_its_columns_the_way_the_source_needs(kind, condition, stored): + """存进目录的有效记录条件：原文照旧，只把列换成表结构里的真名、不带表名，按数据源的需要加引号（和剖析核对查询的写法 + 一致）。以前存的是模型的原文：PostgreSQL 上 orderstatus 会被当成另一列，核对时却按真名加了引号，存下的和核对过的 + 不是同一条，SQL 检查还会让模型照抄这一句。""" + entry = {"table": "orders", "valid_filter": {"condition": condition, "reason": "注释写明"}} + notes = catalog._from_model(entry, MIXED, system=False, source_kind=kind, table="orders") + assert notes["valid_filter"]["value"] == stored + again = sqlcheck.parse_valid_filter(stored, MIXED, kind=kind, table="orders") + assert again.problem is None and sqlcheck.stored_valid_filter(again, kind) == stored, "存下的再整理一遍不变" + + +def test_a_filter_that_grows_past_the_limit_once_quoted_is_dropped_not_written(): + """整理时加了引号、超过 500 字的不收（记下原因）：目录里这一项最多 500 字，写进去会让整张表写入失败。""" + condition = "orderstatus IN (" + ", ".join(f"'{i:03d}'" for i in range(66)) + ") OR updated_at IS NULL" + assert len(condition) == 499 + dropped: list[tuple[str, str]] = [] + notes = catalog._from_model({"table": "orders", "label": "订单", "valid_filter": condition}, MIXED, system=False, + source_kind="postgresql", table="orders", dropped=dropped) + assert "valid_filter" not in notes and notes["label"]["value"] == "订单" + assert dropped == [("有效记录条件按数据源的写法整理后超过 500 字", condition)] + assert not catalog.validate_notes(notes) + + +def test_columns_that_differ_only_in_case_must_be_named_exactly(): + """表结构里有只差大小写的两列：写得和其中一列一模一样的认那一列，否则说不清是哪一列，不收。""" + meta = {**META, "columns": [*META["columns"], _col("Status", "TEXT")]} + problem = sqlcheck.valid_filter_problem("STATUS = 'C'", meta, kind="postgresql", table="orders") + assert problem == ("有效记录条件里的列「STATUS」在表结构里有只差大小写的几列（status、Status），" + "要照表结构的写法写清是哪一列") + for condition, stored in (("\"Status\" = 'C'", "\"Status\" = 'C'"), ("status = 'C'", "status = 'C'")): + parsed = sqlcheck.parse_valid_filter(condition, meta, kind="postgresql", table="orders") + assert parsed.problem is None and sqlcheck.stored_valid_filter(parsed, "postgresql") == stored + + +# -------------------------------------------------------------------------- +# 模型给的两项：校验、依据 +# -------------------------------------------------------------------------- + + +def test_the_model_fields_become_llm_items_with_their_reason(): + entry = {"table": "orders", "label": "订单", + "valid_filter": {"condition": "status NOT IN ('C', 'X')", "reason": "状态注释写明 C 作废、X 测试单"}, + "dedup": {"rule": "同一订单号有多行时只取最后修改的一行", "reason": "订单号可能重复导出"}} + notes = catalog._from_model(entry, META, system=False, source_kind="sqlite", table="orders") + vf, dd = notes["valid_filter"], notes["dedup"] + assert (vf["value"], vf["source"], vf["status"]) == ("status NOT IN ('C', 'X')", "llm", "proposed") + assert vf["note"] == "依据：状态注释写明 C 作废、X 测试单" + assert (dd["value"], dd["source"], dd["status"]) == ("同一订单号有多行时只取最后修改的一行", "llm", "proposed") + assert dd["note"] == "依据：订单号可能重复导出" + assert not catalog.validate_notes(notes) + + +def test_plain_strings_are_accepted_and_bad_filters_dropped(caplog): + entry = {"table": "orders", "label": "订单", "valid_filter": "status <> 'C'", "dedup": "按订单号去重"} + notes = catalog._from_model(entry, META, system=False, source_kind="sqlite", table="orders") + assert notes["valid_filter"]["value"] == "status <> 'C'" and "note" not in notes["valid_filter"] + assert notes["dedup"]["value"] == "按订单号去重" + + with caplog.at_level(logging.INFO, logger="app.data.catalog"): + notes = catalog._from_model({"table": "orders", "label": "订单", + "valid_filter": {"condition": "status IN (SELECT 1)", "reason": "猜的"}}, + META, system=False, source_kind="sqlite", table="orders") + assert "valid_filter" not in notes and notes["label"]["value"] == "订单", "丢掉这一项，别的照收" + assert any("有效行过滤" in r.getMessage() and "orders" in r.getMessage() for r in caplog.records), "日志记一条" + for junk in ({"condition": ""}, {"reason": "只有依据"}, ["status = 1"], 3): + notes = catalog._from_model({"table": "orders", "valid_filter": junk, "dedup": junk}, META, system=False, + source_kind="sqlite", table="orders") + assert "valid_filter" not in notes and "dedup" not in notes, junk + + +def test_the_prompt_asks_for_the_two_facts_only_when_evident(): + system = catalog.CATALOG_DRAFT_SYSTEM + assert "valid_filter" in system and "dedup" in system + assert "拿不准就不写" in system and "reason" in system + assert "UPPER、LOWER、TRIM、COALESCE" in system + props = catalog.CATALOG_DRAFT_PROMPT_SCHEMA["properties"]["tables"]["items"]["properties"] + assert set(props["valid_filter"]["properties"]) == {"condition", "reason"} + assert set(props["dedup"]["properties"]) == {"rule", "reason"} + assert '"valid_filter": {"condition"' in catalog.CATALOG_DRAFT_TEXT_PROMPT + assert '"dedup": {"rule"' in catalog.CATALOG_DRAFT_TEXT_PROMPT + assert "有效行过滤" in catalog.CATALOG_DRAFT_PROMPT_LABELS["only_table"] + + +# -------------------------------------------------------------------------- +# 引擎：没开剖析照收、写明没有核对；开了剖析的源把码值交给模型 +# -------------------------------------------------------------------------- + +REPLY = {"label": "订单", "grain": "每笔订单一行", + "valid_filter": {"condition": "status NOT IN ('C', 'X')", "reason": "状态注释写明 C 作废、X 测试单"}, + "dedup": {"rule": "同一订单号只取最后一行", "reason": "订单号可能重复"}, + "columns": [{"name": "status", "label": "订单状态"}]} + + +async def test_without_profiling_the_filter_is_kept_and_says_it_was_not_checked(): + model = FakeModel({"orders": REPLY}) + got = await D.run_model_draft(model, _plain({"orders": _orders()}), ["orders"]) + o = got["orders"] + assert o.status == "ok" + vf = o.notes["valid_filter"] + assert (vf["source"], vf["status"]) == ("llm", "proposed") + assert vf["note"] == "依据：状态注释写明 C 作废、X 测试单。没有核对排除比例（数据源没有开数据剖析）" + assert o.notes["dedup"]["note"] == "依据：订单号可能重复" + + +async def test_an_invalid_filter_from_the_model_is_dropped_not_failed(): + reply = {**REPLY, "valid_filter": {"condition": "status IN (SELECT code FROM bad_codes)", "reason": "猜的"}} + got = await D.run_model_draft(FakeModel({"orders": reply}), _plain({"orders": _orders()}), ["orders"]) + o = got["orders"] + assert o.status == "ok" and o.error is None, "不算失败" + assert "valid_filter" not in o.notes and o.notes["dedup"]["value"] == "同一订单号只取最后一行" + + +def _codes_existing() -> dict: + return {"orders": {"columns": { + "status": {"codes": make_item({"C": "作废", "X": "", "P": "已支付"}, "profile", "proposed")}, + "order_no": {"codes": make_item({"A1": "甲"}, "profile", "proposed")}, + "amount": {"codes": make_item({"0": "零"}, "profile", "rejected")}, + }}} + + +async def test_codes_go_to_the_model_when_profiling_is_on_but_not_masked_ones(): + on = {PROFILE_OPTION: {"enabled": True}, MASK_COLUMNS_OPTION: ["ORDER_NO"]} + model = FakeModel({"orders": REPLY}) + await D.run_model_draft(model, _plain({"orders": _orders()}, options=on), ["orders"], existing=_codes_existing()) + text = model.calls[0] + head = catalog.CATALOG_DRAFT_PROMPT_LABELS["codes"] + assert head in text + block = text[text.index(head):] + assert "- status：C=作废、P=已支付，X 的含义待填写" in block + assert "A1" not in text, "遮罩的列不给码值" + assert "零" not in text, "驳回的码值不给" + + plain = FakeModel({"orders": REPLY}) + await D.run_model_draft(plain, _plain({"orders": _orders()}), ["orders"], existing=_codes_existing()) + assert head not in plain.calls[0] and "已支付" not in plain.calls[0], "没开剖析不给码值" + + +@pytest.mark.parametrize("options", [ + {PROFILE_OPTION: {"enabled": True}, MASK_COLUMNS_OPTION: ["ORDER_NO"]}, + {MASK_COLUMNS_OPTION: "order_no"}, +], ids=["profiled", "not-profiled"]) +async def test_confirmed_codes_of_a_masked_column_do_not_reach_the_model_either(options): + """遮罩的列不给码值，人工确认过的也一样：码值就是这一列的取值。以前经「已确认的目录项」照给（开没开剖析都给）； + 没遮罩的列确认过的码值照给（以此为准），遮罩列别的确认过的项（中文名）也照给。""" + existing = {"orders": {"columns": { + "order_no": {"codes": make_item({"ZX-0417": "内部测试单"}, "human", "confirmed"), + "label": make_item("订单号", "human", "confirmed")}, + "status": {"codes": make_item({"C": "作废"}, "human", "confirmed")}, + }}} + model = FakeModel({"orders": REPLY}) + await D.run_model_draft(model, _plain({"orders": _orders()}, options=options), ["orders"], existing=existing) + text = model.calls[0] + assert "ZX-0417" not in text and "内部测试单" not in text + confirmed = text[text.index(catalog.CATALOG_DRAFT_PROMPT_LABELS["confirmed"]):] + assert "status：码值：C=作废" in confirmed and "order_no：订单号" in confirmed + + +async def test_codes_reach_the_table_part_of_a_wide_table_and_only_their_column_part(): + cols = [_col(f"c{i:03d}") for i in range(1, 46)] + [_col("state", "TEXT")] + wide = {"qualified": "wide", "schema": None, "is_view": False, "primary_key": ["c001"], "foreign_keys": [], + "comment": None, "columns": cols} + existing = {"wide": {"columns": {"state": {"codes": make_item({"9": "作废"}, "profile", "proposed")}}}} + on = {PROFILE_OPTION: {"enabled": True}} + model = FakeModel({"wide": {"label": "宽表"}}) + await D.run_model_draft(model, _plain({"wide": wide}, options=on), ["wide"], existing=existing) + head = catalog.CATALOG_DRAFT_PROMPT_LABELS["codes"] + with_codes = [requested(c)["wide"] for c in model.calls if "9=作废" in c] + assert ("table", None) in with_codes, "表级那段要提议有效行过滤：给全部码值" + assert all(part == "table" or "state" in (cols or []) for part, cols in with_codes) + assert any(head not in c for c in model.calls), "不含这一列的列分段不给" + + +# -------------------------------------------------------------------------- +# 剖析核对排除比例 +# -------------------------------------------------------------------------- + + +async def test_the_check_counts_excluded_rows_through_the_read_only_guard(scenic, scenic_db, monkeypatch): + seen: list[tuple[str, bool]] = [] + real = catalog_profile.guard.check + + def spy(sql, **kw): + seen.append((sql, kw.get("readonly"))) + return real(sql, **kw) + + monkeypatch.setattr(catalog_profile.guard, "check", spy) + source = await scenic() + meta = source.schema_cache["tables"]["visits"] + found = await catalog_profile.check_valid_filter(source, "visits", meta, "status = 1") + total, excluded = _count(scenic_db, "status = 1") + assert 0 < excluded < total + assert found.keep and (found.rows, found.excluded) == (total, excluded) and not found.sampled + share = catalog_profile.excluded_share(excluded, total) + assert found.note == f"剖析核对：排除 {share} 的行（共 {total:,} 行）" + assert len(seen) == 1 and seen[0][1] is True, "一条查询，过只读守卫" + assert "COUNT(*)" in seen[0][0] and "status = 1" in seen[0][0] + + +async def test_the_check_runs_the_condition_as_it_is_stored(scenic, monkeypatch): + """剖析核对的就是存进目录的那一条：表名去掉、列换成表结构里的真名，别的照原文（NOT IN 不被改写成 NOT x IN）。""" + seen: list[str] = [] + real = catalog_profile.guard.check + + def spy(sql, **kw): + seen.append(sql) + return real(sql, **kw) + + monkeypatch.setattr(catalog_profile.guard, "check", spy) + source = await scenic() + reply = {"label": "入园记录", "valid_filter": {"condition": "VISITS.STATUS NOT IN (0)", "reason": "0 是作废的检票"}} + got = await D.run_model_draft(FakeModel({"visits": reply}), source, ["visits"]) + vf = got["visits"].notes["valid_filter"] + assert vf["value"] == "status NOT IN (0)" + assert len(seen) == 1 and f"CASE WHEN {vf['value']} THEN 0 ELSE 1 END" in seen[0] + assert "剖析核对：排除" in vf["note"] + + +@pytest.mark.parametrize("condition", ["status IN (0, 1)", "status = 7"]) +async def test_a_filter_that_excludes_nothing_or_everything_is_dropped(scenic, condition): + source = await scenic() + found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"], + condition) + assert not found.keep and found.dropped + + +async def test_a_big_table_is_checked_on_its_first_rows(scenic): + source = await scenic(options={PROFILE_OPTION: {"enabled": True, "max_scan_rows": 100, "sample_size": 10}}) + found = await catalog_profile.check_valid_filter(source, "visits", source.schema_cache["tables"]["visits"], + "status = 1") + assert found.sampled and found.rows == 100 + assert found.note.startswith("剖析核对：按前 100 行统计，") + + +async def test_unchecked_filters_are_kept_with_the_reason(scenic, tmp_path): + meta_of = lambda s: s.schema_cache["tables"]["visits"] # noqa: E731 + off = await scenic(options={}) + found = await catalog_profile.check_valid_filter(off, "visits", meta_of(off), "status = 1") + assert found.keep and found.note == "没有核对排除比例（数据源没有开数据剖析）" + + masked = await scenic(options={PROFILE_OPTION: {"enabled": True}, MASK_COLUMNS_OPTION: "Status"}) + found = await catalog_profile.check_valid_filter(masked, "visits", meta_of(masked), "status = 1") + assert found.keep and found.note == "没有核对排除比例（条件里的 status 在数据源设置中被遮罩，不取数）" + + stopped = await scenic(enabled=False) + found = await catalog_profile.check_valid_filter(stopped, "visits", meta_of(stopped), "status = 1") + assert found.keep and found.note == "没有核对排除比例（数据源已停用）" + + gone = await scenic(database=str(tmp_path / "missing" / "nope.db")) + found = await catalog_profile.check_valid_filter(gone, "visits", meta_of(gone), "status = 1") + assert found.keep and found.note.startswith("没有核对排除比例（查询失败：") + + +async def test_the_engine_checks_the_filter_on_a_profiled_source(scenic, scenic_db): + source = await scenic() + good = {"label": "入园记录", "valid_filter": {"condition": "status = 1", "reason": "0 是作废的检票"}} + got = await D.run_model_draft(FakeModel({"visits": good}), source, ["visits"]) + vf = got["visits"].notes["valid_filter"] + total, excluded = _count(scenic_db, "status = 1") + share = catalog_profile.excluded_share(excluded, total) + assert vf["note"] == f"依据：0 是作废的检票。剖析核对：排除 {share} 的行（共 {total:,} 行）" + + useless = {"label": "入园记录", "valid_filter": {"condition": "status IN (0, 1)", "reason": "猜的"}} + got = await D.run_model_draft(FakeModel({"visits": useless}), source, ["visits"]) + assert got["visits"].status == "ok" and "valid_filter" not in got["visits"].notes, "排除 0% 的不收" + assert got["visits"].notes["label"]["value"] == "入园记录" + + +async def test_a_dropped_filter_is_written_into_the_table_log(scenic, scenic_db): + """没收下的有效记录条件不算失败，在这张表的日志里记一条（起草任务逐表的 log，界面看得到）：没通过校验的写原因， + 剖析核对排除 0% 或 100% 的写数到了多少行。收下的、模型没提的不记。""" + bad = {**REPLY, "valid_filter": {"condition": "status IN (SELECT code FROM bad_codes)", "reason": "猜的"}} + got = await D.run_model_draft(FakeModel({"orders": bad}), _plain({"orders": _orders()}), ["orders"]) + assert got["orders"].status == "ok" and got["orders"].error is None + assert got["orders"].log == [ + "有效记录条件里不能有子查询，模型提议的这一条没有收下：status IN (SELECT code FROM bad_codes)"] + + source = await scenic() + total, _ = _count(scenic_db, "1 = 1") + for condition, what in (("status IN (0, 1)", "没有排除任何一行"), ("status = 7", "所有行都被排除了")): + useless = {"label": "入园记录", "valid_filter": {"condition": condition, "reason": "猜的"}} + got = await D.run_model_draft(FakeModel({"visits": useless}), source, ["visits"]) + assert got["visits"].log == [ + f"有效记录条件按剖析核对{what}（共 {total:,} 行），模型提议的这一条没有收下：{condition}"] + + for reply in (REPLY, {"label": "订单"}): + got = await D.run_model_draft(FakeModel({"orders": reply}), _plain({"orders": _orders()}), ["orders"]) + assert got["orders"].log == [] + + +async def test_a_confirmed_filter_is_not_checked_again(scenic, monkeypatch): + source = await scenic() + called: list[str] = [] + + async def counting(*args, **kw): + called.append("x") + return catalog_profile.FilterCheck(True, "不该用到") + + monkeypatch.setattr(catalog_profile, "check_valid_filter", counting) + existing = {"visits": {"valid_filter": make_item("status = 1", "human", "confirmed")}} + reply = {"label": "入园记录", "valid_filter": {"condition": "status <> 0", "reason": "0 是作废"}} + await D.run_model_draft(FakeModel({"visits": reply}), source, ["visits"], existing=existing) + assert called == [], "人工确认过的有效行过滤，起草的提议并入时不会采用，不必再查库" + + +# -------------------------------------------------------------------------- +# SQL 检查：推断的有效行过滤只是提示，确认后才是警告 +# -------------------------------------------------------------------------- + + +async def test_a_drafted_filter_only_hints_until_someone_confirms_it(scenic): + source = await scenic(options={}) + entry = {"table": "visits", "valid_filter": {"condition": "status = 1", "reason": "0 是作废"}} + meta = source.schema_cache["tables"]["visits"] + notes = catalog._from_model(entry, meta, system=False, source_kind="sqlite", table="visits") + sql = "SELECT COUNT(*) FROM visits" + + def levels(table_notes: dict) -> list[tuple[str, str]]: + checker = sqlcheck.SqlChecker(kind="sqlite", schema_cache=source.schema_cache, notes={"visits": table_notes}) + return [(c.code, c.level) for c in checker.check(sql)] + + assert levels(notes) == [("missing_valid_filter", "info")] + confirmed = catalog.review_item(notes, "valid_filter", "confirm") + assert levels(confirmed) == [("missing_valid_filter", "warning")] + assert levels(catalog.review_item(notes, "valid_filter", "reject")) == [] diff --git a/backend/tests/test_catalog_draft_jobs.py b/backend/tests/test_catalog_draft_jobs.py index bc366876..81840b34 100644 --- a/backend/tests/test_catalog_draft_jobs.py +++ b/backend/tests/test_catalog_draft_jobs.py @@ -441,6 +441,11 @@ def test_old_job_tables_get_the_new_columns(): assert conn.execute(text("SELECT timeouts, failed_calls, part_columns, notice FROM catalog_draft_jobs")) \ .one() == (0, 0, None, None) assert conn.execute(text("SELECT blank_columns, blank_sample FROM catalog_draft_tasks")).one() == (0, None) + # 闭环第一步：逐段落盘的段数、块进度。存量行是 0 和 NULL：升级前没记这些 + assert conn.execute(text("SELECT parts_written, chunks_total, chunks_done FROM catalog_draft_tasks")) \ + .one() == (0, None, None) + # 复核后：逐表的日志（没收下的有效记录条件）。存量行是 NULL，接口给空列表 + assert conn.execute(text("SELECT log FROM catalog_draft_tasks")).one() == (None,) async def test_a_job_that_cannot_start_is_failed_with_the_reason(client, monkeypatch): @@ -779,3 +784,271 @@ async def test_jobs_go_away_with_their_source(client): assert (await session.execute(select(CatalogDraftJob).where(CatalogDraftJob.id == job["id"]))).first() is None assert (await session.execute(select(CatalogDraftTask).where(CatalogDraftTask.job_id == job["id"]))).first() \ is None + + +# -------------------------------------------------------------------------- +# 闭环第一步：宽表逐段落盘、停止后的说法、块进度、停止后的留空提示、同步起草与任务互斥 +# -------------------------------------------------------------------------- + + +def _wide(n: int = 95) -> dict: + return {"wide": {"qualified": "wide", "columns": [_col(f"c{i:03d}") for i in range(1, n + 1)], + "primary_key": ["c001"], "comment": None, "is_view": False, "foreign_keys": []}} + + +def _wide_reply(blank: set[str] = frozenset()) -> dict: + """95 列的宽表每列都写中文名；blank 里的列只列出、不写（模型拿不准）。""" + return {"wide": {"label": "宽表", "kind": "fact", + "columns": [{"name": f"c{i:03d}", "label": f"第{i}列"} for i in range(1, 96) + if f"c{i:03d}" not in blank]}} + + +class Hold: + """假模型：请写的表在 tables 里、或请写的列里有 columns 里的列时，等 release 才回（停止时这次调用被取消）；别的照 + FakeModel 回。waiting：有调用在等了。""" + + def __init__(self, replies: dict, *, tables: set[str] = frozenset(), columns: set[str] = frozenset()) -> None: + self.inner = FakeModel(replies) + self.tables, self.columns = set(tables), set(columns) + self.release, self.waiting = asyncio.Event(), asyncio.Event() + self.cancelled = 0 + self.model_name = "hold-model" + + def with_structured_output(self, schema, **_): + outer = self + + class _S: + async def ainvoke(self, messages): + asked = requested("\n".join(str(m.content) for m in messages)) + cols = {c for _, listed in asked.values() for c in listed or []} + if outer.tables & set(asked) or outer.columns & cols: + outer.waiting.set() + try: + await outer.release.wait() + except asyncio.CancelledError: + outer.cancelled += 1 + raise + return outer.inner._reply(messages, "structured") + + return _S() + + async def ainvoke(self, messages): + return await self.inner.ainvoke(messages) + + +async def _task(client, sid: str, job_id: str, table: str) -> dict: + return next(t for t in (await get_job(client, sid, job_id))["tasks"] if t["table_name"] == table) + + +async def _start_held_wide(client, sid: str, monkeypatch) -> tuple[str, Hold]: + """95 列的宽表分 4 段：表级那段和第 2 段照常写，第 3、4 段一直等着。等到前两段都写进目录才返回。""" + model = Hold(_wide_reply(), columns={"c033", "c065"}) + use(monkeypatch, model) + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["wide"], "use_model": True})).json()["job"] + + async def two_written() -> bool: + return (await _task(client, sid, job["id"], "wide"))["parts_written"] == 2 + + await until(two_written) + return job["id"], model + + +async def test_a_stopped_wide_table_keeps_the_parts_already_written(client, monkeypatch): + """宽表每写出一段就并入目录：停止时已经写进去的段留着，这张表照旧记停止，model_error 说清写进了几段、剩几段没起草。""" + sid = await _source(_wide()) + job_id, model = await _start_held_wide(client, sid, monkeypatch) + task = await _task(client, sid, job_id, "wide") + assert task["status"] == "running" and task["parts_total"] == 4 and task["parts_done"] == 2 + assert (task["chunks_total"], task["chunks_done"]) == (4, 2) + assert task["added"] == 2 + 32 and task["version"] >= 1, "表级两项 + 第 2 段 32 列" + + r = await client.post(f"{_base(sid)}/draft-jobs/{job_id}/cancel") + assert r.status_code == 200, r.text + task = next(t for t in r.json()["job"]["tasks"] if t["table_name"] == "wide") + assert task["status"] == "cancelled" and task["error"] is None and task["parts_written"] == 2 + assert task["model_error"] == "已停止：已写进目录 2 段，剩下 2 段没起草" + assert model.cancelled == 2, "第 3、4 段正在进行的调用被停掉" + notes = (await client.get(f"{_base(sid)}/wide")).json()["notes"] + assert notes["label"]["value"] == "宽表" and notes["kind"]["value"] == "fact" + assert set(notes["columns"]) == {f"c{i:03d}" for i in range(1, 33)}, "第 2 段写进去的留着" + row = next(r for r in (await client.get(_base(sid))).json()["tables"] if r["table_name"] == "wide") + assert row["last_draft"]["status"] == "cancelled" and row["last_draft"]["parts_written"] == 2 + assert row["last_draft"]["blank_columns_from"] is None, "停止前写进了新内容：以前的留空列数不再沿用" + + +async def test_an_interrupted_wide_table_keeps_its_written_parts_too(client, monkeypatch): + sid = await _source(_wide()) + job_id, _ = await _start_held_wide(client, sid, monkeypatch) + await catalog_jobs.manager.shutdown(grace=5) + task = await _task(client, sid, job_id, "wide") + assert task["status"] == "cancelled" and task["error"] == "服务重启，起草中断" + assert task["model_error"] == "已中断：已写进目录 2 段，剩下 2 段没起草" + assert len((await client.get(f"{_base(sid)}/wide")).json()["notes"]["columns"]) == 32 + + # 进程没来得及收尾（直接退出）：下次启动时同样写清 + async with SessionLocal() as session: + job = CatalogDraftJob(source_id=sid, status="running", use_model=True) + session.add(job) + await session.flush() + session.add(CatalogDraftTask(job_id=job.id, seq=0, table_name="wide", status="running", parts_total=3, + parts_written=1)) + session.add(CatalogDraftTask(job_id=job.id, seq=1, table_name="gates", status="pending", parts_total=1)) + await session.commit() + crashed = job.id + await catalog_jobs.startup() + by = {t["table_name"]: t for t in (await get_job(client, sid, crashed))["tasks"]} + assert by["wide"]["model_error"] == "已中断：已写进目录 1 段，剩下 2 段没起草" + assert by["gates"]["model_error"] is None and by["gates"]["error"] == "服务重启，起草中断" + + +async def test_a_finished_wide_table_counts_every_write_once(client, monkeypatch): + """逐段落盘之后整张表收尾一次：新增、更新的计数是各次写入的合计（每一项只数一次），块进度和写进去的段数都记下。""" + sid = await _source(_wide()) + job = await _draft(client, sid, monkeypatch, FakeModel(_wide_reply({"c010", "c040", "c070"})), ["wide"]) + task = job["tasks"][0] + assert task["status"] == "ok" and task["parts_written"] == 4 and task["parts_done"] == 4 + assert (task["chunks_total"], task["chunks_done"]) == (4, 4) + assert (task["added"], task["updated"], task["removed"]) == (2 + 92, 0, 0) + assert task["blank_columns"] == 3 and task["blank_sample"] == ["c010", "c040", "c070"] + notes = (await client.get(f"{_base(sid)}/wide")).json()["notes"] + assert len(notes["columns"]) == 92 + + plain = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["wide"]})).json()["job"] + await finished(plain["id"]) + task = (await get_job(client, sid, plain["id"]))["tasks"][0] + assert (task["parts_written"], task["chunks_total"], task["chunks_done"]) == (0, None, None), "不用模型时为 null" + + +async def test_chunk_progress_shows_a_part_that_was_split(client, monkeypatch): + """块进度：这张表实际分成的块数和已结束的块数，一段超时拆成两半后变多。""" + monkeypatch.setattr(catalog, "MODEL_TIMEOUT_S", 0.05) + sid = await _source(_tickets(24)) + replies = {"tickets": {"label": "门票", "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} + for i in range(1, 25)]}} + job = await _draft(client, sid, monkeypatch, FakeModel(replies, hang=1), ["tickets"]) + task = job["tasks"][0] + assert task["status"] == "ok" and (task["chunks_total"], task["chunks_done"]) == (2, 2) + assert task["parts_written"] == 1 and task["parts_total"] == 1 + + +async def test_a_partial_table_also_drops_model_items_of_columns_gone_from_the_schema(client, monkeypatch): + """整张表写完时按完整覆盖的规则收尾，只写出一部分的也一样：只删已经不在表结构里的列的模型起草项。""" + sid = await _source(_tickets(6)) + full = {"label": "门票", "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} for i in range(1, 7)]} + await _draft(client, sid, monkeypatch, FakeModel({"tickets": full}), ["tickets"]) + async with SessionLocal() as session: + row = await session.get(DataSource, sid) + row.schema_cache = {"tables": _tickets(5)} # 重新探查过：c06 已经没有了 + await session.commit() + some = {"label": "门票", "columns": [{"name": f"c{i:02d}", "label": f"第{i}列"} for i in range(1, 5)]} + job = await _draft(client, sid, monkeypatch, FakeModel({"tickets": some}, list_all=False), ["tickets"]) + task = job["tasks"][0] + assert task["status"] == "partial" and task["removed"] == 1 + after = (await client.get(f"{_base(sid)}/tickets")).json()["notes"]["columns"] + assert set(after) == {"c01", "c02", "c03", "c04", "c05"}, "c05 是以前起草的，这次没写出来也留着" + + +async def test_blank_columns_from_carries_the_last_finished_draft_over_a_stop(client, monkeypatch): + """目录清单的 last_draft.blank_columns_from：最近一次结束于完成、部分完成的起草记下的留空列数。之后的起草停止了、 + 没写进新内容时照样带着，界面据此继续提示；不用模型的起草不算。""" + sid = await _source(_tickets(4)) + replies = {"tickets": {"label": "门票", "columns": [{"name": "c01", "label": "编号"}]}} + await _draft(client, sid, monkeypatch, FakeModel(replies), ["tickets"]) + + async def last() -> dict: + rows = (await client.get(_base(sid))).json()["tables"] + return next(r for r in rows if r["table_name"] == "tickets")["last_draft"] + + first = await last() + assert first["status"] == "ok" and first["blank_columns"] == 3 and first["blank_columns_from"] == 3 + assert first["parts_written"] == 1 + + plain = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["tickets"]})).json()["job"] + await finished(plain["id"]) + assert (await last())["blank_columns_from"] == 3, "不用模型的起草不改模型留空的列" + + monkeypatch.setattr(catalog_jobs.manager, "start", lambda job_id: None) + queued = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["tickets"], "use_model": True})).json() + await client.post(f"{_base(sid)}/draft-jobs/{queued['job']['id']}/cancel") + stopped = await last() + assert stopped["status"] == "cancelled" and stopped["blank_columns"] == 0 and stopped["parts_written"] == 0 + assert stopped["blank_columns_from"] == 3 + + other = await _source(_tickets(4)) + rows = (await client.get(_base(other))).json()["tables"] + assert rows[0]["last_draft"] is None + + +async def test_a_sync_draft_and_draft_jobs_exclude_each_other(client, monkeypatch): + """同步起草进行期间，同一个源建任务、重试、再开一次同步起草都 409 catalog_draft_busy（导入照 ensure_idle 认）； + 同步起草结束后照常。""" + sid = await _source() + use(monkeypatch, FakeModel(REPLIES)) + old = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"], "use_model": True})).json()["job"] + await finished(old["id"]) + async with SessionLocal() as session: + await session.execute( + CatalogDraftTask.__table__.update().where(CatalogDraftTask.job_id == old["id"]).values(status="failed")) + await session.commit() + + model = Hold(REPLIES, tables={"visits"}) + use(monkeypatch, model) + sync = asyncio.create_task(client.post(f"{_base(sid)}/draft", json={"tables": ["visits"], "use_model": True})) + await asyncio.wait_for(model.waiting.wait(), 5) + + busy = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]}) + assert busy.status_code == 409 and busy.json()["code"] == "catalog_draft_busy", busy.text + assert "正在起草" in busy.json()["detail"] + retry = await client.post(f"{_base(sid)}/draft-jobs/{old['id']}/retry") + assert retry.status_code == 409 and retry.json()["code"] == "catalog_draft_busy", retry.text + again = await client.post(f"{_base(sid)}/draft", json={"tables": ["gates"]}) + assert again.status_code == 409 and again.json()["code"] == "catalog_draft_busy", again.text + async with SessionLocal() as session: + with pytest.raises(catalog_jobs.DraftBusy): + await catalog_jobs.ensure_idle(session, await session.get(DataSource, sid)) + assert (await client.get(f"{_base(sid)}/draft-jobs/latest")).json()["job"]["id"] == old["id"], "没建新任务" + + model.release.set() + done = await sync + assert done.status_code == 200, done.text + after = await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["gates"]}) + assert after.status_code == 202, after.text + await finished(after.json()["job"]["id"]) + + +async def test_a_wide_table_whose_closing_write_fails_keeps_the_count_of_parts_on_disk(client, monkeypatch): + """整张表收尾那一次写不进去（一直撞上别人刚改过）：已经逐段写进去的段留着，写进目录的段数照库里的记，不按引擎写出来 + 的段数虚报。""" + monkeypatch.setattr(catalog, "MODEL_CONCURRENCY", 1) # 一段一段来：最后一段只在收尾那次写 + sid = await _source(_wide()) + use(monkeypatch, FakeModel(_wide_reply())) + real = catalog.write_draft + + async def conflicted(session, source, table, modeled=None, *, at, actor): + result = await real(session, source, table, None, at=at, actor=actor) + result.error, result.status = catalog.WRITE_CONFLICT_ERROR, "failed" + result.added = result.updated = result.removed = 0 + return result + + monkeypatch.setattr(catalog, "write_draft", conflicted) + job = (await client.post(f"{_base(sid)}/draft-jobs", json={"tables": ["wide"], "use_model": True})).json()["job"] + await finished(job["id"]) + task = await _task(client, sid, job["id"], "wide") + assert task["status"] == "failed" and task["error"] == catalog.WRITE_CONFLICT_ERROR + assert task["parts_written"] == 3, "最后一段没写进去" + assert len((await client.get(f"{_base(sid)}/wide")).json()["notes"]["columns"]) == 64 + + +async def test_a_dropped_filter_is_logged_on_the_task_not_failed(client, monkeypatch): + """模型提议的有效记录条件没通过校验：丢掉、不算失败（表照样完成），逐表的 log 里记一条，接口看得到；没有就是 + 空列表。""" + sid = await _source() + replies = {**REPLIES, "gates": {"label": "闸口", + "valid_filter": {"condition": "gate_code IN (SELECT 1)", "reason": "猜的"}}} + job = await _draft(client, sid, monkeypatch, FakeModel(replies), ["gates", "channels"]) + by = {t["table_name"]: t for t in job["tasks"]} + assert by["gates"]["status"] == "ok" and by["gates"]["model_error"] is None + assert by["gates"]["log"] == ["有效记录条件里不能有子查询，模型提议的这一条没有收下：gate_code IN (SELECT 1)"] + assert by["channels"]["log"] == [] + notes = (await client.get(f"{_base(sid)}/gates")).json()["notes"] + assert "valid_filter" not in notes and notes["label"]["value"] == "闸口"