diff --git a/README.md b/README.md index 21994a83..873e86f1 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,10 @@ Mock 不会写引用标记：探索运行里报告撰写节点照样跑完，但 http://127.0.0.1:8000 。前端产物不存在或比源码旧时才重新构建，`--build` / `--no-build` 强制构建或跳过。 `--port`（或 `AGENTLAB_PORT`）改端口；`--host` 监听所有网卡，同样**没有登录**，只在信得过的网络里开。 数据目录默认为仓库下的 `data/`，`AGENTLAB_DATA_DIR` 可改。只能单进程运行：运行管理器在进程内、数据库是 SQLite。 +上传表格的版本存储同样只支持单进程运行（不要给 uvicorn 加 `--workers`）：启动时会锁住数据目录，同一个数据目录 +起了第二个进程时，后起的那个不做任何清理，上传、启用版本等写操作一律返回 503。 +按期累积的数据源每个历史版本是一份完整副本，磁盘占用随期数和被引用的版本数增长。除当前版本和被运行引用的 +版本外，每个数据源再保留最近启用过的 3 个版本，其余的会被回收、无法再启用。 **Docker 离线包：** `./scripts/package-docker.sh` 构建 Linux 镜像（默认取本机架构，目标机不同时用 `--platform linux/arm64` 或 `linux/amd64`），连同一键部署脚本、配置模板和部署说明打成 diff --git a/backend/app/api/datasources.py b/backend/app/api/datasources.py index 4f7d72ce..e91a0977 100644 --- a/backend/app/api/datasources.py +++ b/backend/app/api/datasources.py @@ -9,7 +9,7 @@ import logging from datetime import datetime, timezone from typing import Any -from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, UploadFile +from fastapi import APIRouter, Depends, File, Form, Header, HTTPException, Query, Request, UploadFile from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse from pydantic import BaseModel, Field @@ -18,6 +18,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.api import health from app.api.coded import CodedHTTPException +from app.api.runs import actor_of from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw from app.core.config import settings from app.core.crypto import encrypt, mask @@ -174,28 +175,40 @@ def _is_upload(row: DataSource) -> bool: async def _snapshot_infos(session: AsyncSession, rows: list[DataSource]) -> dict[str, dict[str, Any]]: - """各上传源当前版本的摘要：{源 id: {id, created_at, file_name, raw_state}}。两次查询取齐，不逐个查。""" + """各上传源当前版本的摘要：{源 id: {id, created_at, file_name, raw_state, mode, periods, period_start, + period_end, activated_at}}。两次查询取齐，不逐个查。 + + 文件名、原件状态取最近的那一期（按统计期排在最后的）；期 3 按期累积时一个快照有多期，卡片另写模式、期数、 + 统计期范围。activated_at 是最近一次成为当前版本的时刻：回滚之后卡片写「…启用」，不再写快照当初的导入时间 + （期 3 之前的快照没有这一列，按 created_at）。""" wanted = {r.current_snapshot_id: r.id for r in rows if _is_upload(r) and r.current_snapshot_id} if not wanted: return {} snaps = list((await session.execute( select(SourceSnapshot).where(SourceSnapshot.id.in_(list(wanted))) )).scalars()) - # 期 1 一个快照只有一期导入；以后按期累积时，显示最近的那一期 - last_import = {s.id: str(s.imports[-1]) for s in snaps if s.imports} + members = {str(x) for s in snaps for x in s.imports or []} imports = { i.id: i for i in (await session.execute( - select(TableImport).where(TableImport.id.in_(list(last_import.values()))) + select(TableImport).where(TableImport.id.in_(list(members))) )).scalars() - } if last_import else {} + } if members else {} out: dict[str, dict[str, Any]] = {} for snap in snaps: - imp = imports.get(last_import.get(snap.id, "")) + mine = [imports[str(x)] for x in snap.imports or [] if str(x) in imports] + imp = mine[-1] if mine else None + starts = [i.period_start for i in mine if i.period_start] + ends = [i.period_end for i in mine if i.period_end] out[snap.source_id] = { "id": snap.id, "created_at": snap.created_at.isoformat() if snap.created_at else None, "file_name": imp.file_name if imp else "", "raw_state": imp.raw_state if imp else "absent", + "mode": snap.mode or ("replace" if imp is not None and imp.recipe_id else None), + "periods": len(snap.imports or []), + "period_start": min(starts) if starts else None, + "period_end": max(ends) if ends else None, + "activated_at": _iso(snap.activated_at or snap.created_at), } return out @@ -475,15 +488,24 @@ async def update_source( @router.delete("/{source_id}", status_code=204) async def delete_source(source_id: str, session: AsyncSession = Depends(get_session)) -> None: row = await _get_or_404(session, source_id) - # 上传源的导入记录置 retired 留作审计；库文件和原件交给回收，被运行引用的版本会保留 - await table_versions.retire_source(session, source_id) + # 上传源的导入记录置 retired 留作审计；库文件和原件交给回收，被运行引用的版本会保留。只读进程（没拿到 + # 版本存储的守卫）不删上传源：retire_source 要删隔离文件、放弃暂存区（第 8 节遗留项 5）。 + # 手工登记的源没有导入记录、配方、暂存区和隔离文件，根本不碰版本存储：不调 retire_source，只读进程里也照常 + # 删除（9.1 只要求上传源回 503，评审意见）；删完也没有文件可回收 + upload = _is_upload(row) + if upload: + try: + await table_versions.retire_source(session, source_id) + except table_versions.StoreUnavailable as e: + raise CodedHTTPException(503, str(e), "store_unavailable") from e await session.delete(row) await session.commit() await engines.invalidate(source_id) - try: - await table_versions.gc(session) - except Exception: # noqa: BLE001 - 回收失败不影响删除，重启或下次上传时再收 - logger.exception("删除数据源后的回收失败") + if upload: + try: + await table_versions.gc(session) + except Exception: # noqa: BLE001 - 回收失败不影响删除，重启或下次上传时再收 + logger.exception("删除数据源后的回收失败") #: 测连接最多等多久。内网库防火墙丢包时驱动默认要等一分钟，弹窗里干转一分钟 @@ -926,6 +948,11 @@ async def upload_table( raise HTTPException(400, str(e)) from e except table_versions.PublishError as e: logger.error("上传表格「%s」发布失败：%s", name, e) + if e.code == "store_unavailable": + raise CodedHTTPException(503, str(e), "store_unavailable") from e + if e.code == "build_conflict": + # 服务端已有的同版本文件被改过、这次上传也恢复不了：重试也会失败，要管理员处理（第 8 节遗留项 1） + raise CodedHTTPException(409, str(e), "build_conflict") from e raise HTTPException(500, str(e)) from e await session.refresh(row) @@ -936,6 +963,8 @@ async def upload_table( "import_id": result.import_id, "snapshot_id": result.snapshot_id, "build_reused": result.build_reused, + # 服务端的同版本数据文件曾被改动，已用本次上传的文件恢复（原文件挪进了隔离区） + "build_restored": result.build_restored, "skipped_sheets": report.get("skipped_sheets") or [], "conversions": report.get("conversions") or [], "warnings": report.get("warnings") or [], @@ -943,76 +972,72 @@ async def upload_table( } -def _import_out(imp: TableImport, current: set[str]) -> dict[str, Any]: - return { - "id": imp.id, "seq": imp.seq, "build_id": imp.build_id, - "file_name": imp.file_name, "file_size": imp.file_size, "raw_sha256": imp.raw_sha256, - "raw_state": imp.raw_state, "status": imp.status, "purged": imp.purged, - "current": imp.id in current, - "created_at": imp.created_at.isoformat() if imp.created_at else None, - "activated_at": imp.activated_at.isoformat() if imp.activated_at else None, - # 按配方导入才有（简单导入为空）：配方、统计期、署名，接受的条数 - "recipe_id": imp.recipe_id, "period_start": imp.period_start, "period_end": imp.period_end, - "signed_by": imp.signed_by, "overrides": len(imp.overrides or []), "waivers": len(imp.waivers or []), - } - - -async def _current_imports(session: AsyncSession, row: DataSource) -> set[str]: - if not row.current_snapshot_id: - return set() - snap = await session.get(SourceSnapshot, row.current_snapshot_id) - return {str(x) for x in (snap.imports or [])} if snap else set() - - @router.get("/{source_id}/imports") async def list_imports(source_id: str, session: AsyncSession = Depends(get_session)) -> list[dict[str, Any]]: - """上传表格的导入记录，新的在前。手工登记的源没有导入记录，返回空列表。""" + """上传表格的导入记录，新的在前。手工登记的源没有导入记录，返回空列表。 + + 期 3（P3-SPEC 7.6）补了接受明细、作废记录、导入清单、配方第几版、各表行数、作废接受的预案（revoke_plan）， + 以及同一份原件在别处的引用（raw_shared_with、raw_open_stagings）：清除对话框在提交前就要列出来。""" + from app.data import source_versions + row = await _get_or_404(session, source_id) - rows = (await session.execute( + rows = list((await session.execute( select(TableImport).where(TableImport.source_id == source_id).order_by(TableImport.seq.desc()) - )).scalars() - current = await _current_imports(session, row) - return [_import_out(i, current) for i in rows] + )).scalars()) + return await source_versions.import_records(session, row, rows) -class PurgeRawIn(BaseModel): - reason: str = Field(min_length=1, max_length=500, description="为什么清除这份原件") - signed_by: str | None = Field( - default=None, max_length=100, - description="署名（未认证）：自填的名字，系统不核实身份，只原样记进清除记录", - ) +#: 清除原件的理由、署名的字数上限 +_PURGE_REASON_MAX = 500 +_SIGNED_MAX = 100 @router.post("/{source_id}/imports/{import_id}/purge-raw") async def purge_raw( - source_id: str, import_id: str, payload: PurgeRawIn, session: AsyncSession = Depends(get_session) + source_id: str, import_id: str, request: Request, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session), ) -> dict[str, Any]: """清除一次导入的原件。导入记录、哈希和库都保留，原件文件删掉，记下时间、署名和理由。 按内容清除：同一份内容可能被别的导入共用（同一个文件传过两次、传给过两个源），清除的 目的是让这份内容从服务器上消失，所以文件照删，那些导入一并标成已清除，回执的 also_purged 逐条列出（期 3 的界面要把它展示出来，不能静默带过）。 + + 期 3：请求体手工读成 dict（评审二-m8：原来用 pydantic，空理由由 FastAPI 自动回 422，响应体不是 {detail, code}）， + 错误一律带机读码。confirm 键收但不强求（7.7：期 1、期 2 的调用方不带它）；带了而不是 true 时拒绝。 """ + from app.data import source_versions + + body = await _json_body(request) row = await _get_or_404(session, source_id) imp = await session.get(TableImport, import_id) if imp is None or imp.source_id != source_id: - raise HTTPException(404, "导入记录不存在") - reason = payload.reason.strip() + raise CodedHTTPException(404, "导入记录不存在", "import_not_found") + if "confirm" in body and body.get("confirm") is not True: + raise CodedHTTPException(422, "请在确认框中确认后再清除原件", "confirm_required") + raw_reason = body.get("reason") + reason = raw_reason.strip() if isinstance(raw_reason, str) else "" if not reason: - raise HTTPException(422, "请填写清除原件的理由") + raise CodedHTTPException(422, "请填写清除原件的理由", "reason_required") + if len(reason) > _PURGE_REASON_MAX: + raise CodedHTTPException(422, f"理由不能超过 {_PURGE_REASON_MAX} 字", "reason_required") + signed = body.get("signed_by") + signed_by = signed.strip()[:_SIGNED_MAX] if isinstance(signed, str) and signed.strip() else actor_of(x_actor) if imp.raw_state == "absent": - raise HTTPException(409, "这次导入没有保存原件") + raise CodedHTTPException(409, "这次导入没有保存原件", "raw_absent") if imp.raw_state == "purged": - raise HTTPException(409, "这次导入的原件已经清除") - purged = await table_versions.purge_import_raw( - session, imp, reason=reason, signed_by=payload.signed_by) + raise CodedHTTPException(409, "这次导入的原件已经清除", "raw_already_purged") + try: + purged = await table_versions.purge_import_raw(session, imp, reason=reason, signed_by=signed_by) + except table_versions.StoreUnavailable as e: + raise CodedHTTPException(503, str(e), "store_unavailable") from e deleted, others = purged.deleted, purged.others - current = await _current_imports(session, row) names = dict((await session.execute( select(DataSource.id, DataSource.name).where(DataSource.id.in_({o.source_id for o in others})) )).all()) if others else {} + [record] = await source_versions.import_records(session, row, [imp]) return { - "import": _import_out(imp, current), + "import": record, "file_deleted": deleted, "also_purged": [ {"id": o.id, "source_id": o.source_id, "source_name": names.get(o.source_id), @@ -1022,3 +1047,17 @@ async def purge_raw( # 引用同一份原件、随之一并放弃的未完成导入（暂存区，已清空） "discarded_stagings": [st.id for st in purged.discarded_stagings], } + + +async def _json_body(request: Request) -> dict[str, Any]: + """JSON 请求体手工读成 dict（空 body 当作 {}）；不是 JSON 对象回 422 body_invalid（不触发 FastAPI 自带的 422）。""" + raw = await request.body() + if not raw.strip(): + return {} + try: + data = await request.json() + except ValueError: + data = None + if not isinstance(data, dict): + raise CodedHTTPException(422, "请求体必须是一个 JSON 对象", "body_invalid") + return data diff --git a/backend/app/api/source_versions.py b/backend/app/api/source_versions.py new file mode 100644 index 00000000..2dbbaf54 --- /dev/null +++ b/backend/app/api/source_versions.py @@ -0,0 +1,201 @@ +"""上传表格的版本页接口（期 3，WP-5；P3-SPEC 第 7 节）：版本列表、启用旧版本（回滚）、查看清单、移除一期、 +作废接受。导入记录列表与清除原件仍在 datasources.py（期 1 的接口，期 3 补了字段和错误码）。 + +流程在 app.data.source_versions 里，这里只做三件事：手工读请求体（不触发 FastAPI 自带的 422：它的 body 是 +{detail: [...]}、不带 code，前端没法分支）、取署名、把业务错误转成带机读码的 CodedHTTPException。 + +**写操作一律要 confirm: true 和界面看到的当前版本（expected_current_snapshot_id）**（7.7）：确认框是按那个版本 +写的后果，别人在这期间提交、启用过，锁内比对不上就 409 base_changed，不能照旧切过去。移除、作废还要写理由。 +署名取请求体的 signed_by，其次请求头 X-Actor，只原样记下（未认证）。 + +路径都是 /{source_id}/snapshots… 和 /{source_id}/imports/{import_id}/(manifest|remove|revoke-acceptance)，与 +datasources.py 的 /{source_id}/imports（GET）、/{source_id}/imports/{import_id}/purge-raw 不重叠；在 main.py 里 +登记在 table_imports 之后、datasources 之前。 +""" +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, Header, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.coded import CodedHTTPException +from app.api.datasources import _out +from app.api.runs import actor_of +from app.data import source_versions, table_versions +from app.data.recipe_imports import ImportRefused +from app.db.base import get_session +from app.db.models import DataSource + +router = APIRouter(prefix="/api/datasources", tags=["source-versions"]) + +_SIGNED_MAX = 100 + + +def _refused(e: Exception) -> CodedHTTPException: + if isinstance(e, table_versions.StoreUnavailable): + return CodedHTTPException(503, str(e), "store_unavailable") + return CodedHTTPException(e.status, e.message, e.code) # type: ignore[attr-defined] + + +_REFUSALS = (ImportRefused, table_versions.StoreUnavailable) + + +async def _body(request: Request) -> dict[str, Any]: + raw = await request.body() + if not raw.strip(): + return {} + try: + data = await request.json() + except ValueError: + data = None + if not isinstance(data, dict): + raise CodedHTTPException(422, "请求体必须是一个 JSON 对象", "body_invalid") + return data + + +def _signed(body: dict[str, Any], header: str | None) -> str | None: + raw = body.get("signed_by") + if isinstance(raw, str) and raw.strip(): + return raw.strip()[:_SIGNED_MAX] + return actor_of(header) + + +async def _source(session: AsyncSession, source_id: str) -> DataSource: + row = await session.get(DataSource, source_id) + if row is None: + raise CodedHTTPException(404, "数据源不存在，可能已被删除", "source_not_found") + return row + + +def _confirmed(body: dict[str, Any]) -> None: + if body.get("confirm") is not True: + raise CodedHTTPException(422, "请在确认框中确认后再操作", "confirm_required") + + +def _expected_current(body: dict[str, Any]) -> str | None: + """界面看到的当前版本 id：必须带这个键（值可以是 null：界面看到的就是「没有当前版本」）。""" + if "expected_current_snapshot_id" not in body: + raise CodedHTTPException(422, "请求里缺少界面看到的当前版本，请刷新版本列表后重试", "expected_current_required") + value = body.get("expected_current_snapshot_id") + if value is not None and not isinstance(value, str): + raise CodedHTTPException(422, "当前版本的写法不对，请刷新版本列表后重试", "expected_current_required") + return value + + +def _reason(body: dict[str, Any], *, required: bool) -> str | None: + raw = body.get("reason") + if raw is not None and not isinstance(raw, str): + raise CodedHTTPException(422, "理由必须是一段文字", "reason_required") + text = (raw or "").strip() + if required and not text: + raise CodedHTTPException(422, "请写明理由", "reason_required") + if len(text) > source_versions.REASON_MAX: + raise CodedHTTPException(422, f"理由不能超过 {source_versions.REASON_MAX} 字", "reason_required") + return text or None + + +def _ack(body: dict[str, Any]) -> list[str] | None: + raw = body.get("ack_mask_lost") + if raw is None: + return None + if not isinstance(raw, list) or not all(isinstance(x, str) for x in raw): + raise CodedHTTPException(422, "确认丢失的遮罩列要写成列名的列表", "body_invalid") + return list(raw) + + +async def _source_out(session: AsyncSession, source_id: str) -> dict[str, Any] | None: + row = await session.get(DataSource, source_id, populate_existing=True) + return (await _out(session, row)).model_dump() if row is not None else None + + +# -------------------------------------------------------------------------- +# 列表、清单 +# -------------------------------------------------------------------------- + + +@router.get("/{source_id}/snapshots") +async def list_snapshots(source_id: str, session: AsyncSession = Depends(get_session)) -> list[dict[str, Any]]: + """最近 50 个版本，当前版本排第一（7.1）。手工源返回空列表。""" + return await source_versions.list_snapshots(session, await _source(session, source_id)) + + +@router.get("/{source_id}/snapshots/{snapshot_id}/manifest") +async def snapshot_manifest(source_id: str, snapshot_id: str, session: AsyncSession = Depends(get_session)) -> Any: + source = await _source(session, source_id) + try: + return await source_versions.snapshot_manifest_out(session, source, snapshot_id) + except _REFUSALS as e: + raise _refused(e) from e + + +@router.get("/{source_id}/imports/{import_id}/manifest") +async def import_manifest(source_id: str, import_id: str, session: AsyncSession = Depends(get_session)) -> Any: + source = await _source(session, source_id) + try: + return await source_versions.import_manifest(session, source, import_id) + except _REFUSALS as e: + raise _refused(e) from e + + +# -------------------------------------------------------------------------- +# 写操作 +# -------------------------------------------------------------------------- + + +@router.post("/{source_id}/snapshots/{snapshot_id}/activate") +async def activate(source_id: str, snapshot_id: str, request: Request, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> Any: + """启用一个还保留着的旧版本（回滚，7.2）。理由可选。""" + body = await _body(request) + _confirmed(body) + expected = _expected_current(body) + reason = _reason(body, required=False) + ack = _ack(body) + source = await _source(session, source_id) + try: + out = await source_versions.activate(session, source, snapshot_id, expected_current=expected, + ack_mask_lost=ack, reason=reason, signed_by=_signed(body, x_actor)) + except _REFUSALS as e: + raise _refused(e) from e + return {"source": await _source_out(session, source_id), **out} + + +@router.post("/{source_id}/imports/{import_id}/remove") +async def remove_period(source_id: str, import_id: str, request: Request, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> Any: + """从当前版本中移除这一期（累积模式，7.4）。配方和导入模式不变。""" + body = await _body(request) + _confirmed(body) + expected = _expected_current(body) + reason = _reason(body, required=True) + source = await _source(session, source_id) + try: + out = await source_versions.remove_period(session, source, import_id, expected_current=expected, + reason=reason or "", signed_by=_signed(body, x_actor)) + except _REFUSALS as e: + raise _refused(e) from e + return {"source": await _source_out(session, source_id), **out} + + +@router.post("/{source_id}/imports/{import_id}/revoke-acceptance") +async def revoke_acceptance(source_id: str, import_id: str, request: Request, + x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> Any: + """撤回这一期（作废接受，不可恢复，7.5）：累积模式移除这一期，替换模式回滚到 revoke_plan 的目标。""" + body = await _body(request) + _confirmed(body) + expected = _expected_current(body) + reason = _reason(body, required=True) + target = body.get("expected_target_snapshot_id") + if target is not None and not isinstance(target, str): + raise CodedHTTPException(422, "回滚目标的写法不对，请刷新后重新确认", "body_invalid") + ack = _ack(body) + source = await _source(session, source_id) + try: + out = await source_versions.revoke_acceptance( + session, source, import_id, expected_current=expected, expected_target=target, ack_mask_lost=ack, + reason=reason or "", signed_by=_signed(body, x_actor)) + except _REFUSALS as e: + raise _refused(e) from e + return {"source": await _source_out(session, source_id), **out} diff --git a/backend/app/api/table_imports.py b/backend/app/api/table_imports.py index f519a4aa..f2309bfa 100644 --- a/backend/app/api/table_imports.py +++ b/backend/app/api/table_imports.py @@ -10,6 +10,10 @@ CodedHTTPException（body 是 {"detail": 中文, "code": 机读码}，前端按 路由路径都以 /imports/ 开头，或是 /{source_id}/ 后面跟固定的 reupload、redraft、recipe，与 datasources.py 现有的 /{source_id}、/{source_id}/test|introspect|schema|imports 不重叠；在 main.py 里登记在 datasources 之前，免得 /imports/… 被通配的 {source_id} 先截走。 + +期 3（WP-5）加了 edits/preview、edits/apply、edits/undo、redraft-rules，PUT recipe 收可选的 origin；只读进程 +（没拿到版本存储的守卫）里会写版本存储的入口一律回 503 store_unavailable（第 8 节遗留项 5），body 照样是 +{detail, code}：前端把不是 JSON 的 503 当成网络断开。 """ from __future__ import annotations @@ -26,8 +30,8 @@ from app.api.coded import CodedHTTPException from app.api.datasources import _NAME_PATTERN, _out, read_upload from app.api.runs import actor_of from app.data import recipe_imports, table_versions -from app.data.recipe_imports import ImportRefused -from app.data.recipe_types import Acceptance, AiAvailability, PeriodInput, Recipe +from app.data.recipe_imports import EditRequest, ImportRefused +from app.data.recipe_types import Acceptance, AiAvailability, PeriodInput, Recipe, Selection from app.db.base import get_session from app.db.models import DataSource, ImportStaging, TableRecipe @@ -38,8 +42,14 @@ _NAME_INVALID = "名称必须以小写字母开头，只能包含字母、数字 _SIGNED_MAX = 100 -def _refused(e: ImportRefused) -> CodedHTTPException: - return CodedHTTPException(e.status, e.message, e.code) +#: 业务上的拒绝：ImportRefused 带着状态和机读码；StoreUnavailable 是只读进程拒写（503） +_REFUSALS = (ImportRefused, table_versions.StoreUnavailable) + + +def _refused(e: Exception) -> CodedHTTPException: + if isinstance(e, table_versions.StoreUnavailable): + return CodedHTTPException(503, str(e), "store_unavailable") + return CodedHTTPException(e.status, e.message, e.code) # type: ignore[attr-defined] def _signed(body: dict[str, Any] | None, header: str | None) -> str | None: @@ -167,7 +177,7 @@ async def stage( staging = await recipe_imports.stage_upload( session, raw=raw, filename=file.filename or name, name=name, description=description or "", signed_by=_signed({"signed_by": signed_by}, x_actor)) - except ImportRefused as e: + except _REFUSALS as e: raise _refused(e) from e except UnsupportedTable as e: raise _unreadable(e) from e @@ -185,7 +195,7 @@ async def discard(staging_id: str, session: AsyncSession = Depends(get_session)) staging = await _open_staging(session, staging_id) try: await recipe_imports.discard_staging(session, staging) - except ImportRefused as e: + except _REFUSALS as e: raise _refused(e) from e return Response(status_code=204) @@ -204,7 +214,7 @@ async def answers(staging_id: str, request: Request, session: AsyncSession = Dep staging = await _open_staging(session, staging_id) try: staging = await recipe_imports.answer(session, staging, given) - except ImportRefused as e: + except _REFUSALS as e: raise _refused(e) from e return await _out_of(session, staging) @@ -215,14 +225,99 @@ async def put_recipe(staging_id: str, request: Request, session: AsyncSession = recipe = body.get("recipe") if not isinstance(recipe, dict): raise CodedHTTPException(422, "配方必须是一个 JSON 对象", "body_invalid") + origin = body.get("origin") + if origin not in (None, "rules_redraft"): + raise CodedHTTPException(422, "配方来源只能不填，或填 rules_redraft（采用按规则重新起草的结果）", "body_invalid") staging = await _open_staging(session, staging_id) try: - staging = await recipe_imports.put_recipe(session, staging, recipe, origin="manual") - except ImportRefused as e: + staging = await recipe_imports.put_recipe(session, staging, recipe, origin=origin or "manual") + except _REFUSALS as e: raise _refused(e) from e return await _out_of(session, staging) +# -------------------------------------------------------------------------- +# 期 3：修复按钮、框选、撤销、按规则重新起草 +# -------------------------------------------------------------------------- + +_EDIT_SHAPE = "请求里要么给修复（fix），要么给框选（selection），二选一" + + +def _edit_request(body: dict[str, Any]) -> EditRequest: + """{"fix": {id, option, reason?}} 或 {"selection": {sheet, ref, as, options?}}，二选一（9.1）。形状不对回 422 + edit_invalid；框选的形状校验交给契约 Selection.from_json（只抛 ValueError），输出时再经 to_json 改回 as。""" + fix, sel = body.get("fix"), body.get("selection") + if (fix is None) == (sel is None): + raise CodedHTTPException(422, _EDIT_SHAPE, "edit_invalid") + if fix is not None: + if (not isinstance(fix, dict) or not isinstance(fix.get("id"), str) or not fix["id"] + or not isinstance(fix.get("option"), str) or not fix["option"]): + raise CodedHTTPException(422, "修复请求要指明是哪条修复建议（id）和选的哪一项（option）", "edit_invalid") + reason = fix.get("reason") + if reason is not None and not isinstance(reason, str): + raise CodedHTTPException(422, "理由必须是一段文字", "edit_invalid") + return EditRequest(kind="fix", fix_id=fix["id"], option=fix["option"], reason=reason) + try: + return EditRequest(kind="selection", selection=Selection.from_json(sel)) + except ValueError as e: + raise CodedHTTPException(422, str(e) or "框选的请求格式不对", "edit_invalid") from e + + +@router.post("/imports/{staging_id}/edits/preview") +async def edits_preview(staging_id: str, request: Request, session: AsyncSession = Depends(get_session)) -> Any: + """修复或框选的预览（不改工作配方）。seq 原样回传：界面只采用最新一次的回包。""" + body = await _body(request) + req = _edit_request(body) + staging = await _open_staging(session, staging_id) + try: + return await recipe_imports.edit_preview(session, staging, req, seq=body.get("seq")) + except _REFUSALS as e: + raise _refused(e) from e + + +@router.post("/imports/{staging_id}/edits/apply") +async def edits_apply(staging_id: str, request: Request, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> Any: + """应用预览过的修复或框选：选项和理由与预览时逐字相同，expected_sha256 是预览给的 recipe_sha256_after。""" + body = await _body(request) + req = _edit_request(body) + expected = body.get("expected_sha256") + staging = await _open_staging(session, staging_id) + try: + staging = await recipe_imports.edit_apply( + session, staging, req, expected_sha256=expected if isinstance(expected, str) else None, + signed_by=_signed(body, x_actor)) + except _REFUSALS as e: + raise _refused(e) from e + return await _out_of(session, staging) + + +@router.post("/imports/{staging_id}/edits/undo") +async def edits_undo(staging_id: str, request: Request, session: AsyncSession = Depends(get_session)) -> Any: + await _body(request) + staging = await _open_staging(session, staging_id) + try: + staging = await recipe_imports.edit_undo(session, staging) + except _REFUSALS as e: + raise _refused(e) from e + return await _out_of(session, staging) + + +@router.post("/imports/{staging_id}/redraft-rules") +async def redraft_rules(staging_id: str, request: Request, session: AsyncSession = Depends(get_session)) -> Any: + """按规则重新起草（不调用模型、不改工作配方），名字对齐到现行配方；「采用」走 PUT recipe（origin=rules_redraft）。""" + from app.data.tabular import UnsupportedTable + + await _body(request) + staging = await _open_staging(session, staging_id) + try: + return await recipe_imports.redraft_rules(session, staging) + except _REFUSALS as e: + raise _refused(e) from e + except UnsupportedTable as e: + raise _unreadable(e) from e + + # -------------------------------------------------------------------------- # AI 起草 # -------------------------------------------------------------------------- @@ -233,7 +328,7 @@ async def draft_ai_preview(staging_id: str, session: AsyncSession = Depends(get_ staging = await _open_staging(session, staging_id) try: return await recipe_imports.ai_preview(session, staging) - except ImportRefused as e: + except _REFUSALS as e: raise _refused(e) from e @@ -247,7 +342,7 @@ async def draft_ai(staging_id: str, request: Request, x_actor: str | None = Head staging = await recipe_imports.run_ai_draft( session, staging, consent=body.get("consent") is True, preview_sha256=sha if isinstance(sha, str) else None, signed_by=_signed(body, x_actor)) - except ImportRefused as e: + except _REFUSALS as e: raise _refused(e) from e return await _out_of(session, staging) @@ -291,7 +386,7 @@ async def trial(staging_id: str, request: Request, x_actor: str | None = Header( staging = await _open_staging(session, staging_id) try: staging = await recipe_imports.run_trial(session, staging, context_inputs=inputs, signed_by=who) - except ImportRefused as e: + except _REFUSALS as e: raise _refused(e) from e return await _out_of(session, staging) @@ -325,13 +420,15 @@ async def commit(staging_id: str, request: Request, x_actor: str | None = Header result = await recipe_imports.commit_staging( session, staging, trial_id=trial_id if isinstance(trial_id, str) else "", confirmations=confirmations, acceptances=_acceptances(body.get("acceptances"), who), signed_by=who) - except ImportRefused as e: + except _REFUSALS as e: raise _refused(e) from e source = await session.get(DataSource, result.source_id, populate_existing=True) return JSONResponse(status_code=201, content={ "source": (await _out(session, source)).model_dump() if source is not None else None, "import_id": result.import_id, "snapshot_id": result.snapshot_id, "build_id": result.build_id, "recipe_id": result.recipe_id, "build_reused": result.build_reused, "unchanged": result.unchanged, + # 期 3：同版本数据文件被改过、已用本次的文件恢复；启用后当前版本含几期；直接启用了内容相同的已有版本 + "build_restored": result.build_restored, "parts": result.parts, "snapshot_reused": result.snapshot_reused, }) @@ -360,7 +457,7 @@ async def reupload( try: staging = await recipe_imports.reupload(session, source, raw=raw, filename=file.filename or source.name, signed_by=_signed({"signed_by": signed_by}, x_actor)) - except ImportRefused as e: + except _REFUSALS as e: raise _refused(e) from e except UnsupportedTable as e: raise _unreadable(e) from e @@ -376,7 +473,7 @@ async def redraft(source_id: str, request: Request, x_actor: str | None = Header source = await _source(session, source_id) try: staging = await recipe_imports.redraft(session, source, signed_by=_signed(body, x_actor)) - except ImportRefused as e: + except _REFUSALS as e: raise _refused(e) from e except UnsupportedTable as e: raise _unreadable(e) from e diff --git a/backend/app/data/recipe.py b/backend/app/data/recipe.py index 98f8afbb..6e4d4a6d 100644 --- a/backend/app/data/recipe.py +++ b/backend/app/data/recipe.py @@ -38,6 +38,7 @@ from app.data.recipe_parsers import ( hour_range_total, looks_numeric_text, match_key, + month_day_or_date, split_unit_suffix, text_label, ) @@ -56,6 +57,7 @@ from app.data.recipe_types import ( Recipe, RecipeProblem, SumEq, + accumulate_blockers, derive_tables, prose_problems, ) @@ -267,6 +269,8 @@ _FIELD_LABEL = { "blank_rows": "空行的处理", "total_row": "合计行", "label_column": "合计标签所在的列", "merged_data": "合并单元格的处理", "grain": "主键", "units": "单位", "note": "说明", "total": "合计列", "parts": "组成列", "claims": "认领", "a": "一方", "b": "另一方", "reason": "理由", + "ignore_rows": "按行标签忽略的行", "ignore_columns": "按表头忽略的列", "ignore_outside": "按同一行文字忽略的数字", + "label": "标签", "anchor": "同一行的文字", } @@ -388,16 +392,45 @@ class _Writer: origin: str = "" +def _measures_segments(r: Recipe) -> list[MeasuresSegment]: + return [s for sheet in r.sheets for b in sheet.blocks if isinstance(b, CrosstabBlock) + for s in b.segments if isinstance(s, MeasuresSegment)] + + +def _find_measures(segs: list[MeasuresSegment], seg_id: Any, labels: list[str]) -> MeasuresSegment | None: + """按 id 找；找不到时按标签找唯一含这些标签的分段。""" + hit = next((s for s in segs if s.id == seg_id), None) + if hit is not None: + return hit + keys = {match_key(x) for x in labels} + found = [s for s in segs if keys <= {match_key(x) for x in s.labels.expect}] + return found[0] if len(found) == 1 else None + + +def _most_labels(segs: list[MeasuresSegment], labels: list[str]) -> MeasuresSegment | None: + """过半标签命中的分段，命中最多且唯一时取它（事实里有标签已被去掉、按全部标签认不出时用）。""" + keys = {match_key(x) for x in labels} + if not keys: + return None + scored = [(len(keys & {match_key(x) for x in s.labels.expect}), s) for s in segs] + best = max((n for n, _s in scored), default=0) + top = [s for n, s in scored if n == best] + return top[0] if best * 2 > len(keys) and len(top) == 1 else None + + def _fact_brief(fact: Fact) -> str: # 「第 5 行 = 第 6 行 + 第 7 行（31 列中 31 列成立）」→ 去掉末尾的成立情况，只留关系本身 return re.sub(r"（[^（）]*）\s*$", "", fact.text).strip() or fact.id class _Checker: - def __init__(self, recipe: Recipe, facts: DraftFacts | None, origin: str): + def __init__(self, recipe: Recipe, facts: DraftFacts | None, origin: str, base: Recipe | None = None): self.r = recipe self.facts = facts self.origin = origin + #: 现行配方（上传新一期、修改配方时），3.3 的「沿用已确认的常量」和「认领只查改动过的部分」用。 + #: 首次导入、从简单导入切换时为 None，照期 2 全查 + self.base = base self.data = recipe.model_dump(mode="json") self.problems: list[RecipeProblem] = [] self._seen: set[tuple[str, str]] = set() @@ -492,17 +525,23 @@ class _Checker: self.check_units() self.check_grain() self.check_list_refs() + self.check_ignores() self.check_relations() if self.facts is not None: self.check_facts() self.check_notes() return self.problems - # ---- 7. 格式版本 ---- + # ---- 7. 格式版本、导入模式 ---- def check_mode(self) -> None: - if self.r.mode == "accumulate": - self.add(("mode",), "mode_unsupported", "导入模式「按期累积」将在后续版本提供，目前只支持每期替换") + # 期 3：按期累积按资格判（契约 accumulate_blockers，起草器、累积计划共用同一份）；mode_unsupported 不再产生 + if self.r.mode != "accumulate": + return + for p in accumulate_blockers(self.r): + if (p.path, p.code) not in self._seen: + self._seen.add((p.path, p.code)) + self.problems.append(p) # ---- 8. 工作表、块、分段的键唯一 ---- @@ -792,11 +831,33 @@ class _Checker: elif not title: self.add(cp, "const_not_candidate", f"常量「{name}」只能从分段标题里选词，但这个分段没有按分段标题定位") - elif c.pick not in cands: + elif c.pick not in cands and not self.const_carried(seg.id, name, c.pick, title): self.add(cp, "const_not_candidate", f"常量「{name}」的取值「{c.pick}」不在分段标题「{title}」的候选词中，" f"可选：{'、'.join(cands) if cands else '（无）'}") + def const_carried(self, seg_id: str, name: str, pick: str, title: str) -> bool: + """3.3 的 (b)(c)：不在新标题的候选词里，但仍可以沿用。两条都要求 canon(pick) 是 canon(新标题) 的子串。 + + (b) 沿用：现行配方里同一分段（同 id）同一常量列的 pick 与它逐字相同。pick 当初是从确认过的标题的候选词里 + 选的，现在仍然逐字出现在新标题里，常量列的值不变，不是新编的文字（D09「日间分时段客流」沿用「日间」）； + (c) 重放：修复后的配方下个月按 replay 校验、又没有 base，执行时 const_missing 本来就只查子串。 + 不放宽的话 D09 只能改 pick，常量列整列换值，是破坏性变更，累积模式下还要重新开始累积。""" + if canon(pick) not in canon(title): + return False + if self.origin == "replay": + return True + if self.base is None: + return False + for sheet in self.base.sheets: + for block in sheet.blocks: + if not isinstance(block, CrosstabBlock): + continue + for seg in block.segments: + if isinstance(seg, DimensionSegment) and seg.id == seg_id and name in seg.const: + return seg.const[name].pick == pick + return False + # ---- 3. 占位符只能映射空值，不能像数 ---- def check_placeholders(self) -> None: @@ -945,6 +1006,78 @@ class _Checker: f"表头「{c.header}」与「{seen[key]}」是同一个表头") seen.setdefault(key, c.header) + # ---- 期 3：忽略规则、表头上方的标题（P3-SPEC 9.2） ---- + + def check_ignores(self) -> None: + """忽略规则的三条：ignore_conflict（同一行、同一列既导入又忽略；交叉表忽略的表头是日期）、ignore_duplicate + （同一列表里写了两遍），以及 period_literal 的扩展范围。 + + period_literal 只在 origin 不是 replay 时报：期 2 已有的配方按重放校验时不受影响；人改配方、用修复按钮或 + 框选时才拦。锚点文字含数字（「注：9月补录」「2026年8月 销售月报」）时，这条规则写进配方以后下一期必然 + 对不上，也违背「配方里不写统计期的值」的原则（评审一-M11、二-m11）。""" + literal = self.origin != "replay" + for si, sheet in enumerate(self.r.sheets): + sp = ("sheets", si) + self.unique_anchors(sp + ("ignore_outside",), [x.anchor for x in sheet.ignore_outside], "anchor") + if literal: + for n, x in enumerate(sheet.ignore_outside): + if any(ch.isdigit() for ch in x.anchor): + self.add(sp + ("ignore_outside", n, "anchor"), "period_literal", + f"按同一行文字忽略数字时用的「{x.anchor}」含数字，下一期很可能对不上" + "（配方里不能写死年份或期次）") + for bi, block in enumerate(sheet.blocks): + bp = sp + ("blocks", bi) + self.unique_anchors(bp + ("ignore_columns",), [x.header for x in block.ignore_columns], "header") + if literal: + for n, x in enumerate(block.ignore_columns): + if any(ch.isdigit() for ch in x.header): + self.add(bp + ("ignore_columns", n, "header"), "period_literal", + f"按表头忽略的列「{x.header}」含数字，下一期很可能对不上（配方里不能写死年份或期次）") + if isinstance(block, CrosstabBlock): + self.unique_anchors(bp + ("ignore_rows",), [x.label for x in block.ignore_rows], "label") + for n, x in enumerate(block.ignore_rows): + xp = bp + ("ignore_rows", n, "label") + if literal and any(ch.isdigit() for ch in x.label): + self.add(xp, "period_literal", + f"按行标签忽略的「{x.label}」含数字，下一期很可能对不上（配方里不能写死年份或期次）") + owner = self.expecting(block, x.label) + if owner is not None: + self.add(xp, "ignore_conflict", + f"按行标签忽略的「{x.label}」同时是分段「{owner}」期望的标签：同一行不能既导入又忽略") + for n, x in enumerate(block.ignore_columns): + if month_day_or_date(x.header) is not None: + self.add(bp + ("ignore_columns", n, "header"), "ignore_conflict", + f"按表头忽略的列「{x.header}」是一个日期：交叉表只能忽略日期表头右侧的列") + continue + if literal and block.after_title and any(ch.isdigit() for ch in block.after_title): + self.add(bp + ("after_title",), "period_literal", + f"表头上方的标题「{block.after_title}」含数字，下一期很可能对不上：配方里不能写死年份或期次，" + "请换一行不含数字的文字，或去掉它") + heads = {match_key(c.header): c.header for c in block.columns} + for n, x in enumerate(block.ignore_columns): + hit = heads.get(match_key(x.header)) + if hit is not None: + self.add(bp + ("ignore_columns", n, "header"), "ignore_conflict", + f"按表头忽略的列「{x.header}」与要导入的列「{hit}」是同一个表头：同一列不能既导入又忽略") + + def unique_anchors(self, parts: tuple[Any, ...], texts: list[str], field: str) -> None: + seen: dict[str, str] = {} + for n, t in enumerate(texts): + k = match_key(t) + if k in seen: + self.add(parts + (n, field), "ignore_duplicate", f"「{t}」与「{seen[k]}」是同一段文字，写了两遍") + seen.setdefault(k, t) + + def expecting(self, block: CrosstabBlock, label: str) -> str | None: + """块里期望这个标签的分段 id（按 match_key，或按该段的解析方式得到同一个规范写法）；没有为 None。""" + key = match_key(label) + for seg in block.segments: + own = self.label_key(seg, label) + for e in seg.labels.expect: + if match_key(e) == key or (own is not None and self.label_key(seg, e) == own): + return seg.id + return None + # ---- 9. 关系 ---- def check_relations(self) -> None: @@ -998,40 +1131,117 @@ class _Checker: def check_facts(self) -> None: assert self.facts is not None facts = {f.id: f for f in self.facts.facts} + scope = self.changed_scope() + # scope 为 None：首次导入、从简单导入切换，照期 2 全查。否则只查改动过的关系、成员落在改动过的 + # measures 段上的关系，以及落在改动过的 measures 段上的事实（3.3 第 11 条，评审一-M3） + rels = set(range(len(self.r.relations))) if scope is None else self.relations_in(scope) claimers: dict[str, list[int]] = {} for i, rel in enumerate(self.r.relations): if rel.claims is None: continue if rel.claims not in facts: - self.add(("relations", i, "claims"), "fact_claim_mismatch", - f"认领的系统发现「{rel.claims}」不存在：请去掉认领，或按本次发现重新登记") + if i in rels: + self.add(("relations", i, "claims"), "fact_claim_mismatch", + f"认领的系统发现「{rel.claims}」不存在：请去掉认领，或按本次发现重新登记") continue claimers.setdefault(rel.claims, []).append(i) for fid, fact in facts.items(): got = claimers.get(fid, []) - if not got: + in_scope = scope is None or self.fact_in(fact, scope) + if not got and in_scope: self.add(("relations",), "fact_unclaimed", f"系统发现的关系「{_fact_brief(fact)}」没有被认领：请登记为每期核对，或说明不登记的理由", where=False) for j in got[1:]: - self.add(("relations", j, "claims"), "fact_claimed_twice", - f"系统发现的关系「{_fact_brief(fact)}」已经由关系「{self.r.relations[got[0]].id}」认领，" - "每条只能认领一次") + if in_scope or j in rels: + self.add(("relations", j, "claims"), "fact_claimed_twice", + f"系统发现的关系「{_fact_brief(fact)}」已经由关系「{self.r.relations[got[0]].id}」认领，" + "每条只能认领一次") for j in got: + if j not in rels: + continue why = self.claim_mismatch(self.r.relations[j], fact) if why: self.add(("relations", j, "claims"), "fact_claim_mismatch", f"认领了系统发现的关系「{_fact_brief(fact)}」，但{why}") + def changed_scope(self) -> tuple[set[str], set[str]] | None: + """与现行配方（base）相比改动过的 (measures 分段 id, 关系 id)；base 为空时 None（全查）。 + + measures 段：同 id 的段 labels.expect 集合、measures 映射、locate 任一不同，或者是新段。 + 关系：同 id 的关系任一字段不同，或者是新关系。 + 理由：用过修复按钮以后工作配方的哈希变了，静态校验改成带本期系统发现。本期 R1 有某天不成立时 _sum_holds + 不出 sum_eq 事实，没改过的 R1 就会报「认领的系统发现不存在」，D4「每期写理由接受」变成永久的配方改动。 + 没改过的部分按重放处理：R1 本期成不成立交给试运行的 R 核对（数据质量类，可以写理由接受）。""" + if self.base is None: + return None + + def measures(r: Recipe) -> dict[str, MeasuresSegment]: + return {s.id: s for s in _measures_segments(r)} + + old = measures(self.base) + segs: set[str] = set() + for sid, seg in measures(self.r).items(): + o = old.get(sid) + if (o is None or set(o.labels.expect) != set(seg.labels.expect) or o.measures != seg.measures + or o.locate != seg.locate): + segs.add(sid) + old_rels = {r.id: r.model_dump(mode="json") for r in self.base.relations} + rels = {r.id for r in self.r.relations if old_rels.get(r.id) != r.model_dump(mode="json")} + return segs, rels + + def relations_in(self, scope: tuple[set[str], set[str]]) -> set[int]: + """要查认领的关系下标：改动过的，以及成员（表 + 列）落在改动过的 measures 段写入的列上的。""" + segs, rels = scope + cols = {(s.table, c) for *_rest, s in self.segs if isinstance(s, MeasuresSegment) and s.id in segs + for c in s.measures.values()} + out: set[int] = set() + for i, rel in enumerate(self.r.relations): + if rel.id in rels: + out.add(i) + elif isinstance(rel, SumEq): + if any((rel.table, c) in cols for c in (rel.total, *rel.parts)): + out.add(i) + elif isinstance(rel, NotComparable): + if (rel.a.table, rel.a.value) in cols or (rel.b.table, rel.b.value) in cols: + out.add(i) + return out + + def fact_in(self, fact: Fact, scope: tuple[set[str], set[str]]) -> bool: + """事实是否落在改动过的 measures 段上。 + + 先在工作配方里按 id、按全部标签认段（measures_seg）。认不出时多半是这一段刚被改过：① 去掉了「分区乙」， + 本期的事实「全日客流 = 分区甲 + 分区乙」还在，分段 id 又是规则起草的写法（remap_facts 要全部标签对上才 + 换 id），按 id、按全部标签都认不出。只到这一步就判「不在范围内」的话，这条事实连同认领它的关系被删掉以后， + 既不报未认领、也没有理由，数据质量核对不声不响地消失（3.3 第 11 条要查的正是这种改动）。所以再按两步认： + 1. 在现行配方（base）里按 id、按全部标签认段：认出的段在工作配方里改动过或已删掉，就在范围内； + 2. 在工作配方里按「过半标签命中」认段（命中最多且唯一）。 + 两步都只会把事实多算进范围（多查一条认领），不会漏查。""" + d = fact.detail or {} + if fact.kind == "sum_eq": + total, parts = d.get("total"), d.get("parts") + labels = [total, *parts] if isinstance(total, str) and isinstance(parts, list) else [] + seg_id = d.get("segment") + else: + b = d.get("b") + labels, seg_id = ([b] if isinstance(b, str) else []), d.get("b_segment") + labels = [x for x in labels if isinstance(x, str)] + seg = self.measures_seg(seg_id, labels) + if seg is not None: + return seg.id in scope[0] + if self.base is not None: + old = _find_measures(_measures_segments(self.base), seg_id, labels) + if old is not None: + return old.id in scope[0] or old.id not in {s.id for s in self.measures_list()} + seg = _most_labels(self.measures_list(), labels) + return seg is not None and seg.id in scope[0] + + def measures_list(self) -> list[MeasuresSegment]: + return [s for *_rest, s in self.segs if isinstance(s, MeasuresSegment)] + def measures_seg(self, seg_id: Any, labels: list[str]) -> MeasuresSegment | None: """Fact 指向的 measures 分段：按 id 找；改过分段名找不到时，按标签找唯一含这些标签的分段。""" - segs = [s for *_rest, s in self.segs if isinstance(s, MeasuresSegment)] - hit = next((s for s in segs if s.id == seg_id), None) - if hit is not None: - return hit - keys = {match_key(x) for x in labels} - found = [s for s in segs if keys <= {match_key(x) for x in s.labels.expect}] - return found[0] if len(found) == 1 else None + return _find_measures(self.measures_list(), seg_id, labels) def dimension_segs(self, seg_ids: list[str], near: MeasuresSegment | None) -> list[DimensionSegment]: """not_equal_sum 的 a 指向的 dimension 分段：按 id 找；有改过名找不到的，取与 b 那个 measures 段同块的全部 dimension 段。 @@ -1128,11 +1338,14 @@ def validate_recipe( data: dict[str, Any] | Recipe, *, facts: DraftFacts | None = None, origin: Origin = "manual", + base: Recipe | dict[str, Any] | None = None, ) -> tuple[Recipe | None, list[RecipeProblem]]: """静态校验，不抛异常。schema 不过时返回 (None, problems)；语义问题返回 (recipe, problems)。 facts 给了（首次导入、改配方）才查认领；重放（origin="replay"）不传。origin="ai" 时 note 必须为空。 - problems 为空才可试运行。 + base（期 3）是现行配方（数据源当前的 current_recipe_id 对应的那份，canonical 或完整形式都行），上传新一期、 + 修改配方时传：常量沿用现行配方里逐字相同的 pick 时放行（3.3 (b)），认领只查改动过的部分（3.3 第 11 条）。 + base 本身不合 schema 时按没给处理（它是已提交过的配方，正常不会出现）。problems 为空才可试运行。 """ if isinstance(data, Recipe): recipe = data @@ -1141,4 +1354,12 @@ def validate_recipe( recipe = parse_recipe(data) except RecipeInvalid as exc: return None, exc.problems - return recipe, _Checker(recipe, facts, origin).run() + base_recipe: Recipe | None = None + if isinstance(base, Recipe): + base_recipe = base + elif isinstance(base, dict): + try: + base_recipe = Recipe.model_validate(base) + except ValidationError: + base_recipe = None + return recipe, _Checker(recipe, facts, origin, base_recipe).run() diff --git a/backend/app/data/recipe_accumulate.py b/backend/app/data/recipe_accumulate.py new file mode 100644 index 00000000..5f6a8b54 --- /dev/null +++ b/backend/app/data/recipe_accumulate.py @@ -0,0 +1,924 @@ +"""按期累积（期 3，WP-4）：配方演进的分档、累积计划、并集的物化（P3-SPEC 2.4–2.7）。 + +**为什么物化，不用视图。** 对外的表是各期最新分区的并集。视图（final.md R1 的否定结论）没法以 immutable 只读打开、 +没法按文件哈希核对，运行固定的版本也就说不清查的是哪一份数据。所以并集是一个真正的 STRICT + PRIMARY KEY 的 +SQLite 文件，和期 1、期 2 的快照库一样内容寻址、只读、按哈希核对；它也登记成一个构建（table_versions）。 + +**为什么在试运行里物化。** 用户启用前看到的行数、结构核对和说明，就是启用后模型看到的那一份（H2）；提交在版本 +存储锁里只做文件 link 和写记录，不在锁里花几秒物化。所以这里的函数都是同步的，调用方放进线程池并占 parse_slot。 + +**确定性。** 写入顺序固定（期按统计期排序，表按列序，行按 rowid），同样的输入物化两次库文件哈希相同。publish_snapshot +复活已回收的快照时靠这一条：重新物化出的文件哈希必须等于登记值。 + +这里不碰数据库记录、不碰数据源：PartInfo 由 WP-5 组装（统计期取自各期导入清单），计划和报告原样交回给它。 +""" +from __future__ import annotations + +import datetime as _dt +import os +import secrets +import sqlite3 +from pathlib import Path +from typing import Any, Iterable +from urllib.parse import quote + +from app.data import raw_store +from app.data.engine import open_checked_sqlite +from app.data.names import collide_key +from app.data.recipe_confirm import table_changes +from app.data.recipe_parsers import hour_range, hour_range_total, match_key, text_label +from app.data.recipe_types import ( + UNION_VER, AccumulatePlan, ChangeClass, CheckResult, ColumnOut, CrosstabBlock, DerivedSegment, + DimensionSegment, PartInfo, Recipe, UnionPart, UnionReport, accumulate_blockers, derive_tables, +) +from app.data.table_versions import settle_journal, sha_json + +#: table_changes 报出这些 kind 就是不兼容：单位、类型、常量取值只要和任何一期不同，同一列里就混了两种口径 +#: （P3-SPEC 2.5）。table_removed / column_removed 不在其中：按「当前版本的表结构」另判退役； +#: placeholder_meaning 属于兼容（占位符本来就存空值，只是说明的写法变了） +INCOMPATIBLE_KINDS = ("type", "unit", "grain", "table_kind", "const_value", "source", "store") + + +class UnionError(ValueError): + """物化失败。code：part_missing（某一期的构建库不在了）、part_tampered（某一期的库哈希对不上，物化第 0 步）、 + union_failed（目标表结构推不出来、某一期的 rowid 不连续、某一期有并集里没有位置的表或列；U1–U3 不通过时 + materialize_union 不抛它、只返回 ok=False 的报告，要异常的调用方用 require_union_ok）。 + part 是出问题的那一期的序号（从 1 起，按统计期排序），与哪一期无关时为 None。message 给人看。""" + + def __init__(self, message: str, *, code: str, part: int | None = None) -> None: + super().__init__(message) + self.code = code + self.part = part + + +# ========================================================================== +# 小工具 +# ========================================================================== + + +def _q(name: str) -> str: + return '"' + str(name).replace('"', '""') + '"' + + +def _period_text(start: str | None, end: str | None) -> str: + return f"{start} 至 {end}" if start and end else "统计期未记录" + + +def _period_key(start: str | None, end: str | None) -> str: + """快照清单、报告里一期的写法：「2026-08-01~2026-08-31」。""" + return f"{start}~{end}" + + +def _day(text: str) -> _dt.date: + return _dt.date.fromisoformat(str(text)) + + +def _intersects(a: tuple[str | None, str | None], b: tuple[str | None, str | None]) -> bool: + """两个统计期有没有交集（ISO 日期按字符串比较即可）。统计期不全的不算相交：替换模式下的旧一期可能没有统计期。""" + if not (a[0] and a[1] and b[0] and b[1]): + return False + return not (a[1] < b[0] or b[1] < a[0]) + + +def _gaps(periods: list[tuple[str, str]]) -> list[dict[str, str]]: + """相邻两期之间的空缺：前一期 end 加一天 < 后一期 start。periods 已按统计期排序。""" + out: list[dict[str, str]] = [] + for (_s1, e1), (s2, _e2) in zip(periods, periods[1:]): + first, last = _day(e1) + _dt.timedelta(days=1), _day(s2) - _dt.timedelta(days=1) + if first <= last: + out.append({"start": first.isoformat(), "end": last.isoformat()}) + return out + + +def _dump(recipe: Recipe) -> dict[str, Any]: + return recipe.model_dump(mode="json") + + +def overlap_message(period: dict[str, Any], overlaps: Iterable[dict[str, Any]]) -> str: + """period_overlap 问题的消息（P3-SPEC 2.3）。WP-5 组装试运行问题时用，免得两边各写一套文字。""" + others = "、".join(_period_text(o.get("start"), o.get("end")) for o in overlaps) + return (f"本期统计期 {_period_text(period.get('start'), period.get('end'))} 与已有的 {others} 部分重叠：按期累积要求" + "各期互不重叠。请检查文件的统计期；如需重新开始累积，可以在数据源卡片的「版本」中启用更早的版本，" + "或在配方中改为每期替换") + + +# ========================================================================== +# 配方演进：兼容、退役、不兼容（P3-SPEC 2.5） +# ========================================================================== + + +def _tables_of_parts(parts: list[PartInfo]) -> dict[str, list[ColumnOut]]: + """没有当前版本的实际表结构时，退回各期配方推出的表结构的并集（按首次出现的顺序）。""" + out: dict[str, list[ColumnOut]] = {} + for part in parts: + if part.recipe is None: + continue + tables, _ = derive_tables(part.recipe) + for name, cols in tables.items(): + have = out.setdefault(name, []) + known = {c.name for c in have} + have.extend(c for c in cols if c.name not in known) + return out + + +def _label_canon(seg: Any, label: str) -> str: + """维度、派生分段的期望标签 → 解析器的规范写法（执行器存进库里的就是它）。解析不了的退回 match_key。""" + if isinstance(seg, DerivedSegment): + parsed = hour_range_total(label) + return parsed.canonical if parsed is not None else match_key(label) + if seg.dim.parser == "hour_range": + parsed = hour_range(label) + return parsed.canonical if parsed is not None else match_key(label) + return text_label(label) or match_key(label) + + +def _dim_segments(recipe: Recipe) -> list[tuple[Any, str, str]]: + """配方里写进表的维度、派生分段：(分段, 表, 维度列)。只核对不另存的派生分段不进表，不算。""" + out: list[tuple[Any, str, str]] = [] + for sheet in recipe.sheets: + for block in sheet.blocks: + if not isinstance(block, CrosstabBlock): + continue + for seg in block.segments: + if isinstance(seg, DimensionSegment): + out.append((seg, seg.table, seg.dim.name)) + elif isinstance(seg, DerivedSegment) and seg.keep_as is not None: + out.append((seg, seg.keep_as.table, seg.keep_as.dim)) + return out + + +def _label_sets(parts: list[PartInfo]) -> list[dict[str, Any]]: + """各期维度取值的差异（评审一-M2）：按分段 id 比较各期配方里 dimension、derived 段的期望标签（规范写法， + 按 match_key 比）。只在含这个分段的各期之间比；集合都相同就不列。 + + missing 是相对全部各期并集少了哪些，extra 是比全部各期都有的（交集）多了哪些：D04 去掉「7-8」以后， + 9 月的 missing 是「7-8」，8 月的 extra 是「7-8」。集合不同不算不兼容（table_changes 为空），但跨期比较会 + 悄悄变得不可比，所以记进计划和快照清单，说明加 dim_values_vary。 + """ + seen: dict[str, dict[str, Any]] = {} + for part in parts: + if part.recipe is None: + continue + for seg, table, column in _dim_segments(part.recipe): + entry = seen.setdefault(seg.id, {"table": table, "column": column, "periods": []}) + labels: dict[str, str] = {} + for label in seg.labels.expect: + canon = _label_canon(seg, label) + labels.setdefault(match_key(canon), canon) + entry["periods"].append((part, labels)) + out: list[dict[str, Any]] = [] + for sid, entry in seen.items(): + periods = entry["periods"] + if len(periods) < 2: + continue + order: dict[str, str] = {} + for _part, labels in periods: + for key, canon in labels.items(): + order.setdefault(key, canon) + common = set.intersection(*(set(labels) for _part, labels in periods)) + if all(set(labels) == set(order) for _part, labels in periods): + continue + out.append({ + "table": entry["table"], "column": entry["column"], "segment": sid, + "periods": [{ + "start": part.start, "end": part.end, + "missing": [canon for key, canon in order.items() if key not in labels], + "extra": [canon for key, canon in order.items() if key in labels and key not in common], + } for part, labels in periods], + }) + return out + + +def classify_changes(parts: list[PartInfo], target: Recipe, *, + current_tables: dict[str, list[ColumnOut]] | None, + retired_status: dict[str, dict[str, dict]] | None = None) -> ChangeClass: + """目标配方相对结果快照各期、当前版本的变化分三档（P3-SPEC 2.5，评审二-M3 后改）。 + + 1. **不兼容**：拿目标配方和 parts 里**每一期的配方**分别调 table_changes，出现 INCOMPATIBLE_KINDS 就是不兼容。 + 逐期比是因为单位、类型、常量取值只要和任何一期不同，同一列里就混了两种口径。 + 2. **新增和退役**：拿目标配方的表结构和**当前版本实际的表结构**（current_tables，含早先已退役、仍保留在并集 + 里的列；为 None 时退回各期配方推出的表结构的并集）比。当前有、目标没有的是退役，其中 retired_status + （当前快照清单的 columns）里早已标为 retired 的记进 retired_existing、不再出确认项，这次才退役的记进 + retired_new。不这样分的话，8 月的配方永远含「分区乙」，以后每个月都会再报一次退役。整张表退役：这张表在 + retired_status 里的每一列都已标 retired，才算早已退役。 + 新增的表名、列名和当前版本里退役的名字 collide_key 相同、写法不同（「PM2_5」退役、「pm2_5」新增）时按 + 不兼容处理：SQLite 的名字不分大小写，并集建表会报重名。 + + label_sets 只在 parts 之间比：要把本期算进去，调用方把本期也作为一期传进来（plan_accumulate 就这么做）。 + change：有不兼容为 semantic；否则有这次才退役的为 retire；否则配方与各期都相同、也没有新增为 same；其余 compatible。 + """ + semantic: list[str] = [] + + def _say(text: str) -> None: + if text not in semantic: + semantic.append(text) + + for k, part in enumerate(parts, 1): + if part.recipe is None: + _say(f"第 {k} 期（{_period_text(part.start, part.end)}）没有配方记录，无法比较表结构") + continue + for ch in table_changes(part.recipe, target): + if ch.kind in INCOMPATIBLE_KINDS: + _say(f"表「{ch.table}」：{ch.message}") + target_tables, problems = derive_tables(target) + for p in problems: + _say(p.message) + current = current_tables if current_tables is not None else _tables_of_parts(parts) + status = retired_status or {} + + added: list[dict[str, Any]] = [] + retired_new: list[dict[str, Any]] = [] + retired_existing: list[dict[str, Any]] = [] + gone_tables: dict[str, str] = {} + for name, cols in current.items(): + info = status.get(name) or {} + tgt = target_tables.get(name) + if tgt is None: + gone_tables[collide_key(name)] = name + names = [c.name for c in cols] + done = bool(names) and all((info.get(c) or {}).get("status") == "retired" for c in names) + (retired_existing if done else retired_new).append({"table": name, "column": None}) + continue + keep = {c.name for c in tgt} + for col in cols: + if col.name not in keep: + entry = {"table": name, "column": col.name} + done = (info.get(col.name) or {}).get("status") == "retired" + (retired_existing if done else retired_new).append(entry) + for name, cols in target_tables.items(): + have = current.get(name) + if have is None: + added.append({"table": name, "column": None}) + other = gone_tables.get(collide_key(name)) + if other is not None and other != name: + _say(f"新增的表「{name}」与当前版本里退役的表「{other}」只差大小写或全半角：数据库中的表名不分大小写，" + "两张表不能并存") + continue + names = {c.name for c in have} + gone = {collide_key(c.name): c.name for c in have if c.name not in {x.name for x in cols}} + for col in cols: + if col.name in names: + continue + added.append({"table": name, "column": col.name}) + other = gone.get(collide_key(col.name)) + if other is not None and other != col.name: + _say(f"表「{name}」新增的列「{col.name}」与退役的列「{other}」只差大小写或全半角：数据库中的列名不分" + "大小写，两列不能并存") + + label_sets = _label_sets(parts) + target_dump = _dump(target) + same = all(p.recipe is not None and _dump(p.recipe) == target_dump for p in parts) + if semantic: + change = "semantic" + elif retired_new: + change = "retire" + elif same and not added and not label_sets: + change = "same" + else: + change = "compatible" + return ChangeClass(change=change, added=added, retired_new=retired_new, retired_existing=retired_existing, + semantic=semantic, label_sets=label_sets) + + +# ========================================================================== +# 累积计划（P3-SPEC 2.4） +# ========================================================================== + + +def _part_dict(part: PartInfo, blockers: list[str]) -> dict[str, Any]: + return {"import_id": part.import_id, "seq": part.seq, "start": part.start, "end": part.end, + "file_name": part.file_name, "new": False, "rows": dict(part.rows), "blockers": list(blockers)} + + +def plan_accumulate(*, new_recipe: Recipe, new_period: tuple[str, str], period_source: str, + staging_kind: str, current_mode: str | None, current_snapshot_kind: str | None, + parts: list[PartInfo], current_tables: dict[str, list[ColumnOut]] | None, + retired_status: dict[str, dict[str, dict]] | None = None) -> AccumulatePlan: + """这次导入启用后，当前版本由哪几期组成（P3-SPEC 2.4 的判定表）。不物化：union 留给调用方物化后填。 + + - 新配方是每期替换：replace，结果只有本期；当前是累积快照时 mode_switch=accumulate->replace，各期列进 dropped。 + - 新配方是按期累积： + - 没有当前快照、当前是简单导入、首次导入或从简单导入切换：first； + - 现行也是累积：先判不兼容（restart 优先于 rejected，评审一-m12）——目标配方对结果快照里留下的任何一期 + 不兼容，或者留下的某一期的配方不满足累积资格（parts[].blockers），restart，结果只有本期，此前各期列进 + dropped，部分重叠的各期写进 overlaps 只作提示。否则按统计期判：部分重叠 rejected；起止都相同 + replace_period；不相交 append（早于已有的某一期时 backfill）。修改配方、不换文件（staging_kind=redraft） + 用的就是最晚一期的原件，替换最晚一期（replace_period）； + - 现行是替换（模式切换 replace->accumulate）：当前那一期有统计期、满足资格、配方兼容时作为第一期保留 + （append / replace_period）；没有统计期或不满足资格时 first（reason 写明），不兼容时 restart。 + + period 除了 start、end 还带 source（period_source：cells / human）：补传早期而本期统计期是人工录入时，差异卡的 + period_backfill 要改为需确认（2.3，评审一-m12），WP-3 只拿得到计划，从这里读。 + parts 里本期标 new: true；本期的 file_name、rows 这里不知道（签名里没有），留空，调用方按试运行回执补上。 + 结果只剩本期时不比新增、退役（没有早期各期可保留，列就跟着本期走，破坏性变化由期 2 的 breaking 确认项兜住）， + label_sets 也为空（restart 的结果只有本期）。rejected 时 parts 是原样不动的当前各期，label_sets 为空。 + current_tables 为 None 时按当前快照各期的配方推出表结构（含要被替换的那一期），新增、退役照样比得出来； + 早已退役的列只在真实的表结构里有，所以有当前快照时调用方应当给 current_tables。 + """ + start, end = (new_period or (None, None))[0], (new_period or (None, None))[1] + period = {"start": start, "end": end, "source": period_source} + ordered = sorted(parts, key=lambda p: (p.start or "", p.end or "", p.seq or 0)) + blockers: dict[int, list[str]] = {} + for p in ordered: + blockers[id(p)] = ([b.message for b in accumulate_blockers(p.recipe)] if p.recipe is not None + else ["这一期没有配方记录"]) + new_entry = {"import_id": None, "seq": None, "start": start, "end": end, "file_name": "", "new": True, + "rows": {}, "blockers": []} + plan = AccumulatePlan( + mode=new_recipe.mode, action="replace", period=period, parts=[new_entry], replaces=None, dropped=[], + overlaps=[], gaps=[], backfill=False, change="same", added=[], retired_new=[], retired_existing=[], + semantic=[], label_sets=[], mode_switch=None, reason=None, union=None, + ) + + def entry(p: PartInfo) -> dict[str, Any]: + return _part_dict(p, blockers[id(p)]) + + if new_recipe.mode != "accumulate": + if current_mode == "accumulate" and current_snapshot_kind == "recipe" and ordered: + plan.mode_switch = "accumulate->replace" + plan.dropped = [entry(p) for p in ordered] + return plan + + if not start or not end: + raise ValueError("按期累积需要本期的统计期：统计期解析不出来时先由人录入，再做累积计划") + new_blockers = [b.message for b in accumulate_blockers(new_recipe)] + if new_blockers: + # 静态校验已经拦过（mode=accumulate 时报 accumulate_blockers），走到这里说明配方没过校验就试运行了 + plan.action, plan.reason, plan.parts = "rejected", new_blockers[0], [entry(p) for p in ordered] + return plan + if (staging_kind in ("first", "switch") or current_snapshot_kind in (None, "simple") or current_mode is None + or not ordered): + plan.action = "first" + return plan + + new_part = PartInfo(import_id="", seq=0, start=start, end=end, build_id="", db_path="", db_sha256="", + manifest_db_sha256=None, recipe_id=None, recipe=new_recipe, recipe_sha256=None, + raw_state="kept", file_name="", manifest_artifact=None) + switching = current_mode != "accumulate" + if switching: + plan.mode_switch = "replace->accumulate" + cur = ordered[-1] + if not (cur.start and cur.end) or blockers[id(cur)]: + plan.action = "first" + plan.dropped = [entry(p) for p in ordered] + if not (cur.start and cur.end): + plan.reason = "当前版本没有统计期，不能作为第一期：启用后当前版本只含本期" + else: + plan.reason = (f"当前版本的配方不满足按期累积的要求（{blockers[id(cur)][0]}），不能作为第一期：" + "启用后当前版本只含本期") + return plan + + if staging_kind == "redraft": + replaced: PartInfo | None = ordered[-1] + kept = ordered[:-1] + clash = [p for p in kept if _intersects((p.start, p.end), (start, end))] + else: + replaced = next((p for p in ordered if (p.start, p.end) == (start, end)), None) + kept = [p for p in ordered if p is not replaced] + clash = [p for p in kept if _intersects((p.start, p.end), (start, end))] + + # 没给当前版本的实际表结构时，按当前快照各期（含要被替换的那一期，它此刻仍在当前版本里）的配方推。不能让 + # classify_changes 自己退回它的 parts：那里面有本期，本期的列也算成「当前已有」，新增列就永远报不出来 + current = current_tables if current_tables is not None else _tables_of_parts(ordered) + cc = classify_changes([*kept, new_part], new_recipe, current_tables=current, retired_status=retired_status) + plan.change, plan.semantic, plan.label_sets = cc.change, cc.semantic, cc.label_sets + if kept: + plan.added, plan.retired_new, plan.retired_existing = cc.added, cc.retired_new, cc.retired_existing + elif cc.change == "retire": + plan.change = "compatible" + blocked = [(k, p) for k, p in enumerate(ordered, 1) if p in kept and blockers[id(p)]] + if cc.semantic or blocked: + plan.action = "restart" + plan.dropped = [entry(p) for p in ordered] + plan.overlaps = [entry(p) for p in clash] + if cc.semantic: + plan.reason = "表结构有不兼容的变化：" + "；".join(cc.semantic) + else: + k, p = blocked[0] + plan.reason = (f"第 {k} 期（{_period_text(p.start, p.end)}）的配方不满足按期累积的要求：" + f"{blockers[id(p)][0]}") + # 结果只有本期：没有早期各期可保留，新增、退役、各期维度取值的差异都无从谈起。label_sets 不清的话， + # 差异卡会说「本期没有「7-8」，此前各期有」，可那些期并不在新版本里 + plan.added, plan.retired_new, plan.retired_existing, plan.label_sets = [], [], [], [] + if plan.change == "retire": + plan.change = "semantic" + return plan + if clash: + plan.action = "rejected" + plan.overlaps = [entry(p) for p in clash] + plan.parts = [entry(p) for p in ordered] + # 拒收时没有结果版本，本期与各期的取值差异没有落点 + plan.label_sets = [] + return plan + + result = sorted([*kept, new_part], key=lambda p: (p.start or "", p.end or "")) + plan.parts = [new_entry if p is new_part else entry(p) for p in result] + if replaced is not None: + plan.action, plan.replaces = "replace_period", entry(replaced) + else: + plan.action = "append" + plan.backfill = any((p.start or "") > start for p in kept) + plan.gaps = _gaps([(p.start, p.end) for p in result if p.start and p.end]) + return plan + + +# ========================================================================== +# 并集 id 与物化（P3-SPEC 2.1、2.6） +# ========================================================================== + + +def retired_columns(change: Any, current_tables: dict[str, list[ColumnOut]] | None) -> dict[str, list[ColumnOut]]: + """把计划（AccumulatePlan）或 classify_changes 的结果里的退役项，换成 materialize_union 的 retired 参数。 + + 顺序有讲究（P3-SPEC 2.6「退役列按退役先后追加」）：**先 retired_existing，再 retired_new**，各自按 + current_tables 里的列序。当前版本的表结构里，早先退役的列本来就排在目标配方的列之后、按退役先后排着；这次才 + 退役的列原是目标配方里的列，排在前面。照 current_tables 的顺序直接拼，新退役的就跑到早先退役的前面去了。 + 退役列的顺序决定并集的建表语句，也就决定库文件的哈希：复活时重新物化要得到同一个哈希，同一组输入就必须 + 拼出同一个顺序，所以由这里统一拼，调用方不要自己拼。整张表退役（column 为 None）给这张表的全部列。 + """ + tables = current_tables or {} + out: dict[str, list[ColumnOut]] = {} + for entry in [*(getattr(change, "retired_existing", None) or []), *(getattr(change, "retired_new", None) or [])]: + table, column = entry.get("table"), entry.get("column") + cols = tables.get(table) or [] + picks = list(cols) if column is None else [c for c in cols if c.name == column] + have = out.setdefault(table, []) + known = {c.name for c in have} + for col in picks: + if col.name not in known: + have.append(_copy_col(col)) + known.add(col.name) + if not have: + del out[table] + return out + + +def retired_layout(retired: dict[str, list[ColumnOut]] | None) -> list[list[Any]]: + """退役列的规范写法 [[表, [[列, 类型], …]], …]，**保留给定的顺序**（表的先后、列的先后都进库文件的哈希）， + 同一张表里重复的列只记第一次，没有列的表不记。并集 id、并集构建的 options 都用它。""" + out: list[list[Any]] = [] + for table, cols in (retired or {}).items(): + seen: set[str] = set() + listed: list[list[str]] = [] + for col in cols: + if col.name in seen: + continue + seen.add(col.name) + listed.append([str(col.name), str(col.type)]) + if listed: + out.append([str(table), listed]) + return out + + +def _ordered_parts(parts: list[tuple[str, str, str]]) -> list[list[str]]: + return sorted(([str(s), str(e), str(b)] for s, e, b in parts), key=lambda x: (x[0], x[1])) + + +def union_id(source_id: str, parts: list[tuple[str, str, str]], target_recipe_sha: str, *, + retired: dict[str, list[ColumnOut]] | None = None) -> str: + """并集 id = sha_json({source_id, parts: [[起, 止, 构建 id]…] 按统计期排序, 目标配方哈希, union_ver})， + 有退役列时再加 "retired": retired_layout(retired)。 + + 只取决于各期构建和并集的表结构，**不含导入记录 id**：同一个 9 月文件换个文件名再传，导入记录是新的， + 并集文件照样复用。 + + **retired 必须和传给 materialize_union 的是同一个**（用 retired_columns 拼）。退役列是有历史的：同样的各期、 + 同样的目标配方，在含「分区乙」的历史上移除某一期，并集里留着全为空值的「分区乙」；回滚之后重新导入同样两个 + 文件，就没有这一列。id 不含退役列的话，两份结构不同的文件同一个 id，发布时目标已存在、哈希等于登记值就直接 + 复用旧文件，用户在试运行里核对的和真正发布的不是同一份（H2）。没有退役列时 id 与不带这个键时相同。 + """ + payload: dict[str, Any] = {"source_id": source_id, "parts": _ordered_parts(parts), + "target_recipe_sha256": target_recipe_sha, "union_ver": UNION_VER} + layout = retired_layout(retired) + if layout: + payload["retired"] = layout + return sha_json(payload) + + +def union_options(parts: list[tuple[str, str, str]], target_recipe_sha: str, *, + retired: dict[str, list[ColumnOut]] | None = None) -> dict[str, Any]: + """并集构建登记的 options（TableBuild.options，P3-SPEC 2.1）：{"kind": "union", "parts", "target_recipe_sha256"}， + 有退役列时加 "retired"。与 union_id 的输入一一对应，管理员按 options 就能复核 id 是怎么来的。""" + options: dict[str, Any] = {"kind": "union", "parts": _ordered_parts(parts), + "target_recipe_sha256": target_recipe_sha} + layout = retired_layout(retired) + if layout: + options["retired"] = layout + return options + + +class _Table: + """并集里的一张表：列（目标配方的在前、退役的按给定顺序追加在后）、主键、日期轴列。""" + + def __init__(self, name: str, cols: list[ColumnOut], grain: list[str]) -> None: + self.name = name + self.cols = cols + self.grain = grain + self.axis = next((c.name for c in cols if c.role == "axis"), None) + self.names = [c.name for c in cols] + + +def _copy_col(col: ColumnOut) -> ColumnOut: + return ColumnOut(name=col.name, type=col.type, header=col.header, unit=col.unit, role=col.role) + + +def _layout(target: Recipe, parts: list[UnionPart], + retired: dict[str, list[ColumnOut]] | None) -> dict[str, _Table]: + """并集的表结构：derive_tables(target) 的列序，加上退役列。退役列、整张退役的表，类型、单位、表头、主键 + 取最后一个还含它的那一期的配方（「类型取最后一个含它的那一期」，说明的单位也要用它）。 + + 退役列按 retired 给定的顺序追加，整张退役的表按 retired 的顺序排在目标配方的表之后。**这个顺序进建表语句， + 也就进库文件的哈希**：retired 要用 retired_columns 拼（先早先退役的、再这次才退役的），union_id 要带上同一个 + retired，否则同一个并集 id 会对应结构不同的文件，复活时重新物化也对不上登记的哈希（build_conflict）。""" + tables, problems = derive_tables(target) + if problems: + raise UnionError("目标配方推不出表结构：" + "；".join(p.message for p in problems[:3]), code="union_failed") + specs = {t.name: t for t in target.tables} + derived = [(derive_tables(p.recipe)[0], {t.name: t for t in p.recipe.tables}) for p in parts] + + def last_with(table: str, column: str | None) -> tuple[ColumnOut | None, list[str] | None]: + for cols, pspecs in reversed(derived): + have = cols.get(table) + if have is None: + continue + grain = list(pspecs[table].grain) if table in pspecs else [] + if column is None: + return None, grain + found = next((c for c in have if c.name == column), None) + if found is not None: + return found, grain + return None, None + + out: dict[str, _Table] = {} + retired = retired or {} + for name, cols in tables.items(): + mine = [_copy_col(c) for c in cols] + known = {c.name for c in mine} + for col in retired.get(name, []): + if col.name in known: + continue + found, _ = last_with(name, col.name) + mine.append(_copy_col(found) if found is not None else _copy_col(col)) + known.add(col.name) + spec = specs.get(name) + out[name] = _Table(name, mine, list(spec.grain) if spec is not None else []) + for name, cols in retired.items(): + if name in out: + continue + base: list[ColumnOut] = [] + for cols_k, _pspecs in reversed(derived): + if name in cols_k: + base = [_copy_col(c) for c in cols_k[name]] + break + known = {c.name for c in base} + for col in cols: + if col.name not in known: + found, _ = last_with(name, col.name) + base.append(_copy_col(found) if found is not None else _copy_col(col)) + known.add(col.name) + _, grain = last_with(name, None) + out[name] = _Table(name, base, [g for g in grain or [] if g in known]) + return out + + +def _create(conn: sqlite3.Connection, table: _Table) -> None: + """建表，写法与执行器相同（recipe_engine._create_tables）：STRICT；主键列 NOT NULL；单列 INTEGER 主键写成 + 列约束的 PRIMARY KEY DESC，不让它成为 rowid 的别名（rowid 必须是插入序号，期 4 按 rowid 区间溯源）。""" + types = {c.name: c.type for c in table.cols} + alias = len(table.grain) == 1 and types.get(table.grain[0]) == "INTEGER" + defs = [] + for col in table.cols: + d = f"{_q(col.name)} {col.type}" + if alias and col.name == table.grain[0]: + d += " PRIMARY KEY DESC NOT NULL" + elif col.name in table.grain: + d += " NOT NULL" + defs.append(d) + if table.grain and not alias: + defs.append(f"PRIMARY KEY ({', '.join(_q(g) for g in table.grain)})") + conn.execute(f"CREATE TABLE {_q(table.name)} ({', '.join(defs)}) STRICT") + + +def _check(cid: str, kind: str, title: str, *, checked: int, failed: list[str], sql: str | None, + params: list[Any] | None = None) -> CheckResult: + return CheckResult(id=cid, kind=kind, title=title, status="mismatch" if failed else "passed", # type: ignore[arg-type] + category="structure", checked=checked, failed=len(failed), sql=sql, + params=list(params or []), details=failed[:20]) + + +def materialize_union(out: Path, *, target: Recipe, parts: list[UnionPart], + retired: dict[str, list[ColumnOut]] | None = None) -> UnionReport: + """按目标配方的表结构，把各期构建库物化成一个并集库（P3-SPEC 2.6）。同步，调用方放进线程池并占 parse_slot。 + + parts 已按统计期排序，可以只有一期（移除、撤销之后剩一期而那一期的配方不是目标配方，7.4）。retired 是退役列： + 表 → 列，整张退役的表给全部的列，**用 retired_columns(计划, current_tables) 拼**（先 retired_existing、再 + retired_new，顺序进库哈希），并且原样传给 union_id、union_options。 + + 0. 逐期算构建库文件的 sha256，和 UnionPart.db_sha256（TableBuild 登记的）、manifest_db_sha256（导入清单里的） + 都比对，不一致抛 UnionError(part_tampered)；文件不在抛 part_missing。各期以 immutable 方式附加，SQLite 不会 + 再校验，被改过的库附加进来就被「洗白」成新版本了，所以必须在附加之前算（评审一-M4）。 + 1. 先写 .tmp-<随机>（open_checked_sqlite，关 DQS，回滚日志、不开 WAL），成功后 os.replace 到 out。 + 2. 建表：目标配方的列在前、退役列追加在后；grain 非空时加 PRIMARY KEY，主键列 NOT NULL。 + 3. 逐期附加（mode=ro&immutable=1）、断言该期每张表 rowid 从 1 连续（不成立抛 union_failed：期 4 按偏移把并集 + rowid 换算回该期构建库，依赖这一条）、INSERT … SELECT … ORDER BY rowid（只选这一期有的列）、每期一个事务、 + 提交后 DETACH（SQLite 不允许在事务里 DETACH；逐期附加也绕开了最多附加 10 个库的上限）。 + 4. 记下每期每张表的 rowid 区间（并集里的、该期构建库里的）。 + 5. 结构核对 U1 行数、U2 主键（写入时撞上主键约束也记成 U2 不通过）、U3 日期落在该期统计期内。 + 6. table_hashes 按 union/1 的组合定义（目标表结构加各期登记的表哈希），不逐行重算；空值数按表、按期分开记 + （按期的只记该期构建库里有的列：不含这一列的期是结构性空值，说明不能说成「原表为空格」）。 + 7. 关连接、settle_journal、算 db_sha256、os.replace。 + + **U1–U3 有不通过的不抛异常**：不产出文件（临时文件删掉，out 上旧的同名文件也删掉，免得被当成这次的结果发布）， + 返回 ok=False、db_sha256 为空的报告。试运行要把核对结果（哪一期哪张表几行）给人看，所以看 report.ok 判 + rejected；版本页的写操作（移除、撤销）没有地方展示核对结果，调 require_union_ok(report) 换成 + UnionError(union_failed)，接口层映射成 409 union_failed（7.4）。漏检也不会静默发布：没有文件，提交时是 + trial_missing。其余失败（part_missing、part_tampered、rowid 不连续等）抛 UnionError。 + """ + out = Path(out) + if not parts: + raise ValueError("物化至少要有一期") + keys = [(p.start, p.end) for p in parts] + if keys != sorted(keys): + raise ValueError("parts 必须按统计期排序") + for k, part in enumerate(parts, 1): + path = Path(part.db_path) + label = f"第 {k} 期（{_period_text(part.start, part.end)}）" + if not path.is_file(): + # 出路照 7.4 写：丢了文件的那一期移出去，或回到更早的版本。接口层直接转发这句 + raise UnionError(f"{label}的数据文件已丢失，无法重建当前版本。可以先移除这一期，" + "或在数据源卡片的「版本」中启用更早的版本", code="part_missing", part=k) + actual = raw_store.sha256_file(path) + expected = {str(part.db_sha256 or "").strip()} + if part.manifest_db_sha256 is not None: + expected.add(str(part.manifest_db_sha256).strip()) + if expected != {actual}: + raise UnionError(f"{label}的数据文件与登记的不一致，可能被修改过", code="part_tampered", part=k) + layout = _layout(target, parts, retired) + targets = {name: {c.name for c in cols} for name, cols in derive_tables(target)[0].items()} + out.parent.mkdir(parents=True, exist_ok=True) + tmp = out.parent / f"{out.name}{raw_store.TMP_MARK}{secrets.token_hex(6)}" + try: + report = _materialize(tmp, layout, parts, targets) + if not report.ok: + _remove_db_files(tmp) + _remove_db_files(out) + return report + settle_journal(tmp) + report.db_sha256 = raw_store.sha256_file(tmp) + os.replace(tmp, out) + return report + except BaseException: + _remove_db_files(tmp) + raise + + +def require_union_ok(report: UnionReport) -> None: + """U1–U3 有不通过的就抛 UnionError(union_failed)，消息写出不通过的细节（前三条）。 + + materialize_union 遇到 U1–U3 不通过只返回 ok=False 的报告（试运行要展示核对结果）；版本页的移除、撤销没有地方 + 展示，用它把报告换成异常，接口层照 7.4 回 409 union_failed。""" + if report.ok: + return + details = [d for c in report.checks if c.status != "passed" for d in (c.details or [c.title])] + raise UnionError("累积后的数据没有通过结构核对，当前版本未受影响：" + "；".join(details[:3]), code="union_failed") + + +def _remove_db_files(path: Path) -> None: + for suffix in ("", "-journal", "-wal", "-shm"): + Path(f"{path}{suffix}").unlink(missing_ok=True) + + +def _materialize(tmp: Path, layout: dict[str, _Table], parts: list[UnionPart], + targets: dict[str, set[str]]) -> UnionReport: + conn = open_checked_sqlite(str(tmp), readonly=False) + conn.isolation_level = None + part_rows: list[dict[str, dict[str, list[int]]]] = [] + part_cols: list[dict[str, list[str]]] = [] + part_counts: list[dict[str, int]] = [] + pk_failures: list[str] = [] + #: 撞上主键约束、整期回滚的那几期（下标从 0 起） + rolled_back: set[int] = set() + try: + conn.execute("BEGIN") + for table in layout.values(): + _create(conn, table) + conn.execute("COMMIT") + for k, part in enumerate(parts, 1): + label = f"第 {k} 期（{_period_text(part.start, part.end)}）" + conn.execute("ATTACH DATABASE ? AS part", (f"file:{quote(os.fspath(part.db_path))}?mode=ro&immutable=1",)) + ranges: dict[str, dict[str, list[int]]] = {} + cols_k: dict[str, list[str]] = {} + counts: dict[str, int] = {} + try: + present = [r[0] for r in conn.execute( + "SELECT name FROM part.sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name")] + stray = [t for t in present if t not in layout] + if stray: + raise UnionError(f"{label}有表" + "、".join(f"「{t}」" for t in stray) + + "在累积后的表结构里没有位置", code="union_failed", part=k) + conn.execute("BEGIN") + try: + for name, table in layout.items(): + if name not in present: + continue + have = [r[1] for r in conn.execute(f"PRAGMA part.table_info({_q(name)})")] + extra = [c for c in have if c not in table.names] + if extra: + raise UnionError(f"{label}的「{name}」有列" + "、".join(f"「{c}」" for c in extra) + + "在累积后的表结构里没有位置", code="union_failed", part=k) + lo, hi, n = conn.execute(f"SELECT MIN(rowid), MAX(rowid), COUNT(*) FROM part.{_q(name)}").fetchone() + if n and (lo != 1 or hi != n): + raise UnionError(f"{label}的「{name}」行号不连续（{lo} 至 {hi}，共 {n} 行），无法按行号溯源", + code="union_failed", part=k) + cols = [c for c in table.names if c in have] + before = conn.execute(f"SELECT COALESCE(MAX(rowid), 0) FROM main.{_q(name)}").fetchone()[0] + listed = ", ".join(_q(c) for c in cols) + conn.execute(f"INSERT INTO main.{_q(name)} ({listed}) SELECT {listed} FROM part.{_q(name)} " + "ORDER BY rowid") + after = conn.execute(f"SELECT COALESCE(MAX(rowid), 0) FROM main.{_q(name)}").fetchone()[0] + if after - before != n: + raise UnionError(f"{label}的「{name}」写入了 {after - before} 行，应为 {n} 行", + code="union_failed", part=k) + ranges[name] = {"union": [before + 1, after], "part": [1, n]} + cols_k[name] = cols + counts[name] = n + conn.execute("COMMIT") + except sqlite3.IntegrityError as e: + conn.execute("ROLLBACK") + pk_failures.append(_pk_conflict(conn, layout, parts, part_rows, k, e)) + ranges, cols_k, counts = {}, {}, {} + rolled_back.add(k - 1) + except BaseException: + if conn.in_transaction: + conn.execute("ROLLBACK") + raise + finally: + conn.execute("DETACH DATABASE part") + part_rows.append(ranges) + part_cols.append(cols_k) + part_counts.append(counts) + return _finish(conn, layout, parts, part_rows, part_cols, part_counts, pk_failures, rolled_back, targets) + finally: + conn.close() + + +def _pk_conflict(conn: sqlite3.Connection, layout: dict[str, _Table], parts: list[UnionPart], + part_rows: list[dict[str, dict[str, list[int]]]], k: int, error: Exception) -> str: + """第 k 期写入时撞上了主键约束：找出和哪一期、在哪个主键上重复（库已回滚到第 k 期之前，part 还附加着）。""" + label = f"第 {k} 期（{_period_text(parts[k - 1].start, parts[k - 1].end)}）" + for name, table in layout.items(): + if not table.grain: + continue + try: + listed = ", ".join(_q(g) for g in table.grain) + nulls = conn.execute(f"SELECT COUNT(*) FROM part.{_q(name)} WHERE " + + " OR ".join(f"{_q(g)} IS NULL" for g in table.grain)).fetchone()[0] + if nulls: + return f"{label}的「{name}」有 {nulls} 行主键列为空值" + dup = conn.execute(f"SELECT {listed} FROM part.{_q(name)} GROUP BY {listed} HAVING COUNT(*) > 1 LIMIT 1" + ).fetchone() + if dup is not None: + key = "，".join(f"{g}={v}" for g, v in zip(table.grain, dup)) + return f"{label}的「{name}」自身在主键（{key}）上重复" + hit = conn.execute(f"SELECT m.rowid, {', '.join('m.' + _q(g) for g in table.grain)} FROM main.{_q(name)} m " + f"JOIN part.{_q(name)} p USING ({listed}) LIMIT 1").fetchone() + except sqlite3.Error: + continue + if hit is None: + continue + rowid, values = hit[0], hit[1:] + other = next((j for j, ranges in enumerate(part_rows, 1) + if name in ranges and ranges[name]["union"][0] <= rowid <= ranges[name]["union"][1]), None) + key = "，".join(f"{g}={v}" for g, v in zip(table.grain, values)) + who = f"第 {other} 期" if other is not None else "此前的某一期" + return f"{who}与第 {k} 期在「{name}」的主键（{key}）上重复" + return f"{label}写入时违反了主键约束（{error}）" + + +def _finish(conn: sqlite3.Connection, layout: dict[str, _Table], parts: list[UnionPart], + part_rows: list[dict[str, dict[str, list[int]]]], part_cols: list[dict[str, list[str]]], + part_counts: list[dict[str, int]], pk_failures: list[str], rolled_back: set[int], + targets: dict[str, set[str]]) -> UnionReport: + rows: dict[str, int] = {} + null_counts: dict[str, dict[str, int]] = {} + for name, table in layout.items(): + exprs = ", ".join(f"COUNT({_q(c)})" for c in table.names) + got = conn.execute(f"SELECT COUNT(*), {exprs} FROM {_q(name)}").fetchone() + rows[name] = got[0] + null_counts[name] = {c: got[0] - n for c, n in zip(table.names, got[1:])} + + # U1 行数：每张表的并集行数 = 各期该表行数之和 + u1: list[str] = [] + for name in layout: + want = sum(c.get(name, 0) for c in part_counts) + # 撞了主键而整期回滚的那几期没有计入 part_counts：按它们在构建库里的行数补上，U1 才会如实报出少了的行 + want += sum(_part_count(parts[k], name) for k in sorted(rolled_back)) + if rows[name] != want: + u1.append(f"表「{name}」累积后 {rows[name]} 行，各期之和 {want} 行") + first = next(iter(layout), None) + checks = [_check("U1", "union_rows", "累积后每张表的行数等于各期行数之和", checked=len(layout), failed=u1, + sql=f"SELECT COUNT(*) FROM {_q(first)}" if first else None)] + + # U2 主键：按目标 grain 分组没有重复；写入时撞上主键约束的也记在这里 + u2 = list(pk_failures) + keyed = [t for t in layout.values() if t.grain] + sql2 = None + for table in keyed: + listed = ", ".join(_q(g) for g in table.grain) + sql2 = f"SELECT COUNT(*) FROM (SELECT 1 FROM {_q(table.name)} GROUP BY {listed} HAVING COUNT(*) > 1)" + dup = conn.execute(sql2).fetchone()[0] + if dup: + u2.append(f"表「{table.name}」有 {dup} 组主键重复") + checks.append(_check("U2", "union_pk", "累积后主键不重复", checked=len(keyed), failed=u2, sql=sql2)) + + # U3 统计期：第 k 期的 rowid 区间里，日期都在 [start_k, end_k] 之内 + u3: list[str] = [] + sql3, params3, checked3 = None, [], 0 + for k, (part, ranges) in enumerate(zip(parts, part_rows), 1): + for name, span in ranges.items(): + axis = layout[name].axis + if axis is None or axis not in part_cols[k - 1].get(name, []): + continue + a, b = span["union"] + if b < a: + continue + checked3 += 1 + sql3 = (f"SELECT COUNT(*) FROM {_q(name)} WHERE rowid BETWEEN ? AND ? " + f"AND ({_q(axis)} < ? OR {_q(axis)} > ?)") + params3 = [a, b, part.start, part.end] + bad = conn.execute(sql3, params3).fetchone()[0] + if bad: + u3.append(f"第 {k} 期（{_period_text(part.start, part.end)}）的「{name}」有 {bad} 行日期不在该期统计期内") + checks.append(_check("U3", "union_period", "各期的日期都在该期的统计期内", checked=checked3, failed=u3, + sql=sql3, params=params3)) + + part_nulls: list[dict[str, dict[str, int]]] = [] + for ranges, cols_k in zip(part_rows, part_cols): + mine: dict[str, dict[str, int]] = {} + for name, span in ranges.items(): + cols = cols_k.get(name, []) + a, b = span["union"] + if not cols: + continue + if b < a: + mine[name] = {c: 0 for c in cols} + continue + got = conn.execute(f"SELECT {', '.join(f'COUNT(*) - COUNT({_q(c)})' for c in cols)} FROM {_q(name)} " + "WHERE rowid BETWEEN ? AND ?", (a, b)).fetchone() + mine[name] = dict(zip(cols, got)) + part_nulls.append(mine) + + table_hashes: dict[str, str] = {} + for name, table in layout.items(): + hashes: list[str | None] = [] + for k, (part, ranges) in enumerate(zip(parts, part_rows), 1): + if name not in ranges: + hashes.append(None) + continue + got = part.table_hashes.get(name) + if not got: + raise UnionError(f"第 {k} 期（{_period_text(part.start, part.end)}）缺少表「{name}」的表哈希登记", + code="union_failed", part=k) + hashes.append(got) + table_hashes[name] = sha_json({"union_ver": UNION_VER, "columns": [[c.name, c.type] for c in table.cols], + "grain": table.grain, "parts": hashes}) + + added: list[dict[str, Any]] = [] + retired: list[dict[str, Any]] = [] + for name, table in layout.items(): + with_table = [_period_key(p.start, p.end) for p, ranges in zip(parts, part_rows) if name in ranges] + if name not in targets: + retired.append({"table": name, "column": None, "periods": with_table}) + continue + if len(with_table) < len(parts): + added.append({"table": name, "column": None, "periods": with_table}) + for col in table.cols: + having = [_period_key(p.start, p.end) for p, cols_k in zip(parts, part_cols) + if col.name in cols_k.get(name, [])] + if col.name not in targets[name]: + retired.append({"table": name, "column": col.name, "periods": having}) + elif len(having) < len(with_table): + added.append({"table": name, "column": col.name, "periods": having}) + + ok = not (u1 or u2 or u3) + return UnionReport( + tables={name: table.cols for name, table in layout.items()}, + grains={name: table.grain for name, table in layout.items()}, + rows=rows, part_rows=part_rows, table_hashes=table_hashes, null_counts=null_counts, + part_null_counts=part_nulls, added=added, retired=retired, checks=checks, ok=ok, db_sha256="", + ) + + +def _part_count(part: UnionPart, name: str) -> int: + """某一期构建库里一张表的行数（只读打开）。表不在时为 0。""" + try: + conn = open_checked_sqlite(part.db_path, readonly=True) + except sqlite3.Error: + return 0 + try: + return int(conn.execute(f"SELECT COUNT(*) FROM {_q(name)}").fetchone()[0]) + except sqlite3.Error: + return 0 + finally: + conn.close() + diff --git a/backend/app/data/recipe_compare.py b/backend/app/data/recipe_compare.py new file mode 100644 index 00000000..2a4e93ea --- /dev/null +++ b/backend/app/data/recipe_compare.py @@ -0,0 +1,234 @@ +"""新旧配方对照（期 3，WP-3；P3-SPEC 6.2、9.4 的 RecipeComparison）。 + +**为什么要有对照。** 期 2 改配方之后只有两种信号：破坏性变更（breaking:*，按表一句）和配方类确认项。修复按钮、 +框选、按规则重新起草都会改分段的定位、标签集合、忽略规则、工作表名，这些不在破坏性变更里，人也不会去逐字 +比两份 JSON。对照把两份配方摆在一起：表、列（类型、单位、来源）、主键、表的种类、分段（标题、标签增删、 +忽略规则）、关系、工作表名、导入模式，以及按期累积的分档。试运行回执、确认清单顶部、修复预览、重新起草的 +「采用」都用它（界面复用同一个 RecipeCompare 组件）。 + +**排序**：破坏性的排在最前，单位变化最前（单位变了，同一列里就混了两种口径，按期累积时尤其要命）。 + +纯函数：不碰数据库、不读文件。配方收 Recipe，也收 dict（canonical 形式也行，按模型补全默认值）。 +""" +from __future__ import annotations + +from typing import Any + +from app.data.recipe_confirm import ( + TableChange, _column_sources, _declared_types, as_recipe, table_changes, +) +from app.data.recipe_diff import plain +from app.data.recipe_parsers import match_key +from app.data.recipe_types import ( + CrosstabBlock, Dismissed, ListBlock, NotComparable, Recipe, SheetRecipe, SumEq, derive_tables, +) + + +def _brief(types: dict[tuple[str, str], str], sources: dict[tuple[str, str], str], table: str, + col: Any) -> dict[str, Any]: + return {"type": types.get((table, col.name), col.type), "unit": col.unit, "source": sources.get((table, col.name))} + + +def _table_entries(old: Recipe, new: Recipe, changes: list[TableChange]) -> list[dict[str, Any]]: + ot, _ = derive_tables(old) + nt, _ = derive_tables(new) + otypes, ntypes = _declared_types(old, ot), _declared_types(new, nt) + osrc, nsrc = _column_sources(old), _column_sources(new) + ospec = {t.name: t for t in old.tables} + nspec = {t.name: t for t in new.tables} + by_table: dict[str, list[TableChange]] = {} + for ch in changes: + by_table.setdefault(ch.table, []).append(ch) + out: list[dict[str, Any]] = [] + for name in list(dict.fromkeys([*nt, *ot])): + ocols = {c.name: c for c in ot.get(name, [])} + ncols = {c.name: c for c in nt.get(name, [])} + mine = by_table.get(name, []) + columns: list[dict[str, Any]] = [] + for cname in list(dict.fromkeys([*ncols, *ocols])): + oc, nc = ocols.get(cname), ncols.get(cname) + ob = _brief(otypes, osrc, name, oc) if oc is not None else None + nb = _brief(ntypes, nsrc, name, nc) if nc is not None else None + kinds = [ch.kind for ch in mine if ch.column == cname and ch.kind != "column_removed"] + if oc is None: + status = "added" + elif nc is None: + status = "removed" + elif kinds or ob != nb: + status = "changed" + else: + status = "same" + columns.append({"name": cname, "status": status, "old": ob, "new": nb, + "changes": list(dict.fromkeys(kinds))}) + os_, ns_ = ospec.get(name), nspec.get(name) + og = list(os_.grain) if os_ is not None and name in ot else None + ng = list(ns_.grain) if ns_ is not None and name in nt else None + grain_changed = og is not None and ng is not None and set(og) != set(ng) + kind = {"old": os_.kind if os_ is not None and name in ot else None, + "new": ns_.kind if ns_ is not None and name in nt else None} + # 单位另列在 units_changed（排最前），这里只放其余的破坏性变化，与确认项 breaking:<表> 同一口径 + breaking = [ch.message for ch in mine if ch.kind != "unit"] + if name not in ot: + status = "added" + elif name not in nt: + status = "removed" + elif mine or grain_changed or kind["old"] != kind["new"] or any(c["status"] != "same" for c in columns): + status = "changed" + else: + status = "same" + out.append({"name": name, "status": status, "columns": columns, + "grain": {"old": og, "new": ng, "changed": grain_changed}, "kind": kind, "breaking": breaking, + "_units": any(ch.kind == "unit" for ch in mine)}) + # 单位变化的表最前，其余破坏性变化其次，再是有变化的、新增删除的，没变的最后（sort 稳定，同档保持配方顺序） + rank = {"changed": 2, "added": 3, "removed": 3, "same": 4} + out.sort(key=lambda t: 0 if t["_units"] else (1 if t["breaking"] else rank.get(t["status"], 4))) + for t in out: + del t["_units"] + return out + + +def _title(seg: Any) -> str | None: + loc = getattr(seg, "locate", None) + return getattr(loc, "title", None) + + +def _diff_texts(old: list[str], new: list[str]) -> tuple[list[str], list[str]]: + """按 match_key 比两组原文：返回 (新增的原文, 去掉的原文)，各自保持原来的顺序。""" + ok, nk = {match_key(x) for x in old}, {match_key(x) for x in new} + return [x for x in new if match_key(x) not in ok], [x for x in old if match_key(x) not in nk] + + +def _block_ignores(block: Any) -> list[str]: + out = [f"按行标签忽略「{r.label}」" for r in getattr(block, "ignore_rows", [])] + out += [f"按表头忽略「{r.header}」" for r in getattr(block, "ignore_columns", [])] + return out + + +def _sheet_ignores(sheet: SheetRecipe) -> list[str]: + return [f"按同一行的文字「{r.anchor}」忽略区域外的数字" for r in sheet.ignore_outside] + + +def _entry(eid: str, status: str, title: tuple[str | None, str | None], labels: tuple[list[str], list[str]], + ignore: tuple[list[str], list[str]]) -> dict[str, Any]: + return {"id": eid, "status": status, "title": {"old": title[0], "new": title[1]}, + "labels": {"added": labels[0], "removed": labels[1]}, "ignore": {"added": ignore[0], "removed": ignore[1]}} + + +def _segment_entries(old: Recipe, new: Recipe) -> list[dict[str, Any]]: + """分段的对照：交叉表的每个分段按 id 配对（标题、标签增删）。块和工作表上的忽略规则、列表块的 after_title + 没有分段可挂，按块 id、工作表 id 各出一条（只在有变化时出），id 就是块或工作表在配方里的 id。""" + def segs(r: Recipe) -> dict[str, Any]: + return {s.id: s for sh in r.sheets for b in sh.blocks if isinstance(b, CrosstabBlock) for s in b.segments} + + def blocks(r: Recipe) -> dict[str, Any]: + return {b.id: b for sh in r.sheets for b in sh.blocks} + + out: list[dict[str, Any]] = [] + os_, ns_ = segs(old), segs(new) + for sid in list(dict.fromkeys([*ns_, *os_])): + a, b = os_.get(sid), ns_.get(sid) + la = list(a.labels.expect) if a is not None else [] + lb = list(b.labels.expect) if b is not None else [] + added, removed = _diff_texts(la, lb) + title = (_title(a) if a is not None else None, _title(b) if b is not None else None) + if a is None: + status = "added" + elif b is None: + status = "removed" + elif added or removed or title[0] != title[1] or a.model_dump(mode="json") != b.model_dump(mode="json"): + status = "changed" + else: + status = "same" + # 新增、去掉的分段：标签全部列出（_diff_texts 对空的一边自然给出全部）。起草器换了分段 id 时，新分段的 + # 标题和标签就是它新认出的定位规则，redraft_adopted 要逐项写出来让人确认（P3-SPEC 6.3） + out.append(_entry(sid, status, title, (added, removed), ([], []))) + ob, nb = blocks(old), blocks(new) + for bid in list(dict.fromkeys([*nb, *ob])): + a, b = ob.get(bid), nb.get(bid) + ig = _diff_texts(_block_ignores(a) if a is not None else [], _block_ignores(b) if b is not None else []) + ta = a.after_title if isinstance(a, ListBlock) else None + tb = b.after_title if isinstance(b, ListBlock) else None + if ig[0] or ig[1] or ta != tb: + out.append(_entry(bid, "changed", (ta, tb), ([], []), ig)) + osh = {s.id: s for s in old.sheets} + nsh = {s.id: s for s in new.sheets} + for sid in list(dict.fromkeys([*nsh, *osh])): + a, b = osh.get(sid), nsh.get(sid) + ig = _diff_texts(_sheet_ignores(a) if a is not None else [], _sheet_ignores(b) if b is not None else []) + if ig[0] or ig[1]: + out.append(_entry(sid, "changed", (None, None), ([], []), ig)) + return out + + +def relation_text(rel: Any) -> str: + """关系的一句人话（对照界面用，与确认项 relation:* 的写法一致）。""" + if isinstance(rel, SumEq): + return f"{rel.total} = " + " + ".join(rel.parts) + if isinstance(rel, NotComparable): + return f"表「{rel.a.table}」与表「{rel.b.table}」口径不同，不互相推算或相加" + if isinstance(rel, Dismissed): + return f"不登记系统发现的关系 {rel.claims}（理由：{rel.reason}）" + return str(getattr(rel, "id", "")) + + +def _relation_entries(old: Recipe, new: Recipe) -> list[dict[str, Any]]: + orel = {r.id: r for r in old.relations} + nrel = {r.id: r for r in new.relations} + out = [] + for rid in list(dict.fromkeys([*nrel, *orel])): + a, b = orel.get(rid), nrel.get(rid) + if a is None: + status = "added" + elif b is None: + status = "removed" + elif a.model_dump(mode="json") != b.model_dump(mode="json"): + status = "changed" + else: + status = "same" + out.append({"id": rid, "status": status, "old": relation_text(a) if a is not None else None, + "new": relation_text(b) if b is not None else None}) + return out + + +def _sheet_entries(old: Recipe, new: Recipe) -> list[dict[str, Any]]: + """工作表按配方里的 id 配对，给出匹配名的新旧（⑨ 更新工作表名就是这里的变化）。""" + osh = {s.id: s.match.name for s in old.sheets} + nsh = {s.id: s.match.name for s in new.sheets} + return [{"id": sid, "name": {"old": osh.get(sid), "new": nsh.get(sid)}} for sid in dict.fromkeys([*nsh, *osh])] + + +def _accumulate(plan: Any) -> dict[str, Any] | None: + p = plain(plan) + if not isinstance(p, dict): + return None + return {"change": str(p.get("change") or ""), "added": list(p.get("added") or []), + "retired_new": list(p.get("retired_new") or []), "retired_existing": list(p.get("retired_existing") or [])} + + +def compare_recipes(old: Recipe | dict[str, Any], new: Recipe | dict[str, Any], *, + plan: dict[str, Any] | None = None) -> dict[str, Any]: + """新旧配方逐项对照（RecipeComparison，P3-SPEC 9.4）。 + + - tables：每张表的列（新增、删除、变化；类型、单位、来源，changes 是 table_changes 的 kind）、主键、种类， + breaking 是这张表除单位以外的破坏性变化（与确认项 breaking:<表> 同一口径）。单位变化的表排最前，其余 + 破坏性的其次；没变的也列出（status=same），界面自己决定显示哪些； + - segments：交叉表的分段（标题、标签增删），以及块、工作表上的忽略规则和列表块的 after_title（有变化时，按块 + 或工作表的 id 出一条）； + - relations、sheets（工作表 id → 匹配名的新旧）、mode； + - breaking：有没有任何破坏性变化（含单位）；units_changed：单位变化的列； + - accumulate：给了累积计划（AccumulatePlan 或它的 asdict）时，抄它的分档、新增、退役，否则 None。 + """ + o, n = as_recipe(old), as_recipe(new) + changes = table_changes(o, n) + return { + "tables": _table_entries(o, n, changes), + "segments": _segment_entries(o, n), + "relations": _relation_entries(o, n), + "sheets": _sheet_entries(o, n), + "mode": {"old": o.mode, "new": n.mode}, + "breaking": bool(changes), + "units_changed": [{"table": ch.table, "column": ch.column, "old": ch.old, "new": ch.new} + for ch in changes if ch.kind == "unit"], + "accumulate": _accumulate(plan), + } + diff --git a/backend/app/data/recipe_confirm.py b/backend/app/data/recipe_confirm.py index 68157715..f2a9419c 100644 --- a/backend/app/data/recipe_confirm.py +++ b/backend/app/data/recipe_confirm.py @@ -17,9 +17,22 @@ id 里的键一律按契约「键的格式」：分段、块用 id，工作表 rows_after_stop 就没了；上传新一期漏传现行配方，破坏性变更和单位变化就没了），而提交时重算用的是同一个 函数，服务端也发现不了。所以 kind 只认四种；reupload、redraft 必须给现行配方和上一期回执；extraction 必须带齐 EXTRACTION_KEYS、不能是部分干跑；checks 必须给；有上一期时差异卡必须给（可以是空列表）。 + +期 3（P3-SPEC 6.2、9.5）： +- **只列有变化的配方类确认项**：首次、切换、采用「按规则重新起草」（recipe_origin=rules_redraft）仍列全部；其余 + 上传新一期、修改配方只列 recipe_item_signatures 与现行配方不同的项。没变的项在现行配方提交时已经由人确认过 + （TableRecipe.confirmations），重复勾选只会让人习惯性地全勾。例外：按本期数据才出、或内容随数据变的几项 + （忽略的列、列表的公式按保存值、年份取自统计期、隐藏行列、忽略规则命中的行），实际内容与上一期回执不同也照列 + （_data_signatures）：配方变了时差异卡不再逐项比较，只看签名的话，上次没出、从没人确认过的这几项就漏了； +- 新确认项：修改记录（fix:*、select:*）、redraft_adopted、按期累积的计划（period_replace、accumulate_restart、 + retire、mode_switch、accumulate_unit_risk）、忽略规则（ignore_rows、ignore_outside）。ConfirmItem.source 标出 + 来源（recipe / edit / accumulate / outside / sheet / context / diff / switch），界面按它分组。 """ from __future__ import annotations +import json +import re +from collections.abc import Callable from dataclasses import dataclass from typing import Any @@ -28,7 +41,7 @@ from app.data.recipe_diff import ( ReceiptIncomplete, col_letter, outside_of, period_annotated, period_of, period_texts, plain, receipt_of, require_receipt, rows_text, same_period_sentence, short, split_coord, ) -from app.data.recipe_parsers import match_key, period_residue, split_unit_suffix +from app.data.recipe_parsers import UNITS, match_key, period_residue, split_unit_suffix from app.data.recipe_types import ( ConfirmItem, CrosstabBlock, DerivedSegment, Dismissed, ListBlock, MeasuresSegment, NotComparable, Recipe, SheetRecipe, SumEq, derive_tables, @@ -75,6 +88,19 @@ class ConfirmContext: facts: Any = None #: 数据源现在设的遮罩列（options.mask_columns）。切换、改配方之后列名变了，遮罩按列名匹配会静默失效（AU-6） mask_columns: list[str] | None = None + #: 期 3：累积计划（AccumulatePlan 或它的 asdict）。新配方 mode=accumulate、或者现行是累积时由试运行给出； + #: period_replace、accumulate_restart、retire、mode_switch、accumulate_unit_risk 由它出 + accumulate: Any = None + #: 期 3：暂存区里已应用的修改（StagingOut.edits 的形状），每条出 fix: 或 select:。调用方只传未被 + #: 覆盖的；superseded 为真的这里也会跳过（PUT 之后它们的效果可能已经不在新配方里） + edits: list[dict[str, Any]] | None = None + #: 期 3：recipe_origin=rules_redraft 时 compare_recipes(现行, 采用的配方) 的结果，redraft_adopted 据此逐条列出 + #: 变化；不给时按 old_recipe 现算 + redraft_compare: dict[str, Any] | None = None + #: 期 3：区域外文字的差异项（accumulate_unit_risk 用）；不给时从 diff 里取 outside_* 和统计期写法的项。 + #: diff_reports 收到追加或替换该期的累积计划时，配方变了也比区域外文字，所以通常不用另给；另算时用 + #: recipe_diff.outside_items(上一期, 本期)，不能传空列表顶替（传了就只看它，单位变化会静默消失） + outside_diff: list[Any] | None = None @dataclass(frozen=True) @@ -336,18 +362,23 @@ def switch_changes(old_schema_cache: dict[str, Any] | None, new_recipe: Recipe | class _Bag: - """按出现顺序收集，id 重复的只留第一条。""" + """按出现顺序收集，id 重复的只留第一条。keep 不为 None 时只收 keep(id) 为真的（只列有变化的配方类项）。""" - def __init__(self) -> None: + def __init__(self, keep: Callable[[str], bool] | None = None) -> None: self.items: list[ConfirmItem] = [] self._ids: set[str] = set() + self.keep = keep def add(self, item_id: str, label: str, detail: str = "", source: str = "recipe") -> None: - if item_id in self._ids: + if item_id in self._ids or (self.keep is not None and not self.keep(item_id)): return self._ids.add(item_id) self.items.append(ConfirmItem(id=item_id, label=label, detail=detail, required=True, source=source)) + def into(self, other: "_Bag") -> None: + for i in self.items: + other.add(i.id, i.label, i.detail, i.source) + def _validate(ctx: ConfirmContext) -> dict[str, Any]: """输入不全就报错（见模块 docstring），返回 extraction 的 dict 形状。""" @@ -378,13 +409,16 @@ def _validate(ctx: ConfirmContext) -> dict[str, Any]: def confirm_items(ctx: ConfirmContext) -> list[ConfirmItem]: - """生成必勾的确认项（P2-SPEC 7.5）。顺序：单位变化（排最前）、其余破坏性变更、切换、配方类、AI 类、本期类、差异卡。 + """生成必勾的确认项（P2-SPEC 7.5、P3-SPEC 6.2、9.5）。顺序：单位变化（排最前）、其余破坏性变更、切换、 + 按期累积（重新开始、切换模式、退役、替换该期、单位风险）、采用重新起草、配方类、修改记录、AI 类、本期类、差异卡。 + 配方类：首次、切换、采用按规则重新起草时列全部；其余只列签名与现行配方不同的（recipe_item_signatures）。 输入不全时抛 ValueError（缺键时是它的子类 ReceiptIncomplete），见模块 docstring。 """ ex = _validate(ctx) recipe = as_recipe(ctx.recipe) old = as_recipe(ctx.old_recipe) if ctx.old_recipe is not None else None + plan = _plan(ctx.accumulate) bag = _Bag() differs = recipe_differs(ctx.kind, recipe, old) @@ -395,10 +429,17 @@ def confirm_items(ctx: ConfirmContext) -> list[ConfirmItem]: bag.add(f"unit_changed:{ch.table}.{ch.column}", f"表「{ch.table}」列「{ch.column}」的单位 {_unit_name(ch.old)} → {_unit_name(ch.new)}：" "数量级或含义可能不同，引用它的报告口径会变") + retired_tables, retired_existing = _retired_scope(plan) grouped: dict[str, list[str]] = {} for ch in changes: - if ch.kind != "unit": - grouped.setdefault(ch.table, []).append(ch.message) + if ch.kind == "unit": + continue + # 同一张表已有 retire:*（按期累积时退役的列和表保留在并集里），删除或改名就不在 breaking 里再列一遍； + # 早已退役的同样不列（P3-SPEC 2.5） + if ch.kind in ("column_removed", "table_removed") and ( + ch.table in retired_tables or (ch.table, ch.column) in retired_existing): + continue + grouped.setdefault(ch.table, []).append(ch.message) for table, msgs in grouped.items(): bag.add(f"breaking:{table}", f"表「{table}」：" + "；".join(msgs)) @@ -410,13 +451,26 @@ def confirm_items(ctx: ConfirmContext) -> list[ConfirmItem]: tail = ("以下表和列将消失或改变：" + "；".join(lines)) if lines else "原有的表和列都保留" bag.add("switch_from_simple", "从简单导入切换为按配方导入：" + tail, source="switch") + if plan is not None: + _accumulate_items(bag, plan, ctx, recipe, old) + if ctx.recipe_origin == "rules_redraft": + _redraft_item(bag, ctx, recipe, old) + if differs and ctx.mask_columns: _mask_items(bag, ctx, recipe, old) if differs: - _recipe_items(bag, recipe, ex, ctx.facts, ai=ctx.recipe_origin in ("ai", "mixed")) - if ctx.recipe_origin in ("ai", "mixed"): - _ai_items(bag, recipe) + # 首次、切换、采用重新起草：列全部（重新起草的分段定位、标签集合、工作表匹配名不在签名里，任何一项都 + # 兜不住，评审一-M6）。其余只列新增的、签名变了的 + full = ctx.kind in ("first", "switch") or ctx.recipe_origin == "rules_redraft" or old is None + sub = _Bag(None if full else _changed_items(recipe, old, ex, ctx.prev)) + _recipe_items(sub, recipe, ex, ctx.facts, ai=ctx.recipe_origin in ("ai", "mixed")) + sub.into(bag) + + _edit_items(bag, ctx.edits) + + if differs and ctx.recipe_origin in ("ai", "mixed"): + _ai_items(bag, recipe) _period_items(bag, recipe, ex, ctx) @@ -427,6 +481,458 @@ def confirm_items(ctx: ConfirmContext) -> list[ConfirmItem]: return bag.items +# -------------------------------------------------------------------------- +# 期 3：只列有变化的配方类确认项（P3-SPEC 6.2） +# -------------------------------------------------------------------------- + + +def _sig(*parts: Any) -> str: + return json.dumps(parts, ensure_ascii=False, sort_keys=True, default=str) + + +def _signatures(recipe: Recipe, sheet_name: Callable[[SheetRecipe], str]) -> dict[str, str]: + """确认项 id → 签名（只看配方内容，不看本期计数）。按工作表出的 id 用 sheet_name(工作表) 拼：比较时两份配方 + 按工作表 id 换成同一个名字（本期实际的工作表名），所以只改了匹配名（⑨ 更新工作表名）不算变化。""" + out: dict[str, str] = {} + for sheet in recipe.sheets: + name = sheet_name(sheet) + if sheet.context: + out[f"cross_check_off:{name}"] = _sig(sheet.context[0].cross_check) + out[f"hidden:{name}"] = _sig(sheet.hidden.rows, sheet.hidden.cols) + out[f"ignore_outside:{name}"] = _sig(sorted((match_key(r.anchor), r.reason) for r in sheet.ignore_outside)) + for block in sheet.blocks: + for p in block.values.placeholders: + out.setdefault(f"placeholder:{p.text}", _sig(p.meaning)) + out[f"text_number:{block.id}"] = _sig(block.values.text_number) + out[f"formula_cached:{block.id}"] = _sig(block.values.formula) + ignore_cols = sorted((match_key(r.header), r.reason) for r in block.ignore_columns) + if isinstance(block, CrosstabBlock): + out[f"blank_null:{block.id}"] = _sig(block.values.blank) + out[f"checks_relaxed:{block.id}"] = _sig(sorted(k for k in _AXIS_CHECK if k not in block.axis.checks)) + out[f"year_from:{block.id}"] = _sig(block.axis.year_from) + out[f"ignored_columns:{block.id}"] = _sig(None, ignore_cols) + out[f"ignore_rows:{block.id}"] = _sig(sorted((match_key(r.label), r.reason) for r in block.ignore_rows)) + for seg in block.segments: + if isinstance(seg, DerivedSegment): + keep = seg.keep_as.table if seg.keep_as else None + out[f"derived:{seg.id}"] = _sig(keep, sorted(match_key(x) for x in seg.labels.expect)) + else: + out[f"merged_fill:{block.id}"] = _sig(block.merged_data) + out[f"ignored_columns:{block.id}"] = _sig(block.extra_columns, ignore_cols) + out[f"blank_skip:{block.id}"] = _sig(block.rows.blank_rows) + total = block.rows.total_row + out[f"total_row:{block.id}"] = _sig( + None if total is None else (total.label_column, total.pick, total.keep_as)) + for r in recipe.relations: + if isinstance(r, SumEq): + out[f"relation:{r.id}"] = _sig("sum_eq", r.table, r.total, sorted(r.parts)) + elif isinstance(r, NotComparable): + out[f"relation:{r.id}"] = _sig("not_comparable", r.a.table, r.a.value, r.b.table, r.b.value, r.by) + elif isinstance(r, Dismissed): + out[f"dismissed:{r.claims}"] = _sig(r.reason) + tables, _ = derive_tables(recipe) + for t, cols in tables.items(): + for c in cols: + out[f"unit:{t}.{c.name}"] = _sig(c.unit, c.header) + out["mode"] = _sig(recipe.mode) + return out + + +def recipe_item_signatures(recipe: Recipe | dict[str, Any]) -> dict[str, str]: + """配方类确认项 id → 签名（P3-SPEC 6.2 的表）。改配方之后只列「新增的、签名变了的」配方类项。 + + 签名只看配方内容：占位符的含义、空格与千分位与公式的处理、合并单元格、缺的日期断言、文件名核对、 + 忽略的列（多出的列怎么处理加按表头忽略的表头和理由）、跳过空行、合计段与合计行、年份来源、关系、不登记的理由、 + 单位与来源标签、隐藏行列的策略、导入模式、按行标签忽略和按同一行文字忽略的锚点加理由。 + 分段定位（标题、标签集合、after_title）、工作表匹配名、列的来源标签不在签名里：来自修复按钮时有 fix:*，来自 + 框选时有 select:*，来自手工修改时有破坏性变更和对照界面，来自按规则重新起草时按首次导入列全部。 + + 按工作表出的 id（cross_check_off、hidden、ignore_outside）这里用配方里的工作表名拼；confirm_items 比较时 + 两份配方按工作表 id 换成本期实际的工作表名。 + + 签名只是一半：confirm_items 另按本期数据和上一期回执比较几项按数据出的确认项（_data_signatures，不在这个公开 + 函数里，因为它要读回执）。 + """ + return _signatures(as_recipe(recipe), lambda s: s.match.name) + + +def _changed_items(recipe: Recipe, old: Recipe, ex: dict[str, Any], + prev: dict[str, Any] | None) -> Callable[[str], bool]: + """「这个配方类确认项要不要列出」：新配方里新增的 id，签名与现行配方不同的，或者按本期数据出的内容与上一期 + 不同的（_data_signatures）。没有签名定义的 id 照列（宁可多勾一项，不能漏）。工作表按 id 换成本期实际的名字， + 两份配方用同一个名字比。""" + matched = (ex.get("sheets") or {}).get("matched") or {} + + def name(sheet: SheetRecipe) -> str: + return str(matched.get(sheet.id) or sheet.match.name) + + new, old_sig = _signatures(recipe, name), _signatures(old, name) + p_rec = receipt_of(prev) if prev is not None else {} + now, before = _data_signatures(recipe, ex), _data_signatures(recipe, p_rec) + + def keep(item_id: str) -> bool: + if item_id not in new or old_sig.get(item_id) != new[item_id]: + return True + return item_id in now and before.get(item_id) != now[item_id] + + return keep + + +def _data_signatures(recipe: Recipe, rec: dict[str, Any]) -> dict[str, str | None]: + """按数据出的那几种配方类确认项 → 本期（或上一期回执）的实际内容（P3-SPEC 6.2 的补充，评审意见 H3）。 + + 为什么要有：签名只看配方内容，但下面几项在期 2 里只在本期数据满足条件时才出，或者内容随数据变： + - ignored_columns:<块>：实际被忽略的表头集合（extra_columns=ignore 时本期没有多出的列就不出）； + - formula_cached:<列表块>：本期有没有按保存值导入的公式格（列表的默认取值就是放宽，有公式格才出）； + - year_from:<块>：日期表头的写法集合（表头都是完整日期时不出）； + - hidden:<工作表>：照常导入或排除的是哪些隐藏行列； + - ignore_rows:<块> / ignore_outside:<工作表>：忽略规则命中的锚点和行数。 + 配方没变时这些变化由差异卡兜住（diff:ignored:、diff:axis_form:、diff:hidden:、diff:ignored_rows:），但配方变了时 + 差异卡只给行数和统计期（recipe_diff 的模块 docstring）。只比签名的话，同一暂存区里应用过一次修复，上一次提交时 + 没出、从未被人确认过的这几项就永远不用确认了。所以配方变了时，这几项的实际内容和上一期回执（ctx.prev，现行 + 导入的导入清单）不同也照列。 + + 回执里缺对应字段（期 3 之前的导入没有 rows_excluded、手写的最小回执）时值为 None：一边 None、一边有值就算不同， + 照列；两边都缺则相同。工作表按 id 换成各自回执里的实际工作表名（改了名的照样对得上）。 + """ + out: dict[str, str | None] = {} + matched = (rec.get("sheets") or {}).get("matched") or {} + + def name(sheet: SheetRecipe) -> str: + return str(matched.get(sheet.id) or sheet.match.name) + + def has(field: str) -> bool: + return field in rec + + ignored = rec.get("ignored_columns") or {} + forms: dict[str, list[str]] = {} + for a in rec.get("axes") or []: + if isinstance(a, dict): + forms.setdefault(str(a.get("block")), []).append(str(a.get("form") or "")) + excluded = [e for e in (plain(x) for x in rec.get("rows_excluded") or []) if isinstance(e, dict)] + + def hits(reason: str, pred: Callable[[dict[str, Any]], bool]) -> str | None: + if not has("rows_excluded"): + return None + rows: dict[str, int] = {} + for e in excluded: + if e.get("reason") == reason and pred(e): + key = match_key(e.get("anchor") or "") + rows[key] = rows.get(key, 0) + sum(int(b) - int(a) + 1 for a, b in e.get("rows") or []) + return _sig(sorted(rows.items())) + + hidden = rec.get("hidden") or {} + for sheet in recipe.sheets: + sname = name(sheet) + h = hidden.get(sname) if isinstance(hidden.get(sname), dict) else {} + out[f"hidden:{sname}"] = _sig(sorted(int(r) for r in h.get("rows") or [] if str(r).isdigit()), + sorted(str(c) for c in h.get("cols") or [])) if has("hidden") else None + out[f"ignore_outside:{sname}"] = hits("ignored_outside", lambda e, s=sname: e.get("sheet") == s) + for block in sheet.blocks: + out[f"ignored_columns:{block.id}"] = (_sig(sorted({match_key(h) for h in ignored.get(block.id) or []})) + if has("ignored_columns") else None) + if isinstance(block, CrosstabBlock): + out[f"year_from:{block.id}"] = _sig(sorted(set(forms.get(block.id, [])))) if has("axes") else None + out[f"ignore_rows:{block.id}"] = hits("ignored_rows", lambda e, b=block.id: e.get("block") == b) + else: + out[f"formula_cached:{block.id}"] = (_sig(int(rec.get("formula_cells_accepted") or 0) > 0) + if has("formula_cells_accepted") else None) + return out + + +# -------------------------------------------------------------------------- +# 期 3：修改记录、重新起草、按期累积（P3-SPEC 9.5） +# -------------------------------------------------------------------------- + + +def _edit_items(bag: _Bag, edits: list[dict[str, Any]] | None) -> None: + """每条已应用、未被覆盖的修改出一项：修复 fix:<目标键>，框选 select::<块或分段 id>。文案是修改的标题， + 细节是人话摘要和署名（未认证）。""" + for e in edits or []: + e = plain(e) + if not isinstance(e, dict) or e.get("superseded") or not e.get("key"): + continue + prefix = "select" if e.get("kind") == "selection" else "fix" + title = str(e.get("title") or "").strip() or ("按框选修改了配方" if prefix == "select" else "用修复按钮修改了配方") + summary = e.get("summary") + detail = "；".join(str(x) for x in summary) if isinstance(summary, list) else str(summary or "") + if e.get("signed_by"): + detail = "；".join(x for x in (detail, f"署名（未认证）：{e['signed_by']}") if x) + bag.add(f"{prefix}:{e['key']}", title, detail, source="edit") + + +def _compare_lines(cmp: dict[str, Any]) -> list[str]: + """compare_recipes 的结果里，分段、标签、工作表名、列来源的变化，逐条写成人话（redraft_adopted 用）。""" + lines: list[str] = [] + for seg in cmp.get("segments") or []: + sid, status = seg.get("id"), seg.get("status") + title = seg.get("title") or {} + labels = seg.get("labels") or {} + if status in ("added", "removed"): + # 新增、去掉的分段写出标题和全部标签：起草器换了分段 id 时，新分段的定位规则只在这里经人确认 + # （配方类确认项里没有分段的标题和标签） + new = status == "added" + head = title.get("new" if new else "old") + tags = labels.get("added" if new else "removed") or [] + what = "".join([f"，标题「{head}」" if head else "", + "，标签" + "".join(f"「{x}」" for x in tags) if tags else ""]) + lines.append(f"{'新增' if new else '去掉'}分段「{sid}」{what}") + continue + if status == "changed" and title.get("old") != title.get("new"): + lines.append(f"分段「{sid}」的标题「{title.get('old') or '无'}」→「{title.get('new') or '无'}」") + if labels.get("added"): + lines.append(f"分段「{sid}」加入标签" + "".join(f"「{x}」" for x in labels["added"])) + if labels.get("removed"): + lines.append(f"分段「{sid}」去掉标签" + "".join(f"「{x}」" for x in labels["removed"])) + ignore = seg.get("ignore") or {} + if ignore.get("added"): + lines.append(f"「{sid}」新增忽略规则：" + "、".join(str(x) for x in ignore["added"])) + if ignore.get("removed"): + lines.append(f"「{sid}」去掉忽略规则：" + "、".join(str(x) for x in ignore["removed"])) + for sh in cmp.get("sheets") or []: + nm = sh.get("name") or {} + if nm.get("old") != nm.get("new"): + lines.append(f"工作表名「{nm.get('old') or '无'}」→「{nm.get('new') or '无'}」") + for t in cmp.get("tables") or []: + for c in t.get("columns") or []: + o, n = c.get("old") or {}, c.get("new") or {} + if o.get("source") != n.get("source") and (o.get("source") or n.get("source")): + lines.append(f"表「{t.get('name')}」列「{c.get('name')}」的来源" + f"「{o.get('source') or '无'}」→「{n.get('source') or '无'}」") + return lines + + +def _redraft_item(bag: _Bag, ctx: ConfirmContext, recipe: Recipe, old: Recipe | None) -> None: + """采用了按规则重新起草的配方（P3-SPEC 6.3）：起草器识别出的定位规则要逐项经过人的确认，不能因为只列 + 有变化的配方类项就直接成为现行配方（H3）。""" + cmp = plain(ctx.redraft_compare) + if not isinstance(cmp, dict): + if old is None: + raise ValueError("采用按规则重新起草的配方时必须给现行配方或对照结果，否则无法列出变化") + from app.data.recipe_compare import compare_recipes # 循环导入：recipe_compare 用本模块的 table_changes + + cmp = compare_recipes(old, recipe) + lines = _compare_lines(cmp) + head = "采用按规则重新起草的配方：" + if not lines: + bag.add("redraft_adopted", head + "分段、标签、工作表名和列的来源与现行配方相同", source="edit") + return + shown = "；".join(lines[:6]) + (f"等 {len(lines)} 处" if len(lines) > 6 else "") + bag.add("redraft_adopted", head + shown, "；".join(lines), source="edit") + + +def _plan(data: Any) -> dict[str, Any] | None: + plan = plain(data) + if plan is None: + return None + if not isinstance(plan, dict) or not plan.get("action"): + raise ValueError("累积计划不完整：缺少 action，无法确定要确认的项") + return plan + + +def _span(p: Any) -> tuple[str, str] | None: + if not isinstance(p, dict) or not p.get("start") or not p.get("end"): + return None + return str(p["start"]), str(p["end"]) + + +def _spans(parts: list[Any]) -> str: + return "、".join(f"{s} 至 {e}" for s, e in (_span(p) for p in parts or []) if s) if parts else "" + + +def _blocker_text(part: dict[str, Any]) -> str: + msgs = [str(b.get("message") if isinstance(b, dict) else b) for b in part.get("blockers") or []] + return "；".join(m for m in msgs if m) + + +def _retired_scope(plan: dict[str, Any] | None) -> tuple[set[str], set[tuple[str, str | None]]]: + """breaking 去重的范围：有 retire:* 的表（只在追加、替换该期时出），以及早已退役的 (表, 列)。""" + if plan is None or plan.get("mode") != "accumulate" or plan.get("action") not in ("append", "replace_period"): + return set(), set() + tables = {str(e.get("table")) for e in plan.get("retired_new") or [] if isinstance(e, dict)} + existing = {(str(e.get("table")), e.get("column")) for e in plan.get("retired_existing") or [] + if isinstance(e, dict)} + return tables, existing + + +#: 区域外文字里提示单位或口径的词（P3-SPEC 2.5） +_SCOPE_WORDS = ("单位", "口径", "统计范围", "计量") +#: 含单位字、却不是在说单位的常见词。按规格「出现即算」，只把这些词先去掉再找单位词：单字单位（人、元、个、件、户） +#: 太容易出现在别的词里，「编制人：测试员甲」→「编制人：测试员乙」这种每期都可能变的落款不该要求确认单位。 +#: 只列确定不是单位的词；拿不准的（人数、件数、户数）不列，宁可多确认一次 +_UNIT_STOPWORDS = ( + "编制人", "填报人", "填表人", "制表人", "负责人", "联系人", "审核人", "复核人", "审批人", "批准人", "经办人", + "统计人", "报送人", "签发人", "核对人", "人员", "人工", "单元", "元旦", "文件", "附件", "条件", "事件", "邮件", + "软件", "用户", "账户", "客户", "门户", "个别", "各个", "整个", "个人", "个月", + # 「单位」在这几个词里指填报的机构，不是计量单位 + "填报单位", "编制单位", "报送单位", "制表单位", "责任单位", +) + + +def _unit_pattern() -> re.Pattern[str]: + """单位词（P3-SPEC 2.5：新旧文字里出现 UNITS 词表中的词，或出现「单位」「口径」「统计范围」「计量」之一）。 + 出现就算，不论前后是什么：「货运量按吨统计」→「按件统计」、「每户」→「每人」、「元/件」→「元/个」都是单位真的变了。 + 误报靠 _UNIT_STOPWORDS 先去掉（_unit_tokens）。单位前面的数量级（万、千、百、亿）算进同一个词：「人次」→「万人次」 + 要能比出不同；多字的排在单字前面，「人次」不会被拆成「人」。""" + units = sorted(UNITS, key=len, reverse=True) + alt = "|".join(re.escape(u) for u in units) + scope = "|".join(map(re.escape, _SCOPE_WORDS)) + return re.compile(rf"(?:{scope})|(?:[万千百亿]?(?:{alt}))") + + +_UNIT_RE = _unit_pattern() +#: 停用词按长度从长到短排进正则：长词先匹配，不会被其中的短词截断 +_STOP_RE = re.compile("|".join(re.escape(w) for w in sorted(_UNIT_STOPWORDS, key=len, reverse=True))) + + +def _unit_tokens(text: str) -> list[str]: + """文字里的单位、口径词（去掉停用词之后）。停用词换成「|」：它不是单位，也不会和前后的字拼成新词。""" + return sorted(m.group(0) for m in _UNIT_RE.finditer(_STOP_RE.sub("|", canon(text)))) + + +#: 区域外文字的差异项，以及统计期格写法的变化（D12 那种）：单位、口径的说明可能写在这些格里 +_OUTSIDE_KINDS = ("outside_added", "outside_removed", "outside_changed", "context_text", "context_source_added") + + +def _unit_risks(ctx: ConfirmContext, recipe: Recipe, old: Recipe | None) -> list[str]: + """按期累积时单位写在配方之外、只表现为区域外文字变化的那几种（评审一-M10）。返回涉及单位或口径的变化。""" + out: list[str] = [] + source = ctx.outside_diff if ctx.outside_diff is not None else ctx.diff + for d in source or []: + d = plain(d) + if not isinstance(d, dict) or d.get("kind") not in _OUTSIDE_KINDS: + continue + # 只看差异项里的原文（detail：全文或「上一期：…；本期：…」），不看 label 里的坐标和工作表名 + if _unit_tokens(str(d.get("detail") or "")): + out.append(str(d.get("label") or "")) + if old is not None: + # 分段标题（D09 那种改字）：只在新旧标题里的单位、口径词不同时算。标题里常年写着「（人次）」，只改了别的字 + # 就报，会让每次改标题都多一项必勾 + titles = {s.id: s.locate.title for s in _segments(old) if getattr(s.locate, "title", None)} + for seg in _segments(recipe): + new_title, old_title = getattr(seg.locate, "title", None), titles.get(seg.id) + if new_title and old_title and _unit_tokens(new_title) != _unit_tokens(old_title): + out.append(f"分段「{seg.id}」的标题「{old_title}」→「{new_title}」") + return out + + +def _segments(recipe: Recipe) -> list[Any]: + return [s for sheet in recipe.sheets for b in sheet.blocks if isinstance(b, CrosstabBlock) for s in b.segments] + + +def _switch_to_accumulate_text(plan: dict[str, Any], ctx: ConfirmContext, dropped: list[dict[str, Any]]) -> str | None: + """mode_switch:replace->accumulate 冒号后面那一句（P3-SPEC 2.4 表）。按 action 分开写，不能只看 parts： + + - append：当前那一期原样保留，parts 里它是 new=False 的那一项； + - replace_period：当前那一期被本次替换（修改配方后重新导入同一个文件，或者同一统计期再传），parts 里只有本期， + 被替换的那一期在 replaces。只看 parts 找不到「第一期」，早先因此落到「当前版本没有统计期」，与同一清单里的 + period_replace 自相矛盾（评审意见）； + - restart：配方变化不兼容； + - first：当前那一期没有统计期，或者它的配方不满足资格，只有这里才写这两种原因； + - rejected：试运行拒收（部分重叠），不能提交，不出这一项（拒收的原因在问题列表里），返回 None。 + """ + action = plan.get("action") + if action == "append": + kept = next((p for p in plan.get("parts") or [] if isinstance(p, dict) and not p.get("new")), None) + sp = _span(kept) + if sp is None: + raise ValueError("累积计划是追加，但结果里没有可作为第一期保留的当前那一期") + return f"当前版本的 {sp[0]} 至 {sp[1]} 作为第一期保留，此前被替换的各期不会补回" + if action == "replace_period": + rep = plan.get("replaces") if isinstance(plan.get("replaces"), dict) else None + sp = _span(rep) + if sp is None: + raise ValueError("累积计划是替换该期，但没有给出被替换那一期的统计期") + if ctx.kind == "redraft": + return f"当前版本的 {sp[0]} 至 {sp[1]} 按修改后的配方重新导入，作为第一期，此前被替换的各期不会补回" + return f"本次导入替换当前版本的 {sp[0]} 至 {sp[1]}，作为第一期，此前被替换的各期不会补回" + if action == "restart": + return "表结构有不兼容的变化，当前版本不能作为第一期：启用后当前版本只含本期" + if action == "first": + blocked = next((p for p in dropped if p.get("blockers")), None) + if blocked is not None: + return (f"当前版本的配方不满足按期累积的要求（{_blocker_text(blocked)}），不能作为第一期：" + "启用后当前版本只含本期") + # 计划里当前那一期是 dropped 的最后一项（plan_accumulate 取最晚的一期判资格） + if not dropped or _span(dropped[-1]) is None: + return "当前版本没有统计期，不能作为第一期：启用后当前版本只含本期" + why = f"（{plan['reason']}）" if plan.get("reason") else "" + return f"当前版本不能作为第一期{why}：启用后当前版本只含本期" + return None + + +def _accumulate_items(bag: _Bag, plan: dict[str, Any], ctx: ConfirmContext, recipe: Recipe, + old: Recipe | None) -> None: + """按期累积的必勾项（P3-SPEC 2.4、2.5、9.5），来源都是 accumulate。""" + action, switch = plan.get("action"), plan.get("mode_switch") + dropped = [p for p in plan.get("dropped") or [] if isinstance(p, dict)] + n = len(dropped) + before = f"此前 {n} 期" if n else "此前各期" + + if action == "restart": + changes = [str(x) for x in plan.get("semantic") or []] + for p in dropped: + # 已是累积模式、某一期的配方不满足资格（blockers）也走 restart，变化一栏写明是哪一期、为什么 + text, sp = _blocker_text(p), _span(p) + if text: + where = f"{sp[0]} 至 {sp[1]} 这一期" if sp else "此前的一期" + changes.append(f"{where}的配方不满足按期累积的要求：{text}") + what = "（" + "；".join(changes) + "）" if changes else "" + bag.add("accumulate_restart", + f"表结构有不兼容的变化{what}：启用后当前版本只含本期，{before}不再出现在当前版本中，之后各期在本期基础上" + "继续累积；可以在数据源卡片的「版本」中启用此前的版本", + "如果不想重新开始累积，请返回修改配方（例如保留原来的单位、列名或常量取值）", source="accumulate") + + if switch == "accumulate->replace": + spans = _spans(dropped) + bag.add("mode_switch:accumulate->replace", + f"从按期累积改为每期替换：启用后当前版本只含本期，{before}" + (f"（{spans}）" if spans else "") + + "不再出现在当前版本中，可以在数据源卡片的「版本」中启用此前的版本", source="accumulate") + elif switch == "replace->accumulate": + text = _switch_to_accumulate_text(plan, ctx, dropped) + if text is not None: + bag.add("mode_switch:replace->accumulate", "从每期替换改为按期累积：" + text, str(plan.get("reason") or ""), + source="accumulate") + + accumulating = plan.get("mode") == "accumulate" and action in ("append", "replace_period") + if accumulating: + for e in plan.get("retired_new") or []: + if not isinstance(e, dict) or not e.get("table"): + continue + t, c = str(e["table"]), e.get("column") + if c: + bag.add(f"retire:{t}.{c}", f"表「{t}」的列「{c}」自本期起不再导入：早期各期保留原值，本期起为空值", + "如果是改名：旧列名保留早期各期的值，新列名从本期起有值，两列并存", source="accumulate") + else: + bag.add(f"retire:{t}", f"表「{t}」自本期起不再导入：早期各期的行保留，本期起没有新的行", + "如果是改名：旧表保留早期各期的行，新表从本期起有行，两张表并存", source="accumulate") + + if action == "replace_period": + rep = plan.get("replaces") if isinstance(plan.get("replaces"), dict) else {} + sp = _span(rep) or _span(plan.get("period")) + if sp is None: + raise ValueError("累积计划是替换该期，但没有给出被替换那一期的统计期") + if ctx.kind == "redraft": + label = f"用修改后的配方重新导入最近一期 {sp[0]} 至 {sp[1]}：启用后该期以本次为准，其余各期不变" + else: + src = [] + if rep.get("seq") is not None: + src.append(f"第 {rep['seq']} 次导入") + if rep.get("file_name"): + src.append(f"「{short(rep['file_name'])}」") + label = (f"替换已有的一期 {sp[0]} 至 {sp[1]}" + (f"（{''.join(src)}）" if src else "") + + "：启用后该期以本次为准") + bag.add(f"period_replace:{sp[0]}~{sp[1]}", label, source="accumulate") + + if accumulating: + risks = _unit_risks(ctx, recipe, old) + if risks: + shown = "；".join(risks[:3]) + (f"等 {len(risks)} 处" if len(risks) > 3 else "") + bag.add("accumulate_unit_risk", + f"区域外的说明文字有变化，且涉及单位或口径（{shown}）：确认后，本期与此前各期按同一单位累积在同一列中。" + "如果单位已经变了，请返回修改配方里的单位（这会重新开始累积）", "；".join(risks), source="accumulate") + + def _mask_items(bag: _Bag, ctx: ConfirmContext, recipe: Recipe, old: Recipe | None) -> None: """遮罩列在新结构里找不到同名列（不区分大小写）：出必勾 mask_lost:<列>（AU-6）。 @@ -509,11 +1015,19 @@ def _recipe_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any], facts: Any, *, bag.add(f"cross_check_off:{name}", f"工作表「{name}」的统计期不与文件名里的日期核对") ignored = ex.get("ignored_columns") or {} - for b in lists: + for _, _, b in blocks: heads = [str(h) for h in ignored.get(b.id) or []] - if b.extra_columns == "ignore" and heads: + extra = isinstance(b, ListBlock) and b.extra_columns == "ignore" + if b.ignore_columns: + # 期 3 的按表头忽略（修复按钮 ⑥、框选）：配置了就出，写明每条规则和理由，细节里是本期实际忽略的列 + rules = "、".join(f"「{short(r.header, 20)}」（理由：{short(r.reason)}）" for r in b.ignore_columns) + tail = "；其余多出的列也不导入" if extra else "" + now = "本期忽略的列：" + "、".join(heads) if heads else "本期没有命中的列" + bag.add(f"ignored_columns:{b.id}", f"「{b.id}」按表头忽略列：{rules}{tail}", now) + elif extra and heads: bag.add(f"ignored_columns:{b.id}", f"「{b.id}」不导入这些列：" + "".join(f"「{short(h, 20)}」" for h in heads), "、".join(heads)) + _ignore_rule_items(bag, recipe, ex) skips = [b for b in lists if b.rows.blank_rows == "skip"] for b in skips: n = f"（本期 {int(ex.get('blank_rows_skipped') or 0)} 行）" if len(skips) == 1 else "" @@ -590,7 +1104,58 @@ def _recipe_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any], facts: Any, *, if parts: bag.add(f"hidden:{name}", f"工作表「{name}」的" + "；".join(parts)) - bag.add("mode", "导入模式：每期替换" if recipe.mode == "replace" else "导入模式：按期累积") + # 两种取值的后果（P3-SPEC 2.2，评审三-m4）：首次导入时由人确认，这就是 D14 的「首次导入时由人确认」 + bag.add("mode", "导入模式：每期替换" if recipe.mode == "replace" else "导入模式：按期累积", MODE_DETAIL[recipe.mode]) + + +#: 导入模式两种取值的后果（q_mode 卡片和确认项 mode 用同一句） +MODE_DETAIL = { + "accumulate": "每期以统计期为键加入当前版本；统计期与已有各期部分重叠的文件会被拒收，同一统计期再传要确认替换", + "replace": "每次上传新一期，当前版本只含新的一期，此前各期留在历史版本中", +} + + +def _excluded_rows(ex: dict[str, Any], reason: str) -> list[dict[str, Any]] | None: + """本期 rows_excluded 里某种原因的项；回执里没有这个字段（期 3 之前）时 None，个数就不写。""" + if "rows_excluded" not in ex: + return None + entries = (plain(x) for x in ex.get("rows_excluded") or []) + return [e for e in entries if isinstance(e, dict) and e.get("reason") == reason] + + +def _row_count(entries: list[dict[str, Any]]) -> int: + return sum(int(b) - int(a) + 1 for e in entries for a, b in (e.get("rows") or [])) + + +def _rule_text(anchor: str, reason: str, entries: list[dict[str, Any]] | None) -> str: + """「「补录（人次）」（理由：…；本期 1 行）」。本期命中几行按 rows_excluded 里锚点相同（match_key）的项数。""" + tail = "" + if entries is not None: + hit = [e for e in entries if match_key(e.get("anchor") or "") == match_key(anchor)] + tail = f"；本期 {_row_count(hit)} 行" + return f"「{short(anchor, 20)}」（理由：{short(reason)}{tail}）" + + +def _ignore_rule_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any]) -> None: + """期 3 的忽略规则（修复按钮 ⑥ 和框选写入）：按行标签忽略 ignore_rows:<块>、按同一行文字忽略区域外的数字 + ignore_outside:<工作表>。配置了就出，写上每条规则的理由和本期命中的行数（可以是 0）。""" + rows = _excluded_rows(ex, "ignored_rows") + for sheet in recipe.sheets: + for b in sheet.blocks: + if isinstance(b, CrosstabBlock) and b.ignore_rows: + mine = None if rows is None else [e for e in rows if e.get("block") == b.id] + rules = "、".join(_rule_text(r.label, r.reason, mine) for r in b.ignore_rows) + bag.add(f"ignore_rows:{b.id}", f"「{b.id}」按行标签忽略：{rules}", + "；".join(f"「{r.label}」：{r.reason}" for r in b.ignore_rows)) + outside = _excluded_rows(ex, "ignored_outside") + for sheet in recipe.sheets: + if not sheet.ignore_outside: + continue + name = _sheet_name(ex, sheet) + mine = None if outside is None else [e for e in outside if e.get("sheet") == name] + rules = "、".join(_rule_text(r.anchor, r.reason, mine) for r in sheet.ignore_outside) + bag.add(f"ignore_outside:{name}", f"工作表「{name}」按同一行的文字忽略导入区域之外的数字：{rules}", + "；".join(f"「{r.anchor}」：{r.reason}" for r in sheet.ignore_outside)) def _coord_of(d: dict[str, Any]) -> str: @@ -744,7 +1309,14 @@ def _outside_exempt(o: dict[str, Any], p_out: dict[str, dict[str, Any]], p_per: def _stop_block(problem: dict[str, Any], recipe: Recipe, ex: dict[str, Any]) -> str: """rows_after_stop 属于哪个列表块。问题对象里没有块键，按工作表和溯源的列范围找：同一工作表上被空行停下的 - 列表块只有一个就是它；有多个时取列范围与问题格相交、数据在问题格之上、最靠近的那个。""" + 列表块只有一个就是它；有多个时取列范围与问题格相交、数据在问题格之上、最靠近的那个。 + + 执行器的消息里写着「列表「块」」，恰好认出一个块时直接用它。跳过空行的列表按表下说明收尾时也会报这个问题 + （WP-8 修补 3 与评审意见 4），下面按 blank_rows=stop 找候选的办法找不到它。""" + message = str(problem.get("message") or "") + named = [b.id for _, _, b in _blocks(recipe, ex) if isinstance(b, ListBlock) and f"列表「{b.id}」" in message] + if len(named) == 1: + return named[0] cells = [split_coord(c) for c in problem.get("cells") or []] sheet = next((s for s, _, _ in cells if s), None) rows = [r for _, r, _ in cells if r is not None] diff --git a/backend/app/data/recipe_diff.py b/backend/app/data/recipe_diff.py index b4e4d251..d4464b21 100644 --- a/backend/app/data/recipe_diff.py +++ b/backend/app/data/recipe_diff.py @@ -28,6 +28,16 @@ TableBuild.report，没有 ledger 键）或配方变了（redraft、改配方） **数字遮盖后的模板**（digit_template）：canon 之后把每一段连续的 ASCII 数字换成 ``{}``。文件名、统计期格、 区域外文字的「写法变了」都按它比，每月必变的日期因此不算变化。 +**期 3**（P3-SPEC 9.5）： +- 给了累积计划（accumulate）时，按计划另出 period_added、period_replaced、period_gap（需确认）、period_backfill + （本期统计期是人工录入时需确认）、columns_added、columns_retired、labels_vary。它们来自计划，与配方变没变无关； +- 追加或替换该期时，区域外文字的差异（outside_*）**配方变了也照比**（outside_items，也公开给调用方单独算 + ConfirmContext.outside_diff）：本期和此前各期进同一列，单位写在区域外时变化只表现在这里， + recipe_confirm 的 accumulate_unit_risk 靠它（P3-SPEC 2.5）； +- 逐项比较时按回执的 rows_excluded（排除的行）比「按配方忽略的行」：锚点集合或行数变了出 ignored_rows（需确认， + ``diff:ignored_rows:<工作表>``，与期 2 ignored_columns 的 ``diff:ignored:<块>`` 不共用前缀）。rows_excluded + 不在 RECEIPT_KEYS 里：上一期（期 3 之前的导入）没有这个字段时出 info「上一期未记录排除的行」，不当作空列表比。 + 纯函数，不碰数据库、不读文件。 """ from __future__ import annotations @@ -44,10 +54,11 @@ _DIGITS = re.compile(r"[0-9]+") #: 差异卡的全部 kind，按界面上的先后（7.6 的表；outside_moved 是表里「只是位置变了的算 info」那一项） DIFF_KINDS: tuple[str, ...] = ( - "period", "rows", "placeholders", "checks", "file_name", "full_calc", "context_text", + "period", "period_added", "period_replaced", "period_gap", "period_backfill", "rows", "columns_added", + "columns_retired", "labels_vary", "placeholders", "checks", "file_name", "full_calc", "context_text", "context_source_added", "column_order", "label_writing", "header_writing", "canon_values", "row_order", "block_order", "derived_form", "axis_form", "outside_added", "outside_removed", "outside_changed", - "outside_moved", "ignored_columns", "hidden", + "outside_moved", "ignored_columns", "ignored_rows", "hidden", ) #: 需确认的 kind → 确认项 id 的前缀（id = 前缀 + 键，键的格式见契约 recipe_types 的模块 docstring） @@ -64,6 +75,10 @@ CONFIRM_PREFIX: dict[str, str] = { "outside_changed": "diff:outside:", "ignored_columns": "diff:ignored:", "hidden": "diff:hidden:", + # 期 3。period_backfill、ignored_rows 只在特定情况下需确认（_item 的 confirm=False 时是 info） + "period_gap": "diff:period_gap:", + "period_backfill": "diff:period_backfill:", + "ignored_rows": "diff:ignored_rows:", } _STATUS = {"passed": "通过", "mismatch": "不一致", "unverifiable": "无法核对", "info": "提示"} @@ -275,7 +290,7 @@ def period_annotated(doc: Any, rec: dict[str, Any]) -> dict[str, str]: def diff_reports(prev: dict[str, Any] | None, cur: dict[str, Any], *, prev_file_name: str | None, cur_file_name: str, recipe: Recipe | dict[str, Any] | None = None, - recipe_changed: bool | None = None) -> list[DiffItem]: + recipe_changed: bool | None = None, accumulate: Any = None) -> list[DiffItem]: """上一期 → 本期的差异卡。prev 为 None（首次）返回 []。需确认的排在前面，同类按 DIFF_KINDS 的顺序。 上一期的回执没有 ledger 键（简单导入，kind=switch）或配方变了（redraft、改配方）时只给 rows、period； @@ -285,6 +300,9 @@ def diff_reports(prev: dict[str, Any] | None, cur: dict[str, Any], *, prev_file_ - recipe_changed：配方与上一期是否不同。不传时看 prev、cur 两边的配方哈希（顶层 ``recipe_sha256``， 或导入清单的 ``recipe.sha256``）；两边没都带、又没传时报错，不默认「没变」。传了、两边也都带了哈希 而两者矛盾时同样报错。 + - accumulate（期 3）：累积计划（AccumulatePlan 或它的 asdict）。给了、而且是按期累积的追加或替换该期时，按计划 + 出统计期、列、维度取值的差异项；配方变了也比区域外文字（outside_items，两边回执缺 OUTSIDE_KEYS 时报错）， + 见模块 docstring。 """ if prev is None: return [] @@ -292,10 +310,18 @@ def diff_reports(prev: dict[str, Any] | None, cur: dict[str, Any], *, prev_file_ for side, doc in (("prev", prev), ("cur", cur)): require_receipt(doc, side, ("tables",), top=()) p_rec, c_rec = receipt_of(prev), receipt_of(cur) + plan = plain(accumulate) items: list[DiffItem] = [] items += _diff_period(prev, cur) + items += _diff_accumulate(plan, cur) items += _diff_rows(p_rec, c_rec) - if "ledger" in p_rec and not _recipe_changed(prev, cur, recipe_changed): + full = "ledger" in p_rec and not _recipe_changed(prev, cur, recipe_changed) + if not full and accumulating(plan): + # 按期累积的追加、替换该期：本期和此前各期进同一列，单位写在区域外时（lab.c01 的「单位：万元」），变化只 + # 表现为区域外文字的差异（P3-SPEC 2.5，评审一-M10），accumulate_unit_risk 靠它。配方变了也照比：同一暂存区里 + # 应用过一次修复，单位从「人次」改成「万人次」也不能没有提示（评审意见 H2） + items += outside_items(prev, cur) + if full: require_receipt(prev, "prev") require_receipt(cur, "cur") if recipe is None: @@ -314,12 +340,40 @@ def diff_reports(prev: dict[str, Any] | None, cur: dict[str, Any], *, prev_file_ items += _diff_forms(p_rec, c_rec) items += _diff_outside(prev, cur, p_rec, c_rec) items += _diff_ignored(p_rec, c_rec) + items += _diff_ignored_rows(p_rec, c_rec) items += _diff_hidden(p_rec, c_rec) order = {k: i for i, k in enumerate(DIFF_KINDS)} items.sort(key=lambda d: (not d.requires_confirm, order.get(d.kind, len(order)))) return items +#: 比较区域外文字要用的回执键（outside_items） +OUTSIDE_KEYS: tuple[str, ...] = ("outside_text", "sheets") + + +def accumulating(plan: Any) -> bool: + """累积计划是不是「本期和此前各期进同一个版本」：mode=accumulate 且 action 为追加或替换该期。first、restart、 + replace 的结果只有本期，rejected 不提交。""" + plan = plain(plan) + return (isinstance(plan, dict) and plan.get("mode") == "accumulate" + and plan.get("action") in ("append", "replace_period")) + + +def outside_items(prev: dict[str, Any], cur: dict[str, Any]) -> list[DiffItem]: + """上一期 → 本期区域外文字的差异项（outside_added / outside_removed / outside_changed / outside_moved / + context_source_added），与 diff_reports 逐项比较时的那几条相同。 + + 公开出来是给按期累积用的（P3-SPEC 9.3 ConfirmContext.outside_diff）：accumulate_unit_risk 看区域外文字涉及单位 + 或口径的变化，而 diff_reports 在配方变了时不做逐项比较。diff_reports 收到追加或替换该期的累积计划时已经调用 + 这里；调用方另算 outside_diff 时也用它。两边回执都必须带 OUTSIDE_KEYS，缺了报 ReceiptIncomplete，不按空处理 + （按空处理时单位变化就静默消失了）。 + """ + prev, cur = plain(prev) or {}, plain(cur) or {} + require_receipt(prev, "prev", OUTSIDE_KEYS, top=()) + require_receipt(cur, "cur", OUTSIDE_KEYS, top=()) + return _diff_outside(prev, cur, receipt_of(prev), receipt_of(cur)) + + def _recipe_changed(prev: dict[str, Any], cur: dict[str, Any], flag: bool | None) -> bool: """配方与上一期是否不同。传了 flag 就用它（两边都带哈希时还要对得上）；没传就比两边的配方哈希。""" a, b = recipe_sha_of(prev), recipe_sha_of(cur) @@ -332,9 +386,11 @@ def _recipe_changed(prev: dict[str, Any], cur: dict[str, Any], flag: bool | None return flag -def _item(kind: str, label: str, detail: str = "", key: str | None = None) -> DiffItem: +def _item(kind: str, label: str, detail: str = "", key: str | None = None, *, confirm: bool = True) -> DiffItem: + """confirm=False：这一类一般需确认，这一条只是信息（period_backfill 的统计期不是人工录入、ignored_rows 的 + 上一期没有记录）。""" prefix = CONFIRM_PREFIX.get(kind) - if prefix is None: + if prefix is None or not confirm: return DiffItem(kind=kind, label=label, detail=detail) return DiffItem(kind=kind, label=label, detail=detail, requires_confirm=True, confirm_id=f"{prefix}{key}") @@ -774,3 +830,147 @@ def _diff_hidden(p_rec: dict[str, Any], c_rec: dict[str, Any]) -> list[DiffItem] if parts: out.append(_item("hidden", f"工作表「{sheet}」的隐藏行列变了：" + "；".join(parts), key=sheet)) return out + + +# -------------------------------------------------------------------------- +# 期 3：按期累积的计划、排除的行 +# -------------------------------------------------------------------------- + + +def _refs(entries: Any) -> list[str]: + """[{"table", "column" | None}] →「表「日客流」的列「分区丙」」「表「X」」。""" + out = [] + for e in entries or []: + if isinstance(e, dict) and e.get("table"): + out.append(f"表「{e['table']}」的列「{e['column']}」" if e.get("column") else f"表「{e['table']}」") + return out + + +def _diff_accumulate(plan: Any, cur: dict[str, Any]) -> list[DiffItem]: + """按期累积的计划给出的差异项（P3-SPEC 2.3、2.5、9.5）。只在 mode=accumulate、action 是追加或替换该期时出： + first / restart / replace 的结果只有本期，这些项无从说起（restart、模式切换另有必勾的确认项）。""" + if not accumulating(plan): + return [] + action = plan.get("action") + out: list[DiffItem] = [] + period = _period_span(plan.get("period")) + parts = [p for p in plan.get("parts") or [] if isinstance(p, dict)] + if action == "append" and period is not None: + out.append(_item("period_added", f"按期累积：本期 {period[0]} 至 {period[1]} 加入当前版本，启用后共 {len(parts)} 期")) + if action == "replace_period": + rep = _period_span(plan.get("replaces")) or period + if rep is not None: + # 替换由必勾项 period_replace 确认，这里只是信息 + out.append(_item("period_replaced", f"按期累积：替换已有的一期 {rep[0]} 至 {rep[1]}")) + for gap in plan.get("gaps") or []: + g = _period_span(gap) + if g is not None: + out.append(_item("period_gap", f"启用后各期之间有空缺：{g[0]} 至 {g[1]} 没有数据", + "比较不同期之前，请先确认空缺的这段时间是否本来就不需要", key=f"{g[0]}~{g[1]}")) + if plan.get("backfill") and period is not None: + human = (period_of(cur) or {}).get("source") == "human" + if human: + # 人工录入时日期的年份也按统计期补全：录错年份，同一份数据会按错误的年份进来（评审一-m12） + out.append(_item("period_backfill", "本期统计期为人工录入，且早于已有各期：请核对年份和月份无误", + f"本期 {period[0]} 至 {period[1]}，按统计期排在已有各期之前", key=f"{period[0]}~{period[1]}")) + else: + out.append(_item("period_backfill", f"本期 {period[0]} 至 {period[1]} 早于已有各期，按统计期排在最前", + confirm=False)) + added = _refs(plan.get("added")) + if added: + # 写「其他各期」不写「此前各期」：补传早期、替换中间某一期时，没有这些列的是更晚的期 + out.append(_item("columns_added", "新增：" + "、".join(added) + "，其他各期没有这些数据，为空值")) + retired = _refs(plan.get("retired_new")) + if retired: + # 退役由必勾项 retire:* 确认，这里只是信息 + out.append(_item("columns_retired", "自本期起不再导入：" + "、".join(retired) + "，早期各期保留原值")) + out += _labels_vary(plan, period) + return out + + +def _labels_vary(plan: dict[str, Any], period: tuple[str, str] | None) -> list[DiffItem]: + """各期维度取值的差异（label_sets）里本期那一项（评审一-M2）。 + + label_sets 的 missing 是相对全部各期（含本期，补传早期时还含更晚的期）的并集少了哪些，extra 是比各期的交集多了 + 哪些，所以只能说「与其他各期相比」：本期没有的，其他期里至少有一期有；本期多的，其他期里至少有一期没有。不写 + 「此前各期有 / 没有」——8 月没有「7-8」、9 月有，上传 10 月（有「7-8」）时说「此前各期没有」就错了。细节里按期 + 列出其他各期各缺哪些取值，人能看出是哪几期不一样。""" + out: list[DiffItem] = [] + for entry in plan.get("label_sets") or []: + if not isinstance(entry, dict): + continue + col = entry.get("column") or entry.get("segment") or "" + periods = [p for p in entry.get("periods") or [] if isinstance(p, dict)] + mine = next((p for p in periods if _period_span(p) == period), None) + if mine is None: + continue + parts = [] + if mine.get("missing"): + parts.append(f"本期「{col}」没有" + "".join(f"「{short(x, 20)}」" for x in mine["missing"])) + if mine.get("extra"): + parts.append(f"本期「{col}」有" + "".join(f"「{short(x, 20)}」" for x in mine["extra"]) + "，但并非各期都有") + if not parts: + continue + others = [] + for p in periods: + span = _period_span(p) + if p is mine or span is None: + continue + gone = "".join(f"「{short(x, 20)}」" for x in p.get("missing") or []) + others.append(f"{span[0]} 至 {span[1]} " + (f"没有{gone}" if gone else "取值齐全")) + detail = f"表「{entry.get('table')}」：跨期比较时各期覆盖的取值不同" + if others: + detail += "；其他各期：" + "；".join(others) + out.append(_item("labels_vary", "与其他各期相比，" + "；".join(parts), detail)) + return out + + +#: 按配方忽略的行（rows_excluded 的这两种原因）：锚点集合或行数和上一期不同就要确认（P3-SPEC 3.2 ⑥） +_IGNORE_REASONS = ("ignored_rows", "ignored_outside") + + +def _ignored_by_sheet(rec: dict[str, Any]) -> dict[str, tuple[set[tuple[str, str]], int, list[str]]]: + """工作表 → (锚点集合 {(原因, match_key)}, 行数, 锚点原文)。""" + out: dict[str, tuple[set[tuple[str, str]], int, list[str]]] = {} + for e in rec.get("rows_excluded") or []: + e = plain(e) + if not isinstance(e, dict) or e.get("reason") not in _IGNORE_REASONS: + continue + keys, n, raws = out.setdefault(str(e.get("sheet") or ""), (set(), 0, [])) + anchor = str(e.get("anchor") or "") + keys.add((str(e["reason"]), match_key(anchor))) + if anchor and anchor not in raws: + raws.append(anchor) + rows = sum(int(b) - int(a) + 1 for a, b in e.get("rows") or []) + out[str(e.get("sheet") or "")] = (keys, n + rows, raws) + return out + + +def _excluded_total(rec: dict[str, Any]) -> int: + return sum(int(b) - int(a) + 1 for e in (plain(x) for x in rec.get("rows_excluded") or []) + if isinstance(e, dict) for a, b in e.get("rows") or []) + + +def _diff_ignored_rows(p_rec: dict[str, Any], c_rec: dict[str, Any]) -> list[DiffItem]: + """按配方忽略的行（ignore_rows、ignore_outside）和上一期不同：需确认 diff:ignored_rows:<工作表>。 + + 上一期回执里没有 rows_excluded（期 3 之前的导入）时不当作空列表比：那样本期一有排除的行就成了「新增」， + 而上一期其实只是没记录。改出 info「上一期未记录排除的行，本期排除了 N 行」（本期没有排除的行就不出）。""" + if "rows_excluded" not in p_rec: + n = _excluded_total(c_rec) + if n <= 0: + return [] + return [_item("ignored_rows", f"上一期未记录排除的行，本期排除了 {n} 行", "排除的行逐项列在本期回执里", + confirm=False)] + a, b = _ignored_by_sheet(p_rec), _ignored_by_sheet(c_rec) + out = [] + for sheet in dict.fromkeys([*b, *a]): + ka, na, ra = a.get(sheet, (set(), 0, [])) + kb, nb, rb = b.get(sheet, (set(), 0, [])) + if ka == kb and na == nb: + continue + now = "".join(f"「{short(x, 20)}」" for x in rb) or "无" + before = "".join(f"「{short(x, 20)}」" for x in ra) or "无" + out.append(_item("ignored_rows", f"工作表「{sheet}」按配方忽略的行变了：本期 {nb} 行（锚点{now}）", + f"上一期 {na} 行（锚点{before}）", key=sheet)) + return out diff --git a/backend/app/data/recipe_engine.py b/backend/app/data/recipe_engine.py index 4f7b090c..d25961cc 100644 --- a/backend/app/data/recipe_engine.py +++ b/backend/app/data/recipe_engine.py @@ -26,6 +26,19 @@ pk_duplicate 同时出现）。块级的定位失败（找不到日期表头、 建库连接用 engine.open_checked_sqlite（关 DQS，和核对模块同一个入口）。建表语句不经 SQL 守卫 （同 tabular.py 开头的理由：导入是我们自己发起的可信操作）。 + +**期 3（P3-SPEC 9.2、3.1、3.2、第 8 节遗留项 4）在这里加了四件事，没用新字段的配方写出的库与期 2 逐字节相同：** +- 三种忽略规则（ignore_rows / ignore_columns / ignore_outside）：都按文件里的文字认，锚点在某一期没出现就什么也 + 不忽略、也不报错；被忽略的格去向是 ignored（按表头忽略的列沿用 ignored_column），不再报「没有去处」「区域外 + 数字」。 +- 修复按钮的参数 Problem.fix_args：构造补丁要的标签、表头、分段标题、锚点原文和坐标，**不含数据格的数值** + （修复按钮直接把它写进配方，混进数值就等于把本期的数写死在配方里）。fix 和 fix_args 同时有或同时没有：拿不出 + 参数的问题（合并成「另有 N 处」的那条、列表找不到标题、标签不是文字的 row_unclaimed）不给修复按钮，界面也就 + 不会放一个点了没用的按钮。 +- 区域标记带块 id，按（去向、块、列段）合矩形：框选的重放比对按块过滤，网格按块描边，相邻两块的格不再合进同一 + 个矩形。 +- 回执里的「排除的行」rows_excluded（H2：排除的行要写进回执）：隐藏行、列表跳过的空行、按规则忽略的行、 + 停止之后没导入的文字行、只核对不另存的合计行。 """ from __future__ import annotations @@ -47,12 +60,12 @@ from app.data import tabular, xlsx_cells from app.data.names import canon from app.data.recipe_parsers import ( AxisDate, HourRange, HourTotal, Period, classify_outside, cn_date_range, cn_year_month, filename_period, - hour_range, hour_range_total, is_pure_period_cell, match_key, month_day_or_date, period_residue, - text_label, thousands_number, + hour_range, hour_range_total, is_pure_period_cell, looks_numeric_text, match_key, month_day_or_date, + period_residue, text_label, thousands_number, ) from app.data.recipe_types import ( AxisOut, CanonEntry, CheckResult, ColumnOut, ContextSpec, CrosstabBlock, DerivedItem, DimensionSegment, - Extraction, Grid, GridCell, LedgerSheet, ListBlock, MeasuresSegment, OutsideText, + ExcludedRows, Extraction, Grid, GridCell, LedgerSheet, ListBlock, MeasuresSegment, OutsideText, PeriodInput, PeriodOut, Problem, Recipe, RegionMark, SegmentLabels, SheetRecipe, TableOut, TableSpec, derive_tables, ) @@ -96,12 +109,22 @@ PROBLEM_CODES: dict[str, str] = { "pk_duplicate": "structure", } -#: 期 3 修复按钮的种类（契约 Problem.fix）。期 2 只透传 +#: 修复按钮的种类（契约 Problem.fix，取值见 FIX_KINDS）。参数 fix_args 的形状见 P3-SPEC 3.2，由报问题的地方随问题 +#: 一起填（合并类问题在 _Engine._fix_args 里补）。期 3 相对期 2 的改动（3.1）： +#: - row_unclaimed 改为 ignore_cells（按行标签忽略）：分段是连续的数据行，隔了空行的那一行把标签加进分段也认领 +#: 不到，加标签只会再多报一条 label_missing； +#: - cell_unclaimed 不给：数据行里单独一格没有能泛化的锚点（同一行的标签会把整行忽略掉），只能改文件或框选； +#: - rows_after_stop 不给：它是需确认类，勾选即可启用，列表的按行忽略不在期 3 范围内； +#: - column_extra（列表）给 ignore_cells（按表头忽略）、hidden_rows / hidden_cols 给 declare_hidden、 +#: sheet_missing 给 rename_sheet。 +#: edit_members 不来自执行器（来自静态校验的 fact_claim_mismatch），rename_sheet 另有不是问题的触发 +#: （Extraction.sheets.renamed），都不在这里。 PROBLEM_FIX: dict[str, str] = { "label_missing": "remove_label", "label_unexpected": "add_label", "title_not_found": "rename_title", - "axis_extra_cells": "ignore_cells", "outside_number": "ignore_cells", "cell_unclaimed": "ignore_cells", - "row_unclaimed": "add_label", "label_unparsed": "declare_total", "rows_after_stop": "ignore_cells", - "value_not_number": "declare_placeholder", + "label_unparsed": "declare_total", "value_not_number": "declare_placeholder", + "axis_extra_cells": "ignore_cells", "column_extra": "ignore_cells", "row_unclaimed": "ignore_cells", + "outside_number": "ignore_cells", + "hidden_rows": "declare_hidden", "hidden_cols": "declare_hidden", "sheet_missing": "rename_sheet", } #: 列表找表头的窗口：从 after_title 的下一行（或已用区域首行）起往下这么多行 @@ -114,6 +137,13 @@ CELLS_MAX = 20 PER_CODE_MAX = 50 #: 规范写法对照最多几条 CANON_MAX = 500 +#: declare_placeholder 的 fix_args.texts 最多几种（P3-SPEC 3.2 ⑦）；_Issue 多收一种，才知道是不是超了 +PLACEHOLDER_TEXTS_MAX = 8 +#: 能当占位符提议的文字最长几个字（canon 之后）。更长的多半是备注、说明，不是「无数据」的记号 +PLACEHOLDER_TEXT_LEN = 8 +#: 排除的行在回执里的顺序（同一工作表内按原因、块、锚点、首行排） +_EXCLUDED_ORDER = ("hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", + "total_not_kept") #: 公式区域展开的上限：更大的区域不可能全落在基表里，直接判「引用了基表之外的格」 REF_CELLS_MAX = 200_000 @@ -123,6 +153,11 @@ _BLOCK_ROLES = frozenset({"value", "derived_value", "derived_label", "col_header #: 块级定位失败的工作表上不再报的「区域外 / 没有去处」类问题（根因已经报了） _NOISE_CODES = frozenset({"outside_number", "outside_digits", "cell_unclaimed", "row_unclaimed", "row_without_label", "rows_after_stop"}) +#: 像表下说明的开头（按 canon 之后比）。跳过空行的列表里，空行之后只有首列一格文字的行一律当表的下边界（与起草器 +#: 一致，_ListRun._note_after_blank）；以这些开头的只记进回执（rows_excluded、区域外文字），其余的另出 rows_after_stop +#: 要人确认：它也可能是表尾一条只填了首列的数据行，静默移出去就少了一条记录（WP-8 评审意见 4） +_NOTE_PREFIXES = tuple(canon(x) for x in ("注", "说明", "备注", "制表", "填表", "填报", "来源", "数据来源", "单位", + "审核")) #: after_title 最长 40 字，算上空白和全角写法，比这更长的格不可能是它 _TITLE_SCAN_MAX = 200 _PARSER_NAME = { @@ -177,6 +212,28 @@ def _rows_text(rows: list[int], limit: int = 10) -> str: return _join([str(a) if a == b else f"{a}–{b}" for a, b in spans], limit) +def _anchor_text(value: Any) -> str | None: + """能写进 fix_args 当锚点的文字（表头、行标签、区域外文字、候选标题）：只认文字格，原样返回；空白、不是文字、 + 像一个数（「1,234」「12.5%」）的一律 None。 + + 为什么要拦「像数的」：fix_args 不许带数据格的数值（契约 Problem.fix_args），而表头位置上的「987654」、标签列上 + 的「2,345」其实就是数，写进配方的忽略规则还等于把本期的数写死。拦下之后界面给不出这条提议，提示改文件。""" + if not isinstance(value, str) or not value.strip() or looks_numeric_text(value.strip()): + return None + return value + + +def _spans_of(rows: list[list[int]]) -> list[list[int]]: + """[[起, 止], …]（可能乱序、重叠、相接）→ 升序、不重叠、相接的合成一段。""" + out: list[list[int]] = [] + for a, b in sorted(rows): + if out and a <= out[-1][1] + 1: + out[-1][1] = max(out[-1][1], b) + else: + out.append([a, b]) + return out + + def _in_spans(spans: list[tuple[int, int]], starts: list[int], x: int) -> bool: i = bisect.bisect_right(starts, x) - 1 return i >= 0 and spans[i][0] <= x <= spans[i][1] @@ -281,26 +338,33 @@ class _Runs: return [[start, sheet, cell_ref(r, c), n, way] for start, sheet, r, c, n, way in self.runs] -def _row_spans(cells: list[tuple[int, GridCell]], claims: dict[int, str], - layout: list[tuple[int, int, str]] | None) -> list[tuple[int, int, str]]: - """一行的区域段 [(起列, 止列, 去向)]。 +#: 一行里的一个区域段：(起列, 止列, 去向, 块 id)。区域外文字、统计期、按区域外锚点忽略的格块 id 为 None +Span = tuple[int, int, str, "str | None"] + + +def _row_spans(cells: list[tuple[int, GridCell]], claims: dict[int, str], layout: list[Span] | None, + blocks: dict[int, str] | str | None) -> list[Span]: + """一行的区域段 [(起列, 止列, 去向, 块)]。blocks：已认领格所在的块（按列给，或者整行同一个块）。 块登记了这一行的固定列段（layout：列表数据行是本块的列范围，交叉表数据行是标签列和日期列）时，按固定列段 出段，不看这一行哪几格碰巧是空的：末列常空的「备注」、隔行为空的金额，否则每行的段都不一样，20 万行的表会 - 出 20 万个矩形（4.9「区域标记按矩形」）。固定列段里有去向对不上的格（块之间认领重叠）时退回按格出段。 - 其余的格：同一去向的相邻已认领格合成一段（中间的空格一并覆盖，着色无妨），没有认领的格把段隔开（未认领的格 - 不在区域标记里，由问题的 cells 指出）。""" - items: list[tuple[int, int, str | None]] = [] + 出 20 万个矩形（4.9「区域标记按矩形」）。固定列段里有去向或块对不上的格（块之间认领重叠）时退回按格出段。 + 其余的格：同一去向、同一块的相邻已认领格合成一段（中间的空格一并覆盖，着色无妨），没有认领的格把段隔开 + （未认领的格不在区域标记里，由问题的 cells 指出）。不同块的格不合段（期 3：框选的重放比对按块过滤区域， + 并排的两张列表的值区合成一个矩形就分不出各自的范围了）。""" + block_of: Callable[[int], str | None] = (blocks.get if isinstance(blocks, dict) # type: ignore[assignment] + else (lambda c: blocks)) + items: list[tuple[int, int, str | None, str | None]] = [] if layout: - fixed = sorted(layout) + fixed = sorted(layout, key=lambda x: (x[0], x[1])) for c, _ in cells: - for a, b, role in fixed: + for a, b, role, blk in fixed: if a <= c <= b: - if claims.get(c) != role: + if claims.get(c) != role or block_of(c) != blk: layout = None break else: - items.append((c, c, claims.get(c))) + items.append((c, c, claims.get(c), block_of(c))) if layout is None: items = [] break @@ -308,60 +372,61 @@ def _row_spans(cells: list[tuple[int, GridCell]], claims: dict[int, str], items.extend(fixed) items.sort(key=lambda x: x[0]) if not layout: - items = [(c, c, claims.get(c)) for c, _ in cells] + items = [(c, c, claims.get(c), block_of(c)) for c, _ in cells] out: list[list[Any]] = [] broken = True - for a, b, role in items: + for a, b, role, blk in items: if role is None: broken = True continue - if out and not broken and out[-1][2] == role: + if out and not broken and out[-1][2] == role and out[-1][3] == blk: out[-1][1] = max(out[-1][1], b) else: - out.append([a, b, role]) + out.append([a, b, role, blk]) broken = False - return [(a, b, role) for a, b, role in out] + return [(a, b, role, blk) for a, b, role, blk in out] class _Regions: - """按行喂入区域段，同一去向、同一列范围、行相接的合成矩形（给网格着色，契约 RegionMark）。 + """按行喂入区域段，同一去向、同一块、同一列范围、行相接的合成矩形（给网格着色，契约 RegionMark）。 行内怎么分段见 _row_spans。喂入不必全局按行有序：乱序的只是多出几个小矩形，覆盖不受影响（区域外文字要等 统计期定下来才分类，喂得晚）。""" def __init__(self, sheet: str) -> None: self.sheet = sheet - self.open: dict[tuple[str, int, int], list[int]] = {} - self.closed: list[tuple[int, int, int, int, str]] = [] + self.open: dict[tuple[str, str | None, int, int], list[int]] = {} + self.closed: list[tuple[int, int, int, int, str, str | None]] = [] def feed_row(self, r: int, cells: list[tuple[int, str]]) -> None: - """单独几格（区域外文字、统计期）：同一去向的相邻格合成一段。""" - spans: list[tuple[int, int, str]] = [] + """单独几格（区域外文字、统计期、按区域外锚点忽略的数字）：不属于任何块，同一去向的相邻格合成一段。""" + spans: list[Span] = [] for c, role in cells: if spans and spans[-1][2] == role: - spans[-1] = (spans[-1][0], c, role) + spans[-1] = (spans[-1][0], c, role, None) else: - spans.append((c, c, role)) + spans.append((c, c, role, None)) self.feed_spans(r, spans) - def feed_spans(self, r: int, spans: list[tuple[int, int, str]]) -> None: - for c1, c2, role in spans: - key = (role, c1, c2) + def feed_spans(self, r: int, spans: list[Span]) -> None: + for c1, c2, role, block in spans: + key = (role, block, c1, c2) rect = self.open.get(key) if rect is not None and rect[1] == r - 1: rect[1] = r continue if rect is not None: - self.closed.append((rect[0], c1, rect[1], c2, role)) + self.closed.append((rect[0], c1, rect[1], c2, role, block)) self.open[key] = [r, r] def marks(self) -> list[RegionMark]: - rects = list(self.closed) + [(r1, c1, r2, c2, role) for (role, c1, c2), (r1, r2) in self.open.items()] - rects.sort(key=lambda x: (x[0], x[1], x[4])) + rects = list(self.closed) + [(r1, c1, r2, c2, role, block) + for (role, block, c1, c2), (r1, r2) in self.open.items()] + rects.sort(key=lambda x: (x[0], x[1], x[4], x[5] or "")) out = [] - for r1, c1, r2, c2, role in rects: + for r1, c1, r2, c2, role, block in rects: ref = cell_ref(r1, c1) if (r1, c1) == (r2, c2) else f"{cell_ref(r1, c1)}:{cell_ref(r2, c2)}" - out.append(RegionMark(self.sheet, role, ref)) # type: ignore[arg-type] + out.append(RegionMark(self.sheet, role, ref, block)) # type: ignore[arg-type] return out @@ -434,6 +499,11 @@ class _Issue: count: int = 0 cells: list[str] = field(default_factory=list) examples: list[str] = field(default_factory=list) + #: 期 3 修复参数要的位置：所在块（value_not_number）、所在行（outside_number 按行合并，一行一条） + block: str | None = None + row: int | None = None + #: value_not_number：能当占位符提议的文字（canon 后）→ 格数，最多收 PLACEHOLDER_TEXTS_MAX + 1 种 + texts: Counter[str] = field(default_factory=Counter) #: 合并类问题的说明：code → (原因（{n} 是个数）, 下一步)。下一步写配方面板上的「字段」和「选项」原文 @@ -470,44 +540,59 @@ class _Problems: self._over: Counter[tuple[str, str | None]] = Counter() def add(self, code: str, message: str, model_message: str, cells: list[str] | tuple[str, ...] = (), *, - sheet: str | None = None) -> None: + sheet: str | None = None, fix_args: dict[str, Any] | None = None) -> Problem | None: + """记一条问题，返回它（超过 PER_CODE_MAX 合进「另有 N 处」时返回 None）。 + + fix 只在 fix_args 也给了时才填：修复补丁全靠 fix_args 构造，没有参数的修复按钮点了也没用（P3-SPEC 3.1）。 + title_not_found 的候选标题要等所有块认领完才知道，调用方拿返回的 Problem 之后再补（_Crosstab.post_claims）。""" key = (code, sheet) self._per[key] += 1 if self._per[key] > PER_CODE_MAX: self._over[key] += 1 - return - self.items.append((sheet, Problem(code=code, category=PROBLEM_CODES[code], message=message, # type: ignore[arg-type] - cells=list(cells)[:CELLS_MAX], model_message=model_message, - fix=PROBLEM_FIX.get(code)))) + return None + fix = PROBLEM_FIX.get(code) if fix_args is not None else None + p = Problem(code=code, category=PROBLEM_CODES[code], message=message, # type: ignore[arg-type] + cells=list(cells)[:CELLS_MAX], model_message=model_message, fix=fix, + fix_args=fix_args if fix is not None else None) + self.items.append((sheet, p)) + return p def issue(self, code: str, sheet: str | None, where: str, cell: str | list[str], example: str | None = None, *, - text: tuple[str, str] | None = None, group: str = "") -> None: - """记一处同类问题；cell 可以是几个坐标（主键重复要列出两处）。""" + text: tuple[str, str] | None = None, group: str = "", block: str | None = None, + row: int | None = None, sample: str | None = None) -> None: + """记一处同类问题；cell 可以是几个坐标（主键重复要列出两处）。block、row、sample 是修复参数要的： + sample 是 value_not_number 那一格的文字（canon 后），能当占位符提议的才收进 texts。""" key = (code, sheet, where + "\x00" + group) it = self.issues.get(key) if it is None: reason, tail = text or _ISSUE_TEXT[code] - it = self.issues[key] = _Issue(code, sheet, where, reason, tail) + it = self.issues[key] = _Issue(code, sheet, where, reason, tail, block=block, row=row) it.count += 1 for one in ([cell] if isinstance(cell, str) else cell): if len(it.cells) < CELLS_MAX and one not in it.cells: it.cells.append(one) if example is not None and len(it.examples) < 3: it.examples.append(example) + if sample is not None and (sample in it.texts or len(it.texts) <= PLACEHOLDER_TEXTS_MAX): + it.texts[sample] += 1 - def finish(self, failed_sheets: set[str]) -> list[Problem]: + def finish(self, failed_sheets: set[str], + args_for: Callable[[_Issue], dict[str, Any] | None] | None = None) -> list[Problem]: + """合并类问题出成 Problem；args_for 给有修复按钮的合并类问题（outside_number、value_not_number）算 fix_args。 + 「另有 N 处」那一条没有坐标和参数，不给修复按钮。""" for it in self.issues.values(): reason = it.reason.format(n=it.count) shown = [c.split("!")[-1] for c in it.cells] ex = f"（如 {'；'.join(it.examples)}）" if it.examples else f"（{_join(shown, 5)}）" model_ex = f"（{_join(shown, 5)}）" + args = args_for(it) if args_for is not None and it.code in PROBLEM_FIX else None self.add(it.code, f"{it.where}：{reason}{ex}{it.tail}", f"{it.where}：{reason}{model_ex}", it.cells, - sheet=it.sheet) + sheet=it.sheet, fix_args=args) for (code, sheet), n in self._over.items(): where = f"工作表「{sheet}」" if sheet else "本文件" text = f"{where}另有 {n} 处同类问题未逐一列出" self.items.append((sheet, Problem(code=code, category=PROBLEM_CODES[code], message=text, # type: ignore[arg-type] - model_message=text, fix=PROBLEM_FIX.get(code)))) + model_message=text))) return [p for sheet, p in self.items if not (sheet in failed_sheets and p.code in _NOISE_CODES)] def blocking(self) -> bool: @@ -626,6 +711,18 @@ class _Stopped: closed: bool = False rows: list[int] = field(default_factory=list) cells: list[str] = field(default_factory=list) + #: 这些行在本块列范围内的非空格数（rows_excluded 的 after_stop 用；cells 只留前 CELLS_MAX 个坐标） + n: int = 0 + #: 跳过空行的列表按「表下说明」收尾时（_ListRun._note_after_blank，WP-8 修补 3）：说明那一行的行号、坐标、文字 + #: （去了首尾空白），以及它像不像说明（_note_like）。stop 模式碰到空行收尾时都是 None + note_row: int | None = None + note_cell: str | None = None + note_text: str | None = None + note_like: bool = True + #: 按表下说明收尾时，说明之下本块列范围内有字、但不到 need 格的行（第二行说明、单独一格的落款）和它们的格数： + #: 修补 3 之前这些行都会读成数据，现在交给了区域外，同样记进 rows_excluded（after_stop），不另出问题 + tail: list[int] = field(default_factory=list) + tail_n: int = 0 class _Sheet: @@ -641,6 +738,9 @@ class _Sheet: self.truncated = False self.claims: dict[tuple[int, int], str] = {} self.owners: dict[tuple[int, int], str] = {} + #: 格 → 认领它的块 id（区域标记按块合矩形）。owners 里交叉表的格记的是分段 id（重叠报错要写分段名）， + #: 这里统一换算成块 id + self.cell_block: dict[tuple[int, int], str] = {} self.overlaps: set[tuple[str, str]] = set() self.data_rows: set[int] = set() self.suppressed: set[tuple[int, int]] = set() @@ -652,7 +752,16 @@ class _Sheet: self.recon: _Reconciler | None = None self.stopped: list[_Stopped] = [] #: 行 → 块登记的固定列段（_row_spans）：格子账收尾那一行时取走，流式路径里同时只有一两行 - self.layouts: dict[int, list[tuple[int, int, str]]] = {} + self.layouts: dict[int, list[Span]] = {} + #: 排除的行：(原因, 块, 锚点) → [[[起, 止], …], 格数]，_phase4 收尾时出成 ExcludedRows + self.excluded: dict[tuple[str, str | None, str | None], list[Any]] = {} + #: ignore_outside 的锚点：[(match_key, 配方里的原文)]，按配方顺序 + self.outside_rules: list[tuple[str, str]] = [(k, x.anchor) for x in sr.ignore_outside + if (k := match_key(x.anchor))] + #: 有 ignore_outside 时，区域外的数字格先压着，等同一行的区域外文字都分类完再定忽略还是报问题 + self.held: list[tuple[int, int, Any]] = [] + #: 行 → 这一行区域外的文字格 [(列, 原文)]（不含统计期来源）：ignore_outside 的锚点、outside_number 的修复参数 + self.outside_texts: dict[int, list[tuple[int, str]]] = {} self.order: list[tuple[int, str]] = [] self.hidden_claimed_rows: set[int] = set() self.hidden_claimed_cols: set[int] = set() @@ -673,7 +782,7 @@ class _Sheet: def coord(self, r: int, c: int) -> str: return _coord(self.name, r, c) - def layout(self, r: int, spans: list[tuple[int, int, str]]) -> None: + def layout(self, r: int, spans: list[Span]) -> None: """登记这一行的固定列段（区域标记按它出矩形，见 _row_spans）。""" got = self.layouts.get(r) if got is None: @@ -681,16 +790,31 @@ class _Sheet: else: got.extend(spans) - def claim(self, r: int, c: int, role: str, owner: str) -> bool: - """认领一格（只认领网格里的非空格）。被别的块认领过就记重叠、不改。""" + def claim(self, r: int, c: int, role: str, owner: str, block: str | None = None) -> bool: + """认领一格（只认领网格里的非空格）。被别的块认领过就记重叠、不改。block 缺省时就是 owner（列表块、 + 交叉表的表头行用块 id 认领；交叉表分段里的格 owner 是分段 id，另传块 id）。""" prev = self.owners.get((r, c)) if prev is not None and prev != owner: self.overlaps.add((min(prev, owner), max(prev, owner))) return False self.claims[(r, c)] = role self.owners[(r, c)] = owner + self.cell_block[(r, c)] = block or owner return True + def exclude(self, reason: str, first: int, last: int, cells: int, *, block: str | None = None, + anchor: str | None = None) -> None: + """记一段排除的行（rows_excluded）。同一原因、同一块、同一锚点的合成一项。""" + got = self.excluded.setdefault((reason, block, anchor), [[], 0]) + got[0].append([first, last]) + got[1] += cells + + def excluded_rows(self) -> list[ExcludedRows]: + out = [ExcludedRows(self.name, reason, _spans_of(rows), n, anchor=anchor, block=block) # type: ignore[arg-type] + for (reason, block, anchor), (rows, n) in self.excluded.items()] + out.sort(key=lambda x: (_EXCLUDED_ORDER.index(x.reason), x.block or "", x.anchor or "", x.rows[0][0])) + return out + # -------------------------------------------------------------------------- # 交叉表（4.3、4.5、4.6） @@ -769,6 +893,16 @@ class _Crosstab: self.excluded: set[tuple[int, int]] = set() self.cell_index: dict[tuple[int, int], tuple[str, int]] = {} self.exclude_hidden = sh.sr.hidden.rows == "exclude" + #: 期 3 忽略规则：match_key → 配方里的原文（同一键取第一条；同键重复是静态校验的 ignore_duplicate） + self.ignore_rows: dict[str, str] = {} + for rule in block.ignore_rows: + if (k := match_key(rule.label)): + self.ignore_rows.setdefault(k, rule.label) + self.ignore_cols = {k for rule in block.ignore_columns if (k := match_key(rule.header))} + #: 按表头忽略的列（轴行右侧） + self.ignored_cols: list[int] = [] + #: 本块报出的 title_not_found（候选标题要等所有块认领完，在 post_claims 里补进 fix_args） + self.title_missing: list[Problem] = [] # ---------------- 定位 ---------------- @@ -821,6 +955,10 @@ class _Crosstab: return self.axis_cells = dates extra = [c for c, _ in row if c > self.last_c] + # 期 3：表头命中 ignore_columns 的列按表头忽略，不报 axis_extra_cells；其余照期 2 + self.ignored_cols = [c for c in extra if self.ignore_cols + and match_key(_text(grid.get(r, c).value)) in self.ignore_cols] # type: ignore[union-attr] + extra = [c for c in extra if c not in self.ignored_cols] if extra: self.extra_cols = set(extra) shown = [f"{cell_ref(r, c)}「{_shown(grid.get(r, c).value)}」" for c in extra] @@ -828,7 +966,10 @@ class _Crosstab: "axis_extra_cells", f"{self.where}：日期表头（第 {r} 行）最后一个日期右边还有内容：{_join(shown)}。请删除这些列，或调整配方", f"{self.where}：日期表头（第 {r} 行）最后一个日期右边还有内容：{_join([cell_ref(r, c) for c in extra])}", - [sh.coord(r, c) for c in extra], sheet=sh.name) + [sh.coord(r, c) for c in extra], sheet=sh.name, + fix_args={"block": block.id, "how": "columns", + "headers": [{"raw": _anchor_text(grid.get(r, c).value), "cell": sh.coord(r, c)} # type: ignore[union-attr] + for c in extra]}) for (rr, cc) in grid.cells: if rr >= r and cc in self.extra_cols: sh.suppressed.add((rr, cc)) @@ -840,6 +981,26 @@ class _Crosstab: self._classify_rows() self._locate_segments() + def _claim_ignored_cols(self, bottom: int) -> None: + """按表头忽略的列：从轴行到本块数据区底部（bottom，本块最后一个数据行，post_claims 算）的格，去向 + ignored_column；表头记进 Extraction.ignored_columns[块]。数据区之下的格不归这条规则管（照常算区域外）， + 免得表下的备注、落款被一并吞掉。 + + 为什么放在 post_claims、底部只按本块自己的行算：kinds 覆盖轴行以下整张工作表，表下另有一张列表时，列表的 + 行同样「标签列有字、日期列有值」。按整张表的 data 行取底部、又在列表认领之前先认领，被忽略的这一列会一直 + 认领到列表的最后一行，列表再认领同一批格就成了 segment_overlap（结构类、没有修复按钮）：用户按了 + 「按表头忽略」，换来一条看不懂的结构错误。已归别的块的格照样跳过，与期 2 多出的列（axis_extra_cells 只 + 压掉、不认领）对别的块的效果一致。""" + if not self.ignored_cols: + return + grid, sh, r = self.grid, self.sh, self.axis_row + for rr in range(r, bottom + 1): + for cc in self.ignored_cols: + if grid.get(rr, cc) is not None and (rr, cc) not in sh.owners: + sh.claim(rr, cc, "ignored_column", self.block.id) + self.eng.ex.ignored_columns[self.block.id] = [_text(grid.get(r, c).value).strip() # type: ignore[union-attr] + for c in self.ignored_cols] + def _label(self, r: int) -> GridCell | None: return self.grid.get(r, self.label_c) @@ -909,7 +1070,10 @@ class _Crosstab: self.failed.add(seg.id) if not self._missing_ok(): msg = f"{where}：没有找到分段标题「{seg.locate.title}」" - eng.problems.add("title_not_found", msg, msg, sheet=sh.name) + p = eng.problems.add("title_not_found", msg, msg, sheet=sh.name, + fix_args={"segment": seg.id, "title": seg.locate.title, "candidates": []}) + if p is not None: + self.title_missing.append(p) return if len(hits) > 1: self.failed.add(seg.id) @@ -1001,15 +1165,24 @@ class _Crosstab: cells = [sh.coord(r, lc) for r in unparsed] shown = [f"{cell_ref(r, lc)}「{_shown(self._label(r).value)}」" for r in unparsed] # type: ignore[union-attr] name = _PARSER_NAME[spec.parser] - eng.problems.add("label_unparsed", f"{where}：{_join(shown)}的标签无法按{name}解析", - f"{where}：{_join([cell_ref(r, lc) for r in unparsed])} 的标签无法按{name}解析", - cells, sheet=sh.name) + # declare_total 的前提之一：解析不了的行都在分段末尾、连成一段（P3-SPEC 3.2 ⑤） + at_end = unparsed == loc.rows[len(loc.rows) - len(unparsed):] + eng.problems.add( + "label_unparsed", f"{where}：{_join(shown)}的标签无法按{name}解析", + f"{where}：{_join([cell_ref(r, lc) for r in unparsed])} 的标签无法按{name}解析", cells, sheet=sh.name, + fix_args={"segment": seg.id, "role": spec.role, "at_end": at_end, + "labels": [{"raw": _text(self._label(r).value), "cell": sh.coord(r, lc), # type: ignore[union-attr] + "total": hour_range_total(self._label(r).value) is not None} # type: ignore[union-attr] + for r in unparsed]}) if unexpected: cells = [sh.coord(r, lc) for r in unexpected] shown = [f"{cell_ref(r, lc)}「{_shown(self._label(r).value)}」" for r in unexpected] # type: ignore[union-attr] - eng.problems.add("label_unexpected", f"{where}：{_join(shown)}的标签不在期望的标签中", - f"{where}：{_join([cell_ref(r, lc) for r in unexpected])} 的标签不在期望的标签中", - cells, sheet=sh.name) + eng.problems.add( + "label_unexpected", f"{where}：{_join(shown)}的标签不在期望的标签中", + f"{where}：{_join([cell_ref(r, lc) for r in unexpected])} 的标签不在期望的标签中", cells, sheet=sh.name, + fix_args={"segment": seg.id, "role": spec.role, + "labels": [{"raw": _text(self._label(r).value), "cell": sh.coord(r, lc)} # type: ignore[union-attr] + for r in unexpected]}) if dup: cells = [sh.coord(r, lc) for r in dup] pairs = [f"{cell_ref(loc.first[canonical[loc.rows.index(r)]], lc)} 与 {cell_ref(r, lc)}" for r in dup] @@ -1022,54 +1195,78 @@ class _Crosstab: cells = [sh.coord(loc.rows[0], lc)] if loc.rows else ( [sh.coord(loc.title_row, lc)] if loc.title_row else []) msg = f"{where}：期望的标签{_quoted(missing)}没有找到" - eng.problems.add("label_missing", msg, msg, cells, sheet=sh.name) + eng.problems.add("label_missing", msg, msg, cells, sheet=sh.name, + fix_args={"segment": seg.id, "labels": list(missing)}) def _claim(self, loc: _SegLoc) -> None: - sh, grid, seg = self.sh, self.grid, loc.spec.seg + sh, grid, seg, bid = self.sh, self.grid, loc.spec.seg, self.block.id derived = loc.spec.role == "derived" + # 只核对、不另存的合计行（keep_as 为空）：参与核对，但不进任何表，记进「排除的行」（遗留项 4 的 total_not_kept） + not_kept = derived and seg.keep_as is None if loc.title_row is not None: - sh.claim(loc.title_row, self.label_c, "section_title", seg.id) + sh.claim(loc.title_row, self.label_c, "section_title", seg.id, bid) for r in loc.rows: sh.data_rows.add(r) excluded = self.exclude_hidden and sh.hidden_row(r) label_role = "hidden_excluded" if excluded else ("derived_label" if derived else "row_label") value_role = "hidden_excluded" if excluded else ("derived_value" if derived else "value") # 区域按标签列和整段日期列出（某天是空格的行不另起矩形） - sh.layout(r, [(self.label_c, self.label_c, label_role), (self.first_c, self.last_c, value_role)]) - sh.claim(r, self.label_c, label_role, seg.id) + sh.layout(r, [(self.label_c, self.label_c, label_role, bid), (self.first_c, self.last_c, value_role, bid)]) + n = int(sh.claim(r, self.label_c, label_role, seg.id, bid)) for c, _ in grid.row(r): if self.first_c <= c <= self.last_c: - sh.claim(r, c, value_role, seg.id) + n += sh.claim(r, c, value_role, seg.id, bid) if excluded: self.excluded.add((r, c)) + if not_kept and not excluded and n: + sh.exclude("total_not_kept", r, r, n, block=bid) def post_claims(self) -> None: """所有块都认领完之后：没有分段认领的数据行、没有标签的行、数据区的合并单元格。""" if not self.located: return sh, grid, eng = self.sh, self.grid, self.eng - lc = self.label_c + lc, bid = self.label_c, self.block.id + # 本块的数据行（按表头忽略的列认领到哪一行为止）：分段认领的行，加上下面没有分段认领、又不归别的块的 + # 数据行（按 ignore_rows 忽略的、报 row_unclaimed 的、分段没定位到而压掉的），都是本块要负责的行 + own = set(self.row_owner) for r, kind in sorted(self.kinds.items()): if kind == "title" or r in self.row_owner: continue row = grid.row(r) - if any((r, c) in sh.owners and sh.owners[(r, c)] != self.block.id for c, _ in row): + if any((r, c) in sh.owners and sh.owners[(r, c)] != bid for c, _ in row): continue # 别的块（列表）认领了这一行 + if kind == "data": + own.add(r) + if kind == "data" and self.ignore_rows and self._ignore_row(r, row): + continue for c, _ in row: sh.suppressed.add((r, c)) if kind == "data": if self.failed: continue # 有分段没定位到：这些行多半就是它的，不再逐行报 coord = sh.coord(r, lc) + label = self._label(r).value # type: ignore[union-attr] + # 修复按钮只有「按行标签忽略」（3.1：把标签加进分段认领不到隔了空行的这一行）。标签不是文字（数、 + # 像数的文字）时没有能写进 ignore_rows 的锚点，不给按钮（fix_args 为空，fix 随之为空），提示也不提 + # 按标签忽略；框选成分段同样要按标签文字认，也不提 + text = _anchor_text(label) + if text is None: + hint = "它的标签不是文字（例如是一个数），无法按标签忽略。请删除这一行，或把标签改成文字后重新上传" + args = None + else: + hint = "请删除这一行，或按行标签忽略这一行；也可以在网格上把它框选为新的分段" + args = {"block": bid, "how": "rows", "labels": [{"raw": text, "cell": coord}]} eng.problems.add( "row_unclaimed", - f"工作表「{sh.name}」第 {r} 行（{cell_ref(r, lc)}「{_shown(self._label(r).value)}」）有数据，" # type: ignore[union-attr] - "但没有分段认领这一行。请在配方面板中把它的标签添加到某个分段，或删除这一行", + f"工作表「{sh.name}」第 {r} 行（{cell_ref(r, lc)}「{_shown(label)}」）有数据，" + f"但没有分段认领这一行。{hint}", f"工作表「{sh.name}」第 {r} 行有数据，但没有分段认领这一行（标签在 {cell_ref(r, lc)}）", - [coord] + [sh.coord(r, c) for c, _ in row if c != lc], sheet=sh.name) + [coord] + [sh.coord(r, c) for c, _ in row if c != lc], sheet=sh.name, fix_args=args) else: msg = f"工作表「{sh.name}」第 {r} 行有数据，但标签列 {cell_ref(r, lc)} 是空的" eng.problems.add("row_without_label", msg, msg, [sh.coord(r, c) for c, _ in row], sheet=sh.name) + self._claim_ignored_cols(max(own, default=self.axis_row)) data = {r for r, k in self.kinds.items() if k == "data"} for r1, c1, r2, c2 in grid.merges: if c2 < self.first_c or c1 > self.last_c: @@ -1077,6 +1274,36 @@ class _Crosstab: if any(r in data for r in range(max(r1, self.axis_row + 1), r2 + 1)): ref = cell_ref(r1, c1) if (r1, c1) == (r2, c2) else f"{cell_ref(r1, c1)}:{cell_ref(r2, c2)}" eng.problems.issue("merged_in_values", sh.name, self.where, sh.coord(r1, c1), ref) + if self.title_missing: + # rename_title 的候选：本块里像分段标题、又没被任何块认领的行（按行号升序）。要等所有块都认领完： + # 后面的分段、列表块可能还会认领标题行（P3-SPEC 3.2 ④） + cands = [] + for r, kind in sorted(self.kinds.items()): + if kind != "title" or (r, lc) in sh.owners: + continue + text = _anchor_text(self._label(r).value) # type: ignore[union-attr] + if text is not None and len(cands) < CELLS_MAX: + cands.append({"cell": sh.coord(r, lc), "text": text}) + for p in self.title_missing: + p.fix_args["candidates"] = list(cands) # type: ignore[index] + + def _ignore_row(self, r: int, row: list[tuple[int, GridCell]]) -> bool: + """没有分段认领的数据行，标签命中 ignore_rows 时按配方忽略：标签格和日期列上的格去向 ignored，不报 + row_unclaimed，记进「排除的行」。这一行别处的格（标签左边、轴行右侧没有表头的列）不归这条规则管，照常 + 算区域外：规则只说了忽略这一行的数据。ignored 不算块认领的格（不在 _BLOCK_ROLES）：这一行即使隐藏了 + 也不报 hidden_rows，反正不导入。""" + sh, bid = self.sh, self.block.id + rule = self.ignore_rows.get(match_key(_text(self._label(r).value))) # type: ignore[union-attr] + if rule is None: + return False + n = 0 + for c, _ in row: + if c == self.label_c or self.first_c <= c <= self.last_c: + n += sh.claim(r, c, "ignored", bid) + sh.layout(r, [(self.label_c, self.label_c, "ignored", bid), (self.first_c, self.last_c, "ignored", bid)]) + if n: + sh.exclude("ignored_rows", r, r, n, block=bid, anchor=rule) + return True # ---------------- 年份与断言 ---------------- @@ -1372,6 +1599,9 @@ class _ListRun: self.last_row = 0 self.done = False self.stop: tuple[str, int | None] | None = None + #: skip 模式下，上一条数据行之后已经按空行跳过的几段（起、止）：随后判定表已结束（表下的说明文字）时要撤回 + #: 计数和排除的行，那几行不是「数据中间的空行」，而是表的下边界 + self.blank_run: list[tuple[int, int]] = [] self.row_rowid: dict[int, int] = {} self.rowid_first: int | None = None self.rowid_last: int | None = None @@ -1386,6 +1616,8 @@ class _ListRun: spec = eng.specs.get(block.table) self.grain = set(spec.grain) if spec else set() self.sink = eng.sinks[block.table] + #: 期 3：按表头忽略的列（ignore_columns）的 match_key。只管命中的列，其余多出的列照 extra_columns + self.ignore_keys = {k for rule in block.ignore_columns if (k := match_key(rule.header))} total = block.rows.total_row self.total_idx = next((i for i, c in enumerate(self.cols) if total and c.name == total.label_column), None) @@ -1592,14 +1824,22 @@ class _ListRun: eng.problems.add("header_ambiguous", msg, msg, [sh.coord(hb, c) for c in cs], sheet=sh.name) bad = True if extra: - if self.block.extra_columns == "ignore": - self.ignored = set(extra) - eng.ex.ignored_columns[self.block.id] = [composed[c] for c in extra] - else: - shown = [f"「{composed[c]}」（{cell_ref(hb, c)}）" for c in extra] + # 期 3：表头命中 ignore_columns 的列按期 2 extra_columns=ignore 的方式忽略；其余多出的列照 extra_columns + ignored = [c for c in extra if self.ignore_keys and match_key(composed[c]) in self.ignore_keys] + rest = [c for c in extra if c not in ignored] + if rest and self.block.extra_columns == "ignore": + ignored, rest = extra, [] + if ignored: + self.ignored = set(ignored) + eng.ex.ignored_columns[self.block.id] = [composed[c] for c in ignored] + if rest: + shown = [f"「{composed[c]}」（{cell_ref(hb, c)}）" for c in rest] eng.problems.add("column_extra", f"{self.where}：表头多出了列{_join(shown)}", - f"{self.where}：表头多出了 {_join([cell_ref(hb, c) for c in extra])} 这几列", - [sh.coord(hb, c) for c in extra], sheet=sh.name) + f"{self.where}：表头多出了 {_join([cell_ref(hb, c) for c in rest])} 这几列", + [sh.coord(hb, c) for c in rest], sheet=sh.name, + fix_args={"block": self.block.id, "how": "columns", + "headers": [{"raw": _anchor_text(composed[c]), "cell": sh.coord(hb, c)} + for c in rest]}) bad = True if bad: self._fail() @@ -1608,15 +1848,16 @@ class _ListRun: self.state = "data" self.last_row = hb # 数据行的固定列段：映射列是 value、忽略的列是 ignored_column，按列相邻合段（区域标记用，_row_spans） - layout: list[tuple[int, int, str]] = [] + bid = self.block.id + layout: list[Span] = [] for c in range(self.c1, self.c2 + 1): role = "ignored_column" if c in self.ignored else "value" if layout and layout[-1][2] == role: - layout[-1] = (layout[-1][0], c, role) + layout[-1] = (layout[-1][0], c, role, bid) else: - layout.append((c, c, role)) + layout.append((c, c, role, bid)) self.layout_data = layout - self.layout_hidden = [(self.c1, self.c2, "hidden_excluded")] + self.layout_hidden: list[Span] = [(self.c1, self.c2, "hidden_excluded", bid)] self.sink.sources.append(f"{sh.sr.id}/{self.block.id}") self.merges = _MergeSweep(sh.merges, self.c1, self.c2) sh.order.append((t, self.block.id)) @@ -1658,10 +1899,65 @@ class _ListRun: # ---------------- 数据行 ---------------- - def _stop_blank(self, row: int) -> None: + def _stop_blank(self, row: int, note: tuple[int, GridCell] | None = None) -> _Stopped: + """列表到 row 之前为止（row 是收尾的那一段空行的第一行）。note：按表下说明收尾时说明那一行（行号, 那一格）， + 记进 _Stopped，_phase4 据此写排除的行、判要不要人确认。""" self.done = True self.stop = ("blank", row) - self.sh.stopped.append(_Stopped(self.block.id, row, self.c1, self.c2, min(2, self.c2 - self.c1 + 1))) + st = _Stopped(self.block.id, row, self.c1, self.c2, min(2, self.c2 - self.c1 + 1)) + if note is not None: + r, cell = note + st.note_row, st.note_cell = r, self.sh.coord(r, self.c1) + st.note_text = _text(cell.value).strip() + st.note_like = canon(st.note_text).startswith(_NOTE_PREFIXES) + self.sh.stopped.append(st) + return st + + def _skip_blank(self, first: int, last: int) -> None: + """skip 模式下按空行跳过 first..last：计数、记排除的行（blank_skipped，整行没有要导入的格，格数为 0）， + 并记进 blank_run，随后若判定表已到下边界（_note_after_blank）就撤回。""" + self.eng.ex.blank_rows_skipped += last - first + 1 + self.sh.exclude("blank_skipped", first, last, 0, block=self.block.id) + self.blank_run.append((first, last)) + + def _unskip_blanks(self) -> int: + """撤回 blank_run 里记的空行（它们其实是表的下边界，不是数据中间的空行），返回第一行。""" + first = self.blank_run[0][0] + got = self.sh.excluded.get(("blank_skipped", self.block.id, None)) + for a, b in self.blank_run: + self.eng.ex.blank_rows_skipped -= b - a + 1 + if got is not None and [a, b] in got[0]: + got[0].remove([a, b]) + if got is not None and not got[0]: + del self.sh.excluded[("blank_skipped", self.block.id, None)] + self.blank_run = [] + return first + + def _note_after_blank(self, inb: dict[int, GridCell]) -> bool: + """skip 模式下，空行之后这一行在本块列范围内只有首列一格文字、其余列全空：是表下隔着空行的说明文字 + （「注：……」「制表人：……」），不是数据行（WP-8 修补，P3-SPEC 1.5 留给 WP-8 的那一条）。 + + 起草器本来就这么认（recipe_suggest._compatible：空行之后要有两格以上、类型对得上才算同一张表），执行器却把 + 空行之后的任何一行都当数据读：说明文字静默成了一条首列是「注：……」、其余全为空值的记录，回执里只多一行， + 区域外文字里也看不到它（H2）。这里按同一条规则把它判为表的下边界：这一行和下面的行交给区域外，文字照常进 + 回执，数字照常报 outside_number。首列是数、日期，或者正是合计行的标签时不算（那是缺数的数据行或合计行）； + 只有一列的块分不出来，照旧当数据。 + + 判成下边界不等于静默（WP-8 评审意见 4）：这一行和其下交给区域外的行都记进排除的行（after_stop，锚点是这格 + 文字）；文字不像说明（_NOTE_PREFIXES）时另出 rows_after_stop 要人确认，见 _Engine._phase4。""" + if self.c2 <= self.c1 or len(inb) != 1 or self.c1 not in inb: + return False + cell = inb[self.c1] + if cell.formula is not None or not isinstance(cell.value, str): + return False + text = cell.value.strip() + if not text or looks_numeric_text(text) or month_day_or_date(text) is not None: + return False + total = self.block.rows.total_row + if (total is not None and self.total_idx is not None and self.colmap.get(self.total_idx) == self.c1 + and match_key(text).startswith(match_key(total.pick))): + return False + return True def _data_row(self, r: int, cells: list[tuple[int, GridCell]]) -> None: if self.done: @@ -1669,25 +1965,34 @@ class _ListRun: return block, eng = self.block, self.eng gap = r - (self.last_row + 1) - if gap > 0: - if block.rows.blank_rows == "stop": - self._stop_blank(self.last_row + 1) - self.emit(r, cells, {}, False) - return - eng.ex.blank_rows_skipped += gap - self.last_row = r inb = {c: cell for c, cell in cells if self.c1 <= c <= self.c2} if inb and self.ignored and all(c in self.ignored for c in inb): # 只有忽略的列有字（表下紧跟的「注：以上为初步统计」写在备注列）：这一行没有一格要导入，按空行处理， # 格子交给区域外（文字进回执、数字照常报）。当成数据行会凭空写进一条全是空值的记录 inb = {} + if (block.rows.blank_rows == "skip" and inb and self.data_rows and (gap > 0 or self.blank_run) + and self._note_after_blank(inb)): + # 表下隔着空行的说明文字：列表到上一条数据行为止，和 stop 模式碰到空行一样收尾（之后的行交给区域外） + self._stop_blank(self._unskip_blanks() if self.blank_run else self.last_row + 1, + note=(r, inb[self.c1])) + self.emit(r, cells, {}, False) + return + if gap > 0: + if block.rows.blank_rows == "stop": + self._stop_blank(self.last_row + 1) + self.emit(r, cells, {}, False) + return + self._skip_blank(self.last_row + 1, r - 1) + self.last_row = r if not inb: if block.rows.blank_rows == "stop": self._stop_blank(r) else: - eng.ex.blank_rows_skipped += 1 + # 这一行的格都交给区域外照常记账，不算被排除 + self._skip_blank(r, r) self.emit(r, cells, {}, False) return + self.blank_run = [] fills = self._merges_at(r, inb) if self.sh.hidden_row(r): self.hidden_seen = True @@ -1823,6 +2128,11 @@ class _ListRun: lin = [(col.name, r, self.colmap[i]) for i, col in enumerate(self.cols) if col.type in ("INTEGER", "REAL")] ks.add([canon(label_raw), *keep_values], sheet=sh.name, at=(r, self.colmap[self.total_idx]), # type: ignore[index] cells=lin, fails=True) + elif not total.keep_as: + # 只核对、不另存的合计行：参与核对，但不进任何表，记进「排除的行」（遗留项 4 的 total_not_kept） + n = sum(1 for role in claims.values() if role in ("total_label", "total_value")) + if n: + sh.exclude("total_not_kept", r, r, n, block=block.id) self.done = True self.stop = ("total", r) self.emit(r, cells, claims, True) @@ -1843,6 +2153,15 @@ class _ListRun: # -------------------------------------------------------------------------- +def _placeholder_sample(key: str) -> str | None: + """value_not_number 的这一格能不能当占位符提议（P3-SPEC 3.2 ⑦）：canon 之后不超过 PLACEHOLDER_TEXT_LEN 个字、 + 不像一个数，而且**不含数字**。规格只要求前两条；多拦「含数字」是因为 fix_args 不许带数据格的数值，「约987654」 + 「9876.5万」这种写法不像一个数，却正是本期的数（含数字的占位符本来也没有意义：「—」「无」「/」才是）。""" + if not key or len(key) > PLACEHOLDER_TEXT_LEN or looks_numeric_text(key) or any(ch.isdigit() for ch in key): + return None + return key + + class _Engine: def __init__(self, recipe: Recipe, raw: bytes, filename: str, scan: WorkbookScan, db_path: str | None, context_inputs: dict[str, PeriodInput] | None, max_rows: int | None) -> None: @@ -1864,6 +2183,8 @@ class _Engine: self.canon_log: dict[tuple[str, str, str, str], CanonEntry] = {} #: 每个块自己的占位符（canon 后 → 配方里的原文）：别的块声明的占位符不能在这里放过 self._ph: dict[int, dict[str, str]] = {} + #: 块的取值规则对象 → 块 id：值的转换只拿到取值规则，declare_placeholder 的 fix_args 要写块 id + self._pol_block: dict[int, str] = {id(b.values): b.id for s in recipe.sheets for b in s.blocks} self.books = xlsx_cells.Workbooks(raw) self.sheets: list[_Sheet] = [] @@ -1924,8 +2245,10 @@ class _Engine: """非原生数字的值（文本、bool、日期……）。pk：主键列，占位符和空白文本都会变成空值，一律拒收。""" p = self.problems - def bad(code: str) -> Any: - p.issue(code, sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」") + def bad(code: str, sample: str | None = None) -> Any: + block = self._pol_block.get(id(pol)) if code == "value_not_number" else None + p.issue(code, sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」", block=block, + sample=sample) return _FAIL if isinstance(v, bool): @@ -1963,6 +2286,7 @@ class _Engine: if pol.text_number != "parse_thousands": return bad("value_text_number") return self._number(num, vtype, pol, sheet, where, r, c, pk) + return bad("value_not_number", _placeholder_sample(key)) return bad("value_not_number") def convert_list(self, cell: GridCell | None, col: Any, is_pk: bool, pol: Any, sheet: str, where: str, @@ -2041,12 +2365,14 @@ class _Engine: # ---------------- 格子账 ---------------- def finish_row(self, sh: _Sheet, r: int, cells: list[tuple[int, GridCell]], claims: dict[int, str], - data_row: bool) -> None: + data_row: bool, blocks: dict[int, str] | str | None = None) -> None: + """格子账的一行。blocks：已认领格所在的块（网格路径按格给，流式路径整行是同一个列表块）。""" sh.read_count += len(cells) if sh.recon is not None: sh.recon.feed(r, len(cells)) layout = sh.layouts.pop(r, None) if sh.layouts else None unclaimed: list[int] = [] + hidden_excluded = 0 for c, cell in cells: role = claims.get(c) if role is not None: @@ -2056,6 +2382,8 @@ class _Engine: sh.hidden_claimed_rows.add(r) if sh.hidden_col(c): sh.hidden_claimed_cols.add(c) + if role == "hidden_excluded": + hidden_excluded += 1 continue if (r, c) in sh.suppressed: continue @@ -2070,12 +2398,17 @@ class _Engine: self._outside_number(sh, r, c, cell.value) else: self._outside_text(sh, r, c, _text(cell.value), kind) + if hidden_excluded: + # 排除的隐藏行按工作表记（契约 ExcludedRows.block：hidden_excluded 为 None），各块的合成一项 + sh.exclude("hidden_excluded", r, r, hidden_excluded) if claims or layout: - sh.regions.feed_spans(r, _row_spans(cells, claims, layout)) + sh.regions.feed_spans(r, _row_spans(cells, claims, layout, blocks)) for c in unclaimed: self.problems.issue("cell_unclaimed", sh.name, f"工作表「{sh.name}」第 {r} 行", sh.coord(r, c)) for st in sh.stopped: - if st.closed or r <= st.stop_row: + # 按表下说明收尾的：说明那一行由 _phase4 单独记，它和上面那段空行都不在这里看。空行里可能有只写在忽略列 + # 的字，网格路径收尾时 _Stopped 已经在了、流式路径还没有，看了两条路径的结果就不一样 + if st.closed or r <= st.stop_row or (st.note_row is not None and r <= st.note_row): continue if claims: st.closed = True # 到了别的块的认领区域 @@ -2083,11 +2416,59 @@ class _Engine: inside = [c for c, _ in cells if st.c1 <= c <= st.c2] if len(inside) >= st.need: st.rows.append(r) + st.n += len(inside) st.cells.extend(sh.coord(r, c) for c in inside[:CELLS_MAX - len(st.cells)]) + elif inside and st.note_row is not None: + st.tail.append(r) + st.tail_n += len(inside) def _outside_number(self, sh: _Sheet, r: int, c: int, value: Any) -> None: + """区域外的数字格。配了 ignore_outside 的工作表先压着：同一行有没有锚点文字，要等区域外的文字都分类完 + （_phase4）才知道。没配的照期 2 当场记问题，问题的先后顺序不变。""" + if sh.outside_rules: + sh.held.append((r, c, value)) + return + self._report_outside_number(sh, r, c, value) + + def _report_outside_number(self, sh: _Sheet, r: int, c: int, value: Any) -> None: self.problems.issue("outside_number", sh.name, f"工作表「{sh.name}」第 {r} 行", sh.coord(r, c), - f"{cell_ref(r, c)}「{_shown(value)}」") + f"{cell_ref(r, c)}「{_shown(value)}」", row=r) + + def _ignore_outside(self, sh: _Sheet) -> None: + """ignore_outside：区域外的数字格所在行，如果有区域外文字格的 match_key 等于某条锚点，这些数字格去向 + ignored，记进「排除的行」；锚点格本身照常是区域外文字（已记录、照常比对）。一行命中几条锚点时取配方里 + 排在前面的那条。没命中的照常报 outside_number。""" + keys = {r: {match_key(t) for _, t in items} for r, items in sh.outside_texts.items()} + for r, c, value in sorted(sh.held, key=lambda x: (x[0], x[1])): + row_keys = keys.get(r) + hit = next((raw for key, raw in sh.outside_rules if key in row_keys), None) if row_keys else None + if hit is None: + self._report_outside_number(sh, r, c, value) + continue + sh.roles["ignored"] += 1 + sh.regions.feed_row(r, [(c, "ignored")]) + sh.exclude("ignored_outside", r, r, 1, anchor=hit) + sh.held = [] + + def _fix_args(self, it: _Issue) -> dict[str, Any] | None: + """合并类问题的修复参数（P3-SPEC 3.2 ⑥ outside、⑦），在所有工作表收尾之后算：区域外文字要等统计期定下来 + 才分类，锚点那时才齐。""" + if it.code == "value_not_number": + if it.block is None: + return None + texts = list(it.texts)[:PLACEHOLDER_TEXTS_MAX] + return {"block": it.block, "texts": texts, "other": it.count - sum(it.texts[t] for t in texts)} + if it.code == "outside_number" and it.row is not None: + sh = next((x for x in self.sheets if x.name == it.sheet), None) + if sh is None: + return None + # 锚点：这一行区域外的文字格，优先取不含数字的（含数字的锚点下一期多半对不上，修复按钮不会采用） + items = [(c, t) for c, t in sorted(sh.outside_texts.get(it.row, [])) if _anchor_text(t) is not None] + pick = next(((c, t) for c, t in items if not any(ch.isdigit() for ch in t)), items[0] if items else None) + return {"sheet": sh.name, "sheet_id": sh.sr.id, "how": "outside", + "rows": [{"row": it.row, "anchor": pick[1] if pick else None, + "anchor_cell": sh.coord(it.row, pick[0]) if pick else None, "cells": list(it.cells)}]} + return None def _outside_text(self, sh: _Sheet, r: int, c: int, text: str, kind: str, *, period_source: bool = False, role: str = "outside_text") -> None: @@ -2245,8 +2626,12 @@ class _Engine: # 块级定位失败的工作表：区域都不知道在哪，「区域外文字」只是噪声（统计期来源照留） self.ex.outside_text = [o for o in self.ex.outside_text if o.period_source or o.sheet not in failed_sheets] + if failed_sheets: + # 同理：区域外按锚点忽略的格、列表停止之后的文字行，在区域定不下来的工作表上也只是噪声 + self.ex.rows_excluded = [x for x in self.ex.rows_excluded if not ( + x.sheet in failed_sheets and x.reason in ("ignored_outside", "after_stop"))] blocking = self.problems.blocking() - self.ex.problems = self.problems.finish(failed_sheets) + self.ex.problems = self.problems.finish(failed_sheets, self._fix_args) t4 = time.perf_counter() if self.conn is not None: if blocking: @@ -2308,26 +2693,31 @@ class _Engine: def _match_sheets(self) -> None: visible = [s for s in self.scan.sheets if s.state == "visible" and s.nonempty > 0] used: set[str] = set() + #: 找不到的工作表先记下，等全部认完再报：rename_sheet 的候选是「配方里其他工作表没认走的」可见工作表 + #: （P3-SPEC 6.1），认完之前算不出来。这一段只报 sheet_missing，挪到循环之后问题的先后顺序不变 + missing: list[tuple[SheetRecipe, str]] = [] for si, sr in enumerate(self.recipe.sheets): key = match_key(sr.match.name) hit = next((s for s in self.scan.sheets if match_key(s.name) == key), None) renamed = False if hit is not None and hit.state != "visible": - msg = f"工作表「{hit.name}」是隐藏的工作表，不读取（配方中的工作表「{sr.match.name}」）" - self.problems.add("sheet_missing", msg, msg) + missing.append((sr, f"工作表「{hit.name}」是隐藏的工作表，不读取（配方中的工作表「{sr.match.name}」）")) continue if hit is None and sr.match.fallback == "only_visible_sheet" and len(visible) == 1: hit, renamed = visible[0], True if hit is None or hit.name in used: names = "、".join(f"「{s.name}」" for s in visible[:8]) or "无" - msg = f"没有找到工作表「{sr.match.name}」（工作簿中有内容的可见工作表：{names}）" - self.problems.add("sheet_missing", msg, msg) + missing.append((sr, f"没有找到工作表「{sr.match.name}」（工作簿中有内容的可见工作表：{names}）")) continue used.add(hit.name) self.ex.sheets.matched[sr.id] = hit.name if renamed: self.ex.sheets.renamed[sr.match.name] = hit.name self.sheets.append(_Sheet(self, si, sr, hit)) + free = [s.name for s in visible if s.name not in used] + for sr, msg in missing: + self.problems.add("sheet_missing", msg, msg, + fix_args={"sheet_id": sr.id, "name": sr.match.name, "candidates": list(free)}) for s in visible: if s.name in used: continue @@ -2366,8 +2756,9 @@ class _Engine: return [] if big and not self.partial and b is not None: sh.recon = _Reconciler(s) + bid = lists[0].id run = _ListRun(self, sh, lists[0], 0, lambda r, cells, claims, data: self.finish_row( - sh, r, cells, claims, data)) + sh, r, cells, claims, data, bid)) for r, cells in xlsx_cells.sheet_rows(self.books, s, min_row=b.min_row, max_row=b.max_row, min_col=b.min_col, max_col=b.max_col): run.feed(r, cells) @@ -2406,8 +2797,14 @@ class _Engine: p.add("segment_overlap", msg, msg, sheet=sh.name) for r in grid.row_numbers(): cells = grid.row(r) - claims = {c: sh.claims[(r, c)] for c, _ in cells if (r, c) in sh.claims} - self.finish_row(sh, r, cells, claims, r in sh.data_rows) + claims: dict[int, str] = {} + blocks: dict[int, str] = {} + for c, _ in cells: + role = sh.claims.get((r, c)) + if role is not None: + claims[c] = role + blocks[c] = sh.cell_block[(r, c)] + self.finish_row(sh, r, cells, claims, r in sh.data_rows, blocks) self._recon_problem(sh) return cts @@ -2436,16 +2833,43 @@ class _Engine: if kind == "number": self._outside_number(sh, r, c, text) else: + sh.outside_texts.setdefault(r, []).append((c, text)) self._outside_text(sh, r, c, text, kind) + if sh.held: + self._ignore_outside(sh) for st in sh.stopped: + where = f"工作表「{sh.name}」的列表「{st.block}」" + if st.note_row is not None and not st.note_like: + # 跳过空行的列表按「表下说明」收尾，可那格文字不像说明：也可能是表尾一条只填了首列的数据行。照样不导入 + # （与起草器一致），但要人确认，首次导入和每期重放都要勾（confirm 类，确认项 rows_after_stop:<块>） + ref = (st.note_cell or "").split("!")[-1] + msg = (f"{where}：第 {st.note_row} 行只有首列文字（{ref}「{_shown(st.note_text)}」），" + "已当作表下说明，没有导入。如果它是一条数据，请在原表中补上这一行其他列的值后重新上传") + model = f"{where}：第 {st.note_row} 行只有首列文字（{ref}），已当作表下说明，没有导入" + self.problems.add("rows_after_stop", msg, model, [st.note_cell or ""], sheet=sh.name) if st.rows: - where = f"工作表「{sh.name}」的列表「{st.block}」" - msg = (f"{where}：第 {st.rows[0]} 行起有 {len(st.rows)} 行文字在空行之后，没有导入" - f"（{_join([c.split('!')[-1] for c in st.cells], 5)}）。" - "如果它们也是数据，请在配方面板的「数据中间的空行」中选择「跳过继续」") - model = (f"{where}：第 {st.rows[0]} 行起有 {len(st.rows)} 行文字在空行之后，没有导入" - f"（{_join([c.split('!')[-1] for c in st.cells], 5)}）") + shown = _join([c.split('!')[-1] for c in st.cells], 5) + if st.note_row is None: + msg = (f"{where}：第 {st.rows[0]} 行起有 {len(st.rows)} 行文字在空行之后，没有导入（{shown}）。" + "如果它们也是数据，请在配方面板的「数据中间的空行」中选择「跳过继续」") + model = f"{where}：第 {st.rows[0]} 行起有 {len(st.rows)} 行文字在空行之后，没有导入（{shown}）" + else: + # 已经是「跳过继续」：叫人去选它没有用。列表停在表下说明那一行，说明下面的行要导入只能挪走说明 + msg = (f"{where}：第 {st.rows[0]} 行起有 {len(st.rows)} 行文字在表下说明（第 {st.note_row} 行）" + f"之后，没有导入（{shown}）。如果它们也是数据，请把原表第 {st.note_row} 行的说明移到表的" + "最下方后重新上传") + model = (f"{where}：第 {st.rows[0]} 行起有 {len(st.rows)} 行文字在表下说明（第 {st.note_row} 行）" + f"之后，没有导入（{shown}）") self.problems.add("rows_after_stop", msg, model, st.cells, sheet=sh.name) + # 排除的行（after_stop）：格数是这些行在本块列范围内的非空格（它们在格子账里照常算区域外） + for i, r in enumerate(st.rows): + sh.exclude("after_stop", r, r, st.n if i == 0 else 0, block=st.block, anchor=st.note_text) + if st.note_row is not None: + # 按表下说明收尾：说明那一行（本块列范围内只有首列一格）和其下交给区域外的行都记进排除的行，锚点是说明 + # 那格的文字，回执里看得出列表停在哪一行、因为什么。像说明的只记在这里和区域外文字里，不出问题 + sh.exclude("after_stop", st.note_row, st.note_row, 1, block=st.block, anchor=st.note_text) + for i, r in enumerate(st.tail): + sh.exclude("after_stop", r, r, st.tail_n if i == 0 else 0, block=st.block, anchor=st.note_text) rows, cols = sorted(sh.hidden_claimed_rows), sorted(sh.hidden_claimed_cols) pol = sh.sr.hidden if rows or cols: @@ -2455,16 +2879,22 @@ class _Engine: hint = "（工作表设置了筛选，可能是筛选隐藏的行）" if sh.scan.autofilter else "" msg = (f"工作表「{sh.name}」导入区域内有隐藏的行：第 {_rows_text(rows)} 行{hint}。" "请在 Excel 中取消隐藏，或在配方面板的「隐藏行」中选择「照常导入」或「跳过」") - self.problems.add("hidden_rows", msg, msg, sheet=sh.name) + self.problems.add("hidden_rows", msg, msg, sheet=sh.name, + fix_args={"sheet": sh.name, "sheet_id": sh.sr.id, "axis": "rows", "rows": rows, + "autofilter": bool(sh.scan.autofilter)}) if cols and pol.cols == "reject_if_any": - msg = (f"工作表「{sh.name}」导入区域内有隐藏的列：{_join([col_letter(c) for c in cols])} 列。" + letters = [col_letter(c) for c in cols] + msg = (f"工作表「{sh.name}」导入区域内有隐藏的列：{_join(letters)} 列。" "请在 Excel 中取消隐藏，或在配方面板的「隐藏列」中选择「照常导入」") - self.problems.add("hidden_cols", msg, msg, sheet=sh.name) + self.problems.add("hidden_cols", msg, msg, sheet=sh.name, + fix_args={"sheet": sh.name, "sheet_id": sh.sr.id, "axis": "cols", "cols": letters, + "autofilter": bool(sh.scan.autofilter)}) roles = dict(sh.roles) self.ex.ledger.append(LedgerSheet(sh.name, sh.scan.nonempty, sh.read_count, roles, max(sh.read_count - sum(roles.values()), 0))) self.ex.regions.extend(sh.regions.marks()) self.ex.block_order[sh.name] = [key for _, key in sorted(sh.order)] + self.ex.rows_excluded.extend(sh.excluded_rows()) def _outputs(self) -> None: ex = self.ex diff --git a/backend/app/data/recipe_fixes.py b/backend/app/data/recipe_fixes.py new file mode 100644 index 00000000..54f6be61 --- /dev/null +++ b/backend/app/data/recipe_fixes.py @@ -0,0 +1,1103 @@ +"""修复按钮（期 3，WP-2，P3-SPEC 第 3 节）：从问题算出封闭的修复提议，再把选中的选项换算成配方补丁。 + +**客户端只发 fix_id、选项值和理由，不发补丁**（3.1）。补丁里的文字只有三种来源： +- 文件里的原文：标签、表头、分段标题、锚点文字，都来自执行器填的 Problem.fix_args（执行器从格子里取）； +- 系统按固定规则推出的名字：起草器第 8 条的列名（recipe_suggest._measure_column）、{基表}_表内合计、{前一段 id}合计； +- 用户写的理由：只用作说明，不参与匹配。 +没有正则，没有用户输入的匹配常量。所以 fix_ops 只认 proposal.options 里有的选项值：带文件原文的选项值 +（「<候选格>|pick:<词>」、新工作表名）都是服务端算出来放进去的，客户端改不了内容。 + +**入参 recipe 一律是完整形式**（Recipe.model_validate(x).model_dump(mode="json")，与 answers_base 相同，评审二-M1）： +暂存区存的是 canonical 形式，等于默认值的字段被去掉了，在它上面写补丁，replace /sheets/{si}/hidden/rows 这类路径会 +不存在。WP-5 调用前先转成完整形式。补丁只用 recipe.apply_patch 支持的 add / replace / remove，路径按 id 在完整 +形式里找下标。 + +提议不存库：staging_out 每次按暂存区里存的问题现算。同样的问题得到同样的提议 id（"fx-" + sha1(kind|目标键)[:12]）， +界面按问题上的 fix_ids 放按钮（FixAnchor，评审三-B2）。 +""" +from __future__ import annotations + +import dataclasses +import hashlib +import re +from typing import Any, Callable + +from pydantic import ValidationError + +from app.data.names import canon, collide_key +from app.data.recipe import apply_patch, recipe_sha256 +from app.data.recipe_parsers import ( + candidate_words, + looks_numeric_text, + match_key, + month_day_or_date, +) +from app.data.recipe_suggest import _measure_column +from app.data.recipe_types import ( + DraftFacts, + EditResult, + Fact, + FixAnchor, + FixOption, + FixProposal, + Problem, + Recipe, + derive_tables, +) + +#: 理由的长度（契约 IgnoreRow.reason、Dismissed.reason 都是 1–200 字） +REASON_MAX = 200 +#: 交叉表的分段上限（契约 CrosstabBlock.segments max_length） +_SEGMENTS_MAX = 16 +#: 分段标题、工作表名、锚点、表头的长度上限（契约 Locate.title、SheetMatch.name、IgnoreRow.label / IgnoreOutside.anchor、 +#: IgnoreColumn.header）。超长的原文写不进配方，不出这个选项 +_TITLE_MAX, _SHEET_MAX, _ANCHOR_MAX, _HEADER_MAX = 40, 31, 40, 80 +#: 占位符：最长 8 字、每块最多 8 个（契约 Placeholder.text、CrossValues.placeholders） +_PLACEHOLDER_MAX, _PLACEHOLDERS_MAX = 8, 8 +#: rename_title 的 pick 类选项最多给几个候选词 +_PICK_OPTIONS_MAX = 3 + + +class EditRequestError(ValueError): + """修改请求本身不对（不是「泛化不了」）。code 是接口层要回的机读码（P3-SPEC 9.1）： + + - edit_invalid：选项不在提议里、理由超长、框选的选项写错（422）； + - reason_required：需要理由的选项没给理由（422；界面要先填理由再预览，10.2）。 + 是 ValueError 的子类：接口层 ``except ValueError`` 时 code 缺省按 edit_invalid。消息可以直接给人看。""" + + def __init__(self, code: str, message: str): + self.code = code + super().__init__(message) + + +class _Stale(Exception): + """提议的目标在当前工作配方里已经找不到了（分段、关系、工作表被改掉）：换算成 ok=False 的 EditResult。""" + + +# ========================================================================== +# 小工具（recipe_select 也用） +# ========================================================================== + + +def esc(token: Any) -> str: + """JSON Pointer 的一段（RFC 6901：~ 写成 ~0、/ 写成 ~1）。""" + return str(token).replace("~", "~0").replace("/", "~1") + + +def ptr(*parts: Any) -> str: + return "".join("/" + esc(p) for p in parts) + + +def quoted(items: list[str] | tuple[str, ...]) -> str: + return "、".join(f"「{x}」" for x in items) + + +def has_digit(text: str) -> bool: + """与静态校验的 period_literal 同一判断：含任何数字（含全角数字）。""" + return any(ch.isdigit() for ch in text) + + +def fix_id(kind: str, key: str) -> str: + return "fx-" + hashlib.sha1(f"{kind}|{key}".encode("utf-8")).hexdigest()[:12] + + +def iter_segments(recipe: dict[str, Any]): + """完整形式的配方里每个交叉表分段：(si, bi, gi, 块, 分段)。""" + for si, sheet in enumerate(recipe.get("sheets") or []): + for bi, block in enumerate(sheet.get("blocks") or []): + if block.get("layout") != "crosstab": + continue + for gi, seg in enumerate(block.get("segments") or []): + yield si, bi, gi, block, seg + + +def find_segment(recipe: dict[str, Any], seg_id: Any) -> tuple[int, int, int, dict, dict] | None: + return next(((si, bi, gi, b, s) for si, bi, gi, b, s in iter_segments(recipe) if s.get("id") == seg_id), None) + + +def find_block(recipe: dict[str, Any], block_id: Any) -> tuple[int, int, dict] | None: + for si, sheet in enumerate(recipe.get("sheets") or []): + for bi, block in enumerate(sheet.get("blocks") or []): + if block.get("id") == block_id: + return si, bi, block + return None + + +def sheet_index(recipe: dict[str, Any], actual: Any, renamed: dict[str, str] | None = None) -> int | None: + """实际工作表名 → 配方里的工作表下标：按 match_key 同名；按 fallback 认的（renamed：配方名 → 实际名）也算； + 配方只有一个工作表时就是它（执行器的 only_visible_sheet 也只在那时生效）。""" + sheets = recipe.get("sheets") or [] + if not isinstance(actual, str): + return None + key = match_key(actual) + for si, sheet in enumerate(sheets): + if match_key((sheet.get("match") or {}).get("name")) == key: + return si + for old, new in (renamed or {}).items(): + if match_key(new) == key: + for si, sheet in enumerate(sheets): + if match_key((sheet.get("match") or {}).get("name")) == match_key(old): + return si + return 0 if len(sheets) == 1 else None + + +def table_index(recipe: dict[str, Any], name: Any) -> int | None: + return next((ti for ti, t in enumerate(recipe.get("tables") or []) if t.get("name") == name), None) + + +def unique_name(name: str, used: set[str], *, limit: int = 48) -> str: + """名字撞了（collide_key）就加 _2、_3……（与起草器同一规则）。used 原地更新。""" + base = name[:limit] + out, n = base, 2 + while collide_key(out) in used: + tail = f"_{n}" + out = base[: limit - len(tail)] + tail + n += 1 + used.add(collide_key(out)) + return out + + +def unique_id(want: str, used: set[str], *, limit: int = 32) -> str: + """分段、块的 id：整份配方内唯一（逐字比较，同静态校验 segment_id_duplicate / block_id_duplicate）。""" + base = want[:limit] + out, n = base, 2 + while out in used: + tail = f"_{n}" + out = base[: limit - len(tail)] + tail + n += 1 + used.add(out) + return out + + +def all_ids(recipe: dict[str, Any]) -> set[str]: + """配方里已用的分段 id 和块 id（两者同在 block_order 里，也不能彼此重名）。""" + out: set[str] = set() + for sheet in recipe.get("sheets") or []: + for block in sheet.get("blocks") or []: + out.add(str(block.get("id"))) + for seg in block.get("segments") or []: + out.add(str(seg.get("id"))) + return out + + +def finish(recipe: dict[str, Any] | None, ops: list[dict[str, Any]], *, kind: str, key: str, title: str, + summary: list[str], anchors: list | None = None, notes: list[str] | None = None, + expected: dict[str, str | None] | None = None, block: str | None = None, + breaking: bool = False) -> EditResult: + """应用补丁、按 schema 解析、算哈希（与暂存区的哈希同一口径：apply_patch → 解析 → recipe_sha256）。 + + 补丁应用不上或解析不过时 ok=False：正常流程不会出现（补丁按当前工作配方写），出现说明提议已经过期， + 界面照原因提示重新打开问题。语义问题（静态校验的其余规则）不在这里判：预览时 WP-5 照常静态校验、干跑。""" + base = recipe if isinstance(recipe, dict) else {} + try: + new = apply_patch(base, ops) + sha = recipe_sha256(Recipe.model_validate(new)) + except (ValueError, ValidationError): + msg = "这条修改已无法应用到当前的配方上：配方在提出修改之后又被改过。请重新查看问题和修复建议" + return EditResult(ok=False, kind=kind, key=key, title=title, summary=[], ops=[], anchors=[], # type: ignore[arg-type] + notes=list(notes or []), problems=[Problem("edit_not_applicable", "recipe", msg, + model_message="edit_not_applicable")]) + return EditResult(ok=True, kind=kind, key=key, title=title, summary=summary, ops=ops, # type: ignore[arg-type] + anchors=list(anchors or []), notes=list(notes or []), problems=[], recipe_sha256_after=sha, + expected=expected, block=block, breaking=breaking) + + +def _get(obj: Any, key: str) -> Any: + if isinstance(obj, dict): + return obj.get(key) + return getattr(obj, key, None) + + +def _facts(facts: dict[str, Any] | DraftFacts | None) -> list[Fact]: + if facts is None: + return [] + if isinstance(facts, DraftFacts): + return list(facts.facts) + out: list[Fact] = [] + for f in (facts.get("facts") or []) if isinstance(facts, dict) else []: + if isinstance(f, Fact): + out.append(f) + elif isinstance(f, dict) and f.get("id") and f.get("kind"): + out.append(Fact(f["id"], f["kind"], f.get("sheet") or "", f.get("text") or "", dict(f.get("detail") or {}))) + return out + + +def _proposal(kind: str, key: str, *, code: str | None, title: str, cells: list[str], target: dict[str, Any], + options: list[FixOption], anchor: FixAnchor) -> FixProposal: + return FixProposal(id=fix_id(kind, key), kind=kind, problem_code=code, title=title, + cells=[c for c in cells if isinstance(c, str)][:40], target=target, options=options, + anchor=anchor) + + +def _key(kind: str, target: dict[str, Any]) -> str: + """目标键（3.2）：确认项 fix:<目标键>、提议 id 都由它来。只取目标里的文件原文和配方里的 id。""" + t = target + if kind == "remove_label": + return f"remove_label:{t['segment']}:{'|'.join(t['labels'])}" + if kind == "add_label": + return f"add_label:{t['segment']}:{'|'.join(x['raw'] for x in t['labels'])}" + if kind == "edit_members": + return f"edit_members:{t['relation']}" + if kind == "rename_title": + return f"rename_title:{t['segment']}" + if kind == "declare_total": + return f"declare_total:{t['segment']}" + if kind == "ignore_cells": + how = t["how"] + texts = t.get("labels") or t.get("headers") or [r["anchor"] for r in t.get("rows") or []] + where = t.get("block") if how != "outside" else t.get("sheet") + return f"ignore_cells:{how}:{where}:{'|'.join(texts)}" + if kind == "declare_placeholder": + return f"declare_placeholder:{t['block']}:{'|'.join(t['texts'])}" + if kind == "declare_hidden": + return f"declare_hidden:{t['sheet']}:{t['axis']}" + if kind == "rename_sheet": + return f"rename_sheet:{t['sheet_id']}" + raise KeyError(kind) + + +# ========================================================================== +# ① 移除标签 +# ========================================================================== + + +def _p_remove_label(recipe: dict, prob: Any, args: dict, anchor: FixAnchor) -> FixProposal | None: + found = find_segment(recipe, args.get("segment")) + labels = [x for x in args.get("labels") or [] if isinstance(x, str)] + if found is None or not labels: + return None + _si, _bi, _gi, _block, seg = found + expect = list((seg.get("labels") or {}).get("expect") or []) + keys = {match_key(x) for x in labels} + removed = [e for e in expect if e in labels or match_key(e) in keys] + if not removed or len(removed) == len(expect): + # 剩下的 expect 为空：分段不能为空，请在配方面板里删掉整个分段 + return None + measures = seg.get("role") == "measures" + cols = [seg.get("measures", {}).get(e) for e in removed if seg.get("measures", {}).get(e)] + if measures: + detail = (f"表「{seg.get('table')}」不再导入列{quoted(cols)}，这是破坏性变更（按期累积时按退役处理，" + f"早期各期保留原值）；今后各期如果又出现{quoted(removed)}，会再次拒收") + else: + detail = f"今后各期如果又出现{quoted(removed)}，会再次拒收" + target = {"segment": seg["id"], "labels": removed} + return _proposal("remove_label", _key("remove_label", target), code=_get(prob, "code"), + title=f"在分段「{seg['id']}」的期望标签中去掉{quoted(removed)}", + cells=list(_get(prob, "cells") or []), target=target, + options=[FixOption("remove", f"去掉标签{quoted(removed)}", detail=detail, breaking=measures)], + anchor=anchor) + + +def _o_remove_label(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]: + found = find_segment(recipe, p.target.get("segment")) + if found is None: + raise _Stale + si, bi, gi, _block, seg = found + sp = ("sheets", si, "blocks", bi, "segments", gi) + expect = list(seg["labels"]["expect"]) + removed = [e for e in expect if e in p.target["labels"]] + remaining = [e for e in expect if e not in removed] + if not removed or not remaining: + raise _Stale + ops: list[dict[str, Any]] = [{"op": "replace", "path": ptr(*sp, "labels", "expect"), "value": remaining}] + summary = [f"分段「{seg['id']}」的期望标签去掉{quoted(removed)}"] + if seg.get("role") == "measures": + ti = table_index(recipe, seg.get("table")) + cols = [] + for e in removed: + col = seg["measures"].get(e) + if col is None: + continue + cols.append(col) + ops.append({"op": "remove", "path": ptr(*sp, "measures", e)}) + if ti is not None and col in (recipe["tables"][ti].get("units") or {}): + ops.append({"op": "remove", "path": ptr("tables", ti, "units", col)}) + if cols: + summary.append(f"表「{seg.get('table')}」不再导入列{quoted(cols)}") + return ops, summary + + +# ========================================================================== +# ② 加入标签 +# ========================================================================== + + +def _new_columns(recipe: dict, seg: dict, raws: list[str]) -> list[tuple[str, str, str | None, str | None]]: + """measures 段新加的标签 → [(原文, 列名, 词表内单位, 括号原文)]。列名和单位照起草器第 8 条推 + （recipe_suggest._measure_column，框选的分段也用它）：括号里的单位在词表里时列名去掉单位、记单位；不在词表里 + 时列名保留原文（「分区丙（千人次）」→「分区丙_千人次」）、不记单位。只按 3.2 ② 的字面去掉括号的话，兄弟列 + 都是「人次」时新列看起来同名同口径，却没有单位，确认项也不提示（unit:* 只对有单位的列出）。 + 与这张表已有的列 collide_key 撞名时加 _2。位置号与起草器相同（第几个标签 + 1，轴列是第 1 列）。""" + try: + cols, _ = derive_tables(Recipe.model_validate(recipe)) + except ValidationError: + cols = {} + used = {collide_key(c.name) for c in cols.get(seg.get("table"), []) if c.name} + start = len((seg.get("labels") or {}).get("expect") or []) + out = [] + for k, raw in enumerate(raws): + name, unit, paren = _measure_column(raw, start + k + 2) + out.append((raw, unique_name(name, used), unit, paren)) + return out + + +def _column_text(col: str, unit: str | None, paren: str | None) -> str: + """新列的人话：「新增列「分区丙」，单位「人次」」；括号里的不是词表里的单位时写明没有登记单位。""" + if unit: + return f"新增列「{col}」，单位「{unit}」" + if paren: + return f"新增列「{col}」（括号里的「{paren}」不在单位词表里，列名保留原文，没有登记单位）" + return f"新增列「{col}」" + + +def _p_add_label(recipe: dict, prob: Any, args: dict, anchor: FixAnchor) -> FixProposal | None: + found = find_segment(recipe, args.get("segment")) + if found is None: + return None + _si, _bi, _gi, _block, seg = found + role = args.get("role") + if role is not None and role != seg.get("role"): + return None + expect = (seg.get("labels") or {}).get("expect") or [] + have = {match_key(e) for e in expect} + labels: list[dict[str, Any]] = [] + for item in args.get("labels") or []: + raw = _get(item, "raw") + if isinstance(raw, str) and raw.strip() and match_key(raw) not in have: + have.add(match_key(raw)) + labels.append({"raw": raw.strip(), "cell": _get(item, "cell")}) + if not labels: + return None + raws = [x["raw"] for x in labels] + if seg.get("role") == "measures": + cols = _new_columns(recipe, seg, raws) + detail = "；".join(_column_text(col, unit, paren) for _r, col, unit, paren in cols) + detail += "。今后各期必须有这些标签，没有会拒收；按期累积时早期各期这一列为空值" + else: + detail = "今后各期必须有这些标签，没有会拒收" + target = {"segment": seg["id"], "role": seg.get("role"), "labels": labels} + return _proposal("add_label", _key("add_label", target), code=_get(prob, "code"), + title=f"在分段「{seg['id']}」的期望标签中加入{quoted(raws)}", + cells=[x["cell"] for x in labels if x.get("cell")] or list(_get(prob, "cells") or []), + target=target, options=[FixOption("add", f"加入标签{quoted(raws)}", detail=detail)], + anchor=anchor) + + +def _o_add_label(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]: + found = find_segment(recipe, p.target.get("segment")) + if found is None: + raise _Stale + si, bi, gi, _block, seg = found + sp = ("sheets", si, "blocks", bi, "segments", gi) + expect = list(seg["labels"]["expect"]) + have = {match_key(e) for e in expect} + raws = [x["raw"] for x in p.target["labels"] if match_key(x["raw"]) not in have] + if not raws: + raise _Stale + ops: list[dict[str, Any]] = [{"op": "replace", "path": ptr(*sp, "labels", "expect"), "value": expect + raws}] + summary = [f"分段「{seg['id']}」的期望标签加入{quoted(raws)}"] + if seg.get("role") == "measures": + ti = table_index(recipe, seg.get("table")) + for raw, col, unit, paren in _new_columns(recipe, seg, raws): + ops.append({"op": "add", "path": ptr(*sp, "measures", raw), "value": col}) + if unit and ti is not None: + # 满足静态校验第 13 条 unit_label_conflict：原表标签写着单位，列的单位也要写 + ops.append({"op": "add", "path": ptr("tables", ti, "units", col), "value": unit}) + summary.append(f"表「{seg.get('table')}」" + _column_text(col, unit if ti is not None else None, paren)) + return ops, summary + + +# ========================================================================== +# ③ 编辑恒等式成员 +# ========================================================================== + +_REL_PATH = re.compile(r"/relations/(\d+)(?:/.*)?") + + +def _measures_of(recipe: dict, seg_id: Any, labels: list[str]) -> dict | None: + """Fact 指向的 measures 分段：按 id 找；找不到时按标签找唯一含这些标签的分段（同静态校验 measures_seg）。""" + segs = [s for *_r, s in iter_segments(recipe) if s.get("role") == "measures"] + hit = next((s for s in segs if s.get("id") == seg_id), None) + if hit is not None: + return hit + keys = {match_key(x) for x in labels} + found = [s for s in segs if keys <= {match_key(e) for e in (s.get("labels") or {}).get("expect") or []}] + return found[0] if len(found) == 1 else None + + +def _fact_columns(recipe: dict, fact: Fact) -> tuple[str, str, list[str]] | None: + """sum_eq 事实 → (表, 合计列, 组成列)：标签原文经该 measures 段的映射换成列名，映射不到为 None。""" + d = fact.detail or {} + total, parts = d.get("total"), d.get("parts") + if fact.kind != "sum_eq" or not isinstance(total, str) or not isinstance(parts, list): + return None + seg = _measures_of(recipe, d.get("segment"), [total, *[p for p in parts if isinstance(p, str)]]) + if seg is None: + return None + by_key = {match_key(e): col for e, col in (seg.get("measures") or {}).items()} + t = by_key.get(match_key(total)) + ps = [by_key.get(match_key(x)) for x in parts if isinstance(x, str)] + if t is None or None in ps or len(ps) < 2: + return None + return seg.get("table"), t, ps # type: ignore[return-value] + + +def _total_col(recipe: dict, table: Any, fact: Fact) -> str | None: + """sum_eq 事实的合计标签在写入 table 的 measures 段里映射到的列（事实所在分段同 id 的优先）。组成标签映射 + 不全也照样算：只用来认「这条事实说的是不是这条关系的合计列」。""" + d = fact.detail or {} + total = d.get("total") + if fact.kind != "sum_eq" or not isinstance(total, str): + return None + segs = [s for *_r, s in iter_segments(recipe) if s.get("role") == "measures" and s.get("table") == table] + for s in sorted(segs, key=lambda x: x.get("id") != d.get("segment")): + col = {match_key(e): c for e, c in (s.get("measures") or {}).items()}.get(match_key(total)) + if col is not None: + return col + return None + + +def _claimable(recipe: dict, rel: dict, facts: list[Fact]) -> Fact | None: + """成员映射不全时，这条关系还能认领的本期 sum_eq 事实： + 1. 它原来认领的那条（按编号），合计标签映射到关系的合计列； + 2. 否则同一张表上合计标签映射到同一合计列的 sum_eq，恰好一条（编号按本期的发现顺序排，可能错开）。 + 已经被别的关系认领的不算：再认领一次会报 fact_claimed_twice。""" + others = {r.get("claims") for r in recipe.get("relations") or [] if r is not rel and r.get("claims")} + ok = [f for f in facts if f.id not in others and _total_col(recipe, rel.get("table"), f) == rel.get("total")] + own = next((f for f in ok if f.id == rel.get("claims")), None) + if own is not None: + return own + return ok[0] if len(ok) == 1 else None + + +def _unmapped(recipe: dict, rel: dict, fact: Fact) -> list[str]: + """事实里在写入这张表的 measures 段里映射不到列的标签（被 ① 去掉的那些）。""" + have = {match_key(e) for *_r, s in iter_segments(recipe) if s.get("role") == "measures" + and s.get("table") == rel.get("table") for e in (s.get("measures") or {})} + d = fact.detail or {} + labels = [d.get("total"), *(d.get("parts") or [])] + return [x for x in labels if isinstance(x, str) and match_key(x) not in have] + + +def _fact_text(fact: Fact) -> str: + d = fact.detail or {} + parts = [x for x in d.get("parts") or [] if isinstance(x, str)] + return f"{d.get('total')} = {' + '.join(parts)}" + + +def _p_edit_members(recipe: dict, rp: Any, facts: list[Fact], anchor: FixAnchor) -> FixProposal | None: + code = _get(rp, "code") + m = _REL_PATH.fullmatch(str(_get(rp, "path") or "")) + if code not in ("fact_claim_mismatch", "column_unknown") or m is None: + return None + rels = recipe.get("relations") or [] + ri = int(m.group(1)) + if ri >= len(rels) or rels[ri].get("kind") != "sum_eq": + return None + rel = rels[ri] + if code == "column_unknown" and not str(_get(rp, "path")).startswith(ptr("relations", ri, "parts")) \ + and _get(rp, "path") != ptr("relations", ri, "total"): + return None + by_id = {f.id: f for f in facts} + fact = by_id.get(rel.get("claims")) + cols = _fact_columns(recipe, fact) if fact is not None else None + if cols is None or cols[0] != rel.get("table"): + # 认领的那条不是能对上的 sum_eq（编号错开、或本期不成立）：找同一张表上合计列相同的那条 sum_eq，恰好一条才用 + hits = [(f, c) for f in facts if (c := _fact_columns(recipe, f)) is not None + and c[0] == rel.get("table") and c[1] == rel.get("total")] + fact, cols = hits[0] if len(hits) == 1 else (None, None) + options: list[FixOption] = [] + manual = "也可以在配方面板中手工修改成员并去掉认领，之后每期照常核对" + if cols is not None and fact is not None: + _t, total, parts = cols + if (total, sorted(parts), fact.id) != (rel.get("total"), sorted(rel.get("parts") or []), rel.get("claims")): + options.append(FixOption("update", f"按系统发现更新为「{total} = {' + '.join(parts)}」", + detail="今后每期按新的成员核对")) + options.append(FixOption("dismiss", "不再登记这条关系", needs_reason=True, + detail="今后不再核对这条关系；系统发现记为已说明不登记")) + elif (fact := _claimable(recipe, rel, facts)) is not None: + # 本期的事实还在、只是成员映射不全（3.2 ① 的连带：去掉的组成列被这条关系引用）：update 给不出， + # 「不登记」认领这条事实（3.2 ① 写明这种情形下 ③ 只给「不登记」）。退回「删除这条关系」的话，事实在 + # 改动范围之外时连未认领都不报，核对连同它的系统发现一起不声不响地消失 + gone = _unmapped(recipe, rel, fact) + why = f"本期的系统发现「{_fact_text(fact)}」含有配方不再导入的标签{quoted(gone)}，无法按它更新成员" if gone \ + else f"本期的系统发现「{_fact_text(fact)}」无法换算成这张表的列" + options.append(FixOption("dismiss", "不再登记这条关系", needs_reason=True, + detail=f"{why}；不再登记后今后不再核对这条关系，系统发现记为已说明不登记。{manual}")) + else: + # 本期没有可以认领的恒等式（D13 加了分项而新恒等式某天不成立时 _sum_holds 不出 sum_eq 事实）。 + # 不登记要认领一条系统发现，这里没有能认领的，只能删掉这条关系；删掉一条数据质量核对要写明理由 + fact = None + options.append(FixOption("remove", "删除这条关系", needs_reason=True, + detail=f"本期未发现可以替换或认领的恒等式，删除后不再核对这条关系；{manual}")) + target: dict[str, Any] = {"relation": rel.get("id"), "fact": fact.id if fact else None, + "total": cols[1] if cols else None, "parts": list(cols[2]) if cols else []} + return _proposal("edit_members", _key("edit_members", target), code=code, + title=f"关系「{rel.get('id')}」的成员与本期的系统发现对不上", cells=[], target=target, + options=options, anchor=anchor) + + +def _o_edit_members(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]: + rels = recipe.get("relations") or [] + rid = p.target.get("relation") + ri = next((i for i, r in enumerate(rels) if r.get("id") == rid), None) + if ri is None: + raise _Stale + rel = rels[ri] + if opt.value == "remove": + return [{"op": "remove", "path": ptr("relations", ri)}], [f"删除关系「{rid}」（理由：{reason}）"] + if opt.value == "dismiss": + value = {"id": rid, "kind": "dismissed", "claims": p.target["fact"], "reason": reason} + return [{"op": "replace", "path": ptr("relations", ri), "value": value}], \ + [f"关系「{rid}」不再登记（理由：{reason}）"] + if rel.get("kind") != "sum_eq": + raise _Stale + total, parts, fid = p.target["total"], list(p.target["parts"]), p.target["fact"] + ops: list[dict[str, Any]] = [{"op": "replace", "path": ptr("relations", ri, "parts"), "value": parts}] + if rel.get("total") != total: + ops.append({"op": "replace", "path": ptr("relations", ri, "total"), "value": total}) + if rel.get("claims") != fid: + ops.append({"op": "add", "path": ptr("relations", ri, "claims"), "value": fid}) + return ops, [f"关系「{rid}」的成员更新为「{total} = {' + '.join(parts)}」"] + + +# ========================================================================== +# ④ 分段标题改名 +# ========================================================================== + + +def _siblings(recipe: dict, seg: dict) -> list[str]: + """写进同一张表的其他分段的分段标题（候选词按兄弟标题去掉公共部分，同静态校验 check_const）。""" + out = [] + for *_r, s in iter_segments(recipe): + loc = s.get("locate") or {} + if (s is not seg and s.get("role") == "dimension" and s.get("table") == seg.get("table") + and loc.get("by") == "section_title" and loc.get("title")): + out.append(loc["title"]) + return out + + +def keep_ok(seg: dict, title: str) -> bool: + """改标题时常量能不能沿用：每个常量的 canon(pick) 都是 canon(新标题) 的子串（3.3 (b)，静态校验据此放行）。""" + return all(canon(c.get("pick", "")) in canon(title) for c in (seg.get("const") or {}).values()) + + +def title_options(recipe: dict, seg: dict, cell: str, text: str) -> list[FixOption]: + """一个候选标题的选项：沿用常量（keep_ok 时）和改用候选词（只有一个常量列时，最多 _PICK_OPTIONS_MAX 个）。""" + consts = seg.get("const") or {} + opts: list[FixOption] = [] + after = "今后各期只认新标题；改回旧标题时会再次拒收" + if keep_ok(seg, text): + kept = "，" + "、".join(f"「{k}」沿用「{v.get('pick')}」" for k, v in consts.items()) if consts else "" + opts.append(FixOption(f"{cell}|keep", f"改为「{text}」{kept}", detail=after)) + if len(consts) == 1: + col, c = next(iter(consts.items())) + words = [w for w in candidate_words(text, _siblings(recipe, seg)) if w != c.get("pick")][:_PICK_OPTIONS_MAX] + for w in words: + opts.append(FixOption(f"{cell}|pick:{w}", f"改为「{text}」，「{col}」改用「{w}」", + detail=f"这是破坏性变更：这一列的值从「{c.get('pick')}」变为「{w}」。{after}", + breaking=True)) + return opts + + +def _p_rename_title(recipe: dict, prob: Any, args: dict, anchor: FixAnchor) -> FixProposal | None: + found = find_segment(recipe, args.get("segment")) + if found is None: + return None + _si, _bi, _gi, _block, seg = found + loc = seg.get("locate") or {} + if loc.get("by") != "section_title" or seg.get("role") == "derived": + return None + cands: list[dict[str, str]] = [] + options: list[FixOption] = [] + for item in args.get("candidates") or []: + cell, text = _get(item, "cell"), _get(item, "text") + if not isinstance(cell, str) or not isinstance(text, str) or not text.strip() or len(text) > _TITLE_MAX: + continue + opts = title_options(recipe, seg, cell, text.strip()) + if opts: + cands.append({"cell": cell, "text": text.strip()}) + options += opts + if not options: + return None + target = {"segment": seg["id"], "title": loc.get("title"), "candidates": cands} + return _proposal("rename_title", _key("rename_title", target), code=_get(prob, "code"), + title=f"分段「{seg['id']}」的分段标题「{loc.get('title')}」没有找到，可以改用文件里的新标题", + cells=[c["cell"] for c in cands], target=target, options=options, anchor=anchor) + + +def _o_rename_title(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]: + found = find_segment(recipe, p.target.get("segment")) + if found is None: + raise _Stale + si, bi, gi, _block, seg = found + sp = ("sheets", si, "blocks", bi, "segments", gi) + cell, _, how = opt.value.rpartition("|") + cand = next((c for c in p.target.get("candidates") or [] if c["cell"] == cell), None) + if cand is None: + raise _Stale + text = cand["text"] + ops: list[dict[str, Any]] = [{"op": "replace", "path": ptr(*sp, "locate", "title"), "value": text}] + summary = [f"分段「{seg['id']}」的分段标题改为「{text}」"] + if how.startswith("pick:"): + consts = seg.get("const") or {} + if len(consts) != 1: + raise _Stale + col = next(iter(consts)) + word = how[len("pick:"):] + ops.append({"op": "replace", "path": ptr(*sp, "const", col, "pick"), "value": word}) + summary.append(f"「{col}」的值改为「{word}」") + elif seg.get("const"): + summary.append("、".join(f"「{k}」沿用「{v.get('pick')}」" for k, v in seg["const"].items())) + return ops, summary + + +# ========================================================================== +# ⑤ 声明为合计行（框选的 derived 也用） +# ========================================================================== + + +def total_blockers(recipe: dict, si: int, bi: int, gi: int) -> str | None: + """分段能不能接一个合计分段：不能时返回给人看的原因（3.2 ⑤ 的五个前提里与配方有关的三条）。""" + block = recipe["sheets"][si]["blocks"][bi] + seg = block["segments"][gi] + if seg.get("role") != "dimension": + return f"分段「{seg.get('id')}」不是按维度展开的分段，合计行无法按时段区间核对" + if sorted((seg.get("dim") or {}).get("derive", {}).values()) != ["end", "start"]: + return f"分段「{seg.get('id')}」没有派生起始小时和结束小时，合计行无法按时段区间核对" + if any(s.get("role") == "derived" and (s.get("locate") or {}).get("segment") == seg.get("id") + for s in block["segments"]): + return f"分段「{seg.get('id')}」后面已经有合计分段" + if len(block["segments"]) >= _SEGMENTS_MAX: + return f"交叉表「{block.get('id')}」的分段已经到上限（{_SEGMENTS_MAX} 个）" + return None + + +def total_ops(recipe: dict, si: int, bi: int, gi: int, labels: list[str], keep: bool) -> tuple[list, list[str], str | None]: + """在分段后面插入合计分段（derived）：(补丁, 摘要, 另存的表名或 None)。 + + 分段 id = 前一段 id +「合计」，撞了加 _2；另存时同一块里已有 derived 段另存到同一张基表的合计表的，沿用那张表的 + table、dim、derive、value（D18 的日间合计和原来的夜间合计存进同一张「时段客流_表内合计」）；否则新建 + {基表}_表内合计，并在表清单里加上（grain = 轴列 + 合计项，单位取基表值列的单位）。""" + block = recipe["sheets"][si]["blocks"][bi] + seg = block["segments"][gi] + bp = ("sheets", si, "blocks", bi) + sp = bp + ("segments", gi) + new_id = unique_id(f"{seg['id']}合计", all_ids(recipe)) + keep_as = None + table_ops: list[dict[str, Any]] = [] + kept: str | None = None + if keep: + same = next((s["keep_as"] for s in block["segments"] if s.get("role") == "derived" and s.get("keep_as") + and (s.get("verify") or {}).get("against_table") == seg.get("table")), None) + if same is not None: + keep_as = {k: same[k] for k in ("table", "dim", "derive", "value")} + else: + used = {collide_key(t.get("name", "")) for t in recipe.get("tables") or []} + name = unique_name(f"{seg['table']}_表内合计", used) + keep_as = {"table": name, "dim": "合计项", "derive": dict((seg.get("dim") or {}).get("derive") or {}), + "value": seg.get("value")} + ti = table_index(recipe, seg.get("table")) + unit = ((recipe["tables"][ti].get("units") or {}).get(seg.get("value")) if ti is not None else None) + table_ops.append({"op": "add", "path": "/tables/-", "value": { + "name": name, "grain": [block.get("axis", {}).get("name", "日期"), "合计项"], "kind": "reported_total", + "units": {seg.get("value"): unit} if unit else {}, "note": ""}}) + kept = keep_as["table"] + derived = {"id": new_id, "role": "derived", "locate": {"by": "after", "title": None, "segment": seg["id"]}, + "labels_parser": "hour_range_total", "labels": {"expect": list(labels)}, + "verify": {"kind": "label_range_sum", "against_table": seg.get("table"), "value": seg.get("value")}, + "keep_as": keep_as} + ops = [{"op": "add", "path": ptr(*sp, "stop_parser"), "value": "hour_range_total"}, + {"op": "add", "path": ptr(*bp, "segments", gi + 1), "value": derived}, *table_ops] + summary = [f"分段「{seg['id']}」遇到合计行就结束", + f"新增合计分段「{new_id}」：{quoted(labels)}改作合计核对" + + (f"，原值另存表「{kept}」" if kept else "，不另存")] + return ops, summary, kept + + +def _p_declare_total(recipe: dict, prob: Any, args: dict, anchor: FixAnchor) -> FixProposal | None: + found = find_segment(recipe, args.get("segment")) + if found is None or args.get("role", "dimension") != "dimension" or not args.get("at_end"): + return None + si, bi, gi, _block, seg = found + labels = list(args.get("labels") or []) + raws = [_get(x, "raw") for x in labels] + if not labels or not all(_get(x, "total") for x in labels) or not all(isinstance(r, str) and r for r in raws): + return None + if total_blockers(recipe, si, bi, gi) is not None: + return None + _ops, _s, kept = total_ops(recipe, si, bi, gi, raws, True) + target = {"segment": seg["id"], "labels": raws} + return _proposal("declare_total", _key("declare_total", target), code=_get(prob, "code"), + title=f"分段「{seg['id']}」末尾的{quoted(raws)}是合计行", + cells=[_get(x, "cell") for x in labels if _get(x, "cell")] or list(_get(prob, "cells") or []), + target=target, + options=[FixOption("keep", f"改作合计核对，原值另存表「{kept}」", + detail="今后各期必须有这一行，没有会拒收"), + FixOption("check_only", "改作合计核对，不另存", + detail="今后各期必须有这一行，没有会拒收；这一行只参与核对，记进回执的「排除的行」")], + anchor=anchor) + + +def _o_declare_total(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]: + found = find_segment(recipe, p.target.get("segment")) + if found is None: + raise _Stale + si, bi, gi, _block, _seg = found + if total_blockers(recipe, si, bi, gi) is not None: + raise _Stale + ops, summary, _kept = total_ops(recipe, si, bi, gi, list(p.target["labels"]), opt.value == "keep") + return ops, summary + + +# ========================================================================== +# ⑥ 忽略（按文字记） +# ========================================================================== + + +def _split_digits(texts: list[str], limit: int) -> tuple[list[str], list[str]]: + """(可以作锚点的, 含数字的)。超长的写不进配方，直接不要；按 match_key 去重。""" + clean: list[str] = [] + digits: list[str] = [] + seen: set[str] = set() + for t in texts: + t = t.strip() + k = match_key(t) + if not t or k in seen or len(t) > limit: + continue + seen.add(k) + (digits if has_digit(t) else clean).append(t) + return clean, digits + + +def _ignore_proposal(target: dict, *, code: Any, cells: list[str], anchor: FixAnchor, label: str, + clean: list[str], digits: list[str], extra: str = "") -> FixProposal | None: + if not clean: + # 锚点都含数字：不出提议（3.2 ⑥「锚点含数字时不出提议」）。出一个不带选项的提议的话，staging_out 按 anchor + # 给问题写上 fix_ids，界面照 10.2 放修复按钮，点开却没有可选的处理方式。部分含数字时只给其余的锚点， + # 含数字的那几个写进选项的 detail（见下）；框选忽略同样拦（selection_anchor_has_digits） + return None + detail = "今后各期按这段文字忽略；某一期没有出现时什么也不忽略，也不报错" + if digits: + detail += f"。{quoted(digits)}含数字，下一期很可能对不上，不按文字忽略：请修改文件，或把它框选为新的分段" + if extra: + detail += f"。{extra}" + return _proposal("ignore_cells", _key("ignore_cells", target), code=code, title=label, cells=cells, + target=target, options=[FixOption("ignore", label, detail=detail, needs_reason=True)], + anchor=anchor) + + +def _p_ignore_cells(recipe: dict, prob: Any, args: dict, anchor: FixAnchor) -> FixProposal | None: + how = args.get("how") + code = _get(prob, "code") + cells = list(_get(prob, "cells") or []) + if how == "rows": + found = find_block(recipe, args.get("block")) + if found is None or found[2].get("layout") != "crosstab": + return None + have = {match_key(x.get("label")) for x in found[2].get("ignore_rows") or []} + raws = [r for x in args.get("labels") or [] if isinstance(r := _get(x, "raw"), str) + and match_key(r) not in have] + clean, digits = _split_digits(raws, _ANCHOR_MAX) + if not clean and not digits: + return None # 标签列为空：那是 row_without_label，没有锚点 + target = {"block": found[2]["id"], "how": "rows", "labels": clean or digits} + return _ignore_proposal(target, code=code, cells=cells, anchor=anchor, clean=clean, digits=digits, + label=f"按行标签忽略{quoted(clean)}" + ("这几行" if len(clean) > 1 else "这一行"), + extra="被忽略的行记进回执的「排除的行」") + if how == "columns": + found = find_block(recipe, args.get("block")) + if found is None: + return None + heads = [_get(x, "raw") for x in args.get("headers") or []] + if not heads or any(not isinstance(h, str) or not h.strip() for h in heads): + return None # 有表头不是文字 + if any(month_day_or_date(h) is not None for h in heads): + return None # 能解析成日期：和日期轴冲突 + have = {match_key(x.get("header")) for x in found[2].get("ignore_columns") or []} + clean, digits = _split_digits([h for h in heads if match_key(h) not in have], _HEADER_MAX) + if not clean and not digits: + return None + target = {"block": found[2]["id"], "how": "columns", "headers": clean or digits} + return _ignore_proposal(target, code=code, cells=cells, anchor=anchor, clean=clean, digits=digits, + label=f"按表头忽略{quoted(clean)}" + ("这几列" if len(clean) > 1 else "这一列")) + if how == "outside": + si = sheet_index(recipe, args.get("sheet")) + if si is None: + return None + have = {match_key(x.get("anchor")) for x in recipe["sheets"][si].get("ignore_outside") or []} + rows = [r for r in args.get("rows") or [] if isinstance(r, dict)] + texts = [a for r in rows if isinstance(a := r.get("anchor"), str) and match_key(a) not in have] + clean, digits = _split_digits(texts, _ANCHOR_MAX) + bare = [r.get("row") for r in rows if not isinstance(r.get("anchor"), str) or not r.get("anchor")] + if not clean and not digits: + return None # 这几格没有可以作锚点的文字，只能修改文件 + lines = {match_key(r["anchor"]): r.get("row") for r in rows if isinstance(r.get("anchor"), str)} + where = "；".join(f"第 {lines.get(match_key(a))} 行有文字「{a}」" for a in clean) + target = {"sheet": args.get("sheet"), "how": "outside", + "rows": [{"row": lines.get(match_key(a)), "anchor": a} for a in (clean or digits)]} + extra = ("另有" + "、".join(f"第 {r} 行" for r in bare if r) + "没有可以作锚点的文字，只能修改文件") if bare else "" + return _ignore_proposal(target, code=code, cells=cells, anchor=anchor, clean=clean, digits=digits, + label=f"{where}：忽略{'这些行' if len(clean) > 1 else '这一行'}导入区域之外的数字", + extra=extra) + return None + + +def _o_ignore_cells(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]: + t = p.target + if t["how"] == "outside": + si = sheet_index(recipe, t.get("sheet")) + if si is None: + raise _Stale + anchors = [r["anchor"] for r in t["rows"]] + ops = [{"op": "add", "path": ptr("sheets", si, "ignore_outside", "-"), "value": {"anchor": a, "reason": reason}} + for a in anchors] + return ops, [f"工作表「{t.get('sheet')}」：同一行有{quoted(anchors)}时，忽略这一行导入区域之外的数字（理由：{reason}）"] + found = find_block(recipe, t.get("block")) + if found is None: + raise _Stale + si, bi, block = found + if t["how"] == "rows": + if block.get("layout") != "crosstab": + raise _Stale + ops = [{"op": "add", "path": ptr("sheets", si, "blocks", bi, "ignore_rows", "-"), + "value": {"label": x, "reason": reason}} for x in t["labels"]] + return ops, [f"交叉表「{block['id']}」按行标签忽略{quoted(t['labels'])}（理由：{reason}）"] + ops = [{"op": "add", "path": ptr("sheets", si, "blocks", bi, "ignore_columns", "-"), + "value": {"header": h, "reason": reason}} for h in t["headers"]] + return ops, [f"块「{block['id']}」按表头忽略{quoted(t['headers'])}（理由：{reason}）"] + + +# ========================================================================== +# ⑦ 声明占位符 +# ========================================================================== + + +def _p_declare_placeholder(recipe: dict, prob: Any, args: dict, anchor: FixAnchor) -> FixProposal | None: + found = find_block(recipe, args.get("block")) + if found is None: + return None + block = found[2] + have = {canon(x.get("text", "")) for x in (block.get("values") or {}).get("placeholders") or []} + room = _PLACEHOLDERS_MAX - len(have) + texts: list[str] = [] + for t in args.get("texts") or []: + if (isinstance(t, str) and t.strip() and len(t.strip()) <= _PLACEHOLDER_MAX + and not looks_numeric_text(t) and canon(t) not in have and t.strip() not in texts): + texts.append(t.strip()) + texts = texts[:max(room, 0)] + if not texts: + return None + target = {"block": block["id"], "texts": texts} + after = "今后各期这种写法都存为空值" + return _proposal("declare_placeholder", _key("declare_placeholder", target), code=_get(prob, "code"), + title=f"块「{block['id']}」的数据格里有{quoted(texts)}，不是数字", + cells=list(_get(prob, "cells") or []), target=target, + options=[FixOption("no_data", f"把{quoted(texts)}也声明为无数据（存为空值）", detail=after), + FixOption("not_applicable", f"把{quoted(texts)}声明为不适用（存为空值）", detail=after)], + anchor=anchor) + + +def _o_declare_placeholder(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]: + found = find_block(recipe, p.target.get("block")) + if found is None: + raise _Stale + si, bi, block = found + meaning = "无数据" if opt.value == "no_data" else "不适用" + ops = [{"op": "add", "path": ptr("sheets", si, "blocks", bi, "values", "placeholders", "-"), + "value": {"text": t, "meaning": meaning}} for t in p.target["texts"]] + return ops, [f"块「{block['id']}」把{quoted(p.target['texts'])}声明为{meaning}（存为空值）"] + + +# ========================================================================== +# ⑧ 声明隐藏行列怎么处理 +# ========================================================================== + + +def _p_declare_hidden(recipe: dict, prob: Any, args: dict, anchor: FixAnchor) -> FixProposal | None: + axis = args.get("axis") + si = sheet_index(recipe, args.get("sheet")) + if axis not in ("rows", "cols") or si is None: + return None + sheet = args.get("sheet") + filt = "可能是筛选隐藏的行。" if args.get("autofilter") else "" + if axis == "rows": + options = [FixOption("include", "隐藏行照常导入", detail=f"{filt}今后各期的隐藏行都照常导入"), + FixOption("exclude", "隐藏行不导入", detail=f"{filt}今后各期的隐藏行都跳过，记进回执的「排除的行」")] + what = "行" + else: + options = [FixOption("include", "隐藏列照常导入", detail="今后各期的隐藏列都照常导入")] + what = "列" + target = {"sheet": sheet, "axis": axis} + return _proposal("declare_hidden", _key("declare_hidden", target), code=_get(prob, "code"), + title=f"工作表「{sheet}」的数据区里有隐藏的{what}", cells=list(_get(prob, "cells") or []), + target=target, options=options, anchor=anchor) + + +def _o_declare_hidden(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]: + si = sheet_index(recipe, p.target.get("sheet")) + if si is None: + raise _Stale + return ([{"op": "replace", "path": ptr("sheets", si, "hidden", p.target["axis"]), "value": opt.value}], + [f"工作表「{p.target.get('sheet')}」：{opt.label}"]) + + +# ========================================================================== +# ⑨ 更新工作表名（P3-SPEC 6.1） +# ========================================================================== + + +def _sheet_option(name: str) -> FixOption: + detail = "下一期不再需要确认改名" + if has_digit(name): + detail = "新名字里有月份或期次数字，下一期很可能又会改名；可以不更新，每期确认改名即可" + return FixOption(name, f"把配方里的工作表名更新为「{name}」", detail=detail) + + +def _p_rename_sheet(recipe: dict, prob: Any, args: dict, anchor: FixAnchor) -> FixProposal | None: + sheets = recipe.get("sheets") or [] + si = next((i for i, s in enumerate(sheets) if s.get("id") == args.get("sheet_id")), None) + if si is None: + return None + cands = [c for c in args.get("candidates") or [] if isinstance(c, str) and c and len(c) <= _SHEET_MAX] + if not cands: + return None + old = (sheets[si].get("match") or {}).get("name") + target = {"sheet_id": sheets[si]["id"], "name": old, "candidates": cands} + return _proposal("rename_sheet", _key("rename_sheet", target), code=_get(prob, "code"), + title=f"没有找到工作表「{old}」，可以改用文件里的另一个工作表", cells=list(_get(prob, "cells") or []), + target=target, options=[_sheet_option(c) for c in cands], anchor=anchor) + + +def _p_sheet_renamed(recipe: dict, old: str, new: str) -> FixProposal | None: + """按 fallback 认出了改名的工作表（Extraction.sheets.renamed）：不是问题，但给提议（6.1 触发一）。""" + sheets = recipe.get("sheets") or [] + si = next((i for i, s in enumerate(sheets) if match_key((s.get("match") or {}).get("name")) == match_key(old)), None) + if si is None or not isinstance(new, str) or not new or len(new) > _SHEET_MAX: + return None + target = {"sheet_id": sheets[si]["id"], "name": old, "candidates": [new]} + return _proposal("rename_sheet", _key("rename_sheet", target), code=None, + title=f"工作表「{old}」改名为「{new}」", cells=[], target=target, options=[_sheet_option(new)], + anchor=FixAnchor("sheet_renamed", None, new)) + + +def _o_rename_sheet(recipe: dict, p: FixProposal, opt: FixOption, reason: str | None) -> tuple[list, list[str]]: + sheets = recipe.get("sheets") or [] + si = next((i for i, s in enumerate(sheets) if s.get("id") == p.target.get("sheet_id")), None) + if si is None or opt.value not in p.target.get("candidates", []): + raise _Stale + return ([{"op": "replace", "path": ptr("sheets", si, "match", "name"), "value": opt.value}], + [f"配方里的工作表名「{p.target.get('name')}」更新为「{opt.value}」"]) + + +# ========================================================================== +# 入口 +# ========================================================================== + +_Builder = Callable[[dict, Any, dict, FixAnchor], "FixProposal | None"] +_BUILDERS: dict[str, _Builder] = { + "remove_label": _p_remove_label, "add_label": _p_add_label, "rename_title": _p_rename_title, + "declare_total": _p_declare_total, "ignore_cells": _p_ignore_cells, "declare_placeholder": _p_declare_placeholder, + "declare_hidden": _p_declare_hidden, "rename_sheet": _p_rename_sheet, +} +_OPS: dict[str, Callable[[dict, FixProposal, FixOption, str | None], tuple[list, list[str]]]] = { + "remove_label": _o_remove_label, "add_label": _o_add_label, "edit_members": _o_edit_members, + "rename_title": _o_rename_title, "declare_total": _o_declare_total, "ignore_cells": _o_ignore_cells, + "declare_placeholder": _o_declare_placeholder, "declare_hidden": _o_declare_hidden, "rename_sheet": _o_rename_sheet, +} + + +def propose_fixes(recipe: dict[str, Any], problems: list[dict[str, Any]], recipe_problems: list[dict[str, Any]], + facts: dict[str, Any] | DraftFacts | None, + renamed: dict[str, str] | None = None) -> list[FixProposal]: + """问题 → 修复提议（纯函数，不抛异常）。 + + - problems：执行器的问题（Problem 的 JSON，或 Problem 本身）。按 fix 字段分派、按 fix_args 构造；没有 fix_args + 的不出提议（界面不能光靠 fix 字符串构造补丁）； + - recipe_problems：静态校验的问题。fact_claim_mismatch（以及关系成员列不存在的 column_unknown）出 ③； + - facts：本期的系统发现（DraftFacts 或它的 JSON），③ 的成员只能取自这里； + - renamed：试运行的 Extraction.sheets.renamed（配方名 → 实际名），出 ⑨。 + 每个提议带 anchor：index 是该问题在传进来的 problems / recipe_problems 里的下标，staging_out 据此把 fix_ids + 写到对应的问题上。同样的目标只出一个提议（按 id 去重，anchor 取第一条）。""" + if not isinstance(recipe, dict): + return [] + out: list[FixProposal] = [] + seen: set[str] = set() + + def push(p: FixProposal | None) -> None: + if p is not None and p.id not in seen: + seen.add(p.id) + out.append(p) + + for i, prob in enumerate(problems or []): + args = _get(prob, "fix_args") + builder = _BUILDERS.get(_get(prob, "fix") or "") + if builder is None or not isinstance(args, dict): + continue + try: + push(builder(recipe, prob, args, FixAnchor("problem", i))) + except (KeyError, TypeError, ValueError, IndexError, AttributeError): + continue # fix_args 的形状不对（执行器的契约问题）：不出提议，不拖垮 staging_out + fl = _facts(facts) + for i, rp in enumerate(recipe_problems or []): + try: + push(_p_edit_members(recipe, rp, fl, FixAnchor("recipe_problem", i))) + except (KeyError, TypeError, ValueError, IndexError, AttributeError): + continue + for old, new in (renamed or {}).items(): + push(_p_sheet_renamed(recipe, old, new)) + return out + + +def _edit_title(p: FixProposal, opt: FixOption) -> str: + """EditResult.title 是确认项 fix:<目标键> 的文案，要写明选了哪一项（选项的 label 加上主语）。""" + t = p.target + if p.kind == "edit_members": + return f"关系「{t.get('relation')}」：{opt.label}" + if p.kind == "rename_title": + return f"分段「{t.get('segment')}」的分段标题{opt.label}" + if p.kind == "declare_total": + return f"分段「{t.get('segment')}」末尾的{quoted(t.get('labels') or [])}{opt.label}" + if p.kind == "declare_placeholder": + return f"块「{t.get('block')}」{opt.label}" + if p.kind == "declare_hidden": + return f"工作表「{t.get('sheet')}」：{opt.label}" + if p.kind == "rename_sheet": + return opt.label + return p.title + + +def fix_ops(recipe: dict[str, Any], proposal: FixProposal, option: str, reason: str | None) -> EditResult: + """选中的选项 → EditResult（补丁、人话摘要、修改后的配方哈希）。 + + 选项不在 proposal.options 里、需要理由而没给（或超长）时抛 EditRequestError（code 见该类）。提议的目标在 + 当前工作配方里已经找不到时返回 ok=False（problems 写原因）。理由去掉首尾空白后写进配方，进配方哈希：预览和 + 应用必须用同一个理由（评审三-B1）。""" + opt = next((o for o in proposal.options if o.value == option), None) + if opt is None: + raise EditRequestError("edit_invalid", "所选的处理方式不在这条修复建议的选项里，请重新打开修复建议") + text = reason.strip() if isinstance(reason, str) else "" + if opt.needs_reason: + if not text: + raise EditRequestError("reason_required", "这个选项需要写明理由") + if len(text) > REASON_MAX: + raise EditRequestError("edit_invalid", f"理由不能超过 {REASON_MAX} 字") + key = _key(proposal.kind, proposal.target) + title = _edit_title(proposal, opt) + try: + ops, summary = _OPS[proposal.kind](recipe, proposal, opt, text if opt.needs_reason else None) + except (_Stale, KeyError, TypeError, IndexError): + msg = "这条修复建议对应的内容已不在当前的配方里：配方在提出建议之后又被改过。请重新查看问题和修复建议" + return EditResult(ok=False, kind="fix", key=key, title=title, summary=[], ops=[], anchors=[], notes=[], + problems=[Problem("edit_not_applicable", "recipe", msg, model_message="edit_not_applicable")]) + return finish(recipe, ops, kind="fix", key=key, title=title, summary=summary, breaking=opt.breaking) + + +def proposal_from_dict(data: dict[str, Any]) -> FixProposal: + """FixProposal 的 JSON（asdict）→ dataclass。WP-5 把提议存进响应、按 fix_id 找回来时用。""" + opts = [FixOption(**{k: v for k, v in o.items() if k in {f.name for f in dataclasses.fields(FixOption)}}) + for o in data.get("options") or []] + a = data.get("anchor") or {} + return FixProposal(id=data["id"], kind=data["kind"], problem_code=data.get("problem_code"), title=data["title"], + cells=list(data.get("cells") or []), target=dict(data.get("target") or {}), options=opts, + anchor=FixAnchor(a.get("kind", "problem"), a.get("index"), a.get("sheet"))) diff --git a/backend/app/data/recipe_imports.py b/backend/app/data/recipe_imports.py index d9f9bf24..40dba2b1 100644 --- a/backend/app/data/recipe_imports.py +++ b/backend/app/data/recipe_imports.py @@ -21,6 +21,15 @@ - 试运行把 asdict(Extraction) 和核对结果整份存进 trial，提交时据此还原、重算必勾确认项；差异卡的本期 一侧用这份完整回执，不用只截了前 50 条规范写法的 TrialOut.receipt。 - 发布前不改会话里的任何对象：对暂存区的修改一律放进 publish_build 的 before_commit（锁内、同一事务）。 + +**期 3（WP-5）补的几条：** +- 修复按钮与框选（edits/preview、apply、undo）：补丁由服务端按封闭选项算，客户端只发提议 id、选项和理由； + 应用时在同一个工作配方上重新换算，哈希对不上就是预览之后配方或理由变了（edit_stale）。每次修改的修改前起点 + 记在暂存区的 edits 里，撤销只撤最近一次，PUT 整份替换会清空撤销栈（之前的修改标 superseded）； +- 改配方之后，回答在新起点上仍然成立的保留（carry_answers），不成立的进 answers_dropped； +- 静态校验的 base 一律取数据源**当前的**现行配方（3.3）：启用旧版本之后，未结束暂存区的三处判断要一致； +- 按期累积：试运行里做累积计划、物化并集（提交时只 link），单期提交走 publish_build，多期走 publish_snapshot； +- 只读进程（没拿到版本存储的守卫）里，会删、写试运行库和原件的入口一律先 require_store_writable（503）。 """ from __future__ import annotations @@ -57,6 +66,8 @@ from app.data.recipe_types import ( DiffItem, Draft, DraftFacts, + EditResult, + ExcludedRows, Extraction, Fact, Grid, @@ -71,8 +82,10 @@ from app.data.recipe_types import ( RegionMark, SchemaNotes, SegmentLabels, + Selection, SheetsOut, TableOut, + UnionPart, ) from app.data.xlsx_scan import WorkbookScan from app.db.base import new_id, utcnow @@ -88,16 +101,22 @@ ANSWER_REASON_MAX = 200 ACCEPT_REASON_MAX = 500 #: 试运行回执里规范写法对照给界面看几条（全部在导入清单里） TRIAL_CANON_MAX = 50 -#: trial 里只给服务端自己用的键：StagingOut 不带它们（整份 Extraction 很大，界面也不需要） -TRIAL_INTERNAL = ("extraction", "null_counts", "inputs", "recipe_sha256", "build_id") +#: trial 里只给服务端自己用的键：StagingOut 不带它们（整份 Extraction 很大，界面也不需要）。union 是物化好的 +#: 并集（试运行文件路径、并集 id、回执），提交时用 +TRIAL_INTERNAL = ("extraction", "null_counts", "inputs", "recipe_sha256", "build_id", "union", "prev_import_id") #: TrialOut.receipt 给界面看的回执字段（7.4）。完整的回执在 trial["extraction"] 和导入清单里 _RECEIPT_VIEW = ( "ledger", "tables", "period", "axes", "placeholders", "labels", "outside_text", "hidden", "sheets", "derived_form", "block_order", "ignored_columns", "full_calc_on_load", "formula_cells_accepted", - "blank_rows_skipped", "table_hashes", "timings", + "blank_rows_skipped", "table_hashes", "timings", "rows_excluded", ) -#: 暂存区的 recipe_origin → 静态校验的 origin。manual / mixed 一律按 manual 校验 -_VALIDATE_ORIGIN = {"rules": "rules", "ai": "ai"} +#: 暂存区的 recipe_origin → 静态校验的 origin。manual / mixed 一律按 manual 校验；采用按规则重新起草的结果 +#: （rules_redraft，6.3）按规则草稿的口径校验 +_VALIDATE_ORIGIN = {"rules": "rules", "ai": "ai", "rules_redraft": "rules"} +#: 修改记录里只给服务端的键：撤销用的修改前起点，StagingOut、导入清单都不带（整份旧配方很大，界面也不需要） +EDIT_INTERNAL = ("before", "base_sha256_after") +#: 修改记录里只进导入清单、不给界面的键（StagingOut 的 edits 不带补丁：界面用不上，也免得回包随补丁变大） +EDIT_MANIFEST_ONLY = ("ops",) #: 统计期上下文的 id（契约 ContextSpec.id 只能是它） PERIOD_KEY = "统计期" _ACTIVE_TRIAL = ("passed", "needs_decision") @@ -183,13 +202,31 @@ class Pipeline: units: tuple[str, ...] = () #: (模型对象, 模型 id, 可用性)：同意与发送只解析一次模型（AU-7）。None 时退回 ai_availability，ai_draft 自己解析 resolve_model: Callable[..., Awaitable[tuple[Any, str, AiAvailability]]] | None = None + # ---- 期 3（P3-SPEC 12.6）。都给默认值 None：期 2 测试里全关键字构造的假 Pipeline 照旧成立，没接上的功能 + # 按「没有」处理（没有修复提议、不做累积计划），状态机的其余分支照常测 + #: WP-2 recipe_fixes：问题 → 修复提议；选中的选项 → 补丁 + propose_fixes: Callable[..., list[Any]] | None = None + fix_ops: Callable[..., EditResult] | None = None + #: WP-2 recipe_select：框选 → 补丁；重放比对 + selection_edit: Callable[..., EditResult] | None = None + replay_compare: Callable[..., Any] | None = None + #: WP-2 recipe_suggest：按规则重新起草的结果按来源对齐到现行配方的名字（6.3） + align_to_contract: Callable[..., tuple[dict[str, Any], list[str]]] | None = None + #: WP-3 recipe_compare：新旧配方对照 + compare_recipes: Callable[..., dict[str, Any]] | None = None + #: WP-4 recipe_accumulate：累积计划、配方演进分档、并集物化 + plan_accumulate: Callable[..., Any] | None = None + classify_changes: Callable[..., Any] | None = None + materialize_union: Callable[..., Any] | None = None + #: WP-3 recipe_notes：物化快照的说明 + build_union_notes: Callable[..., SchemaNotes] | None = None def default_pipeline() -> Pipeline: """接上真模块。函数体里惰性导入：这些模块（openpyxl、执行器、起草器）导入不便宜，接口模块加载时不必付。""" from app.data import ( - recipe, recipe_ai, recipe_checks, recipe_confirm, recipe_diff, recipe_engine, recipe_notes, - recipe_suggest, xlsx_cells, xlsx_scan, + recipe, recipe_accumulate, recipe_ai, recipe_checks, recipe_compare, recipe_confirm, recipe_diff, + recipe_engine, recipe_fixes, recipe_notes, recipe_select, recipe_suggest, xlsx_cells, xlsx_scan, ) return Pipeline( @@ -221,6 +258,16 @@ def default_pipeline() -> Pipeline: draft_max_rows=recipe_suggest.DRAFT_MAX_ROWS, units=tuple(recipe_parsers.UNITS), resolve_model=recipe_ai.resolve_draft_model, + propose_fixes=recipe_fixes.propose_fixes, + fix_ops=recipe_fixes.fix_ops, + selection_edit=recipe_select.selection_edit, + replay_compare=recipe_select.replay_compare, + align_to_contract=recipe_suggest.align_to_contract, + compare_recipes=recipe_compare.compare_recipes, + plan_accumulate=recipe_accumulate.plan_accumulate, + classify_changes=recipe_accumulate.classify_changes, + materialize_union=recipe_accumulate.materialize_union, + build_union_notes=recipe_notes.build_union_notes, ) @@ -240,6 +287,12 @@ class CommitResult: build_reused: bool unchanged: bool source_id: str = "" + #: 服务端已有的同版本数据文件被改过，这次用试运行的文件恢复了它（第 8 节遗留项 1） + build_restored: bool = False + #: 启用后当前版本含几期 + parts: int = 1 + #: 与此前某个版本内容相同，直接启用了那个版本（publish_snapshot 的复用、复活） + snapshot_reused: bool = False # ========================================================================== @@ -284,6 +337,8 @@ _EXTRACTION_PARTS: dict[str, Callable[[Any], Any]] = { "outside_text": lambda v: [OutsideText(**_known(OutsideText, x)) for x in v or []], "sheets": lambda v: SheetsOut(**_known(SheetsOut, v)), "axes": lambda v: [AxisOut(**_known(AxisOut, x)) for x in v or []], + # 期 3：排除的行（契约 ExcludedRows，第 8 节遗留项 4）。期 3 之前的回执没有这个键，按默认值（空列表） + "rows_excluded": lambda v: [ExcludedRows(**_known(ExcludedRows, x)) for x in v or [] if isinstance(x, dict)], } @@ -383,13 +438,18 @@ class _Snap: #: 起点配方（上传新一期的现行配方）的哈希：工作配方与它相同时按重放校验 base_sha: str | None = None context: dict[str, Any] = field(default_factory=dict) + #: 现行配方（canonical 形式）：数据源**当前的** current_recipe_id 对应的那一版（3.3，评审一-m7）。kind 为 + #: reupload / redraft 时给，静态校验据此放宽常量沿用、只查改动过的部分的认领；first / switch 为 None + base_recipe: dict[str, Any] | None = None -def _snap(staging: ImportStaging, *, base_sha: str | None = None, recipe_origin: str | None = None) -> _Snap: +def _snap(staging: ImportStaging, *, base_sha: str | None = None, recipe_origin: str | None = None, + base_recipe: dict[str, Any] | None = None) -> _Snap: return _Snap(id=staging.id, kind=staging.kind, raw_sha256=staging.raw_sha256, scan=copy.deepcopy(staging.scan), file_name=staging.file_name or "", recipe_origin=recipe_origin if recipe_origin is not None else staging.recipe_origin, - base_sha=base_sha, context=copy.deepcopy(staging.context_inputs or {})) + base_sha=base_sha, context=copy.deepcopy(staging.context_inputs or {}), + base_recipe=copy.deepcopy(base_recipe) if isinstance(base_recipe, dict) else None) def _read_raw(sha: str) -> bytes: @@ -447,14 +507,18 @@ def _dry_fn(p: Pipeline, book: _Book, filename: str, ctx: dict[str, Any] | None) def _validate(p: Pipeline, snap: _Snap, work: Any, facts: DraftFacts | None) -> tuple[Recipe | None, list[Any]]: - """静态校验。上传新一期、工作配方仍是现行配方时按重放校验（不查认领）；其余带系统发现查认领。""" + """静态校验。上传新一期、工作配方仍是现行配方时按重放校验（不查认领）；其余带系统发现查认领。 + + 有现行配方（kind 为 reupload / redraft）时把它作为 base 传下去（3.3）：常量可以沿用已确认的 pick，认领检查 + 只查与现行配方相比改动过的部分。base 只在有值时传：期 2 测试里的假 validate 没有这个关键字（12.0）。""" if not isinstance(work, dict): return None, [] if snap.kind == "reupload" and snap.base_sha: parsed, problems = p.validate(work, facts=None, origin="replay") if parsed is not None and p.recipe_sha256(parsed) == snap.base_sha: return parsed, problems - return p.validate(work, facts=facts, origin=_VALIDATE_ORIGIN.get(snap.recipe_origin or "", "manual")) + extra = {"base": snap.base_recipe} if snap.base_recipe is not None else {} + return p.validate(work, facts=facts, origin=_VALIDATE_ORIGIN.get(snap.recipe_origin or "", "manual"), **extra) def _cards(p: Pipeline, recipe: Any, facts: DraftFacts, grids: dict[str, Grid], @@ -671,6 +735,26 @@ def effective_origin(staging: ImportStaging, base_origin: str | None) -> str: return "mixed" if ai_used or base_origin in ("ai", "mixed") else "manual" +#: 配方记录（TableRecipe.origin）、导入清单只记这四种来源：界面按 RECIPE_ORIGIN_LABEL 显示，没有别的键 +STORED_ORIGINS = ("rules", "ai", "manual", "mixed") + + +def stored_origin(origin: str | None, staging: ImportStaging | None = None) -> str: + """配方入库（TableRecipe.origin、导入清单的 recipe.origin）时记的来源，只取 STORED_ORIGINS 之一。 + + rules_redraft 是**这个暂存区**采用了按规则重新起草的结果（6.3）：只用来让本次的确认清单列出全部配方类项并出 + redraft_adopted。它不能原样进配方记录：下个月上传新一期时，reupload 拿现行配方的 origin 给暂存区赋值， + recipe_confirm 见到 rules_redraft 就又出 redraft_adopted、又列全部配方类项，每个月都要再勾一遍并没有发生的 + 「采用重新起草」，违背「每月重传不打扰人」；卡片上还会露出英文键名（评审意见）。所以：没再改过的记 rules（内容 + 就是规则起草器给的，名字按来源对齐过）；采用之后又用修复按钮、框选改过（还有未被覆盖的修改）的记 manual， + 与首次导入时规则草稿加修复记 manual 一致。staging 为 None 时（给暂存区赋初值，防住已经入库的旧值）按没改过算。""" + if origin == "rules_redraft": + live = staging is not None and any(isinstance(e, dict) and not e.get("superseded") + for e in staging.edits or []) + return "manual" if live else "rules" + return origin if origin in STORED_ORIGINS else "manual" + + # ========================================================================== # 暂存、上传新一期、修改配方 # ========================================================================== @@ -747,6 +831,8 @@ async def stage_upload(session: AsyncSession, *, raw: bytes, filename: str, name """ p = pipeline() await table_versions.expire_stagings(session) + # 只读进程（没拿到版本存储的守卫）存不了原件：别先花时间扫描、起草（第 8 节遗留项 5） + table_versions.require_store_writable() existing = (await session.execute(select(DataSource).where(DataSource.name == name))).scalar_one_or_none() if existing is None: kind, source_id = "first", new_id() @@ -797,12 +883,13 @@ async def reupload(session: AsyncSession, source: DataSource, *, raw: bytes, fil """ p = pipeline() await table_versions.expire_stagings(session) + table_versions.require_store_writable() row = await _require_recipe_source(session, source) full = _full_form(row.recipe) staging = _new_staging(source_id=source.id, name=source.name, description=source.description or "", kind="reupload", filename=filename, raw=raw, signed_by=signed_by, base_recipe_id=row.id) - staging.recipe_origin = row.origin or "manual" - snap = _snap(staging, base_sha=row.recipe_sha256) + staging.recipe_origin = stored_origin(row.origin or "manual") + snap = _snap(staging, base_sha=row.recipe_sha256, base_recipe=row.recipe) staged, ev = await _in_slot(_replay_work, p, raw, full, filename, snap, evaluate=False) _fill(staging, staged, ev, answers_base=full) drafts = dict(staging.drafts or {}) @@ -836,6 +923,7 @@ async def redraft(session: AsyncSession, source: DataSource, *, signed_by: str | """POST {source_id}/redraft：取当前导入的原件、以当前配方为工作配方，重做系统发现和干跑 → drafting。""" p = pipeline() await table_versions.expire_stagings(session) + table_versions.require_store_writable() row = await _require_recipe_source(session, source) _snapshot, imp = await current_import(session, source) if imp is None or imp.raw_state != "kept" or not raw_store.raw_exists(imp.raw_sha256): @@ -846,8 +934,8 @@ async def redraft(session: AsyncSession, source: DataSource, *, signed_by: str | staging = _new_staging(source_id=source.id, name=source.name, description=source.description or "", kind="redraft", filename=imp.file_name, raw=raw, signed_by=signed_by, base_recipe_id=row.id) - staging.recipe_origin = row.origin or "manual" - snap = _snap(staging) + staging.recipe_origin = stored_origin(row.origin or "manual") + snap = _snap(staging, base_recipe=row.recipe) staged, ev = await _in_slot(_replay_work, p, raw, full, imp.file_name, snap, evaluate=True) _fill(staging, staged, ev, answers_base=full) drafts = dict(staging.drafts or {}) @@ -912,17 +1000,153 @@ def _check_answers(questions: list[Question], answers: dict[str, Any]) -> dict[s return clean -async def _base_sha(session: AsyncSession, staging: ImportStaging) -> str | None: - if staging.kind != "reupload" or not staging.base_recipe_id: - return None - row = await session.get(TableRecipe, staging.base_recipe_id) - return row.recipe_sha256 if row is not None else None +async def _base_of(session: AsyncSession, staging: ImportStaging) -> tuple[str | None, dict[str, Any] | None]: + """(按重放校验时比的现行配方哈希, 静态校验的 base)。一律取数据源**当前的** current_recipe_id（3.3，评审一-m7）： + 启用旧版本、移除、作废都会改它，静态校验、只列有变化的确认项、试运行三处取的「现行配方」必须一致。暂存区的 + base_recipe_id 只是创建时的记录。哈希只给上传新一期（修改配方期 2 起就一律带系统发现校验）。""" + if staging.kind not in ("reupload", "redraft"): + return None, None + source = await session.get(DataSource, staging.source_id) + row = await current_recipe(session, source) + if row is None: + return None, None + return (row.recipe_sha256 if staging.kind == "reupload" else None), copy.deepcopy(row.recipe) + + +async def _snap_of(session: AsyncSession, staging: ImportStaging, *, recipe_origin: str | None = None) -> _Snap: + sha, base = await _base_of(session, staging) + return _snap(staging, base_sha=sha, recipe_origin=recipe_origin, base_recipe=base) def _reeval_work(p: Pipeline, snap: _Snap, work: Any) -> _Eval: return _evaluate(p, _book_of(p, snap), snap, work) +def _doc_sha(doc: Any) -> str: + """一份起点配方的哈希（canonical 口径，与 recipe_sha256 同一种写法：去掉等于默认值的字段）。合 schema 的按 + canonical 算，不合的（PUT 照存的半成品）按原样算。延续回答、撤销时比对「是不是同一份起点」用。""" + try: + return table_versions.sha_json(Recipe.model_validate(doc).model_dump(mode="json", exclude_defaults=True)) + except Exception: # noqa: BLE001 - 不合 schema 的照样要能比 + return table_versions.sha_json(doc) + + +def carry_answers(p: Pipeline, base: Any, questions: list[Question], old_questions: list[Question], + answers: dict[str, dict[str, Any]], *, + reapply: frozenset[str] | set[str] = frozenset()) -> tuple[dict[str, dict[str, Any]], + list[dict[str, Any]]]: + """起点换了之后（PUT、应用修复或框选、撤销修改），旧回答逐个判断还成不成立（第 8 节遗留项 2）。 + + - 新问题里有同 id、同选项值的：把它的 effects（理由照旧填）应用到新起点上，结果与新起点的 canonical 形式 + 相同（新起点已经体现了这个回答），保留为已选；否则丢弃，reason=changed； + - 新问题里已经没有这个 id：丢弃，reason=gone。 + 返回 (保留的回答, answers_dropped)。answers_dropped 每项 {id, text, reason}：text 取自旧问题（gone 的问题在新 + 问题里已经没有了，界面只拿到 id 的话要么露出 id，要么什么都显示不了）。 + + 为什么不清空（期 2 的写法）：PUT 进来的配方本来就体现了之前的回答（占位符已经在列表里），清空之后看起来像没 + 答过，用户会以为要重答，或者以为自己的选择丢了。 + + reapply（只有撤销修改用）：这几个回答是在那次修改**之后**才做的，撤销恢复的起点当然还没有体现它们。按上面的 + 规则它们一律判 changed，用户在修改之后做的选择就被撤销连带撤掉了（3.1「修改之后又回答过的问题不应该被撤 + 掉」）。所以对它们放宽成「effects 能干净地应用到恢复的起点上就保留」，等于在恢复的起点上把这道题重新答一遍。 + """ + by_id = {q.id: q for q in questions} + old_by = {q.id: q for q in old_questions} + target = _doc_sha(base) + kept: dict[str, dict[str, Any]] = {} + dropped: list[dict[str, Any]] = [] + for qid, a in (answers or {}).items(): + old_q = old_by.get(qid) + text = old_q.text if old_q is not None else (by_id[qid].text if qid in by_id else qid) + q = by_id.get(qid) + if q is None: + dropped.append({"id": qid, "text": text, "reason": "gone"}) + continue + value = str((a or {}).get("value")) + if not any(str(o.get("value")) == value for o in q.options): + dropped.append({"id": qid, "text": text, "reason": "changed"}) + continue + ops = q.effects.get(value, []) + try: + doc = p.apply_patch(copy.deepcopy(base), _fill_reason(ops, (a or {}).get("reason"))) if ops else base + except (ValueError, TypeError, KeyError, IndexError): + doc = None + if doc is not None and (_doc_sha(doc) == target or qid in reapply): + kept[qid] = {"value": value, "reason": (a or {}).get("reason") or None} + else: + dropped.append({"id": qid, "text": text, "reason": "changed"}) + return kept, dropped + + +@dataclass +class _Rebased: + """起点换掉之后的一次评估：按新起点重出的问题、延续下来的回答、丢掉的回答。""" + + ev: _Eval + answers: dict[str, dict[str, Any]] + dropped: list[dict[str, Any]] + + +def _rebase_on(p: Pipeline, book: _Book, snap: _Snap, base: Any, old_questions: list[Question], + answers: dict[str, dict[str, Any]], ev: _Eval | None = None, *, + reapply: frozenset[str] | set[str] = frozenset()) -> _Rebased: + """新起点 → 评估（问题按新起点重出）→ 延续回答。保留下来的回答在新起点上一般都是无效果的，工作配方仍是新起点； + 撤销修改时 reapply 的那几条（修改之后才做的回答）有效果，几条回答叠在一起也可能有效果：这两种情况按工作配方 + 重新评估，问题仍按起点出。重放失败时先丢掉 reapply 的那几条再试（其余各条单独看都是无效果的），还不行才全丢。""" + ev = ev or _evaluate(p, book, snap, base) + kept, dropped = carry_answers(p, base, ev.questions, old_questions, answers, reapply=reapply) + + def drop(ids: list[str]) -> None: + for qid in ids: + dropped.append({"id": qid, "text": next((q.text for q in ev.questions if q.id == qid), qid), + "reason": "changed"}) + kept.pop(qid, None) + + if kept: + try: + work = replay_answers(p, base, ev.questions, kept) + except ValueError: + work = None + if work is None and any(qid in reapply for qid in kept): + drop([qid for qid in kept if qid in reapply]) + try: + work = replay_answers(p, base, ev.questions, kept) if kept else base + except ValueError: + work = None + if work is None: + drop(list(kept)) + work = base + if _doc_sha(work) != _doc_sha(base): + again = _evaluate(p, book, snap, work) + again.questions = ev.questions + ev = again + return _Rebased(ev=ev, answers=kept, dropped=dropped) + + +def _rebase_work(p: Pipeline, snap: _Snap, base: Any, old_questions: list[Question], + answers: dict[str, dict[str, Any]], *, + reapply: frozenset[str] | set[str] = frozenset()) -> _Rebased: + return _rebase_on(p, _book_of(p, snap), snap, base, old_questions, answers, reapply=reapply) + + +def _apply_rebase(staging: ImportStaging, base: Any, rebased: _Rebased, *, origin: str | None) -> str | None: + """把换起点的结果写回暂存区（不提交）。返回旧的试运行库路径（配方变了才有）。""" + if origin is not None: + staging.recipe_origin = origin + staging.answers_base = copy.deepcopy(base) + staging.answers = dict(rebased.answers) + staging.answers_dropped = list(rebased.dropped) + return _apply_eval(staging, rebased.ev, questions=True) + + +def _supersede_edits(staging: ImportStaging) -> None: + """整份替换工作配方（PUT、采用 AI 草稿）：之前的修改标 superseded，不能再撤销，也不再出 fix:* / select:* 确认项 + （它们的效果可能已经不在新配方里），只作为历史进导入清单（3.1，评审丙-M4）。""" + edits = [dict(e) for e in staging.edits or [] if isinstance(e, dict)] + if edits: + staging.edits = [{**e, "superseded": True} for e in edits] + + async def answer(session: AsyncSession, staging: ImportStaging, answers: dict[str, dict[str, Any]]) -> ImportStaging: """POST imports/{id}/answers：工作配方 = answers_base 重放这一组回答（整组替换已存的回答）， 重做系统发现、静态校验、干跑和卡片。问题集合不变（它按 answers_base 生成）。""" @@ -934,39 +1158,423 @@ async def answer(session: AsyncSession, staging: ImportStaging, answers: dict[st if clean: raise ImportRefused(422, "answer_invalid", "还没有可回答的配方草稿") return staging + # 回答会让旧的试运行库作废、被删掉：只读进程不写版本存储（第 8 节遗留项 5） + table_versions.require_store_writable() try: work = replay_answers(p, staging.answers_base, questions, clean) except ValueError as e: raise ImportRefused(422, "answer_invalid", f"回答没能应用到配方上：{e}") from e - snap = _snap(staging, base_sha=await _base_sha(session, staging)) + snap = await _snap_of(session, staging) ev = await _in_slot(_reeval_work, p, snap, work) await _still_open(session, staging) old = _apply_eval(staging, ev, questions=False) staging.answers = clean + # 用户重新回答过了：「需要重新回答」的提示到此为止 + staging.answers_dropped = [] await session.commit() await _remove_trial(old) return staging +_REDRAFT_MISMATCH = "采用的配方与重新起草的结果不同，请重新起草后再采用" + + +def _schema_sha(p: Pipeline, doc: Any) -> str | None: + """配方的哈希（只看 schema，不做语义校验），与 EditResult.recipe_sha256_after、暂存区的哈希同一口径。""" + try: + return p.recipe_sha256(Recipe.model_validate(doc)) + except Exception: # noqa: BLE001 + return None + + async def put_recipe(session: AsyncSession, staging: ImportStaging, recipe: dict[str, Any], *, origin: str = "manual") -> ImportStaging: - """PUT imports/{id}/recipe：整份配方作为新的起点（answers_base 换成它、回答清空），照存；静态校验的问题 - 写进 recipe_problems（有问题也存，只是不能试运行）。""" + """PUT imports/{id}/recipe：整份配方作为新的起点（answers_base 换成它），照存；静态校验的问题写进 + recipe_problems（有问题也存，只是不能试运行）。 + + 期 3：回答在新起点上仍然成立的保留（carry_answers），不成立的进 answers_dropped；撤销栈清空，之前的修改标 + superseded（3.1）。origin="rules_redraft"（采用按规则重新起草的结果，6.3）时配方的哈希必须等于最近一次 + redraft-rules 给出的那份（redraft_mismatch）：服务端据此记 recipe_origin=rules_redraft，确认清单按首次导入 + 列出全部配方类项并出 redraft_adopted，起草器识别出的定位规则才会逐项经过人的确认（H3）。""" p = pipeline() require_open(staging) base = _full_form(recipe) - snap = _snap(staging, base_sha=await _base_sha(session, staging), recipe_origin=origin) - ev = await _in_slot(_reeval_work, p, snap, base) + if origin == "rules_redraft": + sha = _schema_sha(p, base) + if not staging.redraft_sha256 or sha != staging.redraft_sha256: + raise ImportRefused(409, "redraft_mismatch", _REDRAFT_MISMATCH) + table_versions.require_store_writable() + old_questions = [_question_from(q) for q in staging.questions or []] + answers = dict(staging.answers or {}) + snap = await _snap_of(session, staging, recipe_origin=origin) + rebased = await _in_slot(_rebase_work, p, snap, base, old_questions, answers) await _still_open(session, staging) - staging.recipe_origin = origin - staging.answers_base = copy.deepcopy(base) - staging.answers = {} - old = _apply_eval(staging, ev, questions=True) + _supersede_edits(staging) + old = _apply_rebase(staging, base, rebased, origin=origin) await session.commit() await _remove_trial(old) return staging +# ========================================================================== +# 期 3：修复按钮、框选（edits/preview、apply、undo），按规则重新起草 +# ========================================================================== + + +@dataclass +class EditRequest: + """edits/preview、edits/apply 的请求（接口层解析好）：修复给 fix_id、option、reason；框选给 selection。""" + + kind: str + fix_id: str = "" + option: str = "" + reason: str | None = None + selection: Selection | None = None + + +def _fix_inputs(staging: ImportStaging) -> tuple[str, list[dict[str, Any]], list[dict[str, Any]], dict[str, str] | None]: + """提议按哪些问题现算（3.1）：当前试运行没失效且拒收时用 trial.problems，否则用 draft_problems；再加上 + recipe_problems，以及当前试运行的 extraction.sheets.renamed。返回 (问题出自哪里, 问题, 配方问题, 改名)。""" + trial = staging.trial if isinstance(staging.trial, dict) else {} + if staging.trial_key and trial.get("status") == "rejected": + where, problems = "trial", [x for x in trial.get("problems") or [] if isinstance(x, dict)] + else: + where, problems = "draft", [x for x in staging.draft_problems or [] if isinstance(x, dict)] + renamed = None + if staging.trial_key: + sheets = ((trial.get("extraction") or {}).get("sheets") or {}) if isinstance(trial.get("extraction"), dict) else {} + got = sheets.get("renamed") if isinstance(sheets, dict) else None + renamed = {str(k): str(v) for k, v in got.items()} if isinstance(got, dict) and got else None + recipe_problems = [x for x in staging.recipe_problems or [] if isinstance(x, dict)] + return where, problems, recipe_problems, renamed + + +def proposals_for(p: Pipeline, staging: ImportStaging) -> tuple[list[Any], str]: + """当前的修复提议（不存库，每次按暂存区里存的问题现算；同样的问题得到同样的 id）。没接上 propose_fixes、 + 暂存区已结束或还没有配方时为空。出错只记日志：提议是辅助信息，不能拖垮 StagingOut。""" + where, problems, recipe_problems, renamed = _fix_inputs(staging) + if p.propose_fixes is None or not table_versions.staging_is_open(staging) or not isinstance(staging.recipe, dict): + return [], where + try: + return list(p.propose_fixes(_full_form(staging.recipe), problems, recipe_problems, staging.facts or {}, + renamed)), where + except Exception as e: # noqa: BLE001 + logger.warning("propose_fixes failed: %s", type(e).__name__) + return [], where + + +@dataclass +class _EditRun: + edit: EditResult + new: Any = None + rebased: _Rebased | None = None + replay: Any = None + + +def _dry_before(p: Pipeline, book: _Book, snap: _Snap, full: Any) -> Extraction | None: + """框选要的「当前工作配方最近一次干跑」：工作配方合 schema 才干跑（判断选区和已有的块是否重叠）。""" + if not isinstance(full, dict): + return None + try: + parsed = Recipe.model_validate(full) + except Exception: # noqa: BLE001 + return None + return _safe_dry(_dry_fn(p, book, snap.file_name, snap.context), parsed) + + +def _edit_work(p: Pipeline, snap: _Snap, full: Any, req: EditRequest, proposal: Any, + retired: dict[str, list[str]] | None, old_questions: list[Question], + answers: dict[str, dict[str, Any]]) -> _EditRun: + """（线程里跑）换算一次修改，ok 时把补丁应用到工作配方上、静态校验（带系统发现和 base）、干跑、重放比对， + 再按新起点重出问题、延续回答（4.4）。预览和应用走同一段，应用时才把结果写回暂存区。""" + from app.data.recipe_fixes import EditRequestError + + book = _book_of(p, snap) + if req.kind == "fix": + edit = p.fix_ops(full, proposal, req.option, req.reason) + else: + sel = req.selection + grid = book.grids.get(sel.sheet) if sel is not None else None + if grid is None: + raise EditRequestError("edit_invalid", f"这份文件里没有可以框选的工作表「{sel.sheet if sel else ''}」") + before = _dry_before(p, book, snap, full) + + def draft_fn() -> Draft: + return p.draft(book.scan, book.grids, snap.file_name, + validate=lambda data, f, o: p.validate(data, facts=f, origin=o), + dry_run=_Capture(_dry_fn(p, book, snap.file_name, snap.context))) + edit = p.selection_edit(full if isinstance(full, dict) else None, grid, sel, extraction=before, + draft_fn=draft_fn, retired_names=retired or None) + if not edit.ok: + return _EditRun(edit=edit) + new = p.apply_patch(copy.deepcopy(full) if isinstance(full, dict) else {}, edit.ops) + rebased = _rebase_on(p, book, snap, new, old_questions, answers) + replay = None + ext = rebased.ev.extraction + if req.kind == "selection" and ext is not None and p.replay_compare is not None: + window = p.draft_max_rows if ext.partial else None + replay = p.replay_compare(edit, ext, window_rows=window) + return _EditRun(edit=edit, new=new, rebased=rebased, replay=replay) + + +async def _edit_context(session: AsyncSession, staging: ImportStaging, req: EditRequest + ) -> tuple[Pipeline, _Snap, Any, Any, dict[str, list[str]] | None]: + """预览、应用共用的准备：工作配方的完整形式、要用的提议（fix_stale）、退役名字（框选撞名检查用）。""" + p = pipeline() + require_open(staging) + full = _full_form(staging.recipe) if isinstance(staging.recipe, dict) else None + proposal = None + if req.kind == "fix": + if p.fix_ops is None: + raise ImportRefused(409, "fix_stale", "这条修复建议已不在当前的列表里，请刷新后重新查看") + proposal = next((x for x in proposals_for(p, staging)[0] if x.id == req.fix_id), None) + if proposal is None: + raise ImportRefused(409, "fix_stale", "这条修复建议已不在当前的列表里：问题在打开建议之后有了变化。" + "请重新查看问题和修复建议") + elif p.selection_edit is None: + raise ImportRefused(422, "edit_invalid", "这个服务端没有提供框选") + retired: dict[str, list[str]] | None = None + if req.kind == "selection": + from app.data import source_versions + + source = await session.get(DataSource, staging.source_id) + snapshot = (await session.get(SourceSnapshot, source.current_snapshot_id) + if source is not None and source.current_snapshot_id else None) + try: + retired = source_versions.retired_names(source_versions.snapshot_manifest(snapshot)) + except source_versions.PartProblem: + retired = None + snap = await _snap_of(session, staging) + return p, snap, full, proposal, retired + + +async def _run_edit(session: AsyncSession, staging: ImportStaging, req: EditRequest) -> tuple[Pipeline, Any, _EditRun]: + from app.data.recipe_fixes import EditRequestError + + p, snap, full, proposal, retired = await _edit_context(session, staging, req) + old_questions = [_question_from(q) for q in staging.questions or []] + try: + run = await _in_slot(_edit_work, p, snap, full, req, proposal, retired, old_questions, + dict(staging.answers or {})) + except EditRequestError as e: + raise ImportRefused(422, e.code or "edit_invalid", str(e)) from e + except ValueError as e: + raise ImportRefused(422, "edit_invalid", str(e) or "修改请求的格式不对") from e + return p, full, run + + +async def _accumulate_change(session: AsyncSession, p: Pipeline, staging: ImportStaging, new: Any) -> str | None: + """预览里的 accumulate_change：当前版本是按期累积时，修改后的配方对当前各期的分档（2.5）；否则 None。""" + if p.classify_changes is None: + return None + from app.data import source_versions + + source = await session.get(DataSource, staging.source_id) + snapshot = (await session.get(SourceSnapshot, source.current_snapshot_id) + if source is not None and source.current_snapshot_id else None) + if snapshot is None or (snapshot.mode or "replace") != "accumulate" or not source.current_recipe_id: + return None + try: + parts = await source_versions.load_parts(session, snapshot) + tables = await source_versions.current_tables(session, snapshot) + status = source_versions.retired_status(source_versions.snapshot_manifest(snapshot)) + cc = p.classify_changes([x.info for x in parts], Recipe.model_validate(new), current_tables=tables, + retired_status=status) + except Exception as e: # noqa: BLE001 - 分档只是提示 + logger.warning("classify_changes for preview failed: %s", type(e).__name__) + return None + return str(cc.change) + + +async def edit_preview(session: AsyncSession, staging: ImportStaging, req: EditRequest, *, + seq: Any = None) -> dict[str, Any]: + """POST imports/{id}/edits/preview（4.4、9.4 EditPreview）：换算、应用到工作配方的副本上、静态校验、干跑、重放 + 比对、对现行配方的破坏性变化、新旧对照。不改暂存区。换算不了时照样 200，ok=false、problems 写原因。""" + p, full, run = await _run_edit(session, staging, req) + edit = run.edit + out = _jsonable(edit) + out.update({"recipe_sha256_before": staging.recipe_sha256, "recipe_problems": [], "dry_run": None, + "replay": None, "breaking": {}, "accumulate_change": None, "compare": None, "seq": seq}) + if not edit.ok: + return out + ev = run.rebased.ev + out["recipe_problems"] = _problems_json(ev.problems) + if ev.extraction is not None: + out["dry_run"] = {"problems": [_jsonable(x) for x in ev.extraction.problems], + "partial": bool(ev.extraction.partial), "marks": _regions(ev.extraction)} + out["replay"] = _jsonable(run.replay) if run.replay is not None else None + source = await session.get(DataSource, staging.source_id) + row = await current_recipe(session, source) + if row is not None and isinstance(run.new, dict): + # 9.4：breaking 是 table_changes 算出的「表名 → 人话列表」，不是 EditResult.breaking 那个布尔（WP-0 交接） + try: + out["breaking"] = dict(p.breaking_changes(row.recipe, run.new)) + except Exception: # noqa: BLE001 - 修改后的配方不合 schema 时没有可比的表结构 + out["breaking"] = {} + if p.compare_recipes is not None and isinstance(full, dict) and isinstance(run.new, dict): + try: + out["compare"] = p.compare_recipes(full, run.new) + except Exception: # noqa: BLE001 + out["compare"] = None + out["accumulate_change"] = await _accumulate_change(session, p, staging, run.new) + return out + + +def _edit_out(entry: dict[str, Any], *, last: bool, open_: bool) -> dict[str, Any]: + """修改记录给界面的形状：不带撤销用的修改前起点；undoable 只给最后一条、未被覆盖的。""" + out = {k: copy.deepcopy(v) for k, v in entry.items() if k not in EDIT_INTERNAL and k not in EDIT_MANIFEST_ONLY} + out["superseded"] = bool(entry.get("superseded")) + out["undoable"] = bool(last and open_ and not entry.get("superseded")) + return out + + +async def edit_apply(session: AsyncSession, staging: ImportStaging, req: EditRequest, *, expected_sha256: str | None, + signed_by: str | None) -> ImportStaging: + """POST imports/{id}/edits/apply：在同一个工作配方上重新换算，哈希必须等于预览时的 expected_sha256（edit_stale： + 预览之后配方、选项或理由变了；理由进配方哈希，界面必须原样提交预览时的理由）。之后新配方作为新的起点，回答按 + carry_answers 延续，试运行作废。修改记录存修改前的工作配方和回答（撤销用）、修改后起点的哈希（undo_stale 的兜底）， + 以及补丁和所选的选项、理由或选区（进导入清单的 edits）。""" + p, full, run = await _run_edit(session, staging, req) + edit = run.edit + if not edit.ok: + why = "；".join(x.message for x in edit.problems[:3]) or "这次修改已无法应用到当前的配方上" + raise ImportRefused(422, "edit_not_applicable", why) + if not expected_sha256 or edit.recipe_sha256_after != expected_sha256: + raise ImportRefused(409, "edit_stale", "配方在预览之后有变化（或选项、理由与预览时不同），请重新预览") + table_versions.require_store_writable() + await _still_open(session, staging) + edits = [dict(e) for e in staging.edits or [] if isinstance(e, dict)] + # 撤销用的「修改前的起点」存修改前的**工作配方**（完整形式），不存 answers_base：补丁打在工作配方上（新起点 = + # 工作配方 + 补丁，已经体现了此前的回答），撤销时恢复的起点必须与它对称，此前的回答在恢复的起点里也都已体现， + # carry_answers 才会照常保留它们。存 answers_base 的话，有效果的回答（例如导入模式选了每期替换）在撤销时一律 + # 判 changed 丢掉，配方悄悄退回草稿的默认值（评审意见：撤销丢回答）。修改前的回答另存一份，撤销时据此分出 + # 哪些是修改之后才做的（见 edit_undo） + before_base = full if isinstance(full, dict) else copy.deepcopy(staging.answers_base) + entry = { + "seq": max((int(e.get("seq") or 0) for e in edits), default=0) + 1, + "kind": req.kind, "key": edit.key, "title": edit.title, "summary": list(edit.summary), + "at": _now_iso(), "signed_by": (signed_by or staging.signed_by or None), + "recipe_sha256_before": staging.recipe_sha256, "recipe_sha256_after": edit.recipe_sha256_after, + "superseded": False, + # 修改的内容本身（3.1「修改记录（谁、何时、补丁）……只进导入清单 edits」）：补丁，以及人选的是什么——修复 + # 记提议 id、选项和理由，框选记选区（Selection.to_json，键是 as，不用 asdict）。导入清单据此能复原这次 + # 改了什么，不只剩人话摘要。StagingOut 不给 ops（_edit_out 去掉），界面用不上 + "ops": copy.deepcopy(list(edit.ops)), + **({"fix": {"id": req.fix_id, "option": req.option, "reason": req.reason}} if req.kind == "fix" else {}), + **({"selection": req.selection.to_json()} if req.kind == "selection" and req.selection is not None else {}), + "before": {"answers_base": copy.deepcopy(before_base), "answers": copy.deepcopy(dict(staging.answers or {})), + "recipe_origin": staging.recipe_origin}, + "base_sha256_after": _doc_sha(run.new), + } + # 修改是人做的：照 PUT 记 manual（采用过 AI 草稿的由 effective_origin 记 mixed）。采用按规则重新起草的结果之后 + # 再用修复按钮，仍记 rules_redraft：确认清单照旧列出全部配方类项和 redraft_adopted，起草器识别出的定位规则不能 + # 因为补了一个封闭补丁就免于逐项确认（H3） + origin = "rules_redraft" if staging.recipe_origin == "rules_redraft" else "manual" + staging.edits = [*edits, entry] + old = _apply_rebase(staging, run.new, run.rebased, origin=origin) + await session.commit() + await _remove_trial(old) + return staging + + +async def edit_undo(session: AsyncSession, staging: ImportStaging) -> ImportStaging: + """POST imports/{id}/edits/undo：撤销最近一次修改（3.1）。先比对当前起点的哈希等于最后一条修改的 + base_sha256_after（否则 undo_stale：之后又被改过；正常流程不会出现，PUT 会清空撤销栈，这是兜底），再恢复修改前的 + 起点（修改前的工作配方，见 edit_apply）和 recipe_origin。回答按 carry_answers 延续，不是整份恢复成修改前的回答： + 修改前的回答已体现在恢复的起点里，照常保留；修改之后又回答过的问题不该被撤掉，在恢复的起点上重新应用（reapply）。""" + p = pipeline() + require_open(staging) + edits = [dict(e) for e in staging.edits or [] if isinstance(e, dict)] + last = edits[-1] if edits else None + if last is None or last.get("superseded") or not isinstance(last.get("before"), dict): + raise ImportRefused(409, "nothing_to_undo", "没有可以撤销的修改（整份替换配方之后，之前的修改不能再撤销）") + if _doc_sha(staging.answers_base) != last.get("base_sha256_after"): + raise ImportRefused(409, "undo_stale", "这次修改之后配方又被改过，无法直接撤销；请在配方面板中手工改回") + table_versions.require_store_writable() + before = last["before"] + base = copy.deepcopy(before.get("answers_base")) + origin = before.get("recipe_origin") + old_questions = [_question_from(q) for q in staging.questions or []] + # 恢复的起点是修改前的工作配方，修改前的回答都已体现在里面。要判断的回答 = 修改前的回答，再叠上现在的回答： + # 修改时因为补丁而丢掉的回答，撤销后又成立了，照常恢复为已选；修改之后才做（或改过）的回答，恢复的起点还没有 + # 体现，按 reapply 在恢复的起点上重新应用，不跟着撤销一起撤掉（3.1） + prior = {str(k): v for k, v in (before.get("answers") or {}).items() if isinstance(v, dict)} + current = {str(k): v for k, v in (staging.answers or {}).items() if isinstance(v, dict)} + + def same(a: Any, b: Any) -> bool: + return (isinstance(a, dict) and isinstance(b, dict) and str(a.get("value")) == str(b.get("value")) + and (a.get("reason") or None) == (b.get("reason") or None)) + + later = {qid for qid, a in current.items() if not same(prior.get(qid), a)} + snap = await _snap_of(session, staging, recipe_origin=origin) + rebased = await _in_slot(_rebase_work, p, snap, base, old_questions, {**prior, **current}, reapply=later) + # 只对现在的回答报「需要重新回答」：修改前的回答若在修改时已经丢掉、撤销后仍不成立，修改那一刻已经提示过 + rebased.dropped = [d for d in rebased.dropped if d.get("id") in current] + await _still_open(session, staging) + staging.edits = edits[:-1] + old = _apply_rebase(staging, base, rebased, origin=origin) + await session.commit() + await _remove_trial(old) + return staging + + +@dataclass +class _Redrafted: + draft: Draft + aligned: dict[str, Any] | None + alignment: list[str] + compare: dict[str, Any] | None + sha: str | None + + +def _redraft_rules_work(p: Pipeline, snap: _Snap, current: dict[str, Any]) -> _Redrafted: + """（线程里跑）对暂存区的文件跑规则起草（不调用模型），名字按来源对齐到现行配方，再和现行配方对照（6.3）。""" + book = _book_of(p, snap) + dry = _Capture(_dry_fn(p, book, snap.file_name, snap.context)) + d = p.draft(book.scan, book.grids, snap.file_name, validate=lambda data, f, o: p.validate(data, facts=f, origin=o), + dry_run=dry) + if d.recipe is None: + # 失败原因只留在 draft.failures：alignment 是「名字对齐」的报告，界面把它放在那个标题下面；抄一份过去，同一句话 + # 就在错误的标题下再出现一次（评审意见；check-manage 对这种情形伪造的也是 alignment: []） + return _Redrafted(draft=d, aligned=None, alignment=[], compare=None, sha=None) + candidate = _full_form(d.recipe) + if p.align_to_contract is not None: + aligned, report = p.align_to_contract(candidate, current) + else: + aligned, report = candidate, [] + try: + parsed = Recipe.model_validate(aligned) + except Exception: # noqa: BLE001 + return _Redrafted(draft=d, aligned=None, compare=None, sha=None, + alignment=[*report, "对齐后的配方格式不对，无法采用；请在配方面板中修改现行配方"]) + full = parsed.model_dump(mode="json") + cmp = p.compare_recipes(current, full) if p.compare_recipes is not None else None + return _Redrafted(draft=d, aligned=full, alignment=list(report), compare=cmp, sha=p.recipe_sha256(parsed)) + + +async def redraft_rules(session: AsyncSession, staging: ImportStaging) -> dict[str, Any]: + """POST imports/{id}/redraft-rules（6.3）：不改工作配方，只把对齐后配方的哈希记在 redraft_sha256，「采用」时 + PUT recipe（origin=rules_redraft）核对它。首次导入、切换那里已有规则草稿，不提供（redraft_not_offered）。""" + p = pipeline() + require_open(staging) + if staging.kind not in ("reupload", "redraft"): + raise ImportRefused(409, "redraft_not_offered", "首次导入已有规则起草的结果，不需要重新起草") + table_versions.require_store_writable() + source = await session.get(DataSource, staging.source_id) + row = await _require_recipe_source(session, source) if source is not None else None + if row is None: + raise ImportRefused(409, "not_recipe_source", _NOT_RECIPE) + current = _full_form(row.recipe) + snap = await _snap_of(session, staging) + out = await _in_slot(_redraft_rules_work, p, snap, current) + await _still_open(session, staging) + staging.redraft_sha256 = out.sha + staging.updated_at = utcnow() + await session.commit() + return {"draft": _draft_out(_jsonable(out.draft)), "aligned_recipe": out.aligned, "alignment": out.alignment, + "compare": out.compare} + + # ========================================================================== # AI 起草 # ========================================================================== @@ -1188,6 +1796,9 @@ async def _run_ai_draft(session: AsyncSession, staging: ImportStaging, *, staging.recipe_origin = "ai" staging.answers_base = copy.deepcopy(base) staging.answers = {} + staging.answers_dropped = [] + # 整份换成 AI 草稿：同 PUT，之前的修改不能再撤销、不再出确认项 + _supersede_edits(staging) old = _apply_eval(staging, ev, questions=True) if old is None and not was_ai: # AI 草稿与工作配方逐字相同（哈希没变）时 _apply_eval 不作废试运行，但来源从规则 / 手改变成了 AI， @@ -1332,29 +1943,242 @@ async def _context(session: AsyncSession, staging: ImportStaging) -> _Context: return _Context(source=source, recipe=row, snapshot=snapshot, imp=imp, prev=prev) +def _active_edits(staging: ImportStaging) -> list[dict[str, Any]]: + """确认项要的修改记录：只含未被覆盖的（3.1），不带撤销用的修改前起点。""" + return [{k: v for k, v in e.items() if k not in EDIT_INTERNAL} + for e in staging.edits or [] if isinstance(e, dict) and not e.get("superseded")] + + def _confirm_context(staging: ImportStaging, cx: _Context, recipe: Recipe, ex_dict: dict[str, Any], - checks: list[Any], diff: list[Any] | None) -> ConfirmContext: + checks: list[Any], diff: list[Any] | None, *, prev: dict[str, Any] | None = None, + accumulate: dict[str, Any] | None = None) -> ConfirmContext: + """确认项的输入。prev 给了就用它（按期累积时上一期是统计期最晚的那一期，替换该期时是被替换的那一期，9.5）， + 否则用现行导入的清单。""" from app.data.engine import masked_columns + prev = prev if prev is not None else cx.prev old = cx.recipe.recipe if cx.recipe is not None and staging.kind in ("reupload", "redraft") else None return ConfirmContext( kind=staging.kind, recipe=recipe, old_recipe=old, old_schema_cache=(cx.snapshot.schema_cache or {}) if staging.kind == "switch" and cx.snapshot else None, - extraction=ex_dict, checks=checks, diff=(diff or []) if cx.prev is not None else None, prev=cx.prev, + extraction=ex_dict, checks=checks, diff=(diff or []) if prev is not None else None, prev=prev, recipe_origin=effective_origin(staging, cx.recipe.origin if cx.recipe is not None else None), facts=_facts_from(staging.facts), mask_columns=masked_columns(cx.source.options) if cx.source is not None else None, + accumulate=accumulate, edits=_active_edits(staging), ) +def _structure_problem(code: str, message: str) -> dict[str, Any]: + return {"code": code, "category": "structure", "message": message, "cells": [], "model_message": code, + "fix": None} + + +@dataclass +class _Acc: + """试运行里按期累积的那一步（2.4、2.6、2.8）：计划、物化好的并集、说明、额外的拒收问题。""" + + plan: dict[str, Any] | None = None + #: 提交时用的并集（只给服务端，TRIAL_INTERNAL）：试运行文件、并集 id、哈希、登记的选项和回执、各期顺序 + union: dict[str, Any] | None = None + union_checks: list[dict[str, Any]] | None = None + notes: SchemaNotes | None = None + problems: list[dict[str, Any]] = field(default_factory=list) + #: 差异卡的上一期（替换该期时是被替换的那一期，9.5）；None = 用现行导入 + prev_import_id: str | None = None + + +def _union_note_new(parsed: Recipe, ex: Extraction, run: _TrialRun, start: str, end: str, acc: list[Acceptance]) -> Any: + from app.data.recipe_types import UnionNotePart + + return UnionNotePart(recipe=parsed, extraction=ex, checks=list(run.checks), acceptances=list(acc), + null_counts=run.null_counts, start=start, end=end) + + +async def _accumulate_step(session: AsyncSession, p: Pipeline, staging: ImportStaging, cx: _Context, parsed: Recipe, + recipe_sha: str, run: _TrialRun, build_id: str) -> _Acc | None: + """新配方是按期累积、或者当前版本是按期累积时做累积计划；结果有两期及以上时在这里物化并集（提交时只 link）， + 并按各期最弱的结论生成说明预览（本期按「全部未接受」）。不需要累积计划时返回 None。 + + 当前快照某一期的导入清单被改过、统计期与库列对不上（part_tampered）：拒收，当前快照不动。某一期的构建库不在了 + （part_missing）：409，消息指向数据源卡片的「版本」（可以先移除那一期，7.4）。""" + from app.data import recipe_accumulate as ra + from app.data import source_versions as sv + + if p.plan_accumulate is None or run.extraction is None or run.db is None: + return None + ex = run.extraction + snapshot = cx.snapshot if cx.source is not None else None + cur_kind = None if snapshot is None else ("recipe" if cx.source.current_recipe_id else "simple") + cur_mode = (snapshot.mode or "replace") if cur_kind == "recipe" else None + if parsed.mode != "accumulate" and cur_mode != "accumulate": + return None + period = ex.period + new_period = (period.start, period.end) if period is not None and period.start and period.end else None + parts: list[sv.Part] = [] + tables = status = None + try: + if cur_kind == "recipe": + parts = await sv.load_parts(session, snapshot) + # 退役只按结果各期有数据的列判定（WP-8 修补）：当前版本的表结构里，只留结果各期（当前各期去掉本期要替换的 + # 那一期）的配方里有、或者目标配方里有的列。否则两种情况都会出一条任何一期都没有数据的必勾 retire:*， + # 文案还写「早期各期保留原值」：移除唯一含某列的那一期之后（留下的全空目标列），以及替换唯一含某列的那一期 + # （被替换掉的旧数据不在新版本里）。替换该期时这一列就是从当前配方里消失了，由 breaking:<表> 要人确认 + gone = _replaced_part(parts, new_period, staging.kind) + tables = sv.tables_in_parts(await sv.current_tables(session, snapshot), + [x for x in parts if x is not gone], also=parsed) + status = sv.retired_status(sv.snapshot_manifest(snapshot)) + except sv.PartProblem as e: + if e.code == "part_missing": + raise ImportRefused(409, "part_missing", e.message) from e + return _Acc(problems=[_structure_problem(e.code, e.message)]) + try: + plan = p.plan_accumulate(new_recipe=parsed, new_period=new_period, + period_source=(period.source if period is not None else "cells"), + staging_kind=staging.kind, current_mode=cur_mode, current_snapshot_kind=cur_kind, + parts=[x.info for x in parts], current_tables=tables, retired_status=status) + except ValueError as e: + return _Acc(problems=[_structure_problem("accumulate_rejected", str(e))]) + plan_json = _jsonable(plan) + rows = {t.name: int(t.rows) for t in ex.tables} + for entry in plan_json.get("parts") or []: + if entry.get("new"): + entry["file_name"], entry["rows"] = staging.file_name or "", dict(rows) + acc = _Acc(plan=plan_json) + if plan.action == "rejected": + if plan.overlaps: + acc.problems.append(_structure_problem("period_overlap", ra.overlap_message(plan.period, plan.overlaps))) + else: + acc.problems.append(_structure_problem("accumulate_rejected", plan.reason or "这份配方不能按期累积")) + return acc + if plan.action == "replace_period" and plan.replaces: + acc.prev_import_id = plan.replaces.get("import_id") + if len(plan.parts) < 2: + if plan.dropped: + # restart、模式切换后只剩本期：快照库就是本期的构建库，说明照单期写，另加「此前各期不在当前版本中」 + # （2.8 开头，评审一-m10） + acc.notes = _add_single_after_drop(run.notes) if run.notes is not None else None + return acc + by_id = {x.info.import_id: x for x in parts} + start, end = new_period or ("", "") + union_parts, triples, order = [], [], [] + try: + for entry in plan.parts: + if entry.get("new"): + union_parts.append(UnionPart(import_id=None, start=start, end=end, db_path=str(run.db), recipe=parsed, + db_sha256=run.db_sha256 or "", table_hashes=dict(ex.table_hashes))) + triples.append((start, end, build_id)) + order.append(None) + else: + x = by_id[entry["import_id"]] + union_parts.append(x.union_part()) + triples.append((x.info.start, x.info.end, x.info.build_id)) + order.append(x.info.import_id) + except sv.PartProblem as e: + acc.problems.append(_structure_problem(e.code, e.message)) + return acc + retired = ra.retired_columns(plan, tables) + uid = ra.union_id(staging.source_id, triples, recipe_sha, retired=retired) + out = table_versions.union_trial_path(run.db) + try: + report = await _in_slot(p.materialize_union, out, target=parsed, parts=union_parts, retired=retired) + except ra.UnionError as e: + if e.code == "part_missing": + raise ImportRefused(409, "part_missing", str(e)) from e + acc.problems.append(_structure_problem(e.code, str(e))) + return acc + acc.union_checks = [_jsonable(c) for c in report.checks] + plan_json["union"] = {"union_id": uid, "db_sha256": report.db_sha256, "rows": dict(report.rows)} + if not report.ok: + details = [d for c in report.checks if c.status != "passed" for d in (c.details or [c.title])] + acc.problems.append(_structure_problem("union_failed", "累积后的数据没有通过结构核对：" + + "；".join(details[:3]))) + return acc + note_parts = [] + for entry in plan.parts: + if entry.get("new"): + note_parts.append(_union_note_new(parsed, ex, run, start, end, [])) + else: + note_parts.append(await _in_slot(sv.note_part, by_id[entry["import_id"]])) + acc.notes = p.build_union_notes( + parsed, note_parts, union_tables=report.tables, null_counts=report.null_counts, + part_null_counts=report.part_null_counts, added=report.added, retired=report.retired, + label_sets=plan.label_sets, gaps=bool(plan.gaps), dropped=False) + acc.union = {"path": str(out), "union_id": uid, "db_sha256": report.db_sha256, + "options": ra.union_options(triples, recipe_sha, retired=retired), "report": _jsonable(report), + "imports": order} + return acc + + +def _replaced_part(parts: list[Any], new_period: tuple[str, str] | None, staging_kind: str) -> Any | None: + """当前各期里本期要替换的那一期（与 recipe_accumulate.plan_accumulate 的判定一致）：修改配方、不换文件（redraft） + 时是最晚一期；否则是统计期起止与本期都相同的那一期。没有就 None（追加、部分重叠拒收）。""" + if not parts: + return None + if staging_kind == "redraft": + return max(parts, key=lambda x: (x.info.start or "", x.info.end or "", x.info.seq or 0)) + if new_period is None: + return None + return next((x for x in parts if (x.info.start, x.info.end) == tuple(new_period)), None) + + +def _add_single_after_drop(notes: SchemaNotes) -> SchemaNotes: + from app.data.recipe_notes import add_single_after_drop + + try: + return add_single_after_drop(notes) + except Exception: # noqa: BLE001 - 假说明（测试）没有片段时照原样 + return notes + + +async def _prev_of(session: AsyncSession, cx: _Context, import_id: str | None) -> tuple[dict[str, Any] | None, + str | None]: + """差异卡、确认项的上一期：给了导入 id（替换该期）就取那一期的导入清单，否则取现行导入的。""" + if not import_id or cx.imp is None or import_id == cx.imp.id: + return cx.prev, (cx.imp.file_name if cx.imp is not None else None) + imp = await session.get(TableImport, import_id) + doc = await _prev_doc(session, cx.source, imp) + return doc, (imp.file_name if imp is not None else None) + + +async def _prior_acceptances(session: AsyncSession, source_id: str, build_id: str) -> list[dict[str, Any]]: + """同源、build_id 相同的导入记录以前的接受（第 5 节）：只读显示「上次接受的理由」，不预填。 + + 已作废（revoked 非空）的导入不算：那条理由已经被人判定为不成立，再摆在回执旁边当「上次接受的理由」，等于引导 + 用户照着再接受一次（H3）；界面的 PriorAcceptance 也没有「已作废」的标记，用户分辨不出来（评审意见）。""" + rows = (await session.execute( + select(TableImport).where(TableImport.source_id == source_id, TableImport.build_id == build_id) + .order_by(TableImport.seq.desc()))).scalars() + out: list[dict[str, Any]] = [] + for imp in rows: + if imp.revoked: + continue + for rec in [*(imp.overrides or []), *(imp.waivers or [])]: + if isinstance(rec, dict) and rec.get("check_id"): + out.append({"check_id": rec.get("check_id"), "reason": rec.get("reason"), + "signed_by": rec.get("signed_by"), "at": rec.get("at"), "import_seq": imp.seq}) + return out + + +def _same_candidate(cx: _Context, acc: _Acc | None) -> str | None: + """「与某次导入完全相同」看哪一期：替换该期时是被替换的那一期；当前版本只有一期时是它（期 2 的写法）。""" + if acc is not None and acc.plan is not None and acc.plan.get("action") == "replace_period": + return (acc.plan.get("replaces") or {}).get("import_id") + if cx.snapshot is not None and len(cx.snapshot.imports or []) == 1 and cx.imp is not None: + return cx.imp.id + return None + + async def run_trial(session: AsyncSession, staging: ImportStaging, *, context_inputs: dict[str, PeriodInput] | None, signed_by: str | None) -> ImportStaging: - """POST imports/{id}/trial：执行 → 核对 → 说明预览 → 确认项 →（源有当前版本时）差异卡。拒收也照常返回， - 看 trial.status。试运行库写好后删掉同一暂存区旧的试运行库。""" + """POST imports/{id}/trial：执行 → 核对 → 说明预览 →（按期累积时）累积计划、物化并集 → 确认项 →（源有当前 + 版本时）差异卡。拒收也照常返回，看 trial.status。试运行库写好后删掉同一暂存区旧的试运行库。""" p = pipeline() require_open(staging) if not isinstance(staging.recipe, dict): raise ImportRefused(409, "recipe_invalid", "还没有可用的配方，请先在配方面板中填写") + # 试运行要写试运行库（和并集文件）：只读进程不写版本存储（第 8 节遗留项 5） + table_versions.require_store_writable() ctx = copy.deepcopy(staging.context_inputs or {}) if context_inputs: if not _context_allowed(staging): @@ -1363,7 +2187,8 @@ async def run_trial(session: AsyncSession, staging: ImportStaging, *, "signed_by": (v.signed_by or signed_by or None)} for key, v in context_inputs.items()} cx = await _context(session, staging) base_sha = cx.recipe.recipe_sha256 if cx.recipe is not None and staging.kind == "reupload" else None - snap = _snap(staging, base_sha=base_sha) + base_recipe = cx.recipe.recipe if cx.recipe is not None and staging.kind in ("reupload", "redraft") else None + snap = _snap(staging, base_sha=base_sha, base_recipe=base_recipe) snap.context = ctx work = _full_form(staging.recipe) facts = _facts_from(staging.facts) @@ -1381,7 +2206,7 @@ async def run_trial(session: AsyncSession, staging: ImportStaging, *, except UnsupportedTable as e: raise ImportRefused(400, "file_unreadable", str(e) or "无法读取这个文件，请确认是未加密的 Excel 文件") from e # 从这里到提交之间出错（期间暂存区结束、差异卡或确认项报错、提交失败）：这次新写的试运行库没有记录 - # 指着，当场删掉。路径与旧的相同（同一个键重跑）时不删，那是暂存区记着的库 + # 指着，当场删掉（连同附属的并集文件）。路径与旧的相同（同一个键重跑）时不删，那是暂存区记着的库 try: await _still_open(session, staging) if run.extraction is None: @@ -1389,34 +2214,65 @@ async def run_trial(session: AsyncSession, staging: ImportStaging, *, ex = run.extraction notes_problems = list(run.notes.problems) if run.notes is not None else [] problems_out = [_jsonable(x) for x in ex.problems] + status = trial_status(ex.problems, run.checks, notes_problems) + inputs = _build_inputs(ex) + # 执行器语义标记（table_versions.recipe_engine_semantics）：只有受修补 3 影响的配方（列表跳过空行）才有， + # 这类配方的构建因此换 id，不会复用修补前的执行器建的旧库；提交时按同一份配方重算，与这里一致 + build_id = table_versions.recipe_build_id(staging.source_id, staging.raw_sha256, recipe_sha, inputs, + semantics=table_versions.recipe_engine_semantics(parsed)) + acc: _Acc | None = None + notes = run.notes + if status in _ACTIVE_TRIAL: + acc = await _accumulate_step(session, p, staging, cx, parsed, recipe_sha, run, build_id) + if acc is not None: + if acc.notes is not None: + notes = acc.notes + notes_problems = list(acc.notes.problems) + if acc.problems: + problems_out += acc.problems + status = "rejected" + elif notes_problems: + status = "rejected" problems_out += [{"code": "notes_invalid", "category": "recipe", "message": f"说明没有通过检查：{m}", "cells": [], "model_message": "", "fix": None} for m in notes_problems] checks_json = [_jsonable(c) for c in run.checks] - status = trial_status(ex.problems, run.checks, notes_problems) full = _receipt_full(ex) ex_dict = _jsonable(ex) + plan_json = acc.plan if acc is not None else None + prev, prev_name = await _prev_of(session, cx, acc.prev_import_id if acc is not None else None) diff: list[Any] | None = None items: list[ConfirmItem] = [] # 差异卡和确认项只在能提交的试运行（passed / needs_decision）上算。拒收时执行器在结构问题上停下， # 表、格子账可能只写了一半，拿它比会冒出「行数 31 → 0」这类假变化，结构上的变化已经作为拒收问题 # 报了（7.6）；needs_input 缺统计期，录入后再试运行就有。这两种状态 diff 为 null if status in _ACTIVE_TRIAL: - if cx.prev is not None: - prev_name = cx.imp.file_name if cx.imp is not None else None + if prev is not None: cur = {"receipt": full, "checks": checks_json, "period": full.get("period"), "recipe_sha256": recipe_sha} - changed = cx.recipe is None or cx.recipe.recipe_sha256 != recipe_sha - diff = list(p.diff_reports(cx.prev, cur, prev_file_name=prev_name, + # 配方变没变按上一期自己的配方比：按期累积时上一期的配方不一定是现行配方（移除之后剩下的期） + prev_sha = ((prev.get("recipe") or {}).get("sha256") if isinstance(prev.get("recipe"), dict) + else None) or (cx.recipe.recipe_sha256 if cx.recipe is not None else None) + kw: dict[str, Any] = {"accumulate": plan_json} if plan_json is not None else {} + diff = list(p.diff_reports(prev, cur, prev_file_name=prev_name, cur_file_name=staging.file_name or "", recipe=parsed, - recipe_changed=changed)) - items = list(p.confirm_items(_confirm_context(staging, cx, parsed, ex_dict, run.checks, diff))) - inputs = _build_inputs(ex) - build_id = table_versions.recipe_build_id(staging.source_id, staging.raw_sha256, recipe_sha, inputs) + recipe_changed=prev_sha != recipe_sha, **kw)) + items = list(p.confirm_items(_confirm_context(staging, cx, parsed, ex_dict, run.checks, diff, prev=prev, + accumulate=plan_json))) same = None - if cx.imp is not None and cx.imp.build_id == build_id: - same = {"id": cx.imp.id, "seq": cx.imp.seq} - notes_view = ({t: {"comment": n.comment, "columns": dict(n.columns)} for t, n in run.notes.tables.items()} - if run.notes is not None else {}) + cand = _same_candidate(cx, acc) + if cand is not None: + cand_imp = cx.imp if cx.imp is not None and cx.imp.id == cand else await session.get(TableImport, cand) + if cand_imp is not None and cand_imp.build_id == build_id: + same = {"id": cand_imp.id, "seq": cand_imp.seq} + notes_view = ({t: {"comment": n.comment, "columns": dict(n.columns)} for t, n in notes.tables.items()} + if notes is not None else {}) + compare = None + if (p.compare_recipes is not None and cx.recipe is not None and staging.kind in ("reupload", "redraft") + and cx.recipe.recipe_sha256 != recipe_sha): + try: + compare = p.compare_recipes(cx.recipe.recipe, parsed, plan=plan_json) + except Exception as e: # noqa: BLE001 - 对照只是辅助信息 + logger.warning("compare_recipes failed: %s", type(e).__name__) base_snapshot = cx.source.current_snapshot_id if cx.source is not None else None staging.trial = { "trial_id": key[:16], "status": status, "receipt": _receipt_view(full, run.db_sha256), @@ -1424,9 +2280,14 @@ async def run_trial(session: AsyncSession, staging: ImportStaging, *, "acceptable": acceptable_ids(run.checks), "notes": notes_view, "diff": [_jsonable(d) for d in diff] if diff is not None else None, "same_as_import": same, "base_snapshot_id": base_snapshot, - # 以下只给服务端：提交时还原 Extraction、重算必勾项、生成说明 + # 期 3（9.4 TrialOut） + "accumulate": plan_json, "recipe_compare": compare, + "union_checks": acc.union_checks if acc is not None else None, + "prior_acceptances": await _prior_acceptances(session, staging.source_id, build_id), + # 以下只给服务端：提交时还原 Extraction、重算必勾项、生成说明、发布并集 "extraction": ex_dict, "null_counts": run.null_counts, "inputs": inputs, "recipe_sha256": recipe_sha, - "build_id": build_id, + "build_id": build_id, "union": acc.union if acc is not None and status in _ACTIVE_TRIAL else None, + "prev_import_id": acc.prev_import_id if acc is not None else None, } old = staging.trial_path staging.context_inputs = ctx @@ -1499,7 +2360,8 @@ async def _back_to_drafting(session: AsyncSession, staging_id: str) -> None: def _manifest(*, staging_id: str, kind: str, source_id: str, info: table_versions.PublishInfo, trial_sha: str | None, ex_dict: dict[str, Any], file: dict[str, Any], recipe_meta: dict[str, Any], checks: list[dict[str, Any]], overrides: list[dict[str, Any]], waivers: list[dict[str, Any]], confirmed: list[dict[str, Any]], - notes: SchemaNotes, ai: dict[str, Any], signed_by: str | None) -> dict[str, Any]: + notes: SchemaNotes, ai: dict[str, Any], signed_by: str | None, + edits: list[dict[str, Any]] | None = None) -> dict[str, Any]: receipt = copy.deepcopy(ex_dict) receipt.pop("problems", None) return { @@ -1513,20 +2375,52 @@ def _manifest(*, staging_id: str, kind: str, source_id: str, info: table_version "period": copy.deepcopy(ex_dict.get("period")), "checks": checks, "acceptances": {"overrides": overrides, "waivers": waivers}, "confirmations": confirmed, + # receipt 照 asdict(Extraction) 去掉 problems：期 3 的 rows_excluded（排除的行，H2）随之写进来，只取本次 + # 试运行的 Extraction，不取复用构建登记的回执（那里没有这个字段，第 8 节遗留项 4） "receipt": receipt, "notes": {"templates": dict(notes.templates), "rendered": {t: {"comment": n.comment, "columns": dict(n.columns)} for t, n in notes.tables.items()}}, "ai": ai, + # 期 3：修复、框选的修改记录（含已被整份替换覆盖的，作为历史；不带撤销用的修改前起点） + "edits": copy.deepcopy(list(edits or [])), "signed_by": {"name": signed_by, "verified": False}, "staging_id": staging_id, "kind": kind, "created_at": _now_iso(), } +#: 发布原语的错误码 → 接口的状态和机读码（10.2 的错误映射表） +_PUBLISH_STATUS = { + "trial_missing": (409, "trial_required"), "trial_tampered": (409, "trial_tampered"), + "build_conflict": (409, "build_conflict"), "base_changed": (409, "base_changed"), + "part_missing": (409, "part_missing"), "part_tampered": (409, "part_tampered"), + "raw_missing": (409, "raw_missing"), "store_unavailable": (503, "store_unavailable"), +} + + +def _publish_refused(e: table_versions.PublishError) -> ImportRefused: + status, code = _PUBLISH_STATUS.get(e.code or "", (500, "publish_failed")) + return ImportRefused(status, code, str(e) or "导入失败，当前版本未受影响") + + +def _file_sha(path: Path) -> str | None: + try: + return raw_store.sha256_file(path) if path.is_file() and path.stat().st_size > 0 else None + except OSError: + return None + + async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial_id: str, confirmations: list[str], acceptances: list[Acceptance], signed_by: str | None) -> CommitResult: """POST imports/{id}/commit（7.8）：预检 → （与现行导入相同时）不新建记录 → 发布构建、写配方、导入记录、 - 快照、清单，切指针。失败时指针不动，暂存区退回 drafting（试运行库已被发布流程消耗）。""" + 快照、清单，切指针。失败时指针不动，暂存区退回 drafting（试运行库已被发布流程消耗）。 + + 期 3 的三条路径（12.6）： + - 结果只有一期、快照库就是本期的构建库（替换模式，按期累积的 first / restart / 替换唯一的一期）：照期 2 走 + publish_build，带 snapshot_fields（mode、目标配方：累积快照的 id 要用）； + - 结果有两期及以上：试运行时已物化好并集，这里走 publish_snapshot（本期构建、并集都 link，写导入清单和快照清单）； + - 与当前版本里的某一期完全相同（同一构建、同一配方、同一文件名、没有接受、统计期不是人工录入）：不新建任何记录。 + 按期累积时这一期是被替换的那一期（替换该期而构建相同）。""" p = pipeline() require_open(staging) trial = staging.trial or {} @@ -1547,12 +2441,14 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial raise ImportRefused(409, "trial_required", _TRIAL_STALE) if trial.get("status") not in _ACTIVE_TRIAL: raise ImportRefused(409, "trial_not_passed", "试运行没有通过（拒收或需要录入统计期），不能提交") + # 发布会消耗试运行库：只读进程在动任何东西之前就停下（503），别把用户的试运行白白删掉 + table_versions.require_store_writable() cx = await _context(session, staging) source = cx.source base = staging.base_snapshot_id # 先看当前版本有没有被换掉：换了的话上一期回执也换了，按它重算的确认项没有意义（这里只是快速预检， - # 最终判定在 publish_build 的锁内） + # 最终判定在 publish_build / publish_snapshot 的锁内） if source is not None and source.current_snapshot_id != base: raise ImportRefused(409, "base_changed", _BASE_CHANGED) if source is None: @@ -1562,7 +2458,11 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial raise ImportRefused(409, "name_taken", f"名称「{staging.source_name}」已被占用，请放弃这次导入后换一个名称") ex_dict = trial.get("extraction") or {} checks = [check_from_dict(c) for c in trial.get("checks") or []] - items = list(p.confirm_items(_confirm_context(staging, cx, parsed, ex_dict, checks, trial.get("diff")))) + plan = trial.get("accumulate") if isinstance(trial.get("accumulate"), dict) else None + union_info = trial.get("union") if isinstance(trial.get("union"), dict) else None + prev, _prev_name = await _prev_of(session, cx, trial.get("prev_import_id")) + items = list(p.confirm_items(_confirm_context(staging, cx, parsed, ex_dict, checks, trial.get("diff"), prev=prev, + accumulate=plan))) shown = {str(i.get("id")) for i in trial.get("confirm_items") or [] if isinstance(i, dict)} if any(i.required and i.id not in shown for i in items): # 服务端此刻要求的确认项比试运行时给界面的多（配方来源、数据源的遮罩设置等在试运行之后变了）：界面上的 @@ -1595,14 +2495,26 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial if not usable: await _reopen_drafting(session, staging) raise ImportRefused(409, "trial_required", "试运行的数据文件已不在，请重新试运行") + union_path: Path | None = None + if union_info is not None: + union_path = Path(str(union_info.get("path") or "")) + got = await asyncio.to_thread(_file_sha, union_path) + if got is None: + await _reopen_drafting(session, staging) + raise ImportRefused(409, "trial_required", "试运行时生成的累积数据文件已不在，请重新试运行") + if got != union_info.get("db_sha256"): + raise ImportRefused(409, "union_tampered", "试运行之后累积数据文件被改动过，请重新试运行") human = bool(_build_inputs(ex_dict)) same = trial.get("same_as_import") or None - if (same and not given and not human and cx.imp is not None and cx.imp.id == same.get("id") - and cx.imp.file_name == staging.file_name and source is not None): + same_imp = await session.get(TableImport, str(same.get("id"))) if same and same.get("id") else None + current_ids = {str(x) for x in (cx.snapshot.imports or [])} if cx.snapshot is not None else set() + if (same_imp is not None and not given and not human and same_imp.id in current_ids + and same_imp.file_name == staging.file_name and source is not None and cx.recipe is not None + and cx.recipe.recipe_sha256 == recipe_sha): # 与现行导入完全相同：不新建任何记录（final.md 2.6 第 5 条）。结束暂存区也在锁内、重读指针之后： # 预检之后别的提交切走了指针的话，这里返回的「现行导入」已经不是现行的了 - imp_id, imp_build, source_id = cx.imp.id, cx.imp.build_id, source.id + imp_id, imp_build, source_id = same_imp.id, same_imp.build_id, source.id async with table_versions.store_lock(): cur = (await session.execute( select(DataSource.current_snapshot_id, DataSource.current_recipe_id) @@ -1615,16 +2527,33 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial await session.commit() await _remove_trial(old) return CommitResult(import_id=imp_id, snapshot_id=cur[0] or "", build_id=imp_build, - recipe_id=cur[1] or "", build_reused=True, unchanged=True, source_id=source_id) + recipe_id=cur[1] or "", build_reused=True, unchanged=True, source_id=source_id, + parts=max(len(current_ids), 1)) ex = extraction_from_dict(ex_dict) who = signed_by or staging.signed_by acc = [Acceptance(check_id=a.check_id, reason=a.reason.strip(), signed_by=who) for a in given.values()] - notes = p.build_notes(parsed, ex, checks, acc, null_counts=trial.get("null_counts")) - if notes.problems: - raise ImportRefused(422, "notes_invalid", "说明没有通过检查：" + "；".join(notes.problems[:5])) + # 这一期自己的说明（带实际的接受）进它的导入清单：版本页逐期显示「改写后的那一句」从这里取（第 5 节）。 + # 快照冻结的说明在多期时另按各期最弱的结论合并（2.8），只剩本期时就是它 + period_notes = p.build_notes(parsed, ex, checks, acc, null_counts=trial.get("null_counts")) + parts: list[Any] = [] + if union_info is not None: + from app.data import source_versions as sv - # ---- 以下到 publish_build 之前，不改会话里的任何对象（7.2 第 1 条最后一项） + try: + parts = await sv.load_parts(session, cx.snapshot) + except sv.PartProblem as e: + raise ImportRefused(409, e.code, e.message) from e + notes = await _union_notes_for_commit(p, parsed, ex, checks, acc, trial, union_info, plan or {}, parts) + else: + notes = period_notes + if plan is not None and plan.get("dropped"): + notes = _add_single_after_drop(notes) + problems = list(notes.problems) + [m for m in period_notes.problems if m not in notes.problems] + if problems: + raise ImportRefused(422, "notes_invalid", "说明没有通过检查：" + "；".join(problems[:5])) + + # ---- 以下到 publish_build / publish_snapshot 之前，不改会话里的任何对象（7.2 第 1 条最后一项） at = _now_iso() by_id = {c.id: c for c in checks} overrides, waivers = [], [] @@ -1639,13 +2568,17 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial name, file_name, file_size, raw_sha = (staging.source_name, staging.file_name or "", int(staging.file_size or 0), staging.raw_sha256) base_origin = cx.recipe.origin if cx.recipe is not None else None - origin = effective_origin(staging, base_origin) + # 确认项按 effective_origin 算（上面已经算过）；入库只记四种来源之一（rules_redraft → rules / manual） + origin = stored_origin(effective_origin(staging, base_origin), staging) recipe_changed = cx.recipe is None or cx.recipe.recipe_sha256 != recipe_sha recipe_id = new_id() if recipe_changed else cx.recipe.id + recipe_seq_now = cx.recipe.seq if cx.recipe is not None and not recipe_changed else None canonical = p.canonical(parsed) - build_id = table_versions.recipe_build_id(source_id, raw_sha, recipe_sha, inputs) + semantics = table_versions.recipe_engine_semantics(parsed) + build_id = table_versions.recipe_build_id(source_id, raw_sha, recipe_sha, inputs, semantics=semantics) import_id = new_id() ai = {"usage": copy.deepcopy(staging.ai_usage or []), "consents": copy.deepcopy(staging.ai_consents or [])} + edits = [{k: v for k, v in e.items() if k not in EDIT_INTERNAL} for e in staging.edits or [] if isinstance(e, dict)] trial_sha = (trial.get("receipt") or {}).get("db_sha256") report = copy.deepcopy(ex_dict) report.pop("problems", None) @@ -1662,6 +2595,15 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial else: target = source manifest_ids: dict[str, str] = {} + activation = {"kind": "commit", "import_id": import_id, "signed_by": who} + + def import_manifest(info: table_versions.PublishInfo) -> dict[str, Any]: + return _manifest( + staging_id=staging_id, kind=kind, source_id=source_id, info=info, trial_sha=trial_sha, ex_dict=ex_dict, + file={"name": file_name, "size": file_size, "raw_sha256": raw_sha}, + recipe_meta={"id": recipe_id, "sha256": recipe_sha, "origin": origin, "canonical": canonical}, + checks=checks_full, overrides=overrides, waivers=waivers, confirmed=confirmed, + notes=notes if union_info is None else period_notes, ai=ai, signed_by=who, edits=edits) async def schema_cache_for(view: table_versions.SourceView, info: table_versions.PublishInfo) -> dict[str, Any]: from app.core import artifact_store @@ -1672,19 +2614,64 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial raise table_versions.PublishError("导入失败：说明没能写进表结构（" + "；".join(notes.problems[:3]) + "），当前版本未受影响") table_versions.check_probe(cache, info.report, source_id) - manifest = _manifest( - staging_id=staging_id, kind=kind, source_id=source_id, info=info, trial_sha=trial_sha, ex_dict=ex_dict, - file={"name": file_name, "size": file_size, "raw_sha256": raw_sha}, - recipe_meta={"id": recipe_id, "sha256": recipe_sha, "origin": origin, "canonical": canonical}, - checks=checks_full, overrides=overrides, waivers=waivers, confirmed=confirmed, notes=notes, ai=ai, - signed_by=who) - mid = await artifact_store.put_json(manifest, kind="import_manifest", + mid = await artifact_store.put_json(import_manifest(info), kind="import_manifest", meta={"source_id": source_id, "import_id": info.import_id}) manifest_ids["id"] = mid cache["import_manifests"] = [mid] return cache - async def before_commit(imp_id: str, snap_id: str) -> None: + async def union_cache_for(view: table_versions.SourceView, info: table_versions.SnapshotInfo) -> dict[str, Any]: + """多期快照冻结的表结构：探查并集库、写说明，再写本期的导入清单和快照清单（2.9）。""" + from app.core import artifact_store + from app.data import source_versions as sv + + union_report = info.union_report or union_info.get("report") or {} + probe = sv.union_table_report(union_report) + cache = await table_versions.introspect_view(view, probe) + cache = p.apply_notes(cache, notes) + if notes.problems: + raise table_versions.PublishError("导入失败：说明没能写进表结构（" + "；".join(notes.problems[:3]) + + "），当前版本未受影响") + table_versions.check_probe(cache, probe, source_id) + pinfo = table_versions.PublishInfo( + import_id=info.import_id or import_id, snapshot_id=info.snapshot_id, seq=int(info.seq or 0), + build_id=info.build_id or build_id, db_sha256=info.db_sha256 or "", build_reused=info.build_reused, + report=info.report, build_restored=info.build_restored) + mid = await artifact_store.put_json(import_manifest(pinfo), kind="import_manifest", + meta={"source_id": source_id, "import_id": pinfo.import_id}) + manifest_ids["id"] = mid + by_imp = {x.info.import_id: x for x in parts} + ordered: list[Any] = [] + for iid in union_info.get("imports") or []: + if iid is None: + ordered.append({"import_id": pinfo.import_id, "seq": pinfo.seq, + "period": {"start": period.get("start"), "end": period.get("end")}, + "build_id": pinfo.build_id, "db_sha256": pinfo.db_sha256, "recipe_id": recipe_id, + "recipe_sha256": recipe_sha, "manifest_artifact": mid, "file_name": file_name, + "raw_sha256": raw_sha}) + else: + ordered.append(by_imp[iid]) + seq = recipe_seq_now + if seq is None: + last = (await session.execute( + select(func.max(TableRecipe.seq)).where(TableRecipe.source_id == source_id))).scalar() or 0 + seq = last + 1 + doc = sv.manifest_doc( + source_id=source_id, snapshot_id=info.snapshot_id, action=str((plan or {}).get("action") or "append"), + recipe={"id": recipe_id, "seq": seq, "sha256": recipe_sha}, + union={"build_id": info.union_id, "db_sha256": info.union_db_sha256, + "table_hashes": dict(union_report.get("table_hashes") or {}), "union_ver": recipe_types.UNION_VER}, + parts=ordered, report=union_report, label_sets=list((plan or {}).get("label_sets") or []), + gaps=list((plan or {}).get("gaps") or []), replaced=(plan or {}).get("replaces"), removed=None, + notes=notes, signed_by=who) + smid = await artifact_store.put_json(doc, kind="snapshot_manifest", + meta={"source_id": source_id, "snapshot_id": info.snapshot_id}) + cache["import_manifests"] = [mid if iid is None else by_imp[iid].info.manifest_artifact + for iid in union_info.get("imports") or []] + cache["snapshot_manifest"] = smid + return cache + + async def before_commit(imp_id: str | None, snap_id: str) -> None: # 锁内、同一事务：用 select 重读，不信会话里缓存的对象 now = utcnow() cur = (await session.execute( @@ -1710,30 +2697,47 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial signed_by=who, ai_usage={"calls": ai["usage"], "consents": ai["consents"]}, staging_id=staging_id, created_at=now, activated_at=now)) target.current_recipe_id = recipe_id - imp = await session.get(TableImport, imp_id) + imp = await session.get(TableImport, imp_id) if imp_id else None if imp is not None: imp.manifest_artifact = manifest_ids.get("id") staging.committed_import_id = imp_id table_versions.close_staging(staging, "committed", now=now) try: + if union_info is not None: + imports = [import_id if iid is None else str(iid) for iid in union_info.get("imports") or []] + pub = await table_versions.publish_snapshot( + session, target, imports=imports, mode="accumulate", recipe_id=recipe_id, recipe_sha256=recipe_sha, + expected_current=base, + period_build=table_versions.PeriodBuild( + tmp_db=path, build_id=build_id, + options=table_versions.recipe_build_options(recipe_sha, inputs, semantics=semantics), + report=report, expected_sha256=trial_sha or ""), + new_import=table_versions.NewImport(import_id=import_id, raw_sha=raw_sha, file_name=file_name, + file_size=file_size, fields=import_fields), + union=table_versions.UnionBuild( + tmp_db=union_path, union_id=str(union_info["union_id"]), options=dict(union_info.get("options") or {}), + report=dict(union_info.get("report") or {}), expected_sha256=str(union_info.get("db_sha256") or "")), + schema_cache_for=union_cache_for, before_commit=before_commit, activation=activation) + return CommitResult(import_id=pub.import_id or import_id, snapshot_id=pub.snapshot_id, + build_id=pub.build_id or build_id, recipe_id=recipe_id, build_reused=pub.build_reused, + unchanged=False, source_id=source_id, build_restored=pub.build_restored, + parts=len(imports), snapshot_reused=bool(pub.snapshot_reused or pub.snapshot_revived)) result = await table_versions.publish_build( session, target, tmp_db=path, build_id=build_id, - build_options=table_versions.recipe_build_options(recipe_sha, inputs), + build_options=table_versions.recipe_build_options(recipe_sha, inputs, semantics=semantics), engine_ver=recipe_types.RECIPE_ENGINE_VER, report=report, raw=None, raw_sha=raw_sha, file_name=file_name, file_size=file_size, schema_cache_for=schema_cache_for, import_fields=import_fields, expected_db_sha256=trial_sha, before_commit=before_commit, - import_id=import_id) + import_id=import_id, + snapshot_fields={"mode": parsed.mode, "recipe_id": recipe_id, "recipe_sha256": recipe_sha}, + activation=activation) except ImportRefused: await _back_to_drafting(session, staging_id) raise except table_versions.PublishError as e: await _back_to_drafting(session, staging_id) - if e.code == "trial_missing": - raise ImportRefused(409, "trial_required", str(e)) from e - if e.code == "trial_tampered": - raise ImportRefused(409, "trial_tampered", str(e)) from e - raise ImportRefused(500, "publish_failed", str(e)) from e + raise _publish_refused(e) from e except IntegrityError as e: await _back_to_drafting(session, staging_id) if "data_sources.name" in str(e.orig if e.orig is not None else e): @@ -1742,8 +2746,49 @@ async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial except BaseException: await _back_to_drafting(session, staging_id) raise + finally: + if union_path is not None: + # publish_snapshot 无论成败都消耗并集的试运行文件；走 publish_build 的路径用不上它，也一并删掉 + await asyncio.to_thread(_drop_file, union_path) return CommitResult(import_id=result.import_id, snapshot_id=result.snapshot_id, build_id=result.build_id, - recipe_id=recipe_id, build_reused=result.build_reused, unchanged=False, source_id=source_id) + recipe_id=recipe_id, build_reused=result.build_reused, unchanged=False, source_id=source_id, + build_restored=result.build_restored, parts=1) + + +def _drop_file(path: Path) -> None: + if path.parent.name != table_versions.TRIALS_DIR: + return + for suffix in ("", "-journal", "-wal", "-shm"): + Path(f"{path}{suffix}").unlink(missing_ok=True) + + +async def _union_notes_for_commit(p: Pipeline, parsed: Recipe, ex: Extraction, checks: list[CheckResult], + acc: list[Acceptance], trial: dict[str, Any], union_info: dict[str, Any], + plan: dict[str, Any], parts: list[Any]) -> SchemaNotes: + """提交时冻结的并集说明：与试运行预览同一套合并规则，只是本期带上实际的接受（2.8）。""" + from app.data import source_versions as sv + + report = union_info.get("report") or {} + by_imp = {x.info.import_id: x for x in parts} + period = (trial.get("extraction") or {}).get("period") or {} + start, end = str(period.get("start") or ""), str(period.get("end") or "") + run = _TrialRun(parsed=parsed, problems=[], extraction=ex, checks=checks, null_counts=trial.get("null_counts")) + note_parts = [] + for iid in union_info.get("imports") or []: + if iid is None: + note_parts.append(_union_note_new(parsed, ex, run, start, end, acc)) + else: + x = by_imp.get(iid) + if x is None: + raise ImportRefused(409, "base_changed", _BASE_CHANGED) + note_parts.append(await _in_slot(sv.note_part, x)) + tables = {str(k): [ColumnOut(**_known(ColumnOut, c)) for c in v or [] if isinstance(c, dict)] + for k, v in (report.get("tables") or {}).items()} + return p.build_union_notes( + parsed, note_parts, union_tables=tables, null_counts=report.get("null_counts"), + part_null_counts=report.get("part_null_counts"), added=list(report.get("added") or []), + retired=list(report.get("retired") or []), label_sets=list(plan.get("label_sets") or []), + gaps=bool(plan.get("gaps")), dropped=False) # ========================================================================== @@ -1763,8 +2808,9 @@ async def discard_staging(session: AsyncSession, staging: ImportStaging) -> None """DELETE imports/{id}：status → discarded、瘦身、删试运行库；原件已没有别的引用时当场删除。 进锁之后重读状态：排在提交后面等锁的放弃，拿到锁时暂存区可能已经提交了，不能把 committed 改写成 - discarded（SE-8）。""" + discarded（SE-8）。只读进程不放弃：放弃要删试运行库和原件（第 8 节遗留项 5）。""" require_open(staging) + table_versions.require_store_writable() async with table_versions.store_lock(): await session.refresh(staging, ["status"]) require_open(staging) @@ -1847,6 +2893,10 @@ def staging_out(staging: ImportStaging, source: DataSource | None, *, marks = _regions(staging.trial.get("extraction")) if units is None: units = tuple(recipe_parsers.UNITS) + fixes, fix_where = proposals_for(pipeline(), staging) + trial, draft_problems, recipe_problems = _with_fix_ids(fixes, fix_where, trial, staging) + edits = [e for e in staging.edits or [] if isinstance(e, dict)] + open_ = table_versions.staging_is_open(staging) return { "id": staging.id, "kind": staging.kind, "status": staging.status, "source": {"id": staging.source_id, "name": source.name if source is not None else staging.source_name, @@ -1862,10 +2912,10 @@ def staging_out(staging: ImportStaging, source: DataSource | None, *, "provider": ai.provider if ai is not None else ""}, "ai_consents": list(staging.ai_consents or []), "recipe": recipe, "recipe_origin": staging.recipe_origin, - "recipe_problems": list(staging.recipe_problems or []), + "recipe_problems": recipe_problems, "answers": dict(staging.answers or {}), "cards": list(staging.cards or []), "questions": list(staging.questions or []), - "draft_problems": list(staging.draft_problems or []), "draft_partial": bool(staging.draft_partial), + "draft_problems": draft_problems, "draft_partial": bool(staging.draft_partial), "candidates": dict(facts.get("candidates") or {}), "units": list(units), "marks": marks, @@ -1875,14 +2925,47 @@ def staging_out(staging: ImportStaging, source: DataSource | None, *, "committed_import_id": staging.committed_import_id, "created_at": _iso(staging.created_at), "updated_at": _iso(staging.updated_at), "expires_at": _iso(staging.expires_at), + # 期 3（9.4 StagingOut） + "fixes": [_jsonable(f) for f in fixes], + "recipe_sha256": staging.recipe_sha256, + "edits": [_edit_out(e, last=i == len(edits) - 1, open_=open_) for i, e in enumerate(edits)], + "answers_dropped": list(staging.answers_dropped or []), } +def _with_fix_ids(fixes: list[Any], where: str, trial: Any, staging: ImportStaging + ) -> tuple[Any, list[dict[str, Any]], list[dict[str, Any]]]: + """按提议的 anchor 把 fix_ids 写到对应的问题上（3.1，评审丙-B2）：界面只按 fix_ids 放按钮，不靠 code 加坐标去猜 + （title_not_found 没有坐标，两个分段同时找不到标题时只有这样才分得清）。没有提议的问题 fix_ids 为 []。 + 返回 (trial, draft_problems, recipe_problems)，都是新的副本。""" + by_problem: dict[int, list[str]] = {} + by_recipe: dict[int, list[str]] = {} + for f in fixes: + anchor = getattr(f, "anchor", None) + index = getattr(anchor, "index", None) + if index is None: + continue + if getattr(anchor, "kind", None) == "problem": + by_problem.setdefault(int(index), []).append(f.id) + elif getattr(anchor, "kind", None) == "recipe_problem": + by_recipe.setdefault(int(index), []).append(f.id) + + def tag(items: Any, ids: dict[int, list[str]]) -> list[dict[str, Any]]: + return [{**x, "fix_ids": list(ids.get(i, []))} for i, x in enumerate(items or []) if isinstance(x, dict)] + + draft_problems = tag(staging.draft_problems, by_problem if where == "draft" else {}) + recipe_problems = tag(staging.recipe_problems, by_recipe) + if isinstance(trial, dict) and isinstance(trial.get("problems"), list): + trial = {**trial, "problems": tag(trial["problems"], by_problem if where == "trial" else {})} + return trial, draft_problems, recipe_problems + + __all__ = [ - "ACCEPT_REASON_MAX", "ANSWER_REASON_MAX", "CommitConflict", "CommitResult", "ImportRefused", "MANIFEST_FORMAT", - "Pipeline", "acceptable_ids", "ai_offered", "ai_preview", "answer", "check_from_dict", "commit_staging", - "consent_token", + "ACCEPT_REASON_MAX", "ANSWER_REASON_MAX", "CommitConflict", "CommitResult", "EditRequest", "ImportRefused", + "MANIFEST_FORMAT", "Pipeline", "acceptable_ids", "ai_offered", "ai_preview", "answer", "carry_answers", + "check_from_dict", "commit_staging", "consent_token", "edit_apply", "edit_preview", "edit_undo", "proposals_for", + "redraft_rules", "current_import", "current_recipe", "default_pipeline", "discard_staging", "effective_origin", "extraction_from_dict", "import_mode", "pipeline", "put_recipe", "redraft", "replay_answers", "reupload", - "run_ai_draft", "run_trial", "stage_upload", "staging_out", "trial_key", "trial_status", + "run_ai_draft", "run_trial", "stage_upload", "staging_out", "stored_origin", "trial_key", "trial_status", ] diff --git a/backend/app/data/recipe_notes.py b/backend/app/data/recipe_notes.py index e3931e42..b4464cf7 100644 --- a/backend/app/data/recipe_notes.py +++ b/backend/app/data/recipe_notes.py @@ -11,6 +11,10 @@ 原表头、坐标、区域外文字全文只进清单和界面，不进说明。 - **模板是常量，拼接只用 .format()**：模板放在 NOTE_TEXT / COLUMN_TEXT 里，{a} {b} 这类槽位由 _tok 填入引用； 这样模板本身经得起文案检查，填进去的名字也不会被当成格式串再解析一次。 + +期 3（P3-SPEC 2.8）：build_notes 顺带返回每张表说明的片段（SchemaNotes.fragments，NoteFragment 的 key 是片段族、 +subject 是关系 id 或表名），渲染结果与期 2 逐字相同；build_union_notes 对按期累积物化的快照库逐期调 build_notes， +按 (片段族, subject) 合并、取最弱：任一期没核对、或者还没确认接受，合并后的说法就不能比它强。 """ from __future__ import annotations @@ -33,12 +37,14 @@ from app.data.recipe_types import ( ListBlock, MeasuresSegment, NotComparable, + NoteFragment, Placeholder, Recipe, SchemaNotes, SheetRecipe, SumEq, TableNote, + UnionNotePart, derive_tables, prose_problems, render_prose, @@ -98,6 +104,36 @@ NOTE_TEXT = { "period_conflict_pending": "本期统计期的写法在原表或文件名中有冲突，尚未确认接受", "period_human": "本期统计期为人工录入", "outside_digits": "本期原表附有说明文字（可能涉及口径），内容见证据面板", + # ---- 期 3：按配方忽略的行和格（单期也用，P3-SPEC 3.2 ⑥） + "ignored_rows": "原表另有按配方忽略的行，内容见证据面板", + "ignored_cells": "原表另有按配方忽略的单元格，内容见证据面板", + # ---- 期 3：按期累积的并集说明（P3-SPEC 2.8）。各期合并时取最弱：部分期没核对、还没确认接受，都不能写成 + # 「已核对」「已由用户确认接受」；不含「一致」（CONSISTENT_KEYS 之外） + "accumulate": "本表按统计期累积多期数据，各期互不重叠：比较不同期时请按 {col} 筛选", + "accumulate_gap": "本表按统计期累积多期数据，各期互不重叠，但各期之间有空缺：比较不同期之前请先查询 {col} 的覆盖范围", + "single_after_drop": "当前版本只含最近一期的数据，此前各期不在当前版本中", + "table_partial_periods": "部分期没有这张表的数据", + "table_absent": "当前版本的数据中没有这张表的行", + "sum_eq_union_mixed": "{total} 与 {parts} 之和的关系在部分期的个别{unit}不成立或未能核对，详见导入清单：" + "不要据此推算，也不要把这几列相加", + "sum_eq_union_pending": "{total} 与 {parts} 之和的关系在部分期的个别{unit}不成立，尚未确认接受：" + "不要据此推算，也不要把这几列相加", + "sum_eq_union_partial": "{total} 与 {parts} 之和的关系只在部分期登记并核对，其余各期未核对，详见导入清单：" + "不要据此推算，也不要把这几列相加", + "sum_eq_members_vary": "{total} 等于各期登记的分项列之和，各期登记的分项列不同（某期没有的列为空值），导入时已逐期核对：" + "求合计只用 {total}，不要把这几列相加", + "total_union_unverified": "原表写明的合计，按原表保存的值存储。部分期的合计值未能核对，详见导入清单", + "period_conflict_union": "部分期的统计期写法在原表或文件名中有冲突，已由用户确认接受，详见导入清单", + "period_conflict_union_pending": "部分期的统计期写法在原表或文件名中有冲突，尚未确认接受", + "period_human_union": "部分期的统计期为人工录入", + "period_human_all": "各期的统计期均为人工录入", + "outside_digits_union": "部分期的原表附有说明文字（可能涉及口径），内容见证据面板", + "hidden_excluded_union": "部分期的原表有被隐藏的行，本表不含这些行", + "hidden_included_union": "部分期的原表有被隐藏的行，本表包含这些行", + "ignored_columns_union": "部分期的原表另有未导入的列", + "ignored_rows_union": "部分期的原表另有按配方忽略的行，内容见证据面板", + "ignored_cells_union": "部分期的原表另有按配方忽略的单元格，内容见证据面板", + "dim_values_vary": "各期的 {dim} 取值不完全相同：跨期比较前请先确认两期都有的取值", } #: 列说明的片段模板 @@ -108,6 +144,10 @@ COLUMN_TEXT = { "null_both": "空值表示原表为{meaning}占位符或空格，不代表零；求和时请同时统计非空个数", "date": "格式 YYYY-MM-DD", "date_year_from_period": "格式 YYYY-MM-DD，年份按统计期补全", + # ---- 期 3：并集里的列（P3-SPEC 2.8 第 4 步）。缺这一列的期是结构性空值，不能说成「原表为空格」 + "col_partial_periods": "部分期没有这一列的数据，为空值", + "col_absent": "当前版本的数据中没有这一列，为空值", + "col_retired": "这一列自某一期起不再导入，此后各期为空值", } #: 「逐日 / 逐行」「个别日期 / 个别行」：粒度只有一列日期时按日说，否则按行说 @@ -264,18 +304,23 @@ def _per_day(ctx: _Ctx, table: str) -> bool: return len(grain) == 1 and grain[0] in ctx.dates.get(table, set()) -def _grain_fragment(grain: list[str]) -> str | None: +def _frag(key: str, subject: str | None, text: str | None) -> NoteFragment | None: + """一个片段：key 是 NOTE_TEXT 的键（用户写的表说明是 note），subject 是关系 id 或表名。text 为空时 None。""" + return NoteFragment(key=key, subject=subject, template=text) if text else None + + +def _grain_fragment(table: str, grain: list[str]) -> NoteFragment | None: cols = [_col(g) for g in grain] if not cols: return None if len(cols) == 1: - return NOTE_TEXT["grain_1"].format(a=cols[0]) + return _frag("grain_1", table, NOTE_TEXT["grain_1"].format(a=cols[0])) if len(cols) == 2: - return NOTE_TEXT["grain_2"].format(a=cols[0], b=cols[1]) - return NOTE_TEXT["grain_n"].format(a=_LIST.join(cols[:-1]), b=cols[-1]) + return _frag("grain_2", table, NOTE_TEXT["grain_2"].format(a=cols[0], b=cols[1])) + return _frag("grain_n", table, NOTE_TEXT["grain_n"].format(a=_LIST.join(cols[:-1]), b=cols[-1])) -def _sum_eq_fragment(ctx: _Ctx, table: str, rel: SumEq) -> str | None: +def _sum_eq_fragment(ctx: _Ctx, table: str, rel: SumEq) -> NoteFragment | None: c = ctx.check(table, rel.id, f"关系「{rel.id}」") if c is None or _blocking(c): # 核对 SQL 出错时 status 也是 mismatch：不能当成「关系不成立」去写 @@ -283,19 +328,20 @@ def _sum_eq_fragment(ctx: _Ctx, table: str, rel: SumEq) -> str | None: per, unit = _PER[_per_day(ctx, table)] total = _col(rel.total) if c.status == "passed": - return NOTE_TEXT["sum_eq_passed"].format(total=total, parts=_join_and([_col(p) for p in rel.parts]), per=per) + return _frag("sum_eq_passed", rel.id, NOTE_TEXT["sum_eq_passed"].format( + total=total, parts=_join_and([_col(p) for p in rel.parts]), per=per)) parts = _LIST.join(_col(p) for p in rel.parts) if c.status == "mismatch": key = "sum_eq_mismatch_accepted" if rel.id in ctx.accepted else "sum_eq_mismatch_pending" - return NOTE_TEXT[key].format(total=total, parts=parts, unit=unit) + return _frag(key, rel.id, NOTE_TEXT[key].format(total=total, parts=parts, unit=unit)) if c.status == "unverifiable": key = "sum_eq_no_rows" if not c.checked and not c.unverifiable else "sum_eq_nulls" - return NOTE_TEXT[key].format(total=total, parts=parts, unit=unit) + return _frag(key, rel.id, NOTE_TEXT[key].format(total=total, parts=parts, unit=unit)) ctx.problems.append(f"表「{table}」：关系「{rel.id}」的核对状态「{c.status}」没有对应的说明") return None -def _not_comparable_fragment(ctx: _Ctx, table: str, rel: NotComparable) -> str | None: +def _not_comparable_fragment(ctx: _Ctx, table: str, rel: NotComparable) -> NoteFragment | None: c = ctx.check(table, rel.id, f"关系「{rel.id}」") if c is None or _blocking(c): return None @@ -305,49 +351,50 @@ def _not_comparable_fragment(ctx: _Ctx, table: str, rel: NotComparable) -> str | other = rel.b.table if table == rel.a.table else rel.a.table if c.failed <= 0: # 本期全部相等（或没有可比的分组）：不写「不等于」 - return NOTE_TEXT["not_comparable_equal"].format(other=_tab(other)) + return _frag("not_comparable_equal", rel.id, NOTE_TEXT["not_comparable_equal"].format(other=_tab(other))) dims = [g for g in ctx.grains.get(rel.a.table, []) if g != rel.by] if dims: - return NOTE_TEXT["not_comparable_unequal"].format( - other=_tab(other), dims=_LIST.join(_col(d) for d in dims), value=_col(rel.b.value)) - return NOTE_TEXT["not_comparable_unequal_flat"].format( - other=_tab(other), a_value=_col(rel.a.value), by=_col(rel.by), value=_col(rel.b.value)) + return _frag("not_comparable_unequal", rel.id, NOTE_TEXT["not_comparable_unequal"].format( + other=_tab(other), dims=_LIST.join(_col(d) for d in dims), value=_col(rel.b.value))) + return _frag("not_comparable_unequal_flat", rel.id, NOTE_TEXT["not_comparable_unequal_flat"].format( + other=_tab(other), a_value=_col(rel.a.value), by=_col(rel.by), value=_col(rel.b.value))) -def _dim_fragments(w: _Writers) -> list[str]: - out: list[str] = [] +def _add_unique(out: list[NoteFragment], frag: NoteFragment | None) -> None: + """同一句只写一次（几个分段写进同一张表时，它们的提示往往相同）。""" + if frag is not None and all(f.template != frag.template for f in out): + out.append(frag) + + +def _dim_fragments(w: _Writers) -> list[NoteFragment]: + out: list[NoteFragment] = [] for seg in w.dims: if seg.dim.parser != "hour_range": continue roles = {role: name for name, role in seg.dim.derive.items()} if "start" in roles and "end" in roles: - text = NOTE_TEXT["hour_range"].format(dim=_col(seg.dim.name), start=_col(roles["start"]), - end=_col(roles["end"])) + frag = _frag("hour_range", seg.dim.name, NOTE_TEXT["hour_range"].format( + dim=_col(seg.dim.name), start=_col(roles["start"]), end=_col(roles["end"]))) else: - text = NOTE_TEXT["hour_range_plain"].format(dim=_col(seg.dim.name)) - if text not in out: - out.append(text) + frag = _frag("hour_range_plain", seg.dim.name, NOTE_TEXT["hour_range_plain"].format(dim=_col(seg.dim.name))) + _add_unique(out, frag) for seg in w.dims: for name in sorted(seg.const): - text = NOTE_TEXT["const"].format(col=_col(name)) - if text not in out: - out.append(text) + _add_unique(out, _frag("const", name, NOTE_TEXT["const"].format(col=_col(name)))) return out -def _base_total_fragments(w: _Writers) -> list[str]: +def _base_total_fragments(w: _Writers) -> list[NoteFragment]: """明细表上的提示：原表合计另存在哪、或者没导入。""" - out: list[str] = [] + out: list[NoteFragment] = [] kept = [s.keep_as.table for s in w.verified_by if s.keep_as is not None] kept += [b.rows.total_row.keep_as for b in w.lists if b.rows.total_row is not None and b.rows.total_row.keep_as] for t in kept: - text = NOTE_TEXT["total_kept"].format(table=_tab(t)) - if text not in out: - out.append(text) + _add_unique(out, _frag("total_kept", t, NOTE_TEXT["total_kept"].format(table=_tab(t)))) dropped = any(s.keep_as is None for s in w.verified_by) or any( b.rows.total_row is not None and not b.rows.total_row.keep_as for b in w.lists) if dropped: - out.append(NOTE_TEXT["total_not_kept"]) + out.append(NoteFragment("total_not_kept", None, NOTE_TEXT["total_not_kept"])) return out @@ -370,8 +417,8 @@ def _total_key(checks: list[CheckResult], by_reason: dict[str, str], mixed: str, return mixed if reasons else None -def _kept_total_fragments(ctx: _Ctx, table: str, w: _Writers) -> list[str]: - out: list[str] = [] +def _kept_total_fragments(ctx: _Ctx, table: str, w: _Writers) -> list[NoteFragment]: + out: list[NoteFragment] = [] if w.kept_from: # 一张合计表可以由几个 derived 段写入（日间、夜间各有合计行）：所有段的 K、G 一起选一条模板 ks = [ctx.check(table, ctx.plan.k.get(s.id), f"分段「{s.id}」的合计") for s in w.kept_from] @@ -384,11 +431,10 @@ def _kept_total_fragments(ctx: _Ctx, table: str, w: _Writers) -> list[str]: if key is None: ctx.problems.append(f"表「{table}」：合计核对 {_ids(ks + gs)} 的结果没有对应的说明") return out - out.append(NOTE_TEXT[key]) + out.append(NoteFragment(key, table, NOTE_TEXT[key])) for seg in w.kept_from: - text = NOTE_TEXT["total_overlap"].format(dim=_col(seg.keep_as.dim), base=_tab(seg.verify.against_table)) - if text not in out: - out.append(text) + _add_unique(out, _frag("total_overlap", seg.verify.against_table, NOTE_TEXT["total_overlap"].format( + dim=_col(seg.keep_as.dim), base=_tab(seg.verify.against_table)))) for block in w.list_totals: t = ctx.check(table, ctx.plan.t.get(block.id), f"块「{block.id}」的合计行") if t is None or _blocking(t): @@ -399,12 +445,11 @@ def _kept_total_fragments(ctx: _Ctx, table: str, w: _Writers) -> list[str]: continue blank_cols = _uncovered_total_columns(ctx, block) if key == "list_total_passed" else [] if blank_cols: - text = NOTE_TEXT["list_total_passed_partial"].format( - cols=_LIST.join(_col(c) for c in blank_cols), base=_tab(block.table)) + frag = _frag("list_total_passed_partial", table, NOTE_TEXT["list_total_passed_partial"].format( + cols=_LIST.join(_col(c) for c in blank_cols), base=_tab(block.table))) else: - text = NOTE_TEXT[key].format(base=_tab(block.table)) - if text not in out: - out.append(text) + frag = _frag(key, table, NOTE_TEXT[key].format(base=_tab(block.table))) + _add_unique(out, frag) return out @@ -422,7 +467,7 @@ def _ids(checks: list[CheckResult]) -> str: return "、".join(c.id for c in checks) -def _hidden_fragments(ctx: _Ctx, table: str, w: _Writers) -> list[str]: +def _hidden_fragments(ctx: _Ctx, table: str, w: _Writers) -> list[NoteFragment]: excluded = included = False for sheet in w.sheets: actual = _actual_sheet(sheet, ctx.extraction) @@ -440,12 +485,41 @@ def _hidden_fragments(ctx: _Ctx, table: str, w: _Writers) -> list[str]: included |= not spans or any(a <= r <= b for a, b in spans for r in rows) out = [] if excluded: - out.append(NOTE_TEXT["hidden_excluded"]) + out.append(NoteFragment("hidden_excluded", None, NOTE_TEXT["hidden_excluded"])) if included: - out.append(NOTE_TEXT["hidden_included"]) + out.append(NoteFragment("hidden_included", None, NOTE_TEXT["hidden_included"])) return out +def _ignored_fragments(ctx: _Ctx, w: _Writers) -> list[NoteFragment]: + """按配方忽略的列、行、单元格（期 3 ⑥）：本期确实忽略了才写，内容只进证据面板，说明里不写原文。 + + - 列：期 2 的列表 extra_columns=ignore，加上期 3 的 ignore_columns（交叉表、列表），本期 ignored_columns 里 + 这个块有表头才写。期 2 的配方没有 ignore_columns，判断与期 2 完全相同； + - 行：rows_excluded 里 reason=ignored_rows、block 是写入本表的块； + - 单元格：rows_excluded 里 reason=ignored_outside、工作表是写入本表的工作表（区域外的格不属于任何块）。 + """ + out: list[NoteFragment] = [] + ignored = ctx.extraction.ignored_columns + blocks: list[Any] = [*w.lists, *w.crosstabs] + if any((getattr(b, "extra_columns", None) == "ignore" or b.ignore_columns) and ignored.get(b.id) for b in blocks): + out.append(NoteFragment("ignored_columns", None, NOTE_TEXT["ignored_columns"])) + excluded = [_excluded(e) for e in ctx.extraction.rows_excluded or []] + ids = {b.id for b in blocks} + if any(e["reason"] == "ignored_rows" and e["block"] in ids and e["rows"] for e in excluded): + out.append(NoteFragment("ignored_rows", None, NOTE_TEXT["ignored_rows"])) + sheets = {_actual_sheet(s, ctx.extraction) for s in w.sheets} + if any(e["reason"] == "ignored_outside" and e["sheet"] in sheets and e["rows"] for e in excluded): + out.append(NoteFragment("ignored_cells", None, NOTE_TEXT["ignored_cells"])) + return out + + +def _excluded(e: Any) -> dict[str, Any]: + """ExcludedRows 或它的 JSON 形状（暂存区、清单里存的是 asdict 之后的样子）→ 统一的 dict。""" + get = e.get if isinstance(e, dict) else (lambda k: getattr(e, k, None)) + return {"reason": get("reason"), "block": get("block"), "sheet": get("sheet"), "rows": get("rows") or []} + + def _rows_after_stop_tables(ctx: _Ctx) -> set[str]: """rows_after_stop 问题落在哪张表上：同一工作表上被空行停下的列表块；不止一个时取紧挨在这些行之上的那个。""" out: set[str] = set() @@ -456,8 +530,15 @@ def _rows_after_stop_tables(ctx: _Ctx) -> set[str]: if parsed is None or parsed[0] is None: continue sheet_name, first_row = parsed[0], parsed[2] - blocks = [b for s in ctx.recipe.sheets if _actual_sheet(s, ctx.extraction) == sheet_name - for b in s.blocks if isinstance(b, ListBlock) and b.rows.blank_rows == "stop"] + on_sheet = [b for s in ctx.recipe.sheets if _actual_sheet(s, ctx.extraction) == sheet_name + for b in s.blocks if isinstance(b, ListBlock)] + # 消息里写着「列表「块」」：恰好认出一个块就用它。跳过空行的列表按表下说明收尾时也报这个问题（WP-8）， + # 下面按 blank_rows=stop 找的办法找不到它 + named = [b for b in on_sheet if f"列表「{b.id}」" in p.message] + if len(named) == 1: + out.add(named[0].table) + continue + blocks = [b for b in on_sheet if b.rows.blank_rows == "stop"] if len(blocks) > 1: above = [(max(b for _, b in spans), blk) for blk in blocks if (spans := _lineage_spans(ctx.extraction, blk.table, sheet_name)) @@ -468,25 +549,26 @@ def _rows_after_stop_tables(ctx: _Ctx) -> set[str]: return out -def _period_fragments(ctx: _Ctx, w: _Writers) -> list[str]: +def _period_fragments(ctx: _Ctx, w: _Writers) -> list[NoteFragment]: if not any(s.context for s in w.sheets): return [] - out: list[str] = [] + out: list[NoteFragment] = [] # 按传进来的核对结果判（run_checks 把 C1、C2 原样排在最前），与试运行、提交时存下的那份一致 conflicts = [c for c in ctx.checks.values() if c.kind in ("context_agree", "filename_period") and c.status == "mismatch"] if conflicts: - accepted = all(c.id in ctx.accepted for c in conflicts) - out.append(NOTE_TEXT["period_conflict_accepted" if accepted else "period_conflict_pending"]) + key = "period_conflict_accepted" if all(c.id in ctx.accepted for c in conflicts) else "period_conflict_pending" + out.append(NoteFragment(key, None, NOTE_TEXT[key])) if ctx.extraction.period is not None and ctx.extraction.period.source == "human": - out.append(NOTE_TEXT["period_human"]) + out.append(NoteFragment("period_human", None, NOTE_TEXT["period_human"])) return out -def _table_template(ctx: _Ctx, table: str, after_stop: set[str]) -> str: +def _table_fragments(ctx: _Ctx, table: str, after_stop: set[str]) -> list[NoteFragment]: + """一张表说明的全部片段，按拼接顺序。顺序与期 2 的 _table_template 相同：渲染结果逐字不变。""" w = ctx.writers.get(table, _Writers()) spec = next((t for t in ctx.recipe.tables if t.name == table), None) - frags: list[str | None] = [_grain_fragment(ctx.grains.get(table, []))] + frags: list[NoteFragment | None] = [_grain_fragment(table, ctx.grains.get(table, []))] for rel in ctx.recipe.relations: if isinstance(rel, SumEq) and rel.table == table: frags.append(_sum_eq_fragment(ctx, table, rel)) @@ -497,17 +579,26 @@ def _table_template(ctx: _Ctx, table: str, after_stop: set[str]) -> str: frags += _base_total_fragments(w) frags += _kept_total_fragments(ctx, table, w) frags += _hidden_fragments(ctx, table, w) - if any(b.extra_columns == "ignore" and ctx.extraction.ignored_columns.get(b.id) for b in w.lists): - frags.append(NOTE_TEXT["ignored_columns"]) + frags += _ignored_fragments(ctx, w) if table in after_stop: - frags.append(NOTE_TEXT["rows_after_stop"]) + frags.append(NoteFragment("rows_after_stop", None, NOTE_TEXT["rows_after_stop"])) frags += _period_fragments(ctx, w) sheets = {_actual_sheet(s, ctx.extraction) for s in w.sheets} if any(o.kind == "text_digits" and o.sheet in sheets for o in ctx.extraction.outside_text): - frags.append(NOTE_TEXT["outside_digits"]) + frags.append(NoteFragment("outside_digits", None, NOTE_TEXT["outside_digits"])) if spec is not None and spec.note.strip(): - frags.append(spec.note.strip()) - return "。".join(f.strip().rstrip("。") for f in frags if f and f.strip().rstrip("。")) + # 用户写的表说明不是 NOTE_TEXT 里的模板，片段族记作 note + frags.append(NoteFragment("note", table, spec.note.strip())) + return [f for f in frags if f is not None and _clean(f.template)] + + +def _clean(text: str) -> str: + return text.strip().rstrip("。") + + +def _join(frags: list[NoteFragment]) -> str: + """片段用「。」拼接（每段去掉首尾空白和句末的「。」，空段跳过）。""" + return "。".join(_clean(f.template) for f in frags if _clean(f.template)) # ---------------------------------------------------------------- 列说明 @@ -519,21 +610,29 @@ def _occurred(placeholders: list[Placeholder], extraction: Extraction) -> list[P return [p for p in placeholders if canon(p.text) in seen] -def _null_fragment(w: _Writers, extraction: Extraction) -> str | None: +def _null_sources(w: _Writers, extraction: Extraction) -> tuple[set[str], bool]: + """本表空值的来源：本期出现过的占位符的含义集合，以及有没有块把空格存成空值。""" blocks: list[Any] = [*w.crosstabs, *w.lists] - marks = [p for b in blocks for p in _occurred(list(b.values.placeholders), extraction)] - blank = any(b.values.blank == "null" for b in blocks) - meanings = {p.meaning for p in marks} + meanings = {p.meaning for b in blocks for p in _occurred(list(b.values.placeholders), extraction)} + return meanings, any(b.values.blank == "null" for b in blocks) + + +def _null_text(meanings: set[str], blank: bool) -> str | None: + """占位符的含义不止一种（几期、几个块各说各的）时写「无数据或不适用」。""" meaning = next(iter(meanings)) if len(meanings) == 1 else _MEANING_BOTH - if marks and blank: + if meanings and blank: return COLUMN_TEXT["null_both"].format(meaning=meaning) - if marks: + if meanings: return COLUMN_TEXT["null_placeholder"].format(meaning=meaning) if blank: return COLUMN_TEXT["null_blank"] return None +def _null_fragment(w: _Writers, extraction: Extraction) -> str | None: + return _null_text(*_null_sources(w, extraction)) + + def _year_from_period(ctx: _Ctx, w: _Writers) -> bool: """轴的年份是不是从统计期补的：配了 year_from，而且本期表头不全是自带年份的日期格。""" for block in w.crosstabs: @@ -545,25 +644,34 @@ def _year_from_period(ctx: _Ctx, w: _Writers) -> bool: return False +def _date_key(ctx: _Ctx, w: _Writers, table: str, col: ColumnOut) -> str | None: + """日期列说明用哪条模板（None = 不是日期列）。""" + if col.role == "axis": + return "date_year_from_period" if _year_from_period(ctx, w) else "date" + if col.name in {c.name for b in w.lists for c in b.columns if c.type == "DATE"}: + return "date" + return None + + +def _null_eligible(ctx: _Ctx, w: _Writers, table: str, col: ColumnOut) -> bool: + """这一列要不要考虑空值说明：值列；主键不会是空值；表内合计表的空值已由表说明（未能核对）交代。""" + value_col = col.role in ("measure", "value") and col.type in ("INTEGER", "REAL") + is_total = bool(w.kept_from or w.list_totals) + return value_col and not is_total and col.name not in set(ctx.grains.get(table, [])) + + def _column_templates(ctx: _Ctx, table: str, null_counts: Mapping[str, Mapping[str, int]] | None) -> dict[str, str]: w = ctx.writers.get(table, _Writers()) - grain = set(ctx.grains.get(table, [])) - is_total = bool(w.kept_from or w.list_totals) - list_dates = {c.name for b in w.lists for c in b.columns if c.type == "DATE"} counts = null_counts.get(table) if null_counts is not None else None out: dict[str, str] = {} for col in ctx.tables.get(table, []): frags: list[str] = [] - if col.role == "axis": - frags.append(COLUMN_TEXT["date_year_from_period" if _year_from_period(ctx, w) else "date"]) - elif col.name in list_dates: - frags.append(COLUMN_TEXT["date"]) + if (date := _date_key(ctx, w, table, col)) is not None: + frags.append(COLUMN_TEXT[date]) if col.unit: frags.append(COLUMN_TEXT["unit"].format(unit=_tok("单位", col.unit))) - # 值列的空值说明：主键不会是空值；表内合计表的空值已由表说明（未能核对）交代 - value_col = (col.role in ("measure", "value") and col.type in ("INTEGER", "REAL")) has_nulls = counts is None or counts.get(col.name, 0) > 0 - if value_col and not is_total and col.name not in grain and has_nulls: + if _null_eligible(ctx, w, table, col) and has_nulls: if (text := _null_fragment(w, ctx.extraction)) is not None: frags.append(text) if frags: @@ -574,6 +682,37 @@ def _column_templates(ctx: _Ctx, table: str, null_counts: Mapping[str, Mapping[s # ---------------------------------------------------------------- 入口 +def _make_ctx(recipe: Recipe, extraction: Extraction, checks: list[CheckResult], + acceptances: list[Acceptance]) -> _Ctx: + tables, shape_problems = derive_tables(recipe) + problems = [f"配方推出的表结构有问题：{p.message}" for p in shape_problems] + return _Ctx( + recipe=recipe, extraction=extraction, checks={c.id: c for c in checks}, + accepted={a.check_id for a in acceptances}, tables=tables, + grains={t.name: list(t.grain) for t in recipe.tables}, dates=date_columns(recipe), + writers=_writers(recipe), problems=problems, plan=plan_checks(recipe, extraction), + ) + + +def _check_prose(notes: SchemaNotes, table: str, template: str, columns: dict[str, str], + known: dict[str, set[str]]) -> None: + for p in prose_problems(template, known=known): + notes.problems.append(f"表「{table}」的说明：{p}") + for col, tpl in columns.items(): + for p in prose_problems(tpl, known=known): + notes.problems.append(f"表「{table}」列「{col}」的说明：{p}") + + +def _store(notes: SchemaNotes, table: str, frags: list[NoteFragment], columns: dict[str, str]) -> None: + template = _join(frags) + if template: + notes.templates[table] = template + notes.templates.update({f"{table}.{col}": tpl for col, tpl in columns.items()}) + notes.fragments[table] = list(frags) + notes.tables[table] = TableNote(comment=render_prose(template), + columns={col: render_prose(tpl) for col, tpl in columns.items()}) + + def build_notes(recipe: Recipe, extraction: Extraction, checks: list[CheckResult], acceptances: list[Acceptance], *, null_counts: Mapping[str, Mapping[str, int]] | None = None) -> SchemaNotes: @@ -585,35 +724,24 @@ def build_notes(recipe: Recipe, extraction: Extraction, checks: list[CheckResult 或空格存空值的块里，每个值列都写（宁可多写一句「空值表示……」，不能漏写）。 **试运行预览和提交都应传 null_counts**：参考配方（D00）的 全日客流 列说明要是「单位：人次」 （P2-SPEC 7.4、9.7 第 2 步），不传就会多出空值那一句。 + + 期 3：fragments 记下每张表说明的片段（按拼接顺序），给 build_union_notes 逐期合并用；templates、tables 的 + 渲染结果与期 2 逐字相同。 """ - tables, shape_problems = derive_tables(recipe) - problems = [f"配方推出的表结构有问题：{p.message}" for p in shape_problems] - ctx = _Ctx( - recipe=recipe, extraction=extraction, checks={c.id: c for c in checks}, - accepted={a.check_id for a in acceptances}, tables=tables, - grains={t.name: list(t.grain) for t in recipe.tables}, dates=date_columns(recipe), - writers=_writers(recipe), problems=problems, plan=plan_checks(recipe, extraction), - ) + ctx = _make_ctx(recipe, extraction, checks, acceptances) + tables = ctx.tables # 只给真正建出来的表写说明（derive_tables 推出的表）；声明了却没有分段写入的表不在库里，写了也落不了地 names = list(tables) known = {"列": {c.name for cols in tables.values() for c in cols}, "表": set(names) | {t.name for t in recipe.tables}, "单位": set(UNITS)} after_stop = _rows_after_stop_tables(ctx) - notes = SchemaNotes(problems=problems) + notes = SchemaNotes(problems=ctx.problems) notes.kinds = {t.name: t.kind for t in recipe.tables if t.kind == "reported_total" and t.name in tables} for table in names: - template = _table_template(ctx, table, after_stop) + frags = _table_fragments(ctx, table, after_stop) columns = _column_templates(ctx, table, null_counts) - for p in prose_problems(template, known=known): - problems.append(f"表「{table}」的说明：{p}") - for col, tpl in columns.items(): - for p in prose_problems(tpl, known=known): - problems.append(f"表「{table}」列「{col}」的说明：{p}") - if template: - notes.templates[table] = template - notes.templates.update({f"{table}.{col}": tpl for col, tpl in columns.items()}) - notes.tables[table] = TableNote(comment=render_prose(template), - columns={col: render_prose(tpl) for col, tpl in columns.items()}) + _check_prose(notes, table, _join(frags), columns, known) + _store(notes, table, frags, columns) return notes @@ -647,3 +775,392 @@ def apply_notes(schema_cache: dict[str, Any], notes: SchemaNotes) -> dict[str, A target["comment"] = text out["import_mode"] = "recipe" return out + + +# ---------------------------------------------------------------- 期 3：按期累积的并集说明（P3-SPEC 2.8） + +#: 片段 key → 片段族。合并按 (片段族, subject) 进行；同一族的几个 key 是同一件事的几种结论 +_FAMILY: dict[str, str] = { + **{k: "grain" for k in ("grain_1", "grain_2", "grain_n")}, + **{k: "sum_eq" for k in ("sum_eq_passed", "sum_eq_mismatch_accepted", "sum_eq_mismatch_pending", "sum_eq_nulls", + "sum_eq_no_rows")}, + **{k: "not_comparable" for k in ("not_comparable_unequal", "not_comparable_unequal_flat", "not_comparable_equal")}, + "hour_range": "hour_range", "hour_range_plain": "hour_range", "const": "const", + "total_kept": "total_kept", "total_not_kept": "total_not_kept", + **{k: "total_k" for k in NOTE_TEXT if k.startswith("total_k_")}, + **{k: "list_total" for k in NOTE_TEXT if k.startswith("list_total_")}, + "total_overlap": "total_overlap", + "hidden_excluded": "hidden_excluded", "hidden_included": "hidden_included", + "ignored_columns": "ignored_columns", "ignored_rows": "ignored_rows", "ignored_cells": "ignored_cells", + "rows_after_stop": "rows_after_stop", + "period_conflict_accepted": "period_conflict", "period_conflict_pending": "period_conflict", + "period_human": "period_human", "outside_digits": "outside_digits", + "note": "note", +} +#: 合并后的拼接顺序（与 _table_fragments 一致；累积特有的几句排在用户写的表说明之前） +_FAMILY_ORDER = ("grain", "sum_eq", "not_comparable", "hour_range", "const", "total_kept", "total_not_kept", "total_k", + "list_total", "total_overlap", "hidden_excluded", "hidden_included", "ignored_columns", + "ignored_rows", "ignored_cells", "rows_after_stop", "period_conflict", "period_human", + "outside_digits", "union", "note") +#: 与核对结论无关、只描述表结构的片段族：用目标配方那一期的；只在早期出现的 subject（退役的表、另存表）用 +#: 最后一个含它的那一期的 +_STRUCTURAL = frozenset({"grain", "hour_range", "const", "total_kept", "total_not_kept", "total_overlap", "note"}) +#: 「任一期出现就写」的片段族 → 并集里的说法 +_ANY_UNION = {"hidden_excluded": "hidden_excluded_union", "hidden_included": "hidden_included_union", + "ignored_columns": "ignored_columns_union", "ignored_rows": "ignored_rows_union", + "ignored_cells": "ignored_cells_union", "outside_digits": "outside_digits_union"} +_K_PASSED = frozenset({"total_k_passed", "total_k_passed_g_unverifiable"}) + + +@dataclass +class _UPart: + """并集里的一期：它自己的说明（按它自己的配方、核对、接受生成）和推导出的表结构。""" + + index: int + src: UnionNotePart + ctx: _Ctx + notes: SchemaNotes + #: 表 → {列名: ColumnOut} + cols: dict[str, dict[str, ColumnOut]] + + def label(self) -> str: + return f"第 {self.index + 1} 期（{self.src.start} 至 {self.src.end}）" + + +def _canonical(recipe: Recipe) -> dict[str, Any]: + return recipe.model_dump(mode="json", exclude_defaults=True) + + +def _relation(recipe: Recipe, rid: str | None) -> Any: + return next((r for r in recipe.relations if r.id == rid), None) + + +def _pick(group: dict[int, NoteFragment], having: list[int], ref: int) -> tuple[int, NoteFragment] | None: + """结构类片段取哪一期的：目标那一期有就用它；否则用最后一个还有它的那一期。""" + if ref in group: + return ref, group[ref] + for i in reversed(having): + if i in group: + return i, group[i] + return None + + +def _sum_eq_slots(part: _UPart, table: str, rid: str | None) -> dict[str, str] | None: + rel = _relation(part.src.recipe, rid) + if not isinstance(rel, SumEq): + return None + _, unit = _PER[_per_day(part.ctx, table)] + return {"total": _col(rel.total), "parts": _LIST.join(_col(p) for p in rel.parts), "unit": unit} + + +def _sum_eq_members(recipe: Recipe, rid: str | None) -> tuple[str, tuple[str, ...]] | None: + """一期配方里这条 sum_eq 的成员：(合计列, 排好序的分项列)。分项的先后不影响关系的含义。""" + rel = _relation(recipe, rid) + return (rel.total, tuple(sorted(rel.parts))) if isinstance(rel, SumEq) else None + + +def _merge_sum_eq(table: str, subject: str | None, group: dict[int, NoteFragment], having: list[int], ref: int, + parts: list[_UPart], problems: list[str]) -> NoteFragment | None: + """sum_eq 按各期最弱合并（P3-SPEC 2.8 表）。优先级：有一期尚未确认接受 > 部分期没有这条关系 > 有一期没成立或 + 没核对 > 各期成员不同 > 各期都通过且成员相同。理由：前两种都不能写「已核对」「已由用户确认接受」；尚未接受的 + 排最前，是因为那一期还要人处理，界面上要看得见。""" + present = [group[i] for i in having if i in group] + keys = {f.key for f in present} + picked = _pick(group, having, ref) + if picked is None: + return None + src = parts[picked[0]] + slots = _sum_eq_slots(src, table, subject) + if slots is None: + problems.append(f"表「{table}」：{src.label()}的配方里找不到关系「{subject}」，无法生成说明") + return None + partial = len(present) < len(having) + if "sum_eq_mismatch_pending" in keys: + key = "sum_eq_union_pending" + elif partial: + key = "sum_eq_union_partial" + elif keys - {"sum_eq_passed"}: + key = "sum_eq_union_mixed" + elif len(members := {_sum_eq_members(parts[i].src.recipe, subject) for i in having if i in group}) == 1: + # 各期登记的合计列和分项列相同（只比集合：分项只是调了顺序、或者逐日与逐行的说法不同，都不算成员不同）， + # 用目标那一期的句子。按渲染出的句子比的话，调换分项顺序就会写成「各期登记的分项列不同」，而这句会进 + # 模型可见的说明（评审意见） + return NoteFragment("sum_eq_passed", subject, picked[1].template) + else: + totals = {m[0] for m in members if m is not None} + # 各期的合计列都相同，只是分项不同，才能说「{total} 等于各期登记的分项列之和」；合计列也换过的，那几期的 + # 合计不在这一列里，按「只在部分期核对」这种更弱的说法写 + key = "sum_eq_members_vary" if len(totals) == 1 else "sum_eq_union_partial" + return NoteFragment(key, subject, NOTE_TEXT[key].format(**slots)) + + +def _merge_not_comparable(table: str, subject: str | None, group: dict[int, NoteFragment], having: list[int], + ref: int, parts: list[_UPart], problems: list[str]) -> NoteFragment | None: + picked = _pick(group, having, ref) + if picked is None: + return None + present = [group[i] for i in having if i in group] + if len(present) == len(having) and all(f.key.startswith("not_comparable_unequal") for f in present): + return NoteFragment(picked[1].key, subject, picked[1].template) + # 任一期全部相等、或者部分期没有这条关系：不写「不等于」 + rel = _relation(parts[picked[0]].src.recipe, subject) + if not isinstance(rel, NotComparable): + problems.append(f"表「{table}」：{parts[picked[0]].label()}的配方里找不到关系「{subject}」，无法生成说明") + return None + other = rel.b.table if table == rel.a.table else rel.a.table + return NoteFragment("not_comparable_equal", subject, NOTE_TEXT["not_comparable_equal"].format(other=_tab(other))) + + +def _merge_group(table: str, family: str, subject: str | None, group: dict[int, NoteFragment], having: list[int], + ref: int, parts: list[_UPart], problems: list[str]) -> NoteFragment | None: + """一组 (片段族, subject) 跨期合并成一个片段（两期及以上时）。group：期的下标 → 那一期的片段。 + + 「部分期缺失」只在含这张表的那几期里算（having）：不含这张表的期在并集里没有这张表的行，不需要它的核对。 + 这样退役的合计表只在早期出现时，说明取最后一个含它的那一期的（P3-SPEC 2.8 的例子），而同一张表里某一期 + 没登记这条关系，照样按「部分期未核对」写。""" + present = [group[i] for i in having if i in group] + if family in _STRUCTURAL: + picked = _pick(group, having, ref) + return NoteFragment(picked[1].key, subject, picked[1].template) if picked else None + if family == "sum_eq": + return _merge_sum_eq(table, subject, group, having, ref, parts, problems) + if family == "not_comparable": + return _merge_not_comparable(table, subject, group, having, ref, parts, problems) + if family == "total_k": + if len(present) == len(having) and {f.key for f in present} <= _K_PASSED: + key = ("total_k_passed_g_unverifiable" if any(f.key == "total_k_passed_g_unverifiable" for f in present) + else "total_k_passed") + else: + key = "total_union_unverified" + return NoteFragment(key, subject, NOTE_TEXT[key]) + if family in _ANY_UNION: + key = _ANY_UNION[family] + return NoteFragment(key, subject, NOTE_TEXT[key]) + if family == "period_conflict": + key = ("period_conflict_union_pending" if any(f.key == "period_conflict_pending" for f in present) + else "period_conflict_union") + return NoteFragment(key, subject, NOTE_TEXT[key]) + if family == "period_human": + key = "period_human_all" if len(present) == len(having) else "period_human_union" + return NoteFragment(key, subject, NOTE_TEXT[key]) + # list_total、rows_after_stop 只有列表才有，而列表不能按期累积（P3-SPEC 2.2）；不认识的片段族同样没有并集说法。 + # 记 problems（不能保存），绝不沿用某一期的单期说法。消息给人看，不露片段族的键名 + problems.append(f"表「{table}」：{_UNMERGEABLE.get(family, '有一段说明')}不能按期合并，无法生成说明") + return None + + +#: 没有并集说法的片段族 → 给人看的叫法（problems 里不露键名） +_UNMERGEABLE = {"list_total": "列表合计行的核对结论", "rows_after_stop": "列表空行之后的文字行"} + + +def _axis_column(recipe: Recipe, table: str) -> str | None: + tables, _ = derive_tables(recipe) + axes = [c.name for c in tables.get(table, []) if c.role == "axis"] + return axes[0] if len(axes) == 1 else None + + +def _union_table_fragments(table: str, having: list[int], parts: list[_UPart], ref: int, target: Recipe, + target_tables: dict[str, list[ColumnOut]], label_sets: list[dict], gaps: bool, + problems: list[str]) -> list[NoteFragment]: + single = len(parts) == 1 + out: list[tuple[tuple[int, int], NoteFragment]] = [] + if single: + # 物化的单期快照（移除、作废之后，P3-SPEC 7.4）：那一期自己的说法就是准确的，不说「部分期」 + out = [((_FAMILY_ORDER.index(_FAMILY.get(f.key, "note")), n), f) + for n, f in enumerate(parts[0].notes.fragments.get(table, []))] + else: + groups: dict[tuple[str, str | None], dict[int, NoteFragment]] = {} + seen: dict[tuple[str, str | None], int] = {} + # 先按目标那一期的片段顺序，再按其余各期从新到旧，记下每组第一次出现的位置 + visit = [ref] + [i for i in reversed(having) if i != ref] + for i in visit: + for f in parts[i].notes.fragments.get(table, []): + family = _FAMILY.get(f.key) + if family is None: + problems.append(f"表「{table}」：{parts[i].label()}有一段说明不能按期合并，无法生成说明") + continue + k = (family, f.subject) + groups.setdefault(k, {}).setdefault(i, f) + seen.setdefault(k, len(seen)) + for (family, subject), group in groups.items(): + merged = _merge_group(table, family, subject, group, having, ref, parts, problems) + if merged is not None: + out.append(((_FAMILY_ORDER.index(family), seen[(family, subject)]), merged)) + union: list[NoteFragment] = [] + if not having: + if target_table := next((t for t in target.tables if t.name == table), None): + if (g := _grain_fragment(table, list(target_table.grain))) is not None: + union.append(g) + union.append(NoteFragment("table_absent", table, NOTE_TEXT["table_absent"])) + if single: + union.append(NoteFragment("single_after_drop", table, NOTE_TEXT["single_after_drop"])) + elif having: + recipe = target if table in target_tables else parts[having[-1]].src.recipe + axis = _axis_column(recipe, table) + if axis is None: + problems.append(f"表「{table}」：找不到日期列，无法写按期累积的说明") + else: + key = "accumulate_gap" if gaps else "accumulate" + union.append(NoteFragment(key, table, NOTE_TEXT[key].format(col=_col(axis)))) + if len(having) < len(parts): + union.append(NoteFragment("table_partial_periods", table, NOTE_TEXT["table_partial_periods"])) + dims: list[str] = [] + for entry in label_sets or []: + col = str(entry.get("column") or "") if isinstance(entry, dict) else "" + if entry.get("table") == table and col and col not in dims: + dims.append(col) + union.append(NoteFragment("dim_values_vary", col, NOTE_TEXT["dim_values_vary"].format(dim=_col(col)))) + pos = _FAMILY_ORDER.index("union") + out += [((pos, n), f) for n, f in enumerate(union)] + out.sort(key=lambda x: x[0]) + return [f for _, f in out if _clean(f.template)] + + +def _union_column_templates(table: str, columns: list[ColumnOut], having: list[int], parts: list[_UPart], + target_cols: dict[str, ColumnOut] | None, retired: set[tuple[str, str]], + union_kinds: dict[str, str], null_counts: Mapping[str, Mapping[str, int]] | None, + part_null_counts: list[dict[str, dict[str, int]]] | None) -> dict[str, str]: + """并集里每一列的说明（P3-SPEC 2.8 第 4 步）： + - 日期：任一期年份取自统计期就用「年份按统计期补全」； + - 单位：目标配方里有的列取目标配方的，退役列取最后一个还含它的那一期的； + - 空值：只按含这一列的那几期统计（每期的空值数、那几期出现过的占位符、空格存空值的设置）。不含这一列的期是 + 结构性空值，由下一句说明，不能说成「原表为空格」； + - 部分期没有这一列：col_partial_periods；退役列：col_retired；哪一期都没有：col_absent。""" + counts_total = null_counts.get(table) if null_counts is not None else None + out: dict[str, str] = {} + for col in columns: + with_col = [i for i in having if col.name in parts[i].cols.get(table, {})] + frags: list[str] = [] + dates = {_date_key(parts[i].ctx, parts[i].ctx.writers.get(table, _Writers()), table, + parts[i].cols[table][col.name]) for i in with_col} + if "date_year_from_period" in dates: + frags.append(COLUMN_TEXT["date_year_from_period"]) + elif "date" in dates or (not with_col and col.role == "axis"): + frags.append(COLUMN_TEXT["date"]) + if target_cols is not None and col.name in target_cols: + unit = target_cols[col.name].unit + elif with_col: + unit = parts[with_col[-1]].cols[table][col.name].unit + else: + unit = col.unit + if unit: + frags.append(COLUMN_TEXT["unit"].format(unit=_tok("单位", unit))) + meanings: set[str] = set() + blank = False + none_in_union = counts_total is not None and counts_total.get(col.name, 1) == 0 + if table not in union_kinds and not none_in_union: + for i in with_col: + part = parts[i] + w = part.ctx.writers.get(table, _Writers()) + if not _null_eligible(part.ctx, w, table, part.cols[table][col.name]): + continue + counts = part_null_counts[i].get(table) if part_null_counts is not None else None + if counts is not None and counts.get(col.name, 0) <= 0: + continue + m, b = _null_sources(w, part.src.extraction) + meanings |= m + blank |= b + if (text := _null_text(meanings, blank)) is not None: + frags.append(text) + if not with_col: + frags.append(COLUMN_TEXT["col_absent"]) + elif (table, col.name) in retired: + frags.append(COLUMN_TEXT["col_retired"]) + elif len(parts) > 1 and len(with_col) < len(having): + frags.append(COLUMN_TEXT["col_partial_periods"]) + if frags: + out[col.name] = "。".join(frags) + return out + + +def build_union_notes(target: Recipe, parts: list[UnionNotePart], *, union_tables: dict[str, list[ColumnOut]], + null_counts: dict[str, dict[str, int]] | None, + part_null_counts: list[dict[str, dict[str, int]]] | None, + added: list[dict], retired: list[dict], label_sets: list[dict], + gaps: bool, dropped: bool = False) -> SchemaNotes: + """按期累积物化的快照库的说明（P3-SPEC 2.8）。parts 按统计期排序，可以只有一期（物化的单期快照，7.4）。 + + 做法：逐期调 build_notes（该期的配方、Extraction、核对、接受、空值数），按 (片段族, subject) 合并、取最弱： + - 部分期没有这条关系、这张合计表的结论：按「未能核对」写，绝不沿用「已核对」； + - 任一期还没确认接受（试运行预览时本期按「全部未接受」生成）：不写「已由用户确认接受」； + - 结构类的片段（粒度、时段写法、常量列、合计另存在哪、用户写的表说明）用目标配方那一期的，只在早期出现的 + 用最后一个含它的那一期的。「目标那一期」= 配方与目标配方相同的最后一期；没有一期相同（移除、作废之后） + 时取最后一期。 + + union_tables：并集的表结构（目标配方的表加退役的表和列），说明逐表、逐列按它写；某一期有、而它没有的表记 + problems（并集漏了这一期的数据）。null_counts / part_null_counts：materialize_union 的整体与按期空值数， + part_null_counts 与 parts 同序。added / retired：并集新增、退役的表和列（UnionReport），退役以「目标配方没有、 + 并集里有」推出，retired 里列出的一并按退役处理；added 只作交叉核对。label_sets：各期维度取值的差异（2.5）。 + gaps：各期之间有空缺。dropped：计划移出了此前各期（restart、模式切换），结果只能有一期。 + + kinds 取各期配方的并集：任一期把一张表声明为原表写明的合计，并集里就标 reported_total（退役了也标）。 + 每条模板都过 prose_problems（known = 并集的列、表、单位）；problems 非空就不能保存。 + """ + notes = SchemaNotes() + problems = notes.problems + if not parts: + problems.append("没有任何一期的数据，无法生成按期累积的说明") + return notes + if dropped and len(parts) > 1: + problems.append("计划移出了此前各期，但当前版本有多期，说明无法确定") + if part_null_counts is not None and len(part_null_counts) != len(parts): + problems.append("各期空值数的期数与各期不符，无法生成空值说明") + part_null_counts = None + built: list[_UPart] = [] + for i, src in enumerate(parts): + pn = build_notes(src.recipe, src.extraction, src.checks, src.acceptances, null_counts=src.null_counts) + ctx = _make_ctx(src.recipe, src.extraction, src.checks, src.acceptances) + part = _UPart(index=i, src=src, ctx=ctx, notes=pn, + cols={t: {c.name: c for c in cs} for t, cs in ctx.tables.items()}) + problems.extend(f"{part.label()}：{p}" for p in pn.problems) + built.append(part) + target_sig = _canonical(target) + ref = max((i for i, p in enumerate(built) if _canonical(p.src.recipe) == target_sig), default=len(built) - 1) + target_tables, _ = derive_tables(target) + retired_cols = {(t, c.name) for t, cs in union_tables.items() if t in target_tables + for c in cs if c.name not in {x.name for x in target_tables[t]}} + retired_cols |= {(str(r.get("table")), str(r.get("column"))) for r in retired or [] + if isinstance(r, dict) and r.get("column")} + for entry in added or []: + if isinstance(entry, dict) and entry.get("table") not in union_tables: + problems.append(f"新增的表「{entry.get('table')}」不在当前版本的表结构中") + for part in built: + for t in part.cols: + if t not in union_tables: + problems.append(f"表「{t}」在{part.label()}有数据，但不在当前版本的表结构中") + kinds = {t.name: t.kind for p in [*(b.src.recipe for b in built), target] for t in p.tables + if t.kind == "reported_total"} + notes.kinds = {t: k for t, k in kinds.items() if t in union_tables} + for t in target_tables: + if t not in union_tables: + problems.append(f"目标配方的表「{t}」不在当前版本的表结构中") + # 引用只认并集库里真有的表和列：某一期的说明提到了并集里没有的列，就是并集漏了东西，不能保存 + known = {"列": {c.name for cols in union_tables.values() for c in cols}, "表": set(union_tables), + "单位": set(UNITS)} + for table, columns in union_tables.items(): + having = [i for i, p in enumerate(built) if table in p.cols] + frags = _union_table_fragments(table, having, built, ref, target, target_tables, label_sets, gaps, problems) + target_cols = {c.name: c for c in target_tables[table]} if table in target_tables else None + cols = _union_column_templates(table, list(columns), having, built, target_cols, retired_cols, notes.kinds, + null_counts, part_null_counts) + _check_prose(notes, table, _join(frags), cols, known) + _store(notes, table, frags, cols) + return notes + + +def add_single_after_drop(notes: SchemaNotes) -> SchemaNotes: + """累积模式下 restart 的结果：快照库就是本期的构建库（build_notes 生成说明），但此前各期已移出当前版本 + （计划的 dropped 非空）。给每张表加 single_after_drop，免得模型以为当前版本就是全部历史（评审一-m10）。 + 返回新的 SchemaNotes，不改入参。""" + out = copy.deepcopy(notes) + text = NOTE_TEXT["single_after_drop"] + for table, note in out.tables.items(): + frags = list(out.fragments.get(table, [])) + if any(f.key == "single_after_drop" for f in frags): + continue + out.fragments[table] = [*frags, NoteFragment("single_after_drop", table, text)] + template = out.templates.get(table, "") + template = f"{_clean(template)}。{text}" if _clean(template) else text + out.templates[table] = template + note.comment = render_prose(template) + return out diff --git a/backend/app/data/recipe_select.py b/backend/app/data/recipe_select.py new file mode 100644 index 00000000..daa7332c --- /dev/null +++ b/backend/app/data/recipe_select.py @@ -0,0 +1,1453 @@ +"""框选转锚点（期 3，WP-2，P3-SPEC 第 4 节）：网格上框一块区域、选它是什么，换算成按文字认的配方规则。 + +**坐标不进配方**（4.1）：配方 schema 里没有任何坐标字段（extra="forbid"），框选的坐标只在换算的那一刻用两次—— +从格子里取锚点文字（表头、行标签、分段标题、合计词、轴右侧表头、同一行的文字），以及换算完干跑后把执行器认出的 +区域和框比对（replay_compare）。所以下个月行列挪了位置，配方照样按文字找到它（4.6 第 2 条用挪位的文件钉住）。 + +**换算规则全部复用起草器**（期 2 规格 6.1，评审二-m12）：列表直接用起草器的列表计划和块构造函数 +（recipe_suggest.list_block），框只决定表头行和列范围，列名、单位、类型、grain、合计行、跳过空行、占位符、合并 +单元格填充都由起草器按框内的数据推断；交叉表直接取规则起草对整张工作表的结果。换算不了就明确说为什么 +（EditResult.ok=False，problems 的 code 见 4.3），不退回写死坐标，也不猜。 + +客户端传来的文字只有两种会进补丁：理由，以及经过名字校验、撞名检查的表名（options.table，评审一-m6）。 +""" +from __future__ import annotations + +import copy +from dataclasses import dataclass, field +from typing import Any, Callable + +from pydantic import ValidationError + +from app.data import recipe_suggest as S +from app.data.names import canon, collide_key, name_problem, to_sql_name +from app.data.recipe_fixes import ( + REASON_MAX, + EditRequestError, + all_ids, + finish, + has_digit, + keep_ok, + ptr, + quoted, + table_index, + total_blockers, + total_ops, + unique_id, + unique_name, +) +from app.data.recipe_parsers import ( + LABEL_MAX, + candidate_words, + hour_range, + hour_range_total, + looks_numeric_text, + match_key, + month_day_or_date, + split_unit_suffix, + text_label, +) +from app.data.recipe_types import ( + RECIPE_FORMAT, + Anchor, + CrosstabBlock, + Draft, + EditResult, + Extraction, + Grid, + ListBlock, + Problem, + Recipe, + ReplayCompare, + Selection, +) +from app.data.xlsx_scan import cell_ref, col_letter, parse_ref + +#: 交叉表：框与规则草稿认出的交叉表至少重叠这么多（占框的比例）才算框住了它（4.2） +CROSSTAB_OVERLAP = 0.5 +#: 锚点的长度上限（契约 IgnoreRow.label / IgnoreOutside.anchor 40 字、ListBlock.after_title 40 字、IgnoreColumn.header 80 字） +_ANCHOR_MAX, _HEADER_MAX = 40, 80 + +Rect = tuple[int, int, int, int] + + +# ========================================================================== +# 小工具 +# ========================================================================== + + +def _rect(ref: str) -> Rect: + """「C5」或「C5:F8」→ (r1, c1, r2, c2)，左上到右下（反着框也行）。""" + a, _, b = ref.partition(":") + r1, c1 = parse_ref(a) + r2, c2 = parse_ref(b) if b else (r1, c1) + return min(r1, r2), min(c1, c2), max(r1, r2), max(c1, c2) + + +def _a1(rect: Rect | None) -> str | None: + if rect is None: + return None + r1, c1, r2, c2 = rect + return cell_ref(r1, c1) if (r1, c1) == (r2, c2) else f"{cell_ref(r1, c1)}:{cell_ref(r2, c2)}" + + +def _rows_text(r1: int, r2: int) -> str: + return f"第 {r1} 行" if r1 == r2 else f"第 {r1}–{r2} 行" + + +def _intersects(a: Rect, b: Rect) -> bool: + return a[0] <= b[2] and b[0] <= a[2] and a[1] <= b[3] and b[1] <= a[3] + + +def _area(a: Rect) -> int: + return (a[2] - a[0] + 1) * (a[3] - a[1] + 1) + + +def _overlap(a: Rect, b: Rect) -> int: + if not _intersects(a, b): + return 0 + return (min(a[2], b[2]) - max(a[0], b[0]) + 1) * (min(a[3], b[3]) - max(a[1], b[1]) + 1) + + +def _bbox(rects: list[Rect]) -> Rect | None: + if not rects: + return None + return min(r[0] for r in rects), min(r[1] for r in rects), max(r[2] for r in rects), max(r[3] for r in rects) + + +def _cells_bbox(cells: set[tuple[int, int]] | list[tuple[int, int]]) -> Rect | None: + cells = list(cells) + if not cells: + return None + return min(r for r, _ in cells), min(c for _, c in cells), max(r for r, _ in cells), max(c for _, c in cells) + + +def _text_of(grid: Grid, r: int, c: int) -> str | None: + """格子里的文字（去首尾空白）：不是文字、是公式或只有空白时 None。""" + cell = grid.get(r, c) + if cell is None or cell.formula is not None or not isinstance(cell.value, str): + return None + t = cell.value.strip() + return t or None + + +def _textual(grid: Grid, r: int, c: int) -> bool: + """这一格能不能作表头文字：是文字、不像数、不是日期写法。""" + cell = grid.get(r, c) + if cell is None: + return True + if cell.formula is not None or not isinstance(cell.value, str): + return False + return not looks_numeric_text(cell.value) and not S._date_kind(cell.value) + + +def _row_empty(grid: Grid, r: int, c1: int, c2: int) -> bool: + return all(grid.get(r, c) is None for c in range(c1, c2 + 1)) + + +def _reason(sel: Selection) -> str: + """忽略类框选的理由（必填，1–200 字）。理由进配方哈希：预览和应用必须用同一个理由。""" + raw = sel.options.get("reason") + text = raw.strip() if isinstance(raw, str) else "" + if not text: + raise EditRequestError("reason_required", "忽略要写明理由") + if len(text) > REASON_MAX: + raise EditRequestError("edit_invalid", f"理由不能超过 {REASON_MAX} 字") + return text + + +@dataclass +class _Ctx: + recipe: dict[str, Any] | None + parsed: Recipe | None + grid: Grid + sel: Selection + rect: Rect + ext: Extraction | None + retired: dict[str, list[str]] + #: 配方里对应这张工作表的下标（没有为 None） + si: int | None = None + problems: list[Problem] = field(default_factory=list) + + @property + def sheet(self) -> str: + return self.grid.sheet + + def coord(self, r: int, c: int) -> str: + return f"{self.sheet}!{cell_ref(r, c)}" + + def fail(self, code: str, message: str, cells: list[str] | None = None) -> None: + box = f"{self.sheet}!{_a1(self.rect)}" + self.problems.append(Problem(code, "recipe", message, cells=(cells or [box])[:20], model_message=code)) + + +def _failed(ctx: _Ctx) -> EditResult: + return EditResult(ok=False, kind="selection", key=f"{ctx.sel.as_}:", title=_TITLES.get(ctx.sel.as_, "框选"), + summary=[], ops=[], anchors=[], notes=[], problems=list(ctx.problems)) + + +_TITLES = {"list": "框选为列表", "crosstab": "框选为交叉表", "segment": "框选为分段", "derived": "框选为合计行", + "section_title": "框选为分段标题", "ignore_rows": "忽略框选的行", "ignore_columns": "忽略框选的列", + "ignore_outside": "忽略框选行里导入区域之外的数字"} + + +def _sheet_of(recipe: dict[str, Any] | None, grid: Grid, ext: Extraction | None) -> int | None: + """配方里对应这张网格的工作表：干跑认出的（sheets.matched：工作表 id → 实际名）优先，其次按名字。""" + if not recipe: + return None + sheets = recipe.get("sheets") or [] + if ext is not None: + for sid, actual in ext.sheets.matched.items(): + if match_key(actual) == match_key(grid.sheet): + hit = next((i for i, s in enumerate(sheets) if s.get("id") == sid), None) + if hit is not None: + return hit + key = match_key(grid.sheet) + return next((i for i, s in enumerate(sheets) if match_key((s.get("match") or {}).get("name")) == key), None) + + +@dataclass +class _Located: + bi: int + block: dict[str, Any] + rects: list[Rect] + cross: Any = None # recipe_suggest._CrossLoc + lst: Any = None # recipe_suggest._ListLoc + + +def _locate(ctx: _Ctx) -> list[_Located]: + """这张工作表上配方各块认领的区域。干跑结果带块 id（RegionMark.block，期 3）时按它取；否则（期 2 的执行器、 + 没有干跑）用起草器的定位函数在网格上近似。交叉表另外总是用起草器定位一遍：标签列、轴行、各分段的行要用到。""" + if ctx.si is None or ctx.parsed is None: + return [] + sheet = ctx.recipe["sheets"][ctx.si] # type: ignore[index] + model = ctx.parsed.sheets[ctx.si] + tagged: dict[str, list[Rect]] = {} + if ctx.ext is not None and any(m.block for m in ctx.ext.regions): + for m in ctx.ext.regions: + if m.block and match_key(m.sheet) == match_key(ctx.sheet): + tagged.setdefault(m.block, []).append(_rect(m.ref)) + out: list[_Located] = [] + # 已用的表头行只对表头相同的块排除（上下两张同表头的表各认各的）；左右并排的两张表表头在同一行，不能互相排除 + taken: dict[frozenset[str], set[int]] = {} + for bi, (bd, bm) in enumerate(zip(sheet.get("blocks") or [], model.blocks)): + loc = _Located(bi, bd, list(tagged.get(bm.id, []))) + if isinstance(bm, CrosstabBlock): + loc.cross = S._locate_crosstab(bm, ctx.grid) + if not loc.rects and loc.cross is not None: + box = _cells_bbox(loc.cross.region) + loc.rects = [box] if box else [] + elif isinstance(bm, ListBlock): + keys = frozenset(match_key(c.header) for c in bm.columns) + loc.lst = S._locate_list(bm, ctx.grid, taken.setdefault(keys, set())) + if loc.lst is not None: + taken[keys].update(loc.lst.header_rows) + if not loc.rects: + box = _cells_bbox(loc.lst.region) + loc.rects = [box] if box else [] + out.append(loc) + return out + + +def _crosstab_at(ctx: _Ctx, located: list[_Located]) -> _Located | None: + """框落在哪个交叉表块的标签列上：框含标签列、不越过标签列往左、在日期表头之下、不越过最后一个日期往右。""" + r1, c1, r2, c2 = ctx.rect + for loc in located: + x = loc.cross + if x is None: + continue + if c1 == x.label_col and c2 <= x.c2 and r1 > x.axis_row: + return loc + return None + + +def _labels(ctx: _Ctx, col: int) -> list[tuple[int, str]] | None: + """框内各行标签列的文字；有行没有文字标签时报 selection_label_blank 并返回 None。""" + r1, _c1, r2, _c2 = ctx.rect + out: list[tuple[int, str]] = [] + blank: list[int] = [] + for r in range(r1, r2 + 1): + t = _text_of(ctx.grid, r, col) + if t is None: + blank.append(r) + else: + out.append((r, t)) + if blank: + ctx.fail("selection_label_blank", + f"{'、'.join(f'第 {r} 行' for r in blank[:8])}的标签列（{col_letter(col)} 列）没有文字，无法按标签认：请调整框的上下范围，" + "或改选别的类型", [ctx.coord(r, col) for r in blank]) + return None + return out + + +def _new_table_name(ctx: _Ctx, want: str, used: set[str]) -> str | None: + """新表名：客户端给了 options.table 时过名字校验和撞名检查（含当前版本里退役的表名，评审一-m6），否则按 want + 推出并加 _2 去重。不合法或撞名时报问题、返回 None。""" + given = ctx.sel.options.get("table") + if given is None or given == "": + return unique_name(to_sql_name(want, fallback=f"表{len(used) + 1}") if want else f"表{len(used) + 1}", used) + if not isinstance(given, str): + raise EditRequestError("edit_invalid", "表名要是文字") + why = name_problem(given) + if why: + ctx.fail("selection_name_invalid", f"表名不能用：{why}") + return None + if collide_key(given) in used: + ctx.fail("selection_name_taken", f"表名「{given}」与已有的表或已退役的表重名（名字不区分大小写和全角半角），请换一个名字") + return None + used.add(collide_key(given)) + return given + + +def _used_tables(ctx: _Ctx, *, except_: set[str] | frozenset[str] = frozenset()) -> set[str]: + """已用的表名（collide_key）：工作配方里的表和当前版本里退役的表；except_ 里的（被替换的块自己的表）不算。""" + used = {collide_key(t.get("name", "")) for t in (ctx.recipe or {}).get("tables") or [] + if t.get("name") not in except_} + used |= {collide_key(t) for t in ctx.retired if t not in except_} + return used + + +# ========================================================================== +# 列表 +# ========================================================================== + + +def _header_problems(ctx: _Ctx, rows: list[int], cols: list[int]) -> dict[int, str] | None: + """表头：每列按执行器的拼法得到（合并区取左上格、去掉空的和与上一层相同的、用「_」拼接）。任一列为空、 + 是数字或日期、两列 match_key 相同，各报一条（4.2 列表第 1 条）。""" + texts = S._header_texts(ctx.grid, rows, cols) + # 空：表头那几行这一列一格都没有（是数字、日期的另报 not_text，不算空） + blank = [c for c in cols if c not in texts and all(ctx.grid.get(r, c) is None for r in rows)] + if blank: + ctx.fail("selection_header_blank", + f"{_rows_text(rows[0], rows[-1])}的 {'、'.join(col_letter(c) for c in blank)} 列表头为空：列表的每一列都要有表头。" + "请调整框的左右范围，或改选「忽略这些列」", [ctx.coord(rows[-1], c) for c in blank]) + bad = sorted({c for c in cols for r in rows if not _textual(ctx.grid, r, c)}) + if bad: + ctx.fail("selection_header_not_text", + f"{'、'.join(col_letter(c) for c in bad)} 列的表头是数字或日期，不像表头：框的第一行应是表头。" + "请把框的上边移到表头所在的行", [ctx.coord(rows[0], c) for c in bad]) + seen: dict[str, int] = {} + dup: list[int] = [] + for c in cols: + if c in texts: + k = match_key(texts[c]) + if k in seen: + dup.append(c) + seen.setdefault(k, c) + if dup: + ctx.fail("selection_header_duplicate", + f"表头{quoted([texts[c] for c in dup])}出现了不止一次：同一张列表里表头不能重复。请调整框的左右范围", + [ctx.coord(rows[-1], c) for c in dup]) + return None if (blank or bad or dup) else texts + + +def _header_candidates(grid: Grid, headers: list[str], n: int) -> list[int]: + """整张网格上像这组表头的行（执行器的候选规则：过半表头命中）。""" + want = {match_key(h) for h in headers} + out = [] + for t in grid.row_numbers(): + rows = list(range(t, t + n)) + cols = sorted({c for r in rows for c, _ in grid.row(r)}) + keys = {match_key(x) for x in S._header_texts(grid, rows, cols).values()} + if len(want & keys) * 2 > len(want): + out.append(t) + return out + + +def _after_title(ctx: _Ctx, top: int, cols: list[int], headers: list[str], n: int) -> tuple[str, str] | None | bool: + """表头不止一处时找 after_title：表头行往上、块的列范围内最近的、整张表里唯一的、不含数字的文字格（4.2 第 2 条， + 评审一-M11：最近的唯一文字格常常是「2026年8月 销售月报」这种带年月的标题，写进配方等于写死期次）。 + 返回 (文字, 坐标)；不需要时 None；需要但找不到时 False。""" + others = [t for t in _header_candidates(ctx.grid, headers, n) if t != top] + if not others: + return None + above = [t for t in others if t < top] + floor = 1 + if above: + # 上面那张表到它下方第一个空行为止：标题只在那之后找，不能拿上一张表的数据格当标题 + floor = max(above) + n + while floor < top and not _row_empty(ctx.grid, floor, cols[0], cols[-1]): + floor += 1 + counts: dict[str, int] = {} + for (r, c), cell in ctx.grid.cells.items(): + if isinstance(cell.value, str) and cell.formula is None: + counts[match_key(cell.value)] = counts.get(match_key(cell.value), 0) + 1 + for r in range(top - 1, floor - 1, -1): + for c in cols: + t = _text_of(ctx.grid, r, c) + if t and len(t) <= _ANCHOR_MAX and not has_digit(t) and counts.get(match_key(t)) == 1: + return t, ctx.coord(r, c) + return False + + +def _no_extra_columns(ctx: _Ctx, rows: list[int], cols: list[int], ignored: set[str]) -> bool: + """执行器按表头行里包含锚点的最大连续非空段认列范围（期 2 规格 4.4）：框的左右紧挨着还有非空表头时，重放会把 + 它们也算进这张表（多出的列默认拒收），按文字无法表达「到这一列为止」，报 selection_extra_columns（4.2 第 3 条）。 + 被替换的块上已经按表头忽略的列不算（ignored：表头的 match_key）。""" + lo, hi = cols[0], cols[-1] + wide = S._header_texts(ctx.grid, rows, list(range(max(1, lo - 60), hi + 61))) + left, right = lo, hi + while left - 1 in wide: + left -= 1 + while right + 1 in wide: + right += 1 + extra = [c for c in range(left, right + 1) if (c < lo or c > hi) and match_key(wide[c]) not in ignored] + if not extra: + return True + r1, _c1, r2, _c2 = ctx.rect + ctx.fail("selection_extra_columns", + f"框的旁边紧挨着还有表头{quoted([wide[c] for c in extra])}（{'、'.join(col_letter(c) for c in extra)} 列）：" + f"按文字定位时它们会被算进这张表。请把框扩大到 {_a1((r1, left, r2, right))}，或改选「忽略这些列」", + [ctx.coord(rows[-1], c) for c in extra]) + return False + + +def _bottom_box(ctx: _Ctx, plan: Any, first: int) -> bool: + """bottom="box"：框的下边就是数据的末尾（4.2 第 5 条）。合计行照起草器的规则认（期 2 规格 6.1 第 11 条）。 + + - 框内最后一个非空行 last 的第一列以合计词开头、上面还有数据行：它是合计行（rows.total_row），不是数据行。 + 框把合计行也框进来是很自然的操作，当成数据行导入的话求和翻倍，而且干跑、核对、重放比对都看不出来（H2）； + - 合计字样出现在框内别的位置（第一列但不在末尾，或在别的列上）：起草器在这里记起草失败，框选同样报错， + 不把合计当明细导入； + - 合计行下面紧接着还有同一张表的行：它是夹在中间的小计（起草器的 _more_rows），同样报错； + - 末尾不是合计行时，看 last 的下一行（只看框的列范围）：全空或到网格底部就「遇到空行停止」，以合计词开头 + 就是合计行，都不是就报 selection_bottom_not_anchorable。看的是 last+1、不是框的下一行：框底多带几行空行 + 时结果与框到 last 相同，提示里建议的末尾行也从 last+1 往下数，不会让人把下方的备注、制表人行也框进来。""" + r1, c1, r2, c2 = ctx.rect + grid = ctx.grid + filled = [r for r in range(first, r2 + 1) if not _row_empty(grid, r, c1, c2)] + if not filled: + ctx.fail("selection_bottom_not_anchorable", "框里只有表头、没有数据行：请把框扩大到数据的实际末尾") + return False + last = filled[-1] + word = S._total_word(S._label_of(grid.get(last, c1))) + data = filled[:-1] if word is not None else filled + if not data: + ctx.fail("selection_bottom_not_anchorable", + f"框里只有表头和合计行（第 {last} 行），没有数据行：请把框扩大到数据所在的行") + return False + if not _no_stray_totals(ctx, data, c1, c2): + return False + bottom = S._bottom(grid) + acc = S._ColTypes(grid, plan.cols) + for r in data: + acc.add(r) + plan.rows = data + gaps = [r for r in range(first, last + 1) if r not in filled] + if gaps: + # 框内有空行（c08 gap 那种）：跳过空行，执行器跳过空行后照样能碰到末尾的合计行；重放比对把实际的数据区摆出来 + plan.skip_blank, plan.skipped = True, gaps + if word is not None: + return _total_at_end(ctx, plan, last, word, bottom, acc.types()) + below = last + 1 + if below > bottom or _row_empty(grid, below, c1, c2): + return True + word = S._total_word(S._label_of(grid.get(below, c1))) + if word is not None: + return _total_at_end(ctx, plan, below, word, bottom, acc.types()) + end = below + while end <= bottom and not _row_empty(grid, end, c1, c2) and S._total_word(S._label_of(grid.get(end, c1))) is None: + end += 1 + ctx.fail("selection_bottom_not_anchorable", + f"第 {below} 行还有数据：按文字无法表达「到第 {last} 行为止」。请把框扩大到数据的实际末尾（第 {end - 1} 行），" + "改选「下边界按规则推断」，或删掉文件里多出的行", [ctx.coord(below, c1)]) + return False + + +def _no_stray_totals(ctx: _Ctx, data: list[int], c1: int, c2: int) -> bool: + """数据行里不能有合计字样（起草器 _scan_block 的 stray_totals：第一列以合计词开头、或别的列整格以合计词开头）。 + 起草器在这里记起草失败；框选照样不收，否则合计会被当成明细导入、求和翻倍（H2）。""" + grid = ctx.grid + first_col = [r for r in data if S._total_word(S._label_of(grid.get(r, c1))) is not None] + other = [(r, c) for r in data for c in range(c1 + 1, c2 + 1) + if (cell := grid.get(r, c)) is not None and S._total_word(cell.value) is not None] + if first_col: + r = first_col[0] + ctx.fail("selection_bottom_not_anchorable", + f"第 {r} 行像合计行（「{S._label_of(grid.get(r, c1))}」），但它不在框的末尾：无法确定它汇总的是哪几行，" + f"按数据导入会把合计重复计数。合计行就是表的末尾时，请把框的下边移到第 {r} 行；它是夹在中间的小计时，" + "请修改文件", [ctx.coord(x, c1) for x in first_col]) + return False + if other: + r, c = other[0] + ctx.fail("selection_bottom_not_anchorable", + f"第 {r} 行 {col_letter(c)} 列是合计字样（「{grid.get(r, c).value}」），但合计字样不在第一列，无法按合计行" + "核对，按数据导入会把合计重复计数：请修改文件，把合计字样写在第一列，或把框的下边移到这一行之上", + [ctx.coord(x, y) for x, y in other]) + return False + return True + + +def _auto_settled(ctx: _Ctx, plan: Any) -> bool: + """bottom="auto"：起草器收进来、却无法确定的行（紧挨着数据的新表头、不在末尾第一列的合计字样）照起草器记失败 + （recipe_suggest._list_failures，同一套文案）。起草器的 _scan_block 会把这些行照样收进数据行，只是另记失败； + 框选不看失败的话，夹在中间的小计会被当成明细导入、求和翻倍（H2）。""" + failures = S._list_failures(ctx.sheet, ctx.grid, plan) + if not failures: + return True + cells = [ctx.coord(r, c) for r, c in plan.stray_totals] + [ctx.coord(r, plan.cols[0]) for r, _f in plan.adjacent] + for f in failures: + ctx.fail("selection_bottom_not_anchorable", + f"{f.human}。按文字无法表达这张表到哪里为止，按数据导入会出错：请修改文件", cells) + return False + + +def _total_at_end(ctx: _Ctx, plan: Any, r: int, word: str, bottom: int, types: dict[int, str]) -> bool: + """第 r 行是表末尾的合计行：下面紧接着还有同一张表的行时，它是夹在中间的小计（起草器的 _more_rows）， + 按合计行停在这里会把下面的明细丢掉，报错；否则记为合计行（执行器碰到它就停）。""" + if S._more_rows(ctx.grid, r, plan, bottom, types): + ctx.fail("selection_bottom_not_anchorable", + f"第 {r} 行像合计行（「{S._label_of(ctx.grid.get(r, plan.cols[0]))}」），但下面紧接着还有同一张表的数据：" + "它是夹在中间的小计，按文字无法表达这张表到哪里为止，按数据导入又会把小计重复计数。请修改文件", + [ctx.coord(r, plan.cols[0])]) + return False + plan.total_row, plan.total_word = r, word + return True + + +def _select_list(ctx: _Ctx, located: list[_Located]) -> EditResult: + r1, c1, r2, c2 = ctx.rect + grid = ctx.grid + n = ctx.sel.options.get("header_rows", 1) + if not isinstance(n, int) or isinstance(n, bool) or not 1 <= n <= 3: + raise EditRequestError("edit_invalid", "表头行数只能是 1 到 3") + bottom = ctx.sel.options.get("bottom", "box") + if bottom not in ("box", "auto"): + raise EditRequestError("edit_invalid", "下边界只能「以框的下边为准」或「按规则推断」") + rows_h = list(range(r1, r1 + n)) + cols = list(range(c1, c2 + 1)) + if len(cols) < 1 or r2 < r1 + n: + ctx.fail("selection_bottom_not_anchorable", "框里只有表头、没有数据行：请把框扩大到数据的实际末尾") + return _failed(ctx) + # 重叠：恰好一个块时替换它（沿用块 id 和表名，免得关系、单位里的引用断掉），两个以上报错，不相交时新增 + hits = [loc for loc in located if any(_intersects(ctx.rect, r) for r in loc.rects)] + if len(hits) > 1: + ctx.fail("selection_overlap", f"框与{quoted([h.block.get('id') for h in hits])}几块都相交：请只框一张表") + return _failed(ctx) + old = hits[0] if hits else None + if old is not None and old.block.get("layout") != "list": + ctx.fail("selection_overlap", f"框与交叉表「{old.block.get('id')}」相交：列表不能替换交叉表。请调整框的范围") + return _failed(ctx) + texts = _header_problems(ctx, rows_h, cols) + if texts is None: + return _failed(ctx) + ignored = {match_key(x.get("header")) for x in (old.block.get("ignore_columns") or [])} if old is not None else set() + if not _no_extra_columns(ctx, rows_h, cols, ignored): + return _failed(ctx) + plan = S._ListPlan(header_rows=rows_h, cols=cols, headers=dict(texts)) + notes: list[str] = [] + if bottom == "box": + if not _bottom_box(ctx, plan, r1 + n): + return _failed(ctx) + else: + S._scan_block(grid, plan, S._bottom(grid)) + if not plan.rows: + ctx.fail("selection_bottom_not_anchorable", "表头之下没有认出数据行：请检查框的上边是否在表头所在的行") + return _failed(ctx) + if not _auto_settled(ctx, plan): + return _failed(ctx) + last = plan.total_row or plan.rows[-1] + if grid.truncated and last >= S._bottom(grid): + notes.append("数据延续到已读入范围之外，完整范围在试运行时核对") + if grid.truncated and plan.rows[-1] >= S._bottom(grid) and not notes: + notes.append("数据延续到已读入范围之外，完整范围在试运行时核对") + S._finish_block(grid, plan) + old_tables = set() + if old is not None: + old_tables = {old.block.get("table")} + tr = (old.block.get("rows") or {}).get("total_row") or {} + if tr.get("keep_as"): + old_tables.add(tr["keep_as"]) + used = _used_tables(ctx, except_=old_tables) + given = ctx.sel.options.get("table") + table: str | None = None + if given not in (None, ""): + table = _new_table_name(ctx, "", used) + if table is None: + return _failed(ctx) + elif old is not None: + table = old.block.get("table") + block_id = old.block["id"] if old is not None else unique_id(f"列表{_list_count(ctx.recipe) + 1}", + all_ids(ctx.recipe or {})) + block, main, total_spec = S.list_block(ctx.sheet, plan, block_id=block_id, tables_used=used, table=table) + # 表头上方的标题：表头不止一处时才要 + anchors = [Anchor("header", texts[c], ctx.coord(r1 + n - 1, c)) for c in cols] + found = _after_title(ctx, r1, cols, [texts[c] for c in cols], n) + if found is False: + ctx.fail("selection_anchor_ambiguous", + "这组表头在工作表里不止一处，而上方能区分这张表的文字都含年月或数字，下一期很可能对不上：请修改文件，" + "或给这张表加一行不含数字的标题") + return _failed(ctx) + if found: + block["after_title"] = found[0] + anchors.append(Anchor("after_title", found[0], found[1])) + if plan.total_row is not None: + anchors.append(Anchor("total_word", plan.total_word or "", ctx.coord(plan.total_row, c1))) + for c in cols: + if S._YEAR.search(texts[c]): + notes.append(f"表头「{texts[c]}」含年份，下一期年份变化时将无法匹配") + if old is not None and old.block.get("ignore_columns"): + # 被替换的块上原有的按表头忽略照旧保留：框选只决定表头行和列范围，不该悄悄丢掉人确认过的规则 + block["ignore_columns"] = list(old.block["ignore_columns"]) + for col in block["columns"]: + retired = next((x for x in ctx.retired.get(block["table"], []) if collide_key(x) == collide_key(col["name"]) + and x != col["name"]), None) + if retired is not None: + notes.append(f"列「{col['name']}」与已退役的列「{retired}」同名（不区分大小写），沿用原来的写法「{retired}」") + col["name"] = retired + ops = _block_ops(ctx, old, block, main, total_spec, old_tables) + if ops is None: + return _failed(ctx) + total_rect = (plan.total_row, c1, plan.total_row, c2) if plan.total_row else None + expected = {"header": _a1((r1, c1, r1 + n - 1, c2)), "data": _a1((plan.rows[0], c1, plan.rows[-1], c2)), + "total": _a1(total_rect)} + verb = "替换现有的" if old is not None else "新增" + summary = [f"按框选{verb}列表「{block_id}」（表「{block['table']}」）：表头按文字{quoted([texts[c] for c in cols])}定位"] + if found: + summary.append(f"只在文字「{found[0]}」之后找表头") + summary.append("下边界：" + ("到合计行为止" if plan.total_row else "遇到空行停止") + + ("，中间的空行跳过" if plan.skip_blank else "")) + if bottom == "auto": + summary.append(f"重放的数据区到第 {plan.rows[-1]} 行") + return finish(ctx.recipe, ops, kind="selection", key=f"list:{block_id}", + title=f"按框选{verb}列表「{block_id}」：表头按文字定位", summary=summary, anchors=anchors, + notes=notes, expected=expected, block=block_id) + + +def _list_count(recipe: dict[str, Any] | None) -> int: + return sum(1 for s in (recipe or {}).get("sheets") or [] for b in s.get("blocks") or [] if b.get("layout") == "list") + + +def _block_ops(ctx: _Ctx, old: _Located | None, block: dict, main: dict, total_spec: dict | None, + old_tables: set[str]) -> list[dict[str, Any]] | None: + """把一个列表块写进配方：替换旧块（连同表定义）、新增块，或者配方为空时用根替换新建配方（4.2 第 7 条）。""" + recipe = ctx.recipe + if not recipe: + tables = [main] + ([total_spec] if total_spec else []) + return [{"op": "replace", "path": "", "value": { + "recipe_format": RECIPE_FORMAT, "sheets": [{"id": "s1", "match": {"name": ctx.sheet[:31]}, "blocks": [block]}], + "tables": tables, "relations": []}}] + if ctx.si is None: + sid = _new_sheet_id(recipe) + ops: list[dict[str, Any]] = [{"op": "add", "path": "/sheets/-", "value": { + "id": sid, "match": {"name": ctx.sheet[:31]}, "blocks": [block]}}] + return ops + _add_tables(recipe, [main] + ([total_spec] if total_spec else [])) + if old is None: + ops = [{"op": "add", "path": ptr("sheets", ctx.si, "blocks", "-"), "value": block}] + return ops + _add_tables(recipe, [main] + ([total_spec] if total_spec else [])) + # 替换：表定义原位替换（保留用户写的说明），表名变了就是改名；旧块另存的合计表这次不再另存时删掉它的定义 + # （别的块还在写入时不删）。先替换、再删（按下标从大到小）、最后追加，下标不会错位 + ops = [{"op": "replace", "path": ptr("sheets", ctx.si, "blocks", old.bi), "value": block}] + old_total = ((old.block.get("rows") or {}).get("total_row") or {}).get("keep_as") + replaced: set[int] = set() + adds: list[dict] = [] + for spec, was in ((main, old.block.get("table")), (total_spec, old_total)): + if spec is None: + continue + ti = table_index(recipe, was) if was else None + if ti is None: + adds.append(spec) + continue + replaced.add(ti) + ops.append({"op": "replace", "path": ptr("tables", ti), + "value": {**spec, "note": recipe["tables"][ti].get("note") or ""}}) + removes = sorted({ti for name in old_tables if (ti := table_index(recipe, name)) is not None + and ti not in replaced and not _written_elsewhere(recipe, name, ctx.si, old.bi)}, reverse=True) + ops += [{"op": "remove", "path": ptr("tables", ti)} for ti in removes] + ops += [{"op": "add", "path": "/tables/-", "value": spec} for spec in adds] + return ops + + +def _add_tables(recipe: dict, specs: list[dict]) -> list[dict[str, Any]]: + return [{"op": "add", "path": "/tables/-", "value": s} for s in specs] + + +def _new_sheet_id(recipe: dict) -> str: + have = {s.get("id") for s in recipe.get("sheets") or []} + n = len(have) + 1 + while f"s{n}" in have: + n += 1 + return f"s{n}" + + +def _written_elsewhere(recipe: dict, table: str, si: int, bi: int) -> bool: + """除了 (si, bi) 这个块，还有别的块写入这张表吗（替换块时不能删掉别人还在用的表定义）。""" + for sj, sheet in enumerate(recipe.get("sheets") or []): + for bj, block in enumerate(sheet.get("blocks") or []): + if (sj, bj) == (si, bi): + continue + if block.get("layout") == "list": + tr = (block.get("rows") or {}).get("total_row") or {} + if table in (block.get("table"), tr.get("keep_as")): + return True + for seg in block.get("segments") or []: + if table in (seg.get("table"), (seg.get("keep_as") or {}).get("table")): + return True + return False + + +# ========================================================================== +# 交叉表（整块） +# ========================================================================== + + +def _block_tables(block: dict) -> set[str]: + out: set[str] = set() + for seg in block.get("segments") or []: + if seg.get("table"): + out.add(seg["table"]) + if (seg.get("keep_as") or {}).get("table"): + out.add(seg["keep_as"]["table"]) + return out + + +def _rel_tables(rel: dict) -> set[str]: + if rel.get("kind") == "sum_eq": + return {rel.get("table")} + if rel.get("kind") == "not_comparable": + return {(rel.get("a") or {}).get("table"), (rel.get("b") or {}).get("table")} + return set() + + +def _select_crosstab(ctx: _Ctx, located: list[_Located], draft_fn: Callable[[], Draft] | None) -> EditResult: + """框只用来指认是哪一块：draft_fn() 对整张工作表起草，取认领区域与框重叠最多（且 ≥50%）的交叉表块，连同它写入的表、 + 它认领的事实对应的关系、统计期上下文（配方里还没有时）一起并入。关系 id 与已有的重号时，后面的顺延（4.2）。""" + d = draft_fn() if draft_fn is not None else None + dr = d.recipe if d is not None else None + no = ("框住的范围里没有认出交叉表（日期表头加行标签）：请框在交叉表上，或改选「列表」") + if not dr: + ctx.fail("selection_no_crosstab", no) + return _failed(ctx) + try: + dmodel = Recipe.model_validate(dr) + except ValidationError: + ctx.fail("selection_no_crosstab", no) + return _failed(ctx) + best: tuple[int, dict, dict, Any] | None = None + for dsi, dsheet in enumerate(dmodel.sheets): + if match_key(dsheet.match.name) != match_key(ctx.sheet): + continue + for dbi, dblock in enumerate(dsheet.blocks): + if not isinstance(dblock, CrosstabBlock): + continue + loc = S._locate_crosstab(dblock, ctx.grid) + box = _cells_bbox(loc.region) if loc is not None else None + if box is None: + continue + ov = _overlap(ctx.rect, box) + if ov * 1.0 >= CROSSTAB_OVERLAP * _area(ctx.rect) and (best is None or ov > best[0]): + best = (ov, dr["sheets"][dsi], dr["sheets"][dsi]["blocks"][dbi], loc) + if best is None: + ctx.fail("selection_no_crosstab", no) + return _failed(ctx) + _ov, dsheet_d, dblock_d, dloc = best + region = _cells_bbox(dloc.region) + hits = [loc for loc in located if any(_intersects(region, r) for r in loc.rects)] # type: ignore[arg-type] + crosses = [loc for loc in located if loc.block.get("layout") == "crosstab"] + if len(hits) > 1 or (hits and hits[0].block.get("layout") != "crosstab"): + ctx.fail("selection_overlap", f"这块交叉表与{quoted([h.block.get('id') for h in hits])}相交：请先调整或删掉那些块") + return _failed(ctx) + old = hits[0] if hits else None + if old is None and crosses: + ctx.fail("selection_overlap", + f"这个工作表已经有交叉表「{crosses[0].block.get('id')}」：一个工作表只能有一个交叉表，请框在它上面以替换它") + return _failed(ctx) + recipe = ctx.recipe + block = copy.deepcopy(dblock_d) + btables = _block_tables(block) + rels = [copy.deepcopy(r) for r in dr.get("relations") or [] if _rel_tables(r) and _rel_tables(r) <= btables] + tables = [copy.deepcopy(t) for t in dr.get("tables") or [] if t.get("name") in btables] + # 被替换的交叉表只由它自己写入的表，连同引用这些表的关系一起换掉 + drop = ({t for t in _block_tables(old.block) if not _written_elsewhere(recipe, t, ctx.si, old.bi)} + if recipe and old is not None and ctx.si is not None else set()) + if recipe and old is not None: + # 替换已有的交叉表：先把起草器的名字按来源对齐到现行配方（6.3 的 align_to_contract），版式没变时得到的 + # 就是原来的配方，表名、列名、分段 id、关系编号都不会因为换了一种定位方式而变 + sheet = recipe["sheets"][ctx.si] # type: ignore[index] + cand = {"recipe_format": RECIPE_FORMAT, "mode": recipe.get("mode", "replace"), + "sheets": [{"id": sheet.get("id"), "match": sheet.get("match"), + "context": list(dsheet_d.get("context") or []), "blocks": [block]}], + "tables": tables, "relations": rels} + aligned, _report = S.align_to_contract(cand, recipe) + block = aligned["sheets"][0]["blocks"][0] + tables, rels = aligned["tables"], aligned["relations"] + btables = _block_tables(block) + if recipe: + old_ids = ({old.block.get("id")} | {s.get("id") for s in old.block.get("segments") or []}) if old else set() + used_ids = all_ids(recipe) - old_ids + used = _used_tables(ctx, except_=drop) + rename = {t: unique_name(t, used) for t in sorted(btables)} + seg_rename = {seg["id"]: unique_id(seg["id"], used_ids) for seg in block["segments"]} + block["id"] = old.block["id"] if old is not None else unique_id(block["id"], used_ids) + _rename_block(block, rename, seg_rename) + for t in tables: + t["name"] = rename.get(t["name"], t["name"]) + # 关系 id 与留下来的关系重号时，后面的顺延 + left = {str(r.get("id")) for r in recipe.get("relations") or [] if not (_rel_tables(r) & drop)} + next_n = max([int(i[1:]) for i in left if i[1:].isdigit()] + [0]) + for r in rels: + _rename_rel(r, rename) + if r["id"] in left: + next_n += 1 + while f"R{next_n}" in left: + next_n += 1 + r["id"] = f"R{next_n}" + left.add(r["id"]) + expected_rows = sorted({r for rows in dloc.seg_rows.values() for r in rows}) + lc = dloc.label_col + expected = {"axis": _a1((dloc.axis_row, dloc.c1, dloc.axis_row, dloc.c2)), + "labels": _a1((expected_rows[0], lc, expected_rows[-1], lc)) if expected_rows else None, + "values": _a1((expected_rows[0], dloc.c1, expected_rows[-1], dloc.c2)) if expected_rows else None} + anchors = [Anchor("axis", _text_of(ctx.grid, dloc.axis_row, dloc.c1) or str(ctx.grid.get(dloc.axis_row, dloc.c1).value), + ctx.coord(dloc.axis_row, dloc.c1))] + for sid, tr in dloc.title_rows.items(): + anchors.append(Anchor("section_title", _text_of(ctx.grid, tr, lc) or sid, ctx.coord(tr, lc))) + context = list(dsheet_d.get("context") or []) + if not recipe: + root = {"recipe_format": RECIPE_FORMAT, "mode": dr.get("mode", "replace"), + "sheets": [{"id": "s1", "match": dict(dsheet_d.get("match") or {"name": ctx.sheet[:31]}), + "context": context, "blocks": [block]}], + "tables": tables, "relations": rels} + ops: list[dict[str, Any]] = [{"op": "replace", "path": "", "value": root}] + elif ctx.si is None: + ops = [{"op": "add", "path": "/sheets/-", "value": { + "id": _new_sheet_id(recipe), "match": {"name": ctx.sheet[:31]}, "context": context, "blocks": [block]}}] + ops += _add_tables(recipe, tables) + [{"op": "add", "path": "/relations/-", "value": r} for r in rels] + else: + if old is not None: + ops = [{"op": "replace", "path": ptr("sheets", ctx.si, "blocks", old.bi), "value": block}] + else: + ops = [{"op": "add", "path": ptr("sheets", ctx.si, "blocks", "-"), "value": block}] + tis = sorted({ti for t in drop if (ti := table_index(recipe, t)) is not None}, reverse=True) + ris = sorted({i for i, r in enumerate(recipe.get("relations") or []) if _rel_tables(r) & drop}, reverse=True) + ops += [{"op": "remove", "path": ptr("tables", ti)} for ti in tis] + ops += [{"op": "remove", "path": ptr("relations", ri)} for ri in ris] + ops += _add_tables(recipe, tables) + [{"op": "add", "path": "/relations/-", "value": r} for r in rels] + if context and not recipe["sheets"][ctx.si].get("context"): + ops.append({"op": "add", "path": ptr("sheets", ctx.si, "context"), "value": context}) + verb = "替换现有的" if old is not None else "新增" + summary = [f"按框选{verb}交叉表「{block['id']}」：日期表头、行标签、分段标题按文字定位", + f"写入表{quoted([t['name'] for t in tables])}"] + if rels: + summary.append(f"登记系统发现的关系{quoted([r['id'] for r in rels])}") + return finish(recipe, ops, kind="selection", key=f"crosstab:{block['id']}", + title=f"按框选{verb}交叉表「{block['id']}」：按日期表头和行标签定位", summary=summary, + anchors=anchors, expected=expected, block=block["id"]) + + +def _rename_block(block: dict, tables: dict[str, str], segs: dict[str, str]) -> None: + for seg in block.get("segments") or []: + seg["id"] = segs.get(seg["id"], seg["id"]) + if seg.get("table"): + seg["table"] = tables.get(seg["table"], seg["table"]) + loc = seg.get("locate") or {} + if loc.get("segment"): + loc["segment"] = segs.get(loc["segment"], loc["segment"]) + if seg.get("verify"): + seg["verify"]["against_table"] = tables.get(seg["verify"]["against_table"], seg["verify"]["against_table"]) + if seg.get("keep_as"): + seg["keep_as"]["table"] = tables.get(seg["keep_as"]["table"], seg["keep_as"]["table"]) + + +def _rename_rel(rel: dict, tables: dict[str, str]) -> None: + if rel.get("table"): + rel["table"] = tables.get(rel["table"], rel["table"]) + for side in ("a", "b"): + if isinstance(rel.get(side), dict): + rel[side]["table"] = tables.get(rel[side]["table"], rel[side]["table"]) + + +# ========================================================================== +# 分段、合计行、分段标题 +# ========================================================================== + + +def _seg_rows(x: Any) -> dict[str, list[int]]: + return {sid: list(rows) for sid, rows in x.seg_rows.items()} + + +def _select_segment(ctx: _Ctx, located: list[_Located]) -> EditResult: + loc = _crosstab_at(ctx, located) + if loc is None: + ctx.fail("selection_not_in_block", "框要落在交叉表的标签列上（可以带上右边的数据格），并在日期表头之下") + return _failed(ctx) + role = ctx.sel.options.get("role") + if role not in ("measures", "dimension"): + raise EditRequestError("edit_invalid", "框选为分段时要选是指标（每行一个指标）还是维度（每行一个取值）") + x = loc.cross + r1, _c1, r2, _c2 = ctx.rect + labels = _labels(ctx, x.label_col) + if labels is None: + return _failed(ctx) + taken = {r: sid for sid, rows in _seg_rows(x).items() for r in rows} + taken.update({r: sid for sid, r in x.title_rows.items()}) + clash = sorted({taken[r] for r, _ in labels if r in taken}) + if clash: + ctx.fail("selection_overlap", f"框里的行已经属于分段{quoted(clash)}：请只框还没有分段认领的行") + return _failed(ctx) + title_row = r1 - 1 + title = _text_of(ctx.grid, title_row, x.label_col) + if title is None or not _row_empty(ctx.grid, title_row, x.c1, x.c2) or title_row in taken or len(title) > 40: + title, title_row = None, None + texts = [t for _, t in labels] + block = loc.block + axis = (block.get("axis") or {}).get("name", "日期") + used_ids = all_ids(ctx.recipe or {}) + used = _used_tables(ctx) + notes: list[str] = [] + if role == "measures": + base = to_sql_name(ctx.sheet, fallback=f"表{len(used) + 1}")[:45] + table = _new_table_name(ctx, f"{base}_按日", used) + if table is None: + return _failed(ctx) + cols_used = {collide_key(axis)} + measures: dict[str, str] = {} + units: dict[str, str] = {} + for i, lab in enumerate(texts): + col, unit, _raw = S._measure_column(lab, i + 2) + col = unique_name(col, cols_used) + measures[lab] = col + if unit: + units[col] = unit + seg = {"id": unique_id(table, used_ids), "role": "measures", "table": table, + "locate": {"by": "section_title", "title": title, "segment": None} if title + else {"by": "labels", "title": None, "segment": None}, + "labels": {"expect": texts}, "measures": measures} + spec = {"name": table, "grain": [axis], "kind": "data", "units": units, "note": ""} + else: + parser = "hour_range" if all(hour_range(t) is not None for t in texts) else "text" + bad = [t for t in texts if parser == "text" and (text_label(t) is None or len(t) > LABEL_MAX)] + if bad: + ctx.fail("selection_label_unparsed", f"标签{quoted(bad)}为空或超过 {LABEL_MAX} 字，无法作维度的取值") + return _failed(ctx) + merged, blocked = _merge_target(ctx, block, parser, texts, title) + if merged is not None: + return _merged_segment(ctx, loc, merged, texts, title, title_row, labels) + if blocked: + notes.append(blocked) + want = split_unit_suffix(title)[0] if title else "" + n_fallback = len(used) + 1 + plain = (to_sql_name(want, fallback=f"表{n_fallback}") if want else f"表{n_fallback}")[:48] + table = _new_table_name(ctx, plain, used) + if table is None: + return _failed(ctx) + dim = S._HOUR_DIM if parser == "hour_range" else S._TEXT_DIM + value = to_sql_name(plain[-2:], fallback="数值") if len(plain) >= 2 else "数值" + reserved = {collide_key(n) for n in (axis, dim, *(S._DERIVE if parser == "hour_range" else ()))} + if collide_key(value) in reserved or len(value) < 2: + value = "数值" + unit_raw = split_unit_suffix(title)[1] if title else None + seg = {"id": unique_id(table, used_ids), "role": "dimension", "table": table, + "locate": {"by": "section_title", "title": title, "segment": None} if title + else {"by": "labels", "title": None, "segment": None}, + "labels": {"expect": texts}, + "dim": {"name": dim, "parser": parser, "derive": dict(S._DERIVE) if parser == "hour_range" else {}}, + "value": value, "const": {}, "stop_parser": None} + units = {value: unit_raw} if unit_raw and S.P.unit_known(unit_raw) else {} + spec = {"name": table, "grain": [axis, dim], "kind": "data", "units": units, "note": ""} + ops = [{"op": "add", "path": _segment_path(ctx, loc), "value": seg}, {"op": "add", "path": "/tables/-", "value": spec}] + kind = "指标" if role == "measures" else "维度" + return _segment_result(ctx, loc, seg, ops, kind=kind, texts=texts, title=title, title_row=title_row, + labels=labels, where=f"写入新表「{table}」", notes=notes) + + +def _segment_path(ctx: _Ctx, loc: _Located) -> str: + """新分段插在按行号排在它后面的第一个分段之前；合计分段要紧跟它的基础分段，不能插进两者之间。""" + x = loc.cross + r1 = ctx.rect[0] + segs = loc.block.get("segments") or [] + rows_of = _seg_rows(x) + idx = len(segs) + for i, s in enumerate(segs): + rows = rows_of.get(s.get("id")) or ([x.title_rows[s["id"]]] if s.get("id") in x.title_rows else []) + if rows and min(rows) > r1: + idx = i + break + while idx < len(segs) and segs[idx].get("role") == "derived": + idx += 1 + return ptr("sheets", ctx.si, "blocks", loc.bi, "segments", idx) + + +def _segment_result(ctx: _Ctx, loc: _Located, seg: dict, ops: list[dict[str, Any]], *, kind: str, texts: list[str], + title: str | None, title_row: int | None, labels: list[tuple[int, str]], where: str, + notes: list[str], extra: list[str] | None = None) -> EditResult: + x = loc.cross + r1, _c1, r2, _c2 = ctx.rect + anchors = [Anchor("row_label", t, ctx.coord(r, x.label_col)) for r, t in labels] + if title: + anchors.insert(0, Anchor("section_title", title, ctx.coord(title_row, x.label_col))) # type: ignore[arg-type] + expected = {"labels": _a1((r1, x.label_col, r2, x.label_col)), "values": _a1((r1, x.c1, r2, x.c2)), + "title": _a1((title_row, x.label_col, title_row, x.label_col)) if title else None} + how = f"分段标题「{title}」" if title else f"标签{quoted(texts[:5])}" + ("等" if len(texts) > 5 else "") + return finish(ctx.recipe, ops, kind="selection", key=f"segment:{seg['id']}", + title=f"按框选新增{kind}分段「{seg['id']}」：按{'分段标题' if title else '标签'}定位", + summary=[f"交叉表「{loc.block['id']}」新增{kind}分段「{seg['id']}」，按{how}定位，{where}", *(extra or [])], + anchors=anchors, notes=notes, expected=expected, block=loc.block.get("id")) + + +@dataclass +class _Merge: + table: str + template: dict # 那张表里第一个分段（dim、value 照抄它） + col: str # 常量列名 + pick: str # 新分段的常量取值（候选词的第一个，同起草器第 8b 条） + + +def _dim_unit(title: str | None) -> str | None: + return split_unit_suffix(title)[1] if title else None + + +def _merge_target(ctx: _Ctx, block: dict, parser: str, texts: list[str], + title: str | None) -> tuple[_Merge | None, str]: + """起草器第 5 条（P2-SPEC 6.1，recipe_suggest._group_dimensions）：dim.parser 相同、标签规范写法不相交、单位 + 相同的维度分段写进同一张表，各段用常量列区分。框选新的维度分段时照这条找可以并入的表（同一块里、按分段 + 顺序的第一张），常量取值照起草器第 8b 条取 candidate_words(新标题, 兄弟标题) 的第一个，并且要与已有的取值 + 不同（起草器「选不出互不相同的常量就各成一张表」）。返回 (可以并入的表, 说明)： + + - 能并入：(_Merge, "")； + - 有 parser、单位、标签都对得上的表，却并不进去（客户端指定了新表名；那张表的分段没有常量列，并入要给现有 + 分段加列；分段标题里选不出与已有取值不同的词）：(None, 说明)，说明写进 notes——新建了表、没有并入， + 要让人看见； + - 没有对得上的表：(None, "")。 + 只在同一块里找，并且那张表的写入方必须全在这一块里：跨块、跨工作表合表不是起草器的规则。""" + keys = {S._dim_key(parser, t) for t in texts} + unit = _dim_unit(title) + segs = [s for s in block.get("segments") or [] if s.get("role") == "dimension"] + order: list[str] = [] + for s in segs: + if s.get("table") not in order: + order.append(s.get("table")) + for table in order: + group = [s for s in segs if s.get("table") == table] + if any((s.get("dim") or {}).get("parser") != parser or _dim_unit((s.get("locate") or {}).get("title")) != unit + for s in group): + continue + if any(keys & {S._dim_key(parser, e) for e in (s.get("labels") or {}).get("expect") or []} for s in group): + continue + if _written_by_others(ctx.recipe or {}, table, block): + continue + given = ctx.sel.options.get("table") + if given not in (None, ""): + return None, f"按指定的表名新建了表，没有并入同类的表「{table}」" + cols = {tuple((s.get("const") or {}).keys()) for s in group} + if len(cols) != 1 or len(next(iter(cols))) != 1: + return None, (f"没有并入同类的表「{table}」：它的分段没有用来区分分段的常量列，并入需要给现有分段加列。" + "需要并入时请在配方面板中修改") + col = next(iter(cols))[0] + picks = {canon(((s.get("const") or {}).get(col) or {}).get("pick") or "") for s in group} + sibs = [t for s in group if (t := (s.get("locate") or {}).get("title"))] + words = candidate_words(title, sibs) if title else [] + if not words or canon(words[0]) in picks: + return None, (f"没有并入同类的表「{table}」：分段标题里没有能与已有分段区分的词作「{col}」的取值。" + "需要并入时请在配方面板中修改") + return _Merge(table, group[0], col, words[0]), "" + return None, "" + + +def _written_by_others(recipe: dict, table: str, block: dict) -> bool: + """这张表还有这一块以外的写入方（别的块、列表、合计另存）。""" + for sheet in recipe.get("sheets") or []: + for b in sheet.get("blocks") or []: + if b is block: + continue + if b.get("layout") == "list": + tr = (b.get("rows") or {}).get("total_row") or {} + if table in (b.get("table"), tr.get("keep_as")): + return True + for seg in b.get("segments") or []: + if table in (seg.get("table"), (seg.get("keep_as") or {}).get("table")): + return True + return False + + +def _merged_segment(ctx: _Ctx, loc: _Located, m: _Merge, texts: list[str], title: str | None, + title_row: int | None, labels: list[tuple[int, str]]) -> EditResult: + """并入已有的表：dim、值列照抄那张表的分段，常量列取 m.pick，不新增表定义。""" + t = m.template + seg = {"id": unique_id(m.pick, all_ids(ctx.recipe or {})), "role": "dimension", "table": m.table, + "locate": {"by": "section_title", "title": title, "segment": None}, + "labels": {"expect": texts}, "dim": copy.deepcopy(t["dim"]), "value": t["value"], + "const": {m.col: {"pick": m.pick}}, "stop_parser": None} + ops = [{"op": "add", "path": _segment_path(ctx, loc), "value": seg}] + return _segment_result(ctx, loc, seg, ops, kind="维度", texts=texts, title=title, title_row=title_row, + labels=labels, where=f"并入表「{m.table}」", notes=[], + extra=[f"「{m.col}」取「{m.pick}」（取自分段标题，与同表其他分段区分）"]) + + +def _select_derived(ctx: _Ctx, located: list[_Located]) -> EditResult: + loc = _crosstab_at(ctx, located) + if loc is None: + ctx.fail("selection_not_in_block", "框要落在交叉表的标签列上（可以带上右边的数据格），并在日期表头之下") + return _failed(ctx) + x = loc.cross + r1, _c1, r2, _c2 = ctx.rect + labels = _labels(ctx, x.label_col) + if labels is None: + return _failed(ctx) + bad = [t for _, t in labels if hour_range_total(t) is None] + if bad: + ctx.fail("selection_label_unparsed", f"标签{quoted(bad)}无法解析为带区间的合计（如「18-22时合计」）") + return _failed(ctx) + keep = ctx.sel.options.get("keep", True) + if not isinstance(keep, bool): + raise EditRequestError("edit_invalid", "合计行是否另存只能选是或否") + box_rows = set(range(r1, r2 + 1)) + rows_of = _seg_rows(x) + others = {r for rows in rows_of.values() for r in rows} + segs = loc.block.get("segments") or [] + gi = None + for i, s in enumerate(segs): + rows = set(rows_of.get(s.get("id")) or []) + # 框里的行必须紧跟这个分段的最后一行。起草器定位时会把紧跟的、有数据的合计行也收进分段（它还不知道该停）， + # 所以「收进来的」也算紧跟 + if s.get("role") == "dimension" and (r1 - 1) in rows and all(r in rows or r not in others for r in box_rows): + gi = i + break + if gi is None: + ctx.fail("selection_not_after_segment", "框里的行要紧跟在一个按时段展开的分段的最后一行之后") + return _failed(ctx) + why = total_blockers(ctx.recipe, ctx.si, loc.bi, gi) # type: ignore[arg-type] + if why: + ctx.fail("selection_not_after_segment", why) + return _failed(ctx) + texts = [t for _, t in labels] + ops, summary, _kept = total_ops(ctx.recipe, ctx.si, loc.bi, gi, texts, keep) # type: ignore[arg-type] + new_id = ops[1]["value"]["id"] + expected = {"labels": _a1((r1, x.label_col, r2, x.label_col)), "values": _a1((r1, x.c1, r2, x.c2))} + return finish(ctx.recipe, ops, kind="selection", key=f"derived:{new_id}", + title=f"按框选把{quoted(texts)}改作合计核对", summary=summary, + anchors=[Anchor("total_word", t, ctx.coord(r, x.label_col)) for r, t in labels], + expected=expected, block=loc.block.get("id")) + + +def _select_section_title(ctx: _Ctx, located: list[_Located]) -> EditResult: + r1, c1, r2, c2 = ctx.rect + loc = _crosstab_at(ctx, located) + if loc is None or (r1, c1) != (r2, c2): + ctx.fail("selection_not_in_block", "分段标题要框交叉表标签列上的一格") + return _failed(ctx) + seg_id = ctx.sel.options.get("segment") + segs = loc.block.get("segments") or [] + gi = next((i for i, s in enumerate(segs) if s.get("id") == seg_id), None) + if gi is None or segs[gi].get("role") == "derived": + raise EditRequestError("edit_invalid", "要指明是这个交叉表里哪个分段（指标或维度分段）的标题") + seg = segs[gi] + text = _text_of(ctx.grid, r1, c1) + if text is None or len(text) > 40: + ctx.fail("selection_label_blank", "这一格没有文字（或超过 40 字），不能作分段标题") + return _failed(ctx) + sp = ("sheets", ctx.si, "blocks", loc.bi, "segments", gi) + ops: list[dict[str, Any]] = [{"op": "replace", "path": ptr(*sp, "locate"), + "value": {"by": "section_title", "title": text, "segment": None}}] + summary = [f"分段「{seg_id}」改按分段标题「{text}」定位"] + consts = seg.get("const") or {} + if consts and not keep_ok(seg, text): + # 常量沿用不了：常量列整列的值要换成新标题里的哪个词，只能由人选（4.1「不猜」）。框选不替人挑第一个 + # 候选词——那是破坏性变更，按期累积时还要重新开始累积。④ 的修复按钮把候选词逐个列成选项，配方面板也能选 + held = "、".join(f"「{k}」现在取「{v.get('pick')}」" for k, v in consts.items()) + ctx.fail("selection_label_unparsed", + f"分段「{seg_id}」的常量列{held}，新标题「{text}」里没有这个词，无法沿用；改用哪个词需要人工选择。" + "请在配方面板中修改取值；该分段报了找不到分段标题时，也可以用问题旁的修复按钮选词", + [ctx.coord(r1, c1)]) + return _failed(ctx) + if consts: + summary.append("、".join(f"「{k}」沿用「{v.get('pick')}」" for k, v in consts.items())) + return finish(ctx.recipe, ops, kind="selection", key=f"section_title:{seg_id}", + title=f"按框选把「{text}」作为分段「{seg_id}」的分段标题", summary=summary, + anchors=[Anchor("section_title", text, ctx.coord(r1, c1))], + expected={"title": _a1(ctx.rect)}, block=loc.block.get("id")) + + +# ========================================================================== +# 忽略 +# ========================================================================== + + +def _digits_fail(ctx: _Ctx, texts: list[str], cells: list[str]) -> None: + ctx.fail("selection_anchor_has_digits", + f"{quoted(texts)}含数字，下一期很可能对不上，无法按文字忽略：请修改文件，或把它框选为新的分段", cells) + + +def _select_ignore_rows(ctx: _Ctx, located: list[_Located]) -> EditResult: + reason = _reason(ctx.sel) + loc = _crosstab_at(ctx, located) + if loc is None: + ctx.fail("selection_not_in_block", "要忽略的行要框在交叉表的标签列上（可以带上右边的数据格），并在日期表头之下") + return _failed(ctx) + x = loc.cross + labels = _labels(ctx, x.label_col) + if labels is None: + return _failed(ctx) + digits = [(r, t) for r, t in labels if has_digit(t) or len(t) > _ANCHOR_MAX] + if digits: + _digits_fail(ctx, [t for _, t in digits], [ctx.coord(r, x.label_col) for r, _ in digits]) + return _failed(ctx) + have = {match_key(i.get("label")) for i in loc.block.get("ignore_rows") or []} + texts: list[str] = [] + for _r, t in labels: + if match_key(t) not in have: + have.add(match_key(t)) + texts.append(t) + bp = ("sheets", ctx.si, "blocks", loc.bi) + ops = [{"op": "add", "path": ptr(*bp, "ignore_rows", "-"), "value": {"label": t, "reason": reason}} for t in texts] + r1, _c1, r2, _c2 = ctx.rect + return finish(ctx.recipe, ops, kind="selection", key=f"ignore_rows:{loc.block['id']}", + title=f"按框选按行标签忽略{quoted([t for _, t in labels])}", + summary=[f"交叉表「{loc.block['id']}」按行标签忽略{quoted(texts)}（理由：{reason}）"] if texts + else ["这些行已经按行标签忽略，配方不变"], + anchors=[Anchor("row_label", t, ctx.coord(r, x.label_col)) for r, t in labels], + expected={"ignored": _a1((r1, x.label_col, r2, x.label_col))}, block=loc.block.get("id")) + + +def _select_ignore_columns(ctx: _Ctx, located: list[_Located]) -> EditResult: + reason = _reason(ctx.sel) + r1, c1, r2, c2 = ctx.rect + target: tuple[_Located, int, dict[int, str]] | None = None + for loc in located: + if loc.cross is not None and c1 > loc.cross.c2 and loc.cross.axis_row <= r2: + x = loc.cross + texts = {c: t for c in range(c1, c2 + 1) if (t := _text_of(ctx.grid, x.axis_row, c))} + target = (loc, x.axis_row, texts) + break + if loc.lst is not None and loc.lst.cols and loc.lst.cols[0] <= c1 and c2 <= loc.lst.cols[-1]: + hr = loc.lst.header_rows + texts = S._header_texts(ctx.grid, hr, list(range(c1, c2 + 1))) + target = (loc, hr[-1], texts) + break + if target is None: + ctx.fail("selection_not_in_block", "要忽略的列要框在交叉表最后一个日期右侧，或列表的表头范围之内") + return _failed(ctx) + loc, hrow, texts = target + cols = list(range(c1, c2 + 1)) + blank = [c for c in cols if c not in texts and ctx.grid.get(hrow, c) is None] + if blank: + ctx.fail("selection_header_blank", f"{'、'.join(col_letter(c) for c in blank)} 列没有表头，无法按表头忽略", + [ctx.coord(hrow, c) for c in blank]) + return _failed(ctx) + bad = [c for c in cols if c not in texts or not _textual(ctx.grid, hrow, c) + or month_day_or_date(texts[c]) is not None] + if bad: + ctx.fail("selection_header_not_text", f"{'、'.join(col_letter(c) for c in bad)} 列的表头是数字或日期，无法按表头忽略", + [ctx.coord(hrow, c) for c in bad]) + return _failed(ctx) + digits = [c for c in cols if has_digit(texts[c]) or len(texts[c]) > _HEADER_MAX] + if digits: + _digits_fail(ctx, [texts[c] for c in digits], [ctx.coord(hrow, c) for c in digits]) + return _failed(ctx) + have = {match_key(i.get("header")) for i in loc.block.get("ignore_columns") or []} + heads: list[str] = [] + for c in cols: + if match_key(texts[c]) not in have: + have.add(match_key(texts[c])) + heads.append(texts[c]) + bp = ("sheets", ctx.si, "blocks", loc.bi) + ops = [{"op": "add", "path": ptr(*bp, "ignore_columns", "-"), "value": {"header": h, "reason": reason}} + for h in heads] + return finish(ctx.recipe, ops, kind="selection", key=f"ignore_columns:{loc.block['id']}", + title=f"按框选按表头忽略{quoted([texts[c] for c in cols])}", + summary=[f"块「{loc.block['id']}」按表头忽略{quoted(heads)}（理由：{reason}）"] if heads + else ["这些列已经按表头忽略，配方不变"], + anchors=[Anchor("header", texts[c], ctx.coord(hrow, c)) for c in cols], + expected={"ignored": _a1((hrow, c1, hrow, c2))}, block=loc.block.get("id")) + + +def _select_ignore_outside(ctx: _Ctx, located: list[_Located]) -> EditResult: + reason = _reason(ctx.sel) + if ctx.si is None: + ctx.fail("selection_not_in_block", "这个工作表还没有导入区域：请先框选要导入的表") + return _failed(ctx) + r1, c1, r2, c2 = ctx.rect + rects = [r for loc in located for r in loc.rects] + inside = lambda r, c: any(a[0] <= r <= a[2] and a[1] <= c <= a[3] for a in rects) # noqa: E731 + found: list[tuple[int, int, str]] = [] + bare: list[int] = [] + for r in range(r1, r2 + 1): + hit = next(((c, t) for c in range(c1, c2 + 1) if not inside(r, c) and (t := _text_of(ctx.grid, r, c)) + and not looks_numeric_text(t)), None) + if hit is None: + bare.append(r) + else: + found.append((r, hit[0], hit[1])) + if bare: + ctx.fail("selection_no_anchor", f"{'、'.join(f'第 {r} 行' for r in bare[:8])}在导入区域之外没有文字格，" + "没有可以作锚点的文字：只能修改文件", + [ctx.coord(r, c1) for r in bare]) + return _failed(ctx) + digits = [(r, c, t) for r, c, t in found if has_digit(t) or len(t) > _ANCHOR_MAX] + if digits: + _digits_fail(ctx, [t for *_x, t in digits], [ctx.coord(r, c) for r, c, _t in digits]) + return _failed(ctx) + have = {match_key(i.get("anchor")) for i in ctx.recipe["sheets"][ctx.si].get("ignore_outside") or []} # type: ignore[index] + texts: list[str] = [] + for _r, _c, t in found: + if match_key(t) not in have: + have.add(match_key(t)) + texts.append(t) + ops = [{"op": "add", "path": ptr("sheets", ctx.si, "ignore_outside", "-"), "value": {"anchor": t, "reason": reason}} + for t in texts] + sid = ctx.recipe["sheets"][ctx.si]["id"] # type: ignore[index] + return finish(ctx.recipe, ops, kind="selection", key=f"ignore_outside:{sid}", + title=f"按框选忽略同一行有{quoted([t for *_x, t in found])}的导入区域之外的数字", + summary=[f"工作表「{ctx.sheet}」：同一行有{quoted(texts)}时，忽略这一行导入区域之外的数字（理由：{reason}）"] + if texts else ["这些行已经按文字忽略，配方不变"], + anchors=[Anchor("outside_text", t, ctx.coord(r, c)) for r, c, t in found], + expected={"ignored": _a1(ctx.rect)}, block=None) + + +# ========================================================================== +# 入口 +# ========================================================================== + + +def selection_edit(recipe: dict[str, Any] | None, grid: Grid, sel: Selection, *, + extraction: Extraction | None, draft_fn: Callable[[], Draft] | None = None, + retired_names: dict[str, list[str]] | None = None) -> EditResult: + """一次框选 → EditResult（补丁、锚点、期望区域）。 + + - recipe：工作配方的**完整形式**（同 fix_ops），配方为空（规则起草失败）时是 None：list、crosstab 用根替换新建配方， + 这样框选也是起草失败时不用 AI 的兜底； + - grid：该工作表的 read_grid(max_rows=draft_max_rows)，和起草用的同一份； + - extraction：当前工作配方最近一次干跑的结果（判断选区和已有的块是否重叠；带块 id 时按它取各块的区域）； + - draft_fn：只在 as=crosstab 时用，接口层注入一个对当前文件跑规则起草的闭包； + - retired_names：当前版本里退役的表名 → 该表退役的列名（取自快照清单的 columns；整张表退役时键在、值是它的列）， + 新表名的撞名检查要算上这些键，新列名与退役列只差大小写时沿用退役列的写法（SQLite 列名不分大小写，并存会撞）。 + 选项写错（header_rows 不在 1–3、role 不认识……）抛 EditRequestError（edit_invalid）；忽略类没写理由抛 + EditRequestError（reason_required）。框和网格、配方的关系泛化不了时返回 ok=False，problems 写原因（4.3）。""" + rect = _rect(sel.ref) + parsed: Recipe | None = None + if recipe: + try: + parsed = Recipe.model_validate(recipe) + except ValidationError: + parsed = None + ctx = _Ctx(recipe=recipe or None, parsed=parsed, grid=grid, sel=sel, rect=rect, ext=extraction, + retired={str(k): list(v or []) for k, v in (retired_names or {}).items()}) + rows = grid.row_numbers() + if not rows or (grid.truncated and rect[2] > rows[-1]): + ctx.fail("selection_out_of_grid", "框超出了已读入的范围（每张工作表只读入前 500 行）：请框在已显示的范围之内，" + "大表的列表可以改选「下边界按规则推断」") + return _failed(ctx) + if recipe and parsed is None and sel.as_ not in ("list", "crosstab"): + ctx.fail("selection_not_in_block", "当前的配方还有格式问题，无法在它上面框选：请先在配方面板中修改") + return _failed(ctx) + ctx.si = _sheet_of(ctx.recipe, grid, extraction) + located = _locate(ctx) + if sel.as_ == "list": + return _select_list(ctx, located) + if sel.as_ == "crosstab": + return _select_crosstab(ctx, located, draft_fn) + if ctx.si is None and sel.as_ != "ignore_outside": + ctx.fail("selection_not_in_block", "这个工作表还没有交叉表或列表：请先框选要导入的表") + return _failed(ctx) + handler = {"segment": _select_segment, "derived": _select_derived, "section_title": _select_section_title, + "ignore_rows": _select_ignore_rows, "ignore_columns": _select_ignore_columns, + "ignore_outside": _select_ignore_outside}.get(sel.as_) + if handler is None: + raise EditRequestError("edit_invalid", "框选没有指明选中的是什么，或取值不在可选范围内") + return handler(ctx, located) + + +# ========================================================================== +# 重放比对（4.5） +# ========================================================================== + +#: 期望区域的键 → 干跑的格子角色（RegionMark.role） +_ROLES = { + "header": ("col_header",), "data": ("value",), "total": ("total_label", "total_value"), + "axis": ("col_header",), "labels": ("row_label", "derived_label", "section_title"), + "values": ("value", "derived_value"), "title": ("section_title",), "ignored": ("ignored", "ignored_column"), +} +_NAMES = {"header": "表头", "data": "数据区", "total": "合计行", "axis": "日期表头", "labels": "行标签", + "values": "数值区", "title": "分段标题", "ignored": "忽略的格"} + + +def _crop(rect: Rect | None, window: int | None) -> Rect | None: + if rect is None or window is None: + return rect + if rect[0] > window: + return None + return rect[0], rect[1], min(rect[2], window), rect[3] + + +def _covered(rect: Rect, marks: list[Rect]) -> bool: + """rect 里的每一格都落在某个 mark 里（框是人框的，几十格以内，逐格判）。""" + return all(any(m[0] <= r <= m[2] and m[1] <= c <= m[3] for m in marks) + for r in range(rect[0], rect[2] + 1) for c in range(rect[1], rect[3] + 1)) + + +def replay_compare(edit: EditResult, after: Extraction, *, window_rows: int | None = None) -> ReplayCompare: + """框换算出的期望区域与应用后干跑认出的区域比对（4.5）。match=False 不阻止应用：界面把差异摆出来，按钮文字变成 + 「仍然应用（以重放结果为准）」。 + + actual 从 after.regions 里按 RegionMark.block 过滤出这个块的格子、按角色合成外接矩形。列表、交叉表按矩形相等判 + （表头矩形相等、数据矩形的行列范围相等、期望有合计行时合计行就是框下一行）；分段、合计行、分段标题、忽略这几种 + 框只是一部分，按「框里的格都被认成了期望的角色」判。干跑结果里没有块 id（期 2 的执行器）时退回按列范围取。 + window_rows：大表的交互干跑只跑前若干行时，期望和实际都先裁到这些行再比（评审三-M3）。""" + exp = {k: v for k, v in (edit.expected or {}).items()} + tagged = any(m.block for m in after.regions) + exp_rects = {k: _rect(v) for k, v in exp.items() if v} + cols = _bbox(list(exp_rects.values())) + + def marks_for(key: str) -> list[Rect]: + roles = _ROLES.get(key, ()) + out = [] + for m in after.regions: + if m.role not in roles: + continue + if tagged and edit.block is not None and m.block != edit.block: + continue + r = _rect(m.ref) + if (not tagged or edit.block is None) and cols is not None and not (r[1] <= cols[3] and cols[1] <= r[3]): + continue + out.append(r) + return [x for x in (_crop(r, window_rows) for r in out) if x is not None] + + whole = edit.key.split(":", 1)[0] in ("list", "crosstab") + actual: dict[str, str | None] = {} + diffs: list[str] = [] + expected: dict[str, str | None] = {} + match = True + for key in exp: + e = _crop(exp_rects.get(key), window_rows) + expected[key] = _a1(e) + ms = marks_for(key) + if whole: + a = _bbox(ms) + else: + a = _bbox([m for m in ms if e is not None and _intersects(m, e)]) + actual[key] = _a1(a) + name = _NAMES.get(key, key) + if e is None and a is None: + continue + if e is None: + if whole: + match = False + diffs.append(f"重放时多认出了{name}（{_a1(a)}），框里没有") + continue + if a is None: + match = False + diffs.append(f"重放时没有认出{name}（框里是 {_a1(e)}）") + continue + if whole: + if a != e: + match = False + diffs.append(_diff_text(name, e, a)) + elif not _covered(e, ms): + match = False + diffs.append(f"框里的{name}（{_a1(e)}）在重放时没有全部认出，认出的是 {_a1(a)}") + return ReplayCompare(expected=expected, actual=actual, match=match, diffs=diffs, window_rows=window_rows) + + +def _diff_text(name: str, e: Rect, a: Rect) -> str: + """人话：「重放的数据区到第 10 行，比框多 2 行（第 9–10 行）」。""" + parts = [] + if a[2] > e[2]: + parts.append(f"重放的{name}到第 {a[2]} 行，比框多 {a[2] - e[2]} 行（{_rows_text(e[2] + 1, a[2])}）") + elif a[2] < e[2]: + parts.append(f"重放的{name}到第 {a[2]} 行，比框少 {e[2] - a[2]} 行（{_rows_text(a[2] + 1, e[2])}）") + if a[0] != e[0]: + parts.append(f"重放的{name}从第 {a[0]} 行开始，框是第 {e[0]} 行") + if (a[1], a[3]) != (e[1], e[3]): + parts.append(f"重放的{name}在 {col_letter(a[1])}–{col_letter(a[3])} 列，框是 {col_letter(e[1])}–{col_letter(e[3])} 列") + return "；".join(parts) or f"重放的{name}是 {_a1(a)}，框是 {_a1(e)}" diff --git a/backend/app/data/recipe_suggest.py b/backend/app/data/recipe_suggest.py index 51ae5274..53314072 100644 --- a/backend/app/data/recipe_suggest.py +++ b/backend/app/data/recipe_suggest.py @@ -23,6 +23,7 @@ validate）和执行器干跑（注入的 dry_run），再由人逐条确认之 from __future__ import annotations import asyncio +import copy import datetime as _dt import inspect import itertools @@ -54,8 +55,12 @@ from app.data.recipe_types import ( MeasuresSegment, Problem, Question, + NotComparable, Recipe, RecipeProblem, + SumEq, + accumulate_blockers, + derive_tables, ) from app.data.tabular import DATE_HEADER_MIN, EXCEL_ERRORS from app.data.xlsx_scan import WorkbookScan, cell_ref, col_letter, parse_ref @@ -1133,9 +1138,6 @@ def _analyze(scan: WorkbookScan, grids: dict[str, Grid]) -> _Analysis: seg_ids.add(out) return out - def new_table(want: str, n_fallback: int) -> str: - return _unique(to_sql_name(want, fallback=f"表{n_fallback}") if want else f"表{n_fallback}", tables_used) - sheet_dicts: list[dict[str, Any]] = [] candidates: dict[str, list[str]] = {} crosstabs = 0 @@ -1215,42 +1217,13 @@ def _analyze(scan: WorkbookScan, grids: dict[str, Grid]) -> _Analysis: if sh.lists is not None: for b in sh.lists.blocks: lists_n += 1 - b.id = f"列表{lists_n}" - block_ids.add(b.id) - title = b.title if b.title and not any(ch.isdigit() for ch in b.title) else None - want = _ENUM_PREFIX.sub("", P.split_unit_suffix(title)[0]) if title else sh.sheet - b.table = new_table(want, len(tables_used) + 1) - used_cols: set[str] = set() - cols_out: list[dict[str, Any]] = [] - for i, c in enumerate(b.cols): - col, unit, _raw = _measure_column(b.headers[c], i + 1) - col = _unique(col, used_cols) - b.names[c] = col - if unit: - b.units[col] = unit - cols_out.append({"header": b.headers[c][:80], "name": col, "type": b.types[c]}) - total = None - if b.total_row is not None: - keep = f"{b.table}_表内合计" - if collide_key(keep) not in tables_used: - tables_used.add(collide_key(keep)) - num_units = {n: u for c, n in b.names.items() if (u := b.units.get(n)) - and b.types[c] in ("INTEGER", "REAL")} - table(keep, ["合计项"], num_units, kind="reported_total", parent=b.table) - b.keep_table = keep - total = {"label_column": b.names[b.cols[0]], "pick": b.total_word, "keep_as": keep} - grain = [b.names[b.grain_col]] if b.grain_col is not None else [] - table(b.table, grain, dict(b.units)) - fill = bool(b.merged_fill) and all(r2 > r1 and c1 == c2 and b.types.get(c1) == "TEXT" - for (r1, c1, r2, c2) in b.merged_fill) - blocks.append({ - "id": b.id, "layout": "list", "table": b.table, "header_rows": len(b.header_rows), - "after_title": None, "columns": cols_out, "extra_columns": "reject", - "rows": {"blank_rows": "skip" if b.skip_blank else "stop", "total_row": total}, - "values": {"placeholders": [{"text": e[0], "meaning": "无数据"} for e in b.placeholders.values()], - "blank": "null", "text_number": "reject", "formula": "accept_cached"}, - "merged_data": "fill" if fill else "reject", - }) + block_ids.add(f"列表{lists_n}") + block, main, total_spec = list_block(sh.sheet, b, block_id=f"列表{lists_n}", tables_used=tables_used) + table(main["name"], main["grain"], main["units"]) + if total_spec is not None: + table(total_spec["name"], total_spec["grain"], total_spec["units"], kind="reported_total", + parent=main["name"]) + blocks.append(block) if not blocks: continue context: list[dict[str, Any]] = [] @@ -1281,6 +1254,60 @@ def _analyze(scan: WorkbookScan, grids: dict[str, Grid]) -> _Analysis: return _Analysis(sheets, hidden, facts, recipe, failures) +def list_block(sheet: str, b: _ListPlan, *, block_id: str, tables_used: set[str], + table: str | None = None) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any] | None]: + """列表计划 → (块, 主表的表定义, 合计表的表定义或 None)。起草和框选（recipe_select 的 as=list）共用这一个函数 + （P3-SPEC 4.2、评审二-m12）：列名、单位、类型、grain、合计行、blank_rows、占位符、合并单元格填充都由这里按 + 计划推出，框选只决定表头行和列范围。各写一套的话，走了框选就会丢掉占位符、合并填充、跳过空行这些设置， + 框选得到的规则也不再与起草器相同（4.6 第 1 条要求同一份文件框选后配方哈希不变）。 + + tables_used 是已用表名的 collide_key 集合，原地更新（与起草器的去重同一口径）。table 给了时用它作表名 + （框选沿用被替换的块的表名、或客户端给的、已经过名字校验的表名），不再从标题推。 + 合计表名撞了已用的名字时照起草器的老行为：不另建表定义，keep_as 仍写这个名字（交给静态校验报）。""" + b.id = block_id + if table is None: + title = b.title if b.title and not any(ch.isdigit() for ch in b.title) else None + want = _ENUM_PREFIX.sub("", P.split_unit_suffix(title)[0]) if title else sheet + n_fallback = len(tables_used) + 1 + b.table = _unique(to_sql_name(want, fallback=f"表{n_fallback}") if want else f"表{n_fallback}", tables_used) + else: + b.table = table + tables_used.add(collide_key(table)) + used_cols: set[str] = set() + cols_out: list[dict[str, Any]] = [] + for i, c in enumerate(b.cols): + col, unit, _raw = _measure_column(b.headers[c], i + 1) + col = _unique(col, used_cols) + b.names[c] = col + if unit: + b.units[col] = unit + cols_out.append({"header": b.headers[c][:80], "name": col, "type": b.types[c]}) + total = None + total_spec: dict[str, Any] | None = None + if b.total_row is not None: + keep = f"{b.table}_表内合计" + if collide_key(keep) not in tables_used: + tables_used.add(collide_key(keep)) + num_units = {n: u for c, n in b.names.items() if (u := b.units.get(n)) + and b.types[c] in ("INTEGER", "REAL")} + total_spec = {"name": keep, "grain": ["合计项"], "kind": "reported_total", "units": num_units, "note": ""} + b.keep_table = keep + total = {"label_column": b.names[b.cols[0]], "pick": b.total_word, "keep_as": keep} + grain = [b.names[b.grain_col]] if b.grain_col is not None else [] + main = {"name": b.table, "grain": grain, "kind": "data", "units": dict(b.units), "note": ""} + fill = bool(b.merged_fill) and all(r2 > r1 and c1 == c2 and b.types.get(c1) == "TEXT" + for (r1, c1, r2, c2) in b.merged_fill) + block = { + "id": b.id, "layout": "list", "table": b.table, "header_rows": len(b.header_rows), + "after_title": None, "columns": cols_out, "extra_columns": "reject", + "rows": {"blank_rows": "skip" if b.skip_blank else "stop", "total_row": total}, + "values": {"placeholders": [{"text": e[0], "meaning": "无数据"} for e in b.placeholders.values()], + "blank": "null", "text_number": "reject", "formula": "accept_cached"}, + "merged_data": "fill" if fill else "reject", + } + return block, main, total_spec + + def _prefer_prefix(period: list[tuple[int, int, str]]) -> str | None: """统计期格冒号前的文字（≤12 字、不含数字），作回执里优先展示的格。取第一处。""" if not period: @@ -1535,6 +1562,14 @@ def remap_facts(facts: DraftFacts, recipe: dict[str, Any], *, visible: list[str] # ========================================================================== +def _eligible(recipe: dict[str, Any]) -> bool: + """这份配方符合按期累积的资格（契约 accumulate_blockers 为空）。pydantic 不收时不符合。""" + try: + return not accumulate_blockers(Recipe.model_validate(recipe)) + except ValidationError: + return False + + def _full_form(recipe: dict[str, Any]) -> dict[str, Any]: """补全默认值的完整形式：问题的 effects 按它写路径，父对象都在。pydantic 不收时原样返回（交给静态校验报）。""" try: @@ -1559,6 +1594,9 @@ def draft(scan: WorkbookScan, grids: dict[str, Grid], filename: str, *, failures = [f.human for f in an.failures] for_model = [f.model for f in an.failures] recipe = _full_form(an.recipe) if an.recipe is not None else None + if recipe is not None and _eligible(recipe): + # D14 = A：按统计期出的报表默认按期累积，其他默认替换（P3-SPEC 2.2）；首次导入时由人经 q_mode 和确认项确认 + recipe["mode"] = "accumulate" extraction: Extraction | None = None if recipe is None and not failures: failures.append("没有找到有内容的可见工作表") @@ -1995,7 +2033,7 @@ def questions_for(recipe: dict[str, Any], facts: DraftFacts, grids: dict[str, Gr try: parsed = Recipe.model_validate(recipe) except ValidationError: - return [_mode_card()], [] + return [_mode_card(None)], [] relations = recipe.get("relations") if isinstance(recipe.get("relations"), list) else [] claims = {r.claims: i for i, r in enumerate(parsed.relations) if r.claims} next_rid = [max([int(r.id[1:]) for r in parsed.relations] + [0])] @@ -2092,7 +2130,11 @@ def questions_for(recipe: dict[str, Any], facts: DraftFacts, grids: dict[str, Gr if ext is not None and ext.sheets.skipped_hidden: for nm in _dedupe([item.get("sheet", "") for item in ext.sheets.skipped_hidden]): cards.append(_hidden_sheet_card(nm)) - cards.append(_mode_card()) + blockers = accumulate_blockers(parsed) + cards.append(_mode_card(blockers)) + if not blockers: + # q_mode 固定排在问题列表最后：期 2 的测试和性能用例取 questions[0]，不能被它顶掉（评审二-B2） + questions.append(_mode_question()) return cards, questions @@ -2103,9 +2145,39 @@ def _qid(base: str, block_id: str, used: set[str]) -> str: return qid -def _mode_card() -> Card: - return Card("mode", "导入模式：每期替换", - "每次上传新一期时，整份替换上一期的数据（按期累积将在后续版本提供）") +#: 导入模式两种取值的后果（P3-SPEC 2.2，评审三-m4）：卡片的理由里写，确认项 mode 的 detail（WP-3）和界面 +#: q_mode 选项下方的说明（WP-6）也是这两句 +MODE_CONSEQUENCE = { + "accumulate": "每期以统计期为键加入当前版本；统计期与已有各期部分重叠的文件会被拒收，同一统计期再传要确认替换", + "replace": "每次上传新一期，当前版本只含新的一期，此前各期留在历史版本中", +} + + +def _mode_question() -> Question: + """q_mode：符合按期累积资格时才问。effects 作用在 answers_base（完整形式，/mode 总在）上；用 add 是为了 + answers_base 不合 schema、原样保存时也能应用（add 到已有键等于替换，RFC 6902）。没有默认选中：没回答时， + 草稿里的取值生效（起草器在符合资格时已写 accumulate）。""" + return Question("q_mode", "这份报表每期怎么更新", + [{"value": "accumulate", "label": "按期累积（建议）", "needs_reason": False}, + {"value": "replace", "label": "每期替换", "needs_reason": False}], + None, + {"accumulate": [{"op": "add", "path": "/mode", "value": "accumulate"}], + "replace": [{"op": "add", "path": "/mode", "value": "replace"}]}) + + +def _mode_card(blockers: list[RecipeProblem] | None) -> Card: + """导入模式卡片。blockers 为空（符合资格）时建议按期累积并指向 q_mode；不符合时写第一条原因； + None 表示配方还没过 schema，判断不了。""" + if blockers is not None and not blockers: + return Card("mode", "导入模式：按期累积（建议）", + "这是按统计期出的报表：每期以统计期为键累积，跨期可以比较；也可以选每期替换。" + f"按期累积：{MODE_CONSEQUENCE['accumulate']}。每期替换：{MODE_CONSEQUENCE['replace']}", + question="q_mode") + if blockers is None: + reason = "配方还不完整，暂时无法判断能否按期累积" + else: + reason = blockers[0].message + return Card("mode", "导入模式：每期替换", f"{reason}。每期替换：{MODE_CONSEQUENCE['replace']}") def _period_cells(grid: Grid | None, name: str, region: set[tuple[int, int]], @@ -2472,3 +2544,358 @@ def _list_cards(si: int, bi: int, sheet: Any, block: ListBlock, loc: _ListLoc | cells = _coords(name, [(loc.total_row, loc.cols[0])]) if loc is not None and loc.total_row else [] cards.append(Card(f"total_row:{block.id}", f"{where}是合计行，改作核对", reason, cells)) return cards, questions + + +# ========================================================================== +# 期 3：按规则重新起草，并把名字对齐到现行配方（P3-SPEC 6.3） +# ========================================================================== + +#: 表配对的门槛：来源签名的 Jaccard 相似度至少这么多 +ALIGN_MIN_JACCARD = 0.5 + + +def _table_signatures(recipe: Recipe) -> dict[str, set[tuple[str, str]]]: + """表 → 来源签名：写入它的 measures 标签、dimension 标题与标签、合计分段的标签、列表表头的 match_key 集合。 + + 带上来源的种类（指标、维度标题、维度标签、合计、列表、列表合计）：列表的合计表和主表表头相同，不分种类的话 + 两张表的签名一样，会并列最大、谁也配不上。""" + out: dict[str, set[tuple[str, str]]] = {} + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, ListBlock): + out.setdefault(block.table, set()).update(("list", P.match_key(c.header)) for c in block.columns) + total = block.rows.total_row + if total is not None and total.keep_as: + out.setdefault(total.keep_as, set()).update( + ("list_total", P.match_key(c.header)) for c in block.columns if c.type in ("INTEGER", "REAL")) + continue + for seg in block.segments: + if isinstance(seg, MeasuresSegment): + out.setdefault(seg.table, set()).update(("measure", P.match_key(x)) for x in seg.labels.expect) + elif isinstance(seg, DimensionSegment): + sig = out.setdefault(seg.table, set()) + if seg.locate.title: + sig.add(("dim_title", P.match_key(seg.locate.title))) + sig.update(("dim_label", _dim_key(seg.dim.parser, x)) for x in seg.labels.expect) + elif isinstance(seg, DerivedSegment) and seg.keep_as is not None: + out.setdefault(seg.keep_as.table, set()).update( + ("derived", (h.canonical if (h := P.hour_range_total(x)) else canon(x))) for x in seg.labels.expect) + return out + + +def _pair_tables(cand: Recipe, cur: Recipe, report: list[str]) -> dict[str, str]: + """起草的表 → 现行的表，一对一（评审一-m13）：Jaccard ≥ 0.5，按相似度从高到低贪心配对，一张现行表只配一次； + 同一张起草的表有两个并列最大的候选，或者两张起草的表对同一张现行表并列最大，都不配对，记进 report。""" + cs, ks = _table_signatures(cand), _table_signatures(cur) + scored: list[tuple[float, str, str]] = [] + for c, a in cs.items(): + for k, b in ks.items(): + if a or b: + j = len(a & b) / len(a | b) + if j >= ALIGN_MIN_JACCARD: + scored.append((j, c, k)) + out: dict[str, str] = {} + used_c: set[str] = set() + used_k: set[str] = set() + for score in sorted({s for s, _c, _k in scored}, reverse=True): + level = [(c, k) for s, c, k in scored if s == score and c not in used_c and k not in used_k] + n_c: dict[str, int] = {} + n_k: dict[str, int] = {} + for c, k in level: + n_c[c] = n_c.get(c, 0) + 1 + n_k[k] = n_k.get(k, 0) + 1 + tied_c = {c for c, n in n_c.items() if n > 1} + tied_k = {k for k, n in n_k.items() if n > 1} + for c, k in level: + if c in tied_c or k in tied_k: + continue + out[c] = k + for c in sorted(tied_c): + report.append(f"表「{c}」与现行配方里的几张表同样相似，无法确定对应哪一张，保留起草器给的名字") + for k in sorted(tied_k - {out.get(c) for c in tied_c}): + report.append(f"起草的几张表都同样像现行的表「{k}」，无法确定哪一张对应它，保留起草器给的名字") + used_c |= set(out) | tied_c + used_k |= set(out.values()) | tied_k + return out + + +def align_to_contract(candidate: dict[str, Any], current: dict[str, Any]) -> tuple[dict[str, Any], list[str]]: + """把按规则重新起草的配方的名字对齐到现行配方（P3-SPEC 6.3）。规则封闭，只从现行配方里抄名字： + + | 对象 | 怎么配对 | 对上以后 | + | 表 | 来源签名的 Jaccard ≥ 0.5，一对一贪心，并列不配 | 改成现行的表名，并改掉一切引用 | + | measures 列、列表列 | 按标签、表头的 match_key | 改成现行的列名 | + | measures 段 | 同一张表上标签重合最多的段 | 抄 id | + | dimension 段 | 按标题，或按标签集合 | 抄 id、dim.name、value、常量列名（pick 仍要能沿用，3.3）、派生列名 | + | derived 段 | 跟着前一段 | 同上（另存表的合计项列、派生列、值列） | + | 关系 | 按 kind 和改名后的成员 | 抄 id 和 claims | + 另外沿用现行配方的导入模式、日期列名、工作表 id 和块 id：重新起草只是换一套定位规则，不该悄悄改掉这些。 + + 对不上的保留起草器给的名字，记进 report（人话）。返回 (对齐后的配方（完整形式）, report)。不改入参。 + 任何一份不合 schema 时原样返回候选和一句说明。理由：重新起草时大多数「破坏性变更」只是名字不同（起草器把宽表 + 叫「客流汇总_按日」，现行叫「日客流」），按来源对齐能消掉这类假警报，剩下的才是真变化（final.md R11、3.3）。""" + try: + cand_m = Recipe.model_validate(candidate) + cur_m = Recipe.model_validate(current) + except ValidationError: + return copy.deepcopy(candidate), ["配方的格式有问题，名字没有对齐"] + report: list[str] = [] + cand = cand_m.model_dump(mode="json") + tmap = _pair_tables(cand_m, cur_m, report) + for t in derive_tables(cand_m)[0]: + if t not in tmap: + report.append(f"表「{t}」没有在现行配方里找到对应的表") + cmap: dict[str, dict[str, str]] = {} # 起草的表名 → {起草的列名: 现行的列名} + smap: dict[str, str] = {} # 起草的分段 id → 现行的分段 id + bmap: dict[tuple[int, int], str] = {} # 起草的块位置 → 现行的块 id + picks: dict[str, dict[str, str]] = {} # 起草的分段 id → {现行的常量列名: pick} + + def col(t: str, old: str, new: str) -> None: + if old and new and old != new: + cmap.setdefault(t, {})[old] = new + + cur_segs = [(sheet, block, seg) for sheet in cur_m.sheets for block in sheet.blocks + if isinstance(block, CrosstabBlock) for seg in block.segments] + cur_lists = [(sheet, block) for sheet in cur_m.sheets for block in sheet.blocks if isinstance(block, ListBlock)] + cur_cross = [(sheet, block) for sheet in cur_m.sheets for block in sheet.blocks if isinstance(block, CrosstabBlock)] + sheet_ids: dict[int, str] = {} + for si, sheet in enumerate(cand_m.sheets): + same = [s for s in cur_m.sheets if P.match_key(s.match.name) == P.match_key(sheet.match.name)] + if len(same) == 1: + sheet_ids[si] = same[0].id + elif len(cand_m.sheets) == 1 and len(cur_m.sheets) == 1: + sheet_ids[si] = cur_m.sheets[0].id + for bi, block in enumerate(sheet.blocks): + if isinstance(block, ListBlock): + target = tmap.get(block.table) + hit = next((b for _s, b in cur_lists if b.table == target), None) + if hit is None: + continue + bmap[(si, bi)] = hit.id + by_key = {P.match_key(c.header): c.name for c in hit.columns} + for c in block.columns: + new = by_key.get(P.match_key(c.header)) + if new: + col(block.table, c.name, new) + if block.rows.total_row is not None and block.rows.total_row.keep_as: + col(block.rows.total_row.keep_as, c.name, new) + continue + cross = [b for s, b in cur_cross if P.match_key(s.match.name) == P.match_key(sheet.match.name)] + if not cross and len(cur_cross) == 1: + cross = [cur_cross[0][1]] + if len(cross) == 1: + bmap[(si, bi)] = cross[0].id + if cross[0].axis.name != block.axis.name: + for seg in block.segments: + for t in (getattr(seg, "table", None), seg.keep_as.table if isinstance(seg, DerivedSegment) + and seg.keep_as else None): + if t: + col(t, block.axis.name, cross[0].axis.name) + for seg in block.segments: + if isinstance(seg, MeasuresSegment): + pool = [s for _sh, _b, s in cur_segs if isinstance(s, MeasuresSegment) and s.table == tmap.get(seg.table)] + keys = {P.match_key(x): x for x in seg.labels.expect} + best = max(pool, key=lambda s: len(keys.keys() & {P.match_key(x) for x in s.labels.expect}), + default=None) + if best is None or not keys.keys() & {P.match_key(x) for x in best.labels.expect}: + continue + smap[seg.id] = best.id + theirs = {P.match_key(x): best.measures.get(x) for x in best.labels.expect} + for k, raw in keys.items(): + if theirs.get(k): + col(seg.table, seg.measures[raw], theirs[k]) # type: ignore[arg-type] + elif isinstance(seg, DimensionSegment): + pool = [s for _sh, _b, s in cur_segs if isinstance(s, DimensionSegment)] + title = P.match_key(seg.locate.title) if seg.locate.title else None + hit = next((s for s in pool if title and s.locate.title and P.match_key(s.locate.title) == title), None) + if hit is None: + mine = {_dim_key(seg.dim.parser, x) for x in seg.labels.expect} + same_set = [s for s in pool if {_dim_key(s.dim.parser, x) for x in s.labels.expect} == mine] + hit = same_set[0] if len(same_set) == 1 else None + if hit is None: + report.append(f"分段「{seg.id}」没有在现行配方里找到对应的分段") + continue + smap[seg.id] = hit.id + col(seg.table, seg.dim.name, hit.dim.name) + col(seg.table, seg.value, hit.value) + roles = {v: k for k, v in hit.dim.derive.items()} + for name, role in seg.dim.derive.items(): + col(seg.table, name, roles.get(role, name)) + if len(seg.const) == 1 and len(hit.const) == 1: + (mine_c, mine_v), (their_c, their_v) = next(iter(seg.const.items())), next(iter(hit.const.items())) + col(seg.table, mine_c, their_c) + # pick 仍要满足 3.3：现行的 pick 仍是新标题的子串才沿用，否则用起草器从新标题里选的词 + keep = their_v.pick if seg.locate.title and canon(their_v.pick) in canon(seg.locate.title) \ + else mine_v.pick + picks[seg.id] = {their_c: keep} + for seg in block.segments: + if not isinstance(seg, DerivedSegment): + continue + base = smap.get(seg.locate.segment or "") + hit = next((s for _sh, _b, s in cur_segs if isinstance(s, DerivedSegment) + and s.locate.segment == base), None) if base else None + if hit is None: + report.append(f"合计分段「{seg.id}」没有在现行配方里找到对应的分段") + continue + smap[seg.id] = hit.id + if seg.keep_as is not None and hit.keep_as is not None: + t = seg.keep_as.table + col(t, seg.keep_as.dim, hit.keep_as.dim) + col(t, seg.keep_as.value, hit.keep_as.value) + roles = {v: k for k, v in hit.keep_as.derive.items()} + for name, role in seg.keep_as.derive.items(): + col(t, name, roles.get(role, name)) + + def T(name: Any) -> Any: + return tmap.get(name, name) + + def C(table: Any, name: Any) -> Any: + return cmap.get(table, {}).get(name, name) + + cand["mode"] = cur_m.mode + for si, sheet in enumerate(cand["sheets"]): + if si in sheet_ids: + sheet["id"] = sheet_ids[si] + for bi, block in enumerate(sheet["blocks"]): + if (si, bi) in bmap: + block["id"] = bmap[(si, bi)] + if block["layout"] == "list": + old = block["table"] + block["table"] = T(old) + for c in block["columns"]: + c["name"] = C(old, c["name"]) + total = block["rows"].get("total_row") + if total: + total["label_column"] = C(old, total["label_column"]) + total["keep_as"] = T(total["keep_as"]) if total.get("keep_as") else total.get("keep_as") + continue + for seg in block["segments"]: + old_id = seg["id"] + seg["id"] = smap.get(old_id, old_id) + if seg["locate"].get("segment"): + seg["locate"]["segment"] = smap.get(seg["locate"]["segment"], seg["locate"]["segment"]) + if seg["role"] == "measures": + t = seg["table"] + seg["measures"] = {k: C(t, v) for k, v in seg["measures"].items()} + seg["table"] = T(t) + elif seg["role"] == "dimension": + t = seg["table"] + seg["dim"]["name"] = C(t, seg["dim"]["name"]) + seg["dim"]["derive"] = {C(t, k): v for k, v in seg["dim"]["derive"].items()} + seg["value"] = C(t, seg["value"]) + seg["const"] = {C(t, k): v for k, v in seg["const"].items()} + for k, pick in picks.get(old_id, {}).items(): + if k in seg["const"]: + seg["const"][k] = {"pick": pick} + seg["table"] = T(t) + else: + v = seg["verify"] + v["value"] = C(v["against_table"], v["value"]) + v["against_table"] = T(v["against_table"]) + k = seg.get("keep_as") + if k: + t = k["table"] + k.update(dim=C(t, k["dim"]), value=C(t, k["value"]), + derive={C(t, n): r for n, r in k["derive"].items()}, table=T(t)) + cur_block = next((b for _s, b in cur_cross if b.id == block["id"]), None) if (si, bi) in bmap else None + if cur_block is not None: + block["axis"]["name"] = cur_block.axis.name + for t in cand["tables"]: + old = t["name"] + t["name"] = T(old) + t["grain"] = [C(old, g) for g in t["grain"]] + t["units"] = {C(old, k): v for k, v in t["units"].items()} + # 关系：先改引用，再按 kind 和成员与现行的关系配对，抄 id 和 claims；没配上的顺延编号，免得与抄来的 id 重号 + cur_rel: dict[Any, Any] = {} + for r in cur_m.relations: + if isinstance(r, SumEq): + cur_rel[("sum_eq", r.table, r.total, frozenset(r.parts))] = r + elif isinstance(r, NotComparable): + cur_rel[("not_comparable", r.a.table, r.a.value, r.b.table, r.b.value, r.by)] = r + taken: set[str] = set() + loose: list[dict[str, Any]] = [] + for r in cand["relations"]: + if r["kind"] == "sum_eq": + old = r["table"] + r.update(table=T(old), total=C(old, r["total"]), parts=[C(old, p) for p in r["parts"]]) + key: Any = ("sum_eq", r["table"], r["total"], frozenset(r["parts"])) + elif r["kind"] == "not_comparable": + a, b = r["a"], r["b"] + r["by"] = C(a["table"], r["by"]) + a.update(value=C(a["table"], a["value"]), table=T(a["table"])) + b.update(value=C(b["table"], b["value"]), table=T(b["table"])) + key = ("not_comparable", a["table"], a["value"], b["table"], b["value"], r["by"]) + else: + key = None + hit = cur_rel.get(key) if key is not None else None + if hit is not None and hit.id not in taken: + r["id"] = hit.id + r["claims"] = hit.claims + taken.add(hit.id) + else: + loose.append(r) + if r["kind"] != "dismissed": + report.append(f"关系「{r['id']}」没有在现行配方里找到对应的关系") + n = max([int(i[1:]) for i in taken] + [0]) + for r in loose: + if r["id"] in taken: + n += 1 + while f"R{n}" in taken: + n += 1 + r["id"] = f"R{n}" + taken.add(r["id"]) + _dedupe_names(cand, set(tmap.values()), set(smap.values()) | set(bmap.values())) + return cand, _dedupe(report) + + +def _dedupe_names(recipe: dict[str, Any], kept_tables: set[str], kept_ids: set[str]) -> None: + """对齐之后，没配上的起草名字可能和抄来的现行名字撞上：只改没配上的那一方（加 _2）。""" + used = {collide_key(t) for t in kept_tables} + rename: dict[str, str] = {} + for t in recipe["tables"]: + if t["name"] in kept_tables: + continue + if collide_key(t["name"]) in used: + rename[t["name"]] = _unique(t["name"], used) + else: + used.add(collide_key(t["name"])) + if rename: + R = lambda n: rename.get(n, n) # noqa: E731 + for t in recipe["tables"]: + t["name"] = R(t["name"]) + for sheet in recipe["sheets"]: + for block in sheet["blocks"]: + if block["layout"] == "list": + block["table"] = R(block["table"]) + total = block["rows"].get("total_row") + if total and total.get("keep_as"): + total["keep_as"] = R(total["keep_as"]) + for seg in block.get("segments") or []: + if seg.get("table"): + seg["table"] = R(seg["table"]) + if seg.get("verify"): + seg["verify"]["against_table"] = R(seg["verify"]["against_table"]) + if seg.get("keep_as"): + seg["keep_as"]["table"] = R(seg["keep_as"]["table"]) + for r in recipe["relations"]: + if r.get("table"): + r["table"] = R(r["table"]) + for side in ("a", "b"): + if isinstance(r.get(side), dict): + r[side]["table"] = R(r[side]["table"]) + ids = set(kept_ids) + for sheet in recipe["sheets"]: + for block in sheet["blocks"]: + for item in [block, *(block.get("segments") or [])]: + if item["id"] in kept_ids: + continue + if item["id"] in ids: + old = item["id"] + n = 2 + while f"{old[:28]}_{n}" in ids: + n += 1 + item["id"] = f"{old[:28]}_{n}" + for seg in block.get("segments") or []: + if (seg.get("locate") or {}).get("segment") == old: + seg["locate"]["segment"] = item["id"] + ids.add(item["id"]) diff --git a/backend/app/data/recipe_types.py b/backend/app/data/recipe_types.py index f003444d..452c7cb2 100644 --- a/backend/app/data/recipe_types.py +++ b/backend/app/data/recipe_types.py @@ -1,7 +1,13 @@ -"""配方导入的契约（期 2，波次 0 由编排者提交）：配方 JSON 的模型、执行结果和各模块之间传递的数据结构。 +"""配方导入的契约（期 2，波次 0 由编排者提交；期 3 的改动见 P3-SPEC 9.2，同样由波次 0 提交）：配方 JSON 的模型、 +执行结果和各模块之间传递的数据结构。 -各工作包之间只通过这里的类型和 P2-SPEC.md 里写明的函数签名交互。这里**只有声明**，没有业务逻辑 -（prose_problems / render_prose 两个小函数除外：静态校验、说明生成、AI 起草三处共用，必须是同一份）。 +各工作包之间只通过这里的类型和 P2-SPEC.md、P3-SPEC.md 里写明的函数签名交互。这里**只有声明**，没有业务逻辑 +（prose_problems / render_prose / accumulate_blockers 三个小函数除外：静态校验、说明生成、AI 起草、累积计划 +几处共用，必须是同一份）。 + +**哈希前向兼容**（P2-SPEC 2.3、P3-SPEC 9.6）：配方的新字段默认值一律等于「没有这个字段时的行为」，canonical 形式 +去掉默认值，所以老配方的哈希和构建 id 不变；不改已有字段的默认值。执行结果、说明里的新字段也都有默认值：旧回执、 +旧清单里没有它们，读的一方按默认值处理。 改这个文件要编排者拍板：字段的增删会同时影响执行器、核对、起草、接口和前端。 @@ -33,8 +39,11 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator #: 配方格式版本。改结构就要升，静态校验只认这一个 RECIPE_FORMAT = "agentlab-recipe/2" -#: 配方执行器版本：进构建 id 和快照 id（table_versions.recipe_build_id / snapshot_id） +#: 配方执行器版本：进构建 id 和快照 id（table_versions.recipe_build_id / snapshot_id）。期 3 不升：没用新字段的配方， +#: 执行器写出的库与期 2 逐字节相同（P3-SPEC 2.1），期 2 的构建照样复用 RECIPE_ENGINE_VER = "recipe/1" +#: 按期累积的并集库的物化规则版本：进并集构建（TableBuild.engine_ver）、并集 id 和组合表哈希（P3-SPEC 2.1、2.6） +UNION_VER = "union/1" # ========================================================================== # 配方 JSON（agentlab-recipe/2） @@ -211,6 +220,53 @@ class DerivedSegment(_M): Segment = Annotated[Union[MeasuresSegment, DimensionSegment, DerivedSegment], Field(discriminator="role")] +# ---- 忽略规则（期 3，修复按钮 ⑥ 和框选写入，P3-SPEC 3.2、9.2）。都按文件里的文字认，不记坐标；锚点在某一期 +# 没出现，就什么也不忽略、也不报错。锚点文字含数字时静态校验报 period_literal（origin 不是 replay 时）。 +# 注意：Recipe.model_json_schema() 会进 AI 起草的提示词（recipe_ai._schema_text），类的 docstring 是 schema 的 +# description，所以 docstring 只写给读配方的人看的语义，实现细节写在注释里。 +# +# 执行语义（WP-1 实现）： +# - IgnoreRow：轴行以下的行，标签 match_key 在集合里、又没有被分段认领的，归类为 ignored：标签格和轴列格的去向是 +# ignored，不进分段，不报 row_unclaimed；行记进 Extraction.rows_excluded（reason=ignored_rows，anchor=命中的 +# 标签原文）。同一文字同时在本块某个分段的 expect 里是静态校验的 ignore_conflict。 +# - IgnoreColumn（交叉表）：轴行最后一个日期右侧、表头 match_key 在集合里的列，从轴行到本块数据区底部的格，去向是 +# ignored_column，不报 axis_extra_cells；表头记进 Extraction.ignored_columns[块]。表头能被 month_day_or_date +# 解析的是 ignore_conflict。 +# - IgnoreColumn（列表）：表头命中的列按 extra_columns=ignore 的方式处理（去向 ignored_column，表头记进 +# ignored_columns），只针对这些列；其余多出的列照 extra_columns 处理。与 columns 的表头重复是 ignore_conflict。 +# 列不是行，不进 rows_excluded。 +# - IgnoreOutside：区域外的数字格（含区域外的公式格）所在行，如果有区域外文字格的 match_key 等于锚点，这些数字格的 +# 去向是 ignored；锚点格本身仍是区域外文字（照常记录、比对）。行记进 rows_excluded（reason=ignored_outside， +# anchor=锚点原文）。 +# - 同一列表里 match_key 相同是静态校验的 ignore_duplicate。理由（reason）只作说明、不参与匹配，但进配方哈希： +# 改理由就是改配方。 + + +class IgnoreRow(_M): + """按行标签忽略交叉表里的一行：标签与 label 相同、又没有被任何分段认领的行不导入，记进回执的「排除的行」。""" + + #: 交叉表标签列上要忽略的行的标签原文（按 match_key 比；1–40 字） + label: str = Field(min_length=1, max_length=40) + #: 为什么忽略（说明用，不参与匹配；1–200 字） + reason: str = Field(min_length=1, max_length=200) + + +class IgnoreColumn(_M): + """按表头忽略一列。交叉表只认日期表头右侧的列；列表只针对这些表头，其余多出的列照 extra_columns 处理。""" + + #: 交叉表：轴行最后一个日期右侧的表头原文；列表：表头原文（按 match_key 比；1–80 字） + header: str = Field(min_length=1, max_length=80) + reason: str = Field(min_length=1, max_length=200) + + +class IgnoreOutside(_M): + """导入区域之外、同一行有这段文字时，这一行区域外的数字格不导入；这段文字本身照常作为区域外文字记录。""" + + #: 区域外同一行里的文字（按 match_key 比；1–40 字）。这一行区域外的数字格都忽略 + anchor: str = Field(min_length=1, max_length=40) + reason: str = Field(min_length=1, max_length=200) + + class CrosstabBlock(_M): id: str = Field(min_length=1, max_length=32) layout: Literal["crosstab"] @@ -219,6 +275,10 @@ class CrosstabBlock(_M): label_offset: int = Field(default=-1, ge=-1, le=-1) values: CrossValues = Field(default_factory=CrossValues) segments: list[Segment] = Field(min_length=1, max_length=16) + #: 期 3：按行标签忽略的行（IgnoreRow）。默认空 = 期 2 的行为 + ignore_rows: list[IgnoreRow] = Field(default_factory=list, max_length=50) + #: 期 3：轴行最后一个日期右侧、按表头忽略的列（IgnoreColumn）。默认空 = 期 2 的行为 + ignore_columns: list[IgnoreColumn] = Field(default_factory=list, max_length=50) class ListColumn(_M): @@ -270,6 +330,8 @@ class ListBlock(_M): values: ListValues = Field(default_factory=ListValues) #: 数据区里的合并单元格：reject（默认）；fill 在合并范围内用左上格的值填充（只对 TEXT 列） merged_data: Literal["reject", "fill"] = "reject" + #: 期 3：按表头忽略的列（IgnoreColumn），只针对这些列；其余多出的列照 extra_columns。默认空 = 期 2 的行为 + ignore_columns: list[IgnoreColumn] = Field(default_factory=list, max_length=50) Block = Annotated[Union[CrosstabBlock, ListBlock], Field(discriminator="layout")] @@ -281,6 +343,8 @@ class SheetRecipe(_M): hidden: HiddenPolicy = Field(default_factory=HiddenPolicy) context: list[ContextSpec] = Field(default_factory=list, max_length=1) blocks: list[Block] = Field(min_length=1, max_length=8) + #: 期 3：按同一行的文字忽略导入区域之外的数字格（IgnoreOutside）。默认空 = 期 2 的行为 + ignore_outside: list[IgnoreOutside] = Field(default_factory=list, max_length=20) class TableSpec(_M): @@ -337,7 +401,7 @@ Relation = Annotated[Union[SumEq, NotComparable, Dismissed], Field(discriminator class Recipe(_M): recipe_format: Literal["agentlab-recipe/2"] - #: replace 每期替换；accumulate 按期累积（期 3，期 2 静态校验拒收，数据模型兼容） + #: replace 每期替换；accumulate 按期累积（期 3 起可用：静态校验按 accumulate_blockers 判资格，P3-SPEC 2.2） mode: Literal["replace", "accumulate"] = "replace" #: 配方没列到的、有内容的可见工作表：confirm 记进回执且需确认（默认）；reject 拒收 other_visible_sheets: Literal["confirm", "reject"] = "confirm" @@ -366,9 +430,19 @@ class Problem: #: 给 AI 修订看的版本：可以有坐标和标签文字，不能有数据格的数值。执行器产出的每个 code 都必须填 #: （WP-2 的契约测试逐个 code 断言非空）；万一为空，回灌时只发 code 和 cells，**绝不退回 message** model_message: str = "" - #: 期 3 修复按钮的种类（add_label / remove_label / rename_title / declare_total / edit_members / - #: ignore_cells / declare_placeholder）。期 2 只透传 + #: 修复按钮的种类，取值见 FIX_KINDS（执行器按 recipe_engine.PROBLEM_FIX 填；edit_members 只来自静态校验， + #: rename_sheet 另有不是问题的触发：Extraction.sheets.renamed）。没有修复按钮时 None fix: str | None = None + #: 期 3：构造修复补丁要的参数，形状按 fix 的种类见 P3-SPEC 3.2（如 label_missing 的 + #: {"segment": 分段 id, "labels": [原文…]}）。由执行器填写。**只含标签、表头、分段标题、锚点的原文和坐标， + #: 不得含数据格的数值**（declare_placeholder 的 texts 是数据格里的占位符文字，不是数值，允许）。 + #: 没有修复按钮时 None + fix_args: dict[str, Any] | None = None + + +#: 修复按钮的九种（P3-SPEC 3.2 ①–⑨）。Problem.fix、FixProposal.kind、确认项 fix:<目标键> 的 kind 都取自这里 +FIX_KINDS = ("remove_label", "add_label", "edit_members", "rename_title", "declare_total", "ignore_cells", + "declare_placeholder", "declare_hidden", "rename_sheet") @dataclass @@ -388,10 +462,12 @@ CheckStatus = Literal["passed", "mismatch", "unverifiable", "info"] @dataclass class CheckResult: #: 稳定 id：K1… 合计核对、G1… 公式引用、T1… 列表合计、R1… 关系（沿用配方里的 id）、C1 统计期、C2 文件名、 - #: N1… 行数、P1… 主键。F 编号留给系统发现的事实（Fact.id），两者不混用 + #: N1… 行数、P1… 主键；期 3 的并集整体核对 U1 行数、U2 主键、U3 统计期（物化时跑，都是结构类，P3-SPEC 2.7）。 + #: F 编号留给系统发现的事实（Fact.id），两者不混用 id: str kind: Literal["derived_sum", "formula_refs", "column_sum", "relation_sum_eq", "relation_not_comparable", - "context_agree", "filename_period", "row_count", "pk_unique"] + "context_agree", "filename_period", "row_count", "pk_unique", + "union_rows", "union_pk", "union_period"] title: str status: CheckStatus #: 不通过时的类别：structure / data_quality / info @@ -422,12 +498,14 @@ class Acceptance: @dataclass class ConfirmItem: - #: 稳定 id，见 P2-SPEC 第 7.5 节的取值表 + #: 稳定 id，见 P2-SPEC 第 7.5 节、P3-SPEC 9.5 的取值表 id: str label: str detail: str = "" required: bool = True - source: Literal["recipe", "diff", "outside", "sheet", "context", "switch"] = "recipe" + #: 界面按它分组（P3-SPEC 9.5）。期 3 新增 edit（fix:*、select:*、redraft_adopted）和 accumulate + #: （period_replace、retire、mode_switch、accumulate_restart、accumulate_unit_risk）；默认值仍是 recipe + source: Literal["recipe", "diff", "outside", "sheet", "context", "switch", "edit", "accumulate"] = "recipe" @dataclass @@ -503,9 +581,11 @@ class Grid: # 执行结果（执行器 → 核对、说明、差异、回执） # ========================================================================== -#: 格子账的去处（每个非空格恰好一个） +#: 格子账的去处（每个非空格恰好一个）。期 3 新增 ignored：按配方的 ignore_rows / ignore_outside 忽略的格 +#: （按表头忽略的列沿用 ignored_column） Role = Literal["value", "derived_value", "derived_label", "col_header", "row_label", "section_title", - "context", "outside_text", "ignored_column", "hidden_excluded", "total_label", "total_value"] + "context", "outside_text", "ignored_column", "hidden_excluded", "total_label", "total_value", + "ignored"] @dataclass @@ -514,6 +594,13 @@ class RegionMark: role: Role #: A1 区域（「C5:AG7」或单格） ref: str + #: 期 3：块 id（Block.id）。区域外文字、统计期这类不属于任何块的为 None。框选的重放比对按它过滤出新块的区域， + #: 网格按块描边。执行器按 (role, block, 列段) 合矩形，不同块的格不合进同一个矩形。 + #: JSON（asdict）里一律带 block 键，None 也带：界面和 replay_compare 按键取，不判断有没有这个键。 + #: 已知后果：期 2 的 test_recipe_imports.py 的 test_stage_first_does_not_create_the_source_and_keeps_the_raw + #: 逐字比 marks（不含 block），WP-0 合并后它会失败。那个文件归 WP-5（P3-SPEC 12.0），改法是期望值补 + #: "block": None 或只比 sheet / role / ref。不要为了让它通过去掉这个字段或改默认值 + block: str | None = None @dataclass @@ -661,6 +748,34 @@ class SheetsOut: skipped_hidden: list[dict[str, str]] = field(default_factory=list) +#: 排除的行的原因（P3-SPEC 第 8 节遗留项 4）： +#: hidden_excluded = hidden.rows=exclude 排除的隐藏行；blank_skipped = 列表 blank_rows=skip 跳过的空行； +#: ignored_rows = 交叉表 ignore_rows 忽略的行；ignored_outside = ignore_outside 忽略了区域外数字格的行； +#: after_stop = 列表停止之后没导入的文字行；total_not_kept = 参与核对、但不另存的合计行（交叉表 keep_as 为空的 +#: derived 段、列表 total_row.keep_as 为空的合计行） +ExcludedReason = Literal["hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", + "total_not_kept"] + + +@dataclass +class ExcludedRows: + """回执里的「排除的行」（H2：排除的行要写进回执）。同一工作表、同一原因、同一块（和同一锚点）合成一项， + 行号写成闭区间列表。按表头忽略的列不是行，不在这里（记在 Extraction.ignored_columns）。""" + + #: 实际工作表名 + sheet: str + reason: ExcludedReason + #: [[起, 止], …]，1 起的行号闭区间，升序、不重叠 + rows: list[list[int]] + #: 这些行里被排除的非空格个数（与格子账同一口径） + cells: int + #: ignored_rows / ignored_outside：命中的锚点原文（标签或区域外文字）；after_stop 且是跳过空行的列表按「表下说明」 + #: 收尾时（WP-8）：说明那一格的文字（去首尾空白），列表停在哪一行、因为什么由它看出；其余 None + anchor: str | None = None + #: 所在块的 id；ignored_outside、hidden_excluded 这类按工作表算的为 None + block: str | None = None + + @dataclass class Extraction: #: 没有 structure / input 类问题，且库已写好 @@ -706,6 +821,14 @@ class Extraction: table_hashes: dict[str, str] = field(default_factory=dict) #: 各阶段耗时（秒） timings: dict[str, float] = field(default_factory=dict) + #: 期 3：排除的行（ExcludedRows）。只取本次试运行的 Extraction 写进导入清单的 receipt；复用的期 2 构建回执 + #: （TableBuild.report）里没有这个字段，读的一方按空列表处理。差异卡不把它加进 RECEIPT_KEYS：上一期没有这个 + #: 字段时出 info「上一期未记录」，不当作空列表比较（P3-SPEC 第 8 节遗留项 4）。 + #: 要显示它的地方有两处，WP-5 两处都要接上：①试运行回执 TrialOut.receipt——recipe_imports._RECEIPT_VIEW + #: 只抄白名单里的键，要把 "rows_excluded" 加进去，否则向导回执一律显示「这次导入未记录排除的行」（界面把缺这个 + #: 键当作期 3 之前的回执）；②导入清单的 receipt。extraction_from_dict 读回时还要在 _EXTRACTION_PARTS 里把 + #: 每项转回 ExcludedRows + rows_excluded: list[ExcludedRows] = field(default_factory=list) @dataclass @@ -826,6 +949,19 @@ class TableNote: columns: dict[str, str] = field(default_factory=dict) +@dataclass +class NoteFragment: + """表说明里的一个片段（期 3）：build_notes 按片段族拼说明时顺带记下，build_union_notes 按 (key, subject) + 逐期合并、取最弱（P3-SPEC 2.8）。只是多返回结构，渲染结果不变。""" + + #: NOTE_TEXT 的键（片段族，如 sum_eq_passed、total_k_passed） + key: str + #: 关系 id（sum_eq / not_comparable 的 Rn）或表名（合计表、本表）；与 subject 无关的片段为 None + subject: str | None + #: 带 {列:} {表:} {单位:} 引用的模板（未渲染） + template: str + + @dataclass class SchemaNotes: tables: dict[str, TableNote] = field(default_factory=dict) @@ -836,6 +972,8 @@ class SchemaNotes: #: 表名 → 表的种类，只记 reported_total（原表写明的合计）。apply_notes 记进表结构，工具描述、db_schema 的 #: 表清单、写作目录据此标出「不要彼此相加、不要与明细相加」（AU-5） kinds: dict[str, str] = field(default_factory=dict) + #: 期 3：表名 → 这张表说明的片段（NoteFragment），按拼接顺序。旧清单里没有，按空处理 + fragments: dict[str, list[NoteFragment]] = field(default_factory=dict) _TOKEN = re.compile(r"\{(列|表|单位):([^{}]{1,48})\}") @@ -970,3 +1108,385 @@ def derive_tables(recipe: Recipe) -> tuple[dict[str, list[ColumnOut]], list[Reci for c in cols if c.type in ("INTEGER", "REAL")] put(total.keep_as, tcols, f"{base}/rows/total_row") return out, problems + + +# ========================================================================== +# 期 3：按期累积的资格（静态校验、起草器、累积计划共用同一份，P3-SPEC 2.2） +# ========================================================================== + + +def accumulate_blockers(recipe: Recipe) -> list[RecipeProblem]: + """这份配方为什么不能按期累积；空列表 = 符合资格。不看 recipe.mode：静态校验只在 mode=accumulate 时报它们， + 起草器在为空时才写 mode=accumulate，累积计划对每一期的配方都跑一遍（P3-SPEC 2.2、2.4）。 + + 按期累积要求同时满足： + 1. 至少一个工作表配置了统计期上下文（SheetRecipe.context 非空），否则报 accumulate_needs_period（path=/mode）； + 2. derive_tables 推出的每张表都「按统计期分得开」，否则逐表报 accumulate_unkeyed（path=/tables/{ti}，表没在 + tables 里声明时指向第一个写入它的块），消息写明哪一条不满足： + - 只由交叉表块写入（measures、dimension 段，或者 derived 段的 keep_as）——列表写入的表一律不满足； + - 写入它的每个交叉表块，axis.checks 都含 covers_context（日期恰好落在统计期内）； + - 这张表的 grain 含那个块的 axis.name。 + 两条都成立时，各期的日期集合互不相交（统计期不重叠由累积计划保证），主键里不需要另加统计期列。 + """ + problems: list[RecipeProblem] = [] + if not any(sheet.context for sheet in recipe.sheets): + problems.append(RecipeProblem("/mode", "accumulate_needs_period", + "配方没有配置统计期，不能按期累积：各期要按统计期区分")) + tables, _ = derive_tables(recipe) + # 表名 → 写入它的块（去重，按配方顺序）和第一个块的路径 + writers: dict[str, list[CrosstabBlock | ListBlock]] = {} + first_path: dict[str, str] = {} + + def _add(table: str, block: CrosstabBlock | ListBlock, path: str) -> None: + got = writers.setdefault(table, []) + if all(b is not block for b in got): + got.append(block) + first_path.setdefault(table, path) + + for si, sheet in enumerate(recipe.sheets): + for bi, block in enumerate(sheet.blocks): + path = f"/sheets/{si}/blocks/{bi}" + if isinstance(block, CrosstabBlock): + for seg in block.segments: + if isinstance(seg, (MeasuresSegment, DimensionSegment)): + _add(seg.table, block, path) + elif isinstance(seg, DerivedSegment) and seg.keep_as is not None: + _add(seg.keep_as.table, block, path) + else: + _add(block.table, block, path) + total = block.rows.total_row + if total is not None and total.keep_as: + _add(total.keep_as, block, path) + + index = {t.name: ti for ti, t in enumerate(recipe.tables)} + grains = {t.name: list(t.grain) for t in recipe.tables} + for name in tables: + blocks = writers.get(name, []) + if any(isinstance(b, ListBlock) for b in blocks): + reasons = ["列表形态的表目前只支持每期替换"] + else: + reasons = [] + uncovered = [b.id for b in blocks if isinstance(b, CrosstabBlock) and "covers_context" not in b.axis.checks] + if uncovered: + reasons.append("、".join(f"交叉表「{x}」" for x in uncovered) + + "的日期检查没有选「恰好覆盖统计期」，各期的日期可能超出统计期") + grain = grains.get(name, []) + axes = sorted({b.axis.name for b in blocks if isinstance(b, CrosstabBlock) and b.axis.name not in grain}) + if axes: + reasons.append("主键不含日期列" + "、".join(f"「{x}」" for x in axes) + "，各期的行无法按日期区分") + if reasons: + path = f"/tables/{index[name]}" if name in index else first_path.get(name, "/tables") + problems.append(RecipeProblem(path, "accumulate_unkeyed", f"表「{name}」不能按期累积：" + "；".join(reasons))) + return problems + + +# ========================================================================== +# 期 3：修复、框选、累积、并集的数据结构（P3-SPEC 9.2；JSON 形状见 9.4，dataclass 字段与 JSON 同名， +# Selection.as_ 除外）。字段名由 test_recipe_contract.py 钉住，实现者不改名 +# ========================================================================== + + +@dataclass +class FixOption: + """修复提议的一个封闭选项。效果由服务端算：客户端只发 fix_id、value 和理由，不发补丁。""" + + #: 选项值（remove / add / update / dismiss / keep / check_only / ignore / no_data / not_applicable / + #: include / exclude / "<候选格>|keep" / "<候选格>|pick:<词>" / 新工作表名……，P3-SPEC 3.2） + value: str + #: 给人看的一句话（「去掉标签「7-8」」） + label: str + #: 后果说明（「今后各期如果又出现「7-8」，会再次拒收」） + detail: str = "" + #: 选它必须写理由（1–200 字）。理由进配方哈希：改理由要重新预览 + needs_reason: bool = False + #: 这是破坏性变更（如常量换值） + breaking: bool = False + + +@dataclass +class FixAnchor: + """提议对应哪一条问题（P3-SPEC 3.1）。staging_out 按它把 fix_ids 写到对应的问题上，界面只按 fix_ids 放按钮。""" + + kind: Literal["problem", "recipe_problem", "sheet_renamed"] + #: problem / recipe_problem：在传给 propose_fixes 的 problems / recipe_problems 列表里的下标；sheet_renamed 为 None + index: int | None = None + #: sheet_renamed：本期的工作表名；其余 None + sheet: str | None = None + + +@dataclass +class FixProposal: + #: "fx-" + sha1(kind + "|" + 目标键)[:12]：同样的问题得到同样的 id + id: str + #: FIX_KINDS 之一 + kind: str + #: 触发它的问题 code（sheet_renamed 触发时为 None） + problem_code: str | None + title: str + #: 网格上要标出的格（「工作表!A1」） + cells: list[str] + #: 目标（{"segment", "labels"}、{"relation", "fact", "total", "parts"}……，P3-SPEC 3.2） + target: dict[str, Any] + options: list[FixOption] + anchor: FixAnchor + + +#: 框选「选它是什么」的封闭取值（P3-SPEC 4.2） +SELECTION_AS = ("list", "crosstab", "segment", "derived", "section_title", "ignore_rows", "ignore_columns", + "ignore_outside") +#: 框选的 ref：不带工作表名的 A1 区域，列字母大写，行号从 1 起（界面按网格坐标生成，不会有 $ 和小写） +_SELECTION_REF = re.compile(r"[A-Z]{1,3}[1-9][0-9]{0,6}(?::[A-Z]{1,3}[1-9][0-9]{0,6})?") + + +@dataclass +class Selection: + """网格上的一次框选。**JSON 里的键是 as**（Python 关键字），dataclass 字段叫 as_：接口层一律经 + from_json / to_json 改名，不要直接 asdict。""" + + sheet: str + #: A1 区域（「C5:F8」），不带工作表名 + ref: str + #: SELECTION_AS 之一 + as_: str + #: list：header_rows、table、bottom（box / auto）；segment：role、table；derived：keep；section_title：segment； + #: ignore_*：reason + options: dict[str, Any] = field(default_factory=dict) + + @classmethod + def from_json(cls, data: Any) -> "Selection": + """请求体里的 {"sheet", "ref", "as", "options"?} → Selection，同时把形状校验做完。 + + **任何形状不对都只抛 ValueError**（不抛 KeyError、TypeError），消息是可以直接给人看的中文：接口层只要 + ``except ValueError`` 就能映射成 422 edit_invalid（P3-SPEC 9.1），不会因为请求体写得怪而落到 500。 + 这里查的是「请求本身对不对」：body 是对象；sheet 是非空文字；ref 是不带工作表名的 A1 区域（大写列字母， + 「C5」或「C5:F8」）；as 是 SELECTION_AS 之一；options 缺省、为 null 或是对象。框和网格、配方的关系 + （超出网格、重叠、表头为空……）是 selection_edit 的事，报成 EditResult.problems，不在这里。""" + if not isinstance(data, dict): + raise ValueError("框选的请求格式不对：应为一个对象") + sheet, ref, as_, options = data.get("sheet"), data.get("ref"), data.get("as"), data.get("options") + if not isinstance(sheet, str) or not sheet.strip(): + raise ValueError("框选没有指明工作表") + if not isinstance(ref, str) or not _SELECTION_REF.fullmatch(ref): + raise ValueError("框选的区域写法不对：应为「C5」或「C5:F8」这样的单元格区域") + if not isinstance(as_, str) or as_ not in SELECTION_AS: + raise ValueError("框选没有指明选中的是什么，或取值不在可选范围内") + if options is not None and not isinstance(options, dict): + raise ValueError("框选的选项格式不对：应为一个对象") + return cls(sheet=sheet, ref=ref, as_=as_, options=dict(options or {})) + + def to_json(self) -> dict[str, Any]: + return {"sheet": self.sheet, "ref": self.ref, "as": self.as_, "options": dict(self.options)} + + +@dataclass +class Anchor: + """框选换算出的锚点，给界面显示「按文字定位」用。""" + + kind: Literal["header", "row_label", "section_title", "total_word", "axis", "after_title", "outside_text"] + text: str + cell: str | None = None + + +@dataclass +class EditResult: + """一次修复（fix_ops）或框选换算（selection_edit）的结果。ok=False 时 problems 写原因（category=recipe）， + ops 为空，界面不能应用。""" + + ok: bool + kind: Literal["fix", "selection"] + #: 修复：目标键（remove_label:日间:7-8）；框选：:<块或分段 id>。确认项 fix: / select: 用它 + key: str + title: str + #: 人话摘要，确认项和修改记录用 + summary: list[str] + #: 按完整形式写的 JSON Patch（add / replace / remove，不给用户看） + ops: list[dict[str, Any]] + anchors: list[Anchor] + notes: list[str] + problems: list[Problem] + #: apply_patch → validate 解析 → recipe_sha256，与暂存区的哈希同一口径；ok=False 时 None + recipe_sha256_after: str | None = None + #: 框选：换算出的期望区域（列表 {"header", "data", "total"}；交叉表 {"axis", "labels", "values"}） + expected: dict[str, str | None] | None = None + #: 框选：新增或替换的块 id（重放比对按它过滤 RegionMark） + block: str | None = None + #: 只在服务端内部用：这次修改是否带破坏性变化（WP-2 按所选 FixOption.breaking 或换算结果填）。 + #: **注意与 EditPreview JSON 的 breaking 不是同一个值**：预览里的 breaking 是「表名 → 人话列表」（WP-5 用 + #: recipe_confirm.table_changes 拿修改后的配方对现行配方算，P3-SPEC 9.4），没有破坏性变化时是 {}。WP-5 组装 + #: 预览时如果从 asdict(EditResult) 出发，必须总是用那份字典覆盖 breaking：漏了的话界面收到 false， + #: Object.entries(false) 是空数组，破坏性变化会不声不响地从预览里消失 + breaking: bool = False + + +@dataclass +class ReplayCompare: + """框选的期望区域与干跑后执行器认出的区域比对（P3-SPEC 4.5）。match=False 不阻止应用。""" + + expected: dict[str, str | None] + actual: dict[str, str | None] + match: bool + #: 不一致时逐条写人话（「重放的数据区到第 10 行，比框多 2 行（第 9–10 行）」） + diffs: list[str] + #: 部分干跑时只比较前多少行；完整比较为 None + window_rows: int | None = None + + +#: 累积计划的 action（P3-SPEC 2.4） +ACCUMULATE_ACTIONS = ("replace", "first", "append", "replace_period", "restart", "rejected") + + +@dataclass +class ChangeClass: + """classify_changes 的结果（P3-SPEC 2.5）。""" + + change: Literal["same", "compatible", "retire", "semantic"] + #: [{"table", "column" | None}] + added: list[dict[str, Any]] + #: 这次才退役的，出必勾项 retire:* + retired_new: list[dict[str, Any]] + #: 早已退役、仍保留在并集里的，不出确认项 + retired_existing: list[dict[str, Any]] + #: 不兼容变化的人话 + semantic: list[str] + #: 各期维度取值的差异：[{"table", "column", "segment", "periods": [{"start", "end", "missing", "extra"}]}] + label_sets: list[dict[str, Any]] + + +@dataclass +class AccumulatePlan: + """累积计划（WP-4 plan_accumulate 产出，JSON 形状见 P3-SPEC 9.4，TrialOut.accumulate 原样给界面）。""" + + mode: str + #: ACCUMULATE_ACTIONS 之一 + action: str + #: 本期 {"start", "end", "source"}：source 是 "cells"（取自表内统计期）或 "human"（人工录入）； + #: 每期替换且没有统计期时 start、end 为 None + period: dict[str, Any] + #: 结果快照的各期，按统计期排序：[{"import_id", "seq", "start", "end", "file_name", "new", "rows", "blockers"}]， + #: 本期 import_id、seq 为 None、new 为 True + parts: list[dict[str, Any]] + #: 被替换的那一期（replace_period），没有为 None + replaces: dict[str, Any] | None + #: restart、模式切换时移出当前版本的各期 + dropped: list[dict[str, Any]] + #: 部分重叠的各期（rejected；restart 时只作提示） + overlaps: list[dict[str, Any]] + #: 相邻两期之间的空缺 [{"start", "end"}] + gaps: list[dict[str, str]] + #: 本期早于已有各期 + backfill: bool + #: ChangeClass.change + change: str + added: list[dict[str, Any]] + retired_new: list[dict[str, Any]] + retired_existing: list[dict[str, Any]] + semantic: list[str] + label_sets: list[dict[str, Any]] + #: "replace->accumulate" / "accumulate->replace" / None + mode_switch: str | None + #: first / restart 的原因（blockers、没有统计期），其余 None + reason: str | None + #: 物化了的并集 {"union_id", "db_sha256", "rows"}；没有物化为 None + union: dict[str, Any] | None + + +@dataclass +class PartInfo: + """当前快照里的一期（WP-5 组装，交给 plan_accumulate / materialize_union）。""" + + import_id: str + seq: int + #: 统计期取自该期导入清单（内容寻址），再与 table_imports 的库列比对（P3-SPEC 2.3） + start: str + end: str + build_id: str + db_path: str + #: 取自 TableBuild + db_sha256: str + #: 该期导入清单里记的库哈希（物化第 0 步比对用） + manifest_db_sha256: str | None + recipe_id: str | None + recipe: Recipe | None + recipe_sha256: str | None + raw_state: str + file_name: str + manifest_artifact: str | None + #: 表 → 该期行数 + rows: dict[str, int] = field(default_factory=dict) + overrides: int = 0 + waivers: int = 0 + + +@dataclass +class UnionPart: + """物化的一期。""" + + #: None = 本期（试运行库） + import_id: str | None + start: str + end: str + db_path: str + recipe: Recipe + #: 期望的库哈希：已有的期取 TableBuild，本期取试运行回执 + db_sha256: str + manifest_db_sha256: str | None = None + #: 该期登记的表哈希（组合定义要用，P3-SPEC 2.6 第 6 步） + table_hashes: dict[str, str] = field(default_factory=dict) + + +@dataclass +class UnionReport: + """materialize_union 的结果。UnionBuild.report = asdict(UnionReport) 的 JSON，登记进 TableBuild.report。""" + + #: 表 → 列（目标表结构加退役列） + tables: dict[str, list[ColumnOut]] + grains: dict[str, list[str]] + rows: dict[str, int] + #: 与 parts 同序：表 → {"union": [a, b], "part": [1, n]} + part_rows: list[dict[str, dict[str, list[int]]]] + table_hashes: dict[str, str] + null_counts: dict[str, dict[str, int]] + #: 与 parts 同序（说明的空值只按含这一列的期统计） + part_null_counts: list[dict[str, dict[str, int]]] + #: [{"table", "column" | None, "periods": [...]}] + added: list[dict[str, Any]] + retired: list[dict[str, Any]] + #: U1–U3 + checks: list[CheckResult] + ok: bool + db_sha256: str + + +@dataclass +class UnionNotePart: + """build_union_notes 的一期输入。""" + + recipe: Recipe + extraction: Extraction + checks: list[CheckResult] + acceptances: list[Acceptance] + null_counts: dict[str, dict[str, int]] | None + start: str + end: str + + +# ========================================================================== +# 期 3：版本页（P3-SPEC 7.1） +# ========================================================================== + +#: 快照不能启用的原因（SnapshotOut.reason_code，P3-SPEC 7.1 的补充）。7.1 只给了人话 reason，而 available=false +#: 同时覆盖「已回收」和「数据文件已丢失」，界面分不出该用哪句文案，只能去匹配服务端的原话；WP-5 和 WP-7 并行， +#: 各写一套文字必然对不上。所以加一个机器可读的 code： +#: - WP-5 列快照时 reason_code 取这几个值之一（能启用时为 None），reason 一律取 SNAPSHOT_NOT_ACTIVATABLE[code]， +#: 不自己另写文字。几种同时成立时按字典顺序取第一个：已是当前版本 > 已回收 > 数据文件已丢失 > 含作废的导入； +#: - WP-7 按 VERSIONS_TEXT.notActivatable[reason_code] 显示（键与这里相同），没有 reason_code 时退回显示 reason。 +#: 文字与附录 C 的 notActivatable 逐字相同，所以无论界面走哪条路，显示都一样。 +#: 遮罩列丢失（mask_lost）不在这里：它不让快照变成不能启用，只是启用时要确认。哈希校验（snapshot_tampered） +#: 只在启用时算，列表不算，所以也不在这里 +SnapshotReasonCode = Literal["current", "retired", "file_lost", "contains_revoked"] +SNAPSHOT_NOT_ACTIVATABLE: dict[str, str] = { + "current": "已是当前版本", + "retired": "已回收，无法启用", + "file_lost": "数据文件已丢失，无法启用", + "contains_revoked": "包含已作废接受的导入，无法启用", +} diff --git a/backend/app/data/source_versions.py b/backend/app/data/source_versions.py new file mode 100644 index 00000000..a3dcb73f --- /dev/null +++ b/backend/app/data/source_versions.py @@ -0,0 +1,1067 @@ +"""按期累积与版本页的数据侧（期 3，WP-5；P3-SPEC 2.3、2.6、2.9、第 5、7 节）。 + +两类调用方共用这里： +- 试运行、提交（recipe_imports）：组装当前快照的各期（PartInfo）、当前版本实际的表结构、快照清单里的退役记录， + 物化之后写快照清单； +- 版本页（api/source_versions、api/datasources）：列版本、启用旧版本、查看清单、移除一期、作废接受、导入记录的 + 新字段（revoke_plan、raw_shared_with……）。 + +**各期的统计期只信导入清单。** 导入清单是内容寻址的工件（artifact_store.load 会复验内容哈希），库列 +table_imports.period_start / period_end 是可改的：两者对不上说明库列被改过，按 part_tampered 拒绝（2.3， +评审一-m11），不拿被改过的统计期去判重叠、做物化。 + +**写操作一律经 WP-4 的原语**（publish_snapshot、activate_snapshot），锁内比对 expected_current；这里只在锁外 +组装输入、映射错误码。锁外读到的东西在锁内可能已经变了：当前版本变了由 base_changed 兜住，回滚目标被回收、 +被作废由 activate_snapshot 的 snapshot_unavailable、contains_revoked 兜住，都不会悄悄切到别的版本。另外，锁外 +读到的当前版本必须就是确认框依据的那个（_check_base）：留下哪几期、回滚到哪里都按它算，只靠锁内比对挡不住 +「改走又改回」的 ABA。只读进程（没拿到版本存储的守卫）在动手组装、物化之前就回 503。 +""" +from __future__ import annotations + +import asyncio +import copy +import logging +import secrets +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import raw_store, recipe_types, table_versions +from app.data.recipe_imports import ImportRefused, check_from_dict, extraction_from_dict, pipeline +from app.data.recipe_types import ( + SNAPSHOT_NOT_ACTIVATABLE, Acceptance, ColumnOut, PartInfo, Recipe, SchemaNotes, UnionNotePart, UnionPart, +) +from app.db.base import utcnow +from app.db.models import ( + DataSource, ImportStaging, Run, SourceSnapshot, TableBuild, TableImport, TableRecipe, +) + +logger = logging.getLogger(__name__) + +#: 快照清单的格式版本（P3-SPEC 2.9） +SNAPSHOT_MANIFEST_FORMAT = "agentlab-snapshot-manifest/1" +#: GET snapshots 最多列几个（7.1） +SNAPSHOTS_MAX = 50 +#: 理由的字数上限（启用、移除、作废、清除，7.2–7.6） +REASON_MAX = 500 + +_NO_ROLLBACK = "没有可以回滚的版本，请上传修正后的文件" +_TARGET_CHANGED = "可以回滚的版本已有变化，请刷新后重新确认" +_LAST_PERIOD = "这是当前版本里唯一的一期，不能移除；如需清空请删除数据源" +#: raw_shared_with 里另一个源已被删除时的名字（与界面 VERSIONS_TEXT.deletedSource 同文）。名字会拼进「「名字」N 条」， +#: 给 null 的话旧界面会显示成「null」（评审意见）；界面的类型已声明可空并有同文兜底，这里仍给名字，新旧界面都读得通。 +#: 删除数据源只把它的导入置 retired，原件仍可能被本源共用，这条引用照样要列出来 +_DELETED_SOURCE = "已删除的数据源" + + +class PartProblem(Exception): + """当前快照的某一期读不出来（导入清单不在、被改过，统计期与库列对不上，构建记录不在）。 + + code 是 part_tampered / part_missing：试运行把它变成拒收的问题，版本页的写操作回 409。message 给人看。""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +# ========================================================================== +# 当前快照的各期 +# ========================================================================== + + +@dataclass +class Part: + """当前快照里的一期：契约 PartInfo 之外，物化、说明还要用的几样（导入清单、登记的表哈希、原件哈希）。""" + + info: PartInfo + manifest: dict[str, Any] + #: 该期登记的表哈希（组合定义要用，2.6 第 6 步）：取自导入清单（内容寻址），不取可改的构建回执 + table_hashes: dict[str, str] = field(default_factory=dict) + raw_sha: str = "" + #: 该期统计期的来源：cells / human + period_source: str | None = None + + def union_part(self) -> UnionPart: + if self.info.recipe is None: + raise PartProblem("part_tampered", f"第 {self.info.seq} 次导入没有配方记录，无法重建当前版本") + return UnionPart(import_id=self.info.import_id, start=self.info.start, end=self.info.end, + db_path=self.info.db_path, recipe=self.info.recipe, db_sha256=self.info.db_sha256, + manifest_db_sha256=self.info.manifest_db_sha256, table_hashes=dict(self.table_hashes)) + + +def _period_label(start: str | None, end: str | None) -> str: + return f"{start} 至 {end}" if start and end else "统计期未记录" + + +def load_manifest(artifact_id: str | None) -> dict[str, Any] | None: + """取回一份工件（导入清单、快照清单）。内容哈希对不上时抛 ValueError（artifact_store.load 自己复验）；不在为 None。""" + if not artifact_id: + return None + from app.core import artifact_store + + try: + doc = artifact_store.load(artifact_id) + except OSError: + return None + return doc if isinstance(doc, dict) else None + + +def _rows_of(report: dict[str, Any] | None) -> dict[str, int]: + """构建回执里各表的行数（执行器回执、期 1 的解析回执都是 tables: [{name, rows}]）。""" + out: dict[str, int] = {} + for t in (report or {}).get("tables") or []: + if isinstance(t, dict) and t.get("name") is not None: + try: + out[str(t["name"])] = int(t.get("rows") or 0) + except (TypeError, ValueError): + continue + return out + + +def _part_from(imp: TableImport, build: TableBuild | None, manifest: dict[str, Any] | None, seq: int) -> Part: + """一条导入记录 → Part。统计期取自导入清单，再与库列比对（2.3）。seq 是它在当前快照里的序号（报错用）。""" + label = f"第 {seq} 期" + if manifest is None: + raise PartProblem("part_tampered", f"{label}（第 {imp.seq} 次导入）的导入清单不在了，无法核对这一期的数据。" + "可以在数据源卡片的「版本」中启用更早的版本") + period = manifest.get("period") if isinstance(manifest.get("period"), dict) else {} + start, end = period.get("start"), period.get("end") + if (start or None) != (imp.period_start or None) or (end or None) != (imp.period_end or None): + logger.warning("导入记录的统计期与导入清单不一致：import=%s 清单=%s~%s 库列=%s~%s", + imp.id, start, end, imp.period_start, imp.period_end) + raise PartProblem("part_tampered", f"{label}（第 {imp.seq} 次导入）登记的统计期与它的导入清单不一致，" + "可能被修改过。可以在数据源卡片的「版本」中启用更早的版本") + if build is None: + raise PartProblem("part_missing", f"{label}（{_period_label(start, end)}）的数据文件已丢失，无法重建当前版本。" + "可以先移除这一期，或在数据源卡片的「版本」中启用更早的版本") + meta = manifest.get("recipe") if isinstance(manifest.get("recipe"), dict) else {} + recipe: Recipe | None = None + try: + recipe = Recipe.model_validate(meta.get("canonical")) if meta.get("canonical") is not None else None + except ValueError: + recipe = None + hashes = manifest.get("table_hashes") if isinstance(manifest.get("table_hashes"), dict) else {} + info = PartInfo( + import_id=imp.id, seq=imp.seq, start=str(start or ""), end=str(end or ""), build_id=imp.build_id, + db_path=build.db_path, db_sha256=str(build.db_sha256 or ""), + manifest_db_sha256=manifest.get("db_sha256"), recipe_id=imp.recipe_id, recipe=recipe, + recipe_sha256=meta.get("sha256"), raw_state=imp.raw_state, file_name=imp.file_name or "", + manifest_artifact=imp.manifest_artifact, rows=_rows_of(build.report), + overrides=len(imp.overrides or []), waivers=len(imp.waivers or []), + ) + return Part(info=info, manifest=manifest, table_hashes={str(k): str(v) for k, v in hashes.items()}, + raw_sha=imp.raw_sha256 or "", period_source=period.get("source")) + + +async def load_parts(session: AsyncSession, snapshot: SourceSnapshot | None, *, + exclude: str | None = None) -> list[Part]: + """当前快照的各期（按快照记录的顺序，即按统计期）。简单导入的快照没有配方各期，返回空列表。 + + 任何一期读不出来抛 PartProblem：被改过的统计期、不在的清单都不能悄悄跳过，否则结果版本会少一期。 + + exclude：移除、作废这一期时，被移除的那一期**先排除、不校验**（7.4「移除的正好是那一期时，它的文件不参与物化， + 不报这个错」）。先全部校验再过滤的话，坏掉的正是要移除的那一期时反而移不掉，part_missing 的消息还建议「先移除 + 这一期」，走不通（评审意见）。序号（报错里的「第 N 期」）仍按它在当前快照里的位置算，与版本页一致。""" + if snapshot is None or not snapshot.imports: + return [] + ids = [str(x) for x in snapshot.imports] + imps = {i.id: i for i in (await session.execute(select(TableImport).where(TableImport.id.in_(ids)))).scalars()} + if not any(i.recipe_id for i in imps.values()): + return [] + builds = {b.id: b for b in (await session.execute( + select(TableBuild).where(TableBuild.id.in_([i.build_id for i in imps.values()])))).scalars()} + parts: list[Part] = [] + for k, iid in enumerate(ids, 1): + if exclude is not None and iid == exclude: + continue + imp = imps.get(iid) + if imp is None: + raise PartProblem("part_missing", f"第 {k} 期的导入记录不在了，无法重建当前版本。" + "可以在数据源卡片的「版本」中启用更早的版本") + try: + manifest = await asyncio.to_thread(load_manifest, imp.manifest_artifact) + except ValueError: + raise PartProblem("part_tampered", f"第 {k} 期（第 {imp.seq} 次导入）的导入清单与它的哈希不符，" + "可能被修改过") from None + parts.append(_part_from(imp, builds.get(imp.build_id), manifest, k)) + return parts + + +def _columns_from(cols: Any) -> list[ColumnOut]: + out = [] + for c in cols or []: + if isinstance(c, dict) and c.get("name") is not None: + out.append(ColumnOut(name=str(c["name"]), type=str(c.get("type") or "TEXT"), header=c.get("header"), + unit=c.get("unit"), role=c.get("role") or "measure")) + return out + + +def _union_build_id(snapshot: SourceSnapshot) -> str | None: + """快照库是并集库时，它的并集构建 id（文件名）。单期快照的库就是那一期的构建库，返回 None。""" + path = Path(snapshot.db_path or "") + if path.parent.name != table_versions.UNION_DIR: + return None + return path.stem + + +async def current_tables(session: AsyncSession, snapshot: SourceSnapshot | None) -> dict[str, list[ColumnOut]] | None: + """当前版本实际的表结构（含早先退役、仍保留在并集里的列）：并集取并集构建登记的回执（UnionReport.tables）， + 单期取那一期构建回执的 tables。两样都取不到时退回快照冻结的 schema_cache（只有列名和类型）。""" + if snapshot is None: + return None + uid = _union_build_id(snapshot) + if uid is not None: + build = await session.get(TableBuild, uid) + tables = (build.report or {}).get("tables") if build is not None else None + if isinstance(tables, dict) and tables: + return {str(name): _columns_from(cols) for name, cols in tables.items()} + elif snapshot.imports: + imp = await session.get(TableImport, str(snapshot.imports[-1])) + build = await session.get(TableBuild, imp.build_id) if imp is not None else None + tables = (build.report or {}).get("tables") if build is not None else None + if isinstance(tables, list) and tables: + return {str(t["name"]): _columns_from(t.get("columns")) for t in tables + if isinstance(t, dict) and t.get("name") is not None} + cache = snapshot.schema_cache or {} + out: dict[str, list[ColumnOut]] = {} + for name, meta in (cache.get("tables") or {}).items(): + cols = meta.get("columns") if isinstance(meta, dict) else None + out[str(name)] = _columns_from(cols) + return out or None + + +def tables_in_parts(tables: dict[str, list[ColumnOut]] | None, + parts: list[Part], *, also: Recipe | None = None) -> dict[str, list[ColumnOut]] | None: + """当前版本的表结构里，只留下至少一期剩下的构建里有的表和列（顺序照旧）。移除、作废一期时用。 + + also：这份配方（试运行的目标配方）推出的表和列也留下。试运行路径用它（WP-8 修补）：parts 给结果各期（当前版本的 + 各期去掉本期要替换的那一期，recipe_imports._replaced_part），筛掉「结果各期都没有、目标配方也没有」的幽灵列， + 免得累积计划把它当成「这次才退役」、出一条任何一期都没有数据的必勾项（文案还写「早期各期保留原值」）。两种情况： + 移除唯一含「分区丙」的那一期之后，当前并集按目标配方仍留着一列全为空值的分区丙；替换唯一含分区丙的那一期， + 被替换的旧数据不在新版本里。后一种是这一列从现行配方里消失，由 breaking:<表> 要人确认，不会变成静默。结果各期 + 有数据的列照旧留着，真正的退役照样要确认（H2、H3）；目标配方里有的列也照旧留着，不会因此多报「新增列」。 + + 为什么要筛：当前并集的表结构是有历史的。被移除的那一期独有的列（例如只有 9 月有的「分区丙」，10 月起退役）还 + 留在当前并集里，直接拿去比，剩下的 8 月、10 月都没有它，新并集却多出一列全为空值的「幽灵」退役列，快照清单把 + 它记成退役、since 还是第一期；更糟的是并集 id 随之不同，而快照 id 只看（剩下各期, 目标配方, 模式）：复活已回收 + 的同 id 快照时内容对不上，publish_snapshot 报 snapshot_conflict，用户根本移除不了（评审意见）。筛过之后并集只 + 取决于（剩下各期, 目标配方），与快照 id 的口径一致（2.1「同一组导入、同一个目标配方、同为累积模式，得到同一个 + 快照 id，内容也必然相同」）。 + + 各期构建库的表结构就是它的配方 derive_tables 推出的那份（执行器照它建表），所以按各期配方判断，不必打开库。 + 留下的列保持当前并集里的顺序：退役列的先后进建表语句、也就进库文件的哈希，要与这组导入当初物化时一致。""" + if tables is None: + return None + have: dict[str, set[str]] = {} + for rec in [*(part.info.recipe for part in parts), also]: + if rec is None: + continue + derived, _problems = recipe_types.derive_tables(rec) + for name, cols in derived.items(): + have.setdefault(name, set()).update(c.name for c in cols) + out: dict[str, list[ColumnOut]] = {} + for name, cols in tables.items(): + names = have.get(name) + if names is None: + continue + kept = [c for c in cols if c.name in names] + if kept: + out[name] = kept + return out + + +def snapshot_manifest(snapshot: SourceSnapshot | None) -> dict[str, Any] | None: + """快照的快照清单（物化过的累积快照才有）。被改过时抛 PartProblem(part_tampered)：退役记录靠它，不能当作没有。""" + if snapshot is None or not snapshot.manifest_artifact: + return None + try: + doc = load_manifest(snapshot.manifest_artifact) + except ValueError: + raise PartProblem("part_tampered", "当前版本的版本清单与它的哈希不符，可能被修改过") from None + if doc is None: + raise PartProblem("part_tampered", "当前版本的版本清单不在了，无法确认此前退役的列") + return doc + + +def retired_status(manifest: dict[str, Any] | None) -> dict[str, dict[str, dict]] | None: + """快照清单的 columns（表 → 列 → {status, …}），classify_changes 据此区分「早已退役」与「这次才退役」。""" + cols = (manifest or {}).get("columns") + return copy.deepcopy(cols) if isinstance(cols, dict) else None + + +def retired_names(manifest: dict[str, Any] | None) -> dict[str, list[str]]: + """当前版本里退役的表名 → 该表退役的列名（框选的撞名检查用，recipe_select 的 retired_names）。""" + out: dict[str, list[str]] = {} + for table, cols in (retired_status(manifest) or {}).items(): + names = [str(c) for c, meta in (cols or {}).items() + if isinstance(meta, dict) and meta.get("status") == "retired"] + if names: + out[str(table)] = names + return out + + +def acceptances_of(manifest: dict[str, Any]) -> list[Acceptance]: + acc = manifest.get("acceptances") if isinstance(manifest.get("acceptances"), dict) else {} + out = [] + for rec in [*(acc.get("overrides") or []), *(acc.get("waivers") or [])]: + if isinstance(rec, dict) and rec.get("check_id"): + out.append(Acceptance(check_id=str(rec["check_id"]), reason=str(rec.get("reason") or ""), + signed_by=rec.get("signed_by"))) + return out + + +def note_part(part: Part) -> UnionNotePart: + """一期的说明输入（同步，线程里跑：空值数要查该期的构建库）。数据取自该期的导入清单和构建库（2.8）。""" + p = pipeline() + recipe = part.info.recipe + if recipe is None: + raise PartProblem("part_tampered", f"第 {part.info.seq} 次导入没有配方记录，无法生成说明") + receipt = part.manifest.get("receipt") if isinstance(part.manifest.get("receipt"), dict) else {} + ex = extraction_from_dict(receipt) + checks = [check_from_dict(c) for c in part.manifest.get("checks") or [] if isinstance(c, dict)] + nulls = p.column_null_counts(part.info.db_path, recipe) if Path(part.info.db_path).is_file() else None + return UnionNotePart(recipe=recipe, extraction=ex, checks=checks, acceptances=acceptances_of(part.manifest), + null_counts=nulls, start=part.info.start, end=part.info.end) + + +def single_notes(part: Part) -> SchemaNotes: + """一期、快照库就是它的构建库时的说明（同步）：照它导入时的写法重新生成。""" + p = pipeline() + src = note_part(part) + return p.build_notes(src.recipe, src.extraction, src.checks, src.acceptances, null_counts=src.null_counts) + + +def gaps_of(periods: list[tuple[str | None, str | None]]) -> list[dict[str, str]]: + """相邻两期之间的空缺（前一期 end 加一天 < 后一期 start）。没有统计期的期不参与。""" + import datetime as _dt + + spans = sorted((s, e) for s, e in periods if s and e) + out: list[dict[str, str]] = [] + for (_s1, e1), (s2, _e2) in zip(spans, spans[1:]): + try: + after = _dt.date.fromisoformat(str(e1)) + _dt.timedelta(days=1) + before = _dt.date.fromisoformat(str(s2)) - _dt.timedelta(days=1) + except ValueError: + continue + if after <= before: + out.append({"start": after.isoformat(), "end": before.isoformat()}) + return out + + +# ========================================================================== +# 快照清单（2.9） +# ========================================================================== + + +def _since(periods: list[str], having: list[str]) -> str | None: + """一列（或一张表）自哪一期起没有了：最后一个含它的期之后的第一期。""" + last = max((periods.index(p) for p in having if p in periods), default=-1) + return periods[last + 1] if last + 1 < len(periods) else None + + +def columns_doc(report: dict[str, Any], periods: list[str]) -> dict[str, dict[str, dict[str, Any]]]: + """快照清单的 columns：部分期才有的列标 added（periods 列出哪几期有），退役的列标 retired（since、unit）。 + 整张表退役时把它的每一列都标 retired：classify_changes 要每一列都是 retired 才认作「早已退役」（WP-4 交接）。""" + tables = report.get("tables") if isinstance(report.get("tables"), dict) else {} + out: dict[str, dict[str, dict[str, Any]]] = {} + + def unit_of(table: str, column: str) -> str | None: + for c in tables.get(table) or []: + if isinstance(c, dict) and c.get("name") == column: + return c.get("unit") + return None + + for entry in report.get("added") or []: + table, column, having = entry.get("table"), entry.get("column"), list(entry.get("periods") or []) + names = [column] if column else [c.get("name") for c in tables.get(table) or [] if isinstance(c, dict)] + for name in names: + if name: + out.setdefault(str(table), {})[str(name)] = {"status": "added", "periods": having} + for entry in report.get("retired") or []: + table, column, having = entry.get("table"), entry.get("column"), list(entry.get("periods") or []) + names = [column] if column else [c.get("name") for c in tables.get(table) or [] if isinstance(c, dict)] + for name in names: + if name: + out.setdefault(str(table), {})[str(name)] = { + "status": "retired", "since": _since(periods, having), "unit": unit_of(str(table), str(name))} + return out + + +def manifest_doc(*, source_id: str, snapshot_id: str, action: str, recipe: dict[str, Any], + union: dict[str, Any], parts: list[Part | dict[str, Any]], report: dict[str, Any], + label_sets: list[Any], gaps: list[dict[str, str]], replaced: Any, removed: Any, + notes: SchemaNotes, signed_by: str | None) -> dict[str, Any]: + """快照清单（2.9）。parts 按统计期排序，与 report.part_rows 同序；本期写成 dict（导入记录还没落库）。""" + part_rows = report.get("part_rows") or [] + keys = [] + out_parts = [] + for k, part in enumerate(parts): + if isinstance(part, Part): + d = {"import_id": part.info.import_id, "seq": part.info.seq, + "period": {"start": part.info.start, "end": part.info.end}, "build_id": part.info.build_id, + "db_sha256": part.info.db_sha256, "recipe_id": part.info.recipe_id, + "recipe_sha256": part.info.recipe_sha256, "manifest_artifact": part.info.manifest_artifact, + "file_name": part.info.file_name, "raw_sha256": part.raw_sha} + else: + d = dict(part) + d["rows"] = copy.deepcopy(part_rows[k]) if k < len(part_rows) else {} + keys.append(f"{d['period']['start']}~{d['period']['end']}") + out_parts.append(d) + return { + "format": SNAPSHOT_MANIFEST_FORMAT, + "source_id": source_id, "snapshot_id": snapshot_id, "mode": "accumulate", "action": action, + "recipe": recipe, "union": union, "parts": out_parts, + "columns": columns_doc(report, keys), + "label_sets": copy.deepcopy(list(label_sets or [])), + "kinds": dict(notes.kinds), + "gaps": list(gaps), "replaced": copy.deepcopy(replaced), "removed": copy.deepcopy(removed), + "checks": copy.deepcopy(report.get("checks") or []), + "notes": {"templates": dict(notes.templates), + "rendered": {t: {"comment": n.comment, "columns": dict(n.columns)} for t, n in notes.tables.items()}}, + "signed_by": {"name": signed_by, "verified": False}, + "created_at": utcnow().isoformat(), + } + + +def union_table_report(report: dict[str, Any]) -> dict[str, Any]: + """check_probe 要的「回执」形状（tables: [{name}]）：并集回执的 tables 是「表 → 列」的字典。""" + tables = report.get("tables") if isinstance(report.get("tables"), dict) else {} + return {"tables": [{"name": name} for name in tables]} + + +# ========================================================================== +# 版本页：列表（7.1） +# ========================================================================== + + +def _iso(v: Any) -> str | None: + return v.isoformat() if v is not None else None + + +async def _imports_of(session: AsyncSession, ids: set[str]) -> dict[str, TableImport]: + if not ids: + return {} + return {i.id: i for i in (await session.execute(select(TableImport).where(TableImport.id.in_(list(ids))))).scalars()} + + +async def _builds_of(session: AsyncSession, ids: set[str]) -> dict[str, TableBuild]: + if not ids: + return {} + return {b.id: b for b in (await session.execute(select(TableBuild).where(TableBuild.id.in_(list(ids))))).scalars()} + + +async def _recipe_seqs(session: AsyncSession, ids: set[str]) -> dict[str, int]: + ids = {i for i in ids if i} + if not ids: + return {} + return dict((await session.execute(select(TableRecipe.id, TableRecipe.seq).where(TableRecipe.id.in_(list(ids))))) + .all()) + + +def snapshot_recipe_id(snap: SourceSnapshot, imps: dict[str, TableImport]) -> str | None: + """快照的配方：期 3 之前的快照没记 recipe_id，取最后一期导入的（9.6）；简单导入为 None。""" + if snap.recipe_id: + return snap.recipe_id + last = imps.get(str(snap.imports[-1])) if snap.imports else None + return last.recipe_id if last is not None else None + + +def _period_ref(imp: TableImport | None) -> dict[str, Any]: + if imp is None: + return {"start": None, "end": None, "file_name": ""} + return {"start": imp.period_start, "end": imp.period_end, "file_name": imp.file_name or ""} + + +def _period_key(imp: TableImport) -> tuple[str, ...]: + """periods_diff 按 (start, end) 比；没有统计期（简单导入）的期没法按统计期认，按导入记录本身认。""" + if imp.period_start and imp.period_end: + return ("period", imp.period_start, imp.period_end) + return ("import", imp.id) + + +def _file_ok(path: str | None) -> bool: + try: + return bool(path) and Path(path).is_file() + except OSError: + return False + + +def _size(path: str | None) -> int | None: + try: + return Path(path).stat().st_size if path else None + except OSError: + return None + + +async def _pinned_counts(session: AsyncSession) -> dict[str, int]: + """快照 id → 引用它的运行数（任何状态），写法同 gc（table_versions._snapshots_in）。""" + out: dict[str, int] = {} + for (versions,) in (await session.execute(select(Run.data_versions).where(Run.data_versions.is_not(None)))).all(): + for sid in table_versions._snapshots_in(versions): + out[sid] = out.get(sid, 0) + 1 + return out + + +async def list_snapshots(session: AsyncSession, source: DataSource) -> list[dict[str, Any]]: + """GET /{source_id}/snapshots：最近 50 个版本，当前版本排第一，其余按 coalesce(activated_at, created_at) 倒序。""" + if (source.origin or "manual") != "upload": + return [] + snaps = list((await session.execute(select(SourceSnapshot).where(SourceSnapshot.source_id == source.id))).scalars()) + current = next((s for s in snaps if s.id == source.current_snapshot_id), None) + others = sorted((s for s in snaps if s is not current), key=lambda s: (table_versions._snapshot_time(s), s.id), + reverse=True) + listed = ([current] if current is not None else []) + others + listed = listed[:SNAPSHOTS_MAX] + imps = await _imports_of(session, {str(x) for s in listed for x in s.imports or []}) + builds = await _builds_of(session, {i.build_id for i in imps.values()} + | {uid for s in listed if (uid := _union_build_id(s))}) + seqs = await _recipe_seqs(session, {snapshot_recipe_id(s, imps) or "" for s in listed}) + pinned = await _pinned_counts(session) + cur_keys = {_period_key(imps[str(x)]): imps[str(x)] for x in (current.imports or []) if str(x) in imps} \ + if current is not None else {} + out = [] + for s in listed: + members = [imps[str(x)] for x in s.imports or [] if str(x) in imps] + parts = [] + for imp in members: + b = builds.get(imp.build_id) + parts.append({ + "import_id": imp.id, "seq": imp.seq, "period_start": imp.period_start, "period_end": imp.period_end, + "file_name": imp.file_name or "", "raw_state": imp.raw_state, "status": imp.status, + "rows": _rows_of(b.report if b is not None else None), + "overrides": len(imp.overrides or []), "waivers": len(imp.waivers or []), "revoked": bool(imp.revoked), + }) + uid = _union_build_id(s) + if uid is not None and builds.get(uid) is not None: + tables = dict((builds[uid].report or {}).get("rows") or {}) + elif members: + b = builds.get(members[-1].build_id) + tables = _rows_of(b.report if b is not None else None) + else: + tables = {} + available = s.retired_at is None and _file_ok(s.db_path) + revoked = any(i.revoked for i in members) + if s is current: + code = "current" + elif s.retired_at is not None: + code = "retired" + elif not available: + code = "file_lost" + elif revoked: + code = "contains_revoked" + else: + code = None + rid = snapshot_recipe_id(s, imps) + mine = {_period_key(i): i for i in members} + diff = {"added": [_period_ref(i) for k, i in mine.items() if k not in cur_keys], + "removed": [_period_ref(i) for k, i in cur_keys.items() if k not in mine]} + out.append({ + "id": s.id, "current": s is current, "mode": s.mode or ("replace" if rid else None), + "created_at": _iso(s.created_at), "activated_at": _iso(s.activated_at or s.created_at), + "recipe": {"id": rid, "seq": seqs.get(rid)} if rid else None, + "parts": parts, "tables": tables, + "db_sha256_prefix": (s.db_sha256 or "")[:8], "db_size": _size(s.db_path) if available else None, + "available": available, "pinned_runs": pinned.get(s.id, 0), + "activatable": code is None, "reason_code": code, + "reason": SNAPSHOT_NOT_ACTIVATABLE[code] if code else None, + "mask_lost": table_versions.mask_lost(source.options, s.schema_cache), + "periods_diff": diff, + }) + return out + + +# ========================================================================== +# 版本页：清单（7.3） +# ========================================================================== + + +async def import_manifest(session: AsyncSession, source: DataSource, import_id: str) -> dict[str, Any]: + imp = await session.get(TableImport, import_id) + if imp is None or imp.source_id != source.id: + raise ImportRefused(404, "import_not_found", "导入记录不存在") + if not imp.manifest_artifact: + if imp.recipe_id: + raise ImportRefused(404, "manifest_missing", "这次按配方导入的导入清单不在了，请联系管理员检查数据目录") + build = await session.get(TableBuild, imp.build_id) if imp.build_id else None + return {"artifact_id": None, "verified": False, "kind": "build_report", + "content": copy.deepcopy(build.report or {}) if build is not None else {}} + try: + doc = await asyncio.to_thread(load_manifest, imp.manifest_artifact) + except ValueError: + raise ImportRefused(409, "manifest_tampered", "这份导入清单与它的哈希不符，可能被修改过，不能作为证据") from None + if doc is None: + raise ImportRefused(404, "manifest_missing", "这次导入的导入清单不在了，请联系管理员检查数据目录") + return {"artifact_id": imp.manifest_artifact, "verified": True, "kind": "import_manifest", "content": doc} + + +async def snapshot_manifest_out(session: AsyncSession, source: DataSource, snapshot_id: str) -> dict[str, Any]: + snap = await session.get(SourceSnapshot, snapshot_id) + if snap is None or snap.source_id != source.id: + raise ImportRefused(404, "snapshot_not_found", "这个版本不存在，或不属于这个数据源") + if snap.manifest_artifact: + try: + doc = await asyncio.to_thread(load_manifest, snap.manifest_artifact) + except ValueError: + raise ImportRefused(409, "manifest_tampered", "这份版本清单与它的哈希不符，可能被修改过，不能作为证据") from None + if doc is None: + raise ImportRefused(404, "manifest_missing", "这个版本的版本清单不在了，请联系管理员检查数据目录") + return {"artifact_id": snap.manifest_artifact, "verified": True, "kind": "snapshot_manifest", "content": doc} + imps = await _imports_of(session, {str(x) for x in snap.imports or []}) + return {"artifact_id": None, "verified": False, "kind": "snapshot_view", "content": { + "mode": snap.mode or "replace", + "imports": [{"import_id": str(x), "manifest_artifact": imps[str(x)].manifest_artifact if str(x) in imps else None} + for x in snap.imports or []], + "db_sha256": snap.db_sha256, "schema_cache_keys": sorted((snap.schema_cache or {}).keys()), + }} + + +# ========================================================================== +# 版本页：启用旧版本（7.2） +# ========================================================================== + +#: activate_snapshot 的错误码 → HTTP 状态 +_ACTIVATE_STATUS = {"snapshot_not_found": 404} + + +def _publish_refused(e: table_versions.PublishError) -> ImportRefused: + """版本页写操作里 publish_snapshot / activate_snapshot 的错误 → ImportRefused（接口层原样转 CodedHTTPException）。""" + code = e.code or "publish_failed" + if code == "store_unavailable": + return ImportRefused(503, code, str(e)) + if code in _ACTIVATE_STATUS: + return ImportRefused(_ACTIVATE_STATUS[code], code, str(e)) + if code in ("publish_failed", "trial_missing", "raw_missing", "snapshot_conflict"): + return ImportRefused(500, code, str(e) or "操作失败，当前版本未受影响") + return ImportRefused(409, code, str(e)) + + +async def activate(session: AsyncSession, source: DataSource, snapshot_id: str, *, expected_current: str | None, + ack_mask_lost: list[str] | None, reason: str | None, signed_by: str | None) -> dict[str, Any]: + """POST …/snapshots/{id}/activate（7.2）。不存在或不属于这个源的先回 404，其余交给 activate_snapshot（锁内）。""" + snap = await session.get(SourceSnapshot, snapshot_id) + if snap is None or snap.source_id != source.id: + raise ImportRefused(404, "snapshot_not_found", "这个版本不存在，或不属于这个数据源") + source_id = source.id + try: + previous = await table_versions.activate_snapshot( + session, source, snapshot_id, expected_current=expected_current, kind="activate", reason=reason, + signed_by=signed_by, ack_mask_lost=ack_mask_lost) + except table_versions.PublishError as e: + raise _publish_refused(e) from e + fresh = await session.get(DataSource, source_id, populate_existing=True) + return {"snapshot_id": snapshot_id, "previous_snapshot_id": previous, + "recipe_id": fresh.current_recipe_id if fresh is not None else None} + + +# ========================================================================== +# 版本页：移除一期（7.4）、作废接受（7.5） +# ========================================================================== + + +@dataclass +class _Current: + source: DataSource + snapshot: SourceSnapshot + imports: list[str] + mode: str + recipe: TableRecipe | None + + +async def _current(session: AsyncSession, source: DataSource) -> _Current: + if (source.origin or "manual") != "upload" or not source.current_snapshot_id: + raise ImportRefused(409, "not_upload_source", "这个数据源不是上传的表格，没有可以操作的版本") + snap = await session.get(SourceSnapshot, source.current_snapshot_id) + if snap is None: + raise ImportRefused(409, "snapshot_missing", "当前版本的记录不在了，请联系管理员检查数据目录") + imps = await _imports_of(session, {str(x) for x in snap.imports or []}) + rid = snapshot_recipe_id(snap, imps) + recipe = await session.get(TableRecipe, rid) if rid else None + return _Current(source=source, snapshot=snap, imports=[str(x) for x in snap.imports or []], + mode=snap.mode or "replace", recipe=recipe) + + +def _revoked_record(reason: str, signed_by: str | None) -> dict[str, Any]: + return {"at": utcnow().isoformat(), "reason": reason, "signed_by": signed_by or None, "signed_by_verified": False} + + +async def rollback_target(session: AsyncSession, source: DataSource, import_id: str) -> SourceSnapshot | None: + """替换模式下作废接受的回滚目标（第 5 节，评审一-M8）：同源快照里保留着、可以启用（7.1 的 activatable）、不含 + 这条导入、也不含任何已作废导入的，按 coalesce(activated_at, created_at) 倒序的第一个。""" + snaps = list((await session.execute(select(SourceSnapshot).where(SourceSnapshot.source_id == source.id))).scalars()) + revoked = {i for (i,) in (await session.execute( + select(TableImport.id).where(TableImport.source_id == source.id, TableImport.revoked.is_not(None)))).all()} + snaps.sort(key=lambda s: (table_versions._snapshot_time(s), s.id), reverse=True) + for s in snaps: + members = {str(x) for x in s.imports or []} + if (s.id == source.current_snapshot_id or s.retired_at is not None or not _file_ok(s.db_path) + or import_id in members or members & revoked): + continue + return s + return None + + +def _accumulating(cur: _Current) -> bool: + """作废接受走「移除这一期」还是「回滚」：累积模式而且至少两期时移除；只有一期时没有可留下的，按回滚处理。""" + return cur.mode == "accumulate" and len(cur.imports) >= 2 + + +async def revoke_plan(session: AsyncSession, source: DataSource, imp: TableImport, + cur: _Current | None = None) -> dict[str, Any]: + """导入记录的 revoke_plan（7.6）：确认框照它写，服务端在提交时按同一个定义重算、比对。""" + cur = cur or await _current(session, source) + if _accumulating(cur): + keep = [i for i in cur.imports if i != imp.id] + imps = await _imports_of(session, set(keep)) + rows = [imps[i] for i in keep if i in imps] + target = (table_versions.snapshot_id(source.id, keep, recipe_types.RECIPE_ENGINE_VER, mode="accumulate", + recipe_sha=cur.recipe.recipe_sha256) if cur.recipe is not None else None) + return {"action": "remove_period", "target_snapshot_id": target, "target": None, + "result_parts": [_period_ref(i) for i in rows], + "gaps": gaps_of([(i.period_start, i.period_end) for i in rows]), "mask_lost": [], "reason": None} + target = await rollback_target(session, source, imp.id) + if target is None: + return {"action": None, "target_snapshot_id": None, "target": None, "result_parts": [], "gaps": [], + "mask_lost": [], "reason": _NO_ROLLBACK} + imps = await _imports_of(session, {str(x) for x in target.imports or []}) + rows = [imps[str(x)] for x in target.imports or [] if str(x) in imps] + rid = snapshot_recipe_id(target, imps) + seqs = await _recipe_seqs(session, {rid or ""}) + refs = [_period_ref(i) for i in rows] + return {"action": "rollback", "target_snapshot_id": target.id, + "target": {"parts": refs, "recipe_seq": seqs.get(rid) if rid else None, + "mode": (target.mode or "replace") if rid else None, "simple": rid is None}, + "result_parts": refs, "gaps": gaps_of([(i.period_start, i.period_end) for i in rows]), + "mask_lost": table_versions.mask_lost(source.options, target.schema_cache), "reason": None} + + +async def _rebuild_without(session: AsyncSession, cur: _Current, imp: TableImport, *, expected_current: str | None, + kind: str, reason: str, signed_by: str | None, revoke: bool) -> tuple[str, bool]: + """从当前版本去掉一期、生成（或复用、复活）新快照（7.4）。配方和导入模式不变。返回 (快照 id, 是否复用)。 + + revoke=True 时同一个事务里写 table_imports.revoked（7.5）：复用路径在调 activate_snapshot 之前改好（随它的提交 + 生效、失败时一起回滚）；新建、复活路径在 before_commit 里写（publish_snapshot 之前不能改会话里的对象）。""" + source = cur.source + source_id = source.id + target = cur.recipe + if target is None: + raise ImportRefused(409, "not_accumulate", "当前版本没有配方记录，不能按期移除") + try: + target_recipe = Recipe.model_validate(target.recipe) + except ValueError: + raise ImportRefused(409, "not_accumulate", "当前版本的配方记录读不出来，不能按期移除") from None + target_sha, target_id = target.recipe_sha256, target.id + keep = [i for i in cur.imports if i != imp.id] + sid = table_versions.snapshot_id(source_id, keep, recipe_types.RECIPE_ENGINE_VER, mode="accumulate", + recipe_sha=target_sha) + imp_id = imp.id + revoked = _revoked_record(reason, signed_by) if revoke else None + activation = {"kind": kind, "import_id": imp_id, "reason": reason, "signed_by": signed_by} + + existing = await session.get(SourceSnapshot, sid) + if (existing is not None and existing.retired_at is None and existing.source_id == source_id + and await asyncio.to_thread(table_versions._file_matches, existing.db_path, existing.db_sha256)): + # 同一组导入、同一个目标配方、同为累积：内容必然相同，直接切过去（7.4 第 3 步）。遮罩：目标配方不变，表结构 + # 只可能多出退役列，不会少列（7.4 第 6 步），所以把算出的遮罩丢失原样作为确认传下去，不让它额外拦一道 + lost = table_versions.mask_lost(source.options, existing.schema_cache) + if revoked is not None: + imp.revoked = revoked + try: + await table_versions.activate_snapshot( + session, source, sid, expected_current=expected_current, kind=kind, reason=reason, + signed_by=signed_by, import_id=imp_id, ack_mask_lost=lost) + except table_versions.PublishError as e: + raise _publish_refused(e) from e + return sid, True + + # 已回收的同 id 快照要复活时，配方记录可能与目标配方不是同一条（内容相同：快照 id 按配方哈希算），由 + # publish_snapshot 按哈希认出同一个版本，现行配方跟着那个快照记的配方记录走（与上面「直接切过去」的路径一致） + try: + parts = await load_parts(session, cur.snapshot, exclude=imp_id) + manifest = snapshot_manifest(cur.snapshot) + except PartProblem as e: + raise ImportRefused(409, e.code, e.message) from e + cur_tables = tables_in_parts(await current_tables(session, cur.snapshot), parts) + status = retired_status(manifest) + p = pipeline() + single = len(parts) == 1 and parts[0].info.recipe_sha256 == target_sha + union_build: table_versions.UnionBuild | None = None + report_json: dict[str, Any] | None = None + label_sets: list[Any] = [] + gaps = gaps_of([(x.info.start, x.info.end) for x in parts]) + out: Path | None = None + try: + if single: + notes = await table_versions_slot(single_notes, parts[0]) + else: + from app.data import recipe_accumulate as ra + + classify = p.classify_changes or ra.classify_changes + cc = classify([x.info for x in parts], target_recipe, current_tables=cur_tables, retired_status=status) + if cc.semantic: + raise ImportRefused(409, "union_failed", "当前配方与剩下的某一期不兼容，无法重建当前版本：" + + "；".join(cc.semantic[:3])) + label_sets = list(cc.label_sets) + retired = ra.retired_columns(cc, cur_tables) + triples = [(x.info.start, x.info.end, x.info.build_id) for x in parts] + uid = ra.union_id(source_id, triples, target_sha, retired=retired) + final = table_versions.union_path(source_id, uid) + out = final.with_name(f"{final.name}{raw_store.TMP_MARK}{secrets.token_hex(6)}") + materialize = p.materialize_union or ra.materialize_union + union_parts = [x.union_part() for x in parts] + report = await table_versions_slot(materialize, out, target=target_recipe, parts=union_parts, + retired=retired) + ra.require_union_ok(report) + report_json = _jsonable(report) + build_union_notes = p.build_union_notes + if build_union_notes is None: + from app.data.recipe_notes import build_union_notes + note_parts = [await asyncio.to_thread(note_part, x) for x in parts] + notes = build_union_notes( + target_recipe, note_parts, union_tables=report.tables, null_counts=report.null_counts, + part_null_counts=report.part_null_counts, added=report.added, retired=report.retired, + label_sets=label_sets, gaps=bool(gaps), dropped=False) + union_build = table_versions.UnionBuild( + tmp_db=out, union_id=uid, options=ra.union_options(triples, target_sha, retired=retired), + report=report_json, expected_sha256=report.db_sha256) + except PartProblem as e: + _drop(out) + raise ImportRefused(409, e.code, e.message) from e + except ImportRefused: + _drop(out) + raise + except Exception as e: + _drop(out) + from app.data.recipe_accumulate import UnionError + + if isinstance(e, UnionError): + raise ImportRefused(409, e.code, str(e)) from e + raise + if notes.problems: + _drop(out) + raise ImportRefused(422, "notes_invalid", "说明没有通过检查：" + "；".join(notes.problems[:5])) + + manifests = [x.info.manifest_artifact for x in parts] + removed = {"import_id": imp_id, "seq": imp.seq, "start": imp.period_start, "end": imp.period_end, + "file_name": imp.file_name or ""} + action = "revoke_acceptance" if revoke else "remove_period" + + async def schema_cache_for(view: table_versions.SourceView, info: table_versions.SnapshotInfo) -> dict[str, Any]: + from app.core import artifact_store + + if union_build is not None: + probe = union_table_report(info.union_report or report_json or {}) + else: + probe = {"tables": [{"name": name} for name in parts[0].info.rows]} if parts[0].info.rows else \ + {"tables": [{"name": t} for t in notes.tables]} + cache = await table_versions.introspect_view(view, probe) + cache = p.apply_notes(cache, notes) + if notes.problems: + raise table_versions.PublishError("说明没能写进表结构（" + "；".join(notes.problems[:3]) + "），当前版本未受影响") + table_versions.check_probe(cache, probe, source_id) + cache["import_manifests"] = [m for m in manifests if m] + if union_build is not None: + doc = manifest_doc( + source_id=source_id, snapshot_id=info.snapshot_id, action=action, + recipe={"id": target_id, "seq": target.seq, "sha256": target_sha}, + union={"build_id": info.union_id, "db_sha256": info.union_db_sha256, + "table_hashes": dict((info.union_report or report_json or {}).get("table_hashes") or {}), + "union_ver": recipe_types.UNION_VER}, + parts=parts, report=info.union_report or report_json or {}, label_sets=label_sets, gaps=gaps, + replaced=None, removed=removed, notes=notes, signed_by=signed_by) + cache["snapshot_manifest"] = await artifact_store.put_json( + doc, kind="snapshot_manifest", meta={"source_id": source_id, "snapshot_id": info.snapshot_id}) + return cache + + async def before_commit(_new_import: str | None, _snap_id: str) -> None: + if revoked is not None: + row = await session.get(TableImport, imp_id) + if row is not None: + row.revoked = revoked + + try: + pub = await table_versions.publish_snapshot( + session, source, imports=keep, mode="accumulate", recipe_id=target_id, recipe_sha256=target_sha, + expected_current=expected_current, period_build=None, new_import=None, union=union_build, + schema_cache_for=schema_cache_for, before_commit=before_commit, activation=activation) + except table_versions.PublishError as e: + raise _publish_refused(e) from e + finally: + _drop(out) + return pub.snapshot_id, bool(pub.snapshot_reused or pub.snapshot_revived) + + +def _drop(path: Path | None) -> None: + if path is None: + return + for suffix in ("", "-journal", "-wal", "-shm"): + Path(f"{path}{suffix}").unlink(missing_ok=True) + + +def _jsonable(obj: Any) -> Any: + from app.data.recipe_imports import _jsonable as conv + + return conv(obj) + + +async def table_versions_slot(fn: Any, *args: Any, **kw: Any) -> Any: + """CPU 密集的一步（物化、查空值数）：占一个解析名额、放进线程池（与试运行同一套名额）。""" + async with table_versions.parse_slot(): + return await asyncio.to_thread(fn, *args, **kw) + + +def _check_target(cur: _Current, imp: TableImport) -> None: + if imp.id not in cur.imports: + raise ImportRefused(409, "not_in_current", "这一期不在当前版本里，请刷新版本列表后重试") + + +def _check_base(cur: _Current, expected_current: str | None) -> None: + """锁外读到的当前版本必须就是确认框依据的那个（expected_current），否则 409 base_changed。 + + 为什么锁内的比对不够：留下哪几期（keep）、替换模式的回滚目标，都是按这里读到的当前版本算的。锁内只比 + 「指针此刻是不是 expected_current」，挡不住 ABA：用户按 E=[7,8,9] 发「移除 8 月」，处理器读到的却已是别人移除 + 9 月之后的 C=[7,8]，算出 keep=[7]；处理过程中又有人把 E 启用回来，锁内 E==E 通过，接口回 200，当前版本却只剩 + [7]，9 月悄悄丢了，确认框写的是 [7,9]（评审意见）。快照 id 按内容算、含 imports，这里相等就说明 keep 的来源与 + 确认框一致；锁内再比一次，管的是这之后的变化。""" + if expected_current != cur.snapshot.id: + raise ImportRefused(409, "base_changed", table_versions._BASE_CHANGED) + + +async def remove_period(session: AsyncSession, source: DataSource, import_id: str, *, expected_current: str | None, + reason: str, signed_by: str | None) -> dict[str, Any]: + """POST …/imports/{id}/remove（7.4）。""" + imp = await session.get(TableImport, import_id) + if imp is None or imp.source_id != source.id: + raise ImportRefused(404, "import_not_found", "导入记录不存在") + # 只读进程不写版本存储（第 8 节遗留项 5）：在组装各期、物化并集之前就停下。publish_snapshot 进锁后也会拒，但 + # 那时并集的临时文件已经写进 snapshots/，各期哈希、说明也都白算了（评审意见） + table_versions.require_store_writable() + cur = await _current(session, source) + _check_base(cur, expected_current) + _check_target(cur, imp) + if cur.mode != "accumulate": + raise ImportRefused(409, "not_accumulate", "当前版本不是按期累积的，不能移除其中一期；可以在「历史版本」中启用更早的版本") + if len(cur.imports) < 2: + raise ImportRefused(409, "last_period", _LAST_PERIOD) + sid, reused = await _rebuild_without(session, cur, imp, expected_current=expected_current, kind="remove_period", + reason=reason, signed_by=signed_by, revoke=False) + return {"snapshot_id": sid, "removed_import_id": import_id, "reused": reused} + + +async def revoke_acceptance(session: AsyncSession, source: DataSource, import_id: str, *, + expected_current: str | None, expected_target: str | None, + ack_mask_lost: list[str] | None, reason: str, signed_by: str | None) -> dict[str, Any]: + """POST …/imports/{id}/revoke-acceptance（7.5）。累积模式（至少两期）= 移除这一期；否则回滚到第 5 节定义的目标。""" + imp = await session.get(TableImport, import_id) + if imp is None or imp.source_id != source.id: + raise ImportRefused(404, "import_not_found", "导入记录不存在") + if not (imp.overrides or imp.waivers): + raise ImportRefused(409, "no_acceptance", "这一期没有写理由接受的核对，不需要作废") + table_versions.require_store_writable() + cur = await _current(session, source) + # 替换模式的回滚目标也是在锁外按这里读到的当前版本算的：同样先确认它就是确认框依据的那个（ABA，见 _check_base） + _check_base(cur, expected_current) + _check_target(cur, imp) + if _accumulating(cur): + sid, _reused = await _rebuild_without(session, cur, imp, expected_current=expected_current, + kind="revoke_acceptance", reason=reason, signed_by=signed_by, + revoke=True) + return {"snapshot_id": sid, "action": "remove_period"} + target = await rollback_target(session, source, imp.id) + if target is None: + raise ImportRefused(409, "no_rollback_target", _NO_ROLLBACK) + if target.id != expected_target: + raise ImportRefused(409, "revoke_target_changed", _TARGET_CHANGED) + target_id = target.id + imp.revoked = _revoked_record(reason, signed_by) + try: + await table_versions.activate_snapshot( + session, source, target_id, expected_current=expected_current, kind="revoke_acceptance", reason=reason, + signed_by=signed_by, import_id=import_id, ack_mask_lost=ack_mask_lost) + except table_versions.PublishError as e: + raise _publish_refused(e) from e + return {"snapshot_id": target_id, "action": "rollback"} + + +# ========================================================================== +# 导入记录（7.6） +# ========================================================================== + + +def _acceptance_rows(imp: TableImport) -> list[dict[str, Any]]: + out = [] + for kind, recs in (("override", imp.overrides), ("waiver", imp.waivers)): + for rec in recs or []: + if isinstance(rec, dict): + out.append({"check_id": rec.get("check_id"), "kind": kind, "reason": rec.get("reason"), + "signed_by": rec.get("signed_by"), "at": rec.get("at")}) + return out + + +async def import_records(session: AsyncSession, source: DataSource, imps: list[TableImport]) -> list[dict[str, Any]]: + """GET /{source_id}/imports 的行（期 1 的字段照旧，期 3 补字段，7.6）。几样一次查齐，不逐条查。""" + current_ids: set[str] = set() + cur: _Current | None = None + if source.current_snapshot_id: + snap = await session.get(SourceSnapshot, source.current_snapshot_id) + current_ids = {str(x) for x in (snap.imports or [])} if snap is not None else set() + if snap is not None and (source.origin or "manual") == "upload": + try: + cur = await _current(session, source) + except ImportRefused: + cur = None + builds = await _builds_of(session, {i.build_id for i in imps if i.build_id}) + seqs = await _recipe_seqs(session, {i.recipe_id or "" for i in imps}) + shas = {i.raw_sha256 for i in imps if i.raw_sha256} + shared: dict[str, list[tuple[str, str | None]]] = {} + opened: dict[str, int] = {} + if shas: + rows = (await session.execute( + select(TableImport.id, TableImport.raw_sha256, TableImport.source_id, DataSource.name) + .join(DataSource, DataSource.id == TableImport.source_id, isouter=True) + .where(TableImport.raw_sha256.in_(list(shas)), TableImport.raw_state == "kept"))).all() + for iid, sha, _src, name in rows: + shared.setdefault(sha, []).append((iid, name if name is not None else _DELETED_SOURCE)) + opened = dict((await session.execute( + select(ImportStaging.raw_sha256, func.count()) + .where(ImportStaging.raw_sha256.in_(list(shas)), ImportStaging.status.in_(table_versions.STAGING_OPEN)) + .group_by(ImportStaging.raw_sha256))).all()) + out = [] + for imp in imps: + in_current = imp.id in current_ids + refs = [name for iid, name in shared.get(imp.raw_sha256 or "", []) if iid != imp.id] + by_name: dict[str | None, int] = {} + for name in refs: + by_name[name] = by_name.get(name, 0) + 1 + plan = None + if in_current and (imp.overrides or imp.waivers) and not imp.revoked and cur is not None: + plan = await revoke_plan(session, source, imp, cur) + b = builds.get(imp.build_id) + out.append({ + "id": imp.id, "seq": imp.seq, "build_id": imp.build_id, + "file_name": imp.file_name, "file_size": imp.file_size, "raw_sha256": imp.raw_sha256, + "raw_state": imp.raw_state, "status": imp.status, "purged": imp.purged, + "current": in_current, + "created_at": _iso(imp.created_at), "activated_at": _iso(imp.activated_at), + # 按配方导入才有（简单导入为空）：配方、统计期、署名，接受的条数 + "recipe_id": imp.recipe_id, "period_start": imp.period_start, "period_end": imp.period_end, + "signed_by": imp.signed_by, "overrides": len(imp.overrides or []), "waivers": len(imp.waivers or []), + # 期 3（7.6） + "acceptances": _acceptance_rows(imp), "revoked": imp.revoked, "manifest_artifact": imp.manifest_artifact, + "recipe_seq": seqs.get(imp.recipe_id) if imp.recipe_id else None, "in_current": in_current, + "rows": _rows_of(b.report if b is not None else None), "revoke_plan": plan, + "raw_shared_with": [{"source_name": name, "count": n} for name, n in by_name.items()], + "raw_open_stagings": int(opened.get(imp.raw_sha256, 0)) if imp.raw_sha256 else 0, + }) + return out + + +__all__ = [ + "Part", "PartProblem", "REASON_MAX", "activate", "current_tables", "gaps_of", "import_manifest", + "import_records", "list_snapshots", "load_manifest", "load_parts", "manifest_doc", "note_part", "remove_period", + "retired_names", "retired_status", "revoke_acceptance", "revoke_plan", "rollback_target", "single_notes", + "snapshot_manifest", "snapshot_manifest_out", "tables_in_parts", "union_table_report", +] diff --git a/backend/app/data/table_versions.py b/backend/app/data/table_versions.py index 4fcadba7..c2e2e575 100644 --- a/backend/app/data/table_versions.py +++ b/backend/app/data/table_versions.py @@ -24,6 +24,16 @@ 配方导入的提交事务也调它（带期望的库哈希、锁内的提交前回调）； - 暂存区（import_stagings）：没发布的尝试都在这里，提交成功前不碰数据源。它引用的原件、试运行库 归这里的回收与清理管：未结束的暂存区保护原件，结束即瘦身，过期、放弃、清除原件都会结束它。 + +**按期累积与版本页（期 3）。** 三层仍不变，只是一个快照可以有多期： +- 两期及以上（或一期但那一期的配方不是目标配方）时，快照库是物化出的并集（recipe_accumulate.materialize_union）， + 也登记成一个构建（engine_ver=UNION_VER），文件在 tables/<源>/snapshots/<并集 id>.db； +- publish_snapshot 发布多期的快照（提交、移除某一期、作废接受都走它），activate_snapshot 启用一个还保留着的 + 旧快照（回滚）。两者都在锁内核对「当前版本没被别人换掉」（expected_current），快照 id 已存在时复用或复活； +- 回收在当前快照、被运行引用的快照之外，再给每个源保留最近启用过的 SNAPSHOT_KEEP 个快照，回滚才有东西可回； +- 发布撞上被改过的同版本文件时，能用这次的临时库恢复就恢复，原文件挪进隔离区（uploads/quarantine）； +- 版本存储只支持单进程：启动时取数据目录上的文件锁（acquire_store_guard），拿不到就不做任何清理， + 之后的写入一律拒绝（require_store_writable）。 """ from __future__ import annotations @@ -40,9 +50,9 @@ import sqlite3 import stat import weakref from dataclasses import dataclass, field -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any, Awaitable, Callable, Iterator, Sequence +from typing import Any, Awaitable, Callable, Iterator, NamedTuple, Sequence from urllib.parse import unquote, urlsplit from sqlalchemy import and_, delete, func, or_, select, update @@ -54,11 +64,12 @@ from app.core.config import settings from app.core.errors import explain, first_line from app.data import introspect as introspect_mod from app.data import raw_store, recipe_parsers, recipe_types, tabular -from app.data.engine import engines +from app.data.engine import SnapshotTampered as _EngineSnapshotTampered +from app.data.engine import engines, masked_columns from app.db import base as db_base from app.db.base import new_id, utcnow from app.db.models import ( - DataSource, ImportStaging, Run, SourceSnapshot, TableBuild, TableImport, TableRecipe, + DataSource, ImportStaging, Run, SnapshotActivation, SourceSnapshot, TableBuild, TableImport, TableRecipe, ) logger = logging.getLogger(__name__) @@ -80,11 +91,34 @@ _PROBE = "~probe~" #: 上传库只有 main：options 里的这一项对上传源没有意义，留着只会让探查去找不存在的 schema _SCHEMA_OPTION = "schema" +#: 回收在「运行引用的、各源当前的」之外，再按 coalesce(activated_at, created_at) 倒序给每个源保留几个快照。 +#: 当前快照和被运行引用的快照**不占名额**：用户定案 P3-3 的原话是「当前版本、被运行引用的版本之外，再保留最近 +#: 几个」，版本页顶部常驻的保留规则（VERSIONS_TEXT.retentionRule）也这么说，名额要是把当前版本算进去，界面说的 +#: 「保留 3 个」实际只有 2 个可回滚。期 1 只保留前两类（等于这里取 0）：发布 9 月之后 8 月的快照马上被回收，回滚 +#: 在实际中几乎做不了。每个物化过的累积版本是一份全量副本，保留越多越占空间（P3-SPEC 1.4 P3-3 定为 3） +SNAPSHOT_KEEP = 3 +#: 隔离文件（发布时撞上的、内容和登记不一致的同版本文件）保留几天。隔离文件含整张表的数据，不能无限期留着（D13） +QUARANTINE_DAYS = 7 +#: 并集库所在的目录名：tables/<源 id>/snapshots/<并集 id>.db +UNION_DIR = "snapshots" +#: 隔离区：uploads/quarantine/<源 id>/<构建 id>-.db。不在 tables/ 下：孤儿清理和 _sweep_tmp 都不碰它 +QUARANTINE_DIR = "quarantine" + class SnapshotMissing(ValueError): """要的快照不存在、不属于这个源、已回收或文件丢了。message 直接给人看。""" +class SnapshotManifestMismatch(SnapshotMissing, _EngineSnapshotTampered): + """快照的哈希登记与它的快照清单（内容寻址）对不上：库列被改过，拒绝查询（P3-SPEC 2.9）。 + + 同时是 SnapshotMissing 和引擎的 SnapshotTampered：解析源的调用方按前者处理（不退回当前版本）， + 查询路径按后者处理，两边都不会把它漏成 500。 + """ + + code = "snapshot_tampered" + + class PublishError(RuntimeError): """解析成功了，但没能把库文件安全地发布出去。指针没动，旧版照常可查。 @@ -97,6 +131,21 @@ class PublishError(RuntimeError): self.code = code +class StoreUnavailable(PublishError): + """本进程没拿到版本存储的守卫（同一个数据目录被另一个进程占着），不写入版本存储。接口层回 503。""" + + def __init__(self, message: str = "", *, code: str | None = "store_unavailable") -> None: + super().__init__(message, code=code or "store_unavailable") + + +class ActivationError(PublishError): + """启用旧快照失败。code 取 P3-SPEC 7.2 的值（snapshot_not_found、already_current、base_changed、 + snapshot_unavailable、snapshot_tampered、contains_revoked、mask_lost、not_upload_source），message 给人看。""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(message, code=code) + + class ParseFailed(tabular.UnsupportedTable): """解析时出了意料之外的错（不是 UnsupportedTable 那种「认得出、读不了」）。 @@ -116,6 +165,8 @@ class ImportResult: seq: int = 0 #: 实际发布（或复用）的库文件哈希 db_sha256: str = "" + #: 服务端已有的同版本文件被改过，这次用临时库恢复了它（原文件挪进隔离区，第 8 节遗留项 1） + build_restored: bool = False @dataclass @@ -135,6 +186,8 @@ class PublishInfo: build_reused: bool #: 实际冻结的回执：复用构建时是已有构建登记的回执，否则是这次传进来的 report: dict[str, Any] = field(default_factory=dict) + #: 见 ImportResult.build_restored + build_restored: bool = False @dataclass @@ -207,29 +260,80 @@ def build_id(source_id: str, raw_sha: str, options: dict[str, Any]) -> str: }) -def snapshot_id(source_id: str, import_ids: list[str], engine_ver: str = ENGINE_VER) -> str: - return _sha_json({"source_id": source_id, "imports": list(import_ids), "engine_ver": engine_ver}) +def snapshot_id(source_id: str, import_ids: list[str], engine_ver: str = ENGINE_VER, *, + mode: str | None = None, recipe_sha: str | None = None) -> str: + """快照 id。import_ids 按统计期排序。 + + mode == "accumulate" 时把 {"mode", "recipe_sha256"}（目标配方）加进哈希：累积快照的库按目标配方的表结构物化 + （列序、退役列），同一组导入配上不同的目标配方是内容不同的两个版本。只看导入记录的话，「移除该期」「撤销接受」 + 会复用到配方或模式不同的旧快照，悄悄把配方、模式退回去（P3-SPEC 2.1）。其余情况（替换模式、简单导入）哈希内容 + 与期 2 逐字相同，已有的快照 id 不失效：替换模式下快照的配方就是那一期导入的配方，由导入记录决定。 + """ + payload: dict[str, Any] = {"source_id": source_id, "imports": list(import_ids), "engine_ver": engine_ver} + if mode == "accumulate": + if not recipe_sha: + raise ValueError("按期累积的快照 id 必须带目标配方的哈希") + payload.update(mode="accumulate", recipe_sha256=recipe_sha) + return _sha_json(payload) -def recipe_build_options(recipe_sha: str, inputs: dict[str, Any]) -> dict[str, Any]: - """按配方导入的构建登记的选项（TableBuild.options）。options_sha256 = sha_json(它)。""" - return {"kind": "recipe", "recipe_sha256": recipe_sha, - "parser_ver": recipe_parsers.PARSER_VER, "inputs": inputs} +#: 执行器语义标记的键：列表块 blank_rows=skip 时，空行之后本块列范围内只有首列一格文字的行判为表的下边界 +#: （期 3 WP-8 修补 3，recipe_engine._ListRun._note_after_blank），不再读成一条数据 +LIST_NOTE_AFTER_BLANK = "list_note_after_blank" -def recipe_build_id(source_id: str, raw_sha: str, recipe_sha: str, inputs: dict[str, Any]) -> str: - """按配方导入的构建 id：源、原件、配方、解析器版本、执行器版本、人工录入。 +def recipe_engine_semantics(recipe: recipe_types.Recipe | None) -> dict[str, int]: + """这份配方的构建要带的执行器语义标记（进 recipe_build_id 与 recipe_build_options），没有就 {}。 + + 为什么不升 RECIPE_ENGINE_VER：它进每一个配方构建的 id，一升，交叉表配方（期 2 起已导入的报表都是这种）的构建 + 全部换 id、期 2 的构建不再复用，p2_baseline 记的 id 也随之失效（P3-SPEC 2.1：没用新字段的配方必须写出逐字节相同 + 的库）。而修补 3 只改了「列表 + 跳过空行」这一类配方的结果：同一原件、同一配方，修补前的执行器多读一条首列是 + 「注：……」的记录。构建 id 不变的话，数据目录里修补前建的同 id 构建（哈希等于登记值）会被 _publish_file 当成 + 复用，用户核对的试运行内容和发布出去的对不上。所以只给受影响的配方加标记，只有这类配方的构建换 id。 + + 以后再有只影响一类配方的执行器改动，照此另加一个键（值从 1 起），已有的键和值不要改：改了会让已有构建换 id。""" + if recipe is None: + return {} + marks: dict[str, int] = {} + if any(isinstance(b, recipe_types.ListBlock) and b.rows.blank_rows == "skip" + for sheet in recipe.sheets for b in sheet.blocks): + marks[LIST_NOTE_AFTER_BLANK] = 1 + return marks + + +def recipe_build_options(recipe_sha: str, inputs: dict[str, Any], *, + semantics: dict[str, int] | None = None) -> dict[str, Any]: + """按配方导入的构建登记的选项（TableBuild.options）。options_sha256 = sha_json(它)。 + + semantics：recipe_engine_semantics(配方)，有标记时记成 "engine_semantics"，与 recipe_build_id 的输入一一对应； + 没有标记时不写这个键，期 2 起已有构建的选项（和 options_sha256）不变。""" + options: dict[str, Any] = {"kind": "recipe", "recipe_sha256": recipe_sha, + "parser_ver": recipe_parsers.PARSER_VER, "inputs": inputs} + if semantics: + options["engine_semantics"] = dict(semantics) + return options + + +def recipe_build_id(source_id: str, raw_sha: str, recipe_sha: str, inputs: dict[str, Any], *, + semantics: dict[str, int] | None = None) -> str: + """按配方导入的构建 id：源、原件、配方、解析器版本、执行器版本、人工录入，以及执行器语义标记。 inputs 只放人工录入的东西（{"context": {"统计期": {"start", "end"}}}，没有就 {}）；署名和文件名 不进。同一份原件换个名字再传会复用构建，所以凡是取决于文件名的结果（C2）都不能放进构建回执， 得跟着导入记录走（table_imports.checks、导入清单）。解析器或执行器升级会换掉 id：按新规则建出来的 库不是同一个构建，不能复用旧文件。 + + semantics：recipe_engine_semantics(配方)。执行器只对某一类配方改了结果时，用它让这类配方的构建换 id，其余 + 配方的 id 照旧（见 recipe_engine_semantics）。为空时不进哈希，id 与期 2 逐字相同。 """ - return sha_json({ + payload: dict[str, Any] = { "source_id": source_id, "raw_sha256": raw_sha, "recipe_sha256": recipe_sha, "parser_ver": recipe_parsers.PARSER_VER, "engine_ver": recipe_types.RECIPE_ENGINE_VER, "inputs_sha256": sha_json(inputs), - }) + } + if semantics: + payload["engine_semantics"] = dict(semantics) + return sha_json(payload) def legacy_build_id(source_id: str, file_sha: str) -> str: @@ -247,6 +351,19 @@ def build_path(source_id: str, bid: str) -> Path: return tables_root() / source_id / "builds" / f"{bid}.db" +def union_path(source_id: str, union_id: str) -> Path: + """并集库的最终路径。并集也登记成构建（id = 并集 id），内容寻址：同样的各期、同样的目标配方只有一份文件。""" + if not _SOURCE_ID.match(source_id or ""): + raise ValueError("数据源 id 的格式不对") + if not re.fullmatch(r"[0-9a-f]{64}", union_id or ""): + raise ValueError("并集 id 的格式不对") + return tables_root() / source_id / UNION_DIR / f"{union_id}.db" + + +def quarantine_root() -> Path: + return settings.uploads_dir / QUARANTINE_DIR + + #: 试运行库所在的目录名：tables/<源 id>/trials/<暂存区 id>-.db TRIALS_DIR = "trials" @@ -264,6 +381,20 @@ def trial_db_path(source_id: str, staging_id: str, trial_key: str) -> Path: return tables_root() / source_id / TRIALS_DIR / f"{staging_id}-{trial_key[:16]}.db" +#: 附属试运行库（并集）文件名比主文件多的后缀：<暂存区 id>-u.db +_UNION_TRIAL_MARK = "u" + + +def union_trial_path(trial_path: Path | str) -> Path: + """试运行的并集文件：主试运行库去掉 .db 加「u.db」，放在同一目录。 + + 文件名里不加「-」：_trial_owner 按最后一个「-」切出暂存区 id，加了就切错。它跟着主文件走： + remove_trial_file 连带删除，启动清理按主文件判断是否有效。 + """ + path = Path(trial_path) + return path.with_name(f"{path.name.removesuffix('.db')}{_UNION_TRIAL_MARK}.db") + + def _under(path: str | os.PathLike[str], root: Path) -> bool: try: return Path(os.path.realpath(path)).is_relative_to(os.path.realpath(root)) @@ -449,11 +580,49 @@ def upload_options(options: dict[str, Any] | None) -> dict[str, Any]: return {k: v for k, v in (options or {}).items() if k != _SCHEMA_OPTION} +#: 已经和快照清单交叉核对过的 (快照 id, 库列里的哈希, 清单 id)。按进程缓存：清单取回时要读文件、复验内容哈希， +#: 每次查询都做不划算。键里带着库列的值，库列之后再被改，键就对不上，会重新核对 +_MANIFEST_VERIFIED: set[tuple[str, str, str]] = set() + + +def verify_snapshot_manifest(source: Any, snapshot: SourceSnapshot) -> None: + """快照有快照清单时，拿清单里的 union.db_sha256 和库列 db_sha256 比，不一致抛 SnapshotManifestMismatch。 + + 查询时核对文件用的期望哈希取自 source_snapshots.db_sha256 这个库列。有人同时改了库文件和库列（改成与 + 文件一致的假值），文件哈希核对照样通过；快照清单是内容寻址的工件（artifact_store.load 会复验内容哈希）， + 改不了，所以再拿它交叉核对一次（P3-SPEC 2.9）。没有清单的快照（单期）不在此列。清单取不回来、内容哈希 + 不符、没有 union.db_sha256、快照 id 对不上，一律按不一致处理：证据链断了就不能当作没事。 + """ + mid = str(getattr(snapshot, "manifest_artifact", None) or "").strip() + if not mid: + return + registered = str(snapshot.db_sha256 or "").strip() + key = (snapshot.id, registered, mid) + if key in _MANIFEST_VERIFIED: + return + from app.core import artifact_store + + try: + manifest = artifact_store.load(mid) + except (OSError, ValueError): + manifest = None + union = manifest.get("union") if isinstance(manifest, dict) else None + claimed = union.get("db_sha256") if isinstance(union, dict) else None + claimed_id = manifest.get("snapshot_id") if isinstance(manifest, dict) else None + if not claimed or claimed != registered or claimed_id not in (None, snapshot.id): + logger.warning("快照的哈希登记与快照清单不一致，已拒绝：source=%s snapshot=%s manifest=%s", + getattr(source, "id", None), snapshot.id, mid) + raise SnapshotManifestMismatch( + f"「{source.name}」的数据版本 {snapshot.id[:12]} 的哈希登记与版本清单不一致，可能被修改过，已拒绝查询") + _MANIFEST_VERIFIED.add(key) + + def bind_snapshot(source: Any, snapshot: SourceSnapshot) -> SourceView: """数据源 + 快照 → SourceView：只读、库路径和表结构都取快照里冻结的那份。 快照没登记文件哈希（记录损坏、被手工改过）时抛 SnapshotMissing：没有哈希就无从核对文件 是否被改过，不能退回「不核对照样打开」。现有的写入路径（发布、迁移）都会写哈希。 + 快照有快照清单（物化过的累积快照）时，本进程第一次绑定它时再和清单交叉核对一次（verify_snapshot_manifest）。 """ if not str(snapshot.db_sha256 or "").strip(): logger.warning("快照没有登记文件哈希，已拒绝：source=%s snapshot=%s", source.id, snapshot.id) @@ -461,6 +630,7 @@ def bind_snapshot(source: Any, snapshot: SourceSnapshot) -> SourceView: f"「{source.name}」的数据版本 {snapshot.id[:12]} 缺少数据文件的哈希登记，" "无法核对数据文件是否被修改过，已拒绝查询，请重新上传" ) + verify_snapshot_manifest(source, snapshot) return SourceView( id=source.id, name=source.name, kind="sqlite", database=snapshot.db_path, options=upload_options(source.options), readonly=True, @@ -525,6 +695,91 @@ def store_lock() -> asyncio.Lock: _store_lock = store_lock +# -------------------------------------------------------------------------- +# 守卫：版本存储只支持单进程（第 8 节遗留项 5） +# -------------------------------------------------------------------------- + +#: 数据目录上的守卫文件：uploads/.store.lock +_GUARD_FILE = ".store.lock" +_STORE_READONLY = ("数据目录正被另一个 AgentLab 进程使用：上传表格的版本只支持单进程部署，" + "本进程不写入版本存储，请检查部署方式") + + +@dataclass +class _Guard: + #: 上次尝试加锁的守卫文件。数据目录换了（测试里每个用例一个）就对新文件重新加锁 + path: Path | None = None + #: 持有的文件描述符：进程活着就一直持有，flock 跟着它 + fd: int | None = None + writable: bool = True + + +_guard = _Guard() + + +def _guard_path() -> Path: + return settings.uploads_dir / _GUARD_FILE + + +def _release_guard() -> None: + if _guard.fd is not None: + try: + os.close(_guard.fd) + except OSError: + pass + _guard.path, _guard.fd, _guard.writable = None, None, True + + +def acquire_store_guard() -> bool: + """对 uploads/.store.lock 做 flock(LOCK_EX | LOCK_NB)，拿到就在进程生命周期内一直持有。返回本进程能否写。 + + **为什么要有。** 发布、回收、启动清理之间的互斥靠进程内的 asyncio 锁（store_lock），只在单进程里成立。 + 部署本来就是单进程（uvicorn 不带 --workers）；万一同一个数据目录起了两个进程，第二个进程的启动清理会删掉 + 第一个进程正在写的临时文件、已经 link 但记录还没提交的构建库，第一个进程随后提交的快照就指向一个已删的文件。 + 所以 startup 第一步就取它，拿不到就什么都不清理，之后的写入一律拒绝（require_store_writable）。 + + 同一进程里重复调用是幂等的；数据目录换了就放掉旧的、对新文件重新加锁。上次没拿到的，再调一次会重试 + （占着它的进程退出后就能拿到）。没有 fcntl 的平台照常可写，只记一条警告。 + """ + path = _guard_path() + if _guard.path == path and _guard.fd is not None: + return True + _release_guard() + try: + import fcntl + except ImportError: + logger.warning("这个平台没有 fcntl，无法确认数据目录只被一个进程使用：上传表格的版本只支持单进程部署") + _guard.path, _guard.writable = path, True + return True + try: + path.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(path, os.O_RDWR | os.O_CREAT, 0o600) + except OSError: + logger.exception("无法打开版本存储的守卫文件 %s，本进程不写入版本存储", path) + _guard.path, _guard.writable = path, False + return False + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(fd) + _guard.path, _guard.writable = path, False + logger.error(_STORE_READONLY) + return False + _guard.path, _guard.fd, _guard.writable = path, fd, True + return True + + +def store_writable() -> bool: + """本进程能不能写版本存储。没对当前数据目录取过守卫（没调过 startup）的按能写处理：守卫只在启动时取。""" + return _guard.writable or _guard.path != _guard_path() + + +def require_store_writable() -> None: + """写版本存储的入口一律先调它：拿不到守卫的进程抛 StoreUnavailable（接口层回 503 store_unavailable）。""" + if not store_writable(): + raise StoreUnavailable(_STORE_READONLY) + + #: 同时解析的上传最多几个。解析在线程里做，内存和上传数成正比；不设上限的话，几个大文件 #: 同时上传，内存会叠加起来。超出的排队等前面的解析完 MAX_PARALLEL_PARSES = 2 @@ -640,14 +895,64 @@ def _fsync_file(path: Path) -> None: os.close(fd) -def _publish_file(tmp: Path, final: Path, expected_sha: str | None, - trial_sha: str | None = None) -> tuple[str, bool]: - """把临时库发布到最终路径，返回 (库文件 sha256, 是否复用了已有文件)。 +_BUILD_CONFLICT = ("导入失败：服务端已有的同版本数据文件与登记的不一致，且无法用本次上传恢复，当前版本未受影响。" + "请联系管理员检查数据目录") - 先落盘再 link：link 不会覆盖已有文件，已发布的版本因此不可能被半个新文件顶掉。 - 目标已存在时（同一个构建传过），只有它的哈希等于登记的 db_sha256 才复用——0 字节的 - 残骸、被改过的文件都不能冒充。没有登记过（上次发布到一半进程没了）时，和这次刚建的 - 临时库逐字节一致也可以接手。临时文件无论成败都删掉。 + +class Placed(NamedTuple): + """_publish_file 的结果。按前两项解包和期 1 一样：(库文件 sha256, 是否复用了已有文件)。""" + + sha: str + #: 最终路径上的文件算作已有构建的那一份（含「恢复」）：发布失败时不删它 + reused: bool + #: 已有文件被改过，这次用临时库恢复了它，原文件挪进了隔离区（第 8 节遗留项 1） + restored: bool = False + + +def _quarantine(final: Path) -> Path: + """把最终路径上的文件挪进隔离区 uploads/quarantine/<源 id>/<构建 id>-.db（0444）。 + + 不直接删：它可能是被人改过的证据，管理员要能看到改成了什么。隔离文件含整张表的数据，由回收在 + QUARANTINE_DAYS 天后删掉，删除数据源、清除原件时也连带删掉（D13）。 + + 挪进来之后把修改时间刷成此刻：回收按修改时间判断是否过期，而 os.replace、chmod 都不改修改时间。不刷的话， + 隔离文件带着原文件的修改时间（构建文件多半是几周前发布的，磁盘位翻转这类损坏也不改修改时间），发布之后 + 紧接着的那次回收就把刚隔离的证据删了，7 天的保留形同虚设。 + """ + source_id, bid = final.parent.parent.name, final.stem + if not _SOURCE_ID.match(source_id): + raise PublishError(f"导入失败：数据文件的路径不对（{final}），当前版本未受影响") + folder = quarantine_root() / source_id + folder.mkdir(parents=True, exist_ok=True) + stamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + target = folder / f"{bid}-{stamp}.db" + os.replace(final, target) + try: + os.utime(target, None) + except OSError as e: + logger.warning("没能刷新隔离文件的修改时间，它可能早于 %d 天被回收：%s（%s）", QUARANTINE_DAYS, target, e) + try: + os.chmod(target, 0o444) + except OSError: + pass + raw_store.fsync_dir(folder) + raw_store.fsync_dir(final.parent) + return target + + +def _publish_file(tmp: Path, final: Path, expected_sha: str | None, + trial_sha: str | None = None) -> Placed: + """把临时库发布到最终路径，返回 Placed(库文件 sha256, 是否复用了已有文件, 是否恢复了被改过的文件)。 + + 先落盘再 link：link 不会覆盖已有文件，已发布的版本因此不可能被半个新文件顶掉。临时文件无论成败都删掉。 + 目标已存在时（同一个构建传过），设 E = 已有文件的哈希，R = 登记的 db_sha256（没有记录、记录为空时为空）， + T = 本次临时库的哈希（第 8 节遗留项 1）： + - E 等于 R（有记录），或者没有记录而 E 等于 T：复用。后者是「上次发布到一半进程没了」的崩溃恢复； + - 有记录、E 不等于 R、而 T 等于 R：同一构建按同样的输入确定地重建出了登记的那份（执行器、物化都是确定的）， + 把已有文件挪进隔离区、link 临时库，结果算作复用并记 restored，日志写 WARNING。0 字节的残骸、被改过的 + 文件都不能冒充登记的版本，但也不该让同一个文件永远传不上去（期 1 这里一律 500）； + - 没有记录、E 不等于 T：没人登记的孤儿文件（本来就会被启动清理删掉），同样挪进隔离区再 link，restored=False； + - 有记录、E、T 都不等于 R：PublishError(code="build_conflict")，什么都不动。 trial_sha：配方导入试运行时算的库哈希。给了就在 link 之前比对，对不上说明试运行之后有人改过 这个文件，不能把改过的数据当成用户核对过的那份发布出去。目标已存在时也先比它：被改过的试运行库 @@ -662,21 +967,36 @@ def _publish_file(tmp: Path, final: Path, expected_sha: str | None, _fsync_file(tmp) raw_store.fsync_dir(tmp.parent) final.parent.mkdir(parents=True, exist_ok=True) + registered = str(expected_sha or "").strip() or None try: os.link(tmp, final) + linked = True except FileExistsError: - existing = raw_store.sha256_file(final) - if existing == (expected_sha or sha): - return existing, True - raise PublishError( - "导入失败：服务端已有的同版本数据文件与登记的不一致，可能被修改过。" - "当前版本未受影响，请联系管理员检查数据目录" - ) from None + linked = False except OSError as e: raise PublishError(f"导入失败：无法写入数据文件（{first_line(e)}），当前版本未受影响") from e + if not linked: + existing = raw_store.sha256_file(final) + if existing == (registered or sha): + return Placed(existing, True) + if registered is not None and sha != registered: + logger.error("同版本数据文件与登记的哈希不一致，本次上传也恢复不了：%s", final) + raise PublishError(_BUILD_CONFLICT, code="build_conflict") + moved = _quarantine(final) + if registered is not None: + logger.warning("同版本数据文件被改过，已用本次上传恢复，原文件挪进隔离区：%s → %s", final, moved) + else: + logger.warning("目标位置有一个没有登记的文件，已挪进隔离区后发布：%s → %s", final, moved) + try: + os.link(tmp, final) + except OSError as e: + raise PublishError(f"导入失败：无法写入数据文件（{first_line(e)}），当前版本未受影响") from e + os.chmod(final, 0o444) + raw_store.fsync_dir(final.parent) + return Placed(sha, registered is not None, registered is not None) os.chmod(final, 0o444) raw_store.fsync_dir(final.parent) - return sha, False + return Placed(sha, False) finally: _remove_db_files(tmp) @@ -736,7 +1056,26 @@ IMPORT_FIELDS = frozenset({ }) SchemaCacheFor = Callable[[SourceView, PublishInfo], Awaitable[dict[str, Any]]] -BeforeCommit = Callable[[str, str], Awaitable[None]] +#: 提交前回调 (导入 id 或 None, 快照 id)：publish_build 总有新导入；publish_snapshot 的移除、作废没有新导入，传 None +BeforeCommit = Callable[[str | None, str], Awaitable[None]] + +#: 切换当前快照的几种原因（snapshot_activations.kind） +ACTIVATION_KINDS = ("commit", "activate", "remove_period", "revoke_acceptance") + + +def _activation_row(source_id: str, snapshot: str, previous: str | None, activation: dict[str, Any] | None, + *, default_import: str | None = None) -> SnapshotActivation: + """snapshot_activations 的一行。activation 的键对齐列：{"kind", "import_id", "reason", "signed_by"}。""" + act = dict(activation or {}) + kind = act.get("kind") or "commit" + if kind not in ACTIVATION_KINDS: + raise ValueError(f"切换当前快照的原因只能是 {ACTIVATION_KINDS}，不能是 {kind!r}") + reason = str(act.get("reason") or "").strip() or None + signed = str(act.get("signed_by") or "").strip()[:100] or None + return SnapshotActivation( + source_id=source_id, snapshot_id=snapshot, previous_snapshot_id=previous, kind=kind, + import_id=act.get("import_id") or default_import, reason=reason, signed_by=signed, created_at=utcnow(), + ) def _usable_tmp(tmp: Path) -> bool: @@ -755,6 +1094,8 @@ async def publish_build( expected_db_sha256: str | None = None, before_commit: BeforeCommit | None = None, import_id: str | None = None, + snapshot_fields: dict[str, Any] | None = None, + activation: dict[str, Any] | None = None, ) -> ImportResult: """把一个建好的临时库发布成数据源的新版本并启用。期 1 的上传和期 2 的配方提交都走这里。 @@ -770,7 +1111,13 @@ async def publish_build( 事务（session.get 取得到导入记录），**新源还没加入会话、数据源指针还没切**——在里面 select 读到的 current_snapshot_id 是发布前的值，查同名源也不会查到这次要建的新源。配方导入在这里重新核对 base_changed、name_taken，写配方表、清单索引，结束暂存区。 - 7. 加入数据源、切指针、提交。之后回收一次（失败只记日志）。 + 7. 写一行 snapshot_activations（activation 给了就按它，默认 kind=commit）；加入数据源、切指针、提交。 + 之后回收一次（失败只记日志）。 + + snapshot_fields（期 3）：{"mode", "recipe_id", "recipe_sha256"}。mode=accumulate 时快照 id 带上目标配方的哈希 + （snapshot_id 的说明），单期的累积快照（first、restart）也走这里；快照记录写 mode、recipe_id（没给时取 + import_fields 的 recipe_id）、activated_at。不加 expected_current：配方提交的 before_commit 已在锁内判 base_changed， + 简单上传照期 1 的语义，后提交的为准。 任何一步失败（包括 before_commit 抛的异常）都回滚，删掉这次 link 出来的库文件和这次新存的原件， 指针不动，异常原样抛出。**tmp_db 一律被消耗**：成功时 link 走，失败时也删，调用方不能指望它还在。 @@ -797,7 +1144,11 @@ async def publish_build( if raw is not None and hashlib.sha256(raw).hexdigest() != raw_sha: raise ValueError("raw_sha 与传入的原件内容不符") report = report_to_json(report) + fields = dict(snapshot_fields or {}) + snap_mode = fields.get("mode") or "replace" + snap_recipe = fields.get("recipe_id") or import_fields.get("recipe_id") async with store_lock(): + require_store_writable() if not _usable_tmp(tmp_db): raise PublishError(_TRIAL_MISSING, code="trial_missing") # 先读后写：读的时候别把调用方挂起的改动（新源、改说明）提前刷进库。 @@ -807,9 +1158,11 @@ async def publish_build( last_seq = (await session.execute( select(func.max(TableImport.seq)).where(TableImport.source_id == source_id) )).scalar() or 0 + previous = None if fresh else (await session.execute( + select(DataSource.current_snapshot_id).where(DataSource.id == source_id))).scalar() if raw is None and not raw_store.raw_exists(raw_sha): raise PublishError(_RAW_MISSING, code="raw_missing") - db_sha, reused_file = await asyncio.to_thread( + db_sha, reused_file, restored = await asyncio.to_thread( _publish_file, tmp_db, final, build.db_sha256 if build is not None else None, expected_db_sha256) linked = not reused_file @@ -841,7 +1194,8 @@ async def publish_build( imp_id = import_id or new_id() seq = last_seq + 1 - snap_id = snapshot_id(source_id, [imp_id], engine_ver) + snap_id = snapshot_id(source_id, [imp_id], engine_ver, mode=snap_mode, + recipe_sha=fields.get("recipe_sha256")) options_now = upload_options(source.options) view = SourceView( id=source_id, name=source.name, database=str(final), options=options_now, @@ -849,7 +1203,8 @@ async def publish_build( expected_sha256=build.db_sha256, ) info = PublishInfo(import_id=imp_id, snapshot_id=snap_id, seq=seq, build_id=build_id, - db_sha256=db_sha, build_reused=build_reused, report=report) + db_sha256=db_sha, build_reused=build_reused, report=report, + build_restored=restored) cache = await schema_cache_for(view, info) session.add(TableImport( @@ -861,6 +1216,7 @@ async def publish_build( session.add(SourceSnapshot( id=snap_id, source_id=source_id, imports=[imp_id], db_path=str(final), db_sha256=build.db_sha256, schema_cache=cache, created_at=now, + mode=snap_mode, recipe_id=snap_recipe, activated_at=now, )) await session.execute( update(TableImport) @@ -870,6 +1226,7 @@ async def publish_build( ) if before_commit is not None: await before_commit(imp_id, snap_id) + session.add(_activation_row(source_id, snap_id, previous, activation, default_import=imp_id)) session.add(source) source.kind, source.origin, source.readonly = "sqlite", "upload", True @@ -907,7 +1264,8 @@ async def publish_build( except Exception: # noqa: BLE001 - 回收失败不影响这次导入，下次发布或重启再收 logger.exception("上传表格发布后的回收失败") return ImportResult(import_id=imp_id, snapshot_id=snap_id, build_id=build_id, - build_reused=build_reused, report=report, seq=seq, db_sha256=db_sha) + build_reused=build_reused, report=report, seq=seq, db_sha256=db_sha, + build_restored=restored) async def publish_upload( @@ -925,6 +1283,8 @@ async def publish_upload( if not source.id: source.id = new_id() source_id = source.id + # 拿不到守卫的进程别先花时间解析：发布那一步反正会拒 + require_store_writable() raw_sha = hashlib.sha256(raw).hexdigest() options = parse_options(filename, header_row=header_row, mixed=mixed, raw_mode=raw_mode) bid = build_id(source_id, raw_sha, options) @@ -957,6 +1317,543 @@ async def publish_upload( ) +# -------------------------------------------------------------------------- +# 期 3：多期快照的发布、启用旧快照（回滚） +# -------------------------------------------------------------------------- + +_BASE_CHANGED = "数据源的当前版本已被更新，请刷新版本列表后重试" + + +@dataclass +class PeriodBuild: + """这次新导入的那一期：试运行库（tmp_db）、构建 id、登记的选项和回执、试运行时算的库哈希。""" + + tmp_db: Path + build_id: str + options: dict[str, Any] + report: dict[str, Any] + expected_sha256: str + + +@dataclass +class UnionBuild: + """物化好的并集：试运行的并集文件（tmp_db）、并集 id、登记的选项、回执（asdict(UnionReport) 的 JSON， + 登记进 TableBuild.report，版本页的行数从这里取）、物化时算的库哈希。""" + + tmp_db: Path + union_id: str + options: dict[str, Any] + report: dict[str, Any] + expected_sha256: str + + +@dataclass +class NewImport: + """新导入记录：id 由调用方先定（它已经在 imports 里，按统计期排好序）。fields 写进 IMPORT_FIELDS 那些列。""" + + import_id: str + raw_sha: str + file_name: str + file_size: int + fields: dict[str, Any] + + +@dataclass +class SnapshotInfo: + """publish_snapshot 交给 schema_cache_for 的发布结果：库文件已经落位，记录还没写。 + + WP-5 在 schema_cache_for 里写导入清单和快照清单。db_sha256 是这一期**实际发布（或复用）**的构建库哈希， + union_db_sha256 是实际发布（或复用）的并集库哈希，清单取这里的。 + """ + + snapshot_id: str + import_id: str | None + seq: int | None + build_id: str | None + db_sha256: str | None + build_reused: bool + build_restored: bool + union_id: str | None + union_db_sha256: str | None + #: 这一期实际冻结的构建回执（复用构建时是已有构建登记的）；没有新导入时为 {} + report: dict[str, Any] + #: 并集实际冻结的回执（复用并集时是已有并集登记的）；没有并集时为 None + union_report: dict[str, Any] | None + + +@dataclass +class SnapshotPublish: + snapshot_id: str + import_id: str | None + build_id: str | None + union_id: str | None + build_reused: bool + #: 本期构建或并集的已有文件被改过，这次用临时文件恢复了它、原文件挪进了隔离区（第 8 节遗留项 1）。 + #: 并集也算：接口层照这一个字段提示「服务端的同版本数据文件曾被改动」，不用分两处看 + build_restored: bool + union_reused: bool + seq: int | None + #: 快照 id 已存在、没回收：直接切过去，没有新建快照 + snapshot_reused: bool = False + #: 快照 id 已存在、已回收：复活后切过去 + snapshot_revived: bool = False + #: 恢复的是并集文件（build_restored 已含这种情况） + union_restored: bool = False + + +SnapshotSchemaFor = Callable[[SourceView, SnapshotInfo], Awaitable[dict[str, Any]]] + + +async def _current_pointer(session: AsyncSession, source_id: str) -> str | None: + """锁内用 select 重读数据源的当前快照：不信会话里缓存的对象，别人可能刚提交过。""" + return (await session.execute( + select(DataSource.current_snapshot_id).where(DataSource.id == source_id))).scalar() + + +async def _switch_statuses(session: AsyncSession, source_id: str, imports: list[str], + recipe_id: str | None, now: datetime) -> None: + """切当前快照时的状态：快照里的各期导入置 active，其余 active 的置 superseded；目标配方置 active， + 其余 active 的置 superseded（recipe_id 为 None = 简单导入，所有配方都置 superseded）。""" + await session.execute( + update(TableImport) + .where(TableImport.source_id == source_id, TableImport.id.in_(imports), TableImport.status != "active") + .values(status="active", activated_at=now) + ) + await session.execute( + update(TableImport) + .where(TableImport.source_id == source_id, TableImport.status == "active", TableImport.id.not_in(imports)) + .values(status="superseded") + ) + await session.execute( + update(TableRecipe) + .where(TableRecipe.source_id == source_id, TableRecipe.status == "active", + *([TableRecipe.id != recipe_id] if recipe_id else [])) + .values(status="superseded") + ) + if recipe_id: + await session.execute( + update(TableRecipe) + .where(TableRecipe.source_id == source_id, TableRecipe.id == recipe_id, TableRecipe.status != "active") + .values(status="active", activated_at=now) + ) + + +def _union_layout(report: dict[str, Any] | None) -> dict[str, list[list[Any]]] | None: + """并集回执（asdict(UnionReport) 的 JSON）里的表结构：表 → [[列, 类型], …]，列序照原样。没有就 None。""" + tables = (report or {}).get("tables") if isinstance(report, dict) else None + if not isinstance(tables, dict) or not tables: + return None + return {str(name): [[c.get("name"), c.get("type")] for c in cols if isinstance(c, dict)] + for name, cols in tables.items() if isinstance(cols, list)} + + +_UNION_LAYOUT_CONFLICT = ("导入失败：服务端已有的同一份累积数据文件与这次的表结构不一致，当前版本未受影响。" + "请联系管理员检查数据目录") + + +def _file_matches(path: str | None, expected: str | None) -> bool: + """文件在、哈希等于登记值（登记值为空一律不算）。""" + if not path or not str(expected or "").strip() or not Path(path).is_file(): + return False + try: + return raw_store.sha256_file(Path(path)) == str(expected).strip() + except OSError: + return False + + +async def publish_snapshot( + session: AsyncSession, source: DataSource, *, imports: list[str], mode: str, + recipe_id: str | None, recipe_sha256: str | None, expected_current: str | None, + period_build: PeriodBuild | None, new_import: NewImport | None, union: UnionBuild | None, + schema_cache_for: SnapshotSchemaFor, before_commit: BeforeCommit | None = None, + activation: dict[str, Any] | None = None, +) -> SnapshotPublish: + """把一组导入发布成数据源的新快照并启用（P3-SPEC 2.6 发布）。累积提交、移除某一期、作废接受都走这里； + 单期而且快照库就是那一期构建库的提交照旧走 publish_build。 + + imports 按统计期排序（含新导入的 id）；union 为 None 时快照库就是 imports 里唯一那一期的构建库（这次的 + period_build，或者已有构建）。快照 id 按 snapshot_id 的规则算，引擎版本一律 RECIPE_ENGINE_VER。 + + 1. 进锁后先 require_store_writable()；select 重读当前快照，不等于 expected_current 就抛 + PublishError(code="base_changed")：确认框是按旧的当前版本给的，别人在这期间提交或启用过，不能照旧切过去。 + 再检查本期构建、并集的临时文件都在且非空（trial_missing）。 + 2. 本期构建、并集都经 _publish_file 发布（带期望哈希，复用、恢复的规则同 publish_build）。并集 id 已登记、 + 登记的回执里的表结构却与这次的不同时 build_conflict：id 没把退役列算进去，复用旧文件就违背了 H2。 + 恢复了被改过的并集文件也记 build_restored（union_restored 另记一份）。 + 3. 快照 id 已存在（imports、mode、库路径都必须与已有记录相同，配方按内容哈希比、记录 id 可以不同，否则 + snapshot_conflict；现行配方跟着已有快照记的那条配方记录走）： + - 没回收、文件在、哈希等于登记值：不新建，按 activate_snapshot 的步骤切过去（snapshot_reused）； + - 已回收：复活。并集文件被回收删了的，第 2 步已经用这次的临时文件 link 回来，哈希必须等于登记的 + db_sha256（物化是确定的），否则 build_conflict；清掉快照和并集构建的回收标记（snapshot_revived）。 + 复用、复活都用已有快照冻结的表结构，不调 schema_cache_for。 + 4. 新快照：schema_cache_for(view, info) 返回要冻结的表结构（WP-5 在这里写导入清单和快照清单；返回值里的 + snapshot_manifest 记进快照的 manifest_artifact 索引列），再写构建记录（新的才写）、导入记录、快照记录。 + 5. 状态：快照里的各期置 active，其余 active 的置 superseded；目标配方置 active。 + 6. before_commit(新导入 id 或 None, 快照 id)：同一事务、锁内，pointer 还没切。之后写 snapshot_activations、 + 切指针、写 schema_cache，提交。 + + 任何一步失败：回滚，删掉这次新 link 出来的文件（恢复的、复用的不删），指针不动，异常原样抛出。临时文件一律 + 被消耗。成功后回收一次（失败只记日志）。调用之前调用方不得改动会话里的对象（理由同 publish_build）。 + """ + imports = [str(x) for x in imports] + if not imports or len(set(imports)) != len(imports): + raise ValueError("imports 不能为空，也不能有重复的导入记录") + if (period_build is None) != (new_import is None): + raise ValueError("period_build 和 new_import 要么都给，要么都不给") + if new_import is not None and new_import.import_id not in imports: + raise ValueError("新导入记录的 id 必须在 imports 里") + if union is None and len(imports) != 1: + raise ValueError("没有并集时快照只能有一期") + if mode not in ("replace", "accumulate"): + raise ValueError(f"快照的模式只能是 replace 或 accumulate，不能是 {mode!r}") + if new_import is not None: + unknown = set(new_import.fields) - IMPORT_FIELDS + if unknown: + raise ValueError(f"new_import.fields 含有不能经发布写入的列：{sorted(unknown)}") + state = sa_inspect(source) + if state.transient or state.pending: + raise ValueError("publish_snapshot 只用于已有的数据源：首次导入走 publish_build") + source_id = source.id + snap_id = snapshot_id(source_id, imports, recipe_types.RECIPE_ENGINE_VER, mode=mode, recipe_sha=recipe_sha256) + period_final = build_path(source_id, period_build.build_id) if period_build is not None else None + union_final = union_path(source_id, union.union_id) if union is not None else None + tmps = [t.tmp_db for t in (period_build, union) if t is not None] + linked: list[Path] = [] + try: + report = report_to_json(period_build.report) if period_build is not None else {} + union_report = report_to_json(union.report) if union is not None else None + async with store_lock(): + require_store_writable() + with session.no_autoflush: + previous = await _current_pointer(session, source_id) + if previous != expected_current: + raise PublishError(_BASE_CHANGED, code="base_changed") + for tmp in tmps: + if not _usable_tmp(tmp): + raise PublishError(_TRIAL_MISSING, code="trial_missing") + existing = await session.get(SourceSnapshot, snap_id) + pbuild = await session.get(TableBuild, period_build.build_id) if period_build is not None else None + ubuild = await session.get(TableBuild, union.union_id) if union is not None else None + rows = {i.id: i for i in (await session.execute( + select(TableImport).where(TableImport.id.in_(imports)))).scalars()} + last_seq = (await session.execute( + select(func.max(TableImport.seq)).where(TableImport.source_id == source_id) + )).scalar() or 0 + only_build = None + if union is None and period_build is None: + only = rows.get(imports[0]) + only_build = await session.get(TableBuild, only.build_id) if only is not None else None + # 已有的各期必须在、属于这个源；新导入的 id 不能已被占用 + olds = [i for i in imports if new_import is None or i != new_import.import_id] + stray = [i for i in olds if i not in rows or rows[i].source_id != source_id] + if stray: + raise ValueError(f"imports 里有不属于这个数据源的导入记录：{stray}") + if new_import is not None and new_import.import_id in rows: + raise ValueError("新导入记录的 id 已被占用") + if new_import is not None and not raw_store.raw_exists(new_import.raw_sha): + raise PublishError(_RAW_MISSING, code="raw_missing") + if previous is not None and snap_id == previous: + raise PublishError("这个版本已是当前版本", code="already_current") + + # 2. 文件落位 + build_sha, build_reused, build_restored = None, False, False + if period_build is not None: + placed = await asyncio.to_thread( + _publish_file, period_build.tmp_db, period_final, + pbuild.db_sha256 if pbuild is not None else None, period_build.expected_sha256) + if not placed.reused: + linked.append(period_final) + build_sha, build_reused, build_restored = placed + build_reused = build_reused and pbuild is not None + union_sha, union_reused, union_restored = None, False, False + if union is not None: + had = _union_layout(ubuild.report) if ubuild is not None else None + fresh = _union_layout(union_report) + if had is not None and fresh is not None and had != fresh: + # 同一个并集 id、表结构却不同：说明 id 没把退役列算进去（recipe_accumulate.union_id 的 retired）。 + # 照常复用的话，用户在试运行里核对的那份和真正发布的结构不同（H2），宁可停下 + logger.error("并集 id 已登记、表结构却与这次的不同：source=%s union=%s", source_id, union.union_id) + raise PublishError(_UNION_LAYOUT_CONFLICT, code="build_conflict") + uplaced = await asyncio.to_thread( + _publish_file, union.tmp_db, union_final, + ubuild.db_sha256 if ubuild is not None else None, union.expected_sha256) + if not uplaced.reused: + linked.append(union_final) + union_sha, union_reused = uplaced.sha, uplaced.reused and ubuild is not None + union_restored = uplaced.restored + if (ubuild is not None and not uplaced.reused and str(ubuild.db_sha256 or "").strip() + and uplaced.sha != ubuild.db_sha256 and existing is not None): + # 并集文件被回收删了、重新物化出来的却和登记的不同：物化本该是确定的，已有快照不能指向它 + raise PublishError(_BUILD_CONFLICT, code="build_conflict") + + # 快照库：并集 / 本期构建 / 唯一那一期的已有构建 + if union is not None: + db_path, db_sha = str(union_final), union_sha + elif period_build is not None: + db_path, db_sha = str(period_final), build_sha + else: + if only_build is None or only_build.retired_at is not None or not Path(only_build.db_path).is_file(): + # 只剩这一期，「先移除这一期」走不通（不能移除唯一的一期），出路只有回到更早的版本（7.4） + only = rows.get(imports[0]) + when = (f"（{only.period_start} 至 {only.period_end}）" + if only is not None and only.period_start and only.period_end else "") + raise PublishError(f"这一期{when}的数据文件已丢失，无法重建当前版本。可以在数据源卡片的「版本」中" + "启用更早的版本", code="part_missing") + if not await asyncio.to_thread(_file_matches, only_build.db_path, only_build.db_sha256): + raise PublishError("这一期的数据文件与登记的不一致，可能被修改过", code="part_tampered") + db_path, db_sha = only_build.db_path, only_build.db_sha256 + + now = utcnow() + seq = last_seq + 1 if new_import is not None else None + reused_snap = revived = False + if existing is not None: + # 3. 快照 id 已存在：id 相同则内容必然相同。对不上说明 id 规则被破坏了（不应出现）。 + # 期 3 之前的快照没记 recipe_id，按最后一期导入的算（9.6） + had_recipe = existing.recipe_id or (rows[imports[-1]].recipe_id if imports[-1] in rows else None) + same_recipe = (had_recipe or None) == (recipe_id or None) + if not same_recipe and had_recipe and str(recipe_sha256 or "").strip(): + # 快照 id 按配方**内容**（哈希）算，配方记录却可能是两条：例如按 R0 建了快照，之后改成 R1、又改回 + # 与 R0 逐字相同的 R2（新记录）。移除、作废回到那组导入时 id 正是当初那个，按记录 id 比会误报 + # snapshot_conflict，用户移除不了。所以按登记的配方哈希比：内容相同就是同一个版本 + with session.no_autoflush: + had_row = await session.get(TableRecipe, had_recipe) + same_recipe = (had_row is not None and had_row.source_id == source_id + and had_row.recipe_sha256 == str(recipe_sha256).strip()) + if (existing.source_id != source_id or [str(x) for x in existing.imports or []] != imports + or not same_recipe + or (existing.mode or "replace") != mode or not _same_path(existing.db_path, db_path)): + logger.error("快照 id 已存在但内容对不上：source=%s snapshot=%s", source_id, snap_id) + raise PublishError("导入失败：已有的同 id 版本与这次的内容不一致，当前版本未受影响", + code="snapshot_conflict") + if not await asyncio.to_thread(_file_matches, existing.db_path, existing.db_sha256): + raise PublishError(_BUILD_CONFLICT, code="build_conflict") + # 现行配方跟着这个快照记的那条配方记录走（内容相同）：与 activate_snapshot「current_recipe_id 换成 + # 该快照的 recipe_id」一致，免得当前快照记 R0、数据源的现行配方却是 R2，版本页的「第几版」两处对不上 + recipe_id = had_recipe or recipe_id + revived = existing.retired_at is not None + reused_snap = not revived + cache = copy.deepcopy(existing.schema_cache or {}) + db_path, db_sha = existing.db_path, existing.db_sha256 + else: + # 4. 新快照：先冻结表结构（这之前不动会话里的任何对象，理由同 publish_build） + info = SnapshotInfo( + snapshot_id=snap_id, import_id=new_import.import_id if new_import is not None else None, + seq=seq, build_id=period_build.build_id if period_build is not None else None, + db_sha256=build_sha, build_reused=build_reused, build_restored=build_restored, + union_id=union.union_id if union is not None else None, union_db_sha256=union_sha, + report=(pbuild.report or report) if build_reused and pbuild is not None else report, + union_report=((ubuild.report or union_report) if union_reused and ubuild is not None + else union_report), + ) + view = SourceView( + id=source_id, name=source.name, database=db_path, options=upload_options(source.options), + description=source.description or "", snapshot_id=snap_id, expected_sha256=db_sha, + ) + cache = await schema_cache_for(view, info) + + # 构建记录：新的才写；回收过、被恢复的补回路径、清掉回收标记 + if period_build is not None: + if pbuild is None: + session.add(TableBuild( + id=period_build.build_id, source_id=source_id, raw_sha256=new_import.raw_sha, + options_sha256=sha_json(period_build.options), engine_ver=recipe_types.RECIPE_ENGINE_VER, + options=period_build.options, db_path=str(period_final), db_sha256=build_sha, + report=report, created_at=now, + )) + else: + if not build_reused: + pbuild.db_sha256, pbuild.report = build_sha, report + elif not str(pbuild.db_sha256 or "").strip(): + pbuild.db_sha256 = build_sha + pbuild.db_path, pbuild.retired_at = str(period_final), None + if union is not None: + if ubuild is None: + session.add(TableBuild( + id=union.union_id, source_id=source_id, raw_sha256="", + options_sha256=sha_json(union.options), engine_ver=recipe_types.UNION_VER, + options=union.options, db_path=str(union_final), db_sha256=union_sha, + report=union_report, created_at=now, + )) + else: + if not union_reused: + ubuild.db_sha256, ubuild.report = union_sha, union_report + elif not str(ubuild.db_sha256 or "").strip(): + ubuild.db_sha256 = union_sha + ubuild.db_path, ubuild.retired_at = str(union_final), None + + if existing is not None: + existing.retired_at = None + existing.activated_at = now + else: + if new_import is not None: + session.add(TableImport( + id=new_import.import_id, source_id=source_id, seq=seq, build_id=period_build.build_id, + file_name=(new_import.file_name or "")[:500], file_size=int(new_import.file_size), + raw_sha256=new_import.raw_sha, raw_state="kept", status="active", + created_at=now, activated_at=now, **new_import.fields, + )) + manifest = cache.get("snapshot_manifest") if isinstance(cache, dict) else None + session.add(SourceSnapshot( + id=snap_id, source_id=source_id, imports=list(imports), db_path=db_path, db_sha256=db_sha, + schema_cache=cache, created_at=now, mode=mode, recipe_id=recipe_id, activated_at=now, + manifest_artifact=manifest if isinstance(manifest, str) else None, + )) + # 5. 状态（新导入记录 add 时已是 active）。提交路径上新配方由 before_commit 写进配方表， + # 这里先把其余配方置 superseded，before_commit 再加的新配方本来就是 active + await _switch_statuses(session, source_id, imports, recipe_id, now) + if before_commit is not None: + await before_commit(new_import.import_id if new_import is not None else None, snap_id) + session.add(_activation_row(source_id, snap_id, previous, activation, + default_import=new_import.import_id if new_import is not None else None)) + source.current_snapshot_id = snap_id + source.schema_cache, source.schema_synced_at = cache, now + # database 只作显示：查询一律从快照解析 + source.database = db_path + if recipe_id: + source.current_recipe_id = recipe_id + await session.commit() + except BaseException: + try: + await session.rollback() + except Exception: # noqa: BLE001 - 回滚失败也要先把文件收拾掉，再抛原来的异常 + logger.exception("上传表格发布多期版本失败后回滚出错") + for path in linked: + _remove_db_files(path) + raise + finally: + for tmp in tmps: + if _under(tmp, settings.uploads_dir): + _remove_db_files(tmp) + await engines.invalidate(source_id) + + try: + await gc(session) + except Exception: # noqa: BLE001 - 回收失败不影响这次发布，下次发布或重启再收 + logger.exception("上传表格发布多期版本后的回收失败") + return SnapshotPublish( + snapshot_id=snap_id, import_id=new_import.import_id if new_import is not None else None, + build_id=period_build.build_id if period_build is not None else None, + union_id=union.union_id if union is not None else None, build_reused=build_reused, + build_restored=build_restored or union_restored, union_reused=union_reused, seq=seq, + snapshot_reused=reused_snap, snapshot_revived=revived, union_restored=union_restored, + ) + + +def mask_lost(source_options: dict[str, Any] | None, schema_cache: dict[str, Any] | None) -> list[str]: + """数据源当前的遮罩列里，在这份表结构（快照冻结的 schema_cache）各表的列名中都找不到的（P3-SPEC 7.1）。 + + 遮罩按列名生效（不分大小写，同 recipe_imports、evidence 的比法）：回滚到列名不同的版本，遮罩会悄悄失效， + 证据面板和裁判摘录会显示本该遮罩的值。启用、移除、作废之前拿它和目标版本比，丢失的要人确认。 + """ + wanted = masked_columns(source_options) + if not wanted: + return [] + names: set[str] = set() + for meta in ((schema_cache or {}).get("tables") or {}).values(): + cols = meta.get("columns") if isinstance(meta, dict) else None + for col in cols or []: + if isinstance(col, dict) and col.get("name") is not None: + names.add(str(col["name"]).lower()) + return [c for c in wanted if c.lower() not in names] + + +_ACTIVATE_TEXT = { + "not_upload_source": "这个数据源不是上传的表格，没有可以启用的版本", + "snapshot_not_found": "这个版本不存在，或不属于这个数据源", + "already_current": "这个版本已是当前版本", + "retired": "这个版本已回收，无法启用", + "file_lost": "这个版本的数据文件已丢失，无法启用", + "snapshot_tampered": "这个版本的数据文件与登记的不一致，可能被修改过，无法启用", + "contains_revoked": "这个版本包含已作废接受的导入，无法启用", +} + + +async def activate_snapshot( + session: AsyncSession, source: DataSource, snapshot_id: str, *, expected_current: str | None, + kind: str = "activate", reason: str | None, signed_by: str | None, + import_id: str | None = None, ack_mask_lost: list[str] | None = None, +) -> str: + """启用一个还保留着的旧快照（回滚），在版本存储锁内原子完成。返回原来的当前快照 id（P3-SPEC 7.2）。 + + 1. require_store_writable()；select 重读数据源：不是上传源 not_upload_source；当前快照不等于 + expected_current 就 base_changed（确认框按旧的当前版本给的，不能照旧切过去）； + 2. 快照属于这个源（snapshot_not_found）、不是当前快照（already_current）、没回收、文件在（snapshot_unavailable）、 + 文件哈希等于登记值并与快照清单一致（每次都算，snapshot_tampered）、不含作废接受的导入（contains_revoked）； + 遮罩列丢失的集合不为空时必须等于 ack_mask_lost（mask_lost）； + 3. 切 current_snapshot_id、schema_cache 换成快照冻结的那份、database 换成快照库路径（只作显示）、 + current_recipe_id 换成快照的 recipe_id（期 3 之前的快照取最后一期导入的；简单导入为 None，源回到简单导入）； + 4. 快照里的各期导入置 active、其余 superseded；目标配方置 active、其余 superseded； + 5. 写 activated_at 和一行 snapshot_activations（kind、import_id、理由、署名）；提交； + 6. 失效引擎缓存。 + + 任何一步失败都回滚、指针不动，抛 ActivationError(code)（只读进程抛 StoreUnavailable）。回滚之后调用方手里的 + ORM 对象都已过期，在异步会话里再读它们的属性会抛 MissingGreenlet：要用的值先取成本地变量。作废接受时，调用方 + 可以先在同一个会话里改好那条导入记录的 revoked，再调这里：随这次提交一起生效，失败时一起回滚。 + """ + source_id = source.id + if kind not in ACTIVATION_KINDS or kind == "commit": + raise ValueError(f"启用旧版本的原因不能是 {kind!r}") + try: + async with store_lock(): + require_store_writable() + row = (await session.execute( + select(DataSource.origin, DataSource.current_snapshot_id, DataSource.options) + .where(DataSource.id == source_id))).first() + if row is None or (row[0] or "manual") != "upload": + raise ActivationError("not_upload_source", _ACTIVATE_TEXT["not_upload_source"]) + previous, options = row[1], row[2] + if previous != expected_current: + raise ActivationError("base_changed", _BASE_CHANGED) + snap = await session.get(SourceSnapshot, snapshot_id) + if snap is None or snap.source_id != source_id: + raise ActivationError("snapshot_not_found", _ACTIVATE_TEXT["snapshot_not_found"]) + if snap.id == previous: + raise ActivationError("already_current", _ACTIVATE_TEXT["already_current"]) + if snap.retired_at is not None: + raise ActivationError("snapshot_unavailable", _ACTIVATE_TEXT["retired"]) + if not snap.db_path or not Path(snap.db_path).is_file(): + raise ActivationError("snapshot_unavailable", _ACTIVATE_TEXT["file_lost"]) + if not await asyncio.to_thread(_file_matches, snap.db_path, snap.db_sha256): + raise ActivationError("snapshot_tampered", _ACTIVATE_TEXT["snapshot_tampered"]) + try: + verify_snapshot_manifest(source, snap) + except SnapshotManifestMismatch as e: + raise ActivationError("snapshot_tampered", str(e)) from e + members = [str(x) for x in snap.imports or []] + imps = {i.id: i for i in (await session.execute( + select(TableImport).where(TableImport.id.in_(members)))).scalars()} if members else {} + if any(i.revoked for i in imps.values()): + raise ActivationError("contains_revoked", _ACTIVATE_TEXT["contains_revoked"]) + lost = mask_lost(options, snap.schema_cache) + if lost and sorted(set(lost)) != sorted({str(c) for c in ack_mask_lost or []}): + raise ActivationError( + "mask_lost", "以下遮罩列在这个版本中没有同名列：" + "、".join(f"「{c}」" for c in lost) + + "。请在确认框中确认，或先调整遮罩设置") + recipe = snap.recipe_id + if recipe is None and members and members[-1] in imps: + recipe = imps[members[-1]].recipe_id + now = utcnow() + target = await session.get(DataSource, source_id) + await _switch_statuses(session, source_id, members, recipe, now) + target.current_snapshot_id = snap.id + target.schema_cache, target.schema_synced_at = copy.deepcopy(snap.schema_cache or {}), now + target.database = snap.db_path + target.current_recipe_id = recipe + snap.activated_at = now + session.add(_activation_row(source_id, snap.id, previous, { + "kind": kind, "import_id": import_id, "reason": reason, "signed_by": signed_by})) + await session.commit() + except BaseException: + try: + await session.rollback() + except Exception: # noqa: BLE001 - 回滚失败也照样抛原来的异常 + logger.exception("启用旧版本失败后回滚出错") + raise + finally: + await engines.invalidate(source_id) + return previous + + # -------------------------------------------------------------------------- # 暂存区：结束、瘦身、过期、原件的存与放 # -------------------------------------------------------------------------- @@ -977,6 +1874,8 @@ STAGING_SLIM: dict[str, Any] = { "scan": None, "grid_preview": None, "facts": None, "drafts": None, "trial": None, "answers_base": None, "cards": [], "questions": [], "draft_problems": [], "draft_partial": False, "answers": {}, "recipe_problems": [], "context_inputs": {}, "trial_key": None, "trial_path": None, + # 期 3：修改记录（含撤销用的旧 answers_base）、丢弃的回答、重新起草的哈希。提交前修改记录已抄进导入清单 + "edits": [], "answers_dropped": [], "redraft_sha256": None, } @@ -1001,7 +1900,11 @@ def close_staging(staging: ImportStaging, status: str, *, now: datetime | None = def remove_trial_file(path: str | None) -> bool: - """删一个试运行库（连同日志文件）。只删上传目录里 trials/ 下的：记录被改坏了也不能删到别处去。""" + """删一个试运行库（连同日志文件和附属的并集文件）。只删上传目录里 trials/ 下的：记录被改坏了也不能删到别处去。 + + 附属的并集文件（union_trial_path）跟着主文件走：暂存区只记主文件的路径，主文件删了它就没人认领了。 + 不查 require_store_writable：它只在已经拦过的入口和清理路径里被调用，在清理路径里抛异常反而会盖住原来的错误。 + """ if not path: return False target = Path(path) @@ -1011,6 +1914,7 @@ def remove_trial_file(path: str | None) -> bool: existed = target.exists() try: _remove_db_files(target) + _remove_db_files(union_trial_path(target)) except OSError as e: logger.warning("没能删掉试运行库 %s：%s", path, e) return False @@ -1095,7 +1999,12 @@ async def expire_stagings(session: AsyncSession, *, now: datetime | None = None) TTL 不一定生效，所以暂存、上传新一期、查看暂存区时也顺手调一次。调用时会话里不要有没提交的改动。 先在锁外看一眼有没有要处理的：没有就不去等锁，查看暂存区不该因为别处正在发布大文件而排队。 + + 只读进程（没拿到版本存储的守卫）直接跳过、不抛：暂存、上传新一期、修改配方一开头就调它，抛异常的话这些 + 接口在报出更具体的原因之前就一律 503；跳过以后，真正写文件的那一步再报 503。 """ + if not store_writable(): + return [] now = now or utcnow() pending = (await session.execute( select(ImportStaging.id).where(or_(_expiry_due(now), _closed_long_ago(now))).limit(1) @@ -1122,6 +2031,7 @@ async def store_staging(session: AsyncSession, staging: ImportStaging, raw: byte if staging.expires_at is None: staging.expires_at = utcnow() + STAGING_TTL async with store_lock(): + require_store_writable() raw_new = not raw_store.raw_exists(sha) try: await asyncio.to_thread(raw_store.put_raw, raw) @@ -1148,6 +2058,7 @@ async def release_staging_raw(session: AsyncSession, staging: ImportStaging) -> return False sha = staging.raw_sha256 async with store_lock(): + require_store_writable() if await _raw_in_use(session, sha, except_staging=staging.id): return False return await asyncio.to_thread(lambda: raw_store.purge_raw(sha, prune=True)) @@ -1158,11 +2069,34 @@ async def release_staging_raw(session: AsyncSession, staging: ImportStaging) -> # -------------------------------------------------------------------------- +def _drop_quarantine(source_id: str, build_ids: set[str] | None = None) -> int: + """删隔离区里这个源的文件：build_ids 给了只删这些构建的，否则连目录整个删掉。返回删了几个文件。""" + if not _SOURCE_ID.match(source_id or ""): + return 0 + folder = quarantine_root() / source_id + if not folder.is_dir(): + return 0 + removed = 0 + for path in sorted(folder.glob("*.db")): + if build_ids is not None and path.name.split("-", 1)[0] not in build_ids: + continue + try: + path.unlink() + removed += 1 + except OSError as e: + logger.warning("没能删掉隔离文件 %s：%s", path, e) + raw_store.remove_empty_dir(folder) + return removed + + async def retire_source(session: AsyncSession, source_id: str) -> None: """数据源删除时调用（不提交）：导入记录、配方全部置 retired，未结束的暂存区放弃（瘦身）。 文件交给随后的回收：被运行引用的版本会保留；放弃的暂存区留下的试运行库、没人要的原件也由回收删。 + 隔离区里这个源的文件当场删掉：它们含整张表的数据，源都删了就没有留着的理由（D13）。 """ + require_store_writable() + await asyncio.to_thread(_drop_quarantine, source_id) await session.execute( update(TableImport).where(TableImport.source_id == source_id, TableImport.status != "retired") .values(status="retired") @@ -1204,8 +2138,10 @@ async def purge_import_raw( 引用同一份原件的未结束暂存区也在同一个事务里放弃并瘦身、删掉试运行库：不这么做，它们随后 试运行或提交时读不到原件，暂存区里那份网格和回执也照样留着，清除就没有达到目的。 持版本存储的锁：发布正在把同一份原件登记给新导入时，不能在它提交前把文件删掉。 + 这几期构建的隔离文件也一并删掉：隔离文件就是那几期的数据，原件清除了它们不该还留着（第 8 节遗留项 1）。 """ async with store_lock(): + require_store_writable() sha = imp.raw_sha256 others = list((await session.execute( select(TableImport).where( @@ -1227,10 +2163,16 @@ async def purge_import_raw( for other in others: other.raw_state, other.purged = "purged", {**record, "via_import": imp.id} trials = [close_staging(s, "discarded", now=now) for s in stagings] + quarantined: dict[str, set[str]] = {} + for item in (imp, *others): + if item.build_id: + quarantined.setdefault(item.source_id, set()).add(item.build_id) await session.commit() for path in trials: await asyncio.to_thread(remove_trial_file, path) deleted = await asyncio.to_thread(lambda: raw_store.purge_raw(sha, prune=True)) + for src, bids in quarantined.items(): + await asyncio.to_thread(_drop_quarantine, src, bids) return RawPurge(deleted=deleted, others=others, discarded_stagings=stagings) @@ -1279,8 +2221,17 @@ async def gc(session: AsyncSession) -> None: - 原件的保护集合并上未结束暂存区的原件（被拒收、正在修改的那份不能删），以及配方源最近 RECIPE_RAW_KEEP 次导入的原件（被替换、构建已回收的也留着原件，重放要用）； - 已结束的暂存区留下的原件没人要了就删，留下的试运行库也删（放弃、删源时没来得及删的）。 + + 期 3 又多了几条： + - 受保护的快照再加上每个（还在的）源在当前快照、被运行引用的快照之外，按 coalesce(activated_at, created_at) + 倒序的前 SNAPSHOT_KEEP 个：回滚要有东西可回。受保护快照里的各期导入、它们的构建、并集构建（经 kept_paths）、 + 原件都受保护； + - 已回收的快照还可能在「移除」「撤销」时按同一组导入、同一个目标配方被复活（publish_snapshot）； + - 隔离区里超过 QUARANTINE_DAYS 天的文件删掉。 + 只读进程（没拿到守卫）不回收，抛 StoreUnavailable。 """ async with store_lock(): + require_store_writable() await _expire_stagings_locked(session, utcnow()) referenced: set[str] = set() for (versions,) in (await session.execute(select(Run.data_versions))).all(): @@ -1291,6 +2242,8 @@ async def gc(session: AsyncSession) -> None: )).all() } snaps = list((await session.execute(select(SourceSnapshot))).scalars()) + live_sources = {sid for (sid,) in (await session.execute(select(DataSource.id))).all()} + referenced |= _recent_snapshots(snaps, live_sources, referenced) imports = list((await session.execute(select(TableImport))).scalars()) builds = list((await session.execute(select(TableBuild))).scalars()) @@ -1363,6 +2316,7 @@ async def gc(session: AsyncSession) -> None: for path in _trial_files(): if _trial_owner(path) in closed_stagings: _remove_db_files(path) + _expire_quarantine(now) await asyncio.to_thread(_delete) # 被删掉文件的快照可能还有引擎开着文件句柄：已删除的文件要等句柄关了才真正释放空间 @@ -1375,6 +2329,55 @@ async def gc(session: AsyncSession) -> None: # -------------------------------------------------------------------------- +def _snapshot_time(snap: SourceSnapshot) -> datetime: + """快照最近一次成为当前版本的时刻；期 3 之前的快照没有 activated_at，按 created_at。""" + at = snap.activated_at or snap.created_at + if at is None: + return datetime.min.replace(tzinfo=timezone.utc) + return at if at.tzinfo is not None else at.replace(tzinfo=timezone.utc) + + +def _recent_snapshots(snaps: list[SourceSnapshot], live_sources: set[str], protected: set[str]) -> set[str]: + """每个还在的源在 protected（各源当前的、被运行引用的快照）之外，按 coalesce(activated_at, created_at) 倒序、 + 还没回收的前 SNAPSHOT_KEEP 个快照。 + + protected 里的不占名额：它们本来就不回收，占了名额，界面上「除当前版本和被运行引用的版本外，保留最近 3 个」 + 就成了假话（用户定案 P3-3）。已回收的也不参与排名：回收不会把它们复活（那是 publish_snapshot 的事），排进来 + 只会挤掉还能用的。已删除的源不保留：删源之后只有被运行引用的版本还留着。 + """ + by_source: dict[str, list[SourceSnapshot]] = {} + for snap in snaps: + if snap.retired_at is None and snap.source_id in live_sources and snap.id not in protected: + by_source.setdefault(snap.source_id, []).append(snap) + kept: set[str] = set() + for items in by_source.values(): + items.sort(key=lambda x: (_snapshot_time(x), x.id), reverse=True) + kept |= {x.id for x in items[:max(SNAPSHOT_KEEP, 0)]} + return kept + + +def _expire_quarantine(now: datetime) -> int: + """删掉隔离区里修改时间超过 QUARANTINE_DAYS 天的文件（同步，在线程里调）。返回删了几个。""" + root = quarantine_root() + if not root.is_dir(): + return 0 + cutoff = (now if now.tzinfo else now.replace(tzinfo=timezone.utc)).timestamp() - QUARANTINE_DAYS * 86400 + removed = 0 + for folder in root.iterdir(): + if not folder.is_dir(): + continue + for path in folder.glob("*.db"): + try: + if path.stat().st_mtime < cutoff: + path.unlink() + removed += 1 + logger.info("隔离文件超过 %d 天，已删除：%s", QUARANTINE_DAYS, path) + except OSError as e: + logger.warning("没能删掉隔离文件 %s：%s", path, e) + raw_store.remove_empty_dir(folder) + return removed + + def _recent_recipe_raws(imports: list[TableImport], recipe_sources: set[str]) -> set[str]: """配方源最近 RECIPE_RAW_KEEP 次导入（按 seq，不论状态）还留着的原件。""" by_source: dict[str, list[TableImport]] = {} @@ -1394,7 +2397,7 @@ def _trial_files() -> list[Path]: def _trial_owner(path: Path) -> str: - """试运行库文件名里的暂存区 id（<暂存区 id>-.db）。 + """试运行库文件名里的暂存区 id（<暂存区 id>-.db，附属的并集文件多一个 u）。 从右边切：trial_key 只有字母数字，暂存区 id 却可以带「-」。从左边切会把 stg-1 认成 stg， 启动清理就把一个开着的暂存区的库当孤儿删掉，回收也认不出已结束的暂存区。 @@ -1402,6 +2405,15 @@ def _trial_owner(path: Path) -> str: return path.name.removesuffix(".db").rsplit("-", 1)[0] +def _trial_main(path: Path) -> Path: + """试运行库文件对应的主文件：附属的并集文件（键的部分是 16 位加「u」）换回主文件名，主文件就是它自己。""" + stem = path.name.removesuffix(".db") + owner, _, key = stem.rpartition("-") + if owner and len(key) == 17 and key.endswith(_UNION_TRIAL_MARK): + return path.with_name(f"{owner}-{key[:-1]}.db") + return path + + def _same_path(a: str | os.PathLike[str], b: str | os.PathLike[str]) -> bool: try: return os.path.realpath(a) == os.path.realpath(b) @@ -1438,15 +2450,17 @@ async def _sweep_orphans(session: AsyncSession) -> None: kept_raw |= {sha for (_sid, status, _path, sha) in rows if status in STAGING_OPEN and sha} def _stale_trial(path: Path) -> bool: - """试运行库：暂存区不在、已结束、或者它指着的不是这个文件，就是孤儿。""" + """试运行库：暂存区不在、已结束、或者它指着的不是这个文件，就是孤儿。附属的并集文件按主文件判断。""" row = staging_rows.get(_trial_owner(path)) - return row is None or row[0] not in STAGING_OPEN or not row[1] or not _same_path(row[1], path) + main = _trial_main(path) + return row is None or row[0] not in STAGING_OPEN or not row[1] or not _same_path(row[1], main) def _sweep() -> list[Path]: orphans: list[Path] = [] root = tables_root() if root.is_dir(): - for path in root.glob("*/builds/*.db"): + # 构建库和并集库（并集也登记成构建）：文件名不在 table_builds.id 里的就是孤儿 + for path in [*root.glob("*/builds/*.db"), *root.glob(f"*/{UNION_DIR}/*.db")]: if path.stem not in build_ids: orphans.append(path) raw_root = raw_store.raw_root() @@ -1460,7 +2474,7 @@ async def _sweep_orphans(session: AsyncSession) -> None: # 空目录：被拒收的新源留下的 tables/<源 id>/builds/、清除原件后的 raw/<前两位>/。 # 启动时持着锁、还没开始接请求，删了也不会和正在写的发布撞上（发布会按需重建目录） if root.is_dir(): - for builds in [*root.glob("*/builds"), *root.glob(f"*/{TRIALS_DIR}")]: + for builds in [*root.glob("*/builds"), *root.glob(f"*/{TRIALS_DIR}"), *root.glob(f"*/{UNION_DIR}")]: if builds.is_dir() and raw_store.remove_empty_dir(builds): orphans.append(builds) for folder in root.iterdir(): @@ -1628,7 +2642,15 @@ async def _disable_manual_app_files(session: AsyncSession) -> int: async def startup(session: AsyncSession) -> None: """服务启动时调用一次：清理半成品和孤儿文件、迁移老上传、把指向上传目录的手工源改成只读、 - 停用指向应用自己数据文件的手工源、回收。""" + 停用指向应用自己数据文件的手工源、回收。 + + **第一步先取版本存储的守卫**（acquire_store_guard），拿不到就立即返回，什么都不做：同一个数据目录被另一个 + 进程占着时，这里的清理会删掉对方正在写的临时文件、已经 link 但记录还没提交的构建库，本该防止损坏的机制 + 自己造成损坏（第 8 节遗留项 5）。 + """ + if not acquire_store_guard(): + logger.error("上传表格的版本存储被另一个进程占用，本进程跳过启动清理、迁移和回收") + return async with _store_lock(): swept = await asyncio.to_thread(_sweep_tmp) if swept: diff --git a/backend/app/db/base.py b/backend/app/db/base.py index 7251f7d0..374829ae 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -188,6 +188,16 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [ ("table_imports", "manifest_artifact", "VARCHAR(64)"), ("table_imports", "signed_by", "VARCHAR(100)"), ("table_imports", "staging_id", "VARCHAR(32)"), + # 期 3：按期累积、版本页（P3-SPEC 9.6）。新表 snapshot_activations 由 create_all 建；存量行一律 NULL， + # 含义是「期 3 之前的」：快照的 mode 按 replace、recipe_id 取最后一期导入的、activated_at 按 created_at + ("source_snapshots", "mode", "VARCHAR(20)"), + ("source_snapshots", "recipe_id", "VARCHAR(32)"), + ("source_snapshots", "activated_at", "DATETIME"), + ("source_snapshots", "manifest_artifact", "VARCHAR(64)"), + ("table_imports", "revoked", "JSON"), + ("import_stagings", "edits", "JSON"), + ("import_stagings", "answers_dropped", "JSON"), + ("import_stagings", "redraft_sha256", "VARCHAR(64)"), ] diff --git a/backend/app/db/models.py b/backend/app/db/models.py index ae1570bb..4c019cbf 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -150,6 +150,9 @@ class TableImport(Base): # 署名（自报，未认证） signed_by: Mapped[str | None] = mapped_column(String(100), default=None) staging_id: Mapped[str | None] = mapped_column(String(32), default=None) + # 期 3：作废接受（界面叫「撤回这一期」）的记录 {at, reason, signed_by, signed_by_verified: false}。 + # 导入记录本身不删、接受内容不改（H4、H5），只在这里记下作废；含作废导入的快照不能再启用 + revoked: Mapped[dict[str, Any] | None] = mapped_column(default=None) class TableRecipe(Base): @@ -253,26 +256,65 @@ class ImportStaging(Base, TimestampMixin): expires_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) # 进入 committed / discarded / expired 的时刻 closed_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + # 期 3：已应用的修改（修复、框选）。每条含撤销用的 before（answers_base、recipe_origin）、base_sha256_after、 + # superseded；接口不给 before。结束时瘦身清空，提交前先抄进导入清单（含已被覆盖的，作为历史） + edits: Mapped[list[Any] | None] = mapped_column(default=list) + # 期 3：改配方后丢弃的回答 [{id, text, reason: changed | gone}]（遗留项 2）。结束时清空 + answers_dropped: Mapped[list[Any] | None] = mapped_column(default=list) + # 期 3：最近一次「按规则重新起草」给出的对齐配方的哈希。PUT recipe 带 origin=rules_redraft 时核对它；结束时清空 + redraft_sha256: Mapped[str | None] = mapped_column(String(64), default=None) class SourceSnapshot(Base): """数据源的一个可查询版本：若干期导入合成的一个库，加上冻结的表结构。 - 期 1 只有「每期替换」：一个快照就是一期导入，db_path 直接用那次构建的库。 - 运行钉的是快照 id（runs.data_versions），表结构和数据出自同一个版本。 + 「每期替换」：一个快照就是一期导入，db_path 直接用那次构建的库。「按期累积」（期 3）：imports 是 + 各期导入记录、按统计期排序，两期及以上（或一期但那一期的配方不是目标配方）时 db_path 指向物化出的 + 并集库（tables/<源>/snapshots/<并集 id>.db，并集也登记成一个构建）。 + 运行固定的是快照 id（runs.data_versions），表结构和数据出自同一个版本。 """ __tablename__ = "source_snapshots" id: Mapped[str] = mapped_column(String(64), primary_key=True) source_id: Mapped[str] = mapped_column(String(32), index=True) - imports: Mapped[list[Any]] = mapped_column(default=list) # import id 列表 + imports: Mapped[list[Any]] = mapped_column(default=list) # import id 列表（按统计期排序） db_path: Mapped[str] = mapped_column(Text, default="") db_sha256: Mapped[str] = mapped_column(String(64), default="") schema_cache: Mapped[dict[str, Any]] = mapped_column(default=dict) created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) # 没有运行、也不是当前版本时由回收标上；之后钉着它的请求明确报「版本不存在」 retired_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + # 以下是期 3 的新列，存量行一律 NULL，含义是「期 3 之前的」： + # replace / accumulate；NULL 按 replace 处理 + mode: Mapped[str | None] = mapped_column(String(20), default=None) + # 快照库按哪一版配方的表结构建的；NULL 时取 imports 最后一期的 recipe_id，简单导入为 NULL + recipe_id: Mapped[str | None] = mapped_column(String(32), default=None) + # 最近一次成为当前快照的时刻；NULL 按 created_at。回收按它保留最近 SNAPSHOT_KEEP 个 + activated_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + # 快照清单的工件 id（物化过的快照才有）。只作索引：证据链经 schema_cache 的 snapshot_manifest 承诺 + manifest_artifact: Mapped[str | None] = mapped_column(String(64), default=None) + + +class SnapshotActivation(Base): + """数据源的当前快照每切换一次记一行：提交、启用旧版本（回滚）、移除某一期、作废接受。 + + 快照记录只有「最近一次启用」的时刻；谁在什么时候、为什么把当前版本切到哪里，要留完整的历史 + （署名自报、未认证，照实标注）。不挂外键：数据源删掉以后照样留作审计。 + """ + + __tablename__ = "snapshot_activations" + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + source_id: Mapped[str] = mapped_column(String(32), index=True) + snapshot_id: Mapped[str] = mapped_column(String(64)) + previous_snapshot_id: Mapped[str | None] = mapped_column(String(64), default=None) + # commit / activate / remove_period / revoke_acceptance + kind: Mapped[str] = mapped_column(String(20), default="commit") + import_id: Mapped[str | None] = mapped_column(String(32), default=None) + reason: Mapped[str | None] = mapped_column(Text, default=None) + signed_by: Mapped[str | None] = mapped_column(String(100), default=None) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) class Setting(Base, TimestampMixin): diff --git a/backend/app/main.py b/backend/app/main.py index ed470c5e..0a17c1f8 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -10,7 +10,7 @@ from fastapi.responses import JSONResponse from app.api import ( artifacts, conversations, copilot, datasources, evidence, governance, knowledge, runs, - sandbox, settings as settings_api, table_imports, tools, workflows, + sandbox, settings as settings_api, source_versions, table_imports, tools, workflows, ) from app.api.coded import CodedHTTPException, coded_handler from app.core.config import settings @@ -119,6 +119,7 @@ for router in ( governance.router, # 按配方导入的 /api/datasources/imports/… 要在 datasources 的通配 /{source_id} 之前登记 table_imports.router, + source_versions.router, datasources.router, ): app.include_router(router) diff --git a/backend/app/tools/datasource.py b/backend/app/tools/datasource.py index 1d78bb0c..217fbf63 100644 --- a/backend/app/tools/datasource.py +++ b/backend/app/tools/datasource.py @@ -149,8 +149,11 @@ RECIPE_QUERY_HINT = " 中文表名和列名请加双引号。查单个值时， #: 冻结 schema_snapshot 时从 schema_cache 复制的顶层键。import_mode、import_manifests 只有按配方导入的源才有： #: 冻结进来以后，schema_snapshot 的内容哈希就承诺了导入清单（证据链按哈希从封存事件走到清单，中间不经过 -#: 可改的数据库列，H5）；手工源和期 1 的上传源没有这两个键，冻结内容和原来一字不差 -_FROZEN_SCHEMA_KEYS = ("schema", "synced_at", "truncated", "total", "import_mode", "import_manifests") +#: 可改的数据库列，H5）；手工源和期 1 的上传源没有这两个键，冻结内容和原来一字不差。 +#: snapshot_manifest（期 3，P3-SPEC 2.9）：按期累积物化的快照才有，是快照清单的内容哈希。冻结进来以后证据链是 +#: 封存事件 → schema_snapshot → 快照清单 → 各期导入清单 → 原件或清除记录，每一跳都按内容哈希 +_FROZEN_SCHEMA_KEYS = ("schema", "synced_at", "truncated", "total", "import_mode", "import_manifests", + "snapshot_manifest") def _query_description(source: Any) -> str: diff --git a/backend/tests/fixtures/recipes/p2_baseline.json b/backend/tests/fixtures/recipes/p2_baseline.json new file mode 100644 index 00000000..7f5c9b58 --- /dev/null +++ b/backend/tests/fixtures/recipes/p2_baseline.json @@ -0,0 +1,36 @@ +{ + "about": "期 2 基线（P3-SPEC 11.3 WP-1、12.1）：在 feat/excel-p2 2d5e50f 的代码上按参考配方执行 D00 得到的构建库哈希与表哈希。期 3 对没用新字段的配方必须写出逐字节相同的库（RECIPE_ENGINE_VER 不升，期 2 的构建照样复用），WP-1 拿它当测试常量；snapshot_id、recipe_build_id 用固定的假 id 算，替换模式、简单导入的 id 规则不得变化（WP-4）。夹具是合成的假数据", + "commit": "2d5e50fa88ac45aa174ec68f4d3b5b425c8bcbd8", + "procedure": "raw, name = flow_workbook(date(2026, 8, 1), 31)；ex = recipe_engine.execute(Recipe.model_validate(flow_recipe.json), raw, name, xlsx_scan.scan(raw), db)；table_versions.settle_journal(db)；db_sha256 = raw_store.sha256_file(db)；table_hashes = ex.table_hashes", + "fixture": { + "call": "flow_workbook(date(2026, 8, 1), 31)", + "file_name": "月报导出_2026-08-01_2026-08-31.xlsx", + "raw_sha256": "6269f8e67b1f3a9517fd1e2e027d01f05a53205699a00f5bf72a7945bf08c748" + }, + "recipe": { + "path": "tests/fixtures/recipes/flow_recipe.json", + "recipe_sha256": "971a8eba5ac606bd0552c43ff83b5c361d134c77d471bfe2d28e440ebfcc07b4" + }, + "engine_ver": "recipe/1", + "ok": true, + "db_sha256": "986c9128e60b653852330cc496215e634cf37086947c6c9e709cc54737e7f313", + "table_hashes": { + "日客流": "36452a4d651d017f2d7865e63b1543f6afb939f43c2c61c3672a67df421a4b2b", + "时段客流": "3dce5a79d697ca9b0078d2cc34d75d32d7094119385bee874663f5f1d6429c3a", + "时段客流_表内合计": "098eca2473c066b156d7f42650b2545f3ec9bc283ee95e110aa11e560b02ef68" + }, + "rows": { + "日客流": 31, + "时段客流": 527, + "时段客流_表内合计": 93 + }, + "ids": { + "source_id": "src-p2-baseline", + "import_ids": [ + "imp-p2-baseline" + ], + "recipe_build_id": "f0b253448ae17d20b54710c60fed8d28e96fe457e5fe65cd042fd09eb5e0dea4", + "snapshot_id_recipe": "b3ff74d7716ce2ea204b7c9b4fe3148d9263c638f49c5d8696a6eb1c2edfab80", + "snapshot_id_simple": "bbb5efd66da448b991d2706ed4656b9dfc6424f1dfd9247fffab6bc64689b262" + } +} diff --git a/backend/tests/fixtures/xlsx/flow.py b/backend/tests/fixtures/xlsx/flow.py index 2d1999c6..c4abde9e 100644 --- a/backend/tests/fixtures/xlsx/flow.py +++ b/backend/tests/fixtures/xlsx/flow.py @@ -33,12 +33,16 @@ EXTRA_SHEET = "说明" #: 只换起止日期、版式与 D00 相同的代号 _PLAIN = {"D00", "D01", "D02", "D03", "D03b"} -#: 9.2 的全部代号，外加 noperiod（把 B2 删掉，9.7 第 6 步的人工录入统计期用） +#: 9.2 的全部代号，外加 noperiod（把 B2 删掉，9.7 第 6 步的人工录入统计期用）和期 3 的 D24w +#: （时段标签用全角数字和全角减号「７－８」，其余同 D01；P3-SPEC 11.1，验收 A5 用） VARIANTS = frozenset(_PLAIN | { "D04", "D05", "D06", "D07", "D08", "D09", "D10", "D11", "D12", "D13", "D14", "D15", "D16", "D17", "D18", - "D19", "D20", "D21", "D22", "D23", "D24", "D25", "D26", "D27", "D28", "D28b", "noperiod", + "D19", "D20", "D21", "D22", "D23", "D24", "D25", "D26", "D27", "D28", "D28b", "noperiod", "D24w", }) +#: 半角数字 → 全角数字（D24w） +_FULLWIDTH_DIGITS = str.maketrans("0123456789", "０１２３４５６７８９") + #: D00（2026-08-01 起 31 天、参考配方）的期望，照 P2-SPEC 9.1 抄录，WP-7 验收时逐项比 FLOW_EXPECT: dict[str, Any] = { "nonempty": 771, @@ -91,6 +95,8 @@ def _hour_label(h: int, variant: str) -> str: return f"{h:02d}:00-{h + 1:02d}:00" if variant == "D24": return f"{h}–{h + 1}" # en dash + if variant == "D24w": # 全角数字、全角减号（U+FF0D）：按规范写法存成「7-8」 + return f"{h}－{h + 1}".translate(_FULLWIDTH_DIGITS) return f"{h}-{h + 1}" diff --git a/backend/tests/fixtures/xlsx/lab.py b/backend/tests/fixtures/xlsx/lab.py index 9b480e6e..d9c5a90d 100644 --- a/backend/tests/fixtures/xlsx/lab.py +++ b/backend/tests/fixtures/xlsx/lab.py @@ -6,6 +6,8 @@ from __future__ import annotations from openpyxl import Workbook +from openpyxl.utils import column_index_from_string, get_column_letter +from openpyxl.utils.cell import coordinate_from_string from . import excel_saved, save @@ -29,38 +31,65 @@ def _variant(v: str, allowed: tuple[str, ...]) -> str: return v -def c01(variant: str = "literal") -> tuple[bytes, str]: +#: c01 挪位（shift）时，表的上方另加的两行说明（不含数字：只让「挪了位置」这一件事不同，P3-SPEC 11.1） +C01_SHIFT_NOTES = ("说明：本表为合成测试数据", "说明：口径与上月相同") + + +def _shift_ref(ref: str, dr: int, dc: int) -> str: + """「C5」或「C1:F1」整体平移 dr 行、dc 列。""" + def one(a: str) -> str: + col, row = coordinate_from_string(a) + return f"{get_column_letter(column_index_from_string(col) + dc)}{row + dr}" + return ":".join(one(a) for a in ref.split(":")) + + +def c01(variant: str = "literal", *, shift: tuple[int, int] = (0, 0)) -> tuple[bytes, str]: """表头在 C5，上方标题、单位、编制行，下方合计行、空行、备注。 literal：合计写死；uncached：合计是公式、没有缓存值（openpyxl 直接写出的样子）；cached：补上缓存值。 期望：列表块 + total_row(pick=合计, keep_as)；区域外含数字的文字只有 C1、C3 两格（outside_digits ×2）。 + + shift=(行, 列)（期 3，框选回放验收 4.6 第 2 条）：整张表（含 C1:C3 的标题、单位、编制行和下方的备注、 + 合计公式的引用）整体往下挪若干行、往右挪若干列；往下挪了 2 行及以上时，第 1、2 行另加两行不含数字的说明 + （C01_SHIFT_NOTES，写在挪位后的首列）。shift=(3, 2) 时表头落在 E8。默认 (0, 0) 与期 2 逐字节相同。 """ variant = _variant(variant, C01_VARIANTS) + dr, dc = shift + if dr < 0 or dc < 0: + raise ValueError(f"shift 只能往下、往右挪，收到 {shift!r}") + + def at(ref: str) -> str: + return _shift_ref(ref, dr, dc) + book = Workbook() ws = book.active ws.title = "月报" - ws["C1"] = "2026年8月 销售月报" - ws.merge_cells("C1:F1") - ws["C2"] = "单位：万元" - ws["C3"] = "编制：财务部 日期：2026-09-05" + ws[at("C1")] = "2026年8月 销售月报" + ws.merge_cells(at("C1:F1")) + ws[at("C2")] = "单位：万元" + ws[at("C3")] = "编制：财务部 日期：2026-09-05" for i, h in enumerate(("地区", "产品", "销量", "金额")): - ws.cell(5, 3 + i, h) + ws.cell(5 + dr, 3 + dc + i, h) data = [(ZONES[0], PRODUCTS[0], 10, 100), (ZONES[1], PRODUCTS[1], 20, 120), (ZONES[2], PRODUCTS[2], 15, 80)] for r, row in enumerate(data, start=6): for i, v in enumerate(row): - ws.cell(r, 3 + i, v) - ws["C9"] = "合计" + ws.cell(r + dr, 3 + dc + i, v) + ws[at("C9")] = "合计" cached: dict[str, object] = {} if variant == "literal": - ws["E9"], ws["F9"] = 45, 300 + ws[at("E9")], ws[at("F9")] = 45, 300 else: - ws["E9"], ws["F9"] = "=SUM(E6:E8)", "=SUM(F6:F8)" + ws[at("E9")], ws[at("F9")] = f"=SUM({at('E6:E8')})", f"=SUM({at('F6:F8')})" if variant == "cached": - cached = {"E9": 45, "F9": 300} + cached = {at("E9"): 45, at("F9"): 300} # 备注不含数字：这一格若带数字会多出一条 outside_digits，与 9.3 的期望（C1、C3 两格）不符 - ws["C11"] = "注：数据来源于业务系统，金额含税。" - ws["C12"] = "制表人：经办甲" - return excel_saved(save(book), "月报", cached), f"c01_{variant}.xlsx" + ws[at("C11")] = "注：数据来源于业务系统，金额含税。" + ws[at("C12")] = "制表人：经办甲" + if dr >= 2: + for i, note in enumerate(C01_SHIFT_NOTES): + ws.cell(1 + i, 3 + dc, note) + name = f"c01_{variant}.xlsx" if shift == (0, 0) else f"c01_{variant}_shifted.xlsx" + return excel_saved(save(book), "月报", cached), name def c04() -> tuple[bytes, str]: diff --git a/backend/tests/test_datasources_upload.py b/backend/tests/test_datasources_upload.py index 23ca0cd4..51ca5fb7 100644 --- a/backend/tests/test_datasources_upload.py +++ b/backend/tests/test_datasources_upload.py @@ -278,18 +278,26 @@ async def test_upload_still_refuses_to_take_over_other_sources(client, tmp_path) assert "请换一个名称" in resp.json()["detail"] -async def test_publish_failure_is_reported_without_moving_the_pointer(client): +async def test_reupload_over_a_tampered_build_file_restores_it(client, store): + """重传同一文件、服务端的构建文件被改成 0 字节（期 1 这里回 500）：用这次上传恢复，原文件挪进隔离区。 + + 响应里的 build_restored 字段由 WP-5 加，那条断言在 test_p3_api_errors.py；这里只看状态码、隔离区和指针。 + """ name, raw = unique(), xlsx(V1) first = (await upload(client, name, raw)).json() async with SessionLocal() as session: snap = await session.get(SourceSnapshot, first["snapshot_id"]) - path = Path(snap.db_path) + path, registered = Path(snap.db_path), snap.db_sha256 os.chmod(path, 0o644) path.write_bytes(b"") resp = await upload(client, name, raw) - assert resp.status_code == 500 - assert "不一致" in resp.json()["detail"] - assert (await by_name(name)).current_snapshot_id == first["snapshot_id"] + assert resp.status_code == 201, resp.text + body = resp.json() + row = await by_name(name) + assert row.current_snapshot_id == body["snapshot_id"] != first["snapshot_id"] + assert raw_store.sha256_file(path) == registered + moved = sorted(p for p in (store / "uploads" / "quarantine" / row.id).rglob("*.db")) + assert len(moved) == 1 and moved[0].stat().st_size == 0 # -------------------------------------------------------------------------- diff --git a/backend/tests/test_fixtures_xlsx.py b/backend/tests/test_fixtures_xlsx.py index 9d80305d..9adfefd9 100644 --- a/backend/tests/test_fixtures_xlsx.py +++ b/backend/tests/test_fixtures_xlsx.py @@ -268,6 +268,29 @@ def test_drift_case_builds_with_the_intended_change(code): assert case.rows == (days, 17 * days, 3 * days) +def test_d24w_hour_labels_are_fullwidth(): + """期 3 的 D24w（P3-SPEC 11.1）：时段标签用全角数字和全角减号，规范写法仍是「7-8」；其余与 D01 相同。""" + from app.data import recipe_parsers as P + + raw, name = flow_workbook(SEP, 30, variant="D24w") + base, base_name = flow_workbook(SEP, 30) + assert name == base_name + ws, plain = _sheet(raw), _sheet(base) + hours = [*range(10, 21), *range(22, 28)] + assert [ws[f"B{r}"].value for r in (10, 13, 27)] == ["７－８", "１０－１１", "２３－２４"] + for r in hours: + label = ws[f"B{r}"].value + assert all(not ("0" <= ch <= "9") and ch != "-" for ch in label), label + assert "－" in label and P.hour_range(label).canonical == plain[f"B{r}"].value + # 只改了时段标签：其余格（数字、合计标签、公式、统计期）与 D01 逐格相同 + changed = _diff(cells(base), cells(raw)) + assert changed == {f"B{r}" for r in hours} + assert _diff(cells(base, data_only=True), cells(raw, data_only=True)) == changed + assert ws["B28"].value == "18-22 时合计" and not has_full_calc(raw) + assert flow_workbook(SEP, 30, variant="D24w")[0] == raw + xlsx_scan.scan(raw) + + def test_noperiod_variant_drops_b2_only(): raw, _ = flow_workbook(SEP, 30, variant="noperiod") ws = _sheet(raw) @@ -362,7 +385,7 @@ def _all(raw: bytes) -> dict[tuple[str, str], object]: LAB_CASES = ([("c01", v, lab.c01(v)) for v in lab.C01_VARIANTS] + [("c04", "", lab.c04())] + [("c05", v, lab.c05(v)) for v in lab.C05_VARIANTS] + [("c06", "", lab.c06())] + [("c08", v, lab.c08(v)) for v in lab.C08_VARIANTS] + [("c10", m, lab.c10(m)) for m in lab.C10_MONTHS] - + [("kv", "", lab.kv_form())]) + + [("kv", "", lab.kv_form()), ("c01", "shift", lab.c01("literal", shift=(3, 2)))]) @pytest.mark.parametrize("name,variant,built", LAB_CASES, ids=[f"{n}-{v}" for n, v, _ in LAB_CASES]) @@ -394,6 +417,40 @@ def test_lab_c01(): lab.c01("nope") +def test_lab_c01_shift(): + """期 3 的 c01 挪位（P3-SPEC 4.6 第 2 条、11.1）：整张表往下 3 行、往右 2 列，表头落在 E8，上方另加两行 + 不含数字的说明；shift=(0, 0) 与不传参数逐字节相同。""" + for variant in lab.C01_VARIANTS: + assert lab.c01(variant, shift=(0, 0)) == lab.c01(variant) + for variant in lab.C01_VARIANTS: + raw, name = lab.c01(variant, shift=(3, 2)) + assert name == f"c01_{variant}_shifted.xlsx" + ws, cached = book(raw)["月报"], book(raw, data_only=True)["月报"] + assert [ws.cell(8, c).value for c in range(5, 9)] == ["地区", "产品", "销量", "金额"] + assert [ws.cell(9, c).value for c in range(5, 9)] == ["分区甲", "产品甲", 10, 100] + assert ws["E4"].value == "2026年8月 销售月报" and {str(m) for m in ws.merged_cells.ranges} == {"E4:H4"} + assert (ws["E5"].value, ws["E12"].value, ws["E14"].value) == ("单位：万元", "合计", "注：数据来源于业务系统，金额含税。") + assert [ws["E1"].value, ws["E2"].value] == list(lab.C01_SHIFT_NOTES) and ws["E3"].value is None + # 原来的位置都空了：C 列、第 5 行左侧没有任何格 + assert all(ws.cell(r, c).value is None for r in range(1, 16) for c in (1, 2, 3, 4)) + texts = {ref: ws[ref].value for ref in ("E1", "E2", "E4", "E5", "E6", "E14", "E15")} + assert [ref for ref, v in texts.items() if any(ch.isdigit() for ch in v)] == ["E4", "E6"] + if variant == "literal": + assert (ws["G12"].value, ws["H12"].value) == (45, 300) + else: + assert (ws["G12"].value, ws["H12"].value) == ("=SUM(G9:G11)", "=SUM(H9:H11)") + want = (45, 300) if variant == "cached" else (None, None) + assert (cached["G12"].value, cached["H12"].value) == want + assert not has_full_calc(raw) + assert lab.c01(variant, shift=(3, 2))[0] == raw + # 挪位只改坐标：非空格的值（公式里的引用除外）与原表一一对应 + plain = sorted(str(v) for v in cells(lab.c01("literal")[0], "月报").values()) + moved = cells(lab.c01("literal", shift=(3, 2))[0], "月报") + assert sorted(str(v) for v in moved.values() if v not in lab.C01_SHIFT_NOTES) == plain + with pytest.raises(ValueError): + lab.c01("literal", shift=(-1, 0)) + + def test_lab_c04_merges(): ws = book(lab.c04()[0])["合并"] assert {str(m) for m in ws.merged_cells.ranges} == {"A1:F1", "A3:A4", "B3:B4", "C3:D3", "E3:F3", "A5:A7", "A8:A10"} diff --git a/backend/tests/test_p3_api_errors.py b/backend/tests/test_p3_api_errors.py new file mode 100644 index 00000000..6fb872de --- /dev/null +++ b/backend/tests/test_p3_api_errors.py @@ -0,0 +1,421 @@ +"""期 3 接口的错误码与遗留项（WP-5，P3-SPEC 9.1、第 8 节遗留项 1、5）：真管线、经 HTTP。 + +- 遗留项 1（接口部分）：重传撞上被改过的构建文件，用这次的文件恢复（build_restored=true）；记录的哈希也被改掉时 + 409 build_conflict、指针不动。/upload 和配方提交各一例； +- 遗留项 5（接口部分）：另一个进程占着版本存储的守卫时，写版本存储的接口一律 503 store_unavailable（body 照样是 + {detail, code}），只读的查询照常，过期处理跳过不抛； +- 修改请求的形状（edit_invalid）、PUT 的 origin、redraft_mismatch、undo_stale、试运行的 part_tampered / part_missing、 + 提交的 union_tampered、清除原件的机读码。 + +夹具一律合成、假名、随机数；不调用任何模型。 +""" +from __future__ import annotations + +import io +import os +import subprocess +import sys +from pathlib import Path + +import openpyxl +import pytest +from sqlalchemy import update + +from app.core.config import settings +from app.data import table_versions +from app.db.base import SessionLocal +from app.db.models import DataSource, ImportStaging, SourceSnapshot, TableBuild, TableImport +from tests.fixtures.xlsx.drift import DRIFT_CASES, OCT, SEP +from tests.fixtures.xlsx.flow import flow_workbook +from tests.test_table_imports_api_p3 import ( # noqa: F401 - client、store 是夹具 + API, XLSX, client, commit, db_get, first_import, reupload, store, trial, unique, +) + + +def sheet_bytes(seed: int) -> bytes: + """一份最简单的表格（期 1 的简单上传用）：表头加两行，数字随机。""" + import random + + rnd = random.Random(seed) + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "甲表" + ws.append(["分区", "数量"]) + ws.append(["分区甲", rnd.randint(1, 999)]) + ws.append(["分区乙", rnd.randint(1, 999)]) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +async def upload(client, name: str, raw: bytes): + return await client.post(f"{API}/upload", files={"file": ("甲表.xlsx", raw, XLSX)}, data={"name": name}) + + +def corrupt(path: Path, data: bytes = b"") -> None: + os.chmod(path, 0o644) + path.write_bytes(data) + + +# ========================================================================== +# 遗留项 1：被改过的构建文件（接口部分） +# ========================================================================== + + +async def test_upload_restores_a_tampered_build_and_reports_build_restored(client, store): + name, raw = unique(), sheet_bytes(501) + first = (await upload(client, name, raw)).json() + assert first["build_restored"] is False + snap = await db_get(SourceSnapshot, first["snapshot_id"]) + corrupt(Path(snap.db_path)) + resp = await upload(client, name, raw) + assert resp.status_code == 201, resp.text + assert resp.json()["build_restored"] is True + moved = list((store / "uploads" / "quarantine" / first["source"]["id"]).rglob("*.db")) + assert len(moved) == 1 + + +async def test_upload_build_conflict_is_a_409_and_keeps_the_pointer(client): + name, raw = unique(), sheet_bytes(502) + first = (await upload(client, name, raw)).json() + snap = await db_get(SourceSnapshot, first["snapshot_id"]) + imp = await db_get(TableImport, first["import_id"]) + corrupt(Path(snap.db_path)) + async with SessionLocal() as session: + await session.execute(update(TableBuild).where(TableBuild.id == imp.build_id).values(db_sha256="0" * 64)) + await session.commit() + resp = await upload(client, name, raw) + assert resp.status_code == 409 and resp.json()["code"] == "build_conflict" + src = await db_get(DataSource, first["source"]["id"]) + assert src.current_snapshot_id == first["snapshot_id"] + + +async def test_recipe_commit_restores_a_tampered_build_and_reports_build_restored(client): + """配方提交（publish_build 的路径）：8 月 → 9 月 → 8 月原文件再传（构建与第一次相同），第一次的构建文件被改成 + 0 字节：用这次试运行的文件恢复，CommitOut.build_restored=true。""" + name = unique() + base = await first_import(client, name, seed=503, mode="replace") + sid = base["source_id"] + raw9, fn9 = flow_workbook(SEP, 30, seed=504) + st = await reupload(client, sid, raw9, fn9) + assert (await commit(client, st)).status_code == 201 + imp = await db_get(TableImport, base["commit"]["import_id"]) + build = await db_get(TableBuild, imp.build_id) + corrupt(Path(build.db_path)) + st = await reupload(client, sid, base["raw"], base["file_name"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["build_restored"] is True and out["build_id"] == imp.build_id and out["parts"] == 1 + + +# ========================================================================== +# 遗留项 5：只读进程（接口部分） +# ========================================================================== + +_HOLD = """ +import fcntl, os, sys +fd = os.open(sys.argv[1], os.O_RDWR | os.O_CREAT, 0o600) +fcntl.flock(fd, fcntl.LOCK_EX) +print("locked", flush=True) +sys.stdin.read() +""" + + +@pytest.fixture +def held_guard(store): + """另一个进程占着 uploads/.store.lock：本进程取守卫拿不到，进入只读。用完放掉、复原守卫状态。""" + path = settings.uploads_dir / ".store.lock" + proc = subprocess.Popen([sys.executable, "-c", _HOLD, str(path)], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, text=True) + assert proc.stdout.readline().strip() == "locked" + yield proc + proc.stdin.close() + proc.wait(timeout=10) + table_versions._release_guard() + + +async def test_read_only_process_refuses_writes_with_503_and_keeps_reads(client, held_guard): + name = unique() + base = await first_import(client, name, seed=505) + sid = base["source_id"] + raw9, fn9 = flow_workbook(SEP, 30, seed=506) + st = await reupload(client, sid, raw9, fn9) + assert table_versions.acquire_store_guard() is False + + def refused(resp) -> None: + assert resp.status_code == 503, resp.text + assert resp.json()["code"] == "store_unavailable" and "单进程" in resp.json()["detail"] + + refused(await upload(client, unique(), sheet_bytes(507))) + refused(await client.post(f"{API}/imports/stage", files={"file": (fn9, raw9, XLSX)}, data={"name": unique()})) + refused(await client.post(f"{API}/{sid}/reupload", files={"file": (fn9, raw9, XLSX)})) + refused(await client.post(f"{API}/imports/{st['id']}/trial", json={})) + refused(await commit(client, st)) + refused(await client.post(f"{API}/imports/{st['id']}/answers", json={"answers": {}})) + refused(await client.put(f"{API}/imports/{st['id']}/recipe", json={"recipe": st["recipe"]})) + refused(await client.post(f"{API}/imports/{st['id']}/redraft-rules", json={})) + refused(await client.delete(f"{API}/imports/{st['id']}")) + refused(await client.post(f"{API}/{sid}/imports/{base['commit']['import_id']}/purge-raw", + json={"reason": "合成理由"})) + refused(await client.delete(f"{API}/{sid}")) + # 只读的查询照常；查看暂存区会顺手做过期处理，只读进程里跳过、不抛 + assert (await client.get(API)).status_code == 200 + assert (await client.get(f"{API}/{sid}/snapshots")).status_code == 200 + assert (await client.get(f"{API}/{sid}/imports")).status_code == 200 + got = await client.get(f"{API}/imports/{st['id']}") + assert got.status_code == 200 and got.json()["status"] == "trialed" + async with SessionLocal() as session: + assert await table_versions.expire_stagings(session) == [] + + +async def test_read_only_process_refuses_activate_and_commit_keeps_the_trial(client, held_guard): + """启用旧版本 503；提交 503 时不消耗试运行库（守卫拿回来之后照常提交）。""" + name = unique() + base = await first_import(client, name, seed=508) + sid = base["source_id"] + raw9, fn9 = flow_workbook(SEP, 30, seed=509) + st = await reupload(client, sid, raw9, fn9) + s2 = (await commit(client, st)).json()["snapshot_id"] + raw10, fn10 = flow_workbook(OCT, 31, seed=510) + st = await reupload(client, sid, raw10, fn10) + assert table_versions.acquire_store_guard() is False + resp = await client.post(f"{API}/{sid}/snapshots/{base['commit']['snapshot_id']}/activate", + json={"confirm": True, "expected_current_snapshot_id": s2}) + assert resp.status_code == 503 and resp.json()["code"] == "store_unavailable" + resp = await commit(client, st) + assert resp.status_code == 503 + row = await db_get(ImportStaging, st["id"]) + assert row.status == "trialed" and Path(row.trial_path).is_file(), "只读进程不消耗试运行库" + held_guard.stdin.close() + held_guard.wait(timeout=10) + assert table_versions.acquire_store_guard() is True + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert resp.json()["parts"] == 3 + + +async def test_read_only_process_refuses_version_and_edit_writes_before_doing_any_work(client, held_guard, store, + monkeypatch): + """只读进程：移除一期（要物化）、作废接受（累积模式，要物化）、edits/apply、edits/undo、POST /{id}/redraft 一律 503， + 而且在组装各期、物化并集之前就停下：版本存储里不留任何临时文件（第 8 节遗留项 5「本进程不写入版本存储」，评审 + 意见：原来先物化、写进 snapshots/ 之后才由 publish_snapshot 拒绝）。""" + from app.data import source_versions + from tests.test_source_versions_api import accepted_september + from tests.test_table_imports_api_p3 import apply, preview + + v = await accepted_september(client, seed=521, mode="accumulate") + sid = v["source_id"] + raw10, fn10 = flow_workbook(OCT, 31, seed=522) + resp = await commit(client, await reupload(client, sid, raw10, fn10)) + assert resp.status_code == 201, resp.text + s3 = resp.json()["snapshot_id"] + # 一个已应用修复的暂存区（撤销用），另一个源上一个预览好、还没应用的修复（应用用） + raw, fn = DRIFT_CASES["D04"].build(523) + applied = await reupload(client, sid, raw, fn) + fx = next(f for f in applied["fixes"] if f["kind"] == "remove_label") + body = {"fix": {"id": fx["id"], "option": "remove"}} + pv = (await preview(client, applied["id"], body)).json() + assert (await apply(client, applied["id"], body, pv["recipe_sha256_after"])).status_code == 200 + other = await first_import(client, unique(), seed=524) + pending = await reupload(client, other["source_id"], raw, fn) + fx2 = next(f for f in pending["fixes"] if f["kind"] == "remove_label") + body2 = {"fix": {"id": fx2["id"], "option": "remove"}} + pv2 = (await preview(client, pending["id"], body2)).json() + assert pv2["ok"], pv2 + + calls: list[str] = [] + orig = source_versions.table_versions_slot + + async def slot(fn, *a, **kw): + calls.append(getattr(fn, "__name__", str(fn))) + return await orig(fn, *a, **kw) + + monkeypatch.setattr(source_versions, "table_versions_slot", slot) + assert table_versions.acquire_store_guard() is False + + def refused(resp) -> None: + assert resp.status_code == 503, resp.text + assert resp.json()["code"] == "store_unavailable" + + version_body = {"confirm": True, "expected_current_snapshot_id": s3, "reason": "合成理由"} + refused(await client.post(f"{API}/{sid}/imports/{v['sep']}/remove", json=version_body)) + refused(await client.post(f"{API}/{sid}/imports/{v['sep']}/revoke-acceptance", json=version_body)) + refused(await client.post(f"{API}/imports/{applied['id']}/edits/undo", json={})) + refused(await apply(client, pending["id"], body2, pv2["recipe_sha256_after"])) + refused(await client.post(f"{API}/{sid}/redraft", json={})) + assert calls == [], "只读进程不该开始物化" + assert list((store / "uploads").rglob("*.tmp-*")) == [] + assert (await db_get(TableImport, v["sep"])).revoked is None + async with SessionLocal() as session: + assert (await session.get(DataSource, sid)).current_snapshot_id == s3 + assert len((await db_get(ImportStaging, applied["id"])).edits) == 1, "撤销没有生效" + + +async def test_read_only_process_still_deletes_a_manual_source(client, held_guard): + """手工登记的源没有导入记录、配方、暂存区和隔离文件，不碰版本存储：只读进程里照常删除（9.1 只要求上传源 503）。""" + sid = "manual-" + unique("m") + async with SessionLocal() as session: + session.add(DataSource(id=sid, name=unique("man"), kind="sqlite", database="/tmp/none.db", origin="manual")) + await session.commit() + assert table_versions.acquire_store_guard() is False + resp = await client.delete(f"{API}/{sid}") + assert resp.status_code == 204, resp.text + assert await db_get(DataSource, sid) is None + + +# ========================================================================== +# 修改请求、PUT、撤销 +# ========================================================================== + + +async def test_edit_request_shape_errors_are_edit_invalid(client): + name = unique() + base = await first_import(client, name, seed=511) + raw, fn = DRIFT_CASES["D04"].build(511) + st = await reupload(client, base["source_id"], raw, fn) + fx = next(f for f in st["fixes"] if f["kind"] == "remove_label") + url = f"{API}/imports/{st['id']}/edits/preview" + bad = [ + {}, + {"fix": {"id": fx["id"], "option": "remove"}, "selection": {"sheet": "客流汇总", "ref": "B5", "as": "list"}}, + {"fix": {"id": fx["id"]}}, + {"fix": {"id": fx["id"], "option": "nope"}}, + {"selection": {"sheet": "客流汇总", "ref": "b5:c6", "as": "list"}}, + {"selection": {"sheet": "客流汇总", "ref": "B5", "as": "picture"}}, + {"selection": {"sheet": "没有这张表", "ref": "B5:C6", "as": "list"}}, + ] + for body in bad: + resp = await client.post(url, json=body) + assert resp.status_code == 422 and resp.json()["code"] == "edit_invalid", (body, resp.text) + resp = await client.post(url, json={"fix": {"id": "fx-000000000000", "option": "remove"}}) + assert resp.status_code == 409 and resp.json()["code"] == "fix_stale" + resp = await client.post(f"{API}/imports/{st['id']}/edits/apply", + json={"fix": {"id": fx["id"], "option": "remove"}, "expected_sha256": "0" * 64}) + assert resp.status_code == 409 and resp.json()["code"] == "edit_stale" + resp = await client.post(f"{API}/imports/nope/edits/preview", json={"fix": {"id": fx["id"], "option": "remove"}}) + assert resp.status_code == 404 and resp.json()["code"] == "staging_not_found" + + +async def test_put_origin_and_redraft_mismatch_and_undo_stale(client): + name = unique() + base = await first_import(client, name, seed=512) + raw, fn = DRIFT_CASES["D04"].build(512) + st = await reupload(client, base["source_id"], raw, fn) + url = f"{API}/imports/{st['id']}/recipe" + resp = await client.put(url, json={"recipe": st["recipe"], "origin": "ai"}) + assert resp.status_code == 422 and resp.json()["code"] == "body_invalid" + resp = await client.put(url, json={"recipe": st["recipe"], "origin": "rules_redraft"}) + assert resp.status_code == 409 and resp.json()["code"] == "redraft_mismatch", "没有重新起草过就不能按采用记" + fx = next(f for f in st["fixes"] if f["kind"] == "remove_label") + body = {"fix": {"id": fx["id"], "option": "remove"}} + pv = (await client.post(f"{API}/imports/{st['id']}/edits/preview", json=body)).json() + resp = await client.post(f"{API}/imports/{st['id']}/edits/apply", + json={**body, "expected_sha256": pv["recipe_sha256_after"]}) + assert resp.status_code == 200, resp.text + # 起点在修改之后又被改过（正常流程不会出现：PUT 会清空撤销栈），撤销回 undo_stale 兜底 + async with SessionLocal() as session: + row = await session.get(ImportStaging, st["id"]) + changed = dict(row.answers_base) + changed["tables"] = [{**t, "note": "合成说明"} if i == 0 else t for i, t in enumerate(changed["tables"])] + row.answers_base = changed + await session.commit() + resp = await client.post(f"{API}/imports/{st['id']}/edits/undo", json={}) + assert resp.status_code == 409 and resp.json()["code"] == "undo_stale" + + +# ========================================================================== +# 按期累积：被改过、不在了的各期，被改过的并集试运行文件 +# ========================================================================== + + +async def accumulated(client, *, seed: int) -> dict[str, str]: + name = unique() + base = await first_import(client, name, seed=seed) + raw9, fn9 = flow_workbook(SEP, 30, seed=seed + 1) + st = await reupload(client, base["source_id"], raw9, fn9) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + return {"name": name, "source_id": base["source_id"], "aug": base["commit"]["import_id"], + "sep": resp.json()["import_id"], "s2": resp.json()["snapshot_id"]} + + +async def test_tampered_period_column_rejects_the_trial_with_part_tampered(client): + """某一期导入记录的 period_start 库列被改：统计期取自导入清单，对不上 → 试运行拒收（11.4 数据变异）。""" + v = await accumulated(client, seed=513) + async with SessionLocal() as session: + await session.execute(update(TableImport).where(TableImport.id == v["aug"]).values(period_start="2026-08-02")) + await session.commit() + raw, fn = flow_workbook(OCT, 31, seed=515) + st = await reupload(client, v["source_id"], raw, fn) + assert st["trial"]["status"] == "rejected" + tampered = next(p for p in st["trial"]["problems"] if p["code"] == "part_tampered") + assert tampered["category"] == "structure" and "统计期" in tampered["message"] + + +async def test_tampered_part_file_rejects_and_missing_part_file_is_a_409(client): + """已发布的某一期构建库被改字节 → 试运行拒收 part_tampered；构建库不在了 → 409 part_missing，消息指向「版本」。""" + v = await accumulated(client, seed=516) + imp = await db_get(TableImport, v["aug"]) + build = await db_get(TableBuild, imp.build_id) + path = Path(build.db_path) + original = path.read_bytes() + corrupt(path, original[:-1] + bytes([original[-1] ^ 1])) + raw, fn = flow_workbook(OCT, 31, seed=518) + st = await reupload(client, v["source_id"], raw, fn) + assert st["trial"]["status"] == "rejected" + assert "part_tampered" in {p["code"] for p in st["trial"]["problems"]} + path.unlink() + resp = await client.post(f"{API}/imports/{st['id']}/trial", json={}) + assert resp.status_code == 409 and resp.json()["code"] == "part_missing" + assert "版本" in resp.json()["detail"] + + +async def test_tampered_or_missing_union_trial_file_is_refused_at_commit(client): + name = unique() + base = await first_import(client, name, seed=519) + raw9, fn9 = flow_workbook(SEP, 30, seed=520) + st = await reupload(client, base["source_id"], raw9, fn9) + union = table_versions.union_trial_path(Path((await db_get(ImportStaging, st["id"])).trial_path)) + with open(union, "ab") as f: + f.write(b"\0") + resp = await commit(client, st) + assert resp.status_code == 409 and resp.json()["code"] == "union_tampered" + st = await trial(client, st["id"]) + union = table_versions.union_trial_path(Path((await db_get(ImportStaging, st["id"])).trial_path)) + union.unlink() + resp = await commit(client, st) + assert resp.status_code == 409 and resp.json()["code"] == "trial_required" + assert (await db_get(ImportStaging, st["id"])).status == "drafting" + + +# ========================================================================== +# 清除原件（7.6）的机读码 +# ========================================================================== + + +async def test_purge_raw_errors_carry_codes(client): + name, raw = unique(), sheet_bytes(521) + first = (await upload(client, name, raw)).json() + sid, iid = first["source"]["id"], first["import_id"] + url = f"{API}/{sid}/imports/{iid}/purge-raw" + resp = await client.post(url, json={"reason": "x", "confirm": False}) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" + resp = await client.post(url, json={}) + assert resp.status_code == 422 and resp.json()["code"] == "reason_required" + resp = await client.post(url, content=b"[1]", headers={"content-type": "application/json"}) + assert resp.status_code == 422 and resp.json()["code"] == "body_invalid" + resp = await client.post(f"{API}/{sid}/imports/nope/purge-raw", json={"reason": "x"}) + assert resp.status_code == 404 and resp.json()["code"] == "import_not_found" + resp = await client.post(url, json={"reason": "合成理由", "confirm": True}, headers={"X-Actor": "header-actor"}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["import"]["purged"]["signed_by"] == "header-actor" and out["import"]["raw_state"] == "purged" + resp = await client.post(url, json={"reason": "再来"}) + assert resp.status_code == 409 and resp.json()["code"] == "raw_already_purged" + async with SessionLocal() as session: + await session.execute(update(TableImport).where(TableImport.id == iid).values(raw_state="absent")) + await session.commit() + resp = await client.post(url, json={"reason": "x"}) + assert resp.status_code == 409 and resp.json()["code"] == "raw_absent" diff --git a/backend/tests/test_recipe_acceptance.py b/backend/tests/test_recipe_acceptance.py index 8e7663a7..9bea8fc4 100644 --- a/backend/tests/test_recipe_acceptance.py +++ b/backend/tests/test_recipe_acceptance.py @@ -238,7 +238,9 @@ async def flow_trial(client, seed: int = SEED, name: str | None = None) -> dict[ """ raw, fn = flow_workbook(AUG, 31, seed=seed) staged = await stage(client, raw, fn, name) - answers = {q["id"]: {"value": "register" if q["id"].startswith("q_relation") else "null"} + # 期 3：q_mode（每期怎么更新）答「每期替换」，按参考配方比哈希（P3-SPEC 12.0、1.5） + answers = {q["id"]: {"value": "register" if q["id"].startswith("q_relation") + else "replace" if q["id"] == "q_mode" else "null"} for q in staged["questions"]} after = await answer(client, staged["id"], answers) wide = next(t["name"] for t in after["recipe"]["tables"] if t["name"].endswith("_按日")) diff --git a/backend/tests/test_recipe_acceptance_p3.py b/backend/tests/test_recipe_acceptance_p3.py new file mode 100644 index 00000000..a502564c --- /dev/null +++ b/backend/tests/test_recipe_acceptance_p3.py @@ -0,0 +1,2007 @@ +"""期 3 集成验收（WP-8，P3-SPEC 11.2、11.4、12.9）：真管线（default_pipeline）经接口走完整流程。 + +不用假 Pipeline：暂存、规则起草、回答问题、修复按钮、框选、试运行、累积计划、物化、确认项、提交、版本页的启用 / +移除 / 作废 / 清除，全部是各工作包合并后的真代码；接口一律经 AsyncClient(transport=ASGITransport(app=app))，从暂存 +开始，不直接调内部函数拼结果。只有接口不提供的观察点（快照记录、导入记录、构建记录、运行记录、文件）才直接读库、 +看文件；造「数据目录被人改过」这类状态时才直接改库或改文件（11.4 的数据变异）。 + +公共做法（11.2）：D00 8 月暂存，回答问题（系统发现一律登记、占位符存为空值），按期 2 验收的写法把宽表改名「日客流」， +q_mode 选「按期累积」，提交得到 S1。之后各例按「上传新一期」重放。 + +夹具全部合成、假名（tests/fixtures/xlsx：分区甲、分区乙、分区丙……，数字随机）；不调用任何模型（期 3 没有新的模型 +调用，规则起草不完整时的 AI 可用性查询换成「不可用」的桩）。断言照规格写，不放宽、不标 xfail。 +""" +from __future__ import annotations + +import json +import os +import re +import sqlite3 +import subprocess +import sys +import time +import uuid +from contextlib import closing +from pathlib import Path +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from openpyxl import Workbook +from sqlalchemy import select, update + +from app.core import artifact_store +from app.core.config import settings +from app.data import raw_store, table_versions +from app.db.base import SessionLocal +from app.db.models import ( + DataSource, ImportStaging, Run, SnapshotActivation, SourceSnapshot, TableBuild, TableImport, TableRecipe, +) +from app.main import app +from tests.fixtures.xlsx import excel_saved, lab, save +from tests.fixtures.xlsx.drift import AUG, DRIFT_CASES, OCT, SEP +from tests.fixtures.xlsx.flow import SHEET, flow_workbook + +XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +API = "/api/datasources" +TABLES = ("日客流", "时段客流", "时段客流_表内合计") +#: 2.12 的验收数字：8 月 31 天、9 月 30 天，并集 61 / 1037 / 183 +AUG_ROWS = {"日客流": 31, "时段客流": 527, "时段客流_表内合计": 93} +SEP_ROWS = {"日客流": 30, "时段客流": 510, "时段客流_表内合计": 90} +UNION_ROWS = {"日客流": 61, "时段客流": 1037, "时段客流_表内合计": 183} +REASON = "合成理由：验收用例" + + +# ========================================================================== +# 夹具 +# ========================================================================== + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + """每个测试一个独立的数据目录：原件存档、构建库、试运行库、并集、隔离区、工件互不串。""" + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +@pytest.fixture(autouse=True) +def real_pipeline_no_model(monkeypatch): + """验收必须跑真管线：recipe_imports.pipeline() 接的是各工作包的真函数（含期 3 新增的累积、物化、修复、框选）， + 不是别的测试留下的假实现。规则起草不完整时 StagingOut 会查 AI 可用性：换成「不可用」的桩，连模型对象都不建。""" + from app.data import ( + recipe_accumulate, recipe_ai, recipe_engine, recipe_fixes, recipe_imports, recipe_notes, recipe_select, + recipe_suggest, + ) + from app.data.recipe_types import AiAvailability + + p = recipe_imports.pipeline() + assert p.execute is recipe_engine.execute and p.draft is recipe_suggest.draft + assert p.ai_availability is recipe_ai.ai_availability + assert p.plan_accumulate is recipe_accumulate.plan_accumulate + assert p.materialize_union is recipe_accumulate.materialize_union + assert p.build_union_notes is recipe_notes.build_union_notes + assert p.propose_fixes is recipe_fixes.propose_fixes and p.selection_edit is recipe_select.selection_edit + + async def no_model(session): + return None, "", AiAvailability(False, reason="验收测试不接模型") + + monkeypatch.setattr(recipe_ai, "resolve_draft_model", no_model) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +# ========================================================================== +# 小工具（经 HTTP） +# ========================================================================== + + +def unique(prefix: str = "wp8") -> str: + return f"{prefix}_{uuid.uuid4().hex[:10]}" + + +async def db_get(model, key): + async with SessionLocal() as session: + return await session.get(model, key) + + +async def stage(client, raw: bytes, filename: str, name: str | None = None) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/stage", files={"file": (filename, raw, XLSX)}, + data={"name": name or unique()}) + assert resp.status_code == 201, resp.text + return resp.json() + + +async def reupload(client, source_id: str, raw: bytes, filename: str) -> dict[str, Any]: + resp = await client.post(f"{API}/{source_id}/reupload", files={"file": (filename, raw, XLSX)}) + assert resp.status_code == 201, resp.text + return resp.json() + + +async def trial(client, sid: str, **body: Any) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/{sid}/trial", json=body) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def answer(client, sid: str, answers: dict[str, dict[str, Any]]) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/{sid}/answers", json={"answers": answers}) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def put_recipe(client, sid: str, rec: dict[str, Any], **extra: Any) -> dict[str, Any]: + resp = await client.put(f"{API}/imports/{sid}/recipe", json={"recipe": rec, **extra}) + assert resp.status_code == 200, resp.text + return resp.json() + + +def confirm_ids(st: dict[str, Any]) -> list[str]: + return [i["id"] for i in (st.get("trial") or {}).get("confirm_items") or []] + + +async def commit(client, st: dict[str, Any], *, confirmations: list[str] | None = None, + acceptances: list[dict[str, str]] | None = None): + """提交；返回原始响应（调用方自己判状态码）。confirmations 缺省 = 勾上全部确认项。""" + body: dict[str, Any] = {"trial_id": st["trial"]["trial_id"], + "confirmations": confirm_ids(st) if confirmations is None else confirmations} + if acceptances is not None: + body["acceptances"] = acceptances + return await client.post(f"{API}/imports/{st['id']}/commit", json=body) + + +def rows_of(st: dict[str, Any]) -> dict[str, int]: + return {t["name"]: t["rows"] for t in st["trial"]["receipt"]["tables"]} + + +def problems(st: dict[str, Any], code: str | None = None) -> list[dict[str, Any]]: + return [p for p in (st.get("trial") or {}).get("problems") or [] if code is None or p["code"] == code] + + +def check(st: dict[str, Any], cid: str) -> dict[str, Any] | None: + return next((c for c in st["trial"]["checks"] if c["id"] == cid), None) + + +def all_comments(notes: dict[str, Any] | None) -> str: + out = [] + for n in (notes or {}).values(): + out.append(n.get("comment") or "") + out += list((n.get("columns") or {}).values()) + return "\n".join(out) + + +def rename_wide(rec: dict[str, Any], old: str, new: str) -> dict[str, Any]: + """把规则起草的宽表名改成 new：表名、写入它的分段（id 一起改）、关系里引用的表名（期 2 验收 9.7 第 1 步）。""" + rec = json.loads(json.dumps(rec, ensure_ascii=False)) + for sheet in rec["sheets"]: + for block in sheet["blocks"]: + for seg in block.get("segments", []): + if seg.get("table") == old: + seg["table"] = new + if seg.get("id") == old: + seg["id"] = new + for t in rec["tables"]: + if t["name"] == old: + t["name"] = new + for r in rec["relations"]: + if r.get("table") == old: + r["table"] = new + for side in ("a", "b"): + if isinstance(r.get(side), dict) and r[side].get("table") == old: + r[side]["table"] = new + return rec + + +def answers_for(questions: list[dict[str, Any]], mode: str) -> dict[str, dict[str, Any]]: + """系统发现一律「登记」，q_mode 按 mode，其余能存为空值的选「存为空值」（参考配方的写法）。""" + out = {} + for q in questions: + values = [o["value"] for o in q["options"]] + if q["id"].startswith("q_relation"): + out[q["id"]] = {"value": "register"} + elif q["id"] == "q_mode": + out[q["id"]] = {"value": mode} + elif "null" in values: + out[q["id"]] = {"value": "null"} + else: + out[q["id"]] = {"value": values[0]} + return out + + +async def first_import(client, *, mode: str = "accumulate", seed: int = 0, start=AUG, days: int = 31, + variant: str = "D00", name: str | None = None, raw: bytes | None = None, + edit: Any = None) -> dict[str, Any]: + """公共做法：暂存 → 回答问题（q_mode=mode）→ 宽表改名「日客流」→ 试运行 → 勾全部确认项提交，得到 S1。 + raw 给了就用它（文件名照 flow_workbook 的写法）；edit 给了就在改名之后再改一次配方（A18 的列改名）。""" + name = name or unique() + made, fn = flow_workbook(start, days, seed=seed, variant=variant) + raw = raw if raw is not None else made + st = await stage(client, raw, fn, name) + assert st["questions"][-1]["id"] == "q_mode", "q_mode 固定排在问题列表最后（2.2）" + st = await answer(client, st["id"], answers_for(st["questions"], mode)) + wide = next(t["name"] for t in st["recipe"]["tables"] if t["name"] not in TABLES) + rec = rename_wide(st["recipe"], wide, "日客流") + st = await put_recipe(client, st["id"], edit(rec) if edit is not None else rec) + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + return {"name": name, "source_id": out["source"]["id"], "s1": out["snapshot_id"], "import_id": out["import_id"], + "recipe_id": out["recipe_id"], "commit": out, "trial": st, "raw": raw, "file_name": fn} + + +async def add_period(client, source_id: str, raw: bytes, fn: str, *, + acceptances: list[dict[str, str]] | None = None) -> dict[str, Any]: + """上传新一期 → 试运行通过 → 勾全部确认项提交。返回提交响应。""" + st = await reupload(client, source_id, raw, fn) + assert st["trial"]["status"] in ("passed", "needs_decision"), st["trial"]["problems"] + resp = await commit(client, st, acceptances=acceptances) + assert resp.status_code == 201, resp.text + return resp.json() + + +async def august_september(client, *, seed: int = 0, name: str | None = None) -> dict[str, Any]: + """S1=[8 月]、S2=[8 月, 9 月]，同一个配方（按期累积）。""" + base = await first_import(client, seed=seed, name=name) + raw, fn = flow_workbook(SEP, 30, seed=seed + 1) + out = await add_period(client, base["source_id"], raw, fn) + return {**base, "s2": out["snapshot_id"], "aug": base["import_id"], "sep": out["import_id"], "sep_commit": out, + "sep_raw": raw, "sep_file": fn} + + +async def tool_rows(client, source_name: str, sql: str) -> list[list[Any]]: + """经工具接口查当前版本（模型看到的那一份）。""" + resp = await client.post(f"/api/tools/db_query__{source_name}/run", json={"args": {"sql": sql}}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["ok"], out + return json.loads(out["result"])["rows"] + + +async def count(client, name: str, table: str, where: str = "") -> int: + return int((await tool_rows(client, name, f'SELECT COUNT(*) FROM "{table}"{where}'))[0][0]) + + +async def by_month(client, name: str, table: str, where: str = "") -> list[list[Any]]: + return await tool_rows(client, name, f'SELECT substr("日期", 1, 7), COUNT(*) FROM "{table}"{where} ' + "GROUP BY 1 ORDER BY 1") + + +async def current_id(source_id: str) -> str | None: + return (await db_get(DataSource, source_id)).current_snapshot_id + + +async def current_periods(source_id: str) -> list[str]: + snap = await db_get(SourceSnapshot, await current_id(source_id)) + return [(await db_get(TableImport, i)).period_start for i in snap.imports] + + +async def snapshots(client, source_id: str) -> list[dict[str, Any]]: + resp = await client.get(f"{API}/{source_id}/snapshots") + assert resp.status_code == 200, resp.text + return resp.json() + + +async def imports_api(client, source_id: str) -> list[dict[str, Any]]: + resp = await client.get(f"{API}/{source_id}/imports") + assert resp.status_code == 200, resp.text + return resp.json() + + +def date_of(y: int, m: int, d: int): + import datetime as dt + + return dt.date(y, m, d) + + +def sql_file(path: str | Path, query: str, params: tuple[Any, ...] = ()) -> list[tuple[Any, ...]]: + """只读打开库文件查一句（不改文件，库哈希不受影响）。""" + with closing(sqlite3.connect(f"file:{path}?mode=ro", uri=True)) as conn: + return list(conn.execute(query, params)) + + +def columns_of(snap: SourceSnapshot, table: str) -> list[str]: + return [c["name"] for c in snap.schema_cache["tables"][table]["columns"]] + + +# --- 改造夹具（保留公式的缓存值） --- + + +def edit_cells(raw: bytes, edits: dict[str, Any], sheet: str = SHEET) -> bytes: + """在合成夹具上改几格（值为 None 即清空），其余照旧。openpyxl 存盘会丢掉公式的缓存值，这里先按原文件读出缓存值， + 存盘后再用 excel_saved 填回去：表内合计的公式照样有缓存，核对不受影响。""" + import io + + from openpyxl import load_workbook + + book = load_workbook(io.BytesIO(raw)) + values = load_workbook(io.BytesIO(raw), data_only=True) + ws, wv = book[sheet], values[sheet] + cached = {c.coordinate: wv[c.coordinate].value for row in ws.iter_rows() for c in row + if isinstance(c.value, str) and c.value.startswith("=")} + for ref, value in edits.items(): + ws[ref] = value + return excel_saved(save(book), sheet, cached) + + +def without_zone_b(raw: bytes) -> bytes: + """这一期的文件不再有「分区乙（人次）」那一行（第 7 行整行清空）：原表不再登记分区乙。""" + from openpyxl.utils import get_column_letter + + return edit_cells(raw, {f"{get_column_letter(c)}7": None for c in range(2, 40)}) + + +# --- 修复按钮、框选 --- + + +def fix_of(st: dict[str, Any], kind: str) -> dict[str, Any]: + found = [f for f in st["fixes"] if f["kind"] == kind] + assert found, (kind, [f["kind"] for f in st["fixes"]]) + return found[0] + + +async def preview(client, sid: str, body: dict[str, Any]): + return await client.post(f"{API}/imports/{sid}/edits/preview", json=body) + + +async def apply(client, sid: str, body: dict[str, Any], expected: str | None, **extra: Any): + return await client.post(f"{API}/imports/{sid}/edits/apply", json={**body, "expected_sha256": expected, **extra}) + + +async def fixed(client, st: dict[str, Any], kind: str, option: str | None = None, + reason: str | None = None) -> tuple[dict[str, Any], dict[str, Any]]: + """按提议预览再应用（3.1：预览 → 应用，客户端只发 fix_id、选项值和理由）。返回 (应用后的 StagingOut, 预览)。""" + fx = fix_of(st, kind) + opt = next(o for o in fx["options"] if option is None or o["value"] == option) + if opt["needs_reason"] and reason is None: + reason = REASON + body = {"fix": {"id": fx["id"], "option": opt["value"], **({"reason": reason} if reason else {})}} + resp = await preview(client, st["id"], body) + assert resp.status_code == 200, resp.text + pv = resp.json() + assert pv["ok"], pv["problems"] + resp = await apply(client, st["id"], body, pv["recipe_sha256_after"]) + assert resp.status_code == 200, resp.text + return resp.json(), pv + + +# ========================================================================== +# A1–A3：按期累积、重叠拒收、替换该期 +# ========================================================================== + + +async def test_a1_august_plus_september_accumulates_to_61_1037_183(client): + """A1：reupload D01 → 试运行 passed，accumulate.action=append，union.rows={61, 1037, 183}，U1–U3 passed → + 提交 201、parts=2 → 当前快照 imports=[8 月, 9 月]、库在 snapshots/ 下，三表 61 / 1037 / 183，按月 31 / 30。""" + base = await first_import(client, seed=11) + sid, name = base["source_id"], base["name"] + raw, fn = flow_workbook(SEP, 30, seed=12) + st = await reupload(client, sid, raw, fn) + t = st["trial"] + assert st["kind"] == "reupload" and t["status"] == "passed", t["problems"] + assert t["accumulate"]["action"] == "append" and t["accumulate"]["mode"] == "accumulate" + assert t["accumulate"]["union"]["rows"] == UNION_ROWS + assert rows_of(st) == SEP_ROWS, "本期的回执是 9 月这一期" + assert [(c["id"], c["kind"], c["status"]) for c in t["union_checks"]] == [ + ("U1", "union_rows", "passed"), ("U2", "union_pk", "passed"), ("U3", "union_period", "passed")] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["parts"] == 2 and out["unchanged"] is False + snap = await db_get(SourceSnapshot, out["snapshot_id"]) + imps = [await db_get(TableImport, i) for i in snap.imports] + assert snap.imports == [base["import_id"], out["import_id"]] + assert [(i.period_start, i.period_end) for i in imps] == [("2026-08-01", "2026-08-31"), ("2026-09-01", "2026-09-30")] + assert Path(snap.db_path).parent.name == "snapshots" and snap.mode == "accumulate" + for table, n in UNION_ROWS.items(): + assert await count(client, name, table) == n + assert sql_file(snap.db_path, f'SELECT COUNT(*) FROM "{table}"')[0][0] == n + assert await by_month(client, name, "日客流") == [["2026-08", 31], ["2026-09", 30]] + + +async def test_a2_partially_overlapping_period_is_rejected(client): + """A2：S2 之后 reupload flow_workbook(2026-08-15, 31) → 试运行 rejected，problems 有 period_overlap，消息里有两段 + 已有的统计期；commit 409 trial_not_passed；当前快照不变。""" + v = await august_september(client, seed=21) + raw, fn = flow_workbook(AUG.replace(day=15), 31, seed=23) + st = await reupload(client, v["source_id"], raw, fn) + t = st["trial"] + assert t["status"] == "rejected" + [overlap] = problems(st, "period_overlap") + assert overlap["category"] == "structure" and overlap["fix"] is None + assert "2026-08-15 至 2026-09-14" in overlap["message"] + assert "2026-08-01 至 2026-08-31" in overlap["message"] and "2026-09-01 至 2026-09-30" in overlap["message"] + resp = await commit(client, st) + assert resp.status_code == 409 and resp.json()["code"] == "trial_not_passed", resp.text + assert await current_id(v["source_id"]) == v["s2"] + + +async def test_a3_replacing_a_period_needs_confirmation(client): + """A3：reupload 9 月 seed 不同（数字不同、统计期相同）→ passed、action=replace_period；不勾 + period_replace:2026-09-01~2026-09-30 提交 422 confirm_required（detail 里有这个 id）；勾上 201 → 新快照 + imports=[8 月, 9 月新]，行数仍是 61 / 1037 / 183，9 月的值与原来的不同。""" + v = await august_september(client, seed=31) + sid, name = v["source_id"], v["name"] + probe = 'SELECT "全日客流" FROM "日客流" WHERE "日期" = \'2026-09-05\'' + old = (await tool_rows(client, name, probe))[0][0] + raw, fn = flow_workbook(SEP, 30, seed=99) + st = await reupload(client, sid, raw, fn) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert t["accumulate"]["action"] == "replace_period" + item = "period_replace:2026-09-01~2026-09-30" + assert item in confirm_ids(st) + resp = await commit(client, st, confirmations=[i for i in confirm_ids(st) if i != item]) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required", resp.text + assert item in resp.json()["detail"] + assert await current_id(sid) == v["s2"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + snap = await db_get(SourceSnapshot, out["snapshot_id"]) + assert snap.imports == [v["aug"], out["import_id"]] and out["import_id"] != v["sep"] + for table, n in UNION_ROWS.items(): + assert await count(client, name, table) == n + assert (await tool_rows(client, name, probe))[0][0] != old + assert (await db_get(TableImport, v["sep"])).status == "superseded" + # 回到替换之前的版本：被替换掉的旧 9 月重新是 active，新 9 月置 superseded，数字回到原来的 + resp = await client.post(f"{API}/{sid}/snapshots/{v['s2']}/activate", + json={"confirm": True, "expected_current_snapshot_id": out["snapshot_id"]}) + assert resp.status_code == 200, resp.text + assert (await db_get(TableImport, v["sep"])).status == "active" + assert (await db_get(TableImport, out["import_id"])).status == "superseded" + assert (await tool_rows(client, name, probe))[0][0] == old + + +# ========================================================================== +# A4：固定在 8 月快照上的运行不受影响；被运行引用的并集快照不回收 +# ========================================================================== + + +async def keep_run(source_id: str, name: str, snapshot: str) -> str: + """登记一条引用这个快照的中断运行（写法同 test_run_data_versions.keep）。返回运行 id。""" + async with SessionLocal() as session: + run = Run(workflow_name="验收：保留旧版本", status="interrupted", + data_versions={source_id: {"snapshot": snapshot, "name": name}}) + session.add(run) + await session.commit() + return run.id + + +async def tools_for(names: list[str], *, data_versions: Any = None) -> dict[str, Any]: + from app.tools.datasource import build_datasource_tools + from app.tools.registry import ToolContext, Versions + + async with SessionLocal() as session: + built = await build_datasource_tools( + names, ToolContext(run_id="", node_id="n", + data_versions=Versions.CURRENT if data_versions is None else data_versions), session) + return {t.name: t for t in built} + + +async def restart_with_unit_change(client, source_id: str, start=OCT, days: int = 31, seed: int = 0) -> dict[str, Any]: + """这一期的文件把「分区甲（人次）」写成「分区甲（人）」，在这一期的暂存区里用 PUT 把配方的标签和单位一起改掉 + （单位由标签写死，unit_label_conflict 不许单改配方里的单位）：单位变了是不兼容变化 → restart。返回暂存区。""" + raw, fn = flow_workbook(start, days, seed=seed) + st = await reupload(client, source_id, edit_cells(raw, {"B6": "分区甲（人）"}), fn) + assert st["trial"]["status"] == "rejected", "现行配方认不出改了单位的标签" + rec = json.loads(json.dumps(st["recipe"], ensure_ascii=False)) + seg = next(s for s in rec["sheets"][0]["blocks"][0]["segments"] if s["id"] == "日客流") + seg["labels"]["expect"] = ["分区甲（人）" if x == "分区甲（人次）" else x for x in seg["labels"]["expect"]] + seg["measures"] = {("分区甲（人）" if k == "分区甲（人次）" else k): v for k, v in seg["measures"].items()} + next(t for t in rec["tables"] if t["name"] == "日客流")["units"]["分区甲"] = "人" + st = await put_recipe(client, st["id"], rec) + assert st["recipe_problems"] == [], st["recipe_problems"] + return await trial(client, st["id"]) + + +async def test_a4_runs_pinned_to_august_keep_reading_august(client): + """A4 前半：在 S1 上建运行记录（data_versions 固定 S1），之后提交 9 月 → 固定 S1 的查询工具查日客流是 31 行， + 冻结的表结构与 S1 的一致；不固定的调用查到 61。""" + from app.data.engine import engines + + base = await first_import(client, seed=41) + sid, name, s1 = base["source_id"], base["name"], base["s1"] + await keep_run(sid, name, s1) + raw, fn = flow_workbook(SEP, 30, seed=42) + out = await add_period(client, sid, raw, fn) + assert out["snapshot_id"] != s1 and out["parts"] == 2 + pins = {sid: {"snapshot": s1, "name": name}} + try: + query = (await tools_for([f"db_query__{name}"], data_versions=pins))[f"db_query__{name}"] + text = await query.coroutine(sql='SELECT COUNT(*) FROM "日客流"') + payload = json.loads(text) + assert payload["rows"][0][0] == 31 + assert artifact_store.load(payload["artifact"])["data_version"] == s1 + frozen = artifact_store.load(payload["schema_artifact"]) + s1_row = await db_get(SourceSnapshot, s1) + assert set(frozen["tables"]) == set(s1_row.schema_cache["tables"]) == set(TABLES) + assert frozen["tables"]["日客流"]["columns"] == s1_row.schema_cache["tables"]["日客流"]["columns"] + current = (await tools_for([f"db_query__{name}"]))[f"db_query__{name}"] + text = await current.coroutine(sql='SELECT COUNT(*) FROM "日客流"') + assert json.loads(text)["rows"][0][0] == 61 + assert artifact_store.load(json.loads(text)["artifact"])["data_version"] == out["snapshot_id"] + finally: + await engines.invalidate(sid) + + +async def test_a4_a_union_snapshot_referenced_by_a_run_survives_gc(client, monkeypatch): + """A4 后半（保护规则单独验证，评审一-m8；1.5：KEEP 一律用 0）：在 S2（并集）上建运行记录，再走 restart 得到 S3； + 回收之后 S2 的并集文件仍在、记录没有 retired；删掉那条运行记录再回收，S2 被回收、并集文件删掉。""" + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0) + v = await august_september(client, seed=43) + sid = v["source_id"] + run_id = await keep_run(sid, v["name"], v["s2"]) + st = await restart_with_unit_change(client, sid, seed=44) + assert st["trial"]["accumulate"]["action"] == "restart" + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + s3 = resp.json()["snapshot_id"] + s2 = await db_get(SourceSnapshot, v["s2"]) + union_file = Path(s2.db_path) + async with SessionLocal() as session: + await table_versions.gc(session) + assert union_file.is_file() and (await db_get(SourceSnapshot, v["s2"])).retired_at is None + assert (await db_get(SourceSnapshot, v["s1"])).retired_at is not None, "KEEP=0：没人引用的 S1 照期 1 回收" + async with SessionLocal() as session: + await session.delete(await session.get(Run, run_id)) + await session.commit() + await table_versions.gc(session) + assert (await db_get(SourceSnapshot, v["s2"])).retired_at is not None + assert not union_file.exists() + assert await current_id(sid) == s3 + + +# ========================================================================== +# A5：全角或 en-dash 的标签两个月都能查到 +# ========================================================================== + + +@pytest.mark.parametrize("variant", ["D24", "D24w"]) +async def test_a5_dash_and_fullwidth_labels_are_queryable_across_months(client, variant): + """A5：9 月用 en-dash（D24）或全角数字与全角减号（D24w）写时段标签：试运行差异卡有 label_writing、需确认；提交后 + WHERE "时段"='8-9' 是 61，按月 31 / 30（维度值按规范写法存储）。""" + base = await first_import(client, seed=51) + sid, name = base["source_id"], base["name"] + raw, fn = flow_workbook(SEP, 30, seed=52, variant=variant) + st = await reupload(client, sid, raw, fn) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + writing = [d for d in t["diff"] if d["kind"] == "label_writing"] + assert writing and all(d["requires_confirm"] for d in writing), t["diff"] + assert {"diff:label_writing:日间", "diff:label_writing:夜间"} <= set(confirm_ids(st)) + assert t["accumulate"]["action"] == "append" and t["accumulate"]["union"]["rows"] == UNION_ROWS + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + where = ' WHERE "时段" = \'8-9\'' + assert await count(client, name, "时段客流", where) == 61 + assert await by_month(client, name, "时段客流", where) == [["2026-08", 31], ["2026-09", 30]] + labels = {r[0] for r in await tool_rows(client, name, 'SELECT DISTINCT "时段" FROM "时段客流"')} + assert all(re.fullmatch(r"\d{1,2}-\d{1,2}", x) for x in labels), labels + + +# ========================================================================== +# A6、A6b、A6c、A15：拒收的漂移走修复按钮后能试运行并启用（3.4） +# ========================================================================== + +#: 3.4 表与「其他两例」：拒收的问题、fix、坐标（None = 无坐标）、修复步骤 [(kind, 选项)]、修复后要勾的确认项、 +#: 累积后的行数 +FIX_CASES: dict[str, dict[str, Any]] = { + "D04": {"code": "label_missing", "fix": "remove_label", "cell": f"{SHEET}!B10", + "steps": [("remove_label", "remove")], "confirms": {"fix:remove_label:日间:7-8"}, + "rows": {"日客流": 61, "时段客流": 1007, "时段客流_表内合计": 183}}, + "D09": {"code": "title_not_found", "fix": "rename_title", "cell": None, + "steps": [("rename_title", "|keep")], "confirms": {"fix:rename_title:日间"}, "rows": UNION_ROWS}, + "D13": {"code": "label_unexpected", "fix": "add_label", "cell": f"{SHEET}!B8", + "steps": [("add_label", "add"), ("edit_members", "update")], + "confirms": {"unit:日客流.分区丙", "relation:R1"}, "fix_items": 2, "rows": UNION_ROWS}, + "D18": {"code": "label_unparsed", "fix": "declare_total", "cell": f"{SHEET}!B21", + "steps": [("declare_total", "keep")], "confirms": {"fix:declare_total:日间", "derived:日间合计"}, + "accept": "K1", "rows": {"日客流": 61, "时段客流": 1037, "时段客流_表内合计": 213}}, + "D16": {"code": "axis_extra_cells", "fix": "ignore_cells", "cell": None, + "steps": [("ignore_cells", None)], "confirms": set(), "rows": UNION_ROWS}, + "D21": {"code": "row_unclaimed", "fix": "ignore_cells", "cell": f"{SHEET}!B32", + "steps": [("ignore_cells", None)], "confirms": set(), "rows": UNION_ROWS}, +} + + +async def _fix_case(client, code: str, seed: int) -> dict[str, Any]: + case = FIX_CASES[code] + base = await first_import(client, seed=seed) + raw, fn = DRIFT_CASES[code].build(seed + 1) + st = await reupload(client, base["source_id"], raw, fn) + assert st["trial"]["status"] == "rejected" + probs = problems(st, case["code"]) + assert probs and {p["fix"] for p in probs} == {case["fix"]}, [(p["code"], p["fix"]) for p in problems(st)] + if case["cell"] is not None: + assert case["cell"] in {c for p in probs for c in p["cells"]}, [p["cells"] for p in probs] + # 问题旁边的按钮：提议挂在对应的那条问题上（fix_ids，评审三-B2） + fx = fix_of(st, case["fix"]) + assert any(fx["id"] in p["fix_ids"] for p in probs), (fx["id"], [p["fix_ids"] for p in probs]) + for kind, option in case["steps"]: + fx = fix_of(st, kind) + value = next(o["value"] for o in fx["options"] if option is None or o["value"].endswith(option)) + st, pv = await fixed(client, st, kind, value) + assert pv["recipe_sha256_after"] == st["recipe_sha256"] + assert st["recipe_problems"] == [], st["recipe_problems"] + st = await trial(client, st["id"]) + return {"base": base, "st": st, "case": case} + + +@pytest.mark.parametrize("code", ["D04", "D09", "D13", "D18"]) +async def test_a6_drift_fixed_with_the_fix_button_can_be_enabled(client, code): + """A6：reupload → rejected，问题的 fix 与 3.4 表一致 → StagingOut.fixes 有对应提议 → edits/preview → edits/apply + （D13 接着应用 edit_members）→ trial（D18 写理由接受 K1）→ 勾全部确认项（含 fix:*）提交 201 → 并集行数与 3.4 一致。""" + done = await _fix_case(client, code, seed=60 + list(FIX_CASES).index(code)) + st, case, name = done["st"], done["case"], done["base"]["name"] + t = st["trial"] + acceptances = None + if case.get("accept"): + assert t["status"] == "needs_decision" and t["acceptable"] == [case["accept"]], (t["status"], t["acceptable"]) + k1 = check(st, case["accept"]) + # 3.4：K1（新的日间合计）unverifiable，原因 null_detail 30 格（7-8 行整行是占位符，明细缺数） + assert k1["status"] == "unverifiable" and k1["reasons"] == {"null_detail": 30} and k1["acceptable"] is True + assert {c["id"]: c["status"] for c in t["checks"] if c["id"] in ("K2", "G1", "G2")} == { + "K2": "passed", "G1": "passed", "G2": "passed"} + acceptances = [{"check_id": case["accept"], "reason": REASON}] + else: + assert t["status"] == "passed", (t["problems"], [c for c in t["checks"] if c["status"] != "passed"]) + assert t["accumulate"]["action"] == "append" and t["accumulate"]["union"]["rows"] == case["rows"] + ids = confirm_ids(st) + fixes = [i for i in ids if i.startswith("fix:")] + assert len(fixes) == case.get("fix_items", 1), fixes + assert case["confirms"] <= set(ids), (case["confirms"], ids) + assert not [i for i in ids if i.startswith(("breaking:", "accumulate_restart"))], ids + resp = await commit(client, st, confirmations=[i for i in ids if not i.startswith("fix:")], acceptances=acceptances) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required", "fix:* 是必勾项" + resp = await commit(client, st, acceptances=acceptances) + assert resp.status_code == 201, resp.text + assert resp.json()["parts"] == 2 + for table, n in case["rows"].items(): + assert await count(client, name, table) == n, table + if code == "D09": + kinds = await tool_rows(client, name, 'SELECT DISTINCT "时段类别" FROM "时段客流" ORDER BY 1') + assert sorted(k[0] for k in kinds) == ["夜间", "日间"] + if code == "D13": + assert await count(client, name, "日客流", ' WHERE "分区丙" IS NULL') == 31 + assert await by_month(client, name, "日客流", ' WHERE "分区丙" IS NULL') == [["2026-08", 31]] + snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"]) + assert "各期登记的分项列不同" in snap.schema_cache["tables"]["日客流"]["comment"] + + +@pytest.mark.parametrize("code", ["D16", "D21"]) +async def test_a6b_ignore_columns_and_rows_then_enable(client, code): + """A6b：D16 按表头忽略右侧的「合计」列（⑥ columns），D21 按行标签忽略「补录（人次）」（⑥ rows，要理由）；修复后 + 能启用，行数与 D01 相同（30 / 510 / 90，并集 61 / 1037 / 183）；D21 的回执 rows_excluded 有 ignored_rows（第 32 行）。""" + done = await _fix_case(client, code, seed=70 + (code == "D21")) + st, name = done["st"], done["base"]["name"] + t = st["trial"] + assert t["status"] == "passed", (t["problems"], [c for c in t["checks"] if c["status"] != "passed"]) + assert rows_of(st) == SEP_ROWS + excluded = t["receipt"]["rows_excluded"] + if code == "D21": + ignored = [x for x in excluded if x["reason"] == "ignored_rows"] + assert ignored and ignored[0]["rows"] == [[32, 32]] and ignored[0]["anchor"] == "补录（人次）", excluded + assert ignored[0]["cells"] == 3, "标签格加两格数（B32:D32）" + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + for table, n in UNION_ROWS.items(): + assert await count(client, name, table) == n + if code == "D21": + imp = await db_get(TableImport, resp.json()["import_id"]) + manifest = artifact_store.load(imp.manifest_artifact) + assert [x["reason"] for x in manifest["receipt"]["rows_excluded"]] == [x["reason"] for x in excluded] + # 后续每期按锚点忽略：锚点照旧的下一期直接通过；换了一个没见过的标签（列头）照样拒收，不会被这条规则吞掉 + from openpyxl.utils import get_column_letter + + raw, fn = flow_workbook(OCT, 31, seed=72, variant=code) + st = await reupload(client, done["base"]["source_id"], raw, fn) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + await client.delete(f"{API}/imports/{st['id']}") + other = {"B32": "备用分区（人次）"} if code == "D21" else {f"{get_column_letter(3 + 31)}4": "备注"} + st = await reupload(client, done["base"]["source_id"], edit_cells(raw, other), fn) + want = "row_unclaimed" if code == "D21" else "axis_extra_cells" + assert st["trial"]["status"] == "rejected" and problems(st, want), st["trial"]["problems"] + + +async def test_a6c_d04_with_a_failing_r1_day_is_fixed_then_accepted(client): + """A6c（评审一-M3）：D04 上再改一格分区甲，使 R1 有一天不成立 → 应用 ① 后 recipe_problems 为空（R1 的认领不查）→ + 试运行 R1 mismatch、needs_decision → 写理由接受后提交 201；说明用 sum_eq_union_mixed、不含「已逐日核对」。""" + base = await first_import(client, seed=81) + raw, fn = DRIFT_CASES["D04"].build(82) + raw = edit_cells(raw, {"L6": 1}) # 第 10 个日期（L 列）的分区甲改成 1：全日 ≠ 甲 + 乙 + st = await reupload(client, base["source_id"], raw, fn) + assert st["trial"]["status"] == "rejected" and problems(st, "label_missing") + st, _ = await fixed(client, st, "remove_label", "remove") + assert st["recipe_problems"] == [], "改的是 dimension 段：R1 的认领按重放处理，不查（3.3 第 11 条）" + st = await trial(client, st["id"]) + t = st["trial"] + r1 = check(st, "R1") + assert t["status"] == "needs_decision" and r1["status"] == "mismatch" and t["acceptable"] == ["R1"], t["status"] + resp = await commit(client, st) + assert resp.status_code == 422 and resp.json()["code"] == "acceptance_required" + resp = await commit(client, st, acceptances=[{"check_id": "R1", "reason": REASON}]) + assert resp.status_code == 201, resp.text + comment = (await db_get(SourceSnapshot, resp.json()["snapshot_id"])).schema_cache["tables"]["日客流"]["comment"] + assert "在部分期的个别" in comment and "不成立或未能核对" in comment and "已逐日核对" not in comment, comment + + +async def test_a15_after_a_fix_only_changed_recipe_items_are_listed(client): + """A15：D04 修复之后的确认项里，配方类只有 fix:*（source=edit），没有 placeholder:·、unit:*、relation:* 等没有变化 + 的项（6.2：只列签名与现行配方不同的项）。""" + done = await _fix_case(client, "D04", seed=91) + st = done["st"] + items = {i["id"]: i for i in st["trial"]["confirm_items"]} + assert items["fix:remove_label:日间:7-8"]["source"] == "edit" + recipe_items = sorted(i for i, v in items.items() if v.get("source", "recipe") == "recipe") + assert recipe_items == [], recipe_items + for stale in ("placeholder:·", "unit:日客流.全日客流", "relation:R1", "derived:夜间合计", "mode", "year_from:交叉表"): + assert stale not in items, stale + + +# ========================================================================== +# A7：c01 框选回放一致（4.6 的三条） +# ========================================================================== + + +async def test_a7_c01_selection_replays_consistently(client): + """4.6：① 同一份文件：月报!C5:F8 框为列表 → ok，锚点是四个表头，重放与框一致，换算后的配方与规则草稿逐字相同； + ② 整张表挪到 E8（shift=(3, 2)）用同一份配方试运行：passed、T1 passed，列表表和合计表的 table_hashes 与未挪动的 + 逐一相同（配方里没有坐标）；③ 泛化不了时明确说明：C5:F7 → selection_bottom_not_anchorable（建议扩大到第 8 行）， + 同一框改选 bottom=auto → ok、data=C6:F8、重放一致；C6:F8 → selection_header_not_text；C5:G8 → selection_header_blank。""" + raw, fn = lab.c01("literal") + st = await stage(client, raw, fn) + blocks = [b for s in st["recipe"]["sheets"] for b in s["blocks"]] + assert [b["layout"] for b in blocks] == ["list"] + sel = {"sheet": "月报", "ref": "C5:F8", "as": "list", "options": {"header_rows": 1, "bottom": "box"}} + resp = await preview(client, st["id"], {"selection": sel}) + assert resp.status_code == 200, resp.text + pv = resp.json() + assert pv["ok"] is True, pv["problems"] + assert [a["text"] for a in pv["anchors"] if a["kind"] == "header"] == ["地区", "产品", "销量", "金额"] + box = {"header": "C5:F5", "data": "C6:F8", "total": "C9:F9"} + assert pv["replay"]["match"] is True and pv["replay"]["expected"] == box and pv["replay"]["actual"] == box + assert pv["replay"]["diffs"] == [] + assert pv["recipe_sha256_after"] == st["recipe_sha256"], "框选得到的规则与起草器完全一样" + resp = await apply(client, st["id"], {"selection": sel}, pv["recipe_sha256_after"]) + assert resp.status_code == 200, resp.text + st = await trial(client, resp.json()["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert check(st, "T1")["status"] == "passed" + hashes = st["trial"]["receipt"]["table_hashes"] + list_table = blocks[0]["table"] + total_table = blocks[0]["rows"]["total_row"]["keep_as"] + assert {list_table, total_table} <= set(hashes) + + # ② 挪了位置也回放一致：用第 1 条得到的配方（PUT） + raw2, fn2 = lab.c01("literal", shift=(3, 2)) + moved = await stage(client, raw2, fn2) + moved = await put_recipe(client, moved["id"], st["recipe"]) + assert moved["recipe_problems"] == [], moved["recipe_problems"] + moved = await trial(client, moved["id"]) + assert moved["trial"]["status"] == "passed", moved["trial"]["problems"] + assert check(moved, "T1")["status"] == "passed" + got = moved["trial"]["receipt"]["table_hashes"] + assert got[list_table] == hashes[list_table] and got[total_table] == hashes[total_table] + + # ③ 泛化不了时明确说明 + bad = (await preview(client, st["id"], {"selection": {**sel, "ref": "C5:F7"}})).json() + assert bad["ok"] is False and [p["code"] for p in bad["problems"]] == ["selection_bottom_not_anchorable"] + assert "第 8 行" in bad["problems"][0]["message"], bad["problems"][0]["message"] + auto = (await preview(client, st["id"], {"selection": {**sel, "ref": "C5:F7", + "options": {"header_rows": 1, "bottom": "auto"}}})).json() + assert auto["ok"] is True, auto["problems"] + assert auto["expected"]["data"] == "C6:F8" and auto["replay"]["match"] is True + bad = (await preview(client, st["id"], {"selection": {**sel, "ref": "C6:F8"}})).json() + assert bad["ok"] is False and "selection_header_not_text" in [p["code"] for p in bad["problems"]] + bad = (await preview(client, st["id"], {"selection": {**sel, "ref": "C5:G8"}})).json() + assert bad["ok"] is False and "selection_header_blank" in [p["code"] for p in bad["problems"]] + + +# ========================================================================== +# A8：回滚（启用旧版本） +# ========================================================================== + + +async def test_a8_rollback_to_an_earlier_version(client, monkeypatch): + """A8：在 S2（累积）上 activate S1（带 expected_current=S2）→ 当前快照是 S1，current_recipe_id 是 S1 的配方，导入记录 + 状态互换，写了 snapshot_activations；在 S2 上发起的固定运行仍查 61 行；confirm 缺省 422；expected_current 不对 409 + base_changed；遮罩列在目标版本里没有同名列时不带 ack_mask_lost 409 mask_lost，带上成功；activate 已回收的快照 + 409 snapshot_unavailable（KEEP=0 造出已回收的快照，1.5）。 + + 9 月用 D13 走修复按钮（加「分区丙」），S2 的配方 R2 与 S1 的 R0 不是同一条：启用前现行配方是 R2，启用 S1 之后是 R0， + 两条配方的状态互换。两期同一个配方时「现行配方跟着快照走」这一条测不出来（WP-8 评审意见 3）。""" + from app.data.engine import engines + + done = await _fix_case(client, "D13", seed=101) + assert done["st"]["trial"]["status"] == "passed", done["st"]["trial"]["problems"] + resp = await commit(client, done["st"]) + assert resp.status_code == 201, resp.text + base, sep = done["base"], resp.json() + v = {**base, "s2": sep["snapshot_id"], "aug": base["import_id"], "sep": sep["import_id"]} + sid, name = v["source_id"], v["name"] + r0, r2 = base["recipe_id"], sep["recipe_id"] + assert r0 != r2 and (await db_get(SourceSnapshot, v["s1"])).recipe_id == r0 + assert (await db_get(SourceSnapshot, v["s2"])).recipe_id == r2 + assert (await db_get(DataSource, sid)).current_recipe_id == r2 + assert ((await db_get(TableRecipe, r0)).status, (await db_get(TableRecipe, r2)).status) == ("superseded", "active") + await keep_run(sid, name, v["s2"]) + url = f"{API}/{sid}/snapshots/{v['s1']}/activate" + resp = await client.post(url, json={"expected_current_snapshot_id": v["s2"]}) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" + resp = await client.post(url, json={"confirm": True}) + assert resp.status_code == 422 and resp.json()["code"] == "expected_current_required" + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s1"]}) + assert resp.status_code == 409 and resp.json()["code"] == "base_changed" + async with SessionLocal() as session: + await session.execute(update(DataSource).where(DataSource.id == sid).values(options={"mask_columns": "分区丁"})) + await session.commit() + listed = {s["id"]: s for s in await snapshots(client, sid)} + assert listed[v["s1"]]["mask_lost"] == ["分区丁"] and listed[v["s1"]]["activatable"] is True + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s2"]}) + assert resp.status_code == 409 and resp.json()["code"] == "mask_lost" + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s2"], "reason": REASON, + "ack_mask_lost": ["分区丁"], "signed_by": "合成署名甲"}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["snapshot_id"] == v["s1"] and out["previous_snapshot_id"] == v["s2"] + s1 = await db_get(SourceSnapshot, v["s1"]) + src = await db_get(DataSource, sid) + assert src.current_snapshot_id == v["s1"] and src.current_recipe_id == s1.recipe_id == out["recipe_id"] == r0 + assert ((await db_get(TableRecipe, r0)).status, (await db_get(TableRecipe, r2)).status) == ("active", "superseded") + assert (await db_get(TableImport, v["aug"])).status == "active" + assert (await db_get(TableImport, v["sep"])).status == "superseded" + async with SessionLocal() as session: + last = (await session.execute(select(SnapshotActivation).where(SnapshotActivation.source_id == sid) + .order_by(SnapshotActivation.created_at.desc()))).scalars().first() + assert (last.kind, last.snapshot_id, last.previous_snapshot_id, last.signed_by) == ( + "activate", v["s1"], v["s2"], "合成署名甲") + assert await count(client, name, "日客流") == 31 + try: + pinned = (await tools_for([f"db_query__{name}"], data_versions={sid: {"snapshot": v["s2"], "name": name}})) + text = await pinned[f"db_query__{name}"].coroutine(sql='SELECT COUNT(*) FROM "日客流"') + assert json.loads(text)["rows"][0][0] == 61, "在 S2 上发起的固定运行仍查 S2" + finally: + await engines.invalidate(sid) + # 已回收的快照不能启用：KEEP=0 下再提交 10 月，S1（不是当前、没人引用）被回收 + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0) + raw, fn = flow_workbook(OCT, 31, seed=102) + s3 = (await add_period(client, sid, raw, fn))["snapshot_id"] + assert (await db_get(SourceSnapshot, v["s1"])).retired_at is not None + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": s3, + "ack_mask_lost": ["分区丁"]}) + assert resp.status_code == 409 and resp.json()["code"] == "snapshot_unavailable" + + +# ========================================================================== +# A9：作废接受（累积模式 = 移除该期） +# ========================================================================== + + +async def test_a9_revoking_an_acceptance_removes_the_period_and_keeps_recipe_and_mode(client): + """A9：D26（9 月 R1 不成立、写理由接受）累积提交 → 说明含 sum_eq_union_mixed 的片段、不含「已逐日核对」→ + revoke-acceptance → action=remove_period，新快照只有 8 月，快照 id 等于 S1（复用），current_recipe_id 和模式都没变， + 该导入 revoked 非空；再 activate 含已作废导入的快照 409 contains_revoked；remove 唯一一期 409 last_period。""" + base = await first_import(client, seed=111) + sid = base["source_id"] + raw, fn = DRIFT_CASES["D26"].build(112) + st = await reupload(client, sid, raw, fn) + assert st["trial"]["status"] == "needs_decision" and st["trial"]["acceptable"] == ["R1"] + resp = await commit(client, st, acceptances=[{"check_id": "R1", "reason": REASON}]) + assert resp.status_code == 201, resp.text + s2, sep = resp.json()["snapshot_id"], resp.json()["import_id"] + comment = (await db_get(SourceSnapshot, s2)).schema_cache["tables"]["日客流"]["comment"] + assert "在部分期的个别" in comment and "不成立或未能核对" in comment and "已逐日核对" not in comment, comment + recipe_before = (await db_get(DataSource, sid)).current_recipe_id + resp = await client.post(f"{API}/{sid}/imports/{sep}/revoke-acceptance", + json={"confirm": True, "expected_current_snapshot_id": s2, "reason": REASON}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["action"] == "remove_period" and out["snapshot_id"] == base["s1"] + snap = await db_get(SourceSnapshot, out["snapshot_id"]) + assert snap.imports == [base["import_id"]] and snap.mode == "accumulate" + assert (await db_get(DataSource, sid)).current_recipe_id == recipe_before + imp = await db_get(TableImport, sep) + assert imp.revoked and imp.revoked["reason"] == REASON and imp.status == "superseded" + resp = await client.post(f"{API}/{sid}/snapshots/{s2}/activate", + json={"confirm": True, "expected_current_snapshot_id": base["s1"]}) + assert resp.status_code == 409 and resp.json()["code"] == "contains_revoked" + resp = await client.post(f"{API}/{sid}/imports/{base['import_id']}/remove", + json={"confirm": True, "expected_current_snapshot_id": base["s1"], "reason": REASON}) + assert resp.status_code == 409 and resp.json()["code"] == "last_period" + + +# ========================================================================== +# A10：不兼容变更要重新开始累积 +# ========================================================================== + + +async def test_a10_an_incompatible_change_restarts_the_accumulation(client): + """A10：S2 之后 reupload 10 月，在 10 月的暂存区里用 PUT 把一列的单位改掉 → 试运行 action=restart，确认项有 + accumulate_restart；提交后当前快照只有 10 月，说明含 single_after_drop；S2 仍可启用（在 SNAPSHOT_KEEP 之内）。""" + v = await august_september(client, seed=121) + sid = v["source_id"] + # 不兼容与部分重叠同时出现：restart 优先（结果只有本期，不涉及重叠），重叠的各期只作提示（2.4，评审一-m12） + st = await restart_with_unit_change(client, sid, start=date_of(2026, 8, 15), days=31, seed=120) + assert st["trial"]["status"] == "passed" and st["trial"]["accumulate"]["action"] == "restart", st["trial"] + assert [p["start"] for p in st["trial"]["accumulate"]["overlaps"]] == ["2026-08-01", "2026-09-01"] + await client.delete(f"{API}/imports/{st['id']}") + st = await restart_with_unit_change(client, sid, seed=122) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + plan = t["accumulate"] + assert plan["action"] == "restart" and plan["change"] == "semantic" + assert [p["start"] for p in plan["dropped"]] == ["2026-08-01", "2026-09-01"] + assert any("人次" in x and "人" in x for x in plan["semantic"]), plan["semantic"] + assert "accumulate_restart" in confirm_ids(st) + resp = await commit(client, st, confirmations=[i for i in confirm_ids(st) if i != "accumulate_restart"]) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["parts"] == 1 + assert await current_periods(sid) == ["2026-10-01"] + comment = (await db_get(SourceSnapshot, out["snapshot_id"])).schema_cache["tables"]["日客流"]["comment"] + assert "当前版本只含最近一期的数据" in comment, comment + listed = {s["id"]: s for s in await snapshots(client, sid)} + assert listed[v["s2"]]["activatable"] is True and listed[v["s2"]]["available"] is True + + +# ========================================================================== +# A11：退役列保留 +# ========================================================================== + + +async def drop_zone_b(client, st: dict[str, Any]) -> dict[str, Any]: + """这一期的文件不再有分区乙：修复按钮去掉标签（①，连带去掉列和单位），R1（全日 = 甲 + 乙）删掉（③ 的「删除这条 + 关系」），R2 按本期的系统发现重新认领（本期只剩口径不同那一条，编号是 F1；改认领只能在配方面板里 PUT）。""" + st, _ = await fixed(client, st, "remove_label", "remove") + st, _ = await fixed(client, st, "edit_members", "remove") + rec = json.loads(json.dumps(st["recipe"], ensure_ascii=False)) + for r in rec["relations"]: + if r["id"] == "R2": + r["claims"] = "F1" + st = await put_recipe(client, st["id"], rec) + assert st["recipe_problems"] == [], st["recipe_problems"] + assert [r["id"] for r in st["recipe"]["relations"]] == ["R2"] + return await trial(client, st["id"]) + + +async def test_a11_a_retired_column_is_kept_and_not_retired_again(client): + """A11：去掉 measures 的分区乙（R1 不再登记）后累积 9 月 → 确认项有 retire:日客流.分区乙；并集里 8 月有值、9 月为空值； + 列说明含「自某一期起不再导入」，分区乙的单位仍是人次；日客流的说明用 sum_eq_union_partial，不含「已逐日核对」。 + 再累积 10 月（评审二-M3）：确认项里没有 retire:*，也没有 breaking:日客流；计划的 retired_existing 有分区乙。""" + base = await first_import(client, seed=131) + sid, name = base["source_id"], base["name"] + raw, fn = flow_workbook(SEP, 30, seed=132) + st = await reupload(client, sid, without_zone_b(raw), fn) + assert st["trial"]["status"] == "rejected" and problems(st, "label_missing") + st = await drop_zone_b(client, st) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert t["accumulate"]["action"] == "append" and t["accumulate"]["change"] == "retire" + assert t["accumulate"]["retired_new"] == [{"table": "日客流", "column": "分区乙"}] + assert "retire:日客流.分区乙" in confirm_ids(st) + assert not [i for i in confirm_ids(st) if i.startswith("breaking:日客流")], "退役与 breaking 同表去重（2.5）" + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"]) + assert await by_month(client, name, "日客流", ' WHERE "分区乙" IS NOT NULL') == [["2026-08", 31]] + assert await by_month(client, name, "日客流", ' WHERE "分区乙" IS NULL') == [["2026-09", 30]] + table = snap.schema_cache["tables"]["日客流"] + col = next(c for c in table["columns"] if c["name"] == "分区乙") + assert "单位：人次" in col["comment"] and "自某一期起不再导入" in col["comment"], col + assert "只在部分期登记并核对" in table["comment"] and "已逐日核对" not in table["comment"], table["comment"] + manifest = artifact_store.load(snap.manifest_artifact) + assert manifest["columns"]["日客流"]["分区乙"]["status"] == "retired" + + raw, fn = flow_workbook(OCT, 31, seed=133) + st = await reupload(client, sid, without_zone_b(raw), fn) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert t["accumulate"]["retired_new"] == [] + assert {"table": "日客流", "column": "分区乙"} in t["accumulate"]["retired_existing"] + assert not [i for i in confirm_ids(st) if i.startswith(("retire", "breaking:日客流"))], confirm_ids(st) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert await by_month(client, name, "日客流", ' WHERE "分区乙" IS NULL') == [["2026-09", 30], ["2026-10", 31]] + + +# ========================================================================== +# A12：模式切换 +# ========================================================================== + + +async def test_a12_switching_modes_both_ways(client): + """A12：替换模式的源上传新一期并回答 q_mode=accumulate → mode_switch:replace->accumulate，当前那一期作为第一期； + 反向切换出 mode_switch:accumulate->replace，提交后只剩一期。""" + base = await first_import(client, seed=141, mode="replace") + sid, name = base["source_id"], base["name"] + assert (await db_get(SourceSnapshot, base["s1"])).mode == "replace" + raw, fn = flow_workbook(SEP, 30, seed=142) + st = await reupload(client, sid, raw, fn) + assert "q_mode" in {q["id"] for q in st["questions"]} + st = await answer(client, st["id"], {"q_mode": {"value": "accumulate"}}) + st = await trial(client, st["id"]) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert t["accumulate"]["mode_switch"] == "replace->accumulate" and t["accumulate"]["action"] == "append" + assert [p["start"] for p in t["accumulate"]["parts"]] == ["2026-08-01", "2026-09-01"] + assert "mode_switch:replace->accumulate" in confirm_ids(st) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert resp.json()["parts"] == 2 and await count(client, name, "日客流") == 61 + + raw, fn = flow_workbook(OCT, 31, seed=143) + st = await reupload(client, sid, raw, fn) + st = await answer(client, st["id"], {"q_mode": {"value": "replace"}}) + st = await trial(client, st["id"]) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert t["accumulate"]["mode_switch"] == "accumulate->replace" and t["accumulate"]["action"] == "replace" + assert "mode_switch:accumulate->replace" in confirm_ids(st) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert resp.json()["parts"] == 1 + snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"]) + assert snap.mode == "replace" and len(snap.imports) == 1 + assert await count(client, name, "日客流") == 31 + + +# ========================================================================== +# A13：快照清单与证据链 +# ========================================================================== + + +async def test_a13_query_snapshot_commits_to_the_snapshot_and_import_manifests(client): + """A13：在累积快照上跑一次查询工具：query_snapshot → schema_snapshot 里有 snapshot_manifest 和两个 import_manifests → + 快照清单的 union.db_sha256 等于快照的 db_sha256；各期 rows 区间相接，覆盖 1..61。""" + from app.data.engine import engines + + v = await august_september(client, seed=151) + sid, name = v["source_id"], v["name"] + try: + query = (await tools_for([f"db_query__{name}"]))[f"db_query__{name}"] + payload = json.loads(await query.coroutine(sql='SELECT COUNT(*) FROM "日客流"')) + finally: + await engines.invalidate(sid) + assert payload["rows"][0][0] == 61 + qsnap = artifact_store.load(payload["artifact"]) + assert qsnap["data_version"] == v["s2"] + schema = artifact_store.load(payload["schema_artifact"]) + snap = await db_get(SourceSnapshot, v["s2"]) + assert schema["snapshot_manifest"] == snap.manifest_artifact + imps = [await db_get(TableImport, i) for i in snap.imports] + assert schema["import_manifests"] == [i.manifest_artifact for i in imps] and len(imps) == 2 + manifest = artifact_store.load(schema["snapshot_manifest"]) + assert manifest["format"] == "agentlab-snapshot-manifest/1" and manifest["snapshot_id"] == v["s2"] + assert manifest["union"]["db_sha256"] == snap.db_sha256 == raw_store.sha256_file(Path(snap.db_path)) + assert [p["import_id"] for p in manifest["parts"]] == snap.imports + for table, total in UNION_ROWS.items(): + spans = [p["rows"][table]["union"] for p in manifest["parts"]] + assert spans[0][0] == 1 and spans[-1][1] == total, (table, spans) + assert all(a[1] + 1 == b[0] for a, b in zip(spans, spans[1:])), (table, spans) + assert [p["rows"][table]["part"][0] for p in manifest["parts"]] == [1, 1] + for imp, part in zip(imps, manifest["parts"]): + doc = artifact_store.load(part["manifest_artifact"]) + assert doc["import_id"] == imp.id and part["manifest_artifact"] == imp.manifest_artifact + + +# ========================================================================== +# A14：确定性与复用 +# ========================================================================== + + +async def test_a14_same_september_file_under_another_name_reuses_the_union(client): + """A14：9 月同一份文件换个名字再传（替换该期）→ 新导入记录（文件名不同，不是期 2 的「原样重传」），union_id 与 S2 + 相同（并集 id 只取决于各期构建和目标表结构，不含导入记录 id），并集文件复用（新快照的库就是 S2 那个文件）， + db_sha256 相同。对照：同名原样再传不新建记录（unchanged）。""" + v = await august_september(client, seed=161) + sid = v["source_id"] + s2_row = await db_get(SourceSnapshot, v["s2"]) + st = await reupload(client, sid, v["sep_raw"], "九月客流（再传）.xlsx") + t = st["trial"] + assert t["status"] == "passed" and t["accumulate"]["action"] == "replace_period", t["problems"] + assert t["same_as_import"] == {"id": v["sep"], "seq": 2}, "同一个构建" + assert t["accumulate"]["union"]["union_id"] == Path(s2_row.db_path).stem + assert t["accumulate"]["union"]["db_sha256"] == s2_row.db_sha256 + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["unchanged"] is False and out["build_reused"] is True + assert out["import_id"] != v["sep"] and out["snapshot_id"] != v["s2"] + s3 = await db_get(SourceSnapshot, out["snapshot_id"]) + assert s3.db_path == s2_row.db_path and s3.db_sha256 == s2_row.db_sha256 + assert raw_store.sha256_file(Path(s3.db_path)) == s2_row.db_sha256 + assert (await db_get(TableImport, out["import_id"])).file_name == "九月客流（再传）.xlsx" + async with SessionLocal() as session: + unions = (await session.execute(select(TableBuild).where( + TableBuild.source_id == sid, TableBuild.raw_sha256 == ""))).scalars().all() + assert [u.id for u in unions] == [Path(s2_row.db_path).stem], "并集只登记一次" + # 同名原样再传：不新建记录，当前版本不动 + st = await reupload(client, sid, v["sep_raw"], "九月客流（再传）.xlsx") + resp = await commit(client, st) + assert resp.status_code == 201 and resp.json()["unchanged"] is True, resp.text + assert resp.json()["snapshot_id"] == out["snapshot_id"] + + +# ========================================================================== +# A16：并发：移除与提交交错 +# ========================================================================== + + +async def test_a16_remove_with_a_stale_version_after_a_commit_is_base_changed(client): + """A16 前半：S2=[8, 9] 上先取 snapshots 得到当前 id；另一个请求提交 10 月得 S3；再用旧的 expected_current 移除 9 月 + → 409 base_changed，当前快照仍是 S3，10 月仍在。""" + v = await august_september(client, seed=171) + sid = v["source_id"] + seen = (await snapshots(client, sid))[0]["id"] + assert seen == v["s2"] + raw, fn = flow_workbook(OCT, 31, seed=172) + s3 = (await add_period(client, sid, raw, fn))["snapshot_id"] + resp = await client.post(f"{API}/{sid}/imports/{v['sep']}/remove", + json={"confirm": True, "expected_current_snapshot_id": seen, "reason": REASON}) + assert resp.status_code == 409 and resp.json()["code"] == "base_changed", resp.text + assert await current_id(sid) == s3 + assert await current_periods(sid) == ["2026-08-01", "2026-09-01", "2026-10-01"] + + +async def test_a16_interleaved_publishes_the_later_one_is_refused(client, monkeypatch, store): + """A16 后半：两次发布卡在锁前交错。提交 10 月的请求进了 publish_snapshot、还没进锁时，移除 9 月（按 S2 确认）先完成； + 提交随后在锁内重读指针，与暂存区的 base（S2）对不上 → 后到的一方 409 base_changed，当前版本是移除的结果，10 月没进去， + 暂存区退回 drafting、没留下临时文件。反过来（移除停在物化里、提交先完成）由 test_source_versions_api 覆盖。""" + v = await august_september(client, seed=173) + sid = v["source_id"] + raw, fn = flow_workbook(OCT, 31, seed=174) + oct_st = await reupload(client, sid, raw, fn) + assert oct_st["trial"]["status"] == "passed" + orig = table_versions.publish_snapshot + removed: dict[str, Any] = {} + + async def interleaved(session, source, **kw): + if (kw.get("activation") or {}).get("kind") == "commit" and not removed: + removed["resp"] = await client.post( + f"{API}/{sid}/imports/{v['sep']}/remove", + json={"confirm": True, "expected_current_snapshot_id": v["s2"], "reason": REASON}) + return await orig(session, source, **kw) + + monkeypatch.setattr(table_versions, "publish_snapshot", interleaved) + resp = await commit(client, oct_st) + assert removed["resp"].status_code == 200, removed["resp"].text + assert removed["resp"].json()["snapshot_id"] == v["s1"] + assert resp.status_code == 409 and resp.json()["code"] == "base_changed", resp.text + assert await current_id(sid) == v["s1"] and await current_periods(sid) == ["2026-08-01"] + assert (await db_get(ImportStaging, oct_st["id"])).status == "drafting" + assert list((store / "uploads").rglob("*.tmp-*")) == [] + + +# ========================================================================== +# A17：移除时配方、模式不退回；已回收的快照复活 +# ========================================================================== + + +async def test_a17_1_removing_after_a_mode_switch_keeps_the_target_recipe_and_mode(client): + """A17①：替换模式 S1=[8]（配方 r1）→ 9 月 q_mode=accumulate 得 S2=[8, 9]（目标 r2）→ 移除 9 月：新快照 mode=accumulate、 + recipe=r2，id 不等于 S1；8 月的配方（r1）哈希不等于 r2，快照库是物化的单期并集（在 snapshots/ 下），说明含 + single_after_drop、不含「部分期」。""" + base = await first_import(client, seed=181, mode="replace") + sid, r1 = base["source_id"], base["recipe_id"] + raw, fn = flow_workbook(SEP, 30, seed=182) + st = await reupload(client, sid, raw, fn) + st = await answer(client, st["id"], {"q_mode": {"value": "accumulate"}}) + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + s2, r2, sep = resp.json()["snapshot_id"], resp.json()["recipe_id"], resp.json()["import_id"] + assert r2 != r1 + assert (await db_get(TableRecipe, r1)).recipe_sha256 != (await db_get(TableRecipe, r2)).recipe_sha256 + resp = await client.post(f"{API}/{sid}/imports/{sep}/remove", + json={"confirm": True, "expected_current_snapshot_id": s2, "reason": REASON}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["snapshot_id"] not in (base["s1"], s2) and out["reused"] is False + snap = await db_get(SourceSnapshot, out["snapshot_id"]) + assert snap.mode == "accumulate" and snap.recipe_id == r2 and snap.imports == [base["import_id"]] + assert Path(snap.db_path).parent.name == "snapshots", "一期、但配方不是目标配方：按目标配方物化成单期并集" + assert (await db_get(DataSource, sid)).current_recipe_id == r2 + comment = snap.schema_cache["tables"]["日客流"]["comment"] + assert "当前版本只含最近一期的数据" in comment and "部分期" not in comment, comment + + +async def test_a17_2_removing_the_period_that_added_a_column_keeps_the_target_recipe(client): + """A17②：[8, 9, 10] 中 10 月加了分区丙（修复按钮，目标 r3）→ 移除 10 月：目标仍是 r3，并集里分区丙为空值；下一期 + 按 r3 重放（不用再修）。""" + v = await august_september(client, seed=183) + sid, name = v["source_id"], v["name"] + raw, fn = flow_workbook(OCT, 31, seed=184, variant="D13") + st = await reupload(client, sid, raw, fn) + st, _ = await fixed(client, st, "add_label", "add") + st, _ = await fixed(client, st, "edit_members", "update") + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + s3, r3, oct_id = resp.json()["snapshot_id"], resp.json()["recipe_id"], resp.json()["import_id"] + resp = await client.post(f"{API}/{sid}/imports/{oct_id}/remove", + json={"confirm": True, "expected_current_snapshot_id": s3, "reason": REASON}) + assert resp.status_code == 200, resp.text + snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"]) + assert snap.recipe_id == r3 and snap.imports == [v["aug"], v["sep"]] and snap.id != v["s2"] + assert (await db_get(DataSource, sid)).current_recipe_id == r3 + assert "分区丙" in columns_of(snap, "日客流") + assert await count(client, name, "日客流", ' WHERE "分区丙" IS NULL') == 61 + raw, fn = flow_workbook(date_of(2026, 11, 1), 30, seed=185, variant="D13") + st = await reupload(client, sid, raw, fn) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert st["recipe_sha256"] == (await db_get(TableRecipe, r3)).recipe_sha256 + + +async def test_a17_3_removing_revives_a_retired_snapshot_whose_union_file_was_deleted(client, monkeypatch): + """A17③（1.5：KEEP=0）：S2=[8, 9]（并集）在提交 10 月之后被回收、并集文件被删；移除 10 月按同一组导入、同一个目标配方 + 得到 S2 的 id → 复活（重新物化、link 回去），文件哈希等于原登记值。""" + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0) + v = await august_september(client, seed=186) + sid = v["source_id"] + s2 = await db_get(SourceSnapshot, v["s2"]) + registered, union_file = s2.db_sha256, Path(s2.db_path) + raw, fn = flow_workbook(OCT, 31, seed=187) + out = await add_period(client, sid, raw, fn) + assert (await db_get(SourceSnapshot, v["s2"])).retired_at is not None and not union_file.exists() + resp = await client.post(f"{API}/{sid}/imports/{out['import_id']}/remove", + json={"confirm": True, "expected_current_snapshot_id": out["snapshot_id"], + "reason": REASON}) + assert resp.status_code == 200, resp.text + assert resp.json()["snapshot_id"] == v["s2"] and resp.json()["reused"] is True + revived = await db_get(SourceSnapshot, v["s2"]) + assert revived.retired_at is None and revived.db_sha256 == registered + assert union_file.is_file() and raw_store.sha256_file(union_file) == registered + assert (await db_get(TableBuild, union_file.stem)).retired_at is None + assert await count(client, v["name"], "日客流") == 61 + + +# ========================================================================== +# A18：退役名撞名、单位风险 +# ========================================================================== + + +def rename_column(rec: dict[str, Any], old: str, new: str) -> dict[str, Any]: + """把日客流的一列改名：measures 映射、单位、关系的成员一起改（改名在配方里分不出来，按「旧名退役、新名新增」报）。""" + rec = json.loads(json.dumps(rec, ensure_ascii=False)) + for sheet in rec["sheets"]: + for block in sheet["blocks"]: + for seg in block.get("segments", []): + if isinstance(seg.get("measures"), dict): + seg["measures"] = {k: (new if v == old else v) for k, v in seg["measures"].items()} + for t in rec["tables"]: + if t["name"] == "日客流" and old in t.get("units", {}): + t["units"] = {(new if k == old else k): v for k, v in t["units"].items()} + for r in rec["relations"]: + if r.get("table") == "日客流": + r["parts"] = [new if x == old else x for x in r.get("parts", [])] + if r.get("total") == old: + r["total"] = new + return rec + + +async def test_a18_1_a_new_column_colliding_with_a_retired_one_restarts(client): + """A18①：当前版本有退役列「PM2_5」时，新增「pm2_5」→ restart（SQLite 列名不分大小写，并集建表会报重名，评审一-m6）。""" + base = await first_import(client, seed=191, edit=lambda rec: rename_column(rec, "分区乙", "PM2_5")) + sid = base["source_id"] + raw, fn = flow_workbook(SEP, 30, seed=192) + st = await reupload(client, sid, raw, fn) + st = await put_recipe(client, st["id"], rename_column(st["recipe"], "PM2_5", "分区乙")) + st = await trial(client, st["id"]) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert t["accumulate"]["action"] == "append" + assert {"table": "日客流", "column": "PM2_5"} in t["accumulate"]["retired_new"] + assert "retire:日客流.PM2_5" in confirm_ids(st) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + raw, fn = flow_workbook(OCT, 31, seed=193) + st = await reupload(client, sid, raw, fn) + st = await put_recipe(client, st["id"], rename_column(st["recipe"], "分区乙", "pm2_5")) + assert st["recipe_problems"] == [], st["recipe_problems"] + st = await trial(client, st["id"]) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert t["accumulate"]["action"] == "restart" + assert any("pm2_5" in x and "PM2_5" in x for x in t["accumulate"]["semantic"]), t["accumulate"]["semantic"] + assert "accumulate_restart" in confirm_ids(st) + + +async def test_a18_2_unit_wording_outside_the_region_changing_needs_confirmation(client): + """A18②：累积模式下 9 月区域外的「单位：人次」改成「单位：万人次」→ 确认项有 accumulate_unit_risk（单位写在配方之外， + classify_changes 看不到，评审一-M10）。""" + raw8, _ = flow_workbook(AUG, 31, seed=194) + base = await first_import(client, seed=194, raw=edit_cells(raw8, {"B32": "注：单位：人次"})) + raw9, fn9 = flow_workbook(SEP, 30, seed=195) + st = await reupload(client, base["source_id"], edit_cells(raw9, {"B32": "注：单位：万人次"}), fn9) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert t["accumulate"]["action"] == "append" + assert "accumulate_unit_risk" in confirm_ids(st), confirm_ids(st) + item = next(i for i in t["confirm_items"] if i["id"] == "accumulate_unit_risk") + assert "万人次" in item["label"] + (item.get("detail") or "") + # 对照：区域外文字不变时不出 + raw9b, fn9b = flow_workbook(SEP, 30, seed=196) + await client.delete(f"{API}/imports/{st['id']}") + st = await reupload(client, base["source_id"], edit_cells(raw9b, {"B32": "注：单位：人次"}), fn9b) + assert st["trial"]["status"] == "passed" and "accumulate_unit_risk" not in confirm_ids(st) + + +# ========================================================================== +# 版本页：清除原件（7.6） +# ========================================================================== + + +async def test_purging_raw_files_from_the_version_page(client, store): + """7.6：清除原件要理由（422 reason_required）；当前版本里的一期可以清，不在当前版本里的导入（被替换掉的那一期）也能清 + （purge-raw 不看导入状态，1.5）；清除后原件文件删掉、导入记录 raw_state=purged、署名照记；同一份内容在别的导入里的 + 引用一并清除（also_purged），引用它的未完成导入一并放弃；那几期构建的隔离文件一并删掉；清除不影响累积：下一期照常 + 追加（期 3 不重放历史，2.5）。""" + v = await august_september(client, seed=301) + sid = v["source_id"] + aug = await db_get(TableImport, v["aug"]) + # 8 月的构建曾被改过、已恢复：隔离区里有它的一份 + build = await db_get(TableBuild, aug.build_id) + q = store / "uploads" / "quarantine" / sid + q.mkdir(parents=True, exist_ok=True) + stale = q / f"{aug.build_id}-20260101T000000Z.db" + stale.write_bytes(b"quarantined copy") + # 引用同一份 8 月原件的未完成导入（原样再传、还没提交） + open_st = await reupload(client, sid, v["raw"], v["file_name"]) + listed = {r["id"]: r for r in await imports_api(client, sid)} + assert listed[v["aug"]]["raw_open_stagings"] == 1 + url = f"{API}/{sid}/imports/{v['aug']}/purge-raw" + resp = await client.post(url, json={"confirm": True, "reason": " "}) + assert resp.status_code == 422 and resp.json()["code"] == "reason_required" + resp = await client.post(url, json={"confirm": True, "reason": REASON, "signed_by": "合成署名丙"}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["file_deleted"] is True and out["discarded_stagings"] == [open_st["id"]] + assert out["import"]["raw_state"] == "purged" and out["import"]["purged"]["signed_by"] == "合成署名丙" + assert not raw_store.raw_exists(aug.raw_sha256) and not stale.exists() + assert Path(build.db_path).is_file(), "构建库（数据）照旧，只清原件" + assert (await db_get(ImportStaging, open_st["id"])).status == "discarded" + resp = await client.post(url, json={"confirm": True, "reason": REASON}) + assert resp.status_code == 409 and resp.json()["code"] == "raw_already_purged" + # 不在当前版本里的导入：替换掉 9 月之后，旧 9 月的原件照样能清 + raw, fn = flow_workbook(SEP, 30, seed=304) + replaced = await add_period(client, sid, raw, fn) + assert (await db_get(TableImport, v["sep"])).status == "superseded" + resp = await client.post(f"{API}/{sid}/imports/{v['sep']}/purge-raw", json={"confirm": True, "reason": REASON}) + assert resp.status_code == 200, resp.text + # 清除不影响累积 + raw, fn = flow_workbook(OCT, 31, seed=303) + out = await add_period(client, sid, raw, fn) + assert out["parts"] == 3 and out["snapshot_id"] != replaced["snapshot_id"] + assert await count(client, v["name"], "日客流") == 92 + + +# ========================================================================== +# 遗留项 L1：被改过的构建文件（能重建就重建，否则明确报错） +# ========================================================================== + + +def simple_sheet(seed: int) -> bytes: + """一份最简单的表格（期 1 的简单上传用）：表头加两行，数字随机。""" + import io + import random + + rnd = random.Random(seed) + book = Workbook() + ws = book.active + ws.title = "甲表" + ws.append(["分区", "数量"]) + ws.append(["分区甲", rnd.randint(1, 999)]) + ws.append(["分区乙", rnd.randint(1, 999)]) + buf = io.BytesIO() + book.save(buf) + return buf.getvalue() + + +def corrupt(path: Path, data: bytes = b"") -> None: + os.chmod(path, 0o644) + path.write_bytes(data) + + +def quarantined(store: Path, source_id: str) -> list[Path]: + return sorted((store / "uploads" / "quarantine" / source_id).rglob("*.db")) + + +async def test_l1_upload_restores_a_tampered_build_and_cleans_the_quarantine(client, store): + """L1（/upload）：同一文件重传、构建文件被改成 0 字节 → 201，build_restored=true，quarantine/<源 id>/ 里有原文件， + 快照的文件哈希恢复；构建记录的哈希也被改掉时 → 409 build_conflict，指针不动；隔离文件超过 7 天被回收删除；删除 + 数据源后隔离目录被删。""" + name, raw = unique(), simple_sheet(201) + resp = await client.post(f"{API}/upload", files={"file": ("甲表.xlsx", raw, XLSX)}, data={"name": name}) + assert resp.status_code == 201 and resp.json()["build_restored"] is False, resp.text + first = resp.json() + sid = first["source"]["id"] + snap = await db_get(SourceSnapshot, first["snapshot_id"]) + corrupt(Path(snap.db_path)) + resp = await client.post(f"{API}/upload", files={"file": ("甲表.xlsx", raw, XLSX)}, data={"name": name}) + assert resp.status_code == 201, resp.text + assert resp.json()["build_restored"] is True + [moved] = quarantined(store, sid) + assert moved.stat().st_size == 0, "隔离的是被改过的那份" + assert raw_store.sha256_file(Path(snap.db_path)) == snap.db_sha256, "快照的文件哈希恢复" + # 记录的哈希也被改掉：无法用本次上传恢复 → 409 build_conflict，指针不动 + imp = await db_get(TableImport, first["import_id"]) + pointer = await current_id(sid) + corrupt(Path(snap.db_path), b"tampered again") + async with SessionLocal() as session: + await session.execute(update(TableBuild).where(TableBuild.id == imp.build_id).values(db_sha256="0" * 64)) + await session.commit() + resp = await client.post(f"{API}/upload", files={"file": ("甲表.xlsx", raw, XLSX)}, data={"name": name}) + assert resp.status_code == 409 and resp.json()["code"] == "build_conflict", resp.text + assert await current_id(sid) == pointer + # 隔离文件超过 QUARANTINE_DAYS 天：回收删掉 + old = time.time() - (table_versions.QUARANTINE_DAYS + 1) * 86400 + os.utime(moved, (old, old)) + async with SessionLocal() as session: + await table_versions.gc(session) + assert not moved.exists() + # 删除数据源：隔离目录一并删掉 + corrupt(Path(snap.db_path), b"") + async with SessionLocal() as session: + await session.execute(update(TableBuild).where(TableBuild.id == imp.build_id) + .values(db_sha256=snap.db_sha256)) + await session.commit() + resp = await client.post(f"{API}/upload", files={"file": ("甲表.xlsx", raw, XLSX)}, data={"name": name}) + assert resp.status_code == 201 and resp.json()["build_restored"] is True, resp.text + assert quarantined(store, sid) + resp = await client.delete(f"{API}/{sid}") + assert resp.status_code == 204, resp.text + assert not (store / "uploads" / "quarantine" / sid).exists() + + +async def test_l1_recipe_commit_restores_conflicts_and_quarantines_an_orphan(client, store): + """L1（配方提交）：8 月（每期替换）→ 9 月 → 8 月原文件再传（构建与第一次相同），第一次的构建文件被改成 0 字节 → + 201、build_restored=true；构建记录的哈希也被改掉 → 409 build_conflict、指针不动；目标位置有一个没人登记、内容不同的 + 文件（孤儿）→ 隔离后 link，发布成功。""" + base = await first_import(client, seed=211, mode="replace") + sid = base["source_id"] + raw9, fn9 = flow_workbook(SEP, 30, seed=212) + await add_period(client, sid, raw9, fn9) + build = await db_get(TableBuild, (await db_get(TableImport, base["import_id"])).build_id) + corrupt(Path(build.db_path)) + st = await reupload(client, sid, base["raw"], base["file_name"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert resp.json()["build_restored"] is True and resp.json()["build_id"] == build.id + assert raw_store.sha256_file(Path(build.db_path)) == build.db_sha256 + assert len(quarantined(store, sid)) == 1 + + # 记录的哈希也被改掉：9 月原文件再传，那份构建无法恢复 + sep_build = await db_get(TableBuild, (await db_get(TableImport, (await snapshots(client, sid))[1]["parts"][0] + ["import_id"])).build_id) + corrupt(Path(sep_build.db_path)) + async with SessionLocal() as session: + await session.execute(update(TableBuild).where(TableBuild.id == sep_build.id).values(db_sha256="0" * 64)) + await session.commit() + pointer = await current_id(sid) + st = await reupload(client, sid, raw9, fn9) + resp = await commit(client, st) + assert resp.status_code == 409 and resp.json()["code"] == "build_conflict", resp.text + assert await current_id(sid) == pointer + + # 孤儿：10 月的构建位置上已经有一个没人登记的文件 + raw10, fn10 = flow_workbook(OCT, 31, seed=213) + st = await reupload(client, sid, raw10, fn10) + build_id = (await db_get(ImportStaging, st["id"])).trial["build_id"] + orphan = table_versions.build_path(sid, build_id) + orphan.parent.mkdir(parents=True, exist_ok=True) + orphan.write_bytes(b"orphan nobody registered") + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert resp.json()["build_restored"] is False + assert raw_store.sha256_file(orphan) == (await db_get(TableBuild, build_id)).db_sha256 + assert any(p.read_bytes() == b"orphan nobody registered" for p in quarantined(store, sid)) + + +# ========================================================================== +# 遗留项 L2：改配方后回答延续 +# ========================================================================== + + +async def test_l2_answers_survive_a_put_that_embodies_them(client): + """L2：先回答 q_placeholder:·，再 PUT 一份体现了这个回答的配方 → answers 里仍有它，answers_dropped 为空；PUT 一份 + 与回答矛盾的配方（去掉占位符）→ answers_dropped 有它（带问题文字）。""" + raw, fn = flow_workbook(AUG, 31, seed=221) + st = await stage(client, raw, fn) + qid = "q_placeholder:·" + st = await answer(client, st["id"], {qid: {"value": "null"}}) + st = await put_recipe(client, st["id"], st["recipe"]) + assert st["answers"].get(qid, {}).get("value") == "null" and st["answers_dropped"] == [] + rec = json.loads(json.dumps(st["recipe"], ensure_ascii=False)) + for sheet in rec["sheets"]: + for block in sheet["blocks"]: + if isinstance(block.get("values"), dict): + block["values"]["placeholders"] = [] + st = await put_recipe(client, st["id"], rec) + dropped = {d["id"]: d for d in st["answers_dropped"]} + assert qid in dropped and dropped[qid]["text"] and dropped[qid]["reason"] in ("changed", "gone"), dropped + assert qid not in st["answers"] + + +# ========================================================================== +# 遗留项 L3：沿用已确认的常量（3.3），经接口 +# ========================================================================== + + +async def test_l3_keeping_a_confirmed_constant_follows_the_three_rules(client): + """L3（3.3）：D09 改了日间的分段标题。(b) 沿用：base 里同一分段的 pick「日间」仍是新标题的子串 → 静态校验通过； + 反例：换成不在候选词里、base 里也没有的「时段」→ const_not_candidate；含数字的 pick → period_literal；(a) 候选词 + 「日间分」→ 通过（但 breaking）；首次导入没有 base 时沿用不成立 → const_not_candidate；(c) 提交后下一期按重放校验 + （origin=replay，没有 base），子串成立 → 通过。""" + base = await first_import(client, seed=231) + sid = base["source_id"] + raw, fn = DRIFT_CASES["D09"].build(232) + st = await reupload(client, sid, raw, fn) + + def with_title(rec: dict[str, Any], pick: str) -> dict[str, Any]: + rec = json.loads(json.dumps(rec, ensure_ascii=False)) + seg = next(s for s in rec["sheets"][0]["blocks"][0]["segments"] if s["id"] == "日间") + seg["locate"]["title"] = "日间分时段客流（人次）" + seg["const"]["时段类别"]["pick"] = pick + return rec + + codes = lambda out: {p["code"] for p in out["recipe_problems"]} # noqa: E731 + out = await put_recipe(client, st["id"], with_title(st["recipe"], "时段")) + assert "const_not_candidate" in codes(out), out["recipe_problems"] + out = await put_recipe(client, st["id"], with_title(st["recipe"], "9月")) + assert "period_literal" in codes(out), out["recipe_problems"] + out = await put_recipe(client, st["id"], with_title(st["recipe"], "日间分")) + assert out["recipe_problems"] == [], "(a) 候选词照旧放行" + kept = with_title(st["recipe"], "日间") + out = await put_recipe(client, st["id"], kept) + assert out["recipe_problems"] == [], "(b) 沿用：base 里同一分段的 pick，仍是新标题的子串" + st = await trial(client, out["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert not [i for i in confirm_ids(st) if i.startswith("breaking:")], confirm_ids(st) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + # 首次导入没有 base：沿用不成立 + other = await stage(client, raw, fn) + out = await put_recipe(client, other["id"], rename_wide(kept, "日客流", "日客流")) + assert "const_not_candidate" in codes(out), out["recipe_problems"] + # (c) 下一期按重放校验：子串成立就放行 + raw10, fn10 = flow_workbook(OCT, 31, seed=233, variant="D09") + st = await reupload(client, sid, raw10, fn10) + assert st["recipe_problems"] == [] and st["trial"]["status"] == "passed", (st["recipe_problems"], + st["trial"]["problems"]) + + +# ========================================================================== +# 遗留项 L4：排除的行（rows_excluded）；列表只剩表头必须确认（empty_block） +# ========================================================================== + + +def list_book(rows: list[tuple[Any, ...]], *, sheet: str = "明细", extra: list[tuple[Any, ...]] | None = None + ) -> tuple[bytes, str]: + """一张列表：表头「地区、产品、金额」加 rows（None 行即空行），再接 extra。""" + book = Workbook() + ws = book.active + ws.title = sheet + for row in [("地区", "产品", "金额"), *rows, *(extra or [])]: + ws.append(list(row)) + return excel_saved(save(book), sheet, {}), f"{sheet}.xlsx" + + +def excluded_reasons(doc: dict[str, Any]) -> set[str]: + return {x["reason"] for x in doc["rows_excluded"]} + + +async def _commit_and_manifest(client, st: dict[str, Any]) -> dict[str, Any]: + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + imp = await db_get(TableImport, resp.json()["import_id"]) + return artifact_store.load(imp.manifest_artifact) + + +async def test_l4_rows_excluded_reach_the_receipt_and_the_import_manifest(client): + """L4：c06 exclude → hidden_excluded；c08 gap skip → blank_skipped；D21 忽略 → ignored_rows（A6b 已提交过一次，这里 + 再看清单）；列表停止后的文字行 → after_stop。trial.receipt.rows_excluded 经接口给出，导入清单的 receipt 里也有。""" + # c06：隐藏行不导入 + raw, fn = lab.c06() + st = await stage(client, raw, fn) + q = next(q for q in st["questions"] if q["id"].startswith("q_hidden:")) + st = await answer(client, st["id"], {q["id"]: {"value": "exclude"}}) + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert "hidden_excluded" in excluded_reasons(st["trial"]["receipt"]) + assert "hidden_excluded" in excluded_reasons((await _commit_and_manifest(client, st))["receipt"]) + # c08 gap：中间空行跳过 + raw, fn = lab.c08("gap") + st = await trial(client, (await stage(client, raw, fn))["id"]) + assert st["trial"]["status"] == "passed" + gap = [x for x in st["trial"]["receipt"]["rows_excluded"] if x["reason"] == "blank_skipped"] + assert gap and gap[0]["rows"] == [[4, 4]], st["trial"]["receipt"]["rows_excluded"] + assert "blank_skipped" in excluded_reasons((await _commit_and_manifest(client, st))["receipt"]) + # 列表停止后的文字行：空行之后还有两格文字（像数据，没导入）→ rows_after_stop 需确认，记 after_stop + raw, fn = list_book([("分区甲", "产品甲", 10), ("分区乙", "产品乙", 20), (None, None, None), + ("补充说明", "见附件甲", None)]) + st = await stage(client, raw, fn) + rec = json.loads(json.dumps(st["recipe"], ensure_ascii=False)) + rec["sheets"][0]["blocks"][0]["rows"]["blank_rows"] = "stop" + st = await trial(client, (await put_recipe(client, st["id"], rec))["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert problems(st, "rows_after_stop") and "after_stop" in excluded_reasons(st["trial"]["receipt"]) + assert "after_stop" in excluded_reasons((await _commit_and_manifest(client, st))["receipt"]) + # D21：按行标签忽略 + done = await _fix_case(client, "D21", seed=241) + assert "ignored_rows" in excluded_reasons(done["st"]["trial"]["receipt"]) + assert "ignored_rows" in excluded_reasons((await _commit_and_manifest(client, done["st"]))["receipt"]) + + +async def test_l4_a_list_that_drops_to_only_its_header_must_be_confirmed(client): + """遗留项 4 原意（期 2 AU-1 修复③）：上一期有数据行、本期只剩表头的列表重传，必须出 empty_block:<块> 必勾项， + 不勾不能启用；勾上之后这张表是 0 行。""" + raw, fn = list_book([("分区甲", "产品甲", 10), ("分区乙", "产品乙", 20), ("分区丙", "产品丙", 30)]) + st = await trial(client, (await stage(client, raw, fn))["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + sid, name = resp.json()["source"]["id"], resp.json()["source"]["name"] + table = st["recipe"]["sheets"][0]["blocks"][0]["table"] + block = st["recipe"]["sheets"][0]["blocks"][0]["id"] + raw, fn = list_book([]) + st = await reupload(client, sid, raw, fn) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert problems(st, "list_empty"), t["problems"] + item = f"empty_block:{block}" + items = {i["id"]: i for i in t["confirm_items"]} + assert item in items and items[item]["required"] is True, list(items) + resp = await commit(client, st, confirmations=[i for i in confirm_ids(st) if i != item]) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" and item in resp.json()["detail"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert await count(client, name, table) == 0 + + +# ========================================================================== +# 遗留项 L5：另一个进程持有存储守卫 +# ========================================================================== + +_HOLD = """ +import fcntl, os, sys +fd = os.open(sys.argv[1], os.O_RDWR | os.O_CREAT, 0o600) +fcntl.flock(fd, fcntl.LOCK_EX) +print("locked", flush=True) +sys.stdin.read() +""" + + +async def test_l5_another_process_holding_the_store_guard(client, store): + """L5：子进程先占住 .store.lock → 本进程 acquire_store_guard() 返回 False；本进程调 startup：事先放好的临时文件和孤儿 + 构建库一个都没删；/upload、stage、trial、commit、activate 返回 503 store_unavailable，GET 接口照常；过期处理不抛； + 子进程退出后重新调用返回 True，startup 照常清理。""" + v = await august_september(client, seed=251) + sid = v["source_id"] + raw10, fn10 = flow_workbook(OCT, 31, seed=252) + st = await reupload(client, sid, raw10, fn10) + half = table_versions.build_path(sid, "1" * 64).with_name(f"{'1' * 64}.db{raw_store.TMP_MARK}zzz") + half.write_bytes(b"half written") + orphan = table_versions.build_path(sid, "2" * 64) + orphan.write_bytes(b"orphan build") + proc = subprocess.Popen([sys.executable, "-c", _HOLD, str(settings.uploads_dir / ".store.lock")], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True) + try: + assert proc.stdout.readline().strip() == "locked" + assert table_versions.acquire_store_guard() is False + ran: list[str] = [] + real_gc = table_versions.gc + + async def spy_gc(session): + ran.append("gc") + + table_versions.gc = spy_gc + try: + async with SessionLocal() as session: + await table_versions.startup(session) + finally: + table_versions.gc = real_gc + assert ran == [] and half.exists() and orphan.exists(), "拿不到守卫：startup 一个文件都不删，不回收" + + def refused(resp) -> None: + assert resp.status_code == 503, resp.text + assert resp.json()["code"] == "store_unavailable" + + refused(await client.post(f"{API}/upload", files={"file": ("甲表.xlsx", simple_sheet(253), XLSX)}, + data={"name": unique()})) + refused(await client.post(f"{API}/imports/stage", files={"file": (fn10, raw10, XLSX)}, + data={"name": unique()})) + refused(await client.post(f"{API}/imports/{st['id']}/trial", json={})) + refused(await commit(client, st)) + refused(await client.post(f"{API}/{sid}/snapshots/{v['s1']}/activate", + json={"confirm": True, "expected_current_snapshot_id": v["s2"]})) + assert (await client.get(API)).status_code == 200 + assert (await client.get(f"{API}/{sid}/snapshots")).status_code == 200 + assert (await client.get(f"{API}/{sid}/imports")).status_code == 200 + assert (await client.get(f"{API}/imports/{st['id']}")).status_code == 200 + assert await count(client, v["name"], "日客流") == 61 + async with SessionLocal() as session: + assert await table_versions.expire_stagings(session) == [] + assert await current_id(sid) == v["s2"] + finally: + proc.stdin.close() + proc.wait(timeout=10) + try: + assert table_versions.acquire_store_guard() is True + async with SessionLocal() as session: + await table_versions.startup(session) + assert not half.exists() and not orphan.exists() + resp = await commit(client, st) + assert resp.status_code == 201 and resp.json()["parts"] == 3, resp.text + finally: + table_versions._release_guard() + + +# ========================================================================== +# 11.4 数据变异 +# ========================================================================== + + +def shifted_period(raw: bytes, start, days: int) -> bytes: + """把 9 月 D01 的统计期（B2）和日期表头（第 4 行）一起改成从 start 起 days 天：数字照旧，只挪统计期。""" + import datetime as dt + + from openpyxl.utils import get_column_letter + + end = start + dt.timedelta(days=days - 1) + edits: dict[str, Any] = {"B2": f"统计时间范围：{start.year}年{start.month}月{start.day}日至" + f"{end.year}年{end.month}月{end.day}日"} + for i in range(days): + d = start + dt.timedelta(days=i) + edits[f"{get_column_letter(3 + i)}4"] = f"{d.month}月{d.day}日" + return edit_cells(raw, edits) + + +async def test_mutation_september_period_overlapping_august_by_one_day(client): + """11.4 第 1 行：9 月 D01 的 B2 改成 08-31 至 09-29（与 8 月重叠一天）→ period_overlap，当前快照不动。日期表头跟着挪 + （否则先撞上「日期不在统计期内」的结构问题，同样拒收，只是原因不同，见下半）。""" + base = await first_import(client, seed=261) + raw, fn = flow_workbook(SEP, 30, seed=262) + st = await reupload(client, base["source_id"], shifted_period(raw, date_of(2026, 8, 31), 30), fn) + assert st["trial"]["status"] == "rejected" + [overlap] = problems(st, "period_overlap") + assert "2026-08-31 至 2026-09-29" in overlap["message"] and "2026-08-01 至 2026-08-31" in overlap["message"] + resp = await commit(client, st) + assert resp.status_code == 409 and resp.json()["code"] == "trial_not_passed" + await client.delete(f"{API}/imports/{st['id']}") + # 只改 B2、表头不动：9 月 30 日落在统计期之外，执行器先拒收（axis_year_missing），同样不动当前快照 + bad = edit_cells(raw, {"B2": "统计时间范围：2026年8月31日至2026年9月29日"}) + st = await reupload(client, base["source_id"], bad, fn) + assert st["trial"]["status"] == "rejected" and problems(st, "axis_year_missing") + assert await current_id(base["source_id"]) == base["s1"] + + +async def test_mutation_trial_files_tampered_or_deleted_before_commit(client): + """11.4 第 2、3 行：本期构建的试运行库在提交前被改字节 → 409 trial_tampered；并集试运行库被改 → 409 union_tampered； + 并集试运行库被删 → 409 trial_required，暂存区退回 drafting。当前快照都不动。""" + base = await first_import(client, seed=271) + sid = base["source_id"] + raw, fn = flow_workbook(SEP, 30, seed=272) + st = await reupload(client, sid, raw, fn) + row = await db_get(ImportStaging, st["id"]) + main = Path(row.trial_path) + with open(main, "ab") as f: + f.write(b"\0") + resp = await commit(client, st) + assert resp.status_code == 409 and resp.json()["code"] == "trial_tampered", resp.text + st = await trial(client, st["id"]) + union = table_versions.union_trial_path(Path((await db_get(ImportStaging, st["id"])).trial_path)) + with open(union, "ab") as f: + f.write(b"\0") + resp = await commit(client, st) + assert resp.status_code == 409 and resp.json()["code"] == "union_tampered", resp.text + st = await trial(client, st["id"]) + union = table_versions.union_trial_path(Path((await db_get(ImportStaging, st["id"])).trial_path)) + union.unlink() + resp = await commit(client, st) + assert resp.status_code == 409 and resp.json()["code"] == "trial_required", resp.text + assert (await db_get(ImportStaging, st["id"])).status == "drafting" + assert await current_id(sid) == base["s1"] + + +async def test_mutation_published_part_or_its_period_column_tampered(client): + """11.4 第 4、5 行：已发布的某一期构建库被改字节，再上传下一期 → 试运行 rejected、part_tampered；某一期导入记录的 + period_start 库列被改 → rejected、part_tampered（统计期取自导入清单）。""" + v = await august_september(client, seed=281) + sid = v["source_id"] + build = await db_get(TableBuild, (await db_get(TableImport, v["aug"])).build_id) + path = Path(build.db_path) + original = path.read_bytes() + corrupt(path, original[:-1] + bytes([original[-1] ^ 1])) + raw, fn = flow_workbook(OCT, 31, seed=282) + st = await reupload(client, sid, raw, fn) + assert st["trial"]["status"] == "rejected" and problems(st, "part_tampered"), st["trial"]["problems"] + corrupt(path, original) + os.chmod(path, 0o444) + async with SessionLocal() as session: + await session.execute(update(TableImport).where(TableImport.id == v["sep"]).values(period_start="2026-09-02")) + await session.commit() + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "rejected" + tampered = problems(st, "part_tampered") + assert tampered and "统计期" in tampered[0]["message"], st["trial"]["problems"] + assert await current_id(sid) == v["s2"] + + +async def test_mutation_snapshot_hash_column_rewritten_to_match_a_swapped_file(client): + """11.4 第 6 行：并集快照的库文件被换掉，source_snapshots.db_sha256 库列也改成与新文件一致的假值（文件哈希核对照样 + 通过）→ 查询时拿快照清单交叉核对，报 snapshot_tampered（版本清单不一致），不会查到被换掉的数据。""" + v = await august_september(client, seed=291) + snap = await db_get(SourceSnapshot, v["s2"]) + path = Path(snap.db_path) + os.chmod(path, 0o644) + with closing(sqlite3.connect(path)) as conn: + conn.execute('UPDATE "日客流" SET "全日客流" = 0') + conn.commit() + table_versions.settle_journal(path) + fake = raw_store.sha256_file(path) + async with SessionLocal() as session: + await session.execute(update(SourceSnapshot).where(SourceSnapshot.id == v["s2"]).values(db_sha256=fake)) + await session.commit() + from app.data.engine import engines + + await engines.invalidate(v["source_id"]) + resp = await client.post(f"/api/tools/db_query__{v['name']}/run", + json={"args": {"sql": 'SELECT SUM("全日客流") FROM "日客流"'}}) + assert resp.status_code == 200, resp.text + out = resp.json() + # 工具把数据不可用写成给模型看的结果（不抛），内容是拒绝查询的原因，没有数 + assert "查询失败（数据不可用）" in out["result"] and "版本清单不一致" in out["result"], out + with pytest.raises(table_versions.SnapshotManifestMismatch) as err: + async with SessionLocal() as session: + await table_versions.resolve_source(session, await session.get(DataSource, v["source_id"])) + assert err.value.code == "snapshot_tampered" + + +# ========================================================================== +# WP-8 修补 3：跳过空行的列表，表下隔着空行的说明不是数据（经真管线） +# ========================================================================== + + +async def test_fix3_note_under_a_list_with_a_middle_gap_stays_outside(client): + """列表中间有一个空行（起草成 blank_rows=skip），表下再隔一个空行有「注：……」：试运行 4 行，「注：……」进区域外文字， + 跳过的空行只有中间那一行；提交后表里没有首列是「注：……」的记录。框选整块（含中间空行）重放与框一致。曾静默多读进 + 一行首列是说明、其余为空值的记录（p3w1a/fix-2.md「需要你决定的事项」第 5 条）。""" + raw, fn = list_book([("分区甲", "产品甲", 10), ("分区甲", "产品乙", 20), (None, None, None), + ("分区乙", "产品甲", 30), ("分区乙", "产品乙", 40)], + extra=[(None, None, None), ("注：以上为初步统计，以终稿为准。", None, None)]) + st = await stage(client, raw, fn) + block = st["recipe"]["sheets"][0]["blocks"][0] + assert block["rows"]["blank_rows"] == "skip" + sel = {"sheet": "明细", "ref": "A1:C6", "as": "list", "options": {"header_rows": 1, "bottom": "box"}} + pv = (await preview(client, st["id"], {"selection": sel})).json() + assert pv["ok"] is True and pv["replay"]["match"] is True, (pv["problems"], pv["replay"]) + st = await trial(client, st["id"]) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert rows_of(st)[block["table"]] == 4 + assert ("明细!A8", "注：以上为初步统计，以终稿为准。") in [(o["cell"], o["text"]) for o in t["receipt"]["outside_text"]] + # 列表停在哪一行、因为什么也看得出来（评审意见 4）：说明那一行记进排除的行（after_stop，1 格，锚点是说明文字）； + # 像说明，不出 rows_after_stop + note = "注：以上为初步统计，以终稿为准。" + assert [(x["reason"], x["rows"], x["cells"], x["anchor"]) for x in t["receipt"]["rows_excluded"]] == [ + ("blank_skipped", [[4, 4]], 0, None), ("after_stop", [[8, 8]], 1, note)] + assert t["receipt"]["blank_rows_skipped"] == 1 + assert not problems(st, "rows_after_stop") and not [i for i in confirm_ids(st) if i.startswith("rows_after_stop")] + assert "空行之后" not in t["notes"][block["table"]]["comment"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + name = resp.json()["source"]["name"] + doc = artifact_store.load((await db_get(TableImport, resp.json()["import_id"])).manifest_artifact) + assert ("after_stop", [[8, 8]], note) in [(x["reason"], x["rows"], x["anchor"]) for x in doc["receipt"]["rows_excluded"]] + assert await count(client, name, block["table"]) == 4 + assert await count(client, name, block["table"], ' WHERE "地区" LIKE \'注%\'') == 0 + + +def _lone_tail_book(seed: int) -> tuple[bytes, str]: + """起草成 blank_rows=skip 的列表：中间一个空行，表尾隔一个空行是一条只填了首列的「分区丁」（评审意见 4 的探针）。 + 金额随 seed 变，地区和产品不变。""" + import random + + rnd = random.Random(seed) + return list_book([("分区甲", "产品甲", rnd.randint(10, 99)), ("分区乙", "产品甲", rnd.randint(10, 99)), + (None, None, None), ("分区丙", "产品甲", rnd.randint(10, 99)), (None, None, None), + ("分区丁", None, None)]) + + +async def test_fix3_a_lone_first_column_row_that_is_not_a_note_must_be_ticked_every_time(client): + """评审意见 4：表尾只填了首列的「分区丁」照旧当表的下边界（与起草器一致，不导入），但它不像说明：出 confirm 类的 + rows_after_stop（「第 7 行只有首列文字，已当作表下说明，没有导入」），必勾项 rows_after_stop:<块>，不勾不能提交； + 排除的行记 after_stop（锚点「分区丁」）。下一期重放同样的表尾，照样要勾。曾只出现在区域外文字里：确认项只有 + blank_skip 和 mode，排除的行里也没有它。""" + raw, fn = _lone_tail_book(711) + st = await stage(client, raw, fn) + block = st["recipe"]["sheets"][0]["blocks"][0] + assert block["rows"]["blank_rows"] == "skip" + st = await trial(client, st["id"]) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert rows_of(st)[block["table"]] == 3 + found = problems(st, "rows_after_stop") + assert len(found) == 1 and found[0]["category"] == "confirm" and found[0]["cells"] == ["明细!A7"], found + assert "第 7 行只有首列文字" in found[0]["message"] and "已当作表下说明，没有导入" in found[0]["message"] + assert ("after_stop", [[7, 7]], 1, "分区丁") in [(x["reason"], x["rows"], x["cells"], x["anchor"]) + for x in t["receipt"]["rows_excluded"]] + item = f"rows_after_stop:{block['id']}" + items = {i["id"]: i for i in t["confirm_items"]} + assert item in items and items[item]["required"] is True, list(items) + assert "第 7 行只有首列文字" in items[item]["label"] + assert "原表在空行之后另有未导入的文字行" in t["notes"][block["table"]]["comment"] + resp = await commit(client, st, confirmations=[i for i in confirm_ids(st) if i != item]) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" and item in resp.json()["detail"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + sid, name = resp.json()["source"]["id"], resp.json()["source"]["name"] + assert await count(client, name, block["table"]) == 3 + assert await count(client, name, block["table"], ' WHERE "地区" = \'分区丁\'') == 0 + # 下一期（数字不同、表尾照旧）：同样出这一项，不勾照样不能提交 + raw, fn = _lone_tail_book(712) + st = await reupload(client, sid, raw, fn) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert item in confirm_ids(st), confirm_ids(st) + resp = await commit(client, st, confirmations=[i for i in confirm_ids(st) if i != item]) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" and item in resp.json()["detail"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert await count(client, name, block["table"]) == 3 + + +async def test_fix3_a_build_from_the_old_engine_is_not_reused_for_the_same_file(client, monkeypatch): + """评审意见 1：修补 3 改了「列表 + 跳过空行」的执行结果，RECIPE_ENGINE_VER 没升（升了交叉表配方的构建全部换 id）， + 改为这类配方的构建 id 带执行器语义标记（table_versions.recipe_engine_semantics）。先用修补前的执行器（_note_after_blank + 恒 False、构建 id 不带标记）发布一次：5 行，其中一条首列是「注：……」。恢复后把同一文件换个名字重传：试运行 4 行， + 提交不复用旧构建（换了 id），发布出去的库就是这次试运行的库，表里 4 行。曾复用旧构建：核对的 4 行、发布的 5 行。""" + from app.data import recipe_engine + + note = "注：以上为初步统计，以终稿为准。" + raw, fn = list_book([("分区甲", "产品甲", 31), ("分区甲", "产品乙", 47), (None, None, None), + ("分区乙", "产品甲", 52), ("分区乙", "产品乙", 68)], + extra=[(None, None, None), (note, None, None)]) + with monkeypatch.context() as old: + old.setattr(recipe_engine._ListRun, "_note_after_blank", lambda self, inb: False) + old.setattr(table_versions, "recipe_engine_semantics", lambda recipe: {}) + st = await stage(client, raw, fn) + block = st["recipe"]["sheets"][0]["blocks"][0] + assert block["rows"]["blank_rows"] == "skip" + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert rows_of(st)[block["table"]] == 5 + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + first = resp.json() + sid, name, table = first["source"]["id"], first["source"]["name"], block["table"] + assert await count(client, name, table, ' WHERE "地区" LIKE \'注%\'') == 1 + old_build = await db_get(TableBuild, first["build_id"]) + assert "engine_semantics" not in old_build.options + + st = await reupload(client, sid, raw, "另一个名字.xlsx") + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert rows_of(st)[table] == 4 + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["build_reused"] is False and out["build_id"] != first["build_id"] + build = await db_get(TableBuild, out["build_id"]) + assert build.db_sha256 == t["receipt"]["db_sha256"], "发布出去的库就是这次试运行、用户核对过的那一份" + assert build.options["engine_semantics"] == {table_versions.LIST_NOTE_AFTER_BLANK: 1} + assert build.options_sha256 == table_versions.sha_json(build.options) + assert await count(client, name, table) == 4 + assert await count(client, name, table, ' WHERE "地区" LIKE \'注%\'') == 0 diff --git a/backend/tests/test_recipe_accumulate.py b/backend/tests/test_recipe_accumulate.py new file mode 100644 index 00000000..deb1b437 --- /dev/null +++ b/backend/tests/test_recipe_accumulate.py @@ -0,0 +1,719 @@ +"""按期累积（期 3，WP-4 recipe_accumulate）：配方演进的分档、累积计划、并集的物化。 + +要守住的几件事（P3-SPEC 2.4–2.7、11.3 的 WP-4 部分）： +- 计划的判定表逐行成立：不兼容即 restart，而且优先于部分重叠的 rejected；相等的统计期是「替换该期」、要确认； + 模式切换时当前那一期有统计期、满足资格、配方兼容才作为第一期； +- 退役只对「这次才退役的」出确认项：连续三期只有第一次是 retired_new；退役名和新名只差大小写按不兼容； + 没给当前表结构时按当前各期推（新增列照样报得出）；restart、rejected 不带各期维度取值的差异； +- 并集 id 带上退役列的组成和顺序（顺序进库哈希）；retired_columns 先拼早先退役的、再拼这次才退役的； +- 物化：行数、rowid 区间、确定性、U1–U3 各自能失败、退役列、单期物化、组合表哈希；各期库被改过就 part_tampered， + rowid 不连续就 union_failed；先写临时文件再 os.replace。 + +夹具：8 月、9 月用合成客流表（tests/fixtures/xlsx/flow.py，假名、随机数）按参考配方经执行器建库；其余变体的 +期库用 sqlite3 直接建，数字是手写的假数。 +""" +from __future__ import annotations + +import copy +import datetime as dt +import json +import os +import shutil +import sqlite3 +from functools import lru_cache +from pathlib import Path +from typing import Any + +import pytest + +from app.data import raw_store, recipe_accumulate as ra, recipe_engine, table_versions, xlsx_scan +from app.data.recipe_types import ( + UNION_VER, ColumnOut, PartInfo, Recipe, UnionPart, derive_tables, +) +from tests.fixtures.xlsx.flow import flow_workbook + +FLOW = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text(encoding="utf-8")) + + +def recipe(edit=None, *, mode: str = "accumulate") -> Recipe: + data = copy.deepcopy(FLOW) + data["mode"] = mode + if edit is not None: + edit(data) + return Recipe.model_validate(data) + + +def _segments(data: dict) -> list[dict]: + return data["sheets"][0]["blocks"][0]["segments"] + + +def no_zone_b(data: dict) -> None: + """去掉 measures 的分区乙（R1 一并去掉：sum_eq 至少两个成员）。""" + seg = _segments(data)[0] + seg["labels"]["expect"].remove("分区乙（人次）") + del seg["measures"]["分区乙（人次）"] + del data["tables"][0]["units"]["分区乙"] + data["relations"] = [r for r in data["relations"] if r["id"] != "R1"] + + +def with_zone_c(data: dict) -> None: + seg = _segments(data)[0] + seg["labels"]["expect"].append("分区丙（人次）") + seg["measures"]["分区丙（人次）"] = "分区丙" + data["tables"][0]["units"]["分区丙"] = "人次" + + +def unit_changed(data: dict) -> None: + data["tables"][0]["units"]["分区甲"] = "万人次" + + +def no_seven(data: dict) -> None: + """D04 的修法：日间去掉「7-8」。""" + _segments(data)[1]["labels"]["expect"].remove("7-8") + + +def no_covers(data: dict) -> None: + data["sheets"][0]["blocks"][0]["axis"]["checks"] = ["contiguous"] + + +REF = recipe() +AUG = ("2026-08-01", "2026-08-31") +SEP = ("2026-09-01", "2026-09-30") +OCT = ("2026-10-01", "2026-10-31") +NOV = ("2026-11-01", "2026-11-30") + + +def part(period: tuple[str | None, str | None], *, r: Recipe | None = REF, seq: int = 1, name: str = "", + rows: dict[str, int] | None = None) -> PartInfo: + return PartInfo(import_id=f"imp{seq}", seq=seq, start=period[0], end=period[1], build_id=f"b{seq}", + db_path="", db_sha256="", manifest_db_sha256=None, recipe_id=f"r{seq}", recipe=r, + recipe_sha256=None, raw_state="kept", file_name=name or f"第{seq}期.xlsx", manifest_artifact=None, + rows=rows or {}) + + +def tables_of(r: Recipe) -> dict[str, list[ColumnOut]]: + return derive_tables(r)[0] + + +def plan(new: Recipe, period, *, kind: str = "reupload", mode: str | None = "accumulate", + snap: str | None = "recipe", parts: list[PartInfo] = (), current=None, retired_status=None, + source: str = "cells"): + return ra.plan_accumulate(new_recipe=new, new_period=period, period_source=source, staging_kind=kind, + current_mode=mode, current_snapshot_kind=snap, parts=list(parts), + current_tables=current if current is not None else tables_of(REF), + retired_status=retired_status) + + +# ========================================================================== +# 累积计划：判定表 +# ========================================================================== + + +def test_replace_recipe_stays_replace_without_mode_switch(): + p = plan(recipe(mode="replace"), SEP, mode="replace", parts=[part(AUG)]) + assert p.action == "replace" and p.mode == "replace" and p.mode_switch is None + assert [x["new"] for x in p.parts] == [True] and p.dropped == [] + + +def test_accumulate_to_replace_drops_every_period(): + p = plan(recipe(mode="replace"), OCT, parts=[part(AUG), part(SEP, seq=2)]) + assert p.action == "replace" and p.mode_switch == "accumulate->replace" + assert [(d["start"], d["import_id"]) for d in p.dropped] == [("2026-08-01", "imp1"), ("2026-09-01", "imp2")] + assert [x["new"] for x in p.parts] == [True] + + +@pytest.mark.parametrize("kind,mode,snap,parts", [ + ("first", None, None, []), + ("switch", None, "simple", [part((None, None))]), + ("reupload", "replace", None, []), +]) +def test_first_import_and_switch_from_simple_start_from_this_period(kind, mode, snap, parts): + p = plan(REF, AUG, kind=kind, mode=mode, snap=snap, parts=parts) + assert p.action == "first" and p.mode == "accumulate" + assert [(x["start"], x["end"], x["new"], x["import_id"]) for x in p.parts] == [(*AUG, True, None)] + assert p.union is None + + +def test_switch_from_replace_keeps_the_current_period_as_the_first(): + old = recipe(mode="replace") + p = plan(REF, SEP, mode="replace", parts=[part(AUG, r=old, rows={"日客流": 31})]) + assert p.action == "append" and p.mode_switch == "replace->accumulate" + assert [(x["start"], x["new"]) for x in p.parts] == [("2026-08-01", False), ("2026-09-01", True)] + assert p.parts[0]["rows"] == {"日客流": 31} and p.parts[0]["blockers"] == [] + assert p.dropped == [] and p.reason is None + + +def test_switch_from_replace_without_a_period_starts_over(): + p = plan(REF, SEP, mode="replace", parts=[part((None, None), r=recipe(mode="replace"))]) + assert p.action == "first" and p.mode_switch == "replace->accumulate" + assert "没有统计期" in p.reason and [d["import_id"] for d in p.dropped] == ["imp1"] + + +def test_switch_from_replace_whose_recipe_is_not_eligible_starts_over(): + p = plan(REF, SEP, mode="replace", parts=[part(AUG, r=recipe(no_covers, mode="replace"))]) + assert p.action == "first" and p.mode_switch == "replace->accumulate" + assert "不满足按期累积的要求" in p.reason and "恰好覆盖统计期" in p.reason + assert p.dropped[0]["blockers"] + + +def test_switch_from_replace_with_an_incompatible_recipe_restarts(): + p = plan(recipe(unit_changed), SEP, mode="replace", parts=[part(AUG, r=recipe(mode="replace"))]) + assert p.action == "restart" and p.mode_switch == "replace->accumulate" + assert p.change == "semantic" and any("万人次" in s for s in p.semantic) + assert [x["new"] for x in p.parts] == [True] and len(p.dropped) == 1 + + +def test_switch_from_replace_on_the_same_period_replaces_it(): + p = plan(REF, AUG, mode="replace", parts=[part(AUG, r=recipe(mode="replace"))]) + assert p.action == "replace_period" and p.replaces["import_id"] == "imp1" + assert [x["new"] for x in p.parts] == [True] + + +def test_append_a_later_period(): + p = plan(REF, OCT, parts=[part(AUG), part(SEP, seq=2)]) + assert p.action == "append" and p.change == "same" and p.mode_switch is None + assert [x["start"] for x in p.parts] == ["2026-08-01", "2026-09-01", "2026-10-01"] + assert [x["new"] for x in p.parts] == [False, False, True] + assert p.gaps == [] and p.backfill is False and p.overlaps == [] and p.replaces is None + + +def test_backfill_an_earlier_period_is_sorted_in(): + p = plan(REF, SEP, parts=[part(AUG), part(OCT, seq=2)]) + assert p.action == "append" and p.backfill is True + assert [x["start"] for x in p.parts] == ["2026-08-01", "2026-09-01", "2026-10-01"] + assert p.gaps == [] and p.period == {"start": "2026-09-01", "end": "2026-09-30", "source": "cells"} + + +def test_backfill_with_a_period_typed_by_hand_carries_the_source(): + """补传早期、本期统计期是人工录入：差异卡要改为需确认（2.3），计划把来源带给 WP-3。""" + p = plan(REF, AUG, parts=[part(SEP), part(OCT, seq=2)], source="human") + assert p.action == "append" and p.backfill is True and p.period["source"] == "human" + assert [x["new"] for x in p.parts] == [True, False, False] + + +def test_gap_between_periods_is_reported(): + p = plan(REF, NOV, parts=[part(AUG), part(SEP, seq=2)]) + assert p.action == "append" and p.gaps == [{"start": "2026-10-01", "end": "2026-10-31"}] + + +def test_same_period_is_a_replace_candidate(): + p = plan(REF, SEP, parts=[part(AUG), part(SEP, seq=2)]) + assert p.action == "replace_period" and p.replaces["import_id"] == "imp2" + assert [(x["start"], x["new"]) for x in p.parts] == [("2026-08-01", False), ("2026-09-01", True)] + + +def test_partial_overlap_is_rejected_and_listed(): + p = plan(REF, ("2026-08-15", "2026-09-14"), parts=[part(AUG), part(SEP, seq=2)]) + assert p.action == "rejected" + assert [o["import_id"] for o in p.overlaps] == ["imp1", "imp2"] + # 当前版本原样不动 + assert [x["import_id"] for x in p.parts] == ["imp1", "imp2"] + msg = ra.overlap_message(p.period, p.overlaps) + assert "2026-08-15 至 2026-09-14" in msg and "2026-08-01 至 2026-08-31" in msg + assert "2026-09-01 至 2026-09-30" in msg and "部分重叠" in msg + + +def test_one_day_overlap_is_rejected(): + p = plan(REF, ("2026-08-31", "2026-09-29"), parts=[part(AUG)]) + assert p.action == "rejected" and [o["import_id"] for o in p.overlaps] == ["imp1"] + + +def test_incompatible_change_restarts_even_when_periods_overlap(): + """restart 优先于 rejected（评审一-m12）：重叠的各期写进 overlaps 只作提示。""" + p = plan(recipe(unit_changed), ("2026-08-15", "2026-09-14"), parts=[part(AUG), part(SEP, seq=2)]) + assert p.action == "restart" and [o["import_id"] for o in p.overlaps] == ["imp1", "imp2"] + assert "不兼容" in p.reason and len(p.dropped) == 2 + assert [x["new"] for x in p.parts] == [True] + + +def test_incompatible_change_restarts(): + p = plan(recipe(unit_changed), OCT, parts=[part(AUG), part(SEP, seq=2)]) + assert p.action == "restart" and p.change == "semantic" + assert [d["import_id"] for d in p.dropped] == ["imp1", "imp2"] + assert p.added == [] and p.retired_new == [] and p.retired_existing == [] + + +def test_a_kept_period_that_is_not_eligible_restarts(): + p = plan(REF, OCT, parts=[part(AUG, r=recipe(no_covers)), part(SEP, seq=2)]) + assert p.action == "restart" and "第 1 期" in p.reason + assert p.dropped[0]["blockers"] and p.dropped[1]["blockers"] == [] + + +def test_redraft_replaces_the_latest_period(): + p = plan(recipe(no_seven), SEP, kind="redraft", parts=[part(AUG), part(SEP, seq=2)]) + assert p.action == "replace_period" and p.replaces["import_id"] == "imp2" + assert p.change == "compatible" + + +def test_redraft_with_an_incompatible_recipe_restarts(): + p = plan(recipe(unit_changed), SEP, kind="redraft", parts=[part(AUG), part(SEP, seq=2)]) + assert p.action == "restart" and len(p.dropped) == 2 + + +def test_accumulate_needs_a_period(): + with pytest.raises(ValueError, match="统计期"): + plan(REF, (None, None), parts=[part(AUG)]) + + +def test_new_recipe_that_is_not_eligible_is_rejected(): + p = plan(recipe(no_covers), OCT, parts=[part(AUG)]) + assert p.action == "rejected" and "恰好覆盖统计期" in p.reason + + +# ========================================================================== +# 配方演进：兼容、退役、不兼容 +# ========================================================================== + + +def test_identical_recipes_are_the_same(): + cc = ra.classify_changes([part(AUG), part(SEP, seq=2)], REF, current_tables=tables_of(REF)) + assert cc.change == "same" and not (cc.added or cc.retired_new or cc.retired_existing or cc.semantic) + + +def test_new_column_is_compatible(): + target = recipe(with_zone_c) + cc = ra.classify_changes([part(AUG), part(SEP, r=target, seq=2)], target, current_tables=tables_of(REF)) + assert cc.change == "compatible" and cc.added == [{"table": "日客流", "column": "分区丙"}] + assert cc.semantic == [] and cc.retired_new == [] + + +def test_unit_change_is_semantic(): + cc = ra.classify_changes([part(AUG)], recipe(unit_changed), current_tables=tables_of(REF)) + assert cc.change == "semantic" and cc.semantic and "万人次" in cc.semantic[0] + + +def test_removed_column_retires_only_once_over_three_periods(): + """连续三期：去掉分区乙的那一期出 retired_new，之后各期是 retired_existing、不再出确认项（评审二-M3）。""" + target = recipe(no_zone_b) + first = plan(target, OCT, parts=[part(AUG), part(SEP, seq=2)]) + assert first.action == "append" and first.change == "retire" + assert first.retired_new == [{"table": "日客流", "column": "分区乙"}] and first.retired_existing == [] + # 10 月提交后，当前版本的实际表结构仍含分区乙（保留在并集里），快照清单把它标成 retired + current = tables_of(REF) + status = {"日客流": {"分区乙": {"status": "retired", "since": "2026-10-01~2026-10-31", "unit": "人次"}}} + second = plan(target, NOV, parts=[part(AUG), part(SEP, seq=2), part(OCT, r=target, seq=3)], + current=current, retired_status=status) + assert second.action == "append" and second.change == "compatible" + assert second.retired_new == [] and second.retired_existing == [{"table": "日客流", "column": "分区乙"}] + + +def test_retired_table_is_existing_only_when_every_column_is_marked(): + target = recipe() + current = {**tables_of(REF), "旧表": [ColumnOut("日期", "TEXT", role="axis"), ColumnOut("值", "INTEGER")]} + fresh = ra.classify_changes([part(AUG)], target, current_tables=current) + assert {"table": "旧表", "column": None} in fresh.retired_new + half = ra.classify_changes([part(AUG)], target, current_tables=current, + retired_status={"旧表": {"日期": {"status": "retired"}}}) + assert {"table": "旧表", "column": None} in half.retired_new + done = ra.classify_changes([part(AUG)], target, current_tables=current, retired_status={ + "旧表": {"日期": {"status": "retired"}, "值": {"status": "retired"}}}) + assert done.retired_existing == [{"table": "旧表", "column": None}] and done.retired_new == [] + + +def test_new_name_colliding_with_a_retired_name_is_semantic(): + def add_lower(data: dict) -> None: + seg = _segments(data)[0] + seg["labels"]["expect"].append("PM2.5（人次）") + seg["measures"]["PM2.5（人次）"] = "pm2_5" + current = copy.deepcopy(tables_of(REF)) + current["日客流"].append(ColumnOut("PM2_5", "INTEGER")) + status = {"日客流": {"PM2_5": {"status": "retired"}}} + cc = ra.classify_changes([part(AUG)], recipe(add_lower), current_tables=current, retired_status=status) + assert cc.change == "semantic" and "只差大小写" in cc.semantic[0] + p = plan(recipe(add_lower), SEP, parts=[part(AUG)], current=current, retired_status=status) + assert p.action == "restart" + + +def test_label_sets_record_which_periods_lack_a_label(): + target = recipe(no_seven) + cc = ra.classify_changes([part(AUG), part(SEP, r=target, seq=2)], target, current_tables=tables_of(REF)) + assert cc.change == "compatible" and cc.semantic == [] + [entry] = cc.label_sets + assert (entry["table"], entry["column"], entry["segment"]) == ("时段客流", "时段", "日间") + aug, sep = entry["periods"] + assert (aug["start"], aug["missing"], aug["extra"]) == ("2026-08-01", [], ["7-8"]) + assert (sep["start"], sep["missing"], sep["extra"]) == ("2026-09-01", ["7-8"], []) + p = plan(target, SEP, kind="redraft", parts=[part(AUG), part(SEP, seq=2)]) + assert p.label_sets and p.label_sets[0]["segment"] == "日间" + + +def test_label_sets_compare_canonical_writing(): + """全角、en-dash 写法按规范写法比：只是写法不同不算取值不同。""" + def dashed(data: dict) -> None: + seg = _segments(data)[1] + seg["labels"]["expect"] = [x.replace("-", "–") for x in seg["labels"]["expect"]] + cc = ra.classify_changes([part(AUG), part(SEP, r=recipe(dashed), seq=2)], REF, current_tables=tables_of(REF)) + assert cc.label_sets == [] + + +def test_missing_current_tables_falls_back_to_the_parts(): + target = recipe(no_zone_b) + cc = ra.classify_changes([part(AUG)], target, current_tables=None) + assert cc.retired_new == [{"table": "日客流", "column": "分区乙"}] + + +def _raw_plan(new: Recipe, period, parts: list[PartInfo], *, kind: str = "reupload", current=None): + return ra.plan_accumulate(new_recipe=new, new_period=period, period_source="cells", staging_kind=kind, + current_mode="accumulate", current_snapshot_kind="recipe", parts=parts, + current_tables=current) + + +def test_plan_without_current_tables_still_reports_added_columns(): + """current_tables 为 None 时按当前各期的配方推当前表结构，不能把本期也算进「当前已有」：那样新增列永远报不出来。""" + target = recipe(with_zone_c) + p = _raw_plan(target, SEP, [part(AUG)]) + assert p.action == "append" and p.added == [{"table": "日客流", "column": "分区丙"}] + assert p.added == _raw_plan(target, SEP, [part(AUG)], current=tables_of(REF)).added + # 修改配方重建最近一期：分区丙在要被替换的 9 月里已经有了，它此刻仍在当前版本中，不算新增 + redraft = _raw_plan(target, SEP, [part(AUG), part(SEP, r=target, seq=2)], kind="redraft") + assert redraft.action == "replace_period" and redraft.added == [] + # 退役同样比得出来 + gone = _raw_plan(recipe(no_zone_b), SEP, [part(AUG)]) + assert gone.retired_new == [{"table": "日客流", "column": "分区乙"}] + + +def test_restart_and_rejected_plans_carry_no_label_sets(): + """restart 的结果只有本期、rejected 没有结果版本：各期维度取值的差异没有落点，不能让差异卡说「此前各期有」。""" + def no_seven_and_unit(data: dict) -> None: + no_seven(data) + unit_changed(data) + restart = plan(recipe(no_seven_and_unit), SEP, parts=[part(AUG)]) + assert restart.action == "restart" and [x["start"] for x in restart.parts] == ["2026-09-01"] + assert restart.label_sets == [] + rejected = plan(recipe(no_seven), ("2026-08-15", "2026-09-14"), parts=[part(AUG)]) + assert rejected.action == "rejected" and rejected.label_sets == [] + # 对照：同样的取值差异在追加时照常记下 + assert plan(recipe(no_seven), SEP, parts=[part(AUG)]).label_sets + + +# ========================================================================== +# 并集 id +# ========================================================================== + + +def test_union_id_ignores_import_ids_and_input_order(): + a = ra.union_id("src1", [("2026-08-01", "2026-08-31", "b1"), ("2026-09-01", "2026-09-30", "b2")], "r" * 64) + b = ra.union_id("src1", [("2026-09-01", "2026-09-30", "b2"), ("2026-08-01", "2026-08-31", "b1")], "r" * 64) + assert a == b and len(a) == 64 + assert table_versions.sha_json({"source_id": "src1", "parts": [["2026-08-01", "2026-08-31", "b1"], + ["2026-09-01", "2026-09-30", "b2"]], + "target_recipe_sha256": "r" * 64, "union_ver": UNION_VER}) == a + assert ra.union_id("src1", [("2026-08-01", "2026-08-31", "b1")], "r" * 64) != a + assert ra.union_id("src1", [("2026-08-01", "2026-08-31", "b1"), ("2026-09-01", "2026-09-30", "b2")], + "s" * 64) != a + assert ra.union_id("src2", [("2026-08-01", "2026-08-31", "b1"), ("2026-09-01", "2026-09-30", "b2")], + "r" * 64) != a + + +def test_union_id_commits_to_the_retired_layout(): + """退役列有历史：同样的各期、同样的目标配方，退役列的组成、顺序、类型不同就是结构不同的文件，id 必须跟着不同。""" + parts = [("2026-08-01", "2026-08-31", "b1"), ("2026-09-01", "2026-09-30", "b2")] + base = ra.union_id("src1", parts, "r" * 64) + assert ra.union_id("src1", parts, "r" * 64, retired=None) == base + assert ra.union_id("src1", parts, "r" * 64, retired={}) == base + assert ra.union_id("src1", parts, "r" * 64, retired={"日客流": []}) == base + d, e = ColumnOut("分区丁", "INTEGER"), ColumnOut("分区戊", "INTEGER") + one = ra.union_id("src1", parts, "r" * 64, retired={"日客流": [d]}) + assert one != base + assert one == table_versions.sha_json({ + "source_id": "src1", "parts": [list(x) for x in parts], "target_recipe_sha256": "r" * 64, + "union_ver": UNION_VER, "retired": [["日客流", [["分区丁", "INTEGER"]]]]}) + de = ra.union_id("src1", parts, "r" * 64, retired={"日客流": [d, e]}) + ed = ra.union_id("src1", parts, "r" * 64, retired={"日客流": [e, d]}) + assert len({one, de, ed}) == 3 + assert ra.union_id("src1", parts, "r" * 64, retired={"日客流": [ColumnOut("分区丁", "REAL")]}) != one + # 登记的 options 与 id 的输入一一对应 + assert ra.union_options(parts, "r" * 64) == {"kind": "union", "parts": [list(x) for x in parts], + "target_recipe_sha256": "r" * 64} + assert ra.union_options(parts, "r" * 64, retired={"日客流": [d, e]})["retired"] == [ + ["日客流", [["分区丁", "INTEGER"], ["分区戊", "INTEGER"]]]] + + +# ========================================================================== +# 物化 +# ========================================================================== + + +@lru_cache(maxsize=None) +def _built(start: str, days: int, seed: int = 0) -> tuple[str, str, str, str, tuple]: + """按参考配方（累积）经执行器建一期的库，进程内只建一次。返回 (路径, 哈希, 起, 止, 表哈希)。""" + import tempfile + + raw, name = flow_workbook(dt.date.fromisoformat(start), days, seed=seed) + folder = Path(tempfile.mkdtemp(prefix="wp4-built-")) + db = folder / f"{start}-{seed}.db" + ex = recipe_engine.execute(REF, raw, name, xlsx_scan.scan(raw), str(db)) + assert ex.ok, ex.problems + table_versions.settle_journal(db) + os.chmod(db, 0o444) + return str(db), raw_store.sha256_file(db), ex.period.start, ex.period.end, tuple(sorted(ex.table_hashes.items())) + + +def built_part(first_day: str, days: int, *, seed: int = 0, r: Recipe = REF, **kw: Any) -> UnionPart: + """一期的 UnionPart；kw 可以改登记的统计期、路径、哈希、表哈希（造各种失败用）。""" + path, sha, s, e, hashes = _built(first_day, days, seed) + return UnionPart(import_id=kw.pop("import_id", None), start=kw.pop("start", s), end=kw.pop("end", e), + db_path=kw.pop("db_path", path), recipe=r, db_sha256=kw.pop("db_sha256", sha), + table_hashes=kw.pop("table_hashes", dict(hashes)), **kw) + + +def aug_sep() -> list[UnionPart]: + return [built_part("2026-08-01", 31), built_part("2026-09-01", 30)] + + +def count(path: Path, sql: str, params=()) -> Any: + conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) + try: + return conn.execute(sql, params).fetchone()[0] + finally: + conn.close() + + +def test_two_periods_materialize_with_rows_ranges_and_checks(tmp_path): + out = tmp_path / "u.db" + rep = ra.materialize_union(out, target=REF, parts=aug_sep()) + assert rep.ok and [c.status for c in rep.checks] == ["passed"] * 3 + assert [(c.id, c.kind, c.category) for c in rep.checks] == [ + ("U1", "union_rows", "structure"), ("U2", "union_pk", "structure"), ("U3", "union_period", "structure")] + assert rep.rows == {"日客流": 61, "时段客流": 1037, "时段客流_表内合计": 183} + assert rep.part_rows[0]["日客流"] == {"union": [1, 31], "part": [1, 31]} + assert rep.part_rows[1]["日客流"] == {"union": [32, 61], "part": [1, 30]} + assert rep.part_rows[1]["时段客流"] == {"union": [528, 1037], "part": [1, 510]} + assert rep.db_sha256 == raw_store.sha256_file(out) + assert count(out, 'SELECT COUNT(*) FROM "日客流"') == 61 + assert count(out, 'SELECT COUNT(*) FROM "日客流" WHERE substr("日期", 1, 7) = ?', ("2026-09",)) == 30 + assert count(out, 'SELECT COUNT(*) FROM "时段客流" WHERE "时段" = ?', ("8-9",)) == 61 + # 表结构与执行器的写法相同：STRICT、主键 + ddl = count(out, "SELECT sql FROM sqlite_master WHERE name = '日客流'") + assert "STRICT" in ddl and 'PRIMARY KEY ("日期")' in ddl + assert count(out, "PRAGMA journal_mode") == "delete" + assert not list(tmp_path.glob(f"*{raw_store.TMP_MARK}*")) + assert rep.added == [] and rep.retired == [] + assert rep.part_null_counts[1]["时段客流"]["客流"] == count( + out, 'SELECT COUNT(*) FROM "时段客流" WHERE rowid BETWEEN 528 AND 1037 AND "客流" IS NULL') + assert rep.null_counts["时段客流"]["客流"] == count(out, 'SELECT COUNT(*) FROM "时段客流" WHERE "客流" IS NULL') + + +def test_materialize_is_deterministic_and_replaces_an_existing_file(tmp_path): + first = ra.materialize_union(tmp_path / "a.db", target=REF, parts=aug_sep()) + out = tmp_path / "b.db" + out.write_bytes(b"stale union from an older trial") + second = ra.materialize_union(out, target=REF, parts=aug_sep()) + assert first.db_sha256 == second.db_sha256 == raw_store.sha256_file(out) + + +def test_composite_table_hash_uses_registered_part_hashes(tmp_path): + parts = aug_sep() + rep = ra.materialize_union(tmp_path / "u.db", target=REF, parts=parts) + cols = [[c.name, c.type] for c in tables_of(REF)["日客流"]] + want = table_versions.sha_json({"union_ver": UNION_VER, "columns": cols, "grain": ["日期"], + "parts": [parts[0].table_hashes["日客流"], parts[1].table_hashes["日客流"]]}) + assert rep.table_hashes["日客流"] == want + # 换一期登记的表哈希，组合哈希跟着变；导入记录 id 不参与 + other = aug_sep() + other[1].table_hashes = {**other[1].table_hashes, "日客流": "0" * 64} + other[0].import_id = "another-import" + rep2 = ra.materialize_union(tmp_path / "v.db", target=REF, parts=other) + assert rep2.table_hashes["日客流"] != want + assert rep2.table_hashes["时段客流"] == rep.table_hashes["时段客流"] + + +def test_tampered_part_is_refused_before_attaching(tmp_path): + parts = aug_sep() + parts[1].db_sha256 = "0" * 64 + with pytest.raises(ra.UnionError) as err: + ra.materialize_union(tmp_path / "u.db", target=REF, parts=parts) + assert err.value.code == "part_tampered" and err.value.part == 2 and "第 2 期" in str(err.value) + assert not (tmp_path / "u.db").exists() + + +def test_part_whose_bytes_changed_is_tampered(tmp_path): + parts = aug_sep() + copy_path = tmp_path / "aug.db" + shutil.copyfile(parts[0].db_path, copy_path) + with open(copy_path, "ab") as fh: + fh.write(b"\0") + parts[0].db_path = str(copy_path) # 登记的哈希仍是原来的 + with pytest.raises(ra.UnionError) as err: + ra.materialize_union(tmp_path / "u.db", target=REF, parts=parts) + assert err.value.code == "part_tampered" and err.value.part == 1 + + +def test_manifest_hash_must_match_too(tmp_path): + parts = aug_sep() + parts[0].manifest_db_sha256 = "1" * 64 + with pytest.raises(ra.UnionError) as err: + ra.materialize_union(tmp_path / "u.db", target=REF, parts=parts) + assert err.value.code == "part_tampered" and err.value.part == 1 + + +def test_missing_part_file(tmp_path): + parts = aug_sep() + parts[1].db_path = str(tmp_path / "gone.db") + with pytest.raises(ra.UnionError) as err: + ra.materialize_union(tmp_path / "u.db", target=REF, parts=parts) + assert err.value.code == "part_missing" and err.value.part == 2 + # 接口层直接转发这句，所以出路要写在里面（7.4） + msg = str(err.value) + assert "第 2 期（2026-09-01 至 2026-09-30）" in msg and "移除这一期" in msg and "启用更早的版本" in msg + + +def test_rowid_gap_in_a_part_fails(tmp_path): + parts = aug_sep() + holed = tmp_path / "holed.db" + shutil.copyfile(parts[1].db_path, holed) + os.chmod(holed, 0o644) + conn = sqlite3.connect(holed) + conn.execute('DELETE FROM "日客流" WHERE rowid = 5') + conn.commit() + conn.close() + parts[1].db_path, parts[1].db_sha256 = str(holed), raw_store.sha256_file(holed) + out = tmp_path / "u.db" + with pytest.raises(ra.UnionError) as err: + ra.materialize_union(out, target=REF, parts=parts) + assert err.value.code == "union_failed" and err.value.part == 2 and "行号不连续" in str(err.value) + assert not out.exists() and not list(tmp_path.glob(f"*{raw_store.TMP_MARK}*")) + + +def test_duplicate_primary_key_fails_u2_and_produces_no_file(tmp_path): + """两期写了同一天（计划本该拒收的重叠）：写入撞上主键，U2 不通过并指出哪两期、哪个主键。""" + aug = built_part("2026-08-01", 31) + again = built_part("2026-08-01", 31, seed=0) + out = tmp_path / "u.db" + out.write_bytes(b"stale") + rep = ra.materialize_union(out, target=REF, parts=[aug, again]) + assert rep.ok is False and rep.db_sha256 == "" + u1, u2, u3 = rep.checks + assert u2.status == "mismatch" and "第 1 期与第 2 期" in u2.details[0] and "日期=2026-08-01" in u2.details[0] + assert u1.status == "mismatch" # 回滚的那一期没写进去，行数也对不上 + assert u3.status == "passed" + assert not out.exists() and not list(tmp_path.glob(f"*{raw_store.TMP_MARK}*")) + + +def test_dates_outside_the_period_fail_u3(tmp_path): + aug = built_part("2026-08-01", 31, start="2026-09-01", end="2026-09-30") + rep = ra.materialize_union(tmp_path / "u.db", target=REF, parts=[aug]) + assert rep.ok is False + u3 = rep.checks[2] + assert u3.status == "mismatch" and u3.failed == 3 + assert "第 1 期（2026-09-01 至 2026-09-30）的「日客流」有 31 行日期不在该期统计期内" in u3.details + assert u3.sql and u3.params[2:] == ["2026-09-01", "2026-09-30"] + + +def _small_part(path: Path, r: Recipe, start: str, days: int, values: dict[str, list[int]]) -> UnionPart: + """只含「日客流」一张表的期库（sqlite3 直接建，列按 r 推出的表结构、写法同执行器）。""" + cols = tables_of(r)["日客流"] + conn = sqlite3.connect(path) + defs = ", ".join(f'"{c.name}" {c.type}' + (" NOT NULL" if c.name == "日期" else "") for c in cols) + conn.execute(f'CREATE TABLE "日客流" ({defs}, PRIMARY KEY ("日期")) STRICT') + first = dt.date.fromisoformat(start) + for i in range(days): + row = [(first + dt.timedelta(days=i)).isoformat()] + [values[c.name][i] for c in cols[1:]] + conn.execute(f'INSERT INTO "日客流" VALUES ({", ".join("?" * len(cols))})', row) + conn.commit() + conn.close() + table_versions.settle_journal(path) + end = (first + dt.timedelta(days=days - 1)).isoformat() + return UnionPart(import_id=None, start=start, end=end, db_path=str(path), recipe=r, + db_sha256=raw_store.sha256_file(path), table_hashes={"日客流": f"h-{start}"}) + + +def test_retired_column_keeps_old_values_and_later_periods_are_null(tmp_path): + target = recipe(no_zone_b) + aug = built_part("2026-08-01", 31) + sep = _small_part(tmp_path / "sep.db", target, "2026-09-01", 3, + {"全日客流": [30, 31, 32], "分区甲": [30, 31, 32]}) + retired = {"日客流": [ColumnOut("分区乙", "INTEGER")]} + rep = ra.materialize_union(tmp_path / "u.db", target=target, parts=[aug, sep], retired=retired) + assert rep.ok, rep.checks + names = [c.name for c in rep.tables["日客流"]] + assert names == ["日期", "全日客流", "分区甲", "分区乙"] # 目标的列在前、退役的追加在后 + zone_b = rep.tables["日客流"][-1] + assert zone_b.unit == "人次" and zone_b.header == "分区乙（人次）" # 取最后一个含它的那一期 + out = tmp_path / "u.db" + assert count(out, 'SELECT COUNT(*) FROM "日客流" WHERE "分区乙" IS NOT NULL AND rowid <= 31') == 31 + assert count(out, 'SELECT COUNT(*) FROM "日客流" WHERE "分区乙" IS NULL AND rowid > 31') == 3 + assert {"table": "日客流", "column": "分区乙", "periods": ["2026-08-01~2026-08-31"]} in rep.retired + # 按期的空值数只记该期有的列：9 月没有分区乙，就没有这个键（结构性空值不能说成「原表为空格」） + assert "分区乙" in rep.part_null_counts[0]["日客流"] and "分区乙" not in rep.part_null_counts[1]["日客流"] + assert rep.null_counts["日客流"]["分区乙"] == 3 + # 9 月没有另外两张表：那几张表只有 8 月的行，记作只有部分期 + assert rep.rows["时段客流"] == 527 and "时段客流" not in rep.part_rows[1] + assert {"table": "时段客流", "column": None, "periods": ["2026-08-01~2026-08-31"]} in rep.added + + +def test_require_union_ok_turns_a_failed_report_into_union_failed(tmp_path): + """U1–U3 不通过时 materialize_union 只返回 ok=False（试运行要展示核对结果）；版本页的写操作用 require_union_ok + 换成 union_failed，消息带上不通过的细节。""" + bad = ra.materialize_union(tmp_path / "bad.db", target=REF, + parts=[built_part("2026-08-01", 31, start="2026-09-01", end="2026-09-30")]) + assert bad.ok is False + with pytest.raises(ra.UnionError) as err: + ra.require_union_ok(bad) + assert err.value.code == "union_failed" and err.value.part is None + assert "日期不在该期统计期内" in str(err.value) + ra.require_union_ok(ra.materialize_union(tmp_path / "good.db", target=REF, parts=aug_sep())) + + +def test_retired_columns_put_earlier_retirements_first(tmp_path): + """先早先退役的（按当前表结构里的列序）、再这次才退役的（2.6「按退役先后追加」）；照 current_tables 直接拼会 + 把新退役的排到前面。顺序进库哈希，所以 union_id 也要带上同一个 retired。""" + current = copy.deepcopy(tables_of(REF)) + current["日客流"].append(ColumnOut("分区丁", "INTEGER", unit="人次")) + current["旧表"] = [ColumnOut("日期", "TEXT", role="axis"), ColumnOut("值", "INTEGER")] + status = {"日客流": {"分区丁": {"status": "retired"}}, "旧表": {"日期": {"status": "retired"}, + "值": {"status": "retired"}}} + target = recipe(no_zone_b) + p = plan(target, SEP, parts=[part(AUG)], current=current, retired_status=status) + assert p.retired_existing == [{"table": "日客流", "column": "分区丁"}, {"table": "旧表", "column": None}] + assert p.retired_new == [{"table": "日客流", "column": "分区乙"}] + retired = ra.retired_columns(p, current) + assert list(retired) == ["日客流", "旧表"] + assert [c.name for c in retired["日客流"]] == ["分区丁", "分区乙"] + assert [c.name for c in retired["旧表"]] == ["日期", "值"] + cc = ra.classify_changes([part(AUG)], target, current_tables=current, retired_status=status) + assert ra.retired_layout(ra.retired_columns(cc, current)) == ra.retired_layout(retired) + rep = ra.materialize_union(tmp_path / "u.db", target=target, parts=[built_part("2026-08-01", 31)], + retired=retired) + assert rep.ok, rep.checks + assert [c.name for c in rep.tables["日客流"]] == ["日期", "全日客流", "分区甲", "分区丁", "分区乙"] + assert rep.rows["旧表"] == 0 + # 换个顺序就是另一个文件，id 也跟着不同 + swapped = {"日客流": list(reversed(retired["日客流"])), "旧表": retired["旧表"]} + rep2 = ra.materialize_union(tmp_path / "v.db", target=target, parts=[built_part("2026-08-01", 31)], + retired=swapped) + assert rep2.db_sha256 != rep.db_sha256 + ids = [("2026-08-01", "2026-08-31", "b1")] + assert ra.union_id("s", ids, "r" * 64, retired=retired) != ra.union_id("s", ids, "r" * 64, retired=swapped) + assert ra.retired_columns(p, None) == {} + + +def test_part_column_without_a_place_in_the_union_fails(tmp_path): + """目标去掉了分区乙、却没把它作为退役列传进来：那一列的数据没地方放，不能悄悄丢掉。""" + with pytest.raises(ra.UnionError) as err: + ra.materialize_union(tmp_path / "u.db", target=recipe(no_zone_b), parts=[built_part("2026-08-01", 31)]) + assert err.value.code == "union_failed" and "分区乙" in str(err.value) + + +def test_single_period_materialized_by_a_different_target(tmp_path): + """移除、撤销之后只剩一期、而那一期的配方不是目标配方：按目标表结构物化成「一期的并集」。""" + target = recipe(with_zone_c) + rep = ra.materialize_union(tmp_path / "u.db", target=target, parts=[built_part("2026-08-01", 31)]) + assert rep.ok + assert [c.name for c in rep.tables["日客流"]][-1] == "分区丙" + assert rep.null_counts["日客流"]["分区丙"] == 31 + assert {"table": "日客流", "column": "分区丙", "periods": []} in rep.added + assert rep.rows == {"日客流": 31, "时段客流": 527, "时段客流_表内合计": 93} + + +def test_missing_table_hash_is_refused(tmp_path): + parts = aug_sep() + parts[0].table_hashes = {} + with pytest.raises(ra.UnionError) as err: + ra.materialize_union(tmp_path / "u.db", target=REF, parts=parts) + assert err.value.code == "union_failed" and "表哈希" in str(err.value) + + +def test_parts_must_be_sorted(tmp_path): + with pytest.raises(ValueError, match="排序"): + ra.materialize_union(tmp_path / "u.db", target=REF, parts=list(reversed(aug_sep()))) diff --git a/backend/tests/test_recipe_compare.py b/backend/tests/test_recipe_compare.py new file mode 100644 index 00000000..6a71985b --- /dev/null +++ b/backend/tests/test_recipe_compare.py @@ -0,0 +1,171 @@ +"""新旧配方对照（recipe_compare.compare_recipes，P3-SPEC 6.2、9.4 的 RecipeComparison）。 + +配方手写（假名），以参考配方为基线，按修复按钮、框选、重新起草会改到的地方逐项改。 +""" +from __future__ import annotations + +import copy +import json +from typing import Any + +from app.data.recipe_compare import compare_recipes, relation_text +from app.data.recipe_types import Recipe +from tests.test_recipe_confirm import LIST +from tests.test_recipe_confirm_p3 import plan +from tests.test_recipe_diff import FLOW + + +def flow() -> dict[str, Any]: + return copy.deepcopy(FLOW) + + +def seg(r: dict[str, Any], i: int) -> dict[str, Any]: + return r["sheets"][0]["blocks"][0]["segments"][i] + + +def table(cmp: dict[str, Any], name: str) -> dict[str, Any]: + return next(t for t in cmp["tables"] if t["name"] == name) + + +def column(cmp: dict[str, Any], tname: str, cname: str) -> dict[str, Any]: + return next(c for c in table(cmp, tname)["columns"] if c["name"] == cname) + + +def segment(cmp: dict[str, Any], sid: str) -> dict[str, Any]: + return next(s for s in cmp["segments"] if s["id"] == sid) + + +def test_identical_recipes(): + cmp = compare_recipes(FLOW, Recipe.model_validate(FLOW).model_dump(mode="json", exclude_defaults=True)) + assert cmp["breaking"] is False and cmp["units_changed"] == [] and cmp["accumulate"] is None + assert {t["status"] for t in cmp["tables"]} == {"same"} + assert all(c["status"] == "same" and c["changes"] == [] for t in cmp["tables"] for c in t["columns"]) + assert {s["status"] for s in cmp["segments"]} == {"same"} + assert {r["status"] for r in cmp["relations"]} == {"same"} + assert cmp["sheets"] == [{"id": "s1", "name": {"old": "客流汇总", "new": "客流汇总"}}] + assert cmp["mode"] == {"old": "replace", "new": "replace"} + json.dumps(cmp, ensure_ascii=False) # 原样进 TrialOut、EditPreview + + +def test_d13_add_column_and_update_members(): + new = flow() + s = seg(new, 0) + s["labels"]["expect"].append("分区丙（人次）") + s["measures"]["分区丙（人次）"] = "分区丙" + new["tables"][0]["units"]["分区丙"] = "人次" + new["relations"][0]["parts"] = ["分区甲", "分区乙", "分区丙"] + cmp = compare_recipes(FLOW, new) + t = table(cmp, "日客流") + assert t["status"] == "changed" and t["breaking"] == [] + assert column(cmp, "日客流", "分区丙") == {"name": "分区丙", "status": "added", "old": None, + "new": {"type": "INTEGER", "unit": "人次", "source": "分区丙（人次）"}, + "changes": []} + assert segment(cmp, "日客流")["labels"] == {"added": ["分区丙（人次）"], "removed": []} + r1 = next(r for r in cmp["relations"] if r["id"] == "R1") + assert r1 == {"id": "R1", "status": "changed", "old": "全日客流 = 分区甲 + 分区乙", + "new": "全日客流 = 分区甲 + 分区乙 + 分区丙"} + assert cmp["breaking"] is False + + +def test_d04_remove_label_is_not_breaking(): + new = flow() + seg(new, 1)["labels"]["expect"] = seg(new, 1)["labels"]["expect"][1:] + cmp = compare_recipes(FLOW, new) + assert segment(cmp, "日间")["status"] == "changed" + assert segment(cmp, "日间")["labels"] == {"added": [], "removed": ["7-8"]} + assert table(cmp, "时段客流")["status"] == "same" and cmp["breaking"] is False + + +def test_title_change_and_const_value(): + new = flow() + seg(new, 1)["locate"]["title"] = "日间分时段客流（人次）" + cmp = compare_recipes(FLOW, new) + assert segment(cmp, "日间")["title"] == {"old": "日间时段客流（人次）", "new": "日间分时段客流（人次）"} + assert cmp["breaking"] is False + seg(new, 1)["const"]["时段类别"]["pick"] = "日间分" + cmp = compare_recipes(FLOW, new) + assert column(cmp, "时段客流", "时段类别")["changes"] == ["const_value"] + assert column(cmp, "时段客流", "时段类别")["status"] == "changed" + assert cmp["breaking"] is True and table(cmp, "时段客流")["breaking"] + + +def test_unit_change_first_and_separate_from_breaking(): + new = flow() + new["tables"][2]["units"]["客流"] = "人" + new["sheets"][0]["blocks"][0]["values"]["type"] = "REAL" + cmp = compare_recipes(FLOW, new) + assert cmp["tables"][0]["name"] == "时段客流_表内合计" # 单位变化的表最前 + assert cmp["units_changed"] == [{"table": "时段客流_表内合计", "column": "客流", "old": "人次", "new": "人"}] + assert all("单位" not in m for t in cmp["tables"] for m in t["breaking"]) + assert column(cmp, "时段客流_表内合计", "客流")["changes"] == ["type", "unit"] + assert cmp["breaking"] is True + # 其余破坏性的在没变化的表之前 + statuses = [(bool(t["breaking"]), t["status"]) for t in cmp["tables"][1:]] + assert statuses == sorted(statuses, key=lambda x: (not x[0], x[1] == "same")) + + +def test_removed_and_added_tables_and_grain(): + new = flow() + seg(new, 3)["keep_as"] = None + new["tables"].pop() + new["tables"][0]["grain"] = ["日期", "全日客流"] + cmp = compare_recipes(FLOW, new) + assert table(cmp, "时段客流_表内合计")["status"] == "removed" + assert table(cmp, "时段客流_表内合计")["kind"] == {"old": "reported_total", "new": None} + assert table(cmp, "日客流")["grain"] == {"old": ["日期"], "new": ["日期", "全日客流"], "changed": True} + back = compare_recipes(new, FLOW) + assert table(back, "时段客流_表内合计")["status"] == "added" + + +def test_added_and_removed_segments_list_all_labels(): + """起草器换了分段 id（「日间」→「日间段」）：新增的分段列出标题和全部标签，去掉的分段列出原有的标题和标签， + redraft_adopted 据此逐项写出新认出的定位规则（6.3；早先新增分段的 labels.added 是空的）。""" + new = flow() + renamed = copy.deepcopy(seg(new, 1)) + renamed["id"] = "日间段" + new["sheets"][0]["blocks"][0]["segments"][1] = renamed + cmp = compare_recipes(FLOW, new) + labels = list(seg(FLOW, 1)["labels"]["expect"]) + added, removed = segment(cmp, "日间段"), segment(cmp, "日间") + assert added["status"] == "added" and added["labels"] == {"added": labels, "removed": []} + assert added["title"] == {"old": None, "new": "日间时段客流（人次）"} + assert removed["status"] == "removed" and removed["labels"] == {"added": [], "removed": labels} + assert removed["title"] == {"old": "日间时段客流（人次）", "new": None} + + +def test_sheet_name_mode_and_ignore_rules(): + new = flow() + new["sheets"][0]["match"]["name"] = "客流汇总（新）" + new["mode"] = "accumulate" + new["sheets"][0]["blocks"][0]["ignore_rows"] = [{"label": "补录（人次）", "reason": "表下补录"}] + new["sheets"][0]["ignore_outside"] = [{"anchor": "补录说明", "reason": "说明里的数字"}] + cmp = compare_recipes(FLOW, new) + assert cmp["sheets"] == [{"id": "s1", "name": {"old": "客流汇总", "new": "客流汇总（新）"}}] + assert cmp["mode"] == {"old": "replace", "new": "accumulate"} + assert segment(cmp, "交叉表")["ignore"] == {"added": ["按行标签忽略「补录（人次）」"], "removed": []} + assert segment(cmp, "s1")["ignore"]["added"] == ["按同一行的文字「补录说明」忽略区域外的数字"] + assert compare_recipes(new, FLOW)["segments"][-1]["ignore"]["removed"] + + +def test_list_after_title_and_header_source(): + new = copy.deepcopy(LIST) + block = new["sheets"][0]["blocks"][0] + block["after_title"] = "一、销售" + block["columns"][3]["header"] = "销量（件数）" + cmp = compare_recipes(LIST, new) + assert segment(cmp, "列表1")["title"] == {"old": None, "new": "一、销售"} + c = column(cmp, "销售", "销量") + assert c["changes"] == ["source"] and c["old"]["source"] == "销量（件）" and c["new"]["source"] == "销量（件数）" + + +def test_plan_is_copied_into_accumulate(): + p = plan(change="compatible", added=[{"table": "日客流", "column": "分区丙"}]) + cmp = compare_recipes(FLOW, FLOW, plan=p) + assert cmp["accumulate"] == {"change": "compatible", "added": [{"table": "日客流", "column": "分区丙"}], + "retired_new": [], "retired_existing": []} + + +def test_relation_text(): + r = Recipe.model_validate(FLOW) + assert relation_text(r.relations[0]) == "全日客流 = 分区甲 + 分区乙" + assert relation_text(r.relations[1]) == "表「时段客流」与表「日客流」口径不同，不互相推算或相加" diff --git a/backend/tests/test_recipe_confirm.py b/backend/tests/test_recipe_confirm.py index bd7376b3..bfbc0a11 100644 --- a/backend/tests/test_recipe_confirm.py +++ b/backend/tests/test_recipe_confirm.py @@ -234,12 +234,16 @@ def test_recipe_class_only_when_recipe_differs(): assert not recipe_differs("reupload", explicit, compact) assert ids(ConfirmContext(kind="reupload", recipe=explicit, old_recipe=compact, extraction=ex, checks=[], diff=[], prev=BASE[0])) == set() - # 改了配方：配方类全部重新出 + # 改了配方：期 3 起只列签名有变化的配方类项（P3-SPEC 6.2），没变的已在现行配方提交时确认过 r = flow() r["sheets"][0]["blocks"][0]["values"]["blank"] = "null" got = ids(ConfirmContext(kind="redraft", recipe=r, old_recipe=FLOW, extraction=ex, checks=[], diff=[], prev=BASE[0])) - assert D00_CONFIRMS | {"blank_null:交叉表"} <= got + assert got == {"blank_null:交叉表"} + # 采用按规则重新起草的配方：按首次导入列全部，另出 redraft_adopted + got = ids(ConfirmContext(kind="redraft", recipe=r, old_recipe=FLOW, extraction=ex, checks=[], diff=[], + prev=BASE[0], recipe_origin="rules_redraft")) + assert D00_CONFIRMS | {"blank_null:交叉表", "redraft_adopted"} <= got # 首次、切换、没有现行配方都算不同 assert recipe_differs("first", FLOW, FLOW) and recipe_differs("switch", FLOW, None) assert recipe_differs("reupload", FLOW, None) @@ -612,6 +616,23 @@ def test_rows_after_stop_two_blocks_uses_lineage(): assert "rows_after_stop:列表1" in got + +def test_rows_after_stop_from_a_skip_list_goes_to_the_block_named_in_the_message(): + """跳过空行的列表按表下说明收尾、那格文字不像说明时也报 rows_after_stop（WP-8 评审意见 4）。同一工作表上另有一个 + stop 模式的列表：只按 blank_rows=stop 找候选会把它归到列表1；消息里写着「列表「列表2」」，确认项按它记在列表2 上。""" + r = copy.deepcopy(LIST) + r["sheets"][0]["blocks"][0]["rows"] = {} + second = copy.deepcopy(r["sheets"][0]["blocks"][0]) + second.update(id="列表2", table="费用", after_title="二、费用", rows={"blank_rows": "skip"}) + r["sheets"][0]["blocks"].append(second) + r["tables"] = [{"name": "销售"}, {"name": "费用"}] + lineage = {"销售": {"地区": [[1, "销售", "C3", 4, "down"]]}, "费用": {"地区": [[1, "销售", "C12", 3, "down"]]}} + prob = {"code": "rows_after_stop", "category": "confirm", "cells": ["销售!C16"], + "message": "工作表「销售」的列表「列表2」：第 16 行只有首列文字（C16「地区丁」），已当作表下说明，没有导入。"} + got = ids(first(r, list_ex(problems=[prob], lineage=lineage, derived=[]))) + assert "rows_after_stop:列表2" in got and "rows_after_stop:列表1" not in got + + def test_relation_observed(): cur, _ = sep() assert not any(i.startswith("relation_observed:") for i in ids(reupload(cur))) # 30 天中 0 天相等 diff --git a/backend/tests/test_recipe_confirm_p3.py b/backend/tests/test_recipe_confirm_p3.py new file mode 100644 index 00000000..ad302b3f --- /dev/null +++ b/backend/tests/test_recipe_confirm_p3.py @@ -0,0 +1,654 @@ +"""期 3 的确认项（recipe_confirm，P3-SPEC 6.2、9.5）：只列有变化的配方类项、修改记录、采用重新起草、按期累积。 + +回执和配方都是手写的（假名、假数），构造器在 test_recipe_diff、test_recipe_confirm 里；累积计划按契约 +AccumulatePlan 的 JSON 形状手写（WP-4 的 plan_accumulate 在这一波还没合并，这里不依赖它）。 +""" +from __future__ import annotations + +import copy +from typing import Any, Callable + +import pytest + +from app.data.recipe_confirm import ConfirmContext, confirm_items, recipe_item_signatures +from app.data.recipe_types import AccumulatePlan, ConfirmItem, DiffItem +from tests.test_recipe_confirm import LIST, flow, list_ex +from tests.test_recipe_diff import BASE, D00_CONFIRMS, FLOW, SHEET, extraction_of, sep + +#: 配方类确认项的前缀（6.2 的签名表 + 期 3 新增的两种） +RECIPE_PREFIXES = ("placeholder:", "blank_null:", "text_number:", "formula_cached:", "merged_fill:", + "checks_relaxed:", "cross_check_off:", "ignored_columns:", "blank_skip:", "derived:", "total_row:", + "year_from:", "relation:", "dismissed:", "unit:", "hidden:", "mode", "ignore_rows:", + "ignore_outside:") + + +def recipe_ids(items: list[ConfirmItem]) -> set[str]: + return {i.id for i in items if i.id.startswith(RECIPE_PREFIXES)} + + +def redraft(new: dict[str, Any], old: dict[str, Any] = FLOW, ex: dict[str, Any] | None = None, + **kw: Any) -> list[ConfirmItem]: + kw.setdefault("checks", []) + kw.setdefault("diff", []) + kw.setdefault("prev", BASE[0]) + return confirm_items(ConfirmContext(kind=kw.pop("kind", "redraft"), recipe=new, old_recipe=old, + extraction=ex if ex is not None else extraction_of(sep()[0]), **kw)) + + +def by_id(items: list[ConfirmItem]) -> dict[str, ConfirmItem]: + return {i.id: i for i in items} + + +def seg(r: dict[str, Any], i: int) -> dict[str, Any]: + return r["sheets"][0]["blocks"][0]["segments"][i] + + +def block(r: dict[str, Any]) -> dict[str, Any]: + return r["sheets"][0]["blocks"][0] + + +# -------------------------------------------------------------------------- +# 6.2：签名表的每一行「变了才出」 +# -------------------------------------------------------------------------- + + +def _set(path: Callable[[dict[str, Any]], dict[str, Any]], key: str, value: Any) -> Callable[[dict[str, Any]], None]: + def go(r: dict[str, Any]) -> None: + path(r)[key] = value + return go + + +def _values(r: dict[str, Any]) -> dict[str, Any]: + return block(r)["values"] + + +def _ctx0(r: dict[str, Any]) -> dict[str, Any]: + return r["sheets"][0]["context"][0] + + +def _r1(r: dict[str, Any]) -> dict[str, Any]: + return r["relations"][0] + + +def _add_placeholder(r: dict[str, Any]) -> None: + _values(r)["placeholders"].append({"text": "—", "meaning": "不适用"}) + + +def _ignore_rows(reason: str) -> Callable[[dict[str, Any]], None]: + def go(r: dict[str, Any]) -> None: + block(r)["ignore_rows"] = [{"label": "补录（人次）", "reason": reason}] + return go + + +def _ignore_outside(reason: str) -> Callable[[dict[str, Any]], None]: + def go(r: dict[str, Any]) -> None: + r["sheets"][0]["ignore_outside"] = [{"anchor": "补录说明", "reason": reason}] + return go + + +def _derived_not_kept(r: dict[str, Any]) -> None: + seg(r, 3)["keep_as"] = None + r["tables"].pop() + + +def _derived_expect(r: dict[str, Any]) -> None: + seg(r, 3)["labels"]["expect"] = ["18-22 时合计", "18-24 时合计"] + + +def _relaxed(checks: list[str]) -> Callable[[dict[str, Any]], None]: + def go(r: dict[str, Any]) -> None: + block(r)["axis"]["checks"] = checks + return go + + +def _source_label(r: dict[str, Any]) -> None: + s = seg(r, 0) + s["labels"]["expect"][0] = "全天客流（人次）" + s["measures"] = {"全天客流（人次）": "全日客流", "分区甲（人次）": "分区甲", "分区乙（人次）": "分区乙"} + + +#: (说明, 改法, 期望列出的配方类 id)。每一行都是「改了这一项 → 只列这一项」 +FLOW_ROWS: list[tuple[str, Callable[[dict[str, Any]], None], set[str]]] = [ + ("占位符含义", lambda r: _values(r)["placeholders"][0].update(meaning="不适用"), {"placeholder:·"}), + ("新增占位符", _add_placeholder, {"placeholder:—"}), + ("空格存空值", _set(_values, "blank", "null"), {"blank_null:交叉表"}), + ("千分位", _set(_values, "text_number", "parse_thousands"), {"text_number:交叉表"}), + ("公式按保存值", _set(_values, "formula", "accept_cached"), {"formula_cached:交叉表"}), + ("缺的日期断言", _relaxed(["contiguous"]), {"checks_relaxed:交叉表"}), + ("文件名核对", _set(_ctx0, "cross_check", "none"), {"cross_check_off:客流汇总"}), + ("合计不另存", _derived_not_kept, {"derived:夜间合计"}), + ("合计标签集合", _derived_expect, {"derived:夜间合计"}), + ("关系成员", _set(_r1, "parts", ["分区乙", "全日客流"]), {"relation:R1"}), + ("关系换成不登记", lambda r: r["relations"].__setitem__(0, {"id": "R1", "kind": "dismissed", "claims": "F1", + "reason": "巧合"}), {"dismissed:F1"}), + ("单位", lambda r: r["tables"][0]["units"].update(分区甲="人"), {"unit:日客流.分区甲"}), + ("单位的来源标签", _source_label, {"unit:日客流.全日客流"}), + ("隐藏行", lambda r: r["sheets"][0].update(hidden={"rows": "exclude"}), {"hidden:客流汇总"}), + ("导入模式", lambda r: r.update(mode="accumulate"), {"mode"}), + ("按行标签忽略", _ignore_rows("表下补录，不属于任何分区"), {"ignore_rows:交叉表"}), + ("按同一行文字忽略", _ignore_outside("说明里的数字"), {"ignore_outside:客流汇总"}), + ("按表头忽略列", lambda r: block(r).update(ignore_columns=[{"header": "合计", "reason": "月合计"}]), + {"ignored_columns:交叉表"}), +] + + +@pytest.mark.parametrize("what,mutate,want", FLOW_ROWS, ids=[r[0] for r in FLOW_ROWS]) +def test_signature_row_listed_only_when_changed(what, mutate, want): + new = flow() + mutate(new) + assert recipe_ids(redraft(new)) == want + # 反例：改了的配方作为现行配方，同一份再提交（只改了签名表之外的东西）：这一项不再列出 + again = copy.deepcopy(new) + seg(again, 1)["locate"]["title"] = "日间时段客流" + assert recipe_ids(redraft(again, old=new)) == set() + + +def test_year_from_row(): + old = flow() + block(old)["axis"]["year_from"] = None + assert recipe_ids(redraft(FLOW, old=old)) == {"year_from:交叉表"} + assert recipe_ids(redraft(FLOW, old=FLOW)) == set() + + +def test_checks_relaxed_same_missing_set_not_listed_again(): + old = flow() + block(old)["axis"]["checks"] = ["contiguous"] + new = copy.deepcopy(old) + new["sheets"][0]["blocks"][0]["values"]["blank"] = "null" + assert recipe_ids(redraft(new, old=old)) == {"blank_null:交叉表"} + block(new)["axis"]["checks"] = [] # 缺的断言集合变了：再列 + assert "checks_relaxed:交叉表" in recipe_ids(redraft(new, old=old)) + + +def test_reordered_members_not_listed(): + new = flow() + _r1(new)["parts"] = ["分区乙", "分区甲"] + assert recipe_ids(redraft(new)) == set() + + +def test_reason_change_relists_ignore_rule(): + old = flow() + _ignore_rows("第一次写的理由")(old) + new = copy.deepcopy(old) + _ignore_rows("改过的理由")(new) + assert recipe_ids(redraft(new, old=old)) == {"ignore_rows:交叉表"} + + +LIST_ROWS: list[tuple[str, Callable[[dict[str, Any]], None], set[str]]] = [ + ("合并填充", lambda r: block(r).update(merged_data="fill"), {"merged_fill:列表1"}), + ("跳过空行", lambda r: block(r)["rows"].update(blank_rows="skip"), {"blank_skip:列表1"}), + ("合计行的词", lambda r: block(r)["rows"]["total_row"].update(pick="总计"), {"total_row:列表1"}), + ("多出的列", lambda r: block(r).update(extra_columns="ignore"), {"ignored_columns:列表1"}), +] + + +@pytest.mark.parametrize("what,mutate,want", LIST_ROWS, ids=[r[0] for r in LIST_ROWS]) +def test_list_signature_rows(what, mutate, want): + new = copy.deepcopy(LIST) + mutate(new) + ex = list_ex(ignored_columns={"列表1": ["备注"]}, blank_rows_skipped=1) + assert recipe_ids(redraft(new, old=LIST, ex=ex, prev={"receipt": {"tables": []}})) == want + + +# -------------------------------------------------------------------------- +# 6.2 的补充：按本期数据才出的配方类项，数据变了也照列（评审意见 H3） +# -------------------------------------------------------------------------- + + +def _list_recipe_items(new: dict[str, Any], old: dict[str, Any], ex: dict[str, Any], + prev_ex: dict[str, Any]) -> set[str]: + return recipe_ids(redraft(new, old=old, ex=ex, prev={"receipt": prev_ex}, kind="reupload")) + + +def test_newly_ignored_columns_listed_even_if_signature_same(): + """extra_columns=ignore：上一期没有多出的列（上次提交时没出 ignored_columns，从没人确认过），本期冒出「备注金额」； + 同一暂存区里另改了合并填充。只比签名会只列 merged_fill，被忽略的列没人看。""" + old = copy.deepcopy(LIST) + block(old)["extra_columns"] = "ignore" + new = copy.deepcopy(old) + block(new)["merged_data"] = "fill" + now = list_ex(ignored_columns={"列表1": ["备注金额"]}) + assert _list_recipe_items(new, old, now, list_ex()) == {"merged_fill:列表1", "ignored_columns:列表1"} + # 上一期已经忽略同样的列（match_key 相同）：签名和数据都没变，不再列 + same = list_ex(ignored_columns={"列表1": ["备注 金额"]}) + assert _list_recipe_items(new, old, now, same) == {"merged_fill:列表1"} + + +def test_list_formula_cells_appearing_listed(): + """列表的公式默认按保存值导入，有公式格才出 formula_cached。上一期 0 格、本期 12 格：列出。""" + new = copy.deepcopy(LIST) + block(new)["merged_data"] = "fill" + got = _list_recipe_items(new, LIST, list_ex(formula_cells_accepted=12), list_ex(formula_cells_accepted=0)) + assert got == {"merged_fill:列表1", "formula_cached:列表1"} + # 两期都有公式格（个数不同）：上次已经确认过「按保存值导入」，不再列 + got = _list_recipe_items(new, LIST, list_ex(formula_cells_accepted=12), list_ex(formula_cells_accepted=3)) + assert got == {"merged_fill:列表1"} + + +def _flow_prev(**receipt: Any) -> dict[str, Any]: + doc = copy.deepcopy(BASE[0]) + doc["receipt"].update(receipt) + return doc + + +def test_year_from_listed_when_header_form_changes(): + """上一期表头是完整日期（year_from 不出），本期变成「9月1日」这种写法（年份取自统计期）：列出 year_from。""" + new = flow() + _values(new)["blank"] = "null" + ex = extraction_of(sep()[0]) + prev = _flow_prev(axes=[{"block": "交叉表", "form": "date"}]) + assert recipe_ids(redraft(new, ex=ex, prev=prev, kind="reupload")) == {"blank_null:交叉表", "year_from:交叉表"} + # 两期都是文字表头：不再列 + assert recipe_ids(redraft(new, ex=ex, kind="reupload")) == {"blank_null:交叉表"} + + +def test_ignore_rows_hits_change_listed(): + """忽略规则命中从 1 行变成 9 行：配方没改时差异卡出 diff:ignored_rows:，配方改了别处时由这里兜住。""" + old = flow() + _ignore_rows("表下补录")(old) + new = copy.deepcopy(old) + _values(new)["blank"] = "null" + hit = {"sheet": SHEET, "reason": "ignored_rows", "anchor": "补录（人次）", "block": "交叉表", "cells": 3} + ex = extraction_of(sep()[0]) + ex["rows_excluded"] = [{**hit, "rows": [[32, 40]]}] + prev = _flow_prev(rows_excluded=[{**hit, "rows": [[32, 32]]}]) + assert recipe_ids(redraft(new, old=old, ex=ex, prev=prev)) == {"blank_null:交叉表", "ignore_rows:交叉表"} + same = _flow_prev(rows_excluded=[{**hit, "rows": [[30, 38]]}]) # 行号挪了、行数和锚点没变 + assert recipe_ids(redraft(new, old=old, ex=ex, prev=same)) == {"blank_null:交叉表"} + # 上一期回执没有 rows_excluded（期 3 之前的导入）：不知道上次命中几行，照列 + assert "ignore_rows:交叉表" in recipe_ids(redraft(new, old=old, ex=ex, prev=BASE[0])) + + +def test_ignore_outside_and_hidden_content_change_listed(): + old = flow() + _ignore_outside("说明里的数字")(old) + old["sheets"][0]["hidden"] = {"rows": "exclude"} + new = copy.deepcopy(old) + _values(new)["blank"] = "null" + hit = {"sheet": SHEET, "reason": "ignored_outside", "anchor": "补录说明", "block": None, "cells": 2, + "rows": [[33, 34]]} + ex = extraction_of(sep()[0]) + ex["rows_excluded"] = [hit] + ex["hidden"] = {SHEET: {"rows": [12], "cols": [], "policy_rows": "exclude", "policy_cols": "reject_if_any"}} + prev = _flow_prev(rows_excluded=[], hidden={}) + assert recipe_ids(redraft(new, old=old, ex=ex, prev=prev)) == { + "blank_null:交叉表", "ignore_outside:客流汇总", "hidden:客流汇总"} + prev = _flow_prev(rows_excluded=[hit], hidden=copy.deepcopy(ex["hidden"])) + assert recipe_ids(redraft(new, old=old, ex=ex, prev=prev)) == {"blank_null:交叉表"} + + +def test_data_signature_does_not_apply_when_recipe_unchanged(): + """配方没变时不列配方类项（期 2 起如此），数据的变化由差异卡兜住。""" + ex = extraction_of(sep()[0]) + prev = _flow_prev(axes=[{"block": "交叉表", "form": "date"}]) + assert recipe_ids(redraft(FLOW, ex=ex, prev=prev, kind="reupload")) == set() + + +def test_d04_fix_lists_only_the_fix(): + """A15：D04 修复（日间去掉「7-8」）后，配方类只剩 fix:*，没有 placeholder:·、unit:* 等没变的项。""" + new = flow() + seg(new, 1)["labels"]["expect"] = seg(new, 1)["labels"]["expect"][1:] + edits = [{"seq": 1, "kind": "fix", "key": "remove_label:日间:7-8", "title": "在分段「日间」的期望标签中去掉「7-8」", + "signed_by": "测试员甲", "superseded": False, "undoable": True}] + items = redraft(new, kind="reupload", edits=edits) + assert [i.id for i in items] == ["fix:remove_label:日间:7-8"] + fix = items[0] + assert fix.source == "edit" and fix.label == "在分段「日间」的期望标签中去掉「7-8」" + assert "署名（未认证）：测试员甲" in fix.detail + + +def test_sheet_compared_by_recipe_id_not_name(): + """⑨ 更新工作表名：匹配名变了、工作表 id 没变。按 id 换成本期实际的名字比，cross_check_off 不重复列出。""" + old = flow() + _ctx0(old)["cross_check"] = "none" + new = copy.deepcopy(old) + new["sheets"][0]["match"]["name"] = "客流汇总（新）" + ex = extraction_of(sep()[0]) + ex["sheets"]["matched"] = {"s1": "客流汇总（新）"} + assert recipe_ids(redraft(new, old=old, ex=ex)) == set() + _ctx0(new)["cross_check"] = "filename" + _ctx0(old)["cross_check"] = "filename" + new["sheets"][0]["hidden"] = {"rows": "include"} + assert recipe_ids(redraft(new, old=old, ex=ex)) == {"hidden:客流汇总（新）"} + + +def test_recipe_item_signatures_public_shape(): + sig = recipe_item_signatures(FLOW) + assert {"placeholder:·", "derived:夜间合计", "year_from:交叉表", "relation:R1", "relation:R2", "mode", + "unit:日客流.全日客流", "cross_check_off:客流汇总", "hidden:客流汇总"} <= set(sig) + assert all(isinstance(v, str) for v in sig.values()) + # 写全默认值和省略默认值是同一份配方：签名相同 + from app.data.recipe_types import Recipe + + compact = Recipe.model_validate(FLOW).model_dump(mode="json", exclude_defaults=True) + assert recipe_item_signatures(compact) == sig == recipe_item_signatures(Recipe.model_validate(FLOW)) + changed = flow(mode="accumulate") + assert {k for k in sig if recipe_item_signatures(changed)[k] != sig[k]} == {"mode"} + + +def test_first_and_switch_still_list_everything(): + ex = extraction_of(BASE[0]) + first = confirm_items(ConfirmContext(kind="first", recipe=FLOW, extraction=ex, checks=[])) + assert {i.id for i in first} == D00_CONFIRMS + mode = by_id(first)["mode"] + assert mode.detail == "每次上传新一期，当前版本只含新的一期，此前各期留在历史版本中" + acc = confirm_items(ConfirmContext(kind="first", recipe=flow(mode="accumulate"), extraction=ex, checks=[])) + assert by_id(acc)["mode"].label == "导入模式：按期累积" + assert "部分重叠的文件会被拒收" in by_id(acc)["mode"].detail + + +# -------------------------------------------------------------------------- +# 采用按规则重新起草（6.3） +# -------------------------------------------------------------------------- + + +def test_rules_redraft_lists_everything_and_redraft_adopted(): + new = flow() + seg(new, 1)["locate"]["title"] = "日间分时段客流（人次）" + new["sheets"][0]["match"]["name"] = "客流汇总表" + items = redraft(new, recipe_origin="rules_redraft") + got = by_id(items) + assert D00_CONFIRMS <= set(got) + adopted = got["redraft_adopted"] + assert adopted.source == "edit" + assert adopted.label.startswith("采用按规则重新起草的配方：") + assert "分段「日间」的标题「日间时段客流（人次）」→「日间分时段客流（人次）」" in adopted.label + assert "工作表名「客流汇总」→「客流汇总表」" in adopted.label + # 普通的手工修改：只列有变化的，没有 redraft_adopted + assert "redraft_adopted" not in by_id(redraft(new, recipe_origin="manual")) + + +def test_redraft_adopted_uses_given_compare_and_lists_labels_and_sources(): + cmp = {"segments": [{"id": "日间", "status": "changed", "title": {"old": "a", "new": "a"}, + "labels": {"added": ["6-7"], "removed": ["7-8"]}, "ignore": {"added": [], "removed": []}}], + "sheets": [], "tables": [{"name": "日客流", "columns": [ + {"name": "分区甲", "old": {"source": "分区甲（人次）"}, "new": {"source": "东区（人次）"}}]}]} + adopted = by_id(redraft(FLOW, recipe_origin="rules_redraft", redraft_compare=cmp))["redraft_adopted"] + assert "分段「日间」加入标签「6-7」" in adopted.label and "分段「日间」去掉标签「7-8」" in adopted.label + assert "表「日客流」列「分区甲」的来源「分区甲（人次）」→「东区（人次）」" in adopted.label + same = by_id(redraft(FLOW, recipe_origin="rules_redraft"))["redraft_adopted"] + assert same.label == "采用按规则重新起草的配方：分段、标签、工作表名和列的来源与现行配方相同" + + +def test_redraft_adopted_lists_title_and_labels_of_added_and_removed_segments(): + """起草器把「日间」段换成新 id「日间段」：新分段的标题和全部标签就是它新认出的定位规则，要逐项写出来（6.3）。""" + new = flow() + segs = block(new)["segments"] + renamed = copy.deepcopy(segs[1]) + renamed["id"] = "日间段" + segs[1] = renamed + adopted = by_id(redraft(new, recipe_origin="rules_redraft"))["redraft_adopted"] + labels = "".join(f"「{x}」" for x in seg(FLOW, 1)["labels"]["expect"]) + assert f"新增分段「日间段」，标题「日间时段客流（人次）」，标签{labels}" in adopted.detail + assert f"去掉分段「日间」，标题「日间时段客流（人次）」，标签{labels}" in adopted.detail + + +def test_rules_redraft_without_old_or_compare_raises(): + with pytest.raises(ValueError, match="重新起草"): + confirm_items(ConfirmContext(kind="first", recipe=FLOW, extraction=extraction_of(BASE[0]), checks=[], + recipe_origin="rules_redraft")) + + +# -------------------------------------------------------------------------- +# 修改记录 +# -------------------------------------------------------------------------- + + +def test_edit_items_fix_select_and_superseded(): + edits = [ + {"kind": "fix", "key": "add_label:日客流:分区丙（人次）", "title": "在分段「日客流」加入标签「分区丙（人次）」", + "summary": ["加入标签「分区丙（人次）」", "新增列「分区丙」，单位 人次"], "superseded": False}, + {"kind": "selection", "key": "list:列表1", "title": "按框选替换列表「列表1」：表头按文字定位", "superseded": False}, + {"kind": "fix", "key": "remove_label:日间:7-8", "title": "去掉「7-8」", "superseded": True}, + ] + got = by_id(redraft(FLOW, kind="reupload", edits=edits)) + assert set(got) == {"fix:add_label:日客流:分区丙（人次）", "select:list:列表1"} + assert got["fix:add_label:日客流:分区丙（人次）"].detail == "加入标签「分区丙（人次）」；新增列「分区丙」，单位 人次" + assert all(i.source == "edit" for i in got.values()) + + +# -------------------------------------------------------------------------- +# 按期累积（2.4、2.5、9.5） +# -------------------------------------------------------------------------- + + +def plan(action: str = "append", **kw: Any) -> dict[str, Any]: + base = AccumulatePlan( + mode="accumulate", action=action, period={"start": "2026-09-01", "end": "2026-09-30"}, + parts=[{"import_id": "i1", "seq": 1, "start": "2026-08-01", "end": "2026-08-31", "file_name": "八月.xlsx", + "new": False, "rows": {}, "blockers": []}, + {"import_id": None, "seq": None, "start": "2026-09-01", "end": "2026-09-30", "file_name": "九月.xlsx", + "new": True, "rows": {}, "blockers": []}], + replaces=None, dropped=[], overlaps=[], gaps=[], backfill=False, change="same", added=[], retired_new=[], + retired_existing=[], semantic=[], label_sets=[], mode_switch=None, reason=None, union=None) + out = {k: getattr(base, k) for k in base.__dataclass_fields__} + out.update(kw) + return out + + +def acc_items(p: Any, new: dict[str, Any] | None = None, **kw: Any) -> dict[str, ConfirmItem]: + r = new if new is not None else flow(mode="accumulate") + kw.setdefault("old", flow(mode="accumulate")) + kw.setdefault("kind", "reupload") + return by_id(redraft(r, accumulate=p, **kw)) + + +def test_append_has_no_accumulate_items_and_accepts_dataclass(): + got = acc_items(plan()) + assert not {i for i in got if i.startswith(("period_replace", "accumulate_", "retire:", "mode_switch"))} + dc = AccumulatePlan(**plan()) + assert acc_items(dc) == got + + +def test_period_replace_reupload_and_redraft(): + rep = {"import_id": "i2", "seq": 2, "start": "2026-09-01", "end": "2026-09-30", "file_name": "九月.xlsx"} + item = acc_items(plan("replace_period", replaces=rep))["period_replace:2026-09-01~2026-09-30"] + assert item.label == "替换已有的一期 2026-09-01 至 2026-09-30（第 2 次导入「九月.xlsx」）：启用后该期以本次为准" + assert item.source == "accumulate" + red = acc_items(plan("replace_period", replaces=rep), kind="redraft")["period_replace:2026-09-01~2026-09-30"] + assert red.label == "用修改后的配方重新导入最近一期 2026-09-01 至 2026-09-30：启用后该期以本次为准，其余各期不变" + with pytest.raises(ValueError, match="统计期"): + acc_items(plan("replace_period", period={}, replaces=None)) + + +def test_accumulate_restart_lists_changes_and_count(): + dropped = [{"import_id": "i1", "seq": 1, "start": "2026-08-01", "end": "2026-08-31", "blockers": []}, + {"import_id": "i2", "seq": 2, "start": "2026-09-01", "end": "2026-09-30", + "blockers": [{"path": "/tables/0", "code": "accumulate_unkeyed", "message": "表「日客流」不能按期累积"}]}] + p = plan("restart", dropped=dropped, semantic=["表「日客流」列「全日客流」的单位 人次 → 人"], change="semantic") + item = acc_items(p)["accumulate_restart"] + assert item.source == "accumulate" + assert "表「日客流」列「全日客流」的单位 人次 → 人" in item.label + assert "2026-09-01 至 2026-09-30 这一期的配方不满足按期累积的要求：表「日客流」不能按期累积" in item.label + assert "此前 2 期不再出现在当前版本中" in item.label and "数据源卡片的「版本」" in item.label + assert "如果不想重新开始累积，请返回修改配方" in item.detail + assert "accumulate_restart" not in acc_items(plan()) + + +def test_retire_only_for_retired_new_and_dedupes_breaking(): + """去掉分区乙：期 2 的 breaking:日客流（删除或改名了列）在有 retire:* 时不再重复；早已退役的不出确认项。""" + new = flow(mode="accumulate") + s = seg(new, 0) + s["labels"]["expect"] = ["全日客流（人次）", "分区甲（人次）"] + s["measures"] = {"全日客流（人次）": "全日客流", "分区甲（人次）": "分区甲"} + new["tables"][0]["units"] = {"全日客流": "人次", "分区甲": "人次"} + new["relations"] = new["relations"][1:] + p = plan(retired_new=[{"table": "日客流", "column": "分区乙"}], change="retire") + got = acc_items(p, new) + item = got["retire:日客流.分区乙"] + assert item.label == "表「日客流」的列「分区乙」自本期起不再导入：早期各期保留原值，本期起为空值" + assert item.source == "accumulate" and "两列并存" in item.detail + assert "breaking:日客流" not in got + # 没有计划（每期替换）：照期 2 出 breaking + assert "breaking:日客流" in by_id(redraft(new, old=flow(mode="accumulate"), kind="reupload")) + # 只在早已退役里：不出 retire，也不出 breaking + got = acc_items(plan(retired_existing=[{"table": "日客流", "column": "分区乙"}]), new) + assert not {i for i in got if i.startswith(("retire:", "breaking:"))} + # 退役的表 + assert "retire:时段客流_表内合计" in acc_items(plan(retired_new=[{"table": "时段客流_表内合计", "column": None}])) + # restart：结果只有本期，不出 retire，breaking 照出 + got = acc_items(plan("restart", retired_new=[{"table": "日客流", "column": "分区乙"}], semantic=["x"]), new) + assert "retire:日客流.分区乙" not in got and "breaking:日客流" in got + + +def test_mode_switch_texts(): + dropped = [{"import_id": "i1", "seq": 1, "start": "2026-08-01", "end": "2026-08-31", "blockers": []}] + a2r = acc_items(plan("replace", mode="replace", dropped=dropped, mode_switch="accumulate->replace"), + flow())["mode_switch:accumulate->replace"] + assert a2r.label == ("从按期累积改为每期替换：启用后当前版本只含本期，此前 1 期（2026-08-01 至 2026-08-31）" + "不再出现在当前版本中，可以在数据源卡片的「版本」中启用此前的版本") + r2a = acc_items(plan(mode_switch="replace->accumulate"), old=FLOW)["mode_switch:replace->accumulate"] + assert r2a.label == "从每期替换改为按期累积：当前版本的 2026-08-01 至 2026-08-31 作为第一期保留，此前被替换的各期不会补回" + no_period = [{"import_id": "i1", "seq": 1, "start": None, "end": None, "blockers": []}] + first = acc_items(plan("first", dropped=no_period, mode_switch="replace->accumulate", reason="当前版本没有统计期"), + old=FLOW)["mode_switch:replace->accumulate"] + assert "当前版本没有统计期，不能作为第一期：启用后当前版本只含本期" in first.label + blocked = [{**dropped[0], "blockers": [{"message": "交叉表「交叉表」的日期检查没有选「恰好覆盖统计期」"}]}] + first = acc_items(plan("first", dropped=blocked, mode_switch="replace->accumulate"), + old=FLOW)["mode_switch:replace->accumulate"] + assert "当前版本的配方不满足按期累积的要求（交叉表「交叉表」的日期检查没有选「恰好覆盖统计期」），不能作为第一期" in first.label + assert first.source == "accumulate" + + +def test_mode_switch_with_replace_period(): + """替换改累积走「修改配方」（redraft）或同一统计期再传：plan_accumulate 给 action=replace_period、parts 只有本期、 + dropped 为空、replaces 是当前那一期。mode_switch 要写那一期由本次重新导入后作为第一期，不能落到「没有统计期」， + 与同一清单里的 period_replace 自相矛盾（评审意见）。""" + cur = {"import_id": "i1", "seq": 1, "start": "2026-08-01", "end": "2026-08-31", "file_name": "八月.xlsx", + "new": False, "rows": {}, "blockers": []} + new_part = {"import_id": None, "seq": None, "start": "2026-08-01", "end": "2026-08-31", "file_name": "八月.xlsx", + "new": True, "rows": {}, "blockers": []} + p = plan("replace_period", period={"start": "2026-08-01", "end": "2026-08-31"}, parts=[new_part], replaces=cur, + dropped=[], mode_switch="replace->accumulate") + got = acc_items(p, old=FLOW, kind="redraft") + switch = got["mode_switch:replace->accumulate"] + assert switch.label == ("从每期替换改为按期累积：当前版本的 2026-08-01 至 2026-08-31 按修改后的配方重新导入，作为第一期，" + "此前被替换的各期不会补回") + assert "没有统计期" not in switch.label and "不能作为第一期" not in switch.label + assert got["period_replace:2026-08-01~2026-08-31"].label.startswith("用修改后的配方重新导入最近一期 2026-08-01 至") + again = acc_items(p, old=FLOW)["mode_switch:replace->accumulate"] + assert again.label == ("从每期替换改为按期累积：本次导入替换当前版本的 2026-08-01 至 2026-08-31，作为第一期，" + "此前被替换的各期不会补回") + # 计划不完整（替换该期却没有被替换的那一期、追加却没有保留的那一期）：报错，不按「没有统计期」写 + with pytest.raises(ValueError, match="被替换"): + acc_items({**p, "replaces": None}, old=FLOW) + with pytest.raises(ValueError, match="第一期"): + acc_items(plan(parts=[new_part], mode_switch="replace->accumulate"), old=FLOW) + + +def test_mode_switch_rejected_has_no_item(): + """部分重叠被拒收：不能提交，不出 mode_switch（原因在试运行的问题里）。""" + p = plan("rejected", mode_switch="replace->accumulate", + overlaps=[{"start": "2026-08-15", "end": "2026-09-14"}]) + assert "mode_switch:replace->accumulate" not in acc_items(p, old=FLOW) + + +def outside_change(old: str, new: str, cell: str = "客流汇总!B33") -> DiffItem: + return DiffItem("outside_changed", f"{cell} 的区域外文字改了：「{old}」→「{new}」", f"上一期：{old}；本期：{new}", + requires_confirm=True, confirm_id=f"diff:outside:{cell}") + + +@pytest.mark.parametrize("old,new,risky", [ + ("单位：人次", "单位：万人次", True), + ("注：含东门", "注：含东门（万人次）", True), + ("金额（元）", "金额（千元）", True), + ("统计范围：东区", "统计范围：东区、西区", True), + ("注：12% 为估算", "注：15% 为估算", True), + ("编制人：测试员甲", "编制人：测试员乙", False), + ("注：东门闸机故障", "注：西门闸机故障", False), + # 单字单位出现即算（规格 2.5），不只认紧跟数字或在括号里的写法（评审意见） + ("货运量按吨统计", "货运量按件统计", True), + ("金额：元", "金额：万", True), + ("每户", "每人", True), + ("单价 元/件", "单价 元/个", True), + ("注：按人计", "注：按户计", True), + # 停用词：含单位字、却不是在说单位 + ("填报人：测试员甲", "填报人：测试员乙", False), + ("联系人：测试员甲；审核人：测试员乙", "联系人：测试员丙；审核人：测试员丁", False), + ("填报单位：东区站", "填报单位：西区站", False), + ("附件：说明一", "附件：说明二", False), + ("注：个别日期闸机故障", "注：整个月闸机故障", False), +]) +def test_accumulate_unit_risk(old, new, risky): + got = acc_items(plan(), diff=[outside_change(old, new)]) + assert ("accumulate_unit_risk" in got) is risky + if risky: + item = got["accumulate_unit_risk"] + assert item.source == "accumulate" and "涉及单位或口径" in item.label and "返回修改配方里的单位" in item.label + + +def test_accumulate_unit_risk_scope(): + d = [outside_change("单位：人次", "单位：万人次")] + # 每期替换只影响一期：不出 + assert "accumulate_unit_risk" not in acc_items(plan("replace", mode="replace"), flow(), diff=d) + assert "accumulate_unit_risk" not in acc_items(plan("restart", semantic=["x"]), diff=d) + # 替换该期也出 + assert "accumulate_unit_risk" in acc_items(plan("replace_period", replaces={"start": "2026-09-01", + "end": "2026-09-30"}), diff=d) + # outside_diff 给了就用它，不看 diff + assert "accumulate_unit_risk" not in acc_items(plan(), diff=d, outside_diff=[]) + assert "accumulate_unit_risk" in acc_items(plan(), diff=[], outside_diff=d) + # 统计期格的写法变化（context_text）也看 + ctx = DiffItem("context_text", "统计期格 客流汇总!B2 的写法变了", "上一期：统计（人次）；本期：统计（万人次）") + assert "accumulate_unit_risk" in acc_items(plan(), diff=[ctx]) + # 分段标题的变化（配方里的 locate.title）：单位词变了才算；D09 只改了别的字、单位还是人次，不算 + new = flow(mode="accumulate") + seg(new, 1)["locate"]["title"] = "日间时段客流（万人次）" + assert "分段「日间」的标题" in acc_items(plan(), new)["accumulate_unit_risk"].label + seg(new, 1)["locate"]["title"] = "日间分时段客流（人次）" + assert "accumulate_unit_risk" not in acc_items(plan(), new) + + +def test_ignore_rule_items_with_counts(): + r = flow() + _ignore_rows("表下补录，不属于任何分区")(r) + _ignore_outside("说明里的数字")(r) + ex = extraction_of(BASE[0]) + ex["rows_excluded"] = [ + {"sheet": SHEET, "reason": "ignored_rows", "rows": [[32, 32]], "cells": 3, "anchor": "补录（人次）", + "block": "交叉表"}, + {"sheet": SHEET, "reason": "ignored_outside", "rows": [[33, 34]], "cells": 2, "anchor": "补录说明", + "block": None}] + got = by_id(confirm_items(ConfirmContext(kind="first", recipe=r, extraction=ex, checks=[]))) + assert got["ignore_rows:交叉表"].label == "「交叉表」按行标签忽略：「补录（人次）」（理由：表下补录，不属于任何分区；本期 1 行）" + assert got["ignore_outside:客流汇总"].label == ( + "工作表「客流汇总」按同一行的文字忽略导入区域之外的数字：「补录说明」（理由：说明里的数字；本期 2 行）") + assert got["ignore_rows:交叉表"].source == "recipe" + # 本期没有命中：照出，0 行；回执里没有 rows_excluded（期 3 之前）：不写行数 + ex["rows_excluded"] = [] + assert "本期 0 行" in by_id(confirm_items(ConfirmContext(kind="first", recipe=r, extraction=ex, + checks=[])))["ignore_rows:交叉表"].label + del ex["rows_excluded"] + assert "本期" not in by_id(confirm_items(ConfirmContext(kind="first", recipe=r, extraction=ex, + checks=[])))["ignore_rows:交叉表"].label + + +def test_crosstab_ignore_columns_item(): + r = flow() + block(r)["ignore_columns"] = [{"header": "合计", "reason": "右侧月合计列"}] + ex = extraction_of(BASE[0]) + ex["ignored_columns"] = {"交叉表": ["合计"]} + item = by_id(confirm_items(ConfirmContext(kind="first", recipe=r, extraction=ex, checks=[])))["ignored_columns:交叉表"] + assert item.label == "「交叉表」按表头忽略列：「合计」（理由：右侧月合计列）" and item.detail == "本期忽略的列：合计" + + +def test_sources_and_order(): + """ConfirmItem.source 的新取值；高风险的一档排在配方类、本期类、差异卡之前（9.5 的排序）。""" + new = flow(mode="accumulate") + new["tables"][0]["units"]["分区甲"] = "人" + edits = [{"kind": "fix", "key": "k", "title": "修改", "superseded": False}] + diff = [outside_change("单位：人次", "单位：万人次")] + items = redraft(new, old=flow(mode="accumulate"), kind="reupload", edits=edits, diff=diff, + accumulate=plan("replace_period", replaces={"start": "2026-09-01", "end": "2026-09-30"})) + order = [i.id for i in items] + assert order[0] == "unit_changed:日客流.分区甲" + src = {i.id: i.source for i in items} + assert src["period_replace:2026-09-01~2026-09-30"] == "accumulate" == src["accumulate_unit_risk"] + assert src["fix:k"] == "edit" and src["unit:日客流.分区甲"] == "recipe" and src["diff:outside:客流汇总!B33"] == "diff" + assert order.index("accumulate_unit_risk") < order.index("unit:日客流.分区甲") < order.index("fix:k") \ + < order.index("diff:outside:客流汇总!B33") + + +def test_plan_without_action_raises(): + with pytest.raises(ValueError, match="action"): + acc_items({"mode": "accumulate"}) diff --git a/backend/tests/test_recipe_contract.py b/backend/tests/test_recipe_contract.py index 1c0740fb..34800e71 100644 --- a/backend/tests/test_recipe_contract.py +++ b/backend/tests/test_recipe_contract.py @@ -1,8 +1,14 @@ -"""期 2 契约文件（recipe_types、recipe_parsers）的测试：只用合成字符串，不读任何文件。""" +"""期 2、期 3 契约文件（recipe_types、recipe_parsers）的测试：只用合成字符串和合成夹具（参考配方、期 2 基线、 +flow 夹具），不读任何真实文件。""" from __future__ import annotations +import dataclasses import datetime as dt +import hashlib import json +import tempfile +import typing +from dataclasses import MISSING, asdict from pathlib import Path import pytest @@ -10,6 +16,7 @@ from pydantic import ValidationError from app.data import recipe_parsers as P from app.data import recipe_types as T +from app.data.recipe import canonical_recipe, recipe_sha256 FLOW = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text(encoding="utf-8")) @@ -233,3 +240,424 @@ def test_axis_checks_are_a_set(): a = T.Axis.model_validate({"checks": ["covers_context", "contiguous", "contiguous"]}) assert a.checks == ["contiguous", "covers_context"] assert a.model_dump(exclude_defaults=True) == {} + + +# ========================================================================== +# 期 3 契约（P3-SPEC 9.2、12.1 WP-0）：新字段的默认值等于旧行为、语言仍封闭、资格函数、字段名钉住 +# ========================================================================== + +#: 参考配方的哈希（期 2 实测值）：期 3 加了字段也不能变，期 2 的构建 id 才不失效 +REF_RECIPE_SHA = "971a8eba5ac606bd0552c43ff83b5c361d134c77d471bfe2d28e440ebfcc07b4" +P2_BASELINE = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "p2_baseline.json").read_text(encoding="utf-8")) + + +def _flow(**changes) -> dict: + data = json.loads(json.dumps(FLOW)) + data.update(changes) + return data + + +def test_reference_recipe_hash_unchanged_with_p3_fields(): + recipe = T.Recipe.model_validate(FLOW) + assert recipe_sha256(recipe) == REF_RECIPE_SHA == P2_BASELINE["recipe"]["recipe_sha256"] + # 完整形式里新字段都在、都取默认值；再解析一遍哈希不变（新字段的默认值不进 canonical） + full = recipe.model_dump(mode="json") + sheet, block = full["sheets"][0], full["sheets"][0]["blocks"][0] + assert sheet["ignore_outside"] == [] and block["ignore_rows"] == [] and block["ignore_columns"] == [] + assert recipe_sha256(T.Recipe.model_validate(full)) == REF_RECIPE_SHA + canon = json.dumps(canonical_recipe(recipe), ensure_ascii=False) + assert "ignore_" not in canon + # 用了新字段，哈希才变 + full["sheets"][0]["blocks"][0]["ignore_rows"] = [{"label": "补录（人次）", "reason": "补录数据另行核对"}] + assert recipe_sha256(T.Recipe.model_validate(full)) != REF_RECIPE_SHA + + +def test_list_block_ignore_columns_default_keeps_hash(): + data = json.loads(json.dumps(C01_LIKE)) + base = recipe_sha256(T.Recipe.model_validate(data)) + data["sheets"][0]["blocks"][0]["ignore_columns"] = [] + data["sheets"][0]["ignore_outside"] = [] + assert recipe_sha256(T.Recipe.model_validate(data)) == base + data["sheets"][0]["blocks"][0]["ignore_columns"] = [{"header": "备注", "reason": "备注列不导入"}] + parsed = T.Recipe.model_validate(data) + assert parsed.sheets[0].blocks[0].ignore_columns[0].header == "备注" + assert recipe_sha256(parsed) != base + + +@pytest.mark.parametrize("where,key,item", [ + ("block", "ignore_rows", {"label": "补录（人次）", "reason": "补录"}), + ("block", "ignore_columns", {"header": "合计", "reason": "右侧合计列"}), + ("sheet", "ignore_outside", {"anchor": "补录说明", "reason": "说明行"}), +]) +def test_ignore_rules_parse_and_are_closed(where, key, item): + def put(value): + data = json.loads(json.dumps(FLOW)) + target = data["sheets"][0]["blocks"][0] if where == "block" else data["sheets"][0] + target[key] = value + return data + + T.Recipe.model_validate(put([item])) + with pytest.raises(ValidationError): # extra="forbid"：不能多写字段（如坐标、正则） + T.Recipe.model_validate(put([{**item, "cell": "B32"}])) + with pytest.raises(ValidationError): + T.Recipe.model_validate(put([{**item, "regex": ".*"}])) + first = next(iter(item)) + with pytest.raises(ValidationError): # 锚点文字不能为空 + T.Recipe.model_validate(put([{**item, first: ""}])) + with pytest.raises(ValidationError): # 理由必填 + T.Recipe.model_validate(put([{k: v for k, v in item.items() if k != "reason"}])) + with pytest.raises(ValidationError): + T.Recipe.model_validate(put([{**item, "reason": "x" * 201}])) + limit = 20 if key == "ignore_outside" else 50 + T.Recipe.model_validate(put([item] * limit)) + with pytest.raises(ValidationError): + T.Recipe.model_validate(put([item] * (limit + 1))) + + +def test_ignore_rule_lengths(): + T.IgnoreRow(label="x" * 40, reason="r") + T.IgnoreColumn(header="x" * 80, reason="r") + T.IgnoreOutside(anchor="x" * 40, reason="r") + for bad in (lambda: T.IgnoreRow(label="x" * 41, reason="r"), + lambda: T.IgnoreColumn(header="x" * 81, reason="r"), + lambda: T.IgnoreOutside(anchor="x" * 41, reason="r")): + with pytest.raises(ValidationError): + bad() + + +def test_ignore_fields_only_where_the_spec_puts_them(): + """ignore_rows 只在交叉表块上；列表块只有 ignore_columns；ignore_outside 只在工作表上。""" + data = json.loads(json.dumps(C01_LIKE)) + data["sheets"][0]["blocks"][0]["ignore_rows"] = [{"label": "合计", "reason": "r"}] + with pytest.raises(ValidationError): + T.Recipe.model_validate(data) + data = json.loads(json.dumps(FLOW)) + data["sheets"][0]["blocks"][0]["ignore_outside"] = [{"anchor": "注", "reason": "r"}] + with pytest.raises(ValidationError): + T.Recipe.model_validate(data) + data = json.loads(json.dumps(FLOW)) + data["sheets"][0]["ignore_columns"] = [{"header": "合计", "reason": "r"}] + with pytest.raises(ValidationError): + T.Recipe.model_validate(data) + + +# ---- accumulate_blockers（P3-SPEC 2.2） + +#: c01 规则草稿的形状（列表块，表头 地区 产品 销量 金额，合计行另存；没有可解析的统计期区间） +C01_LIKE = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{ + "id": "s1", "match": {"name": "月报"}, + "blocks": [{ + "id": "列表1", "layout": "list", "table": "月报", + "columns": [{"header": "地区", "name": "地区", "type": "TEXT"}, + {"header": "产品", "name": "产品", "type": "TEXT"}, + {"header": "销量", "name": "销量", "type": "INTEGER"}, + {"header": "金额", "name": "金额", "type": "INTEGER"}], + "rows": {"total_row": {"label_column": "地区", "pick": "合计", "keep_as": "月报_表内合计"}}, + }], + }], + "tables": [{"name": "月报", "grain": ["地区", "产品"], "units": {"金额": "万元"}}, + {"name": "月报_表内合计", "grain": ["合计项"], "kind": "reported_total", "units": {"金额": "万元"}}], +} + + +def test_accumulate_blockers_reference_recipe_is_eligible(): + assert T.accumulate_blockers(T.Recipe.model_validate(_flow(mode="accumulate"))) == [] + # 不看 mode：替换模式的同一份配方也符合资格（起草器据此决定写不写 mode=accumulate） + assert T.accumulate_blockers(T.Recipe.model_validate(FLOW)) == [] + + +def test_accumulate_blockers_list_shape_is_unkeyed(): + data = json.loads(json.dumps(C01_LIKE)) + data["sheets"][0]["context"] = [{}] # 即使配了统计期，列表写入的表也不能累积 + got = T.accumulate_blockers(T.Recipe.model_validate(data)) + assert [(p.code, p.path) for p in got] == [("accumulate_unkeyed", "/tables/0"), ("accumulate_unkeyed", "/tables/1")] + assert all("列表形态的表目前只支持每期替换" in p.message for p in got) + assert "「月报」" in got[0].message and "「月报_表内合计」" in got[1].message + # c01 的草稿本来就没有统计期：两条都报，统计期那条在前（起草卡片取第一条的人话） + got = T.accumulate_blockers(T.Recipe.model_validate(C01_LIKE)) + assert [p.code for p in got] == ["accumulate_needs_period", "accumulate_unkeyed", "accumulate_unkeyed"] + + +def test_accumulate_blockers_needs_period(): + data = _flow(mode="accumulate") + data["sheets"][0]["context"] = [] + got = T.accumulate_blockers(T.Recipe.model_validate(data)) + assert [(p.code, p.path) for p in got] == [("accumulate_needs_period", "/mode")] + assert "统计期" in got[0].message + + +def test_accumulate_blockers_axis_checks_and_grain(): + data = _flow(mode="accumulate") + data["sheets"][0]["blocks"][0]["axis"]["checks"] = ["contiguous"] + got = T.accumulate_blockers(T.Recipe.model_validate(data)) + assert [(p.code, p.path) for p in got] == [("accumulate_unkeyed", f"/tables/{i}") for i in range(3)] + assert all("恰好覆盖统计期" in p.message and "交叉表「交叉表」" in p.message for p in got) + + data = _flow(mode="accumulate") + data["tables"][1]["grain"] = ["时段"] + got = T.accumulate_blockers(T.Recipe.model_validate(data)) + assert [(p.code, p.path) for p in got] == [("accumulate_unkeyed", "/tables/1")] + assert "「时段客流」" in got[0].message and "主键不含日期列「日期」" in got[0].message + + +def test_accumulate_blockers_messages_follow_copy_rules(): + data = json.loads(json.dumps(C01_LIKE)) + data["tables"][0]["grain"] = [] + msgs = [p.message for p in T.accumulate_blockers(T.Recipe.model_validate(data))] + for m in msgs: + assert not any(w in m for w in ("跑", "后端", "放行", "钉", "accumulate", "covers_context")), m + + +# ---- 新字段的默认值等于旧行为 + +def test_new_fields_default_to_old_behaviour(): + assert T.RegionMark("s", "value", "C5").block is None + assert T.Problem("x", "structure", "m").fix_args is None + assert T.Extraction(ok=True).rows_excluded == [] + assert T.SchemaNotes().fragments == {} + assert T.ConfirmItem("mode", "导入模式").source == "recipe" + assert T.RECIPE_ENGINE_VER == "recipe/1" and T.UNION_VER == "union/1" + # 旧回执里没有 rows_excluded / block / fix_args：asdict 之后多出来的键都取默认值 + ex = asdict(T.Extraction(ok=True, regions=[T.RegionMark("s", "value", "C5")], + problems=[T.Problem("x", "structure", "m")])) + assert ex["rows_excluded"] == [] and ex["regions"][0]["block"] is None and ex["problems"][0]["fix_args"] is None + + +def test_region_mark_json_always_carries_block(): + """RegionMark 的 JSON 一律带 block 键（None 也带）。期 2 的 test_recipe_imports.py 的 + test_stage_first_does_not_create_the_source_and_keeps_the_raw 逐字比 marks、不含 block，WP-0 合并后会失败； + 那个文件归 WP-5（P3-SPEC 12.0），由 WP-5 改期望值。这条测试防止有人为了让那条期 2 断言通过而去掉这个字段 + 或者改成不输出 None。""" + assert asdict(T.RegionMark("甲表", "value", "A2:B3")) == {"sheet": "甲表", "role": "value", "ref": "A2:B3", + "block": None} + assert asdict(T.RegionMark("甲表", "value", "A2:B3", block="列表1"))["block"] == "列表1" + + +def test_trial_receipt_view_carries_rows_excluded_once_wp5_lands(): + """评审：TrialOut.receipt 只抄 recipe_imports._RECEIPT_VIEW 里的键。不把 rows_excluded 加进去，向导回执就一律 + 显示「这次导入未记录排除的行」。这一行归 WP-5（recipe_imports.py），所以这条测试在 WP-5 合并前跳过： + 以 Pipeline 有没有 propose_fixes 字段（WP-5 按 P3-SPEC 12.6 加的接缝）判断 WP-5 是否已合并。""" + from app.data import recipe_imports + + if "propose_fixes" not in {f.name for f in dataclasses.fields(recipe_imports.Pipeline)}: + pytest.skip("WP-5 还没有合并：试运行回执的白名单由 WP-5 补 rows_excluded") + rows = [asdict(T.ExcludedRows("甲表", "blank_skipped", [[5, 5]], 2, block="列表1"))] + view = recipe_imports._receipt_view({"tables": [], "rows_excluded": rows}, None) + assert view.get("rows_excluded") == rows, "TrialOut.receipt 缺 rows_excluded：把它加进 recipe_imports._RECEIPT_VIEW" + + +def test_edit_result_breaking_is_internal_bool(): + """EditResult.breaking 是布尔；EditPreview JSON 的 breaking 是「表名 → 人话列表」，由 WP-5 另算并覆盖（9.4）。 + 这里只钉住契约这一侧的类型，免得有人把两者改成同一个值。""" + assert typing.get_type_hints(T.EditResult)["breaking"] is bool + assert T.EditResult(True, "fix", "k", "t", [], [], [], [], []).breaking is False + + +def test_snapshot_not_activatable_codes_match_versions_text(): + """SnapshotOut.reason_code 的取值与文字（契约补充，7.1 只有人话 reason）。文字与附录 C 的 notActivatable + 逐字相同：界面按 code 取 VERSIONS_TEXT（types.ts 的 SnapshotReasonCode 用 satisfies 钉住键），没有 code 时 + 显示 reason，两条路显示一样。顺序即几种同时成立时的优先顺序。""" + assert typing.get_args(T.SnapshotReasonCode) == ("current", "retired", "file_lost", "contains_revoked") + assert list(T.SNAPSHOT_NOT_ACTIVATABLE) == list(typing.get_args(T.SnapshotReasonCode)) + assert T.SNAPSHOT_NOT_ACTIVATABLE == { + "current": "已是当前版本", + "retired": "已回收，无法启用", + "file_lost": "数据文件已丢失，无法启用", + "contains_revoked": "包含已作废接受的导入，无法启用", + } + + +def _literal(cls, name: str) -> tuple: + return typing.get_args(typing.get_type_hints(cls)[name]) + + +def test_closed_value_sets(): + assert _literal(T.ConfirmItem, "source") == ( + "recipe", "diff", "outside", "sheet", "context", "switch", "edit", "accumulate") + assert {"union_rows", "union_pk", "union_period"} <= set(_literal(T.CheckResult, "kind")) + assert "ignored" in typing.get_args(T.Role) and "ignored_column" in typing.get_args(T.Role) + assert typing.get_args(T.ExcludedReason) == ( + "hidden_excluded", "blank_skipped", "ignored_rows", "ignored_outside", "after_stop", "total_not_kept") + assert _literal(T.FixAnchor, "kind") == ("problem", "recipe_problem", "sheet_renamed") + assert _literal(T.Anchor, "kind") == ( + "header", "row_label", "section_title", "total_word", "axis", "after_title", "outside_text") + assert _literal(T.EditResult, "kind") == ("fix", "selection") + assert _literal(T.ChangeClass, "change") == ("same", "compatible", "retire", "semantic") + assert T.FIX_KINDS == ("remove_label", "add_label", "edit_members", "rename_title", "declare_total", + "ignore_cells", "declare_placeholder", "declare_hidden", "rename_sheet") + assert T.SELECTION_AS == ("list", "crosstab", "segment", "derived", "section_title", "ignore_rows", + "ignore_columns", "ignore_outside") + assert T.ACCUMULATE_ACTIONS == ("replace", "first", "append", "replace_period", "restart", "rejected") + + +# ---- Selection：JSON 里的键是 as + +def test_selection_json_key_is_as(): + body = {"sheet": "月报", "ref": "C5:F8", "as": "list", "options": {"header_rows": 1, "bottom": "box"}} + sel = T.Selection.from_json(body) + assert (sel.sheet, sel.ref, sel.as_, sel.options) == ("月报", "C5:F8", "list", {"header_rows": 1, "bottom": "box"}) + out = sel.to_json() + assert out == body and "as_" not in out + assert T.Selection.from_json({"sheet": "月报", "ref": "C5", "as": "ignore_rows"}).options == {} + assert T.Selection.from_json({"sheet": "月报", "ref": "C5", "as": "ignore_rows", "options": None}).options == {} + # 写成 as_ 的请求不认（JSON 里的键是 as）：和别的形状错误一样抛 ValueError + with pytest.raises(ValueError): + T.Selection.from_json({"sheet": "月报", "ref": "C5", "as_": "list"}) + # 不经 to_json 直接 asdict 得到的是 as_：接口层必须用 to_json + assert "as_" in asdict(sel) + + +@pytest.mark.parametrize("body", [ + None, [], ["月报", "C5", "list"], "月报", + {"ref": "C5", "as": "list"}, {"sheet": "", "ref": "C5", "as": "list"}, {"sheet": " ", "ref": "C5", "as": "list"}, + {"sheet": 3, "ref": "C5", "as": "list"}, + {"sheet": "月报", "as": "list"}, {"sheet": "月报", "ref": "", "as": "list"}, {"sheet": "月报", "ref": 5, "as": "list"}, + {"sheet": "月报", "ref": "月报!C5", "as": "list"}, {"sheet": "月报", "ref": "c5", "as": "list"}, + {"sheet": "月报", "ref": "$C$5", "as": "list"}, {"sheet": "月报", "ref": "C0", "as": "list"}, + {"sheet": "月报", "ref": "C5:", "as": "list"}, {"sheet": "月报", "ref": "C:F", "as": "list"}, + {"sheet": "月报", "ref": "C5", "as": "table"}, {"sheet": "月报", "ref": "C5", "as": None}, + {"sheet": "月报", "ref": "C5", "as": ["list"]}, + {"sheet": "月报", "ref": "C5", "as": "list", "options": "x"}, + {"sheet": "月报", "ref": "C5", "as": "list", "options": [1]}, + {"sheet": "月报", "ref": "C5", "as": "list", "options": [["header_rows", 1]]}, + {"sheet": "月报", "ref": "C5", "as": "list", "options": 1}, +]) +def test_selection_from_json_rejects_malformed_with_value_error_only(body): + """评审：options 写成字符串、列表时原来抛的是 dict() 的 ValueError / TypeError，文档却只说 KeyError，接口层照 + 文档只接 KeyError 的话这些请求会变成 500。现在任何形状不对都只抛 ValueError（不是 KeyError、TypeError 的 + 子类），消息是给人看的中文，接口层 except ValueError → 422 edit_invalid。""" + with pytest.raises(ValueError) as exc: + T.Selection.from_json(body) + assert type(exc.value) is ValueError + assert any("一" <= ch <= "鿿" for ch in str(exc.value)) + for word in ("sheet", "ref", "options", "as_", "SELECTION_AS"): + assert word not in str(exc.value) + + +@pytest.mark.parametrize("ref", ["C5", "C5:F8", "AA10:AB12", "XFD1048576", "A1:A1"]) +def test_selection_from_json_accepts_a1_ranges(ref): + for as_ in T.SELECTION_AS: + assert T.Selection.from_json({"sheet": "月报", "ref": ref, "as": as_}).to_json() == { + "sheet": "月报", "ref": ref, "as": as_, "options": {}} + + +# ---- 9.2 每个 dataclass 的字段名、顺序和默认值（防止实现者改名；REQ = 必填） + +REQ = "<必填>" +FACTORY = "<默认工厂>" + + +def _shape(cls) -> list[tuple[str, object]]: + out = [] + for f in dataclasses.fields(cls): + if f.default is not MISSING: + out.append((f.name, f.default)) + elif f.default_factory is not MISSING: + out.append((f.name, FACTORY)) + else: + out.append((f.name, REQ)) + return out + + +CONTRACT_SHAPES = { + "ExcludedRows": [("sheet", REQ), ("reason", REQ), ("rows", REQ), ("cells", REQ), ("anchor", None), + ("block", None)], + "NoteFragment": [("key", REQ), ("subject", REQ), ("template", REQ)], + "FixOption": [("value", REQ), ("label", REQ), ("detail", ""), ("needs_reason", False), ("breaking", False)], + "FixAnchor": [("kind", REQ), ("index", None), ("sheet", None)], + "FixProposal": [("id", REQ), ("kind", REQ), ("problem_code", REQ), ("title", REQ), ("cells", REQ), + ("target", REQ), ("options", REQ), ("anchor", REQ)], + "Selection": [("sheet", REQ), ("ref", REQ), ("as_", REQ), ("options", FACTORY)], + "Anchor": [("kind", REQ), ("text", REQ), ("cell", None)], + "EditResult": [("ok", REQ), ("kind", REQ), ("key", REQ), ("title", REQ), ("summary", REQ), ("ops", REQ), + ("anchors", REQ), ("notes", REQ), ("problems", REQ), ("recipe_sha256_after", None), + ("expected", None), ("block", None), ("breaking", False)], + "ReplayCompare": [("expected", REQ), ("actual", REQ), ("match", REQ), ("diffs", REQ), ("window_rows", None)], + "ChangeClass": [("change", REQ), ("added", REQ), ("retired_new", REQ), ("retired_existing", REQ), + ("semantic", REQ), ("label_sets", REQ)], + "AccumulatePlan": [(n, REQ) for n in ( + "mode", "action", "period", "parts", "replaces", "dropped", "overlaps", "gaps", "backfill", "change", + "added", "retired_new", "retired_existing", "semantic", "label_sets", "mode_switch", "reason", "union")], + "PartInfo": [(n, REQ) for n in ( + "import_id", "seq", "start", "end", "build_id", "db_path", "db_sha256", "manifest_db_sha256", "recipe_id", + "recipe", "recipe_sha256", "raw_state", "file_name", "manifest_artifact")] + + [("rows", FACTORY), ("overrides", 0), ("waivers", 0)], + "UnionPart": [("import_id", REQ), ("start", REQ), ("end", REQ), ("db_path", REQ), ("recipe", REQ), + ("db_sha256", REQ), ("manifest_db_sha256", None), ("table_hashes", FACTORY)], + "UnionReport": [(n, REQ) for n in ( + "tables", "grains", "rows", "part_rows", "table_hashes", "null_counts", "part_null_counts", "added", + "retired", "checks", "ok", "db_sha256")], + "UnionNotePart": [(n, REQ) for n in ( + "recipe", "extraction", "checks", "acceptances", "null_counts", "start", "end")], +} + + +@pytest.mark.parametrize("name", sorted(CONTRACT_SHAPES)) +def test_p3_dataclass_shapes(name): + assert _shape(getattr(T, name)) == CONTRACT_SHAPES[name] + + +def test_p3_fields_added_to_p2_dataclasses(): + """期 2 已有的 dataclass 只在末尾追加字段（位置构造照旧成立），新字段都有默认值。""" + assert _shape(T.RegionMark)[-1] == ("block", None) and [n for n, _ in _shape(T.RegionMark)] == [ + "sheet", "role", "ref", "block"] + assert _shape(T.Problem)[-2:] == [("fix", None), ("fix_args", None)] + assert _shape(T.Extraction)[-1] == ("rows_excluded", FACTORY) + assert _shape(T.SchemaNotes) == [("tables", FACTORY), ("templates", FACTORY), ("problems", FACTORY), + ("kinds", FACTORY), ("fragments", FACTORY)] + assert _shape(T.ConfirmItem) == [("id", REQ), ("label", REQ), ("detail", ""), ("required", True), + ("source", "recipe")] + + +def test_p3_json_shapes_match_spec_examples(): + """asdict 出来的键与 P3-SPEC 9.4 的 JSON 示例一致（dataclass 字段与 JSON 同名）。""" + proposal = T.FixProposal( + id="fx-1a2b3c4d5e6f", kind="remove_label", problem_code="label_missing", + title="在分段「日间」的期望标签中去掉「7-8」", cells=["客流汇总!B10"], + target={"segment": "日间", "labels": ["7-8"]}, + options=[T.FixOption("remove", "去掉标签「7-8」", detail="今后各期如果又出现「7-8」，会再次拒收")], + anchor=T.FixAnchor("problem", 0)) + d = asdict(proposal) + assert list(d) == ["id", "kind", "problem_code", "title", "cells", "target", "options", "anchor"] + assert d["options"][0] == {"value": "remove", "label": "去掉标签「7-8」", "needs_reason": False, "breaking": False, + "detail": "今后各期如果又出现「7-8」，会再次拒收"} + assert d["anchor"] == {"kind": "problem", "index": 0, "sheet": None} + plan_keys = {"mode", "action", "period", "parts", "replaces", "dropped", "overlaps", "gaps", "backfill", "change", + "added", "retired_new", "retired_existing", "semantic", "label_sets", "mode_switch", "reason", "union"} + assert {f.name for f in dataclasses.fields(T.AccumulatePlan)} == plan_keys + replay = asdict(T.ReplayCompare({"header": "C5:F5"}, {"header": "C5:F5"}, True, [])) + assert set(replay) == {"expected", "actual", "match", "diffs", "window_rows"} + ex_rows = asdict(T.ExcludedRows("客流汇总", "ignored_rows", [[32, 32]], 3, anchor="补录（人次）", block="交叉表")) + assert ex_rows == {"sheet": "客流汇总", "reason": "ignored_rows", "rows": [[32, 32]], "cells": 3, + "anchor": "补录（人次）", "block": "交叉表"} + + +# ---- 期 2 基线（WP-1 的测试常量；这里先确认现在的代码能复现，免得基线本身算错） + +def test_p2_baseline_reproduces(): + from app.data import raw_store, recipe_engine, table_versions, xlsx_scan + from tests.fixtures.xlsx.flow import flow_workbook + + raw, name = flow_workbook(dt.date(2026, 8, 1), 31) + assert name == P2_BASELINE["fixture"]["file_name"] + assert hashlib.sha256(raw).hexdigest() == P2_BASELINE["fixture"]["raw_sha256"] + with tempfile.TemporaryDirectory() as d: + db = Path(d) / "d00.db" + ex = recipe_engine.execute(T.Recipe.model_validate(FLOW), raw, name, xlsx_scan.scan(raw), str(db)) + table_versions.settle_journal(db) + assert ex.ok + assert raw_store.sha256_file(db) == P2_BASELINE["db_sha256"] + assert ex.table_hashes == P2_BASELINE["table_hashes"] + assert {t.name: t.rows for t in ex.tables} == P2_BASELINE["rows"] + ids = P2_BASELINE["ids"] + raw_sha = P2_BASELINE["fixture"]["raw_sha256"] + assert table_versions.recipe_build_id(ids["source_id"], raw_sha, REF_RECIPE_SHA, {}) == ids["recipe_build_id"] + # 参考配方（交叉表）不带执行器语义标记（WP-8 评审意见 1）：提交路径按配方算出的构建 id 仍是基线记的那个 + marks = table_versions.recipe_engine_semantics(T.Recipe.model_validate(FLOW)) + assert marks == {} + assert table_versions.recipe_build_id(ids["source_id"], raw_sha, REF_RECIPE_SHA, {}, + semantics=marks) == ids["recipe_build_id"] + assert table_versions.snapshot_id(ids["source_id"], ids["import_ids"], T.RECIPE_ENGINE_VER) == ids["snapshot_id_recipe"] + assert table_versions.snapshot_id(ids["source_id"], ids["import_ids"]) == ids["snapshot_id_simple"] diff --git a/backend/tests/test_recipe_diff_p3.py b/backend/tests/test_recipe_diff_p3.py new file mode 100644 index 00000000..6cede926 --- /dev/null +++ b/backend/tests/test_recipe_diff_p3.py @@ -0,0 +1,243 @@ +"""期 3 的差异卡（recipe_diff，P3-SPEC 9.5、第 8 节遗留项 4）：按期累积计划的差异项、按配方忽略的行。 + +回执手写（构造器在 test_recipe_diff），累积计划按契约 AccumulatePlan 的 JSON 形状手写。 +""" +from __future__ import annotations + +import copy +from typing import Any + +import pytest + +from app.data import recipe_diff as D +from app.data.recipe_confirm import ConfirmContext, confirm_items +from app.data.recipe_types import AccumulatePlan +from tests.test_recipe_confirm_p3 import plan +from tests.test_recipe_diff import BASE, FLOW, SHEET, add_outside, extraction_of, kinds, run, sep + + +def by_kind(items: list[Any], kind: str) -> list[Any]: + return [d for d in items if d.kind == kind] + + +# -------------------------------------------------------------------------- +# 按期累积的计划 +# -------------------------------------------------------------------------- + + +def test_append_period_added_info(): + diff = run(BASE, sep(), accumulate=plan()) + item, = by_kind(diff, "period_added") + assert item.label == "按期累积：本期 2026-09-01 至 2026-09-30 加入当前版本，启用后共 2 期" + assert not item.requires_confirm + assert kinds(diff) == {"period", "rows", "period_added"} + + +def test_replace_period_is_info_and_confirmed_elsewhere(): + rep = {"start": "2026-09-01", "end": "2026-09-30", "seq": 2} + item, = by_kind(run(BASE, sep(), accumulate=plan("replace_period", replaces=rep)), "period_replaced") + assert item.label == "按期累积：替换已有的一期 2026-09-01 至 2026-09-30" and not item.requires_confirm + + +def test_gap_requires_confirmation(): + p = plan(period={"start": "2026-10-01", "end": "2026-10-31"}, gaps=[{"start": "2026-09-01", "end": "2026-09-30"}]) + item, = by_kind(run(BASE, sep(), accumulate=p), "period_gap") + assert item.requires_confirm and item.confirm_id == "diff:period_gap:2026-09-01~2026-09-30" + assert "2026-09-01 至 2026-09-30 没有数据" in item.label + assert not by_kind(run(BASE, sep(), accumulate=plan()), "period_gap") + + +def test_backfill_info_unless_period_entered_by_hand(): + p = plan(period={"start": "2026-07-01", "end": "2026-07-31"}, backfill=True) + info, = by_kind(run(BASE, sep(), accumulate=p), "period_backfill") + assert not info.requires_confirm and info.confirm_id is None and "早于已有各期" in info.label + cur = sep() + cur[0]["period"] = {**cur[0]["period"], "source": "human"} + item, = by_kind(run(BASE, cur, accumulate=p), "period_backfill") + assert item.requires_confirm and item.confirm_id == "diff:period_backfill:2026-07-01~2026-07-31" + assert item.label == "本期统计期为人工录入，且早于已有各期：请核对年份和月份无误" + + +def test_columns_added_retired_and_labels_vary_are_info(): + ls = [{"table": "时段客流", "column": "时段", "segment": "日间", + "periods": [{"start": "2026-08-01", "end": "2026-08-31", "missing": [], "extra": ["7-8"]}, + {"start": "2026-09-01", "end": "2026-09-30", "missing": ["7-8"], "extra": []}]}] + p = plan(added=[{"table": "日客流", "column": "分区丙"}], retired_new=[{"table": "日客流", "column": "分区乙"}], + label_sets=ls, change="retire") + diff = run(BASE, sep(), accumulate=p) + added, = by_kind(diff, "columns_added") + retired, = by_kind(diff, "columns_retired") + vary, = by_kind(diff, "labels_vary") + assert "表「日客流」的列「分区丙」" in added.label and "其他各期没有这些数据，为空值" in added.label + assert "表「日客流」的列「分区乙」" in retired.label + assert vary.label == "与其他各期相比，本期「时段」没有「7-8」" + assert vary.detail == "表「时段客流」：跨期比较时各期覆盖的取值不同；其他各期：2026-08-01 至 2026-08-31 取值齐全" + assert not any(d.requires_confirm for d in (added, retired, vary)) + + +def test_labels_vary_is_relative_to_all_other_periods_not_earlier_ones(): + """label_sets 的 missing / extra 相对全部各期（评审意见）：8 月没有「7-8」，9 月、10 月有。上传 10 月时 extra 是 + 「7-8」，但 9 月也有它，不能写「此前各期没有」；细节按期写出是哪一期缺。补传早期时同样不写「此前」。""" + ls = [{"table": "时段客流", "column": "时段", "segment": "日间", + "periods": [{"start": "2026-08-01", "end": "2026-08-31", "missing": ["7-8"], "extra": []}, + {"start": "2026-09-01", "end": "2026-09-30", "missing": [], "extra": ["7-8"]}, + {"start": "2026-10-01", "end": "2026-10-31", "missing": [], "extra": ["7-8"]}]}] + p = plan(period={"start": "2026-10-01", "end": "2026-10-31"}, label_sets=ls) + vary, = by_kind(run(BASE, sep(), accumulate=p), "labels_vary") + assert vary.label == "与其他各期相比，本期「时段」有「7-8」，但并非各期都有" + assert "此前" not in vary.label + vary.detail + assert vary.detail.endswith("其他各期：2026-08-01 至 2026-08-31 没有「7-8」；2026-09-01 至 2026-09-30 取值齐全") + back = plan(period={"start": "2026-08-01", "end": "2026-08-31"}, label_sets=ls, backfill=True) + vary, = by_kind(run(BASE, sep(), accumulate=back), "labels_vary") + assert vary.label == "与其他各期相比，本期「时段」没有「7-8」" and "此前" not in vary.detail + + +@pytest.mark.parametrize("p", [ + plan("restart", semantic=["x"]), plan("first"), plan("replace", mode="replace"), plan("rejected"), + plan(mode="replace"), +]) +def test_no_accumulate_items_unless_appending_or_replacing_a_period(p): + p = {**p, "gaps": [{"start": "2026-09-01", "end": "2026-09-30"}], "added": [{"table": "日客流", "column": "x"}]} + assert kinds(run(BASE, sep(), accumulate=p)) == {"period", "rows"} + + +def test_accumulate_items_even_when_recipe_changed_and_dataclass_accepted(): + p = plan(gaps=[{"start": "2026-09-01", "end": "2026-09-30"}]) + diff = run(BASE, sep(), accumulate=AccumulatePlan(**p), recipe_changed=True) + assert kinds(diff) == {"period", "rows", "period_added", "period_gap"} + # 需确认的排在前面 + assert diff[0].kind == "period_gap" + + +def test_new_kinds_registered(): + for k in ("period_added", "period_replaced", "period_gap", "period_backfill", "columns_added", "columns_retired", + "labels_vary", "ignored_rows"): + assert k in D.DIFF_KINDS + assert D.CONFIRM_PREFIX["period_gap"] == "diff:period_gap:" + assert D.CONFIRM_PREFIX["ignored_rows"] == "diff:ignored_rows:" + assert D.CONFIRM_PREFIX["ignored_rows"] != D.CONFIRM_PREFIX["ignored_columns"] + assert "rows_excluded" not in D.RECEIPT_KEYS + + +# -------------------------------------------------------------------------- +# 按配方忽略的行（rows_excluded） +# -------------------------------------------------------------------------- + + +def excl(reason: str = "ignored_rows", rows: list[list[int]] | None = None, anchor: str = "补录（人次）", + sheet: str = SHEET, block: str | None = "交叉表") -> dict[str, Any]: + return {"sheet": sheet, "reason": reason, "rows": rows or [[32, 32]], "cells": 3, "anchor": anchor, "block": block} + + +def with_excluded(doc: tuple[dict[str, Any], str], entries: list[dict[str, Any]] | None) -> tuple[dict[str, Any], str]: + d = copy.deepcopy(doc) + if entries is None: + d[0]["receipt"].pop("rows_excluded", None) + else: + d[0]["receipt"]["rows_excluded"] = entries + return d + + +def test_ignored_rows_change_requires_confirmation(): + prev = with_excluded(BASE, []) + cur = with_excluded(sep(), [excl()]) + item, = by_kind(run(prev, cur), "ignored_rows") + assert item.requires_confirm and item.confirm_id == f"diff:ignored_rows:{SHEET}" + assert "本期 1 行" in item.label and "「补录（人次）」" in item.label and "上一期 0 行" in item.detail + # 同样的锚点、同样的行数：不出 + assert not by_kind(run(with_excluded(BASE, [excl()]), cur), "ignored_rows") + # 行数变了、锚点没变：要确认 + assert by_kind(run(with_excluded(BASE, [excl()]), with_excluded(sep(), [excl(rows=[[32, 33]])])), "ignored_rows") + # 区域外的忽略也算 + assert by_kind(run(prev, with_excluded(sep(), [excl("ignored_outside", block=None)])), "ignored_rows") + # 别的原因（跳过的空行、隐藏行）不在这里比 + assert not by_kind(run(prev, with_excluded(sep(), [excl("blank_skipped")])), "ignored_rows") + + +def test_prev_without_rows_excluded_is_info_not_empty_list(): + item, = by_kind(run(with_excluded(BASE, None), with_excluded(sep(), [excl(), excl("hidden_excluded")])), + "ignored_rows") + assert not item.requires_confirm and item.confirm_id is None + assert item.label == "上一期未记录排除的行，本期排除了 2 行" + # 本期也没有排除的行：不出 + assert not by_kind(run(with_excluded(BASE, None), with_excluded(sep(), [])), "ignored_rows") + + +def test_ignored_rows_and_ignored_columns_ids_do_not_collide(): + """块 id 与工作表名相同（都叫「客流汇总」）时，期 2 的 diff:ignored:<块> 与 diff:ignored_rows:<工作表> 是两条： + 确认清单按 id 去重（_Bag），共用前缀的话后一条会被吞掉（评审一-m4）。""" + prev = with_excluded(BASE, []) + prev[0]["receipt"]["ignored_columns"] = {SHEET: []} + cur = with_excluded(sep(), [excl()]) + cur[0]["receipt"]["ignored_columns"] = {SHEET: ["合计"]} + diff = run(prev, cur) + ids = {d.confirm_id for d in diff if d.requires_confirm} + assert {f"diff:ignored:{SHEET}", f"diff:ignored_rows:{SHEET}"} <= ids + items = confirm_items(ConfirmContext(kind="reupload", recipe=FLOW, old_recipe=FLOW, + extraction=extraction_of(cur[0]), checks=cur[0]["checks"], diff=diff, + prev=prev[0])) + got = {i.id for i in items} + assert {f"diff:ignored:{SHEET}", f"diff:ignored_rows:{SHEET}"} <= got + + +# -------------------------------------------------------------------------- +# 区域外文字：按期累积时配方变了也比（评审意见 H2） +# -------------------------------------------------------------------------- + + +def _unit_moved(cell: str = "B34") -> tuple[tuple[dict[str, Any], str], tuple[dict[str, Any], str]]: + prev = copy.deepcopy(BASE) + add_outside(prev[0], cell, "单位：人次", kind="text") + cur = sep() + add_outside(cur[0], cell, "单位：万人次", kind="text") + return prev, cur + + +def _blank_null() -> dict[str, Any]: + r = copy.deepcopy(FLOW) + r["mode"] = "accumulate" + r["sheets"][0]["blocks"][0]["values"]["blank"] = "null" + return r + + +@pytest.mark.parametrize("p", [plan(), plan("replace_period", replaces={"start": "2026-09-01", "end": "2026-09-30"})]) +def test_outside_diff_runs_when_recipe_changed_under_accumulate(p): + """同一暂存区里应用过一次修复（配方变了），区域外「单位：人次」→「单位：万人次」照样出差异项， + accumulate_unit_risk 照样出（只看配方没变时，单位变化会静默累积进同一列）。""" + prev, cur = _unit_moved() + new = _blank_null() + diff = run(prev, cur, recipe=new, recipe_changed=True, accumulate=p) + item, = by_kind(diff, "outside_changed") + assert item.requires_confirm and item.confirm_id == f"diff:outside:{SHEET}!B34" + # 其余逐项比较照旧不做（配方变了，分段和块不一定对得上） + assert not kinds(diff) & {"label_writing", "axis_form", "ignored_columns", "hidden", "file_name"} + old = copy.deepcopy(FLOW) + old["mode"] = "accumulate" + items = {i.id for i in confirm_items(ConfirmContext( + kind="reupload", recipe=new, old_recipe=old, extraction=extraction_of(cur[0]), checks=cur[0]["checks"], + diff=diff, prev=prev[0], accumulate=p))} + assert {"accumulate_unit_risk", f"diff:outside:{SHEET}!B34"} <= items + + +@pytest.mark.parametrize("p", [None, plan("restart", semantic=["x"]), plan("first"), plan("replace", mode="replace"), + plan("rejected")]) +def test_outside_diff_not_run_when_recipe_changed_without_accumulating(p): + """每期替换、重新开始累积、首次：只影响一期，配方变了照期 2 只给行数和统计期。""" + prev, cur = _unit_moved() + assert kinds(run(prev, cur, recipe=_blank_null(), recipe_changed=True, accumulate=p)) == {"period", "rows"} + + +def test_outside_items_public_and_requires_keys(): + prev, cur = _unit_moved() + got = D.outside_items(prev[0], cur[0]) + assert [d.kind for d in got] == ["outside_changed"] + assert D.outside_items(prev[0], prev[0]) == [] + broken = copy.deepcopy(prev[0]) + del broken["receipt"]["outside_text"] + with pytest.raises(D.ReceiptIncomplete) as e: + D.outside_items(broken, cur[0]) + assert e.value.side == "prev" and e.value.missing == ["outside_text"] + # diff_reports 在累积、配方变了时同样要求两边带齐，不按空处理 + with pytest.raises(D.ReceiptIncomplete): + D.diff_reports(broken, cur[0], prev_file_name=prev[1], cur_file_name=cur[1], recipe=_blank_null(), + recipe_changed=True, accumulate=plan()) diff --git a/backend/tests/test_recipe_e2e.py b/backend/tests/test_recipe_e2e.py index eb958ab6..3eb9d8db 100644 --- a/backend/tests/test_recipe_e2e.py +++ b/backend/tests/test_recipe_e2e.py @@ -195,6 +195,10 @@ def answer_all(questions: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: values = [o["value"] for o in q["options"]] if q["id"].startswith("q_relation"): out[q["id"]] = {"value": "register"} + elif q["id"] == "q_mode": + # 期 3：q_mode 的 values[0] 是「按期累积」，配方哈希就不再等于参考配方。期 2 的用例都按每期替换走 + # （P3-SPEC 12.0、1.5）；要累积的用例显式答 accumulate + out[q["id"]] = {"value": "replace"} elif "null" in values: out[q["id"]] = {"value": "null"} else: @@ -257,11 +261,13 @@ async def test_step01_02_first_import_matches_the_reference_recipe_and_9_1(clien assert resp.status_code == 200, resp.text st = resp.json() assert set(st["answers"]) == {q["id"] for q in st["questions"]} + answered = st["answers"] wide = next(t["name"] for t in st["recipe"]["tables"] if t["name"] not in ("时段客流", "时段客流_表内合计")) resp = await client.put(f"{API}/imports/{sid}/recipe", json={"recipe": rename_wide(st["recipe"], wide, "日客流")}) assert resp.status_code == 200, resp.text st = resp.json() - assert st["recipe_problems"] == [] and st["answers"] == {} + # 期 3（第 8 节遗留项 2）：改名之后的配方仍然体现每一个回答，回答照旧保留，没有被丢弃的 + assert st["recipe_problems"] == [] and st["answers"] == answered and st["answers_dropped"] == [] ref, ref_problems = recipe.validate_recipe(FLOW_RECIPE, origin="manual") mine, mine_problems = recipe.validate_recipe(st["recipe"], origin="manual") assert ref_problems == [] and mine_problems == [] @@ -980,27 +986,16 @@ async def test_rejected_reupload_is_fixed_by_editing_the_recipe_and_committed(cl assert resp.status_code == 200, resp.text st = resp.json() assert st["status"] == "drafting" - # 2.2 第 2 条：pick 必须是本段新标题的候选词。「日间」不是「日间分时段客流（人次）」的候选词，静态校验 - # 报 const_not_candidate，path 指到 pick（表单把问题显示在常量下拉旁边），下拉的选项来自 candidates - assert [p["code"] for p in st["recipe_problems"]] == ["const_not_candidate"], st["recipe_problems"] - assert st["recipe_problems"][0]["path"].endswith("/const/时段类别/pick") - options = st["candidates"].get("日间分时段客流（人次）") - assert options, st["candidates"] - seg["const"]["时段类别"]["pick"] = options[0] - resp = await client.put(f"{API}/imports/{sid}/recipe", json={"recipe": fixed}) - assert resp.status_code == 200, resp.text - st = resp.json() - assert st["recipe_problems"] == [] and st["status"] == "drafting" + # 期 3（P3-SPEC 3.3 (b)，第 8 节遗留项 3）：pick「日间」是现行配方里确认过的、仍逐字出现在新标题里，沿用它 + # 不是新编的文字，静态校验放行；常量列的值不变，不是破坏性变更 + assert st["recipe_problems"] == [], st["recipe_problems"] resp = await trial(client, sid) st = resp.json() assert st["trial"]["status"] == "passed", st["trial"]["problems"] ids = all_confirms(st) - # 配方变了：配方类确认项重新出现（mode、单位……）。表结构没变，但常量「时段类别」只能另选新标题的候选词， - # 这一列的值整列从「日间」变了：7.5 的 breaking_changes 报常量取值变化（集成验收补，原来这里静默换值）， - # 必须出 breaking:时段客流 让人确认，别的表不报 - assert "mode" in ids and [i for i in ids if i.startswith("breaking:")] == ["breaking:时段客流"], ids - label = next(i["label"] for i in st["trial"]["confirm_items"] if i["id"] == "breaking:时段客流") - assert f"列「时段类别」取值「日间」→「{options[0]}」" in label, label + # 6.2：只列签名与现行配方不同的配方类项，导入模式没变就不再出；只改标题、不改 pick，表结构和常量都没变， + # 没有 breaking:* + assert "mode" not in ids and [i for i in ids if i.startswith("breaking:")] == [], ids resp = await commit(client, st) assert resp.status_code == 201, resp.text out = resp.json() @@ -1013,6 +1008,41 @@ async def test_rejected_reupload_is_fixed_by_editing_the_recipe_and_committed(cl assert await count(client, name, "日客流") == 30 +async def test_rejected_reupload_fixed_by_picking_a_new_word_still_reports_breaking(client): + """D09 改标题时另选新标题的候选词（期 2 的修法）：常量「时段类别」整列从「日间」换了值，7.5 的 breaking_changes + 报常量取值变化，必须出 breaking:时段客流 让人确认，别的表不报（期 3 起只改标题可以沿用旧词，见上一个用例）。""" + name = unique() + base = await first_import(client, name, seed=119) + source_id = base["commit"]["source"]["id"] + raw, fn = DRIFT_CASES["D09"].build(119) + st = (await reupload(client, source_id, raw, fn)).json() + sid = st["id"] + assert st["trial"]["status"] == "rejected" + fixed = json.loads(json.dumps(st["recipe"], ensure_ascii=False)) + seg = next(s for s in fixed["sheets"][0]["blocks"][0]["segments"] if s["id"] == "日间") + seg["locate"]["title"] = "日间分时段客流（人次）" + resp = await client.put(f"{API}/imports/{sid}/recipe", json={"recipe": fixed}) + assert resp.status_code == 200, resp.text + options = resp.json()["candidates"].get("日间分时段客流（人次）") + assert options and options[0] != "日间", resp.json()["candidates"] + seg["const"]["时段类别"]["pick"] = options[0] + resp = await client.put(f"{API}/imports/{sid}/recipe", json={"recipe": fixed}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["recipe_problems"] == [] and st["status"] == "drafting" + resp = await trial(client, sid) + st = resp.json() + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + ids = all_confirms(st) + assert "mode" not in ids and [i for i in ids if i.startswith("breaking:")] == ["breaking:时段客流"], ids + label = next(i["label"] for i in st["trial"]["confirm_items"] if i["id"] == "breaking:时段客流") + assert f"列「时段类别」取值「日间」→「{options[0]}」" in label, label + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert resp.json()["source"]["current_recipe"]["seq"] == 2 + assert await count(client, name, "日客流") == 30 + + # ========================================================================== # 修改配方（不换文件）、放弃导入 # ========================================================================== diff --git a/backend/tests/test_recipe_engine_crosstab.py b/backend/tests/test_recipe_engine_crosstab.py index ac03f415..08f2d53e 100644 --- a/backend/tests/test_recipe_engine_crosstab.py +++ b/backend/tests/test_recipe_engine_crosstab.py @@ -546,7 +546,8 @@ def test_row_codes(): ex = run(flow_book(days=3, edit=extra_row)) assert codes(ex) == {"row_unclaimed"} p = problem(ex, "row_unclaimed") - assert p.fix == "add_label" and p.cells[0] == f"{SHEET}!B32" + # 期 3 起改为按行标签忽略（P3-SPEC 3.1：隔了空行的那一行加标签也认领不到）；fix_args 见 test_recipe_engine_p3 + assert p.fix == "ignore_cells" and p.cells[0] == f"{SHEET}!B32" ex = run(flow_book(days=3, edit=lambda ws, info: set_value(ws, "D32", 987654))) assert codes(ex) == {"row_without_label"} diff --git a/backend/tests/test_recipe_engine_list.py b/backend/tests/test_recipe_engine_list.py index f26bbece..f4bf351f 100644 --- a/backend/tests/test_recipe_engine_list.py +++ b/backend/tests/test_recipe_engine_list.py @@ -376,7 +376,8 @@ def test_side_by_side_tables_split_by_blank_column(): one["sheets"][0]["blocks"].pop() one["tables"].pop() p = problem(run(build(), one), "cell_unclaimed") # 右边那张表的格在左表的数据行上，没有去处 - assert p.fix == "ignore_cells" + # 期 3 起单格没有修复按钮（P3-SPEC 3.1：单格没有能泛化的锚点，只能改文件或框选） + assert p.fix is None and p.fix_args is None # -------------------------------------------------------------------------- @@ -461,6 +462,120 @@ def test_blank_row_stop_and_skip(): assert ex.lineage["销售"]["地区"] == [[1, SHEET, "A4", 2, "down"], [3, SHEET, "A7", 3, "down"]] +@pytest.mark.parametrize("stream", [False, True]) +def test_note_below_a_skipped_blank_row_is_outside_text_not_a_record(tmp_path, monkeypatch, stream): + """跳过空行（blank_rows=skip）时，表下隔一个空行、只有首列一格文字的说明行（「注：……」）是表的下边界，不是数据行 + （WP-8 修补）：照起草器 _compatible 的认法交给区域外，文字进回执；那个空行不算「数据中间的空行」，从计数和排除的 + 行里撤回。曾静默写进一条首列是「注：……」、其余全为空值的记录，回执里只多一行。""" + def gap_and_note(ws): + ws.insert_rows(6) # 数据中间一行空白（第 6 行）：明细到第 9 行 + ws["G10"] = "附注" # 第 10 行只在块外有字：先按空行跳过，随后要一并撤回 + ws["A11"] = "注：以上为初步统计，以终稿为准" + ws["A12"] = "制表人：经办甲" + data = list_recipe(rows={"blank_rows": "skip"}) + if stream: + monkeypatch.setattr(xlsx_cells, "GRID_MAX_CELLS", 20) + db = str(tmp_path / "n.db") + ex = run(sales_book(edit=gap_and_note, total=False), data, db=db) + assert ex.ok and not ex.problems, [(p.code, p.message) for p in ex.problems] + assert ex.expected_rows["销售"] == 5 and ex.blank_rows_skipped == 1 + assert rows_of(db, "SELECT COUNT(*) FROM 销售 WHERE 产品 IS NULL") == [(0,)] + assert [(o.cell, o.kind) for o in ex.outside_text if o.cell.endswith(("11", "12"))] == [ + (f"{SHEET}!A11", "text"), (f"{SHEET}!A12", "text")] + # 判成表的下边界也看得出来（WP-8 评审意见 4）：说明那一行和其下交给区域外的行（第 12 行的落款）记进排除的行 + # after_stop，锚点是说明那格的文字、格数是本块列范围内的非空格；像说明的不出问题 + assert [(x.reason, x.rows, x.cells, x.anchor, x.block) for x in ex.rows_excluded] == [ + ("blank_skipped", [[6, 6]], 0, None, "列表1"), + ("after_stop", [[11, 12]], 2, "注：以上为初步统计，以终稿为准", "列表1")] + # 说明行之后再有数据：照样不读，数字报 outside_number（响亮地拒收，而不是悄悄截断或读进去） + def note_then_data(ws): + gap_and_note(ws) + ws["D12"] = 987654 + ex = run(sales_book(edit=note_then_data, total=False), data) + assert "outside_number" in codes(ex) and ex.expected_rows["销售"] == 5 + stop = problem(ex, "rows_after_stop") + assert "表下说明（第 11 行）之后" in stop.message and "跳过继续" not in stop.message, stop.message + assert [(x.rows, x.cells) for x in ex.rows_excluded if x.reason == "after_stop"] == [([[11, 12]], 3)] + + +def _gap_and_lone(*tail: tuple[str, Any]) -> Callable[[Any], None]: + """数据中间一行空白（第 6 行，明细到第 9 行），第 10 行空，第 11 行起按 tail 逐行写（(格, 值)，值为 None 则这一行空着）。""" + def edit(ws): + ws.insert_rows(6) + for i, (cell, value) in enumerate(tail): + if value is not None: + ws[f"{cell}{11 + i}"] = value + return edit + + +def _seen(ex: Extraction) -> tuple[Any, ...]: + """两条路径要一致的部分：问题（code、类别、消息、格）、排除的行、区域外文字、各表行数、跳过的空行数。""" + return ([(p.code, p.category, p.message, p.cells) for p in ex.problems], + [(x.reason, x.rows, x.cells, x.anchor, x.block) for x in ex.rows_excluded], + [(o.cell, o.text, o.kind) for o in ex.outside_text], ex.expected_rows, ex.blank_rows_skipped) + + +def test_a_lone_first_column_row_that_does_not_read_like_a_note_must_be_confirmed(tmp_path, monkeypatch): + """空行之后只有首列一格文字的行照旧当表的下边界（与起草器一致），但文字不像说明（不以「注」「说明」「备注」……开头） + 时，它也可能是表尾一条只填了首列的数据行：出 confirm 类的 rows_after_stop（消息写明第几行只有首列文字、已当作表下 + 说明、没有导入），并记进排除的行（WP-8 评审意见 4）。曾只出现在区域外文字里，确认项、排除的行都没有它。网格路径和 + 流式路径结果逐项相同；下面再接一行只有首列的「地区戊」，一并记进排除的行。""" + data = list_recipe(rows={"blank_rows": "skip"}) + book = sales_book(edit=_gap_and_lone(("A", "地区丁"), ("A", "地区戊")), total=False) + db = str(tmp_path / "g.db") + grid = run(book, data, db=db) + monkeypatch.setattr(xlsx_cells, "GRID_MAX_CELLS", 20) + stream = run(book, data, db=str(tmp_path / "s.db")) + assert _seen(grid) == _seen(stream) + for ex in (grid, stream): + assert ex.ok and ex.expected_rows["销售"] == 5 and ex.blank_rows_skipped == 1 + assert [p.code for p in ex.problems] == ["rows_after_stop"], [(p.code, p.message) for p in ex.problems] + p = ex.problems[0] + assert p.category == "confirm" and p.cells == [f"{SHEET}!A11"] + assert "第 11 行只有首列文字" in p.message and "已当作表下说明，没有导入" in p.message, p.message + assert "地区丁" not in p.model_message + assert [(x.reason, x.rows, x.cells, x.anchor) for x in ex.rows_excluded] == [ + ("blank_skipped", [[6, 6]], 0, None), ("after_stop", [[11, 12]], 2, "地区丁")] + assert [(o.cell, o.text) for o in ex.outside_text if o.cell != f"{SHEET}!A1"] == [ + (f"{SHEET}!A11", "地区丁"), (f"{SHEET}!A12", "地区戊")] + assert rows_of(db, "SELECT COUNT(*) FROM 销售 WHERE 地区 IN ('地区丁', '地区戊')") == [(0,)] + + +@pytest.mark.parametrize("text", ["注：本表为初步统计", "说明：不含退货", "　备注：见附件", "\u200b注：零宽字符开头", + "制表：经办甲", "填表人：经办乙", + "填报单位：合成单位", "来源：合成系统", "数据来源：合成系统", "单位：万元", + "审核：经办丙", "ＮＯＴＥ"]) +def test_which_lone_rows_read_like_a_note(text): + """像说明的开头按 canon 之后比（全角空格、零宽字符开头的照样认）：像说明的只记进排除的行和区域外文字，不出问题； + 不像的（这里用全角字母写的 NOTE）出 rows_after_stop。""" + ex = run(sales_book(edit=_gap_and_lone(("A", text)), total=False), list_recipe(rows={"blank_rows": "skip"})) + assert ex.expected_rows["销售"] == 5 + assert [(x.reason, x.rows, x.anchor) for x in ex.rows_excluded if x.reason == "after_stop"] == [ + ("after_stop", [[11, 11]], text.strip())] + assert (codes(ex) == {"rows_after_stop"}) is (text == "ＮＯＴＥ"), [(p.code, p.message) for p in ex.problems] + + +def test_lone_first_column_rows_that_are_data_or_total_are_still_read(): + """只认「首列一格文字」：首列是数、日期，或者正是合计行的标签时仍按原样读（缺数的数据行、合计行）， + 中间的空行照旧算跳过。""" + data = list_recipe(rows={"blank_rows": "skip"}) + data["tables"][0]["grain"] = [] + for cell, value in (("C11", dt.datetime(2026, 8, 30)), ("A11", "2026-08-30"), ("A11", "1,234")): + def gap_and_lone(ws, cell=cell, value=value): + ws.insert_rows(6) + ws[cell] = value + ex = run(sales_book(edit=gap_and_lone, total=False), data) + assert ex.expected_rows["销售"] == 6 and ex.blank_rows_skipped == 2, (cell, value) + def gap_and_bare_total(ws): + ws.insert_rows(6) + ws.insert_rows(10) # 合计行（第 11 行）上方也空一行，而且只写了「合计」两个字 + ws["D11"], ws["E11"] = None, None + ex = run(sales_book(edit=gap_and_bare_total), list_recipe(rows={"blank_rows": "skip", "total_row": { + "label_column": "地区", "pick": "合计", "keep_as": "销售_表内合计"}})) + assert "total_row_missing" not in codes(ex), [(p.code, p.message) for p in ex.problems] + assert ex.expected_rows == {"销售": 5, "销售_表内合计": 1} and ex.blank_rows_skipped == 2 + + def test_total_row_formulas_and_hidden_rows(): def subtotal(ws): ws["D9"] = "=SUBTOTAL(9,D4:D8)" diff --git a/backend/tests/test_recipe_engine_p3.py b/backend/tests/test_recipe_engine_p3.py new file mode 100644 index 00000000..a4c6ad63 --- /dev/null +++ b/backend/tests/test_recipe_engine_p3.py @@ -0,0 +1,729 @@ +"""配方执行器的期 3 改动（P3-SPEC 9.2、3.1、3.2、第 8 节遗留项 4、11.3 WP-1、11.4 WP-1 守卫点）。 + +- 三种忽略规则（ignore_rows / ignore_columns / ignore_outside）的执行语义，以及「只忽略命中的」这几条守卫点； +- 修复按钮的参数 Problem.fix_args：每个有修复按钮的 code 一个形状断言，另有一组用特征数 987654 放在数据格里， + 断言任何 fix_args 都不含它； +- 区域标记带块 id，按块合矩形； +- 回执里「排除的行」的六种原因及其行号、格数； +- 没用新字段的配方，D00 构建库与期 2 逐字节相同（期 2 基线由 WP-0 在 2d5e50f 上算好，见 p2_baseline.json）。 + +夹具一律现造：期 2 引擎测试的 flow_book / sales_book，加上 fixtures/xlsx 的漂移夹具（假名、随机数）。 +""" +from __future__ import annotations + +import copy +import datetime as dt +import json +from dataclasses import asdict +from pathlib import Path +from typing import Any + +import pytest + +from app.data import raw_store, recipe_engine, table_versions, xlsx_cells, xlsx_scan +from app.data.names import canon +from app.data.recipe_types import FIX_KINDS, Extraction, Recipe +from tests.fixtures.xlsx import lab +from tests.fixtures.xlsx.flow import flow_workbook +from tests.test_recipe_engine_crosstab import FLOW, flow_book, recipe_dict +from tests.test_recipe_engine_crosstab import SHEET as FS +from tests.test_recipe_engine_list import LIST, list_recipe, sales_book +from tests.test_recipe_engine_list import SHEET as LS + +N = 987654 +SEP = dt.date(2026, 9, 1) +BASELINE = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "p2_baseline.json").read_text(encoding="utf-8")) + + +def run(book: tuple[bytes, str], recipe: dict[str, Any], db: str | None = None, **kw: Any) -> Extraction: + raw, filename = book + return recipe_engine.execute(Recipe.model_validate(recipe), raw, filename, xlsx_scan.scan(raw), db, **kw) + + +def drift(code: str, recipe: dict[str, Any] | None = None, db: str | None = None) -> Extraction: + return run(flow_workbook(SEP, 30, variant=code), recipe or FLOW, db) + + +def codes(ex: Extraction) -> set[str]: + return {p.code for p in ex.problems} + + +def problems(ex: Extraction, code: str) -> list[Any]: + found = [p for p in ex.problems if p.code == code] + assert found, f"没有 {code}：{[(p.code, p.message) for p in ex.problems]}" + return found + + +def excluded(ex: Extraction) -> list[dict[str, Any]]: + return [asdict(x) for x in ex.rows_excluded] + + +def flow_with(**block: Any) -> dict[str, Any]: + """参考配方，交叉表块上加期 3 的字段（ignore_rows / ignore_columns）。""" + data = copy.deepcopy(FLOW) + data["sheets"][0]["blocks"][0].update(block) + return data + + +def rule(key: str, text: str) -> dict[str, str]: + return {key: text, "reason": "合成测试：按文字忽略"} + + +def _set(**cells: Any): + def edit(ws: Any, info: dict[str, Any]) -> None: + for ref, value in cells.items(): + ws[ref] = value + return edit + + +# ========================================================================== +# 期 2 基线：没用新字段的配方，库与期 2 逐字节相同 +# ========================================================================== + + +def test_d00_build_is_byte_identical_to_phase_2(tmp_path): + raw, name = flow_workbook(dt.date(2026, 8, 1), 31) + assert name == BASELINE["fixture"]["file_name"] + db = tmp_path / "d00.db" + ex = run((raw, name), FLOW, str(db)) + table_versions.settle_journal(db) + assert ex.ok and not ex.problems + assert raw_store.sha256_file(db) == BASELINE["db_sha256"] + assert ex.table_hashes == BASELINE["table_hashes"] + assert {t.name: t.rows for t in ex.tables} == BASELINE["rows"] + assert ex.rows_excluded == [] + + +def test_rules_whose_anchors_do_not_appear_change_nothing(tmp_path): + """锚点在某一期没出现：什么也不忽略、也不报错，库和期 2 一样（9.2「锚点在某一期没出现就什么也不忽略」）。""" + data = flow_with(ignore_rows=[rule("label", "补录（人次）")], ignore_columns=[rule("header", "合计")]) + data["sheets"][0]["ignore_outside"] = [rule("anchor", "补录说明")] + raw, name = flow_workbook(dt.date(2026, 8, 1), 31) + db = tmp_path / "d00.db" + ex = run((raw, name), data, str(db)) + table_versions.settle_journal(db) + assert ex.ok and not ex.problems + assert raw_store.sha256_file(db) == BASELINE["db_sha256"] + assert ex.table_hashes == BASELINE["table_hashes"] + assert ex.rows_excluded == [] and ex.ignored_columns == {} and "ignored" not in ex.ledger[0].roles + + +# ========================================================================== +# PROBLEM_FIX（3.1 的取值表） +# ========================================================================== + + +def test_problem_fix_table_follows_spec_3_1(): + assert recipe_engine.PROBLEM_FIX == { + "label_missing": "remove_label", "label_unexpected": "add_label", "title_not_found": "rename_title", + "label_unparsed": "declare_total", "value_not_number": "declare_placeholder", + "axis_extra_cells": "ignore_cells", "column_extra": "ignore_cells", "row_unclaimed": "ignore_cells", + "outside_number": "ignore_cells", "hidden_rows": "declare_hidden", "hidden_cols": "declare_hidden", + "sheet_missing": "rename_sheet"} + assert set(recipe_engine.PROBLEM_FIX.values()) <= set(FIX_KINDS) + # edit_members 只来自静态校验；cell_unclaimed、rows_after_stop 没有修复按钮 + assert "edit_members" not in recipe_engine.PROBLEM_FIX.values() + assert not {"cell_unclaimed", "rows_after_stop"} & set(recipe_engine.PROBLEM_FIX) + + +# ========================================================================== +# fix_args 的形状（3.2），按 3.4 的漂移例 +# ========================================================================== + + +def test_fix_args_of_the_self_service_drift_cases(): + """3.4 表和 final.md 2.8「拒收、修复按钮能自助」那一组：fix 和坐标与 3.4 一致，fix_args 按 3.2 的形状。""" + (p,) = problems(drift("D04"), "label_missing") + assert (p.fix, p.cells, p.fix_args) == ("remove_label", [f"{FS}!B10"], {"segment": "日间", "labels": ["7-8"]}) + (p,) = problems(drift("D09"), "title_not_found") + assert p.fix == "rename_title" and p.cells == [] + assert p.fix_args == {"segment": "日间", "title": "日间时段客流（人次）", + "candidates": [{"cell": f"{FS}!B9", "text": "日间分时段客流（人次）"}]} + (p,) = problems(drift("D13"), "label_unexpected") + assert p.fix == "add_label" and p.fix_args == { + "segment": "日客流", "role": "measures", "labels": [{"raw": "分区丙（人次）", "cell": f"{FS}!B8"}]} + (p,) = problems(drift("D16"), "axis_extra_cells") + assert p.fix == "ignore_cells" and p.fix_args == { + "block": "交叉表", "how": "columns", "headers": [{"raw": "合计", "cell": f"{FS}!AG4"}]} + (p,) = problems(drift("D18"), "label_unparsed") + assert p.fix == "declare_total" and p.fix_args == { + "segment": "日间", "role": "dimension", "at_end": True, + "labels": [{"raw": "7-18 时合计", "cell": f"{FS}!B21", "total": True}]} + (p,) = problems(drift("D21"), "row_unclaimed") + assert p.fix == "ignore_cells" and p.fix_args == { + "block": "交叉表", "how": "rows", "labels": [{"raw": "补录（人次）", "cell": f"{FS}!B32"}]} + + +def test_title_not_found_candidates_are_unclaimed_titles_in_row_order(): + """两个分段同时找不到标题：两条问题各自带 fix_args（界面按 fix_ids 分得清），候选是本块里像分段标题、 + 又没被认领的行，按行号升序；被别的分段认领的标题不在候选里。""" + def edit(ws, info): + ws.unmerge_cells(f"B{info['rows']['日间标题']}:{info['last']}{info['rows']['日间标题']}") + ws.unmerge_cells(f"B{info['rows']['夜间标题']}:{info['last']}{info['rows']['夜间标题']}") + ws[f"B{info['rows']['日间标题']}"] = "白天时段客流（人次）" + ws[f"B{info['rows']['夜间标题']}"] = "晚上时段客流（人次）" + ws["B35"] = "附注说明" # 表下另起的一行文字：也像分段标题，照样列为候选 + ws["B36"] = 2026 # 标签列上的一个数：不是文字，不能当候选 + ex = run(flow_book(days=3, edit=edit), FLOW) + ps = problems(ex, "title_not_found") + assert [p.fix_args["segment"] for p in ps] == ["日间", "夜间"] + want = [{"cell": f"{FS}!B9", "text": "白天时段客流（人次）"}, {"cell": f"{FS}!B21", "text": "晚上时段客流（人次）"}, + {"cell": f"{FS}!B35", "text": "附注说明"}] + assert all(p.fix_args["candidates"] == want for p in ps) + # 只有日间找不到时，夜间的标题已被认领，不是候选 + ex = run(flow_book(days=3, edit=_set(B9="白天时段客流（人次）")), FLOW) + (p,) = problems(ex, "title_not_found") + assert p.fix_args["candidates"] == [{"cell": f"{FS}!B9", "text": "白天时段客流（人次）"}] + + +def test_label_unparsed_at_end_only_when_trailing_and_contiguous(): + def middle(ws, info): + ws["B12"] = "补测" # 日间第 3 行写成解析不了的文字：不在分段末尾 + (p,) = problems(run(flow_book(days=3, edit=middle), FLOW), "label_unparsed") + assert p.fix_args == {"segment": "日间", "role": "dimension", "at_end": False, + "labels": [{"raw": "补测", "cell": f"{FS}!B12", "total": False}]} + + +def test_label_unexpected_in_a_dimension_segment(): + def extra_hour(ws, info): + ws.insert_rows(21) + ws["B21"] = "18-19" # 日间末尾多出一个时段（属于夜间的写法），值随便写 + for j in range(3): + ws.cell(21, 3 + j, 10 + j) + ex = run(flow_book(days=3, edit=extra_hour, uncached=True), FLOW) + (p,) = [p for p in problems(ex, "label_unexpected") if p.fix_args["segment"] == "日间"] + assert p.fix_args == {"segment": "日间", "role": "dimension", "labels": [{"raw": "18-19", "cell": f"{FS}!B21"}]} + + +def test_column_extra_fix_args_and_list_title_not_found_has_no_fix(): + def extra(ws): + ws["F3"], ws["F4"] = "备注", "说明文字" + (p,) = problems(run(sales_book(edit=extra), LIST), "column_extra") + assert p.fix == "ignore_cells" and p.fix_args == { + "block": "列表1", "how": "columns", "headers": [{"raw": "备注", "cell": f"{LS}!F3"}]} + # 列表的 after_title 找不到：rename_title 是分段标题的修复，列表没有能写的参数，不给按钮（交给框选） + (p,) = problems(run(sales_book(), list_recipe(after_title="不存在的标题")), "title_not_found") + assert p.fix is None and p.fix_args is None + + +def test_outside_number_fix_args_carry_the_row_anchor(): + """⑥ outside：一行一条；锚点取这一行区域外的文字格，优先不含数字的；没有文字格时锚点为 None。""" + def edit(ws, info): + ws["B33"], ws["G33"] = "补录说明", 12 # 同一行有文字 + ws["B34"], ws["G34"], ws["H34"] = "注：第2期", 13, "附注" # 两格文字：取不含数字的那格 + ws["G35"] = 14 # 这一行没有文字 + ex = run(flow_book(days=3, edit=edit), FLOW) + got = {p.fix_args["rows"][0]["row"]: p.fix_args for p in problems(ex, "outside_number")} + assert got[33] == {"sheet": FS, "sheet_id": "s1", "how": "outside", + "rows": [{"row": 33, "anchor": "补录说明", "anchor_cell": f"{FS}!B33", "cells": [f"{FS}!G33"]}]} + assert got[34]["rows"][0]["anchor"] == "附注" and got[34]["rows"][0]["anchor_cell"] == f"{FS}!H34" + assert got[35]["rows"][0]["anchor"] is None and got[35]["rows"][0]["anchor_cell"] is None + assert all(p.fix == "ignore_cells" for p in problems(ex, "outside_number")) + + +def test_value_not_number_collects_placeholder_texts(): + """⑦：texts 是 canon 后、不超过 8 个字、不像数、不含数字的写法（最多 8 种，按出现顺序），其余计入 other（格数）。""" + def edit(ws, info): + ws["C11"], ws["D11"], ws["E11"] = "—", "—", "无" + ws["C12"] = "约5000" # 含数字：不收 + ws["D12"] = "这一格是很长的一段说明文字" # 超过 8 个字：不收 + (p,) = problems(run(flow_book(days=3, edit=edit), FLOW), "value_not_number") + assert p.fix == "declare_placeholder" + # texts 是 canon 后的写法（「—」canon 成「-」）：静态校验和执行器都按 canon 比占位符，写进配方照样认得出「—」 + assert p.fix_args == {"block": "交叉表", "texts": [canon("—"), "无"], "other": 2} + words = list("甲乙丙丁戊己庚辛壬癸") # 十种不同的单字写法 + def ten(ws, info): + for i, w in enumerate(words): + ws.cell(10 + i, 3, w) + (p,) = problems(run(flow_book(days=3, edit=ten), FLOW), "value_not_number") + assert p.fix_args["texts"] == words[:8] and p.fix_args["other"] == 2 + + +def test_value_not_number_in_a_list_names_the_list_block(): + data = copy.deepcopy(LIST) + data["sheets"][0]["blocks"][0]["columns"][3]["type"] = "INTEGER" + (p,) = problems(run(sales_book(edit=lambda ws: ws.cell(4, 4, "暂缺")), data), "value_not_number") + assert p.fix_args == {"block": "列表1", "texts": ["暂缺"], "other": 0} + + +def test_hidden_rows_and_cols_fix_args(): + def edit(ws, info): + ws.row_dimensions[23].hidden = True + ws.column_dimensions["D"].hidden = True + ex = run(flow_book(days=3, edit=edit), FLOW) + (rows,) = problems(ex, "hidden_rows") + (cols,) = problems(ex, "hidden_cols") + assert rows.fix == cols.fix == "declare_hidden" + assert rows.fix_args == {"sheet": FS, "sheet_id": "s1", "axis": "rows", "rows": [23], "autofilter": False} + assert cols.fix_args == {"sheet": FS, "sheet_id": "s1", "axis": "cols", "cols": ["D"], "autofilter": False} + # c06：自动筛选隐藏的行，autofilter 为真（界面据此提示「可能是筛选隐藏的行」） + data = {"recipe_format": "agentlab-recipe/2", "sheets": [{"id": "m", "match": {"name": "明细"}, "blocks": [{ + "id": "明细表", "layout": "list", "table": "明细", + "columns": [{"header": h, "name": h, "type": t} for h, t in + (("地区", "TEXT"), ("产品", "TEXT"), ("成本", "INTEGER"), ("金额", "INTEGER"))]}]}], + "tables": [{"name": "明细", "grain": ["产品"]}]} + (p,) = problems(run(lab.c06(), data), "hidden_rows") + assert p.fix_args["autofilter"] is True and p.fix_args["rows"] == [3, 4, 5, 7, 9, 11] + + +def test_sheet_missing_candidates_are_unmatched_visible_sheets(): + def edit(ws, info): + ws.parent.create_sheet("9月客流")["A1"] = "另一张表" + data = recipe_dict() + data["sheets"][0]["match"] = {"name": "客流统计", "fallback": "only_visible_sheet"} + (p,) = problems(run(flow_book(days=3, edit=edit), data), "sheet_missing") + assert p.fix == "rename_sheet" + assert p.fix_args == {"sheet_id": "s1", "name": "客流统计", "candidates": [FS, "9月客流"]} + + +def test_no_fix_for_single_cells_rows_after_stop_and_overflow(monkeypatch): + ex = run(flow_book(days=3, edit=_set(G11=5)), FLOW) + (p,) = problems(ex, "cell_unclaimed") + assert p.fix is None and p.fix_args is None + def after(ws): + ws["A11"], ws["B11"] = "名单甲", "名单乙" + (p,) = problems(run(sales_book(total=False, edit=after), list_recipe(rows={})), "rows_after_stop") + assert p.fix is None and p.fix_args is None + # 超过逐条列出的上限、合成「另有 N 处」的那一条：没有坐标和参数，不给按钮 + monkeypatch.setattr(recipe_engine, "PER_CODE_MAX", 1) + ex = run(flow_book(days=3, edit=_set(G33=1, G34=2)), FLOW) + first, rest = problems(ex, "outside_number") + assert first.fix == "ignore_cells" and first.fix_args is not None + assert "另有 1 处" in rest.message and rest.fix is None and rest.fix_args is None + + +def test_row_unclaimed_offers_the_ignore_button_only_for_a_text_label(): + """row_unclaimed 的修复只有「按行标签忽略」（3.1）。标签是文字时给按钮，提示与按钮一致、不再让人把标签加进 + 分段（那条路认领不到隔了空行的这一行）；标签是数或像数的文字时没有能写进配方的锚点：不给按钮（fix 与 + fix_args 都为空），提示也不提按标签忽略（评审意见：原来给了 raw 为 null 的按钮，点了写不进 IgnoreRow）。""" + (p,) = problems(run(flow_book(days=3, edit=_set(B32="补录（人次）", C32=5)), FLOW), "row_unclaimed") + assert p.fix == "ignore_cells" and p.fix_args["labels"] == [{"raw": "补录（人次）", "cell": f"{FS}!B32"}] + assert "按行标签忽略" in p.message and "添加到某个分段" not in p.message + for label in (N, f"{N:,}"): + (p,) = problems(run(flow_book(days=3, edit=_set(B32=label, C32=5)), FLOW), "row_unclaimed") + assert p.fix is None and p.fix_args is None and p.cells[0] == f"{FS}!B32" + assert "按行标签忽略" not in p.message and "无法按标签忽略" in p.message + + +def test_fix_and_fix_args_come_together(): + """有 fix 就有 fix_args，反之亦然：界面和 propose_fixes 只靠 fix_args 构造补丁。""" + cases = [drift(c) for c in ("D04", "D09", "D13", "D16", "D18", "D21")] + cases.append(run(flow_book(days=3, edit=_set(G11=5, B33="说明", G33=6, C12="—")), FLOW)) + for ex in cases: + for p in ex.problems: + assert (p.fix is None) == (p.fix_args is None), (p.code, p.fix, p.fix_args) + + +# ========================================================================== +# fix_args 不含数据格的值（守卫点：特征数 987654 放在数据格里） +# ========================================================================== + + +def _hide_row(ws, info): + ws["C23"] = N + ws.row_dimensions[23].hidden = True + + +def _hide_col(ws, info): + ws["D11"] = N + ws.column_dimensions["D"].hidden = True + + +def _new_measure(ws, info): + ws.unmerge_cells(f"B8:{info['last']}8") + ws["B8"] = "分区丙（人次）" + for j in range(3): + ws.cell(8, 3 + j, N + j) + + +def _subtotal(ws, info): + ws.insert_rows(21) + ws["B21"] = "7-18 时合计" + for j in range(3): + ws.cell(21, 3 + j, N + j) + + +def _crosstab_cases() -> dict[str, tuple[Any, dict[str, Any]]]: + missing_sheet = recipe_dict() + missing_sheet["sheets"][0]["match"] = {"name": "别的表", "fallback": "none"} + return { + "label_missing": (flow_book(days=3, day_hours=tuple(range(8, 18)), edit=_set(C10=N)), FLOW), + "label_unexpected": (flow_book(days=3, edit=_new_measure), FLOW), + "title_not_found": (flow_book(days=3, edit=_set(B9="白天时段客流（人次）", C10=N)), FLOW), + "label_unparsed": (flow_book(days=3, edit=_subtotal, uncached=True), FLOW), + "row_unclaimed": (flow_book(days=3, edit=_set(B32="补录（人次）", C32=N, D32=f"{N:,}")), FLOW), + "axis_extra_cells": (flow_book(days=3, edit=_set(F4="合计", F5=N, G4=N + 1)), FLOW), + "outside_number": (flow_book(days=3, edit=_set(B33="补录说明", G33=N, B34=str(N), G34=N + 1)), FLOW), + "value_not_number": (flow_book(days=3, edit=_set(C11=f"约{N}", D11="—", E11=f"{N}万元整")), FLOW), + "hidden_rows": (flow_book(days=3, edit=_hide_row), FLOW), + "hidden_cols": (flow_book(days=3, edit=_hide_col), FLOW), + "sheet_missing": (flow_book(days=3, edit=_set(C11=N)), missing_sheet), + } + + +def _column_extra(): + def edit(ws): + ws["F3"], ws["F4"], ws["G3"], ws["G4"] = "备注", N, N + 1, N + 2 + return sales_book(edit=edit), LIST + + +@pytest.mark.parametrize("code", sorted(recipe_engine.PROBLEM_FIX)) +def test_fix_args_never_carry_cell_values(code): + book, recipe = _column_extra() if code == "column_extra" else _crosstab_cases()[code] + ex = run(book, recipe) + hits = [p for p in ex.problems if p.code == code and p.fix_args is not None] + assert hits, f"{code}：{[(p.code, p.fix, p.message) for p in ex.problems]}" + for p in ex.problems: + text = json.dumps(p.fix_args, ensure_ascii=False) + for form in (str(N), str(N + 1), str(N + 2), f"{N:,}"): + assert form not in text, (p.code, text) + # 不是文字或像一个数的表头、标签、锚点：raw 为 None（界面据此不出提议，提示改文件） + if code == "axis_extra_cells": + assert [h["raw"] for h in hits[0].fix_args["headers"]] == ["合计", None] + if code == "column_extra": + assert [h["raw"] for h in hits[0].fix_args["headers"]] == ["备注", None] + if code == "row_unclaimed": + assert hits[0].fix_args["labels"][0]["raw"] == "补录（人次）" + if code == "value_not_number": + assert hits[0].fix_args["texts"] == [canon("—")] and hits[0].fix_args["other"] == 2 + + +# ========================================================================== +# ignore_rows（交叉表） +# ========================================================================== + + +def test_ignore_rows_skips_the_matching_unclaimed_row(tmp_path): + """D21 加 ignore_rows「补录（人次）」：不报 row_unclaimed，标签格和日期列上的格去向 ignored，记进排除的行； + 三张表与 D01 逐表相同。""" + db = str(tmp_path / "d21.db") + ex = drift("D21", flow_with(ignore_rows=[rule("label", "补录（人次）")]), db) + assert ex.ok and not ex.problems + assert ex.ledger[0].roles["ignored"] == 3 and ex.ledger[0].unclaimed == 0 + assert excluded(ex) == [{"sheet": FS, "reason": "ignored_rows", "rows": [[32, 32]], "cells": 3, + "anchor": "补录（人次）", "block": "交叉表"}] + assert ex.table_hashes == drift("D01", db=str(tmp_path / "d01.db")).table_hashes + assert ("ignored", "B32:AF32", "交叉表") in [(m.role, m.ref, m.block) for m in ex.regions] + + +def test_ignore_rows_only_ignores_matching_rows(): + """守卫点「ignore_rows 只忽略匹配的行」：标签不同的行照样报 row_unclaimed；按 match_key 比，全角括号、 + 空白不同也算命中；被分段认领的行不受影响（规则只管没有分段认领的行）。""" + def edit(ws, info): + ws["B32"], ws["C32"] = "补录 (人次)", 5 # 写法不同，match_key 相同 + ws["B34"], ws["C34"] = "补录乙", 6 + data = flow_with(ignore_rows=[rule("label", "补录（人次）"), rule("label", "8-9")]) + ex = run(flow_book(days=3, edit=edit), data) + (p,) = problems(ex, "row_unclaimed") + assert p.cells[0] == f"{FS}!B34" + assert [x["rows"] for x in excluded(ex)] == [[[32, 32]]] + assert ex.ledger[0].roles["ignored"] == 2 + assert ex.expected_rows["时段客流"] == 17 * 3 # 8-9 是日间的期望标签：照常导入，没有被忽略 + + +def test_ignore_rows_leaves_the_rest_of_the_row_to_outside_accounting(): + """规则只忽略标签格和日期列上的格：同一行标签左边的数字照常算区域外数字。""" + ex = run(flow_book(days=3, edit=_set(B32="补录（人次）", C32=5, A32=7)), flow_with( + ignore_rows=[rule("label", "补录（人次）")])) + (p,) = problems(ex, "outside_number") + assert p.cells == [f"{FS}!A32"] and excluded(ex)[0]["cells"] == 2 + + +# ========================================================================== +# ignore_columns（交叉表、列表） +# ========================================================================== + + +def test_crosstab_ignore_columns_right_of_the_axis(tmp_path): + """D16 加 ignore_columns「合计」：不报 axis_extra_cells，从轴行到数据区底部的格去向 ignored_column，表头记进 + ignored_columns；三张表与 D01 相同。列不是行，不进排除的行。""" + ex = drift("D16", flow_with(ignore_columns=[rule("header", "合计")]), str(tmp_path / "d16.db")) + assert ex.ok and not ex.problems + assert ex.ignored_columns == {"交叉表": ["合计"]} + assert ex.ledger[0].roles["ignored_column"] == 24 and ex.rows_excluded == [] + assert ex.table_hashes == drift("D01", db=str(tmp_path / "d01.db")).table_hashes + + +def test_crosstab_ignore_columns_only_right_of_the_axis_and_within_the_block(): + """守卫点「ignore_columns 只在轴行右侧生效」：只认最后一个日期右侧、表头命中的列；没命中的列照样报 + axis_extra_cells；数据区之下这一列的数字不归这条规则管，照常报区域外数字；轴行左边的同名文字不受影响。""" + def edit(ws, info): + ws["F4"], ws["F5"], ws["F30"] = "合计", 1, 2 # 命中：轴行到数据区底部（第 30 行） + ws["G4"], ws["G5"] = "备注", "说明" # 没命中 + ws["B33"], ws["F33"] = "附注", 3 # 数据区之下（表下的文字行不算数据区） + ws["A4"] = "合计" # 轴行左边：不是多出的列 + ex = run(flow_book(days=3, edit=edit), flow_with(ignore_columns=[rule("header", "合计")])) + (extra,) = problems(ex, "axis_extra_cells") + assert extra.cells == [f"{FS}!G4"] and extra.fix_args["headers"] == [{"raw": "备注", "cell": f"{FS}!G4"}] + (out,) = problems(ex, "outside_number") + assert out.cells == [f"{FS}!F33"] + assert ex.ignored_columns == {"交叉表": ["合计"]} and ex.ledger[0].roles["ignored_column"] == 3 + assert (f"{FS}!A4", "text") in [(o.cell, o.kind) for o in ex.outside_text] + + +def _total_column(ws: Any, rows: range = range(5, 31)) -> int: + """F4「合计」，交叉表每个有数的行在 F 列写一个合计（手写假数）。返回 F 列这一段的非空格数（含表头）。""" + ws["F4"] = "合计" + n = 1 + for r in rows: + if ws.cell(r, 3).value is not None: + ws.cell(r, 6, 11) + n += 1 + return n + + +def _list_below(ws: Any) -> None: + """交叉表下方隔几行放一张列表，列范围 B:F，盖到被忽略的 F 列（假名、手写假数）。""" + for j, h in enumerate(("地区", "甲", "乙", "丙", "丁")): + ws.cell(34, 2 + j, h) + for i, name in enumerate(("分区子", "分区丑")): + ws.cell(35 + i, 2, name) + for j in range(4): + ws.cell(35 + i, 3 + j, 100 + i * 10 + j) + + +def _with_list_below(data: dict[str, Any]) -> dict[str, Any]: + data["sheets"][0]["blocks"].append({ + "id": "列表", "layout": "list", "table": "地区表", + "columns": [{"header": h, "name": h, "type": t} for h, t in + (("地区", "TEXT"), ("甲", "INTEGER"), ("乙", "INTEGER"), ("丙", "INTEGER"), ("丁", "INTEGER"))]}) + data["tables"].append({"name": "地区表", "grain": ["地区"]}) + return data + + +def test_crosstab_ignore_columns_stop_at_the_blocks_own_rows_above_a_list(tmp_path): + """评审意见：数据区底部只按本块自己的行算。表下另有一张列表、列范围盖到被忽略的列时，原来按整张表的数据行 + 取底部、又赶在列表之前认领，被忽略的列一直认领到列表的最后一行，列表再认领就报 segment_overlap（结构类、 + 没有修复按钮）。现在按表头忽略的列只到交叉表最后一个数据行，列表照常导入，F 列上列表的格归列表。""" + got: dict[str, int] = {} + + def edit(ws, info): + got["n"] = _total_column(ws) + _list_below(ws) + book = flow_book(days=3, edit=edit) + # 期 2 配方：只报多出的列（它的修复按钮正是「按表头忽略」） + assert codes(run(book, _with_list_below(copy.deepcopy(FLOW)))) == {"axis_extra_cells"} + db = str(tmp_path / "list.db") + ex = run(book, _with_list_below(flow_with(ignore_columns=[rule("header", "合计")])), db) + assert ex.ok and not ex.problems, [(p.code, p.message) for p in ex.problems] + assert ex.ledger[0].roles["ignored_column"] == got["n"] and ex.ledger[0].unclaimed == 0 + assert ex.ignored_columns == {"交叉表": ["合计"]} + assert ex.expected_rows["地区表"] == 2 + marks = [(m.role, m.ref, m.block) for m in ex.regions] + assert ("ignored_column", "F22:F30", "交叉表") in marks # 到交叉表最后一个数据行（第 30 行）为止 + assert ("col_header", "B34:F34", "列表") in marks and ("value", "B35:F36", "列表") in marks + + +def test_crosstab_ignore_columns_cover_rows_ignored_by_label(): + """按标签忽略的行也是本块的行：D16 + D21 同时出现、两条规则都加上时，被忽略那一行在合计列上的格也归 + ignored_column，不报区域外数字；只加按表头忽略时，那一行照常报 row_unclaimed（一条，不另报区域外数字）。""" + got: dict[str, int] = {} + + def edit(ws, info): + got["n"] = _total_column(ws) + ws["B32"], ws["C32"], ws["D32"], ws["E32"], ws["F32"] = "补录（人次）", 5, 6, 7, 18 + book = flow_book(days=3, edit=edit) + ex = run(book, flow_with(ignore_columns=[rule("header", "合计")], ignore_rows=[rule("label", "补录（人次）")])) + assert ex.ok and not ex.problems, [(p.code, p.message) for p in ex.problems] + assert ex.ledger[0].roles["ignored_column"] == got["n"] + 1 and ex.ledger[0].roles["ignored"] == 4 + assert excluded(ex) == [{"sheet": FS, "reason": "ignored_rows", "rows": [[32, 32]], "cells": 4, + "anchor": "补录（人次）", "block": "交叉表"}] + ex = run(book, flow_with(ignore_columns=[rule("header", "合计")])) + assert codes(ex) == {"row_unclaimed"} + + +def test_list_ignore_columns_only_for_matching_headers(): + """列表：命中的列按 extra_columns=ignore 的方式忽略；其余多出的列照 extra_columns（默认拒收）处理。""" + def extra(ws): + ws["F3"], ws["F4"] = "备注", "说明文字" + ws["G3"], ws["G4"] = "核对人", "某甲" + data = list_recipe(ignore_columns=[rule("header", "备注")]) + ex = run(sales_book(edit=extra), data) + (p,) = problems(ex, "column_extra") + assert p.cells == [f"{LS}!G3"] and p.fix_args["headers"] == [{"raw": "核对人", "cell": f"{LS}!G3"}] + assert ex.ignored_columns == {"列表1": ["备注"]} + ex = run(sales_book(edit=extra), list_recipe(ignore_columns=[rule("header", "备注")], extra_columns="ignore")) + assert ex.ok and ex.ignored_columns == {"列表1": ["备注", "核对人"]} + ex = run(sales_book(edit=lambda ws: (ws.cell(3, 6, "备注"), ws.cell(4, 6, "说明"))), data) + assert ex.ok and ex.ledger[0].roles["ignored_column"] == 2 and ex.rows_excluded == [] + + +# ========================================================================== +# ignore_outside +# ========================================================================== + + +def test_ignore_outside_needs_an_anchor_on_the_same_row(): + """守卫点「ignore_outside 要求同一行有锚点」：锚点所在行区域外的数字（含公式格）去向 ignored，锚点本身照常 + 是区域外文字；别的行的数字照常报 outside_number。""" + def edit(ws, info): + ws["B33"], ws["G33"], ws["H33"] = "补录说明", 5, "=G33*2" + info["cache"]["H33"] = 10 + ws["G34"] = 6 # 下一行没有锚点 + ws["B35"], ws["G35"] = "另一条说明", 7 # 有文字，但不是锚点 + data = recipe_dict() + data["sheets"][0]["ignore_outside"] = [rule("anchor", "补录说明")] + ex = run(flow_book(days=3, edit=edit), data) + assert sorted(p.cells[0] for p in problems(ex, "outside_number")) == [f"{FS}!G34", f"{FS}!G35"] + assert ex.ledger[0].roles["ignored"] == 2 + assert (f"{FS}!B33", "补录说明") in [(o.cell, o.text) for o in ex.outside_text] + assert excluded(ex) == [{"sheet": FS, "reason": "ignored_outside", "rows": [[33, 33]], "cells": 2, + "anchor": "补录说明", "block": None}] + marks = [(m.role, m.ref, m.block) for m in ex.regions] + assert ("ignored", "G33", None) in marks and ("ignored", "H33", None) in marks + + +def test_ignore_outside_matches_text_numbers_too(): + """像数的文本（「1,234」）在区域外也是数字格，同样按锚点忽略。""" + data = recipe_dict() + data["sheets"][0]["ignore_outside"] = [rule("anchor", "补录说明")] + ex = run(flow_book(days=3, edit=_set(B33="补录 说明", G33="1,234")), data) + assert ex.ok and not ex.problems and ex.ledger[0].roles["ignored"] == 1 + + +# ========================================================================== +# 区域标记的块 id +# ========================================================================== + + +def test_region_marks_carry_the_block_id(): + ex = run(flow_book(days=3), FLOW) + blocks = {(m.role, m.block) for m in ex.regions} + assert blocks == {("context", None), ("outside_text", None), ("col_header", "交叉表"), ("row_label", "交叉表"), + ("value", "交叉表"), ("section_title", "交叉表"), ("derived_label", "交叉表"), + ("derived_value", "交叉表")} + assert all("block" in asdict(m) for m in ex.regions) + + +def test_regions_of_different_blocks_are_not_merged(): + """并排的两张列表（中间隔一空列）：期 2 的值区合成一个矩形，期 3 按块分开。""" + raw, fn = lab.c08("side") + data = {"recipe_format": "agentlab-recipe/2", "sheets": [{"id": "s", "match": {"name": "并排"}, "blocks": [ + {"id": "左表", "layout": "list", "table": "地区金额", + "columns": [{"header": "地区", "name": "地区", "type": "TEXT"}, {"header": "金额", "name": "金额", "type": "INTEGER"}]}, + {"id": "右表", "layout": "list", "table": "部门费用", + "columns": [{"header": "部门", "name": "部门", "type": "TEXT"}, {"header": "费用", "name": "费用", "type": "INTEGER"}]}, + ]}], "tables": [{"name": "地区金额", "grain": ["地区"]}, {"name": "部门费用", "grain": ["部门"]}]} + ex = run((raw, fn), data) + assert ex.ok + assert [(m.role, m.ref, m.block) for m in ex.regions] == [ + ("col_header", "A1:B1", "左表"), ("col_header", "D1:E1", "右表"), + ("value", "A2:B3", "左表"), ("value", "D2:E3", "右表")] + + +def test_stream_and_grid_agree_on_blocks_and_excluded_rows(monkeypatch): + def gaps(ws): + for r in (6, 7, 10): + for c in range(1, 6): + ws.cell(r, c).value = None + data = list_recipe(rows={"blank_rows": "skip"}) + book = sales_book(n=12, total=False, edit=gaps) + grid = run(book, data) + monkeypatch.setattr(xlsx_cells, "GRID_MAX_CELLS", 30) + stream = run(book, data) + assert grid.ok and stream.ok + assert stream.regions == grid.regions and stream.rows_excluded == grid.rows_excluded + assert {m.block for m in grid.regions if m.role in ("col_header", "value")} == {"列表1"} + assert excluded(grid) == [{"sheet": LS, "reason": "blank_skipped", "rows": [[6, 7], [10, 10]], "cells": 0, + "anchor": None, "block": "列表1"}] + assert grid.blank_rows_skipped == 3 + + +# ========================================================================== +# 排除的行：六种原因（遗留项 4、L4） +# ========================================================================== + + +def test_excluded_hidden_rows_crosstab_and_list(): + exclude = recipe_dict(sheets__0__hidden={"rows": "exclude", "cols": "reject_if_any"}) + def hide(ws, info): + ws.row_dimensions[12].hidden = True + ws.row_dimensions[13].hidden = True + ex = run(flow_book(days=3, edit=hide), exclude) + assert ex.ok + assert excluded(ex) == [{"sheet": FS, "reason": "hidden_excluded", "rows": [[12, 13]], "cells": 8, + "anchor": None, "block": None}] + data = {"recipe_format": "agentlab-recipe/2", "sheets": [{ + "id": "m", "match": {"name": "明细"}, "hidden": {"rows": "exclude", "cols": "include"}, "blocks": [{ + "id": "明细表", "layout": "list", "table": "明细", + "columns": [{"header": h, "name": h, "type": t} for h, t in + (("地区", "TEXT"), ("产品", "TEXT"), ("成本", "INTEGER"), ("金额", "INTEGER"))]}]}], + "tables": [{"name": "明细", "grain": ["产品"]}]} + ex = run(lab.c06(), data) + assert ex.ok and ex.expected_rows == {"明细": 4} + assert excluded(ex) == [{"sheet": "明细", "reason": "hidden_excluded", "rows": [[3, 5], [7, 7], [9, 9], [11, 11]], + "cells": 24, "anchor": None, "block": None}] + + +def test_excluded_blank_rows_in_c08_gap(): + raw, fn = lab.c08("gap") + data = {"recipe_format": "agentlab-recipe/2", "sheets": [{"id": "s", "match": {"name": "明细"}, "blocks": [{ + "id": "明细表", "layout": "list", "table": "明细", "rows": {"blank_rows": "skip"}, + "columns": [{"header": "地区", "name": "地区", "type": "TEXT"}, {"header": "产品", "name": "产品", "type": "TEXT"}, + {"header": "金额", "name": "金额", "type": "INTEGER"}]}]}], + "tables": [{"name": "明细", "grain": ["地区", "产品"]}]} + ex = run((raw, fn), data) + assert ex.ok and ex.blank_rows_skipped == 1 + assert excluded(ex) == [{"sheet": "明细", "reason": "blank_skipped", "rows": [[4, 4]], "cells": 0, + "anchor": None, "block": "明细表"}] + + +def test_excluded_rows_after_stop(): + def after(ws): + ws["A11"], ws["B11"] = "名单甲", "名单乙" + ws["A12"], ws["B12"], ws["C12"] = "名单丙", "名单丁", "名单戊" + ex = run(sales_book(total=False, edit=after), list_recipe(rows={})) + assert "rows_after_stop" in codes(ex) + assert excluded(ex) == [{"sheet": LS, "reason": "after_stop", "rows": [[11, 12]], "cells": 5, + "anchor": None, "block": "列表1"}] + + +def test_excluded_totals_that_are_checked_but_not_kept(): + data = copy.deepcopy(LIST) + data["sheets"][0]["blocks"][0]["rows"]["total_row"].pop("keep_as") + data["tables"].pop() + ex = run(sales_book(), data) + assert ex.ok and [d.base_value for d in ex.derived] == ["销量", "金额"] + assert excluded(ex) == [{"sheet": LS, "reason": "total_not_kept", "rows": [[9, 9]], "cells": 3, + "anchor": None, "block": "列表1"}] + cross = recipe_dict() + cross["sheets"][0]["blocks"][0]["segments"][3]["keep_as"] = None + cross["tables"].pop() + ex = run(flow_book(days=3), cross) + assert ex.ok and len(ex.derived) == 9 + assert excluded(ex) == [{"sheet": FS, "reason": "total_not_kept", "rows": [[28, 30]], "cells": 12, + "anchor": None, "block": "交叉表"}] + + +def test_kept_totals_are_not_excluded(): + assert run(sales_book(), LIST).rows_excluded == [] and run(flow_book(days=3), FLOW).rows_excluded == [] + + +def test_excluded_rows_are_listed_in_a_fixed_order(): + """同一工作表里按原因的固定顺序列出；asdict 之后就是回执里的形状。""" + exclude = recipe_dict(sheets__0__hidden={"rows": "exclude", "cols": "reject_if_any"}) + exclude["sheets"][0]["blocks"][0]["ignore_rows"] = [rule("label", "补录（人次）")] + exclude["sheets"][0]["ignore_outside"] = [rule("anchor", "补录说明")] + exclude["sheets"][0]["blocks"][0]["segments"][3]["keep_as"] = None + exclude["tables"].pop() + def edit(ws, info): + ws.row_dimensions[12].hidden = True + ws["B32"], ws["C32"] = "补录（人次）", 5 + ws["B34"], ws["G34"] = "补录说明", 6 + ex = run(flow_book(days=3, edit=edit), exclude) + assert ex.ok + assert [(x["reason"], x["rows"]) for x in excluded(ex)] == [ + ("hidden_excluded", [[12, 12]]), ("ignored_rows", [[32, 32]]), ("ignored_outside", [[34, 34]]), + ("total_not_kept", [[28, 30]])] + + +def test_excluded_rows_dropped_on_sheets_whose_blocks_failed(): + """块级定位失败的工作表：区域都不知道在哪，区域外按锚点忽略的格只是噪声，不进排除的行。""" + data = recipe_dict() + data["sheets"][0]["ignore_outside"] = [rule("anchor", "补录说明")] + def edit(ws, info): + for c in range(3, 6): + ws.cell(4, c).value = None # 日期表头没了：axis_not_found + ws["B33"], ws["G33"] = "补录说明", 5 + ex = run(flow_book(days=3, edit=edit), data) + assert "axis_not_found" in codes(ex) and ex.rows_excluded == [] diff --git a/backend/tests/test_recipe_fixes.py b/backend/tests/test_recipe_fixes.py new file mode 100644 index 00000000..17b293bb --- /dev/null +++ b/backend/tests/test_recipe_fixes.py @@ -0,0 +1,579 @@ +"""修复按钮（recipe_fixes，WP-2，P3-SPEC 第 3 节）：九种提议各自出与不出的条件、补丁形状、应用后静态校验通过。 + +问题的 fix_args 按 3.2 的形状手写（执行器填写它是 WP-1 的事，这里不依赖）；干跑用期 2 的执行器 +（recipe_engine.execute）。D04、D09、D13、D18 在期 2 执行器上就能验证「修复后干跑通过」；D16、D21 要用到忽略规则的 +执行语义（WP-1），这里只验证补丁和静态校验，干跑留给集成验收（WP-8 A6b）。夹具全部合成、假名。 +""" +from __future__ import annotations + +import copy +import datetime as dt +import json +import re +from dataclasses import asdict +from pathlib import Path +from typing import Any + +import pytest + +from app.data import recipe_engine, xlsx_cells, xlsx_scan +from app.data import recipe_fixes as F +from app.data.recipe import apply_patch, recipe_sha256, validate_recipe +from app.data.recipe_suggest import _measure_column, detect_facts +from app.data.recipe_types import FIX_KINDS, DraftFacts, Fact, FixProposal, Recipe +from tests.fixtures.xlsx.flow import flow_workbook + +FLOW = Recipe.model_validate(json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json") + .read_text(encoding="utf-8"))).model_dump(mode="json") +SEP = dt.date(2026, 9, 1) +SHEET = "客流汇总" +S = "/sheets/0/blocks/0/segments" + + +def full() -> dict[str, Any]: + return copy.deepcopy(FLOW) + + +def book(variant: str): + raw, fn = flow_workbook(SEP, 30, variant=variant) + sc = xlsx_scan.scan(raw) + grids = {s.name: xlsx_cells.read_grid(raw, sc, s.name, max_rows=500) for s in sc.sheets if s.state == "visible"} + return raw, fn, sc, grids + + +def dry(variant: str, recipe: dict[str, Any]): + raw, fn, sc, _ = book(variant) + return recipe_engine.execute(Recipe.model_validate(recipe), raw, fn, sc, None) + + +def one(problem: dict[str, Any], recipe: dict[str, Any] | None = None) -> FixProposal: + got = F.propose_fixes(recipe or full(), [problem], [], None) + assert len(got) == 1, got + return got[0] + + +def apply(recipe: dict[str, Any], p: FixProposal, option: str, reason: str | None = None) -> dict[str, Any]: + r = F.fix_ops(recipe, p, option, reason) + assert r.ok, r.problems + new = apply_patch(recipe, r.ops) + assert r.recipe_sha256_after == recipe_sha256(Recipe.model_validate(new)) + return new + + +def problems_after(new: dict[str, Any], facts: DraftFacts | None = None) -> list[tuple[str, str]]: + _r, probs = validate_recipe(new, facts=facts, origin="manual", base=FLOW) + return [(p.code, p.path) for p in probs] + + +P_D04 = {"code": "label_missing", "category": "structure", "message": "…", "cells": [f"{SHEET}!B10"], + "fix": "remove_label", "fix_args": {"segment": "日间", "labels": ["7-8"]}} +P_D09 = {"code": "title_not_found", "category": "structure", "message": "…", "cells": [], "fix": "rename_title", + "fix_args": {"segment": "日间", "title": "日间时段客流（人次）", + "candidates": [{"cell": f"{SHEET}!B9", "text": "日间分时段客流（人次）"}]}} +P_D13 = {"code": "label_unexpected", "category": "structure", "message": "…", "cells": [f"{SHEET}!B8"], + "fix": "add_label", "fix_args": {"segment": "日客流", "role": "measures", + "labels": [{"raw": "分区丙（人次）", "cell": f"{SHEET}!B8"}]}} +P_D18 = {"code": "label_unparsed", "category": "structure", "message": "…", "cells": [f"{SHEET}!B21"], + "fix": "declare_total", "fix_args": {"segment": "日间", "role": "dimension", + "labels": [{"raw": "7-18 时合计", "cell": f"{SHEET}!B21", "total": True}], + "at_end": True}} +P_D16 = {"code": "axis_extra_cells", "category": "structure", "message": "…", "cells": [f"{SHEET}!AG4"], + "fix": "ignore_cells", "fix_args": {"block": "交叉表", "how": "columns", + "headers": [{"raw": "合计", "cell": f"{SHEET}!AG4"}]}} +P_D21 = {"code": "row_unclaimed", "category": "structure", "message": "…", "cells": [f"{SHEET}!B32"], + "fix": "ignore_cells", "fix_args": {"block": "交叉表", "how": "rows", + "labels": [{"raw": "补录（人次）", "cell": f"{SHEET}!B32"}]}} +P_OUT = {"code": "outside_number", "category": "structure", "message": "…", "cells": [f"{SHEET}!C33"], + "fix": "ignore_cells", "fix_args": {"sheet": SHEET, "how": "outside", "rows": [ + {"row": 33, "anchor": "补录说明", "anchor_cell": f"{SHEET}!B33", "cells": [f"{SHEET}!C33"]}]}} +P_PH = {"code": "value_not_number", "category": "structure", "message": "…", "cells": [f"{SHEET}!C11"], + "fix": "declare_placeholder", "fix_args": {"block": "交叉表", "texts": ["—"], "other": 0}} +P_HID = {"code": "hidden_rows", "category": "structure", "message": "…", "cells": [], + "fix": "declare_hidden", "fix_args": {"sheet": SHEET, "axis": "rows", "rows": [12], "autofilter": True}} +P_SHEET = {"code": "sheet_missing", "category": "structure", "message": "…", "cells": [], + "fix": "rename_sheet", "fix_args": {"sheet_id": "s1", "name": SHEET, "candidates": ["客流汇总（新）"]}} + + +# ========================================================================== +# 公共 +# ========================================================================== + + +def test_ids_are_deterministic_and_prefixed(): + a = F.propose_fixes(full(), [P_D04, P_D13], [], None) + b = F.propose_fixes(full(), [P_D04, P_D13], [], None) + assert [p.id for p in a] == [p.id for p in b] + assert all(re.fullmatch(r"fx-[0-9a-f]{12}", p.id) for p in a) + assert a[0].id != a[1].id + assert {p.kind for p in a} <= set(FIX_KINDS) + + +def test_problem_without_fix_args_gets_no_proposal(): + """期 2 的执行器只透传 fix、没有 fix_args：界面不能光靠 fix 字符串构造补丁，不出提议。""" + p = dict(P_D04) + p.pop("fix_args") + assert F.propose_fixes(full(), [p], [], None) == [] + assert F.propose_fixes(full(), [dict(P_D04, fix_args={"labels": 3})], [], None) == [] + + +def test_anchor_index_points_at_the_right_problem(): + """两个分段同时找不到标题：title_not_found 没有坐标，只有 anchor 下标分得清（评审三-B2）。""" + night = {"code": "title_not_found", "category": "structure", "message": "…", "cells": [], "fix": "rename_title", + "fix_args": {"segment": "夜间", "title": "夜间时段客流（人次）", + "candidates": [{"cell": f"{SHEET}!B21", "text": "夜间分时段客流（人次）"}]}} + other = {"code": "outside_digits", "category": "confirm", "message": "…", "cells": []} + got = F.propose_fixes(full(), [other, P_D09, night], [], None) + assert [(p.target["segment"], p.anchor.kind, p.anchor.index) for p in got] == [ + ("日间", "problem", 1), ("夜间", "problem", 2)] + + +def test_fix_ops_rejects_options_not_in_the_proposal_and_missing_reason(): + p = one(P_D21) + with pytest.raises(F.EditRequestError) as e: + F.fix_ops(full(), p, "remove", None) + assert e.value.code == "edit_invalid" + with pytest.raises(F.EditRequestError) as e: + F.fix_ops(full(), p, "ignore", " ") + assert e.value.code == "reason_required" + with pytest.raises(F.EditRequestError) as e: + F.fix_ops(full(), p, "ignore", "长" * 201) + assert e.value.code == "edit_invalid" + # 客户端改了选项值里的文件原文（pick 的词）：不在选项里，拒收 + t = one(P_D09) + with pytest.raises(F.EditRequestError): + F.fix_ops(full(), t, f"{SHEET}!B9|pick:随便写的词", None) + + +def test_stale_target_returns_not_ok(): + p = one(P_D04) + r = full() + r["sheets"][0]["blocks"][0]["segments"][1]["id"] = "白天" + r["sheets"][0]["blocks"][0]["segments"][3]["locate"]["segment"] = "夜间" + res = F.fix_ops(r, p, "remove", None) + assert res.ok is False and res.ops == [] and res.problems[0].code == "edit_not_applicable" + assert res.recipe_sha256_after is None + + +def test_patch_text_comes_only_from_file_text_rules_and_reason(): + """补丁里的文字只有三种来源：文件原文（fix_args）、系统按规则推出的名字、理由（3.1）。""" + for prob in (P_D04, P_D09, P_D13, P_D18, P_D16, P_D21, P_OUT, P_PH, P_HID, P_SHEET): + for p in F.propose_fixes(full(), [prob], [], None): + for o in p.options: + res = F.fix_ops(full(), p, o.value, "合成理由" if o.needs_reason else None) + blob = json.dumps(res.ops, ensure_ascii=False) + assert "随便写的词" not in blob + if not o.needs_reason: + assert "合成理由" not in blob + + +# ========================================================================== +# ① 移除标签（D04） +# ========================================================================== + + +def test_remove_label_dimension_d04_end_to_end(): + p = one(P_D04) + assert p.kind == "remove_label" and p.problem_code == "label_missing" + assert p.title == "在分段「日间」的期望标签中去掉「7-8」" and p.cells == [f"{SHEET}!B10"] + assert p.target == {"segment": "日间", "labels": ["7-8"]} + assert [(o.value, o.label, o.breaking) for o in p.options] == [("remove", "去掉标签「7-8」", False)] + res = F.fix_ops(full(), p, "remove", None) + assert res.key == "remove_label:日间:7-8" and res.title == p.title + assert res.ops == [{"op": "replace", "path": f"{S}/1/labels/expect", + "value": ["8-9", "9-10", "10-11", "11-12", "12-13", "13-14", "14-15", "15-16", "16-17", "17-18"]}] + new = apply(full(), p, "remove") + assert problems_after(new) == [] + ext = dry("D04", new) + assert ext.ok and ext.problems == [] + assert ext.expected_rows == {"日客流": 30, "时段客流": 480, "时段客流_表内合计": 90} + + +def test_remove_label_measures_removes_column_and_unit(): + prob = dict(P_D04, fix_args={"segment": "日客流", "labels": ["分区乙（人次）"]}) + p = one(prob) + assert p.options[0].breaking is True and "破坏性变更" in p.options[0].detail + res = F.fix_ops(full(), p, "remove", None) + assert res.ops == [ + {"op": "replace", "path": f"{S}/0/labels/expect", "value": ["全日客流（人次）", "分区甲（人次）"]}, + {"op": "remove", "path": f"{S}/0/measures/分区乙（人次）"}, + {"op": "remove", "path": "/tables/0/units/分区乙"}] + assert res.breaking is True + # 关系 R1 引用了去掉的列：静态校验报关系问题，接着出 ③（只剩一个成员，只能删掉或不登记） + new = apply_patch(full(), res.ops) + _r, rps = validate_recipe(new, origin="manual", base=FLOW) + assert ("column_unknown", "/relations/0/parts/1") in [(x.code, x.path) for x in rps] + # 本期没有能认领的恒等式（文件里没有分区乙了）：不登记要认领一条系统发现，这里只能删掉这条关系，并且要写理由 + m = F.propose_fixes(new, [], [asdict(x) for x in rps], DraftFacts()) + assert [(x.kind, [(o.value, o.needs_reason) for o in x.options]) for x in m] == [ + ("edit_members", [("remove", True)])] + with pytest.raises(F.EditRequestError) as e: + F.fix_ops(new, m[0], "remove", None) + assert e.value.code == "reason_required" + gone = F.fix_ops(new, m[0], "remove", "合成理由：分区乙撤销") + assert gone.summary == ["删除关系「R1」（理由：合成理由：分区乙撤销）"] + assert problems_after(apply_patch(new, gone.ops)) == [] + + +def test_remove_label_measures_then_edit_members_offers_dismiss_when_the_fact_is_still_there(): + """3.2 ① 的连带：去掉的组成列被 R1 引用、本期的事实 F1「全日客流 = 分区甲 + 分区乙」还在（文件没变， + 只是不再导入分区乙）。update 给不出（分区乙映射不到列），③ 只给「不登记」，认领 F1，要写理由。 + 退回不要理由的「删除这条关系」的话，F1 的分段 id 是规则起草的写法、按全部标签又认不出，3.3 第 11 条判它 + 不在改动范围内，删掉 R1 以后连未认领都不报，核对和系统发现一起不声不响地消失。""" + prob = dict(P_D04, cells=[f"{SHEET}!B7"], fix_args={"segment": "日客流", "labels": ["分区乙（人次）"]}) + new = Recipe.model_validate(apply_patch(full(), F.fix_ops(full(), one(prob), "remove", None).ops)) \ + .model_dump(mode="json") + _raw, _fn, sc, grids = book("D01") + facts = detect_facts(grids, sc, recipe=new) + f1 = next(f for f in facts.facts if f.kind == "sum_eq") + assert f1.detail["parts"] == ["分区甲（人次）", "分区乙（人次）"] + _r, rps = validate_recipe(new, facts=facts, origin="manual", base=FLOW) + got = F.propose_fixes(new, [], [asdict(x) for x in rps], facts) + assert len(got) == 1 and got[0].kind == "edit_members" + m = got[0] + assert [(o.value, o.needs_reason) for o in m.options] == [("dismiss", True)] + assert m.target["fact"] == f1.id and "「分区乙（人次）」" in m.options[0].detail + dis = apply_patch(new, F.fix_ops(new, m, "dismiss", "合成理由：分区乙另行统计").ops) + assert dis["relations"][0] == {"id": "R1", "kind": "dismissed", "claims": f1.id, "reason": "合成理由：分区乙另行统计"} + assert problems_after(dis, facts) == [] + # 绕过提议、直接删掉 R1：带现行配方校验时照样报未认领（F1 按现行配方认出落在改动过的「日客流」上） + bare = copy.deepcopy(new) + del bare["relations"][0] + assert ("fact_unclaimed", "/relations") in problems_after(bare, facts) + + +def test_remove_label_not_offered_when_nothing_would_remain_or_label_unknown(): + r = full() + r["sheets"][0]["blocks"][0]["segments"][1]["labels"]["expect"] = ["7-8"] + assert F.propose_fixes(r, [P_D04], [], None) == [] + assert F.propose_fixes(full(), [dict(P_D04, fix_args={"segment": "日间", "labels": ["6-7"]})], [], None) == [] + assert F.propose_fixes(full(), [dict(P_D04, fix_args={"segment": "早间", "labels": ["7-8"]})], [], None) == [] + + +# ========================================================================== +# ② 加入标签 → ③ 编辑恒等式成员（D13） +# ========================================================================== + + +def test_add_label_then_edit_members_d13_end_to_end(): + p = one(P_D13) + assert p.kind == "add_label" and p.target["labels"] == [{"raw": "分区丙（人次）", "cell": f"{SHEET}!B8"}] + assert "新增列「分区丙」，单位「人次」" in p.options[0].detail + res = F.fix_ops(full(), p, "add", None) + assert res.ops == [ + {"op": "replace", "path": f"{S}/0/labels/expect", + "value": ["全日客流（人次）", "分区甲（人次）", "分区乙（人次）", "分区丙（人次）"]}, + {"op": "add", "path": f"{S}/0/measures/分区丙（人次）", "value": "分区丙"}, + {"op": "add", "path": "/tables/0/units/分区丙", "value": "人次"}] + work = Recipe.model_validate(apply_patch(full(), res.ops)).model_dump(mode="json") + _raw, _fn, sc, grids = book("D13") + facts = detect_facts(grids, sc, recipe=work) + _r, rps = validate_recipe(work, facts=facts, origin="manual", base=FLOW) + assert [(x.code, x.path) for x in rps] == [("fact_claim_mismatch", "/relations/0/claims")] + fixes = F.propose_fixes(work, [], [asdict(x) for x in rps], facts) + assert len(fixes) == 1 + m = fixes[0] + assert m.kind == "edit_members" and m.anchor.kind == "recipe_problem" and m.anchor.index == 0 + assert m.target == {"relation": "R1", "fact": "F1", "total": "全日客流", "parts": ["分区甲", "分区乙", "分区丙"]} + assert [o.value for o in m.options] == ["update", "dismiss"] + assert m.options[0].label == "按系统发现更新为「全日客流 = 分区甲 + 分区乙 + 分区丙」" + assert m.options[1].needs_reason is True + upd = F.fix_ops(work, m, "update", None) + assert upd.key == "edit_members:R1" and upd.title == f"关系「R1」：{m.options[0].label}" + work2 = apply_patch(work, upd.ops) + _r, rps2 = validate_recipe(work2, facts=facts, origin="manual", base=FLOW) + assert rps2 == [] + ext = dry("D13", work2) + assert ext.ok and ext.problems == [] + # 不登记：整条换成 dismissed，认领本期的那条事实，理由进配方 + dis = apply_patch(work, F.fix_ops(work, m, "dismiss", "合成理由：分区丙另有口径").ops) + assert dis["relations"][0] == {"id": "R1", "kind": "dismissed", "claims": "F1", "reason": "合成理由：分区丙另有口径"} + assert validate_recipe(dis, facts=facts, origin="manual", base=FLOW)[1] == [] + + +def test_add_label_column_name_collision_gets_suffix(): + prob = dict(P_D13, fix_args={"segment": "日客流", "role": "measures", + "labels": [{"raw": "分区甲（人）", "cell": f"{SHEET}!B8"}]}) + p = one(prob) + res = F.fix_ops(full(), p, "add", None) + assert {"op": "add", "path": f"{S}/0/measures/分区甲（人）", "value": "分区甲_2"} in res.ops + assert {"op": "add", "path": "/tables/0/units/分区甲_2", "value": "人"} in res.ops + + +def test_add_label_unit_outside_vocabulary_keeps_the_raw_text_like_the_drafter(): + """括号里的单位不在词表中：照起草器第 8 条（_measure_column）列名保留原文、不记单位，与起草器、框选的分段 + 同一规则；选项和摘要写明单位没有登记，兄弟列都是「人次」时不会看起来同名同口径。""" + odd = dict(P_D13, fix_args={"segment": "日客流", "role": "measures", + "labels": [{"raw": "分区丁（千人次）", "cell": f"{SHEET}!B8"}]}) + p = one(odd) + assert "「千人次」不在单位词表里" in p.options[0].detail + res = F.fix_ops(full(), p, "add", None) + assert res.ops[1:] == [{"op": "add", "path": f"{S}/0/measures/分区丁（千人次）", + "value": _measure_column("分区丁（千人次）", 4)[0]}] + assert res.ops[1]["value"] == "分区丁_千人次" + assert any("没有登记单位" in x for x in res.summary) + assert problems_after(apply_patch(full(), res.ops)) == [] + + +def test_add_label_not_offered_for_known_label_or_role_mismatch(): + already = dict(P_D13, fix_args={"segment": "日客流", "role": "measures", + "labels": [{"raw": "分区甲（人次）", "cell": f"{SHEET}!B6"}]}) + assert F.propose_fixes(full(), [already], [], None) == [] + wrong = dict(P_D13, fix_args={**P_D13["fix_args"], "role": "dimension"}) + assert F.propose_fixes(full(), [wrong], [], None) == [] + + +def test_edit_members_without_a_replacing_fact_offers_only_removal(): + """本期没有可以替换、也没有可以认领的恒等式（新恒等式有一天不成立，_sum_holds 不出 sum_eq）：update、dismiss + 都给不出，只能删掉这条关系；删掉一条数据质量核对要写理由。""" + work = apply_patch(full(), F.fix_ops(full(), one(P_D13), "add", None).ops) + rp = {"path": "/relations/0/claims", "code": "fact_claim_mismatch", "message": "…"} + ne = Fact("F1", "not_equal_sum", SHEET, "…", {"a": ["日间", "夜间"], "b_segment": "日客流", "b": "全日客流（人次）"}) + got = F.propose_fixes(work, [], [rp], DraftFacts(facts=[ne])) + assert len(got) == 1 and [(o.value, o.needs_reason) for o in got[0].options] == [("remove", True)] + assert "本期未发现可以替换或认领的恒等式" in got[0].options[0].detail + new = apply_patch(work, F.fix_ops(work, got[0], "remove", "合成理由").ops) + assert [r["id"] for r in new["relations"]] == ["R2"] + + +def test_edit_members_dismiss_skips_a_fact_claimed_by_another_relation(): + """可以认领的事实已经被别的关系认领：再认领会报 fact_claimed_twice，不拿它当「不登记」的认领对象。""" + prob = dict(P_D04, cells=[f"{SHEET}!B7"], fix_args={"segment": "日客流", "labels": ["分区乙（人次）"]}) + new = apply_patch(full(), F.fix_ops(full(), one(prob), "remove", None).ops) + sum_eq = Fact("F1", "sum_eq", SHEET, "…", {"segment": "客流汇总_按日", "total": "全日客流（人次）", + "parts": ["分区甲（人次）", "分区乙（人次）"]}) + rp = {"path": "/relations/0/parts/1", "code": "column_unknown", "message": "…"} + got = F.propose_fixes(new, [], [rp], DraftFacts(facts=[sum_eq])) + assert [o.value for o in got[0].options] == ["dismiss"] and got[0].target["fact"] == "F1" + taken = copy.deepcopy(new) + taken["relations"].append({"id": "R9", "kind": "dismissed", "claims": "F1", "reason": "合成理由"}) + got = F.propose_fixes(taken, [], [rp], DraftFacts(facts=[sum_eq])) + assert [o.value for o in got[0].options] == ["remove"] + + +def test_edit_members_only_for_sum_eq_relations(): + rp = {"path": "/relations/1/claims", "code": "fact_claim_mismatch", "message": "…"} + assert F.propose_fixes(full(), [], [rp], DraftFacts()) == [] + assert F.propose_fixes(full(), [], [{"path": "/relations/0", "code": "fact_unclaimed", "message": "…"}], + DraftFacts()) == [] + + +# ========================================================================== +# ④ 分段标题改名（D09） +# ========================================================================== + + +def test_rename_title_keep_d09_end_to_end(): + p = one(P_D09) + assert p.kind == "rename_title" and p.cells == [f"{SHEET}!B9"] + values = [o.value for o in p.options] + assert values[0] == f"{SHEET}!B9|keep" and f"{SHEET}!B9|pick:日间分" in values + keep = p.options[0] + assert keep.label == "改为「日间分时段客流（人次）」，「时段类别」沿用「日间」" and keep.breaking is False + pick = next(o for o in p.options if o.value.endswith("pick:日间分")) + assert pick.breaking is True and "这是破坏性变更：这一列的值从「日间」变为「日间分」" in pick.detail + res = F.fix_ops(full(), p, keep.value, None) + assert res.ops == [{"op": "replace", "path": f"{S}/1/locate/title", "value": "日间分时段客流（人次）"}] + assert res.title == "分段「日间」的分段标题改为「日间分时段客流（人次）」，「时段类别」沿用「日间」" + new = apply_patch(full(), res.ops) + # 3.3 (b)：沿用现行配方的 pick，静态校验通过；没有 base 时照期 2 报 const_not_candidate + assert problems_after(new) == [] + assert [p.code for p in validate_recipe(new, origin="manual")[1]] == ["const_not_candidate"] + ext = dry("D09", new) + assert ext.ok and ext.problems == [] + # pick 选项：再改常量 + res2 = F.fix_ops(full(), p, pick.value, None) + assert res2.ops[1] == {"op": "replace", "path": f"{S}/1/const/时段类别/pick", "value": "日间分"} + assert res2.breaking is True + + +def test_rename_title_keep_only_when_pick_is_substring(): + prob = dict(P_D09, fix_args={**P_D09["fix_args"], + "candidates": [{"cell": f"{SHEET}!B9", "text": "白天时段客流（人次）"}]}) + p = one(prob) + assert all("|pick:" in o.value for o in p.options) + + +def test_rename_title_skips_overlong_candidates(): + prob = dict(P_D09, fix_args={**P_D09["fix_args"], + "candidates": [{"cell": f"{SHEET}!B9", "text": "日间" + "长" * 40}]}) + assert F.propose_fixes(full(), [prob], [], None) == [] + + +# ========================================================================== +# ⑤ 声明为合计行（D18） +# ========================================================================== + + +def test_declare_total_keep_d18_end_to_end(): + p = one(P_D18) + assert [o.value for o in p.options] == ["keep", "check_only"] + assert p.options[0].label == "改作合计核对，原值另存表「时段客流_表内合计」" + res = F.fix_ops(full(), p, "keep", None) + assert res.ops[0] == {"op": "add", "path": f"{S}/1/stop_parser", "value": "hour_range_total"} + assert res.ops[1]["path"] == f"{S}/2" + seg = res.ops[1]["value"] + assert seg["id"] == "日间合计" and seg["locate"]["segment"] == "日间" + assert seg["labels"] == {"expect": ["7-18 时合计"]} + assert seg["verify"] == {"kind": "label_range_sum", "against_table": "时段客流", "value": "客流"} + # 同一块里已有另存到同一张基表的合计表：沿用，不新建表 + assert seg["keep_as"] == {"table": "时段客流_表内合计", "dim": "合计项", + "derive": {"起始小时": "start", "结束小时": "end"}, "value": "客流"} + assert len(res.ops) == 2 + new = apply_patch(full(), res.ops) + assert problems_after(new) == [] + ext = dry("D18", new) + assert ext.ok and ext.expected_rows == {"日客流": 30, "时段客流": 510, "时段客流_表内合计": 120} + # 只核对不另存 + co = apply_patch(full(), F.fix_ops(full(), p, "check_only", None).ops) + assert co["sheets"][0]["blocks"][0]["segments"][2]["keep_as"] is None + assert problems_after(co) == [] + + +def test_declare_total_new_table_when_no_sibling_total(): + r = full() + del r["sheets"][0]["blocks"][0]["segments"][3] # 去掉夜间合计 + r["sheets"][0]["blocks"][0]["segments"][2]["stop_parser"] = None + r["tables"] = r["tables"][:2] + res = F.fix_ops(r, one(P_D18, r), "keep", None) + assert res.ops[-1] == {"op": "add", "path": "/tables/-", "value": { + "name": "时段客流_表内合计", "grain": ["日期", "合计项"], "kind": "reported_total", "units": {"客流": "人次"}, + "note": ""}} + assert problems_after(apply_patch(r, res.ops)) == [] + + +@pytest.mark.parametrize("change", ["role", "not_total", "not_at_end", "no_derive", "derived_after"]) +def test_declare_total_five_preconditions(change): + prob = copy.deepcopy(P_D18) + r = full() + args = prob["fix_args"] + if change == "role": + args["role"] = "measures" + elif change == "not_total": + args["labels"][0]["total"] = False + elif change == "not_at_end": + args["at_end"] = False + elif change == "no_derive": + r["sheets"][0]["blocks"][0]["segments"][1]["dim"]["derive"] = {"起始小时": "start"} + else: + args["segment"] = "夜间" # 夜间后面已经有「夜间合计」 + assert F.propose_fixes(r, [prob], [], None) == [] + + +# ========================================================================== +# ⑥ 忽略（D16 列、D21 行、区域外） +# ========================================================================== + + +def test_ignore_columns_d16(): + p = one(P_D16) + assert p.kind == "ignore_cells" and p.title == "按表头忽略「合计」这一列" + assert p.options[0].needs_reason is True + res = F.fix_ops(full(), p, "ignore", "右侧合计列不导入") + assert res.key == "ignore_cells:columns:交叉表:合计" + assert res.ops == [{"op": "add", "path": "/sheets/0/blocks/0/ignore_columns/-", + "value": {"header": "合计", "reason": "右侧合计列不导入"}}] + assert problems_after(apply_patch(full(), res.ops)) == [] + + +def test_ignore_rows_d21(): + p = one(P_D21) + assert p.title == "按行标签忽略「补录（人次）」这一行" + res = F.fix_ops(full(), p, "ignore", "补录行不导入") + assert res.ops == [{"op": "add", "path": "/sheets/0/blocks/0/ignore_rows/-", + "value": {"label": "补录（人次）", "reason": "补录行不导入"}}] + assert problems_after(apply_patch(full(), res.ops)) == [] + + +def test_ignore_outside(): + p = one(P_OUT) + assert p.title == "第 33 行有文字「补录说明」：忽略这一行导入区域之外的数字" + res = F.fix_ops(full(), p, "ignore", "说明行的数字不导入") + assert res.ops == [{"op": "add", "path": "/sheets/0/ignore_outside/-", + "value": {"anchor": "补录说明", "reason": "说明行的数字不导入"}}] + assert problems_after(apply_patch(full(), res.ops)) == [] + no_anchor = copy.deepcopy(P_OUT) + no_anchor["fix_args"]["rows"][0]["anchor"] = None + assert F.propose_fixes(full(), [no_anchor], [], None) == [] + + +def test_ignore_anchor_with_digits_is_not_offered(): + """锚点含数字时这一项不给（3.2 ⑥）：全部含数字时不出提议（不出一个没有选项的修复按钮）；部分含数字时只给 + 其余的，含数字的写进选项的说明。三种形状都一样。""" + prob = copy.deepcopy(P_D21) + prob["fix_args"]["labels"] = [{"raw": "注：9月补录", "cell": f"{SHEET}!B32"}] + assert F.propose_fixes(full(), [prob], [], None) == [] + cols = copy.deepcopy(P_D16) + cols["fix_args"]["headers"] = [{"raw": "第3周合计", "cell": f"{SHEET}!AG4"}] + out = copy.deepcopy(P_OUT) + out["fix_args"]["rows"][0]["anchor"] = "9月补录说明" + assert F.propose_fixes(full(), [cols, out], [], None) == [] + prob["fix_args"]["labels"].append({"raw": "补录（人次）", "cell": f"{SHEET}!B33"}) + p2 = one(prob) + assert p2.target["labels"] == ["补录（人次）"] and "「注：9月补录」含数字" in p2.options[0].detail + + +def test_ignore_columns_not_offered_for_non_text_or_date_headers(): + for head in (None, "9月31日", ""): + prob = copy.deepcopy(P_D16) + prob["fix_args"]["headers"] = [{"raw": head, "cell": f"{SHEET}!AG4"}] + assert F.propose_fixes(full(), [prob], [], None) == [], head + + +def test_ignore_rows_not_offered_without_label_or_on_list_block(): + prob = copy.deepcopy(P_D21) + prob["fix_args"]["labels"] = [{"raw": None, "cell": f"{SHEET}!B32"}] + assert F.propose_fixes(full(), [prob], [], None) == [] + + +# ========================================================================== +# ⑦ 声明占位符、⑧ 隐藏行列、⑨ 工作表改名 +# ========================================================================== + + +def test_declare_placeholder(): + p = one(P_PH) + assert [o.value for o in p.options] == ["no_data", "not_applicable"] + assert p.options[0].label == "把「—」也声明为无数据（存为空值）" + res = F.fix_ops(full(), p, "not_applicable", None) + assert res.ops == [{"op": "add", "path": "/sheets/0/blocks/0/values/placeholders/-", + "value": {"text": "—", "meaning": "不适用"}}] + assert problems_after(apply_patch(full(), res.ops)) == [] + for texts in ([], ["123"], ["·"], ["太长的占位符文字超过"]): + prob = copy.deepcopy(P_PH) + prob["fix_args"]["texts"] = texts + assert F.propose_fixes(full(), [prob], [], None) == [], texts + + +def test_declare_hidden_applies_on_full_form_defaults(): + """⑧ 在 hidden 取默认值的配方上也能应用：入参是完整形式，hidden.rows 总在（评审二-M1）。""" + canonical_like = Recipe.model_validate(full()).model_dump(mode="json", exclude_defaults=True) + assert "hidden" not in canonical_like["sheets"][0] + p = one(P_HID) + assert [o.value for o in p.options] == ["include", "exclude"] + assert "可能是筛选隐藏的行" in p.options[0].detail + res = F.fix_ops(full(), p, "exclude", None) + assert res.ops == [{"op": "replace", "path": "/sheets/0/hidden/rows", "value": "exclude"}] + assert apply_patch(full(), res.ops)["sheets"][0]["hidden"]["rows"] == "exclude" + cols = copy.deepcopy(P_HID) + cols["fix_args"] = {"sheet": SHEET, "axis": "cols", "cols": [5], "autofilter": False} + assert [o.value for o in one(cols).options] == ["include"] + + +def test_rename_sheet_from_missing_and_from_fallback(): + p = one(P_SHEET) + assert [o.value for o in p.options] == ["客流汇总（新）"] + res = F.fix_ops(full(), p, "客流汇总（新）", None) + assert res.ops == [{"op": "replace", "path": "/sheets/0/match/name", "value": "客流汇总（新）"}] + assert res.key == "rename_sheet:s1" + got = F.propose_fixes(full(), [], [], None, {SHEET: "9月客流"}) + assert len(got) == 1 and got[0].anchor.kind == "sheet_renamed" and got[0].anchor.sheet == "9月客流" + assert got[0].problem_code is None + assert "下一期很可能又会改名" in got[0].options[0].detail + assert problems_after(apply_patch(full(), F.fix_ops(full(), got[0], "9月客流", None).ops)) == [] + + +def test_proposal_round_trips_through_json(): + for prob in (P_D09, P_D18, P_HID): + p = one(prob) + back = F.proposal_from_dict(json.loads(json.dumps(asdict(p), ensure_ascii=False))) + assert back == p diff --git a/backend/tests/test_recipe_imports.py b/backend/tests/test_recipe_imports.py index 00f36f70..cdfed879 100644 --- a/backend/tests/test_recipe_imports.py +++ b/backend/tests/test_recipe_imports.py @@ -130,8 +130,9 @@ def make_trial_db(path: str, rows: int, tag: str) -> None: def fake_pipeline(state: FakeState) -> Pipeline: - def validate(data, *, facts=None, origin="manual"): - state.calls["validate"].append({"origin": origin, "facts": facts}) + def validate(data, *, facts=None, origin="manual", **kw): + # 期 3：现行配方作为 base 传进来（只在有现行配方时传，P3-SPEC 12.0） + state.calls["validate"].append({"origin": origin, "facts": facts, **kw}) try: parsed = Recipe.model_validate(data) except Exception: # noqa: BLE001 @@ -238,10 +239,11 @@ def fake_pipeline(state: FakeState) -> Pipeline: usage = [AiUsage("假模型", 120, 80, 0.0012, True, "2026-10-01T00:00:00+00:00", 1)] return AiDraftResult(d, usage, 1, None, compressed.chars, compressed.sha256) - def diff_reports(prev, cur, *, prev_file_name, cur_file_name, recipe=None, recipe_changed=None): + def diff_reports(prev, cur, *, prev_file_name, cur_file_name, recipe=None, recipe_changed=None, **kw): + # 期 3：按期累积时多一个 accumulate=（只在有累积计划时传，P3-SPEC 12.0） state.calls["diff_reports"].append({"prev": prev, "cur": cur, "prev_file_name": prev_file_name, "cur_file_name": cur_file_name, "recipe": recipe, - "recipe_changed": recipe_changed}) + "recipe_changed": recipe_changed, **kw}) return copy.deepcopy(state.diff) def confirm_items(ctx): @@ -460,7 +462,7 @@ async def test_stage_first_does_not_create_the_source_and_keeps_the_raw(client, assert st["grids"][0]["cells"] and st["units"] == ["人次", "元"] assert st["candidates"] == FACTS.candidates assert [q["id"] for q in st["questions"]] == [q.id for q in QUESTIONS] - assert st["marks"] == [{"sheet": "甲表", "role": "value", "ref": "A2:B3"}] + assert st["marks"] == [{"sheet": "甲表", "role": "value", "ref": "A2:B3", "block": None}] assert st["expires_at"] and st["file"]["sha256_prefix"] assert await get(DataSource, st["source"]["id"]) is None row = await get(ImportStaging, st["id"]) @@ -485,12 +487,14 @@ async def test_answers_replay_from_base_and_reset_by_put(client, fake): assert b["recipe"] == FLOW_FULL and b["answers"] == {"q_relation:F1": {"value": "register", "reason": None}} # 每次回答前按工作配方取系统发现 assert fake.calls["detect_facts"][-1] == FLOW_FULL - # PUT 之后回答清空、起点换掉 + # PUT 之后起点换掉。期 3（第 8 节遗留项 2）：新起点仍体现「登记」这个回答（R1 已在配方里），回答保留、 + # answers_dropped 为空，不再一律清空 changed = copy.deepcopy(FLOW_FULL) changed["tables"][0]["note"] = "口径说明" resp = await client.put(f"/api/datasources/imports/{sid}/recipe", json={"recipe": changed}) put = resp.json() - assert put["answers"] == {} and put["recipe"]["tables"][0]["note"] == "口径说明" + assert put["answers"] == {"q_relation:F1": {"value": "register", "reason": None}} + assert put["answers_dropped"] == [] and put["recipe"]["tables"][0]["note"] == "口径说明" assert put["recipe_origin"] == "manual" row = await get(ImportStaging, sid) assert row.answers_base["tables"][0]["note"] == "口径说明" diff --git a/backend/tests/test_recipe_imports_p3.py b/backend/tests/test_recipe_imports_p3.py new file mode 100644 index 00000000..53c6dbe1 --- /dev/null +++ b/backend/tests/test_recipe_imports_p3.py @@ -0,0 +1,301 @@ +"""按配方导入期 3 的状态机（WP-5，P3-SPEC 3.1、3.3、第 8 节遗留项 2、12.6）：用假 Pipeline 测。 + +假管线沿用 test_recipe_imports 的（合成的扫描、网格、Extraction），只换上期 3 的几个接缝：propose_fixes、fix_ops +用桩，补丁照真的 recipe_fixes.finish 算哈希（与暂存区同一口径）。真管线的流程在 test_table_imports_api_p3.py。 +""" +from __future__ import annotations + +import copy +import dataclasses + +from app.data import recipe, recipe_imports, table_versions +from app.data.recipe_fixes import EditRequestError, finish +from app.data.recipe_imports import Pipeline +from app.data.recipe_types import FixAnchor, FixOption, FixProposal, Problem, Question +from app.db.models import ImportStaging, TableImport +from tests.test_recipe_imports import ( # noqa: F401 - client、fake、store 是夹具 + FLOW_FULL, FakeState, client, committed, fake, fake_pipeline, get, recipe_source, staged, store, trialed, +) + +API = "/api/datasources" + + +# ========================================================================== +# 接缝 +# ========================================================================== + + +def test_pipeline_new_fields_default_to_none_and_default_pipeline_wires_them(): + """期 2 测试里全关键字构造的假 Pipeline 照旧成立：期 3 的接缝都有默认值 None（12.6）。""" + new = ("propose_fixes", "fix_ops", "selection_edit", "replay_compare", "align_to_contract", "compare_recipes", + "plan_accumulate", "classify_changes", "materialize_union", "build_union_notes") + fields = {f.name: f for f in dataclasses.fields(Pipeline)} + assert all(fields[n].default is None for n in new) + real = recipe_imports.default_pipeline() + assert all(callable(getattr(real, n)) for n in new) + assert fake_pipeline(FakeState()).propose_fixes is None + + +# ========================================================================== +# 遗留项 2：carry_answers +# ========================================================================== + + +def test_carry_answers_keeps_embodied_answers_and_drops_changed_and_gone(): + p = recipe_imports.default_pipeline() + base = copy.deepcopy(FLOW_FULL) + old_q = [Question("q_a", "问题甲", [{"value": "x"}, {"value": "y"}], None, + {"x": [], "y": [{"op": "add", "path": "/tables/0/note", "value": "乙"}]}), + Question("q_gone", "已经没有的问题", [{"value": "x"}], None, {"x": []})] + new_q = [old_q[0], Question("q_b", "问题乙", [{"value": "z"}], None, + {"z": [{"op": "add", "path": "/tables/0/note", "value": "丙"}]})] + kept, dropped = recipe_imports.carry_answers( + p, base, new_q, old_q, {"q_a": {"value": "x"}, "q_gone": {"value": "x"}, "q_b": {"value": "z"}}) + assert kept == {"q_a": {"value": "x", "reason": None}} + assert {d["id"]: (d["reason"], d["text"]) for d in dropped} == {"q_gone": ("gone", "已经没有的问题"), + "q_b": ("changed", "问题乙")} + # 回答的效果已经体现在新起点上：保留 + embodied = copy.deepcopy(base) + embodied["tables"][0]["note"] = "乙" + kept, dropped = recipe_imports.carry_answers(p, embodied, new_q, old_q, {"q_a": {"value": "y"}}) + assert kept == {"q_a": {"value": "y", "reason": None}} and dropped == [] + # 选项值在新问题里不存在：changed + kept, dropped = recipe_imports.carry_answers(p, base, new_q, old_q, {"q_a": {"value": "w"}}) + assert kept == {} and dropped[0]["reason"] == "changed" + # reapply（撤销修改时，修改之后才做的回答）：恢复的起点没体现它，effects 能干净地应用就保留 + kept, dropped = recipe_imports.carry_answers(p, base, new_q, old_q, {"q_b": {"value": "z"}}, reapply={"q_b"}) + assert kept == {"q_b": {"value": "z", "reason": None}} and dropped == [] + bad = [Question("q_c", "问题丙", [{"value": "z"}], None, + {"z": [{"op": "replace", "path": "/no_such_field/0", "value": 1}]})] + kept, dropped = recipe_imports.carry_answers(p, base, bad, bad, {"q_c": {"value": "z"}}, reapply={"q_c"}) + assert kept == {} and dropped == [{"id": "q_c", "text": "问题丙", "reason": "changed"}] + + +def test_stored_origin_keeps_rules_redraft_out_of_the_recipe_record(): + """配方记录、导入清单只记 rules / ai / manual / mixed（界面按 RECIPE_ORIGIN_LABEL 显示）。rules_redraft 只管本暂存区 + 的确认清单：没再改过记 rules，采用之后又有未被覆盖的修改记 manual；给暂存区赋初值（没有暂存区）按没改过算。""" + st = ImportStaging(edits=[]) + assert recipe_imports.stored_origin("rules_redraft", st) == "rules" + st.edits = [{"seq": 1, "superseded": True}] + assert recipe_imports.stored_origin("rules_redraft", st) == "rules", "采用之前的修改已被覆盖" + st.edits = [{"seq": 1, "superseded": True}, {"seq": 2, "superseded": False}] + assert recipe_imports.stored_origin("rules_redraft", st) == "manual" + assert recipe_imports.stored_origin("rules_redraft") == "rules" + for origin in recipe_imports.STORED_ORIGINS: + assert recipe_imports.stored_origin(origin, st) == origin + assert recipe_imports.stored_origin(None) == "manual" + + +# ========================================================================== +# 3.3：base 一律取数据源当前的现行配方 +# ========================================================================== + + +async def test_validate_gets_the_current_recipe_as_base_only_for_reupload(client, fake): + await staged(client) + assert all("base" not in c for c in fake.calls["validate"]), "首次导入没有现行配方，不传 base（12.0）" + done = await recipe_source(client) + source_id = done["source"]["id"] + st = (await client.post(f"{API}/{source_id}/reupload", files={"file": ("x.xlsx", b"PK p3 base", "x")})).json() + # 现行配方原样重放按 replay 校验（不查认领，也就用不着 base）；工作配方改过之后带系统发现校验，base 是现行配方 + fake.calls["validate"].clear() + changed = copy.deepcopy(FLOW_FULL) + changed["tables"][0]["note"] = "改过的说明" + resp = await client.put(f"{API}/imports/{st['id']}/recipe", json={"recipe": changed}) + assert resp.status_code == 200, resp.text + with_base = [c for c in fake.calls["validate"] if "base" in c] + ref = recipe.recipe_sha256(recipe.Recipe.model_validate(FLOW_FULL)) + assert with_base and all(recipe.recipe_sha256(recipe.Recipe.model_validate(c["base"])) == ref for c in with_base) + + +async def test_base_follows_the_current_recipe_not_the_one_at_creation(client, fake): + """评审一-m7：暂存区 A 创建时现行配方是 r1；之后别的提交把现行换成 r2。A 再评估时 base 取 r2（三处一致）。""" + done = await recipe_source(client) + source_id = done["source"]["id"] + a = (await client.post(f"{API}/{source_id}/reupload", files={"file": ("a.xlsx", b"PK p3 a", "x")})).json() + b = (await client.post(f"{API}/{source_id}/reupload", files={"file": ("b.xlsx", b"PK p3 b", "x")})).json() + changed = copy.deepcopy(FLOW_FULL) + changed["tables"][0]["note"] = "第二版的说明" + resp = await client.put(f"{API}/imports/{b['id']}/recipe", json={"recipe": changed}) + assert resp.status_code == 200, resp.text + b = await trialed(client, b["id"]) + await committed(client, b) + fake.calls["validate"].clear() + resp = await client.post(f"{API}/imports/{a['id']}/answers", json={"answers": {}}) + assert resp.status_code == 200, resp.text + bases = [c["base"] for c in fake.calls["validate"] if "base" in c] + assert bases and all(x["tables"][0].get("note") == "第二版的说明" for x in bases) + row = await get(ImportStaging, a["id"]) + assert row.base_recipe_id == done["recipe_id"], "base_recipe_id 只是创建时的记录，不改" + + +# ========================================================================== +# 3.1：提议、fix_ids、修改与撤销 +# ========================================================================== + + +def _note_fix(state: FakeState, *, index: int = 0, needs_reason: bool = False) -> None: + """给假管线装上修复接缝：任何问题都出一个「给第一张表加说明」的提议，锚在 problems 的第 index 条。""" + + def propose(rec, problems, recipe_problems, facts, renamed=None): + state.calls["propose"].append({"problems": problems, "recipe_problems": recipe_problems, "renamed": renamed}) + if len(problems) <= index: + return [] + return [FixProposal(id="fx-aaaaaaaaaaaa", kind="ignore_cells", problem_code=problems[index].get("code"), + title="合成提议", cells=[], target={"table": 0}, + options=[FixOption("note", "加说明", needs_reason=needs_reason)], + anchor=FixAnchor("problem", index))] + + def fix_ops(rec, proposal, option, reason): + if option != "note": + raise EditRequestError("edit_invalid", "选项不在提议里") + if needs_reason and not reason: + raise EditRequestError("reason_required", "这个选项需要写明理由") + text = f"合成说明：{reason}" if reason else "合成说明" + return finish(rec, [{"op": "add", "path": "/tables/0/note", "value": text}], kind="fix", + key="ignore_cells:合成", title="合成修改", summary=["给第一张表加说明"]) + + pipe = recipe_imports.pipeline() + pipe.propose_fixes = propose + pipe.fix_ops = fix_ops + + +async def test_fix_ids_follow_the_anchor_into_the_problem_they_belong_to(client, fake): + fake.draft_problems = [Problem("label_unexpected", "structure", "第一条"), Problem("label_missing", "structure", "第二条")] + _note_fix(fake, index=1) + st = await staged(client) + assert [p["fix_ids"] for p in st["draft_problems"]] == [[], ["fx-aaaaaaaaaaaa"]] + assert st["fixes"][0]["anchor"] == {"kind": "problem", "index": 1, "sheet": None} + assert all(p["fix_ids"] == [] for p in st["recipe_problems"]) + # 试运行拒收之后，提议按试运行的问题现算，fix_ids 写到 trial.problems 上 + fake.trial_problems = [Problem("x1", "structure", "甲"), Problem("x2", "structure", "乙")] + st = await trialed(client, st["id"]) + assert st["trial"]["status"] == "rejected" + assert [p["fix_ids"] for p in st["trial"]["problems"]] == [[], ["fx-aaaaaaaaaaaa"]] + assert [p["fix_ids"] for p in st["draft_problems"]] == [[], []] + + +async def _apply_note(client, sid: str, body: dict) -> dict: + pv = (await client.post(f"{API}/imports/{sid}/edits/preview", json=body)).json() + assert pv["ok"], pv + resp = await client.post(f"{API}/imports/{sid}/edits/apply", + json={"fix": body["fix"], "expected_sha256": pv["recipe_sha256_after"]}) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def test_preview_apply_undo_and_put_supersede_the_edit_stack(client, fake): + fake.draft_problems = [Problem("label_missing", "structure", "合成问题")] + _note_fix(fake) + st = await staged(client) + resp = await client.post(f"{API}/imports/{st['id']}/answers", + json={"answers": {"q_relation:F1": {"value": "register"}}}) + st = resp.json() + before = st["recipe_sha256"] + body = {"fix": {"id": "fx-aaaaaaaaaaaa", "option": "note"}, "seq": 7} + pv = (await client.post(f"{API}/imports/{st['id']}/edits/preview", json=body)).json() + assert pv["ok"] and pv["seq"] == 7 and pv["recipe_sha256_before"] == before + assert pv["breaking"] == {} and pv["compare"] is None and pv["accumulate_change"] is None + assert (await get(ImportStaging, st["id"])).recipe_sha256 == before, "预览不改暂存区" + resp = await client.post(f"{API}/imports/{st['id']}/edits/apply", + json={"fix": body["fix"], "expected_sha256": pv["recipe_sha256_after"]}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["recipe"]["tables"][0]["note"] == "合成说明" and st["recipe_sha256"] == pv["recipe_sha256_after"] + assert st["answers"] == {"q_relation:F1": {"value": "register", "reason": None}}, "修改之后回答照样延续" + assert st["recipe_origin"] == "manual" + row = await get(ImportStaging, st["id"]) + assert row.edits[0]["before"]["recipe_origin"] == "rules" and row.edits[0]["base_sha256_after"] + # 修改记录带补丁和人选的选项、理由（3.1：进导入清单 edits）；StagingOut 不给补丁 + assert row.edits[0]["ops"] == [{"op": "add", "path": "/tables/0/note", "value": "合成说明"}] + assert row.edits[0]["fix"] == {"id": "fx-aaaaaaaaaaaa", "option": "note", "reason": None} + assert "ops" not in st["edits"][0] and "before" not in st["edits"][0] + # 修改之后又回答过（改成不登记）：撤销只撤这次修改，修改之后的回答不跟着撤掉（3.1），工作配方 = 修改前的配方 + # 加上这次回答，与「不做修改、直接这样回答」的配方逐字相同 + resp = await client.post(f"{API}/imports/{st['id']}/answers", + json={"answers": {"q_relation:F1": {"value": "dismiss", "reason": "合成理由"}}}) + assert resp.status_code == 200, resp.text + resp = await client.post(f"{API}/imports/{st['id']}/edits/undo", json={}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["edits"] == [] and st["recipe_origin"] == "rules" + assert "note" not in st["recipe"]["tables"][0] or not st["recipe"]["tables"][0]["note"] + assert st["answers"] == {"q_relation:F1": {"value": "dismiss", "reason": "合成理由"}} + assert st["answers_dropped"] == [] + assert st["recipe"]["relations"][0]["kind"] == "dismissed" + other = await staged(client) + resp = await client.post(f"{API}/imports/{other['id']}/answers", + json={"answers": {"q_relation:F1": {"value": "dismiss", "reason": "合成理由"}}}) + dismissed_sha = resp.json()["recipe_sha256"] + assert dismissed_sha != before and st["recipe_sha256"] == dismissed_sha + # 先回答（有效果的「不登记」）、再修改、再撤销：修改前的回答原样保留，配方回到修改前（评审意见：撤销丢回答） + st = await _apply_note(client, st["id"], body) + assert st["answers"] == {"q_relation:F1": {"value": "dismiss", "reason": "合成理由"}} + resp = await client.post(f"{API}/imports/{st['id']}/edits/undo", json={}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["recipe_sha256"] == dismissed_sha and st["edits"] == [] + assert st["answers"] == {"q_relation:F1": {"value": "dismiss", "reason": "合成理由"}} + assert st["answers_dropped"] == [] + # 再改一次，然后 PUT：之前的修改标「已被覆盖」，不能撤销 + pv = (await client.post(f"{API}/imports/{st['id']}/edits/preview", json=body)).json() + resp = await client.post(f"{API}/imports/{st['id']}/edits/apply", + json={"fix": body["fix"], "expected_sha256": pv["recipe_sha256_after"]}) + st = resp.json() + resp = await client.put(f"{API}/imports/{st['id']}/recipe", json={"recipe": st["recipe"]}) + st = resp.json() + assert [(e["superseded"], e["undoable"]) for e in st["edits"]] == [(True, False)] + resp = await client.post(f"{API}/imports/{st['id']}/edits/undo", json={}) + assert resp.status_code == 409 and resp.json()["code"] == "nothing_to_undo" + # 被覆盖的修改不进确认项：确认清单只拿到未被覆盖的 + st = await trialed(client, st["id"]) + assert fake.calls["confirm_items"][-1].edits == [] + out = await committed(client, st) + imp = await get(TableImport, out["import_id"]) + from app.core import artifact_store + + manifest = artifact_store.load(imp.manifest_artifact) + assert [e["superseded"] for e in manifest["edits"]] == [True] and "before" not in manifest["edits"][0] + + +async def test_reason_changes_after_preview_are_edit_stale(client, fake): + fake.draft_problems = [Problem("row_unclaimed", "structure", "合成问题")] + _note_fix(fake, needs_reason=True) + st = await staged(client) + url = f"{API}/imports/{st['id']}/edits" + resp = await client.post(f"{url}/preview", json={"fix": {"id": "fx-aaaaaaaaaaaa", "option": "note"}}) + assert resp.status_code == 422 and resp.json()["code"] == "reason_required" + body = {"fix": {"id": "fx-aaaaaaaaaaaa", "option": "note", "reason": "甲"}} + pv = (await client.post(f"{url}/preview", json=body)).json() + resp = await client.post(f"{url}/apply", json={"fix": {**body["fix"], "reason": "乙"}, + "expected_sha256": pv["recipe_sha256_after"]}) + assert resp.status_code == 409 and resp.json()["code"] == "edit_stale" + resp = await client.post(f"{url}/apply", json={**body, "expected_sha256": pv["recipe_sha256_after"]}) + assert resp.status_code == 200 and resp.json()["recipe"]["tables"][0]["note"] == "合成说明：甲" + + +async def test_apply_on_a_closed_staging_and_unknown_fix(client, fake): + fake.draft_problems = [Problem("label_missing", "structure", "合成问题")] + _note_fix(fake) + st = await staged(client) + resp = await client.post(f"{API}/imports/{st['id']}/edits/preview", + json={"fix": {"id": "fx-bbbbbbbbbbbb", "option": "note"}}) + assert resp.status_code == 409 and resp.json()["code"] == "fix_stale" + assert (await client.delete(f"{API}/imports/{st['id']}")).status_code == 204 + resp = await client.post(f"{API}/imports/{st['id']}/edits/preview", + json={"fix": {"id": "fx-aaaaaaaaaaaa", "option": "note"}}) + assert resp.status_code == 409 and resp.json()["code"] == "staging_closed" + got = (await client.get(f"{API}/imports/{st['id']}")).json() + assert got["fixes"] == [], "已结束的暂存区不再给提议" + + +async def test_trial_and_staging_out_carry_the_new_fields_and_hide_internal_ones(client, fake): + """TrialOut 的新字段（没有累积计划时为 null）、StagingOut 的新字段；只给服务端的键（并集、上一期）不出去。 + 修改记录、丢弃的回答、重新起草的哈希在暂存区结束时一并瘦身（STAGING_SLIM）。""" + st = await trialed(client, (await staged(client))["id"]) + t = st["trial"] + assert not set(recipe_imports.TRIAL_INTERNAL) & set(t) + assert t["accumulate"] is None and t["union_checks"] is None and t["recipe_compare"] is None + assert t["prior_acceptances"] == [] + assert st["fixes"] == [] and st["recipe_sha256"] and st["edits"] == [] and st["answers_dropped"] == [] + assert {"edits", "answers_dropped", "redraft_sha256"} <= set(table_versions.STAGING_SLIM) diff --git a/backend/tests/test_recipe_notes.py b/backend/tests/test_recipe_notes.py index e56e861c..4936e233 100644 --- a/backend/tests/test_recipe_notes.py +++ b/backend/tests/test_recipe_notes.py @@ -623,6 +623,24 @@ def test_rows_after_stop_goes_to_block_above(): assert "空行之后" not in notes.tables["销售"].comment + +def test_rows_after_stop_from_a_skip_list_goes_to_the_block_named_in_the_message(): + """跳过空行的列表按表下说明收尾、那格文字不像说明时报的 rows_after_stop（WP-8 评审意见 4）也写进那张表的说明： + 消息里写着「列表「费用」」，按它找块；只按 blank_rows=stop 找的话，这张表的说明里没有这一句。""" + data = deepcopy(LIST_RECIPE) + second = deepcopy(data["sheets"][0]["blocks"][0]) + second.update(id="费用", table="费用", after_title="二、费用") + second["rows"] = {"blank_rows": "skip"} + data["sheets"][0]["blocks"].append(second) + data["tables"].append({"name": "费用"}) + lineage = {"销售": {"金额": [[1, "销售", "D2", 6, "down"]]}, "费用": {"金额": [[1, "销售", "D20", 3, "down"]]}} + notes = list_notes(data=data, lineage=lineage, problems=[Problem( + "rows_after_stop", "confirm", "工作表「销售」的列表「费用」：第 25 行只有首列文字（A25「地区丁」），" + "已当作表下说明，没有导入。", cells=["销售!A25"])]) + assert "空行之后" in notes.tables["费用"].comment + assert "空行之后" not in notes.tables["销售"].comment + + # ---------------------------------------------------------------- 列说明：空值、日期、单位 diff --git a/backend/tests/test_recipe_notes_p3.py b/backend/tests/test_recipe_notes_p3.py new file mode 100644 index 00000000..ba454c55 --- /dev/null +++ b/backend/tests/test_recipe_notes_p3.py @@ -0,0 +1,582 @@ +"""期 3 的说明（recipe_notes，P3-SPEC 2.8、3.2 ⑥）：build_notes 返回片段、单期的忽略说明、按期累积的并集说明。 + +不依赖执行器、核对模块的库和 WP-4 的物化：核对结果、Extraction、各期空值数都在测试里手写（假名、假数）， +并集的表结构用 derive_tables 加手写的退役列模拟 materialize_union 的 UnionReport.tables。 +""" +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +import pytest + +from app.data import recipe_parsers as P +from app.data.recipe_notes import ( + COLUMN_TEXT, CONSISTENT_KEYS, NOTE_TEXT, add_single_after_drop, build_notes, build_union_notes, +) +from app.data.recipe_types import ( + CheckResult, ColumnOut, ExcludedRows, Extraction, OutsideText, PeriodOut, Recipe, SheetsOut, UnionNotePart, + derive_tables, prose_problems, +) +from tests.test_recipe_notes import ( + FLOW, LIST_RECIPE, NULLS, SHEET, TOTALS, accept, chk, flow_checks, flow_extraction, list_item, notes_for, +) + +AUG = ("2026-08-01", "2026-08-31") +SEP = ("2026-09-01", "2026-09-30") +OCT = ("2026-10-01", "2026-10-31") + +#: 期 3 新增的模板（P3-SPEC 2.8 的代码块，加上单期的 ignored_rows、ignored_cells） +NEW_NOTE_KEYS = { + "accumulate", "accumulate_gap", "single_after_drop", "table_partial_periods", "table_absent", + "sum_eq_union_mixed", "sum_eq_union_pending", "sum_eq_union_partial", "sum_eq_members_vary", + "total_union_unverified", "period_conflict_union", "period_conflict_union_pending", "period_human_union", + "period_human_all", "outside_digits_union", "hidden_excluded_union", "hidden_included_union", + "ignored_columns_union", "ignored_rows", "ignored_rows_union", "ignored_cells", "ignored_cells_union", + "dim_values_vary", +} +NEW_COLUMN_KEYS = {"col_partial_periods", "col_absent", "col_retired"} + + +def acc(data: dict | None = None) -> dict: + d = deepcopy(data or FLOW) + d["mode"] = "accumulate" + return d + + +def part(data: dict | None = None, span: tuple[str, str] = AUG, *, checks: list[CheckResult] | None = None, + acceptances: list | None = None, nulls: Any = NULLS, **ex: Any) -> UnionNotePart: + ex.setdefault("period", PeriodOut(span[0], span[1], "cells", [f"{SHEET}!B2"])) + return UnionNotePart(Recipe.model_validate(acc(data)), flow_extraction(**ex), + flow_checks() if checks is None else checks, acceptances or [], nulls, span[0], span[1]) + + +def union(parts: list[UnionNotePart], target: dict | None = None, *, + extra_cols: dict[str, list[ColumnOut]] | None = None, extra_tables: dict[str, list[ColumnOut]] | None = None, + part_nulls: list | None = None, **kw: Any): + """并集的表结构 = 目标配方推出的表 + 退役的列（extra_cols）+ 退役的表（extra_tables），模拟 UnionReport.tables。""" + tgt = Recipe.model_validate(acc(target)) if target is not None else parts[-1].recipe + tables, _ = derive_tables(tgt) + tables = {t: list(cs) for t, cs in tables.items()} + for t, cols in (extra_cols or {}).items(): + tables[t] = tables.get(t, []) + cols + tables.update(extra_tables or {}) + kw.setdefault("null_counts", None) + kw.setdefault("added", []) + kw.setdefault("retired", []) + kw.setdefault("label_sets", []) + kw.setdefault("gaps", False) + return build_union_notes(tgt, parts, union_tables=tables, + part_null_counts=part_nulls if part_nulls is not None else [p.null_counts for p in parts], + **kw) + + +def comment(notes, table: str = "日客流") -> str: + return notes.tables[table].comment + + +def without_r1_and_b() -> dict: + """A11：去掉 measures 的分区乙，R1 不再登记（关系删掉）。""" + d = deepcopy(FLOW) + seg = d["sheets"][0]["blocks"][0]["segments"][0] + seg["labels"]["expect"] = ["全日客流（人次）", "分区甲（人次）"] + seg["measures"] = {"全日客流（人次）": "全日客流", "分区甲（人次）": "分区甲"} + d["tables"][0]["units"] = {"全日客流": "人次", "分区甲": "人次"} + d["relations"] = d["relations"][1:] + return d + + +def with_c() -> dict: + """D13：日客流 加「分区丙（人次）」，R1 成员按系统发现更新。""" + d = deepcopy(FLOW) + seg = d["sheets"][0]["blocks"][0]["segments"][0] + seg["labels"]["expect"].append("分区丙（人次）") + seg["measures"]["分区丙（人次）"] = "分区丙" + d["tables"][0]["units"]["分区丙"] = "人次" + d["relations"][0]["parts"] = ["分区甲", "分区乙", "分区丙"] + return d + + +def no_totals() -> dict: + """合计行改为不另存（keep_as 置空）：时段客流_表内合计 退役。""" + d = deepcopy(FLOW) + d["sheets"][0]["blocks"][0]["segments"][3]["keep_as"] = None + d["tables"] = d["tables"][:2] + return d + + +NULLS_C = {**NULLS, "日客流": {**NULLS["日客流"], "分区丙": 0}} +NULLS_NO_B = {**NULLS, "日客流": {k: v for k, v in NULLS["日客流"].items() if k != "分区乙"}} + + +# ---------------------------------------------------------------- build_notes 返回片段，渲染不变 + + +@pytest.mark.parametrize("override", [ + {}, + {"R1": chk("R1", "relation_sum_eq", "mismatch", "data_quality", checked=30, failed=1, acceptable=True)}, + {"K1": chk("K1", "derived_sum", "unverifiable", acceptable=True, reasons={"uncached": 3})}, + {"R2": chk("R2", "relation_not_comparable", "info", "info", checked=31, failed=0)}, + {"C1": chk("C1", "context_agree", "mismatch", "data_quality", acceptable=True)}, +]) +def test_fragments_join_to_the_same_template(override): + """片段只是多返回的结构：按顺序用「。」拼起来，就是期 2 的表说明模板，一字不差。""" + notes = notes_for(checks=flow_checks(**override)) + for table, frags in notes.fragments.items(): + assert "。".join(f.template for f in frags) == notes.templates.get(table, "") + assert set(notes.fragments) == set(notes.tables) + + +def test_reference_fragments_keys_and_subjects(): + notes = notes_for() + got = {t: [(f.key, f.subject) for f in fs] for t, fs in notes.fragments.items()} + assert got["日客流"] == [("grain_1", "日客流"), ("sum_eq_passed", "R1"), ("not_comparable_unequal", "R2")] + assert got["时段客流"] == [("grain_2", "时段客流"), ("not_comparable_unequal", "R2"), ("hour_range", "时段"), + ("const", "时段类别"), ("total_kept", TOTALS)] + assert got[TOTALS] == [("grain_2", TOTALS), ("total_k_passed", TOTALS), ("total_overlap", "时段客流")] + # 每个片段的 key 都是 NOTE_TEXT 的键（用户写的表说明记作 note） + data = deepcopy(FLOW) + data["tables"][0]["note"] = "{列:分区甲} 只统计闸机" + keys = {f.key for fs in notes_for(data).fragments.values() for f in fs} + assert keys <= set(NOTE_TEXT) | {"note"} and "note" in keys + + +# ---------------------------------------------------------------- 单期：按配方忽略的行、单元格、列（⑥） + + +def test_ignored_rows_note_only_for_tables_of_that_block(): + ex = flow_extraction(rows_excluded=[ExcludedRows(SHEET, "ignored_rows", [[32, 32]], 3, "补录（人次）", "交叉表")]) + notes = notes_for(extraction=ex) + assert notes.problems == [] + for t in ("日客流", "时段客流", TOTALS): + assert "原表另有按配方忽略的行，内容见证据面板" in comment(notes, t) + assert "补录" not in comment(notes, t) + # 别的块、别的原因、没有行：都不写 + for e in (ExcludedRows(SHEET, "ignored_rows", [[32, 32]], 3, "补录", "别的块"), + ExcludedRows(SHEET, "blank_skipped", [[9, 9]], 0, None, "交叉表"), + ExcludedRows(SHEET, "ignored_rows", [], 0, "补录", "交叉表")): + assert "按配方忽略的行" not in comment(notes_for(extraction=flow_extraction(rows_excluded=[e]))) + + +def test_ignored_cells_note_for_tables_of_that_sheet_and_dict_form(): + e = ExcludedRows(SHEET, "ignored_outside", [[33, 33]], 2, "补录说明", None) + notes = notes_for(extraction=flow_extraction(rows_excluded=[e])) + assert all("原表另有按配方忽略的单元格，内容见证据面板" in n.comment for n in notes.tables.values()) + # 暂存区、清单里存的是 JSON 形状：dict 一样认 + as_dict = {"sheet": SHEET, "reason": "ignored_outside", "rows": [[33, 33]], "cells": 2, "anchor": "补录说明", + "block": None} + assert "按配方忽略的单元格" in comment(notes_for(extraction=flow_extraction(rows_excluded=[as_dict]))) + other = ExcludedRows("别的表", "ignored_outside", [[33, 33]], 2, "补录说明", None) + assert "按配方忽略的单元格" not in comment(notes_for(extraction=flow_extraction(rows_excluded=[other]))) + + +def test_crosstab_ignore_columns_reuse_ignored_columns_note(): + data = deepcopy(FLOW) + data["sheets"][0]["blocks"][0]["ignore_columns"] = [{"header": "合计", "reason": "右侧的月合计列不导入"}] + ex = flow_extraction(ignored_columns={"交叉表": ["合计"]}) + assert "原表另有未导入的列" in comment(notes_for(data, extraction=ex)) + # 配了规则、本期没有命中：不写；没配规则（期 2 的交叉表）：照期 2 不写 + assert "未导入的列" not in comment(notes_for(data)) + assert "未导入的列" not in comment(notes_for(extraction=ex)) + + +# ---------------------------------------------------------------- 新模板：过数字和引用检查，不含「一致」 + +_SLOTS = {"col": "{列:日期}", "dim": "{列:时段}", "total": "{列:全日客流}", "parts": "{列:分区甲}、{列:分区乙}", + "unit": "日期"} +_KNOWN = {"列": {"日期", "时段", "全日客流", "分区甲", "分区乙"}, "表": set(), "单位": set(P.UNITS)} + + +@pytest.mark.parametrize("key", sorted(NEW_NOTE_KEYS)) +def test_new_table_templates_pass_prose_check_and_never_say_consistent(key): + assert key in NOTE_TEXT and key not in CONSISTENT_KEYS + text = NOTE_TEXT[key].format(**_SLOTS) + assert prose_problems(text, known=_KNOWN) == [] + assert "一致" not in text + + +@pytest.mark.parametrize("key", sorted(NEW_COLUMN_KEYS)) +def test_new_column_templates_pass_prose_check(key): + assert prose_problems(COLUMN_TEXT[key], known=_KNOWN) == [] and "一致" not in COLUMN_TEXT[key] + + +def test_pending_templates_never_say_accepted(): + for key in ("sum_eq_union_pending", "period_conflict_union_pending"): + assert "已由用户确认接受" not in NOTE_TEXT[key] and "尚未确认接受" in NOTE_TEXT[key] + for key in ("sum_eq_union_partial", "sum_eq_union_mixed", "sum_eq_union_pending"): + assert "已逐" not in NOTE_TEXT[key] and "导入时已" not in NOTE_TEXT[key] + + +# ---------------------------------------------------------------- 并集：参考配方两期 + + +def test_two_periods_reference_union(): + notes = union([part(), part(span=SEP)]) + assert notes.problems == [] + assert comment(notes) == ( + "粒度：每个日期。全日客流 等于 分区甲 与 分区乙 之和（导入时已逐日核对）：求合计只用 全日客流，不要把这几列相加。" + "与 时段客流 口径不同：各时段之和不等于 全日客流，不能互相推算或相加。" + "本表按统计期累积多期数据，各期互不重叠：比较不同期时请按 日期 筛选") + assert comment(notes, TOTALS).startswith("粒度：每个日期的每个合计项。原表写明的合计，已按明细重算核对一致") + assert notes.kinds == {TOTALS: "reported_total"} + assert notes.tables["日客流"].columns == {"日期": "格式 YYYY-MM-DD，年份按统计期补全", "全日客流": "单位：人次", + "分区甲": "单位：人次", "分区乙": "单位：人次"} + # 片段就是渲染前的模板，按顺序拼起来 + for t, frags in notes.fragments.items(): + assert "。".join(f.template for f in frags) == notes.templates[t] + assert all("当前版本只含最近一期" not in n.comment for n in notes.tables.values()) + + +def test_gap_uses_gap_sentence(): + notes = union([part(), part(span=OCT)], gaps=True) + assert "各期之间有空缺：比较不同期之前请先查询 日期 的覆盖范围" in comment(notes) + assert "各期互不重叠：比较不同期时请按" not in comment(notes) + + +# ---------------------------------------------------------------- 并集：sum_eq 的每一行 + +R1_MISMATCH = chk("R1", "relation_sum_eq", "mismatch", "data_quality", checked=30, failed=1, acceptable=True) + + +def test_sum_eq_one_period_accepted_mismatch_is_mixed(): + """D26 与 8 月累积（P3-SPEC 2.8 的例子）：不再出现「已逐日核对」，用 sum_eq_union_mixed。""" + notes = union([part(), part(span=SEP, checks=flow_checks(R1=R1_MISMATCH), acceptances=accept("R1"))]) + assert notes.problems == [] + c = comment(notes) + assert ("全日客流 与 分区甲、分区乙 之和的关系在部分期的个别日期不成立或未能核对，详见导入清单：" + "不要据此推算，也不要把这几列相加") in c + assert "已逐日核对" not in c and "求合计只用" not in c + + +@pytest.mark.parametrize("r1", [ + chk("R1", "relation_sum_eq", "unverifiable", "data_quality", unverifiable=2, acceptable=True), + chk("R1", "relation_sum_eq", "unverifiable", "data_quality", checked=0, unverifiable=0, acceptable=True), +]) +def test_sum_eq_nulls_or_no_rows_is_mixed(r1): + c = comment(union([part(checks=flow_checks(R1=r1), acceptances=accept("R1")), part(span=SEP)])) + assert "在部分期的个别日期不成立或未能核对" in c and "已逐日核对" not in c + + +def test_sum_eq_pending_never_says_accepted(): + """试运行预览：本期按「全部未接受」生成。早期那一期接受过也不行：合并结果不得出现「已由用户确认接受」。""" + old = part(checks=flow_checks(R1=R1_MISMATCH), acceptances=accept("R1")) + new = part(span=SEP, checks=flow_checks(R1=R1_MISMATCH)) + c = comment(union([old, new])) + assert "尚未确认接受" in c and "已由用户确认接受" not in c and "已逐日核对" not in c + # 反例：两期都接受了，才写「不成立或未能核对」（不是「尚未确认接受」） + both = comment(union([old, part(span=SEP, checks=flow_checks(R1=R1_MISMATCH), acceptances=accept("R1"))])) + assert "尚未确认接受" not in both and "不成立或未能核对" in both + + +def test_sum_eq_members_vary(): + """D13 修复后累积：8 月 R1 = 甲 + 乙，9 月 R1 = 甲 + 乙 + 丙，都通过：sum_eq_members_vary。""" + notes = union([part(), part(with_c(), SEP, nulls=NULLS_C)], part_nulls=[NULLS, NULLS_C]) + assert notes.problems == [] + c = comment(notes) + assert "全日客流 等于各期登记的分项列之和，各期登记的分项列不同（某期没有的列为空值），导入时已逐期核对" in c + assert "已逐日核对" not in c + # 列：8 月没有分区丙，结构性空值由 col_partial_periods 说明；9 月这一列本身没有空值，不写「原表为空格」 + assert notes.tables["日客流"].columns["分区丙"] == "单位：人次。部分期没有这一列的数据，为空值" + + +def test_sum_eq_reordered_parts_are_the_same_members(): + """9 月的 R1 只把分项倒了个序：成员相同，用目标那一期（9 月）的「已逐日核对」句子，不写「各期登记的分项列不同」 + （这句会进模型可见的说明，评审意见）。""" + swapped = deepcopy(FLOW) + swapped["relations"][0]["parts"] = list(reversed(swapped["relations"][0]["parts"])) + notes = union([part(), part(swapped, SEP)]) + assert notes.problems == [] + c = comment(notes) + assert "全日客流 等于 分区乙 与 分区甲 之和（导入时已逐日核对）" in c + assert "各期登记的分项列不同" not in c + # 反例：成员真的不同（D13 加了分区丙）仍是 sum_eq_members_vary + assert "各期登记的分项列不同" in comment(union([part(), part(with_c(), SEP, nulls=NULLS_C)], + part_nulls=[NULLS, NULLS_C])) + + +def test_sum_eq_partial_when_target_period_drops_relation(): + """A11：9 月去掉分区乙、R1 不再登记。8 月有 R1（通过），9 月没有：sum_eq_union_partial，成员取 8 月的。""" + notes = union([part(), part(without_r1_and_b(), SEP, nulls=NULLS_NO_B)], + extra_cols={"日客流": [ColumnOut("分区乙", "INTEGER", header=None, unit=None, role="measure")]}, + part_nulls=[NULLS, {**NULLS_NO_B, "日客流": {**NULLS_NO_B["日客流"], "分区乙": 30}}]) + assert notes.problems == [] + c = comment(notes) + assert ("全日客流 与 分区甲、分区乙 之和的关系只在部分期登记并核对，其余各期未核对，详见导入清单：" + "不要据此推算，也不要把这几列相加") in c + assert "已逐日核对" not in c + # 退役列：单位取最后一个含它的那一期（8 月）的，并集给的 unit 为空也一样；说明 col_retired + col = notes.tables["日客流"].columns["分区乙"] + assert col == "单位：人次。这一列自某一期起不再导入，此后各期为空值" + assert "原表为空格" not in col + + +def test_sum_eq_partial_when_only_new_period_registers_it(): + """反过来：8 月没登记 R1，9 月登记了。照样是「只在部分期登记并核对」，不沿用 9 月的「已逐日核对」。""" + old = deepcopy(FLOW) + old["relations"] = old["relations"][1:] + c = comment(union([part(old, checks=flow_checks(R1=None)), part(span=SEP)])) + assert "只在部分期登记并核对" in c and "已逐日核对" not in c + + +# ---------------------------------------------------------------- 并集：not_comparable、K、其余各族 + + +def test_not_comparable_equal_in_one_period_drops_not_equal(): + eq = chk("R2", "relation_not_comparable", "info", "info", checked=30, failed=0) + notes = union([part(), part(span=SEP, checks=flow_checks(R2=eq))]) + for t in ("日客流", "时段客流"): + assert "口径不同，不能互相推算或相加" in comment(notes, t) and "不等于" not in comment(notes, t) + both = union([part(), part(span=SEP)]) + assert "各时段之和不等于 全日客流" in comment(both) + + +@pytest.mark.parametrize("reason", ["uncached", "blank", "null_detail", "tiling", "hidden"]) +def test_k_not_passed_in_any_period_never_says_consistent(reason): + k = chk("K1", "derived_sum", "unverifiable", acceptable=True, reasons={reason: 3}) + for order in ((part(), part(span=SEP, checks=flow_checks(K1=k), acceptances=accept("K1"))), + (part(checks=flow_checks(K1=k), acceptances=accept("K1")), part(span=SEP))): + c = comment(union(list(order)), TOTALS) + assert "原表写明的合计，按原表保存的值存储。部分期的合计值未能核对，详见导入清单" in c + assert "一致" not in c + + +def test_k_passed_g_unverifiable_in_one_period(): + g = chk("G1", "formula_refs", "unverifiable", acceptable=True, reasons={"unrecognized": 1}) + c = comment(union([part(), part(span=SEP, checks=flow_checks(G1=g), acceptances=accept("G1"))]), TOTALS) + assert "原表写明的合计，合计值已按明细核对一致，公式引用的格子未能核对" in c + + +def test_k_blocking_in_new_period_is_not_inherited(): + """本期 K 不一致（试运行本来就拒收）：本期的合计片段不写，合并时不能沿用 8 月的「核对一致」。""" + bad = chk("K1", "derived_sum", "mismatch", failed=1) + c = comment(union([part(), part(span=SEP, checks=flow_checks(K1=bad))]), TOTALS) + assert "一致" not in c and "部分期的合计值未能核对" in c + + +@pytest.mark.parametrize("ex,expect", [ + ({"hidden": {SHEET: {"rows": [12], "cols": [], "policy_rows": "exclude"}}}, "部分期的原表有被隐藏的行，本表不含这些行"), + ({"outside_text": [OutsideText(SHEET, "B32", "注：9月15日闸机故障", "text_digits")]}, + "部分期的原表附有说明文字（可能涉及口径），内容见证据面板"), + ({"rows_excluded": [ExcludedRows(SHEET, "ignored_rows", [[32, 32]], 3, "补录", "交叉表")]}, + "部分期的原表另有按配方忽略的行，内容见证据面板"), + ({"rows_excluded": [ExcludedRows(SHEET, "ignored_outside", [[33, 33]], 2, "补录说明", None)]}, + "部分期的原表另有按配方忽略的单元格，内容见证据面板"), +]) +def test_any_period_families_use_union_wording(ex, expect): + data = deepcopy(FLOW) + if "hidden" in ex: + data["sheets"][0]["hidden"] = {"rows": "exclude"} + notes = union([part(data), part(data, SEP, **ex)], target=data) + assert notes.problems == [] + assert expect in comment(notes) + assert "本期" not in comment(notes) + assert expect not in comment(union([part(data), part(data, SEP)], target=data)) + + +def test_hidden_included_and_ignored_columns_union(): + data = deepcopy(FLOW) + data["sheets"][0]["hidden"] = {"rows": "include"} + data["sheets"][0]["blocks"][0]["ignore_columns"] = [{"header": "合计", "reason": "月合计列不导入"}] + p = part(data, SEP, hidden={SHEET: {"rows": [12], "cols": [], "policy_rows": "include"}}, + ignored_columns={"交叉表": ["合计"]}) + c = comment(union([part(data), p], target=data)) + assert "部分期的原表有被隐藏的行，本表包含这些行" in c and "部分期的原表另有未导入的列" in c + + +def test_period_conflict_pending_and_accepted(): + c1 = chk("C1", "context_agree", "mismatch", "data_quality", acceptable=True) + pending = comment(union([part(checks=flow_checks(C1=c1), acceptances=accept("C1")), + part(span=SEP, checks=flow_checks(C1=c1))])) + assert "部分期的统计期写法在原表或文件名中有冲突，尚未确认接受" in pending + assert "已由用户确认接受" not in pending + accepted = comment(union([part(checks=flow_checks(C1=c1), acceptances=accept("C1")), part(span=SEP)])) + assert "部分期的统计期写法在原表或文件名中有冲突，已由用户确认接受，详见导入清单" in accepted + assert "冲突" not in comment(union([part(), part(span=SEP)])) + + +def test_period_human_some_and_all(): + human = {"period": PeriodOut(SEP[0], SEP[1], "human", signed_by="甲")} + some = comment(union([part(), part(span=SEP, **human)])) + assert "部分期的统计期为人工录入" in some and "均为" not in some + both = comment(union([part(period=PeriodOut(AUG[0], AUG[1], "human")), part(span=SEP, **human)])) + assert "各期的统计期均为人工录入" in both + + +def test_dim_values_vary(): + ls = [{"table": "时段客流", "column": "时段", "segment": "日间", + "periods": [{"start": SEP[0], "end": SEP[1], "missing": ["7-8"], "extra": []}]}] + notes = union([part(), part(span=SEP)], label_sets=ls) + assert "各期的 时段 取值不完全相同：跨期比较前请先确认两期都有的取值" in comment(notes, "时段客流") + assert "取值不完全相同" not in comment(notes) + + +# ---------------------------------------------------------------- 并集：表的种类、退役的表、部分期的表 + + +def test_retired_total_table_keeps_kind_and_last_period_description(): + """目标配方把合计行改为不另存：时段客流_表内合计 退役。并集里仍标 reported_total，说明取 8 月的（只有 8 月有行）； + 明细表同时写着「合计在 时段客流_表内合计」（8 月）和「合计行未导入本表」（9 月）。""" + totals_cols = derive_tables(Recipe.model_validate(FLOW))[0][TOTALS] + notes = union([part(), part(no_totals(), SEP, checks=flow_checks(K1=None, G1=None))], + extra_tables={TOTALS: list(totals_cols)}) + assert notes.problems == [] + assert notes.kinds == {TOTALS: "reported_total"} + t = comment(notes, TOTALS) + assert "原表写明的合计，已按明细重算核对一致" in t and "部分期没有这张表的数据" in t + base = comment(notes, "时段客流") + assert "原表的合计在 时段客流_表内合计，不要与本表相加" in base and "原表的合计行未导入本表" in base + + +def test_new_table_only_in_new_period_is_partial(): + totals_free = no_totals() + notes = union([part(totals_free, checks=flow_checks(K1=None, G1=None)), part(span=SEP)]) + assert notes.problems == [] + t = comment(notes, TOTALS) + assert "部分期没有这张表的数据" in t and "原表写明的合计，已按明细重算核对一致" in t + assert "部分期没有这张表的数据" not in comment(notes) + + +# ---------------------------------------------------------------- 并集：空值只按含这一列的期 + + +def test_nulls_only_from_periods_that_have_the_column(): + """8 月没有分区丙（并集里是结构性空值），9 月分区丙没有空值：不写「空值表示原表为…」；9 月真有空值才写。""" + sep_nulls = {**NULLS_C, "日客流": {**NULLS_C["日客流"], "分区丙": 2}} + data = with_c() + data["sheets"][0]["blocks"][0]["values"]["blank"] = "null" + clean = union([part(), part(data, SEP, nulls=NULLS_C)], target=data, part_nulls=[NULLS, NULLS_C], + null_counts={"日客流": {"分区丙": 31}}) + assert clean.tables["日客流"].columns["分区丙"] == "单位：人次。部分期没有这一列的数据，为空值" + dirty = union([part(), part(data, SEP, nulls=sep_nulls)], target=data, part_nulls=[NULLS, sep_nulls]) + col = dirty.tables["日客流"].columns["分区丙"] + assert "空值表示原表为无数据占位符或空格" in col and col.endswith("部分期没有这一列的数据，为空值") + + +def test_placeholder_meanings_differ_between_periods(): + sep_data = deepcopy(FLOW) + sep_data["sheets"][0]["blocks"][0]["values"]["placeholders"] = [{"text": "—", "meaning": "不适用"}] + notes = union([part(), part(sep_data, SEP, placeholders={"—": 5})], target=sep_data) + assert notes.tables["时段客流"].columns["客流"] == ( + "单位：人次。空值表示原表为无数据或不适用占位符，不代表零；求和时请同时统计非空个数") + same = union([part(), part(span=SEP)]) + assert "无数据占位符" in same.tables["时段客流"].columns["客流"] + + +def test_union_null_counts_zero_skips_null_note(): + notes = union([part(), part(span=SEP)], null_counts={"时段客流": {"客流": 0}}) + assert notes.tables["时段客流"].columns["客流"] == "单位：人次" + + +# ---------------------------------------------------------------- 单期物化（移除、作废之后）与 restart + + +def test_single_materialized_part_uses_col_absent_and_no_partial_wording(): + """A17：只剩 8 月、而目标配方是 9 月加了分区丙的 r3：分区丙 col_absent，不说「部分期」；加 single_after_drop。""" + notes = union([part()], target=with_c()) + assert notes.problems == [] + assert notes.tables["日客流"].columns["分区丙"] == "单位：人次。当前版本的数据中没有这一列，为空值" + for n in notes.tables.values(): + assert "部分期" not in n.comment and "当前版本只含最近一期的数据，此前各期不在当前版本中" in n.comment + assert "按统计期累积多期数据" not in n.comment + # 单期时那一期自己的说法就是准确的 + assert "导入时已逐日核对" in comment(notes) + + +def test_single_materialized_part_table_absent(): + notes = union([part(no_totals(), checks=flow_checks(K1=None, G1=None))], target=FLOW) + assert notes.problems == [] + assert comment(notes, TOTALS) == ("粒度：每个日期的每个合计项。当前版本的数据中没有这张表的行。" + "当前版本只含最近一期的数据，此前各期不在当前版本中") + assert set(notes.tables[TOTALS].columns.values()) >= {"单位：人次。当前版本的数据中没有这一列，为空值"} + + +def test_add_single_after_drop_appends_without_touching_input(): + notes = notes_for() + before = {t: n.comment for t, n in notes.tables.items()} + out = add_single_after_drop(notes) + assert {t: n.comment for t, n in notes.tables.items()} == before + for t, n in out.tables.items(): + assert n.comment == before[t] + "。当前版本只含最近一期的数据，此前各期不在当前版本中" + assert out.fragments[t][-1].key == "single_after_drop" + assert "。".join(f.template for f in out.fragments[t]) == out.templates[t] + assert add_single_after_drop(out).tables["日客流"].comment == out.tables["日客流"].comment + + +# ---------------------------------------------------------------- 「目标那一期」与结构类片段 + + +def test_structural_fragments_come_from_target_period_even_when_backfilled(): + """补传早期：本期（目标配方）排在最前。用户写的表说明取目标那一期的，不取最晚那一期的。""" + old, new = deepcopy(FLOW), deepcopy(FLOW) + old["tables"][0]["note"] = "{列:分区甲} 只统计东门" + new["tables"][0]["note"] = "{列:分区甲} 只统计闸机" + notes = union([part(new, ("2026-07-01", "2026-07-31")), part(old, AUG)], target=new) + assert comment(notes).endswith("分区甲 只统计闸机") and "东门" not in comment(notes) + + +# ---------------------------------------------------------------- 输入不对：记 problems，不能保存 + + +def test_problems_for_inconsistent_inputs(): + assert union([part(), part(span=SEP)], dropped=True).problems + assert build_union_notes(Recipe.model_validate(acc()), [], union_tables={}, null_counts=None, + part_null_counts=None, added=[], retired=[], label_sets=[], gaps=False).problems + # 并集的表结构漏了某一期的表 + tgt = Recipe.model_validate(acc()) + tables = {t: cs for t, cs in derive_tables(tgt)[0].items() if t != TOTALS} + notes = build_union_notes(tgt, [part(), part(span=SEP)], union_tables=tables, null_counts=None, + part_null_counts=None, added=[], retired=[], label_sets=[], gaps=False) + assert any(TOTALS in p for p in notes.problems) + # 各期空值数的期数对不上 + assert union([part(), part(span=SEP)], part_nulls=[NULLS]).problems + + +def test_list_tables_cannot_be_merged_and_are_problems(): + """列表不能按期累积（P3-SPEC 2.2）：真把两期列表并起来，列表合计没有并集说法，记 problems，绝不沿用单期说法。""" + data = deepcopy(LIST_RECIPE) + + def lpart(span): + ex = Extraction(ok=True, derived=[list_item(c) for c in ("金额", "现金", "刷卡")], + sheets=SheetsOut(matched={"s1": "销售"}), lineage={"销售": {"金额": [[1, "销售", "D2", 6, "down"]]}}) + checks = [chk("T1", "column_sum"), chk("R1", "relation_sum_eq", category="data_quality")] + return UnionNotePart(Recipe.model_validate(data), ex, checks, accept("T1"), None, span[0], span[1]) + + tgt = Recipe.model_validate(data) + notes = build_union_notes(tgt, [lpart(AUG), lpart(SEP)], union_tables=derive_tables(tgt)[0], null_counts=None, + part_null_counts=None, added=[], retired=[], label_sets=[], gaps=False) + assert any("列表合计行的核对结论不能按期合并" in p for p in notes.problems) + assert not any("list_total" in p for p in notes.problems) # 不露键名 + assert "一致" not in notes.tables["销售_表内合计"].comment + + +def test_every_union_note_passes_prose_check(): + notes = union([part(), part(with_c(), SEP, nulls=NULLS_C, checks=flow_checks(R1=R1_MISMATCH))], + part_nulls=[NULLS, NULLS_C], gaps=True, + label_sets=[{"table": "时段客流", "column": "时段", "periods": []}]) + assert notes.problems == [] + known = {"列": {c.name for cs in derive_tables(Recipe.model_validate(with_c()))[0].values() for c in cs}, + "表": {"日客流", "时段客流", TOTALS}, "单位": set(P.UNITS)} + for key, tpl in notes.templates.items(): + assert prose_problems(tpl, known=known) == [], key + + +# ---------------------------------------------------------------- 冻结白名单 + + +async def test_store_schema_freezes_snapshot_manifest(): + from types import SimpleNamespace + + from app.core.artifact_store import load + from app.tools.datasource import _FROZEN_SCHEMA_KEYS, _store_schema + from tests.test_recipe_notes import _cache, _source + + assert "snapshot_manifest" in _FROZEN_SCHEMA_KEYS + ctx = SimpleNamespace(run_id="", node_id="") + cache = {**_cache(), "import_mode": "recipe", "import_manifests": ["a" * 64, "b" * 64], + "snapshot_manifest": "s" * 64} + frozen = load(await _store_schema(_source(cache), ctx)) + assert frozen["snapshot_manifest"] == "s" * 64 and frozen["import_manifests"] == ["a" * 64, "b" * 64] + # 没有快照清单的源（单期、手工源）：冻结内容里没有这个键 + one = {**_cache(), "import_mode": "recipe", "import_manifests": ["a" * 64]} + single = load(await _store_schema(_source(one), ctx)) + assert "snapshot_manifest" not in single diff --git a/backend/tests/test_recipe_perf.py b/backend/tests/test_recipe_perf.py index 2721306b..d22676dc 100644 --- a/backend/tests/test_recipe_perf.py +++ b/backend/tests/test_recipe_perf.py @@ -338,3 +338,163 @@ def teardown_module(_module) -> None: for row in RESULTS: target = f"目标 {row['target']} {row['unit']}，{row['ratio']} 倍" if row["target"] is not None else "无目标" print(f"[perf] {row['item']}: {row['value']} {row['unit']}（{target}，{row['verdict']}）") + + +# -------------------------------------------------------------------------- +# 期 3（WP-8，P3-SPEC 11.5）：按期累积的物化、累积试运行、修复与框选的预览 +# -------------------------------------------------------------------------- + +#: 12 期 × 20 万行 × 10 列的合成表（同 p3-probe/probe_perf.py 的结构：每期 30 天 × 6667 个项目，主键 (日期, 项目)） +UNION_PERIODS = 12 +UNION_COLS = ["日期", "项目", "类别"] + [f"值{i}" for i in range(7)] + + +def _union_target() -> Any: + """物化的目标配方：一张 10 列的表，主键 (日期, 项目)。用列表块写出这个表结构（交叉表块一段只有一个值列，造不出 + 10 列 × 20 万行）；物化只看 derive_tables 推出的表结构，与块的形态无关。列表表没有日期轴，U3 不跑（U3 每期每表 + 一条带 rowid 区间的 COUNT，量级可以忽略）。""" + from app.data.recipe_types import Recipe + + return Recipe.model_validate({ + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": SHEET}, "blocks": [{ + "id": "列表1", "layout": "list", "table": "明细", + "columns": [{"header": c, "name": c, "type": "DATE" if c == "日期" else "TEXT" if i < 3 else "INTEGER"} + for i, c in enumerate(UNION_COLS)]}]}], + "tables": [{"name": "明细", "grain": ["日期", "项目"]}], + }) + + +def _union_part_db(path: Path, start: dt.date, seed: int) -> None: + """一期的构建库：执行器建的那种 STRICT 表，rowid 从 1 连续（物化前要断言）。""" + import sqlite3 + + rnd = random.Random(seed) + per = ROWS // 30 + conn = sqlite3.connect(path) + conn.execute("PRAGMA journal_mode=DELETE") + conn.execute('CREATE TABLE "明细" ("日期" TEXT NOT NULL, "项目" TEXT NOT NULL, "类别" TEXT, ' + + ", ".join(f'"值{i}" INTEGER' for i in range(7)) + ', PRIMARY KEY ("日期", "项目")) STRICT') + rows = [] + for d in range(30): + day = (start + dt.timedelta(days=d)).isoformat() + for k in range(per): + rows.append((day, f"项目{k:05d}", "甲" if k % 2 else "乙", *[rnd.randint(0, 99999) for _ in range(7)])) + conn.executemany(f'INSERT INTO "明细" VALUES ({", ".join("?" * 10)})', rows) + conn.commit() + conn.close() + + +@pytest.fixture(scope="module") +def union_parts(tmp_path_factory) -> list[Any]: + from app.data import raw_store + from app.data.recipe_types import UnionPart + + target = _union_target() + root = tmp_path_factory.mktemp("union_parts") + parts = [] + for m in range(UNION_PERIODS): + start = dt.date(2025, 1, 1) + dt.timedelta(days=31 * m) + path = root / f"p{m:02d}.db" + _union_part_db(path, start, seed=700 + m) + end = start + dt.timedelta(days=29) + parts.append(UnionPart(import_id=f"imp{m:02d}", start=start.isoformat(), end=end.isoformat(), + db_path=str(path), recipe=target, db_sha256=raw_store.sha256_file(path), + table_hashes={"明细": f"{m:064x}"})) + return parts + + +def test_perf_materialize_twelve_periods_of_two_hundred_thousand_rows(record_property, union_parts, tmp_path, + monkeypatch): + """11.5：12 期 × 20 万行的累积物化（各期哈希核对 + INSERT…SELECT + U1/U2 + 空值数 + 组合表哈希 + 库哈希）≤ 8 s； + 其中表哈希这一步（union/1 的组合定义，不逐行重算）≤ 0.2 s。进程内存增量只记录。""" + from app.data import recipe_accumulate as ra + + hashing = {"s": 0.0} + real_sha = ra.sha_json + + def timed_sha(obj): + t0 = time.perf_counter() + try: + return real_sha(obj) + finally: + hashing["s"] += time.perf_counter() - t0 + + monkeypatch.setattr(ra, "sha_json", timed_sha) + out = tmp_path / "union.db" + with _Peak() as peak: + t0 = time.perf_counter() + report = ra.materialize_union(out, target=_union_target(), parts=union_parts) + elapsed = time.perf_counter() - t0 + assert report.ok, [c for c in report.checks if c.status != "passed"] + assert report.rows == {"明细": UNION_PERIODS * (ROWS // 30) * 30} + rows = [judge(record_property, "期 3：12 期 × 20 万行累积物化（含核对、哈希）", elapsed, 8.0), + judge(record_property, "期 3：其中组合表哈希", hashing["s"], 0.2), + judge(record_property, "期 3：物化的进程内存增量", peak.delta, None, "MB"), + judge(record_property, "期 3：并集库大小", out.stat().st_size / 1e6, None, "MB")] + assert_all(rows) + + +async def _accumulate_first(client, seed: int) -> str: + """参考夹具 8 月按期累积首次导入（回答问题、宽表改名、提交），返回数据源 id。""" + from tests.test_recipe_acceptance_p3 import answers_for, rename_wide + + raw, fn = flow_workbook(dt.date(2026, 8, 1), 31, seed=seed) + resp = await client.post("/api/datasources/imports/stage", files={"file": (fn, raw, XLSX)}, + data={"name": f"perf_{uuid.uuid4().hex[:8]}"}) + st = resp.json() + st = (await client.post(f"/api/datasources/imports/{st['id']}/answers", + json={"answers": answers_for(st["questions"], "accumulate")})).json() + wide = next(t["name"] for t in st["recipe"]["tables"] if t["name"].endswith("_按日")) + st = (await client.put(f"/api/datasources/imports/{st['id']}/recipe", + json={"recipe": rename_wide(st["recipe"], wide, "日客流")})).json() + st = (await client.post(f"/api/datasources/imports/{st['id']}/trial", json={})).json() + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": _confirms(st)}) + assert resp.status_code == 201, resp.text + return resp.json()["source"]["id"] + + +async def test_perf_flow_fixture_accumulate_trial_and_previews(client, record_property): + """11.5：参考夹具 8 月加 9 月，试运行（含物化）≤ 1.5 s；交互路径：修复预览（D04 去掉「7-8」）、框选预览（c01 框为 + 列表；参考夹具框为交叉表整块，要对整张表跑规则起草）≤ 1 s。上传新一期（扫描 + 试运行 + 物化）只记录。""" + from tests.fixtures.xlsx import lab + from tests.fixtures.xlsx.drift import DRIFT_CASES + + source_id = await _accumulate_first(client, seed=31) + raw, fn = flow_workbook(dt.date(2026, 9, 1), 30, seed=32) + resp, up_s = await _timed(client.post(f"/api/datasources/{source_id}/reupload", + files={"file": (fn, raw, XLSX)})) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["trial"]["status"] == "passed" and st["trial"]["accumulate"]["action"] == "append" + rows = [judge(record_property, "期 3：客流夹具 8+9 月上传新一期（扫描 + 试运行 + 物化）", up_s, None)] + resp, trial_s = await _timed(client.post(f"/api/datasources/imports/{st['id']}/trial", json={})) + assert resp.status_code == 200, resp.text + assert resp.json()["trial"]["accumulate"]["union"]["rows"] == {"日客流": 61, "时段客流": 1037, + "时段客流_表内合计": 183} + rows.append(judge(record_property, "期 3：客流夹具 8+9 月试运行（含物化）", trial_s, 1.5)) + await client.delete(f"/api/datasources/imports/{st['id']}") + + raw, fn = DRIFT_CASES["D04"].build(33) + st = (await client.post(f"/api/datasources/{source_id}/reupload", files={"file": (fn, raw, XLSX)})).json() + fx = next(f for f in st["fixes"] if f["kind"] == "remove_label") + resp, fix_s = await _timed(client.post(f"/api/datasources/imports/{st['id']}/edits/preview", + json={"fix": {"id": fx["id"], "option": "remove"}})) + assert resp.status_code == 200 and resp.json()["ok"], resp.text + rows.append(judge(record_property, "期 3：修复预览（D04 去掉标签）", fix_s, 1.0)) + sel = {"sheet": "客流汇总", "ref": "B4:AG30", "as": "crosstab", "options": {}} + resp, cross_s = await _timed(client.post(f"/api/datasources/imports/{st['id']}/edits/preview", + json={"selection": sel})) + assert resp.status_code == 200, resp.text + rows.append(judge(record_property, "期 3：框选预览（客流夹具框为交叉表整块）", cross_s, 1.0)) + + raw, fn = lab.c01("literal") + st = (await client.post("/api/datasources/imports/stage", files={"file": (fn, raw, XLSX)}, + data={"name": f"perf_{uuid.uuid4().hex[:8]}"})).json() + sel = {"sheet": "月报", "ref": "C5:F8", "as": "list", "options": {"header_rows": 1, "bottom": "box"}} + resp, list_s = await _timed(client.post(f"/api/datasources/imports/{st['id']}/edits/preview", + json={"selection": sel})) + assert resp.status_code == 200 and resp.json()["ok"], resp.text + rows.append(judge(record_property, "期 3：框选预览（c01 框为列表）", list_s, 1.0)) + assert_all(rows) diff --git a/backend/tests/test_recipe_select.py b/backend/tests/test_recipe_select.py new file mode 100644 index 00000000..6824450b --- /dev/null +++ b/backend/tests/test_recipe_select.py @@ -0,0 +1,829 @@ +"""框选转锚点（recipe_select，WP-2，P3-SPEC 第 4 节）：八种 as 各一例成功，泛化不了的每个 code 各一例；c01 的规则 +等价（框选结果与草稿 canonical 相同）；列表复用起草器（占位符、合并填充、跳过空行）；after_title 跳过含数字的格； +客户端表名的校验和撞名（含退役名）；重放比对。 + +网格用读取层（xlsx_cells.read_grid）从合成夹具读出；干跑用期 2 的执行器。期 2 的执行器不给 RegionMark 填块 id， +recipe_select 在没有块 id 时退回用起草器的定位函数近似，这里两种都测（块 id 的那一路用手写的 Extraction）。 +""" +from __future__ import annotations + +import copy +import datetime as dt +import io +import json +from pathlib import Path +from typing import Any + +import pytest +from openpyxl import Workbook, load_workbook + +from app.data import recipe_engine, xlsx_cells, xlsx_scan +from app.data import recipe_suggest as S +from app.data.names import collide_key +from app.data.recipe import apply_patch, recipe_sha256, validate_recipe +from app.data.recipe_fixes import EditRequestError +from app.data.recipe_select import replay_compare, selection_edit +from app.data.recipe_types import ( + EditResult, + Extraction, + GridCell, + Recipe, + RegionMark, + Selection, +) +from tests.fixtures.xlsx import lab, save +from tests.fixtures.xlsx.flow import flow_workbook + +FLOW = Recipe.model_validate(json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json") + .read_text(encoding="utf-8"))).model_dump(mode="json") +SEP = dt.date(2026, 9, 1) +SHEET = "客流汇总" + + +class Book: + """一份合成文件：原件、扫描、网格、规则草稿（完整形式）和按某份配方的干跑。""" + + def __init__(self, raw: bytes, fn: str, *, max_rows: int = 500): + self.raw, self.fn = raw, fn + self.scan = xlsx_scan.scan(raw) + self.grids = {s.name: xlsx_cells.read_grid(raw, self.scan, s.name, max_rows=max_rows) + for s in self.scan.sheets if s.state == "visible" and s.nonempty} + + @property + def grid(self): + return next(iter(self.grids.values())) + + def draft(self): + return S.draft(self.scan, self.grids, self.fn, + validate=lambda r, f, o: validate_recipe(r, facts=f, origin=o), dry_run=self.dry) + + def dry(self, recipe: Any) -> Extraction: + r = recipe if isinstance(recipe, Recipe) else Recipe.model_validate(recipe) + return recipe_engine.execute(r, self.raw, self.fn, self.scan, None) + + +def full(recipe: dict[str, Any]) -> dict[str, Any]: + return Recipe.model_validate(recipe).model_dump(mode="json") + + +def sha(recipe: dict[str, Any]) -> str: + return recipe_sha256(Recipe.model_validate(recipe)) + + +def sel(ref: str, as_: str, sheet: str = SHEET, **options) -> Selection: + return Selection(sheet, ref, as_, options) + + +def codes(res: EditResult) -> list[str]: + return [p.code for p in res.problems] + + +@pytest.fixture(scope="module") +def c01(): + b = Book(*lab.c01("literal")) + d = b.draft() + assert d.complete, d.failures + work = full(d.recipe) + return b, work, b.dry(work) + + +@pytest.fixture(scope="module") +def flow_d01(): + return Book(*flow_workbook(SEP, 30, variant="D01")) + + +def flow_book(variant: str) -> tuple[Book, Extraction]: + b = Book(*flow_workbook(SEP, 30, variant=variant)) + return b, b.dry(FLOW) + + +# ========================================================================== +# 列表：c01（4.6 第 1、3 条） +# ========================================================================== + + +def test_c01_box_gives_the_same_recipe_as_the_rule_draft(c01): + b, work, ext = c01 + res = selection_edit(work, b.grid, sel("C5:F8", "list", "月报"), extraction=ext) + assert res.ok, res.problems + assert res.kind == "selection" and res.key == "list:列表1" and res.block == "列表1" + # 框换算出的块替换规则草稿的同一个块，工作配方的哈希不变：框选得到的规则和起草器完全一样 + assert res.recipe_sha256_after == sha(work) + assert res.expected == {"header": "C5:F5", "data": "C6:F8", "total": "C9:F9"} + assert [(a.kind, a.text) for a in res.anchors] == [ + ("header", "地区"), ("header", "产品"), ("header", "销量"), ("header", "金额"), ("total_word", "合计")] + new = apply_patch(work, res.ops) + rc = replay_compare(res, b.dry(new)) + assert rc.match and rc.diffs == [] and rc.actual == rc.expected + + +def test_c01_shifted_file_replays_with_the_same_table_hashes(c01): + """4.6 第 2 条：整张表挪了位置，按框选得到的配方照样认出，表哈希逐一相同（配方里没有坐标）。""" + b, work, ext = c01 + res = selection_edit(work, b.grid, sel("C5:F8", "list", "月报"), extraction=ext) + new = apply_patch(work, res.ops) + moved = Book(*lab.c01("literal", shift=(3, 2))) + ext2 = moved.dry(new) + assert ext2.ok and [p.code for p in ext2.problems if p.category == "structure"] == [] + assert ext2.table_hashes == b.dry(new).table_hashes + + +def test_c01_box_too_short_is_not_anchorable(c01): + b, work, ext = c01 + res = selection_edit(work, b.grid, sel("C5:F7", "list", "月报"), extraction=ext) + assert not res.ok and res.ops == [] and codes(res) == ["selection_bottom_not_anchorable"] + msg = res.problems[0].message + assert "第 8 行还有数据" in msg and "第 8 行）" in msg and "下边界按规则推断" in msg + assert res.problems[0].category == "recipe" + + +def test_c01_bottom_auto_infers_the_end(c01): + b, work, ext = c01 + res = selection_edit(work, b.grid, sel("C5:F7", "list", "月报", bottom="auto"), extraction=ext) + assert res.ok and res.expected == {"header": "C5:F5", "data": "C6:F8", "total": "C9:F9"} + assert res.recipe_sha256_after == sha(work) + assert any("重放的数据区到第 8 行" in s for s in res.summary) + rc = replay_compare(res, b.dry(apply_patch(work, res.ops))) + assert rc.match + + +@pytest.mark.parametrize("ref", ["C5:F9", "C5:F10"]) +def test_c01_box_including_the_total_row_reads_it_as_total(c01, ref): + """把合计行（和它下面的空行）也框进来：合计行照起草器认作合计行，不当数据行导入（H2：当数据行的话 + SUM(金额) 翻倍，干跑、核对、重放比对都看不出来）。框底多带空行时与框到合计行相同，配方与规则草稿逐字相同。""" + b, work, ext = c01 + res = selection_edit(work, b.grid, sel(ref, "list", "月报"), extraction=ext) + assert res.ok, res.problems + assert res.expected == {"header": "C5:F5", "data": "C6:F8", "total": "C9:F9"} + assert res.recipe_sha256_after == sha(work) + assert ("total_word", "合计") in [(a.kind, a.text) for a in res.anchors] + # 配方为空时同样：框选新建的配方与规则草稿相同 + fresh = selection_edit(None, b.grid, sel(ref, "list", "月报"), extraction=None) + assert fresh.ok and fresh.recipe_sha256_after == sha(work) + new = apply_patch(work, res.ops) + assert replay_compare(res, b.dry(new)).match + + +def test_c01_box_down_to_the_notes_reports_the_total_in_the_middle(c01): + """框一直拉到下方的备注、制表人行：合计行夹在框中间，无法确定它汇总的是哪几行，报错并建议把下边移到合计行， + 不把合计、备注当数据导入。""" + b, work, ext = c01 + res = selection_edit(work, b.grid, sel("C5:F12", "list", "月报"), extraction=ext) + assert not res.ok and codes(res) == ["selection_bottom_not_anchorable"] + msg = res.problems[0].message + assert "第 9 行像合计行" in msg and "把框的下边移到第 9 行" in msg + assert res.problems[0].cells == ["月报!C9"] + + +def test_box_with_trailing_blank_rows_ends_at_the_last_filled_row(): + """框底多带了空行、空行下面是备注：下边界看框内最后一个非空行的下一行（空行，遇到空行停止），与框到最后 + 一个数据行相同。看框的下一行的话会撞上备注，提示「把框扩大到第 5 行」，等于让人把备注也框进来。""" + wb = Workbook() + ws = wb.active + ws.title = "名单" + for r, row in enumerate([("地区", "销量"), ("分区甲", 3), ("分区乙", 5), (None, None), ("注：合成备注", None)], + start=1): + for c, v in enumerate(row, start=1): + if v is not None: + ws.cell(r, c, v) + b = Book(save(wb), "tail.xlsx") + tight = selection_edit(None, b.grid, sel("A1:B3", "list", "名单"), extraction=None) + loose = selection_edit(None, b.grid, sel("A1:B4", "list", "名单"), extraction=None) + assert tight.ok and loose.ok, loose.problems + assert loose.expected == tight.expected == {"header": "A1:B1", "data": "A2:B3", "total": None} + assert loose.recipe_sha256_after == tight.recipe_sha256_after + assert apply_patch({}, loose.ops)["sheets"][0]["blocks"][0]["rows"] == {"blank_rows": "stop", "total_row": None} + + +def _subtotals() -> Book: + """列表中间夹着小计：分区甲两行、小计、分区乙两行、合计。""" + wb = Workbook() + ws = wb.active + ws.title = "明细" + rows = [("地区", "产品", "金额"), ("分区甲", "产品甲", 10), ("分区甲", "产品乙", 20), ("小计", None, 30), + ("分区乙", "产品甲", 5), ("分区乙", "产品乙", 6), ("合计", None, 41)] + for r, row in enumerate(rows, start=1): + for c, v in enumerate(row, start=1): + if v is not None: + ws.cell(r, c, v) + return Book(save(wb), "subtotal.xlsx") + + +@pytest.mark.parametrize("ref,bottom,where", [ + ("A1:C7", "box", "第 4 行像合计行"), # 小计夹在框中间 + ("A1:C4", "box", "第 4 行像合计行"), # 框到小计为止：它下面紧接着还有明细，是夹在中间的小计 + ("A1:C3", "box", "第 4 行像合计行"), # 框到小计上一行：下一行是小计，同上 + ("A1:C3", "auto", "第 4 行像合计行"), # 按规则推断：起草器在这里记失败，框选同样不收 +]) +def test_subtotal_in_the_middle_is_never_imported_as_data(ref, bottom, where): + b = _subtotals() + res = selection_edit(None, b.grid, sel(ref, "list", "明细", bottom=bottom), extraction=None) + assert not res.ok and set(codes(res)) == {"selection_bottom_not_anchorable"}, res.problems + assert where in res.problems[0].message and "明细!A4" in res.problems[0].cells + + +def test_total_word_in_another_column_is_not_imported_as_data(): + """合计字样写在第二列（第一列空着）：起草器记失败，框选同样报错，不当明细导入。""" + wb = Workbook() + ws = wb.active + ws.title = "明细" + for r, row in enumerate([("地区", "产品", "金额"), ("分区甲", "产品甲", 10), ("分区乙", "产品乙", 20), + (None, "合计", 30)], start=1): + for c, v in enumerate(row, start=1): + if v is not None: + ws.cell(r, c, v) + b = Book(save(wb), "total_b.xlsx") + res = selection_edit(None, b.grid, sel("A1:C4", "list", "明细"), extraction=None) + assert codes(res) == ["selection_bottom_not_anchorable"] + assert "B 列是合计字样" in res.problems[0].message and res.problems[0].cells == ["明细!B4"] + + +def test_c01_box_on_data_row_header_not_text(c01): + b, work, ext = c01 + res = selection_edit(work, b.grid, sel("C6:F8", "list", "月报"), extraction=ext) + assert codes(res) == ["selection_header_not_text"] + + +def test_c01_box_with_blank_header_column(c01): + b, work, ext = c01 + res = selection_edit(work, b.grid, sel("C5:G8", "list", "月报"), extraction=ext) + assert codes(res) == ["selection_header_blank"] and res.problems[0].cells == ["月报!G5"] + + +def test_list_without_recipe_creates_one_by_root_replace(c01): + """配方为空（规则起草失败）时用根替换新建配方：框选也是起草失败时不用 AI 的兜底（4.2 第 7 条）。""" + b, work, _ext = c01 + res = selection_edit(None, b.grid, sel("C5:F8", "list", "月报"), extraction=None) + assert res.ok and res.ops[0]["op"] == "replace" and res.ops[0]["path"] == "" + assert res.recipe_sha256_after == sha(work) + + +def test_header_duplicate(): + wb = Workbook() + ws = wb.active + ws.title = "明细" + for r, row in enumerate([("地区", "金额", "金 额"), ("分区甲", 1, 2), ("分区乙", 3, 4)], start=1): + for c, v in enumerate(row, start=1): + ws.cell(r, c, v) + b = Book(save(wb), "dup.xlsx") + res = selection_edit(None, b.grid, sel("A1:C3", "list", "明细"), extraction=None) + assert codes(res) == ["selection_header_duplicate"] + + +def test_extra_columns_next_to_the_box(c01): + b, work, ext = c01 + res = selection_edit(work, b.grid, sel("C5:E8", "list", "月报"), extraction=ext) + assert codes(res) == ["selection_extra_columns"] + assert "「金额」" in res.problems[0].message and "C5:F8" in res.problems[0].message + assert res.problems[0].cells == ["月报!F5"] + # 被替换的块上已经按表头忽略了这一列：不算多出的列 + with_ignore = copy.deepcopy(work) + with_ignore["sheets"][0]["blocks"][0]["ignore_columns"] = [{"header": "金额", "reason": "合成理由"}] + ok = selection_edit(with_ignore, b.grid, sel("C5:E8", "list", "月报"), extraction=ext) + assert ok.ok, ok.problems + + +def test_box_bottom_on_blank_row_stops_there(): + b = _stacked("补充明细") + res = selection_edit(None, b.grid, sel("A2:B4", "list", "汇总"), extraction=None) + assert res.ok, res.problems + blk = apply_patch({}, res.ops)["sheets"][0]["blocks"][0] + assert blk["rows"] == {"blank_rows": "stop", "total_row": None} + assert res.expected == {"header": "A2:B2", "data": "A3:B4", "total": None} + + +def test_list_options_are_validated(c01): + b, work, ext = c01 + with pytest.raises(EditRequestError) as e: + selection_edit(work, b.grid, sel("C5:F8", "list", "月报", header_rows=4), extraction=ext) + assert e.value.code == "edit_invalid" + with pytest.raises(EditRequestError): + selection_edit(work, b.grid, sel("C5:F8", "list", "月报", bottom="下面"), extraction=ext) + + +def test_out_of_grid_on_truncated_grid(): + b = Book(*lab.c01("literal"), max_rows=6) + assert b.grid.truncated + res = selection_edit(None, b.grid, sel("C5:F8", "list", "月报"), extraction=None) + assert codes(res) == ["selection_out_of_grid"] + auto = selection_edit(None, b.grid, sel("C5:F6", "list", "月报", bottom="auto"), extraction=None) + assert auto.ok and "数据延续到已读入范围之外，完整范围在试运行时核对" in auto.notes + + +# ========================================================================== +# 列表复用起草器：合并表头、占位符、合并填充、跳过空行 +# ========================================================================== + + +@pytest.mark.parametrize("build,sheet,ref,n", [ + (lambda: lab.c08("gap"), "明细", "A1:C6", 1), # 中间一行空白：blank_rows=skip + (lambda: lab.c04(), "合并", "A3:F10", 2), # 两行表头、地区纵向合并：merged_data=fill +]) +def test_list_selection_keeps_draft_settings(build, sheet, ref, n): + b = Book(*build()) + d = b.draft() + work = full(d.recipe) + res = selection_edit(work, b.grid, sel(ref, "list", sheet, header_rows=n), extraction=b.dry(work)) + assert res.ok, res.problems + assert res.recipe_sha256_after == sha(work), "框选的列表规则与起草器相同" + blk = apply_patch(work, res.ops)["sheets"][0]["blocks"][0] + if sheet == "明细": + assert blk["rows"]["blank_rows"] == "skip" + assert any("中间的空行跳过" in s for s in res.summary) + else: + assert blk["merged_data"] == "fill" and blk["header_rows"] == 2 + + +def test_list_selection_keeps_placeholders(): + wb = Workbook() + ws = wb.active + ws.title = "名单" + for r, row in enumerate([("地区", "销量"), ("分区甲", 3), ("分区乙", "-"), ("分区丙", 7)], start=1): + for c, v in enumerate(row, start=1): + ws.cell(r, c, v) + b = Book(save(wb), "ph.xlsx") + res = selection_edit(None, b.grid, sel("A1:B4", "list", "名单"), extraction=None) + assert res.ok + blk = apply_patch({}, res.ops)["sheets"][0]["blocks"][0] + assert blk["values"]["placeholders"] == [{"text": "-", "meaning": "无数据"}] + assert blk["columns"][1]["type"] == "INTEGER" + + +# ========================================================================== +# after_title、表名 +# ========================================================================== + + +def _stacked(title2: str | None) -> Book: + """上下两张表头相同的列表：上面一张有标题「销售汇总」，下面一张上方是含年月的说明（和可选的第二行标题）。""" + wb = Workbook() + ws = wb.active + ws.title = "汇总" + ws["A1"] = "销售汇总" + rows = [(2, ("地区", "销量")), (3, ("分区甲", 1)), (4, ("分区乙", 2)), + (8, ("地区", "销量")), (9, ("分区丙", 3)), (10, ("分区丁", 4))] + for r, row in rows: + for c, v in enumerate(row, start=1): + ws.cell(r, c, v) + ws["A7"] = "2026年9月 补充" + if title2: + ws["A6"] = title2 + return Book(save(wb), "stacked.xlsx") + + +def test_after_title_skips_cells_with_digits(): + b = _stacked("补充明细") + res = selection_edit(None, b.grid, sel("A8:B10", "list", "汇总"), extraction=None) + assert res.ok, res.problems + blk = apply_patch({}, res.ops)["sheets"][0]["blocks"][0] + assert blk["after_title"] == "补充明细" + assert ("after_title", "补充明细") in [(a.kind, a.text) for a in res.anchors] + assert validate_recipe(apply_patch({}, res.ops))[1] == [] + + +def test_after_title_ambiguous_when_only_titles_with_digits(): + b = _stacked(None) + res = selection_edit(None, b.grid, sel("A8:B10", "list", "汇总"), extraction=None) + assert codes(res) == ["selection_anchor_ambiguous"] + assert "含年月或数字" in res.problems[0].message + + +def test_client_table_name_is_validated_and_checked_for_collisions(c01): + b, work, ext = c01 + bad = selection_edit(None, b.grid, sel("C5:F8", "list", "月报", table="1号表"), extraction=None) + assert codes(bad) == ["selection_name_invalid"] + taken = selection_edit(None, b.grid, sel("C5:F8", "list", "月报", table="旧销售"), extraction=None, + retired_names={"旧销售": ["销量"]}) + assert codes(taken) == ["selection_name_taken"] + taken2 = selection_edit(None, b.grid, sel("C5:F8", "list", "月报", table="ＳＡＬＥＳ"), extraction=None, + retired_names={"sales": []}) + assert codes(taken2) in (["selection_name_invalid"], ["selection_name_taken"]) + ok = selection_edit(work, b.grid, sel("C5:F8", "list", "月报", table="销售月报"), extraction=ext) + assert ok.ok + new = apply_patch(work, ok.ops) + assert [t["name"] for t in new["tables"]] == ["销售月报", "销售月报_表内合计"] + assert validate_recipe(new)[1] == [] + + +def test_new_column_name_follows_retired_spelling(c01): + b, work, ext = c01 + res = selection_edit(work, b.grid, sel("C5:F8", "list", "月报"), extraction=ext, + retired_names={"月报": ["销量"]}) + assert res.ok and res.notes == [] + wb = Workbook() + ws = wb.active + ws.title = "表" + for r, row in enumerate([("地区", "pm2_5"), ("分区甲", 1), ("分区乙", 2)], start=1): + for c, v in enumerate(row, start=1): + ws.cell(r, c, v) + b2 = Book(save(wb), "pm.xlsx") + first = selection_edit(None, b2.grid, sel("A1:B3", "list", "表", table="监测"), extraction=None) + rec = full(apply_patch({}, first.ops)) + assert rec["sheets"][0]["blocks"][0]["columns"][1]["name"] == "pm2_5" + # 当前版本里「监测」表的「PM2_5」已退役：重新框选这张表时，新列沿用退役列的写法（SQLite 列名不分大小写） + res2 = selection_edit(rec, b2.grid, sel("A1:B3", "list", "表"), extraction=b2.dry(rec), + retired_names={"监测": ["PM2_5"]}) + assert res2.ok and apply_patch(rec, res2.ops)["sheets"][0]["blocks"][0]["columns"][1]["name"] == "PM2_5" + assert any("PM2_5" in n for n in res2.notes) + + +# ========================================================================== +# 重叠 +# ========================================================================== + + +def test_overlap_with_two_blocks(): + b = Book(*lab.c08("side")) + d = b.draft() + work = full(d.recipe) + assert len(work["sheets"][0]["blocks"]) == 2 + res = selection_edit(work, b.grid, sel("A1:E3", "list", "并排"), extraction=b.dry(work)) + assert codes(res) == ["selection_overlap"] + # 只框一张：替换它，沿用块 id + one = selection_edit(work, b.grid, sel("D1:E3", "list", "并排"), extraction=b.dry(work)) + assert one.ok and one.key == "list:列表2" and one.recipe_sha256_after == sha(work) + + +def test_non_overlapping_box_adds_a_block(c01): + b, work, ext = c01 + wb = Workbook() + ws = wb.active + ws.title = "月报" + for r, row in enumerate([("地区", "销量"), ("分区甲", 1), ("分区乙", 2)], start=1): + for c, v in enumerate(row, start=1): + ws.cell(r, c, v) + for r, row in enumerate([("部门", "费用"), ("部门甲", 5), ("部门乙", 7)], start=6): + for c, v in enumerate(row, start=1): + ws.cell(r, c, v) + bk = Book(save(wb), "two.xlsx") + first = selection_edit(None, bk.grid, sel("A1:B3", "list", "月报"), extraction=None) + rec = full(apply_patch({}, first.ops)) + second = selection_edit(rec, bk.grid, sel("A6:B8", "list", "月报"), extraction=bk.dry(rec)) + assert second.ok and second.ops[0] == {"op": "add", "path": "/sheets/0/blocks/-", "value": second.ops[0]["value"]} + new = apply_patch(rec, second.ops) + assert [blk["id"] for blk in new["sheets"][0]["blocks"]] == ["列表1", "列表2"] + assert validate_recipe(new)[1] == [] + + +# ========================================================================== +# 交叉表（整块） +# ========================================================================== + + +def test_crosstab_from_rule_draft(flow_d01): + b = flow_d01 + res = selection_edit(None, b.grid, sel("C5:H12", "crosstab"), extraction=None, draft_fn=b.draft) + assert res.ok, res.problems + assert res.expected == {"axis": "C4:AF4", "labels": "B5:B30", "values": "C5:AF30"} + assert res.anchors[0].kind == "axis" + new = apply_patch({}, res.ops) + assert validate_recipe(new)[1] == [] + rc = replay_compare(res, b.dry(new)) + assert rc.match, rc.diffs + # 已有交叉表：框在它上面替换它。起草器的名字按来源对齐到现行配方，版式没变时就是原来的配方 + rep = selection_edit(FLOW, b.grid, sel("C5:H12", "crosstab"), extraction=b.dry(FLOW), draft_fn=b.draft) + assert rep.ok and rep.key == "crosstab:交叉表" + assert rep.recipe_sha256_after == sha(FLOW) + # 现行配方里另有一条关系占了 R2：起草的关系与它重号时顺延 + extra = copy.deepcopy(FLOW) + extra["relations"][1]["id"] = "R5" + rep2 = selection_edit(extra, b.grid, sel("C5:H12", "crosstab"), extraction=b.dry(extra), draft_fn=b.draft) + new2 = apply_patch(extra, rep2.ops) + assert validate_recipe(new2)[1] == [] and len(new2["sheets"][0]["blocks"]) == 1 + assert sorted(r["id"] for r in new2["relations"]) == ["R1", "R5"] + + +def test_crosstab_on_new_sheet_renumbers_clashing_relations(flow_d01): + b = flow_d01 + other = copy.deepcopy(FLOW) + other["sheets"][0]["match"]["name"] = "另一张表" + other["sheets"][0]["id"] = "s9" + res = selection_edit(other, b.grid, sel("C5:H12", "crosstab"), extraction=None, draft_fn=b.draft) + assert res.ok, res.problems + new = apply_patch(other, res.ops) + assert [s["id"] for s in new["sheets"]] == ["s9", "s2"] + assert [r["id"] for r in new["relations"]] == ["R1", "R2", "R3", "R4"] + names = [t["name"] for t in new["tables"]] + assert len({collide_key(n) for n in names}) == len(names), names + assert validate_recipe(new)[1] == [] + + +def test_selection_options_text_never_enters_the_patch(c01): + b, work, ext = c01 + res = selection_edit(work, b.grid, sel("C5:F8", "list", "月报", note="随便写的说明", segment="恶意"), + extraction=ext) + assert res.ok and "随便写的说明" not in json.dumps(res.ops, ensure_ascii=False) + + +def test_crosstab_needs_enough_overlap(flow_d01): + b = flow_d01 + res = selection_edit(None, b.grid, sel("AG40:AH45", "crosstab"), extraction=None, draft_fn=b.draft) + assert codes(res) == ["selection_no_crosstab"] + lst = Book(*lab.c01("literal")) + res2 = selection_edit(None, lst.grid, sel("C5:F8", "crosstab", "月报"), extraction=None, draft_fn=lst.draft) + assert codes(res2) == ["selection_no_crosstab"] + + +# ========================================================================== +# 分段、合计行、分段标题 +# ========================================================================== + + +def test_segment_measures_on_unclaimed_rows(): + b, ext = flow_book("D21") + res = selection_edit(FLOW, b.grid, sel("B32:AF32", "segment", role="measures", table="补录"), extraction=ext) + assert res.ok, res.problems + assert res.key == "segment:补录" and res.expected["labels"] == "B32" + new = apply_patch(FLOW, res.ops) + seg = new["sheets"][0]["blocks"][0]["segments"][-1] + assert seg["role"] == "measures" and seg["locate"]["by"] == "labels" and seg["labels"]["expect"] == ["补录（人次）"] + assert seg["measures"] == {"补录（人次）": "补录"} + assert new["tables"][-1] == {"name": "补录", "grain": ["日期"], "kind": "data", "units": {"补录": "人次"}, "note": ""} + assert validate_recipe(new)[1] == [] + rc = replay_compare(res, b.dry(new)) + assert rc.match, rc.diffs + + +def test_segment_problems(flow_d01): + b, ext = flow_book("D21") + with pytest.raises(EditRequestError): + selection_edit(FLOW, b.grid, sel("B32:AF32", "segment"), extraction=ext) + assert codes(selection_edit(FLOW, b.grid, sel("C32:AF32", "segment", role="measures"), extraction=ext)) == [ + "selection_not_in_block"] + assert codes(selection_edit(FLOW, b.grid, sel("B31:AF32", "segment", role="measures"), extraction=ext)) == [ + "selection_label_blank"] + assert codes(selection_edit(FLOW, b.grid, sel("B6:AF7", "segment", role="measures"), extraction=ext)) == [ + "selection_overlap"] + taken = selection_edit(FLOW, b.grid, sel("B32:AF32", "segment", role="measures", table="日客流"), extraction=ext) + assert codes(taken) == ["selection_name_taken"] + + +def _early_grid(b: Book, title: str = "早间时段客流（人次）", labels: tuple[str, ...] = ("5-6", "6-7")): + """D21 的网格上，第 32 行改成分段标题、下面两行是新的时段（框选为维度分段用）。""" + grid = copy.deepcopy(b.grid) + grid.cells[(32, 2)] = GridCell(title) + for c in range(3, 5): + grid.cells.pop((32, c), None) + for k, h in enumerate(labels): + grid.cells[(33 + k, 2)] = GridCell(h) + for c in range(3, 33): + grid.cells[(33 + k, c)] = GridCell(10 + c) + grid.__dict__.pop("_rows", None) + return grid + + +def test_segment_dimension_with_section_title_joins_the_matching_table(): + """起草器第 5 条：解析器相同（hour_range）、标签不相交、单位相同的维度分段写进同一张表「时段客流」，常量列 + 「时段类别」照第 8b 条取分段标题的候选词「早间」，与「日间」「夜间」不同。不新建表。""" + b, ext = flow_book("D21") + grid = _early_grid(b) + res = selection_edit(FLOW, grid, sel("B33:AF34", "segment", role="dimension"), extraction=ext) + assert res.ok, res.problems + assert len(res.ops) == 1, "并入已有的表，不新增表定义" + seg = res.ops[0]["value"] + assert seg == {"id": "早间", "role": "dimension", "table": "时段客流", + "locate": {"by": "section_title", "title": "早间时段客流（人次）", "segment": None}, + "labels": {"expect": ["5-6", "6-7"]}, + "dim": {"name": "时段", "parser": "hour_range", "derive": {"起始小时": "start", "结束小时": "end"}}, + "value": "客流", "const": {"时段类别": {"pick": "早间"}}, "stop_parser": None} + assert res.ops[0]["path"] == "/sheets/0/blocks/0/segments/4", "插在已有分段之后" + assert res.expected["title"] == "B32" and res.notes == [] + assert any("并入表「时段客流」" in x for x in res.summary) and any("「时段类别」取「早间」" in x for x in res.summary) + assert validate_recipe(apply_patch(FLOW, res.ops))[1] == [] + + +def test_segment_dimension_merged_replays_on_the_file(): + """并入「时段客流」后按真文件干跑：新时段的 60 行写进同一张表，没有问题，重放比对一致。""" + raw, fn = flow_workbook(SEP, 30, variant="D21") + wb = load_workbook(io.BytesIO(raw), data_only=True) + ws = wb.active + ws.cell(32, 2).value = "早间时段客流（人次）" + for c in range(3, 33): + ws.cell(32, c).value = None + for k, h in enumerate(("5-6", "6-7")): + ws.cell(33 + k, 2).value = h + for c in range(3, 33): + ws.cell(33 + k, c).value = 10 + c + k + b = Book(save(wb), fn) + res = selection_edit(FLOW, b.grid, sel("B33:AF34", "segment", role="dimension"), extraction=b.dry(FLOW)) + assert res.ok and res.ops[0]["value"]["table"] == "时段客流" + new = apply_patch(FLOW, res.ops) + ext = b.dry(new) + assert ext.ok and ext.problems == [] + assert ext.expected_rows["时段客流"] == b.dry(FLOW).expected_rows["时段客流"] + 60 + assert replay_compare(res, ext).match + + +def test_segment_dimension_new_table_says_why_it_was_not_merged(): + """有同类的表却并不进去时新建表，并在说明里写明没有并入、为什么：客户端指定了新表名；那张表的分段没有常量列 + （并入要给现有分段加列）；分段标题里选不出与已有取值不同的词。标签和已有分段重叠的不算同类，不提。""" + b, ext = flow_book("D21") + named = selection_edit(FLOW, _early_grid(b), sel("B33:AF34", "segment", role="dimension", table="早间客流"), + extraction=ext) + assert named.ok and named.ops[1]["value"]["name"] == "早间客流" + assert named.notes == ["按指定的表名新建了表，没有并入同类的表「时段客流」"] + # 那张表只有一个分段、没有常量列 + single = copy.deepcopy(FLOW) + segs = single["sheets"][0]["blocks"][0]["segments"] + segs[1]["const"] = {} + del segs[2:] + alone = selection_edit(single, _early_grid(b), sel("B33:AF34", "segment", role="dimension"), extraction=ext) + assert alone.ok and alone.ops[0]["value"]["table"] == "早间时段客流" + assert "没有并入同类的表「时段客流」" in alone.notes[0] and "没有用来区分分段的常量列" in alone.notes[0] + # 标题里能区分的词与已有取值相同 + same = selection_edit(FLOW, _early_grid(b, title="日间时段客流（人次）"), + sel("B33:AF34", "segment", role="dimension"), extraction=ext) + assert same.ok and same.ops[0]["value"]["table"] != "时段客流" + assert "分段标题里没有能与已有分段区分的词" in same.notes[0] + # 标签与日间重叠：不是同类，新建表、不提并入 + overlap = selection_edit(FLOW, _early_grid(b, labels=("7-8", "8-9")), sel("B33:AF34", "segment", role="dimension"), + extraction=ext) + assert overlap.ok and overlap.ops[0]["value"]["table"] == "早间时段客流" and overlap.notes == [] + + +def test_segment_dimension_text_label_too_long(): + b, ext = flow_book("D21") + grid = copy.deepcopy(b.grid) + grid.cells[(32, 2)] = GridCell("补" * 41) + grid.__dict__.pop("_rows", None) + res = selection_edit(FLOW, grid, sel("B32:AF32", "segment", role="dimension"), extraction=ext) + assert codes(res) == ["selection_label_unparsed"] + + +def test_derived_d18(): + b, ext = flow_book("D18") + res = selection_edit(FLOW, b.grid, sel("B21:AF21", "derived", keep=True), extraction=ext) + assert res.ok and res.key == "derived:日间合计" + new = apply_patch(FLOW, res.ops) + assert validate_recipe(new, base=FLOW)[1] == [] + ext2 = b.dry(new) + assert ext2.ok + assert replay_compare(res, ext2).match + + +def test_derived_problems(): + b, ext = flow_book("D18") + assert codes(selection_edit(FLOW, b.grid, sel("B11:AF11", "derived"), extraction=ext)) == ["selection_label_unparsed"] + b2, ext2 = flow_book("D21") + res = selection_edit(FLOW, b2.grid, sel("B32:AF32", "derived"), extraction=ext2) + assert codes(res) == ["selection_label_unparsed"] + grid = copy.deepcopy(b2.grid) + grid.cells[(32, 2)] = GridCell("7-12 时合计") + grid.__dict__.pop("_rows", None) + assert codes(selection_edit(FLOW, grid, sel("B32:AF32", "derived"), extraction=ext2)) == [ + "selection_not_after_segment"] + + +def test_section_title_d09_keeps_const(): + b, ext = flow_book("D09") + res = selection_edit(FLOW, b.grid, sel("B9", "section_title", segment="日间"), extraction=ext) + assert res.ok and res.key == "section_title:日间" and res.breaking is False + assert res.ops == [{"op": "replace", "path": "/sheets/0/blocks/0/segments/1/locate", + "value": {"by": "section_title", "title": "日间分时段客流（人次）", "segment": None}}] + new = apply_patch(FLOW, res.ops) + assert validate_recipe(new, base=FLOW)[1] == [] + assert b.dry(new).ok + + +def test_section_title_does_not_guess_a_new_word_when_old_pick_is_gone(): + """新标题里没有原来的常量取值：常量列整列换成哪个词只能由人选（4.1「不猜」）。框选不替人挑第一个候选词， + 报 selection_label_unparsed，指到修复按钮（它把候选词逐个列成选项）和配方面板。""" + b, ext = flow_book("D09") + grid = copy.deepcopy(b.grid) + grid.cells[(9, 2)] = GridCell("白天时段客流（人次）") + grid.__dict__.pop("_rows", None) + res = selection_edit(FLOW, grid, sel("B9", "section_title", segment="日间"), extraction=ext) + assert not res.ok and res.ops == [] and codes(res) == ["selection_label_unparsed"] + msg = res.problems[0].message + assert "常量列「时段类别」现在取「日间」" in msg and "修复按钮" in msg and "配方面板" in msg + assert res.problems[0].cells == [f"{SHEET}!B9"] + with pytest.raises(EditRequestError): + selection_edit(FLOW, grid, sel("B9", "section_title", segment="夜间合计"), extraction=ext) + assert codes(selection_edit(FLOW, grid, sel("B9:B10", "section_title", segment="日间"), extraction=ext)) == [ + "selection_not_in_block"] + + +# ========================================================================== +# 忽略 +# ========================================================================== + + +def test_ignore_rows_d21(): + b, ext = flow_book("D21") + res = selection_edit(FLOW, b.grid, sel("B32:D32", "ignore_rows", reason="补录行不导入"), extraction=ext) + assert res.ok and res.key == "ignore_rows:交叉表" + assert res.ops == [{"op": "add", "path": "/sheets/0/blocks/0/ignore_rows/-", + "value": {"label": "补录（人次）", "reason": "补录行不导入"}}] + assert validate_recipe(apply_patch(FLOW, res.ops))[1] == [] + with pytest.raises(EditRequestError) as e: + selection_edit(FLOW, b.grid, sel("B32:D32", "ignore_rows"), extraction=ext) + assert e.value.code == "reason_required" + + +def test_ignore_rows_with_digits(): + b, ext = flow_book("D21") + grid = copy.deepcopy(b.grid) + grid.cells[(32, 2)] = GridCell("注：9月补录") + grid.__dict__.pop("_rows", None) + res = selection_edit(FLOW, grid, sel("B32:D32", "ignore_rows", reason="x"), extraction=ext) + assert codes(res) == ["selection_anchor_has_digits"] + + +def test_ignore_columns_d16(): + b, ext = flow_book("D16") + res = selection_edit(FLOW, b.grid, sel("AG4:AG30", "ignore_columns", reason="右侧合计列"), extraction=ext) + assert res.ok and res.key == "ignore_columns:交叉表" + assert res.ops == [{"op": "add", "path": "/sheets/0/blocks/0/ignore_columns/-", + "value": {"header": "合计", "reason": "右侧合计列"}}] + assert validate_recipe(apply_patch(FLOW, res.ops))[1] == [] + assert codes(selection_edit(FLOW, b.grid, sel("AE4:AE9", "ignore_columns", reason="x"), extraction=ext)) == [ + "selection_not_in_block"] + + +def test_ignore_columns_header_problems(): + b, ext = flow_book("D16") + grid = copy.deepcopy(b.grid) + grid.cells[(4, 34)] = GridCell(123) + grid.cells[(4, 35)] = GridCell("第2组合计") + grid.__dict__.pop("_rows", None) + assert codes(selection_edit(FLOW, grid, sel("AH4", "ignore_columns", reason="x"), extraction=ext)) == [ + "selection_header_not_text"] + assert codes(selection_edit(FLOW, grid, sel("AI4", "ignore_columns", reason="x"), extraction=ext)) == [ + "selection_anchor_has_digits"] + assert codes(selection_edit(FLOW, grid, sel("AJ4", "ignore_columns", reason="x"), extraction=ext)) == [ + "selection_header_blank"] + + +def test_ignore_outside(): + b, ext = flow_book("D01") + grid = copy.deepcopy(b.grid) + grid.cells[(33, 2)] = GridCell("补录说明") + grid.cells[(33, 3)] = GridCell(123) + grid.cells[(34, 3)] = GridCell(456) + grid.cells[(35, 2)] = GridCell("9月补录") + grid.cells[(35, 3)] = GridCell(789) + grid.__dict__.pop("_rows", None) + res = selection_edit(FLOW, grid, sel("B33:C33", "ignore_outside", reason="说明行"), extraction=ext) + assert res.ok and res.key == "ignore_outside:s1" + assert res.ops == [{"op": "add", "path": "/sheets/0/ignore_outside/-", + "value": {"anchor": "补录说明", "reason": "说明行"}}] + assert validate_recipe(apply_patch(FLOW, res.ops))[1] == [] + assert codes(selection_edit(FLOW, grid, sel("B34:C34", "ignore_outside", reason="x"), extraction=ext)) == [ + "selection_no_anchor"] + assert codes(selection_edit(FLOW, grid, sel("B35:C35", "ignore_outside", reason="x"), extraction=ext)) == [ + "selection_anchor_has_digits"] + + +def test_selection_on_sheet_without_blocks(): + b, _ext = flow_book("D21") + res = selection_edit(None, b.grid, sel("B32:D32", "ignore_rows", reason="x"), extraction=None) + assert codes(res) == ["selection_not_in_block"] + + +# ========================================================================== +# 重放比对（4.5） +# ========================================================================== + + +def _edit(key: str, expected: dict, block: str | None = "列表1") -> EditResult: + return EditResult(ok=True, kind="selection", key=key, title="", summary=[], ops=[], anchors=[], notes=[], + problems=[], expected=expected, block=block) + + +def test_replay_compare_reports_extra_rows_in_words(): + ext = Extraction(ok=True, regions=[ + RegionMark("月报", "col_header", "C5:F5", "列表1"), RegionMark("月报", "value", "C6:F10", "列表1"), + RegionMark("月报", "value", "H6:I20", "列表2"), RegionMark("月报", "outside_text", "C1:C3", None)]) + rc = replay_compare(_edit("list:列表1", {"header": "C5:F5", "data": "C6:F8", "total": None}), ext) + assert not rc.match and rc.actual == {"header": "C5:F5", "data": "C6:F10", "total": None} + assert rc.diffs == ["重放的数据区到第 10 行，比框多 2 行（第 9–10 行）"] + + +def test_replay_compare_missing_total_and_window(): + ext = Extraction(ok=True, regions=[ + RegionMark("月报", "col_header", "C5:F5", "列表1"), RegionMark("月报", "value", "C6:F900", "列表1")]) + rc = replay_compare(_edit("list:列表1", {"header": "C5:F5", "data": "C6:F900", "total": "C901:F901"}), ext, + window_rows=500) + # 部分干跑：期望和实际都裁到前 500 行再比，窗口外的合计行不算缺 + assert rc.match and rc.window_rows == 500 and rc.expected["data"] == "C6:F500" + rc2 = replay_compare(_edit("list:列表1", {"header": "C5:F5", "data": "C6:F900", "total": "C901:F901"}), ext) + assert not rc2.match and "重放时没有认出合计行（框里是 C901:F901）" in rc2.diffs + + +def test_replay_compare_partial_kinds_check_coverage(): + ext = Extraction(ok=True, regions=[RegionMark(SHEET, "ignored", "B32:D32", "交叉表"), + RegionMark(SHEET, "row_label", "B5:B30", "交叉表")]) + ok = replay_compare(_edit("ignore_rows:交叉表", {"ignored": "B32"}, "交叉表"), ext) + assert ok.match and ok.actual == {"ignored": "B32:D32"} + bad = replay_compare(_edit("ignore_rows:交叉表", {"ignored": "B33"}, "交叉表"), ext) + assert not bad.match and bad.diffs + + +def test_edit_result_keys_do_not_leak_coordinates_into_recipe(c01): + """坐标不进配方：换算出的补丁里没有任何 A1 写法（表头上方的标题、锚点都是文字）。""" + import re + b, work, ext = c01 + res = selection_edit(work, b.grid, sel("C5:F8", "list", "月报"), extraction=ext) + blob = json.dumps(res.ops, ensure_ascii=False) + assert not re.search(r"(? dict[str, Any]: @@ -255,8 +262,9 @@ def test_reference_layout_cards_and_questions(flow): assert titles[6] == "第 28–30 行是合计，改作核对并另存" assert titles[7] == "各时段之和与全日客流 31 天中 0 天相等" assert titles[8] == "B3 是区域外文字" - assert titles[9] == "导入模式：每期替换" - assert "按期累积将在后续版本提供" in d.cards[9].reason + assert titles[9] == "导入模式：按期累积（建议）" + assert d.cards[9].question == "q_mode" + assert "每期以统计期为键" in d.cards[9].reason and "此前各期留在历史版本中" in d.cards[9].reason assert d.cards[1].cells == ["客流汇总!B2"] assert d.cards[8].cells == ["客流汇总!B3"] by_id = {c.id: c for c in d.cards} @@ -264,7 +272,8 @@ def test_reference_layout_cards_and_questions(flow): assert by_id["fact:F2"].question == "q_relation:F2" qs = {q.id: q for q in d.questions} - assert [q.id for q in d.questions] == ["q_relation:F1", "q_placeholder:·", "q_relation:F2"] + # q_mode 固定排在最后（期 2 的测试和性能用例取 questions[0]） + assert [q.id for q in d.questions] == ["q_relation:F1", "q_placeholder:·", "q_relation:F2", "q_mode"] assert all(q.default is None for q in d.questions) for qid in ("q_relation:F1", "q_relation:F2"): q = qs[qid] @@ -1112,3 +1121,146 @@ def test_long_list_with_many_blank_numbers_stays_one_block_and_fast(): assert time.perf_counter() - t0 < 1.0 assert d.complete, d.failures assert len(d.recipe["sheets"][0]["blocks"]) == 1 + + +# ========================================================================== +# 期 3（WP-2）：导入模式的默认值、q_mode、卡片（P3-SPEC 2.2） +# ========================================================================== + + +def _sales_list() -> Workbook: + wb = Workbook() + ws = wb.active + ws.title = "月报" + ws.append(["地区", "销量"]) + for row in (["分区甲", 10], ["分区乙", 20]): + ws.append(row) + return wb + + +def test_list_draft_stays_replace_without_q_mode(): + """列表形态的表不符合按期累积的资格：草稿不写 mode（每期替换），不问 q_mode，卡片写第一条原因。""" + sc, grids = build(_sales_list()) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert d.complete, d.failures + assert d.recipe["mode"] == "replace" + assert all(q.id != "q_mode" for q in d.questions) + card = next(c for c in d.cards if c.id == "mode") + assert card.title == "导入模式：每期替换" and card.question is None + assert "配方没有配置统计期" in card.reason and "此前各期留在历史版本中" in card.reason + + +def test_q_mode_effects_and_position(flow): + sc, grids, _ = flow + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + q = d.questions[-1] + assert q.id == "q_mode" and q.text == "这份报表每期怎么更新" and q.default is None + assert [(o["value"], o["label"]) for o in q.options] == [("accumulate", "按期累积（建议）"), ("replace", "每期替换")] + assert q.effects == {"accumulate": [{"op": "add", "path": "/mode", "value": "accumulate"}], + "replace": [{"op": "add", "path": "/mode", "value": "replace"}]} + # 在 canonical 形式（去掉了 mode）上也能应用：add 到不存在的键等于新增 + canon_form = canonical(d.recipe) + assert apply_ops(canon_form, q.effects["replace"])["mode"] == "replace" + # 改成每期替换以后，问题照出（资格看配方结构，不看当前取值），卡片照旧建议按期累积 + replaced = apply_ops(d.recipe, q.effects["replace"]) + cards, qs = S.questions_for(replaced, d.facts, grids) + assert qs[-1].id == "q_mode" and next(c for c in cards if c.id == "mode").title == "导入模式：按期累积（建议）" + + +def test_ineligible_crosstab_has_no_q_mode(flow): + sc, grids, _ = flow + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + relaxed = copy.deepcopy(d.recipe) + relaxed["sheets"][0]["blocks"][0]["axis"]["checks"] = ["contiguous"] + cards, qs = S.questions_for(relaxed, d.facts, grids) + assert all(q.id != "q_mode" for q in qs) + mode = next(c for c in cards if c.id == "mode") + assert mode.title == "导入模式：每期替换" and "不能按期累积" in mode.reason + + +# ========================================================================== +# 期 3（WP-2）：按规则重新起草，并把名字对齐到现行配方（P3-SPEC 6.3） +# ========================================================================== + + +def test_align_to_contract_gives_the_reference_recipe(flow): + sc, grids, _ = flow + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert canonical(d.recipe) != canonical(FLOW) + aligned, report = S.align_to_contract(d.recipe, FLOW) + assert report == [] + assert canonical(aligned) == canonical(FLOW) + assert d.recipe["tables"][0]["name"] == "客流汇总_按日", "入参不变" + + +def test_align_to_contract_copies_renamed_columns_ids_and_relations(flow): + sc, grids, _ = flow + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + cur = json.loads(json.dumps(FLOW, ensure_ascii=False).replace('"时段客流"', '"分时客流"') + .replace('"客流"', '"人数"').replace('"分区甲"', '"甲区"')) + segs = cur["sheets"][0]["blocks"][0]["segments"] + segs[1]["id"] = "白天" + segs[2]["const"]["时段类别"] = {"pick": "夜"} # 现行的 pick 仍是新标题的子串：沿用 + cur["relations"][0]["id"], cur["relations"][1]["id"] = "R7", "R3" + Recipe.model_validate(cur) + aligned, report = S.align_to_contract(d.recipe, cur) + assert report == [] + assert canonical(aligned) == canonical(cur) + + +def test_align_to_contract_reports_what_did_not_match(flow): + sc, grids, _ = flow + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + cur = copy.deepcopy(FLOW) + seg = cur["sheets"][0]["blocks"][0]["segments"][0] + seg["labels"]["expect"] = ["甲", "乙", "丙"] + seg["measures"] = {"甲": "甲", "乙": "乙", "丙": "丙"} + cur["tables"][0]["units"] = {} + cur["relations"] = [cur["relations"][1]] + aligned, report = S.align_to_contract(d.recipe, cur) + assert any("表「客流汇总_按日」没有在现行配方里找到对应的表" in r for r in report) + assert any(r.startswith("关系「R1」") for r in report) + assert aligned["tables"][0]["name"] == "客流汇总_按日" + # 宽表没配上，两条关系的成员都对不上现行的关系：保留起草器的编号，各记一句 + assert [r["id"] for r in aligned["relations"]] == ["R1", "R2"] + assert any(r.startswith("关系「R2」") for r in report) + Recipe.model_validate(aligned) + + +def _two_tables(names: tuple[str, str], labels: tuple[list[str], list[str]]) -> dict: + """两个工作表各一张宽表的配方（只用来比较表的来源签名）。""" + sheets, tables = [], [] + for i, (name, labs) in enumerate(zip(names, labels)): + sheets.append({"id": f"s{i + 1}", "match": {"name": f"表{i + 1}"}, "context": [{}], "blocks": [{ + "id": f"块{i + 1}", "layout": "crosstab", "segments": [{ + "id": name, "role": "measures", "table": name, "locate": {"by": "labels"}, + "labels": {"expect": labs}, "measures": {x: f"列{j}" for j, x in enumerate(labs)}}]}]}) + tables.append({"name": name, "grain": ["日期"]}) + return {"recipe_format": "agentlab-recipe/2", "sheets": sheets, "tables": tables} + + +def test_align_to_contract_tie_is_not_paired(): + """同一张起草的表有两个并列最大的候选：不配对，记进 report（评审一-m13）。""" + cand = _two_tables(("起草甲", "起草乙"), (["a", "b"], ["x", "y"])) + cand["sheets"] = cand["sheets"][:1] + cand["tables"] = cand["tables"][:1] + cur = _two_tables(("现行一", "现行二"), (["a", "b"], ["a", "b"])) + aligned, report = S.align_to_contract(cand, cur) + assert aligned["tables"][0]["name"] == "起草甲" + assert any("同样相似" in r for r in report) + + +def test_align_to_contract_two_candidates_for_one_current_table_are_not_paired(): + cand = _two_tables(("起草甲", "起草乙"), (["a", "b"], ["a", "b"])) + cur = _two_tables(("现行一", "现行二"), (["a", "b"], ["x", "y"])) + aligned, report = S.align_to_contract(cand, cur) + assert [t["name"] for t in aligned["tables"]] == ["起草甲", "起草乙"] + assert any("都同样像现行的表「现行一」" in r for r in report) + + +def test_align_to_contract_is_one_to_one(): + cand = _two_tables(("起草甲", "起草乙"), (["a", "b", "c"], ["a", "b"])) + cur = _two_tables(("现行一", "现行二"), (["a", "b", "c"], ["a", "b", "d"])) + aligned, _ = S.align_to_contract(cand, cur) + names = [t["name"] for t in aligned["tables"]] + assert names[0] == "现行一" and names[1] != "现行一" and len(set(names)) == 2 diff --git a/backend/tests/test_recipe_validate.py b/backend/tests/test_recipe_validate.py index 83494a39..0d2f945e 100644 --- a/backend/tests/test_recipe_validate.py +++ b/backend/tests/test_recipe_validate.py @@ -297,10 +297,28 @@ def test_year_in_list_header_is_not_an_error(): # ========================================================================== -def test_accumulate_mode_unsupported(): +def test_accumulate_mode_follows_eligibility(): + """期 3 改写（P3-SPEC 2.2）：按期累积不再一律报 mode_unsupported，改按资格判（契约 accumulate_blockers）。""" r = flow() r["mode"] = "accumulate" - only(r, "mode_unsupported", "/mode") + assert run(r) == [] # 参考配方符合资格：交叉表、有统计期、covers_context、grain 含日期 + # 去掉统计期：accumulate_needs_period（同时断言 covers_context、补年份都要求统计期，照期 2 报） + r2 = flow() + r2["mode"] = "accumulate" + r2["sheets"][0]["context"] = [] + codes = [p.code for p in run(r2)] + assert "accumulate_needs_period" in codes and "mode_unsupported" not in codes + # 列表写入的表：accumulate_unkeyed，path 指向那张表 + lr = list_recipe() + lr["mode"] = "accumulate" + lr["sheets"][0]["context"] = [{}] + probs = run(lr, facts=None) + assert [(p.code, p.path) for p in probs] == [("accumulate_unkeyed", "/tables/0"), + ("accumulate_unkeyed", "/tables/1")] + assert "列表形态的表目前只支持每期替换" in probs[0].message + # 同一份配方每期替换时不报 + lr["mode"] = "replace" + assert run(lr, facts=None) == [] # ========================================================================== diff --git a/backend/tests/test_recipe_validate_p3.py b/backend/tests/test_recipe_validate_p3.py new file mode 100644 index 00000000..a2e2f775 --- /dev/null +++ b/backend/tests/test_recipe_validate_p3.py @@ -0,0 +1,357 @@ +"""配方静态校验的期 3 改动（WP-2，P3-SPEC 2.2、3.3、9.2）：按期累积的资格、沿用已确认的常量（L3）、认领只查改动过的 +部分、period_literal 的扩展范围、忽略规则的 ignore_conflict / ignore_duplicate。 + +只用契约里的参考配方（flow_recipe.json）和手写的小配方，不读 Excel。每条规则一正一反。 +""" +from __future__ import annotations + +import copy +import json +from pathlib import Path + +import pytest + +from app.data.recipe import canonical_recipe, validate_recipe +from app.data.recipe_types import DraftFacts, Fact, Recipe + +FLOW = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text(encoding="utf-8")) +SEG = "/sheets/0/blocks/0/segments" +F1 = Fact("F1", "sum_eq", "客流汇总", "第 5 行 = 第 6 行 + 第 7 行（30 列中 30 列成立）", + {"segment": "日客流", "total": "全日客流（人次）", "parts": ["分区甲（人次）", "分区乙（人次）"], "rows": [5, 6, 7]}) +F2 = Fact("F2", "not_equal_sum", "客流汇总", "日间、夜间各行之和 与 第 5 行（30 列中 0 列相等）", + {"a": ["日间", "夜间"], "b_segment": "日客流", "b": "全日客流（人次）", "equal": 0, "checked": 30}) +#: D13 加了分区丙之后本期的系统发现：全日 = 甲 + 乙 + 丙 +F1_C = Fact("F1", "sum_eq", "客流汇总", "第 5 行 = 第 6 行 + 第 7 行 + 第 8 行（30 列中 30 列成立）", + {"segment": "日客流", "total": "全日客流（人次）", + "parts": ["分区甲（人次）", "分区乙（人次）", "分区丙（人次）"], "rows": [5, 6, 7, 8]}) + + +def flow() -> dict: + return copy.deepcopy(FLOW) + + +def segs(r: dict) -> list[dict]: + return r["sheets"][0]["blocks"][0]["segments"] + + +def codes(data, **kw) -> list[tuple[str, str]]: + _r, problems = validate_recipe(data, **kw) + return [(p.code, p.path) for p in problems] + + +def list_recipe() -> dict: + return { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": "月报"}, "blocks": [{ + "id": "列表1", "layout": "list", "table": "销售", + "columns": [{"header": "地区", "name": "地区", "type": "TEXT"}, + {"header": "销量", "name": "销量", "type": "INTEGER"}]}]}], + "tables": [{"name": "销售", "grain": ["地区"]}], + } + + +# ========================================================================== +# 2.2 按期累积的资格（mode_unsupported 不再产生） +# ========================================================================== + + +def test_accumulate_eligible_reference_recipe_passes(): + r = flow() + r["mode"] = "accumulate" + assert codes(r, facts=DraftFacts(facts=[F1, F2])) == [] + + +def test_accumulate_without_covers_context_reports_unkeyed_on_every_written_table(): + r = flow() + r["mode"] = "accumulate" + r["sheets"][0]["blocks"][0]["axis"]["checks"] = ["contiguous"] + got = codes(r) + assert ("accumulate_unkeyed", "/tables/0") in got and ("accumulate_unkeyed", "/tables/2") in got + assert all(c != "mode_unsupported" for c, _ in got) + # 每期替换时同一份配方只报 checks_relaxed 那一类（确认项），静态校验不报资格 + r["mode"] = "replace" + assert codes(r) == [] + + +def test_accumulate_list_table_message_does_not_promise_future_versions(): + r = list_recipe() + r["mode"] = "accumulate" + r["sheets"][0]["context"] = [{}] + _rec, problems = validate_recipe(r) + assert [p.code for p in problems] == ["accumulate_unkeyed"] + assert "列表形态的表目前只支持每期替换" in problems[0].message and "后续版本" not in problems[0].message + + +# ========================================================================== +# 3.3 沿用已确认的常量（L3） +# ========================================================================== + + +def d09(pick: str = "日间") -> dict: + """D09：日间的分段标题改成「日间分时段客流（人次）」，常量 pick 照旧或改掉。""" + r = flow() + segs(r)[1]["locate"]["title"] = "日间分时段客流（人次）" + segs(r)[1]["const"]["时段类别"]["pick"] = pick + return r + + +def test_const_rule_a_candidate_word_still_passes(): + # (a) 期 2 原规则：pick 是新标题的候选词（「日间分」） + assert codes(d09("日间分"), origin="manual") == [] + + +def test_const_rule_a_negative_without_base(): + # 没有 base、不是重放：沿用「日间」不在候选词里，照期 2 报 const_not_candidate + assert codes(d09(), origin="manual") == [("const_not_candidate", f"{SEG}/1/const/时段类别/pick")] + + +def test_const_rule_b_carried_from_base_passes(): + # (b) 沿用：现行配方里同一分段同一常量列的 pick 逐字相同，且仍是新标题的子串 + assert codes(d09(), origin="manual", base=flow()) == [] + # base 给 canonical 形式也一样 + assert codes(d09(), origin="manual", base=canonical_recipe(Recipe.model_validate(flow()))) == [] + + +def test_const_rule_b_negative_when_base_pick_differs(): + base = flow() + segs(base)[1]["const"]["时段类别"]["pick"] = "白天" + segs(base)[1]["locate"]["title"] = "白天时段客流（人次）" + assert codes(d09(), origin="manual", base=base) == [("const_not_candidate", f"{SEG}/1/const/时段类别/pick")] + + +def test_const_rule_b_negative_when_pick_not_in_new_title(): + r = flow() + segs(r)[1]["locate"]["title"] = "白天时段客流（人次）" + # 现行配方的 pick「日间」逐字相同，但已不是新标题的子串：不放行 + assert codes(r, origin="manual", base=flow()) == [("const_not_candidate", f"{SEG}/1/const/时段类别/pick")] + + +def test_const_rule_b_negative_when_base_segment_id_differs(): + base = flow() + segs(base)[1]["id"] = "白天段" + segs(base)[3]["locate"]["segment"] = "夜间" + assert codes(d09(), origin="manual", base=base) == [("const_not_candidate", f"{SEG}/1/const/时段类别/pick")] + + +def test_const_rule_c_replay_substring_passes_and_negative(): + # (c) 重放：修复后的配方下个月按 replay 校验、又没有 base + assert codes(d09(), origin="replay") == [] + r = flow() + segs(r)[1]["locate"]["title"] = "白天时段客流（人次）" + assert codes(r, origin="replay") == [("const_not_candidate", f"{SEG}/1/const/时段类别/pick")] + + +@pytest.mark.parametrize("kw", [{"origin": "replay"}, {"origin": "manual", "base": FLOW}]) +def test_const_with_digits_still_period_literal(kw): + r = flow() + segs(r)[1]["locate"]["title"] = "日间9时段客流（人次）" + segs(r)[1]["const"]["时段类别"]["pick"] = "日间9" + base = copy.deepcopy(r) if "base" in kw else None + kw = {**kw, "base": base} if base is not None else kw + assert codes(r, **kw) == [("period_literal", f"{SEG}/1/const/时段类别/pick")] + + +# ========================================================================== +# 3.3 第 11 条：认领只查改动过的部分 +# ========================================================================== + + +def test_unchanged_measures_do_not_check_claims_with_base(): + """D04 的修复改的是 dimension 段：本期 R1 有一天不成立、系统发现里没有 F1 时，R1 的认领不查。""" + r = flow() + segs(r)[1]["labels"]["expect"] = segs(r)[1]["labels"]["expect"][1:] # 去掉 7-8 + # 本期只发现了口径不同那一条，编号还错开成了 F1 + f_ne = Fact("F1", "not_equal_sum", "客流汇总", F2.text, dict(F2.detail)) + facts = DraftFacts(facts=[f_ne]) + assert codes(r, facts=facts, origin="manual", base=flow()) == [] + # 没有 base（首次导入）照期 2 全查 + got = codes(r, facts=facts, origin="manual") + assert ("fact_claim_mismatch", "/relations/0/claims") in got + + +def test_changed_measures_still_check_claims_with_base(): + """D13 给日客流加了分区丙：R1 的成员落在改动过的段上，照查，对不上本期的新事实。""" + r = flow() + seg = segs(r)[0] + seg["labels"]["expect"].append("分区丙（人次）") + seg["measures"]["分区丙（人次）"] = "分区丙" + r["tables"][0]["units"]["分区丙"] = "人次" + got = codes(r, facts=DraftFacts(facts=[F1_C, F2]), origin="manual", base=flow()) + assert got == [("fact_claim_mismatch", "/relations/0/claims")] + # 按事实更新成员之后通过 + r["relations"][0]["parts"] = ["分区甲", "分区乙", "分区丙"] + assert codes(r, facts=DraftFacts(facts=[F1_C, F2]), origin="manual", base=flow()) == [] + + +def test_measures_locate_change_alone_still_checks_claims_with_base(): + """measures 段只改了定位方式（expect、measures 都没变，改按分段标题定位）：仍算改动过的段（3.3 第 11 条的 + 「locate 任一不同」），这一段上的关系认领照查。本期的发现里没有 R1 认领的那条 sum_eq（编号错开成口径不同那条）。""" + r = flow() + assert segs(r)[0]["locate"]["by"] == "labels" + segs(r)[0]["locate"] = {"by": "section_title", "title": "日客流汇总", "segment": None} + f_ne = Fact("F1", "not_equal_sum", "客流汇总", F2.text, dict(F2.detail)) + facts = DraftFacts(facts=[f_ne]) + got = codes(r, facts=facts, origin="manual", base=flow()) + assert ("fact_claim_mismatch", "/relations/0/claims") in got + assert ("fact_claim_mismatch", "/relations/1/claims") in got + # 对照：定位方式没变时这两条关系按重放处理，不查认领 + assert codes(flow(), facts=facts, origin="manual", base=flow()) == [] + + +def test_fact_with_a_removed_label_is_located_through_the_base_recipe(): + """去掉了「分区乙」（① remove_label）：本期的事实 F1 还写着分区乙，分段 id 又是规则起草的写法，在工作配方里 + 按 id、按全部标签都认不出。按现行配方认出它落在「日客流」上、这一段改动过，F1 在改动范围内：没人认领时报 + fact_unclaimed，不让它连同被删掉的关系一起不声不响地消失。""" + r = flow() + seg = segs(r)[0] + seg["labels"]["expect"].remove("分区乙（人次）") + del seg["measures"]["分区乙（人次）"] + del r["tables"][0]["units"]["分区乙"] + del r["relations"][0] + drafted = Fact("F1", "sum_eq", "客流汇总", F1.text, {**F1.detail, "segment": "客流汇总_按日"}) + f_ne = Fact("F2", "not_equal_sum", "客流汇总", F2.text, dict(F2.detail)) + got = codes(r, facts=DraftFacts(facts=[drafted, f_ne]), origin="manual", base=flow()) + assert got == [("fact_unclaimed", "/relations")] + # 分区甲、分区乙都换成了分区丙：工作配方里只剩「全日客流」一个标签对得上，过半命中也认不出，只能靠现行配方 + seg["labels"]["expect"] = ["全日客流（人次）", "分区丙（人次）"] + seg["measures"] = {"全日客流（人次）": "全日客流", "分区丙（人次）": "分区丙"} + r["tables"][0]["units"] = {"全日客流": "人次", "分区丙": "人次"} + got = codes(r, facts=DraftFacts(facts=[drafted, f_ne]), origin="manual", base=flow()) + assert got == [("fact_unclaimed", "/relations")] + + +def test_fact_located_by_most_labels_when_neither_recipe_has_all_of_them(): + """同时加了「分区丙」、去掉了「分区乙」：事实「全日 = 甲 + 乙 + 丙」在工作配方和现行配方里都按全部标签认不出， + 按「过半标签命中」认出落在改动过的「日客流」上，没人认领时报 fact_unclaimed；只命中一个标签时不认， + 按重放处理（不在改动范围内，不报）。""" + r = flow() + seg = segs(r)[0] + seg["labels"]["expect"] = ["全日客流（人次）", "分区甲（人次）", "分区丙（人次）"] + seg["measures"] = {"全日客流（人次）": "全日客流", "分区甲（人次）": "分区甲", "分区丙（人次）": "分区丙"} + r["tables"][0]["units"] = {"全日客流": "人次", "分区甲": "人次", "分区丙": "人次"} + del r["relations"][0] + f_ne = Fact("F2", "not_equal_sum", "客流汇总", F2.text, dict(F2.detail)) + mixed = Fact("F7", "sum_eq", "客流汇总", "…", {"segment": "客流汇总_按日", "total": "全日客流（人次）", + "parts": ["分区甲（人次）", "分区乙（人次）", "分区丙（人次）"]}) + assert codes(r, facts=DraftFacts(facts=[mixed, f_ne]), origin="manual", base=flow()) == [ + ("fact_unclaimed", "/relations")] + stray = Fact("F7", "sum_eq", "客流汇总", "…", {"segment": "别处", "total": "全日客流（人次）", + "parts": ["别处甲", "别处乙"]}) + assert codes(r, facts=DraftFacts(facts=[stray, f_ne]), origin="manual", base=flow()) == [] + + +def test_changed_relation_is_checked_even_if_segments_unchanged(): + r = flow() + r["relations"][0]["claims"] = "F9" + got = codes(r, facts=DraftFacts(facts=[F1, F2]), origin="manual", base=flow()) + assert ("fact_claim_mismatch", "/relations/0/claims") in got + + +def test_fact_on_changed_segment_must_be_claimed(): + """新发现落在改动过的 measures 段上、又没有人认领：fact_unclaimed 照报。""" + r = flow() + segs(r)[0]["labels"]["expect"].append("分区丙（人次）") + segs(r)[0]["measures"]["分区丙（人次）"] = "分区丙" + r["tables"][0]["units"]["分区丙"] = "人次" + extra = Fact("F3", "sum_eq", "客流汇总", "第 8 行 = 第 6 行 + 第 7 行（30 列中 30 列成立）", + {"segment": "日客流", "total": "分区丙（人次）", "parts": ["分区甲（人次）", "分区乙（人次）"], + "rows": [8, 6, 7]}) + got = codes(r, facts=DraftFacts(facts=[F1, F2, extra]), origin="manual", base=flow()) + assert ("fact_unclaimed", "/relations") in got + # 落在没改过的段上的新发现：不查 + got2 = codes(flow(), facts=DraftFacts(facts=[F1, F2, extra]), origin="manual", base=flow()) + assert got2 == [] + + +def test_invalid_base_is_ignored(): + assert codes(d09(), origin="manual", base={"recipe_format": "x"}) == [ + ("const_not_candidate", f"{SEG}/1/const/时段类别/pick")] + + +# ========================================================================== +# 9.2 period_literal 的扩展：after_title 和忽略规则的锚点（origin 不是 replay 时） +# ========================================================================== + + +def test_after_title_with_digits_is_period_literal_except_replay(): + r = list_recipe() + r["sheets"][0]["blocks"][0]["after_title"] = "2026年8月 销售月报" + assert codes(r) == [("period_literal", "/sheets/0/blocks/0/after_title")] + assert codes(r, origin="replay") == [] + r["sheets"][0]["blocks"][0]["after_title"] = "销售月报" + assert codes(r) == [] + + +@pytest.mark.parametrize("where,item,path", [ + ("ignore_rows", {"label": "注：9月补录", "reason": "补录不导入"}, "/sheets/0/blocks/0/ignore_rows/0/label"), + ("ignore_columns", {"header": "2026合计", "reason": "合计列"}, "/sheets/0/blocks/0/ignore_columns/0/header"), + ("ignore_outside", {"anchor": "9月补录说明", "reason": "说明行"}, "/sheets/0/ignore_outside/0/anchor"), +]) +def test_ignore_anchor_with_digits_is_period_literal(where, item, path): + r = flow() + target = r["sheets"][0] if where == "ignore_outside" else r["sheets"][0]["blocks"][0] + target[where] = [item] + assert codes(r) == [("period_literal", path)] + assert codes(r, origin="replay") == [] + item = {k: (v.replace("9", "").replace("2026", "") if k != "reason" else v) for k, v in item.items()} + target[where] = [item] + assert codes(r) == [] + + +# ========================================================================== +# 9.2 ignore_conflict、ignore_duplicate +# ========================================================================== + + +def test_ignore_row_that_a_segment_expects_conflicts(): + r = flow() + r["sheets"][0]["blocks"][0]["ignore_rows"] = [{"label": "分区甲（人次）", "reason": "不要"}] + assert codes(r) == [("ignore_conflict", "/sheets/0/blocks/0/ignore_rows/0/label")] + # 时段按解析后的规范写法比：全角写法也算同一个标签 + r["sheets"][0]["blocks"][0]["ignore_rows"] = [{"label": "８－９", "reason": "不要"}] + assert ("ignore_conflict", "/sheets/0/blocks/0/ignore_rows/0/label") in codes(r) + r["sheets"][0]["blocks"][0]["ignore_rows"] = [{"label": "补录（人次）", "reason": "补录不导入"}] + assert codes(r) == [] + + +def test_crosstab_ignore_column_that_is_a_date_conflicts(): + r = flow() + r["sheets"][0]["blocks"][0]["ignore_columns"] = [{"header": "九月合计", "reason": "合计列"}] + assert codes(r) == [] + r["sheets"][0]["blocks"][0]["ignore_columns"] = [{"header": "九月合计", "reason": "合计列"}, + {"header": "九月一日", "reason": "x"}] + assert codes(r) == [] + r["sheets"][0]["blocks"][0]["ignore_columns"] = [{"header": "九月合计", "reason": "合计列"}, + {"header": "9月1日", "reason": "x"}] + got = codes(r, origin="replay") + assert got == [("ignore_conflict", "/sheets/0/blocks/0/ignore_columns/1/header")] + + +def test_list_ignore_column_same_as_imported_header_conflicts(): + r = list_recipe() + r["sheets"][0]["blocks"][0]["ignore_columns"] = [{"header": "销 量", "reason": "不要"}] + assert codes(r) == [("ignore_conflict", "/sheets/0/blocks/0/ignore_columns/0/header")] + r["sheets"][0]["blocks"][0]["ignore_columns"] = [{"header": "备注", "reason": "不要"}] + assert codes(r) == [] + + +@pytest.mark.parametrize("where,key,a,b", [ + ("ignore_rows", "label", "补录（人次）", "补录(人次)"), + ("ignore_columns", "header", "合计", " 合 计 "), + ("ignore_outside", "anchor", "补录说明", "补录说明"), +]) +def test_ignore_duplicate(where, key, a, b): + r = flow() + target = r["sheets"][0] if where == "ignore_outside" else r["sheets"][0]["blocks"][0] + target[where] = [{key: a, "reason": "一"}, {key: b, "reason": "二"}] + prefix = "/sheets/0" if where == "ignore_outside" else "/sheets/0/blocks/0" + assert codes(r) == [("ignore_duplicate", f"{prefix}/{where}/1/{key}")] + + +def test_new_problem_messages_do_not_expose_field_names(): + r = flow() + r["sheets"][0]["blocks"][0]["ignore_rows"] = [{"label": "分区甲（人次）", "reason": "x"}, + {"label": "分区甲(人次)", "reason": "y"}] + _rec, problems = validate_recipe(r) + for p in problems: + assert not any(k in p.message for k in ("ignore_rows", "label", "reason")), p.message diff --git a/backend/tests/test_run_data_versions.py b/backend/tests/test_run_data_versions.py index d28d2cdd..28a06c8c 100644 --- a/backend/tests/test_run_data_versions.py +++ b/backend/tests/test_run_data_versions.py @@ -435,7 +435,9 @@ async def test_tools_follow_the_pinned_snapshot_and_record_data_version(): await engines.invalidate(sid) -async def test_pinned_snapshot_collected_by_gc_gives_a_clear_error_and_never_reads_current(): +async def test_pinned_snapshot_collected_by_gc_gives_a_clear_error_and_never_reads_current(monkeypatch): + # 期 3 起回收另外保留最近 SNAPSHOT_KEEP 个快照；调成 0（期 1 的规则），没人引用的 S1 才会在重传后被回收 + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0) src = uniq() sid, s1 = await publish(src, V1) # 没有运行引用 S1：重传后的回收把它标成已回收、删掉文件 diff --git a/backend/tests/test_source_versions_api.py b/backend/tests/test_source_versions_api.py new file mode 100644 index 00000000..acefc900 --- /dev/null +++ b/backend/tests/test_source_versions_api.py @@ -0,0 +1,722 @@ +"""版本页接口（期 3，WP-5；P3-SPEC 第 7 节）：版本列表、启用旧版本、清单、移除一期、作废接受、导入记录的新字段。 +真管线、经 HTTP；只有接口不提供的观察点（快照记录、导入记录的状态、snapshot_activations）才直接读库。 + +夹具一律合成、假名（客流表：分区甲、分区乙……），数字是随机数；不调用任何模型。 +""" +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +from sqlalchemy import select, update + +from app.core import artifact_store +from app.db.base import SessionLocal, utcnow +from app.db.models import DataSource, SnapshotActivation, SourceSnapshot, TableImport +from tests.fixtures.xlsx.drift import AUG, DRIFT_CASES, SEP +from tests.fixtures.xlsx.flow import flow_workbook +from tests.test_table_imports_api_p3 import ( # noqa: F401 - client、store 是夹具 + API, client, commit, confirms, count, db_get, first_import, reupload, store, trial, unique, +) + + +async def snapshots(client, source_id: str) -> list[dict[str, Any]]: + resp = await client.get(f"{API}/{source_id}/snapshots") + assert resp.status_code == 200, resp.text + return resp.json() + + +async def imports(client, source_id: str) -> list[dict[str, Any]]: + resp = await client.get(f"{API}/{source_id}/imports") + assert resp.status_code == 200, resp.text + return resp.json() + + +async def two_periods(client, *, seed: int) -> dict[str, Any]: + """8 月（按期累积）+ 9 月：S1=[8 月]、S2=[8 月, 9 月]，同一个配方。""" + name = unique() + base = await first_import(client, name, seed=seed) + raw, fn = flow_workbook(SEP, 30, seed=seed + 1) + st = await reupload(client, base["source_id"], raw, fn) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + return {"name": name, "source_id": base["source_id"], "s1": base["commit"]["snapshot_id"], + "s2": resp.json()["snapshot_id"], "aug": base["commit"]["import_id"], "sep": resp.json()["import_id"]} + + +async def activations(source_id: str) -> list[SnapshotActivation]: + async with SessionLocal() as session: + return list((await session.execute( + select(SnapshotActivation).where(SnapshotActivation.source_id == source_id) + .order_by(SnapshotActivation.created_at))).scalars()) + + +# ========================================================================== +# 列表（7.1） +# ========================================================================== + + +async def test_snapshot_list_puts_current_first_with_parts_rows_and_periods_diff(client): + v = await two_periods(client, seed=401) + items = await snapshots(client, v["source_id"]) + assert [s["id"] for s in items[:2]] == [v["s2"], v["s1"]] + cur, old = items[0], items[1] + assert cur["current"] is True and cur["activatable"] is False and cur["reason_code"] == "current" + assert cur["reason"] == "已是当前版本" and cur["mode"] == "accumulate" + assert [p["period_start"] for p in cur["parts"]] == ["2026-08-01", "2026-09-01"] + assert cur["parts"][1]["rows"] == {"日客流": 30, "时段客流": 510, "时段客流_表内合计": 90} + assert cur["tables"] == {"日客流": 61, "时段客流": 1037, "时段客流_表内合计": 183} + assert cur["recipe"]["seq"] == 1 and cur["available"] is True and cur["db_size"] > 0 + assert len(cur["db_sha256_prefix"]) == 8 and cur["pinned_runs"] == 0 and cur["mask_lost"] == [] + assert old["activatable"] is True and old["reason_code"] is None and old["reason"] is None + assert old["periods_diff"] == {"added": [], "removed": [ + {"start": "2026-09-01", "end": "2026-09-30", "file_name": cur["parts"][1]["file_name"]}]} + assert old["tables"] == {"日客流": 31, "时段客流": 527, "时段客流_表内合计": 93} + # 手工源返回空列表；不存在的源 404 + async with SessionLocal() as session: + manual = DataSource(name=unique("manual"), kind="sqlite", database=":memory:") + session.add(manual) + await session.commit() + manual_id = manual.id + assert await snapshots(client, manual_id) == [] + resp = await client.get(f"{API}/nope/snapshots") + assert resp.status_code == 404 and resp.json()["code"] == "source_not_found" + + +# ========================================================================== +# 启用旧版本（7.2） +# ========================================================================== + + +async def test_activate_rolls_back_and_guards_confirm_expected_current_and_mask(client): + v = await two_periods(client, seed=403) + sid, url = v["source_id"], f"{API}/{v['source_id']}/snapshots/{v['s1']}/activate" + resp = await client.post(url, json={"expected_current_snapshot_id": v["s2"]}) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" + resp = await client.post(url, json={"confirm": True}) + assert resp.status_code == 422 and resp.json()["code"] == "expected_current_required" + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s1"]}) + assert resp.status_code == 409 and resp.json()["code"] == "base_changed" + resp = await client.post(f"{API}/{sid}/snapshots/{'0' * 64}/activate", + json={"confirm": True, "expected_current_snapshot_id": v["s2"]}) + assert resp.status_code == 404 and resp.json()["code"] == "snapshot_not_found" + resp = await client.post(f"{API}/{sid}/snapshots/{v['s2']}/activate", + json={"confirm": True, "expected_current_snapshot_id": v["s2"]}) + assert resp.status_code == 409 and resp.json()["code"] == "already_current" + # 遮罩列在目标版本里没有同名列：不带确认 409 mask_lost，带上同样的列表才成 + async with SessionLocal() as session: + await session.execute(update(DataSource).where(DataSource.id == sid) + .values(options={"mask_columns": "分区丁"})) + await session.commit() + items = await snapshots(client, sid) + assert next(s for s in items if s["id"] == v["s1"])["mask_lost"] == ["分区丁"] + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s2"]}) + assert resp.status_code == 409 and resp.json()["code"] == "mask_lost" and "分区丁" in resp.json()["detail"] + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s2"], + "ack_mask_lost": ["分区丁"], "reason": "合成理由：回到八月", + "signed_by": "合成署名乙"}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["snapshot_id"] == v["s1"] and out["previous_snapshot_id"] == v["s2"] + assert out["source"]["current_snapshot"]["id"] == v["s1"] and out["recipe_id"] + assert await count(client, v["name"], "日客流") == 31 + assert (await db_get(TableImport, v["sep"])).status == "superseded" + assert (await db_get(TableImport, v["aug"])).status == "active" + last = (await activations(sid))[-1] + assert (last.kind, last.snapshot_id, last.previous_snapshot_id) == ("activate", v["s1"], v["s2"]) + assert last.reason == "合成理由：回到八月" and last.signed_by == "合成署名乙" + + +# ========================================================================== +# 移除一期（7.4） +# ========================================================================== + + +async def test_remove_period_reuses_the_earlier_snapshot_and_refuses_the_last_one(client): + """8 月、9 月同一个配方时移除 9 月：结果就是原来的 8 月快照（同一组导入、同一个目标配方、同为累积，复用）。""" + v = await two_periods(client, seed=405) + sid = v["source_id"] + url = f"{API}/{sid}/imports/{v['sep']}/remove" + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s2"]}) + assert resp.status_code == 422 and resp.json()["code"] == "reason_required" + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s1"], "reason": "传错"}) + assert resp.status_code == 409 and resp.json()["code"] == "base_changed" + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s2"], + "reason": "合成理由：九月传错了文件"}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["snapshot_id"] == v["s1"] and out["reused"] is True and out["removed_import_id"] == v["sep"] + assert out["source"]["current_snapshot"]["id"] == v["s1"] + assert await count(client, v["name"], "日客流") == 31 + assert (await db_get(TableImport, v["sep"])).status == "superseded" + last = (await activations(sid))[-1] + assert (last.kind, last.import_id) == ("remove_period", v["sep"]) + # 只剩一期：不能移除；不在当前版本里的：not_in_current；不存在的：404 + resp = await client.post(f"{API}/{sid}/imports/{v['aug']}/remove", + json={"confirm": True, "expected_current_snapshot_id": v["s1"], "reason": "x"}) + assert resp.status_code == 409 and resp.json()["code"] == "last_period" + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s1"], "reason": "x"}) + assert resp.status_code == 409 and resp.json()["code"] == "not_in_current" + resp = await client.post(f"{API}/{sid}/imports/nope/remove", + json={"confirm": True, "expected_current_snapshot_id": v["s1"], "reason": "x"}) + assert resp.status_code == 404 and resp.json()["code"] == "import_not_found" + + +async def test_remove_keeps_recipe_and_mode_and_materializes_a_single_period_union(client): + """A17①：替换模式 S1=[8 月]（配方 r1）→ 9 月改为按期累积得 S2=[8 月, 9 月]（目标 r2）→ 移除 9 月：新快照的模式是 + accumulate、配方是 r2，id 不等于 S1；8 月的配方不是目标配方，快照库是物化的单期并集，说明含 single_after_drop。""" + name = unique() + base = await first_import(client, name, seed=407, mode="replace") + sid, s1 = base["source_id"], base["commit"]["snapshot_id"] + raw, fn = flow_workbook(SEP, 30, seed=408) + st = await reupload(client, sid, raw, fn) + assert "q_mode" in {q["id"] for q in st["questions"]} + resp = await client.post(f"{API}/imports/{st['id']}/answers", json={"answers": {"q_mode": {"value": "accumulate"}}}) + assert resp.status_code == 200, resp.text + st = await trial(client, st["id"]) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert t["accumulate"]["action"] == "append" and t["accumulate"]["mode_switch"] == "replace->accumulate" + assert "mode_switch:replace->accumulate" in confirms(st) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + s2, r2 = resp.json()["snapshot_id"], resp.json()["recipe_id"] + sep = resp.json()["import_id"] + resp = await client.post(f"{API}/{sid}/imports/{sep}/remove", + json={"confirm": True, "expected_current_snapshot_id": s2, "reason": "合成理由"}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["reused"] is False and out["snapshot_id"] not in (s1, s2) + snap = await db_get(SourceSnapshot, out["snapshot_id"]) + assert snap.mode == "accumulate" and snap.recipe_id == r2 + assert Path(snap.db_path).parent.name == "snapshots" and snap.manifest_artifact + src = await db_get(DataSource, sid) + assert src.current_recipe_id == r2, "移除一期不把配方退回去" + comment = snap.schema_cache["tables"]["日客流"]["comment"] + assert "当前版本只含最近一期的数据" in comment and "部分期" not in comment + manifest = artifact_store.load(snap.manifest_artifact) + assert manifest["action"] == "remove_period" and manifest["removed"]["import_id"] == sep + assert await count(client, name, "日客流") == 31 + + +async def test_remove_on_a_replace_mode_source_is_not_accumulate(client): + name = unique() + base = await first_import(client, name, seed=409, mode="replace") + resp = await client.post(f"{API}/{base['source_id']}/imports/{base['commit']['import_id']}/remove", + json={"confirm": True, "expected_current_snapshot_id": base["commit"]["snapshot_id"], + "reason": "x"}) + assert resp.status_code == 409 and resp.json()["code"] == "not_accumulate" + + +# ========================================================================== +# 作废接受（7.5）与导入记录的新字段（7.6） +# ========================================================================== + + +async def accepted_september(client, *, seed: int, mode: str) -> dict[str, Any]: + """8 月（mode）+ 9 月 D26（R1 某日不成立、写理由接受）提交。返回各快照、导入 id。""" + name = unique() + base = await first_import(client, name, seed=seed, mode=mode) + raw, fn = DRIFT_CASES["D26"].build(seed) + st = await reupload(client, base["source_id"], raw, fn) + t = st["trial"] + assert t["status"] == "needs_decision", t["problems"] + resp = await commit(client, st, acceptances=[{"check_id": c, "reason": "合成理由：设备故障，人工补录"} + for c in t["acceptable"]]) + assert resp.status_code == 201, resp.text + return {"name": name, "source_id": base["source_id"], "s1": base["commit"]["snapshot_id"], + "s2": resp.json()["snapshot_id"], "aug": base["commit"]["import_id"], "sep": resp.json()["import_id"], + "trial": st} + + +async def test_revoke_in_accumulate_mode_removes_the_period_and_blocks_reactivation(client): + """A9：D26 累积提交 → 说明是「部分期不成立或未能核对」、不含「已逐日核对」→ 作废接受 → remove_period，结果复用 S1， + 配方和模式不变，导入记录 revoked 非空；之后启用含作废导入的版本 409 contains_revoked。""" + v = await accepted_september(client, seed=411, mode="accumulate") + sid = v["source_id"] + snap = await db_get(SourceSnapshot, v["s2"]) + comment = snap.schema_cache["tables"]["日客流"]["comment"] + assert "已逐日核对" not in comment and "部分期的个别" in comment, comment + recs = {r["id"]: r for r in await imports(client, sid)} + sep = recs[v["sep"]] + assert sep["in_current"] is True and sep["current"] is True and sep["recipe_seq"] == 1 + assert sep["acceptances"] and sep["acceptances"][0]["kind"] == "override" + assert sep["revoke_plan"]["action"] == "remove_period" + assert [p["start"] for p in sep["revoke_plan"]["result_parts"]] == ["2026-08-01"] + assert sep["rows"]["日客流"] == 30 and sep["raw_shared_with"] == [] and sep["raw_open_stagings"] == 0 + assert recs[v["aug"]]["revoke_plan"] is None, "没有接受的期不给作废预案" + url = f"{API}/{sid}/imports/{v['sep']}/revoke-acceptance" + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s2"], + "reason": "合成理由：补录的数字有误"}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["action"] == "remove_period" and out["snapshot_id"] == v["s1"] + imp = await db_get(TableImport, v["sep"]) + assert imp.revoked["reason"] == "合成理由：补录的数字有误" and imp.revoked["signed_by_verified"] is False + assert imp.status == "superseded" + src = await db_get(DataSource, sid) + assert (await db_get(SourceSnapshot, src.current_snapshot_id)).mode == "accumulate" + items = {s["id"]: s for s in await snapshots(client, sid)} + assert items[v["s2"]]["reason_code"] == "contains_revoked" and items[v["s2"]]["activatable"] is False + resp = await client.post(f"{API}/{sid}/snapshots/{v['s2']}/activate", + json={"confirm": True, "expected_current_snapshot_id": v["s1"]}) + assert resp.status_code == 409 and resp.json()["code"] == "contains_revoked" + last = (await activations(sid))[-1] + assert (last.kind, last.import_id) == ("revoke_acceptance", v["sep"]) + # 没有接受的导入不需要作废 + resp = await client.post(f"{API}/{sid}/imports/{v['aug']}/revoke-acceptance", + json={"confirm": True, "expected_current_snapshot_id": v["s1"], "reason": "x"}) + assert resp.status_code == 409 and resp.json()["code"] == "no_acceptance" + + +async def test_revoke_in_replace_mode_rolls_back_to_the_planned_target(client): + """替换模式：revoke_plan 给回滚目标（S1），请求带 expected_target_snapshot_id；目标对不上 409 revoke_target_changed。""" + v = await accepted_september(client, seed=413, mode="replace") + sid = v["source_id"] + plan = {r["id"]: r for r in await imports(client, sid)}[v["sep"]]["revoke_plan"] + assert plan["action"] == "rollback" and plan["target_snapshot_id"] == v["s1"] + assert plan["target"]["recipe_seq"] == 1 and plan["target"]["mode"] == "replace" + assert plan["target"]["simple"] is False and plan["mask_lost"] == [] + assert [p["start"] for p in plan["target"]["parts"]] == ["2026-08-01"] + url = f"{API}/{sid}/imports/{v['sep']}/revoke-acceptance" + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s2"], "reason": "合成理由", + "expected_target_snapshot_id": "0" * 64}) + assert resp.status_code == 409 and resp.json()["code"] == "revoke_target_changed" + resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s2"], "reason": "合成理由", + "expected_target_snapshot_id": v["s1"]}) + assert resp.status_code == 200, resp.text + assert resp.json()["action"] == "rollback" and resp.json()["snapshot_id"] == v["s1"] + assert (await db_get(TableImport, v["sep"])).revoked is not None + assert await count(client, v["name"], "日客流") == 31 + # 同一份文件再传（同一个构建）：作废那次的接受理由已被判定为不成立，不再作为「上次接受的理由」给出（第 5 节、H3） + assert v["trial"]["trial"]["acceptable"], "D26 有要写理由接受的核对" + raw, fn = DRIFT_CASES["D26"].build(413) + st = await reupload(client, sid, raw, fn) + assert st["trial"]["status"] == "needs_decision", st["trial"]["problems"] + assert st["trial"]["prior_acceptances"] == [] + + +# ========================================================================== +# 清单（7.3） +# ========================================================================== + + +async def test_manifests_verify_their_hash_and_single_snapshots_get_a_view(client): + v = await two_periods(client, seed=415) + sid = v["source_id"] + resp = await client.get(f"{API}/{sid}/imports/{v['sep']}/manifest") + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["kind"] == "import_manifest" and out["verified"] is True and out["content"]["import_id"] == v["sep"] + resp = await client.get(f"{API}/{sid}/snapshots/{v['s2']}/manifest") + assert resp.status_code == 200 and resp.json()["kind"] == "snapshot_manifest" + resp = await client.get(f"{API}/{sid}/snapshots/{v['s1']}/manifest") + view = resp.json() + assert view["kind"] == "snapshot_view" and view["artifact_id"] is None + assert [i["import_id"] for i in view["content"]["imports"]] == [v["aug"]] + resp = await client.get(f"{API}/{sid}/imports/nope/manifest") + assert resp.status_code == 404 and resp.json()["code"] == "import_not_found" + resp = await client.get(f"{API}/{sid}/snapshots/nope/manifest") + assert resp.status_code == 404 and resp.json()["code"] == "snapshot_not_found" + # 改掉导入清单的内容：取回时内容哈希对不上 → 409 manifest_tampered + imp = await db_get(TableImport, v["sep"]) + path = artifact_store._path_of(imp.manifest_artifact) + os.chmod(path, 0o644) + path.write_text(path.read_text("utf-8").replace("2026-09-01", "2026-09-02"), "utf-8") + resp = await client.get(f"{API}/{sid}/imports/{v['sep']}/manifest") + assert resp.status_code == 409 and resp.json()["code"] == "manifest_tampered" + + +async def test_every_version_route_is_reached(client): + """9.1：版本页的每条路由都打一次，确认没被 datasources 的通配路由截走（截走时回的是没有 code 的 404/405）。""" + v = await two_periods(client, seed=417) + sid = v["source_id"] + hits = [ + ("GET", f"{API}/{sid}/snapshots", None, 200), + ("GET", f"{API}/{sid}/snapshots/{v['s1']}/manifest", None, 200), + ("GET", f"{API}/{sid}/imports/{v['aug']}/manifest", None, 200), + ("POST", f"{API}/{sid}/snapshots/{v['s1']}/activate", {}, 422), + ("POST", f"{API}/{sid}/imports/{v['sep']}/remove", {}, 422), + ("POST", f"{API}/{sid}/imports/{v['sep']}/revoke-acceptance", {}, 422), + ] + for method, url, body, status in hits: + resp = await client.request(method, url, json=body) + assert resp.status_code == status, (url, resp.text) + if status != 200: + assert resp.json().get("code"), (url, resp.text) + # 期 1 的两条照旧：导入记录列表、清除原件 + assert (await client.get(f"{API}/{sid}/imports")).status_code == 200 + resp = await client.post(f"{API}/{sid}/imports/{v['aug']}/purge-raw", json={"reason": " "}) + assert resp.status_code == 422 and resp.json()["code"] == "reason_required" + + +async def test_retired_and_lost_snapshots_cannot_be_activated(client): + """已回收的版本 reason_code=retired、数据文件丢了的 file_lost，都不能启用（409 snapshot_unavailable）。""" + v = await two_periods(client, seed=419) + sid = v["source_id"] + snap = await db_get(SourceSnapshot, v["s1"]) + path = Path(snap.db_path) + moved = path.with_name(path.name + ".bak") + os.rename(path, moved) + items = {s["id"]: s for s in await snapshots(client, sid)} + assert items[v["s1"]]["reason_code"] == "file_lost" and items[v["s1"]]["available"] is False + assert items[v["s1"]]["db_size"] is None + resp = await client.post(f"{API}/{sid}/snapshots/{v['s1']}/activate", + json={"confirm": True, "expected_current_snapshot_id": v["s2"]}) + assert resp.status_code == 409 and resp.json()["code"] == "snapshot_unavailable" + os.rename(moved, path) + async with SessionLocal() as session: + await session.execute(update(SourceSnapshot).where(SourceSnapshot.id == v["s1"]) + .values(retired_at=utcnow())) + await session.commit() + items = {s["id"]: s for s in await snapshots(client, sid)} + assert items[v["s1"]]["reason_code"] == "retired" and items[v["s1"]]["reason"] == "已回收，无法启用" + resp = await client.post(f"{API}/{sid}/snapshots/{v['s1']}/activate", + json={"confirm": True, "expected_current_snapshot_id": v["s2"]}) + assert resp.status_code == 409 and resp.json()["code"] == "snapshot_unavailable" + + +async def test_removing_a_middle_period_recomputes_the_gap(client): + """[7 月, 8 月, 9 月] 移除 8 月：两边之间出现空缺，说明换成带空缺的累积句，快照清单的 gaps 记下（7.4 第 5 步）。""" + name = unique() + base = await first_import(client, name, seed=421, start=AUG.replace(month=7), days=31) + sid = base["source_id"] + ids = {"jul": base["commit"]["import_id"]} + current = base["commit"]["snapshot_id"] + for key, start, days, seed in (("aug", AUG, 31, 422), ("sep", SEP, 30, 423)): + raw, fn = flow_workbook(start, days, seed=seed) + st = await reupload(client, sid, raw, fn) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + ids[key], current = resp.json()["import_id"], resp.json()["snapshot_id"] + resp = await client.post(f"{API}/{sid}/imports/{ids['aug']}/remove", + json={"confirm": True, "expected_current_snapshot_id": current, "reason": "合成理由"}) + assert resp.status_code == 200, resp.text + snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"]) + manifest = artifact_store.load(snap.manifest_artifact) + assert manifest["gaps"] == [{"start": "2026-08-01", "end": "2026-08-31"}] + assert "各期之间有空缺" in snap.schema_cache["tables"]["日客流"]["comment"] + assert await count(client, name, "日客流") == 61 + + +async def test_revoke_of_a_middle_period_publishes_a_new_version_and_records_revoked(client): + """[8 月, 9 月 D26（写理由接受）, 10 月] 作废 9 月：剩下 [8 月, 10 月] 没有现成的版本，按目标配方新物化一份； + revoked 与切换在同一个事务里写进去，含 9 月的版本都不能再启用。""" + from tests.fixtures.xlsx.drift import OCT + + v = await accepted_september(client, seed=425, mode="accumulate") + sid = v["source_id"] + raw, fn = flow_workbook(OCT, 31, seed=426) + st = await reupload(client, sid, raw, fn) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + s3 = resp.json()["snapshot_id"] + resp = await client.post(f"{API}/{sid}/imports/{v['sep']}/revoke-acceptance", + json={"confirm": True, "expected_current_snapshot_id": s3, "reason": "合成理由"}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["action"] == "remove_period" and out["snapshot_id"] not in (v["s1"], v["s2"], s3) + imp = await db_get(TableImport, v["sep"]) + assert imp.revoked is not None and imp.status == "superseded" + snap = await db_get(SourceSnapshot, out["snapshot_id"]) + assert len(snap.imports) == 2 and artifact_store.load(snap.manifest_artifact)["action"] == "revoke_acceptance" + for old in (v["s2"], s3): + resp = await client.post(f"{API}/{sid}/snapshots/{old}/activate", + json={"confirm": True, "expected_current_snapshot_id": out["snapshot_id"]}) + assert resp.status_code == 409 and resp.json()["code"] == "contains_revoked" + assert await count(client, v["name"], "日客流") == 62 + + +# ========================================================================== +# 评审后补：锁外读到的当前版本与确认框绑定（ABA）、过期的 expected_current、移除时复活、并集只取决于剩下的各期、 +# 移除坏掉的正是那一期、原件共用的另一个源已删除 +# ========================================================================== + +NOV = SEP.replace(month=11) + + +async def three_periods(client, *, seed: int) -> dict[str, Any]: + """8 月、9 月、10 月按期累积（同一个配方）：S1=[8]、S2=[8, 9]、S3=[8, 9, 10]。""" + from tests.fixtures.xlsx.drift import OCT + + v = await two_periods(client, seed=seed) + raw, fn = flow_workbook(OCT, 31, seed=seed + 2) + st = await reupload(client, v["source_id"], raw, fn) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + return {**v, "s3": resp.json()["snapshot_id"], "oct": resp.json()["import_id"]} + + +async def current_periods(source_id: str) -> list[str]: + async with SessionLocal() as session: + src = await session.get(DataSource, source_id) + snap = await session.get(SourceSnapshot, src.current_snapshot_id) + rows = {i.id: i for i in (await session.execute( + select(TableImport).where(TableImport.id.in_(snap.imports)))).scalars()} + return [rows[i].period_start for i in snap.imports] + + +async def test_revoke_with_a_stale_expected_current_is_base_changed_in_both_modes(client): + """作废接受带过期的 expected_current：两种模式都 409 base_changed，revoked 不写、指针不动（11.4「base_changed + 在锁内」）。替换模式的回滚目标恰好等于过期的那个版本时也一样，不能悄悄切过去。""" + for mode, seed in (("accumulate", 431), ("replace", 433)): + v = await accepted_september(client, seed=seed, mode=mode) + sid = v["source_id"] + body = {"confirm": True, "expected_current_snapshot_id": v["s1"], "reason": "合成理由"} + if mode == "replace": + body["expected_target_snapshot_id"] = v["s1"] + resp = await client.post(f"{API}/{sid}/imports/{v['sep']}/revoke-acceptance", json=body) + assert resp.status_code == 409 and resp.json()["code"] == "base_changed", (mode, resp.text) + assert (await db_get(TableImport, v["sep"])).revoked is None, mode + assert (await db_get(DataSource, sid)).current_snapshot_id == v["s2"], mode + + +async def test_removing_a_middle_period_with_a_stale_expected_current_is_base_changed(client, store): + """移除中间一期（要物化）带过期的 expected_current：409 base_changed，当前版本三期都在，不留并集临时文件。""" + v = await three_periods(client, seed=435) + resp = await client.post(f"{API}/{v['source_id']}/imports/{v['sep']}/remove", + json={"confirm": True, "expected_current_snapshot_id": v["s2"], "reason": "合成理由"}) + assert resp.status_code == 409 and resp.json()["code"] == "base_changed", resp.text + assert (await db_get(DataSource, v["source_id"])).current_snapshot_id == v["s3"] + assert list((store / "uploads").rglob("*.tmp-*")) == [] + + +async def test_remove_binds_the_version_it_read_to_the_confirmed_one(client, monkeypatch): + """ABA（评审意见）：用户看到的当前版本是 E=[8, 9, 10]；别人先移除 10 月，当前变成 C=[8, 9]；用户随后按 E 发「移除 + 9 月」。处理器读到的是 C，算出 keep=[8]；若处理中又有人把 E 启用回来，锁内 E==E 照样通过，结果只剩 [8]，10 月 + 悄悄丢了。必须在读到当前版本之后立刻比对 expected_current，409 base_changed，一个切换都不做。""" + from app.data import table_versions + + v = await three_periods(client, seed=437) + sid = v["source_id"] + resp = await client.post(f"{API}/{sid}/imports/{v['oct']}/remove", + json={"confirm": True, "expected_current_snapshot_id": v["s3"], "reason": "合成理由：别人"}) + assert resp.status_code == 200, resp.text + c = resp.json()["snapshot_id"] + assert c == v["s2"] + orig = table_versions.activate_snapshot + fired: list[str] = [] + + async def activate(session, source, snapshot_id, **kw): + if kw.get("kind") == "remove_period" and not fired: + fired.append(snapshot_id) + async with SessionLocal() as other: + src = await other.get(DataSource, sid) + await orig(other, src, v["s3"], expected_current=c, kind="activate", reason="合成：回到 E", + signed_by="合成署名乙") + return await orig(session, source, snapshot_id, **kw) + + monkeypatch.setattr(table_versions, "activate_snapshot", activate) + resp = await client.post(f"{API}/{sid}/imports/{v['sep']}/remove", + json={"confirm": True, "expected_current_snapshot_id": v["s3"], "reason": "合成理由：用户"}) + assert resp.status_code == 409 and resp.json()["code"] == "base_changed", resp.text + assert fired == [], "比对在任何切换之前" + assert await current_periods(sid) == ["2026-08-01", "2026-09-01"] + + +async def test_a_commit_landing_during_the_materialization_makes_the_remove_base_changed(client, monkeypatch, store): + """移除中间一期停在物化里，期间 11 月的提交先完成：移除 409 base_changed（锁内比对），当前版本四期都在，物化出的 + 临时文件收拾掉。""" + import asyncio + + from app.data import source_versions + + v = await three_periods(client, seed=439) + sid = v["source_id"] + raw, fn = flow_workbook(NOV, 30, seed=442) + nov = await reupload(client, sid, raw, fn) + assert nov["trial"]["status"] == "passed" and nov["trial"]["accumulate"]["action"] == "append" + entered, release = asyncio.Event(), asyncio.Event() + orig = source_versions.table_versions_slot + + async def slot(fn, *a, **kw): + if getattr(fn, "__name__", "") == "materialize_union": + entered.set() + await release.wait() + return await orig(fn, *a, **kw) + + monkeypatch.setattr(source_versions, "table_versions_slot", slot) + task = asyncio.create_task(client.post(f"{API}/{sid}/imports/{v['sep']}/remove", json={ + "confirm": True, "expected_current_snapshot_id": v["s3"], "reason": "合成理由"})) + await asyncio.wait_for(entered.wait(), 30) + resp = await commit(client, nov) + assert resp.status_code == 201 and resp.json()["parts"] == 4, resp.text + release.set() + resp = await task + assert resp.status_code == 409 and resp.json()["code"] == "base_changed", resp.text + assert await current_periods(sid) == ["2026-08-01", "2026-09-01", "2026-10-01", "2026-11-01"] + assert list((store / "uploads").rglob("*.tmp-*")) == [] + + +async def test_remove_revives_a_retired_snapshot_of_the_same_imports(client, monkeypatch): + """SNAPSHOT_KEEP=0：9 月提交后 [8 月] 的 S1 被回收；移除 9 月按同一组导入、同一个目标配方得到 S1 的 id，复活它 + （7.4 第 4 步、2.6 发布第 4 步），不新建快照。""" + from app.data import table_versions + + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0) + v = await two_periods(client, seed=443) + assert (await db_get(SourceSnapshot, v["s1"])).retired_at is not None + resp = await client.post(f"{API}/{v['source_id']}/imports/{v['sep']}/remove", + json={"confirm": True, "expected_current_snapshot_id": v["s2"], "reason": "合成理由"}) + assert resp.status_code == 200, resp.text + assert resp.json()["snapshot_id"] == v["s1"] and resp.json()["reused"] is True + assert (await db_get(SourceSnapshot, v["s1"])).retired_at is None + assert await count(client, v["name"], "日客流") == 31 + + +async def _fix_until_passed(client, st: dict[str, Any]) -> dict[str, Any]: + """按提议依次修（加、去标签，更新关系成员），直到试运行通过。""" + from tests.test_table_imports_api_p3 import fixed + + for _ in range(6): + trial_ok = (st.get("trial") or {}).get("status") == "passed" + if trial_ok and not st["recipe_problems"]: + return st + kinds = [f["kind"] for f in st["fixes"]] + kind = next((k for k in ("add_label", "remove_label", "edit_members") if k in kinds), None) + if kind is None: + st = await trial(client, st["id"]) + continue + fx = next(f for f in st["fixes"] if f["kind"] == kind) + opt = fx["options"][0] + st = await fixed(client, st, kind, opt["value"], reason="合成理由" if opt["needs_reason"] else None) + if not st["recipe_problems"]: + st = await trial(client, st["id"]) + return st + + +async def test_removing_the_only_period_with_a_column_leaves_no_phantom_retired_column(client): + """并集只取决于（剩下各期, 目标配方）：8 月 R0；9 月 D13 用修复加「分区丙」（R1）；10 月修复去掉「分区丙」（R2， + 退役）；移除 9 月之后剩下的 8 月、10 月都没有分区丙，新并集不该再有这一列、快照清单也不记它（评审意见：原来按 + 当前并集的表结构推退役列，出现全为空值的「幽灵」列，since 还写成第一期）。""" + from tests.fixtures.xlsx.drift import OCT + + name = unique() + base = await first_import(client, name, seed=445) + sid = base["source_id"] + raw, fn = DRIFT_CASES["D13"].build(445) + st = await _fix_until_passed(client, await reupload(client, sid, raw, fn)) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + sep = resp.json()["import_id"] + raw, fn = flow_workbook(OCT, 31, seed=446) + st = await _fix_until_passed(client, await reupload(client, sid, raw, fn)) + assert st["trial"]["status"] == "passed", (st["trial"]["problems"], st["recipe_problems"]) + assert {"table": "日客流", "column": "分区丙"} in st["trial"]["accumulate"]["retired_new"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + s3 = resp.json()["snapshot_id"] + snap = await db_get(SourceSnapshot, s3) + assert "分区丙" in [c["name"] for c in snap.schema_cache["tables"]["日客流"]["columns"]], "9 月的分区丙照旧保留" + + resp = await client.post(f"{API}/{sid}/imports/{sep}/remove", + json={"confirm": True, "expected_current_snapshot_id": s3, "reason": "合成理由"}) + assert resp.status_code == 200, resp.text + snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"]) + assert "分区丙" not in [c["name"] for c in snap.schema_cache["tables"]["日客流"]["columns"]] + assert "分区丙" not in artifact_store.load(snap.manifest_artifact)["columns"].get("日客流", {}) + assert await count(client, name, "日客流") == 62 + + +async def test_removing_back_to_an_earlier_set_revives_it_even_after_the_recipe_returned(client, monkeypatch): + """SNAPSHOT_KEEP=0。8 月、10 月按 R0 得 S_ao（已回收）→ 补传 9 月 D13（修复加分区丙，R1）→ 11 月修复去掉分区丙（R2， + 与 R0 逐字相同、记录不同）→ 移除 9 月 → 移除 11 月：剩 [8 月, 10 月]、目标 R2，快照 id 正是 S_ao。复活它，不报 + snapshot_conflict（评审意见：原来并集带着幽灵列、id 不同；配方记录 id 也不同）。现行配方回到 S_ao 记的那条，内容不变。""" + from app.data import table_versions + from app.db.models import TableRecipe + from tests.fixtures.xlsx.drift import OCT + + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0) + name = unique() + base = await first_import(client, name, seed=447) + sid = base["source_id"] + raw, fn = flow_workbook(OCT, 31, seed=448) + resp = await commit(client, await reupload(client, sid, raw, fn)) + assert resp.status_code == 201, resp.text + s_ao, r0 = resp.json()["snapshot_id"], resp.json()["recipe_id"] + raw, fn = DRIFT_CASES["D13"].build(449) + st = await _fix_until_passed(client, await reupload(client, sid, raw, fn)) + assert st["trial"]["status"] == "passed" and st["trial"]["accumulate"]["backfill"] is True + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + sep = resp.json()["import_id"] + raw, fn = flow_workbook(NOV, 30, seed=450) + st = await _fix_until_passed(client, await reupload(client, sid, raw, fn)) + assert st["trial"]["status"] == "passed", (st["trial"]["problems"], st["recipe_problems"]) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + nov, cur, r2 = resp.json()["import_id"], resp.json()["snapshot_id"], resp.json()["recipe_id"] + assert r2 != r0 + assert (await db_get(TableRecipe, r2)).recipe_sha256 == (await db_get(TableRecipe, r0)).recipe_sha256 + resp = await client.post(f"{API}/{sid}/imports/{sep}/remove", + json={"confirm": True, "expected_current_snapshot_id": cur, "reason": "合成理由"}) + assert resp.status_code == 200, resp.text + assert (await db_get(SourceSnapshot, s_ao)).retired_at is not None + resp = await client.post(f"{API}/{sid}/imports/{nov}/remove", + json={"confirm": True, "expected_current_snapshot_id": resp.json()["snapshot_id"], + "reason": "合成理由"}) + assert resp.status_code == 200, resp.text + assert resp.json()["snapshot_id"] == s_ao and resp.json()["reused"] is True + snap = await db_get(SourceSnapshot, s_ao) + assert snap.retired_at is None + assert (await db_get(DataSource, sid)).current_recipe_id == r0 + assert await count(client, name, "日客流") == 62 + + +async def test_removing_the_broken_period_itself_is_allowed(client): + """被移除的那一期先排除、不校验（7.4「移除的正好是那一期时……不报这个错」）：它的构建记录不在了、或者统计期库列 + 被改过，照样能把它移出去；坏的是别的期时照旧拒绝（评审意见：原来 part_missing 的消息建议「先移除这一期」，却走 + 不通）。""" + from sqlalchemy import delete + + from app.db.models import TableBuild + + v = await three_periods(client, seed=451) + imp = await db_get(TableImport, v["sep"]) + async with SessionLocal() as session: + await session.execute(delete(TableBuild).where(TableBuild.id == imp.build_id)) + await session.commit() + resp = await client.post(f"{API}/{v['source_id']}/imports/{v['oct']}/remove", + json={"confirm": True, "expected_current_snapshot_id": v["s3"], "reason": "合成理由"}) + assert resp.status_code == 200, resp.text + assert resp.json()["snapshot_id"] == v["s2"], "复用 [8, 9] 的现成版本，不经组装" + resp = await client.post(f"{API}/{v['source_id']}/snapshots/{v['s3']}/activate", + json={"confirm": True, "expected_current_snapshot_id": v["s2"]}) + assert resp.status_code == 200, resp.text + resp = await client.post(f"{API}/{v['source_id']}/imports/{v['aug']}/remove", + json={"confirm": True, "expected_current_snapshot_id": v["s3"], "reason": "合成理由"}) + assert resp.status_code == 409 and resp.json()["code"] == "part_missing", "坏的是别的期：照旧拒绝" + assert "可以先移除这一期" in resp.json()["detail"] + resp = await client.post(f"{API}/{v['source_id']}/imports/{v['sep']}/remove", + json={"confirm": True, "expected_current_snapshot_id": v["s3"], "reason": "合成理由"}) + assert resp.status_code == 200, resp.text + assert await current_periods(v["source_id"]) == ["2026-08-01", "2026-10-01"] + + w = await three_periods(client, seed=453) + async with SessionLocal() as session: + await session.execute(update(TableImport).where(TableImport.id == w["sep"]).values(period_start="2026-09-02")) + await session.commit() + resp = await client.post(f"{API}/{w['source_id']}/imports/{w['sep']}/remove", + json={"confirm": True, "expected_current_snapshot_id": w["s3"], "reason": "合成理由"}) + assert resp.status_code == 200, resp.text + assert await current_periods(w["source_id"]) == ["2026-08-01", "2026-10-01"] + + +async def test_raw_shared_with_names_a_deleted_source_instead_of_null(client): + """同一份原件传给两个源，删掉其中一个：另一个源的 raw_shared_with 仍列出这条引用（原件还在、清除时会一并清除）， + 名字给「已删除的数据源」而不是 null（界面的类型是 string，直接拼进文案，评审意见）。""" + from tests.test_p3_api_errors import sheet_bytes, upload + + raw = sheet_bytes(455) + ra, rb = await upload(client, unique("sha"), raw), await upload(client, unique("shb"), raw) + assert ra.status_code == 201 and rb.status_code == 201, (ra.text, rb.text) + sid_a, b = ra.json()["source"]["id"], rb.json()["source"] + [rec] = await imports(client, sid_a) + assert rec["raw_shared_with"] == [{"source_name": b["name"], "count": 1}] + assert (await client.delete(f"{API}/{b['id']}")).status_code == 204 + [rec] = await imports(client, sid_a) + assert rec["raw_shared_with"] == [{"source_name": "已删除的数据源", "count": 1}] diff --git a/backend/tests/test_table_imports_api.py b/backend/tests/test_table_imports_api.py index b8213049..6146d18d 100644 --- a/backend/tests/test_table_imports_api.py +++ b/backend/tests/test_table_imports_api.py @@ -113,7 +113,9 @@ async def first_import(client, name: str, seed: int = 1) -> dict[str, Any]: assert resp.status_code == 201, resp.text st = resp.json() sid = st["id"] - answers = {q["id"]: {"value": "register" if q["id"].startswith("q_relation") else "null"} + # 期 3：q_mode（这份报表每期怎么更新）答「每期替换」，与参考配方的哈希一致（P3-SPEC 12.0、1.5） + answers = {q["id"]: {"value": "register" if q["id"].startswith("q_relation") + else "replace" if q["id"] == "q_mode" else "null"} for q in st["questions"]} resp = await client.post(f"/api/datasources/imports/{sid}/answers", json={"answers": answers}) assert resp.status_code == 200, resp.text @@ -158,19 +160,23 @@ async def test_real_pipeline_first_import_stage_draft_answer_trial_commit(client "q_relation:F1": {"value": "dismiss", "reason": "合成理由：判断为巧合"}}}) assert first.status_code == 200, first.text assert next(r for r in first.json()["recipe"]["relations"] if r["id"] == "R1")["kind"] == "dismissed" - answers = {q: {"value": "register" if q.startswith("q_relation") else "null"} for q in qids} + answers = {q: {"value": "register" if q.startswith("q_relation") else "replace" if q == "q_mode" else "null"} + for q in qids} resp = await client.post(f"/api/datasources/imports/{sid}/answers", json={"answers": answers}) assert resp.status_code == 200, resp.text st = resp.json() assert st["answers"]["q_relation:F1"] == {"value": "register", "reason": None} assert [q["id"] for q in st["questions"]] == qids, "问题集合按起点配方生成，回答不改变它" + answered = st["answers"] - # 改表名（PUT）：起点换成它，回答清空；与参考配方逐字相同 + # 改表名（PUT）：起点换成它；与参考配方逐字相同。期 3（第 8 节遗留项 2）：PUT 进来的配方已经体现了之前的 + # 回答，回答照旧保留（与新起点一致），没有被丢弃的回答 resp = await client.put(f"/api/datasources/imports/{sid}/recipe", json={"recipe": rename_wide(st["recipe"], "客流汇总_按日", "日客流")}) assert resp.status_code == 200, resp.text st = resp.json() - assert st["answers"] == {} and st["recipe_origin"] == "manual" and st["recipe_problems"] == [] + assert st["answers"] == answered and st["answers_dropped"] == [] + assert st["recipe_origin"] == "manual" and st["recipe_problems"] == [] ref, _ = recipe.validate_recipe(FLOW_RECIPE, origin="manual") mine, _ = recipe.validate_recipe(st["recipe"], origin="manual") assert recipe.recipe_sha256(mine) == recipe.recipe_sha256(ref) diff --git a/backend/tests/test_table_imports_api_p3.py b/backend/tests/test_table_imports_api_p3.py new file mode 100644 index 00000000..ff2230c2 --- /dev/null +++ b/backend/tests/test_table_imports_api_p3.py @@ -0,0 +1,964 @@ +"""按配方导入期 3 的接口（WP-5，P3-SPEC 9.1、12.6）：真管线、经 HTTP。 + +覆盖按期累积的试运行与提交（追加、重叠拒收、替换该期、复用同一构建）、修复按钮（预览、应用、撤销、过期、 +PUT 覆盖）、按规则重新起草与采用、改配方后回答的延续、StagingOut 与 TrialOut 的新字段。版本页的接口在 +test_source_versions_api.py，错误码与遗留项在 test_p3_api_errors.py。 + +夹具一律合成、假名（tests/fixtures/xlsx 的客流表：分区甲、分区乙……），数字是随机数；不调用任何模型。 +""" +from __future__ import annotations + +import json +import uuid +from pathlib import Path +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select + +from app.core import artifact_store +from app.core.config import settings +from app.data import table_versions +from app.db.base import SessionLocal +from app.db.models import ImportStaging, SourceSnapshot, TableImport +from app.main import app +from tests.fixtures.xlsx.drift import AUG, DRIFT_CASES, OCT, SEP +from tests.fixtures.xlsx.flow import flow_workbook + +XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +API = "/api/datasources" +TABLES = ("日客流", "时段客流", "时段客流_表内合计") + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +# ========================================================================== +# 小工具（经 HTTP） +# ========================================================================== + + +def unique(prefix: str = "p3") -> str: + return f"{prefix}_{uuid.uuid4().hex[:10]}" + + +def rename_wide(rec: dict[str, Any], old: str, new: str) -> dict[str, Any]: + """把规则起草的宽表名改成参考配方的名字（写法同 test_recipe_e2e.rename_wide）。""" + rec = json.loads(json.dumps(rec, ensure_ascii=False)) + for sheet in rec["sheets"]: + for block in sheet["blocks"]: + for seg in block.get("segments", []): + if seg.get("table") == old: + seg["table"] = new + if seg.get("id") == old: + seg["id"] = new + for t in rec["tables"]: + if t["name"] == old: + t["name"] = new + for r in rec["relations"]: + if r.get("table") == old: + r["table"] = new + for side in ("a", "b"): + if isinstance(r.get(side), dict) and r[side].get("table") == old: + r[side]["table"] = new + return rec + + +async def stage(client, name: str, raw: bytes, filename: str): + return await client.post(f"{API}/imports/stage", files={"file": (filename, raw, XLSX)}, data={"name": name}) + + +async def reupload(client, source_id: str, raw: bytes, filename: str) -> dict[str, Any]: + resp = await client.post(f"{API}/{source_id}/reupload", files={"file": (filename, raw, XLSX)}) + assert resp.status_code == 201, resp.text + return resp.json() + + +async def trial(client, sid: str, **body: Any) -> dict[str, Any]: + resp = await client.post(f"{API}/imports/{sid}/trial", json=body) + assert resp.status_code == 200, resp.text + return resp.json() + + +def confirms(st: dict[str, Any]) -> list[str]: + return [i["id"] for i in (st.get("trial") or {}).get("confirm_items") or []] + + +async def commit(client, st: dict[str, Any], *, confirmations: list[str] | None = None, + acceptances: list[dict[str, str]] | None = None): + body: dict[str, Any] = {"trial_id": st["trial"]["trial_id"], + "confirmations": confirms(st) if confirmations is None else confirmations} + if acceptances is not None: + body["acceptances"] = acceptances + return await client.post(f"{API}/imports/{st['id']}/commit", json=body) + + +def answers_for(questions: list[dict[str, Any]], mode: str) -> dict[str, dict[str, Any]]: + out = {} + for q in questions: + values = [o["value"] for o in q["options"]] + if q["id"].startswith("q_relation"): + out[q["id"]] = {"value": "register"} + elif q["id"] == "q_mode": + out[q["id"]] = {"value": mode} + elif "null" in values: + out[q["id"]] = {"value": "null"} + else: + out[q["id"]] = {"value": values[0]} + return out + + +async def first_import(client, name: str, *, seed: int, mode: str = "accumulate", start=AUG, days: int = 31, + variant: str = "D00") -> dict[str, Any]: + """首次导入（回答问题、宽表改名「日客流」、试运行、提交），q_mode 按 mode 答。返回提交响应和试运行。""" + raw, fn = flow_workbook(start, days, seed=seed, variant=variant) + resp = await stage(client, name, raw, fn) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["questions"][-1]["id"] == "q_mode", "q_mode 固定排在问题列表最后" + resp = await client.post(f"{API}/imports/{st['id']}/answers", + json={"answers": answers_for(st["questions"], mode)}) + assert resp.status_code == 200, resp.text + st = resp.json() + wide = next(t["name"] for t in st["recipe"]["tables"] if t["name"] not in TABLES) + resp = await client.put(f"{API}/imports/{st['id']}/recipe", json={"recipe": rename_wide(st["recipe"], wide, "日客流")}) + assert resp.status_code == 200, resp.text + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + return {"commit": out, "trial": st, "source_id": out["source"]["id"], "raw": raw, "file_name": fn} + + +async def tool_rows(client, source_name: str, sql: str) -> list[list[Any]]: + resp = await client.post(f"/api/tools/db_query__{source_name}/run", json={"args": {"sql": sql}}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["ok"], out + return json.loads(out["result"])["rows"] + + +async def count(client, name: str, table: str, where: str = "") -> int: + return int((await tool_rows(client, name, f'SELECT COUNT(*) FROM "{table}"{where}'))[0][0]) + + +async def db_get(model, key): + async with SessionLocal() as session: + return await session.get(model, key) + + +async def source_row(client, source_id: str) -> dict[str, Any]: + resp = await client.get(API) + return next(s for s in resp.json() if s["id"] == source_id) + + +# ========================================================================== +# 按期累积：追加 +# ========================================================================== + + +async def test_accumulate_august_then_september_materializes_and_publishes_the_union(client): + """8 月首次导入（按期累积）+ 9 月上传新一期：试运行出累积计划、物化并集（U1–U3 通过），提交走 publish_snapshot， + 当前版本两期、库在 snapshots/ 下，三表 61 / 1037 / 183，按月 31 / 30（2.12）。""" + name = unique() + base = await first_import(client, name, seed=301) + source_id = base["source_id"] + first_snapshot = base["commit"]["snapshot_id"] + snap = await db_get(SourceSnapshot, first_snapshot) + assert snap.mode == "accumulate" and Path(snap.db_path).parent.name == "builds", "单期快照库就是那一期的构建库" + card = await source_row(client, source_id) + assert card["current_snapshot"]["mode"] == "accumulate" and card["current_snapshot"]["periods"] == 1 + + raw, fn = flow_workbook(SEP, 30, seed=302) + st = await reupload(client, source_id, raw, fn) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + plan = t["accumulate"] + assert plan["action"] == "append" and plan["mode"] == "accumulate" + assert [(p["start"], p["new"]) for p in plan["parts"]] == [("2026-08-01", False), ("2026-09-01", True)] + assert plan["parts"][1]["rows"] == {"日客流": 30, "时段客流": 510, "时段客流_表内合计": 90} + assert plan["parts"][1]["file_name"] == fn + assert plan["union"]["rows"] == {"日客流": 61, "时段客流": 1037, "时段客流_表内合计": 183} + assert [(c["id"], c["status"]) for c in t["union_checks"]] == [("U1", "passed"), ("U2", "passed"), + ("U3", "passed")] + assert t["recipe_compare"] is None, "配方没变：不给对照" + assert t["prior_acceptances"] == [] + assert "union" not in t and "prev_import_id" not in t, "并集的内部信息不给界面" + assert t["receipt"]["rows_excluded"] == [], "TrialOut.receipt 带排除的行（期 3 的试运行一律带这个键）" + union_file = table_versions.union_trial_path(Path((await db_get(ImportStaging, st["id"])).trial_path)) + assert union_file.is_file(), "并集在试运行里物化，和主试运行库放在同一目录" + assert "本表按统计期累积多期数据" in t["notes"]["日客流"]["comment"], t["notes"]["日客流"] + + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["parts"] == 2 and out["snapshot_reused"] is False and out["build_restored"] is False + assert not union_file.exists(), "并集的试运行文件被发布消耗" + snap = await db_get(SourceSnapshot, out["snapshot_id"]) + imps = [await db_get(TableImport, i) for i in snap.imports] + assert [i.period_start for i in imps] == ["2026-08-01", "2026-09-01"] + assert [i.status for i in imps] == ["active", "active"], "当前快照里的各期都是 active（2.1）" + assert snap.mode == "accumulate" and Path(snap.db_path).parent.name == "snapshots" + assert snap.manifest_artifact and snap.schema_cache["snapshot_manifest"] == snap.manifest_artifact + for table, n in zip(TABLES, (61, 1037, 183)): + assert await count(client, name, table) == n + months = await tool_rows(client, name, 'SELECT substr("日期", 1, 7), COUNT(*) FROM "日客流" GROUP BY 1 ORDER BY 1') + assert months == [["2026-08", 31], ["2026-09", 30]] + # 快照清单与证据链（2.9）：union.db_sha256 等于快照库的哈希，各期 rowid 区间相接 + manifest = artifact_store.load(snap.manifest_artifact) + assert manifest["format"] == "agentlab-snapshot-manifest/1" and manifest["action"] == "append" + assert manifest["union"]["db_sha256"] == snap.db_sha256 + rows = [p["rows"]["日客流"] for p in manifest["parts"]] + assert rows[0]["union"] == [1, 31] and rows[1]["union"] == [32, 61] and rows[1]["part"] == [1, 30] + assert snap.schema_cache["import_manifests"] == [i.manifest_artifact for i in imps] + # 新一期的导入清单：receipt 带排除的行，edits 是空列表 + new_manifest = artifact_store.load(imps[1].manifest_artifact) + assert new_manifest["receipt"]["rows_excluded"] == [] and new_manifest["edits"] == [] + # 卡片：按期累积 · 2 期，统计期范围 + card = await source_row(client, source_id) + cur = card["current_snapshot"] + assert cur["mode"] == "accumulate" and cur["periods"] == 2 + assert (cur["period_start"], cur["period_end"]) == ("2026-08-01", "2026-09-30") and cur["activated_at"] + + +async def test_overlapping_period_is_rejected_and_the_current_version_stays(client): + """统计期与已有的部分重叠：试运行拒收（period_overlap，消息写两段统计期），提交 409 trial_not_passed（A2）。""" + name = unique() + base = await first_import(client, name, seed=303) + source_id = base["source_id"] + raw, fn = flow_workbook(SEP, 30, seed=304) + st = await reupload(client, source_id, raw, fn) + assert (await commit(client, st)).status_code == 201 + current = (await source_row(client, source_id))["current_snapshot"]["id"] + raw, fn = flow_workbook(AUG.replace(day=15), 31, seed=305) + st = await reupload(client, source_id, raw, fn) + t = st["trial"] + assert t["status"] == "rejected" + overlap = next(p for p in t["problems"] if p["code"] == "period_overlap") + assert overlap["category"] == "structure" + assert "2026-08-01 至 2026-08-31" in overlap["message"] and "2026-09-01 至 2026-09-30" in overlap["message"] + assert t["accumulate"]["action"] == "rejected" and t["union_checks"] is None + resp = await commit(client, st) + assert resp.status_code == 409 and resp.json()["code"] == "trial_not_passed" + assert (await source_row(client, source_id))["current_snapshot"]["id"] == current + + +async def test_replace_period_needs_its_confirmation_and_keeps_the_row_counts(client): + """同一统计期再传（数字不同）：replace_period，必勾 period_replace:<起>~<止>；勾上后新快照是 [8 月, 9 月新]（A3）。""" + name = unique() + base = await first_import(client, name, seed=306) + source_id = base["source_id"] + raw, fn = flow_workbook(SEP, 30, seed=307) + st = await reupload(client, source_id, raw, fn) + assert (await commit(client, st)).status_code == 201 + old_sep = (await tool_rows(client, name, 'SELECT "全日客流" FROM "日客流" WHERE "日期" = \'2026-09-05\''))[0][0] + raw2, fn2 = flow_workbook(SEP, 30, seed=308) + st = await reupload(client, source_id, raw2, fn2) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert t["accumulate"]["action"] == "replace_period" + assert t["accumulate"]["replaces"]["start"] == "2026-09-01" + item = "period_replace:2026-09-01~2026-09-30" + got = {i["id"]: i for i in t["confirm_items"]} + assert item in got and got[item]["source"] == "accumulate" and got[item]["required"] + resp = await commit(client, st, confirmations=[i for i in confirms(st) if i != item]) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" and item in resp.json()["detail"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"]) + assert len(snap.imports) == 2 + for table, n in zip(TABLES, (61, 1037, 183)): + assert await count(client, name, table) == n + new_sep = (await tool_rows(client, name, 'SELECT "全日客流" FROM "日客流" WHERE "日期" = \'2026-09-05\''))[0][0] + assert new_sep != old_sep + + +async def test_same_file_again_in_accumulate_mode_is_unchanged(client): + """累积模式下同一个 9 月文件原样再传：替换该期且构建相同 → 不新建任何记录（same_as_import 的累积扩展）。""" + name = unique() + base = await first_import(client, name, seed=309) + source_id = base["source_id"] + raw, fn = flow_workbook(SEP, 30, seed=310) + st = await reupload(client, source_id, raw, fn) + first = await commit(client, st) + assert first.status_code == 201 + st = await reupload(client, source_id, raw, fn) + assert st["trial"]["accumulate"]["action"] == "replace_period" + assert st["trial"]["same_as_import"]["id"] == first.json()["import_id"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["unchanged"] is True and out["snapshot_id"] == first.json()["snapshot_id"] and out["parts"] == 2 + async with SessionLocal() as session: + n = len(list((await session.execute(select(TableImport).where(TableImport.source_id == source_id))).scalars())) + assert n == 2 + + +# ========================================================================== +# 修复按钮：预览、应用、撤销、过期、PUT 覆盖 +# ========================================================================== + + +def problem_with(st: dict[str, Any], code: str) -> dict[str, Any]: + trial_problems = (st.get("trial") or {}).get("problems") or [] + return next(p for p in [*trial_problems, *st["recipe_problems"], *st["draft_problems"]] if p["code"] == code) + + +def fix_of(st: dict[str, Any], kind: str) -> dict[str, Any]: + return next(f for f in st["fixes"] if f["kind"] == kind) + + +async def preview(client, sid: str, body: dict[str, Any]): + return await client.post(f"{API}/imports/{sid}/edits/preview", json=body) + + +async def apply(client, sid: str, body: dict[str, Any], expected: str | None, **extra: Any): + return await client.post(f"{API}/imports/{sid}/edits/apply", json={**body, "expected_sha256": expected, **extra}) + + +async def fixed(client, st: dict[str, Any], kind: str, option: str, reason: str | None = None) -> dict[str, Any]: + """按提议预览再应用，返回应用后的 StagingOut。""" + fx = fix_of(st, kind) + body = {"fix": {"id": fx["id"], "option": option, **({"reason": reason} if reason else {})}} + resp = await preview(client, st["id"], body) + assert resp.status_code == 200, resp.text + pv = resp.json() + assert pv["ok"], pv["problems"] + resp = await apply(client, st["id"], body, pv["recipe_sha256_after"]) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def test_d04_remove_label_fix_preview_apply_trial_and_commit(client): + """D04（日间少了 7-8）：拒收的问题带 fix_ids，提议去掉标签；预览不改暂存区，应用后试运行通过，确认清单的配方类只有 + fix:*（A6、A15），并集 61 / 1007 / 183（3.4），修改记录进导入清单。""" + name = unique() + base = await first_import(client, name, seed=311) + source_id = base["source_id"] + raw, fn = DRIFT_CASES["D04"].build(311) + st = await reupload(client, source_id, raw, fn) + assert st["trial"]["status"] == "rejected" + missing = problem_with(st, "label_missing") + fx = fix_of(st, "remove_label") + assert missing["fix_ids"] == [fx["id"]] and fx["anchor"]["kind"] == "problem" + assert all("fix_ids" in p for p in st["trial"]["problems"]), "每条问题都带 fix_ids（没有提议时是 []）" + assert st["recipe_sha256"] and st["edits"] == [] and st["answers_dropped"] == [] + before_sha = st["recipe_sha256"] + + body = {"fix": {"id": fx["id"], "option": "remove"}, "seq": 3} + resp = await preview(client, st["id"], body) + assert resp.status_code == 200, resp.text + pv = resp.json() + assert pv["ok"] is True and pv["seq"] == 3 and pv["kind"] == "fix" + assert pv["key"] == "remove_label:日间:7-8" and pv["recipe_sha256_before"] == before_sha + assert pv["recipe_sha256_after"] and pv["recipe_sha256_after"] != before_sha + assert pv["breaking"] == {} and isinstance(pv["breaking"], dict), "dimension 段去标签不改表结构" + assert pv["recipe_problems"] == [] and pv["dry_run"]["problems"] == [] and pv["dry_run"]["marks"] + assert pv["accumulate_change"] == "compatible" + seg = next(s for s in pv["compare"]["segments"] if s["id"] == "日间") + assert seg["labels"]["removed"] == ["7-8"] + # 预览不改暂存区 + again = (await client.get(f"{API}/imports/{st['id']}")).json() + assert again["recipe_sha256"] == before_sha and again["edits"] == [] + + resp = await apply(client, st["id"], {"fix": body["fix"]}, pv["recipe_sha256_after"], signed_by="合成署名甲") + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["status"] == "drafting" and st["recipe_sha256"] == pv["recipe_sha256_after"] + [edit] = st["edits"] + assert edit["key"] == "remove_label:日间:7-8" and edit["undoable"] is True and edit["superseded"] is False + assert edit["signed_by"] == "合成署名甲" and "before" not in edit and "base_sha256_after" not in edit + st = await trial(client, st["id"]) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert t["accumulate"]["action"] == "append" + assert t["accumulate"]["union"]["rows"] == {"日客流": 61, "时段客流": 1007, "时段客流_表内合计": 183} + assert t["recipe_compare"] is not None, "工作配方与现行配方不同：给新旧对照" + items = {i["id"]: i for i in t["confirm_items"]} + assert items["fix:remove_label:日间:7-8"]["source"] == "edit" + recipe_class = [i for i, v in items.items() if v.get("source", "recipe") == "recipe"] + assert recipe_class == [], recipe_class + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + imp = await db_get(TableImport, resp.json()["import_id"]) + manifest = artifact_store.load(imp.manifest_artifact) + assert [e["key"] for e in manifest["edits"]] == ["remove_label:日间:7-8"] + assert "before" not in manifest["edits"][0] + # 3.1：修改记录（谁、何时、补丁）进导入清单——补丁和所选的选项都在，不只剩人话摘要 + assert manifest["edits"][0]["fix"] == {"id": fx["id"], "option": "remove", "reason": None} + assert {"op": "replace", "path": manifest["edits"][0]["ops"][0]["path"], + "value": manifest["edits"][0]["ops"][0]["value"]} == manifest["edits"][0]["ops"][0] + assert manifest["edits"][0]["ops"][0]["path"].endswith("/labels/expect") + assert "7-8" not in manifest["edits"][0]["ops"][0]["value"] + assert await count(client, name, "时段客流") == 1007 + + +async def test_undo_restores_the_start_and_put_supersedes_earlier_edits(client): + """撤销恢复修改前的起点；撤销栈空了回 nothing_to_undo。PUT 之后之前的修改标「已被覆盖」，不能撤销、不出 fix:*（3.1）。""" + name = unique() + base = await first_import(client, name, seed=312) + raw, fn = DRIFT_CASES["D04"].build(312) + st = await reupload(client, base["source_id"], raw, fn) + before_sha = st["recipe_sha256"] + st = await fixed(client, st, "remove_label", "remove") + resp = await client.post(f"{API}/imports/{st['id']}/edits/undo", json={}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["recipe_sha256"] == before_sha and st["edits"] == [] + resp = await client.post(f"{API}/imports/{st['id']}/edits/undo", json={}) + assert resp.status_code == 409 and resp.json()["code"] == "nothing_to_undo" + + st = await trial(client, st["id"]) + st = await fixed(client, st, "remove_label", "remove") + resp = await client.put(f"{API}/imports/{st['id']}/recipe", json={"recipe": st["recipe"]}) + assert resp.status_code == 200, resp.text + st = resp.json() + [edit] = st["edits"] + assert edit["superseded"] is True and edit["undoable"] is False + resp = await client.post(f"{API}/imports/{st['id']}/edits/undo", json={}) + assert resp.status_code == 409 and resp.json()["code"] == "nothing_to_undo" + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert not [i for i in confirms(st) if i.startswith("fix:")], "被覆盖的修改不再出确认项" + + +async def test_needs_reason_fix_refuses_a_changed_reason_and_records_rows_excluded(client): + """D21（表下补录一行）：按行标签忽略要理由。没理由预览 422 reason_required；预览之后改了理由，应用 409 edit_stale + （理由进配方哈希）；照原样应用后试运行通过，回执的 rows_excluded 有 ignored_rows（A6b）。提议不在了回 fix_stale。""" + name = unique() + base = await first_import(client, name, seed=313) + raw, fn = DRIFT_CASES["D21"].build(313) + st = await reupload(client, base["source_id"], raw, fn) + assert st["trial"]["status"] == "rejected" + fx = fix_of(st, "ignore_cells") + assert fx["options"][0]["needs_reason"] is True + option = fx["options"][0]["value"] + resp = await preview(client, st["id"], {"fix": {"id": fx["id"], "option": option}}) + assert resp.status_code == 422 and resp.json()["code"] == "reason_required" + body = {"fix": {"id": fx["id"], "option": option, "reason": "合成理由：补录行另有出处"}} + pv = (await preview(client, st["id"], body)).json() + assert pv["ok"], pv["problems"] + changed = {"fix": {**body["fix"], "reason": "合成理由：改过的理由"}} + resp = await apply(client, st["id"], changed, pv["recipe_sha256_after"]) + assert resp.status_code == 409 and resp.json()["code"] == "edit_stale" + resp = await apply(client, st["id"], body, pv["recipe_sha256_after"]) + assert resp.status_code == 200, resp.text + resp = await preview(client, st["id"], body) + assert resp.status_code == 409 and resp.json()["code"] == "fix_stale", "应用之后问题没了，提议也不在了" + st = await trial(client, st["id"]) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + reasons = {r["reason"] for r in t["receipt"]["rows_excluded"]} + assert "ignored_rows" in reasons, t["receipt"]["rows_excluded"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + imp = await db_get(TableImport, resp.json()["import_id"]) + manifest = artifact_store.load(imp.manifest_artifact) + assert {r["reason"] for r in manifest["receipt"]["rows_excluded"]} == reasons + + +async def test_d13_add_label_then_edit_members_two_steps(client): + """D13（多一个分区）：加标签之后静态校验报 fact_claim_mismatch，配方问题带 fix_ids，提议「更新关系成员」；两步 + 修复后试运行通过，确认项有两条 fix:*，提交后 8 月的分区丙为空值（3.4）。""" + name = unique() + base = await first_import(client, name, seed=314) + raw, fn = DRIFT_CASES["D13"].build(314) + st = await reupload(client, base["source_id"], raw, fn) + assert st["trial"]["status"] == "rejected" + st = await fixed(client, st, "add_label", "add") + mismatch = problem_with(st, "fact_claim_mismatch") + fx = fix_of(st, "edit_members") + assert mismatch["fix_ids"] == [fx["id"]] and fx["anchor"]["kind"] == "recipe_problem" + st = await fixed(client, st, "edit_members", "update") + assert st["recipe_problems"] == [], st["recipe_problems"] + assert [e["kind"] for e in st["edits"]] == ["fix", "fix"] and [e["undoable"] for e in st["edits"]] == [False, True] + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + ids = confirms(st) + assert len([i for i in ids if i.startswith("fix:")]) == 2, ids + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert await count(client, name, "日客流", ' WHERE "分区丙" IS NULL') == 31 + + +async def test_d09_rename_title_keeps_the_confirmed_word(client): + """D09（日间标题改字）：改分段标题、沿用「日间」（3.3 (b)）。传 base 之后静态校验放行，确认项没有 breaking:*。""" + name = unique() + base = await first_import(client, name, seed=315) + raw, fn = DRIFT_CASES["D09"].build(315) + st = await reupload(client, base["source_id"], raw, fn) + fx = fix_of(st, "rename_title") + keep = next(o["value"] for o in fx["options"] if o["value"].endswith("|keep")) + st = await fixed(client, st, "rename_title", keep) + assert st["recipe_problems"] == [], st["recipe_problems"] + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert not [i for i in confirms(st) if i.startswith("breaking:")], confirms(st) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + kinds = await tool_rows(client, name, 'SELECT DISTINCT "时段类别" FROM "时段客流" ORDER BY 1') + assert sorted(k[0] for k in kinds) == ["夜间", "日间"] + + +# ========================================================================== +# 按规则重新起草、改配方后回答的延续 +# ========================================================================== + + +async def test_redraft_rules_then_adopt_lists_every_recipe_item_and_redraft_adopted(client): + """redraft-rules 不改工作配方，记下对齐后的哈希；PUT（origin=rules_redraft）核对它，对不上 409 redraft_mismatch； + 采用之后确认清单按首次导入列出全部配方类项，另出 redraft_adopted（6.3，评审一-M6）。首次导入不提供。""" + name = unique() + base = await first_import(client, name, seed=316, mode="replace") + # 与现行配方逐字相同的重新起草（同一版式的 9 月）：采用之后没有可确认的变化，只出 redraft_adopted 说明「相同」 + raw, fn = flow_workbook(SEP, 30, seed=317) + same = await reupload(client, base["source_id"], raw, fn) + out = (await client.post(f"{API}/imports/{same['id']}/redraft-rules", json={})).json() + assert out["aligned_recipe"] is not None + resp = await client.put(f"{API}/imports/{same['id']}/recipe", + json={"recipe": out["aligned_recipe"], "origin": "rules_redraft"}) + assert resp.json()["recipe_sha256"] == same["recipe_sha256"], "按来源对齐之后与现行配方逐字相同" + resp = await client.delete(f"{API}/imports/{same['id']}") + assert resp.status_code == 204 + # D04（日间少了 7-8）：重新起草出的分段标签与现行不同 + raw, fn = DRIFT_CASES["D04"].build(317) + st = await reupload(client, base["source_id"], raw, fn) + before = st["recipe_sha256"] + resp = await client.post(f"{API}/imports/{st['id']}/redraft-rules", json={}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["aligned_recipe"] is not None and isinstance(out["alignment"], list) + assert out["compare"] is not None and "failures_for_model" not in out["draft"] + assert (await client.get(f"{API}/imports/{st['id']}")).json()["recipe_sha256"] == before, "不改工作配方" + tampered = json.loads(json.dumps(out["aligned_recipe"], ensure_ascii=False)) + tampered["tables"][0]["note"] = "合成说明" + resp = await client.put(f"{API}/imports/{st['id']}/recipe", json={"recipe": tampered, "origin": "rules_redraft"}) + assert resp.status_code == 409 and resp.json()["code"] == "redraft_mismatch" + resp = await client.put(f"{API}/imports/{st['id']}/recipe", + json={"recipe": out["aligned_recipe"], "origin": "rules_redraft"}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["recipe_origin"] == "rules_redraft" + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + items = {i["id"]: i for i in st["trial"]["confirm_items"]} + assert items["redraft_adopted"]["source"] == "edit" and "7-8" in items["redraft_adopted"]["label"] + assert "mode" in items, "采用重新起草的结果时配方类项全部列出（只改了分段标签也一样）" + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + + raw, fn = flow_workbook(AUG, 31, seed=318) + first = (await stage(client, unique(), raw, fn)).json() + resp = await client.post(f"{API}/imports/{first['id']}/redraft-rules", json={}) + assert resp.status_code == 409 and resp.json()["code"] == "redraft_not_offered" + + +async def test_put_keeps_answers_the_new_start_embodies_and_drops_the_rest(client): + """L2：先回答占位符「·」存为空值，再 PUT 一份体现了这个回答的配方：回答仍在、answers_dropped 为空；PUT 一份去掉 + 占位符的配方：answers_dropped 列出它（带问题文字，不只给 id）。""" + raw, fn = flow_workbook(AUG, 31, seed=319) + st = (await stage(client, unique(), raw, fn)).json() + qid = "q_placeholder:·" + assert qid in {q["id"] for q in st["questions"]} + resp = await client.post(f"{API}/imports/{st['id']}/answers", json={"answers": {qid: {"value": "null"}}}) + st = resp.json() + resp = await client.put(f"{API}/imports/{st['id']}/recipe", json={"recipe": st["recipe"]}) + st = resp.json() + assert st["answers"] == {qid: {"value": "null", "reason": None}} and st["answers_dropped"] == [] + rec = json.loads(json.dumps(st["recipe"], ensure_ascii=False)) + for sheet in rec["sheets"]: + for block in sheet["blocks"]: + if isinstance(block.get("values"), dict): + block["values"]["placeholders"] = [] + resp = await client.put(f"{API}/imports/{st['id']}/recipe", json={"recipe": rec}) + assert resp.status_code == 200, resp.text + st = resp.json() + dropped = {d["id"]: d for d in st["answers_dropped"]} + assert qid in dropped and dropped[qid]["text"] and dropped[qid]["reason"] in ("changed", "gone") + assert qid not in st["answers"] + + +# ========================================================================== +# 框选转锚点（4.4、4.6） +# ========================================================================== + + +async def test_c01_selection_preview_matches_the_rule_draft_and_applies(client): + """lab.c01 暂存后在「月报!C5:F8」框选为列表：预览 ok，锚点是表头，重放与框一致，换算后的配方与规则草稿逐字相同 + （哈希不变）；框到第 7 行时说不清下边界，照样 200，ok=false、problems 写原因；应用后修改记录是 selection。""" + from tests.fixtures.xlsx import lab + + raw, fn = lab.c01("literal") + st = (await stage(client, unique(), raw, fn)).json() + sel = {"sheet": "月报", "ref": "C5:F8", "as": "list", "options": {"header_rows": 1, "bottom": "box"}} + resp = await preview(client, st["id"], {"selection": sel, "seq": 1}) + assert resp.status_code == 200, resp.text + pv = resp.json() + assert pv["ok"] is True and pv["kind"] == "selection" and pv["seq"] == 1 + assert [a["text"] for a in pv["anchors"] if a["kind"] == "header"] == ["地区", "产品", "销量", "金额"] + assert pv["replay"]["match"] is True, pv["replay"] + assert pv["replay"]["expected"] == {"header": "C5:F5", "data": "C6:F8", "total": "C9:F9"} + assert pv["recipe_sha256_after"] == st["recipe_sha256"], "框选得到的规则与起草器完全一样" + assert pv["breaking"] == {} and pv["expected"] == pv["replay"]["expected"] + bad = (await preview(client, st["id"], {"selection": {**sel, "ref": "C5:F7"}})).json() + assert bad["ok"] is False and [p["code"] for p in bad["problems"]] == ["selection_bottom_not_anchorable"] + assert bad["breaking"] == {} and bad["recipe_problems"] == [] and bad["dry_run"] is None + resp = await apply(client, st["id"], {"selection": {**sel, "ref": "C5:F7"}}, None) + assert resp.status_code == 422 and resp.json()["code"] == "edit_not_applicable" + resp = await apply(client, st["id"], {"selection": sel}, pv["recipe_sha256_after"]) + assert resp.status_code == 200, resp.text + st = resp.json() + [edit] = st["edits"] + assert edit["kind"] == "selection" and edit["key"].startswith("list:") and edit["undoable"] is True + # 修改记录记下选区（键是 as，Selection.to_json）；补丁只进导入清单，StagingOut 不给 + assert edit["selection"] == sel and "ops" not in edit + row = await db_get(ImportStaging, st["id"]) + assert row.edits[0]["ops"] and row.edits[0]["selection"] == sel + + +# ========================================================================== +# 评审后补：撤销不丢回答、预览的 breaking、采用重新起草之后的来源、重新起草没得到配方、带接受的重传、退役链 +# ========================================================================== + + +async def test_undo_keeps_the_answers_given_before_the_edit(client): + """先回答（导入模式选每期替换）、再用修复按钮、再撤销：撤销只撤这一次修改，修改前的回答照旧、不报「需要重新 + 回答」，工作配方回到修改前（3.1）。原来撤销恢复的是还没叠加回答的起点，有效果的回答一律判 changed 丢掉，导入 + 模式悄悄退回草稿默认的按期累积（评审意见）。""" + raw, fn = DRIFT_CASES["D21"].build(331) + resp = await stage(client, unique(), raw, fn) + assert resp.status_code == 201, resp.text + st = resp.json() + resp = await client.post(f"{API}/imports/{st['id']}/answers", + json={"answers": answers_for(st["questions"], "replace")}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["recipe"].get("mode", "replace") == "replace" + before_sha, before_answers = st["recipe_sha256"], st["answers"] + assert before_answers["q_mode"] == {"value": "replace", "reason": None} + fx = fix_of(st, "ignore_cells") + opt = fx["options"][0] + body = {"fix": {"id": fx["id"], "option": opt["value"], + **({"reason": "合成理由：表下另有补录行"} if opt["needs_reason"] else {})}} + pv = (await preview(client, st["id"], body)).json() + assert pv["ok"], pv["problems"] + resp = await apply(client, st["id"], body, pv["recipe_sha256_after"]) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["answers"] == before_answers and st["answers_dropped"] == [] + resp = await client.post(f"{API}/imports/{st['id']}/edits/undo", json={}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["edits"] == [] and st["recipe_sha256"] == before_sha + assert st["answers"] == before_answers and st["answers_dropped"] == [] + assert st["recipe"].get("mode", "replace") == "replace" + + +async def test_preview_breaking_lists_the_table_changes_of_a_breaking_option(client): + """预览的 breaking 是 table_changes 算出的「表名 → 人话列表」（WP-0 交接）：D09 改分段标题时另选新词（pick，常量 + 取值变了），「时段客流」列出取值的变化；沿用原词（keep）为空字典。只断言「是字典」抓不住漏算（评审意见）。""" + base = await first_import(client, unique(), seed=336) + raw, fn = DRIFT_CASES["D09"].build(336) + st = await reupload(client, base["source_id"], raw, fn) + fx = fix_of(st, "rename_title") + pick = next(o for o in fx["options"] if "|pick:" in o["value"]) + keep = next(o for o in fx["options"] if o["value"].endswith("|keep")) + assert pick["breaking"] is True and keep["breaking"] is False + resp = await preview(client, st["id"], {"fix": {"id": fx["id"], "option": pick["value"]}}) + assert resp.status_code == 200, resp.text + pv = resp.json() + assert pv["ok"], pv["problems"] + word = pick["value"].split("|pick:", 1)[1] + assert list(pv["breaking"]) == ["时段客流"], pv["breaking"] + assert any("日间" in line and word in line for line in pv["breaking"]["时段客流"]), pv["breaking"] + pv = (await preview(client, st["id"], {"fix": {"id": fx["id"], "option": keep["value"]}})).json() + assert pv["ok"] and pv["breaking"] == {} + + +async def test_an_adopted_redraft_is_stored_as_rules_and_next_month_does_not_ask_again(client): + """采用按规则重新起草的配方并提交：配方记录、导入清单记 rules（只取 rules / ai / manual / mixed，界面按 + RECIPE_ORIGIN_LABEL 显示）。下个月照常上传同一版式：暂存区的来源是 rules，不再出 redraft_adopted，也不再列出全部 + 配方类项（6.2、6.3：只有在本暂存区采用时才出）。原来 rules_redraft 原样入库，之后每个月都要再勾一次（评审意见）。""" + name = unique() + base = await first_import(client, name, seed=337, mode="replace") + source_id = base["source_id"] + raw, fn = DRIFT_CASES["D04"].build(337) + st = await reupload(client, source_id, raw, fn) + out = (await client.post(f"{API}/imports/{st['id']}/redraft-rules", json={})).json() + assert out["aligned_recipe"] is not None + resp = await client.put(f"{API}/imports/{st['id']}/recipe", + json={"recipe": out["aligned_recipe"], "origin": "rules_redraft"}) + assert resp.status_code == 200, resp.text + assert resp.json()["recipe_origin"] == "rules_redraft" + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert "redraft_adopted" in confirms(st), "本暂存区采用了重新起草：照旧出 redraft_adopted" + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + card = await source_row(client, source_id) + assert card["current_recipe"]["origin"] == "rules" + imp = await db_get(TableImport, resp.json()["import_id"]) + assert artifact_store.load(imp.manifest_artifact)["recipe"]["origin"] == "rules" + + raw, fn = DRIFT_CASES["D04"].build(338) + nxt = await reupload(client, source_id, raw, fn) + assert nxt["recipe_origin"] == "rules" + t = nxt["trial"] + assert t["status"] == "passed", t["problems"] + ids = confirms(nxt) + assert "redraft_adopted" not in ids, ids + assert [i["id"] for i in t["confirm_items"] if i.get("source", "recipe") == "recipe"] == [], ids + + +async def test_a_stored_rules_redraft_origin_is_read_as_rules_for_the_next_staging(client): + """防住已经入库的 rules_redraft（修复之前提交的配方记录）：上传新一期、修改配方给暂存区赋来源时同样映射成 rules， + 不出 redraft_adopted。""" + from sqlalchemy import update + + from app.db.models import TableRecipe + + base = await first_import(client, unique(), seed=344, mode="replace") + async with SessionLocal() as session: + await session.execute(update(TableRecipe).where(TableRecipe.id == base["commit"]["recipe_id"]) + .values(origin="rules_redraft")) + await session.commit() + raw, fn = flow_workbook(SEP, 30, seed=345) + st = await reupload(client, base["source_id"], raw, fn) + assert st["recipe_origin"] == "rules" and st["trial"]["status"] == "passed", st["trial"]["problems"] + assert "redraft_adopted" not in confirms(st) + resp = await client.delete(f"{API}/imports/{st['id']}") + assert resp.status_code in (200, 204), resp.text + resp = await client.post(f"{API}/{base['source_id']}/redraft", json={}) + assert resp.status_code == 201, resp.text + assert resp.json()["recipe_origin"] == "rules" + + +async def test_redraft_rules_without_a_recipe_leaves_alignment_empty(client): + """规则起草没得到配方时，失败原因只在 draft.failures；alignment 是「名字对齐」的报告，为空（界面把它放在那个标题下， + 抄一份过去同一句话就出现两次，check-manage 伪造的也是 []，评审意见）。""" + import io + + import openpyxl + + base = await first_import(client, unique(), seed=339) + wb = openpyxl.Workbook() + ws = wb.active + ws.title = "客流汇总" + ws["A1"] = "说明文字甲" + ws["A3"] = "说明文字乙" + buf = io.BytesIO() + wb.save(buf) + st = await reupload(client, base["source_id"], buf.getvalue(), "说明.xlsx") + resp = await client.post(f"{API}/imports/{st['id']}/redraft-rules", json={}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["draft"]["recipe"] is None and out["draft"]["failures"] + assert out["aligned_recipe"] is None and out["compare"] is None and out["alignment"] == [] + + +async def test_same_file_again_with_an_acceptance_is_a_new_import(client): + """带接受的重传照常新建导入记录，接受要重写（第 5 节：unchanged 只适用于没有接受的情况）。回执旁给出上次（没作废的） + 接受理由作参考，不预填。""" + name = unique() + base = await first_import(client, name, seed=340) + source_id = base["source_id"] + raw, fn = DRIFT_CASES["D26"].build(340) + st = await reupload(client, source_id, raw, fn) + t = st["trial"] + assert t["status"] == "needs_decision", t["problems"] + first = await commit(client, st, acceptances=[{"check_id": c, "reason": "合成理由甲"} for c in t["acceptable"]]) + assert first.status_code == 201, first.text + st = await reupload(client, source_id, raw, fn) + t = st["trial"] + assert t["accumulate"]["action"] == "replace_period" and t["same_as_import"]["id"] == first.json()["import_id"] + assert [a["reason"] for a in t["prior_acceptances"]] == ["合成理由甲"] * len(t["acceptable"]) + resp = await commit(client, st, acceptances=[{"check_id": c, "reason": "合成理由乙"} for c in t["acceptable"]]) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["unchanged"] is False and out["import_id"] != first.json()["import_id"] and out["parts"] == 2 + imp = await db_get(TableImport, out["import_id"]) + assert [o["reason"] for o in imp.overrides or []] + [w["reason"] for w in imp.waivers or []] \ + == ["合成理由乙"] * len(t["acceptable"]) + async with SessionLocal() as session: + n = len(list((await session.execute(select(TableImport).where(TableImport.source_id == source_id))).scalars())) + assert n == 3 + + +def _drop_total_table(recipe: dict[str, Any]) -> dict[str, Any]: + """合计行改为「不另存」（keep_as 置空），「时段客流_表内合计」从目标配方里去掉。""" + rec = json.loads(json.dumps(recipe, ensure_ascii=False)) + n = 0 + for sheet in rec["sheets"]: + for block in sheet["blocks"]: + for seg in block.get("segments", []): + if seg.get("keep_as"): + seg["keep_as"] = None + n += 1 + rec["tables"] = [t for t in rec["tables"] if t["name"] != "时段客流_表内合计"] + assert n + return rec + + +async def test_a_table_retired_once_is_not_retired_again_next_month(client): + """退役链（2.5）：9 月去掉合计表（必勾 retire:时段客流_表内合计，早期各期保留原值）；10 月同一配方，这张表已是 + 「早已退役」（快照清单 columns 的 status），不再出 retire:*。试运行要把快照清单的退役记录传给分档，漏传就每个月 + 都再报一次退役（评审意见：变异「不传 retired_status」没有测试抓到）。""" + name = unique() + base = await first_import(client, name, seed=341) + source_id = base["source_id"] + raw, fn = flow_workbook(SEP, 30, seed=342) + st = await reupload(client, source_id, raw, fn) + resp = await client.put(f"{API}/imports/{st['id']}/recipe", json={"recipe": _drop_total_table(st["recipe"])}) + assert resp.status_code == 200, resp.text + assert resp.json()["recipe_problems"] == [] + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + plan = st["trial"]["accumulate"] + assert plan["change"] == "retire" and plan["retired_new"] == [{"table": "时段客流_表内合计", "column": None}] + assert "retire:时段客流_表内合计" in confirms(st) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"]) + columns = artifact_store.load(snap.manifest_artifact)["columns"]["时段客流_表内合计"] + assert columns and all(c["status"] == "retired" and c["since"] == "2026-09-01~2026-09-30" for c in columns.values()) + assert await count(client, name, "时段客流_表内合计") == 93 + + raw, fn = flow_workbook(OCT, 31, seed=343) + st = await reupload(client, source_id, raw, fn) + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert t["accumulate"]["retired_new"] == [] + assert t["accumulate"]["retired_existing"] == [{"table": "时段客流_表内合计", "column": None}] + assert not [i for i in confirms(st) if i.startswith("retire")], confirms(st) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert await count(client, name, "时段客流_表内合计") == 93 + + +async def test_appending_after_removing_the_only_period_with_a_column_asks_no_phantom_retire(client): + """试运行路径的幽灵退役项（WP-8 修补，WP-5 交付说明「需要你处理」第 2 条）：8 月 R0；9 月 D13 修复加「分区丙」（R1）； + 移除 9 月之后当前版本按 R1 物化成单期并集，「分区丙」是一列全为空值的目标列；10 月修复去掉分区丙（R2）追加。 + 结果各期（8 月、10 月）和当前版本的任何一期都没有分区丙的数据，不该出 retire:日客流.分区丙，并集里也不再有这一列。 + 改回「把未筛选的当前表结构传给累积计划」，这里会多出一条任何一期都没有数据的必勾退役项。""" + from tests.test_source_versions_api import _fix_until_passed + + name = unique() + base = await first_import(client, name, seed=351) + sid = base["source_id"] + raw, fn = DRIFT_CASES["D13"].build(351) + st = await _fix_until_passed(client, await reupload(client, sid, raw, fn)) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + sep, s2 = resp.json()["import_id"], resp.json()["snapshot_id"] + resp = await client.post(f"{API}/{sid}/imports/{sep}/remove", + json={"confirm": True, "expected_current_snapshot_id": s2, "reason": "合成理由"}) + assert resp.status_code == 200, resp.text + s3 = await db_get(SourceSnapshot, resp.json()["snapshot_id"]) + assert Path(s3.db_path).parent.name == "snapshots", "8 月的配方不是目标配方：按 R1 物化成单期并集" + assert "分区丙" in [c["name"] for c in s3.schema_cache["tables"]["日客流"]["columns"]], "目标列照旧在，全为空值" + + raw, fn = flow_workbook(OCT, 31, seed=352) + st = await _fix_until_passed(client, await reupload(client, sid, raw, fn)) + t = st["trial"] + assert t["status"] == "passed", (t["problems"], st["recipe_problems"]) + assert t["accumulate"]["action"] == "append" + assert {"table": "日客流", "column": "分区丙"} not in t["accumulate"]["retired_new"], t["accumulate"] + assert {"table": "日客流", "column": "分区丙"} not in t["accumulate"]["retired_existing"], t["accumulate"] + assert not [i for i in confirms(st) if i.startswith("retire")], confirms(st) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"]) + assert "分区丙" not in [c["name"] for c in snap.schema_cache["tables"]["日客流"]["columns"]] + assert "分区丙" not in artifact_store.load(snap.manifest_artifact)["columns"].get("日客流", {}) + assert await count(client, name, "日客流") == 62 + + + +async def test_a_target_column_without_data_is_not_reported_as_added_again(client): + """WP-8 修补的另一半：筛当前表结构时，目标配方里有的列照旧留着。8 月 R0；9 月 D13 修复加「分区丙」（R1）；移除 9 月之后 + 当前版本按 R1 物化，分区丙是全为空值的目标列；10 月的文件又有分区丙、按 R1 直接重放：当前版本本来就有这一列，计划里 + 不该再报「新增列」（2.5 第 2 步按当前版本实际的表结构比）。只按各期配方筛、不留目标配方的列，这里会多出一条新增。""" + from tests.test_source_versions_api import _fix_until_passed + + name = unique() + base = await first_import(client, name, seed=361) + sid = base["source_id"] + raw, fn = DRIFT_CASES["D13"].build(361) + st = await _fix_until_passed(client, await reupload(client, sid, raw, fn)) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + sep, s2 = resp.json()["import_id"], resp.json()["snapshot_id"] + resp = await client.post(f"{API}/{sid}/imports/{sep}/remove", + json={"confirm": True, "expected_current_snapshot_id": s2, "reason": "合成理由"}) + assert resp.status_code == 200, resp.text + raw, fn = flow_workbook(OCT, 31, seed=362, variant="D13") + st = await reupload(client, sid, raw, fn) + t = st["trial"] + assert t["status"] == "passed", (t["problems"], st["recipe_problems"]) + assert t["accumulate"]["action"] == "append" + assert t["accumulate"]["added"] == [] and t["accumulate"]["retired_new"] == [], t["accumulate"] + assert not [d for d in t["diff"] or [] if d["kind"] == "columns_added"], t["diff"] + + +async def test_replacing_the_only_period_with_a_column_asks_breaking_not_a_phantom_retire(client): + """替换该期的幽灵退役项（WP-8 评审意见 2）：8 月 R0；9 月 D13 修复加「分区丙」（R1）后提交；再上传不含分区丙的 9 月 + （修复去掉分区丙后为 R2），替换该期。结果各期（8 月 R0、新 9 月 R2）都没有分区丙：不出 retire:*（文案写「早期各期 + 保留原值」，可哪一期都没有这一列），改出必勾的 breaking:日客流（列从现行配方里消失，仍要人确认，不会变成静默）； + 提交后并集里没有分区丙，也没有全为空值的列。改回「把要被替换的那一期也算进结果各期」，这里会出 retire:日客流.分区丙， + 并集多出一列全空的分区丙。""" + import sqlite3 + from contextlib import closing + + from tests.test_source_versions_api import _fix_until_passed + + name = unique() + base = await first_import(client, name, seed=361) + sid = base["source_id"] + raw, fn = DRIFT_CASES["D13"].build(361) + st = await _fix_until_passed(client, await reupload(client, sid, raw, fn)) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + sep_r1 = resp.json()["import_id"] + + raw, fn = flow_workbook(SEP, 30, seed=362) + st = await _fix_until_passed(client, await reupload(client, sid, raw, fn)) + t = st["trial"] + assert t["status"] == "passed", (t["problems"], st["recipe_problems"]) + assert t["accumulate"]["action"] == "replace_period" and t["accumulate"]["replaces"]["import_id"] == sep_r1 + assert {"table": "日客流", "column": "分区丙"} not in t["accumulate"]["retired_new"], t["accumulate"] + assert {"table": "日客流", "column": "分区丙"} not in t["accumulate"]["retired_existing"], t["accumulate"] + ids = confirms(st) + assert not [i for i in ids if i.startswith("retire")], ids + assert "breaking:日客流" in ids and "period_replace:2026-09-01~2026-09-30" in ids, ids + resp = await commit(client, st, confirmations=[i for i in ids if i != "breaking:日客流"]) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required", resp.text + assert "breaking:日客流" in resp.json()["detail"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"]) + assert "分区丙" not in [c["name"] for c in snap.schema_cache["tables"]["日客流"]["columns"]] + assert "分区丙" not in artifact_store.load(snap.manifest_artifact)["columns"].get("日客流", {}) + empty = [] + with closing(sqlite3.connect(f"file:{snap.db_path}?mode=ro", uri=True)) as conn: + for table, info in snap.schema_cache["tables"].items(): + for col in info["columns"]: + n = conn.execute(f'SELECT COUNT("{col["name"]}") FROM "{table}"').fetchone()[0] + if n == 0: + empty.append(f"{table}.{col['name']}") + assert empty == [], f"并集里不该有全为空值的列：{empty}" + assert await count(client, name, "日客流") == 61 diff --git a/backend/tests/test_table_versions.py b/backend/tests/test_table_versions.py index 50b8730a..f21211ef 100644 --- a/backend/tests/test_table_versions.py +++ b/backend/tests/test_table_versions.py @@ -465,8 +465,13 @@ async def test_new_source_is_not_created_when_publishing_fails(store, monkeypatc assert not files_under(store / "uploads" / "raw") -async def test_existing_target_with_wrong_hash_fails_and_keeps_pointer(store): - """同一个构建的目标文件已存在、哈希和登记的不一样（0 字节残骸、被改过）：导入失败，指针不动。""" +def quarantined(store: Path, source_id: str) -> list[Path]: + return files_under(store / "uploads" / "quarantine" / source_id) + + +async def test_existing_target_with_wrong_hash_is_restored_and_quarantined(store): + """同一个构建的目标文件已存在、哈希和登记的不一样（0 字节残骸、被改过）：这次的临时库哈希等于登记值， + 就把已有文件挪进隔离区、用临时库恢复（期 3 遗留项 1；期 1 这里一律失败，同一个文件永远传不上去）。""" name = unique() raw = xlsx(V1) sid, first = await publish(name, raw=raw) @@ -474,15 +479,43 @@ async def test_existing_target_with_wrong_hash_fails_and_keeps_pointer(store): path = Path(build.db_path) os.chmod(path, 0o644) path.write_bytes(b"") - with pytest.raises(table_versions.PublishError, match="不一致"): + _, second = await publish(name, raw=raw) + assert second.build_reused is True and second.build_restored is True + assert second.build_id == first.build_id + assert (await load(sid)).current_snapshot_id == second.snapshot_id + # 恢复的是登记的那一份：旧快照、新快照指着同一个文件，哈希都对得上 + assert raw_store.sha256_file(path) == build.db_sha256 == (await get(TableBuild, first.build_id)).db_sha256 + assert stat.S_IMODE(path.stat().st_mode) == 0o444 + [moved] = quarantined(store, sid) + assert moved.name.startswith(f"{first.build_id}-") and moved.stat().st_size == 0 + assert stat.S_IMODE(moved.stat().st_mode) == 0o444 + assert await scalar(await resolve(sid), 'SELECT SUM("数量") FROM "甲表"') == 3 + + +async def test_existing_target_and_record_both_wrong_is_a_build_conflict(store): + """文件和构建记录的哈希都被改了，本次上传也恢复不了：build_conflict，指针不动，什么都不挪。""" + name = unique() + raw = xlsx(V1) + sid, first = await publish(name, raw=raw) + async with SessionLocal() as session: + row = await session.get(TableBuild, first.build_id) + row.db_sha256 = "f" * 64 + await session.commit() + path = Path((await get(TableBuild, first.build_id)).db_path) + os.chmod(path, 0o644) + path.write_bytes(b"") + with pytest.raises(table_versions.PublishError, match="无法用本次上传恢复") as err: await publish(name, raw=raw) + assert err.value.code == "build_conflict" assert (await load(sid)).current_snapshot_id == first.snapshot_id assert len(await imports_of(sid)) == 1 assert path.exists() and path.stat().st_size == 0 # 不是我们建的，不替它删 - assert raw_store.raw_exists(build.raw_sha256) # 原件早就在，不能被这次失败删掉 + assert not quarantined(store, sid) + assert raw_store.raw_exists((await get(TableBuild, first.build_id)).raw_sha256) -async def test_zero_byte_file_at_target_without_record_fails(store): +async def test_unregistered_file_at_target_is_quarantined_and_publish_succeeds(store): + """目标位置有一个没有记录、内容不同的文件（没人登记的孤儿）：挪进隔离区再 link，发布成功。""" sid = uuid.uuid4().hex raw = xlsx(V1) opts = table_versions.parse_options("客流.xlsx", header_row=1, mixed="reject", raw_mode=False) @@ -492,10 +525,13 @@ async def test_zero_byte_file_at_target_without_record_fails(store): name = unique() async with SessionLocal() as session: row = DataSource(id=sid, name=name, kind="sqlite", readonly=True, origin="upload") - with pytest.raises(table_versions.PublishError): - await table_versions.publish_upload(session, row, raw, "客流.xlsx") - assert await get(DataSource, sid) is None - assert not files_under(store / "uploads" / "raw") + result = await table_versions.publish_upload(session, row, raw, "客流.xlsx") + assert result.build_reused is False and result.build_restored is False + assert (await load(sid)).current_snapshot_id == result.snapshot_id + assert raw_store.sha256_file(target) == (await get(TableBuild, result.build_id)).db_sha256 + [moved] = quarantined(store, sid) + assert moved.stat().st_size == 0 + assert await scalar(await resolve(sid), 'SELECT SUM("数量") FROM "甲表"') == 3 async def test_reupload_probes_the_new_file_even_when_the_old_engine_is_cached(store): @@ -717,7 +753,10 @@ async def test_reusing_a_build_with_an_empty_hash_fills_it_in(store): # -------------------------------------------------------------------------- -async def test_old_versions_are_reclaimed_when_nothing_references_them(store): +async def test_old_versions_are_reclaimed_when_nothing_references_them(store, monkeypatch): + # 期 3 起回收在当前快照、运行引用的之外另外保留每个源最近 SNAPSHOT_KEEP 个快照；调成 0（期 1 的规则） + # 才测得出「没人引用就回收」 + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0) name = unique() sid, first = await publish(name, fresh(V1)) old = await get(TableBuild, first.build_id) @@ -734,7 +773,9 @@ async def test_old_versions_are_reclaimed_when_nothing_references_them(store): await resolve(sid, first.snapshot_id) -async def test_gc_keeps_snapshots_referenced_by_interrupted_runs(store): +async def test_gc_keeps_snapshots_referenced_by_interrupted_runs(store, monkeypatch): + # 同上：SNAPSHOT_KEEP 调成 0，保护只来自运行引用，运行记录删掉以后才会被回收 + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0) name = unique() sid, first = await publish(name, fresh(V1)) async with SessionLocal() as session: @@ -951,7 +992,9 @@ async def test_startup_leaves_other_sqlite_sources_alone(store, tmp_path): assert sqlite3.connect(nested / "x.db").execute('SELECT COUNT(*) FROM "明细"').fetchone()[0] == 2 -async def test_legacy_upload_can_be_updated_by_a_new_upload(store): +async def test_legacy_upload_can_be_updated_by_a_new_upload(store, monkeypatch): + # 同上：SNAPSHOT_KEEP 调成 0，迁移出的 v0 版本没人引用，随回收删除 + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0) tables = table_versions.tables_root() tables.mkdir(parents=True) legacy = tables / "old_c.db" diff --git a/backend/tests/test_table_versions_p2.py b/backend/tests/test_table_versions_p2.py index 8602ef7d..bacd95e6 100644 --- a/backend/tests/test_table_versions_p2.py +++ b/backend/tests/test_table_versions_p2.py @@ -301,6 +301,32 @@ def test_recipe_build_options_shape(): opts = table_versions.recipe_build_options("b" * 64, {}) assert opts == {"kind": "recipe", "recipe_sha256": "b" * 64, "parser_ver": recipe_parsers.PARSER_VER, "inputs": {}} + assert table_versions.recipe_build_options("b" * 64, {}, semantics={}) == opts + + +def test_engine_semantics_marks_only_lists_that_skip_blank_rows(): + """执行器语义标记（WP-8 评审意见 1）：只有含 blank_rows=skip 列表块的配方带 list_note_after_blank，构建 id 和 + 构建选项随之变；交叉表、stop 模式的列表不带，构建 id 与期 2 逐字相同（不升 RECIPE_ENGINE_VER 的前提）。""" + def recipe(**rows: Any) -> recipe_types.Recipe: + return recipe_types.Recipe.model_validate({ + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": "明细"}, "blocks": [{ + "id": "列表1", "layout": "list", "table": "明细", + "columns": [{"header": "地区", "name": "地区", "type": "TEXT"}, + {"header": "金额", "name": "金额", "type": "INTEGER"}], + "rows": rows}]}], + "tables": [{"name": "明细", "grain": ["地区"]}]}) + + mark = {table_versions.LIST_NOTE_AFTER_BLANK: 1} + assert table_versions.recipe_engine_semantics(recipe(blank_rows="skip")) == mark + assert table_versions.recipe_engine_semantics(recipe()) == {} + assert table_versions.recipe_engine_semantics(recipe(blank_rows="stop")) == {} + assert table_versions.recipe_engine_semantics(None) == {} + base = table_versions.recipe_build_id("s1", "a" * 64, "b" * 64, {}) + assert table_versions.recipe_build_id("s1", "a" * 64, "b" * 64, {}, semantics={}) == base + assert table_versions.recipe_build_id("s1", "a" * 64, "b" * 64, {}, semantics=mark) != base + opts = table_versions.recipe_build_options("b" * 64, {}, semantics=mark) + assert opts["engine_semantics"] == mark def test_public_names_keep_the_old_aliases(): @@ -445,8 +471,10 @@ async def test_wal_trial_settled_before_hashing_matches_at_publish(store): assert info.value.code == "trial_tampered" -async def test_reused_build_reports_the_existing_file_hash(store): +async def test_reused_build_reports_the_existing_file_hash(store, monkeypatch): """同一个构建 id 再发布：复用已有文件，PublishInfo.db_sha256 是已有文件的，不是这次试运行库的。""" + # 期 3 起回收另外保留最近 SNAPSHOT_KEEP 个快照；调成 0（期 1、期 2 的规则）才测得出「被替换的那一期随回收置 retired」 + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0) name, raw = unique(), raw_bytes() sid = uuid.uuid4().hex bid = table_versions.recipe_build_id(sid, hashlib.sha256(raw).hexdigest(), RECIPE_SHA, {}) @@ -682,8 +710,10 @@ async def test_open_staging_raw_survives_gc_and_startup(store): assert (await get(ImportStaging, sid)).status == "rejected" -async def test_open_staging_protects_a_raw_shared_with_a_replaced_import(store): +async def test_open_staging_protects_a_raw_shared_with_a_replaced_import(store, monkeypatch): """上传新一期被拒收、正在修改时，那份原件和一次旧导入相同：旧导入被回收，原件不能跟着删。""" + # 同上：SNAPSHOT_KEEP 调成 0，旧导入才会随替换被回收 + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0) name, raw = unique(), raw_bytes() src, first = await publish_trial(name, new_trial(uuid.uuid4().hex), raw) sha = hashlib.sha256(raw).hexdigest() diff --git a/backend/tests/test_table_versions_p3.py b/backend/tests/test_table_versions_p3.py new file mode 100644 index 00000000..13494004 --- /dev/null +++ b/backend/tests/test_table_versions_p3.py @@ -0,0 +1,1269 @@ +"""按期累积与版本页（期 3，WP-4）的版本底座：快照 id、多期发布、启用旧快照、回收、隔离区、单进程守卫。 + +要守住的几件事（P3-SPEC 2.1、2.6、2.9、2.10、7.2、第 8 节遗留项 1 与 5、11.3 的 WP-4 部分）： +- 累积快照的 id 带目标配方的哈希，替换模式、简单导入的 id 与期 2 逐字相同； +- publish_snapshot / activate_snapshot 在锁内核对 expected_current，对不上 base_changed、什么都不动； + 快照 id 已存在时复用（没回收）或复活（已回收：文件在就清标记，文件没了就用这次的临时文件 link 回去， + 哈希必须等于登记值）；任何一步失败都回滚、删掉这次新 link 的文件； +- 启用旧快照的每个错误码；遮罩列丢失要确认；查询时拿快照清单交叉核对库列里的哈希； +- 回收在当前快照、被运行引用的快照之外，另外保留每个源最近 SNAPSHOT_KEEP 个快照（前两类不占名额，已回收的也不占）； + 被运行引用的并集快照不回收；孤儿清理认得并集库和附属试运行库； +- 隔离区的三条清理规则，保留期从挪进隔离区的那一刻算；拿不到守卫时 startup 什么都不删、写入一律 StoreUnavailable。 + +夹具：8 月、9 月的期库用合成客流表经执行器建（假名、随机数，见 test_recipe_accumulate）；其余用 sqlite3 现造。 +""" +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import sqlite3 +import subprocess +import sys +import time +import uuid +from dataclasses import asdict +from datetime import timedelta +from pathlib import Path +from typing import Any + +import pytest +from sqlalchemy import select + +from app.core import artifact_store +from app.core.config import settings +from app.data import raw_store, recipe_accumulate as ra, recipe_types, table_versions +from app.data.engine import SnapshotTampered, engines, run_query +from app.db.base import SessionLocal, new_id, utcnow +from app.db.models import ( + DataSource, ImportStaging, Run, SnapshotActivation, SourceSnapshot, TableBuild, TableImport, TableRecipe, +) +from tests.test_recipe_accumulate import REF, _built + +P2_BASELINE = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "p2_baseline.json").read_text("utf-8")) + +ENGINE = recipe_types.RECIPE_ENGINE_VER +SHA_A, SHA_B = "a" * 64, "b" * 64 + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + """每个测试一个独立的数据目录：上传目录、原件、工件、隔离区都在里面。""" + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +# -------------------------------------------------------------------------- +# 工具 +# -------------------------------------------------------------------------- + + +def unique(prefix: str = "wp4") -> str: + return f"{prefix}_{uuid.uuid4().hex[:10]}" + + +def raw_bytes() -> bytes: + return f"合成原件 {uuid.uuid4().hex}".encode() + + +def trial_copy(src: str | Path, source_id: str) -> Path: + """把一份期库拷成这个源的试运行库（发布时从这里 link）。""" + path = table_versions.trial_db_path(source_id, uuid.uuid4().hex, uuid.uuid4().hex) + path.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(src, path) + os.chmod(path, 0o644) + return path + + +def make_db(path: Path, rows: list[tuple[int, str]]) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(path) + conn.execute('CREATE TABLE "t" ("n" INTEGER, "s" TEXT) STRICT') + conn.executemany('INSERT INTO "t" VALUES (?, ?)', rows) + conn.commit() + conn.close() + table_versions.settle_journal(path) + return path + + +async def plain_schema(view, info) -> dict[str, Any]: + return await table_versions.introspect_view(view, {}) + + +async def get(model, key): + async with SessionLocal() as session: + return await session.get(model, key) + + +async def imports_of(source_id: str) -> list[TableImport]: + async with SessionLocal() as session: + return list((await session.execute( + select(TableImport).where(TableImport.source_id == source_id).order_by(TableImport.seq))).scalars()) + + +async def activations(source_id: str) -> list[SnapshotActivation]: + async with SessionLocal() as session: + return list((await session.execute( + select(SnapshotActivation).where(SnapshotActivation.source_id == source_id) + .order_by(SnapshotActivation.created_at))).scalars()) + + +async def current(source_id: str) -> str | None: + return (await get(DataSource, source_id)).current_snapshot_id + + +async def count_rows(source_id: str, sql: str, pinned: str | None = None) -> Any: + async with SessionLocal() as session: + view = await table_versions.resolve_source(session, await session.get(DataSource, source_id), pinned) + try: + return (await run_query(view, sql)).rows[0][0] + finally: + await engines.invalidate(source_id) + + +async def gc() -> None: + async with SessionLocal() as session: + await table_versions.gc(session) + + +async def publish_small(name: str, rows: list[tuple[int, str]] | None = None, *, recipe_id: str | None = None, + snapshot_fields: dict | None = None, activation: dict | None = None, raw: bytes | None = None): + """经 publish_build 发布一期小库（表 t）。返回 (源 id, ImportResult)。raw 相同、rows 相同就是同一个构建。""" + raw = raw if raw is not None else raw_bytes() + raw_sha = hashlib.sha256(raw).hexdigest() + async with SessionLocal() as session: + row = (await session.execute(select(DataSource).where(DataSource.name == name))).scalar_one_or_none() + if row is None: + row = DataSource(id=uuid.uuid4().hex, name=name, kind="sqlite", readonly=True, origin="upload") + sid = row.id + tmp = make_db(table_versions.trial_db_path(sid, uuid.uuid4().hex, uuid.uuid4().hex), + rows or [(1, "分区甲"), (2, "分区乙")]) + result = await table_versions.publish_build( + session, row, tmp_db=tmp, build_id=table_versions.recipe_build_id(sid, raw_sha, SHA_A, {}), + build_options=table_versions.recipe_build_options(SHA_A, {}), engine_ver=ENGINE, + report={"tables": [{"name": "t"}]}, raw=raw, raw_sha=raw_sha, file_name="合成.xlsx", + file_size=len(raw), schema_cache_for=plain_schema, + import_fields={"recipe_id": recipe_id} if recipe_id else {}, expected_db_sha256=raw_store.sha256_file(tmp), + snapshot_fields=snapshot_fields, activation=activation) + return sid, result + + +# -------------------------------------------------------------------------- +# 8 月 + 9 月的累积链 +# -------------------------------------------------------------------------- + + +class Chain: + """一个累积源：S1 = [8 月]（publish_build，快照库就是 8 月的构建库），S2 = [8 月, 9 月]（publish_snapshot，并集）。""" + + def __init__(self) -> None: + self.name = unique() + self.sid = uuid.uuid4().hex + self.aug_path, self.aug_sha, self.aug_start, self.aug_end, hashes = _built("2026-08-01", 31) + self.aug_hashes = dict(hashes) + self.sep_path, self.sep_sha, self.sep_start, self.sep_end, hashes = _built("2026-09-01", 30) + self.sep_hashes = dict(hashes) + self.aug_raw, self.sep_raw = raw_bytes(), raw_bytes() + self.aug_bid = table_versions.recipe_build_id(self.sid, hashlib.sha256(self.aug_raw).hexdigest(), SHA_A, {}) + self.sep_bid = table_versions.recipe_build_id(self.sid, hashlib.sha256(self.sep_raw).hexdigest(), SHA_A, {}) + self.aug_import = self.sep_import = self.s1 = self.s2 = self.uid = None + self.manifest: str | None = None + + async def first(self) -> table_versions.ImportResult: + raw_sha = hashlib.sha256(self.aug_raw).hexdigest() + async with SessionLocal() as session: + row = DataSource(id=self.sid, name=self.name, kind="sqlite", readonly=True, origin="upload") + result = await table_versions.publish_build( + session, row, tmp_db=trial_copy(self.aug_path, self.sid), build_id=self.aug_bid, + build_options=table_versions.recipe_build_options(SHA_A, {}), engine_ver=ENGINE, + report={"tables": [], "table_hashes": self.aug_hashes}, raw=self.aug_raw, raw_sha=raw_sha, + file_name="8月.xlsx", file_size=len(self.aug_raw), schema_cache_for=plain_schema, + import_fields={"recipe_id": "rA", "period_start": self.aug_start, "period_end": self.aug_end}, + expected_db_sha256=self.aug_sha, + snapshot_fields={"mode": "accumulate", "recipe_id": "rA", "recipe_sha256": SHA_A}) + self.aug_import, self.s1 = result.import_id, result.snapshot_id + return result + + def union_parts(self, sep_db: str | Path) -> list[recipe_types.UnionPart]: + build = Path(table_versions.build_path(self.sid, self.aug_bid)) + return [ + recipe_types.UnionPart(import_id=self.aug_import, start=self.aug_start, end=self.aug_end, + db_path=str(build), recipe=REF, db_sha256=self.aug_sha, + table_hashes=self.aug_hashes), + recipe_types.UnionPart(import_id=None, start=self.sep_start, end=self.sep_end, db_path=str(sep_db), + recipe=REF, db_sha256=self.sep_sha, table_hashes=self.sep_hashes), + ] + + def union_build(self, utrial: Path, rep: recipe_types.UnionReport) -> table_versions.UnionBuild: + return table_versions.UnionBuild( + tmp_db=utrial, union_id=self.uid, + options=ra.union_options([(self.aug_start, self.aug_end, self.aug_bid), + (self.sep_start, self.sep_end, self.sep_bid)], SHA_A), + report=table_versions.report_to_json(asdict(rep)), expected_sha256=rep.db_sha256) + + async def schema_with_manifest(self, view, info: table_versions.SnapshotInfo) -> dict[str, Any]: + cache = await table_versions.introspect_view(view, {}) + self.manifest = await artifact_store.put_json( + {"format": "agentlab-snapshot-manifest/1", "snapshot_id": info.snapshot_id, + "union": {"build_id": info.union_id, "db_sha256": info.union_db_sha256}}, + kind="snapshot_manifest") + cache["snapshot_manifest"] = self.manifest + return cache + + async def second(self, *, expected: str | None = "S1", before_commit=None, drop_union_tmp: bool = False, + recipe_id: str = "rA") -> table_versions.SnapshotPublish: + trial = trial_copy(self.sep_path, self.sid) + utrial = table_versions.union_trial_path(trial) + rep = ra.materialize_union(utrial, target=REF, parts=self.union_parts(trial)) + assert rep.ok + self.uid = ra.union_id(self.sid, [(self.aug_start, self.aug_end, self.aug_bid), + (self.sep_start, self.sep_end, self.sep_bid)], SHA_A) + if drop_union_tmp: + utrial.unlink() + raw_store.put_raw(self.sep_raw) + self.sep_import = new_id() + raw_sha = hashlib.sha256(self.sep_raw).hexdigest() + self.sep_trial, self.union_trial = trial, utrial + async with SessionLocal() as session: + source = await session.get(DataSource, self.sid) + result = await table_versions.publish_snapshot( + session, source, imports=[self.aug_import, self.sep_import], mode="accumulate", + recipe_id=recipe_id, recipe_sha256=SHA_A, + expected_current=self.s1 if expected == "S1" else expected, + period_build=table_versions.PeriodBuild( + tmp_db=trial, build_id=self.sep_bid, options=table_versions.recipe_build_options(SHA_A, {}), + report={"tables": [], "table_hashes": self.sep_hashes}, expected_sha256=self.sep_sha), + new_import=table_versions.NewImport( + import_id=self.sep_import, raw_sha=raw_sha, file_name="9月.xlsx", file_size=len(self.sep_raw), + fields={"recipe_id": recipe_id, "period_start": self.sep_start, "period_end": self.sep_end}), + union=self.union_build(utrial, rep), schema_cache_for=self.schema_with_manifest, + before_commit=before_commit(session) if before_commit else None) + self.s2 = result.snapshot_id + return result + + async def publish_existing(self, imports: list[str], *, expected: str, union: bool, kind: str = "remove_period", + union_tmp: Path | None = None, recipe_id: str = "rA") -> table_versions.SnapshotPublish: + """不带新导入的发布（移除、作废）：union=False 时快照库就是唯一那一期的构建库；union=True 时按 8 月、9 月的 + 已发布构建重新物化（union_tmp 给了就拿它冒充这次物化的结果）。""" + ub = None + if union and union_tmp is None: + out = table_versions.union_trial_path( + table_versions.trial_db_path(self.sid, uuid.uuid4().hex, uuid.uuid4().hex)) + rep = ra.materialize_union(out, target=REF, parts=self.union_parts( + table_versions.build_path(self.sid, self.sep_bid))) + ub = self.union_build(out, rep) + elif union: + ub = table_versions.UnionBuild(tmp_db=union_tmp, union_id=self.uid, options={"kind": "union"}, report={}, + expected_sha256=raw_store.sha256_file(union_tmp)) + async with SessionLocal() as session: + source = await session.get(DataSource, self.sid) + return await table_versions.publish_snapshot( + session, source, imports=imports, mode="accumulate", recipe_id=recipe_id, recipe_sha256=SHA_A, + expected_current=expected, period_build=None, new_import=None, union=ub, + schema_cache_for=plain_schema, activation={"kind": kind, "import_id": self.sep_import, + "reason": "合成理由", "signed_by": "测试员"}) + + +async def chain() -> Chain: + c = Chain() + await c.first() + await c.second() + return c + + +# ========================================================================== +# id 与路径 +# ========================================================================== + + +def test_snapshot_id_for_replace_is_unchanged_and_accumulate_carries_the_recipe(): + ids = P2_BASELINE["ids"] + assert table_versions.snapshot_id(ids["source_id"], ids["import_ids"], ENGINE, mode="replace") \ + == ids["snapshot_id_recipe"] + assert table_versions.snapshot_id(ids["source_id"], ids["import_ids"], ENGINE, mode=None, recipe_sha=SHA_A) \ + == ids["snapshot_id_recipe"] + acc = table_versions.snapshot_id("s1", ["i1", "i2"], ENGINE, mode="accumulate", recipe_sha=SHA_A) + assert acc == table_versions.sha_json({"source_id": "s1", "imports": ["i1", "i2"], "engine_ver": ENGINE, + "mode": "accumulate", "recipe_sha256": SHA_A}) + assert acc != table_versions.snapshot_id("s1", ["i1", "i2"], ENGINE) + assert acc != table_versions.snapshot_id("s1", ["i1", "i2"], ENGINE, mode="accumulate", recipe_sha=SHA_B) + assert acc != table_versions.snapshot_id("s1", ["i2", "i1"], ENGINE, mode="accumulate", recipe_sha=SHA_A) + with pytest.raises(ValueError): + table_versions.snapshot_id("s1", ["i1"], ENGINE, mode="accumulate") + + +def test_union_paths_and_the_trial_union_file_name(store): + uid = "c" * 64 + assert table_versions.union_path("src1", uid) == store / "uploads" / "tables" / "src1" / "snapshots" / f"{uid}.db" + with pytest.raises(ValueError): + table_versions.union_path("../x", uid) + with pytest.raises(ValueError): + table_versions.union_path("src1", "not-a-hash") + trial = table_versions.trial_db_path("src1", "stg-1", "0123456789abcdef0123") + union = table_versions.union_trial_path(trial) + assert union.name == "stg-1-0123456789abcdefu.db" and union.parent == trial.parent + # 文件名里不加「-」：按最后一个「-」切出的仍是暂存区 id；附属文件认得回主文件 + assert table_versions._trial_owner(union) == "stg-1" + assert table_versions._trial_main(union) == trial and table_versions._trial_main(trial) == trial + + +def test_new_models_and_migrations_are_registered(): + from app.db import base as db_base + + migrated = {(t, c) for t, c, _ in db_base._COLUMN_MIGRATIONS} + assert {("source_snapshots", c) for c in ("mode", "recipe_id", "activated_at", "manifest_artifact")} <= migrated + assert {("table_imports", "revoked"), ("import_stagings", "edits"), ("import_stagings", "answers_dropped"), + ("import_stagings", "redraft_sha256")} <= migrated + assert SnapshotActivation.__tablename__ == "snapshot_activations" + assert set(SnapshotActivation.__table__.columns.keys()) == { + "id", "source_id", "snapshot_id", "previous_snapshot_id", "kind", "import_id", "reason", "signed_by", + "created_at"} + # 暂存区结束时期 3 的三个字段也清空 + assert {"edits", "answers_dropped", "redraft_sha256"} <= set(table_versions.STAGING_SLIM) + assert table_versions.SNAPSHOT_KEEP == 3 and table_versions.QUARANTINE_DAYS == 7 + + +def test_migration_adds_phase3_columns_to_an_old_database(tmp_path): + from sqlalchemy import create_engine, text + + from app.db import base as db_base + + path = tmp_path / "old.db" + conn = sqlite3.connect(path) + conn.execute("CREATE TABLE source_snapshots (id VARCHAR(64) PRIMARY KEY, source_id VARCHAR(32))") + conn.execute("INSERT INTO source_snapshots VALUES ('s1', 'src')") + conn.commit() + conn.close() + engine = create_engine(f"sqlite:///{path}") + try: + with engine.begin() as sync: + db_base.Base.metadata.create_all(sync) + db_base._migrate(sync) + row = sync.execute(text("SELECT mode, recipe_id, activated_at, manifest_artifact FROM source_snapshots") + ).one() + tables = {r[0] for r in sync.execute(text("SELECT name FROM sqlite_master WHERE type='table'"))} + assert tuple(row) == (None, None, None, None) and "snapshot_activations" in tables + finally: + engine.dispose() + + +# ========================================================================== +# publish_build 的期 3 字段 +# ========================================================================== + + +async def test_publish_build_records_mode_recipe_and_an_activation(store): + name = unique() + sid, first = await publish_small(name, recipe_id="r1") + snap = await get(SourceSnapshot, first.snapshot_id) + assert snap.mode == "replace" and snap.recipe_id == "r1" and snap.activated_at is not None + assert first.snapshot_id == table_versions.snapshot_id(sid, [first.import_id], ENGINE) + _, second = await publish_small(name, snapshot_fields={"mode": "accumulate", "recipe_id": "r2", + "recipe_sha256": SHA_B}, + activation={"kind": "commit", "signed_by": "测试员"}) + assert second.snapshot_id == table_versions.snapshot_id(sid, [second.import_id], ENGINE, mode="accumulate", + recipe_sha=SHA_B) + snap2 = await get(SourceSnapshot, second.snapshot_id) + assert snap2.mode == "accumulate" and snap2.recipe_id == "r2" + rows = await activations(sid) + assert [(a.kind, a.snapshot_id, a.previous_snapshot_id) for a in rows] == [ + ("commit", first.snapshot_id, None), ("commit", second.snapshot_id, first.snapshot_id)] + assert rows[0].import_id == first.import_id and rows[1].signed_by == "测试员" + assert second.build_restored is False + + +# ========================================================================== +# publish_snapshot +# ========================================================================== + + +async def test_publish_snapshot_appends_a_period_as_a_union(store): + c = Chain() + first = await c.first() + assert first.snapshot_id == table_versions.snapshot_id(c.sid, [c.aug_import], ENGINE, mode="accumulate", + recipe_sha=SHA_A) + res = await c.second() + assert res.snapshot_id == table_versions.snapshot_id(c.sid, [c.aug_import, c.sep_import], ENGINE, + mode="accumulate", recipe_sha=SHA_A) + assert (res.union_id, res.build_id, res.import_id) == (c.uid, c.sep_bid, c.sep_import) + assert not (res.build_reused or res.union_reused or res.snapshot_reused or res.snapshot_revived) + assert res.seq == 2 + snap = await get(SourceSnapshot, res.snapshot_id) + upath = table_versions.union_path(c.sid, c.uid) + assert snap.imports == [c.aug_import, c.sep_import] and snap.mode == "accumulate" and snap.recipe_id == "rA" + assert Path(snap.db_path) == upath and snap.db_sha256 == raw_store.sha256_file(upath) + assert snap.manifest_artifact == c.manifest and snap.schema_cache["snapshot_manifest"] == c.manifest + assert oct(upath.stat().st_mode & 0o777) == "0o444" + union = await get(TableBuild, c.uid) + assert union.engine_ver == recipe_types.UNION_VER and union.raw_sha256 == "" + assert union.options["kind"] == "union" and union.report["rows"]["日客流"] == 61 + assert (await get(TableBuild, c.sep_bid)).db_sha256 == c.sep_sha + assert [(i.id, i.status) for i in await imports_of(c.sid)] == [(c.aug_import, "active"), (c.sep_import, "active")] + sep = await get(TableImport, c.sep_import) + assert sep.period_start == "2026-09-01" and sep.recipe_id == "rA" and sep.raw_state == "kept" + row = await get(DataSource, c.sid) + assert row.current_snapshot_id == res.snapshot_id and row.current_recipe_id == "rA" + assert row.database == str(upath) + assert [(a.kind, a.previous_snapshot_id) for a in await activations(c.sid)][-1] == ("commit", c.s1) + assert await count_rows(c.sid, 'SELECT COUNT(*) FROM "日客流"') == 61 + # 固定在 S1 上的查询仍是 8 月那 31 行 + assert await count_rows(c.sid, 'SELECT COUNT(*) FROM "日客流"', c.s1) == 31 + # 临时文件都被消耗 + assert not c.sep_trial.exists() and not c.union_trial.exists() + + +async def test_publish_snapshot_refuses_a_stale_base(store): + c = Chain() + await c.first() + with pytest.raises(table_versions.PublishError) as err: + await c.second(expected="0" * 64) + assert err.value.code == "base_changed" and "刷新" in str(err.value) + assert await current(c.sid) == c.s1 + assert not table_versions.union_path(c.sid, c.uid).exists() + assert not table_versions.build_path(c.sid, c.sep_bid).exists() + assert not c.sep_trial.exists() and not c.union_trial.exists() + assert len(await imports_of(c.sid)) == 1 + + +async def test_publish_snapshot_rolls_back_when_before_commit_fails(store): + c = Chain() + await c.first() + + def failing(session): + async def hook(import_id, snap_id): + assert import_id == c.sep_import + assert await session.get(TableImport, import_id) is not None + # 锁内、指针还没切 + cur = (await session.execute(select(DataSource.current_snapshot_id) + .where(DataSource.id == c.sid))).scalar() + assert cur == c.s1 + raise RuntimeError("模拟提交前回调失败") + return hook + + with pytest.raises(RuntimeError, match="模拟"): + await c.second(before_commit=failing) + assert await current(c.sid) == c.s1 + assert not table_versions.union_path(c.sid, c.uid).exists() + assert not table_versions.build_path(c.sid, c.sep_bid).exists() + assert await get(TableBuild, c.uid) is None and await get(TableImport, c.sep_import) is None + assert [i.status for i in await imports_of(c.sid)] == ["active"] + + +async def test_publish_snapshot_needs_its_union_trial_file(store): + c = Chain() + await c.first() + with pytest.raises(table_versions.PublishError) as err: + await c.second(drop_union_tmp=True) + assert err.value.code == "trial_missing" + assert await current(c.sid) == c.s1 and not table_versions.build_path(c.sid, c.sep_bid).exists() + + +async def test_publish_snapshot_with_a_tampered_union_trial_is_refused(store): + c = Chain() + await c.first() + real = ra.materialize_union + + def tampering(out, **kw): + # 物化之后、提交之前，并集试运行库被人改了一行 + rep = real(out, **kw) + conn = sqlite3.connect(out) + conn.execute('UPDATE "日客流" SET "分区甲" = 0 WHERE rowid = 1') + conn.commit() + conn.close() + return rep + + ra.materialize_union = tampering + try: + with pytest.raises(table_versions.PublishError) as err: + await c.second() + finally: + ra.materialize_union = real + assert err.value.code == "trial_tampered" and await current(c.sid) == c.s1 + assert not table_versions.union_path(c.sid, c.uid).exists() + + +async def test_removing_the_latest_period_reuses_the_earlier_snapshot(store): + """8 月、9 月同一个配方：移除 9 月就是原来的 S1，直接切回去（配方、模式都不变）。""" + c = await chain() + res = await c.publish_existing([c.aug_import], expected=c.s2, union=False) + assert res.snapshot_id == c.s1 and res.snapshot_reused is True and res.snapshot_revived is False + assert res.import_id is None and res.union_id is None + row = await get(DataSource, c.sid) + assert row.current_snapshot_id == c.s1 and row.current_recipe_id == "rA" + assert [(i.id, i.status) for i in await imports_of(c.sid)] == [(c.aug_import, "active"), + (c.sep_import, "superseded")] + last = (await activations(c.sid))[-1] + assert (last.kind, last.snapshot_id, last.previous_snapshot_id) == ("remove_period", c.s1, c.s2) + assert (last.import_id, last.reason, last.signed_by) == (c.sep_import, "合成理由", "测试员") + assert await count_rows(c.sid, 'SELECT COUNT(*) FROM "日客流"') == 31 + + +async def test_retired_snapshot_whose_file_is_there_is_revived(store): + c = await chain() + async with SessionLocal() as session: + (await session.get(SourceSnapshot, c.s1)).retired_at = utcnow() + await session.commit() + res = await c.publish_existing([c.aug_import], expected=c.s2, union=False) + assert res.snapshot_id == c.s1 and res.snapshot_revived is True and res.snapshot_reused is False + snap = await get(SourceSnapshot, c.s1) + assert snap.retired_at is None and snap.activated_at is not None + assert await current(c.sid) == c.s1 + + +async def _back_to_s1_and_retire_s2(c: Chain) -> Path: + """切回 S1，再把 S2 和它的并集构建标成已回收、删掉并集文件（模拟回收）。返回并集路径。""" + await c.publish_existing([c.aug_import], expected=c.s2, union=False) + upath = table_versions.union_path(c.sid, c.uid) + async with SessionLocal() as session: + (await session.get(SourceSnapshot, c.s2)).retired_at = utcnow() + (await session.get(TableBuild, c.uid)).retired_at = utcnow() + await session.commit() + upath.unlink() + return upath + + +async def test_retired_union_snapshot_is_revived_by_relinking_a_rematerialized_file(store): + c = await chain() + registered = (await get(SourceSnapshot, c.s2)).db_sha256 + upath = await _back_to_s1_and_retire_s2(c) + res = await c.publish_existing([c.aug_import, c.sep_import], expected=c.s1, union=True, + kind="revoke_acceptance") + assert res.snapshot_id == c.s2 and res.snapshot_revived is True and res.union_reused is False + assert raw_store.sha256_file(upath) == registered + assert (await get(SourceSnapshot, c.s2)).retired_at is None + assert (await get(TableBuild, c.uid)).retired_at is None + assert await current(c.sid) == c.s2 + assert await count_rows(c.sid, 'SELECT COUNT(*) FROM "日客流"') == 61 + + +async def test_reviving_with_a_different_file_is_a_build_conflict(store, tmp_path): + c = await chain() + upath = await _back_to_s1_and_retire_s2(c) + bogus = table_versions.union_trial_path(table_versions.trial_db_path(c.sid, uuid.uuid4().hex, uuid.uuid4().hex)) + make_db(bogus, [(9, "不是这份")]) + with pytest.raises(table_versions.PublishError) as err: + await c.publish_existing([c.aug_import, c.sep_import], expected=c.s1, union=True, union_tmp=bogus) + assert err.value.code == "build_conflict" + assert not upath.exists() and not bogus.exists() # 这次 link 出来的删掉，临时文件消耗掉 + assert await current(c.sid) == c.s1 + assert (await get(SourceSnapshot, c.s2)).retired_at is not None + + +@pytest.mark.parametrize("field", ["recipe_id", "imports", "mode", "db_path"]) +async def test_existing_snapshot_with_other_content_is_a_conflict(store, field): + """快照 id 相同、记录里的配方、各期、模式、库路径有一样对不上：id 规则被破坏了，不能复用（每一项单独钉住）。""" + c = await chain() + async with SessionLocal() as session: + snap = await session.get(SourceSnapshot, c.s1) + if field == "recipe_id": + snap.recipe_id = "someone-else" + elif field == "imports": + snap.imports = [c.aug_import, "someone-else"] + elif field == "mode": + snap.mode = "replace" + else: + snap.db_path = str(Path(snap.db_path).with_name("elsewhere.db")) + await session.commit() + with pytest.raises(table_versions.PublishError) as err: + await c.publish_existing([c.aug_import], expected=c.s2, union=False) + assert err.value.code == "snapshot_conflict" and await current(c.sid) == c.s2 + + +@pytest.mark.parametrize("registered", [SHA_A, SHA_B]) +async def test_existing_snapshot_matches_on_recipe_content_not_record_id(store, registered): + """快照 id 按配方内容哈希算（WP-8 修补）：已有快照记的配方记录 rA 与这次传的 rB 不是同一条、内容相同时照样复用， + 现行配方跟着快照记的那条（rA，与 activate_snapshot 一致）；rA 登记的哈希与这次的不同才是 snapshot_conflict。 + 改回按记录 id 比，内容相同的那一半会误报冲突。""" + c = await chain() + ra_id, rb_id = new_id(), new_id() # 配方表在各测试间共用，记录 id 不能写死 + async with SessionLocal() as session: + session.add(TableRecipe(id=ra_id, source_id=c.sid, seq=1, recipe_sha256=registered, status="superseded")) + session.add(TableRecipe(id=rb_id, source_id=c.sid, seq=2, recipe_sha256=SHA_A, status="active")) + (await session.get(SourceSnapshot, c.s1)).recipe_id = ra_id + await session.commit() + if registered != SHA_A: + with pytest.raises(table_versions.PublishError) as err: + await c.publish_existing([c.aug_import], expected=c.s2, union=False, recipe_id=rb_id) + assert err.value.code == "snapshot_conflict" and await current(c.sid) == c.s2 + return + res = await c.publish_existing([c.aug_import], expected=c.s2, union=False, recipe_id=rb_id) + assert res.snapshot_id == c.s1 and res.snapshot_reused is True + row = await get(DataSource, c.sid) + assert row.current_snapshot_id == c.s1 and row.current_recipe_id == ra_id + assert (await get(SourceSnapshot, c.s1)).recipe_id == ra_id + assert (await get(TableRecipe, ra_id)).status == "active" and (await get(TableRecipe, rb_id)).status == "superseded" + + +async def test_reusing_a_snapshot_whose_record_disagrees_with_its_file_is_a_build_conflict(store): + """快照记录的库哈希被改了（文件、构建记录都没动）：不能把它切成当前版本，之后的查询会一律报篡改。""" + c = await chain() + async with SessionLocal() as session: + (await session.get(SourceSnapshot, c.s1)).db_sha256 = "0" * 64 + await session.commit() + with pytest.raises(table_versions.PublishError) as err: + await c.publish_existing([c.aug_import], expected=c.s2, union=False) + assert err.value.code == "build_conflict" and await current(c.sid) == c.s2 + + +async def test_reviving_when_the_union_record_disagrees_with_the_snapshot_is_a_build_conflict(store): + """复活时重新物化的文件等于快照记的哈希、却不等于并集构建登记的：两处登记对不上，以构建登记为准，不复活。""" + c = await chain() + upath = await _back_to_s1_and_retire_s2(c) + async with SessionLocal() as session: + (await session.get(TableBuild, c.uid)).db_sha256 = "0" * 64 + await session.commit() + with pytest.raises(table_versions.PublishError) as err: + await c.publish_existing([c.aug_import, c.sep_import], expected=c.s1, union=True, kind="revoke_acceptance") + assert err.value.code == "build_conflict" + assert not upath.exists() and await current(c.sid) == c.s1 + assert (await get(SourceSnapshot, c.s2)).retired_at is not None + assert (await get(TableBuild, c.uid)).db_sha256 == "0" * 64 + + +async def test_publishing_the_current_snapshot_again_is_already_current(store): + c = await chain() + with pytest.raises(table_versions.PublishError) as err: + await c.publish_existing([c.aug_import, c.sep_import], expected=c.s2, union=True) + assert err.value.code == "already_current" and await current(c.sid) == c.s2 + assert [a.kind for a in await activations(c.sid)] == ["commit", "commit"] + # 这次物化的临时文件照样被消耗 + assert not [p for p in table_versions._trial_files() if p.name.endswith("u.db")] + + +async def test_publish_snapshot_restores_a_tampered_union_file(store): + """回到 S1 之后 S2 的并集文件被改成 0 字节（修改时间还是一个月前）：撤销时重新物化出登记的那一份，恢复它， + 原文件挪进隔离区，并且挺过发布后紧接着的回收。""" + c = await chain() + registered = (await get(TableBuild, c.uid)).db_sha256 + await c.publish_existing([c.aug_import], expected=c.s2, union=False) + upath = table_versions.union_path(c.sid, c.uid) + os.chmod(upath, 0o644) + upath.write_bytes(b"") + month_ago = time.time() - 30 * 86400 + os.utime(upath, (month_ago, month_ago)) + res = await c.publish_existing([c.aug_import, c.sep_import], expected=c.s1, union=True, kind="revoke_acceptance") + assert res.snapshot_id == c.s2 and res.snapshot_reused is True + assert res.union_restored is True and res.build_restored is True and res.union_reused is True + assert raw_store.sha256_file(upath) == registered and oct(upath.stat().st_mode & 0o777) == "0o444" + [moved] = quarantined(c.sid) + assert moved.name.startswith(f"{c.uid}-") and moved.stat().st_size == 0 + assert await current(c.sid) == c.s2 + assert await count_rows(c.sid, 'SELECT COUNT(*) FROM "日客流"') == 61 + + +async def test_publish_snapshot_refuses_a_union_id_whose_layout_differs(store, tmp_path): + """同一个并集 id、表结构却不同（调用方算 id 时漏了退役列）：照常复用会把用户没核对过的那份发布出去（H2）， + 宁可 build_conflict。""" + c = await chain() + await c.publish_existing([c.aug_import], expected=c.s2, union=False) # 当前回到 S1，S2 还保留着 + out = table_versions.union_trial_path(table_versions.trial_db_path(c.sid, uuid.uuid4().hex, uuid.uuid4().hex)) + retired = {"日客流": [recipe_types.ColumnOut("分区丁", "INTEGER")]} + rep = ra.materialize_union(out, target=REF, parts=c.union_parts(table_versions.build_path(c.sid, c.sep_bid)), + retired=retired) + assert rep.ok and [x.name for x in rep.tables["日客流"]][-1] == "分区丁" + wrong = table_versions.UnionBuild(tmp_db=out, union_id=c.uid, options={"kind": "union"}, + report=table_versions.report_to_json(asdict(rep)), + expected_sha256=rep.db_sha256) + async with SessionLocal() as session: + source = await session.get(DataSource, c.sid) + with pytest.raises(table_versions.PublishError) as err: + await table_versions.publish_snapshot( + session, source, imports=[c.aug_import, c.sep_import], mode="accumulate", recipe_id="rA", + recipe_sha256=SHA_A, expected_current=c.s1, period_build=None, new_import=None, union=wrong, + schema_cache_for=plain_schema, activation={"kind": "revoke_acceptance"}) + assert err.value.code == "build_conflict" and "表结构" in str(err.value) + assert await current(c.sid) == c.s1 and not out.exists() + # 带上退役列算 id 就是另一个并集，各走各的 + assert ra.union_id(c.sid, [(c.aug_start, c.aug_end, c.aug_bid), (c.sep_start, c.sep_end, c.sep_bid)], SHA_A, + retired=retired) != c.uid + + +async def test_single_period_snapshot_whose_build_is_gone_says_where_to_go(store): + c = await chain() + async with SessionLocal() as session: + await session.delete(await session.get(SourceSnapshot, c.s1)) # 逼它新建，而不是复用 + await session.commit() + table_versions.build_path(c.sid, c.aug_bid).unlink() + with pytest.raises(table_versions.PublishError) as err: + await c.publish_existing([c.aug_import], expected=c.s2, union=False) + assert err.value.code == "part_missing" and await current(c.sid) == c.s2 + assert "2026-08-01 至 2026-08-31" in str(err.value) and "启用更早的版本" in str(err.value) + + +async def test_single_period_snapshot_needs_its_build_file(store): + c = await chain() + async with SessionLocal() as session: + await session.delete(await session.get(SourceSnapshot, c.s1)) # 逼它新建，而不是复用 + await session.commit() + path = table_versions.build_path(c.sid, c.aug_bid) + os.chmod(path, 0o644) + with open(path, "ab") as fh: + fh.write(b"\0") + with pytest.raises(table_versions.PublishError) as err: + await c.publish_existing([c.aug_import], expected=c.s2, union=False) + assert err.value.code == "part_tampered" and await current(c.sid) == c.s2 + + +async def test_publish_snapshot_rejects_inconsistent_arguments(store): + c = Chain() + await c.first() + async with SessionLocal() as session: + source = await session.get(DataSource, c.sid) + kw = dict(mode="accumulate", recipe_id="rA", recipe_sha256=SHA_A, expected_current=c.s1, + period_build=None, new_import=None, schema_cache_for=plain_schema) + with pytest.raises(ValueError): + await table_versions.publish_snapshot(session, source, imports=[c.aug_import, "x"], union=None, **kw) + with pytest.raises(ValueError): + await table_versions.publish_snapshot(session, source, imports=[], union=None, **kw) + with pytest.raises(ValueError): + await table_versions.publish_snapshot(session, DataSource(name=unique()), imports=["x"], union=None, + **kw) + + +# ========================================================================== +# 启用旧快照（回滚） +# ========================================================================== + + +async def activate(c: Chain, snapshot_id: str, *, expected: str | None = None, ack=None, source_id=None, + kind: str = "activate") -> str: + async with SessionLocal() as session: + source = await session.get(DataSource, source_id or c.sid) + return await table_versions.activate_snapshot( + session, source, snapshot_id, expected_current=c.s2 if expected is None else expected, kind=kind, + reason="回到上一版", signed_by="测试员", import_id=c.sep_import if kind != "activate" else None, + ack_mask_lost=ack) + + +async def test_activate_switches_back_atomically(store): + c = await chain() + async with SessionLocal() as session: + session.add(TableRecipe(id="rA", source_id=c.sid, seq=1, status="superseded")) + session.add(TableRecipe(id="rB", source_id=c.sid, seq=2, status="active")) + (await session.get(SourceSnapshot, c.s2)).recipe_id = "rB" + (await session.get(DataSource, c.sid)).current_recipe_id = "rB" + await session.commit() + previous = await activate(c, c.s1) + assert previous == c.s2 + row = await get(DataSource, c.sid) + s1 = await get(SourceSnapshot, c.s1) + assert row.current_snapshot_id == c.s1 and row.current_recipe_id == "rA" + assert row.schema_cache == s1.schema_cache and row.database == s1.db_path + assert [(i.id, i.status) for i in await imports_of(c.sid)] == [(c.aug_import, "active"), + (c.sep_import, "superseded")] + assert (await get(TableRecipe, "rA")).status == "active" and (await get(TableRecipe, "rB")).status == "superseded" + assert s1.activated_at >= (await get(SourceSnapshot, c.s2)).activated_at + last = (await activations(c.sid))[-1] + assert (last.kind, last.snapshot_id, last.previous_snapshot_id, last.reason, last.signed_by) == ( + "activate", c.s1, c.s2, "回到上一版", "测试员") + # 固定在 S2 上的查询不受影响，不固定的查到 S1 + assert await count_rows(c.sid, 'SELECT COUNT(*) FROM "日客流"', c.s2) == 61 + assert await count_rows(c.sid, 'SELECT COUNT(*) FROM "日客流"') == 31 + + +async def test_activating_a_pre_phase3_snapshot_takes_the_recipe_of_its_last_import(store): + c = await chain() + async with SessionLocal() as session: + (await session.get(SourceSnapshot, c.s1)).recipe_id = None + await session.commit() + await activate(c, c.s1) + assert (await get(DataSource, c.sid)).current_recipe_id == "rA" + + +async def _break(c: Chain, how: str) -> None: + async with SessionLocal() as session: + snap = await session.get(SourceSnapshot, c.s1) + if how == "retired": + snap.retired_at = utcnow() + elif how == "file_lost": + snap.db_path = str(Path(snap.db_path).with_name("gone.db")) + elif how == "tampered": + snap.db_sha256 = "0" * 64 + elif how == "revoked": + (await session.get(TableImport, c.aug_import)).revoked = {"at": "x", "reason": "合成", "signed_by": None, + "signed_by_verified": False} + await session.commit() + + +@pytest.mark.parametrize("how,code,text", [ + ("retired", "snapshot_unavailable", "已回收"), + ("file_lost", "snapshot_unavailable", "已丢失"), + ("tampered", "snapshot_tampered", "不一致"), + ("revoked", "contains_revoked", "作废"), +]) +async def test_activate_refuses_unusable_snapshots(store, how, code, text): + c = await chain() + await _break(c, how) + with pytest.raises(table_versions.ActivationError) as err: + await activate(c, c.s1) + assert err.value.code == code and text in str(err.value) + assert await current(c.sid) == c.s2 + assert [a.kind for a in await activations(c.sid)] == ["commit", "commit"] + + +async def test_activate_checks_the_base_the_target_and_the_source(store): + c = await chain() + for snapshot_id, expected, code in [ + (c.s1, c.s1, "base_changed"), + ("f" * 64, None, "snapshot_not_found"), + (c.s2, None, "already_current"), + ]: + with pytest.raises(table_versions.ActivationError) as err: + await activate(c, snapshot_id, expected=expected) + assert err.value.code == code + other = Chain() + await other.first() + with pytest.raises(table_versions.ActivationError) as err: + await activate(c, other.s1) + assert err.value.code == "snapshot_not_found" + async with SessionLocal() as session: + manual = DataSource(name=unique(), kind="sqlite", database="/tmp/none.db") + session.add(manual) + await session.commit() + manual_id = manual.id + with pytest.raises(table_versions.ActivationError) as err: + await activate(c, c.s1, source_id=manual_id, expected=None) + assert err.value.code == "not_upload_source" + assert await current(c.sid) == c.s2 + + +async def test_activate_needs_an_ack_when_masked_columns_get_lost(store): + c = await chain() + async with SessionLocal() as session: + (await session.get(DataSource, c.sid)).options = {"mask_columns": ["分区甲", "手机号"]} + await session.commit() + with pytest.raises(table_versions.ActivationError) as err: + await activate(c, c.s1) + assert err.value.code == "mask_lost" and "「手机号」" in str(err.value) and "分区甲" not in str(err.value) + with pytest.raises(table_versions.ActivationError): + await activate(c, c.s1, ack=["分区甲", "手机号"]) + await activate(c, c.s1, ack=["手机号"]) + assert await current(c.sid) == c.s1 + + +def test_mask_lost_compares_names_without_case(): + cache = {"tables": {"日客流": {"columns": [{"name": "日期"}, {"name": "Phone"}]}}} + assert table_versions.mask_lost({"mask_columns": "phone, 邮箱"}, cache) == ["邮箱"] + assert table_versions.mask_lost({}, cache) == [] + assert table_versions.mask_lost({"mask_columns": ["日期"]}, None) == ["日期"] + + +# ========================================================================== +# 查询时用快照清单交叉核对（P3-SPEC 2.9） +# ========================================================================== + + +async def test_bind_snapshot_cross_checks_the_snapshot_manifest(store, monkeypatch): + c = await chain() + assert await count_rows(c.sid, 'SELECT COUNT(*) FROM "日客流"') == 61 + # 文件和库列一起被改成一致的假值：文件哈希核对照样通过，靠快照清单抓出来 + upath = table_versions.union_path(c.sid, c.uid) + os.chmod(upath, 0o644) + conn = sqlite3.connect(upath) + conn.execute('UPDATE "日客流" SET "分区甲" = 0') + conn.commit() + conn.close() + async with SessionLocal() as session: + (await session.get(SourceSnapshot, c.s2)).db_sha256 = raw_store.sha256_file(upath) + await session.commit() + with pytest.raises(table_versions.SnapshotManifestMismatch, match="版本清单不一致") as err: + await count_rows(c.sid, 'SELECT COUNT(*) FROM "日客流"') + assert isinstance(err.value, table_versions.SnapshotMissing) and isinstance(err.value, SnapshotTampered) + assert err.value.code == "snapshot_tampered" + + +async def test_manifest_cross_check_is_cached_per_process(store, monkeypatch): + c = await chain() + calls: list[str] = [] + real = artifact_store.load + + def spy(mid): + calls.append(mid) + return real(mid) + + monkeypatch.setattr(artifact_store, "load", spy) + for _ in range(3): + assert await count_rows(c.sid, 'SELECT COUNT(*) FROM "日客流"') == 61 + assert calls == [c.manifest] + + +async def test_snapshot_whose_manifest_is_gone_is_refused(store): + c = await chain() + await c.publish_existing([c.aug_import], expected=c.s2, union=False) # 当前回到 S1 + async with SessionLocal() as session: + (await session.get(SourceSnapshot, c.s2)).manifest_artifact = "0" * 64 + await session.commit() + with pytest.raises(table_versions.SnapshotManifestMismatch): + await count_rows(c.sid, 'SELECT COUNT(*) FROM "日客流"', c.s2) + with pytest.raises(table_versions.ActivationError) as err: + await activate(c, c.s2, expected=c.s1) + assert err.value.code == "snapshot_tampered" and await current(c.sid) == c.s1 + + +# ========================================================================== +# 回收：SNAPSHOT_KEEP、被运行引用的并集快照 +# ========================================================================== + + +async def test_gc_keeps_the_latest_snapshots_of_each_source(store, monkeypatch): + """当前版本之外再保留 SNAPSHOT_KEEP 个（P3-3）：KEEP=2、发布 5 次，留下当前的和此前的 2 个。""" + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 2) + name = unique() + published = [await publish_small(name, [(i, "分区甲")]) for i in range(5)] + sid = published[0][0] + snaps = [await get(SourceSnapshot, r.snapshot_id) for _, r in published] + assert [s.retired_at is None for s in snaps] == [False, False, True, True, True] + builds = [await get(TableBuild, r.build_id) for _, r in published] + assert [Path(b.db_path).exists() for b in builds] == [False, False, True, True, True] + assert [i.status for i in await imports_of(sid)] == ["retired", "retired", "superseded", "superseded", "active"] + # 启用一个还保留着的旧版本：它排到最前，最早的那个保留版本被挤出去 + async with SessionLocal() as session: + source = await session.get(DataSource, sid) + await table_versions.activate_snapshot(session, source, published[2][1].snapshot_id, + expected_current=published[4][1].snapshot_id, reason=None, + signed_by=None) + _, extra = await publish_small(name, [(9, "分区乙")]) + alive = {s.id for s in [await get(SourceSnapshot, r.snapshot_id) for _, r in published] + if s.retired_at is None} + assert alive == {published[2][1].snapshot_id, published[4][1].snapshot_id} + assert (await get(SourceSnapshot, extra.snapshot_id)).retired_at is None + + +async def test_gc_quota_does_not_count_the_current_or_run_referenced_snapshots(store, monkeypatch): + """界面写的是「除当前版本和被运行引用的版本外，保留最近 N 个」：这两类不占名额。""" + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 1) + name = unique() + sid, p0 = await publish_small(name, [(0, "分区甲")]) + _, p1 = await publish_small(name, [(1, "分区甲")]) + async with SessionLocal() as session: + run = Run(workflow_name="wp4-quota", status="succeeded", data_versions={sid: {"snapshot": p1.snapshot_id}}) + session.add(run) + await session.commit() + run_id = run.id + _, p2 = await publish_small(name, [(2, "分区甲")]) + # 当前的 p2、运行引用的 p1 之外，名额给了 p0 + assert [(await get(SourceSnapshot, r.snapshot_id)).retired_at is None for r in (p0, p1, p2)] == [True] * 3 + async with SessionLocal() as session: + await session.delete(await session.get(Run, run_id)) + await session.commit() + await gc() + # 运行没了，p1 回到名额里排名：它比 p0 新，p0 被挤出去 + assert [(await get(SourceSnapshot, r.snapshot_id)).retired_at is None for r in (p0, p1, p2)] == [ + False, True, True] + + +async def test_gc_quota_does_not_count_retired_snapshots(store, monkeypatch): + """已回收的快照不占名额：排进来只会挤掉还能启用的。""" + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 5) + name = unique() + published = [await publish_small(name, [(i, "分区甲")]) for i in range(3)] + async with SessionLocal() as session: + (await session.get(SourceSnapshot, published[1][1].snapshot_id)).retired_at = utcnow() + await session.commit() + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 1) + await gc() + alive = [(await get(SourceSnapshot, r.snapshot_id)).retired_at is None for _, r in published] + assert alive == [True, False, True] + assert Path((await get(TableBuild, published[0][1].build_id)).db_path).exists() + + +async def test_gc_does_not_keep_snapshots_of_deleted_sources(store, monkeypatch): + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 3) + name = unique() + sid, first = await publish_small(name) + async with SessionLocal() as session: + await table_versions.retire_source(session, sid) + await session.delete(await session.get(DataSource, sid)) + await session.commit() + await gc() + assert (await get(SourceSnapshot, first.snapshot_id)).retired_at is not None + assert not Path((await get(TableBuild, first.build_id)).db_path).exists() + + +async def test_run_referenced_union_snapshot_survives_gc_until_the_run_is_gone(store, monkeypatch): + """A4 的保护规则单独验证（评审一-m8）：SNAPSHOT_KEEP=0（当前版本本来就不占名额），S2（并集）只靠运行引用保护。""" + monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0) + c = await chain() + async with SessionLocal() as session: + run = Run(workflow_name="wp4-gc", status="succeeded", data_versions={c.sid: {"snapshot": c.s2}}) + session.add(run) + await session.commit() + run_id = run.id + await c.publish_existing([c.aug_import], expected=c.s2, union=False) # 当前回到 S1 + upath = table_versions.union_path(c.sid, c.uid) + assert upath.exists() and (await get(SourceSnapshot, c.s2)).retired_at is None + assert (await get(TableBuild, c.uid)).retired_at is None + assert await count_rows(c.sid, 'SELECT COUNT(*) FROM "日客流"', c.s2) == 61 + async with SessionLocal() as session: + await session.delete(await session.get(Run, run_id)) + await session.commit() + await gc() + assert not upath.exists() + assert (await get(SourceSnapshot, c.s2)).retired_at is not None + assert (await get(TableBuild, c.uid)).retired_at is not None + # 8 月还在当前版本里，构建库不动 + assert table_versions.build_path(c.sid, c.aug_bid).exists() + + +# ========================================================================== +# 孤儿清理、附属试运行库 +# ========================================================================== + + +async def test_startup_sweeps_orphan_union_files_and_follows_the_main_trial(store): + c = await chain() + upath = table_versions.union_path(c.sid, c.uid) + orphan = upath.with_name("d" * 64 + ".db") + make_db(orphan, [(1, "孤儿")]) + sid = uuid.uuid4().hex + key = uuid.uuid4().hex + async with SessionLocal() as session: + staging = ImportStaging(id=uuid.uuid4().hex, source_id=sid, source_name=unique(), status="trialed", + file_name="合成.xlsx") + main = make_db(table_versions.trial_db_path(sid, staging.id, key), [(1, "主")]) + staging.trial_key, staging.trial_path = key, str(main) + session.add(staging) + await session.commit() + union_trial = make_db(table_versions.union_trial_path(main), [(1, "并集")]) + stale = table_versions.trial_db_path(sid, staging.id, uuid.uuid4().hex) + stale_union = make_db(table_versions.union_trial_path(stale), [(1, "旧的并集")]) + async with SessionLocal() as session: + await table_versions.startup(session) + assert not orphan.exists() and upath.exists() + assert main.exists() and union_trial.exists() # 附属文件跟着主文件：主文件有效就留着 + assert not stale_union.exists() # 主文件不是暂存区记着的那个：孤儿 + assert table_versions.remove_trial_file(str(main)) is True + assert not main.exists() and not union_trial.exists() + + +# ========================================================================== +# 隔离区的清理（第 8 节遗留项 1） +# ========================================================================== + + +def quarantine(source_id: str, build_id: str, *, age_days: float = 0) -> Path: + folder = table_versions.quarantine_root() / source_id + folder.mkdir(parents=True, exist_ok=True) + path = folder / f"{build_id}-20260101T000000000000Z.db" + path.write_bytes(b"quarantined") + if age_days: + then = time.time() - age_days * 86400 + os.utime(path, (then, then)) + return path + + +async def test_retire_source_deletes_its_quarantine(store): + name = unique() + sid, first = await publish_small(name) + mine = quarantine(sid, first.build_id) + other = quarantine(uuid.uuid4().hex, "e" * 64) + async with SessionLocal() as session: + await table_versions.retire_source(session, sid) + await session.commit() + assert not mine.exists() and not mine.parent.exists() and other.exists() + + +async def test_purging_a_raw_deletes_the_quarantined_files_of_those_builds(store): + name = unique() + sid, first = await publish_small(name) + _, second = await publish_small(name, [(5, "分区丙")]) + gone = quarantine(sid, first.build_id) + kept = quarantine(sid, second.build_id) + async with SessionLocal() as session: + imp = await session.get(TableImport, first.import_id) + await table_versions.purge_import_raw(session, imp, reason="合成理由", signed_by="测试员") + assert not gone.exists() and kept.exists() + + +def quarantined(source_id: str) -> list[Path]: + folder = table_versions.quarantine_root() / source_id + return sorted(folder.glob("*.db")) if folder.is_dir() else [] + + +async def test_quarantine_keeps_a_restored_file_for_seven_days_from_the_move(store): + """走真实路径（_publish_file → _quarantine）：原文件的修改时间早在 30 天前（构建多半是几周前发布的，磁盘位 + 翻转这类损坏也不改修改时间），保留期仍从挪进隔离区的那一刻算。发布之后紧接着的回收不能把证据删掉。""" + name, raw = unique(), raw_bytes() + sid, first = await publish_small(name, raw=raw) + path = Path((await get(TableBuild, first.build_id)).db_path) + os.chmod(path, 0o644) + path.write_bytes(b"") + month_ago = time.time() - 30 * 86400 + os.utime(path, (month_ago, month_ago)) + _, second = await publish_small(name, raw=raw) # publish_build 提交后紧接着回收一次 + assert second.build_restored is True and second.build_id == first.build_id + [moved] = quarantined(sid) + assert moved.stat().st_size == 0 and time.time() - moved.stat().st_mtime < 3600 + await gc() + assert moved.exists() + # 过了保留期才删 + stale = time.time() - (table_versions.QUARANTINE_DAYS + 1) * 86400 + os.utime(moved, (stale, stale)) + await gc() + assert not moved.exists() + + +async def test_gc_deletes_quarantined_files_older_than_seven_days(store): + old = quarantine(uuid.uuid4().hex, "1" * 64, age_days=table_versions.QUARANTINE_DAYS + 1) + young = quarantine(uuid.uuid4().hex, "2" * 64, age_days=table_versions.QUARANTINE_DAYS - 1) + await gc() + assert not old.exists() and young.exists() + + +async def test_startup_does_not_touch_the_quarantine(store): + path = quarantine(uuid.uuid4().hex, "3" * 64) + async with SessionLocal() as session: + await table_versions.startup(session) + assert path.exists() + + +# ========================================================================== +# 单进程守卫（第 8 节遗留项 5） +# ========================================================================== + +_HOLD = """ +import fcntl, os, sys +fd = os.open(sys.argv[1], os.O_RDWR | os.O_CREAT, 0o600) +fcntl.flock(fd, fcntl.LOCK_EX) +print("locked", flush=True) +sys.stdin.read() +""" + + +@pytest.fixture +def guard_reset(): + yield + table_versions._release_guard() + + +def hold_guard() -> subprocess.Popen: + path = settings.uploads_dir / ".store.lock" + proc = subprocess.Popen([sys.executable, "-c", _HOLD, str(path)], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, text=True) + assert proc.stdout.readline().strip() == "locked" + return proc + + +def release(proc: subprocess.Popen) -> None: + proc.stdin.close() + proc.wait(timeout=10) + + +async def test_guard_is_idempotent_in_one_process(store, guard_reset): + assert table_versions.acquire_store_guard() is True + fd = table_versions._guard.fd + assert table_versions.acquire_store_guard() is True and table_versions._guard.fd == fd + table_versions.require_store_writable() + async with SessionLocal() as session: + await table_versions.startup(session) + await table_versions.startup(session) + + +async def test_second_process_skips_startup_cleanup_and_refuses_writes(store, guard_reset, monkeypatch): + """L5：子进程先占住守卫，本进程启动时一个文件都不删、不回收，写入一律 StoreUnavailable；子进程退出后恢复。""" + name = unique() + sid, first = await publish_small(name) + half = table_versions.build_path(sid, "1" * 64).with_name(f"{'1' * 64}.db{raw_store.TMP_MARK}zzz") + half.write_bytes(b"half written") + orphan = make_db(table_versions.build_path(uuid.uuid4().hex, "0" * 64), [(1, "孤儿")]) + _orphan_sha, orphan_raw = raw_store.put_raw(b"raw nobody registered") + expired_id = uuid.uuid4().hex + async with SessionLocal() as session: + session.add(ImportStaging(id=expired_id, source_id=uuid.uuid4().hex, source_name=unique(), status="drafting", + expires_at=utcnow() - timedelta(days=1))) + await session.commit() + ran: list[str] = [] + + async def spy_gc(session): + ran.append("gc") + + real_gc = table_versions.gc + proc = hold_guard() + try: + assert table_versions.acquire_store_guard() is False + monkeypatch.setattr(table_versions, "gc", spy_gc) + async with SessionLocal() as session: + await table_versions.startup(session) + monkeypatch.setattr(table_versions, "gc", real_gc) + assert ran == [] + assert half.exists() and orphan.exists() and orphan_raw.exists() + with pytest.raises(table_versions.StoreUnavailable) as err: + table_versions.require_store_writable() + assert err.value.code == "store_unavailable" and "单进程" in str(err.value) + # 写入的入口一律拒；临时库照样被消耗 + tmp = make_db(table_versions.trial_db_path(sid, uuid.uuid4().hex, uuid.uuid4().hex), [(3, "分区甲")]) + with pytest.raises(table_versions.StoreUnavailable): + async with SessionLocal() as session: + source = await session.get(DataSource, sid) + raw = raw_bytes() + await table_versions.publish_build( + session, source, tmp_db=tmp, build_id="9" * 64, build_options={}, engine_ver=ENGINE, + report={}, raw=raw, raw_sha=hashlib.sha256(raw).hexdigest(), file_name="x.xlsx", + file_size=len(raw), schema_cache_for=plain_schema, import_fields={}) + assert not tmp.exists() + # 每个入口用新会话：失败时会话回滚，里面的对象随之过期 + refused = { + "publish_upload": lambda session, source: table_versions.publish_upload(session, source, b"raw", "x.xlsx"), + "activate_snapshot": lambda session, source: table_versions.activate_snapshot( + session, source, first.snapshot_id, expected_current=first.snapshot_id, reason=None, signed_by=None), + "publish_snapshot": lambda session, source: table_versions.publish_snapshot( + session, source, imports=[first.import_id], mode="replace", recipe_id=None, recipe_sha256=None, + expected_current=first.snapshot_id, period_build=None, new_import=None, union=None, + schema_cache_for=plain_schema), + "gc": lambda session, source: table_versions.gc(session), + "retire_source": lambda session, source: table_versions.retire_source(session, source.id), + "store_staging": lambda session, source: table_versions.store_staging( + session, ImportStaging(source_id=uuid.uuid4().hex, source_name=unique()), raw_bytes()), + "release_staging_raw": lambda session, source: table_versions.release_staging_raw( + session, ImportStaging(source_id=uuid.uuid4().hex, status="discarded", raw_sha256="4" * 64)), + } + for what, call in refused.items(): + async with SessionLocal() as session: + source = await session.get(DataSource, sid) + with pytest.raises(table_versions.StoreUnavailable): + await call(session, source) + async with SessionLocal() as session: + with pytest.raises(table_versions.StoreUnavailable): + await table_versions.purge_import_raw(session, await session.get(TableImport, first.import_id), + reason="合成", signed_by=None) + async with SessionLocal() as session: + # 过期处理直接跳过、不抛：具体的写入那一步再报 503 + assert await table_versions.expire_stagings(session) == [] + assert (await get(ImportStaging, expired_id)).status == "drafting" + assert await current(sid) == first.snapshot_id + finally: + release(proc) + assert table_versions.acquire_store_guard() is True + table_versions.require_store_writable() + async with SessionLocal() as session: + await table_versions.startup(session) + assert not half.exists() and not orphan.exists() and not orphan_raw.exists() + + +async def test_guard_state_does_not_leak_into_another_data_dir(store, guard_reset, monkeypatch, tmp_path): + proc = hold_guard() + try: + assert table_versions.acquire_store_guard() is False + other = tmp_path / "other" + (other / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", other) + # 换了数据目录、没对它取过守卫：按能写处理（只有启动时取守卫） + table_versions.require_store_writable() + assert table_versions.store_writable() is True + finally: + release(proc) + + +def test_versions_page_keep_count_matches_backend(): + """版本页的保留规则那句话写死了保留数（快照列表接口只返回数组，不带这个数）：前后端必须一致，改一边忘了另一边， + 界面就会把保留规则说错。""" + import re + from pathlib import Path + + page = Path(__file__).resolve().parents[2] / "frontend" / "src" / "pages" / "import" / "VersionsDialog.tsx" + found = re.search(r"^const KEEP_RECENT = (\d+)$", page.read_text(encoding="utf-8"), re.M) + assert found, "VersionsDialog.tsx 里找不到 KEEP_RECENT" + assert int(found.group(1)) == table_versions.SNAPSHOT_KEEP diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 096541fd..8bae6170 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,5 +1,7 @@ import type { - AiPreview, CommitOut, CurrentRecipe, QuestionAnswer, Recipe, Staging, + ActivateSnapshotBody, ActivateSnapshotOut, AiPreview, CommitOut, CurrentRecipe, EditPreview, EditRequest, ImportRecord, + ManifestOut, PurgeRawBody, PurgeRawOut, QuestionAnswer, Recipe, RedraftRulesOut, RemovePeriodBody, RemovePeriodOut, + RevokeAcceptanceBody, RevokeAcceptanceOut, SnapshotOut, Staging, } from '../types' import type { Approval, AutofixResult, Conversation, ConversationDetail, ConversationTurn, CustomTool, DataSource, EvidenceGraph, @@ -572,9 +574,13 @@ export const api = { /** 已选的全部回答（服务端从起点配方按问题顺序重放）。needs_reason 的选项要带 reason */ answers: (id: string, answers: Record) => post(`/datasources/imports/${encodeURIComponent(id)}/answers`, { answers }), - /** 整份配方替换工作配方：有静态校验问题也照存，只是不能试运行；已选的回答清空 */ - putRecipe: (id: string, recipe: Recipe) => - put(`/datasources/imports/${encodeURIComponent(id)}/recipe`, { recipe }), + /** + * 整份配方替换工作配方：有静态校验问题也照存，只是不能试运行。期 3：仍然成立的回答保留（不成立的进 + * answers_dropped），撤销栈清空、之前的修改标「已被覆盖」。origin 只在采用「按规则重新起草」的结果时给 + * 'rules_redraft'（服务端核对配方哈希，对不上回 409 redraft_mismatch）；不给时请求体与期 2 相同 + */ + putRecipe: (id: string, recipe: Recipe, origin?: 'rules_redraft') => + put(`/datasources/imports/${encodeURIComponent(id)}/recipe`, origin ? { recipe, origin } : { recipe }), /** 将要发给模型的全文（同意框里给人看的就是这一份） */ draftAiPreview: (id: string) => get(`/datasources/imports/${encodeURIComponent(id)}/draft-ai/preview`), /** 用户同意后才调：preview_sha256 与服务端重算的不一致时回 409 ai_preview_stale */ @@ -595,6 +601,52 @@ export const api = { /** 修改配方（不换文件）：用当前导入的原件、以当前配方为工作配方 */ redraft: (sourceId: string) => post(`/datasources/${encodeURIComponent(sourceId)}/redraft`, {}), recipe: (sourceId: string) => get(`/datasources/${encodeURIComponent(sourceId)}/recipe`), + /** + * 期 3：修复或框选的预览（不改工作配方）。body 是 {fix: {id, option, reason?}} 或 {selection: {sheet, ref, as, + * options?}} 二选一，seq 原样回传（只采用最新一次的回包）。ok=false 时照样 200，原因在 problems + */ + editPreview: (id: string, body: EditRequest) => + post(`/datasources/imports/${encodeURIComponent(id)}/edits/preview`, body), + /** + * 期 3：应用预览过的修复或框选。选项和理由必须与预览时逐字相同，expected_sha256 是预览返回的 + * recipe_sha256_after：对不上回 409 edit_stale（需要重新预览）；提议已不在当前列表里回 409 fix_stale + */ + editApply: (id: string, body: EditRequest & { expected_sha256: string; signed_by?: string | null }) => + post(`/datasources/imports/${encodeURIComponent(id)}/edits/apply`, body), + /** 期 3：撤销上一次修改。没有可撤销的回 409 nothing_to_undo；之后配方又被改过回 409 undo_stale */ + editUndo: (id: string) => post(`/datasources/imports/${encodeURIComponent(id)}/edits/undo`, {}), + /** 期 3：按规则重新起草（不调用模型、不改工作配方），名字对齐到现行配方；「采用」走 putRecipe(…, 'rules_redraft') */ + redraftRules: (id: string) => + post(`/datasources/imports/${encodeURIComponent(id)}/redraft-rules`, {}), + }, + + /** + * 期 3 的版本页（数据源卡片上的「版本」，P3-SPEC 第 7 节）：版本列表、启用旧版本、清单、移除一期、作废接受、 + * 导入记录与清除原件。写操作都要 confirm: true；启用、移除、作废还要带界面看到的当前版本 id + * （expected_current_snapshot_id），别人在这期间改过当前版本时回 409 base_changed。错误按 ApiError.code 分支 + */ + versions: { + /** 最近 50 个版本，当前版本排第一。手工源返回空列表 */ + snapshots: (sourceId: string) => get(`/datasources/${encodeURIComponent(sourceId)}/snapshots`), + activate: (sourceId: string, snapshotId: string, body: ActivateSnapshotBody) => + post( + `/datasources/${encodeURIComponent(sourceId)}/snapshots/${encodeURIComponent(snapshotId)}/activate`, body), + snapshotManifest: (sourceId: string, snapshotId: string) => + get(`/datasources/${encodeURIComponent(sourceId)}/snapshots/${encodeURIComponent(snapshotId)}/manifest`), + importManifest: (sourceId: string, importId: string) => + get(`/datasources/${encodeURIComponent(sourceId)}/imports/${encodeURIComponent(importId)}/manifest`), + /** 从当前版本中移除这一期（累积模式；配方和导入模式不变） */ + removePeriod: (sourceId: string, importId: string, body: RemovePeriodBody) => + post(`/datasources/${encodeURIComponent(sourceId)}/imports/${encodeURIComponent(importId)}/remove`, body), + /** 撤回这一期（作废接受，不可恢复）：替换模式回滚到 revoke_plan 的目标，累积模式移除这一期 */ + revokeAcceptance: (sourceId: string, importId: string, body: RevokeAcceptanceBody) => + post( + `/datasources/${encodeURIComponent(sourceId)}/imports/${encodeURIComponent(importId)}/revoke-acceptance`, body), + /** 全部导入记录，新的在前（含已被替换的） */ + imports: (sourceId: string) => get(`/datasources/${encodeURIComponent(sourceId)}/imports`), + /** 清除一次导入的原件（同一份内容的其他导入一并清除，引用它的未完成导入一并放弃）。理由必填 */ + purgeRaw: (sourceId: string, importId: string, body: PurgeRawBody) => + post(`/datasources/${encodeURIComponent(sourceId)}/imports/${encodeURIComponent(importId)}/purge-raw`, body), }, runs: { diff --git a/frontend/src/components/ui.tsx b/frontend/src/components/ui.tsx index fcf76564..79f52597 100644 --- a/frontend/src/components/ui.tsx +++ b/frontend/src/components/ui.tsx @@ -602,8 +602,14 @@ export interface DialogOptions { initial?: string /** promptDialog 的输入框标签 */ label?: string - /** promptDialog：返回错误文字则不让提交 */ + /** promptDialog：返回错误文字则不让提交。allowEmpty 时空输入不经过它 */ validate?: (value: string) => string | null | undefined + /** + * promptDialog：允许不填，空输入（含只有空白）也能确认，返回空字符串。默认关：多数输入框（名称、必填的理由） + * 空着提交没有意义，照旧报「不能为空」。用于可选的理由这类「填了更好、不填也成立」的输入，否则调用方只能 + * 退回不带输入框的 confirmDialog，想写的人也没处写 + */ + allowEmpty?: boolean } interface DialogRequest { @@ -654,7 +660,7 @@ export function confirmDialog(opts: DialogOptions): Promise { return enqueue('confirm', opts) } -/** 输入对话框，替代 window.prompt。取消返回 null，确定返回去掉首尾空白的文字 */ +/** 输入对话框，替代 window.prompt。取消返回 null，确定返回去掉首尾空白的文字（allowEmpty 时可能是空字符串） */ export function promptDialog(opts: DialogOptions): Promise { if (!dialogHosts.length) { const v = window.prompt(plainText(opts), opts.initial ?? '') @@ -700,7 +706,7 @@ function DialogView({ req }: { req: DialogRequest }) { const trimmed = value.trim() const invalid = kind === 'prompt' - ? (!trimmed ? '不能为空' : opts.validate?.(trimmed) || null) + ? (!trimmed ? (opts.allowEmpty ? null : '不能为空') : opts.validate?.(trimmed) || null) : null const textOk = opts.requireText == null || value === opts.requireText const canConfirm = kind === 'prompt' ? !invalid && textOk : textOk @@ -925,6 +931,30 @@ export function ErrorNotice(props: { error: unknown; onRetry?: () => void; class return } +export type NoticeTone = 'warn' | 'err' | 'info' + +/** + * 页面里的提示条：一句结论加可选的细节（children），按语气配色和图标。info 是 status（读屏礼貌播报）， + * warn / err 是 alert。attr 原样挂到根节点上，给检查脚本定位（data-wizard-notice 之类）。 + * + * 原来是导入向导的局部组件，期 3 的修复面板、累积计划和版本页都要用，挪到这里共用，不各造一份 + */ +export function Notice({ tone, children, attr, className }: { + tone: NoticeTone; children: ReactNode; attr?: Record; className?: string +}) { + const color = tone === 'err' ? 'var(--err)' : tone === 'warn' ? 'var(--warn)' : 'var(--accent)' + return ( +
+ {tone === 'info' + ? + : } +
{children}
+
+ ) +} + /** * 每 ms 毫秒重渲染一次的「现在」，给倒计时、「3 分钟前」这类相对时间保鲜。 * 只在 active 时计时，不做常驻定时器；页面不可见时不跳，回到前台立刻补一拍， diff --git a/frontend/src/dev/ui-harness.tsx b/frontend/src/dev/ui-harness.tsx index 49cb1067..17f9a971 100644 --- a/frontend/src/dev/ui-harness.tsx +++ b/frontend/src/dev/ui-harness.tsx @@ -4,10 +4,11 @@ import { createRoot } from 'react-dom/client' import { BrowserRouter } from 'react-router-dom' import { Database, Inbox, Plus, Trash2, RefreshCw } from 'lucide-react' import { - DeleteButton, EmptyState, ErrorState, Field, HealthPill, IconButton, Kbd, Modal, OfflineBanner, PageHeader, + DeleteButton, EmptyState, ErrorState, Field, HealthPill, IconButton, Kbd, Modal, Notice, OfflineBanner, PageHeader, SectionBar, Skeleton, StatusBadge, StatusPill, Tabs, TabPanel, ToastHost, confirmDialog, deferDelete, promptDialog, toast, useRadioGroup, withoutDeferred, } from '../components/ui' +import { ReceiptSummary } from '../pages/import/ReceiptBlocks' import { ApiError, api, streamCopilot } from '../api/client' import { useCatalog, useDatasources, useOnReconnect } from '../store/catalog' import { STATUS } from '../lib/status' @@ -32,7 +33,7 @@ import evidenceVerdict from '../run/__tests__/evidence-verdict.json' import { EvidenceGuessView } from '../run/EvidenceGuess' import { IssuanceBanner } from '../run/AssistantStream' import mdPinned from '../run/__tests__/markdown-pinned.json' -import type { EvidenceDocData, EvidenceGraph } from '../types' +import type { EvidenceDocData, EvidenceGraph, TrialReceipt } from '../types' import '../index.css' /** @@ -225,6 +226,44 @@ function longDoc(doc: EvidenceDocData, times: number): EvidenceDocData { } const EV_LONG = longDoc(EV_DOC, 16) +/** 回执块的样例（合成的假数据）：三种排除原因里有一种是界面不认识的，要写「其他原因」，不露枚举值 */ +const DEMO_RECEIPT: TrialReceipt = { + ledger: [{ sheet: '客流汇总', nonempty_scan: 771, nonempty_read: 771, roles: { value: 771 }, unclaimed: 0 }], + tables: [{ name: '日客流', sheet: '客流汇总', kind: 'data', columns: [], grain: ['日期'], rows: 30 }], + placeholders: { '·': 30 }, + canonicalized_total: 17, + outside_text: [{ sheet: '客流汇总', cell: '客流汇总!B3', text: '客流汇总表', kind: 'text' }], + rows_excluded: [ + { sheet: '客流汇总', reason: 'ignored_rows', rows: [[32, 32]], cells: 3, anchor: '补录（人次）', block: '交叉表' }, + { sheet: '客流汇总', reason: 'total_not_kept', rows: [[28, 30]], cells: 93, block: '交叉表' }, + { sheet: '客流汇总', reason: 'mystery_reason', rows: [[40, 41]], cells: 2 }, + ], +} + +/** 期 3 的共用件：提示条 Notice（从导入向导挪进 ui.tsx）、回执块（向导回执和版本页清单视图共用） */ +function NoticeKitDemo() { + const [focused, setFocused] = useState('') + return ( +
+
+ 已应用修复，请重新试运行 + 服务端的同版本数据文件曾被改动，已用本次上传的文件恢复 + + 导入失败：当前版本未受影响 +
细节写在第二行
+ +
+
+ +
定位到：{focused}
+
+
+ +
+
+ ) +} + /** * 可点击证据的演示：宽栏（侧边面板 / 窄屏时底部抽屉）、360px 窄栏（栏内展开）、旧契约的 * 按位置标记。文档是后端 compose_doc 真跑出来的夹具（evidence-doc.json，只用通用名）； @@ -650,6 +689,10 @@ function Harness() { + + + + diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index f5f3a4c9..2c3ca503 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -38,7 +38,7 @@ * 枚举值（formal / degraded / withheld 等）不动，只换显示。 */ -import type { NodeType, ToolTrust } from '../types' +import type { NodeType, SnapshotReasonCode, ToolTrust } from '../types' import { formatNumber } from './format' export const TERMS = { @@ -904,6 +904,183 @@ export const RAW_STATE_LABEL: Record = { absent: '未保存原件', } +/** + * 版本页（数据源卡片上的「版本」，P3-SPEC 7.8、附录 C）。整页不出现「快照」「并集」「构建」，一律写「版本」 + * 「这一期」「数据文件」；署名一律标「署名（未认证）」；指向这个入口时写「数据源卡片上的「版本」」，不写「版本页」。 + * 服务端返回的错误原话直接显示，不在这里重复。 + * + * 这一段是连续的一段，由版本页维护；本文件的其余部分归导入向导 + */ +export const VERSIONS_TEXT = { + open: '版本', + openHint: '查看各期和历史版本：启用旧版本、移除某一期、查看导入清单、清除原件', + title: (name: string) => `「${name}」的版本`, + retentionRule: (keep: number) => + `除当前版本和被运行引用的版本外，系统保留最近 ${formatNumber(keep)} 个版本；更早的版本会被回收，回收后无法再启用`, + tabs: { current: '当前版本', history: '历史版本', imports: '全部导入记录' }, + modeLabel: { accumulate: '按期累积', replace: '每期替换' } as Record, + /** + * 卡片上的一句：按期累积写期数，每期替换只写「每期替换」。服务端没给期数（老后端、字段漏了）时只写「按期累积」： + * 缺值不能替它说成「1 期」，实际可能是多期 + */ + cardSummary: (mode: string | null | undefined, n: number | null | undefined) => + (mode === 'accumulate' ? (typeof n === 'number' ? `按期累积 · ${formatNumber(n)} 期` : '按期累积') : '每期替换'), + /** at 是 formatRelative 或 formatDateTime 的结果：以数字结尾时补一个空格（同 UPLOAD_TEXT.importedAt） */ + activatedAt: (at: string) => `${at}${/\d$/.test(at) ? ' ' : ''}启用`, + /** 卡片悬停：回滚之后启用时间晚于导入时间，两个都写 */ + activatedTitle: (at: string, created: string) => `当前版本于 ${at} 启用，导入于 ${created}`, + period: (start: string, end: string) => `${start} 至 ${end}`, + periodUnknown: '统计期未记录', + /** 没有统计期的一期（简单导入、期 3 之前的导入）用文件名指代 */ + periodFile: (file: string) => `${file}（统计期未记录）`, + periods: (n: number) => `${formatNumber(n)} 期`, + /** 当前版本的各期之间本来就有空缺（移除过中间一期、某个月没导入）：每次打开都写，不只在移除时的确认框里写一次 */ + currentGaps: (list: string) => `各期之间有空缺（${list}），比较不同期之前需要先查询日期的覆盖范围`, + rows: (n: number) => `${formatNumber(n)} 行`, + recipeSeq: (n: number) => `配方第 ${formatNumber(n)} 版`, + simpleImport: '简单导入', + importSeq: (n: number) => `第 ${formatNumber(n)} 次导入`, + /** 导入记录的状态（table_imports.status） */ + importStatus: { active: '在当前版本中', superseded: '已被替换', retired: '已回收' } as Record, + signedBy: (name: string) => `署名（未认证）：${name}`, + /** 本机没有填署名时，确认框里「署名（未认证）：」后面写这个 */ + unsigned: '未填写', + /** 当前版本的概览 */ + overview: { mode: '导入模式', periods: '期数', recipe: '配方', activated: '启用时间', tables: '各表行数' }, + acceptCount: (n: number) => `接受 ${formatNumber(n)} 条`, + acceptRow: (check: string, reason: string) => `${check}：${reason}`, + /** 接受的种类：override 是数据质量类不成立，waiver 是合计无法核对 */ + acceptKind: { override: '数据质量', waiver: '无法核对' } as Record, + pinnedRuns: (n: number) => `被 ${formatNumber(n)} 次运行引用`, + /** text 是 formatBytes 的结果 */ + size: (text: string) => `文件大小 ${text}`, + retiredGroup: (n: number) => `已回收 ${formatNumber(n)} 个`, + /** 不能启用的原因，按 SnapshotOut.reason_code 取（键就是代码；文字与后端 SNAPSHOT_NOT_ACTIVATABLE 逐字相同， + * 改这里的文字时请编排者同步改后端）。没有 reason_code 时显示服务端的 reason，不按原文反查键 */ + notActivatable: { + current: '已是当前版本', + retired: '已回收，无法启用', + file_lost: '数据文件已丢失，无法启用', + contains_revoked: '包含已作废接受的导入，无法启用', + } satisfies Record, + maskLostBadge: '部分遮罩列在这个版本中没有同名列', + historyEmpty: '没有其他版本', + importsEmpty: '还没有导入记录', + // ---- 启用（回滚） + activate: '启用这个版本', + activateTitle: '启用这个版本？', + activateConsequence: { + removed: (list: string) => `当前版本将不再包含：${list}`, + added: (list: string) => `当前版本将增加：${list}`, + newRuns: '启用后，新发起的运行使用这个版本', + running: '已经在进行的运行不受影响', + recipe: (n: number) => `配方回到这个版本使用的第 ${formatNumber(n)} 版`, + simple: '这个数据源回到简单导入，之后上传新一期时不再按配方导入', + /** label 取 modeLabel 的值 */ + mode: (label: string) => `导入模式回到${label}`, + nextUpload: '之后上传新一期时，以启用后的版本为基础', + stagings: '未完成的导入需要重新试运行', + maskLost: (cols: string) => `以下遮罩列在这个版本中没有同名列，启用后不再遮罩：${cols}`, + }, + activated: '已启用这个版本', + // ---- 移除这一期（按期累积） + removePeriod: '移除这一期', + removeTitle: (period: string) => `从当前版本中移除 ${period}？`, + removeConsequence: { + result: (list: string) => `移除后当前版本包含：${list}`, + gap: (period: string) => `移除后各期之间出现空缺（${period}），比较不同期之前需要先查询日期的覆盖范围`, + recipe: '配方和导入模式不变', + reactivate: '可以在「历史版本」中重新启用移除前的版本（在保留期内）', + running: '已经在进行的运行不受影响', + stagings: '未完成的导入需要重新试运行', + }, + removeLastDisabled: '这是当前版本里唯一的一期，不能移除', + removed: (period: string) => `已从当前版本中移除 ${period}`, + /** 移除后的各期与此前某个版本相同：服务端直接启用了那个版本 */ + removedReused: '与此前的某个版本内容相同，已直接启用该版本', + // ---- 撤回这一期（作废接受，不可恢复） + revoke: '撤回这一期（作废接受）', + revokeTitle: '作废这一期的接受？', + revokeConsequence: { + irreversible: '作废后无法恢复：作废本身不能撤回，包含这一期的版本都不能再启用', + rollback: (desc: string) => `当前版本将回到：${desc}`, + remove: (list: string) => `这一期将从当前版本中移除，当前版本包含：${list}；配方和导入模式不变`, + record: '导入记录和接受理由保留，标记为已作废', + running: '已经在进行的运行不受影响', + }, + revokeUnavailable: '没有可以回滚的版本，请上传修正后的文件', + revoked: '已作废这一期的接受', + // ---- 清除原件 + purge: '清除原件', + purgeTitle: '清除这一期的原件？', + purgeConsequence: { + evidence: '证据面板将显示「原件已清除」', + redraft: '原件清除后，这一期无法再按修改后的配方重新导入', + shared: (list: string) => `同一份内容的其他导入记录一并清除：${list}`, + stagings: (n: number) => `引用它的 ${formatNumber(n)} 个未完成导入一并放弃`, + }, + /** 清除前列出同一份原件的其他引用：按数据源汇总 */ + sharedItem: (name: string, n: number) => `「${name}」${formatNumber(n)} 条`, + purgeDone: (also: number, discarded: number) => + `已清除。另外清除了 ${formatNumber(also)} 条导入记录的原件，放弃了 ${formatNumber(discarded)} 个未完成导入`, + purgedItem: (name: string, seq: number, file: string) => `「${name}」第 ${formatNumber(seq)} 次导入（${file}）`, + /** 清除确认框的正文开头：哪一次导入、哪一期。同一统计期可能导入过几次（替换前后），带上导入次序才分得清 */ + purgeTarget: (seq: number, period: string) => `第 ${formatNumber(seq)} 次导入，${period}`, + /** 一并清除的导入记录属于已经删除的数据源时，名字的位置写这个 */ + deletedSource: '已删除的数据源', + /** 移除这一期、作废接受、清除原件的理由输入框 */ + reasonLabel: '理由（必填）', + /** 启用旧版本的理由输入框（7.2 理由可选）：空着也能启用，不填就不随请求发送 */ + reasonOptional: '理由（可选）', + reasonTooLong: '理由不能超过 500 字', + // ---- 清单 + manifest: '查看导入清单', + manifestRaw: '查看原始清单', + manifestTitle: (seq: number) => `第 ${formatNumber(seq)} 次导入的导入清单`, + back: '返回版本列表', + /** 清单视图的各块标题（回执摘要、排除的行也用在向导的回执里） */ + manifestBlocks: { + file: '文件', + recipe: '配方', + period: '统计期', + checks: '核对', + acceptances: '接受', + confirmations: '确认清单', + edits: '修改记录', + notes: '说明', + receipt: '回执摘要', + excluded: '排除的行', + ai: 'AI 用量', + }, + manifestUnverified: '这份清单的内容哈希校验失败，可能已被修改，不能作为证据', + /** 简单导入没有导入清单，服务端给的是导入回执 */ + manifestSimple: '简单导入没有导入清单，以下是这次导入的回执', + sha: (prefix: string) => `内容哈希 ${prefix}`, + recipeJson: '查看配方 JSON', + periodHuman: '人工录入', + sql: '查看 SQL', + /** 核对的计数：checked 是核对了几处，failed、unverifiable 为 0 时不写 */ + checkCounts: (checked: number, failed: number, unverifiable: number) => + [`核对 ${formatNumber(checked)} 处`, failed ? `不一致 ${formatNumber(failed)} 处` : '', + unverifiable ? `无法核对 ${formatNumber(unverifiable)} 处` : ''].filter(Boolean).join('，'), + editKind: { fix: '修复', selection: '框选' } as Record, + editSuperseded: '已被覆盖', + editsNone: '这次导入没有修改', + editsUnrecorded: '这次导入未记录修改', + checksNone: '没有核对结果', + acceptancesNone: '没有接受的核对', + confirmationsNone: '没有勾选的确认项', + notesNone: '没有说明', + /** tokens 是 formatTokens 的结果（带单位），cost 是 formatCost 的结果 */ + aiUsage: (calls: number, tokens: string, cost: string) => `调用模型 ${formatNumber(calls)} 次，共 ${tokens}，约 ${cost}`, + aiNone: '这次导入没有调用模型', + outsideTextNote: '区域外文字只在这里显示，不提供给模型', + revokedBadge: '接受已作废', + /** 当前版本或回滚目标被别人改过（base_changed、revoke_target_changed）：服务端原话说了变的是什么，这里只说列表已刷新 */ + refreshHint: '列表已刷新，请重新确认', + empty: '还没有版本', +} as const + /** * 按配方导入（期 2）：向导、原始网格、建议卡片、配方面板、试运行回执、确认清单、差异卡、AI 起草。 * 后端返回的问题、确认项、核对标题是整句人话，界面原样显示；这里只放界面自己的文字和枚举的显示名。 @@ -1085,6 +1262,12 @@ export const RECIPE_TEXT = { notComparable: (a: string, b: string) => `「${a}」与「${b}」口径不同`, sumEq: (total: string, parts: string) => `${total} = ${parts}`, dismissed: (claims: string) => `不登记系统发现的关系 ${claims}`, + ignoreRules: '忽略规则', + ignoreRow: (label: string) => `按行标签忽略「${label}」这一行`, + ignoreColumn: (header: string) => `按表头忽略「${header}」这一列`, + ignoreOutside: (anchor: string) => `同一行有文字「${anchor}」时，忽略这一行导入区域之外的数字`, + ignoreReason: (reason: string) => `理由：${reason}`, + removeIgnore: (text: string) => `移除忽略规则：${text}`, // ---- 试运行回执 receipt: '试运行回执', ledger: (n: string) => `单元格去向：非空单元格 ${n} 个`, @@ -1158,6 +1341,279 @@ export const RECIPE_TEXT = { diffRequires: '需确认', diffNone: '与上一期相比没有需要说明的差异', sameAsImport: (seq: string) => `与第 ${seq} 次导入相同，无需重复导入`, + + // ======== 期 3（P3-SPEC 10.2）。向导新增的部分和版本页一样不出现「快照」「并集」「构建」，「锚点」是内部术语也不上界面 + // ---- 网格框选 + selectToggle: '框选', + selectToggleHint: '在表格上按住拖动选出一块区域，或在下方输入区域；按文字定位，不记坐标', + selectionNone: '在表格上拖动，或输入区域', + selected: (ref: string) => `已选 ${ref}`, + selectionRef: '区域', + selectionRefPlaceholder: '如 C5:F8', + selectionRefInvalid: '区域写法不对，请写成「C5:F8」这样的形式', + selectionMenu: '框选为…', + selectionCancel: '取消', + outOfPreview: '这一格不在预览范围内（仅显示前 300 行、60 列）', + // ---- 框选面板 + selectionTitle: (label: string) => `框选为：${label}`, + selectionHeaderRows: '表头行数', + selectionBottom: '下边界', + selectionBottomBox: '以框为准', + selectionBottomAuto: '按规则推断', + selectionTable: '新表名（可不填）', + selectionTablePlaceholder: '不填时按规则取名', + selectionRole: '这组行是', + selectionRoleMeasures: '各行是不同的量（每行一列）', + selectionRoleDimension: '各行是同一列的不同取值', + selectionKeep: '合计行的原值', + selectionKeepYes: '另存一张表', + selectionKeepNo: '只核对，不另存', + selectionSegment: '哪个分段', + selectionSegmentPick: '请选择分段', + selectionCrosstabNote: '框只用来指认是哪一块，范围按日期表头和行标签确定', + selectionByText: '按文字定位，不记坐标', + selectionReplayMatch: '重放结果与框选一致', + selectionReplayDiffer: '重放结果与框选不一致，应用后以重放结果为准', + selectionWindow: (n: string) => `仅比对前 ${n} 行，完整范围在试运行时核对`, + selectionExpected: '框选的范围', + selectionActual: '重放识别的范围', + selectionRegion: { header: '表头', data: '数据', total: '合计行', axis: '日期表头', labels: '行标签', values: '数据' } as Record, + selectionCannot: '无法按这个框选修改配方', + selectionApply: '应用', + selectionApplyAnyway: '仍然应用（以重放结果为准）', + selectionApplied: '已按框选修改配方，请重新试运行', + selectionNotes: '提示', + // ---- 修复面板 + fixBack: '返回', + fixOptions: '怎么修改', + fixReasonLabel: '理由（必填，最多 200 字）', + fixReasonPlaceholder: '说明为什么这样处理；理由会写进配方', + fixPreview: '预览', + fixPreviewAgain: '重新预览', + fixPreviewing: '正在预览', + fixReasonChanged: '理由已修改，请重新预览', + fixOptionChanged: '选项已修改，请重新预览', + fixReasonFirst: '请先填写理由，再预览', + fixSummary: '将做的修改', + fixAfter: '修改后的检查结果', + fixAfterClean: '修改后没有发现问题', + fixBreaking: '对现行配方的破坏性变化', + fixBreakingBadge: '破坏性变更', + fixApply: '应用修复', + fixApplying: '正在应用', + fixApplied: '已应用修复，请重新试运行', + fixStale: '问题已变化，请重新查看修复建议', + fixCannot: '无法应用这条修复', + /** 预览没通过、服务端却没有给出原因（不该出现）：不留一块空的预览区 */ + editCannotUnknown: '服务端未说明原因，请返回后重新查看', + editStale: '配方在预览之后有变化，请重新预览', + previewToggle: '网格显示', + previewBefore: '修改前', + previewAfter: '修改后', + recipeProblemsBar: (n: string) => `配方有 ${n} 个问题`, + recipeProblemsHint: '修改之前不能试运行', + // ---- 已做的修改 + editsTitle: '已做的修改', + editSuperseded: '已被覆盖', + editSupersededHint: '之后整份替换过配方：这项修改不能再撤销，也不再列入确认清单', + editUndo: '撤销上一次修改', + editUndone: '已撤销上一次修改，请重新试运行', + editSigned: (who: string) => `署名（未认证）：${who}`, + // ---- 改配方后的回答 + answersChanged: (list: string) => `改配方后，以下问题需要重新回答：${list}`, + answersGone: (list: string) => `以下问题已不适用，回答已移除：${list}`, + // ---- 整份替换工作配方之前的确认 + replaceTitle: '替换当前工作配方？', + saveRecipeTitle: '保存对配方的修改？', + replaceEdits: (n: number) => (n > 0 ? `当前工作配方（含已做的 ${formatNumber(n)} 项修改）将被替换` : '当前工作配方将被替换'), + replaceUndo: '之前的修改标为「已被覆盖」，不能再撤销，也不再列入确认清单', + replaceConfirm: '替换', + saveRecipeConfirm: '保存', + // ---- 按规则重新起草 + redraftRules: '按规则重新起草（不调用模型）', + redraftRulesHint: '按本次的文件重新起草，并把表名、列名对齐到现行配方；采用之前不改工作配方', + redraftRunning: '正在重新起草', + redraftTitle: '按规则重新起草的结果', + redraftAlignment: '名字对齐', + redraftNoRecipe: '规则起草未能得到完整的配方，无法采用', + redraftAdopt: '采用', + redraftAdoptTitle: '采用重新起草的配方？', + redraftAdoptAfter: '采用后，确认清单按首次导入逐项列出配方的各项设置', + redraftDiscard: '不采用', + redraftMismatch: '采用的配方与重新起草的结果不同，请重新起草后再采用', + // ---- 回执、确认清单 + sheetRenamed: '工作表改名', + sheetRenamedLine: (from: string, to: string) => `配方里的工作表「${from}」，本期叫「${to}」`, + priorAcceptance: (reason: string, seq: string, who: string) => + `上次接受的理由：${reason}（第 ${seq} 次导入，署名（未认证）：${who}）`, + priorUnsigned: '未署名', + commitConflict: '服务端的数据文件与登记的不一致，重试也会失败：请联系管理员检查数据目录', + // ---- 完成 + doneRestored: '服务端的同版本数据文件曾被改动，已用本次上传的文件恢复', + doneSnapshotReused: '与此前的某个版本内容相同，已直接启用该版本', +} as const + +/** 修复按钮的文字（契约 FIX_KINDS）：按钮放在对应的问题旁，文字写会做什么，不写内部的叫法 */ +export const FIX_KIND_LABEL: Record = { + remove_label: '去掉标签', + add_label: '加入标签', + edit_members: '更新关系成员', + rename_title: '改分段标题', + declare_total: '改作合计核对', + ignore_cells: '按文字忽略', + declare_placeholder: '声明占位符', + declare_hidden: '设置隐藏行处理', + rename_sheet: '更新工作表名', +} + +/** 「框选为…」的取值（契约 SELECTION_AS），顺序就是下拉框里的顺序 */ +export const SELECT_AS_LABEL: Record = { + list: '列表（含表头）', + crosstab: '交叉表（整块）', + segment: '分段', + derived: '合计行', + section_title: '分段标题', + ignore_rows: '忽略这些行', + ignore_columns: '忽略这些列', + ignore_outside: '忽略区域外的数字', +} + +/** 框选换算出的定位文字的种类（契约 Anchor.kind）：「表头：地区、产品…」 */ +export const ANCHOR_KIND_LABEL: Record = { + header: '表头', + row_label: '行标签', + section_title: '分段标题', + total_word: '合计词', + axis: '日期表头', + after_title: '上方的标题', + outside_text: '同一行的文字', +} + +/** 确认清单的分组（契约 ConfirmItem.source）。区域外文字、工作表、统计期都是「本期」才有的项，合成一组 */ +export const CONFIRM_SOURCE_LABEL: Record = { + recipe: '配方', + edit: '修改', + accumulate: '累积', + outside: '本期', + sheet: '本期', + context: '本期', + diff: '差异', + switch: '切换', +} + +/** + * 按期累积（P3-SPEC 2.2–2.5、10.2）：累积计划、导入模式。行数分「本期」「启用后当前版本」两栏， + * 不写「并集行数」 + */ +export const ACCUMULATE_TEXT = { + title: '各期', + action: { + append: '新增一期', + replace_period: '替换该期', + restart: '重新开始累积', + replace: '改为每期替换', + first: '作为第一期', + rejected: '与已有各期部分重叠，不能累积', + } as Record, + thisPeriod: '本期', + afterEnable: '启用后当前版本', + table: '表', + partNew: '本期', + partReplaced: '将被替换', + partDropped: '将不在当前版本中', + period: (start: string, end: string) => `${start} 至 ${end}`, + periodUnknown: '统计期未记录', + importSeq: (n: string) => `第 ${n} 次导入`, + overlaps: '部分重叠的各期', + gaps: (list: string) => `各期之间有空缺：${list}`, + backfill: '本期早于已有各期，按统计期排序插入', + semantic: '与当前版本不兼容的变化', + restartHint: '如果不想重新开始累积，请返回修改配方（例如保留原来的单位、列名或常量取值）', + added: '新增的列', + retired: '自本期起不再导入的列', + wholeTable: '整张表', + labelSets: '各期取值不完全相同', + labelMissing: (period: string, list: string) => `${period} 没有：${list}`, + labelExtra: (period: string, list: string) => `${period} 另有：${list}`, + blockers: (period: string, reason: string) => `${period} 的配方不满足按期累积的要求：${reason}`, + checks: '整体核对', + modeField: '导入模式', + modeLabel: { accumulate: '按期累积', replace: '每期替换' } as Record, + /** 两种取值的后果（P3-SPEC 2.2，评审三-m4）：问题选项下方、配方面板单选下方各显示一句 */ + modeConsequence: { + accumulate: '每期以统计期为键加入当前版本；统计期与已有各期部分重叠的文件会被拒收，同一统计期再传要确认替换', + replace: '每次上传新一期，当前版本只含新的一期，此前各期留在历史版本中', + } as Record, +} as const + +/** 新旧配方对照（P3-SPEC 6.2、9.4）：试运行回执、确认清单顶部、修复预览、重新起草共用 */ +export const COMPARE_TEXT = { + title: '配方前后对照', + none: '配方没有变化', + breakingBadge: '破坏性', + nothing: '无', + tableAdded: (t: string) => `新增表「${t}」`, + tableRemoved: (t: string) => `表「${t}」不再产出（删除或改名）`, + columnAdded: (t: string, c: string, unit: string) => `表「${t}」新增列「${c}」${unit ? `（${unit}）` : ''}`, + columnRemoved: (t: string, c: string) => `表「${t}」删除或改名了列「${c}」`, + unit: (t: string, c: string, from: string, to: string) => `表「${t}」列「${c}」的单位 ${from} → ${to}`, + noUnit: '无单位', + change: (t: string, c: string, what: string, from: string, to: string) => + (from || to ? `表「${t}」列「${c}」的${what} ${from || '无'} → ${to || '无'}` : `表「${t}」列「${c}」的${what}有变化`), + changeKind: { + type: '类型', unit: '单位', source: '来源', store: '文字存法', const_value: '常量取值', placeholder_meaning: '占位符含义', + } as Record, + changeOther: '设置', + grain: (t: string, from: string, to: string) => `表「${t}」的主键 ${from} → ${to}`, + tableKind: (t: string, from: string, to: string) => `表「${t}」的种类 ${from} → ${to}`, + tableKindLabel: { data: '数据表', reported_total: '原表写明的合计' } as Record, + /** 服务端的破坏性原话（不带表名）前面补上是哪张表 */ + breakingLine: (t: string, msg: string) => `表「${t}」：${msg}`, + /** + * 新增、去掉的分段：标题和全部标签一起写在这一条里（list 是已经加好引号的标签）。起草器换了分段 id 时，新分段的 + * 标题和标签就是它新认出的定位规则，确认时得看得到；去掉的分段写原来认的是什么，免得只剩一个 id 看不出丢了哪些行 + */ + segAdded: (id: string, title: string, list: string) => + `新增分段「${id}」${title ? `，标题「${title}」` : ''}${list ? `，标签：${list}` : ''}`, + segRemoved: (id: string, title: string, list: string) => + `去掉分段「${id}」${title ? `，原标题「${title}」` : ''}${list ? `，原有标签：${list}` : ''}`, + segTitle: (id: string, from: string, to: string) => `分段「${id}」的标题「${from}」→「${to}」`, + labelsAdded: (id: string, list: string) => `分段「${id}」加入标签：${list}`, + labelsRemoved: (id: string, list: string) => `分段「${id}」去掉标签：${list}`, + ignoreAdded: (id: string, list: string) => `「${id}」新增忽略规则：${list}`, + ignoreRemoved: (id: string, list: string) => `「${id}」去掉忽略规则：${list}`, + relation: (id: string, from: string, to: string) => `关系 ${id}：${from} → ${to}`, + relationNone: '不登记', + sheetName: (from: string, to: string) => `工作表名「${from}」→「${to}」`, + mode: (from: string, to: string) => `导入模式：${from} → ${to}`, + accumulate: { + compatible: '与当前版本兼容：早期各期没有的列为空值', + retire: '部分列自本期起不再导入，早期各期保留原值', + semantic: '与当前版本不兼容：启用后将重新开始累积', + } as Record, +} as const + +/** + * 回执的只读展示块（ReceiptBlocks.tsx）：「排除的行」、回执摘要。向导回执和版本页的清单视图共用。 + * 排除原因的说法不露枚举值：认不出的原因写「其他原因」 + */ +export const RECEIPT_BLOCK_TEXT = { + excludedReason: { + hidden_excluded: '隐藏的行（按配方不导入）', + blank_skipped: '跳过的空行', + ignored_rows: '按配方忽略的行', + ignored_outside: '按配方忽略了区域外数字的行', + after_stop: '列表结束之后的文字行', + total_not_kept: '只核对、不另存的合计行', + } as Record, + excludedOther: '其他原因', + excludedTitle: '排除的行', + rowSpan: (a: number, b: number) => (a === b ? `第 ${formatNumber(a)} 行` : `第 ${formatNumber(a)}–${formatNumber(b)} 行`), + cells: (n: number) => `${formatNumber(n)} 格`, + groupTotal: (rows: number, cells: number) => `共 ${formatNumber(rows)} 行、${formatNumber(cells)} 格`, + excludedNone: '没有排除的行', + excludedUnrecorded: '这次导入未记录排除的行', + canonicalized: (n: number) => `按规范写法存储的取值 ${formatNumber(n)} 种`, } as const /** 格子的去向（契约 Role）：网格图例、回执里的格子账 */ @@ -1174,6 +1630,8 @@ export const LEDGER_ROLE_LABEL: Record = { hidden_excluded: '排除的隐藏行', total_label: '合计标签', total_value: '合计格', + /** 期 3：按配方忽略的行、区域外数字（ignore_rows、ignore_outside） */ + ignored: '按配方忽略', } /** 核对结果的状态 */ @@ -1218,6 +1676,15 @@ export const DIFF_KIND_LABEL: Record = { outside_moved: '区域外文字挪了位置', ignored_columns: '忽略的列', hidden: '隐藏的行列', + // 期 3：按期累积、忽略规则（P3-SPEC 9.5） + period_added: '新增的一期', + period_replaced: '替换的一期', + period_gap: '各期之间的空缺', + period_backfill: '补传早期', + columns_added: '新增的列', + columns_retired: '不再导入的列', + labels_vary: '各期取值不同', + ignored_rows: '按配方忽略的行', } /** diff --git a/frontend/src/pages/DataSourcesTab.tsx b/frontend/src/pages/DataSourcesTab.tsx index 6c8a3fb2..a9ebaee7 100644 --- a/frontend/src/pages/DataSourcesTab.tsx +++ b/frontend/src/pages/DataSourcesTab.tsx @@ -1,8 +1,8 @@ import { useEffect, useId, useMemo, useRef, useState } from 'react' import { useNavigate } from 'react-router-dom' import { - AlertTriangle, ChevronRight, Database, EyeOff, FileSpreadsheet, Info, KeyRound, Layers, Lock, Plug, Plus, RefreshCw, - Search, Table2, Upload, X, + AlertTriangle, ChevronRight, Database, EyeOff, FileSpreadsheet, History, Info, KeyRound, Layers, Lock, Plug, Plus, + RefreshCw, Search, Table2, Upload, X, } from 'lucide-react' import clsx from 'clsx' import { ApiError, api, uploadDecision } from '../api/client' @@ -21,11 +21,12 @@ import { checkHealth, forgetHealth, healthFromServer, setHealth, useHealth } fro import type { HealthRecord } from '../lib/health' import { workflowList, workflowsMentioning } from '../lib/mentions' import { - RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT, + RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT, VERSIONS_TEXT, } from '../lib/terms' import { useRunClock } from '../run/useRunClock' import { ImportWizard } from './import/ImportWizard' import type { WizardEntry } from './import/ImportWizard' +import { VersionsDialog } from './import/VersionsDialog' // =========================================================================== // 数据源 @@ -183,6 +184,8 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { onRecipeReupload={() => setWizard({ kind: 'reupload', source: row })} onRedraft={() => setWizard({ kind: 'redraft', source: row })} onResume={() => row.open_staging && setWizard({ kind: 'resume', stagingId: row.open_staging.id, source: row })} + // 版本页里清除原件之后服务端不返回数据源，卡片上的原件状态靠重新取列表 + onReload={() => void load()} kick={kick && kick.id === row.id ? kick.seq : 0} /> ))} @@ -257,9 +260,11 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { /** 结构同步超过这么久就提示可能过期：库表会变，Copilot 照旧表写 SQL 只会报错 */ const STALE_SCHEMA_MS = 7 * 24 * 3600_000 -function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onRecipeReupload, onRedraft, onResume, kick }: { +function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onRecipeReupload, onRedraft, onResume, onReload, kick }: { row: any; meta?: any onChange: (row: any) => void + /** 重新取整个列表（版本页里的写操作没有返回数据源时用） */ + onReload: () => void onRemoved: (id: string) => void onEdit: () => void onReupload: () => void @@ -279,6 +284,8 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci // 传上来的表通常只有一两张，直接摊开；库动辄几十上百个对象，默认收着 const [open, setOpen] = useState(() => uploaded && row.table_count > 0 && row.table_count <= 3) const [masking, setMasking] = useState(false) + /** 版本页（当前版本的各期、历史版本、导入记录） */ + const [versions, setVersions] = useState(false) const masks = maskList(row.options?.[MASK_KEY]) const clock = useRunClock(!!busy) const busySince = useRef(0) @@ -476,6 +483,12 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci {RECIPE_TEXT.openStaging} )} + {uploaded && ( + // 上传的表格才有版本：手工登记的库没有导入记录，也没有可回滚的版本 + + )} {uploaded && row.import_mode === 'recipe' ? ( // 按配方导入的源：每月按已确认的配方重放，不走简单上传（服务端也会拒绝） <> @@ -512,6 +525,10 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onReci setMasking(false)} onSaved={(next) => { setMasking(false); onChange(next); syncCatalog() }} /> )} + {versions && ( + // onChange 是父级的「更新这一行并刷新全局目录」：回滚之后表结构可能变了，检查器、问数据读的目录要跟上 + setVersions(false)} onChange={onChange} onReload={onReload} /> + )} {failed && (
@@ -647,21 +664,45 @@ function Address({ row, meta, uploaded }: { row: any; meta?: any; uploaded: bool ) } +/** + * 启用时间比导入时间晚这么多，才算「后来又启用过」（回滚、移除一期复用了旧版本）。新导入的版本创建和启用在 + * 同一次提交里，两个时间只差毫秒级；留一分钟余量，不把同一次提交误写成「…启用」 + */ +const REACTIVATED_AFTER_MS = 60_000 + /** * 上传表格当前启用的是哪一版：哪个文件、什么时候导入的。库文件的路径按版本变（而且不是用户 - * 传的那个文件），写出来没有意义；同名重传之后，这一行变了才说明新版本真的启用了 + * 传的那个文件），写出来没有意义；同名重传之后，这一行变了才说明新版本真的启用了。 + * + * 期 3：在版本页里启用旧版本之后，「3 小时前导入」说的是那个版本当初的导入时间，不是它成为当前版本的时间， + * 这时改写「…启用」（评审三-M8）；按期累积的写期数 */ function UploadVersion({ snap }: { snap: CurrentSnapshot }) { const legacy = isLegacySnapshot(snap) // 迁移补建的版本：created_at 是迁移的时间，写成「刚刚导入」是假话 const at = legacy ? null : parseServerTime(snap.created_at) + const activated = legacy ? null : parseServerTime(snap.activated_at) + const reactivated = !!at && !!activated && activated.getTime() - at.getTime() > REACTIVATED_AFTER_MS const rawNote = snap.raw_state !== 'kept' ? RAW_STATE_LABEL[snap.raw_state] : '' + const title = legacy + ? UPLOAD_TEXT.legacyTitle + : reactivated + ? VERSIONS_TEXT.activatedTitle(formatDateTime(snap.activated_at), formatDateTime(snap.created_at)) + : at ? UPLOAD_TEXT.importedTitle(formatDateTime(snap.created_at)) : undefined return ( - + {UPLOAD_TEXT.currentVersion} {snap.file_name || UPLOAD_TEXT.legacyFile} - {at && · {UPLOAD_TEXT.importedAt(formatRelative(snap.created_at))}} + {reactivated + ? · {VERSIONS_TEXT.activatedAt(formatRelative(snap.activated_at))} + : at && · {UPLOAD_TEXT.importedAt(formatRelative(snap.created_at))}} + {/* 只有服务端写明了导入模式才写：简单导入、期 3 之前的版本没有 mode，不替它们说「每期替换」 */} + {snap.mode && ( + + · {VERSIONS_TEXT.cardSummary(snap.mode, snap.periods)} + + )} {rawNote && · {rawNote}} ) diff --git a/frontend/src/pages/import/AccumulatePlan.tsx b/frontend/src/pages/import/AccumulatePlan.tsx new file mode 100644 index 00000000..c4ba81bc --- /dev/null +++ b/frontend/src/pages/import/AccumulatePlan.tsx @@ -0,0 +1,156 @@ +import type { AccumulatePeriodRef, AccumulatePlan as Plan, CheckResult } from '../../types' +import { formatNumber } from '../../lib/format' +import { ACCUMULATE_TEXT } from '../../lib/terms' +import { CheckStatusChip } from './ImportReceipt' + +// =========================================================================== +// 累积计划（P3-SPEC 2.4、10.2）：启用之后当前版本由哪几期组成、本期是新增还是替换、哪几期会移出当前版本、 +// 行数（本期 / 启用后当前版本），以及整体的结构核对。全部来自服务端的计划，界面不推算：行数照服务端给的写， +// 没有物化（只有一期）时「启用后当前版本」就是本期 +// =========================================================================== + +type Row = AccumulatePeriodRef & { state: 'new' | 'kept' | 'replaced' | 'dropped' } + +const periodKey = (p: { start: string | null; end: string | null }) => `${p.start ?? ''}~${p.end ?? ''}` +const periodText = (p: { start: string | null; end: string | null }) => + (p.start && p.end ? ACCUMULATE_TEXT.period(p.start, p.end) : ACCUMULATE_TEXT.periodUnknown) + +const STATE_LABEL: Record = { + new: ACCUMULATE_TEXT.partNew, + kept: '', + replaced: ACCUMULATE_TEXT.partReplaced, + dropped: ACCUMULATE_TEXT.partDropped, +} + +export function AccumulatePlan({ plan, checks }: { + plan: Plan + /** 并集的结构核对 U1–U3（TrialOut.union_checks；物化了才有） */ + checks?: CheckResult[] | null +}) { + const rejected = plan.action === 'rejected' + // 结果里的各期、被替换的那一期、移出当前版本的各期，按统计期排在一起：一眼看出启用后是哪几期 + const rows: Row[] = [ + ...(plan.parts ?? []).map((p) => ({ ...p, state: (p.new ? 'new' : 'kept') as Row['state'] })), + ...(plan.replaces ? [{ ...plan.replaces, state: 'replaced' as const }] : []), + ...(plan.dropped ?? []).map((p) => ({ ...p, state: 'dropped' as const })), + ].sort((a, b) => (a.start ?? '').localeCompare(b.start ?? '') || (a.state === 'new' ? 1 : -1)) + const current = (plan.parts ?? []).find((p) => p.new) + const after = plan.union?.rows ?? current?.rows ?? {} + const tables = [...new Set([...Object.keys(current?.rows ?? {}), ...Object.keys(after)])] + const blockers = (plan.parts ?? []).filter((p) => p.blockers?.length) + const columnText = (x: { table: string; column: string | null }) => + (x.column ? `「${x.table}」的「${x.column}」` : `「${x.table}」（${ACCUMULATE_TEXT.wholeTable}）`) + + return ( +
+

+ {ACCUMULATE_TEXT.title} + + {ACCUMULATE_TEXT.action[plan.action] ?? plan.action} + +

+ {plan.reason &&

{plan.reason}

} + +
    + {rows.map((p, i) => ( +
  • + {periodText(p)} + {p.seq != null && {ACCUMULATE_TEXT.importSeq(formatNumber(p.seq))}} + {p.file_name && {p.file_name}} + {STATE_LABEL[p.state] && ( + + {STATE_LABEL[p.state]} + + )} +
    • + ))} +
+ + {!rejected && tables.length > 0 && ( + + + + + + + + + + {tables.map((t) => ( + + + + + + ))} + +
{ACCUMULATE_TEXT.table}{ACCUMULATE_TEXT.thisPeriod}{ACCUMULATE_TEXT.afterEnable}
{t}{current?.rows?.[t] != null ? formatNumber(current.rows[t]) : '—'}{after[t] != null ? formatNumber(after[t]) : '—'}
+ )} + + {!!plan.overlaps?.length && ( +
+
{ACCUMULATE_TEXT.overlaps}
+
    + {plan.overlaps.map((o, i) =>
  • {periodText(o)}
    • )} +
+
+ )} + {!!plan.gaps?.length && ( +

{ACCUMULATE_TEXT.gaps(plan.gaps.map(periodText).join('、'))}

+ )} + {plan.backfill &&

{ACCUMULATE_TEXT.backfill}

} + {!!plan.semantic?.length && ( +
+
{ACCUMULATE_TEXT.semantic}
+
    {plan.semantic.map((s, i) =>
  • {s}
    • )}
+
+ )} + {plan.action === 'restart' &&

{ACCUMULATE_TEXT.restartHint}

} + {blockers.map((p, i) => ( +

+ {ACCUMULATE_TEXT.blockers(periodText(p), p.blockers.map((b) => b.message).join('；'))} +

+ ))} + {!!plan.added?.length && ( +

{ACCUMULATE_TEXT.added}：{plan.added.map(columnText).join('、')}

+ )} + {!!plan.retired_new?.length && ( +

{ACCUMULATE_TEXT.retired}：{plan.retired_new.map(columnText).join('、')}

+ )} + {!!plan.label_sets?.length && ( +
+
{ACCUMULATE_TEXT.labelSets}
+
    + {plan.label_sets.flatMap((l) => l.periods.flatMap((p, j) => [ + ...(p.missing.length ? [
  • + 「{l.column}」{ACCUMULATE_TEXT.labelMissing(periodText(p), p.missing.map((x) => `「${x}」`).join('、'))}
    • ] : []), + ...(p.extra.length ? [
  • + 「{l.column}」{ACCUMULATE_TEXT.labelExtra(periodText(p), p.extra.map((x) => `「${x}」`).join('、'))}
    • ] : []), + ]))} +
+
+ )} + {!!checks?.length && ( +
+
{ACCUMULATE_TEXT.checks}
+
    + {checks.map((c) => ( +
  • +
    + + {c.title} +
    + {!!c.details?.length &&
      {c.details.slice(0, 4).map((d, i) =>
    • {d}
      • )}
    } +
    • + ))} +
+
+ )} +
+ ) +} diff --git a/frontend/src/pages/import/ConfirmList.tsx b/frontend/src/pages/import/ConfirmList.tsx index 0c100773..fd09a80f 100644 --- a/frontend/src/pages/import/ConfirmList.tsx +++ b/frontend/src/pages/import/ConfirmList.tsx @@ -1,32 +1,52 @@ import { useEffect, useId, useMemo, useState } from 'react' import { AlertTriangle } from 'lucide-react' import clsx from 'clsx' -import type { CheckResult, ConfirmItem } from '../../types' +import type { CheckResult, ConfirmItem, FixProposal, PriorAcceptance } from '../../types' import { Spinner } from '../../components/ui' import { localActor } from '../../lib/actor' -import { PROBLEM_CATEGORY_LABEL, RECIPE_TEXT } from '../../lib/terms' -import { CheckStatusChip } from './ImportReceipt' +import { CONFIRM_SOURCE_LABEL, PROBLEM_CATEGORY_LABEL, RECIPE_TEXT } from '../../lib/terms' +import { FixButtons } from './FixPanel' +import { CheckStatusChip, PriorAcceptances, renameFixIds } from './ImportReceipt' // =========================================================================== // 启用前的确认清单：逐条勾选（不提供「全选」），可接受的核对逐条写理由，署名（未认证）。 // 必勾的集合由服务端按试运行结果重算，少勾一项服务端照样拒绝（422 confirm_required） // =========================================================================== -/** 单位变化排最前（数量级变了，引用它的报告口径会变），其次是破坏性变更和差异卡里需确认的项 */ -function rank(id: string): number { +/** 和破坏性变更同一档的确认项（P3-SPEC 9.5）：都会改变当前版本的内容或口径 */ +const TIER1 = ['breaking:', 'mode_switch:', 'retire:', 'period_replace:'] +const TIER1_IDS = new Set(['switch_from_simple', 'accumulate_restart', 'accumulate_unit_risk', 'redraft_adopted']) + +/** + * 排序（P3-SPEC 9.5）：单位变化排最前（数量级变了，引用它的报告口径会变）；破坏性变更、切换、重新开始累积、 + * 模式切换、退役、替换该期、单位口径风险、采用重新起草同一档；然后是差异卡里需确认的项；其余在后 + */ +export function rank(id: string): number { if (id.startsWith('unit_changed:')) return 0 - if (id.startsWith('breaking:') || id === 'switch_from_simple') return 1 + if (TIER1_IDS.has(id) || TIER1.some((p) => id.startsWith(p))) return 1 if (id.startsWith('diff:')) return 2 return 3 } +/** 分组的先后：组里最靠前的那一项的档次在前，同档按配方、修改、累积、本期、差异、切换 */ +const GROUP_ORDER = ['recipe', 'edit', 'accumulate', 'current', 'diff', 'switch'] +/** 区域外文字、工作表、统计期都是「本期」才有的项，合成一组 */ +const groupOf = (it: ConfirmItem) => { + const s = it.source ?? 'recipe' + return s === 'outside' || s === 'sheet' || s === 'context' ? 'current' : GROUP_ORDER.includes(s) ? s : 'recipe' +} +const GROUP_LABEL: Record = { + recipe: CONFIRM_SOURCE_LABEL.recipe, edit: CONFIRM_SOURCE_LABEL.edit, accumulate: CONFIRM_SOURCE_LABEL.accumulate, + current: CONFIRM_SOURCE_LABEL.outside, diff: CONFIRM_SOURCE_LABEL.diff, switch: CONFIRM_SOURCE_LABEL.switch, +} + export interface CommitBody { confirmations: string[] acceptances: { check_id: string; reason: string }[] signed_by: string | null } -export function ConfirmList({ items, checks, acceptable, reupload, busy, missing, error, onSubmit }: { +export function ConfirmList({ items, checks, acceptable, reupload, busy, missing, error, blocked = false, prior, fixes, sheets, onFix, onSubmit }: { items: ConfirmItem[] checks: CheckResult[] /** 可写理由接受、本次未通过的核对 id */ @@ -37,10 +57,32 @@ export function ConfirmList({ items, checks, acceptable, reupload, busy, missing /** 服务端说还没勾的项（422 confirm_required / acceptance_required 的 detail 里点名的） */ missing: Set error: string | null + /** 重试也会失败（build_conflict）：「确认并启用」一直禁用，等管理员处理 */ + blocked?: boolean + /** 同一份构建以前被接受过的理由：只读显示在对应的核对旁，理由框不预填 */ + prior?: PriorAcceptance[] + /** 工作表改名的确认项旁放「更新工作表名」（修复提议的 anchor.kind=sheet_renamed） */ + fixes?: FixProposal[] + /** + * 试运行回执的 sheets：matched 是「配方里的工作表 id → 本期工作表名」，renamed 是「配方里的名字 → 本期的名字」。 + * 确认项 sheet_renamed: 的 sid 一般是配方里的工作表 id，查 matched；配方里找不到那张表时服务端退回用旧名当 sid， + * 查 renamed。查到的本期表名再和提议的 anchor.sheet 对，只放这张表的那一个按钮 + */ + sheets?: { matched?: Record; renamed?: Record } | null + onFix?: (id: string) => void onSubmit: (body: CommitBody) => void }) { - const sorted = useMemo(() => items.map((it, i) => ({ it, i })) - .sort((a, b) => rank(a.it.id) - rank(b.it.id) || a.i - b.i).map((x) => x.it), [items]) + // 先按来源分组，组内按档次、再按服务端给的先后；组的先后按组里最靠前的一项（单位变化所在的组排第一） + const groups = useMemo(() => { + const by = new Map() + items.forEach((it, i) => { const g = groupOf(it); by.set(g, [...(by.get(g) ?? []), { it, i }]) }) + return [...by.entries()] + .map(([g, list]) => ({ g, list: list.sort((a, b) => rank(a.it.id) - rank(b.it.id) || a.i - b.i).map((x) => x.it) })) + .sort((a, b) => rank(a.list[0].id) - rank(b.list[0].id) || GROUP_ORDER.indexOf(a.g) - GROUP_ORDER.indexOf(b.g)) + }, [items]) + const sorted = useMemo(() => groups.flatMap((x) => x.list), [groups]) + const renameFix = useMemo(() => new Map((fixes ?? []).map((f) => [f.id, f])), [fixes]) + const renameIds = (sid: string) => renameFixIds(fixes, sheets?.matched?.[sid] ?? sheets?.renamed?.[sid]) const [checked, setChecked] = useState>(() => new Set()) const [reasons, setReasons] = useState>({}) const [signer, setSigner] = useState(() => localActor() ?? '') @@ -57,7 +99,7 @@ export function ConfirmList({ items, checks, acceptable, reupload, busy, missing const v = (reasons[id] ?? '').trim() return v.length > 0 && v.length <= 500 }) - const ready = allChecked && allReasons && !busy + const ready = allChecked && allReasons && !busy && !blocked const submit = () => { if (!ready) return @@ -81,32 +123,42 @@ export function ConfirmList({ items, checks, acceptable, reupload, busy, missing {error}
)} -
    - {sorted.map((it) => { - const miss = missing.has(it.id) - return ( -
  • - -
    • - ) - })} -
+ {groups.map(({ g, list }) => ( +
+

{GROUP_LABEL[g]}

+
    + {list.map((it) => { + const miss = missing.has(it.id) + return ( +
  • + + {it.id.startsWith('sheet_renamed:') && ( +
    + +
    + )} +
    • + ) + })} +
+
+ ))} {accepts.length > 0 && (

{RECIPE_TEXT.acceptTitle}

@@ -122,6 +174,7 @@ export function ConfirmList({ items, checks, acceptable, reupload, busy, missing {check?.title ?? id} {check &&
{PROBLEM_CATEGORY_LABEL[check.category] ?? ''}
} + p.check_id === id)} />