diff --git a/backend/app/api/datasources.py b/backend/app/api/datasources.py index 49944e9c..4f7d72ce 100644 --- a/backend/app/api/datasources.py +++ b/backend/app/api/datasources.py @@ -17,6 +17,7 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from app.api import health +from app.api.coded import CodedHTTPException from app.core.errors import AUTH, NETWORK, TIMEOUT, classify, explain, raw from app.core.config import settings from app.core.crypto import encrypt, mask @@ -27,7 +28,7 @@ from app.data.engine import ( engine_args, engines, mask_columns_problem, query_timeout_problem, ) from app.db.base import get_session, new_id -from app.db.models import DataSource, SourceSnapshot, TableImport +from app.db.models import DataSource, ImportStaging, SourceSnapshot, TableImport, TableRecipe from app.tools.datasource import tool_names router = APIRouter(prefix="/api/datasources", tags=["datasources"]) @@ -100,6 +101,12 @@ class DataSourceOut(BaseModel): origin: str = "manual" #: 上传表格当前启用的版本：{id, created_at, file_name, raw_state}。手工源恒为 None current_snapshot: dict[str, Any] | None = None + #: simple：简单导入的上传表格；recipe：按配方导入；手工源为 None + import_mode: str | None = None + #: 按配方导入的源当前启用的配方：{id, seq, origin, activated_at, signed_by} + current_recipe: dict[str, Any] | None = None + #: 这个源最近一个未结束的导入暂存区：{id, kind, status, created_at} + open_staging: dict[str, Any] | None = None def _cached_schema(cache: dict[str, Any]) -> str | None: @@ -108,11 +115,21 @@ def _cached_schema(cache: dict[str, Any]) -> str | None: return str(cache.get("schema") or "") -def _to_out(row: DataSource, snapshot: dict[str, Any] | None = None) -> DataSourceOut: +def _import_mode(row: DataSource) -> str | None: + if row.current_recipe_id: + return "recipe" + return "simple" if _is_upload(row) else None + + +def _to_out(row: DataSource, snapshot: dict[str, Any] | None = None, + recipe_info: dict[str, Any] | None = None) -> DataSourceOut: cache = row.schema_cache or {} tables = cache.get("tables") or {} + recipe_info = recipe_info or {} return DataSourceOut( origin=row.origin or "manual", current_snapshot=snapshot, + import_mode=_import_mode(row), current_recipe=recipe_info.get("current_recipe"), + open_staging=recipe_info.get("open_staging"), cached_schema=_cached_schema(cache), schema_error=str(cache.get("error") or "") if cache.get("failed") else "", available_schemas=list(cache.get("available_schemas") or []), @@ -183,8 +200,41 @@ async def _snapshot_infos(session: AsyncSession, rows: list[DataSource]) -> dict return out +def _iso(v: datetime | None) -> str | None: + return v.isoformat() if v else None + + +async def _recipe_infos(session: AsyncSession, rows: list[DataSource]) -> dict[str, dict[str, Any]]: + """各源的当前配方摘要和最近一个未结束的暂存区：{源 id: {current_recipe, open_staging}}。两次查询取齐。""" + out: dict[str, dict[str, Any]] = {} + recipe_ids = {r.current_recipe_id: r.id for r in rows if r.current_recipe_id} + if recipe_ids: + for rec in (await session.execute( + select(TableRecipe).where(TableRecipe.id.in_(list(recipe_ids))) + )).scalars(): + sid = recipe_ids.get(rec.id) + if sid is not None: + out.setdefault(sid, {})["current_recipe"] = { + "id": rec.id, "seq": rec.seq, "origin": rec.origin, + "activated_at": _iso(rec.activated_at), "signed_by": rec.signed_by, + } + ids = [r.id for r in rows if _is_upload(r)] + if ids: + for st in (await session.execute( + select(ImportStaging).where( + ImportStaging.source_id.in_(ids), ImportStaging.status.in_(table_versions.STAGING_OPEN), + ).order_by(ImportStaging.created_at) + )).scalars(): + # 按创建时间升序，后面的覆盖前面的：留下最近的那一个 + out.setdefault(st.source_id, {})["open_staging"] = { + "id": st.id, "kind": st.kind, "status": st.status, "created_at": _iso(st.created_at), + } + return out + + async def _out(session: AsyncSession, row: DataSource) -> DataSourceOut: - return _to_out(row, (await _snapshot_infos(session, [row])).get(row.id)) + return _to_out(row, (await _snapshot_infos(session, [row])).get(row.id), + (await _recipe_infos(session, [row])).get(row.id)) #: 上传源上由系统维护的字段：连接一律从快照解析，database 只作显示 @@ -252,7 +302,8 @@ def _locked_refusal(changed: list[str]) -> str: async def list_sources(session: AsyncSession = Depends(get_session)) -> list[DataSourceOut]: rows = list((await session.execute(select(DataSource).order_by(DataSource.name))).scalars()) snaps = await _snapshot_infos(session, rows) - return [_to_out(r, snaps.get(r.id)) for r in rows] + recipes = await _recipe_infos(session, rows) + return [_to_out(r, snaps.get(r.id), recipes.get(r.id)) for r in rows] #: 每种库都有的「查询时限」。数据库按它自己停下语句，不只是后端不再等 @@ -785,6 +836,22 @@ def _table_out(t: dict[str, Any]) -> dict[str, Any]: } +async def read_upload(file: UploadFile) -> bytes: + """读上传的文件，边读边数，超了立刻停（413 file_too_large）。简单上传和按配方导入共用。 + + 和知识库上传同一套：那个检查拦的是「入库」，不是「占内存」，读完再判等于先把内存吃掉。 + """ + limit = settings.max_upload_mb * 1024 * 1024 + pieces: list[bytes] = [] + total = 0 + while piece := await file.read(1024 * 1024): + total += len(piece) + if total > limit: + raise CodedHTTPException(413, f"文件超过 {settings.max_upload_mb} MB", "file_too_large") + pieces.append(piece) + return b"".join(pieces) + + @router.post("/upload", response_model=dict, status_code=201) async def upload_table( file: UploadFile = File(...), @@ -823,21 +890,14 @@ async def upload_table( if mixed not in _MIXED_CHOICES: raise HTTPException(400, "数字列混入非数字时，只能选择拒收或将非数字的值存为空值") - # 边读边数，超了立刻停。和知识库上传同一套：那个检查拦的是"入库"， - # 不是"占内存"，读完再判等于先把内存吃掉 - limit = settings.max_upload_mb * 1024 * 1024 - pieces: list[bytes] = [] - total = 0 - while piece := await file.read(1024 * 1024): - total += len(piece) - if total > limit: - raise HTTPException(413, f"文件超过 {settings.max_upload_mb} MB") - pieces.append(piece) - raw = b"".join(pieces) + raw = await read_upload(file) existing = (await session.execute( select(DataSource).where(DataSource.name == name) )).scalar_one_or_none() + # 按配方导入的源：重传要按已确认的配方重放（上传新一期），不能退回简单解析把配方和核对绕过去 + if existing is not None and existing.current_recipe_id: + raise CodedHTTPException(409, f"「{name}」按配方导入，请使用「上传新一期」", "recipe_source") # 同名的只有上传表格可以更新。迁移前的老上传（路径还在上传目录里）也算 if existing is not None and not _is_upload(existing) and not ( existing.kind == "sqlite" and table_versions.under_uploads(existing.database) @@ -891,6 +951,9 @@ def _import_out(imp: TableImport, current: set[str]) -> dict[str, Any]: "current": imp.id in current, "created_at": imp.created_at.isoformat() if imp.created_at else None, "activated_at": imp.activated_at.isoformat() if imp.activated_at else None, + # 按配方导入才有（简单导入为空）：配方、统计期、署名，接受的条数 + "recipe_id": imp.recipe_id, "period_start": imp.period_start, "period_end": imp.period_end, + "signed_by": imp.signed_by, "overrides": len(imp.overrides or []), "waivers": len(imp.waivers or []), } @@ -941,8 +1004,9 @@ async def purge_raw( raise HTTPException(409, "这次导入没有保存原件") if imp.raw_state == "purged": raise HTTPException(409, "这次导入的原件已经清除") - deleted, others = await table_versions.purge_import_raw( + purged = await table_versions.purge_import_raw( session, imp, reason=reason, signed_by=payload.signed_by) + deleted, others = purged.deleted, purged.others current = await _current_imports(session, row) names = dict((await session.execute( select(DataSource.id, DataSource.name).where(DataSource.id.in_({o.source_id for o in others})) @@ -955,4 +1019,6 @@ async def purge_raw( "seq": o.seq, "file_name": o.file_name} for o in others ], + # 引用同一份原件、随之一并放弃的未完成导入（暂存区，已清空） + "discarded_stagings": [st.id for st in purged.discarded_stagings], } diff --git a/backend/app/api/table_imports.py b/backend/app/api/table_imports.py new file mode 100644 index 00000000..f519a4aa --- /dev/null +++ b/backend/app/api/table_imports.py @@ -0,0 +1,400 @@ +"""按配方导入的接口（期 2，WP-5c；P2-SPEC 7.3）：暂存、回答问题、改配方、AI 起草、试运行、提交、放弃， +以及「上传新一期」「修改配方」「查看当前配方」。 + +流程和状态都在 app.data.recipe_imports 里，这里只做三件事：解析请求、取署名、把业务错误转成带机读码的 +CodedHTTPException（body 是 {"detail": 中文, "code": 机读码}，前端按 code 分支）。 + +**请求模型里不放会触发 FastAPI 自带 422 的必填项**：它的 body 是 {detail: [...]}、不带 code，前端没法分支。 +所以 JSON 请求体一律手工读成 dict（不是 JSON 对象才回 422 body_invalid），各字段由业务代码检查。 + +路由路径都以 /imports/ 开头，或是 /{source_id}/ 后面跟固定的 reupload、redraft、recipe，与 datasources.py +现有的 /{source_id}、/{source_id}/test|introspect|schema|imports 不重叠；在 main.py 里登记在 datasources +之前，免得 /imports/… 被通配的 {source_id} 先截走。 +""" +from __future__ import annotations + +import datetime as _dt +import logging +import re +from typing import Any + +from fastapi import APIRouter, Depends, File, Form, Header, Request, UploadFile +from fastapi.responses import JSONResponse, Response +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.coded import CodedHTTPException +from app.api.datasources import _NAME_PATTERN, _out, read_upload +from app.api.runs import actor_of +from app.data import recipe_imports, table_versions +from app.data.recipe_imports import ImportRefused +from app.data.recipe_types import Acceptance, AiAvailability, PeriodInput, Recipe +from app.db.base import get_session +from app.db.models import DataSource, ImportStaging, TableRecipe + +router = APIRouter(prefix="/api/datasources", tags=["table-imports"]) +logger = logging.getLogger(__name__) + +_NAME_INVALID = "名称必须以小写字母开头，只能包含字母、数字和下划线（名称会成为工具名的一部分）" +_SIGNED_MAX = 100 + + +def _refused(e: ImportRefused) -> CodedHTTPException: + return CodedHTTPException(e.status, e.message, e.code) + + +def _signed(body: dict[str, Any] | None, header: str | None) -> str | None: + """署名：请求体的 signed_by 优先，其次请求头 X-Actor。只原样记下（未认证）。""" + raw = (body or {}).get("signed_by") if isinstance(body, dict) else None + if isinstance(raw, str) and raw.strip(): + return raw.strip()[:_SIGNED_MAX] + return actor_of(header) + + +#: JSON 请求体的字节上限和嵌套层数上限（SE-5）。最大的合法配方（200 个标签、几十个分段）也只有一两百 KB、 +#: 十来层；更大更深的不是配方，照存会让静态校验跑几十秒、占着解析名额，深层嵌套还会让响应序列化失败 +BODY_MAX_BYTES = 1_000_000 +BODY_MAX_DEPTH = 32 + + +def _json_depth_over(raw: bytes, limit: int) -> bool: + depth = 0 + in_str = esc = False + for ch in raw.decode("utf-8", errors="replace"): + if in_str: + if esc: + esc = False + elif ch == "\\": + esc = True + elif ch == '"': + in_str = False + elif ch == '"': + in_str = True + elif ch in "[{": + depth += 1 + if depth > limit: + return True + elif ch in "]}": + depth -= 1 + return False + + +async def _body(request: Request) -> dict[str, Any]: + """JSON 请求体。空 body 当作 {}；超过字节上限回 413 body_too_large；不是 JSON 对象、嵌套过深回 422 body_invalid。""" + length = request.headers.get("content-length") + if length and length.isdigit() and int(length) > BODY_MAX_BYTES: + raise CodedHTTPException(413, f"请求体超过 {BODY_MAX_BYTES // 1_000_000} MB 的上限", "body_too_large") + raw = await request.body() + if len(raw) > BODY_MAX_BYTES: + raise CodedHTTPException(413, f"请求体超过 {BODY_MAX_BYTES // 1_000_000} MB 的上限", "body_too_large") + if not raw.strip(): + return {} + if _json_depth_over(raw, BODY_MAX_DEPTH): + raise CodedHTTPException(422, f"请求体的 JSON 嵌套超过 {BODY_MAX_DEPTH} 层", "body_invalid") + try: + data = await request.json() + except (ValueError, RecursionError): + data = None + if not isinstance(data, dict): + raise CodedHTTPException(422, "请求体必须是一个 JSON 对象", "body_invalid") + return data + + +async def _staging(session: AsyncSession, staging_id: str) -> ImportStaging: + row = await session.get(ImportStaging, staging_id) + if row is None: + raise CodedHTTPException(404, "这次导入不存在，可能已过期并被清理。请重新开始导入", "staging_not_found") + return row + + +async def _open_staging(session: AsyncSession, staging_id: str) -> ImportStaging: + """写操作要的暂存区：先让过期的过期（不能指望只有回收时才过期），再要求它未结束。""" + await table_versions.expire_stagings(session) + row = await _staging(session, staging_id) + if not table_versions.staging_is_open(row): + raise CodedHTTPException(409, recipe_imports._CLOSED, "staging_closed") + return row + + +async def _source(session: AsyncSession, source_id: str) -> DataSource: + row = await session.get(DataSource, source_id) + if row is None: + raise CodedHTTPException(404, "数据源不存在，可能已被删除", "source_not_found") + return row + + +async def _out_of(session: AsyncSession, staging: ImportStaging) -> dict[str, Any]: + """StagingOut。提供 AI 起草时才去查模型接入的可用性（不提供时不必多一次查询）。""" + source = await session.get(DataSource, staging.source_id) + pipe = recipe_imports.pipeline() + ai: AiAvailability | None = None + if table_versions.staging_is_open(staging) and recipe_imports.ai_offered(staging): + try: + ai = await pipe.ai_availability(session) + except Exception as e: # noqa: BLE001 - 可用性只是提示，查不到按不可用显示 + logger.warning("ai availability failed: %s", type(e).__name__) + ai = AiAvailability(False, reason="暂时无法确认模型接入是否可用。请到「设置 → 模型接入」检查") + return recipe_imports.staging_out(staging, source, ai=ai, units=tuple(pipe.units) or None) + + +def _unreadable(e: Exception) -> CodedHTTPException: + return CodedHTTPException(400, str(e) or "无法读取这个文件，请确认是未加密的 Excel 文件", "file_unreadable") + + +# -------------------------------------------------------------------------- +# 暂存、查看、放弃 +# -------------------------------------------------------------------------- + + +@router.post("/imports/stage", status_code=201) +async def stage( + file: UploadFile | None = File(None), + name: str = Form(""), + description: str = Form(""), + signed_by: str = Form(""), + x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session), +) -> Any: + """选文件开始一次按配方导入：原件存档、扫描、网格预览、系统发现、规则起草。新名字 kind=first； + 同名的简单导入源 kind=switch。""" + from app.data.tabular import UnsupportedTable + + if file is None: + raise CodedHTTPException(422, "请选择要导入的文件", "body_invalid") + if not re.match(_NAME_PATTERN, name or ""): + raise CodedHTTPException(400, _NAME_INVALID, "name_invalid") + raw = await read_upload(file) + try: + staging = await recipe_imports.stage_upload( + session, raw=raw, filename=file.filename or name, name=name, description=description or "", + signed_by=_signed({"signed_by": signed_by}, x_actor)) + except ImportRefused as e: + raise _refused(e) from e + except UnsupportedTable as e: + raise _unreadable(e) from e + return JSONResponse(status_code=201, content=await _out_of(session, staging)) + + +@router.get("/imports/{staging_id}") +async def get_staging(staging_id: str, session: AsyncSession = Depends(get_session)) -> Any: + await table_versions.expire_stagings(session) + return await _out_of(session, await _staging(session, staging_id)) + + +@router.delete("/imports/{staging_id}", status_code=204) +async def discard(staging_id: str, session: AsyncSession = Depends(get_session)) -> Response: + staging = await _open_staging(session, staging_id) + try: + await recipe_imports.discard_staging(session, staging) + except ImportRefused as e: + raise _refused(e) from e + return Response(status_code=204) + + +# -------------------------------------------------------------------------- +# 回答问题、改配方 +# -------------------------------------------------------------------------- + + +@router.post("/imports/{staging_id}/answers") +async def answers(staging_id: str, request: Request, session: AsyncSession = Depends(get_session)) -> Any: + body = await _body(request) + given = body.get("answers", {}) + if not isinstance(given, dict): + raise CodedHTTPException(422, "回答必须是「问题 → 选项」的对象", "body_invalid") + staging = await _open_staging(session, staging_id) + try: + staging = await recipe_imports.answer(session, staging, given) + except ImportRefused as e: + raise _refused(e) from e + return await _out_of(session, staging) + + +@router.put("/imports/{staging_id}/recipe") +async def put_recipe(staging_id: str, request: Request, session: AsyncSession = Depends(get_session)) -> Any: + body = await _body(request) + recipe = body.get("recipe") + if not isinstance(recipe, dict): + raise CodedHTTPException(422, "配方必须是一个 JSON 对象", "body_invalid") + staging = await _open_staging(session, staging_id) + try: + staging = await recipe_imports.put_recipe(session, staging, recipe, origin="manual") + except ImportRefused as e: + raise _refused(e) from e + return await _out_of(session, staging) + + +# -------------------------------------------------------------------------- +# AI 起草 +# -------------------------------------------------------------------------- + + +@router.get("/imports/{staging_id}/draft-ai/preview") +async def draft_ai_preview(staging_id: str, session: AsyncSession = Depends(get_session)) -> Any: + staging = await _open_staging(session, staging_id) + try: + return await recipe_imports.ai_preview(session, staging) + except ImportRefused as e: + raise _refused(e) from e + + +@router.post("/imports/{staging_id}/draft-ai") +async def draft_ai(staging_id: str, request: Request, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> Any: + body = await _body(request) + sha = body.get("preview_sha256") + staging = await _open_staging(session, staging_id) + try: + staging = await recipe_imports.run_ai_draft( + session, staging, consent=body.get("consent") is True, + preview_sha256=sha if isinstance(sha, str) else None, signed_by=_signed(body, x_actor)) + except ImportRefused as e: + raise _refused(e) from e + return await _out_of(session, staging) + + +# -------------------------------------------------------------------------- +# 试运行、提交 +# -------------------------------------------------------------------------- + + +def _context_inputs(raw: Any, signed_by: str | None) -> dict[str, PeriodInput] | None: + """{"统计期": {"start": "2026-09-01", "end": "2026-09-30"}} → PeriodInput。不合法回 422 context_invalid。""" + if raw is None or raw == {}: + return None + bad = CodedHTTPException(422, "统计期的录入不对：请填写起止日期（YYYY-MM-DD），起点不能晚于终点", "context_invalid") + if not isinstance(raw, dict): + raise bad + out: dict[str, PeriodInput] = {} + for key, value in raw.items(): + if key != recipe_imports.PERIOD_KEY or not isinstance(value, dict): + raise bad + try: + start = _dt.date.fromisoformat(str(value.get("start") or "")) + end = _dt.date.fromisoformat(str(value.get("end") or "")) + except ValueError: + raise bad from None + if start > end: + raise bad + who = value.get("signed_by") + out[key] = PeriodInput(start=start, end=end, + signed_by=(str(who).strip()[:_SIGNED_MAX] if isinstance(who, str) and who.strip() + else signed_by)) + return out + + +@router.post("/imports/{staging_id}/trial") +async def trial(staging_id: str, request: Request, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> Any: + body = await _body(request) + who = _signed(body, x_actor) + inputs = _context_inputs(body.get("context_inputs"), who) + staging = await _open_staging(session, staging_id) + try: + staging = await recipe_imports.run_trial(session, staging, context_inputs=inputs, signed_by=who) + except ImportRefused as e: + raise _refused(e) from e + return await _out_of(session, staging) + + +def _acceptances(raw: Any, who: str | None) -> list[Acceptance]: + if raw is None: + return [] + if not isinstance(raw, list): + raise CodedHTTPException(422, "接受理由必须是「核对 → 理由」的列表", "body_invalid") + out = [] + for a in raw: + if not isinstance(a, dict) or not isinstance(a.get("check_id"), str): + raise CodedHTTPException(422, "接受理由必须是「核对 → 理由」的列表", "body_invalid") + reason = a.get("reason") + out.append(Acceptance(check_id=a["check_id"], reason=reason if isinstance(reason, str) else "", + signed_by=who)) + return out + + +@router.post("/imports/{staging_id}/commit", status_code=201) +async def commit(staging_id: str, request: Request, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> Any: + body = await _body(request) + who = _signed(body, x_actor) + confirmations = body.get("confirmations", []) + if not isinstance(confirmations, list) or not all(isinstance(x, str) for x in confirmations): + raise CodedHTTPException(422, "确认项必须是 id 的列表", "body_invalid") + trial_id = body.get("trial_id") + staging = await _open_staging(session, staging_id) + try: + result = await recipe_imports.commit_staging( + session, staging, trial_id=trial_id if isinstance(trial_id, str) else "", + confirmations=confirmations, acceptances=_acceptances(body.get("acceptances"), who), signed_by=who) + except ImportRefused as e: + raise _refused(e) from e + source = await session.get(DataSource, result.source_id, populate_existing=True) + return JSONResponse(status_code=201, content={ + "source": (await _out(session, source)).model_dump() if source is not None else None, + "import_id": result.import_id, "snapshot_id": result.snapshot_id, "build_id": result.build_id, + "recipe_id": result.recipe_id, "build_reused": result.build_reused, "unchanged": result.unchanged, + }) + + +# -------------------------------------------------------------------------- +# 上传新一期、修改配方、当前配方 +# -------------------------------------------------------------------------- + + +@router.post("/{source_id}/reupload", status_code=201) +async def reupload( + source_id: str, + file: UploadFile | None = File(None), + signed_by: str = Form(""), + x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session), +) -> Any: + """上传新一期：按当前配方自动试运行（不调模型），回来时已带回执和差异卡。""" + from app.data.tabular import UnsupportedTable + + source = await _source(session, source_id) + if not source.current_recipe_id: + raise CodedHTTPException(409, recipe_imports._NOT_RECIPE, "not_recipe_source") + if file is None: + raise CodedHTTPException(422, "请选择要导入的文件", "body_invalid") + raw = await read_upload(file) + try: + staging = await recipe_imports.reupload(session, source, raw=raw, filename=file.filename or source.name, + signed_by=_signed({"signed_by": signed_by}, x_actor)) + except ImportRefused as e: + raise _refused(e) from e + except UnsupportedTable as e: + raise _unreadable(e) from e + return JSONResponse(status_code=201, content=await _out_of(session, staging)) + + +@router.post("/{source_id}/redraft", status_code=201) +async def redraft(source_id: str, request: Request, x_actor: str | None = Header(default=None), + session: AsyncSession = Depends(get_session)) -> Any: + from app.data.tabular import UnsupportedTable + + body = await _body(request) + source = await _source(session, source_id) + try: + staging = await recipe_imports.redraft(session, source, signed_by=_signed(body, x_actor)) + except ImportRefused as e: + raise _refused(e) from e + except UnsupportedTable as e: + raise _unreadable(e) from e + return JSONResponse(status_code=201, content=await _out_of(session, staging)) + + +@router.get("/{source_id}/recipe") +async def get_recipe(source_id: str, session: AsyncSession = Depends(get_session)) -> Any: + source = await _source(session, source_id) + row = await session.get(TableRecipe, source.current_recipe_id) if source.current_recipe_id else None + if row is None: + raise CodedHTTPException(409, recipe_imports._NOT_RECIPE, "not_recipe_source") + try: + full = Recipe.model_validate(row.recipe).model_dump(mode="json") + except Exception: # noqa: BLE001 - 存下的配方都是校验过的；万一读不出，原样给出 + full = row.recipe + return { + "recipe_id": row.id, "seq": row.seq, "origin": row.origin, "recipe": full, + "recipe_sha256": row.recipe_sha256, "confirmations": list(row.confirmations or []), + "signed_by": row.signed_by, "activated_at": row.activated_at.isoformat() if row.activated_at else None, + } diff --git a/backend/app/data/engine.py b/backend/app/data/engine.py index 395e8216..925166eb 100644 --- a/backend/app/data/engine.py +++ b/backend/app/data/engine.py @@ -249,6 +249,25 @@ def sqlite_dqs_self_check() -> None: raise RuntimeError(f"SQLite 自检失败：无法确认写错的双引号列名会报错，服务不启动。{problem}") +def open_checked_sqlite(path: str, *, readonly: bool = True) -> sqlite3.Connection: + """同步打开一个本地 SQLite 文件，DQS 已关（配方导入的建库、核对用）。 + + 配方执行器建临时库、核对模块跑 SQL 都不经 SQLAlchemy，直接用 sqlite3。它们同样要关 DQS： + 核对 SQL 里写错一个双引号列名，在默认连接上会被当成字符串常量，`TOTAL("客流")` 得 0、 + `WHERE "时段类别" = ?` 一行都对不上，于是「合计与明细一致」或「不一致」都可能是假的。 + 所以和连接层共用 _DqsOffConnection：关不掉就抛 SqliteHardeningError，连接不交出去。 + + readonly=True 以 mode=ro 打开（核对只读，文件不存在直接报错，不会悄悄建一个空库）； + readonly=False 以 mode=rwc 打开（执行器建库）。不设日志模式：沿用 SQLite 默认的回滚日志， + 不开 WAL——WAL 会改写文件头，库文件哈希就对不上了（P2-SPEC 4.2 第 9 步）。 + 路径按 URI 转义：文件名里的空格、中文、「?」「#」都不会被当成 URI 的参数或片段。 + """ + from urllib.parse import quote + + mode = "ro" if readonly else "rwc" + return sqlite3.connect(f"file:{quote(os.fspath(path))}?mode={mode}", uri=True, factory=_DqsOffConnection) + + def _sqlite_extra() -> dict[str, Any]: """每个 SQLite engine 都带的连接参数：关 DQS。""" return {"connect_args": {"factory": _DqsOffConnection}} diff --git a/backend/app/data/recipe.py b/backend/app/data/recipe.py new file mode 100644 index 00000000..98f8afbb --- /dev/null +++ b/backend/app/data/recipe.py @@ -0,0 +1,1144 @@ +"""配方语言的静态校验、规范形式与哈希、JSON Patch（期 2，WP-1）。 + +配方保存时（起草、回答问题、PUT、AI 草稿）都过这里：schema 由契约里的 pydantic 模型保证（recipe_types， +extra="forbid"、取值能枚举的一律 Literal），这里补上 pydantic 表达不了的语义规则（P2-SPEC 2.2 的 13 条）。 +problems 为空才能试运行；validate_recipe 不抛异常，问题按「位置说明：原因」的整句给人看，path 是 JSON +Pointer（向导表单据此把问题挂到对应字段旁边）。 + +**配方内部的一切引用逐字比较**（契约「键的格式」）：segment.table、verify、keep_as、relations、grain、 +tables[].units 的键、TotalRow.label_column 都必须与 derive_tables 推出的名字逐字相等，不用 name_key、不用 +match_key。文件里写法的变化（全角、空白）由执行器按 match_key 认，与配方内部的引用无关；配方内部若也宽容， +「Sales」和「sales」这种差一点的写法会被静默认成同一张表，下游 SQL 却按另一个名字查。 + +**哈希的前向兼容**（期 3 要给配方加修复类字段）。canonical 形式去掉等于默认值的字段（pydantic 的 +exclude_defaults 对 default_factory 同样生效；axis.checks 在契约里按集合规范化，顺序不同的默认值也会被去掉）， +于是以后新增一个带默认值的字段，已存配方的哈希和构建 id 都不变。配套的规矩： +- **不改已有字段的默认值**（改了的话，省略写法的旧配方含义就变了，哈希却不变）； +- 新字段的默认值必须等于「没有这个字段时的行为」； +- 做不到时升 recipe_format，并写旧格式到新格式的迁移，旧版本按旧规则算哈希。 +dict 的键序没有语义（derive_tables 不按键序定列序），所以哈希里的 sort_keys 不会让两份不同的配方撞上同一个哈希。 +""" +from __future__ import annotations + +import copy +import hashlib +import json +import re +from dataclasses import dataclass +from typing import Any, Literal + +from pydantic import ValidationError + +from app.data.names import canon, collide_key, name_key, name_problem +from app.data.recipe_parsers import ( + LABEL_MAX, + UNITS, + candidate_words, + hour_range, + hour_range_total, + looks_numeric_text, + match_key, + split_unit_suffix, + text_label, +) +from app.data.recipe_types import ( + LIST_TOTAL_DIM, + ColumnOut, + CrosstabBlock, + DerivedSegment, + DimensionSegment, + Dismissed, + DraftFacts, + Fact, + ListBlock, + MeasuresSegment, + NotComparable, + Recipe, + RecipeProblem, + SumEq, + derive_tables, + prose_problems, +) + +Origin = Literal["rules", "ai", "manual", "replay"] + +#: pydantic 在 loc 里插在列表下标后面的判别字段标签：转 JSON Pointer 时去掉，否则 path 对不上配方里的位置 +_UNION_TAGS = frozenset({"crosstab", "list", "measures", "dimension", "derived", "sum_eq", "not_comparable", + "dismissed"}) +_NUMERIC = ("INTEGER", "REAL") + + +class RecipeInvalid(ValueError): + """配方没过 schema（parse_recipe 抛）。problems 的 code 都是 schema，带 path。""" + + def __init__(self, problems: list[RecipeProblem]): + self.problems = list(problems) + super().__init__("；".join(p.message for p in self.problems) or "配方不合法") + + +# ========================================================================== +# 规范形式、哈希 +# ========================================================================== + + +def canonical_recipe(recipe: Recipe) -> dict[str, Any]: + """去掉等于默认值的字段的紧凑形式：存库（table_recipes.recipe、import_stagings.recipe）和算哈希都用它。""" + return recipe.model_dump(mode="json", exclude_defaults=True) + + +def recipe_sha256(recipe: Recipe) -> str: + """配方哈希：进构建 id。与 table_versions._sha_json 同一种写法（sort_keys、紧凑分隔符、UTF-8）。""" + text = json.dumps(canonical_recipe(recipe), sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +# ========================================================================== +# JSON Patch（RFC 6902 的 add / replace / remove） +# ========================================================================== + + +def _pointer_tokens(path: str) -> list[str]: + if path == "": + return [] + if not path.startswith("/"): + raise ValueError(f"路径「{path}」写法不对：应以「/」开头") + return [t.replace("~1", "/").replace("~0", "~") for t in path[1:].split("/")] + + +def _list_index(token: str, path: str) -> int: + # RFC 6901：数组下标只能是不带前导零的非负整数 + if not token.isdigit() or (len(token) > 1 and token[0] == "0") or not token.isascii(): + raise ValueError(f"路径「{path}」不存在：「{token}」不是列表下标") + return int(token) + + +def _walk(doc: Any, tokens: list[str], path: str) -> Any: + node = doc + for t in tokens: + if isinstance(node, dict): + if t not in node: + raise ValueError(f"路径「{path}」不存在") + node = node[t] + elif isinstance(node, list): + i = _list_index(t, path) + if i >= len(node): + raise ValueError(f"路径「{path}」不存在：列表只有 {len(node)} 项") + node = node[i] + else: + raise ValueError(f"路径「{path}」不存在") + return node + + +def apply_patch(recipe: dict[str, Any], ops: list[dict[str, Any]]) -> dict[str, Any]: + """按顺序应用 add / replace / remove，返回新 dict（入参不变）。任何一步出错抛 ValueError，前面的修改也不生效。 + + 起草问题的 effects、向导表单的修改都走这里；add 到已有的键等于替换（RFC 6902），add 到列表下标是插入、 + 「-」是追加；replace、remove 的目标必须存在。 + """ + doc: Any = copy.deepcopy(recipe) + for n, op in enumerate(ops, start=1): + if not isinstance(op, dict): + raise ValueError(f"第 {n} 项修改不是对象") + kind, path = op.get("op"), op.get("path") + if kind not in ("add", "replace", "remove"): + raise ValueError(f"第 {n} 项修改的操作「{kind}」不支持") + if not isinstance(path, str): + raise ValueError(f"第 {n} 项修改缺少路径") + if kind != "remove" and "value" not in op: + raise ValueError(f"第 {n} 项修改缺少新值") + tokens = _pointer_tokens(path) + value = copy.deepcopy(op.get("value")) + if not tokens: + if kind == "remove": + raise ValueError("不能删除整个配方") + if not isinstance(value, dict): + raise ValueError("整个配方只能换成一个 JSON 对象") + doc = value + continue + parent = _walk(doc, tokens[:-1], path) + last = tokens[-1] + if isinstance(parent, dict): + if kind != "add" and last not in parent: + raise ValueError(f"路径「{path}」不存在") + if kind == "remove": + del parent[last] + else: + parent[last] = value + elif isinstance(parent, list): + if kind == "add" and last == "-": + parent.append(value) + continue + i = _list_index(last, path) + if kind == "add": + if i > len(parent): + raise ValueError(f"路径「{path}」不存在：列表只有 {len(parent)} 项") + parent.insert(i, value) + else: + if i >= len(parent): + raise ValueError(f"路径「{path}」不存在：列表只有 {len(parent)} 项") + if kind == "replace": + parent[i] = value + else: + del parent[i] + else: + raise ValueError(f"路径「{path}」不存在") + return doc + + +# ========================================================================== +# 位置说明（给人看的「第 1 个工作表「客流汇总」的分段「夜间」」） +# ========================================================================== + + +def _esc(token: Any) -> str: + return str(token).replace("~", "~0").replace("/", "~1") + + +def _pointer(parts: tuple[Any, ...] | list[Any]) -> str: + return "".join("/" + _esc(p) for p in parts) + + +def _get(node: Any, key: Any) -> Any: + if isinstance(node, dict) and isinstance(key, str): + return node.get(key) + if isinstance(node, list) and isinstance(key, int) and 0 <= key < len(node): + return node[key] + return None + + +def _text(node: Any, *keys: str) -> str | None: + for k in keys: + node = _get(node, k) + return node if isinstance(node, str) and node else None + + +def _where(parts: tuple[Any, ...], data: Any) -> tuple[str, int]: + """路径 → (位置说明, 用掉了几段)。说明用人话、不露键名；名字取自配方本身（schema 没过时也尽量取）。""" + if len(parts) >= 2 and parts[0] == "sheets" and isinstance(parts[1], int): + sheet = _get(_get(data, "sheets"), parts[1]) + name = _text(sheet, "match", "name") + desc = f"第 {parts[1] + 1} 个工作表" + (f"「{name}」" if name else "") + used = 2 + if len(parts) >= 4 and parts[2] == "blocks" and isinstance(parts[3], int): + block = _get(_get(sheet, "blocks"), parts[3]) + used = 4 + if len(parts) >= 6 and parts[4] == "segments" and isinstance(parts[5], int): + seg = _get(_get(block, "segments"), parts[5]) + sid = _text(seg, "id") + desc += f"的分段「{sid}」" if sid else f"的第 {parts[5] + 1} 个分段" + used = 6 + else: + bid = _text(block, "id") + desc += f"的块「{bid}」" if bid else f"的第 {parts[3] + 1} 个块" + if len(parts) >= 6 and parts[4] == "columns" and isinstance(parts[5], int): + col = _get(_get(block, "columns"), parts[5]) + header = _text(col, "header") or _text(col, "name") + desc += f"的列「{header}」" if header else f"的第 {parts[5] + 1} 列" + used = 6 + elif len(parts) >= 3 and parts[2] == "context": + desc += "的统计期" + used = 4 if len(parts) >= 4 and isinstance(parts[3], int) else 3 + return desc, used + if len(parts) >= 2 and parts[0] == "tables" and isinstance(parts[1], int): + name = _text(_get(_get(data, "tables"), parts[1]), "name") + return (f"表「{name}」" if name else f"第 {parts[1] + 1} 张表"), 2 + if len(parts) >= 2 and parts[0] == "relations" and isinstance(parts[1], int): + rid = _text(_get(_get(data, "relations"), parts[1]), "id") + return (f"关系「{rid}」" if rid else f"第 {parts[1] + 1} 条关系"), 2 + return "配方", 0 + + +#: schema 报错时字段的中文名（不在句子里裸写键名） +_FIELD_LABEL = { + "recipe_format": "配方格式", "mode": "导入模式", "other_visible_sheets": "其他可见工作表的处理", + "sheets": "工作表", "tables": "表", "relations": "关系", "id": "名称", "match": "工作表匹配", + "name": "名称", "fallback": "找不到同名工作表时的处理", "hidden": "隐藏行列的处理", "rows": "行", + "cols": "列", "context": "统计期", "kind": "类型", "parser": "解析方式", "prefer_prefix": "优先展示的开头文字", + "cross_check": "旁证", "blocks": "块", "layout": "版式", "axis": "日期表头", "find": "日期表头的查找", + "min": "最少日期格数", "type": "类型", "year_from": "年份来源", "checks": "检查项", + "label_offset": "标签列位置", "values": "取值规则", "placeholders": "占位符", "text": "原文", + "meaning": "含义", "blank": "空格的处理", "text_number": "文本数字的处理", "formula": "公式的处理", + "segments": "分段", "role": "分段类型", "table": "表", "locate": "定位方式", "by": "定位依据", + "title": "分段标题", "segment": "紧跟的分段", "labels": "标签", "expect": "期望的标签", + "measures": "指标列名对照", "dim": "维度", "derive": "派生列", "value": "值列", "const": "常量列", + "pick": "取值", "stop_parser": "结束本段的标签", "labels_parser": "标签解析方式", "verify": "核对方式", + "against_table": "核对的基表", "keep_as": "另存", "header_rows": "表头行数", "after_title": "表头上方的标题", + "columns": "列", "header": "表头", "store": "存储写法", "extra_columns": "多出的列", + "blank_rows": "空行的处理", "total_row": "合计行", "label_column": "合计标签所在的列", + "merged_data": "合并单元格的处理", "grain": "主键", "units": "单位", "note": "说明", "total": "合计列", + "parts": "组成列", "claims": "认领", "a": "一方", "b": "另一方", "reason": "理由", +} + + +def _field_path(parts: tuple[Any, ...]) -> str: + out: list[str] = [] + for p in parts: + if isinstance(p, int): + out.append(f"第 {p + 1} 项") + else: + out.append(_FIELD_LABEL.get(p, f"「{p}」")) + return "的".join(out) + + +def _quoted_options(expected: str) -> str: + opts = re.findall(r"'([^']*)'", expected) + return "、".join(f"「{o}」" for o in opts) if opts else expected + + +def _schema_message(err: dict[str, Any], parts: tuple[Any, ...], data: Any) -> str: + where, used = _where(parts, data) + rest = parts[used:] + typ = err.get("type", "") + ctx = err.get("ctx") or {} + raw = err.get("input") + if typ == "missing" and rest: + parent = _field_path(rest[:-1]) + return f"{where}：{parent + '中' if parent else ''}缺少必填项「{_FIELD_LABEL.get(rest[-1], rest[-1])}」" + if typ == "extra_forbidden" and rest: + parent = _field_path(rest[:-1]) + return f"{where}：{parent + '中' if parent else ''}有配方语言不支持的字段「{rest[-1]}」" + field = _field_path(rest) or "取值" + if typ == "literal_error": + reason = f"取值「{raw}」不在可选范围内（可选：{_quoted_options(str(ctx.get('expected', '')))}）" + elif typ == "union_tag_invalid": + disc = str(ctx.get("discriminator", "")).strip("'") + field = _field_path(rest + (disc,)) if disc else field + reason = (f"「{ctx.get('tag')}」不在可选范围内" + f"（可选：{_quoted_options(str(ctx.get('expected_tags', '')))}）") + elif typ == "union_tag_not_found": + disc = str(ctx.get("discriminator", "")).strip("'") + reason = f"缺少「{_FIELD_LABEL.get(disc, disc)}」" + elif typ == "string_too_long": + reason = f"长度不能超过 {ctx.get('max_length')} 字" + elif typ == "string_too_short": + reason = "不能为空" if ctx.get("min_length") == 1 else f"长度至少 {ctx.get('min_length')} 字" + elif typ == "too_short": + reason = f"至少需要 {ctx.get('min_length')} 项" + elif typ == "too_long": + reason = f"最多只能有 {ctx.get('max_length')} 项" + elif typ in ("greater_than_equal", "greater_than"): + reason = f"不能小于 {ctx.get('ge', ctx.get('gt'))}" + elif typ in ("less_than_equal", "less_than"): + reason = f"不能大于 {ctx.get('le', ctx.get('lt'))}" + elif typ == "string_pattern_mismatch": + reason = { + "id": "写法不对：工作表编号以小写字母开头，只含小写字母、数字和下划线；关系编号写成 R1 到 R99", + "claims": "写法不对：认领写成 F1、F2 这样的编号", + }.get(str(rest[-1]) if rest else "", "写法不对") + elif typ in ("int_type", "int_parsing", "int_from_float"): + reason = "需要是整数" + elif typ in ("string_type",): + reason = "需要是文字" + elif typ in ("list_type",): + reason = "需要是列表" + elif typ in ("dict_type", "model_type", "model_attributes_type"): + reason = "需要是对象" + else: + reason = "取值不合法" + return f"{where}：{field}{reason}" + + +def _schema_problems(exc: ValidationError, data: Any) -> list[RecipeProblem]: + out: list[RecipeProblem] = [] + for err in exc.errors(include_url=False): + parts: list[Any] = [] + prev: Any = None + for p in err.get("loc", ()): + # 只去掉紧跟在列表下标后面的那一个标签：measures 分段缺 measures 字段时 loc 是 + # (…, 0, "measures", "measures")，第二个是真字段 + tag = isinstance(p, str) and p in _UNION_TAGS and isinstance(prev, int) + prev = p + if p == "[key]" or tag: + continue + parts.append(p) + tparts = tuple(parts) + out.append(RecipeProblem(_pointer(tparts), "schema", _schema_message(err, tparts, data))) + return out + + +def parse_recipe(data: dict[str, Any]) -> Recipe: + """pydantic 校验（封闭语言：多出来的键、写死的值都在这里被拒）。失败抛 RecipeInvalid，code 都是 schema。""" + if not isinstance(data, dict): + raise RecipeInvalid([RecipeProblem("", "schema", "配方：必须是一个 JSON 对象")]) + try: + return Recipe.model_validate(data) + except ValidationError as exc: + raise RecipeInvalid(_schema_problems(exc, data)) from None + + +# ========================================================================== +# 语义校验（P2-SPEC 2.2） +# ========================================================================== + + +@dataclass +class _Writer: + """写入一张表的一处（分段、keep_as、列表块、列表合计）：建表的列由它决定。""" + + table: str + #: 表名字段的路径（table_unknown 报在这里） + table_parts: tuple[Any, ...] + #: 这一处应写进的表的 kind + kind: Literal["data", "reported_total"] + #: (列名, 定义这个名字的字段路径)，顺序与 derive_tables 一致 + columns: list[tuple[str, tuple[Any, ...]]] + #: 交叉表产出的表：主键必须含轴列 + axis: str | None = None + #: derive_tables 报问题时用的路径（分段、列表块、列表合计行），据此认出是哪张表的列推不出来 + origin: str = "" + + +def _fact_brief(fact: Fact) -> str: + # 「第 5 行 = 第 6 行 + 第 7 行（31 列中 31 列成立）」→ 去掉末尾的成立情况，只留关系本身 + return re.sub(r"（[^（）]*）\s*$", "", fact.text).strip() or fact.id + + +class _Checker: + def __init__(self, recipe: Recipe, facts: DraftFacts | None, origin: str): + self.r = recipe + self.facts = facts + self.origin = origin + self.data = recipe.model_dump(mode="json") + self.problems: list[RecipeProblem] = [] + self._seen: set[tuple[str, str]] = set() + #: _near 的缓存：名字 → (name_key, match_key)；名字列表 → 键索引（SE-5：几万个未知键逐个对全表各列 + #: 重算 NFKC 是 O(键×列)，几百 KB 的配方能让静态校验跑几十秒） + self._kk: dict[str, tuple[str, str]] = {} + self._near_idx: dict[tuple[str, ...], dict[tuple[str, str], list[tuple[int, str]]]] = {} + # 分段：(si, bi, gi, 块, 分段) + self.segs: list[tuple[int, int, int, CrosstabBlock, Any]] = [] + self.lists: list[tuple[int, int, ListBlock]] = [] + for si, sheet in enumerate(recipe.sheets): + for bi, block in enumerate(sheet.blocks): + if isinstance(block, CrosstabBlock): + for gi, seg in enumerate(block.segments): + self.segs.append((si, bi, gi, block, seg)) + else: + self.lists.append((si, bi, block)) + # 表名 → 第一次声明的位置（重名的另报 name_collision） + self.declared = {t.name: i for i, t in reversed(list(enumerate(recipe.tables)))} + self.cols: dict[str, list[ColumnOut]] = {} + self.writers: list[_Writer] = [] + #: derive_tables 推不准列的表（形状冲突、measures 对不上标签）：根因已报，不再逐个报「列不存在」， + #: 免得一处写错连带出一串问题（AI 修订时回灌的也只是根因） + self.broken: set[str] = set() + + # ---- 小工具 ---- + + def add(self, parts: tuple[Any, ...], code: str, reason: str, *, where: bool = True) -> None: + path = _pointer(parts) + if (path, code) in self._seen: + return + self._seen.add((path, code)) + msg = f"{_where(parts, self.data)[0]}：{reason}" if where else reason + self.problems.append(RecipeProblem(path, code, msg)) + + @staticmethod + def seg_parts(si: int, bi: int, gi: int) -> tuple[Any, ...]: + return ("sheets", si, "blocks", bi, "segments", gi) + + def col_names(self, table: str) -> list[str]: + return [c.name for c in self.cols.get(table, []) if c.name] + + def col(self, table: str, name: str) -> ColumnOut | None: + return next((c for c in self.cols.get(table, []) if c.name == name), None) + + def _keys(self, name: str) -> tuple[str, str]: + k = self._kk.get(name) + if k is None: + k = self._kk[name] = (name_key(name), match_key(name)) + return k + + def _near(self, wanted: str, names: list[str]) -> str: + # 只给提示，不放行：配方内部引用一律逐字比较。按名字列表建一次键索引，之后每次查表（结果与逐个比较相同： + # 取列表里最靠前的、名字不同而 name_key 或 match_key 相同的那个） + key = tuple(names) + idx = self._near_idx.get(key) + if idx is None: + idx = {} + for i, n in enumerate(names): + nk, mk = self._keys(n) + for k in (("n", nk), ("m", mk)): + hits = idx.setdefault(k, []) + if len(hits) < 3: + hits.append((i, n)) + self._near_idx[key] = idx + wk, wm = self._keys(wanted) + for _, n in sorted(idx.get(("n", wk), []) + idx.get(("m", wm), [])): + if n != wanted: + return f"（有「{n}」，名字必须逐字相同）" + return "" + + # ---- 主流程 ---- + + def run(self) -> list[RecipeProblem]: + self.check_mode() + self.check_ids() + self.writers = self.build_writers() + self.cols, shape = derive_tables(self.r) + for p in shape: + if (p.path, p.code) not in self._seen: + self._seen.add((p.path, p.code)) + self.problems.append(p) + origin = p.path.removesuffix("/measures") + self.broken |= {w.table for w in self.writers if w.origin == origin} + self.check_names() + self.check_tables() + self.check_context() + self.check_locate() + self.check_labels() + self.check_const() + self.check_placeholders() + self.check_units() + self.check_grain() + self.check_list_refs() + self.check_relations() + if self.facts is not None: + self.check_facts() + self.check_notes() + return self.problems + + # ---- 7. 格式版本 ---- + + def check_mode(self) -> None: + if self.r.mode == "accumulate": + self.add(("mode",), "mode_unsupported", "导入模式「按期累积」将在后续版本提供，目前只支持每期替换") + + # ---- 8. 工作表、块、分段的键唯一 ---- + + def check_ids(self) -> None: + sheet_ids: dict[str, int] = {} + sheet_names: dict[str, int] = {} + for si, sheet in enumerate(self.r.sheets): + if sheet.id in sheet_ids: + self.add(("sheets", si, "id"), "sheet_id_duplicate", + f"工作表编号「{sheet.id}」与第 {sheet_ids[sheet.id] + 1} 个工作表重复") + sheet_ids.setdefault(sheet.id, si) + key = match_key(sheet.match.name) + if key in sheet_names: + self.add(("sheets", si, "match", "name"), "sheet_name_duplicate", + f"与第 {sheet_names[key] + 1} 个工作表对应的是同一张工作表「{sheet.match.name}」") + sheet_names.setdefault(key, si) + crosstabs = [bi for bi, b in enumerate(sheet.blocks) if isinstance(b, CrosstabBlock)] + for bi in crosstabs[1:]: + self.add(("sheets", si, "blocks", bi), "crosstab_twice", "一个工作表最多只能有一个交叉表块") + # 分段键和块键都在整份配方内唯一；两者同在 Extraction.block_order 里排先后，所以也不能彼此重名 + seg_ids: dict[str, tuple[Any, ...]] = {} + for si, bi, gi, _block, seg in self.segs: + parts = self.seg_parts(si, bi, gi) + if seg.id in seg_ids: + self.add(parts + ("id",), "segment_id_duplicate", + f"分段名「{seg.id}」在配方里重复：分段名在整份配方内必须唯一") + seg_ids.setdefault(seg.id, parts) + block_ids: set[str] = set() + for si, sheet in enumerate(self.r.sheets): + for bi, block in enumerate(sheet.blocks): + parts = ("sheets", si, "blocks", bi, "id") + if block.id in block_ids: + self.add(parts, "block_id_duplicate", f"块名「{block.id}」在配方里重复：块名在整份配方内必须唯一") + elif block.id in seg_ids: + self.add(parts, "block_id_duplicate", + f"块名「{block.id}」与一个分段同名：分段名和块名在整份配方内不能重复") + block_ids.add(block.id) + + # ---- 写入各表的位置与列 ---- + + def build_writers(self) -> list[_Writer]: + out: list[_Writer] = [] + for si, bi, gi, block, seg in self.segs: + base = ("sheets", si, "blocks", bi) + sp = self.seg_parts(si, bi, gi) + axis = (block.axis.name, base + ("axis", "name")) + if isinstance(seg, MeasuresSegment): + cols = [axis] + [(seg.measures[label], sp + ("measures", label)) + for label in seg.labels.expect if label in seg.measures] + out.append(_Writer(seg.table, sp + ("table",), "data", cols, block.axis.name, _pointer(sp))) + elif isinstance(seg, DimensionSegment): + cols = [axis, (seg.dim.name, sp + ("dim", "name"))] + cols += [(k, sp + ("const", k)) for k in sorted(seg.const)] + cols += [(k, sp + ("dim", "derive", k)) for k, _ in _derive_order(seg.dim.derive)] + cols.append((seg.value, sp + ("value",))) + out.append(_Writer(seg.table, sp + ("table",), "data", cols, block.axis.name, _pointer(sp))) + elif isinstance(seg, DerivedSegment) and seg.keep_as is not None: + k = seg.keep_as + kp = sp + ("keep_as",) + cols = [axis, (k.dim, kp + ("dim",))] + cols += [(n, kp + ("derive", n)) for n, _ in _derive_order(k.derive)] + cols.append((k.value, kp + ("value",))) + out.append(_Writer(k.table, kp + ("table",), "reported_total", cols, block.axis.name, _pointer(sp))) + for si, bi, block in self.lists: + base = ("sheets", si, "blocks", bi) + cols = [(c.name, base + ("columns", n, "name")) for n, c in enumerate(block.columns)] + out.append(_Writer(block.table, base + ("table",), "data", cols, origin=_pointer(base))) + total = block.rows.total_row + if total is not None and total.keep_as: + tp = base + ("rows", "total_row", "keep_as") + tcols = [(LIST_TOTAL_DIM, tp)] + tcols += [(c.name, base + ("columns", n, "name")) + for n, c in enumerate(block.columns) if c.type in _NUMERIC] + out.append(_Writer(total.keep_as, tp, "reported_total", tcols, + origin=_pointer(base + ("rows", "total_row")))) + return out + + # ---- 8. 名字 ---- + + def check_names(self) -> None: + seen: dict[str, str] = {} + for i, t in enumerate(self.r.tables): + why = name_problem(t.name) + if why: + self.add(("tables", i, "name"), "name_invalid", why) + key = collide_key(t.name) + if key in seen: + self.add(("tables", i, "name"), "name_collision", + f"与表「{seen[key]}」重名（名字不区分大小写和全角半角）") + seen.setdefault(key, t.name) + for w in self.writers: + names: dict[str, str] = {} + for name, parts in w.columns: + if not name: + continue # measures 对不上标签的空列名：derive_tables 已报 measures_keys_mismatch + why = name_problem(name) + if why: + self.add(parts, "name_invalid", f"列名有问题：{why}") + key = collide_key(name) + if key in names: + self.add(parts, "name_collision", + f"表「{w.table}」里列名「{name}」与「{names[key]}」重名（名字不区分大小写和全角半角）") + names.setdefault(key, name) + + # ---- 9. 表的引用、10. 表的种类 ---- + + def check_tables(self) -> None: + declared = list(self.declared) + written: dict[str, set[str]] = {} + for w in self.writers: + if w.table not in self.declared: + self.add(w.table_parts, "table_unknown", + f"表「{w.table}」不在表清单里{self._near(w.table, declared)}") + written.setdefault(w.table, set()).add(w.kind) + for i, t in enumerate(self.r.tables): + kinds = written.get(t.name) + if kinds is None: + if self.declared.get(t.name) == i: + self.add(("tables", i), "table_unused", "没有任何分段或列表写入这张表") + continue + if len(kinds) == 1 and t.kind not in kinds: + want = "原表合计（另存的合计行）" if "reported_total" in kinds else "数据" + self.add(("tables", i, "kind"), "table_kind_mismatch", + f"这张表写入的是{want}，表的类型要与之一致") + elif len(kinds) > 1: + self.add(("tables", i, "kind"), "table_kind_mismatch", "同一张表不能既存数据又存另存的合计行") + + # ---- 统计期：覆盖断言和补年份都要求同一工作表配了统计期 ---- + + def check_context(self) -> None: + for si, sheet in enumerate(self.r.sheets): + for ci, ctx in enumerate(sheet.context): + if ctx.prefer_prefix and any(ch.isdigit() for ch in ctx.prefer_prefix): + self.add(("sheets", si, "context", ci, "prefer_prefix"), "period_literal", + f"优先展示的开头文字「{ctx.prefer_prefix}」不能含数字（配方里不能写死年份或期次）") + if sheet.context: + continue + for bi, block in enumerate(sheet.blocks): + if not isinstance(block, CrosstabBlock): + continue + ap = ("sheets", si, "blocks", bi, "axis") + if "covers_context" in block.axis.checks: + self.add(ap + ("checks",), "covers_without_context", + "要求日期恰好覆盖统计期，但这个工作表没有配置统计期") + if block.axis.year_from is not None: + self.add(ap + ("year_from",), "covers_without_context", + "日期的年份取自统计期，但这个工作表没有配置统计期") + + # ---- 10. 定位与合计分段 ---- + + def check_locate(self) -> None: + after_seen: dict[tuple[int, int, str], str] = {} + by_block: dict[tuple[int, int], dict[str, Any]] = {} + for si, bi, _gi, _block, seg in self.segs: + by_block.setdefault((si, bi), {}).setdefault(seg.id, seg) + for si, bi, gi, _block, seg in self.segs: + sp = self.seg_parts(si, bi, gi) + lp = sp + ("locate",) + loc = seg.locate + if isinstance(seg, DerivedSegment): + if loc.by != "after": + self.add(lp + ("by",), "derived_after_invalid", "合计分段只能按「紧跟在另一个分段之后」定位") + continue + if not loc.segment: + self.add(lp + ("segment",), "derived_after_invalid", "合计分段要写明紧跟在哪个分段之后") + continue + target = by_block[(si, bi)].get(loc.segment) + if target is None: + self.add(lp + ("segment",), "derived_after_invalid", + f"紧跟的分段「{loc.segment}」不在同一个交叉表块里") + continue + if not isinstance(target, DimensionSegment): + self.add(lp + ("segment",), "derived_after_invalid", + f"紧跟的分段「{loc.segment}」不是按维度展开的分段，合计行无法按时段区间核对") + continue + if target.stop_parser != seg.labels_parser: + self.add(lp + ("segment",), "derived_after_invalid", + f"分段「{target.id}」没有设置遇到合计行就结束（或结束方式与合计分段的标签解析方式不同），" + "合计行会被当成时段明细") + key = (si, bi, loc.segment) + if key in after_seen: + self.add(lp + ("segment",), "derived_after_invalid", + f"分段「{after_seen[key]}」已经紧跟在「{loc.segment}」之后") + after_seen.setdefault(key, seg.id) + if loc.title is not None: + self.add(lp + ("title",), "derived_after_invalid", "合计分段按位置定位，不写分段标题") + self.check_verify(sp, seg, target) + continue + if loc.by == "after": + self.add(lp + ("by",), "locate_invalid", "只有合计分段才能按「紧跟在另一个分段之后」定位") + elif loc.by == "section_title" and not loc.title: + self.add(lp + ("title",), "locate_invalid", "按分段标题定位时必须写分段标题") + if loc.by != "section_title" and loc.title is not None: + self.add(lp + ("title",), "locate_invalid", "只有按分段标题定位时才写分段标题") + if loc.segment is not None: + self.add(lp + ("segment",), "locate_invalid", "只有合计分段才写紧跟的分段") + if isinstance(seg, DimensionSegment): + self.check_derive(sp + ("dim", "derive"), seg.dim.derive) + if seg.dim.parser != "hour_range" and seg.dim.derive: + self.add(sp + ("dim", "derive"), "derive_invalid", "只有按时段解析的维度才能派生起止小时") + + def check_derive(self, parts: tuple[Any, ...], derive: dict[str, str]) -> None: + roles = list(derive.values()) + for role in ("start", "end"): + if roles.count(role) > 1: + names = "、".join(f"「{k}」" for k, v in derive.items() if v == role) + word = "起始小时" if role == "start" else "结束小时" + self.add(parts, "derive_role_duplicate", f"派生列{names}都取{word}，同一种只能派生一列") + + def check_verify(self, sp: tuple[Any, ...], seg: DerivedSegment, target: DimensionSegment) -> None: + v = seg.verify + vp = sp + ("verify",) + if seg.keep_as is not None: + self.check_derive(sp + ("keep_as", "derive"), seg.keep_as.derive) + if v.against_table not in self.declared and v.against_table not in self.cols: + self.add(vp + ("against_table",), "table_unknown", + f"核对的基表「{v.against_table}」不在表清单里{self._near(v.against_table, list(self.declared))}") + return + if v.against_table != target.table: + self.add(vp + ("against_table",), "verify_against_invalid", + f"核对的基表必须是紧邻的分段「{target.id}」写入的表「{target.table}」") + return + if target.dim.parser != "hour_range" or sorted(target.dim.derive.values()) != ["end", "start"]: + self.add(vp + ("against_table",), "verify_against_invalid", + f"按标签区间核对要求基表「{v.against_table}」按时段解析、并派生起始小时和结束小时") + col = self.col(v.against_table, v.value) + if col is None and v.against_table in self.broken: + return + if col is None: + self.add(vp + ("value",), "column_unknown", + f"基表「{v.against_table}」里没有列「{v.value}」{self._near(v.value, self.col_names(v.against_table))}") + elif col.role != "value": + self.add(vp + ("value",), "verify_against_invalid", + f"核对要加总的是基表的值列「{target.value}」，不是「{v.value}」") + + # ---- 标签：期望的标签按该段的比较方式不能重复、必须解析得出 ---- + + def check_labels(self) -> None: + for si, bi, gi, _block, seg in self.segs: + sp = self.seg_parts(si, bi, gi) + seen: dict[str, str] = {} + for n, label in enumerate(seg.labels.expect): + key = self.label_key(seg, label) + lp = sp + ("labels", "expect", n) + if key is None: + how = "时段（如「7-8」）" if isinstance(seg, DimensionSegment) else "带区间的合计（如「18-22时合计」）" + if isinstance(seg, DimensionSegment) and seg.dim.parser == "text": + self.add(lp, "label_unparsed", f"标签「{label}」为空或超过 {LABEL_MAX} 字") + else: + self.add(lp, "label_unparsed", f"标签「{label}」无法解析为{how}") + continue + if key in seen: + self.add(lp, "label_duplicate", f"标签「{label}」与「{seen[key]}」是同一个标签") + seen.setdefault(key, label) + + @staticmethod + def label_key(seg: Any, label: str) -> str | None: + if isinstance(seg, MeasuresSegment): + return match_key(label) + if isinstance(seg, DimensionSegment): + if seg.dim.parser == "hour_range": + h = hour_range(label) + return h.canonical if h else None + return text_label(label) + t = hour_range_total(label) + return t.canonical if t else None + + # ---- 2、6. 常量只能 pick 分段标题的候选词，不能含数字 ---- + + def check_const(self) -> None: + titles: dict[str, list[tuple[str, str]]] = {} + for _si, _bi, _gi, _block, seg in self.segs: + if isinstance(seg, DimensionSegment) and seg.locate.by == "section_title" and seg.locate.title: + titles.setdefault(seg.table, []).append((seg.id, seg.locate.title)) + for si, bi, gi, _block, seg in self.segs: + if not isinstance(seg, DimensionSegment) or not seg.const: + continue + sp = self.seg_parts(si, bi, gi) + title = seg.locate.title if seg.locate.by == "section_title" else None + siblings = [t for sid, t in titles.get(seg.table, []) if sid != seg.id] + cands = candidate_words(title, siblings) if title else [] + for name, c in seg.const.items(): + cp = sp + ("const", name, "pick") + if any(ch.isdigit() for ch in c.pick): + self.add(cp, "period_literal", + f"常量「{name}」的取值「{c.pick}」不能含数字（配方里不能写死年份或期次）") + elif not title: + self.add(cp, "const_not_candidate", + f"常量「{name}」只能从分段标题里选词，但这个分段没有按分段标题定位") + elif c.pick not in cands: + self.add(cp, "const_not_candidate", + f"常量「{name}」的取值「{c.pick}」不在分段标题「{title}」的候选词中，" + f"可选：{'、'.join(cands) if cands else '（无）'}") + + # ---- 3. 占位符只能映射空值，不能像数 ---- + + def check_placeholders(self) -> None: + for si, sheet in enumerate(self.r.sheets): + for bi, block in enumerate(sheet.blocks): + for n, ph in enumerate(block.values.placeholders): + if looks_numeric_text(ph.text): + self.add(("sheets", si, "blocks", bi, "values", "placeholders", n, "text"), + "placeholder_numeric", + f"占位符「{ph.text}」看起来是一个数，不能当作占位符（占位符一律存为空值）") + for si, bi, block in self.lists: + total = block.rows.total_row + if total is not None and any(ch.isdigit() for ch in total.pick): + self.add(("sheets", si, "blocks", bi, "rows", "total_row", "pick"), "period_literal", + f"合计行的开头词「{total.pick}」不能含数字（配方里不能写死年份或期次）") + + # ---- 4、9、13. 单位：键是本表列名，值在词表里，与原表标签、表头里的单位一致 ---- + + def check_units(self) -> None: + for i, t in enumerate(self.r.tables): + names = self.col_names(t.name) + for col, unit in t.units.items(): + up = ("tables", i, "units", col) + # 没人写入的表（已报 table_unused）没有列可比，不再逐个报列不存在 + if names and col not in names and t.name not in self.broken: + self.add(up, "column_unknown", f"单位对应的列「{col}」不在这张表里{self._near(col, names)}") + if unit not in UNITS: + self.add(up, "unit_unknown", f"列「{col}」的单位「{unit}」不在单位词表中") + # 原表里写着单位的地方：标签（指标列）、表头（列表）、分段标题（长表的值列）。 + # 另存的合计表（keep_as）也要查：它的数字列和原表同出一处（列表合计表的列就是那几个表头，交叉表的 + # 表内合计与紧跟的明细同在一个分段标题下），不查的话合计表就成了能把「万元」静默写成「元」的口子 + for si, bi, _gi, _block, seg in self.segs: + if isinstance(seg, MeasuresSegment): + for label in seg.labels.expect: + if label in seg.measures: + self.unit_against(seg.table, seg.measures[label], label, "标签") + elif isinstance(seg, DimensionSegment) and seg.locate.by == "section_title" and seg.locate.title: + self.unit_against(seg.table, seg.value, seg.locate.title, "分段标题") + elif isinstance(seg, DerivedSegment) and seg.keep_as is not None: + target = self.after_target(si, bi, seg) + if target is not None and target.locate.by == "section_title" and target.locate.title: + self.unit_against(seg.keep_as.table, seg.keep_as.value, target.locate.title, "分段标题") + for _si, _bi, block in self.lists: + total = block.rows.total_row + for c in block.columns: + self.unit_against(block.table, c.name, c.header, "表头") + if total is not None and total.keep_as and c.type in _NUMERIC: + self.unit_against(total.keep_as, c.name, c.header, "表头") + + def after_target(self, si: int, bi: int, seg: DerivedSegment) -> DimensionSegment | None: + """合计分段紧跟的那个 dimension 分段（同一块里按 id 找）；找不到或不是 dimension 段时已报 derived_after_invalid。""" + return next((s for sj, bj, _gi, _block, s in self.segs + if (sj, bj) == (si, bi) and isinstance(s, DimensionSegment) and s.id == seg.locate.segment), None) + + def unit_against(self, table: str, column: str, source: str, what: str) -> None: + i = self.declared.get(table) + if i is None or not column or (table in self.broken and column not in self.col_names(table)): + return + unit = split_unit_suffix(source)[1] + if unit is None: + return + declared = self.r.tables[i].units.get(column) + parts = ("tables", i, "units", column) if declared is not None else ("tables", i, "units") + if unit in UNITS: + if declared is None: + self.add(parts, "unit_label_conflict", + f"原表{what}「{source}」写着单位「{unit}」，列「{column}」的单位也必须写「{unit}」") + elif declared != unit: + self.add(parts, "unit_label_conflict", + f"列「{column}」的单位写的是「{declared}」，原表{what}「{source}」写的是「{unit}」：" + f"配方不做单位换算，请改成「{unit}」") + elif declared is not None: + self.add(parts, "unit_label_conflict", + f"原表{what}「{source}」的单位「{unit}」不在单位词表中，列「{column}」不能写单位") + + # ---- 9、10. 主键 ---- + + def check_grain(self) -> None: + axis_of: dict[str, str] = {} + for w in self.writers: + if w.axis is not None: + axis_of.setdefault(w.table, w.axis) + dims: dict[str, list[DimensionSegment]] = {} + for _si, _bi, _gi, _block, seg in self.segs: + if isinstance(seg, DimensionSegment): + dims.setdefault(seg.table, []).append(seg) + for i, t in enumerate(self.r.tables): + gp = ("tables", i, "grain") + names = self.col_names(t.name) + seen: set[str] = set() + for n, g in enumerate(t.grain): + if g not in names and names and t.name not in self.broken: + self.add(gp + (n,), "column_unknown", f"主键列「{g}」不在这张表里{self._near(g, names)}") + if g in seen: + self.add(gp + (n,), "grain_invalid", f"主键列「{g}」写了两次") + seen.add(g) + if self.declared.get(t.name) != i: + continue + axis = axis_of.get(t.name) + if axis is not None and axis not in t.grain: + self.add(gp, "grain_invalid", f"交叉表产出的表，主键必须含日期列「{axis}」") + segs = dims.get(t.name, []) + for a in range(len(segs)): + for b in range(a + 1, len(segs)): + sa, sb = segs[a], segs[b] + common = self.dim_keys(sa) & self.dim_keys(sb) + if not common: + continue + if any(g in sa.const and g in sb.const and canon(sa.const[g].pick) != canon(sb.const[g].pick) + for g in t.grain): + continue + ex = "、".join(f"「{x}」" for x in sorted(common)[:3]) + self.add(gp, "grain_invalid", + f"分段「{sa.id}」和「{sb.id}」写进同一张表，标签有重复（{ex}），" + "主键必须含能区分它们的常量列") + + @staticmethod + def dim_keys(seg: DimensionSegment) -> set[str]: + out: set[str] = set() + for label in seg.labels.expect: + if seg.dim.parser == "hour_range": + h = hour_range(label) + if h: + out.add(h.canonical) + else: + k = text_label(label) + if k: + out.add(k) + return out + + # ---- 9. 列表：合计标签列、表头不重复 ---- + + def check_list_refs(self) -> None: + for si, bi, block in self.lists: + base = ("sheets", si, "blocks", bi) + names = [c.name for c in block.columns] + total = block.rows.total_row + if total is not None and total.label_column not in names: + self.add(base + ("rows", "total_row", "label_column"), "column_unknown", + f"合计标签所在的列「{total.label_column}」不在这个列表的列里{self._near(total.label_column, names)}") + seen: dict[str, str] = {} + for n, c in enumerate(block.columns): + key = match_key(c.header) + if key in seen: + self.add(base + ("columns", n, "header"), "label_duplicate", + f"表头「{c.header}」与「{seen[key]}」是同一个表头") + seen.setdefault(key, c.header) + + # ---- 9. 关系 ---- + + def check_relations(self) -> None: + ids: set[str] = set() + for i, rel in enumerate(self.r.relations): + rp = ("relations", i) + if rel.id in ids: + self.add(rp + ("id",), "relation_invalid", f"关系编号「{rel.id}」重复") + ids.add(rel.id) + if isinstance(rel, SumEq): + if not self.table_ref(rp + ("table",), rel.table): + continue + cols = [(rp + ("total",), rel.total)] + [(rp + ("parts", n), p) for n, p in enumerate(rel.parts)] + for parts, name in cols: + self.numeric_col(parts, rel.table, name) + if rel.total in rel.parts: + self.add(rp + ("total",), "relation_invalid", f"合计列「{rel.total}」不能同时是组成列") + if len(set(rel.parts)) != len(rel.parts): + self.add(rp + ("parts",), "relation_invalid", "组成列有重复") + elif isinstance(rel, NotComparable): + ok_a = self.table_ref(rp + ("a", "table"), rel.a.table) + ok_b = self.table_ref(rp + ("b", "table"), rel.b.table) + if ok_a: + self.numeric_col(rp + ("a", "value"), rel.a.table, rel.a.value) + if ok_b: + self.numeric_col(rp + ("b", "value"), rel.b.table, rel.b.value) + for side, ok in ((rel.a.table, ok_a), (rel.b.table, ok_b)): + if ok and self.col(side, rel.by) is None and side not in self.broken: + self.add(rp + ("by",), "column_unknown", + f"分组列「{rel.by}」不在表「{side}」里{self._near(rel.by, self.col_names(side))}") + if ok_a and ok_b and rel.a.table == rel.b.table: + self.add(rp + ("b", "table"), "relation_invalid", "口径不同的两方必须是两张不同的表") + + def table_ref(self, parts: tuple[Any, ...], table: str) -> bool: + if table in self.cols and table in self.declared: + return True + self.add(parts, "table_unknown", f"表「{table}」不在配方产出的表里{self._near(table, list(self.declared))}") + return False + + def numeric_col(self, parts: tuple[Any, ...], table: str, name: str) -> None: + col = self.col(table, name) + if col is None and table in self.broken: + return + if col is None: + self.add(parts, "column_unknown", f"表「{table}」里没有列「{name}」{self._near(name, self.col_names(table))}") + elif col.type not in _NUMERIC: + self.add(parts, "relation_invalid", f"列「{name}」不是数字列，不能参与求和核对") + + # ---- 11. 认领 ---- + + def check_facts(self) -> None: + assert self.facts is not None + facts = {f.id: f for f in self.facts.facts} + claimers: dict[str, list[int]] = {} + for i, rel in enumerate(self.r.relations): + if rel.claims is None: + continue + if rel.claims not in facts: + self.add(("relations", i, "claims"), "fact_claim_mismatch", + f"认领的系统发现「{rel.claims}」不存在：请去掉认领，或按本次发现重新登记") + continue + claimers.setdefault(rel.claims, []).append(i) + for fid, fact in facts.items(): + got = claimers.get(fid, []) + if not got: + self.add(("relations",), "fact_unclaimed", + f"系统发现的关系「{_fact_brief(fact)}」没有被认领：请登记为每期核对，或说明不登记的理由", + where=False) + for j in got[1:]: + self.add(("relations", j, "claims"), "fact_claimed_twice", + f"系统发现的关系「{_fact_brief(fact)}」已经由关系「{self.r.relations[got[0]].id}」认领，" + "每条只能认领一次") + for j in got: + why = self.claim_mismatch(self.r.relations[j], fact) + if why: + self.add(("relations", j, "claims"), "fact_claim_mismatch", + f"认领了系统发现的关系「{_fact_brief(fact)}」，但{why}") + + def measures_seg(self, seg_id: Any, labels: list[str]) -> MeasuresSegment | None: + """Fact 指向的 measures 分段：按 id 找；改过分段名找不到时，按标签找唯一含这些标签的分段。""" + segs = [s for *_rest, s in self.segs if isinstance(s, MeasuresSegment)] + hit = next((s for s in segs if s.id == seg_id), None) + if hit is not None: + return hit + keys = {match_key(x) for x in labels} + found = [s for s in segs if keys <= {match_key(x) for x in s.labels.expect}] + return found[0] if len(found) == 1 else None + + def dimension_segs(self, seg_ids: list[str], near: MeasuresSegment | None) -> list[DimensionSegment]: + """not_equal_sum 的 a 指向的 dimension 分段：按 id 找；有改过名找不到的，取与 b 那个 measures 段同块的全部 dimension 段。 + + 起草时 a 就是交叉表里的全部 dimension 段（P2-SPEC 6.1 第 10 条），段 id 取自常量的 pick（8b）；用户在向导 + 里改了分段名、或 AI 起草时另起了名字，id 就对不上。Fact 只记 a 的分段 id、不记标签，没法像 measures 段那样 + 按标签认，所以按「同块的全部 dimension 段」认，并且个数必须与 a 相同、没改名的那几段也必须在其中： + 多了少了说明内容变了，照样报对不上。 + """ + dims = [(si, bi, s) for si, bi, _gi, _block, s in self.segs if isinstance(s, DimensionSegment)] + wanted = set(seg_ids) + hit = [s for *_rest, s in dims if s.id in wanted] + if {s.id for s in hit} == wanted or near is None: + return hit + where = next(((si, bi) for si, bi, _gi, _block, s in self.segs if s is near), None) + same = [s for si, bi, s in dims if (si, bi) == where] + if len(same) == len(wanted) and all(any(s is t for t in same) for s in hit): + return same + return hit + + def claim_mismatch(self, rel: Any, fact: Fact) -> str | None: + d = fact.detail or {} + if isinstance(rel, Dismissed): + return None + if fact.kind == "sum_eq": + if not isinstance(rel, SumEq): + return "这条发现只能登记为每期核对（合计等于各项之和），或说明不登记的理由" + total, parts = d.get("total"), d.get("parts") + if not isinstance(total, str) or not isinstance(parts, list): + return "系统发现缺少标签信息，无法核对认领的内容" + seg = self.measures_seg(d.get("segment"), [total, *parts]) + if seg is None: + return f"配方里找不到这条发现所在的分段「{d.get('segment')}」" + if rel.table != seg.table: + return f"它在分段「{seg.id}」写入的表「{seg.table}」里，关系写的是表「{rel.table}」" + back = {col: label for label, col in seg.measures.items()} + got_total = back.get(rel.total) + got_parts = [back.get(p) for p in rel.parts] + ok = (got_total is not None and match_key(got_total) == match_key(total) + and None not in got_parts + and sorted(match_key(x) for x in got_parts if x is not None) == sorted(match_key(x) for x in parts)) + if not ok: + want = f"「{total}」=" + "+".join(f"「{x}」" for x in parts) + have = f"「{got_total or rel.total}」=" + "+".join(f"「{back.get(p) or p}」" for p in rel.parts) + return f"内容对不上：应为{want}，关系写的是{have}" + return None + # not_equal_sum + if not isinstance(rel, NotComparable): + return "这条发现只能登记为口径不同，或说明不登记的理由" + a_ids, b_label = d.get("a"), d.get("b") + if not isinstance(a_ids, list) or not isinstance(b_label, str): + return "系统发现缺少分段信息，无法核对认领的内容" + seg = self.measures_seg(d.get("b_segment"), [b_label]) + a_segs = self.dimension_segs([x for x in a_ids if isinstance(x, str)], seg) + a_tables = {s.table for s in a_segs} + if rel.a.table not in a_tables: + shown = "、".join(f"「{x}」" for x in sorted(a_tables)) or "（配方里找不到这些分段）" + return f"一方应是分段{'、'.join(f'「{x}」' for x in a_ids)}写入的表{shown}，关系写的是「{rel.a.table}」" + # 发现说的是这些分段「各行之和」，加总的是值列；换成派生列（起始小时）或常量列，口径就写错了 + a_values = sorted({s.value for s in a_segs if s.table == rel.a.table}) + if rel.a.value not in a_values: + shown = "、".join(f"「{x}」" for x in a_values) + return f"一方应是表「{rel.a.table}」的值列{shown}，关系写的是列「{rel.a.value}」" + if seg is None: + return f"配方里找不到这条发现所在的分段「{d.get('b_segment')}」" + back = {col: label for label, col in seg.measures.items()} + got = back.get(rel.b.value) + if rel.b.table != seg.table or got is None or match_key(got) != match_key(b_label): + return f"另一方应是表「{seg.table}」里标签「{b_label}」对应的列，关系写的是表「{rel.b.table}」的列「{rel.b.value}」" + return None + + # ---- 12. 说明散文 ---- + + def check_notes(self) -> None: + all_tables = {t.name for t in self.r.tables} + for i, t in enumerate(self.r.tables): + if not t.note: + continue + np_ = ("tables", i, "note") + if self.origin == "ai": + self.add(np_, "ai_note_forbidden", "AI 起草的配方不能写说明：说明由系统按核对结果生成") + known = {"列": set(self.col_names(t.name)), "表": all_tables, "单位": set(UNITS)} + for p in prose_problems(t.note, known=known): + code = "note_token_unknown" if p.startswith("说明引用了不存在的") else "note_numbers" + self.problems.append(RecipeProblem(_pointer(np_), code, f"表「{t.name}」的说明：{p}")) + + +def _derive_order(derive: dict[str, str]) -> list[tuple[str, str]]: + # 与 derive_tables 同一个顺序：start 在前、end 在后，同角色按列名 + return sorted(derive.items(), key=lambda kv: (kv[1] != "start", kv[0])) + + +def validate_recipe( + data: dict[str, Any] | Recipe, *, + facts: DraftFacts | None = None, + origin: Origin = "manual", +) -> tuple[Recipe | None, list[RecipeProblem]]: + """静态校验，不抛异常。schema 不过时返回 (None, problems)；语义问题返回 (recipe, problems)。 + + facts 给了（首次导入、改配方）才查认领；重放（origin="replay"）不传。origin="ai" 时 note 必须为空。 + problems 为空才可试运行。 + """ + if isinstance(data, Recipe): + recipe = data + else: + try: + recipe = parse_recipe(data) + except RecipeInvalid as exc: + return None, exc.problems + return recipe, _Checker(recipe, facts, origin).run() diff --git a/backend/app/data/recipe_ai.py b/backend/app/data/recipe_ai.py new file mode 100644 index 00000000..54cba7ba --- /dev/null +++ b/backend/app/data/recipe_ai.py @@ -0,0 +1,1178 @@ +"""AI 兜底起草（期 2，WP-4）：规则起草不完整时，经用户同意把表格**结构**发给模型，由模型写配方。 + +**模型只写规则、不碰数据**（用户决定 B）。发出去的是压缩表示（compress）：结构文字照发，数字格只给类型、 +公式先脱敏、隐藏行列只给个数、列表数据列只给画像，任何一个数据值都不出门。模型写回来的配方走和规则草稿 +同一条路：静态校验（注入的 validate，origin="ai"）、执行器干跑（注入的 dry_run）、交给用户逐条确认， +必须人工确认之后才能启用。每月重传按已确认的配方重放，不再调用模型。 + +**为什么不用 get_chat_model。** 它在没有接入时会静默退回演示模型；拿演示模型起草等于假装起草了。这里 +自己按助手的模型设置解析接入，没有、停用、只有演示接入、缺 key 都在调用模型之前判定为不可用。 + +**调用参数照 c837cdb 修过的问题来**：关思考（thinking="off"）、额度 32768、服务方嫌额度大时退回 8192； +正文一律经 message_text 抽取（回复可能是块列表）；输出被截断单独记一种失败，不当成「不是 JSON」去修订。 + +写给模型的文字（系统提示、提示词、压缩表示里的固定片段、回灌的问题模板）都放在名字含 _SYSTEM / _PROMPT +的模块级常量里，拼接只用 .format()。 +""" +from __future__ import annotations + +import asyncio +import datetime as _dt +import hashlib +import json +import logging +import re +from dataclasses import dataclass +from typing import Any + +from pydantic import ValidationError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import recipe_parsers as P +from app.data import recipe_suggest as S +from app.data.names import canon, to_sql_name +from app.data.recipe_suggest import AsyncDryRunFn, DryRunFn, ValidateFn +from app.data.recipe_types import ( + AiAvailability, + AiUsage, + Card, + Draft, + DraftFacts, + Grid, + GridCell, + Recipe, +) +from app.data.tabular import EXCEL_ERRORS +from app.data.xlsx_scan import WorkbookScan, cell_ref, col_letter + +logger = logging.getLogger(__name__) + + +class AiUnavailable(RuntimeError): + """没有可用的模型（没有接入、停用、只有演示接入、缺 key）。message 给人看；在调用模型之前判定。""" + + +class AiTooLarge(ValueError): + """压缩表示超过字数上限。同样在调用模型之前判定（接口 413 ai_too_large）。""" + + +# 没有 AiFailed：调用之后的任何失败都体现在返回值里（draft=None、error、usage 照记） + +#: 助手的模型设置键（与 app/api/copilot.py 的 COPILOT_SETTING_KEY 同值；数据层不导入接口层，测试断言两者相等） +COPILOT_SETTING_KEY = "copilot" +#: 1 次起草 + 最多 2 次修订 +AI_MAX_ATTEMPTS = 3 +#: 单次调用的超时（秒）；测试 monkeypatch 调小 +AI_TIMEOUT_S = 90 +#: 输出额度（与 copilot.COPILOT_MAX_TOKENS 同值）：会思考的模型的思考也算在里面，8192 时常常想完就没额度写正文 +AI_MAX_TOKENS = 32768 +#: 服务方嫌额度太大时退回这个（同 copilot._FALLBACK_MAX_TOKENS） +AI_FALLBACK_MAX_TOKENS = 8192 +COMPRESS_MAX_CHARS_PER_SHEET = 12_000 +COMPRESS_MAX_CHARS = 20_000 +#: 行数超过这个数的工作表，只给前 COMPRESS_HEAD_ROWS 行、后 COMPRESS_TAIL_ROWS 行，中间折成列画像 +COMPRESS_FOLD_ROWS = 300 +COMPRESS_HEAD_ROWS = 200 +COMPRESS_TAIL_ROWS = 20 +#: 结构文字超过这个字数就截断 +COMPRESS_TEXT_MAX = 40 +#: 同模板的连续格至少几格折成「序列」；序列不超过这个数时把每一格都列出来 +SEQ_MIN, SEQ_LIST_MAX = 3, 12 +#: 服务方嫌额度太大时报错里会出现的字样（同 copilot._BUDGET_WORDS） +_BUDGET_WORDS = ("max_tokens", "max_completion_tokens", "max_output_tokens", "maximum context length") + + +@dataclass +class Compressed: + """压缩表示。text 就是提示词的前三段（工作表、系统发现、分段标题候选词），ai_draft 只用 text： + 接口层按这四个字段存库、取回再传进来，发出去的和预览给用户看的逐字相同。""" + + text: str + chars: int + truncated: bool + #: text 的 sha256：同意记录和清单里存它，事后能核对发出去的是哪一份 + sha256: str + + +@dataclass +class AiDraftResult: + #: origin="ai"；None 表示没有得到静态校验通过的配方 + draft: Draft | None + usage: list[AiUsage] + attempts: int + #: 给人看的失败原因：「AI 未能起草出合法的配方…」「输出被截断…」「调用超时…」 + error: str | None + compressed_chars: int + compressed_sha256: str + + +# ========================================================================== +# 写给模型的文字（check-copy 按常量名排除） +# ========================================================================== + +RECIPE_DRAFT_SYSTEM = ( + "你为 Excel 工作表起草「取数配方」：一个符合下面 JSON Schema 的 JSON 对象。系统会按配方确定性地执行、" + "用 SQL 核对，再交给用户逐条确认；你只写规则，不接触数据。\n" + "规则：\n" + "1. 只输出一个 JSON 对象：不要解释，不要代码块标记，不要在思考里草拟 JSON。\n" + "2. 配方语言是封闭的：没有正则，不写单元格坐标；只能用 Schema 里有的字段和取值。\n" + "3. measures 的键必须与 labels.expect 里的标签逐字相同、一一对应。\n" + "4. const 的 pick 只能从「分段标题候选词」里选。\n" + "5. 占位符只能表示空值（无数据或不适用）。\n" + "6. 单位只能取自「单位词表」；标签或表头末尾括号里的单位拆进 tables[].units，列名去掉单位，" + "单位必须与括号里写的一致；括号里的单位不在词表里时，列名保留括号、不写单位。\n" + "7. 合计、小计行用 role=derived 核对，并用 keep_as 另存成一张表。\n" + "8. 「系统发现」的每一条必须被 relations 恰好认领一次：成立写 sum_eq，口径不同写 not_comparable，" + "判断为巧合写 dismissed 并给理由；认领的成员必须正是该发现里的那几行，分段 id 必须与发现里写的一致。\n" + "9. 分段 id、块 id 在整份配方内不能重复。\n" + "10. 不写年份、日期和表格里的任何数值；tables[].note 留空。\n" + "11. 表名、列名用简短的中文。" +) + +#: 前三段：就是压缩表示（Compressed.text），预览里展示的也是这一段 +RECIPE_SECTIONS_PROMPT = "# 工作表压缩表示\n{sheets}\n\n# 系统发现（F1…）\n{facts}\n\n# 分段标题候选词\n{candidates}" + +#: 后三段：单位词表、规则起草的半成品与失败原因、配方 JSON Schema +RECIPE_REQUEST_PROMPT = ( + "# 单位词表\n{units}\n\n" + "# 规则起草的半成品与失败原因\n{rules_draft}\n\n" + "# 配方 JSON Schema\n{schema}\n\n" + "请输出配方 JSON。" +) + +#: 完整的首次提示词（.format(sheets=, facts=, candidates=, units=, rules_draft=, schema=)）。ai_draft 发的是 +#: Compressed.text + 「\n\n」+ RECIPE_REQUEST_PROMPT，与它逐字相同，只是前三段直接取压缩表示、不再重算 +RECIPE_DRAFT_PROMPT = RECIPE_SECTIONS_PROMPT + "\n\n" + RECIPE_REQUEST_PROMPT + +RECIPE_REPAIR_PROMPT = ( + "上一次的配方有以下问题，请输出修正后的完整配方 JSON（只输出一个 JSON 对象）：\n{problems}" +) + +#: 规则草稿那一段、回灌的问题行 +RULES_DRAFT_PROMPT = { + "none": "无", + "recipe": "半成品配方：\n{recipe}", + "failures": "失败原因：\n{lines}", + "line": "- {text}", + "not_json": "上一次的回复不是合法的 JSON 对象（{detail}）", +} + +#: 压缩表示里的固定片段 +COMPRESS_PROMPT_LABELS = { + "sheet_head": "## 工作表「{name}」 已用区域 {area}，非空格 {count}", + "sheet_cut": "（起草时只读取了前 {n} 行）", + "merges": "合并：{items}", + "none": "无", + "texts": "文字：", + "text_item": " {ref}「{text}」", + "long_text": "{head}…（共 {n} 字）", + "seq_all": " {ref} 序列「{tpl}」{n} 格：{items}", + "seq_ends": " {ref} 序列「{tpl}」{n} 格：{first} … {last}", + "repeat": " {ref}「{text}」×{n}", + "numeric_text": "<文本数字>", + "other_text": " {ref} 文字（{n} 字）", + "profile": " {col} 列「{header}」是数据列（只给画像）：{n} 格，不同取值 {distinct} 个，最长 {longest} 字，空值 {blank}%", + "masked": " {col} 列「{header}」是遮罩列（只给画像）：{n} 格，不同取值 {distinct} 个，最长 {longest} 字，空值 {blank}%", + "numbers": "数字（只有类型）：", + "rect": " {ref} {sig}", + "int": "整数", + "float": "小数", + "date": "日期", + "time": "时间", + "bool": "布尔", + "error": "错误值", + "text_formula": "文字", + "formula": "，公式", + "uncached": "，无保存值", + "formulas": "公式形状：", + "shape_one": " {ref} {formula}", + "shape_many": " {ref} {formula} {how}（{n} 格，{cache}）", + "by_col": "逐列平移", + "by_row": "逐行平移", + "by_cell": "逐格平移", + "all_cached": "均有保存值", + "some_uncached": "{n} 格无保存值", + "hidden": "隐藏行：{rows} 隐藏列：{cols}", + "hidden_rows": "第 {span} 行（文字 {t} 格、数字 {n} 格）", + "hidden_cols": "{span} 列（文字 {t} 格、数字 {n} 格）", + "fold": "第 {a}–{b} 行（共 {n} 行）折叠为列画像：", + "fold_col": " {col} 列：{kinds}，空值 {blank}%，不同取值 {distinct} 个", + "kind_count": "{kind} {n}", + "hidden_sheets": "另有隐藏工作表 {n} 个（不发送内容）", + "facts_none": "无", + "fact_sum": "{id}（等式：用 sum_eq 认领，或判断为巧合写 dismissed）：{text}；分段 id「{segment}」，合计行「{total}」，组成行{parts}", + "fact_ne": "{id}（口径不同：用 not_comparable 认领，或判断为巧合写 dismissed）：{text}；明细分段 id {a}，对比分段 id「{b_segment}」的「{b}」", + "quoted": "「{text}」", + "candidates_none": "无", + "candidate": "「{title}」：{words}", + "hidden_text": "<隐藏>", +} + +_L = COMPRESS_PROMPT_LABELS + +#: 给人看的失败原因。只说出了什么事：下一步取决于暂存区有没有工作配方（没有时配方面板里只能粘贴一份）， +#: 由接口层（recipe_imports）补上 +_ERR_INVALID = "AI 未能起草出合法的配方" +_ERR_TRUNCATED = "AI 的输出被截断，配方没有写完（输出被截断）" +_ERR_TIMEOUT = "调用超时：{n} 秒内没有收到 AI 的回复" +_ERR_CALL = "调用失败：{reason}" + + +# ========================================================================== +# 用哪个模型 +# ========================================================================== + + +def _budget_rejected(e: BaseException) -> bool: + """服务方一开口就嫌额度太大（400 / 422，报错里带 max_tokens 等字样）。写法同 copilot._budget_rejected。""" + status = getattr(e, "status_code", None) or getattr(getattr(e, "response", None), "status_code", None) + return status in (400, 422) and any(w in str(e).lower() for w in _BUDGET_WORDS) + + +class _BudgetFallback: + """先按大额度调；服务方嫌额度太大时按小额度重来，之后的调用（修订）都用小的。同 copilot._BudgetFallback， + 只是这里是一次性的 ainvoke，不是流式。""" + + def __init__(self, big: Any, small: Any, model_name: str = "") -> None: + self.model, self._small = big, small + #: 模型 id：ai_draft(model=…) 拿它记用量。不给的话 getattr(model, "model") 拿到的是底层的模型对象 + self.model_name = model_name + + @property + def max_tokens(self) -> Any: + return getattr(self.model, "max_tokens", None) + + async def ainvoke(self, messages: list[Any]) -> Any: + try: + return await self.model.ainvoke(messages) + except Exception as e: + if self.model is self._small or not _budget_rejected(e): + raise + self.model = self._small + return await self.model.ainvoke(messages) + + +def _unconfigured(e: BaseException) -> str: + """写法照 copilot._unconfigured。""" + from app.core.errors import not_configured + + return f"助手使用的模型尚未配置完成：{not_configured(e, with_hint=False)}。请到「设置 → 模型接入」检查" + + +async def resolve_draft_model(session: AsyncSession) -> tuple[Any | None, str, AiAvailability]: + """(可调用的模型, 模型 id, 可用性)；不可用时模型为 None。只建模型对象，不发请求。""" + from app.db.models import Setting + from app.providers.factory import ModelSpec, ProviderNotConfigured, build_chat_model, resolve_provider + + row = await session.get(Setting, COPILOT_SETTING_KEY) + saved = (row.value if row else None) or {} + spec = ModelSpec(provider=saved.get("provider") or None, model=saved.get("model") or None, + max_tokens=AI_MAX_TOKENS, temperature=0, thinking="off") + try: + provider = await resolve_provider(session, spec.provider, spec.model) + except ProviderNotConfigured as e: + # 只存了模型名、提供它的接入已停用：resolve_provider 自己会报出来 + return None, "", AiAvailability(False, reason=_unconfigured(e)) + if provider is None: + if spec.provider: + reason = f"助手设置里的模型接入「{spec.provider}」不存在。请到「设置 → 模型接入」检查" + else: + reason = "未配置模型接入，无法使用 AI 起草。请到「设置 → 模型接入」添加" + return None, "", AiAvailability(False, reason=reason) + if provider.kind == "mock": + return None, "", AiAvailability(False, reason="当前只有演示用的模型接入，无法用于 AI 起草。" + "请到「设置 → 模型接入」添加真实的模型接入") + if not provider.enabled: + # 存的是接入名时 resolve_provider 会把停用的接入原样返回，build_chat_model 也不查 enabled + return None, "", AiAvailability(False, reason=f"助手使用的模型接入「{provider.name}」已停用。" + "请到「设置 → 模型接入」启用它，或为助手换一个模型") + try: + big = build_chat_model(provider, spec) + small = build_chat_model(provider, spec.model_copy(update={"max_tokens": AI_FALLBACK_MAX_TOKENS})) + except ProviderNotConfigured as e: + return None, "", AiAvailability(False, reason=_unconfigured(e)) + model_id = spec.model or provider.default_model or "" + return (_BudgetFallback(big, small, model_id), model_id, + AiAvailability(True, model=model_id, provider=provider.name)) + + +async def ai_availability(session: AsyncSession) -> AiAvailability: + return (await resolve_draft_model(session))[2] + + +# ========================================================================== +# 压缩表示 +# ========================================================================== + +_DIGITS = re.compile(r"\d+") +_WS = re.compile(r"\s+") + + +def _tpl(text: str) -> str: + return _DIGITS.sub("{}", text) + + +def _rng(r1: int, c1: int, r2: int, c2: int) -> str: + a, b = cell_ref(r1, c1), cell_ref(r2, c2) + return a if a == b else f"{a}:{b}" + + +def _flat(text: str) -> str: + return _WS.sub(" ", text.strip()) + + +def _clip(text: str) -> str: + flat = _flat(text) + n = len(canon(flat)) + if n <= COMPRESS_TEXT_MAX: + return flat + return _L["long_text"].format(head=flat[:20], n=n) + + +def _num(v: Any) -> bool: + return isinstance(v, (int, float)) and not isinstance(v, bool) + + +def _value_sig(cell: GridCell) -> str | None: + """数字格、日期格、布尔、公式格的类型签名；文字格 None。""" + v = cell.value + formula = cell.formula is not None + if formula and v is None: + return _L["formula"].lstrip("，") + _L["uncached"] + if isinstance(v, bool): + base = _L["bool"] + elif _num(v): + base = _L["int"] if isinstance(v, int) or float(v).is_integer() else _L["float"] + elif isinstance(v, (_dt.datetime, _dt.date)): + base = _L["date"] + elif isinstance(v, _dt.time): + base = _L["time"] + elif isinstance(v, str) and v.strip().upper() in EXCEL_ERRORS: + base = _L["error"] + elif formula: + base = _L["text_formula"] + else: + return None + return base + (_L["formula"] if formula else "") + + +def _rects(cells: dict[tuple[int, int], str]) -> list[tuple[int, int, int, int, str]]: + """同签名的格合成矩形：先按行成段，再把上下相同的段接起来。""" + by_row: dict[int, list[int]] = {} + for (r, c) in cells: + by_row.setdefault(r, []).append(c) + segs: dict[int, list[tuple[int, int, str]]] = {} + for r in sorted(by_row): + cols = sorted(by_row[r]) + start = prev = cols[0] + for c in cols[1:] + [None]: # type: ignore[list-item] + if c is not None and c == prev + 1 and cells[(r, c)] == cells[(r, start)]: + prev = c + continue + segs.setdefault(r, []).append((start, prev, cells[(r, start)])) + if c is not None: + start = prev = c + rects: list[list[Any]] = [] + open_: dict[tuple[int, int, str], list[Any]] = {} + for r in sorted(segs): + nxt: dict[tuple[int, int, str], list[Any]] = {} + for c0, c1, sig in segs[r]: + key = (c0, c1, sig) + if key in open_ and open_[key][2] == r - 1: + open_[key][2] = r + nxt[key] = open_[key] + else: + rect = [r, c0, r, c1, sig] + rects.append(rect) + nxt[key] = rect + open_ = nxt + return sorted((tuple(x) for x in rects), key=lambda t: (t[0], t[1])) # type: ignore[misc] + + +def sanitize_formula(formula: str) -> str: + """公式脱敏：数字常量 → <数>，字符串常量 → <文本>，跨工作表的引用只留表名（'内部'!<格>）。 + + 「=3000+2500」这种把数写成算式的格本身就是数据值；「=C5*1.0675」里的税率、「=IF(B5>50000,…)」里的阈值 + 是业务数据，字符串常量同理。同表的单元格引用、区域、函数名、运算符照留：模型要靠它们认出合计。 + """ + from openpyxl.formula.tokenizer import Token, Tokenizer + + try: + tok = Tokenizer(formula if formula.startswith("=") else "=" + formula) + except Exception: # noqa: BLE001 - 切不开的公式整条不发 + return "=<公式>" + out: list[str] = [] + for t in tok.items: + if t.type == Token.OPERAND and t.subtype == Token.NUMBER: + out.append("<数>") + elif t.type == Token.OPERAND and t.subtype == Token.TEXT: + out.append("<文本>") + elif t.type == Token.OPERAND and t.subtype == Token.RANGE and "!" in t.value: + out.append(t.value.rsplit("!", 1)[0] + "!<格>") + else: + out.append(t.value) + return "=" + "".join(out) + + +def _translate(formula: str, origin: str, dest: str) -> str | None: + from openpyxl.formula.translate import Translator + + try: + return Translator(formula, origin=origin).translate_formula(dest) + except Exception: # noqa: BLE001 + return None + + +def _formula_shapes(grid: Grid, cells: list[tuple[int, int]]) -> list[str]: + """公式形状：同一行里逐列平移的折成一段，上下相同的段再接成矩形（Translator 判平移，比较的是原文）。""" + by_row: dict[int, list[int]] = {} + for (r, c) in cells: + by_row.setdefault(r, []).append(c) + runs: list[list[int]] = [] # [r, c0, c1] + for r in sorted(by_row): + cols = sorted(by_row[r]) + cur = [r, cols[0], cols[0]] + for c in cols[1:]: + first = grid.get(r, cur[1]) + if (c == cur[2] + 1 and first is not None and first.formula + and _translate(first.formula, cell_ref(r, cur[1]), cell_ref(r, c)) == grid.get(r, c).formula): # type: ignore[union-attr] + cur[2] = c + else: + runs.append(cur) + cur = [r, c, c] + runs.append(cur) + rects: list[list[int]] = [] # [r0, c0, r1, c1] + for r, c0, c1 in runs: + prev = next((x for x in rects if x[2] == r - 1 and x[1] == c0 and x[3] == c1), None) + top = grid.get(prev[0], c0) if prev else None + cur = grid.get(r, c0) + if (prev is not None and top is not None and cur is not None and top.formula + and _translate(top.formula, cell_ref(prev[0], c0), cell_ref(r, c0)) == cur.formula): + prev[2] = r + else: + rects.append([r, c0, r, c1]) + lines: list[str] = [] + for r0, c0, r1, c1 in sorted(rects): + first = grid.get(r0, c0) + formula = sanitize_formula(first.formula or "") if first is not None else "=<公式>" + n = (r1 - r0 + 1) * (c1 - c0 + 1) + if n == 1: + lines.append(_L["shape_one"].format(ref=cell_ref(r0, c0), formula=formula)) + continue + how = _L["by_cell"] if r1 > r0 and c1 > c0 else (_L["by_row"] if r1 > r0 else _L["by_col"]) + uncached = sum(1 for rr in range(r0, r1 + 1) for cc in range(c0, c1 + 1) + if (x := grid.get(rr, cc)) is not None and x.value is None) + cache = _L["all_cached"] if not uncached else _L["some_uncached"].format(n=uncached) + lines.append(_L["shape_many"].format(ref=_rng(r0, c0, r1, c1), formula=formula, how=how, n=n, cache=cache)) + return lines + + +def _spans(nums: list[int]) -> list[tuple[int, int]]: + out: list[tuple[int, int]] = [] + for n in sorted(set(nums)): + if out and n == out[-1][1] + 1: + out[-1] = (out[-1][0], n) + else: + out.append((n, n)) + return out + + +def _fold_items(items: list[tuple[int, int, str]]) -> list[tuple[int, int, str]]: + """结构文字：同值的连续格折成「×N」，同模板的连续格折成「序列」（横向优先，再纵向）。""" + texts = {(r, c): t for r, c, t in items} + used: set[tuple[int, int]] = set() + out: list[tuple[int, int, str]] = [] + for horizontal in (True, False): + keys = sorted(texts, key=(lambda k: (k[0], k[1])) if horizontal else (lambda k: (k[1], k[0]))) + i = 0 + while i < len(keys): + k = keys[i] + if k in used: + i += 1 + continue + run = [k] + j = i + 1 + while j < len(keys): + nk, pk = keys[j], run[-1] + adj = (nk[0] == pk[0] and nk[1] == pk[1] + 1) if horizontal else (nk[1] == pk[1] and nk[0] == pk[0] + 1) + if not adj or nk in used or _tpl(texts[nk]) != _tpl(texts[run[0]]): + break + run.append(nk) + j += 1 + vals = [texts[x] for x in run] + ref = _rng(run[0][0], run[0][1], run[-1][0], run[-1][1]) + if len(run) >= 2 and len(set(vals)) == 1: + out.append((run[0][0], run[0][1], _L["repeat"].format(ref=ref, text=vals[0], n=len(run)))) + elif len(run) >= SEQ_MIN and "{}" in _tpl(vals[0]): + if len(run) <= SEQ_LIST_MAX: + line = _L["seq_all"].format(ref=ref, tpl=_tpl(vals[0]), n=len(run), items="、".join(vals)) + else: + line = _L["seq_ends"].format(ref=ref, tpl=_tpl(vals[0]), n=len(run), first=vals[0], last=vals[-1]) + out.append((run[0][0], run[0][1], line)) + else: + i += 1 + continue + used.update(run) + i = j + for (r, c), t in texts.items(): + if (r, c) not in used: + out.append((r, c, _L["text_item"].format(ref=cell_ref(r, c), text=t))) + return out + + +def _structural_parse(v: Any) -> bool: + return any(f(v) is not None for f in (P.hour_range, P.hour_range_total, P.month_day_or_date, + P.cn_date_range, P.cn_year_month)) + + +def _profile(cells: list[tuple[int, str]], span: tuple[int, int]) -> tuple[int, int, int, int]: + """(个数, 不同取值个数, 最长字数, 空值百分比)。""" + n = len(cells) + distinct = len({canon(t) for _, t in cells}) + longest = max((len(canon(t)) for _, t in cells), default=0) + rows = span[1] - span[0] + 1 + blank = round(100 * (rows - n) / rows) if rows > 0 else 0 + return n, distinct, longest, max(blank, 0) + + +def _compress_sheet(grid: Grid, scan: WorkbookScan, mask_keys: set[str]) -> tuple[str, bool, set[str]]: + """一张工作表的压缩表示。返回 (文本, 是否截断/折叠, 这张表上被隐藏或遮罩的文字)。""" + name = grid.sheet + sheet_scan = scan.sheet(name) + hints = S.layout_hints(grid) + rows_all = grid.row_numbers() + hidden_r, hidden_c = set(grid.hidden_rows), set(grid.hidden_cols) + secret: set[str] = set() + lines: list[str] = [] + if grid.bounds is not None: + r1, c1, r2, c2 = grid.bounds + elif rows_all: + r1, r2 = rows_all[0], rows_all[-1] + c1 = min(c for (_, c) in grid.cells) + c2 = max(c for (_, c) in grid.cells) + else: + r1 = c1 = r2 = c2 = 1 + count = sheet_scan.nonempty if sheet_scan is not None else len(grid.cells) + lines.append(_L["sheet_head"].format(name=name, area=_rng(r1, c1, r2, c2), count=count)) + truncated = grid.truncated + if grid.truncated: + lines.append(_L["sheet_cut"].format(n=rows_all[-1] if rows_all else 0)) + + # 大表：中间的行折成列画像 + folded: set[int] = set() + fold_range: tuple[int, int] | None = None + if rows_all and r2 - r1 + 1 > COMPRESS_FOLD_ROWS: + a, b = r1 + COMPRESS_HEAD_ROWS, r2 - COMPRESS_TAIL_ROWS + folded = {r for r in rows_all if a <= r <= b} + fold_range = (a, b) + truncated = True + + merges = [m for m in grid.merges if not all(r in hidden_r for r in range(m[0], m[2] + 1))] + lines.append(_L["merges"].format(items="、".join(_rng(*m) for m in sorted(merges)) or _L["none"])) + + # 遮罩列：表头格之下的整列只给画像 + masked: dict[int, tuple[int, str]] = {} + if mask_keys: + for (r, c), cell in sorted(grid.cells.items()): + if isinstance(cell.value, str) and P.match_key(cell.value) in mask_keys and c not in masked: + masked[c] = (r, _flat(cell.value)) + + axis_row, label_col = hints.axis_row, hints.label_col + header_rows = hints.header_rows + top_header = hints.top_header_row + + def positional(r: int, c: int) -> bool: + if axis_row is not None: + return c == label_col or r <= axis_row + if top_header is not None: + return r in header_rows or r < top_header + return False + + numbers: dict[tuple[int, int], str] = {} + formulas: list[tuple[int, int]] = [] + structural: list[tuple[int, int, str]] = [] + numeric_like: list[tuple[int, int, str]] = [] + others: list[tuple[int, int, str]] = [] + data_candidates: dict[int, list[tuple[int, str]]] = {} + masked_cells: dict[int, list[tuple[int, str]]] = {} + hidden_rows_count: dict[int, list[int]] = {} + hidden_cols_count: dict[int, list[int]] = {} + + #: 每行最左的可见非空格所在的列：列表合计行的标签（「合计」）在这一列 + leftmost: dict[int, int] = {} + for (r, c) in grid.cells: + if r not in hidden_r and c not in hidden_c and (r not in leftmost or c < leftmost[r]): + leftmost[r] = c + + for (r, c), cell in sorted(grid.cells.items()): + if r in hidden_r or c in hidden_c: + is_text = isinstance(cell.value, str) and cell.formula is None + bucket = hidden_rows_count if r in hidden_r else hidden_cols_count + key = r if r in hidden_r else c + pair = bucket.setdefault(key, [0, 0]) + pair[0 if is_text else 1] += 1 + if isinstance(cell.value, str): + secret.add(_flat(cell.value)) + continue + if r in folded: + continue + sig = _value_sig(cell) + if sig is not None: + # 日期格（包括日期轴上的）只给类型签名、不给值（P2-SPEC 6.2）：表格里没有统计期格时，轴上的 + # 年月日就是这一期的数据；模型从「C4:AG4 日期」已经看得出这一行是日期表头 + numbers[(r, c)] = sig + if cell.formula is not None: + formulas.append((r, c)) + continue + text = cell.value if isinstance(cell.value, str) else str(cell.value) + if c in masked and r > masked[c][0]: + masked_cells.setdefault(c, []).append((r, text)) + secret.add(_flat(text)) + continue + if P.looks_numeric_text(text): + numeric_like.append((r, c, _L["text_item"].format(ref=cell_ref(r, c), text=_L["numeric_text"]))) + secret.add(_flat(text)) + continue + if c in masked and r == masked[c][0]: + continue # 遮罩列的表头放在画像那一行里 + # 合计字样只在结构位置或一行最左的格（列表合计行的标签）里照发：数据列中间以「合计」「累计」开头的格 + # （「累计消费12000元」）是数据，和同列其他格一样只进画像（SE-3） + if (positional(r, c) or (S._total_word(text) and leftmost.get(r) == c) + or S._placeholder_like(text)): + structural.append((r, c, _clip(text))) + continue + if axis_row is None: + data_candidates.setdefault(c, []).append((r, text)) + continue + if _structural_parse(text): + structural.append((r, c, _clip(text))) + else: + others.append((r, c, _L["other_text"].format(ref=cell_ref(r, c), n=len(canon(text))))) + secret.add(_flat(text)) + + # 没有日期轴的工作表：同一列纵向连续 ≥4 个文字格（至多隔一个空行）、所在行还有别的非空格的，是数据列 + profiles: list[tuple[int, int, str]] = [] + row_count = {r: len(grid.row(r)) for r in rows_all} + for c, items in sorted(data_candidates.items()): + items.sort() + runs: list[list[tuple[int, str]]] = [] + for r, t in items: + if runs and r - runs[-1][-1][0] <= 2: + runs[-1].append((r, t)) + else: + runs.append([(r, t)]) + data: list[tuple[int, str]] = [] + span: list[int] = [] + for run in runs: + # 冒号结尾的是表单的标签（「项目名称：」），是结构文字；它右边那一列才是要藏起来的内容 + labels = sum(1 for _, t in run if canon(t).endswith(":")) * 2 >= len(run) + if len(run) >= 4 and not labels and all(row_count.get(r, 0) >= 2 for r, _ in run): + data.extend(run) + span += [run[0][0], run[-1][0]] + else: + for r, t in run: + if labels or _structural_parse(t): + structural.append((r, c, _clip(t))) + else: + others.append((r, c, _L["other_text"].format(ref=cell_ref(r, c), n=len(canon(t))))) + secret.add(_flat(t)) + if data: + for _, t in data: + secret.add(_flat(t)) + header = _header_above(grid, c, data[0][0], header_rows) + n, distinct, longest, blank = _profile(data, (min(span), max(span))) + profiles.append((data[0][0], c, _L["profile"].format(col=col_letter(c), header=header, n=n, + distinct=distinct, longest=longest, blank=blank))) + for c, items in sorted(masked_cells.items()): + rows = [r for r, _ in items] + n, distinct, longest, blank = _profile(items, (min(rows), max(rows))) + profiles.append((masked[c][0], c, _L["masked"].format(col=col_letter(c), header=masked[c][1], n=n, + distinct=distinct, longest=longest, blank=blank))) + + text_lines = _fold_items(structural) + numeric_like + others + profiles + lines.append(_L["texts"]) + lines.extend(t for _, _, t in sorted(text_lines, key=lambda x: (x[0], x[1]))) + rects = _rects(numbers) + if rects: + lines.append(_L["numbers"]) + lines.extend(_L["rect"].format(ref=_rng(a, b, c, d), sig=sig) for a, b, c, d, sig in rects) + shapes = _formula_shapes(grid, formulas) + if shapes: + lines.append(_L["formulas"]) + lines.extend(shapes) + hr = [_L["hidden_rows"].format(span=f"{a}" if a == b else f"{a}–{b}", + t=sum(hidden_rows_count.get(x, [0, 0])[0] for x in range(a, b + 1)), + n=sum(hidden_rows_count.get(x, [0, 0])[1] for x in range(a, b + 1))) + for a, b in _spans(sorted(hidden_r))] + hc = [_L["hidden_cols"].format(span=col_letter(a) if a == b else f"{col_letter(a)}–{col_letter(b)}", + t=sum(hidden_cols_count.get(x, [0, 0])[0] for x in range(a, b + 1)), + n=sum(hidden_cols_count.get(x, [0, 0])[1] for x in range(a, b + 1))) + for a, b in _spans(sorted(hidden_c))] + lines.append(_L["hidden"].format(rows="、".join(hr) or _L["none"], cols="、".join(hc) or _L["none"])) + if fold_range is not None: + a, b = fold_range + lines.append(_L["fold"].format(a=a, b=b, n=b - a + 1)) + cols: dict[int, list[Any]] = {} + for (r, c), cell in grid.cells.items(): + if r in folded and r not in hidden_r and c not in hidden_c: + cols.setdefault(c, []).append(cell) + if isinstance(cell.value, str): + secret.add(_flat(cell.value)) + for c in sorted(cols): + kinds: dict[str, int] = {} + for cell in cols[c]: + k = _value_sig(cell) or _L["text_formula"] + kinds[k] = kinds.get(k, 0) + 1 + distinct = len({repr(cell.value) for cell in cols[c]}) + blank = round(100 * (b - a + 1 - len(cols[c])) / (b - a + 1)) + lines.append(_L["fold_col"].format( + col=col_letter(c), kinds="、".join(_L["kind_count"].format(kind=k, n=n) for k, n in sorted(kinds.items())), + blank=max(blank, 0), distinct=distinct)) + return "\n".join(lines), truncated, secret + + +def _header_above(grid: Grid, c: int, r: int, header_rows: set[int]) -> str: + rows = sorted((h for h in header_rows if h < r), reverse=True) or list(range(r - 1, max(r - 4, 0), -1)) + for h in rows: + cell = grid.get(h, c) + if cell is not None and isinstance(cell.value, str) and cell.value.strip(): + return _clip(cell.value) + return "" + + +def _hide_text(text: str, secret: set[str]) -> str: + """把隐藏行列、遮罩列、数据列里的文字从要发出去的段落里换掉（标签可能来自隐藏行）。 + + secret 里每个文字同时按它的派生写法比：去掉末尾括号单位（「绝密分区（人次）」→「绝密分区」，起草时列名、 + 单位的键、关系成员就是这么来的）、再去掉「一、」这类序号前缀，以及它们经 to_sql_name 的写法（SE-1、SE-2）。 + """ + return _hide_in(text, _variants(secret)) + + +def _hide_in(text: str, words: set[str]) -> str: + """把 words（已含派生写法）里 ≥2 字的文字在 text 里按子串换成 <隐藏>，长的先换。""" + for s in sorted((x for x in words if len(x) >= 2), key=len, reverse=True): + if s in text: + text = text.replace(s, _L["hidden_text"]) + return text + + +def _variants(texts: set[str]) -> set[str]: + """文字 → 它本身和起草时可能派生出的写法（去单位后缀、去序号前缀、to_sql_name）。""" + out: set[str] = set() + for x in texts: + f = _flat(x) + stem = _flat(P.split_unit_suffix(f)[0]) + for v in (f, stem, _flat(S._ENUM_PREFIX.sub("", stem))): + if not v: + continue + out.add(v) + n = to_sql_name(v, fallback="") + if n: + out.add(n) + return out + + +def _hidden_texts(grids: dict[str, Grid]) -> set[str]: + """隐藏行列里的文字（只这些，不含数据列、遮罩列）：规则半成品里只要有字符串是它的一部分，或包含它，就遮掉。""" + out: set[str] = set() + for g in grids.values(): + hr, hc = set(g.hidden_rows), set(g.hidden_cols) + if not hr and not hc: + continue + for (r, c), cell in g.cells.items(): + if (r in hr or c in hc) and isinstance(cell.value, str) and _flat(cell.value): + out.add(_flat(cell.value)) + return out + + +class _Outbound: + """发给模型之前的最后一道清洗：首次调用里的规则半成品、失败原因，以及修订轮回灌的问题列表都过它。 + + - 隐藏行列、遮罩列、数据列、认不出用途的文字（secret）及其派生写法：整串相等就换成 <隐藏>，自由文字里按 + 子串换掉； + - 隐藏行列的文字再多两道（规则半成品里的字符串）：是隐藏文字的一部分（≥3 字，列名可能被截短、取公共后缀） + 或包含隐藏文字的派生写法（≥2 字，表名、分段 id 可能拼了后缀），也换成 <隐藏>； + - 像数的文字：整串相等、或在问题行里被「」引着的，换成 <文本数字>。 + + 修订轮的问题来自静态校验和干跑，消息里会引用系统发现的标签原文（「应为「全日客流（人次）」=「分区甲（人次）」 + +「…」」），系统发现并不排除隐藏行（SE-1）；首轮同意框里展示的全文不含这些，所以必须在这里清洗。 + """ + + def __init__(self, secret: set[str], grids: dict[str, Grid]) -> None: + self.secret = set(secret) + self.exact = _variants(self.secret) + hidden = _hidden_texts(grids) + self.hidden_canon = [canon(h) for h in hidden] + self.hidden_parts = {v for v in _variants(hidden) if len(canon(v)) >= 2} + self.numeric = {S._shown(c.value) for g in grids.values() for c in g.cells.values() + if isinstance(c.value, str) and P.looks_numeric_text(c.value)} + self.numeric = {_flat(x) for x in self.numeric if _flat(x)} + + def _secret_str(self, s: str) -> bool: + f = _flat(s) + if f in self.exact or s in self.exact: + return True + c = canon(f) + if len(c) >= 3 and any(c in h for h in self.hidden_canon): + return True + return any(v in f for v in self.hidden_parts) + + def text(self, s: str) -> str: + f = _flat(s) + if f in self.numeric: + return _L["numeric_text"] + if self._secret_str(s): + return _L["hidden_text"] + return s + + def json(self, node: Any) -> Any: + if isinstance(node, dict): + return {self.text(k) if isinstance(k, str) else k: self.json(v) for k, v in node.items()} + if isinstance(node, list): + return [self.json(v) for v in node] + if isinstance(node, str): + return self.text(node) + return node + + def line(self, text: str) -> str: + out = _hide_in(text, self.exact) + for part in sorted(self.hidden_parts, key=len, reverse=True): + if part in out: + out = out.replace(part, _L["hidden_text"]) + for n in sorted(self.numeric, key=len, reverse=True): + q = _L["quoted"].format(text=n) + if q in out: + out = out.replace(q, _L["quoted"].format(text=_L["numeric_text"])) + return out + + +def _facts_text(facts: DraftFacts, secret: set[str]) -> str: + words = _variants(secret) if facts.facts else set() + lines: list[str] = [] + for f in facts.facts: + d = f.detail + q = _L["quoted"] + if f.kind == "sum_eq": + line = _L["fact_sum"].format(id=f.id, text=f.text, segment=d.get("segment", ""), + total=d.get("total", ""), + parts="".join(q.format(text=p) for p in d.get("parts", []))) + else: + line = _L["fact_ne"].format(id=f.id, text=f.text, a="".join(q.format(text=a) for a in d.get("a", [])), + b_segment=d.get("b_segment", ""), b=d.get("b", "")) + lines.append(_hide_in(line, words)) + return "\n".join(lines) or _L["facts_none"] + + +def _candidates_text(facts: DraftFacts, secret: set[str]) -> str: + lines = [_L["candidate"].format(title=_flat(t), words="、".join(w)) for t, w in facts.candidates.items() + if _flat(t) not in secret] + return "\n".join(lines) or _L["candidates_none"] + + +def compress(grids: dict[str, Grid], scan: WorkbookScan, facts: DraftFacts, *, + mask_headers: set[str] | frozenset[str] = frozenset()) -> Compressed: + """只发结构、不发数据的压缩表示。判定规则全部是确定性的，同一份原件的压缩文本相同。超限抛 AiTooLarge。""" + mask_keys = {P.match_key(h) for h in mask_headers} + order = [s.name for s in scan.sheets if s.state == "visible"] if scan.sheets else list(grids) + names = [n for n in order if n in grids and grids[n].cells] + [ + n for n, g in grids.items() if n not in order and g.cells] + parts: list[str] = [] + truncated = False + secret: set[str] = set() + for n in names: + text, cut, hidden = _compress_sheet(grids[n], scan, mask_keys) + if len(text) > COMPRESS_MAX_CHARS_PER_SHEET: + raise AiTooLarge(f"工作表「{n}」的结构压缩后有 {len(text)} 字，超过上限 {COMPRESS_MAX_CHARS_PER_SHEET} 字，" + "无法交给 AI 起草") + parts.append(text) + truncated = truncated or cut + secret |= hidden + hidden_sheets = [s for s in scan.sheets if s.state != "visible"] + if hidden_sheets: + parts.append(_L["hidden_sheets"].format(n=len(hidden_sheets))) + sheets = "\n\n".join(parts) + facts_text = _facts_text(facts, secret) + cand_text = _candidates_text(facts, secret) + text = RECIPE_SECTIONS_PROMPT.format(sheets=sheets, facts=facts_text, candidates=cand_text) + if len(text) > COMPRESS_MAX_CHARS: + raise AiTooLarge(f"表格结构压缩后有 {len(text)} 字，超过上限 {COMPRESS_MAX_CHARS} 字，无法交给 AI 起草") + return Compressed(text=text, chars=len(text), truncated=truncated, + sha256=hashlib.sha256(text.encode("utf-8")).hexdigest()) + + +def _secret_texts(grids: dict[str, Grid], scan: WorkbookScan) -> set[str]: + """不发给模型的文字（隐藏行列、数据列、认不出用途的文字）：拼规则草稿那一段时据此遮掉。 + + 不存进 Compressed：那个对象可能被接口层留存，隐藏行里的姓名不该跟着落库。 + """ + out: set[str] = set() + for g in grids.values(): + if g.cells: + out |= _compress_sheet(g, scan, set())[2] + return out + + +# ========================================================================== +# 起草 +# ========================================================================== + + +def _rules_section(rules: Draft | None, secret: set[str], grids: dict[str, Grid]) -> str: + """规则起草的半成品（紧凑形式）和 failures_for_model（不用 failures：后者可以引用格子原文）。""" + if rules is None or (rules.recipe is None and not rules.failures_for_model): + return RULES_DRAFT_PROMPT["none"] + scrub = _Outbound(secret, grids) + out: list[str] = [] + if rules.recipe is not None: + try: + compact = Recipe.model_validate(rules.recipe).model_dump(mode="json", exclude_defaults=True) + except ValidationError: + compact = rules.recipe + compact = scrub.json(compact) + out.append(RULES_DRAFT_PROMPT["recipe"].format( + recipe=json.dumps(compact, ensure_ascii=False, separators=(",", ":")))) + lines = [RULES_DRAFT_PROMPT["line"].format(text=scrub.line(x)) for x in rules.failures_for_model] + if lines: + out.append(RULES_DRAFT_PROMPT["failures"].format(lines="\n".join(lines))) + return "\n".join(out) or RULES_DRAFT_PROMPT["none"] + + +def _scrub_json(node: Any, secret: set[str], numeric: set[str]) -> Any: + """配方里等于隐藏文字、像数的文字的字符串（值和键）换掉：配方语言写不了数，但标签可能是「2025」这种文字。 + + 列名是表头经 to_sql_name 变来的（「137-2222-3333」→「c_137_2222_3333」），按原文比对不上，所以也按 + to_sql_name 之后的写法、去掉单位后缀的写法比（见 _variants）：起草万一把数据行认成了表头，列名同样不能把 + 数据带出去。首次调用实际用的是 _Outbound（还会按隐藏行列的子串比），这个函数留给只有文字集合的调用方。 + """ + names = _variants(secret) + + def fix(s: str) -> str: + f = _flat(s) + if f in numeric: + return _L["numeric_text"] + if f in secret or f in names or s in names: + return _L["hidden_text"] + return s + if isinstance(node, dict): + return {fix(k) if isinstance(k, str) else k: _scrub_json(v, secret, numeric) for k, v in node.items()} + if isinstance(node, list): + return [_scrub_json(v, secret, numeric) for v in node] + if isinstance(node, str): + return fix(node) + return node + + +def _schema_text() -> str: + return json.dumps(Recipe.model_json_schema(), ensure_ascii=False, separators=(",", ":")) + + +#: 模型回复的 JSON 最多嵌套几层。配方本身不到 10 层；更深的一律当成不合法的 JSON（深层嵌套会让 json.loads +#: 抛 RecursionError，它不是 ValueError，会逃出 ai_draft，见 SE-6） +JSON_MAX_DEPTH = 64 + + +def _too_deep(s: str, limit: int = JSON_MAX_DEPTH) -> bool: + depth = 0 + in_str = esc = False + for ch in s: + if in_str: + if esc: + esc = False + elif ch == "\\": + esc = True + elif ch == '"': + in_str = False + elif ch == '"': + in_str = True + elif ch in "[{": + depth += 1 + if depth > limit: + return True + elif ch in "]}": + depth -= 1 + return False + + +def _extract_json(text: str) -> dict[str, Any]: + """回复 → JSON 对象。任何解析失败（含嵌套过深、RecursionError）都按 ValueError 抛，走修订分支。""" + s = text.strip() + s = re.sub(r"^```[A-Za-z]*\s*", "", s) + s = re.sub(r"\s*```\s*$", "", s) + i, j = s.find("{"), s.rfind("}") + if i < 0 or j <= i: + raise ValueError("no JSON object found") + body = s[i:j + 1] + if _too_deep(body): + raise ValueError(f"JSON nested deeper than {JSON_MAX_DEPTH} levels") + try: + data = json.loads(body) + except RecursionError: + raise ValueError("JSON nested too deeply") from None + if not isinstance(data, dict): + raise ValueError("JSON is not an object") + return data + + +def _cut_off(reply: Any) -> bool: + """输出被截断：Anthropic 是 stop_reason=max_tokens，OpenAI 系是 finish_reason=length（同 copilot._cut_off）。""" + meta = getattr(reply, "response_metadata", None) or {} + reason = meta.get("finish_reason") or meta.get("stop_reason") + return reason in ("length", "max_tokens") + + +def _now() -> str: + return _dt.datetime.now(_dt.timezone.utc).isoformat(timespec="seconds") + + +def _usage(reply: Any, model_id: str, attempt: int, ok: bool) -> AiUsage: + from app.providers import catalog + + meta = getattr(reply, "usage_metadata", None) or {} + inp, out = int(meta.get("input_tokens") or 0), int(meta.get("output_tokens") or 0) + return AiUsage(model=model_id, input_tokens=inp, output_tokens=out, + cost_usd=round(catalog.estimate_cost(model_id, inp, out), 6), ok=ok, at=_now(), attempt=attempt) + + +def _failed_usage(model_id: str, attempt: int) -> AiUsage: + return AiUsage(model=model_id, input_tokens=0, output_tokens=0, cost_usd=0.0, ok=False, at=_now(), attempt=attempt) + + +def _problems_text(lines: list[str]) -> str: + return "\n".join(RULES_DRAFT_PROMPT["line"].format(text=x) for x in lines[:S.MODEL_LINES_MAX]) + + +def first_call_text(compressed: Compressed, *, grids: dict[str, Grid], scan: WorkbookScan, + rules_draft: Draft | None) -> str: + """ai_draft 第一次调用模型时实际发出的全文：系统提示 + 首次提示词（两段之间空一行）。 + + 同意框展示的必须是这份全文，而不只是压缩表示：首次提示词里还有单位词表、规则起草的半成品与失败原因、 + 配方 JSON Schema，这些同样会发给模型。拼法与 ai_draft 里的逐字相同（同一组常量、同一个 + _rules_section、同一个 _schema_text），接口层按这份全文算 sha256，用户同意的是哪一份、发出去的就是哪一份。 + """ + secret = _secret_texts(grids, scan) + prompt = "\n\n".join([compressed.text, RECIPE_REQUEST_PROMPT.format( + units="、".join(P.UNITS), + rules_draft=_rules_section(rules_draft, secret, grids), + schema=_schema_text())]) + return "\n\n".join([RECIPE_DRAFT_SYSTEM, prompt]) + + +async def ai_draft(session: AsyncSession, *, grids: dict[str, Grid], scan: WorkbookScan, facts: DraftFacts, + rules_draft: Draft | None, filename: str, validate: ValidateFn, + dry_run: AsyncDryRunFn | DryRunFn | None, model: Any | None = None, + compressed: Compressed | None = None, model_id: str | None = None) -> AiDraftResult: + """AI 起草。只抛 AiUnavailable / AiTooLarge（都在第一次调用模型之前）；之后的失败一律返回 draft=None。 + + 这是一个跑在事件循环上的协程，CPU 活一律不在循环线程上做：静态校验（check_recipe）和卡片问题 + （questions_for）放进线程池；干跑经 S.run_dry_async——dry_run 是协程函数就 await（接口层注入的那份 + 占一个解析名额、在线程池里执行），是普通函数也放进线程池。 + + model 给了就用（接口层把同意时解析出的那个模型交进来，同意记录、实际调用、用量是同一个；测试注入假模型， + 需有 async ainvoke(messages) -> AIMessage），model_id 是它的模型 id；没给 model 时 resolve_draft_model。 + compressed 给了就用（接口层先在解析名额内算好、预览时给用户看的就是这一份），否则现算。 + filename 不发给模型（文件名里可能有人名、单位名）；只为与规则起草同签名。 + """ + from langchain_core.messages import HumanMessage, SystemMessage + + from app.engine.state import message_text + + if model is None: + model, model_id, avail = await resolve_draft_model(session) + if model is None or not avail.available: + raise AiUnavailable(avail.reason or "AI 起草不可用") + elif not model_id: + named = getattr(model, "model_name", None) or getattr(model, "model", None) + model_id = named if isinstance(named, str) else "" + if compressed is None: + compressed = compress(grids, scan, facts) + secret = _secret_texts(grids, scan) + scrub = _Outbound(secret, grids) + # 前三段整段取压缩表示的 text（不再重算系统发现、候选词）：接口层存库再取回的只有 text 等四个字段， + # 发出去的必须与同意框里展示、sha256 记下的那一份逐字相同 + prompt = "\n\n".join([compressed.text, RECIPE_REQUEST_PROMPT.format( + units="、".join(P.UNITS), + rules_draft=_rules_section(rules_draft, secret, grids), + schema=_schema_text())]) + messages: list[Any] = [SystemMessage(content=RECIPE_DRAFT_SYSTEM), HumanMessage(content=prompt)] + usage: list[AiUsage] = [] + attempts = 0 + last_error = _ERR_INVALID + best: tuple[Recipe, Any, list[str], list[str]] | None = None + while attempts < AI_MAX_ATTEMPTS: + attempts += 1 + try: + reply = await asyncio.wait_for(model.ainvoke(list(messages)), timeout=AI_TIMEOUT_S) + except asyncio.TimeoutError: + usage.append(_failed_usage(model_id, attempts)) + last_error = _ERR_TIMEOUT.format(n=AI_TIMEOUT_S) + logger.info("recipe ai draft attempt %d timed out (model=%s)", attempts, model_id) + continue + except Exception as e: # noqa: BLE001 - 调用失败只能记一次失败，不能变成起草成功 + from app.core.errors import describe_exception + + usage.append(_failed_usage(model_id, attempts)) + last_error = _ERR_CALL.format(reason=describe_exception(e)) + logger.info("recipe ai draft attempt %d failed (model=%s): %s", attempts, model_id, type(e).__name__) + continue + truncated = _cut_off(reply) + usage.append(_usage(reply, model_id, attempts, ok=not truncated)) + u = usage[-1] + logger.info("recipe ai draft attempt %d model=%s in=%d out=%d truncated=%s", + attempts, model_id, u.input_tokens, u.output_tokens, truncated) + if truncated: + # 半截文本不拿去修订：同样的消息列表重来 + last_error = _ERR_TRUNCATED + continue + last_error = _ERR_INVALID + # 拿到回复之后的任何异常都只算这一次没起草成：用量已记，结果走返回值（SE-6） + try: + text = message_text(reply) + try: + data = _extract_json(text) + except (ValueError, json.JSONDecodeError) as e: + detail = e.msg if isinstance(e, json.JSONDecodeError) else str(e) + messages += [reply, HumanMessage(content=RECIPE_REPAIR_PROMPT.format( + problems=_problems_text([RULES_DRAFT_PROMPT["not_json"].format(detail=detail)])))] + continue + human, for_model, parsed = await asyncio.to_thread(S.check_recipe, data, facts, "ai", validate) + if parsed is None: + # 静态校验的消息会引用系统发现的标签原文（可能来自隐藏行），回灌前清洗（SE-1） + messages += [reply, HumanMessage(content=RECIPE_REPAIR_PROMPT.format( + problems=_problems_text([scrub.line(x) for x in for_model])))] + continue + ext, dh, dm = (None, [], []) + if dry_run is not None: + ext, dh, dm = await S.run_dry_async(parsed, dry_run, grids) + best = (parsed, ext, dh, dm) + if dh and attempts < AI_MAX_ATTEMPTS: + messages += [reply, HumanMessage(content=RECIPE_REPAIR_PROMPT.format( + problems=_problems_text([scrub.line(x) for x in dm])))] + continue + except Exception as e: # noqa: BLE001 + logger.warning("recipe ai draft attempt %d: handling the reply failed: %s", attempts, type(e).__name__) + last_error = _ERR_INVALID + break + break + if best is None: + return AiDraftResult(None, usage, attempts, last_error, compressed.chars, compressed.sha256) + parsed, ext, dh, dm = best + recipe = parsed.model_dump(mode="json") + try: + cards, questions = await asyncio.to_thread(S.questions_for, recipe, facts, grids, extraction=ext) + except Exception: # noqa: BLE001 - 卡片是辅助信息 + cards, questions = [], [] + cards.insert(0, Card("ai_origin", "AI 起草：请逐项核对", + "这份配方由 AI 根据表格结构起草，系统已做静态校验和检查；导入前请逐项核对下面的卡片和问题，" + "提交时需要逐条确认")) + result = Draft(recipe=recipe, complete=not dh, origin="ai", cards=cards, questions=questions, facts=facts, + failures=dh, failures_for_model=dm) + return AiDraftResult(result, usage, attempts, None, compressed.chars, compressed.sha256) + + +__all__ = [ + "AI_FALLBACK_MAX_TOKENS", "AI_MAX_ATTEMPTS", "AI_MAX_TOKENS", "AI_TIMEOUT_S", "AiDraftResult", "AiTooLarge", + "AiUnavailable", "COMPRESS_MAX_CHARS", "COMPRESS_MAX_CHARS_PER_SHEET", "COMPRESS_PROMPT_LABELS", "JSON_MAX_DEPTH", + "COPILOT_SETTING_KEY", "Compressed", "RECIPE_DRAFT_PROMPT", "RECIPE_DRAFT_SYSTEM", "RECIPE_REPAIR_PROMPT", + "ai_availability", "ai_draft", "compress", "first_call_text", "resolve_draft_model", "sanitize_formula", +] diff --git a/backend/app/data/recipe_checks.py b/backend/app/data/recipe_checks.py new file mode 100644 index 00000000..9bc80f27 --- /dev/null +++ b/backend/app/data/recipe_checks.py @@ -0,0 +1,655 @@ +"""配方导入的核对（P2-SPEC 5.2）：在执行器写好的临时库上用 SQL 重算，得出「通过 / 不一致 / 无法核对」。 + +为什么算术放在这里、而且只用 SQL：执行器只搬数、记来历，不做加法（H1）。核对用库里真正存下的值重算， +SQL 原文和参数原样进 CheckResult、进导入清单，复核的人拿同一条 SQL 在同一个库上能得到同一个数。 + +三种状态要分清（final.md 5 节、critic C6）： +- 「通过」只在真的比上了、而且相等时给； +- 明细没铺满、合计没缓存、合计格是空的、明细里有空值、有隐藏行或分类汇总函数——这些情况下合计 + 「等于什么」本身说不准，判「无法核对」（可写理由接受），绝不退回「通过」，也不硬判「不一致」； +- 「不一致」只在比得上、确实不等、又没有上面那些口径疑点时给。 + +各项核对的编号（K / G / T 按出现顺序编号，R 沿用配方）由 plan_checks 一处决定：说明生成 +（recipe_notes）要按编号找回某个分段的核对结果，两边必须是同一套编号。 +""" +from __future__ import annotations + +import math +import re +import sqlite3 +from collections import Counter +from dataclasses import dataclass, field +from typing import Any, Callable, Iterable + +from app.data.engine import open_checked_sqlite +from app.data.recipe_types import ( + CheckResult, + ColumnOut, + CrosstabBlock, + DerivedItem, + DerivedSegment, + Extraction, + ListBlock, + NotComparable, + Recipe, + SumEq, + derive_tables, +) + +#: 无法核对的原因（契约 CheckResult.reasons 的键）的固定顺序：细节、说明都按这个顺序列 +REASON_ORDER = ("tiling", "uncached", "blank", "null_detail", "hidden", "unrecognized") + +#: 无法核对的原因 → 给人看的说法（交叉表合计、公式引用） +REASON_TEXT = { + "tiling": "明细时段没有恰好铺满合计的范围", + "uncached": "文件未经计算保存", + "blank": "合计格为空", + "null_detail": "明细含无数据占位符", + "hidden": "存在隐藏行或使用了分类汇总函数，合计口径无法确定", + "unrecognized": "公式写法无法识别", +} +#: 列表合计行的说法：明细空值来自空格或占位符，不只是占位符；「没铺满」在列表上是合计行之上一行明细都没有 +_LIST_REASON_TEXT = {**REASON_TEXT, "null_detail": "明细含空值", "tiling": "合计行之上没有明细行"} + +#: 合计公式是这两种函数时，不等只能判「无法核对」：SUBTOTAL(109) 本来就不算隐藏行，SUM 却算 +_SUBTOTAL_FUNCS = frozenset({"SUBTOTAL", "AGGREGATE"}) +#: 细节最多逐条列出这么多格（其余写一句「另有 N 格」）；cells 同样封顶 +_DETAIL_MAX = 20 + +_FULL_CALC_NOTE = "工作簿设置了打开时重算，保存值与重算一致" +_A1 = re.compile(r"^\$?([A-Za-z]{1,3})\$?([1-9][0-9]*)$") + + +# ---------------------------------------------------------------- 编号 + + +@dataclass +class CheckPlan: + """K / G / T 的编号与各自要核对的合计格。run_checks 和 recipe_notes 共用。""" + + #: derived 分段键 → K 编号（每个有合计格的 derived 段一条） + k: dict[str, str] = field(default_factory=dict) + #: derived 分段键 → G 编号（只给有公式格的 derived 段） + g: dict[str, str] = field(default_factory=dict) + #: 列表块键 → T 编号（每个有合计行的列表块一条） + t: dict[str, str] = field(default_factory=dict) + items: dict[str, list[DerivedItem]] = field(default_factory=dict) + + +def _recipe_keys(recipe: Recipe) -> tuple[list[str], list[str]]: + """配方里 derived 分段键、带合计行的列表块键，按配方里的先后。""" + derived: list[str] = [] + totals: list[str] = [] + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, CrosstabBlock): + derived += [s.id for s in block.segments if isinstance(s, DerivedSegment)] + elif block.rows.total_row is not None: + totals.append(block.id) + return derived, totals + + +def _ordered(recipe_keys: list[str], items: list[DerivedItem]) -> list[str]: + """先按配方里的先后；配方里找不到的分段（不该出现）按执行结果里第一次出现的先后排在后面，不丢。""" + present: list[str] = [] + for it in items: + if it.segment not in present: + present.append(it.segment) + known = set(recipe_keys) + return [k for k in recipe_keys if k in present] + [k for k in present if k not in known] + + +def plan_checks(recipe: Recipe, extraction: Extraction) -> CheckPlan: + """按配方顺序给 K / G / T 编号。同一份配方和执行结果永远得到同一套编号。""" + derived_keys, total_keys = _recipe_keys(recipe) + sums = [d for d in extraction.derived if d.kind == "label_range_sum"] + cols = [d for d in extraction.derived if d.kind == "column_sum"] + plan = CheckPlan() + for key in _ordered(derived_keys, sums): + plan.items[key] = [d for d in sums if d.segment == key] + plan.k[key] = f"K{len(plan.k) + 1}" + if any(d.is_formula for d in plan.items[key]): + plan.g[key] = f"G{len(plan.g) + 1}" + for key in _ordered(total_keys, cols): + plan.items[key] = [d for d in cols if d.segment == key] + plan.t[key] = f"T{len(plan.t) + 1}" + return plan + + +# ---------------------------------------------------------------- 小工具 + + +def _q(name: str) -> str: + """SQL 标识符：一律双引号、内部双引号加倍（库是 DQS 关闭的连接，写错的名字会直接报错）。""" + return '"' + str(name).replace('"', '""') + '"' + + +def col_index(letters: str) -> int: + n = 0 + for ch in letters.upper(): + n = n * 26 + (ord(ch) - 64) + return n + + +def col_letters(index: int) -> str: + out = "" + while index > 0: + index, rem = divmod(index - 1, 26) + out = chr(65 + rem) + out + return out + + +def split_coord(coord: str) -> tuple[str | None, int, int] | None: + """「工作表!C5」或「C5」→ (工作表或 None, 列号, 行号)；认不出返回 None。""" + sheet, _, ref = str(coord).rpartition("!") + m = _A1.match(ref.strip()) + if not m: + return None + return (sheet or None), col_index(m.group(1)), int(m.group(2)) + + +def lineage_cell(lineage: dict[str, dict[str, list[list[Any]]]], table: str, column: str, + rowid: int) -> str | None: + """按溯源段把 (表, 列, rowid) 反查成「工作表!A1」。溯源段：[起始 rowid, 工作表, 起始格, 个数, 方向]。""" + for run in (lineage.get(table) or {}).get(column) or []: + try: + start, sheet, cell, count, direction = run[0], run[1], run[2], run[3], run[4] + except (IndexError, TypeError): + continue + if not (start <= rowid < start + count): + continue + parsed = split_coord(cell) + if parsed is None: + return None + _, c, r = parsed + offset = rowid - start + if direction == "down": + r += offset + else: + c += offset + return f"{sheet}!{col_letters(c)}{r}" + return None + + +def _short(item: DerivedItem) -> str: + """细节里写的格子：不带工作表名（「G28」），工作表在 cells 里。""" + return str(item.cell).rpartition("!")[2] + + +def _coord(item: DerivedItem) -> str: + return item.cell if "!" in str(item.cell) else f"{item.sheet}!{item.cell}" + + +def _fmt(x: Any) -> str: + if x is None: + return "空" + if isinstance(x, bool) or not isinstance(x, (int, float)): + return str(x) + if isinstance(x, float): + if math.isfinite(x) and x.is_integer() and abs(x) < 1e15: + return str(int(x)) + return format(x, ".10g") + return str(x) + + +def _is_int_value(x: Any) -> bool: + return isinstance(x, int) or (isinstance(x, float) and math.isfinite(x) and x.is_integer()) + + +def same_number(a: Any, b: Any) -> bool: + """两边都是整数值时精确相等，否则相对容差 1e-9（P2-SPEC 5.2 的比较规则）。""" + if isinstance(a, bool) or isinstance(b, bool) or not isinstance(a, (int, float)) \ + or not isinstance(b, (int, float)): + return False + if _is_int_value(a) and _is_int_value(b): + return int(a) == int(b) + return abs(a - b) <= 1e-9 * max(1.0, abs(a), abs(b)) + + +def _range_text(item: DerivedItem) -> str: + return f"{item.start}-{item.end} 时" + + +class _Broken(Exception): + """合计格缺了核对必需的信息（执行器的错）：这一条核对判结构类不一致，不硬算。""" + + +class _Sql: + """执行 SQL 并记下最后一条：出错时 CheckResult 里也要有那条 SQL 原文。""" + + def __init__(self, conn: sqlite3.Connection) -> None: + self.conn = conn + self.last: tuple[str | None, list[Any]] = (None, []) + + def one(self, sql: str, params: list[Any] | tuple[Any, ...] = ()) -> tuple[Any, ...]: + self.last = (sql, list(params)) + return self.conn.execute(sql, list(params)).fetchone() + + def all(self, sql: str, params: list[Any] | tuple[Any, ...] = ()) -> list[tuple[Any, ...]]: + self.last = (sql, list(params)) + return self.conn.execute(sql, list(params)).fetchall() + + +@dataclass +class _Tally: + """逐格判定的汇总：K、T、G 共用。""" + + checked: int = 0 + mismatches: list[tuple[str, str]] = field(default_factory=list) # (格, 细节) + reasons: Counter = field(default_factory=Counter) + unverifiable_cells: list[str] = field(default_factory=list) + extra: dict[str, list[str]] = field(default_factory=dict) # 原因 → 附加说明（如没铺满的区间） + sql: str | None = None + params: list[Any] = field(default_factory=list) + _sql_rank: int = 3 + + def keep_sql(self, sql: str, params: list[Any], rank: int) -> None: + # 代表性的 SQL：优先取第一处不一致，其次第一处无法核对，再次第一格 + if rank < self._sql_rank: + self.sql, self.params, self._sql_rank = sql, list(params), rank + + def ok(self) -> None: + self.checked += 1 + + def bad(self, cell: str, detail: str) -> None: + self.checked += 1 + self.mismatches.append((cell, detail)) + + def unverifiable(self, cell: str, reason: str, note: str | None = None) -> None: + self.reasons[reason] += 1 + self.unverifiable_cells.append(cell) + if note and note not in self.extra.setdefault(reason, []): + self.extra[reason].append(note) + + +def _finish(cid: str, kind: Any, title: str, tally: _Tally, *, reason_text: dict[str, str], + full_calc: bool = False) -> CheckResult: + unverifiable = sum(tally.reasons.values()) + details: list[str] = [d for _, d in tally.mismatches[:_DETAIL_MAX]] + if len(tally.mismatches) > _DETAIL_MAX: + details.append(f"另有 {len(tally.mismatches) - _DETAIL_MAX} 格不一致") + for reason in REASON_ORDER + tuple(r for r in tally.reasons if r not in REASON_ORDER): + n = tally.reasons.get(reason, 0) + if not n: + continue + text = reason_text.get(reason, reason) + if reason == "tiling" and tally.extra.get("tiling"): + text = "明细时段没有恰好铺满 " + "、".join(tally.extra["tiling"]) + details.append(f"{n} 格无法核对：{text}") + if tally.mismatches: + status = "mismatch" + elif unverifiable: + status = "unverifiable" + else: + status = "passed" + details.append(f"{tally.checked} 格全部一致") + if full_calc: + details.append(_FULL_CALC_NOTE) + cells = [c for c, _ in tally.mismatches] + tally.unverifiable_cells + # category 一律 structure：不一致是结构类（拒收），无法核对时 acceptable=True、可写理由接受（进 waivers）。 + # 所以判「拒收」要看 status != passed 且 category == structure 且 acceptable 为假，不能只看 category + return CheckResult( + id=cid, kind=kind, title=title, status=status, category="structure", + checked=tally.checked, failed=len(tally.mismatches), unverifiable=unverifiable, + sql=tally.sql, params=tally.params, details=details, cells=cells[:_DETAIL_MAX], + acceptable=status == "unverifiable", + reasons={r: n for r, n in tally.reasons.items() if n}, + ) + + +# ---------------------------------------------------------------- K：合计按标签区间重算 + + +def _require_range(item: DerivedItem) -> None: + if item.start is None or item.end is None or not item.start_col or not item.end_col \ + or not item.base_table or not item.base_value: + raise _Broken(f"{_short(item)}：合计格缺少核对需要的区间或基表信息") + + +def _range_where(item: DerivedItem, filters: Iterable[tuple[str, Any]]) -> tuple[str, list[Any]]: + """WHERE 各分组列 = ? AND 起始列 >= ? AND 结束列 <= ?，以及对应的参数。""" + filters = list(filters) + where = [f"{_q(k)} = ?" for k, _ in filters] + [f"{_q(item.start_col)} >= ?", f"{_q(item.end_col)} <= ?"] + return " AND ".join(where), [v for _, v in filters] + [item.start, item.end] + + +def _tiled(sql: _Sql, item: DerivedItem, filters: tuple[tuple[str, Any], ...]) -> bool: + """区间 [start, end] 里的明细时段是否恰好铺满：首尾对上、相邻首尾相接、不重叠。""" + where, params = _range_where(item, filters) + rows = sql.all(f"SELECT DISTINCT {_q(item.start_col)}, {_q(item.end_col)} FROM {_q(item.base_table)} " + f"WHERE {where} ORDER BY {_q(item.start_col)}, {_q(item.end_col)}", params) + if not rows or rows[0][0] != item.start or rows[-1][1] != item.end: + return False + prev_end = None + for s, e in rows: + if s is None or e is None or s >= e or (prev_end is not None and s != prev_end): + return False + prev_end = e + return True + + +def _sum_sql(item: DerivedItem) -> tuple[str, list[Any]]: + # WHERE 带上 key 的全部列（轴列 + 前一分段的常量列）：同表别的类别的行不会被加进来 + where, params = _range_where(item, item.key.items()) + return (f"SELECT TOTAL({_q(item.base_value)}), COUNT({_q(item.base_value)}), COUNT(*) " + f"FROM {_q(item.base_table)} WHERE {where}"), params + + +def _judge_value(tally: _Tally, item: DerivedItem, total: float, nonnull: int, count: int, + sql: str, params: list[Any], *, mismatch_detail: Callable[[], str]) -> None: + """判定顺序第 2–6 条（第 1 条铺满由调用方先判）。K 与 T 共用。""" + cell = _coord(item) + if item.value is None: + tally.unverifiable(cell, "uncached" if item.is_formula else "blank") + tally.keep_sql(sql, params, 1) + elif nonnull < count: + # 明细缺数时合计等于什么都说不准：不能判一致，也不能判不一致 + tally.unverifiable(cell, "null_detail") + tally.keep_sql(sql, params, 1) + elif same_number(item.value, total): + tally.ok() + tally.keep_sql(sql, params, 2) + elif item.hidden_in_range or (item.func or "").upper() in _SUBTOTAL_FUNCS: + tally.unverifiable(cell, "hidden") + tally.keep_sql(sql, params, 1) + else: + tally.bad(cell, mismatch_detail()) + tally.keep_sql(sql, params, 0) + + +def _check_k(sql: _Sql, cid: str, title: str, items: list[DerivedItem], axis: str | None, + full_calc: bool) -> CheckResult: + tally = _Tally() + tiling: dict[tuple[Any, ...], bool] = {} + for item in items: + _require_range(item) + # 铺满只看区间和常量列（分段类别），不看日期：长表里每个日期的时段行是同一组 + filters = tuple((k, v) for k, v in item.key.items() if k != axis) + tkey = (item.base_table, item.start_col, item.end_col, item.start, item.end, filters) + if tkey not in tiling: + tiling[tkey] = _tiled(sql, item, filters) + q, params = _sum_sql(item) + if not tiling[tkey]: + tally.unverifiable(_coord(item), "tiling", _range_text(item)) + tally.keep_sql(q, params, 1) + continue + total, nonnull, count = sql.one(q, params) + if count == 0: + # 这个日期一行明细都没有：同样是没铺满 + tally.unverifiable(_coord(item), "tiling", _range_text(item)) + tally.keep_sql(q, params, 1) + continue + _judge_value(tally, item, total, nonnull, count, q, params, mismatch_detail=lambda: ( + f"{_short(item)}：表内为 {_fmt(item.value)}，按 {_range_text(item)}的明细重算为 {_fmt(total)}")) + return _finish(cid, "derived_sum", title, tally, reason_text=REASON_TEXT, full_calc=full_calc) + + +# ---------------------------------------------------------------- G：公式引用与标签一致 + + +def _check_g(sql: _Sql, cid: str, title: str, items: list[DerivedItem]) -> CheckResult: + tally = _Tally() + for item in items: + if not item.is_formula: + continue + cell = _coord(item) + problem = item.ref_problem + if problem == "outside": + tally.bad(cell, f"{_short(item)}：公式引用了不在「{item.base_table}」明细里的格子") + continue + if problem == "hidden" or (problem is None and (item.func or "").upper() in _SUBTOTAL_FUNCS): + tally.unverifiable(cell, "hidden") + continue + if problem is not None or item.ref_rowids is None: + tally.unverifiable(cell, "unrecognized") + continue + _require_range(item) + where, params = _range_where(item, item.key.items()) + q = f"SELECT rowid FROM {_q(item.base_table)} WHERE {where}" + got = {r[0] for r in sql.all(q, params)} + if got == set(item.ref_rowids): + tally.ok() + tally.keep_sql(q, params, 2) + else: + tally.bad(cell, f"{_short(item)}：标签「{item.label}」与公式引用的时段不一致") + tally.keep_sql(q, params, 0) + return _finish(cid, "formula_refs", title, tally, reason_text=REASON_TEXT) + + +# ---------------------------------------------------------------- T：列表合计行按列求和 + + +def _check_t(sql: _Sql, cid: str, title: str, items: list[DerivedItem], full_calc: bool) -> CheckResult: + tally = _Tally() + for item in items: + if not item.base_table or not item.base_value: + raise _Broken(f"{_short(item)}：合计格缺少基表信息") + first, last = item.rowid_first, item.rowid_last + if (first is None) != (last is None): + raise _Broken(f"{_short(item)}：合计行对应的明细行区间只有一端") + q = (f"SELECT TOTAL({_q(item.base_value)}), COUNT({_q(item.base_value)}), COUNT(*) " + f"FROM {_q(item.base_table)} WHERE rowid BETWEEN ? AND ?") + params = [first, last] + total, nonnull, count = sql.one(q, params) + if count == 0: + # 区间里一行明细都没有：表头下面紧接着就是合计行（执行器用两端都是 None 表示空区间， + # BETWEEN NULL AND NULL 一行都选不到），或者区间对不上库。TOTAL 得 0，合计写 0 时会被判「一致」； + # 没有可比的明细，判无法核对 + tally.unverifiable(_coord(item), "tiling") + tally.keep_sql(q, params, 1) + continue + _judge_value(tally, item, total, nonnull, count, q, params, mismatch_detail=lambda: ( + f"{_short(item)}：表内为 {_fmt(item.value)}，按明细求和为 {_fmt(total)}")) + return _finish(cid, "column_sum", title, tally, reason_text=_LIST_REASON_TEXT, full_calc=full_calc) + + +# ---------------------------------------------------------------- R：关系 + + +def _check_sum_eq(sql: _Sql, rel: SumEq, title: str, columns: list[ColumnOut], grain: list[str], + lineage: dict[str, dict[str, list[list[Any]]]]) -> CheckResult: + table, total, parts = rel.table, rel.total, list(rel.parts) + names = [total, *parts] + nonnull = " AND ".join(f"{_q(c)} IS NOT NULL" for c in names) + anynull = " OR ".join(f"{_q(c)} IS NULL" for c in names) + checked = sql.one(f"SELECT COUNT(*) FROM {_q(table)} WHERE {nonnull}")[0] + nulls = sql.one(f"SELECT COUNT(*) FROM {_q(table)} WHERE {anynull}")[0] + types = {c.name: c.type for c in columns} + added = " + ".join(_q(p) for p in parts) + if any(types.get(c) == "REAL" for c in names): + cond = f"ABS({_q(total)} - ({added})) > 1e-9 * MAX(1, ABS({_q(total)}))" + else: + cond = f"{_q(total)} <> {added}" + where = f"{nonnull} AND {cond}" + failed = sql.one(f"SELECT COUNT(*) FROM {_q(table)} WHERE {where}")[0] + picked = ", ".join(["rowid", *(_q(g) for g in grain), _q(total), f"({added})"]) + rows_sql = f"SELECT {picked} FROM {_q(table)} WHERE {where} ORDER BY rowid" + rows = sql.all(rows_sql + f" LIMIT {_DETAIL_MAX}") if failed else [] + details: list[str] = [] + cells: list[str] = [] + for row in rows: + rowid, keys, value, summed = row[0], row[1:1 + len(grain)], row[-2], row[-1] + where_text = "、".join(f"{g} {v}" for g, v in zip(grain, keys)) or f"第 {rowid} 条记录" + cell = lineage_cell(lineage, table, total, rowid) + if cell: + cells.append(cell) + at = f"（{cell.rpartition('!')[2]}）" if cell else "" + details.append(f"{where_text}{at}：{total} 为 {_fmt(value)}，{'、'.join(parts)} 之和为 {_fmt(summed)}") + if failed > len(rows): + details.append(f"另有 {failed - len(rows)} 行不成立") + if nulls: + details.append(f"{nulls} 行含空值，未能核对") + if failed: + status = "mismatch" + elif nulls: + status = "unverifiable" + elif checked == 0: + # 一行都没有核对过（表头下面没有数据行）：不能当「通过」，与 T 的空区间一样判无法核对（AU-1）。 + # 重传时文件导出出错只剩表头，上一期的数据会被整表替换成 0 行，要让人写理由接受才能提交 + status = "unverifiable" + details.append("本期这张表没有数据行，关系未能核对") + else: + status = "passed" + details.append(f"{checked} 行全部成立") + return CheckResult( + id=rel.id, kind="relation_sum_eq", title=title, status=status, category="data_quality", + checked=checked, failed=failed, unverifiable=nulls, sql=rows_sql, params=[], + details=details, cells=cells, acceptable=status != "passed", + ) + + +def _check_not_comparable(sql: _Sql, rel: NotComparable, title: str, tables: dict[str, list[ColumnOut]], + grains: dict[str, list[str]], date_cols: dict[str, set[str]]) -> CheckResult: + a, b, by = rel.a, rel.b, rel.by + inner = (f"SELECT {_q(by)}, TOTAL({_q(a.value)}) AS __sum FROM {_q(a.table)} GROUP BY {_q(by)}") + joined = f"SELECT COUNT(*) FROM ({inner}) a JOIN {_q(b.table)} b ON a.{_q(by)} = b.{_q(by)}" + groups = sql.one(joined)[0] + b_type = next((c.type for c in tables.get(b.table, []) if c.name == b.value), "INTEGER") + if b_type == "REAL": + cond = f"ABS(a.__sum - b.{_q(b.value)}) <= 1e-9 * MAX(1, ABS(b.{_q(b.value)}))" + else: + cond = f"a.__sum = b.{_q(b.value)}" + equal_sql = f"{joined} WHERE {cond}" + equal = sql.one(equal_sql)[0] + dims = [g for g in grains.get(a.table, []) if g != by] + unit = "天" if by in date_cols.get(a.table, set()) else "组" + head = f"各{'、'.join(dims)}之和与{b.value}" if dims else f"{a.value}按{by}汇总与{b.value}" + if groups and equal == groups: + # 声明了口径不同、本期却处处相等：说明换不含「不等于」的写法，确认项要人看一眼（7.5） + detail = f"{head}：{groups} {unit}全部相等" + else: + detail = f"{head}：{groups} {unit}中 {equal} {unit}相等" + return CheckResult( + id=rel.id, kind="relation_not_comparable", title=title, + status="info", category="info", checked=groups, failed=groups - equal, + sql=equal_sql, params=[], details=[detail], + ) + + +# ---------------------------------------------------------------- N、P：落库 + + +def _check_rows(sql: _Sql, cid: str, title: str, table: str, expected: int | None) -> CheckResult: + q = f"SELECT COUNT(*) FROM {_q(table)}" + n = sql.one(q)[0] + if expected is None: + status, detail = "mismatch", f"缺少按格子账推出的期望行数，库里 {n} 行" + elif n != expected: + status, detail = "mismatch", f"库里 {n} 行，按格子账应有 {expected} 行" + else: + status, detail = "passed", f"{n} 行，与格子账一致" + return CheckResult(id=cid, kind="row_count", title=title, status=status, category="structure", + checked=1, failed=int(status != "passed"), sql=q, params=[], details=[detail]) + + +def _check_pk(sql: _Sql, cid: str, title: str, table: str, grain: list[str]) -> CheckResult: + q = (f"SELECT COUNT(*) FROM (SELECT 1 FROM {_q(table)} GROUP BY {', '.join(_q(g) for g in grain)} " + f"HAVING COUNT(*) > 1)") + dup = sql.one(q)[0] + detail = f"{dup} 组主键重复" if dup else "主键没有重复" + return CheckResult(id=cid, kind="pk_unique", title=title, status="passed" if not dup else "mismatch", + category="structure", checked=1, failed=int(bool(dup)), sql=q, params=[], details=[detail]) + + +# ---------------------------------------------------------------- 入口 + + +def _axis_names(recipe: Recipe) -> dict[str, str]: + """derived 分段键 → 所在交叉表块的轴列名（K 的铺满判定不按日期分）。""" + out: dict[str, str] = {} + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, CrosstabBlock): + for seg in block.segments: + if isinstance(seg, DerivedSegment): + out[seg.id] = block.axis.name + return out + + +def date_columns(recipe: Recipe) -> dict[str, set[str]]: + """表 → 日期列（交叉表的轴列、列表里 type=DATE 的列）。「逐日」「天」的措辞按它定。""" + tables, _ = derive_tables(recipe) + out: dict[str, set[str]] = {t: {c.name for c in cols if c.role == "axis"} for t, cols in tables.items()} + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, ListBlock): + out.setdefault(block.table, set()).update(c.name for c in block.columns if c.type == "DATE") + return out + + +def _guarded(sql: _Sql, cid: str, kind: Any, title: str, + run: Callable[[str], CheckResult]) -> CheckResult: + """核对 SQL 出错（表或列不存在）或合计格缺信息，都是执行器或配方的错：判结构类不一致，不让整次核对崩掉。""" + sql.last = (None, []) + try: + return run(title) + except (sqlite3.Error, _Broken) as e: + text, params = sql.last + reason = str(e) if isinstance(e, _Broken) else f"核对 SQL 执行失败：{e}" + return CheckResult(id=cid, kind=kind, title=title, status="mismatch", category="structure", + sql=text, params=params, details=[reason]) + + +def run_checks(db_path: str, extraction: Extraction, recipe: Recipe) -> list[CheckResult]: + """在临时库上跑全部核对。同步（调用方放进线程池）。 + + 结果 = extraction.context_checks（C1、C2 原样排在最前）+ K… + G… + T… + R…（配方顺序）+ N… + P…， + 顺序只取决于配方和执行结果。库以只读、DQS 关闭的连接打开（engine.open_checked_sqlite）。 + """ + plan = plan_checks(recipe, extraction) + tables, _ = derive_tables(recipe) + grains = {t.name: list(t.grain) for t in recipe.tables} + axis = _axis_names(recipe) + dates = date_columns(recipe) + full_calc = extraction.full_calc_on_load + conn = open_checked_sqlite(db_path, readonly=True) + sql = _Sql(conn) + # (编号, 种类, 标题, 核对函数)：标题在这里定一次，出错时的那条结果也用它 + jobs: list[tuple[str, Any, str, Callable[[str], CheckResult]]] = [] + for seg, cid in plan.k.items(): + jobs.append((cid, "derived_sum", f"合计按明细重算（分段「{seg}」）", + lambda t, seg=seg, cid=cid: _check_k(sql, cid, t, plan.items[seg], axis.get(seg), full_calc))) + for seg, cid in plan.g.items(): + jobs.append((cid, "formula_refs", f"合计公式引用（分段「{seg}」）", + lambda t, seg=seg, cid=cid: _check_g(sql, cid, t, plan.items[seg]))) + for block, cid in plan.t.items(): + jobs.append((cid, "column_sum", f"合计行按列求和（块「{block}」）", + lambda t, block=block, cid=cid: _check_t(sql, cid, t, plan.items[block], full_calc))) + for rel in recipe.relations: + if isinstance(rel, SumEq): + jobs.append((rel.id, "relation_sum_eq", f"「{rel.table}」：{rel.total} = {' + '.join(rel.parts)}", + lambda t, rel=rel: _check_sum_eq(sql, rel, t, tables.get(rel.table, []), + grains.get(rel.table, []), extraction.lineage))) + elif isinstance(rel, NotComparable): + jobs.append((rel.id, "relation_not_comparable", f"口径不同：「{rel.a.table}」与「{rel.b.table}」", + lambda t, rel=rel: _check_not_comparable(sql, rel, t, tables, grains, dates))) + for i, table in enumerate(tables, 1): + jobs.append((f"N{i}", "row_count", f"行数：「{table}」", + lambda t, table=table, i=i: _check_rows(sql, f"N{i}", t, table, + extraction.expected_rows.get(table)))) + for i, table in enumerate([t for t in tables if grains.get(t)], 1): + jobs.append((f"P{i}", "pk_unique", f"主键唯一：「{table}」", + lambda t, table=table, i=i: _check_pk(sql, f"P{i}", t, table, grains[table]))) + try: + return list(extraction.context_checks) + [_guarded(sql, *job) for job in jobs] + finally: + conn.close() + + +def column_null_counts(db_path: str, recipe: Recipe) -> dict[str, dict[str, int]]: + """表 → 列 → 空值行数。给 recipe_notes.build_notes 的 null_counts：只有本期真有空值的值列才写空值说明。 + + 说明生成拿不到库（契约签名里没有库路径），Extraction 里也没有按列的空值来源，所以单独查一次。 + """ + tables, _ = derive_tables(recipe) + out: dict[str, dict[str, int]] = {} + conn = open_checked_sqlite(db_path, readonly=True) + try: + for table, cols in tables.items(): + names = [c.name for c in cols] + row = conn.execute( + f"SELECT COUNT(*), {', '.join(f'COUNT({_q(n)})' for n in names)} FROM {_q(table)}").fetchone() + out[table] = {n: row[0] - row[i] for i, n in enumerate(names, 1)} + finally: + conn.close() + return out diff --git a/backend/app/data/recipe_confirm.py b/backend/app/data/recipe_confirm.py new file mode 100644 index 00000000..68157715 --- /dev/null +++ b/backend/app/data/recipe_confirm.py @@ -0,0 +1,776 @@ +"""确认项（期 2，WP-5b；P2-SPEC 7.5）：试运行之后、提交之前必须逐项勾选的清单。 + +**为什么要有确认项。** 配方语言是封闭的，但仍有不少取值是「放宽」：占位符存空值、千分位文本按数字、公式按 +保存值、跳过空行、忽略多出的列、不核对文件名里的日期……每一项都可能让本该拒收的文件被导入。原则（评审甲-5）： +配方里每个放宽取值，**只要配置了**就出确认项，计数类写上本期个数；配方与现行的不同时（首次、切换、改配方后 +的第一次提交，或配方哈希变了）列出全部配方类；每期都要重新看的（含数字的区域外文字、空行之后的文字、 +统计期人工录入、工作表改名……）每次都按本期情况算；差异卡里需确认的项原样收进来。 + +提交时服务端按存下的试运行重算一遍必勾集合（同一个函数），缺一项就 422 confirm_required；界面不提供全选。 +id 里的键一律按契约「键的格式」：分段、块用 id，工作表用实际工作表名（sheet_renamed 用配方里的工作表 id）， +格子用「工作表!A1」。 + +纯函数，不碰数据库。输入的 Extraction、核对结果、差异卡收 dataclass，也收 JSON 形状的 dict（暂存区里存的 +就是 asdict 之后的样子）；配方收 Recipe，也收 dict（canonical 形式也行，按模型补全默认值）。 + +**输入不全就报错，不按空处理**：少一个键、少一个参数，对应的必勾项就会静默消失（extraction 没有 problems， +rows_after_stop 就没了；上传新一期漏传现行配方，破坏性变更和单位变化就没了），而提交时重算用的是同一个 +函数，服务端也发现不了。所以 kind 只认四种；reupload、redraft 必须给现行配方和上一期回执；extraction +必须带齐 EXTRACTION_KEYS、不能是部分干跑；checks 必须给；有上一期时差异卡必须给（可以是空列表）。 +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from app.data.names import canon, to_sql_name +from app.data.recipe_diff import ( + ReceiptIncomplete, col_letter, outside_of, period_annotated, period_of, period_texts, plain, receipt_of, + require_receipt, rows_text, same_period_sentence, short, split_coord, +) +from app.data.recipe_parsers import match_key, period_residue, split_unit_suffix +from app.data.recipe_types import ( + ConfirmItem, CrosstabBlock, DerivedSegment, Dismissed, ListBlock, MeasuresSegment, NotComparable, Recipe, + SheetRecipe, SumEq, derive_tables, +) + +_AXIS_CHECK = {"contiguous": "逐日连续", "covers_context": "恰好覆盖统计期"} +_TABLE_KIND = {"data": "明细数据", "reported_total": "原表合计"} +#: 统计期上下文的 id（契约 ContextSpec.id 只能是它）。确认项 id「context_human:统计期」由它拼出来 +_PERIOD_ID = "统计期" +#: ConfirmContext.kind 的取值（P2-SPEC 7.2、7.3） +KINDS: tuple[str, ...] = ("first", "reupload", "redraft", "switch") +#: confirm_items 读到的 Extraction 字段：缺一个，对应的确认项就会静默消失或写错个数 +EXTRACTION_KEYS: tuple[str, ...] = ( + "problems", "outside_text", "ignored_columns", "sheets", "period", "placeholders", "formula_cells_accepted", + "blank_rows_skipped", "labels", "derived", "axes", "hidden", "lineage", +) +#: 有上一期时，本期类确认项读上一期回执里的这几个键（含数字文字的免确认、新增工作表） +_PREV_KEYS: tuple[str, ...] = ("outside_text", "sheets") + + +@dataclass +class ConfirmContext: + """confirm_items 的输入（字段顺序同 P2-SPEC 7.5）。""" + + #: first / reupload / redraft / switch + kind: str + #: 本次要提交的工作配方 + recipe: Recipe | dict[str, Any] + #: 现行配方；首次、切换时为 None，reupload、redraft 必须给（漏传就报错：否则破坏性变更、单位变化会静默消失） + old_recipe: Recipe | dict[str, Any] | None = None + #: 切换时旧快照（简单导入）的 schema_cache，用来列出将消失或改变的表和列 + old_schema_cache: dict[str, Any] | None = None + #: 本次试运行的 Extraction（或它的 asdict），必须带齐 EXTRACTION_KEYS；默认值只为保持字段顺序，不传会报错 + extraction: Any = None + #: 本次的核对结果（CheckResult 或 dict）；必须给（没有核对也传空列表） + checks: list[Any] | None = None + #: 本次的差异卡（DiffItem 或 dict）；有上一期时必须给（没有差异也传空列表） + diff: list[Any] | None = None + #: 上一期的导入回执（diff_reports 的 prev 形状）；首次为 None，reupload、redraft 必须给 + prev: dict[str, Any] | None = None + #: rules / ai / manual / mixed。AI 起草后又人手改过的，调用方要传 mixed（暂存区里此时记的是 manual） + recipe_origin: str = "rules" + #: 系统发现（DraftFacts 或 dict），只用来给 dismissed 的标题写上那条关系的原话；不给也行 + facts: Any = None + #: 数据源现在设的遮罩列（options.mask_columns）。切换、改配方之后列名变了，遮罩按列名匹配会静默失效（AU-6） + mask_columns: list[str] | None = None + + +@dataclass(frozen=True) +class TableChange: + """新旧配方之间一张表的一处破坏性变化。kind：table_removed / column_removed / type / unit / grain / + table_kind / placeholder_meaning / const_value / source / store。""" + + table: str + kind: str + message: str + column: str | None = None + old: Any = None + new: Any = None + + +def as_recipe(data: Recipe | dict[str, Any]) -> Recipe: + return data if isinstance(data, Recipe) else Recipe.model_validate(data) + + +def recipe_canonical(recipe: Recipe) -> dict[str, Any]: + """与 recipe.canonical_recipe 同一种写法（去掉等于默认值的字段）：两份配方这个相等，哈希就相等。""" + return recipe.model_dump(mode="json", exclude_defaults=True) + + +def recipe_differs(kind: str, recipe: Recipe | dict[str, Any], old: Recipe | dict[str, Any] | None) -> bool: + """「配方与现行的不同」：首次、切换，或者与现行配方的 canonical 形式不同。""" + if kind in ("first", "switch") or old is None: + return True + return recipe_canonical(as_recipe(recipe)) != recipe_canonical(as_recipe(old)) + + +# -------------------------------------------------------------------------- +# 破坏性变更 +# -------------------------------------------------------------------------- + + +def _declared_types(recipe: Recipe, tables: dict[str, list[Any]]) -> dict[tuple[str, str], str]: + """(表, 列) → 配方声明的类型。DATE 存成 TEXT，只看 derive_tables 的 SQL 类型就分不出 TEXT ↔ DATE。""" + out = {(t, c.name): ("DATE" if c.role == "axis" else c.type) for t, cols in tables.items() for c in cols} + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, ListBlock): + for col in block.columns: + out[(block.table, col.name)] = col.type + return out + + +def _placeholder_meanings(recipe: Recipe) -> dict[str, dict[str, str]]: + """表 → {占位符（canon）: 含义}。占位符按块配置，作用于这个块写入的所有表。""" + out: dict[str, dict[str, str]] = {} + for sheet in recipe.sheets: + for block in sheet.blocks: + meanings = {canon(p.text): p.meaning for p in block.values.placeholders} + if not meanings: + continue + if isinstance(block, CrosstabBlock): + tables = [s.table for s in block.segments if not isinstance(s, DerivedSegment)] + tables += [s.keep_as.table for s in block.segments if isinstance(s, DerivedSegment) and s.keep_as] + else: + tables = [block.table] + if block.rows.total_row is not None and block.rows.total_row.keep_as: + tables.append(block.rows.total_row.keep_as) + for t in tables: + for text, meaning in meanings.items(): + out.setdefault(t, {}).setdefault(text, meaning) + return out + + +def _unit_name(unit: str | None) -> str: + return unit or "未标注" + + +def _const_values(recipe: Recipe) -> dict[tuple[str, str, str], str]: + """(表, 分段 id, 常量列) → pick。常量列的值整列取自 pick：pick 换了，这一列已有的值就换了。""" + out: dict[tuple[str, str, str], str] = {} + for sheet in recipe.sheets: + for block in sheet.blocks: + if not isinstance(block, CrosstabBlock): + continue + for seg in block.segments: + for col, const in (getattr(seg, "const", None) or {}).items(): + out[(seg.table, seg.id, col)] = const.pick + return out + + +def _const_sets(recipe: Recipe) -> dict[tuple[str, str], set[str]]: + """(表, 常量列) → 所有分段的 pick。不按分段 id 配对：id 和 pick 一起改（把新文件的规则草稿粘进来时， + 规则起草把分段 id 设成 pick 本身）时按 id 配不上对，取值换了也比不到（AU-2）。""" + out: dict[tuple[str, str], set[str]] = {} + for (table, _seg, col), pick in _const_values(recipe).items(): + out.setdefault((table, col), set()).add(pick) + return out + + +def _column_sources(recipe: Recipe) -> dict[tuple[str, str], str]: + """(表, 列) → 这一列取自原表的哪个标签（交叉表 measures）或哪个表头（列表）。""" + out: dict[tuple[str, str], str] = {} + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, CrosstabBlock): + for seg in block.segments: + if isinstance(seg, MeasuresSegment): + for label in seg.labels.expect: + col = seg.measures.get(label) + if col: + out[(seg.table, col)] = label + else: + for c in block.columns: + out[(block.table, c.name)] = c.header + return out + + +def _text_stores(recipe: Recipe) -> dict[tuple[str, str], tuple[str, bool]]: + """(表, 列表的 TEXT 列) → (实际生效的存法, 是否显式写了)。缺省时主键列 canonical、其余 raw。""" + grains = {t.name: set(t.grain) for t in recipe.tables} + out: dict[tuple[str, str], tuple[str, bool]] = {} + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, ListBlock): + for c in block.columns: + if c.type == "TEXT": + eff = c.store or ("canonical" if c.name in grains.get(block.table, set()) else "raw") + out[(block.table, c.name)] = (eff, c.store is not None) + return out + + +_STORE_NAME = {"canonical": "规范写法", "raw": "原文"} + + +def table_changes(old_recipe: Recipe | dict[str, Any], new_recipe: Recipe | dict[str, Any]) -> list[TableChange]: + """新旧配方逐表比较（P2-SPEC 7.5）。新增表、新增列不算；比较范围：表删除或改名、列删除或改名、 + 类型（含 TEXT ↔ DATE 的写法）、单位、grain、kind、值列占位符的含义、常量列的取值，以及每列取自原表的 + 哪个标签或表头、列表文字列的存法（列名类型都没变、内容却整列换了的几种，AU-2）。 + + 期 2 是替换模式，这些变化会让已启用的工作流和报告的口径静默改变；期 3 按期累积时，同一列前后各期单位 + 不一致是灾难性的。表改名和删除分不开（配方里没有改名记录），一律报「删除或改名」。 + """ + old, new = as_recipe(old_recipe), as_recipe(new_recipe) + ot, _ = derive_tables(old) + nt, _ = derive_tables(new) + ospec = {t.name: t for t in old.tables} + nspec = {t.name: t for t in new.tables} + otypes, ntypes = _declared_types(old, ot), _declared_types(new, nt) + oph, nph = _placeholder_meanings(old), _placeholder_meanings(new) + oconst, nconst = _const_values(old), _const_values(new) + osets, nsets = _const_sets(old), _const_sets(new) + osrc, nsrc = _column_sources(old), _column_sources(new) + ostore, nstore = _text_stores(old), _text_stores(new) + out: list[TableChange] = [] + for t, ocols in ot.items(): + if t not in nt: + out.append(TableChange(t, "table_removed", f"表「{t}」不再产出（删除或改名）")) + continue + ncols = {c.name: c for c in nt[t]} + for oc in ocols: + nc = ncols.get(oc.name) + if nc is None: + out.append(TableChange(t, "column_removed", f"删除或改名了列「{oc.name}」", column=oc.name)) + continue + ta, tb = otypes.get((t, oc.name), oc.type), ntypes.get((t, nc.name), nc.type) + if ta != tb: + if {ta, tb} == {"TEXT", "DATE"}: + msg = f"列「{oc.name}」的写法 {ta} → {tb}（日期存为 YYYY-MM-DD 文本）" + else: + msg = f"列「{oc.name}」类型 {ta} → {tb}" + out.append(TableChange(t, "type", msg, column=oc.name, old=ta, new=tb)) + if (oc.unit or None) != (nc.unit or None): + out.append(TableChange(t, "unit", f"列「{oc.name}」的单位 {_unit_name(oc.unit)} → {_unit_name(nc.unit)}", + column=oc.name, old=oc.unit, new=nc.unit)) + # 列名、类型、单位都没变，取自原表的标签或表头换了（两列对调、改成另一行）：这一列的内容整列换了（AU-2） + sa, sb = osrc.get((t, oc.name)), nsrc.get((t, oc.name)) + if sa is not None and sb is not None and match_key(sa) != match_key(sb): + out.append(TableChange(t, "source", f"列「{oc.name}」的来源「{sa}」→「{sb}」", + column=oc.name, old=sa, new=sb)) + # 文字的存法（规范写法 / 原文）换了：按这一列分组、筛选的结果会变。两边都是缺省、只因主键变了而变的， + # 已由 grain 项报出 + ea, eb = ostore.get((t, oc.name)), nstore.get((t, oc.name)) + if ea is not None and eb is not None and ea[0] != eb[0] and (ea[1] or eb[1]): + out.append(TableChange(t, "store", f"列「{oc.name}」的文字存法 {_STORE_NAME[ea[0]]} → {_STORE_NAME[eb[0]]}", + column=oc.name, old=ea[0], new=eb[0])) + os_, ns_ = ospec.get(t), nspec.get(t) + if os_ is not None and ns_ is not None: + if set(os_.grain) != set(ns_.grain): + out.append(TableChange(t, "grain", f"主键 {'、'.join(os_.grain) or '无'} → {'、'.join(ns_.grain) or '无'}", + old=list(os_.grain), new=list(ns_.grain))) + if os_.kind != ns_.kind: + out.append(TableChange(t, "table_kind", + f"表的种类 {_TABLE_KIND[os_.kind]} → {_TABLE_KIND[ns_.kind]}", + old=os_.kind, new=ns_.kind)) + pa, pb = oph.get(t, {}), nph.get(t, {}) + for text in pa: + if text in pb and pa[text] != pb[text]: + out.append(TableChange(t, "placeholder_meaning", f"值列的占位符「{text}」含义 {pa[text]} → {pb[text]}", + old=pa[text], new=pb[text])) + # 常量列的取值换了（分段标题改了写法，常量只能从新标题的候选词里另选，D09）：列名、类型都没变， + # 但这一列的值整列变了，按它筛选、分组的查询和报告会静默变口径 + reported: set[str] = set() + for (table, seg, col), pick in oconst.items(): + new_pick = nconst.get((table, seg, col)) + if table == t and new_pick is not None and new_pick != pick: + out.append(TableChange(t, "const_value", f"分段「{seg}」的列「{col}」取值「{pick}」→「{new_pick}」", + column=col, old=pick, new=new_pick)) + reported.add(col) + # 按 (表, 列) 比全部取值：分段 id 跟着 pick 一起改时上面配不上对（AU-2） + for (table, col), picks in osets.items(): + new_picks = nsets.get((table, col)) + if table == t and col not in reported and new_picks is not None and new_picks != picks: + out.append(TableChange(t, "const_value", + f"列「{col}」的取值「{'、'.join(sorted(picks))}」→「{'、'.join(sorted(new_picks))}」", + column=col, old=sorted(picks), new=sorted(new_picks))) + return out + + +def breaking_changes(old_recipe: Recipe | dict[str, Any], new_recipe: Recipe | dict[str, Any]) -> dict[str, list[str]]: + """表名 → 破坏性变化的说明（含单位变化；确认项里单位另出 unit_changed:*，不在 breaking:* 里重复）。""" + out: dict[str, list[str]] = {} + for ch in table_changes(old_recipe, new_recipe): + out.setdefault(ch.table, []).append(ch.message) + return out + + +def _affinity(sql_type: str) -> str: + """手工源、期 1 上传源的列类型写法 → INTEGER / REAL / TEXT（SQLite 的类型亲和规则，够比较用）。""" + t = sql_type.upper() + if "INT" in t: + return "INTEGER" + if any(k in t for k in ("CHAR", "CLOB", "TEXT")): + return "TEXT" + if any(k in t for k in ("REAL", "FLOA", "DOUB")): + return "REAL" + return t + + +def switch_changes(old_schema_cache: dict[str, Any] | None, new_recipe: Recipe | dict[str, Any]) -> list[str]: + """从简单导入切换为按配方导入时，旧快照里哪些表和列会消失或改变（逐表一句）。""" + nt, _ = derive_tables(as_recipe(new_recipe)) + lines = [] + for name, meta in ((old_schema_cache or {}).get("tables") or {}).items(): + if name not in nt: + lines.append(f"表「{name}」将消失") + continue + ncols = {c.name: c for c in nt[name]} + parts = [] + for col in (meta or {}).get("columns") or []: + cname = str(col.get("name")) + otype = str(col.get("type") or "") + if cname not in ncols: + parts.append(f"列「{cname}」将消失") + elif otype and _affinity(otype) != ncols[cname].type: + parts.append(f"列「{cname}」类型 {_affinity(otype)} → {ncols[cname].type}") + if parts: + lines.append(f"表「{name}」：" + "、".join(parts)) + return lines + + +# -------------------------------------------------------------------------- +# 确认项 +# -------------------------------------------------------------------------- + + +class _Bag: + """按出现顺序收集，id 重复的只留第一条。""" + + def __init__(self) -> None: + self.items: list[ConfirmItem] = [] + self._ids: set[str] = set() + + def add(self, item_id: str, label: str, detail: str = "", source: str = "recipe") -> None: + if item_id in self._ids: + return + self._ids.add(item_id) + self.items.append(ConfirmItem(id=item_id, label=label, detail=detail, required=True, source=source)) + + +def _validate(ctx: ConfirmContext) -> dict[str, Any]: + """输入不全就报错（见模块 docstring），返回 extraction 的 dict 形状。""" + if ctx.kind not in KINDS: + raise ValueError(f"未知的导入方式：{ctx.kind}") + if ctx.kind in ("reupload", "redraft"): + if ctx.old_recipe is None: + raise ValueError(f"{ctx.kind} 必须给现行配方，否则破坏性变更和单位变化无从比较") + if ctx.prev is None: + raise ValueError(f"{ctx.kind} 必须给上一期的导入回执") + if ctx.checks is None: + raise ValueError("必须给本次的核对结果（没有核对时传空列表）") + if ctx.prev is not None and ctx.diff is None: + raise ValueError("有上一期时必须给差异卡（没有差异时传空列表）") + ex = plain(ctx.extraction) + if not isinstance(ex, dict): + raise ReceiptIncomplete("extraction", ["extraction"]) + missing = [k for k in EXTRACTION_KEYS if k not in ex] + if ex.get("partial"): + # 部分干跑只看了前若干行，区域外文字、空行之后的文字都不全，不能拿来定必勾集合 + missing.append("partial") + if missing: + raise ReceiptIncomplete("extraction", missing) + prev = plain(ctx.prev) if ctx.prev is not None else None + if prev is not None and "ledger" in receipt_of(prev): + require_receipt(prev, "prev", _PREV_KEYS, top=()) + return ex + + +def confirm_items(ctx: ConfirmContext) -> list[ConfirmItem]: + """生成必勾的确认项（P2-SPEC 7.5）。顺序：单位变化（排最前）、其余破坏性变更、切换、配方类、AI 类、本期类、差异卡。 + + 输入不全时抛 ValueError（缺键时是它的子类 ReceiptIncomplete），见模块 docstring。 + """ + ex = _validate(ctx) + recipe = as_recipe(ctx.recipe) + old = as_recipe(ctx.old_recipe) if ctx.old_recipe is not None else None + bag = _Bag() + differs = recipe_differs(ctx.kind, recipe, old) + + if differs and old is not None and ctx.kind != "switch": + changes = table_changes(old, recipe) + for ch in changes: + if ch.kind == "unit": + bag.add(f"unit_changed:{ch.table}.{ch.column}", + f"表「{ch.table}」列「{ch.column}」的单位 {_unit_name(ch.old)} → {_unit_name(ch.new)}：" + "数量级或含义可能不同，引用它的报告口径会变") + grouped: dict[str, list[str]] = {} + for ch in changes: + if ch.kind != "unit": + grouped.setdefault(ch.table, []).append(ch.message) + for table, msgs in grouped.items(): + bag.add(f"breaking:{table}", f"表「{table}」：" + "；".join(msgs)) + + if ctx.kind == "switch": + if ctx.old_schema_cache is None: + tail = "原有的表结构未知，切换后以本次导入的表为准" + else: + lines = switch_changes(ctx.old_schema_cache, recipe) + tail = ("以下表和列将消失或改变：" + "；".join(lines)) if lines else "原有的表和列都保留" + bag.add("switch_from_simple", "从简单导入切换为按配方导入：" + tail, source="switch") + + if differs and ctx.mask_columns: + _mask_items(bag, ctx, recipe, old) + + if differs: + _recipe_items(bag, recipe, ex, ctx.facts, ai=ctx.recipe_origin in ("ai", "mixed")) + if ctx.recipe_origin in ("ai", "mixed"): + _ai_items(bag, recipe) + + _period_items(bag, recipe, ex, ctx) + + for d in ctx.diff or []: + d = plain(d) + if isinstance(d, dict) and d.get("requires_confirm") and d.get("confirm_id"): + bag.add(str(d["confirm_id"]), str(d.get("label") or ""), str(d.get("detail") or ""), source="diff") + return bag.items + + +def _mask_items(bag: _Bag, ctx: ConfirmContext, recipe: Recipe, old: Recipe | None) -> None: + """遮罩列在新结构里找不到同名列（不区分大小写）：出必勾 mask_lost:<列>（AU-6）。 + + 遮罩按列名匹配：从简单导入切换过来，「金额（元）」的列名从期 1 的「金额_元」变成「金额」；改配方改了列名也一样。 + 只报这次变化弄丢的：旧结构里本来就没有这一列的遮罩（早就不生效了）不算。 + """ + new_cols = {c.name.lower() for cols in derive_tables(recipe)[0].values() for c in cols} + old_cols: set[str] | None = None + if ctx.kind == "switch" and ctx.old_schema_cache is not None: + old_cols = {str(c.get("name")).lower() for meta in (ctx.old_schema_cache.get("tables") or {}).values() + for c in (meta or {}).get("columns") or [] if isinstance(c, dict)} + elif old is not None: + old_cols = {c.name.lower() for cols in derive_tables(old)[0].values() for c in cols} + for m in ctx.mask_columns or []: + key = str(m).strip() + if not key or key.lower() in new_cols or (old_cols is not None and key.lower() not in old_cols): + continue + bag.add(f"mask_lost:{key}", + f"遮罩列「{key}」在新的表结构里没有同名列：提交后这一列的遮罩不再生效，证据面板和裁判摘录会显示它的值。" + "如需继续遮罩，提交后请在数据源设置里改成新的列名", + source="switch" if ctx.kind == "switch" else "recipe") + + +def _sheet_name(ex: dict[str, Any], sheet: SheetRecipe) -> str: + """配方工作表 → 本期实际的工作表名（改了名的按 fallback 认到的那张）。""" + matched = (ex.get("sheets") or {}).get("matched") or {} + return str(matched.get(sheet.id) or sheet.match.name) + + +def _blocks(recipe: Recipe, ex: dict[str, Any]) -> list[tuple[SheetRecipe, str, Any]]: + return [(sheet, _sheet_name(ex, sheet), block) for sheet in recipe.sheets for block in sheet.blocks] + + +def _count(counts: dict[str, Any], text: str) -> int: + if text in counts: + return int(counts[text] or 0) + return sum(int(v or 0) for k, v in counts.items() if canon(k) == canon(text)) + + +def _recipe_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any], facts: Any, *, ai: bool = False) -> None: + blocks = _blocks(recipe, ex) + crosstabs = [b for _, _, b in blocks if isinstance(b, CrosstabBlock)] + lists = [b for _, _, b in blocks if isinstance(b, ListBlock)] + + # 占位符：配置了就出，写上本期个数（可以是 0） + counts = ex.get("placeholders") or {} + meanings: dict[str, str] = {} + for _, _, b in blocks: + for p in b.values.placeholders: + meanings.setdefault(p.text, p.meaning) + for text, meaning in meanings.items(): + bag.add(f"placeholder:{text}", f"「{text}」存为空值（表示{meaning}，本期 {_count(counts, text)} 格）") + + for b in crosstabs: + if b.values.blank == "null": + bag.add(f"blank_null:{b.id}", f"「{b.id}」数据区的空格存为空值") + for _, _, b in blocks: + if b.values.text_number == "parse_thousands": + bag.add(f"text_number:{b.id}", f"「{b.id}」中千分位写法的数字文本按数字保存") + + # 公式按保存值：交叉表配置了就出；列表的默认值就是放宽，本期确有公式格才出。回执只有全表一个计数， + # 只有一个块按保存值导入时才写个数（多个块分不开，宁可不写也不写错） + accepted = int(ex.get("formula_cells_accepted") or 0) + cached = [b for _, _, b in blocks if b.values.formula == "accept_cached"] + for b in cached: + if isinstance(b, CrosstabBlock) or accepted > 0: + n = f"（本期 {accepted} 格）" if len(cached) == 1 else "" + bag.add(f"formula_cached:{b.id}", f"「{b.id}」中的公式格按文件里保存的值导入{n}") + + for b in lists: + if b.merged_data == "fill": + bag.add(f"merged_fill:{b.id}", f"「{b.id}」数据区的合并单元格按左上格的值填充") + for b in crosstabs: + missing = [name for key, name in _AXIS_CHECK.items() if key not in b.axis.checks] + if missing: + bag.add(f"checks_relaxed:{b.id}", f"「{b.id}」不检查日期" + "、".join(f"「{m}」" for m in missing)) + for sheet in recipe.sheets: + if sheet.context and sheet.context[0].cross_check == "none": + name = _sheet_name(ex, sheet) + bag.add(f"cross_check_off:{name}", f"工作表「{name}」的统计期不与文件名里的日期核对") + + ignored = ex.get("ignored_columns") or {} + for b in lists: + heads = [str(h) for h in ignored.get(b.id) or []] + if b.extra_columns == "ignore" and heads: + bag.add(f"ignored_columns:{b.id}", f"「{b.id}」不导入这些列：" + "".join(f"「{short(h, 20)}」" for h in heads), + "、".join(heads)) + skips = [b for b in lists if b.rows.blank_rows == "skip"] + for b in skips: + n = f"（本期 {int(ex.get('blank_rows_skipped') or 0)} 行）" if len(skips) == 1 else "" + bag.add(f"blank_skip:{b.id}", f"「{b.id}」跳过数据中间的空行{n}") + + # 合计段、合计行改作核对 + seg_rows = {str(s.get("segment")): s.get("rows") for s in ex.get("labels") or [] if isinstance(s, dict)} + for b in crosstabs: + for seg in b.segments: + if isinstance(seg, DerivedSegment): + prefix = rows_text(seg_rows.get(seg.id)) or f"分段「{seg.id}」的合计行" + keep = f"，原值另存表「{seg.keep_as.table}」" if seg.keep_as else "，原值不另存" + bag.add(f"derived:{seg.id}", f"{prefix}改作核对{keep}") + derived_rows: dict[str, set[int]] = {} + for d in ex.get("derived") or []: + if isinstance(d, dict) and d.get("kind") == "column_sum": + _, r, _ = split_coord(_coord_of(d)) + if r is not None: + derived_rows.setdefault(str(d.get("segment")), set()).add(r) + for b in lists: + total = b.rows.total_row + if total is not None: + rows = rows_text(derived_rows.get(b.id)) + prefix = f"「{b.id}」的合计行" + (f"（{rows}）" if rows else "") + keep = f"，原值另存表「{total.keep_as}」" if total.keep_as else "，原值不另存" + # 只有数字列按明细核对：文字列（AI 可能把数字列写成 TEXT）合计行里的数不核对，写在细节里（SE-10） + nums = [c.name for c in b.columns if c.type in ("INTEGER", "REAL")] + detail = (f"按明细核对的列：{'、'.join(nums)}；其余列合计行里的内容不核对" if nums + else "这个块没有数字列，合计行里的内容不核对") + bag.add(f"total_row:{b.id}", f"{prefix}改作核对{keep}", detail) + + # 年份取自统计期：表头是日期格时年份来自格子本身，不算 + forms: dict[str, set[str]] = {} + for a in ex.get("axes") or []: + if isinstance(a, dict): + forms.setdefault(str(a.get("block")), set()).add(str(a.get("form") or "")) + for b in crosstabs: + if b.axis.year_from is not None and forms.get(b.id) != {"date"}: + bag.add(f"year_from:{b.id}", f"「{b.id}」的日期年份取自统计期") + + fact_text = _fact_texts(facts) + for r in recipe.relations: + if isinstance(r, SumEq): + formula = f"{r.total} = " + " + ".join(r.parts) + bag.add(f"relation:{r.id}", f"每期核对「{formula}」（表「{r.table}」）") + elif isinstance(r, NotComparable): + bag.add(f"relation:{r.id}", f"表「{r.a.table}」与表「{r.b.table}」口径不同，不互相推算或相加", + f"按「{r.by}」汇总的「{r.a.value}」不作为「{r.b.value}」的核对依据") + for r in recipe.relations: + if isinstance(r, Dismissed): + what = f"「{fact_text[r.claims]}」" if r.claims in fact_text else f"系统发现的关系 {r.claims}" + # AI 起草的配方里，理由可能出自模型（SE-10）：标明来源，别让说服性的文字看起来像系统的结论 + who = "；理由可能由 AI 填写，请核实" if ai else "" + bag.add(f"dismissed:{r.claims}", f"不登记{what}（理由：{r.reason}{who}）") + + tables, _ = derive_tables(recipe) + for t, cols in tables.items(): + for c in cols: + if c.unit: + head = f"「{c.header}」→ " if c.header else "" + bag.add(f"unit:{t}.{c.name}", f"{head}表「{t}」的列「{c.name}」，单位 {c.unit}") + + hidden = ex.get("hidden") or {} + for sheet in recipe.sheets: + name = _sheet_name(ex, sheet) + h = hidden.get(name) or {} + parts = [] + if sheet.hidden.rows != "reject_if_any": + word = "包含" if sheet.hidden.rows == "include" else "排除" + parts.append(f"隐藏行：{word}（{rows_text(h.get('rows')) or '本期没有隐藏行'}）") + if sheet.hidden.cols != "reject_if_any": + cols = [col_letter(c) if isinstance(c, int) else str(c) for c in h.get("cols") or []] + parts.append(f"隐藏列：包含（{'、'.join(cols) + ' 列' if cols else '本期没有隐藏列'}）") + if parts: + bag.add(f"hidden:{name}", f"工作表「{name}」的" + "；".join(parts)) + + bag.add("mode", "导入模式：每期替换" if recipe.mode == "replace" else "导入模式：按期累积") + + +def _coord_of(d: dict[str, Any]) -> str: + cell = str(d.get("cell") or "") + return cell if "!" in cell or not d.get("sheet") else f"{d['sheet']}!{cell}" + + +def _fact_texts(facts: Any) -> dict[str, str]: + data = plain(facts) + rows = data.get("facts") if isinstance(data, dict) else None + return {str(f["id"]): str(f.get("text") or "") for f in rows or [] if isinstance(f, dict) and f.get("id")} + + +_TYPE_NAME = {"TEXT": "文字", "INTEGER": "整数", "REAL": "小数", "DATE": "日期"} + + +def _col_desc(types: dict[tuple[str, str], str], table: str, c: Any) -> str: + t = types.get((table, c.name), c.type) + parts = [_TYPE_NAME.get(t, t)] + ([c.unit] if c.unit else []) + return f"{c.name}（{'，'.join(parts)}）" + + +def _ai_items(bag: _Bag, recipe: Recipe) -> None: + """AI 起草的配方：列名偏离机械推导的逐列确认（rename），每张表的列（带类型、单位）再整体看一遍（columns）。""" + bag.add("ai_origin", "本配方由 AI 起草，已逐项核对") + tables, _ = derive_tables(recipe) + table_types = _declared_types(recipe, tables) + pos = {(t, c.name): i for t, cols in tables.items() for i, c in enumerate(cols, 1)} + + def check(table: str, source: str, name: str, word: str) -> None: + expected = to_sql_name(split_unit_suffix(source)[0], fallback=f"列{pos.get((table, name), 1)}") + if name != expected: + bag.add(f"rename:{table}.{name}", f"原{word}「{source}」改名为列「{name}」", + f"按{word}推出的列名是「{expected}」") + + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, CrosstabBlock): + for seg in block.segments: + if isinstance(seg, MeasuresSegment): + for label in seg.labels.expect: + check(seg.table, label, seg.measures.get(label, ""), "标签") + else: + for col in block.columns: + check(block.table, col.header, col.name, "表头") + # 每列带上类型和单位（SE-10）：AI 把数字列写成 TEXT 能过静态校验，之后既不做数字检查、合计行也不核对这一列 + for t, cols in tables.items(): + bag.add(f"columns:{t}", f"表「{t}」的列：" + "、".join(_col_desc(table_types, t, c) for c in cols)) + + +def _period_items(bag: _Bag, recipe: Recipe, ex: dict[str, Any], ctx: ConfirmContext) -> None: + """本期类：每次都按本期情况算，与配方变没变无关。""" + prev = plain(ctx.prev) if ctx.prev is not None else None + p_rec = receipt_of(prev) if prev is not None else {} + # 上一期是简单导入时（回执里没有格子账），逐格文字比不了，按首次处理 + has_prev = prev is not None and "ledger" in p_rec + p_out = {o["coord"]: o for o in outside_of(p_rec)} if has_prev else {} + p_per = period_texts(prev, p_rec) if has_prev else {} + c_per = period_texts(ex, ex) + c_ann = period_annotated(ex, ex) + + for o in outside_of(ex): + if o["kind"] != "text_digits": + continue + if has_prev and _outside_exempt(o, p_out, p_per, c_per, c_ann): + continue + bag.add(f"outside_digits:{o['coord']}", f"区域外有含数字的文字「{short(o['text'])}」", + f"{o['coord']}：{o['text']}", source="outside") + + grouped: dict[str, list[dict[str, Any]]] = {} + for p in ex.get("problems") or []: + p = plain(p) + if isinstance(p, dict) and p.get("code") == "rows_after_stop": + grouped.setdefault(_stop_block(p, recipe, ex), []).append(p) + for blk, probs in grouped.items(): + cells = [str(c) for p in probs for c in p.get("cells") or []] + bag.add(f"rows_after_stop:{blk}", "；".join(str(p.get("message") or "") for p in probs), + "、".join(cells[:20]), source="outside") + + # 列表块只有表头、没有数据行（AU-1）：问题对象里没有块键，按消息里的「列表「块」」认 + for p in ex.get("problems") or []: + p = plain(p) + if isinstance(p, dict) and p.get("code") == "list_empty": + blk = next((b.id for _, _, b in _blocks(recipe, ex) + if isinstance(b, ListBlock) and f"列表「{b.id}」" in str(p.get("message") or "")), None) + table = next((b.table for _, _, b in _blocks(recipe, ex) if isinstance(b, ListBlock) and b.id == blk), "") + key = blk or "、".join(str(c) for c in p.get("cells") or []) + what = f"表「{table}」" if table else "这张表" + bag.add(f"empty_block:{key}", f"「{key}」表头之下没有数据行：本期{what}将导入 0 行（上一期的数据会被替换掉）", + "、".join(str(c) for c in p.get("cells") or []), source="outside") + + relations = {r.id: r for r in recipe.relations} + for c in ctx.checks or []: + c = plain(c) + if not isinstance(c, dict) or c.get("kind") != "relation_not_comparable": + continue + checked, failed = int(c.get("checked") or 0), int(c.get("failed") or 0) + # 约定（P2-SPEC 5.2）：failed 是不相等的分组数；全部相等时 failed 记 0、checked 记分组数 + if checked > 0 and failed == 0: + r = relations.get(str(c.get("id"))) + if isinstance(r, NotComparable): + label = (f"表「{r.a.table}」与表「{r.b.table}」声明为口径不同，" + f"但本期按「{r.by}」分组的 {checked} 组全部相等") + else: + label = f"{c.get('title') or c.get('id')}：声明为口径不同，但本期 {checked} 组全部相等" + bag.add(f"relation_observed:{c.get('id')}", label) + + sheets = ex.get("sheets") or {} + p_other = {str(s) for s in (p_rec.get("sheets") or {}).get("other_visible") or []} if has_prev else set() + for name in sheets.get("other_visible") or []: + if str(name) not in p_other: + bag.add(f"sheet_extra:{name}", f"另有可见工作表「{name}」，不导入", source="sheet") + matched = {str(v): str(k) for k, v in (sheets.get("matched") or {}).items()} + for old_name, new_name in (sheets.get("renamed") or {}).items(): + sid = next((s.id for s in recipe.sheets if s.match.name == old_name), None) + if sid is None: + sid = next((s.id for s in recipe.sheets if match_key(s.match.name) == match_key(old_name)), None) + sid = sid or matched.get(str(new_name)) or str(old_name) + bag.add(f"sheet_renamed:{sid}", f"工作表「{old_name}」现在叫「{new_name}」", source="sheet") + + period = period_of(ex) or {} + if period.get("source") == "human": + signed = f"署名（未认证）：{period['signed_by']}" if period.get("signed_by") else "" + bag.add(f"context_human:{_PERIOD_ID}", f"本期统计期为人工录入：{period.get('start')} 至 {period.get('end')}", + signed, source="context") + + +def _outside_exempt(o: dict[str, Any], p_out: dict[str, dict[str, Any]], p_per: dict[str, str], + c_per: dict[str, str], c_ann: dict[str, str]) -> bool: + """带附加文字的统计期格免确认的两种情况（P2-SPEC 7.5 outside_digits）： + + (a) 上一期同一格是区域外纯文字，且它的 match_key 等于本期这格去掉统计期之后的文字（标题加了年月，D17、D27）； + (b) 上一期同一格也是统计期来源，且两期是同一句（每月重复的同一句）：数字遮盖后的模板相同，**并且**只去掉 + 统计期那一段之后的文字（period_residue，其余数字照留）也相同。只比遮盖模板的话，「注：2026年9月数据只含 + 3个分区」→「注：2026年10月数据只含2个分区」会被当成同一句，口径数字变了没人看（AU-3）。任一期认不出 + 统计期（period_residue 为 None）就不免。 + 不是统计期来源的含数字文字（D28 的「注：9月15日…」）每期都要确认；上一期没有这一格的（D28b）也要。 + """ + cell = o["coord"] + if not (o["period_source"] or cell in c_per or cell in c_ann): + return False + prev = p_out.get(cell) + if prev is not None and prev["kind"] == "text": + residue = c_ann.get(cell) + if residue is None: + residue = period_residue(o["text"]) + if residue is not None and match_key(prev["text"]) == match_key(residue): + return True + return cell in p_per and same_period_sentence(p_per[cell], o["text"]) + + +def _stop_block(problem: dict[str, Any], recipe: Recipe, ex: dict[str, Any]) -> str: + """rows_after_stop 属于哪个列表块。问题对象里没有块键，按工作表和溯源的列范围找：同一工作表上被空行停下的 + 列表块只有一个就是它；有多个时取列范围与问题格相交、数据在问题格之上、最靠近的那个。""" + cells = [split_coord(c) for c in problem.get("cells") or []] + sheet = next((s for s, _, _ in cells if s), None) + rows = [r for _, r, _ in cells if r is not None] + cols = {c for _, _, c in cells if c is not None} + cands = [(name, b) for _, name, b in _blocks(recipe, ex) + if isinstance(b, ListBlock) and b.rows.blank_rows == "stop" and (sheet is None or name == sheet)] + if not cands: + lists = [b for _, _, b in _blocks(recipe, ex) if isinstance(b, ListBlock)] + return lists[0].id if lists else str(sheet or "") + if len(cands) == 1 or not rows: + return cands[0][1].id + best: tuple[int, str] | None = None + lineage = ex.get("lineage") or {} + for name, b in cands: + span_cols: set[int] = set() + last = 0 + for runs in (lineage.get(b.table) or {}).values(): + for run in runs or []: + if not isinstance(run, (list, tuple)) or len(run) < 4: + continue + s, r, c = split_coord(f"{run[1]}!{run[2]}" if "!" not in str(run[2]) else run[2]) + if (s or name) != name or r is None or c is None: + continue + span_cols.add(c) + last = max(last, r + int(run[3] or 1) - 1 if (run[4] if len(run) > 4 else "down") == "down" else r) + if span_cols and (not cols or span_cols & cols) and last < min(rows): + if best is None or last > best[0]: + best = (last, b.id) + return best[1] if best else cands[0][1].id diff --git a/backend/app/data/recipe_diff.py b/backend/app/data/recipe_diff.py new file mode 100644 index 00000000..b4e4d251 --- /dev/null +++ b/backend/app/data/recipe_diff.py @@ -0,0 +1,776 @@ +"""差异卡（期 2，WP-5b；P2-SPEC 7.6）：上传新一期时，拿本期的导入回执和上一期的比，列出哪里和上次不一样。 + +**为什么要有差异卡。** 配方能通过试运行，只说明这一期的文件「对得上结构」；标签写法、块的先后、合计由 +公式变成写死的数、区域外多了一句含数字的说明，这些都不会让试运行失败，却可能改变数据的口径（E3、E7、 +R3）。差异卡把它们逐条摆出来：只是信息的（统计期、行数）标成 info，可能改变口径的标成「需确认」， +对应的确认项（`diff::<键>`）由 recipe_confirm 收进必勾清单。 + +**输入形状。** prev / cur 都是「导入回执」``{"receipt": 回执, "checks": [核对摘要], "period": PeriodOut}``， +JSON 形状的 dict（dataclass 也收，先转成 dict）。回执就是 ``dataclasses.asdict(Extraction)`` 去掉 +problems 之后的样子；逐项比较时用到其中的 RECEIPT_KEYS（tables、labels、block_order、derived_form、axes、 +outside_text、ignored_columns、hidden、placeholders、canonicalized、sheets、lineage、full_calc_on_load； +列表的列顺序只能靠溯源里各列首格的列号推出来），外加顶层的 checks。 + +**缺键就报错（ReceiptIncomplete），不按空处理。** 早先的写法是「缺的键按空处理、比不了的项不出」，结果 +调用方少带一个键（清单里没存 axes、cur 用了只有前 50 条规范写法对照的 TrialOut.receipt），需确认的 +axis_form、label_writing、canon_values 就静默消失了（H2）。所以： +- prev、cur 都必须有 ``receipt``，回执里必须有 tables； +- 逐项比较时（上一期也是按配方导入的、配方没变），两边的回执都必须带齐 RECEIPT_KEYS、顶层带 checks； + 规范写法对照被截断（canonicalized_total 大于条数）、或是部分干跑（partial）的结果，同样报错； +- 判断不了配方变没变（没传 recipe_changed，两边又没都带配方哈希）时报错，不默认「没变」去跨配方逐项比； +- 逐项比较时必须给本期配方：列表表头写法的确认项按**块**出，表 → 块只能从配方里查。 + +cur 取自本次试运行；prev 取自**现行导入的导入清单**（7.7），不取 TableBuild.report：构建可能被复用， +回执里没有 C1、C2，文件名也不是那一次导入的（评审甲-16）。上一期是简单导入（kind=switch，prev 是期 1 的 +TableBuild.report，没有 ledger 键）或配方变了（redraft、改配方）时，两边的分段、块、表不一定对得上， +只给 info 类的 rows、period；破坏性变更由 recipe_confirm 按新旧 derive_tables 另算（breaking:*）。 + +**数字遮盖后的模板**（digit_template）：canon 之后把每一段连续的 ASCII 数字换成 ``{}``。文件名、统计期格、 +区域外文字的「写法变了」都按它比，每月必变的日期因此不算变化。 + +纯函数，不碰数据库、不读文件。 +""" +from __future__ import annotations + +import dataclasses +import re +from typing import Any + +from app.data.names import canon +from app.data.recipe_parsers import match_key, period_residue +from app.data.recipe_types import DiffItem, ListBlock, Recipe + +_DIGITS = re.compile(r"[0-9]+") + +#: 差异卡的全部 kind，按界面上的先后（7.6 的表；outside_moved 是表里「只是位置变了的算 info」那一项） +DIFF_KINDS: tuple[str, ...] = ( + "period", "rows", "placeholders", "checks", "file_name", "full_calc", "context_text", + "context_source_added", "column_order", "label_writing", "header_writing", "canon_values", "row_order", + "block_order", "derived_form", "axis_form", "outside_added", "outside_removed", "outside_changed", + "outside_moved", "ignored_columns", "hidden", +) + +#: 需确认的 kind → 确认项 id 的前缀（id = 前缀 + 键，键的格式见契约 recipe_types 的模块 docstring） +CONFIRM_PREFIX: dict[str, str] = { + "label_writing": "diff:label_writing:", + "header_writing": "diff:header_writing:", + "canon_values": "diff:canon:", + "row_order": "diff:row_order:", + "block_order": "diff:block_order:", + "derived_form": "diff:derived_form:", + "axis_form": "diff:axis_form:", + "outside_added": "diff:outside:", + "outside_removed": "diff:outside:", + "outside_changed": "diff:outside:", + "ignored_columns": "diff:ignored:", + "hidden": "diff:hidden:", +} + +_STATUS = {"passed": "通过", "mismatch": "不一致", "unverifiable": "无法核对", "info": "提示"} +_DERIVED_FORM = {"formula": "公式", "literal": "写死的数", "mixed": "公式与写死的数混用"} +_AXIS_FORM = {"text": "文字", "date": "日期格", "mixed": "文字与日期格混用"} +_SHORT = 40 +_A1 = re.compile(r"\$?([A-Za-z]{1,3})\$?([0-9]{1,7})$") +_WS = re.compile(r"\s+") + + +#: 逐项比较时两边回执都必须有的键（7.7 的导入清单 receipt 照 asdict(Extraction) 去掉 problems 写就都有） +RECEIPT_KEYS: tuple[str, ...] = ( + "tables", "labels", "block_order", "derived_form", "axes", "outside_text", "ignored_columns", "hidden", + "placeholders", "canonicalized", "sheets", "lineage", "full_calc_on_load", +) + + +class ReceiptIncomplete(ValueError): + """导入回执缺了比较要用的键、被截断或只是部分干跑：调用方的错，报出来，不按空处理（见模块 docstring）。 + + side 是 prev / cur / extraction，missing 是缺的键（截断、部分干跑时写成 canonicalized_total、partial）。 + """ + + def __init__(self, side: str, missing: list[str]) -> None: + self.side = side + self.missing = list(missing) + super().__init__(f"{side} 不完整，缺少比较要用的内容：{'、'.join(self.missing)}") + + +# -------------------------------------------------------------------------- +# 小工具（recipe_confirm 也用） +# -------------------------------------------------------------------------- + + +def digit_template(text: Any) -> str: + """数字遮盖后的模板：「统计时间范围：2026年9月1日至2026年9月30日」→「统计时间范围：{}年{}月{}日至{}年{}月{}日」。 + + 先 canon（全角数字经 NFKC 变成 ASCII、横线统一、空白折叠），再把每一段连续的 ASCII 数字换成 {}。 + """ + return _DIGITS.sub("{}", canon("" if text is None else str(text))) + + +def same_period_sentence(old: Any, new: Any) -> bool: + """两期的统计期格是不是「每月重复的同一句」：数字遮盖后的模板相同，且只去掉统计期那一段之后剩下的文字 + （period_residue：其余数字照留）相同。任一期认不出统计期（None）就不算同一句。 + + 只比遮盖模板不够（AU-3）：「注：2026年9月数据只含3个分区」与「注：2026年10月数据只含2个分区」模板相同， + 但统计期以外的口径数字变了。确认项（outside_digits 免确认的条件 (b)）与差异卡（outside_changed）共用这一个判定。 + """ + if digit_template(old) != digit_template(new): + return False + ra, rb = period_residue(old), period_residue(new) + return ra is not None and rb is not None and match_key(ra) == match_key(rb) + + +def plain(x: Any) -> Any: + """dataclass（含嵌套）转成 JSON 形状；dict、list 逐层处理，其余原样。调用方传 Extraction 或 asdict 都行。""" + if dataclasses.is_dataclass(x) and not isinstance(x, type): + return dataclasses.asdict(x) + if isinstance(x, dict): + return {k: plain(v) for k, v in x.items()} + if isinstance(x, (list, tuple)): + return [plain(v) for v in x] + return x + + +def coord(sheet: Any, cell: Any) -> str: + """「工作表!A1」。cell 已经带工作表名的原样返回；不知道工作表时只给 A1。""" + cell = str(cell or "") + if "!" in cell or not sheet: + return cell + return f"{sheet}!{cell}" + + +def split_coord(text: Any) -> tuple[str | None, int | None, int | None]: + """「工作表!C5」→ (工作表, 行, 列号)；认不出的部分为 None。""" + s = str(text or "") + sheet, _, a1 = s.rpartition("!") + m = _A1.match(a1.strip()) + if m is None: + return (sheet or None), None, None + col = 0 + for ch in m.group(1).upper(): + col = col * 26 + (ord(ch) - 64) + return (sheet or None), int(m.group(2)), col + + +def col_letter(n: int) -> str: + out = "" + while n > 0: + n, r = divmod(n - 1, 26) + out = chr(65 + r) + out + return out + + +def short(text: Any, limit: int = _SHORT) -> str: + """给标题用的短写法：空白折叠，超长的截断加「…」，全文放 detail。不做 canon：界面上照原文显示， + canon 的 NFKC 会把全角标点变成半角。""" + s = _WS.sub(" ", "" if text is None else str(text)).strip() + return s if len(s) <= limit else s[:limit] + "…" + + +def rows_text(rows: Any) -> str: + """[28, 29, 30] →「第 28–30 行」；不连续的用顿号；空的给空串。""" + nums = sorted({int(r) for r in rows or [] if isinstance(r, int) or str(r).isdigit()}) + if not nums: + return "" + if len(nums) > 1 and nums[-1] - nums[0] == len(nums) - 1: + return f"第 {nums[0]}–{nums[-1]} 行" + return "第 " + "、".join(str(n) for n in nums) + " 行" + + +def receipt_of(doc: Any) -> dict[str, Any]: + doc = plain(doc) or {} + rec = doc.get("receipt") if isinstance(doc, dict) else None + return rec if isinstance(rec, dict) else {} + + +def require_receipt(doc: Any, side: str, keys: tuple[str, ...] = RECEIPT_KEYS, *, + top: tuple[str, ...] = ("checks",)) -> None: + """检查导入回执带齐了 keys（回执里）和 top（顶层），没有被截断、不是部分干跑；不满足就抛 ReceiptIncomplete。""" + doc = plain(doc) + if not isinstance(doc, dict) or not isinstance(doc.get("receipt"), dict): + raise ReceiptIncomplete(side, ["receipt"]) + rec = doc["receipt"] + missing = [k for k in top if k not in doc] + [k for k in keys if k not in rec] + if "canonicalized" in keys and "canonicalized" in rec and "canonicalized_total" in rec: + # TrialOut.receipt 里的 canonicalized 只有前 50 条；拿它比会漏掉新写法 + if int(rec.get("canonicalized_total") or 0) > len(rec.get("canonicalized") or []): + missing.append("canonicalized_total") + if rec.get("partial"): + missing.append("partial") + if missing: + raise ReceiptIncomplete(side, missing) + + +def recipe_sha_of(doc: Any) -> str | None: + """导入回执上的配方哈希：顶层 recipe_sha256，或导入清单的 recipe.sha256（7.7）。""" + doc = plain(doc) or {} + if not isinstance(doc, dict): + return None + sha = doc.get("recipe_sha256") + if not sha and isinstance(doc.get("recipe"), dict): + sha = doc["recipe"].get("sha256") + return str(sha) if sha else None + + +def period_of(doc: Any) -> dict[str, Any] | None: + """导入回执的统计期：顶层 period 优先（人工录入只记在那里），没有再看回执里的。""" + doc = plain(doc) or {} + p = doc.get("period") if isinstance(doc, dict) else None + if not isinstance(p, dict): + p = receipt_of(doc).get("period") + return p if isinstance(p, dict) else None + + +def checks_of(doc: Any) -> list[dict[str, Any]]: + doc = plain(doc) or {} + rows = doc.get("checks") if isinstance(doc, dict) else None + return [c for c in rows or [] if isinstance(c, dict) and c.get("id")] + + +def check_status(doc: Any, check_id: str) -> str | None: + for c in checks_of(doc): + if c.get("id") == check_id: + return str(c.get("status") or "") + return None + + +def default_sheet(rec: dict[str, Any]) -> str | None: + """只有一张配方工作表时它的实际名字：统计期格的坐标万一没带工作表名，靠它补上。""" + matched = (rec.get("sheets") or {}).get("matched") or {} + names = {str(v) for v in matched.values() if v} + if len(names) == 1: + return next(iter(names)) + sheets = {str(o.get("sheet")) for o in rec.get("outside_text") or [] if isinstance(o, dict) and o.get("sheet")} + return next(iter(sheets)) if len(sheets) == 1 else None + + +def outside_of(rec: dict[str, Any]) -> list[dict[str, Any]]: + """区域外文字（含带附加文字的统计期格），坐标统一成「工作表!A1」。""" + out = [] + for o in rec.get("outside_text") or []: + if not isinstance(o, dict): + continue + out.append({"coord": coord(o.get("sheet"), o.get("cell")), "text": str(o.get("text") or ""), + "kind": str(o.get("kind") or "text"), "period_source": bool(o.get("period_source"))}) + return out + + +def period_texts(doc: Any, rec: dict[str, Any]) -> dict[str, str]: + """统计期来源格：坐标 → 全文。""" + p = period_of(doc) or {} + sheet = default_sheet(rec) + return {coord(sheet, k): str(v or "") for k, v in (p.get("texts") or {}).items()} + + +def period_annotated(doc: Any, rec: dict[str, Any]) -> dict[str, str]: + """带附加文字的统计期格：坐标 → period_residue。""" + p = period_of(doc) or {} + sheet = default_sheet(rec) + return {coord(sheet, k): str(v or "") for k, v in (p.get("annotated") or {}).items()} + + +# -------------------------------------------------------------------------- +# 差异卡 +# -------------------------------------------------------------------------- + + +def diff_reports(prev: dict[str, Any] | None, cur: dict[str, Any], *, prev_file_name: str | None, + cur_file_name: str, recipe: Recipe | dict[str, Any] | None = None, + recipe_changed: bool | None = None) -> list[DiffItem]: + """上一期 → 本期的差异卡。prev 为 None（首次）返回 []。需确认的排在前面，同类按 DIFF_KINDS 的顺序。 + + 上一期的回执没有 ledger 键（简单导入，kind=switch）或配方变了（redraft、改配方）时只给 rows、period； + 否则逐项比较。规格之外多了两个参数： + - recipe：本期的配方，逐项比较时必须给。列表的表头写法变化要按**块**出确认项 + （``diff:header_writing:<块>``），回执里只有表，「表 → 块」只能从配方里查。 + - recipe_changed：配方与上一期是否不同。不传时看 prev、cur 两边的配方哈希（顶层 ``recipe_sha256``， + 或导入清单的 ``recipe.sha256``）；两边没都带、又没传时报错，不默认「没变」。传了、两边也都带了哈希 + 而两者矛盾时同样报错。 + """ + if prev is None: + return [] + prev, cur = plain(prev) or {}, plain(cur) or {} + for side, doc in (("prev", prev), ("cur", cur)): + require_receipt(doc, side, ("tables",), top=()) + p_rec, c_rec = receipt_of(prev), receipt_of(cur) + items: list[DiffItem] = [] + items += _diff_period(prev, cur) + items += _diff_rows(p_rec, c_rec) + if "ledger" in p_rec and not _recipe_changed(prev, cur, recipe_changed): + require_receipt(prev, "prev") + require_receipt(cur, "cur") + if recipe is None: + raise ValueError("逐项比较需要本期配方：列表表头写法的确认项按块出，表与块的对应只能从配方里查") + blocks = _list_blocks(recipe) + items += _diff_placeholders(p_rec, c_rec) + items += _diff_checks(prev, cur) + items += _diff_file_name(prev_file_name, cur_file_name) + items += _diff_full_calc(p_rec, c_rec) + items += _diff_context_text(prev, cur, p_rec, c_rec) + items += _diff_column_order(p_rec, c_rec) + items += _diff_labels(p_rec, c_rec) + items += _diff_header_writing(p_rec, c_rec, blocks) + items += _diff_canon(p_rec, c_rec) + items += _diff_block_order(p_rec, c_rec) + items += _diff_forms(p_rec, c_rec) + items += _diff_outside(prev, cur, p_rec, c_rec) + items += _diff_ignored(p_rec, c_rec) + items += _diff_hidden(p_rec, c_rec) + order = {k: i for i, k in enumerate(DIFF_KINDS)} + items.sort(key=lambda d: (not d.requires_confirm, order.get(d.kind, len(order)))) + return items + + +def _recipe_changed(prev: dict[str, Any], cur: dict[str, Any], flag: bool | None) -> bool: + """配方与上一期是否不同。传了 flag 就用它（两边都带哈希时还要对得上）；没传就比两边的配方哈希。""" + a, b = recipe_sha_of(prev), recipe_sha_of(cur) + if a and b: + if flag is not None and flag != (a != b): + raise ValueError("recipe_changed 与 prev、cur 上的配方哈希不一致") + return a != b + if flag is None: + raise ValueError("无法判断配方是否与上一期相同：请传 recipe_changed，或在 prev、cur 上都带配方哈希") + return flag + + +def _item(kind: str, label: str, detail: str = "", key: str | None = None) -> DiffItem: + prefix = CONFIRM_PREFIX.get(kind) + if prefix is None: + return DiffItem(kind=kind, label=label, detail=detail) + return DiffItem(kind=kind, label=label, detail=detail, requires_confirm=True, confirm_id=f"{prefix}{key}") + + +def _period_span(p: dict[str, Any] | None) -> tuple[str, str] | None: + if not p or not p.get("start") or not p.get("end"): + return None + return str(p["start"]), str(p["end"]) + + +def _diff_period(prev: dict[str, Any], cur: dict[str, Any]) -> list[DiffItem]: + a, b = _period_span(period_of(prev)), _period_span(period_of(cur)) + if a == b: + return [] + if a is None: + return [_item("period", f"统计期：本期 {b[0]} 至 {b[1]}", "上一期没有记录统计期")] + if b is None: + return [_item("period", f"统计期：上一期 {a[0]} 至 {a[1]}，本期未能确定")] + return [_item("period", f"统计期 {a[0]} 至 {a[1]} → {b[0]} 至 {b[1]}")] + + +def _table_rows(rec: dict[str, Any]) -> dict[str, int]: + out: dict[str, int] = {} + for t in rec.get("tables") or []: + if isinstance(t, dict) and t.get("name") is not None: + out[str(t["name"])] = int(t.get("rows") or 0) + return out + + +def _placeholder_counts(rec: dict[str, Any]) -> dict[str, int]: + return {str(k): int(v or 0) for k, v in (rec.get("placeholders") or {}).items()} + + +def _diff_rows(p_rec: dict[str, Any], c_rec: dict[str, Any]) -> list[DiffItem]: + a, b = _table_rows(p_rec), _table_rows(c_rec) + if a == b: + return [] + parts = [] + for name in list(dict.fromkeys([*b, *a])): + if name in a and name in b: + if a[name] != b[name]: + parts.append(f"「{name}」{a[name]} → {b[name]}") + elif name in b: + parts.append(f"「{name}」本期新增，{b[name]} 行") + else: + parts.append(f"「{name}」本期没有，上一期 {a[name]} 行") + pa, pb = _placeholder_counts(p_rec), _placeholder_counts(c_rec) + notes = [f"占位符「{t}」{pa.get(t, 0)} → {pb.get(t, 0)} 格" + for t in dict.fromkeys([*pa, *pb]) if pa.get(t, 0) != pb.get(t, 0)] + return [_item("rows", "行数：" + "；".join(parts), "；".join(notes))] + + +def _diff_placeholders(p_rec: dict[str, Any], c_rec: dict[str, Any]) -> list[DiffItem]: + # 只比原文集合：每月个数都会变（31 天 → 30 天），个数写在 rows 的细节里 + a = {t for t, n in _placeholder_counts(p_rec).items() if n > 0} + b = {t for t, n in _placeholder_counts(c_rec).items() if n > 0} + if a == b: + return [] + parts = [] + if b - a: + parts.append("新出现" + "".join(f"「{t}」" for t in sorted(b - a))) + if a - b: + parts.append("不再出现" + "".join(f"「{t}」" for t in sorted(a - b))) + return [_item("placeholders", "占位符：" + "；".join(parts))] + + +def _diff_checks(prev: dict[str, Any], cur: dict[str, Any]) -> list[DiffItem]: + # 比 (id, 状态) 的集合：核对数每期都变，不算 + a = {c["id"]: str(c.get("status") or "") for c in checks_of(prev)} + b = {c["id"]: str(c.get("status") or "") for c in checks_of(cur)} + if set(a.items()) == set(b.items()): + return [] + parts = [] + for cid in dict.fromkeys([*b, *a]): + if cid in a and cid in b: + if a[cid] != b[cid]: + parts.append(f"{cid} {_STATUS.get(a[cid], a[cid])} → {_STATUS.get(b[cid], b[cid])}") + elif cid in b: + parts.append(f"{cid} 本期新出现（{_STATUS.get(b[cid], b[cid])}）") + else: + parts.append(f"{cid} 本期不再出现") + return [_item("checks", "核对结果：" + "；".join(parts))] + + +def _base_name(name: str | None) -> str: + return str(name or "").replace("\\", "/").rsplit("/", 1)[-1] + + +def _diff_file_name(prev_name: str | None, cur_name: str) -> list[DiffItem]: + if prev_name is None: + return [] + a, b = _base_name(prev_name), _base_name(cur_name) + if digit_template(a) == digit_template(b): + return [] + return [_item("file_name", f"文件名的写法变了：「{short(a)}」→「{short(b)}」")] + + +def _diff_full_calc(p_rec: dict[str, Any], c_rec: dict[str, Any]) -> list[DiffItem]: + """本期设了打开时重算就出（规格 7.6「这一期设了」，不论上一期设没设）：导入用的是保存值，每期都提醒。""" + if not c_rec.get("full_calc_on_load"): + return [] + before = "上一期也设置了" if p_rec.get("full_calc_on_load") else "上一期没有设置" + return [_item("full_calc", "本期文件设置了打开时重新计算公式", f"导入使用文件里保存的值；{before}")] + + +def _diff_context_text(prev: dict[str, Any], cur: dict[str, Any], p_rec: dict[str, Any], + c_rec: dict[str, Any]) -> list[DiffItem]: + """两期都是统计期来源、而且都只是在说统计期的格（不在区域外文字里）：写法变了、C1 不是 mismatch 时出 info（D12）。 + + 带附加文字的统计期格同时是区域外文字，写法变化归 outside_*（要确认），不在这里重复。 + """ + if check_status(cur, "C1") == "mismatch": + return [] + a, b = period_texts(prev, p_rec), period_texts(cur, c_rec) + in_outside = {o["coord"] for o in outside_of(p_rec)} | {o["coord"] for o in outside_of(c_rec)} + out = [] + for cell in b: + if cell in a and cell not in in_outside and digit_template(a[cell]) != digit_template(b[cell]): + out.append(_item("context_text", f"统计期格 {cell} 的写法变了：「{short(a[cell])}」→「{short(b[cell])}」", + f"上一期：{a[cell]}；本期：{b[cell]}")) + return out + + +def _is_list_table(t: dict[str, Any]) -> bool: + # 交叉表产出的表一定有轴列（宽表、长表、表内合计都以它开头）；列表合计表是 reported_total + cols = [c for c in t.get("columns") or [] if isinstance(c, dict)] + return str(t.get("kind") or "data") == "data" and not any(c.get("role") == "axis" for c in cols) + + +def _tables(rec: dict[str, Any]) -> dict[str, dict[str, Any]]: + return {str(t["name"]): t for t in rec.get("tables") or [] if isinstance(t, dict) and t.get("name") is not None} + + +def _column_positions(rec: dict[str, Any], table: str) -> dict[str, int]: + """溯源里各列首格的列号：列表一列一段（向下），首格的列号就是这一列在工作表里的位置。""" + out: dict[str, int] = {} + for col, runs in ((rec.get("lineage") or {}).get(table) or {}).items(): + for run in runs or []: + if isinstance(run, (list, tuple)) and len(run) >= 3: + _, _, c = split_coord(run[2]) + if c is not None: + out[str(col)] = min(out.get(str(col), c), c) + return out + + +def _diff_column_order(p_rec: dict[str, Any], c_rec: dict[str, Any]) -> list[DiffItem]: + pa, pb = _tables(p_rec), _tables(c_rec) + out = [] + for name, t in pb.items(): + if name not in pa or not _is_list_table(t) or not _is_list_table(pa[name]): + continue + a, b = _column_positions(p_rec, name), _column_positions(c_rec, name) + common = set(a) & set(b) + if len(common) < 2: + continue + oa = sorted(common, key=lambda c: a[c]) + ob = sorted(common, key=lambda c: b[c]) + if oa != ob: + out.append(_item("column_order", f"表「{name}」的列顺序变了", + f"上一期：{'、'.join(oa)}；本期：{'、'.join(ob)}")) + return out + + +def _segment_labels(rec: dict[str, Any]) -> dict[str, tuple[list[str], list[str]]]: + out: dict[str, tuple[list[str], list[str]]] = {} + for s in rec.get("labels") or []: + if isinstance(s, dict) and s.get("segment") is not None: + out[str(s["segment"])] = ([str(x) for x in s.get("raw") or []], + [str(x) for x in s.get("canonical") or []]) + return out + + +def _diff_labels(p_rec: dict[str, Any], c_rec: dict[str, Any]) -> list[DiffItem]: + """label_writing（同一规范写法的原文变了，D11、D22、D24）与 row_order（分段内顺序变了，D19）。 + + 原文逐字比：en dash 的「7–8」和「7-8」canon 之后相同，正是要报的那种写法变化（E3）。 + """ + a, b = _segment_labels(p_rec), _segment_labels(c_rec) + writing, order = [], [] + for seg, (raw_b, can_b) in b.items(): + if seg not in a: + continue + raw_a, can_a = a[seg] + map_a = dict(zip(reversed(can_a), reversed(raw_a))) + map_b = dict(zip(reversed(can_b), reversed(raw_b))) + pairs = [(map_a[c], map_b[c]) for c in dict.fromkeys(can_b) if c in map_a and map_a[c] != map_b[c]] + if pairs: + shown = "；".join(f"「{x}」→「{y}」" for x, y in pairs[:3]) + ("等" if len(pairs) > 3 else "") + writing.append(_item("label_writing", f"分段「{seg}」的标签写法变了：{shown}", + f"共 {len(pairs)} 个标签；按规范写法存储的值不变", key=seg)) + common = set(can_a) & set(can_b) + seq_a = [c for c in dict.fromkeys(can_a) if c in common] + seq_b = [c for c in dict.fromkeys(can_b) if c in common] + if seq_a != seq_b: + order.append(_item("row_order", f"分段「{seg}」的行顺序变了", + f"上一期：{'、'.join(seq_a)}；本期：{'、'.join(seq_b)}", key=seg)) + return writing + order + + +def _list_blocks(recipe: Recipe | dict[str, Any]) -> dict[str, str]: + """列表的表名 → 块 id（只看本期配方）。""" + r = recipe if isinstance(recipe, Recipe) else Recipe.model_validate(recipe) + out: dict[str, str] = {} + for sheet in r.sheets: + for block in sheet.blocks: + if isinstance(block, ListBlock): + out.setdefault(block.table, block.id) + return out + + +def _diff_header_writing(p_rec: dict[str, Any], c_rec: dict[str, Any], blocks: dict[str, str]) -> list[DiffItem]: + pa, pb = _tables(p_rec), _tables(c_rec) + out = [] + for name, t in pb.items(): + if name not in pa or not _is_list_table(t): + continue + if name not in blocks: + # 回执里的列表表在配方里找不到块：传进来的配方不是产出这份回执的那份 + raise ValueError(f"本期配方里没有写入表「{name}」的列表块，配方与回执对不上") + ha = {str(c.get("name")): c.get("header") for c in pa[name].get("columns") or [] if isinstance(c, dict)} + pairs = [] + for c in t.get("columns") or []: + if not isinstance(c, dict): + continue + old, new = ha.get(str(c.get("name"))), c.get("header") + if old is not None and new is not None and str(old) != str(new): + pairs.append((str(old), str(new))) + if pairs: + shown = "；".join(f"「{short(x)}」→「{short(y)}」" for x, y in pairs[:3]) + ("等" if len(pairs) > 3 else "") + out.append(_item("header_writing", f"表「{name}」的表头写法变了：{shown}", + f"共 {len(pairs)} 列；列名和导入的值不变", key=blocks[name])) + return out + + +def _diff_canon(p_rec: dict[str, Any], c_rec: dict[str, Any]) -> list[DiffItem]: + """列表的主键列、canonical 存储的 TEXT 列出现新的「原文 ≠ 规范写法」对（来自标签的值不在 canonicalized 里）。""" + def pairs(rec: dict[str, Any]) -> list[tuple[str, str, str, str]]: + return [(str(e.get("table")), str(e.get("column")), str(e.get("raw")), str(e.get("canonical"))) + for e in rec.get("canonicalized") or [] if isinstance(e, dict)] + + seen = set(pairs(p_rec)) + grouped: dict[tuple[str, str], list[tuple[str, str]]] = {} + for t, c, raw, can in pairs(c_rec): + if (t, c, raw, can) not in seen: + grouped.setdefault((t, c), []).append((raw, can)) + out = [] + for (t, c), new in grouped.items(): + shown = "；".join(f"「{short(r)}」按「{short(k)}」保存" for r, k in new[:3]) + ("等" if len(new) > 3 else "") + out.append(_item("canon_values", f"表「{t}」列「{c}」出现新的写法：{shown}", + f"共 {len(new)} 种", key=f"{t}.{c}")) + return out + + +def _sheet_ids(rec: dict[str, Any]) -> dict[str, str]: + """实际工作表名 → 配方工作表 id：工作表改了名，块的先后照样能和上一期对上。""" + return {str(v): str(k) for k, v in ((rec.get("sheets") or {}).get("matched") or {}).items()} + + +def _diff_block_order(p_rec: dict[str, Any], c_rec: dict[str, Any]) -> list[DiffItem]: + a = {str(k): [str(x) for x in v or []] for k, v in (p_rec.get("block_order") or {}).items()} + b = {str(k): [str(x) for x in v or []] for k, v in (c_rec.get("block_order") or {}).items()} + ida, idb = _sheet_ids(p_rec), _sheet_ids(c_rec) + by_id_a = {ida[k]: v for k, v in a.items() if k in ida} + out = [] + for sheet, seq_b in b.items(): + seq_a = a.get(sheet) + if seq_a is None and sheet in idb: + seq_a = by_id_a.get(idb[sheet]) + if seq_a is None: + continue + common = set(seq_a) & set(seq_b) + fa = [k for k in seq_a if k in common] + fb = [k for k in seq_b if k in common] + if fa != fb: + out.append(_item("block_order", f"工作表「{sheet}」上各部分的先后变了", + f"上一期：{'、'.join(fa)}；本期：{'、'.join(fb)}", key=sheet)) + return out + + +def _diff_forms(p_rec: dict[str, Any], c_rec: dict[str, Any]) -> list[DiffItem]: + out = [] + a = {str(k): str(v) for k, v in (p_rec.get("derived_form") or {}).items()} + for seg, form in ((c_rec.get("derived_form") or {}).items()): + seg, form = str(seg), str(form) + if seg in a and a[seg] != form: + out.append(_item("derived_form", + f"分段「{seg}」的合计由{_DERIVED_FORM.get(a[seg], a[seg])}改为" + f"{_DERIVED_FORM.get(form, form)}", + "合计仍按明细核对", key=seg)) + + def axes(rec: dict[str, Any]) -> dict[str, str]: + return {str(x.get("block")): str(x.get("form") or "") for x in rec.get("axes") or [] + if isinstance(x, dict) and x.get("block") is not None} + + aa, ab = axes(p_rec), axes(c_rec) + for blk, form in ab.items(): + if blk in aa and aa[blk] != form: + out.append(_item("axis_form", + f"「{blk}」的日期表头由{_AXIS_FORM.get(aa[blk], aa[blk])}改为{_AXIS_FORM.get(form, form)}", + key=blk)) + return out + + +def _diff_outside(prev: dict[str, Any], cur: dict[str, Any], p_rec: dict[str, Any], + c_rec: dict[str, Any]) -> list[DiffItem]: + """区域外文字（含带附加文字的统计期格）的增、删、改，以及「区域外纯文字变成统计期来源」（D17、D27）。 + + 顺序：先按全文配对（同格的先配；只是位置变了的出 info）；再看同一格从纯文字变成统计期来源、满足条件的 + 归入 context_source_added；剩下同一格的按数字遮盖后的模板比（只是日期变了不算改）；其余算增、删。 + """ + p_out, c_out = outside_of(p_rec), outside_of(c_rec) + p_per, c_per = period_texts(prev, p_rec), period_texts(cur, c_rec) + c_ann = period_annotated(cur, c_rec) + c1_ok = check_status(cur, "C1") == "passed" + info: list[DiffItem] = [] + confirm: list[DiffItem] = [] + + def take(pool: list[dict[str, Any]], pred: Any) -> dict[str, Any] | None: + for i, x in enumerate(pool): + if pred(x): + return pool.pop(i) + return None + + # 1. 全文配对 + p_left, c_left = list(p_out), [] + for c in c_out: + if take(p_left, lambda p, c=c: p["coord"] == c["coord"] and canon(p["text"]) == canon(c["text"])) is None: + c_left.append(c) + rest = [] + for c in c_left: + p = take(p_left, lambda p, c=c: canon(p["text"]) == canon(c["text"])) + if p is None: + rest.append(c) + else: + info.append(_item("outside_moved", f"区域外文字「{short(c['text'])}」从 {p['coord']} 移到 {c['coord']}")) + c_left = rest + + # 2. 纯文字变成统计期来源 + for p in list(p_left): + if p["kind"] != "text" or p["coord"] not in c_per: + continue + c = next((x for x in c_left if x["coord"] == p["coord"]), None) + # 本期这一格还在区域外文字里，按契约就是带附加文字的统计期格（period_source 漏标也一样） + annotated = p["coord"] in c_ann or c is not None + residue = c_ann.get(p["coord"]) + if annotated and residue is None and c is not None: + residue = period_residue(c["text"]) + if not c1_ok or (annotated and (residue is None or match_key(residue) != match_key(p["text"]))): + continue + p_left.remove(p) + if c is not None: + c_left.remove(c) + new_text = c_per[p["coord"]] + info.append(_item("context_source_added", + f"{p['coord']} 现在也写明了统计期：「{short(p['text'])}」→「{short(new_text)}」", + "统计期与其他来源一致；附加的文字与上一期这一格相同")) + + # 3. 同一格：写法（数字遮盖后的模板）变了才算改。两期都是统计期来源的格，还要比去掉统计期之后的文字 + # （其余数字照留）：附加文字里的口径数字变了也算改（AU-3） + def changed(cell: str, old: str, new: str) -> None: + if cell in p_per and cell in c_per: + differ = not same_period_sentence(old, new) + else: + differ = digit_template(old) != digit_template(new) + if differ: + confirm.append(_item("outside_changed", f"{cell} 的区域外文字改了：「{short(old)}」→「{short(new)}」", + f"上一期：{old}；本期：{new}", key=cell)) + + for p in list(p_left): + c = take(c_left, lambda x, p=p: x["coord"] == p["coord"]) + if c is not None: + p_left.remove(p) + changed(p["coord"], p["text"], c["text"]) + elif p["coord"] in c_per: + # 这一格本期只是在说统计期了，但不满足 context_source_added 的条件 + p_left.remove(p) + changed(p["coord"], p["text"], c_per[p["coord"]]) + for c in list(c_left): + if c["coord"] in p_per: + # 上一期只是统计期格，本期多了附加文字 + c_left.remove(c) + changed(c["coord"], p_per[c["coord"]], c["text"]) + + # 4. 其余是增、删 + for p in p_left: + confirm.append(_item("outside_removed", f"区域外文字「{short(p['text'])}」本期没有了（上一期在 {p['coord']}）", + p["text"], key=p["coord"])) + for c in c_left: + confirm.append(_item("outside_added", f"区域外新增文字「{short(c['text'])}」（{c['coord']}）", + c["text"], key=c["coord"])) + return confirm + info + + +def _diff_ignored(p_rec: dict[str, Any], c_rec: dict[str, Any]) -> list[DiffItem]: + a = {str(k): [str(x) for x in v or []] for k, v in (p_rec.get("ignored_columns") or {}).items()} + b = {str(k): [str(x) for x in v or []] for k, v in (c_rec.get("ignored_columns") or {}).items()} + out = [] + for blk in dict.fromkeys([*b, *a]): + ka = {match_key(x) for x in a.get(blk, [])} + kb = {match_key(x) for x in b.get(blk, [])} + if ka != kb: + now = "".join(f"「{short(x, 20)}」" for x in b.get(blk, [])) or "无" + before = "".join(f"「{short(x, 20)}」" for x in a.get(blk, [])) or "无" + out.append(_item("ignored_columns", f"「{blk}」忽略的列变了：本期忽略{now}", + f"上一期忽略{before}", key=blk)) + return out + + +def _hidden_parts(h: dict[str, Any]) -> tuple[set[int] | None, set[str] | None]: + rows = cols = None + if str(h.get("policy_rows") or "") in ("include", "exclude"): + rows = {int(r) for r in h.get("rows") or [] if isinstance(r, int) or str(r).isdigit()} + if str(h.get("policy_cols") or "") == "include": + cols = {col_letter(c) if isinstance(c, int) else str(c).upper() for c in h.get("cols") or []} + return rows, cols + + +def _diff_hidden(p_rec: dict[str, Any], c_rec: dict[str, Any]) -> list[DiffItem]: + """照常导入（include）或排除（exclude）隐藏行列时，隐藏的是哪些行列变了。 + + 规格只写了 include；exclude 下被排除的行变了同样会静默改变导入的数据，一并报（见交付说明）。 + reject_if_any 时有隐藏行列就拒收，不会走到这里。 + """ + a = {str(k): v for k, v in (p_rec.get("hidden") or {}).items() if isinstance(v, dict)} + b = {str(k): v for k, v in (c_rec.get("hidden") or {}).items() if isinstance(v, dict)} + out = [] + for sheet in dict.fromkeys([*b, *a]): + rb, cb = _hidden_parts(b.get(sheet, {})) + ra, ca = _hidden_parts(a.get(sheet, {})) + if rb is None and cb is None and ra is None and ca is None: + continue + parts = [] + if (ra or set()) != (rb or set()) and (ra is not None or rb is not None): + parts.append("隐藏行：" + (rows_text(rb) or "本期没有")) + if (ca or set()) != (cb or set()) and (ca is not None or cb is not None): + parts.append("隐藏列：" + ("、".join(sorted(cb or ())) + " 列" if cb else "本期没有")) + if parts: + out.append(_item("hidden", f"工作表「{sheet}」的隐藏行列变了：" + "；".join(parts), key=sheet)) + return out diff --git a/backend/app/data/recipe_engine.py b/backend/app/data/recipe_engine.py new file mode 100644 index 00000000..4f7b090c --- /dev/null +++ b/backend/app/data/recipe_engine.py @@ -0,0 +1,2523 @@ +"""配方执行器（期 2，WP-2）：按已确认的配方把 Excel 工作表确定性地抽成表，写进一个临时 SQLite 库。 + +**为什么要有执行器而不是「按原样导入」。** 交叉表、多块报表按原样进库，同一列里混着不同口径的行，模型只能 +猜（期 1 的「未规整」）。配方把「哪些格是数据、落进哪张表哪一列、合计怎么核对」写成封闭的规则，执行器逐格 +执行、逐格记账，任何对不上的地方都报出来，绝不猜（H1–H3）。 + +**每个非空格恰好一个去向（格子账）。** 去向见契约 Role：值、合计、表头、标签、分段标题、统计期、区域外文字、 +忽略的列、排除的隐藏行……没有去向的格是问题：数据行上的叫「没有去处」（cell_unclaimed），区域外的数字叫 +「区域外数字」（outside_number），没有分段认领的数据行叫 row_unclaimed。读取层和扫描层各数一遍非空格， +逐行对账（pass_mismatch），两个解析器看到的必须是同一张表。 + +**问题收集策略。** 定位失败的分段（或块）跳过抽取，其余照常执行，一次报全（9.4 要求 label_duplicate 和 +pk_duplicate 同时出现）。块级的定位失败（找不到日期表头、找不到表头）时，这张工作表的区域外数字、没有去处的格 +不再逐个报：区域都不知道在哪，报出来的只是噪声，根因已经报了。 + +**给模型看的问题文字（model_message）只有位置和配方里的名字。** 坐标、行号、个数、分段 id、表名列名、配方 +写的期望标签和表头；格子里的任何内容（数值、文字、日期）都不放——AI 修订时回灌的就是它（P2-SPEC 6.2）， +模型要看格子写了什么，去压缩表示里按坐标找。message 是给人看的，可以带格子原文。 + +**执行顺序**（P2-SPEC 4.2）：认工作表 → 选读法（网格 / 流式）→ 定位各块（交叉表定位不需要年份；列表直接抽完） +→ 统计期（只看区域外的文字格）→ 交叉表补年份、断言、抽取 → 区域外文字分类 → 隐藏行列 → 格子账收尾 → 写库。 + +**写库。** 一个事务写完全部表（默认的回滚日志模式，不开 WAL：WAL 下 settle 会改写文件头，库哈希跟着变）； +有 structure / input / recipe 类问题时不留下库文件；任何异常都先删掉库文件再抛。table_hashes 在关掉写连接之后 +用 SQL 排序流式算，不在 Python 里整表排序。库文件哈希不由这里算（调用方 settle_journal 之后再算）。 + +建库连接用 engine.open_checked_sqlite（关 DQS，和核对模块同一个入口）。建表语句不经 SQL 守卫 +（同 tabular.py 开头的理由：导入是我们自己发起的可信操作）。 +""" +from __future__ import annotations + +import bisect +import datetime as _dt +import hashlib +import json +import os +import re +import sqlite3 +import time +from collections import Counter +from collections.abc import Callable, Iterator +from dataclasses import dataclass, field +from typing import Any + +from app.data import engine as _data_engine +from app.data import tabular, xlsx_cells +from app.data.names import canon +from app.data.recipe_parsers import ( + AxisDate, HourRange, HourTotal, Period, classify_outside, cn_date_range, cn_year_month, filename_period, + hour_range, hour_range_total, is_pure_period_cell, match_key, month_day_or_date, period_residue, + text_label, thousands_number, +) +from app.data.recipe_types import ( + AxisOut, CanonEntry, CheckResult, ColumnOut, ContextSpec, CrosstabBlock, DerivedItem, DimensionSegment, + Extraction, Grid, GridCell, LedgerSheet, ListBlock, MeasuresSegment, OutsideText, + PeriodInput, PeriodOut, Problem, Recipe, RegionMark, SegmentLabels, SheetRecipe, TableOut, TableSpec, + derive_tables, +) +from app.data.xlsx_scan import MAX_MERGED, SheetScan, WorkbookScan, cell_ref, col_letter + +__all__ = ["ANCHOR_WINDOW", "PROBLEM_CODES", "PROBLEM_FIX", "execute"] + +# -------------------------------------------------------------------------- +# 常量 +# -------------------------------------------------------------------------- + +#: 执行器能产出的全部问题 code → 类别。test_recipe_engine_messages 维护一份同样的表，逐个 code 造夹具断言 +#: model_message 非空、且不含触发格的值：这里新增 code 不补用例，那个测试就失败 +PROBLEM_CODES: dict[str, str] = { + # 配方本身（derive_tables 报的形状问题：调用方必须拦下，不能拿它产出的列去建表） + "table_shape_conflict": "recipe", "measures_keys_mismatch": "recipe", + # 工作表与读取 + "sheet_missing": "structure", "sheet_unexpected": "structure", "sheet_too_large_for_layout": "structure", + "merged_too_many": "structure", "sparse": "structure", "pass_mismatch": "structure", + # 格子账、区域外 + "cell_unclaimed": "structure", "outside_number": "structure", "outside_digits": "confirm", + "hidden_rows": "structure", "hidden_cols": "structure", + # 统计期 + "period_missing": "input", + # 交叉表定位 + "axis_not_found": "structure", "axis_ambiguous": "structure", "axis_gap": "structure", + "axis_extra_cells": "structure", "axis_year_missing": "structure", "axis_year_ambiguous": "structure", + "axis_duplicate": "structure", "axis_not_contiguous": "structure", "axis_coverage": "structure", + "row_without_label": "structure", "row_unclaimed": "structure", "title_not_found": "structure", + "title_ambiguous": "structure", "segment_not_found": "structure", "segment_overlap": "structure", + "label_missing": "structure", "label_unexpected": "structure", "label_duplicate": "structure", + "label_unparsed": "structure", "const_missing": "structure", "merged_in_values": "structure", + # 列表定位 + "header_not_found": "structure", "header_ambiguous": "structure", "column_missing": "structure", + "column_extra": "structure", "rows_after_stop": "confirm", "total_row_missing": "structure", + "total_row_blank": "structure", "list_empty": "confirm", + # 值 + "value_not_integer": "structure", "value_text_number": "structure", "value_blank": "structure", + "value_error": "structure", "value_not_number": "structure", "value_not_date": "structure", + "value_formula": "structure", "formula_uncached": "structure", "formula_full_calc": "structure", + "pk_duplicate": "structure", +} + +#: 期 3 修复按钮的种类（契约 Problem.fix）。期 2 只透传 +PROBLEM_FIX: dict[str, str] = { + "label_missing": "remove_label", "label_unexpected": "add_label", "title_not_found": "rename_title", + "axis_extra_cells": "ignore_cells", "outside_number": "ignore_cells", "cell_unclaimed": "ignore_cells", + "row_unclaimed": "add_label", "label_unparsed": "declare_total", "rows_after_stop": "ignore_cells", + "value_not_number": "declare_placeholder", +} + +#: 列表找表头的窗口：从 after_title 的下一行（或已用区域首行）起往下这么多行 +ANCHOR_WINDOW = 200 +#: 写库时每批 executemany 的行数 +WRITE_BATCH = 5000 +#: 问题的格子坐标最多几个（契约 Problem.cells） +CELLS_MAX = 20 +#: 同一工作表、同一 code 最多逐条列出的问题数，其余合成一条「另有 N 处」 +PER_CODE_MAX = 50 +#: 规范写法对照最多几条 +CANON_MAX = 500 +#: 公式区域展开的上限：更大的区域不可能全落在基表里，直接判「引用了基表之外的格」 +REF_CELLS_MAX = 200_000 + +#: 块认领的去向（区域外文字、统计期不算）：隐藏行列只看这些格所在的行列 +_BLOCK_ROLES = frozenset({"value", "derived_value", "derived_label", "col_header", "row_label", "section_title", + "ignored_column", "hidden_excluded", "total_label", "total_value"}) +#: 块级定位失败的工作表上不再报的「区域外 / 没有去处」类问题（根因已经报了） +_NOISE_CODES = frozenset({"outside_number", "outside_digits", "cell_unclaimed", "row_unclaimed", + "row_without_label", "rows_after_stop"}) +#: after_title 最长 40 字，算上空白和全角写法，比这更长的格不可能是它 +_TITLE_SCAN_MAX = 200 +_PARSER_NAME = { + "hour_range": "时段写法（如 7-8）", "hour_range_total": "时段合计写法（如 18-22时合计）", + "text": "文字", "measures": "标签", +} + + +# -------------------------------------------------------------------------- +# 小工具 +# -------------------------------------------------------------------------- + + +def _coord(sheet: str, r: int, c: int) -> str: + return f"{sheet}!{cell_ref(r, c)}" + + +def _q(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def _text(value: Any) -> str: + """格子的文字：文本原样，其余按简单导入的写法（tabular.as_text）。""" + if isinstance(value, str): + return value + return tabular.as_text(value) + + +def _shown(value: Any, limit: int = 20) -> str: + """给人看的一格内容（截断）。""" + text = _text(value).strip() + return text if len(text) <= limit else text[:limit] + "…" + + +def _join(items: list[str], limit: int = 10) -> str: + shown = "、".join(items[:limit]) + return shown + (f" 等 {len(items)} 处" if len(items) > limit else "") + + +def _quoted(items: list[str], limit: int = 10) -> str: + return _join([f"「{x}」" for x in items], limit) + + +def _rows_text(rows: list[int], limit: int = 10) -> str: + """行号列表 → 「10–20、23」：相邻的合成一段。""" + spans: list[list[int]] = [] + for r in sorted(set(rows)): + if spans and r == spans[-1][1] + 1: + spans[-1][1] = r + else: + spans.append([r, r]) + return _join([str(a) if a == b else f"{a}–{b}" for a, b in spans], limit) + + +def _in_spans(spans: list[tuple[int, int]], starts: list[int], x: int) -> bool: + i = bisect.bisect_right(starts, x) - 1 + return i >= 0 and spans[i][0] <= x <= spans[i][1] + + +def _open_sqlite(path: str, *, readonly: bool) -> sqlite3.Connection: + """关 DQS 的 sqlite3 连接，和核对模块共用 engine.open_checked_sqlite：关不掉就拒绝连接。""" + return _data_engine.open_checked_sqlite(path, readonly=readonly) + + +def _remove_db(path: str) -> None: + for p in (path, path + "-journal", path + "-wal", path + "-shm"): + try: + os.unlink(p) + except FileNotFoundError: + pass + + +# -------------------------------------------------------------------------- +# 公式形状（合计格的引用核对，4.6） +# -------------------------------------------------------------------------- + +_REF = r"\$?([A-Za-z]{1,3})\$?([0-9]{1,7})" +_AREA = rf"{_REF}(?:\s*:\s*{_REF})?" +_AREAS = rf"{_AREA}(?:\s*,\s*{_AREA})*" +_F_SUM = re.compile(rf"=\s*SUM\s*\(\s*({_AREAS})\s*\)\s*", re.IGNORECASE | re.ASCII) +_F_SUBTOTAL = re.compile(rf"=\s*SUBTOTAL\s*\(\s*(?:9|109)\s*,\s*({_AREAS})\s*\)\s*", re.IGNORECASE | re.ASCII) +_F_AGGREGATE = re.compile(rf"=\s*AGGREGATE\s*\(\s*9\s*,\s*[0-9]{{1,2}}\s*,\s*({_AREAS})\s*\)\s*", + re.IGNORECASE | re.ASCII) +_F_PLUS = re.compile(rf"=\s*{_REF}(?:\s*\+\s*{_REF})*\s*", re.ASCII) +_ONE_AREA = re.compile(_AREA, re.ASCII) +_ONE_REF = re.compile(_REF, re.ASCII) + + +def _col_no(letters: str) -> int: + n = 0 + for ch in letters.upper(): + n = n * 26 + ord(ch) - 64 + return n + + +def formula_refs(formula: str) -> tuple[str, list[tuple[int, int]]] | None: + """合计公式 → (函数, 引用的格 [(行, 列)…])。认四种形状：SUM(区域…)、单格相加、SUBTOTAL(9/109, 区域)、 + AGGREGATE(9, 选项, 区域)，允许 $ 和空白；别的工作表的引用、别的函数一律认不出（None）。区域太大时 + 给出的格子列表是空的、函数照给（调用方按「引用了基表之外的格」处理）。""" + text = formula.strip() + for func, rx in (("SUM", _F_SUM), ("SUBTOTAL", _F_SUBTOTAL), ("AGGREGATE", _F_AGGREGATE)): + m = rx.fullmatch(text) + if m is None: + continue + cells: list[tuple[int, int]] = [] + for a in _ONE_AREA.finditer(m.group(1)): + c1, r1 = _col_no(a.group(1)), int(a.group(2)) + c2, r2 = (_col_no(a.group(3)), int(a.group(4))) if a.group(3) else (c1, r1) + r1, r2 = min(r1, r2), max(r1, r2) + c1, c2 = min(c1, c2), max(c1, c2) + if (r2 - r1 + 1) * (c2 - c1 + 1) + len(cells) > REF_CELLS_MAX: + return func, [] + cells.extend((r, c) for r in range(r1, r2 + 1) for c in range(c1, c2 + 1)) + return func, cells + if _F_PLUS.fullmatch(text): + return "plus", [(int(m.group(2)), _col_no(m.group(1))) for m in _ONE_REF.finditer(text)] + return None + + +# -------------------------------------------------------------------------- +# 溯源、区域标记、两遍对账、合并区 +# -------------------------------------------------------------------------- + + +class _Runs: + """一列的溯源：连续 rowid 且格子沿同一行向右（或同一列向下）的合成一段（recipe-ui 原型的 _lineage_runs）。""" + + __slots__ = ("runs",) + + def __init__(self) -> None: + #: [起始 rowid, 工作表, 起始行, 起始列, 个数, 方向] + self.runs: list[list[Any]] = [] + + def add(self, rowid: int, sheet: str, r: int, c: int) -> None: + runs = self.runs + if runs: + last = runs[-1] + start, lsheet, lr, lc, n, way = last + if lsheet == sheet and start + n == rowid: + if way == "right" and r == lr and c == lc + n: + last[4] = n + 1 + return + if way == "down" and c == lc and r == lr + n: + last[4] = n + 1 + return + if n == 1: + if r == lr and c == lc + 1: + last[4], last[5] = 2, "right" + return + if c == lc and r == lr + 1: + last[4], last[5] = 2, "down" + return + runs.append([rowid, sheet, r, c, 1, "right"]) + + def out(self) -> list[list[Any]]: + return [[start, sheet, cell_ref(r, c), n, way] for start, sheet, r, c, n, way in self.runs] + + +def _row_spans(cells: list[tuple[int, GridCell]], claims: dict[int, str], + layout: list[tuple[int, int, str]] | None) -> list[tuple[int, int, str]]: + """一行的区域段 [(起列, 止列, 去向)]。 + + 块登记了这一行的固定列段（layout：列表数据行是本块的列范围，交叉表数据行是标签列和日期列）时，按固定列段 + 出段，不看这一行哪几格碰巧是空的：末列常空的「备注」、隔行为空的金额，否则每行的段都不一样，20 万行的表会 + 出 20 万个矩形（4.9「区域标记按矩形」）。固定列段里有去向对不上的格（块之间认领重叠）时退回按格出段。 + 其余的格：同一去向的相邻已认领格合成一段（中间的空格一并覆盖，着色无妨），没有认领的格把段隔开（未认领的格 + 不在区域标记里，由问题的 cells 指出）。""" + items: list[tuple[int, int, str | None]] = [] + if layout: + fixed = sorted(layout) + for c, _ in cells: + for a, b, role in fixed: + if a <= c <= b: + if claims.get(c) != role: + layout = None + break + else: + items.append((c, c, claims.get(c))) + if layout is None: + items = [] + break + if layout: + items.extend(fixed) + items.sort(key=lambda x: x[0]) + if not layout: + items = [(c, c, claims.get(c)) for c, _ in cells] + out: list[list[Any]] = [] + broken = True + for a, b, role in items: + if role is None: + broken = True + continue + if out and not broken and out[-1][2] == role: + out[-1][1] = max(out[-1][1], b) + else: + out.append([a, b, role]) + broken = False + return [(a, b, role) for a, b, role in out] + + +class _Regions: + """按行喂入区域段，同一去向、同一列范围、行相接的合成矩形（给网格着色，契约 RegionMark）。 + + 行内怎么分段见 _row_spans。喂入不必全局按行有序：乱序的只是多出几个小矩形，覆盖不受影响（区域外文字要等 + 统计期定下来才分类，喂得晚）。""" + + def __init__(self, sheet: str) -> None: + self.sheet = sheet + self.open: dict[tuple[str, int, int], list[int]] = {} + self.closed: list[tuple[int, int, int, int, str]] = [] + + def feed_row(self, r: int, cells: list[tuple[int, str]]) -> None: + """单独几格（区域外文字、统计期）：同一去向的相邻格合成一段。""" + spans: list[tuple[int, int, str]] = [] + for c, role in cells: + if spans and spans[-1][2] == role: + spans[-1] = (spans[-1][0], c, role) + else: + spans.append((c, c, role)) + self.feed_spans(r, spans) + + def feed_spans(self, r: int, spans: list[tuple[int, int, str]]) -> None: + for c1, c2, role in spans: + key = (role, c1, c2) + rect = self.open.get(key) + if rect is not None and rect[1] == r - 1: + rect[1] = r + continue + if rect is not None: + self.closed.append((rect[0], c1, rect[1], c2, role)) + self.open[key] = [r, r] + + def marks(self) -> list[RegionMark]: + rects = list(self.closed) + [(r1, c1, r2, c2, role) for (role, c1, c2), (r1, r2) in self.open.items()] + rects.sort(key=lambda x: (x[0], x[1], x[4])) + out = [] + for r1, c1, r2, c2, role in rects: + ref = cell_ref(r1, c1) if (r1, c1) == (r2, c2) else f"{cell_ref(r1, c1)}:{cell_ref(r2, c2)}" + out.append(RegionMark(self.sheet, role, ref)) # type: ignore[arg-type] + return out + + +class _Reconciler: + """两遍对账：读取层逐行数到的非空格数，和扫描层的 row_runs 逐行比（行号升序喂入）。""" + + def __init__(self, sheet: SheetScan) -> None: + self.scan = sheet + self.total = 0 + self.bad: list[tuple[int, int, int]] = [] + self.by_rows = bool(sheet.row_runs) or sheet.nonempty == 0 + self._it = self._expected() + self._next = next(self._it, None) + + def _expected(self) -> Iterator[tuple[int, int]]: + for lo, hi, n in self.scan.row_runs: + for r in range(lo, hi + 1): + yield r, n + + def feed(self, r: int, n: int) -> None: + self.total += n + if not self.by_rows: + return + while self._next is not None and self._next[0] < r: + self.bad.append((self._next[0], self._next[1], 0)) + self._next = next(self._it, None) + expect = 0 + if self._next is not None and self._next[0] == r: + expect = self._next[1] + self._next = next(self._it, None) + if expect != n: + self.bad.append((r, expect, n)) + + def finish(self) -> None: + while self.by_rows and self._next is not None: + self.bad.append((self._next[0], self._next[1], 0)) + self._next = next(self._it, None) + + +class _MergeSweep: + """按行号递增查询「这一行有哪些合并区」：合并区按首行排序，活动集合随行推进（列表块逐行处理时用）。""" + + def __init__(self, merges: list[tuple[int, int, int, int]], c1: int, c2: int) -> None: + self.todo = sorted((m for m in merges if m[3] >= c1 and m[1] <= c2), key=lambda m: m[0]) + self.i = 0 + self.active: list[tuple[int, int, int, int]] = [] + + def at(self, r: int) -> list[tuple[int, int, int, int]]: + while self.i < len(self.todo) and self.todo[self.i][0] <= r: + self.active.append(self.todo[self.i]) + self.i += 1 + self.active = [m for m in self.active if m[2] >= r] + return self.active + + +# -------------------------------------------------------------------------- +# 问题 +# -------------------------------------------------------------------------- + + +@dataclass +class _Issue: + """按（code、位置）合并的一组同类格子问题：值的类型、区域外数字、没有去处的格……""" + + code: str + sheet: str | None + where: str + reason: str + tail: str + count: int = 0 + cells: list[str] = field(default_factory=list) + examples: list[str] = field(default_factory=list) + + +#: 合并类问题的说明：code → (原因（{n} 是个数）, 下一步)。下一步写配方面板上的「字段」和「选项」原文 +#: （frontend/src/lib/terms.ts 的 RECIPE_TEXT、RECIPE_CHOICE_LABEL），界面上改了标签这里要跟着改 +_ISSUE_TEXT: dict[str, tuple[str, str]] = { + "value_not_integer": ("{n} 个单元格不是整数", "。该列按整数导入：请检查原表，或在配方面板中把「类型」改为「小数」"), + "value_text_number": ("{n} 个单元格是文本写的数字", + "。如需按数字导入，请在配方面板的「文本写的数字」中选择「千分位写法按数字保存」"), + "value_blank": ("{n} 个单元格为空", "。如原表确实为空，请在配方面板的「空格」中选择「存为空值」"), + "value_error": ("{n} 个单元格是错误值", "。请在 Excel 中修正后重新上传"), + "value_not_number": ("{n} 个单元格不是数字", "。如果它表示无数据，请在配方面板中添加占位符"), + "value_not_date": ("{n} 个单元格不是日期", "。请检查原表，或在配方面板中把这一列的「类型」改为「文字」"), + "value_formula": ("{n} 个数据格是公式", + "。数据区的公式通常是没有识别出来的合计；如确为数据，请在配方面板的「公式」中选择「按保存值导入」"), + "formula_uncached": ("{n} 个公式格没有保存计算结果", "，文件未经计算保存。请在 Excel 中打开并保存后重新上传"), + "formula_full_calc": ("工作簿设置了打开时重新计算，{n} 个公式格的保存值可能只是占位", + "。请在 Excel 中打开并保存后重新上传"), + "pk_duplicate": ("主键有 {n} 处重复", "。同一主键只能出现一次，请检查原表（例如同一项写成了两种写法）"), + "cell_unclaimed": ("{n} 个单元格没有去处", "。请调整配方，或删除这些单元格"), + "outside_number": ("导入区域之外有 {n} 个数字", "。请删除这些单元格，或调整配方让它们进入导入区域"), + "merged_in_values": ("数据区有 {n} 个合并单元格", "。请在 Excel 中取消合并后重新上传"), +} +_PK_BLANK = ("主键列有 {n} 个单元格为空", "。主键不能为空，请检查原表") +_PK_PLACEHOLDER = ("主键列有 {n} 个单元格是占位符", "。占位符按空值存，主键不能为空，请检查原表") +#: 交叉表、合计另存表的主键列是空值（占位符、blank=null 的空格、没有保存值的合计格、跳过的隐藏行） +_PK_NULL = ("有 {n} 处为空值", "。主键不能为空：请检查原表，或在配方面板的「主键」中去掉这一列") + + +class _Problems: + def __init__(self) -> None: + self.items: list[tuple[str | None, Problem]] = [] + self.issues: dict[tuple[str, str | None, str], _Issue] = {} + self._per: Counter[tuple[str, str | None]] = Counter() + self._over: Counter[tuple[str, str | None]] = Counter() + + def add(self, code: str, message: str, model_message: str, cells: list[str] | tuple[str, ...] = (), *, + sheet: str | None = None) -> None: + key = (code, sheet) + self._per[key] += 1 + if self._per[key] > PER_CODE_MAX: + self._over[key] += 1 + return + self.items.append((sheet, Problem(code=code, category=PROBLEM_CODES[code], message=message, # type: ignore[arg-type] + cells=list(cells)[:CELLS_MAX], model_message=model_message, + fix=PROBLEM_FIX.get(code)))) + + def issue(self, code: str, sheet: str | None, where: str, cell: str | list[str], example: str | None = None, *, + text: tuple[str, str] | None = None, group: str = "") -> None: + """记一处同类问题；cell 可以是几个坐标（主键重复要列出两处）。""" + key = (code, sheet, where + "\x00" + group) + it = self.issues.get(key) + if it is None: + reason, tail = text or _ISSUE_TEXT[code] + it = self.issues[key] = _Issue(code, sheet, where, reason, tail) + it.count += 1 + for one in ([cell] if isinstance(cell, str) else cell): + if len(it.cells) < CELLS_MAX and one not in it.cells: + it.cells.append(one) + if example is not None and len(it.examples) < 3: + it.examples.append(example) + + def finish(self, failed_sheets: set[str]) -> list[Problem]: + for it in self.issues.values(): + reason = it.reason.format(n=it.count) + shown = [c.split("!")[-1] for c in it.cells] + ex = f"（如 {'；'.join(it.examples)}）" if it.examples else f"（{_join(shown, 5)}）" + model_ex = f"（{_join(shown, 5)}）" + self.add(it.code, f"{it.where}：{reason}{ex}{it.tail}", f"{it.where}：{reason}{model_ex}", it.cells, + sheet=it.sheet) + for (code, sheet), n in self._over.items(): + where = f"工作表「{sheet}」" if sheet else "本文件" + text = f"{where}另有 {n} 处同类问题未逐一列出" + self.items.append((sheet, Problem(code=code, category=PROBLEM_CODES[code], message=text, # type: ignore[arg-type] + model_message=text, fix=PROBLEM_FIX.get(code)))) + return [p for sheet, p in self.items if not (sheet in failed_sheets and p.code in _NOISE_CODES)] + + def blocking(self) -> bool: + return any(p.category in ("structure", "input", "recipe") for _, p in self.items) or any( + PROBLEM_CODES[i.code] in ("structure", "input", "recipe") for i in self.issues.values()) + + +# -------------------------------------------------------------------------- +# 表的写入口 +# -------------------------------------------------------------------------- + + +class _Sink: + """一张表的行：主键去重、计数、溯源、批量写库（有库时）。rowid 就是插入序号（新库、1 起）。""" + + def __init__(self, eng: _Engine, name: str, cols: list[ColumnOut], spec: TableSpec | None) -> None: + self.eng = eng + self.name = name + self.cols = cols + names = [c.name for c in cols] + self.grain = [g for g in (spec.grain if spec else []) if g in names] + self.grain_idx = [names.index(g) for g in self.grain] + self.check_pk = bool(self.grain_idx) and not eng.partial + self.seen: dict[tuple[Any, ...], tuple[int, int]] = {} + self.rows = 0 + self.lineage: dict[str, _Runs] = {} + self.buffer: list[tuple[Any, ...]] = [] + self.sheet: str | None = None + self.sources: list[str] = [] + self.insert_sql = f"INSERT INTO {_q(name)} VALUES ({', '.join('?' * len(cols))})" + + def add(self, values: list[Any], *, sheet: str, at: tuple[int, int], + cells: list[tuple[str, int, int]] | None = None, fails: bool = False) -> int | None: + """加一行；主键重复或主键为空时不加（问题已记），返回 None。at：代表这一行的格（报主键重复用）； + cells：[(列名, 行, 列)] 用于溯源。 + + fails=True 时 values 里可以有 _FAIL：转换时已经报过问题的格（交叉表的值、合计格），按空值存。 + 主键是 _FAIL 的行不再另报；主键是空值（占位符、blank=null 的空格、没有保存值的合计格、排除的隐藏行） + 一律报 value_blank：不加这一行又不出声，库里就静默少一行，N 核对（COUNT(*) = expected_rows）照样通过（H2）。 + 列表的失败行不进这里（转换失败就不加），主键列的空格、占位符在转换时已经拒收。""" + if self.grain_idx: + key = tuple(values[i] for i in self.grain_idx) + if any(k is None or k is _FAIL for k in key): + if not any(k is _FAIL for k in key): + self._pk_null(values, sheet, at, cells) + return None + if fails: + values = [None if v is _FAIL else v for v in values] + if self.grain_idx: + if self.check_pk: + first = self.seen.get(key) + if first is not None: + shown = "、".join(str(k) for k in key) + self.eng.problems.issue( + "pk_duplicate", sheet, f"表「{self.name}」（主键：{'、'.join(self.grain)}）", + [_coord(sheet, *first), _coord(sheet, *at)], + f"「{shown}」出现在 {cell_ref(*first)} 和 {cell_ref(*at)}") + return None + self.seen[key] = at + if self.sheet is None: + self.sheet = sheet + self.rows += 1 + rowid = self.rows + if cells: + for col, r, c in cells: + runs = self.lineage.get(col) + if runs is None: + runs = self.lineage[col] = _Runs() + runs.add(rowid, sheet, r, c) + if self.eng.conn is not None: + self.buffer.append(tuple(values)) + if len(self.buffer) >= WRITE_BATCH: + self.flush() + return rowid + + def _pk_null(self, values: list[Any], sheet: str, at: tuple[int, int], + cells: list[tuple[str, int, int]] | None) -> None: + """主键列是空值：按主键列分组记 value_blank，坐标取溯源里那一列的格（没有就用代表格）。""" + names = [self.grain[j] for j, i in enumerate(self.grain_idx) if values[i] is None] + where_of = {col: (r, c) for col, r, c in (cells or [])} + coords = [_coord(sheet, *where_of.get(n, at)) for n in names] + self.eng.problems.issue("value_blank", sheet, f"工作表「{sheet}」的表「{self.name}」的主键列{_quoted(names)}", + coords, text=_PK_NULL, group="pk_null") + + def flush(self) -> None: + if self.buffer and self.eng.conn is not None: + self.eng.conn.executemany(self.insert_sql, self.buffer) + self.buffer.clear() + + +# -------------------------------------------------------------------------- +# 值的转换（4.5） +# -------------------------------------------------------------------------- + +_FAIL = object() +#: 列表数据区里不能填充的合并单元格覆盖的空格（已经报了 merged_in_values） +_MERGED = object() + + +# -------------------------------------------------------------------------- +# 工作表上下文 +# -------------------------------------------------------------------------- + + +@dataclass +class _Stopped: + """被空行停下的列表块：往下看紧接着的行有没有像数据的文字行（rows_after_stop，4.4）。""" + + block: str + stop_row: int + c1: int + c2: int + #: 本块列范围内几格非空就算「像数据的行」：规格写的是 2 格（挡住单独一格的落款、页码），只有 1 列的块 + #: 按 1 格算——只有一列的名单在空行之后被截掉，正是这条检查要让人看见的情形 + need: int = 2 + closed: bool = False + rows: list[int] = field(default_factory=list) + cells: list[str] = field(default_factory=list) + + +class _Sheet: + def __init__(self, eng: _Engine, si: int, sr: SheetRecipe, scan: SheetScan) -> None: + self.eng = eng + self.si = si + self.sr = sr + self.scan = scan + self.name = scan.name + self.context: ContextSpec | None = sr.context[0] if sr.context else None + self.grid: Grid | None = None + self.limit = scan.bounds.max_row if scan.bounds else 0 + self.truncated = False + self.claims: dict[tuple[int, int], str] = {} + self.owners: dict[tuple[int, int], str] = {} + self.overlaps: set[tuple[str, str]] = set() + self.data_rows: set[int] = set() + self.suppressed: set[tuple[int, int]] = set() + self.failed = False + self.deferred: list[tuple[int, int, str]] = [] + self.roles: Counter[str] = Counter() + self.read_count = 0 + self.regions = _Regions(self.name) + self.recon: _Reconciler | None = None + self.stopped: list[_Stopped] = [] + #: 行 → 块登记的固定列段（_row_spans）：格子账收尾那一行时取走，流式路径里同时只有一两行 + self.layouts: dict[int, list[tuple[int, int, str]]] = {} + self.order: list[tuple[int, str]] = [] + self.hidden_claimed_rows: set[int] = set() + self.hidden_claimed_cols: set[int] = set() + self._hr, self._hr_starts = scan.hidden_rows, [a for a, _ in scan.hidden_rows] + self._hc, self._hc_starts = scan.hidden_cols, [a for a, _ in scan.hidden_cols] + self.merges: list[tuple[int, int, int, int]] = [] + for ref in scan.merged: + span = xlsx_cells.parse_range(ref) + if span is not None: + self.merges.append(span) + + def hidden_row(self, r: int) -> bool: + return bool(self._hr) and _in_spans(self._hr, self._hr_starts, r) + + def hidden_col(self, c: int) -> bool: + return bool(self._hc) and _in_spans(self._hc, self._hc_starts, c) + + def coord(self, r: int, c: int) -> str: + return _coord(self.name, r, c) + + def layout(self, r: int, spans: list[tuple[int, int, str]]) -> None: + """登记这一行的固定列段（区域标记按它出矩形，见 _row_spans）。""" + got = self.layouts.get(r) + if got is None: + self.layouts[r] = list(spans) + else: + got.extend(spans) + + def claim(self, r: int, c: int, role: str, owner: str) -> bool: + """认领一格（只认领网格里的非空格）。被别的块认领过就记重叠、不改。""" + prev = self.owners.get((r, c)) + if prev is not None and prev != owner: + self.overlaps.add((min(prev, owner), max(prev, owner))) + return False + self.claims[(r, c)] = role + self.owners[(r, c)] = owner + return True + + +# -------------------------------------------------------------------------- +# 交叉表（4.3、4.5、4.6） +# -------------------------------------------------------------------------- + + +@dataclass +class _SegSpec: + seg: Any + role: str + #: 规范写法 → 配方里写的期望标签 + expect: dict[str, str] + parser: str + + def parsed(self, value: Any) -> Any: + if self.role == "measures": + key = match_key(_text(value)) + return key or None + if self.role == "derived": + return hour_range_total(value) + if self.parser == "hour_range": + return hour_range(value) + return text_label(_text(value)) + + def canonical(self, value: Any) -> str | None: + p = self.parsed(value) + if p is None: + return None + return p if isinstance(p, str) else p.canonical + + +def _seg_spec(seg: Any) -> _SegSpec: + if isinstance(seg, MeasuresSegment): + spec = _SegSpec(seg, "measures", {}, "measures") + elif isinstance(seg, DimensionSegment): + spec = _SegSpec(seg, "dimension", {}, seg.dim.parser) + else: + spec = _SegSpec(seg, "derived", {}, "hour_range_total") + for label in seg.labels.expect: + key = spec.canonical(label) + if key is not None: + spec.expect.setdefault(key, label) + return spec + + +@dataclass +class _SegLoc: + spec: _SegSpec + rows: list[int] + title_row: int | None = None + #: 行 → 解析结果（HourRange / HourTotal / 规范写法）；解析不了的行不在这里 + parsed: dict[int, Any] = field(default_factory=dict) + #: 规范写法 → 第一次出现的行 + first: dict[str, int] = field(default_factory=dict) + #: 不进表的行（标签不在期望中的指标行、解析不了的行） + skip: set[int] = field(default_factory=set) + + +class _Crosstab: + def __init__(self, eng: _Engine, sh: _Sheet, block: CrosstabBlock, bi: int) -> None: + self.eng, self.sh, self.block, self.bi = eng, sh, block, bi + self.grid: Grid = sh.grid # type: ignore[assignment] + self.where = f"工作表「{sh.name}」的交叉表「{block.id}」" + self.located = False + self.axis_row = 0 + self.label_c = 0 + self.first_c = self.last_c = 0 + self.axis_cells: list[tuple[int, AxisDate]] = [] + self.kinds: dict[int, str] = {} + self.specs = [_seg_spec(s) for s in block.segments] + self.locs: dict[str, _SegLoc] = {} + self.failed: set[str] = set() + self.row_owner: dict[int, str] = {} + self.extra_cols: set[int] = set() + self.dates: dict[int, _dt.date] | None = None + self.excluded: set[tuple[int, int]] = set() + self.cell_index: dict[tuple[int, int], tuple[str, int]] = {} + self.exclude_hidden = sh.sr.hidden.rows == "exclude" + + # ---------------- 定位 ---------------- + + def _seg_where(self, seg_id: str) -> str: + return f"工作表「{self.sh.name}」的分段「{seg_id}」" + + def _axis_cols(self) -> range: + return range(self.first_c, self.last_c + 1) + + def locate(self) -> None: + grid, sh, block = self.grid, self.sh, self.block + need = block.axis.find.min + cands: list[tuple[int, list[tuple[int, AxisDate]], list[tuple[int, GridCell]]]] = [] + for r in grid.row_numbers(): + row = grid.row(r) + dates = [(c, d) for c, cell in row if (d := month_day_or_date(cell.value)) is not None] + if len(dates) < need or dates[0][0] < 2: + continue + label_c = dates[0][0] + block.label_offset + right = sum(1 for c, _ in row if c >= label_c) + if len(dates) * 2 > right: + cands.append((r, dates, row)) + if not cands: + msg = f"{self.where}：未找到日期表头行：没有一行有至少 {need} 格日期" + self.eng.problems.add("axis_not_found", msg, msg, sheet=sh.name) + sh.failed = True + return + if len(cands) > 1: + rows = [r for r, _, _ in cands] + msg = f"{self.where}：有多行像日期表头（第 {_rows_text(rows)} 行），无法确定用哪一行" + self.eng.problems.add("axis_ambiguous", msg, msg, [sh.coord(r, d[0][0]) for r, d, _ in cands], + sheet=sh.name) + sh.failed = True + return + r, dates, row = cands[0] + self.axis_row = r + self.first_c, self.last_c = dates[0][0], dates[-1][0] + self.label_c = self.first_c + block.label_offset + by_col = dict(dates) + gaps = [c for c in self._axis_cols() if c not in by_col] + if gaps: + cells = [sh.coord(r, c) for c in gaps] + shown = [f"{cell_ref(r, c)}「{_shown(grid.get(r, c).value)}」" if grid.get(r, c) else + f"{cell_ref(r, c)}（空）" for c in gaps] + self.eng.problems.add( + "axis_gap", f"{self.where}：日期表头（第 {r} 行）中间有不是日期的格：{_join(shown)}", + f"{self.where}：日期表头（第 {r} 行）中间有不是日期的格：{_join([cell_ref(r, c) for c in gaps])}", + cells, sheet=sh.name) + sh.failed = True + return + self.axis_cells = dates + extra = [c for c, _ in row if c > self.last_c] + if extra: + self.extra_cols = set(extra) + shown = [f"{cell_ref(r, c)}「{_shown(grid.get(r, c).value)}」" for c in extra] + self.eng.problems.add( + "axis_extra_cells", + f"{self.where}：日期表头（第 {r} 行）最后一个日期右边还有内容：{_join(shown)}。请删除这些列，或调整配方", + f"{self.where}：日期表头（第 {r} 行）最后一个日期右边还有内容：{_join([cell_ref(r, c) for c in extra])}", + [sh.coord(r, c) for c in extra], sheet=sh.name) + for (rr, cc) in grid.cells: + if rr >= r and cc in self.extra_cols: + sh.suppressed.add((rr, cc)) + self.located = True + if grid.get(r, self.label_c) is not None: + sh.claim(r, self.label_c, "col_header", block.id) + for c, _ in dates: + sh.claim(r, c, "col_header", block.id) + self._classify_rows() + self._locate_segments() + + def _label(self, r: int) -> GridCell | None: + return self.grid.get(r, self.label_c) + + def _classify_rows(self) -> None: + grid, lo, hi = self.grid, self.first_c, self.last_c + for r in grid.row_numbers(): + if r <= self.axis_row: + continue + label = self._label(r) + axis = any(lo <= c <= hi for c, _ in grid.row(r)) + if label is None and not axis: + continue + if label is None: + self.kinds[r] = "orphan" + elif axis: + self.kinds[r] = "data" + elif any((k := s.canonical(label.value)) is not None and k in s.expect for s in self.specs): + # 整期为空格的时段行：标签认得出、又在期望里，按数据行处理（否则会被当成分段标题） + self.kinds[r] = "data" + else: + self.kinds[r] = "title" + + def _run(self, start: int, stop_total: bool) -> list[int]: + rows = [] + r = start + while self.kinds.get(r) == "data": + label = self._label(r) + if stop_total and label is not None and hour_range_total(label.value) is not None: + break + rows.append(r) + r += 1 + return rows + + def _locate_segments(self) -> None: + pending = list(self.specs) + progress = True + while pending and progress: + progress = False + for spec in list(pending): + seg = spec.seg + if seg.locate.by == "after" and seg.locate.segment not in self.locs: + if seg.locate.segment in self.failed or not any(s.seg.id == seg.locate.segment for s in self.specs): + pending.remove(spec) + self.failed.add(seg.id) + progress = True + continue + pending.remove(spec) + progress = True + self._locate_one(spec) + for spec in pending: + self.failed.add(spec.seg.id) + + def _missing_ok(self) -> bool: + """部分干跑且网格读不全时，「没找到」类的问题可能只是还没读到，不报。""" + return self.eng.partial and self.sh.truncated + + def _locate_one(self, spec: _SegSpec) -> None: + seg, sh, eng = spec.seg, self.sh, self.eng + where = self._seg_where(seg.id) + stop_total = isinstance(seg, DimensionSegment) and seg.stop_parser == "hour_range_total" + title_row = None + if seg.locate.by == "section_title": + key = match_key(seg.locate.title or "") + hits = [r for r, k in sorted(self.kinds.items()) if k == "title" + and match_key(_text(self._label(r).value)) == key] # type: ignore[union-attr] + if not hits: + self.failed.add(seg.id) + if not self._missing_ok(): + msg = f"{where}：没有找到分段标题「{seg.locate.title}」" + eng.problems.add("title_not_found", msg, msg, sheet=sh.name) + return + if len(hits) > 1: + self.failed.add(seg.id) + cells = [sh.coord(r, self.label_c) for r in hits] + msg = f"{where}：分段标题「{seg.locate.title}」出现了不止一次（{_join([c.split('!')[-1] for c in cells])}）" + eng.problems.add("title_ambiguous", msg, msg, cells, sheet=sh.name) + return + title_row = hits[0] + rows = self._run(title_row + 1, stop_total) + elif seg.locate.by == "labels": + first = next((r for r, k in sorted(self.kinds.items()) if k == "data" + and (key := spec.canonical(self._label(r).value)) is not None # type: ignore[union-attr] + and key in spec.expect), None) + if first is None: + self.failed.add(seg.id) + if not self._missing_ok(): + msg = f"{where}：没有找到期望标签所在的行（期望：{_quoted(list(seg.labels.expect))}）" + eng.problems.add("segment_not_found", msg, msg, sheet=sh.name) + return + start = first + while self.kinds.get(start - 1) == "data": + start -= 1 + rows = self._run(start, stop_total) + if first not in rows: # 第一处命中本身就是停止行（不会发生在正常配方里） + rows = self._run(first, False) + else: + prev = self.locs[seg.locate.segment] + rows = self._run(prev.rows[-1] + 1, False) if prev.rows else [] + loc = _SegLoc(spec, rows, title_row) + # 每行只能属于一个分段 + clash = [r for r in rows if r in self.row_owner] + if clash: + self.failed.add(seg.id) + other = self.row_owner[clash[0]] + msg = f"{self.where}：分段「{other}」和「{seg.id}」认领了同一批行（第 {_rows_text(clash)} 行）" + eng.problems.add("segment_overlap", msg, msg, [sh.coord(r, self.label_c) for r in clash], sheet=sh.name) + return + for r in rows: + self.row_owner[r] = seg.id + self.locs[seg.id] = loc + self._check_labels(loc) + self._claim(loc) + if isinstance(seg, DimensionSegment) and seg.const and title_row is not None: + title_raw = _text(self._label(title_row).value) # type: ignore[union-attr] + for name in sorted(seg.const): + pick = seg.const[name].pick + if canon(pick) not in canon(title_raw): + ref = cell_ref(title_row, self.label_c) + eng.problems.add( + "const_missing", + f"{where}：常量「{name}」的取值「{pick}」不在分段标题「{_shown(title_raw, 40)}」中", + f"{where}：常量「{name}」的取值「{pick}」不在 {ref} 的分段标题中", + [sh.coord(title_row, self.label_c)], sheet=sh.name) + first_row = title_row if title_row is not None else (rows[0] if rows else None) + if first_row is not None: + sh.order.append((first_row, seg.id)) + + def _check_labels(self, loc: _SegLoc) -> None: + spec, sh, eng = loc.spec, self.sh, self.eng + seg = spec.seg + where = self._seg_where(seg.id) + raw, canonical = [], [] + unparsed, dup, unexpected = [], [], [] + for r in loc.rows: + value = self._label(r).value # type: ignore[union-attr] + raw.append(_text(value)) + p = spec.parsed(value) + if p is None: + canonical.append("") + unparsed.append(r) + loc.skip.add(r) + continue + key = p if isinstance(p, str) else p.canonical + canonical.append(key) + loc.parsed[r] = p + if key in loc.first: + dup.append(r) + if spec.role == "measures": + loc.skip.add(r) + continue + loc.first[key] = r + if key not in spec.expect: + unexpected.append(r) + if spec.role == "measures": + loc.skip.add(r) + eng.ex.labels.append(SegmentLabels(seg.id, raw, canonical, list(loc.rows))) + lc = self.label_c + if unparsed: + cells = [sh.coord(r, lc) for r in unparsed] + shown = [f"{cell_ref(r, lc)}「{_shown(self._label(r).value)}」" for r in unparsed] # type: ignore[union-attr] + name = _PARSER_NAME[spec.parser] + eng.problems.add("label_unparsed", f"{where}：{_join(shown)}的标签无法按{name}解析", + f"{where}：{_join([cell_ref(r, lc) for r in unparsed])} 的标签无法按{name}解析", + cells, sheet=sh.name) + if unexpected: + cells = [sh.coord(r, lc) for r in unexpected] + shown = [f"{cell_ref(r, lc)}「{_shown(self._label(r).value)}」" for r in unexpected] # type: ignore[union-attr] + eng.problems.add("label_unexpected", f"{where}：{_join(shown)}的标签不在期望的标签中", + f"{where}：{_join([cell_ref(r, lc) for r in unexpected])} 的标签不在期望的标签中", + cells, sheet=sh.name) + if dup: + cells = [sh.coord(r, lc) for r in dup] + pairs = [f"{cell_ref(loc.first[canonical[loc.rows.index(r)]], lc)} 与 {cell_ref(r, lc)}" for r in dup] + shown = [f"「{canonical[loc.rows.index(r)]}」（{p}）" for r, p in zip(dup, pairs)] + eng.problems.add("label_duplicate", f"{where}：标签重复：{_join(shown)}", + f"{where}：标签重复：{_join(pairs)}", cells, sheet=sh.name) + missing = [label for key, label in spec.expect.items() if key not in loc.first] + edge = self._missing_ok() and (not loc.rows or loc.rows[-1] >= self.sh.limit - 1) + if missing and not edge: + cells = [sh.coord(loc.rows[0], lc)] if loc.rows else ( + [sh.coord(loc.title_row, lc)] if loc.title_row else []) + msg = f"{where}：期望的标签{_quoted(missing)}没有找到" + eng.problems.add("label_missing", msg, msg, cells, sheet=sh.name) + + def _claim(self, loc: _SegLoc) -> None: + sh, grid, seg = self.sh, self.grid, loc.spec.seg + derived = loc.spec.role == "derived" + if loc.title_row is not None: + sh.claim(loc.title_row, self.label_c, "section_title", seg.id) + for r in loc.rows: + sh.data_rows.add(r) + excluded = self.exclude_hidden and sh.hidden_row(r) + label_role = "hidden_excluded" if excluded else ("derived_label" if derived else "row_label") + value_role = "hidden_excluded" if excluded else ("derived_value" if derived else "value") + # 区域按标签列和整段日期列出（某天是空格的行不另起矩形） + sh.layout(r, [(self.label_c, self.label_c, label_role), (self.first_c, self.last_c, value_role)]) + sh.claim(r, self.label_c, label_role, seg.id) + for c, _ in grid.row(r): + if self.first_c <= c <= self.last_c: + sh.claim(r, c, value_role, seg.id) + if excluded: + self.excluded.add((r, c)) + + def post_claims(self) -> None: + """所有块都认领完之后：没有分段认领的数据行、没有标签的行、数据区的合并单元格。""" + if not self.located: + return + sh, grid, eng = self.sh, self.grid, self.eng + lc = self.label_c + for r, kind in sorted(self.kinds.items()): + if kind == "title" or r in self.row_owner: + continue + row = grid.row(r) + if any((r, c) in sh.owners and sh.owners[(r, c)] != self.block.id for c, _ in row): + continue # 别的块（列表）认领了这一行 + for c, _ in row: + sh.suppressed.add((r, c)) + if kind == "data": + if self.failed: + continue # 有分段没定位到：这些行多半就是它的，不再逐行报 + coord = sh.coord(r, lc) + eng.problems.add( + "row_unclaimed", + f"工作表「{sh.name}」第 {r} 行（{cell_ref(r, lc)}「{_shown(self._label(r).value)}」）有数据，" # type: ignore[union-attr] + "但没有分段认领这一行。请在配方面板中把它的标签添加到某个分段，或删除这一行", + f"工作表「{sh.name}」第 {r} 行有数据，但没有分段认领这一行（标签在 {cell_ref(r, lc)}）", + [coord] + [sh.coord(r, c) for c, _ in row if c != lc], sheet=sh.name) + else: + msg = f"工作表「{sh.name}」第 {r} 行有数据，但标签列 {cell_ref(r, lc)} 是空的" + eng.problems.add("row_without_label", msg, msg, [sh.coord(r, c) for c, _ in row], sheet=sh.name) + data = {r for r, k in self.kinds.items() if k == "data"} + for r1, c1, r2, c2 in grid.merges: + if c2 < self.first_c or c1 > self.last_c: + continue + if any(r in data for r in range(max(r1, self.axis_row + 1), r2 + 1)): + ref = cell_ref(r1, c1) if (r1, c1) == (r2, c2) else f"{cell_ref(r1, c1)}:{cell_ref(r2, c2)}" + eng.problems.issue("merged_in_values", sh.name, self.where, sh.coord(r1, c1), ref) + + # ---------------- 年份与断言 ---------------- + + def complete_axis(self, period: Period | None, period_reported: bool) -> None: + if not self.located: + return + sh, eng, axis = self.sh, self.eng, self.block.axis + r = self.axis_row + resolved: dict[int, _dt.date] = {} + missing: list[tuple[int, str]] = [] + ambiguous: list[int] = [] + blocked = False + for c, d in self.axis_cells: + if d.year is not None: + try: + resolved[c] = _dt.date(d.year, d.month, d.day) + except ValueError: + missing.append((c, "（日期不合法）")) + continue + if axis.year_from is None: + missing.append((c, "（表头没有写年份，配方也没有指定年份来源）")) + continue + if period is None: + if period_reported: + blocked = True # 统计期待录入：已经报了 period_missing，这里不重复 + else: + missing.append((c, "（没有统计期，无法补全年份）")) + continue + hits = [] + for y in range(period.start.year, period.end.year + 1): + try: + day = _dt.date(y, d.month, d.day) + except ValueError: + continue + if period.start <= day <= period.end: + hits.append(day) + if len(hits) == 1: + resolved[c] = hits[0] + elif hits: + ambiguous.append(c) + else: + missing.append((c, "（不在统计期内）")) + grid = self.grid + if missing: + cells = [sh.coord(r, c) for c, _ in missing] + shown = [f"{cell_ref(r, c)}「{_shown(grid.get(r, c).value)}」{why}" for c, why in missing] # type: ignore[union-attr] + eng.problems.add("axis_year_missing", f"{self.where}：日期表头无法确定年份：{_join(shown)}", + f"{self.where}：日期表头无法确定年份：{_join([cell_ref(r, c) + why for c, why in missing])}", + cells, sheet=sh.name) + if ambiguous: + cells = [sh.coord(r, c) for c in ambiguous] + msg = f"{self.where}：统计期跨了不止一年，日期表头的年份有多种可能：{_join([cell_ref(r, c) for c in ambiguous])}" + eng.problems.add("axis_year_ambiguous", msg, msg, cells, sheet=sh.name) + form = {d.form for _, d in self.axis_cells} + first_txt = _shown(grid.get(r, self.first_c).value) # type: ignore[union-attr] + last_txt = _shown(grid.get(r, self.last_c).value) # type: ignore[union-attr] + if missing or ambiguous or blocked: + eng.ex.axes.append(AxisOut(self.block.id, r, first_txt, last_txt, len(self.axis_cells), + form.pop() if len(form) == 1 else "mixed")) + return + ordered = [(c, resolved[c]) for c, _ in self.axis_cells] + seen: dict[_dt.date, int] = {} + dups: list[tuple[int, int]] = [] + for c, day in ordered: + if day in seen: + dups.append((seen[day], c)) + else: + seen[day] = c + if dups: + cells = [sh.coord(r, c) for pair in dups for c in pair] + eng.problems.add( + "axis_duplicate", + f"{self.where}：日期表头有重复的日期：" + + _join([f"{resolved[a].isoformat()} 出现在 {cell_ref(r, a)} 和 {cell_ref(r, b)}" for a, b in dups]), + f"{self.where}：日期表头有重复的日期：" + _join([f"{cell_ref(r, a)} 和 {cell_ref(r, b)}" for a, b in dups]), + cells, sheet=sh.name) + # 有重复日期时「不连续」「没有恰好覆盖」都只是它的后果，不再另报 + if "contiguous" in axis.checks and not dups: + for (ca, da), (cb, db) in zip(ordered, ordered[1:]): + if db - da != _dt.timedelta(days=1): + eng.problems.add( + "axis_not_contiguous", + f"{self.where}：日期表头不是逐日连续的：{cell_ref(r, ca)}（{da.isoformat()}）之后是 " + f"{cell_ref(r, cb)}（{db.isoformat()}）", + f"{self.where}：日期表头不是逐日连续的：{cell_ref(r, ca)} 之后是 {cell_ref(r, cb)}", + [sh.coord(r, ca), sh.coord(r, cb)], sheet=sh.name) + break + if "covers_context" in axis.checks and period is not None and not eng.partial and not dups: + days = {period.start + _dt.timedelta(days=i) for i in range(period.days)} + if set(resolved.values()) != days or len(ordered) != len(days): + eng.problems.add( + "axis_coverage", + f"{self.where}：第 {r} 行的日期是 {first_txt} 至 {last_txt} 共 {len(ordered)} 格，" + f"统计期 {period.start.isoformat()} 至 {period.end.isoformat()} 共 {period.days} 天", + f"{self.where}：第 {r} 行的日期表头共 {len(ordered)} 格，没有恰好覆盖统计期（共 {period.days} 天）", + [sh.coord(r, self.first_c), sh.coord(r, self.last_c)], sheet=sh.name) + eng.ex.axes.append(AxisOut(self.block.id, r, ordered[0][1].isoformat(), ordered[-1][1].isoformat(), + len(ordered), form.pop() if len(form) == 1 else "mixed")) + if not dups: + self.dates = resolved + + # ---------------- 抽取 ---------------- + + def extract(self) -> None: + if not self.located: + return + eng, sh = self.eng, self.sh + order = sorted(self.locs.values(), key=lambda loc: (loc.title_row or (loc.rows[0] if loc.rows else 0))) + for loc in order: + if loc.spec.seg.id in self.failed: + continue + if loc.spec.role == "measures": + self._measures(loc) + elif loc.spec.role == "dimension": + self._dimension(loc) + for loc in order: + if loc.spec.role == "derived" and loc.spec.seg.id not in self.failed: + self._derived(loc) + + def _value(self, r: int, c: int, where: str) -> Any: + return self.eng.convert_number(self.grid.get(r, c), self.block.values.type, self.block.values, + self.sh.name, where, r, c, data_area=True) + + def _measures(self, loc: _SegLoc) -> None: + eng, sh, seg = self.eng, self.sh, loc.spec.seg + where = self._seg_where(seg.id) + sink = eng.sinks[seg.table] + sink.sources.append(f"{sh.sr.id}/{self.block.id}/{seg.id}") + rows_by_label: dict[str, int] = {} + for key, label in loc.spec.expect.items(): + r = loc.first.get(key) + if r is not None and r not in loc.skip: + rows_by_label[label] = r + eng.headers[(seg.table, seg.measures.get(label, ""))] = _text(self._label(r).value) # type: ignore[union-attr] + # 值保留 _FAIL（已报过问题），交给 _Sink 按空值存；没找到的标签（已报 label_missing）也按 _FAIL + values: dict[tuple[str, int], Any] = {} + for label, r in rows_by_label.items(): + excluded = self.exclude_hidden and sh.hidden_row(r) + for c in self._axis_cols(): + values[(label, c)] = None if excluded else self._value(r, c, where) + if self.dates is None: + return + for c in self._axis_cols(): + row: list[Any] = [self.dates[c].isoformat()] + cells: list[tuple[str, int, int]] = [] + for label in seg.labels.expect: + row.append(values.get((label, c), _FAIL)) + r = rows_by_label.get(label) + if r is not None: + cells.append((seg.measures.get(label, ""), r, c)) + sink.add(row, sheet=sh.name, at=(self.axis_row, c), cells=cells, fails=True) + + def _derive_values(self, derive: dict[str, str], hr: Any) -> list[Any]: + ordered = sorted(derive.items(), key=lambda kv: (kv[1] != "start", kv[0])) + if not isinstance(hr, (HourRange, HourTotal)): + return [None for _ in ordered] + return [hr.start if role == "start" else hr.end for _, role in ordered] + + def _dimension(self, loc: _SegLoc) -> None: + eng, sh, seg = self.eng, self.sh, loc.spec.seg + where = self._seg_where(seg.id) + sink = eng.sinks[seg.table] + sink.sources.append(f"{sh.sr.id}/{self.block.id}/{seg.id}") + consts = [seg.const[name].pick for name in sorted(seg.const)] + for r in loc.rows: + if r in loc.skip or r not in loc.parsed: + continue + excluded = self.exclude_hidden and sh.hidden_row(r) + if excluded: + continue + p = loc.parsed[r] + dim = p if isinstance(p, str) else p.canonical + derives = self._derive_values(seg.dim.derive, p) + for c in self._axis_cols(): + v = self._value(r, c, where) + if self.dates is None: + continue + row = [self.dates[c].isoformat(), dim, *consts, *derives, v] + rowid = sink.add(row, sheet=sh.name, at=(r, c), cells=[(seg.value, r, c)], fails=True) + if rowid is not None: + self.cell_index[(r, c)] = (seg.table, rowid) + + def _derived(self, loc: _SegLoc) -> None: + eng, sh, seg, grid = self.eng, self.sh, loc.spec.seg, self.grid + where = self._seg_where(seg.id) + prev = self.locs.get(seg.locate.segment or "") + prev_seg = prev.spec.seg if prev is not None else None + base = seg.verify.against_table + start_col = end_col = None + writers = [s.seg for s in self.specs if isinstance(s.seg, DimensionSegment) and s.seg.table == base] + if isinstance(prev_seg, DimensionSegment) and prev_seg.table == base: + writers.insert(0, prev_seg) + for w in writers: + for name, role in w.dim.derive.items(): + if role == "start" and start_col is None: + start_col = name + if role == "end" and end_col is None: + end_col = name + consts = {name: c.pick for name, c in sorted(prev_seg.const.items())} if isinstance( + prev_seg, DimensionSegment) else {} + keep = seg.keep_as + sink = eng.sinks.get(keep.table) if keep is not None else None + if sink is not None: + sink.sources.append(f"{sh.sr.id}/{self.block.id}/{seg.id}") + forms: set[str] = set() + for r in loc.rows: + for c in self._axis_cols(): + cell = grid.get(r, c) + if cell is not None: + forms.add("formula" if cell.formula is not None else "literal") + eng.ex.derived_form[seg.id] = forms.pop() if len(forms) == 1 else ("mixed" if forms else "literal") + if self.dates is None: + for r in loc.rows: # 类型问题照样报 + for c in self._axis_cols(): + eng.convert_number(grid.get(r, c), self.block.values.type, self.block.values, sh.name, where, + r, c, data_area=False) + return + for r in loc.rows: + total = loc.parsed.get(r) + if not isinstance(total, HourTotal) or r in loc.skip: + continue + if self.exclude_hidden and sh.hidden_row(r): + continue # 排除的隐藏行：合计行本身也不核对、不另存 + label_raw = _text(self._label(r).value) # type: ignore[union-attr] + hidden_in_range = prev is not None and any( + sh.hidden_row(pr) for pr, hr in prev.parsed.items() + if isinstance(hr, HourRange) and hr.start >= total.start and hr.end <= total.end) + derives = self._derive_values(keep.derive, total) if keep is not None else [] + for c in self._axis_cols(): + cell = grid.get(r, c) + coord = sh.coord(r, c) + got = eng.convert_number(cell, self.block.values.type, self.block.values, sh.name, where, r, c, + data_area=False) + value = None if got is _FAIL else got + item = DerivedItem( + kind="label_range_sum", segment=seg.id, sheet=sh.name, cell=coord, label_raw=label_raw, + label=total.canonical, value=value, is_formula=cell is not None and cell.formula is not None, + formula=cell.formula if cell is not None else None, base_table=base, + base_value=seg.verify.value, start=total.start, end=total.end, + key={self.block.axis.name: self.dates[c].isoformat(), **consts}, + start_col=start_col, end_col=end_col, hidden_in_range=hidden_in_range, + keep_table=keep.table if keep is not None else None) + if item.is_formula: + eng.resolve_refs(item, item.formula or "", lambda rc: self._ref_lookup(rc, base)) + eng.ex.derived.append(item) + if sink is not None and keep is not None: + sink.add([self.dates[c].isoformat(), total.canonical, *derives, got], sheet=sh.name, + at=(r, c), cells=[(keep.value, r, c)], fails=True) + + def _ref_lookup(self, rc: tuple[int, int], base: str) -> str | int: + hit = self.cell_index.get(rc) + if hit is not None and hit[0] == base: + return hit[1] + if rc in self.excluded: + return "hidden" + return "outside" + + +# -------------------------------------------------------------------------- +# 列表（4.4、4.5） +# -------------------------------------------------------------------------- + +Emit = Callable[[int, list[tuple[int, GridCell]], dict[int, str], bool], None] + + +class _ListRun: + """一个列表块在一张工作表上的一遍执行。逐行喂入（行号递增、只喂有非空格的行），网格和流式共用。 + + 每一行最后都经 emit 交出去（行号、格、本块的认领、是不是本块的数据行）：网格路径把认领记进工作表， + 流式路径当场做格子账。找表头要先看完窗口里的行，所以窗口内的行先缓存，定下表头再按顺序交出。""" + + def __init__(self, eng: _Engine, sh: _Sheet, block: ListBlock, bi: int, emit: Emit) -> None: + self.eng, self.sh, self.block, self.bi, self.emit = eng, sh, block, bi, emit + self.where = f"工作表「{sh.name}」的列表「{block.id}」" + self.n = block.header_rows + self.cols = block.columns + self.title_key = match_key(block.after_title) if block.after_title else None + #: 同一工作表上别的列表块的 after_title：窗口里已经有候选表头之后再碰到它，窗口就到它为止（上下两张 + #: 表头相同的时候，窗口不越过下一张表的标题，「恰好一处」才判得出来）。还没有候选表头时碰到的别块标题 + #: 不截断：没写 after_title 的块从已用区域首行找起，首行往往正是上面那张表的标题 + self.stop_keys = {match_key(b.after_title) for b in sh.sr.blocks + if isinstance(b, ListBlock) and b.after_title and b.id != block.id} - {self.title_key} + self.state = "title" if self.title_key else "header" + self.title_hits: list[str] = [] + self.window_start: int | None = None + self.buffer: list[tuple[int, list[tuple[int, GridCell]]]] = [] + self.located = False + self.header_top = 0 + self.c1 = self.c2 = 0 + self.colmap: dict[int, int] = {} + self.ignored: set[int] = set() + self.headers_read: dict[int, str] = {} + self.last_row = 0 + self.done = False + self.stop: tuple[str, int | None] | None = None + self.row_rowid: dict[int, int] = {} + self.rowid_first: int | None = None + self.rowid_last: int | None = None + #: 表头之下读到的数据行数（不管写没写进库）：一行都没有时报 list_empty（AU-1） + self.data_rows = 0 + self.hidden_seen = False + self.excluded_rows: set[int] = set() + self.merges: _MergeSweep | None = None + self.merge_judged: dict[tuple[int, int, int, int], bool] = {} + self.merge_bad: list[tuple[int, int, int, int]] = [] + self.topleft: dict[tuple[int, int, int, int], GridCell | None] = {} + spec = eng.specs.get(block.table) + self.grain = set(spec.grain) if spec else set() + self.sink = eng.sinks[block.table] + total = block.rows.total_row + self.total_idx = next((i for i, c in enumerate(self.cols) if total and c.name == total.label_column), None) + + # ---------------- 喂入 ---------------- + + def feed(self, r: int, cells: list[tuple[int, GridCell]]) -> None: + if self.title_key is not None: + # 标题是文字格：只对不太长的文本算 match_key（大列表逐格规范化要好几秒） + hit = next((c for c, cell in cells if type(cell.value) is str and len(cell.value) <= _TITLE_SCAN_MAX + and match_key(cell.value) == self.title_key), None) + if hit is not None: + self.title_hits.append(self.sh.coord(r, hit)) + if self.state == "title": + self.state = "header" + self.window_start = r + 1 + self.emit(r, cells, {}, False) + return + if self.state == "title": + self.emit(r, cells, {}, False) + return + if self.state == "header": + if self.window_start is None: + self.window_start = r + in_window = r <= self.window_start + ANCHOR_WINDOW - 1 + self.n - 1 + cands = None + if in_window and self.stop_keys and self.buffer and any( + type(cell.value) is str and len(cell.value) <= _TITLE_SCAN_MAX + and match_key(cell.value) in self.stop_keys for _, cell in cells): + cands = self._candidates(self._window_top(r)) or None + if in_window and cands is None: + self.buffer.append((r, cells)) + return + self._decide(before=r if cands is not None else None, cands=cands) + if self.state == "data": + self._data_row(r, cells) + else: + self.emit(r, cells, {}, False) + + def finish(self) -> None: + if self.state == "title": + if not (self.eng.partial and self.sh.truncated): + msg = f"工作表「{self.sh.name}」中没有找到标题「{self.block.after_title}」（列表「{self.block.id}」只在这行之后找表头）" + self.eng.problems.add("title_not_found", msg, msg, sheet=self.sh.name) + self.sh.failed = True + elif self.state == "header": + self._decide() + if self.state == "data" and not self.done: + self.done = True + self.stop = ("end", None) + if self.located and self.data_rows == 0 and not (self.eng.partial and self.sh.truncated): + # 只有表头、没有数据行（导出出错，或本月确实没有记录）：表会被整个替换成 0 行。不拒收（没有记录也是 + # 合理的），但要人确认（AU-1）；没有数据行时关系核对判无法核对，也不能写「已核对」 + head = self.sh.coord(self.header_top, self.c1) + msg = (f"{self.where}的表头之下没有数据行：本期这张表将导入 0 行。" + "如果原表本期确实没有记录，确认后可以导入；否则请检查导出的文件") + model = f"{self.where}的表头（{head.split('!')[-1]}）之下没有数据行" + self.eng.problems.add("list_empty", msg, model, [head], sheet=self.sh.name) + self._check_total_found() + if self.title_key is not None and len(self.title_hits) > 1: + msg = (f"工作表「{self.sh.name}」中标题「{self.block.after_title}」出现了不止一次" + f"（{_join([c.split('!')[-1] for c in self.title_hits])}），列表「{self.block.id}」无法确定从哪里找表头") + self.eng.problems.add("title_ambiguous", msg, msg, self.title_hits, sheet=self.sh.name) + for m in self.merge_bad: + r1, c1, r2, c2 = m + ref = cell_ref(r1, c1) if (r1, c1) == (r2, c2) else f"{cell_ref(r1, c1)}:{cell_ref(r2, c2)}" + self.eng.problems.issue("merged_in_values", self.sh.name, self.where, self.sh.coord(r1, c1), ref) + if self.located: + for i, text in self.headers_read.items(): + self.eng.headers[(self.block.table, self.cols[i].name)] = text + + # ---------------- 表头 ---------------- + + def _compose(self, rows: dict[int, dict[int, GridCell]], t: int, + merges: list[tuple[int, int, int, int]]) -> tuple[dict[int, str], dict[int, str]]: + """以 t 为首行的表头：(列 → 表头文字, 列 → 表头原文)。表头文字去了各段首尾空白，比对、报错用； + 原文保留每段的原样（「地区　」「销量\\n」），填进 TableOut.columns[].header，差异卡的 header_writing + 才看得到这类写法变化（4.4、9.3）。判空、判重（与上一层相同）都按去了空白的文字。""" + n = self.n + cols: set[int] = set() + for hr in range(t, t + n): + cols.update(rows.get(hr, {})) + local: list[tuple[int, int, int, int]] = [] + if n > 1: + local = [m for m in merges if t <= m[0] <= t + n - 1] + for m in local: + if (m[0], m[1]) in ((hr, c) for hr in range(t, t + n) for c in rows.get(hr, {})): + cols.update(range(m[1], m[3] + 1)) + out: dict[int, str] = {} + raw: dict[int, str] = {} + for c in sorted(cols): + parts: list[str] = [] + raws: list[str] = [] + for hr in range(t, t + n): + cell = rows.get(hr, {}).get(c) + original = _text(cell.value) if cell is not None else "" + text = original.strip() + if not text and local: + m = next((m for m in local if m[0] <= hr <= m[2] and m[1] <= c <= m[3]), None) + if m is not None: + tl = rows.get(m[0], {}).get(m[1]) + original = _text(tl.value) if tl is not None else "" + text = original.strip() + if text and (not parts or canon(parts[-1]) != canon(text)): + parts.append(text) + raws.append(original) + if parts: + out[c] = "_".join(parts) + raw[c] = "_".join(raws) + return out, raw + + def _keys(self) -> dict[str, int]: + """配方表头的 match_key → 列序号（同名的取第一个）。""" + keys = self.__dict__.get("_keys_cache") + if keys is None: + keys = {} + for i, col in enumerate(self.cols): + keys.setdefault(match_key(col.header), i) + self.__dict__["_keys_cache"] = keys + return keys + + def _window_top(self, before: int | None) -> int: + """表头首行最晚可以在哪一行：窗口末尾；碰到别块标题时，表头必须整个在那一行之上。""" + last_top = (self.window_start or 0) + ANCHOR_WINDOW - 1 + if before is not None: + last_top = min(last_top, before - self.n) + return last_top + + def _candidates(self, last_top: int) -> list[tuple[int, int, list[int], dict[int, str], dict[int, str]]]: + """缓存的窗口里首行不晚于 last_top 的候选表头：(命中列数, 首行, 列段, 表头文字, 表头原文)。""" + rows = {r: dict(cells) for r, cells in self.buffer} + start = self.window_start or 0 + keys = self._keys() + need = len(self.cols) + merges = [m for m in self.sh.merges if m[0] <= last_top + self.n and m[2] >= start] if self.n > 1 else [] + cands: list[tuple[int, int, list[int], dict[int, str], dict[int, str]]] = [] + for t in sorted(rows): + if t > last_top: + break + composed, raw = self._compose(rows, t, merges) + run: list[int] = [] + for c in sorted(composed) + [None]: # type: ignore[list-item] + if c is not None and (not run or c == run[-1] + 1): + run.append(c) + continue + if run: + hit = {keys[k] for cc in run if (k := match_key(composed[cc])) in keys} + if len(hit) * 2 > need: + cands.append((len(hit), t, run, composed, raw)) + run = [c] if c is not None else [] + return cands + + def _decide(self, before: int | None = None, + cands: list[tuple[int, int, list[int], dict[int, str], dict[int, str]]] | None = None) -> None: + sh, eng = self.sh, self.eng + start = self.window_start or 0 + keys = self._keys() + need = len(self.cols) + last_top = self._window_top(before) + if cands is None: + cands = self._candidates(last_top) + if not cands: + if not (eng.partial and sh.truncated and not self.buffer): + # 窗口按实际读到的范围写：已用区域只有 10 行时不写「第 1–200 行」；窗口是空的（标题就在最后 + # 一行、工作表没有内容）时不拼出「第 5–4 行」 + hi = min(last_top, sh.limit) if sh.limit else last_top + span = (f"在第 {start} 行中" if hi == start else f"在第 {start}–{hi} 行中") if ( + self.window_start is not None and hi >= start) else "" + msg = f"{self.where}：{span}没有找到表头（配方的表头：{_quoted([c.header for c in self.cols])}）" + eng.problems.add("header_not_found", msg, msg, sheet=sh.name) + self._fail() + return + best = max(x[0] for x in cands) + tops = [x for x in cands if x[0] == best] + if len(tops) > 1: + msg = f"{self.where}：有多处都像表头（{_join([cell_ref(t, run[0]) for _, t, run, _, _ in tops])}），无法确定用哪一处" + eng.problems.add("header_ambiguous", msg, msg, [sh.coord(t, run[0]) for _, t, run, _, _ in tops], + sheet=sh.name) + self._fail() + return + _, t, run, composed, raw = tops[0] + self.header_top, self.c1, self.c2 = t, run[0], run[-1] + extra: list[int] = [] + dup: dict[int, list[int]] = {} + for c in run: + k = match_key(composed[c]) + i = keys.get(k) + if i is None: + extra.append(c) + elif i in self.colmap: + dup.setdefault(i, [self.colmap[i]]).append(c) + else: + self.colmap[i] = c + self.headers_read[i] = raw[c] + hb = t + self.n - 1 + bad = False + missing = [self.cols[i].header for i in range(need) if i not in self.colmap] + if missing: + msg = f"{self.where}：表头（第 {t} 行）中没有找到列{_quoted(missing)}" + eng.problems.add("column_missing", msg, msg, [sh.coord(t, self.c1)], sheet=sh.name) + bad = True + if dup: + for i, cs in dup.items(): + msg = f"{self.where}：表头「{self.cols[i].header}」出现了不止一次（{_join([cell_ref(hb, c) for c in cs])}）" + eng.problems.add("header_ambiguous", msg, msg, [sh.coord(hb, c) for c in cs], sheet=sh.name) + bad = True + if extra: + if self.block.extra_columns == "ignore": + self.ignored = set(extra) + eng.ex.ignored_columns[self.block.id] = [composed[c] for c in extra] + else: + shown = [f"「{composed[c]}」（{cell_ref(hb, c)}）" for c in extra] + eng.problems.add("column_extra", f"{self.where}：表头多出了列{_join(shown)}", + f"{self.where}：表头多出了 {_join([cell_ref(hb, c) for c in extra])} 这几列", + [sh.coord(hb, c) for c in extra], sheet=sh.name) + bad = True + if bad: + self._fail() + return + self.located = True + self.state = "data" + self.last_row = hb + # 数据行的固定列段：映射列是 value、忽略的列是 ignored_column，按列相邻合段（区域标记用，_row_spans） + layout: list[tuple[int, int, str]] = [] + for c in range(self.c1, self.c2 + 1): + role = "ignored_column" if c in self.ignored else "value" + if layout and layout[-1][2] == role: + layout[-1] = (layout[-1][0], c, role) + else: + layout.append((c, c, role)) + self.layout_data = layout + self.layout_hidden = [(self.c1, self.c2, "hidden_excluded")] + self.sink.sources.append(f"{sh.sr.id}/{self.block.id}") + self.merges = _MergeSweep(sh.merges, self.c1, self.c2) + sh.order.append((t, self.block.id)) + buffered, self.buffer = self.buffer, [] + for r, cells in buffered: + if r < t: + self.emit(r, cells, {}, False) + elif r <= hb: + claims = {c: ("ignored_column" if c in self.ignored else "col_header") + for c, _ in cells if self.c1 <= c <= self.c2} + self.emit(r, cells, claims, False) + else: + self._data_row(r, cells) + + def _fail(self) -> None: + self.state = "failed" + self.sh.failed = True + buffered, self.buffer = self.buffer, [] + for r, cells in buffered: + self.emit(r, cells, {}, False) + + def _check_total_found(self) -> None: + """配方里有合计行、本期列表里却没碰到：明确报出来（结构类）。不报的话这一期照常导入，核对里没有 T、 + 说明照旧写「原表的合计行未导入本表」，另存的合计表是空的、说明生成又找不到核对结果——都是误导。 + 部分干跑（只读了前几百行）时合计行可能还在后面，不报。""" + total = self.block.rows.total_row + if total is None or self.state != "data" or (self.stop and self.stop[0] == "total"): + return + if self.eng.partial and self.sh.truncated: + return + c = self.colmap.get(self.total_idx) if self.total_idx is not None else None + cells = [self.sh.coord(self.last_row, c)] if c is not None and self.last_row else [] + where = f"{self.where}：配方里有合计行（「{total.label_column}」列以「{total.pick}」开头的行）" + self.eng.problems.add( + "total_row_missing", + f"{where}，本期文件中没有找到。请检查文件是否缺了合计行；本期确实没有合计行时，" + "在配方面板的「合计行」中取消「有合计行」", + f"{where}，本期没有找到", cells, sheet=self.sh.name) + + # ---------------- 数据行 ---------------- + + def _stop_blank(self, row: int) -> None: + self.done = True + self.stop = ("blank", row) + self.sh.stopped.append(_Stopped(self.block.id, row, self.c1, self.c2, min(2, self.c2 - self.c1 + 1))) + + def _data_row(self, r: int, cells: list[tuple[int, GridCell]]) -> None: + if self.done: + self.emit(r, cells, {}, False) + return + block, eng = self.block, self.eng + gap = r - (self.last_row + 1) + if gap > 0: + if block.rows.blank_rows == "stop": + self._stop_blank(self.last_row + 1) + self.emit(r, cells, {}, False) + return + eng.ex.blank_rows_skipped += gap + self.last_row = r + inb = {c: cell for c, cell in cells if self.c1 <= c <= self.c2} + if inb and self.ignored and all(c in self.ignored for c in inb): + # 只有忽略的列有字（表下紧跟的「注：以上为初步统计」写在备注列）：这一行没有一格要导入，按空行处理， + # 格子交给区域外（文字进回执、数字照常报）。当成数据行会凭空写进一条全是空值的记录 + inb = {} + if not inb: + if block.rows.blank_rows == "stop": + self._stop_blank(r) + else: + eng.ex.blank_rows_skipped += 1 + self.emit(r, cells, {}, False) + return + fills = self._merges_at(r, inb) + if self.sh.hidden_row(r): + self.hidden_seen = True + if self.sh.sr.hidden.rows == "exclude": + self.excluded_rows.add(r) + self.sh.layout(r, self.layout_hidden) + self.emit(r, cells, {c: "hidden_excluded" for c in inb}, True) + return + if self.total_idx is not None: + label = inb.get(self.colmap.get(self.total_idx, -1)) + pick = match_key(block.rows.total_row.pick) # type: ignore[union-attr] + if label is not None and match_key(_text(label.value)).startswith(pick): + self._total_row(r, inb, cells, label) + return + self._record(r, inb, cells, fills) + + def _plan(self) -> list[tuple[Any, int, bool, str]]: + """每列的（列定义、所在列号、是不是主键、位置说明）：定下表头之后算一次，逐行复用。""" + plan = self.__dict__.get("_plan_cache") + if plan is None: + plan = [(col, self.colmap[i], col.name in self.grain, f"{self.where}的列「{col.name}」") + for i, col in enumerate(self.cols)] + self.__dict__["_plan_cache"] = plan + self._grain_c = next((c for col, c, pk, _ in plan if pk), self.c1) + return plan + + def _merges_at(self, r: int, inb: dict[int, GridCell]) -> dict[int, GridCell | None]: + """这一行碰到的合并区：不能填充的记问题（合并区里的空格不再另报为空）；fill 时给 TEXT 列返回要填的值 + （列号 → 左上格）。返回值里值为 _MERGED 的列是「已按合并单元格报过、按空值处理」。""" + if self.merges is None: + return {} + fills: dict[int, Any] = {} + first_data = self.header_top + self.n + for m in self.merges.at(r): + if m[0] == r: + self.topleft[m] = inb.get(m[1]) + ok = self.merge_judged.get(m) + if ok is None: + ok = self.block.merged_data == "fill" and m[0] >= first_data and all( + self.cols[i].type == "TEXT" for i, c in self.colmap.items() if m[1] <= c <= m[3]) + self.merge_judged[m] = ok + if not ok: + self.merge_bad.append(m) + if not ok: + for c in range(max(m[1], self.c1), min(m[3], self.c2) + 1): + if c not in inb: + fills[c] = _MERGED + continue + for c in range(max(m[1], self.c1), min(m[3], self.c2) + 1): + if (r, c) != (m[0], m[1]) and c not in inb: + fills[c] = self.topleft.get(m) + return fills + + def _record(self, r: int, inb: dict[int, GridCell], cells: list[tuple[int, GridCell]], + fills: dict[int, GridCell | None]) -> None: + eng, sh, block = self.eng, self.sh, self.block + convert, pol, name, table = eng.convert_list, block.values, sh.name, block.table + values: list[Any] = [] + lineage: list[tuple[str, int, int]] = [] + failed = False + for col, c, is_pk, where in self._plan(): + cell = inb.get(c) + if cell is None and c in fills: + if fills[c] is _MERGED: + values.append(None) # 合并单元格已经报过，这一格不再按空格另报 + lineage.append((col.name, r, c)) + failed = True + continue + cell = fills[c] + v = convert(cell, col, is_pk, pol, name, where, r, c, table) + if v is _FAIL: + failed = True + v = None + values.append(v) + lineage.append((col.name, r, c)) + ignored = self.ignored + claims = {c: ("ignored_column" if c in ignored else "value") for c in inb} + self.sh.layout(r, self.layout_data) + self.data_rows += 1 + if not failed: + rowid = self.sink.add(values, sheet=name, at=(r, self._grain_c), cells=lineage) + if rowid is not None: + self.row_rowid[r] = rowid + if self.rowid_first is None: + self.rowid_first = rowid + self.rowid_last = rowid + self.emit(r, cells, claims, True) + + def _total_row(self, r: int, inb: dict[int, GridCell], cells: list[tuple[int, GridCell]], + label: GridCell) -> None: + eng, sh, block = self.eng, self.sh, self.block + total = block.rows.total_row + assert total is not None + claims = {c: ("ignored_column" if c in self.ignored else "total_label") for c in inb} + label_raw = _text(label.value) + numeric = [self.colmap[i] for i, col in enumerate(self.cols) if col.type in ("INTEGER", "REAL")] + if numeric and not any(c in inb for c in numeric): + # 合计行在，数字格却全空：没有一列能核对，另存的合计表也只会多一行空值。明确拒收，不让核对和说明 + # 当成「有合计行」去写（说明会找不到核对结果，或者写出原表并没有的合计） + coord = sh.coord(r, self.colmap[self.total_idx]) # type: ignore[index] + where = f"{self.where}：合计行（{coord.split('!')[-1]}）的数字格全部为空" + eng.problems.add( + "total_row_blank", + f"{where}，无法按各列明细核对。请在文件中补上合计后重新上传；原表不再提供合计时，删掉这一行，" + "并在配方面板的「合计行」中取消「有合计行」", + f"{where}，无法核对", [coord], sheet=sh.name) + keep_values: list[Any] = [] + for i, col in enumerate(self.cols): + if col.type not in ("INTEGER", "REAL"): + continue + c = self.colmap[i] + cell = inb.get(c) + if cell is None: + keep_values.append(None) + continue + claims[c] = "total_value" + coord = sh.coord(r, c) + got = eng.convert_number(cell, col.type, block.values, sh.name, f"{self.where}的合计行", r, c, + data_area=False) + value = None if got is _FAIL else got + keep_values.append(got) + item = DerivedItem( + kind="column_sum", segment=block.id, sheet=sh.name, cell=coord, label_raw=label_raw, + label=canon(label_raw), value=value, is_formula=cell.formula is not None, formula=cell.formula, + base_table=block.table, base_value=col.name, rowid_first=self.rowid_first, + rowid_last=self.rowid_last, hidden_in_range=self.hidden_seen, keep_table=total.keep_as) + if item.is_formula: + eng.resolve_refs(item, cell.formula or "", self._ref_lookup) + eng.ex.derived.append(item) + if total.keep_as and total.keep_as in eng.sinks: + ks = eng.sinks[total.keep_as] + ks.sources.append(f"{sh.sr.id}/{block.id}") + lin = [(col.name, r, self.colmap[i]) for i, col in enumerate(self.cols) if col.type in ("INTEGER", "REAL")] + ks.add([canon(label_raw), *keep_values], sheet=sh.name, at=(r, self.colmap[self.total_idx]), # type: ignore[index] + cells=lin, fails=True) + self.done = True + self.stop = ("total", r) + self.emit(r, cells, claims, True) + + def _ref_lookup(self, rc: tuple[int, int]) -> str | int: + r, c = rc + if self.c1 <= c <= self.c2: + rowid = self.row_rowid.get(r) + if rowid is not None: + return rowid + if r in self.excluded_rows: + return "hidden" + return "outside" + + +# -------------------------------------------------------------------------- +# 执行 +# -------------------------------------------------------------------------- + + +class _Engine: + def __init__(self, recipe: Recipe, raw: bytes, filename: str, scan: WorkbookScan, db_path: str | None, + context_inputs: dict[str, PeriodInput] | None, max_rows: int | None) -> None: + self.recipe = recipe + self.raw = raw + self.filename = filename + self.scan = scan + self.db_path = db_path + self.context_inputs = context_inputs or {} + self.max_rows = max_rows + self.partial = max_rows is not None + self.ex = Extraction(ok=False, partial=self.partial, full_calc_on_load=scan.full_calc_on_load) + self.problems = _Problems() + self.conn: sqlite3.Connection | None = None + self.specs: dict[str, TableSpec] = {t.name: t for t in recipe.tables} + self.cols: dict[str, list[ColumnOut]] = {} + self.sinks: dict[str, _Sink] = {} + self.headers: dict[tuple[str, str], str] = {} + self.canon_log: dict[tuple[str, str, str, str], CanonEntry] = {} + #: 每个块自己的占位符（canon 后 → 配方里的原文）：别的块声明的占位符不能在这里放过 + self._ph: dict[int, dict[str, str]] = {} + self.books = xlsx_cells.Workbooks(raw) + self.sheets: list[_Sheet] = [] + + def _placeholders(self, pol: Any) -> dict[str, str]: + hit = self._ph.get(id(pol)) + if hit is None: + hit = self._ph[id(pol)] = {canon(p.text): p.text for p in pol.placeholders} + return hit + + # ---------------- 值的转换 ---------------- + + def convert_number(self, cell: GridCell | None, vtype: str, pol: Any, sheet: str, where: str, r: int, c: int, *, + data_area: bool, pk: bool = False) -> Any: + """数值格 → 入库值；有问题记问题、返回 _FAIL。data_area=False 是合计格：空格、无缓存的公式都只是 None， + 公式不受「数据区不许公式」约束（合计本来就该是公式，由核对模块按标签区间重算）。 + 坐标只在出问题时才拼（20 万行的列表每格都拼一次字符串，光这一项就要好几秒）。""" + if cell is None: + if not data_area: + return None + if pk: + self.problems.issue("value_blank", sheet, where, _coord(sheet, r, c), text=_PK_BLANK, group="pk") + return _FAIL + if pol.blank == "null": + return None + self.problems.issue("value_blank", sheet, where, _coord(sheet, r, c)) + return _FAIL + v = cell.value + if cell.formula is not None: + if data_area: + v = self._accept_formula(cell, pol, sheet, where, r, c) + if v is _FAIL: + return _FAIL + elif v is None: + return None + kind = type(v) + if kind is int: + return v if vtype == "INTEGER" else float(v) + if kind is float and (vtype == "REAL" or v.is_integer()): + return v if vtype == "REAL" else int(v) + return self._number(v, vtype, pol, sheet, where, r, c, pk) + + def _accept_formula(self, cell: GridCell, pol: Any, sheet: str, where: str, r: int, c: int) -> Any: + """数据区的公式格：拒收、无缓存、打开时重算都是问题；否则按保存值导入并计数。""" + p = self.problems + if pol.formula == "reject": + p.issue("value_formula", sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)} 的 {cell.formula[:40]}") # type: ignore[index] + return _FAIL + if cell.value is None: + p.issue("formula_uncached", sheet, where, _coord(sheet, r, c)) + return _FAIL + if self.scan.full_calc_on_load: + p.issue("formula_full_calc", sheet, where, _coord(sheet, r, c)) + return _FAIL + self.ex.formula_cells_accepted += 1 + return cell.value + + def _number(self, v: Any, vtype: str, pol: Any, sheet: str, where: str, r: int, c: int, pk: bool = False) -> Any: + """非原生数字的值（文本、bool、日期……）。pk：主键列，占位符和空白文本都会变成空值，一律拒收。""" + p = self.problems + + def bad(code: str) -> Any: + p.issue(code, sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」") + return _FAIL + + if isinstance(v, bool): + return bad("value_not_number") + if isinstance(v, int): + return v if vtype == "INTEGER" else float(v) + if isinstance(v, float): + if vtype == "REAL": + return v + return int(v) if v.is_integer() else bad("value_not_integer") + if isinstance(v, str): + t = v.strip() + key = canon(t) + ph = self._placeholders(pol) + if key in ph: + if pk: + p.issue("value_blank", sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」", + text=_PK_PLACEHOLDER, group="pk_placeholder") + return _FAIL + name = ph[key] + self.ex.placeholders[name] = self.ex.placeholders.get(name, 0) + 1 + return None + if not t: + if pk: + p.issue("value_blank", sheet, where, _coord(sheet, r, c), text=_PK_BLANK, group="pk") + return _FAIL + if pol.blank == "null": + return None + p.issue("value_blank", sheet, where, _coord(sheet, r, c)) + return _FAIL + if t in tabular.EXCEL_ERRORS: + return bad("value_error") + num = thousands_number(t) + if num is not None: + if pol.text_number != "parse_thousands": + return bad("value_text_number") + return self._number(num, vtype, pol, sheet, where, r, c, pk) + return bad("value_not_number") + + def convert_list(self, cell: GridCell | None, col: Any, is_pk: bool, pol: Any, sheet: str, where: str, + r: int, c: int, table: str) -> Any: + ctype = col.type + if cell is not None and cell.formula is not None: + v = self._accept_formula(cell, pol, sheet, where, r, c) + if v is _FAIL: + return _FAIL + cell = GridCell(v) + if ctype == "INTEGER" or ctype == "REAL": + return self.convert_number(cell, ctype, pol, sheet, where, r, c, data_area=True, pk=is_pk) + v = cell.value if cell is not None else None + if v is None or (type(v) is str and not v.strip()): + if is_pk: + self.problems.issue("value_blank", sheet, where, _coord(sheet, r, c), text=_PK_BLANK, group="pk") + return _FAIL + if ctype == "DATE" and pol.blank == "reject": + self.problems.issue("value_blank", sheet, where, _coord(sheet, r, c)) + return _FAIL + return None + if ctype == "DATE": + if isinstance(v, _dt.datetime): + if (v.hour, v.minute, v.second, v.microsecond) == (0, 0, 0, 0): + return v.date().isoformat() + elif isinstance(v, _dt.date): + return v.isoformat() + elif isinstance(v, str): + d = month_day_or_date(v) + if d is not None and d.year is not None: + try: + return _dt.date(d.year, d.month, d.day).isoformat() + except ValueError: + pass + self.problems.issue("value_not_date", sheet, where, _coord(sheet, r, c), f"{cell_ref(r, c)}「{_shown(v)}」") + return _FAIL + # 文本的写法与简单导入一致（tabular.as_text）；文本格就是去首尾空白，不必再走一遍数字识别 + text = v.strip() if type(v) is str else tabular.as_text(v) + store = col.store or ("canonical" if is_pk else "raw") + if store == "raw": + return text + stored = canon(text) + if stored != text: + key = (table, col.name, text, stored) + hit = self.canon_log.get(key) + if hit is not None: + hit.count += 1 + elif len(self.canon_log) < CANON_MAX: + self.canon_log[key] = CanonEntry(table, col.name, text, stored, 1, _coord(sheet, r, c)) + return stored + + def resolve_refs(self, item: DerivedItem, formula: str, lookup: Callable[[tuple[int, int]], str | int]) -> None: + """合计公式的引用经溯源：全在基表 → ref_rowids；有隐藏行或分类汇总函数 → hidden； + 有不在基表的 → outside；形状认不出 → unrecognized。执行器不做加法（H1）。""" + shape = formula_refs(formula) + if shape is None: + item.ref_problem = "unrecognized" + return + func, cells = shape + item.func = func + rowids: list[int] = [] + hidden = func in ("SUBTOTAL", "AGGREGATE") + outside = not cells + for rc in cells: + hit = lookup(rc) + if isinstance(hit, int): + rowids.append(hit) + elif hit == "hidden": + hidden = True + outside = True + else: + outside = True + item.ref_rowids = sorted(rowids) if not outside else None + item.ref_problem = "hidden" if hidden else ("outside" if outside else None) + + # ---------------- 格子账 ---------------- + + def finish_row(self, sh: _Sheet, r: int, cells: list[tuple[int, GridCell]], claims: dict[int, str], + data_row: bool) -> None: + sh.read_count += len(cells) + if sh.recon is not None: + sh.recon.feed(r, len(cells)) + layout = sh.layouts.pop(r, None) if sh.layouts else None + unclaimed: list[int] = [] + for c, cell in cells: + role = claims.get(c) + if role is not None: + sh.roles[role] += 1 + if role in _BLOCK_ROLES: + if sh.hidden_row(r): + sh.hidden_claimed_rows.add(r) + if sh.hidden_col(c): + sh.hidden_claimed_cols.add(c) + continue + if (r, c) in sh.suppressed: + continue + if data_row: + unclaimed.append(c) + continue + if cell.formula is None and isinstance(cell.value, str): + sh.deferred.append((r, c, cell.value)) # 统计期定下来之后再分类 + continue + kind = "number" if cell.formula is not None else classify_outside(cell.value) + if kind == "number": + self._outside_number(sh, r, c, cell.value) + else: + self._outside_text(sh, r, c, _text(cell.value), kind) + if claims or layout: + sh.regions.feed_spans(r, _row_spans(cells, claims, layout)) + for c in unclaimed: + self.problems.issue("cell_unclaimed", sh.name, f"工作表「{sh.name}」第 {r} 行", sh.coord(r, c)) + for st in sh.stopped: + if st.closed or r <= st.stop_row: + continue + if claims: + st.closed = True # 到了别的块的认领区域 + continue + inside = [c for c, _ in cells if st.c1 <= c <= st.c2] + if len(inside) >= st.need: + st.rows.append(r) + st.cells.extend(sh.coord(r, c) for c in inside[:CELLS_MAX - len(st.cells)]) + + def _outside_number(self, sh: _Sheet, r: int, c: int, value: Any) -> None: + self.problems.issue("outside_number", sh.name, f"工作表「{sh.name}」第 {r} 行", sh.coord(r, c), + f"{cell_ref(r, c)}「{_shown(value)}」") + + def _outside_text(self, sh: _Sheet, r: int, c: int, text: str, kind: str, *, period_source: bool = False, + role: str = "outside_text") -> None: + coord = sh.coord(r, c) + sh.roles[role] += 1 + sh.regions.feed_row(r, [(c, role)]) + self.ex.outside_text.append(OutsideText(sh.name, coord, text, kind, period_source)) # type: ignore[arg-type] + if kind == "text_digits" and not period_source: + self.problems.add( + "outside_digits", + f"工作表「{sh.name}」{cell_ref(r, c)} 在导入区域之外有含数字的文字「{_shown(text, 40)}」，不会导入，请确认", + f"工作表「{sh.name}」{cell_ref(r, c)} 在导入区域之外有含数字的文字，不会导入，需要确认", + [coord], sheet=sh.name) + + # ---------------- 统计期（4.2 第 4 步） ---------------- + + def _period(self) -> tuple[Period | None, bool, set[tuple[str, int, int]]]: + """返回 (统计期, 是否报了 period_missing, 作为统计期来源的格)。""" + ctx_sheets = [sh for sh in self.sheets if sh.context is not None] + if not ctx_sheets: + return None, False, set() + found: list[tuple[_Sheet, int, int, str, Period]] = [] + months: list[tuple[_Sheet, int, int, str, Any]] = [] + for sh in ctx_sheets: + for r, c, text in sorted(sh.deferred): + p = cn_date_range(text) + if p is not None: + found.append((sh, r, c, text, p)) + continue + m = cn_year_month(text) + if m is not None: + months.append((sh, r, c, text, m)) + spec0 = ctx_sheets[0].context + assert spec0 is not None + sources: set[tuple[str, int, int]] = set() + if not found: + entered = self.context_inputs.get("统计期") + if entered is not None: + self.ex.period = PeriodOut(entered.start.isoformat(), entered.end.isoformat(), "human", + signed_by=entered.signed_by) + period = Period(entered.start, entered.end) + self._c2(period, ctx_sheets) + return period, False, sources + hint = "" + fp = filename_period(self.filename) + if fp is not None: + hint = f"（文件名中的日期为 {fp.start.isoformat()} 至 {fp.end.isoformat()}，可作参考）" + self.problems.add("period_missing", f"未能从表格中解析出统计期，请为本期录入统计期{hint}", + "未能从表格中解析出统计期，需要人工录入") + return None, True, sources + distinct = {p for *_, p in found} + chosen = found[0] + if len(distinct) > 1: + for item in found: + prefer = item[0].context.prefer_prefix if item[0].context else None + if prefer and match_key(item[3]).startswith(match_key(prefer)): + chosen = item + break + period = chosen[4] + out = PeriodOut(period.start.isoformat(), period.end.isoformat(), "cells") + details: list[str] = [] + failed = 0 + for sh, r, c, text, p in found: + coord = sh.coord(r, c) + if p != period: + failed += 1 + details.append(f"{coord}：{p.start.isoformat()} 至 {p.end.isoformat()}") + for sh, r, c, text, m in months: + coord = sh.coord(r, c) + if not m.contains(period): + failed += 1 + details.append(f"{coord}：{m.year}年{m.month}月") + for sh, r, c, text, _ in [*found, *months]: + coord = sh.coord(r, c) + out.cells.append(coord) + out.texts[coord] = text + sources.add((sh.name, r, c)) + prefer = sh.context.prefer_prefix if sh.context else None + sh.roles["context"] += 1 + sh.regions.feed_row(r, [(c, "context")]) + if not is_pure_period_cell(text, prefer): + out.annotated[coord] = period_residue(text) or "" + self.ex.outside_text.append(OutsideText(sh.name, coord, text, "text_digits", True)) + self.ex.period = out + cells = list(out.cells) + if failed: + self.ex.context_checks.append(CheckResult( + id="C1", kind="context_agree", title="统计期多处一致", status="mismatch", category="data_quality", + checked=len(cells), failed=failed, + details=[f"统计期取自 {chosen[0].coord(chosen[1], chosen[2])}"] + details, cells=cells[:CELLS_MAX], + acceptable=True)) + else: + self.ex.context_checks.append(CheckResult( + id="C1", kind="context_agree", title="统计期多处一致", status="passed", category="data_quality", + checked=len(cells), details=details, cells=cells[:CELLS_MAX])) + self._c2(period, ctx_sheets) + return period, False, sources + + def _c2(self, period: Period, ctx_sheets: list[_Sheet]) -> None: + if not any(sh.context is not None and sh.context.cross_check == "filename" for sh in ctx_sheets): + return + fp = filename_period(self.filename) + title = "文件名中的日期与统计期一致" + if fp is None: + self.ex.context_checks.append(CheckResult( + id="C2", kind="filename_period", title=title, status="info", category="info", + details=["文件名中没有日期区间"])) + elif fp == period: + self.ex.context_checks.append(CheckResult( + id="C2", kind="filename_period", title=title, status="passed", category="data_quality", checked=1)) + else: + self.ex.context_checks.append(CheckResult( + id="C2", kind="filename_period", title=title, status="mismatch", category="data_quality", checked=1, + failed=1, acceptable=True, + details=[f"文件名中的日期为 {fp.start.isoformat()} 至 {fp.end.isoformat()}，" + f"统计期为 {period.start.isoformat()} 至 {period.end.isoformat()}"])) + + # ---------------- 主流程 ---------------- + + def run(self) -> Extraction: + t0 = time.perf_counter() + cols, shape = derive_tables(self.recipe) + if shape: + for rp in shape: + self.problems.add(rp.code, rp.message, rp.message) + self.ex.problems = self.problems.finish(set()) + return self.ex + self.cols = cols + for name, cs in cols.items(): + self.sinks[name] = _Sink(self, name, cs, self.specs.get(name)) + if self.db_path is not None: + _remove_db(self.db_path) + self.conn = _open_sqlite(self.db_path, readonly=False) + self.conn.isolation_level = None + self.conn.execute("BEGIN") + self._create_tables() + self._match_sheets() + t1 = time.perf_counter() + crosstabs: list[_Crosstab] = [] + for sh in self.sheets: + crosstabs.extend(self._phase1(sh)) + t2 = time.perf_counter() + period, period_reported, _ = self._period() + for ct in crosstabs: + ct.complete_axis(period, period_reported) + ct.extract() + for sh in self.sheets: + self._phase4(sh) + self.ex.sheets.skipped_hidden = [{"sheet": s.name, "state": s.state} for s in self.scan.sheets + if s.state != "visible"] + t3 = time.perf_counter() + self._outputs() + failed_sheets = {sh.name for sh in self.sheets if sh.failed} + if failed_sheets: + # 块级定位失败的工作表：区域都不知道在哪，「区域外文字」只是噪声（统计期来源照留） + self.ex.outside_text = [o for o in self.ex.outside_text + if o.period_source or o.sheet not in failed_sheets] + blocking = self.problems.blocking() + self.ex.problems = self.problems.finish(failed_sheets) + t4 = time.perf_counter() + if self.conn is not None: + if blocking: + self.conn.close() + self.conn = None + _remove_db(self.db_path) # type: ignore[arg-type] + else: + for sink in self.sinks.values(): + sink.flush() + self.conn.execute("COMMIT") + self.conn.close() + self.conn = None + self.ex.table_hashes = self._hashes() + self.ex.ok = not blocking and (self.db_path is None or bool(self.ex.table_hashes) or not self.cols) + t5 = time.perf_counter() + self.ex.timings = {"match": round(t1 - t0, 4), "read_locate": round(t2 - t1, 4), + "extract": round(t3 - t2, 4), "receipt": round(t4 - t3, 4), "write": round(t5 - t4, 4), + "total": round(t5 - t0, 4)} + return self.ex + + def _create_tables(self) -> None: + assert self.conn is not None + for name, cs in self.cols.items(): + sink = self.sinks[name] + grain = sink.grain + defs = [] + alias = len(grain) == 1 and next(c.type for c in cs if c.name == grain[0]) == "INTEGER" + for c in cs: + d = f"{_q(c.name)} {c.type}" + if alias and c.name == grain[0]: + # 单列 INTEGER 主键写成表约束会变成 rowid 的别名（rowid 就不再是插入序号，溯源和列表合计的 + # rowid 区间都会错）；列约束的 PRIMARY KEY DESC 不是别名（SQLite 文档写明的历史行为） + d += " PRIMARY KEY DESC NOT NULL" + elif c.name in grain: + d += " NOT NULL" + defs.append(d) + if grain and not alias: + defs.append(f"PRIMARY KEY ({', '.join(_q(g) for g in grain)})") + self.conn.execute(f"CREATE TABLE {_q(name)} ({', '.join(defs)}) STRICT") + + def _hashes(self) -> dict[str, str]: + out: dict[str, str] = {} + conn = _open_sqlite(self.db_path, readonly=True) # type: ignore[arg-type] + try: + for name, cs in self.cols.items(): + grain = self.sinks[name].grain + order = grain or [c.name for c in cs] + sql = (f"SELECT {', '.join(_q(c.name) for c in cs)} FROM {_q(name)} " + f"ORDER BY {', '.join(_q(x) for x in order)}") + h = hashlib.sha256() + for row in conn.execute(sql): + h.update(json.dumps(list(row), ensure_ascii=False, separators=(",", ":")).encode("utf-8")) + h.update(b"\n") + out[name] = h.hexdigest() + finally: + conn.close() + return out + + def _match_sheets(self) -> None: + visible = [s for s in self.scan.sheets if s.state == "visible" and s.nonempty > 0] + used: set[str] = set() + for si, sr in enumerate(self.recipe.sheets): + key = match_key(sr.match.name) + hit = next((s for s in self.scan.sheets if match_key(s.name) == key), None) + renamed = False + if hit is not None and hit.state != "visible": + msg = f"工作表「{hit.name}」是隐藏的工作表，不读取（配方中的工作表「{sr.match.name}」）" + self.problems.add("sheet_missing", msg, msg) + continue + if hit is None and sr.match.fallback == "only_visible_sheet" and len(visible) == 1: + hit, renamed = visible[0], True + if hit is None or hit.name in used: + names = "、".join(f"「{s.name}」" for s in visible[:8]) or "无" + msg = f"没有找到工作表「{sr.match.name}」（工作簿中有内容的可见工作表：{names}）" + self.problems.add("sheet_missing", msg, msg) + continue + used.add(hit.name) + self.ex.sheets.matched[sr.id] = hit.name + if renamed: + self.ex.sheets.renamed[sr.match.name] = hit.name + self.sheets.append(_Sheet(self, si, sr, hit)) + for s in visible: + if s.name in used: + continue + self.ex.sheets.other_visible.append(s.name) + if self.recipe.other_visible_sheets == "reject": + msg = (f"工作簿中另有可见工作表「{s.name}」，配方没有列出它，按配方设置不能导入。" + "请删除该工作表，或在配方面板的「配方没有列出的其他工作表」中选择「记入回执，需确认」") + self.problems.add("sheet_unexpected", msg, msg) + + def _phase1(self, sh: _Sheet) -> list[_Crosstab]: + """读取、定位各块、列表直接抽完、格子账的第一遍（区域外的文字先存着，等统计期）。""" + s = sh.scan + p = self.problems + if s.merged_total > len(s.merged) or s.merged_total > MAX_MERGED: + msg = (f"工作表「{sh.name}」的合并单元格超过 {MAX_MERGED:,} 个，无法逐一核对。" + "请在 Excel 中取消不需要的合并后重新上传") + p.add("merged_too_many", msg, msg, sheet=sh.name) + sh.failed = True + return [] + b = s.bounds + if b is not None and not self.partial and b.area > tabular.SPARSE_FACTOR * max(s.nonempty, tabular.SPARSE_FLOOR): + far = "、".join(list(dict.fromkeys(s.far_cells))[:3]) + msg = (f"工作表「{sh.name}」的已用区域 {b.a1()} 共 {b.area:,} 个单元格，但只有 {s.nonempty:,} 个单元格有内容；" + f"远处的单元格（如 {far}）可能是误输入或残留格式。请在 Excel 中删除这些单元格后重新上传") + p.add("sparse", msg, msg, [f"{sh.name}!{x}" for x in s.far_cells[:CELLS_MAX]], sheet=sh.name) + sh.failed = True + return [] + lists = [blk for blk in sh.sr.blocks if isinstance(blk, ListBlock)] + tabs = [blk for blk in sh.sr.blocks if isinstance(blk, CrosstabBlock)] + big = s.nonempty > xlsx_cells.GRID_MAX_CELLS + if big and (tabs or len(lists) != 1): + msg = (f"工作表「{sh.name}」有 {s.nonempty:,} 个非空单元格，超过逐格核对的上限" + f"（{xlsx_cells.GRID_MAX_CELLS:,} 个）；这么大的工作表只支持一个列表，不支持交叉表或多个表格") + p.add("sheet_too_large_for_layout", msg, msg, sheet=sh.name) + sh.failed = True + return [] + if big and not self.partial and b is not None: + sh.recon = _Reconciler(s) + run = _ListRun(self, sh, lists[0], 0, lambda r, cells, claims, data: self.finish_row( + sh, r, cells, claims, data)) + for r, cells in xlsx_cells.sheet_rows(self.books, s, min_row=b.min_row, max_row=b.max_row, + min_col=b.min_col, max_col=b.max_col): + run.feed(r, cells) + run.finish() + self._recon_problem(sh) + return [] + grid = xlsx_cells.grid_from(self.books, s, max_rows=self.max_rows) + sh.grid = grid + sh.truncated = grid.truncated + if self.partial and self.max_rows is not None and b is not None: + sh.limit = min(b.max_row, b.min_row + self.max_rows - 1) + if not grid.truncated: + sh.recon = _Reconciler(s) + cts = [_Crosstab(self, sh, blk, bi) for bi, blk in enumerate(sh.sr.blocks) if isinstance(blk, CrosstabBlock)] + for ct in cts: + ct.locate() + + def store(r: int, cells: list[tuple[int, GridCell]], claims: dict[int, str], data: bool, + owner: str = "") -> None: + for c, role in claims.items(): + sh.claim(r, c, role, owner) + if data: + sh.data_rows.add(r) + + for bi, blk in enumerate(sh.sr.blocks): + if not isinstance(blk, ListBlock): + continue + run = _ListRun(self, sh, blk, bi, lambda r, cells, claims, data, o=blk.id: store(r, cells, claims, data, o)) + for r in grid.row_numbers(): + run.feed(r, grid.row(r)) + run.finish() + for ct in cts: + ct.post_claims() + for a, bb in sorted(sh.overlaps): + msg = f"工作表「{sh.name}」：「{a}」和「{bb}」认领了同一批单元格" + p.add("segment_overlap", msg, msg, sheet=sh.name) + for r in grid.row_numbers(): + cells = grid.row(r) + claims = {c: sh.claims[(r, c)] for c, _ in cells if (r, c) in sh.claims} + self.finish_row(sh, r, cells, claims, r in sh.data_rows) + self._recon_problem(sh) + return cts + + def _recon_problem(self, sh: _Sheet) -> None: + rc = sh.recon + if rc is None or self.partial: + return + rc.finish() + if not rc.bad and rc.total == sh.scan.nonempty: + return + parts = [f"第 {r} 行：扫描到 {a} 个非空格，读取到 {b} 个" for r, a, b in rc.bad[:CELLS_MAX]] + if len(rc.bad) > CELLS_MAX: + parts.append(f"另有 {len(rc.bad) - CELLS_MAX} 行不一致") + parts.append(f"合计：扫描到 {sh.scan.nonempty} 个，读取到 {rc.total} 个") + msg = (f"工作表「{sh.name}」两遍读取的非空单元格数不一致：{'；'.join(parts)}。" + "文件可能由非常规程序生成，请在 Excel 中另存为 .xlsx 后重新上传") + self.problems.add("pass_mismatch", msg, msg, sheet=sh.name) + + def _phase4(self, sh: _Sheet) -> None: + """区域外文字分类（统计期之后）、停止之后的行、隐藏行列、格子账收尾。""" + period_cells = set(self.ex.period.cells) if self.ex.period and self.ex.period.source == "cells" else set() + for r, c, text in sh.deferred: + if sh.coord(r, c) in period_cells: + continue + kind = classify_outside(text) + if kind == "number": + self._outside_number(sh, r, c, text) + else: + self._outside_text(sh, r, c, text, kind) + for st in sh.stopped: + if st.rows: + where = f"工作表「{sh.name}」的列表「{st.block}」" + msg = (f"{where}：第 {st.rows[0]} 行起有 {len(st.rows)} 行文字在空行之后，没有导入" + f"（{_join([c.split('!')[-1] for c in st.cells], 5)}）。" + "如果它们也是数据，请在配方面板的「数据中间的空行」中选择「跳过继续」") + model = (f"{where}：第 {st.rows[0]} 行起有 {len(st.rows)} 行文字在空行之后，没有导入" + f"（{_join([c.split('!')[-1] for c in st.cells], 5)}）") + self.problems.add("rows_after_stop", msg, model, st.cells, sheet=sh.name) + rows, cols = sorted(sh.hidden_claimed_rows), sorted(sh.hidden_claimed_cols) + pol = sh.sr.hidden + if rows or cols: + self.ex.hidden[sh.name] = {"rows": rows, "cols": [col_letter(c) for c in cols], + "policy_rows": pol.rows, "policy_cols": pol.cols} + if rows and pol.rows == "reject_if_any": + hint = "（工作表设置了筛选，可能是筛选隐藏的行）" if sh.scan.autofilter else "" + msg = (f"工作表「{sh.name}」导入区域内有隐藏的行：第 {_rows_text(rows)} 行{hint}。" + "请在 Excel 中取消隐藏，或在配方面板的「隐藏行」中选择「照常导入」或「跳过」") + self.problems.add("hidden_rows", msg, msg, sheet=sh.name) + if cols and pol.cols == "reject_if_any": + msg = (f"工作表「{sh.name}」导入区域内有隐藏的列：{_join([col_letter(c) for c in cols])} 列。" + "请在 Excel 中取消隐藏，或在配方面板的「隐藏列」中选择「照常导入」") + self.problems.add("hidden_cols", msg, msg, sheet=sh.name) + roles = dict(sh.roles) + self.ex.ledger.append(LedgerSheet(sh.name, sh.scan.nonempty, sh.read_count, roles, + max(sh.read_count - sum(roles.values()), 0))) + self.ex.regions.extend(sh.regions.marks()) + self.ex.block_order[sh.name] = [key for _, key in sorted(sh.order)] + + def _outputs(self) -> None: + ex = self.ex + ex.canonicalized = list(self.canon_log.values()) + for name, cs in self.cols.items(): + sink = self.sinks[name] + spec = self.specs.get(name) + columns = [] + for c in cs: + header = self.headers.get((name, c.name), c.header) + columns.append(ColumnOut(c.name, c.type, header, c.unit, c.role)) + ex.tables.append(TableOut( + name=name, sheet=sink.sheet or (self.sheets[0].name if self.sheets else ""), + kind=spec.kind if spec else "data", columns=columns, grain=list(sink.grain), rows=sink.rows, + sources=list(dict.fromkeys(sink.sources)))) + ex.expected_rows[name] = sink.rows + if sink.lineage: + ex.lineage[name] = {col: runs.out() for col, runs in sink.lineage.items()} + + def close(self) -> None: + self.books.close() + if self.conn is not None: + try: + self.conn.close() + finally: + self.conn = None + + +def execute(recipe: Recipe, raw: bytes, filename: str, scan: WorkbookScan, db_path: str | None, *, + context_inputs: dict[str, PeriodInput] | None = None, + max_rows: int | None = None) -> Extraction: + """按配方执行一份原件（同步，调用方放进线程池）。 + + db_path=None 是干跑：完整定位、认领、转换、溯源，不建库（起草器判完整性、AI 修订回灌用），ok 表示没有 + structure / input / recipe 类问题。有库时一个事务写完全部表（默认回滚日志模式，不开 WAL），关掉连接后算 + table_hashes；有 structure / input / recipe 类问题时不留库文件、ok=False；任何异常都先删掉库文件再抛。 + db_path 上已有的文件会先删掉（调用方给的应当是新路径，试运行库先写 .tmp- 再改名是调用方的事）。 + + 坐标一律写成「工作表!A1」（Problem.cells、DerivedItem.cell、OutsideText.cell、PeriodOut 的键、 + CanonEntry.first_cell）；只有溯源的起始格和 RegionMark.ref 不带工作表（它们旁边另有工作表字段）。 + + max_rows（只许和 db_path=None 一起用）：每张工作表只读已用区域的前 max_rows 行做部分干跑，partial=True， + 跳过依赖整张表的检查（两遍对账、窗口之外的未认领、轴覆盖、稀疏、主键全表去重）。 + """ + if max_rows is not None and db_path is not None: + raise ValueError("只读前若干行的预检（max_rows）不能写库") + eng = _Engine(recipe, raw, filename, scan, db_path, context_inputs, max_rows) + try: + return eng.run() + except BaseException: + eng.close() + if db_path is not None: + _remove_db(db_path) + raise + finally: + eng.close() diff --git a/backend/app/data/recipe_imports.py b/backend/app/data/recipe_imports.py new file mode 100644 index 00000000..d9f9bf24 --- /dev/null +++ b/backend/app/data/recipe_imports.py @@ -0,0 +1,1888 @@ +"""按配方导入的流程（期 2，WP-5c；P2-SPEC 第 7 节）：暂存 → 起草 → 回答问题 / 改配方 → 试运行 → 确认并启用， +以及「上传新一期」（按已确认的配方重放）和「修改配方」（不换文件）。 + +**暂存区是唯一的工作台。** 起草、回答、改配方、试运行都只改 import_stagings 里的一行，提交成功的那个事务 +（table_versions.publish_build）才碰数据源、导入记录和快照。所以任何拒收、放弃、过期都不动当前快照， +暂存区结束即瘦身（table_versions.close_staging）。 + +**各模块经 Pipeline 接缝调用。** 静态校验、读取、执行、核对、说明、起草、差异卡、确认项都来自别的工作包， +这里只编排：default_pipeline() 在函数体里惰性导入真模块，测试替换 pipeline() 返回假实现，状态机的每条 +分支（错误码、提交事务、并发）都能在不依赖真表格的情况下测到。 + +**几条贯穿全文件的规矩：** +- CPU 密集的步骤（扫描、读网格、干跑、试运行、核对、压缩表示）在线程池里，受 table_versions.parse_slot() + 限并发；等模型回复期间不占名额（三次调用、每次 90 秒，占着名额会堵住所有上传和试运行）。 +- 工作配方 = answers_base 按问题顺序重放全部回答。问题（含 effects）按 answers_base 生成，只在 answers_base + 换掉（暂存、PUT、AI 草稿、上传新一期、修改配方）时重算；每次回答只重算卡片。这样先选 A 再改选 B 等于 + 直接选 B，effects 的路径也不会打在已经改过的配方上。 +- 每次回答、PUT、修改配方之前用 detect_facts(grids, scan, recipe=工作配方) 取系统发现，静态校验带上它 + （认领检查）；上传新一期的现行配方原样重放时按 origin="replay" 校验、不查认领（配方已经确认过，本期的 + 数据变化由试运行报，不该在静态校验这一步就把 D26 这类数据质量问题挡成「配方不合法」）。 +- 试运行把 asdict(Extraction) 和核对结果整份存进 trial，提交时据此还原、重算必勾确认项；差异卡的本期 + 一侧用这份完整回执，不用只截了前 50 条规范写法的 TrialOut.receipt。 +- 发布前不改会话里的任何对象：对暂存区的修改一律放进 publish_build 的 before_commit（锁内、同一事务）。 +""" +from __future__ import annotations + +import asyncio +import copy +import dataclasses +import datetime as _dt +import hashlib +import json +import logging +import os +import secrets +from dataclasses import dataclass, field, fields +from pathlib import Path +from typing import Any, Awaitable, Callable + +from sqlalchemy import func, select, update +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import raw_store, recipe_parsers, recipe_types, table_versions +from app.data.recipe_confirm import ConfirmContext +from app.data.recipe_types import ( + REASON_SLOT, + Acceptance, + AiAvailability, + AxisOut, + CanonEntry, + Card, + CheckResult, + ColumnOut, + ConfirmItem, + DerivedItem, + DiffItem, + Draft, + DraftFacts, + Extraction, + Fact, + Grid, + LedgerSheet, + OutsideText, + PeriodInput, + PeriodOut, + Problem, + Question, + Recipe, + RecipeProblem, + RegionMark, + SchemaNotes, + SegmentLabels, + SheetsOut, + TableOut, +) +from app.data.xlsx_scan import WorkbookScan +from app.db.base import new_id, utcnow +from app.db.models import DataSource, ImportStaging, SourceSnapshot, TableBuild, TableImport, TableRecipe + +logger = logging.getLogger(__name__) + +#: 导入清单的格式版本（7.7） +MANIFEST_FORMAT = "agentlab-import-manifest/1" +#: 回答里理由的字数上限（Question.options 的 needs_reason） +ANSWER_REASON_MAX = 200 +#: 接受（Acceptance）理由的字数上限（5.3） +ACCEPT_REASON_MAX = 500 +#: 试运行回执里规范写法对照给界面看几条（全部在导入清单里） +TRIAL_CANON_MAX = 50 +#: trial 里只给服务端自己用的键：StagingOut 不带它们（整份 Extraction 很大，界面也不需要） +TRIAL_INTERNAL = ("extraction", "null_counts", "inputs", "recipe_sha256", "build_id") +#: TrialOut.receipt 给界面看的回执字段（7.4）。完整的回执在 trial["extraction"] 和导入清单里 +_RECEIPT_VIEW = ( + "ledger", "tables", "period", "axes", "placeholders", "labels", "outside_text", "hidden", "sheets", + "derived_form", "block_order", "ignored_columns", "full_calc_on_load", "formula_cells_accepted", + "blank_rows_skipped", "table_hashes", "timings", +) +#: 暂存区的 recipe_origin → 静态校验的 origin。manual / mixed 一律按 manual 校验 +_VALIDATE_ORIGIN = {"rules": "rules", "ai": "ai"} +#: 统计期上下文的 id（契约 ContextSpec.id 只能是它） +PERIOD_KEY = "统计期" +_ACTIVE_TRIAL = ("passed", "needs_decision") + + +# ========================================================================== +# 错误 +# ========================================================================== + + +class ImportRefused(Exception): + """业务上拒绝这次操作。status、code 给接口层原样转成 CodedHTTPException；message 给人看。""" + + def __init__(self, status: int, code: str, message: str) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + + +class CommitConflict(ImportRefused): + """提交时在版本存储锁内发现的冲突（base_changed / name_taken）：409。""" + + def __init__(self, code: str, message: str) -> None: + super().__init__(409, code, message) + + +_CLOSED = "这次导入已经结束（已提交、已放弃或已过期），不能再修改。请重新开始导入" +_NOT_RECIPE = "这个数据源不是按配方导入的。请用「上传表格」更新，或通过按配方导入重新开始" +_RAW_GONE = "这次导入的原件已不在服务端（可能已被清除），请重新上传文件" +_TRIAL_STALE = "试运行之后配方或录入有变化，请重新试运行" +_BASE_CHANGED = "试运行之后当前版本已被更新，请重新试运行后再提交" +#: AI 起草失败（502 ai_failed、413 ai_too_large）之后能做什么，取决于有没有工作配方：有的话配方面板里是表单， +#: 可以接着改；没有的话面板里只能粘贴一份完整的配方（没有表单可填）。recipe_ai 的失败原因只说出了什么事 +_AI_NEXT = { + (True, True): "可以重试，或在配方面板中修改现有配方", + (True, False): "可以重试，或在配方面板中粘贴一份完整的配方", + (False, True): "请在配方面板中修改现有配方", + (False, False): "请在配方面板中粘贴一份完整的配方", +} + + +# ========================================================================== +# 接缝 +# ========================================================================== + + +@dataclass +class Pipeline: + """各工作包的函数。default_pipeline() 接真模块；测试替换 pipeline() 返回假实现。 + + 在 P2-SPEC 10 节的清单上多了几项：scan（暂存、上传新一期要扫描）、column_null_counts（说明只给本期真有 + 空值的列写空值说明，WP-3 的要求）、first_call_text（同意框展示首次调用要发出的全文）、grid_max_cells / + draft_max_rows（大表部分干跑的阈值）、units（单位词表，StagingOut 给表单用）。 + """ + + validate: Callable[..., tuple[Recipe | None, list[RecipeProblem]]] + recipe_sha256: Callable[[Recipe], str] + canonical: Callable[[Recipe], dict[str, Any]] + apply_patch: Callable[[dict[str, Any], list[dict[str, Any]]], dict[str, Any]] + scan: Callable[[bytes], WorkbookScan] + scan_to_json: Callable[[WorkbookScan], dict[str, Any]] + scan_from_json: Callable[[dict[str, Any]], WorkbookScan] + read_grid: Callable[..., Grid] + preview: Callable[..., dict[str, Any]] + execute: Callable[..., Extraction] + run_checks: Callable[[str, Extraction, Recipe], list[CheckResult]] + column_null_counts: Callable[[str, Recipe], dict[str, dict[str, int]]] + build_notes: Callable[..., SchemaNotes] + apply_notes: Callable[[dict[str, Any], SchemaNotes], dict[str, Any]] + detect_facts: Callable[..., DraftFacts] + draft: Callable[..., Draft] + questions_for: Callable[..., tuple[list[Card], list[Question]]] + ai_availability: Callable[..., Awaitable[AiAvailability]] + ai_draft: Callable[..., Awaitable[Any]] + compress: Callable[..., Any] + first_call_text: Callable[..., str] + diff_reports: Callable[..., list[DiffItem]] + confirm_items: Callable[[ConfirmContext], list[ConfirmItem]] + breaking_changes: Callable[..., dict[str, list[str]]] + grid_max_cells: int = 200_000 + draft_max_rows: int = 500 + units: tuple[str, ...] = () + #: (模型对象, 模型 id, 可用性)：同意与发送只解析一次模型（AU-7）。None 时退回 ai_availability，ai_draft 自己解析 + resolve_model: Callable[..., Awaitable[tuple[Any, str, AiAvailability]]] | None = None + + +def default_pipeline() -> Pipeline: + """接上真模块。函数体里惰性导入：这些模块（openpyxl、执行器、起草器）导入不便宜，接口模块加载时不必付。""" + from app.data import ( + recipe, recipe_ai, recipe_checks, recipe_confirm, recipe_diff, recipe_engine, recipe_notes, + recipe_suggest, xlsx_cells, xlsx_scan, + ) + + return Pipeline( + validate=recipe.validate_recipe, + recipe_sha256=recipe.recipe_sha256, + canonical=recipe.canonical_recipe, + apply_patch=recipe.apply_patch, + scan=xlsx_scan.scan, + scan_to_json=xlsx_scan.scan_to_json, + scan_from_json=xlsx_scan.scan_from_json, + read_grid=xlsx_cells.read_grid, + preview=xlsx_cells.preview, + execute=recipe_engine.execute, + run_checks=recipe_checks.run_checks, + column_null_counts=recipe_checks.column_null_counts, + build_notes=recipe_notes.build_notes, + apply_notes=recipe_notes.apply_notes, + detect_facts=recipe_suggest.detect_facts, + draft=recipe_suggest.draft, + questions_for=recipe_suggest.questions_for, + ai_availability=recipe_ai.ai_availability, + ai_draft=recipe_ai.ai_draft, + compress=recipe_ai.compress, + first_call_text=recipe_ai.first_call_text, + diff_reports=recipe_diff.diff_reports, + confirm_items=recipe_confirm.confirm_items, + breaking_changes=recipe_confirm.breaking_changes, + grid_max_cells=xlsx_cells.GRID_MAX_CELLS, + draft_max_rows=recipe_suggest.DRAFT_MAX_ROWS, + units=tuple(recipe_parsers.UNITS), + resolve_model=recipe_ai.resolve_draft_model, + ) + + +def pipeline() -> Pipeline: + """本模块实际用的接缝。测试 monkeypatch 它。""" + return default_pipeline() + + +@dataclass +class CommitResult: + """提交的结果（CommitOut 除 source 以外的字段，接口层补上 source）。""" + + import_id: str + snapshot_id: str + build_id: str + recipe_id: str + build_reused: bool + unchanged: bool + source_id: str = "" + + +# ========================================================================== +# JSON 往返 +# ========================================================================== + + +def _jsonable(obj: Any) -> Any: + """dataclass（含嵌套）、日期 → 能存进 JSON 列的形状。""" + if dataclasses.is_dataclass(obj) and not isinstance(obj, type): + obj = dataclasses.asdict(obj) + return json.loads(json.dumps(obj, ensure_ascii=False, default=str)) + + +def _known(cls: type, data: Any) -> dict[str, Any]: + names = {f.name for f in fields(cls)} + return {k: v for k, v in (data or {}).items() if k in names} + + +def check_from_dict(data: dict[str, Any]) -> CheckResult: + """试运行存下的核对结果 → CheckResult（提交时还原用，7.8 第 3 步）。""" + return CheckResult(**_known(CheckResult, data)) + + +def _table_from(data: dict[str, Any]) -> TableOut: + d = _known(TableOut, data) + d["columns"] = [ColumnOut(**_known(ColumnOut, c)) for c in d.get("columns") or []] + return TableOut(**d) + + +#: Extraction 里需要还原成 dataclass 的字段 +_EXTRACTION_PARTS: dict[str, Callable[[Any], Any]] = { + "problems": lambda v: [Problem(**_known(Problem, x)) for x in v or []], + "tables": lambda v: [_table_from(x) for x in v or []], + "ledger": lambda v: [LedgerSheet(**_known(LedgerSheet, x)) for x in v or []], + "regions": lambda v: [RegionMark(**_known(RegionMark, x)) for x in v or []], + "period": lambda v: PeriodOut(**_known(PeriodOut, v)) if v else None, + "context_checks": lambda v: [check_from_dict(x) for x in v or []], + "derived": lambda v: [DerivedItem(**_known(DerivedItem, x)) for x in v or []], + "canonicalized": lambda v: [CanonEntry(**_known(CanonEntry, x)) for x in v or []], + "labels": lambda v: [SegmentLabels(**_known(SegmentLabels, x)) for x in v or []], + "outside_text": lambda v: [OutsideText(**_known(OutsideText, x)) for x in v or []], + "sheets": lambda v: SheetsOut(**_known(SheetsOut, v)), + "axes": lambda v: [AxisOut(**_known(AxisOut, x)) for x in v or []], +} + + +def extraction_from_dict(data: dict[str, Any]) -> Extraction: + """试运行存下的 asdict(Extraction) → Extraction（提交时还原，说明生成要用 dataclass）。缺的字段取默认值。""" + kw: dict[str, Any] = {} + for f in fields(Extraction): + if f.name not in data: + continue + conv = _EXTRACTION_PARTS.get(f.name) + kw[f.name] = conv(data[f.name]) if conv is not None else copy.deepcopy(data[f.name]) + kw.setdefault("ok", bool(data.get("ok"))) + return Extraction(**kw) + + +def _facts_from(data: Any) -> DraftFacts: + data = data or {} + return DraftFacts(facts=[Fact(**_known(Fact, f)) for f in data.get("facts") or []], + candidates=dict(data.get("candidates") or {}), + nonnumeric=dict(data.get("nonnumeric") or {})) + + +def _question_from(data: dict[str, Any]) -> Question: + return Question(**_known(Question, data)) + + +def _card_from(data: dict[str, Any]) -> Card: + return Card(**_known(Card, data)) + + +def _draft_from(data: Any) -> Draft | None: + if not data: + return None + return Draft(recipe=data.get("recipe"), complete=bool(data.get("complete")), origin=data.get("origin") or "rules", + cards=[_card_from(c) for c in data.get("cards") or []], + questions=[_question_from(q) for q in data.get("questions") or []], + facts=_facts_from(data.get("facts")), failures=list(data.get("failures") or []), + failures_for_model=list(data.get("failures_for_model") or [])) + + +def _draft_out(data: Any) -> dict[str, Any] | None: + """Draft 给界面的形状：不含给模型看的那份原因（failures_for_model），也不重复带系统发现。""" + if not data: + return None + return {k: v for k, v in data.items() if k not in ("failures_for_model", "facts")} + + +def _full_form(recipe: Any) -> Any: + """补全默认值的完整形式：问题的 effects 按它写路径、界面表单按它显示。schema 不过时原样返回。""" + if not isinstance(recipe, dict): + return recipe + try: + return Recipe.model_validate(recipe).model_dump(mode="json") + except Exception: # noqa: BLE001 - 不合法的配方照存，由静态校验报问题 + return copy.deepcopy(recipe) + + +def _period_inputs(ctx: dict[str, Any] | None) -> dict[str, PeriodInput] | None: + """暂存区存的人工录入 → 执行器要的 PeriodInput。""" + out: dict[str, PeriodInput] = {} + for key, v in (ctx or {}).items(): + try: + out[key] = PeriodInput(start=_dt.date.fromisoformat(str(v["start"])), + end=_dt.date.fromisoformat(str(v["end"])), signed_by=v.get("signed_by")) + except (KeyError, TypeError, ValueError): + continue + return out or None + + +def _now_iso() -> str: + return utcnow().isoformat() + + +# ========================================================================== +# 读原件、扫描、网格（线程里跑） +# ========================================================================== + + +@dataclass +class _Book: + raw: bytes + scan: WorkbookScan + #: 每个有内容的可见工作表的前 draft_max_rows 行（起草、系统发现、压缩表示用） + grids: dict[str, Grid] + + +@dataclass +class _Snap: + """暂存区在线程里要用的几项。ORM 对象不进线程：属性过期时会回库加载，线程里没有事件循环。""" + + id: str + kind: str + raw_sha256: str + scan: dict[str, Any] | None + file_name: str + recipe_origin: str | None + #: 起点配方（上传新一期的现行配方）的哈希：工作配方与它相同时按重放校验 + base_sha: str | None = None + context: dict[str, Any] = field(default_factory=dict) + + +def _snap(staging: ImportStaging, *, base_sha: str | None = None, recipe_origin: str | None = None) -> _Snap: + return _Snap(id=staging.id, kind=staging.kind, raw_sha256=staging.raw_sha256, + scan=copy.deepcopy(staging.scan), file_name=staging.file_name or "", + recipe_origin=recipe_origin if recipe_origin is not None else staging.recipe_origin, + base_sha=base_sha, context=copy.deepcopy(staging.context_inputs or {})) + + +def _read_raw(sha: str) -> bytes: + try: + path = raw_store.raw_path(sha) + except ValueError: + raise ImportRefused(409, "raw_missing", _RAW_GONE) from None + if not path.is_file(): + raise ImportRefused(409, "raw_missing", _RAW_GONE) + return path.read_bytes() + + +def _open_book(p: Pipeline, raw: bytes, scan: WorkbookScan) -> _Book: + grids: dict[str, Grid] = {} + for s in scan.sheets: + if s.state != "visible" or s.bounds is None or not s.nonempty: + continue + grids[s.name] = p.read_grid(raw, scan, s.name, max_rows=p.draft_max_rows) + return _Book(raw=raw, scan=scan, grids=grids) + + +def _book_of(p: Pipeline, snap: _Snap) -> _Book: + raw = _read_raw(snap.raw_sha256) + scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw) + return _open_book(p, raw, scan) + + +async def _in_slot(fn: Callable[..., Any], *args: Any, **kw: Any) -> Any: + """CPU 密集的一步：占一个解析名额、放进线程池。只包这一步，用完即还。""" + async with table_versions.parse_slot(): + return await asyncio.to_thread(fn, *args, **kw) + + +class _Capture: + """包一层干跑：记下最后一次的 Extraction（起草器内部干跑过，不必再跑一遍）。""" + + def __init__(self, fn: Callable[[Recipe], Extraction]) -> None: + self.fn = fn + self.last: Extraction | None = None + + def __call__(self, recipe: Recipe) -> Extraction: + self.last = self.fn(recipe) + return self.last + + +def _dry_fn(p: Pipeline, book: _Book, filename: str, ctx: dict[str, Any] | None) -> Callable[[Recipe], Extraction]: + """交互时的干跑（不建库）。有工作表的非空格超过 GRID_MAX_CELLS 时只在前 draft_max_rows 行上部分干跑（4.9）。""" + big = any(s.state == "visible" and s.nonempty > p.grid_max_cells for s in book.scan.sheets) + inputs = _period_inputs(ctx) + + def run(recipe: Recipe) -> Extraction: + return p.execute(recipe, book.raw, filename, book.scan, None, context_inputs=inputs, + max_rows=p.draft_max_rows if big else None) + return run + + +def _validate(p: Pipeline, snap: _Snap, work: Any, facts: DraftFacts | None) -> tuple[Recipe | None, list[Any]]: + """静态校验。上传新一期、工作配方仍是现行配方时按重放校验（不查认领）；其余带系统发现查认领。""" + if not isinstance(work, dict): + return None, [] + if snap.kind == "reupload" and snap.base_sha: + parsed, problems = p.validate(work, facts=None, origin="replay") + if parsed is not None and p.recipe_sha256(parsed) == snap.base_sha: + return parsed, problems + return p.validate(work, facts=facts, origin=_VALIDATE_ORIGIN.get(snap.recipe_origin or "", "manual")) + + +def _cards(p: Pipeline, recipe: Any, facts: DraftFacts, grids: dict[str, Grid], + ext: Extraction | None) -> tuple[list[Card], list[Question]]: + if not isinstance(recipe, dict): + return [], [] + try: + return p.questions_for(recipe, facts, grids, extraction=ext) + except Exception as e: # noqa: BLE001 - 卡片和问题是辅助信息，出错不能拖垮回答、改配方 + logger.warning("questions_for failed: %s", type(e).__name__) + return [], [] + + +#: 交互时的干跑自己出错时给界面的问题（异常只进日志：消息里可能带着格子的值） +_DRY_FAILED = "按配方检查表格未能完成：检查时出错。请修改配方后重试，或直接试运行查看完整结果" + + +def _safe_dry(run: Callable[[Recipe], Extraction], recipe: Recipe) -> Extraction: + """交互时的干跑只给着色和问题列表用：出错时变成一条问题，不让回答、改配方整个失败。试运行不走这里。""" + from app.data.xlsx_scan import UnsupportedTable + + try: + return run(recipe) + except UnsupportedTable as e: + message = str(e) or _DRY_FAILED + except Exception as e: # noqa: BLE001 + logger.warning("interactive dry run failed: %s", type(e).__name__) + message = _DRY_FAILED + return Extraction(ok=False, problems=[Problem("dry_run_failed", "structure", message, + model_message="dry_run_failed")]) + + +@dataclass +class _Eval: + """一次「静态校验 + 干跑 + 卡片」的结果。""" + + recipe: Any + sha: str | None + parsed: Recipe | None + problems: list[Any] + facts: DraftFacts + extraction: Extraction | None + cards: list[Card] + questions: list[Question] + + +def _evaluate(p: Pipeline, book: _Book, snap: _Snap, work: Any, *, facts: DraftFacts | None = None) -> _Eval: + """工作配方 → 系统发现（按这份配方的写法）、静态校验、干跑、卡片和问题（同步，线程里跑）。""" + if facts is None: + facts = p.detect_facts(book.grids, book.scan, recipe=work if isinstance(work, dict) else None) + parsed, problems = _validate(p, snap, work, facts) + ext = None + if parsed is not None and not problems: + ext = _safe_dry(_dry_fn(p, book, snap.file_name, snap.context), parsed) + cards, questions = _cards(p, work, facts, book.grids, ext) + stored = p.canonical(parsed) if parsed is not None else (copy.deepcopy(work) if isinstance(work, dict) else None) + sha = p.recipe_sha256(parsed) if parsed is not None else None + return _Eval(recipe=stored, sha=sha, parsed=parsed, problems=list(problems), facts=facts, extraction=ext, + cards=cards, questions=questions) + + +def _problem_json(p: Any) -> dict[str, Any]: + if isinstance(p, RecipeProblem): + return {"path": p.path, "code": p.code, "message": p.message} + return _jsonable(p) + + +#: 静态校验的问题存库、回显最多几条（SE-5：病态配方能产生几万条，整份存两份再回显）。多出的合成一条 +RECIPE_PROBLEMS_MAX = 200 + + +def _problems_json(problems: list[Any]) -> list[dict[str, Any]]: + out = [_problem_json(x) for x in problems[:RECIPE_PROBLEMS_MAX]] + if len(problems) > RECIPE_PROBLEMS_MAX: + out.append({"path": "", "code": "too_many_problems", + "message": f"另有 {len(problems) - RECIPE_PROBLEMS_MAX} 个问题没有列出（共 {len(problems)} 个），" + "请先按上面的提示修改"}) + return out + + +def _regions(ext: Extraction | dict[str, Any] | None) -> list[dict[str, Any]]: + if ext is None: + return [] + if isinstance(ext, dict): + return [dict(r) for r in ext.get("regions") or [] if isinstance(r, dict)] + return [_jsonable(r) for r in ext.regions] + + +def _ai_card(staging: ImportStaging) -> dict[str, Any] | None: + """AI 草稿那张「请逐项核对」的卡片（ai_draft 生成，存在 drafts.ai 里）。工作配方仍出自 AI 时一直显示。""" + if staging.recipe_origin != "ai": + return None + draft = ((staging.drafts or {}).get("ai_adopted") or {}).get("draft") or {} + return next((c for c in draft.get("cards") or [] if c.get("id") == "ai_origin"), None) + + +def _invalidate_trial(staging: ImportStaging) -> str | None: + """工作配方或录入变了：试运行作废（键和库清空、状态退回 drafting），回执留着给界面标「已失效」。 + + 返回旧的试运行库路径：提交之后再删（remove_trial_file）。 + """ + old = staging.trial_path + staging.trial_key = None + staging.trial_path = None + if staging.status in ("trialed", "rejected"): + staging.status = "drafting" + return old + + +def _apply_eval(staging: ImportStaging, ev: _Eval, *, questions: bool) -> str | None: + """把一次评估写回暂存区（不提交）。配方变了就让试运行作废，返回旧的试运行库路径。""" + old_sha = staging.recipe_sha256 + staging.recipe = ev.recipe + staging.recipe_sha256 = ev.sha + staging.recipe_problems = _problems_json(ev.problems) + staging.facts = _jsonable(ev.facts) + staging.draft_problems = [_jsonable(x) for x in ev.extraction.problems] if ev.extraction is not None else [] + staging.draft_partial = bool(ev.extraction is not None and ev.extraction.partial) + cards = [_jsonable(c) for c in ev.cards] + card = _ai_card(staging) + if card is not None and not any(c.get("id") == "ai_origin" for c in cards): + cards.insert(0, card) + staging.cards = cards + if questions: + staging.questions = [_jsonable(q) for q in ev.questions] + drafts = dict(staging.drafts or {}) + drafts["marks"] = _regions(ev.extraction) + staging.drafts = drafts + staging.updated_at = utcnow() + if ev.sha is None or ev.sha != old_sha: + return _invalidate_trial(staging) + return None + + +async def _remove_trial(path: str | None) -> None: + if path: + await asyncio.to_thread(table_versions.remove_trial_file, path) + + +def require_open(staging: ImportStaging) -> None: + if not table_versions.staging_is_open(staging): + raise ImportRefused(409, "staging_closed", _CLOSED) + + +async def _still_open(session: AsyncSession, staging: ImportStaging) -> None: + """慢步骤之后再看一眼：期间被放弃、过期或提交了的，不再往上写。""" + await session.refresh(staging, ["status"]) + require_open(staging) + + +# ========================================================================== +# 现行版本：配方、导入、上一期回执 +# ========================================================================== + + +async def current_recipe(session: AsyncSession, source: DataSource | None) -> TableRecipe | None: + if source is None or not source.current_recipe_id: + return None + return await session.get(TableRecipe, source.current_recipe_id) + + +async def _require_recipe_source(session: AsyncSession, source: DataSource) -> TableRecipe: + row = await current_recipe(session, source) + if row is None: + raise ImportRefused(409, "not_recipe_source", _NOT_RECIPE) + return row + + +async def current_import(session: AsyncSession, source: DataSource | None) -> tuple[SourceSnapshot | None, + TableImport | None]: + """源的当前快照和其中最近的那一期导入（期 2 替换模式下一个快照只有一期）。""" + if source is None or not source.current_snapshot_id: + return None, None + snap = await session.get(SourceSnapshot, source.current_snapshot_id) + if snap is None or not snap.imports: + return snap, None + return snap, await session.get(TableImport, str(snap.imports[-1])) + + +async def _prev_doc(session: AsyncSession, source: DataSource | None, + imp: TableImport | None) -> dict[str, Any] | None: + """差异卡、确认项的「上一期导入回执」。 + + 按配方导入的源取现行导入的导入清单（7.7：回执、核对含 C1 C2、统计期、配方哈希都在里面），不取 + TableBuild.report——构建可能被复用，回执里没有 C1、C2，文件名也不是那一次的。简单导入的源（切换） + 取期 1 的构建回执，没有格子账，差异卡只给 rows、period。 + """ + if source is None or imp is None: + return None + if source.current_recipe_id: + from app.core import artifact_store + + manifest = artifact_store.load(imp.manifest_artifact) if imp.manifest_artifact else None + if not isinstance(manifest, dict): + raise ImportRefused(500, "manifest_missing", + "现行版本的导入清单不在了，无法与上一期比较。请联系管理员检查数据目录") + return manifest + build = await session.get(TableBuild, imp.build_id) if imp.build_id else None + report = dict(build.report or {}) if build is not None else {} + report.setdefault("tables", []) + return {"receipt": report} + + +def effective_origin(staging: ImportStaging, base_origin: str | None) -> str: + """提交时配方记的 origin，也是确认项的 recipe_origin。 + + 人手改过（recipe_origin=manual）而起点出自 AI（这个暂存区采用过 AI 草稿，或现行配方本来就是 ai / mixed） + 时记 mixed：AI 类确认项（ai_origin、rename、columns）照出。规则草稿改过的记 manual，不出 AI 类。 + """ + origin = staging.recipe_origin or "manual" + if origin != "manual": + return origin + ai_used = bool(((staging.drafts or {}).get("ai_adopted") or {}).get("draft")) + return "mixed" if ai_used or base_origin in ("ai", "mixed") else "manual" + + +# ========================================================================== +# 暂存、上传新一期、修改配方 +# ========================================================================== + + +@dataclass +class _Staged: + scan_json: dict[str, Any] + previews: list[dict[str, Any]] + facts: DraftFacts + draft: Draft | None + ev: _Eval | None + + +def _stage_work(p: Pipeline, raw: bytes, filename: str) -> _Staged: + """首次导入（或切换）的暂存：扫描、网格预览、规则起草（带干跑）、对草稿做一次静态校验。""" + scan = p.scan(raw) + book = _open_book(p, raw, scan) + dry = _Capture(_dry_fn(p, book, filename, {})) + d = p.draft(scan, book.grids, filename, validate=lambda data, f, o: p.validate(data, facts=f, origin=o), + dry_run=dry) + previews = [p.preview(g) for g in book.grids.values()] + ev = None + if d.recipe is not None: + snap = _Snap(id="", kind="first", raw_sha256="", scan=None, file_name=filename, recipe_origin="rules") + parsed, problems = _validate(p, snap, d.recipe, d.facts) + ext = dry.last if parsed is not None and not problems else None + if parsed is not None and not problems and ext is None: + ext = _safe_dry(dry, parsed) + ev = _Eval(recipe=p.canonical(parsed) if parsed is not None else copy.deepcopy(d.recipe), + sha=p.recipe_sha256(parsed) if parsed is not None else None, parsed=parsed, + problems=list(problems), facts=d.facts, extraction=ext, cards=list(d.cards), + questions=list(d.questions)) + return _Staged(scan_json=p.scan_to_json(scan), previews=previews, facts=d.facts, draft=d, ev=ev) + + +def _new_staging(*, source_id: str, name: str, description: str, kind: str, filename: str, raw: bytes, + signed_by: str | None, base_recipe_id: str | None = None) -> ImportStaging: + now = utcnow() + return ImportStaging( + id=new_id(), source_id=source_id, source_name=name, description=description or "", kind=kind, + status="drafting", raw_sha256=hashlib.sha256(raw).hexdigest(), file_name=(filename or "")[:500], + file_size=len(raw), answers={}, cards=[], questions=[], recipe_problems=[], draft_problems=[], + draft_partial=False, context_inputs={}, ai_usage=[], ai_consents=[], base_recipe_id=base_recipe_id, + signed_by=(signed_by or None), created_at=now, updated_at=now, + ) + + +def _fill(staging: ImportStaging, staged: _Staged, ev: _Eval | None, *, answers_base: Any) -> None: + staging.scan = staged.scan_json + staging.grid_preview = staged.previews + staging.facts = _jsonable(staged.facts) + staging.answers_base = copy.deepcopy(answers_base) + if ev is not None: + staging.recipe = ev.recipe + staging.recipe_sha256 = ev.sha + staging.recipe_problems = _problems_json(ev.problems) + staging.facts = _jsonable(ev.facts) + staging.draft_problems = [_jsonable(x) for x in ev.extraction.problems] if ev.extraction else [] + staging.draft_partial = bool(ev.extraction is not None and ev.extraction.partial) + staging.cards = [_jsonable(c) for c in ev.cards] + staging.questions = [_jsonable(q) for q in ev.questions] + drafts = dict(staging.drafts or {}) + drafts["marks"] = _regions(ev.extraction if ev is not None else None) + staging.drafts = drafts + + +async def stage_upload(session: AsyncSession, *, raw: bytes, filename: str, name: str, description: str, + signed_by: str | None) -> ImportStaging: + """POST imports/stage：原件存档、扫描、网格预览、系统发现、规则起草（加干跑）。 + + 同名的简单导入源 → kind=switch（提交时原地切成按配方导入，源 id 和工具名不变）；没有同名源 → first， + 这时就分配源 id，提交时用它建 DataSource。名字格式由接口层查。扫描认不出时 UnsupportedTable 原样抛出。 + """ + p = pipeline() + await table_versions.expire_stagings(session) + existing = (await session.execute(select(DataSource).where(DataSource.name == name))).scalar_one_or_none() + if existing is None: + kind, source_id = "first", new_id() + elif existing.current_recipe_id: + raise ImportRefused(409, "recipe_source_exists", + f"「{name}」已经按配方导入。更新数据请用「上传新一期」，调整取数规则请用「修改配方」") + elif (existing.origin or "manual") == "upload": + kind, source_id = "switch", existing.id + else: + raise ImportRefused(409, "name_taken", f"已存在名为「{name}」的数据源（非上传表格），请换一个名称") + staged = await _in_slot(_stage_work, p, raw, filename) + staging = _new_staging(source_id=source_id, name=name, description=description, kind=kind, + filename=filename, raw=raw, signed_by=signed_by) + staging.recipe_origin = "rules" + d = staged.draft + _fill(staging, staged, staged.ev, answers_base=_full_form(d.recipe) if d and d.recipe is not None else None) + drafts = dict(staging.drafts or {}) + drafts.update({"rules": _jsonable(d) if d is not None else None, "ai": None}) + staging.drafts = drafts + await table_versions.store_staging(session, staging, raw) + return staging + + +def _replay_work(p: Pipeline, raw: bytes, current: dict[str, Any], filename: str, snap: _Snap, + *, evaluate: bool) -> tuple[_Staged, _Eval | None]: + """上传新一期 / 修改配方：扫描（或取存下的扫描）、按现行配方的写法取系统发现，修改配方时再评估一遍。""" + scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw) + book = _open_book(p, raw, scan) + facts = p.detect_facts(book.grids, scan, recipe=current) + staged = _Staged(scan_json=p.scan_to_json(scan), previews=[p.preview(g) for g in book.grids.values()], + facts=facts, draft=None, ev=None) + if not evaluate: + # 上传新一期：现行配方原样重放（不查认领），试运行紧接着做，这里不另干跑 + parsed, problems = _validate(p, snap, current, facts) + cards, questions = _cards(p, current, facts, book.grids, None) + return staged, _Eval(recipe=p.canonical(parsed) if parsed is not None else copy.deepcopy(current), + sha=p.recipe_sha256(parsed) if parsed is not None else None, parsed=parsed, + problems=list(problems), facts=facts, extraction=None, cards=cards, + questions=questions) + return staged, _evaluate(p, book, snap, current, facts=facts) + + +async def reupload(session: AsyncSession, source: DataSource, *, raw: bytes, filename: str, + signed_by: str | None) -> ImportStaging: + """POST {source_id}/reupload：原件存档、扫描、系统发现，按当前配方自动试运行（不调模型）→ 回执 + 差异卡。 + + 系统发现按现行配方的写法存下，用户随后改配方时静态校验用。现行配方原样重放不查认领（见模块 docstring）。 + """ + p = pipeline() + await table_versions.expire_stagings(session) + row = await _require_recipe_source(session, source) + full = _full_form(row.recipe) + staging = _new_staging(source_id=source.id, name=source.name, description=source.description or "", + kind="reupload", filename=filename, raw=raw, signed_by=signed_by, base_recipe_id=row.id) + staging.recipe_origin = row.origin or "manual" + snap = _snap(staging, base_sha=row.recipe_sha256) + staged, ev = await _in_slot(_replay_work, p, raw, full, filename, snap, evaluate=False) + _fill(staging, staged, ev, answers_base=full) + drafts = dict(staging.drafts or {}) + drafts.update({"rules": None, "ai": None}) + staging.drafts = drafts + await table_versions.store_staging(session, staging, raw) + if staging.recipe_sha256 and not staging.recipe_problems: + staging_id = staging.id + try: + staging = await run_trial(session, staging, context_inputs=None, signed_by=signed_by) + except Exception: + # 自动试运行没跑完（全量执行时文件读不了、上一期的清单不在了……）：接口回的是错误，前端拿不到 + # 这个暂存区的 id，留着它只会占着原件直到过期。放弃它（瘦身、原件没有别的引用就删），再抛原错误 + await _discard_quietly(session, staging_id) + raise + return staging + + +async def _discard_quietly(session: AsyncSession, staging_id: str) -> None: + """出错之后放弃刚建的暂存区。收拾本身出错只记日志，不盖住原来的错误。""" + try: + await session.rollback() + row = await session.get(ImportStaging, staging_id, populate_existing=True) + if row is not None and table_versions.staging_is_open(row): + await discard_staging(session, row) + except Exception: # noqa: BLE001 + logger.exception("failed to discard staging %s after an error", staging_id) + + +async def redraft(session: AsyncSession, source: DataSource, *, signed_by: str | None) -> ImportStaging: + """POST {source_id}/redraft：取当前导入的原件、以当前配方为工作配方，重做系统发现和干跑 → drafting。""" + p = pipeline() + await table_versions.expire_stagings(session) + row = await _require_recipe_source(session, source) + _snapshot, imp = await current_import(session, source) + if imp is None or imp.raw_state != "kept" or not raw_store.raw_exists(imp.raw_sha256): + raise ImportRefused(409, "raw_missing", + "当前版本的原件已清除或没有保存，无法在原文件上修改配方。请用「上传新一期」带上文件进入修改") + raw = await asyncio.to_thread(_read_raw, imp.raw_sha256) + full = _full_form(row.recipe) + staging = _new_staging(source_id=source.id, name=source.name, description=source.description or "", + kind="redraft", filename=imp.file_name, raw=raw, signed_by=signed_by, + base_recipe_id=row.id) + staging.recipe_origin = row.origin or "manual" + snap = _snap(staging) + staged, ev = await _in_slot(_replay_work, p, raw, full, imp.file_name, snap, evaluate=True) + _fill(staging, staged, ev, answers_base=full) + drafts = dict(staging.drafts or {}) + drafts.update({"rules": None, "ai": None}) + staging.drafts = drafts + await table_versions.store_staging(session, staging, raw) + return staging + + +# ========================================================================== +# 回答问题、改配方 +# ========================================================================== + + +def _fill_reason(node: Any, reason: str | None) -> Any: + """effects 里整个字符串等于 REASON_SLOT 的值换成理由（不做子串替换，契约 Question.effects）。""" + if isinstance(node, str): + return (reason or "") if node == REASON_SLOT else node + if isinstance(node, list): + return [_fill_reason(x, reason) for x in node] + if isinstance(node, dict): + return {k: _fill_reason(v, reason) for k, v in node.items()} + return node + + +def replay_answers(p: Pipeline, base: Any, questions: list[Question], answers: dict[str, dict[str, Any]]) -> Any: + """工作配方 = answers_base 按问题顺序重放全部回答的 effects（7.2）。effects 用不上时抛 ValueError。""" + doc = copy.deepcopy(base) + for q in questions: + a = answers.get(q.id) + if a is None: + continue + ops = q.effects.get(str(a.get("value")), []) + if ops: + doc = p.apply_patch(doc, _fill_reason(ops, a.get("reason"))) + return doc + + +def _check_answers(questions: list[Question], answers: dict[str, Any]) -> dict[str, dict[str, Any]]: + """回答必须指向存在的问题和选项；needs_reason 的选项要有 1–200 字的理由。返回规范后的回答。""" + by_id = {q.id: q for q in questions} + clean: dict[str, dict[str, Any]] = {} + for qid, a in answers.items(): + q = by_id.get(qid) + if q is None: + raise ImportRefused(422, "answer_invalid", f"问题「{qid}」不存在，请刷新后重新回答") + if not isinstance(a, dict): + raise ImportRefused(422, "answer_invalid", f"问题「{q.text}」的回答格式不对，请重新选择") + value = a.get("value") + option = next((o for o in q.options if str(o.get("value")) == str(value)), None) + if option is None: + raise ImportRefused(422, "answer_invalid", f"问题「{q.text}」没有「{value}」这个选项，请重新选择") + reason = a.get("reason") + reason = str(reason).strip() if reason is not None else "" + if len(reason) > ANSWER_REASON_MAX: + raise ImportRefused(422, "answer_invalid", + f"问题「{q.text}」的理由超过 {ANSWER_REASON_MAX} 字，请精简后重新提交") + if option.get("needs_reason") and not reason: + raise ImportRefused(422, "answer_invalid", + f"问题「{q.text}」选择「{option.get('label') or value}」时需要写明理由") + clean[qid] = {"value": str(option.get("value")), "reason": reason or None} + return clean + + +async def _base_sha(session: AsyncSession, staging: ImportStaging) -> str | None: + if staging.kind != "reupload" or not staging.base_recipe_id: + return None + row = await session.get(TableRecipe, staging.base_recipe_id) + return row.recipe_sha256 if row is not None else None + + +def _reeval_work(p: Pipeline, snap: _Snap, work: Any) -> _Eval: + return _evaluate(p, _book_of(p, snap), snap, work) + + +async def answer(session: AsyncSession, staging: ImportStaging, answers: dict[str, dict[str, Any]]) -> ImportStaging: + """POST imports/{id}/answers：工作配方 = answers_base 重放这一组回答（整组替换已存的回答）， + 重做系统发现、静态校验、干跑和卡片。问题集合不变（它按 answers_base 生成）。""" + p = pipeline() + require_open(staging) + questions = [_question_from(q) for q in staging.questions or []] + clean = _check_answers(questions, answers or {}) + if staging.answers_base is None: + if clean: + raise ImportRefused(422, "answer_invalid", "还没有可回答的配方草稿") + return staging + try: + work = replay_answers(p, staging.answers_base, questions, clean) + except ValueError as e: + raise ImportRefused(422, "answer_invalid", f"回答没能应用到配方上：{e}") from e + snap = _snap(staging, base_sha=await _base_sha(session, staging)) + ev = await _in_slot(_reeval_work, p, snap, work) + await _still_open(session, staging) + old = _apply_eval(staging, ev, questions=False) + staging.answers = clean + await session.commit() + await _remove_trial(old) + return staging + + +async def put_recipe(session: AsyncSession, staging: ImportStaging, recipe: dict[str, Any], *, + origin: str = "manual") -> ImportStaging: + """PUT imports/{id}/recipe：整份配方作为新的起点（answers_base 换成它、回答清空），照存；静态校验的问题 + 写进 recipe_problems（有问题也存，只是不能试运行）。""" + p = pipeline() + require_open(staging) + base = _full_form(recipe) + snap = _snap(staging, base_sha=await _base_sha(session, staging), recipe_origin=origin) + ev = await _in_slot(_reeval_work, p, snap, base) + await _still_open(session, staging) + staging.recipe_origin = origin + staging.answers_base = copy.deepcopy(base) + staging.answers = {} + old = _apply_eval(staging, ev, questions=True) + await session.commit() + await _remove_trial(old) + return staging + + +# ========================================================================== +# AI 起草 +# ========================================================================== + + +def ai_offered(staging: ImportStaging) -> bool: + """首次导入（含切换）、规则草稿不完整时才提供 AI 起草（6.2）。""" + if staging.kind not in ("first", "switch"): + return False + rules = (staging.drafts or {}).get("rules") or {} + return not rules.get("complete") + + +@dataclass +class _Gate: + avail: AiAvailability + #: 解析出的模型对象和 id（pipeline 没有 resolve_model 时为 None / ""，由 ai_draft 自己解析） + model: Any = None + model_id: str = "" + + +async def _ai_gate(session: AsyncSession, staging: ImportStaging) -> _Gate: + require_open(staging) + if staging.kind not in ("first", "switch"): + raise ImportRefused(409, "ai_not_offered", + "上传新一期和修改配方时不提供 AI 起草：工作配方是已确认的现行配方。请在配方面板中修改") + if not ai_offered(staging): + raise ImportRefused(409, "ai_not_needed", "规则起草已经完整，不需要 AI 起草。请检查配方后试运行") + p = pipeline() + if p.resolve_model is not None: + model, model_id, avail = await p.resolve_model(session) + else: + model, model_id, avail = None, "", await p.ai_availability(session) + if not avail.available or (p.resolve_model is not None and model is None): + raise ImportRefused(409, "ai_unavailable", avail.reason or "AI 起草不可用。请到「设置 → 模型接入」检查") + return _Gate(avail=avail, model=model, model_id=model_id or avail.model or "") + + +def consent_token(text_sha256: str, avail: AiAvailability) -> str: + """同意框绑定的令牌：发送的全文 + 接入名 + 模型 id（AU-7、SE-4）。 + + 只按全文算的话，看完预览之后把助手的模型从本机换成外部服务，全文不变、令牌不变，内容就发给了同意框里 + 没写的那个接入。三者任一变了，POST draft-ai 就回 409 ai_preview_stale，要重新查看。 + """ + return table_versions.sha_json({"text_sha256": text_sha256, "provider": avail.provider or "", + "model": avail.model or ""}) + + +async def _mask_headers(session: AsyncSession, staging: ImportStaging) -> set[str]: + """切换时，原数据源遮罩的列在原表里的表头（压缩表示里这些列只给画像）。首次导入没有遮罩列。""" + if staging.kind != "switch": + return set() + from app.data.engine import masked_columns + + source = await session.get(DataSource, staging.source_id) + # 遮罩列名不区分大小写（与证据面板、裁判的遮罩一致，SE-9） + masked = {m.lower() for m in masked_columns(source.options if source is not None else None)} + if not masked: + return set() + _snapshot, imp = await current_import(session, source) + build = await session.get(TableBuild, imp.build_id) if imp is not None and imp.build_id else None + heads: set[str] = set() + for t in ((build.report or {}).get("tables") or []) if build is not None else []: + for c in t.get("columns") or [] if isinstance(t, dict) else []: + if isinstance(c, dict) and str(c.get("name") or "").lower() in masked and c.get("header"): + heads.add(str(c["header"])) + return heads + + +def _ai_failure(staging: ImportStaging, message: str, *, retry: bool = True) -> str: + """AI 起草失败的原话补上下一步（见 _AI_NEXT）。""" + has_work = isinstance(staging.recipe, dict) and bool(staging.recipe) + return f"{message.rstrip('。')}。{_AI_NEXT[(retry, has_work)]}" + + +@dataclass +class _AiPrep: + book: _Book + facts: DraftFacts + compressed: Any + text: str + sha256: str + + +def _ai_prep_work(p: Pipeline, snap: _Snap, rules: Draft | None, mask: set[str]) -> _AiPrep: + """读原件、系统发现（规则起草的分段 id，与半成品配方一致）、压缩表示、首次调用的全文。""" + book = _book_of(p, snap) + facts = p.detect_facts(book.grids, book.scan) + compressed = p.compress(book.grids, book.scan, facts, mask_headers=frozenset(mask)) + text = p.first_call_text(compressed, grids=book.grids, scan=book.scan, rules_draft=rules) + return _AiPrep(book=book, facts=facts, compressed=compressed, text=text, + sha256=hashlib.sha256(text.encode("utf-8")).hexdigest()) + + +async def _ai_prep(session: AsyncSession, staging: ImportStaging) -> _AiPrep: + from app.data.recipe_ai import AiTooLarge + + p = pipeline() + rules = _draft_from((staging.drafts or {}).get("rules")) + mask = await _mask_headers(session, staging) + try: + return await _in_slot(_ai_prep_work, p, _snap(staging), rules, mask) + except AiTooLarge as e: + raise ImportRefused(413, "ai_too_large", _ai_failure(staging, str(e), retry=False)) from e + + +async def ai_preview(session: AsyncSession, staging: ImportStaging) -> dict[str, Any]: + """GET imports/{id}/draft-ai/preview：首次调用要发给模型的全文（系统提示 + 首次提示词，含单位词表、 + 规则半成品、格式说明）。sha256 是同意令牌（全文 + 接入名 + 模型 id，见 consent_token），POST draft-ai 时 + 带回来比对；text_sha256 是全文本身的哈希，留痕用。""" + gate = await _ai_gate(session, staging) + prep = await _ai_prep(session, staging) + return {"text": prep.text, "chars": len(prep.text), "sha256": consent_token(prep.sha256, gate.avail), + "text_sha256": prep.sha256, "model": gate.avail.model, "provider": gate.avail.provider} + + +def _usage_total(usage: list[dict[str, Any]]) -> tuple[int, float]: + tokens = sum(int(u.get("input_tokens") or 0) + int(u.get("output_tokens") or 0) for u in usage) + return tokens, round(sum(float(u.get("cost_usd") or 0) for u in usage), 6) + + +def _adopt_ai_work(p: Pipeline, book: _Book, snap: _Snap, work: Any) -> _Eval: + return _evaluate(p, book, snap, work) + + +#: 正在 AI 起草的暂存区（进程内）：同一暂存区不并发起草，否则两次发送各花一份钱，同意记录、用量还会互相覆盖 +#: （SE-11）。事件循环单线程，查和加之间没有 await，不会有两个请求同时通过；进程重启后自然清空 +_AI_RUNNING: set[str] = set() + + +async def run_ai_draft(session: AsyncSession, staging: ImportStaging, *, consent: bool, + preview_sha256: str | None, signed_by: str | None) -> ImportStaging: + """POST imports/{id}/draft-ai（流程见 _run_ai_draft）。同一暂存区同时只允许一次 AI 起草：已有一次在跑时回 + 409 ai_in_progress，不记同意、不调模型。""" + if consent is not True: + raise ImportRefused(422, "ai_consent_required", "需要先查看并同意发送的内容，才能让 AI 起草") + if staging.id in _AI_RUNNING: + raise ImportRefused(409, "ai_in_progress", "这次导入正在由 AI 起草，请等它完成后再试") + _AI_RUNNING.add(staging.id) + try: + return await _run_ai_draft(session, staging, preview_sha256=preview_sha256, signed_by=signed_by) + finally: + _AI_RUNNING.discard(staging.id) + + +async def _run_ai_draft(session: AsyncSession, staging: ImportStaging, *, + preview_sha256: str | None, signed_by: str | None) -> ImportStaging: + """POST imports/{id}/draft-ai：用户同意后才发送。 + + 发送的全文必须就是预览给用户看的那份：服务端重算的 sha256 与 preview_sha256 不同 → 409 ai_preview_stale。 + 压缩表示在解析名额内先算好、经 compressed= 交给 ai_draft；等模型回复期间不占名额。同意记录在发送前落库； + 用量（AiDraftResult.usage）拿到就单独提交，之后才看暂存区是否还开着（期间结束了 → 409 staging_closed， + 用量照留）、草稿能不能用（不能 → 502 ai_failed）。成功时工作配方换成 AI 草稿（answers_base 换掉、 + 回答清空、recipe_origin=ai）。 + """ + from app.data.recipe_ai import AiTooLarge, AiUnavailable + + p = pipeline() + gate = await _ai_gate(session, staging) + avail = gate.avail + prep = await _ai_prep(session, staging) + token = consent_token(prep.sha256, avail) + if not preview_sha256 or preview_sha256 != token: + raise ImportRefused(409, "ai_preview_stale", + "要发送的内容或接收它的模型与查看时不同（原件、设置或助手的模型有变化），请重新查看后再同意") + await _still_open(session, staging) + consent_rec = {"signed_by": signed_by or None, "at": _now_iso(), "model": avail.model, + "provider": avail.provider, "compressed_sha256": prep.compressed.sha256, + "preview_sha256": token, "text_sha256": prep.sha256, "chars": len(prep.text)} + staging.ai_consents = [*(staging.ai_consents or []), consent_rec] + staging.updated_at = utcnow() + await session.commit() + + rules = _draft_from((staging.drafts or {}).get("rules")) + # AI 每次尝试之后的干跑：协程函数，占一个解析名额、在线程池里执行（ai_draft 经 run_dry_async await 它）。 + # 等模型回复期间不占名额，只有这一步占 + dry_sync = _dry_fn(p, prep.book, staging.file_name or "", staging.context_inputs or {}) + + async def dry(recipe: Recipe) -> Extraction: + return await _in_slot(dry_sync, recipe) + # 同意记录里的模型就是实际调用的模型：把同意前解析出的那个交给 ai_draft，不让它再解析一次（AU-7） + model_kw: dict[str, Any] = {"model": gate.model, "model_id": gate.model_id} if gate.model is not None else {} + try: + result = await p.ai_draft( + session, grids=prep.book.grids, scan=prep.book.scan, facts=prep.facts, rules_draft=rules, + filename=staging.file_name or "", validate=lambda d, f, o: p.validate(d, facts=f, origin=o), + dry_run=dry, compressed=prep.compressed, **model_kw) + except AiUnavailable as e: + raise ImportRefused(409, "ai_unavailable", str(e) or "AI 起草不可用。请到「设置 → 模型接入」检查") from e + except AiTooLarge as e: + raise ImportRefused(413, "ai_too_large", _ai_failure(staging, str(e), retry=False)) from e + + # 用量先落库，再看暂存区还开不开、草稿能不能用：等模型期间暂存区可能已被放弃、过期或提交了（瘦身后的 + # 行也留着用量），采用草稿时也可能出错，这几次调用花的钱都要记下。已结束的暂存区只追加用量，不写草稿 + await session.refresh(staging) + usage = [_jsonable(u) for u in result.usage or []] + tokens, cost = _usage_total(usage) + draft = result.draft + summary = {"draft": _jsonable(draft) if draft is not None else None, "usage": usage, + "attempts": int(result.attempts or 0), "total_tokens": tokens, "cost_usd": cost, + "error": result.error, "compressed_chars": result.compressed_chars, + "compressed_sha256": result.compressed_sha256} + staging.ai_usage = [*(staging.ai_usage or []), *usage] + if table_versions.staging_is_open(staging): + # ai：最近一次起草的摘要（成败都记，界面显示它）；ai_adopted：被采用为工作配方的那一次 + staging.drafts = {**(staging.drafts or {}), "ai": summary} + staging.updated_at = utcnow() + await session.commit() + require_open(staging) + if draft is None or draft.recipe is None: + # 工作配方不变 + raise ImportRefused(502, "ai_failed", _ai_failure(staging, result.error or "AI 未能起草出合法的配方")) + base = _full_form(draft.recipe) + snap = _snap(staging, recipe_origin="ai") + ev = await _in_slot(_adopt_ai_work, p, prep.book, snap, base) + await _still_open(session, staging) + was_ai = effective_origin(staging, None) in ("ai", "mixed") + staging.drafts = {**(staging.drafts or {}), "ai_adopted": summary} + staging.recipe_origin = "ai" + staging.answers_base = copy.deepcopy(base) + staging.answers = {} + old = _apply_eval(staging, ev, questions=True) + if old is None and not was_ai: + # AI 草稿与工作配方逐字相同（哈希没变）时 _apply_eval 不作废试运行，但来源从规则 / 手改变成了 AI， + # 提交时要多勾 ai_origin、rename、columns：存下的确认清单已经不全，试运行一并作废（AU-4） + old = _invalidate_trial(staging) + await session.commit() + await _remove_trial(old) + return staging + + +# ========================================================================== +# 试运行 +# ========================================================================== + + +def trial_key(staging: ImportStaging, recipe_sha: str, ctx: dict[str, Any]) -> str: + """试运行的键：工作配方、人工录入、文件名、解析器与执行器版本（外加原件和暂存区）。任何一项变了就要重新试运行。""" + return table_versions.sha_json({ + "staging_id": staging.id, "raw_sha256": staging.raw_sha256, "recipe_sha256": recipe_sha, + "context": ctx or {}, "file_name": staging.file_name or "", + "parser_ver": recipe_parsers.PARSER_VER, "engine_ver": recipe_types.RECIPE_ENGINE_VER, + }) + + +def _context_allowed(staging: ImportStaging) -> bool: + """录入统计期只在需要时收：上一次试运行要求录入（needs_input），或者上一次用的就是人工录入。""" + trial = staging.trial or {} + if trial.get("status") == "needs_input": + return True + period = (trial.get("receipt") or {}).get("period") or {} + return period.get("source") == "human" + + +def _build_inputs(ex: Extraction | dict[str, Any]) -> dict[str, Any]: + """进构建 id 的人工录入：统计期是人工录入时才有（署名不进）。""" + period = ex.get("period") if isinstance(ex, dict) else (dataclasses.asdict(ex.period) if ex.period else None) + if isinstance(period, dict) and period.get("source") == "human": + return {"context": {PERIOD_KEY: {"start": period.get("start"), "end": period.get("end")}}} + return {} + + +@dataclass +class _TrialRun: + parsed: Recipe | None + problems: list[Any] + facts: DraftFacts | None = None + extraction: Extraction | None = None + db: Path | None = None + db_sha256: str | None = None + checks: list[CheckResult] = field(default_factory=list) + null_counts: dict[str, dict[str, int]] | None = None + notes: SchemaNotes | None = None + + +def _trial_work(p: Pipeline, snap: _Snap, work: dict[str, Any], facts: DraftFacts | None, path: Path) -> _TrialRun: + """执行（写临时库）→ settle → 改名 → 算库哈希 → 核对 → 各列空值数 → 说明预览（同步，线程里跑）。""" + parsed, problems = _validate(p, snap, work, facts) + if parsed is None or problems: + return _TrialRun(parsed=parsed, problems=list(problems)) + raw = _read_raw(snap.raw_sha256) + scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw) + path.parent.mkdir(parents=True, exist_ok=True) + # 先写 .tmp- 再改名：半成品归启动清理（_sweep_tmp）管，发布、回收不会把半个库当成试运行库 + tmp = path.parent / f"{path.name}{raw_store.TMP_MARK}{secrets.token_hex(6)}" + ex = p.execute(parsed, raw, snap.file_name, scan, str(tmp), context_inputs=_period_inputs(snap.context)) + if not tmp.is_file(): + return _TrialRun(parsed=parsed, problems=[], extraction=ex, checks=list(ex.context_checks)) + try: + # 先 settle 再算哈希：发布时 _publish_file 还会 settle 一次，库已是回滚日志模式，文件头不会再变 + table_versions.settle_journal(tmp) + os.replace(tmp, path) + except BaseException: + for suffix in ("", "-journal", "-wal", "-shm"): + Path(f"{tmp}{suffix}").unlink(missing_ok=True) + raise + db_sha = raw_store.sha256_file(path) + checks = p.run_checks(str(path), ex, parsed) + nulls = p.column_null_counts(str(path), parsed) + notes = p.build_notes(parsed, ex, checks, [], null_counts=nulls) + return _TrialRun(parsed=parsed, problems=[], extraction=ex, db=path, db_sha256=db_sha, checks=checks, + null_counts=nulls, notes=notes) + + +def trial_status(problems: list[Problem], checks: list[CheckResult], notes_problems: list[str]) -> str: + """5.3：recipe / structure 类问题、不可接受的结构类核对没过 → rejected；input 类问题 → needs_input； + 有可接受、没通过的核对 → needs_decision；否则 passed。confirm 类问题不影响状态。""" + if notes_problems or any(p.category in ("recipe", "structure") for p in problems): + return "rejected" + if any(c.status != "passed" and c.category == "structure" and not c.acceptable for c in checks): + return "rejected" + if any(p.category == "input" for p in problems): + return "needs_input" + if acceptable_ids(checks): + return "needs_decision" + return "passed" + + +def acceptable_ids(checks: list[CheckResult]) -> list[str]: + """可以写理由接受、本次没通过的核对。""" + return [c.id for c in checks if c.acceptable and c.status in ("mismatch", "unverifiable")] + + +def _receipt_full(ex: Extraction) -> dict[str, Any]: + """导入回执 = asdict(Extraction) 去掉 problems（7.7、7.6）：axes、tables、ledger、period……一样不少。""" + data = _jsonable(ex) + data.pop("problems", None) + return data + + +def _receipt_view(full: dict[str, Any], db_sha: str | None) -> dict[str, Any]: + out = {k: copy.deepcopy(full.get(k)) for k in _RECEIPT_VIEW if k in full} + canon = list(full.get("canonicalized") or []) + out["canonicalized"] = canon[:TRIAL_CANON_MAX] + out["canonicalized_total"] = len(canon) + out["db_sha256"] = db_sha + return out + + +def _checks_summary(checks: list[dict[str, Any]]) -> list[dict[str, Any]]: + keys = ("id", "kind", "title", "status", "category", "checked", "failed", "unverifiable", "reasons", "acceptable") + return [{k: c.get(k) for k in keys} for c in checks] + + +@dataclass +class _Context: + """试运行、提交都要的「现行版本」上下文。""" + + source: DataSource | None + recipe: TableRecipe | None + snapshot: SourceSnapshot | None + imp: TableImport | None + prev: dict[str, Any] | None + + +async def _context(session: AsyncSession, staging: ImportStaging) -> _Context: + source = await session.get(DataSource, staging.source_id) + row = await current_recipe(session, source) + snapshot, imp = await current_import(session, source) + prev = await _prev_doc(session, source, imp) + if staging.kind in ("reupload", "redraft") and (row is None or prev is None): + raise ImportRefused(409, "not_recipe_source", _NOT_RECIPE) + return _Context(source=source, recipe=row, snapshot=snapshot, imp=imp, prev=prev) + + +def _confirm_context(staging: ImportStaging, cx: _Context, recipe: Recipe, ex_dict: dict[str, Any], + checks: list[Any], diff: list[Any] | None) -> ConfirmContext: + from app.data.engine import masked_columns + + old = cx.recipe.recipe if cx.recipe is not None and staging.kind in ("reupload", "redraft") else None + return ConfirmContext( + kind=staging.kind, recipe=recipe, old_recipe=old, + old_schema_cache=(cx.snapshot.schema_cache or {}) if staging.kind == "switch" and cx.snapshot else None, + extraction=ex_dict, checks=checks, diff=(diff or []) if cx.prev is not None else None, prev=cx.prev, + recipe_origin=effective_origin(staging, cx.recipe.origin if cx.recipe is not None else None), + facts=_facts_from(staging.facts), + mask_columns=masked_columns(cx.source.options) if cx.source is not None else None, + ) + + +async def run_trial(session: AsyncSession, staging: ImportStaging, *, + context_inputs: dict[str, PeriodInput] | None, signed_by: str | None) -> ImportStaging: + """POST imports/{id}/trial：执行 → 核对 → 说明预览 → 确认项 →（源有当前版本时）差异卡。拒收也照常返回， + 看 trial.status。试运行库写好后删掉同一暂存区旧的试运行库。""" + p = pipeline() + require_open(staging) + if not isinstance(staging.recipe, dict): + raise ImportRefused(409, "recipe_invalid", "还没有可用的配方，请先在配方面板中填写") + ctx = copy.deepcopy(staging.context_inputs or {}) + if context_inputs: + if not _context_allowed(staging): + raise ImportRefused(422, "context_not_needed", "这份表格里能解析出统计期，不需要人工录入") + ctx = {key: {"start": v.start.isoformat(), "end": v.end.isoformat(), + "signed_by": (v.signed_by or signed_by or None)} for key, v in context_inputs.items()} + cx = await _context(session, staging) + base_sha = cx.recipe.recipe_sha256 if cx.recipe is not None and staging.kind == "reupload" else None + snap = _snap(staging, base_sha=base_sha) + snap.context = ctx + work = _full_form(staging.recipe) + facts = _facts_from(staging.facts) + parsed, problems = _validate(p, snap, work, facts) + if parsed is None or problems: + raise ImportRefused(409, "recipe_invalid", "配方没有通过检查，请先按配方面板里的提示修改") + recipe_sha = p.recipe_sha256(parsed) + key = trial_key(staging, recipe_sha, ctx) + path = table_versions.trial_db_path(staging.source_id, staging.id, key) + from app.data.xlsx_scan import UnsupportedTable + + prev_path = staging.trial_path + try: + run = await _in_slot(_trial_work, p, snap, work, facts, path) + except UnsupportedTable as e: + raise ImportRefused(400, "file_unreadable", str(e) or "无法读取这个文件，请确认是未加密的 Excel 文件") from e + # 从这里到提交之间出错（期间暂存区结束、差异卡或确认项报错、提交失败）：这次新写的试运行库没有记录 + # 指着，当场删掉。路径与旧的相同（同一个键重跑）时不删，那是暂存区记着的库 + try: + await _still_open(session, staging) + if run.extraction is None: + raise ImportRefused(409, "recipe_invalid", "配方没有通过检查，请先按配方面板里的提示修改") + ex = run.extraction + notes_problems = list(run.notes.problems) if run.notes is not None else [] + problems_out = [_jsonable(x) for x in ex.problems] + problems_out += [{"code": "notes_invalid", "category": "recipe", "message": f"说明没有通过检查：{m}", + "cells": [], "model_message": "", "fix": None} for m in notes_problems] + checks_json = [_jsonable(c) for c in run.checks] + status = trial_status(ex.problems, run.checks, notes_problems) + full = _receipt_full(ex) + ex_dict = _jsonable(ex) + diff: list[Any] | None = None + items: list[ConfirmItem] = [] + # 差异卡和确认项只在能提交的试运行（passed / needs_decision）上算。拒收时执行器在结构问题上停下， + # 表、格子账可能只写了一半，拿它比会冒出「行数 31 → 0」这类假变化，结构上的变化已经作为拒收问题 + # 报了（7.6）；needs_input 缺统计期，录入后再试运行就有。这两种状态 diff 为 null + if status in _ACTIVE_TRIAL: + if cx.prev is not None: + prev_name = cx.imp.file_name if cx.imp is not None else None + cur = {"receipt": full, "checks": checks_json, "period": full.get("period"), + "recipe_sha256": recipe_sha} + changed = cx.recipe is None or cx.recipe.recipe_sha256 != recipe_sha + diff = list(p.diff_reports(cx.prev, cur, prev_file_name=prev_name, + cur_file_name=staging.file_name or "", recipe=parsed, + recipe_changed=changed)) + items = list(p.confirm_items(_confirm_context(staging, cx, parsed, ex_dict, run.checks, diff))) + inputs = _build_inputs(ex) + build_id = table_versions.recipe_build_id(staging.source_id, staging.raw_sha256, recipe_sha, inputs) + same = None + if cx.imp is not None and cx.imp.build_id == build_id: + same = {"id": cx.imp.id, "seq": cx.imp.seq} + notes_view = ({t: {"comment": n.comment, "columns": dict(n.columns)} for t, n in run.notes.tables.items()} + if run.notes is not None else {}) + base_snapshot = cx.source.current_snapshot_id if cx.source is not None else None + staging.trial = { + "trial_id": key[:16], "status": status, "receipt": _receipt_view(full, run.db_sha256), + "problems": problems_out, "checks": checks_json, "confirm_items": [_jsonable(i) for i in items], + "acceptable": acceptable_ids(run.checks), "notes": notes_view, + "diff": [_jsonable(d) for d in diff] if diff is not None else None, + "same_as_import": same, "base_snapshot_id": base_snapshot, + # 以下只给服务端：提交时还原 Extraction、重算必勾项、生成说明 + "extraction": ex_dict, "null_counts": run.null_counts, "inputs": inputs, "recipe_sha256": recipe_sha, + "build_id": build_id, + } + old = staging.trial_path + staging.context_inputs = ctx + staging.trial_key = key + staging.trial_path = str(run.db) if run.db is not None else None + staging.base_snapshot_id = base_snapshot + staging.status = "rejected" if status == "rejected" else "trialed" + drafts = dict(staging.drafts or {}) + drafts["marks"] = _regions(ex) + staging.drafts = drafts + if signed_by: + staging.signed_by = signed_by + staging.updated_at = utcnow() + await session.commit() + except Exception: + if run.db is not None and str(run.db) != prev_path: + await _remove_trial(str(run.db)) + raise + if old and old != staging.trial_path: + await _remove_trial(old) + return staging + + +# ========================================================================== +# 提交 +# ========================================================================== + + +async def _reopen_drafting(session: AsyncSession, staging: ImportStaging) -> None: + """提交预检发现试运行用不了：试运行作废、退回 drafting，删掉试运行库。 + + 用带状态条件的 UPDATE：预检期间（几次 await）暂存区可能已被放弃或过期，不能拿会话里过期的对象把它改回 + drafting（SE-8）。已结束的回 409 staging_closed。""" + old = staging.trial_path + done = await session.execute( + update(ImportStaging) + .where(ImportStaging.id == staging.id, ImportStaging.status.in_(table_versions.STAGING_OPEN)) + .values(trial_path=None, trial_key=None, status="drafting", updated_at=utcnow()) + .execution_options(synchronize_session=False)) + await session.commit() + if not done.rowcount: + raise ImportRefused(409, "staging_closed", _CLOSED) + await _remove_trial(old) + + +def _missing_text(items: list[ConfirmItem]) -> str: + shown = "；".join(f"{i.id}「{i.label}」" for i in items[:20]) + more = f"等 {len(items)} 项" if len(items) > 20 else "" + return f"还有确认项没有勾选{more}：{shown}" + + +async def _back_to_drafting(session: AsyncSession, staging_id: str) -> None: + """发布失败：试运行库已被发布流程消耗（或本来就不在），另开一个事务把暂存区退回 drafting。回执留着。""" + try: + await session.rollback() + except Exception: # noqa: BLE001 + pass + row = await session.get(ImportStaging, staging_id, populate_existing=True) + if row is None or not table_versions.staging_is_open(row): + return + old = row.trial_path + row.trial_path = None + row.trial_key = None + row.status = "drafting" + row.updated_at = utcnow() + await session.commit() + await _remove_trial(old) + + +def _manifest(*, staging_id: str, kind: str, source_id: str, info: table_versions.PublishInfo, trial_sha: str | None, + ex_dict: dict[str, Any], file: dict[str, Any], recipe_meta: dict[str, Any], checks: list[dict[str, Any]], + overrides: list[dict[str, Any]], waivers: list[dict[str, Any]], confirmed: list[dict[str, Any]], + notes: SchemaNotes, ai: dict[str, Any], signed_by: str | None) -> dict[str, Any]: + receipt = copy.deepcopy(ex_dict) + receipt.pop("problems", None) + return { + "format": MANIFEST_FORMAT, + "source_id": source_id, "import_id": info.import_id, "seq": info.seq, "build_id": info.build_id, + "build_reused": info.build_reused, "snapshot_id": info.snapshot_id, + "db_sha256": info.db_sha256, "trial_db_sha256": trial_sha, + "table_hashes": copy.deepcopy(ex_dict.get("table_hashes") or {}), + "file": file, "recipe": recipe_meta, + "engine_ver": recipe_types.RECIPE_ENGINE_VER, "parser_ver": recipe_parsers.PARSER_VER, + "period": copy.deepcopy(ex_dict.get("period")), + "checks": checks, + "acceptances": {"overrides": overrides, "waivers": waivers}, "confirmations": confirmed, + "receipt": receipt, + "notes": {"templates": dict(notes.templates), + "rendered": {t: {"comment": n.comment, "columns": dict(n.columns)} for t, n in notes.tables.items()}}, + "ai": ai, + "signed_by": {"name": signed_by, "verified": False}, + "staging_id": staging_id, "kind": kind, + "created_at": _now_iso(), + } + + +async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial_id: str, confirmations: list[str], + acceptances: list[Acceptance], signed_by: str | None) -> CommitResult: + """POST imports/{id}/commit（7.8）：预检 → （与现行导入相同时）不新建记录 → 发布构建、写配方、导入记录、 + 快照、清单，切指针。失败时指针不动，暂存区退回 drafting（试运行库已被发布流程消耗）。""" + p = pipeline() + require_open(staging) + trial = staging.trial or {} + if not isinstance(staging.recipe, dict) or not trial: + raise ImportRefused(409, "trial_required", "还没有试运行，请先试运行") + # 试运行作废时回执留着（界面标「已失效」），键清空；这时工作配方可能已改成 schema 不过的样子 + # （PUT 照存），先看键，别让 model_validate 的英文原文经全局 ValueError 处理器漏成 400 + if not staging.trial_key: + raise ImportRefused(409, "trial_required", _TRIAL_STALE) + try: + parsed = Recipe.model_validate(staging.recipe) + except ValueError: + raise ImportRefused(409, "trial_required", _TRIAL_STALE) from None + recipe_sha = p.recipe_sha256(parsed) + ctx = staging.context_inputs or {} + key = trial_key(staging, recipe_sha, ctx) + if staging.trial_key != key or trial.get("trial_id") != key[:16] or trial_id != key[:16]: + raise ImportRefused(409, "trial_required", _TRIAL_STALE) + if trial.get("status") not in _ACTIVE_TRIAL: + raise ImportRefused(409, "trial_not_passed", "试运行没有通过（拒收或需要录入统计期），不能提交") + + cx = await _context(session, staging) + source = cx.source + base = staging.base_snapshot_id + # 先看当前版本有没有被换掉：换了的话上一期回执也换了，按它重算的确认项没有意义（这里只是快速预检， + # 最终判定在 publish_build 的锁内） + if source is not None and source.current_snapshot_id != base: + raise ImportRefused(409, "base_changed", _BASE_CHANGED) + if source is None: + taken = (await session.execute( + select(DataSource.id).where(DataSource.name == staging.source_name))).first() + if taken is not None: + raise ImportRefused(409, "name_taken", f"名称「{staging.source_name}」已被占用，请放弃这次导入后换一个名称") + ex_dict = trial.get("extraction") or {} + checks = [check_from_dict(c) for c in trial.get("checks") or []] + items = list(p.confirm_items(_confirm_context(staging, cx, parsed, ex_dict, checks, trial.get("diff")))) + shown = {str(i.get("id")) for i in trial.get("confirm_items") or [] if isinstance(i, dict)} + if any(i.required and i.id not in shown for i in items): + # 服务端此刻要求的确认项比试运行时给界面的多（配方来源、数据源的遮罩设置等在试运行之后变了）：界面上的 + # 清单已经不全，用户勾不到缺的那几项。试运行作废，请重新试运行，而不是回一个对不上清单的 422（AU-4） + await _reopen_drafting(session, staging) + raise ImportRefused(409, "trial_required", "试运行之后确认项有变化（配方来源或数据源设置变了），请重新试运行") + ticked = set(confirmations or []) + missing = [i for i in items if i.required and i.id not in ticked] + if missing: + raise ImportRefused(422, "confirm_required", _missing_text(missing)) + acceptable = list(trial.get("acceptable") or []) + given: dict[str, Acceptance] = {} + for a in acceptances or []: + if a.check_id in acceptable and (a.reason or "").strip(): + given[a.check_id] = a + too_long = [cid for cid, a in given.items() if len(a.reason.strip()) > ACCEPT_REASON_MAX] + if too_long: + raise ImportRefused(422, "acceptance_required", + f"接受理由不能超过 {ACCEPT_REASON_MAX} 字：{'、'.join(too_long)}") + lacking = [cid for cid in acceptable if cid not in given] + if lacking: + raise ImportRefused(422, "acceptance_required", + f"以下核对没有通过，接受前需要写明理由：{'、'.join(lacking)}") + + path = Path(staging.trial_path) if staging.trial_path else None + try: + usable = path is not None and path.is_file() and path.stat().st_size > 0 + except OSError: + usable = False + if not usable: + await _reopen_drafting(session, staging) + raise ImportRefused(409, "trial_required", "试运行的数据文件已不在，请重新试运行") + + human = bool(_build_inputs(ex_dict)) + same = trial.get("same_as_import") or None + if (same and not given and not human and cx.imp is not None and cx.imp.id == same.get("id") + and cx.imp.file_name == staging.file_name and source is not None): + # 与现行导入完全相同：不新建任何记录（final.md 2.6 第 5 条）。结束暂存区也在锁内、重读指针之后： + # 预检之后别的提交切走了指针的话，这里返回的「现行导入」已经不是现行的了 + imp_id, imp_build, source_id = cx.imp.id, cx.imp.build_id, source.id + async with table_versions.store_lock(): + cur = (await session.execute( + select(DataSource.current_snapshot_id, DataSource.current_recipe_id) + .where(DataSource.id == source_id))).first() + if cur is None or cur[0] != base: + raise ImportRefused(409, "base_changed", _BASE_CHANGED) + await _require_open_locked(session, staging.id) + old = table_versions.close_staging(staging, "committed") + staging.committed_import_id = imp_id + await session.commit() + await _remove_trial(old) + return CommitResult(import_id=imp_id, snapshot_id=cur[0] or "", build_id=imp_build, + recipe_id=cur[1] or "", build_reused=True, unchanged=True, source_id=source_id) + + ex = extraction_from_dict(ex_dict) + who = signed_by or staging.signed_by + acc = [Acceptance(check_id=a.check_id, reason=a.reason.strip(), signed_by=who) for a in given.values()] + notes = p.build_notes(parsed, ex, checks, acc, null_counts=trial.get("null_counts")) + if notes.problems: + raise ImportRefused(422, "notes_invalid", "说明没有通过检查：" + "；".join(notes.problems[:5])) + + # ---- 以下到 publish_build 之前，不改会话里的任何对象（7.2 第 1 条最后一项） + at = _now_iso() + by_id = {c.id: c for c in checks} + overrides, waivers = [], [] + for a in acc: + rec = {"check_id": a.check_id, "reason": a.reason, "signed_by": a.signed_by, "at": at} + (overrides if by_id[a.check_id].category == "data_quality" else waivers).append(rec) + confirmed = [{"id": i.id, "label": i.label, "at": at} for i in items if i.id in ticked] + checks_full = [_jsonable(c) for c in checks] + period = ex_dict.get("period") or {} + inputs = trial.get("inputs") or {} + source_id, staging_id, kind = staging.source_id, staging.id, staging.kind + name, file_name, file_size, raw_sha = (staging.source_name, staging.file_name or "", int(staging.file_size or 0), + staging.raw_sha256) + base_origin = cx.recipe.origin if cx.recipe is not None else None + origin = effective_origin(staging, base_origin) + recipe_changed = cx.recipe is None or cx.recipe.recipe_sha256 != recipe_sha + recipe_id = new_id() if recipe_changed else cx.recipe.id + canonical = p.canonical(parsed) + build_id = table_versions.recipe_build_id(source_id, raw_sha, recipe_sha, inputs) + import_id = new_id() + ai = {"usage": copy.deepcopy(staging.ai_usage or []), "consents": copy.deepcopy(staging.ai_consents or [])} + trial_sha = (trial.get("receipt") or {}).get("db_sha256") + report = copy.deepcopy(ex_dict) + report.pop("problems", None) + report.pop("context_checks", None) + import_fields = { + "recipe_id": recipe_id, "period_start": period.get("start"), "period_end": period.get("end"), + "context": copy.deepcopy(period) or None, "checks": _checks_summary(checks_full), + "overrides": overrides, "waivers": waivers, "confirmations": confirmed, "signed_by": who, + "staging_id": staging_id, + } + if source is None: + target = DataSource(id=source_id, name=name, kind="sqlite", readonly=True, + description=staging.description or "", origin="upload") + else: + target = source + manifest_ids: dict[str, str] = {} + + async def schema_cache_for(view: table_versions.SourceView, info: table_versions.PublishInfo) -> dict[str, Any]: + from app.core import artifact_store + + cache = await table_versions.introspect_view(view, info.report) + cache = p.apply_notes(cache, notes) + if notes.problems: + raise table_versions.PublishError("导入失败：说明没能写进表结构（" + "；".join(notes.problems[:3]) + + "），当前版本未受影响") + table_versions.check_probe(cache, info.report, source_id) + manifest = _manifest( + staging_id=staging_id, kind=kind, source_id=source_id, info=info, trial_sha=trial_sha, ex_dict=ex_dict, + file={"name": file_name, "size": file_size, "raw_sha256": raw_sha}, + recipe_meta={"id": recipe_id, "sha256": recipe_sha, "origin": origin, "canonical": canonical}, + checks=checks_full, overrides=overrides, waivers=waivers, confirmed=confirmed, notes=notes, ai=ai, + signed_by=who) + mid = await artifact_store.put_json(manifest, kind="import_manifest", + meta={"source_id": source_id, "import_id": info.import_id}) + manifest_ids["id"] = mid + cache["import_manifests"] = [mid] + return cache + + async def before_commit(imp_id: str, snap_id: str) -> None: + # 锁内、同一事务：用 select 重读，不信会话里缓存的对象 + now = utcnow() + cur = (await session.execute( + select(DataSource.current_snapshot_id).where(DataSource.id == source_id))).first() + if (cur[0] if cur is not None else None) != base: + raise CommitConflict("base_changed", _BASE_CHANGED) + # 暂存区本身也要在锁内重读：预检之后它可能已被放弃（放弃的请求先拿到了锁），已放弃的不能再发布（SE-8） + await _require_open_locked(session, staging_id) + if kind == "first" or source is None: + taken = (await session.execute( + select(DataSource.id).where(DataSource.name == name, DataSource.id != source_id))).first() + if taken is not None: + raise CommitConflict("name_taken", f"名称「{name}」已被占用，请放弃这次导入后换一个名称") + if recipe_changed: + last = (await session.execute( + select(func.max(TableRecipe.seq)).where(TableRecipe.source_id == source_id))).scalar() or 0 + await session.execute( + update(TableRecipe).where(TableRecipe.source_id == source_id, TableRecipe.status == "active") + .values(status="superseded")) + session.add(TableRecipe( + id=recipe_id, source_id=source_id, seq=last + 1, recipe=canonical, recipe_sha256=recipe_sha, + recipe_format=recipe_types.RECIPE_FORMAT, origin=origin, status="active", confirmations=confirmed, + signed_by=who, ai_usage={"calls": ai["usage"], "consents": ai["consents"]}, staging_id=staging_id, + created_at=now, activated_at=now)) + target.current_recipe_id = recipe_id + imp = await session.get(TableImport, imp_id) + if imp is not None: + imp.manifest_artifact = manifest_ids.get("id") + staging.committed_import_id = imp_id + table_versions.close_staging(staging, "committed", now=now) + + try: + result = await table_versions.publish_build( + session, target, tmp_db=path, build_id=build_id, + build_options=table_versions.recipe_build_options(recipe_sha, inputs), + engine_ver=recipe_types.RECIPE_ENGINE_VER, report=report, raw=None, raw_sha=raw_sha, + file_name=file_name, file_size=file_size, schema_cache_for=schema_cache_for, + import_fields=import_fields, expected_db_sha256=trial_sha, before_commit=before_commit, + import_id=import_id) + except ImportRefused: + await _back_to_drafting(session, staging_id) + raise + except table_versions.PublishError as e: + await _back_to_drafting(session, staging_id) + if e.code == "trial_missing": + raise ImportRefused(409, "trial_required", str(e)) from e + if e.code == "trial_tampered": + raise ImportRefused(409, "trial_tampered", str(e)) from e + raise ImportRefused(500, "publish_failed", str(e)) from e + except IntegrityError as e: + await _back_to_drafting(session, staging_id) + if "data_sources.name" in str(e.orig if e.orig is not None else e): + raise ImportRefused(409, "name_taken", f"名称「{name}」已被占用，请放弃这次导入后换一个名称") from e + raise ImportRefused(500, "publish_failed", "导入失败：写入版本记录时出错，当前版本未受影响") from e + except BaseException: + await _back_to_drafting(session, staging_id) + raise + return CommitResult(import_id=result.import_id, snapshot_id=result.snapshot_id, build_id=result.build_id, + recipe_id=recipe_id, build_reused=result.build_reused, unchanged=False, source_id=source_id) + + +# ========================================================================== +# 放弃 +# ========================================================================== + + +async def _require_open_locked(session: AsyncSession, staging_id: str) -> None: + """锁内按库里的状态判断暂存区是否还开着（select 重读，不信会话里缓存的对象）。""" + status = (await session.execute( + select(ImportStaging.status).where(ImportStaging.id == staging_id))).scalar_one_or_none() + if status not in table_versions.STAGING_OPEN: + raise CommitConflict("staging_closed", _CLOSED) + + +async def discard_staging(session: AsyncSession, staging: ImportStaging) -> None: + """DELETE imports/{id}：status → discarded、瘦身、删试运行库；原件已没有别的引用时当场删除。 + + 进锁之后重读状态：排在提交后面等锁的放弃，拿到锁时暂存区可能已经提交了，不能把 committed 改写成 + discarded（SE-8）。""" + require_open(staging) + async with table_versions.store_lock(): + await session.refresh(staging, ["status"]) + require_open(staging) + old = table_versions.close_staging(staging, "discarded") + await session.commit() + await _remove_trial(old) + await table_versions.release_staging_raw(session, staging) + + +# ========================================================================== +# 响应形状（7.4 StagingOut） +# ========================================================================== + + +def _span_count(spans: Any) -> int: + total = 0 + for s in spans or []: + try: + total += int(s[1]) - int(s[0]) + 1 + except (TypeError, ValueError, IndexError): + continue + return total + + +def _bounds_a1(b: Any) -> str | None: + if not isinstance(b, dict): + return None + from app.data.xlsx_scan import cell_ref + + try: + return f"{cell_ref(int(b['min_row']), int(b['min_col']))}:{cell_ref(int(b['max_row']), int(b['max_col']))}" + except (KeyError, TypeError, ValueError): + return None + + +def import_mode(source: DataSource | None) -> str | None: + if source is None: + return None + if source.current_recipe_id: + return "recipe" + return "simple" if (source.origin or "manual") == "upload" else None + + +def _iso(v: Any) -> str | None: + return v.isoformat() if v is not None else None + + +def staging_out(staging: ImportStaging, source: DataSource | None, *, + ai: AiAvailability | None = None, units: tuple[str, ...] | None = None) -> dict[str, Any]: + """7.4 的 StagingOut。ai 是模型接入的可用性（只在提供 AI 起草时由接口层查好传进来）。结束（瘦身）后的 + 暂存区照样能给出：大字段都是空的。""" + scan = staging.scan or {} + sheets, skipped = [], [] + for s in scan.get("sheets") or []: + if not isinstance(s, dict): + continue + if s.get("state") != "visible": + skipped.append({"sheet": s.get("name"), "state": s.get("state")}) + continue + sheets.append({ + "name": s.get("name"), "state": s.get("state"), "bounds": _bounds_a1(s.get("bounds")), + "nonempty": int(s.get("nonempty") or 0), + "merged": int(s.get("merged_total") or len(s.get("merged") or [])), + "formulas": int(s.get("formulas") or 0), + "formulas_uncached": int((s.get("formulas_uncached") or {}).get("total") or 0), + "hidden_rows": _span_count(s.get("hidden_rows")), "hidden_cols": _span_count(s.get("hidden_cols")), + }) + drafts = staging.drafts or {} + trial = staging.trial + if isinstance(trial, dict): + trial = {k: copy.deepcopy(v) for k, v in trial.items() if k not in TRIAL_INTERNAL} + ai_sum = drafts.get("ai") + if isinstance(ai_sum, dict): + ai_sum = {**ai_sum, "draft": _draft_out(ai_sum.get("draft"))} + offered = table_versions.staging_is_open(staging) and ai_offered(staging) + recipe = _full_form(staging.recipe) if isinstance(staging.recipe, dict) else None + facts = staging.facts or {} + marks = drafts.get("marks") or [] + if not marks and isinstance(staging.trial, dict): + marks = _regions(staging.trial.get("extraction")) + if units is None: + units = tuple(recipe_parsers.UNITS) + return { + "id": staging.id, "kind": staging.kind, "status": staging.status, + "source": {"id": staging.source_id, "name": source.name if source is not None else staging.source_name, + "exists": source is not None, "import_mode": import_mode(source)}, + "file": {"name": staging.file_name, "size": int(staging.file_size or 0), + "sha256_prefix": (staging.raw_sha256 or "")[:8]}, + "sheets": sheets, "skipped_sheets": skipped, "full_calc_on_load": bool(scan.get("full_calc_on_load")), + "grids": list(staging.grid_preview or []), + "draft": _draft_out(drafts.get("rules")), + "ai_draft": ai_sum, + "ai": {"offered": offered, "available": bool(ai.available) if ai is not None else False, + "reason": ai.reason if ai is not None else "", "model": ai.model if ai is not None else "", + "provider": ai.provider if ai is not None else ""}, + "ai_consents": list(staging.ai_consents or []), + "recipe": recipe, "recipe_origin": staging.recipe_origin, + "recipe_problems": list(staging.recipe_problems or []), + "answers": dict(staging.answers or {}), + "cards": list(staging.cards or []), "questions": list(staging.questions or []), + "draft_problems": list(staging.draft_problems or []), "draft_partial": bool(staging.draft_partial), + "candidates": dict(facts.get("candidates") or {}), + "units": list(units), + "marks": marks, + "trial": trial, + "context_inputs": dict(staging.context_inputs or {}), + "base_snapshot_id": staging.base_snapshot_id, + "committed_import_id": staging.committed_import_id, + "created_at": _iso(staging.created_at), "updated_at": _iso(staging.updated_at), + "expires_at": _iso(staging.expires_at), + } + + +__all__ = [ + "ACCEPT_REASON_MAX", "ANSWER_REASON_MAX", "CommitConflict", "CommitResult", "ImportRefused", "MANIFEST_FORMAT", + "Pipeline", "acceptable_ids", "ai_offered", "ai_preview", "answer", "check_from_dict", "commit_staging", + "consent_token", + "current_import", "current_recipe", "default_pipeline", "discard_staging", "effective_origin", + "extraction_from_dict", "import_mode", "pipeline", "put_recipe", "redraft", "replay_answers", "reupload", + "run_ai_draft", "run_trial", "stage_upload", "staging_out", "trial_key", "trial_status", +] diff --git a/backend/app/data/recipe_notes.py b/backend/app/data/recipe_notes.py new file mode 100644 index 00000000..e3931e42 --- /dev/null +++ b/backend/app/data/recipe_notes.py @@ -0,0 +1,649 @@ +"""按核对结果生成表说明和列说明（P2-SPEC 5.4），再写进要冻结的表结构（schema_cache）。 + +说明进模型可见文本：introspect.describe_table 把表和列的 comment 拼进 db_schema 的输出，摘要里也有。 +所以这里守三条规矩： +- **说明跟着核对结果走，规则写死**：K / T 没通过，这张表的说明里不出现「一致」；一条核对有几种无法核对的 + 原因，用最弱的「原因混合」说法；找不到对应模板就记 problems（不能保存），绝不退回到别的模板—— + 宁可导不进去，也不让模型读到「已核对一致」而事实上没核对。核对本身是不可接受的结构类失败（试运行已经 + 拒收）时，那一段不写、也不记 problems，免得用户被引去改配方（见 _blocking）。 +- **不带数字和坐标**：模板里的列名、表名、单位一律写成 {列:名} {表:名} {单位:名} 引用，每张表、每列的模板都过 + 契约的 prose_problems(known=…)（遮盖引用后不许抽出数字、不许有坐标，引用必须指向真实的名字）。 + 原表头、坐标、区域外文字全文只进清单和界面，不进说明。 +- **模板是常量，拼接只用 .format()**：模板放在 NOTE_TEXT / COLUMN_TEXT 里，{a} {b} 这类槽位由 _tok 填入引用； + 这样模板本身经得起文案检查，填进去的名字也不会被当成格式串再解析一次。 +""" +from __future__ import annotations + +import copy +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any + +from app.data.names import canon +from app.data.recipe_checks import date_columns, plan_checks, split_coord +from app.data.recipe_parsers import UNITS +from app.data.recipe_types import ( + Acceptance, + CheckResult, + ColumnOut, + CrosstabBlock, + DerivedSegment, + DimensionSegment, + Extraction, + ListBlock, + MeasuresSegment, + NotComparable, + Placeholder, + Recipe, + SchemaNotes, + SheetRecipe, + SumEq, + TableNote, + derive_tables, + prose_problems, + render_prose, +) + +#: 表说明的片段模板。槽位里填的是 {列:名} {表:名} 引用（_tok 生成），片段之间用「。」拼接 +NOTE_TEXT = { + "grain_1": "粒度：每个{a}", + "grain_2": "粒度：每个{a}的每个{b}", + "grain_n": "粒度：{a} 与 {b} 的每个组合", + "sum_eq_passed": "{total} 等于 {parts} 之和（导入时已{per}核对）：求合计只用 {total}，不要把这几列相加", + "sum_eq_mismatch_accepted": "{total} 与 {parts} 之和的关系在本期个别{unit}不成立，已由用户确认接受，详见导入清单：" + "不要据此推算，也不要把这几列相加", + # 试运行回执的预览按「全部未接受」生成（P2-SPEC 7.4），这时还不能写「已由用户确认接受」 + "sum_eq_mismatch_pending": "{total} 与 {parts} 之和的关系在本期个别{unit}不成立，尚未确认接受：" + "不要据此推算，也不要把这几列相加", + "sum_eq_nulls": "{total} 与 {parts} 之和的关系在本期部分{unit}因空值未能核对：不要据此推算，也不要把这几列相加", + # 本期一行数据都没有（AU-1）：不能写「已核对」，也不是「因空值」 + "sum_eq_no_rows": "{total} 与 {parts} 之和的关系在本期没有数据行，未能核对：不要据此推算，也不要把这几列相加", + "not_comparable_unequal": "与 {other} 口径不同：各{dims}之和不等于 {value}，不能互相推算或相加", + # 基表除了分组列没有别的粒度列时，「各……之和」无从说起，改说按分组列汇总 + "not_comparable_unequal_flat": "与 {other} 口径不同：{a_value} 按 {by} 汇总不等于 {value}，不能互相推算或相加", + "not_comparable_equal": "与 {other} 口径不同，不能互相推算或相加", + "hour_range": "{dim} 按规范写法存储：{start}-{end}，不带单位", + # 配方没派生起止小时列时，没有列可以引用 + "hour_range_plain": "{dim} 按规范写法存储：起始小时与结束小时用连字符相连，不带单位", + "const": "{col} 取自原表的分段标题", + "total_kept": "原表的合计在 {table}，不要与本表相加", + "total_not_kept": "原表的合计行未导入本表", + "total_k_passed": "原表写明的合计，已按明细重算核对一致", + "total_k_passed_g_unverifiable": "原表写明的合计，合计值已按明细核对一致，公式引用的格子未能核对", + "total_k_uncached": "原表写明的合计。本期原表未保存计算结果，合计值为空，未能核对", + "total_k_blank": "原表写明的合计。本期原表合计格为空，未能核对", + "total_k_null_detail": "原表写明的合计，按原表保存的值存储。本期明细含无数据占位符，未能核对", + "total_k_tiling": "原表写明的合计，按原表保存的值存储。本期明细未能覆盖合计的全部范围，未能核对", + "total_k_hidden": "原表写明的合计，按原表保存的值存储。本期存在隐藏行或使用了分类汇总函数，合计口径无法确定，未能核对", + "total_k_mixed": "原表写明的合计，按原表保存的值存储。本期合计值未能核对", + "total_overlap": "各{dim}覆盖的时段可能互相重叠：不要彼此相加，也不要与 {base} 相加", + "list_total_passed": "原表合计行的值，已按各列明细求和核对一致：不要与 {base} 相加", + # 合计行里空着的数字格不生成合计格（P2-SPEC 4.4），T 只核对了其余的列：不能说「各列」都一致 + "list_total_passed_partial": "原表合计行的值。本期原表 {cols} 的合计格为空，其余各列已按明细求和核对一致：" + "不要与 {base} 相加", + "list_total_uncached": "原表合计行的值。本期原表未保存计算结果，合计值为空，未能核对：不要与 {base} 相加", + # 合计格写的是占位符（转成空值）时走到这里，与交叉表的 total_k_blank 对应 + "list_total_blank": "原表合计行的值。本期原表合计格为空，未能核对：不要与 {base} 相加", + "list_total_tiling": "原表合计行的值，按原表保存的值存储。本期合计行之上没有明细，未能核对：不要与 {base} 相加", + "list_total_null_detail": "原表合计行的值，按原表保存的值存储。本期明细含空值，未能核对：不要与 {base} 相加", + "list_total_hidden": "原表合计行的值，按原表保存的值存储。本期存在隐藏行或使用了分类汇总函数，合计口径无法确定，" + "未能核对：不要与 {base} 相加", + "list_total_mixed": "原表合计行的值，按原表保存的值存储。本期未能核对：不要与 {base} 相加", + "hidden_excluded": "本表不含原表中被隐藏的行", + "hidden_included": "本表包含原表中被隐藏的行", + "ignored_columns": "原表另有未导入的列", + "rows_after_stop": "原表在空行之后另有未导入的文字行，内容见证据面板", + # 不写「不一致」：K / T 没通过的表不许出现「一致」二字，按子串判断时「不一致」也会撞上 + "period_conflict_accepted": "本期统计期的写法在原表或文件名中有冲突，已由用户确认接受，详见导入清单", + "period_conflict_pending": "本期统计期的写法在原表或文件名中有冲突，尚未确认接受", + "period_human": "本期统计期为人工录入", + "outside_digits": "本期原表附有说明文字（可能涉及口径），内容见证据面板", +} + +#: 列说明的片段模板 +COLUMN_TEXT = { + "unit": "单位：{unit}", + "null_placeholder": "空值表示原表为{meaning}占位符，不代表零；求和时请同时统计非空个数", + "null_blank": "空值表示原表为空格，不代表零；求和时请同时统计非空个数", + "null_both": "空值表示原表为{meaning}占位符或空格，不代表零；求和时请同时统计非空个数", + "date": "格式 YYYY-MM-DD", + "date_year_from_period": "格式 YYYY-MM-DD，年份按统计期补全", +} + +#: 「逐日 / 逐行」「个别日期 / 个别行」：粒度只有一列日期时按日说，否则按行说 +_PER = {True: ("逐日", "日期"), False: ("逐行", "行")} +_AND = " 与 " +_LIST = "、" +_MEANING_BOTH = "无数据或不适用" +#: 交叉表合计的单一原因 → 模板键；列表合计同理。列表的 blank：合计格写了占位符（非空格，转成空值，照样生成 +#: 合计格）；tiling：合计行之上没有明细行。unrecognized 只出现在公式引用（G）上 +_K_REASON = {r: f"total_k_{r}" for r in ("uncached", "blank", "null_detail", "tiling", "hidden")} +_T_REASON = {r: f"list_total_{r}" for r in ("uncached", "blank", "null_detail", "tiling", "hidden")} +#: 允许出现「一致」的模板：只在对应的 K / T 通过时才会选用 +CONSISTENT_KEYS = frozenset({"total_k_passed", "total_k_passed_g_unverifiable", "list_total_passed", + "list_total_passed_partial"}) + + +def _tok(kind: str, name: str) -> str: + """结构化引用 {列:名} / {表:名} / {单位:名}。渲染时换成名字，检查时遮盖。""" + return "{" + kind + ":" + name + "}" + + +def _col(name: str) -> str: + return _tok("列", name) + + +def _tab(name: str) -> str: + return _tok("表", name) + + +def _join_and(names: list[str]) -> str: + """a、b 与 c。""" + if len(names) <= 1: + return "".join(names) + return _LIST.join(names[:-1]) + _AND + names[-1] + + +# ---------------------------------------------------------------- 配方结构：每张表从哪来 + + +@dataclass +class _Writers: + """一张表在配方里的来历（哪些工作表、分段、块写进它、它是谁的合计另存表）。""" + + sheets: list[SheetRecipe] = field(default_factory=list) + crosstabs: list[CrosstabBlock] = field(default_factory=list) + dims: list[DimensionSegment] = field(default_factory=list) + measures: list[MeasuresSegment] = field(default_factory=list) + #: 以本表为基表核对的 derived 段（本表是明细表） + verified_by: list[DerivedSegment] = field(default_factory=list) + #: 把合计另存进本表的 derived 段（本表是交叉表的表内合计表） + kept_from: list[DerivedSegment] = field(default_factory=list) + lists: list[ListBlock] = field(default_factory=list) + #: 把合计行另存进本表的列表块 + list_totals: list[ListBlock] = field(default_factory=list) + + def add_sheet(self, sheet: SheetRecipe) -> None: + if all(s is not sheet for s in self.sheets): + self.sheets.append(sheet) + + +def _writers(recipe: Recipe) -> dict[str, _Writers]: + out: dict[str, _Writers] = {} + + def w(table: str, sheet: SheetRecipe) -> _Writers: + entry = out.setdefault(table, _Writers()) + entry.add_sheet(sheet) + return entry + + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, CrosstabBlock): + for seg in block.segments: + if isinstance(seg, MeasuresSegment): + e = w(seg.table, sheet) + e.measures.append(seg) + if block not in e.crosstabs: + e.crosstabs.append(block) + elif isinstance(seg, DimensionSegment): + e = w(seg.table, sheet) + e.dims.append(seg) + if block not in e.crosstabs: + e.crosstabs.append(block) + elif isinstance(seg, DerivedSegment): + w(seg.verify.against_table, sheet).verified_by.append(seg) + if seg.keep_as is not None: + e = w(seg.keep_as.table, sheet) + e.kept_from.append(seg) + if block not in e.crosstabs: + e.crosstabs.append(block) + else: + w(block.table, sheet).lists.append(block) + total = block.rows.total_row + if total is not None and total.keep_as: + w(total.keep_as, sheet).list_totals.append(block) + return out + + +def _actual_sheet(sheet: SheetRecipe, extraction: Extraction) -> str: + return extraction.sheets.matched.get(sheet.id) or sheet.match.name + + +def _lineage_spans(extraction: Extraction, table: str, sheet: str) -> list[tuple[int, int]]: + """本表的值在这张工作表上占的行区间（来自溯源段）。判断隐藏行、停止后的行属于哪张表用。""" + spans: list[tuple[int, int]] = [] + for runs in (extraction.lineage.get(table) or {}).values(): + for run in runs: + try: + run_sheet, cell, count, direction = run[1], run[2], int(run[3]), run[4] + except (IndexError, TypeError, ValueError): + continue + parsed = split_coord(cell) + if run_sheet != sheet or parsed is None: + continue + row = parsed[2] + spans.append((row, row + count - 1) if direction == "down" else (row, row)) + return spans + + +# ---------------------------------------------------------------- 表说明的各个片段 + + +@dataclass +class _Ctx: + recipe: Recipe + extraction: Extraction + checks: dict[str, CheckResult] + accepted: set[str] + tables: dict[str, list[ColumnOut]] + grains: dict[str, list[str]] + dates: dict[str, set[str]] + writers: dict[str, _Writers] + problems: list[str] + plan: Any + + def check(self, table: str, cid: str | None, what: str) -> CheckResult | None: + c = self.checks.get(cid or "") + if c is None: + self.problems.append(f"表「{table}」：找不到{what}的核对结果，无法生成说明") + return c + + +def _blocking(c: CheckResult) -> bool: + """不可接受的结构类失败（K / G / T 不一致、核对 SQL 出错、落库不对）：试运行已经因此拒收、也提交不了。 + + 这时不写这一段说明、也不记 problems：problems 会被当成「配方本身不合法 / 说明生成失败」显示， + 把用户引去改配方，而真正的问题是那条核对。problems 只留给真正缺模板的情况。 + 不写比写错安全：缺了这一句，说明里就不会有「一致」。 + """ + return c.status == "mismatch" and c.category == "structure" and not c.acceptable + + +def _per_day(ctx: _Ctx, table: str) -> bool: + grain = ctx.grains.get(table, []) + return len(grain) == 1 and grain[0] in ctx.dates.get(table, set()) + + +def _grain_fragment(grain: list[str]) -> str | None: + cols = [_col(g) for g in grain] + if not cols: + return None + if len(cols) == 1: + return NOTE_TEXT["grain_1"].format(a=cols[0]) + if len(cols) == 2: + return NOTE_TEXT["grain_2"].format(a=cols[0], b=cols[1]) + return NOTE_TEXT["grain_n"].format(a=_LIST.join(cols[:-1]), b=cols[-1]) + + +def _sum_eq_fragment(ctx: _Ctx, table: str, rel: SumEq) -> str | None: + c = ctx.check(table, rel.id, f"关系「{rel.id}」") + if c is None or _blocking(c): + # 核对 SQL 出错时 status 也是 mismatch：不能当成「关系不成立」去写 + return None + per, unit = _PER[_per_day(ctx, table)] + total = _col(rel.total) + if c.status == "passed": + return NOTE_TEXT["sum_eq_passed"].format(total=total, parts=_join_and([_col(p) for p in rel.parts]), per=per) + parts = _LIST.join(_col(p) for p in rel.parts) + if c.status == "mismatch": + key = "sum_eq_mismatch_accepted" if rel.id in ctx.accepted else "sum_eq_mismatch_pending" + return NOTE_TEXT[key].format(total=total, parts=parts, unit=unit) + if c.status == "unverifiable": + key = "sum_eq_no_rows" if not c.checked and not c.unverifiable else "sum_eq_nulls" + return NOTE_TEXT[key].format(total=total, parts=parts, unit=unit) + ctx.problems.append(f"表「{table}」：关系「{rel.id}」的核对状态「{c.status}」没有对应的说明") + return None + + +def _not_comparable_fragment(ctx: _Ctx, table: str, rel: NotComparable) -> str | None: + c = ctx.check(table, rel.id, f"关系「{rel.id}」") + if c is None or _blocking(c): + return None + if c.status != "info": + ctx.problems.append(f"表「{table}」：关系「{rel.id}」的核对没有完成，无法生成说明") + return None + other = rel.b.table if table == rel.a.table else rel.a.table + if c.failed <= 0: + # 本期全部相等（或没有可比的分组）：不写「不等于」 + return NOTE_TEXT["not_comparable_equal"].format(other=_tab(other)) + dims = [g for g in ctx.grains.get(rel.a.table, []) if g != rel.by] + if dims: + return NOTE_TEXT["not_comparable_unequal"].format( + other=_tab(other), dims=_LIST.join(_col(d) for d in dims), value=_col(rel.b.value)) + return NOTE_TEXT["not_comparable_unequal_flat"].format( + other=_tab(other), a_value=_col(rel.a.value), by=_col(rel.by), value=_col(rel.b.value)) + + +def _dim_fragments(w: _Writers) -> list[str]: + out: list[str] = [] + for seg in w.dims: + if seg.dim.parser != "hour_range": + continue + roles = {role: name for name, role in seg.dim.derive.items()} + if "start" in roles and "end" in roles: + text = NOTE_TEXT["hour_range"].format(dim=_col(seg.dim.name), start=_col(roles["start"]), + end=_col(roles["end"])) + else: + text = NOTE_TEXT["hour_range_plain"].format(dim=_col(seg.dim.name)) + if text not in out: + out.append(text) + for seg in w.dims: + for name in sorted(seg.const): + text = NOTE_TEXT["const"].format(col=_col(name)) + if text not in out: + out.append(text) + return out + + +def _base_total_fragments(w: _Writers) -> list[str]: + """明细表上的提示：原表合计另存在哪、或者没导入。""" + out: list[str] = [] + kept = [s.keep_as.table for s in w.verified_by if s.keep_as is not None] + kept += [b.rows.total_row.keep_as for b in w.lists if b.rows.total_row is not None and b.rows.total_row.keep_as] + for t in kept: + text = NOTE_TEXT["total_kept"].format(table=_tab(t)) + if text not in out: + out.append(text) + dropped = any(s.keep_as is None for s in w.verified_by) or any( + b.rows.total_row is not None and not b.rows.total_row.keep_as for b in w.lists) + if dropped: + out.append(NOTE_TEXT["total_not_kept"]) + return out + + +def _total_key(checks: list[CheckResult], by_reason: dict[str, str], mixed: str, *, + g_checks: list[CheckResult] | tuple[CheckResult, ...] = (), passed: str, + passed_g: str | None = None) -> str | None: + """表内合计表按「状态 × 原因」选模板。返回 None 表示没有对应模板（调用方记 problems）。""" + statuses = {c.status for c in checks} | {c.status for c in g_checks} + if not checks or "mismatch" in statuses or statuses - {"passed", "unverifiable"}: + return None + if all(c.status == "passed" for c in checks): + if any(g.status == "unverifiable" for g in g_checks): + return passed_g + return passed + reasons = {r for c in checks if c.status == "unverifiable" for r, n in c.reasons.items() if n} + some_passed = any(c.status == "passed" for c in checks) + if len(reasons) == 1 and not some_passed: + return by_reason.get(next(iter(reasons))) + # 几种原因，或者一部分核对通过、一部分没有：用最弱的说法，不挑其中一种来写 + return mixed if reasons else None + + +def _kept_total_fragments(ctx: _Ctx, table: str, w: _Writers) -> list[str]: + out: list[str] = [] + if w.kept_from: + # 一张合计表可以由几个 derived 段写入（日间、夜间各有合计行）：所有段的 K、G 一起选一条模板 + ks = [ctx.check(table, ctx.plan.k.get(s.id), f"分段「{s.id}」的合计") for s in w.kept_from] + gs = [ctx.check(table, ctx.plan.g[s.id], f"分段「{s.id}」的公式引用") for s in w.kept_from if s.id in ctx.plan.g] + if None in ks or None in gs: + return out + if not any(_blocking(c) for c in ks + gs): + key = _total_key(ks, _K_REASON, "total_k_mixed", g_checks=gs, passed="total_k_passed", + passed_g="total_k_passed_g_unverifiable") + if key is None: + ctx.problems.append(f"表「{table}」：合计核对 {_ids(ks + gs)} 的结果没有对应的说明") + return out + out.append(NOTE_TEXT[key]) + for seg in w.kept_from: + text = NOTE_TEXT["total_overlap"].format(dim=_col(seg.keep_as.dim), base=_tab(seg.verify.against_table)) + if text not in out: + out.append(text) + for block in w.list_totals: + t = ctx.check(table, ctx.plan.t.get(block.id), f"块「{block.id}」的合计行") + if t is None or _blocking(t): + continue + key = _total_key([t], _T_REASON, "list_total_mixed", passed="list_total_passed") + if key is None: + ctx.problems.append(f"表「{table}」：合计核对 {t.id} 的结果没有对应的说明") + continue + blank_cols = _uncovered_total_columns(ctx, block) if key == "list_total_passed" else [] + if blank_cols: + text = NOTE_TEXT["list_total_passed_partial"].format( + cols=_LIST.join(_col(c) for c in blank_cols), base=_tab(block.table)) + else: + text = NOTE_TEXT[key].format(base=_tab(block.table)) + if text not in out: + out.append(text) + return out + + +def _uncovered_total_columns(ctx: _Ctx, block: ListBlock) -> list[str]: + """列表合计行里没有生成合计格的数字列（原表合计格是空的，P2-SPEC 4.4），按配方列序。 + + 这些列在合计表里存的是空值、T 也没核对过。按合计格算而不按库里的空值数算：4 参数调用（不带 null_counts） + 时照样成立。 + """ + covered = {it.base_value for it in ctx.plan.items.get(block.id, [])} + return [c.name for c in block.columns if c.type in ("INTEGER", "REAL") and c.name not in covered] + + +def _ids(checks: list[CheckResult]) -> str: + return "、".join(c.id for c in checks) + + +def _hidden_fragments(ctx: _Ctx, table: str, w: _Writers) -> list[str]: + excluded = included = False + for sheet in w.sheets: + actual = _actual_sheet(sheet, ctx.extraction) + info = ctx.extraction.hidden.get(actual) or {} + rows = [int(r) for r in info.get("rows") or []] + if not rows: + continue + policy = info.get("policy_rows") or sheet.hidden.rows + spans = _lineage_spans(ctx.extraction, table, actual) + if policy == "exclude": + # 排除的行不在溯源里：落在本表占的行范围之内就算本表排除了它们 + lo, hi = (min(a for a, _ in spans), max(b for _, b in spans)) if spans else (None, None) + excluded |= not spans or any(lo <= r <= hi for r in rows) + elif policy == "include": + included |= not spans or any(a <= r <= b for a, b in spans for r in rows) + out = [] + if excluded: + out.append(NOTE_TEXT["hidden_excluded"]) + if included: + out.append(NOTE_TEXT["hidden_included"]) + return out + + +def _rows_after_stop_tables(ctx: _Ctx) -> set[str]: + """rows_after_stop 问题落在哪张表上：同一工作表上被空行停下的列表块；不止一个时取紧挨在这些行之上的那个。""" + out: set[str] = set() + for p in ctx.extraction.problems: + if p.code != "rows_after_stop" or not p.cells: + continue + parsed = split_coord(p.cells[0]) + if parsed is None or parsed[0] is None: + continue + sheet_name, first_row = parsed[0], parsed[2] + blocks = [b for s in ctx.recipe.sheets if _actual_sheet(s, ctx.extraction) == sheet_name + for b in s.blocks if isinstance(b, ListBlock) and b.rows.blank_rows == "stop"] + if len(blocks) > 1: + above = [(max(b for _, b in spans), blk) for blk in blocks + if (spans := _lineage_spans(ctx.extraction, blk.table, sheet_name)) + and max(b for _, b in spans) < first_row] + if above: + blocks = [max(above, key=lambda x: x[0])[1]] + out.update(b.table for b in blocks) + return out + + +def _period_fragments(ctx: _Ctx, w: _Writers) -> list[str]: + if not any(s.context for s in w.sheets): + return [] + out: list[str] = [] + # 按传进来的核对结果判（run_checks 把 C1、C2 原样排在最前），与试运行、提交时存下的那份一致 + conflicts = [c for c in ctx.checks.values() + if c.kind in ("context_agree", "filename_period") and c.status == "mismatch"] + if conflicts: + accepted = all(c.id in ctx.accepted for c in conflicts) + out.append(NOTE_TEXT["period_conflict_accepted" if accepted else "period_conflict_pending"]) + if ctx.extraction.period is not None and ctx.extraction.period.source == "human": + out.append(NOTE_TEXT["period_human"]) + return out + + +def _table_template(ctx: _Ctx, table: str, after_stop: set[str]) -> str: + w = ctx.writers.get(table, _Writers()) + spec = next((t for t in ctx.recipe.tables if t.name == table), None) + frags: list[str | None] = [_grain_fragment(ctx.grains.get(table, []))] + for rel in ctx.recipe.relations: + if isinstance(rel, SumEq) and rel.table == table: + frags.append(_sum_eq_fragment(ctx, table, rel)) + for rel in ctx.recipe.relations: + if isinstance(rel, NotComparable) and table in (rel.a.table, rel.b.table): + frags.append(_not_comparable_fragment(ctx, table, rel)) + frags += _dim_fragments(w) + frags += _base_total_fragments(w) + frags += _kept_total_fragments(ctx, table, w) + frags += _hidden_fragments(ctx, table, w) + if any(b.extra_columns == "ignore" and ctx.extraction.ignored_columns.get(b.id) for b in w.lists): + frags.append(NOTE_TEXT["ignored_columns"]) + if table in after_stop: + frags.append(NOTE_TEXT["rows_after_stop"]) + frags += _period_fragments(ctx, w) + sheets = {_actual_sheet(s, ctx.extraction) for s in w.sheets} + if any(o.kind == "text_digits" and o.sheet in sheets for o in ctx.extraction.outside_text): + frags.append(NOTE_TEXT["outside_digits"]) + if spec is not None and spec.note.strip(): + frags.append(spec.note.strip()) + return "。".join(f.strip().rstrip("。") for f in frags if f and f.strip().rstrip("。")) + + +# ---------------------------------------------------------------- 列说明 + + +def _occurred(placeholders: list[Placeholder], extraction: Extraction) -> list[Placeholder]: + """本期真的出现过的占位符（按 canon 比对原文）。""" + seen = {canon(k) for k, n in extraction.placeholders.items() if n} + return [p for p in placeholders if canon(p.text) in seen] + + +def _null_fragment(w: _Writers, extraction: Extraction) -> str | None: + blocks: list[Any] = [*w.crosstabs, *w.lists] + marks = [p for b in blocks for p in _occurred(list(b.values.placeholders), extraction)] + blank = any(b.values.blank == "null" for b in blocks) + meanings = {p.meaning for p in marks} + meaning = next(iter(meanings)) if len(meanings) == 1 else _MEANING_BOTH + if marks and blank: + return COLUMN_TEXT["null_both"].format(meaning=meaning) + if marks: + return COLUMN_TEXT["null_placeholder"].format(meaning=meaning) + if blank: + return COLUMN_TEXT["null_blank"] + return None + + +def _year_from_period(ctx: _Ctx, w: _Writers) -> bool: + """轴的年份是不是从统计期补的：配了 year_from，而且本期表头不全是自带年份的日期格。""" + for block in w.crosstabs: + if block.axis.year_from is None: + continue + forms = [a.form for a in ctx.extraction.axes if a.block == block.id] + if not forms or any(f != "date" for f in forms): + return True + return False + + +def _column_templates(ctx: _Ctx, table: str, null_counts: Mapping[str, Mapping[str, int]] | None) -> dict[str, str]: + w = ctx.writers.get(table, _Writers()) + grain = set(ctx.grains.get(table, [])) + is_total = bool(w.kept_from or w.list_totals) + list_dates = {c.name for b in w.lists for c in b.columns if c.type == "DATE"} + counts = null_counts.get(table) if null_counts is not None else None + out: dict[str, str] = {} + for col in ctx.tables.get(table, []): + frags: list[str] = [] + if col.role == "axis": + frags.append(COLUMN_TEXT["date_year_from_period" if _year_from_period(ctx, w) else "date"]) + elif col.name in list_dates: + frags.append(COLUMN_TEXT["date"]) + if col.unit: + frags.append(COLUMN_TEXT["unit"].format(unit=_tok("单位", col.unit))) + # 值列的空值说明：主键不会是空值；表内合计表的空值已由表说明（未能核对）交代 + value_col = (col.role in ("measure", "value") and col.type in ("INTEGER", "REAL")) + has_nulls = counts is None or counts.get(col.name, 0) > 0 + if value_col and not is_total and col.name not in grain and has_nulls: + if (text := _null_fragment(w, ctx.extraction)) is not None: + frags.append(text) + if frags: + out[col.name] = "。".join(frags) + return out + + +# ---------------------------------------------------------------- 入口 + + +def build_notes(recipe: Recipe, extraction: Extraction, checks: list[CheckResult], + acceptances: list[Acceptance], *, + null_counts: Mapping[str, Mapping[str, int]] | None = None) -> SchemaNotes: + """按核对结果生成全部表说明、列说明。problems 非空就不能保存（试运行判 recipe 类问题，提交拒绝）。 + + acceptances：写了理由接受的核对。试运行回执的预览传空列表（按「全部未接受」生成），提交时传实际的接受。 + null_counts（recipe_checks.column_null_counts 的结果）：给了就只对本期真有空值的值列写空值说明。 + 不给时只能按配方推断——Extraction.placeholders 是整份文件的计数、不分列，于是配了占位符且本期出现过、 + 或空格存空值的块里，每个值列都写（宁可多写一句「空值表示……」，不能漏写）。 + **试运行预览和提交都应传 null_counts**：参考配方（D00）的 全日客流 列说明要是「单位：人次」 + （P2-SPEC 7.4、9.7 第 2 步），不传就会多出空值那一句。 + """ + tables, shape_problems = derive_tables(recipe) + problems = [f"配方推出的表结构有问题：{p.message}" for p in shape_problems] + ctx = _Ctx( + recipe=recipe, extraction=extraction, checks={c.id: c for c in checks}, + accepted={a.check_id for a in acceptances}, tables=tables, + grains={t.name: list(t.grain) for t in recipe.tables}, dates=date_columns(recipe), + writers=_writers(recipe), problems=problems, plan=plan_checks(recipe, extraction), + ) + # 只给真正建出来的表写说明（derive_tables 推出的表）；声明了却没有分段写入的表不在库里，写了也落不了地 + names = list(tables) + known = {"列": {c.name for cols in tables.values() for c in cols}, + "表": set(names) | {t.name for t in recipe.tables}, "单位": set(UNITS)} + after_stop = _rows_after_stop_tables(ctx) + notes = SchemaNotes(problems=problems) + notes.kinds = {t.name: t.kind for t in recipe.tables if t.kind == "reported_total" and t.name in tables} + for table in names: + template = _table_template(ctx, table, after_stop) + columns = _column_templates(ctx, table, null_counts) + for p in prose_problems(template, known=known): + problems.append(f"表「{table}」的说明：{p}") + for col, tpl in columns.items(): + for p in prose_problems(tpl, known=known): + problems.append(f"表「{table}」列「{col}」的说明：{p}") + if template: + notes.templates[table] = template + notes.templates.update({f"{table}.{col}": tpl for col, tpl in columns.items()}) + notes.tables[table] = TableNote(comment=render_prose(template), + columns={col: render_prose(tpl) for col, tpl in columns.items()}) + return notes + + +def apply_notes(schema_cache: dict[str, Any], notes: SchemaNotes) -> dict[str, Any]: + """把说明写进表结构，返回新 dict（不改入参）：表 comment、列 comment，顶层加 import_mode=recipe。 + + 表或列在 schema_cache 里找不到（不该发生：说明和库出自同一份配方）时记进 notes.problems，不悄悄跳过。 + 表说明为空时保留原来的 comment。 + """ + out = copy.deepcopy(schema_cache) if schema_cache else {} + tables = out.get("tables") + if not isinstance(tables, dict): + tables = {} + for name, kind in notes.kinds.items(): + # 表的种类记进表结构：说明（不要相加）只有 db_schema 查单表时看得到，工具描述和写作目录靠它点名（AU-5） + if isinstance(tables.get(name), dict): + tables[name]["kind"] = kind + for name, note in notes.tables.items(): + meta = tables.get(name) + if not isinstance(meta, dict): + notes.problems.append(f"表「{name}」不在表结构中，说明没有写入") + continue + if note.comment: + meta["comment"] = note.comment + columns = {c.get("name"): c for c in meta.get("columns") or [] if isinstance(c, dict)} + for col, text in note.columns.items(): + target = columns.get(col) + if target is None: + notes.problems.append(f"表「{name}」没有列「{col}」，说明没有写入") + continue + target["comment"] = text + out["import_mode"] = "recipe" + return out diff --git a/backend/app/data/recipe_parsers.py b/backend/app/data/recipe_parsers.py new file mode 100644 index 00000000..d6152ef9 --- /dev/null +++ b/backend/app/data/recipe_parsers.py @@ -0,0 +1,442 @@ +"""配方的固定解析器（期 2 契约文件，波次 0 由编排者提交，实现者不改签名）。 + +**为什么是固定集合。** 配方里没有正则字段（ReDoS、模型写错正则都会静默改数据）。会变的写法 +（各类横线、HH:MM、「时」「点」后缀、全角、第二个日期省略年份、日期格）都在这里宽容归一， +规范写法统一；认不出的一律返回 None，由执行器判成结构类问题，绝不猜。新增一种写法要改代码、 +升 PARSER_VER（进构建 id），并对历史原件重放回归。 + +**输入限长。** 标签 ≤ LABEL_MAX，上下文 ≤ CONTEXT_MAX：超长的直接 None，正则只跑在有界输入上。 +所有正则都用 re.ASCII：\\d 只认 ASCII 数字（全角数字先经 canon 的 NFKC 变成 ASCII，别的文字的数字不认）。 +""" +from __future__ import annotations + +import datetime as _dt +import re +from dataclasses import dataclass +from typing import Any, Literal + +from app.data.names import canon + +#: 解析器版本：进构建 id（table_versions.recipe_build_id）。解析规则改了就要升 +PARSER_VER = "parsers/2" +#: 标签（行标签、表头、分段标题、合计标签）长度上限 +LABEL_MAX = 40 +#: 上下文格（统计期所在的格、文件名）长度上限 +CONTEXT_MAX = 80 +#: 单位封闭词表。新增单位要改代码（说明里由系统渲染「单位：万元」，数字检查时遮盖） +UNITS: tuple[str, ...] = ("人次", "人", "元", "万元", "千元", "亿元", "千克", "吨", "个", "件", "户", "%") +#: 列表合计行标签的候选词（起草器据此认合计行；配方里的 pick 必须取自实际格子文字的候选词） +TOTAL_WORDS: tuple[str, ...] = ("合计", "总计", "小计", "汇总", "累计") +#: 期 2 认得的解析器名（配方 JSON 里出现的取值，recipe_types 的 Literal 与此一致） +LABEL_PARSERS = ("hour_range", "hour_range_total", "text") +AXIS_PARSERS = ("month_day_or_date",) +CONTEXT_PARSERS = ("cn_date_range",) + +_WS = re.compile(r"\s+") + + +def match_key(text: Any) -> str: + """锚点、分段标题、标签集合比对用的键：canon 之后去掉全部空白。 + + 「金 额」「销量\\n」「地区　」和「金额」「销量」「地区」是同一个表头。存储用 canon（保留单个空格）， + 比对用这个；两者不同的写法进清单的 canonicalized，并在差异卡里报「写法变化」。 + """ + return _WS.sub("", canon("" if text is None else str(text))) + + +# -------------------------------------------------------------------------- +# 结果 +# -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class HourRange: + start: int + end: int + #: 规范写法：「7-8」（不带单位） + canonical: str + + +@dataclass(frozen=True) +class HourTotal: + start: int + end: int + #: 合计 / 小计 / 总计 + word: str + #: 规范写法：「18-22时合计」 + canonical: str + + +@dataclass(frozen=True) +class Period: + start: _dt.date + end: _dt.date + + @property + def days(self) -> int: + return (self.end - self.start).days + 1 + + def iso(self) -> tuple[str, str]: + return self.start.isoformat(), self.end.isoformat() + + +@dataclass(frozen=True) +class MonthMention: + """只写到月的「2026年9月」：只能作旁证（统计期须落在这个月内才算一致），不能单独定统计期。""" + + year: int + month: int + + def contains(self, period: Period) -> bool: + return (period.start.year, period.start.month) == (self.year, self.month) == ( + period.end.year, period.end.month) + + +@dataclass(frozen=True) +class AxisDate: + """日期轴上的一格：「8月1日」（year=None，form=text）、「2026-08-01」（text）、日期格（form=date）。""" + + year: int | None + month: int + day: int + form: Literal["text", "date"] + + +# -------------------------------------------------------------------------- +# 时段 +# -------------------------------------------------------------------------- + +_H = r"(\d{1,2})(?::00)?\s*[时点]?" +_HOUR = re.compile(rf"{_H}\s*-\s*{_H}\s*[时点]?", re.ASCII) +_HOUR_TOTAL = re.compile(rf"{_H}\s*-\s*{_H}\s*[时点]?\s*(合计|小计|总计)", re.ASCII) + + +def _bounded(text: Any, limit: int) -> str | None: + if not isinstance(text, str) or len(text) > limit: + return None + return canon(text) + + +def hour_range(text: Any) -> HourRange | None: + """「7-8」「7–8」「07:00-08:00」「7时-8时」「7点-8点」「7-8时」「７－８」→ HourRange(7, 8, "7-8")。 + + 拒收：起点不小于终点（「8-7」）、超出 0..24（「25-26」）、半点（「7:30-8:00」）、带合计字样、非文本。 + """ + s = _bounded(text, LABEL_MAX) + m = _HOUR.fullmatch(s) if s is not None else None + if m is None: + return None + a, b = int(m.group(1)), int(m.group(2)) + return HourRange(a, b, f"{a}-{b}") if 0 <= a < b <= 24 else None + + +def hour_range_total(text: Any) -> HourTotal | None: + """「18-22 时合计」「18:00-22:00合计」「18-22时小计」「18－24 时合计」→ HourTotal(18, 22, "合计", "18-22时合计")。""" + s = _bounded(text, LABEL_MAX) + m = _HOUR_TOTAL.fullmatch(s) if s is not None else None + if m is None: + return None + a, b = int(m.group(1)), int(m.group(2)) + return HourTotal(a, b, m.group(3), f"{a}-{b}时{m.group(3)}") if 0 <= a < b <= 24 else None + + +def text_label(text: Any) -> str | None: + """不解析的标签（parser="text"）：规范写法就是 canon，空的、超长的、非文本的 None。""" + s = _bounded(text, LABEL_MAX) + return s or None + + +# -------------------------------------------------------------------------- +# 日期区间（统计期） +# -------------------------------------------------------------------------- + +_CN_RANGE = re.compile( + r"(\d{4})\s*年\s*(\d{1,2})\s*月\s*(\d{1,2})\s*日\s*(?:至|到|~|-)\s*" + r"(?:(\d{4})\s*年\s*)?(\d{1,2})\s*月\s*(\d{1,2})\s*日", + re.ASCII) +_ISO_RANGE = re.compile( + r"(? Period | None: + try: + a, b = _dt.date(y1, m1, d1), _dt.date(y2, m2, d2) + except ValueError: + return None + if a > b or (b - a).days + 1 > PERIOD_MAX_DAYS: + return None + return Period(a, b) + + +def _ranges(s: str) -> list[Period | None]: + found: list[Period | None] = [] + for m in _CN_RANGE.finditer(s): + y1, m1, d1 = int(m.group(1)), int(m.group(2)), int(m.group(3)) + m2, d2 = int(m.group(5)), int(m.group(6)) + # 第二个日期省略年份：不早于第一个日期就是同年，否则跨年（12-15 至 01-14） + y2 = int(m.group(4)) if m.group(4) else (y1 if (m2, d2) >= (m1, d1) else y1 + 1) + found.append(_period(y1, m1, d1, y2, m2, d2)) + for m in _ISO_RANGE.finditer(s): + g = [int(x) for x in m.groups()] + found.append(_period(*g)) + return found + + +def cn_date_range(text: Any) -> Period | None: + """格子文字里的统计期：「统计时间范围：2026年9月1日至2026年9月30日」「…至9月30日」「（2026年9月1日至…）」。 + + 在文字中**查找**（不要求整格只有区间，标题里带区间也认），但必须恰好一处、日期合法、起点不晚于终点； + 零处、多处或日期不合法都返回 None。ISO 写法「2026-09-01至2026-09-30」同样认（分隔只认 至/到/~/_， + 「-」和日期内部的横线分不开）。 + """ + s = _bounded(text, CONTEXT_MAX) + if s is None: + return None + found = _ranges(s) + return found[0] if len(found) == 1 else None + + +def cn_year_month(text: Any) -> MonthMention | None: + """「（2026年9月）」这类只写到月的提法；文字里同时有完整区间的不算（那由 cn_date_range 认）。恰好一处才返回。""" + s = _bounded(text, CONTEXT_MAX) + if s is None or _ranges(s): + return None + hits = list(_CN_MONTH.finditer(s)) + if len(hits) != 1: + return None + y, mo = int(hits[0].group(1)), int(hits[0].group(2)) + return MonthMention(y, mo) if 1 <= mo <= 12 else None + + +def filename_period(name: str) -> Period | None: + """文件名里的区间（旁证）：「月报导出_2026-08-01_2026-08-31.xlsx」或中文写法。只看文件名本身，不看路径。""" + base = str(name or "").replace("\\", "/").rsplit("/", 1)[-1] + stem = base.rsplit(".", 1)[0] if "." in base else base + s = _bounded(stem, CONTEXT_MAX) + if s is None: + return None + found = _ranges(s) + return found[0] if len(found) == 1 else None + + +#: 统计期格里「只是在说统计期」的词：去掉区间之后只剩这些（或配方的 prefer_prefix），这格才只算统计期来源 +PERIOD_WORDS: tuple[str, ...] = ("统计时间范围", "统计时间", "统计期", "统计周期", "时间范围", "日期范围", + "报告期", "期间", "日期", "时间") +_RESIDUE_STRIP = re.compile(r"[\s:：()\[\]【】<>《》,，。.、;；~_\-至到]+") + + +def period_residue(text: Any) -> str | None: + """统计期格去掉区间（或「2026年9月」这类年月）之后剩下的文字，去标点、去空白；认不出统计期返回 None。 + + 用来区分两种格子（D15、E7）：「统计时间范围：2026年9月1日至…」去掉区间只剩「统计时间范围」，只是在说 + 统计期；「注：2026年9月数据为初步统计，待修订」剩「注数据为初步统计待修订」，是带着统计期的说明文字—— + 后者除了参与统计期一致性判断，还要按含数字的区域外文字处理（需确认，说明里加提示）。判定见 + is_pure_period_cell。 + """ + s = _bounded(text, CONTEXT_MAX) + if s is None: + return None + spans = [m.span() for m in _CN_RANGE.finditer(s)] + [m.span() for m in _ISO_RANGE.finditer(s)] + if not spans: + spans = [m.span() for m in _CN_MONTH.finditer(s)] + if len(spans) != 1: + return None + a, b = spans[0] + return _RESIDUE_STRIP.sub("", s[:a] + " " + s[b:]) + + +def is_pure_period_cell(text: Any, prefer_prefix: str | None = None) -> bool: + """去掉区间后什么都不剩、只剩 prefer_prefix、或只剩 PERIOD_WORDS 里的词：这格只是统计期来源。""" + rest = period_residue(text) + if rest is None: + return False + allowed = {"", *(match_key(w) for w in PERIOD_WORDS)} + if prefer_prefix: + allowed.add(_RESIDUE_STRIP.sub("", canon(prefer_prefix))) + return rest in allowed + + +# -------------------------------------------------------------------------- +# 日期轴 +# -------------------------------------------------------------------------- + +_MD = re.compile(r"(\d{1,2})月(\d{1,2})日", re.ASCII) +_YMD_CN = re.compile(r"(\d{4})年(\d{1,2})月(\d{1,2})日", re.ASCII) +_YMD_ISO = re.compile(r"(\d{4})[-/](\d{1,2})[-/](\d{1,2})", re.ASCII) + + +def month_day_or_date(value: Any) -> AxisDate | None: + """日期轴的一格。认：「8月1日」、「2026年8月1日」、「2026-08-01」「2026/8/1」文本，以及日期格（零点的 datetime 或 date）。 + + 不认：带时刻的 datetime、数字（没有日期格式的序列号）、「8/1」（中美写法分不清）。月日的合法性在补年份时再查。 + """ + if isinstance(value, _dt.datetime): + if (value.hour, value.minute, value.second, value.microsecond) != (0, 0, 0, 0): + return None + return AxisDate(value.year, value.month, value.day, "date") + if isinstance(value, _dt.date): + return AxisDate(value.year, value.month, value.day, "date") + s = _bounded(value, LABEL_MAX) + if not s: + return None + s = _WS.sub("", s) + if m := _MD.fullmatch(s): + mo, d = int(m.group(1)), int(m.group(2)) + return AxisDate(None, mo, d, "text") if 1 <= mo <= 12 and 1 <= d <= 31 else None + for rx in (_YMD_CN, _YMD_ISO): + if m := rx.fullmatch(s): + y, mo, d = (int(x) for x in m.groups()) + try: + _dt.date(y, mo, d) + except ValueError: + return None + return AxisDate(y, mo, d, "text") + return None + + +def complete_year(cell: AxisDate, period: Period) -> _dt.date | None: + """给轴上的一格定日期。自带年份的原样（必须落在统计期内）；只有月日的，在统计期覆盖的年份里找唯一解。""" + if cell.year is not None: + try: + d = _dt.date(cell.year, cell.month, cell.day) + except ValueError: + return None + return d if period.start <= d <= period.end else None + hits = [] + for y in range(period.start.year, period.end.year + 1): + try: + d = _dt.date(y, cell.month, cell.day) + except ValueError: + continue + if period.start <= d <= period.end: + hits.append(d) + return hits[0] if len(hits) == 1 else None + + +# -------------------------------------------------------------------------- +# 单位、数字、区域外格子的分类 +# -------------------------------------------------------------------------- + +_UNIT_SUFFIX = re.compile(r"(.+?)\s*\(\s*([^()]{1,8}?)\s*\)") + + +def split_unit_suffix(label: Any) -> tuple[str, str | None]: + """「全日客流（人次）」→ ("全日客流", "人次")；没有括号后缀 → (canon 原文, None)。 + + 只拆末尾一对括号（canon 之后全角括号已是半角）。括号里的字不在 UNITS 里也照样拆出来返回， + 由调用方判「单位不在词表中」——不能因为认不出就把它留在列名里。 + """ + s = canon("" if label is None else str(label)) + m = _UNIT_SUFFIX.fullmatch(s) + if m is None: + return s, None + return m.group(1).strip(), m.group(2) + + +def unit_known(unit: str | None) -> bool: + return unit in UNITS + + +_PLAIN_NUMBER = re.compile(r"[-+]?\d+(?:\.\d+)?", re.ASCII) +_THOUSANDS = re.compile(r"-?[1-9]\d{0,2}(?:,\d{3})+(?:\.\d+)?", re.ASCII) +#: 数字后面跟的汉字后缀里不许出现的日期、期间字。「50万」「12万人次」「3天」是数加量级或单位； +#: 「2026年上半年」「2026年度」「9月」「15日」「7时」「第3期」是日期或期间的写法，是文字（含数字）， +#: 不是一个数：区域外按 text_digits 只需确认，列表表头里的「2026年上半年」也照样算表头文字。 +_DATE_CHARS = "年月日号季期周旬时点" +_NUMBERISH = re.compile( + r"[¥$€£]?\s*[-+]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?\s*" + r"(?:%|‰|(?:(?![" + _DATE_CHARS + r"])[一-鿿]){1,4}|[A-Za-z]{1,4})?") + + +def thousands_number(text: Any) -> int | float | None: + """千分位写法的数字文本（「1,234」「12,345.5」）→ 数；别的写法 None。只在配方选了 parse_thousands 时用。""" + s = _bounded(text, LABEL_MAX) + if s is None or not _THOUSANDS.fullmatch(s): + return None + plain = s.replace(",", "") + return float(plain) if "." in plain else int(plain) + + +def looks_numeric_text(text: Any) -> bool: + """看起来是一个数的文本：「123」「1,234」「12.5%」「¥1234」「50万」「12kg」。区域外遇到这种格按「区域外数字」拒收。 + + 数后面的汉字后缀只认量级、单位一类（不含 _DATE_CHARS）：「2026年上半年」「2026年度」「9月」不是数。 + """ + s = _bounded(text, LABEL_MAX) + return bool(s) and _NUMBERISH.fullmatch(s) is not None + + +OutsideKind = Literal["number", "text_digits", "text"] + + +def classify_outside(value: Any) -> OutsideKind: + """区域外的一格：number（数字格、像数的文本）→ 拒收；text_digits（含数字的文字、日期格）→ 需确认；text → 记录。""" + if isinstance(value, bool): + return "text" + if isinstance(value, (int, float)): + return "number" + if isinstance(value, (_dt.datetime, _dt.date, _dt.time)): + return "text_digits" + s = canon("" if value is None else str(value)) + if looks_numeric_text(s): + return "number" + return "text_digits" if any(ch.isdigit() for ch in s) else "text" + + +# -------------------------------------------------------------------------- +# 常量的候选词 +# -------------------------------------------------------------------------- + +_SPLIT = re.compile(r"[\s、，,/·\-_:：;；|()]+") +CANDIDATE_MAX = 12 + + +def _strip_unit(title: str) -> str: + return split_unit_suffix(title)[0] + + +def candidate_words(title: str, siblings: tuple[str, ...] | list[str] = ()) -> list[str]: + """分段标题里可以拿来当常量的词。配方里的 const.pick 必须是其中之一（保存时静态校验）； + 重放时只查「pick 仍是新标题的子串」，不在就判结构类问题。 + + 取法（都在 canon 之后、去掉末尾单位括号之后）： + 1. 有兄弟标题（写进同一张表的其他分段）时，去掉所有标题的公共前缀和公共后缀，剩下的中段。公共部分要收缩到 + 每个中段至少 2 个字：「日间时段客流」「夜间时段客流」的公共后缀按字算是「间时段客流」，收缩成「时段客流」， + 中段才是「日间」「夜间」； + 2. 按分隔符（空白、顿号、逗号、斜杠、间隔号、横线、下划线、冒号、分号、竖线、括号）切开的各段； + 3. 整个标题。 + 去重保序；含数字的不要（配方里不能写死年份、期次）；超过 CANDIDATE_MAX 个字的不要。 + """ + base = _strip_unit(title) + group = [base] + [_strip_unit(s) for s in siblings if _strip_unit(s) != base] + out: list[str] = [] + if len(group) > 1: + pre = 0 + while all(len(g) > pre for g in group) and len({g[pre] for g in group}) == 1: + pre += 1 + suf = 0 + while all(len(g) - pre > suf for g in group) and len({g[len(g) - 1 - suf] for g in group}) == 1: + suf += 1 + # 先收缩后缀、再收缩前缀，直到每个中段至少 2 个字；再补上「不去后缀」「不去前缀」两种切法， + # 中文没有词边界，「工作日 早高峰 / 工作日 晚高峰」收缩后是「早高」，不去后缀才是「早高峰」 + p, q = pre, suf + while (p or q) and any(len(g) - p - q < 2 for g in group): + if q: + q -= 1 + else: + p -= 1 + for a, b in ((p, q), (p, 0), (0, q)): + middle = base[a:len(base) - b].strip() + if len(middle) >= 2 or (middle and len(base) < 2): + out.append(middle) + out.extend(p for p in _SPLIT.split(base) if p) + out.append(base) + seen: list[str] = [] + for w in out: + w = w.strip() + if w and w not in seen and len(w) <= CANDIDATE_MAX and not any(ch.isdigit() for ch in w): + seen.append(w) + return seen diff --git a/backend/app/data/recipe_suggest.py b/backend/app/data/recipe_suggest.py new file mode 100644 index 00000000..51ae5274 --- /dev/null +++ b/backend/app/data/recipe_suggest.py @@ -0,0 +1,2474 @@ +"""确定性起草（期 2，WP-4）：从扫描事实和网格推出一份配方草稿、系统发现的关系、建议卡片和待确认问题。 + +**所有产物只是草稿**（H3）。起草器只读格子、不做任何判定性的导入：它写出的配方要经静态校验（注入的 +validate）和执行器干跑（注入的 dry_run），再由人逐条确认之后才能启用。所以这里宁可认不出就说认不出 +（complete=False，failures 写人话原因），也不猜。 + +**为什么 detect_facts 只给关系不给数值。** 「第 5 行 = 第 6 行 + 第 7 行」这种事实要交给用户确认、交给 AI +起草参考；数值本身不需要离开表格，系统也不该替用户判断某个数「对不对」——对不对由每期的 SQL 核对说了算。 + +**为什么 questions_for 吃任意配方。** 规则草稿、AI 草稿、用户在面板里改过的配方都要给同一套卡片和问题， +接口层每次回答、PUT、AI 草稿之后都重算一遍。给了最近一次干跑（extraction）时，区域类卡片（统计期取自哪格、 +哪些是区域外文字、哪些格没有去处）直接用执行器的结论：同一件事只有执行器一套判法，这里自己定位只是没有 +干跑时的近似。 + +**问题的 effects 怎么写。** 每个选项是一组 JSON Patch，路径按传进来的配方写；接口层从 answers_base 起、按 +问题顺序重放全部回答（P2-SPEC 7.2）。因此： +- 关系的两个选项都用整条 replace（「登记」写回完整的 sum_eq / not_comparable，「不登记」写 dismissed）， + 与当前状态无关，先选 A 再改选 B 等于直接选 B； +- 占位符的「不是」用 remove，同一块里有多个占位符时问题按下标从大到小排列，按问题顺序重放不会错位； +- 改某个字段用 add（对象成员上 add 等于覆盖），父对象缺失时（传进来的是去掉默认值的紧凑形式）才连父对象 + 一起建。起草器自己产出的配方一律是补全默认值的完整形式，父对象都在。 +""" +from __future__ import annotations + +import asyncio +import datetime as _dt +import inspect +import itertools +import logging +import os +import re +import traceback +from dataclasses import dataclass, field +from typing import Any, Awaitable, Callable + +from pydantic import ValidationError + +from app.data import recipe_parsers as P +from app.data.names import canon, collide_key, to_sql_name +from app.data.recipe_types import ( + RECIPE_FORMAT, + REASON_SLOT, + Card, + CrosstabBlock, + DerivedSegment, + DimensionSegment, + Draft, + DraftFacts, + Extraction, + Fact, + Grid, + GridCell, + ListBlock, + MeasuresSegment, + Problem, + Question, + Recipe, + RecipeProblem, +) +from app.data.tabular import DATE_HEADER_MIN, EXCEL_ERRORS +from app.data.xlsx_scan import WorkbookScan, cell_ref, col_letter, parse_ref + +logger = logging.getLogger(__name__) + +ValidateFn = Callable[[dict[str, Any], DraftFacts | None, str], tuple[Recipe | None, list[RecipeProblem]]] +#: 第三个参数是 origin（"rules" / "ai"）：draft 传 "rules"，ai_draft 传 "ai"——ai_note_forbidden 靠它触发 +DryRunFn = Callable[[Recipe], Extraction] +#: AI 起草（recipe_ai.ai_draft）用的干跑可以是协程函数：WP-5c 注入的实现占一个解析名额、在线程池里跑执行器。 +#: 一次干跑可能要几秒，绝不能同步跑在事件循环上（会堵住同进程的所有请求）；见 run_dry_async +AsyncDryRunFn = Callable[[Recipe], Awaitable[Extraction]] +#: WP-5c 注入：网格路径的工作表完整干跑，超过 GRID_MAX_CELLS 的工作表 execute(max_rows=500) 部分干跑（4.9） + +#: 起草时每张工作表最多读多少行（接口层 read_grid 的 max_rows） +DRAFT_MAX_ROWS = 500 +#: 列表的第一个表头只在前这么多行里找（P2-SPEC 6.1 第 11 条） +LIST_HEADER_SCAN_ROWS = 50 +#: 卡片最多指向多少个格子（界面着色用，再多也看不过来） +CARD_CELLS_MAX = 40 +#: sum_eq 事实：合计行等于另外几行之和，最多几行 +FACT_PARTS_MIN, FACT_PARTS_MAX = 2, 4 +#: 组合数会爆炸：指标段超过这么多行时只找「两行之和」，超过 FACT_SEARCH_MAX_ROWS 行时不找 sum_eq +#: （指标段一般只有几行；几十行的指标段多半是认错了版式，起草本来也不会完整） +FACT_DEEP_MAX_ROWS = 20 +FACT_SEARCH_MAX_ROWS = 60 +#: 回灌给模型的问题最多几条（P2-SPEC 6.2） +MODEL_LINES_MAX = 20 + +#: 给模型看的问题行（起草失败原因、AI 修订回灌）：静态校验 = 路径 + 消息；干跑 = code + model_message + 坐标 +MODEL_LINE_PROMPT = {"static": "{path}：{message}", "dry": "{code}", "dry_msg": "{code}：{message}", + "cells": "{line}（{cells}）"} + +_HOUR_DIM, _TEXT_DIM = "时段", "项目" +_DERIVE = {"起始小时": "start", "结束小时": "end"} +_TOTAL_HINT = ("合计", "小计", "总计") +_YEAR = re.compile(r"(? bool: + return isinstance(v, (int, float)) and not isinstance(v, bool) + + +def _int_like(v: Any) -> bool: + return isinstance(v, int) or (isinstance(v, float) and v.is_integer()) + + +def _text(v: Any) -> str | None: + """格子里的文字（去首尾空白）；不是文字或只有空白时 None。""" + if isinstance(v, str): + s = v.strip() + return s or None + return None + + +def _shown(v: Any) -> str: + """界面上显示一个格子的值（只给人看）。""" + if isinstance(v, _dt.datetime): + return v.date().isoformat() if (v.hour, v.minute, v.second) == (0, 0, 0) else v.isoformat(sep=" ") + if isinstance(v, _dt.date): + return v.isoformat() + return str(v).strip() + + +def _placeholder_like(text: str) -> bool: + """占位符候选：canon 后不超过 2 个字、没有字母数字、不像数（「·」「-」「/」「—」）。""" + s = canon(text) + return 0 < len(s) <= 2 and not any(ch.isalnum() for ch in s) and not P.looks_numeric_text(s) + + +def _is_error(v: Any) -> bool: + return isinstance(v, str) and v.strip().upper() in EXCEL_ERRORS + + +def _total_word(text: Any) -> str | None: + """列表合计行的标签：match_key 后以 TOTAL_WORDS 之一开头时返回那个词。""" + if not isinstance(text, str): + return None + key = P.match_key(text) + return next((w for w in P.TOTAL_WORDS if key.startswith(w)), None) + + +def _a1(r: int, c: int) -> str: + return cell_ref(r, c) + + +def _coord(sheet: str, r: int, c: int) -> str: + return f"{sheet}!{cell_ref(r, c)}" + + +def _coords(sheet: str, cells: list[tuple[int, int]]) -> list[str]: + return [_coord(sheet, r, c) for r, c in sorted(set(cells))[:CARD_CELLS_MAX]] + + +def _rows_text(rows: list[int]) -> str: + rows = sorted(set(rows)) + if not rows: + return "" + if len(rows) == 1: + return f"第 {rows[0]} 行" + if rows[-1] - rows[0] + 1 == len(rows): + return f"第 {rows[0]}–{rows[-1]} 行" + return "第 " + "、".join(str(r) for r in rows[:8]) + (" 等" if len(rows) > 8 else "") + " 行" + + +def _a1_list(cells: list[tuple[int, int]], limit: int = 5) -> str: + cells = sorted(set(cells)) + head = "、".join(_a1(r, c) for r, c in cells[:limit]) + return head + (f" 等 {len(cells)} 格" if len(cells) > limit else "") + + +def _ptr(*parts: Any) -> str: + return "".join("/" + str(p).replace("~", "~0").replace("/", "~1") for p in parts) + + +def _bottom(grid: Grid) -> int: + rows = grid.row_numbers() + return rows[-1] if rows else 0 + + +def _cell_kind(cell: GridCell | None) -> str: + """数值区里的一格：blank / formula / num / ph（占位符候选）/ error / text / other。""" + if cell is None: + return "blank" + if cell.formula is not None: + return "formula" + v = cell.value + if _num(v): + return "num" + if isinstance(v, str): + if _is_error(v): + return "error" + return "ph" if _placeholder_like(v) else "text" + return "other" + + +def _value_num(cell: GridCell | None) -> int | float | None: + """格子的数（公式取保存值）；不是数时 None。""" + if cell is None: + return None + return cell.value if _num(cell.value) else None + + +def _equal(a: float, b: float) -> bool: + if _int_like(a) and _int_like(b): + return a == b + return abs(a - b) <= 1e-9 * max(1.0, abs(a), abs(b)) + + +def _unique(name: str, used: set[str], *, limit: int = 48) -> str: + """名字撞了（collide_key）就加 _2、_3……；超长截断。""" + base = name[:limit] + out, n = base, 2 + while collide_key(out) in used: + tail = f"_{n}" + out = base[: limit - len(tail)] + tail + n += 1 + used.add(collide_key(out)) + return out + + +# ========================================================================== +# 版式分析（起草和压缩表示共用） +# ========================================================================== + + +def _axis_row(grid: Grid) -> tuple[int, list[int]] | None: + """日期表头行：至少 DATE_HEADER_MIN 格能被 month_day_or_date 解析，取个数最多的那行（并列取最上面）。""" + best: tuple[int, list[int]] | None = None + for r in grid.row_numbers(): + cols = [c for c, cell in grid.row(r) if P.month_day_or_date(cell.value) is not None] + if len(cols) >= DATE_HEADER_MIN and (best is None or len(cols) > len(best[1])): + best = (r, cols) + return best + + +def _header_texts(grid: Grid, rows: list[int], cols: list[int]) -> dict[int, str]: + """多行表头每列的文字：合并区内取左上格（左上格要在这几行里），去掉空的和与上一层相同的，用「_」拼接。""" + tops: dict[tuple[int, int], tuple[int, int]] = {} + lo, hi = rows[0], rows[-1] + for (r1, c1, r2, c2) in grid.merges: + if lo <= r1 <= hi: + for rr in range(r1, min(r2, hi) + 1): + for cc in range(c1, c2 + 1): + tops[(rr, cc)] = (r1, c1) + out: dict[int, str] = {} + for c in cols: + parts: list[str] = [] + for r in rows: + src = tops.get((r, c), (r, c)) + cell = grid.get(*src) + t = _text(cell.value) if cell is not None else None + if t and (not parts or parts[-1] != t): + parts.append(t) + if parts: + out[c] = "_".join(parts) + return out + + +def _header_like(grid: Grid, r: int) -> bool: + """表头行：≥2 个非空格、全是文本（不像数、不以冒号结尾）且互不相同，下面 3 行里至少有一行有数字。 + + 冒号结尾的是表单的标签（「项目名称：」），不是表头：键值表单不能被认成列表。 + """ + cells = grid.row(r) + if len(cells) < 2: + return False + keys: set[str] = set() + for _, cell in cells: + if cell.formula is not None or not isinstance(cell.value, str): + return False + s = canon(cell.value) + if not s or P.looks_numeric_text(s) or s.endswith(":"): + return False + k = P.match_key(s) + if k in keys: + return False + keys.add(k) + lo, hi = cells[0][0], cells[-1][0] + return any(any(_num(cell.value) for c, cell in grid.row(rr) if lo <= c <= hi) for rr in range(r + 1, r + 4)) + + +def _all_text_row(grid: Grid, r: int) -> bool: + cells = grid.row(r) + return len(cells) >= 2 and all(isinstance(cell.value, str) and cell.formula is None + and not P.looks_numeric_text(cell.value) for _, cell in cells) + + +def _header_rows_at(grid: Grid, r: int) -> list[int]: + """r 是表头行时，表头占哪几行：上一行（或本行）有横向合并区覆盖表头的 ≥2 列时是两行。""" + cols = {c for c, _ in grid.row(r)} + for (r1, c1, r2, c2) in grid.merges: + if r1 == r2 == r - 1 and len(cols & set(range(c1, c2 + 1))) >= 2: + return [r - 1, r] + # 本行就是上一层（「销售」横跨「数量」「金额」），下一行是全文字的下一层 + if any(r1 == r2 == r and c2 > c1 for (r1, c1, r2, c2) in grid.merges) and _all_text_row(grid, r + 1): + return [r, r + 1] + return [r] + + +@dataclass +class LayoutHints: + """压缩表示要知道的版式事实：哪一行是日期轴、标签列在哪、哪些行是列表的表头。""" + + axis_row: int | None = None + label_col: int | None = None + header_rows: set[int] = field(default_factory=set) + top_header_row: int | None = None + + +def layout_hints(grid: Grid) -> LayoutHints: + """压缩表示据此决定哪些文字格按结构文字照发。 + + 列表的表头只认两种：最上面那个表头（和与它同一行的并排表头），以及上方隔着空行（或「空行 + 小标题」）的 + 后续表头。紧挨着上一张表的「表头」不认：数据行的数字格空着时（「王五 | 电话 | (空) | 备注」）它也像表头， + 认了就会把姓名、电话当结构文字发给模型（P2-SPEC 6.2「列表数据列的文字只发列画像」）。起草那边已经不会在 + 这种行上切开（_scan_block），这里再守一道：判错的代价是外发数据，宁可让模型少看一个表头。 + """ + axis = _axis_row(grid) + if axis is not None: + return LayoutHints(axis_row=axis[0], label_col=min(axis[1]) - 1) + hints = LayoutHints() + blocks = _analyze_lists(grid).blocks + if not blocks: + return hints + top = min(b.header_rows[0] for b in blocks) + for b in blocks: + first = b.header_rows[0] + if first == top or _separated_above(grid, first, b.cols): + hints.header_rows.update(range(first, b.header_rows[-1] + 1)) + hints.top_header_row = top + return hints + + +# -------------------------------------------------------------------------- +# 交叉表 +# -------------------------------------------------------------------------- + + +@dataclass +class _Seg: + role: str # measures / dimension / derived + rows: list[int] + labels: list[str] # 原文（去首尾空白） + title: str | None = None + title_row: int | None = None + parser: str | None = None # dimension：hour_range / text + # 以下由命名阶段填 + id: str = "" + table: str = "" + columns: dict[str, str] = field(default_factory=dict) # measures：标签原文 → 列名 + unit: str | None = None # dimension：值列单位 + unit_raw: str | None = None # 分段标题括号里的原文（可能不在词表） + value: str = "" + dim: str = "" + const: dict[str, str] = field(default_factory=dict) + base: "_Seg | None" = None # derived：紧跟的分段 + stop: bool = False + keep_table: str = "" + + +@dataclass +class _Failure: + human: str + model: str + + +@dataclass +class _Cross: + sheet: str + axis_row: int + label_col: int + c1: int + c2: int + date_cols: list[int] + year_less: bool + segments: list[_Seg] = field(default_factory=list) + value_type: str = "INTEGER" + #: 占位符：canon → [首个原文, 个数, 格子] + placeholders: dict[str, list[Any]] = field(default_factory=dict) + nonnumeric: dict[str, int] = field(default_factory=dict) + blanks: list[tuple[int, int]] = field(default_factory=list) + formulas: list[tuple[int, int]] = field(default_factory=list) + region: set[tuple[int, int]] = field(default_factory=set) + failures: list[_Failure] = field(default_factory=list) + #: 按行号：data 行的数值（公式取保存值，不是数为 None），只给 detect_facts 用 + values: dict[int, list[int | float | None]] = field(default_factory=dict) + + +def _label_of(cell: GridCell | None) -> str | None: + if cell is None or cell.value is None: + return None + if isinstance(cell.value, str): + return _text(cell.value) + return _shown(cell.value) or None + + +def _sums_above(cell: GridCell | None, r: int) -> bool: + """这格是不是「汇总上方连续几行」的公式（=SUM(C22:C25) 写在 C26）：表内合计行的信号。""" + if cell is None or not cell.formula: + return False + m = re.fullmatch(r"=\s*SUM\(\s*\$?([A-Z]{1,3})\$?(\d+)\s*:\s*\$?([A-Z]{1,3})\$?(\d+)\s*\)\s*", cell.formula.upper()) + return bool(m) and m.group(1) == m.group(3) and int(m.group(4)) == r - 1 + + +def _analyze_crosstab(grid: Grid, axis_row: int, date_cols: list[int]) -> _Cross: + sheet = grid.sheet + c1, c2 = min(date_cols), max(date_cols) + label_col = c1 - 1 + x = _Cross(sheet=sheet, axis_row=axis_row, label_col=label_col, c1=c1, c2=c2, date_cols=date_cols, + year_less=any((cell := grid.get(axis_row, c)) is not None + and (d := P.month_day_or_date(cell.value)) is not None and d.year is None + for c in date_cols)) + fail = x.failures.append + if label_col < 1: + fail(_Failure(f"工作表「{sheet}」：第 {axis_row} 行的日期从 A 列开始，左边没有行标签列", + f"工作表「{sheet}」第 {axis_row} 行的日期从 A 列开始，左边没有行标签列")) + return x + gaps = [c for c in range(c1, c2 + 1) if c not in set(date_cols)] + if gaps: + where = "、".join(f"{col_letter(c)} 列" for c in gaps[:5]) + msg = f"工作表「{sheet}」：第 {axis_row} 行的日期中间夹着不是日期的格（{where}）" + fail(_Failure(msg, msg)) + extra = [c for c, _ in grid.row(axis_row) if c > c2] + if extra: + msg = f"工作表「{sheet}」：第 {axis_row} 行最后一个日期右边还有格子（{_a1_list([(axis_row, c) for c in extra])}）" + fail(_Failure(msg, msg)) + + # ---- 行分类（P2-SPEC 4.3）：只看标签列和轴列 + bottom = _bottom(grid) + kinds: list[tuple[int, str, str | None]] = [] + for r in range(axis_row + 1, bottom + 1): + lab_cell = grid.get(r, label_col) + label = _label_of(lab_cell) + has = any(grid.get(r, c) is not None for c in range(c1, c2 + 1)) + if lab_cell is not None and lab_cell.value is not None and not isinstance(lab_cell.value, str): + # 数字、日期当行标签：配方的标签只收文字，也不能把格子里的数写进配方（配方会发给模型看） + kinds.append((r, "badlabel", None)) + elif label is None: + kinds.append((r, "orphan" if has else "blank", None)) + elif has: + kinds.append((r, "data", label)) + elif P.hour_range(label) is not None or P.hour_range_total(label) is not None: + # 整期是空格的时段行：照常按 data 行处理，不能误判成分段标题（D11） + kinds.append((r, "data", label)) + else: + kinds.append((r, "title", label)) + + # ---- 分组：空行结束一组；分段标题开始一组 + groups: list[dict[str, Any]] = [] + cur: dict[str, Any] | None = None + for r, kind, label in kinds: + if kind == "blank": + cur = None + elif kind == "title": + cur = {"title": label, "title_row": r, "rows": [], "labels": []} + groups.append(cur) + elif kind == "badlabel": + cur = None + msg = f"工作表「{sheet}」：第 {r} 行的行标签（{_a1(r, label_col)}）不是文字" + fail(_Failure(msg, msg)) + x.region.update((r, c) for c in range(label_col, c2 + 1) if grid.get(r, c) is not None) + elif kind == "orphan": + cur = None + cells = [(r, c) for c in range(c1, c2 + 1) if grid.get(r, c) is not None] + msg = f"工作表「{sheet}」：第 {r} 行有数值但标签列（{col_letter(label_col)} 列）是空的" + fail(_Failure(msg, msg)) + x.region.update(cells) # 已经报了失败，不再按区域外重复报 + else: + if cur is None: + cur = {"title": None, "title_row": None, "rows": [], "labels": []} + groups.append(cur) + cur["rows"].append(r) + cur["labels"].append(label) + + seen_title = False + for g in groups: + if g["title"] is None: + if seen_title or any(s.role == "measures" for s in x.segments): + msg = (f"工作表「{sheet}」：{_rows_text(g['rows'])}的数据行上方没有分段标题，" + "和前面的行之间又隔着空行，无法确定归属") + fail(_Failure(msg, msg)) + continue + x.segments.append(_Seg("measures", g["rows"], g["labels"])) + continue + seen_title = True + if not g["rows"]: + continue # 下面没有数据行的标题：区域外文字 + rows, labels = g["rows"], g["labels"] + cut = next((i for i, lab in enumerate(labels) if P.hour_range_total(lab) is not None), len(rows)) + drows, dlabels, trows, tlabels = rows[:cut], labels[:cut], rows[cut:], labels[cut:] + bad_total = False + for r, lab in zip(drows, dlabels): + cells = [grid.get(r, c) for c in range(c1, c2 + 1)] + if any(w in lab for w in _TOTAL_HINT) or (any(c is not None for c in cells) + and all(_sums_above(c, r) for c in cells if c is not None)): + msg = f"工作表「{sheet}」：第 {r} 行像合计行，但标签无法确定区间" + fail(_Failure(f"{msg}（「{lab}」）", f"{msg}（「{lab}」）")) + bad_total = True + for r, lab in zip(trows, tlabels): + if P.hour_range_total(lab) is None: + msg = f"工作表「{sheet}」：第 {r} 行在合计行之后，但不是能确定区间的合计行（「{lab}」）" + fail(_Failure(msg, msg)) + bad_total = True + if bad_total: + continue + if not drows: + msg = f"工作表「{sheet}」：分段标题「{g['title']}」下面只有合计行，没有明细" + fail(_Failure(msg, msg)) + continue + parser = "hour_range" if all(P.hour_range(lab) is not None for lab in dlabels) else "text" + if parser == "text": + bad = [(r, lab) for r, lab in zip(drows, dlabels) if P.text_label(lab) is None] + for r, lab in bad: + msg = f"工作表「{sheet}」：第 {r} 行的标签超过 {P.LABEL_MAX} 字或不是文字" + fail(_Failure(msg, msg)) + if bad: + continue + seg = _Seg("dimension", drows, dlabels, title=g["title"], title_row=g["title_row"], parser=parser) + x.segments.append(seg) + if trows: + if parser != "hour_range": + msg = (f"工作表「{sheet}」：{_rows_text(trows)}是按时段区间的合计，" + f"但上方「{g['title']}」的标签不全是时段") + fail(_Failure(msg, msg)) + continue + seg.stop = True + x.segments.append(_Seg("derived", trows, tlabels, base=seg)) + + # ---- 数值区：类型、占位符、空格、公式（P2-SPEC 6.1 第 9 条） + ints = True + for seg in x.segments: + for r in seg.rows: + row_vals: list[int | float | None] = [] + for c in range(c1, c2 + 1): + cell = grid.get(r, c) + kind = _cell_kind(cell) + num = _value_num(cell) + row_vals.append(num) + if num is not None and not _int_like(num): + ints = False + if seg.role == "derived": + continue + if kind == "blank": + x.blanks.append((r, c)) + elif kind == "formula": + x.formulas.append((r, c)) + elif kind == "ph": + key = canon(cell.value) + entry = x.placeholders.setdefault(key, [cell.value.strip(), 0, []]) + entry[1] += 1 + entry[2].append((r, c)) + x.nonnumeric[cell.value.strip()] = x.nonnumeric.get(cell.value.strip(), 0) + 1 + elif kind in ("text", "error", "other"): + shown = _shown(cell.value) + if isinstance(cell.value, str): + x.nonnumeric[shown] = x.nonnumeric.get(shown, 0) + 1 + why = "是错误值" if kind == "error" else "不是数字，也不像表示无数据的符号" + fail(_Failure(f"工作表「{sheet}」：{_a1(r, c)} 的「{shown}」{why}", + f"工作表「{sheet}」：{_a1(r, c)} {why}")) + x.values[r] = row_vals + x.value_type = "INTEGER" if ints else "REAL" + + # ---- 认领区域、合并区 + claimed_rows = [r for s in x.segments for r in s.rows] + x.region.update((axis_row, c) for c in range(label_col, c2 + 1)) + for r in claimed_rows: + x.region.update((r, c) for c in range(label_col, c2 + 1)) + for s in x.segments: + if s.title_row is not None: + x.region.add((s.title_row, label_col)) + data_rows = {r for s in x.segments if s.role != "derived" for r in s.rows} + for (r1, mc1, r2, mc2) in grid.merges: + if any(r1 <= r <= r2 for r in data_rows) and mc2 >= c1 and mc1 <= c2: + msg = f"工作表「{sheet}」：合并单元格 {_a1(r1, mc1)}:{_a1(r2, mc2)} 落在数值区" + fail(_Failure(msg, msg)) + return x + + +# -------------------------------------------------------------------------- +# 列表 +# -------------------------------------------------------------------------- + + +@dataclass +class _ListPlan: + header_rows: list[int] + cols: list[int] + headers: dict[int, str] + rows: list[int] = field(default_factory=list) + types: dict[int, str] = field(default_factory=dict) + total_row: int | None = None + total_word: str | None = None + skip_blank: bool = False + skipped: list[int] = field(default_factory=list) + placeholders: dict[str, list[Any]] = field(default_factory=dict) + nonnumeric: dict[str, int] = field(default_factory=dict) + grain_col: int | None = None + merged_fill: list[tuple[int, int, int, int]] = field(default_factory=list) + title: str | None = None + region: set[tuple[int, int]] = field(default_factory=set) + #: 紧挨着数据、像另一张表表头的行：没有空行或小标题隔开，不在这里切开（切错了会把数据行当表头）， + #: 按数据行收进来并记起草失败 + adjacent: list[tuple[int, list[int]]] = field(default_factory=list) + #: 不在末尾第一列的合计字样：(行, 列)。第一列在中间或最上面、或者落在别的列上，都无法按合计行核对， + #: 记起草失败；不记就会把合计当明细导入，求和翻倍（H2） + stray_totals: list[tuple[int, int]] = field(default_factory=list) + # 命名阶段填 + id: str = "" + table: str = "" + names: dict[int, str] = field(default_factory=dict) + units: dict[str, str] = field(default_factory=dict) + keep_table: str = "" + + +@dataclass +class _Lists: + sheet: str + blocks: list[_ListPlan] = field(default_factory=list) + failures: list[_Failure] = field(default_factory=list) + + +def _date_kind(v: Any) -> bool: + if isinstance(v, _dt.datetime): + return (v.hour, v.minute, v.second, v.microsecond) == (0, 0, 0, 0) + if isinstance(v, _dt.date): + return True + if isinstance(v, str): + d = P.month_day_or_date(v) + return d is not None and d.year is not None + return False + + +def _list_kind(cell: GridCell | None) -> str | None: + """列表数据格：num / date / ph / text；空格 None。公式按保存值算。""" + if cell is None: + return None + v = cell.value + if v is None: + return "text" if cell.formula is None else "num" + if _num(v): + return "num" + if _date_kind(v): + return "date" + if isinstance(v, str) and _placeholder_like(v): + return "ph" + return "text" + + +def _col_type(kinds: set[str], all_int: bool) -> str: + if kinds and kinds <= {"num", "ph"} and "num" in kinds: + return "INTEGER" if all_int else "REAL" + if kinds and kinds <= {"date", "ph"} and "date" in kinds: + return "DATE" + return "TEXT" + + +def _types_of(grid: Grid, rows: list[int], cols: list[int]) -> dict[int, str]: + acc = _ColTypes(grid, cols) + for r in rows: + acc.add(r) + return acc.types() + + +class _ColTypes: + """各列到目前为止的类型（_col_type 的增量版）。收数据行时每一行都要问一次「和本块对不对得上」， + 每次从头推一遍是行数的平方（文字格还要试着按日期解析），几百行的名单就要一两秒。""" + + def __init__(self, grid: Grid, cols: list[int]) -> None: + self.grid, self.cols = grid, cols + self.kinds: dict[int, set[str]] = {c: set() for c in cols} + self.ints: dict[int, bool] = {c: True for c in cols} + self._types: dict[int, str] | None = None + + def add(self, r: int) -> None: + for c in self.cols: + cell = self.grid.get(r, c) + k = _list_kind(cell) + if k is None: + continue + self.kinds[c].add(k) + if _num(cell.value) and not _int_like(cell.value): # type: ignore[union-attr] + self.ints[c] = False + self._types = None + + def types(self) -> dict[int, str]: + if self._types is None: + self._types = {c: _col_type(self.kinds[c], self.ints[c]) for c in self.cols} + return self._types + + +def _compatible(grid: Grid, r: int, cols: list[int], types: dict[int, str]) -> bool: + """空行之后的这一行和上方是不是同一张表：≥2 个非空格，每格都合各列的类型。""" + cells = [(c, grid.get(r, c)) for c in cols if grid.get(r, c) is not None] + if len(cells) < 2: + return False + for c, cell in cells: + k = _list_kind(cell) + t = types.get(c, "TEXT") + if t in ("INTEGER", "REAL") and k not in ("num", "ph"): + return False + if t == "DATE" and k not in ("date", "ph"): + return False + return True + + +def _next_row(grid: Grid, r: int, cols: list[int], bottom: int) -> int | None: + lo, hi = cols[0], cols[-1] + for rr in grid.row_numbers(): + if rr > r and rr <= bottom and any(lo <= c <= hi for c, _ in grid.row(rr)): + return rr + return None + + +_TYPE_CLASS = {"INTEGER": "num", "REAL": "num", "DATE": "date"} + + +def _foreign_cols(grid: Grid, r: int, cols: list[int], types: dict[int, str]) -> list[int]: + """r 行里落在数字或日期列上的文字格所在的列。""" + return [c for c in cols if types.get(c) in _TYPE_CLASS and _list_kind(grid.get(r, c)) == "text"] + + +def _title_row(grid: Grid, r: int, cols: list[int]) -> bool: + """块的列范围里只有一格文字（「二、费用」）：下一张表的小标题。""" + cells = [cell for c, cell in grid.row(r) if cols[0] <= c <= cols[-1]] + if len(cells) != 1 or cells[0].formula is not None: + return False + t = _text(cells[0].value) + return t is not None and not P.looks_numeric_text(t) and _total_word(t) is None + + +def _separated_above(grid: Grid, r: int, cols: list[int]) -> bool: + """r 行（表头）上方在这几列里隔着空行，或者是「空行 + 小标题」，或者已经到了工作表顶上。""" + rr = r - 1 + if rr >= 1 and _title_row(grid, rr, cols): + rr -= 1 + return rr < 1 or not any(cols[0] <= c <= cols[-1] for c, _ in grid.row(rr)) + + +def _new_header(grid: Grid, r: int, plan: _ListPlan, bottom: int, types: dict[int, str]) -> bool: + """r 行是不是下一张表的表头：像表头，并且和本块对不上——有文字落在本块的数字或日期列上，或者它下面 + 几行的列类型和本块不同。 + + 只看「像表头」不够：数据行的数字格空着时（「王五 | 电话 | (空) | 备注」）这一行全是文字、互不相同、下面 + 有数，也像表头；可它的文字都在本块的文字列上，下面的行也和本块同类型。把它当表头，姓名、电话就成了列名， + 还会当结构文字发给模型。 + """ + if not plan.rows or not _header_like(grid, r): + return False + if _foreign_cols(grid, r, plan.cols, types): + return True + below = [rr for rr in range(r + 1, min(r + 3, bottom) + 1) + if any(grid.get(rr, c) is not None for c in plan.cols)] + if not below: + return False + under = _types_of(grid, below, plan.cols) + return any(_TYPE_CLASS.get(under[c]) and _TYPE_CLASS.get(under[c]) != _TYPE_CLASS.get(types.get(c, "")) + for c in plan.cols) + + +def _more_rows(grid: Grid, r: int, plan: _ListPlan, bottom: int, types: dict[int, str]) -> bool: + """合计字样那一行下面紧接着还有同一张表的行（明细，或另一行合计）：它不在末尾，是夹在中间的小计。""" + nr = r + 1 + if nr > bottom or all(grid.get(nr, c) is None for c in plan.cols): + return False + if _title_row(grid, nr, plan.cols) or _new_header(grid, nr, plan, bottom, types): + return False + return _compatible(grid, nr, plan.cols, types) + + +def _scan_block(grid: Grid, plan: _ListPlan, bottom: int) -> None: + """从表头下一行往下收数据行，到空行（后面不是同一张表）、小标题 + 新表头、或末尾的合计行为止。 + + 新表头只在空行或小标题之后才认（P2-SPEC 6.1 第 11 条的「第二个表头」）：块里上下行都是数据时，一行 + 「像表头」多半只是数字格空着，不能在那里切开。真像另一张表、却紧挨着数据的行记进 adjacent（起草失败）。 + """ + cols = plan.cols + acc = _ColTypes(grid, cols) + r = plan.header_rows[-1] + 1 + while r <= bottom: + cells = [grid.get(r, c) for c in cols] + types = acc.types() + if all(c is None for c in cells): + nxt = _next_row(grid, r, cols, bottom) + if nxt is None or not plan.rows or _new_header(grid, nxt, plan, bottom, types): + break + if _compatible(grid, nxt, cols, types) and _total_word( + _label_of(grid.get(nxt, cols[0]))) is None: + plan.skip_blank = True + plan.skipped.extend(range(r, nxt)) + r = nxt + continue + break + if plan.rows and _title_row(grid, r, cols) and _new_header(grid, r + 1, plan, bottom, types): + break + if _new_header(grid, r, plan, bottom, types): + plan.adjacent.append((r, _foreign_cols(grid, r, cols, types))) + word = _total_word(_label_of(grid.get(r, cols[0]))) + if word is not None and plan.rows and not _more_rows(grid, r, plan, bottom, types): + plan.total_row, plan.total_word = r, word + break + if word is not None: + plan.stray_totals.append((r, cols[0])) + plan.stray_totals.extend((r, c) for c, cell in zip(cols[1:], cells[1:]) + if cell is not None and _total_word(cell.value) is not None) + plan.rows.append(r) + acc.add(r) + r += 1 + + +def _list_failures(sheet: str, grid: Grid, plan: _ListPlan) -> list[_Failure]: + """块里收进来、但起草无法确定的行：紧挨着数据的新表头、不在末尾第一列的合计字样（照交叉表第 7 条的做法）。 + + 模型版只写行号、列号和命中的合计字样（「合计」「小计」本身是结构文字），不带格子原文。 + """ + out: list[_Failure] = [] + for r, foreign in plan.adjacent: + where = f"，{'、'.join(col_letter(c) for c in foreign)} 列应是数字或日期" if foreign else "" + msg = (f"工作表「{sheet}」：第 {r} 行像另一张表的表头{where}，但它紧挨着上方的数据，" + "中间没有空行或标题，无法确定两张表的分界") + out.append(_Failure(msg, msg)) + seen: set[int] = set() + for r, c in plan.stray_totals: + if r in seen: + continue + seen.add(r) + cell = grid.get(r, c) + text = _text(cell.value) if cell is not None else None + word = _total_word(text) or "" + shown = (text or "")[:20] + if c == plan.cols[0]: + why = "但不在列表末尾，无法确定它汇总的是哪几行" + else: + why = "但合计字样不在第一列，无法按合计行核对" + out.append(_Failure(f"工作表「{sheet}」：第 {r} 行像合计行（{col_letter(c)} 列「{shown}」），{why}", + f"工作表「{sheet}」：第 {r} 行像合计行（{col_letter(c)} 列以「{word}」开头），{why}")) + return out + + +def _analyze_lists(grid: Grid) -> _Lists: + out = _Lists(sheet=grid.sheet) + rows = grid.row_numbers() + if not rows: + return out + bottom = rows[-1] + first_limit = rows[0] + LIST_HEADER_SCAN_ROWS - 1 + r_idx = 0 + seen_keys: list[frozenset[str]] = [] + while r_idx < len(rows): + r = rows[r_idx] + if not out.blocks and r > first_limit: + break + if not _header_like(grid, r): + r_idx += 1 + continue + hrows = _header_rows_at(grid, r) + if hrows[0] < r and out.blocks and hrows[0] <= max(b.rows[-1] if b.rows else b.header_rows[-1] + for b in out.blocks): + hrows = [r] + all_cols = sorted({c for hr in hrows for c, _ in grid.row(hr)} + | {c for (r1, c1, r2, c2) in grid.merges if hrows[0] <= r1 <= hrows[-1] + for c in range(c1, c2 + 1)}) + texts = _header_texts(grid, hrows, all_cols) + runs: list[list[int]] = [] + for c in sorted(texts): + if runs and c == runs[-1][-1] + 1: + runs[-1].append(c) + else: + runs.append([c]) + made: list[_ListPlan] = [] + for run in runs: + if len(run) < 2: + continue + plan = _ListPlan(header_rows=hrows, cols=run, headers={c: texts[c] for c in run}) + _scan_block(grid, plan, bottom) + if not plan.rows: + continue + made.append(plan) + if not made: + r_idx += 1 + continue + for plan in made: + keys = frozenset(P.match_key(h) for h in plan.headers.values()) + if keys in seen_keys: + msg = (f"工作表「{grid.sheet}」：{_rows_text(plan.header_rows)}的表头和上方另一张表完全相同，" + "无法区分两张表") + out.failures.append(_Failure(msg, msg)) + seen_keys.append(keys) + _finish_block(grid, plan) + out.failures.extend(_list_failures(grid.sheet, grid, plan)) + out.blocks.append(plan) + end = max((p.total_row or (p.rows[-1] if p.rows else p.header_rows[-1])) for p in made) + while r_idx < len(rows) and rows[r_idx] <= end: + r_idx += 1 + return out + + +def _finish_block(grid: Grid, plan: _ListPlan) -> None: + """类型、占位符、主键、合并区、标题、认领区域。""" + cols, rows = plan.cols, plan.rows + plan.types = _types_of(grid, rows, cols) + for c in cols: + if plan.types[c] not in ("INTEGER", "REAL", "DATE"): + continue + for r in rows: + cell = grid.get(r, c) + if _list_kind(cell) == "ph": + key = canon(cell.value) + entry = plan.placeholders.setdefault(key, [cell.value.strip(), 0, []]) + entry[1] += 1 + entry[2].append((r, c)) + plan.nonnumeric[cell.value.strip()] = plan.nonnumeric.get(cell.value.strip(), 0) + 1 + for c in cols: + if plan.types[c] not in ("TEXT", "DATE"): + continue + keys: list[str] = [] + for r in rows: + cell = grid.get(r, c) + if cell is None or cell.value is None: + break + keys.append(canon(_shown(cell.value))) + if len(keys) == len(rows) and len(set(keys)) == len(keys): + plan.grain_col = c + break + for m in grid.merges: + r1, c1, r2, c2 = m + if not any(r1 <= r <= r2 for r in rows) or c2 < cols[0] or c1 > cols[-1]: + continue + plan.merged_fill.append(m) + # 表头正上方的单格文字当标题（表名用它，不含数字时） + above = plan.header_rows[0] - 1 + for rr in (above, above - 1): + cells = [(c, cell) for c, cell in grid.row(rr) if cols[0] <= c <= cols[-1]] + if len(cells) == 1 and _text(cells[0][1].value): + plan.title = _text(cells[0][1].value) + break + if cells: + break + last = plan.total_row or rows[-1] + for r in range(plan.header_rows[0], last + 1): + for c in cols: + if grid.get(r, c) is not None or r in plan.header_rows: + plan.region.add((r, c)) + for (r1, c1, r2, c2) in plan.merged_fill: + for r in range(r1, r2 + 1): + for c in range(c1, c2 + 1): + plan.region.add((r, c)) + + +# -------------------------------------------------------------------------- +# 一张工作表 +# -------------------------------------------------------------------------- + + +@dataclass +class _Sheet: + sheet: str + sid: str + cross: _Cross | None = None + lists: _Lists | None = None + region: set[tuple[int, int]] = field(default_factory=set) + period: list[tuple[int, int, str]] = field(default_factory=list) + outside: list[tuple[int, int]] = field(default_factory=list) + failures: list[_Failure] = field(default_factory=list) + hidden_rows: list[int] = field(default_factory=list) + hidden_cols: list[int] = field(default_factory=list) + + +def _analyze_sheet(grid: Grid, sid: str) -> _Sheet: + sh = _Sheet(sheet=grid.sheet, sid=sid) + axis = _axis_row(grid) + if axis is not None: + sh.cross = _analyze_crosstab(grid, *axis) + sh.region = set(sh.cross.region) + sh.failures.extend(sh.cross.failures) + else: + sh.lists = _analyze_lists(grid) + sh.failures.extend(sh.lists.failures) + for b in sh.lists.blocks: + sh.region |= b.region + for (r1, c1, r2, c2) in b.merged_fill: + bad = [c for c in range(c1, c2 + 1) if b.types.get(c) != "TEXT"] + if bad or c2 > c1: + msg = (f"工作表「{grid.sheet}」：合并单元格 {_a1(r1, c1)}:{_a1(r2, c2)} 落在数据区，" + "只有文字列的纵向合并能按左上格填充") + sh.failures.append(_Failure(msg, msg)) + if not any(t != "TEXT" for t in b.types.values()): + msg = (f"工作表「{grid.sheet}」：{_rows_text(b.header_rows)}下面没有一列是统一的数字或日期，" + "看起来不是列表（可能是表单）") + sh.failures.append(_Failure(msg, msg)) + if not sh.lists.blocks: + msg = (f"工作表「{grid.sheet}」：认不出版式。没有一行有 {DATE_HEADER_MIN} 格以上的日期（交叉表），" + "也没有一行像列表的表头（几个互不相同的文字格、下方有数字）") + sh.failures.append(_Failure(msg, msg)) + rows_in = {r for r, _ in sh.region} + cols_in = {c for _, c in sh.region} + sh.hidden_rows = sorted(r for r in grid.hidden_rows if r in rows_in) + sh.hidden_cols = sorted(c for c in grid.hidden_cols if c in cols_in) + # 区域外：统计期、区域外文字、区域外数字 + for (r, c), cell in sorted(grid.cells.items()): + if (r, c) in sh.region: + continue + v = cell.value + if cell.formula is not None: + msg = f"工作表「{grid.sheet}」：{_a1(r, c)} 是数据区外的公式" + sh.failures.append(_Failure(msg, msg)) + continue + if isinstance(v, str) and P.cn_date_range(v) is not None: + sh.period.append((r, c, v.strip())) + sh.outside.append((r, c)) + continue + kind = P.classify_outside(v) + if kind == "number": + sh.failures.append(_Failure( + f"工作表「{grid.sheet}」：数据区外的 {_a1(r, c)} 是数字（「{_shown(v)}」），不知道它属于哪张表", + f"工作表「{grid.sheet}」：数据区外的 {_a1(r, c)} 是数字，不知道它属于哪张表")) + continue + sh.outside.append((r, c)) + return sh + + +# ========================================================================== +# 整个工作簿：命名、配方、事实 +# ========================================================================== + + +@dataclass +class _Analysis: + sheets: list[_Sheet] + hidden_sheets: list[str] + facts: DraftFacts + recipe: dict[str, Any] | None + failures: list[_Failure] + + +def _visible_grids(scan: WorkbookScan, grids: dict[str, Grid]) -> list[Grid]: + order = [s.name for s in scan.sheets if s.state == "visible"] if scan.sheets else list(grids) + out = [grids[n] for n in order if n in grids and grids[n].cells] + out += [g for n, g in grids.items() if n not in order and g.cells] + return out + + +def _common_suffix(titles: list[str]) -> str: + """表名 = 标题去掉单位后缀后的最长公共后缀，收缩到每个前缀至少 2 字；只有一段时就是去单位后的标题。""" + bases = [P.split_unit_suffix(t)[0] for t in titles] + if len(bases) == 1: + return bases[0] + suf = 0 + while all(len(b) > suf for b in bases) and len({b[len(b) - 1 - suf] for b in bases}) == 1: + suf += 1 + while suf and any(len(b) - suf < 2 for b in bases): + suf -= 1 + name = bases[0][len(bases[0]) - suf:].strip(" _-") if suf else "" + return name if len(name) >= 2 else bases[0] + + +def _group_dimensions(segs: list[_Seg]) -> list[list[_Seg]]: + """合表：dim.parser 相同、标签规范写法集合两两不相交、单位相同的分段写进同一张表（P2-SPEC 6.1 第 5 条）。""" + groups: list[list[_Seg]] = [] + for s in segs: + keys = {_dim_key(s.parser, lab) for lab in s.labels} + unit = P.split_unit_suffix(s.title or "")[1] + for g in groups: + if g[0].parser != s.parser or P.split_unit_suffix(g[0].title or "")[1] != unit: + continue + if any(keys & {_dim_key(o.parser, lab) for lab in o.labels} for o in g): + continue + g.append(s) + break + else: + groups.append([s]) + # 合表要能给每段选出互不相同的常量；选不出来就各成一张表 + out: list[list[_Seg]] = [] + for g in groups: + if len(g) > 1: + titles = [s.title or "" for s in g] + picks = [(P.candidate_words(t, [o for o in titles if o != t]) or [None])[0] for t in titles] + if None in picks or len(set(picks)) != len(picks): + out.extend([s] for s in g) + continue + out.append(g) + return out + + +def _dim_key(parser: str | None, label: str) -> str: + if parser == "hour_range": + hr = P.hour_range(label) + return hr.canonical if hr else canon(label) + return canon(label) + + +def _measure_column(label: str, pos: int) -> tuple[str, str | None, str | None]: + """标签 → (列名, 词表内单位, 括号原文)。单位在词表里：列名去单位；不在词表：列名保留原文（P2-SPEC 6.1 第 8 条）。""" + name, unit = P.split_unit_suffix(label) + if unit is not None and P.unit_known(unit): + return to_sql_name(name, fallback=f"列{pos}"), unit, unit + if unit is not None: + return to_sql_name(label, fallback=f"列{pos}"), None, unit + return to_sql_name(name, fallback=f"列{pos}"), None, None + + +def _analyze(scan: WorkbookScan, grids: dict[str, Grid]) -> _Analysis: + visible = _visible_grids(scan, grids) + hidden = [s.name for s in scan.sheets if s.state != "visible"] + sheets = [_analyze_sheet(g, f"s{i + 1}") for i, g in enumerate(visible)] + failures = [f for sh in sheets for f in sh.failures] + + tables_used: set[str] = set() + seg_ids: set[str] = set() + block_ids: set[str] = set() + tables: list[dict[str, Any]] = [] + children: dict[str, list[dict[str, Any]]] = {} + + def table(name: str, grain: list[str], units: dict[str, str], kind: str = "data", + parent: str | None = None) -> None: + spec = {"name": name, "grain": grain, "kind": kind, "units": units, "note": ""} + if parent is None: + tables.append(spec) + else: + children.setdefault(parent, []).append(spec) + + def seg_id(want: str, sid: str) -> str: + want = want[:32] + out = want if want not in seg_ids else f"{want[:32 - len(sid) - 1]}_{sid}" + n = 2 + while out in seg_ids: + out = f"{want[:28]}_{n}" + n += 1 + seg_ids.add(out) + return out + + def new_table(want: str, n_fallback: int) -> str: + return _unique(to_sql_name(want, fallback=f"表{n_fallback}") if want else f"表{n_fallback}", tables_used) + + sheet_dicts: list[dict[str, Any]] = [] + candidates: dict[str, list[str]] = {} + crosstabs = 0 + lists_n = 0 + for sh in sheets: + blocks: list[dict[str, Any]] = [] + x = sh.cross + if x is not None and x.segments: + crosstabs += 1 + bid = "交叉表" if crosstabs == 1 else f"交叉表_{sh.sid}" + block_ids.add(bid) + segs_out: list[dict[str, Any]] = [] + for s in x.segments: + if s.role != "measures": + continue + base = to_sql_name(sh.sheet, fallback=f"表{len(tables_used) + 1}")[:45] + s.table = _unique(f"{base}_按日", tables_used) + s.id = seg_id(s.table, sh.sid) + used_cols = {collide_key("日期")} + units: dict[str, str] = {} + for i, lab in enumerate(s.labels): + col, unit, _raw = _measure_column(lab, i + 2) + col = _unique(col, used_cols) + s.columns[lab] = col + if unit: + units[col] = unit + table(s.table, ["日期"], units) + dims = [s for s in x.segments if s.role == "dimension"] + for g in _group_dimensions(dims): + titles = [s.title or "" for s in g] + n_fallback = len(tables_used) + 1 + want = _common_suffix(titles) + plain = (to_sql_name(want, fallback=f"表{n_fallback}") if want else f"表{n_fallback}")[:48] + tname = _unique(plain, tables_used) + dim = _HOUR_DIM if g[0].parser == "hour_range" else _TEXT_DIM + const_col = f"{dim}类别" if len(g) > 1 else "" + # 值列名取去重之前的表名：撞名后的「时段客流_2」最后两个字是「_2」，会得到无意义的「c_2」 + value = to_sql_name(plain[-2:], fallback="数值") if len(plain) >= 2 else "数值" + reserved = {collide_key(n) for n in ("日期", dim, const_col, *(_DERIVE if g[0].parser == "hour_range" + else ())) if n} + if collide_key(value) in reserved or len(value) < 2: + value = "数值" + unit_raw = P.split_unit_suffix(titles[0])[1] + unit = unit_raw if unit_raw and P.unit_known(unit_raw) else None + for s in g: + s.table, s.dim, s.value, s.unit, s.unit_raw = tname, dim, value, unit, unit_raw + siblings = [t for t in titles if t != s.title] + cands = P.candidate_words(s.title or "", siblings) + candidates[s.title or ""] = cands + if const_col: + s.const = {const_col: cands[0]} + s.id = seg_id(cands[0], sh.sid) + else: + s.id = seg_id(tname, sh.sid) + table(tname, ["日期", dim], {value: unit} if unit else {}) + for s in x.segments: + if s.role == "derived" and s.base is not None: + s.id = seg_id(f"{s.base.id}合计", sh.sid) + keep = f"{s.base.table}_表内合计" + if collide_key(keep) not in tables_used: + tables_used.add(collide_key(keep)) + table(keep, ["日期", "合计项"], {s.base.value: s.base.unit} if s.base.unit else {}, + kind="reported_total", parent=s.base.table) + s.keep_table = keep + for s in x.segments: + segs_out.append(_segment_dict(s)) + placeholders = [{"text": e[0], "meaning": "无数据"} for e in x.placeholders.values()] + blocks.append({ + "id": bid, "layout": "crosstab", + "axis": {"find": {"parser": "month_day_or_date", "min": 2}, "name": "日期", "type": "DATE", + "year_from": "统计期", "checks": ["contiguous", "covers_context"]}, + "label_offset": -1, + "values": {"type": x.value_type, "placeholders": placeholders, "blank": "reject", + "text_number": "reject", "formula": "reject"}, + "segments": segs_out, + }) + if sh.lists is not None: + for b in sh.lists.blocks: + lists_n += 1 + b.id = f"列表{lists_n}" + block_ids.add(b.id) + title = b.title if b.title and not any(ch.isdigit() for ch in b.title) else None + want = _ENUM_PREFIX.sub("", P.split_unit_suffix(title)[0]) if title else sh.sheet + b.table = new_table(want, len(tables_used) + 1) + used_cols: set[str] = set() + cols_out: list[dict[str, Any]] = [] + for i, c in enumerate(b.cols): + col, unit, _raw = _measure_column(b.headers[c], i + 1) + col = _unique(col, used_cols) + b.names[c] = col + if unit: + b.units[col] = unit + cols_out.append({"header": b.headers[c][:80], "name": col, "type": b.types[c]}) + total = None + if b.total_row is not None: + keep = f"{b.table}_表内合计" + if collide_key(keep) not in tables_used: + tables_used.add(collide_key(keep)) + num_units = {n: u for c, n in b.names.items() if (u := b.units.get(n)) + and b.types[c] in ("INTEGER", "REAL")} + table(keep, ["合计项"], num_units, kind="reported_total", parent=b.table) + b.keep_table = keep + total = {"label_column": b.names[b.cols[0]], "pick": b.total_word, "keep_as": keep} + grain = [b.names[b.grain_col]] if b.grain_col is not None else [] + table(b.table, grain, dict(b.units)) + fill = bool(b.merged_fill) and all(r2 > r1 and c1 == c2 and b.types.get(c1) == "TEXT" + for (r1, c1, r2, c2) in b.merged_fill) + blocks.append({ + "id": b.id, "layout": "list", "table": b.table, "header_rows": len(b.header_rows), + "after_title": None, "columns": cols_out, "extra_columns": "reject", + "rows": {"blank_rows": "skip" if b.skip_blank else "stop", "total_row": total}, + "values": {"placeholders": [{"text": e[0], "meaning": "无数据"} for e in b.placeholders.values()], + "blank": "null", "text_number": "reject", "formula": "accept_cached"}, + "merged_data": "fill" if fill else "reject", + }) + if not blocks: + continue + context: list[dict[str, Any]] = [] + if sh.period or x is not None: + context.append({"id": "统计期", "kind": "period", "parser": "cn_date_range", + "prefer_prefix": _prefer_prefix(sh.period), "cross_check": "filename"}) + sheet_dicts.append({"id": sh.sid, "match": {"name": sh.sheet[:31], "fallback": "only_visible_sheet"}, + "hidden": {"rows": "reject_if_any", "cols": "reject_if_any"}, + "context": context, "blocks": blocks}) + + facts = _facts_of(sheets) + facts.candidates = candidates + facts.nonnumeric = {} + for sh in sheets: + for src in ([sh.cross.nonnumeric] if sh.cross else []) + [b.nonnumeric for b in + (sh.lists.blocks if sh.lists else [])]: + for k, v in src.items(): + facts.nonnumeric[k] = facts.nonnumeric.get(k, 0) + v + + recipe: dict[str, Any] | None = None + if sheet_dicts: + ordered: list[dict[str, Any]] = [] + for t in tables: + ordered.append(t) + ordered.extend(children.get(t["name"], [])) + recipe = {"recipe_format": RECIPE_FORMAT, "mode": "replace", "other_visible_sheets": "confirm", + "sheets": sheet_dicts, "tables": ordered, "relations": _relations_for(facts, sheets)} + return _Analysis(sheets, hidden, facts, recipe, failures) + + +def _prefer_prefix(period: list[tuple[int, int, str]]) -> str | None: + """统计期格冒号前的文字（≤12 字、不含数字），作回执里优先展示的格。取第一处。""" + if not period: + return None + s = canon(period[0][2]) + if ":" not in s: + return None + head = s.split(":", 1)[0].strip() + return head if 0 < len(head) <= 12 and not any(ch.isdigit() for ch in head) else None + + +def _segment_dict(s: _Seg) -> dict[str, Any]: + if s.role == "measures": + return {"id": s.id, "role": "measures", "table": s.table, "locate": {"by": "labels"}, + "labels": {"expect": list(s.labels)}, "measures": dict(s.columns)} + if s.role == "dimension": + out: dict[str, Any] = { + "id": s.id, "role": "dimension", "table": s.table, + "locate": {"by": "section_title", "title": s.title}, + "labels": {"expect": list(s.labels)}, + "dim": {"name": s.dim, "parser": s.parser, "derive": dict(_DERIVE) if s.parser == "hour_range" else {}}, + "value": s.value, "const": {k: {"pick": v} for k, v in s.const.items()}, + } + if s.stop: + out["stop_parser"] = "hour_range_total" + return out + base = s.base + assert base is not None + return {"id": s.id, "role": "derived", "locate": {"by": "after", "segment": base.id}, + "labels_parser": "hour_range_total", "labels": {"expect": list(s.labels)}, + "verify": {"kind": "label_range_sum", "against_table": base.table, "value": base.value}, + "keep_as": {"table": s.keep_table, "dim": "合计项", "derive": dict(_DERIVE), "value": base.value}} + + +# -------------------------------------------------------------------------- +# 系统发现 +# -------------------------------------------------------------------------- + + +def _facts_of(sheets: list[_Sheet]) -> DraftFacts: + """sum_eq 与 not_equal_sum（P2-SPEC 6.1 第 10 条）。编号按工作表、分段、合计行的先后。""" + facts: list[Fact] = [] + for sh in sheets: + x = sh.cross + if x is None: + continue + ncols = x.c2 - x.c1 + 1 + sum_totals: dict[str, int] = {} + for seg in x.segments: + if seg.role != "measures" or not FACT_PARTS_MIN + 1 <= len(seg.rows) <= FACT_SEARCH_MAX_ROWS: + continue + deepest = FACT_PARTS_MAX if len(seg.rows) <= FACT_DEEP_MAX_ROWS else FACT_PARTS_MIN + for ti, t in enumerate(seg.rows): + others = [i for i in range(len(seg.rows)) if i != ti] + hit = None + for size in range(FACT_PARTS_MIN, min(deepest, len(others)) + 1): + for combo in itertools.combinations(others, size): + ok = _sum_holds(x, t, [seg.rows[i] for i in combo]) + if ok is not None: + hit = (combo, ok) + break + if hit: + break + if hit is None: + continue + combo, checked = hit + parts = [seg.rows[i] for i in combo] + fid = f"F{len(facts) + 1}" + text = (f"第 {t} 行 = " + " + ".join(f"第 {p} 行" for p in parts) + + f"（{ncols} 列中 {checked} 列成立）") + facts.append(Fact(fid, "sum_eq", sh.sheet, text, { + "segment": seg.id, "total": seg.labels[ti], "parts": [seg.labels[i] for i in combo], + "rows": [t, *parts]})) + sum_totals.setdefault(seg.id, ti) + measures = next((s for s in x.segments if s.role == "measures"), None) + if measures is None: + continue + bi = sum_totals.get(measures.id, 0) + b_row = measures.rows[bi] + by_table: dict[str, list[_Seg]] = {} + for s in x.segments: + if s.role == "dimension": + by_table.setdefault(s.table, []).append(s) + for tname, segs in by_table.items(): + equal = checked = 0 + for ci in range(ncols): + b = x.values.get(b_row, [None] * ncols)[ci] + parts = [x.values.get(r, [None] * ncols)[ci] for s in segs for r in s.rows] + nums = [p for p in parts if p is not None] + if b is None or not nums: + continue + checked += 1 + equal += _equal(float(sum(nums)), float(b)) + if checked and equal < checked: + fid = f"F{len(facts) + 1}" + names = "、".join(s.id for s in segs) + facts.append(Fact(fid, "not_equal_sum", sh.sheet, + f"{names}各行之和 与 第 {b_row} 行：{checked} 列中 {equal} 列相等", + {"a": [s.id for s in segs], "b_segment": measures.id, + "b": measures.labels[bi], "equal": equal, "checked": checked})) + return DraftFacts(facts=facts) + + +def _sum_holds(x: _Cross, total: int, parts: list[int]) -> int | None: + """total 行在两边都是数的列上都等于 parts 之和（≥2 列、全部成立）时返回核对的列数。全零的组成行不算。""" + tv = x.values.get(total) + pv = [x.values.get(p) for p in parts] + if tv is None or any(v is None for v in pv): + return None + checked = 0 + for ci, t in enumerate(tv): + col = [v[ci] for v in pv] # type: ignore[index] + if t is None or any(c is None for c in col): + continue + if not _equal(float(sum(col)), float(t)): + return None + checked += 1 + if checked < 2: + return None + for v in pv: + if all((c or 0) == 0 for c in v): # type: ignore[union-attr] + return None + return checked + + +def _relations_for(facts: DraftFacts, sheets: list[_Sheet]) -> list[dict[str, Any]]: + """每条事实一条认领：sum_eq 先编 R1…，not_comparable 接着编（P2-SPEC 6.1 第 8b 条）。""" + segs = {s.id: s for sh in sheets if sh.cross for s in sh.cross.segments} + out: list[dict[str, Any]] = [] + for f in facts.facts: + if f.kind != "sum_eq": + continue + seg = segs.get(f.detail["segment"]) + if seg is None: + continue + out.append({"id": f"R{len(out) + 1}", "kind": "sum_eq", "table": seg.table, + "total": seg.columns[f.detail["total"]], + "parts": [seg.columns[p] for p in f.detail["parts"]], "claims": f.id}) + for f in facts.facts: + if f.kind != "not_equal_sum": + continue + a = segs.get(f.detail["a"][0]) + b = segs.get(f.detail["b_segment"]) + if a is None or b is None: + continue + out.append({"id": f"R{len(out) + 1}", "kind": "not_comparable", + "a": {"table": a.table, "value": a.value}, + "b": {"table": b.table, "value": b.columns[f.detail["b"]]}, "by": "日期", "claims": f.id}) + return out + + +def detect_facts(grids: dict[str, Grid], scan: WorkbookScan, *, + recipe: dict[str, Any] | None = None) -> DraftFacts: + """系统发现（只给关系不给数值）。分段 id 按规则起草的取法（同一份原件得到同一组 id）。 + + 给了 recipe（改配方、上传新一期时的现行或工作配方）时，再按标签把事实里的分段 id 和标签原文换成 + 这份配方里对应的写法（remap_facts）：用户把规则起草的「客流汇总_按日」改名成「日客流」之后，静态校验的 + 认领检查才对得上；事实本身（哪几行之和等于哪一行）不变，所以「甲 + 乙」配「甲 + 乙 + 丙」照样报出来。 + """ + an = _analyze(scan, grids) + if recipe is None: + return an.facts + titles = {s.id: s.title for sh in an.sheets if sh.cross for s in sh.cross.segments + if s.role == "dimension" and s.title} + return remap_facts(an.facts, recipe, visible=[sh.sheet for sh in an.sheets], titles=titles) + + +def remap_facts(facts: DraftFacts, recipe: dict[str, Any], *, visible: list[str] | None = None, + titles: dict[str, str] | None = None) -> DraftFacts: + """把事实的 detail 里的分段 id、标签原文换成配方里按 match_key 对得上的分段和 expect 写法。对不上的原样留着。 + + **只在事实所在的那张工作表里找**：配方里 match.name 与 Fact.sheet 按 match_key 相同的工作表；没有同名的， + 且文件里恰好只有一张有内容的可见工作表（visible）、配方里恰好一张 fallback=only_visible_sheet 时，用那一张 + （与执行器认工作表的办法一致）。不限定工作表的话，多张同结构工作表时后面那张的事实会被改指到第一张的分段， + 原样的规则草稿也会报 fact_claim_mismatch。visible 不给时不做回退。 + + not_equal_sum 的明细分段 a 改过 id 时，按分段标题认（titles：规则起草的分段 id → 标题，detect_facts 给）； + 认不全时，只有这张工作表上恰好一张长表才取它的全部分段——两张长表时取全部会把别的表拼进来。 + """ + try: + parsed = Recipe.model_validate(recipe) + except ValidationError: + return facts + titles = titles or {} + + def scope(fact_sheet: str) -> list[Any]: + key = P.match_key(fact_sheet) + named = [s for s in parsed.sheets if P.match_key(s.match.name) == key] + if named: + return named + if visible is not None and len(visible) == 1 and P.match_key(visible[0]) == key: + fb = [s for s in parsed.sheets if s.match.fallback == "only_visible_sheet"] + if len(fb) == 1: + return fb + return [] + + def segments(sheets: list[Any]) -> tuple[list[MeasuresSegment], list[DimensionSegment]]: + ms: list[MeasuresSegment] = [] + ds: list[DimensionSegment] = [] + for sheet in sheets: + for block in sheet.blocks: + if isinstance(block, CrosstabBlock): + ms += [s for s in block.segments if isinstance(s, MeasuresSegment)] + ds += [s for s in block.segments if isinstance(s, DimensionSegment)] + return ms, ds + + def find_measures(ms: list[MeasuresSegment], seg_id: str, + labels: list[str]) -> tuple[MeasuresSegment, dict[str, str]] | None: + want = [P.match_key(x) for x in labels] + # 同 id 的分段优先（没改名时不动 id，只换标签写法） + for seg in sorted(ms, key=lambda s: s.id != seg_id): + keys = {P.match_key(e): e for e in seg.labels.expect} + if all(k in keys for k in want): + return seg, keys + return None + + out: list[Fact] = [] + for f in facts.facts: + d = dict(f.detail) + ms, ds = segments(scope(f.sheet)) + if f.kind == "sum_eq": + hit = find_measures(ms, d.get("segment", ""), [d["total"], *d["parts"]]) + if hit is not None: + seg, keys = hit + d["segment"] = seg.id + d["total"] = keys[P.match_key(d["total"])] + d["parts"] = [keys[P.match_key(p)] for p in d["parts"]] + else: + hit = find_measures(ms, d.get("b_segment", ""), [d["b"]]) + if hit is not None: + seg, keys = hit + d["b_segment"] = seg.id + d["b"] = keys[P.match_key(d["b"])] + ids = {s.id for s in ds} + if ds and not all(a in ids for a in d["a"]): + by_title = {P.match_key(s.locate.title): s.id for s in ds + if s.locate.by == "section_title" and s.locate.title} + mapped: list[str | None] = [] + for a in d["a"]: + title = titles.get(a) + mapped.append(a if a in ids else by_title.get(P.match_key(title)) if title else None) + if all(mapped): + d["a"] = mapped + elif len({s.table for s in ds}) == 1: + d["a"] = [s.id for s in ds] + out.append(Fact(f.id, f.kind, f.sheet, f.text, d)) + return DraftFacts(facts=out, candidates=dict(facts.candidates), nonnumeric=dict(facts.nonnumeric)) + + +# ========================================================================== +# 起草 +# ========================================================================== + + +def _full_form(recipe: dict[str, Any]) -> dict[str, Any]: + """补全默认值的完整形式：问题的 effects 按它写路径，父对象都在。pydantic 不收时原样返回（交给静态校验报）。""" + try: + return Recipe.model_validate(recipe).model_dump(mode="json") + except ValidationError: + return recipe + + +def draft(scan: WorkbookScan, grids: dict[str, Grid], filename: str, *, + validate: ValidateFn, dry_run: DryRunFn | None = None) -> Draft: + """规则起草。不抛异常；认不出时 recipe=None、complete=False、failures 写原因。 + + filename 只为与接口层的调用一致：文件名旁证一律写 cross_check="filename"，由执行器在每期导入时核对， + 起草时不看文件名（文件名每期都变，不能进配方）。 + """ + try: + an = _analyze(scan, grids) + except Exception as e: # noqa: BLE001 - 起草器出任何错都只能变成「起草不完整」 + _log_failure("rule draft", e) + return Draft(None, False, "rules", failures=[_FAIL_TEXT["draft"]], + failures_for_model=[_FAIL_TEXT["draft_model"]]) + failures = [f.human for f in an.failures] + for_model = [f.model for f in an.failures] + recipe = _full_form(an.recipe) if an.recipe is not None else None + extraction: Extraction | None = None + if recipe is None and not failures: + failures.append("没有找到有内容的可见工作表") + for_model.append("没有找到有内容的可见工作表") + if recipe is not None: + h, m, parsed = check_recipe(recipe, an.facts, "rules", validate) + failures += h + for_model += m + if parsed is not None and not h and dry_run is not None: + extraction, h, m = run_dry(parsed, dry_run, grids) + failures += h + for_model += m + cards: list[Card] = [] + questions: list[Question] = [] + if recipe is not None: + try: + cards, questions = questions_for(recipe, an.facts, grids, extraction=extraction) + except Exception: # noqa: BLE001 - 卡片是辅助信息，出错不能拖垮起草 + cards, questions = [], [] + # 给了干跑时 questions_for 已经按执行器的 skipped_hidden 出过这些卡片：同 id 的不再补，免得界面上 key 重复 + have = {c.id for c in cards} + extra = [_hidden_sheet_card(name) for name in an.hidden_sheets if f"hidden_sheet:{name}" not in have] + at = next((i for i, c in enumerate(cards) if c.id == "mode"), len(cards)) + cards[at:at] = extra + return Draft(recipe=recipe, complete=recipe is not None and not failures, origin="rules", cards=cards, + questions=questions, facts=an.facts, failures=_dedupe(failures), + failures_for_model=_dedupe(for_model)) + + +#: 起草器、注入的校验和干跑自己出错时的失败原因。异常类名只进日志：界面上不露内部实现，报错写「发生了什么 + +#: 下一步」（terms.ts 的规范，后端返回给界面的文案同样适用） +_FAIL_TEXT = { + "draft": "规则起草未能完成：分析表格版式时出错。请在配方面板中粘贴一份完整的配方", # 没有半成品：面板里没有表单可填 + "draft_model": "规则起草未能完成，没有半成品配方", + "validate": "配方检查未能完成：检查时出错。请在配方面板中修改配方后重试", + "validate_model": "配方检查未能完成", + "dry_run": "按草稿检查表格未能完成：检查时出错。请在配方面板中修改配方后重试", + "dry_run_model": "按草稿检查表格未能完成", +} + + +def _log_failure(what: str, e: BaseException) -> None: + """异常只进日志：记类名和出错的位置，不记异常消息（消息里可能带着格子的值）。""" + tb = traceback.extract_tb(e.__traceback__) + where = f"{os.path.basename(tb[-1].filename)}:{tb[-1].lineno}" if tb else "?" + logger.warning("%s failed: %s at %s", what, type(e).__name__, where) + + +def _hidden_sheet_card(name: str) -> Card: + return Card(f"hidden_sheet:{name}", f"工作表「{name}」是隐藏的，默认不导入", + "隐藏的工作表不读取内容；需要导入时请在 Excel 里取消隐藏后重新上传") + + +def _dedupe(items: list[str]) -> list[str]: + seen: list[str] = [] + for x in items: + if x not in seen: + seen.append(x) + return seen + + +def check_recipe(recipe: dict[str, Any], facts: DraftFacts | None, origin: str, + validate: ValidateFn) -> tuple[list[str], list[str], Recipe | None]: + """静态校验：(给人看的问题, 给模型看的问题, 通过时的 Recipe)。校验器出错也算问题，不抛。""" + try: + parsed, problems = validate(recipe, facts, origin) + except Exception as e: # noqa: BLE001 + _log_failure("recipe validation", e) + return [_FAIL_TEXT["validate"]], [_FAIL_TEXT["validate_model"]], None + if problems: + return ([p.message for p in problems], + [MODEL_LINE_PROMPT["static"].format(path=p.path or "/", message=p.message) + for p in problems[:MODEL_LINES_MAX]], None) + if parsed is None: + return ["静态校验没有给出配方"], ["静态校验没有给出配方"], None + return [], [], parsed + + +def _dry_outcome(ext: Extraction, grids: dict[str, Grid]) -> tuple[Extraction, list[str], list[str]]: + structure = [p for p in ext.problems if p.category == "structure"] + return ext, [p.message for p in structure], problem_lines_for_model(structure, grids) + + +def run_dry(recipe: Recipe, dry_run: DryRunFn, + grids: dict[str, Grid]) -> tuple[Extraction | None, list[str], list[str]]: + """干跑：(Extraction, 结构类问题的人话, 同一组问题的模型版)。只有 structure 类算起草不完整。""" + try: + ext = dry_run(recipe) + except Exception as e: # noqa: BLE001 + _log_failure("dry run", e) + return None, [_FAIL_TEXT["dry_run"]], [_FAIL_TEXT["dry_run_model"]] + return _dry_outcome(ext, grids) + + +def _is_async(fn: Any) -> bool: + return inspect.iscoroutinefunction(fn) or inspect.iscoroutinefunction(getattr(fn, "__call__", None)) + + +async def run_dry_async(recipe: Recipe, dry_run: DryRunFn | AsyncDryRunFn, + grids: dict[str, Grid]) -> tuple[Extraction | None, list[str], list[str]]: + """run_dry 的协程版（AI 起草在事件循环上用）：协程函数直接 await（WP-5c 注入的那份自己占解析名额、 + 进线程池）；普通函数放进线程池跑，不在事件循环上执行。出错同 run_dry，变成一条起草失败原因。""" + try: + if _is_async(dry_run): + ext = await dry_run(recipe) # type: ignore[misc] + else: + ext = await asyncio.to_thread(dry_run, recipe) + if inspect.isawaitable(ext): + ext = await ext + except Exception as e: # noqa: BLE001 + _log_failure("dry run", e) + return None, [_FAIL_TEXT["dry_run"]], [_FAIL_TEXT["dry_run_model"]] + return _dry_outcome(ext, grids) # type: ignore[arg-type] + + +# ========================================================================== +# 给模型看的问题文字（起草失败原因、AI 修订回灌共用） +# ========================================================================== + + +def _cell_tokens(value: Any) -> list[str]: + if isinstance(value, bool) or value is None: + return [] + if _num(value): + out = {str(value)} + if isinstance(value, float) and value.is_integer(): + out.add(str(int(value))) + try: + out.add(f"{value:,}") + if isinstance(value, float) and value.is_integer(): + out.add(f"{int(value):,}") + except (TypeError, ValueError): + pass + return sorted(out, key=len, reverse=True) + if isinstance(value, str) and P.looks_numeric_text(value): + return sorted({value.strip(), canon(value)}, key=len, reverse=True) + return [] + + +def scrub_cells(text: str, cells: list[str], grids: dict[str, Grid]) -> str: + """把这条问题自己的格子里的数（数字格的值、像数的文字）在 text 里出现的地方换成 <值>。 + + 只按数的边界换（前面不是字母数字、后面不是数字），坐标「C5」里的 5 不会被误换。 + """ + for coord in cells: + sheet, _, ref = coord.rpartition("!") + grid = grids.get(sheet) + if grid is None or not ref: + continue + try: + r, c = parse_ref(ref.replace("$", "").split(":")[0]) + except (ValueError, IndexError): + continue + cell = grid.get(r, c) + if cell is None: + continue + for tok in _cell_tokens(cell.value): + if tok: + text = re.sub(r"(?", text) + return text + + +def problem_lines_for_model(problems: list[Problem], grids: dict[str, Grid]) -> list[str]: + """干跑问题的模型版：code + model_message + 坐标；model_message 为空时只有 code 和坐标，绝不退回 message。""" + out: list[str] = [] + for p in problems[:MODEL_LINES_MAX]: + cells = "、".join(p.cells[:20]) + line = (MODEL_LINE_PROMPT["dry_msg"].format(code=p.code, message=p.model_message) if p.model_message + else MODEL_LINE_PROMPT["dry"].format(code=p.code)) + if cells: + line = MODEL_LINE_PROMPT["cells"].format(line=line, cells=cells) + out.append(scrub_cells(line, p.cells, grids)) + return out + + +# ========================================================================== +# 卡片与问题（任意配方） +# ========================================================================== + + +@dataclass +class _CrossLoc: + axis_row: int + label_col: int + c1: int + c2: int + dates: list[int] + seg_rows: dict[str, list[int]] = field(default_factory=dict) + title_rows: dict[str, int] = field(default_factory=dict) + data_rows: list[int] = field(default_factory=list) + region: set[tuple[int, int]] = field(default_factory=set) + + +@dataclass +class _ListLoc: + header_rows: list[int] + cols: list[int] + rows: list[int] + total_row: int | None + region: set[tuple[int, int]] = field(default_factory=set) + + +def _find_grid(name: str, grids: dict[str, Grid]) -> Grid | None: + for g in grids.values(): + if P.match_key(g.sheet) == P.match_key(name): + return g + live = [g for g in grids.values() if g.cells] + return live[0] if len(live) == 1 else None + + +def _locate_crosstab(block: CrosstabBlock, grid: Grid) -> _CrossLoc | None: + best: tuple[int, list[int]] | None = None + for r in grid.row_numbers(): + cols = [c for c, cell in grid.row(r) if P.month_day_or_date(cell.value) is not None] + if len(cols) < block.axis.find.min: + continue + right = [c for c, _ in grid.row(r) if c >= min(cols) - 1] + if len(cols) * 2 <= len(right): + continue + if best is None or len(cols) > len(best[1]): + best = (r, cols) + if best is None: + return None + axis_row, dates = best + c1, c2 = min(dates), max(dates) + loc = _CrossLoc(axis_row, c1 - 1, c1, c2, dates) + if loc.label_col < 1: + return None + + def parse_key(seg: Any, label: str) -> str | None: + if isinstance(seg, MeasuresSegment): + return P.match_key(label) + if isinstance(seg, DimensionSegment): + if seg.dim.parser == "hour_range": + hr = P.hour_range(label) + return hr.canonical if hr else None + return P.text_label(label) + hr = P.hour_range_total(label) + return hr.canonical if hr else None + + expects: list[tuple[Any, set[str]]] = [] + for seg in block.segments: + keys = {k for e in seg.labels.expect if (k := parse_key(seg, e)) is not None} + expects.append((seg, keys)) + kinds: dict[int, str] = {} + labels: dict[int, str] = {} + for r in range(axis_row + 1, _bottom(grid) + 1): + label = _label_of(grid.get(r, loc.label_col)) + has = any(grid.get(r, c) is not None for c in range(c1, c2 + 1)) + if label is None: + kinds[r] = "orphan" if has else "blank" + continue + labels[r] = label + if has or any(parse_key(seg, label) in keys for seg, keys in expects): + kinds[r] = "data" + else: + kinds[r] = "title" + + def run_from(start: int, accept: Callable[[str], bool] | None = None, + stop: Callable[[str], bool] | None = None) -> list[int]: + out: list[int] = [] + r = start + while kinds.get(r) == "data": + lab = labels[r] + if stop is not None and stop(lab): + break + if accept is not None and not accept(lab): + break + out.append(r) + r += 1 + return out + + for seg, keys in expects: + rows: list[int] = [] + if isinstance(seg, DerivedSegment): + prev = loc.seg_rows.get(seg.locate.segment or "") + if prev: + rows = run_from(prev[-1] + 1, accept=lambda lab: P.hour_range_total(lab) is not None) + elif seg.locate.by == "section_title": + want = P.match_key(seg.locate.title or "") + tr = next((r for r, k in kinds.items() if k == "title" and P.match_key(labels[r]) == want), None) + if tr is not None: + loc.title_rows[seg.id] = tr + stop = ((lambda lab: P.hour_range_total(lab) is not None) + if isinstance(seg, DimensionSegment) and seg.stop_parser else None) + rows = run_from(tr + 1, stop=stop) + else: + first = next((r for r, k in kinds.items() if k == "data" and parse_key(seg, labels[r]) in keys), None) + if first is not None: + start = first + while kinds.get(start - 1) == "data": + start -= 1 + rows = run_from(start) + loc.seg_rows[seg.id] = rows + if not isinstance(seg, DerivedSegment): + loc.data_rows.extend(rows) + loc.region.update((axis_row, c) for c in range(loc.label_col, c2 + 1)) + for rows in loc.seg_rows.values(): + for r in rows: + loc.region.update((r, c) for c in range(loc.label_col, c2 + 1)) + for tr in loc.title_rows.values(): + loc.region.add((tr, loc.label_col)) + return loc + + +def _locate_list(block: ListBlock, grid: Grid, taken: set[int]) -> _ListLoc | None: + n = block.header_rows + want = [P.match_key(c.header) for c in block.columns] + start = grid.row_numbers()[0] if grid.row_numbers() else 1 + if block.after_title: + key = P.match_key(block.after_title) + hit = next((r for r in grid.row_numbers() if any(P.match_key(_label_of(cell) or "") == key + for _, cell in grid.row(r))), None) + if hit is not None: + start = hit + 1 + best: tuple[int, list[int], dict[int, str]] | None = None + best_hits = 0 + want_set = set(want) + for r in grid.row_numbers(): + if r < start + n - 1 or r in taken: + continue + if n == 1 and not any(P.match_key(_label_of(cell) or "") in want_set for _, cell in grid.row(r)): + continue + hrows = list(range(r - n + 1, r + 1)) + cols = sorted({c for hr in hrows for c, _ in grid.row(hr)}) + texts = _header_texts(grid, hrows, cols) + keys = {P.match_key(t): c for c, t in texts.items()} + hits = sum(1 for k in want if k in keys) + if hits * 2 > len(want) and hits > best_hits: + best, best_hits = (r, hrows, texts), hits + if best is None: + return None + r, hrows, texts = best + keys = {P.match_key(t): c for c, t in texts.items()} + anchor = [keys[k] for k in want if k in keys] + lo, hi = min(anchor), max(anchor) + while lo - 1 in texts: + lo -= 1 + while hi + 1 in texts: + hi += 1 + cols = list(range(lo, hi + 1)) + rows: list[int] = [] + total = None + bottom = _bottom(grid) + rr = r + 1 + pick = P.match_key(block.rows.total_row.pick) if block.rows.total_row else None + while rr <= bottom: + if all(grid.get(rr, c) is None for c in cols): + if block.rows.blank_rows == "skip": + rr += 1 + continue + break + if pick and P.match_key(_label_of(grid.get(rr, cols[0])) or "").startswith(pick): + total = rr + break + rows.append(rr) + rr += 1 + loc = _ListLoc(hrows, cols, rows, total) + for row in [*hrows, *rows, *([total] if total else [])]: + loc.region.update((row, c) for c in cols) + return loc + + +def _registered_relation(fact: Fact, recipe: Recipe, rid: str) -> dict[str, Any] | None: + """按事实和配方拼出「登记」时的整条关系（sum_eq / not_comparable）；拼不出来时 None。""" + measures: list[tuple[MeasuresSegment, CrosstabBlock]] = [] + dims: list[DimensionSegment] = [] + for sheet in recipe.sheets: + for block in sheet.blocks: + if isinstance(block, CrosstabBlock): + for seg in block.segments: + if isinstance(seg, MeasuresSegment): + measures.append((seg, block)) + elif isinstance(seg, DimensionSegment): + dims.append(seg) + + def measure_col(seg: MeasuresSegment, label: str) -> str | None: + keys = {P.match_key(e): e for e in seg.labels.expect} + e = keys.get(P.match_key(label)) + return seg.measures.get(e) if e is not None else None + + def find(seg_id: str, labels: list[str]) -> tuple[MeasuresSegment, CrosstabBlock] | None: + for seg, block in measures: + if seg.id == seg_id: + return seg, block + for seg, block in measures: + if all(measure_col(seg, lab) for lab in labels): + return seg, block + return None + + d = fact.detail + if fact.kind == "sum_eq": + hit = find(d.get("segment", ""), [d.get("total", ""), *d.get("parts", [])]) + if hit is None: + return None + seg, _ = hit + total = measure_col(seg, d.get("total", "")) + parts = [measure_col(seg, p) for p in d.get("parts", [])] + if total is None or None in parts: + return None + return {"id": rid, "kind": "sum_eq", "table": seg.table, "total": total, "parts": parts, "claims": fact.id} + hit = find(d.get("b_segment", ""), [d.get("b", "")]) + a = next((s for s in dims if s.id in d.get("a", [])), None) + if hit is None or a is None: + return None + seg, block = hit + b_col = measure_col(seg, d.get("b", "")) + if b_col is None: + return None + return {"id": rid, "kind": "not_comparable", "a": {"table": a.table, "value": a.value}, + "b": {"table": seg.table, "value": b_col}, "by": block.axis.name, "claims": fact.id} + + +def _add_ops(recipe: dict[str, Any], parts: list[Any], value: Any) -> list[dict[str, Any]]: + """把 parts 指向的成员设成 value：父对象都在时一条 add；缺了哪一层就在那一层 add 一个嵌套对象。""" + node: Any = recipe + for i, key in enumerate(parts[:-1]): + nxt = node[key] if isinstance(node, list) and isinstance(key, int) and key < len(node) else ( + node.get(key) if isinstance(node, dict) else None) + if nxt is None: + nested: Any = value + for k in reversed(parts[i + 1:]): + nested = {k: nested} + return [{"op": "add", "path": _ptr(*parts[: i + 1]), "value": nested}] + node = nxt + return [{"op": "add", "path": _ptr(*parts), "value": value}] + + +def questions_for(recipe: dict[str, Any], facts: DraftFacts, grids: dict[str, Grid], *, + extraction: Extraction | None = None) -> tuple[list[Card], list[Question]]: + """给任意配方生成建议卡片和待确认问题。配方不合法（pydantic 不收）时只给导入模式卡片。""" + cards: list[Card] = [] + questions: list[Question] = [] + try: + parsed = Recipe.model_validate(recipe) + except ValidationError: + return [_mode_card()], [] + relations = recipe.get("relations") if isinstance(recipe.get("relations"), list) else [] + claims = {r.claims: i for i, r in enumerate(parsed.relations) if r.claims} + next_rid = [max([int(r.id[1:]) for r in parsed.relations] + [0])] + facts_left = list(facts.facts) + ext = extraction + used: set[str] = set() + + def relation_question(f: Fact) -> Question: + idx = claims.get(f.id) + rid = parsed.relations[idx].id if idx is not None else None + if rid is None: + next_rid[0] += 1 + rid = f"R{next_rid[0]}" + current = parsed.relations[idx] if idx is not None else None + reg = (relations[idx] if current is not None and current.kind in ("sum_eq", "not_comparable") + and idx < len(relations) else _registered_relation(f, parsed, rid)) + dismissed = {"id": rid, "kind": "dismissed", "claims": f.id, "reason": REASON_SLOT} + + def put(value: dict[str, Any]) -> list[dict[str, Any]]: + if idx is not None: + return [{"op": "replace", "path": _ptr("relations", idx), "value": value}] + if "relations" not in recipe: + return [{"op": "add", "path": "/relations", "value": [value]}] + return [{"op": "add", "path": "/relations/-", "value": value}] + + verb = "每期核对" if f.kind == "sum_eq" else "口径不同" + options = [] + effects: dict[str, list[dict[str, Any]]] = {} + if reg is not None: + options.append({"value": "register", "label": "登记", "needs_reason": False}) + effects["register"] = put(reg) + options.append({"value": "dismiss", "label": "不登记", "needs_reason": True}) + effects["dismiss"] = put(dismissed) + return Question(f"q_relation:{f.id}", f"{f.text}：要登记为{verb}吗", options, None, effects) + + for si, sheet in enumerate(parsed.sheets): + grid = _find_grid(sheet.match.name, grids) + name = grid.sheet if grid is not None else sheet.match.name + region: set[tuple[int, int]] = set() + cross_locs: dict[int, _CrossLoc] = {} + list_locs: dict[int, _ListLoc] = {} + taken: set[int] = set() + if grid is not None: + for bi, block in enumerate(sheet.blocks): + if isinstance(block, CrosstabBlock): + cl = _locate_crosstab(block, grid) + if cl is not None: + cross_locs[bi] = cl + region |= cl.region + else: + ll = _locate_list(block, grid, taken) + if ll is not None: + list_locs[bi] = ll + region |= ll.region + taken.update(ll.header_rows) + sheet_facts = [f for f in facts_left if P.match_key(f.sheet) == P.match_key(name)] + facts_left = [f for f in facts_left if f not in sheet_facts] + + for bi, block in enumerate(sheet.blocks): + if isinstance(block, CrosstabBlock): + cards_q = _crosstab_cards(si, bi, sheet, block, cross_locs.get(bi), grid, name, recipe, parsed, + sheet_facts, relation_question, ext, region, used) + else: + cards_q = _list_cards(si, bi, sheet, block, list_locs.get(bi), grid, name, recipe, ext, used) + cards.extend(cards_q[0]) + questions.extend(cards_q[1]) + if not any(isinstance(b, CrosstabBlock) for b in sheet.blocks): + facts_left.extend(sheet_facts) # 没有交叉表来认领：放到最后统一出问题 + # 列表工作表也可能有统计期 + c, q = _period_cards(si, sheet, grid, name, region, ext) + cards.extend(c) + questions.extend(q) + c2 = _outside_card(sheet, grid, name, region, ext) + cards.extend(c2) + # 隐藏行列 + hq = _hidden_question(si, sheet, grid, name, region, ext) + if hq is not None: + cards.append(hq[0]) + questions.append(hq[1]) + # 干跑报的「没有去处」的格 + if ext is not None: + bad = [p for p in ext.problems if p.code in ("cell_unclaimed", "row_unclaimed") + and any(c.startswith(f"{name}!") for c in p.cells)] + if bad: + cells = [c for p in bad for c in p.cells][:CARD_CELLS_MAX] + cards.append(Card(f"unclaimed:{sheet.id}", "有单元格没有去处", + "；".join(p.message for p in bad[:5]), cells)) + # 没落到任何工作表的事实（工作表改名等）也要能认领 + for f in facts_left: + q = relation_question(f) + questions.append(q) + cards.append(Card(f"fact:{f.id}", f.text, "系统发现的关系需要逐条认领：登记，或写明理由不登记", + [], q.id)) + if ext is not None and ext.sheets.skipped_hidden: + for nm in _dedupe([item.get("sheet", "") for item in ext.sheets.skipped_hidden]): + cards.append(_hidden_sheet_card(nm)) + cards.append(_mode_card()) + return cards, questions + + +def _qid(base: str, block_id: str, used: set[str]) -> str: + """问题 id 在整份配方内唯一：同一个占位符出现在两个块里时，后面的带上块 id。""" + qid = base if base not in used else f"{base}@{block_id}" + used.add(qid) + return qid + + +def _mode_card() -> Card: + return Card("mode", "导入模式：每期替换", + "每次上传新一期时，整份替换上一期的数据（按期累积将在后续版本提供）") + + +def _period_cells(grid: Grid | None, name: str, region: set[tuple[int, int]], + ext: Extraction | None) -> list[str]: + """统计期取自哪些格：有干跑就用执行器的结论；否则在区域外找能解析出区间的格。""" + if ext is not None: + if ext.period is None or ext.period.source != "cells": + return [] + return [c if "!" in c else f"{name}!{c}" for c in ext.period.cells + if "!" not in c or c.startswith(f"{name}!")] + if grid is None: + return [] + out = [] + for (r, c), cell in sorted(grid.cells.items()): + if (r, c) not in region and isinstance(cell.value, str) and P.cn_date_range(cell.value) is not None: + out.append(_coord(name, r, c)) + return out + + +def _cell_text(grid: Grid | None, coord: str) -> str: + if grid is None: + return "" + try: + r, c = parse_ref(coord.rpartition("!")[2]) + except ValueError: + return "" + cell = grid.get(r, c) + return _shown(cell.value) if cell is not None and cell.value is not None else "" + + +def _period_cards(si: int, sheet: Any, grid: Grid | None, name: str, region: set[tuple[int, int]], + ext: Extraction | None) -> tuple[list[Card], list[Question]]: + if not sheet.context: + return [], [] + ctx = sheet.context[0] + cells = _period_cells(grid, name, region, ext) + if cells: + shown = "、".join(c.rpartition("!")[2] for c in cells[:5]) + texts = "；".join(f"{c.rpartition('!')[2]}「{_cell_text(grid, c)}」" for c in cells[:5]) + reason = f"能解析出日期区间的格：{texts}" + if ctx.cross_check == "filename": + reason += "。另与文件名里的日期区间核对" + return [Card(f"period:{sheet.id}", f"统计期取自 {shown}", reason, cells[:CARD_CELLS_MAX])], [] + q = Question(f"q_period:{sheet.id}" if si else "q_period", "表格里找不到统计期，每次导入时怎么确定", + [{"value": "human", "label": "每期人工录入", "needs_reason": False}], "human", {"human": []}) + return [Card(f"period:{sheet.id}", "未找到统计期：每期人工录入", + "表格里没有能解析出日期区间的文字，每次导入时需要人工录入本期的统计期", [], q.id)], [q] + + +def _outside_card(sheet: Any, grid: Grid | None, name: str, region: set[tuple[int, int]], + ext: Extraction | None) -> list[Card]: + items: list[tuple[str, str, bool]] = [] # (坐标, 全文, 含数字) + if ext is not None: + for o in ext.outside_text: + if o.sheet == name: + items.append((f"{o.sheet}!{o.cell}" if "!" not in o.cell else o.cell, o.text, + o.kind == "text_digits")) + elif grid is not None: + for (r, c), cell in sorted(grid.cells.items()): + if (r, c) in region or cell.formula is not None: + continue + v = cell.value + if isinstance(v, str) and P.cn_date_range(v) is not None and P.is_pure_period_cell( + v, sheet.context[0].prefer_prefix if sheet.context else None): + continue + kind = P.classify_outside(v) + if kind == "number": + continue + items.append((_coord(name, r, c), _shown(v), kind == "text_digits")) + if not items: + return [] + refs = [i[0].rpartition("!")[2] for i in items] + head = "、".join(refs[:5]) + (f" 等 {len(refs)} 格" if len(refs) > 5 else "") + reason = "；".join(f"{i[0].rpartition('!')[2]}「{i[1][:40]}」" for i in items[:5]) + reason += "：不在任何数据区内，记录在回执里，不导入" + if any(i[2] for i in items): + reason += "。其中含数字的文字需要在提交时逐条确认" + return [Card(f"outside:{sheet.id}", f"{head} 是区域外文字", reason, [i[0] for i in items][:CARD_CELLS_MAX])] + + +def _hidden_question(si: int, sheet: Any, grid: Grid | None, name: str, region: set[tuple[int, int]], + ext: Extraction | None) -> tuple[Card, Question] | None: + rows: list[int] = [] + cols: list[int] = [] + if ext is not None and name in ext.hidden: + rows = list(ext.hidden[name].get("rows") or []) + cols = list(ext.hidden[name].get("cols") or []) + elif grid is not None: + rows_in = {r for r, _ in region} + cols_in = {c for _, c in region} + rows = sorted(r for r in grid.hidden_rows if r in rows_in) + cols = sorted(c for c in grid.hidden_cols if c in cols_in) + if not rows and not cols: + return None + parts = [] + if rows: + parts.append(f"隐藏行：{_rows_text([int(r) for r in rows])}") + if cols: + parts.append("隐藏列：" + "、".join(col_letter(c) if isinstance(c, int) else str(c) for c in cols[:10])) + options = [{"value": "reject_if_any", "label": "拒收（文件里不应有隐藏的行列）", "needs_reason": False}, + {"value": "include", "label": "照常导入隐藏的行列", "needs_reason": False}] + policy = {"reject_if_any": {"rows": "reject_if_any", "cols": "reject_if_any"}, + "include": {"rows": "include", "cols": "include"}} + if rows: + options.append({"value": "exclude", "label": "跳过隐藏的行", "needs_reason": False}) + policy["exclude"] = {"rows": "exclude", "cols": "include" if cols else "reject_if_any"} + effects = {k: [{"op": "add", "path": _ptr("sheets", si, "hidden"), "value": v}] for k, v in policy.items()} + q = Question(f"q_hidden:{name}", f"工作表「{name}」的数据区里有隐藏的行或列，怎么处理", options, None, effects) + card = Card(f"hidden:{name}", "数据区里有隐藏的行或列", "；".join(parts) + "。请选择怎么处理", [], q.id) + return card, q + + +def _unit_cards(labels: list[tuple[str, str]], sheet_id: str) -> list[Card]: + out = [] + for label, where in labels: + _, unit = P.split_unit_suffix(label) + if unit is not None and not P.unit_known(unit): + out.append(Card(f"unit_unknown:{sheet_id}:{label}", f"「{label}」的单位「{unit}」不在单位词表中", + f"{where}：列名保留原文，单位不写进单位字段")) + return out + + +def _crosstab_cards(si: int, bi: int, sheet: Any, block: CrosstabBlock, loc: _CrossLoc | None, grid: Grid | None, + name: str, recipe: dict[str, Any], parsed: Recipe, sheet_facts: list[Fact], + relation_question: Callable[[Fact], Question], ext: Extraction | None, + region: set[tuple[int, int]], used: set[str]) -> tuple[list[Card], list[Question]]: + cards: list[Card] = [] + questions: list[Question] = [] + lc = loc.label_col if loc is not None else None + + def label_cells(rows: list[int]) -> list[str]: + return _coords(name, [(r, lc) for r in rows]) if lc is not None else [] + + # 1 日期表头 + axis_first = axis_last = "" + count = 0 + axis_cells: list[str] = [] + if ext is not None and (ax := next((a for a in ext.axes if a.block == block.id), None)) is not None: + axis_first, axis_last, count = ax.first, ax.last, ax.count + row = ax.row + axis_cells = [_coord(name, row, c) for c in (loc.dates if loc else [])][:CARD_CELLS_MAX] + elif loc is not None and grid is not None: + row = loc.axis_row + count = len(loc.dates) + axis_first = _shown(grid.get(row, loc.dates[0]).value) # type: ignore[union-attr] + axis_last = _shown(grid.get(row, loc.dates[-1]).value) # type: ignore[union-attr] + axis_cells = _coords(name, [(row, c) for c in loc.dates]) + else: + row = 0 + measures = [s for s in block.segments if isinstance(s, MeasuresSegment)] + has_unit = any(P.unit_known(P.split_unit_suffix(e)[1]) for s in measures for e in s.labels.expect) + if row: + reason = f"第 {row} 行有 {count} 格日期（{axis_first} 至 {axis_last}）" + if lc is not None: + reason += f"，行标签在 {col_letter(lc)} 列" + if block.axis.year_from and grid is not None and loc is not None and any( + (cell := grid.get(loc.axis_row, c)) is not None + and (d := P.month_day_or_date(cell.value)) is not None and d.year is None for c in loc.dates): + reason += "；日期只写了月日，年份取自统计期" + if has_unit: + reason += "；行标签末尾括号里的单位拆进单位字段，列名去掉单位后缀" + cards.append(Card(f"axis:{block.id}", f"第 {row} 行是日期表头，按交叉表导入", reason, axis_cells)) + else: + cards.append(Card(f"axis:{block.id}", "按交叉表导入", "表格里没有找到符合配方的日期表头行，试运行时会报出具体原因")) + cards.extend(_unit_cards([(e, f"分段「{s.id}」") for s in measures for e in s.labels.expect], sheet.id)) + + # 2 统计期 + c, q = _period_cards(si, sheet, grid, name, region, ext) + cards.extend(c) + questions.extend(q) + + # 3 分段标题 + dims = [s for s in block.segments if isinstance(s, DimensionSegment)] + titled = [s for s in dims if s.locate.by == "section_title"] + if titled: + rows = [loc.title_rows[s.id] for s in titled if loc is not None and s.id in loc.title_rows] + refs = "、".join(_a1(r, lc) for r in rows) if lc is not None and rows else "" + title = f"分段标题 {refs}" if refs else "分段标题" + reason = "".join(f"「{s.locate.title}」" for s in titled) + "下面的行各成一段，按标题定位" + cards.append(Card(f"titles:{block.id}", title, reason, label_cells(rows))) + cards.extend(_unit_cards([(s.locate.title or "", f"分段「{s.id}」") for s in titled], sheet.id)) + + # 4 sum_eq + for f in sheet_facts: + if f.kind != "sum_eq": + continue + q = relation_question(f) + questions.append(q) + cards.append(Card(f"fact:{f.id}", f.text, + "建议登记为每期核对：以后每期导入都会逐行核对这个等式；如果只是巧合，可以选择不登记并写明理由", + label_cells(list(f.detail.get("rows", []))), q.id)) + + # 5 合表 + by_table: dict[str, list[DimensionSegment]] = {} + for s in dims: + by_table.setdefault(s.table, []).append(s) + for tname, segs in by_table.items(): + if len(segs) < 2: + continue + picks = ["、".join(v.pick for v in s.const.values()) or s.id for s in segs] + consts = sorted({k for s in segs for k in s.const}) + reason = f"这几段的{segs[0].dim.name}互不重叠，写进同一张表" + if consts: + reason += "，用列" + "、".join(f"「{k}」" for k in consts) + "区分，取值从分段标题的候选词里选：" + "、".join(picks) + rows = [loc.title_rows[s.id] for s in segs if loc is not None and s.id in loc.title_rows] + cards.append(Card(f"merge:{tname}", f"{'、'.join(picks)}合进一张表「{tname}」", reason, label_cells(rows))) + + # 6 占位符（按下标从大到小出问题：remove 重放不错位） + phs = list(enumerate(block.values.placeholders)) + ph_cards: list[Card] = [] + ph_qs: list[Question] = [] + for pi, ph in phs: + key = canon(ph.text) + cells: list[tuple[int, int]] = [] + if loc is not None and grid is not None: + for r in loc.data_rows: + for cc in range(loc.c1, loc.c2 + 1): + cell = grid.get(r, cc) + if cell is not None and isinstance(cell.value, str) and canon(cell.value) == key: + cells.append((r, cc)) + n = ext.placeholders.get(ph.text, len(cells)) if ext is not None else len(cells) + qid = _qid(f"q_placeholder:{ph.text}", block.id, used) + ph_cards.append(Card(f"placeholder:{block.id}:{ph.text}", f"「{ph.text}」表示无数据吗", + f"数值区有 {n} 格「{ph.text}」：确认后这些格存为空值；" + "如果它不是「无数据」的意思，这份文件无法按当前配方导入", + _coords(name, cells), qid)) + path = ["sheets", si, "blocks", bi, "values", "placeholders", pi] + ph_qs.append(Question(qid, f"「{ph.text}」表示无数据吗", + [{"value": "null", "label": "是，存为空值", "needs_reason": False}, + {"value": "reject", "label": "不是，不导入", "needs_reason": False}], + None, {"null": [], "reject": [{"op": "remove", "path": _ptr(*path)}]})) + cards.extend(ph_cards) + questions.extend(reversed(ph_qs)) + + # 7 合计段 + for s in block.segments: + if not isinstance(s, DerivedSegment): + continue + rows = [] + if ext is not None: + lab = next((x for x in ext.labels if x.segment == s.id), None) + rows = list(lab.rows) if lab is not None else [] + if not rows and loc is not None: + rows = loc.seg_rows.get(s.id, []) + where = _rows_text(rows) or f"分段「{s.id}」" + first = s.labels.expect[0] if s.labels.expect else "" + reason = f"「{first}」等 {len(s.labels.expect)} 行按时段区间与上方明细逐日核对" + reason += f"，原值另存表「{s.keep_as.table}」" if s.keep_as else "，只核对不另存" + cards.append(Card(f"derived:{s.id}", f"{where}是合计，改作核对并另存" if s.keep_as else f"{where}是合计，改作核对", + reason, label_cells(rows))) + + # 8 not_equal_sum + for f in sheet_facts: + if f.kind != "not_equal_sum": + continue + q = relation_question(f) + questions.append(q) + a = next((s for s in dims if s.id in f.detail.get("a", [])), None) + bseg = next((s for s in measures if s.id == f.detail.get("b_segment")), None) + bcol = None + if bseg is not None: + keys = {P.match_key(e): e for e in bseg.labels.expect} + e = keys.get(P.match_key(f.detail.get("b", ""))) + bcol = bseg.measures.get(e) if e else None + unit = "天" if block.axis.type == "DATE" else "组" + if a is not None and bcol: + title = (f"各{a.dim.name}之和与{bcol} {f.detail.get('checked', 0)} {unit}中 " + f"{f.detail.get('equal', 0)} {unit}相等") + else: + title = f.text + rows = [loc.title_rows[s.id] for s in dims if loc is not None and s.id in loc.title_rows + and s.id in f.detail.get("a", [])] + cards.append(Card(f"fact:{f.id}", title, + "两者的口径不同（例如统计范围不同），建议登记为口径不同：以后每期只记录相等的天数，不作核对；" + "如果判断为巧合，可以选择不登记并写明理由", label_cells(rows), q.id)) + + # 9 区域外文字 + cards.extend(_outside_card(sheet, grid, name, region, ext)) + + # 数值区的空格、公式（规格没有给默认处理：保守地按拒收起草，出问题让人选） + if loc is not None and grid is not None: + blanks = [(r, cc) for r in loc.data_rows for cc in range(loc.c1, loc.c2 + 1) if grid.get(r, cc) is None] + formulas = [(r, cc) for r in loc.data_rows for cc in range(loc.c1, loc.c2 + 1) + if (cell := grid.get(r, cc)) is not None and cell.formula is not None] + vpath = ["sheets", si, "blocks", bi, "values"] + if blanks: + q = Question(f"q_blank:{block.id}", f"数值区有 {len(blanks)} 个空格，怎么处理", + [{"value": "null", "label": "存为空值", "needs_reason": False}, + {"value": "reject", "label": "拒收（文件里不应有空格）", "needs_reason": False}], None, + {"null": _add_ops(recipe, [*vpath, "blank"], "null"), + "reject": _add_ops(recipe, [*vpath, "blank"], "reject")}) + questions.append(q) + cards.append(Card(f"blank:{block.id}", f"数值区有 {len(blanks)} 个空格", + f"{_a1_list(blanks)} 是空的：存为空值，还是拒收", _coords(name, blanks), q.id)) + if formulas: + q = Question(f"q_formula:{block.id}", f"数值区有 {len(formulas)} 个公式格，怎么处理", + [{"value": "accept_cached", "label": "按保存值导入", "needs_reason": False}, + {"value": "reject", "label": "拒收（可能是没认出来的合计）", "needs_reason": False}], None, + {"accept_cached": _add_ops(recipe, [*vpath, "formula"], "accept_cached"), + "reject": _add_ops(recipe, [*vpath, "formula"], "reject")}) + questions.append(q) + cards.append(Card(f"formula:{block.id}", f"数值区有 {len(formulas)} 个公式格", + f"{_a1_list(formulas)} 是公式：按保存值导入，还是拒收", _coords(name, formulas), q.id)) + return cards, questions + + +def _list_cards(si: int, bi: int, sheet: Any, block: ListBlock, loc: _ListLoc | None, grid: Grid | None, name: str, + recipe: dict[str, Any], ext: Extraction | None, used: set[str]) -> tuple[list[Card], list[Question]]: + cards: list[Card] = [] + questions: list[Question] = [] + headers = "、".join(f"「{c.header}」" for c in block.columns[:12]) + if loc is not None: + where = _rows_text(loc.header_rows) + rows = f"；{_rows_text(loc.rows)}是数据" if loc.rows else "" + cards.append(Card(f"list:{block.id}", f"{where}是表头，按列表导入（表「{block.table}」）", + f"列：{headers}{rows}", _coords(name, [(r, c) for r in loc.header_rows for c in loc.cols]))) + else: + cards.append(Card(f"list:{block.id}", f"按列表导入（表「{block.table}」）", + f"列：{headers}。表格里没有找到这组表头，试运行时会报出具体原因")) + cards.extend(_unit_cards([(c.header, f"列「{c.name}」") for c in block.columns], sheet.id)) + years = [c.header for c in block.columns if _YEAR.search(c.header)] + if years: + cards.append(Card(f"header_year:{block.id}", "表头里含年份：下一期年份变化时将无法匹配", + "、".join(f"「{h}」" for h in years[:6]) + "。下一期表头里的年份变了，需要修改配方")) + # 占位符 + ph_qs: list[Question] = [] + for pi, ph in enumerate(block.values.placeholders): + cells: list[tuple[int, int]] = [] + if loc is not None and grid is not None: + for r in loc.rows: + for c in loc.cols: + cell = grid.get(r, c) + if cell is not None and isinstance(cell.value, str) and canon(cell.value) == canon(ph.text): + cells.append((r, c)) + n = ext.placeholders.get(ph.text, len(cells)) if ext is not None else len(cells) + qid = _qid(f"q_placeholder:{ph.text}", block.id, used) + cards.append(Card(f"placeholder:{block.id}:{ph.text}", f"「{ph.text}」表示无数据吗", + f"数字列里有 {n} 格「{ph.text}」：确认后这些格存为空值", _coords(name, cells), qid)) + path = ["sheets", si, "blocks", bi, "values", "placeholders", pi] + ph_qs.append(Question(qid, f"「{ph.text}」表示无数据吗", + [{"value": "null", "label": "是，存为空值", "needs_reason": False}, + {"value": "reject", "label": "不是，不导入", "needs_reason": False}], + None, {"null": [], "reject": [{"op": "remove", "path": _ptr(*path)}]})) + questions.extend(reversed(ph_qs)) + if block.merged_data == "fill": + q = Question(f"q_merged:{block.id}", "数据区的文字列有纵向合并的单元格，怎么处理", + [{"value": "fill", "label": "用合并区左上格的值填充", "needs_reason": False}, + {"value": "reject", "label": "拒收", "needs_reason": False}], None, + {"fill": _add_ops(recipe, ["sheets", si, "blocks", bi, "merged_data"], "fill"), + "reject": _add_ops(recipe, ["sheets", si, "blocks", bi, "merged_data"], "reject")}) + questions.append(q) + cards.append(Card(f"merged:{block.id}", "文字列有纵向合并的单元格", + "合并区里每一行都按左上格的值填充（例如按地区分组时，地区只写在第一行）", [], q.id)) + if block.rows.blank_rows == "skip": + n = ext.blank_rows_skipped if ext is not None else None + extra = f"（本期 {n} 行）" if n else "" + cards.append(Card(f"blank_skip:{block.id}", "数据中间有空行，跳过继续", + f"空行下面的行与上方同类型，按同一张表继续读取{extra}；提交时需要确认")) + total = block.rows.total_row + if total is not None: + where = _rows_text([loc.total_row]) if loc is not None and loc.total_row else f"「{total.pick}」开头的行" + reason = f"「{total.pick}」行按列求和与上方明细核对" + reason += f"，原值另存表「{total.keep_as}」" if total.keep_as else "，只核对不另存" + cells = _coords(name, [(loc.total_row, loc.cols[0])]) if loc is not None and loc.total_row else [] + cards.append(Card(f"total_row:{block.id}", f"{where}是合计行，改作核对", reason, cells)) + return cards, questions diff --git a/backend/app/data/recipe_types.py b/backend/app/data/recipe_types.py new file mode 100644 index 00000000..f003444d --- /dev/null +++ b/backend/app/data/recipe_types.py @@ -0,0 +1,972 @@ +"""配方导入的契约（期 2，波次 0 由编排者提交）：配方 JSON 的模型、执行结果和各模块之间传递的数据结构。 + +各工作包之间只通过这里的类型和 P2-SPEC.md 里写明的函数签名交互。这里**只有声明**，没有业务逻辑 +（prose_problems / render_prose 两个小函数除外：静态校验、说明生成、AI 起草三处共用，必须是同一份）。 + +改这个文件要编排者拍板：字段的增删会同时影响执行器、核对、起草、接口和前端。 + +配方 JSON 的封闭性靠结构本身保证： +- 所有模型 extra="forbid"，没有正则字段，取值能枚举的一律 Literal； +- 常量只有 pick（从确认过的分段标题的候选词里选，recipe_parsers.candidate_words）； +- 占位符只能映射成空值（Placeholder 没有「映射成什么」这个字段）； +- 单位只能取自 recipe_parsers.UNITS（静态校验查）； +- 没有任何字段能写年份、统计期的值：统计期只能从格子解析，解析不出来时由人录入、记在导入记录上。 + +**键的格式**（执行器产出、确认项和差异卡消费，各包必须一致；P2-SPEC 第 7.5 节的 id 都按这张表拼）： +- 分段键 = 分段 id（Segment.id）；块键 = 块 id（Block.id）。两者都**在整份配方内唯一**（静态校验 + segment_id_duplicate / block_id_duplicate），所以不用带工作表前缀。 +- 工作表键 = **实际的工作表名**（与坐标「工作表!A1」里的一致）；只有 SheetsOut.matched / renamed 和确认项 + sheet_renamed: 用配方里的工作表 id。 +- 坐标 = 「实际工作表名!A1」；区域 = 「实际工作表名!A1:B2」或 RegionMark 的 (sheet, ref)。 +- 表名、列名 = derive_tables 推出的名字，配方内部的一切引用（segment.table、verify、keep_as、relations、 + tables[].units 的键、grain、TotalRow.label_column）都必须与它**逐字相等**，不用 name_key、不用 match_key。 +- TableOut.sources 里的「s1/交叉表/日间」只给人看，不作键。 +""" +from __future__ import annotations + +import datetime as _dt +import re +from dataclasses import dataclass, field +from typing import Annotated, Any, Literal, Union + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +#: 配方格式版本。改结构就要升，静态校验只认这一个 +RECIPE_FORMAT = "agentlab-recipe/2" +#: 配方执行器版本：进构建 id 和快照 id（table_versions.recipe_build_id / snapshot_id） +RECIPE_ENGINE_VER = "recipe/1" + +# ========================================================================== +# 配方 JSON（agentlab-recipe/2） +# ========================================================================== + + +class _M(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class SheetMatch(_M): + #: 工作表名，按 recipe_parsers.match_key 比对 + name: str = Field(min_length=1, max_length=31) + #: 找不到同名工作表时：only_visible_sheet = 恰好只有一个有内容的可见工作表就用它（差异卡报「工作表改名」，需确认） + fallback: Literal["only_visible_sheet", "none"] = "only_visible_sheet" + + +class HiddenPolicy(_M): + #: 认领区域里有隐藏行时：reject_if_any 拒收（默认）；include 照常导入；exclude 跳过（这些格记作 hidden_excluded） + rows: Literal["reject_if_any", "include", "exclude"] = "reject_if_any" + #: 认领区域里有隐藏列时：reject_if_any（默认）或 include。期 2 不支持排除列（要排除请用 extra_columns=ignore） + cols: Literal["reject_if_any", "include"] = "reject_if_any" + + +class ContextSpec(_M): + """统计期。固定行为（不可配）：所有能解析出区间的区域外文字格都列出，一致即可、冲突算数据质量； + 一处都没有时请人为本期录入（记在导入记录上，不写进配方）；文件名区间作旁证，冲突算数据质量，没有只记一笔。""" + + id: Literal["统计期"] = "统计期" + kind: Literal["period"] = "period" + parser: Literal["cn_date_range"] = "cn_date_range" + #: 多处一致时，回执里优先展示以这个词开头的格子（如「统计时间范围」）。只作展示优先，不作必需；不能含数字 + prefer_prefix: str | None = Field(default=None, max_length=12) + #: 旁证：filename = 和文件名里的区间核对；none = 不核对 + cross_check: Literal["filename", "none"] = "filename" + + +class AxisFind(_M): + parser: Literal["month_day_or_date"] = "month_day_or_date" + #: 轴行至少要有几格可解析的日期（还要占该行标签列右侧非空格的多数） + min: int = Field(default=2, ge=2, le=400) + + +class Axis(_M): + find: AxisFind = Field(default_factory=AxisFind) + #: 日期列的列名 + name: str = "日期" + #: 存成 TEXT，写法 YYYY-MM-DD + type: Literal["DATE"] = "DATE" + #: 年份来源：上下文 id（只写月日的表头靠它补年份，自带年份的也要落在统计期内）；None = 表头格必须自带年份 + year_from: Literal["统计期"] | None = "统计期" + #: 断言：contiguous 逐日连续；covers_context 恰好覆盖统计期（不多、不少）。不重复是固定断言。 + #: 少了任何一项都是放宽，要进确认清单（P2-SPEC 7.5 的 checks_relaxed） + checks: list[Literal["contiguous", "covers_context"]] = Field( + default_factory=lambda: ["contiguous", "covers_context"]) + + @field_validator("checks") + @classmethod + def _checks_as_set(cls, v: list[str]) -> list[str]: + # 按集合处理、固定顺序：["covers_context", "contiguous"] 和默认值是同一个意思，哈希也要一样 + return [x for x in ("contiguous", "covers_context") if x in v] + + +class Placeholder(_M): + #: 占位符原文，按 canon 比对。不能像数（「0」「-1」静态校验拒） + text: str = Field(min_length=1, max_length=8) + #: 只用于说明文字；入库一律是空值 + meaning: Literal["无数据", "不适用"] = "无数据" + + +class CrossValues(_M): + type: Literal["INTEGER", "REAL"] = "INTEGER" + placeholders: list[Placeholder] = Field(default_factory=list, max_length=8) + #: 数据区真正的空格：reject 拒收（默认）；null 存空值（要进确认清单） + blank: Literal["reject", "null"] = "reject" + #: 文本写的数字（「1,234」）：reject（默认）；parse_thousands 千分位按数字 + text_number: Literal["reject", "parse_thousands"] = "reject" + #: 数据区出现公式：reject（默认，数据区的公式多半是没认出来的合计）；accept_cached 按保存值导入 + formula: Literal["reject", "accept_cached"] = "reject" + + +class Labels(_M): + #: 期望的标签集合（原文，按解析器的规范写法或 match_key 比对）。按集合比：多了少了是结构类问题，顺序变化进差异卡 + expect: list[str] = Field(min_length=1, max_length=200) + + +class Locate(_M): + """分段怎么定位。labels：按标签集合；section_title：按分段标题（标签列有字、值格全空的行，不要求合并）； + after：紧跟在另一个分段之后（只给 derived 用）。""" + + by: Literal["labels", "section_title", "after"] + title: str | None = Field(default=None, max_length=40) + segment: str | None = Field(default=None, max_length=32) + + +class Const(_M): + #: 从本分段的分段标题的候选词里选（recipe_parsers.candidate_words） + pick: str = Field(min_length=1, max_length=12) + + +class DimSpec(_M): + #: 维度列的列名（如「时段」）。一律存规范写法 + name: str + parser: Literal["hour_range", "text"] + #: 从解析结果派生的列（只对 hour_range）：{"起始小时": "start", "结束小时": "end"}，INTEGER。 + #: 建表顺序固定为 start 在前、end 在后（与键序无关：配方哈希按键排序，键序不能有语义） + derive: dict[str, Literal["start", "end"]] = Field(default_factory=dict) + + +class MeasuresSegment(_M): + """每一行是一个指标：落成宽表，一个指标一列。""" + + id: str = Field(min_length=1, max_length=32) + role: Literal["measures"] + table: str + locate: Locate + labels: Labels + #: 标签原文 → 列名。键必须和 labels.expect 的元素**逐字相同**、一一对应（全角半角括号不同也不行： + #: derive_tables 按原文查，查不到报 measures_keys_mismatch，绝不产出空列名）。文件里标签的写法变化由 + #: 执行器按 match_key 认，与这里无关 + measures: dict[str, str] + + +class DimensionSegment(_M): + """每一行是维度的一个取值：落成长表（轴 × 维度 → 一个值列）。""" + + id: str = Field(min_length=1, max_length=32) + role: Literal["dimension"] + table: str + locate: Locate + labels: Labels + dim: DimSpec + #: 值列的列名（如「客流」） + value: str + #: 常量列：列名 → 取自分段标题的词。建表顺序按列名排序（同上：键序不能有语义） + const: dict[str, Const] = Field(default_factory=dict) + #: 遇到能被这个解析器解析的标签就结束本段（后面通常是 derived 段） + stop_parser: Literal["hour_range_total"] | None = None + + +class Verify(_M): + """按标签语义区间重算核对：「18-22时合计」= 基表里 起始小时>=18 且 结束小时<=22 的明细之和（逐个轴日期）。 + 固定行为：明细时段必须恰好铺满区间，否则「无法核对」；是公式的，公式引用经溯源后必须正好是这些明细格； + 公式没有缓存值是「无法核对」（默认阻断，可写理由接受）；公式和写死之间的变化进差异卡。""" + + kind: Literal["label_range_sum"] = "label_range_sum" + against_table: str + #: 基表的值列 + value: str + + +class KeepAs(_M): + """表内合计另存成一张表（kind=reported_total）。""" + + table: str + #: 合计项列（存规范写法「18-22时合计」） + dim: str + derive: dict[str, Literal["start", "end"]] = Field(default_factory=dict) + value: str + + +class DerivedSegment(_M): + """合计 / 小计行：不作数据，按 verify 核对；keep_as 给了就另存一张表。""" + + id: str = Field(min_length=1, max_length=32) + role: Literal["derived"] + locate: Locate + labels_parser: Literal["hour_range_total"] = "hour_range_total" + labels: Labels + verify: Verify + keep_as: KeepAs | None = None + + +Segment = Annotated[Union[MeasuresSegment, DimensionSegment, DerivedSegment], Field(discriminator="role")] + + +class CrosstabBlock(_M): + id: str = Field(min_length=1, max_length=32) + layout: Literal["crosstab"] + axis: Axis = Field(default_factory=Axis) + #: 标签列相对第一个轴格的列偏移。期 2 只支持紧挨着（-1），字段留作以后放宽 + label_offset: int = Field(default=-1, ge=-1, le=-1) + values: CrossValues = Field(default_factory=CrossValues) + segments: list[Segment] = Field(min_length=1, max_length=16) + + +class ListColumn(_M): + #: 表头原文（多行表头在合并跨度内用「_」拼接）。按 match_key 比对 + header: str = Field(min_length=1, max_length=80) + name: str + type: Literal["TEXT", "INTEGER", "REAL", "DATE"] + #: 只对 TEXT：canonical 存规范写法、raw 存原文。缺省时主键列 canonical、其余 raw + store: Literal["canonical", "raw"] | None = None + + +class TotalRow(_M): + #: 合计标签所在的列（列名） + label_column: str + #: 合计行的标签以这个词开头（match_key 比较）。取自实际格子文字的候选词 + pick: str = Field(min_length=1, max_length=12) + #: 原表合计另存成哪张表（kind=reported_total）；None 只核对不另存 + keep_as: str | None = None + + +class ListRows(_M): + #: 数据中间的空行：stop 到此为止（默认）；skip 跳过继续（回执记个数） + blank_rows: Literal["stop", "skip"] = "stop" + #: 合计行：到合计行为止，合计行按列求和核对 + total_row: TotalRow | None = None + + +class ListValues(_M): + placeholders: list[Placeholder] = Field(default_factory=list, max_length=8) + #: 数字列、日期列的空格：null 存空值（默认，列表里常见）；reject 拒收。主键列的空格一律拒收 + blank: Literal["null", "reject"] = "null" + text_number: Literal["reject", "parse_thousands"] = "reject" + #: 列表里的公式（金额 = 单价 × 数量）：accept_cached 按保存值（默认）；reject 拒收。 + #: 固定行为：工作簿设了打开时重算（fullCalcOnLoad）时，未经核对的公式格一律拒收 + formula: Literal["accept_cached", "reject"] = "accept_cached" + + +class ListBlock(_M): + id: str = Field(min_length=1, max_length=32) + layout: Literal["list"] + table: str + header_rows: int = Field(default=1, ge=1, le=3) + #: 只在这行文字之后找表头（同一工作表上下两张表头相同的时候用）。按 match_key 比，必须恰好一处 + after_title: str | None = Field(default=None, max_length=40) + columns: list[ListColumn] = Field(min_length=1, max_length=64) + #: 表头里多出的列：reject 拒收（默认）；ignore 忽略（按表头文字记进回执） + extra_columns: Literal["reject", "ignore"] = "reject" + rows: ListRows = Field(default_factory=ListRows) + values: ListValues = Field(default_factory=ListValues) + #: 数据区里的合并单元格：reject（默认）；fill 在合并范围内用左上格的值填充（只对 TEXT 列） + merged_data: Literal["reject", "fill"] = "reject" + + +Block = Annotated[Union[CrosstabBlock, ListBlock], Field(discriminator="layout")] + + +class SheetRecipe(_M): + id: str = Field(pattern=r"^[a-z][a-z0-9_]{0,31}$") + match: SheetMatch + hidden: HiddenPolicy = Field(default_factory=HiddenPolicy) + context: list[ContextSpec] = Field(default_factory=list, max_length=1) + blocks: list[Block] = Field(min_length=1, max_length=8) + + +class TableSpec(_M): + name: str + #: 粒度 = 主键列（按 canon 唯一）。交叉表产出的表必须含轴列；列表可以为空（不建主键） + grain: list[str] = Field(default_factory=list, max_length=4) + kind: Literal["data", "reported_total"] = "data" + #: 列名 → 单位（取自 recipe_parsers.UNITS） + units: dict[str, str] = Field(default_factory=dict) + #: 用户补充的口径说明（散文）：不许有数字和坐标，列名表名用 {列:名} {表:名} 引用。AI 起草的配方这里必须为空 + note: str = Field(default="", max_length=300) + + +class SumEq(_M): + """恒等式：total = parts 之和，逐行用 SQL 核对。固定为数据质量类（不成立时可写理由接受）。""" + + id: str = Field(pattern=r"^R[0-9]{1,2}$") + kind: Literal["sum_eq"] + table: str + total: str + parts: list[str] = Field(min_length=2, max_length=8) + #: 认领的系统发现（DraftFacts 里的 F 编号）；人自己加的为 None + claims: str | None = Field(default=None, pattern=r"^F[0-9]{1,3}$") + + +class Side(_M): + table: str + value: str + + +class NotComparable(_M): + """口径不同：a 按 by 分组求和 ≠ b（只作说明，导入时记一次观察到的相等个数，不阻断）。""" + + id: str = Field(pattern=r"^R[0-9]{1,2}$") + kind: Literal["not_comparable"] + a: Side + b: Side + #: 两表共有的分组列（如「日期」） + by: str + claims: str | None = Field(default=None, pattern=r"^F[0-9]{1,3}$") + + +class Dismissed(_M): + """系统发现了一条关系，但用户判定不登记（如巧合）。也算认领。""" + + id: str = Field(pattern=r"^R[0-9]{1,2}$") + kind: Literal["dismissed"] + claims: str = Field(pattern=r"^F[0-9]{1,3}$") + reason: str = Field(min_length=1, max_length=200) + + +Relation = Annotated[Union[SumEq, NotComparable, Dismissed], Field(discriminator="kind")] + + +class Recipe(_M): + recipe_format: Literal["agentlab-recipe/2"] + #: replace 每期替换；accumulate 按期累积（期 3，期 2 静态校验拒收，数据模型兼容） + mode: Literal["replace", "accumulate"] = "replace" + #: 配方没列到的、有内容的可见工作表：confirm 记进回执且需确认（默认）；reject 拒收 + other_visible_sheets: Literal["confirm", "reject"] = "confirm" + sheets: list[SheetRecipe] = Field(min_length=1, max_length=8) + tables: list[TableSpec] = Field(min_length=1, max_length=32) + relations: list[Relation] = Field(default_factory=list, max_length=32) + + +# ========================================================================== +# 问题、核对、确认、差异 +# ========================================================================== + +#: 问题类别：structure 结构类（不能接受，只能改配方或改文件）；data_quality 数据质量类（可写理由接受）； +#: confirm 需确认（勾选后可启用）；input 需要人录入（统计期）；recipe 配方本身不合法（不能试运行） +Category = Literal["structure", "data_quality", "confirm", "input", "recipe"] + + +@dataclass +class Problem: + code: str + category: Category + #: 给人看的中文，可以含格子里的文字和数 + message: str + #: 坐标「工作表!A1」，最多 20 个 + cells: list[str] = field(default_factory=list) + #: 给 AI 修订看的版本：可以有坐标和标签文字，不能有数据格的数值。执行器产出的每个 code 都必须填 + #: （WP-2 的契约测试逐个 code 断言非空）；万一为空，回灌时只发 code 和 cells，**绝不退回 message** + model_message: str = "" + #: 期 3 修复按钮的种类（add_label / remove_label / rename_title / declare_total / edit_members / + #: ignore_cells / declare_placeholder）。期 2 只透传 + fix: str | None = None + + +@dataclass +class RecipeProblem: + #: JSON Pointer，如 /sheets/0/blocks/0/segments/2/const/时段类别/pick + path: str + code: str + message: str + + def __str__(self) -> str: + return self.message + + +CheckStatus = Literal["passed", "mismatch", "unverifiable", "info"] + + +@dataclass +class CheckResult: + #: 稳定 id：K1… 合计核对、G1… 公式引用、T1… 列表合计、R1… 关系（沿用配方里的 id）、C1 统计期、C2 文件名、 + #: N1… 行数、P1… 主键。F 编号留给系统发现的事实（Fact.id），两者不混用 + id: str + kind: Literal["derived_sum", "formula_refs", "column_sum", "relation_sum_eq", "relation_not_comparable", + "context_agree", "filename_period", "row_count", "pk_unique"] + title: str + status: CheckStatus + #: 不通过时的类别：structure / data_quality / info + category: Literal["structure", "data_quality", "info"] + checked: int = 0 + failed: int = 0 + unverifiable: int = 0 + #: 跑过的 SQL 原文（一条代表性的，参数写在 params 里） + sql: str | None = None + params: list[Any] = field(default_factory=list) + details: list[str] = field(default_factory=list) + cells: list[str] = field(default_factory=list) + #: 可以写理由接受（数据质量类不成立、合计无法核对） + acceptable: bool = False + #: 无法核对的原因 → 格数（K / T / G）：tiling 明细没铺满、uncached 公式无缓存、blank 合计格为空、 + #: null_detail 明细含空值、hidden 隐藏行或分类汇总、unrecognized 公式形状认不出。说明按它选模板 + reasons: dict[str, int] = field(default_factory=dict) + + +@dataclass +class Acceptance: + """对一条核对结果写理由接受。data_quality 不成立 → overrides；合计无法核对 → waivers。""" + + check_id: str + reason: str + signed_by: str | None = None + + +@dataclass +class ConfirmItem: + #: 稳定 id，见 P2-SPEC 第 7.5 节的取值表 + id: str + label: str + detail: str = "" + required: bool = True + source: Literal["recipe", "diff", "outside", "sheet", "context", "switch"] = "recipe" + + +@dataclass +class DiffItem: + kind: str + label: str + detail: str = "" + requires_confirm: bool = False + #: requires_confirm 时对应的 ConfirmItem.id + confirm_id: str | None = None + + +# ========================================================================== +# 网格（读取层 → 执行器、起草器、压缩表示） +# ========================================================================== + + +@dataclass +class GridCell: + #: 保存值（openpyxl data_only）：int / float / str / bool / datetime / date / time / None + value: Any + #: 公式原文（带「=」），不是公式时 None + formula: str | None = None + + @property + def has_cache(self) -> bool: + return self.formula is None or self.value is not None + + +@dataclass +class Grid: + """一张可见工作表的物化网格（只给小表：交叉表、预览、起草；大列表走 xlsx_cells.iter_rows）。""" + + sheet: str + #: (min_row, min_col, max_row, max_col)；空表 None + bounds: tuple[int, int, int, int] | None + #: 非空格（口径和 xlsx_scan 一致：有公式的格不算空；只有空白字符的文本算空） + cells: dict[tuple[int, int], GridCell] = field(default_factory=dict) + #: 合并区 (r1, c1, r2, c2) + merges: list[tuple[int, int, int, int]] = field(default_factory=list) + hidden_rows: set[int] = field(default_factory=set) + hidden_cols: set[int] = field(default_factory=set) + #: 读到 max_rows 就停了（大表的预览、起草） + truncated: bool = False + + def get(self, r: int, c: int) -> GridCell | None: + return self.cells.get((r, c)) + + def _index(self) -> dict[int, list[int]]: + # 行号 → 该行非空格的列号（升序）。第一次按行访问时建一次：逐行扫的调用方（执行器、起草器） + # 总代价是 O(格数)，而不是每次 row() 都把整张表过一遍。**构造完成后不要再改 cells** + # （改了要 del grid.__dict__["_rows"]）。存在实例字典里而不是 dataclass 字段：不进 asdict、不进比较 + idx = self.__dict__.get("_rows") + if idx is None: + idx = {} + for (rr, c) in self.cells: + idx.setdefault(rr, []).append(c) + for cols in idx.values(): + cols.sort() + self.__dict__["_rows"] = idx + return idx + + def row(self, r: int) -> list[tuple[int, GridCell]]: + """第 r 行的非空格 [(列号, 格)]，按列升序。O(该行格数)（首次调用时建索引，O(格数)）。""" + return [(c, self.cells[(r, c)]) for c in self._index().get(r, ())] + + def row_numbers(self) -> list[int]: + """有非空格的行号，升序。""" + return sorted(self._index()) + + +# ========================================================================== +# 执行结果（执行器 → 核对、说明、差异、回执） +# ========================================================================== + +#: 格子账的去处（每个非空格恰好一个） +Role = Literal["value", "derived_value", "derived_label", "col_header", "row_label", "section_title", + "context", "outside_text", "ignored_column", "hidden_excluded", "total_label", "total_value"] + + +@dataclass +class RegionMark: + sheet: str + role: Role + #: A1 区域（「C5:AG7」或单格） + ref: str + + +@dataclass +class LedgerSheet: + sheet: str + #: 第一遍（xlsx_scan）数到的非空格 + nonempty_scan: int + #: 第二遍（读取层）数到的非空格 + nonempty_read: int + roles: dict[str, int] = field(default_factory=dict) + unclaimed: int = 0 + + +@dataclass +class ColumnOut: + name: str + #: INTEGER / REAL / TEXT（DATE 存 TEXT） + type: str + #: 原表头或原标签（只进清单和界面，不进模型可见文本） + header: str | None = None + unit: str | None = None + #: axis / dim / derive / const / measure / value / text + role: str = "value" + + +@dataclass +class TableOut: + name: str + sheet: str + kind: Literal["data", "reported_total"] + columns: list[ColumnOut] + grain: list[str] + rows: int + #: 写入这张表的分段 / 块（「s1/交叉表/日间」，只给人看，不作键） + sources: list[str] = field(default_factory=list) + + +@dataclass +class DerivedItem: + """一个要核对的合计格（交叉表的合计行或列表的合计行）。核对模块用 SQL 重算，执行器不做算术。""" + + kind: Literal["label_range_sum", "column_sum"] + #: 分段键（derived 段的 id）或块键（列表块的 id） + segment: str + sheet: str + cell: str + label_raw: str + label: str + #: 保存值（写死的数或公式的缓存值）；公式没有缓存值时 None + value: int | float | None + is_formula: bool + formula: str | None + base_table: str + base_value: str + #: label_range_sum：区间与分组键。key = {轴列名: 日期, 前一分段（locate.segment）的每个常量列: pick}： + #: 核对 SQL 按 key 的全部列过滤，同一张表里别的分段（别的类别）写入的行不会被加进来 + start: int | None = None + end: int | None = None + key: dict[str, Any] = field(default_factory=dict) + start_col: str | None = None + end_col: str | None = None + #: column_sum：基表里这一块数据行的 rowid 闭区间 + rowid_first: int | None = None + rowid_last: int | None = None + #: 公式引用经溯源后对应的基表 rowid；不是公式、或认不出公式形状时 None + ref_rowids: list[int] | None = None + #: unrecognized（公式形状认不出）/ outside（引用了不在基表里的格）/ hidden（引用里有隐藏行、 + #: 或是 SUBTOTAL / AGGREGATE：合计口径无法确定）/ None + ref_problem: str | None = None + #: 合计公式的函数：SUM / SUBTOTAL / AGGREGATE / plus（单格相加）/ None（写死的数） + func: str | None = None + #: 本合计覆盖的明细行里有隐藏行（include 或 exclude）：不一致时只能判「无法核对」（critic C6） + hidden_in_range: bool = False + keep_table: str | None = None + + +@dataclass +class CanonEntry: + table: str + column: str + raw: str + canonical: str + count: int + first_cell: str + + +@dataclass +class SegmentLabels: + segment: str + #: 按工作表里的顺序 + raw: list[str] + canonical: list[str] + rows: list[int] + + +@dataclass +class OutsideText: + sheet: str + #: 带工作表名的坐标「工作表!A1」（与 Problem.cells、差异卡的 diff:outside:<工作表>!<格> 同一写法），不是纯 A1 + cell: str + #: 全文（含数字也照录：只给回执、面板、裁判看，不进模型可见文本） + text: str + kind: Literal["text", "text_digits"] + #: 这格同时是统计期来源（带附加文字的统计期格，PeriodOut.annotated） + period_source: bool = False + + +@dataclass +class AxisOut: + block: str + row: int + first: str + last: str + count: int + #: text（「8月1日」之类）/ date（日期格）/ mixed + form: str + + +@dataclass +class PeriodOut: + start: str + end: str + #: cells 从格子解析；human 本期人工录入 + source: Literal["cells", "human"] + #: 解析出统计期的格子（含只写到月的旁证） + cells: list[str] = field(default_factory=list) + #: 人工录入时的署名（未认证） + signed_by: str | None = None + #: 坐标 → 这格的全文（差异卡按「数字遮盖后的模板」比较统计期格的写法变化，D12） + texts: dict[str, str] = field(default_factory=dict) + #: 带附加文字的统计期格：坐标 → recipe_parsers.period_residue。这些格同时记进 outside_text + #: （kind=text_digits），按含数字的区域外文字处理（D15、E7）；只是在说统计期的格不在这里 + annotated: dict[str, str] = field(default_factory=dict) + + +@dataclass +class SheetsOut: + #: 配方工作表 id → 实际用的工作表名 + matched: dict[str, str] = field(default_factory=dict) + #: 按 fallback 认的（工作表改了名）：配方里的名字 → 实际名字 + renamed: dict[str, str] = field(default_factory=dict) + #: 配方没列到的、有内容的可见工作表 + other_visible: list[str] = field(default_factory=list) + #: 没读的隐藏工作表 [{sheet, state}] + skipped_hidden: list[dict[str, str]] = field(default_factory=list) + + +@dataclass +class Extraction: + #: 没有 structure / input 类问题，且库已写好 + ok: bool + problems: list[Problem] = field(default_factory=list) + #: 只在每张工作表的前若干行上干跑（execute(max_rows=…)，交互时用）：依赖整张表的检查（两遍对账、 + #: 窗口之外的未认领、轴覆盖、稀疏、主键全表去重）没做。partial 的结果只能给界面着色和起草判断， + #: 不能当试运行用 + partial: bool = False + tables: list[TableOut] = field(default_factory=list) + ledger: list[LedgerSheet] = field(default_factory=list) + regions: list[RegionMark] = field(default_factory=list) + #: 表 → 列 → [[起始 rowid, 工作表, 起始格, 个数, "right" | "down"]] + lineage: dict[str, dict[str, list[list[Any]]]] = field(default_factory=dict) + period: PeriodOut | None = None + #: 统计期多处一致、文件名旁证的核对结果（C1、C2）。C2 取决于文件名、而构建 id 不含文件名： + #: 构建回执（TableBuild.report）不存这一项，它跟着导入记录走（table_imports.checks、导入清单） + context_checks: list[CheckResult] = field(default_factory=list) + derived: list[DerivedItem] = field(default_factory=list) + canonicalized: list[CanonEntry] = field(default_factory=list) + labels: list[SegmentLabels] = field(default_factory=list) + outside_text: list[OutsideText] = field(default_factory=list) + #: 占位符原文 → 个数 + placeholders: dict[str, int] = field(default_factory=dict) + #: 认领区域里的隐藏行列：{sheet: {"rows": [...], "cols": [...], "policy_rows": ..., "policy_cols": ...}} + hidden: dict[str, dict[str, Any]] = field(default_factory=dict) + sheets: SheetsOut = field(default_factory=SheetsOut) + axes: list[AxisOut] = field(default_factory=list) + #: 工作表键（实际工作表名）→ 该表上各分段键（交叉表）和块键（列表）按首行从上到下的先后 + block_order: dict[str, list[str]] = field(default_factory=dict) + #: derived 分段键 → formula / literal / mixed + derived_form: dict[str, str] = field(default_factory=dict) + full_calc_on_load: bool = False + #: 按保存值导入的公式格个数（accept_cached） + formula_cells_accepted: int = 0 + #: 列表里跳过的空行个数（blank_rows=skip） + blank_rows_skipped: int = 0 + #: 块键 → 按 extra_columns=ignore 忽略的列的表头原文（确认项 ignored_columns:<块>、差异卡 ignored_columns） + ignored_columns: dict[str, list[str]] = field(default_factory=dict) + #: 表 → 期望行数（= 格子账推出来的），核对模块和 SELECT COUNT(*) 比 + expected_rows: dict[str, int] = field(default_factory=dict) + #: 表 → 按主键排序后规范化行的 sha256（写库之后算） + table_hashes: dict[str, str] = field(default_factory=dict) + #: 各阶段耗时（秒） + timings: dict[str, float] = field(default_factory=dict) + + +@dataclass +class PeriodInput: + """本期人工录入的统计期（只在解析不出来时）。记在导入记录上，计入构建哈希，不写进配方。""" + + start: _dt.date + end: _dt.date + signed_by: str | None = None + + +# ========================================================================== +# 起草 +# ========================================================================== + + +@dataclass +class Fact: + """系统发现的一条事实（只有关系，不含数值）。首次确认时每条都要被配方认领。""" + + id: str + kind: Literal["sum_eq", "not_equal_sum"] + sheet: str + #: 人能看懂的一句话：「第 5 行 = 第 6 行 + 第 7 行（31/31 列成立）」 + text: str + #: sum_eq：{"segment": measures 分段 id, "total": 标签原文, "parts": [...], "rows": [...]}； + #: not_equal_sum：{"a": [dimension 分段 id...], "b_segment": measures 分段 id, "b": 标签原文, "equal": n, "checked": m}。 + #: 静态校验据此做认领的语义检查（fact_claim_mismatch）：认领它的 relation 经 measures 映射后必须正好是这些标签 + detail: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class DraftFacts: + facts: list[Fact] = field(default_factory=list) + #: 分段标题（原文）→ 候选词，界面上给常量做选择 + candidates: dict[str, list[str]] = field(default_factory=dict) + #: 数值区里出现的非数字文字 → 个数（占位符候选） + nonnumeric: dict[str, int] = field(default_factory=dict) + + +@dataclass +class Card: + """建议卡片：一条建议加一句理由，指向网格上的格子。""" + + id: str + title: str + reason: str + cells: list[str] = field(default_factory=list) + #: 对应的待确认问题（Question.id），没有就是 None + question: str | None = None + + +@dataclass +class Question: + id: str + text: str + #: [{"value": str, "label": str, "needs_reason": bool}]：只给封闭的选项。needs_reason 为真的选项 + #: （如「不登记」）提交回答时必须带理由（1–200 字） + options: list[dict[str, Any]] = field(default_factory=list) + default: str | None = None + #: 选项值 → 对配方的修改（[{"op": "replace" | "add" | "remove", "path": JSON Pointer, "value": ...}]），由服务端应用。 + #: value 里的字符串 REASON_SLOT 由服务端换成用户写的理由（只换整个字符串等于它的值，不做子串替换） + effects: dict[str, list[dict[str, Any]]] = field(default_factory=dict) + + +#: Question.effects 里给理由占位的字符串 +REASON_SLOT = "{reason}" + + +@dataclass +class Draft: + recipe: dict[str, Any] | None + #: 起草得出了能通过静态校验、并能认领全部非空格（干跑）的配方 + complete: bool + origin: Literal["rules", "ai"] + cards: list[Card] = field(default_factory=list) + questions: list[Question] = field(default_factory=list) + facts: DraftFacts = field(default_factory=DraftFacts) + #: 起草不完整的原因（给人看，可以引用格子原文） + failures: list[str] = field(default_factory=list) + #: 同一组原因给模型看的版本：只有坐标、标签和表头文字，不含任何数字格的值、像数的文字 + #: （AI 起草的提示词只用这一份，不用 failures） + failures_for_model: list[str] = field(default_factory=list) + + +@dataclass +class AiUsage: + model: str + input_tokens: int + output_tokens: int + cost_usd: float + ok: bool + at: str + #: 第几次调用（1 起草，2、3 修订） + attempt: int = 1 + + +@dataclass +class AiAvailability: + available: bool + #: 不可用时给人看的原因（「未配置模型接入」） + reason: str = "" + #: 可用时的模型名（界面告知会发给谁） + model: str = "" + provider: str = "" + + +# ========================================================================== +# 说明（按核对结果生成） +# ========================================================================== + + +@dataclass +class TableNote: + #: 已渲染的表说明（注入 schema_cache 的 comment） + comment: str + #: 列名 → 已渲染的列说明（单位、空值含义、日期格式） + columns: dict[str, str] = field(default_factory=dict) + + +@dataclass +class SchemaNotes: + tables: dict[str, TableNote] = field(default_factory=dict) + #: 渲染前的模板（带 {列:名} 标记），进清单，复核时可以重查。键：表名或「表名.列名」 + templates: dict[str, str] = field(default_factory=dict) + #: 数字检查没过的（非空就不能保存） + problems: list[str] = field(default_factory=list) + #: 表名 → 表的种类，只记 reported_total（原表写明的合计）。apply_notes 记进表结构，工具描述、db_schema 的 + #: 表清单、写作目录据此标出「不要彼此相加、不要与明细相加」（AU-5） + kinds: dict[str, str] = field(default_factory=dict) + + +_TOKEN = re.compile(r"\{(列|表|单位):([^{}]{1,48})\}") +_COORD = re.compile(r"(? str: + """把 {列:名} {表:名} {单位:名} 换成名字本身。""" + return _TOKEN.sub(lambda m: m.group(2), template) + + +#: prose_problems 的 known 参数里三类引用的键 +TOKEN_KINDS = ("列", "表", "单位") + + +def prose_problems(template: str, *, known: dict[str, set[str]] | None = None) -> list[str]: + """说明散文的检查：遮盖结构化引用（列名、表名、单位）之后，不许抽出数字，也不许出现单元格坐标。 + + 数字用出具校验同一套规则抽（app.engine.issuance.extract_numbers，app.engine.evidence 里的同名函数就是它）。 + 单位、列名由系统按结构渲染，检查时遮盖：「单位：万元」抽出「万」、「3号门客流」抽出 3，都不是说明在报数。 + + **遮盖的前提是引用真的指向结构化的名字。** 给了 known（{"列": 列名集合, "表": 表名集合, "单位": 单位集合}， + 逐字比较）时，不认识的引用不遮盖、并报一条问题：用户写的 note 不能靠「{列:增长30%}」把数字藏过检查。 + 静态校验（用户的 note）和说明生成都必须传 known；只有契约测试里检查模板本身时才不传。 + 返回给人看的问题列表；空列表表示通过。 + """ + from app.engine.issuance import extract_numbers + + out: list[str] = [] + + def _mask(m: re.Match[str]) -> str: + kind, name = m.group(1), m.group(2) + if known is not None and name not in known.get(kind, set()): + label = {"列": "列", "表": "表", "单位": "单位"}[kind] + out.append(f"说明引用了不存在的{label}「{name}」") + return m.group(0) + return " " * len(m.group(0)) + + masked = _TOKEN.sub(_mask, template) + out += [f"说明中含有数字「{t.raw}」" for t in extract_numbers(masked)] + out += [f"说明中含有单元格坐标「{m.group(0)}」" for m in _COORD.finditer(masked)] + return out + + +# ========================================================================== +# 由配方推出每张表的列（执行器建表、静态校验、说明、破坏性变更比对共用同一份） +# ========================================================================== + +#: 列表合计行另存成表时，合计项列的固定列名 +LIST_TOTAL_DIM = "合计项" +_SQL_TYPE = {"INTEGER": "INTEGER", "REAL": "REAL", "TEXT": "TEXT", "DATE": "TEXT"} + + +def derive_tables(recipe: Recipe) -> tuple[dict[str, list[ColumnOut]], list[RecipeProblem]]: + """配方 → {表名: 列（按建表顺序）}，以及形状冲突（同一张表被两个分段写成不同的列）。 + + 列的顺序：交叉表的宽表 = 轴、各指标（labels.expect 的顺序）；长表 = 轴、维度、常量（按列名排序）、 + 派生（start 在前、end 在后）、值；表内合计 = 轴、合计项、派生、值；列表 = columns 的顺序；列表合计表 = + 合计项、各数字列。常量和派生不按 dict 的键序：配方哈希用 sort_keys，键序若决定列序，两份列序不同的配方 + 会得到同一个构建 id。单位取 tables[].units（逐字查列名）。 + + measures 的键按原文逐字查（不按 match_key）：expect 里某个标签在 measures 里没有、或 measures 里多出 + expect 没有的键，报 measures_keys_mismatch——列名照样产出（空字符串），但带着问题，调用方必须拦下， + 绝不能拿空列名去建表。不查名字合法性、不查 tables 里有没有声明（那是静态校验的事）。 + """ + units = {t.name: t.units for t in recipe.tables} + out: dict[str, list[ColumnOut]] = {} + where: dict[str, str] = {} + problems: list[RecipeProblem] = [] + + def _derive_cols(derive: dict[str, str]) -> list[ColumnOut]: + ordered = sorted(derive.items(), key=lambda kv: (kv[1] != "start", kv[0])) + return [ColumnOut(name, "INTEGER", role="derive") for name, _ in ordered] + + def put(table: str, cols: list[ColumnOut], path: str) -> None: + for c in cols: + c.unit = units.get(table, {}).get(c.name) + if table not in out: + out[table], where[table] = cols, path + return + old = [(c.name, c.type, c.role) for c in out[table]] + new = [(c.name, c.type, c.role) for c in cols] + if old != new: + problems.append(RecipeProblem(path, "table_shape_conflict", + f"表「{table}」在两处写入的列不一致（另一处：{where[table]}）")) + + for si, sheet in enumerate(recipe.sheets): + for bi, block in enumerate(sheet.blocks): + base = f"/sheets/{si}/blocks/{bi}" + if isinstance(block, CrosstabBlock): + vtype = block.values.type + axis_name = block.axis.name + for gi, seg in enumerate(block.segments): + path = f"{base}/segments/{gi}" + if isinstance(seg, MeasuresSegment): + missing = [label for label in seg.labels.expect if label not in seg.measures] + extra = [key for key in seg.measures if key not in seg.labels.expect] + if missing or extra: + parts = [] + if missing: + parts.append("标签" + "、".join(f"「{x}」" for x in missing) + "没有对应的列名") + if extra: + parts.append("列名对照里的" + "、".join(f"「{x}」" for x in extra) + + "不在期望的标签中（必须与标签逐字相同）") + problems.append(RecipeProblem(f"{path}/measures", "measures_keys_mismatch", + f"分段「{seg.id}」：" + "；".join(parts))) + cols = [ColumnOut(axis_name, "TEXT", role="axis")] + cols += [ColumnOut(seg.measures.get(label, ""), vtype, header=label, role="measure") + for label in seg.labels.expect] + put(seg.table, cols, path) + elif isinstance(seg, DimensionSegment): + cols = [ColumnOut(axis_name, "TEXT", role="axis"), ColumnOut(seg.dim.name, "TEXT", role="dim")] + cols += [ColumnOut(name, "TEXT", role="const") for name in sorted(seg.const)] + cols += _derive_cols(seg.dim.derive) + cols.append(ColumnOut(seg.value, vtype, role="value")) + put(seg.table, cols, path) + elif isinstance(seg, DerivedSegment) and seg.keep_as is not None: + k = seg.keep_as + cols = [ColumnOut(axis_name, "TEXT", role="axis"), ColumnOut(k.dim, "TEXT", role="dim")] + cols += _derive_cols(k.derive) + cols.append(ColumnOut(k.value, vtype, role="value")) + put(k.table, cols, path) + else: + cols = [ColumnOut(c.name, _SQL_TYPE[c.type], header=c.header, + role="measure" if c.type in ("INTEGER", "REAL") else "text") + for c in block.columns] + put(block.table, cols, base) + total = block.rows.total_row + if total is not None and total.keep_as: + tcols = [ColumnOut(LIST_TOTAL_DIM, "TEXT", role="dim")] + tcols += [ColumnOut(c.name, c.type, header=c.header, role="value") + for c in cols if c.type in ("INTEGER", "REAL")] + put(total.keep_as, tcols, f"{base}/rows/total_row") + return out, problems diff --git a/backend/app/data/table_versions.py b/backend/app/data/table_versions.py index 9bd0c9bd..4fcadba7 100644 --- a/backend/app/data/table_versions.py +++ b/backend/app/data/table_versions.py @@ -18,6 +18,12 @@ **回收。** 任何运行引用的快照、各源当前快照，以及它们用到的构建和原件受保护；其余的删文件、 记录标 retired 留作审计。 + +**按配方导入（期 2）。** 构建、导入记录、快照三层和查询路径都不变，只是多了两样： +- 发布拆成两步：publish_upload 只管解析，「把一个建好的临时库发布成新版本」抽成 publish_build， + 配方导入的提交事务也调它（带期望的库哈希、锁内的提交前回调）； +- 暂存区（import_stagings）：没发布的尝试都在这里，提交成功前不碰数据源。它引用的原件、试运行库 + 归这里的回收与清理管：未结束的暂存区保护原件，结束即瘦身，过期、放弃、清除原件都会结束它。 """ from __future__ import annotations @@ -34,12 +40,12 @@ import sqlite3 import stat import weakref from dataclasses import dataclass, field -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path -from typing import Any, Sequence +from typing import Any, Awaitable, Callable, Iterator, Sequence from urllib.parse import unquote, urlsplit -from sqlalchemy import func, or_, select, update +from sqlalchemy import and_, delete, func, or_, select, update from sqlalchemy import inspect as sa_inspect from sqlalchemy.engine import make_url from sqlalchemy.ext.asyncio import AsyncSession @@ -47,11 +53,13 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.config import settings from app.core.errors import explain, first_line from app.data import introspect as introspect_mod -from app.data import raw_store, tabular +from app.data import raw_store, recipe_parsers, recipe_types, tabular from app.data.engine import engines from app.db import base as db_base from app.db.base import new_id, utcnow -from app.db.models import DataSource, Run, SourceSnapshot, TableBuild, TableImport +from app.db.models import ( + DataSource, ImportStaging, Run, SourceSnapshot, TableBuild, TableImport, TableRecipe, +) logger = logging.getLogger(__name__) @@ -78,7 +86,15 @@ class SnapshotMissing(ValueError): class PublishError(RuntimeError): - """解析成功了，但没能把库文件安全地发布出去。指针没动，旧版照常可查。""" + """解析成功了，但没能把库文件安全地发布出去。指针没动，旧版照常可查。 + + code 是给接口层分支用的机读码（trial_missing / trial_tampered / raw_missing）；期 1 的抛出处 + 不带，为 None，接口层照旧按 500 回原话。 + """ + + def __init__(self, message: str = "", *, code: str | None = None) -> None: + super().__init__(message) + self.code = code class ParseFailed(tabular.UnsupportedTable): @@ -94,8 +110,31 @@ class ImportResult: snapshot_id: str build_id: str build_reused: bool - #: 解析回执（tabular.LoadReport 的 JSON 形状） + #: 解析回执（tabular.LoadReport 的 JSON 形状；配方导入是执行器回执） report: dict[str, Any] + #: 新导入记录的序号 + seq: int = 0 + #: 实际发布（或复用）的库文件哈希 + db_sha256: str = "" + + +@dataclass +class PublishInfo: + """publish_build 交给 schema_cache_for 的发布结果：这时库文件已经落位，记录还没写。 + + 配方导入在 schema_cache_for 里写导入清单，清单要记的导入 id、快照 id、序号、实际发布的库哈希 + 此时都已确定。db_sha256 一律是**实际发布（或复用）**的那个文件的哈希：复用构建时它可能和试运行 + 回执里的不同（比如 SQLite 升级后文件头里的版本号变了），清单取这里的，不取回执里的。 + """ + + import_id: str + snapshot_id: str + seq: int + build_id: str + db_sha256: str + build_reused: bool + #: 实际冻结的回执：复用构建时是已有构建登记的回执，否则是这次传进来的 + report: dict[str, Any] = field(default_factory=dict) @dataclass @@ -135,11 +174,15 @@ class SourceView: # -------------------------------------------------------------------------- -def _sha_json(obj: Any) -> str: +def sha_json(obj: Any) -> str: + """JSON 的规范写法（键排序、紧凑分隔）的 sha256。构建 id、快照 id、选项哈希都用它。""" text = json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(",", ":")) return hashlib.sha256(text.encode("utf-8")).hexdigest() +_sha_json = sha_json + + def parse_options(filename: str, *, header_row: int, mixed: str, raw_mode: bool) -> dict[str, Any]: """影响解析结果的全部选项。进构建 id：选项不同，建出来的库就不是同一个。 @@ -168,6 +211,27 @@ def snapshot_id(source_id: str, import_ids: list[str], engine_ver: str = ENGINE_ return _sha_json({"source_id": source_id, "imports": list(import_ids), "engine_ver": engine_ver}) +def recipe_build_options(recipe_sha: str, inputs: dict[str, Any]) -> dict[str, Any]: + """按配方导入的构建登记的选项（TableBuild.options）。options_sha256 = sha_json(它)。""" + return {"kind": "recipe", "recipe_sha256": recipe_sha, + "parser_ver": recipe_parsers.PARSER_VER, "inputs": inputs} + + +def recipe_build_id(source_id: str, raw_sha: str, recipe_sha: str, inputs: dict[str, Any]) -> str: + """按配方导入的构建 id：源、原件、配方、解析器版本、执行器版本、人工录入。 + + inputs 只放人工录入的东西（{"context": {"统计期": {"start", "end"}}}，没有就 {}）；署名和文件名 + 不进。同一份原件换个名字再传会复用构建，所以凡是取决于文件名的结果（C2）都不能放进构建回执， + 得跟着导入记录走（table_imports.checks、导入清单）。解析器或执行器升级会换掉 id：按新规则建出来的 + 库不是同一个构建，不能复用旧文件。 + """ + return sha_json({ + "source_id": source_id, "raw_sha256": raw_sha, "recipe_sha256": recipe_sha, + "parser_ver": recipe_parsers.PARSER_VER, "engine_ver": recipe_types.RECIPE_ENGINE_VER, + "inputs_sha256": sha_json(inputs), + }) + + def legacy_build_id(source_id: str, file_sha: str) -> str: """迁移前老上传的 v0 构建 id。带上 source_id：两个源的老库内容相同，id 也不同。""" return hashlib.sha256(f"legacy{source_id}{file_sha}".encode("utf-8")).hexdigest() @@ -183,6 +247,23 @@ def build_path(source_id: str, bid: str) -> Path: return tables_root() / source_id / "builds" / f"{bid}.db" +#: 试运行库所在的目录名：tables/<源 id>/trials/<暂存区 id>-.db +TRIALS_DIR = "trials" + + +def trial_db_path(source_id: str, staging_id: str, trial_key: str) -> Path: + """试运行库的路径。和构建库同一个文件系统，发布时 link 过去，不用拷贝。 + + 文件名带暂存区 id：启动清理按它找回暂存区，暂存区不在、已结束、或者指着的不是这个文件就删。 + 写的时候先写 .tmp- 再改名（半成品归 _sweep_tmp 管）。 + """ + if not _SOURCE_ID.match(source_id or "") or not _SOURCE_ID.match(staging_id or ""): + raise ValueError("数据源或暂存区 id 的格式不对") + if not re.fullmatch(r"[0-9A-Za-z]{16,}", trial_key or ""): + raise ValueError("试运行的键格式不对") + return tables_root() / source_id / TRIALS_DIR / f"{staging_id}-{trial_key[:16]}.db" + + def _under(path: str | os.PathLike[str], root: Path) -> bool: try: return Path(os.path.realpath(path)).is_relative_to(os.path.realpath(root)) @@ -427,8 +508,8 @@ async def resolve_source( _locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock]" = weakref.WeakKeyDictionary() -def _store_lock() -> asyncio.Lock: - """版本存储的写锁，按事件循环各一把。 +def store_lock() -> asyncio.Lock: + """版本存储的写锁，按事件循环各一把。不可重入：持着它时别再调会自己取锁的函数。 回收要先看「哪些文件有记录」再删：发布正处在「文件已 link、记录未提交」之间时，回收 会把它当孤儿删掉；复用一个刚被判为无人引用的构建，也会撞上正在删它的回收。上传不频繁， @@ -441,14 +522,17 @@ def _store_lock() -> asyncio.Lock: return lock +_store_lock = store_lock + + #: 同时解析的上传最多几个。解析在线程里做，内存和上传数成正比；不设上限的话，几个大文件 #: 同时上传，内存会叠加起来。超出的排队等前面的解析完 MAX_PARALLEL_PARSES = 2 _parse_slots: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Semaphore]" = weakref.WeakKeyDictionary() -def _parse_slot() -> asyncio.Semaphore: - """解析的并发名额，按事件循环各一份（理由同 _store_lock）。""" +def parse_slot() -> asyncio.Semaphore: + """解析的并发名额，按事件循环各一份（理由同 store_lock）。配方导入的扫描、干跑、试运行也占它。""" loop = asyncio.get_running_loop() slot = _parse_slots.get(loop) if slot is None: @@ -456,6 +540,9 @@ def _parse_slot() -> asyncio.Semaphore: return slot +_parse_slot = parse_slot + + def pin_lock() -> asyncio.Lock: """固定数据版本时持有的锁（就是版本存储的写锁）：从读当前指针到把它写进运行记录，全程持有。 @@ -465,7 +552,7 @@ def pin_lock() -> asyncio.Lock: 这次运行只能报「固定的数据版本已不存在」。持着这把锁，回收要么在读指针之前做完，要么等运行 记录提交之后才开始。只包住读指针到提交这一小段，别在里面做慢事（发布和回收也在等它）。 """ - return _store_lock() + return store_lock() # -------------------------------------------------------------------------- @@ -512,11 +599,15 @@ def _parse_to_tmp(final: Path, raw: bytes, filename: str, header_row: int, mixed _JOURNAL_STUCK = "数据文件的日志未能合并，已停止发布，原数据未受影响" -def _settle_journal(path: Path) -> None: +def settle_journal(path: Path) -> None: """切成回滚日志模式并确认没有残留的 -wal / -journal：发布后以 immutable 打开，SQLite 不会再看日志。 解析器要是用了 WAL、又有连接没关，切换会报「database is locked」或者原样留在 WAL： 这时库里的一部分数据还在 -wal 里，按 immutable 打开会静默少数据，只能停下。 + + 配方导入的试运行写完库先调它、再算回执里的 db_sha256：WAL 模式下切换会改写文件头第 18、19 + 字节，先切再算，发布时 _publish_file 再切一次就不会改动文件，两次哈希才对得上。 + 注意文件不存在时 sqlite3.connect 会新建一个空库：调用方要先确认文件在。 """ try: conn = sqlite3.connect(str(path)) @@ -534,6 +625,13 @@ def _settle_journal(path: Path) -> None: Path(f"{path}-shm").unlink(missing_ok=True) +_settle_journal = settle_journal + +_TRIAL_MISSING = "试运行的数据文件已不在，请重新试运行" +_TRIAL_TAMPERED = "试运行之后数据文件被改动过，请重新试运行" +_RAW_MISSING = "这次导入的原件已不在服务端（可能已被清除），请重新上传文件" + + def _fsync_file(path: Path) -> None: fd = os.open(path, os.O_RDONLY) try: @@ -542,19 +640,28 @@ def _fsync_file(path: Path) -> None: os.close(fd) -def _publish_file(tmp: Path, final: Path, expected_sha: str | None) -> tuple[str, bool]: +def _publish_file(tmp: Path, final: Path, expected_sha: str | None, + trial_sha: str | None = None) -> tuple[str, bool]: """把临时库发布到最终路径，返回 (库文件 sha256, 是否复用了已有文件)。 先落盘再 link：link 不会覆盖已有文件，已发布的版本因此不可能被半个新文件顶掉。 目标已存在时（同一个构建传过），只有它的哈希等于登记的 db_sha256 才复用——0 字节的 残骸、被改过的文件都不能冒充。没有登记过（上次发布到一半进程没了）时，和这次刚建的 临时库逐字节一致也可以接手。临时文件无论成败都删掉。 + + trial_sha：配方导入试运行时算的库哈希。给了就在 link 之前比对，对不上说明试运行之后有人改过 + 这个文件，不能把改过的数据当成用户核对过的那份发布出去。目标已存在时也先比它：被改过的试运行库 + 本身就是要停下的信号，不因为这次碰巧复用旧文件而放过。 """ try: - _settle_journal(tmp) + settle_journal(tmp) sha = raw_store.sha256_file(tmp) + if trial_sha is not None and sha != trial_sha: + logger.warning("试运行库的哈希与试运行回执不符，已停止发布：%s", tmp) + raise PublishError(_TRIAL_TAMPERED, code="trial_tampered") _fsync_file(tmp) raw_store.fsync_dir(tmp.parent) + final.parent.mkdir(parents=True, exist_ok=True) try: os.link(tmp, final) except FileExistsError: @@ -574,7 +681,7 @@ def _publish_file(tmp: Path, final: Path, expected_sha: str | None) -> tuple[str _remove_db_files(tmp) -async def _introspect_view(view: SourceView, report: dict[str, Any]) -> dict[str, Any]: +async def introspect_view(view: SourceView, report: dict[str, Any]) -> dict[str, Any]: """探查快照库的结构，给未规整的表注入说明。 走 introspect 而不是自己读 sqlite_master：表结构的形状必须和手工源的一模一样，下游 @@ -597,7 +704,10 @@ async def _introspect_view(view: SourceView, report: dict[str, Any]) -> dict[str return cache -def _check_probe(cache: dict[str, Any], report: dict[str, Any], source_id: str) -> None: +_introspect_view = introspect_view + + +def check_probe(cache: dict[str, Any], report: dict[str, Any], source_id: str) -> None: """冻结进快照之前核对：探查成功，且探到的表正是解析回执里的那些。 快照的结构一旦冻结就改不了（上传源不能重新探查），所以宁可这次发布失败、旧版照常可查， @@ -615,66 +725,105 @@ def _check_probe(cache: dict[str, Any], report: dict[str, Any], source_id: str) raise PublishError("导入失败：新数据的表结构与解析结果对不上，已停止发布，当前版本未受影响") -async def publish_upload( - session: AsyncSession, source: DataSource, raw: bytes, filename: str, *, - header_row: int = 1, mixed: str = "reject", raw_mode: bool = False, +_check_probe = check_probe + + +#: publish_build 的 import_fields 能写的列：配方导入在 table_imports 上新加的那些。id、状态、序号、 +#: 原件这些由发布流程自己定，不许经这里改 +IMPORT_FIELDS = frozenset({ + "recipe_id", "period_start", "period_end", "context", "checks", "overrides", "waivers", + "confirmations", "manifest_artifact", "signed_by", "staging_id", +}) + +SchemaCacheFor = Callable[[SourceView, PublishInfo], Awaitable[dict[str, Any]]] +BeforeCommit = Callable[[str, str], Awaitable[None]] + + +def _usable_tmp(tmp: Path) -> bool: + """临时库在、非空、在上传目录里。不在上传目录里的一律不认：发布结束要删它，不能删到别处去。""" + try: + return tmp.is_file() and tmp.stat().st_size > 0 and _under(tmp, settings.uploads_dir) + except OSError: + return False + + +async def publish_build( + session: AsyncSession, source: DataSource, *, tmp_db: Path, build_id: str, + build_options: dict[str, Any], engine_ver: str, report: dict[str, Any], + raw: bytes | None, raw_sha: str, file_name: str, file_size: int, + schema_cache_for: SchemaCacheFor, import_fields: dict[str, Any], + expected_db_sha256: str | None = None, + before_commit: BeforeCommit | None = None, + import_id: str | None = None, ) -> ImportResult: - """解析一份上传、发布成新版本并启用。返回导入回执。 + """把一个建好的临时库发布成数据源的新版本并启用。期 1 的上传和期 2 的配方提交都走这里。 - 1. 线程池里解析到临时库；NeedsDecision / UnsupportedTable 原样抛出——不留原件、不动指针。 - 2. 发布库文件：落盘、link 到构建路径、0444。 - 3. 存原件；一个事务里写构建、导入记录、快照，旧的 active 导入置 superseded，切数据源指针。 - 任何一步失败都回滚，并删掉这次新建的文件，指针不动。 - 4. 回收一次（失败只记日志）。 + 1. 进锁后先确认临时库在且非空：settle_journal 的 sqlite3.connect 碰到不存在的文件会新建一个空库， + 不先查就会把空库当成新版本发布出去（PublishError code=trial_missing）。 + 2. 原件：raw 给了就存档；为 None 时原件必须已在存档里（配方导入暂存时存过），否则 raw_missing。 + 3. 发布库文件：落盘、（给了 expected_db_sha256 就先比哈希，code=trial_tampered）、link 到构建路径、0444。 + 4. 写构建记录（复用时照期 1 的规则）；schema_cache_for(view, info) 返回要冻结进快照的表结构。 + 它在库文件落位之后、写任何导入记录之前调用：配方导入在这里写导入清单，这时导入 id、快照 id、 + 序号、实际发布的库哈希都已确定。 + 5. 写导入记录（import_fields 写进新列）、快照，旧的 active 导入置 superseded。 + 6. before_commit(imp_id, snap_id)：同一事务、版本存储锁内。调用时新导入记录、快照、构建已刷进当前 + 事务（session.get 取得到导入记录），**新源还没加入会话、数据源指针还没切**——在里面 select 读到的 + current_snapshot_id 是发布前的值，查同名源也不会查到这次要建的新源。配方导入在这里重新核对 + base_changed、name_taken，写配方表、清单索引，结束暂存区。 + 7. 加入数据源、切指针、提交。之后回收一次（失败只记日志）。 + 任何一步失败（包括 before_commit 抛的异常）都回滚，删掉这次 link 出来的库文件和这次新存的原件， + 指针不动，异常原样抛出。**tmp_db 一律被消耗**：成功时 link 走，失败时也删，调用方不能指望它还在。 + + 调用之前，调用方不得改动会话里的任何对象（暂存区也不行，放进 before_commit）：schema_cache_for + 里写工件会另开会话提交，主会话要是已经有待刷的改动、占着应用库的写锁，两边会互等。 source 可以是还没入库的新 DataSource：失败时它也不会被建出来。 """ if not source.id: source.id = new_id() # 失败回滚后 source 的属性会过期，再读就要回库加载；后面一律用这份本地值 source_id = source.id - raw_sha = hashlib.sha256(raw).hexdigest() - options = parse_options(filename, header_row=header_row, mixed=mixed, raw_mode=raw_mode) - bid = build_id(source_id, raw_sha, options) - final = build_path(source_id, bid) - # 还没入库的新源（接口层每次给一个新 id）：失败时它的 tables/<源 id>/builds/ 目录也不该留下。 - # 被拒收的上传在「422 → 用户选择 → 重传」里每轮都是一个新 id，不收拾的话每轮多一个空目录。 - # 新 id 只有这一个请求在用，删目录不会和别的发布撞上；已有的源目录里有当前版本，不动 + final = build_path(source_id, build_id) + # 还没入库的新源：失败时它的 tables/<源 id>/builds/ 目录也不该留下（见 publish_upload） state = sa_inspect(source) fresh = state.transient or state.pending - try: - async with _parse_slot(): - tmp, parsed = await asyncio.to_thread( - _parse_to_tmp, final, raw, filename, header_row, mixed, raw_mode) - except BaseException: - if fresh: - await asyncio.to_thread(_prune_build_dirs, final) - raise - report = report_to_json(parsed) - linked = raw_new = False try: - async with _store_lock(): + unknown = set(import_fields) - IMPORT_FIELDS + if unknown: + raise ValueError(f"import_fields 含有不能经发布写入的列：{sorted(unknown)}") + # 导入记录上的原件哈希必须就是这份内容的：对不上的话，清除、回收、重放都会找错文件 + if raw is not None and hashlib.sha256(raw).hexdigest() != raw_sha: + raise ValueError("raw_sha 与传入的原件内容不符") + report = report_to_json(report) + async with store_lock(): + if not _usable_tmp(tmp_db): + raise PublishError(_TRIAL_MISSING, code="trial_missing") # 先读后写：读的时候别把调用方挂起的改动（新源、改说明）提前刷进库。 # 应用库的写锁只在最后提交那一下持有，解析、哈希、fsync 都不占着它 with session.no_autoflush: - build = await session.get(TableBuild, bid) + build = await session.get(TableBuild, build_id) last_seq = (await session.execute( select(func.max(TableImport.seq)).where(TableImport.source_id == source_id) )).scalar() or 0 + if raw is None and not raw_store.raw_exists(raw_sha): + raise PublishError(_RAW_MISSING, code="raw_missing") db_sha, reused_file = await asyncio.to_thread( - _publish_file, tmp, final, build.db_sha256 if build is not None else None) + _publish_file, tmp_db, final, build.db_sha256 if build is not None else None, + expected_db_sha256) linked = not reused_file - raw_new = not raw_store.raw_exists(raw_sha) - await asyncio.to_thread(raw_store.put_raw, raw) + if raw is not None: + raw_new = not raw_store.raw_exists(raw_sha) + await asyncio.to_thread(raw_store.put_raw, raw) now = utcnow() if build is None: build = TableBuild( - id=bid, source_id=source_id, raw_sha256=raw_sha, - options_sha256=_sha_json(options), engine_ver=ENGINE_VER, options=options, - db_path=str(final), db_sha256=db_sha, report=report, created_at=now, + id=build_id, source_id=source_id, raw_sha256=raw_sha, + options_sha256=sha_json(build_options), engine_ver=engine_ver, + options=build_options, db_path=str(final), db_sha256=db_sha, report=report, + created_at=now, ) session.add(build) build_reused = False @@ -690,22 +839,24 @@ async def publish_upload( build_reused = reused_file report = build.report or report - imp_id = new_id() - snap_id = snapshot_id(source_id, [imp_id]) + imp_id = import_id or new_id() + seq = last_seq + 1 + snap_id = snapshot_id(source_id, [imp_id], engine_ver) options_now = upload_options(source.options) view = SourceView( id=source_id, name=source.name, database=str(final), options=options_now, description=source.description or "", snapshot_id=snap_id, expected_sha256=build.db_sha256, ) - cache = await _introspect_view(view, report) - _check_probe(cache, report, source_id) + info = PublishInfo(import_id=imp_id, snapshot_id=snap_id, seq=seq, build_id=build_id, + db_sha256=db_sha, build_reused=build_reused, report=report) + cache = await schema_cache_for(view, info) - session.add(source) session.add(TableImport( - id=imp_id, source_id=source_id, seq=last_seq + 1, build_id=bid, - file_name=(filename or "")[:500], file_size=len(raw), raw_sha256=raw_sha, + id=imp_id, source_id=source_id, seq=seq, build_id=build_id, + file_name=(file_name or "")[:500], file_size=int(file_size), raw_sha256=raw_sha, raw_state="kept", status="active", created_at=now, activated_at=now, + **import_fields, )) session.add(SourceSnapshot( id=snap_id, source_id=source_id, imports=[imp_id], db_path=str(final), @@ -717,6 +868,10 @@ async def publish_upload( TableImport.id != imp_id) .values(status="superseded") ) + if before_commit is not None: + await before_commit(imp_id, snap_id) + + session.add(source) source.kind, source.origin, source.readonly = "sqlite", "upload", True source.host = source.port = source.username = source.password = None if options_now != (source.options or {}): @@ -737,11 +892,13 @@ async def publish_upload( if raw_new: raw_store.purge_raw(raw_sha) if fresh: - _remove_db_files(tmp) + if _under(tmp_db, settings.uploads_dir): + _remove_db_files(tmp_db) _prune_build_dirs(final) raise finally: - _remove_db_files(tmp) + if _under(tmp_db, settings.uploads_dir): + _remove_db_files(tmp_db) # 指针换了（或没换成）：按源 id 清掉缓存的引擎，下次按新的快照重建 await engines.invalidate(source_id) @@ -749,50 +906,332 @@ async def publish_upload( await gc(session) except Exception: # noqa: BLE001 - 回收失败不影响这次导入，下次发布或重启再收 logger.exception("上传表格发布后的回收失败") - return ImportResult(import_id=imp_id, snapshot_id=snap_id, build_id=bid, - build_reused=build_reused, report=report) + return ImportResult(import_id=imp_id, snapshot_id=snap_id, build_id=build_id, + build_reused=build_reused, report=report, seq=seq, db_sha256=db_sha) + + +async def publish_upload( + session: AsyncSession, source: DataSource, raw: bytes, filename: str, *, + header_row: int = 1, mixed: str = "reject", raw_mode: bool = False, +) -> ImportResult: + """解析一份上传、发布成新版本并启用。返回导入回执。 + + 1. 线程池里解析到临时库；NeedsDecision / UnsupportedTable 原样抛出——不留原件、不动指针。 + 2. 交给 publish_build：发布库文件、存原件、一个事务里写构建 / 导入记录 / 快照、切指针， + 失败时回滚并删掉这次新建的文件；之后回收一次。 + + source 可以是还没入库的新 DataSource：失败时它也不会被建出来。 + """ + if not source.id: + source.id = new_id() + source_id = source.id + raw_sha = hashlib.sha256(raw).hexdigest() + options = parse_options(filename, header_row=header_row, mixed=mixed, raw_mode=raw_mode) + bid = build_id(source_id, raw_sha, options) + final = build_path(source_id, bid) + # 还没入库的新源（接口层每次给一个新 id）：失败时它的 tables/<源 id>/builds/ 目录也不该留下。 + # 被拒收的上传在「422 → 用户选择 → 重传」里每轮都是一个新 id，不收拾的话每轮多一个空目录。 + # 新 id 只有这一个请求在用，删目录不会和别的发布撞上；已有的源目录里有当前版本，不动 + state = sa_inspect(source) + fresh = state.transient or state.pending + + try: + async with parse_slot(): + tmp, parsed = await asyncio.to_thread( + _parse_to_tmp, final, raw, filename, header_row, mixed, raw_mode) + except BaseException: + if fresh: + await asyncio.to_thread(_prune_build_dirs, final) + raise + + async def schema(view: SourceView, info: PublishInfo) -> dict[str, Any]: + # 经模块里的旧名调用：期 1 的测试按这个名字替换探查，模拟发布到一半失败 + cache = await _introspect_view(view, info.report) + _check_probe(cache, info.report, source_id) + return cache + + return await publish_build( + session, source, tmp_db=tmp, build_id=bid, build_options=options, engine_ver=ENGINE_VER, + report=report_to_json(parsed), raw=raw, raw_sha=raw_sha, file_name=filename, + file_size=len(raw), schema_cache_for=schema, import_fields={}, + ) # -------------------------------------------------------------------------- -# 删除数据源 +# 暂存区：结束、瘦身、过期、原件的存与放 +# -------------------------------------------------------------------------- + +#: 暂存区多久不提交就过期（D9 未拍板，推测；常量可调） +STAGING_TTL = timedelta(days=7) +#: 结束（瘦身）后的暂存区行保留多久。之后由回收删掉：用量和同意记录已经抄进配方表和导入清单 +STAGING_KEEP_CLOSED = timedelta(days=90) +#: 配方源保留最近几次导入的原件（按 seq，被替换的也算）。期 3 的「破坏性变更用新配方重放历史原件」、 +#: 解析器升级的回归重放都要用到历史原件；构建库照常回收，需要时按原件重建。N 的取值待拍板 +RECIPE_RAW_KEEP = 12 +STAGING_OPEN = ("drafting", "trialed", "rejected") +STAGING_CLOSED = ("committed", "discarded", "expired") +#: 暂存区结束时清空的字段和清空后的值。网格预览里有数字格的值和隐藏行列，试运行回执里有区域外文字 +#: 全文，回答里有用户写的理由：用完就不该留在库里（接口没有认证）。留下的是 id、来源、文件哈希、 +#: 状态、工作配方及其哈希、用量、同意记录、署名和时间 +STAGING_SLIM: dict[str, Any] = { + "scan": None, "grid_preview": None, "facts": None, "drafts": None, "trial": None, + "answers_base": None, "cards": [], "questions": [], "draft_problems": [], "draft_partial": False, + "answers": {}, "recipe_problems": [], "context_inputs": {}, "trial_key": None, "trial_path": None, +} + + +def staging_is_open(staging: ImportStaging) -> bool: + return staging.status in STAGING_OPEN + + +def close_staging(staging: ImportStaging, status: str, *, now: datetime | None = None) -> str | None: + """暂存区进入结束状态并瘦身（不提交）。返回它原来的试运行库路径：提交之后交给 remove_trial_file 删。 + + 文件放到提交之后再删：提交失败时暂存区还开着，库还在，用户不用重新试运行。提交成功、删文件失败 + 只是留下一个孤儿，回收和启动清理会按「暂存区已结束」把它删掉。 + """ + if status not in STAGING_CLOSED: + raise ValueError(f"暂存区的结束状态只能是 {STAGING_CLOSED}，不能是 {status!r}") + now = now or utcnow() + old_trial = staging.trial_path + for name, value in STAGING_SLIM.items(): + setattr(staging, name, copy.deepcopy(value)) + staging.status, staging.closed_at, staging.updated_at = status, now, now + return old_trial + + +def remove_trial_file(path: str | None) -> bool: + """删一个试运行库（连同日志文件）。只删上传目录里 trials/ 下的：记录被改坏了也不能删到别处去。""" + if not path: + return False + target = Path(path) + if target.parent.name != TRIALS_DIR or not _under(target, tables_root()): + logger.warning("没有删除试运行库以外的文件：%s", path) + return False + existed = target.exists() + try: + _remove_db_files(target) + except OSError as e: + logger.warning("没能删掉试运行库 %s：%s", path, e) + return False + return existed + + +def _expiry_due(now: datetime) -> Any: + """「该过期了」的条件。expires_at 没写的按 created_at 加 TTL 算，免得漏写一列就永不过期。""" + return and_( + ImportStaging.status.in_(STAGING_OPEN), + or_( + and_(ImportStaging.expires_at.is_not(None), ImportStaging.expires_at <= now), + and_(ImportStaging.expires_at.is_(None), ImportStaging.created_at <= now - STAGING_TTL), + ), + ) + + +def _closed_long_ago(now: datetime) -> Any: + """「结束超过 90 天」的条件。closed_at 没写的按 updated_at 算（同 _expiry_due 的兜底）： + 哪条结束路径只改了状态、没经过 close_staging，行也不会因此永远留着。""" + cutoff = now - STAGING_KEEP_CLOSED + return and_( + ImportStaging.status.in_(STAGING_CLOSED), + or_( + and_(ImportStaging.closed_at.is_not(None), ImportStaging.closed_at <= cutoff), + and_(ImportStaging.closed_at.is_(None), ImportStaging.updated_at <= cutoff), + ), + ) + + +async def _raw_in_use(session: AsyncSession, sha: str, *, except_staging: str | None = None) -> bool: + """这份原件还有没有人要：留着原件的导入记录（raw_state=kept），或者未结束的暂存区。""" + if (await session.execute( + select(TableImport.id).where(TableImport.raw_sha256 == sha, TableImport.raw_state == "kept") + .limit(1) + )).first() is not None: + return True + cond = [ImportStaging.raw_sha256 == sha, ImportStaging.status.in_(STAGING_OPEN)] + if except_staging: + cond.append(ImportStaging.id != except_staging) + return (await session.execute(select(ImportStaging.id).where(*cond).limit(1))).first() is not None + + +async def _release_raws_locked(session: AsyncSession, shas: set[str]) -> set[str]: + """（持锁）这些原件没人要了就删掉，返回删了的。暂存区引用的原件没有导入记录记着，不在这里删就成了孤儿。""" + unused = {sha for sha in shas if sha and not await _raw_in_use(session, sha)} + + def _purge() -> set[str]: + return {sha for sha in unused if raw_store.purge_raw(sha, prune=True)} + + return await asyncio.to_thread(_purge) if unused else set() + + +async def _expire_stagings_locked(session: AsyncSession, now: datetime) -> list[str]: + """（持锁）过期的暂存区 → expired、瘦身、删试运行库、放掉没人要的原件；结束超过 90 天的行删掉（提交）。""" + expired = list((await session.execute( + select(ImportStaging).where(_expiry_due(now)).order_by(ImportStaging.created_at) + )).scalars()) + ids, shas = [s.id for s in expired], {s.raw_sha256 for s in expired} + trials = [close_staging(s, "expired", now=now) for s in expired] + old = (await session.execute( + select(func.count()).select_from(ImportStaging).where(_closed_long_ago(now)) + )).scalar() or 0 + if old: + await session.execute(delete(ImportStaging).where(_closed_long_ago(now)) + .execution_options(synchronize_session=False)) + if not expired and not old: + return [] + await session.commit() + for path in trials: + await asyncio.to_thread(remove_trial_file, path) + await _release_raws_locked(session, shas) + if expired: + logger.info("%d 个导入暂存区已过期并清空", len(expired)) + return ids + + +async def expire_stagings(session: AsyncSession, *, now: datetime | None = None) -> list[str]: + """过期的暂存区 → expired（瘦身、删试运行库、原件没人要了就删）；结束超过 90 天的行删除。提交。 + + 返回这次过期的暂存区 id。回收（gc）开头会做一遍；但回收只在发布、删源、启动时跑，光靠它 7 天的 + TTL 不一定生效，所以暂存、上传新一期、查看暂存区时也顺手调一次。调用时会话里不要有没提交的改动。 + + 先在锁外看一眼有没有要处理的：没有就不去等锁，查看暂存区不该因为别处正在发布大文件而排队。 + """ + now = now or utcnow() + pending = (await session.execute( + select(ImportStaging.id).where(or_(_expiry_due(now), _closed_long_ago(now))).limit(1) + )).first() + if pending is None: + return [] + async with store_lock(): + return await _expire_stagings_locked(session, now) + + +async def store_staging(session: AsyncSession, staging: ImportStaging, raw: bytes) -> None: + """存原件、把暂存区写进库（提交），全程持版本存储的锁。 + + 原件和引用它的暂存区必须在同一把锁里落地：回收、清除原件、放弃导入都持着这把锁判断「这份原件 + 还有没有人要」，锁外先存原件、过一会儿才提交暂存区，中间插进来的回收会把它当成没人要的删掉。 + expires_at 没写的按现在加 STAGING_TTL 补上。提交失败时，这次新存的原件删掉。 + """ + sha = hashlib.sha256(raw).hexdigest() + if staging.raw_sha256 and staging.raw_sha256 != sha: + raise ValueError("暂存区登记的原件哈希与这份内容不符") + staging.raw_sha256 = sha + if staging.file_size in (None, 0): + staging.file_size = len(raw) + if staging.expires_at is None: + staging.expires_at = utcnow() + STAGING_TTL + async with store_lock(): + raw_new = not raw_store.raw_exists(sha) + try: + await asyncio.to_thread(raw_store.put_raw, raw) + session.add(staging) + await session.commit() + except BaseException: + try: + await session.rollback() + except Exception: # noqa: BLE001 - 回滚失败也要先把原件收拾掉，再抛原来的异常 + logger.exception("导入暂存区写入失败后回滚出错") + if raw_new: + raw_store.purge_raw(sha, prune=True) + raise + + +async def release_staging_raw(session: AsyncSession, staging: ImportStaging) -> bool: + """放弃导入之后调用：这份原件已经没有别的引用（没有留着原件的导入记录、没有别的未结束暂存区）就 + 当场删掉，不等回收。返回是否删了文件。 + + 自己取版本存储的锁，**不要在持锁时调用**。暂存区必须已经结束（一般是刚提交成 discarded）， + 还开着的一律不删：它自己就是这份原件的引用。 + """ + if staging.status not in STAGING_CLOSED or not staging.raw_sha256: + return False + sha = staging.raw_sha256 + async with store_lock(): + if await _raw_in_use(session, sha, except_staging=staging.id): + return False + return await asyncio.to_thread(lambda: raw_store.purge_raw(sha, prune=True)) + + +# -------------------------------------------------------------------------- +# 删除数据源、清除原件 # -------------------------------------------------------------------------- async def retire_source(session: AsyncSession, source_id: str) -> None: - """数据源删除时调用（不提交）：它的导入记录全部置 retired。文件交给回收，被运行引用的会保留。""" + """数据源删除时调用（不提交）：导入记录、配方全部置 retired，未结束的暂存区放弃（瘦身）。 + + 文件交给随后的回收：被运行引用的版本会保留；放弃的暂存区留下的试运行库、没人要的原件也由回收删。 + """ await session.execute( update(TableImport).where(TableImport.source_id == source_id, TableImport.status != "retired") .values(status="retired") ) + await session.execute( + update(TableRecipe).where(TableRecipe.source_id == source_id, TableRecipe.status != "retired") + .values(status="retired") + ) + now = utcnow() + for staging in (await session.execute( + select(ImportStaging).where(ImportStaging.source_id == source_id, + ImportStaging.status.in_(STAGING_OPEN)) + )).scalars(): + close_staging(staging, "discarded", now=now) + + +@dataclass +class RawPurge: + """清除原件的结果。按两元组解包得到 (是否删了文件, 一并标成已清除的其他导入记录)，和期 1 一样。""" + + deleted: bool + others: list[TableImport] + #: 引用同一份原件、随之一并放弃的未结束暂存区（已瘦身） + discarded_stagings: list[ImportStaging] = field(default_factory=list) + + def __iter__(self) -> Iterator[Any]: + return iter((self.deleted, self.others)) async def purge_import_raw( session: AsyncSession, imp: TableImport, *, reason: str, signed_by: str | None -) -> tuple[bool, list[TableImport]]: - """清除一次导入的原件（提交）。返回 (是否删了文件, 一并标成已清除的其他导入记录)。 +) -> RawPurge: + """清除一次导入的原件（提交）。按两元组解包得到 (是否删了文件, 一并标成已清除的其他导入记录)， + discarded_stagings 是一并放弃的暂存区。 **按内容清除。** 原件按哈希只存一份，同一个文件传过两次、传给过两个源，共用的是同一个 文件。清除的目的是让含敏感内容的原件从服务器上消失（D13），所以文件一定删；同一份内容 下其他仍保留原件的导入一并标成已清除，记录里写明是随哪次导入清除的，回执逐条列出。 + 引用同一份原件的未结束暂存区也在同一个事务里放弃并瘦身、删掉试运行库：不这么做，它们随后 + 试运行或提交时读不到原件，暂存区里那份网格和回执也照样留着，清除就没有达到目的。 持版本存储的锁：发布正在把同一份原件登记给新导入时，不能在它提交前把文件删掉。 """ - async with _store_lock(): + async with store_lock(): + sha = imp.raw_sha256 others = list((await session.execute( select(TableImport).where( - TableImport.raw_sha256 == imp.raw_sha256, TableImport.raw_state == "kept", + TableImport.raw_sha256 == sha, TableImport.raw_state == "kept", TableImport.id != imp.id, ).order_by(TableImport.source_id, TableImport.seq) )).scalars()) + stagings = list((await session.execute( + select(ImportStaging).where( + ImportStaging.raw_sha256 == sha, ImportStaging.status.in_(STAGING_OPEN), + ).order_by(ImportStaging.created_at) + )).scalars()) if sha else [] + now = utcnow() record = { - "at": utcnow().isoformat(), "reason": reason, + "at": now.isoformat(), "reason": reason, "signed_by": (signed_by or "").strip() or None, "signed_by_verified": False, } imp.raw_state, imp.purged = "purged", record for other in others: other.raw_state, other.purged = "purged", {**record, "via_import": imp.id} + trials = [close_staging(s, "discarded", now=now) for s in stagings] await session.commit() - deleted = await asyncio.to_thread(lambda: raw_store.purge_raw(imp.raw_sha256, prune=True)) - return deleted, others + for path in trials: + await asyncio.to_thread(remove_trial_file, path) + deleted = await asyncio.to_thread(lambda: raw_store.purge_raw(sha, prune=True)) + return RawPurge(deleted=deleted, others=others, discarded_stagings=stagings) # -------------------------------------------------------------------------- @@ -834,8 +1273,15 @@ async def gc(session: AsyncSession) -> None: 快照 ∪ 各源当前的快照。受保护的是这些快照、它们的导入记录、构建库，以及这些导入记录里 还留着的原件。其余的：记录先标 retired 并提交，再删文件——删到一半出错，下次回收会接着删 （已 retired、没被保护的文件照样会被处理）。 + + 按配方导入之后多了几条： + - 开头先让过期的暂存区过期（expire_stagings 的同一套处理）； + - 原件的保护集合并上未结束暂存区的原件（被拒收、正在修改的那份不能删），以及配方源最近 + RECIPE_RAW_KEEP 次导入的原件（被替换、构建已回收的也留着原件，重放要用）； + - 已结束的暂存区留下的原件没人要了就删，留下的试运行库也删（放弃、删源时没来得及删的）。 """ - async with _store_lock(): + async with store_lock(): + await _expire_stagings_locked(session, utcnow()) referenced: set[str] = set() for (versions,) in (await session.execute(select(Run.data_versions))).all(): referenced |= _snapshots_in(versions) @@ -857,6 +1303,17 @@ async def gc(session: AsyncSession) -> None: kept_builds = {i.build_id for i in imports if i.id in kept_imports} kept_raw = {i.raw_sha256 for i in imports if i.id in kept_imports and i.raw_state == "kept" and i.raw_sha256} + kept_raw |= _recent_recipe_raws(imports, { + sid for (sid,) in (await session.execute( + select(DataSource.id).where(DataSource.current_recipe_id.is_not(None)) + )).all() + }) + stagings = (await session.execute( + select(ImportStaging.id, ImportStaging.status, ImportStaging.raw_sha256) + )).all() + open_raw = {sha for (_sid, status, sha) in stagings if status in STAGING_OPEN and sha} + closed_stagings = {sid for (sid, status, _sha) in stagings if status in STAGING_CLOSED} + kept_raw |= open_raw build_paths = {b.db_path for b in builds} now = utcnow() @@ -889,6 +1346,10 @@ async def gc(session: AsyncSession) -> None: imp.purged = {"at": now.isoformat(), "reason": "版本已回收，原件随之删除", "signed_by": None, "auto": True} raws.add(imp.raw_sha256) + # 已结束的暂存区引用过的原件：没有导入记录留着它、也没有未结束的暂存区要它，就是没人要了 + still_kept = {i.raw_sha256 for i in imports if i.raw_state == "kept" and i.raw_sha256} | open_raw + raws |= {sha for (_sid, status, sha) in stagings + if status in STAGING_CLOSED and sha and sha not in still_kept} await session.commit() def _delete() -> None: @@ -897,6 +1358,11 @@ async def gc(session: AsyncSession) -> None: _delete_store_file(path) for sha in raws: raw_store.purge_raw(sha, prune=True) + # 只删「暂存区已结束」的试运行库。暂存区不存在的不在这里删：上传新一期可能先试运行、后写暂存区， + # 发布后的回收撞上这个空档会删掉正要登记的库；这类孤儿留给启动清理（那时没有请求在跑） + for path in _trial_files(): + if _trial_owner(path) in closed_stagings: + _remove_db_files(path) await asyncio.to_thread(_delete) # 被删掉文件的快照可能还有引擎开着文件句柄：已删除的文件要等句柄关了才真正释放空间 @@ -909,6 +1375,40 @@ async def gc(session: AsyncSession) -> None: # -------------------------------------------------------------------------- +def _recent_recipe_raws(imports: list[TableImport], recipe_sources: set[str]) -> set[str]: + """配方源最近 RECIPE_RAW_KEEP 次导入（按 seq，不论状态）还留着的原件。""" + by_source: dict[str, list[TableImport]] = {} + for imp in imports: + if imp.source_id in recipe_sources: + by_source.setdefault(imp.source_id, []).append(imp) + kept: set[str] = set() + for items in by_source.values(): + items.sort(key=lambda i: i.seq or 0, reverse=True) + kept |= {i.raw_sha256 for i in items[:RECIPE_RAW_KEEP] if i.raw_state == "kept" and i.raw_sha256} + return kept + + +def _trial_files() -> list[Path]: + root = tables_root() + return sorted(root.glob(f"*/{TRIALS_DIR}/*.db")) if root.is_dir() else [] + + +def _trial_owner(path: Path) -> str: + """试运行库文件名里的暂存区 id（<暂存区 id>-.db）。 + + 从右边切：trial_key 只有字母数字，暂存区 id 却可以带「-」。从左边切会把 stg-1 认成 stg， + 启动清理就把一个开着的暂存区的库当孤儿删掉，回收也认不出已结束的暂存区。 + """ + return path.name.removesuffix(".db").rsplit("-", 1)[0] + + +def _same_path(a: str | os.PathLike[str], b: str | os.PathLike[str]) -> bool: + try: + return os.path.realpath(a) == os.path.realpath(b) + except (OSError, ValueError): + return False + + def _sweep_tmp() -> int: """删掉上次崩溃留下的 *.tmp-* 半成品（库和原件都是先写临时文件再 link）。""" removed = 0 @@ -930,6 +1430,17 @@ async def _sweep_orphans(session: AsyncSession) -> None: select(TableImport.raw_sha256).where(TableImport.raw_state == "kept") )).all() if sha } + rows = (await session.execute( + select(ImportStaging.id, ImportStaging.status, ImportStaging.trial_path, ImportStaging.raw_sha256) + )).all() + staging_rows = {sid: (status, trial_path) for (sid, status, trial_path, _sha) in rows} + # 未结束的暂存区引用的原件没有导入记录记着，不并进来就会被当成孤儿删掉 + kept_raw |= {sha for (_sid, status, _path, sha) in rows if status in STAGING_OPEN and sha} + + def _stale_trial(path: Path) -> bool: + """试运行库：暂存区不在、已结束、或者它指着的不是这个文件，就是孤儿。""" + row = staging_rows.get(_trial_owner(path)) + return row is None or row[0] not in STAGING_OPEN or not row[1] or not _same_path(row[1], path) def _sweep() -> list[Path]: orphans: list[Path] = [] @@ -943,12 +1454,13 @@ async def _sweep_orphans(session: AsyncSession) -> None: for path in raw_root.glob("*/*"): if path.is_file() and path.name not in kept_raw and _TMP not in path.name: orphans.append(path) + orphans.extend(path for path in _trial_files() if _stale_trial(path)) for path in orphans: _remove_db_files(path) # 空目录：被拒收的新源留下的 tables/<源 id>/builds/、清除原件后的 raw/<前两位>/。 # 启动时持着锁、还没开始接请求，删了也不会和正在写的发布撞上（发布会按需重建目录） if root.is_dir(): - for builds in root.glob("*/builds"): + for builds in [*root.glob("*/builds"), *root.glob(f"*/{TRIALS_DIR}")]: if builds.is_dir() and raw_store.remove_empty_dir(builds): orphans.append(builds) for folder in root.iterdir(): diff --git a/backend/app/data/tabular.py b/backend/app/data/tabular.py index 7b4014a1..512d9f1b 100644 --- a/backend/app/data/tabular.py +++ b/backend/app/data/tabular.py @@ -46,7 +46,7 @@ from app.data.xlsx_scan import SheetScan, TableScan, UnsupportedTable, WorkbookS __all__ = [ "CONVERSION_KINDS", "SHAPE_KINDS", "UNSHAPED_NOTE", "LoadReport", "LoadedTable", "NeedsDecision", - "UnsupportedTable", "infer_type", "load_into", + "UnsupportedTable", "as_text", "infer_type", "load_into", "open_book", "worksheet_for", ] #: 按原样导入（未规整）的表写进表结构的说明（data/table_versions.py 注入 schema_cache 的 comment）。 @@ -309,6 +309,21 @@ def _as_text(value: Any, cls: int, payload: Any) -> str: return str(value) +def as_text(value: Any) -> str: + """一个单元格要存进 TEXT 列的写法，与简单导入完全一致（配方导入的列表 TEXT 列、主键列共用）。 + + 零点的日期写成 YYYY-MM-DD，带时刻的 YYYY-MM-DD HH:MM:SS，time 写成 HH:MM:SS，原生数字按常规写法， + 文本去首尾空白。两条导入路径各写一套的话，期 1 修掉的「日期存成 … 00:00:00」（#23）会在另一条路上重现。 + 空值（None）是空字符串：调用方先判空，这里只保证不写出「None」。 + """ + if value is None: + return "" + cls, payload, _ = _classify(value) + if cls == _EMPTY: + return "" + return _as_text(value, cls, payload) + + def _is_date_like(value: Any) -> bool: return _date_like(_classify(value)) @@ -884,17 +899,52 @@ def _csv_sources(raw: bytes, filename: str, header_row: int) -> list[_Source]: csv=True)] -def _open_book(raw: bytes) -> Any: +def _quick_get_size(self: Any) -> None: + """openpyxl ReadOnlyWorksheet._get_size 的等价实现，只是读到 开头就停。 + + 原实现用 iterparse 的结束事件找 ， 的结束事件要等整张表读完才出现：文件里没有 + 时（openpyxl write_only 等导出工具都不写），每次打开工作簿都把每张表整个解析一遍（20 万行约 + 3 秒）。按 OOXML 的元素顺序 在 之前，所以改用开始事件、碰到 就停， + 结果与原实现完全相同（有就取它，没有就留空）；读取层本来也先 reset_dimensions、按扫描算出的边界读。 + """ + from openpyxl.worksheet._reader import DATA_TAG, DIMENSION_TAG + from openpyxl.worksheet.dimensions import SheetDimension + from openpyxl.xml.functions import iterparse + + src = self._get_source() + try: + for _event, element in iterparse(src, events=("start",)): + if element.tag == DIMENSION_TAG: + self._min_column, self._min_row, self._max_column, self._max_row = \ + SheetDimension.from_tree(element).boundaries + return + if element.tag == DATA_TAG: + return + finally: + src.close() + + +def _install_quick_size() -> None: + """装上 _quick_get_size（幂等）。openpyxl 改了内部结构（没有 _get_size）就什么都不做，照原样慢一点。""" + from openpyxl.worksheet._read_only import ReadOnlyWorksheet + + if getattr(ReadOnlyWorksheet, "_get_size", None) not in (None, _quick_get_size): + ReadOnlyWorksheet._get_size = _quick_get_size + + +def open_book(raw: bytes, *, data_only: bool = True) -> Any: + """只读打开工作簿。data_only=False 取公式本身（配方读取层的公式一份，xlsx_cells.Workbooks）。""" try: from openpyxl import load_workbook except ImportError as e: raise UnsupportedTable( "读取 Excel 需要安装解析库：pip install 'agentlab-backend[docs]'" ) from e + _install_quick_size() try: # read_only 走流式；data_only 取公式算出来的值而不是公式本身——用户要的是数。 # keep_links=False：外部链接里是别的工作簿的缓存数据（可以很大），导入用不上，不让它整份建树 - return load_workbook(io.BytesIO(raw), read_only=True, data_only=True, keep_links=False) + return load_workbook(io.BytesIO(raw), read_only=True, data_only=data_only, keep_links=False) except Exception as e: # noqa: BLE001 # 这句话原样进上传的 400：异常类名留在日志（from e），界面上只说能照着做的 raise UnsupportedTable( @@ -902,7 +952,7 @@ def _open_book(raw: bytes) -> Any: ) from e -def _worksheet_for(book: Any, sheet: xlsx_scan.SheetScan) -> Any: +def worksheet_for(book: Any, sheet: xlsx_scan.SheetScan) -> Any: """扫描结果里的一张工作表 → openpyxl 里的同一张。按部件路径配对，不按名字。 按名字取（book[名字]）时，openpyxl 返回第一个同名的表；扫描说「可见」的那张和导入实际读的 @@ -916,6 +966,11 @@ def _worksheet_for(book: Any, sheet: xlsx_scan.SheetScan) -> Any: return found[0] +#: 旧名字：期 1 的调用方和测试照旧可用（配方导入的读取层 xlsx_cells 用上面的公开名） +_open_book = open_book +_worksheet_for = worksheet_for + + def _excel_sources(scan: WorkbookScan, book: Any, header_row: int, report: LoadReport) -> list[_Source]: sources: list[_Source] = [] #: 因为表头行号超出了内容而跳过的可见工作表：(表名, 最后一行)。全被这样跳过时报的是行号填错了 diff --git a/backend/app/data/xlsx_cells.py b/backend/app/data/xlsx_cells.py new file mode 100644 index 00000000..28f137e0 --- /dev/null +++ b/backend/app/data/xlsx_cells.py @@ -0,0 +1,325 @@ +"""配方导入的读取层：一张可见工作表读成网格（小表）或逐行流（大列表），外加给界面的网格预览。 + +**两遍读取，口径一致。** 第一遍是 xlsx_scan（标准库流式扫描：边界、合并区、隐藏行列、逐行非空格数）， +这里是第二遍（openpyxl read_only）。两遍各数一次非空格，配方执行器逐行对账：对不上说明两个解析器看到的 +不是同一张表（XML 行号乱序、重复的行 openpyxl 会静默跳过），整份拒收，不猜。所以「非空」的口径必须和 +xlsx_scan 一字不差：值不是 None、也不是只含空白的文本，或者是公式格（哪怕没有缓存值）；0 和 False 不算空。 + +**值和公式各读一份。** 值来自 data_only 的工作簿（用户要的是数），公式原文只有 data_only=False 才拿得到。 +工作表有公式（SheetScan.formulas > 0）时再开一份 data_only=False 的只读工作簿，两份按行同步迭代：两个生成器 +解析的是同一段 XML，行与行一一对应。openpyxl 只读模式会把共享公式展开成各格自己的写法（B5 的 +`` 读成 `=SUM(B1:B4)`），合计公式的引用核对靠的就是这一点。 + +**边界用扫描算出来的。** `` 是写文件的程序自己填的，可能陈旧或者干脆没有：先 reset_dimensions， +再按 SheetScan.bounds 传 min/max 行列（与期 1 的 tabular.load_into 相同）。 + +隐藏工作表一律不读（H10）：worksheet_for 只认可见的工作表，按部件路径配对，名字、路径、状态对不上就拒收。 +""" +from __future__ import annotations + +import datetime as _dt +import logging +import zipfile +from collections.abc import Iterator +from typing import Any +from xml.etree import ElementTree as ET + +from app.data import tabular +from app.data.recipe_types import Grid, GridCell +from app.data.xlsx_scan import SheetScan, UnsupportedTable, WorkbookScan, _broken, cell_ref, parse_ref + +logger = logging.getLogger(__name__) +#: 读格子时 openpyxl、zipfile、XML 解析会抛的异常（数字格的 写坏了是 ValueError）：一律转成 UnsupportedTable， +#: 固定中文文案。不转的话会经全局 ValueError 处理器把英文原文连同格子内容回成不带 code 的 400（SE-7） +_READ_ERRORS = (ValueError, KeyError, ET.ParseError, zipfile.BadZipFile, EOFError) + +__all__ = [ + "GRID_MAX_CELLS", "PREVIEW_MAX_COLS", "PREVIEW_MAX_ROWS", "GridTooLarge", "Workbooks", "display", "grid_from", + "iter_rows", "parse_range", "preview", "read_grid", "sheet_rows", "visible_sheet", +] + +#: 物化网格的非空格上限。更大的工作表走流式（iter_rows），配方执行器在那条路上只支持一个列表块 +GRID_MAX_CELLS = 200_000 +#: 网格预览最多给多少行、多少列（界面只读展示，再多也看不过来） +PREVIEW_MAX_ROWS, PREVIEW_MAX_COLS = 300, 60 +#: 预览里一格文字最多多少字（长备注截断，原文在证据面板） +PREVIEW_TEXT_MAX = 200 + + +class GridTooLarge(UnsupportedTable): + """非空格超过 GRID_MAX_CELLS、又没有给 max_rows：不物化，调用方改走 iter_rows。""" + + +# -------------------------------------------------------------------------- +# 工作簿 +# -------------------------------------------------------------------------- + + +def _open_formula_book(raw: bytes) -> Any: + return tabular.open_book(raw, data_only=False) + + +class Workbooks: + """一次执行里复用的两份只读工作簿：值一份（data_only），公式一份（第一次要读有公式的工作表时才开）。 + + 打开一次工作簿要解析样式表和共享字符串表，多张工作表、多次读取各开一遍很浪费。用完 close()： + 只读工作簿一直开着压缩包。 + """ + + def __init__(self, raw: bytes) -> None: + self.raw = raw + self._values: Any = None + self._formulas: Any = None + + def values(self) -> Any: + if self._values is None: + self._values = tabular.open_book(self.raw) + return self._values + + def formulas(self) -> Any: + if self._formulas is None: + self._formulas = _open_formula_book(self.raw) + return self._formulas + + def close(self) -> None: + for book in (self._values, self._formulas): + if book is not None: + try: + book.close() + except Exception: # noqa: BLE001 关闭失败不影响已经读出来的结果 + pass + self._values = self._formulas = None + + def __enter__(self) -> Workbooks: + return self + + def __exit__(self, *_exc: object) -> None: + self.close() + + +def visible_sheet(scan: WorkbookScan, sheet: str) -> SheetScan: + """扫描结果里的一张可见工作表；没有这张表、或者它是隐藏的，抛 UnsupportedTable。""" + found = scan.sheet(sheet) + if found is None: + raise UnsupportedTable(f"工作簿中没有名为「{sheet}」的工作表") + if found.state != "visible": + raise UnsupportedTable(f"工作表「{sheet}」是隐藏的工作表，不读取") + return found + + +# -------------------------------------------------------------------------- +# 逐行读取 +# -------------------------------------------------------------------------- + + +def _formula_text(value: Any) -> str: + """openpyxl 给的公式：普通公式是「=…」字符串，数组公式是 ArrayFormula（.text 是原文）。""" + text = value if isinstance(value, str) else getattr(value, "text", None) or str(value) + return text if text.startswith("=") else f"={text}" + + +def sheet_rows(books: Workbooks, sheet: SheetScan, *, min_row: int, max_row: int, min_col: int, + max_col: int) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]: + """按行流式读一个矩形：每行给 (行号, [(列号, 格)…])，只给非空格、按列升序；整行为空的不给。 + + 值和公式两份工作簿按行同步迭代（zip）：openpyxl 只读模式按行号补齐缺的行，两边解析同一段 XML， + 行数和每行的宽度都一样。文件内容写坏了（见 _READ_ERRORS）抛 UnsupportedTable，日志只记异常类型。 + """ + try: + yield from _sheet_rows(books, sheet, min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col) + except UnsupportedTable: + raise + except _READ_ERRORS as e: + logger.info("reading sheet failed: %s", type(e).__name__) + raise UnsupportedTable(_broken()) from e + + +def _sheet_rows(books: Workbooks, sheet: SheetScan, *, min_row: int, max_row: int, min_col: int, + max_col: int) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]: + if min_row > max_row or min_col > max_col: + return + ws = tabular.worksheet_for(books.values(), sheet) + ws.reset_dimensions() # 不信 ：边界用扫描算出来的 + values = ws.iter_rows(min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col, values_only=True) + formulas: Iterator[Any] | None = None + if sheet.formulas > 0: + fws = tabular.worksheet_for(books.formulas(), sheet) + fws.reset_dimensions() + formulas = iter(fws.iter_rows(min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col, + values_only=False)) + row = min_row + for vrow in values: + frow = next(formulas, None) if formulas is not None else None + out: list[tuple[int, GridCell]] = [] + col = min_col + for i, v in enumerate(vrow): + f = None + if frow is not None and i < len(frow): + fc = frow[i] + if getattr(fc, "data_type", None) == "f" and fc.value is not None: + f = _formula_text(fc.value) + if f is None and (v is None or (type(v) is str and not v.strip())): + col += 1 + continue + out.append((col, GridCell(v, f))) + col += 1 + if out: + yield row, out + row += 1 + + +def iter_rows(raw: bytes, scan: WorkbookScan, sheet: str, *, min_row: int, max_row: int, + min_col: int, max_col: int) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]: + """流式读一张可见工作表的一个矩形（大列表用）：每行只给非空格（按列排序），整行为空的行不给。""" + found = visible_sheet(scan, sheet) + books = Workbooks(raw) + try: + yield from sheet_rows(books, found, min_row=min_row, max_row=max_row, min_col=min_col, max_col=max_col) + finally: + books.close() + + +# -------------------------------------------------------------------------- +# 网格 +# -------------------------------------------------------------------------- + + +def parse_range(ref: str) -> tuple[int, int, int, int] | None: + """「B2:AG2」→ (2, 2, 2, 33)；单格「B2」→ (2, 2, 2, 2)；写坏了 None。""" + a, _, b = ref.replace("$", "").partition(":") + try: + r1, c1 = parse_ref(a) + r2, c2 = parse_ref(b or a) + except ValueError: + return None + if r1 <= 0 or c1 <= 0 or r2 <= 0 or c2 <= 0: + return None + return min(r1, r2), min(c1, c2), max(r1, r2), max(c1, c2) + + +def _expand(spans: list[tuple[int, int]], lo: int, hi: int) -> set[int]: + out: set[int] = set() + for a, b in spans: + a, b = max(a, lo), min(b, hi) + if a <= b: + out.update(range(a, b + 1)) + return out + + +def grid_from(books: Workbooks, sheet: SheetScan, *, max_rows: int | None = None) -> Grid: + """read_grid 的内部版本：用调用方开好的工作簿（配方执行器一次执行里读多张表时复用）。""" + b = sheet.bounds + if b is None: + return Grid(sheet=sheet.name, bounds=None) + if max_rows is None and sheet.nonempty > GRID_MAX_CELLS: + raise GridTooLarge( + f"工作表「{sheet.name}」有 {sheet.nonempty:,} 个非空单元格，超过网格读取的上限（{GRID_MAX_CELLS:,} 个）") + last = b.max_row if max_rows is None else min(b.max_row, b.min_row + max(max_rows, 1) - 1) + truncated = last < b.max_row + cells: dict[tuple[int, int], GridCell] = {} + for r, row in sheet_rows(books, sheet, min_row=b.min_row, max_row=last, min_col=b.min_col, max_col=b.max_col): + if max_rows is not None and len(cells) + len(row) > GRID_MAX_CELLS: + # 前 max_rows 行就已经超过上限（极宽的表）：读到上一行为止 + truncated, last = True, r - 1 + break + for c, cell in row: + cells[(r, c)] = cell + merges = [] + for ref in sheet.merged: + span = parse_range(ref) + if span is not None and span[0] <= last and span[2] >= b.min_row: + merges.append(span) + return Grid( + sheet=sheet.name, bounds=(b.min_row, b.min_col, b.max_row, b.max_col), cells=cells, merges=merges, + hidden_rows=_expand(sheet.hidden_rows, b.min_row, last), + hidden_cols=_expand(sheet.hidden_cols, b.min_col, b.max_col), + truncated=truncated, + ) + + +def read_grid(raw: bytes, scan: WorkbookScan, sheet: str, *, max_rows: int | None = None) -> Grid: + """物化一张可见工作表。 + + 隐藏工作表抛 UnsupportedTable。非空格超过 GRID_MAX_CELLS 且没给 max_rows 抛 GridTooLarge；给了 max_rows + 就只读已用区域的前 max_rows 行（起草、预览用），读不完时 truncated=True。Grid.bounds 始终是整张表的真实边界 + （界面要知道表有多大），读到哪一行看 truncated 和 cells。隐藏行列、合并区只保留落在读取范围内的。 + """ + found = visible_sheet(scan, sheet) + with Workbooks(raw) as books: + return grid_from(books, found, max_rows=max_rows) + + +# -------------------------------------------------------------------------- +# 预览 +# -------------------------------------------------------------------------- + + +def display(cell: GridCell) -> tuple[str, str]: + """一格在预览里的 (写法, 种类)。种类：text / number / date / bool / error / formula / formula_uncached。""" + v = cell.value + if cell.formula is not None: + if v is None: + return "", "formula_uncached" + return _clip(tabular.as_text(v)), "formula" + if isinstance(v, bool): + return ("TRUE" if v else "FALSE"), "bool" + if isinstance(v, (int, float)): + return tabular.as_text(v), "number" + if isinstance(v, str): + text = v.strip() + return _clip(text), ("error" if text in tabular.EXCEL_ERRORS else "text") + text = tabular.as_text(v) + return _clip(text), ("date" if isinstance(v, (_dt.datetime, _dt.date, _dt.time)) else "text") + + +def _clip(text: str) -> str: + return text if len(text) <= PREVIEW_TEXT_MAX else text[:PREVIEW_TEXT_MAX] + "…" + + +def _range_a1(span: tuple[int, int, int, int]) -> str: + r1, c1, r2, c2 = span + return cell_ref(r1, c1) if (r1, c1) == (r2, c2) else f"{cell_ref(r1, c1)}:{cell_ref(r2, c2)}" + + +def preview(grid: Grid, *, max_rows: int = PREVIEW_MAX_ROWS, max_cols: int = PREVIEW_MAX_COLS) -> dict[str, Any]: + """给界面的网格预览（P2-SPEC 7.4 的 GridPreview）：左上角起最多 max_rows 行、max_cols 列。 + + cells 是 [行, 列, 写法, 种类]；formulas 是 {坐标: 公式原文}（悬停时显示）。这是给用户看的，数字照给； + 发给模型的压缩表示另有一套（recipe_ai.compress），不从这里取。 + """ + if grid.bounds is None: + return {"sheet": grid.sheet, "bounds": None, "total_rows": 0, "total_cols": 0, "truncated": grid.truncated, + "rows": [], "cols": [], "cells": [], "formulas": {}, "merges": [], "hidden_rows": [], + "hidden_cols": []} + r1, c1, r2, c2 = grid.bounds + total_rows, total_cols = r2 - r1 + 1, c2 - c1 + 1 + last_r = min(r2, r1 + max(max_rows, 1) - 1) + last_c = min(c2, c1 + max(max_cols, 1) - 1) + cells: list[list[Any]] = [] + formulas: dict[str, str] = {} + for r in grid.row_numbers(): + if r > last_r: + break + for c, cell in grid.row(r): + if c > last_c: + break + text, kind = display(cell) + cells.append([r, c, text, kind]) + if cell.formula is not None: + formulas[cell_ref(r, c)] = cell.formula + merges = [_range_a1(m) for m in grid.merges + if m[0] <= last_r and m[2] >= r1 and m[1] <= last_c and m[3] >= c1] + return { + "sheet": grid.sheet, + "bounds": f"{cell_ref(r1, c1)}:{cell_ref(r2, c2)}", + "total_rows": total_rows, + "total_cols": total_cols, + "truncated": bool(grid.truncated or last_r < r2 or last_c < c2), + "rows": list(range(r1, last_r + 1)), + "cols": list(range(c1, last_c + 1)), + "cells": cells, + "formulas": formulas, + "merges": merges, + "hidden_rows": sorted(r for r in grid.hidden_rows if r1 <= r <= last_r), + "hidden_cols": sorted(c for c in grid.hidden_cols if c1 <= c <= last_c), + } diff --git a/backend/app/data/xlsx_scan.py b/backend/app/data/xlsx_scan.py index 4b3fd43f..4910c6ef 100644 --- a/backend/app/data/xlsx_scan.py +++ b/backend/app/data/xlsx_scan.py @@ -31,8 +31,8 @@ import xml.etree.ElementTree as ET import zipfile from xml.parsers import expat from contextlib import closing -from dataclasses import dataclass, field -from typing import IO +from dataclasses import asdict, dataclass, field, fields +from typing import IO, Any #: 解压后的总大小上限。上传本身限 50 MB，正常的 xlsx 压缩比在 5 到 20 倍之间； #: 超过这个数的基本是 zip 炸弹。可调 @@ -172,6 +172,10 @@ class SheetScan: far_cells: list[str] = field(default_factory=list) #: 文件自己声明的 ，只作记录，不用来定边界 dimension: str | None = None + #: 每行非空格数的游程：(起始行, 结束行, 每行个数)，只记个数 > 0 的行，按行号升序。 + #: 配方执行器拿它和第二遍（openpyxl）逐行读到的个数对账：两边数的口径一致（见 nonempty）， + #: 对不上就说明两个解析器看到的不是同一张表（XML 行号乱序、重复的行，openpyxl 会静默跳过） + row_runs: list[tuple[int, int, int]] = field(default_factory=list) @dataclass @@ -860,6 +864,11 @@ def _scan_sheet(archive: _Archive, sheet: SheetScan, sst_flags: bytearray | None alive = rows = 0 where = f"工作表「{sheet.name}」" + # 逐行非空格数的游程。行号单调递增（正常文件）时就地合并，O(1)；遇到行号回退或重复（构造的、 + # 或别的程序写乱的 XML）改用 dict 累加，扫完再压缩成游程 + runs = sheet.row_runs + by_row: dict[int, int] | None = None + last_filled_row = 0 # 上限先取到局部变量：每个元素都要比一次，模块属性每次现查要多花时间（测试里照样能改模块常量） max_alive, max_rows, max_cols = MAX_ALIVE, EXCEL_MAX_ROWS, EXCEL_MAX_COLS max_text, max_formula = EXCEL_MAX_TEXT, EXCEL_MAX_FORMULA @@ -904,6 +913,7 @@ def _scan_sheet(archive: _Archive, sheet: SheetScan, sst_flags: bytearray | None col = 0 #: 这一行清空时随之释放的元素个数（行本身的空壳留着，不算） freed = 0 + row_filled = 0 for cell in el: freed += 1 if tags.get(cell.tag) != "c": @@ -969,6 +979,7 @@ def _scan_sheet(archive: _Archive, sheet: SheetScan, sst_flags: bytearray | None if not filled: continue nonempty += 1 + row_filled += 1 if row < min_row: min_row = row if col < min_col: @@ -985,11 +996,98 @@ def _scan_sheet(archive: _Archive, sheet: SheetScan, sst_flags: bytearray | None max_row_cell = cell_ref(row, col) if jumped and len(far) < MAX_FAR: far.append(cell_ref(row, col)) + if row_filled: + if by_row is None and row > last_filled_row: + if runs and runs[-1][1] + 1 == row and runs[-1][2] == row_filled: + runs[-1] = (runs[-1][0], row, row_filled) + else: + runs.append((row, row, row_filled)) + last_filled_row = row + else: + if by_row is None: + by_row = {} + for lo, hi, n in runs: + for rr in range(lo, hi + 1): + by_row[rr] = n + by_row[row] = by_row.get(row, 0) + row_filled el.clear() alive -= freed + if by_row is not None: + runs.clear() + for rr in sorted(by_row): + n = by_row[rr] + if runs and runs[-1][1] + 1 == rr and runs[-1][2] == n: + runs[-1] = (runs[-1][0], rr, n) + else: + runs.append((rr, rr, n)) sheet.nonempty = nonempty if nonempty: sheet.bounds = Bounds(min_row, min_col, max_row, max_col) # 没有明显的跳跃（比如区域是慢慢变稀的），就拿撑出右边界、下边界的格子当示例 sheet.far_cells = far or list(dict.fromkeys(c for c in (max_col_cell, max_row_cell) if c)) + + +# -------------------------------------------------------------------------- +# 序列化：暂存区存扫描结果，试运行不必再扫一遍（配方导入，P2-SPEC 4.1） +# -------------------------------------------------------------------------- + + +def _jsonable(value: Any) -> Any: + """asdict 的结果里元组照样是元组：转成列表，返回值和 json 往返一次之后逐字相等。""" + if isinstance(value, dict): + return {k: _jsonable(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_jsonable(v) for v in value] + return value + + +def scan_to_json(scan: WorkbookScan) -> dict[str, Any]: + """WorkbookScan → 可以 json.dumps 的 dict（dataclasses.asdict 的形状，元组写成列表）。""" + return _jsonable(asdict(scan)) + + +def _pairs(items: Any, width: int) -> list[tuple[int, ...]]: + out = [] + for item in items or []: + seq = tuple(int(x) for x in item) + if len(seq) != width: + raise ValueError(f"扫描结果里的区间应有 {width} 项：{item!r}") + out.append(seq) + return out + + +def _cell_list(data: Any) -> CellList: + data = data or {} + return CellList(cells=list(data.get("cells") or []), total=int(data.get("total") or 0), + max_row=int(data.get("max_row") or 0), texts=list(data.get("texts") or [])) + + +def _known(cls: type, data: dict[str, Any]) -> dict[str, Any]: + """只取 cls 认识的键：老版本存下的 JSON 少了新字段时取默认值，多出的键（更新的版本写的）忽略。""" + names = {f.name for f in fields(cls)} + return {k: v for k, v in data.items() if k in names} + + +def scan_from_json(data: dict[str, Any]) -> WorkbookScan: + """scan_to_json 的逆操作。字段缺了取默认值（老暂存区里没有 row_runs），多了忽略。""" + sheets = [] + for raw in data.get("sheets") or []: + d = _known(SheetScan, dict(raw)) + bounds = d.get("bounds") + sheets.append(SheetScan( + name=str(d["name"]), state=str(d.get("state") or "visible"), path=str(d.get("path") or ""), + bounds=Bounds(**_known(Bounds, bounds)) if bounds else None, + nonempty=int(d.get("nonempty") or 0), + merged=list(d.get("merged") or []), merged_total=int(d.get("merged_total") or 0), + hidden_rows=_pairs(d.get("hidden_rows"), 2), hidden_cols=_pairs(d.get("hidden_cols"), 2), + autofilter=d.get("autofilter"), formulas=int(d.get("formulas") or 0), + formulas_uncached=_cell_list(d.get("formulas_uncached")), errors=_cell_list(d.get("errors")), + formulas_above=_cell_list(d.get("formulas_above")), far_cells=list(d.get("far_cells") or []), + dimension=d.get("dimension"), row_runs=_pairs(d.get("row_runs"), 3), + )) + tables = [TableScan(**_known(TableScan, dict(t))) for t in data.get("tables") or []] + for t in tables: + t.columns = list(t.columns) + return WorkbookScan(sheets=sheets, full_calc_on_load=bool(data.get("full_calc_on_load")), tables=tables, + defined_names=dict(data.get("defined_names") or {})) diff --git a/backend/app/db/base.py b/backend/app/db/base.py index e6b0d711..7251f7d0 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -174,6 +174,20 @@ _COLUMN_MIGRATIONS: list[tuple[str, str, str]] = [ ("data_sources", "current_snapshot_id", "VARCHAR(64)"), # 存量运行没有钉过版本，NULL 正好表示「升级前发起的」 ("runs", "data_versions", "JSON"), + # 按配方导入（期 2）。新表 table_recipes、import_stagings 由 create_all 建；这些是已有表的新列， + # 存量行一律 NULL：存量的源都是简单导入或手工源，存量的导入记录都不是按配方导入的 + ("data_sources", "current_recipe_id", "VARCHAR(32)"), + ("table_imports", "recipe_id", "VARCHAR(32)"), + ("table_imports", "period_start", "VARCHAR(10)"), + ("table_imports", "period_end", "VARCHAR(10)"), + ("table_imports", "context", "JSON"), + ("table_imports", "checks", "JSON"), + ("table_imports", "overrides", "JSON"), + ("table_imports", "waivers", "JSON"), + ("table_imports", "confirmations", "JSON"), + ("table_imports", "manifest_artifact", "VARCHAR(64)"), + ("table_imports", "signed_by", "VARCHAR(100)"), + ("table_imports", "staging_id", "VARCHAR(32)"), ] diff --git a/backend/app/db/models.py b/backend/app/db/models.py index d0be5c7b..ae1570bb 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -67,6 +67,9 @@ class DataSource(Base, TimestampMixin): # 上传源当前启用的快照（source_snapshots.id）。查询、探查都从快照解析， # database 只作显示；手工源恒为 None current_snapshot_id: Mapped[str | None] = mapped_column(String(64), default=None) + # 按配方导入的源当前启用的配方（table_recipes.id）。非空即「按配方导入」：重传走「上传新一期」， + # 旧的简单上传对它拒收。简单导入的上传源和手工源恒为 None + current_recipe_id: Mapped[str | None] = mapped_column(String(32), default=None) # -------------------------------------------------------------------------- @@ -124,6 +127,133 @@ class TableImport(Base): created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) activated_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + # 以下是按配方导入（期 2）才有的列，简单导入一律为空。状态仍只有 active / superseded / retired： + # 没发布的尝试记在 import_stagings 上，不进这张表——回收的保护集合、seq 的递增、导入列表都按 + # 这张表算，塞进未发布的尝试会同时扰动这三处 + recipe_id: Mapped[str | None] = mapped_column(String(32), default=None) + # 统计期（YYYY-MM-DD）。期 3 按期累积时按它排序 + period_start: Mapped[str | None] = mapped_column(String(10), default=None) + period_end: Mapped[str | None] = mapped_column(String(10), default=None) + # 统计期的来源与原文（PeriodOut） + context: Mapped[dict[str, Any] | None] = mapped_column(default=None) + # 核对摘要（id、状态、核对数、不一致数、无法核对数、原因）。含 C1、C2：它们取决于文件名， + # 而构建按内容复用、不含文件名，所以不能放在构建回执里 + checks: Mapped[list[Any] | None] = mapped_column(default=None) + # 数据质量类的接受、无法核对的接受（各带理由） + overrides: Mapped[list[Any] | None] = mapped_column(default=None) + waivers: Mapped[list[Any] | None] = mapped_column(default=None) + # 本次提交时勾过的确认项（id、label、at） + confirmations: Mapped[list[Any] | None] = mapped_column(default=None) + # 导入清单的工件 id。只作索引：证据链经快照 schema_cache 里的 import_manifests 承诺（内容寻址）， + # 不经过这个可改的列 + manifest_artifact: Mapped[str | None] = mapped_column(String(64), default=None) + # 署名（自报，未认证） + signed_by: Mapped[str | None] = mapped_column(String(100), default=None) + staging_id: Mapped[str | None] = mapped_column(String(32), default=None) + + +class TableRecipe(Base): + """按配方导入的源的一版配方。同一个源从 1 起编号，只增不改：改配方就是新的一版。 + + recipe 存去掉默认值的紧凑形式（recipe.canonical_recipe），recipe_sha256 由它算，进构建 id。 + 不挂外键：数据源删掉以后配方置 retired 留作审计，导入清单里引用的配方 id 仍查得到。 + """ + + __tablename__ = "table_recipes" + # 同源的 seq 不重复：发布在版本存储的锁里取「最大 seq + 1」，这条约束兜住锁外误用 + __table_args__ = (UniqueConstraint("source_id", "seq"),) + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + source_id: Mapped[str] = mapped_column(String(32), index=True) + seq: Mapped[int] = mapped_column(Integer, default=0) + recipe: Mapped[dict[str, Any]] = mapped_column(default=dict) + recipe_sha256: Mapped[str] = mapped_column(String(64), default="") + # 配方格式版本，与 recipe_types.RECIPE_FORMAT 一致（这里不 import 数据层，测试核对两边相等） + recipe_format: Mapped[str] = mapped_column(String(40), default="agentlab-recipe/2") + # rules：规则起草；ai：AI 起草；manual：手工；mixed：起草后又改过 + origin: Mapped[str] = mapped_column(String(20), default="manual") + # active：现行；superseded：被新的一版替换；retired：数据源已删除 + status: Mapped[str] = mapped_column(String(20), default="active", index=True) + # 确认时勾过的确认项（id、label、at） + confirmations: Mapped[list[Any]] = mapped_column(default=list) + # 署名（自报，未认证） + signed_by: Mapped[str | None] = mapped_column(String(100), default=None) + # {"calls": [AiUsage], "consents": [同意记录]}；不是 AI 起草的为空 + ai_usage: Mapped[dict[str, Any]] = mapped_column(default=dict) + staging_id: Mapped[str | None] = mapped_column(String(32), default=None) + created_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) + activated_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + + +class ImportStaging(Base, TimestampMixin): + """一次还没发布的配方导入：暂存、起草、试运行都在这里，提交成功时才影响数据源。 + + 「拒收、放弃、过期都不动当前快照」由此天然成立：在提交那个事务之前，这里的任何东西都不碰 + 数据源、导入记录和快照。新源在暂存时就分配 source_id，提交时用它建 DataSource。 + + **结束即瘦身**（table_versions.close_staging）：进入 committed / discarded / expired 的同一个事务里， + 网格预览（有数字格的值、隐藏行列）、试运行回执（有区域外文字全文）这些大字段一律清空、试运行库 + 删掉，只留 id、来源、文件哈希、状态、工作配方及其哈希、用量、同意记录、署名和时间。接口没有认证， + 这些内容不该在用完之后还留在库里。结束超过 90 天的行由回收删除。 + """ + + __tablename__ = "import_stagings" + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + source_id: Mapped[str] = mapped_column(String(32), index=True) + source_name: Mapped[str] = mapped_column(String(100), default="") + description: Mapped[str] = mapped_column(Text, default="") + # first：首次导入；reupload：上传新一期；redraft：修改配方（不换文件）；switch：从简单导入切换 + kind: Mapped[str] = mapped_column(String(20), default="first") + # 未结束：drafting / trialed / rejected；已结束：committed / discarded / expired + status: Mapped[str] = mapped_column(String(20), default="drafting", index=True) + # 原件按内容存档（raw_store）。未结束的暂存区保护它不被回收；清除原件时按它找到要一并放弃的暂存区 + raw_sha256: Mapped[str] = mapped_column(String(64), default="", index=True) + file_name: Mapped[str] = mapped_column(String(500), default="") + file_size: Mapped[int] = mapped_column(Integer, default=0) + # 以下几项结束即清空 + scan: Mapped[dict[str, Any] | None] = mapped_column(default=None) + grid_preview: Mapped[list[Any] | None] = mapped_column(default=None) + facts: Mapped[dict[str, Any] | None] = mapped_column(default=None) + # {"rules": Draft 或 null, "ai": AI 起草结果的摘要或 null} + drafts: Mapped[dict[str, Any] | None] = mapped_column(default=None) + # 工作配方 + recipe: Mapped[dict[str, Any] | None] = mapped_column(default=None) + recipe_origin: Mapped[str | None] = mapped_column(String(20), default=None) + recipe_sha256: Mapped[str | None] = mapped_column(String(64), default=None) + # 回答问题的起点配方。工作配方 = 它按问题顺序重放全部回答；PUT 配方、AI 草稿会换掉它并清空回答 + answers_base: Mapped[dict[str, Any] | None] = mapped_column(default=None) + # {question_id: {"value": 选项值, "reason": 理由或 null}} + answers: Mapped[dict[str, Any]] = mapped_column(default=dict) + cards: Mapped[list[Any] | None] = mapped_column(default=list) + questions: Mapped[list[Any] | None] = mapped_column(default=list) + # 最近一次静态校验 + recipe_problems: Mapped[list[Any]] = mapped_column(default=list) + # 最近一次干跑的问题、是否只在前若干行上干跑 + draft_problems: Mapped[list[Any] | None] = mapped_column(default=list) + draft_partial: Mapped[bool] = mapped_column(default=False) + # 起点配方（上传新一期、修改配方时是现行配方） + base_recipe_id: Mapped[str | None] = mapped_column(String(32), default=None) + # 试运行时源的当前快照；提交时在版本存储的锁里核对它没变 + base_snapshot_id: Mapped[str | None] = mapped_column(String(64), default=None) + # {"统计期": {"start", "end", "signed_by"}}：人工录入 + context_inputs: Mapped[dict[str, Any]] = mapped_column(default=dict) + # 最近一次试运行的回执（含 extraction、checks 的整份 JSON，提交时据此还原） + trial: Mapped[dict[str, Any] | None] = mapped_column(default=None) + # 试运行库的键和路径。库被发布消耗或删除后两者都清空 + trial_key: Mapped[str | None] = mapped_column(String(64), default=None) + trial_path: Mapped[str | None] = mapped_column(Text, default=None) + # 每次调用模型的用量；用户同意发送的记录 {signed_by, at, model, provider, compressed_sha256, chars} + ai_usage: Mapped[list[Any]] = mapped_column(default=list) + ai_consents: Mapped[list[Any]] = mapped_column(default=list) + # 署名（自报，未认证） + signed_by: Mapped[str | None] = mapped_column(String(100), default=None) + committed_import_id: Mapped[str | None] = mapped_column(String(32), default=None) + # 过期时刻（创建后 table_versions.STAGING_TTL）。为空时按 created_at 加 TTL 算 + expires_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + # 进入 committed / discarded / expired 的时刻 + closed_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + class SourceSnapshot(Base): """数据源的一个可查询版本：若干期导入合成的一个库，加上冻结的表结构。 diff --git a/backend/app/engine/evidence.py b/backend/app/engine/evidence.py index 64021a26..b2e7a7a2 100644 --- a/backend/app/engine/evidence.py +++ b/backend/app/engine/evidence.py @@ -3122,6 +3122,20 @@ def unshaped_table(entry: dict[str, Any], fetch: Callable[[Any], Any]) -> bool: #: 写作目录里未规整的表名后面跟的那句。不含数字（H11） _UNSHAPED_MARK = "（按原样导入、未经规整：同一列里混有不同口径的行，不能直接对列求和）" +#: 写作目录里「原表写明的合计」表（按配方导入另存的表内合计，表结构里 kind=reported_total）后面跟的那句（AU-5） +_REPORTED_TOTAL_MARK = "（原表写明的合计：不要彼此相加，也不要与明细相加）" + + +def reported_total_table(entry: dict[str, Any], fetch: Callable[[Any], Any]) -> bool: + """目录里的这张表是不是原表写明的合计（冻结的表结构里 kind 为 reported_total）。口径同 unshaped_table。""" + name = entry.get("name") or (entry.get("locator") or {}).get("table") + for origin in entry.get("sources") or []: + if not (isinstance(origin, dict) and origin.get("kind") == "schema" and origin.get("artifact")): + continue + info = schema_table(fetch(origin["artifact"]), name) + if info is not None: + return info.get("kind") == "reported_total" + return False def _entity_prompt(catalog: dict[str, Any], *, partial: bool = False, snaps: _Snapshots | None = None) -> list[str]: @@ -3134,7 +3148,11 @@ def _entity_prompt(catalog: dict[str, Any], *, partial: bool = False, snaps: _Sn return snaps._fetch(artifact, "表结构快照")[0] if snaps is not None else None def mark(entry: dict[str, Any]) -> str: - return _UNSHAPED_MARK if snaps is not None and unshaped_table(entry, fetch) else "" + if snaps is None: + return "" + if unshaped_table(entry, fetch): + return _UNSHAPED_MARK + return _REPORTED_TOTAL_MARK if reported_total_table(entry, fetch) else "" tables = [e for e in catalog.values() if e.get("kind") == "table"] fields: dict[str, list[str]] = {} diff --git a/backend/app/engine/judge.py b/backend/app/engine/judge.py index ae4d9ccc..ec1f5639 100644 --- a/backend/app/engine/judge.py +++ b/backend/app/engine/judge.py @@ -499,6 +499,9 @@ def _table_lines(entry: dict[str, Any], catalog: dict[str, Any], fetch: _Fetch, if info.get("comment") == UNSHAPED_NOTE: # 上传时选了「按原样导入（未规整）」：对这张表的列直接求和得出的数，裁判要知道它不可靠（R4、D16） lines.append(f"表的说明：{UNSHAPED_NOTE}") + elif info.get("kind") == "reported_total": + # 按配方导入另存的「原表写明的合计」：各合计项可能重叠，相加得出的数不可靠（AU-5） + lines.append("表的说明：原表写明的合计，各合计项可能互相重叠，不要彼此相加，也不要与明细相加") if truncated: lines.append("注意：表结构快照不完整（数据源的表太多，只保存了一部分），不能据此断定某张表或某个字段不存在") return lines diff --git a/backend/app/main.py b/backend/app/main.py index 09b046f2..ed470c5e 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -10,7 +10,7 @@ from fastapi.responses import JSONResponse from app.api import ( artifacts, conversations, copilot, datasources, evidence, governance, knowledge, runs, - sandbox, settings as settings_api, tools, workflows, + sandbox, settings as settings_api, table_imports, tools, workflows, ) from app.api.coded import CodedHTTPException, coded_handler from app.core.config import settings @@ -117,6 +117,8 @@ for router in ( conversations.router, artifacts.router, governance.router, + # 按配方导入的 /api/datasources/imports/… 要在 datasources 的通配 /{source_id} 之前登记 + table_imports.router, datasources.router, ): app.include_router(router) diff --git a/backend/app/tools/datasource.py b/backend/app/tools/datasource.py index 34110cce..1d78bb0c 100644 --- a/backend/app/tools/datasource.py +++ b/backend/app/tools/datasource.py @@ -134,6 +134,25 @@ def unshaped_tables(source: Any) -> list[str]: if isinstance(meta, dict) and meta.get("comment") == UNSHAPED_NOTE] +def reported_total_tables(source: Any) -> list[str]: + """按配方导入时另存的「原表写明的合计」表（表结构里 kind=reported_total，recipe_notes.apply_notes 记的）。""" + tables = (getattr(source, "schema_cache", None) or {}).get("tables") or {} + return [meta.get("qualified", name) for name, meta in tables.items() + if isinstance(meta, dict) and meta.get("kind") == "reported_total"] + + +#: 原表写明的合计表在工具描述、db_schema 表清单里的标记。不含数字（H11） +REPORTED_TOTAL_MARK = "（原表写明的合计，不要彼此相加，也不要与明细相加）" + +#: 按配方导入的源在工具描述「可用表」之后多加的两句（P2-SPEC 5.4，不含数字） +RECIPE_QUERY_HINT = " 中文表名和列名请加双引号。查单个值时，把主键列一起选出来，证据面板才能追到原表格子。" + +#: 冻结 schema_snapshot 时从 schema_cache 复制的顶层键。import_mode、import_manifests 只有按配方导入的源才有： +#: 冻结进来以后，schema_snapshot 的内容哈希就承诺了导入清单（证据链按哈希从封存事件走到清单，中间不经过 +#: 可改的数据库列，H5）；手工源和期 1 的上传源没有这两个键，冻结内容和原来一字不差 +_FROZEN_SCHEMA_KEYS = ("schema", "synced_at", "truncated", "total", "import_mode", "import_manifests") + + def _query_description(source: Any) -> str: tables = introspect.table_names(source) head = f"在数据源「{source.name}」上执行 SQL 查询。" @@ -144,11 +163,18 @@ def _query_description(source: Any) -> str: listed = "、".join(tables[:25]) more = f" 等 {len(tables)} 张表" if len(tables) > 25 else "" head += f" 可用表：{listed}{more}。" + # 按配方导入的源：表名列名是中文（不加引号在多数方言里报错），主键是证据面板追溯原表格子的钥匙 + if (getattr(source, "schema_cache", None) or {}).get("import_mode") == "recipe": + head += RECIPE_QUERY_HINT # 未规整的说明以前只在 db_schema 查单表时看得到：只绑了查询工具的 Agent、调用工具节点接报告的 # 链路都看不到，照样对列求和。工具描述里点名（不含数字，H11） if raw := unshaped_tables(source): shown = "、".join(raw[:10]) + (f" 等 {len(raw)} 张" if len(raw) > 10 else "") head += f" 其中 {shown} 按原样导入、未经规整：同一列里混有不同口径的行，不能直接对列求和。" + # 原表写明的合计表同理：「不要相加」的说明只在 db_schema 查单表时看得到（AU-5） + if totals := reported_total_tables(source): + shown = "、".join(totals[:10]) + (f" 等 {len(totals)} 张" if len(totals) > 10 else "") + head += f" 其中 {shown} 是原表写明的合计：各合计项可能互相重叠，不要彼此相加，也不要与明细表相加。" head += " 字段不确定时先查结构，不要猜字段名。" else: head += _no_schema_note(source) @@ -473,7 +499,7 @@ async def _store_schema(source: Any, ctx: ToolContext) -> str | None: from app.engine.evidence import SCHEMA_SNAPSHOT content = {"source": source.name, "tables": tables, - **{k: cache[k] for k in ("schema", "synced_at", "truncated", "total") if k in cache}} + **{k: cache[k] for k in _FROZEN_SCHEMA_KEYS if k in cache}} try: return await put_json(content, kind=SCHEMA_SNAPSHOT, run_id=ctx.run_id or "", node_id=ctx.node_id or "", meta={"source": source.name, "tables": len(tables)}) @@ -497,8 +523,10 @@ def _make_schema_tool(source: Any, *, fixed: bool = False) -> StructuredTool: # 把实情交出去，而不是让它去点一个它点不到的按钮 return f"数据源「{source.name}」的结构信息不可用。" + _no_schema_note(source) raw = set(unshaped_tables(source)) + totals = set(reported_total_tables(source)) return f"数据源「{source.name}」共 {len(tables)} 张表：\n" + "\n".join( - f" {n}" + ("（按原样导入、未经规整，不能直接对列求和）" if n in raw else "") for n in tables + f" {n}" + ("（按原样导入、未经规整，不能直接对列求和）" if n in raw else "") + + (REPORTED_TOTAL_MARK if n in totals else "") for n in tables ) return StructuredTool( diff --git a/backend/tests/fixtures/recipes/flow_recipe.json b/backend/tests/fixtures/recipes/flow_recipe.json new file mode 100644 index 00000000..53ad2d81 --- /dev/null +++ b/backend/tests/fixtures/recipes/flow_recipe.json @@ -0,0 +1,85 @@ +{ + "recipe_format": "agentlab-recipe/2", + "mode": "replace", + "other_visible_sheets": "confirm", + "sheets": [ + { + "id": "s1", + "match": {"name": "客流汇总", "fallback": "only_visible_sheet"}, + "hidden": {"rows": "reject_if_any", "cols": "reject_if_any"}, + "context": [ + {"id": "统计期", "kind": "period", "parser": "cn_date_range", "prefer_prefix": "统计时间范围", "cross_check": "filename"} + ], + "blocks": [ + { + "id": "交叉表", + "layout": "crosstab", + "axis": { + "find": {"parser": "month_day_or_date", "min": 2}, + "name": "日期", + "type": "DATE", + "year_from": "统计期", + "checks": ["contiguous", "covers_context"] + }, + "label_offset": -1, + "values": { + "type": "INTEGER", + "placeholders": [{"text": "·", "meaning": "无数据"}], + "blank": "reject", + "text_number": "reject", + "formula": "reject" + }, + "segments": [ + { + "id": "日客流", + "role": "measures", + "table": "日客流", + "locate": {"by": "labels"}, + "labels": {"expect": ["全日客流（人次）", "分区甲（人次）", "分区乙（人次）"]}, + "measures": {"全日客流（人次）": "全日客流", "分区甲（人次）": "分区甲", "分区乙（人次）": "分区乙"} + }, + { + "id": "日间", + "role": "dimension", + "table": "时段客流", + "locate": {"by": "section_title", "title": "日间时段客流（人次）"}, + "labels": {"expect": ["7-8", "8-9", "9-10", "10-11", "11-12", "12-13", "13-14", "14-15", "15-16", "16-17", "17-18"]}, + "dim": {"name": "时段", "parser": "hour_range", "derive": {"起始小时": "start", "结束小时": "end"}}, + "value": "客流", + "const": {"时段类别": {"pick": "日间"}} + }, + { + "id": "夜间", + "role": "dimension", + "table": "时段客流", + "locate": {"by": "section_title", "title": "夜间时段客流（人次）"}, + "labels": {"expect": ["18-19", "19-20", "20-21", "21-22", "22-23", "23-24"]}, + "dim": {"name": "时段", "parser": "hour_range", "derive": {"起始小时": "start", "结束小时": "end"}}, + "value": "客流", + "const": {"时段类别": {"pick": "夜间"}}, + "stop_parser": "hour_range_total" + }, + { + "id": "夜间合计", + "role": "derived", + "locate": {"by": "after", "segment": "夜间"}, + "labels_parser": "hour_range_total", + "labels": {"expect": ["18-22 时合计", "22-24 时合计", "18-24 时合计"]}, + "verify": {"kind": "label_range_sum", "against_table": "时段客流", "value": "客流"}, + "keep_as": {"table": "时段客流_表内合计", "dim": "合计项", "derive": {"起始小时": "start", "结束小时": "end"}, "value": "客流"} + } + ] + } + ] + } + ], + "tables": [ + {"name": "日客流", "grain": ["日期"], "kind": "data", "units": {"全日客流": "人次", "分区甲": "人次", "分区乙": "人次"}, "note": ""}, + {"name": "时段客流", "grain": ["日期", "时段"], "kind": "data", "units": {"客流": "人次"}, "note": ""}, + {"name": "时段客流_表内合计", "grain": ["日期", "合计项"], "kind": "reported_total", "units": {"客流": "人次"}, "note": ""} + ], + "relations": [ + {"id": "R1", "kind": "sum_eq", "table": "日客流", "total": "全日客流", "parts": ["分区甲", "分区乙"], "claims": "F1"}, + {"id": "R2", "kind": "not_comparable", "a": {"table": "时段客流", "value": "客流"}, "b": {"table": "日客流", "value": "全日客流"}, "by": "日期", "claims": "F2"} + ] +} diff --git a/backend/tests/fixtures/xlsx/__init__.py b/backend/tests/fixtures/xlsx/__init__.py new file mode 100644 index 00000000..223071d1 --- /dev/null +++ b/backend/tests/fixtures/xlsx/__init__.py @@ -0,0 +1,106 @@ +"""合成 xlsx 夹具（期 2，WP-1）：全部在测试里用 openpyxl 现造，标签一律假名，数字随机或手写假数。 + +- flow.py：「结构仿照客流表」的交叉表（P2-SPEC 9.1），带 variant 参数造 9.2 的漂移用例； +- drift.py：9.2 的 31 例（30 例 + D28b）及各例的期望； +- lab.py：9.3 的实验室难缠用例（结构照 xlsx-lab/case*.py 重写，名字全换成假名）； +- mutate.py：9.4 的数据变异（在 XML 层改 D00 的字节，保留缓存值）。 + +**用户的真实 Excel 文件不得以任何形式进入这里。** openpyxl 写公式不带缓存值、默认写 fullCalcOnLoad， +「在 Excel 里打开并保存过」的文件由这里的小工具在压缩包层面补上。这些函数不写磁盘，只返回字节。 +""" +from __future__ import annotations + +import io +import re +import zipfile +from typing import Callable +from xml.etree import ElementTree as ET + +_NS_MAIN = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" +_NS_REL = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" +_NS_PKG = "http://schemas.openxmlformats.org/package/2006/relationships" + + +#: 压缩包条目和文档属性里的时间一律写成这个：同样的参数造出同样的字节（原件哈希、构建 id 才可复现） +_FIXED_TIME = (2026, 1, 1, 0, 0, 0) +_STAMP = re.compile(r"(]*>)[^<]*()") + + +def save(book) -> bytes: + """openpyxl 工作簿 → 字节（不落盘）。openpyxl 存盘时把「修改时间」写成当前时刻，这里改成固定值。""" + buf = io.BytesIO() + book.save(buf) + src = zipfile.ZipFile(io.BytesIO(buf.getvalue())) + out = io.BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as dst: + for info in src.infolist(): + data = src.read(info.filename) + if info.filename == "docProps/core.xml": + data = _STAMP.sub(r"\g<1>2026-01-01T00:00:00Z\3", data.decode("utf-8")).encode("utf-8") + entry = zipfile.ZipInfo(info.filename, date_time=_FIXED_TIME) + entry.compress_type = zipfile.ZIP_DEFLATED + entry.external_attr = info.external_attr + dst.writestr(entry, data) + return out.getvalue() + + +def patch_parts(raw: bytes, edits: dict[str, Callable[[str], str]]) -> bytes: + """改压缩包里的部件：{部件路径: 文本 → 文本}。其余部件原样拷贝（含压缩参数）。""" + src = zipfile.ZipFile(io.BytesIO(raw)) + missing = set(edits) - set(src.namelist()) + if missing: + raise KeyError(f"压缩包里没有这些部件：{sorted(missing)}") + out = io.BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as dst: + for info in src.infolist(): + data = src.read(info.filename) + if info.filename in edits: + data = edits[info.filename](data.decode("utf-8")).encode("utf-8") + dst.writestr(info, data) + return out.getvalue() + + +def read_part(raw: bytes, name: str) -> str: + return zipfile.ZipFile(io.BytesIO(raw)).read(name).decode("utf-8") + + +def sheet_part(raw: bytes, sheet: str) -> str: + """工作表名 → 它在压缩包里的 XML 路径（按 workbook.xml 和关系文件配对，不靠 sheetN 的序号猜）。""" + z = zipfile.ZipFile(io.BytesIO(raw)) + book = ET.fromstring(z.read("xl/workbook.xml")) + rels = ET.fromstring(z.read("xl/_rels/workbook.xml.rels")) + targets = {r.get("Id"): r.get("Target") for r in rels.iter(f"{{{_NS_PKG}}}Relationship")} + for el in book.iter(f"{{{_NS_MAIN}}}sheet"): + if el.get("name") == sheet: + target = targets[el.get(f"{{{_NS_REL}}}id")] + return target.lstrip("/") if target.startswith("/") else f"xl/{target}" + raise KeyError(f"没有工作表「{sheet}」") + + +def _cell_re(ref: str) -> re.Pattern[str]: + # openpyxl 写出的格：SUM(C22:C25)（可能带 s=、t= 属性） + return re.compile(rf'[^>]*)>(?P[^<]*)(?P|[^<]*)?') + + +def fill_cached(xml: str, values: dict[str, object]) -> str: + """给公式格补上 缓存值（模拟 Excel 保存过）。每个坐标必须恰好命中一个公式格。""" + for ref, v in values.items(): + text = repr(v) if isinstance(v, float) else str(v) + xml, n = _cell_re(ref).subn(lambda m: f'{m.group("f")}{text}', xml) + if n != 1: + raise AssertionError(f"公式格 {ref} 命中 {n} 次") + return xml + + +def excel_saved(raw: bytes, sheet: str, values: dict[str, object], *, full_calc: bool = False) -> bytes: + """补缓存值，并按 full_calc 决定是否保留 workbook.xml 的 fullCalcOnLoad（openpyxl 默认写它）。""" + edits: dict[str, Callable[[str], str]] = {} + if values: + edits[sheet_part(raw, sheet)] = lambda t: fill_cached(t, values) + if not full_calc: + edits["xl/workbook.xml"] = lambda t: t.replace(' fullCalcOnLoad="1"', "") + return patch_parts(raw, edits) if edits else raw + + +def has_full_calc(raw: bytes) -> bool: + return 'fullCalcOnLoad="1"' in read_part(raw, "xl/workbook.xml") diff --git a/backend/tests/fixtures/xlsx/drift.py b/backend/tests/fixtures/xlsx/drift.py new file mode 100644 index 00000000..34575d55 --- /dev/null +++ b/backend/tests/fixtures/xlsx/drift.py @@ -0,0 +1,135 @@ +"""9.2 的 31 例漂移夹具（30 例 + D28b）及各例的期望。 + +基线是 D00 首次导入并启用（8 月 31 天）；其余各例作为「上传新一期」按参考配方重放，与 D00 比差异。 +配方没变，所以 7.5 的「配方类」确认项都不出现，confirms 只列「本期类」和 diff:*（D00 首次导入的确认项见 +flow.FLOW_EXPECT["confirms"]）。差异卡的基础集合是 {period, rows}：文件名只变日期不出 file_name，占位符 +原文集合不变不出 placeholders，核对的 (id, 状态) 集合不变不出 checks。diff_kinds 是**精确集合**。 +""" +from __future__ import annotations + +import datetime as dt +from dataclasses import dataclass +from typing import Callable + +from .flow import flow_workbook + +AUG = dt.date(2026, 8, 1) +SEP = dt.date(2026, 9, 1) +OCT = dt.date(2026, 10, 1) + +#: 三张表的行数：日客流 / 时段客流 / 时段客流_表内合计 +ROWS_31 = (31, 527, 93) +ROWS_30 = (30, 510, 90) + + +@dataclass +class DriftCase: + code: str + title: str + #: seed → (字节, 文件名) + build: Callable[[int], tuple[bytes, str]] + #: passed / confirm / rejected / decision + expect: str + #: 三张表的行数（拒收为 None） + rows: tuple[int, int, int] | None + #: 期望出现的问题 code 或核对 id（rejected / decision） + codes: tuple[str, ...] + #: 期望出现的「本期类」「diff」确认项 id（精确 id；带 * 的按前缀） + confirms: tuple[str, ...] + #: 差异卡里出现的 kind 的精确集合（拒收为 None） + diff_kinds: frozenset[str] | None + #: 说明里必须出现的片段 + note_has: tuple[str, ...] = () + #: 说明里不得出现的片段 + note_lacks: tuple[str, ...] = () + + +def _build(start: dt.date, days: int, variant: str) -> Callable[[int], tuple[bytes, str]]: + def build(seed: int) -> tuple[bytes, str]: + return flow_workbook(start, days, seed=seed, variant=variant) + return build + + +_BASE = frozenset({"period", "rows"}) +_NOTE_E7 = ("本期原表附有说明文字",) + + +def _case(code: str, title: str, expect: str, *, start: dt.date = SEP, days: int = 30, + rows: tuple[int, int, int] | None = ROWS_30, codes: tuple[str, ...] = (), + confirms: tuple[str, ...] = (), diff: frozenset[str] | set[str] | None = _BASE, + note_has: tuple[str, ...] = (), note_lacks: tuple[str, ...] = ()) -> DriftCase: + if expect == "rejected": + rows, diff = None, None + return DriftCase(code=code, title=title, build=_build(start, days, code), expect=expect, rows=rows, + codes=codes, confirms=confirms, diff_kinds=None if diff is None else frozenset(diff), + note_has=note_has, note_lacks=note_lacks) + + +_CASES = [ + # ---- 通过（12） ---- + _case("D00", "基线：8 月 31 天（同一文件再传是 same_as_import，提交 unchanged）", "passed", start=AUG, days=31, + rows=ROWS_31, diff=frozenset(), note_has=("已逐日核对",), note_lacks=_NOTE_E7), + _case("D01", "9 月 30 天", "passed"), + _case("D02", "10 月 31 天（行数与 D00 相同）", "passed", start=OCT, days=31, rows=ROWS_31, diff={"period"}), + _case("D03", "10 月 1–15 日", "passed", start=OCT, days=15, rows=(15, 255, 45)), + _case("D03b", "10 月 1–7 日", "passed", start=OCT, days=7, rows=(7, 119, 21)), + _case("D12", "B2 第二个日期省略年份", "passed", diff=_BASE | {"context_text"}), + _case("D14", "去掉第 8 行空行", "passed"), + _case("D15", "分段标题不合并", "passed"), + _case("D17", "B3 改成「客流汇总表（2026年9月）」：进 annotated，免 outside_digits", "passed", + diff=_BASE | {"context_source_added"}, note_has=_NOTE_E7), + _case("D23", "文件名改成「9月客流.xlsx」：C2 passed → info", "passed", diff=_BASE | {"file_name", "checks"}), + _case("D25", "保留 fullCalcOnLoad、缓存正确：K1 passed 带「保存值与重算一致」", "passed", + diff=_BASE | {"full_calc"}), + _case("D27", "B3 写成「客流汇总表（2026年9月1日至2026年9月30日）」：C1 两处一致、进 annotated", "passed", + diff=_BASE | {"context_source_added"}, note_has=_NOTE_E7), + # ---- 需要确认（11） ---- + _case("D05", "夜间块（标题、时段、合计）移到日间块之前", "confirm", diff=_BASE | {"block_order"}, + confirms=("diff:block_order:客流汇总",)), + _case("D06", "日客流三行移到最后，前面加标题「日客流（人次）」", "confirm", + diff=_BASE | {"block_order", "outside_added"}, + confirms=("diff:block_order:客流汇总", "diff:outside:客流汇总!B*")), + _case("D07", "合计行改成写死的数（值正确）：K1 passed，没有 G1", "confirm", + diff=_BASE | {"checks", "derived_form"}, confirms=("diff:derived_form:夜间合计",)), + _case("D10", "表头改成日期格（显示格式 m\"月\"d\"日\"）", "confirm", diff=_BASE | {"axis_form"}, + confirms=("diff:axis_form:交叉表",)), + _case("D11", "时段写成 07:00-08:00（时段仍存 7-8，没有 diff:canon:*）", "confirm", + diff=_BASE | {"label_writing"}, confirms=("diff:label_writing:日间", "diff:label_writing:夜间")), + _case("D19", "日客流三行顺序改成 分区甲、分区乙、全日客流", "confirm", diff=_BASE | {"row_order"}, + confirms=("diff:row_order:日客流",)), + _case("D22", "合计标签写成 18:00-22:00合计（合计项仍是 18-22时合计）", "confirm", + diff=_BASE | {"label_writing"}, confirms=("diff:label_writing:夜间合计",)), + _case("D24", "时段用 en-dash", "confirm", diff=_BASE | {"label_writing"}, + confirms=("diff:label_writing:日间", "diff:label_writing:夜间")), + _case("D20", "多一个可见工作表「说明」", "confirm", confirms=("sheet_extra:说明",)), + _case("D28", "表下新增「注：9月15日闸机故障，当日客流为估算值」", "confirm", diff=_BASE | {"outside_added"}, + confirms=("outside_digits:客流汇总!B32", "diff:outside:客流汇总!B32"), note_has=_NOTE_E7), + _case("D28b", "表下新增「注：2026年9月数据为初步统计，待修订」：进 annotated、C1 两处一致，仍要确认", "confirm", + diff=_BASE | {"outside_added"}, confirms=("outside_digits:客流汇总!B32", "diff:outside:客流汇总!B32"), + note_has=_NOTE_E7), + # ---- 拒收：结构类（6） ---- + _case("D04", "日间少了 7-8 行", "rejected", codes=("label_missing",)), + _case("D09", "日间标题改成「日间分时段客流（人次）」", "rejected", codes=("title_not_found",)), + _case("D13", "日客流加「分区丙（人次）」，全日 = 甲 + 乙 + 丙", "rejected", codes=("label_unexpected",)), + _case("D16", "右侧多一列「合计」（每行 SUM 公式，有缓存）", "rejected", codes=("axis_extra_cells",)), + _case("D18", "日间末尾多一行公式小计「7-18 时合计」", "rejected", codes=("label_unparsed",)), + _case("D21", "表下空一行补录「补录（人次）」「1,234」「2,345」（B32:D32）", "rejected", codes=("row_unclaimed",)), + # ---- 拒收：合计核对（1）、需要决定（1） ---- + _case("D08", "写死的合计，「18-22 时合计」第 5 个日期 +100：K1 mismatch，定位 G28", "rejected", codes=("K1",)), + # D26 的说明片段是「写理由接受 R1 之后」重新生成的说明 + _case("D26", "第 10 个日期全日 = 甲 + 乙 + 7：R1 mismatch，写理由接受后可提交", "decision", + codes=("R1",), diff=_BASE | {"checks"}, note_has=("已由用户确认接受",), note_lacks=("已逐日核对",)), +] + +DRIFT_CASES: dict[str, DriftCase] = {c.code: c for c in _CASES} + +#: final.md 2.8 的分组（12 / 8 / 2 / 6 / 2；D28b 归「需要确认才能启用」，那一组变成 3 例）。 +#: 与 expect 的对应：通过 → passed；版式变化需确认、需要确认才能启用 → confirm；拒收可自助修复、D08 → rejected； +#: D26 → decision（数据质量类，写理由接受后可提交） +GROUPS = { + "通过": ("D00", "D01", "D02", "D03", "D03b", "D12", "D14", "D15", "D17", "D23", "D25", "D27"), + "版式变化需确认": ("D05", "D06", "D07", "D10", "D11", "D19", "D22", "D24"), + "需要确认才能启用": ("D20", "D28", "D28b"), + "拒收可自助修复": ("D04", "D09", "D13", "D16", "D18", "D21"), + "正确拒收": ("D08", "D26"), +} diff --git a/backend/tests/fixtures/xlsx/flow.py b/backend/tests/fixtures/xlsx/flow.py new file mode 100644 index 00000000..2d1999c6 --- /dev/null +++ b/backend/tests/fixtures/xlsx/flow.py @@ -0,0 +1,272 @@ +"""「结构仿照客流表」的合成交叉表（P2-SPEC 9.1）。名字全是假名（分区甲 / 分区乙 / 分区丙），数字随机。 + +坐标与真实报表一致（B2 统计期、第 4 行日期、第 5–7 行指标、第 8 行空、第 9 / 21 行分段标题、第 28–30 行 +公式合计），这样执行器、核对和说明对着它跑出来的格子账、行数、核对结果才有可比的期望（FLOW_EXPECT）。 +variant 取 9.2 的用例代号，在同一套生成逻辑上改出各种漂移：行的增删挪动都在「行清单」上做，行号和公式 +引用跟着重新算，不在 XML 里硬改坐标。 +""" +from __future__ import annotations + +import datetime as dt +import random +from dataclasses import dataclass +from typing import Any + +from openpyxl import Workbook +from openpyxl.utils import get_column_letter + +from . import excel_saved, save + +SHEET = "客流汇总" +TITLE = "客流汇总表" +PLACEHOLDER = "·" +TOTAL_LABEL, ZONE_A_LABEL, ZONE_B_LABEL = "全日客流（人次）", "分区甲（人次）", "分区乙（人次）" +ZONE_C_LABEL = "分区丙（人次）" +DAY_TITLE = "日间时段客流（人次）" +NIGHT_TITLE = "夜间时段客流（人次）" +DAY_HOURS = tuple(range(7, 18)) +NIGHT_HOURS = tuple(range(18, 24)) +#: 表内合计：(标签, 起始小时, 结束小时) +TOTALS = (("18-22 时合计", 18, 22), ("22-24 时合计", 22, 24), ("18-24 时合计", 18, 24)) +LABEL_COL, FIRST_COL = 2, 3 # B 列是标签，C 列起是日期 +EXTRA_SHEET = "说明" + +#: 只换起止日期、版式与 D00 相同的代号 +_PLAIN = {"D00", "D01", "D02", "D03", "D03b"} +#: 9.2 的全部代号，外加 noperiod（把 B2 删掉，9.7 第 6 步的人工录入统计期用） +VARIANTS = frozenset(_PLAIN | { + "D04", "D05", "D06", "D07", "D08", "D09", "D10", "D11", "D12", "D13", "D14", "D15", "D16", "D17", "D18", + "D19", "D20", "D21", "D22", "D23", "D24", "D25", "D26", "D27", "D28", "D28b", "noperiod", +}) + +#: D00（2026-08-01 起 31 天、参考配方）的期望，照 P2-SPEC 9.1 抄录，WP-7 验收时逐项比 +FLOW_EXPECT: dict[str, Any] = { + "nonempty": 771, + "bounds": "B2:AG30", + "merged": 5, + "formulas": 93, + #: 格子账去向（契约 Role）：数据值、合计格、合计标签、列表头（日期）、行标签、分段标题、统计期、区域外文字 + "ledger": {"value": 620, "derived_value": 93, "derived_label": 3, "col_header": 31, "row_label": 20, + "section_title": 2, "context": 1, "outside_text": 1}, + "rows": {"日客流": 31, "时段客流": 527, "时段客流_表内合计": 93}, + "placeholders": {PLACEHOLDER: 31}, + "outside_text": [("客流汇总!B3", TITLE, "text")], + "period_cells": ["客流汇总!B2"], + "checks": {"C1": "passed", "C2": "passed", "K1": "passed", "G1": "passed", "R1": "passed", "R2": "info", + "N1": "passed", "N2": "passed", "N3": "passed", "P1": "passed", "P2": "passed", "P3": "passed"}, + #: R2：31 天中 0 天相等 + "r2_equal_days": 0, + "canonicalized": 0, + "confirms": frozenset({ + "placeholder:·", "derived:夜间合计", "year_from:交叉表", "relation:R1", "relation:R2", + "unit:日客流.全日客流", "unit:日客流.分区甲", "unit:日客流.分区乙", "unit:时段客流.客流", + "unit:时段客流_表内合计.客流", "mode", + }), + "note_has": ("已逐日核对",), + "note_lacks": ("本期原表附有说明文字",), +} + + +@dataclass +class _Row: + #: empty / period / text / axis / measure / blank / section / hour / total / subtotal / note / raw + kind: str + label: Any = None + values: list[Any] | None = None + merged: bool = False + #: hour：小时；total / subtotal：(起始小时, 结束小时) + key: Any = None + r: int = 0 + + +def flow_filename(start: dt.date, days: int, variant: str = "D00") -> str: + end = start + dt.timedelta(days=days - 1) + if variant == "D23": + return f"{start.month}月客流.xlsx" + return f"月报导出_{start.isoformat()}_{end.isoformat()}.xlsx" + + +def _hour_label(h: int, variant: str) -> str: + if variant == "D11": + return f"{h:02d}:00-{h + 1:02d}:00" + if variant == "D24": + return f"{h}–{h + 1}" # en dash + return f"{h}-{h + 1}" + + +def _numbers(days: int, seed: int, variant: str): + rnd = random.Random(seed) + zone_a = [rnd.randint(3000, 9000) for _ in range(days)] + zone_b = [rnd.randint(2000, 8000) for _ in range(days)] + hours: dict[int, list[Any]] = {} + for h in DAY_HOURS + NIGHT_HOURS: + # 第 10 行（7-8）整行是占位符：真实报表早上这一档没有数 + hours[h] = [PLACEHOLDER] * days if h == 7 else [rnd.randint(50, 900) for _ in range(days)] + # 变体专用的数另起一个随机源，基线的数不跟着变 + extra = random.Random(seed * 7919 + 13) + zone_c = [extra.randint(1000, 5000) for _ in range(days)] if variant == "D13" else None + total = [zone_a[i] + zone_b[i] + (zone_c[i] if zone_c else 0) for i in range(days)] + if variant == "D26" and days >= 10: + total[9] += 7 # 第 10 个日期：全日 = 甲 + 乙 + 7 + # 保证每个日期「各时段之和 ≠ 全日客流」（R2 的口径不同在每一天都看得出来） + for i in range(days): + s = sum(v[i] for h, v in hours.items() if h != 7) + if s == total[i]: + hours[8][i] += 1 if hours[8][i] < 900 else -1 + return zone_a, zone_b, zone_c, total, hours + + +def _period_text(start: dt.date, end: dt.date, variant: str) -> str: + if variant == "D12": # 第二个日期省略年份 + return f"统计时间范围：{start.year}年{start.month}月{start.day}日至{end.month}月{end.day}日" + return f"统计时间范围：{start.year}年{start.month}月{start.day}日至{end.year}年{end.month}月{end.day}日" + + +def _title_text(start: dt.date, end: dt.date, variant: str) -> str: + if variant == "D17": + return f"{TITLE}（{start.year}年{start.month}月）" + if variant == "D27": + return f"{TITLE}（{start.year}年{start.month}月{start.day}日至{end.year}年{end.month}月{end.day}日）" + return TITLE + + +def _layout(start: dt.date, end: dt.date, days: int, seed: int, variant: str) -> list[_Row]: + zone_a, zone_b, zone_c, total, hours = _numbers(days, seed, variant) + rows: list[_Row] = [] + if variant == "noperiod": + rows.append(_Row("empty")) + else: + rows.append(_Row("period", _period_text(start, end, variant), merged=True)) + rows.append(_Row("text", _title_text(start, end, variant), merged=True)) + rows.append(_Row("axis")) + + measures = [_Row("measure", TOTAL_LABEL, total), _Row("measure", ZONE_A_LABEL, zone_a), + _Row("measure", ZONE_B_LABEL, zone_b)] + if zone_c is not None: # D13：多一个分区，全日 = 甲 + 乙 + 丙 + measures.append(_Row("measure", ZONE_C_LABEL, zone_c)) + if variant == "D19": # 顺序改成 甲、乙、全日 + measures = [measures[1], measures[2], measures[0]] + blank = [] if variant == "D14" else [_Row("blank", merged=True)] + + titled = variant != "D15" # D15：分段标题不合并 + day = [_Row("section", "日间分时段客流（人次）" if variant == "D09" else DAY_TITLE, merged=titled)] + for h in DAY_HOURS: + if variant == "D04" and h == 7: # D04：日间少了 7-8 + continue + day.append(_Row("hour", _hour_label(h, variant), hours[h], key=h)) + if variant == "D18": # 日间末尾多一行公式小计 + day.append(_Row("subtotal", "7-18 时合计", key=(7, 18))) + night = [_Row("section", NIGHT_TITLE, merged=titled)] + night += [_Row("hour", _hour_label(h, variant), hours[h], key=h) for h in NIGHT_HOURS] + for label, s, e in TOTALS: + night.append(_Row("total", f"{s}:00-{e}:00合计" if variant == "D22" else label, key=(s, e))) + + if variant == "D05": # 夜间块（标题、时段、合计）移到日间块之前 + rows += measures + blank + night + day + elif variant == "D06": # 日客流三行移到最后，前面加标题 + rows += day + night + [_Row("empty"), _Row("text", "日客流（人次）")] + measures + else: + rows += measures + blank + day + night + + if variant == "D21": # 表下空一行补录一行数据 + rows += [_Row("empty"), _Row("raw", values=["补录（人次）", "1,234", "2,345"])] + elif variant == "D28": + rows += [_Row("empty"), _Row("note", f"注：{start.month}月15日闸机故障，当日客流为估算值")] + elif variant == "D28b": + rows += [_Row("empty"), _Row("note", f"注：{start.year}年{start.month}月数据为初步统计，待修订")] + for i, row in enumerate(rows): + row.r = 2 + i # 第 1 行空着，和真实报表一样从第 2 行开始 + return rows + + +def flow_workbook(start: dt.date, days: int, *, seed: int = 0, variant: str = "D00") -> tuple[bytes, str]: + """返回 (xlsx 字节, 文件名)。variant 见 VARIANTS（9.2 的用例代号）。""" + if variant not in VARIANTS: + raise ValueError(f"不认识的 variant：{variant}") + if days < 1: + raise ValueError("days 至少为 1") + end = start + dt.timedelta(days=days - 1) + dates = [start + dt.timedelta(days=i) for i in range(days)] + rows = _layout(start, end, days, seed, variant) + last = FIRST_COL + days - 1 + last_letter = get_column_letter(last) + hour_row = {row.key: row.r for row in rows if row.kind == "hour"} + hour_vals = {row.key: row.values for row in rows if row.kind == "hour"} + + book = Workbook() + ws = book.active + ws.title = SHEET + cached: dict[str, object] = {} + computed: dict[int, list[int]] = {} # 合计行 → 各日期的值（D16 的右侧合计列要用） + + def merge(r: int) -> None: + ws.merge_cells(start_row=r, start_column=LABEL_COL, end_row=r, end_column=last) + + for row in rows: + r = row.r + if row.kind == "empty": + continue + if row.kind in ("period", "text", "section", "note"): + ws.cell(r, LABEL_COL, row.label) + if row.merged: + merge(r) + elif row.kind == "blank": + if row.merged: + merge(r) + elif row.kind == "axis": + for i, d in enumerate(dates): + if variant == "D10": # 表头是日期格，显示成「9月1日」 + cell = ws.cell(r, FIRST_COL + i, dt.datetime(d.year, d.month, d.day)) + cell.number_format = 'm"月"d"日"' + else: + ws.cell(r, FIRST_COL + i, f"{d.month}月{d.day}日") + if variant == "D16": + ws.cell(r, last + 1, "合计") + elif row.kind in ("measure", "hour"): + ws.cell(r, LABEL_COL, row.label) + for i, v in enumerate(row.values or []): + ws.cell(r, FIRST_COL + i, v) + elif row.kind in ("total", "subtotal"): + ws.cell(r, LABEL_COL, row.label) + s, e = row.key + span = [h for h in range(s, e) if h in hour_row] + r1, r2 = hour_row[span[0]], hour_row[span[-1]] + assert r2 - r1 == len(span) - 1, "合计覆盖的时段行必须连续" + vals = [] + for i in range(days): + v = sum(x for h in span if isinstance(x := hour_vals[h][i], int)) + col = get_column_letter(FIRST_COL + i) + literal = variant in ("D07", "D08") and row.kind == "total" + if variant == "D08" and row.key == (18, 22) and i == 4: + v += 100 # 写死的「18-22 时合计」第 5 个日期多 100 + vals.append(v) + if literal: + ws.cell(r, FIRST_COL + i, v) + else: + ws.cell(r, FIRST_COL + i, f"=SUM({col}{r1}:{col}{r2})") + cached[f"{col}{r}"] = v + computed[r] = vals + elif row.kind == "raw": + for j, v in enumerate(row.values or []): + ws.cell(r, LABEL_COL + j, v) + else: # pragma: no cover - 行清单只会有上面这些 + raise AssertionError(row.kind) + + if variant == "D16": # 右侧多一列「合计」：每个数据行一个 SUM 公式，有缓存 + col = get_column_letter(last + 1) + for row in rows: + if row.kind in ("measure", "hour"): + vals = [v for v in row.values or [] if isinstance(v, int)] + elif row.kind in ("total", "subtotal"): + vals = computed[row.r] + else: + continue + ws.cell(row.r, last + 1, f"=SUM(C{row.r}:{last_letter}{row.r})") + cached[f"{col}{row.r}"] = sum(vals) + + if variant == "D20": # 多一个有内容的可见工作表 + extra = book.create_sheet(EXTRA_SHEET) + extra["A1"] = "本工作簿为合成测试数据，仅供测试使用" + + raw = excel_saved(save(book), SHEET, cached, full_calc=variant == "D25") + return raw, flow_filename(start, days, variant) diff --git a/backend/tests/fixtures/xlsx/lab.py b/backend/tests/fixtures/xlsx/lab.py new file mode 100644 index 00000000..9b480e6e --- /dev/null +++ b/backend/tests/fixtures/xlsx/lab.py @@ -0,0 +1,225 @@ +"""9.3 的实验室难缠用例，按 xlsx-lab/case*.py 的结构重写成合成夹具：地区、产品、人名一律换成假名。 + +各函数返回 (xlsx 字节, 文件名)，不写磁盘。openpyxl 写不出来的形态（公式缓存值、XlsxWriter 式的「缓存为 0 加 +打开时重算」）在压缩包层面补。期望（规则草稿 → 试运行）见 P2-SPEC 9.3，WP-7 验收。 +""" +from __future__ import annotations + +from openpyxl import Workbook + +from . import excel_saved, save + +ZONES = ("分区甲", "分区乙", "分区丙") +PRODUCTS = ("产品甲", "产品乙", "产品丙") + +C01_VARIANTS = ("literal", "uncached", "cached") +C05_VARIANTS = ("uncached", "zero_full_calc", "cached") +C08_VARIANTS = ("stacked", "gap", "side") +C10_MONTHS = (7, 8, 9, 10) + +#: 9.3 表里的叫法 → 这里的 variant +_ALIASES = {"写死": "literal", "公式无缓存": "uncached", "公式有缓存": "cached", "有缓存": "cached", + "openpyxl": "uncached", "xlsxwriter": "zero_full_calc"} + + +def _variant(v: str, allowed: tuple[str, ...]) -> str: + v = _ALIASES.get(v, v) + if v not in allowed: + raise ValueError(f"variant 只能是 {allowed}，收到 {v!r}") + return v + + +def c01(variant: str = "literal") -> tuple[bytes, str]: + """表头在 C5，上方标题、单位、编制行，下方合计行、空行、备注。 + + literal：合计写死；uncached：合计是公式、没有缓存值（openpyxl 直接写出的样子）；cached：补上缓存值。 + 期望：列表块 + total_row(pick=合计, keep_as)；区域外含数字的文字只有 C1、C3 两格（outside_digits ×2）。 + """ + variant = _variant(variant, C01_VARIANTS) + book = Workbook() + ws = book.active + ws.title = "月报" + ws["C1"] = "2026年8月 销售月报" + ws.merge_cells("C1:F1") + ws["C2"] = "单位：万元" + ws["C3"] = "编制：财务部 日期：2026-09-05" + for i, h in enumerate(("地区", "产品", "销量", "金额")): + ws.cell(5, 3 + i, h) + data = [(ZONES[0], PRODUCTS[0], 10, 100), (ZONES[1], PRODUCTS[1], 20, 120), (ZONES[2], PRODUCTS[2], 15, 80)] + for r, row in enumerate(data, start=6): + for i, v in enumerate(row): + ws.cell(r, 3 + i, v) + ws["C9"] = "合计" + cached: dict[str, object] = {} + if variant == "literal": + ws["E9"], ws["F9"] = 45, 300 + else: + ws["E9"], ws["F9"] = "=SUM(E6:E8)", "=SUM(F6:F8)" + if variant == "cached": + cached = {"E9": 45, "F9": 300} + # 备注不含数字：这一格若带数字会多出一条 outside_digits，与 9.3 的期望（C1、C3 两格）不符 + ws["C11"] = "注：数据来源于业务系统，金额含税。" + ws["C12"] = "制表人：经办甲" + return excel_saved(save(book), "月报", cached), f"c01_{variant}.xlsx" + + +def c04() -> tuple[bytes, str]: + """合并单元格：A1:F1 标题；两行表头（地区、产品纵向合并，上下半年横向合并、下层销量 / 金额）；地区纵向合并。""" + book = Workbook() + ws = book.active + ws.title = "合并" + ws["A1"] = "分地区销售（万元）" + ws.merge_cells("A1:F1") + ws["A3"], ws["B3"], ws["C3"], ws["E3"] = "地区", "产品", "2026年上半年", "2026年下半年" + for m in ("A3:A4", "B3:B4", "C3:D3", "E3:F3"): + ws.merge_cells(m) + for i, h in enumerate(("销量", "金额", "销量", "金额")): + ws.cell(4, 3 + i, h) + data = [(PRODUCTS[0], 1, 10, 2, 20), (PRODUCTS[1], 3, 30, 4, 40), (PRODUCTS[2], 5, 50, 6, 60), + (PRODUCTS[0], 7, 70, 8, 80), (PRODUCTS[1], 9, 90, 1, 10), (PRODUCTS[2], 2, 20, 3, 30)] + for r, row in enumerate(data, start=5): + for i, v in enumerate(row): + ws.cell(r, 2 + i, v) + ws["A5"], ws["A8"] = ZONES[0], ZONES[1] + ws.merge_cells("A5:A7") + ws.merge_cells("A8:A10") + return excel_saved(save(book), "合并", {}), "c04_merged.xlsx" + + +def c05(variant: str = "cached") -> tuple[bytes, str]: + """公式与缓存值：金额列是公式（单价 × 数量），末行合计。 + + uncached：openpyxl 直接写出，公式没有缓存值；zero_full_calc：模拟 XlsxWriter 的默认写法，缓存值全是 0、 + 并设打开时重算（fullCalcOnLoad）；cached：补上正确的缓存值（模拟 Excel 保存过）。 + """ + variant = _variant(variant, C05_VARIANTS) + book = Workbook() + ws = book.active + ws.title = "明细" + rows = [("产品", "单价", "数量", "金额"), (PRODUCTS[0], 2.5, 4), (PRODUCTS[1], 3, 10), (PRODUCTS[2], 10, 1)] + for row in rows: + ws.append(list(row)) + for r in (2, 3, 4): + ws.cell(r, 4, f"=B{r}*C{r}") + ws["A5"], ws["D5"] = "合计", "=SUM(D2:D4)" + raw = save(book) + if variant == "uncached": + return excel_saved(raw, "明细", {}), "c05_uncached.xlsx" + if variant == "zero_full_calc": + return excel_saved(raw, "明细", {"D2": 0, "D3": 0, "D4": 0, "D5": 0}, full_calc=True), "c05_zero_full_calc.xlsx" + return excel_saved(raw, "明细", {"D2": 10, "D3": 30, "D4": 10, "D5": 50}), "c05_cached.xlsx" + + +def c06() -> tuple[bytes, str]: + """隐藏：明细表第 4 行手动隐藏、C 列隐藏、自动筛选（地区 = 分区甲）隐藏其余行；另有 hidden、veryHidden 工作表。""" + book = Workbook() + ws = book.active + ws.title = "明细" + ws.append(["地区", "产品", "成本", "金额"]) + regions = [ZONES[0], ZONES[1], ZONES[0], ZONES[2], ZONES[0], ZONES[1], ZONES[0], ZONES[2], ZONES[0], ZONES[1]] + for i, reg in enumerate(regions): + ws.append([reg, f"型号{'甲乙丙丁戊己庚辛壬癸'[i]}", i, 10 * (i + 1)]) + ws.row_dimensions[4].hidden = True + ws.column_dimensions["C"].hidden = True + ws.auto_filter.ref = "A1:D11" + ws.auto_filter.add_filter_column(0, [ZONES[0]]) + for r in range(2, 12): # Excel 保存筛选结果时就是给不满足条件的行打 hidden + if ws.cell(r, 1).value != ZONES[0]: + ws.row_dimensions[r].hidden = True + draft = book.create_sheet("草稿") + draft.append(["甲", "乙"]) + draft.append([1, 2]) + draft.sheet_state = "hidden" + conf = book.create_sheet("配置") + conf.append(["键", "值"]) + conf.append(["参数甲", "示例值"]) + conf.sheet_state = "veryHidden" + return excel_saved(save(book), "明细", {}), "c06_hidden.xlsx" + + +def c08(variant: str = "stacked") -> tuple[bytes, str]: + """同一工作表里的多张表。stacked：上下两张（各有小标题）；gap：一张表中间一行空白；side：左右两张、隔一空列。""" + variant = _variant(variant, C08_VARIANTS) + book = Workbook() + ws = book.active + if variant == "stacked": + ws.title = "汇总" + ws["B1"] = "一、销售" + for r, row in enumerate([("地区", "销量", "金额"), (ZONES[0], 10, 100), (ZONES[1], 20, 120), + (ZONES[2], 15, 80)], start=2): + for c, v in enumerate(row, start=2): + ws.cell(r, c, v) + ws["B7"] = "二、费用" + for r, row in enumerate([("部门", "费用"), ("部门甲", 5), ("部门乙", 7), ("部门丙", 8)], start=8): + for c, v in enumerate(row, start=2): + ws.cell(r, c, v) + sheet = "汇总" + elif variant == "gap": + ws.title = "明细" + for row in [("地区", "产品", "金额"), (ZONES[0], PRODUCTS[0], 10), (ZONES[0], PRODUCTS[1], 20), + (None, None, None), (ZONES[1], PRODUCTS[0], 30), (ZONES[1], PRODUCTS[1], 40)]: + ws.append(list(row)) + sheet = "明细" + else: + ws.title = "并排" + for r, (a, b) in enumerate([("地区", "金额"), (ZONES[0], 1), (ZONES[1], 2)], start=1): + ws.cell(r, 1, a) + ws.cell(r, 2, b) + for r, (a, b) in enumerate([("部门", "费用"), ("部门甲", 5), ("部门乙", 7)], start=1): + ws.cell(r, 4, a) + ws.cell(r, 5, b) + sheet = "并排" + return excel_saved(save(book), sheet, {}), f"c08_{variant}.xlsx" + + +_C10_DATA = ((ZONES[0], PRODUCTS[0], 10, 100), (ZONES[1], PRODUCTS[1], 20, 120), (ZONES[2], PRODUCTS[2], 15, 80)) + + +def c10(month: int) -> tuple[bytes, str]: + """版式漂移（以 7 月起草并启用）： + + 7 月：表头在 C5：地区 产品 销量 金额，下面合计行； + 8 月：表头挪到 C7（上面多两行说明），产品和销量之间多「单价」列 → column_extra； + 9 月：表头在 B6，外观差异「地区　」「销量\\n」「金 额」→ 通过、差异卡 header_writing； + 10 月：「金额」改名「销售额」→ column_missing 加 column_extra。 + """ + if month not in C10_MONTHS: + raise ValueError(f"month 只能是 {C10_MONTHS}") + top, left, headers, extra_rows, with_price = { + 7: (5, 3, ["地区", "产品", "销量", "金额"], 0, False), + 8: (7, 3, ["地区", "产品", "单价", "销量", "金额"], 2, True), + 9: (6, 2, ["地区　", "产品", "销量\n", "金 额"], 1, False), + 10: (5, 3, ["地区", "产品", "销量", "销售额"], 0, False), + }[month] + book = Workbook() + ws = book.active + ws.title = "月报" + ws.cell(1, left, "销售月报") + ws.cell(2, left, "单位：万元") + for i in range(extra_rows): # 说明行不含数字：只让表头的变化产生差异 + ws.cell(3 + i, left, f"说明{'甲乙丙'[i]}：口径同上月") + for j, h in enumerate(headers): + ws.cell(top, left + j, h) + for r, (zone, prod, qty, amt) in enumerate(_C10_DATA, start=top + 1): + vals = [zone, prod] + ([amt / qty] if with_price else []) + [qty, amt] + for j, v in enumerate(vals): + ws.cell(r, left + j, v) + tot = top + 1 + len(_C10_DATA) + ws.cell(tot, left, "合计") + ws.cell(tot, left + len(headers) - 2, sum(d[2] for d in _C10_DATA)) + ws.cell(tot, left + len(headers) - 1, sum(d[3] for d in _C10_DATA)) + return excel_saved(save(book), "月报", {}), f"c10_2026-{month:02d}.xlsx" + + +def kv_form() -> tuple[bytes, str]: + """键值表单式的版式（规则起草认不出、交给 AI 兜底起草的用例，9.7 第 10 步）：两列「项目：值」，没有表头行。""" + book = Workbook() + ws = book.active + ws.title = "基本情况" + ws["A1"] = "基本情况登记表" + items = [("单位名称", "示例单位甲"), ("所在分区", ZONES[0]), ("从业人数", 120), ("营业面积", 860), + ("年度客流", 35600), ("负责人", "经办甲")] + for r, (k, v) in enumerate(items, start=3): + ws.cell(r, 1, k) + ws.cell(r, 2, v) + return excel_saved(save(book), "基本情况", {}), "kv_form.xlsx" diff --git a/backend/tests/fixtures/xlsx/mutate.py b/backend/tests/fixtures/xlsx/mutate.py new file mode 100644 index 00000000..2236744f --- /dev/null +++ b/backend/tests/fixtures/xlsx/mutate.py @@ -0,0 +1,141 @@ +"""9.4 的数据变异：在 XML 层改 D00 的字节（flow_workbook(2026-08-01, 31)），保留缓存值。 + +做法同 xlsx-lab 评审时的 mutate.py，但输入是合成夹具。openpyxl 写不出来的形态（去掉公式留缓存、陈旧的 +、右侧多出的格子不进 ）只能这么造。每种变异只改预定的格子，其余字节级不动 +（test_fixtures_xlsx 逐格比对）。 +""" +from __future__ import annotations + +import re + +from . import patch_parts, sheet_part +from .flow import SHEET + +#: 代号 → (改法, 期望的结果, 期望出现的问题 code 或核对 id)。照 P2-SPEC 9.4 抄录，WP-7 验收时逐项比 +MUTATIONS: dict[str, tuple[str, str, tuple[str, ...]]] = { + "P1": ("第 28–30 行去掉公式、保留缓存值", "passed", ()), + "P1b": ("第 28–30 行保留公式、去掉缓存值", "needs_decision", ("K1",)), + "P2": ("D15 改成空", "rejected", ("value_blank",)), + "P6": ("R4 改成与 Q4 相同的日期", "rejected", ("axis_duplicate",)), + "P9": ("C28 改成 =SUM(C22:C24)，缓存值自洽", "rejected", ("K1", "G1")), + "P20": ("第 32 行加「备用分区（人次）」和两个数， 不更新", "rejected", ("row_unclaimed",)), + "P23": (" 截到第 27 行", "passed", ()), + "P24": (" 截到倒数第二个日期列", "passed", ()), + "P25": ("右侧多一列：表头「合计」、各数据行 999， 不更新", "rejected", ("axis_extra_cells",)), + "P26": ("删掉最后一个日期列的所有格， 跟着改，B2 仍写到月末", "rejected", ("axis_coverage",)), + # 9.4「外加」：同一分段里一行写 8-9、另一行写 8－9（全角横线） + "PKDUP": ("B12（9-10）改成「8－9」，与 B11 的「8-9」规范写法相同", "rejected", ("pk_duplicate", "label_duplicate")), +} + +#: D00 的数据行（指标、日间、夜间、合计） +DATA_ROWS = (5, 6, 7, *range(10, 21), *range(22, 31)) +_TOTAL_ROWS = "(?:28|29|30)" + + +def _cell_span(xml: str, ref: str) -> re.Match[str]: + m = re.search(rf']*/>|[^>]*>.*?)', xml) + if m is None: + raise AssertionError(f"没有格子 {ref}") + return m + + +def _replace_cell(xml: str, ref: str, new: str) -> str: + m = _cell_span(xml, ref) + return xml[:m.start()] + new + xml[m.end():] + + +def _number(xml: str, ref: str) -> float: + m = re.search(r"([^<]*)", _cell_span(xml, ref).group(0)) + if m is None: + raise AssertionError(f"格子 {ref} 没有值") + return float(m.group(1)) + + +def _text(xml: str, ref: str) -> str: + m = re.search(r"]*>([^<]*)", _cell_span(xml, ref).group(0)) + if m is None: + raise AssertionError(f"格子 {ref} 不是文字") + return m.group(1) + + +def _inline(ref: str, text: str) -> str: + return f'{text}' + + +def _num(ref: str, v: float | int) -> str: + return f'{v}' + + +def _append_to_row(xml: str, row: int, cell: str) -> str: + xml, n = re.subn(rf'(]*>.*?)()', lambda m: m.group(1) + cell + m.group(2), xml, + count=1, flags=re.S) + if n != 1: + raise AssertionError(f"没有第 {row} 行") + return xml + + +def _dimension(xml: str) -> tuple[str, int, str, int]: + m = re.search(r'', xml) + if m is None: + raise AssertionError("没有 ") + return m.group(1), int(m.group(2)), m.group(3), int(m.group(4)) + + +def _set_dimension(xml: str, ref: str) -> str: + return re.sub(r'', f'', xml, count=1) + + +def _col_shift(letters: str, delta: int) -> str: + n = 0 + for ch in letters: + n = n * 26 + ord(ch) - 64 + n += delta + out = "" + while n: + n, rem = divmod(n - 1, 26) + out = chr(65 + rem) + out + return out + + +def _mutate_sheet(xml: str, case: str) -> str: + c1, r1, last, r2 = _dimension(xml) + if (c1, r1, r2) != ("B", 2, 30): + raise ValueError(f"mutate 只接受 D00 的版式（ 是 B2:…30），这份是 {c1}{r1}:{last}{r2}") + if case == "P1": + return re.sub(rf'(]*>)[^<]*', r"\1", xml) + if case == "P1b": + return re.sub(rf'(]*>[^<]*)(?:[^<]*|)', r"\1", xml) + if case == "P2": + return _replace_cell(xml, "D15", "") + if case == "P6": + return _replace_cell(xml, "R4", _inline("R4", _text(xml, "Q4"))) + if case == "P9": + v = sum(_number(xml, f"C{r}") for r in (22, 23, 24)) + return _replace_cell(xml, "C28", f'SUM(C22:C24){int(v)}') + if case == "P20": + row = ('' + _inline("B32", "备用分区（人次）") + _num("C32", 12) + _num("D32", 15) + "") + return xml.replace("", row + "", 1) + if case == "P23": + return _set_dimension(xml, f"B2:{last}27") + if case == "P24": + return _set_dimension(xml, f"B2:{_col_shift(last, -1)}30") + if case == "P25": + extra = _col_shift(last, 1) + xml = _append_to_row(xml, 4, _inline(f"{extra}4", "合计")) + for r in DATA_ROWS: + xml = _append_to_row(xml, r, _num(f"{extra}{r}", 999)) + return xml + if case == "P26": + xml = re.sub(rf']*/>|[^>]*>.*?)', "", xml) + return _set_dimension(xml, f"B2:{_col_shift(last, -1)}30") + if case == "PKDUP": + return _replace_cell(xml, "B12", _inline("B12", "8－9")) + raise ValueError(f"不认识的变异：{case}") + + +def mutate(xlsx: bytes, case: str) -> bytes: + """在 XML 层改 D00 的字节。case 见 MUTATIONS。""" + if case not in MUTATIONS: + raise ValueError(f"不认识的变异：{case}") + part = sheet_part(xlsx, SHEET) + return patch_parts(xlsx, {part: lambda t: _mutate_sheet(t, case)}) diff --git a/backend/tests/test_fixtures_xlsx.py b/backend/tests/test_fixtures_xlsx.py new file mode 100644 index 00000000..9d80305d --- /dev/null +++ b/backend/tests/test_fixtures_xlsx.py @@ -0,0 +1,456 @@ +"""合成 xlsx 夹具的自检（P2-SPEC 9.1–9.4）：坐标与规格的表一致、公式有缓存、变异只改到预定的格子。 + +夹具是执行器、核对、差异卡验收的输入；它自己错了，下游的验收全都失去意义，所以这里逐格对。 +""" +from __future__ import annotations + +import datetime as dt +import io +import re + +import pytest +from openpyxl import load_workbook +from openpyxl.utils import get_column_letter + +from app.data import xlsx_scan +from tests.fixtures.xlsx import has_full_calc, read_part, sheet_part +from tests.fixtures.xlsx import lab +from tests.fixtures.xlsx.drift import AUG, DRIFT_CASES, GROUPS, OCT, SEP +from tests.fixtures.xlsx.flow import ( + DAY_TITLE, + FLOW_EXPECT, + NIGHT_TITLE, + PLACEHOLDER, + SHEET, + VARIANTS, + flow_workbook, +) +from tests.fixtures.xlsx.mutate import MUTATIONS, mutate + +D00_RAW, D00_NAME = flow_workbook(AUG, 31) + + +def book(raw: bytes, *, data_only: bool = False): + return load_workbook(io.BytesIO(raw), data_only=data_only) + + +def cells(raw: bytes, sheet: str = SHEET, *, data_only: bool = False) -> dict[str, object]: + """非空格 → 值（data_only=False 时公式格是公式原文）。""" + ws = book(raw, data_only=data_only)[sheet] + out = {} + for row in ws.iter_rows(): + for c in row: + if c.value is not None: + out[c.coordinate] = c.value + return out + + +def col(i: int) -> str: + """第 i 个日期（0 起）所在的列字母：C 起。""" + return get_column_letter(3 + i) + + +# ========================================================================== +# 9.1 D00 +# ========================================================================== + + +def test_d00_layout_matches_spec(): + assert D00_NAME == "月报导出_2026-08-01_2026-08-31.xlsx" + wb = book(D00_RAW) + assert wb.sheetnames == [SHEET] + ws = wb[SHEET] + assert ws.sheet_state == "visible" + assert {str(m) for m in ws.merged_cells.ranges} == {"B2:AG2", "B3:AG3", "B8:AG8", "B9:AG9", "B21:AG21"} + assert ws["B2"].value == "统计时间范围：2026年8月1日至2026年8月31日" + assert ws["B3"].value == "客流汇总表" + assert ws["B4"].value is None + assert [ws[f"{col(i)}4"].value for i in range(31)] == [f"8月{d}日" for d in range(1, 32)] + assert ws["AH4"].value is None + assert [ws[f"B{r}"].value for r in (5, 6, 7)] == ["全日客流（人次）", "分区甲（人次）", "分区乙（人次）"] + assert all(ws.cell(8, c).value is None for c in range(2, 34)) + assert ws["B9"].value == DAY_TITLE and ws["B21"].value == NIGHT_TITLE + assert [ws[f"B{r}"].value for r in range(10, 21)] == [f"{h}-{h + 1}" for h in range(7, 18)] + assert [ws[f"B{r}"].value for r in range(22, 28)] == [f"{h}-{h + 1}" for h in range(18, 24)] + assert [ws[f"B{r}"].value for r in (28, 29, 30)] == ["18-22 时合计", "22-24 时合计", "18-24 时合计"] + assert all(ws[f"{col(i)}10"].value == PLACEHOLDER for i in range(31)) + assert all(ws.cell(9, c).value is None and ws.cell(21, c).value is None for c in range(3, 34)) + assert ws.max_row == 30 + + +def test_d00_values_and_formulas(): + ws = book(D00_RAW)[SHEET] + cached = book(D00_RAW, data_only=True)[SHEET] + for i in range(31): + c = col(i) + a, b, total = ws[f"{c}6"].value, ws[f"{c}7"].value, ws[f"{c}5"].value + assert 3000 <= a <= 9000 and 2000 <= b <= 8000 and total == a + b + hours = {r: ws[f"{c}{r}"].value for r in [*range(11, 21), *range(22, 28)]} + assert all(isinstance(v, int) and 50 <= v <= 900 for v in hours.values()) + assert sum(hours.values()) != total # 各时段之和 ≠ 全日客流 + assert ws[f"{c}28"].value == f"=SUM({c}22:{c}25)" + assert ws[f"{c}29"].value == f"=SUM({c}26:{c}27)" + assert ws[f"{c}30"].value == f"=SUM({c}22:{c}27)" + assert cached[f"{c}28"].value == sum(hours[r] for r in range(22, 26)) + assert cached[f"{c}29"].value == sum(hours[r] for r in (26, 27)) + assert cached[f"{c}30"].value == sum(hours[r] for r in range(22, 28)) + assert not has_full_calc(D00_RAW) + + +def test_d00_scan_matches_expectations(): + scan = xlsx_scan.scan(D00_RAW) + assert not scan.full_calc_on_load and [s.name for s in scan.sheets] == [SHEET] + s = scan.sheets[0] + assert s.bounds.a1() == FLOW_EXPECT["bounds"] + assert s.nonempty == FLOW_EXPECT["nonempty"] == sum(FLOW_EXPECT["ledger"].values()) + assert len(s.merged) == FLOW_EXPECT["merged"] + assert s.formulas == FLOW_EXPECT["formulas"] and s.formulas_uncached.total == 0 + assert s.hidden_rows == [] and s.hidden_cols == [] + + +def test_flow_workbook_is_deterministic_and_seeded(): + assert flow_workbook(AUG, 31)[0] == D00_RAW + assert flow_workbook(AUG, 31, seed=1)[0] != D00_RAW + with pytest.raises(ValueError): + flow_workbook(AUG, 31, variant="D99") + + +def test_r2_never_equal_across_seeds(): + for seed in range(20): + ws = book(flow_workbook(SEP, 30, seed=seed)[0])[SHEET] + for i in range(30): + c = col(i) + hours = [ws[f"{c}{r}"].value for r in [*range(11, 21), *range(22, 28)]] + assert sum(hours) != ws[f"{c}5"].value + + +# ========================================================================== +# 9.2 漂移用例 +# ========================================================================== + + +def test_drift_cases_catalogue(): + assert len(DRIFT_CASES) == 31 and "D28b" in DRIFT_CASES + assert set(DRIFT_CASES) <= VARIANTS + assert sorted(c for g in GROUPS.values() for c in g) == sorted(DRIFT_CASES) + assert [len(GROUPS[k]) for k in GROUPS] == [12, 8, 3, 6, 2] + for code, case in DRIFT_CASES.items(): + assert case.code == code + assert case.expect in ("passed", "confirm", "rejected", "decision") + if case.expect == "rejected": + assert case.rows is None and case.diff_kinds is None and case.codes + else: + assert case.rows is not None and case.diff_kinds is not None + if code != "D00" and case.diff_kinds is not None: + assert "period" in case.diff_kinds + assert {c for c, k in DRIFT_CASES.items() if k.expect == "passed"} == set(GROUPS["通过"]) + assert DRIFT_CASES["D26"].expect == "decision" and DRIFT_CASES["D08"].expect == "rejected" + + +def _sheet(raw: bytes, *, data_only: bool = False): + return book(raw, data_only=data_only)[SHEET] + + +def _labels(ws) -> list[object]: + return [ws.cell(r, 2).value for r in range(1, ws.max_row + 1)] + + +def _row_of(ws, label: str) -> int: + return _labels(ws).index(label) + 1 + + +#: 每例的「改到了」检查（ws 是 data_only=False 的工作表，cached 是 data_only=True 的） +def _check_variant(code: str, raw: bytes, name: str, start: dt.date, days: int) -> None: + ws, cached = _sheet(raw), _sheet(raw, data_only=True) + end = start + dt.timedelta(days=days - 1) + last = col(days - 1) + labels = _labels(ws) + period = f"统计时间范围：{start.year}年{start.month}月{start.day}日至{end.year}年{end.month}月{end.day}日" + expect_name = f"月报导出_{start.isoformat()}_{end.isoformat()}.xlsx" + if code == "D23": + assert name == "9月客流.xlsx" + else: + assert name == expect_name + if code in ("D00", "D01", "D02", "D03", "D03b", "D23"): # D23 只改文件名（上面已查） + assert ws["B2"].value == period and ws[f"{last}4"].value == f"{end.month}月{end.day}日" + assert ws.cell(4, 4 + days).value is None + elif code == "D12": + assert ws["B2"].value == f"统计时间范围：{start.year}年{start.month}月{start.day}日至{end.month}月{end.day}日" + elif code == "D14": + assert ws["B8"].value == DAY_TITLE and ws["B29"].value == "18-24 时合计" and ws.max_row == 29 + assert ws["C27"].value == "=SUM(C21:C24)" + elif code == "D15": + assert {str(m) for m in ws.merged_cells.ranges} == {f"B2:{last}2", f"B3:{last}3", f"B8:{last}8"} + elif code == "D17": + assert ws["B3"].value == "客流汇总表（2026年9月）" + elif code == "D25": + assert has_full_calc(raw) and cached["C28"].value is not None + elif code == "D27": + assert ws["B3"].value == "客流汇总表（2026年9月1日至2026年9月30日）" + elif code == "D05": + assert ws["B9"].value == NIGHT_TITLE and ws["B19"].value == DAY_TITLE + assert ws["B16"].value == "18-22 时合计" and ws["C16"].value == "=SUM(C10:C13)" + elif code == "D06": + assert ws["B5"].value == DAY_TITLE and ws["B28"].value == "日客流（人次）" + assert labels[28:31] == ["全日客流（人次）", "分区甲（人次）", "分区乙（人次）"] and ws["B27"].value is None + elif code in ("D07", "D08"): + for r, span in ((28, range(22, 26)), (29, (26, 27)), (30, range(22, 28))): + for i in range(days): + c = col(i) + v = ws[f"{c}{r}"].value + assert isinstance(v, int) + want = sum(ws[f"{c}{h}"].value for h in span) + assert v == want + (100 if (code == "D08" and r == 28 and i == 4) else 0) + assert not any(isinstance(v, str) and v.startswith("=") for v in cells(raw).values()) + elif code == "D10": + assert ws["C4"].value == dt.datetime(2026, 9, 1) and ws["C4"].number_format == 'm"月"d"日"' + elif code == "D11": + assert ws["B10"].value == "07:00-08:00" and ws["B27"].value == "23:00-24:00" + elif code == "D19": + assert [ws[f"B{r}"].value for r in (5, 6, 7)] == ["分区甲（人次）", "分区乙（人次）", "全日客流（人次）"] + elif code == "D22": + assert [ws[f"B{r}"].value for r in (28, 29, 30)] == ["18:00-22:00合计", "22:00-24:00合计", "18:00-24:00合计"] + elif code == "D24": + assert ws["B10"].value == "7–8" and ws["B28"].value == "18-22 时合计" + elif code == "D20": + wb = book(raw) + assert wb.sheetnames == [SHEET, "说明"] and wb["说明"].sheet_state == "visible" and wb["说明"]["A1"].value + elif code == "D28": + assert ws["B31"].value is None and ws["B32"].value == "注：9月15日闸机故障，当日客流为估算值" + elif code == "D28b": + assert ws["B31"].value is None and ws["B32"].value == "注：2026年9月数据为初步统计，待修订" + elif code == "D04": + assert "7-8" not in labels and ws["B10"].value == "8-9" and ws["B20"].value == NIGHT_TITLE + elif code == "D09": + assert ws["B9"].value == "日间分时段客流（人次）" + elif code == "D13": + assert ws["B8"].value == "分区丙（人次）" and ws["B10"].value == DAY_TITLE + assert all(ws[f"{col(i)}5"].value == sum(ws[f"{col(i)}{r}"].value for r in (6, 7, 8)) for i in range(days)) + elif code == "D16": + extra = get_column_letter(4 + days - 1) + assert ws[f"{extra}4"].value == "合计" and ws[f"{extra}9"].value is None + for r in (5, 10, 28): + assert ws[f"{extra}{r}"].value == f"=SUM(C{r}:{last}{r})" + nums = [ws[f"{col(i)}{r}"].value for i in range(days)] + want = sum(v for v in nums if isinstance(v, int)) if r != 28 else sum( + cached[f"{col(i)}28"].value for i in range(days)) + assert cached[f"{extra}{r}"].value == want + elif code == "D18": + assert ws["B21"].value == "7-18 时合计" and ws["C21"].value == "=SUM(C10:C20)" + assert cached["C21"].value == sum(ws[f"C{r}"].value for r in range(11, 21)) + assert ws["B22"].value == NIGHT_TITLE + elif code == "D21": + assert ws["B31"].value is None + assert [ws[f"{c}32"].value for c in "BCD"] == ["补录（人次）", "1,234", "2,345"] + elif code == "D26": + c = col(9) + assert ws[f"{c}5"].value == ws[f"{c}6"].value + ws[f"{c}7"].value + 7 + assert all(ws[f"{col(i)}5"].value == ws[f"{col(i)}6"].value + ws[f"{col(i)}7"].value + for i in range(days) if i != 9) + else: # pragma: no cover + raise AssertionError(f"没有为 {code} 写检查") + + +_STARTS = {"D00": (AUG, 31), "D02": (OCT, 31), "D03": (OCT, 15), "D03b": (OCT, 7)} + + +@pytest.mark.parametrize("code", sorted(DRIFT_CASES)) +def test_drift_case_builds_with_the_intended_change(code): + case = DRIFT_CASES[code] + start, days = _STARTS.get(code, (SEP, 30)) + for seed in (0, 1): + raw, name = case.build(seed) + _check_variant(code, raw, name, start, days) + scan = xlsx_scan.scan(raw) # 扫描层能读（不在扫描阶段被拒） + assert scan.sheets[0].formulas_uncached.total == 0 + assert has_full_calc(raw) is (code == "D25") + if case.rows is not None and code not in ("D13",): + assert case.rows == (days, 17 * days, 3 * days) + + +def test_noperiod_variant_drops_b2_only(): + raw, _ = flow_workbook(SEP, 30, variant="noperiod") + ws = _sheet(raw) + assert ws["B2"].value is None and ws["B3"].value == "客流汇总表" and ws["C4"].value == "9月1日" + assert "B2:AF2" not in {str(m) for m in ws.merged_cells.ranges} + + +# ========================================================================== +# 9.4 数据变异 +# ========================================================================== + + +def _diff(a: dict[str, object], b: dict[str, object]) -> set[str]: + return {k for k in a.keys() | b.keys() if a.get(k) != b.get(k)} + + +def _dimension(raw: bytes) -> str: + return re.search(r'', read_part(raw, sheet_part(raw, SHEET))).group(1) + + +TOTAL_CELLS = {f"{col(i)}{r}" for i in range(31) for r in (28, 29, 30)} +AH_CELLS = {"AH4"} | {f"AH{r}" for r in (5, 6, 7, *range(10, 21), *range(22, 31))} +AG_CELLS = {"AG4"} | {f"AG{r}" for r in (5, 6, 7, *range(10, 21), *range(22, 31))} + +#: 代号 → (公式视图里变了的格, 缓存值视图里变了的格, 变异后的 ) +MUTATION_TARGETS = { + "P1": (TOTAL_CELLS, set(), "B2:AG30"), + "P1b": (set(), TOTAL_CELLS, "B2:AG30"), + "P2": ({"D15"}, {"D15"}, "B2:AG30"), + "P6": ({"R4"}, {"R4"}, "B2:AG30"), + "P9": ({"C28"}, {"C28"}, "B2:AG30"), + "P20": ({"B32", "C32", "D32"}, {"B32", "C32", "D32"}, "B2:AG30"), + "P23": (set(), set(), "B2:AG27"), + "P24": (set(), set(), "B2:AF30"), + "P25": (AH_CELLS, AH_CELLS, "B2:AG30"), + "P26": (AG_CELLS, AG_CELLS, "B2:AF30"), + "PKDUP": ({"B12"}, {"B12"}, "B2:AG30"), +} + + +def test_mutation_catalogue(): + assert set(MUTATIONS) == set(MUTATION_TARGETS) + assert {"P1", "P1b", "P2", "P6", "P9", "P20", "P23", "P24", "P25", "P26"} <= set(MUTATIONS) + with pytest.raises(ValueError): + mutate(D00_RAW, "P99") + with pytest.raises(ValueError): # 只接受 D00 的版式 + mutate(flow_workbook(SEP, 30, variant="D21")[0], "P1") + + +@pytest.mark.parametrize("case", sorted(MUTATION_TARGETS)) +def test_mutation_changes_only_the_intended_cells(case): + raw = mutate(D00_RAW, case) + formula_cells, cached_cells, dimension = MUTATION_TARGETS[case] + assert _diff(cells(D00_RAW), cells(raw)) == formula_cells + assert _diff(cells(D00_RAW, data_only=True), cells(raw, data_only=True)) == cached_cells + assert _dimension(raw) == dimension + assert has_full_calc(raw) is False + xlsx_scan.scan(raw) + + +def test_mutation_values(): + before, before_v = cells(D00_RAW), cells(D00_RAW, data_only=True) + after = cells(mutate(D00_RAW, "P1"), data_only=False) + assert all(isinstance(after[c], int) and after[c] == before_v[c] for c in TOTAL_CELLS) + after_v = cells(mutate(D00_RAW, "P1b"), data_only=True) + assert all(c not in after_v for c in TOTAL_CELLS) + assert "D15" not in cells(mutate(D00_RAW, "P2")) + p6 = cells(mutate(D00_RAW, "P6")) + assert p6["R4"] == p6["Q4"] == "8月15日" + p9, p9v = cells(mutate(D00_RAW, "P9")), cells(mutate(D00_RAW, "P9"), data_only=True) + assert p9["C28"] == "=SUM(C22:C24)" and p9v["C28"] == sum(before[f"C{r}"] for r in (22, 23, 24)) + p20 = cells(mutate(D00_RAW, "P20")) + assert (p20["B32"], p20["C32"], p20["D32"]) == ("备用分区（人次）", 12, 15) + p25 = cells(mutate(D00_RAW, "P25")) + assert p25["AH4"] == "合计" and all(p25[c] == 999 for c in AH_CELLS - {"AH4"}) + assert cells(mutate(D00_RAW, "PKDUP"))["B12"] == "8－9" + scan = xlsx_scan.scan(mutate(D00_RAW, "P1b")).sheets[0] + assert scan.formulas_uncached.total == 93 + + +# ========================================================================== +# 9.3 实验室用例 +# ========================================================================== + + +def _all(raw: bytes) -> dict[tuple[str, str], object]: + wb = book(raw) + return {(ws.title, c.coordinate): c.value for ws in wb.worksheets for row in ws.iter_rows() for c in row + if c.value is not None} + + +LAB_CASES = ([("c01", v, lab.c01(v)) for v in lab.C01_VARIANTS] + [("c04", "", lab.c04())] + + [("c05", v, lab.c05(v)) for v in lab.C05_VARIANTS] + [("c06", "", lab.c06())] + + [("c08", v, lab.c08(v)) for v in lab.C08_VARIANTS] + [("c10", m, lab.c10(m)) for m in lab.C10_MONTHS] + + [("kv", "", lab.kv_form())]) + + +@pytest.mark.parametrize("name,variant,built", LAB_CASES, ids=[f"{n}-{v}" for n, v, _ in LAB_CASES]) +def test_lab_cases_build_with_fake_names(name, variant, built): + raw, filename = built + assert filename.endswith(".xlsx") + xlsx_scan.scan(raw) + text = " ".join(str(v) for v in _all(raw).values()) + for real in ("华东", "华北", "华南", "张三", "secret", "口令"): + assert real not in text + + +def test_lab_c01(): + for variant in lab.C01_VARIANTS: + raw, _ = lab.c01(variant) + ws, cached = book(raw)["月报"], book(raw, data_only=True)["月报"] + assert [ws.cell(5, c).value for c in range(3, 7)] == ["地区", "产品", "销量", "金额"] + assert ws["C9"].value == "合计" and {str(m) for m in ws.merged_cells.ranges} == {"C1:F1"} + digits = [ref for ref in ("C1", "C2", "C3", "C11", "C12") if any(ch.isdigit() for ch in ws[ref].value)] + assert digits == ["C1", "C3"] + if variant == "literal": + assert (ws["E9"].value, ws["F9"].value) == (45, 300) + else: + assert (ws["E9"].value, ws["F9"].value) == ("=SUM(E6:E8)", "=SUM(F6:F8)") + want = (45, 300) if variant == "cached" else (None, None) + assert (cached["E9"].value, cached["F9"].value) == want + assert not has_full_calc(raw) + with pytest.raises(ValueError): + lab.c01("nope") + + +def test_lab_c04_merges(): + ws = book(lab.c04()[0])["合并"] + assert {str(m) for m in ws.merged_cells.ranges} == {"A1:F1", "A3:A4", "B3:B4", "C3:D3", "E3:F3", "A5:A7", "A8:A10"} + assert (ws["C3"].value, ws["E3"].value) == ("2026年上半年", "2026年下半年") + assert [ws.cell(4, c).value for c in range(3, 7)] == ["销量", "金额", "销量", "金额"] + assert (ws["A5"].value, ws["A8"].value, ws["A6"].value) == ("分区甲", "分区乙", None) + + +def test_lab_c05_formula_variants(): + want = {"uncached": ([None] * 4, False), "zero_full_calc": ([0] * 4, True), "cached": ([10, 30, 10, 50], False)} + for variant, (values, full) in want.items(): + raw, _ = lab.c05(variant) + ws, cached = book(raw)["明细"], book(raw, data_only=True)["明细"] + assert [ws[f"D{r}"].value for r in range(2, 6)] == ["=B2*C2", "=B3*C3", "=B4*C4", "=SUM(D2:D4)"] + assert [cached[f"D{r}"].value for r in range(2, 6)] == values + assert has_full_calc(raw) is full + assert lab.c05("openpyxl")[0] == lab.c05("uncached")[0] + + +def test_lab_c06_hidden(): + raw, _ = lab.c06() + wb = book(raw) + assert [(ws.title, ws.sheet_state) for ws in wb.worksheets] == [ + ("明细", "visible"), ("草稿", "hidden"), ("配置", "veryHidden")] + ws = wb["明细"] + hidden = sorted(r for r, d in ws.row_dimensions.items() if d.hidden) + assert hidden == [3, 4, 5, 7, 9, 11] + assert ws.column_dimensions["C"].hidden and ws.auto_filter.ref == "A1:D11" + s = xlsx_scan.scan(raw).sheets[0] + assert s.hidden_rows == [(3, 5), (7, 7), (9, 9), (11, 11)] and s.hidden_cols == [(3, 3)] + + +def test_lab_c08_layouts(): + ws = book(lab.c08("stacked")[0])["汇总"] + assert (ws["B1"].value, ws["B2"].value, ws["B7"].value, ws["B8"].value) == ("一、销售", "地区", "二、费用", "部门") + ws = book(lab.c08("gap")[0])["明细"] + assert [ws.cell(4, c).value for c in (1, 2, 3)] == [None, None, None] and ws["A5"].value == "分区乙" + ws = book(lab.c08("side")[0])["并排"] + assert (ws["A1"].value, ws["C1"].value, ws["D1"].value) == ("地区", None, "部门") + + +def test_lab_c10_months(): + want = {7: ("C5", ["地区", "产品", "销量", "金额"]), 8: ("C7", ["地区", "产品", "单价", "销量", "金额"]), + 9: ("B6", ["地区　", "产品", "销量\n", "金 额"]), 10: ("C5", ["地区", "产品", "销量", "销售额"])} + for month, (anchor, headers) in want.items(): + raw, name = lab.c10(month) + ws = book(raw)["月报"] + cell = ws[anchor] + assert [ws.cell(cell.row, cell.column + j).value for j in range(len(headers))] == headers + assert ws.cell(cell.row + 4, cell.column).value == "合计" + assert name == f"c10_2026-{month:02d}.xlsx" + notes = [ws.cell(r, cell.column).value for r in range(1, cell.row)] + assert not any(isinstance(v, str) and any(ch.isdigit() for ch in v) for v in notes) + with pytest.raises(ValueError): + lab.c10(11) + + +def test_lab_kv_form_has_no_header_row(): + ws = book(lab.kv_form()[0])["基本情况"] + assert ws["A1"].value == "基本情况登记表" and ws["A3"].value == "单位名称" and ws.max_column == 2 diff --git a/backend/tests/test_recipe_acceptance.py b/backend/tests/test_recipe_acceptance.py new file mode 100644 index 00000000..8e7663a7 --- /dev/null +++ b/backend/tests/test_recipe_acceptance.py @@ -0,0 +1,903 @@ +"""期 2 集成验收（WP-7，P2-SPEC 9.1 / 9.2 / 9.3）：真管线（default_pipeline）经接口走完整流程。 + +不用假 Pipeline：暂存、规则起草、回答问题、试运行、差异卡、确认项、提交全部是各工作包合并后的真代码， +接口一律经 AsyncClient(transport=ASGITransport(app=app))。夹具全部合成、假名（tests/fixtures/xlsx）； +不调用任何真实模型（这几组用例都走规则起草，用不到 AI）。 + +- 9.1：flow_workbook(2026-08-01, 31 天) 走 暂存 → 规则起草 → 回答问题 → 改宽表名 → 试运行，逐项对 FLOW_EXPECT。 +- 9.2：31 例漂移（30 例 + D28b）逐例参数化。每例先把 D00 首次导入并启用（D00 的已确认配方就是现行配方、 + D00 的导入就是上一期），再把该例文件作为「上传新一期」重放，断言试运行状态、行数、差异卡 kind 的精确集合、 + 确认项的精确集合、拒收原因与坐标、说明片段，以及能否提交。 +- 9.3：实验室用例 c01 / c04 / c05 / c06 / c08 / c10 的各变体。 + +断言照规格写；跑不过的保留失败，由验收报告逐条诊断，不放宽、不标 xfail。 +""" +from __future__ import annotations + +import json +import re +import sqlite3 +import uuid +from contextlib import closing +from pathlib import Path +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.core import artifact_store +from app.core.config import settings +from app.data import recipe, recipe_parsers +from app.data.recipe_types import prose_problems +from app.db.base import SessionLocal +from app.db.models import DataSource, SourceSnapshot, TableImport +from app.main import app +from tests.fixtures.xlsx import lab +from tests.fixtures.xlsx.drift import AUG, DRIFT_CASES, DriftCase +from tests.fixtures.xlsx.flow import FLOW_EXPECT, flow_workbook + +XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +REF_RECIPE = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text("utf-8")) +#: 基线和各例共用的随机种子：D00 例「同一文件再传」要求基线与它字节相同 +SEED = 0 +SHEET = "客流汇总" +#: 9.2 的 expect → TrialOut.status。确认项不影响状态（5.3），所以「需要确认」的各例试运行是 passed +TRIAL_STATUS = {"passed": "passed", "confirm": "passed", "rejected": "rejected", "decision": "needs_decision"} +#: 2.4 末尾实测的 derive_tables 结果：表 → [(列名, SQLite 类型)]，以及主键 +REF_COLUMNS = { + "日客流": [("日期", "TEXT"), ("全日客流", "INTEGER"), ("分区甲", "INTEGER"), ("分区乙", "INTEGER")], + "时段客流": [("日期", "TEXT"), ("时段", "TEXT"), ("时段类别", "TEXT"), ("起始小时", "INTEGER"), + ("结束小时", "INTEGER"), ("客流", "INTEGER")], + "时段客流_表内合计": [("日期", "TEXT"), ("合计项", "TEXT"), ("起始小时", "INTEGER"), ("结束小时", "INTEGER"), + ("客流", "INTEGER")], +} +REF_GRAIN = {"日客流": ["日期"], "时段客流": ["日期", "时段"], "时段客流_表内合计": ["日期", "合计项"]} +TABLE_ORDER = ("日客流", "时段客流", "时段客流_表内合计") + + +# ========================================================================== +# 夹具与小工具 +# ========================================================================== + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + """每个测试一个独立的数据目录：原件存档、构建库、试运行库、工件互不串。""" + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +@pytest.fixture(autouse=True) +def real_pipeline_no_model(monkeypatch): + """验收必须跑真管线：确认 recipe_imports.pipeline() 接的是各工作包的真函数（不是别的测试留下的假实现）。 + + 规则起草不完整的用例（c04、c05、c06）在 StagingOut 里会查 AI 可用性；这里把模型解析换成「不可用」的桩， + 测试里连模型对象都不建，更不会发请求。 + """ + from app.data import recipe_ai, recipe_engine, recipe_imports, recipe_suggest + from app.data.recipe_types import AiAvailability + + p = recipe_imports.pipeline() + assert p.execute is recipe_engine.execute and p.draft is recipe_suggest.draft + assert p.ai_availability is recipe_ai.ai_availability + + async def no_model(session): + return None, "", AiAvailability(False, reason="验收测试不接模型") + + monkeypatch.setattr(recipe_ai, "resolve_draft_model", no_model) + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def unique(prefix: str = "wp7a") -> str: + return f"{prefix}_{uuid.uuid4().hex[:10]}" + + +async def get(model, key): + async with SessionLocal() as session: + return await session.get(model, key) + + +async def stage(client, raw: bytes, filename: str, name: str | None = None) -> dict[str, Any]: + resp = await client.post("/api/datasources/imports/stage", files={"file": (filename, raw, XLSX)}, + data={"name": name or unique()}) + assert resp.status_code == 201, resp.text + return resp.json() + + +async def trial(client, sid: str, **body) -> dict[str, Any]: + resp = await client.post(f"/api/datasources/imports/{sid}/trial", json=body) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def answer(client, sid: str, answers: dict[str, dict[str, Any]]) -> dict[str, Any]: + resp = await client.post(f"/api/datasources/imports/{sid}/answers", json={"answers": answers}) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def put_recipe(client, sid: str, rec: dict[str, Any]) -> dict[str, Any]: + resp = await client.put(f"/api/datasources/imports/{sid}/recipe", json={"recipe": rec}) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def commit(client, st: dict[str, Any], confirmations: list[str] | None = None, + acceptances: list[dict[str, str]] | None = None): + """提交；返回原始响应（调用方自己判状态码）。confirmations 缺省 = 勾上全部确认项。""" + body: dict[str, Any] = {"trial_id": st["trial"]["trial_id"], + "confirmations": confirm_ids(st) if confirmations is None else confirmations} + if acceptances is not None: + body["acceptances"] = acceptances + return await client.post(f"/api/datasources/imports/{st['id']}/commit", json=body) + + +async def reupload(client, source_id: str, raw: bytes, filename: str) -> dict[str, Any]: + resp = await client.post(f"/api/datasources/{source_id}/reupload", files={"file": (filename, raw, XLSX)}) + assert resp.status_code == 201, resp.text + return resp.json() + + +def confirm_ids(st: dict[str, Any]) -> list[str]: + return [i["id"] for i in st["trial"]["confirm_items"]] + + +def check(st: dict[str, Any], cid: str) -> dict[str, Any] | None: + return next((c for c in st["trial"]["checks"] if c["id"] == cid), None) + + +def problems(st: dict[str, Any], code: str | None = None) -> list[dict[str, Any]]: + return [p for p in st["trial"]["problems"] if code is None or p["code"] == code] + + +def rows_of(st: dict[str, Any]) -> dict[str, int]: + return {t["name"]: t["rows"] for t in st["trial"]["receipt"]["tables"]} + + +def all_comments(st: dict[str, Any]) -> str: + """试运行的说明预览里全部表说明和列说明拼在一起（找片段用）。""" + notes = st["trial"]["notes"] or {} + parts = [] + for n in notes.values(): + parts.append(n.get("comment") or "") + parts += list((n.get("columns") or {}).values()) + return "\n".join(parts) + + +def match_confirms(actual: list[str], expected: tuple[str, ...] | frozenset[str]) -> tuple[list[str], list[str]]: + """确认项的精确集合比较；期望里带 * 的按前缀。返回 (多出来的, 缺少的)。""" + def hit(item: str, pattern: str) -> bool: + return item.startswith(pattern[:-1]) if pattern.endswith("*") else item == pattern + + extra = [a for a in actual if not any(hit(a, p) for p in expected)] + missing = [p for p in expected if not any(hit(a, p) for a in actual)] + return extra, missing + + +def rename_wide(rec: dict[str, Any], old: str, new: str) -> dict[str, Any]: + """把规则起草的宽表名改成 new：表名、写入它的分段（id 一起改）、关系里引用的表名（9.7 第 1 步）。""" + rec = json.loads(json.dumps(rec, ensure_ascii=False)) + for sheet in rec["sheets"]: + for block in sheet["blocks"]: + for seg in block.get("segments", []): + if seg.get("table") == old: + seg["table"] = new + if seg.get("id") == old: + seg["id"] = new + for t in rec["tables"]: + if t["name"] == old: + t["name"] = new + for r in rec["relations"]: + if r.get("table") == old: + r["table"] = new + for side in ("a", "b"): + if isinstance(r.get(side), dict) and r[side].get("table") == old: + r[side]["table"] = new + return rec + + +def sql(path: str, query: str, params: tuple[Any, ...] = ()) -> list[tuple[Any, ...]]: + """只读打开库文件查一句（不改文件，库哈希不受影响）。""" + with closing(sqlite3.connect(f"file:{path}?mode=ro", uri=True)) as conn: + return list(conn.execute(query, params)) + + +async def snapshot_db(snapshot_id: str) -> str: + snap = await get(SourceSnapshot, snapshot_id) + assert snap is not None and snap.db_path, snapshot_id + return snap.db_path + + +async def current_snapshot(source_id: str) -> str | None: + return (await get(DataSource, source_id)).current_snapshot_id + + +async def schema_of(client, source_id: str, table: str) -> dict[str, Any]: + resp = await client.get(f"/api/datasources/{source_id}/schema", params={"table": table}) + assert resp.status_code == 200, resp.text + return resp.json() + + +# ========================================================================== +# D00：暂存 → 规则起草 → 回答问题 → 改宽表名 → 试运行（→ 提交） +# ========================================================================== + + +async def flow_trial(client, seed: int = SEED, name: str | None = None) -> dict[str, Any]: + """D00 走到试运行，返回 {"staged": 暂存时的 StagingOut, "st": 试运行后的 StagingOut}。 + + 回答：系统发现一律「登记」，占位符「·」选「存为空值」（参考配方的写法）；然后把规则起草的宽表名 + （「客流汇总_按日」）改成参考配方的「日客流」。 + """ + raw, fn = flow_workbook(AUG, 31, seed=seed) + staged = await stage(client, raw, fn, name) + answers = {q["id"]: {"value": "register" if q["id"].startswith("q_relation") else "null"} + for q in staged["questions"]} + after = await answer(client, staged["id"], answers) + wide = next(t["name"] for t in after["recipe"]["tables"] if t["name"].endswith("_按日")) + await put_recipe(client, staged["id"], rename_wide(after["recipe"], wide, "日客流")) + st = await trial(client, staged["id"]) + return {"staged": staged, "st": st, "raw": raw, "file_name": fn} + + +async def baseline(client, seed: int = SEED) -> dict[str, Any]: + """D00 首次导入并启用（勾全部确认项）。返回 flow_trial 的结果加上提交响应。""" + done = await flow_trial(client, seed) + assert done["st"]["trial"]["status"] == "passed", done["st"]["trial"]["problems"] + resp = await commit(client, done["st"]) + assert resp.status_code == 201, resp.text + done["commit"] = resp.json() + done["source_id"] = done["commit"]["source"]["id"] + return done + + +# ========================================================================== +# 9.1 合成「真实结构」夹具：D00 的期望（FLOW_EXPECT） +# ========================================================================== + + +async def test_9_1_rules_draft_answers_and_rename_give_the_reference_recipe(client): + """规则起草完整；回答问题、改宽表名之后，工作配方与参考配方 recipe_sha256 相同（9.1 的期望以参考配方为前提）。""" + raw, fn = flow_workbook(AUG, 31, seed=SEED) + staged = await stage(client, raw, fn) + assert staged["draft"]["complete"] is True, staged["draft"]["failures"] + assert staged["recipe_origin"] == "rules" and staged["recipe_problems"] == [] + assert len(staged["cards"]) == 10, [c["id"] for c in staged["cards"]] + qids = {q["id"] for q in staged["questions"]} + assert {"q_placeholder:·", "q_relation:F1", "q_relation:F2"} <= qids + # 6.1 第 9、10 条：占位符和系统发现的问题都没有默认值，必须由人选 + assert all(q["default"] is None for q in staged["questions"] if q["id"] in + ("q_placeholder:·", "q_relation:F1", "q_relation:F2")) + done = await flow_trial(client) + st = done["st"] + assert st["recipe_problems"] == [] + ref, ref_problems = recipe.validate_recipe(REF_RECIPE, origin="manual") + mine, mine_problems = recipe.validate_recipe(st["recipe"], origin="manual") + assert ref_problems == [] and mine_problems == [] + assert recipe.recipe_sha256(mine) == recipe.recipe_sha256(ref) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + + +async def test_9_1_sheet_scan_and_cell_ledger(client): + """格子账：非空格 771 = 各去向之和；未认领 0；两遍一致（扫描数 = 读取数）。""" + st = (await flow_trial(client))["st"] + sheet = next(s for s in st["sheets"] if s["name"] == SHEET) + assert sheet["bounds"] == FLOW_EXPECT["bounds"] + assert sheet["nonempty"] == FLOW_EXPECT["nonempty"] + assert sheet["merged"] == FLOW_EXPECT["merged"] + assert sheet["formulas"] == FLOW_EXPECT["formulas"] and sheet["formulas_uncached"] == 0 + ledger = st["trial"]["receipt"]["ledger"] + assert [x["sheet"] for x in ledger] == [SHEET] + led = ledger[0] + assert led["nonempty_scan"] == FLOW_EXPECT["nonempty"] + assert led["nonempty_read"] == led["nonempty_scan"], "两遍对账一致" + assert led["unclaimed"] == 0 + assert {k: v for k, v in led["roles"].items() if v} == FLOW_EXPECT["ledger"] + assert sum(led["roles"].values()) == FLOW_EXPECT["nonempty"] + + +async def test_9_1_tables_columns_types_rows_strict_and_primary_key(client): + done = await flow_trial(client) + st = done["st"] + tables = {t["name"]: t for t in st["trial"]["receipt"]["tables"]} + assert list(tables) == list(TABLE_ORDER) + assert rows_of(st) == FLOW_EXPECT["rows"] + for name, cols in REF_COLUMNS.items(): + assert [(c["name"], c["type"]) for c in tables[name]["columns"]] == cols, name + assert tables[name]["grain"] == REF_GRAIN[name], name + assert tables["时段客流_表内合计"]["kind"] == "reported_total" + assert tables["日客流"]["kind"] == tables["时段客流"]["kind"] == "data" + + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + db = await snapshot_db(resp.json()["snapshot_id"]) + for name, cols in REF_COLUMNS.items(): + ddl = sql(db, "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", (name,))[0][0] + assert re.search(r"\)\s*STRICT\s*$", ddl), ddl + assert "PRIMARY KEY" in ddl.upper(), ddl + info = sql(db, f'PRAGMA table_info("{name}")') + assert [(r[1], r[2]) for r in info] == cols, name + assert sorted((r[5], r[1]) for r in info if r[5]) == list(enumerate(REF_GRAIN[name], start=1)), name + assert sql(db, f'SELECT COUNT(*) FROM "{name}"')[0][0] == FLOW_EXPECT["rows"][name] + + +async def test_9_1_placeholders_keep_null_rows(client): + """占位符 {"·": 31}；时段='7-8' 的 31 行客流为 NULL（D11：整期无数据的行保留为空值行）。""" + done = await flow_trial(client) + st = done["st"] + assert st["trial"]["receipt"]["placeholders"] == FLOW_EXPECT["placeholders"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + db = await snapshot_db(resp.json()["snapshot_id"]) + rows = sql(db, 'SELECT "时段类别", "起始小时", "结束小时", "客流" FROM "时段客流" WHERE "时段" = ?', ("7-8",)) + assert len(rows) == 31 + assert {r[:3] for r in rows} == {("日间", 7, 8)} + assert all(r[3] is None for r in rows) + assert sql(db, 'SELECT COUNT(*) FROM "时段客流" WHERE "客流" IS NULL')[0][0] == 31 + + +async def test_9_1_checks(client): + """C1 passed（一处）、C2 passed；K1 passed（3 × 31 格）；G1 passed；R1 passed（31/31）；R2 info（31 天中 0 天相等）； + N1–N3、P1–P3 passed。""" + st = (await flow_trial(client))["st"] + assert {c["id"]: c["status"] for c in st["trial"]["checks"]} == FLOW_EXPECT["checks"] + c1 = check(st, "C1") + assert c1["cells"] == FLOW_EXPECT["period_cells"] + k1 = check(st, "K1") + assert k1["checked"] == 3 * 31 and k1["failed"] == 0 and k1["unverifiable"] == 0 + assert k1["category"] == "structure" + g1 = check(st, "G1") + assert g1["checked"] == 3 * 31 and g1["failed"] == 0 + r1 = check(st, "R1") + assert r1["checked"] == 31 and r1["failed"] == 0 and r1["unverifiable"] == 0 + assert r1["category"] == "data_quality" + r2 = check(st, "R2") + assert r2["category"] == "info" + assert any(f"31 天中 {FLOW_EXPECT['r2_equal_days']} 天相等" in d for d in r2["details"]), r2["details"] + # 每条 SQL 核对的原文进 CheckResult（5.1）；C1、C2 来自执行器的统计期判定，不是 SQL + for c in st["trial"]["checks"]: + if c["id"] not in ("C1", "C2"): + assert c.get("sql"), c["id"] + assert st["trial"]["acceptable"] == [] + + +async def test_9_1_canonicalized_outside_text_and_period(client): + """规范写法对照 0 条（合计标签的写法记在 SegmentLabels）；区域外文字只有 B3（text）；PeriodOut.texts 只有 B2、 + annotated 为空。""" + st = (await flow_trial(client))["st"] + rec = st["trial"]["receipt"] + assert rec["canonicalized_total"] == FLOW_EXPECT["canonicalized"] and rec["canonicalized"] == [] + totals = next(x for x in rec["labels"] if x["segment"] == "夜间合计") + assert totals["raw"] == ["18-22 时合计", "22-24 时合计", "18-24 时合计"] + assert totals["canonical"] == ["18-22时合计", "22-24时合计", "18-24时合计"] + assert [(o["cell"], o["text"], o["kind"]) for o in rec["outside_text"]] == FLOW_EXPECT["outside_text"] + period = rec["period"] + assert period["source"] == "cells" + assert (period["start"], period["end"]) == ("2026-08-01", "2026-08-31") + assert period["cells"] == FLOW_EXPECT["period_cells"] + assert list(period["texts"]) == FLOW_EXPECT["period_cells"] + assert period["annotated"] == {} + + +async def test_9_1_confirm_items_exact(client): + st = (await flow_trial(client))["st"] + ids = confirm_ids(st) + assert len(ids) == len(set(ids)), ids + assert set(ids) == FLOW_EXPECT["confirms"] + assert all(i["required"] for i in st["trial"]["confirm_items"]) + + +async def test_9_1_notes(client): + """三张表的说明（模板与渲染后的文字）过 prose_problems(known=…)；日客流含「已逐日核对」；都不含 E7 那句。 + 提交后表结构里的说明照样，列「全日客流」的说明是「单位：人次」。""" + st = (await flow_trial(client))["st"] + notes = st["trial"]["notes"] + assert set(notes) == set(TABLE_ORDER) + for frag in FLOW_EXPECT["note_has"]: + assert frag in notes["日客流"]["comment"], notes["日客流"]["comment"] + for name in TABLE_ORDER: + for frag in FLOW_EXPECT["note_lacks"]: + assert frag not in notes[name]["comment"], (name, notes[name]["comment"]) + assert notes["日客流"]["columns"]["全日客流"] == "单位：人次" + # 5.4 按核对结果选模板：K1、G1 都通过 → 表内合计表写「已按明细重算核对一致」；基表指向表内合计表； + # 值列的空值只来自占位符 → 列说明写占位符的空值含义 + assert "原表写明的合计，已按明细重算核对一致" in notes["时段客流_表内合计"]["comment"] + assert "原表的合计在 时段客流_表内合计，不要与本表相加" in notes["时段客流"]["comment"] + assert "空值表示原表为无数据占位符，不代表零" in notes["时段客流"]["columns"]["客流"] + assert "空值" not in notes["时段客流_表内合计"]["columns"].get("客流", "") + + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + imp = await get(TableImport, out["import_id"]) + manifest = artifact_store.load(imp.manifest_artifact) + known = {"列": {c for cols in REF_COLUMNS.values() for c, _ in cols}, "表": set(TABLE_ORDER), + "单位": set(recipe_parsers.UNITS)} + templates = manifest["notes"]["templates"] + for name in TABLE_ORDER: + assert name in templates, sorted(templates) + assert prose_problems(templates[name], known=known) == [], (name, templates[name]) + assert prose_problems(manifest["notes"]["rendered"][name]["comment"], known=known) == [], name + for key, text in templates.items(): + assert prose_problems(text, known=known) == [], (key, text) + schema = await schema_of(client, out["source"]["id"], "日客流") + assert "已逐日核对" in (schema["comment"] or "") + col = next(c for c in schema["columns"] if c["name"] == "全日客流") + assert col.get("comment") == "单位：人次" + + +# ========================================================================== +# 9.2 31 例漂移：以 D00 的已确认配方为现行配方、D00 的导入为上一期，逐例「上传新一期」 +# ========================================================================== + +#: 拒收各例的 fix（9.2 表「其他关键结果」） +REJECT_FIX = {"D04": {"remove_label"}, "D09": {"rename_title"}, "D13": {"add_label"}, "D16": {"ignore_cells"}, + "D18": {"declare_total"}, "D21": {"add_label", "ignore_cells"}} +#: 拒收各例的问题坐标（按 flow.py 的版式推出：9 月 30 天，日期列 C..AF） +REJECT_CELLS = { + "D13": {"客流汇总!B8"}, # 「分区丙（人次）」在第 8 行 + "D16": {"客流汇总!AG4"}, # 右侧多出的「合计」表头（同 P25 的 AH4） + "D18": {"客流汇总!B21"}, # 日间末尾的「7-18 时合计」 + "D21": {"客流汇总!B32"}, # 第 32 行补录 +} + + +#: 各例的统计期（9.2：9 月 = 2026-09-01 起 30 天，10 月 = 2026-10-01 起；D00 是 8 月 31 天） +PERIOD = {"D00": ("2026-08-01", "2026-08-31"), "D02": ("2026-10-01", "2026-10-31"), + "D03": ("2026-10-01", "2026-10-15"), "D03b": ("2026-10-01", "2026-10-07")} +SEP_PERIOD = ("2026-09-01", "2026-09-30") + + +def _check_id(code: str) -> bool: + return re.fullmatch(r"[A-Z]\d+", code) is not None + + +async def _extra_D00(client, ctx): + """同一文件再传：same_as_import 指向基线导入，提交 unchanged、返回原导入。""" + st = ctx["st"] + assert st["trial"]["same_as_import"] is not None + assert st["trial"]["same_as_import"]["id"] == ctx["base"]["commit"]["import_id"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["unchanged"] is True and out["import_id"] == ctx["base"]["commit"]["import_id"] + assert out["snapshot_id"] == ctx["base"]["commit"]["snapshot_id"] + return True + + +async def _extra_D02(client, ctx): + assert rows_of(ctx["st"]) == FLOW_EXPECT["rows"], "行数与 D00 相同" + + +async def _annotated_title(ctx, cell: str = f"{SHEET}!B3", title: str = "客流汇总表"): + period = ctx["st"]["trial"]["receipt"]["period"] + assert cell in period["annotated"], period + assert recipe_parsers.match_key(period["annotated"][cell]) == recipe_parsers.match_key(title) + assert cell in period["texts"], period + out = next((o for o in ctx["st"]["trial"]["receipt"]["outside_text"] if o["cell"] == cell), None) + assert out is not None and out["kind"] == "text_digits" and out["period_source"] is True, out + assert check(ctx["st"], "C1")["status"] == "passed" + + +async def _extra_D17(client, ctx): + await _annotated_title(ctx) + + +async def _extra_D27(client, ctx): + await _annotated_title(ctx) + c1 = check(ctx["st"], "C1") + assert {f"{SHEET}!B2", f"{SHEET}!B3"} <= set(c1["cells"]), c1 + + +async def _extra_D23(client, ctx): + assert check(ctx["st"], "C2")["status"] == "info" + + +async def _extra_D25(client, ctx): + k1 = check(ctx["st"], "K1") + assert k1["status"] == "passed" + assert any("保存值与重算一致" in d for d in k1["details"]), k1["details"] + assert ctx["st"]["trial"]["receipt"]["full_calc_on_load"] is True + + +async def _extra_D07(client, ctx): + assert check(ctx["st"], "K1")["status"] == "passed" + assert check(ctx["st"], "G1") is None, "合计写死：没有 G1" + assert ctx["st"]["trial"]["receipt"]["derived_form"].get("夜间合计") == "literal" + + +async def _extra_D11(client, ctx): + labels = next(x for x in ctx["st"]["trial"]["receipt"]["labels"] if x["segment"] == "日间") + assert labels["raw"][0] == "07:00-08:00" and labels["canonical"][0] == "7-8" + assert not [i for i in confirm_ids(ctx["st"]) if i.startswith("diff:canon:")] + resp = await commit(client, ctx["st"]) + assert resp.status_code == 201, resp.text + db = await snapshot_db(resp.json()["snapshot_id"]) + got = {r[0] for r in sql(db, 'SELECT DISTINCT "时段" FROM "时段客流"')} + assert "7-8" in got and all(re.fullmatch(r"\d{1,2}-\d{1,2}", x) for x in got), got + return True + + +async def _extra_D22(client, ctx): + labels = next(x for x in ctx["st"]["trial"]["receipt"]["labels"] if x["segment"] == "夜间合计") + assert labels["canonical"] == ["18-22时合计", "22-24时合计", "18-24时合计"] + resp = await commit(client, ctx["st"]) + assert resp.status_code == 201, resp.text + db = await snapshot_db(resp.json()["snapshot_id"]) + got = {r[0] for r in sql(db, 'SELECT DISTINCT "合计项" FROM "时段客流_表内合计"')} + assert got == {"18-22时合计", "22-24时合计", "18-24时合计"} + return True + + +async def _extra_D20(client, ctx): + assert ctx["st"]["trial"]["receipt"]["sheets"]["other_visible"] == ["说明"] + + +async def _extra_D28(client, ctx): + out = next((o for o in ctx["st"]["trial"]["receipt"]["outside_text"] if o["cell"] == f"{SHEET}!B32"), None) + assert out is not None and out["kind"] == "text_digits", out + + +async def _extra_D28b(client, ctx): + period = ctx["st"]["trial"]["receipt"]["period"] + assert f"{SHEET}!B32" in period["annotated"], period + c1 = check(ctx["st"], "C1") + assert c1["status"] == "passed" and {f"{SHEET}!B2", f"{SHEET}!B32"} <= set(c1["cells"]), c1 + + +async def _extra_D08(client, ctx): + k1 = check(ctx["st"], "K1") + assert k1["status"] == "mismatch" and k1["category"] == "structure" + assert any(d.startswith("G28") for d in k1["details"]), k1["details"] + + +async def _extra_D26(client, ctx): + """R1 mismatch（data_quality、可接受）；不带接受 → 422；写理由接受 → 201，说明改写。""" + st = ctx["st"] + r1 = check(st, "R1") + assert r1["status"] == "mismatch" and r1["category"] == "data_quality" and r1["acceptable"] is True + assert st["trial"]["acceptable"] == ["R1"] + text = " ".join(r1["details"] + r1["cells"]) + assert "2026-09-10" in text and "L5" in text, r1 + resp = await commit(client, st) + assert resp.status_code == 422 and resp.json()["code"] == "acceptance_required", resp.text + resp = await commit(client, st, acceptances=[{"check_id": "R1", "reason": "合成理由：第 10 天的全日数据为人工补录"}]) + assert resp.status_code == 201, resp.text + out = resp.json() + imp = await get(TableImport, out["import_id"]) + assert [o["check_id"] for o in imp.overrides or []] == ["R1"] + comment = (await schema_of(client, ctx["source_id"], "日客流"))["comment"] or "" + for frag in ctx["case"].note_has: + assert frag in comment, comment + for frag in ctx["case"].note_lacks: + assert frag not in comment, comment + return True + + +#: 各例「其他关键结果」。返回 True 的表示已自己提交过 +EXTRA = {"D00": _extra_D00, "D02": _extra_D02, "D17": _extra_D17, "D27": _extra_D27, "D23": _extra_D23, + "D25": _extra_D25, "D07": _extra_D07, "D11": _extra_D11, "D22": _extra_D22, "D20": _extra_D20, + "D28": _extra_D28, "D28b": _extra_D28b, "D08": _extra_D08, "D26": _extra_D26} + + +@pytest.mark.parametrize("code", list(DRIFT_CASES)) +async def test_9_2_drift_case(client, code): + case: DriftCase = DRIFT_CASES[code] + base = await baseline(client) + source_id = base["source_id"] + snap_before = await current_snapshot(source_id) + assert snap_before == base["commit"]["snapshot_id"] + raw, fn = case.build(SEED) + st = await reupload(client, source_id, raw, fn) + tr = st["trial"] + # 上传新一期按现行配方重放、自动试运行，不起草、不提供 AI（7.1、7.4） + assert st["kind"] == "reupload" and st["draft"] is None and st["ai"]["offered"] is False + assert tr["base_snapshot_id"] == snap_before + + # 试运行状态 + assert tr["status"] == TRIAL_STATUS[case.expect], (tr["status"], tr["problems"], + [c for c in tr["checks"] if c["status"] != "passed"]) + # 行数、统计期 + if case.rows is not None: + assert tuple(rows_of(st)[t] for t in TABLE_ORDER) == case.rows + period = tr["receipt"]["period"] + assert (period["start"], period["end"]) == PERIOD.get(code, SEP_PERIOD), period + assert period["source"] == "cells" + if code != "D00": + assert tr["same_as_import"] is None + # 差异卡 kind 的精确集合（拒收时没有差异卡） + if case.diff_kinds is None: + assert tr["diff"] is None + else: + assert tr["diff"] is not None + assert {d["kind"] for d in tr["diff"]} == set(case.diff_kinds), [(d["kind"], d["label"]) for d in tr["diff"]] + for d in tr["diff"]: + if d["requires_confirm"]: + assert d["confirm_id"] and d["confirm_id"] in confirm_ids(st), d + # 确认项的精确集合（配方没变：只有本期类和 diff:*） + extra, missing = match_confirms(confirm_ids(st), case.confirms) + assert not extra and not missing, {"多出": extra, "缺少": missing} + assert all(i["required"] for i in tr["confirm_items"]), tr["confirm_items"] + # 拒收原因 / 核对 id + for c in case.codes: + if _check_id(c): + chk = check(st, c) + assert chk is not None and chk["status"] != "passed", (c, chk) + else: + assert problems(st, c), (c, [(p["code"], p["cells"]) for p in tr["problems"]]) + if case.expect == "rejected": + structural = {p["code"] for p in tr["problems"] if p["category"] in ("structure", "recipe")} + assert structural == {c for c in case.codes if not _check_id(c)}, structural + if code in REJECT_FIX: + fixes = {p["fix"] for p in problems(st, case.codes[0])} + assert fixes and fixes <= REJECT_FIX[code], fixes + if code in REJECT_CELLS: + cells = {c for p in problems(st, case.codes[0]) for c in p["cells"]} + assert REJECT_CELLS[code] <= cells, cells + if code == "D21": + assert all(c.startswith(f"{SHEET}!") and c.endswith("32") for c in cells), cells + # 说明片段（D26 的片段是接受之后重新生成的，在 _extra_D26 里查） + if case.expect != "decision": + comments = all_comments(st) + for frag in case.note_has: + assert frag in comments, (frag, st["trial"]["notes"]) + for frag in case.note_lacks: + assert frag not in comments, (frag, st["trial"]["notes"]) + + committed = False + if code in EXTRA: + committed = bool(await EXTRA[code](client, {"st": st, "base": base, "source_id": source_id, "case": case})) + + # 能否提交 + if case.expect == "rejected": + assert st["status"] == "rejected" + resp = await commit(client, st) + assert resp.status_code == 409 and resp.json()["code"] == "trial_not_passed", resp.text + assert await current_snapshot(source_id) == snap_before, "拒收不动当前快照" + elif not committed: + if case.confirms: + resp = await commit(client, st, confirmations=[]) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required", resp.text + for item in confirm_ids(st): + assert item in resp.json()["detail"], (item, resp.json()["detail"]) + assert await current_snapshot(source_id) == snap_before + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["unchanged"] is False and out["recipe_id"] == base["commit"]["recipe_id"] + assert await current_snapshot(source_id) == out["snapshot_id"] != snap_before + + +# ========================================================================== +# 9.3 实验室难缠用例 +# ========================================================================== + + +def blocks_of(rec: dict[str, Any]) -> list[dict[str, Any]]: + return [b for s in rec["sheets"] for b in s["blocks"]] + + +async def test_9_3_c01_literal_total(client): + """写死合计：列表块 + total_row(pick=合计, keep_as)；passed；T1 passed；确认项含 outside_digits ×2（C1、C3）。""" + raw, fn = lab.c01("literal") + staged = await stage(client, raw, fn) + blocks = blocks_of(staged["recipe"]) + assert [b["layout"] for b in blocks] == ["list"] + total = blocks[0]["rows"]["total_row"] + assert total is not None and total["pick"] == "合计" and total["keep_as"], total + st = await trial(client, staged["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert check(st, "T1")["status"] == "passed" + outside = sorted(i for i in confirm_ids(st) if i.startswith("outside_digits:")) + assert outside == ["outside_digits:月报!C1", "outside_digits:月报!C3"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + + +async def test_9_3_c01_formula_without_cache(client): + """公式无缓存：needs_decision，T1 unverifiable（可接受）；接受后提交，表内合计值为 NULL，说明写「未能核对」。""" + raw, fn = lab.c01("uncached") + staged = await stage(client, raw, fn) + st = await trial(client, staged["id"]) + tr = st["trial"] + assert tr["status"] == "needs_decision", tr["problems"] + t1 = check(st, "T1") + assert t1["status"] == "unverifiable" and t1["acceptable"] is True + assert t1["reasons"].get("uncached"), t1 + assert tr["acceptable"] == ["T1"] + resp = await commit(client, st) + assert resp.status_code == 422 and resp.json()["code"] == "acceptance_required", resp.text + resp = await commit(client, st, acceptances=[{"check_id": "T1", "reason": "合成理由：导出工具不保存计算结果"}]) + assert resp.status_code == 201, resp.text + out = resp.json() + total_table = blocks_of(staged["recipe"])[0]["rows"]["total_row"]["keep_as"] + db = await snapshot_db(out["snapshot_id"]) + rows = sql(db, f'SELECT "销量", "金额" FROM "{total_table}"') + assert rows and all(v is None for r in rows for v in r), rows + imp = await get(TableImport, out["import_id"]) + assert [w["check_id"] for w in imp.waivers or []] == ["T1"] + comment = (await schema_of(client, out["source"]["id"], total_table))["comment"] or "" + assert "未能核对" in comment and "一致" not in comment, comment + + +async def test_9_3_c01_formula_with_cache(client): + raw, fn = lab.c01("cached") + staged = await stage(client, raw, fn) + st = await trial(client, staged["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert check(st, "T1")["status"] == "passed" + + +def _option_with(question: dict[str, Any], value: str) -> str: + """选项里 effects 把某个字段设成 value 的那个选项值。""" + for opt in question["options"]: + if value in json.dumps(question.get("effects", {}).get(opt["value"], []), ensure_ascii=False): + return opt["value"] + raise AssertionError(f"问题 {question['id']} 没有设成「{value}」的选项：{question['options']}") + + +async def test_9_3_c04_merged_cells(client): + """合并单元格：header_rows=2，列表头「2026年上半年_销量」等；merged_data=fill 的问题；回答后 passed，确认项 + merged_fill:*；卡片提示表头含年份。""" + raw, fn = lab.c04() + staged = await stage(client, raw, fn) + assert staged["recipe"] is not None, staged["draft"]["failures"] + blocks = blocks_of(staged["recipe"]) + assert [b["layout"] for b in blocks] == ["list"] + block = blocks[0] + assert block["header_rows"] == 2 + headers = [c["header"] for c in block["columns"]] + assert headers == ["地区", "产品", "2026年上半年_销量", "2026年上半年_金额", "2026年下半年_销量", + "2026年下半年_金额"], headers + assert block["merged_data"] == "fill" + assert any("年份" in (c["title"] + c["reason"]) for c in staged["cards"]), staged["cards"] + merged_q = [q for q in staged["questions"] if "fill" in json.dumps(q.get("effects", {}), ensure_ascii=False)] + assert merged_q, [q["id"] for q in staged["questions"]] + st = await answer(client, staged["id"], {merged_q[0]["id"]: {"value": _option_with(merged_q[0], "fill")}}) + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert [i for i in confirm_ids(st) if i.startswith("merged_fill:")], confirm_ids(st) + assert rows_of(st)[block["table"]] == 6 + + +@pytest.mark.parametrize(("variant", "status", "code"), [ + ("openpyxl", "rejected", "formula_uncached"), + ("xlsxwriter", "rejected", "formula_full_calc"), + ("有缓存", "passed", None), +]) +async def test_9_3_c05_formulas(client, variant, status, code): + raw, fn = lab.c05(variant) + staged = await stage(client, raw, fn) + st = await trial(client, staged["id"]) + tr = st["trial"] + assert tr["status"] == status, tr["problems"] + if code is not None: + assert problems(st, code), [(p["code"], p["cells"]) for p in tr["problems"]] + assert problems(st, code)[0]["category"] == "structure" + else: + assert tr["receipt"]["formula_cells_accepted"] == 3 + assert [i for i in confirm_ids(st) if i.startswith("formula_cached:")], confirm_ids(st) + + +async def test_9_3_c06_hidden_rows_cols_and_sheets(client): + """默认 rejected：hidden_rows（提示可能是筛选）、hidden_cols；隐藏工作表列在 skipped_hidden 且不读； + 回答 include → passed；rows=exclude → passed、行数减少、排除的行列在回执。""" + raw, fn = lab.c06() + staged = await stage(client, raw, fn) + assert [g["sheet"] for g in staged["grids"]] == ["明细"], "隐藏工作表不读" + assert {(s["sheet"], s["state"]) for s in staged["skipped_sheets"]} == {("草稿", "hidden"), ("配置", "veryHidden")} + st = await trial(client, staged["id"]) + tr = st["trial"] + assert tr["status"] == "rejected" + hr = problems(st, "hidden_rows") + assert hr and "筛选" in hr[0]["message"], hr + assert problems(st, "hidden_cols") + skipped = tr["receipt"]["sheets"]["skipped_hidden"] + assert {(s["sheet"], s["state"]) for s in skipped} == {("草稿", "hidden"), ("配置", "veryHidden")} + assert [x["sheet"] for x in tr["receipt"]["ledger"]] == ["明细"] + + q = next(q for q in staged["questions"] if q["id"].startswith("q_hidden:")) + assert q["default"] is None, "6.1 第 12 条：隐藏行列的问题没有默认值" + st = await answer(client, staged["id"], {q["id"]: {"value": "include"}}) + st = await trial(client, st["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + full_rows = rows_of(st)["明细"] + assert full_rows == 10 + assert "hidden:明细" in confirm_ids(st) + + st = await answer(client, staged["id"], {q["id"]: {"value": "exclude"}}) + assert st["recipe"]["sheets"][0]["hidden"]["rows"] == "exclude" + st = await trial(client, st["id"]) + tr = st["trial"] + assert tr["status"] == "passed", tr["problems"] + assert rows_of(st)["明细"] == full_rows - 6 < full_rows + hidden = tr["receipt"]["hidden"]["明细"] + assert hidden["rows"] == [3, 4, 5, 7, 9, 11] and hidden["cols"] == ["C"], hidden + led = tr["receipt"]["ledger"][0] + assert led["roles"].get("hidden_excluded") == 6 * 4, led + assert "hidden:明细" in confirm_ids(st) + + +async def test_9_3_c08_stacked_tables(client): + """上下两表：两个列表块；passed；标题是区域外文字。""" + raw, fn = lab.c08("stacked") + staged = await stage(client, raw, fn) + assert [b["layout"] for b in blocks_of(staged["recipe"])] == ["list", "list"] + st = await trial(client, staged["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + outside = {(o["cell"], o["text"], o["kind"]) for o in st["trial"]["receipt"]["outside_text"]} + assert {("汇总!B1", "一、销售", "text"), ("汇总!B7", "二、费用", "text")} <= outside + assert sorted(rows_of(st).values()) == [3, 3] + + +async def test_9_3_c08_blank_row_in_the_middle(client): + """中间空行：起草 blank_rows=skip + 卡片；passed，blank_rows_skipped=1；改成 stop → rejected（空行下面的数据没有去处）。""" + raw, fn = lab.c08("gap") + staged = await stage(client, raw, fn) + blocks = blocks_of(staged["recipe"]) + assert len(blocks) == 1 and blocks[0]["rows"]["blank_rows"] == "skip" + assert any("空行" in (c["title"] + c["reason"]) for c in staged["cards"]), staged["cards"] + st = await trial(client, staged["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert st["trial"]["receipt"]["blank_rows_skipped"] == 1 + assert rows_of(st)[blocks[0]["table"]] == 4 + + rec = json.loads(json.dumps(staged["recipe"], ensure_ascii=False)) + rec["sheets"][0]["blocks"][0]["rows"]["blank_rows"] = "stop" + st = await put_recipe(client, staged["id"], rec) + st = await trial(client, st["id"]) + tr = st["trial"] + assert tr["status"] == "rejected", tr["problems"] + structural = [p for p in tr["problems"] if p["category"] == "structure"] + cells = {c for p in structural for c in p["cells"]} + assert {"明细!C5", "明细!C6"} <= cells, [(p["code"], p["cells"]) for p in tr["problems"]] + + +async def test_9_3_c08_side_by_side(client): + raw, fn = lab.c08("side") + staged = await stage(client, raw, fn) + assert [b["layout"] for b in blocks_of(staged["recipe"])] == ["list", "list"] + st = await trial(client, staged["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert sorted(rows_of(st).values()) == [2, 2] + + +async def test_9_3_c10_drift(client): + """以 07 起草并启用；08（表头挪到 C7、多「单价」）rejected column_extra；09（表头外观差异）passed + + diff:header_writing:*；10（「金额」改名「销售额」）rejected column_missing + column_extra。""" + raw, fn = lab.c10(7) + staged = await stage(client, raw, fn) + st = await trial(client, staged["id"]) + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + source_id = out["source"]["id"] + + raw, fn = lab.c10(8) + st = await reupload(client, source_id, raw, fn) + assert st["trial"]["status"] == "rejected", st["trial"]["problems"] + assert problems(st, "column_extra"), [(p["code"], p["cells"]) for p in st["trial"]["problems"]] + await client.delete(f"/api/datasources/imports/{st['id']}") + + raw, fn = lab.c10(9) + st = await reupload(client, source_id, raw, fn) + tr = st["trial"] + assert tr["status"] == "passed", tr["problems"] + assert "header_writing" in {d["kind"] for d in tr["diff"]}, tr["diff"] + assert [i for i in confirm_ids(st) if i.startswith("diff:header_writing:")], confirm_ids(st) + await client.delete(f"/api/datasources/imports/{st['id']}") + + raw, fn = lab.c10(10) + st = await reupload(client, source_id, raw, fn) + assert st["trial"]["status"] == "rejected", st["trial"]["problems"] + codes = {p["code"] for p in st["trial"]["problems"]} + assert {"column_missing", "column_extra"} <= codes, codes + assert await current_snapshot(source_id) == out["snapshot_id"] diff --git a/backend/tests/test_recipe_ai.py b/backend/tests/test_recipe_ai.py new file mode 100644 index 00000000..631da85f --- /dev/null +++ b/backend/tests/test_recipe_ai.py @@ -0,0 +1,862 @@ +"""WP-4 AI 兜底起草（recipe_ai）的单元测试（P2-SPEC 9.6 的 WP-4 部分）。 + +不调用任何真实模型：ai_draft 注入假模型（async ainvoke(messages) -> AIMessage）；可用性的用例用独立的内存库建接入， +resolve_draft_model 走到 build_chat_model 时 monkeypatch 成假模型。夹具一律合成、假名。 +""" +from __future__ import annotations + +import asyncio +import datetime as dt +import json +from typing import Any + +import pytest +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage +from openpyxl import Workbook + +from app.data import recipe_ai as A +from app.data import recipe_suggest as S +from app.data.recipe_types import Draft, DraftFacts, Extraction, Fact, Grid, GridCell, Problem, Recipe, RecipeProblem +from tests.test_recipe_suggest import FLOW, build, flow_book, ok_validate + +# ========================================================================== +# 假模型与桩 +# ========================================================================== + + +def reply(content: Any, *, inp: int = 100, out: int = 50, meta: dict[str, Any] | None = None) -> AIMessage: + return AIMessage(content=content, usage_metadata={"input_tokens": inp, "output_tokens": out, + "total_tokens": inp + out}, + response_metadata=meta or {}) + + +class FakeModel: + """按预设顺序返回；元素是异常就抛，是协程函数就 await 它。记下每次收到的消息列表。""" + + def __init__(self, replies: list[Any], name: str = "fake-model") -> None: + self.replies = list(replies) + self.calls: list[list[Any]] = [] + self.model_name = name + + async def ainvoke(self, messages: list[Any]) -> Any: + self.calls.append(list(messages)) + r = self.replies.pop(0) + if isinstance(r, BaseException): + raise r + if callable(r): + return await r() + return r + + +def flow_json(**patch: Any) -> str: + data = json.loads(json.dumps(FLOW)) + data.update(patch) + return json.dumps(data, ensure_ascii=False) + + +def claims_validate(calls: list | None = None): + """静态校验桩：pydantic、AI 的 note 必须为空（ai_note_forbidden）、每条事实恰好被认领一次（fact_unclaimed）。""" + def _v(data: dict[str, Any], facts: DraftFacts | None, origin: str): + if calls is not None: + calls.append((data, facts, origin)) + try: + r = Recipe.model_validate(data) + except Exception as e: # noqa: BLE001 + return None, [RecipeProblem("/", "schema", str(e)[:200])] + probs = [] + if origin == "ai": + for i, t in enumerate(r.tables): + if t.note: + probs.append(RecipeProblem(f"/tables/{i}/note", "ai_note_forbidden", f"表「{t.name}」：AI 起草的配方说明必须为空")) + claimed = [x.claims for x in r.relations if x.claims] + for f in (facts.facts if facts else []): + if claimed.count(f.id) != 1: + probs.append(RecipeProblem("/relations", "fact_unclaimed", f"系统发现的关系「{f.text}」没有被认领")) + return (r if not probs else r), probs + return _v + + +FACTS = DraftFacts(facts=[ + Fact("F1", "sum_eq", "客流汇总", "第 5 行 = 第 6 行 + 第 7 行（31 列中 31 列成立）", + {"segment": "日客流", "total": "全日客流（人次）", "parts": ["分区甲（人次）", "分区乙（人次）"], "rows": [5, 6, 7]}), + Fact("F2", "not_equal_sum", "客流汇总", "日间、夜间各行之和 与 第 5 行：31 列中 0 列相等", + {"a": ["日间", "夜间"], "b_segment": "日客流", "b": "全日客流（人次）", "equal": 0, "checked": 31}), +], candidates={"日间时段客流（人次）": ["日间", "日间时段客流"], "夜间时段客流（人次）": ["夜间", "夜间时段客流"]}) + + +@pytest.fixture(scope="module") +def flow(): + wb, cache, nums = flow_book() + sc, grids = build(wb, cache) + return sc, grids, nums + + +async def run(flow, model, *, validate=None, dry_run=None, rules: Draft | None = None, facts=FACTS, **kw): + sc, grids, _ = flow + return await A.ai_draft(None, grids=grids, scan=sc, facts=facts, rules_draft=rules, filename="x.xlsx", # type: ignore[arg-type] + validate=validate or claims_validate(), dry_run=dry_run, model=model, **kw) + + +# ========================================================================== +# 常量 +# ========================================================================== + + +def test_constants_match_copilot(): + from app.api import copilot + + assert A.COPILOT_SETTING_KEY == copilot.COPILOT_SETTING_KEY + assert A.AI_MAX_TOKENS == copilot.COPILOT_MAX_TOKENS + assert A.AI_FALLBACK_MAX_TOKENS == copilot._FALLBACK_MAX_TOKENS + assert A._BUDGET_WORDS == copilot._BUDGET_WORDS + assert A.AI_MAX_ATTEMPTS == 3 + + +# ========================================================================== +# 可用性（独立的内存库） +# ========================================================================== + + +class _MemDB: + """独立的内存库：不碰测试会话共用的库，也不受别的测试建的接入影响。""" + + async def __aenter__(self): + from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + from sqlalchemy.pool import StaticPool + + import app.db.models # noqa: F401 - 建表前要先登记全部模型 + from app.db.base import Base + + self.engine = create_async_engine("sqlite+aiosqlite:///:memory:", poolclass=StaticPool) + async with self.engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + self.session = async_sessionmaker(self.engine, expire_on_commit=False, class_=AsyncSession)() + return self.session + + async def __aexit__(self, *exc): + await self.session.close() + await self.engine.dispose() + + +@pytest.fixture +async def mem_session(): + async with _MemDB() as session: + yield session + + +async def _setup(session, providers: list[dict[str, Any]], saved: dict[str, str] | None = None) -> None: + from app.core.crypto import encrypt + from app.db.models import Provider, Setting + + for p in providers: + p = dict(p) + if p.get("api_key"): + p["api_key"] = encrypt(p["api_key"]) + session.add(Provider(**p)) + if saved is not None: + session.add(Setting(key=A.COPILOT_SETTING_KEY, value=saved)) + await session.commit() + + +@pytest.fixture +def no_env_keys(monkeypatch): + for k in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "OPENAI_API_KEY", "ANTHROPIC_BASE_URL"): + monkeypatch.delenv(k, raising=False) + + +CASES = { + "none": ([], None), + "mock_only": ([{"name": "演示", "kind": "mock", "models": [{"id": "mock-fast"}], "default_model": "mock-fast"}], None), + "name_disabled": ([{"name": "甲接入", "kind": "anthropic", "api_key": "k", "models": [{"id": "claude-x"}], + "default_model": "claude-x", "enabled": False}], {"provider": "甲接入", "model": ""}), + "model_disabled": ([{"name": "乙接入", "kind": "anthropic", "api_key": "k", "models": [{"id": "claude-y"}], + "default_model": "claude-y", "enabled": False}], {"provider": "", "model": "claude-y"}), + "missing_key": ([{"name": "丙接入", "kind": "anthropic", "api_key": None, "models": [{"id": "claude-z"}], + "default_model": "claude-z"}], None), +} + + +@pytest.mark.asyncio +async def test_unavailable_cases(no_env_keys, monkeypatch, flow): + """没有接入、只有演示接入、存的接入名已停用、存的模型名所属接入已停用、缺 key：都不可用，原因各不相同， + 假模型一次都没被调。""" + from app.providers import factory + + built: list[FakeModel] = [] + real = factory.build_chat_model + + def spy(provider, spec): + real(provider, spec) # 缺 key 时这里就抛 ProviderNotConfigured + fake = FakeModel([]) + built.append(fake) + return fake + monkeypatch.setattr(factory, "build_chat_model", spy) + sc, grids, _ = flow + reasons: dict[str, str] = {} + for case, (providers, saved) in CASES.items(): + async with _MemDB() as session: + await _setup(session, providers, saved) + avail = await A.ai_availability(session) + assert not avail.available and avail.reason, case + with pytest.raises(A.AiUnavailable) as e: + await A.ai_draft(session, grids=grids, scan=sc, facts=FACTS, rules_draft=None, filename="x.xlsx", + validate=claims_validate(), dry_run=None) + assert str(e.value) == avail.reason + reasons[case] = avail.reason + assert all(not m.calls for m in built) + assert len(set(reasons.values())) == len(CASES), reasons + assert "已停用" in reasons["name_disabled"] and "甲接入" in reasons["name_disabled"] + assert "已停用" in reasons["model_disabled"] and "乙接入" in reasons["model_disabled"] + assert "API Key" in reasons["missing_key"] + assert "演示" in reasons["mock_only"] + assert "未配置模型接入" in reasons["none"] + + +@pytest.mark.asyncio +async def test_available_builds_model_without_request(mem_session, no_env_keys, monkeypatch): + from app.providers import factory + + await _setup(mem_session, [{"name": "丁接入", "kind": "anthropic", "api_key": "sk-test", + "models": [{"id": "claude-w"}], "default_model": "claude-w"}], + {"provider": "丁接入", "model": "claude-w"}) + specs: list[Any] = [] + + def fake_build(provider, spec): + specs.append(spec) + return FakeModel([], name=spec.model) + monkeypatch.setattr(factory, "build_chat_model", fake_build) + model, model_id, avail = await A.resolve_draft_model(mem_session) + assert avail.available and avail.model == "claude-w" and avail.provider == "丁接入" and model_id == "claude-w" + assert [s.max_tokens for s in specs] == [A.AI_MAX_TOKENS, A.AI_FALLBACK_MAX_TOKENS] + assert all(s.thinking == "off" and s.temperature == 0 for s in specs) + assert not model.model.calls # 只建对象，不发请求 + + +@pytest.mark.asyncio +async def test_budget_rejection_falls_back_to_small_budget(mem_session, no_env_keys, monkeypatch, flow): + """服务方以 400「max_tokens too large」拒绝 → 退回 8192 重来，之后的调用都用 8192。""" + from app.providers import factory + + await _setup(mem_session, [{"name": "戊接入", "kind": "anthropic", "api_key": "sk-test", + "models": [{"id": "claude-v"}], "default_model": "claude-v"}]) + + class BudgetError(Exception): + status_code = 400 + + models: dict[int, FakeModel] = {} + + def fake_build(provider, spec): + if spec.max_tokens == A.AI_MAX_TOKENS: + m = FakeModel([BudgetError("max_tokens: 32768 > 8192, which is the maximum allowed")]) + else: + m = FakeModel([reply("不是 JSON"), reply(flow_json())]) + models[spec.max_tokens] = m + return m + monkeypatch.setattr(factory, "build_chat_model", fake_build) + sc, grids, _ = flow + res = await A.ai_draft(mem_session, grids=grids, scan=sc, facts=FACTS, rules_draft=None, filename="x.xlsx", + validate=claims_validate(), dry_run=None) + assert res.draft is not None and res.attempts == 2 + assert len(models[A.AI_MAX_TOKENS].calls) == 1 + assert len(models[A.AI_FALLBACK_MAX_TOKENS].calls) == 2 + + +# ========================================================================== +# 压缩表示不外泄 +# ========================================================================== + + +def _leaky_book(): + """一张交叉表 + 一张列表：特征数、像数的文字、带常量的公式、姓名电话、隐藏行列、遮罩列。""" + wb = Workbook() + ws = wb.active + ws.title = "客流汇总" + ws["B2"] = "统计时间范围：2026年8月1日至2026年8月8日" + for i in range(8): + ws.cell(4, 3 + i, f"8月{i + 1}日") + ws["B5"] = "全日客流（人次）" + ws["B6"] = "分区甲（人次）" + ws["B7"] = "分区乙（人次）" + ws["B8"] = "秘密指标（人次）" + for i in range(8): + ws.cell(5, 3 + i, 987654 + i) + ws.cell(6, 3 + i, 123457 + i) + ws.cell(7, 3 + i, 5555.75) + ws.cell(8, 3 + i, 424242) + ws["C9"] = "1,234" + ws["B9"] = "补录（人次）" + ws["D9"] = "=3000+2500" + ws["E9"] = "=C5*1.0675" + ws["F9"] = '="内部"&B5' + ws["G9"] = "='附表'!C5+1" + ws.row_dimensions[8].hidden = True + ws["L5"] = "隐藏列里的话" + ws.column_dimensions["L"].hidden = True + lst = wb.create_sheet("名单") + lst.append(["姓名", "电话", "金额", "工号"]) + people = [("张三", "138-1234-5678", 10, "X-保密-1"), ("李四", "139-8765-4321", 20, "X-保密-2"), + ("王五", "137-0000-1111", 30, "X-保密-3"), ("赵六", "136-2222-3333", 40, "X-保密-4"), + ("钱七", "135-4444-5555", 50, "X-保密-5")] + for p in people: + lst.append(list(p)) + cache = {"客流汇总": {"D9": 5500, "E9": 1054320.6, "F9": "内部987654", "G9": 99}} + return wb, cache + + +def test_compress_does_not_leak_values(): + wb, cache = _leaky_book() + sc, grids = build(wb, cache) + facts = S.detect_facts(grids, sc) + c = A.compress(grids, sc, facts, mask_headers={"工号"}) + text = c.text + for g in grids.values(): + for cell in g.cells.values(): + # 特征数逐个断言（小数字会和坐标、统计期里的年份撞上，不在这里查） + if isinstance(cell.value, (int, float)) and not isinstance(cell.value, bool) and abs(cell.value) >= 5000: + assert str(cell.value) not in text, cell.value + for secret in ("987654", "123457", "5555.75", "424242", "1,234", "3000", "2500", "1.0675", "内部", "1054320", + "张三", "李四", "138-1234-5678", "139-8765-4321", "秘密指标", "隐藏列里的话", "保密"): + assert secret not in text, secret + assert "<文本数字>" in text + assert "=<数>+<数>" in text and "=C5*<数>" in text and "=<文本>&B5" in text + assert "G9 ='附表'!<格>+<数>" in text # 跨工作表引用只留表名 + assert "隐藏行：第 8 行（文字 1 格、数字 8 格）" in text + assert "隐藏列：L 列（文字 1 格、数字 0 格）" in text + assert "A 列「姓名」是数据列（只给画像）：5 格，不同取值 5 个" in text + assert "B 列「电话」是数据列" in text + assert "D 列「工号」是遮罩列（只给画像）" in text + assert "C9「<文本数字>」" in text + # 结构文字照发 + for shown in ("全日客流（人次）", "统计时间范围：2026年8月1日至2026年8月8日", "序列「{}月{}日」8 格", "姓名", "金额"): + assert shown in text, shown + # 同一份网格两次压缩，sha256 相同 + again = A.compress(grids, sc, facts, mask_headers={"工号"}) + assert again.sha256 == c.sha256 and again.text == text and c.chars == len(text) + + +def test_sanitize_formula(): + assert A.sanitize_formula("=3000+2500") == "=<数>+<数>" + assert A.sanitize_formula("=C5*1.0675") == "=C5*<数>" + assert A.sanitize_formula('=IF(B5>50000,"高","低")') == "=IF(B5><数>,<文本>,<文本>)" + assert A.sanitize_formula("='内部'!C5+SUM(C1:C4)") == "='内部'!<格>+SUM(C1:C4)" + + +def test_compress_flow_shape(flow): + sc, grids, nums = flow + c = A.compress(grids, sc, S.detect_facts(grids, sc)) + assert "## 工作表「客流汇总」 已用区域 B2:AG30，非空格 771" in c.text + assert "C28:AG28 =SUM(C22:C25) 逐列平移（31 格，均有保存值）" in c.text + assert "C10:AG10「·」×31" in c.text + assert "C5:AG7 整数" in c.text + assert "F1（等式" in c.text and "分段 id「客流汇总_按日」" in c.text + assert "「日间时段客流（人次）」：日间、日间时段客流" in c.text + assert not any(str(v) in c.text for v in nums["jia"] + nums["yi"]) + # 压缩表示只有规格里的四个字段（接口层照此存库） + assert set(c.__dataclass_fields__) == {"text", "chars", "truncated", "sha256"} + + +def test_compress_folds_large_sheet(): + wb = Workbook() + ws = wb.active + ws.title = "大表" + ws.append(["编号", "金额"]) + for i in range(400): + ws.append([f"N{i:04d}", 1000 + i]) + sc, grids = build(wb) + c = A.compress(grids, sc, DraftFacts()) + assert c.truncated + assert "第 201–381 行（共 181 行）折叠为列画像：" in c.text # 前 200 行、后 20 行照常 + assert "N0300" not in c.text + + +@pytest.mark.asyncio +async def test_too_large_raises_before_calling_model(flow, monkeypatch): + monkeypatch.setattr(A, "COMPRESS_MAX_CHARS_PER_SHEET", 100) + model = FakeModel([reply(flow_json())]) + with pytest.raises(A.AiTooLarge): + await run(flow, model) + assert model.calls == [] + monkeypatch.setattr(A, "COMPRESS_MAX_CHARS_PER_SHEET", 12_000) + monkeypatch.setattr(A, "COMPRESS_MAX_CHARS", 100) + with pytest.raises(A.AiTooLarge): + await run(flow, model) + assert model.calls == [] + + +# ========================================================================== +# 起草、修订、失败 +# ========================================================================== + + +@pytest.mark.asyncio +async def test_invalid_json_then_valid(flow): + model = FakeModel([reply("我先想一想……"), reply(flow_json(), inp=200, out=80)]) + res = await run(flow, model) + assert res.draft is not None and res.draft.origin == "ai" and res.error is None + assert res.attempts == 2 and len(res.usage) == 2 + assert [u.ok for u in res.usage] == [True, True] + assert res.usage[1].input_tokens == 200 and res.usage[1].output_tokens == 80 + assert all(u.model == "fake-model" and u.attempt == i + 1 for i, u in enumerate(res.usage)) + assert res.draft.cards[0].id == "ai_origin" and res.draft.cards[-1].id == "mode" + assert res.draft.complete + assert Recipe.model_validate(res.draft.recipe).model_dump(mode="json", exclude_defaults=True) == \ + Recipe.model_validate(FLOW).model_dump(mode="json", exclude_defaults=True) + assert res.compressed_chars > 0 and len(res.compressed_sha256) == 64 + # 修订是同一轮对话：system + 首次 prompt + 上一轮回复（原样）+ repair + second = model.calls[1] + assert len(second) == 4 + assert isinstance(second[0], SystemMessage) and isinstance(second[1], HumanMessage) + assert second[0].content == A.RECIPE_DRAFT_SYSTEM + assert second[1].content == model.calls[0][1].content + assert isinstance(second[2], AIMessage) and second[2].content == "我先想一想……" + assert isinstance(second[3], HumanMessage) and "不是合法的 JSON" in second[3].content + + +@pytest.mark.asyncio +async def test_block_list_content_is_extracted(flow): + content = [{"type": "thinking", "thinking": "先写交叉表 {\"x\": 1}", "signature": "s"}, + {"type": "text", "text": "```json\n" + flow_json() + "\n```"}] + model = FakeModel([reply(content)]) + res = await run(flow, model) + assert res.draft is not None and res.attempts == 1 + + +@pytest.mark.asyncio +async def test_truncated_reply_is_not_repaired(flow): + model = FakeModel([reply(flow_json()[:200], meta={"stop_reason": "max_tokens"}), + reply(flow_json())]) + res = await run(flow, model) + assert res.draft is not None and res.attempts == 2 + assert [u.ok for u in res.usage] == [False, True] + assert len(model.calls[1]) == 2 # 没有 repair：同样的消息列表重来 + assert [m.content for m in model.calls[1]] == [m.content for m in model.calls[0]] + # 三次都截断：error 是截断的原因，不是「不是 JSON」 + cut = reply("{", meta={"finish_reason": "length"}) + res2 = await run(flow, FakeModel([cut, cut, cut])) + assert res2.draft is None and "输出被截断" in (res2.error or "") + assert [u.ok for u in res2.usage] == [False, False, False] + + +@pytest.mark.asyncio +async def test_timeouts_fail_without_raising(flow, monkeypatch): + monkeypatch.setattr(A, "AI_TIMEOUT_S", 0.05) + + async def slow(): + await asyncio.sleep(1) + return reply(flow_json()) + model = FakeModel([slow, slow, slow]) + res = await run(flow, model) + assert res.draft is None and res.attempts == 3 + assert "调用超时" in (res.error or "") + assert len(res.usage) == 3 and not any(u.ok for u in res.usage) + assert all(len(c) == 2 for c in model.calls) + + +@pytest.mark.asyncio +async def test_call_exception_then_success(flow): + model = FakeModel([RuntimeError("connection reset"), reply(flow_json())]) + res = await run(flow, model) + assert res.draft is not None and res.attempts == 2 + assert [u.ok for u in res.usage] == [False, True] + res2 = await run(flow, FakeModel([RuntimeError("boom")] * 3)) + assert res2.draft is None and (res2.error or "").startswith("调用失败：") + + +@pytest.mark.asyncio +async def test_note_and_unclaimed_fact_trigger_repair(flow): + with_note = json.loads(flow_json()) + with_note["tables"][0]["note"] = "这是AI写的说明" + unclaimed = json.loads(flow_json()) + unclaimed["relations"] = unclaimed["relations"][:1] + calls: list = [] + model = FakeModel([reply(json.dumps(with_note, ensure_ascii=False)), + reply(json.dumps(unclaimed, ensure_ascii=False)), reply(flow_json())]) + res = await run(flow, model, validate=claims_validate(calls)) + assert res.draft is not None and res.attempts == 3 + assert [c[2] for c in calls] == ["ai", "ai", "ai"] + assert "/tables/0/note：" in model.calls[1][-1].content and "ai_note_forbidden" not in model.calls[1][-1].content + assert "没有被认领" in model.calls[2][-1].content + + +@pytest.mark.asyncio +async def test_dry_run_problems_are_fed_back_with_model_message_only(flow): + sc, grids, _ = flow + grids = dict(grids) + g = grids["客流汇总"] + cells = dict(g.cells) + cells[(31, 3)] = GridCell(987654) + cells[(31, 4)] = GridCell("约7777") + grids["客流汇总"] = Grid(g.sheet, g.bounds, cells, g.merges, g.hidden_rows, g.hidden_cols) + n = {"k": 0} + + def dry(recipe): + n["k"] += 1 + if n["k"] == 1: + return Extraction(ok=False, problems=[ + Problem("outside_number", "structure", "第 31 行 C31 的 987654 在数据区外", ["客流汇总!C31"]), + Problem("value_not_number", "structure", "D31 的「约7777」不是数字", ["客流汇总!D31"], + model_message="D31 不是数字"), + Problem("outside_digits", "confirm", "区域外有含数字的文字"), + ]) + return Extraction(ok=True) + model = FakeModel([reply(flow_json()), reply(flow_json())]) + res = await A.ai_draft(None, grids=grids, scan=sc, facts=FACTS, rules_draft=None, filename="x.xlsx", # type: ignore[arg-type] + validate=claims_validate(), dry_run=dry, model=model) + assert res.draft is not None and res.draft.complete and res.attempts == 2 + fed = model.calls[1][-1].content + assert "outside_number（客流汇总!C31）" in fed + assert "value_not_number：D31 不是数字（客流汇总!D31）" in fed + assert "987654" not in fed and "7777" not in fed and "outside_digits" not in fed + + +@pytest.mark.asyncio +async def test_cpu_work_runs_off_the_event_loop_and_async_dry_run_is_awaited(flow, monkeypatch): + """ai_draft 跑在事件循环上：静态校验、干跑、卡片问题都不能在循环线程上同步执行（一次干跑可能要几秒， + 会堵住同进程的所有请求）。协程函数的 dry_run 直接 await（接口层那份自己占解析名额、进线程池）； + 普通函数的 dry_run 放进线程池。""" + import threading + + loop_thread = threading.get_ident() + where: dict[str, list[int]] = {"validate": [], "questions_for": [], "sync_dry": []} + inner = claims_validate() + + def validate(data, facts, origin): + where["validate"].append(threading.get_ident()) + return inner(data, facts, origin) + + real_questions = S.questions_for + + def questions_for(*a, **kw): + where["questions_for"].append(threading.get_ident()) + return real_questions(*a, **kw) + monkeypatch.setattr(S, "questions_for", questions_for) + + awaited: list[str] = [] + + async def dry_async(recipe): + awaited.append(type(recipe).__name__) + await asyncio.sleep(0) + return Extraction(ok=True) + + res = await run(flow, FakeModel([reply(flow_json())]), validate=validate, dry_run=dry_async) + assert res.draft is not None and res.draft.complete and awaited == ["Recipe"] + assert where["validate"] and all(t != loop_thread for t in where["validate"]) + assert where["questions_for"] and all(t != loop_thread for t in where["questions_for"]) + + def dry_sync(recipe): + where["sync_dry"].append(threading.get_ident()) + return Extraction(ok=True) + + res = await run(flow, FakeModel([reply(flow_json())]), dry_run=dry_sync) + assert res.draft is not None and where["sync_dry"] and all(t != loop_thread for t in where["sync_dry"]) + + # 协程版干跑出错：同普通版，变成一条起草失败原因，不让起草整个失败 + async def boom(recipe): + raise RuntimeError("内部细节") + res = await run(flow, FakeModel([reply(flow_json())] * 3), dry_run=boom) + assert res.draft is not None and not res.draft.complete + assert res.draft.failures == [S._FAIL_TEXT["dry_run"]] and "内部细节" not in json.dumps(res.draft.failures) + + +@pytest.mark.asyncio +async def test_last_dry_run_problems_still_return_draft(flow): + """次数用完时，静态校验通过但干跑仍有结构问题的配方照样返回（问题随草稿交给用户）。""" + prob = Problem("cell_unclaimed", "structure", "第 31 行没有去处", ["客流汇总!B31"], model_message="第 31 行没有去处") + model = FakeModel([reply(flow_json())] * 3) + res = await run(flow, model, dry_run=lambda r: Extraction(ok=False, problems=[prob])) + assert res.attempts == 3 and res.draft is not None and not res.draft.complete + assert res.draft.failures == ["第 31 行没有去处"] and res.error is None + # 后面一次退化成不合法的 JSON：仍返回之前那份静态校验通过的配方 + model2 = FakeModel([reply(flow_json()), reply("坏了"), reply("还是坏的")]) + res2 = await run(flow, model2, dry_run=lambda r: Extraction(ok=False, problems=[prob])) + assert res2.draft is not None and res2.attempts == 3 + + +@pytest.mark.asyncio +async def test_never_valid_returns_none_with_usage(flow): + model = FakeModel([reply("x"), reply("{\"recipe_format\": 1}"), reply("y")]) + res = await run(flow, model) + assert res.draft is None and res.error == "AI 未能起草出合法的配方" + assert len(res.usage) == 3 and res.attempts == 3 + + +@pytest.mark.asyncio +async def test_prompt_uses_failures_for_model_not_failures(flow): + rules = Draft(recipe=None, complete=False, origin="rules", + failures=["C12 的「约987654」不是数字"], failures_for_model=["工作表「客流汇总」：C12 不是数字"]) + model = FakeModel([reply(flow_json())]) + await run(flow, model, rules=rules) + prompt = model.calls[0][1].content + assert "987654" not in prompt + assert "工作表「客流汇总」：C12 不是数字" in prompt + assert "# 配方 JSON Schema" in prompt and '"recipe_format"' in prompt + assert "人次、人、元" in prompt + # 半成品配方也带上（紧凑形式） + sc, grids, _ = flow + rd = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + model2 = FakeModel([reply(flow_json())]) + await run(flow, model2, rules=rd) + assert "半成品配方" in model2.calls[0][1].content and "客流汇总_按日" in model2.calls[0][1].content + + +@pytest.mark.asyncio +async def test_given_compressed_is_what_is_sent(flow): + sc, grids, _ = flow + c = A.compress(grids, sc, FACTS) + model = FakeModel([reply(flow_json())]) + res = await run(flow, model, compressed=c) + assert c.text in model.calls[0][1].content + assert res.compressed_sha256 == c.sha256 + + +@pytest.mark.asyncio +async def test_rules_draft_section_hides_hidden_and_numeric_labels(): + wb = Workbook() + ws = wb.active + ws.title = "表" + ws["B2"] = "2026年8月1日至2026年8月8日" + for i in range(8): + ws.cell(4, 3 + i, f"8月{i + 1}日") + for r, lab in ((5, "甲（人次）"), (6, "藏起来的指标"), (7, "2025")): + ws.cell(r, 2, lab) + for i in range(8): + ws.cell(r, 3 + i, 10 + r + i) + ws.row_dimensions[6].hidden = True + sc, grids = build(wb) + rd = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert rd.recipe is not None and "藏起来的指标" in json.dumps(rd.recipe, ensure_ascii=False) + model = FakeModel([reply(flow_json())]) + await A.ai_draft(None, grids=grids, scan=sc, facts=rd.facts, rules_draft=rd, filename="x.xlsx", # type: ignore[arg-type] + validate=ok_validate(), dry_run=None, model=model) + prompt = model.calls[0][1].content + assert "藏起来的指标" not in prompt + assert '"2025"' not in prompt + + +# ========================================================================== +# 评审修复：数据行不当表头外发、压缩表示只靠 text、日期轴不发值 +# ========================================================================== + + +@pytest.mark.asyncio +async def test_list_row_with_blank_number_does_not_leak(): + """名单第 4 行的金额空着：这一行不能被当成表头，姓名、电话、备注都不出现在压缩表示和提示词里。""" + from tests.test_recipe_suggest import _roster + + for blank in (False, True): + sc, grids = build(_roster(blank_before_wang=blank)) + rd = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + c = A.compress(grids, sc, rd.facts) + model = FakeModel([reply(flow_json())]) + await A.ai_draft(None, grids=grids, scan=sc, facts=rd.facts, rules_draft=rd, filename="x.xlsx", # type: ignore[arg-type] + validate=ok_validate(), dry_run=None, model=model) + prompt = model.calls[0][1].content + for secret in ("张三", "王五", "钱七", "138-1234-5678", "137-2222-3333", "137_2222_3333", "转介绍", "老客户"): + assert secret not in c.text, (blank, secret) + assert secret not in prompt, (blank, secret) + assert "A 列「姓名」是数据列（只给画像）：5 格" in c.text + assert "B 列「电话」是数据列" in c.text and "D 列「备注」是数据列" in c.text + + +@pytest.mark.asyncio +async def test_rules_section_scrubs_column_names_made_from_hidden_text(): + """万一起草把数据行认成了表头：配方里的列名是 to_sql_name 变过的写法，同样要遮掉。""" + from tests.test_recipe_suggest import _roster + + sc, grids = build(_roster()) + rd = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + bad = json.loads(json.dumps(rd.recipe, ensure_ascii=False)) + cols = bad["sheets"][0]["blocks"][0]["columns"] + cols[0].update(header="王五", name="王五") + cols[1].update(header="137-2222-3333", name="c_137_2222_3333") + bad["tables"][0]["grain"] = ["王五"] + leaky = Draft(recipe=bad, complete=False, origin="rules", failures=[], failures_for_model=[]) + model = FakeModel([reply(flow_json())]) + await A.ai_draft(None, grids=grids, scan=sc, facts=rd.facts, rules_draft=leaky, filename="x.xlsx", # type: ignore[arg-type] + validate=ok_validate(), dry_run=None, model=model) + prompt = model.calls[0][1].content + assert "半成品配方" in prompt + assert "王五" not in prompt and "137" not in prompt + + +@pytest.mark.asyncio +async def test_compressed_with_only_the_four_fields_is_sent_once(flow): + """接口层把压缩表示按四个字段存库再取回：提示词的前三段就是 text，不重复、不重算。""" + sc, grids, _ = flow + facts = S.detect_facts(grids, sc) + c = A.compress(grids, sc, facts) + stored = A.Compressed(text=c.text, chars=c.chars, truncated=c.truncated, sha256=c.sha256) + m1, m2 = FakeModel([reply(flow_json())]), FakeModel([reply(flow_json())]) + await run(flow, m1, facts=facts, compressed=stored) + await run(flow, m2, facts=facts) + prompt = m1.calls[0][1].content + for head in ("# 工作表压缩表示", "# 系统发现", "# 分段标题候选词", "# 单位词表", "# 规则起草的半成品与失败原因", + "# 配方 JSON Schema"): + assert prompt.count(head) == 1, head + assert prompt.startswith(c.text + "\n\n# 单位词表\n") + assert prompt == m2.calls[0][1].content + # 与 RECIPE_DRAFT_PROMPT 的写法逐字相同（前三段取压缩表示） + assert A.RECIPE_DRAFT_PROMPT == A.RECIPE_SECTIONS_PROMPT + "\n\n" + A.RECIPE_REQUEST_PROMPT + + +def test_date_axis_cells_send_type_only(): + """日期轴上是日期格（不是「8月1日」这样的文字）时只发类型签名：表格里没有统计期格时，轴上的年月日就是数据。""" + wb = Workbook() + ws = wb.active + ws.title = "日报" + for i in range(8): + ws.cell(4, 3 + i, dt.datetime(2026, 8, 1 + i)) + for r, lab in ((5, "全日客流（人次）"), (6, "分区甲（人次）")): + ws.cell(r, 2, lab) + for i in range(8): + ws.cell(r, 3 + i, 1000 + r * 10 + i) + sc, grids = build(wb) + assert S.layout_hints(grids["日报"]).axis_row == 4 + c = A.compress(grids, sc, S.detect_facts(grids, sc)) + assert "C4:J4 日期" in c.text + for leak in ("2026-08", "2026年", "8月1日", "08-01"): + assert leak not in c.text, leak + assert "B5「全日客流（人次）」" in c.text + + +# ========================================================================== +# 整体审查（AU / SE）修复 +# ========================================================================== + + +def _hidden_unit_book() -> Workbook: + """交叉表：第 7 行隐藏，标签带单位后缀（「绝密分区（人次）」），并且是 F1（5 = 6 + 7）的成员。""" + wb = Workbook() + ws = wb.active + ws.title = "表" + ws["B2"] = "统计时间范围：2026年8月1日至2026年8月8日" + for i in range(8): + ws.cell(4, 3 + i, f"8月{i + 1}日") + labels = ((5, "全日客流（人次）"), (6, "分区甲（人次）"), (7, "绝密分区（人次）")) + for r, lab in labels: + ws.cell(r, 2, lab) + for i in range(8): + a, b = 100 + i, 30 + i + ws.cell(6, 3 + i, a) + ws.cell(7, 3 + i, b) + ws.cell(5, 3 + i, a + b) + ws.row_dimensions[7].hidden = True + return wb + + +@pytest.mark.asyncio +async def test_hidden_label_with_unit_suffix_never_leaves_in_any_call(): + """SE-1、SE-2：隐藏行的标签（含去掉单位后缀的「绝密分区」）不出现在首轮全文、也不出现在任何一轮修订消息里。""" + sc, grids = build(_hidden_unit_book()) + rd = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + facts = S.detect_facts(grids, sc) + assert rd.recipe is not None and "绝密分区" in json.dumps(rd.recipe, ensure_ascii=False) + assert any("绝密分区" in json.dumps(f.detail, ensure_ascii=False) for f in facts.facts) + c = A.compress(grids, sc, facts) + first = A.first_call_text(c, grids=grids, scan=sc, rules_draft=rd) + assert "绝密" not in first + assert "半成品配方" in first and "人次" in first # 半成品照发，只遮隐藏的部分 + + # 模型把首轮看到的半成品原样抄回来（<隐藏> 对不上真实标签）：静态校验的消息会引用系统发现的标签原文 + echoed = json.dumps(A._Outbound(A._secret_texts(grids, sc), grids).json(rd.recipe), ensure_ascii=False) + from app.data import recipe as R + + def validate(data, f, origin): + return R.validate_recipe(data, facts=f, origin=origin) + model = FakeModel([reply(echoed), reply(echoed), reply(echoed)]) + res = await A.ai_draft(None, grids=grids, scan=sc, facts=facts, rules_draft=rd, filename="x.xlsx", # type: ignore[arg-type] + validate=validate, dry_run=None, model=model, compressed=c) + assert res.attempts == 3 and len(model.calls) == 3 + repairs = [m.content for call in model.calls for m in call if isinstance(m, HumanMessage)] + assert len(repairs) >= 3 and any("上一次的配方有以下问题" in x for x in repairs) + for text in repairs: + assert "绝密" not in text + + +def test_hidden_column_header_with_unit_suffix_is_scrubbed(): + """SE-2：隐藏列的表头「机密奖金（元）」去掉单位后成了列名「机密奖金」和 units 的键，同样不能外发。""" + wb = Workbook() + ws = wb.active + ws.title = "明细" + for j, h in enumerate(["日期", "门店", "机密奖金（元）", "数量（件）"]): + ws.cell(1, j + 1, h) + for i in range(10): + ws.cell(2 + i, 1, dt.datetime(2026, 8, 1 + i)) + ws.cell(2 + i, 2, ["店甲", "店乙"][i % 2]) + ws.cell(2 + i, 3, 100 + i) + ws.cell(2 + i, 4, 5 + i) + ws.column_dimensions["C"].hidden = True + sc, grids = build(wb) + rd = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert rd.recipe is not None and "机密奖金" in json.dumps(rd.recipe, ensure_ascii=False) + text = A.first_call_text(A.compress(grids, sc, rd.facts), grids=grids, scan=sc, + rules_draft=Draft(recipe=rd.recipe, complete=False, origin="rules")) + assert "机密" not in text and "半成品配方" in text and "数量" in text + + +def test_outbound_line_scrubs_quoted_numeric_text_and_hidden_variants(): + sc, grids = build(_hidden_unit_book()) + ob = A._Outbound(A._secret_texts(grids, sc), grids) + line = ob.line("应为「全日客流（人次）」=「分区甲（人次）」+「绝密分区（人次）」；列「绝密分区」；表「绝密分区_人次」") + assert "绝密" not in line and "全日客流（人次）" in line and "分区甲（人次）" in line + + +def test_total_word_in_middle_of_data_column_is_profiled_not_sent(): + """SE-3：备注列中间一格以「累计」开头：它是数据，只进画像，不当合计标签原样外发；最左一列的「合计」照发。""" + wb = Workbook() + ws = wb.active + ws.title = "名单" + ws.append(["姓名", "产品", "数量", "金额（元）", "备注"]) + rows = [("张一", "甲品", 3, 30, "老客户来电"), ("张二", "甲品", 2, 20, "累计消费12000元"), + ("张三", "甲品", 1, 10, "新客户到店"), ("张四", "甲品", 5, 50, "转介绍来的"), + ("张五", "甲品", 4, 40, "门店活动价"), ("张六", "甲品", 6, 60, "线上下单的")] + for r in rows: + ws.append(list(r)) + ws.append(["合计", None, 21, 210, None]) + sc, grids = build(wb) + c = A.compress(grids, sc, S.detect_facts(grids, sc)) + assert "累计消费" not in c.text and "12000" not in c.text + assert "E 列「备注」是数据列（只给画像）：6 格" in c.text + assert "A8「合计」" in c.text + + +def test_extract_json_deep_nesting_is_value_error(): + """SE-6：深层嵌套（json.loads 会抛 RecursionError）一律按「不是合法的 JSON」处理。""" + deep = '{"x":' + "[" * 12000 + "}" + with pytest.raises(ValueError): + A._extract_json(deep) + closed = '{"x":' + "[" * 100 + "]" * 100 + "}" + with pytest.raises(ValueError): + A._extract_json(closed) + assert A._extract_json('{"a": "[[[[{{{{", "b": [[1]]}') == {"a": "[[[[{{{{", "b": [[1]]} + + +@pytest.mark.asyncio +async def test_deeply_nested_reply_returns_none_with_usage(flow): + """SE-6：回复是约 1.2 万层不闭合的「[」时 ai_draft 不抛异常，返回 draft=None、每次调用的用量都在。""" + deep = '{"x":' + "[" * 12000 + "}" + model = FakeModel([reply(deep), reply(deep), reply(deep)]) + res = await run(flow, model) + assert res.draft is None and res.error and len(res.usage) == 3 == res.attempts + + +@pytest.mark.asyncio +async def test_exception_after_reply_is_returned_not_raised(flow, monkeypatch): + """SE-6：拿到回复之后的处理（这里是 check_recipe 本身）抛了异常：不逃出 ai_draft，用量照记。""" + def boom(*a, **kw): + raise RecursionError("maximum recursion depth exceeded") + monkeypatch.setattr(S, "check_recipe", boom) + model = FakeModel([reply(flow_json())]) + res = await run(flow, model) + assert res.draft is None and len(res.usage) == 1 and res.error + + +@pytest.mark.asyncio +async def test_given_model_id_is_recorded_and_budget_wrapper_has_a_name(flow): + """AU-7 / SE-4：接口层把同意时解析出的模型和它的 id 交进来，用量记的就是这个 id；_BudgetFallback 的 + model_name 是 id，不是底层模型对象的 repr。""" + model = FakeModel([reply(flow_json())], name="ignored") + res = await run(flow, model, model_id="local-model-A") + assert [u.model for u in res.usage] == ["local-model-A"] + wrapped = A._BudgetFallback(object(), object(), "cloud-model-B") + assert wrapped.model_name == "cloud-model-B" diff --git a/backend/tests/test_recipe_checks.py b/backend/tests/test_recipe_checks.py new file mode 100644 index 00000000..957357a2 --- /dev/null +++ b/backend/tests/test_recipe_checks.py @@ -0,0 +1,826 @@ +"""核对模块（recipe_checks）与 engine.open_checked_sqlite 的测试。 + +不依赖执行器：库和 Extraction 都在测试里按「结构仿照客流表」的参考配方手写（假名、随机假数）。 +""" +from __future__ import annotations + +import hashlib +import json +import random +import sqlite3 +from copy import deepcopy +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable + +import pytest + +from app.data import engine +from app.data.engine import SqliteHardeningError, open_checked_sqlite +from app.data.recipe_checks import ( + col_letters, + column_null_counts, + lineage_cell, + plan_checks, + run_checks, + same_number, +) +from app.data.recipe_types import CheckResult, DerivedItem, Extraction, Recipe, derive_tables + +FLOW = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text(encoding="utf-8")) +SHEET = "客流汇总" +DAY = [f"{h}-{h + 1}" for h in range(7, 18)] # 第 10–20 行 +NIGHT = [f"{h}-{h + 1}" for h in range(18, 24)] # 第 22–27 行 +TOTALS = [("18-22时合计", 18, 22, 28), ("22-24时合计", 22, 24, 29), ("18-24时合计", 18, 24, 30)] + + +def _q(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def make_db(path: Path, recipe: Recipe, rows: dict[str, list[dict[str, Any]]], *, pk: bool = True) -> None: + """按 derive_tables 建 STRICT 表、按给定顺序插入（rowid 就是插入序号）。""" + tables, problems = derive_tables(recipe) + assert problems == [] + grains = {t.name: t.grain for t in recipe.tables} + conn = open_checked_sqlite(str(path), readonly=False) + try: + for table, cols in tables.items(): + defs = [f"{_q(c.name)} {c.type}" for c in cols] + if pk and grains.get(table): + defs.append(f"PRIMARY KEY ({', '.join(_q(g) for g in grains[table])})") + conn.execute(f"CREATE TABLE {_q(table)} ({', '.join(defs)}) STRICT") + names = [c.name for c in cols] + conn.executemany( + f"INSERT INTO {_q(table)} ({', '.join(_q(n) for n in names)}) VALUES ({', '.join('?' * len(names))})", + [[r.get(n) for n in names] for r in rows.get(table, [])]) + conn.commit() + finally: + conn.close() + + +@dataclass +class Flow: + recipe: Recipe + db: str + extraction: Extraction + dates: list[str] + rows: dict[str, list[dict[str, Any]]] + + +def flow(tmp_path: Path, days: int = 3, *, recipe_data: dict | None = None, + rows_hook: Callable[[dict], None] | None = None, + items_hook: Callable[[list[DerivedItem]], None] | None = None, + pk: bool = True, seed: int = 7, full_calc: bool = False) -> Flow: + rnd = random.Random(seed) + recipe = Recipe.model_validate(deepcopy(recipe_data or FLOW)) + dates = [f"2026-08-{d:02d}" for d in range(1, days + 1)] + daily = [] + for d in dates: + a, b = rnd.randint(3000, 9000), rnd.randint(2000, 8000) + daily.append({"日期": d, "全日客流": a + b, "分区甲": a, "分区乙": b}) + hourly = [] + for cat, labels in (("日间", DAY), ("夜间", NIGHT)): + for label in labels: + s, e = (int(x) for x in label.split("-")) + for d in dates: + v = None if label == "7-8" else rnd.randint(50, 900) + hourly.append({"日期": d, "时段": label, "时段类别": cat, "起始小时": s, "结束小时": e, "客流": v}) + for row in daily: # 各时段之和 ≠ 全日客流（生成器保证） + tot = sum(h["客流"] or 0 for h in hourly if h["日期"] == row["日期"]) + if tot == row["全日客流"]: + next(h for h in hourly if h["日期"] == row["日期"] and h["客流"] is not None)["客流"] += 1 + + def night_sum(d: str, s: int, e: int, rows: list[dict]) -> int: + return sum(h["客流"] for h in rows if h["日期"] == d and h["时段类别"] == "夜间" + and h["起始小时"] >= s and h["结束小时"] <= e) + + totals = [{"日期": d, "合计项": label, "起始小时": s, "结束小时": e, "客流": night_sum(d, s, e, hourly)} + for label, s, e, _ in TOTALS for d in dates] + original = deepcopy(hourly) + rows = {"日客流": daily, "时段客流": hourly, "时段客流_表内合计": totals} + if rows_hook: + rows_hook(rows) + db = tmp_path / "trial.db" + make_db(db, recipe, rows, pk=pk) + + items: list[DerivedItem] = [] + for label, s, e, r in TOTALS: + for i, d in enumerate(dates): + refs = [n for n, h in enumerate(rows["时段客流"], 1) if h["日期"] == d and h["时段类别"] == "夜间" + and h["起始小时"] >= s and h["结束小时"] <= e] + items.append(DerivedItem( + kind="label_range_sum", segment="夜间合计", sheet=SHEET, cell=f"{col_letters(3 + i)}{r}", + label_raw=label.replace("时", " 时"), label=label, value=night_sum(d, s, e, original), + is_formula=True, formula=f"=SUM({col_letters(3 + i)}22:{col_letters(3 + i)}25)", + base_table="时段客流", base_value="客流", start=s, end=e, + key={"日期": d, "时段类别": "夜间"}, start_col="起始小时", end_col="结束小时", + ref_rowids=refs, func="SUM", keep_table="时段客流_表内合计")) + if items_hook: + items_hook(items) + lineage = { + "日客流": {c: [[1, SHEET, f"C{r}", days, "right"]] for c, r in (("全日客流", 5), ("分区甲", 6), ("分区乙", 7))}, + "时段客流": {"客流": [[1 + j * days, SHEET, f"C{r}", days, "right"] + for j, r in enumerate([*range(10, 21), *range(22, 28)])]}, + } + ctx = [CheckResult("C1", "context_agree", "统计期", "passed", "data_quality", checked=1), + CheckResult("C2", "filename_period", "文件名", "passed", "data_quality", checked=1)] + ex = Extraction(ok=True, derived=items, lineage=lineage, context_checks=ctx, full_calc_on_load=full_calc, + expected_rows={t: len(v) for t, v in rows.items()}) + return Flow(recipe, str(db), ex, dates, rows) + + +def by_id(results: list[CheckResult]) -> dict[str, CheckResult]: + return {c.id: c for c in results} + + +# ---------------------------------------------------------------- open_checked_sqlite + + +def test_open_checked_sqlite_readonly_and_dqs_off(tmp_path): + path = tmp_path / "a b?#中文.db" + conn = open_checked_sqlite(str(path), readonly=False) + conn.execute("CREATE TABLE t (a INTEGER)") + conn.execute("INSERT INTO t VALUES (1)") + conn.commit() + # 默认回滚日志，不开 WAL（WAL 改写文件头，库哈希会变） + assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "delete" + conn.close() + assert path.exists() and sorted(p.name for p in tmp_path.iterdir()) == [path.name] + + ro = open_checked_sqlite(str(path)) + try: + assert ro.execute("SELECT a FROM t").fetchall() == [(1,)] + with pytest.raises(sqlite3.OperationalError, match="readonly"): + ro.execute("INSERT INTO t VALUES (2)") + # DQS 关了：写错的双引号列名报错，而不是当成字符串 + with pytest.raises(sqlite3.OperationalError, match="no such column"): + ro.execute('SELECT "no_such" FROM t') + finally: + ro.close() + + +def test_open_checked_sqlite_readonly_does_not_create(tmp_path): + with pytest.raises(sqlite3.OperationalError): + open_checked_sqlite(str(tmp_path / "missing.db")) + assert not (tmp_path / "missing.db").exists() + + +def test_open_checked_sqlite_refuses_when_dqs_cannot_be_disabled(tmp_path, monkeypatch): + def broken(conn): + raise SqliteHardeningError("关不掉") + + monkeypatch.setattr(engine, "_disable_dqs", broken) + with pytest.raises(SqliteHardeningError): + open_checked_sqlite(str(tmp_path / "x.db"), readonly=False) + + +# ---------------------------------------------------------------- 小工具 + + +def test_same_number_rules(): + assert same_number(100, 100.0) + assert not same_number(100, 100.0000001 + 1) + assert same_number(0.3, 0.1 + 0.2) # 非整数值按相对容差 + assert not same_number(5, 6.0) + assert not same_number(None, 0.0) + + +def test_lineage_cell_reverse_lookup(): + lin = {"t": {"v": [[1, "表一", "C5", 31, "right"], [32, "表一", "B10", 4, "down"]]}} + assert lineage_cell(lin, "t", "v", 1) == "表一!C5" + assert lineage_cell(lin, "t", "v", 10) == "表一!L5" + assert lineage_cell(lin, "t", "v", 33) == "表一!B11" + assert lineage_cell(lin, "t", "v", 99) is None + assert col_letters(27) == "AA" and col_letters(33) == "AG" + + +# ---------------------------------------------------------------- 全部通过（D00 的形状） + + +def test_reference_flow_all_checks(tmp_path): + f = flow(tmp_path, days=5) + before = hashlib.sha256(Path(f.db).read_bytes()).hexdigest() + results = run_checks(f.db, f.extraction, f.recipe) + assert [c.id for c in results] == ["C1", "C2", "K1", "G1", "R1", "R2", "N1", "N2", "N3", "P1", "P2", "P3"] + got = by_id(results) + assert got["K1"].status == "passed" and got["K1"].checked == 15 and got["K1"].reasons == {} + assert got["K1"].kind == "derived_sum" and got["K1"].acceptable is False + assert got["G1"].status == "passed" and got["G1"].checked == 15 + assert got["R1"].status == "passed" and got["R1"].checked == 5 and got["R1"].category == "data_quality" + assert got["R2"].status == "info" and got["R2"].details == ["各时段之和与全日客流：5 天中 0 天相等"] + assert got["R2"].checked == 5 and got["R2"].failed == 5 + assert all(got[k].status == "passed" for k in ("N1", "N2", "N3", "P1", "P2", "P3")) + # 只读打开：核对不改库 + assert hashlib.sha256(Path(f.db).read_bytes()).hexdigest() == before + assert sorted(p.name for p in tmp_path.iterdir()) == ["trial.db"] + + +def test_sql_text_and_params_recorded(tmp_path): + f = flow(tmp_path) + got = by_id(run_checks(f.db, f.extraction, f.recipe)) + k = got["K1"] + assert k.sql == ('SELECT TOTAL("客流"), COUNT("客流"), COUNT(*) FROM "时段客流" ' + 'WHERE "日期" = ? AND "时段类别" = ? AND "起始小时" >= ? AND "结束小时" <= ?') + assert k.params == ["2026-08-01", "夜间", 18, 22] + assert '"全日客流" <> "分区甲" + "分区乙"' in got["R1"].sql + assert got["N1"].sql == 'SELECT COUNT(*) FROM "日客流"' + assert 'GROUP BY "日期", "时段"' in got["P2"].sql + assert got["G1"].sql.startswith('SELECT rowid FROM "时段客流" WHERE "日期" = ?') + # 重放：同一条 SQL、同样的参数，在同一个库上得到同一个数 + conn = sqlite3.connect(f.db) + assert conn.execute(k.sql, k.params).fetchone()[0] == f.extraction.derived[0].value + conn.close() + + +def test_full_calc_on_load_consistent_passes_with_note(tmp_path): + f = flow(tmp_path, full_calc=True) + k = by_id(run_checks(f.db, f.extraction, f.recipe))["K1"] + assert k.status == "passed" + assert "工作簿设置了打开时重算，保存值与重算一致" in k.details + + +# ---------------------------------------------------------------- K：各种结果 + + +def test_k_mismatch_points_at_cell(tmp_path): + def bump(items): + items[4].value += 100 # 「18-22 时合计」第 5 个日期（G28） + + f = flow(tmp_path, days=5, items_hook=bump) + k = by_id(run_checks(f.db, f.extraction, f.recipe))["K1"] + assert k.status == "mismatch" and k.category == "structure" and k.acceptable is False + assert k.failed == 1 and k.checked == 15 + item = f.extraction.derived[4] + assert k.details[0] == f"G28：表内为 {item.value}，按 18-22 时的明细重算为 {item.value - 100}" + assert k.cells == [f"{SHEET}!G28"] + assert k.params == ["2026-08-05", "夜间", 18, 22] # 代表性 SQL 取第一处不一致 + + +@pytest.mark.parametrize("slot,ranges", [ + ("19-20", "18-22 时、18-24 时"), # 中间缺一段 + ("18-19", "18-22 时、18-24 时"), # 缺起点 + ("23-24", "22-24 时、18-24 时"), # 缺终点 +]) +def test_k_tiling_unverifiable(tmp_path, slot, ranges): + def drop(rows): + rows["时段客流"] = [h for h in rows["时段客流"] if h["时段"] != slot] + + f = flow(tmp_path, rows_hook=drop) + k = by_id(run_checks(f.db, f.extraction, f.recipe))["K1"] + assert k.status == "unverifiable" and k.acceptable is True + assert k.reasons == {"tiling": 6} # 两个合计各 3 个日期；另一个合计照常一致 + assert k.checked == 3 and k.failed == 0 and k.unverifiable == 6 + assert f"6 格无法核对：明细时段没有恰好铺满 {ranges}" in k.details + + +def test_k_overlap_is_not_tiling(tmp_path): + """区间里多出一条跨时段的明细（18-20）：首尾相接不成立，不能判一致。""" + data = deepcopy(FLOW) + data["tables"][1]["grain"] = ["日期", "时段类别", "时段"] + + def extra(rows): + for d in ("2026-08-01", "2026-08-02", "2026-08-03"): + rows["时段客流"].append({"日期": d, "时段": "18-20", "时段类别": "夜间", "起始小时": 18, + "结束小时": 20, "客流": 0}) + + f = flow(tmp_path, recipe_data=data, rows_hook=extra) + k = by_id(run_checks(f.db, f.extraction, f.recipe))["K1"] + assert k.status == "unverifiable" and k.reasons == {"tiling": 6} + + +@pytest.mark.parametrize("is_formula,reason", [(True, "uncached"), (False, "blank")]) +def test_k_missing_value(tmp_path, is_formula, reason): + def blank(items): + for it in items: + it.value = None + it.is_formula = is_formula + + f = flow(tmp_path, items_hook=blank) + k = by_id(run_checks(f.db, f.extraction, f.recipe))["K1"] + assert k.status == "unverifiable" and k.reasons == {reason: 9} and k.acceptable + assert k.checked == 0 and k.unverifiable == 9 + + +def test_k_null_detail_never_passes(tmp_path): + """明细含空值（占位符）：哪怕保存的合计恰好等于非空明细之和，也只能判无法核对。""" + removed: dict[str, int] = {} + + def hole(rows): + for h in rows["时段客流"]: + if h["时段"] == "23-24" and h["日期"] == "2026-08-02": + removed["v"], h["客流"] = h["客流"], None + + def match_nonnull(items): + for it in items: + if it.key["日期"] == "2026-08-02" and it.end == 24: + it.value -= removed["v"] # 合计 = 非空明细之和：TOTAL 对得上，仍不能判一致 + + f = flow(tmp_path, rows_hook=hole, items_hook=match_nonnull) + k = by_id(run_checks(f.db, f.extraction, f.recipe))["K1"] + assert k.status == "unverifiable" and k.reasons == {"null_detail": 2} # 22-24、18-24 那天 + assert k.failed == 0 and k.checked == 7 + assert "2 格无法核对：明细含无数据占位符" in k.details + + +@pytest.mark.parametrize("mark", ["hidden_in_range", "SUBTOTAL", "AGGREGATE"]) +def test_k_hidden_turns_mismatch_into_unverifiable(tmp_path, mark): + def tweak(items): + it = items[0] + it.value += 5 + if mark == "hidden_in_range": + it.hidden_in_range = True + else: + it.func = mark + + f = flow(tmp_path, items_hook=tweak) + k = by_id(run_checks(f.db, f.extraction, f.recipe))["K1"] + assert k.status == "unverifiable" and k.reasons == {"hidden": 1} and k.acceptable + + +def test_k_hidden_but_equal_is_consistent(tmp_path): + def tweak(items): + for it in items: + it.hidden_in_range = True + + f = flow(tmp_path, items_hook=tweak) + assert by_id(run_checks(f.db, f.extraction, f.recipe))["K1"].status == "passed" + + +def test_k_key_with_const_column_ignores_other_category(tmp_path): + """同一张表里别的分段（别的类别）也有 18-22 时的行：key 带常量列，不会被加进来。""" + data = deepcopy(FLOW) + data["tables"][1]["grain"] = ["日期", "时段类别", "时段"] + + def other(rows): + # 别的类别按两小时一段：和夜间的逐小时混在一起就互相重叠、也会被加进合计 + for d in ("2026-08-01", "2026-08-02", "2026-08-03"): + for s in (18, 20, 22): + rows["时段客流"].append({"日期": d, "时段": f"{s}-{s + 2}", "时段类别": "其他", "起始小时": s, + "结束小时": s + 2, "客流": 999}) + + f = flow(tmp_path, recipe_data=data, rows_hook=other) + got = by_id(run_checks(f.db, f.extraction, f.recipe)) + assert got["K1"].status == "passed" and got["G1"].status == "passed" + + +def test_k_day_without_detail_rows_is_unverifiable(tmp_path): + """某个日期的夜间明细一行都没有（别的日期都有，所以按区间判的「铺满」照样成立）： + COUNT(*) = 0，TOTAL 是 0，不能拿它和保存的合计比，判无法核对（tiling），不判一致也不判不一致。""" + def drop_day(rows): + rows["时段客流"] = [h for h in rows["时段客流"] + if not (h["日期"] == "2026-08-02" and h["时段类别"] == "夜间")] + + f = flow(tmp_path, rows_hook=drop_day) + k = by_id(run_checks(f.db, f.extraction, f.recipe))["K1"] + assert k.status == "unverifiable" and k.acceptable + assert k.reasons == {"tiling": 3} and k.checked == 6 and k.failed == 0 + assert sorted(k.cells) == [f"{SHEET}!D28", f"{SHEET}!D29", f"{SHEET}!D30"] + + +def test_k_mixed_reasons_counted(tmp_path): + def mix(items): + items[0].value = None # 18-22 第 1 天：无缓存 + items[1].value += 1 + items[1].func = "SUBTOTAL" # 18-22 第 2 天：分类汇总 + items[3].value = None + items[3].is_formula = False # 22-24 第 1 天：空格 + + f = flow(tmp_path, items_hook=mix) + k = by_id(run_checks(f.db, f.extraction, f.recipe))["K1"] + assert k.status == "unverifiable" + assert k.reasons == {"uncached": 1, "hidden": 1, "blank": 1} + assert k.unverifiable == 3 and k.checked == 6 + assert any(d.startswith("1 格无法核对：文件未经计算保存") for d in k.details) + + +def test_k_mismatch_wins_over_unverifiable(tmp_path): + def mix(items): + items[0].value = None + items[2].value += 3 + + f = flow(tmp_path, items_hook=mix) + k = by_id(run_checks(f.db, f.extraction, f.recipe))["K1"] + assert k.status == "mismatch" and k.acceptable is False and k.reasons == {"uncached": 1} + + +def test_k_sql_error_is_structure_mismatch_not_crash(tmp_path): + """列名写错：DQS 关闭的连接直接报错，核对判结构类不一致，而不是把「客 流」当字符串算出 0。""" + def typo(items): + for it in items: + it.base_value = "客 流" + + f = flow(tmp_path, items_hook=typo) + got = by_id(run_checks(f.db, f.extraction, f.recipe)) + k = got["K1"] + assert k.status == "mismatch" and k.category == "structure" and not k.acceptable + assert k.details[0].startswith("核对 SQL 执行失败") and "no such column" in k.details[0] + assert k.sql and '"客 流"' in k.sql + assert got["R1"].status == "passed" # 别的核对照常 + + +# ---------------------------------------------------------------- G：公式引用 + + +def test_g_p9_label_and_refs_disagree(tmp_path): + def p9(items): + it = items[0] # C28 改成 =SUM(C22:C24)：只到 21 时 + it.ref_rowids = it.ref_rowids[:3] + + f = flow(tmp_path, items_hook=p9) + g = by_id(run_checks(f.db, f.extraction, f.recipe))["G1"] + assert g.status == "mismatch" and g.category == "structure" + assert g.details[0] == "C28：标签「18-22时合计」与公式引用的时段不一致" + assert g.cells == [f"{SHEET}!C28"] + + +@pytest.mark.parametrize("extra_slot,category", [ + ("22-23", "夜间"), # 同一类别里范围外的一行：「18-22 时合计」把 22-23 时也引用进来 + ("17-18", "日间"), # 别的类别的一行 +]) +def test_g_refs_superset_of_label_range_is_mismatch(tmp_path, extra_slot, category): + """公式引用比标签范围多出一行（如「8-12时合计」=SUM(C10:C14) 把整行占位符的 7-8 时也引进来）： + K 按标签区间重算照样一致，能拦住的只有 G 的集合相等——「引用 ⊆ 标签范围」不够，必须相等。""" + seen: dict[str, list[dict]] = {} + + def keep(rows): + seen["rows"] = rows["时段客流"] + + def widen(items): + it = items[0] # C28「18-22时合计」第 1 个日期 + extra = next(n for n, h in enumerate(seen["rows"], 1) + if h["日期"] == it.key["日期"] and h["时段"] == extra_slot and h["时段类别"] == category) + it.ref_rowids = [*it.ref_rowids, extra] + + f = flow(tmp_path, rows_hook=keep, items_hook=widen) + got = by_id(run_checks(f.db, f.extraction, f.recipe)) + g = got["G1"] + assert g.status == "mismatch" and g.category == "structure" and not g.acceptable + assert g.failed == 1 and g.cells == [f"{SHEET}!C28"] + assert g.details[0] == "C28：标签「18-22时合计」与公式引用的时段不一致" + assert got["K1"].status == "passed" + + +@pytest.mark.parametrize("problem,status,reason", [ + ("unrecognized", "unverifiable", "unrecognized"), ("hidden", "unverifiable", "hidden"), + ("outside", "mismatch", None), +]) +def test_g_ref_problems(tmp_path, problem, status, reason): + def mark(items): + items[0].ref_problem = problem + items[0].ref_rowids = None + + f = flow(tmp_path, items_hook=mark) + g = by_id(run_checks(f.db, f.extraction, f.recipe))["G1"] + assert g.status == status + assert g.acceptable is (status == "unverifiable") + assert g.reasons == ({reason: 1} if reason else {}) + + +def test_g_absent_when_no_formula(tmp_path): + def literal(items): + for it in items: + it.is_formula, it.formula, it.func, it.ref_rowids = False, None, None, None + + f = flow(tmp_path, items_hook=literal) + ids = [c.id for c in run_checks(f.db, f.extraction, f.recipe)] + assert "K1" in ids and "G1" not in ids + + +def test_uncached_formula_still_checks_refs(tmp_path): + """P1b：公式无缓存——K1 无法核对（uncached），G1 照常核对引用并通过。""" + def p1b(items): + for it in items: + it.value = None + + f = flow(tmp_path, items_hook=p1b) + got = by_id(run_checks(f.db, f.extraction, f.recipe)) + assert got["K1"].status == "unverifiable" and got["K1"].reasons == {"uncached": 9} + assert got["G1"].status == "passed" + + +# ---------------------------------------------------------------- T：列表合计 + + +LIST_RECIPE = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": "销售"}, "blocks": [{ + "id": "明细", "layout": "list", "table": "销售", + "columns": [{"header": "地区", "name": "地区", "type": "TEXT"}, + {"header": "金额", "name": "金额", "type": "INTEGER"}, + {"header": "数量", "name": "数量", "type": "REAL"}], + "rows": {"total_row": {"label_column": "地区", "pick": "合计", "keep_as": "销售_表内合计"}}}]}], + "tables": [{"name": "销售", "grain": ["地区"]}, {"name": "销售_表内合计", "kind": "reported_total"}], +} + + +def list_case(tmp_path, *, hook=None, null_amount=False, hidden=False): + recipe = Recipe.model_validate(deepcopy(LIST_RECIPE)) + sales = [{"地区": f"分区{n}", "金额": 100 * i, "数量": 0.1 * i} for i, n in enumerate("甲乙丙", 1)] + if null_amount: + sales[1]["金额"] = None + totals = [{"合计项": "合计", "金额": 600, "数量": 0.6}] + make_db(tmp_path / "list.db", recipe, {"销售": sales, "销售_表内合计": totals}) + items = [DerivedItem(kind="column_sum", segment="明细", sheet="销售", cell=f"{c}5", label_raw="合计", + label="合计", value=v, is_formula=False, formula=None, base_table="销售", + base_value=col, rowid_first=1, rowid_last=3, hidden_in_range=hidden, + keep_table="销售_表内合计") + for c, col, v in (("B", "金额", 600), ("C", "数量", 0.6))] + if hook: + hook(items) + ex = Extraction(ok=True, derived=items, expected_rows={"销售": 3, "销售_表内合计": 1}) + return recipe, str(tmp_path / "list.db"), ex + + +def test_t_passed_with_real_tolerance(tmp_path): + recipe, db, ex = list_case(tmp_path) + got = by_id(run_checks(db, ex, recipe)) + assert [c.id for c in run_checks(db, ex, recipe)] == ["T1", "N1", "N2", "P1"] + t = got["T1"] + assert t.kind == "column_sum" and t.status == "passed" and t.checked == 2 # 0.1+0.2+0.3 ≈ 0.6 + assert t.sql == 'SELECT TOTAL("金额"), COUNT("金额"), COUNT(*) FROM "销售" WHERE rowid BETWEEN ? AND ?' + assert t.params == [1, 3] + + +def test_t_mismatch(tmp_path): + def bump(items): + items[0].value = 601 + + recipe, db, ex = list_case(tmp_path, hook=bump) + t = by_id(run_checks(db, ex, recipe))["T1"] + assert t.status == "mismatch" and t.details[0] == "B5：表内为 601，按明细求和为 600" + + +def test_t_uncached(tmp_path): + def unc(items): + items[0].value, items[0].is_formula, items[0].formula = None, True, "=SUM(B2:B4)" + + recipe, db, ex = list_case(tmp_path, hook=unc) + t = by_id(run_checks(db, ex, recipe))["T1"] + assert t.status == "unverifiable" and t.reasons == {"uncached": 1} and t.acceptable + + +def test_t_null_detail(tmp_path): + recipe, db, ex = list_case(tmp_path, null_amount=True) + t = by_id(run_checks(db, ex, recipe))["T1"] + assert t.status == "unverifiable" and t.reasons == {"null_detail": 1} + assert "1 格无法核对：明细含空值" in t.details + + +def test_t_hidden_rows_turn_mismatch_unverifiable(tmp_path): + def bump(items): + items[0].value = 500 + + recipe, db, ex = list_case(tmp_path, hook=bump, hidden=True) + t = by_id(run_checks(db, ex, recipe))["T1"] + assert t.status == "unverifiable" and t.reasons == {"hidden": 1} + + +def test_t_without_detail_rows_is_unverifiable(tmp_path): + """表头下面紧接着就是合计行：执行器给的明细区间两端都是 None。不能拿 BETWEEN NULL AND NULL 算出的 0 + 去比（合计写 0 时会被判「2 格全部一致」），判无法核对。""" + def empty(items): + for it in items: + it.rowid_first = it.rowid_last = None + it.value = 0 + + recipe, db, ex = list_case(tmp_path, hook=empty) + t = by_id(run_checks(db, ex, recipe))["T1"] + assert t.status == "unverifiable" and t.acceptable and t.category == "structure" + assert t.reasons == {"tiling": 2} and t.checked == 0 + assert "2 格无法核对：合计行之上没有明细行" in t.details + assert all("一致" not in d for d in t.details) + assert t.params == [None, None] + + +def test_t_empty_rowid_range_is_unverifiable(tmp_path): + def far(items): + for it in items: + it.rowid_first, it.rowid_last = 10, 12 + + recipe, db, ex = list_case(tmp_path, hook=far) + t = by_id(run_checks(db, ex, recipe))["T1"] + assert t.status == "unverifiable" and t.reasons == {"tiling": 2} + assert t.params == [10, 12] + + +@pytest.mark.parametrize("first,last", [(None, 3), (1, None)]) +def test_t_half_open_rowid_range_is_structure_failure(tmp_path, first, last): + """区间只有一端：执行器的错，和 K 缺区间信息一样判结构类不一致，不硬算。""" + def half(items): + items[0].rowid_first, items[0].rowid_last = first, last + + recipe, db, ex = list_case(tmp_path, hook=half) + t = by_id(run_checks(db, ex, recipe))["T1"] + assert t.status == "mismatch" and t.category == "structure" and not t.acceptable + assert t.details == ["B5：合计行对应的明细行区间只有一端"] + + +# ---------------------------------------------------------------- R：关系 + + +def test_r_sum_eq_mismatch_located_by_lineage(tmp_path): + def seven(rows): + rows["日客流"][1]["全日客流"] += 7 # 第 2 个日期 + + f = flow(tmp_path, rows_hook=seven) + r1 = by_id(run_checks(f.db, f.extraction, f.recipe))["R1"] + assert r1.status == "mismatch" and r1.category == "data_quality" and r1.acceptable + assert r1.checked == 3 and r1.failed == 1 + assert r1.cells == [f"{SHEET}!D5"] + row = f.rows["日客流"][1] + assert r1.details[0] == (f"日期 2026-08-02（D5）：全日客流 为 {row['全日客流']}，" + f"分区甲、分区乙 之和为 {row['全日客流'] - 7}") + + +def test_r_sum_eq_nulls_unverifiable(tmp_path): + def hole(rows): + rows["日客流"][0]["分区甲"] = None + + f = flow(tmp_path, rows_hook=hole) + r1 = by_id(run_checks(f.db, f.extraction, f.recipe))["R1"] + assert r1.status == "unverifiable" and r1.acceptable and r1.category == "data_quality" + assert r1.checked == 2 and r1.unverifiable == 1 and r1.failed == 0 + + +def test_r_sum_eq_no_rows_is_unverifiable_not_passed(tmp_path): + """AU-1：表里一行都没有（只有表头）：一行都没核对过，不能判通过，也不能写「0 行全部成立」。""" + def empty(rows): + rows["日客流"].clear() + + f = flow(tmp_path, rows_hook=empty) + r1 = by_id(run_checks(f.db, f.extraction, f.recipe))["R1"] + assert r1.status == "unverifiable" and r1.acceptable and r1.category == "data_quality" + assert r1.checked == 0 and r1.unverifiable == 0 and r1.failed == 0 + assert r1.details == ["本期这张表没有数据行，关系未能核对"] + + +def test_r_sum_eq_real_uses_tolerance(tmp_path): + data = deepcopy(FLOW) + data["sheets"][0]["blocks"][0]["values"]["type"] = "REAL" + + def floats(rows): + for r in rows["日客流"]: + r["分区甲"], r["分区乙"], r["全日客流"] = 0.1, 0.2, 0.3 + for t in ("时段客流", "时段客流_表内合计"): + for r in rows[t]: + r["客流"] = None if r["客流"] is None else float(r["客流"]) + + f = flow(tmp_path, recipe_data=data, rows_hook=floats) + r1 = by_id(run_checks(f.db, f.extraction, f.recipe))["R1"] + assert r1.status == "passed", r1.details + assert "ABS(" in r1.sql + # 同样的数按整数写法（<>）比，就会误判不成立 + conn = sqlite3.connect(f.db) + assert conn.execute('SELECT COUNT(*) FROM "日客流" WHERE "全日客流" <> "分区甲" + "分区乙"').fetchone()[0] == 3 + conn.close() + + +def test_r_not_comparable_all_equal(tmp_path): + def equalize(rows): + for r in rows["日客流"]: + r["全日客流"] = sum(h["客流"] or 0 for h in rows["时段客流"] if h["日期"] == r["日期"]) + r["分区甲"] = r["全日客流"] - r["分区乙"] + + f = flow(tmp_path, rows_hook=equalize) + got = by_id(run_checks(f.db, f.extraction, f.recipe)) + r2 = got["R2"] + assert r2.status == "info" and r2.category == "info" + assert r2.details == ["各时段之和与全日客流：3 天全部相等"] + assert r2.checked == 3 and r2.failed == 0 + assert got["R1"].status == "passed" + + +def test_r_not_comparable_partly_equal_stays_info(tmp_path): + def one(rows): + r = rows["日客流"][0] + r["全日客流"] = sum(h["客流"] or 0 for h in rows["时段客流"] if h["日期"] == r["日期"]) + + f = flow(tmp_path, rows_hook=one) + r2 = by_id(run_checks(f.db, f.extraction, f.recipe))["R2"] + assert r2.status == "info" and r2.details == ["各时段之和与全日客流：3 天中 1 天相等"] and r2.failed == 2 + + +# ---------------------------------------------------------------- N、P + + +def test_n_row_count_mismatch(tmp_path): + f = flow(tmp_path) + f.extraction.expected_rows["时段客流"] += 1 + n2 = by_id(run_checks(f.db, f.extraction, f.recipe))["N2"] + assert n2.status == "mismatch" and n2.category == "structure" + assert n2.details == ["库里 51 行，按格子账应有 52 行"] + + +def test_p_duplicate_key(tmp_path): + def dup(rows): + rows["日客流"].append(dict(rows["日客流"][0])) + + f = flow(tmp_path, rows_hook=dup, pk=False) + got = by_id(run_checks(f.db, f.extraction, f.recipe)) + assert got["P1"].status == "mismatch" and got["P1"].details == ["1 组主键重复"] + assert got["P2"].status == "passed" + + +# ---------------------------------------------------------------- 编号与空值统计 + + +def test_plan_numbering_follows_recipe_order(tmp_path): + f = flow(tmp_path) + plan = plan_checks(f.recipe, f.extraction) + assert plan.k == {"夜间合计": "K1"} and plan.g == {"夜间合计": "G1"} and plan.t == {} + assert len(plan.items["夜间合计"]) == 9 + + +def test_column_null_counts(tmp_path): + f = flow(tmp_path) + counts = column_null_counts(f.db, f.recipe) + assert counts["时段客流"]["客流"] == 3 # 7-8 那一行整期是占位符 + assert counts["日客流"] == {"日期": 0, "全日客流": 0, "分区甲": 0, "分区乙": 0} + + +# ---------------------------------------------------------------- 核对结果 → 说明（同一套编号） + + +def test_checks_feed_notes_end_to_end(tmp_path): + from app.data.recipe_notes import build_notes + from app.data.recipe_types import Acceptance, AxisOut, SheetsOut + + def p1b(items): + for it in items: + it.value = None + + f = flow(tmp_path, items_hook=p1b) + f.extraction.placeholders = {"·": 3} + f.extraction.sheets = SheetsOut(matched={"s1": SHEET}) + f.extraction.axes = [AxisOut("交叉表", 4, "8月1日", "8月3日", 3, "text")] + checks = run_checks(f.db, f.extraction, f.recipe) + counts = column_null_counts(f.db, f.recipe) + pending = build_notes(f.recipe, f.extraction, checks, [], null_counts=counts) + assert pending.problems == [] # 未接受的「无法核对」只影响状态，不让说明生成失败 + assert by_id(checks)["K1"].acceptable + notes = build_notes(f.recipe, f.extraction, checks, [Acceptance("K1", "原表未经计算保存")], null_counts=counts) + assert notes.problems == [] + total = notes.tables["时段客流_表内合计"].comment + assert "本期原表未保存计算结果，合计值为空，未能核对" in total and "一致" not in total + assert "已逐日核对" in notes.tables["日客流"].comment + assert notes.tables["日客流"].columns["全日客流"] == "单位：人次" + assert "无数据占位符" in notes.tables["时段客流"].columns["客流"] + + +def test_structure_mismatch_feeds_notes_without_problem(tmp_path): + """P9：K1、G1 都不一致，试运行因结构类核对拒收。说明不写合计那一句、也不额外记「说明没有对应模板」， + 免得被当成配方问题显示。""" + from app.data.recipe_notes import build_notes + from app.data.recipe_types import SheetsOut + + def p9(items): + it = items[0] + it.ref_rowids = it.ref_rowids[:3] + it.value += 11 + + f = flow(tmp_path, items_hook=p9) + f.extraction.sheets = SheetsOut(matched={"s1": SHEET}) + checks = by_id(run_checks(f.db, f.extraction, f.recipe)) + assert checks["K1"].status == "mismatch" and checks["G1"].status == "mismatch" + notes = build_notes(f.recipe, f.extraction, list(checks.values()), [], + null_counts=column_null_counts(f.db, f.recipe)) + assert notes.problems == [] + total = notes.tables["时段客流_表内合计"].comment + assert "原表写明的合计" not in total and "一致" not in total + + +def _list_notes_from_db(tmp_path, hook): + from app.data.recipe_notes import build_notes + from app.data.recipe_types import Acceptance + + recipe, db, ex = list_case(tmp_path, hook=hook) + checks = run_checks(db, ex, recipe) + acc = [Acceptance(c.id, "已与业务方核实") for c in checks if c.acceptable] + return by_id(checks), build_notes(recipe, ex, checks, acc, null_counts=column_null_counts(db, recipe)) + + +def test_list_total_placeholder_cell_gets_blank_template(tmp_path): + """合计行的 金额 格写了已声明的占位符：非空格照样生成合计格（P2-SPEC 4.4），值转成空值（4.5）， + T 判无法核对（blank）。说明要有对应的模板，不能记「没有对应的说明」把合法文件当配方问题拒收。""" + def placeholder(items): + items[0].value = None + + checks, notes = _list_notes_from_db(tmp_path, placeholder) + assert checks["T1"].status == "unverifiable" and checks["T1"].reasons == {"blank": 1} + assert notes.problems == [] + total = notes.tables["销售_表内合计"].comment + assert total == "原表合计行的值。本期原表合计格为空，未能核对：不要与 销售 相加" + assert "一致" not in total + + +def test_list_total_blank_cell_not_claimed_as_checked(tmp_path): + """合计行的 数量 格是空的：不生成合计格，T 只核对 金额 并通过。说明不能写「各列都核对一致」。""" + def drop(items): + del items[1] + + checks, notes = _list_notes_from_db(tmp_path, drop) + assert checks["T1"].status == "passed" and checks["T1"].checked == 1 + assert notes.problems == [] + assert notes.tables["销售_表内合计"].comment == ( + "原表合计行的值。本期原表 数量 的合计格为空，其余各列已按明细求和核对一致：不要与 销售 相加") diff --git a/backend/tests/test_recipe_confirm.py b/backend/tests/test_recipe_confirm.py new file mode 100644 index 00000000..bd7376b3 --- /dev/null +++ b/backend/tests/test_recipe_confirm.py @@ -0,0 +1,757 @@ +"""确认项（recipe_confirm）的测试：7.5 每行一正一反、AI 类、破坏性变更、切换、本期类。 + +回执和配方都是手写的（假名、假数），构造器在 test_recipe_diff 里；不依赖执行器、核对、静态校验（各包的桩 +都不需要：confirm_items 只读回执和配方）。 +""" +from __future__ import annotations + +import copy +from typing import Any + +import pytest + +from app.data.recipe_confirm import ( + EXTRACTION_KEYS, ConfirmContext, breaking_changes, confirm_items, recipe_differs, switch_changes, table_changes, +) +from app.data.recipe_diff import ReceiptIncomplete, diff_reports +from app.data.recipe_types import DiffItem, Recipe +from app.data.recipe_parsers import match_key, period_residue +from tests.test_recipe_diff import ( + AUG, BASE, D00_CONFIRMS, FLOW, SEP, SHEET, add_outside, extraction_of, file_name, flow_doc, sep, set_period_cell, +) + +LIST: dict[str, Any] = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": "销售"}, "blocks": [{ + "id": "列表1", "layout": "list", "table": "销售", + "columns": [{"header": "地区", "name": "地区", "type": "TEXT"}, + {"header": "产品", "name": "产品", "type": "TEXT"}, + {"header": "日期", "name": "日期", "type": "DATE"}, + {"header": "销量（件）", "name": "销量", "type": "INTEGER"}, + {"header": "金额（元）", "name": "金额", "type": "REAL"}], + "rows": {"total_row": {"label_column": "地区", "pick": "合计", "keep_as": "销售_表内合计"}}, + }]}], + "tables": [{"name": "销售", "grain": ["地区", "产品"], "units": {"销量": "件", "金额": "元"}}, + {"name": "销售_表内合计", "grain": ["合计项"], "kind": "reported_total", + "units": {"销量": "件", "金额": "元"}}], +} + + +def flow(**patch: Any) -> dict[str, Any]: + r = copy.deepcopy(FLOW) + for path, value in patch.items(): + cur = r + keys = path.split("__") + for k in keys[:-1]: + cur = cur[int(k)] if k.isdigit() else cur[k] + last = keys[-1] + cur[int(last) if last.isdigit() else last] = value + return r + + +def lst(**patch: Any) -> dict[str, Any]: + r = copy.deepcopy(LIST) + block = r["sheets"][0]["blocks"][0] + for k, v in patch.items(): + block[k] = v + return r + + +def list_ex(**extra: Any) -> dict[str, Any]: + ex = {"ok": True, "problems": [], "tables": [], "placeholders": {}, "labels": [], "outside_text": [], + "sheets": {"matched": {"s1": "销售"}, "renamed": {}, "other_visible": [], "skipped_hidden": []}, + "derived": [{"kind": "column_sum", "segment": "列表1", "sheet": "销售", "cell": "F12"}], + "axes": [], "hidden": {}, "ignored_columns": {}, "formula_cells_accepted": 0, "blank_rows_skipped": 0, + "period": None, "lineage": {}} + ex.update(extra) + return ex + + +def ids(ctx: ConfirmContext) -> set[str]: + return {i.id for i in confirm_items(ctx)} + + +def first(recipe: dict[str, Any], ex: dict[str, Any] | None = None, **kw: Any) -> ConfirmContext: + kw.setdefault("checks", []) + return ConfirmContext(kind=kw.pop("kind", "first"), recipe=recipe, + extraction=ex if ex is not None else extraction_of(BASE[0]), **kw) + + +def label(ctx: ConfirmContext, item_id: str) -> str: + return next(i.label for i in confirm_items(ctx) if i.id == item_id) + + +# -------------------------------------------------------------------------- +# 配方类：每行一正一反 +# -------------------------------------------------------------------------- + + +def test_reference_recipe_first_import_exact(): + items = confirm_items(first(FLOW)) + assert {i.id for i in items} == D00_CONFIRMS + got = {i.id: i for i in items} + assert got["placeholder:·"].label == "「·」存为空值（表示无数据，本期 31 格）" + assert got["derived:夜间合计"].label == "第 28–30 行改作核对，原值另存表「时段客流_表内合计」" + assert got["relation:R1"].label == "每期核对「全日客流 = 分区甲 + 分区乙」（表「日客流」）" + assert got["unit:日客流.全日客流"].label == "「全日客流（人次）」→ 表「日客流」的列「全日客流」，单位 人次" + assert got["mode"].label == "导入模式：每期替换" + assert all(i.required and i.source == "recipe" for i in items) + + +def test_placeholder_counts_and_absence(): + r = flow() + r["sheets"][0]["blocks"][0]["values"]["placeholders"] = [] + assert not any(i.startswith("placeholder:") for i in ids(first(r))) + r["sheets"][0]["blocks"][0]["values"]["placeholders"] = [{"text": "·"}, {"text": "-", "meaning": "不适用"}] + ctx = first(r) + assert label(ctx, "placeholder:-") == "「-」存为空值（表示不适用，本期 0 格）" # 配置了就出，个数可以是 0 + + +@pytest.mark.parametrize("field,value,item", [ + ("blank", "null", "blank_null:交叉表"), + ("text_number", "parse_thousands", "text_number:交叉表"), + ("formula", "accept_cached", "formula_cached:交叉表"), +]) +def test_crosstab_value_relaxations(field, value, item): + assert item not in ids(first(FLOW)) + r = flow() + r["sheets"][0]["blocks"][0]["values"][field] = value + assert item in ids(first(r)) + + +def test_formula_cached_list_only_when_formulas_seen(): + assert "formula_cached:列表1" not in ids(first(LIST, list_ex())) + ctx = first(LIST, list_ex(formula_cells_accepted=3)) + assert label(ctx, "formula_cached:列表1") == "「列表1」中的公式格按文件里保存的值导入（本期 3 格）" + assert "formula_cached:列表1" not in ids(first(lst(values={"formula": "reject"}), list_ex(formula_cells_accepted=3))) + + +def test_merged_fill(): + assert "merged_fill:列表1" not in ids(first(LIST, list_ex())) + assert "merged_fill:列表1" in ids(first(lst(merged_data="fill"), list_ex())) + + +def test_text_number_list(): + assert "text_number:列表1" in ids(first(lst(values={"text_number": "parse_thousands"}), list_ex())) + + +def test_checks_relaxed(): + assert not any(i.startswith("checks_relaxed:") for i in ids(first(FLOW))) + r = flow() + r["sheets"][0]["blocks"][0]["axis"]["checks"] = ["contiguous"] + assert label(first(r), "checks_relaxed:交叉表") == "「交叉表」不检查日期「恰好覆盖统计期」" + r["sheets"][0]["blocks"][0]["axis"]["checks"] = [] + assert "「逐日连续」、「恰好覆盖统计期」" in label(first(r), "checks_relaxed:交叉表") + + +def test_cross_check_off_uses_actual_sheet_name(): + assert not any(i.startswith("cross_check_off:") for i in ids(first(FLOW))) + r = flow() + r["sheets"][0]["context"][0]["cross_check"] = "none" + assert "cross_check_off:客流汇总" in ids(first(r)) + ex = extraction_of(BASE[0]) + ex["sheets"]["matched"] = {"s1": "汇总"} + assert "cross_check_off:汇总" in ids(first(r, ex)) + + +def test_ignored_columns_only_when_something_ignored(): + assert "ignored_columns:列表1" not in ids(first(lst(extra_columns="ignore"), list_ex())) + ex = list_ex(ignored_columns={"列表1": ["备注", "经办人"]}) + assert label(first(lst(extra_columns="ignore"), ex), "ignored_columns:列表1") == "「列表1」不导入这些列：「备注」「经办人」" + assert "ignored_columns:列表1" not in ids(first(LIST, ex)) + + +def test_blank_skip(): + assert "blank_skip:列表1" not in ids(first(LIST, list_ex())) + r = lst(rows={"blank_rows": "skip"}) + assert label(first(r, list_ex(blank_rows_skipped=2)), "blank_skip:列表1") == "「列表1」跳过数据中间的空行（本期 2 行）" + # 配置了就出，本期 0 行也出：首期恰好没有空行时不出，之后几期真跳过了行、配方又没变，就没人确认过了 + assert label(first(r, list_ex(blank_rows_skipped=0)), "blank_skip:列表1") == "「列表1」跳过数据中间的空行（本期 0 行）" + + +def test_total_row_and_derived(): + ctx = first(LIST, list_ex()) + assert label(ctx, "total_row:列表1") == "「列表1」的合计行（第 12 行）改作核对，原值另存表「销售_表内合计」" + detail = next(i.detail for i in confirm_items(ctx) if i.id == "total_row:列表1") + assert detail == "按明细核对的列：销量、金额；其余列合计行里的内容不核对" # SE-10 + assert "total_row:列表1" not in ids(first(lst(rows={}), list_ex())) + r = flow() + r["sheets"][0]["blocks"][0]["segments"][3]["keep_as"] = None + r["tables"].pop() + assert label(first(r), "derived:夜间合计").endswith("改作核对，原值不另存") + r["sheets"][0]["blocks"][0]["segments"].pop() + r["relations"] = [] + assert "derived:夜间合计" not in ids(first(r)) + + +def test_year_from(): + assert "year_from:交叉表" in ids(first(FLOW)) + ex = extraction_of(BASE[0]) + ex["axes"][0]["form"] = "date" # 日期格自带年份 + assert "year_from:交叉表" not in ids(first(FLOW, ex)) + r = flow() + r["sheets"][0]["blocks"][0]["axis"]["year_from"] = None + assert "year_from:交叉表" not in ids(first(r)) + + +def test_relations_and_dismissed(): + r = flow() + r["relations"] = [{"id": "R1", "kind": "dismissed", "claims": "F1", "reason": "巧合"}] + facts = {"facts": [{"id": "F1", "kind": "sum_eq", "sheet": SHEET, "text": "第 5 行 = 第 6 行 + 第 7 行", "detail": {}}]} + got = ids(first(r, facts=facts)) + assert "dismissed:F1" in got and not any(i.startswith("relation:") for i in got) + assert label(first(r, facts=facts), "dismissed:F1") == "不登记「第 5 行 = 第 6 行 + 第 7 行」（理由：巧合）" + assert label(first(r), "dismissed:F1") == "不登记系统发现的关系 F1（理由：巧合）" + assert "relation:R2" in ids(first(FLOW)) + assert "口径不同" in label(first(FLOW), "relation:R2") + + +def test_units_each_column(): + r = flow() + for t in r["tables"]: + t["units"] = {} + assert not any(i.startswith("unit:") for i in ids(first(r))) + list_ids = ids(first(LIST, list_ex())) + assert {"unit:销售.销量", "unit:销售.金额", "unit:销售_表内合计.销量", "unit:销售_表内合计.金额"} <= list_ids + + +def test_hidden_policy(): + assert not any(i.startswith("hidden:") for i in ids(first(FLOW))) + r = flow() + r["sheets"][0]["hidden"] = {"rows": "exclude", "cols": "include"} + ex = extraction_of(BASE[0]) + ex["hidden"] = {SHEET: {"rows": [4, 9], "cols": [3], "policy_rows": "exclude", "policy_cols": "include"}} + assert label(first(r, ex), "hidden:客流汇总") == "工作表「客流汇总」的隐藏行：排除（第 4、9 行）；隐藏列：包含（C 列）" + r["sheets"][0]["hidden"] = {"rows": "include"} + assert "本期没有隐藏行" in label(first(r), "hidden:客流汇总") + + +def test_recipe_class_only_when_recipe_differs(): + ex = extraction_of(sep()[0]) + # 同一份配方（一份写全默认值、一份省略）：canonical 相同，配方类全不出 + explicit = Recipe.model_validate(FLOW).model_dump(mode="json") + compact = Recipe.model_validate(FLOW).model_dump(mode="json", exclude_defaults=True) + assert not recipe_differs("reupload", explicit, compact) + assert ids(ConfirmContext(kind="reupload", recipe=explicit, old_recipe=compact, extraction=ex, checks=[], diff=[], + prev=BASE[0])) == set() + # 改了配方：配方类全部重新出 + r = flow() + r["sheets"][0]["blocks"][0]["values"]["blank"] = "null" + got = ids(ConfirmContext(kind="redraft", recipe=r, old_recipe=FLOW, extraction=ex, checks=[], diff=[], + prev=BASE[0])) + assert D00_CONFIRMS | {"blank_null:交叉表"} <= got + # 首次、切换、没有现行配方都算不同 + assert recipe_differs("first", FLOW, FLOW) and recipe_differs("switch", FLOW, None) + assert recipe_differs("reupload", FLOW, None) + + +# -------------------------------------------------------------------------- +# AI 类 +# -------------------------------------------------------------------------- + + +def test_ai_origin_items(): + r = flow() + r["sheets"][0]["blocks"][0]["segments"][0]["measures"]["全日客流（人次）"] = "全天客流" + r["tables"][0]["units"] = {"全天客流": "人次", "分区甲": "人次", "分区乙": "人次"} + r["relations"][0]["total"] = "全天客流" + r["relations"][1]["b"]["value"] = "全天客流" + got = ids(first(r, recipe_origin="ai")) + assert {"ai_origin", "rename:日客流.全天客流", "columns:日客流", "columns:时段客流", "columns:时段客流_表内合计"} <= got + assert not any(i.startswith("rename:日客流.分区") for i in got) # 按标签推出来的名字不算改名 + assert label(first(r, recipe_origin="ai"), "rename:日客流.全天客流") == "原标签「全日客流（人次）」改名为列「全天客流」" + # 每列带类型和单位（SE-10） + assert label(first(r, recipe_origin="ai"), "columns:时段客流") == ( + "表「时段客流」的列：日期（日期）、时段（文字）、时段类别（文字）、起始小时（整数）、结束小时（整数）、客流（整数，人次）") + assert "ai_origin" in ids(first(r, recipe_origin="mixed")) + assert not {i for i in ids(first(r, recipe_origin="rules")) if i.startswith(("ai_", "rename:", "columns:"))} + assert not {i for i in ids(first(r, recipe_origin="manual")) if i.startswith(("ai_", "rename:", "columns:"))} + + +def test_ai_items_not_repeated_on_reupload(): + # 现行配方出自 AI：每月重传配方没变，不再要求 AI 类确认 + got = ids(ConfirmContext(kind="reupload", recipe=FLOW, old_recipe=FLOW, extraction=extraction_of(sep()[0]), + checks=[], diff=[], prev=BASE[0], recipe_origin="ai")) + assert "ai_origin" not in got + + +def test_ai_rename_list_columns(): + r = lst() + r["sheets"][0]["blocks"][0]["columns"][4]["name"] = "销售额" + r["tables"][0]["units"] = {"销量": "件", "销售额": "元"} + r["tables"][1]["units"] = {"销量": "件", "销售额": "元"} + got = ids(first(r, list_ex(), recipe_origin="ai")) + assert "rename:销售.销售额" in got + assert "rename:销售.金额" not in got and "rename:销售.销量" not in got # 「销量（件）」→ 销量 不算改名 + + +# -------------------------------------------------------------------------- +# 破坏性变更 +# -------------------------------------------------------------------------- + + +def test_breaking_changes_scope(): + new = flow() + new["sheets"][0]["blocks"][0]["values"]["type"] = "REAL" # 类型 + new["tables"][1]["units"] = {"客流": "人"} # 单位 + new["tables"][0]["grain"] = ["日期", "全日客流"] # 主键 + new["tables"][2]["kind"] = "data" # 种类 + new["sheets"][0]["blocks"][0]["values"]["placeholders"] = [{"text": "·", "meaning": "不适用"}] # 占位符含义 + out = breaking_changes(FLOW, new) + assert "列「全日客流」类型 INTEGER → REAL" in out["日客流"] + assert "列「客流」的单位 人次 → 人" in out["时段客流"] + assert "主键 日期 → 日期、全日客流" in out["日客流"] + assert "表的种类 原表合计 → 明细数据" in out["时段客流_表内合计"] + assert "值列的占位符「·」含义 无数据 → 不适用" in out["时段客流"] + assert breaking_changes(FLOW, FLOW) == {} + + +def test_breaking_changes_const_pick(): + """分段标题改了写法、常量只能另选候选词（D09：「日间分时段客流」的候选词只有「日间分」）：列名、类型都没变， + 这一列的值却整列变了。必须进 breaking:<表> 让人确认，不能静默换值。""" + new = flow() + seg = new["sheets"][0]["blocks"][0]["segments"][1] + seg["locate"]["title"] = "日间分时段客流（人次）" + seg["const"]["时段类别"]["pick"] = "日间分" + out = breaking_changes(FLOW, new) + assert out == {"时段客流": [f"分段「{seg['id']}」的列「时段类别」取值「日间」→「日间分」"]} + assert [c.kind for c in table_changes(FLOW, new)] == ["const_value"] + ex = extraction_of(sep()[0]) + got = [i.id for i in confirm_items(ConfirmContext(kind="redraft", recipe=new, old_recipe=FLOW, extraction=ex, + checks=[], diff=[], prev=BASE[0]))] + assert "breaking:时段客流" in got + # 只改了标题、pick 不变：不算 + same = flow() + same["sheets"][0]["blocks"][0]["segments"][1]["locate"]["title"] = "日间时段客流" + assert breaking_changes(FLOW, same) == {} + + +def test_ai_columns_label_shows_text_type_of_a_numeric_column(): + """SE-10：AI 把列表的金额列声明成 TEXT 能过静态校验：columns:<表> 的文案里看得见「文字」。""" + r = copy.deepcopy(LIST) + col = next(c for c in r["sheets"][0]["blocks"][0]["columns"] if c["name"] == "金额") + col["type"] = "TEXT" + text = label(first(r, list_ex(), recipe_origin="ai"), "columns:销售") + assert "金额（文字，元）" in text and "销量（整数，件）" in text + + +def test_dismissed_reason_marked_as_possibly_ai_written(): + """SE-10：AI（或 AI 起草后改过的）配方里，dismissed 的理由标明可能出自 AI；规则起草的不标。""" + r = flow() + r["relations"][0] = {"id": "R1", "kind": "dismissed", "claims": "F1", "reason": "巧合，无需核对"} + for origin, marked in (("ai", True), ("mixed", True), ("rules", False), ("manual", False)): + text = next(i.label for i in confirm_items(first(r, recipe_origin=origin)) if i.id == "dismissed:F1") + assert ("可能由 AI 填写" in text) is marked, (origin, text) + + +def test_mask_lost_when_switch_or_redraft_drops_a_masked_column(): + """AU-6：源上遮罩的列在新结构里没有同名列（不区分大小写）→ 必勾 mask_lost:<列>；旧结构里本来就没有的不报。""" + old_cache = {"tables": {"销售": {"columns": [{"name": "地区", "type": "TEXT"}, {"name": "金额_万元", "type": "REAL"}]}}} + ctx = first(LIST, list_ex(), kind="switch", old_schema_cache=old_cache, mask_columns=["金额_万元", "不存在的列"]) + items = {i.id: i for i in confirm_items(ctx)} + assert "mask_lost:金额_万元" in items and items["mask_lost:金额_万元"].source == "switch" + assert "mask_lost:不存在的列" not in items + # 新结构里有同名列（大小写不同也算）：不报 + ctx = first(LIST, list_ex(), kind="switch", old_schema_cache=old_cache, mask_columns=["金额", "地区"]) + assert not {i for i in ids(ctx) if i.startswith("mask_lost:")} + # 改配方把列改了名 + new = copy.deepcopy(LIST) + next(c for c in new["sheets"][0]["blocks"][0]["columns"] if c["name"] == "金额")["name"] = "销售额" + new["tables"][0]["units"] = {"销量": "件", "销售额": "元"} + ex = list_ex() + ctx = ConfirmContext(kind="redraft", recipe=new, old_recipe=LIST, extraction=ex, checks=[], diff=[], + prev={"receipt": {"tables": []}}, mask_columns=["金额"]) + assert "mask_lost:金额" in ids(ctx) + + +def test_breaking_changes_column_source_swapped(): + """AU-2：交叉表两列的来源标签对调、列表两列的表头对调：列名类型都没变，内容整列换了，要进 breaking。""" + new = flow() + m = new["sheets"][0]["blocks"][0]["segments"][0]["measures"] + m["分区甲（人次）"], m["分区乙（人次）"] = "分区乙", "分区甲" + out = breaking_changes(FLOW, new) + assert "列「分区甲」的来源「分区甲（人次）」→「分区乙（人次）」" in out["日客流"] + assert "列「分区乙」的来源「分区乙（人次）」→「分区甲（人次）」" in out["日客流"] + ex = extraction_of(sep()[0]) + got = [i.id for i in confirm_items(ConfirmContext(kind="redraft", recipe=new, old_recipe=FLOW, extraction=ex, + checks=[], diff=[], prev=BASE[0]))] + assert "breaking:日客流" in got + lst2 = copy.deepcopy(LIST) + cols = lst2["sheets"][0]["blocks"][0]["columns"] + cols[0]["name"], cols[1]["name"] = cols[1]["name"], cols[0]["name"] + kinds = [c.kind for c in table_changes(LIST, lst2)] + assert kinds.count("source") == 2 + # 只是写法（空白、全角）不同：match_key 相同，不算 + same = flow() + seg = same["sheets"][0]["blocks"][0]["segments"][0] + seg["labels"]["expect"] = [x.replace("（", "(").replace("）", ")") for x in seg["labels"]["expect"]] + seg["measures"] = {k.replace("（", "(").replace("）", ")"): v for k, v in seg["measures"].items()} + assert not [c for c in table_changes(FLOW, same) if c.kind == "source"] + + +def test_breaking_changes_const_pick_with_segment_id_changed(): + """AU-2：分段 id 跟着 pick 一起改（把新文件的规则草稿粘进来）：按 (表, 列) 比全部取值，照样报 const_value。""" + new = flow() + seg = new["sheets"][0]["blocks"][0]["segments"][1] + old_id = seg["id"] + seg["id"] = old_id + "分段" + seg["locate"]["title"] = "日间分时段客流（人次）" + seg["const"]["时段类别"]["pick"] = "日间分" + for other in new["sheets"][0]["blocks"][0]["segments"]: + if (other.get("locate") or {}).get("segment") == old_id: + other["locate"]["segment"] = seg["id"] + changes = [c for c in table_changes(FLOW, new) if c.kind == "const_value"] + assert len(changes) == 1 and "日间分" in changes[0].message and "日间" in changes[0].message + + +def test_breaking_changes_text_store(): + """AU-2：列表文字列的存法（规范写法 / 原文）显式改了要报；缺省与显式写成同一个生效值不报。""" + base = copy.deepcopy(LIST) + base["tables"][0]["grain"] = ["地区"] # 产品不在主键里：缺省按原文存 + raw_explicit = copy.deepcopy(base) + raw_explicit["sheets"][0]["blocks"][0]["columns"][1]["store"] = "raw" + assert not [c for c in table_changes(base, raw_explicit) if c.kind == "store"] + canon_ = copy.deepcopy(base) + canon_["sheets"][0]["blocks"][0]["columns"][1]["store"] = "canonical" + ch = [c for c in table_changes(base, canon_) if c.kind == "store"] + assert len(ch) == 1 and ch[0].message == "列「产品」的文字存法 原文 → 规范写法" + + +def test_breaking_changes_removed_and_added(): + new = flow() + seg = new["sheets"][0]["blocks"][0]["segments"][0] + seg["labels"]["expect"] = ["全日客流（人次）", "分区甲（人次）", "分区丙（人次）"] + seg["measures"] = {"全日客流（人次）": "全日客流", "分区甲（人次）": "分区甲", "分区丙（人次）": "分区丙"} + new["sheets"][0]["blocks"][0]["segments"].pop() # 夜间合计没了，表内合计表随之消失 + out = breaking_changes(FLOW, new) + assert out["日客流"] == ["删除或改名了列「分区乙」"] # 新增的「分区丙」不算 + assert out["时段客流_表内合计"] == ["表「时段客流_表内合计」不再产出（删除或改名）"] + # 反过来：表内合计表是新增的，不算；分区丙被删了，照报 + assert breaking_changes(new, FLOW) == {"日客流": ["删除或改名了列「分区丙」"]} + + +def test_breaking_changes_text_date(): + new = lst() + new["sheets"][0]["blocks"][0]["columns"][2]["type"] = "TEXT" + out = breaking_changes(LIST, new) + assert out == {"销售": ["列「日期」的写法 DATE → TEXT（日期存为 YYYY-MM-DD 文本）"]} + assert [c.kind for c in table_changes(LIST, new)] == ["type"] + + +def test_breaking_and_unit_changed_items(): + new = flow() + new["tables"][0]["units"] = {"全日客流": "人", "分区甲": "人次", "分区乙": "人次"} + ex = extraction_of(sep()[0]) + items = confirm_items(ConfirmContext(kind="redraft", recipe=new, old_recipe=FLOW, extraction=ex, checks=[], diff=[], + prev=BASE[0])) + assert items[0].id == "unit_changed:日客流.全日客流" # 排最前 + assert "人次 → 人" in items[0].label + assert not any(i.id.startswith("breaking:") for i in items) # 只变了单位：不重复出 breaking + new["sheets"][0]["blocks"][0]["values"]["type"] = "REAL" + got = [i.id for i in confirm_items(ConfirmContext(kind="redraft", recipe=new, old_recipe=FLOW, extraction=ex, + checks=[], diff=[], prev=BASE[0]))] + assert got[0] == "unit_changed:日客流.全日客流" + assert {"breaking:日客流", "breaking:时段客流", "breaking:时段客流_表内合计"} <= set(got) + # 首次导入没有现行配方，不报破坏性变更 + assert not any(i.startswith(("breaking:", "unit_changed:")) for i in ids(first(new))) + + +def test_switch_from_simple(): + cache = {"tables": { + "客流汇总": {"columns": [{"name": "项目", "type": "TEXT"}]}, + "日客流": {"columns": [{"name": "日期", "type": "TEXT"}, {"name": "全日客流", "type": "TEXT"}, + {"name": "备注", "type": "TEXT"}, {"name": "分区甲", "type": "BIGINT"}]}, + }} + lines = switch_changes(cache, FLOW) + assert lines == ["表「客流汇总」将消失", "表「日客流」：列「全日客流」类型 TEXT → INTEGER、列「备注」将消失"] + items = confirm_items(ConfirmContext(kind="switch", recipe=FLOW, old_schema_cache=cache, + extraction=extraction_of(BASE[0]), checks=[], prev=None)) + sw = next(i for i in items if i.id == "switch_from_simple") + assert sw.source == "switch" and "表「客流汇总」将消失" in sw.label + assert D00_CONFIRMS <= {i.id for i in items} # 切换时配方类照出 + assert not any(i.id.startswith("breaking:") for i in items) + assert "switch_from_simple" not in ids(first(FLOW)) + keep = confirm_items(ConfirmContext(kind="switch", recipe=FLOW, old_schema_cache={"tables": {}}, + extraction=extraction_of(BASE[0]), checks=[])) + assert next(i for i in keep if i.id == "switch_from_simple").label.endswith("原有的表和列都保留") + + +# -------------------------------------------------------------------------- +# 本期类 +# -------------------------------------------------------------------------- + + +def reupload(cur_doc: dict[str, Any], prev_doc: dict[str, Any] | None = BASE[0], **kw: Any) -> ConfirmContext: + kw.setdefault("diff", []) + return ConfirmContext(kind="reupload", recipe=FLOW, old_recipe=FLOW, extraction=kw.pop("ex", extraction_of(cur_doc)), + checks=cur_doc["checks"], prev=prev_doc, **kw) + + +def test_outside_digits_plain_text_with_digits_every_period(): + cur, _ = sep() + add_outside(cur, "B32", "注：9月15日闸机故障，当日客流为估算值") + item = next(i for i in confirm_items(reupload(cur)) if i.id == "outside_digits:客流汇总!B32") + assert item.source == "outside" and "注：9月15日" in item.label + # 下一期同一格同一句（只是日期变了）：不是统计期来源，仍要确认 + nxt = flow_doc(dt_oct(), 31) + add_outside(nxt, "B32", "注：10月3日闸机故障，当日客流为估算值") + assert "outside_digits:客流汇总!B32" in ids(reupload(nxt, cur)) + # 纯文字不出 + assert not any(i.startswith("outside_digits:") for i in ids(reupload(sep()[0]))) + + +def dt_oct(): + import datetime as dt + return dt.date(2026, 10, 1) + + +def test_outside_digits_exempt_a_title_gained_period(): + for text in ("客流汇总表（2026年9月）", "客流汇总表（2026年9月1日至2026年9月30日）"): + cur, _ = sep() + set_period_cell(cur, "B3", text, annotated=True) + assert "outside_digits:客流汇总!B3" not in ids(reupload(cur)) # D17、D27 + assert "outside_digits:客流汇总!B3" in ids(first(FLOW, extraction_of(cur))) # 首次没有上一期，不免 + + +def test_outside_digits_exempt_a_requires_same_text(): + cur, _ = sep() + set_period_cell(cur, "B3", "客流汇总初步表（2026年9月）", annotated=True) + assert "outside_digits:客流汇总!B3" in ids(reupload(cur)) + + +def test_outside_digits_exempt_a_requires_prev_plain_text(): + # 上一期这一格就含数字（不是纯文字）：(a) 不适用，即使附加文字和它相同 + prev = copy.deepcopy(BASE[0]) + prev["receipt"]["outside_text"] = [{"sheet": SHEET, "cell": "B3", "text": "客流汇总表3号门", "kind": "text_digits", + "period_source": False}] + cur, _ = sep() + set_period_cell(cur, "B3", "客流汇总表3号门（2026年9月）", annotated=True) + assert "outside_digits:客流汇总!B3" in ids(reupload(cur, prev)) + + +def test_outside_digits_exempt_only_for_period_source_cells(): + # 上一期 B3 是纯文字「客流汇总表」，本期 B3 加了年月，但没被认成统计期来源（kind=text_digits、 + # period_source=False、不在 PeriodOut.texts 里）：(a) 不适用，要确认。去掉「只有统计期来源格才能免」的 + # 入口判断，这里就会被 (a) 免掉（D15、E7） + cur, _ = sep() + cur["receipt"]["outside_text"] = [o for o in cur["receipt"]["outside_text"] if o["cell"] != "B3"] + add_outside(cur, "B3", "客流汇总表（2026年9月）") + assert match_key(period_residue("客流汇总表（2026年9月）")) == match_key("客流汇总表") # 真走到 (a) 是会免的 + assert "outside_digits:客流汇总!B3" in ids(reupload(cur)) + + +def test_outside_digits_d28b_not_exempt(): + cur, _ = sep() + set_period_cell(cur, "B32", "注：2026年9月数据为初步统计，待修订", annotated=True) + assert "outside_digits:客流汇总!B32" in ids(reupload(cur)) + + +def test_outside_digits_exempt_b_same_sentence_monthly(): + prev, _ = sep() + set_period_cell(prev, "B32", "注：2026年9月数据为初步统计，待修订", annotated=True) + cur = flow_doc(dt_oct(), 31) + set_period_cell(cur, "B32", "注：2026年10月数据为初步统计，待修订", annotated=True) + assert "outside_digits:客流汇总!B32" not in ids(reupload(cur, prev)) + set_period_cell(cur, "B32", "注：2026年10月数据为正式统计", annotated=True) + assert "outside_digits:客流汇总!B32" in ids(reupload(cur, prev)) + + +def test_outside_digits_exempt_b_other_digits_changed(): + # AU-3：附加文字里统计期以外的数字（口径）变了，遮盖模板相同也不算同一句，要确认 + prev, _ = sep() + set_period_cell(prev, "B32", "注：2026年9月数据只含3个分区", annotated=True) + cur = flow_doc(dt_oct(), 31) + set_period_cell(cur, "B32", "注：2026年10月数据只含2个分区", annotated=True) + assert "outside_digits:客流汇总!B32" in ids(reupload(cur, prev)) + set_period_cell(cur, "B32", "注：2026年10月数据只含3个分区", annotated=True) + assert "outside_digits:客流汇总!B32" not in ids(reupload(cur, prev)) + + +def test_outside_digits_prev_simple_import_not_exempt(): + cur, _ = sep() + set_period_cell(cur, "B3", "客流汇总表（2026年9月）", annotated=True) + simple = {"receipt": {"tables": [{"name": "客流汇总", "rows": 29}], + "outside_text": [{"sheet": SHEET, "cell": "B3", "text": "客流汇总表", "kind": "text"}]}, + "checks": [], "period": None} + assert "outside_digits:客流汇总!B3" in ids(reupload(cur, simple)) + + +def test_rows_after_stop(): + prob = {"code": "rows_after_stop", "category": "confirm", "message": "第 14 行起有 2 行文字在空行之后，没有导入", + "cells": ["销售!C14", "销售!D14", "销售!C15", "销售!D15"]} + ctx = first(LIST, list_ex(problems=[prob])) + item = next(i for i in confirm_items(ctx) if i.id == "rows_after_stop:列表1") + assert item.label == prob["message"] and item.source == "outside" and "销售!C14" in item.detail + assert not any(i.startswith("rows_after_stop:") for i in ids(first(LIST, list_ex()))) + + +def test_empty_list_block_needs_confirmation(): + """AU-1：执行器报 list_empty（列表块只有表头）→ 必勾 empty_block:<块>，首次、重传都出。""" + prob = {"code": "list_empty", "category": "confirm", "cells": ["销售!A3"], + "message": "工作表「销售」的列表「列表1」的表头之下没有数据行：本期这张表将导入 0 行。"} + item = next(i for i in confirm_items(first(LIST, list_ex(problems=[prob]))) if i.id == "empty_block:列表1") + assert item.required and item.source == "outside" and "0 行" in item.label and "表「销售」" in item.label + assert not any(i.startswith("empty_block:") for i in ids(first(LIST, list_ex()))) + + +def test_rows_after_stop_two_blocks_uses_lineage(): + r = copy.deepcopy(LIST) + r["sheets"][0]["blocks"][0]["rows"] = {} + second = copy.deepcopy(r["sheets"][0]["blocks"][0]) + second.update(id="列表2", table="费用", after_title="二、费用") + r["sheets"][0]["blocks"].append(second) + r["tables"] = [{"name": "销售"}, {"name": "费用"}] + lineage = {"销售": {"地区": [[1, "销售", "C3", 4, "down"]], "金额": [[1, "销售", "G3", 4, "down"]]}, + "费用": {"地区": [[1, "销售", "C12", 3, "down"]], "金额": [[1, "销售", "G12", 3, "down"]]}} + prob = {"code": "rows_after_stop", "category": "confirm", "message": "第 16 行起有 1 行文字在空行之后，没有导入", + "cells": ["销售!C16", "销售!D16"]} + got = ids(first(r, list_ex(problems=[prob], lineage=lineage, derived=[]))) + assert "rows_after_stop:列表2" in got and "rows_after_stop:列表1" not in got + prob["cells"] = ["销售!C8", "销售!D8"] + got = ids(first(r, list_ex(problems=[prob], lineage=lineage, derived=[]))) + assert "rows_after_stop:列表1" in got + + +def test_relation_observed(): + cur, _ = sep() + assert not any(i.startswith("relation_observed:") for i in ids(reupload(cur))) # 30 天中 0 天相等 + for c in cur["checks"]: + if c["id"] == "R2": + c["failed"] = 0 + lab = label(reupload(cur), "relation_observed:R2") + assert lab == "表「时段客流」与表「日客流」声明为口径不同，但本期按「日期」分组的 30 组全部相等" + + +def test_sheet_extra(): + cur, _ = sep() + cur["receipt"]["sheets"]["other_visible"] = ["说明"] + assert "sheet_extra:说明" in ids(first(FLOW, extraction_of(cur))) # 首次就有 + prev = copy.deepcopy(BASE[0]) + prev["receipt"]["sheets"]["other_visible"] = ["说明"] + assert "sheet_extra:说明" not in ids(reupload(cur, prev)) # 上一期也有 + cur["receipt"]["sheets"]["other_visible"] = ["说明", "备注"] + got = ids(reupload(cur, prev)) + assert "sheet_extra:备注" in got and "sheet_extra:说明" not in got + + +def test_sheet_renamed(): + cur, _ = sep() + cur["receipt"]["sheets"].update(matched={"s1": "汇总"}, renamed={"客流汇总": "汇总"}) + item = next(i for i in confirm_items(reupload(cur)) if i.id == "sheet_renamed:s1") + assert item.label == "工作表「客流汇总」现在叫「汇总」" and item.source == "sheet" + assert not any(i.startswith("sheet_renamed:") for i in ids(reupload(sep()[0]))) + + +def test_context_human(): + cur, _ = sep() + cur["period"] = {"start": "2026-09-01", "end": "2026-09-30", "source": "human", "cells": [], + "signed_by": "测试员甲", "texts": {}, "annotated": {}} + item = next(i for i in confirm_items(reupload(cur)) if i.id == "context_human:统计期") + assert item.label == "本期统计期为人工录入：2026-09-01 至 2026-09-30" and item.source == "context" + assert "测试员甲" in item.detail + assert "context_human:统计期" not in ids(reupload(sep()[0])) + + +def test_diff_items_collected(): + diff = [DiffItem("label_writing", "分段「日间」的标签写法变了", requires_confirm=True, + confirm_id="diff:label_writing:日间"), + DiffItem("period", "统计期变了")] + items = confirm_items(reupload(sep()[0], diff=diff)) + assert [(i.id, i.source) for i in items] == [("diff:label_writing:日间", "diff")] + # dict 形状也收 + items = confirm_items(reupload(sep()[0], diff=[{"kind": "row_order", "label": "x", "detail": "", + "requires_confirm": True, "confirm_id": "diff:row_order:日客流"}])) + assert [i.id for i in items] == ["diff:row_order:日客流"] + + +def test_accepts_dataclass_extraction_and_checks(): + from app.data.recipe_types import CheckResult, Extraction, OutsideText, SheetsOut + + ex = Extraction(ok=True, outside_text=[OutsideText(SHEET, "B32", "注：9月15日", "text_digits")], + sheets=SheetsOut(matched={"s1": SHEET})) + checks = [CheckResult("R2", "relation_not_comparable", "R2", "info", "info", checked=30, failed=0)] + got = ids(ConfirmContext(kind="reupload", recipe=FLOW, old_recipe=FLOW, extraction=ex, checks=checks, diff=[], + prev=BASE[0])) + assert got == {"outside_digits:客流汇总!B32", "relation_observed:R2"} + + +def test_end_to_end_with_diff_reports(): + # D28：差异卡和确认项一起跑，必勾集合正好是 outside_digits + diff:outside + cur, name = sep() + add_outside(cur, "B32", "注：9月15日闸机故障，当日客流为估算值") + diff = diff_reports(BASE[0], cur, prev_file_name=BASE[1], cur_file_name=name, recipe=FLOW, recipe_changed=False) + got = ids(reupload(cur, diff=diff)) + assert got == {"outside_digits:客流汇总!B32", "diff:outside:客流汇总!B32"} + assert file_name(AUG, 31) == BASE[1] and SEP.month == 9 + + +# -------------------------------------------------------------------------- +# 输入不全就报错：漏一个键、一个参数，对应的必勾项会静默消失，提交时重算也发现不了 +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("kind", ["reupload", "redraft"]) +def test_old_recipe_required_for_reupload_and_redraft(kind): + # 漏传现行配方：早先 breaking:*、unit_changed:* 整段跳过（万元 → 元 静默放过） + new = flow() + new["tables"][0]["units"] = {"全日客流": "人", "分区甲": "人次", "分区乙": "人次"} + ctx = ConfirmContext(kind=kind, recipe=new, old_recipe=None, extraction=extraction_of(sep()[0]), checks=[], + diff=[], prev=BASE[0]) + with pytest.raises(ValueError, match="现行配方"): + confirm_items(ctx) + ctx.old_recipe = FLOW + assert "unit_changed:日客流.全日客流" in ids(ctx) + + +@pytest.mark.parametrize("kind", ["reupload", "redraft"]) +def test_prev_required_for_reupload_and_redraft(kind): + ctx = ConfirmContext(kind=kind, recipe=FLOW, old_recipe=FLOW, extraction=extraction_of(sep()[0]), checks=[], + diff=[], prev=None) + with pytest.raises(ValueError, match="上一期"): + confirm_items(ctx) + + +def test_unknown_kind_raises(): + with pytest.raises(ValueError, match="未知的导入方式"): + confirm_items(first(FLOW, kind="upload")) + + +def test_checks_and_diff_required(): + with pytest.raises(ValueError, match="核对结果"): + confirm_items(ConfirmContext(kind="first", recipe=FLOW, extraction=extraction_of(BASE[0]))) + with pytest.raises(ValueError, match="差异卡"): + confirm_items(ConfirmContext(kind="reupload", recipe=FLOW, old_recipe=FLOW, extraction=extraction_of(sep()[0]), + checks=[], prev=BASE[0])) + # 首次没有上一期，不用给差异卡 + assert ids(ConfirmContext(kind="first", recipe=FLOW, extraction=extraction_of(BASE[0]), checks=[])) == D00_CONFIRMS + + +@pytest.mark.parametrize("key", EXTRACTION_KEYS) +def test_missing_extraction_key_raises(key): + # 例如 extraction 没有 problems：rows_after_stop 早先直接没了 + ex = extraction_of(BASE[0]) + del ex[key] + with pytest.raises(ReceiptIncomplete) as err: + confirm_items(first(FLOW, ex)) + assert err.value.side == "extraction" and err.value.missing == [key] + + +def test_missing_extraction_raises(): + with pytest.raises(ReceiptIncomplete, match="extraction"): + confirm_items(ConfirmContext(kind="first", recipe=FLOW, checks=[])) + + +def test_partial_extraction_raises(): + ex = extraction_of(BASE[0]) + ex["partial"] = True + with pytest.raises(ReceiptIncomplete, match="partial"): + confirm_items(first(FLOW, ex)) + + +@pytest.mark.parametrize("key", ["outside_text", "sheets"]) +def test_prev_receipt_missing_key_raises(key): + prev = copy.deepcopy(BASE[0]) + del prev["receipt"][key] + with pytest.raises(ReceiptIncomplete) as err: + confirm_items(reupload(sep()[0], prev)) + assert err.value.side == "prev" and err.value.missing == [key] diff --git a/backend/tests/test_recipe_contract.py b/backend/tests/test_recipe_contract.py new file mode 100644 index 00000000..1c0740fb --- /dev/null +++ b/backend/tests/test_recipe_contract.py @@ -0,0 +1,235 @@ +"""期 2 契约文件（recipe_types、recipe_parsers）的测试：只用合成字符串，不读任何文件。""" +from __future__ import annotations + +import datetime as dt +import json +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from app.data import recipe_parsers as P +from app.data import recipe_types as T + +FLOW = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("text,canonical", [ + ("7-8", "7-8"), ("7–8", "7-8"), ("7—8", "7-8"), ("07:00-08:00", "7-8"), ("7时-8时", "7-8"), + ("7点-8点", "7-8"), ("7-8时", "7-8"), ("８－９", "8-9"), ("7-8 ", "7-8"), ("23-24", "23-24"), +]) +def test_hour_range_accepts_tolerant_forms(text, canonical): + assert P.hour_range(text).canonical == canonical + + +@pytest.mark.parametrize("text", ["25-26", "8-7", "7:30-8:00", "18-22 时合计", "x" * 41, 7, None, ""]) +def test_hour_range_rejects(text): + assert P.hour_range(text) is None + + +@pytest.mark.parametrize("text,canonical", [ + ("18-22 时合计", "18-22时合计"), ("18:00-22:00合计", "18-22时合计"), ("18-22时小计", "18-22时小计"), + ("18－24 时合计", "18-24时合计"), ("18–22时总计", "18-22时总计"), +]) +def test_hour_range_total(text, canonical): + assert P.hour_range_total(text).canonical == canonical + + +def test_hour_range_total_rejects_plain_range(): + assert P.hour_range_total("18-22") is None + + +@pytest.mark.parametrize("text,start,end", [ + ("统计时间范围：2026年9月1日至2026年9月30日", "2026-09-01", "2026-09-30"), + ("统计时间范围：2026年9月1日至9月30日", "2026-09-01", "2026-09-30"), + ("2026年12月15日至1月14日", "2026-12-15", "2027-01-14"), + ("客流汇总表（2026年9月1日至2026年9月30日）", "2026-09-01", "2026-09-30"), + ("2026-09-01至2026-09-30", "2026-09-01", "2026-09-30"), +]) +def test_cn_date_range(text, start, end): + assert P.cn_date_range(text).iso() == (start, end) + + +@pytest.mark.parametrize("text", [ + "客流汇总表（2026年9月）", "2026年9月1日至2026年9月30日；2026年10月1日至10月31日", "2026年2月30日至3月1日", + "注：9月15日闸机故障，当日客流为估算值", "x" * 81, +]) +def test_cn_date_range_rejects(text): + assert P.cn_date_range(text) is None + + +def test_year_month_is_only_supporting_evidence(): + m = P.cn_year_month("客流汇总表（2026年9月）") + assert (m.year, m.month) == (2026, 9) + assert m.contains(P.Period(dt.date(2026, 9, 1), dt.date(2026, 9, 30))) + assert m.contains(P.Period(dt.date(2026, 9, 1), dt.date(2026, 9, 15))) + assert not m.contains(P.Period(dt.date(2026, 10, 1), dt.date(2026, 10, 31))) + assert P.cn_year_month("2026年9月1日至9月30日") is None + + +def test_filename_period(): + assert P.filename_period("月报导出_2026-08-01_2026-08-31.xlsx").iso() == ("2026-08-01", "2026-08-31") + assert P.filename_period("9月客流.xlsx") is None + + +@pytest.mark.parametrize("value,expected", [ + ("8月1日", (None, 8, 1, "text")), (" 8月1日 ", (None, 8, 1, "text")), ("2026年8月1日", (2026, 8, 1, "text")), + ("2026-08-01", (2026, 8, 1, "text")), ("2026/8/1", (2026, 8, 1, "text")), + (dt.datetime(2026, 8, 1), (2026, 8, 1, "date")), (dt.date(2026, 8, 1), (2026, 8, 1, "date")), +]) +def test_axis_cells(value, expected): + got = P.month_day_or_date(value) + assert (got.year, got.month, got.day, got.form) == expected + + +@pytest.mark.parametrize("value", ["8/1", "13月1日", dt.datetime(2026, 8, 1, 12), 45000, "合计", None]) +def test_axis_cells_rejected(value): + assert P.month_day_or_date(value) is None + + +def test_complete_year_unique_solution_across_new_year(): + period = P.Period(dt.date(2026, 12, 15), dt.date(2027, 1, 14)) + assert P.complete_year(P.AxisDate(None, 1, 3, "text"), period) == dt.date(2027, 1, 3) + assert P.complete_year(P.AxisDate(None, 12, 20, "text"), period) == dt.date(2026, 12, 20) + assert P.complete_year(P.AxisDate(None, 2, 1, "text"), period) is None + assert P.complete_year(P.AxisDate(2026, 11, 1, "date"), period) is None + + +def test_unit_suffix(): + assert P.split_unit_suffix("全日客流（人次）") == ("全日客流", "人次") + assert P.split_unit_suffix("金额") == ("金额", None) + assert P.split_unit_suffix("销量（箱）") == ("销量", "箱") + assert P.unit_known("人次") and not P.unit_known("箱") + + +@pytest.mark.parametrize("value,kind", [ + ("客流汇总表", "text"), ("注：9月15日闸机故障，当日客流为估算值", "text_digits"), ("1,234", "number"), + ("12.5%", "number"), ("50万", "number"), (12.0, "number"), (dt.date(2026, 9, 5), "text_digits"), + ("单位：万元", "text"), ("补录（人次）", "text"), +]) +def test_classify_outside(value, kind): + assert P.classify_outside(value) == kind + + +@pytest.mark.parametrize("text,numeric", [ + ("1,234", True), ("12.5%", True), ("50万", True), ("¥1234", True), ("12kg", True), ("12万人次", True), + ("3天", True), ("100元", True), + # 日期、期间写法是文字（含数字），不是一个数（lab c04 的两行表头「2026年上半年」） + ("2026年上半年", False), ("2026年度", False), ("2026上半年", False), ("9月", False), ("15日", False), + ("7时", False), ("3号", False), ("2季度", False), ("3期", False), +]) +def test_looks_numeric_text_date_suffix_is_text(text, numeric): + assert P.looks_numeric_text(text) is numeric + assert P.classify_outside(text) == ("number" if numeric else "text_digits") + + +def test_candidate_words_strip_common_affixes(): + assert P.candidate_words("日间时段客流（人次）", ["夜间时段客流（人次）"])[0] == "日间" + assert P.candidate_words("夜间时段客流（人次）", ["日间时段客流（人次）"])[0] == "夜间" + assert "早高峰" in P.candidate_words("工作日 早高峰", ["工作日 晚高峰"]) + assert all(not any(ch.isdigit() for ch in w) for w in P.candidate_words("2026年日间", ["2026年夜间"])) + + +def test_match_key_ignores_whitespace_and_width(): + assert P.match_key("金 额") == P.match_key("销 售".replace("销 售", "金额")) == "金额" + assert P.match_key("地区\u3000") == "地区" + + +def test_flow_recipe_parses_and_derives_tables(): + recipe = T.Recipe.model_validate(FLOW) + tables, problems = T.derive_tables(recipe) + assert problems == [] + assert [c.name for c in tables["时段客流"]] == ["日期", "时段", "时段类别", "起始小时", "结束小时", "客流"] + assert [c.name for c in tables["日客流"]] == ["日期", "全日客流", "分区甲", "分区乙"] + assert tables["日客流"][1].unit == "人次" + + +@pytest.mark.parametrize("mutate", [ + lambda r: r["sheets"][0].__setitem__("regex", "x"), + lambda r: r["sheets"][0]["blocks"][0]["segments"][1]["const"].__setitem__("时段类别", {"value": "日间"}), + lambda r: r["sheets"][0]["blocks"][0]["values"]["placeholders"][0].__setitem__("to", 0), + lambda r: r["sheets"][0]["blocks"][0]["axis"].__setitem__("year", 2026), +]) +def test_closed_language_rejects_open_fields(mutate): + data = json.loads(json.dumps(FLOW)) + mutate(data) + with pytest.raises(ValidationError): + T.Recipe.model_validate(data) + + +@pytest.mark.parametrize("text,ok", [ + ("粒度：每个{列:日期}。{列:全日客流} 等于 {列:分区甲} 与 {列:分区乙} 之和（导入时已逐日核对）", True), + ("单位：{单位:万元}", True), ("单位：万元", False), ("{列:3号门客流} 与 {列:指标2} 不能相加", True), + ("18-22 时合计不要相加", False), ("见 C5 格", False), ("格式 YYYY-MM-DD，年份按统计期补全", True), +]) +def test_prose_problems(text, ok): + assert (T.prose_problems(text) == []) is ok + + +# ---- 评审后补的契约用例（P2-SPEC「评审意见处理」） ---- + + +def test_measures_keys_must_match_expect_verbatim(): + """键用半角括号、expect 用全角：match_key 相等，但 derive_tables 必须报问题，不能静默产出空列名。""" + data = json.loads(json.dumps(FLOW)) + seg = data["sheets"][0]["blocks"][0]["segments"][0] + seg["measures"] = {"全日客流(人次)": "全日客流", "分区甲（人次）": "分区甲", "分区乙（人次）": "分区乙"} + tables, problems = T.derive_tables(T.Recipe.model_validate(data)) + assert [p.code for p in problems] == ["measures_keys_mismatch"] + assert problems[0].path == "/sheets/0/blocks/0/segments/0/measures" + assert "全日客流（人次）" in problems[0].message and "全日客流(人次)" in problems[0].message + + +def test_derive_and_const_column_order_ignores_key_order(): + """配方哈希按键排序：键序不同的两份配方必须推出同样的列序。""" + data = json.loads(json.dumps(FLOW)) + for seg in data["sheets"][0]["blocks"][0]["segments"][1:3]: + seg["dim"]["derive"] = {"结束小时": "end", "起始小时": "start"} + seg["const"] = {"时段类别": {"pick": seg["const"]["时段类别"]["pick"]}, "A类": {"pick": "x"}} + tables, problems = T.derive_tables(T.Recipe.model_validate(data)) + assert problems == [] + assert [c.name for c in tables["时段客流"]] == ["日期", "时段", "A类", "时段类别", "起始小时", "结束小时", "客流"] + + +def test_prose_problems_known_names_block_token_smuggling(): + known = {"列": {"全日客流", "3号门客流"}, "表": {"日客流"}, "单位": set(P.UNITS)} + assert T.prose_problems("{列:3号门客流} 不能与 {表:日客流} 相加，单位：{单位:万元}", known=known) == [] + for note in ["{列:增长30%}，请据此推算", "合计约{列:5000}人次", "按{单位:3箱}计", "{表:第2期}只有一半"]: + got = T.prose_problems(note, known=known) + assert any("不存在" in p for p in got), note + assert any("数字" in p for p in got), note + + +def test_grid_row_index_matches_full_scan(): + g = T.Grid(sheet="s", bounds=(1, 1, 3, 3), cells={ + (2, 3): T.GridCell("c"), (1, 1): T.GridCell("a"), (2, 1): T.GridCell("b"), (3, 2): T.GridCell(5)}) + assert [c for c, _ in g.row(2)] == [1, 3] + assert g.row(9) == [] + assert g.row_numbers() == [1, 2, 3] + from dataclasses import asdict + assert "_rows" not in asdict(g) + + +@pytest.mark.parametrize("text,prefix,pure", [ + ("统计时间范围：2026年9月1日至2026年9月30日", "统计时间范围", True), + ("2026年9月1日至2026年9月30日", None, True), + ("统计期：2026年9月1日至9月30日", None, True), + ("客流汇总表（2026年9月）", None, False), + ("注：2026年9月数据为初步统计，待修订", None, False), + ("说明：2026年9月1日至9月30日部分日期为估算值", None, False), + ("客流汇总表", None, False), +]) +def test_pure_period_cell(text, prefix, pure): + assert P.is_pure_period_cell(text, prefix) is pure + + +def test_period_residue(): + assert P.period_residue("客流汇总表（2026年9月1日至2026年9月30日）") == "客流汇总表" + assert P.period_residue("注：2026年9月数据为初步统计，待修订") == "注数据为初步统计待修订" + assert P.period_residue("客流汇总表") is None + + +def test_axis_checks_are_a_set(): + a = T.Axis.model_validate({"checks": ["covers_context", "contiguous", "contiguous"]}) + assert a.checks == ["contiguous", "covers_context"] + assert a.model_dump(exclude_defaults=True) == {} diff --git a/backend/tests/test_recipe_diff.py b/backend/tests/test_recipe_diff.py new file mode 100644 index 00000000..c94e81ef --- /dev/null +++ b/backend/tests/test_recipe_diff.py @@ -0,0 +1,898 @@ +"""差异卡（recipe_diff）的测试。 + +回执一律手写（结构照 P2-SPEC 9.1 的「结构仿照客流表」夹具，假名、假数），不依赖执行器（WP-2）和夹具 +生成器（WP-1）：差异卡是纯函数，输入就是回执 dict。WP-7 会再用真夹具把 9.2 的 31 例跑一遍。 + +本文件还放着两个测试文件共用的回执构造器（flow_doc 等），test_recipe_confirm 从这里导入。 +""" +from __future__ import annotations + +import copy +import datetime as dt +import json +from pathlib import Path +from typing import Any, Callable + +import pytest + +from app.data import recipe_diff as D +from app.data.recipe_confirm import ConfirmContext, confirm_items +from app.data.recipe_parsers import match_key, period_residue + +FLOW: dict[str, Any] = json.loads( + (Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text(encoding="utf-8")) + +SHEET = "客流汇总" +AUG = dt.date(2026, 8, 1) +SEP = dt.date(2026, 9, 1) +OCT = dt.date(2026, 10, 1) +DAILY = ["全日客流（人次）", "分区甲（人次）", "分区乙（人次）"] +DAY = ["7-8", "8-9", "9-10", "10-11", "11-12", "12-13", "13-14", "14-15", "15-16", "16-17", "17-18"] +NIGHT = ["18-19", "19-20", "20-21", "21-22", "22-23", "23-24"] +TOTAL_RAW = ["18-22 时合计", "22-24 时合计", "18-24 时合计"] +TOTAL_CAN = ["18-22时合计", "22-24时合计", "18-24时合计"] + + +# -------------------------------------------------------------------------- +# 回执构造器（两个测试文件共用） +# -------------------------------------------------------------------------- + + +def b2_text(start: dt.date, end: dt.date, *, omit_year: bool = False) -> str: + second = f"{end.month}月{end.day}日" if omit_year else f"{end.year}年{end.month}月{end.day}日" + return f"统计时间范围：{start.year}年{start.month}月{start.day}日至{second}" + + +def file_name(start: dt.date, days: int) -> str: + end = start + dt.timedelta(days=days - 1) + return f"月报导出_{start.isoformat()}_{end.isoformat()}.xlsx" + + +def _col(name: str, type_: str, role: str, header: str | None = None, unit: str | None = None) -> dict[str, Any]: + return {"name": name, "type": type_, "header": header, "unit": unit, "role": role} + + +def _seg(segment: str, raw: list[str], canonical: list[str], rows: list[int]) -> dict[str, Any]: + return {"segment": segment, "raw": list(raw), "canonical": list(canonical), "rows": list(rows)} + + +def _check(cid: str, kind: str, status: str, *, checked: int = 0, failed: int = 0, + category: str = "info") -> dict[str, Any]: + return {"id": cid, "kind": kind, "title": cid, "status": status, "category": category, "checked": checked, + "failed": failed, "unverifiable": 0, "sql": None, "params": [], "details": [], "cells": [], + "acceptable": False, "reasons": {}} + + +def flow_doc(start: dt.date = AUG, days: int = 31) -> dict[str, Any]: + """「导入回执」：{"receipt": 回执, "checks": [...], "period": PeriodOut}，按参考配方、参考布局手写。""" + end = start + dt.timedelta(days=days - 1) + period = {"start": start.isoformat(), "end": end.isoformat(), "source": "cells", "cells": [f"{SHEET}!B2"], + "signed_by": None, "texts": {f"{SHEET}!B2": b2_text(start, end)}, "annotated": {}} + tables = [ + {"name": "日客流", "sheet": SHEET, "kind": "data", "grain": ["日期"], "rows": days, + "columns": [_col("日期", "TEXT", "axis"), _col("全日客流", "INTEGER", "measure", DAILY[0], "人次"), + _col("分区甲", "INTEGER", "measure", DAILY[1], "人次"), + _col("分区乙", "INTEGER", "measure", DAILY[2], "人次")], + "sources": ["s1/交叉表/日客流"]}, + {"name": "时段客流", "sheet": SHEET, "kind": "data", "grain": ["日期", "时段"], "rows": days * 17, + "columns": [_col("日期", "TEXT", "axis"), _col("时段", "TEXT", "dim"), _col("时段类别", "TEXT", "const"), + _col("起始小时", "INTEGER", "derive"), _col("结束小时", "INTEGER", "derive"), + _col("客流", "INTEGER", "value", None, "人次")], + "sources": ["s1/交叉表/日间", "s1/交叉表/夜间"]}, + {"name": "时段客流_表内合计", "sheet": SHEET, "kind": "reported_total", "grain": ["日期", "合计项"], + "rows": days * 3, + "columns": [_col("日期", "TEXT", "axis"), _col("合计项", "TEXT", "dim"), _col("起始小时", "INTEGER", "derive"), + _col("结束小时", "INTEGER", "derive"), _col("客流", "INTEGER", "value", None, "人次")], + "sources": ["s1/交叉表/夜间合计"]}, + ] + receipt = { + "ledger": [{"sheet": SHEET, "nonempty_scan": 20 * days + 3 * days + 47, + "nonempty_read": 20 * days + 3 * days + 47, "roles": {}, "unclaimed": 0}], + "tables": tables, + "period": copy.deepcopy(period), + "axes": [{"block": "交叉表", "row": 4, "first": f"{start.month}月{start.day}日", + "last": f"{end.month}月{end.day}日", "count": days, "form": "text"}], + "placeholders": {"·": days}, + "canonicalized": [], + "canonicalized_total": 0, + "labels": [_seg("日客流", DAILY, [match_key(x) for x in DAILY], [5, 6, 7]), + _seg("日间", DAY, DAY, list(range(10, 21))), + _seg("夜间", NIGHT, NIGHT, list(range(22, 28))), + _seg("夜间合计", TOTAL_RAW, TOTAL_CAN, [28, 29, 30])], + "outside_text": [{"sheet": SHEET, "cell": "B3", "text": "客流汇总表", "kind": "text", "period_source": False}], + "hidden": {}, + "sheets": {"matched": {"s1": SHEET}, "renamed": {}, "other_visible": [], "skipped_hidden": []}, + "derived_form": {"夜间合计": "formula"}, + "block_order": {SHEET: ["日客流", "日间", "夜间", "夜间合计"]}, + "ignored_columns": {}, + "full_calc_on_load": False, + "formula_cells_accepted": 0, + "blank_rows_skipped": 0, + "derived": [], + "lineage": {}, + "db_sha256": "0" * 64, + "table_hashes": {}, + "timings": {}, + } + checks = [ + _check("C1", "context_agree", "passed", checked=1), + _check("C2", "filename_period", "passed", checked=1), + _check("K1", "derived_sum", "passed", checked=3 * days, category="structure"), + _check("G1", "formula_refs", "passed", checked=3 * days, category="structure"), + _check("R1", "relation_sum_eq", "passed", checked=days, category="data_quality"), + _check("R2", "relation_not_comparable", "info", checked=days, failed=days), + ] + for i, t in enumerate(("日客流", "时段客流", "时段客流_表内合计"), 1): + checks.append(_check(f"N{i}", "row_count", "passed", checked=1, category="structure")) + checks.append(_check(f"P{i}", "pk_unique", "passed", checked=1, category="structure")) + return {"receipt": receipt, "checks": checks, "period": period} + + +def extraction_of(doc: dict[str, Any], problems: list[dict[str, Any]] | None = None) -> dict[str, Any]: + """回执 → 确认项要的 Extraction dict 形状（回执就是 Extraction 去掉 problems 和 C1、C2）。""" + ex = copy.deepcopy(doc["receipt"]) + ex["period"] = copy.deepcopy(doc["period"]) + ex["problems"] = list(problems or []) + ex["ok"] = True + return ex + + +def set_period_cell(doc: dict[str, Any], cell: str, text: str, *, annotated: bool) -> None: + """把 cell 设成统计期来源；annotated=True 时同时记成区域外文字（kind=text_digits、period_source）。""" + key = f"{SHEET}!{cell}" + for p in (doc["period"], doc["receipt"]["period"]): + if key not in p["cells"]: + p["cells"].append(key) + p["texts"][key] = text + if annotated: + p["annotated"][key] = period_residue(text) + out = [o for o in doc["receipt"]["outside_text"] if o["cell"] != cell] + if annotated: + out.append({"sheet": SHEET, "cell": cell, "text": text, "kind": "text_digits", "period_source": True}) + doc["receipt"]["outside_text"] = out + + +def add_outside(doc: dict[str, Any], cell: str, text: str, kind: str = "text_digits") -> None: + doc["receipt"]["outside_text"].append({"sheet": SHEET, "cell": cell, "text": text, "kind": kind, + "period_source": False}) + + +def set_check(doc: dict[str, Any], cid: str, status: str) -> None: + for c in doc["checks"]: + if c["id"] == cid: + c["status"] = status + + +def set_labels(doc: dict[str, Any], segment: str, raw: list[str] | None = None, + canonical: list[str] | None = None, rows: list[int] | None = None) -> None: + for s in doc["receipt"]["labels"]: + if s["segment"] == segment: + if raw is not None: + s["raw"] = list(raw) + if canonical is not None: + s["canonical"] = list(canonical) + if rows is not None: + s["rows"] = list(rows) + + +def kinds(items: list[Any]) -> set[str]: + return {i.kind for i in items} + + +def sep() -> tuple[dict[str, Any], str]: + return flow_doc(SEP, 30), file_name(SEP, 30) + + +def run(prev: tuple[dict[str, Any], str] | None, cur: tuple[dict[str, Any], str], **kw: Any) -> list[Any]: + """默认按参考配方、配方没变（上传新一期）调用；要测别的就显式传。""" + kw.setdefault("recipe", FLOW) + kw.setdefault("recipe_changed", False) + return D.diff_reports(prev[0] if prev else None, cur[0], prev_file_name=prev[1] if prev else None, + cur_file_name=cur[1], **kw) + + +BASE = (flow_doc(AUG, 31), file_name(AUG, 31)) + + +# -------------------------------------------------------------------------- +# 9.2 的 31 例：差异的精确集合、本期类与 diff 确认项的精确集合 +# -------------------------------------------------------------------------- + + +def _d00(): + return flow_doc(AUG, 31), file_name(AUG, 31) + + +def _d02(): + return flow_doc(OCT, 31), file_name(OCT, 31) + + +def _d03(): + return flow_doc(OCT, 15), file_name(OCT, 15) + + +def _d03b(): + return flow_doc(OCT, 7), file_name(OCT, 7) + + +def _d12(): + doc, name = sep() + text = b2_text(SEP, dt.date(2026, 9, 30), omit_year=True) + doc["period"]["texts"][f"{SHEET}!B2"] = text + doc["receipt"]["period"]["texts"][f"{SHEET}!B2"] = text + return doc, name + + +def _d14(): + # 去掉第 8 行空行：第 8 行以下整体上移一行，标签的行号都变了，顺序没变 + doc, name = sep() + for s in doc["receipt"]["labels"]: + s["rows"] = [r - 1 if r > 8 else r for r in s["rows"]] + return doc, name + + +def _d17(): + doc, name = sep() + set_period_cell(doc, "B3", "客流汇总表（2026年9月）", annotated=True) + return doc, name + + +def _d23(): + doc, _ = sep() + set_check(doc, "C2", "info") + return doc, "9月客流.xlsx" + + +def _d25(): + doc, name = sep() + doc["receipt"]["full_calc_on_load"] = True + return doc, name + + +def _d27(): + doc, name = sep() + set_period_cell(doc, "B3", "客流汇总表（2026年9月1日至2026年9月30日）", annotated=True) + return doc, name + + +def _d05(): + doc, name = sep() + doc["receipt"]["block_order"][SHEET] = ["日客流", "夜间", "夜间合计", "日间"] + set_labels(doc, "夜间", rows=list(range(10, 16))) + set_labels(doc, "夜间合计", rows=[16, 17, 18]) + set_labels(doc, "日间", rows=list(range(21, 32))) + return doc, name + + +def _d06(): + doc, name = sep() + doc["receipt"]["block_order"][SHEET] = ["日间", "夜间", "夜间合计", "日客流"] + add_outside(doc, "B28", "日客流（人次）", kind="text") + return doc, name + + +def _d07(): + doc, name = sep() + doc["receipt"]["derived_form"]["夜间合计"] = "literal" + doc["checks"] = [c for c in doc["checks"] if c["id"] != "G1"] + return doc, name + + +def _d10(): + doc, name = sep() + doc["receipt"]["axes"][0]["form"] = "date" + return doc, name + + +def _d11(): + doc, name = sep() + set_labels(doc, "日间", raw=[f"{int(a):02d}:00-{int(b):02d}:00" for a, b in (x.split("-") for x in DAY)]) + set_labels(doc, "夜间", raw=[f"{int(a):02d}:00-{int(b):02d}:00" for a, b in (x.split("-") for x in NIGHT)]) + return doc, name + + +def _d19(): + doc, name = sep() + order = [DAILY[1], DAILY[2], DAILY[0]] + set_labels(doc, "日客流", raw=order, canonical=[match_key(x) for x in order]) + return doc, name + + +def _d22(): + doc, name = sep() + set_labels(doc, "夜间合计", raw=["18:00-22:00合计", "22:00-24:00合计", "18:00-24:00合计"]) + return doc, name + + +def _d24(): + doc, name = sep() + set_labels(doc, "日间", raw=[x.replace("-", "–") for x in DAY]) + set_labels(doc, "夜间", raw=[x.replace("-", "–") for x in NIGHT]) + return doc, name + + +def _d20(): + doc, name = sep() + doc["receipt"]["sheets"]["other_visible"] = ["说明"] + return doc, name + + +def _d28(): + doc, name = sep() + add_outside(doc, "B32", "注：9月15日闸机故障，当日客流为估算值") + return doc, name + + +def _d28b(): + doc, name = sep() + set_period_cell(doc, "B32", "注：2026年9月数据为初步统计，待修订", annotated=True) + return doc, name + + +def _d26(): + doc, name = sep() + set_check(doc, "R1", "mismatch") + return doc, name + + +#: 代号 → (构造, 差异的精确集合, 本期类与 diff 确认项的精确集合；带 * 的按前缀)。P2-SPEC 9.2 的表 +DRIFT: dict[str, tuple[Callable[[], tuple[dict[str, Any], str]], set[str], set[str]]] = { + "D01": (sep, {"period", "rows"}, set()), + "D02": (_d02, {"period"}, set()), + "D03": (_d03, {"period", "rows"}, set()), + "D03b": (_d03b, {"period", "rows"}, set()), + "D12": (_d12, {"period", "rows", "context_text"}, set()), + "D14": (_d14, {"period", "rows"}, set()), + "D15": (sep, {"period", "rows"}, set()), + "D17": (_d17, {"period", "rows", "context_source_added"}, set()), + "D23": (_d23, {"period", "rows", "file_name", "checks"}, set()), + "D25": (_d25, {"period", "rows", "full_calc"}, set()), + "D27": (_d27, {"period", "rows", "context_source_added"}, set()), + "D05": (_d05, {"period", "rows", "block_order"}, {"diff:block_order:客流汇总"}), + "D06": (_d06, {"period", "rows", "block_order", "outside_added"}, + {"diff:block_order:客流汇总", "diff:outside:客流汇总!B*"}), + "D07": (_d07, {"period", "rows", "checks", "derived_form"}, {"diff:derived_form:夜间合计"}), + "D10": (_d10, {"period", "rows", "axis_form"}, {"diff:axis_form:交叉表"}), + "D11": (_d11, {"period", "rows", "label_writing"}, {"diff:label_writing:日间", "diff:label_writing:夜间"}), + "D19": (_d19, {"period", "rows", "row_order"}, {"diff:row_order:日客流"}), + "D22": (_d22, {"period", "rows", "label_writing"}, {"diff:label_writing:夜间合计"}), + "D24": (_d24, {"period", "rows", "label_writing"}, {"diff:label_writing:日间", "diff:label_writing:夜间"}), + "D20": (_d20, {"period", "rows"}, {"sheet_extra:说明"}), + "D28": (_d28, {"period", "rows", "outside_added"}, {"outside_digits:客流汇总!B32", "diff:outside:客流汇总!B32"}), + "D28b": (_d28b, {"period", "rows", "outside_added"}, + {"outside_digits:客流汇总!B32", "diff:outside:客流汇总!B32"}), + "D26": (_d26, {"period", "rows", "checks"}, set()), +} +#: 拒收的 7 例：试运行就拒收了，9.2 不给差异和确认项的期望（diff_kinds=None），这里不跑 +REJECTED = {"D04", "D09", "D13", "D16", "D18", "D21", "D08"} + +#: 9.1：D00 首次导入时的确认项（精确集合） +D00_CONFIRMS = { + "placeholder:·", "derived:夜间合计", "year_from:交叉表", "relation:R1", "relation:R2", + "unit:日客流.全日客流", "unit:日客流.分区甲", "unit:日客流.分区乙", "unit:时段客流.客流", + "unit:时段客流_表内合计.客流", "mode", +} + + +def match_ids(actual: set[str], expected: set[str]) -> None: + """精确集合比对；expected 里以 * 结尾的按前缀匹配（至少命中一个）。""" + exact = {e for e in expected if not e.endswith("*")} + prefixes = [e[:-1] for e in expected if e.endswith("*")] + rest = actual - exact + assert exact <= actual, f"缺：{exact - actual}" + for p in prefixes: + assert any(a.startswith(p) for a in rest), f"没有以 {p} 开头的" + stray = {a for a in rest if not any(a.startswith(p) for p in prefixes)} + assert not stray, f"多出：{stray}" + + +def test_drift_table_covers_all_31_cases(): + assert len(DRIFT) + len(REJECTED) + 1 == 31 # + D00 + + +def test_d00_first_import(): + doc, name = _d00() + assert run(None, (doc, name)) == [] + items = confirm_items(ConfirmContext(kind="first", recipe=FLOW, extraction=extraction_of(doc), + checks=doc["checks"], diff=[], prev=None, recipe_origin="rules")) + assert {i.id for i in items} == D00_CONFIRMS + assert all(i.required for i in items) + + +def test_d00_same_file_again_has_no_diff(): + assert run(BASE, _d00()) == [] + + +@pytest.mark.parametrize("code", sorted(DRIFT)) +def test_drift_case(code): + build, want_kinds, want_confirms = DRIFT[code] + cur = build() + diff = run(BASE, cur, recipe=FLOW) + assert kinds(diff) == want_kinds + items = confirm_items(ConfirmContext(kind="reupload", recipe=FLOW, old_recipe=FLOW, + extraction=extraction_of(cur[0]), checks=cur[0]["checks"], diff=diff, + prev=BASE[0], recipe_origin="rules")) + match_ids({i.id for i in items}, want_confirms) + + +def test_drift_outside_added_cell_for_d06(): + diff = run(BASE, _d06()) + assert [d.confirm_id for d in diff if d.kind == "outside_added"] == ["diff:outside:客流汇总!B28"] + + +# -------------------------------------------------------------------------- +# 数字遮盖后的模板 +# -------------------------------------------------------------------------- + + +def test_digit_template(): + # 先 canon（NFKC 会把全角冒号变成半角），再遮盖 + assert D.digit_template("统计时间范围：2026年9月1日至2026年9月30日") == "统计时间范围:{}年{}月{}日至{}年{}月{}日" + assert D.digit_template("统计时间范围：2026年9月1日至9月30日") == "统计时间范围:{}年{}月{}日至{}月{}日" + assert D.digit_template("月报导出_2026-08-01_2026-08-31.xlsx") == "月报导出_{}-{}-{}_{}-{}-{}.xlsx" + # 全角数字先经 canon 变成 ASCII，再遮盖；连续的一段数字只算一个 {} + assert D.digit_template("２０２６年９月") == "{}年{}月" + assert D.digit_template("12345") == "{}" + assert D.digit_template("第 3 号") != D.digit_template("第 3 号门") + assert D.digit_template(None) == "" + + +# -------------------------------------------------------------------------- +# 各 kind 一正一反 +# -------------------------------------------------------------------------- + + +def test_first_import_returns_empty(): + assert run(None, sep()) == [] + + +def test_period_and_rows(): + diff = run(BASE, sep()) + p = next(d for d in diff if d.kind == "period") + assert "2026-08-01 至 2026-08-31" in p.label and "2026-09-01 至 2026-09-30" in p.label + r = next(d for d in diff if d.kind == "rows") + assert "「日客流」31 → 30" in r.label and "「时段客流」527 → 510" in r.label + assert "占位符「·」31 → 30 格" in r.detail + assert not p.requires_confirm and not r.requires_confirm + assert kinds(run(BASE, _d02())) == {"period"} # 行数相同不出 rows + + +def test_rows_lists_added_and_removed_tables(): + cur = sep() + cur[0]["receipt"]["tables"].pop() + cur[0]["receipt"]["tables"].append({"name": "新表", "rows": 3, "columns": []}) + r = next(d for d in run(BASE, cur) if d.kind == "rows") + assert "「新表」本期新增，3 行" in r.label and "「时段客流_表内合计」本期没有，上一期 93 行" in r.label + + +def test_placeholders_set_not_count(): + assert "placeholders" not in kinds(run(BASE, sep())) # 31 → 30 只是个数 + cur = sep() + cur[0]["receipt"]["placeholders"] = {"·": 30, "-": 2} + item = next(d for d in run(BASE, cur) if d.kind == "placeholders") + assert "新出现「-」" in item.label and not item.requires_confirm + cur[0]["receipt"]["placeholders"] = {"-": 2, "·": 0} + item = next(d for d in run(BASE, cur) if d.kind == "placeholders") + assert "不再出现「·」" in item.label + + +def test_checks_set_not_count(): + assert "checks" not in kinds(run(BASE, sep())) # K1 93 → 90 只是核对数 + item = next(d for d in run(BASE, _d23()) if d.kind == "checks") + assert "C2 通过 → 提示" in item.label + item = next(d for d in run(BASE, _d07()) if d.kind == "checks") + assert "G1 本期不再出现" in item.label + + +def test_file_name_template(): + assert "file_name" not in kinds(run(BASE, sep())) + item = next(d for d in run(BASE, _d23()) if d.kind == "file_name") + assert "9月客流.xlsx" in item.label and not item.requires_confirm + # 目录不同不算，只看文件名本身 + assert "file_name" not in kinds(D.diff_reports(BASE[0], sep()[0], prev_file_name="a/" + BASE[1], + cur_file_name="b\\" + sep()[1], recipe=FLOW, + recipe_changed=False)) + assert "file_name" not in kinds(D.diff_reports(BASE[0], sep()[0], prev_file_name=None, cur_file_name="x.xlsx", + recipe=FLOW, recipe_changed=False)) + + +def test_full_calc_whenever_set_this_period(): + # 规格 7.6：「这一期设了打开时重算」就出 info，不论上一期设没设 + item = next(d for d in run(BASE, _d25()) if d.kind == "full_calc") + assert not item.requires_confirm and "上一期没有设置" in item.detail + prev = (copy.deepcopy(BASE[0]), BASE[1]) + prev[0]["receipt"]["full_calc_on_load"] = True + item = next(d for d in run(prev, _d25()) if d.kind == "full_calc") + assert "上一期也设置了" in item.detail + assert "full_calc" not in kinds(run(BASE, sep())) + assert "full_calc" not in kinds(run(prev, sep())) # 上一期设了、本期没设：不出 + + +def test_context_text(): + item = next(d for d in run(BASE, _d12()) if d.kind == "context_text") + assert "客流汇总!B2" in item.label and not item.requires_confirm + cur = _d12() + set_check(cur[0], "C1", "mismatch") + assert "context_text" not in kinds(run(BASE, cur)) + assert "context_text" not in kinds(run(BASE, sep())) # 只有日期变了 + + +def test_context_source_added_d17_d27(): + for build in (_d17, _d27): + diff = run(BASE, build()) + item = next(d for d in diff if d.kind == "context_source_added") + assert not item.requires_confirm and "客流汇总!B3" in item.label + assert not any(d.kind.startswith("outside_") for d in diff) + + +def test_context_source_added_needs_same_residue(): + # 附加文字不是上一期的标题：按区域外文字改了处理，要确认 + cur = sep() + set_period_cell(cur[0], "B3", "客流汇总初步表（2026年9月）", annotated=True) + diff = run(BASE, cur) + assert "context_source_added" not in kinds(diff) + item = next(d for d in diff if d.kind == "outside_changed") + assert item.confirm_id == "diff:outside:客流汇总!B3" + + +def test_context_source_added_needs_c1_passed(): + cur = _d17() + set_check(cur[0], "C1", "mismatch") + diff = run(BASE, cur) + assert "context_source_added" not in kinds(diff) + assert "outside_changed" in kinds(diff) + + +def test_context_source_added_pure_period_cell(): + # B3 换成只是在说统计期的格：不在区域外文字里，C1 一致就是 context_source_added + cur = sep() + set_period_cell(cur[0], "B3", "2026年9月1日至2026年9月30日", annotated=False) + diff = run(BASE, cur) + assert "context_source_added" in kinds(diff) and "outside_removed" not in kinds(diff) + set_check(cur[0], "C1", "mismatch") + diff = run(BASE, cur) + assert [d.confirm_id for d in diff if d.kind == "outside_changed"] == ["diff:outside:客流汇总!B3"] + + +def _list_doc(order: list[str], headers: dict[str, str] | None = None) -> dict[str, Any]: + """一张列表表（销售），lineage 里每列一段向下的溯源，列号按 order 排。""" + doc = flow_doc(SEP, 30) + headers = headers or {} + cols = ["地区", "产品", "销量", "金额"] + doc["receipt"]["tables"].append({ + "name": "销售", "sheet": "销售", "kind": "data", "grain": ["地区", "产品"], "rows": 4, + "columns": [_col(c, "TEXT" if c in ("地区", "产品") else "INTEGER", "text" if c in ("地区", "产品") else "measure", + headers.get(c, c)) for c in cols]}) + doc["receipt"]["lineage"]["销售"] = {c: [[1, "销售", f"{chr(67 + order.index(c))}6", 4, "down"]] for c in cols} + return doc + + +LIST_RECIPE: dict[str, Any] = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s2", "match": {"name": "销售"}, "blocks": [{ + "id": "列表1", "layout": "list", "table": "销售", + "columns": [{"header": "地区", "name": "地区", "type": "TEXT"}, {"header": "产品", "name": "产品", "type": "TEXT"}, + {"header": "销量", "name": "销量", "type": "INTEGER"}, {"header": "金额", "name": "金额", "type": "INTEGER"}], + }]}], + "tables": [{"name": "销售", "grain": ["地区", "产品"]}], +} + + +def test_column_order(): + prev = (_list_doc(["地区", "产品", "销量", "金额"]), "a.xlsx") + cur = (_list_doc(["产品", "地区", "销量", "金额"]), "a.xlsx") + item = next(d for d in run(prev, cur, recipe=LIST_RECIPE) if d.kind == "column_order") + assert "「销售」" in item.label and not item.requires_confirm + assert "column_order" not in kinds(run(prev, (_list_doc(["地区", "产品", "销量", "金额"]), "a.xlsx"), + recipe=LIST_RECIPE)) + # 交叉表的表不比列顺序（它们的列来自配方，不是工作表上的位置） + assert "column_order" not in kinds(run(BASE, _d19())) + + +def test_header_writing_keyed_by_block(): + prev = (_list_doc(["地区", "产品", "销量", "金额"]), "a.xlsx") + cur = (_list_doc(["地区", "产品", "销量", "金额"], {"金额": "金 额", "地区": "地区　"}), "a.xlsx") + item = next(d for d in run(prev, cur, recipe=LIST_RECIPE) if d.kind == "header_writing") + assert item.requires_confirm and item.confirm_id == "diff:header_writing:列表1" + assert "「金额」→「金 额」" in item.label + assert "header_writing" not in kinds(run(prev, prev, recipe=LIST_RECIPE)) + # 配方里找不到写入这张表的列表块（传错了配方）：报错，不拿表名凑一个键 + with pytest.raises(ValueError, match="列表块"): + run(prev, cur, recipe=FLOW) + # 逐项比较时不给配方：报错 + with pytest.raises(ValueError, match="本期配方"): + run(prev, cur, recipe=None) + + +def test_label_writing_and_row_order(): + diff = run(BASE, _d11()) + lw = {d.confirm_id for d in diff if d.kind == "label_writing"} + assert lw == {"diff:label_writing:日间", "diff:label_writing:夜间"} + assert "row_order" not in kinds(diff) + item = next(d for d in diff if d.confirm_id == "diff:label_writing:日间") + assert "「7-8」→「07:00-08:00」" in item.label + diff = run(BASE, _d19()) + assert "label_writing" not in kinds(diff) + assert [d.confirm_id for d in diff if d.kind == "row_order"] == ["diff:row_order:日客流"] + + +def test_label_writing_compares_raw_exactly(): + # en dash canon 之后与「-」相同，仍要报（E3）；只是行号变了不报 + assert "label_writing" in kinds(run(BASE, _d24())) + assert "label_writing" not in kinds(run(BASE, _d14())) + + +def test_canon_values(): + prev = (_list_doc(["地区", "产品", "销量", "金额"]), "a.xlsx") + prev[0]["receipt"]["canonicalized"] = [{"table": "销售", "column": "地区", "raw": "华 东", "canonical": "华 东", + "count": 1, "first_cell": "销售!C7"}] + cur = copy.deepcopy(prev) + assert "canon_values" not in kinds(run(prev, cur, recipe=LIST_RECIPE)) + cur[0]["receipt"]["canonicalized"].append({"table": "销售", "column": "地区", "raw": "华北　", + "canonical": "华北", "count": 2, "first_cell": "销售!C9"}) + item = next(d for d in run(prev, cur, recipe=LIST_RECIPE) if d.kind == "canon_values") + assert item.confirm_id == "diff:canon:销售.地区" and item.requires_confirm + # 标签的写法变化不进 canonicalized，所以 D11 没有 canon_values + assert "canon_values" not in kinds(run(BASE, _d11())) + + +def test_block_order(): + diff = run(BASE, _d05()) + assert [d.confirm_id for d in diff if d.kind == "block_order"] == ["diff:block_order:客流汇总"] + assert "block_order" not in kinds(run(BASE, _d14())) + + +def test_block_order_follows_renamed_sheet(): + cur = _d05() + rec = cur[0]["receipt"] + rec["block_order"] = {"汇总": rec["block_order"][SHEET]} + rec["sheets"]["matched"] = {"s1": "汇总"} + item = next(d for d in run(BASE, cur) if d.kind == "block_order") + assert item.confirm_id == "diff:block_order:汇总" + + +def test_derived_and_axis_form(): + item = next(d for d in run(BASE, _d07()) if d.kind == "derived_form") + assert item.confirm_id == "diff:derived_form:夜间合计" and "由公式改为写死的数" in item.label + item = next(d for d in run(BASE, _d10()) if d.kind == "axis_form") + assert item.confirm_id == "diff:axis_form:交叉表" and "由文字改为日期格" in item.label + assert not kinds(run(BASE, sep())) & {"derived_form", "axis_form"} + + +def test_outside_added_removed_changed(): + cur = _d28() + item = next(d for d in run(BASE, cur) if d.kind == "outside_added") + assert item.confirm_id == "diff:outside:客流汇总!B32" and "注：9月15日" in item.label + # 下一期同一句、只是日期变了：模板相同，不算改 + nxt = flow_doc(OCT, 31) + add_outside(nxt, "B32", "注：10月3日闸机故障，当日客流为估算值") + assert not kinds(run(cur, (nxt, file_name(OCT, 31)))) & {"outside_added", "outside_removed", "outside_changed"} + # 写法变了：要确认 + add_outside(nxt, "B33", "x") + nxt["receipt"]["outside_text"][-2]["text"] = "注：10月3日闸机检修，当日客流为估算值" + diff = run(cur, (nxt, file_name(OCT, 31))) + assert {d.confirm_id for d in diff if d.kind == "outside_changed"} == {"diff:outside:客流汇总!B32"} + assert {d.confirm_id for d in diff if d.kind == "outside_added"} == {"diff:outside:客流汇总!B33"} + # 删掉了 + item = next(d for d in run(cur, sep()) if d.kind == "outside_removed") + assert item.confirm_id == "diff:outside:客流汇总!B32" + + +def test_outside_pairs_full_text_first(): + # 只是位置变了：info，不要求确认 + prev = (copy.deepcopy(BASE[0]), BASE[1]) + add_outside(prev[0], "B32", "注：本表为测试") + cur = sep() + add_outside(cur[0], "B33", "注：本表为测试") + diff = run(prev, cur) + moved = [d for d in diff if d.kind == "outside_moved"] + assert len(moved) == 1 and not moved[0].requires_confirm + assert not kinds(diff) & {"outside_added", "outside_removed", "outside_changed"} + + +def test_outside_full_coordinates_accepted(): + # OutsideText.cell 已经带工作表名时，确认项 id 一样 + cur = sep() + cur[0]["receipt"]["outside_text"].append({"sheet": SHEET, "cell": f"{SHEET}!B32", "text": "注：9月15日", + "kind": "text_digits", "period_source": False}) + item = next(d for d in run(BASE, cur) if d.kind == "outside_added") + assert item.confirm_id == "diff:outside:客流汇总!B32" + + +def test_annotated_cell_monthly_same_sentence(): + # 上一期、本期同一格都是带附加文字的统计期格、句式相同：不出差异 + prev = sep() + set_period_cell(prev[0], "B32", "注：2026年9月数据为初步统计，待修订", annotated=True) + cur = (flow_doc(OCT, 31), file_name(OCT, 31)) + set_period_cell(cur[0], "B32", "注：2026年10月数据为初步统计，待修订", annotated=True) + diff = run(prev, cur) + assert not kinds(diff) & {"outside_added", "outside_removed", "outside_changed", "context_text"} + set_period_cell(cur[0], "B32", "注：2026年10月数据为正式统计", annotated=True) + assert "outside_changed" in kinds(run(prev, cur)) + + +def test_annotated_cell_other_digits_changed(): + # AU-3：两期都是带附加文字的统计期格，统计期以外的数字变了（只含3个分区 → 只含2个分区）要出 outside_changed + prev = sep() + set_period_cell(prev[0], "B32", "注：2026年9月数据只含3个分区", annotated=True) + cur = (flow_doc(OCT, 31), file_name(OCT, 31)) + set_period_cell(cur[0], "B32", "注：2026年10月数据只含2个分区", annotated=True) + diff = run(prev, cur) + assert [d.confirm_id for d in diff if d.kind == "outside_changed"] == ["diff:outside:客流汇总!B32"] + set_period_cell(cur[0], "B32", "注：2026年10月数据只含3个分区", annotated=True) + assert "outside_changed" not in kinds(run(prev, cur)) + + +def test_same_period_sentence(): + assert D.same_period_sentence("注：2026年9月数据为初步统计", "注：2026年10月数据为初步统计") + assert not D.same_period_sentence("注：2026年9月数据只含3个分区", "注：2026年10月数据只含2个分区") + # 认不出统计期（没有或有两段）就不算同一句 + assert not D.same_period_sentence("注：数据只含3个分区", "注：数据只含4个分区") + assert not D.same_period_sentence("2026年9月与2026年8月对比", "2026年10月与2026年9月对比") + + +def test_pure_period_cell_becomes_annotated(): + cur = sep() + set_period_cell(cur[0], "B2", b2_text(SEP, dt.date(2026, 9, 30)) + "（初步）", annotated=True) + diff = run(BASE, cur) + assert [d.confirm_id for d in diff if d.kind == "outside_changed"] == ["diff:outside:客流汇总!B2"] + assert "context_text" not in kinds(diff) + + +def test_ignored_columns(): + prev = (copy.deepcopy(BASE[0]), BASE[1]) + prev[0]["receipt"]["ignored_columns"] = {"列表1": ["备注"]} + cur = sep() + cur[0]["receipt"]["ignored_columns"] = {"列表1": ["备 注"]} + assert "ignored_columns" not in kinds(run(prev, cur)) # 只是空白 + cur[0]["receipt"]["ignored_columns"] = {"列表1": ["备注", "经办人"]} + item = next(d for d in run(prev, cur) if d.kind == "ignored_columns") + assert item.confirm_id == "diff:ignored:列表1" and "「经办人」" in item.label + + +def test_hidden(): + prev = (copy.deepcopy(BASE[0]), BASE[1]) + prev[0]["receipt"]["hidden"] = {SHEET: {"rows": [4], "cols": [], "policy_rows": "include", + "policy_cols": "reject_if_any"}} + cur = sep() + cur[0]["receipt"]["hidden"] = {SHEET: {"rows": [4], "cols": [], "policy_rows": "include", + "policy_cols": "reject_if_any"}} + assert "hidden" not in kinds(run(prev, cur)) + cur[0]["receipt"]["hidden"][SHEET]["rows"] = [4, 9] + item = next(d for d in run(prev, cur) if d.kind == "hidden") + assert item.confirm_id == "diff:hidden:客流汇总" and "第 4、9 行" in item.label + # exclude 下被排除的行变了同样要确认（会改变导入的数据） + for doc in (prev[0], cur[0]): + doc["receipt"]["hidden"][SHEET]["policy_rows"] = "exclude" + assert "hidden" in kinds(run(prev, cur)) + # 隐藏列（include） + cur[0]["receipt"]["hidden"][SHEET].update(rows=[4], policy_cols="include", cols=[3]) + prev[0]["receipt"]["hidden"][SHEET].update(policy_cols="include", cols=[]) + item = next(d for d in run(prev, cur) if d.kind == "hidden") + assert "C 列" in item.label + + +def test_limited_when_prev_is_simple_import(): + # 期 1 的 TableBuild.report：没有 ledger，只有表和行数 + prev = ({"receipt": {"tables": [{"name": "客流汇总", "rows": 29, "columns": []}]}, "checks": [], "period": None}, + "客流.xlsx") + diff = run(prev, _d28()) + assert kinds(diff) == {"period", "rows"} + assert all(not d.requires_confirm for d in diff) + + +def test_limited_when_recipe_changed(): + assert kinds(run(BASE, _d11(), recipe_changed=True)) == {"period", "rows"} + prev = (dict(BASE[0], recipe_sha256="a" * 64), BASE[1]) + cur = _d11() + assert kinds(run(prev, (dict(cur[0], recipe_sha256="b" * 64), cur[1]), recipe_changed=None)) == {"period", "rows"} + assert "label_writing" in kinds(run(prev, (dict(cur[0], recipe_sha256="a" * 64), cur[1]), recipe_changed=None)) + + +def test_recipe_sha_from_manifest_shape(): + # 导入清单（7.7）把配方哈希放在 recipe.sha256：直接拿清单当 prev 也认得 + manifest = dict(copy.deepcopy(BASE[0]), recipe={"id": "r1", "sha256": "a" * 64, "origin": "rules"}) + cur = _d11() + changed = dict(cur[0], recipe={"sha256": "b" * 64}) + assert kinds(run((manifest, BASE[1]), (changed, cur[1]), recipe_changed=None)) == {"period", "rows"} + same = dict(cur[0], recipe_sha256="a" * 64) + assert "label_writing" in kinds(run((manifest, BASE[1]), (same, cur[1]), recipe_changed=None)) + + +def test_recipe_changed_undecidable_raises(): + # 没传 recipe_changed、两边又没都带哈希：不默认「没变」去跨配方逐项比 + manifest = dict(copy.deepcopy(BASE[0]), recipe={"sha256": "a" * 64}) + with pytest.raises(ValueError, match="recipe_changed"): + run((manifest, BASE[1]), _d11(), recipe_changed=None) + with pytest.raises(ValueError, match="recipe_changed"): + run(BASE, _d11(), recipe_changed=None) + # 传了、两边哈希也都在，但互相矛盾 + cur = _d11() + with pytest.raises(ValueError, match="不一致"): + run((manifest, BASE[1]), (dict(cur[0], recipe_sha256="b" * 64), cur[1]), recipe_changed=False) + with pytest.raises(ValueError, match="不一致"): + run((manifest, BASE[1]), (dict(cur[0], recipe_sha256="a" * 64), cur[1]), recipe_changed=True) + # 上一期是简单导入：只给 rows、period，用不着判断配方 + simple = ({"receipt": {"tables": [{"name": "客流汇总", "rows": 29, "columns": []}]}, "checks": [], "period": None}, + "客流.xlsx") + assert kinds(run(simple, _d11(), recipe=None, recipe_changed=None)) == {"period", "rows"} + + +@pytest.mark.parametrize("key", D.RECEIPT_KEYS) +@pytest.mark.parametrize("side", ["prev", "cur"]) +def test_missing_receipt_key_raises(side, key): + # 早先缺键按空处理：prev 少了 axes 配 D10、少了 block_order 配 D05……需确认的项就静默消失了 + prev, cur = copy.deepcopy(BASE[0]), _d10()[0] + del (prev if side == "prev" else cur)["receipt"][key] + with pytest.raises(D.ReceiptIncomplete) as err: + run((prev, BASE[1]), (cur, "a.xlsx")) + assert err.value.side == side and err.value.missing == [key] + + +@pytest.mark.parametrize("side", ["prev", "cur"]) +def test_missing_checks_or_receipt_raises(side): + prev, cur = copy.deepcopy(BASE[0]), sep()[0] + del (prev if side == "prev" else cur)["checks"] + with pytest.raises(D.ReceiptIncomplete, match="checks"): + run((prev, BASE[1]), (cur, "a.xlsx")) + prev, cur = copy.deepcopy(BASE[0]), sep()[0] + del (prev if side == "prev" else cur)["receipt"] + with pytest.raises(D.ReceiptIncomplete, match="receipt"): + run((prev, BASE[1]), (cur, "a.xlsx")) + # 只给行数和统计期时（简单导入）回执里也得有 tables + simple = {"receipt": {"warnings": []}, "checks": [], "period": None} + with pytest.raises(D.ReceiptIncomplete, match="tables"): + run((simple, "a.xlsx"), sep()) + + +def test_truncated_canonicalized_raises(): + # TrialOut.receipt 的 canonicalized 只有前 50 条：拿它当 cur 会漏掉新写法 + cur = sep() + cur[0]["receipt"]["canonicalized_total"] = 51 + cur[0]["receipt"]["canonicalized"] = [{"table": "销售", "column": "地区", "raw": f"华{i} ", "canonical": f"华{i}", + "count": 1, "first_cell": "销售!C7"} for i in range(50)] + with pytest.raises(D.ReceiptIncomplete, match="canonicalized_total"): + run(BASE, cur) + cur[0]["receipt"]["canonicalized_total"] = 50 + assert "canon_values" in kinds(run(BASE, cur)) + + +def test_partial_receipt_raises(): + cur = sep() + cur[0]["receipt"]["partial"] = True + with pytest.raises(D.ReceiptIncomplete, match="partial"): + run(BASE, cur) + + +def test_requires_confirm_first(): + cur = _d06() + diff = run(BASE, cur) + flags = [d.requires_confirm for d in diff] + assert flags == sorted(flags, reverse=True) and flags[0] and not flags[-1] + for d in diff: + assert d.kind in D.DIFF_KINDS + assert (d.confirm_id is not None) == d.requires_confirm + if d.requires_confirm: + assert d.confirm_id.startswith(D.CONFIRM_PREFIX[d.kind]) + + +def test_accepts_dataclass_inputs(): + from app.data.recipe_types import ColumnOut, Extraction, PeriodOut, TableOut + + def doc(rows: int, start: str) -> dict[str, Any]: + cols = [ColumnOut("日期", "TEXT", role="axis")] + ex = Extraction(ok=True, tables=[TableOut("日客流", SHEET, "data", cols, ["日期"], rows)], + period=PeriodOut(start, start, "cells"), ledger=[]) + return {"receipt": ex, "checks": [], "period": ex.period} + + diff = D.diff_reports(doc(31, "2026-08-01"), doc(30, "2026-09-01"), prev_file_name="a", cur_file_name="a", + recipe=FLOW, recipe_changed=False) + assert kinds(diff) == {"period", "rows"} + + +def test_period_missing_sides(): + cur = sep() + cur[0]["period"] = None + cur[0]["receipt"]["period"] = None + item = next(d for d in run(BASE, cur) if d.kind == "period") + assert "本期未能确定" in item.label diff --git a/backend/tests/test_recipe_e2e.py b/backend/tests/test_recipe_e2e.py new file mode 100644 index 00000000..eb958ab6 --- /dev/null +++ b/backend/tests/test_recipe_e2e.py @@ -0,0 +1,1142 @@ +"""按配方导入的端到端验收（期 2，WP-7；P2-SPEC 9.7 的 13 步）。 + +**流程一律经 HTTP 接口**（`AsyncClient(transport=ASGITransport(app=app))`）：暂存、回答、改配方、AI 起草、 +试运行、提交、上传新一期、清除原件、发起运行、恢复运行、取工件、试跑工具，全部打真实的路由，管线是 +`default_pipeline()` 接的真模块。只有接口不提供的观察点（回收、过期时间、构建 id 的期望值、快照的库哈希、 +运行固定的版本）才直接读库或调模块函数，而且只读、不替流程做任何一步。 + +夹具：tests/fixtures/xlsx 的合成表格（假名、随机数），AI 用假模型（monkeypatch `recipe_ai.resolve_draft_model`）， +不调用任何真实模型，不读 .env。 + +这份文件是验收：断言照规格写，跑出来失败的用例保留失败，不放宽断言、不标 xfail，根因在交付说明里逐条写。 +""" +from __future__ import annotations + +import asyncio +import datetime as dt +import hashlib +import json +import uuid +from pathlib import Path +from typing import Any, Callable + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select, update + +from app.core.config import settings +from app.data import raw_store, recipe, table_versions +from app.data.recipe_types import prose_problems +from app.db.base import SessionLocal +from app.db.models import DataSource, ImportStaging, Run, SourceSnapshot, TableImport, TableRecipe +from app.engine.runner import run_manager +from app.main import app +from tests.fixtures.xlsx.drift import AUG, DRIFT_CASES, OCT, SEP, DriftCase +from tests.fixtures.xlsx.flow import FLOW_EXPECT, flow_workbook + +XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +API = "/api/datasources" +FLOW_RECIPE = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text("utf-8")) +#: 基线和各漂移用例共用的随机种子：D00 用例与基线逐字节相同，就是「同一文件再传」 +SEED = 5 + + +# ========================================================================== +# 夹具 +# ========================================================================== + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + """每个测试一个独立的数据目录：上传目录、原件存档、工件、试运行库都在里面（库是会话级的临时库）。""" + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +async def engine_up(store): + """发起运行要的引擎（checkpoint 库跟着 data_dir 走，所以依赖 store）。""" + await run_manager.setup() + yield + await run_manager.shutdown() + + +# ========================================================================== +# 小工具（全部经 HTTP） +# ========================================================================== + + +def unique(prefix: str = "e2e") -> str: + return f"{prefix}_{uuid.uuid4().hex[:10]}" + + +async def stage(client, name: str, raw: bytes, filename: str, **form: str): + return await client.post(f"{API}/imports/stage", files={"file": (filename, raw, XLSX)}, + data={"name": name, **form}) + + +async def reupload(client, source_id: str, raw: bytes, filename: str, **form: str): + return await client.post(f"{API}/{source_id}/reupload", files={"file": (filename, raw, XLSX)}, data=form) + + +async def trial(client, sid: str, **body: Any): + return await client.post(f"{API}/imports/{sid}/trial", json=body) + + +async def commit(client, st: dict[str, Any], confirmations: list[str] | None = None, + acceptances: list[dict[str, str]] | None = None, **extra: Any): + body: dict[str, Any] = {"trial_id": st["trial"]["trial_id"], + "confirmations": all_confirms(st) if confirmations is None else confirmations} + if acceptances is not None: + body["acceptances"] = acceptances + body.update(extra) + return await client.post(f"{API}/imports/{st['id']}/commit", json=body) + + +def all_confirms(st: dict[str, Any]) -> list[str]: + return [i["id"] for i in (st.get("trial") or {}).get("confirm_items") or []] + + +def rename_wide(rec: dict[str, Any], old: str, new: str) -> dict[str, Any]: + """把规则起草的宽表名改成 new：表名、写入它的分段（id 一起改）、关系里引用的表名。""" + rec = json.loads(json.dumps(rec, ensure_ascii=False)) + for sheet in rec["sheets"]: + for block in sheet["blocks"]: + for seg in block.get("segments", []): + if seg.get("table") == old: + seg["table"] = new + if seg.get("id") == old: + seg["id"] = new + for t in rec["tables"]: + if t["name"] == old: + t["name"] = new + for r in rec["relations"]: + if r.get("table") == old: + r["table"] = new + for side in ("a", "b"): + if isinstance(r.get(side), dict) and r[side].get("table") == old: + r[side]["table"] = new + return rec + + +async def query(client, source_name: str, sql: str) -> list[list[Any]]: + """工具库「试一下」：上传表格查当前版本（POST /api/tools/db_query__名/run）。""" + resp = await client.post(f"/api/tools/db_query__{source_name}/run", json={"args": {"sql": sql}}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["ok"], out + return json.loads(out["result"])["rows"] + + +async def count(client, source_name: str, table: str, where: str = "") -> int: + rows = await query(client, source_name, f'SELECT COUNT(*) AS n FROM "{table}"{where}') + return int(rows[0][0]) + + +async def schema(client, source_id: str, table: str) -> dict[str, Any]: + resp = await client.get(f"{API}/{source_id}/schema", params={"table": table}) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def source_row(client, source_id: str) -> dict[str, Any]: + resp = await client.get(API) + assert resp.status_code == 200, resp.text + return next(s for s in resp.json() if s["id"] == source_id) + + +async def artifact(client, artifact_id: str) -> dict[str, Any]: + resp = await client.get(f"/api/artifacts/{artifact_id}") + assert resp.status_code == 200, resp.text + return resp.json()["content"] + + +async def db_get(model, key): + async with SessionLocal() as session: + return await session.get(model, key) + + +def notes_text(trial_or_notes: dict[str, Any]) -> str: + """说明预览（TrialOut.notes）的全部文字：表说明 + 列说明。""" + notes = trial_or_notes.get("notes", trial_or_notes) + parts = [] + for t in (notes or {}).values(): + parts.append(t.get("comment") or "") + parts.extend((t.get("columns") or {}).values()) + return "\n".join(parts) + + +async def schema_notes(client, source_id: str, tables: list[str]) -> str: + parts = [] + for t in tables: + s = await schema(client, source_id, t) + parts.append(s.get("comment") or "") + parts.extend(c.get("comment") or "" for c in s.get("columns") or []) + return "\n".join(parts) + + +def _ref(o: dict[str, Any]) -> str: + """OutsideText 的坐标：cell 已带工作表名（「客流汇总!B3」）时原样用，否则拼上 sheet。""" + return o["cell"] if "!" in o["cell"] else f"{o['sheet']}!{o['cell']}" + + +def answer_all(questions: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + """D00 的问题：关系一律登记，占位符存为空值（9.1 的期望配方）。""" + out = {} + for q in questions: + values = [o["value"] for o in q["options"]] + if q["id"].startswith("q_relation"): + out[q["id"]] = {"value": "register"} + elif "null" in values: + out[q["id"]] = {"value": "null"} + else: + out[q["id"]] = {"value": values[0]} + return out + + +async def first_import(client, name: str, *, seed: int = SEED, start: dt.date = AUG, days: int = 31, + variant: str = "D00", signed_by: str = "验收员甲") -> dict[str, Any]: + """第 1、2 步：stage → 回答 → 改宽表名（PUT）→ trial → commit（勾全部确认项）。返回各步的响应。""" + raw, fn = flow_workbook(start, days, seed=seed, variant=variant) + resp = await stage(client, name, raw, fn) + assert resp.status_code == 201, resp.text + staged = resp.json() + sid = staged["id"] + resp = await client.post(f"{API}/imports/{sid}/answers", json={"answers": answer_all(staged["questions"])}) + assert resp.status_code == 200, resp.text + answered = resp.json() + wide = next(t["name"] for t in answered["recipe"]["tables"] if t["name"] not in ("时段客流", "时段客流_表内合计")) + resp = await client.put(f"{API}/imports/{sid}/recipe", json={"recipe": rename_wide(answered["recipe"], wide, "日客流")}) + assert resp.status_code == 200, resp.text + put = resp.json() + resp = await trial(client, sid) + assert resp.status_code == 200, resp.text + tried = resp.json() + assert tried["trial"]["status"] == "passed", tried["trial"]["problems"] + resp = await commit(client, tried, signed_by=signed_by) + assert resp.status_code == 201, resp.text + return {"raw": raw, "file_name": fn, "staged": staged, "answered": answered, "put": put, "trial": tried, + "commit": resp.json()} + + +# ========================================================================== +# 第 1、2 步：首次导入 +# ========================================================================== + + +async def test_step01_02_first_import_matches_the_reference_recipe_and_9_1(client): + name = unique() + raw, fn = flow_workbook(AUG, 31, seed=101) + resp = await stage(client, name, raw, fn) + assert resp.status_code == 201, resp.text + st = resp.json() + sid = st["id"] + # 第 1 步：规则草稿完整、卡片 10 张、不提供 AI + assert st["kind"] == "first" and st["status"] == "drafting" + assert st["draft"]["complete"] is True + assert len(st["cards"]) == 10 + assert all(c.get("reason") for c in st["cards"]), "每张卡片都有理由" + assert st["ai"]["offered"] is False + assert st["recipe_problems"] == [] + assert st["sheets"][0]["nonempty"] == FLOW_EXPECT["nonempty"] + assert st["sheets"][0]["bounds"] == FLOW_EXPECT["bounds"] + assert st["sheets"][0]["merged"] == FLOW_EXPECT["merged"] + assert st["sheets"][0]["formulas"] == FLOW_EXPECT["formulas"] + # 问题没有默认选中 + assert all(q.get("default") in (None, "") for q in st["questions"]) + + resp = await client.post(f"{API}/imports/{sid}/answers", json={"answers": answer_all(st["questions"])}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert set(st["answers"]) == {q["id"] for q in st["questions"]} + wide = next(t["name"] for t in st["recipe"]["tables"] if t["name"] not in ("时段客流", "时段客流_表内合计")) + resp = await client.put(f"{API}/imports/{sid}/recipe", json={"recipe": rename_wide(st["recipe"], wide, "日客流")}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["recipe_problems"] == [] and st["answers"] == {} + ref, ref_problems = recipe.validate_recipe(FLOW_RECIPE, origin="manual") + mine, mine_problems = recipe.validate_recipe(st["recipe"], origin="manual") + assert ref_problems == [] and mine_problems == [] + assert recipe.recipe_sha256(mine) == recipe.recipe_sha256(ref), "回答 + 改名之后应与参考配方逐字相同" + + # 第 2 步：trial → passed，回执与 9.1 一致 + resp = await trial(client, sid) + assert resp.status_code == 200, resp.text + st = resp.json() + t = st["trial"] + assert st["status"] == "trialed" and t["status"] == "passed", t["problems"] + rc = t["receipt"] + ledger = rc["ledger"][0] + assert ledger["nonempty_scan"] == ledger["nonempty_read"] == FLOW_EXPECT["nonempty"], "两遍一致" + assert ledger["unclaimed"] == 0 + assert {k: v for k, v in ledger["roles"].items() if v} == FLOW_EXPECT["ledger"] + assert {x["name"]: x["rows"] for x in rc["tables"]} == FLOW_EXPECT["rows"] + assert rc["placeholders"] == FLOW_EXPECT["placeholders"] + assert rc["canonicalized_total"] == FLOW_EXPECT["canonicalized"] + assert [(_ref(o), o["text"], o["kind"]) for o in rc["outside_text"]] == FLOW_EXPECT["outside_text"] + period = rc["period"] + assert period["source"] == "cells" and (period["start"], period["end"]) == ("2026-08-01", "2026-08-31") + assert len(period["texts"]) == 1 and next(iter(period["texts"])).endswith("B2"), period["texts"] + assert period["annotated"] == {} + checks = {c["id"]: c for c in t["checks"]} + assert {k: c["status"] for k, c in checks.items()} == FLOW_EXPECT["checks"] + assert checks["K1"]["checked"] == 93, "K1 核对 3 × 31 格" + assert checks["R1"]["checked"] == 31 and checks["R1"]["failed"] == 0 + assert checks["C1"]["checked"] == 1, "C1 一处" + assert checks["R2"]["checked"] == 31 and "31 天中 0 天相等" in " ".join(checks["R2"]["details"]) + assert set(all_confirms(st)) == set(FLOW_EXPECT["confirms"]), "确认项是精确集合" + assert t["diff"] is None and t["same_as_import"] is None + for table, note in t["notes"].items(): + assert prose_problems(note["comment"], known={table, *note["columns"]}) == [], table + assert "已逐日核对" in t["notes"]["日客流"]["comment"] + assert "本期原表附有说明文字" not in notes_text(t) + assert t["notes"]["日客流"]["columns"]["全日客流"] == "单位：人次" + + resp = await commit(client, st, signed_by="验收员甲") + assert resp.status_code == 201, resp.text + out = resp.json() + src = out["source"] + assert src["name"] == name and src["import_mode"] == "recipe" and src["origin"] == "upload" + # 当前配方就是参考配方 + resp = await client.get(f"{API}/{src['id']}/recipe") + assert resp.status_code == 200, resp.text + assert resp.json()["recipe_sha256"] == recipe.recipe_sha256(ref) + # 快照里三张表的行数（经工具库试跑，查的是当前版本） + for table, n in FLOW_EXPECT["rows"].items(): + assert await count(client, name, table) == n, table + # D11：7-8 时段整行是「·」，31 行保留为空值行 + assert await count(client, name, "时段客流", " WHERE \"时段\" = '7-8' AND \"客流\" IS NULL") == 31 + s = await schema(client, src["id"], "日客流") + assert s["found"] and "已逐日核对" in (s["comment"] or "") + col = next(c for c in s["columns"] if c["name"] == "全日客流") + assert col["comment"] == "单位：人次" + # 主键：STRICT + PRIMARY KEY（grain） + assert [c["name"] for c in s["columns"] if c["pk"]] == ["日期"] + + +# ========================================================================== +# 第 3 步：运行固定版本 +# ========================================================================== + + +def _node(nid, ntype, **config): + return {"id": nid, "type": ntype, "position": {"x": 0, "y": 0}, "data": {"label": nid, "config": config}} + + +def _chain(*nodes): + return {"nodes": list(nodes), "edges": [{"source": a["id"], "target": b["id"]} for a, b in zip(nodes, nodes[1:])]} + + +async def _wait_run(client, run_id: str, statuses: tuple[str, ...], timeout: float = 30.0) -> dict[str, Any]: + deadline = asyncio.get_event_loop().time() + timeout + while asyncio.get_event_loop().time() < deadline: + resp = await client.get(f"/api/runs/{run_id}") + if resp.status_code == 200 and resp.json()["status"] in statuses and run_manager._tasks.get(run_id) is None: + return resp.json() + await asyncio.sleep(0.05) + raise AssertionError(f"运行 {run_id} 没有进入 {statuses}") + + +def _rows_of(value: Any) -> list[list[Any]]: + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + return json.loads(text[text.index("{"):])["rows"] + + +async def test_step03_run_started_on_d00_keeps_reading_d00_after_reupload_d01_is_enabled(client, engine_up): + name = unique() + done = await first_import(client, name, seed=103) + source_id, s1 = done["commit"]["source"]["id"], done["commit"]["snapshot_id"] + sql = 'SELECT COUNT(*) AS n FROM "日客流"' + graph = _chain( + _node("start", "input"), + _node("t1", "tool", tool=f"db_query__{name}", args={"sql": sql}), + _node("gate", "human", mode="approve", title="看一眼"), + _node("t2", "tool", tool=f"db_query__{name}", args={"sql": sql}), + _node("out", "output", fields=[{"name": "t1", "value": "{{ nodes.t1 }}"}, + {"name": "t2", "value": "{{ nodes.t2 }}"}]), + ) + resp = await client.post("/api/runs", json={"graph": graph, "input": {}}) + assert resp.status_code == 201, resp.text + run_id = resp.json()["id"] + await _wait_run(client, run_id, ("interrupted",)) + await run_manager.wait_idle(run_id) + + # 运行停在审批上：上传新一期 D01 并启用 + raw, fn = flow_workbook(SEP, 30, seed=1031) + resp = await reupload(client, source_id, raw, fn) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + s2 = resp.json()["snapshot_id"] + assert s2 != s1 + assert await count(client, name, "日客流") == 30, "当前版本已是 D01" + + resp = await client.post(f"/api/runs/{run_id}/resume", json={"response": {"approved": True}}) + assert resp.status_code == 200, resp.text + done_run = await _wait_run(client, run_id, ("succeeded", "failed")) + assert done_run["status"] == "succeeded", done_run["error"] + assert _rows_of(done_run["output"]["t1"]) == [[31]] + assert _rows_of(done_run["output"]["t2"]) == [[31]], "恢复之后查到了新一期，运行没有固定在 D00 的快照上" + row = await db_get(Run, run_id) + assert row.data_versions == {source_id: {"snapshot": s1, "name": name}} + + +# ========================================================================== +# 第 4 步：31 例漂移（30 例 + D28b） +# ========================================================================== + + +def _match(actual: str, pattern: str) -> bool: + return actual.startswith(pattern[:-1]) if pattern.endswith("*") else actual == pattern + + +def _confirm_diff(actual: list[str], expected: tuple[str, ...]) -> tuple[list[str], list[str]]: + missing = [p for p in expected if not any(_match(a, p) for a in actual)] + extra = [a for a in actual if not any(_match(a, p) for p in expected)] + return missing, extra + + +def _labels(rc: dict[str, Any], segment: str) -> dict[str, Any]: + return next(x for x in rc.get("labels") or [] if x["segment"] == segment) + + +def _cell_endswith(cells: list[str], ref: str) -> bool: + return any(c == ref or c.endswith(f"!{ref}") for c in cells or []) + + +#: 9.2「其他关键结果」一栏里，单看回执就能断言的部分 +def _extra_d23(t): + assert {c["id"]: c["status"] for c in t["checks"]}["C2"] == "info", "D23：C2 passed → info" + + +def _extra_d25(t): + k1 = next(c for c in t["checks"] if c["id"] == "K1") + assert k1["status"] == "passed" and "保存值与重算一致" in json.dumps(k1, ensure_ascii=False), k1 + + +def _extra_annotated_b3(t): + ann = t["receipt"]["period"]["annotated"] + assert any(k.endswith("B3") for k in ann), f"B3 应进 annotated：{ann}" + + +def _extra_d27(t): + _extra_annotated_b3(t) + cells = t["receipt"]["period"]["cells"] + assert any(c.endswith("B2") for c in cells) and any(c.endswith("B3") for c in cells), f"C1 两处一致：{cells}" + assert {c["id"]: c["status"] for c in t["checks"]}["C1"] == "passed" + + +def _extra_d28b(t): + ann = t["receipt"]["period"]["annotated"] + assert any(k.endswith("B32") for k in ann), f"B32 应进 annotated：{ann}" + cells = t["receipt"]["period"]["cells"] + assert any(c.endswith("B2") for c in cells) and any(c.endswith("B32") for c in cells), f"C1 两处一致：{cells}" + + +def _extra_d07(t): + ids = {c["id"]: c["status"] for c in t["checks"]} + assert ids.get("K1") == "passed" and "G1" not in ids, ids + + +def _extra_d11(t): + assert _labels(t["receipt"], "日间")["canonical"][0] == "7-8" + + +def _extra_d22(t): + assert "18-22时合计" in _labels(t["receipt"], "夜间合计")["canonical"] + + +def _extra_d08(t): + k1 = next(c for c in t["checks"] if c["id"] == "K1") + assert k1["status"] == "mismatch" and k1["category"] == "structure", k1 + assert _cell_endswith(k1["cells"], "G28"), f"K1 应定位到 G28：{k1['cells']}" + + +def _extra_d26(t): + r1 = next(c for c in t["checks"] if c["id"] == "R1") + assert r1["status"] == "mismatch" and r1["category"] == "data_quality" and r1["acceptable"] is True, r1 + assert t["acceptable"] == ["R1"] + + +EXTRA: dict[str, Callable[[dict[str, Any]], None]] = { + "D23": _extra_d23, "D25": _extra_d25, "D17": _extra_annotated_b3, "D27": _extra_d27, "D28b": _extra_d28b, + "D07": _extra_d07, "D11": _extra_d11, "D22": _extra_d22, "D08": _extra_d08, "D26": _extra_d26, +} + + +@pytest.mark.parametrize("code", list(DRIFT_CASES)) +async def test_step04_drift_case(client, code): + case: DriftCase = DRIFT_CASES[code] + name = unique(f"d_{code.lower()}") + base = await first_import(client, name, seed=SEED) + source_id = base["commit"]["source"]["id"] + base_snapshot = base["commit"]["snapshot_id"] + raw, fn = case.build(SEED) + resp = await reupload(client, source_id, raw, fn) + assert resp.status_code == 201, resp.text + st = resp.json() + t = st["trial"] + problems = {p["code"] for p in t["problems"]} + failing = {c["id"] for c in t["checks"] if c["status"] in ("mismatch", "unverifiable")} + status_want = {"passed": "passed", "confirm": "passed", "rejected": "rejected", "decision": "needs_decision"} + assert t["status"] == status_want[case.expect], (code, t["status"], t["problems"], + [(c["id"], c["status"]) for c in t["checks"]]) + for want in case.codes: + assert want in problems or want in failing, (code, want, problems, failing) + if code in EXTRA: + EXTRA[code](t) + + if case.expect == "rejected": + # 拒收不动当前快照；也没有「启用」可点（提交 409 trial_not_passed） + assert (await source_row(client, source_id))["current_snapshot"]["id"] == base_snapshot + resp = await commit(client, st, confirmations=[]) + assert resp.status_code == 409 and resp.json()["code"] == "trial_not_passed", resp.text + assert (await client.delete(f"{API}/imports/{st['id']}")).status_code == 204 + return + + assert {x["name"]: x["rows"] for x in t["receipt"]["tables"]} == dict(zip(FLOW_EXPECT["rows"], case.rows)) + assert t["diff"] is not None + assert {d["kind"] for d in t["diff"]} == set(case.diff_kinds), (code, [(d["kind"], d["label"]) for d in t["diff"]]) + ids = all_confirms(st) + missing, extra = _confirm_diff(ids, case.confirms) + assert not missing and not extra, (code, "缺", missing, "多", extra) + # 差异卡里需确认的项都在确认清单里，并排在差异卡前面 + for d in t["diff"]: + if d["requires_confirm"]: + assert d["confirm_id"] in ids, (code, d) + flags = [d["requires_confirm"] for d in t["diff"]] + assert flags == sorted(flags, reverse=True), (code, "需确认的差异项应排在前面", flags) + + if code == "D00": + # 同一文件再传：same_as_import，提交 unchanged，不新建记录 + assert t["same_as_import"] and t["same_as_import"]["id"] == base["commit"]["import_id"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert resp.json()["unchanged"] is True and resp.json()["import_id"] == base["commit"]["import_id"] + return + + if case.expect == "decision": + resp = await commit(client, st) + assert resp.status_code == 422 and resp.json()["code"] == "acceptance_required", resp.text + resp = await commit(client, st, acceptances=[{"check_id": "R1", "reason": "合成理由：闸机补录造成的差额"}]) + assert resp.status_code == 201, resp.text + text = await schema_notes(client, source_id, list(FLOW_EXPECT["rows"])) + else: + text = notes_text(t) + if ids: + resp = await commit(client, st, confirmations=[]) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required", resp.text + for i in ids: + assert i in resp.json()["detail"], (code, i) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + assert resp.json()["unchanged"] is False + for frag in case.note_has: + assert frag in text, (code, frag) + for frag in case.note_lacks: + assert frag not in text, (code, frag) + assert await count(client, name, "日客流") == case.rows[0] + + +# ========================================================================== +# 第 5 步：D26 写理由接受 +# ========================================================================== + + +async def test_step05_d26_acceptance_requires_a_reason_and_rewrites_the_note(client): + name = unique() + base = await first_import(client, name, seed=105) + source_id = base["commit"]["source"]["id"] + raw, fn = DRIFT_CASES["D26"].build(105) + st = (await reupload(client, source_id, raw, fn)).json() + t = st["trial"] + assert t["status"] == "needs_decision" and t["acceptable"] == ["R1"], t["problems"] + # 说明预览按「全部未接受」生成：R1 不成立，不能写「已逐日核对」 + assert "已逐日核对" not in notes_text(t) + resp = await commit(client, st) + assert resp.status_code == 422 and resp.json()["code"] == "acceptance_required", resp.text + assert "R1" in resp.json()["detail"] + resp = await commit(client, st, acceptances=[{"check_id": "R1", "reason": " "}]) + assert resp.status_code == 422 and resp.json()["code"] == "acceptance_required", "空白理由不算理由" + resp = await commit(client, st, acceptances=[{"check_id": "R1", "reason": "合成理由：第 10 日分区数据补录"}], + signed_by="验收员乙") + assert resp.status_code == 201, resp.text + s = await schema(client, source_id, "日客流") + assert "已逐日核对" not in (s["comment"] or "") + assert "已由用户确认接受" in (s["comment"] or "") + imports = (await client.get(f"{API}/{source_id}/imports")).json() + assert imports[0]["overrides"] == 1 and imports[0]["waivers"] == 0 + imp = await db_get(TableImport, resp.json()["import_id"]) + assert imp.overrides[0]["check_id"] == "R1" and imp.overrides[0]["signed_by"] == "验收员乙" + + +# ========================================================================== +# 第 6 步：统计期人工录入 +# ========================================================================== + + +PERIOD_SEP = {"统计期": {"start": "2026-09-01", "end": "2026-09-30"}} + + +async def test_step06_human_period_input_end_to_end(client): + name = unique() + base = await first_import(client, name, seed=106) + source_id = base["commit"]["source"]["id"] + raw, fn = flow_workbook(SEP, 30, seed=106, variant="noperiod") + resp = await reupload(client, source_id, raw, fn) + assert resp.status_code == 201, resp.text + st = resp.json() + sid = st["id"] + assert st["trial"]["status"] == "needs_input", st["trial"]["problems"] + assert any(p["category"] == "input" for p in st["trial"]["problems"]) + # 不需要录入时不收；录入不合法 → 422 context_invalid + resp = await trial(client, sid, context_inputs={"统计期": {"start": "2026-09-30", "end": "2026-09-01"}}) + assert resp.status_code == 422 and resp.json()["code"] == "context_invalid", resp.text + resp = await trial(client, sid, context_inputs=PERIOD_SEP, signed_by="录入员甲") + assert resp.status_code == 200, resp.text + st = resp.json() + t = st["trial"] + assert t["status"] == "passed", t["problems"] + assert t["receipt"]["period"]["source"] == "human" + assert t["receipt"]["period"]["signed_by"] == "录入员甲" + assert (t["receipt"]["period"]["start"], t["receipt"]["period"]["end"]) == ("2026-09-01", "2026-09-30") + ids = all_confirms(st) + assert "context_human:统计期" in ids + assert "本期统计期为人工录入" in notes_text(t) + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + imp = await db_get(TableImport, out["import_id"]) + assert imp.context["source"] == "human" and imp.context["signed_by"] == "录入员甲" + assert "context_human:统计期" in {c["id"] for c in imp.confirmations} + # 录入值计入构建哈希：与「同一原件、同一配方、不录入」的构建 id 不同 + rec = (await client.get(f"{API}/{source_id}/recipe")).json() + raw_sha = hashlib.sha256(raw).hexdigest() + with_input = table_versions.recipe_build_id( + source_id, raw_sha, rec["recipe_sha256"], {"context": {"统计期": {"start": "2026-09-01", "end": "2026-09-30"}}}) + without = table_versions.recipe_build_id(source_id, raw_sha, rec["recipe_sha256"], {}) + assert out["build_id"] == with_input != without + text = await schema_notes(client, source_id, list(FLOW_EXPECT["rows"])) + assert "本期统计期为人工录入" in text + # 导入清单（经 table_imports.manifest_artifact 取回） + manifest = await artifact(client, imp.manifest_artifact) + assert manifest["period"]["source"] == "human" and manifest["period"]["signed_by"] == "录入员甲" + assert manifest["receipt"]["period"]["source"] == "human" + assert manifest["receipt"]["period"]["signed_by"] == "录入员甲" + assert "context_human:统计期" in {c["id"] for c in manifest["confirmations"]} + + # 能解析出统计期的文件不收录入：422 context_not_needed + raw2, fn2 = flow_workbook(OCT, 31, seed=1061) + st2 = (await reupload(client, source_id, raw2, fn2)).json() + assert st2["trial"]["status"] == "passed", st2["trial"]["problems"] + resp = await trial(client, st2["id"], context_inputs=PERIOD_SEP) + assert resp.status_code == 422 and resp.json()["code"] == "context_not_needed", resp.text + + +# ========================================================================== +# 第 7 步：简单上传对配方源 409 +# ========================================================================== + + +async def test_step07_simple_upload_to_a_recipe_source_is_refused(client): + name = unique() + base = await first_import(client, name, seed=107) + resp = await client.post(f"{API}/upload", files={"file": (base["file_name"], base["raw"], XLSX)}, + data={"name": name}) + assert resp.status_code == 409 and resp.json()["code"] == "recipe_source", resp.text + # 再按配方导入同名 → 409 recipe_source_exists + resp = await stage(client, name, base["raw"], base["file_name"]) + assert resp.status_code == 409 and resp.json()["code"] == "recipe_source_exists", resp.text + + +# ========================================================================== +# 第 8 步：回收 +# ========================================================================== + + +async def test_step08_open_staging_raw_survives_gc_and_expired_staging_is_cleaned(client): + name = unique() + base = await first_import(client, name, seed=108) + source_id = base["commit"]["source"]["id"] + d00_sha = hashlib.sha256(base["raw"]).hexdigest() + raw, fn = flow_workbook(SEP, 30, seed=1081) + st = (await reupload(client, source_id, raw, fn)).json() + assert st["status"] == "trialed", st["trial"]["problems"] + sha = hashlib.sha256(raw).hexdigest() + row = await db_get(ImportStaging, st["id"]) + trial_path = Path(row.trial_path) + assert trial_path.is_file() and raw_store.raw_exists(sha) + + async with SessionLocal() as session: + await table_versions.gc(session) + async with SessionLocal() as session: + await table_versions.startup(session) + assert raw_store.raw_exists(sha), "未结束暂存区的原件不能被回收" + assert trial_path.is_file(), "未结束暂存区的试运行库不能被清理" + assert (await client.get(f"{API}/imports/{st['id']}")).json()["status"] == "trialed" + + # 过期：GET 时顺手过期 → 瘦身、删试运行库、原件没人要了随即删掉 + async with SessionLocal() as session: + await session.execute(update(ImportStaging).where(ImportStaging.id == st["id"]) + .values(expires_at=dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=1))) + await session.commit() + resp = await client.get(f"{API}/imports/{st['id']}") + assert resp.status_code == 200 + out = resp.json() + assert out["status"] == "expired" and out["grids"] == [] and out["trial"] is None + assert not trial_path.exists() + async with SessionLocal() as session: + await table_versions.gc(session) + assert not raw_store.raw_exists(sha), "过期暂存区的原件随后应被回收" + # 写操作 409 staging_closed + resp = await trial(client, st["id"]) + assert resp.status_code == 409 and resp.json()["code"] == "staging_closed" + # 提交过的不受影响 + assert raw_store.raw_exists(d00_sha) + assert (await source_row(client, source_id))["current_snapshot"]["id"] == base["commit"]["snapshot_id"] + assert await count(client, name, "日客流") == 31 + + +# ========================================================================== +# 第 9 步：重放幂等 +# ========================================================================== + + +async def test_step09_replay_is_idempotent(client): + name = unique() + base = await first_import(client, name, seed=109) + source_id = base["commit"]["source"]["id"] + # 同一文件、同一配方 → 同一构建；same_as_import → unchanged + st = (await reupload(client, source_id, base["raw"], base["file_name"])).json() + assert st["trial"]["same_as_import"]["id"] == base["commit"]["import_id"] + resp = await commit(client, st) + assert resp.status_code == 201 and resp.json()["unchanged"] is True + # 同一原件换个文件名：复用构建（D23），但要新建导入记录（文件名不同，不是 unchanged） + st = (await reupload(client, source_id, base["raw"], "换个名字.xlsx")).json() + assert st["trial"]["status"] in ("passed",), st["trial"]["problems"] + assert st["trial"]["same_as_import"] is not None + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["unchanged"] is False and out["build_reused"] is True + assert out["build_id"] == base["commit"]["build_id"] + + # 同一录入再传 → 同一构建 + raw, fn = flow_workbook(SEP, 30, seed=109, variant="noperiod") + builds = [] + for _ in range(2): + st = (await reupload(client, source_id, raw, fn)).json() + assert st["trial"]["status"] == "needs_input" + st = (await trial(client, st["id"], context_inputs=PERIOD_SEP)).json() + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + builds.append(resp.json()) + assert builds[0]["build_id"] == builds[1]["build_id"] + assert builds[1]["build_reused"] is True + + +# ========================================================================== +# 第 10 步：AI 起草全链路（假模型） +# ========================================================================== + +KV_RECIPE = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": "基本情况"}, "blocks": [{ + "id": "列表1", "layout": "list", "table": "基本情况", "after_title": "基本情况登记表", + "columns": [{"header": "单位名称", "name": "项目", "type": "TEXT"}, + {"header": "示例单位甲", "name": "取值", "type": "TEXT"}]}]}], + "tables": [{"name": "基本情况"}], +} + + +async def test_step10_ai_draft_full_chain_with_fake_model(client, monkeypatch): + from langchain_core.messages import AIMessage + + from app.data import recipe_ai + from app.data.recipe_types import AiAvailability + from tests.fixtures.xlsx.lab import kv_form + + calls: list[list[Any]] = [] + + class FakeModel: + async def ainvoke(self, messages): + calls.append(list(messages)) + return AIMessage(content=json.dumps(KV_RECIPE, ensure_ascii=False), + usage_metadata={"input_tokens": 2100, "output_tokens": 300, "total_tokens": 2400}) + + async def resolve(session): + return FakeModel(), "fake-model", AiAvailability(True, model="fake-model", provider="假接入") + + monkeypatch.setattr(recipe_ai, "resolve_draft_model", resolve) + raw, fn = kv_form() + name = unique() + resp = await stage(client, name, raw, fn) + assert resp.status_code == 201, resp.text + st = resp.json() + sid = st["id"] + assert st["draft"]["complete"] is False + assert st["ai"] == {"offered": True, "available": True, "reason": "", "model": "fake-model", "provider": "假接入"} + + resp = await client.get(f"{API}/imports/{sid}/draft-ai/preview") + assert resp.status_code == 200, resp.text + pv = resp.json() + assert set(pv) >= {"text", "chars", "sha256", "model", "provider"} + assert pv["chars"] == len(pv["text"]) and pv["text_sha256"] == hashlib.sha256(pv["text"].encode()).hexdigest() + for number in ("120", "860", "35600"): + assert number not in pv["text"], f"数字格的值 {number} 不外发" + assert calls == [], "预览不调用模型" + + # 同意框的门槛：不同意 → 422；sha 对不上 → 409；都不调模型 + resp = await client.post(f"{API}/imports/{sid}/draft-ai", json={"preview_sha256": pv["sha256"]}) + assert resp.status_code == 422 and resp.json()["code"] == "ai_consent_required", resp.text + resp = await client.post(f"{API}/imports/{sid}/draft-ai", json={"consent": True, "preview_sha256": "0" * 64}) + assert resp.status_code == 409 and resp.json()["code"] == "ai_preview_stale", resp.text + assert calls == [] + + resp = await client.post(f"{API}/imports/{sid}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"], "signed_by": "验收员丙"}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert calls and calls[0][0].content + "\n\n" + calls[0][1].content == pv["text"], "发出去的就是预览的全文" + assert st["recipe_origin"] == "ai" and st["recipe_problems"] == [] and st["answers"] == {} + assert st["ai_draft"]["total_tokens"] == 2400 and st["ai_draft"]["error"] is None + assert st["ai_consents"][-1]["signed_by"] == "验收员丙" + assert st["ai_consents"][-1]["model"] == "fake-model" + + resp = await trial(client, sid) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + ids = all_confirms(st) + assert "ai_origin" in ids + assert any(i.startswith("rename:") for i in ids), ids + assert any(i.startswith("columns:") for i in ids), ids + resp = await commit(client, st, confirmations=[i for i in ids if i != "ai_origin"]) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" and "ai_origin" in resp.json()["detail"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["source"]["current_recipe"]["origin"] == "ai" + imp = await db_get(TableImport, out["import_id"]) + manifest = await artifact(client, imp.manifest_artifact) + assert manifest["ai"]["usage"] and manifest["ai"]["usage"][0]["input_tokens"] == 2100 + assert manifest["ai"]["consents"] and manifest["ai"]["consents"][0]["preview_sha256"] == pv["sha256"] + assert manifest["recipe"]["origin"] == "ai" + assert await count(client, name, "基本情况") > 0 + + +# ========================================================================== +# 第 11 步：证据链 +# ========================================================================== + + +async def test_step11_evidence_chain_from_query_snapshot_to_manifest(client, engine_up): + name = unique() + base = await first_import(client, name, seed=111) + snapshot_id = base["commit"]["snapshot_id"] + graph = _chain( + _node("start", "input"), + _node("t1", "tool", tool=f"db_query__{name}", args={"sql": 'SELECT "日期", "全日客流" FROM "日客流" LIMIT 3'}), + _node("out", "output", fields=[{"name": "t1", "value": "{{ nodes.t1 }}"}]), + ) + resp = await client.post("/api/runs", json={"graph": graph, "input": {}}) + assert resp.status_code == 201, resp.text + run = await _wait_run(client, resp.json()["id"], ("succeeded", "failed")) + assert run["status"] == "succeeded", run["error"] + value = run["output"]["t1"] + payload = json.loads(value[value.index("{"):]) if isinstance(value, str) else value + qs = await artifact(client, payload["artifact"]) + assert qs["data_version"] == snapshot_id + schema_id = qs.get("schema_artifact") or payload.get("schema_artifact") + assert schema_id, "查询快照应指向冻结的表结构" + ss = await artifact(client, schema_id) + assert ss.get("import_mode") == "recipe" + assert ss.get("import_manifests") and len(ss["import_manifests"]) == 1 + manifest = await artifact(client, ss["import_manifests"][0]) + snap = await db_get(SourceSnapshot, snapshot_id) + assert manifest["db_sha256"] == snap.db_sha256 + assert manifest["snapshot_id"] == snapshot_id + assert manifest["table_hashes"] == base["trial"]["trial"]["receipt"]["table_hashes"] + assert manifest["file"]["raw_sha256"] == hashlib.sha256(base["raw"]).hexdigest() + + +# ========================================================================== +# 第 12 步：并发 +# ========================================================================== + + +async def test_step12_two_reuploads_committed_in_turn_second_gets_base_changed(client): + name = unique() + base = await first_import(client, name, seed=112) + source_id = base["commit"]["source"]["id"] + a = (await reupload(client, source_id, *flow_workbook(SEP, 30, seed=1121))).json() + b = (await reupload(client, source_id, *flow_workbook(OCT, 31, seed=1122))).json() + assert a["trial"]["status"] == b["trial"]["status"] == "passed" + first = await commit(client, a) + assert first.status_code == 201, first.text + second = await commit(client, b) + assert second.status_code == 409 and second.json()["code"] == "base_changed", second.text + assert (await source_row(client, source_id))["current_snapshot"]["id"] == first.json()["snapshot_id"] + assert await count(client, name, "日客流") == 30 + + +async def test_step12_two_reuploads_committed_concurrently_one_wins(client): + name = unique() + base = await first_import(client, name, seed=113) + source_id = base["commit"]["source"]["id"] + a = (await reupload(client, source_id, *flow_workbook(SEP, 30, seed=1131))).json() + b = (await reupload(client, source_id, *flow_workbook(OCT, 31, seed=1132))).json() + ra, rb = await asyncio.gather(commit(client, a), commit(client, b)) + codes = sorted([ra.status_code, rb.status_code]) + assert codes == [201, 409], (ra.text, rb.text) + loser = ra if ra.status_code == 409 else rb + winner = ra if ra.status_code == 201 else rb + assert loser.json()["code"] == "base_changed" + assert (await source_row(client, source_id))["current_snapshot"]["id"] == winner.json()["snapshot_id"] + + +async def test_step12_same_new_name_committed_concurrently_one_gets_name_taken(client): + name = unique() + stagings = [] + for seed in (1141, 1142): + raw, fn = flow_workbook(AUG, 31, seed=seed) + st = (await stage(client, name, raw, fn)).json() + assert st["kind"] == "first" + st = (await trial(client, st["id"])).json() + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + stagings.append(st) + ra, rb = await asyncio.gather(commit(client, stagings[0]), commit(client, stagings[1])) + assert sorted([ra.status_code, rb.status_code]) == [201, 409], (ra.text, rb.text) + loser = ra if ra.status_code == 409 else rb + assert loser.json()["code"] == "name_taken", loser.text + async with SessionLocal() as session: + rows = list((await session.execute(select(DataSource).where(DataSource.name == name))).scalars()) + assert len(rows) == 1 + + +# ========================================================================== +# 第 13 步：清除原件连带暂存区 +# ========================================================================== + + +async def test_step13_purge_raw_discards_open_stagings_on_the_same_raw(client): + name = unique() + base = await first_import(client, name, seed=114) + source_id = base["commit"]["source"]["id"] + st = (await reupload(client, source_id, base["raw"], base["file_name"])).json() + assert st["status"] == "trialed" + row = await db_get(ImportStaging, st["id"]) + trial_path = Path(row.trial_path) + assert trial_path.is_file() + resp = await client.post(f"{API}/{source_id}/imports/{base['commit']['import_id']}/purge-raw", + json={"reason": "合成理由：原件含敏感信息", "signed_by": "验收员丁"}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["file_deleted"] is True + assert out["discarded_stagings"] == [st["id"]] + got = (await client.get(f"{API}/imports/{st['id']}")).json() + assert got["status"] == "discarded" and got["grids"] == [] and got["trial"] is None + assert not trial_path.exists() + assert not raw_store.raw_exists(hashlib.sha256(base["raw"]).hexdigest()) + # 当前版本照常可查；修改配方要原件 → 409 raw_missing + assert await count(client, name, "日客流") == 31 + resp = await client.post(f"{API}/{source_id}/redraft", json={}) + assert resp.status_code == 409 and resp.json()["code"] == "raw_missing", resp.text + + +# ========================================================================== +# 拒收后修复（8.1：D09 这类拒收在期 2 靠配方面板修好） +# ========================================================================== + + +async def test_rejected_reupload_is_fixed_by_editing_the_recipe_and_committed(client): + name = unique() + base = await first_import(client, name, seed=115) + source_id = base["commit"]["source"]["id"] + raw, fn = DRIFT_CASES["D09"].build(115) + st = (await reupload(client, source_id, raw, fn)).json() + sid = st["id"] + assert st["trial"]["status"] == "rejected" + assert "title_not_found" in {p["code"] for p in st["trial"]["problems"]} + assert st["ai"]["offered"] is False, "上传新一期不提供 AI 起草" + resp = await client.get(f"{API}/imports/{sid}/draft-ai/preview") + assert resp.status_code == 409 and resp.json()["code"] == "ai_not_offered" + # 「修改配方」：在同一个暂存区里改分段标题 + fixed = json.loads(json.dumps(st["recipe"], ensure_ascii=False)) + seg = next(s for s in fixed["sheets"][0]["blocks"][0]["segments"] if s["id"] == "日间") + seg["locate"]["title"] = "日间分时段客流（人次）" + resp = await client.put(f"{API}/imports/{sid}/recipe", json={"recipe": fixed}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["status"] == "drafting" + # 2.2 第 2 条：pick 必须是本段新标题的候选词。「日间」不是「日间分时段客流（人次）」的候选词，静态校验 + # 报 const_not_candidate，path 指到 pick（表单把问题显示在常量下拉旁边），下拉的选项来自 candidates + assert [p["code"] for p in st["recipe_problems"]] == ["const_not_candidate"], st["recipe_problems"] + assert st["recipe_problems"][0]["path"].endswith("/const/时段类别/pick") + options = st["candidates"].get("日间分时段客流（人次）") + assert options, st["candidates"] + seg["const"]["时段类别"]["pick"] = options[0] + resp = await client.put(f"{API}/imports/{sid}/recipe", json={"recipe": fixed}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["recipe_problems"] == [] and st["status"] == "drafting" + resp = await trial(client, sid) + st = resp.json() + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + ids = all_confirms(st) + # 配方变了：配方类确认项重新出现（mode、单位……）。表结构没变，但常量「时段类别」只能另选新标题的候选词， + # 这一列的值整列从「日间」变了：7.5 的 breaking_changes 报常量取值变化（集成验收补，原来这里静默换值）， + # 必须出 breaking:时段客流 让人确认，别的表不报 + assert "mode" in ids and [i for i in ids if i.startswith("breaking:")] == ["breaking:时段客流"], ids + label = next(i["label"] for i in st["trial"]["confirm_items"] if i["id"] == "breaking:时段客流") + assert f"列「时段类别」取值「日间」→「{options[0]}」" in label, label + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["recipe_id"] != base["commit"]["recipe_id"] + assert out["source"]["current_recipe"]["seq"] == 2 + async with SessionLocal() as session: + recipes = {r.seq: r.status for r in (await session.execute( + select(TableRecipe).where(TableRecipe.source_id == source_id))).scalars()} + assert recipes == {1: "superseded", 2: "active"} + assert await count(client, name, "日客流") == 30 + + +# ========================================================================== +# 修改配方（不换文件）、放弃导入 +# ========================================================================== + + +async def test_redraft_on_the_current_raw_without_changes_is_unchanged(client): + name = unique() + base = await first_import(client, name, seed=116) + source_id = base["commit"]["source"]["id"] + resp = await client.post(f"{API}/{source_id}/redraft", json={}) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["kind"] == "redraft" and st["status"] == "drafting" and st["draft"] is None + assert st["ai"]["offered"] is False and st["recipe_problems"] == [] + assert st["file"]["name"] == base["file_name"] + cur = (await client.get(f"{API}/{source_id}/recipe")).json() + mine, _ = recipe.validate_recipe(st["recipe"], origin="manual") + assert recipe.recipe_sha256(mine) == cur["recipe_sha256"], "工作配方 = 现行配方" + st = (await trial(client, st["id"])).json() + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + assert st["trial"]["same_as_import"]["id"] == base["commit"]["import_id"] + resp = await commit(client, st) + assert resp.status_code == 201 and resp.json()["unchanged"] is True, resp.text + + +async def test_discard_deletes_an_unreferenced_raw_and_closes_the_staging(client): + raw, fn = flow_workbook(AUG, 31, seed=117) + st = (await stage(client, unique(), raw, fn)).json() + sha = hashlib.sha256(raw).hexdigest() + assert raw_store.raw_exists(sha) + resp = await client.delete(f"{API}/imports/{st['id']}") + assert resp.status_code == 204, resp.text + assert not raw_store.raw_exists(sha), "放弃即删原件（没有别的引用）" + got = (await client.get(f"{API}/imports/{st['id']}")).json() + assert got["status"] == "discarded" and got["grids"] == [] and got["recipe"] is not None + resp = await client.delete(f"{API}/imports/{st['id']}") + assert resp.status_code == 409 and resp.json()["code"] == "staging_closed" + resp = await client.post(f"{API}/imports/{st['id']}/answers", json={"answers": {}}) + assert resp.status_code == 409 and resp.json()["code"] == "staging_closed" + + +# ========================================================================== +# 差异卡的种类与界面对得上（7.6 的取值表就是前端 DIFF_KIND_LABEL 的键） +# ========================================================================== + +#: P2-SPEC 7.6 差异卡表格里的全部 kind（前端 frontend/src/lib/terms.ts 的 DIFF_KIND_LABEL 照它逐个给了中文名） +SPEC_DIFF_KINDS = frozenset({ + "period", "rows", "placeholders", "checks", "file_name", "full_calc", "context_text", "context_source_added", + "column_order", "label_writing", "header_writing", "canon_values", "row_order", "block_order", "derived_form", + "axis_form", "outside_moved", "outside_added", "outside_removed", "outside_changed", "ignored_columns", "hidden", +}) + + +def _move_note_down(raw: bytes) -> bytes: + """D28 的表下注释从 B32 挪到 B33（文字一字不改，只是位置变了）。""" + import re + + from tests.fixtures.xlsx import patch_parts, sheet_part + from tests.fixtures.xlsx.flow import SHEET + + def edit(xml: str) -> str: + xml = xml.replace('', r'', xml) + + return patch_parts(raw, {sheet_part(raw, SHEET): edit}) + + +async def test_outside_text_only_moved_is_an_info_item_of_a_spec_kind(client): + """7.6：区域外文字先按全文配对，只是位置变了的算 info、不要求确认。差异项的 kind 必须是 7.6 表里的取值 + （界面按它取中文名，表外的 kind 会把英文键名原样露在差异卡上）。""" + name = unique() + raw, fn = flow_workbook(SEP, 30, seed=118, variant="D28") + resp = await stage(client, name, raw, fn) + assert resp.status_code == 201, resp.text + st = resp.json() + st = (await client.post(f"{API}/imports/{st['id']}/answers", json={"answers": answer_all(st["questions"])})).json() + st = (await trial(client, st["id"])).json() + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await commit(client, st) + assert resp.status_code == 201, resp.text + source_id = resp.json()["source"]["id"] + + st = (await reupload(client, source_id, _move_note_down(raw), fn)).json() + t = st["trial"] + assert t["status"] == "passed", t["problems"] + kinds = {d["kind"] for d in t["diff"]} + assert kinds <= SPEC_DIFF_KINDS, f"差异卡出现了 7.6 表外的 kind：{sorted(kinds - SPEC_DIFF_KINDS)}" + assert kinds == {"outside_moved"}, kinds + assert not any(d["requires_confirm"] for d in t["diff"]), "只是挪了位置：info，不要求确认" + assert "outside_digits:客流汇总!B33" in all_confirms(st), "本期类：含数字的区域外文字每期都要确认" + + +async def test_reported_total_table_is_marked_everywhere_the_model_and_judge_read(client): + """AU-5：原表写明的合计表（时段客流_表内合计）的「不要相加」不只在 db_schema 查单表时看得到：查询工具描述、 + db_schema 的表清单、写作目录、裁判摘录都要标出，而且不含数字。明细表不标。""" + from app.core.artifact_store import load as load_artifact + from app.data.engine import engines + from app.engine import judge + from app.engine.evidence import build_catalog, catalog_prompt, extract_numbers, schema_ledger_entry + from app.tools.datasource import REPORTED_TOTAL_MARK, build_datasource_tools + from app.tools.registry import ToolContext, Versions + + name = unique() + done = await first_import(client, name) + sid = done["commit"]["source"]["id"] + async with SessionLocal() as session: + src = await session.get(DataSource, sid) + assert src.schema_cache["tables"]["时段客流_表内合计"]["kind"] == "reported_total" + assert "kind" not in src.schema_cache["tables"]["时段客流"] + tools = {t.name: t for t in await build_datasource_tools( + [f"db_query__{name}", f"db_schema__{name}"], ToolContext(data_versions=Versions.CURRENT), session)} + try: + query, schema = tools[f"db_query__{name}"], tools[f"db_schema__{name}"] + assert "时段客流_表内合计 是原表写明的合计" in query.description and "不要与明细表相加" in query.description + listing = await schema.coroutine() + assert f"时段客流_表内合计{REPORTED_TOTAL_MARK}" in listing + assert f"时段客流{REPORTED_TOTAL_MARK}" not in listing.replace(f"时段客流_表内合计{REPORTED_TOTAL_MARK}", "") + assert extract_numbers(REPORTED_TOTAL_MARK) == [] + payload = json.loads(await query.coroutine(sql='SELECT COUNT(*) AS n FROM "时段客流_表内合计"')) + frozen = schema_ledger_entry(payload["schema_artifact"], node_id="fetch", exec_no=1) + catalog = build_catalog(nodes={}, ledger=[frozen]) + prompt = catalog_prompt(catalog, loader=load_artifact) + assert "时段客流_表内合计（原表写明的合计：不要彼此相加，也不要与明细相加）" in prompt + excerpt = judge._excerpt("t:时段客流_表内合计", catalog["t:时段客流_表内合计"], [], load_artifact, None, catalog) + assert "表的说明：原表写明的合计" in excerpt + finally: + await engines.invalidate(sid) diff --git a/backend/tests/test_recipe_engine_crosstab.py b/backend/tests/test_recipe_engine_crosstab.py new file mode 100644 index 00000000..ac03f415 --- /dev/null +++ b/backend/tests/test_recipe_engine_crosstab.py @@ -0,0 +1,873 @@ +"""配方执行器：交叉表（P2-SPEC 4.2、4.3、4.5、4.6、4.7）。 + +夹具是测试里现造的「结构仿照客流表」（坐标同 9.1，名字全是假名、数字随机），按需要改几格造出各种漂移； +配方用契约里的参考配方 flow_recipe.json。不依赖 WP-1 的夹具。 +""" +from __future__ import annotations + +import copy +import datetime as dt +import hashlib +import io +import json +import os +import random +import re +import sqlite3 +import zipfile +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest +from openpyxl import Workbook +from openpyxl.utils import get_column_letter + +from app.data import recipe_engine, xlsx_scan +from app.data.recipe_types import Extraction, PeriodInput, Recipe + +FLOW = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text(encoding="utf-8")) +SHEET = "客流汇总" + + +# -------------------------------------------------------------------------- +# 夹具 +# -------------------------------------------------------------------------- + + +def inject_cache(raw: bytes, values: dict[str, Any], *, keep_full_calc: bool = False, + sheet_part: str = "xl/worksheets/sheet1.xml") -> bytes: + """openpyxl 写公式不带缓存值：在 zip 层给指定格补上 ；默认同时去掉 workbook.xml 的 fullCalcOnLoad。""" + zin = zipfile.ZipFile(io.BytesIO(raw)) + out = io.BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zout: + for it in zin.infolist(): + data = zin.read(it.filename) + if it.filename == sheet_part and values: + t = data.decode("utf-8") + for ref, v in values.items(): + t, n = re.subn(rf'(]*>)(.*?)(?:|)?', + lambda m, v=v: f"{m.group(1)}{m.group(2)}{v}", t) + assert n == 1, ref + data = t.encode("utf-8") + if it.filename == "xl/workbook.xml" and not keep_full_calc: + data = re.sub(rb'\s*fullCalcOnLoad="1"', b"", data) + zout.writestr(it, data) + return out.getvalue() + + +def patch_part(raw: bytes, part: str, fn: Callable[[str], str]) -> bytes: + zin = zipfile.ZipFile(io.BytesIO(raw)) + out = io.BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zout: + for it in zin.infolist(): + data = zin.read(it.filename) + if it.filename == part: + data = fn(data.decode("utf-8")).encode("utf-8") + zout.writestr(it, data) + return out.getvalue() + + +def flow_book(start: dt.date = dt.date(2026, 8, 1), days: int = 31, seed: int = 0, *, + day_hours: tuple[int, ...] = tuple(range(7, 18)), night_first: bool = False, blank_row: bool = True, + edit: Callable[[Any, dict[str, Any]], None] | None = None, uncached: bool = False, + keep_full_calc: bool = False, filename: str | None = None, period: str | None = None + ) -> tuple[bytes, str]: + """「结构仿照客流表」：B2 统计期、B3 标题、第 4 行日期、B5:B7 指标、空行、日间段（第一行整行「·」）、 + 夜间段和三行合计公式（补了缓存值）。edit(ws, info) 在保存前改格子；info["cache"] 是公式缓存值， + info["rows"] 记各段所在的行。""" + rnd = random.Random(seed) + wb = Workbook() + ws = wb.active + ws.title = SHEET + end = start + dt.timedelta(days=days - 1) + last = get_column_letter(2 + days) + ws["B2"] = period if period is not None else ( + f"统计时间范围：{start.year}年{start.month}月{start.day}日至{end.year}年{end.month}月{end.day}日") + ws.merge_cells(f"B2:{last}2") + ws["B3"] = "客流汇总表" + ws.merge_cells(f"B3:{last}3") + for j in range(days): + d = start + dt.timedelta(days=j) + ws.cell(4, 3 + j, f"{d.month}月{d.day}日") + info: dict[str, Any] = {"cache": {}, "rows": {}, "last": last, "days": days} + + def measures(r: int) -> int: + for k, label in enumerate(("全日客流（人次）", "分区甲（人次）", "分区乙（人次）")): + ws.cell(r + k, 2, label) + for j in range(days): + a, b = rnd.randint(3000, 9000), rnd.randint(2000, 8000) + ws.cell(r, 3 + j, a + b) + ws.cell(r + 1, 3 + j, a) + ws.cell(r + 2, 3 + j, b) + info["rows"]["日客流"] = [r, r + 1, r + 2] + return r + 3 + + def day(r: int) -> int: + ws.cell(r, 2, "日间时段客流（人次）") + ws.merge_cells(f"B{r}:{last}{r}") + info["rows"]["日间标题"] = r + rows = [] + for i, h in enumerate(day_hours): + rr = r + 1 + i + ws.cell(rr, 2, f"{h}-{h + 1}") + for j in range(days): + ws.cell(rr, 3 + j, "·" if h == 7 else rnd.randint(50, 900)) + rows.append(rr) + info["rows"]["日间"] = rows + return r + 1 + len(day_hours) + + def night(r: int) -> int: + ws.cell(r, 2, "夜间时段客流（人次）") + ws.merge_cells(f"B{r}:{last}{r}") + info["rows"]["夜间标题"] = r + vals: dict[tuple[int, int], int] = {} + rows = [] + for i, h in enumerate(range(18, 24)): + rr = r + 1 + i + ws.cell(rr, 2, f"{h}-{h + 1}") + for j in range(days): + v = rnd.randint(50, 900) + ws.cell(rr, 3 + j, v) + vals[(rr, j)] = v + rows.append(rr) + info["rows"]["夜间"] = rows + totals = [] + for k, (label, a, b) in enumerate((("18-22 时合计", 0, 3), ("22-24 时合计", 4, 5), ("18-24 时合计", 0, 5))): + rr = r + 7 + k + ws.cell(rr, 2, label) + for j in range(days): + col = get_column_letter(3 + j) + ws.cell(rr, 3 + j, f"=SUM({col}{rows[a]}:{col}{rows[b]})") + info["cache"][f"{col}{rr}"] = sum(vals[(rows[x], j)] for x in range(a, b + 1)) + totals.append(rr) + info["rows"]["合计"] = totals + return r + 10 + + r = measures(5) + if blank_row: + ws.merge_cells(f"B{r}:{last}{r}") + r += 1 + for block in ((night, day) if night_first else (day, night)): + r = block(r) + info["bottom"] = r - 1 + if edit is not None: + edit(ws, info) + buf = io.BytesIO() + wb.save(buf) + raw = inject_cache(buf.getvalue(), {} if uncached else info["cache"], keep_full_calc=keep_full_calc) + return raw, filename or f"月报导出_{start.isoformat()}_{end.isoformat()}.xlsx" + + +def recipe_dict(**patch: Any) -> dict[str, Any]: + data = copy.deepcopy(FLOW) + for path, value in patch.items(): + cur: Any = data + keys = path.split("__") + for k in keys[:-1]: + cur = cur[int(k)] if k.isdigit() else cur[k] + last = keys[-1] + if isinstance(cur, list): + cur[int(last)] = value + else: + cur[last] = value + return data + + +def run(book: tuple[bytes, str], recipe: dict[str, Any] | None = None, db: str | None = None, + scan: xlsx_scan.WorkbookScan | None = None, **kw: Any) -> Extraction: + raw, filename = book + return recipe_engine.execute(Recipe.model_validate(recipe or FLOW), raw, filename, + scan or xlsx_scan.scan(raw), db, **kw) + + +def codes(ex: Extraction) -> set[str]: + return {p.code for p in ex.problems} + + +def problem(ex: Extraction, code: str): + found = [p for p in ex.problems if p.code == code] + assert found, f"没有 {code}：{[(p.code, p.message) for p in ex.problems]}" + return found[0] + + +def set_value(ws: Any, ref: str, value: Any) -> None: + ws[ref] = value + + +# -------------------------------------------------------------------------- +# 参考布局（9.1 的期望） +# -------------------------------------------------------------------------- + + +def test_reference_layout_matches_receipt(tmp_path): + db = str(tmp_path / "flow.db") + ex = run(flow_book(), db=db) + assert ex.ok and not ex.problems and not ex.partial + (ledger,) = ex.ledger + assert ledger.nonempty_scan == ledger.nonempty_read == 771 and ledger.unclaimed == 0 + assert ledger.roles == {"value": 620, "derived_value": 93, "derived_label": 3, "col_header": 31, + "row_label": 20, "section_title": 2, "context": 1, "outside_text": 1} + assert [(t.name, t.rows, t.kind, t.grain) for t in ex.tables] == [ + ("日客流", 31, "data", ["日期"]), ("时段客流", 527, "data", ["日期", "时段"]), + ("时段客流_表内合计", 93, "reported_total", ["日期", "合计项"])] + assert ex.expected_rows == {"日客流": 31, "时段客流": 527, "时段客流_表内合计": 93} + assert ex.placeholders == {"·": 31} + assert [(c.id, c.status) for c in ex.context_checks] == [("C1", "passed"), ("C2", "passed")] + assert ex.canonicalized == [] + assert [(o.cell, o.kind, o.period_source) for o in ex.outside_text] == [(f"{SHEET}!B3", "text", False)] + assert ex.period.source == "cells" and ex.period.texts == { + f"{SHEET}!B2": "统计时间范围：2026年8月1日至2026年8月31日"} and ex.period.annotated == {} + assert ex.block_order == {SHEET: ["日客流", "日间", "夜间", "夜间合计"]} + assert ex.derived_form == {"夜间合计": "formula"} + assert [(a.row, a.first, a.last, a.count, a.form) for a in ex.axes] == [(4, "2026-08-01", "2026-08-31", 31, "text")] + assert ex.sheets.matched == {"s1": SHEET} and ex.sheets.renamed == {} and ex.sheets.other_visible == [] + assert len(ex.derived) == 93 and all(d.ref_problem is None and d.func == "SUM" for d in ex.derived) + assert ex.lineage["日客流"]["全日客流"] == [[1, SHEET, "C5", 31, "right"]] + assert ex.lineage["时段客流"]["客流"][:2] == [[1, SHEET, "C10", 31, "right"], [32, SHEET, "C11", 31, "right"]] + assert set(ex.table_hashes) == {"日客流", "时段客流", "时段客流_表内合计"} + columns = {t.name: [(c.name, c.type, c.role, c.unit) for c in t.columns] for t in ex.tables} + assert columns["时段客流"] == [("日期", "TEXT", "axis", None), ("时段", "TEXT", "dim", None), + ("时段类别", "TEXT", "const", None), ("起始小时", "INTEGER", "derive", None), + ("结束小时", "INTEGER", "derive", None), ("客流", "INTEGER", "value", "人次")] + assert ex.tables[0].sources == ["s1/交叉表/日客流"] + assert ex.tables[1].sources == ["s1/交叉表/日间", "s1/交叉表/夜间"] + conn = sqlite3.connect(db) + try: + assert conn.execute("PRAGMA journal_mode").fetchone()[0] == "delete" + ddl = conn.execute("SELECT sql FROM sqlite_master WHERE name='时段客流'").fetchone()[0] + assert ddl.endswith("STRICT") and 'PRIMARY KEY ("日期", "时段")' in ddl + assert conn.execute("SELECT COUNT(*), COUNT(客流) FROM 时段客流 WHERE 时段='7-8'").fetchone() == (31, 0) + assert conn.execute("SELECT 时段类别, 起始小时, 结束小时 FROM 时段客流 WHERE 时段='18-19' LIMIT 1" + ).fetchone() == ("夜间", 18, 19) + assert conn.execute("SELECT COUNT(*) FROM 日客流 WHERE 全日客流 <> 分区甲 + 分区乙").fetchone() == (0,) + assert conn.execute("SELECT DISTINCT 合计项 FROM 时段客流_表内合计 ORDER BY rowid").fetchall() == [ + ("18-22时合计",), ("22-24时合计",), ("18-24时合计",)] + finally: + conn.close() + assert not os.path.exists(db + "-wal") and not os.path.exists(db + "-journal") + + +def test_regions_cover_every_claimed_cell(): + raw, fn = flow_book(days=3) + ex = run((raw, fn)) + from app.data.xlsx_cells import parse_range + rects = [(m.role, parse_range(m.ref)) for m in ex.regions] + assert {m.role for m in ex.regions} == {"context", "outside_text", "col_header", "row_label", "value", + "section_title", "derived_label", "derived_value"} + def covered(role: str, r: int, c: int) -> bool: + return any(rl == role and s[0] <= r <= s[2] and s[1] <= c <= s[3] for rl, s in rects) + assert covered("context", 2, 2) and covered("outside_text", 3, 2) and covered("value", 5, 3) + assert covered("derived_value", 30, 5) and covered("section_title", 21, 2) + + +def test_regions_do_not_split_on_blank_edge_days(): + # 首日、末日是空格（blank=null）的行，值的区域仍按整段日期列出，不另起矩形（4.9） + base = [(m.role, m.ref) for m in run(flow_book(days=3)).regions] + def blank_edges(ws, info): + for r in info["rows"]["日间"][1::2]: + ws.cell(r, 5).value = None + ws.cell(info["rows"]["日间"][3], 3).value = None + ex = run(flow_book(days=3, edit=blank_edges), recipe_dict(sheets__0__blocks__0__values__blank="null")) + assert ex.ok and [(m.role, m.ref) for m in ex.regions] == base + assert ("value", "C10:E20") in base + + +# -------------------------------------------------------------------------- +# 评审后新增的行为 +# -------------------------------------------------------------------------- + + +def test_null_in_a_primary_key_column_is_reported_not_dropped(): + # grain 含指标列、blank=null 时那一格存空值：主键为空的行曾被静默丢掉（ok=True、少一行、N 核对照样通过） + data = recipe_dict(sheets__0__blocks__0__values__blank="null") + data["tables"][0]["grain"] = ["日期", "分区甲"] + ex = run(flow_book(days=3, edit=lambda ws, info: set_value(ws, "D6", None)), data) + assert not ex.ok and codes(ex) == {"value_blank"} + p = problem(ex, "value_blank") + assert p.cells == [f"{SHEET}!D6"] and "主键列「分区甲」" in p.message and "主键" in p.model_message + # 合计另存表的主键含值列、合计格没有保存值：同样报出来，不静默少存 + data = recipe_dict() + data["tables"][2]["grain"] = ["日期", "合计项", "客流"] + ex = run(flow_book(days=3, uncached=True), data) + assert not ex.ok and codes(ex) == {"value_blank"} + p = problem(ex, "value_blank") + assert len(p.cells) == 9 and p.cells[0] == f"{SHEET}!C28" and "有 9 处为空值" in p.message + # 转换失败的格（已经报过）不再另报主键为空 + data = recipe_dict() + data["tables"][0]["grain"] = ["日期", "分区甲"] + ex = run(flow_book(days=3, edit=lambda ws, info: set_value(ws, "D6", "若干")), data) + assert codes(ex) == {"value_not_number"} + + +def test_merged_blank_row_is_not_merged_in_values_but_merge_over_values_is(): + ex = run(flow_book(days=3)) + assert "merged_in_values" not in codes(ex) # B8:E8 空行合并（乙-5） + def merge_values(ws, info): + r = info["rows"]["日间"][2] + ws.merge_cells(f"C{r}:D{r}") + p = problem(run(flow_book(days=3, edit=merge_values)), "merged_in_values") + assert p.category == "structure" and p.cells == [f"{SHEET}!C12"] + + +def _blank_day_row(ws, info): + r = info["rows"]["日间"][0] + for j in range(info["days"]): + ws.cell(r, 3 + j).value = None + + +def test_all_blank_period_row_is_a_data_row(tmp_path): + ex = run(flow_book(days=3, edit=_blank_day_row)) + assert codes(ex) == {"value_blank"} # 不是 title_not_found / label_missing / row_unclaimed + assert problem(ex, "value_blank").cells == [f"{SHEET}!C10", f"{SHEET}!D10", f"{SHEET}!E10"] + db = str(tmp_path / "null.db") + ex = run(flow_book(days=3, edit=_blank_day_row), + recipe_dict(sheets__0__blocks__0__values__blank="null"), db=db) + assert ex.ok + conn = sqlite3.connect(db) + assert conn.execute("SELECT COUNT(*), COUNT(客流) FROM 时段客流 WHERE 时段='7-8'").fetchone() == (3, 0) + conn.close() + + +def test_annotated_period_cells_go_to_outside_text_pure_ones_do_not(): + ex = run(flow_book(days=3, edit=lambda ws, info: set_value(ws, "B3", "客流汇总表（2026年8月）"))) + assert ex.ok and ex.period.annotated == {f"{SHEET}!B3": "客流汇总表"} + assert ex.period.cells == [f"{SHEET}!B2", f"{SHEET}!B3"] + assert [(o.cell, o.kind, o.period_source) for o in ex.outside_text] == [(f"{SHEET}!B3", "text_digits", True)] + assert ex.context_checks[0].status == "passed" and ex.context_checks[0].checked == 2 + assert "outside_digits" not in codes(ex) # 免不免确认由确认项按上一期判断（7.5），不在这里报 + assert ex.ledger[0].roles["context"] == 2 and "outside_text" not in ex.ledger[0].roles + # D28b：脚注里写了年月，同样是统计期来源、同样记进 outside_text + ex = run(flow_book(days=3, edit=lambda ws, info: set_value(ws, "B32", "注：2026年8月数据为初步统计，待修订"))) + assert ex.period.annotated == {f"{SHEET}!B32": "注数据为初步统计待修订"} + assert (f"{SHEET}!B32", "text_digits", True) in [(o.cell, o.kind, o.period_source) for o in ex.outside_text] + + +def test_outside_text_with_digits_needs_confirmation(): + ex = run(flow_book(days=3, edit=lambda ws, info: set_value(ws, "B32", "注：8月15日闸机故障，当日客流为估算值"))) + assert ex.ok + p = problem(ex, "outside_digits") + assert p.category == "confirm" and p.cells == [f"{SHEET}!B32"] + assert (f"{SHEET}!B32", "text_digits", False) in [(o.cell, o.kind, o.period_source) for o in ex.outside_text] + + +def test_outside_number_is_structure_and_merges_per_row(): + def edit(ws, info): + ws["B1"] = 987654 # 日期表头以上：区域外 + ws["C1"] = "1,234" + ex = run(flow_book(days=3, edit=edit)) + ps = [p for p in ex.problems if p.code == "outside_number"] + assert len(ps) == 1 and ps[0].fix == "ignore_cells" and ps[0].cells == [f"{SHEET}!B1", f"{SHEET}!C1"] + assert not ex.ok + + +def test_subtotal_formula_and_hidden_rows_make_refs_hidden(): + def subtotal(ws, info): + ws[f"C{info['rows']['合计'][0]}"] = f"=SUBTOTAL(9,C{info['rows']['夜间'][0]}:C{info['rows']['夜间'][3]})" + info["cache"].pop("C28") + ex = run(flow_book(days=3, edit=subtotal)) + item = next(d for d in ex.derived if d.cell == f"{SHEET}!C28") + assert item.func == "SUBTOTAL" and item.ref_problem == "hidden" and item.value is None # 改了公式、没有缓存 + def hide(ws, info): + ws.row_dimensions[info["rows"]["夜间"][1]].hidden = True + include = recipe_dict(sheets__0__hidden={"rows": "include", "cols": "reject_if_any"}) + ex = run(flow_book(days=3, edit=hide), include) + assert ex.ok and ex.hidden == {SHEET: {"rows": [23], "cols": [], "policy_rows": "include", + "policy_cols": "reject_if_any"}} + item = next(d for d in ex.derived if d.cell == f"{SHEET}!C28") + assert item.hidden_in_range and item.ref_problem is None and len(item.ref_rowids) == 4 + assert not next(d for d in ex.derived if d.cell == f"{SHEET}!C29").hidden_in_range + exclude = recipe_dict(sheets__0__hidden={"rows": "exclude", "cols": "reject_if_any"}) + ex = run(flow_book(days=3, edit=hide), exclude) + assert ex.ok and ex.expected_rows["时段客流"] == 16 * 3 + item = next(d for d in ex.derived if d.cell == f"{SHEET}!C28") + assert item.ref_problem == "hidden" and item.ref_rowids is None and item.hidden_in_range + assert ex.ledger[0].roles["hidden_excluded"] == 4 + + +def test_excluded_hidden_total_row_is_neither_checked_nor_kept(): + def hide(ws, info): + ws.row_dimensions[info["rows"]["合计"][0]].hidden = True + exclude = recipe_dict(sheets__0__hidden={"rows": "exclude", "cols": "reject_if_any"}) + ex = run(flow_book(days=3, edit=hide), exclude) + assert ex.ok and len(ex.derived) == 6 and ex.expected_rows["时段客流_表内合计"] == 6 + assert {d.label for d in ex.derived} == {"22-24时合计", "18-24时合计"} + + +def test_hidden_rows_and_cols_rejected_by_default(): + def hide(ws, info): + ws.row_dimensions[info["rows"]["夜间"][1]].hidden = True + ws.column_dimensions["D"].hidden = True + ws.auto_filter.ref = "B4:E30" + ex = run(flow_book(days=3, edit=hide)) + rows, cols = problem(ex, "hidden_rows"), problem(ex, "hidden_cols") + assert "第 23 行" in rows.message and "筛选" in rows.message and "D 列" in cols.message + assert not ex.ok + + +def test_derived_item_key_carries_the_const_column(tmp_path): + db = str(tmp_path / "k.db") + ex = run(flow_book(days=3), db=db) + item = next(d for d in ex.derived if d.cell == f"{SHEET}!D29") + assert item.key == {"日期": "2026-08-02", "时段类别": "夜间"} + assert (item.label, item.label_raw, item.start, item.end) == ("22-24时合计", "22-24 时合计", 22, 24) + assert (item.base_table, item.base_value, item.start_col, item.end_col) == ("时段客流", "客流", "起始小时", "结束小时") + assert item.keep_table == "时段客流_表内合计" and item.is_formula and item.formula == "=SUM(D26:D27)" + conn = sqlite3.connect(db) + rowids = [r for (r,) in conn.execute( + "SELECT rowid FROM 时段客流 WHERE 日期='2026-08-02' AND 时段类别='夜间' AND 起始小时>=22 AND 结束小时<=24")] + conn.close() + assert item.ref_rowids == sorted(rowids) + + +def test_literal_totals_and_unrecognized_formulas(): + def literal(ws, info): + for ref, v in info["cache"].items(): + ws[ref] = v + info["cache"].clear() + ex = run(flow_book(days=3, edit=literal)) + assert ex.ok and ex.derived_form == {"夜间合计": "literal"} + assert all(not d.is_formula and d.ref_rowids is None and d.func is None for d in ex.derived) + def odd(ws, info): + ws["C28"] = "=C22+C23+C24+C25" + ws["D28"] = "=SUM(D22:D24)*1" + ws["E28"] = "=SUM(E22:E26)" + info["cache"].update({"C28": 1, "D28": 1, "E28": 1}) + ex = run(flow_book(days=3, edit=odd)) + by = {d.cell.split("!")[1]: d for d in ex.derived} + assert by["C28"].func == "plus" and by["C28"].ref_problem is None and len(by["C28"].ref_rowids) == 4 + assert by["D28"].ref_problem == "unrecognized" and by["D28"].func is None + assert by["E28"].ref_problem is None and len(by["E28"].ref_rowids) == 5 # 引用对不对由核对模块判 + assert ex.derived_form == {"夜间合计": "formula"} + def mixed(ws, info): + ws["C28"] = info["cache"].pop("C28") + assert run(flow_book(days=3, edit=mixed)).derived_form == {"夜间合计": "mixed"} + + +def test_uncached_totals_are_none_not_problems(): + ex = run(flow_book(days=3, uncached=True)) + assert ex.ok and all(d.value is None and d.is_formula for d in ex.derived) + + +# -------------------------------------------------------------------------- +# 问题收集策略 +# -------------------------------------------------------------------------- + + +def test_one_failed_segment_does_not_stop_the_others(): + ex = run(flow_book(days=3, edit=lambda ws, info: set_value(ws, "B9", "日间分时段客流（人次）"))) # D09 + assert codes(ex) == {"title_not_found"} # 日间的 11 行不再逐行报 row_unclaimed + p = problem(ex, "title_not_found") + assert p.fix == "rename_title" and p.category == "structure" + assert ex.expected_rows == {"日客流": 3, "时段客流": 6 * 3, "时段客流_表内合计": 9} + assert len(ex.derived) == 9 + + +def test_label_duplicate_and_pk_duplicate_reported_together(): + ex = run(flow_book(days=3, edit=lambda ws, info: set_value(ws, "B12", "8－9"))) + assert {"label_duplicate", "label_missing", "pk_duplicate"} <= codes(ex) + dup = problem(ex, "pk_duplicate") + assert dup.cells[:2] == [f"{SHEET}!C11", f"{SHEET}!C12"] and "C11 和 C12" in dup.message + + +# -------------------------------------------------------------------------- +# 标签集合、分段定位 +# -------------------------------------------------------------------------- + + +def test_label_set_problems(): + ex = run(flow_book(days=3, day_hours=tuple(range(8, 18)))) # D04：日间少了 7-8 + assert codes(ex) == {"label_missing"} and problem(ex, "label_missing").fix == "remove_label" + def add_zone(ws, info): # D13 + ws.unmerge_cells("B8:E8") + ws["B8"] = "分区丙（人次）" + for j in range(3): + ws.cell(8, 3 + j, 5) + p = problem(run(flow_book(days=3, edit=add_zone)), "label_unexpected") + assert p.fix == "add_label" and p.cells == [f"{SHEET}!B8"] + def subtotal_row(ws, info): # D18 + ws.insert_rows(21) + ws["B21"] = "7-18 时合计" + for j in range(3): + ws.cell(21, 3 + j, 1) + raw, fn = flow_book(days=3, edit=subtotal_row, uncached=True) + p = problem(run((raw, fn)), "label_unparsed") + assert p.fix == "declare_total" and p.cells == [f"{SHEET}!B21"] + + +def test_label_order_and_writing_changes_are_not_problems(): + def reorder(ws, info): # D19 + for j in range(3 + 1): + c = 2 + j + a, b, t = ws.cell(5, c).value, ws.cell(6, c).value, ws.cell(7, c).value + ws.cell(5, c).value, ws.cell(6, c).value, ws.cell(7, c).value = b, t, a + ex = run(flow_book(days=3, edit=reorder)) + assert ex.ok + assert ex.labels[0].raw == ["分区甲（人次）", "分区乙（人次）", "全日客流（人次）"] + def writing(ws, info): # D11 / D24 + ws["B11"] = "08:00-09:00" + ws["B12"] = "9–10" + ex = run(flow_book(days=3, edit=writing)) + assert ex.ok and ex.canonicalized == [] + day = next(lab for lab in ex.labels if lab.segment == "日间") + assert day.raw[1:3] == ["08:00-09:00", "9–10"] and day.canonical[1:3] == ["8-9", "9-10"] + + +def test_title_and_segment_codes(): + def twice(ws, info): + ws.unmerge_cells("B8:E8") + ws["B8"] = "日间时段客流（人次）" + p = problem(run(flow_book(days=3, edit=twice)), "title_ambiguous") + assert p.cells == [f"{SHEET}!B8", f"{SHEET}!B9"] + bad_pick = recipe_dict() + bad_pick["sheets"][0]["blocks"][0]["segments"][2]["const"] = {"时段类别": {"pick": "晚上"}} + p = problem(run(flow_book(days=3), bad_pick), "const_missing") + assert p.cells == [f"{SHEET}!B21"] and "晚上" in p.message + gone = recipe_dict() + seg = gone["sheets"][0]["blocks"][0]["segments"][0] + seg["labels"]["expect"] = ["甲（人次）", "乙（人次）"] + seg["measures"] = {"甲（人次）": "甲", "乙（人次）": "乙"} + gone["tables"][0]["units"] = {} + gone["relations"] = [] + assert "segment_not_found" in codes(run(flow_book(days=3), gone)) + overlap = recipe_dict() + overlap["sheets"][0]["blocks"][0]["segments"].append({ + "id": "日间按标签", "role": "dimension", "table": "另表", "locate": {"by": "labels"}, + "labels": {"expect": ["8-9", "9-10"]}, "dim": {"name": "时段", "parser": "hour_range"}, "value": "客流"}) + overlap["tables"].append({"name": "另表", "grain": ["日期", "时段"]}) + assert "segment_overlap" in codes(run(flow_book(days=3), overlap)) + + +def test_row_codes(): + def extra_row(ws, info): # D21 + ws["B32"] = "补录（人次）" + ws["C32"] = "1,234" + ws["D32"] = "2,345" + ex = run(flow_book(days=3, edit=extra_row)) + assert codes(ex) == {"row_unclaimed"} + p = problem(ex, "row_unclaimed") + assert p.fix == "add_label" and p.cells[0] == f"{SHEET}!B32" + ex = run(flow_book(days=3, edit=lambda ws, info: set_value(ws, "D32", 987654))) + assert codes(ex) == {"row_without_label"} + + +def test_cells_right_of_the_axis_on_data_rows_have_nowhere_to_go(): + def edit(ws, info): + ws["G11"] = "备注" + ex = run(flow_book(days=3, edit=edit)) + p = problem(ex, "cell_unclaimed") + assert p.cells == [f"{SHEET}!G11"] and ex.ledger[0].unclaimed == 1 + + +# -------------------------------------------------------------------------- +# 日期轴 +# -------------------------------------------------------------------------- + + +def test_axis_location_codes(): + def no_dates(ws, info): + for j in range(3): + ws.cell(4, 3 + j).value = f"第{j}列" + ex = run(flow_book(days=3, edit=no_dates)) + assert codes(ex) == {"axis_not_found"} # 块级失败：别的格不再逐个报 + def twice(ws, info): + for j in range(3): + ws.cell(40, 3 + j).value = ws.cell(4, 3 + j).value + assert "axis_ambiguous" in codes(run(flow_book(days=3, edit=twice))) + p = problem(run(flow_book(days=5, edit=lambda ws, info: set_value(ws, "E4", "备注"))), "axis_gap") + assert p.cells == [f"{SHEET}!E4"] + def extra(ws, info): # D16 / P25 + ws["F4"] = "合计" + for r in range(5, 31): + ws.cell(r, 6).value = 999 + ex = run(flow_book(days=3, edit=extra)) + assert codes(ex) == {"axis_extra_cells"} # F 列的 999 不再报成区域外数字 + assert problem(ex, "axis_extra_cells").fix == "ignore_cells" + + +def test_axis_year_and_assertions(): + sep = flow_book(days=3, period="统计时间范围：2026年9月1日至2026年9月3日", + filename="月报导出_2026-09-01_2026-09-03.xlsx") + assert "axis_year_missing" in codes(run(sep)) + long = flow_book(start=dt.date(2026, 1, 5), days=3, + period="统计时间范围：2026年1月1日至2027年1月31日", filename="x.xlsx") + assert "axis_year_ambiguous" in codes(run(long)) + ex = run(flow_book(days=5, edit=lambda ws, info: set_value(ws, "E4", "8月2日"))) # P6 + assert codes(ex) == {"axis_duplicate"} + def swap(ws, info): + ws["D4"], ws["E4"] = "8月3日", "8月2日" + ex = run(flow_book(days=5, edit=swap)) + assert codes(ex) == {"axis_not_contiguous"} + relaxed = recipe_dict() + relaxed["sheets"][0]["blocks"][0]["axis"]["checks"] = ["covers_context"] + assert run(flow_book(days=5, edit=swap), relaxed).ok + p = problem(run(flow_book(days=5, period="统计时间范围：2026年8月1日至2026年8月6日")), "axis_coverage") + assert "共 5 格" in p.message and "共 6 天" in p.message + + +def test_date_cells_axis_form(): + def as_dates(ws, info): + for j in range(3): + ws.cell(4, 3 + j).value = dt.datetime(2026, 8, 1 + j) + ex = run(flow_book(days=3, edit=as_dates)) + assert ex.ok and ex.axes[0].form == "date" + ex = run(flow_book(days=3, edit=lambda ws, info: set_value(ws, "C4", dt.date(2026, 8, 1)))) + assert ex.ok and ex.axes[0].form == "mixed" + + +# -------------------------------------------------------------------------- +# 值的转换（4.5） +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("value,policy,expected", [ + (123, {}, 123), + (123.0, {}, 123), + (1.5, {}, "value_not_integer"), + (7, {"type": "REAL"}, 7.0), + (2.25, {"type": "REAL"}, 2.25), + ("·", {}, None), + (" · ", {}, None), + ("1,234", {"text_number": "parse_thousands"}, 1234), + ("1,234", {}, "value_text_number"), + (None, {"blank": "null"}, None), + (None, {}, "value_blank"), + ("#N/A", {}, "value_error"), + ("约5000", {}, "value_not_number"), + ("1234", {}, "value_not_number"), + (True, {}, "value_not_number"), + (dt.datetime(2026, 8, 1), {}, "value_not_number"), +]) +def test_value_conversion(tmp_path, value, policy, expected): + data = recipe_dict() + data["sheets"][0]["blocks"][0]["values"].update(policy) + book = flow_book(days=3, edit=lambda ws, info: set_value(ws, "C11", value)) + db = str(tmp_path / "v.db") + ex = run(book, data, db=db) + if isinstance(expected, str): + assert codes(ex) == {expected} and not ex.ok and not os.path.exists(db) + return + assert ex.ok, [(p.code, p.message) for p in ex.problems] + conn = sqlite3.connect(db) + got = conn.execute("SELECT 客流 FROM 时段客流 WHERE 日期='2026-08-01' AND 时段='8-9'").fetchone()[0] + conn.close() + assert got == expected and type(got) is type(expected) + + +def test_formula_policies_in_the_data_area(): + def formula(ws, info): + ws["C11"] = "=C12+1" + info["cache"]["C11"] = 42 + ex = run(flow_book(days=3, edit=formula)) + assert codes(ex) == {"value_formula"} + accept = recipe_dict(sheets__0__blocks__0__values__formula="accept_cached") + ex = run(flow_book(days=3, edit=formula), accept) + assert ex.ok and ex.formula_cells_accepted == 1 + def uncached(ws, info): + ws["C11"] = "=C12+1" + assert codes(run(flow_book(days=3, edit=uncached), accept)) == {"formula_uncached"} + ex = run(flow_book(days=3, edit=formula, keep_full_calc=True), accept) + assert codes(ex) == {"formula_full_calc"} and ex.full_calc_on_load + # 合计格是公式不受这两条限制：交给核对模块按标签区间重算 + ex = run(flow_book(days=3, keep_full_calc=True)) + assert ex.ok and ex.full_calc_on_load + + +# -------------------------------------------------------------------------- +# 统计期 +# -------------------------------------------------------------------------- + + +def test_period_conflict_prefers_prefix(): + ex = run(flow_book(days=3, edit=lambda ws, info: set_value(ws, "B3", "客流汇总表（2026年8月2日至2026年8月4日）"))) + c1 = ex.context_checks[0] + assert (c1.id, c1.status, c1.category, c1.acceptable, c1.failed) == ("C1", "mismatch", "data_quality", True, 1) + assert ex.period.start == "2026-08-01" # prefer_prefix「统计时间范围」那格 + assert ex.ok # 数据质量类：不阻断执行，由核对结果决定 + ex = run(flow_book(days=3, edit=lambda ws, info: set_value(ws, "B3", "客流汇总表（2026年9月）"))) + assert ex.context_checks[0].status == "mismatch" + + +def test_period_missing_and_human_input(): + no_period = flow_book(days=3, period="客流汇总") + ex = run(no_period) + p = problem(ex, "period_missing") + assert p.category == "input" and "2026-08-01 至 2026-08-03" in p.message + assert codes(ex) == {"period_missing"} and not ex.ok + entered = {"统计期": PeriodInput(dt.date(2026, 8, 1), dt.date(2026, 8, 3), signed_by="张三")} + ex = run(no_period, context_inputs=entered) + assert ex.ok and ex.period.source == "human" and ex.period.signed_by == "张三" + assert [(c.id, c.status) for c in ex.context_checks] == [("C2", "passed")] + ex = run(flow_book(days=3, filename="9月客流.xlsx")) + assert [(c.id, c.status) for c in ex.context_checks] == [("C1", "passed"), ("C2", "info")] + ex = run(flow_book(days=3, filename="月报导出_2026-07-01_2026-07-03.xlsx")) + c2 = ex.context_checks[1] + assert (c2.status, c2.acceptable, c2.category) == ("mismatch", True, "data_quality") and ex.ok + off = recipe_dict() + off["sheets"][0]["context"][0]["cross_check"] = "none" + assert [c.id for c in run(flow_book(days=3), off).context_checks] == ["C1"] + + +# -------------------------------------------------------------------------- +# 工作表、写库、哈希、部分干跑、两遍对账 +# -------------------------------------------------------------------------- + + +def test_sheet_matching(tmp_path): + def rename(wb_raw: bytes) -> bytes: + return patch_part(wb_raw, "xl/workbook.xml", lambda t: t.replace('name="客流汇总"', 'name="客流汇总九月"')) + raw, fn = flow_book(days=3) + ex = run((rename(raw), fn)) + assert ex.ok and ex.sheets.renamed == {"客流汇总": "客流汇总九月"} and ex.sheets.matched == {"s1": "客流汇总九月"} + strict = recipe_dict() + strict["sheets"][0]["match"]["fallback"] = "none" + assert codes(run((rename(raw), fn), strict)) == {"sheet_missing"} + + def with_extra(ws, info): + wb = ws.parent + wb.create_sheet("说明")["A1"] = "填表说明" + hidden = wb.create_sheet("配置") + hidden["A1"] = 987654 + hidden.sheet_state = "veryHidden" + book = flow_book(days=3, edit=with_extra) + ex = run(book) + assert ex.ok and ex.sheets.other_visible == ["说明"] + assert ex.sheets.skipped_hidden == [{"sheet": "配置", "state": "veryHidden"}] + reject = recipe_dict(other_visible_sheets="reject") + assert codes(run(book, reject)) == {"sheet_unexpected"} + + +def test_db_only_when_clean(tmp_path): + db = str(tmp_path / "bad.db") + ex = run(flow_book(days=3, day_hours=tuple(range(8, 18))), db=db) + assert not ex.ok and not os.path.exists(db) and ex.table_hashes == {} + ex = run(flow_book(days=3), db=None) + assert ex.ok and ex.table_hashes == {} and ex.expected_rows["时段客流"] == 51 + with pytest.raises(ValueError): + run(flow_book(days=3), db=str(tmp_path / "x.db"), max_rows=10) + + +def test_exception_removes_the_db(tmp_path, monkeypatch): + db = str(tmp_path / "boom.db") + def boom(self): + raise RuntimeError("中途出错") + monkeypatch.setattr(recipe_engine._Engine, "_hashes", boom) + with pytest.raises(RuntimeError): + run(flow_book(days=3), db=db) + assert not os.path.exists(db) + + +def test_recipe_shape_problem_blocks_everything(tmp_path): + data = recipe_dict() + seg = data["sheets"][0]["blocks"][0]["segments"][0] + seg["measures"] = {"全日客流(人次)": "全日客流", "分区甲（人次）": "分区甲", "分区乙（人次）": "分区乙"} + db = str(tmp_path / "shape.db") + ex = run(flow_book(days=3), data, db=db) + assert [(p.code, p.category) for p in ex.problems] == [("measures_keys_mismatch", "recipe")] + assert not ex.ok and not os.path.exists(db) + + +def _sqlite_order_key(row: tuple[Any, ...]) -> tuple[Any, ...]: + # SQLite 的排序：NULL < 数 < 文本 + return tuple((0, 0) if v is None else (1, v) if isinstance(v, (int, float)) else (2, v) for v in row) + + +def test_table_hash_without_primary_key_matches_python_sort(tmp_path): + data = recipe_dict() + data["tables"][1]["grain"] = [] + db = str(tmp_path / "nopk.db") + ex = run(flow_book(days=3), data, db=db) + assert ex.ok + conn = sqlite3.connect(db) + ddl = conn.execute("SELECT sql FROM sqlite_master WHERE name='时段客流'").fetchone()[0] + rows = conn.execute("SELECT * FROM 时段客流").fetchall() + conn.close() + assert "PRIMARY KEY" not in ddl + h = hashlib.sha256() + for row in sorted(rows, key=_sqlite_order_key): + h.update(json.dumps(list(row), ensure_ascii=False, separators=(",", ":")).encode("utf-8") + b"\n") + assert ex.table_hashes["时段客流"] == h.hexdigest() + + +def test_partial_dry_run_skips_whole_sheet_checks(): + ex = run(flow_book(days=3), max_rows=12) # 第 2–13 行 + assert ex.partial and ex.ok and not ex.problems + assert ex.expected_rows == {"日客流": 3, "时段客流": 4 * 3, "时段客流_表内合计": 0} + assert ex.ledger[0].nonempty_read < ex.ledger[0].nonempty_scan + ex = run(flow_book(days=3, edit=lambda ws, info: set_value(ws, "B32", "补录")), max_rows=25) + assert ex.partial and not ex.problems # 窗口之外的行不报 + + +def test_pass_mismatch_when_row_runs_disagree(tmp_path): + raw, fn = flow_book(days=3) + sc = xlsx_scan.scan(raw) + a, b, n = sc.sheets[0].row_runs[0] + sc.sheets[0].row_runs[0] = (a, b, n - 1) + db = str(tmp_path / "m.db") + ex = run((raw, fn), scan=sc, db=db) + p = problem(ex, "pass_mismatch") + assert "第 2 行：扫描到 0 个非空格，读取到 1 个" in p.message and not ex.ok and not os.path.exists(db) + + +def test_pass_mismatch_by_total_when_row_runs_missing(tmp_path): + """老暂存区的扫描经 scan_from_json 读回来时没有 row_runs：逐行比不了，只剩合计数把关（4.1：两遍对账的 + 合计数还要等于 SheetScan.nonempty）。少读一格也必须报 pass_mismatch、不建库。""" + raw, fn = flow_book(days=3) + sc = xlsx_scan.scan(raw) + sc.sheets[0].row_runs = [] + sc.sheets[0].nonempty += 1 + db = str(tmp_path / "t.db") + ex = run((raw, fn), scan=sc, db=db) + problem(ex, "pass_mismatch") + assert not ex.ok and not os.path.exists(db) + + +def test_outside_formula_cells_are_numbers_whatever_the_cache(tmp_path): + """区域外的公式格一律按 number 处理（4.2 第 6 步）：缓存值是文字（="备"&"注"）、或者没有缓存值， + 也按区域外数字拒收，不能当成文字只记录。""" + raw, fn = flow_book(days=3) + row = ('"备"&"注"备注' + 'LEN(B32)') + raw = patch_part(raw, "xl/worksheets/sheet1.xml", lambda t: t.replace("", row + "", 1)) + ex = run((raw, fn), db=str(tmp_path / "f.db")) + ps = [p for p in ex.problems if p.code == "outside_number"] # 按行各一条 + assert {p.category for p in ps} == {"structure"} + assert [c for p in ps for c in p.cells] == [f"{SHEET}!B32", f"{SHEET}!B34"], [p.cells for p in ps] + assert {o.cell for o in ex.outside_text} == {f"{SHEET}!B3"} # 只有原本的标题是区域外文字 + assert not ex.ok + + +def test_stale_dimension_gives_identical_hashes(tmp_path): + raw, fn = flow_book(days=3) + a = run((raw, fn), db=str(tmp_path / "a.db")) + for ref in ("B2:E27", "B2:D30"): # P23 截行、P24 截列 + stale = patch_part(raw, "xl/worksheets/sheet1.xml", + lambda t, ref=ref: re.sub(r'', f'', t)) + b = run((stale, fn), db=str(tmp_path / f"{ref.replace(':', '')}.db")) + assert b.ok and b.table_hashes == a.table_hashes + + +def test_measures_moved_below_with_a_new_title(): + def move(ws, info): # D06 + ws.unmerge_cells("B8:E8") + ws["B31"] = "日客流（人次）" + for k in range(3): + for c in range(2, 6): + ws.cell(32 + k, c).value = ws.cell(5 + k, c).value + ws.cell(5 + k, c).value = None + ex = run(flow_book(days=3, edit=move)) + assert ex.ok + assert ex.block_order == {SHEET: ["日间", "夜间", "夜间合计", "日客流"]} + assert ex.lineage["日客流"]["全日客流"] == [[1, SHEET, "C32", 3, "right"]] + assert (f"{SHEET}!B31", "text") in [(o.cell, o.kind) for o in ex.outside_text] + + +def test_block_order_follows_the_sheet(): + ex = run(flow_book(days=3, night_first=True)) # D05 + assert ex.ok and ex.block_order == {SHEET: ["日客流", "夜间", "夜间合计", "日间"]} + # 长表按分段在工作表里的先后写：夜间在前，rowid 从夜间第一行起 + assert ex.lineage["时段客流"]["客流"][0] == [1, SHEET, "C10", 3, "right"] + first_total = next(d for d in ex.derived if d.cell == f"{SHEET}!C16") + assert first_total.ref_rowids == [1, 4, 7, 10] + ex = run(flow_book(days=3, blank_row=False)) # D14：没有空行 + assert ex.ok diff --git a/backend/tests/test_recipe_engine_list.py b/backend/tests/test_recipe_engine_list.py new file mode 100644 index 00000000..f26bbece --- /dev/null +++ b/backend/tests/test_recipe_engine_list.py @@ -0,0 +1,707 @@ +"""配方执行器：列表（P2-SPEC 4.4、4.5、4.6 的列表部分、4.9 的流式路径）。 + +夹具是测试里现造的销售明细（地区甲/乙/丙、产品甲…，数字手写的假数），不依赖 WP-1 的夹具。 +""" +from __future__ import annotations + +import copy +import datetime as dt +import io +import os +import sqlite3 +from collections.abc import Callable +from typing import Any + +import pytest +from openpyxl import Workbook + +from app.data import recipe_engine, xlsx_cells, xlsx_scan +from app.data.recipe_types import Extraction, Recipe + +SHEET = "销售" +REGIONS = ("地区甲", "地区乙", "地区丙") + +LIST = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{ + "id": "s1", "match": {"name": SHEET}, + "blocks": [{ + "id": "列表1", "layout": "list", "table": "销售", + "columns": [{"header": "地区", "name": "地区", "type": "TEXT"}, + {"header": "产品", "name": "产品", "type": "TEXT"}, + {"header": "日期", "name": "日期", "type": "DATE"}, + {"header": "销量", "name": "销量", "type": "INTEGER"}, + {"header": "金额（万元）", "name": "金额", "type": "REAL"}], + "rows": {"total_row": {"label_column": "地区", "pick": "合计", "keep_as": "销售_表内合计"}}, + }], + }], + "tables": [{"name": "销售", "grain": ["地区", "产品"], "units": {"金额": "万元"}}, + {"name": "销售_表内合计", "grain": ["合计项"], "kind": "reported_total", "units": {"金额": "万元"}}], +} + + +def sales_book(n: int = 5, *, edit: Callable[[Any], None] | None = None, total: bool = True, + title: str | None = "销售月报") -> tuple[bytes, str]: + """A1 标题（合并 A1:E1）、第 3 行表头、第 4 行起 n 行明细、合计行（写死的数）。""" + wb = Workbook() + ws = wb.active + ws.title = SHEET + if title: + ws["A1"] = title + ws.merge_cells("A1:E1") + for c, h in enumerate(("地区", "产品", "日期", "销量", "金额（万元）"), 1): + ws.cell(3, c, h) + for i in range(n): + r = 4 + i + ws.cell(r, 1, REGIONS[i % 3]) + ws.cell(r, 2, f"产品{i:03d}") + ws.cell(r, 3, dt.datetime(2026, 8, 1) + dt.timedelta(days=i)) + ws.cell(r, 4, 10 + i) + ws.cell(r, 5, 1.5 * (i + 1)) + if total: + r = 4 + n + ws.cell(r, 1, "合计") + ws.cell(r, 4, sum(10 + i for i in range(n))) + ws.cell(r, 5, sum(1.5 * (i + 1) for i in range(n))) + if edit is not None: + edit(ws) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue(), "销售明细.xlsx" + + +def list_recipe(**block: Any) -> dict[str, Any]: + data = copy.deepcopy(LIST) + data["sheets"][0]["blocks"][0].update(block) + return data + + +def run(book: tuple[bytes, str], recipe: dict[str, Any] | None = None, db: str | None = None, + scan: xlsx_scan.WorkbookScan | None = None, **kw: Any) -> Extraction: + raw, filename = book + return recipe_engine.execute(Recipe.model_validate(recipe or LIST), raw, filename, + scan or xlsx_scan.scan(raw), db, **kw) + + +def codes(ex: Extraction) -> set[str]: + return {p.code for p in ex.problems} + + +def problem(ex: Extraction, code: str): + found = [p for p in ex.problems if p.code == code] + assert found, f"没有 {code}：{[(p.code, p.message) for p in ex.problems]}" + return found[0] + + +def rows_of(db: str, sql: str) -> list[tuple[Any, ...]]: + conn = sqlite3.connect(db) + try: + return conn.execute(sql).fetchall() + finally: + conn.close() + + +# -------------------------------------------------------------------------- +# 基线 +# -------------------------------------------------------------------------- + + +def test_list_reference(tmp_path): + db = str(tmp_path / "l.db") + ex = run(sales_book(), db=db) + assert ex.ok and not ex.problems + assert ex.ledger[0].roles == {"col_header": 5, "value": 25, "total_label": 1, "total_value": 2, "outside_text": 1} + assert ex.ledger[0].unclaimed == 0 and ex.ledger[0].nonempty_read == ex.ledger[0].nonempty_scan == 34 + assert ex.expected_rows == {"销售": 5, "销售_表内合计": 1} + assert [c.header for c in ex.tables[0].columns] == ["地区", "产品", "日期", "销量", "金额（万元）"] + assert ex.tables[0].sources == ["s1/列表1"] and ex.block_order == {SHEET: ["列表1"]} + assert ex.lineage["销售"]["金额"] == [[1, SHEET, "E4", 5, "down"]] + assert ex.lineage["销售_表内合计"]["销量"] == [[1, SHEET, "D9", 1, "right"]] + assert [(d.kind, d.cell, d.base_value, d.value, d.rowid_first, d.rowid_last, d.label, d.keep_table) + for d in ex.derived] == [ + ("column_sum", f"{SHEET}!D9", "销量", 60, 1, 5, "合计", "销售_表内合计"), + ("column_sum", f"{SHEET}!E9", "金额", 22.5, 1, 5, "合计", "销售_表内合计")] + assert rows_of(db, "SELECT rowid, * FROM 销售 ORDER BY rowid LIMIT 2") == [ + (1, "地区甲", "产品000", "2026-08-01", 10, 1.5), (2, "地区乙", "产品001", "2026-08-02", 11, 3.0)] + assert rows_of(db, "SELECT * FROM 销售_表内合计") == [("合计", 60, 22.5)] + assert [(o.cell, o.kind) for o in ex.outside_text] == [(f"{SHEET}!A1", "text")] + + +def test_text_column_dates_use_simple_import_writing(tmp_path): + def stamp(ws): + ws["C5"] = dt.datetime(2026, 8, 2, 9, 30) + data = copy.deepcopy(LIST) + data["sheets"][0]["blocks"][0]["columns"][2]["type"] = "TEXT" + db = str(tmp_path / "t.db") + ex = run(sales_book(edit=stamp), data, db=db) + assert ex.ok + assert [d for (d,) in rows_of(db, "SELECT 日期 FROM 销售 ORDER BY rowid LIMIT 2")] == [ + "2026-08-01", "2026-08-02 09:30:00"] # 期 1 的 #23：零点日期不写成「… 00:00:00」 + + +def test_single_integer_primary_key_is_not_a_rowid_alias(tmp_path): + def codes_col(ws): + for i, v in enumerate((100, 50, 75, 20, 90)): + ws.cell(4 + i, 4, v) + data = copy.deepcopy(LIST) + data["tables"][0]["grain"] = ["销量"] + data["sheets"][0]["blocks"][0]["rows"] = {} # 夹具没有合计行：配方也不要合计行（否则报 total_row_missing） + db = str(tmp_path / "i.db") + ex = run(sales_book(edit=codes_col, total=False), data, db=db) + assert ex.ok + assert rows_of(db, "SELECT rowid, 销量 FROM 销售 ORDER BY rowid") == [(1, 100), (2, 50), (3, 75), (4, 20), (5, 90)] + info = rows_of(db, "PRAGMA table_info('销售')") + assert [name for _, name, _, _, _, pk in info if pk] == ["销量"] + + +# -------------------------------------------------------------------------- +# 表头 +# -------------------------------------------------------------------------- + + +def test_header_codes(): + def rename_all(ws): + for c in range(1, 6): + ws.cell(3, c).value = f"列{c}" + assert codes(run(sales_book(edit=rename_all))) == {"header_not_found"} + def second(ws): + for c, h in enumerate(("地区", "产品", "日期", "销量", "金额（万元）"), 1): + ws.cell(20, c, h) + p = problem(run(sales_book(edit=second)), "header_ambiguous") + assert p.cells == [f"{SHEET}!A3", f"{SHEET}!A20"] + ex = run(sales_book(edit=lambda ws: ws.cell(3, 5, "销售额（万元）"))) # c10 第 10 期：改名 + assert codes(ex) == {"column_missing", "column_extra"} + assert "金额（万元）" in problem(ex, "column_missing").message + ex = run(sales_book(edit=lambda ws: (ws.cell(3, 6, "单价"), ws.cell(4, 6, 3)))) # c10 第 08 期：多一列 + assert codes(ex) == {"column_extra"} and problem(ex, "column_extra").cells == [f"{SHEET}!F3"] + + +def test_header_writing_variants_still_match(): + def writing(ws): + ws["E3"] = "金 额（万元）" + ws["D3"] = "销量\n" + ws["A3"] = "地区　" + ex = run(sales_book(edit=writing)) + assert ex.ok + # 本期读到的表头原文（首尾空白也保留），差异卡的 header_writing 才看得到这三处写法变化（9.3） + assert [c.header for c in ex.tables[0].columns] == ["地区　", "产品", "日期", "销量\n", "金 额（万元）"] + + +def test_header_not_found_names_the_rows_actually_searched(): + def rename_all(ws): + for c in range(1, 6): + ws.cell(3, c).value = f"列{c}" + p = problem(run(sales_book(edit=rename_all)), "header_not_found") + assert "在第 1–9 行中没有找到表头" in p.message # 已用区域只有 9 行，不写「第 1–200 行」 + wb = Workbook() + wb.active.title = SHEET + wb.active["A1"] = "销售月报" # 标题在最后一行：标题之后没有可找的行 + buf = io.BytesIO() + wb.save(buf) + p = problem(run((buf.getvalue(), "x.xlsx"), list_recipe(after_title="销售月报")), "header_not_found") + assert "行中" not in p.message and "没有找到表头" in p.message + + +def test_extra_columns_ignored(): + def remark(ws): + ws["F3"] = "备注" + ws["F4"] = "加急" + ex = run(sales_book(edit=remark), list_recipe(extra_columns="ignore")) + assert ex.ok and ex.ignored_columns == {"列表1": ["备注"]} + assert ex.ledger[0].roles["ignored_column"] == 2 + + +def test_multi_row_header_composes_merged_spans(tmp_path): + def build() -> tuple[bytes, str]: + wb = Workbook() + ws = wb.active + ws.title = SHEET + ws["A1"], ws["B1"], ws["C1"] = "地区", "产品", "2026年上半年" + ws.merge_cells("A1:A2") + ws.merge_cells("B1:B2") + ws.merge_cells("C1:D1") + ws["C2"], ws["D2"] = "销量", "金额" + for i, region in enumerate(REGIONS): + ws.cell(3 + i, 1, region) + ws.cell(3 + i, 2, "产品甲") + ws.cell(3 + i, 3, 5 + i) + ws.cell(3 + i, 4, 2.5) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue(), "上半年.xlsx" + data = {"recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": SHEET}, "blocks": [{ + "id": "列表1", "layout": "list", "table": "销售", "header_rows": 2, + "columns": [{"header": "地区", "name": "地区", "type": "TEXT"}, + {"header": "产品", "name": "产品", "type": "TEXT"}, + {"header": "2026年上半年_销量", "name": "上半年销量", "type": "INTEGER"}, + {"header": "2026年上半年_金额", "name": "上半年金额", "type": "REAL"}]}]}], + "tables": [{"name": "销售", "grain": ["地区"]}]} + ex = run(build(), data, db=str(tmp_path / "h.db")) + assert ex.ok, [(p.code, p.message) for p in ex.problems] + assert [c.header for c in ex.tables[0].columns] == ["地区", "产品", "2026年上半年_销量", "2026年上半年_金额"] + assert ex.ledger[0].roles["col_header"] == 5 and ex.expected_rows["销售"] == 3 + + +def test_after_title_stacked_tables(): + def build(second_title: str = "二、费用", dup: bool = False) -> tuple[bytes, str]: + wb = Workbook() + ws = wb.active + ws.title = SHEET + ws["B1"] = "一、销售" + ws["B2"], ws["C2"] = "项目", "金额" + ws["B3"], ws["C3"] = "项目甲", 10 + ws["B4"], ws["C4"] = "项目乙", 20 + ws["B6"] = second_title + ws["B7"], ws["C7"] = "项目", "金额" + ws["B8"], ws["C8"] = "项目丙", 30 + if dup: + ws["B10"] = "一、销售" + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue(), "两张表.xlsx" + def block(bid: str, table: str, title: str) -> dict[str, Any]: + return {"id": bid, "layout": "list", "table": table, "after_title": title, + "columns": [{"header": "项目", "name": "项目", "type": "TEXT"}, + {"header": "金额", "name": "金额", "type": "INTEGER"}]} + data = {"recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": SHEET}, + "blocks": [block("列表1", "销售", "一、销售"), block("列表2", "费用", "二、费用")]}], + "tables": [{"name": "销售", "grain": ["项目"]}, {"name": "费用", "grain": ["项目"]}]} + ex = run(build(), data) + assert ex.ok and ex.expected_rows == {"销售": 2, "费用": 1} + assert ex.block_order == {SHEET: ["列表1", "列表2"]} + assert {o.text for o in ex.outside_text} == {"一、销售", "二、费用"} + assert "title_not_found" in codes(run(build("二、支出"), data)) + p = problem(run(build(dup=True), data), "title_ambiguous") + assert p.cells == [f"{SHEET}!B1", f"{SHEET}!B10"] + + +def _two_tables(*, upper_title: bool, same_header: bool) -> tuple[tuple[bytes, str], dict[str, Any]]: + """上下两张表：上面一张（B1 可选标题「一、销售」），下面一张从 B7 起、没有标题。""" + wb = Workbook() + ws = wb.active + ws.title = SHEET + if upper_title: + ws["B1"] = "一、销售" + ws["B2"], ws["C2"] = "地区", "销量" + ws["B3"], ws["C3"] = "地区甲", 1 + ws["B4"], ws["C4"] = "地区乙", 2 + lower = ("地区", "销量") if same_header else ("部门", "费用") + ws["B7"], ws["C7"] = lower + ws["B8"], ws["C8"] = f"{lower[0]}丙", 3 + ws["B9"], ws["C9"] = f"{lower[0]}丁", 4 + buf = io.BytesIO() + wb.save(buf) + def block(bid: str, head: tuple[str, str], title: str | None) -> dict[str, Any]: + return {"id": bid, "layout": "list", "table": bid, "after_title": title, + "columns": [{"header": head[0], "name": head[0], "type": "TEXT"}, + {"header": head[1], "name": head[1], "type": "INTEGER"}]} + data = {"recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": SHEET}, "blocks": [ + block("上表", ("地区", "销量"), "一、销售" if upper_title else None), + block("下表", lower, None)]}], + "tables": [{"name": "上表", "grain": ["地区"]}, {"name": "下表", "grain": [lower[0]]}]} + return (buf.getvalue(), "两张表.xlsx"), data + + +def test_block_without_title_below_a_titled_block_is_found(): + # 下表没写 after_title：窗口从第 1 行起，第 1 行正是上表的标题。窗口里还没有候选表头时不截断（评审：曾报 + # 「在第 1–0 行中没有找到表头」，这份合法的配方永远过不了） + book, data = _two_tables(upper_title=True, same_header=False) + ex = run(book, data) + assert ex.ok and not ex.problems and ex.expected_rows == {"上表": 2, "下表": 2} + assert ex.block_order == {SHEET: ["上表", "下表"]} + flipped = copy.deepcopy(data) + flipped["sheets"][0]["blocks"].reverse() + ex = run(book, flipped) + assert ex.ok and ex.expected_rows == {"下表": 2, "上表": 2} + + +def test_untitled_block_window_still_stops_at_the_next_title_after_its_header(): + # 上表没写标题、下表写了：两张表头相同。上表的窗口里已经有候选表头之后再碰到下表的标题，窗口到此为止， + # 下表的同名表头不进上表的窗口，「恰好一处」判得出来 + book, data = _two_tables(upper_title=False, same_header=True) + raw = book[0] + from openpyxl import load_workbook + wb = load_workbook(io.BytesIO(raw)) + wb.active["B6"] = "二、续表" + buf = io.BytesIO() + wb.save(buf) + data["sheets"][0]["blocks"][1]["after_title"] = "二、续表" + ex = run((buf.getvalue(), book[1]), data) + assert ex.ok and not ex.problems and ex.expected_rows == {"上表": 2, "下表": 2} + # 两张表都没写标题、表头又相同：照实报「多处都像表头」，不猜 + data["sheets"][0]["blocks"][1]["after_title"] = None + assert "header_ambiguous" in codes(run((buf.getvalue(), book[1]), data)) + + +def test_side_by_side_tables_split_by_blank_column(): + def build() -> tuple[bytes, str]: + wb = Workbook() + ws = wb.active + ws.title = SHEET + ws["A1"], ws["B1"], ws["D1"], ws["E1"] = "项目", "金额", "科目", "费用" + ws["A2"], ws["B2"], ws["D2"], ws["E2"] = "项目甲", 1, "科目甲", 5 + ws["A3"], ws["B3"], ws["D3"], ws["E3"] = "项目乙", 2, "科目乙", 6 + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue(), "左右.xlsx" + data = {"recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": SHEET}, "blocks": [ + {"id": "列表1", "layout": "list", "table": "左表", + "columns": [{"header": "项目", "name": "项目", "type": "TEXT"}, + {"header": "金额", "name": "金额", "type": "INTEGER"}]}, + {"id": "列表2", "layout": "list", "table": "右表", + "columns": [{"header": "科目", "name": "科目", "type": "TEXT"}, + {"header": "费用", "name": "费用", "type": "INTEGER"}]}]}], + "tables": [{"name": "左表", "grain": ["项目"]}, {"name": "右表", "grain": ["科目"]}]} + ex = run(build(), data) + assert ex.ok and ex.expected_rows == {"左表": 2, "右表": 2} and ex.ledger[0].unclaimed == 0 + # 占位符按块声明：左表声明的「—」在右表里不算占位符 + marked = copy.deepcopy(data) + marked["sheets"][0]["blocks"][0]["values"] = {"placeholders": [{"text": "—"}]} + def dash(ws): + ws["E3"] = "—" + raw, fn = build() + from openpyxl import load_workbook + wb = load_workbook(io.BytesIO(raw)) + dash(wb.active) + buf = io.BytesIO() + wb.save(buf) + ex = run((buf.getvalue(), fn), marked) + p = problem(ex, "value_not_number") + assert p.cells == [f"{SHEET}!E3"] and ex.placeholders == {} + one = copy.deepcopy(data) + one["sheets"][0]["blocks"].pop() + one["tables"].pop() + p = problem(run(build(), one), "cell_unclaimed") # 右边那张表的格在左表的数据行上，没有去处 + assert p.fix == "ignore_cells" + + +# -------------------------------------------------------------------------- +# 数据行：空行、合计行、停止之后 +# -------------------------------------------------------------------------- + + +def test_rows_after_stop_need_confirmation(): + def names(ws): + ws["A11"], ws["B11"] = "名单甲", "名单乙" + ws["A12"], ws["B12"] = "名单丙", "名单丁" + ws["A14"] = "单独一格" # 只有一格：不算像数据的行 + ex = run(sales_book(total=False, edit=names), list_recipe(rows={})) + p = problem(ex, "rows_after_stop") + assert p.category == "confirm" and "第 11 行起有 2 行" in p.message + assert p.cells == [f"{SHEET}!A11", f"{SHEET}!B11", f"{SHEET}!A12", f"{SHEET}!B12"] + assert ex.ok + + +def test_header_without_data_rows_needs_confirmation(): + """AU-1：只有表头、没有数据行：不拒收（本月可能确实没有记录），但出 confirm 类问题，表里 0 行。""" + def wipe(ws): + for row in ws.iter_rows(min_row=4): + for cell in row: + cell.value = None + ex = run(sales_book(total=False, edit=wipe), list_recipe(rows={})) + p = problem(ex, "list_empty") + assert p.category == "confirm" and "列表「列表1」" in p.message and "0 行" in p.message + assert p.cells == [f"{SHEET}!A3"] + assert ex.ok and next(t for t in ex.tables if t.name == "销售").rows == 0 + # 有数据行时不报 + assert not [x for x in run(sales_book(total=False), list_recipe(rows={})).problems if x.code == "list_empty"] + + +def test_rows_after_stop_in_a_one_column_list(): + # 只有 1 列的名单：空行之后的名字每行只有 1 格，也要出确认（规格写「2 格及以上」，1 列的块按 1 格算） + wb = Workbook() + ws = wb.active + ws.title = "名单" + ws["A1"], ws["A2"], ws["A3"], ws["A5"], ws["A6"] = "姓名", "甲某", "乙某", "丙某", "丁某" + buf = io.BytesIO() + wb.save(buf) + data = {"recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": "名单"}, "blocks": [ + {"id": "名单", "layout": "list", "table": "名单", + "columns": [{"header": "姓名", "name": "姓名", "type": "TEXT"}]}]}], + "tables": [{"name": "名单", "grain": []}]} + ex = run((buf.getvalue(), "名单.xlsx"), data) + p = problem(ex, "rows_after_stop") + assert "第 5 行起有 2 行" in p.message and p.cells == ["名单!A5", "名单!A6"] + assert ex.ok and ex.expected_rows == {"名单": 2} + + +def test_row_with_only_ignored_columns_is_not_a_record(tmp_path): + # 紧跟数据的下一行只在忽略的「备注」列写了字：不是数据行，按空行处理（默认 stop），那一格进区域外文字。 + # 曾当成数据行写进一条全是空值的记录（没有主键时完全不出声） + def remark(ws): + ws["F3"] = "备注" + ws["F4"] = "加急" + ws["F9"] = "注：以上为初步统计" + data = list_recipe(extra_columns="ignore", rows={}) # 夹具没有合计行 + data["tables"][0]["grain"] = [] + db = str(tmp_path / "r.db") + ex = run(sales_book(edit=remark, total=False), data, db=db) + assert ex.ok and not ex.problems and ex.expected_rows["销售"] == 5 + assert rows_of(db, "SELECT COUNT(*) FROM 销售 WHERE 地区 IS NULL") == [(0,)] + assert (f"{SHEET}!F9", "text") in [(o.cell, o.kind) for o in ex.outside_text] + assert ex.ledger[0].roles["ignored_column"] == 2 and ex.ledger[0].unclaimed == 0 + ex = run(sales_book(edit=remark, total=False), dict(data, sheets=[ + dict(data["sheets"][0], blocks=[dict(data["sheets"][0]["blocks"][0], rows={"blank_rows": "skip"})])])) + assert ex.ok and ex.expected_rows["销售"] == 5 and ex.blank_rows_skipped == 1 + + +def test_blank_row_stop_and_skip(): + def gap(ws): + ws.insert_rows(6) + ex = run(sales_book(edit=gap, total=False), list_recipe(rows={})) + assert {"rows_after_stop", "outside_number"} <= codes(ex) and not ex.ok + assert ex.expected_rows == {"销售": 2} + ex = run(sales_book(edit=gap, total=False), list_recipe(rows={"blank_rows": "skip"})) + assert ex.ok and ex.blank_rows_skipped == 1 and ex.expected_rows["销售"] == 5 + assert ex.lineage["销售"]["地区"] == [[1, SHEET, "A4", 2, "down"], [3, SHEET, "A7", 3, "down"]] + + +def test_total_row_formulas_and_hidden_rows(): + def subtotal(ws): + ws["D9"] = "=SUBTOTAL(9,D4:D8)" + ws["E9"] = "=SUM(E4:E8)" + ex = run(sales_book(edit=subtotal)) + assert ex.ok + d, e = ex.derived + assert (d.func, d.ref_problem, d.value, d.ref_rowids) == ("SUBTOTAL", "hidden", None, [1, 2, 3, 4, 5]) + assert (e.func, e.ref_problem, e.is_formula) == ("SUM", None, True) + def hide(ws): + ws.row_dimensions[5].hidden = True + ex = run(sales_book(edit=hide), copy.deepcopy(LIST) | {"sheets": [ + dict(LIST["sheets"][0], hidden={"rows": "exclude"})]}) + assert ex.ok and ex.expected_rows["销售"] == 4 + assert all(d.hidden_in_range and (d.rowid_first, d.rowid_last) == (1, 4) for d in ex.derived) + assert ex.hidden == {SHEET: {"rows": [5], "cols": [], "policy_rows": "exclude", "policy_cols": "reject_if_any"}} + ex = run(sales_book(edit=hide)) + assert "hidden_rows" in codes(ex) + + +# -------------------------------------------------------------------------- +# 值 +# -------------------------------------------------------------------------- + + +def test_placeholder_in_a_number_primary_key_is_rejected(): + # INTEGER 主键列里的占位符按空值存，主键不能为空：曾静默丢掉这一行（ok=True、没有问题、少一行） + data = copy.deepcopy(LIST) + data["tables"][0]["grain"] = ["销量"] + data["sheets"][0]["blocks"][0]["values"] = {"placeholders": [{"text": "·"}]} + data["sheets"][0]["blocks"][0]["rows"] = {} # 夹具没有合计行 + ex = run(sales_book(edit=lambda ws: ws.cell(5, 4, "·"), total=False), data) + assert not ex.ok and codes(ex) == {"value_blank"} + p = problem(ex, "value_blank") + assert p.cells == [f"{SHEET}!D5"] and "占位符" in p.message and "·" not in p.model_message + assert ex.placeholders == {} + data["tables"][0]["grain"] = [] # 不是主键：照常存空值、计数 + ex = run(sales_book(edit=lambda ws: ws.cell(5, 4, "·"), total=False), data) + assert ex.ok and ex.placeholders == {"·": 1} and ex.expected_rows["销售"] == 5 + + +def test_total_row_configured_but_missing_or_blank_is_reported(tmp_path): + """配方里有合计行：本期没有找到（或合计行的数字格全空）时明确拒收，不能沉默——否则核对里没有 T、说明照旧 + 写「原表的合计行未导入本表」，另存的合计表是空的（或多一行空值）。部分干跑合计行可能还在窗口之外，不报。""" + db = str(tmp_path / "m.db") + ex = run(sales_book(total=False), LIST, db=db) + p = problem(ex, "total_row_missing") + assert not ex.ok and p.category == "structure" and not os.path.exists(db) + assert "「地区」列以「合计」开头" in p.message and "取消「有合计行」" in p.message and p.cells == [f"{SHEET}!A8"] + assert not any(d.kind == "column_sum" for d in ex.derived) + # 合计行在、数字格全空 + def blank(ws): + ws["D9"] = ws["E9"] = None + ex = run(sales_book(edit=blank), LIST) + p = problem(ex, "total_row_blank") + assert not ex.ok and p.category == "structure" and p.cells == [f"{SHEET}!A9"] and "全部为空" in p.message + assert "total_row_missing" not in codes(ex) + # 只空了一列：照常核对另一列（不报） + def one_blank(ws): + ws["D9"] = None + ex = run(sales_book(edit=one_blank), list_recipe()) + assert "total_row_blank" not in codes(ex) and "total_row_missing" not in codes(ex) + # 部分干跑：只读前几行，合计行在窗口之外，不报 + ex = run(sales_book(n=40), LIST, max_rows=10) + assert ex.partial and "total_row_missing" not in codes(ex) + + +def test_blank_number_in_kept_total_primary_key_is_reported(): + # 合计行另存的表把数字列放进主键：合计格是空的，那一行不能静默不存 + data = copy.deepcopy(LIST) + data["tables"][1]["grain"] = ["合计项", "销量"] + def blank(ws): + ws["D9"] = None + ex = run(sales_book(edit=blank), data) + assert not ex.ok and codes(ex) == {"value_blank"} + p = problem(ex, "value_blank") + assert p.cells == [f"{SHEET}!D9"] and "主键列「销量」" in p.message + + +@pytest.mark.parametrize("ref,value,block,expected", [ + ("D4", None, {}, None), # 列表的空格默认存空值 + ("D4", None, {"values": {"blank": "reject"}}, "value_blank"), + ("A4", None, {}, "value_blank"), # 主键列的空格一律拒收 + ("D4", "1,234", {}, "value_text_number"), + ("D4", "1,234", {"values": {"text_number": "parse_thousands"}}, 1234), + ("D4", "—", {"values": {"placeholders": [{"text": "—", "meaning": "不适用"}]}}, None), + ("D4", 2.5, {}, "value_not_integer"), + ("D4", "#REF!", {}, "value_error"), + ("D4", "若干", {}, "value_not_number"), + ("C4", "2026-08-09", {}, "2026-08-09"), + ("C4", "8月9日", {}, "value_not_date"), # 列表的日期必须自带年份 + ("C4", "8/9", {}, "value_not_date"), + ("C4", dt.datetime(2026, 8, 9, 10, 0), {}, "value_not_date"), + ("C4", None, {"values": {"blank": "reject"}}, "value_blank"), +]) +def test_list_value_conversion(tmp_path, ref, value, block, expected): + def edit(ws): + ws[ref] = value + db = str(tmp_path / "v.db") + ex = run(sales_book(edit=edit), list_recipe(**block), db=db) + if isinstance(expected, str) and expected.startswith("value_"): + assert codes(ex) == {expected} and not ex.ok + return + assert ex.ok, [(p.code, p.message) for p in ex.problems] + column = {"C": "日期", "D": "销量"}[ref[0]] + (got,) = rows_of(db, f"SELECT {column} FROM 销售 WHERE rowid = 1")[0] + assert got == expected + + +def test_list_formulas(tmp_path): + from tests.test_recipe_engine_crosstab import inject_cache + + def amount(ws): + for r in range(4, 9): + ws.cell(r, 5).value = f"=D{r}*0.15" + raw, fn = sales_book(edit=amount) + cached = inject_cache(raw, {f"E{r}": (10 + r - 4) * 0.15 for r in range(4, 9)}) + ex = run((cached, fn)) + assert ex.ok and ex.formula_cells_accepted == 5 + assert codes(run((cached, fn), list_recipe(values={"formula": "reject"}))) == {"value_formula"} + assert codes(run((raw, fn))) == {"formula_uncached"} + full = inject_cache(raw, {f"E{r}": 0 for r in range(4, 9)}, keep_full_calc=True) + assert codes(run((full, fn))) == {"formula_full_calc"} + + +def test_canonical_text_and_canonicalized(tmp_path): + def writing(ws): + ws["A4"] = "地区甲 " # 首尾空白：as_text 已去掉，不算规范化 + ws["B5"] = "产品００１" # 全角数字：主键列按规范写法存 + ws["A6"] = "地区　丙" + db = str(tmp_path / "c.db") + ex = run(sales_book(edit=writing), db=db) + assert ex.ok + assert [(c.column, c.raw, c.canonical, c.count, c.first_cell) for c in ex.canonicalized] == [ + ("产品", "产品００１", "产品001", 1, f"{SHEET}!B5"), ("地区", "地区　丙", "地区 丙", 1, f"{SHEET}!A6")] + assert rows_of(db, "SELECT 地区, 产品 FROM 销售 WHERE rowid IN (2, 3) ORDER BY rowid") == [ + ("地区乙", "产品001"), ("地区 丙", "产品002")] + raw_store = copy.deepcopy(LIST) + raw_store["sheets"][0]["blocks"][0]["columns"][1]["store"] = "raw" + ex = run(sales_book(edit=writing), raw_store) + assert [c.column for c in ex.canonicalized] == ["地区"] + + +def test_pk_duplicate_lists_both_cells(): + def dup(ws): + ws["A5"], ws["B5"] = "地区甲", "产品000" + p = problem(run(sales_book(edit=dup)), "pk_duplicate") + assert "A4 和 A5" in p.message and p.cells == [f"{SHEET}!A4", f"{SHEET}!A5"] + + +def test_merged_data_reject_and_fill(tmp_path): + def merged(ws): + ws["A5"] = None + ws.merge_cells("A4:A5") + ex = run(sales_book(edit=merged)) + assert codes(ex) == {"merged_in_values"} # 合并区里的空格不再另报为空 + db = str(tmp_path / "f.db") + ex = run(sales_book(edit=merged), list_recipe(merged_data="fill"), db=db) + assert ex.ok and rows_of(db, "SELECT 地区 FROM 销售 WHERE rowid = 2") == [("地区甲",)] + def merged_number(ws): + ws["D5"] = None + ws.merge_cells("D4:D5") + assert codes(run(sales_book(edit=merged_number), list_recipe(merged_data="fill"))) == {"merged_in_values"} + + +def test_empty_matched_sheet_reports_header_not_found(): + wb = Workbook() + wb.active.title = SHEET + wb.create_sheet("别的")["A1"] = "填表说明" + buf = io.BytesIO() + wb.save(buf) + ex = run((buf.getvalue(), "空表.xlsx")) + assert "header_not_found" in codes(ex) and not ex.ok + + +# -------------------------------------------------------------------------- +# 流式路径、部分干跑 +# -------------------------------------------------------------------------- + + +def test_stream_path_matches_grid_path(tmp_path, monkeypatch): + book = sales_book(n=40) + grid = run(book, db=str(tmp_path / "g.db")) + monkeypatch.setattr(xlsx_cells, "GRID_MAX_CELLS", 50) + stream = run(book, db=str(tmp_path / "s.db")) + assert grid.ok and stream.ok + assert stream.table_hashes == grid.table_hashes + assert stream.ledger == grid.ledger and stream.lineage == grid.lineage + assert stream.derived == grid.derived and stream.regions == grid.regions + assert stream.outside_text == grid.outside_text + + +def test_stream_path_layout_limits(monkeypatch): + monkeypatch.setattr(xlsx_cells, "GRID_MAX_CELLS", 10) + two = copy.deepcopy(LIST) + second = copy.deepcopy(two["sheets"][0]["blocks"][0]) + second.update(id="列表2", table="销售", after_title="不存在") + two["sheets"][0]["blocks"].append(second) + assert codes(run(sales_book(), two)) == {"sheet_too_large_for_layout"} + + +def test_stream_path_problems(tmp_path, monkeypatch): + monkeypatch.setattr(xlsx_cells, "GRID_MAX_CELLS", 10) + raw, fn = sales_book() + sc = xlsx_scan.scan(raw) + a, b, n = sc.sheets[0].row_runs[-1] + sc.sheets[0].row_runs[-1] = (a, b, n + 1) + db = str(tmp_path / "s.db") + ex = run((raw, fn), scan=sc, db=db) + assert codes(ex) == {"pass_mismatch"} and not os.path.exists(db) + ex = run(sales_book(edit=lambda ws: ws.cell(3, 5, "销售额")), db=db) + assert codes(ex) == {"column_missing", "column_extra"} and not os.path.exists(db) + def after(ws): + ws["A12"], ws["B12"] = 987654, "说明" + ex = run(sales_book(total=False, edit=after), list_recipe(rows={})) + assert {"rows_after_stop", "outside_number"} <= codes(ex) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_regions_stay_rectangles_when_edge_columns_are_sparse(monkeypatch, stream): + # 末列（金额）隔行为空、首列偶有空格：区域按块的列范围出矩形，不会每行一个（4.9「区域标记按矩形」） + def sparse(ws): + for i in range(400): + r = 4 + i + ws.cell(r, 1, REGIONS[i % 3] if i % 7 else None) + ws.cell(r, 2, f"产品{i:03d}") + ws.cell(r, 3, dt.datetime(2026, 8, 1)) + ws.cell(r, 4, i) + ws.cell(r, 5, 1.5 if i % 2 else None) + data = list_recipe(rows={}) + data["tables"][0]["grain"] = ["产品"] + if stream: + monkeypatch.setattr(xlsx_cells, "GRID_MAX_CELLS", 50) + ex = run(sales_book(n=0, edit=sparse, total=False), data) + assert ex.ok + assert [(m.role, m.ref) for m in ex.regions] == [ + ("outside_text", "A1"), ("col_header", "A3:E3"), ("value", "A4:E403")] + + +def test_partial_dry_run_on_a_long_list(): + ex = run(sales_book(n=60), max_rows=10) + assert ex.partial and ex.ok and not ex.problems + assert ex.expected_rows == {"销售": 7, "销售_表内合计": 0} # 第 1–10 行：表头在第 3 行 + assert ex.table_hashes == {} diff --git a/backend/tests/test_recipe_engine_messages.py b/backend/tests/test_recipe_engine_messages.py new file mode 100644 index 00000000..26dd3d84 --- /dev/null +++ b/backend/tests/test_recipe_engine_messages.py @@ -0,0 +1,444 @@ +"""model_message 契约测试（P2-SPEC 6.2、第 10 节 WP-2）。 + +AI 修订时回灌给模型的是 Problem.model_message，绝不退回 message（message 可以带格子里的值）。这里为执行器 +能产出的**每个** problem code 各造一个触发夹具，触发格里放特征数（987654）或特征文字，断言： +- 这个 code 真的出现了； +- 它的 model_message 非空； +- model_message 里没有触发格的值。 +测试里维护一份全量 code 表，和 recipe_engine.PROBLEM_CODES 比对：执行器新增 code 不补用例，这个测试就失败。 +""" +from __future__ import annotations + +import datetime as dt +import re +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from app.data import recipe_engine, xlsx_cells, xlsx_scan +from app.data.recipe_types import Extraction, Recipe +from tests.test_recipe_engine_crosstab import FLOW, flow_book, recipe_dict +from tests.test_recipe_engine_list import LIST, list_recipe, sales_book + +N = 987654 + + +def _flow(edit: Callable[[Any, dict[str, Any]], None] | None = None, **kw: Any) -> tuple[bytes, str]: + return flow_book(days=3, edit=edit, **kw) + + +def _set(**cells: Any) -> Callable[[Any, dict[str, Any]], None]: + def edit(ws: Any, info: dict[str, Any]) -> None: + for ref, value in cells.items(): + ws[ref] = value + return edit + + +def _run(book: tuple[bytes, str], recipe: dict[str, Any], *, scan_fix: Callable[[Any], None] | None = None, + **kw: Any) -> Extraction: + raw, filename = book + scan = xlsx_scan.scan(raw) + if scan_fix is not None: + scan_fix(scan) + return recipe_engine.execute(Recipe.model_validate(recipe), raw, filename, scan, None, **kw) + + +# -------------------------------------------------------------------------- +# 每个 code 的触发夹具：返回 (Extraction, 触发格的值) +# -------------------------------------------------------------------------- + +Case = Callable[[pytest.MonkeyPatch], tuple[Extraction, list[Any]]] + + +def c_table_shape_conflict(mp): + data = recipe_dict() + data["sheets"][0]["blocks"][0]["segments"][2]["table"] = "日客流" + return _run(_flow(_set(C11=N)), data), [N] + + +def c_measures_keys_mismatch(mp): + data = recipe_dict() + seg = data["sheets"][0]["blocks"][0]["segments"][0] + seg["measures"] = {"全日客流(人次)": "全日客流", "分区甲（人次）": "分区甲", "分区乙（人次）": "分区乙"} + return _run(_flow(_set(C5=N)), data), [N] + + +def c_sheet_missing(mp): + data = recipe_dict() + data["sheets"][0]["match"] = {"name": "另一张表", "fallback": "none"} + return _run(_flow(_set(C5=N)), data), [N] + + +def c_sheet_unexpected(mp): + def edit(ws, info): + ws.parent.create_sheet("说明")["A1"] = N + return _run(_flow(edit), recipe_dict(other_visible_sheets="reject")), [N] + + +def c_sheet_too_large_for_layout(mp): + mp.setattr(xlsx_cells, "GRID_MAX_CELLS", 10) + return _run(_flow(_set(C11=N)), FLOW), [N] + + +def c_merged_too_many(mp): + def fix(scan): + scan.sheets[0].merged_total = len(scan.sheets[0].merged) + 1 + return _run(_flow(_set(C11=N)), FLOW, scan_fix=fix), [N] + + +def c_sparse(mp): + return _run(_flow(_set(Z100000=N)), FLOW), [N] + + +def c_pass_mismatch(mp): + def fix(scan): + a, b, n = scan.sheets[0].row_runs[0] + scan.sheets[0].row_runs[0] = (a, b, n + 1) + return _run(_flow(_set(C11=N)), FLOW, scan_fix=fix), [N] + + +def c_cell_unclaimed(mp): + return _run(_flow(_set(G11=N)), FLOW), [N] + + +def c_outside_number(mp): + return _run(_flow(_set(B1=N)), FLOW), [N] + + +def c_outside_digits(mp): + text = f"注：{N}号闸机故障" + return _run(_flow(_set(B32=text)), FLOW), [text, N] + + +def c_hidden_rows(mp): + def edit(ws, info): + ws["C23"] = N + ws.row_dimensions[23].hidden = True + return _run(_flow(edit), FLOW), [N] + + +def c_hidden_cols(mp): + def edit(ws, info): + ws["D11"] = N + ws.column_dimensions["D"].hidden = True + return _run(_flow(edit), FLOW), [N] + + +def c_period_missing(mp): + text = f"统计时间：第{N}期" + return _run(_flow(period=text, filename=f"客流{N}.xlsx"), FLOW), [text, N] + + +def c_axis_not_found(mp): + return _run(_flow(_set(C4=N, D4=N + 1, E4=N + 2)), FLOW), [N, N + 1, N + 2] + + +def c_axis_ambiguous(mp): + def edit(ws, info): + for j in range(3): + ws.cell(40, 3 + j).value = ws.cell(4, 3 + j).value + ws["B40"] = N + return _run(_flow(edit), FLOW), ["8月1日", "8月3日", N] + + +def c_axis_gap(mp): + return _run(flow_book(days=5, edit=_set(E4=N)), FLOW), [N] + + +def c_axis_extra_cells(mp): + text = f"合计{N}" + def edit(ws, info): + ws["F4"] = text + ws["F5"] = N + return _run(_flow(edit), FLOW), [text, N] + + +def c_axis_year_missing(mp): + book = _flow(period="统计时间范围：2026年9月1日至2026年9月3日") + return _run(book, FLOW), ["8月1日", "8月2日", "2026年9月1日"] + + +def c_axis_year_ambiguous(mp): + book = flow_book(start=dt.date(2026, 1, 5), days=3, period="统计时间范围：2026年1月1日至2027年1月31日") + return _run(book, FLOW), ["1月5日", "1月6日"] + + +def c_axis_duplicate(mp): + return _run(flow_book(days=5, edit=_set(E4="8月2日")), FLOW), ["8月2日", "2026-08-02"] + + +def c_axis_not_contiguous(mp): + return _run(flow_book(days=5, edit=_set(D4="8月3日", E4="8月2日")), FLOW), ["8月3日", "2026-08-03", + "2026-08-02"] + + +def c_axis_coverage(mp): + book = flow_book(days=5, period="统计时间范围：2026年8月1日至2026年8月6日") + return _run(book, FLOW), ["8月1日", "8月5日", "2026-08-01", "2026-08-06"] + + +def c_row_without_label(mp): + return _run(_flow(_set(D32=N)), FLOW), [N] + + +def c_row_unclaimed(mp): + label = f"补录{N}" + return _run(_flow(_set(B32=label, C32=N + 1)), FLOW), [label, N + 1] + + +def c_title_not_found(mp): + title = f"日间{N}时段" + return _run(_flow(_set(B9=title)), FLOW), [title] + + +def c_title_ambiguous(mp): + title = "日间时段客流 （人次）" # 与配方的标题 match_key 相同，原文不同 + def edit(ws, info): + ws.unmerge_cells("B8:E8") + ws["B8"] = title + return _run(_flow(edit), FLOW), [title] + + +def c_segment_not_found(mp): + data = recipe_dict() + seg = data["sheets"][0]["blocks"][0]["segments"][0] + seg["labels"]["expect"] = ["甲（人次）", "乙（人次）"] + seg["measures"] = {"甲（人次）": "甲", "乙（人次）": "乙"} + data["tables"][0]["units"] = {} + data["relations"] = [] + return _run(_flow(_set(B5=f"全日{N}")), data), [f"全日{N}"] + + +def c_segment_overlap(mp): + data = recipe_dict() + data["sheets"][0]["blocks"][0]["segments"].append({ + "id": "日间按标签", "role": "dimension", "table": "另表", "locate": {"by": "labels"}, + "labels": {"expect": ["8-9", "9-10"]}, "dim": {"name": "时段", "parser": "hour_range"}, "value": "客流"}) + data["tables"].append({"name": "另表", "grain": ["日期", "时段"]}) + return _run(_flow(_set(C11=N)), data), [N] + + +def c_label_missing(mp): + return _run(flow_book(days=3, day_hours=tuple(range(8, 18)), edit=_set(C10=N)), FLOW), [N] + + +def c_label_unexpected(mp): + label = f"分区{N}（人次）" + def edit(ws, info): + ws.unmerge_cells("B8:E8") + ws["B8"] = label + for j in range(3): + ws.cell(8, 3 + j, N + 1) + return _run(_flow(edit), FLOW), [label, N + 1] + + +def c_label_duplicate(mp): + return _run(_flow(_set(B12="8－9")), FLOW), ["8－9"] + + +def c_label_unparsed(mp): + label = "7-18 时合计" + def edit(ws, info): + ws.insert_rows(21) + ws["B21"] = label + for j in range(3): + ws.cell(21, 3 + j, N) + return _run(flow_book(days=3, edit=edit, uncached=True), FLOW), [label, N] + + +def c_const_missing(mp): + data = recipe_dict() + data["sheets"][0]["blocks"][0]["segments"][2]["const"] = {"时段类别": {"pick": "晚上"}} + return _run(_flow(), data), ["夜间时段客流（人次）"] + + +def c_merged_in_values(mp): + def edit(ws, info): + ws["C12"] = N + ws.merge_cells("C12:D12") + return _run(_flow(edit), FLOW), [N] + + +def c_header_not_found(mp): + def edit(ws): + for c in range(1, 6): + ws.cell(3, c).value = f"列{N}{c}" + return _run(sales_book(edit=edit), LIST), [f"列{N}1", f"列{N}5"] + + +def c_header_ambiguous(mp): + def edit(ws): + for c, h in enumerate(("地区 ", "产品", "日期", "销量", "金额（万元）"), 1): + ws.cell(20, c, h) + ws["D21"] = N + return _run(sales_book(edit=edit), LIST), ["地区 ", N] + + +def c_column_missing(mp): + header = f"销售额{N}" + return _run(sales_book(edit=lambda ws: ws.cell(3, 5, header)), LIST), [header] + + +def c_column_extra(mp): + header = f"备注{N}" + def edit(ws): + ws["F3"] = header + ws["F4"] = N + return _run(sales_book(edit=edit), LIST), [header, N] + + +def c_rows_after_stop(mp): + text = f"名单{N}" + def edit(ws): + ws["A11"], ws["B11"] = text, "名单乙" + return _run(sales_book(total=False, edit=edit), list_recipe(rows={})), [text, "名单乙"] + + +def c_list_empty(mp): + """AU-1：只有表头、没有数据行。触发格是表头之下本该有数据的地方，这里放不了值；断言表头文字之外的特征不外露。""" + def edit(ws): + for row in ws.iter_rows(min_row=4): + for cell in row: + cell.value = None + return _run(sales_book(total=False, edit=edit), list_recipe(rows={})), [N] + + +def c_total_row_missing(mp): + def edit(ws): + ws["D4"] = N + return _run(sales_book(total=False, edit=edit), LIST), [N] + + +def c_total_row_blank(mp): + def edit(ws): + ws["D4"] = N + ws["D9"] = ws["E9"] = None # 第 9 行是合计行：标签还在，数字格全空 + return _run(sales_book(edit=edit), LIST), [N] + + +def c_value_not_integer(mp): + return _run(_flow(_set(C11=9876.54)), FLOW), [9876.54] + + +def c_value_text_number(mp): + return _run(_flow(_set(C11="987,654")), FLOW), ["987,654", N] + + +def c_value_blank(mp): + return _run(_flow(_set(C11=None, D11=N)), FLOW), [N] + + +def c_value_error(mp): + return _run(_flow(_set(C11="#DIV/0!")), FLOW), ["#DIV/0!"] + + +def c_value_not_number(mp): + return _run(_flow(_set(C11=f"约{N}")), FLOW), [f"约{N}", N] + + +def c_value_not_date(mp): + return _run(sales_book(edit=lambda ws: ws.cell(4, 3, f"{N}日")), LIST), [f"{N}日", N] + + +def c_value_formula(mp): + def edit(ws, info): + ws["C11"] = f"=C12+{N}" + info["cache"]["C11"] = N + 1 + return _run(_flow(edit), FLOW), [N, N + 1] + + +def c_formula_uncached(mp): + data = recipe_dict(sheets__0__blocks__0__values__formula="accept_cached") + return _run(_flow(_set(C11=f"=C12+{N}")), data), [N] + + +def c_formula_full_calc(mp): + data = recipe_dict(sheets__0__blocks__0__values__formula="accept_cached") + def edit(ws, info): + ws["C11"] = f"=C12+{N}" + info["cache"]["C11"] = N + 1 + return _run(flow_book(days=3, edit=edit, keep_full_calc=True), data), [N, N + 1] + + +def c_pk_duplicate(mp): + def edit(ws): + ws["A4"] = ws["A5"] = f"地区{N}" + ws["B5"] = ws["B4"].value + return _run(sales_book(edit=edit), LIST), [f"地区{N}", "产品000"] + + +CASES: dict[str, Case] = { + name[2:]: fn for name, fn in globals().items() if name.startswith("c_") and callable(fn) +} + + +def test_case_table_covers_every_problem_code(): + assert set(CASES) == set(recipe_engine.PROBLEM_CODES) + assert set(recipe_engine.PROBLEM_FIX) <= set(recipe_engine.PROBLEM_CODES) + + +def _forms(value: Any) -> list[str]: + if isinstance(value, float): + return [repr(value), str(value)] + return [str(value)] + + +@pytest.mark.parametrize("code", sorted(recipe_engine.PROBLEM_CODES)) +def test_model_message_is_filled_and_has_no_cell_values(code, monkeypatch): + ex, triggers = CASES[code](monkeypatch) + hits = [p for p in ex.problems if p.code == code] + assert hits, f"{code} 没有出现：{[(p.code, p.message) for p in ex.problems]}" + for p in ex.problems: # 同一夹具里顺带出现的别的 code 也要守这条 + assert p.model_message.strip(), (p.code, p.message) + assert p.category == recipe_engine.PROBLEM_CODES[p.code] + assert len(p.cells) <= recipe_engine.CELLS_MAX + for p in hits: + for value in triggers: + for form in _forms(value): + assert form not in p.model_message, (code, form, p.model_message) + assert p.message # 给人看的那句照样有 + + +def test_message_may_carry_values_but_model_message_does_not(): + ex, _ = c_value_not_number(None) + p = next(p for p in ex.problems if p.code == "value_not_number") + assert f"约{N}" in p.message and f"约{N}" not in p.model_message + assert "客流汇总!C11" in p.cells and "C11" in p.model_message + + +#: 下一步指到配方面板上真实存在的「字段」和「选项」（terms.ts：报错写界面上的中文标签）。check-copy 看不出这种 +#: 跨包的不一致，这里把引用的标签逐个钉住 +NEXT_STEPS = { + "value_text_number": "在配方面板的「文本写的数字」中选择「千分位写法按数字保存」", + "value_blank": "在配方面板的「空格」中选择「存为空值」", + "value_formula": "在配方面板的「公式」中选择「按保存值导入」", + "value_not_number": "在配方面板中添加占位符", + "value_not_integer": "在配方面板中把「类型」改为「小数」", + "value_not_date": "在配方面板中把这一列的「类型」改为「文字」", + "sheet_unexpected": "在配方面板的「配方没有列出的其他工作表」中选择「记入回执，需确认」", + "rows_after_stop": "在配方面板的「数据中间的空行」中选择「跳过继续」", + "hidden_rows": "在配方面板的「隐藏行」中选择「照常导入」或「跳过」", + "hidden_cols": "在配方面板的「隐藏列」中选择「照常导入」", + "total_row_missing": "在配方面板的「合计行」中取消「有合计行」", + "total_row_blank": "在配方面板的「合计行」中取消「有合计行」", +} + + +@pytest.mark.parametrize("code", sorted(NEXT_STEPS)) +def test_next_steps_name_recipe_panel_labels(code, monkeypatch): + ex, _ = CASES[code](monkeypatch) + p = next(p for p in ex.problems if p.code == code) + assert NEXT_STEPS[code] in p.message, p.message + + +def test_next_step_labels_exist_in_the_recipe_panel(): + """引用的每个标签都要在前端的配方面板文案里（RECIPE_TEXT / RECIPE_CHOICE_LABEL）。前端的配方面板合入之前跳过。""" + terms = Path(__file__).resolve().parents[2] / "frontend" / "src" / "lib" / "terms.ts" + text = terms.read_text(encoding="utf-8") if terms.exists() else "" + if "RECIPE_CHOICE_LABEL" not in text: + pytest.skip("前端的配方面板文案还没有合入") + labels = {x for step in NEXT_STEPS.values() for x in re.findall(r"「(.+?)」", step)} | {"主键"} + missing = sorted(x for x in labels if f"'{x}" not in text) + assert not missing, missing diff --git a/backend/tests/test_recipe_imports.py b/backend/tests/test_recipe_imports.py new file mode 100644 index 00000000..00f36f70 --- /dev/null +++ b/backend/tests/test_recipe_imports.py @@ -0,0 +1,1692 @@ +"""按配方导入的流程（recipe_imports，WP-5c）：用假 Pipeline 测状态机、提交事务、并发与失败回滚。 + +假 Pipeline 返回固定的扫描、网格、Extraction、核对和说明；试运行写一个真的 SQLite 小库（publish_build、 +探查、哈希核对都是真的）。各用例按需改 FakeState。接口层的错误码在 test_table_imports_api 里逐个打。 +夹具一律合成、假名；不调用任何真实模型（AI 起草换成桩，first_call_text 的一致性用 recipe_ai 自己的 +ai_draft 加假模型核对）。 +""" +from __future__ import annotations + +import asyncio +import copy +import dataclasses +import datetime as dt +import hashlib +import inspect +import json +import sqlite3 +import threading +import uuid +from collections import defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import func, select + +from app.core.config import settings +from app.data import ( + recipe, recipe_ai, recipe_imports, recipe_notes, recipe_parsers, recipe_types, table_versions, xlsx_cells, + xlsx_scan, +) +from app.data.recipe_ai import AiDraftResult, AiTooLarge, Compressed +from app.data.recipe_imports import Pipeline +from app.data.recipe_types import ( + REASON_SLOT, AiAvailability, AiUsage, Card, CheckResult, ColumnOut, ConfirmItem, DiffItem, Draft, DraftFacts, + Extraction, Grid, GridCell, PeriodOut, Problem, Question, Recipe, RecipeProblem, RegionMark, SchemaNotes, + TableNote, TableOut, +) +from app.data.xlsx_scan import Bounds, SheetScan, UnsupportedTable, WorkbookScan +from app.db.base import SessionLocal +from app.db.models import DataSource, ImportStaging, SourceSnapshot, TableImport, TableRecipe +from app.main import app + +FLOW_RECIPE = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text("utf-8")) +FLOW_FULL = Recipe.model_validate(FLOW_RECIPE).model_dump(mode="json") +XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + + +# ========================================================================== +# 假 Pipeline +# ========================================================================== + + +def ok_check(cid: str, kind: str = "row_count", category: str = "structure") -> CheckResult: + return CheckResult(id=cid, kind=kind, title=f"合成核对 {cid}", status="passed", category=category, checked=1) + + +QUESTIONS = [ + Question("q_placeholder:·", "「·」表示无数据吗", + [{"value": "null", "label": "是，存为空值", "needs_reason": False}, + {"value": "reject", "label": "不是，不导入", "needs_reason": False}], None, + {"null": [], "reject": [{"op": "remove", "path": "/sheets/0/blocks/0/values/placeholders/0"}]}), + Question("q_relation:F1", "第 5 行 = 第 6 行 + 第 7 行：要登记为每期核对吗", + [{"value": "register", "label": "登记", "needs_reason": False}, + {"value": "dismiss", "label": "不登记", "needs_reason": True}], None, + {"register": [{"op": "replace", "path": "/relations/0", "value": FLOW_FULL["relations"][0]}], + "dismiss": [{"op": "replace", "path": "/relations/0", + "value": {"id": "R1", "kind": "dismissed", "claims": "F1", "reason": REASON_SLOT}}]}), +] +FACTS = DraftFacts(candidates={"日间时段客流（人次）": ["日间", "日间时段客流"]}) + + +@dataclass +class FakeState: + complete: bool = True + recipe_problems: list[RecipeProblem] = field(default_factory=list) + draft_problems: list[Problem] = field(default_factory=list) + trial_problems: list[Problem] = field(default_factory=list) + needs_period: bool = False + checks: list[CheckResult] = field(default_factory=lambda: [ + ok_check("C1", "context_agree", "data_quality"), ok_check("R1", "relation_sum_eq", "data_quality"), + ok_check("N1")]) + notes_problems: list[str] = field(default_factory=list) + #: 只在带着接受（提交时）生成说明才报的问题：试运行能过、提交时 notes_invalid + notes_problems_on_accept: list[str] = field(default_factory=list) + #: 说明里多写一张库里没有的表：apply_notes 写不进表结构 + notes_extra_table: bool = False + diff: list[DiffItem] = field(default_factory=list) + confirm_extra: list[ConfirmItem] = field(default_factory=list) + availability: AiAvailability = field(default_factory=lambda: AiAvailability(True, model="假模型", + provider="假接入")) + compress_too_large: bool = False + #: 规则起草连半成品都没有（Draft.recipe=None）：暂存区没有工作配方 + no_recipe: bool = False + #: 测试所在事件循环的解析名额（execute 里记下当时还剩几个） + slot: Any = None + ai_result: Any = None + ai_hook: Any = None + rows: int = 2 + #: 让工作表大于网格上限：交互时只做部分干跑 + big_sheet: bool = False + calls: dict[str, list[Any]] = field(default_factory=lambda: defaultdict(list)) + + +def fake_scan(raw: bytes, state: FakeState) -> WorkbookScan: + if raw.startswith(b"bad"): + raise UnsupportedTable("无法读取这个文件：不是有效的 Excel 文件") + return WorkbookScan(sheets=[ + SheetScan(name="甲表", state="visible", path="xl/worksheets/sheet1.xml", bounds=Bounds(1, 1, 3, 2), + nonempty=10 ** 6 if state.big_sheet else 4, row_runs=[(1, 3, 2)]), + SheetScan(name="配置", state="hidden", path="xl/worksheets/sheet2.xml"), + ]) + + +def fake_grid(raw: bytes, scan: WorkbookScan, sheet: str, *, max_rows: int | None = None) -> Grid: + return Grid(sheet=sheet, bounds=(1, 1, 3, 2), cells={ + (1, 1): GridCell("区域"), (1, 2): GridCell("数量"), (2, 1): GridCell("分区甲"), (2, 2): GridCell(1)}) + + +def make_trial_db(path: str, rows: int, tag: str) -> None: + conn = sqlite3.connect(path) + try: + conn.execute('CREATE TABLE "t" ("n" INTEGER, "s" TEXT) STRICT') + conn.executemany('INSERT INTO "t" VALUES (?, ?)', [(i, f"{tag}-{i}") for i in range(rows)]) + conn.commit() + finally: + conn.close() + + +def fake_pipeline(state: FakeState) -> Pipeline: + def validate(data, *, facts=None, origin="manual"): + state.calls["validate"].append({"origin": origin, "facts": facts}) + try: + parsed = Recipe.model_validate(data) + except Exception: # noqa: BLE001 + return None, [RecipeProblem("/sheets", "schema", "第 1 个工作表：缺少必填项")] + return parsed, list(state.recipe_problems) + + def execute(rec, raw, filename, scan, db_path, *, context_inputs=None, max_rows=None): + state.calls["execute"].append({"db": db_path, "ctx": context_inputs, "max_rows": max_rows, + "filename": filename, "thread": threading.get_ident(), + "slots_free": state.slot._value if state.slot is not None else None}) + problems = list(state.trial_problems if db_path else state.draft_problems) + period = PeriodOut("2026-08-01", "2026-08-31", "cells", cells=["甲表!A1"], + texts={"甲表!A1": "统计时间范围：2026年8月1日至2026年8月31日"}) + if state.needs_period: + pi = (context_inputs or {}).get("统计期") + if pi is not None: + period = PeriodOut(pi.start.isoformat(), pi.end.isoformat(), "human", signed_by=pi.signed_by) + else: + period = None + problems.append(Problem("period_missing", "input", "未能从表格中解析出统计期，请为本期录入统计期", + model_message="未能解析出统计期")) + blocking = any(p.category in ("structure", "input", "recipe") for p in problems) + ex = Extraction( + ok=not blocking, problems=problems, partial=max_rows is not None, + tables=[TableOut("t", "甲表", "data", [ColumnOut("n", "INTEGER", role="measure"), + ColumnOut("s", "TEXT", role="text")], [], state.rows)], + regions=[RegionMark("甲表", "value", "A2:B3")], period=period, + context_checks=[ok_check("C1", "context_agree", "data_quality")], expected_rows={"t": state.rows}, + table_hashes={"t": "0" * 64}, + ) + if db_path and not blocking: + make_trial_db(db_path, state.rows, hashlib.sha256(raw).hexdigest()[:8]) + return ex + + def run_checks(db_path, ex, rec): + state.calls["run_checks"].append(db_path) + assert Path(db_path).is_file() + return copy.deepcopy(state.checks) + + def null_counts(db_path, rec): + state.calls["null_counts"].append(db_path) + return {"t": {"n": 0, "s": 0}} + + def build_notes(rec, ex, checks, acceptances, *, null_counts=None): + state.calls["build_notes"].append({"acceptances": list(acceptances), "null_counts": null_counts}) + tables = {"t": TableNote(comment="合成说明", columns={"n": "单位：人次"})} + if state.notes_extra_table: + tables["缺表"] = TableNote(comment="合成说明") + problems = list(state.notes_problems) + (list(state.notes_problems_on_accept) if acceptances else []) + return SchemaNotes(tables=tables, templates={"t": "合成说明"}, problems=problems) + + def detect_facts(grids, scan, *, recipe=None): + state.calls["detect_facts"].append(recipe) + return copy.deepcopy(FACTS) + + def draft(scan, grids, filename, *, validate, dry_run): + state.calls["draft"].append(filename) + rec = copy.deepcopy(FLOW_FULL) + parsed, problems = validate(rec, FACTS, "rules") + if parsed is not None and not problems and dry_run is not None: + dry_run(parsed) + failures = [] if state.complete else ["第 9 行像合计行，但标签无法确定区间"] + if state.no_recipe: + rec = None + return Draft(recipe=rec, complete=state.complete, origin="rules", cards=[Card("mode", "导入模式：每期替换", "合成")], + questions=copy.deepcopy(QUESTIONS), facts=copy.deepcopy(FACTS), failures=failures, + failures_for_model=["第 9 行像合计行"] if failures else []) + + def questions_for(rec, facts, grids, *, extraction=None): + state.calls["questions_for"].append({"recipe": rec, "extraction": extraction}) + return [Card("mode", "导入模式：每期替换", "合成")], copy.deepcopy(QUESTIONS) + + async def ai_availability(session): + state.calls["ai_availability"].append(True) + return state.availability + + def compress(grids, scan, facts, *, mask_headers=frozenset()): + state.calls["compress"].append(set(mask_headers)) + if state.compress_too_large: + raise AiTooLarge("表格结构压缩后有 30000 字，超过上限 20000 字，无法交给 AI 起草") + text = "## 工作表「甲表」 已用区域 A1:B3，非空格 4" + return Compressed(text=text, chars=len(text), truncated=False, + sha256=hashlib.sha256(text.encode()).hexdigest()) + + def first_call_text(compressed, *, grids, scan, rules_draft): + tail = "；".join(rules_draft.failures_for_model) if rules_draft else "无" + return "系统提示：只输出 JSON\n\n" + compressed.text + "\n\n# 单位词表\n人次\n\n# 规则起草的半成品\n" + tail + + async def ai_draft(session, *, grids, scan, facts, rules_draft, filename, validate, dry_run, compressed): + state.calls["ai_draft"].append({"compressed": compressed, "rules": rules_draft}) + if state.ai_hook is not None: + await state.ai_hook() + if state.ai_result is not None: + return state.ai_result + rec = copy.deepcopy(FLOW_FULL) + rec["other_visible_sheets"] = "reject" + parsed, problems = validate(rec, facts, "ai") + state.calls["ai_dry_is_coroutine"].append(inspect.iscoroutinefunction(dry_run)) + ext = await dry_run(parsed) + d = Draft(recipe=rec, complete=True, origin="ai", + cards=[Card("ai_origin", "AI 起草：请逐项核对", "合成"), Card("mode", "导入模式：每期替换", "合成")], + questions=copy.deepcopy(QUESTIONS), facts=facts) + state.calls["ai_dry"].append(ext) + usage = [AiUsage("假模型", 120, 80, 0.0012, True, "2026-10-01T00:00:00+00:00", 1)] + return AiDraftResult(d, usage, 1, None, compressed.chars, compressed.sha256) + + def diff_reports(prev, cur, *, prev_file_name, cur_file_name, recipe=None, recipe_changed=None): + state.calls["diff_reports"].append({"prev": prev, "cur": cur, "prev_file_name": prev_file_name, + "cur_file_name": cur_file_name, "recipe": recipe, + "recipe_changed": recipe_changed}) + return copy.deepcopy(state.diff) + + def confirm_items(ctx): + state.calls["confirm_items"].append(ctx) + items = [ConfirmItem("mode", "导入模式：每期替换")] + if ctx.recipe_origin in ("ai", "mixed"): + items.append(ConfirmItem("ai_origin", "本配方由 AI 起草，已逐项核对")) + items += copy.deepcopy(state.confirm_extra) + for d in ctx.diff or []: + d = d if isinstance(d, dict) else dataclasses.asdict(d) + if d.get("requires_confirm"): + items.append(ConfirmItem(d["confirm_id"], d["label"], source="diff")) + return items + + return Pipeline( + validate=validate, recipe_sha256=recipe.recipe_sha256, canonical=recipe.canonical_recipe, + apply_patch=recipe.apply_patch, scan=lambda raw: fake_scan(raw, state), + scan_to_json=xlsx_scan.scan_to_json, scan_from_json=xlsx_scan.scan_from_json, read_grid=fake_grid, + preview=xlsx_cells.preview, execute=execute, run_checks=run_checks, column_null_counts=null_counts, + build_notes=build_notes, apply_notes=recipe_notes.apply_notes, detect_facts=detect_facts, draft=draft, + questions_for=questions_for, ai_availability=ai_availability, ai_draft=ai_draft, compress=compress, + first_call_text=first_call_text, diff_reports=diff_reports, confirm_items=confirm_items, + breaking_changes=lambda old, new: {}, grid_max_cells=1000, draft_max_rows=500, units=("人次", "元"), + ) + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + """每个测试一个独立的数据目录：上传目录、原件存档、工件都在里面。""" + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +@pytest.fixture +def fake(monkeypatch) -> FakeState: + state = FakeState() + pipe = fake_pipeline(state) + monkeypatch.setattr(recipe_imports, "pipeline", lambda: pipe) + return state + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def unique(prefix: str = "wp5c") -> str: + return f"{prefix}_{uuid.uuid4().hex[:10]}" + + +def raw_bytes() -> bytes: + """内容独一份的「原件」：原件按内容共用，别的测试传过同样字节就会互相影响。""" + return f"PK 合成原件 {uuid.uuid4().hex}".encode() + + +async def get(model, key): + async with SessionLocal() as session: + return await session.get(model, key) + + +async def api_stage(client, name: str, raw: bytes | None = None, filename: str = "月报导出_2026-08-01_2026-08-31.xlsx"): + return await client.post("/api/datasources/imports/stage", + files={"file": (filename, raw if raw is not None else raw_bytes(), XLSX)}, + data={"name": name}) + + +async def staged(client, name: str | None = None, **kw) -> dict[str, Any]: + resp = await api_stage(client, name or unique(), **kw) + assert resp.status_code == 201, resp.text + return resp.json() + + +async def trialed(client, sid: str, **body) -> dict[str, Any]: + resp = await client.post(f"/api/datasources/imports/{sid}/trial", json=body) + assert resp.status_code == 200, resp.text + return resp.json() + + +async def committed(client, st: dict[str, Any], **body) -> dict[str, Any]: + body.setdefault("confirmations", [i["id"] for i in st["trial"]["confirm_items"]]) + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], **body}) + assert resp.status_code == 201, resp.text + return resp.json() + + +async def recipe_source(client, name: str | None = None) -> dict[str, Any]: + """用假管线建一个按配方导入的源：暂存 → 试运行 → 提交。返回提交的响应。""" + st = await staged(client, name) + st = await trialed(client, st["id"]) + return await committed(client, st) + + +# ========================================================================== +# 纯函数 +# ========================================================================== + + +def test_replay_answers_choosing_a_then_b_equals_choosing_b(): + p = recipe_imports.default_pipeline() + base = copy.deepcopy(FLOW_FULL) + a = recipe_imports.replay_answers(p, base, QUESTIONS, {"q_relation:F1": {"value": "dismiss", "reason": "巧合"}}) + assert a["relations"][0] == {"id": "R1", "kind": "dismissed", "claims": "F1", "reason": "巧合"} + b_direct = recipe_imports.replay_answers(p, base, QUESTIONS, {"q_relation:F1": {"value": "register"}}) + # 在 A 的结果上不会再叠一次：重放总是从起点开始 + b_after_a = recipe_imports.replay_answers(p, base, QUESTIONS, {"q_relation:F1": {"value": "register"}}) + assert b_direct == b_after_a == base + both = recipe_imports.replay_answers(p, base, QUESTIONS, {"q_placeholder:·": {"value": "reject"}, + "q_relation:F1": {"value": "register"}}) + assert both["sheets"][0]["blocks"][0]["values"]["placeholders"] == [] + assert base == FLOW_FULL, "起点配方不被修改" + + +def test_replay_answers_reports_effects_that_do_not_apply(): + p = recipe_imports.default_pipeline() + broken = copy.deepcopy(FLOW_FULL) + broken["relations"] = [] + with pytest.raises(ValueError): + recipe_imports.replay_answers(p, broken, QUESTIONS, {"q_relation:F1": {"value": "register"}}) + + +@pytest.mark.parametrize(("problems", "checks", "notes", "expected"), [ + ([], [ok_check("N1")], [], "passed"), + ([Problem("label_missing", "structure", "合成")], [], [], "rejected"), + ([], [], ["说明中含有数字「3」"], "rejected"), + ([], [CheckResult("K1", "derived_sum", "合计", "mismatch", "structure")], [], "rejected"), + ([], [CheckResult("K1", "derived_sum", "合计", "unverifiable", "structure", acceptable=True)], [], "needs_decision"), + ([], [CheckResult("R1", "relation_sum_eq", "关系", "mismatch", "data_quality", acceptable=True)], [], + "needs_decision"), + ([Problem("period_missing", "input", "合成")], [], [], "needs_input"), + ([Problem("outside_digits", "confirm", "合成")], [CheckResult("R2", "relation_not_comparable", "口径", "info", "info")], + [], "passed"), +]) +def test_trial_status(problems, checks, notes, expected): + assert recipe_imports.trial_status(problems, checks, notes) == expected + + +def test_extraction_round_trip_through_json(): + from tests.fixtures.xlsx.flow import flow_workbook + + raw, fn = flow_workbook(dt.date(2026, 8, 1), 31, seed=5) + scan = xlsx_scan.scan(raw) + parsed = Recipe.model_validate(FLOW_RECIPE) + from app.data import recipe_engine + + ex = recipe_engine.execute(parsed, raw, fn, scan, None) + data = json.loads(json.dumps(dataclasses.asdict(ex), ensure_ascii=False, default=str)) + back = recipe_imports.extraction_from_dict(data) + assert json.loads(json.dumps(dataclasses.asdict(back), ensure_ascii=False, default=str)) == data + assert isinstance(back.period, PeriodOut) and isinstance(back.tables[0], TableOut) + assert isinstance(back.tables[0].columns[0], ColumnOut) + + +async def test_first_call_text_is_exactly_what_ai_draft_sends_first(): + """同意框展示的全文 = ai_draft 第一次调用的系统提示 + 首次提示词（含单位词表、规则半成品、格式说明）。""" + from langchain_core.messages import AIMessage + + from tests.fixtures.xlsx.lab import kv_form + + raw, fn = kv_form() + scan = xlsx_scan.scan(raw) + grids = {s.name: xlsx_cells.read_grid(raw, scan, s.name, max_rows=500) for s in scan.sheets + if s.state == "visible" and s.nonempty} + from app.data import recipe_suggest + + facts = recipe_suggest.detect_facts(grids, scan) + rules = recipe_suggest.draft(scan, grids, fn, + validate=lambda d, f, o: recipe.validate_recipe(d, facts=f, origin=o)) + compressed = recipe_ai.compress(grids, scan, facts) + text = recipe_ai.first_call_text(compressed, grids=grids, scan=scan, rules_draft=rules) + + class Model: + calls: list = [] + + async def ainvoke(self, messages): + self.calls.append(list(messages)) + return AIMessage(content="不是 JSON", usage_metadata={"input_tokens": 1, "output_tokens": 1, + "total_tokens": 2}) + + model = Model() + result = await recipe_ai.ai_draft(None, grids=grids, scan=scan, facts=facts, rules_draft=rules, filename=fn, + validate=lambda d, f, o: recipe.validate_recipe(d, facts=f, origin=o), + dry_run=None, model=model, compressed=compressed) + assert result.draft is None + first = model.calls[0] + assert text == first[0].content + "\n\n" + first[1].content + assert compressed.text in text and "# 单位词表" in text and "# 配方 JSON Schema" in text + assert "# 规则起草的半成品与失败原因" in text + + +def test_effective_origin(): + st = ImportStaging(recipe_origin="manual", drafts={"ai_adopted": {"draft": {"recipe": {}}}}) + assert recipe_imports.effective_origin(st, None) == "mixed" + st = ImportStaging(recipe_origin="manual", drafts={"rules": {}}) + assert recipe_imports.effective_origin(st, None) == "manual" + assert recipe_imports.effective_origin(st, "ai") == "mixed" + st = ImportStaging(recipe_origin="rules", drafts={}) + assert recipe_imports.effective_origin(st, None) == "rules" + + +# ========================================================================== +# 暂存与回答 +# ========================================================================== + + +async def test_stage_first_does_not_create_the_source_and_keeps_the_raw(client, fake, store): + name = unique() + st = await staged(client, name) + assert st["kind"] == "first" and st["status"] == "drafting" and st["recipe_origin"] == "rules" + assert st["source"] == {"id": st["source"]["id"], "name": name, "exists": False, "import_mode": None} + assert st["skipped_sheets"] == [{"sheet": "配置", "state": "hidden"}] + assert st["sheets"][0]["name"] == "甲表" and st["sheets"][0]["bounds"] == "A1:B3" + assert st["grids"][0]["cells"] and st["units"] == ["人次", "元"] + assert st["candidates"] == FACTS.candidates + assert [q["id"] for q in st["questions"]] == [q.id for q in QUESTIONS] + assert st["marks"] == [{"sheet": "甲表", "role": "value", "ref": "A2:B3"}] + assert st["expires_at"] and st["file"]["sha256_prefix"] + assert await get(DataSource, st["source"]["id"]) is None + row = await get(ImportStaging, st["id"]) + assert (store / "uploads" / "raw" / row.raw_sha256[:2] / row.raw_sha256).is_file() + assert row.answers_base == FLOW_FULL and row.recipe == recipe.canonical_recipe(Recipe.model_validate(FLOW_FULL)) + + +async def test_answers_replay_from_base_and_reset_by_put(client, fake): + st = await staged(client) + sid = st["id"] + resp = await client.post(f"/api/datasources/imports/{sid}/answers", json={"answers": { + "q_relation:F1": {"value": "dismiss", "reason": "合成理由"}, "q_placeholder:·": {"value": "reject"}}}) + assert resp.status_code == 200, resp.text + a = resp.json() + assert a["recipe"]["relations"][0]["kind"] == "dismissed" and a["recipe"]["relations"][0]["reason"] == "合成理由" + assert a["recipe"]["sheets"][0]["blocks"][0]["values"]["placeholders"] == [] + assert a["answers"]["q_relation:F1"] == {"value": "dismiss", "reason": "合成理由"} + # 改选 B：等于直接选 B + resp = await client.post(f"/api/datasources/imports/{sid}/answers", json={"answers": { + "q_relation:F1": {"value": "register"}}}) + b = resp.json() + assert b["recipe"] == FLOW_FULL and b["answers"] == {"q_relation:F1": {"value": "register", "reason": None}} + # 每次回答前按工作配方取系统发现 + assert fake.calls["detect_facts"][-1] == FLOW_FULL + # PUT 之后回答清空、起点换掉 + changed = copy.deepcopy(FLOW_FULL) + changed["tables"][0]["note"] = "口径说明" + resp = await client.put(f"/api/datasources/imports/{sid}/recipe", json={"recipe": changed}) + put = resp.json() + assert put["answers"] == {} and put["recipe"]["tables"][0]["note"] == "口径说明" + assert put["recipe_origin"] == "manual" + row = await get(ImportStaging, sid) + assert row.answers_base["tables"][0]["note"] == "口径说明" + + +@pytest.mark.parametrize(("answers", "why"), [ + ({"q_nope": {"value": "x"}}, "问题不存在"), + ({"q_relation:F1": {"value": "maybe"}}, "选项不存在"), + ({"q_relation:F1": {"value": "dismiss"}}, "needs_reason 没给理由"), + ({"q_relation:F1": {"value": "dismiss", "reason": " "}}, "理由是空白"), + ({"q_relation:F1": {"value": "dismiss", "reason": "长" * 201}}, "理由超过 200 字"), + ({"q_relation:F1": "register"}, "回答不是对象"), +]) +async def test_answer_invalid(client, fake, answers, why): + st = await staged(client) + resp = await client.post(f"/api/datasources/imports/{st['id']}/answers", json={"answers": answers}) + assert resp.status_code == 422, why + assert resp.json()["code"] == "answer_invalid" + + +async def test_partial_dry_run_for_big_sheets(client, fake): + fake.big_sheet = True + st = await staged(client) + assert st["draft_partial"] is True + assert all(c["max_rows"] == 500 for c in fake.calls["execute"] if c["db"] is None) + resp = await client.post(f"/api/datasources/imports/{st['id']}/answers", + json={"answers": {"q_relation:F1": {"value": "register"}}}) + assert resp.json()["draft_partial"] is True and fake.calls["execute"][-1]["max_rows"] == 500 + # 试运行一律完整执行 + st = await trialed(client, st["id"]) + assert fake.calls["execute"][-1]["max_rows"] is None and fake.calls["execute"][-1]["db"] + + +# ========================================================================== +# 试运行 +# ========================================================================== + + +async def test_trial_stores_extraction_checks_and_null_counts(client, fake): + st = await staged(client) + st = await trialed(client, st["id"]) + trial = st["trial"] + assert st["status"] == "trialed" and trial["status"] == "passed" + assert trial["receipt"]["db_sha256"] and trial["receipt"]["canonicalized_total"] == 0 + assert set(recipe_imports.TRIAL_INTERNAL).isdisjoint(trial) + assert trial["notes"] == {"t": {"comment": "合成说明", "columns": {"n": "单位：人次"}}} + assert trial["confirm_items"] == [{"id": "mode", "label": "导入模式：每期替换", "detail": "", "required": True, + "source": "recipe"}] + assert trial["diff"] is None + # 试运行预览的说明拿到了各列的空值数 + assert fake.calls["build_notes"][-1]["null_counts"] == {"t": {"n": 0, "s": 0}} + assert fake.calls["null_counts"] + row = await get(ImportStaging, st["id"]) + assert row.trial["extraction"]["tables"][0]["name"] == "t" and row.trial["null_counts"] + assert Path(row.trial_path).is_file() and row.trial_path.endswith(f"{row.trial_key[:16]}.db") + ctx = fake.calls["confirm_items"][-1] + assert ctx.kind == "first" and ctx.prev is None and ctx.diff is None and ctx.old_recipe is None + assert isinstance(ctx.extraction, dict) and "problems" in ctx.extraction + + +async def test_retrial_replaces_the_old_trial_db(client, fake): + st = await staged(client) + await trialed(client, st["id"]) + first = (await get(ImportStaging, st["id"])).trial_path + resp = await client.put(f"/api/datasources/imports/{st['id']}/recipe", + json={"recipe": {**FLOW_FULL, "other_visible_sheets": "reject"}}) + assert resp.json()["status"] == "drafting" + assert not Path(first).exists(), "配方变了：旧试运行库作废并删掉" + await trialed(client, st["id"]) + second = (await get(ImportStaging, st["id"])).trial_path + assert second != first and Path(second).is_file() + + +async def test_trial_rejected_keeps_no_db_and_commit_refuses(client, fake): + fake.trial_problems = [Problem("label_missing", "structure", "分段「日间」少了标签「7-8」", fix="remove_label")] + st = await staged(client) + st = await trialed(client, st["id"]) + assert st["status"] == "rejected" and st["trial"]["status"] == "rejected" + assert st["trial"]["problems"][0]["code"] == "label_missing" + assert st["trial"]["confirm_items"] == [] and st["trial"]["diff"] is None + assert (await get(ImportStaging, st["id"])).trial_path is None + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": []}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_not_passed" + + +async def test_notes_problems_reject_the_trial(client, fake): + fake.notes_problems = ["表「t」的说明：说明中含有数字「3」"] + st = await trialed(client, (await staged(client))["id"]) + assert st["trial"]["status"] == "rejected" + assert any(p["code"] == "notes_invalid" and p["category"] == "recipe" for p in st["trial"]["problems"]) + + +async def test_needs_input_then_human_period_changes_the_build(client, fake): + fake.needs_period = True + st = await staged(client) + sid = st["id"] + # 第一次试运行就给录入：上一次没有要求录入 → 422 + resp = await client.post(f"/api/datasources/imports/{sid}/trial", + json={"context_inputs": {"统计期": {"start": "2026-08-01", "end": "2026-08-31"}}}) + assert resp.status_code == 422 and resp.json()["code"] == "context_not_needed" + st = await trialed(client, sid) + assert st["trial"]["status"] == "needs_input" and st["status"] == "trialed" + resp = await client.post(f"/api/datasources/imports/{sid}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_not_passed" + for bad in ({"统计期": {"start": "2026-02-30", "end": "2026-03-01"}}, + {"统计期": {"start": "2026-09-30", "end": "2026-09-01"}}, {"别的": {"start": "2026-09-01"}}): + resp = await client.post(f"/api/datasources/imports/{sid}/trial", json={"context_inputs": bad}) + assert resp.status_code == 422 and resp.json()["code"] == "context_invalid" + st = await trialed(client, sid, context_inputs={"统计期": {"start": "2026-08-01", "end": "2026-08-31"}}, + signed_by="录入员") + assert st["trial"]["status"] == "passed" + assert st["context_inputs"] == {"统计期": {"start": "2026-08-01", "end": "2026-08-31", "signed_by": "录入员"}} + row = await get(ImportStaging, sid) + assert row.trial["inputs"] == {"context": {"统计期": {"start": "2026-08-01", "end": "2026-08-31"}}} + out = await committed(client, st) + imp = await get(TableImport, out["import_id"]) + assert imp.context["source"] == "human" and imp.context["signed_by"] == "录入员" + without = table_versions.recipe_build_id(out["source"]["id"], row.raw_sha256, row.recipe_sha256, {}) + assert out["build_id"] != without, "人工录入计入构建哈希" + + +# ========================================================================== +# 提交 +# ========================================================================== + + +async def test_commit_writes_everything_in_one_go(client, fake): + name = unique() + st = await trialed(client, (await staged(client, name))["id"]) + out = await committed(client, st, signed_by="测试员") + source = await get(DataSource, out["source"]["id"]) + assert source.name == name and source.origin == "upload" and source.current_recipe_id == out["recipe_id"] + assert source.current_snapshot_id == out["snapshot_id"] + snap = await get(SourceSnapshot, out["snapshot_id"]) + imp = await get(TableImport, out["import_id"]) + rec = await get(TableRecipe, out["recipe_id"]) + staging = await get(ImportStaging, st["id"]) + assert snap.schema_cache["import_manifests"] == [imp.manifest_artifact] and imp.manifest_artifact + assert snap.schema_cache["tables"]["t"]["comment"] == "合成说明" + assert imp.recipe_id == rec.id and imp.staging_id == st["id"] and imp.signed_by == "测试员" + assert imp.checks[0]["id"] == "C1" and imp.overrides == [] and imp.waivers == [] + assert [c["id"] for c in imp.confirmations] == ["mode"] + assert rec.origin == "rules" and rec.seq == 1 and rec.recipe == staging.recipe + assert staging.status == "committed" and staging.committed_import_id == imp.id + assert staging.scan is None and staging.trial is None and staging.trial_path is None + from app.core import artifact_store + + manifest = artifact_store.load(imp.manifest_artifact) + assert manifest["import_id"] == imp.id and manifest["snapshot_id"] == snap.id + assert manifest["db_sha256"] == snap.db_sha256 and manifest["recipe"]["sha256"] == rec.recipe_sha256 + assert "problems" not in manifest["receipt"] and "axes" in manifest["receipt"] + # 提交时说明拿到的是实际的接受（这里没有）和空值数 + assert fake.calls["build_notes"][-1] == {"acceptances": [], "null_counts": {"t": {"n": 0, "s": 0}}} + + +async def test_commit_requires_every_confirm_item(client, fake): + fake.confirm_extra = [ConfirmItem("outside_digits:甲表!B9", "区域外有含数字的文字「注：9月15日…」", + source="outside")] + st = await trialed(client, (await staged(client))["id"]) + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" + assert "outside_digits:甲表!B9" in resp.json()["detail"] + await committed(client, st) + + +async def test_commit_requires_reasons_and_records_overrides_and_waivers(client, fake): + fake.checks = [ok_check("C1", "context_agree", "data_quality"), + CheckResult("R1", "relation_sum_eq", "全日客流 = 分区甲 + 分区乙", "mismatch", "data_quality", + checked=31, failed=1, acceptable=True), + CheckResult("K1", "derived_sum", "合计", "unverifiable", "structure", checked=93, + unverifiable=3, acceptable=True, reasons={"uncached": 3})] + st = await trialed(client, (await staged(client))["id"]) + assert st["trial"]["status"] == "needs_decision" and st["trial"]["acceptable"] == ["R1", "K1"] + url = f"/api/datasources/imports/{st['id']}/commit" + base = {"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]} + resp = await client.post(url, json=base) + assert resp.status_code == 422 and resp.json()["code"] == "acceptance_required" + resp = await client.post(url, json={**base, "acceptances": [{"check_id": "R1", "reason": "第 10 日为补录"}, + {"check_id": "K1", "reason": " "}]}) + assert resp.status_code == 422 and resp.json()["code"] == "acceptance_required" and "K1" in resp.json()["detail"] + out = await committed(client, st, acceptances=[{"check_id": "R1", "reason": "第 10 日为补录"}, + {"check_id": "K1", "reason": "原表未经计算保存"}], + signed_by="测试员") + imp = await get(TableImport, out["import_id"]) + assert [(o["check_id"], o["reason"]) for o in imp.overrides] == [("R1", "第 10 日为补录")] + assert [(w["check_id"], w["reason"]) for w in imp.waivers] == [("K1", "原表未经计算保存")] + assert {a.check_id for a in fake.calls["build_notes"][-1]["acceptances"]} == {"R1", "K1"} + + +async def test_trial_key_invalidated_by_recipe_change(client, fake): + st = await trialed(client, (await staged(client))["id"]) + old_trial = st["trial"] + resp = await client.post(f"/api/datasources/imports/{st['id']}/answers", + json={"answers": {"q_placeholder:·": {"value": "reject"}}}) + after = resp.json() + assert after["status"] == "drafting" and after["trial"]["trial_id"] == old_trial["trial_id"], "回执留着标「已失效」" + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": old_trial["trial_id"], "confirmations": ["mode"]}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_required" + # 选回原样：配方哈希回到试运行时的值，但试运行库已作废，仍要重新试运行 + await client.post(f"/api/datasources/imports/{st['id']}/answers", json={"answers": {}}) + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": old_trial["trial_id"], "confirmations": ["mode"]}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_required" + + +async def test_commit_with_wrong_trial_id_or_missing_db(client, fake): + st = await trialed(client, (await staged(client))["id"]) + url = f"/api/datasources/imports/{st['id']}/commit" + resp = await client.post(url, json={"trial_id": "0" * 16, "confirmations": ["mode"]}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_required" + row = await get(ImportStaging, st["id"]) + Path(row.trial_path).unlink() + resp = await client.post(url, json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_required" + row = await get(ImportStaging, st["id"]) + assert row.status == "drafting" and row.trial_path is None and row.trial_key is None and row.trial + + +async def test_failed_publish_leaves_pointer_and_returns_staging_to_drafting(client, fake, monkeypatch): + done = await recipe_source(client) + source_id = done["source"]["id"] + resp = await client.post(f"/api/datasources/{source_id}/reupload", + files={"file": ("下月.xlsx", raw_bytes(), XLSX)}) + st = resp.json() + assert st["trial"]["status"] == "passed" + + def boom(cache, report, sid): + raise table_versions.PublishError("导入失败：新数据的表结构与解析结果对不上，已停止发布，当前版本未受影响") + + monkeypatch.setattr(table_versions, "check_probe", boom) + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + assert resp.status_code == 500 and resp.json()["code"] == "publish_failed" + source = await get(DataSource, source_id) + assert source.current_snapshot_id == done["snapshot_id"] + row = await get(ImportStaging, st["id"]) + assert row.status == "drafting" and row.trial_path is None and row.trial_key is None and row.trial + async with SessionLocal() as session: + n = len(list((await session.execute(select(TableImport).where(TableImport.source_id == source_id))).scalars())) + assert n == 1 + + +async def test_failed_first_commit_does_not_create_the_source(client, fake, monkeypatch): + st = await trialed(client, (await staged(client))["id"]) + + def boom(cache, report, sid): + raise table_versions.PublishError("导入失败：合成的探查错误，当前版本未受影响") + + monkeypatch.setattr(table_versions, "check_probe", boom) + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + assert resp.status_code == 500 + assert await get(DataSource, st["source"]["id"]) is None + + +async def test_tampered_trial_db_is_refused(client, fake): + st = await trialed(client, (await staged(client))["id"]) + row = await get(ImportStaging, st["id"]) + conn = sqlite3.connect(row.trial_path) + conn.execute('INSERT INTO "t" VALUES (999, "改过")') + conn.commit() + conn.close() + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_tampered" + assert (await get(ImportStaging, st["id"])).status == "drafting" + + +async def test_base_changed_is_judged_inside_the_lock(client, fake): + """两个上传新一期都试运行通过；两个提交都过了锁外预检、在锁上排队，先拿到锁的 201，另一个 409。""" + done = await recipe_source(client) + source_id = done["source"]["id"] + a = (await client.post(f"/api/datasources/{source_id}/reupload", + files={"file": ("甲.xlsx", raw_bytes(), XLSX)})).json() + b = (await client.post(f"/api/datasources/{source_id}/reupload", + files={"file": ("乙.xlsx", raw_bytes(), XLSX)})).json() + assert a["trial"]["status"] == b["trial"]["status"] == "passed" + + async def commit(st): + return await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + + lock = table_versions.store_lock() + await lock.acquire() + try: + tasks = [asyncio.create_task(commit(a)), asyncio.create_task(commit(b))] + # 等两个提交都排到锁上（预检都已通过） + for _ in range(200): + await asyncio.sleep(0.01) + if len(getattr(lock, "_waiters", None) or []) >= 2: + break + assert len(lock._waiters) >= 2 + finally: + lock.release() + results = await asyncio.gather(*tasks) + codes = sorted(r.status_code for r in results) + assert codes == [201, 409], [r.text for r in results] + loser = next(r for r in results if r.status_code == 409) + assert loser.json()["code"] == "base_changed" + winner = next(r for r in results if r.status_code == 201).json() + assert (await get(DataSource, source_id)).current_snapshot_id == winner["snapshot_id"] + + +async def test_same_new_name_committed_concurrently(client, fake): + name = unique() + a = await trialed(client, (await staged(client, name))["id"]) + b = await trialed(client, (await staged(client, name))["id"]) + assert a["source"]["id"] != b["source"]["id"] + + async def commit(st): + return await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + + lock = table_versions.store_lock() + await lock.acquire() + try: + tasks = [asyncio.create_task(commit(a)), asyncio.create_task(commit(b))] + for _ in range(200): + await asyncio.sleep(0.01) + if len(getattr(lock, "_waiters", None) or []) >= 2: + break + finally: + lock.release() + results = await asyncio.gather(*tasks) + assert sorted(r.status_code for r in results) == [201, 409], [r.text for r in results] + assert next(r for r in results if r.status_code == 409).json()["code"] == "name_taken" + async with SessionLocal() as session: + rows = list((await session.execute(select(DataSource).where(DataSource.name == name))).scalars()) + assert len(rows) == 1 + + +async def _wait_on_lock(lock: asyncio.Lock, n: int) -> None: + """等 n 个协程排到版本存储锁上（它们都已过了锁外预检）。""" + for _ in range(300): + await asyncio.sleep(0.01) + if len(getattr(lock, "_waiters", None) or []) >= n: + return + + +async def test_name_taken_is_judged_inside_the_lock(client, fake, monkeypatch): + """锁外预检之后、进锁之前别处建了同名源：锁内的检查拦下（CommitConflict），不靠 UNIQUE 约束兜底。""" + name = unique() + st = await trialed(client, (await staged(client, name))["id"]) + seen: list[BaseException] = [] + real = table_versions.publish_build + + async def spy(*a, **kw): + try: + return await real(*a, **kw) + except BaseException as e: + seen.append(e) + raise + + monkeypatch.setattr(table_versions, "publish_build", spy) + lock = table_versions.store_lock() + await lock.acquire() + try: + task = asyncio.create_task(client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]})) + await _wait_on_lock(lock, 1) + assert len(lock._waiters) >= 1 + async with SessionLocal() as session: + session.add(DataSource(name=name, kind="sqlite", database="/tmp/合成.db")) + await session.commit() + finally: + lock.release() + resp = await task + assert resp.status_code == 409 and resp.json()["code"] == "name_taken", resp.text + assert [type(e) for e in seen] == [recipe_imports.CommitConflict], seen + assert await get(DataSource, st["source"]["id"]) is None + assert (await get(ImportStaging, st["id"])).status == "drafting" + + +async def test_name_taken_falls_back_on_the_unique_constraint(client, fake, monkeypatch): + """锁内检查之后才冒出来的同名源（这里在 before_commit 之后、同一事务里插一条）：UNIQUE 约束的 + IntegrityError 兜底转成 409 name_taken，不是 500。""" + name = unique() + st = await trialed(client, (await staged(client, name))["id"]) + real = table_versions.publish_build + + async def sneaky(session, source, **kw): + inner = kw["before_commit"] + + async def before_commit(imp_id, snap_id): + await inner(imp_id, snap_id) + session.add(DataSource(name=name, kind="sqlite", database="/tmp/合成.db")) + + return await real(session, source, **{**kw, "before_commit": before_commit}) + + monkeypatch.setattr(table_versions, "publish_build", sneaky) + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + assert resp.status_code == 409 and resp.json()["code"] == "name_taken", resp.text + assert await get(DataSource, st["source"]["id"]) is None + async with SessionLocal() as session: + assert (await session.execute(select(DataSource).where(DataSource.name == name))).first() is None + assert (await get(ImportStaging, st["id"])).status == "drafting" + + +async def test_commit_after_putting_a_schema_broken_recipe_is_trial_required(client, fake): + """试运行后 PUT 一份 schema 不过的配方（照存、试运行作废但回执留着），拿旧的 trial_id 提交： + 409 trial_required，不是 pydantic 英文原文的 400。""" + st = await trialed(client, (await staged(client))["id"]) + sid = st["id"] + url = f"/api/datasources/imports/{sid}/commit" + body = {"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]} + resp = await client.put(f"/api/datasources/imports/{sid}/recipe", + json={"recipe": {"recipe_format": "agentlab-recipe/2", "tables": []}}) + assert resp.status_code == 200 and resp.json()["status"] == "drafting" + assert resp.json()["trial"]["trial_id"] == st["trial"]["trial_id"], "回执留着标「已失效」" + resp = await client.post(url, json=body) + assert resp.status_code == 409 and resp.json()["code"] == "trial_required", resp.text + # 键还在、存下的配方却读不成 Recipe（记录被改坏）：同样 409 trial_required + await client.put(f"/api/datasources/imports/{sid}/recipe", json={"recipe": FLOW_FULL}) + st = await trialed(client, sid) + async with SessionLocal() as session: + row = await session.get(ImportStaging, sid) + row.recipe = {"recipe_format": "agentlab-recipe/2", "tables": []} + await session.commit() + resp = await client.post(url, json={**body, "trial_id": st["trial"]["trial_id"]}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_required", resp.text + assert "validation error" not in resp.json()["detail"] + + +@pytest.mark.parametrize("change", ["parser_ver", "engine_ver", "file_name"]) +async def test_commit_recomputes_the_trial_key(client, fake, monkeypatch, change): + """提交时按当前的解析器、执行器版本和文件名重算 trial_key，不信存下的键。""" + st = await trialed(client, (await staged(client))["id"]) + if change == "parser_ver": + monkeypatch.setattr(recipe_parsers, "PARSER_VER", recipe_parsers.PARSER_VER + "-next") + elif change == "engine_ver": + monkeypatch.setattr(recipe_types, "RECIPE_ENGINE_VER", recipe_types.RECIPE_ENGINE_VER + "-next") + else: + async with SessionLocal() as session: + row = await session.get(ImportStaging, st["id"]) + row.file_name = "改过名字的月报.xlsx" + await session.commit() + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_required", resp.text + + +async def test_commit_notes_invalid_keeps_the_trial_and_the_pointer(client, fake): + """7.8 第 3 步：试运行通过，提交时带上接受理由生成的说明有问题 → 422 notes_invalid；指针不动， + 暂存区仍是 trialed、试运行库还在（没有进发布流程）。""" + done = await recipe_source(client) + source_id = done["source"]["id"] + fake.checks = [ok_check("C1", "context_agree", "data_quality"), + CheckResult("R1", "relation_sum_eq", "全日客流 = 分区甲 + 分区乙", "mismatch", "data_quality", + checked=31, failed=1, acceptable=True)] + fake.notes_problems_on_accept = ["表「t」的说明：说明中含有数字「10」"] + st = (await client.post(f"/api/datasources/{source_id}/reupload", + files={"file": ("下月.xlsx", raw_bytes(), XLSX)})).json() + assert st["trial"]["status"] == "needs_decision" + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", json={ + "trial_id": st["trial"]["trial_id"], "confirmations": [i["id"] for i in st["trial"]["confirm_items"]], + "acceptances": [{"check_id": "R1", "reason": "当日为补录"}]}) + assert resp.status_code == 422 and resp.json()["code"] == "notes_invalid", resp.text + assert "说明中含有数字" in resp.json()["detail"] + assert (await get(DataSource, source_id)).current_snapshot_id == done["snapshot_id"] + row = await get(ImportStaging, st["id"]) + assert row.status == "trialed" and row.trial_key and Path(row.trial_path).is_file() + async with SessionLocal() as session: + n = len(list((await session.execute(select(TableImport).where(TableImport.source_id == source_id))).scalars())) + assert n == 1 + + +async def test_notes_that_do_not_fit_the_schema_fail_the_publish(client, fake): + """说明写不进表结构（apply_notes 记了问题）→ 发布中止：500 publish_failed，指针不动，暂存区退回 drafting。""" + done = await recipe_source(client) + source_id = done["source"]["id"] + fake.notes_extra_table = True + st = (await client.post(f"/api/datasources/{source_id}/reupload", + files={"file": ("下月.xlsx", raw_bytes(), XLSX)})).json() + assert st["trial"]["status"] == "passed" + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + assert resp.status_code == 500 and resp.json()["code"] == "publish_failed", resp.text + assert "缺表" in resp.json()["detail"] + assert (await get(DataSource, source_id)).current_snapshot_id == done["snapshot_id"] + row = await get(ImportStaging, st["id"]) + assert row.status == "drafting" and row.trial_path is None and row.trial + + +async def test_same_as_import_rechecks_base_inside_the_lock(client, fake): + """同一文件再传（same_as_import）和另一期的提交排在锁上，另一期先拿到锁、切走了指针：same_as_import + 那次在锁内重读指针 → 409 base_changed，不返回已被替换的那期导入。""" + raw = raw_bytes() + first = await committed(client, await trialed(client, (await staged(client, raw=raw))["id"])) + source_id = first["source"]["id"] + same = (await client.post(f"/api/datasources/{source_id}/reupload", + files={"file": ("月报导出_2026-08-01_2026-08-31.xlsx", raw, XLSX)})).json() + other = (await client.post(f"/api/datasources/{source_id}/reupload", + files={"file": ("下月.xlsx", raw_bytes(), XLSX)})).json() + assert same["trial"]["same_as_import"]["id"] == first["import_id"] + assert other["trial"]["status"] == "passed" and other["trial"]["same_as_import"] is None + + async def commit(st): + return await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + + lock = table_versions.store_lock() + await lock.acquire() + try: + t_other = asyncio.create_task(commit(other)) + await _wait_on_lock(lock, 1) + t_same = asyncio.create_task(commit(same)) + await _wait_on_lock(lock, 2) + assert len(lock._waiters) >= 2, "same_as_import 的提交也要排到锁上" + finally: + lock.release() + r_other, r_same = await asyncio.gather(t_other, t_same) + assert r_other.status_code == 201, r_other.text + assert r_same.status_code == 409 and r_same.json()["code"] == "base_changed", r_same.text + assert (await get(DataSource, source_id)).current_snapshot_id == r_other.json()["snapshot_id"] + row = await get(ImportStaging, same["id"]) + assert row.status == "trialed" and row.committed_import_id is None + + +# ========================================================================== +# 上传新一期、修改配方 +# ========================================================================== + + +async def test_reupload_replays_without_claims_and_diff_gets_the_full_receipt(client, fake): + fake.diff = [DiffItem("period", "统计期：2026-08-01 至 2026-08-31 → 2026-09-01 至 2026-09-30"), + DiffItem("label_writing", "分段「日间」的标签写法变了", requires_confirm=True, + confirm_id="diff:label_writing:日间")] + done = await recipe_source(client) + source_id = done["source"]["id"] + fake.calls.clear() + resp = await client.post(f"/api/datasources/{source_id}/reupload", + files={"file": ("月报导出_2026-09-01_2026-09-30.xlsx", raw_bytes(), XLSX)}) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["kind"] == "reupload" and st["status"] == "trialed" + # 现行配方原样重放：静态校验按 replay、不带系统发现；系统发现按现行配方的写法算过、存下 + assert all(v["origin"] == "replay" and v["facts"] is None for v in fake.calls["validate"]) + assert fake.calls["detect_facts"][0] == FLOW_FULL + call = fake.calls["diff_reports"][-1] + assert call["recipe_changed"] is False and isinstance(call["recipe"], Recipe) + assert call["prev"]["format"] == recipe_imports.MANIFEST_FORMAT, "上一期取现行导入的导入清单" + assert call["prev_file_name"] == "月报导出_2026-08-01_2026-08-31.xlsx" + assert {"axes", "tables", "ledger", "period", "lineage", "canonicalized"} <= set(call["cur"]["receipt"]) + assert "problems" not in call["cur"]["receipt"] and call["cur"]["recipe_sha256"] + ctx = fake.calls["confirm_items"][-1] + assert ctx.kind == "reupload" and ctx.old_recipe is not None and ctx.prev is call["prev"] + assert [d.kind for d in ctx.diff] == ["period", "label_writing"] + assert [i["id"] for i in st["trial"]["confirm_items"]] == ["mode", "diff:label_writing:日间"] + # 在上传新一期的暂存区里改配方：之后的静态校验带系统发现 + fake.calls.clear() + resp = await client.put(f"/api/datasources/imports/{st['id']}/recipe", + json={"recipe": {**FLOW_FULL, "other_visible_sheets": "reject"}}) + assert resp.status_code == 200 + assert fake.calls["validate"][-1]["facts"] is not None and fake.calls["validate"][-1]["origin"] == "manual" + + +async def test_redraft_uses_the_current_raw_and_recipe(client, fake): + done = await recipe_source(client) + source_id = done["source"]["id"] + fake.calls.clear() + resp = await client.post(f"/api/datasources/{source_id}/redraft", json={}) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["kind"] == "redraft" and st["status"] == "drafting" and st["recipe"] == FLOW_FULL + assert st["ai"]["offered"] is False and st["draft"] is None + assert fake.calls["detect_facts"][0] == FLOW_FULL + assert fake.calls["validate"][-1]["facts"] is not None, "修改配方：静态校验带系统发现" + imp = await get(TableImport, done["import_id"]) + row = await get(ImportStaging, st["id"]) + assert row.raw_sha256 == imp.raw_sha256 and row.base_recipe_id == done["recipe_id"] + # 改了配方再提交：新配方 seq=2，旧的 superseded + resp = await client.put(f"/api/datasources/imports/{st['id']}/recipe", + json={"recipe": {**FLOW_FULL, "other_visible_sheets": "reject"}}) + st = await trialed(client, st["id"]) + assert fake.calls["diff_reports"][-1]["recipe_changed"] is True + out = await committed(client, st) + assert out["recipe_id"] != done["recipe_id"] + old, new = await get(TableRecipe, done["recipe_id"]), await get(TableRecipe, out["recipe_id"]) + assert old.status == "superseded" and new.status == "active" and new.seq == 2 and new.origin == "manual" + + +async def test_switch_from_simple_upload(client, fake): + from tests.test_table_versions import V1, xlsx + + name = unique() + resp = await client.post("/api/datasources/upload", files={"file": ("客流.xlsx", xlsx(V1), XLSX)}, + data={"name": name}) + assert resp.status_code == 201, resp.text + old = resp.json() + assert old["source"]["import_mode"] == "simple" and old["source"]["current_recipe"] is None + st = await staged(client, name) + assert st["kind"] == "switch" and st["source"]["id"] == old["source"]["id"] + assert st["source"]["exists"] is True and st["source"]["import_mode"] == "simple" + st = await trialed(client, st["id"]) + call = fake.calls["diff_reports"][-1] + assert "ledger" not in call["prev"]["receipt"] and call["prev"]["receipt"]["tables"] + ctx = fake.calls["confirm_items"][-1] + assert ctx.kind == "switch" and ctx.old_schema_cache and ctx.old_recipe is None + out = await committed(client, st) + assert out["source"]["id"] == old["source"]["id"] and out["source"]["import_mode"] == "recipe" + + +async def test_redraft_without_raw_and_non_recipe_sources(client, fake): + from tests.test_table_versions import V1, xlsx + + name = unique() + resp = await client.post("/api/datasources/upload", files={"file": ("客流.xlsx", xlsx(V1), XLSX)}, + data={"name": name}) + simple_id = resp.json()["source"]["id"] + for path in (f"/api/datasources/{simple_id}/redraft", f"/api/datasources/{simple_id}/reupload"): + kw = {"json": {}} if path.endswith("redraft") else {"files": {"file": ("x.xlsx", raw_bytes(), XLSX)}} + resp = await client.post(path, **kw) + assert resp.status_code == 409 and resp.json()["code"] == "not_recipe_source", path + resp = await client.get(f"/api/datasources/{simple_id}/recipe") + assert resp.status_code == 409 and resp.json()["code"] == "not_recipe_source" + for path in ("/api/datasources/nope/redraft", "/api/datasources/nope/recipe"): + resp = await (client.post(path, json={}) if path.endswith("redraft") else client.get(path)) + assert resp.status_code == 404, path + + done = await recipe_source(client) + source_id = done["source"]["id"] + resp = await client.post(f"/api/datasources/{source_id}/imports/{done['import_id']}/purge-raw", + json={"reason": "合成理由：含敏感信息"}) + assert resp.status_code == 200, resp.text + resp = await client.post(f"/api/datasources/{source_id}/redraft", json={}) + assert resp.status_code == 409 and resp.json()["code"] == "raw_missing" + + +async def test_purge_raw_discards_open_stagings_on_the_same_raw(client, fake): + raw = raw_bytes() + st = await trialed(client, (await staged(client, raw=raw))["id"]) + out = await committed(client, st) + source_id = out["source"]["id"] + resp = await client.post(f"/api/datasources/{source_id}/reupload", files={"file": ("同一份.xlsx", raw, XLSX)}) + again = resp.json() + trial_path = (await get(ImportStaging, again["id"])).trial_path + assert trial_path and Path(trial_path).is_file() + resp = await client.post(f"/api/datasources/{source_id}/imports/{out['import_id']}/purge-raw", + json={"reason": "合成理由"}) + assert resp.status_code == 200, resp.text + assert resp.json()["discarded_stagings"] == [again["id"]] + row = await get(ImportStaging, again["id"]) + assert row.status == "discarded" and row.grid_preview is None and row.trial is None + assert not Path(trial_path).exists() + + +async def test_commit_recomputes_required_items_from_the_stored_diff(client, fake): + """提交时按试运行存下的差异卡重算必勾项：差异卡里需确认的那一项不勾 → 422，detail 里有它。""" + done = await recipe_source(client) + fake.diff = [DiffItem("label_writing", "分段「日间」的标签写法变了", requires_confirm=True, + confirm_id="diff:label_writing:日间")] + st = (await client.post(f"/api/datasources/{done['source']['id']}/reupload", + files={"file": ("下月.xlsx", raw_bytes(), XLSX)})).json() + assert [i["id"] for i in st["trial"]["confirm_items"]] == ["mode", "diff:label_writing:日间"] + fake.diff = [] + n = len(fake.calls["diff_reports"]) + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required", resp.text + assert "diff:label_writing:日间" in resp.json()["detail"] + assert len(fake.calls["diff_reports"]) == n, "提交时不重算差异卡，用存下的那份" + await committed(client, st) + + +@pytest.mark.parametrize("where", ["execute", "confirm_items"]) +async def test_reupload_whose_auto_trial_fails_is_discarded(client, fake, store, monkeypatch, where): + """上传新一期的自动试运行出错：接口回错误，刚建的暂存区放弃（瘦身）、原件和试运行库都不留。""" + done = await recipe_source(client) + source_id = done["source"]["id"] + pipe = recipe_imports.pipeline() + if where == "execute": + def broken_execute(rec, raw, filename, scan, db_path, **kw): + raise UnsupportedTable("工作表「甲表」读取失败：文件已损坏") + monkeypatch.setattr(pipe, "execute", broken_execute) + else: + def broken_confirm(ctx): + raise ValueError("合成的调用方错误") + monkeypatch.setattr(pipe, "confirm_items", broken_confirm) + raw = raw_bytes() + resp = await client.post(f"/api/datasources/{source_id}/reupload", files={"file": ("下月.xlsx", raw, XLSX)}) + assert resp.status_code == 400, resp.text + if where == "execute": + assert resp.json()["code"] == "file_unreadable" + async with SessionLocal() as session: + rows = list((await session.execute(select(ImportStaging).where( + ImportStaging.source_id == source_id, ImportStaging.kind == "reupload"))).scalars()) + assert [r.status for r in rows] == ["discarded"] and rows[0].scan is None + sha = hashlib.sha256(raw).hexdigest() + assert not (store / "uploads" / "raw" / sha[:2] / sha).exists() + trials = table_versions.tables_root() / source_id / table_versions.TRIALS_DIR + assert not list(trials.glob(f"{rows[0].id}-*")), "这次新写的试运行库也删掉" + listing = {s["id"]: s for s in (await client.get("/api/datasources")).json()} + assert listing[source_id]["open_staging"] is None + assert (await get(DataSource, source_id)).current_snapshot_id == done["snapshot_id"] + + +# ========================================================================== +# 放弃、过期 +# ========================================================================== + + +async def test_discard_slims_and_releases_the_raw(client, fake, store): + st = await trialed(client, (await staged(client))["id"]) + row = await get(ImportStaging, st["id"]) + raw_path = store / "uploads" / "raw" / row.raw_sha256[:2] / row.raw_sha256 + trial = Path(row.trial_path) + assert raw_path.is_file() and trial.is_file() + resp = await client.delete(f"/api/datasources/imports/{st['id']}") + assert resp.status_code == 204 + row = await get(ImportStaging, st["id"]) + assert row.status == "discarded" and row.closed_at and row.scan is None and row.trial is None + assert not raw_path.exists() and not trial.exists() + resp = await client.delete(f"/api/datasources/imports/{st['id']}") + assert resp.status_code == 409 and resp.json()["code"] == "staging_closed" + for method, path, kw in ( + ("post", "answers", {"json": {"answers": {}}}), ("put", "recipe", {"json": {"recipe": FLOW_FULL}}), + ("post", "trial", {"json": {}}), ("post", "commit", {"json": {"trial_id": "x"}}), + ("get", "draft-ai/preview", {}), ("post", "draft-ai", {"json": {"consent": True}}), + ): + resp = await getattr(client, method)(f"/api/datasources/imports/{st['id']}/{path}", **kw) + assert resp.status_code == 409 and resp.json()["code"] == "staging_closed", path + + +async def test_expiry_happens_on_get(client, fake): + st = await trialed(client, (await staged(client))["id"]) + async with SessionLocal() as session: + row = await session.get(ImportStaging, st["id"]) + trial = Path(row.trial_path) + row.expires_at = row.created_at - dt.timedelta(days=1) + await session.commit() + resp = await client.get(f"/api/datasources/imports/{st['id']}") + out = resp.json() + assert out["status"] == "expired" and out["grids"] == [] and out["trial"] is None + assert not trial.exists() + + +# ========================================================================== +# AI 起草（P2-SPEC 9.6 的 WP-5c 部分） +# ========================================================================== + + +async def test_ai_gates(client, fake): + # 规则草稿完整 → 不需要 + st = await staged(client) + assert st["ai"]["offered"] is False + resp = await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview") + assert resp.status_code == 409 and resp.json()["code"] == "ai_not_needed" + resp = await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", json={"consent": True}) + assert resp.status_code == 409 and resp.json()["code"] == "ai_not_needed" + # 上传新一期 → 不提供 + done = await recipe_source(client) + re_ = (await client.post(f"/api/datasources/{done['source']['id']}/reupload", + files={"file": ("x.xlsx", raw_bytes(), XLSX)})).json() + resp = await client.post(f"/api/datasources/imports/{re_['id']}/draft-ai", json={"consent": True}) + assert resp.status_code == 409 and resp.json()["code"] == "ai_not_offered" + # 规则不完整、接入不可用 → 409，桩没被调 + fake.complete = False + fake.availability = AiAvailability(False, reason="未配置模型接入，无法使用 AI 起草。请到「设置 → 模型接入」添加") + st = await staged(client) + assert st["ai"] == {"offered": True, "available": False, "reason": fake.availability.reason, "model": "", + "provider": ""} + resp = await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview") + assert resp.status_code == 409 and resp.json()["code"] == "ai_unavailable" + resp = await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", json={"consent": True, + "preview_sha256": "x"}) + assert resp.status_code == 409 and resp.json()["code"] == "ai_unavailable" + assert fake.calls["ai_draft"] == [] + # consent 缺省或不是 true → 422 + for body in ({}, {"consent": False}, {"consent": "true"}): + resp = await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", json=body) + assert resp.status_code == 422 and resp.json()["code"] == "ai_consent_required" + # 太大 → 413 + fake.availability = AiAvailability(True, model="假模型", provider="假接入") + fake.compress_too_large = True + resp = await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview") + assert resp.status_code == 413 and resp.json()["code"] == "ai_too_large" + resp = await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", json={"consent": True, + "preview_sha256": "x"}) + assert resp.status_code == 413 and fake.calls["ai_draft"] == [] + + +async def test_ai_preview_is_the_full_first_call_and_stale_sha_is_refused(client, fake): + fake.complete = False + st = await staged(client) + resp = await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview") + assert resp.status_code == 200, resp.text + pv = resp.json() + assert pv["model"] == "假模型" and pv["provider"] == "假接入" and pv["chars"] == len(pv["text"]) + assert pv["text_sha256"] == hashlib.sha256(pv["text"].encode()).hexdigest() + # 同意令牌绑定全文、接入名、模型 id（AU-7） + assert pv["sha256"] == recipe_imports.consent_token(pv["text_sha256"], fake.availability) + assert pv["sha256"] != pv["text_sha256"] + assert pv["text"].startswith("系统提示") and "第 9 行像合计行" in pv["text"], "规则半成品的失败原因也在全文里" + resp = await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", + json={"consent": True, "preview_sha256": "0" * 64}) + assert resp.status_code == 409 and resp.json()["code"] == "ai_preview_stale" + assert fake.calls["ai_draft"] == [] + resp = await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"], "signed_by": "测试员"}) + assert resp.status_code == 200, resp.text + out = resp.json() + sent = fake.calls["ai_draft"][-1]["compressed"] + assert sent.text in pv["text"] + assert out["recipe_origin"] == "ai" and out["answers"] == {} and out["recipe"]["other_visible_sheets"] == "reject" + assert out["cards"][0]["id"] == "ai_origin" + consent = out["ai_consents"][-1] + assert consent["preview_sha256"] == pv["sha256"] and consent["compressed_sha256"] == sent.sha256 + assert consent["text_sha256"] == pv["text_sha256"] + assert consent["signed_by"] == "测试员" and consent["chars"] == len(pv["text"]) and consent["model"] == "假模型" + assert out["ai_draft"]["usage"][0]["input_tokens"] == 120 and out["ai_draft"]["total_tokens"] == 200 + assert "failures_for_model" not in (out["ai_draft"]["draft"] or {}) + # AI 起草时静态校验用 origin=ai + assert fake.calls["validate"][-1]["origin"] == "ai" + + +async def test_ai_failed_keeps_the_recipe_and_records_usage_first(client, fake): + fake.complete = False + fake.ai_result = AiDraftResult(None, [AiUsage("假模型", 10, 0, 0.0, False, "2026-10-01T00:00:00+00:00", 1), + AiUsage("假模型", 10, 5, 0.0, False, "2026-10-01T00:00:01+00:00", 2)], + 2, "AI 的输出被截断，配方没有写完（输出被截断）", 40, "a" * 64) + st = await staged(client) + assert st["recipe"] + pv = (await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview")).json() + resp = await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"]}) + assert resp.status_code == 502 and resp.json()["code"] == "ai_failed" + # 原话 + 下一步：有工作配方（规则半成品）时叫用户在面板里接着改 + assert resp.json()["detail"] == fake.ai_result.error + "。可以重试，或在配方面板中修改现有配方" + out = (await client.get(f"/api/datasources/imports/{st['id']}")).json() + assert out["recipe"] == st["recipe"] and out["recipe_origin"] == "rules" + assert out["ai_draft"]["error"] == fake.ai_result.error and out["ai_draft"]["draft"] is None + assert len(out["ai_consents"]) == 1 + row = await get(ImportStaging, st["id"]) + assert [u["attempt"] for u in row.ai_usage] == [1, 2] + + +async def test_ai_failed_without_a_working_recipe_points_to_pasting_one(client, fake): + """规则起草连半成品都没有时，配方面板里只有「粘贴配方」、没有表单可填：AI 起草失败（502）和表格太大 + （413）的下一步都说粘贴一份完整的配方，不说「填写」。""" + fake.complete = False + fake.no_recipe = True + fake.ai_result = AiDraftResult(None, [AiUsage("假模型", 10, 0, 0.0, False, "2026-10-01T00:00:00+00:00", 1)], + 1, "AI 未能起草出合法的配方", 40, "a" * 64) + st = await staged(client) + assert st["recipe"] is None and st["ai"]["offered"] is True + pv = (await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview")).json() + resp = await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"]}) + assert resp.status_code == 502 and resp.json()["code"] == "ai_failed" + assert resp.json()["detail"] == "AI 未能起草出合法的配方。可以重试，或在配方面板中粘贴一份完整的配方" + fake.compress_too_large = True + resp = await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview") + assert resp.status_code == 413 and resp.json()["code"] == "ai_too_large" + assert resp.json()["detail"].endswith("无法交给 AI 起草。请在配方面板中粘贴一份完整的配方"), resp.json()["detail"] + + +async def test_ai_draft_dry_run_is_awaited_inside_a_parse_slot_off_the_event_loop(client, fake): + """AI 每次尝试之后的干跑（WP-4 遗留）：接口层交给 ai_draft 的是协程函数，执行器在线程池里跑、 + 占着一个解析名额；不在事件循环线程上同步执行。""" + fake.complete = False + fake.slot = table_versions.parse_slot() + st = await staged(client) + pv = (await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview")).json() + n = len(fake.calls["execute"]) + resp = await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"]}) + assert resp.status_code == 200, resp.text + assert fake.calls["ai_dry_is_coroutine"] == [True] + dry = fake.calls["execute"][n] # ai_draft 里那次干跑（之后才是采用草稿时的重算） + assert dry["db"] is None and dry["thread"] != threading.get_ident(), "干跑不在事件循环线程上" + assert dry["slots_free"] == table_versions.MAX_PARALLEL_PARSES - 1, "干跑时占着一个解析名额" + + +async def test_ai_draft_does_not_hold_a_parse_slot_while_waiting(client, fake): + fake.complete = False + seen: dict[str, Any] = {} + + async def wait_for_model(): + slot = table_versions.parse_slot() + seen["free"] = slot._value + # 名额全占满，「等模型」这一步照样走得完 + for _ in range(table_versions.MAX_PARALLEL_PARSES): + await slot.acquire() + try: + await asyncio.sleep(0.01) + finally: + for _ in range(table_versions.MAX_PARALLEL_PARSES): + slot.release() + + fake.ai_hook = wait_for_model + st = await staged(client) + pv = (await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview")).json() + resp = await asyncio.wait_for(client.post(f"/api/datasources/imports/{st['id']}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"]}), timeout=10) + assert resp.status_code == 200, resp.text + assert seen["free"] == table_versions.MAX_PARALLEL_PARSES + + +@pytest.mark.parametrize("outcome", ["drafted", "failed"]) +async def test_ai_usage_is_recorded_even_if_the_staging_closes_while_waiting(client, fake, outcome): + """等模型期间暂存区被放弃：用量照样落库（瘦身后的行也留用量），草稿不写回已瘦身的行，回 409 staging_closed。""" + fake.complete = False + if outcome == "failed": + fake.ai_result = AiDraftResult(None, [AiUsage("假模型", 10, 5, 0.0, False, "2026-10-01T00:00:00+00:00", 1)], + 1, "AI 没有返回合法的配方", 40, "a" * 64) + st = await staged(client) + pv = (await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview")).json() + + async def discard_meanwhile(): + resp = await client.delete(f"/api/datasources/imports/{st['id']}") + assert resp.status_code == 204 + + fake.ai_hook = discard_meanwhile + resp = await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"]}) + assert resp.status_code == 409 and resp.json()["code"] == "staging_closed", resp.text + row = await get(ImportStaging, st["id"]) + assert row.status == "discarded" and [u["model"] for u in row.ai_usage] == ["假模型"] + assert row.drafts is None, "已瘦身的行不再写回草稿" + assert len(row.ai_consents) == 1 + + +async def test_ai_usage_and_summary_are_kept_when_adopting_the_draft_fails(client, fake, monkeypatch): + """采用 AI 草稿那一步出错（例如原件已不在）：用量和这次起草的摘要已经落库，工作配方不变。""" + fake.complete = False + st = await staged(client) + pv = (await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview")).json() + + def adopt_fails(*a, **kw): + raise recipe_imports.ImportRefused(409, "raw_missing", recipe_imports._RAW_GONE) + + monkeypatch.setattr(recipe_imports, "_adopt_ai_work", adopt_fails) + resp = await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"]}) + assert resp.status_code == 409 and resp.json()["code"] == "raw_missing", resp.text + out = (await client.get(f"/api/datasources/imports/{st['id']}")).json() + assert out["recipe"] == st["recipe"] and out["recipe_origin"] == "rules" + assert out["ai_draft"]["usage"][0]["input_tokens"] == 120 and out["ai_draft"]["draft"]["recipe"] + row = await get(ImportStaging, st["id"]) + assert [u["model"] for u in row.ai_usage] == ["假模型"] and "ai_adopted" not in row.drafts + + +async def test_ai_draft_commit_needs_ai_origin_and_records_usage(client, fake): + fake.complete = False + st = await staged(client) + pv = (await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview")).json() + resp = await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"]}) + assert resp.status_code == 200 + st = await trialed(client, st["id"]) + assert st["trial"]["status"] == "passed" + url = f"/api/datasources/imports/{st['id']}/commit" + resp = await client.post(url, json={"trial_id": st["trial"]["trial_id"], "confirmations": ["mode"]}) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" + assert "ai_origin" in resp.json()["detail"] + out = await committed(client, st) + rec = await get(TableRecipe, out["recipe_id"]) + assert rec.origin == "ai" and rec.ai_usage["calls"][0]["model"] == "假模型" + assert rec.ai_usage["consents"][0]["preview_sha256"] == pv["sha256"] + imp = await get(TableImport, out["import_id"]) + from app.core import artifact_store + + manifest = artifact_store.load(imp.manifest_artifact) + assert manifest["ai"]["usage"] and manifest["ai"]["consents"][0]["preview_sha256"] == pv["sha256"] + assert manifest["recipe"]["origin"] == "ai" + + +async def test_ai_then_manual_edit_commits_as_mixed(client, fake): + fake.complete = False + st = await staged(client) + pv = (await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview")).json() + await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"]}) + resp = await client.put(f"/api/datasources/imports/{st['id']}/recipe", + json={"recipe": {**FLOW_FULL, "other_visible_sheets": "confirm"}}) + assert resp.json()["recipe_origin"] == "manual" + st = await trialed(client, st["id"]) + assert fake.calls["confirm_items"][-1].recipe_origin == "mixed" + out = await committed(client, st) + assert (await get(TableRecipe, out["recipe_id"])).origin == "mixed" + + +async def test_interactive_dry_run_failure_becomes_a_problem(client, fake, monkeypatch): + st = await staged(client) + pipe = recipe_imports.pipeline() + real = pipe.execute + + def flaky(rec, raw, filename, scan, db_path, **kw): + if db_path is None: + raise RuntimeError("合成的执行器错误 12345") + return real(rec, raw, filename, scan, db_path, **kw) + + monkeypatch.setattr(pipe, "execute", flaky) + resp = await client.post(f"/api/datasources/imports/{st['id']}/answers", + json={"answers": {"q_relation:F1": {"value": "register"}}}) + assert resp.status_code == 200, resp.text + probs = resp.json()["draft_problems"] + assert [p["code"] for p in probs] == ["dry_run_failed"] and "12345" not in probs[0]["message"] + # 试运行不吞错：照常执行（这里写库那一步是好的） + st = await trialed(client, st["id"]) + assert st["trial"]["status"] == "passed" + + +async def test_trial_unreadable_file_is_a_400(client, fake, monkeypatch): + st = await staged(client) + pipe = recipe_imports.pipeline() + + def unreadable(rec, raw, filename, scan, db_path, **kw): + raise UnsupportedTable("工作表「甲表」读取失败：文件已损坏") + + monkeypatch.setattr(pipe, "execute", unreadable) + resp = await client.post(f"/api/datasources/imports/{st['id']}/trial", json={}) + assert resp.status_code == 400 and resp.json()["code"] == "file_unreadable" + assert (await get(ImportStaging, st["id"])).trial_path is None + + +# -------------------------------------------------------------------------- +# 整体审查（AU / SE）修复 +# -------------------------------------------------------------------------- + + +async def test_ai_draft_identical_to_working_recipe_invalidates_the_trial(client, fake): + """AU-4：AI 草稿与已试运行的工作配方逐字相同（哈希没变），来源却从手改变成了 AI：试运行作废， + 重跑之后清单里有 ai_origin，照它勾就能提交。""" + fake.complete = False + st = await staged(client) + resp = await client.put(f"/api/datasources/imports/{st['id']}/recipe", + json={"recipe": {**FLOW_FULL, "other_visible_sheets": "reject"}}) + assert resp.status_code == 200 and resp.json()["recipe_origin"] == "manual" + st = await trialed(client, st["id"]) + assert "ai_origin" not in [i["id"] for i in st["trial"]["confirm_items"]] + pv = (await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview")).json() + resp = await client.post(f"/api/datasources/imports/{st['id']}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"]}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["recipe_origin"] == "ai" and out["recipe_sha256" if "recipe_sha256" in out else "recipe"] is not None + assert out["status"] == "drafting", "来源变了，试运行作废" + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], + "confirmations": [i["id"] for i in st["trial"]["confirm_items"]]}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_required" + st = await trialed(client, st["id"]) + assert "ai_origin" in [i["id"] for i in st["trial"]["confirm_items"]] + await committed(client, st) + + +async def test_commit_requiring_items_the_trial_did_not_show_asks_for_a_new_trial(client, fake): + """AU-4（通用）：提交时服务端重算出的必勾项比试运行给界面的多：试运行作废、回 409 trial_required， + 而不是回一个界面清单里根本没有的 422 confirm_required。""" + st = await trialed(client, (await staged(client))["id"]) + fake.confirm_extra = [ConfirmItem("mask_lost:金额_元", "遮罩列在新结构里找不到")] + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], + "confirmations": [i["id"] for i in st["trial"]["confirm_items"]]}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_required", resp.text + row = await get(ImportStaging, st["id"]) + assert row.status == "drafting" and row.trial_key is None and row.trial_path is None + st = await trialed(client, st["id"]) + assert "mask_lost:金额_元" in [i["id"] for i in st["trial"]["confirm_items"]] + await committed(client, st) + + +async def test_concurrent_ai_drafts_on_one_staging_are_refused(client, fake): + """SE-11：同一暂存区并发两个 POST draft-ai：第二个回 409 ai_in_progress、不调模型；同意记录和用量各一条。""" + fake.complete = False + st = await staged(client) + pv = (await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview")).json() + gate = asyncio.Event() + + async def hold(): + await gate.wait() + + fake.ai_hook = hold + url = f"/api/datasources/imports/{st['id']}/draft-ai" + first = asyncio.create_task(client.post(url, json={"consent": True, "preview_sha256": pv["sha256"], + "signed_by": "甲"})) + for _ in range(200): + if fake.calls["ai_draft"]: + break + await asyncio.sleep(0.01) + assert len(fake.calls["ai_draft"]) == 1 + try: + second = await asyncio.wait_for( + client.post(url, json={"consent": True, "preview_sha256": pv["sha256"], "signed_by": "乙"}), timeout=5) + finally: + gate.set() + assert second.status_code == 409 and second.json()["code"] == "ai_in_progress", second.text + resp = await asyncio.wait_for(first, timeout=10) + assert resp.status_code == 200, resp.text + assert len(fake.calls["ai_draft"]) == 1 + row = await get(ImportStaging, st["id"]) + assert [c["signed_by"] for c in row.ai_consents] == ["甲"] and len(row.ai_usage) == 1 + # 结束后标记清掉：可以再起草一次 + fake.ai_hook = None + pv = (await client.get(f"/api/datasources/imports/{st['id']}/draft-ai/preview")).json() + resp = await client.post(url, json={"consent": True, "preview_sha256": pv["sha256"]}) + assert resp.status_code == 200, resp.text + + +async def test_discard_waiting_behind_a_commit_does_not_overwrite_committed(client, fake, monkeypatch): + """SE-8：提交持锁发布时，放弃请求已过了开头的检查、在锁上排队；提交完成后放弃拿到锁：回 409 staging_closed， + 暂存区仍是 committed。""" + st = await trialed(client, (await staged(client))["id"]) + release = asyncio.Event() + entered = asyncio.Event() + real_close = table_versions.close_staging + + def slow_close(staging, status, **kw): + if status == "committed": + entered.set() + return real_close(staging, status, **kw) + + real_publish = table_versions.publish_build + + async def publish(*a, **kw): + before = kw["before_commit"] + + async def wait_then(imp_id, snap_id): + await before(imp_id, snap_id) + entered.set() + await release.wait() + kw["before_commit"] = wait_then + return await real_publish(*a, **kw) + + monkeypatch.setattr(table_versions, "publish_build", publish) + commit = asyncio.create_task(client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], + "confirmations": [i["id"] for i in st["trial"]["confirm_items"]]})) + await asyncio.wait_for(entered.wait(), timeout=10) + discard = asyncio.create_task(client.delete(f"/api/datasources/imports/{st['id']}")) + for _ in range(200): + if table_versions.store_lock()._waiters: + break + await asyncio.sleep(0.01) + assert table_versions.store_lock()._waiters, "放弃请求在锁上排队" + release.set() + resp = await asyncio.wait_for(commit, timeout=10) + assert resp.status_code == 201, resp.text + d = await asyncio.wait_for(discard, timeout=10) + assert d.status_code == 409 and d.json()["code"] == "staging_closed", d.text + row = await get(ImportStaging, st["id"]) + assert row.status == "committed" and row.committed_import_id == resp.json()["import_id"] + + +async def test_commit_after_discard_finished_during_preflight_is_refused(client, fake, monkeypatch): + """SE-8：提交预检期间放弃已经完成：提交不发布、不把已放弃的暂存区改回 drafting。""" + st = await trialed(client, (await staged(client))["id"]) + real_context = recipe_imports._context + done = {"n": 0} + + async def context_then_discard(session, staging): + cx = await real_context(session, staging) + if done["n"] == 0: + done["n"] += 1 + resp = await client.delete(f"/api/datasources/imports/{st['id']}") + assert resp.status_code == 204 + return cx + + monkeypatch.setattr(recipe_imports, "_context", context_then_discard) + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], + "confirmations": [i["id"] for i in st["trial"]["confirm_items"]]}) + assert resp.status_code == 409 and resp.json()["code"] in ("staging_closed", "trial_required"), resp.text + row = await get(ImportStaging, st["id"]) + assert row.status == "discarded" and row.committed_import_id is None + + +async def test_publish_after_discard_is_refused_inside_the_lock(client, fake, monkeypatch): + """SE-8：预检都过了、发布前一刻暂存区被放弃（试运行库碰巧还在）：锁内重读状态，不发布已放弃的暂存区。""" + st = await trialed(client, (await staged(client))["id"]) + monkeypatch.setattr(table_versions, "remove_trial_file", lambda path: None) + + async def keep_raw(session, staging): # 原件还被别的暂存区引用着 + return None + monkeypatch.setattr(table_versions, "release_staging_raw", keep_raw) + real_publish = table_versions.publish_build + + async def discard_then_publish(*a, **kw): + resp = await client.delete(f"/api/datasources/imports/{st['id']}") + assert resp.status_code == 204 + return await real_publish(*a, **kw) + + monkeypatch.setattr(table_versions, "publish_build", discard_then_publish) + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": st["trial"]["trial_id"], + "confirmations": [i["id"] for i in st["trial"]["confirm_items"]]}) + assert resp.status_code == 409 and resp.json()["code"] == "staging_closed", resp.text + row = await get(ImportStaging, st["id"]) + assert row.status == "discarded" and row.committed_import_id is None + async with SessionLocal() as session: + n = (await session.execute(select(func.count()).select_from(TableImport) + .where(TableImport.source_id == st["source"]["id"]))).scalar() + assert n == 0 diff --git a/backend/tests/test_recipe_mutations.py b/backend/tests/test_recipe_mutations.py new file mode 100644 index 00000000..04479c1e --- /dev/null +++ b/backend/tests/test_recipe_mutations.py @@ -0,0 +1,351 @@ +"""9.4 的数据变异（WP-7 验收）：在 XML 层改 D00 的字节（tests/fixtures/xlsx/mutate.py），按参考配方重放。 + +每个用例先把 D00（2026-08-01 起 31 天）按参考配方首次导入并启用，再把变异后的字节作为「上传新一期」重放： +上传新一期自动试运行，回来时已带状态、问题、核对、回执和差异卡。期望照 P2-SPEC 9.4 的表逐条断言， +另加 MUTATIONS 里的「外加」用例 PKDUP（同一分段里一行写 8-9、另一行写 8－9）。 + +拒收的用例另外断言：不能提交（409 trial_not_passed），当前快照没变（9.7 第 4 步「拒收的确认当前快照没变」）。 +夹具全部合成、假名；不调用任何模型。 +""" +from __future__ import annotations + +import datetime as dt +import json +import re +import uuid +from pathlib import Path +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.core.config import settings +from app.data import table_versions +from app.data.engine import engines, run_query +from app.db.base import SessionLocal +from app.db.models import DataSource, SourceSnapshot +from app.main import app +from tests.fixtures.xlsx.flow import FLOW_EXPECT, flow_workbook +from tests.fixtures.xlsx.mutate import MUTATIONS, mutate + +XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +FLOW_RECIPE = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text("utf-8")) +AUG = dt.date(2026, 8, 1) +SHEET = "客流汇总" +TOTALS_TABLE = "时段客流_表内合计" + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + """每个测试一个独立的数据目录：上传目录、原件存档、试运行库、工件都在里面。""" + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +# -------------------------------------------------------------------------- +# 工具 +# -------------------------------------------------------------------------- + + +def _cell(ref: str) -> str: + return f"{SHEET}!{ref}" + + +def _confirm_ids(st: dict[str, Any]) -> list[str]: + return [i["id"] for i in st["trial"]["confirm_items"]] + + +def _problems(trial: dict[str, Any], code: str) -> list[dict[str, Any]]: + return [p for p in trial["problems"] if p["code"] == code] + + +def _check(trial: dict[str, Any], cid: str) -> dict[str, Any] | None: + return next((c for c in trial["checks"] if c["id"] == cid), None) + + +async def _baseline(client, seed: int) -> dict[str, Any]: + """D00 按参考配方首次导入并启用：暂存 → PUT 参考配方 → 试运行（passed）→ 勾全部确认项提交。""" + raw, fn = flow_workbook(AUG, 31, seed=seed) + name = f"wp7mut_{uuid.uuid4().hex[:10]}" + resp = await client.post("/api/datasources/imports/stage", files={"file": (fn, raw, XLSX)}, data={"name": name}) + assert resp.status_code == 201, resp.text + sid = resp.json()["id"] + resp = await client.put(f"/api/datasources/imports/{sid}/recipe", json={"recipe": FLOW_RECIPE}) + assert resp.status_code == 200, resp.text + assert resp.json()["recipe_problems"] == [] + resp = await client.post(f"/api/datasources/imports/{sid}/trial", json={}) + assert resp.status_code == 200, resp.text + st = resp.json() + trial = st["trial"] + assert trial["status"] == "passed", trial["problems"] + assert {t["name"]: t["rows"] for t in trial["receipt"]["tables"]} == FLOW_EXPECT["rows"] + resp = await client.post(f"/api/datasources/imports/{sid}/commit", + json={"trial_id": trial["trial_id"], "confirmations": _confirm_ids(st)}) + assert resp.status_code == 201, resp.text + out = resp.json() + return {"raw": raw, "fn": fn, "source_id": out["source"]["id"], "snapshot_id": out["snapshot_id"], + "import_id": out["import_id"], "table_hashes": trial["receipt"]["table_hashes"]} + + +async def _replay(client, base: dict[str, Any], case: str) -> dict[str, Any]: + """变异后的 D00 作为「上传新一期」重放（文件名不变），返回自动试运行后的暂存区。""" + raw = mutate(base["raw"], case) + assert raw != base["raw"] + resp = await client.post(f"/api/datasources/{base['source_id']}/reupload", + files={"file": (base["fn"], raw, XLSX)}) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["kind"] == "reupload" and st["trial"] is not None, st + return st + + +async def _current_snapshot(source_id: str) -> str | None: + async with SessionLocal() as session: + row = await session.get(DataSource, source_id) + return row.current_snapshot_id + + +async def _assert_rejected(client, base: dict[str, Any], st: dict[str, Any], codes: tuple[str, ...]) -> None: + """拒收：状态 rejected、期望的问题 code 都在、提交不了、当前快照没变。""" + trial = st["trial"] + assert trial["status"] == "rejected", trial["problems"] + assert st["status"] == "rejected" + got = {p["code"] for p in trial["problems"]} | {c["id"] for c in trial["checks"] if c["status"] != "passed"} + for code in codes: + assert code in got, (code, trial["problems"], trial["checks"]) + # 拒收时不算差异卡和确认项（7.6） + assert trial["diff"] is None and trial["confirm_items"] == [] + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": trial["trial_id"], "confirmations": []}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_not_passed", resp.text + assert await _current_snapshot(base["source_id"]) == base["snapshot_id"] + + +async def _scalar(source_id: str, sql: str) -> Any: + async with SessionLocal() as session: + row = await session.get(DataSource, source_id) + view = await table_versions.resolve_source(session, row) + try: + return (await run_query(view, sql)).rows[0][0] + finally: + await engines.invalidate(view.id) + + +def _table_meta(schema_cache: dict[str, Any], table: str) -> dict[str, Any]: + """schema_cache 里某张表的结构（introspect 的形状：tables = {表名: {comment, columns: [...]}}）。""" + tables = schema_cache.get("tables") or {} + assert table in tables, sorted(tables) + return tables[table] + + +# -------------------------------------------------------------------------- +# 夹具本身：MUTATIONS 照 9.4 抄录 +# -------------------------------------------------------------------------- + + +def test_mutation_table_matches_spec(): + expect = { + "P1": ("passed", ()), "P1b": ("needs_decision", ("K1",)), "P2": ("rejected", ("value_blank",)), + "P6": ("rejected", ("axis_duplicate",)), "P9": ("rejected", ("K1", "G1")), + "P20": ("rejected", ("row_unclaimed",)), "P23": ("passed", ()), "P24": ("passed", ()), + "P25": ("rejected", ("axis_extra_cells",)), "P26": ("rejected", ("axis_coverage",)), + "PKDUP": ("rejected", ("pk_duplicate", "label_duplicate")), + } + assert {k: v[1:] for k, v in MUTATIONS.items()} == expect + + +# -------------------------------------------------------------------------- +# 能通过的变异 +# -------------------------------------------------------------------------- + + +async def test_p1_totals_without_formula_pass_by_label_range(client): + """P1：第 28–30 行去掉公式、保留缓存值 → passed（K1 按标签区间核对通过，derived_form=literal）。""" + base = await _baseline(client, seed=21) + st = await _replay(client, base, "P1") + trial = st["trial"] + assert trial["status"] == "passed", trial["problems"] + assert [p for p in trial["problems"] if p["category"] in ("structure", "recipe", "input")] == [] + k1 = _check(trial, "K1") + assert k1 is not None and k1["status"] == "passed" and k1["checked"] == 93, k1 + # 写死的合计没有公式：不出 G1（9.2 D07 同理） + assert _check(trial, "G1") is None + assert trial["receipt"]["derived_form"] == {"夜间合计": "literal"} + # 值没变：三张表的哈希与 D00 相同 + assert trial["receipt"]["table_hashes"] == base["table_hashes"] + # 合计从公式变成写死的数：差异卡要求确认（同 9.2 D07 的 derived_form；G1 消失记 checks） + kinds = {d["kind"] for d in trial["diff"]} + assert kinds == {"checks", "derived_form"}, trial["diff"] + assert "diff:derived_form:夜间合计" in _confirm_ids(st) + + +async def test_p1b_uncached_totals_need_decision_then_commit_with_null_totals(client): + """P1b：第 28–30 行保留公式、去掉缓存值 → needs_decision：K1 unverifiable（uncached: 93，可接受）、 + G1 passed；写理由接受后提交，合计表的客流全为 NULL、表结构保留，说明写「未能核对」、不写「一致」。""" + base = await _baseline(client, seed=22) + st = await _replay(client, base, "P1b") + trial = st["trial"] + assert trial["status"] == "needs_decision", trial["problems"] + k1 = _check(trial, "K1") + assert k1 is not None and k1["status"] == "unverifiable", k1 + assert k1["reasons"] == {"uncached": 93} and k1["acceptable"] is True + g1 = _check(trial, "G1") + assert g1 is not None and g1["status"] == "passed", g1 + assert trial["acceptable"] == ["K1"] + # 试运行时的说明预览（还没接受）也不能说「一致」 + preview = trial["notes"][TOTALS_TABLE]["comment"] + assert "一致" not in preview, preview + + # 不带接受 → 422 acceptance_required；理由是空白 → 同样 422 + confirms = _confirm_ids(st) + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": trial["trial_id"], "confirmations": confirms}) + assert resp.status_code == 422 and resp.json()["code"] == "acceptance_required", resp.text + assert "K1" in resp.json()["detail"] + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": trial["trial_id"], "confirmations": confirms, + "acceptances": [{"check_id": "K1", "reason": " "}]}) + assert resp.status_code == 422 and resp.json()["code"] == "acceptance_required", resp.text + assert await _current_snapshot(base["source_id"]) == base["snapshot_id"] + + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": trial["trial_id"], "confirmations": confirms, + "acceptances": [{"check_id": "K1", "reason": "合成理由：导出工具未保存计算结果"}]}) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["snapshot_id"] != base["snapshot_id"] + sid = base["source_id"] + # 合计表：行数照旧 93，客流全为 NULL；明细表不受影响 + assert await _scalar(sid, f'SELECT COUNT(*) FROM "{TOTALS_TABLE}"') == 93 + assert await _scalar(sid, f'SELECT COUNT("客流") FROM "{TOTALS_TABLE}"') == 0 + assert await _scalar(sid, 'SELECT COUNT(*) FROM "时段客流"') == FLOW_EXPECT["rows"]["时段客流"] + async with SessionLocal() as session: + snap = await session.get(SourceSnapshot, out["snapshot_id"]) + cache = snap.schema_cache + meta = _table_meta(cache, TOTALS_TABLE) + cols = [c["name"] for c in meta["columns"]] + assert cols == ["日期", "合计项", "起始小时", "结束小时", "客流"], cols + comment = meta.get("comment") or "" + assert "本期原表未保存计算结果，合计值为空，未能核对" in comment, comment + assert "一致" not in comment, comment + for c in meta["columns"]: + assert "一致" not in (c.get("comment") or ""), c + + +async def test_p23_dimension_cut_rows_still_reads_everything(client): + """P23： 截到第 27 行 → passed，table_hashes 与 D00 完全相同。""" + base = await _baseline(client, seed=23) + st = await _replay(client, base, "P23") + trial = st["trial"] + assert trial["status"] == "passed", trial["problems"] + assert trial["receipt"]["table_hashes"] == base["table_hashes"] + assert {t["name"]: t["rows"] for t in trial["receipt"]["tables"]} == FLOW_EXPECT["rows"] + # 陈旧的 不影响任何东西：内容与上一期逐格相同，没有差异 + assert trial["diff"] == [], trial["diff"] + + +async def test_p24_dimension_cut_columns_still_reads_everything(client): + """P24： 截到倒数第二个日期列 → passed，table_hashes 与 D00 完全相同。""" + base = await _baseline(client, seed=24) + st = await _replay(client, base, "P24") + trial = st["trial"] + assert trial["status"] == "passed", trial["problems"] + assert trial["receipt"]["table_hashes"] == base["table_hashes"] + assert {t["name"]: t["rows"] for t in trial["receipt"]["tables"]} == FLOW_EXPECT["rows"] + assert trial["diff"] == [], trial["diff"] + + +# -------------------------------------------------------------------------- +# 拒收的变异 +# -------------------------------------------------------------------------- + + +async def test_p2_blank_value_is_rejected_at_d15(client): + """P2：D15 改成空 → rejected value_blank（D15）。""" + base = await _baseline(client, seed=25) + st = await _replay(client, base, "P2") + await _assert_rejected(client, base, st, ("value_blank",)) + hits = _problems(st["trial"], "value_blank") + assert len(hits) == 1 and hits[0]["category"] == "structure" + assert hits[0]["cells"] == [_cell("D15")], hits[0] + + +async def test_p6_duplicate_axis_date_is_rejected(client): + """P6：R4 改成「8月15日」（与 Q4 重复）→ rejected axis_duplicate。""" + base = await _baseline(client, seed=26) + st = await _replay(client, base, "P6") + await _assert_rejected(client, base, st, ("axis_duplicate",)) + hit = _problems(st["trial"], "axis_duplicate")[0] + assert hit["category"] == "structure" + assert {_cell("Q4"), _cell("R4")} <= set(hit["cells"]), hit + + +async def test_p9_wrong_formula_range_fails_k1_and_g1(client): + """P9：C28 改成 =SUM(C22:C24)，缓存值自洽 → rejected：K1 mismatch（C28）且 G1 mismatch。""" + base = await _baseline(client, seed=27) + st = await _replay(client, base, "P9") + await _assert_rejected(client, base, st, ("K1", "G1")) + trial = st["trial"] + k1, g1 = _check(trial, "K1"), _check(trial, "G1") + for c in (k1, g1): + assert c["status"] == "mismatch" and c["category"] == "structure" and c["acceptable"] is False, c + assert c["cells"] == [_cell("C28")], c + assert c["failed"] == 1, c + # 只有核对没过：执行器本身没有结构问题 + assert [p for p in trial["problems"] if p["category"] in ("structure", "recipe")] == [] + + +async def test_p20_unclaimed_data_row_32_is_rejected(client): + """P20：第 32 行加「备用分区（人次）」和两个数， 不更新 → rejected row_unclaimed（第 32 行）。""" + base = await _baseline(client, seed=28) + st = await _replay(client, base, "P20") + await _assert_rejected(client, base, st, ("row_unclaimed",)) + hits = _problems(st["trial"], "row_unclaimed") + assert len(hits) == 1, hits + assert hits[0]["category"] == "structure" + assert all(re.fullmatch(rf"{SHEET}![A-Z]+32", c) for c in hits[0]["cells"]), hits[0] + assert _cell("B32") in hits[0]["cells"] + + +async def test_p25_extra_total_column_right_of_axis_is_rejected(client): + """P25：AH4「合计」、AH 列各数据行写 999， 不更新 → rejected axis_extra_cells（AH4）。""" + base = await _baseline(client, seed=29) + st = await _replay(client, base, "P25") + await _assert_rejected(client, base, st, ("axis_extra_cells",)) + hit = _problems(st["trial"], "axis_extra_cells")[0] + assert hit["category"] == "structure" and _cell("AH4") in hit["cells"], hit + assert hit["fix"] == "ignore_cells" + + +async def test_p26_missing_last_date_column_fails_coverage(client): + """P26：删掉 AG 列所有格，B2 仍写到 31 日 → rejected axis_coverage。""" + base = await _baseline(client, seed=30) + st = await _replay(client, base, "P26") + await _assert_rejected(client, base, st, ("axis_coverage",)) + hit = _problems(st["trial"], "axis_coverage")[0] + assert hit["category"] == "structure" + assert "30" in hit["message"] and "31" in hit["message"], hit + + +async def test_pkdup_mixed_dash_labels_report_pk_and_label_duplicates(client): + """9.4 外加：同一分段里一行写 8-9、另一行写 8－9 → rejected pk_duplicate（两处坐标），label_duplicate + 也出现（4.2 第 3 步：定位失败的分段跳过，其余照常，一次报全）。""" + base = await _baseline(client, seed=31) + st = await _replay(client, base, "PKDUP") + await _assert_rejected(client, base, st, ("pk_duplicate", "label_duplicate")) + trial = st["trial"] + pk = _problems(trial, "pk_duplicate") + assert len(pk) == 1 and pk[0]["category"] == "structure", pk + # 两处坐标：B11（8-9）和 B12（8－9）那两行同一日期的格子 + assert {_cell("C11"), _cell("C12")} <= set(pk[0]["cells"]), pk[0] + assert "C11" in pk[0]["message"] and "C12" in pk[0]["message"], pk[0]["message"] + dup = _problems(trial, "label_duplicate") + assert dup and _cell("B12") in dup[0]["cells"], dup diff --git a/backend/tests/test_recipe_notes.py b/backend/tests/test_recipe_notes.py new file mode 100644 index 00000000..e56e861c --- /dev/null +++ b/backend/tests/test_recipe_notes.py @@ -0,0 +1,776 @@ +"""说明生成（recipe_notes）、写进表结构（apply_notes）与工具描述、冻结白名单（tools/datasource）的测试。 + +不依赖执行器和核对模块的库：核对结果、Extraction 都在测试里手写（假名）。 +""" +from __future__ import annotations + +import json +from copy import deepcopy +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +import app.db.models # noqa: F401 - 会话级建表（conftest）按已导入的模型建：冻结 schema_snapshot 要用工件表 +from app.data import recipe_parsers as P +from app.data.recipe_notes import CONSISTENT_KEYS, COLUMN_TEXT, NOTE_TEXT, apply_notes, build_notes +from app.data.recipe_types import ( + Acceptance, + AxisOut, + CheckResult, + DerivedItem, + Extraction, + OutsideText, + PeriodOut, + Problem, + Recipe, + SchemaNotes, + SheetsOut, + prose_problems, +) + +FLOW = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text(encoding="utf-8")) +SHEET = "客流汇总" +TOTALS = "时段客流_表内合计" +NULLS = {"日客流": {"日期": 0, "全日客流": 0, "分区甲": 0, "分区乙": 0}, + "时段客流": {"日期": 0, "时段": 0, "时段类别": 0, "起始小时": 0, "结束小时": 0, "客流": 31}, + TOTALS: {"日期": 0, "合计项": 0, "起始小时": 0, "结束小时": 0, "客流": 0}} + + +def chk(cid: str, kind: str, status: str = "passed", category: str = "structure", **kw: Any) -> CheckResult: + return CheckResult(cid, kind, cid, status, category, **kw) + + +def flow_checks(**override: CheckResult) -> list[CheckResult]: + base = { + "C1": chk("C1", "context_agree", category="data_quality"), + "C2": chk("C2", "filename_period", category="data_quality"), + "K1": chk("K1", "derived_sum", checked=93), + "G1": chk("G1", "formula_refs", checked=93), + "R1": chk("R1", "relation_sum_eq", category="data_quality", checked=31), + "R2": chk("R2", "relation_not_comparable", "info", "info", checked=31, failed=31), + **{f"N{i}": chk(f"N{i}", "row_count") for i in (1, 2, 3)}, + **{f"P{i}": chk(f"P{i}", "pk_unique") for i in (1, 2, 3)}, + } + base.update(override) + return [c for c in base.values() if c is not None] + + +def night_item(**kw: Any) -> DerivedItem: + args = dict(kind="label_range_sum", segment="夜间合计", sheet=SHEET, cell="C28", label_raw="18-22 时合计", + label="18-22时合计", value=1, is_formula=True, formula="=SUM(C22:C25)", base_table="时段客流", + base_value="客流", start=18, end=22, key={"日期": "2026-08-01", "时段类别": "夜间"}, + start_col="起始小时", end_col="结束小时", ref_rowids=[1], func="SUM") + args.update(kw) + return DerivedItem(**args) + + +def flow_extraction(**kw: Any) -> Extraction: + args: dict[str, Any] = dict( + ok=True, derived=[night_item()], placeholders={"·": 31}, + axes=[AxisOut("交叉表", 4, "8月1日", "8月31日", 31, "text")], + outside_text=[OutsideText(SHEET, "B3", "客流汇总表", "text")], + sheets=SheetsOut(matched={"s1": SHEET}), + period=PeriodOut("2026-08-01", "2026-08-31", "cells", [f"{SHEET}!B2"]), + ) + args.update(kw) + return Extraction(**args) + + +def notes_for(data: dict | None = None, *, extraction: Extraction | None = None, + checks: list[CheckResult] | None = None, acceptances: list[Acceptance] | None = None, + null_counts: Any = NULLS): + recipe = Recipe.model_validate(deepcopy(data or FLOW)) + return build_notes(recipe, extraction or flow_extraction(), flow_checks() if checks is None else checks, + acceptances or [], null_counts=null_counts) + + +def accept(*ids: str) -> list[Acceptance]: + return [Acceptance(i, "已与业务方核实") for i in ids] + + +# ---------------------------------------------------------------- 参考配方（D00） + + +def test_reference_flow_notes(): + notes = notes_for() + assert notes.problems == [] + assert notes.tables["日客流"].comment == ( + "粒度：每个日期。全日客流 等于 分区甲 与 分区乙 之和（导入时已逐日核对）：求合计只用 全日客流，不要把这几列相加。" + "与 时段客流 口径不同：各时段之和不等于 全日客流，不能互相推算或相加") + assert notes.tables["时段客流"].comment == ( + "粒度：每个日期的每个时段。与 日客流 口径不同：各时段之和不等于 全日客流，不能互相推算或相加。" + "时段 按规范写法存储：起始小时-结束小时，不带单位。时段类别 取自原表的分段标题。" + "原表的合计在 时段客流_表内合计，不要与本表相加") + assert notes.tables[TOTALS].comment == ( + "粒度：每个日期的每个合计项。原表写明的合计，已按明细重算核对一致。" + "各合计项覆盖的时段可能互相重叠：不要彼此相加，也不要与 时段客流 相加") + date = "格式 YYYY-MM-DD，年份按统计期补全" + assert notes.tables["日客流"].columns == {"日期": date, "全日客流": "单位：人次", "分区甲": "单位：人次", + "分区乙": "单位：人次"} + assert notes.tables["时段客流"].columns == { + "日期": date, "客流": "单位：人次。空值表示原表为无数据占位符，不代表零；求和时请同时统计非空个数"} + assert notes.tables[TOTALS].columns == {"日期": date, "客流": "单位：人次"} + # 模板带引用，进清单可复查；渲染后的说明没有引用标记 + assert "{列:全日客流} 等于 {列:分区甲} 与 {列:分区乙} 之和" in notes.templates["日客流"] + assert notes.templates["时段客流.客流"].startswith("单位：{单位:人次}") + # B3「客流汇总表」是纯文字的区域外文字：不加 E7 提示 + assert all("附有说明文字" not in t.comment for t in notes.tables.values()) + + +def test_all_rendered_notes_pass_prose_check_with_known_names(): + recipe = Recipe.model_validate(deepcopy(FLOW)) + notes = notes_for() + known = {"列": {"日期", "全日客流", "分区甲", "分区乙", "时段", "时段类别", "起始小时", "结束小时", "客流", "合计项"}, + "表": {"日客流", "时段客流", TOTALS}, "单位": set(P.UNITS)} + assert set(notes.templates) >= {t.name for t in recipe.tables} + for key, tpl in notes.templates.items(): + assert prose_problems(tpl, known=known) == [], key + + +# ---------------------------------------------------------------- 每个模板都过数字和引用检查 + +_SLOTS = { + "a": "{列:日期}", "b": "{列:时段}", "total": "{列:全日客流}", "parts": "{列:分区甲} 与 {列:分区乙}", "per": "逐日", + "unit": "日期", "other": "{表:时段客流}", "dims": "{列:时段}", "value": "{列:全日客流}", "a_value": "{列:客流}", + "by": "{列:日期}", "dim": "{列:时段}", "start": "{列:起始小时}", "end": "{列:结束小时}", "col": "{列:时段类别}", + "table": "{表:时段客流_表内合计}", "base": "{表:时段客流}", "cols": "{列:现金}、{列:刷卡}", +} +_KNOWN = {"列": {"日期", "时段", "全日客流", "分区甲", "分区乙", "客流", "起始小时", "结束小时", "时段类别", "现金", + "刷卡"}, + "表": {"时段客流", TOTALS}, "单位": set(P.UNITS)} + + +@pytest.mark.parametrize("key", sorted(NOTE_TEXT)) +def test_every_table_template_passes_prose_check(key): + assert prose_problems(NOTE_TEXT[key].format(**_SLOTS), known=_KNOWN) == [] + + +@pytest.mark.parametrize("key", sorted(COLUMN_TEXT)) +@pytest.mark.parametrize("unit", P.UNITS) +def test_every_column_template_passes_prose_check(key, unit): + text = COLUMN_TEXT[key].format(unit="{单位:" + unit + "}", meaning="不适用") + assert prose_problems(text, known=_KNOWN) == [] + + +def test_only_passed_templates_say_consistent(): + """「一致」只许出现在 K / T 通过时才选的模板里。别的模板（包括统计期冲突那两条）一个「一致」都没有： + K / T 没通过的表，说明由这些片段拼成，按子串判断也不会误报。""" + assert CONSISTENT_KEYS <= set(NOTE_TEXT) + for key, text in NOTE_TEXT.items(): + if key not in CONSISTENT_KEYS: + assert "一致" not in text, key + assert all("一致" not in text for text in COLUMN_TEXT.values()) + + +# ---------------------------------------------------------------- 交叉表的表内合计：状态 × 原因 + +_K_EXPECT = { + "uncached": "原表写明的合计。本期原表未保存计算结果，合计值为空，未能核对", + "blank": "原表写明的合计。本期原表合计格为空，未能核对", + "null_detail": "原表写明的合计，按原表保存的值存储。本期明细含无数据占位符，未能核对", + "tiling": "原表写明的合计，按原表保存的值存储。本期明细未能覆盖合计的全部范围，未能核对", + "hidden": "原表写明的合计，按原表保存的值存储。本期存在隐藏行或使用了分类汇总函数，合计口径无法确定，未能核对", +} + + +@pytest.mark.parametrize("reason", sorted(_K_EXPECT)) +@pytest.mark.parametrize("accepted", [True, False]) +def test_k_unverifiable_each_reason_never_says_consistent(reason, accepted): + k = chk("K1", "derived_sum", "unverifiable", unverifiable=93, acceptable=True, reasons={reason: 93}) + notes = notes_for(checks=flow_checks(K1=k), acceptances=accept("K1") if accepted else []) + assert notes.problems == [] + comment = notes.tables[TOTALS].comment + assert _K_EXPECT[reason] in comment + assert "一致" not in comment + assert "各合计项覆盖的时段可能互相重叠" in comment + + +def test_k_mixed_reasons_use_weakest_wording(): + k = chk("K1", "derived_sum", "unverifiable", acceptable=True, reasons={"uncached": 2, "hidden": 1}) + notes = notes_for(checks=flow_checks(K1=k), acceptances=accept("K1")) + comment = notes.tables[TOTALS].comment + assert "原表写明的合计，按原表保存的值存储。本期合计值未能核对" in comment + assert "一致" not in comment and "未保存计算结果" not in comment and "分类汇总" not in comment + + +def test_k_passed_but_g_unverifiable(): + g = chk("G1", "formula_refs", "unverifiable", acceptable=True, reasons={"unrecognized": 3}) + notes = notes_for(checks=flow_checks(G1=g), acceptances=accept("G1")) + assert notes.problems == [] + assert "原表写明的合计，合计值已按明细核对一致，公式引用的格子未能核对" in notes.tables[TOTALS].comment + + +def test_k_passed_without_formulas_has_no_g(): + ex = flow_extraction(derived=[night_item(is_formula=False, formula=None, func=None, ref_rowids=None)]) + notes = notes_for(extraction=ex, checks=flow_checks(G1=None)) + assert notes.problems == [] + assert "原表写明的合计，已按明细重算核对一致" in notes.tables[TOTALS].comment + + +@pytest.mark.parametrize("k,g", [ + (chk("K1", "derived_sum", "unverifiable", acceptable=True, reasons={"unrecognized": 1}), None), + (chk("K1", "derived_sum", "unverifiable", acceptable=True, reasons={}), None), + (chk("K1", "derived_sum", "info", "info"), None), + (chk("K1", "derived_sum", "mismatch", acceptable=True), None), +]) +def test_no_template_records_problem_and_never_falls_back(k, g): + """找不到对应模板就是 bug：记 problems（不能保存），绝不退回到「核对一致」之类的别的说法。""" + override = {} + if k is not None: + override["K1"] = k + if g is not None: + override["G1"] = g + notes = notes_for(checks=flow_checks(**override)) + assert any(TOTALS in p for p in notes.problems) + comment = notes.tables[TOTALS].comment + assert "原表写明的合计" not in comment and "一致" not in comment + + +@pytest.mark.parametrize("override", [ + {"K1": chk("K1", "derived_sum", "mismatch", failed=1)}, + {"G1": chk("G1", "formula_refs", "mismatch", failed=1)}, + {"K1": chk("K1", "derived_sum", "mismatch", failed=1), "G1": chk("G1", "formula_refs", "mismatch", failed=1)}, + # K 不一致、G 无法核对：只要有一条不可接受的结构类失败，整段都不写 + {"K1": chk("K1", "derived_sum", "mismatch", failed=1), + "G1": chk("G1", "formula_refs", "unverifiable", acceptable=True, reasons={"hidden": 1})}, +]) +def test_blocking_structure_failure_skips_total_note_without_problem(override): + """K / G 不一致（D08、P9）：试运行本来就因结构类核对拒收。说明里不写合计那一句（更不会写「一致」）， + 也不记 problems——problems 会被当成「配方不合法 / 说明生成失败」，把用户引去改配方。""" + notes = notes_for(checks=flow_checks(**override)) + assert notes.problems == [] + comment = notes.tables[TOTALS].comment + assert "原表写明的合计" not in comment and "一致" not in comment + # 与核对无关、本身成立的提示照写 + assert "各合计项覆盖的时段可能互相重叠" in comment + + +def test_blocking_relation_failure_skips_relation_note(): + """关系核对的 SQL 出错（_guarded：mismatch、structure、不可接受）：不能当成「关系不成立、尚未确认」去写。""" + broken = {"R1": chk("R1", "relation_sum_eq", "mismatch", details=["核对 SQL 执行失败"]), + "R2": chk("R2", "relation_not_comparable", "mismatch", details=["核对 SQL 执行失败"])} + notes = notes_for(checks=flow_checks(**broken)) + assert notes.problems == [] + daily = notes.tables["日客流"].comment + assert "分区甲" not in daily and "口径不同" not in daily and "尚未确认" not in daily + + +# ---------------------------------------------------------------- 一张合计表由两个 derived 段写入 + + +def two_totals_recipe() -> dict: + """日间、夜间各有一组合计行，都另存进 时段客流_表内合计（P2-SPEC 6.1 允许两段时段合表）。""" + data = deepcopy(FLOW) + segs = data["sheets"][0]["blocks"][0]["segments"] + segs[1]["stop_parser"] = "hour_range_total" + day = deepcopy(segs[3]) + day.update(id="日间合计", locate={"by": "after", "segment": "日间"}, labels={"expect": ["8-12 时合计"]}) + segs.insert(2, day) + return data + + +def day_item(**kw: Any) -> DerivedItem: + args = dict(segment="日间合计", cell="C21", label_raw="8-12 时合计", label="8-12时合计", formula="=SUM(C11:C14)", + start=8, end=12, key={"日期": "2026-08-01", "时段类别": "日间"}) + args.update(kw) + return night_item(**args) + + +def two_totals(**override: CheckResult) -> SchemaNotes: + """K1 / G1 = 日间合计，K2 / G2 = 夜间合计（plan_checks 按配方里分段的先后编号）。""" + base = {"K1": chk("K1", "derived_sum", checked=31), "K2": chk("K2", "derived_sum", checked=93), + "G1": chk("G1", "formula_refs", checked=31), "G2": chk("G2", "formula_refs", checked=93)} + base.update(override) + ex = flow_extraction(derived=[day_item(), night_item()]) + return notes_for(two_totals_recipe(), extraction=ex, checks=flow_checks(**base), acceptances=accept(*base)) + + +_UNC = {"status": "unverifiable", "acceptable": True} + + +def test_two_segments_into_one_total_table_all_passed(): + notes = two_totals() + assert notes.problems == [] + assert "原表写明的合计，已按明细重算核对一致" in notes.tables[TOTALS].comment + + +@pytest.mark.parametrize("k1,k2", [ + ({"reasons": {"uncached": 31}, **_UNC}, {}), # 日间无法核对、夜间通过 + ({}, {"reasons": {"hidden": 2}, **_UNC}), # 反过来 + ({"reasons": {"uncached": 31}, **_UNC}, {"reasons": {"hidden": 2}, **_UNC}), # 两段原因不同 +]) +def test_two_segments_partly_unverifiable_use_mixed_template(k1, k2): + """几个 derived 段写进同一张合计表：只要有一段没通过，就不能写「一致」；一段通过、一段无法核对，或者两段 + 原因不同，都用最弱的「原因混合」说法，不挑其中一段的原因来写。""" + notes = two_totals(K1=chk("K1", "derived_sum", **k1), K2=chk("K2", "derived_sum", **k2)) + assert notes.problems == [] + comment = notes.tables[TOTALS].comment + assert "一致" not in comment + assert "原表写明的合计，按原表保存的值存储。本期合计值未能核对" in comment + assert "未保存计算结果" not in comment and "分类汇总" not in comment + assert comment.count("各合计项覆盖的时段可能互相重叠") == 1 + + +def test_two_segments_same_reason_use_that_reason(): + unc = {"reasons": {"uncached": 3}, **_UNC} + notes = two_totals(K1=chk("K1", "derived_sum", **unc), K2=chk("K2", "derived_sum", **unc)) + assert notes.problems == [] + comment = notes.tables[TOTALS].comment + assert "原表写明的合计。本期原表未保存计算结果，合计值为空，未能核对" in comment and "一致" not in comment + + +def test_two_segments_one_g_unverifiable(): + notes = two_totals(G2=chk("G2", "formula_refs", "unverifiable", acceptable=True, reasons={"unrecognized": 1})) + assert notes.problems == [] + assert "原表写明的合计，合计值已按明细核对一致，公式引用的格子未能核对" in notes.tables[TOTALS].comment + + +def test_two_segments_one_blocking_skips_without_problem(): + notes = two_totals(K1=chk("K1", "derived_sum", **{"reasons": {"uncached": 31}, **_UNC}), + K2=chk("K2", "derived_sum", "mismatch", failed=1)) + assert notes.problems == [] + assert "原表写明的合计" not in notes.tables[TOTALS].comment + + +# ---------------------------------------------------------------- 统计期冲突与「一致」 + + +@pytest.mark.parametrize("reason", sorted(_K_EXPECT)) +def test_period_conflict_with_unverifiable_k_never_says_consistent(reason): + """K 无法核对、C1 / C2 冲突都已接受：合计表的说明里同时有统计期那一句，仍然一个「一致」都没有。""" + k = chk("K1", "derived_sum", "unverifiable", acceptable=True, reasons={reason: 3}) + c1 = chk("C1", "context_agree", "mismatch", "data_quality", acceptable=True) + c2 = chk("C2", "filename_period", "mismatch", "data_quality", acceptable=True) + notes = notes_for(checks=flow_checks(K1=k, C1=c1, C2=c2), acceptances=accept("K1", "C1", "C2")) + assert notes.problems == [] + comment = notes.tables[TOTALS].comment + assert "本期统计期的写法在原表或文件名中有冲突，已由用户确认接受" in comment + assert "一致" not in comment + + +def test_missing_check_is_problem(): + notes = notes_for(checks=flow_checks(R1=None)) + assert any("R1" in p and "日客流" in p for p in notes.problems) + + +# ---------------------------------------------------------------- 关系 + +def test_sum_eq_acceptance_rewrites_note(): + """D26：R1 不成立。接受前后都不再写「已逐日核对」；接受后写「已由用户确认接受」。""" + assert "已逐日核对" in notes_for().tables["日客流"].comment + r1 = chk("R1", "relation_sum_eq", "mismatch", "data_quality", checked=30, failed=1, acceptable=True) + accepted = notes_for(checks=flow_checks(R1=r1), acceptances=accept("R1")) + pending = notes_for(checks=flow_checks(R1=r1)) + assert accepted.problems == [] and pending.problems == [] + a, p = accepted.tables["日客流"].comment, pending.tables["日客流"].comment + assert "已逐日核对" not in a and "已逐日核对" not in p + assert ("全日客流 与 分区甲、分区乙 之和的关系在本期个别日期不成立，已由用户确认接受，详见导入清单：" + "不要据此推算，也不要把这几列相加") in a + assert "尚未确认接受" in p and "已由用户确认接受" not in p + assert "求合计只用" not in a + + +def test_sum_eq_with_nulls(): + r1 = chk("R1", "relation_sum_eq", "unverifiable", "data_quality", unverifiable=2, acceptable=True) + comment = notes_for(checks=flow_checks(R1=r1), acceptances=accept("R1")).tables["日客流"].comment + assert "全日客流 与 分区甲、分区乙 之和的关系在本期部分日期因空值未能核对：不要据此推算，也不要把这几列相加" in comment + assert "已逐日核对" not in comment + + +def test_sum_eq_without_rows_says_so_and_not_checked(): + """AU-1：没有数据行时 R 判无法核对（checked=0、unverifiable=0）：说明写「没有数据行」，不写「已核对」「因空值」。""" + r1 = chk("R1", "relation_sum_eq", "unverifiable", "data_quality", checked=0, unverifiable=0, acceptable=True) + comment = notes_for(checks=flow_checks(R1=r1), acceptances=accept("R1")).tables["日客流"].comment + assert "全日客流 与 分区甲、分区乙 之和的关系在本期没有数据行，未能核对：不要据此推算，也不要把这几列相加" in comment + assert "已逐日核对" not in comment and "因空值" not in comment + + +def test_not_comparable_all_equal_drops_not_equal_wording(): + r2 = chk("R2", "relation_not_comparable", "info", "info", checked=31, failed=0) + notes = notes_for(checks=flow_checks(R2=r2)) + for table in ("日客流", "时段客流"): + comment = notes.tables[table].comment + assert "口径不同，不能互相推算或相加" in comment and "不等于" not in comment + assert "与 时段客流 口径不同" in notes.tables["日客流"].comment + assert "与 日客流 口径不同" in notes.tables["时段客流"].comment + + +def test_not_comparable_flat_variant(): + data = deepcopy(FLOW) + data["relations"][1]["a"] = {"table": "日客流", "value": "分区甲"} + data["relations"][1]["b"] = {"table": TOTALS, "value": "客流"} + notes = notes_for(data) + assert notes.problems == [] + assert "与 时段客流_表内合计 口径不同：分区甲 按 日期 汇总不等于 客流，不能互相推算或相加" in notes.tables["日客流"].comment + + +# ---------------------------------------------------------------- 单位、列名里的数字 + + +def test_digit_names_and_units_pass_checks(): + data = deepcopy(FLOW) + seg = data["sheets"][0]["blocks"][0]["segments"][0] + seg["labels"]["expect"] = ["1月（万元）", "3号门客流（千克）", "指标2（亿元）"] + seg["measures"] = {"1月（万元）": "1月", "3号门客流（千克）": "3号门客流", "指标2（亿元）": "指标2"} + data["tables"][0]["units"] = {"1月": "万元", "3号门客流": "千克", "指标2": "亿元"} + data["relations"][0].update(total="1月", parts=["3号门客流", "指标2"]) + data["relations"][1]["b"]["value"] = "1月" + data["tables"][0]["note"] = "{列:3号门客流} 只统计闸机，{单位:万元} 与 {单位:亿元} 不要混算" + notes = notes_for(data, null_counts=None) + assert notes.problems == [] + comment = notes.tables["日客流"].comment + assert "1月 等于 3号门客流 与 指标2 之和（导入时已逐日核对）" in comment + assert comment.endswith("3号门客流 只统计闸机，万元 与 亿元 不要混算") + assert notes.tables["日客流"].columns["1月"].startswith("单位：万元") + assert notes.tables["日客流"].columns["指标2"].startswith("单位：亿元") + + +def test_unknown_unit_in_column_note_is_problem(): + """列说明同样过引用检查：单位不在系统单位表里（「万人次」），{单位:…} 不遮盖、记问题，不能保存。""" + data = deepcopy(FLOW) + data["tables"][0]["units"]["全日客流"] = "万人次" + notes = notes_for(data) + assert any("全日客流" in p and "不存在的单位「万人次」" in p for p in notes.problems) + + +def test_note_smuggling_numbers_is_problem(): + data = deepcopy(FLOW) + data["tables"][0]["note"] = "{列:增长30%} 要单独看" + notes = notes_for(data) + assert any("日客流" in p and "不存在" in p for p in notes.problems) + assert any("数字" in p for p in notes.problems) + + +# ---------------------------------------------------------------- 本期情况：E7、统计期、隐藏行、忽略列、停止之后 + + +def test_e7_only_for_text_digits(): + plain = notes_for() + assert all("附有说明文字" not in t.comment for t in plain.tables.values()) + for extra in (OutsideText(SHEET, "B32", "注：9月15日闸机故障，当日客流为估算值", "text_digits"), + OutsideText(SHEET, "B3", "客流汇总表（2026年9月）", "text_digits", period_source=True)): + ex = flow_extraction(outside_text=[OutsideText(SHEET, "B3", "客流汇总表", "text"), extra]) + notes = notes_for(extraction=ex) + assert notes.problems == [] + for t in notes.tables.values(): + assert "本期原表附有说明文字（可能涉及口径），内容见证据面板" in t.comment + assert "9月" not in t.comment and "B32" not in t.comment + + +def test_period_conflict_and_human_period(): + c1 = chk("C1", "context_agree", "mismatch", "data_quality", acceptable=True) + accepted = notes_for(checks=flow_checks(C1=c1), acceptances=accept("C1")) + pending = notes_for(checks=flow_checks(C1=c1)) + assert "本期统计期的写法在原表或文件名中有冲突，已由用户确认接受，详见导入清单" in accepted.tables["日客流"].comment + assert "本期统计期的写法在原表或文件名中有冲突，尚未确认接受" in pending.tables["日客流"].comment + human = flow_extraction(period=PeriodOut("2026-09-01", "2026-09-30", "human", signed_by="甲")) + assert "本期统计期为人工录入" in notes_for(extraction=human).tables["时段客流"].comment + assert "人工录入" not in notes_for().tables["时段客流"].comment + + +LIST_RECIPE = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": "销售"}, "hidden": {"rows": "exclude"}, "blocks": [{ + "id": "明细", "layout": "list", "table": "销售", + "columns": [{"header": "日期", "name": "日期", "type": "DATE"}, + {"header": "地区", "name": "地区", "type": "TEXT"}, + {"header": "产品", "name": "产品", "type": "TEXT"}, + {"header": "金额（元）", "name": "金额", "type": "INTEGER"}, + {"header": "现金", "name": "现金", "type": "INTEGER"}, + {"header": "刷卡", "name": "刷卡", "type": "INTEGER"}], + "rows": {"total_row": {"label_column": "地区", "pick": "合计", "keep_as": "销售_表内合计"}}}]}], + "tables": [{"name": "销售", "grain": ["日期", "地区", "产品"], "units": {"金额": "元"}}, + {"name": "销售_表内合计", "kind": "reported_total", "units": {"金额": "元"}}], + "relations": [{"id": "R1", "kind": "sum_eq", "table": "销售", "total": "金额", "parts": ["现金", "刷卡"]}], +} + + +LIST_NUMBERS = ("金额", "现金", "刷卡") + + +def list_item(col: str = "金额") -> DerivedItem: + cell = {"金额": "D9", "现金": "E9", "刷卡": "F9"}.get(col, "D9") + return DerivedItem(kind="column_sum", segment="明细", sheet="销售", cell=cell, label_raw="合计", label="合计", + value=1, is_formula=False, formula=None, base_table="销售", base_value=col, + rowid_first=1, rowid_last=6) + + +def list_notes(t1: CheckResult | None = None, *, data: dict | None = None, accepted: bool = True, + **ex: Any): + checks = [t1 or chk("T1", "column_sum"), chk("R1", "relation_sum_eq", category="data_quality")] + args = dict(ok=True, derived=[list_item(c) for c in LIST_NUMBERS], sheets=SheetsOut(matched={"s1": "销售"}), + lineage={"销售": {"金额": [[1, "销售", "D2", 6, "down"]]}}) + args.update(ex) + recipe = Recipe.model_validate(deepcopy(data or LIST_RECIPE)) + return build_notes(recipe, Extraction(**args), checks, accept("T1") if accepted else []) + + +def test_list_reference_notes(): + notes = list_notes() + assert notes.problems == [] + assert notes.tables["销售"].comment == ( + "粒度：日期、地区 与 产品 的每个组合。金额 等于 现金 与 刷卡 之和（导入时已逐行核对）：求合计只用 金额，" + "不要把这几列相加。原表的合计在 销售_表内合计，不要与本表相加") + assert notes.tables["销售_表内合计"].comment == "原表合计行的值，已按各列明细求和核对一致：不要与 销售 相加" + cols = notes.tables["销售"].columns + assert cols["日期"] == "格式 YYYY-MM-DD" + assert cols["金额"] == "单位：元。空值表示原表为空格，不代表零；求和时请同时统计非空个数" + assert "地区" not in cols + # 列表合计表的空值不写列说明（表说明已交代） + assert notes.tables["销售_表内合计"].columns == {"金额": "单位：元"} + + +_T_EXPECT = { + "uncached": "原表合计行的值。本期原表未保存计算结果，合计值为空，未能核对：不要与 销售 相加", + "blank": "原表合计行的值。本期原表合计格为空，未能核对：不要与 销售 相加", + "tiling": "原表合计行的值，按原表保存的值存储。本期合计行之上没有明细，未能核对：不要与 销售 相加", + "null_detail": "原表合计行的值，按原表保存的值存储。本期明细含空值，未能核对：不要与 销售 相加", + "hidden": "原表合计行的值，按原表保存的值存储。本期存在隐藏行或使用了分类汇总函数，合计口径无法确定，未能核对：不要与 销售 相加", +} + + +@pytest.mark.parametrize("reason", sorted(_T_EXPECT)) +def test_t_unverifiable_each_reason_never_says_consistent(reason): + notes = list_notes(chk("T1", "column_sum", "unverifiable", acceptable=True, reasons={reason: 1})) + assert notes.problems == [] + comment = notes.tables["销售_表内合计"].comment + assert comment == _T_EXPECT[reason] + assert "一致" not in comment + + +def test_t_mixed_and_missing_templates(): + mixed = list_notes(chk("T1", "column_sum", "unverifiable", acceptable=True, reasons={"uncached": 1, "hidden": 1})) + assert mixed.tables["销售_表内合计"].comment == "原表合计行的值，按原表保存的值存储。本期未能核对：不要与 销售 相加" + # 列表合计没有「公式写法无法识别」这种原因（那是公式引用 G 的）：真出现了就是没有模板，记问题 + odd = list_notes(chk("T1", "column_sum", "unverifiable", acceptable=True, reasons={"unrecognized": 1})) + assert any("T1" in p for p in odd.problems) + assert "一致" not in odd.tables["销售_表内合计"].comment + # T 不一致：试运行已因结构类核对拒收，不写这一句、不记问题 + mismatch = list_notes(chk("T1", "column_sum", "mismatch", failed=1)) + assert mismatch.problems == [] + assert "原表合计行的值" not in mismatch.tables["销售_表内合计"].comment + + +def test_list_total_with_blank_cells_does_not_claim_every_column(): + """合计行里 现金、刷卡 的格子是空的（P2-SPEC 4.4：空格不生成合计格），T 只核对了 金额 并通过： + 不能写「各列都核对一致」，要点名哪几列的合计格为空。只看合计格，不依赖 null_counts。""" + notes = list_notes(derived=[list_item("金额")]) + assert notes.problems == [] + assert notes.tables["销售_表内合计"].comment == ( + "原表合计行的值。本期原表 现金、刷卡 的合计格为空，其余各列已按明细求和核对一致：不要与 销售 相加") + assert "各列明细求和核对一致" not in notes.tables["销售_表内合计"].comment + one = list_notes(derived=[list_item("金额"), list_item("刷卡")]) + assert "本期原表 现金 的合计格为空，其余各列" in one.tables["销售_表内合计"].comment + + +def test_list_total_not_kept(): + data = deepcopy(LIST_RECIPE) + data["sheets"][0]["blocks"][0]["rows"]["total_row"]["keep_as"] = None + data["tables"] = data["tables"][:1] + notes = list_notes(data=data) + assert notes.problems == [] + assert "原表的合计行未导入本表" in notes.tables["销售"].comment + assert "销售_表内合计" not in notes.tables + + +def test_crosstab_total_not_kept(): + data = deepcopy(FLOW) + data["sheets"][0]["blocks"][0]["segments"][3]["keep_as"] = None + data["tables"] = data["tables"][:2] + notes = notes_for(data) + assert notes.problems == [] + assert "原表的合计行未导入本表" in notes.tables["时段客流"].comment + + +def test_hidden_rows_excluded_and_included(): + excluded = list_notes(hidden={"销售": {"rows": [4], "cols": [], "policy_rows": "exclude"}}) + assert "本表不含原表中被隐藏的行" in excluded.tables["销售"].comment + data = deepcopy(LIST_RECIPE) + data["sheets"][0]["hidden"] = {"rows": "include"} + included = list_notes(data=data, hidden={"销售": {"rows": [4], "cols": [], "policy_rows": "include"}}) + assert "本表包含原表中被隐藏的行" in included.tables["销售"].comment + # 隐藏行在本表占的行之外：不写 + far = list_notes(data=data, hidden={"销售": {"rows": [40], "cols": [], "policy_rows": "include"}}) + assert "隐藏" not in far.tables["销售"].comment + assert "隐藏" not in list_notes().tables["销售"].comment + + +def test_ignored_columns_and_rows_after_stop(): + data = deepcopy(LIST_RECIPE) + data["sheets"][0]["blocks"][0]["extra_columns"] = "ignore" + notes = list_notes(data=data, ignored_columns={"明细": ["备注"]}, problems=[Problem( + "rows_after_stop", "confirm", "第 9 行起有 2 行文字在空行之后，没有导入", cells=["销售!A9", "销售!B9"])]) + comment = notes.tables["销售"].comment + assert "原表另有未导入的列" in comment + assert "原表在空行之后另有未导入的文字行，内容见证据面板" in comment + assert "备注" not in comment + assert "未导入的列" not in list_notes(data=data).tables["销售"].comment + + +def test_rows_after_stop_goes_to_block_above(): + data = deepcopy(LIST_RECIPE) + second = deepcopy(data["sheets"][0]["blocks"][0]) + second.update(id="费用", table="费用", after_title="二、费用") + second["rows"] = {} + data["sheets"][0]["blocks"].append(second) + data["tables"].append({"name": "费用"}) + lineage = {"销售": {"金额": [[1, "销售", "D2", 6, "down"]]}, "费用": {"金额": [[1, "销售", "D20", 3, "down"]]}} + notes = list_notes(data=data, lineage=lineage, problems=[Problem( + "rows_after_stop", "confirm", "第 25 行起有 1 行文字在空行之后，没有导入", cells=["销售!A25", "销售!B25"])]) + assert "空行之后" in notes.tables["费用"].comment + assert "空行之后" not in notes.tables["销售"].comment + + +# ---------------------------------------------------------------- 列说明：空值、日期、单位 + + +def test_null_notes_follow_actual_nulls_when_counts_given(): + """给了每列空值数：只有本期真有空值的值列写空值说明（全日客流 只写单位）。不给时按配方推断。""" + with_counts = notes_for() + assert with_counts.tables["日客流"].columns["全日客流"] == "单位：人次" + without = notes_for(null_counts=None) + assert without.tables["日客流"].columns["全日客流"].startswith("单位：人次。空值表示原表为无数据占位符") + + +def test_four_arg_call_without_null_counts_is_conservative(): + """规格里 build_notes 是 4 参数调用（P2-SPEC 5.4、7.8 第 3 步）。Extraction 里的占位符计数是整份文件的、 + 不分列，4 参数调用时只能对「可能有空值」的值列都写空值那一句，宁多勿漏。 + + 这条钉住这层依赖：9.7 第 2 步要的 全日客流 列说明是「单位：人次」，只有调用方把 + recipe_checks.column_null_counts 的结果作为 null_counts 传进来才成立（试运行预览和提交都要传）。 + """ + recipe = Recipe.model_validate(deepcopy(FLOW)) + four = build_notes(recipe, flow_extraction(), flow_checks(), []) + assert four.problems == [] + cols = four.tables["日客流"].columns + assert cols["全日客流"] == "单位：人次。空值表示原表为无数据占位符，不代表零；求和时请同时统计非空个数" + assert four.tables["时段客流"].columns["客流"].startswith("单位：人次。空值表示原表为无数据占位符") + # 合计表的空值由表说明交代，哪种调用都不写 + assert four.tables[TOTALS].columns["客流"] == "单位：人次" + given = build_notes(recipe, flow_extraction(), flow_checks(), [], null_counts=NULLS) + assert given.tables["日客流"].columns["全日客流"] == "单位：人次" + # 表说明不受影响 + assert {t: n.comment for t, n in four.tables.items()} == {t: n.comment for t, n in given.tables.items()} + + +def test_null_note_variants(): + data = deepcopy(FLOW) + data["sheets"][0]["blocks"][0]["values"]["placeholders"] = [{"text": "—", "meaning": "不适用"}] + ex = flow_extraction(placeholders={"—": 31}) + assert notes_for(data, extraction=ex).tables["时段客流"].columns["客流"] == ( + "单位：人次。空值表示原表为不适用占位符，不代表零；求和时请同时统计非空个数") + data["sheets"][0]["blocks"][0]["values"]["blank"] = "null" + data["sheets"][0]["blocks"][0]["values"]["placeholders"].append({"text": "·"}) + ex = flow_extraction(placeholders={"—": 3, "·": 4}) + assert notes_for(data, extraction=ex).tables["时段客流"].columns["客流"] == ( + "单位：人次。空值表示原表为无数据或不适用占位符或空格，不代表零；求和时请同时统计非空个数") + # 配了占位符但本期没出现、又不收空格：没有空值来源，不写 + ex = flow_extraction(placeholders={}) + data["sheets"][0]["blocks"][0]["values"]["blank"] = "reject" + assert notes_for(data, extraction=ex, null_counts=None).tables["时段客流"].columns["客流"] == "单位：人次" + + +def test_axis_with_own_year_drops_year_hint(): + ex = flow_extraction(axes=[AxisOut("交叉表", 4, "2026-08-01", "2026-08-31", 31, "date")]) + assert notes_for(extraction=ex).tables["日客流"].columns["日期"] == "格式 YYYY-MM-DD" + data = deepcopy(FLOW) + data["sheets"][0]["blocks"][0]["axis"]["year_from"] = None + assert notes_for(data).tables["日客流"].columns["日期"] == "格式 YYYY-MM-DD" + + +def test_hour_range_without_derive_uses_plain_template(): + data = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": SHEET}, "blocks": [{ + "id": "交叉表", "layout": "crosstab", "axis": {"year_from": None}, + "segments": [{"id": "日间", "role": "dimension", "table": "时段客流", + "locate": {"by": "section_title", "title": "日间时段客流（人次）"}, + "labels": {"expect": ["7-8", "8-9"]}, "dim": {"name": "时段", "parser": "hour_range"}, + "value": "客流"}]}]}], + "tables": [{"name": "时段客流", "grain": ["日期", "时段"]}], + } + notes = build_notes(Recipe.model_validate(data), Extraction(ok=True), [], []) + assert notes.problems == [] + assert "时段 按规范写法存储：起始小时与结束小时用连字符相连，不带单位" in notes.tables["时段客流"].comment + + +# ---------------------------------------------------------------- apply_notes + + +def _cache() -> dict[str, Any]: + def cols(*names): + return [{"name": n, "type": "INTEGER", "nullable": True} for n in names] + + return {"tables": { + "日客流": {"columns": cols("日期", "全日客流", "分区甲", "分区乙"), "primary_key": ["日期"]}, + "时段客流": {"columns": cols("日期", "时段", "时段类别", "起始小时", "结束小时", "客流"), + "primary_key": ["日期", "时段"]}, + TOTALS: {"columns": cols("日期", "合计项", "起始小时", "结束小时", "客流"), "primary_key": ["日期", "合计项"]}, + }, "synced_at": "2026-09-01T00:00:00"} + + +def test_apply_notes_writes_comments_without_touching_input(): + cache = _cache() + frozen = json.dumps(cache, sort_keys=True, ensure_ascii=False) + notes = notes_for() + out = apply_notes(cache, notes) + assert json.dumps(cache, sort_keys=True, ensure_ascii=False) == frozen + assert out["import_mode"] == "recipe" and out["synced_at"] == cache["synced_at"] + assert out["tables"]["日客流"]["comment"] == notes.tables["日客流"].comment + cols = {c["name"]: c for c in out["tables"]["时段客流"]["columns"]} + assert cols["客流"]["comment"] == notes.tables["时段客流"].columns["客流"] + assert "comment" not in cols["时段"] + assert notes.problems == [] + + +def test_apply_notes_records_unknown_tables_and_columns(): + cache = _cache() + del cache["tables"][TOTALS] + cache["tables"]["日客流"]["columns"] = cache["tables"]["日客流"]["columns"][:2] + notes = notes_for() + out = apply_notes(cache, notes) + assert any(TOTALS in p for p in notes.problems) + assert any("分区甲" in p for p in notes.problems) + assert TOTALS not in out["tables"] + + +# ---------------------------------------------------------------- 工具描述与冻结白名单 + + +def _source(cache: dict[str, Any]) -> SimpleNamespace: + return SimpleNamespace(name="passenger_flow", description="", readonly=True, schema_cache=cache, kind="sqlite") + + +def test_query_description_adds_hint_only_for_recipe_sources(): + from app.tools.datasource import RECIPE_QUERY_HINT, _query_description + + recipe = _query_description(_source({**_cache(), "import_mode": "recipe"})) + manual = _query_description(_source(_cache())) + simple = _query_description(_source({**_cache(), "import_mode": "simple"})) + hint = "中文表名和列名请加双引号。查单个值时，把主键列一起选出来，证据面板才能追到原表格子。" + assert hint in recipe and RECIPE_QUERY_HINT.strip() == hint + assert recipe.index("可用表") < recipe.index(hint) + assert hint not in manual and hint not in simple + assert recipe.replace(RECIPE_QUERY_HINT, "") == manual + assert prose_problems(hint) == [] + + +async def test_store_schema_freezes_import_manifests_and_keeps_manual_sources_identical(): + from app.core.artifact_store import canonical_json, content_hash, load + from app.tools.datasource import _store_schema + + ctx = SimpleNamespace(run_id="", node_id="") + manual = {**_cache(), "schema": "main", "truncated": False, "total": 3, "available_schemas": ["x"]} + got = await _store_schema(_source(manual), ctx) + # 改动前的写法：只复制 schema、synced_at、truncated、total 四个键 + before = {"source": "passenger_flow", "tables": manual["tables"], + **{k: manual[k] for k in ("schema", "synced_at", "truncated", "total") if k in manual}} + assert got == content_hash(canonical_json(before)) + + recipe = {**_cache(), "import_mode": "recipe", "import_manifests": ["m" * 64]} + frozen_id = await _store_schema(_source(recipe), ctx) + frozen = load(frozen_id) + assert frozen["import_manifests"] == ["m" * 64] and frozen["import_mode"] == "recipe" + assert frozen_id != got diff --git a/backend/tests/test_recipe_perf.py b/backend/tests/test_recipe_perf.py new file mode 100644 index 00000000..2721306b --- /dev/null +++ b/backend/tests/test_recipe_perf.py @@ -0,0 +1,340 @@ +"""9.5 的性能（WP-7）：合成 20 万行 × 10 列的列表（同 p2-probe/perf.py 的结构），量 4.9 的各项目标。 + +默认跳过；`RECIPE_PERF=1` 时跑（仓库里没有现成的慢测试标记，按规格用环境变量开关）。量到的数字打印出来 +（加 -s 看得到），也经 record_property 进 junit 报告。判定照 9.5：超过目标 1.5 倍算不通过（断言失败）， +在 1–1.5 倍之间记为「待优化」（只打印，不失败）。 + +量的都是真管线（default_pipeline）经接口走一遍的耗时，和用户点按钮时服务端做的事一样： +- 暂存（POST imports/stage：扫描 + 预览 + 规则起草 + 部分干跑）≤ 扫描时间 + 3 s； +- 单次改配方 / 回答问题（静态校验 + 部分干跑）≤ 1 s； +- 暂存后的试运行（复用 scan_from_json，不再扫描）≤ 18 s，进程内存增量 ≤ 200 MB； +- 上传新一期（扫描 + 试运行）≤ 25 s；整列有公式（有缓存值）的同样大小 ≤ 40 s（整列公式的表暂存后试运行 + 规格没给目标，只记录）； +- 结构仿照客流表的夹具：暂存 ≤ 1 s，单次回答 ≤ 0.5 s，试运行 ≤ 1 s。 +20 万行的夹具与 p2-probe/perf.py 一样用 openpyxl 的 write_only 写出，它不写 ``；Excel 存盘的文件都有 +这个元素。openpyxl 只读模式打开工作簿时，缺 `` 的工作表会被整张解析一遍（parse_dimensions 一直读到 +），所以每项都分「无 dimension（同 perf.py）」和「有 dimension（同 Excel 存盘）」两种各量一次， +目标对两种一视同仁。 +夹具全部合成（假名、随机数），不调用任何模型。 +""" +from __future__ import annotations + +import datetime as dt +import io +import json +import os +import random +import re +import subprocess +import threading +import time +import uuid +from pathlib import Path +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient + +from app.core.config import settings +from app.data import xlsx_cells, xlsx_scan +from app.main import app +from tests.fixtures.xlsx import patch_parts, sheet_part +from tests.fixtures.xlsx.flow import flow_workbook + +pytestmark = pytest.mark.skipif(os.environ.get("RECIPE_PERF") != "1", + reason="性能测试默认跳过：RECIPE_PERF=1 时运行（20 万行的夹具要生成十几秒）") + +XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +ROWS = 200_000 +SHEET = "明细" +HEADERS = ["地区", "产品", "渠道", "门店", "日期", "销量", "单价", "金额", "成本", "备注"] +TYPES = ["TEXT", "TEXT", "TEXT", "TEXT", "DATE", "INTEGER", "REAL", "INTEGER", "INTEGER", "TEXT"] +FLOW_RECIPE = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text("utf-8")) + +#: 超过目标的这个倍数算不通过（9.5） +FAIL_FACTOR = 1.5 + +RESULTS: list[dict[str, Any]] = [] + + +# -------------------------------------------------------------------------- +# 夹具 +# -------------------------------------------------------------------------- + + +def big_list(seed: int, *, formulas: bool = False) -> bytes: + """20 万行 × 10 列的列表：一列日期，无公式；formulas=True 时「金额」整列是公式（=销量+成本），并在 + 压缩包层面补上缓存值、去掉 fullCalcOnLoad（模拟 Excel 保存过）。""" + from openpyxl import Workbook + + rnd = random.Random(seed) + wb = Workbook(write_only=True) + ws = wb.create_sheet(SHEET) + ws.append(HEADERS) + cached: dict[int, int] = {} + base = dt.date(2026, 8, 1) + for i in range(ROWS): + r = i + 2 + qty, cost = rnd.randint(1, 999), rnd.randint(1, 9999) + amount: Any = rnd.randint(1, 99999) + if formulas: + amount = f"=F{r}+I{r}" + cached[r] = qty + cost + ws.append([f"区{i % 7}", f"品{i % 50}", "线上" if i % 2 else "线下", f"店{i % 300}", + base + dt.timedelta(days=i % 31), qty, round(rnd.random() * 100, 2), amount, cost, "无"]) + buf = io.BytesIO() + wb.save(buf) + raw = buf.getvalue() + edits = {"xl/workbook.xml": lambda t: t.replace(' fullCalcOnLoad="1"', "")} + if formulas: + pat = re.compile(r'([^<]*)(?:)?') + + def fill(xml: str) -> str: + out, n = pat.subn(lambda m: f'{m.group(2)}{cached[int(m.group(1))]}', + xml) + assert n == ROWS, n + return out + edits[sheet_part(raw, SHEET)] = fill + return patch_parts(raw, edits) + + +def with_dimension(raw: bytes) -> bytes: + """在 前补上 （Excel 存盘的文件都有它）。""" + def add(xml: str) -> str: + assert "", f'', 1) + return patch_parts(raw, {sheet_part(raw, SHEET): add}) + + +def list_recipe() -> dict[str, Any]: + return { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": SHEET}, "blocks": [{ + "id": "明细", "layout": "list", "table": "销售明细", + "columns": [{"header": h, "name": h, "type": t} for h, t in zip(HEADERS, TYPES)]}]}], + "tables": [{"name": "销售明细"}], + } + + +@pytest.fixture(scope="module") +def plain_files() -> tuple[bytes, bytes]: + return big_list(1), big_list(2) + + +@pytest.fixture(scope="module") +def formula_files() -> tuple[bytes, bytes]: + return big_list(3, formulas=True), big_list(4, formulas=True) + + +def _variant(files: tuple[bytes, bytes], dimension: bool) -> tuple[bytes, bytes]: + return (with_dimension(files[0]), with_dimension(files[1])) if dimension else files + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test", timeout=600) as c: + yield c + + +# -------------------------------------------------------------------------- +# 计时与内存 +# -------------------------------------------------------------------------- + + +def _rss_mb() -> float: + """当前进程的常驻内存（MB）。没有 psutil，用 ps 读（macOS、Linux 都有）。""" + out = subprocess.run(["ps", "-o", "rss=", "-p", str(os.getpid())], capture_output=True, text=True).stdout + return int(out.strip() or 0) / 1024 + + +class _Peak: + """后台线程每 50 ms 采一次 RSS，记下这一段里的峰值。""" + + def __init__(self) -> None: + self.base = _rss_mb() + self.peak = self.base + self._stop = threading.Event() + self._t = threading.Thread(target=self._run, daemon=True) + + def _run(self) -> None: + while not self._stop.wait(0.05): + self.peak = max(self.peak, _rss_mb()) + + def __enter__(self) -> _Peak: + self._t.start() + return self + + def __exit__(self, *_exc: object) -> None: + self._stop.set() + self._t.join() + self.peak = max(self.peak, _rss_mb()) + + @property + def delta(self) -> float: + return self.peak - self.base + + +def judge(record_property, name: str, value: float, target: float | None, unit: str = "s") -> dict[str, Any]: + """记录一项实测；超过目标 1.5 倍判「不通过」，1–1.5 倍记「待优化」。target=None 是规格没给目标的项， + 只记录。返回这一行，由测试末尾的 assert_all 一并断言（量全了再判，前一项不通过不挡住后面各项的实测）。""" + if target is None: + row = {"item": name, "value": round(value, 2), "target": None, "unit": unit, "ratio": 0.0, + "verdict": "仅记录（规格未给目标）"} + RESULTS.append(row) + record_property(name, json.dumps(row, ensure_ascii=False)) + print(f"\n[perf] {name}: {value:.2f} {unit}（规格未给目标，仅记录）") + return row + ratio = value / target + verdict = "通过" if ratio <= 1 else ("待优化" if ratio <= FAIL_FACTOR else "不通过") + row = {"item": name, "value": round(value, 2), "target": round(target, 2), "unit": unit, + "ratio": round(ratio, 2), "verdict": verdict} + RESULTS.append(row) + record_property(name, json.dumps(row, ensure_ascii=False)) + print(f"\n[perf] {name}: {value:.2f} {unit}（目标 {target:.2f} {unit}，{ratio:.2f} 倍）{verdict}") + return row + + +def assert_all(rows: list[dict[str, Any]]) -> None: + bad = [r for r in rows if r["ratio"] > FAIL_FACTOR] + assert not bad, "超过目标 1.5 倍：" + "；".join( + f"{r['item']} {r['value']} {r['unit']}（目标 {r['target']}，{r['ratio']} 倍）" for r in bad) + + +async def _timed(coro) -> tuple[Any, float]: + t0 = time.perf_counter() + out = await coro + return out, time.perf_counter() - t0 + + +def _confirms(st: dict[str, Any]) -> list[str]: + return [i["id"] for i in st["trial"]["confirm_items"]] + + +# -------------------------------------------------------------------------- +# 20 万行 × 10 列 +# -------------------------------------------------------------------------- + + +async def _big_flow(client, record_property, files: tuple[bytes, bytes], *, label: str, + reupload_target: float, trial_target: float | None) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + first, second = files + fn1, fn2 = f"明细_{label}_甲.xlsx", f"明细_{label}_乙.xlsx" + + # 扫描单独量一次：暂存的目标是「扫描时间 + 3 s」 + t0 = time.perf_counter() + scan = xlsx_scan.scan(first) + scan_s = time.perf_counter() - t0 + sheet = scan.sheets[0] + assert sheet.nonempty == ROWS * 10 + 10 and sheet.nonempty > xlsx_cells.GRID_MAX_CELLS + print(f"\n[perf] {label} 文件 {len(first) / 1e6:.1f} MB，扫描 {scan_s:.2f} s") + record_property(f"{label}：扫描", round(scan_s, 2)) + + # 暂存：扫描 + 预览 + 规则起草 + 部分干跑 + name = f"perf_{uuid.uuid4().hex[:8]}" + resp, stage_s = await _timed(client.post("/api/datasources/imports/stage", + files={"file": (fn1, first, XLSX)}, data={"name": name})) + assert resp.status_code == 201, resp.text + st = resp.json() + sid = st["id"] + assert st["draft_partial"] is True, "大表的交互干跑只看前 500 行（4.9）" + rows.append(judge(record_property, f"{label}：暂存（扫描 + 预览 + 规则起草 + 部分干跑）", stage_s, scan_s + 3)) + + # 单次回答问题（有问题时）、单次改配方：静态校验 + 部分干跑 + if st["questions"]: + answers = {q["id"]: {"value": q["options"][0]["value"], "reason": "合成理由"} for q in st["questions"][:1]} + resp, ans_s = await _timed(client.post(f"/api/datasources/imports/{sid}/answers", json={"answers": answers})) + assert resp.status_code == 200, resp.text + rows.append(judge(record_property, f"{label}：单次回答问题", ans_s, 1.0)) + resp, put_s = await _timed(client.put(f"/api/datasources/imports/{sid}/recipe", json={"recipe": list_recipe()})) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["recipe_problems"] == [], st["recipe_problems"] + assert st["draft_partial"] is True, "大表的交互干跑只看前 500 行（4.9）" + rows.append(judge(record_property, f"{label}：单次改配方（PUT recipe）", put_s, 1.0)) + + # 暂存后的试运行：复用 scan_from_json，不再扫描；量内存增量 + with _Peak() as peak: + resp, trial_s = await _timed(client.post(f"/api/datasources/imports/{sid}/trial", json={})) + assert resp.status_code == 200, resp.text + st = resp.json() + trial = st["trial"] + assert trial["status"] == "passed", trial["problems"][:3] + assert {t["name"]: t["rows"] for t in trial["receipt"]["tables"]} == {"销售明细": ROWS} + rows.append(judge(record_property, f"{label}：暂存后的试运行", trial_s, trial_target)) + rows.append(judge(record_property, f"{label}：试运行的进程内存增量", peak.delta, 200.0, "MB")) + resp = await client.post(f"/api/datasources/imports/{sid}/commit", + json={"trial_id": trial["trial_id"], "confirmations": _confirms(st)}) + assert resp.status_code == 201, resp.text + source_id = resp.json()["source"]["id"] + + # 上传新一期：扫描 + 按现行配方自动试运行 + with _Peak() as peak: + resp, re_s = await _timed(client.post(f"/api/datasources/{source_id}/reupload", + files={"file": (fn2, second, XLSX)})) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["trial"]["status"] == "passed", st["trial"]["problems"][:3] + rows.append(judge(record_property, f"{label}：上传新一期（扫描 + 试运行）", re_s, reupload_target)) + rows.append(judge(record_property, f"{label}：上传新一期的进程内存增量", peak.delta, 200.0, "MB")) + return rows + + +@pytest.mark.parametrize("dimension", [False, True], ids=["no_dimension", "with_dimension"]) +async def test_perf_big_list_without_formulas(client, record_property, plain_files, dimension): + label = f"20万行无公式（{'有' if dimension else '无'} dimension）" + assert_all(await _big_flow(client, record_property, _variant(plain_files, dimension), label=label, + reupload_target=25.0, trial_target=18.0)) + + +@pytest.mark.parametrize("dimension", [False, True], ids=["no_dimension", "with_dimension"]) +async def test_perf_big_list_with_formula_column(client, record_property, formula_files, dimension): + label = f"20万行整列公式（{'有' if dimension else '无'} dimension）" + # 4.9 对整列公式的表只给了「上传新一期 ≤ 40 s」；暂存后的试运行没有单独的目标，只记录 + assert_all(await _big_flow(client, record_property, _variant(formula_files, dimension), label=label, + reupload_target=40.0, trial_target=None)) + + +# -------------------------------------------------------------------------- +# 结构仿照客流表的夹具 +# -------------------------------------------------------------------------- + + +async def test_perf_flow_fixture(client, record_property): + raw, fn = flow_workbook(dt.date(2026, 8, 1), 31, seed=5) + name = f"perf_{uuid.uuid4().hex[:8]}" + resp, stage_s = await _timed(client.post("/api/datasources/imports/stage", + files={"file": (fn, raw, XLSX)}, data={"name": name})) + assert resp.status_code == 201, resp.text + st = resp.json() + sid = st["id"] + rows = [judge(record_property, "客流夹具：暂存", stage_s, 1.0)] + q = st["questions"][0] + resp, ans_s = await _timed(client.post(f"/api/datasources/imports/{sid}/answers", json={"answers": { + q["id"]: {"value": q["options"][0]["value"], "reason": "合成理由"}}})) + assert resp.status_code == 200, resp.text + rows.append(judge(record_property, "客流夹具：单次回答", ans_s, 0.5)) + resp, put_s = await _timed(client.put(f"/api/datasources/imports/{sid}/recipe", json={"recipe": FLOW_RECIPE})) + assert resp.status_code == 200, resp.text + rows.append(judge(record_property, "客流夹具：单次改配方", put_s, 0.5)) + resp, trial_s = await _timed(client.post(f"/api/datasources/imports/{sid}/trial", json={})) + assert resp.status_code == 200, resp.text + assert resp.json()["trial"]["status"] == "passed" + rows.append(judge(record_property, "客流夹具：试运行", trial_s, 1.0)) + assert_all(rows) + + +def teardown_module(_module) -> None: + if RESULTS: + print("\n[perf] 汇总：") + for row in RESULTS: + target = f"目标 {row['target']} {row['unit']}，{row['ratio']} 倍" if row["target"] is not None else "无目标" + print(f"[perf] {row['item']}: {row['value']} {row['unit']}（{target}，{row['verdict']}）") diff --git a/backend/tests/test_recipe_suggest.py b/backend/tests/test_recipe_suggest.py new file mode 100644 index 00000000..9559030b --- /dev/null +++ b/backend/tests/test_recipe_suggest.py @@ -0,0 +1,1114 @@ +"""WP-4 确定性起草（recipe_suggest）的测试。 + +夹具一律在这里用 openpyxl 现造（假名：分区甲 / 分区乙，表名「客流汇总」），网格由 grid_from_ws 从 openpyxl 普通模式 +直接造出来，不依赖读取层（WP-2）；validate / dry_run 用桩，不依赖静态校验（WP-1）和执行器（WP-2）。 +""" +from __future__ import annotations + +import datetime as dt +import io +import json +import random +from pathlib import Path +from typing import Any + +import pytest +from openpyxl import Workbook +from openpyxl.utils import column_index_from_string, get_column_letter + +from app.data import recipe_suggest as S +from app.data.recipe_types import ( + REASON_SLOT, + AxisOut, + Extraction, + Grid, + GridCell, + OutsideText, + PeriodOut, + Problem, + Recipe, + RecipeProblem, +) +from app.data.xlsx_scan import scan as xlsx_scan + +FLOW = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text(encoding="utf-8")) + + +# ========================================================================== +# 夹具工具（test_recipe_ai.py 也用） +# ========================================================================== + + +def grid_from_ws(ws: Any, cache: dict[str, Any] | None = None) -> Grid: + """openpyxl 普通模式的工作表 → Grid。公式格的保存值由 cache（坐标 → 值）给出（openpyxl 写公式不带缓存）。""" + cache = cache or {} + cells: dict[tuple[int, int], GridCell] = {} + for row in ws.iter_rows(): + for cell in row: + v = cell.value + if v is None: + continue + if isinstance(v, str) and v.startswith("="): + cells[(cell.row, cell.column)] = GridCell(cache.get(cell.coordinate), v) + elif isinstance(v, str) and not v.strip(): + continue + else: + cells[(cell.row, cell.column)] = GridCell(v) + merges = [(m.min_row, m.min_col, m.max_row, m.max_col) for m in ws.merged_cells.ranges] + hidden_rows = {r for r, d in ws.row_dimensions.items() if d.hidden} + hidden_cols: set[int] = set() + for key, d in ws.column_dimensions.items(): + if d.hidden: + lo = d.min or column_index_from_string(key) + hi = d.max or lo + hidden_cols.update(range(lo, hi + 1)) + bounds = None + if cells: + rs = [r for r, _ in cells] + cs = [c for _, c in cells] + bounds = (min(rs), min(cs), max(rs), max(cs)) + return Grid(ws.title, bounds, cells, merges, hidden_rows, hidden_cols) + + +def build(wb: Workbook, cache: dict[str, dict[str, Any]] | None = None): + """保存成字节、跑期 1 的扫描、按可见工作表造网格。返回 (scan, grids)。""" + bio = io.BytesIO() + wb.save(bio) + sc = xlsx_scan(bio.getvalue()) + grids = {ws.title: grid_from_ws(ws, (cache or {}).get(ws.title)) + for ws in wb.worksheets if ws.sheet_state == "visible"} + return sc, grids + + +def flow_book(start: dt.date = dt.date(2026, 8, 1), days: int = 31, *, seed: int = 0, + big: tuple[int, int] = (3000, 9000)) -> tuple[Workbook, dict[str, dict[str, Any]], dict[str, Any]]: + """按 P2-SPEC 9.1 的坐标造「结构仿照客流表」：返回 (工作簿, 公式缓存, 填进去的数)。""" + rnd = random.Random(seed) + wb = Workbook() + ws = wb.active + ws.title = "客流汇总" + last = 2 + days + end = start + dt.timedelta(days=days - 1) + L = get_column_letter + ws["B2"] = f"统计时间范围：{start.year}年{start.month}月{start.day}日至{end.year}年{end.month}月{end.day}日" + ws.merge_cells(f"B2:{L(last)}2") + ws["B3"] = "客流汇总表" + ws.merge_cells(f"B3:{L(last)}3") + for i in range(days): + d = start + dt.timedelta(days=i) + ws.cell(4, 3 + i, f"{d.month}月{d.day}日") + labels = ["全日客流（人次）", "分区甲（人次）", "分区乙（人次）"] + for r, lab in zip((5, 6, 7), labels): + ws.cell(r, 2, lab) + nums: dict[str, Any] = {"jia": [], "yi": []} + ws.merge_cells(f"B8:{L(last)}8") + ws["B9"] = "日间时段客流（人次）" + ws.merge_cells(f"B9:{L(last)}9") + for k, h in enumerate(range(7, 18)): + ws.cell(10 + k, 2, f"{h}-{h + 1}") + ws["B21"] = "夜间时段客流（人次）" + ws.merge_cells(f"B21:{L(last)}21") + for k, h in enumerate(range(18, 24)): + ws.cell(22 + k, 2, f"{h}-{h + 1}") + for r, lab in zip((28, 29, 30), ["18-22 时合计", "22-24 时合计", "18-24 时合计"]): + ws.cell(r, 2, lab) + cache: dict[str, Any] = {} + for i in range(days): + c = 3 + i + jia, yi = rnd.randint(*big), rnd.randint(2000, 8000) + nums["jia"].append(jia) + nums["yi"].append(yi) + ws.cell(5, c, jia + yi) + ws.cell(6, c, jia) + ws.cell(7, c, yi) + ws.cell(10, c, "·") + slots = {} + for r in list(range(11, 21)) + list(range(22, 28)): + slots[r] = rnd.randint(50, 900) + if sum(slots.values()) == jia + yi: + slots[11] += 1 + for r, v in slots.items(): + ws.cell(r, c, v) + col = L(c) + ws.cell(28, c, f"=SUM({col}22:{col}25)") + ws.cell(29, c, f"=SUM({col}26:{col}27)") + ws.cell(30, c, f"=SUM({col}22:{col}27)") + cache[f"{col}28"] = sum(slots[r] for r in range(22, 26)) + cache[f"{col}29"] = sum(slots[r] for r in range(26, 28)) + cache[f"{col}30"] = sum(slots[r] for r in range(22, 28)) + return wb, {"客流汇总": cache}, nums + + +def ok_validate(calls: list | None = None): + """静态校验桩：pydantic 收就算通过。calls 记下 (配方, facts, origin)。""" + def _v(data: dict[str, Any], facts: Any, origin: str): + if calls is not None: + calls.append((data, facts, origin)) + return Recipe.model_validate(data), [] + return _v + + +def canonical(d: dict[str, Any]) -> dict[str, Any]: + return Recipe.model_validate(d).model_dump(mode="json", exclude_defaults=True) + + +def rename_wide(recipe: dict[str, Any], old: str, new: str) -> dict[str, Any]: + """把宽表名、该段 id 和关系里引用的表名一并换掉（P2-SPEC 第 10 节 WP-4 测试的做法）。""" + text = json.dumps(recipe, ensure_ascii=False) + return json.loads(text.replace(json.dumps(old, ensure_ascii=False), json.dumps(new, ensure_ascii=False))) + + +# ========================================================================== +# 参考布局 +# ========================================================================== + + +@pytest.fixture(scope="module") +def flow(): + wb, cache, nums = flow_book() + sc, grids = build(wb, cache) + return sc, grids, nums + + +def test_reference_layout_draft_matches_reference_recipe(flow): + sc, grids, _ = flow + calls: list = [] + d = S.draft(sc, grids, "月报导出_2026-08-01_2026-08-31.xlsx", validate=ok_validate(calls)) + assert d.complete, d.failures + assert d.origin == "rules" and d.failures == [] and d.failures_for_model == [] + assert calls and calls[0][2] == "rules" and calls[0][1] is d.facts + wide = d.recipe["tables"][0]["name"] + assert wide == "客流汇总_按日" + assert canonical(rename_wide(d.recipe, wide, "日客流")) == canonical(FLOW) + + +def apply_ops(doc: dict[str, Any], ops: list[dict[str, Any]], reason: str = "巧合") -> dict[str, Any]: + """RFC 6902 的 add / replace / remove 子集（测试自用；生产里是 WP-1 的 recipe.apply_patch）。""" + doc = json.loads(json.dumps(doc, ensure_ascii=False)) + + def fill(v: Any) -> Any: + if v == REASON_SLOT: + return reason + if isinstance(v, dict): + return {k: fill(x) for k, x in v.items()} + if isinstance(v, list): + return [fill(x) for x in v] + return v + + for op in ops: + parts = [p.replace("~1", "/").replace("~0", "~") for p in op["path"].split("/")[1:]] + node: Any = doc + for p in parts[:-1]: + node = node[int(p)] if isinstance(node, list) else node[p] + last = parts[-1] + if isinstance(node, list): + if op["op"] == "remove": + node.pop(int(last)) + elif last == "-": + node.append(fill(op["value"])) + elif op["op"] == "add": + node.insert(int(last), fill(op["value"])) + else: + node[int(last)] = fill(op["value"]) + else: + if op["op"] == "remove": + del node[last] + else: + if op["op"] == "replace": + assert last in node, op + node[last] = fill(op["value"]) + return doc + + +def test_reference_layout_facts(flow): + sc, grids, _ = flow + facts = S.detect_facts(grids, sc) + assert [(f.id, f.kind) for f in facts.facts] == [("F1", "sum_eq"), ("F2", "not_equal_sum")] + f1, f2 = facts.facts + assert f1.text == "第 5 行 = 第 6 行 + 第 7 行（31 列中 31 列成立）" + assert f1.detail == {"segment": "客流汇总_按日", "total": "全日客流（人次）", + "parts": ["分区甲（人次）", "分区乙（人次）"], "rows": [5, 6, 7]} + assert f2.text == "日间、夜间各行之和 与 第 5 行：31 列中 0 列相等" + assert f2.detail == {"a": ["日间", "夜间"], "b_segment": "客流汇总_按日", "b": "全日客流（人次）", + "equal": 0, "checked": 31} + assert facts.candidates["日间时段客流（人次）"][0] == "日间" + assert facts.nonnumeric == {"·": 31} + # 只有关系不给数值：事实的文字里没有任何一个填进去的数 + _, _, nums = flow + blob = json.dumps([f.__dict__ for f in facts.facts], ensure_ascii=False) + assert not any(str(v) in blob for v in nums["jia"] + nums["yi"] if v > 100) + + +def test_reference_layout_cards_and_questions(flow): + sc, grids, _ = flow + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert [c.id for c in d.cards] == [ + "axis:交叉表", "period:s1", "titles:交叉表", "fact:F1", "merge:时段客流", "placeholder:交叉表:·", + "derived:夜间合计", "fact:F2", "outside:s1", "mode"] + titles = [c.title for c in d.cards] + assert titles[0] == "第 4 行是日期表头，按交叉表导入" + assert titles[1] == "统计期取自 B2" + assert titles[2] == "分段标题 B9、B21" + assert titles[3].startswith("第 5 行 = 第 6 行 + 第 7 行") + assert titles[4] == "日间、夜间合进一张表「时段客流」" + assert titles[5] == "「·」表示无数据吗" + assert titles[6] == "第 28–30 行是合计，改作核对并另存" + assert titles[7] == "各时段之和与全日客流 31 天中 0 天相等" + assert titles[8] == "B3 是区域外文字" + assert titles[9] == "导入模式：每期替换" + assert "按期累积将在后续版本提供" in d.cards[9].reason + assert d.cards[1].cells == ["客流汇总!B2"] + assert d.cards[8].cells == ["客流汇总!B3"] + by_id = {c.id: c for c in d.cards} + assert by_id["placeholder:交叉表:·"].question == "q_placeholder:·" + assert by_id["fact:F2"].question == "q_relation:F2" + + qs = {q.id: q for q in d.questions} + assert [q.id for q in d.questions] == ["q_relation:F1", "q_placeholder:·", "q_relation:F2"] + assert all(q.default is None for q in d.questions) + for qid in ("q_relation:F1", "q_relation:F2"): + q = qs[qid] + opts = {o["value"]: o for o in q.options} + assert opts["register"]["needs_reason"] is False + assert opts["dismiss"]["needs_reason"] is True + assert REASON_SLOT in json.dumps(q.effects["dismiss"], ensure_ascii=False) + ph = qs["q_placeholder:·"] + assert [o["value"] for o in ph.options] == ["null", "reject"] + + +def test_question_effects_apply_and_are_state_independent(flow): + sc, grids, _ = flow + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + qs = {q.id: q for q in d.questions} + base = d.recipe + dismissed = apply_ops(base, qs["q_relation:F1"].effects["dismiss"]) + rel = dismissed["relations"][0] + assert rel == {"id": "R1", "kind": "dismissed", "claims": "F1", "reason": "巧合"} + Recipe.model_validate(dismissed) + # 先选「不登记」再改选「登记」，等于直接选「登记」（整条 replace，与状态无关） + again = apply_ops(dismissed, qs["q_relation:F1"].effects["register"]) + assert canonical(again) == canonical(base) + # 在改过的配方上重算问题，effects 仍然把它写回同一条 + _, qs2 = S.questions_for(dismissed, d.facts, grids) + q2 = next(q for q in qs2 if q.id == "q_relation:F1") + assert canonical(apply_ops(dismissed, q2.effects["register"])) == canonical(base) + rejected = apply_ops(base, qs["q_placeholder:·"].effects["reject"]) + assert rejected["sheets"][0]["blocks"][0]["values"]["placeholders"] == [] + assert apply_ops(base, qs["q_placeholder:·"].effects["null"]) == base + + +def test_detect_facts_remaps_to_renamed_recipe(flow): + """用户把宽表和该段 id 改名成「日客流」：给了 recipe 时，事实里的分段 id 跟着换，认领检查才对得上。""" + sc, grids, _ = flow + renamed = rename_wide(S.draft(sc, grids, "x.xlsx", validate=ok_validate()).recipe, "客流汇总_按日", "日客流") + facts = S.detect_facts(grids, sc, recipe=renamed) + assert facts.facts[0].detail["segment"] == "日客流" + assert facts.facts[1].detail["b_segment"] == "日客流" + assert facts.facts[1].detail["a"] == ["日间", "夜间"] + # 不给 recipe 时是规则起草的取法 + assert S.detect_facts(grids, sc).facts[0].detail["segment"] == "客流汇总_按日" + # 标签写法只按 match_key 对：换成配方里 expect 的写法 + flow2 = json.loads(json.dumps(FLOW)) + seg = flow2["sheets"][0]["blocks"][0]["segments"][0] + seg["labels"]["expect"][1] = "分区甲 （人次）" + seg["measures"] = {"全日客流（人次）": "全日客流", "分区甲 （人次）": "分区甲", "分区乙（人次）": "分区乙"} + f = S.remap_facts(S.detect_facts(grids, sc), flow2).facts[0] + assert f.detail["parts"] == ["分区甲 （人次）", "分区乙（人次）"] + + +def test_draft_with_dry_run_structure_problem_is_incomplete(flow): + sc, grids, _ = flow + seen: list = [] + + def dry(recipe): + seen.append(recipe) + return Extraction(ok=False, problems=[ + Problem("cell_unclaimed", "structure", "第 31 行有 2 个单元格没有去处：987654", ["客流汇总!C31"], + model_message="第 31 行有 2 个单元格没有去处"), + Problem("period_missing", "input", "未能从表格中解析出统计期"), + Problem("outside_digits", "confirm", "区域外有含数字的文字"), + ]) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate(), dry_run=dry) + assert seen and isinstance(seen[0], Recipe) + assert not d.complete + assert d.failures == ["第 31 行有 2 个单元格没有去处：987654"] + assert d.failures_for_model == ["cell_unclaimed：第 31 行有 2 个单元格没有去处（客流汇总!C31）"] + assert any(c.id == "unclaimed:s1" for c in d.cards) + # input、confirm 类不算起草不完整 + d2 = S.draft(sc, grids, "x.xlsx", validate=ok_validate(), + dry_run=lambda r: Extraction(ok=False, problems=[Problem("period_missing", "input", "x")])) + assert d2.complete + + +def test_draft_with_static_problems_is_incomplete(flow): + sc, grids, _ = flow + + def bad(data, facts, origin): + return None, [RecipeProblem("/tables/0/units/全日客流", "unit_unknown", "表「日客流」：单位不在单位词表中")] + d = S.draft(sc, grids, "x.xlsx", validate=bad, dry_run=lambda r: pytest.fail("静态校验没过不该干跑")) + assert not d.complete and d.recipe is not None + assert d.failures == ["表「日客流」：单位不在单位词表中"] + assert d.failures_for_model == ["/tables/0/units/全日客流：表「日客流」：单位不在单位词表中"] + + def boom(data, facts, origin): + raise RuntimeError("x") + d2 = S.draft(sc, grids, "x.xlsx", validate=boom) + assert not d2.complete and d2.failures + + +def test_questions_for_region_cards_come_from_extraction(flow): + sc, grids, _ = flow + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + ext = Extraction( + ok=True, + period=PeriodOut("2026-08-01", "2026-08-31", "cells", cells=["客流汇总!B40"]), + outside_text=[OutsideText("客流汇总", "B41", "备注说明", "text"), + OutsideText("客流汇总", "B42", "注：9月15日闸机故障", "text_digits")], + axes=[AxisOut("交叉表", 4, "8月1日", "8月31日", 31, "text")], + problems=[Problem("row_unclaimed", "structure", "第 44 行没有分段认领", ["客流汇总!B44"])], + placeholders={"·": 29}, + ) + cards, _ = S.questions_for(d.recipe, d.facts, grids, extraction=ext) + by_id = {c.id: c for c in cards} + assert by_id["period:s1"].cells == ["客流汇总!B40"] + assert by_id["period:s1"].title == "统计期取自 B40" + assert by_id["outside:s1"].cells == ["客流汇总!B41", "客流汇总!B42"] + assert "提交时逐条确认" in by_id["outside:s1"].reason + assert by_id["unclaimed:s1"].cells == ["客流汇总!B44"] + assert "29 格" in by_id["placeholder:交叉表:·"].reason + # 干跑说统计期是人工录入的：统计期卡片换成「每期人工录入」 + ext2 = Extraction(ok=True, period=PeriodOut("2026-08-01", "2026-08-31", "human")) + cards2, qs2 = S.questions_for(d.recipe, d.facts, grids, extraction=ext2) + assert next(c for c in cards2 if c.id == "period:s1").question == "q_period" + assert any(q.id == "q_period" for q in qs2) + + +def test_questions_for_invalid_recipe_does_not_raise(flow): + sc, grids, _ = flow + cards, qs = S.questions_for({"recipe_format": "x"}, S.detect_facts(grids, sc), grids) + assert [c.id for c in cards] == ["mode"] and qs == [] + + +# ========================================================================== +# 交叉表的变体 +# ========================================================================== + + +def _crosstab(ws, labels_rows: list[tuple[int, str]], days: int = 8, start_col: int = 3, axis_row: int = 4, + value=lambda r, c: (r * 7 + c * 3) % 50 + 10): + for i in range(days): + ws.cell(axis_row, start_col + i, f"8月{i + 1}日") + for r, lab in labels_rows: + ws.cell(r, start_col - 1, lab) + if lab.endswith("时段客流（人次）") or lab.endswith("（标题）"): + continue + for i in range(days): + ws.cell(r, start_col + i, value(r, i)) + + +def test_overlapping_hour_segments_are_not_merged(): + wb = Workbook() + ws = wb.active + ws.title = "时段" + ws["B2"] = "统计期：2026年8月1日至2026年8月8日" + rows = [(5, "工作日时段客流（人次）"), (6, "7-8"), (7, "8-9"), (9, "周末时段客流（人次）"), (10, "7-8"), (11, "8-9")] + _crosstab(ws, rows) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert d.complete, d.failures + segs = d.recipe["sheets"][0]["blocks"][0]["segments"] + assert [s["table"] for s in segs] == ["工作日时段客流", "周末时段客流"] + assert all(s["const"] == {} for s in segs) + assert [s["id"] for s in segs] == ["工作日时段客流", "周末时段客流"] + assert [t["grain"] for t in d.recipe["tables"]] == [["日期", "时段"], ["日期", "时段"]] + assert not any(c.id.startswith("merge:") for c in d.cards) + assert d.recipe["sheets"][0]["context"][0]["prefer_prefix"] == "统计期" + + +def test_multiple_sheets_get_unique_segment_and_block_ids(): + wb = Workbook() + for k, name in enumerate(["一号", "二号"]): + ws = wb.active if k == 0 else wb.create_sheet(name) + ws.title = name + ws["B2"] = "2026年8月1日至2026年8月8日" + rows = [(5, "合计（人次）"), (6, "甲（人次）"), (7, "乙（人次）"), + (9, "日间时段客流（人次）"), (10, "7-8"), (11, "8-9"), + (13, "夜间时段客流（人次）"), (14, "18-19"), (15, "19-20")] + _crosstab(ws, rows) + hidden = wb.create_sheet("底稿") + hidden["A1"] = "不导入" + hidden.sheet_state = "hidden" + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert d.complete, d.failures + sheets = d.recipe["sheets"] + assert [s["id"] for s in sheets] == ["s1", "s2"] + block_ids = [b["id"] for s in sheets for b in s["blocks"]] + seg_ids = [g["id"] for s in sheets for b in s["blocks"] for g in b["segments"]] + assert block_ids == ["交叉表", "交叉表_s2"] + assert len(seg_ids) == len(set(seg_ids)) + assert "日间" in seg_ids and "日间_s2" in seg_ids + names = [t["name"] for t in d.recipe["tables"]] + assert len(names) == len(set(names)) + assert names[:2] == ["一号_按日", "时段客流"] and "时段客流_2" in names + assert any(c.id == "hidden_sheet:底稿" for c in d.cards) + assert d.cards[-1].id == "mode" + + +def test_unparseable_total_row_fails_with_reason(): + wb = Workbook() + ws = wb.active + ws.title = "表" + ws["B2"] = "2026年8月1日至2026年8月8日" + rows = [(5, "日间时段客流（人次）"), (6, "7-8"), (7, "8-9"), (8, "上午合计")] + _crosstab(ws, rows) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert not d.complete + assert any("第 8 行像合计行，但标签无法确定区间" in f for f in d.failures) + + +def test_missing_period_asks_for_human_input(): + wb = Workbook() + ws = wb.active + ws.title = "表" + _crosstab(ws, [(5, "甲（人次）"), (6, "乙（人次）")]) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert d.complete + assert d.recipe["sheets"][0]["context"][0]["prefer_prefix"] is None + q = next(q for q in d.questions if q.id == "q_period") + assert [o["value"] for o in q.options] == ["human"] and q.effects == {"human": []} + assert next(c for c in d.cards if c.id == "period:s1").title == "未找到统计期：每期人工录入" + + +def test_value_area_text_fails_but_model_version_has_no_value(): + wb = Workbook() + ws = wb.active + ws.title = "表" + ws["B2"] = "2026年8月1日至2026年8月8日" + _crosstab(ws, [(5, "甲（人次）"), (6, "乙（人次）")]) + ws["E6"] = "约987654" + ws["F6"] = "暂缺" + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert not d.complete + assert any("约987654" in f for f in d.failures) + assert any("E6" in f for f in d.failures_for_model) + assert not any("987654" in f or "暂缺" in f for f in d.failures_for_model) + + +def test_failures_for_model_scrubs_cell_values_from_dry_run(flow): + """model_message 为空时只发 code 和坐标；即使 model_message 写进了格子里的数，也换成 <值>。""" + sc, grids, _ = flow + grids = dict(grids) + g = grids["客流汇总"] + cells = dict(g.cells) + cells[(31, 3)] = GridCell(987654) + cells[(31, 4)] = GridCell("123,457") + grids["客流汇总"] = Grid(g.sheet, g.bounds, cells, g.merges, g.hidden_rows, g.hidden_cols) + probs = [Problem("outside_number", "structure", "C31 的 987654 在数据区外", ["客流汇总!C31"]), + Problem("value_not_number", "structure", "D31「123,457」", ["客流汇总!D31"], + model_message="D31 写着 123,457，不是数")] + lines = S.problem_lines_for_model(probs, grids) + assert lines[0] == "outside_number（客流汇总!C31）" + assert lines[1] == "value_not_number：D31 写着 <值>，不是数（客流汇总!D31）" + assert not any("987654" in x or "123,457" in x for x in lines) + + +def test_blank_and_formula_cells_in_value_area_become_questions(): + wb = Workbook() + ws = wb.active + ws.title = "表" + ws["B2"] = "2026年8月1日至2026年8月8日" + _crosstab(ws, [(5, "甲（人次）"), (6, "乙（人次）")]) + ws["D5"] = None + ws["E6"] = "=E5*2" + sc, grids = build(wb, {"表": {"E6": 40}}) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + values = d.recipe["sheets"][0]["blocks"][0]["values"] + assert values["blank"] == "reject" and values["formula"] == "reject" + qs = {q.id: q for q in d.questions} + assert qs["q_blank:交叉表"].default is None and qs["q_formula:交叉表"].default is None + after = apply_ops(d.recipe, qs["q_blank:交叉表"].effects["null"] + qs["q_formula:交叉表"].effects["accept_cached"]) + v = after["sheets"][0]["blocks"][0]["values"] + assert v["blank"] == "null" and v["formula"] == "accept_cached" + # 紧凑形式（去掉默认值，values 整个不在）上重算的 effects 也能应用 + compact = canonical(d.recipe) + assert "values" not in compact["sheets"][0]["blocks"][0] + _, qs2 = S.questions_for(compact, d.facts, grids) + q = next(x for x in qs2 if x.id == "q_blank:交叉表") + assert apply_ops(compact, q.effects["null"])["sheets"][0]["blocks"][0]["values"]["blank"] == "null" + + +def test_hidden_rows_in_region_ask_without_default(): + wb = Workbook() + ws = wb.active + ws.title = "表" + ws["B2"] = "2026年8月1日至2026年8月8日" + _crosstab(ws, [(5, "甲（人次）"), (6, "乙（人次）"), (7, "丙（人次）")]) + ws.row_dimensions[6].hidden = True + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + q = next(q for q in d.questions if q.id == "q_hidden:表") + assert q.default is None + assert [o["value"] for o in q.options] == ["reject_if_any", "include", "exclude"] + after = apply_ops(d.recipe, q.effects["exclude"]) + assert after["sheets"][0]["hidden"] == {"rows": "exclude", "cols": "reject_if_any"} + + +def test_non_text_row_label_is_a_failure(): + wb = Workbook() + ws = wb.active + ws.title = "表" + ws["B2"] = "2026年8月1日至2026年8月8日" + _crosstab(ws, [(5, "甲（人次）"), (6, "乙（人次）")]) + ws["B6"] = 2025 + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert not d.complete + assert any("B6" in f and "不是文字" in f for f in d.failures) + assert "2025" not in json.dumps(d.recipe, ensure_ascii=False) + + +def test_unknown_unit_keeps_raw_name_and_gets_a_card(): + wb = Workbook() + ws = wb.active + ws.title = "表" + ws["B2"] = "2026年8月1日至2026年8月8日" + _crosstab(ws, [(5, "销量（箱）"), (6, "金额（万元）")]) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + seg = d.recipe["sheets"][0]["blocks"][0]["segments"][0] + assert seg["measures"] == {"销量（箱）": "销量_箱", "金额（万元）": "金额"} + assert d.recipe["tables"][0]["units"] == {"金额": "万元"} + assert any(c.title == "「销量（箱）」的单位「箱」不在单位词表中" for c in d.cards) + + +# ========================================================================== +# 列表（P2-SPEC 6.1 第 11 条） +# ========================================================================== + + +def test_list_header_types_and_grain(): + wb = Workbook() + ws = wb.active + ws.title = "销售明细" + ws.append(["订单号", "日期", "数量", "金额（元）", "备注"]) + rows = [["A001", dt.datetime(2026, 8, 1), 3, 12.5, "首单"], + ["A002", dt.datetime(2026, 8, 2), 1, 7.0, None], + ["A003", dt.datetime(2026, 8, 3), 2, 9.25, "加急"]] + for r in rows: + ws.append(r) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert d.complete, d.failures + block = d.recipe["sheets"][0]["blocks"][0] + assert block["id"] == "列表1" and block["layout"] == "list" and block["table"] == "销售明细" + assert [(c["header"], c["name"], c["type"]) for c in block["columns"]] == [ + ("订单号", "订单号", "TEXT"), ("日期", "日期", "DATE"), ("数量", "数量", "INTEGER"), + ("金额（元）", "金额", "REAL"), ("备注", "备注", "TEXT")] + assert d.recipe["tables"] == [{"name": "销售明细", "grain": ["订单号"], "kind": "data", + "units": {"金额": "元"}, "note": ""}] + assert d.recipe["sheets"][0]["context"] == [] + assert d.cards[0].id == "list:列表1" + + +def test_list_two_row_header_with_merged_upper_row(): + wb = Workbook() + ws = wb.active + ws.title = "汇总" + ws["A1"] = "地区" + ws.merge_cells("A1:A2") + ws["B1"] = "销售" + ws.merge_cells("B1:C1") + ws["B2"] = "数量" + ws["C2"] = "金额" + for r, row in enumerate([["华东", 10, 100], ["华北", 20, 120]], start=3): + for c, v in enumerate(row, start=1): + ws.cell(r, c, v) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + block = d.recipe["sheets"][0]["blocks"][0] + assert block["header_rows"] == 2 + assert [c["header"] for c in block["columns"]] == ["地区", "销售_数量", "销售_金额"] + assert d.complete, d.failures + + +def test_list_blank_row_in_middle_is_skipped_with_card(): + wb = Workbook() + ws = wb.active + ws.title = "明细" + for row in [["地区", "产品", "金额"], ["华东", "甲", 10], ["华东", "乙", 20], [None, None, None], + ["华北", "甲", 30], ["华北", "乙", 40]]: + ws.append(row) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + block = d.recipe["sheets"][0]["blocks"][0] + assert block["rows"]["blank_rows"] == "skip" + assert len(d.recipe["sheets"][0]["blocks"]) == 1 + assert any(c.id == "blank_skip:列表1" for c in d.cards) + + +def test_list_stacked_tables_become_two_blocks(): + wb = Workbook() + ws = wb.active + ws.title = "汇总" + ws["B1"] = "一、销售" + for r, row in enumerate([["地区", "销量", "金额"], ["华东", 10, 100], ["华北", 20, 120], ["华南", 15, 80]], start=2): + for c, v in enumerate(row, start=2): + ws.cell(r, c, v) + ws["B7"] = "二、费用" + for r, row in enumerate([["部门", "费用"], ["甲部", 5], ["乙部", 7], ["丙部", 8]], start=8): + for c, v in enumerate(row, start=2): + ws.cell(r, c, v) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + blocks = d.recipe["sheets"][0]["blocks"] + assert [(b["id"], b["table"]) for b in blocks] == [("列表1", "销售"), ("列表2", "费用")] + assert [c["header"] for c in blocks[1]["columns"]] == ["部门", "费用"] + assert d.complete, d.failures + out = next(c for c in d.cards if c.id == "outside:s1") + assert out.cells == ["汇总!B1", "汇总!B7"] + + +def test_list_side_by_side_tables_split_on_empty_column(): + wb = Workbook() + ws = wb.active + ws.title = "并排" + for r, row in enumerate([["地区", "金额"], ["华东", 1], ["华北", 2]], start=1): + ws.cell(r, 1, row[0]) + ws.cell(r, 2, row[1]) + for r, row in enumerate([["部门", "费用"], ["甲部", 5], ["乙部", 7]], start=1): + ws.cell(r, 4, row[0]) + ws.cell(r, 5, row[1]) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + blocks = d.recipe["sheets"][0]["blocks"] + assert [[c["header"] for c in b["columns"]] for b in blocks] == [["地区", "金额"], ["部门", "费用"]] + assert [b["table"] for b in blocks] == ["并排", "并排_2"] + + +def test_list_total_row_is_verified_and_kept(): + wb = Workbook() + ws = wb.active + ws.title = "销售" + for row in [["地区", "销量", "金额"], ["华东", 10, 100], ["华北", 20, 120], ["合计", 30, 220]]: + ws.append(row) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + block = d.recipe["sheets"][0]["blocks"][0] + assert block["rows"]["total_row"] == {"label_column": "地区", "pick": "合计", "keep_as": "销售_表内合计"} + assert [t["name"] for t in d.recipe["tables"]] == ["销售", "销售_表内合计"] + assert d.recipe["tables"][1]["grain"] == ["合计项"] and d.recipe["tables"][1]["kind"] == "reported_total" + assert any(c.id == "total_row:列表1" and "第 4 行" in c.title for c in d.cards) + + +def test_list_vertical_merge_in_text_column_fills_with_question(): + wb = Workbook() + ws = wb.active + ws.title = "分组" + for row in [["地区", "产品", "金额"], ["华东", "甲", 10], [None, "乙", 20], ["华北", "甲", 30], [None, "乙", 40]]: + ws.append(row) + ws.merge_cells("A2:A3") + ws.merge_cells("A4:A5") + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + block = d.recipe["sheets"][0]["blocks"][0] + assert block["merged_data"] == "fill" + q = next(q for q in d.questions if q.id == "q_merged:列表1") + assert q.default is None + assert apply_ops(d.recipe, q.effects["reject"])["sheets"][0]["blocks"][0]["merged_data"] == "reject" + + +def test_list_header_with_year_gets_a_card(): + wb = Workbook() + ws = wb.active + ws.title = "年度" + for row in [["地区", "2026年上半年_销量"], ["华东", 10], ["华北", 20]]: + ws.append(row) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert any(c.id == "header_year:列表1" for c in d.cards) + + +def test_list_period_cell_outside_becomes_context(): + wb = Workbook() + ws = wb.active + ws.title = "销售" + ws["A1"] = "统计时间范围：2026年8月1日至2026年8月31日" + for r, row in enumerate([["地区", "金额"], ["华东", 1], ["华北", 2]], start=3): + ws.cell(r, 1, row[0]) + ws.cell(r, 2, row[1]) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert d.recipe["sheets"][0]["context"][0]["prefer_prefix"] == "统计时间范围" + assert any(c.id == "period:s1" for c in d.cards) + + +def test_unrecognized_key_value_form(): + wb = Workbook() + ws = wb.active + ws.title = "登记表" + ws["A1"] = "项目登记表" + for r, (k, v) in enumerate([("项目名称：", "某工程"), ("负责人：", "张三"), ("预算（万元）：", 120), + ("开始日期：", dt.datetime(2026, 8, 1)), ("备注：", "无")], start=3): + ws.cell(r, 1, k) + ws.cell(r, 2, v) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=lambda *a: pytest.fail("认不出时不该静态校验")) + assert d.recipe is None and not d.complete + assert any("认不出版式" in f for f in d.failures) + assert d.failures_for_model and not any("张三" in f or "120" in f for f in d.failures_for_model) + + # 不带冒号的键值表单：表头像，但下面没有一列是统一的数字或日期 + wb2 = Workbook() + ws2 = wb2.active + ws2.title = "登记表" + for k, v in [("项目名称", "某工程"), ("负责人", "张三"), ("预算", 120), ("部门", "工程部")]: + ws2.append([k, v]) + sc2, grids2 = build(wb2) + d2 = S.draft(sc2, grids2, "x.xlsx", validate=ok_validate()) + assert not d2.complete + assert any("不是列表" in f for f in d2.failures) + + +def test_draft_never_raises_on_odd_input(): + wb = Workbook() + ws = wb.active + ws.title = "空" + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert d.recipe is None and not d.complete and d.failures + + +def test_layout_hints(): + wb, cache, _ = flow_book() + sc, grids = build(wb, cache) + h = S.layout_hints(grids["客流汇总"]) + assert (h.axis_row, h.label_col) == (4, 2) + wb2 = Workbook() + ws = wb2.active + ws.title = "表" + ws["A1"] = "标题" + for row in [["地区", "金额"], ["华东", 1], ["华北", 2]]: + ws.append(row) + _, grids2 = build(wb2) + h2 = S.layout_hints(grids2["表"]) + assert h2.axis_row is None and h2.header_rows == {2} and h2.top_header_row == 2 + + +def test_constants_match_period1(): + from app.data import tabular + + assert S.DATE_HEADER_MIN == tabular.DATE_HEADER_MIN == 7 + assert S.DRAFT_MAX_ROWS == 500 + + +def test_fact_search_stays_fast_on_large_measures_segment(): + """几十行的指标段不能让组合数爆炸（暂存时起草要在一秒量级内完成）。""" + import time + + wb = Workbook() + ws = wb.active + ws.title = "大段" + ws["B2"] = "2026年8月1日至2026年8月31日" + rows = [(5 + i, f"指标{chr(0x4e00 + i)}（人次）") for i in range(45)] + _crosstab(ws, rows, days=31, value=lambda r, c: (r * 131 + c * 17) % 997 + 3) + sc, grids = build(wb) + t0 = time.perf_counter() + S.detect_facts(grids, sc) + assert time.perf_counter() - t0 < 2.0 + + +def test_question_ids_are_unique_across_blocks(): + wb = Workbook() + for k, name in enumerate(["一号", "二号"]): + ws = wb.active if k == 0 else wb.create_sheet(name) + ws.title = name + ws["B2"] = "2026年8月1日至2026年8月8日" + _crosstab(ws, [(5, "甲（人次）"), (6, "乙（人次）")]) + ws["D6"] = "·" + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + ids = [q.id for q in d.questions] + assert len(ids) == len(set(ids)) + assert "q_placeholder:·" in ids and "q_placeholder:·@交叉表_s2" in ids + assert {c.question for c in d.cards if c.question} <= set(ids) + + +# ========================================================================== +# 列表：数据行不当表头、合计字样不在第一列或在中间 +# ========================================================================== + + +def _roster(blank_before_wang: bool = False) -> Workbook: + """名单：第 4 行（王五）的金额空着。这一行全是文字、互不相同、下面有数，单看「像表头」。""" + wb = Workbook() + ws = wb.active + ws.title = "名单" + ws.append(["姓名", "电话", "金额", "备注"]) + people = [("张三", "138-1234-5678", 10, "老客户"), ("李四", "139-8765-4321", 20, "新客户"), + ("王五", "137-2222-3333", None, "转介绍"), ("赵六", "136-2222-4444", 40, "老客户"), + ("钱七", "135-4444-5555", 50, "新客户")] + for i, p in enumerate(people): + if blank_before_wang and i == 2: + ws.append([None, None, None, None]) + ws.append(list(p)) + return wb + + +def test_list_row_with_blank_number_is_not_a_header(): + sc, grids = build(_roster()) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert d.complete, d.failures + blocks = d.recipe["sheets"][0]["blocks"] + assert len(blocks) == 1 + assert [c["name"] for c in blocks[0]["columns"]] == ["姓名", "电话", "金额", "备注"] + assert blocks[0]["columns"][2]["type"] == "INTEGER" + assert S.layout_hints(grids["名单"]).header_rows == {1} + blob = json.dumps(d.recipe, ensure_ascii=False) + assert "王五" not in blob and "137" not in blob + assert any(c.id == "list:列表1" and "第 2–6 行是数据" in c.reason for c in d.cards) + # 空行之后紧跟着这种行：和上方同类型，跳过空行继续，不当成第二张表 + sc2, grids2 = build(_roster(blank_before_wang=True)) + d2 = S.draft(sc2, grids2, "x.xlsx", validate=ok_validate()) + assert d2.complete, d2.failures + assert len(d2.recipe["sheets"][0]["blocks"]) == 1 + assert d2.recipe["sheets"][0]["blocks"][0]["rows"]["blank_rows"] == "skip" + assert S.layout_hints(grids2["名单"]).header_rows == {1} + + +def test_list_stacked_with_title_but_no_blank_row_splits(): + """「小标题 + 表头」也算两张表的分界（没有空行时）。""" + wb = Workbook() + ws = wb.active + ws.title = "汇总" + for row in [["地区", "销量", "金额"], ["分区甲", 10, 100], ["分区乙", 20, 120], ["二、费用", None, None], + ["部门", "费用", None], ["部门甲", 5, None], ["部门乙", 7, None]]: + ws.append(row) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + blocks = d.recipe["sheets"][0]["blocks"] + assert [[c["header"] for c in b["columns"]] for b in blocks] == [["地区", "销量", "金额"], ["部门", "费用"]] + assert blocks[1]["table"] == "费用" + assert d.complete, d.failures + + +def test_list_header_glued_to_data_is_a_failure_not_a_split(): + """另一张表的表头紧挨着上方的数据（没有空行或标题）：不切开、不把它当表头，起草失败写明原因。""" + wb = Workbook() + ws = wb.active + ws.title = "汇总" + for row in [["地区", "销量"], ["分区甲", 10], ["分区乙", 20], ["部门", "费用"], ["部门甲", 5], ["部门乙", 7]]: + ws.append(row) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert not d.complete + assert len(d.recipe["sheets"][0]["blocks"]) == 1 + assert any("第 4 行像另一张表的表头，B 列应是数字或日期" in f and "没有空行或标题" in f for f in d.failures) + assert S.layout_hints(grids["汇总"]).header_rows == {1} + + +def test_list_total_word_outside_first_column_fails(): + wb = Workbook() + ws = wb.active + ws.title = "销售" + ws.append(["日期", "地区", "金额"]) + for i, (zone, v) in enumerate([("分区甲", 100), ("分区乙", 120), ("分区丙", 130), ("分区丁", 140), + ("分区戊", 160)]): + ws.append([dt.datetime(2026, 8, 1 + i), zone, v]) + ws.append([None, "合计", 650]) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert not d.complete + assert "工作表「销售」：第 7 行像合计行（B 列「合计」），但合计字样不在第一列，无法按合计行核对" in d.failures + assert any("第 7 行像合计行（B 列以「合计」开头）" in f for f in d.failures_for_model) + assert not any("650" in f for f in d.failures + d.failures_for_model) + assert d.recipe["sheets"][0]["blocks"][0]["rows"]["total_row"] is None + + +def test_list_subtotal_rows_in_middle_fail(): + # 小计写在第二列、夹在中间 + wb = Workbook() + ws = wb.active + ws.title = "门店" + for row in [["地区", "门店", "金额"], ["分区甲", "门店一", 100], ["分区甲", "门店二", 200], ["分区甲", "小计", 300], + ["分区乙", "门店三", 150], ["分区乙", "门店四", 150], ["分区乙", "小计", 300]]: + ws.append(row) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert not d.complete + assert [f for f in d.failures if "像合计行" in f] == [ + "工作表「门店」：第 4 行像合计行（B 列「小计」），但合计字样不在第一列，无法按合计行核对", + "工作表「门店」：第 7 行像合计行（B 列「小计」），但合计字样不在第一列，无法按合计行核对"] + # 小计写在第一列、但下面还有明细：不是末尾的合计行，整块照样收进来，不另报「数据区外的数字」 + wb2 = Workbook() + ws2 = wb2.active + ws2.title = "门店" + for row in [["门店", "金额"], ["门店一", 100], ["门店二", 200], ["小计", 300], ["门店三", 150], ["门店四", 150], + ["合计", 600]]: + ws2.append(row) + sc2, grids2 = build(wb2) + d2 = S.draft(sc2, grids2, "x.xlsx", validate=ok_validate()) + assert not d2.complete + assert d2.failures == ["工作表「门店」：第 4 行像合计行（A 列「小计」），但不在列表末尾，无法确定它汇总的是哪几行"] + block = d2.recipe["sheets"][0]["blocks"][0] + assert block["rows"]["total_row"]["pick"] == "合计" + # 末尾第一列的合计行照旧改作核对，不算失败（test_list_total_row_is_verified_and_kept） + + +# ========================================================================== +# 系统发现按配方改写：只在事实所在的工作表里找 +# ========================================================================== + + +def _two_flow_sheets(names=("客流汇总", "客流汇总二")) -> Workbook: + wb = Workbook() + for k, name in enumerate(names): + ws = wb.active if k == 0 else wb.create_sheet(name) + ws.title = name + ws["B2"] = "2026年8月1日至2026年8月8日" + rows = [(5, "全日（人次）"), (6, "甲（人次）"), (7, "乙（人次）"), (9, "日间时段客流（人次）"), (10, "7-8"), + (11, "8-9")] + + def val(r, c, k=k): + jia, yi = 10 + c + k, 20 + 2 * c + return {5: jia + yi, 6: jia, 7: yi}.get(r, (r * 7 + c * 3) % 50 + 10) + _crosstab(ws, rows, value=val) + return wb + + +def test_remap_facts_stays_on_the_fact_sheet(): + sc, grids = build(_two_flow_sheets()) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert d.complete, d.failures + by_id = {f.id: f for f in d.facts.facts} + assert by_id["F3"].sheet == "客流汇总二" and by_id["F3"].detail["segment"] == "客流汇总二_按日" + # 规则草稿原样按配方改写：一个字没改，事实也不能被改指到第一张表的分段 + remapped = {f.id: f.detail for f in S.detect_facts(grids, sc, recipe=d.recipe).facts} + assert remapped == {f.id: f.detail for f in d.facts.facts} + assert remapped["F4"]["b_segment"] == "客流汇总二_按日" + # 第二张表的宽表和明细分段改了名：按名字认得上的同表分段，明细分段按标题认 + renamed = rename_wide(d.recipe, "客流汇总二_按日", "日客流二") + renamed = json.loads(json.dumps(renamed, ensure_ascii=False).replace('"id": "日间时段客流_2"', '"id": "日间二"')) + seg_ids = [g["id"] for b in renamed["sheets"][1]["blocks"] for g in b["segments"]] + assert seg_ids == ["日客流二", "日间二"] + remapped = {f.id: f.detail for f in S.detect_facts(grids, sc, recipe=renamed).facts} + assert remapped["F1"]["segment"] == "客流汇总_按日" and remapped["F2"]["a"] == ["日间时段客流"] + assert remapped["F3"]["segment"] == "日客流二" + assert remapped["F4"]["b_segment"] == "日客流二" and remapped["F4"]["a"] == ["日间二"] + + +def test_remap_facts_two_long_tables_on_one_sheet(): + """一张工作表上两张长表（工作日、周末的时段相交，不合表）：改过 id 的明细分段按标题认，不把两张表拼在一起。""" + wb = Workbook() + ws = wb.active + ws.title = "时段" + ws["B2"] = "统计期：2026年8月1日至2026年8月8日" + rows = [(5, "全日（人次）"), (7, "工作日时段客流（人次）"), (8, "7-8"), (9, "8-9"), + (11, "周末时段客流（人次）"), (12, "7-8"), (13, "8-9")] + _crosstab(ws, rows) + sc, grids = build(wb) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + ne = [f for f in d.facts.facts if f.kind == "not_equal_sum"] + assert [f.detail["a"] for f in ne] == [["工作日时段客流"], ["周末时段客流"]] + renamed = json.loads(json.dumps(d.recipe, ensure_ascii=False) + .replace('"id": "工作日时段客流"', '"id": "工作日"') + .replace('"id": "周末时段客流"', '"id": "周末"')) + got = [f.detail["a"] for f in S.detect_facts(grids, sc, recipe=renamed).facts if f.kind == "not_equal_sum"] + assert got == [["工作日"], ["周末"]] + # 不给标题（直接调 remap_facts）时认不全：两张长表，不取「整张工作表的全部分段」，原样留着 + plain = S.remap_facts(d.facts, renamed) + assert [f.detail["a"] for f in plain.facts if f.kind == "not_equal_sum"] == [["工作日时段客流"], ["周末时段客流"]] + + +def test_remap_facts_follows_renamed_sheet_only_when_unambiguous(): + sc, grids = build(_two_flow_sheets(names=("客流汇总",))) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + moved = json.loads(json.dumps(d.recipe, ensure_ascii=False)) + moved["sheets"][0]["match"]["name"] = "上月的表名" + moved = rename_wide(moved, "客流汇总_按日", "日客流") + # 文件里只有一张可见工作表、配方里一张可回退：按回退认 + assert S.detect_facts(grids, sc, recipe=moved).facts[0].detail["segment"] == "日客流" + # 不告诉有几张可见工作表时不回退 + assert S.remap_facts(d.facts, moved).facts[0].detail["segment"] == "客流汇总_按日" + + +# ========================================================================== +# 卡片、命名、失败原因 +# ========================================================================== + + +def test_hidden_sheet_card_is_not_duplicated_with_dry_run(): + from app.data.recipe_types import SheetsOut + + wb, cache, _ = flow_book() + spare = wb.create_sheet("备用") + spare["A1"] = "不导入" + spare.sheet_state = "hidden" + sc, grids = build(wb, cache) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate(), + dry_run=lambda r: Extraction(ok=True, sheets=SheetsOut(skipped_hidden=[{"sheet": "备用"}]))) + ids = [c.id for c in d.cards] + assert len(ids) == len(set(ids)) + assert ids[-2:] == ["hidden_sheet:备用", "mode"] + # 没有干跑时由 draft 按扫描结果补上 + d2 = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert [c.id for c in d2.cards][-2:] == ["hidden_sheet:备用", "mode"] + + +def test_deduped_long_table_keeps_a_meaningful_value_column(): + sc, grids = build(_two_flow_sheets()) + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + segs = [g for s in d.recipe["sheets"] for b in s["blocks"] for g in b["segments"] if g["role"] == "dimension"] + assert [(g["table"], g["value"]) for g in segs] == [("日间时段客流", "客流"), ("日间时段客流_2", "客流")] + ne = [r for r in d.recipe["relations"] if r["kind"] == "not_comparable"] + assert [r["a"] for r in ne] == [{"table": "日间时段客流", "value": "客流"}, + {"table": "日间时段客流_2", "value": "客流"}] + + +def test_internal_errors_do_not_expose_exception_names(flow, monkeypatch, caplog): + sc, grids, _ = flow + + def boom(*a, **k): + raise KeyError("张三") + monkeypatch.setattr(S, "_analyze", boom) + with caplog.at_level("WARNING", logger="app.data.recipe_suggest"): + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + assert not d.complete and d.recipe is None + assert d.failures == ["规则起草未能完成：分析表格版式时出错。请在配方面板中粘贴一份完整的配方"] + assert "KeyError" in caplog.text and "张三" not in caplog.text + monkeypatch.undo() + + def bad_validate(data, facts, origin): + raise RuntimeError("x") + d2 = S.draft(sc, grids, "x.xlsx", validate=bad_validate) + assert d2.failures == ["配方检查未能完成：检查时出错。请在配方面板中修改配方后重试"] + + def bad_dry(recipe): + raise ValueError("x") + d3 = S.draft(sc, grids, "x.xlsx", validate=ok_validate(), dry_run=bad_dry) + assert d3.failures == ["按草稿检查表格未能完成：检查时出错。请在配方面板中修改配方后重试"] + for f in [*d.failures, *d2.failures, *d3.failures, + *d.failures_for_model, *d2.failures_for_model, *d3.failures_for_model]: + assert not any(name in f for name in ("Error", "Exception")) + + +def test_long_list_with_many_blank_numbers_stays_one_block_and_fast(): + """几百行、隔行空着金额的名单：每一行都「像表头」，既不能切成几百块，也不能逐行从头推类型（行数的平方）。""" + import time + + wb = Workbook() + ws = wb.active + ws.title = "名单" + ws.append(["姓名", "部门", "金额", "备注"]) + for i in range(499): + ws.append([f"人员{i:03d}", f"部门{'甲乙丙'[i % 3]}", None if i % 2 else i + 1, f"说明{i}"]) + sc, grids = build(wb) + t0 = time.perf_counter() + d = S.draft(sc, grids, "x.xlsx", validate=ok_validate()) + S.layout_hints(grids["名单"]) + assert time.perf_counter() - t0 < 1.0 + assert d.complete, d.failures + assert len(d.recipe["sheets"][0]["blocks"]) == 1 diff --git a/backend/tests/test_recipe_validate.py b/backend/tests/test_recipe_validate.py new file mode 100644 index 00000000..83494a39 --- /dev/null +++ b/backend/tests/test_recipe_validate.py @@ -0,0 +1,1108 @@ +"""配方静态校验（recipe.py，P2-SPEC 2.2、2.3）：每条规则一正一反，反例断言 code 和 path。 + +只用契约里的参考配方（flow_recipe.json）和在测试里手写的小配方，不读任何 Excel。 +""" +from __future__ import annotations + +import copy +import hashlib +import json +from pathlib import Path + +import pytest + +from app.data.recipe import ( + RecipeInvalid, + apply_patch, + canonical_recipe, + parse_recipe, + recipe_sha256, + validate_recipe, +) +from app.data.recipe_types import DraftFacts, Fact, Recipe + +FLOW = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text(encoding="utf-8")) +SEG = "/sheets/0/blocks/0/segments" + +F1 = Fact("F1", "sum_eq", "客流汇总", "第 5 行 = 第 6 行 + 第 7 行（31 列中 31 列成立）", + {"segment": "日客流", "total": "全日客流（人次）", "parts": ["分区甲（人次）", "分区乙（人次）"], "rows": [5, 6, 7]}) +F2 = Fact("F2", "not_equal_sum", "客流汇总", "日间、夜间各行之和 与 第 5 行（31 列中 0 列相等）", + {"a": ["日间", "夜间"], "b_segment": "日客流", "b": "全日客流（人次）", "equal": 0, "checked": 31}) +FACTS = DraftFacts(facts=[F1, F2]) + + +def flow() -> dict: + return copy.deepcopy(FLOW) + + +def segs(r: dict) -> list[dict]: + return r["sheets"][0]["blocks"][0]["segments"] + + +def table(r: dict, name: str) -> dict: + return next(t for t in r["tables"] if t["name"] == name) + + +def list_recipe() -> dict: + """lab c01 那种列表：表头 地区 / 产品 / 销量 / 金额（万元），合计行另存。""" + return { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": "月报"}, "blocks": [{ + "id": "列表1", "layout": "list", "table": "销售", + "columns": [{"header": "地区", "name": "地区", "type": "TEXT"}, + {"header": "产品", "name": "产品", "type": "TEXT"}, + {"header": "销量", "name": "销量", "type": "INTEGER"}, + {"header": "金额（万元）", "name": "金额", "type": "REAL"}], + "rows": {"total_row": {"label_column": "地区", "pick": "合计", "keep_as": "销售_表内合计"}}}]}], + "tables": [{"name": "销售", "grain": ["地区", "产品"], "units": {"金额": "万元"}}, + {"name": "销售_表内合计", "kind": "reported_total", "grain": ["合计项"], "units": {"金额": "万元"}}], + } + + +def run(data, *, facts=FACTS, origin="manual"): + _recipe, problems = validate_recipe(data, facts=facts, origin=origin) + return problems + + +def only(data, code, path=None, **kw): + """恰好一条问题，code（和 path）对得上：反例不能连带一串别的问题，否则看不出规则本身对不对。""" + problems = run(data, **kw) + assert [p.code for p in problems] == [code], [(p.code, p.path, p.message) for p in problems] + if path is not None: + assert problems[0].path == path + return problems[0] + + +def has(data, code, path=None, **kw): + problems = run(data, **kw) + hits = [p for p in problems if p.code == code and (path is None or p.path == path)] + assert hits, [(p.code, p.path, p.message) for p in problems] + return hits[0] + + +# ========================================================================== +# 参考配方 +# ========================================================================== + + +@pytest.mark.parametrize("origin", ["rules", "ai", "manual", "replay"]) +def test_reference_recipe_has_no_problems(origin): + recipe, problems = validate_recipe(flow(), facts=FACTS, origin=origin) + assert problems == [] + assert isinstance(recipe, Recipe) + + +def test_reference_recipe_without_facts_and_as_model(): + assert run(flow(), facts=None) == [] + model = Recipe.model_validate(flow()) + recipe, problems = validate_recipe(model) + assert recipe is model and problems == [] + + +def test_list_recipe_has_no_problems(): + assert run(list_recipe(), facts=None) == [] + + +# ========================================================================== +# schema（规则 1、5、7）：parse_recipe 转 pydantic 的错误 +# ========================================================================== + + +def test_closed_language_extra_fields_are_schema_problems_with_clean_paths(): + r = flow() + r["sheets"][0]["regex"] = "x" + segs(r)[1]["const"]["时段类别"] = {"value": "日间"} + recipe, problems = validate_recipe(r) + assert recipe is None + assert {p.code for p in problems} == {"schema"} + # loc 里 pydantic 插的判别字段（crosstab、dimension）去掉，path 对得上配方里的位置 + assert {p.path for p in problems} == {"/sheets/0/regex", f"{SEG}/1/const/时段类别/value", + f"{SEG}/1/const/时段类别/pick"} + by_path = {p.path: p.message for p in problems} + assert by_path["/sheets/0/regex"] == "第 1 个工作表「客流汇总」：有配方语言不支持的字段「regex」" + assert by_path[f"{SEG}/1/const/时段类别/pick"].startswith("第 1 个工作表「客流汇总」的分段「日间」：") + + +@pytest.mark.parametrize("mutate,path", [ + (lambda r: r.__setitem__("recipe_format", "agentlab-recipe/1"), "/recipe_format"), + (lambda r: segs(r)[1]["dim"].__setitem__("parser", "regex"), f"{SEG}/1/dim/parser"), + (lambda r: r["relations"][0].__setitem__("id", "X1"), "/relations/0/id"), + (lambda r: segs(r)[0].pop("measures"), f"{SEG}/0/measures"), + (lambda r: r["sheets"][0]["blocks"][0].__setitem__("layout", "grid"), "/sheets/0/blocks/0"), + (lambda r: r["sheets"][0]["blocks"][0]["values"]["placeholders"][0].__setitem__("to", 0), + "/sheets/0/blocks/0/values/placeholders/0/to"), +]) +def test_schema_problems_point_at_the_field(mutate, path): + r = flow() + mutate(r) + recipe, problems = validate_recipe(r) + assert recipe is None + assert [(p.code, p.path) for p in problems] == [("schema", path)] + assert "：" in problems[0].message + + +def test_parse_recipe_raises_with_problems(): + with pytest.raises(RecipeInvalid) as info: + parse_recipe({"recipe_format": "agentlab-recipe/2"}) + assert {p.code for p in info.value.problems} == {"schema"} + assert {p.path for p in info.value.problems} == {"/sheets", "/tables"} + with pytest.raises(RecipeInvalid): + parse_recipe(["not", "a", "dict"]) # type: ignore[arg-type] + assert isinstance(parse_recipe(flow()), Recipe) + + +# ========================================================================== +# 规则 2：常量只能 pick 分段标题的候选词 +# ========================================================================== + + +def test_const_pick_must_be_a_candidate_word(): + r = flow() + segs(r)[2]["const"]["时段类别"]["pick"] = "晚上" + p = only(r, "const_not_candidate", f"{SEG}/2/const/时段类别/pick") + assert p.message == ("第 1 个工作表「客流汇总」的分段「夜间」：常量「时段类别」的取值「晚上」不在分段标题" + "「夜间时段客流（人次）」的候选词中，可选：夜间、夜间时段客流") + + +def test_const_pick_other_candidate_is_fine(): + r = flow() + segs(r)[2]["const"]["时段类别"]["pick"] = "夜间时段客流" + assert run(r) == [] + + +def test_const_needs_a_section_title(): + r = flow() + segs(r)[1]["locate"] = {"by": "labels"} + has(r, "const_not_candidate", f"{SEG}/1/const/时段类别/pick") + + +# ========================================================================== +# 规则 3：占位符不能像数 +# ========================================================================== + + +@pytest.mark.parametrize("text", ["0", "-1", "1,234", "12.5%"]) +def test_placeholder_must_not_look_numeric(text): + r = flow() + r["sheets"][0]["blocks"][0]["values"]["placeholders"][0]["text"] = text + only(r, "placeholder_numeric", "/sheets/0/blocks/0/values/placeholders/0/text") + + +@pytest.mark.parametrize("text", ["·", "—", "无", "N/A"]) +def test_placeholder_text_ok(text): + r = flow() + r["sheets"][0]["blocks"][0]["values"]["placeholders"][0]["text"] = text + assert run(r) == [] + + +def test_list_placeholder_must_not_look_numeric(): + r = list_recipe() + r["sheets"][0]["blocks"][0]["values"] = {"placeholders": [{"text": "0"}]} + only(r, "placeholder_numeric", "/sheets/0/blocks/0/values/placeholders/0/text", facts=None) + + +# ========================================================================== +# 规则 4：单位封闭词表 +# ========================================================================== + + +def test_unit_must_be_in_vocabulary(): + r = flow() + # 标签不带单位，只看词表这一条 + seg = segs(r)[0] + seg["labels"]["expect"][0] = "全日客流" + seg["measures"] = {"全日客流": "全日客流", "分区甲（人次）": "分区甲", "分区乙（人次）": "分区乙"} + table(r, "日客流")["units"]["全日客流"] = "人头" + p = only(r, "unit_unknown", "/tables/0/units/全日客流", facts=None) + assert p.message == "表「日客流」：列「全日客流」的单位「人头」不在单位词表中" + + +def test_unit_in_vocabulary_ok(): + r = list_recipe() + r["sheets"][0]["blocks"][0]["columns"][2]["header"] = "销量（件）" + r["tables"][0]["units"]["销量"] = "件" + r["tables"][1]["units"]["销量"] = "件" + assert run(r, facts=None) == [] + + +# ========================================================================== +# 规则 5：解析器固定集合（标签解析不了的、重复的在保存时就报） +# ========================================================================== + + +def test_dimension_labels_must_parse(): + r = flow() + segs(r)[1]["labels"]["expect"][1] = "8:30-9:00" + only(r, "label_unparsed", f"{SEG}/1/labels/expect/1") + + +def test_derived_labels_must_parse_as_totals(): + r = flow() + segs(r)[3]["labels"]["expect"][0] = "18-22" + only(r, "label_unparsed", f"{SEG}/3/labels/expect/0") + + +def test_duplicate_labels_by_canonical_form(): + r = flow() + segs(r)[1]["labels"]["expect"][1] = "07:00-08:00" # 规范写法是 7-8，与第一项重复 + only(r, "label_duplicate", f"{SEG}/1/labels/expect/1") + + +def test_text_dimension_labels_ok(): + """文本维度：标签只要非空、不超长；不能派生起止小时。日间改成文本维度、单独成表。""" + r = flow() + seg = segs(r)[1] + seg.update(table="日间客流", dim={"name": "时段", "parser": "text"}, const={}) + seg["labels"]["expect"] = ["早上", "中午", "下午"] + r["tables"].insert(1, {"name": "日间客流", "grain": ["日期", "时段"], "units": {"客流": "人次"}}) + # 夜间独占「时段客流」后没有兄弟标题，候选词只剩整个标题 + has(r, "const_not_candidate", f"{SEG}/2/const/时段类别/pick") + segs(r)[2]["const"]["时段类别"]["pick"] = "夜间时段客流" + assert run(r) == [] + seg["labels"]["expect"].append("很" * 41) + only(r, "label_unparsed", f"{SEG}/1/labels/expect/3") + + +# ========================================================================== +# 规则 6：不能写死年份或统计期 +# ========================================================================== + + +def test_const_pick_with_digits_is_period_literal(): + r = flow() + segs(r)[1]["const"]["时段类别"]["pick"] = "2026日间" + only(r, "period_literal", f"{SEG}/1/const/时段类别/pick") + + +def test_prefer_prefix_with_digits_is_period_literal(): + r = flow() + r["sheets"][0]["context"][0]["prefer_prefix"] = "2026年统计" + only(r, "period_literal", "/sheets/0/context/0/prefer_prefix") + + +def test_total_row_pick_with_digits_is_period_literal(): + r = list_recipe() + r["sheets"][0]["blocks"][0]["rows"]["total_row"]["pick"] = "2026合计" + only(r, "period_literal", "/sheets/0/blocks/0/rows/total_row/pick", facts=None) + + +def test_year_in_list_header_is_not_an_error(): + r = list_recipe() + r["sheets"][0]["blocks"][0]["columns"][2]["header"] = "2026年上半年_销量" + assert run(r, facts=None) == [] + + +# ========================================================================== +# 规则 7：格式版本 +# ========================================================================== + + +def test_accumulate_mode_unsupported(): + r = flow() + r["mode"] = "accumulate" + only(r, "mode_unsupported", "/mode") + + +# ========================================================================== +# 规则 8：名字、键唯一 +# ========================================================================== + + +def test_table_name_must_be_valid(): + r = flow() + table(r, "时段客流_表内合计")["name"] = "select" + segs(r)[3]["keep_as"]["table"] = "select" + p = only(r, "name_invalid", "/tables/2/name") + assert "关键字" in p.message + + +def test_column_name_must_be_valid(): + r = flow() + for seg in segs(r)[1:3]: + seg["const"] = {"1类别": seg["const"]["时段类别"]} + problems = run(r) + assert [(p.code, p.path) for p in problems] == [ + ("name_invalid", f"{SEG}/1/const/1类别"), ("name_invalid", f"{SEG}/2/const/1类别")] + + +def test_column_names_collide_within_a_table(): + r = flow() + for seg in segs(r)[1:3]: + seg["const"] = {"时段": seg["const"]["时段类别"]} + problems = run(r) + assert [(p.code, p.path) for p in problems] == [ + ("name_collision", f"{SEG}/1/const/时段"), ("name_collision", f"{SEG}/2/const/时段")] + + +def test_column_names_collide_case_insensitively(): + r = list_recipe() + cols = r["sheets"][0]["blocks"][0]["columns"] + cols[0]["name"], cols[1]["name"] = "Area", "area" + r["sheets"][0]["blocks"][0]["rows"]["total_row"]["label_column"] = "Area" + r["tables"][0]["grain"] = ["Area"] + only(r, "name_collision", "/sheets/0/blocks/0/columns/1/name", facts=None) + + +def test_table_names_collide(): + r = list_recipe() + r["sheets"][0]["blocks"][0]["rows"]["total_row"]["keep_as"] = "ＳＡＬＥＳ" + r["sheets"][0]["blocks"][0]["table"] = "sales" + r["tables"][0]["name"] = "sales" + r["tables"][1]["name"] = "ＳＡＬＥＳ" + problems = run(r, facts=None) + assert ("name_collision", "/tables/1/name") in [(p.code, p.path) for p in problems] + + +def _second_sheet(r: dict, block: dict, tables: list[dict]) -> dict: + r["sheets"].append({"id": "s2", "match": {"name": "附表"}, "blocks": [block]}) + r["tables"] += tables + return r + + +def _extra_measures(seg_id: str, block_id: str = "交叉表2") -> tuple[dict, list[dict]]: + block = {"id": block_id, "layout": "crosstab", "axis": {"year_from": None, "checks": ["contiguous"]}, + "segments": [{"id": seg_id, "role": "measures", "table": "附表客流", "locate": {"by": "labels"}, + "labels": {"expect": ["附表客流"]}, "measures": {"附表客流": "附表客流"}}]} + return block, [{"name": "附表客流", "grain": ["日期"]}] + + +def test_second_sheet_ok(): + block, tables = _extra_measures("附表") + assert run(_second_sheet(flow(), block, tables)) == [] + + +def test_segment_id_unique_across_blocks(): + block, tables = _extra_measures("日间") + only(_second_sheet(flow(), block, tables), "segment_id_duplicate", "/sheets/1/blocks/0/segments/0/id") + + +def test_block_id_unique_across_sheets(): + block, tables = _extra_measures("附表", block_id="交叉表") + only(_second_sheet(flow(), block, tables), "block_id_duplicate", "/sheets/1/blocks/0/id") + + +def test_block_id_must_not_reuse_a_segment_id(): + block, tables = _extra_measures("附表", block_id="夜间") + only(_second_sheet(flow(), block, tables), "block_id_duplicate", "/sheets/1/blocks/0/id") + + +def test_sheet_id_and_sheet_name_unique(): + block, tables = _extra_measures("附表") + r = _second_sheet(flow(), block, tables) + r["sheets"][1]["id"] = "s1" + only(r, "sheet_id_duplicate", "/sheets/1/id") + r["sheets"][1]["id"] = "s2" + r["sheets"][1]["match"]["name"] = "客流 汇总" + only(r, "sheet_name_duplicate", "/sheets/1/match/name") + + +def test_one_crosstab_per_sheet(): + r = flow() + block, tables = _extra_measures("附表") + r["sheets"][0]["blocks"].append(block) + r["tables"] += tables + only(r, "crosstab_twice", "/sheets/0/blocks/1") + + +# ========================================================================== +# 规则 9：引用完整（逐字比较） +# ========================================================================== + + +def test_segment_table_must_match_verbatim(): + r = list_recipe() + r["sheets"][0]["blocks"][0]["table"] = "Sales" + r["tables"][0]["name"] = "sales" + problems = run(r, facts=None) + assert [(p.code, p.path) for p in problems] == [("table_unknown", "/sheets/0/blocks/0/table"), + ("table_unused", "/tables/0")] + assert "「sales」" in problems[0].message + + +def test_crosstab_segment_table_case_differs(): + r = flow() + table(r, "日客流")["name"] = "sales" + segs(r)[0]["table"] = "Sales" + r["relations"][0]["table"] = r["relations"][1]["b"]["table"] = "sales" + p = has(r, "table_unknown", f"{SEG}/0/table") + assert "「sales」" in p.message + + +def test_crosstab_segment_table_unknown(): + r = flow() + segs(r)[0]["table"] = "日客流表" + has(r, "table_unknown", f"{SEG}/0/table") + + +def test_keep_as_table_unknown(): + r = flow() + segs(r)[3]["keep_as"]["table"] = "时段客流_合计" + problems = run(r) + assert [(p.code, p.path) for p in problems] == [("table_unknown", f"{SEG}/3/keep_as/table"), + ("table_unused", "/tables/2")] + + +def test_verify_against_table_unknown_and_invalid(): + r = flow() + segs(r)[3]["verify"]["against_table"] = "时段明细" + only(r, "table_unknown", f"{SEG}/3/verify/against_table") + r["sheets"][0]["blocks"][0]["segments"][3]["verify"]["against_table"] = "日客流" + only(r, "verify_against_invalid", f"{SEG}/3/verify/against_table") + + +def test_verify_value_must_be_the_value_column(): + r = flow() + segs(r)[3]["verify"]["value"] = "人次" + only(r, "column_unknown", f"{SEG}/3/verify/value") + segs(r)[3]["verify"]["value"] = "起始小时" + only(r, "verify_against_invalid", f"{SEG}/3/verify/value") + + +@pytest.mark.parametrize("key", ["销 量", "Qty"]) +def test_units_key_must_be_a_column_verbatim(key): + r = list_recipe() + if key == "Qty": + r["sheets"][0]["blocks"][0]["columns"][2]["name"] = "qty" + r["tables"][0]["units"][key] = "件" + only(r, "column_unknown", f"/tables/0/units/{key}", facts=None) + + +def test_units_key_with_extra_space_in_flow(): + r = flow() + units = table(r, "日客流")["units"] + units["全日客流 "] = units.pop("全日客流") + has(r, "column_unknown", "/tables/0/units/全日客流 ") + + +def test_grain_column_unknown(): + r = flow() + table(r, "时段客流")["grain"] = ["日期", "时段名"] + only(r, "column_unknown", "/tables/1/grain/1") + + +def test_relation_columns_and_types(): + r = flow() + r["relations"][0]["parts"] = ["分区甲", "分区丁"] + has(r, "column_unknown", "/relations/0/parts/1") + r = flow() + r["relations"][0]["total"] = "日期" + has(r, "relation_invalid", "/relations/0/total", facts=None) + r = flow() + r["relations"][0]["table"] = "日客流表" + only(r, "table_unknown", "/relations/0/table", facts=None) + + +def test_not_comparable_by_must_exist_on_both_sides(): + r = flow() + r["relations"][1]["by"] = "时段" + only(r, "column_unknown", "/relations/1/by") + + +def test_relation_ids_unique(): + r = flow() + r["relations"][1]["id"] = "R1" + only(r, "relation_invalid", "/relations/1/id") + + +def test_total_row_label_column_must_exist(): + r = list_recipe() + r["sheets"][0]["blocks"][0]["rows"]["total_row"]["label_column"] = "区域" + only(r, "column_unknown", "/sheets/0/blocks/0/rows/total_row/label_column", facts=None) + + +def ascii_flow() -> dict: + """参考配方换成英文表名、列名（标签和分段 id 不动，FACTS 照样能认领）。 + + 「差一点」的引用要两种各造一个才能把逐字比较钉死：只差 ASCII 大小写的（name_key 相同、match_key 不同） + 和只差空白的（match_key 相同、name_key 不同）。中文名没有大小写，前一种只能用英文名造。 + """ + r = flow() + s = segs(r) + r["sheets"][0]["blocks"][0]["axis"]["name"] = "Day" + s[0]["table"] = "Daily" + s[0]["measures"] = {"全日客流（人次）": "Total", "分区甲（人次）": "PartA", "分区乙（人次）": "PartB"} + for seg in s[1:3]: + seg.update(table="Hourly", value="Flow", const={"Kind": seg["const"]["时段类别"]}) + seg["dim"].update(name="Slot", derive={"StartH": "start", "EndH": "end"}) + s[3]["verify"].update(against_table="Hourly", value="Flow") + s[3]["keep_as"] = {"table": "Hourly_total", "dim": "Item", "derive": {"StartH": "start", "EndH": "end"}, + "value": "Flow"} + r["tables"] = [ + {"name": "Daily", "grain": ["Day"], "units": {"Total": "人次", "PartA": "人次", "PartB": "人次"}}, + {"name": "Hourly", "grain": ["Day", "Slot"], "units": {"Flow": "人次"}}, + {"name": "Hourly_total", "kind": "reported_total", "grain": ["Day", "Item"], "units": {"Flow": "人次"}}, + ] + r["relations"] = [ + {"id": "R1", "kind": "sum_eq", "table": "Daily", "total": "Total", "parts": ["PartA", "PartB"], "claims": "F1"}, + {"id": "R2", "kind": "not_comparable", "a": {"table": "Hourly", "value": "Flow"}, + "b": {"table": "Daily", "value": "Total"}, "by": "Day", "claims": "F2"}, + ] + return r + + +def ascii_list() -> dict: + r = list_recipe() + block = r["sheets"][0]["blocks"][0] + block["columns"][0]["name"] = "Region" + block["rows"]["total_row"]["label_column"] = "Region" + r["tables"][0]["grain"] = ["Region", "产品"] + return r + + +#: 差一点的写法：只差大小写（name_key 当成同一个）、多一个空格（match_key 当成同一个） +NEAR = {"case": str.lower, "space": lambda s: s + " "} + + +def _get(doc, pointer: str): + for part in pointer.split("/")[1:]: + doc = doc[int(part)] if isinstance(doc, list) else doc[part] + return doc + + +def test_ascii_variants_have_no_problems(): + assert run(ascii_flow()) == [] + assert run(ascii_list(), facts=None) == [] + + +@pytest.mark.parametrize("near", NEAR) +@pytest.mark.parametrize("base,pointer,expected", [ + # 关系里的表和列（table_ref、numeric_col、not_comparable.by） + (ascii_flow, "/relations/0/table", [("table_unknown", "/relations/0/table")]), + (ascii_flow, "/relations/0/total", [("column_unknown", "/relations/0/total")]), + (ascii_flow, "/relations/0/parts/0", [("column_unknown", "/relations/0/parts/0")]), + (ascii_flow, "/relations/1/a/table", [("table_unknown", "/relations/1/a/table")]), + (ascii_flow, "/relations/1/b/table", [("table_unknown", "/relations/1/b/table")]), + (ascii_flow, "/relations/1/a/value", [("column_unknown", "/relations/1/a/value")]), + (ascii_flow, "/relations/1/b/value", [("column_unknown", "/relations/1/b/value")]), + (ascii_flow, "/relations/1/by", [("column_unknown", "/relations/1/by")]), + # 主键 + (ascii_flow, "/tables/1/grain/1", [("column_unknown", "/tables/1/grain/1")]), + # 合计分段的核对基表、核对的值列、另存的表 + (ascii_flow, f"{SEG}/3/verify/against_table", [("table_unknown", f"{SEG}/3/verify/against_table")]), + (ascii_flow, f"{SEG}/3/verify/value", [("column_unknown", f"{SEG}/3/verify/value")]), + (ascii_flow, f"{SEG}/3/keep_as/table", [("table_unknown", f"{SEG}/3/keep_as/table"), + ("table_unused", "/tables/2")]), + # 分段写入的表：日客流一改，表清单里的 Daily 没人写，关系里的 Daily 也就不是配方产出的表 + (ascii_flow, f"{SEG}/0/table", [("table_unknown", f"{SEG}/0/table"), ("table_unused", "/tables/0"), + ("table_unknown", "/relations/0/table"), + ("table_unknown", "/relations/1/b/table")]), + # 列表合计行的标签列 + (ascii_list, "/sheets/0/blocks/0/rows/total_row/label_column", + [("column_unknown", "/sheets/0/blocks/0/rows/total_row/label_column")]), +]) +def test_internal_references_are_verbatim(near, base, pointer, expected): + """规则 9：配方内的引用逐字比较。每类引用各改成「差一点」的写法，必须报出来、且只报这些。""" + r = base() + wrong = NEAR[near](_get(r, pointer)) + r = apply_patch(r, [{"op": "replace", "path": pointer, "value": wrong}]) + problems = run(r, facts=None) + assert [(p.code, p.path) for p in problems] == expected, [(p.code, p.path, p.message) for p in problems] + + +@pytest.mark.parametrize("near", NEAR) +def test_verify_against_table_must_be_the_target_table_verbatim(near): + """核对基表在表清单里、与紧邻分段写入的表只差大小写或空格：仍不是那张表（名字本身另报撞名或不合规）。""" + r = ascii_flow() + other = NEAR[near]("Hourly") + r["tables"].append({"name": other}) + segs(r)[3]["verify"]["against_table"] = other + has(r, "verify_against_invalid", f"{SEG}/3/verify/against_table", facts=None) + + +def test_table_kind_must_match_writer(): + r = flow() + table(r, "时段客流_表内合计")["kind"] = "data" + only(r, "table_kind_mismatch", "/tables/2/kind") + + +def test_declared_table_without_writer(): + r = flow() + r["tables"].append({"name": "多余的表"}) + only(r, "table_unused", "/tables/3") + + +# ========================================================================== +# 规则 10：结构一致 +# ========================================================================== + + +def test_measures_keys_half_width_brackets(): + r = flow() + segs(r)[0]["measures"] = {"全日客流(人次)": "全日客流", "分区甲（人次）": "分区甲", "分区乙（人次）": "分区乙"} + # 只报根因：引用「全日客流」的单位、关系不再连带报「列不存在」 + p = only(r, "measures_keys_mismatch", f"{SEG}/0/measures") + assert "全日客流(人次)" in p.message + + +def test_table_shape_conflict(): + r = flow() + segs(r)[2]["value"] = "人次数" + only(r, "table_shape_conflict", f"{SEG}/2") + + +def test_derived_must_follow_a_dimension_with_stop_parser(): + r = flow() + segs(r)[2]["stop_parser"] = None + only(r, "derived_after_invalid", f"{SEG}/3/locate/segment") + r = flow() + segs(r)[3]["locate"]["segment"] = "日客流" + only(r, "derived_after_invalid", f"{SEG}/3/locate/segment") + r = flow() + segs(r)[3]["locate"]["segment"] = "不存在" + only(r, "derived_after_invalid", f"{SEG}/3/locate/segment") + r = flow() + segs(r)[3]["locate"] = {"by": "labels"} + only(r, "derived_after_invalid", f"{SEG}/3/locate/by") + + +def test_locate_rules_for_measures_and_dimension(): + r = flow() + segs(r)[0]["locate"] = {"by": "after", "segment": "日间"} + has(r, "locate_invalid", f"{SEG}/0/locate/by") + r = flow() + segs(r)[1]["locate"] = {"by": "section_title"} + has(r, "locate_invalid", f"{SEG}/1/locate/title") + + +def test_crosstab_grain_must_contain_axis(): + r = flow() + table(r, "日客流")["grain"] = [] + only(r, "grain_invalid", "/tables/0/grain") + + +def test_derive_role_duplicate(): + r = flow() + segs(r)[3]["keep_as"]["derive"] = {"起始小时": "start", "结束小时": "start"} + only(r, "derive_role_duplicate", f"{SEG}/3/keep_as/derive") + + +def test_derive_only_for_hour_range(): + r = flow() + for seg in segs(r)[1:3]: + seg["dim"]["parser"] = "text" + has(r, "derive_invalid", f"{SEG}/1/dim/derive") + + +def test_overlapping_dimension_segments_need_a_distinguishing_const_in_grain(): + r = flow() + segs(r)[1]["labels"]["expect"].append("18-19") + only(r, "grain_invalid", "/tables/1/grain") + table(r, "时段客流")["grain"] = ["日期", "时段", "时段类别"] + assert run(r) == [] + + +def test_overlapping_segments_with_same_pick_still_invalid(): + r = flow() + segs(r)[1]["labels"]["expect"].append("18-19") + segs(r)[2]["const"]["时段类别"]["pick"] = "夜间时段客流" + table(r, "时段客流")["grain"] = ["日期", "时段", "时段类别"] + assert run(r) == [] + segs(r)[1]["locate"]["title"] = "夜间时段客流（人次）" + segs(r)[1]["const"]["时段类别"]["pick"] = "夜间时段客流" + has(r, "grain_invalid", "/tables/1/grain") + + +def test_covers_context_and_year_need_a_context(): + r = flow() + r["sheets"][0]["context"] = [] + problems = run(r) + assert [(p.code, p.path) for p in problems] == [ + ("covers_without_context", "/sheets/0/blocks/0/axis/checks"), + ("covers_without_context", "/sheets/0/blocks/0/axis/year_from")] + r["sheets"][0]["blocks"][0]["axis"]["checks"] = ["contiguous"] + r["sheets"][0]["blocks"][0]["axis"]["year_from"] = None + assert run(r) == [] + + +# ========================================================================== +# 规则 11：认领（计数与语义） +# ========================================================================== + + +def test_fact_unclaimed(): + r = flow() + r["relations"][0]["claims"] = None + p = only(r, "fact_unclaimed", "/relations") + assert p.message == "系统发现的关系「第 5 行 = 第 6 行 + 第 7 行」没有被认领：请登记为每期核对，或说明不登记的理由" + + +def test_fact_claimed_twice(): + r = flow() + r["relations"].append({"id": "R3", "kind": "sum_eq", "table": "日客流", "total": "全日客流", + "parts": ["分区乙", "分区甲"], "claims": "F1"}) + only(r, "fact_claimed_twice", "/relations/2/claims") + + +def test_claim_of_unknown_fact(): + r = flow() + r["relations"].append({"id": "R3", "kind": "dismissed", "claims": "F9", "reason": "巧合"}) + only(r, "fact_claim_mismatch", "/relations/2/claims") + + +def test_claim_with_wrong_members(): + r = flow() + r["relations"][0].update(total="分区甲", parts=["全日客流", "分区乙"]) + p = only(r, "fact_claim_mismatch", "/relations/0/claims") + assert "「全日客流（人次）」" in p.message + + +def test_claim_with_wrong_kind(): + r = flow() + r["relations"][0], r["relations"][1] = ( + {"id": "R1", "kind": "sum_eq", "table": "日客流", "total": "全日客流", "parts": ["分区甲", "分区乙"], "claims": "F2"}, + {**r["relations"][1], "claims": "F1"}) + problems = run(r) + assert sorted((p.code, p.path) for p in problems) == [("fact_claim_mismatch", "/relations/0/claims"), + ("fact_claim_mismatch", "/relations/1/claims")] + + +def test_not_comparable_sides_must_match_fact(): + r = flow() + rel = r["relations"][1] + rel["a"], rel["b"] = rel["b"], rel["a"] + only(r, "fact_claim_mismatch", "/relations/1/claims") + r = flow() + r["relations"][1]["b"]["value"] = "分区甲" + only(r, "fact_claim_mismatch", "/relations/1/claims") + + +def test_sum_eq_claim_must_be_on_the_fact_segment_table(): + """另一张表的列名碰巧相同：R1 指过去，按 Fact 那个分段的 measures 反查列名能对上，但表不是那张表。""" + block = {"id": "交叉表2", "layout": "crosstab", "axis": {"year_from": None, "checks": ["contiguous"]}, + "segments": [{"id": "附表", "role": "measures", "table": "附表客流", "locate": {"by": "labels"}, + "labels": {"expect": ["全日客流", "分区甲", "分区乙"]}, + "measures": {"全日客流": "全日客流", "分区甲": "分区甲", "分区乙": "分区乙"}}]} + r = _second_sheet(flow(), block, [{"name": "附表客流", "grain": ["日期"]}]) + assert run(r) == [] + r["relations"][0]["table"] = "附表客流" + assert run(r, facts=None) == [] + p = only(r, "fact_claim_mismatch", "/relations/0/claims") + assert "「日客流」" in p.message and "「附表客流」" in p.message + + +def test_not_comparable_a_value_must_be_the_value_column(): + """F2 说的是日间、夜间「各行之和」，加总的是值列；换成派生列「起始小时」（也是数字列）口径就写错了。""" + r = flow() + r["relations"][1]["a"]["value"] = "起始小时" + assert run(r, facts=None) == [] + p = only(r, "fact_claim_mismatch", "/relations/1/claims") + assert "值列「客流」" in p.message and "「起始小时」" in p.message + + +def _rename_dimension_segments(r: dict, day: str | None, night: str | None) -> dict: + s = segs(r) + if day: + s[1]["id"] = day + if night: + s[2]["id"] = night + s[3]["locate"]["segment"] = night + return r + + +@pytest.mark.parametrize("day,night", [("白天", "晚上"), ("白天", None), (None, "晚上")]) +def test_renamed_dimension_segments_are_found_in_the_b_block(day, night): + """F2 的 a 只记分段 id（起草时取 pick：日间、夜间）；改名后按 b 那个 measures 段同块的全部 dimension 段认， + 照样核对内容，而不是报一条看不懂的认领错误。""" + r = _rename_dimension_segments(flow(), day, night) + assert run(r) == [] + r["relations"][1]["a"]["value"] = "起始小时" + only(r, "fact_claim_mismatch", "/relations/1/claims") + + +def test_renamed_dimension_segments_with_a_different_count_do_not_match(): + """改名后同块的 dimension 段个数与 a 不同（少了日间）：内容变了，不按同块认，照样报对不上。""" + r = flow() + del segs(r)[1] + seg = segs(r)[1] + seg["id"], seg["const"] = "晚上", {} + segs(r)[2]["locate"]["segment"] = "晚上" + assert run(r, facts=None) == [] + p = only(r, "fact_claim_mismatch", "/relations/1/claims") + assert "（配方里找不到这些分段）" in p.message + + +def test_dismissed_claims_any_fact(): + r = flow() + r["relations"][1] = {"id": "R2", "kind": "dismissed", "claims": "F2", "reason": "两块口径不同，不作关系登记"} + assert run(r) == [] + + +def test_claims_not_checked_without_facts(): + r = flow() + r["relations"][0]["claims"] = None + r["relations"][1]["claims"] = "F9" + assert run(r, facts=None) == [] + + +def test_renamed_measures_segment_is_found_by_labels(): + """起草时分段 id 是表名，改名后 Fact.detail.segment 对不上：按标签找到唯一的分段，照样核对内容。""" + r = flow() + segs(r)[0]["id"] = "按日" + assert run(r) == [] + r["relations"][0].update(total="分区甲", parts=["全日客流", "分区乙"]) + only(r, "fact_claim_mismatch", "/relations/0/claims") + + +def test_new_fact_numbering_is_not_an_empty_claim(): + """改配方后重新探测：F1 现在是「甲 + 乙 + 丙」（D13），旧的 R1 claims=F1 报出来，而不是空认领通过。""" + new_f1 = Fact("F1", "sum_eq", "客流汇总", "第 5 行 = 第 6 行 + 第 7 行 + 第 8 行（30 列中 30 列成立）", + {"segment": "日客流", "total": "全日客流（人次）", + "parts": ["分区甲（人次）", "分区乙（人次）", "分区丙（人次）"], "rows": [5, 6, 7, 8]}) + only(flow(), "fact_claim_mismatch", "/relations/0/claims", facts=DraftFacts(facts=[new_f1, F2])) + + +# ========================================================================== +# 规则 12：说明散文 +# ========================================================================== + + +def test_note_ok(): + r = flow() + table(r, "日客流")["note"] = "粒度：每个{列:日期}。{列:全日客流} 等于 {列:分区甲} 与 {列:分区乙} 之和，单位{单位:人次}" + assert run(r) == [] + + +@pytest.mark.parametrize("note", ["导入时已逐日核对，共 31 天", "数据见 C5 格", "约 5000 人次"]) +def test_note_with_numbers_or_coordinates(note): + r = flow() + table(r, "日客流")["note"] = note + has(r, "note_numbers", "/tables/0/note") + assert {p.code for p in run(r)} == {"note_numbers"} + + +@pytest.mark.parametrize("note", ["{列:增长30%}，请据此推算", "{列:客流} 不能相加", "{表:第2期}只有一半", "按{单位:箱}计"]) +def test_note_with_unknown_token(note): + r = flow() + table(r, "日客流")["note"] = note + has(r, "note_token_unknown", "/tables/0/note") + + +def test_ai_note_forbidden(): + r = flow() + table(r, "日客流")["note"] = "按{列:日期}逐日记录" + only(r, "ai_note_forbidden", "/tables/0/note", origin="ai") + assert run(r, origin="rules") == [] + + +# ========================================================================== +# 规则 13：单位与标签一致 +# ========================================================================== + + +def test_unit_label_conflict_wrong_unit(): + r = list_recipe() + r["tables"][0]["units"]["金额"] = "元" + p = only(r, "unit_label_conflict", "/tables/0/units/金额", facts=None) + assert "「金额（万元）」" in p.message and "万元" in p.message + + +def test_unit_label_conflict_missing_unit(): + r = list_recipe() + del r["tables"][0]["units"]["金额"] + p = only(r, "unit_label_conflict", "/tables/0/units", facts=None) + assert p.message == "表「销售」：原表表头「金额（万元）」写着单位「万元」，列「金额」的单位也必须写「万元」" + + +def test_unit_label_conflict_unit_outside_vocabulary(): + r = list_recipe() + r["sheets"][0]["blocks"][0]["columns"][2]["header"] = "销量（箱）" + assert run(r, facts=None) == [] + r["tables"][0]["units"]["销量"] = "件" + only(r, "unit_label_conflict", "/tables/0/units/销量", facts=None) + + +def test_unit_label_conflict_measures_and_section_title(): + r = flow() + table(r, "日客流")["units"]["分区甲"] = "人" + only(r, "unit_label_conflict", "/tables/0/units/分区甲") + r = flow() + table(r, "时段客流")["units"]["客流"] = "人" + only(r, "unit_label_conflict", "/tables/1/units/客流") + + +def test_unit_label_conflict_list_total_table(): + """列表合计表的数字列就是那几个表头：主表写对了、合计表写成「元」或漏写，同样要报。""" + r = list_recipe() + r["tables"][1]["units"]["金额"] = "元" + p = only(r, "unit_label_conflict", "/tables/1/units/金额", facts=None) + assert p.message.startswith("表「销售_表内合计」：") and "「金额（万元）」" in p.message + del r["tables"][1]["units"]["金额"] + only(r, "unit_label_conflict", "/tables/1/units", facts=None) + + +def test_unit_label_conflict_list_total_table_unit_outside_vocabulary(): + r = list_recipe() + r["sheets"][0]["blocks"][0]["columns"][2]["header"] = "销量（箱）" + r["tables"][1]["units"]["销量"] = "件" + only(r, "unit_label_conflict", "/tables/1/units/销量", facts=None) + + +def test_list_without_kept_total_has_no_total_table_units(): + r = list_recipe() + r["sheets"][0]["blocks"][0]["rows"]["total_row"]["keep_as"] = None + del r["tables"][1] + assert run(r, facts=None) == [] + + +def test_unit_label_conflict_crosstab_total_table(): + """表内合计和它紧跟的夜间明细同在「夜间时段客流（人次）」下：合计表的客流写错或漏写，同样要报。""" + r = flow() + table(r, "时段客流_表内合计")["units"]["客流"] = "人" + p = only(r, "unit_label_conflict", "/tables/2/units/客流") + assert p.message.startswith("表「时段客流_表内合计」：") and "「夜间时段客流（人次）」" in p.message + del table(r, "时段客流_表内合计")["units"]["客流"] + only(r, "unit_label_conflict", "/tables/2/units") + + +# ========================================================================== +# 规范形式与哈希 +# ========================================================================== + + +def _strip_defaults(r: dict) -> dict: + """手写的省略写法：去掉参考配方里显式写出的默认值。""" + r = copy.deepcopy(r) + for k in ("mode", "other_visible_sheets"): + r.pop(k) + sheet = r["sheets"][0] + sheet["match"].pop("fallback") + sheet.pop("hidden") + ctx = sheet["context"][0] + for k in ("id", "kind", "parser", "cross_check"): + ctx.pop(k) + block = sheet["blocks"][0] + block.pop("axis") + block.pop("label_offset") + for k in ("type", "blank", "text_number", "formula"): + block["values"].pop(k) + block["values"]["placeholders"][0].pop("meaning") + segs(r)[3].pop("labels_parser") + for t in r["tables"]: + t.pop("note") + if t["kind"] == "data": + t.pop("kind") + return r + + +def test_hash_ignores_explicit_defaults(): + assert recipe_sha256(parse_recipe(_strip_defaults(FLOW))) == recipe_sha256(parse_recipe(flow())) + + +def test_hash_ignores_axis_checks_order_and_key_order(): + r = flow() + r["sheets"][0]["blocks"][0]["axis"]["checks"] = ["covers_context", "contiguous"] + assert recipe_sha256(parse_recipe(r)) == recipe_sha256(parse_recipe(flow())) + shuffled = json.loads(json.dumps(flow(), sort_keys=True)) + shuffled = {k: shuffled[k] for k in reversed(list(shuffled))} + assert recipe_sha256(parse_recipe(shuffled)) == recipe_sha256(parse_recipe(flow())) + + +def test_hash_changes_with_content(): + r = flow() + r["sheets"][0]["blocks"][0]["axis"]["checks"] = ["contiguous"] + assert recipe_sha256(parse_recipe(r)) != recipe_sha256(parse_recipe(flow())) + + +def test_canonical_form_is_compact_and_round_trips(): + recipe = parse_recipe(flow()) + canon = canonical_recipe(recipe) + assert "mode" not in canon and "hidden" not in canon["sheets"][0] + assert "axis" not in canon["sheets"][0]["blocks"][0] + assert parse_recipe(canon) == recipe + assert canonical_recipe(parse_recipe(canon)) == canon + text = json.dumps(canon, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + assert recipe_sha256(recipe) == hashlib.sha256(text.encode("utf-8")).hexdigest() + + +# ========================================================================== +# apply_patch +# ========================================================================== + + +def test_apply_patch_add_replace_remove(): + doc = {"a": {"b": 1}, "l": [1, 2], "x/y": {"~k": 0}} + out = apply_patch(doc, [ + {"op": "add", "path": "/a/c", "value": [1]}, + {"op": "add", "path": "/l/1", "value": 9}, + {"op": "add", "path": "/l/-", "value": 3}, + {"op": "replace", "path": "/a/b", "value": {"n": 2}}, + {"op": "remove", "path": "/l/0"}, + {"op": "replace", "path": "/x~1y/~0k", "value": 5}, + {"op": "add", "path": "/a/b", "value": "overwritten"}, + ]) + assert out == {"a": {"b": "overwritten", "c": [1]}, "l": [9, 2, 3], "x/y": {"~k": 5}} + assert doc == {"a": {"b": 1}, "l": [1, 2], "x/y": {"~k": 0}} + + +def test_apply_patch_question_effects_on_recipe(): + """起草问题「不登记」的 effects：把关系整条换成 dismissed；占位符选「不是」：删掉占位符。""" + out = apply_patch(flow(), [ + {"op": "replace", "path": "/relations/1", + "value": {"id": "R2", "kind": "dismissed", "claims": "F2", "reason": "口径不同"}}, + {"op": "remove", "path": "/sheets/0/blocks/0/values/placeholders/0"}, + {"op": "replace", "path": f"{SEG}/2/const/时段类别/pick", "value": "夜间时段客流"}, + ]) + assert out["relations"][1]["kind"] == "dismissed" + assert out["sheets"][0]["blocks"][0]["values"]["placeholders"] == [] + assert segs(out)[2]["const"]["时段类别"]["pick"] == "夜间时段客流" + assert segs(FLOW)[2]["const"]["时段类别"]["pick"] == "夜间" + assert run(out) == [] + + +def test_apply_patch_root_replace(): + assert apply_patch({"a": 1}, [{"op": "replace", "path": "", "value": {"b": 2}}]) == {"b": 2} + + +@pytest.mark.parametrize("ops", [ + [{"op": "replace", "path": "/nope", "value": 1}], + [{"op": "remove", "path": "/a/zz"}], + [{"op": "add", "path": "/l/5", "value": 1}], + [{"op": "replace", "path": "/l/2", "value": 1}], + [{"op": "remove", "path": "/l/01"}], + [{"op": "remove", "path": "/l/x"}], + [{"op": "add", "path": "/q/r", "value": 1}], + [{"op": "add", "path": "a", "value": 1}], + [{"op": "move", "from": "/a", "path": "/b"}], + [{"op": "add", "path": "/b"}], + [{"path": "/a"}], + [{"op": "remove", "path": ""}], + [{"op": "replace", "path": "", "value": [1]}], + [{"op": "add", "path": "/a/b/c", "value": 1}], + ["not an op"], +]) +def test_apply_patch_errors(ops): + doc = {"a": {"b": 1}, "l": [1, 2]} + with pytest.raises(ValueError): + apply_patch(doc, ops) + assert doc == {"a": {"b": 1}, "l": [1, 2]} + + +def test_apply_patch_is_all_or_nothing(): + doc = {"a": 1} + with pytest.raises(ValueError): + apply_patch(doc, [{"op": "replace", "path": "/a", "value": 2}, {"op": "remove", "path": "/zz"}]) + assert doc == {"a": 1} + + +def test_many_unknown_unit_keys_on_a_wide_table_validate_quickly(): + """SE-5：200 列的宽表上 2 万个不存在的单位键：_near 按名字列表建一次键索引，不再逐键对全表各列重算 NFKC + （改之前约 11 秒）。提示照旧：大小写、全半角不同的名字仍会点出来。""" + import time + + from app.data.recipe import validate_recipe + + data = copy.deepcopy(FLOW) + seg = data["sheets"][0]["blocks"][0]["segments"][0] + labels = [f"指标{i}（人次）" for i in range(200)] + seg["labels"]["expect"] = labels + seg["measures"] = {lab: f"指标{i}" for i, lab in enumerate(labels)} + data["tables"][0]["units"] = {f"指标{i}": "人次" for i in range(200)} + data["relations"] = [] + data["tables"][0]["units"].update({f"col_{i}_x": "人次" for i in range(20000)}) + data["tables"][0]["units"]["指标７"] = "人次" # 全角数字：提示「有「指标7」」 + t = time.perf_counter() + _parsed, probs = validate_recipe(data) + assert time.perf_counter() - t < 4 + assert sum(1 for p in probs if p.code == "column_unknown") == 20001 + hint = next(p.message for p in probs if "指标７" in p.message) + assert "（有「指标7」，名字必须逐字相同）" in hint diff --git a/backend/tests/test_table_imports_api.py b/backend/tests/test_table_imports_api.py new file mode 100644 index 00000000..b8213049 --- /dev/null +++ b/backend/tests/test_table_imports_api.py @@ -0,0 +1,698 @@ +"""按配方导入的接口（期 2，WP-5c）：真管线的端到端，加上用假 Pipeline 测的接口形状与错误码。 + +真管线（default_pipeline）跑「结构仿照客流表」的合成夹具（tests/fixtures/xlsx/flow.py，假名、随机数）： +- 暂存 → 规则起草 → 回答问题 → 改表名（PUT）→ 试运行 → 提交； +- 同版式的下月文件「上传新一期」→ 回执与差异卡 → 启用； +- 同一文件再传 → same_as_import → 提交不新建记录； +- 简单导入的源切换为按配方导入。 +其余（错误码、AI 起草的门槛、并发、路由不被截走）用 test_recipe_imports 里的假 Pipeline。不调用任何真实模型。 +""" +from __future__ import annotations + +import datetime as dt +import json +import uuid +from pathlib import Path +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import select + +from app.core import artifact_store +from app.core.config import settings +from app.data import recipe, table_versions +from app.data.engine import engines, run_query +from app.db.base import SessionLocal +from app.db.models import DataSource, ImportStaging, SourceSnapshot, TableImport, TableRecipe +from app.main import app +from tests.fixtures.xlsx.flow import FLOW_EXPECT, flow_workbook +from tests.test_recipe_imports import ( # noqa: F401 - fake 是夹具，导入即可用 + FLOW_FULL, api_stage, fake, raw_bytes, recipe_source, staged, +) + +XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" +FLOW_RECIPE = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text("utf-8")) +AUG, SEP = dt.date(2026, 8, 1), dt.date(2026, 9, 1) + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + """每个测试一个独立的数据目录：上传目录、原件存档、工件都在里面。""" + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +@pytest.fixture +async def client(): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +def unique(prefix: str = "wp5c") -> str: + return f"{prefix}_{uuid.uuid4().hex[:10]}" + + +async def stage(client, name: str, raw: bytes, filename: str, **form: str): + return await client.post("/api/datasources/imports/stage", files={"file": (filename, raw, XLSX)}, + data={"name": name, **form}) + + +async def scalar(source_id: str, sql: str): + async with SessionLocal() as session: + row = await session.get(DataSource, source_id) + view = await table_versions.resolve_source(session, row) + try: + return (await run_query(view, sql)).rows[0][0] + finally: + await engines.invalidate(view.id) + + +async def get(model, key): + async with SessionLocal() as session: + return await session.get(model, key) + + +def rename_wide(rec: dict[str, Any], old: str, new: str) -> dict[str, Any]: + """把规则起草的宽表名改成 new：表名、写入它的分段（id 一起改）、关系里引用的表名。""" + rec = json.loads(json.dumps(rec, ensure_ascii=False)) + for sheet in rec["sheets"]: + for block in sheet["blocks"]: + for seg in block.get("segments", []): + if seg.get("table") == old: + seg["table"] = new + if seg.get("id") == old: + seg["id"] = new + for t in rec["tables"]: + if t["name"] == old: + t["name"] = new + for r in rec["relations"]: + if r.get("table") == old: + r["table"] = new + for side in ("a", "b"): + if isinstance(r.get(side), dict) and r[side].get("table") == old: + r[side]["table"] = new + return rec + + +def all_confirms(staging: dict[str, Any]) -> list[str]: + return [i["id"] for i in staging["trial"]["confirm_items"]] + + +# -------------------------------------------------------------------------- +# 真管线：首次导入 +# -------------------------------------------------------------------------- + + +async def first_import(client, name: str, seed: int = 1) -> dict[str, Any]: + """D00 走完首次导入，返回提交的响应。""" + raw, fn = flow_workbook(AUG, 31, seed=seed) + resp = await stage(client, name, raw, fn) + assert resp.status_code == 201, resp.text + st = resp.json() + sid = st["id"] + answers = {q["id"]: {"value": "register" if q["id"].startswith("q_relation") else "null"} + for q in st["questions"]} + resp = await client.post(f"/api/datasources/imports/{sid}/answers", json={"answers": answers}) + assert resp.status_code == 200, resp.text + wide = next(t["name"] for t in st["recipe"]["tables"] if t["name"].endswith("_按日")) + resp = await client.put(f"/api/datasources/imports/{sid}/recipe", + json={"recipe": rename_wide(resp.json()["recipe"], wide, "日客流")}) + assert resp.status_code == 200, resp.text + resp = await client.post(f"/api/datasources/imports/{sid}/trial", json={}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + resp = await client.post(f"/api/datasources/imports/{sid}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": all_confirms(st)}) + assert resp.status_code == 201, resp.text + return {"staging": st, "commit": resp.json()} + + +async def test_real_pipeline_first_import_stage_draft_answer_trial_commit(client): + name = unique() + raw, fn = flow_workbook(AUG, 31, seed=3) + resp = await stage(client, name, raw, fn) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["kind"] == "first" and st["status"] == "drafting" + assert st["source"]["exists"] is False and st["source"]["import_mode"] is None + assert st["draft"]["complete"] is True and "failures_for_model" not in st["draft"] + assert st["ai"]["offered"] is False + assert len(st["cards"]) == 10 + assert st["recipe_problems"] == [] and st["recipe_origin"] == "rules" + assert st["grids"][0]["sheet"] == "客流汇总" and st["grids"][0]["bounds"] == "B2:AG30" + assert st["sheets"][0]["nonempty"] == FLOW_EXPECT["nonempty"] + assert st["marks"], "干跑的区域标记" + qids = [q["id"] for q in st["questions"]] + assert {"q_relation:F1", "q_relation:F2"} <= set(qids) + sid = st["id"] + # 新源在提交前不存在 + async with SessionLocal() as session: + assert (await session.execute(select(DataSource).where(DataSource.name == name))).first() is None + + # 回答问题：先选「不登记」再改选「登记」，等于直接选「登记」 + first = await client.post(f"/api/datasources/imports/{sid}/answers", json={"answers": { + "q_relation:F1": {"value": "dismiss", "reason": "合成理由：判断为巧合"}}}) + assert first.status_code == 200, first.text + assert next(r for r in first.json()["recipe"]["relations"] if r["id"] == "R1")["kind"] == "dismissed" + answers = {q: {"value": "register" if q.startswith("q_relation") else "null"} for q in qids} + resp = await client.post(f"/api/datasources/imports/{sid}/answers", json={"answers": answers}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["answers"]["q_relation:F1"] == {"value": "register", "reason": None} + assert [q["id"] for q in st["questions"]] == qids, "问题集合按起点配方生成，回答不改变它" + + # 改表名（PUT）：起点换成它，回答清空；与参考配方逐字相同 + resp = await client.put(f"/api/datasources/imports/{sid}/recipe", + json={"recipe": rename_wide(st["recipe"], "客流汇总_按日", "日客流")}) + assert resp.status_code == 200, resp.text + st = resp.json() + assert st["answers"] == {} and st["recipe_origin"] == "manual" and st["recipe_problems"] == [] + ref, _ = recipe.validate_recipe(FLOW_RECIPE, origin="manual") + mine, _ = recipe.validate_recipe(st["recipe"], origin="manual") + assert recipe.recipe_sha256(mine) == recipe.recipe_sha256(ref) + + resp = await client.post(f"/api/datasources/imports/{sid}/trial", json={}) + assert resp.status_code == 200, resp.text + st = resp.json() + trial = st["trial"] + assert st["status"] == "trialed" and trial["status"] == "passed", trial["problems"] + assert set(all_confirms(st)) == set(FLOW_EXPECT["confirms"]) + assert {c["id"]: c["status"] for c in trial["checks"]} == FLOW_EXPECT["checks"] + assert trial["diff"] is None and trial["same_as_import"] is None + assert "extraction" not in trial and "null_counts" not in trial + assert trial["receipt"]["canonicalized_total"] == 0 and trial["receipt"]["db_sha256"] + assert {t["name"]: t["rows"] for t in trial["receipt"]["tables"]} == FLOW_EXPECT["rows"] + assert "已逐日核对" in trial["notes"]["日客流"]["comment"] + assert trial["notes"]["日客流"]["columns"]["全日客流"] == "单位：人次" + + # 少勾一项 → 422，detail 列出缺的那一项 + some = all_confirms(st) + resp = await client.post(f"/api/datasources/imports/{sid}/commit", + json={"trial_id": trial["trial_id"], "confirmations": some[1:]}) + assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" + assert some[0] in resp.json()["detail"] + + resp = await client.post(f"/api/datasources/imports/{sid}/commit", + json={"trial_id": trial["trial_id"], "confirmations": some, "signed_by": "测试员"}) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["unchanged"] is False and out["build_reused"] is False + src = out["source"] + assert src["name"] == name and src["origin"] == "upload" and src["import_mode"] == "recipe" + assert src["current_recipe"]["id"] == out["recipe_id"] and src["current_recipe"]["seq"] == 1 + assert src["current_recipe"]["origin"] == "manual" + assert src["open_staging"] is None + assert await scalar(src["id"], 'SELECT COUNT(*) FROM "时段客流"') == FLOW_EXPECT["rows"]["时段客流"] + + # 表结构里的说明；schema_cache 承诺了导入清单 + resp = await client.get(f"/api/datasources/{src['id']}/schema", params={"table": "日客流"}) + assert resp.status_code == 200, resp.text + schema = resp.json() + assert "已逐日核对" in json.dumps(schema, ensure_ascii=False) + snap = await get(SourceSnapshot, out["snapshot_id"]) + imp = await get(TableImport, out["import_id"]) + assert snap.schema_cache["import_mode"] == "recipe" + assert snap.schema_cache["import_manifests"] == [imp.manifest_artifact] + manifest = artifact_store.load(imp.manifest_artifact) + assert manifest["format"] == "agentlab-import-manifest/1" + assert manifest["db_sha256"] == snap.db_sha256 == manifest["trial_db_sha256"] + for key in ("axes", "tables", "ledger", "period", "labels", "outside_text", "lineage", "canonicalized"): + assert key in manifest["receipt"], key + assert "problems" not in manifest["receipt"] + assert manifest["recipe"]["id"] == out["recipe_id"] and manifest["signed_by"] == {"name": "测试员", "verified": False} + assert {c["id"] for c in manifest["checks"]} >= {"C1", "C2"} + assert {c["id"] for c in imp.checks} >= {"C1", "C2"} + assert imp.period_start == "2026-08-01" and imp.period_end == "2026-08-31" and imp.signed_by == "测试员" + assert imp.recipe_id == out["recipe_id"] and imp.staging_id == sid + assert {c["id"] for c in imp.confirmations} == set(some) + rec = await get(TableRecipe, out["recipe_id"]) + assert rec.status == "active" and rec.seq == 1 and rec.staging_id == sid + # 暂存区结束即瘦身 + resp = await client.get(f"/api/datasources/imports/{sid}") + st = resp.json() + assert st["status"] == "committed" and st["grids"] == [] and st["trial"] is None + assert st["committed_import_id"] == out["import_id"] + row = await get(ImportStaging, sid) + assert row.scan is None and row.grid_preview is None and row.trial_path is None and row.recipe_sha256 + # 已结束的暂存区不能再写 + resp = await client.post(f"/api/datasources/imports/{sid}/trial", json={}) + assert resp.status_code == 409 and resp.json()["code"] == "staging_closed" + # 按配方导入的源：简单上传 409、当前配方、导入记录的新字段 + resp = await client.post("/api/datasources/upload", files={"file": (fn, raw, XLSX)}, data={"name": name}) + assert resp.status_code == 409 and resp.json()["code"] == "recipe_source" + resp = await client.get(f"/api/datasources/{src['id']}/recipe") + assert resp.status_code == 200, resp.text + cur = resp.json() + assert cur["recipe_id"] == out["recipe_id"] and cur["recipe"]["mode"] == "replace" + resp = await client.get(f"/api/datasources/{src['id']}/imports") + item = resp.json()[0] + assert item["recipe_id"] == out["recipe_id"] and item["period_start"] == "2026-08-01" + assert item["overrides"] == 0 and item["waivers"] == 0 and item["signed_by"] == "测试员" + + +# -------------------------------------------------------------------------- +# 真管线：上传新一期 +# -------------------------------------------------------------------------- + + +async def test_real_pipeline_reupload_next_month_receipt_diff_and_activate(client): + name = unique() + done = await first_import(client, name) + source_id = done["commit"]["source"]["id"] + first_snapshot = done["commit"]["snapshot_id"] + raw, fn = flow_workbook(SEP, 30, seed=7) + resp = await client.post(f"/api/datasources/{source_id}/reupload", files={"file": (fn, raw, XLSX)}) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["kind"] == "reupload" and st["status"] == "trialed" and st["draft"] is None + assert st["ai"]["offered"] is False and st["source"]["import_mode"] == "recipe" + trial = st["trial"] + assert trial["status"] == "passed", trial["problems"] + assert {d["kind"] for d in trial["diff"]} == {"period", "rows"} + assert not any(d["requires_confirm"] for d in trial["diff"]) + # 配方没变：配方类确认项都不出，只剩本期类（这里没有） + assert all_confirms(st) == [] + assert trial["base_snapshot_id"] == first_snapshot + assert {t["name"]: t["rows"] for t in trial["receipt"]["tables"]} == {"日客流": 30, "时段客流": 510, + "时段客流_表内合计": 90} + # 数据源列表里看得到这个未结束的导入 + resp = await client.get("/api/datasources") + row = next(s for s in resp.json() if s["id"] == source_id) + assert row["open_staging"]["id"] == st["id"] and row["open_staging"]["kind"] == "reupload" + + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": trial["trial_id"], "confirmations": []}) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["unchanged"] is False and out["recipe_id"] == done["commit"]["recipe_id"] + assert out["source"]["current_snapshot"]["id"] == out["snapshot_id"] != first_snapshot + assert await scalar(source_id, 'SELECT COUNT(*) FROM "日客流"') == 30 + async with SessionLocal() as session: + recipes = list((await session.execute( + select(TableRecipe).where(TableRecipe.source_id == source_id))).scalars()) + imps = list((await session.execute( + select(TableImport).where(TableImport.source_id == source_id).order_by(TableImport.seq))).scalars()) + assert len(recipes) == 1, "配方没变：不新建配方" + # 被替换的那一期：发布后的回收会把没有运行引用的版本标成 retired（原件按配方源最近 12 次保留） + assert imps[0].status in ("superseded", "retired") and imps[1].status == "active" + assert imps[0].raw_state == "kept" + assert imps[1].recipe_id == recipes[0].id and imps[1].period_start == "2026-09-01" + + +async def test_real_pipeline_same_file_again_is_unchanged(client): + name = unique() + raw, fn = flow_workbook(AUG, 31, seed=11) + done = await first_import(client, name, seed=11) + source_id = done["commit"]["source"]["id"] + resp = await client.post(f"/api/datasources/{source_id}/reupload", files={"file": (fn, raw, XLSX)}) + assert resp.status_code == 201, resp.text + st = resp.json() + trial = st["trial"] + assert trial["same_as_import"]["id"] == done["commit"]["import_id"] + assert trial["diff"] == [] + resp = await client.post(f"/api/datasources/imports/{st['id']}/commit", + json={"trial_id": trial["trial_id"], "confirmations": all_confirms(st)}) + assert resp.status_code == 201, resp.text + out = resp.json() + assert out["unchanged"] is True and out["import_id"] == done["commit"]["import_id"] + assert out["snapshot_id"] == done["commit"]["snapshot_id"] + async with SessionLocal() as session: + n = len(list((await session.execute( + select(TableImport).where(TableImport.source_id == source_id))).scalars())) + assert n == 1 + row = await get(ImportStaging, st["id"]) + assert row.status == "committed" and row.trial_path is None + + +async def test_real_pipeline_changed_context_input_invalidates_earlier_trial_id(client): + """trial_key 含人工录入（WP-5c：改配方、改录入后失效）：同一配方、同一统计期换了署名再试运行，trial_id + 必须换；拿前一次（另一份录入）的 trial_id 提交返回 409 trial_required，不能发布成最新那次的结果。""" + raw, fn = flow_workbook(AUG, 31, seed=1201, variant="noperiod") + resp = await stage(client, unique(), raw, fn) + assert resp.status_code == 201, resp.text + sid = resp.json()["id"] + resp = await client.put(f"/api/datasources/imports/{sid}/recipe", json={"recipe": FLOW_RECIPE}) + assert resp.status_code == 200, resp.text + resp = await client.post(f"/api/datasources/imports/{sid}/trial", json={}) + assert resp.status_code == 200 and resp.json()["trial"]["status"] == "needs_input", resp.text + period = {"统计期": {"start": "2026-08-01", "end": "2026-08-31"}} + resp = await client.post(f"/api/datasources/imports/{sid}/trial", + json={"context_inputs": period, "signed_by": "录入员甲"}) + first = resp.json() + assert first["trial"]["status"] == "passed", first["trial"]["problems"] + resp = await client.post(f"/api/datasources/imports/{sid}/trial", + json={"context_inputs": period, "signed_by": "录入员乙"}) + second = resp.json()["trial"] + assert second["status"] == "passed" and second["trial_id"] != first["trial"]["trial_id"] + resp = await client.post(f"/api/datasources/imports/{sid}/commit", json={ + "trial_id": first["trial"]["trial_id"], "confirmations": all_confirms(first)}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_required", resp.text + assert (await get(ImportStaging, sid)).status != "committed" + + +# -------------------------------------------------------------------------- +# 真管线：AI 起草（假模型） +# -------------------------------------------------------------------------- + +KV_RECIPE = { + "recipe_format": "agentlab-recipe/2", + "sheets": [{"id": "s1", "match": {"name": "基本情况"}, "blocks": [{ + "id": "列表1", "layout": "list", "table": "基本情况", "after_title": "基本情况登记表", + "columns": [{"header": "单位名称", "name": "项目", "type": "TEXT"}, + {"header": "示例单位甲", "name": "取值", "type": "TEXT"}]}]}], + "tables": [{"name": "基本情况"}], +} + + +async def test_real_pipeline_ai_draft_sends_exactly_the_previewed_text(client, monkeypatch): + """规则起草认不出的表单（lab.kv_form）→ 查看要发送的全文 → 同意 → 假模型起草 → 试运行 → 提交。 + + 模型第一次收到的系统提示 + 提示词，与同意框里展示的全文逐字相同；不调用任何真实模型。 + """ + from langchain_core.messages import AIMessage + + from app.data import recipe_ai + from app.data.recipe_types import AiAvailability + from tests.fixtures.xlsx.lab import kv_form + + calls: list[list[Any]] = [] + + class Model: + async def ainvoke(self, messages): + calls.append(list(messages)) + return AIMessage(content=json.dumps(KV_RECIPE, ensure_ascii=False), + usage_metadata={"input_tokens": 3000, "output_tokens": 400, "total_tokens": 3400}) + + async def resolve(session): + return Model(), "fake-model", AiAvailability(True, model="fake-model", provider="假接入") + + monkeypatch.setattr(recipe_ai, "resolve_draft_model", resolve) + raw, fn = kv_form() + resp = await stage(client, unique(), raw, fn) + assert resp.status_code == 201, resp.text + st = resp.json() + assert st["draft"]["complete"] is False and st["ai"]["offered"] is True and st["ai"]["available"] is True + sid = st["id"] + resp = await client.get(f"/api/datasources/imports/{sid}/draft-ai/preview") + assert resp.status_code == 200, resp.text + pv = resp.json() + assert pv["model"] == "fake-model" and "# 配方 JSON Schema" in pv["text"] and "# 单位词表" in pv["text"] + assert "示例单位甲" not in pv["text"].split("# 工作表压缩表示")[0], "系统提示里没有表格内容" + assert "35600" not in pv["text"] and "860" not in pv["text"], "数字格的值不外发" + resp = await client.post(f"/api/datasources/imports/{sid}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"], "signed_by": "测试员"}) + assert resp.status_code == 200, resp.text + st = resp.json() + first = calls[0] + assert first[0].content + "\n\n" + first[1].content == pv["text"] + assert st["recipe_origin"] == "ai" and st["recipe_problems"] == [] and st["cards"][0]["id"] == "ai_origin" + assert st["ai_consents"][0]["preview_sha256"] == pv["sha256"] and st["ai_draft"]["total_tokens"] == 3400 + + st = (await client.post(f"/api/datasources/imports/{sid}/trial", json={})).json() + assert st["trial"]["status"] == "passed", st["trial"]["problems"] + ids = all_confirms(st) + assert {"ai_origin", "rename:基本情况.项目", "columns:基本情况"} <= set(ids) + resp = await client.post(f"/api/datasources/imports/{sid}/commit", + json={"trial_id": st["trial"]["trial_id"], + "confirmations": [i for i in ids if i != "ai_origin"]}) + assert resp.status_code == 422 and "ai_origin" in resp.json()["detail"] + resp = await client.post(f"/api/datasources/imports/{sid}/commit", + json={"trial_id": st["trial"]["trial_id"], "confirmations": ids}) + assert resp.status_code == 201, resp.text + out = resp.json() + rec = await get(TableRecipe, out["recipe_id"]) + assert rec.origin == "ai" and rec.ai_usage["calls"][0]["input_tokens"] == 3000 + imp = await get(TableImport, out["import_id"]) + manifest = artifact_store.load(imp.manifest_artifact) + assert manifest["ai"]["consents"][0]["preview_sha256"] == pv["sha256"] + assert await scalar(out["source"]["id"], 'SELECT COUNT(*) FROM "基本情况"') > 0 + + +async def test_real_pipeline_model_switch_after_preview_is_stale_and_model_resolved_once(client, monkeypatch): + """AU-7 / SE-4：看完预览后助手的模型从本机换成外部服务 → 409 ai_preview_stale，不发送；重新查看再同意时， + 同意记录、实际调用、用量记的是同一个模型，POST 里只解析一次模型。""" + from langchain_core.messages import AIMessage + + from app.data import recipe_ai + from app.data.recipe_types import AiAvailability + from tests.fixtures.xlsx.lab import kv_form + + sent: list[str] = [] + current = {"name": "local-model-A", "provider": "本机接入"} + resolves: list[str] = [] + + class Model: + def __init__(self, name: str) -> None: + self.name = name + + async def ainvoke(self, messages): + sent.append(self.name) + resolves.append("send") + return AIMessage(content=json.dumps(KV_RECIPE, ensure_ascii=False), + usage_metadata={"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}) + + async def resolve(session): + resolves.append(current["name"]) + return (Model(current["name"]), current["name"], + AiAvailability(True, model=current["name"], provider=current["provider"])) + + monkeypatch.setattr(recipe_ai, "resolve_draft_model", resolve) + raw, fn = kv_form() + sid = (await stage(client, unique(), raw, fn)).json()["id"] + pv = (await client.get(f"/api/datasources/imports/{sid}/draft-ai/preview")).json() + assert pv["model"] == "local-model-A" + current.update(name="cloud-model-B", provider="外部接入") + resp = await client.post(f"/api/datasources/imports/{sid}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"]}) + assert resp.status_code == 409 and resp.json()["code"] == "ai_preview_stale", resp.text + assert sent == [] + pv = (await client.get(f"/api/datasources/imports/{sid}/draft-ai/preview")).json() + assert pv["model"] == "cloud-model-B" + resolves.clear() + resp = await client.post(f"/api/datasources/imports/{sid}/draft-ai", + json={"consent": True, "preview_sha256": pv["sha256"]}) + assert resp.status_code == 200, resp.text + st = resp.json() + # 发送之前只解析了一次（同意的门槛里那次）；之后的是响应里查可用性，与发送无关 + assert resolves[:resolves.index("send")] == ["cloud-model-B"], resolves + assert sent == ["cloud-model-B"] + assert st["ai_consents"][-1]["model"] == "cloud-model-B" and st["ai_consents"][-1]["provider"] == "外部接入" + assert [u["model"] for u in st["ai_draft"]["usage"]] == ["cloud-model-B"] + + +# -------------------------------------------------------------------------- +# 假管线：接口形状与错误码 +# -------------------------------------------------------------------------- + + +async def test_stage_errors(client, fake, monkeypatch): + resp = await api_stage(client, "Bad-Name") + assert resp.status_code == 400 and resp.json()["code"] == "name_invalid" + resp = await client.post("/api/datasources/imports/stage", data={"name": unique()}) + assert resp.status_code == 422 and resp.json()["code"] == "body_invalid" + resp = await api_stage(client, unique(), raw=b"bad file") + assert resp.status_code == 400 and resp.json()["code"] == "file_unreadable" + assert "无法读取" in resp.json()["detail"] + # 手工源同名 → name_taken；已按配方导入 → recipe_source_exists + manual = unique() + async with SessionLocal() as session: + session.add(DataSource(name=manual, kind="sqlite", database="/tmp/合成.db")) + await session.commit() + resp = await api_stage(client, manual) + assert resp.status_code == 409 and resp.json()["code"] == "name_taken" + done = await recipe_source(client) + resp = await api_stage(client, done["source"]["name"]) + assert resp.status_code == 409 and resp.json()["code"] == "recipe_source_exists" + # 超限 → 413（两条上传路径共用 read_upload） + monkeypatch.setattr(settings, "max_upload_mb", 0) + resp = await api_stage(client, unique()) + assert resp.status_code == 413 and resp.json()["code"] == "file_too_large" + resp = await client.post("/api/datasources/upload", files={"file": ("x.xlsx", b"PK", XLSX)}, + data={"name": unique()}) + assert resp.status_code == 413 and resp.json()["code"] == "file_too_large" + + +async def test_not_found_and_body_invalid(client, fake): + for method, path in (("get", "/api/datasources/imports/nope"), ("delete", "/api/datasources/imports/nope")): + resp = await getattr(client, method)(path) + assert resp.status_code == 404 and resp.json()["code"] == "staging_not_found", path + resp = await client.post("/api/datasources/imports/nope/trial", json={}) + assert resp.status_code == 404 and resp.json()["code"] == "staging_not_found" + st = await staged(client) + sid = st["id"] + for method, path, kw in ( + ("post", "answers", {"content": b"[1, 2]"}), ("put", "recipe", {"content": b"not json"}), + ("put", "recipe", {"json": {"recipe": [1]}}), ("post", "answers", {"json": {"answers": ["x"]}}), + ("post", "commit", {"json": {"trial_id": "x", "confirmations": "mode"}}), + ("post", "commit", {"json": {"trial_id": "x", "acceptances": {"R1": "理由"}}}), + ): + headers = {"content-type": "application/json"} if "content" in kw else None + resp = await getattr(client, method)(f"/api/datasources/imports/{sid}/{path}", headers=headers, **kw) + assert resp.status_code == 422 and resp.json()["code"] == "body_invalid", (path, kw) + resp = await client.post(f"/api/datasources/imports/{sid}/commit", json={}) + assert resp.status_code == 409 and resp.json()["code"] == "trial_required" + + +async def test_put_recipe_with_schema_problems_is_stored_but_cannot_trial(client, fake): + st = await staged(client) + sid = st["id"] + broken = {"recipe_format": "agentlab-recipe/2", "tables": []} + resp = await client.put(f"/api/datasources/imports/{sid}/recipe", json={"recipe": broken}) + assert resp.status_code == 200, resp.text + out = resp.json() + assert out["recipe"] == broken and out["recipe_problems"][0]["code"] == "schema" + assert out["recipe_problems"][0]["path"] + assert fake.calls["detect_facts"][-1] == broken + resp = await client.post(f"/api/datasources/imports/{sid}/trial", json={}) + assert resp.status_code == 409 and resp.json()["code"] == "recipe_invalid" + # 改回合法的配方：问题集合按新的起点重算 + fake.calls["questions_for"].clear() + resp = await client.put(f"/api/datasources/imports/{sid}/recipe", json={"recipe": FLOW_FULL}) + assert resp.json()["recipe_problems"] == [] and fake.calls["questions_for"][-1]["recipe"] == FLOW_FULL + + +async def test_signed_by_from_header_and_body(client, fake): + st = await staged(client) + from urllib.parse import quote + + resp = await client.post(f"/api/datasources/imports/{st['id']}/trial", json={}, + headers={"X-Actor": quote("页眉署名")}) + assert resp.status_code == 200 + assert (await get(ImportStaging, st["id"])).signed_by == "页眉署名" + resp = await client.post(f"/api/datasources/imports/{st['id']}/trial", json={"signed_by": "请求体署名"}, + headers={"X-Actor": quote("页眉署名")}) + assert (await get(ImportStaging, st["id"])).signed_by == "请求体署名" + + +async def test_datasource_out_new_fields(client, fake): + manual = unique() + async with SessionLocal() as session: + session.add(DataSource(name=manual, kind="sqlite", database="/tmp/合成.db")) + await session.commit() + done = await recipe_source(client) + rows = {s["name"]: s for s in (await client.get("/api/datasources")).json()} + assert rows[manual]["import_mode"] is None and rows[manual]["current_recipe"] is None + assert rows[manual]["open_staging"] is None + mine = rows[done["source"]["name"]] + assert mine["import_mode"] == "recipe" + assert set(mine["current_recipe"]) == {"id", "seq", "origin", "activated_at", "signed_by"} + re_ = (await client.post(f"/api/datasources/{mine['id']}/reupload", + files={"file": ("x.xlsx", raw_bytes(), XLSX)})).json() + rows = {s["name"]: s for s in (await client.get("/api/datasources")).json()} + assert rows[done["source"]["name"]]["open_staging"]["id"] == re_["id"] + assert set(rows[done["source"]["name"]]["open_staging"]) == {"id", "kind", "status", "created_at"} + + +async def test_routes_are_not_taken_by_datasource_wildcards(client, fake): + """/imports/… 和 /{source_id}/reupload|redraft|recipe 不被 datasources 的通配路由截走：每个打一次，看 code。""" + resp = await client.get("/api/datasources/imports/imports") + assert resp.json().get("code") == "staging_not_found" + resp = await client.post("/api/datasources/nope/reupload", files={"file": ("x.xlsx", b"PK", XLSX)}) + assert resp.status_code == 404 and resp.json().get("code") == "source_not_found" + resp = await client.get("/api/datasources/nope/recipe") + assert resp.status_code == 404 and resp.json().get("code") == "source_not_found" + resp = await client.delete("/api/datasources/imports/nope") + assert resp.json().get("code") == "staging_not_found" + resp = await client.get("/api/datasources/imports/nope/draft-ai/preview") + assert resp.json().get("code") == "staging_not_found" + + +async def test_switch_from_simple_reports_mask_lost_and_compress_masks_case_insensitively(client, monkeypatch): + """AU-6 / SE-9：简单导入遮罩了「工资_元」，切换后规则起草的列名是「工资」：确认项里必勾 mask_lost:工资_元。 + 遮罩写成大小写不同的「PHONE」时，AI 压缩表示同样按遮罩列处理（与证据面板一致，不区分大小写）。""" + import io + + from openpyxl import Workbook + + from app.data import recipe_imports + + wb = Workbook() + ws = wb.active + ws.title = "员工" + ws.append(["工号", "姓名", "Phone", "工资（元）"]) + for i in range(12): + ws.append([f"E{i:03d}", f"员工{i}", f"1380000{i:04d}", 5000 + i * 137]) + buf = io.BytesIO() + wb.save(buf) + raw = buf.getvalue() + name = unique() + resp = await client.post("/api/datasources/upload", files={"file": ("员工.xlsx", raw, XLSX)}, data={"name": name}) + assert resp.status_code == 201, resp.text + src = resp.json()["source"]["id"] + resp = await client.patch(f"/api/datasources/{src}", json={"options": {"mask_columns": ["工资_元", "PHONE"]}}) + assert resp.status_code == 200, resp.text + st = (await stage(client, name, raw, "员工.xlsx")).json() + assert st["kind"] == "switch" + st = (await client.post(f"/api/datasources/imports/{st['id']}/trial", json={})).json() + items = {i["id"]: i for i in st["trial"]["confirm_items"]} + assert "mask_lost:工资_元" in items, list(items) + assert "mask_lost:PHONE" not in items # 新结构里还有 Phone（不区分大小写） + # SE-9：压缩表示的遮罩表头不区分大小写 + from app.db.base import SessionLocal + from app.db.models import ImportStaging + + async with SessionLocal() as session: + staging = await session.get(ImportStaging, st["id"]) + heads = await recipe_imports._mask_headers(session, staging) + assert "Phone" in heads and "工资（元）" in heads + + +async def test_bad_number_cell_is_file_unreadable_with_fixed_text(client): + """SE-7：数字格的 里写了非数字：stage 回 400 file_unreadable（带 code、固定中文文案），不回英文原文和格子内容。""" + from openpyxl import Workbook + + from tests.fixtures.xlsx import patch_parts, save + + wb = Workbook() + ws = wb.active + ws.title = "S" + ws["A1"], ws["B1"] = "项目", "数量" + for r in range(2, 8): + ws.cell(r, 1, f"甲{r}") + ws.cell(r, 2, r) + raw = patch_parts(save(wb), {"xl/worksheets/sheet1.xml": lambda x: x.replace("3", "机密值abc", 1)}) + resp = await stage(client, unique(), raw, "bad.xlsx") + assert resp.status_code == 400 and resp.json()["code"] == "file_unreadable", resp.text + assert "机密值abc" not in resp.text and "invalid literal" not in resp.text + + +async def test_json_body_size_and_depth_limits(client, fake): + """SE-5：超过 1 MB 的请求体回 413 body_too_large；嵌套超过 32 层回 422 body_invalid，不入库，之后 GET 照常。""" + from app.api import table_imports as api_mod + + sid = (await stage(client, unique(), raw_bytes(), "a.xlsx")).json()["id"] + url = f"/api/datasources/imports/{sid}/recipe" + big = {"recipe": {"recipe_format": "agentlab-recipe/2", "junk": "x" * (api_mod.BODY_MAX_BYTES + 10)}} + resp = await client.put(url, json=big) + assert resp.status_code == 413 and resp.json()["code"] == "body_too_large" + deep: Any = "x" + for _ in range(300): + deep = [deep] + resp = await client.put(url, json={"recipe": {"recipe_format": "agentlab-recipe/2", "junk": deep}}) + assert resp.status_code == 422 and resp.json()["code"] == "body_invalid" + resp = await client.get(f"/api/datasources/imports/{sid}") + assert resp.status_code == 200, resp.text + # 字符串里的括号不算嵌套 + ok = {"recipe": {"recipe_format": "agentlab-recipe/2", "note": "[" * 100}} + resp = await client.put(url, json=ok) + assert resp.status_code == 200, resp.text + + +async def test_recipe_problems_are_capped(client, monkeypatch): + """SE-5：几千个问题只存、只回前 200 条，另附一条总数。""" + from app.data import recipe_imports + from tests.fixtures.xlsx.lab import kv_form + + raw, fn = kv_form() + sid = (await stage(client, unique(), raw, fn)).json()["id"] + rec = json.loads(json.dumps(KV_RECIPE, ensure_ascii=False)) + rec["tables"][0]["units"] = {f"不存在的列{i}": "元" for i in range(1000)} + resp = await client.put(f"/api/datasources/imports/{sid}/recipe", json={"recipe": rec}) + assert resp.status_code == 200, resp.text + probs = resp.json()["recipe_problems"] + assert len(probs) == recipe_imports.RECIPE_PROBLEMS_MAX + 1 + assert probs[-1]["code"] == "too_many_problems" and "共 1000 个" in probs[-1]["message"] diff --git a/backend/tests/test_table_versions_p2.py b/backend/tests/test_table_versions_p2.py new file mode 100644 index 00000000..8602ef7d --- /dev/null +++ b/backend/tests/test_table_versions_p2.py @@ -0,0 +1,930 @@ +"""按配方导入（期 2）的版本底座：数据模型、publish_build、暂存区的回收与清理保护。 + +要守住的几件事： +- publish_build 发布的必须是用户试运行、核对过的那份库：哈希对不上不 link；试运行库不在时不能 + 新建一个空库发布出去；任何一步（包括锁内的提交前回调）失败，指针不动、不留新文件； +- 发布交给 schema_cache_for 的是实际发布的库哈希（复用构建时是已有文件的），导入清单据此记录； +- 未结束暂存区的原件、配方源最近 12 次导入的原件不被回收；暂存区一结束就瘦身； +- 清除原件连带放弃引用它的暂存区；放弃导入时原件没有别的引用就当场删掉。 + +夹具一律现造：试运行库用 sqlite3 直接建，原件是随机字节，标签用假名。 +""" +from __future__ import annotations + +import asyncio +import hashlib +import io +import sqlite3 +import stat +import uuid +from datetime import timedelta +from pathlib import Path +from typing import Any + +import pytest +from sqlalchemy import create_engine, select, text + +from app.core.config import settings +from app.data import raw_store, recipe_parsers, recipe_types, table_versions +from app.data.engine import engines, run_query +from app.db import base as db_base +from app.db.base import SessionLocal, utcnow +from app.db.models import ( + DataSource, ImportStaging, SourceSnapshot, TableBuild, TableImport, TableRecipe, +) + + +@pytest.fixture(autouse=True) +def store(monkeypatch, tmp_path): + """每个测试一个独立的数据目录：上传目录、原件存档、工件都在里面。""" + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + yield data + + +# -------------------------------------------------------------------------- +# 工具 +# -------------------------------------------------------------------------- + +RECIPE_SHA = "c" * 64 + + +def unique(prefix: str = "wp5a") -> str: + return f"{prefix}_{uuid.uuid4().hex[:10]}" + + +def raw_bytes() -> bytes: + """内容独一份的「原件」：原件按内容共用，别的测试传过同样字节就会互相影响。""" + return f"合成原件 {uuid.uuid4().hex}".encode() + + +def make_db(path: Path, rows: list[tuple[int, str]] | None = None, *, wal: bool = False) -> Path: + """一张 STRICT 表 t 的小库。wal=True 时切成 WAL 模式再写（文件头第 18、19 字节是 2）。""" + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(path) + try: + if wal: + conn.execute("PRAGMA journal_mode=WAL") + conn.execute('CREATE TABLE "t" ("n" INTEGER, "s" TEXT) STRICT') + conn.executemany('INSERT INTO "t" VALUES (?, ?)', rows or [(1, "分区甲"), (2, "分区乙")]) + conn.commit() + finally: + conn.close() + return path + + +def new_trial(source_id: str, staging_id: str | None = None, **kw: Any) -> Path: + return make_db(table_versions.trial_db_path( + source_id, staging_id or uuid.uuid4().hex, uuid.uuid4().hex), **kw) + + +def settled_sha(path: Path) -> str: + table_versions.settle_journal(path) + return raw_store.sha256_file(path) + + +def files_under(root: Path) -> list[Path]: + return sorted(p for p in root.rglob("*") if p.is_file()) if root.exists() else [] + + +async def get(model, key): + async with SessionLocal() as session: + return await session.get(model, key) + + +async def imports_of(source_id: str) -> list[TableImport]: + async with SessionLocal() as session: + return list((await session.execute( + select(TableImport).where(TableImport.source_id == source_id).order_by(TableImport.seq) + )).scalars()) + + +async def publish_trial( + name: str, tmp: Path, raw: bytes, *, source_id: str | None = None, pass_raw: bool = True, + expected: str | None | bool = True, build_id: str | None = None, + import_fields: dict[str, Any] | None = None, import_id: str | None = None, + before_commit=None, seen: list | None = None, schema_extra=None, +) -> tuple[str, table_versions.ImportResult]: + """按配方导入的方式发布一个试运行库。expected=True 时先 settle 再算哈希作期望值（同 run_trial）。 + + before_commit 是 (session, source_id) -> 协程函数 的工厂：回调要用同一个会话。 + """ + raw_sha = hashlib.sha256(raw).hexdigest() + async with SessionLocal() as session: + row = (await session.execute(select(DataSource).where(DataSource.name == name))).scalar_one_or_none() + if row is None: + row = DataSource(id=source_id or uuid.uuid4().hex, name=name, kind="sqlite", + readonly=True, origin="upload") + sid = row.id + bid = build_id or table_versions.recipe_build_id(sid, raw_sha, RECIPE_SHA, {}) + if expected is True: + expected = settled_sha(tmp) + + async def schema(view, info): + if seen is not None: + seen.append((view, info)) + cache = await table_versions.introspect_view(view, {}) + table_versions.check_probe(cache, {"tables": [{"name": "t"}]}, sid) + if schema_extra is not None: + await schema_extra(cache, info) + return cache + + result = await table_versions.publish_build( + session, row, tmp_db=tmp, build_id=bid, + build_options=table_versions.recipe_build_options(RECIPE_SHA, {}), + engine_ver=recipe_types.RECIPE_ENGINE_VER, report={"tables": [{"name": "t"}]}, + raw=raw if pass_raw else None, raw_sha=raw_sha, file_name="月报导出.xlsx", + file_size=len(raw), schema_cache_for=schema, import_fields=import_fields or {}, + expected_db_sha256=expected or None, + before_commit=before_commit(session, sid) if before_commit else None, + import_id=import_id, + ) + return sid, result + + +async def total(source_id: str) -> int: + async with SessionLocal() as session: + view = await table_versions.resolve_source(session, await session.get(DataSource, source_id)) + try: + return (await run_query(view, 'SELECT SUM("n") FROM "t"')).rows[0][0] + finally: + await engines.invalidate(source_id) + + +SLIMMED = {"scan": None, "grid_preview": None, "facts": None, "drafts": None, "trial": None, + "answers_base": None, "cards": [], "questions": [], "draft_problems": [], + "answers": {}, "recipe_problems": [], "context_inputs": {}, + "trial_key": None, "trial_path": None} + + +async def new_staging(*, raw: bytes | None = None, source_id: str | None = None, + status: str = "drafting", trial: bool = False, **kw: Any) -> tuple[str, Path | None]: + """一个填满了大字段的暂存区。raw 给了就经 store_staging 存原件。返回 (暂存区 id, 试运行库路径)。""" + kw.setdefault("id", uuid.uuid4().hex) + staging = ImportStaging( + source_id=source_id or uuid.uuid4().hex, source_name=unique(), + kind="first", status=status, file_name="月报导出.xlsx", + scan={"sheets": [{"name": "客流汇总"}]}, + grid_preview=[{"sheet": "客流汇总", "cells": [[5, 3, "6123", "number"]]}], + facts={"facts": [{"id": "F1"}]}, drafts={"rules": {"complete": True}, "ai": None}, + recipe={"format": "agentlab-recipe/2"}, recipe_origin="rules", recipe_sha256=RECIPE_SHA, + answers_base={"format": "agentlab-recipe/2"}, + answers={"q_relation:F2": {"value": "dismiss", "reason": "合成理由"}}, + cards=[{"id": "c1"}], questions=[{"id": "q_relation:F2"}], recipe_problems=[{"code": "x"}], + draft_problems=[{"code": "y"}], draft_partial=True, context_inputs={"统计期": {"start": "2026-09-01"}}, + trial={"trial_id": "t", "receipt": {"outside_text": [{"text": "注：合成说明 12 处"}]}}, + ai_usage=[{"total_tokens": 10}], ai_consents=[{"signed_by": "测试员"}], signed_by="测试员", + **kw, + ) + path = None + if trial: + key = uuid.uuid4().hex + path = make_db(table_versions.trial_db_path(staging.source_id, staging.id, key)) + staging.trial_key, staging.trial_path = key, str(path) + async with SessionLocal() as session: + if raw is not None: + await table_versions.store_staging(session, staging, raw) + else: + session.add(staging) + await session.commit() + return staging.id, path + + +def assert_slimmed(staging: ImportStaging, status: str) -> None: + assert staging.status == status and staging.closed_at is not None + for name, value in SLIMMED.items(): + assert getattr(staging, name) == value, name + # 留下的：来源、文件哈希、工作配方、用量、同意记录、署名 + assert staging.recipe == {"format": "agentlab-recipe/2"} and staging.recipe_sha256 == RECIPE_SHA + assert staging.ai_usage == [{"total_tokens": 10}] and staging.ai_consents == [{"signed_by": "测试员"}] + assert staging.signed_by == "测试员" and staging.file_name == "月报导出.xlsx" + + +async def gc() -> None: + async with SessionLocal() as session: + await table_versions.gc(session) + + +async def startup() -> None: + async with SessionLocal() as session: + await table_versions.startup(session) + + +# -------------------------------------------------------------------------- +# 数据模型与迁移 +# -------------------------------------------------------------------------- + +NEW_IMPORT_COLUMNS = { + "recipe_id", "period_start", "period_end", "context", "checks", "overrides", "waivers", + "confirmations", "manifest_artifact", "signed_by", "staging_id", +} + + +def test_new_columns_are_mapped_and_migrated(): + migrated = {(t, c) for t, c, _ in db_base._COLUMN_MIGRATIONS} + assert ("data_sources", "current_recipe_id") in migrated + assert {("table_imports", c) for c in NEW_IMPORT_COLUMNS} <= migrated + assert "current_recipe_id" in DataSource.__table__.columns + assert NEW_IMPORT_COLUMNS <= set(TableImport.__table__.columns.keys()) + # publish_build 能写的列正是这些，不多不少 + assert table_versions.IMPORT_FIELDS == NEW_IMPORT_COLUMNS + # 配方格式的默认值与契约一致（模型不 import 数据层） + assert TableRecipe.__table__.columns["recipe_format"].default.arg == recipe_types.RECIPE_FORMAT + + +def test_migration_adds_the_new_columns_to_an_old_database(tmp_path): + """期 1 的库升级：新列补上、存量行为 NULL；新表由 create_all 建。""" + path = tmp_path / "old.db" + conn = sqlite3.connect(path) + conn.execute("CREATE TABLE data_sources (id VARCHAR(32) PRIMARY KEY, name VARCHAR(100))") + conn.execute("CREATE TABLE table_imports (id VARCHAR(32) PRIMARY KEY, source_id VARCHAR(32), seq INTEGER)") + conn.execute("INSERT INTO table_imports VALUES ('i1', 's1', 1)") + conn.commit() + conn.close() + engine = create_engine(f"sqlite:///{path}") + try: + with engine.begin() as sync: + # 和 init_db 同序：先 create_all（只建新表），再补列 + db_base.Base.metadata.create_all(sync) + db_base._migrate(sync) + imp_cols = {row[1] for row in sync.execute(text("PRAGMA table_info(table_imports)"))} + src_cols = {row[1] for row in sync.execute(text("PRAGMA table_info(data_sources)"))} + tables = {row[0] for row in sync.execute(text("SELECT name FROM sqlite_master WHERE type='table'"))} + row = sync.execute(text("SELECT recipe_id, checks, manifest_artifact FROM table_imports")).one() + assert NEW_IMPORT_COLUMNS <= imp_cols and "current_recipe_id" in src_cols + assert {"table_recipes", "import_stagings"} <= tables + assert tuple(row) == (None, None, None) + # 再跑一遍不报错（幂等） + with engine.begin() as sync: + db_base._migrate(sync) + finally: + engine.dispose() + + +async def test_recipe_seq_is_unique_per_source(): + sid = uuid.uuid4().hex + async with SessionLocal() as session: + session.add(TableRecipe(source_id=sid, seq=1, recipe={}, recipe_sha256=RECIPE_SHA)) + await session.commit() + session.add(TableRecipe(source_id=sid, seq=1, recipe={}, recipe_sha256=RECIPE_SHA)) + with pytest.raises(Exception, match="UNIQUE"): + await session.commit() + + +# -------------------------------------------------------------------------- +# id、公开名、错误码 +# -------------------------------------------------------------------------- + + +def test_recipe_build_id_covers_every_input(monkeypatch): + base = table_versions.recipe_build_id("s1", "a" * 64, "b" * 64, {}) + assert len(base) == 64 + assert table_versions.recipe_build_id("s1", "a" * 64, "b" * 64, {}) == base + assert table_versions.recipe_build_id("s2", "a" * 64, "b" * 64, {}) != base + assert table_versions.recipe_build_id("s1", "d" * 64, "b" * 64, {}) != base + assert table_versions.recipe_build_id("s1", "a" * 64, "e" * 64, {}) != base + period = {"context": {"统计期": {"start": "2026-09-01", "end": "2026-09-30"}}} + assert table_versions.recipe_build_id("s1", "a" * 64, "b" * 64, period) != base + # 与期 1 的构建 id 不会撞：组成不同 + opts = table_versions.parse_options("a.xlsx", header_row=1, mixed="reject", raw_mode=False) + assert table_versions.build_id("s1", "a" * 64, opts) != base + # 解析器、执行器升级换掉构建 id + monkeypatch.setattr(recipe_parsers, "PARSER_VER", recipe_parsers.PARSER_VER + "-next") + assert table_versions.recipe_build_id("s1", "a" * 64, "b" * 64, {}) != base + monkeypatch.undo() + monkeypatch.setattr(recipe_types, "RECIPE_ENGINE_VER", "recipe/2") + assert table_versions.recipe_build_id("s1", "a" * 64, "b" * 64, {}) != base + + +def test_recipe_build_options_shape(): + opts = table_versions.recipe_build_options("b" * 64, {}) + assert opts == {"kind": "recipe", "recipe_sha256": "b" * 64, + "parser_ver": recipe_parsers.PARSER_VER, "inputs": {}} + + +def test_public_names_keep_the_old_aliases(): + tv = table_versions + assert tv._store_lock is tv.store_lock and tv._parse_slot is tv.parse_slot + assert tv._introspect_view is tv.introspect_view and tv._check_probe is tv.check_probe + assert tv._sha_json is tv.sha_json and tv._settle_journal is tv.settle_journal + assert tv.PublishError("x").code is None + assert tv.PublishError("x", code="trial_missing").code == "trial_missing" + + +def test_trial_db_path_layout_and_validation(store): + path = table_versions.trial_db_path("src1", "stg1", "f" * 64) + assert path == store / "uploads" / "tables" / "src1" / "trials" / f"stg1-{'f' * 16}.db" + assert table_versions._trial_owner(path) == "stg1" + # 暂存区 id 带「-」也认得回来（trial_key 只有字母数字，从右边切） + dashed = table_versions.trial_db_path("src1", "stg-1_x", "a" * 64) + assert dashed.name == f"stg-1_x-{'a' * 16}.db" and table_versions._trial_owner(dashed) == "stg-1_x" + for bad in (("../x", "stg", "f" * 64), ("src", "a/b", "f" * 64), ("src", "stg", "short")): + with pytest.raises(ValueError): + table_versions.trial_db_path(*bad) + + +# -------------------------------------------------------------------------- +# publish_build +# -------------------------------------------------------------------------- + + +async def test_publish_build_publishes_a_trial_db_with_recipe_fields(store): + name, raw = unique(), raw_bytes() + sid = uuid.uuid4().hex + tmp = new_trial(sid) + seen: list = [] + manifests: list[str] = [] + imp_id = uuid.uuid4().hex + + async def write_manifest(cache, info): + # 配方导入在这里写导入清单：put_json 另开会话提交，主会话此时不能占着写锁 + from app.core import artifact_store + + mid = await artifact_store.put_json( + {"import_id": info.import_id, "db_sha256": info.db_sha256}, kind="import_manifest") + manifests.append(mid) + cache["import_manifests"] = [mid] + + fields = {"recipe_id": "r" * 32, "period_start": "2026-09-01", "period_end": "2026-09-30", + "checks": [{"id": "C2", "status": "passed"}], "signed_by": "测试员", "staging_id": "s" * 32} + _, result = await publish_trial(name, tmp, raw, source_id=sid, import_fields=fields, + import_id=imp_id, seen=seen, schema_extra=write_manifest) + + assert not tmp.exists() # 试运行库被消耗 + row = await get(DataSource, sid) + assert row.origin == "upload" and row.current_snapshot_id == result.snapshot_id + assert result.import_id == imp_id and result.seq == 1 and result.build_reused is False + assert result.snapshot_id == table_versions.snapshot_id(sid, [imp_id], recipe_types.RECIPE_ENGINE_VER) + build = await get(TableBuild, result.build_id) + assert build.engine_ver == recipe_types.RECIPE_ENGINE_VER + assert build.options == table_versions.recipe_build_options(RECIPE_SHA, {}) + assert build.options_sha256 == table_versions.sha_json(build.options) + assert stat.S_IMODE(Path(build.db_path).stat().st_mode) == 0o444 + assert result.db_sha256 == build.db_sha256 == raw_store.sha256_file(Path(build.db_path)) + [imp] = await imports_of(sid) + for key, value in fields.items(): + assert getattr(imp, key) == value + assert imp.raw_state == "kept" and raw_store.raw_exists(imp.raw_sha256) + # schema_cache_for 收到绑定新快照的视图和实际发布的结果 + [(view, info)] = seen + assert view.snapshot_id == result.snapshot_id and view.database == build.db_path + assert (info.import_id, info.snapshot_id, info.seq, info.build_id) == (imp_id, result.snapshot_id, 1, result.build_id) + assert info.db_sha256 == build.db_sha256 and info.build_reused is False + snap = await get(SourceSnapshot, result.snapshot_id) + assert snap.schema_cache["import_manifests"] == manifests and set(snap.schema_cache["tables"]) == {"t"} + assert await total(sid) == 3 + + +async def test_publish_build_refuses_a_tampered_trial(store): + """试运行之后库被改过：不 link、不写任何记录、指针不动，试运行库照样被消耗。""" + name, raw = unique(), raw_bytes() + sid, first = await publish_trial(name, new_trial(uuid.uuid4().hex), raw_bytes()) + before = files_under(store / "uploads" / "tables" / sid / "builds") + tmp = new_trial(sid) + expected = settled_sha(tmp) + conn = sqlite3.connect(tmp) + conn.execute('UPDATE "t" SET "n" = 1000') + conn.commit() + conn.close() + with pytest.raises(table_versions.PublishError) as info: + await publish_trial(name, tmp, raw, expected=expected) + assert info.value.code == "trial_tampered" and "请重新试运行" in str(info.value) + assert files_under(store / "uploads" / "tables" / sid / "builds") == before + assert not tmp.exists() + assert (await get(DataSource, sid)).current_snapshot_id == first.snapshot_id + assert len(await imports_of(sid)) == 1 + assert not raw_store.raw_exists(hashlib.sha256(raw).hexdigest()) # 这次新存的原件也删了 + assert await total(sid) == 3 + + +@pytest.mark.parametrize("state", ["missing", "empty"]) +async def test_publish_build_without_a_trial_db_never_creates_an_empty_one(store, state): + name, raw = unique(), raw_bytes() + sid = uuid.uuid4().hex + tmp = table_versions.trial_db_path(sid, uuid.uuid4().hex, uuid.uuid4().hex) + if state == "empty": + tmp.parent.mkdir(parents=True) + tmp.write_bytes(b"") + with pytest.raises(table_versions.PublishError) as info: + await publish_trial(name, tmp, raw, source_id=sid, expected=None) + assert info.value.code == "trial_missing" + assert not tmp.exists() + assert not files_under(store / "uploads" / "tables" / sid / "builds") + assert await get(DataSource, sid) is None + assert not raw_store.raw_exists(hashlib.sha256(raw).hexdigest()) + + +async def test_publish_build_ignores_a_trial_path_outside_the_uploads_dir(store, tmp_path): + outside = make_db(tmp_path / "elsewhere" / "x.db") + before = outside.read_bytes() + with pytest.raises(table_versions.PublishError) as info: + await publish_trial(unique(), outside, raw_bytes(), expected=None) + assert info.value.code == "trial_missing" + assert outside.read_bytes() == before # 不是我们的文件，不删 + + +async def test_wal_trial_settled_before_hashing_matches_at_publish(store): + """试运行库要是 WAL 模式，先 settle 再算回执哈希；发布时再 settle 一次不改文件，哈希对得上。""" + sid = uuid.uuid4().hex + tmp = new_trial(sid, wal=True) + assert tmp.read_bytes()[18:20] == b"\x02\x02" + unsettled = raw_store.sha256_file(tmp) + settled = settled_sha(tmp) + assert settled != unsettled # settle 改了文件头 + _, result = await publish_trial(unique(), tmp, raw_bytes(), source_id=sid, expected=settled) + build = await get(TableBuild, result.build_id) + assert raw_store.sha256_file(Path(build.db_path)) == settled == result.db_sha256 + assert Path(build.db_path).read_bytes()[18:20] == b"\x01\x01" + + # 反例：没 settle 就算的哈希，发布时对不上 + tmp2 = new_trial(sid, wal=True) + early = raw_store.sha256_file(tmp2) + with pytest.raises(table_versions.PublishError) as info: + await publish_trial(unique(), tmp2, raw_bytes(), expected=early) + assert info.value.code == "trial_tampered" + + +async def test_reused_build_reports_the_existing_file_hash(store): + """同一个构建 id 再发布：复用已有文件，PublishInfo.db_sha256 是已有文件的，不是这次试运行库的。""" + name, raw = unique(), raw_bytes() + sid = uuid.uuid4().hex + bid = table_versions.recipe_build_id(sid, hashlib.sha256(raw).hexdigest(), RECIPE_SHA, {}) + _, first = await publish_trial(name, new_trial(sid), raw, source_id=sid, build_id=bid) + # 内容相同、字节不同的试运行库（比如 SQLite 升级后文件头的版本号不同） + tmp = new_trial(sid, rows=[(2, "分区乙"), (1, "分区甲")]) + tmp_sha = settled_sha(tmp) + assert tmp_sha != first.db_sha256 + seen: list = [] + _, second = await publish_trial(name, tmp, raw, build_id=bid, expected=tmp_sha, seen=seen) + [(_view, info)] = seen + assert second.build_reused is True and info.build_reused is True + assert info.db_sha256 == first.db_sha256 == second.db_sha256 + assert info.seq == 2 and second.seq == 2 + assert not tmp.exists() + # 第一次导入随回收置 retired；构建文件仍被第二次导入用着，原件相同也留着 + imps = await imports_of(sid) + assert [i.status for i in imps] == ["retired", "active"] + assert [i.raw_state for i in imps] == ["kept", "kept"] + assert Path((await get(TableBuild, bid)).db_path).exists() + + +async def test_before_commit_failure_rolls_back_and_removes_new_files(store): + name = unique() + sid, first = await publish_trial(name, new_trial(uuid.uuid4().hex), raw_bytes()) + builds = store / "uploads" / "tables" / sid / "builds" + before = files_under(builds) + raw = raw_bytes() + tmp = new_trial(sid) + + def failing(session, source_id): + async def hook(imp_id, snap_id): + raise RuntimeError("模拟锁内核对失败") + return hook + + with pytest.raises(RuntimeError, match="模拟锁内核对失败"): + await publish_trial(name, tmp, raw, before_commit=failing) + assert files_under(builds) == before and not tmp.exists() + assert (await get(DataSource, sid)).current_snapshot_id == first.snapshot_id + assert [i.status for i in await imports_of(sid)] == ["active"] + assert not raw_store.raw_exists(hashlib.sha256(raw).hexdigest()) + assert await total(sid) == 3 + + +async def test_before_commit_failure_does_not_create_a_new_source(store): + sid = uuid.uuid4().hex + name = unique() + + def failing(session, source_id): + async def hook(imp_id, snap_id): + raise RuntimeError("同名源已存在") + return hook + + with pytest.raises(RuntimeError): + await publish_trial(name, new_trial(sid), raw_bytes(), source_id=sid, before_commit=failing) + assert await get(DataSource, sid) is None + assert not files_under(store / "uploads" / "tables" / sid / "builds") + + +async def test_before_commit_runs_in_the_transaction_before_the_pointer_moves(store): + """回调里：select 读到发布前的指针；新源还看不到；导入记录取得到；写的东西和发布一起提交。""" + name = unique() + sid, first = await publish_trial(name, new_trial(uuid.uuid4().hex), raw_bytes()) + observed: dict[str, Any] = {} + + def hook_for(session, source_id): + async def hook(imp_id, snap_id): + observed["pointer"] = (await session.execute( + select(DataSource.current_snapshot_id).where(DataSource.id == source_id))).scalar() + observed["active"] = (await session.execute( + select(TableImport.id).where(TableImport.source_id == source_id, + TableImport.status == "active"))).scalars().all() + imp = await session.get(TableImport, imp_id) + imp.manifest_artifact = "m" * 64 + session.add(TableRecipe(id=uuid.uuid4().hex, source_id=source_id, seq=1, recipe={}, + recipe_sha256=RECIPE_SHA, status="active")) + observed["snap"] = snap_id + return hook + + _, second = await publish_trial(name, new_trial(sid), raw_bytes(), before_commit=hook_for) + assert observed["pointer"] == first.snapshot_id # 指针还没切 + assert observed["active"] == [second.import_id] # 旧导入已置 superseded + assert observed["snap"] == second.snapshot_id + imps = await imports_of(sid) + assert imps[-1].manifest_artifact == "m" * 64 + async with SessionLocal() as session: + recipes = (await session.execute(select(TableRecipe).where(TableRecipe.source_id == sid))).scalars().all() + assert len(recipes) == 1 + + # 新源：回调里按名字查不到这次要建的源 + fresh_id, fresh_name = uuid.uuid4().hex, unique() + found: list = [] + + def lookup(session, source_id): + async def hook(imp_id, snap_id): + found.extend((await session.execute( + select(DataSource.id).where(DataSource.name == fresh_name))).scalars().all()) + return hook + + await publish_trial(fresh_name, new_trial(fresh_id), raw_bytes(), source_id=fresh_id, before_commit=lookup) + assert found == [] and (await get(DataSource, fresh_id)).name == fresh_name + + +async def test_publish_build_with_raw_none_needs_the_archived_raw(store): + raw = raw_bytes() + sid = uuid.uuid4().hex + tmp = new_trial(sid) + with pytest.raises(table_versions.PublishError) as info: + await publish_trial(unique(), tmp, raw, source_id=sid, pass_raw=False) + assert info.value.code == "raw_missing" + assert not tmp.exists() and await get(DataSource, sid) is None + assert not files_under(store / "uploads" / "tables" / sid / "builds") + + sha, _ = raw_store.put_raw(raw) # 暂存时存过 + _, result = await publish_trial(unique(), new_trial(sid), raw, source_id=sid, pass_raw=False) + [imp] = await imports_of(sid) + assert imp.raw_sha256 == sha and imp.raw_state == "kept" and raw_store.raw_exists(sha) + assert imp.file_size == len(raw) and imp.file_name == "月报导出.xlsx" + + +async def test_import_fields_cannot_touch_other_columns(store): + sid = uuid.uuid4().hex + tmp = new_trial(sid) + with pytest.raises(ValueError, match="status"): + await publish_trial(unique(), tmp, raw_bytes(), source_id=sid, import_fields={"status": "retired"}) + assert not tmp.exists() and await get(DataSource, sid) is None + + +async def test_publish_build_checks_raw_against_its_hash(store): + sid = uuid.uuid4().hex + tmp = new_trial(sid) + raw = raw_bytes() + + async def run(): + async with SessionLocal() as session: + row = DataSource(id=sid, name=unique(), kind="sqlite", readonly=True, origin="upload") + await table_versions.publish_build( + session, row, tmp_db=tmp, build_id="b" * 64, build_options={}, engine_ver="recipe/1", + report={}, raw=raw, raw_sha=hashlib.sha256(b"other").hexdigest(), file_name="x.xlsx", + file_size=len(raw), schema_cache_for=None, import_fields={}) + + with pytest.raises(ValueError, match="原件"): + await run() + assert not tmp.exists() and await get(DataSource, sid) is None + + +async def test_publish_upload_goes_through_publish_build(monkeypatch): + calls: list[dict] = [] + real = table_versions.publish_build + + async def spy(session, source, **kw): + calls.append(kw) + return await real(session, source, **kw) + + monkeypatch.setattr(table_versions, "publish_build", spy) + from openpyxl import Workbook + + book = Workbook() + book.active.title = "甲表" + book.active.append(["区域", "数量"]) + book.active.append(["分区甲", 1]) + buf = io.BytesIO() + book.save(buf) + async with SessionLocal() as session: + row = DataSource(id=uuid.uuid4().hex, name=unique(), kind="sqlite", readonly=True, origin="upload") + result = await table_versions.publish_upload(session, row, buf.getvalue(), "客流.xlsx") + [kw] = calls + assert kw["engine_ver"] == table_versions.ENGINE_VER and kw["import_fields"] == {} + assert "expected_db_sha256" not in kw and "before_commit" not in kw + assert result.seq == 1 and result.db_sha256 + + +# -------------------------------------------------------------------------- +# 暂存区：结束即瘦身、写入 +# -------------------------------------------------------------------------- + + +async def test_close_staging_slims_and_returns_the_trial_path(): + sid, path = await new_staging(trial=True) + async with SessionLocal() as session: + staging = await session.get(ImportStaging, sid) + with pytest.raises(ValueError): + table_versions.close_staging(staging, "drafting") + old = table_versions.close_staging(staging, "discarded") + await session.commit() + assert old == str(path) and path.exists() # 文件留给提交之后删 + assert_slimmed(await get(ImportStaging, sid), "discarded") + assert table_versions.remove_trial_file(old) is True and not path.exists() + + +def test_remove_trial_file_only_touches_trial_dbs(store, tmp_path): + outside = make_db(tmp_path / "trials" / "x.db") + assert table_versions.remove_trial_file(str(outside)) is False and outside.exists() + build = make_db(table_versions.tables_root() / "src" / "builds" / "x.db") + assert table_versions.remove_trial_file(str(build)) is False and build.exists() + assert table_versions.remove_trial_file(None) is False + + +async def test_store_staging_saves_raw_and_row_under_the_lock(store): + raw = raw_bytes() + sid, _ = await new_staging(raw=raw) + staging = await get(ImportStaging, sid) + assert staging.raw_sha256 == hashlib.sha256(raw).hexdigest() and raw_store.raw_exists(staging.raw_sha256) + assert staging.file_size == len(raw) + assert staging.expires_at is not None + assert abs((staging.expires_at - staging.created_at) - table_versions.STAGING_TTL) < timedelta(minutes=1) + + +async def test_store_staging_failure_removes_a_new_raw(store): + raw = raw_bytes() + dup = uuid.uuid4().hex + await new_staging(raw=raw_bytes(), id=dup) + staging = ImportStaging(id=dup, source_id=uuid.uuid4().hex, kind="first") + async with SessionLocal() as session: + with pytest.raises(Exception): + await table_versions.store_staging(session, staging, raw) + assert not raw_store.raw_exists(hashlib.sha256(raw).hexdigest()) + + +# -------------------------------------------------------------------------- +# 回收与清理保护 +# -------------------------------------------------------------------------- + + +async def test_open_staging_raw_survives_gc_and_startup(store): + raw = raw_bytes() + sha = hashlib.sha256(raw).hexdigest() + sid, _ = await new_staging(raw=raw, status="rejected") + await gc() + assert raw_store.raw_exists(sha) + await startup() + assert raw_store.raw_exists(sha) + assert (await get(ImportStaging, sid)).status == "rejected" + + +async def test_open_staging_protects_a_raw_shared_with_a_replaced_import(store): + """上传新一期被拒收、正在修改时，那份原件和一次旧导入相同：旧导入被回收，原件不能跟着删。""" + name, raw = unique(), raw_bytes() + src, first = await publish_trial(name, new_trial(uuid.uuid4().hex), raw) + sha = hashlib.sha256(raw).hexdigest() + await new_staging(raw=raw, source_id=src, status="rejected") + await publish_trial(name, new_trial(src), raw_bytes()) # 替换掉第一期，回收一次 + old = (await imports_of(src))[0] + assert old.status == "retired" and old.raw_state == "kept" + assert raw_store.raw_exists(sha) + + +async def test_gc_removes_raws_and_trials_left_by_closed_stagings(store): + """放弃、删源时没来得及删的原件和试运行库：暂存区已结束、没人要了，回收删掉。""" + lone = raw_bytes() + shared = raw_bytes() + lone_id, lone_trial = await new_staging(raw=lone, trial=True) + shared_id, _ = await new_staging(raw=shared) + keeper_id, _ = await new_staging(raw=shared) # 同一份原件还有一个未结束的暂存区 + async with SessionLocal() as session: + for sid in (lone_id, shared_id): + staging = await session.get(ImportStaging, sid) + staging.status = "discarded" # 模拟只改了状态的结束路径 + await session.commit() + open_id, open_trial = await new_staging(trial=True) + orphan = new_trial(uuid.uuid4().hex) # 暂存区不存在：回收不碰，留给启动清理 + await gc() + assert not raw_store.raw_exists(hashlib.sha256(lone).hexdigest()) + assert raw_store.raw_exists(hashlib.sha256(shared).hexdigest()) + assert not lone_trial.exists() + assert open_trial.exists() and orphan.exists() + await startup() + assert open_trial.exists() and not orphan.exists() + + +async def test_recipe_sources_keep_the_raws_of_their_last_12_imports(store): + recipe_src, plain_src = uuid.uuid4().hex, uuid.uuid4().hex + shas: dict[str, list[str]] = {recipe_src: [], plain_src: []} + async with SessionLocal() as session: + session.add(DataSource(id=recipe_src, name=unique(), kind="sqlite", readonly=True, + origin="upload", current_recipe_id="r" * 32)) + session.add(DataSource(id=plain_src, name=unique(), kind="sqlite", readonly=True, origin="upload")) + for src in (recipe_src, plain_src): + for seq in range(1, 15): + sha, _ = raw_store.put_raw(raw_bytes()) + shas[src].append(sha) + session.add(TableImport(source_id=src, seq=seq, raw_sha256=sha, raw_state="kept", + status="active" if seq == 14 else "superseded")) + await session.commit() + await gc() + recipe = await imports_of(recipe_src) + assert [i.raw_state for i in recipe] == ["purged"] * 2 + ["kept"] * 12 + assert [raw_store.raw_exists(s) for s in shas[recipe_src]] == [False] * 2 + [True] * 12 + assert {i.status for i in recipe[:13]} == {"retired"} and recipe[13].status == "active" + plain = await imports_of(plain_src) + assert [i.raw_state for i in plain] == ["purged"] * 13 + ["kept"] + assert [raw_store.raw_exists(s) for s in shas[plain_src]] == [False] * 13 + [True] + # 原件留着的导入，启动清理也不当孤儿删 + await startup() + assert [raw_store.raw_exists(s) for s in shas[recipe_src]] == [False] * 2 + [True] * 12 + assert table_versions.RECIPE_RAW_KEEP == 12 + + +async def test_startup_sweeps_stale_trial_dbs(store): + current_id, current = await new_staging(trial=True) + # 同一个暂存区的旧试运行库（trial_path 已经指向新的） + stale_same = new_trial( + (await get(ImportStaging, current_id)).source_id, current_id) + closed_id, _ = await new_staging() + async with SessionLocal() as session: + staging = await session.get(ImportStaging, closed_id) + staging.status = "committed" + await session.commit() + closed_trial = new_trial(uuid.uuid4().hex, closed_id) + missing = new_trial(uuid.uuid4().hex) + Path(f"{missing}-journal").write_bytes(b"") + await startup() + assert current.exists() + assert not stale_same.exists() and not closed_trial.exists() and not missing.exists() + assert not Path(f"{missing}-journal").exists() + assert not missing.parent.exists() # 空了的 trials/ 和源目录一并删 + + +async def test_trial_dbs_of_dashed_staging_ids_are_matched_to_their_staging(store): + """暂存区 id 带「-」时，回收和启动清理按文件名找回的必须是它本身，不是「-」前面那一截。 + + 切错了：开着的暂存区的库被启动清理当孤儿删掉，已结束的暂存区的库回收又认不出来。 + """ + tag = uuid.uuid4().hex[:8] + open_id, open_trial = await new_staging(trial=True, id=f"stg-{tag}-open") + closed_id, closed_trial = await new_staging(trial=True, id=f"stg-{tag}-closed") + async with SessionLocal() as session: + staging = await session.get(ImportStaging, closed_id) + staging.status = "discarded" # 只改状态，库留给回收删 + await session.commit() + await gc() + assert open_trial.exists() and not closed_trial.exists() + await startup() + assert open_trial.exists() + assert (await get(ImportStaging, open_id)).trial_path == str(open_trial) + + +async def test_expire_stagings_slims_deletes_trials_and_releases_raws(store): + past = utcnow() - timedelta(seconds=1) + lone, shared = raw_bytes(), raw_bytes() + src, _ = await publish_trial(unique(), new_trial(uuid.uuid4().hex), shared) + expired_id, trial = await new_staging(raw=lone, trial=True, expires_at=past) + shared_id, _ = await new_staging(raw=shared, source_id=src, status="trialed", expires_at=past) + fresh_id, fresh_trial = await new_staging(raw=raw_bytes(), trial=True) + # 没写 expires_at 的：按 created_at 加 TTL 算 + legacy_id, _ = await new_staging(created_at=utcnow() - table_versions.STAGING_TTL - timedelta(hours=1)) + async with SessionLocal() as session: + expired = await table_versions.expire_stagings(session) + assert set(expired) >= {expired_id, shared_id, legacy_id} and fresh_id not in expired + for sid in (expired_id, shared_id, legacy_id): + assert_slimmed(await get(ImportStaging, sid), "expired") + assert not trial.exists() and fresh_trial.exists() + assert not raw_store.raw_exists(hashlib.sha256(lone).hexdigest()) + assert raw_store.raw_exists(hashlib.sha256(shared).hexdigest()) # 导入记录还留着它 + assert (await get(ImportStaging, fresh_id)).status == "drafting" + async with SessionLocal() as session: + assert await table_versions.expire_stagings(session) == [] # 没有要处理的就不等锁 + + +async def test_closed_stagings_older_than_90_days_are_deleted(store): + old_id, _ = await new_staging(status="discarded", + closed_at=utcnow() - table_versions.STAGING_KEEP_CLOSED - timedelta(days=1)) + recent_id, _ = await new_staging(status="committed", closed_at=utcnow() - timedelta(days=30)) + open_id, _ = await new_staging(created_at=utcnow() - timedelta(days=200), + expires_at=utcnow() + timedelta(days=1)) + # 没写 closed_at 的（某条结束路径只改了状态）：按 updated_at 算，不会永远留着 + unstamped_old, _ = await new_staging( + status="expired", updated_at=utcnow() - table_versions.STAGING_KEEP_CLOSED - timedelta(days=1)) + unstamped_recent, _ = await new_staging(status="committed", updated_at=utcnow() - timedelta(days=30)) + assert (await get(ImportStaging, unstamped_old)).closed_at is None + await gc() # 回收开头顺手处理 + assert await get(ImportStaging, old_id) is None + assert await get(ImportStaging, recent_id) is not None + assert (await get(ImportStaging, open_id)).status == "drafting" + assert await get(ImportStaging, unstamped_old) is None + assert await get(ImportStaging, unstamped_recent) is not None + + +async def test_gc_expires_stagings_first(store): + sid, trial = await new_staging(raw=raw_bytes(), trial=True, expires_at=utcnow() - timedelta(minutes=1)) + await gc() + assert_slimmed(await get(ImportStaging, sid), "expired") + assert not trial.exists() + + +# -------------------------------------------------------------------------- +# 清除原件、放弃导入、删除数据源 +# -------------------------------------------------------------------------- + + +async def test_purging_a_raw_discards_open_stagings_that_use_it(store): + raw = raw_bytes() + sha = hashlib.sha256(raw).hexdigest() + src, _ = await publish_trial(unique(), new_trial(uuid.uuid4().hex), raw) + hit_id, hit_trial = await new_staging(raw=raw, source_id=src, status="trialed", trial=True) + closed_id, _ = await new_staging(raw=raw, status="committed") + other_raw = raw_bytes() + other_id, other_trial = await new_staging(raw=other_raw, trial=True) + [imp] = await imports_of(src) + async with SessionLocal() as session: + imp = await session.get(TableImport, imp.id) + result = await table_versions.purge_import_raw(session, imp, reason="合成理由", signed_by="测试员") + deleted, others = result # 期 1 的两元组解包照旧 + assert deleted is True and others == [] + assert [s.id for s in result.discarded_stagings] == [hit_id] + assert_slimmed(await get(ImportStaging, hit_id), "discarded") + assert not hit_trial.exists() and not raw_store.raw_exists(sha) + assert (await get(ImportStaging, closed_id)).status == "committed" + assert (await get(ImportStaging, other_id)).status == "drafting" and other_trial.exists() + assert raw_store.raw_exists(hashlib.sha256(other_raw).hexdigest()) + + +async def test_release_staging_raw_only_deletes_unreferenced_raws(store): + raw = raw_bytes() + sha = hashlib.sha256(raw).hexdigest() + a_id, _ = await new_staging(raw=raw) + b_id, _ = await new_staging(raw=raw) + + async def discard(sid: str) -> bool: + async with SessionLocal() as session: + staging = await session.get(ImportStaging, sid) + table_versions.remove_trial_file(table_versions.close_staging(staging, "discarded")) + await session.commit() + return await table_versions.release_staging_raw(session, staging) + + async with SessionLocal() as session: + still_open = await session.get(ImportStaging, a_id) + assert await table_versions.release_staging_raw(session, still_open) is False + assert await discard(a_id) is False and raw_store.raw_exists(sha) # b 还开着 + assert await discard(b_id) is True and not raw_store.raw_exists(sha) + + # 有留着原件的导入记录时也不删 + kept = raw_bytes() + await publish_trial(unique(), new_trial(uuid.uuid4().hex), kept) + c_id, _ = await new_staging(raw=kept) + assert await discard(c_id) is False and raw_store.raw_exists(hashlib.sha256(kept).hexdigest()) + + +async def test_release_staging_raw_keeps_the_raw_of_a_staging_still_open(store): + """暂存区还开着就不删原件，哪怕它是这份原件唯一的引用：删了它之后的试运行、提交就拿不到原件。""" + raw = raw_bytes() + sha = hashlib.sha256(raw).hexdigest() + sid, _ = await new_staging(raw=raw, status="trialed") + async with SessionLocal() as session: + staging = await session.get(ImportStaging, sid) + assert await table_versions.release_staging_raw(session, staging) is False + assert raw_store.raw_exists(sha) + assert (await get(ImportStaging, sid)).status == "trialed" + + +async def test_retire_source_retires_recipes_and_discards_open_stagings(store): + name, raw = unique(), raw_bytes() + src, _ = await publish_trial(name, new_trial(uuid.uuid4().hex), raw_bytes()) + staging_raw = raw_bytes() + stg_id, trial = await new_staging(raw=staging_raw, source_id=src, status="rejected", trial=True) + async with SessionLocal() as session: + session.add(TableRecipe(source_id=src, seq=1, recipe={}, recipe_sha256=RECIPE_SHA, status="active")) + await session.commit() + await table_versions.retire_source(session, src) + await session.delete(await session.get(DataSource, src)) + await session.commit() + await table_versions.gc(session) + async with SessionLocal() as session: + recipes = (await session.execute(select(TableRecipe).where(TableRecipe.source_id == src))).scalars().all() + assert {r.status for r in recipes} == {"retired"} + assert_slimmed(await get(ImportStaging, stg_id), "discarded") + assert not trial.exists() + assert not raw_store.raw_exists(hashlib.sha256(staging_raw).hexdigest()) + assert {i.status for i in await imports_of(src)} == {"retired"} + + +async def test_concurrent_publish_builds_are_serialized(store): + """同一个源同时来两次发布：序号不重、最后只有一条 active。""" + name = unique() + src, _ = await publish_trial(name, new_trial(uuid.uuid4().hex), raw_bytes()) + results = await asyncio.gather( + publish_trial(name, new_trial(src), raw_bytes()), publish_trial(name, new_trial(src), raw_bytes())) + imps = await imports_of(src) + assert [i.seq for i in imps] == [1, 2, 3] + assert [i.status for i in imps].count("active") == 1 + assert (await get(DataSource, src)).current_snapshot_id in {r.snapshot_id for _, r in results} diff --git a/backend/tests/test_tabular.py b/backend/tests/test_tabular.py index 8016b2d6..b733dfb4 100644 --- a/backend/tests/test_tabular.py +++ b/backend/tests/test_tabular.py @@ -326,6 +326,70 @@ def test_same_named_sheets_are_rejected_and_nothing_is_imported(tmp_path): assert not db.exists() or tables(db) == [] +def _write_only_xlsx(rows: int) -> bytes: + """openpyxl write_only 写出的工作簿：不带 （不少导出工具都这样写）。""" + from openpyxl import Workbook + + wb = Workbook(write_only=True) + ws = wb.create_sheet("明细") + ws.append(["地区", "产品", "销量", "金额", "备注"]) + for i in range(rows): + ws.append([f"区{i % 7}", f"品{i % 50}", i % 997, i * 3, "无"]) + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def test_open_book_does_not_parse_a_whole_sheet_to_find_a_missing_dimension(): + """openpyxl 只读打开时要找 ，原实现一直读到 ：没有这个元素的文件，每打开一次就把 + 整张表解析一遍（20 万行约 3 秒，交互路径每次回答都要开两遍）。open_book 装的等价实现读到 + 开头就停：有 的照样取到，没有的照样留空（读取层本来就按扫描的边界读）。""" + raw = _write_only_xlsx(20_000) + part = next(n for n in zipfile.ZipFile(io.BytesIO(raw)).namelist() if n.startswith("xl/worksheets/sheet")) + size = zipfile.ZipFile(io.BytesIO(raw)).getinfo(part).file_size + assert size > 1_000_000 and b" 开头附近 + counted: list[int] = [] + original = ws._get_source + + class Counting: + def __init__(self, f): + self.f, self.n = f, 0 + + def read(self, n=-1): + b = self.f.read(n) + self.n += len(b) + return b + + def close(self): + counted.append(self.n) + self.f.close() + + ws._get_source = lambda: Counting(original()) + ws._get_size() + del ws._get_source + assert counted and counted[0] < size // 10, (counted, size) + assert ws.max_row is None + # 值照常读得出来 + assert list(ws.iter_rows(min_row=2, max_row=2, min_col=1, max_col=5, values_only=True)) == \ + [("区0", "品0", 0, 0, "无")] + finally: + book.close() + # 有 的（Excel 存盘的文件）照样取到；公式一份（data_only=False）同样 + with_dim = xlsx({"甲": [["a", "b"], [1, 2], [3, 4]]}) + for data_only in (True, False): + book = tabular.open_book(with_dim, data_only=data_only) + try: + ws = book.worksheets[0] + assert (ws.min_row, ws.min_column, ws.max_row, ws.max_column) == (1, 1, 3, 2) + finally: + book.close() + + def test_worksheets_are_matched_by_part_path(tmp_path): """扫描结果和 openpyxl 的工作表按部件路径配对。钉住 openpyxl 的私有属性 _worksheet_path： 它改名或换了格式，这里先失败，而不是导入时静默配错。""" diff --git a/backend/tests/test_xlsx_cells.py b/backend/tests/test_xlsx_cells.py new file mode 100644 index 00000000..1af1c822 --- /dev/null +++ b/backend/tests/test_xlsx_cells.py @@ -0,0 +1,299 @@ +"""读取层（xlsx_cells）、扫描的逐行游程与序列化（xlsx_scan.row_runs / scan_to_json）、tabular 的公开名。 + +夹具都在测试里用 openpyxl 现造，标签用假名；需要 openpyxl 写不出来的形状（共享公式、行号乱序、陈旧的 +、公式缓存值）时在 zip 层改 XML。 +""" +from __future__ import annotations + +import datetime as dt +import io +import json +import re +import zipfile + +import pytest +from openpyxl import Workbook + +from app.data import tabular, xlsx_cells, xlsx_scan +from app.data.xlsx_scan import UnsupportedTable + + +def _save(wb: Workbook) -> bytes: + buf = io.BytesIO() + wb.save(buf) + return buf.getvalue() + + +def _patch(raw: bytes, part: str, fn) -> bytes: + zin = zipfile.ZipFile(io.BytesIO(raw)) + out = io.BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as zout: + for it in zin.infolist(): + data = zin.read(it.filename) + if it.filename == part: + data = fn(data.decode("utf-8")).encode("utf-8") + zout.writestr(it, data) + return out.getvalue() + + +def _shared_formula_book() -> bytes: + """B5 是 A5 的共享公式（），两格都有缓存值。""" + wb = Workbook() + ws = wb.active + ws.title = "表甲" + for r in range(1, 5): + ws.cell(r, 1, r) + ws.cell(r, 2, r * 10) + ws["A5"] = "=SUM(A1:A4)" + ws["B5"] = "=SUM(B1:B4)" + + def fix(t: str) -> str: + t = re.sub(r']*)>SUM\(A1:A4\)(?:|)?', + r'SUM(A1:A4)10', t) + t = re.sub(r']*)>SUM\(B1:B4\)(?:|)?', + r'100', t) + assert 'si="0"/>' in t + return t + + return _patch(_save(wb), "xl/worksheets/sheet1.xml", fix) + + +def _mixed_book() -> bytes: + """各种取值：文本、只含空白的文本、0、False、日期、错误值、无缓存的公式；一张隐藏工作表。""" + wb = Workbook() + ws = wb.active + ws.title = "表甲" + ws["B2"] = "分区甲" + ws["C2"] = " " # 只含空白：扫描、读取都算空 + ws["D2"] = 0 # 0 不算空 + ws["E2"] = False # False 不算空 + ws["B3"] = dt.datetime(2026, 8, 1) + ws["C3"] = "#DIV/0!" + ws["D3"] = "=B9*2" # openpyxl 写公式不带缓存值：读出来值是空、公式照给 + ws["B4"] = 12.5 + ws.merge_cells("B6:D6") + ws["B6"] = "合并标题" + ws.row_dimensions[4].hidden = True + ws.column_dimensions["E"].hidden = True + hidden = wb.create_sheet("隐藏表") + hidden["A1"] = "口令" + hidden.sheet_state = "hidden" + return _save(wb) + + +# -------------------------------------------------------------------------- +# tabular 的公开名 +# -------------------------------------------------------------------------- + + +def test_tabular_public_names_keep_old_aliases(): + assert tabular._open_book is tabular.open_book + assert tabular._worksheet_for is tabular.worksheet_for + + +@pytest.mark.parametrize("value,text", [ + (dt.datetime(2026, 8, 1), "2026-08-01"), # 期 1 的 #23：零点日期不能写成「… 00:00:00」 + (dt.datetime(2026, 8, 1, 9, 30), "2026-08-01 09:30:00"), + (dt.date(2026, 8, 1), "2026-08-01"), + (dt.time(7, 5), "07:05:00"), + (3.0, "3"), (2.5, "2.5"), (12, "12"), (True, "TRUE"), + (" 分区甲 ", "分区甲"), (" ", ""), (None, ""), +]) +def test_as_text_matches_simple_import(value, text): + assert tabular.as_text(value) == text + + +# -------------------------------------------------------------------------- +# 扫描：逐行游程、序列化 +# -------------------------------------------------------------------------- + + +def test_row_runs_merge_consecutive_rows_with_same_count(): + wb = Workbook() + ws = wb.active + for r in (1, 2, 3): + ws.cell(r, 1, "甲") + ws.cell(r, 2, r) + ws.cell(5, 1, "乙") # 第 4 行空、第 5 行 1 格 + ws.cell(6, 1, "丙") + ws.cell(7, 1, "丁") + ws.cell(7, 3, 1) + sc = xlsx_scan.scan(_save(wb)) + assert sc.sheets[0].row_runs == [(1, 3, 2), (5, 6, 1), (7, 7, 2)] + assert sum((b - a + 1) * n for a, b, n in sc.sheets[0].row_runs) == sc.sheets[0].nonempty + + +def test_row_runs_fall_back_when_xml_rows_out_of_order(): + wb = Workbook() + ws = wb.active + for r in (1, 2, 3): + ws.cell(r, 1, f"标签{r}") + + def swap(t: str) -> str: + rows = re.findall(r"]*>.*?", t) + assert len(rows) == 3 + return t.replace(rows[0] + rows[1] + rows[2], rows[0] + rows[2] + rows[1]) + + raw = _patch(_save(wb), "xl/worksheets/sheet1.xml", swap) + sc = xlsx_scan.scan(raw) + assert sc.sheets[0].row_runs == [(1, 3, 1)] + + +def test_row_runs_add_up_duplicate_rows(): + wb = Workbook() + ws = wb.active + ws["A1"] = "甲" + ws["A2"] = "乙" + + def dup(t: str) -> str: + row = re.search(r']*>.*?', t).group(0) + return t.replace(row, row + row.replace('r="A2"', 'r="B2"')) + + sc = xlsx_scan.scan(_patch(_save(wb), "xl/worksheets/sheet1.xml", dup)) + assert sc.sheets[0].row_runs == [(1, 1, 1), (2, 2, 2)] + + +def test_scan_json_round_trip(): + sc = xlsx_scan.scan(_mixed_book()) + data = xlsx_scan.scan_to_json(sc) + again = xlsx_scan.scan_from_json(json.loads(json.dumps(data, ensure_ascii=False))) + assert again == sc + assert data == json.loads(json.dumps(data)) # 返回值本身就是 JSON 形状（元组已写成列表） + + +def test_scan_from_json_tolerates_old_and_newer_shapes(): + sc = xlsx_scan.scan(_mixed_book()) + data = xlsx_scan.scan_to_json(sc) + for sheet in data["sheets"]: + del sheet["row_runs"] # 老暂存区没有这个字段 + sheet["future_field"] = 1 # 更新的版本多写的字段 + data["future_top"] = True + again = xlsx_scan.scan_from_json(data) + assert [s.row_runs for s in again.sheets] == [[] for _ in again.sheets] + assert [s.nonempty for s in again.sheets] == [s.nonempty for s in sc.sheets] + assert again.sheets[0].hidden_rows == sc.sheets[0].hidden_rows + + +# -------------------------------------------------------------------------- +# 网格 +# -------------------------------------------------------------------------- + + +def test_read_grid_values_and_nonempty_rule_match_scan(): + raw = _mixed_book() + sc = xlsx_scan.scan(raw) + grid = xlsx_cells.read_grid(raw, sc, "表甲") + s = sc.sheet("表甲") + assert len(grid.cells) == s.nonempty + per_row = {r: len(grid.row(r)) for r in grid.row_numbers()} + expected = {r: n for a, b, n in s.row_runs for r in range(a, b + 1)} + assert per_row == expected + assert (2, 3) not in grid.cells # 只含空白的文本 + assert grid.get(2, 4).value == 0 and grid.get(2, 5).value is False + assert grid.get(3, 2).value == dt.datetime(2026, 8, 1) + assert grid.get(3, 3).value == "#DIV/0!" + uncached = grid.get(3, 4) + assert uncached.value is None and uncached.formula == "=B9*2" and not uncached.has_cache + assert grid.bounds == (2, 2, 6, 5) + assert grid.merges == [(6, 2, 6, 4)] + assert grid.hidden_rows == {4} and grid.hidden_cols == {5} + assert grid.truncated is False + + +def test_read_grid_refuses_hidden_and_unknown_sheets(): + raw = _mixed_book() + sc = xlsx_scan.scan(raw) + with pytest.raises(UnsupportedTable): + xlsx_cells.read_grid(raw, sc, "隐藏表") + with pytest.raises(UnsupportedTable): + xlsx_cells.read_grid(raw, sc, "没有这张表") + + +def test_shared_formula_reads_expanded_text(): + raw = _shared_formula_book() + sc = xlsx_scan.scan(raw) + grid = xlsx_cells.read_grid(raw, sc, "表甲") + assert grid.get(5, 1).formula == "=SUM(A1:A4)" and grid.get(5, 1).value == 10 + assert grid.get(5, 2).formula == "=SUM(B1:B4)" and grid.get(5, 2).value == 100 + rows = list(xlsx_cells.iter_rows(raw, sc, "表甲", min_row=5, max_row=5, min_col=1, max_col=2)) + assert [(r, [(c, g.formula) for c, g in cells]) for r, cells in rows] == [ + (5, [(1, "=SUM(A1:A4)"), (2, "=SUM(B1:B4)")])] + + +def test_iter_rows_gives_only_nonempty_cells_in_column_order(): + raw = _mixed_book() + sc = xlsx_scan.scan(raw) + rows = list(xlsx_cells.iter_rows(raw, sc, "表甲", min_row=1, max_row=6, min_col=1, max_col=5)) + assert [r for r, _ in rows] == [2, 3, 4, 6] + assert [c for c, _ in rows[0][1]] == [2, 4, 5] + assert all(cells for _, cells in rows) + + +def test_grid_too_large_and_max_rows(monkeypatch): + wb = Workbook() + ws = wb.active + ws.title = "表甲" + for r in range(1, 21): + ws.cell(r, 1, f"项{r}") + ws.cell(r, 2, r) + raw = _save(wb) + sc = xlsx_scan.scan(raw) + monkeypatch.setattr(xlsx_cells, "GRID_MAX_CELLS", 30) + with pytest.raises(xlsx_cells.GridTooLarge): + xlsx_cells.read_grid(raw, sc, "表甲") + grid = xlsx_cells.read_grid(raw, sc, "表甲", max_rows=5) + assert grid.truncated and grid.row_numbers() == [1, 2, 3, 4, 5] + assert grid.bounds == (1, 1, 20, 2) # 边界仍是整张表的 + # 给了 max_rows 也不会超过格数上限：前 50 行就超了，读到上一行为止 + capped = xlsx_cells.read_grid(raw, sc, "表甲", max_rows=50) + assert capped.truncated and len(capped.cells) == 30 and capped.row_numbers() == list(range(1, 16)) + monkeypatch.setattr(xlsx_cells, "GRID_MAX_CELLS", 200_000) + whole = xlsx_cells.read_grid(raw, sc, "表甲", max_rows=50) + assert not whole.truncated and len(whole.cells) == 40 + + +def test_stale_dimension_does_not_change_the_grid(): + raw = _mixed_book() + stale = _patch(raw, "xl/worksheets/sheet1.xml", lambda t: re.sub(r'', + '', t)) + assert stale != raw + g1 = xlsx_cells.read_grid(raw, xlsx_scan.scan(raw), "表甲") + g2 = xlsx_cells.read_grid(stale, xlsx_scan.scan(stale), "表甲") + assert g1.cells == g2.cells + + +# -------------------------------------------------------------------------- +# 预览 +# -------------------------------------------------------------------------- + + +def test_preview_shape_and_kinds(): + raw = _mixed_book() + sc = xlsx_scan.scan(raw) + p = xlsx_cells.preview(xlsx_cells.read_grid(raw, sc, "表甲")) + assert set(p) == {"sheet", "bounds", "total_rows", "total_cols", "truncated", "rows", "cols", "cells", + "formulas", "merges", "hidden_rows", "hidden_cols"} + assert p["bounds"] == "B2:E6" and p["total_rows"] == 5 and p["total_cols"] == 4 and not p["truncated"] + kinds = {(r, c): (text, kind) for r, c, text, kind in p["cells"]} + assert kinds[(2, 2)] == ("分区甲", "text") + assert kinds[(2, 4)] == ("0", "number") + assert kinds[(2, 5)] == ("FALSE", "bool") + assert kinds[(3, 2)] == ("2026-08-01", "date") + assert kinds[(3, 3)] == ("#DIV/0!", "error") + assert kinds[(3, 4)] == ("", "formula_uncached") + assert kinds[(4, 2)] == ("12.5", "number") + assert p["formulas"] == {"D3": "=B9*2"} + assert p["merges"] == ["B6:D6"] and p["hidden_rows"] == [4] and p["hidden_cols"] == [5] + json.dumps(p, ensure_ascii=False) + + +def test_preview_window_truncates(): + raw = _shared_formula_book() + sc = xlsx_scan.scan(raw) + p = xlsx_cells.preview(xlsx_cells.read_grid(raw, sc, "表甲"), max_rows=2, max_cols=1) + assert p["truncated"] and p["rows"] == [1, 2] and p["cols"] == [1] + assert {(r, c) for r, c, _, _ in p["cells"]} == {(1, 1), (2, 1)} + assert p["formulas"] == {} + formula = xlsx_cells.preview(xlsx_cells.read_grid(raw, sc, "表甲")) + assert [k for _, _, _, k in formula["cells"] if k == "formula"] == ["formula", "formula"] + assert formula["formulas"] == {"A5": "=SUM(A1:A4)", "B5": "=SUM(B1:B4)"} diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 6a5647c8..096541fd 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1,3 +1,6 @@ +import type { + AiPreview, CommitOut, CurrentRecipe, QuestionAnswer, Recipe, Staging, +} from '../types' import type { Approval, AutofixResult, Conversation, ConversationDetail, ConversationTurn, CustomTool, DataSource, EvidenceGraph, EvidenceAudit, EvidenceJudgeResult, EvidenceSegmentDetail, GraphSpec, KbDocument, MemoryItem, Provider, PublishCheck, PublishLevel, ReviewResult, Run, @@ -552,6 +555,48 @@ export const api = { get(`/datasources/${id}/schema?table=${encodeURIComponent(table)}`), }, + /** + * 按配方导入（期 2）。暂存区（staging）是一次还没提交的导入：起草、回答问题、改配方、试运行都在它上面做， + * 提交（commit）成功之前不动数据源的当前版本。错误按 ApiError.code 分支（staging_closed、trial_required……） + */ + tableImports: { + /** 选文件开始一次导入（首次，或同名的简单导入切换为按配方导入）。带字节进度 */ + stage: (file: File, body: { name: string; description?: string }, opts?: UploadOptions) => { + const form = new FormData() + form.append('file', file) + form.append('name', body.name) + form.append('description', body.description ?? '') + return upload('/datasources/imports/stage', form, opts) + }, + get: (id: string) => get(`/datasources/imports/${encodeURIComponent(id)}`), + /** 已选的全部回答（服务端从起点配方按问题顺序重放）。needs_reason 的选项要带 reason */ + answers: (id: string, answers: Record) => + post(`/datasources/imports/${encodeURIComponent(id)}/answers`, { answers }), + /** 整份配方替换工作配方：有静态校验问题也照存，只是不能试运行；已选的回答清空 */ + putRecipe: (id: string, recipe: Recipe) => + put(`/datasources/imports/${encodeURIComponent(id)}/recipe`, { recipe }), + /** 将要发给模型的全文（同意框里给人看的就是这一份） */ + draftAiPreview: (id: string) => get(`/datasources/imports/${encodeURIComponent(id)}/draft-ai/preview`), + /** 用户同意后才调：preview_sha256 与服务端重算的不一致时回 409 ai_preview_stale */ + draftAi: (id: string, previewSha256: string) => + post(`/datasources/imports/${encodeURIComponent(id)}/draft-ai`, { consent: true, preview_sha256: previewSha256 }), + trial: (id: string, body: { context_inputs?: Record; signed_by?: string | null } = {}) => + post(`/datasources/imports/${encodeURIComponent(id)}/trial`, body), + commit: (id: string, body: { + trial_id: string; confirmations: string[]; acceptances: { check_id: string; reason: string }[]; signed_by?: string | null + }) => post(`/datasources/imports/${encodeURIComponent(id)}/commit`, body), + discard: (id: string) => del(`/datasources/imports/${encodeURIComponent(id)}`), + /** 上传新一期：按当前配方自动试运行，回来时已带回执和差异卡。带字节进度 */ + reupload: (sourceId: string, file: File, opts?: UploadOptions) => { + const form = new FormData() + form.append('file', file) + return upload(`/datasources/${encodeURIComponent(sourceId)}/reupload`, form, opts) + }, + /** 修改配方（不换文件）：用当前导入的原件、以当前配方为工作配方 */ + redraft: (sourceId: string) => post(`/datasources/${encodeURIComponent(sourceId)}/redraft`, {}), + recipe: (sourceId: string) => get(`/datasources/${encodeURIComponent(sourceId)}/recipe`), + }, + runs: { /** * status 可以给多个（逗号连接）；q 按工作流名模糊匹配；before 是翻页游标 diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index dd2014ef..f5f3a4c9 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -842,7 +842,9 @@ export const UPLOAD_TEXT = { // 交叉表、多块结构 shapeTitle: '该表格不是一行一条记录的明细表', shapeLead: '检测到以下结构，默认不导入：', - shapeRecipe: '这类表格需要按配方导入（指定表头、分段和合计行的处理方式），该功能将在后续版本提供。', + shapeRecipe: '这类表格可以按配方导入：指定表头、分段和合计行的处理方式，试运行核对、逐条确认后再启用。', + /** CSV 没有配方这条路（配方只认 Excel 的工作表结构） */ + shapeRecipeCsv: '按配方导入目前只支持 Excel（.xlsx）文件：可另存为 .xlsx 后再按配方导入。', shapeHeader: (row: number) => `如果表头不在第 ${row} 行，请取消后修改表头行号，再重新上传。`, /** 结构问题之外，预告的混合列：选按原样导入以后还要再选一次 */ shapeMixedLead: '另外，以下各列以数字为主，但混有非数字的值。选择按原样导入后，还需要选择这些值的处理方式：', @@ -901,3 +903,352 @@ export const RAW_STATE_LABEL: Record = { purged: '原件已清除', absent: '未保存原件', } + +/** + * 按配方导入（期 2）：向导、原始网格、建议卡片、配方面板、试运行回执、确认清单、差异卡、AI 起草。 + * 后端返回的问题、确认项、核对标题是整句人话，界面原样显示；这里只放界面自己的文字和枚举的显示名。 + * + * 配方面板里不出现「正则」一类字样：配方语言是封闭的，没有任何字段能写匹配规则 + */ +export const RECIPE_TEXT = { + // ---- 入口 + entry: '按配方导入', + entryHint: '交叉表、分段表格按配方导入：起草配方、试运行核对，逐条确认后启用', + reupload: '上传新一期', + reuploadHint: '按已确认的配方导入新一期的文件：先试运行，看过回执和差异后再启用', + redraft: '修改配方', + redraftHint: '不换文件，修改当前配方后重新试运行', + openStaging: '有未完成的导入', + openStagingHint: '继续上次未完成的导入', + // ---- 向导 + titleFirst: '按配方导入', + titleNamed: (name: string) => `按配方导入「${name}」`, + titleReupload: (name: string) => `上传新一期「${name}」`, + titleRedraft: (name: string) => `修改配方「${name}」`, + steps: { pick: '选择文件', draft: '起草配方', trial: '试运行', confirm: '确认启用', done: '完成' }, + pickReupload: '选择新一期的文件：将按当前配方试运行，回执和差异核对无误后再启用。', + nameHint: '将用作工具名的一部分；启用之前不会创建数据源', + nameLabel: '数据源名', + nameInvalid: '须以小写字母开头，只能包含小写字母、数字和下划线', + nameTakenDb: (name: string) => `「${name}」已被其他数据库用作标识，请更换名称`, + descriptionLabel: '说明（供助手参考）', + pickDrop: '拖入文件，或点击选择', + /** 与选择框的 accept 一致 */ + pickFormats: '支持 Excel（.xlsx、.xlsm）', + pickAria: '选择表格文件', + pickWrongType: (name: string) => `「${name}」不是 Excel 文件：按配方导入只支持 .xlsx、.xlsm`, + cancelUpload: '取消上传', + cancel: '取消', + close: '关闭', + staging: '正在读取表格、起草配方', + reuploading: '正在读取表格、按当前配方试运行', + loading: '正在读取这次导入', + discard: '放弃这次导入', + discardTitle: '放弃这次导入？', + discardConsequences: ['已起草的配方、回答和试运行结果都不保留', '数据源的当前版本不受影响'], + discarded: '已放弃这次导入', + closed: '这次导入已结束（过期或已放弃），请重新开始', + rawMissing: '当前导入的原始文件已清除，无法直接修改配方：请用「上传新一期」带着文件进入修改', + later: '稍后继续', + laterHint: '关闭向导，进度已保存在服务端；可从卡片上的「有未完成的导入」继续', + // ---- 起草 + trialRun: '试运行', + trialRunning: '试运行中', + trialBlocked: '配方有问题，修改后才能试运行', + trialStale: '上次的试运行已失效（之后修改过配方或录入），请重新试运行', + trialStaleConfirm: '上次的试运行已失效，不能据此启用：请回到配方重新试运行', + partial: '仅检查了前 500 行，完整检查在试运行时进行', + partialHint: '表格较大，起草时只检查了每张工作表的前 500 行；试运行会检查全部', + draftIncomplete: '规则起草未能完成', + cardsTitle: '建议', + questionsTitle: '待确认', + noCards: '没有需要说明的建议', + reasonLabel: '理由（必填）', + reasonPlaceholder: '说明为什么这样选', + reasonSubmit: '提交', + reasonRequired: '请填写理由', + answering: '正在应用', + suggested: (label: string) => `建议：${label}`, + // ---- 网格 + gridLabel: '原始表格', + legendTitle: '图例', + unclaimed: '没有去处', + hiddenCells: '隐藏的行列', + gridTruncated: (n: string, total: string) => `仅显示前 ${n} 行，共 ${total} 行`, + noGrid: '没有可显示的工作表', + // ---- AI 起草 + aiOffer: '让 AI 起草（会把表格结构发给模型）', + aiUnavailable: '未配置模型接入，无法使用 AI 起草（设置 → 模型接入）', + aiConsentTitle: '发送表格结构给模型', + aiConsentLead: (model: string) => `将把下面这段内容发送给「${model}」：工作表名称、表头和行标签等结构文字` + + '（写成文字的日期和统计期说明也会发送，如「8月1日」「统计时间范围：…」）、数字单元格和日期格式单元格的类型、' + + '合并区域、去掉了常量的公式形状、系统发现的关系。', + aiConsentNot: '不会发送数字单元格的值、日期格式单元格里的日期、像数字的文字和隐藏行列的内容；' + + '其他文字只发字数，列表数据列只发个数统计、不发内容（每行最左一格的「合计」「小计」这类标签和占位符照发）。' + + '配方需要修订时，还会把配方的问题列表发给同一个模型，其中的这些内容同样已遮掉。', + aiConsentAfter: 'AI 只起草配方，导入前仍需试运行并逐条确认。', + aiPreviewLabel: (chars: string) => `将要发送的全文（${chars} 字）`, + aiAgree: '同意并发送', + aiPreviewing: '正在准备将要发送的内容', + aiDrafting: 'AI 起草中', + /** cost 是 formatCost 的结果（「$0.012」） */ + aiDone: (tokens: string, cost: string) => `AI 起草完成，用了 ${tokens} token（约 ${cost}）`, + aiStale: '表格内容或模型设置已变化，请重新查看将要发送的内容', + // ---- 配方面板 + panel: '配方', + panelHint: '每项修改都会立即保存并重新检查；有问题的字段旁会标出原因', + panelProblems: '配方有以下问题', + viewJson: '查看配方 JSON', + hideJson: '收起配方 JSON', + pasteJson: '粘贴配方', + pasteLead: '粘贴一份完整的配方 JSON，保存后同样经过检查。', + pasteSave: '保存配方', + pasteInvalid: 'JSON 格式有误，未保存', + pasteNotObject: '配方必须是一个 JSON 对象', + saving: '正在保存', + noRecipe: '还没有配方：请回答上面的问题，或粘贴一份配方', + readonlyNote: '表单没有覆盖的字段只能通过粘贴配方修改', + sheet: (name: string) => `工作表「${name}」`, + table: (name: string) => `表「${name}」`, + crosstab: '交叉表', + listBlock: (id: string) => `列表「${id}」`, + segment: (id: string) => `分段「${id}」`, + derivedSegment: (id: string) => `合计段「${id}」`, + tableName: '表名', + columnName: '列名', + unit: '单位', + noUnit: '无单位', + grain: '主键', + grainHint: '用作主键的列（最多 4 列）', + labels: '标签', + totalLabels: '合计标签', + addLabel: '添加', + labelPlaceholder: '从表格中选择或输入', + removeLabel: (label: string) => `移除标签「${label}」`, + locateBy: '定位方式', + sectionTitle: '分段标题', + stopAtTotal: '遇到时段合计行时结束本段', + dimName: '维度列名', + dimParser: '标签写法', + valueName: '值列名', + constants: '常量列', + constPick: '取值', + deriveCols: '由时段派生的列', + deriveRole: { start: '区间起点', end: '区间终点' } as Record, + keepDim: '合计项列名', + keepAs: '原表合计另存为表', + keepTable: '另存的表名', + values: '数据区的值', + valueType: '类型', + placeholders: '占位符', + placeholderText: '占位符文字', + placeholderMeaning: '含义', + placeholderStore: '一律存为空值', + addPlaceholder: '添加占位符', + removePlaceholder: (text: string) => `移除占位符「${text}」`, + blank: '空格', + textNumber: '文本写的数字', + formula: '公式', + period: '统计期', + preferPrefix: '优先展示以此开头的格子', + crossCheck: '与文件名里的日期核对', + addPeriod: '从格子里解析统计期', + noPeriod: '未配置统计期', + axisChecks: '日期检查', + axisChecksHint: '取消任一项，确认清单里会多出一条需确认的放宽', + headerRows: '表头行数', + afterTitle: '只在这行文字之后找表头', + columns: '列', + colHeader: '表头', + colType: '类型', + colStore: '存法', + addColumn: '添加列', + removeColumn: (name: string) => `移除列「${name}」`, + extraColumns: '表头里多出的列', + blankRows: '数据中间的空行', + totalRow: '合计行', + totalRowOn: '有合计行', + totalLabelColumn: '合计标签所在的列', + totalPick: '合计行以此开头', + totalKeep: '原表合计另存为表（留空则只核对）', + hiddenRows: '隐藏行', + hiddenCols: '隐藏列', + fallback: '工作表改名时', + otherSheets: '配方没有列出的其他工作表', + relations: '关系', + relationRegister: '登记', + relationDismiss: '不登记', + relationReason: '不登记的理由（必填）', + relationTotal: '合计列', + relationParts: '组成列', + relationNoCache: '请在建议卡片的问题里选择「登记」', + notComparable: (a: string, b: string) => `「${a}」与「${b}」口径不同`, + sumEq: (total: string, parts: string) => `${total} = ${parts}`, + dismissed: (claims: string) => `不登记系统发现的关系 ${claims}`, + // ---- 试运行回执 + receipt: '试运行回执', + ledger: (n: string) => `单元格去向：非空单元格 ${n} 个`, + ledgerAll: '全部有去处', + ledgerUnclaimed: (n: string) => `其中 ${n} 个没有去处`, + /** 格子账为空：配方里的工作表一张都没读到（如工作表改名后对不上），去向和两遍读取都无从谈起 */ + ledgerNone: '未读取到配方中的工作表，无法核对单元格去向', + ledgerSheetsMissing: (n: string) => `配方中有 ${n} 张工作表未读取到，以上只计入已读取的工作表`, + ledgerUnbalanced: (sum: string, read: string) => `各去向合计 ${sum} 个，与读取到的 ${read} 个不符`, + passAgree: '两遍读取一致', + passDiffer: (scan: string, read: string) => `两遍读取不一致：第一遍 ${scan} 个、第二遍 ${read} 个`, + tables: '表与行数', + rows: (n: string) => `${n} 行`, + /** 拒收、需要录入时执行器没有写完（或根本没写库）：只列表和列，不显示行数 */ + tablesUnwritten: '试运行没有完成写入，以下只列出表和列；行数以通过后的试运行为准', + rowsUnwritten: '未写入', + periodLine: (start: string, end: string) => `统计期：${start} 至 ${end}`, + periodFromCells: (cells: string) => `取自 ${cells}`, + periodHuman: (who: string) => `人工录入（${who}）`, + checks: '核对', + notes: '说明预览', + problems: '问题', + outsideText: '区域外文字', + placeholderCounts: '占位符', + placeholderCount: (text: string, n: string) => `「${text}」${n} 格`, + statusPassed: '试运行通过：请逐条确认后启用', + statusNeedsDecision: '有核对未通过但可以接受：在确认清单里写明理由', + statusNeedsInput: '无法从表格中确定统计期：请录入本期统计期', + statusRejected: '试运行未通过：请修改配方或文件', + backToRecipe: '修改配方', + toConfirm: '下一步：逐条确认', + backToReceipt: '返回回执', + periodStart: '统计期起', + periodEnd: '统计期止', + periodSuggest: (start: string, end: string) => `文件名中的日期：${start} 至 ${end}`, + periodUseSuggest: '填入', + periodSubmit: '按此统计期试运行', + periodInvalid: '请填写两个日期，且起点不晚于终点', + // ---- 确认清单 + confirmTitle: '启用前请逐条确认', + confirmLead: '以下各项需逐条勾选；可接受的核对须写明理由。', + acceptTitle: '可接受的核对', + acceptLabel: '接受理由（必填）', + acceptPlaceholder: '说明为什么可以接受', + signLabel: '署名', + signNote: '署名（未认证）', + commitFirst: '确认并启用', + commitReupload: '启用', + committing: '正在启用', + commitBlocked: '请勾选全部必选的确认项，并为每条可接受的核对写明理由', + /** 不必勾的确认项（ConfirmItem.required=false）旁的标注 */ + confirmOptional: '可选', + missing: '这一项尚未确认', + rerunTrial: '请重新试运行', + baseChanged: '在你试运行之后，当前版本已被更新，请重新试运行', + nameTaken: '该名称已被占用：请放弃这次导入，换一个名称后重新导入', + // ---- 完成 + done: (name: string) => `已启用「${name}」`, + doneUnchanged: (seq: string) => `与第 ${seq} 次导入相同，未新建版本`, + doneReused: '内容与以前的某次导入相同，沿用了那次的数据文件', + finish: '完成', + // ---- 卡片上的配方信息 + recipeVersion: (seq: string) => `配方第 ${seq} 版`, + recipeActivated: (when: string) => `${when}${/\d$/.test(when) ? ' ' : ''}启用`, + recipeActivatedTitle: (when: string) => `当前配方于 ${when} 启用`, + recipeSigned: (who: string) => `署名「${who}」（未认证）`, + // ---- 差异卡 + diffTitle: '与上一期相比', + /** 差异卡的 kind 不在 DIFF_KIND_LABEL 里时的通用名（不露键名） */ + diffKindOther: '其他变化', + diffRequires: '需确认', + diffNone: '与上一期相比没有需要说明的差异', + sameAsImport: (seq: string) => `与第 ${seq} 次导入相同，无需重复导入`, +} as const + +/** 格子的去向（契约 Role）：网格图例、回执里的格子账 */ +export const LEDGER_ROLE_LABEL: Record = { + value: '数据值', + derived_value: '合计格', + derived_label: '合计标签', + col_header: '列表头', + row_label: '行标签', + section_title: '分段标题', + context: '统计期', + outside_text: '区域外文字', + ignored_column: '忽略的列', + hidden_excluded: '排除的隐藏行', + total_label: '合计标签', + total_value: '合计格', +} + +/** 核对结果的状态 */ +export const CHECK_STATUS_LABEL: Record = { + passed: '通过', + mismatch: '不一致', + unverifiable: '无法核对', + info: '说明', +} + +/** 问题和核对的类别：结构问题不能接受，只能改配方或文件；数据质量可以写明理由后接受 */ +export const PROBLEM_CATEGORY_LABEL: Record = { + structure: '结构问题（需修改配方或文件）', + data_quality: '数据质量（可写明理由后接受）', + confirm: '需确认', + input: '需要录入', + recipe: '配方问题', + info: '说明', +} + +/** 上传新一期的差异种类（recipe_diff 的 kind） */ +export const DIFF_KIND_LABEL: Record = { + period: '统计期', + rows: '行数', + placeholders: '占位符', + checks: '核对结果', + file_name: '文件名', + full_calc: '打开时重算', + context_text: '统计期的写法', + context_source_added: '统计期来源', + column_order: '列顺序', + label_writing: '标签写法', + header_writing: '表头写法', + canon_values: '规范写法', + row_order: '行顺序', + block_order: '各块的先后', + derived_form: '合计的形态', + axis_form: '日期表头的形态', + outside_added: '新增的区域外文字', + outside_removed: '不再出现的区域外文字', + outside_changed: '区域外文字有变化', + outside_moved: '区域外文字挪了位置', + ignored_columns: '忽略的列', + hidden: '隐藏的行列', +} + +/** + * 配方的起草方式（table_recipes.origin）：卡片上「配方第 N 版 · 规则起草」。manual 是在配方面板里改过或 + * 粘贴的规则草稿，mixed 是 AI 草稿又经人改过 + */ +export const RECIPE_ORIGIN_LABEL: Record = { + rules: '规则起草', + ai: 'AI 起草', + manual: '手工编辑', + mixed: 'AI 起草后手工修改', +} + +/** 配方面板里各取值的显示名（封闭取值，界面不露英文枚举） */ +export const RECIPE_CHOICE_LABEL = { + locateBy: { labels: '按标签', section_title: '按分段标题' } as Record, + dimParser: { hour_range: '时段（如 7-8）', text: '文字' } as Record, + valueType: { INTEGER: '整数', REAL: '小数' } as Record, + blankCross: { reject: '拒收', null: '存为空值（需确认）' } as Record, + blankList: { null: '存为空值', reject: '拒收' } as Record, + textNumber: { reject: '拒收', parse_thousands: '千分位写法按数字保存（需确认）' } as Record, + formulaCross: { reject: '拒收', accept_cached: '按保存值导入（需确认）' } as Record, + formulaList: { accept_cached: '按保存值导入', reject: '拒收' } as Record, + meaning: { 无数据: '无数据', 不适用: '不适用' } as Record, + hiddenRows: { reject_if_any: '有隐藏行时拒收', include: '照常导入', exclude: '跳过' } as Record, + hiddenCols: { reject_if_any: '有隐藏列时拒收', include: '照常导入' } as Record, + fallback: { only_visible_sheet: '用唯一有内容的可见工作表（需确认）', none: '必须同名' } as Record, + otherSheets: { confirm: '记入回执，需确认', reject: '拒收' } as Record, + extraColumns: { reject: '拒收', ignore: '忽略（需确认）' } as Record, + blankRows: { stop: '到此为止', skip: '跳过继续（需确认）' } as Record, + colType: { TEXT: '文字', INTEGER: '整数', REAL: '小数', DATE: '日期' } as Record, + colStore: { '': '默认', canonical: '规范写法', raw: '原文' } as Record, + axisChecks: { contiguous: '逐日连续', covers_context: '恰好覆盖统计期' } as Record, +} diff --git a/frontend/src/pages/DataSourcesTab.tsx b/frontend/src/pages/DataSourcesTab.tsx index 688094ac..6c8a3fb2 100644 --- a/frontend/src/pages/DataSourcesTab.tsx +++ b/frontend/src/pages/DataSourcesTab.tsx @@ -1,13 +1,13 @@ import { useEffect, useId, useMemo, useRef, useState } from 'react' import { useNavigate } from 'react-router-dom' import { - AlertTriangle, ChevronRight, Database, EyeOff, FileSpreadsheet, Info, KeyRound, Lock, Plug, Plus, RefreshCw, + AlertTriangle, ChevronRight, Database, EyeOff, FileSpreadsheet, Info, KeyRound, Layers, Lock, Plug, Plus, RefreshCw, Search, Table2, Upload, X, } from 'lucide-react' import clsx from 'clsx' import { ApiError, api, uploadDecision } from '../api/client' import type { IntrospectPreview, TableSchema, UploadProgress } from '../api/client' -import type { CurrentSnapshot, UploadDecision, UploadMixedColumn, UploadResult } from '../types' +import type { CurrentSnapshot, DataSource, UploadDecision, UploadMixedColumn, UploadResult } from '../types' import { useCatalog, useOnReconnect } from '../store/catalog' import { confirmDialog, DeleteButton, EmptyState, ErrorState, Field, HealthPill, IconButton, Modal, promptDialog, @@ -20,8 +20,12 @@ import { import { checkHealth, forgetHealth, healthFromServer, setHealth, useHealth } from '../lib/health' import type { HealthRecord } from '../lib/health' import { workflowList, workflowsMentioning } from '../lib/mentions' -import { RAW_STATE_LABEL, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT } from '../lib/terms' +import { + RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, UPLOAD_CONVERSION_LABEL, UPLOAD_SHAPE_LABEL, UPLOAD_TEXT, +} from '../lib/terms' import { useRunClock } from '../run/useRunClock' +import { ImportWizard } from './import/ImportWizard' +import type { WizardEntry } from './import/ImportWizard' // =========================================================================== // 数据源 @@ -70,6 +74,8 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { const [loadError, setLoadError] = useState(null) const [editing, setEditing] = useState(null) const [uploading, setUploading] = useState<{ name?: string } | null>(null) + /** 按配方导入的向导（首次、上传新一期、修改配方、继续未完成的导入） */ + const [wizard, setWizard] = useState(null) const [kick, setKick] = useState<{ id: string; seq: number } | null>(null) const loadSeq = useRef(0) const hasRows = useRef(false) @@ -116,6 +122,9 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { title="表格" hint="上传 Excel / CSV 文件，每个工作表会转为一张可用 SQL 查询的表，数字可计算、可溯源。同名文件重新上传会直接替换数据，工具名不变。" > + @@ -171,6 +180,9 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { onRemoved={drop} onEdit={() => setEditing(row)} onReupload={() => setUploading({ name: row.name })} + onRecipeReupload={() => setWizard({ kind: 'reupload', source: row })} + onRedraft={() => setWizard({ kind: 'redraft', source: row })} + onResume={() => row.open_staging && setWizard({ kind: 'resume', stagingId: row.open_staging.id, source: row })} kick={kick && kick.id === row.id ? kick.seq : 0} /> ))} @@ -215,6 +227,27 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { syncCatalog() if (!tables) navigate('/data/tables', { replace: true }) }} + onRecipe={(file, name, description) => { + // 交叉表决定页的「按配方导入」：带着同一个文件、名字和说明进向导，上传弹窗让位 + setUploading(null) + setWizard({ kind: 'stage', file, name, description }) + }} + /> + )} + + {wizard && ( + !isUploadedTable(r) && r.kind !== 'sqlite').map((r) => r.name)} + onClose={() => setWizard(null)} + onCommitted={(row) => { + upsert(row) + syncCatalog() + if (!tables) navigate('/data/tables', { replace: true }) + }} + // 暂存区开始、放弃、关掉：卡片上的「有未完成的导入」要跟上 + onChanged={() => void load()} /> )} @@ -224,12 +257,16 @@ export function DataSourcesTab({ view = 'databases' }: { view?: DataView }) { /** 结构同步超过这么久就提示可能过期：库表会变，Copilot 照旧表写 SQL 只会报错 */ const STALE_SCHEMA_MS = 7 * 24 * 3600_000 -function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, kick }: { +function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, onRecipeReupload, onRedraft, onResume, kick }: { row: any; meta?: any onChange: (row: any) => void onRemoved: (id: string) => void onEdit: () => void onReupload: () => void + /** 按配方导入的源：上传新一期、修改配方；有未完成的导入时继续它 */ + onRecipeReupload: () => void + onRedraft: () => void + onResume: () => void /** 非 0 时自动点一次「探查结构」（新建后 toast 上的按钮） */ kick: number }) { @@ -433,7 +470,19 @@ function SourceCard({ row, meta, onChange, onRemoved, onEdit, onReupload, kick } : <> 探查结构} )} - {uploaded + {row.open_staging && ( + + )} + {uploaded && row.import_mode === 'recipe' ? ( + // 按配方导入的源：每月按已确认的配方重放，不走简单上传（服务端也会拒绝） + <> + + + + ) : uploaded ? : } void remove()} /> @@ -579,6 +628,7 @@ function Address({ row, meta, uploaded }: { row: any; meta?: any; uploaded: bool <> 上传的表格{row.table_count ? ` · ${row.table_count} 张表` : ''} {snap && } + {row.import_mode === 'recipe' && row.current_recipe && } ) } @@ -617,6 +667,23 @@ function UploadVersion({ snap }: { snap: CurrentSnapshot }) { ) } +/** + * 按配方导入的源当前用的是哪一版配方：第几版、怎么起草的、什么时候启用、谁署的名（署名未认证）。 + * 每月按配方重放，配方换了版本，引用它的报告口径可能跟着变，所以和当前版本写在同一处 + */ +function RecipeVersion({ recipe }: { recipe: NonNullable }) { + const at = parseServerTime(recipe.activated_at) + return ( + + {RECIPE_TEXT.recipeVersion(formatNumber(recipe.seq))} + · {RECIPE_ORIGIN_LABEL[recipe.origin] ?? recipe.origin} + {at && · {RECIPE_TEXT.recipeActivated(formatRelative(recipe.activated_at))}} + {recipe.signed_by && · {RECIPE_TEXT.recipeSigned(recipe.signed_by)}} + + ) +} + /** 一次最多画这么多行：上千个对象的库全画出来会卡，过滤一下就够用了 */ const SCHEMA_ROWS = 300 @@ -1552,12 +1619,14 @@ const quoteList = (values: unknown[], max = 3) => values.slice(0, max).map((v) = * 只有后端给的两种（数字列混入非数字、交叉表或多块结构）。选「取消」回到表单：文件、名字、 * 表头行号都还在——表头行号设错是这两种问题最常见的来由，改一下行号可能就不用选了 */ -function UploadDecisionBody({ decision, choices, headerRow, sheetNoun }: { +function UploadDecisionBody({ decision, choices, headerRow, sheetNoun, onRecipe }: { decision: UploadDecision choices: UploadChoices headerRow: number /** 「工作表」；CSV 没有工作表，是「文件」 */ sheetNoun: string + /** 按配方导入（CSV 没有这条路：配方只认 Excel） */ + onRecipe?: () => void }) { if (decision.kind === 'mixed') { return ( @@ -1601,7 +1670,14 @@ function UploadDecisionBody({ decision, choices, headerRow, sheetNoun }: { )} )} -

{UPLOAD_TEXT.shapeRecipe}

+
+

{onRecipe ? UPLOAD_TEXT.shapeRecipe : UPLOAD_TEXT.shapeRecipeCsv}

+ {onRecipe && ( + + )} +

{UPLOAD_TEXT.shapeHeader(headerRow)}

@@ -1762,7 +1838,7 @@ function UploadNotes({ result }: { result: UploadResult }) { * 选（UploadDecisionBody），选完带着答案重传。答案跟着这一份文件和表头行号走，换文件、 * 改行号就清掉，重新问 */ -function TableUploader({ initialName, taken, sqliteTaken, onClose, onImported }: { +function TableUploader({ initialName, taken, sqliteTaken, onClose, onImported, onRecipe }: { initialName?: string /** 已经被数据库占用的名字（传表格只能就地替换表格，不能顶掉数据库） */ taken: string[] @@ -1773,6 +1849,8 @@ function TableUploader({ initialName, taken, sqliteTaken, onClose, onImported }: sqliteTaken: string[] onClose: () => void onImported: (row: any) => void + /** 交叉表决定页上选「按配方导入」：带着这份文件、名字和说明交给向导 */ + onRecipe: (file: File, name: string, description: string) => void }) { const [file, setFile] = useState(null) const [name, setName] = useState(initialName ?? '') @@ -1917,7 +1995,8 @@ function TableUploader({ initialName, taken, sqliteTaken, onClose, onImported }: )} > - + onRecipe(file, name, description) : undefined} /> ) } diff --git a/frontend/src/pages/import/AiDraftDialog.tsx b/frontend/src/pages/import/AiDraftDialog.tsx new file mode 100644 index 00000000..6f4519dc --- /dev/null +++ b/frontend/src/pages/import/AiDraftDialog.tsx @@ -0,0 +1,136 @@ +import { useState } from 'react' +import { AlertTriangle, Sparkles } from 'lucide-react' +import { ApiError, api } from '../../api/client' +import type { AiPreview, Staging } from '../../types' +import { Modal, Spinner } from '../../components/ui' +import { errorMessage } from '../../lib/errors' +import { formatCost, formatNumber } from '../../lib/format' +import { RECIPE_TEXT } from '../../lib/terms' + +// =========================================================================== +// AI 兜底起草：规则起草不完整时，经用户同意把表格结构（不含数字的值）发给模型起草配方。 +// 先取「将要发送的全文」给人看，同意才发；发的时候带上那份全文的 sha256，服务端重算对不上就拒绝 +// =========================================================================== + +/** + * 卡片区顶部的 AI 入口和结果。只在服务端说 offered 时出现（首次导入、规则草稿不完整）； + * 模型接入不可用时按钮置灰、写出原因。staging_closed 之类要关掉向导的错误交给 onError + */ +export function AiDraftOffer({ staging, busy, setBusy, onStaging, onError }: { + staging: Staging + busy: boolean + setBusy: (v: boolean) => void + onStaging: (s: Staging) => void + /** 返回 true 表示已经处理（比如这次导入已结束、向导关了），这里不再显示 */ + onError: (e: unknown) => boolean +}) { + const [phase, setPhase] = useState<'idle' | 'previewing' | 'consent' | 'drafting'>('idle') + const [preview, setPreview] = useState(null) + const [error, setError] = useState<{ message: string; next?: string } | null>(null) + const ai = staging.ai + const result = staging.ai_draft + + const fail = (e: unknown) => { + if (onError(e)) return + const code = e instanceof ApiError ? e.code : undefined + // 服务端的原话照写。ai_failed、ai_too_large 的原话里已经带了下一步（有没有工作配方，能做的事不一样： + // 没有时配方面板里只能粘贴一份），界面不再另补；过期补一句重新查看 + if (code === 'ai_preview_stale') setError({ message: errorMessage(e), next: RECIPE_TEXT.aiStale }) + else setError({ message: errorMessage(e) }) + } + + const open = async () => { + setError(null) + setPhase('previewing') + setBusy(true) + try { + setPreview(await api.tableImports.draftAiPreview(staging.id)) + setPhase('consent') + } catch (e) { + setPhase('idle') + fail(e) + } finally { + setBusy(false) + } + } + + const send = async () => { + if (!preview) return + setPhase('drafting') + setBusy(true) + try { + onStaging(await api.tableImports.draftAi(staging.id, preview.sha256)) + setPreview(null) + setPhase('idle') + } catch (e) { + setPreview(null) + setPhase('idle') + fail(e) + } finally { + setBusy(false) + } + } + + const usage = result && result.draft && result.total_tokens != null + ? RECIPE_TEXT.aiDone(formatNumber(result.total_tokens), formatCost(result.cost_usd ?? 0)) + : null + + if (!ai?.offered && !usage) return null + return ( +
+ {ai?.offered && ( +
+ + {!ai.available && ( + + {ai.reason || RECIPE_TEXT.aiUnavailable} + + )} +
+ )} + {usage &&

{usage}

} + {error && ( +
+ + {error.message}{error.next && （{error.next}）} +
+ )} + {phase === 'consent' && preview && ( + { setPreview(null); setPhase('idle') }} onAgree={() => void send()} /> + )} +
+ ) +} + +/** 同意框：说清发什么、不发什么，下面是将要发送的全文（等宽、可滚动）。取消不发任何请求 */ +function AiConsentDialog({ preview, model, busy, onCancel, onAgree }: { + preview: AiPreview; model: string; busy: boolean; onCancel: () => void; onAgree: () => void +}) { + return ( + + + + + }> +
+

{RECIPE_TEXT.aiConsentLead(model)}

+

{RECIPE_TEXT.aiConsentNot}

+

{RECIPE_TEXT.aiConsentAfter}

+
{RECIPE_TEXT.aiPreviewLabel(formatNumber(preview.chars ?? preview.text.length))}
+ 
{preview.text}
+
+ + ) +} diff --git a/frontend/src/pages/import/ConfirmList.tsx b/frontend/src/pages/import/ConfirmList.tsx new file mode 100644 index 00000000..0c100773 --- /dev/null +++ b/frontend/src/pages/import/ConfirmList.tsx @@ -0,0 +1,150 @@ +import { useEffect, useId, useMemo, useState } from 'react' +import { AlertTriangle } from 'lucide-react' +import clsx from 'clsx' +import type { CheckResult, ConfirmItem } from '../../types' +import { Spinner } from '../../components/ui' +import { localActor } from '../../lib/actor' +import { PROBLEM_CATEGORY_LABEL, RECIPE_TEXT } from '../../lib/terms' +import { CheckStatusChip } from './ImportReceipt' + +// =========================================================================== +// 启用前的确认清单：逐条勾选（不提供「全选」），可接受的核对逐条写理由，署名（未认证）。 +// 必勾的集合由服务端按试运行结果重算，少勾一项服务端照样拒绝（422 confirm_required） +// =========================================================================== + +/** 单位变化排最前（数量级变了，引用它的报告口径会变），其次是破坏性变更和差异卡里需确认的项 */ +function rank(id: string): number { + if (id.startsWith('unit_changed:')) return 0 + if (id.startsWith('breaking:') || id === 'switch_from_simple') return 1 + if (id.startsWith('diff:')) return 2 + return 3 +} + +export interface CommitBody { + confirmations: string[] + acceptances: { check_id: string; reason: string }[] + signed_by: string | null +} + +export function ConfirmList({ items, checks, acceptable, reupload, busy, missing, error, onSubmit }: { + items: ConfirmItem[] + checks: CheckResult[] + /** 可写理由接受、本次未通过的核对 id */ + acceptable: string[] + /** 上传新一期：按钮写「启用」 */ + reupload: boolean + busy: boolean + /** 服务端说还没勾的项（422 confirm_required / acceptance_required 的 detail 里点名的） */ + missing: Set + error: string | null + onSubmit: (body: CommitBody) => void +}) { + const sorted = useMemo(() => items.map((it, i) => ({ it, i })) + .sort((a, b) => rank(a.it.id) - rank(b.it.id) || a.i - b.i).map((x) => x.it), [items]) + const [checked, setChecked] = useState>(() => new Set()) + const [reasons, setReasons] = useState>({}) + const [signer, setSigner] = useState(() => localActor() ?? '') + const signId = useId() + // 换了一次试运行（确认项变了）：勾过的里面只留还在的 + useEffect(() => { + setChecked((cur) => new Set([...cur].filter((id) => items.some((it) => it.id === id)))) + }, [items]) + + const required = sorted.filter((it) => it.required !== false) + const accepts = acceptable.map((id) => ({ id, check: checks.find((c) => c.id === id) })) + const allChecked = required.every((it) => checked.has(it.id)) + const allReasons = accepts.every(({ id }) => { + const v = (reasons[id] ?? '').trim() + return v.length > 0 && v.length <= 500 + }) + const ready = allChecked && allReasons && !busy + + const submit = () => { + if (!ready) return + onSubmit({ + confirmations: sorted.filter((it) => checked.has(it.id)).map((it) => it.id), + acceptances: accepts.map(({ id }) => ({ check_id: id, reason: reasons[id].trim() })), + signed_by: signer.trim() || null, + }) + } + + return ( +
+
+

{RECIPE_TEXT.confirmTitle}

+

{RECIPE_TEXT.confirmLead}

+
+ {error && ( +
+ + {error} +
+ )} +
    + {sorted.map((it) => { + const miss = missing.has(it.id) + return ( +
  • + +
    • + ) + })} +
+ {accepts.length > 0 && ( +
+

{RECIPE_TEXT.acceptTitle}

+ {accepts.map(({ id, check }) => { + const fieldId = `${signId}-accept-${id}` + const miss = missing.has(id) + return ( +
+
+ {check && } + {id} + {check?.title ?? id} +
+ {check &&
{PROBLEM_CATEGORY_LABEL[check.category] ?? ''}
} + +