Files
AgentLab/backend/tests/test_recipe_acceptance_p3.py

2009 lines
115 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""期 3 集成验收(WP-8,P3-SPEC 11.2、11.4、12.9):真管线(default_pipeline)经接口走完整流程。
不用假 Pipeline:暂存、规则起草、回答问题、修复按钮、框选、试运行、累积计划、物化、确认项、提交、版本页的启用 /
移除 / 作废 / 清除,全部是各工作包合并后的真代码;接口一律经 AsyncClient(transport=ASGITransport(app=app)),从暂存
开始,不直接调内部函数拼结果。只有接口不提供的观察点(快照记录、导入记录、构建记录、运行记录、文件)才直接读库、
看文件;造「数据目录被人改过」这类状态时才直接改库或改文件(11.4 的数据变异)。
公共做法(11.2):D00 8 月暂存,回答问题(系统发现一律登记、占位符存为空值),按期 2 验收的写法把宽表改名「日客流」,
q_mode 选「按期累积」,提交得到 S1。之后各例按「上传新一期」重放。
夹具全部合成、假名(tests/fixtures/xlsx:分区甲、分区乙、分区丙……,数字随机);不调用任何模型(期 3 没有新的模型
调用,规则起草不完整时的 AI 可用性查询换成「不可用」的桩)。断言照规格写,不放宽、不标 xfail。
"""
from __future__ import annotations
import json
import os
import re
import sqlite3
import subprocess
import sys
import time
import uuid
from contextlib import closing
from pathlib import Path
from typing import Any
import pytest
from httpx import ASGITransport, AsyncClient
from openpyxl import Workbook
from sqlalchemy import select, update
from app.core import artifact_store
from app.core.config import settings
from app.data import raw_store, table_versions
from app.db.base import SessionLocal
from app.db.models import (
DataSource, ImportStaging, Run, SnapshotActivation, SourceSnapshot, TableBuild, TableImport, TableRecipe,
)
from app.main import app
from tests.fixtures.xlsx import excel_saved, lab, save
from tests.fixtures.xlsx.drift import AUG, DRIFT_CASES, OCT, SEP
from tests.fixtures.xlsx.flow import SHEET, flow_workbook
XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
API = "/api/datasources"
TABLES = ("日客流", "时段客流", "时段客流_表内合计")
#: 2.12 的验收数字:8 月 31 天、9 月 30 天,并集 61 / 1037 / 183
AUG_ROWS = {"日客流": 31, "时段客流": 527, "时段客流_表内合计": 93}
SEP_ROWS = {"日客流": 30, "时段客流": 510, "时段客流_表内合计": 90}
UNION_ROWS = {"日客流": 61, "时段客流": 1037, "时段客流_表内合计": 183}
REASON = "合成理由:验收用例"
# ==========================================================================
# 夹具
# ==========================================================================
@pytest.fixture(autouse=True)
def store(monkeypatch, tmp_path):
"""每个测试一个独立的数据目录:原件存档、构建库、试运行库、并集、隔离区、工件互不串。"""
data = tmp_path / "data"
(data / "uploads").mkdir(parents=True)
monkeypatch.setattr(settings, "data_dir", data)
yield data
@pytest.fixture(autouse=True)
def real_pipeline_no_model(monkeypatch):
"""验收必须跑真管线:recipe_imports.pipeline() 接的是各工作包的真函数(含期 3 新增的累积、物化、修复、框选),
不是别的测试留下的假实现。规则起草不完整时 StagingOut 会查 AI 可用性:换成「不可用」的桩,连模型对象都不建。"""
from app.data import (
recipe_accumulate, recipe_ai, recipe_engine, recipe_fixes, recipe_imports, recipe_notes, recipe_select,
recipe_suggest,
)
from app.data.recipe_types import AiAvailability
p = recipe_imports.pipeline()
assert p.execute is recipe_engine.execute and p.draft is recipe_suggest.draft
assert p.ai_availability is recipe_ai.ai_availability
assert p.plan_accumulate is recipe_accumulate.plan_accumulate
assert p.materialize_union is recipe_accumulate.materialize_union
assert p.build_union_notes is recipe_notes.build_union_notes
assert p.propose_fixes is recipe_fixes.propose_fixes and p.selection_edit is recipe_select.selection_edit
async def no_model(session):
return None, "", AiAvailability(False, reason="验收测试不接模型")
monkeypatch.setattr(recipe_ai, "resolve_draft_model", no_model)
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
# ==========================================================================
# 小工具(经 HTTP)
# ==========================================================================
def unique(prefix: str = "wp8") -> str:
return f"{prefix}_{uuid.uuid4().hex[:10]}"
async def db_get(model, key):
async with SessionLocal() as session:
return await session.get(model, key)
async def stage(client, raw: bytes, filename: str, name: str | None = None) -> dict[str, Any]:
resp = await client.post(f"{API}/imports/stage", files={"file": (filename, raw, XLSX)},
data={"name": name or unique()})
assert resp.status_code == 201, resp.text
return resp.json()
async def reupload(client, source_id: str, raw: bytes, filename: str) -> dict[str, Any]:
resp = await client.post(f"{API}/{source_id}/reupload", files={"file": (filename, raw, XLSX)})
assert resp.status_code == 201, resp.text
return resp.json()
async def trial(client, sid: str, **body: Any) -> dict[str, Any]:
resp = await client.post(f"{API}/imports/{sid}/trial", json=body)
assert resp.status_code == 200, resp.text
return resp.json()
async def answer(client, sid: str, answers: dict[str, dict[str, Any]]) -> dict[str, Any]:
resp = await client.post(f"{API}/imports/{sid}/answers", json={"answers": answers})
assert resp.status_code == 200, resp.text
return resp.json()
async def put_recipe(client, sid: str, rec: dict[str, Any], **extra: Any) -> dict[str, Any]:
resp = await client.put(f"{API}/imports/{sid}/recipe", json={"recipe": rec, **extra})
assert resp.status_code == 200, resp.text
return resp.json()
def confirm_ids(st: dict[str, Any]) -> list[str]:
return [i["id"] for i in (st.get("trial") or {}).get("confirm_items") or []]
async def commit(client, st: dict[str, Any], *, confirmations: list[str] | None = None,
acceptances: list[dict[str, str]] | None = None):
"""提交;返回原始响应(调用方自己判状态码)。confirmations 缺省 = 勾上全部确认项。"""
body: dict[str, Any] = {"trial_id": st["trial"]["trial_id"],
"confirmations": confirm_ids(st) if confirmations is None else confirmations}
if acceptances is not None:
body["acceptances"] = acceptances
return await client.post(f"{API}/imports/{st['id']}/commit", json=body)
def rows_of(st: dict[str, Any]) -> dict[str, int]:
return {t["name"]: t["rows"] for t in st["trial"]["receipt"]["tables"]}
def problems(st: dict[str, Any], code: str | None = None) -> list[dict[str, Any]]:
return [p for p in (st.get("trial") or {}).get("problems") or [] if code is None or p["code"] == code]
def check(st: dict[str, Any], cid: str) -> dict[str, Any] | None:
return next((c for c in st["trial"]["checks"] if c["id"] == cid), None)
def all_comments(notes: dict[str, Any] | None) -> str:
out = []
for n in (notes or {}).values():
out.append(n.get("comment") or "")
out += list((n.get("columns") or {}).values())
return "\n".join(out)
def rename_wide(rec: dict[str, Any], old: str, new: str) -> dict[str, Any]:
"""把规则起草的宽表名改成 new:表名、写入它的分段(id 一起改)、关系里引用的表名(期 2 验收 9.7 第 1 步)。"""
rec = json.loads(json.dumps(rec, ensure_ascii=False))
for sheet in rec["sheets"]:
for block in sheet["blocks"]:
for seg in block.get("segments", []):
if seg.get("table") == old:
seg["table"] = new
if seg.get("id") == old:
seg["id"] = new
for t in rec["tables"]:
if t["name"] == old:
t["name"] = new
for r in rec["relations"]:
if r.get("table") == old:
r["table"] = new
for side in ("a", "b"):
if isinstance(r.get(side), dict) and r[side].get("table") == old:
r[side]["table"] = new
return rec
def answers_for(questions: list[dict[str, Any]], mode: str) -> dict[str, dict[str, Any]]:
"""系统发现一律「登记」,q_mode 按 mode,其余能存为空值的选「存为空值」(参考配方的写法)。"""
out = {}
for q in questions:
values = [o["value"] for o in q["options"]]
if q["id"].startswith("q_relation"):
out[q["id"]] = {"value": "register"}
elif q["id"] == "q_mode":
out[q["id"]] = {"value": mode}
elif "null" in values:
out[q["id"]] = {"value": "null"}
else:
out[q["id"]] = {"value": values[0]}
return out
async def first_import(client, *, mode: str = "accumulate", seed: int = 0, start=AUG, days: int = 31,
variant: str = "D00", name: str | None = None, raw: bytes | None = None,
edit: Any = None) -> dict[str, Any]:
"""公共做法:暂存 → 回答问题(q_mode=mode)→ 宽表改名「日客流」→ 试运行 → 勾全部确认项提交,得到 S1。
raw 给了就用它(文件名照 flow_workbook 的写法);edit 给了就在改名之后再改一次配方(A18 的列改名)。"""
name = name or unique()
made, fn = flow_workbook(start, days, seed=seed, variant=variant)
raw = raw if raw is not None else made
st = await stage(client, raw, fn, name)
assert st["questions"][-1]["id"] == "q_mode", "q_mode 固定排在问题列表最后(2.2)"
st = await answer(client, st["id"], answers_for(st["questions"], mode))
wide = next(t["name"] for t in st["recipe"]["tables"] if t["name"] not in TABLES)
rec = rename_wide(st["recipe"], wide, "日客流")
st = await put_recipe(client, st["id"], edit(rec) if edit is not None else rec)
st = await trial(client, st["id"])
assert st["trial"]["status"] == "passed", st["trial"]["problems"]
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
out = resp.json()
return {"name": name, "source_id": out["source"]["id"], "s1": out["snapshot_id"], "import_id": out["import_id"],
"recipe_id": out["recipe_id"], "commit": out, "trial": st, "raw": raw, "file_name": fn}
async def add_period(client, source_id: str, raw: bytes, fn: str, *,
acceptances: list[dict[str, str]] | None = None) -> dict[str, Any]:
"""上传新一期 → 试运行通过 → 勾全部确认项提交。返回提交响应。"""
st = await reupload(client, source_id, raw, fn)
assert st["trial"]["status"] in ("passed", "needs_decision"), st["trial"]["problems"]
resp = await commit(client, st, acceptances=acceptances)
assert resp.status_code == 201, resp.text
return resp.json()
async def august_september(client, *, seed: int = 0, name: str | None = None) -> dict[str, Any]:
"""S1=[8 月]、S2=[8 月, 9 月],同一个配方(按期累积)。"""
base = await first_import(client, seed=seed, name=name)
raw, fn = flow_workbook(SEP, 30, seed=seed + 1)
out = await add_period(client, base["source_id"], raw, fn)
return {**base, "s2": out["snapshot_id"], "aug": base["import_id"], "sep": out["import_id"], "sep_commit": out,
"sep_raw": raw, "sep_file": fn}
async def tool_rows(client, source_name: str, sql: str) -> list[list[Any]]:
"""经工具接口查当前版本(模型看到的那一份)。"""
resp = await client.post(f"/api/tools/db_query__{source_name}/run", json={"args": {"sql": sql}})
assert resp.status_code == 200, resp.text
out = resp.json()
assert out["ok"], out
return json.loads(out["result"])["rows"]
async def count(client, name: str, table: str, where: str = "") -> int:
return int((await tool_rows(client, name, f'SELECT COUNT(*) FROM "{table}"{where}'))[0][0])
async def by_month(client, name: str, table: str, where: str = "") -> list[list[Any]]:
return await tool_rows(client, name, f'SELECT substr("日期", 1, 7), COUNT(*) FROM "{table}"{where} '
"GROUP BY 1 ORDER BY 1")
async def current_id(source_id: str) -> str | None:
return (await db_get(DataSource, source_id)).current_snapshot_id
async def current_periods(source_id: str) -> list[str]:
snap = await db_get(SourceSnapshot, await current_id(source_id))
return [(await db_get(TableImport, i)).period_start for i in snap.imports]
async def snapshots(client, source_id: str) -> list[dict[str, Any]]:
resp = await client.get(f"{API}/{source_id}/snapshots")
assert resp.status_code == 200, resp.text
return resp.json()
async def imports_api(client, source_id: str) -> list[dict[str, Any]]:
resp = await client.get(f"{API}/{source_id}/imports")
assert resp.status_code == 200, resp.text
return resp.json()
def date_of(y: int, m: int, d: int):
import datetime as dt
return dt.date(y, m, d)
def sql_file(path: str | Path, query: str, params: tuple[Any, ...] = ()) -> list[tuple[Any, ...]]:
"""只读打开库文件查一句(不改文件,库哈希不受影响)。"""
with closing(sqlite3.connect(f"file:{path}?mode=ro", uri=True)) as conn:
return list(conn.execute(query, params))
def columns_of(snap: SourceSnapshot, table: str) -> list[str]:
return [c["name"] for c in snap.schema_cache["tables"][table]["columns"]]
# --- 改造夹具(保留公式的缓存值) ---
def edit_cells(raw: bytes, edits: dict[str, Any], sheet: str = SHEET) -> bytes:
"""在合成夹具上改几格(值为 None 即清空),其余照旧。openpyxl 存盘会丢掉公式的缓存值,这里先按原文件读出缓存值,
存盘后再用 excel_saved 填回去:表内合计的公式照样有缓存,核对不受影响。"""
import io
from openpyxl import load_workbook
book = load_workbook(io.BytesIO(raw))
values = load_workbook(io.BytesIO(raw), data_only=True)
ws, wv = book[sheet], values[sheet]
cached = {c.coordinate: wv[c.coordinate].value for row in ws.iter_rows() for c in row
if isinstance(c.value, str) and c.value.startswith("=")}
for ref, value in edits.items():
ws[ref] = value
return excel_saved(save(book), sheet, cached)
def without_zone_b(raw: bytes) -> bytes:
"""这一期的文件不再有「分区乙(人次)」那一行(第 7 行整行清空):原表不再登记分区乙。"""
from openpyxl.utils import get_column_letter
return edit_cells(raw, {f"{get_column_letter(c)}7": None for c in range(2, 40)})
# --- 修复按钮、框选 ---
def fix_of(st: dict[str, Any], kind: str) -> dict[str, Any]:
found = [f for f in st["fixes"] if f["kind"] == kind]
assert found, (kind, [f["kind"] for f in st["fixes"]])
return found[0]
async def preview(client, sid: str, body: dict[str, Any]):
return await client.post(f"{API}/imports/{sid}/edits/preview", json=body)
async def apply(client, sid: str, body: dict[str, Any], expected: str | None, **extra: Any):
return await client.post(f"{API}/imports/{sid}/edits/apply", json={**body, "expected_sha256": expected, **extra})
async def fixed(client, st: dict[str, Any], kind: str, option: str | None = None,
reason: str | None = None) -> tuple[dict[str, Any], dict[str, Any]]:
"""按提议预览再应用(3.1:预览 → 应用,客户端只发 fix_id、选项值和理由)。返回 (应用后的 StagingOut, 预览)。"""
fx = fix_of(st, kind)
opt = next(o for o in fx["options"] if option is None or o["value"] == option)
if opt["needs_reason"] and reason is None:
reason = REASON
body = {"fix": {"id": fx["id"], "option": opt["value"], **({"reason": reason} if reason else {})}}
resp = await preview(client, st["id"], body)
assert resp.status_code == 200, resp.text
pv = resp.json()
assert pv["ok"], pv["problems"]
resp = await apply(client, st["id"], body, pv["recipe_sha256_after"])
assert resp.status_code == 200, resp.text
return resp.json(), pv
# ==========================================================================
# A1–A3:按期累积、重叠拒收、替换该期
# ==========================================================================
async def test_a1_august_plus_september_accumulates_to_61_1037_183(client):
"""A1:reupload D01 → 试运行 passed,accumulate.action=append,union.rows={61, 1037, 183},U1–U3 passed →
提交 201、parts=2 → 当前快照 imports=[8 月, 9 月]、库在 snapshots/ 下,三表 61 / 1037 / 183,按月 31 / 30。"""
base = await first_import(client, seed=11)
sid, name = base["source_id"], base["name"]
raw, fn = flow_workbook(SEP, 30, seed=12)
st = await reupload(client, sid, raw, fn)
t = st["trial"]
assert st["kind"] == "reupload" and t["status"] == "passed", t["problems"]
assert t["accumulate"]["action"] == "append" and t["accumulate"]["mode"] == "accumulate"
assert t["accumulate"]["union"]["rows"] == UNION_ROWS
assert rows_of(st) == SEP_ROWS, "本期的回执是 9 月这一期"
assert [(c["id"], c["kind"], c["status"]) for c in t["union_checks"]] == [
("U1", "union_rows", "passed"), ("U2", "union_pk", "passed"), ("U3", "union_period", "passed")]
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
out = resp.json()
assert out["parts"] == 2 and out["unchanged"] is False
snap = await db_get(SourceSnapshot, out["snapshot_id"])
imps = [await db_get(TableImport, i) for i in snap.imports]
assert snap.imports == [base["import_id"], out["import_id"]]
assert [(i.period_start, i.period_end) for i in imps] == [("2026-08-01", "2026-08-31"), ("2026-09-01", "2026-09-30")]
assert Path(snap.db_path).parent.name == "snapshots" and snap.mode == "accumulate"
for table, n in UNION_ROWS.items():
assert await count(client, name, table) == n
assert sql_file(snap.db_path, f'SELECT COUNT(*) FROM "{table}"')[0][0] == n
assert await by_month(client, name, "日客流") == [["2026-08", 31], ["2026-09", 30]]
async def test_a2_partially_overlapping_period_is_rejected(client):
"""A2:S2 之后 reupload flow_workbook(2026-08-15, 31) → 试运行 rejected,problems 有 period_overlap,消息里有两段
已有的统计期;commit 409 trial_not_passed;当前快照不变。"""
v = await august_september(client, seed=21)
raw, fn = flow_workbook(AUG.replace(day=15), 31, seed=23)
st = await reupload(client, v["source_id"], raw, fn)
t = st["trial"]
assert t["status"] == "rejected"
[overlap] = problems(st, "period_overlap")
assert overlap["category"] == "structure" and overlap["fix"] is None
assert "2026-08-15 至 2026-09-14" in overlap["message"]
assert "2026-08-01 至 2026-08-31" in overlap["message"] and "2026-09-01 至 2026-09-30" in overlap["message"]
resp = await commit(client, st)
assert resp.status_code == 409 and resp.json()["code"] == "trial_not_passed", resp.text
assert await current_id(v["source_id"]) == v["s2"]
async def test_a3_replacing_a_period_needs_confirmation(client):
"""A3:reupload 9 月 seed 不同(数字不同、统计期相同)→ passed、action=replace_period;不勾
period_replace:2026-09-01~2026-09-30 提交 422 confirm_required(detail 里有这个 id);勾上 201 → 新快照
imports=[8 月, 9 月新],行数仍是 61 / 1037 / 183,9 月的值与原来的不同。"""
v = await august_september(client, seed=31)
sid, name = v["source_id"], v["name"]
probe = 'SELECT "全日客流" FROM "日客流" WHERE "日期" = \'2026-09-05\''
old = (await tool_rows(client, name, probe))[0][0]
raw, fn = flow_workbook(SEP, 30, seed=99)
st = await reupload(client, sid, raw, fn)
t = st["trial"]
assert t["status"] == "passed", t["problems"]
assert t["accumulate"]["action"] == "replace_period"
item = "period_replace:2026-09-01~2026-09-30"
assert item in confirm_ids(st)
resp = await commit(client, st, confirmations=[i for i in confirm_ids(st) if i != item])
assert resp.status_code == 422 and resp.json()["code"] == "confirm_required", resp.text
assert item in resp.json()["detail"]
assert await current_id(sid) == v["s2"]
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
out = resp.json()
snap = await db_get(SourceSnapshot, out["snapshot_id"])
assert snap.imports == [v["aug"], out["import_id"]] and out["import_id"] != v["sep"]
for table, n in UNION_ROWS.items():
assert await count(client, name, table) == n
assert (await tool_rows(client, name, probe))[0][0] != old
assert (await db_get(TableImport, v["sep"])).status == "superseded"
# 回到替换之前的版本:被替换掉的旧 9 月重新是 active,新 9 月置 superseded,数字回到原来的
resp = await client.post(f"{API}/{sid}/snapshots/{v['s2']}/activate",
json={"confirm": True, "expected_current_snapshot_id": out["snapshot_id"]})
assert resp.status_code == 200, resp.text
assert (await db_get(TableImport, v["sep"])).status == "active"
assert (await db_get(TableImport, out["import_id"])).status == "superseded"
assert (await tool_rows(client, name, probe))[0][0] == old
# ==========================================================================
# A4:固定在 8 月快照上的运行不受影响;被运行引用的并集快照不回收
# ==========================================================================
async def keep_run(source_id: str, name: str, snapshot: str) -> str:
"""登记一条引用这个快照的中断运行(写法同 test_run_data_versions.keep)。返回运行 id。"""
async with SessionLocal() as session:
run = Run(workflow_name="验收:保留旧版本", status="interrupted",
data_versions={source_id: {"snapshot": snapshot, "name": name}})
session.add(run)
await session.commit()
return run.id
async def tools_for(names: list[str], *, data_versions: Any = None) -> dict[str, Any]:
from app.tools.datasource import build_datasource_tools
from app.tools.registry import ToolContext, Versions
async with SessionLocal() as session:
built = await build_datasource_tools(
names, ToolContext(run_id="", node_id="n",
data_versions=Versions.CURRENT if data_versions is None else data_versions), session)
return {t.name: t for t in built}
async def restart_with_unit_change(client, source_id: str, start=OCT, days: int = 31, seed: int = 0) -> dict[str, Any]:
"""这一期的文件把「分区甲(人次)」写成「分区甲(人)」,在这一期的暂存区里用 PUT 把配方的标签和单位一起改掉
(单位由标签写死,unit_label_conflict 不许单改配方里的单位):单位变了是不兼容变化 → restart。返回暂存区。"""
raw, fn = flow_workbook(start, days, seed=seed)
st = await reupload(client, source_id, edit_cells(raw, {"B6": "分区甲(人)"}), fn)
assert st["trial"]["status"] == "rejected", "现行配方认不出改了单位的标签"
rec = json.loads(json.dumps(st["recipe"], ensure_ascii=False))
seg = next(s for s in rec["sheets"][0]["blocks"][0]["segments"] if s["id"] == "日客流")
seg["labels"]["expect"] = ["分区甲(人)" if x == "分区甲(人次)" else x for x in seg["labels"]["expect"]]
seg["measures"] = {("分区甲(人)" if k == "分区甲(人次)" else k): v for k, v in seg["measures"].items()}
next(t for t in rec["tables"] if t["name"] == "日客流")["units"]["分区甲"] = "人"
st = await put_recipe(client, st["id"], rec)
assert st["recipe_problems"] == [], st["recipe_problems"]
return await trial(client, st["id"])
async def test_a4_runs_pinned_to_august_keep_reading_august(client):
"""A4 前半:在 S1 上建运行记录(data_versions 固定 S1),之后提交 9 月 → 固定 S1 的查询工具查日客流是 31 行,
冻结的表结构与 S1 的一致;不固定的调用查到 61。"""
from app.data.engine import engines
base = await first_import(client, seed=41)
sid, name, s1 = base["source_id"], base["name"], base["s1"]
await keep_run(sid, name, s1)
raw, fn = flow_workbook(SEP, 30, seed=42)
out = await add_period(client, sid, raw, fn)
assert out["snapshot_id"] != s1 and out["parts"] == 2
pins = {sid: {"snapshot": s1, "name": name}}
try:
query = (await tools_for([f"db_query__{name}"], data_versions=pins))[f"db_query__{name}"]
text = await query.coroutine(sql='SELECT COUNT(*) FROM "日客流"')
payload = json.loads(text)
assert payload["rows"][0][0] == 31
assert artifact_store.load(payload["artifact"])["data_version"] == s1
frozen = artifact_store.load(payload["schema_artifact"])
s1_row = await db_get(SourceSnapshot, s1)
assert set(frozen["tables"]) == set(s1_row.schema_cache["tables"]) == set(TABLES)
assert frozen["tables"]["日客流"]["columns"] == s1_row.schema_cache["tables"]["日客流"]["columns"]
current = (await tools_for([f"db_query__{name}"]))[f"db_query__{name}"]
text = await current.coroutine(sql='SELECT COUNT(*) FROM "日客流"')
assert json.loads(text)["rows"][0][0] == 61
assert artifact_store.load(json.loads(text)["artifact"])["data_version"] == out["snapshot_id"]
finally:
await engines.invalidate(sid)
async def test_a4_a_union_snapshot_referenced_by_a_run_survives_gc(client, monkeypatch):
"""A4 后半(保护规则单独验证,评审一-m8;1.5:KEEP 一律用 0):在 S2(并集)上建运行记录,再走 restart 得到 S3;
回收之后 S2 的并集文件仍在、记录没有 retired;删掉那条运行记录再回收,S2 被回收、并集文件删掉。"""
monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0)
v = await august_september(client, seed=43)
sid = v["source_id"]
run_id = await keep_run(sid, v["name"], v["s2"])
st = await restart_with_unit_change(client, sid, seed=44)
assert st["trial"]["accumulate"]["action"] == "restart"
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
s3 = resp.json()["snapshot_id"]
s2 = await db_get(SourceSnapshot, v["s2"])
union_file = Path(s2.db_path)
async with SessionLocal() as session:
await table_versions.gc(session)
assert union_file.is_file() and (await db_get(SourceSnapshot, v["s2"])).retired_at is None
assert (await db_get(SourceSnapshot, v["s1"])).retired_at is not None, "KEEP=0:没人引用的 S1 照期 1 回收"
async with SessionLocal() as session:
await session.delete(await session.get(Run, run_id))
await session.commit()
await table_versions.gc(session)
assert (await db_get(SourceSnapshot, v["s2"])).retired_at is not None
assert not union_file.exists()
assert await current_id(sid) == s3
# ==========================================================================
# A5:全角或 en-dash 的标签两个月都能查到
# ==========================================================================
@pytest.mark.parametrize("variant", ["D24", "D24w"])
async def test_a5_dash_and_fullwidth_labels_are_queryable_across_months(client, variant):
"""A5:9 月用 en-dash(D24)或全角数字与全角减号(D24w)写时段标签:试运行差异卡有 label_writing、需确认;提交后
WHERE "时段"='8-9' 是 61,按月 31 / 30(维度值按规范写法存储)。"""
base = await first_import(client, seed=51)
sid, name = base["source_id"], base["name"]
raw, fn = flow_workbook(SEP, 30, seed=52, variant=variant)
st = await reupload(client, sid, raw, fn)
t = st["trial"]
assert t["status"] == "passed", t["problems"]
writing = [d for d in t["diff"] if d["kind"] == "label_writing"]
assert writing and all(d["requires_confirm"] for d in writing), t["diff"]
assert {"diff:label_writing:日间", "diff:label_writing:夜间"} <= set(confirm_ids(st))
assert t["accumulate"]["action"] == "append" and t["accumulate"]["union"]["rows"] == UNION_ROWS
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
where = ' WHERE "时段" = \'8-9\''
assert await count(client, name, "时段客流", where) == 61
assert await by_month(client, name, "时段客流", where) == [["2026-08", 31], ["2026-09", 30]]
labels = {r[0] for r in await tool_rows(client, name, 'SELECT DISTINCT "时段" FROM "时段客流"')}
assert all(re.fullmatch(r"\d{1,2}-\d{1,2}", x) for x in labels), labels
# ==========================================================================
# A6、A6b、A6c、A15:拒收的漂移走修复按钮后能试运行并启用(3.4)
# ==========================================================================
#: 3.4 表与「其他两例」:拒收的问题、fix、坐标(None = 无坐标)、修复步骤 [(kind, 选项)]、修复后要勾的确认项、
#: 累积后的行数
FIX_CASES: dict[str, dict[str, Any]] = {
"D04": {"code": "label_missing", "fix": "remove_label", "cell": f"{SHEET}!B10",
"steps": [("remove_label", "remove")], "confirms": {"fix:remove_label:日间:7-8"},
"rows": {"日客流": 61, "时段客流": 1007, "时段客流_表内合计": 183}},
"D09": {"code": "title_not_found", "fix": "rename_title", "cell": None,
"steps": [("rename_title", "|keep")], "confirms": {"fix:rename_title:日间"}, "rows": UNION_ROWS},
"D13": {"code": "label_unexpected", "fix": "add_label", "cell": f"{SHEET}!B8",
"steps": [("add_label", "add"), ("edit_members", "update")],
"confirms": {"unit:日客流.分区丙", "relation:R1"}, "fix_items": 2, "rows": UNION_ROWS},
"D18": {"code": "label_unparsed", "fix": "declare_total", "cell": f"{SHEET}!B21",
"steps": [("declare_total", "keep")], "confirms": {"fix:declare_total:日间", "derived:日间合计"},
"accept": "K1", "rows": {"日客流": 61, "时段客流": 1037, "时段客流_表内合计": 213}},
"D16": {"code": "axis_extra_cells", "fix": "ignore_cells", "cell": None,
"steps": [("ignore_cells", None)], "confirms": set(), "rows": UNION_ROWS},
"D21": {"code": "row_unclaimed", "fix": "ignore_cells", "cell": f"{SHEET}!B32",
"steps": [("ignore_cells", None)], "confirms": set(), "rows": UNION_ROWS},
}
async def _fix_case(client, code: str, seed: int) -> dict[str, Any]:
case = FIX_CASES[code]
base = await first_import(client, seed=seed)
raw, fn = DRIFT_CASES[code].build(seed + 1)
st = await reupload(client, base["source_id"], raw, fn)
assert st["trial"]["status"] == "rejected"
probs = problems(st, case["code"])
assert probs and {p["fix"] for p in probs} == {case["fix"]}, [(p["code"], p["fix"]) for p in problems(st)]
if case["cell"] is not None:
assert case["cell"] in {c for p in probs for c in p["cells"]}, [p["cells"] for p in probs]
# 问题旁边的按钮:提议挂在对应的那条问题上(fix_ids,评审三-B2)
fx = fix_of(st, case["fix"])
assert any(fx["id"] in p["fix_ids"] for p in probs), (fx["id"], [p["fix_ids"] for p in probs])
for kind, option in case["steps"]:
fx = fix_of(st, kind)
value = next(o["value"] for o in fx["options"] if option is None or o["value"].endswith(option))
st, pv = await fixed(client, st, kind, value)
assert pv["recipe_sha256_after"] == st["recipe_sha256"]
assert st["recipe_problems"] == [], st["recipe_problems"]
st = await trial(client, st["id"])
return {"base": base, "st": st, "case": case}
@pytest.mark.parametrize("code", ["D04", "D09", "D13", "D18"])
async def test_a6_drift_fixed_with_the_fix_button_can_be_enabled(client, code):
"""A6:reupload → rejected,问题的 fix 与 3.4 表一致 → StagingOut.fixes 有对应提议 → edits/preview → edits/apply
(D13 接着应用 edit_members)→ trial(D18 写理由接受 K1)→ 勾全部确认项(含 fix:*)提交 201 → 并集行数与 3.4 一致。"""
done = await _fix_case(client, code, seed=60 + list(FIX_CASES).index(code))
st, case, name = done["st"], done["case"], done["base"]["name"]
t = st["trial"]
acceptances = None
if case.get("accept"):
assert t["status"] == "needs_decision" and t["acceptable"] == [case["accept"]], (t["status"], t["acceptable"])
k1 = check(st, case["accept"])
# 3.4:K1(新的日间合计)unverifiable,原因 null_detail 30 格(7-8 行整行是占位符,明细缺数)
assert k1["status"] == "unverifiable" and k1["reasons"] == {"null_detail": 30} and k1["acceptable"] is True
assert {c["id"]: c["status"] for c in t["checks"] if c["id"] in ("K2", "G1", "G2")} == {
"K2": "passed", "G1": "passed", "G2": "passed"}
acceptances = [{"check_id": case["accept"], "reason": REASON}]
else:
assert t["status"] == "passed", (t["problems"], [c for c in t["checks"] if c["status"] != "passed"])
assert t["accumulate"]["action"] == "append" and t["accumulate"]["union"]["rows"] == case["rows"]
ids = confirm_ids(st)
fixes = [i for i in ids if i.startswith("fix:")]
assert len(fixes) == case.get("fix_items", 1), fixes
assert case["confirms"] <= set(ids), (case["confirms"], ids)
assert not [i for i in ids if i.startswith(("breaking:", "accumulate_restart"))], ids
resp = await commit(client, st, confirmations=[i for i in ids if not i.startswith("fix:")], acceptances=acceptances)
assert resp.status_code == 422 and resp.json()["code"] == "confirm_required", "fix:* 是必勾项"
resp = await commit(client, st, acceptances=acceptances)
assert resp.status_code == 201, resp.text
assert resp.json()["parts"] == 2
for table, n in case["rows"].items():
assert await count(client, name, table) == n, table
if code == "D09":
kinds = await tool_rows(client, name, 'SELECT DISTINCT "时段类别" FROM "时段客流" ORDER BY 1')
assert sorted(k[0] for k in kinds) == ["夜间", "日间"]
if code == "D13":
assert await count(client, name, "日客流", ' WHERE "分区丙" IS NULL') == 31
assert await by_month(client, name, "日客流", ' WHERE "分区丙" IS NULL') == [["2026-08", 31]]
snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"])
assert "各期登记的分项列不同" in snap.schema_cache["tables"]["日客流"]["comment"]
@pytest.mark.parametrize("code", ["D16", "D21"])
async def test_a6b_ignore_columns_and_rows_then_enable(client, code):
"""A6b:D16 按表头忽略右侧的「合计」列(⑥ columns),D21 按行标签忽略「补录(人次)」(⑥ rows,要理由);修复后
能启用,行数与 D01 相同(30 / 510 / 90,并集 61 / 1037 / 183);D21 的回执 rows_excluded 有 ignored_rows(第 32 行)。"""
done = await _fix_case(client, code, seed=70 + (code == "D21"))
st, name = done["st"], done["base"]["name"]
t = st["trial"]
assert t["status"] == "passed", (t["problems"], [c for c in t["checks"] if c["status"] != "passed"])
assert rows_of(st) == SEP_ROWS
excluded = t["receipt"]["rows_excluded"]
if code == "D21":
ignored = [x for x in excluded if x["reason"] == "ignored_rows"]
assert ignored and ignored[0]["rows"] == [[32, 32]] and ignored[0]["anchor"] == "补录(人次)", excluded
assert ignored[0]["cells"] == 3, "标签格加两格数(B32:D32)"
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
for table, n in UNION_ROWS.items():
assert await count(client, name, table) == n
if code == "D21":
imp = await db_get(TableImport, resp.json()["import_id"])
manifest = artifact_store.load(imp.manifest_artifact)
assert [x["reason"] for x in manifest["receipt"]["rows_excluded"]] == [x["reason"] for x in excluded]
# 后续每期按锚点忽略:锚点照旧的下一期直接通过;换了一个没见过的标签(列头)照样拒收,不会被这条规则吞掉
from openpyxl.utils import get_column_letter
raw, fn = flow_workbook(OCT, 31, seed=72, variant=code)
st = await reupload(client, done["base"]["source_id"], raw, fn)
assert st["trial"]["status"] == "passed", st["trial"]["problems"]
await client.delete(f"{API}/imports/{st['id']}")
other = {"B32": "备用分区(人次)"} if code == "D21" else {f"{get_column_letter(3 + 31)}4": "备注"}
st = await reupload(client, done["base"]["source_id"], edit_cells(raw, other), fn)
want = "row_unclaimed" if code == "D21" else "axis_extra_cells"
assert st["trial"]["status"] == "rejected" and problems(st, want), st["trial"]["problems"]
async def test_a6c_d04_with_a_failing_r1_day_is_fixed_then_accepted(client):
"""A6c(评审一-M3):D04 上再改一格分区甲,使 R1 有一天不成立 → 应用 ① 后 recipe_problems 为空(R1 的认领不查)→
试运行 R1 mismatch、needs_decision → 写理由接受后提交 201;说明用 sum_eq_union_mixed、不含「已逐日核对」。"""
base = await first_import(client, seed=81)
raw, fn = DRIFT_CASES["D04"].build(82)
raw = edit_cells(raw, {"L6": 1}) # 第 10 个日期(L 列)的分区甲改成 1:全日 ≠ 甲 + 乙
st = await reupload(client, base["source_id"], raw, fn)
assert st["trial"]["status"] == "rejected" and problems(st, "label_missing")
st, _ = await fixed(client, st, "remove_label", "remove")
assert st["recipe_problems"] == [], "改的是 dimension 段:R1 的认领按重放处理,不查(3.3 第 11 条)"
st = await trial(client, st["id"])
t = st["trial"]
r1 = check(st, "R1")
assert t["status"] == "needs_decision" and r1["status"] == "mismatch" and t["acceptable"] == ["R1"], t["status"]
resp = await commit(client, st)
assert resp.status_code == 422 and resp.json()["code"] == "acceptance_required"
resp = await commit(client, st, acceptances=[{"check_id": "R1", "reason": REASON}])
assert resp.status_code == 201, resp.text
comment = (await db_get(SourceSnapshot, resp.json()["snapshot_id"])).schema_cache["tables"]["日客流"]["comment"]
assert "在部分期的个别" in comment and "不成立或未能核对" in comment and "已逐日核对" not in comment, comment
async def test_a15_after_a_fix_only_changed_recipe_items_are_listed(client):
"""A15:D04 修复之后的确认项里,配方类只有 fix:*(source=edit),没有 placeholder:·、unit:*、relation:* 等没有变化
的项(6.2:只列签名与现行配方不同的项)。"""
done = await _fix_case(client, "D04", seed=91)
st = done["st"]
items = {i["id"]: i for i in st["trial"]["confirm_items"]}
assert items["fix:remove_label:日间:7-8"]["source"] == "edit"
recipe_items = sorted(i for i, v in items.items() if v.get("source", "recipe") == "recipe")
assert recipe_items == [], recipe_items
for stale in ("placeholder:·", "unit:日客流.全日客流", "relation:R1", "derived:夜间合计", "mode", "year_from:交叉表"):
assert stale not in items, stale
# ==========================================================================
# A7:c01 框选回放一致(4.6 的三条)
# ==========================================================================
async def test_a7_c01_selection_replays_consistently(client):
"""4.6:① 同一份文件:月报!C5:F8 框为列表 → ok,锚点是四个表头,重放与框一致,换算后的配方与规则草稿逐字相同;
② 整张表挪到 E8(shift=(3, 2))用同一份配方试运行:passed、T1 passed,列表表和合计表的 table_hashes 与未挪动的
逐一相同(配方里没有坐标);③ 泛化不了时明确说明:C5:F7 → selection_bottom_not_anchorable(建议扩大到第 8 行),
同一框改选 bottom=auto → ok、data=C6:F8、重放一致;C6:F8 → selection_header_not_text;C5:G8 → selection_header_blank。"""
raw, fn = lab.c01("literal")
st = await stage(client, raw, fn)
blocks = [b for s in st["recipe"]["sheets"] for b in s["blocks"]]
assert [b["layout"] for b in blocks] == ["list"]
sel = {"sheet": "月报", "ref": "C5:F8", "as": "list", "options": {"header_rows": 1, "bottom": "box"}}
resp = await preview(client, st["id"], {"selection": sel})
assert resp.status_code == 200, resp.text
pv = resp.json()
assert pv["ok"] is True, pv["problems"]
assert [a["text"] for a in pv["anchors"] if a["kind"] == "header"] == ["地区", "产品", "销量", "金额"]
box = {"header": "C5:F5", "data": "C6:F8", "total": "C9:F9"}
assert pv["replay"]["match"] is True and pv["replay"]["expected"] == box and pv["replay"]["actual"] == box
assert pv["replay"]["diffs"] == []
assert pv["recipe_sha256_after"] == st["recipe_sha256"], "框选得到的规则与起草器完全一样"
resp = await apply(client, st["id"], {"selection": sel}, pv["recipe_sha256_after"])
assert resp.status_code == 200, resp.text
st = await trial(client, resp.json()["id"])
assert st["trial"]["status"] == "passed", st["trial"]["problems"]
assert check(st, "T1")["status"] == "passed"
hashes = st["trial"]["receipt"]["table_hashes"]
list_table = blocks[0]["table"]
total_table = blocks[0]["rows"]["total_row"]["keep_as"]
assert {list_table, total_table} <= set(hashes)
# ② 挪了位置也回放一致:用第 1 条得到的配方(PUT)
raw2, fn2 = lab.c01("literal", shift=(3, 2))
moved = await stage(client, raw2, fn2)
moved = await put_recipe(client, moved["id"], st["recipe"])
assert moved["recipe_problems"] == [], moved["recipe_problems"]
moved = await trial(client, moved["id"])
assert moved["trial"]["status"] == "passed", moved["trial"]["problems"]
assert check(moved, "T1")["status"] == "passed"
got = moved["trial"]["receipt"]["table_hashes"]
assert got[list_table] == hashes[list_table] and got[total_table] == hashes[total_table]
# ③ 泛化不了时明确说明
bad = (await preview(client, st["id"], {"selection": {**sel, "ref": "C5:F7"}})).json()
assert bad["ok"] is False and [p["code"] for p in bad["problems"]] == ["selection_bottom_not_anchorable"]
assert "第 8 行" in bad["problems"][0]["message"], bad["problems"][0]["message"]
auto = (await preview(client, st["id"], {"selection": {**sel, "ref": "C5:F7",
"options": {"header_rows": 1, "bottom": "auto"}}})).json()
assert auto["ok"] is True, auto["problems"]
assert auto["expected"]["data"] == "C6:F8" and auto["replay"]["match"] is True
bad = (await preview(client, st["id"], {"selection": {**sel, "ref": "C6:F8"}})).json()
assert bad["ok"] is False and "selection_header_not_text" in [p["code"] for p in bad["problems"]]
bad = (await preview(client, st["id"], {"selection": {**sel, "ref": "C5:G8"}})).json()
assert bad["ok"] is False and "selection_header_blank" in [p["code"] for p in bad["problems"]]
# ==========================================================================
# A8:回滚(启用旧版本)
# ==========================================================================
async def test_a8_rollback_to_an_earlier_version(client, monkeypatch):
"""A8:在 S2(累积)上 activate S1(带 expected_current=S2)→ 当前快照是 S1,current_recipe_id 是 S1 的配方,导入记录
状态互换,写了 snapshot_activations;在 S2 上发起的固定运行仍查 61 行;confirm 缺省 422;expected_current 不对 409
base_changed;遮罩列在目标版本里没有同名列时不带 ack_mask_lost 409 mask_lost,带上成功;activate 已回收的快照
409 snapshot_unavailable(KEEP=0 造出已回收的快照,1.5)。
9 月用 D13 走修复按钮(加「分区丙」),S2 的配方 R2 与 S1 的 R0 不是同一条:启用前现行配方是 R2,启用 S1 之后是 R0,
两条配方的状态互换。两期同一个配方时「现行配方跟着快照走」这一条测不出来(WP-8 评审意见 3)。"""
from app.data.engine import engines
done = await _fix_case(client, "D13", seed=101)
assert done["st"]["trial"]["status"] == "passed", done["st"]["trial"]["problems"]
resp = await commit(client, done["st"])
assert resp.status_code == 201, resp.text
base, sep = done["base"], resp.json()
v = {**base, "s2": sep["snapshot_id"], "aug": base["import_id"], "sep": sep["import_id"]}
sid, name = v["source_id"], v["name"]
r0, r2 = base["recipe_id"], sep["recipe_id"]
assert r0 != r2 and (await db_get(SourceSnapshot, v["s1"])).recipe_id == r0
assert (await db_get(SourceSnapshot, v["s2"])).recipe_id == r2
assert (await db_get(DataSource, sid)).current_recipe_id == r2
assert ((await db_get(TableRecipe, r0)).status, (await db_get(TableRecipe, r2)).status) == ("superseded", "active")
await keep_run(sid, name, v["s2"])
url = f"{API}/{sid}/snapshots/{v['s1']}/activate"
resp = await client.post(url, json={"expected_current_snapshot_id": v["s2"]})
assert resp.status_code == 422 and resp.json()["code"] == "confirm_required"
resp = await client.post(url, json={"confirm": True})
assert resp.status_code == 422 and resp.json()["code"] == "expected_current_required"
resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s1"]})
assert resp.status_code == 409 and resp.json()["code"] == "base_changed"
async with SessionLocal() as session:
await session.execute(update(DataSource).where(DataSource.id == sid).values(options={"mask_columns": "分区丁"}))
await session.commit()
listed = {s["id"]: s for s in await snapshots(client, sid)}
assert listed[v["s1"]]["mask_lost"] == ["分区丁"] and listed[v["s1"]]["activatable"] is True
resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s2"]})
assert resp.status_code == 409 and resp.json()["code"] == "mask_lost"
resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": v["s2"], "reason": REASON,
"ack_mask_lost": ["分区丁"], "signed_by": "合成署名甲"})
assert resp.status_code == 200, resp.text
out = resp.json()
assert out["snapshot_id"] == v["s1"] and out["previous_snapshot_id"] == v["s2"]
s1 = await db_get(SourceSnapshot, v["s1"])
src = await db_get(DataSource, sid)
assert src.current_snapshot_id == v["s1"] and src.current_recipe_id == s1.recipe_id == out["recipe_id"] == r0
assert ((await db_get(TableRecipe, r0)).status, (await db_get(TableRecipe, r2)).status) == ("active", "superseded")
assert (await db_get(TableImport, v["aug"])).status == "active"
assert (await db_get(TableImport, v["sep"])).status == "superseded"
async with SessionLocal() as session:
last = (await session.execute(select(SnapshotActivation).where(SnapshotActivation.source_id == sid)
.order_by(SnapshotActivation.created_at.desc()))).scalars().first()
assert (last.kind, last.snapshot_id, last.previous_snapshot_id, last.signed_by) == (
"activate", v["s1"], v["s2"], "合成署名甲")
assert await count(client, name, "日客流") == 31
try:
pinned = (await tools_for([f"db_query__{name}"], data_versions={sid: {"snapshot": v["s2"], "name": name}}))
text = await pinned[f"db_query__{name}"].coroutine(sql='SELECT COUNT(*) FROM "日客流"')
assert json.loads(text)["rows"][0][0] == 61, "在 S2 上发起的固定运行仍查 S2"
finally:
await engines.invalidate(sid)
# 已回收的快照不能启用:KEEP=0 下再提交 10 月,S1(不是当前、没人引用)被回收
monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0)
raw, fn = flow_workbook(OCT, 31, seed=102)
s3 = (await add_period(client, sid, raw, fn))["snapshot_id"]
assert (await db_get(SourceSnapshot, v["s1"])).retired_at is not None
resp = await client.post(url, json={"confirm": True, "expected_current_snapshot_id": s3,
"ack_mask_lost": ["分区丁"]})
assert resp.status_code == 409 and resp.json()["code"] == "snapshot_unavailable"
# ==========================================================================
# A9:作废接受(累积模式 = 移除该期)
# ==========================================================================
async def test_a9_revoking_an_acceptance_removes_the_period_and_keeps_recipe_and_mode(client):
"""A9:D26(9 月 R1 不成立、写理由接受)累积提交 → 说明含 sum_eq_union_mixed 的片段、不含「已逐日核对」→
revoke-acceptance → action=remove_period,新快照只有 8 月,快照 id 等于 S1(复用),current_recipe_id 和模式都没变,
该导入 revoked 非空;再 activate 含已作废导入的快照 409 contains_revoked;remove 唯一一期 409 last_period。"""
base = await first_import(client, seed=111)
sid = base["source_id"]
raw, fn = DRIFT_CASES["D26"].build(112)
st = await reupload(client, sid, raw, fn)
assert st["trial"]["status"] == "needs_decision" and st["trial"]["acceptable"] == ["R1"]
resp = await commit(client, st, acceptances=[{"check_id": "R1", "reason": REASON}])
assert resp.status_code == 201, resp.text
s2, sep = resp.json()["snapshot_id"], resp.json()["import_id"]
comment = (await db_get(SourceSnapshot, s2)).schema_cache["tables"]["日客流"]["comment"]
assert "在部分期的个别" in comment and "不成立或未能核对" in comment and "已逐日核对" not in comment, comment
recipe_before = (await db_get(DataSource, sid)).current_recipe_id
resp = await client.post(f"{API}/{sid}/imports/{sep}/revoke-acceptance",
json={"confirm": True, "expected_current_snapshot_id": s2, "reason": REASON})
assert resp.status_code == 200, resp.text
out = resp.json()
assert out["action"] == "remove_period" and out["snapshot_id"] == base["s1"]
snap = await db_get(SourceSnapshot, out["snapshot_id"])
assert snap.imports == [base["import_id"]] and snap.mode == "accumulate"
assert (await db_get(DataSource, sid)).current_recipe_id == recipe_before
imp = await db_get(TableImport, sep)
assert imp.revoked and imp.revoked["reason"] == REASON and imp.status == "superseded"
resp = await client.post(f"{API}/{sid}/snapshots/{s2}/activate",
json={"confirm": True, "expected_current_snapshot_id": base["s1"]})
assert resp.status_code == 409 and resp.json()["code"] == "contains_revoked"
resp = await client.post(f"{API}/{sid}/imports/{base['import_id']}/remove",
json={"confirm": True, "expected_current_snapshot_id": base["s1"], "reason": REASON})
assert resp.status_code == 409 and resp.json()["code"] == "last_period"
# ==========================================================================
# A10:不兼容变更要重新开始累积
# ==========================================================================
async def test_a10_an_incompatible_change_restarts_the_accumulation(client):
"""A10:S2 之后 reupload 10 月,在 10 月的暂存区里用 PUT 把一列的单位改掉 → 试运行 action=restart,确认项有
accumulate_restart;提交后当前快照只有 10 月,说明含 single_after_drop;S2 仍可启用(在 SNAPSHOT_KEEP 之内)。"""
v = await august_september(client, seed=121)
sid = v["source_id"]
# 不兼容与部分重叠同时出现:restart 优先(结果只有本期,不涉及重叠),重叠的各期只作提示(2.4,评审一-m12)
st = await restart_with_unit_change(client, sid, start=date_of(2026, 8, 15), days=31, seed=120)
assert st["trial"]["status"] == "passed" and st["trial"]["accumulate"]["action"] == "restart", st["trial"]
assert [p["start"] for p in st["trial"]["accumulate"]["overlaps"]] == ["2026-08-01", "2026-09-01"]
await client.delete(f"{API}/imports/{st['id']}")
st = await restart_with_unit_change(client, sid, seed=122)
t = st["trial"]
assert t["status"] == "passed", t["problems"]
plan = t["accumulate"]
assert plan["action"] == "restart" and plan["change"] == "semantic"
assert [p["start"] for p in plan["dropped"]] == ["2026-08-01", "2026-09-01"]
assert any("人次" in x and "人" in x for x in plan["semantic"]), plan["semantic"]
assert "accumulate_restart" in confirm_ids(st)
resp = await commit(client, st, confirmations=[i for i in confirm_ids(st) if i != "accumulate_restart"])
assert resp.status_code == 422 and resp.json()["code"] == "confirm_required"
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
out = resp.json()
assert out["parts"] == 1
assert await current_periods(sid) == ["2026-10-01"]
comment = (await db_get(SourceSnapshot, out["snapshot_id"])).schema_cache["tables"]["日客流"]["comment"]
assert "当前版本只含最近一期的数据" in comment, comment
listed = {s["id"]: s for s in await snapshots(client, sid)}
assert listed[v["s2"]]["activatable"] is True and listed[v["s2"]]["available"] is True
# ==========================================================================
# A11:退役列保留
# ==========================================================================
async def drop_zone_b(client, st: dict[str, Any]) -> dict[str, Any]:
"""这一期的文件不再有分区乙:修复按钮去掉标签(①,连带去掉列和单位),R1(全日 = 甲 + 乙)删掉(③ 的「删除这条
关系」),R2 按本期的系统发现重新认领(本期只剩口径不同那一条,编号是 F1;改认领只能在配方面板里 PUT)。"""
st, _ = await fixed(client, st, "remove_label", "remove")
st, _ = await fixed(client, st, "edit_members", "remove")
rec = json.loads(json.dumps(st["recipe"], ensure_ascii=False))
for r in rec["relations"]:
if r["id"] == "R2":
r["claims"] = "F1"
st = await put_recipe(client, st["id"], rec)
assert st["recipe_problems"] == [], st["recipe_problems"]
assert [r["id"] for r in st["recipe"]["relations"]] == ["R2"]
return await trial(client, st["id"])
async def test_a11_a_retired_column_is_kept_and_not_retired_again(client):
"""A11:去掉 measures 的分区乙(R1 不再登记)后累积 9 月 → 确认项有 retire:日客流.分区乙;并集里 8 月有值、9 月为空值;
列说明含「自某一期起不再导入」,分区乙的单位仍是人次;日客流的说明用 sum_eq_union_partial,不含「已逐日核对」。
再累积 10 月(评审二-M3):确认项里没有 retire:*,也没有 breaking:日客流;计划的 retired_existing 有分区乙。"""
base = await first_import(client, seed=131)
sid, name = base["source_id"], base["name"]
raw, fn = flow_workbook(SEP, 30, seed=132)
st = await reupload(client, sid, without_zone_b(raw), fn)
assert st["trial"]["status"] == "rejected" and problems(st, "label_missing")
st = await drop_zone_b(client, st)
t = st["trial"]
assert t["status"] == "passed", t["problems"]
assert t["accumulate"]["action"] == "append" and t["accumulate"]["change"] == "retire"
assert t["accumulate"]["retired_new"] == [{"table": "日客流", "column": "分区乙"}]
assert "retire:日客流.分区乙" in confirm_ids(st)
assert not [i for i in confirm_ids(st) if i.startswith("breaking:日客流")], "退役与 breaking 同表去重(2.5)"
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"])
assert await by_month(client, name, "日客流", ' WHERE "分区乙" IS NOT NULL') == [["2026-08", 31]]
assert await by_month(client, name, "日客流", ' WHERE "分区乙" IS NULL') == [["2026-09", 30]]
table = snap.schema_cache["tables"]["日客流"]
col = next(c for c in table["columns"] if c["name"] == "分区乙")
assert "单位:人次" in col["comment"] and "自某一期起不再导入" in col["comment"], col
assert "只在部分期登记并核对" in table["comment"] and "已逐日核对" not in table["comment"], table["comment"]
manifest = artifact_store.load(snap.manifest_artifact)
assert manifest["columns"]["日客流"]["分区乙"]["status"] == "retired"
raw, fn = flow_workbook(OCT, 31, seed=133)
st = await reupload(client, sid, without_zone_b(raw), fn)
t = st["trial"]
assert t["status"] == "passed", t["problems"]
assert t["accumulate"]["retired_new"] == []
assert {"table": "日客流", "column": "分区乙"} in t["accumulate"]["retired_existing"]
assert not [i for i in confirm_ids(st) if i.startswith(("retire", "breaking:日客流"))], confirm_ids(st)
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
assert await by_month(client, name, "日客流", ' WHERE "分区乙" IS NULL') == [["2026-09", 30], ["2026-10", 31]]
# ==========================================================================
# A12:模式切换
# ==========================================================================
async def test_a12_switching_modes_both_ways(client):
"""A12:替换模式的源上传新一期并回答 q_mode=accumulate → mode_switch:replace->accumulate,当前那一期作为第一期;
反向切换出 mode_switch:accumulate->replace,提交后只剩一期。"""
base = await first_import(client, seed=141, mode="replace")
sid, name = base["source_id"], base["name"]
assert (await db_get(SourceSnapshot, base["s1"])).mode == "replace"
raw, fn = flow_workbook(SEP, 30, seed=142)
st = await reupload(client, sid, raw, fn)
assert "q_mode" in {q["id"] for q in st["questions"]}
st = await answer(client, st["id"], {"q_mode": {"value": "accumulate"}})
st = await trial(client, st["id"])
t = st["trial"]
assert t["status"] == "passed", t["problems"]
assert t["accumulate"]["mode_switch"] == "replace->accumulate" and t["accumulate"]["action"] == "append"
assert [p["start"] for p in t["accumulate"]["parts"]] == ["2026-08-01", "2026-09-01"]
assert "mode_switch:replace->accumulate" in confirm_ids(st)
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
assert resp.json()["parts"] == 2 and await count(client, name, "日客流") == 61
raw, fn = flow_workbook(OCT, 31, seed=143)
st = await reupload(client, sid, raw, fn)
st = await answer(client, st["id"], {"q_mode": {"value": "replace"}})
st = await trial(client, st["id"])
t = st["trial"]
assert t["status"] == "passed", t["problems"]
assert t["accumulate"]["mode_switch"] == "accumulate->replace" and t["accumulate"]["action"] == "replace"
assert "mode_switch:accumulate->replace" in confirm_ids(st)
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
assert resp.json()["parts"] == 1
snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"])
assert snap.mode == "replace" and len(snap.imports) == 1
assert await count(client, name, "日客流") == 31
# ==========================================================================
# A13:快照清单与证据链
# ==========================================================================
async def test_a13_query_snapshot_commits_to_the_snapshot_and_import_manifests(client):
"""A13:在累积快照上跑一次查询工具:query_snapshot → schema_snapshot 里有 snapshot_manifest 和两个 import_manifests →
快照清单的 union.db_sha256 等于快照的 db_sha256;各期 rows 区间相接,覆盖 1..61。"""
from app.data.engine import engines
v = await august_september(client, seed=151)
sid, name = v["source_id"], v["name"]
try:
query = (await tools_for([f"db_query__{name}"]))[f"db_query__{name}"]
payload = json.loads(await query.coroutine(sql='SELECT COUNT(*) FROM "日客流"'))
finally:
await engines.invalidate(sid)
assert payload["rows"][0][0] == 61
qsnap = artifact_store.load(payload["artifact"])
assert qsnap["data_version"] == v["s2"]
schema = artifact_store.load(payload["schema_artifact"])
snap = await db_get(SourceSnapshot, v["s2"])
assert schema["snapshot_manifest"] == snap.manifest_artifact
imps = [await db_get(TableImport, i) for i in snap.imports]
assert schema["import_manifests"] == [i.manifest_artifact for i in imps] and len(imps) == 2
manifest = artifact_store.load(schema["snapshot_manifest"])
assert manifest["format"] == "agentlab-snapshot-manifest/1" and manifest["snapshot_id"] == v["s2"]
assert manifest["union"]["db_sha256"] == snap.db_sha256 == raw_store.sha256_file(Path(snap.db_path))
assert [p["import_id"] for p in manifest["parts"]] == snap.imports
for table, total in UNION_ROWS.items():
spans = [p["rows"][table]["union"] for p in manifest["parts"]]
assert spans[0][0] == 1 and spans[-1][1] == total, (table, spans)
assert all(a[1] + 1 == b[0] for a, b in zip(spans, spans[1:])), (table, spans)
assert [p["rows"][table]["part"][0] for p in manifest["parts"]] == [1, 1]
for imp, part in zip(imps, manifest["parts"]):
doc = artifact_store.load(part["manifest_artifact"])
assert doc["import_id"] == imp.id and part["manifest_artifact"] == imp.manifest_artifact
# ==========================================================================
# A14:确定性与复用
# ==========================================================================
async def test_a14_same_september_file_under_another_name_reuses_the_union(client):
"""A14:9 月同一份文件换个名字再传(替换该期)→ 新导入记录(文件名不同,不是期 2 的「原样重传」),union_id 与 S2
相同(并集 id 只取决于各期构建和目标表结构,不含导入记录 id),并集文件复用(新快照的库就是 S2 那个文件),
db_sha256 相同。对照:同名原样再传不新建记录(unchanged)。"""
v = await august_september(client, seed=161)
sid = v["source_id"]
s2_row = await db_get(SourceSnapshot, v["s2"])
st = await reupload(client, sid, v["sep_raw"], "九月客流(再传).xlsx")
t = st["trial"]
assert t["status"] == "passed" and t["accumulate"]["action"] == "replace_period", t["problems"]
assert t["same_as_import"] == {"id": v["sep"], "seq": 2}, "同一个构建"
assert t["accumulate"]["union"]["union_id"] == Path(s2_row.db_path).stem
assert t["accumulate"]["union"]["db_sha256"] == s2_row.db_sha256
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
out = resp.json()
assert out["unchanged"] is False and out["build_reused"] is True
assert out["import_id"] != v["sep"] and out["snapshot_id"] != v["s2"]
s3 = await db_get(SourceSnapshot, out["snapshot_id"])
assert s3.db_path == s2_row.db_path and s3.db_sha256 == s2_row.db_sha256
assert raw_store.sha256_file(Path(s3.db_path)) == s2_row.db_sha256
assert (await db_get(TableImport, out["import_id"])).file_name == "九月客流(再传).xlsx"
async with SessionLocal() as session:
unions = (await session.execute(select(TableBuild).where(
TableBuild.source_id == sid, TableBuild.raw_sha256 == ""))).scalars().all()
assert [u.id for u in unions] == [Path(s2_row.db_path).stem], "并集只登记一次"
# 同名原样再传:不新建记录,当前版本不动
st = await reupload(client, sid, v["sep_raw"], "九月客流(再传).xlsx")
resp = await commit(client, st)
assert resp.status_code == 201 and resp.json()["unchanged"] is True, resp.text
assert resp.json()["snapshot_id"] == out["snapshot_id"]
# ==========================================================================
# A16:并发:移除与提交交错
# ==========================================================================
async def test_a16_remove_with_a_stale_version_after_a_commit_is_base_changed(client):
"""A16 前半:S2=[8, 9] 上先取 snapshots 得到当前 id;另一个请求提交 10 月得 S3;再用旧的 expected_current 移除 9 月
→ 409 base_changed,当前快照仍是 S3,10 月仍在。"""
v = await august_september(client, seed=171)
sid = v["source_id"]
seen = (await snapshots(client, sid))[0]["id"]
assert seen == v["s2"]
raw, fn = flow_workbook(OCT, 31, seed=172)
s3 = (await add_period(client, sid, raw, fn))["snapshot_id"]
resp = await client.post(f"{API}/{sid}/imports/{v['sep']}/remove",
json={"confirm": True, "expected_current_snapshot_id": seen, "reason": REASON})
assert resp.status_code == 409 and resp.json()["code"] == "base_changed", resp.text
assert await current_id(sid) == s3
assert await current_periods(sid) == ["2026-08-01", "2026-09-01", "2026-10-01"]
async def test_a16_interleaved_publishes_the_later_one_is_refused(client, monkeypatch, store):
"""A16 后半:两次发布卡在锁前交错。提交 10 月的请求进了 publish_snapshot、还没进锁时,移除 9 月(按 S2 确认)先完成;
提交随后在锁内重读指针,与暂存区的 base(S2)对不上 → 后到的一方 409 base_changed,当前版本是移除的结果,10 月没进去,
暂存区退回 drafting、没留下临时文件。反过来(移除停在物化里、提交先完成)由 test_source_versions_api 覆盖。"""
v = await august_september(client, seed=173)
sid = v["source_id"]
raw, fn = flow_workbook(OCT, 31, seed=174)
oct_st = await reupload(client, sid, raw, fn)
assert oct_st["trial"]["status"] == "passed"
orig = table_versions.publish_snapshot
removed: dict[str, Any] = {}
async def interleaved(session, source, **kw):
if (kw.get("activation") or {}).get("kind") == "commit" and not removed:
removed["resp"] = await client.post(
f"{API}/{sid}/imports/{v['sep']}/remove",
json={"confirm": True, "expected_current_snapshot_id": v["s2"], "reason": REASON})
return await orig(session, source, **kw)
monkeypatch.setattr(table_versions, "publish_snapshot", interleaved)
resp = await commit(client, oct_st)
assert removed["resp"].status_code == 200, removed["resp"].text
assert removed["resp"].json()["snapshot_id"] == v["s1"]
assert resp.status_code == 409 and resp.json()["code"] == "base_changed", resp.text
assert await current_id(sid) == v["s1"] and await current_periods(sid) == ["2026-08-01"]
assert (await db_get(ImportStaging, oct_st["id"])).status == "drafting"
assert list((store / "uploads").rglob("*.tmp-*")) == []
# ==========================================================================
# A17:移除时配方、模式不退回;已回收的快照复活
# ==========================================================================
async def test_a17_1_removing_after_a_mode_switch_keeps_the_target_recipe_and_mode(client):
"""A17①:替换模式 S1=[8](配方 r1)→ 9 月 q_mode=accumulate 得 S2=[8, 9](目标 r2)→ 移除 9 月:新快照 mode=accumulate、
recipe=r2,id 不等于 S1;8 月的配方(r1)哈希不等于 r2,快照库是物化的单期并集(在 snapshots/ 下),说明含
single_after_drop、不含「部分期」。"""
base = await first_import(client, seed=181, mode="replace")
sid, r1 = base["source_id"], base["recipe_id"]
raw, fn = flow_workbook(SEP, 30, seed=182)
st = await reupload(client, sid, raw, fn)
st = await answer(client, st["id"], {"q_mode": {"value": "accumulate"}})
st = await trial(client, st["id"])
assert st["trial"]["status"] == "passed", st["trial"]["problems"]
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
s2, r2, sep = resp.json()["snapshot_id"], resp.json()["recipe_id"], resp.json()["import_id"]
assert r2 != r1
assert (await db_get(TableRecipe, r1)).recipe_sha256 != (await db_get(TableRecipe, r2)).recipe_sha256
resp = await client.post(f"{API}/{sid}/imports/{sep}/remove",
json={"confirm": True, "expected_current_snapshot_id": s2, "reason": REASON})
assert resp.status_code == 200, resp.text
out = resp.json()
assert out["snapshot_id"] not in (base["s1"], s2) and out["reused"] is False
snap = await db_get(SourceSnapshot, out["snapshot_id"])
assert snap.mode == "accumulate" and snap.recipe_id == r2 and snap.imports == [base["import_id"]]
assert Path(snap.db_path).parent.name == "snapshots", "一期、但配方不是目标配方:按目标配方物化成单期并集"
assert (await db_get(DataSource, sid)).current_recipe_id == r2
comment = snap.schema_cache["tables"]["日客流"]["comment"]
assert "当前版本只含最近一期的数据" in comment and "部分期" not in comment, comment
async def test_a17_2_removing_the_period_that_added_a_column_keeps_the_target_recipe(client):
"""A17②:[8, 9, 10] 中 10 月加了分区丙(修复按钮,目标 r3)→ 移除 10 月:目标仍是 r3,并集里分区丙为空值;下一期
按 r3 重放(不用再修)。"""
v = await august_september(client, seed=183)
sid, name = v["source_id"], v["name"]
raw, fn = flow_workbook(OCT, 31, seed=184, variant="D13")
st = await reupload(client, sid, raw, fn)
st, _ = await fixed(client, st, "add_label", "add")
st, _ = await fixed(client, st, "edit_members", "update")
st = await trial(client, st["id"])
assert st["trial"]["status"] == "passed", st["trial"]["problems"]
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
s3, r3, oct_id = resp.json()["snapshot_id"], resp.json()["recipe_id"], resp.json()["import_id"]
resp = await client.post(f"{API}/{sid}/imports/{oct_id}/remove",
json={"confirm": True, "expected_current_snapshot_id": s3, "reason": REASON})
assert resp.status_code == 200, resp.text
snap = await db_get(SourceSnapshot, resp.json()["snapshot_id"])
assert snap.recipe_id == r3 and snap.imports == [v["aug"], v["sep"]] and snap.id != v["s2"]
assert (await db_get(DataSource, sid)).current_recipe_id == r3
assert "分区丙" in columns_of(snap, "日客流")
assert await count(client, name, "日客流", ' WHERE "分区丙" IS NULL') == 61
raw, fn = flow_workbook(date_of(2026, 11, 1), 30, seed=185, variant="D13")
st = await reupload(client, sid, raw, fn)
assert st["trial"]["status"] == "passed", st["trial"]["problems"]
assert st["recipe_sha256"] == (await db_get(TableRecipe, r3)).recipe_sha256
async def test_a17_3_removing_revives_a_retired_snapshot_whose_union_file_was_deleted(client, monkeypatch):
"""A17③(1.5:KEEP=0):S2=[8, 9](并集)在提交 10 月之后被回收、并集文件被删;移除 10 月按同一组导入、同一个目标配方
得到 S2 的 id → 复活(重新物化、link 回去),文件哈希等于原登记值。"""
monkeypatch.setattr(table_versions, "SNAPSHOT_KEEP", 0)
v = await august_september(client, seed=186)
sid = v["source_id"]
s2 = await db_get(SourceSnapshot, v["s2"])
registered, union_file = s2.db_sha256, Path(s2.db_path)
raw, fn = flow_workbook(OCT, 31, seed=187)
out = await add_period(client, sid, raw, fn)
assert (await db_get(SourceSnapshot, v["s2"])).retired_at is not None and not union_file.exists()
resp = await client.post(f"{API}/{sid}/imports/{out['import_id']}/remove",
json={"confirm": True, "expected_current_snapshot_id": out["snapshot_id"],
"reason": REASON})
assert resp.status_code == 200, resp.text
assert resp.json()["snapshot_id"] == v["s2"] and resp.json()["reused"] is True
revived = await db_get(SourceSnapshot, v["s2"])
assert revived.retired_at is None and revived.db_sha256 == registered
assert union_file.is_file() and raw_store.sha256_file(union_file) == registered
assert (await db_get(TableBuild, union_file.stem)).retired_at is None
assert await count(client, v["name"], "日客流") == 61
# ==========================================================================
# A18:退役名撞名、单位风险
# ==========================================================================
def rename_column(rec: dict[str, Any], old: str, new: str) -> dict[str, Any]:
"""把日客流的一列改名:measures 映射、单位、关系的成员一起改(改名在配方里分不出来,按「旧名退役、新名新增」报)。"""
rec = json.loads(json.dumps(rec, ensure_ascii=False))
for sheet in rec["sheets"]:
for block in sheet["blocks"]:
for seg in block.get("segments", []):
if isinstance(seg.get("measures"), dict):
seg["measures"] = {k: (new if v == old else v) for k, v in seg["measures"].items()}
for t in rec["tables"]:
if t["name"] == "日客流" and old in t.get("units", {}):
t["units"] = {(new if k == old else k): v for k, v in t["units"].items()}
for r in rec["relations"]:
if r.get("table") == "日客流":
r["parts"] = [new if x == old else x for x in r.get("parts", [])]
if r.get("total") == old:
r["total"] = new
return rec
async def test_a18_1_a_new_column_colliding_with_a_retired_one_restarts(client):
"""A18①:当前版本有退役列「PM2_5」时,新增「pm2_5」→ restart(SQLite 列名不分大小写,并集建表会报重名,评审一-m6)。"""
base = await first_import(client, seed=191, edit=lambda rec: rename_column(rec, "分区乙", "PM2_5"))
sid = base["source_id"]
raw, fn = flow_workbook(SEP, 30, seed=192)
st = await reupload(client, sid, raw, fn)
st = await put_recipe(client, st["id"], rename_column(st["recipe"], "PM2_5", "分区乙"))
st = await trial(client, st["id"])
t = st["trial"]
assert t["status"] == "passed", t["problems"]
assert t["accumulate"]["action"] == "append"
assert {"table": "日客流", "column": "PM2_5"} in t["accumulate"]["retired_new"]
assert "retire:日客流.PM2_5" in confirm_ids(st)
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
raw, fn = flow_workbook(OCT, 31, seed=193)
st = await reupload(client, sid, raw, fn)
st = await put_recipe(client, st["id"], rename_column(st["recipe"], "分区乙", "pm2_5"))
assert st["recipe_problems"] == [], st["recipe_problems"]
st = await trial(client, st["id"])
t = st["trial"]
assert t["status"] == "passed", t["problems"]
assert t["accumulate"]["action"] == "restart"
assert any("pm2_5" in x and "PM2_5" in x for x in t["accumulate"]["semantic"]), t["accumulate"]["semantic"]
assert "accumulate_restart" in confirm_ids(st)
async def test_a18_2_unit_wording_outside_the_region_changing_needs_confirmation(client):
"""A18②:累积模式下 9 月区域外的「单位:人次」改成「单位:万人次」→ 确认项有 accumulate_unit_risk(单位写在配方之外,
classify_changes 看不到,评审一-M10)。"""
raw8, _ = flow_workbook(AUG, 31, seed=194)
base = await first_import(client, seed=194, raw=edit_cells(raw8, {"B32": "注:单位:人次"}))
raw9, fn9 = flow_workbook(SEP, 30, seed=195)
st = await reupload(client, base["source_id"], edit_cells(raw9, {"B32": "注:单位:万人次"}), fn9)
t = st["trial"]
assert t["status"] == "passed", t["problems"]
assert t["accumulate"]["action"] == "append"
assert "accumulate_unit_risk" in confirm_ids(st), confirm_ids(st)
item = next(i for i in t["confirm_items"] if i["id"] == "accumulate_unit_risk")
assert "万人次" in item["label"] + (item.get("detail") or "")
# 对照:区域外文字不变时不出
raw9b, fn9b = flow_workbook(SEP, 30, seed=196)
await client.delete(f"{API}/imports/{st['id']}")
st = await reupload(client, base["source_id"], edit_cells(raw9b, {"B32": "注:单位:人次"}), fn9b)
assert st["trial"]["status"] == "passed" and "accumulate_unit_risk" not in confirm_ids(st)
# ==========================================================================
# 版本页:清除原件(7.6)
# ==========================================================================
async def test_purging_raw_files_from_the_version_page(client, store):
"""7.6:清除原件要理由(422 reason_required);当前版本里的一期可以清,不在当前版本里的导入(被替换掉的那一期)也能清
(purge-raw 不看导入状态,1.5);清除后原件文件删掉、导入记录 raw_state=purged、署名照记;同一份内容在别的导入里的
引用一并清除(also_purged),引用它的未完成导入一并放弃;那几期构建的隔离文件一并删掉;清除不影响累积:下一期照常
追加(期 3 不重放历史,2.5)。"""
v = await august_september(client, seed=301)
sid = v["source_id"]
aug = await db_get(TableImport, v["aug"])
# 8 月的构建曾被改过、已恢复:隔离区里有它的一份
build = await db_get(TableBuild, aug.build_id)
q = store / "uploads" / "quarantine" / sid
q.mkdir(parents=True, exist_ok=True)
stale = q / f"{aug.build_id}-20260101T000000Z.db"
stale.write_bytes(b"quarantined copy")
# 引用同一份 8 月原件的未完成导入(原样再传、还没提交)
open_st = await reupload(client, sid, v["raw"], v["file_name"])
listed = {r["id"]: r for r in await imports_api(client, sid)}
assert listed[v["aug"]]["raw_open_stagings"] == 1
url = f"{API}/{sid}/imports/{v['aug']}/purge-raw"
resp = await client.post(url, json={"confirm": True, "reason": " "})
assert resp.status_code == 422 and resp.json()["code"] == "reason_required"
resp = await client.post(url, json={"confirm": True, "reason": REASON, "signed_by": "合成署名丙"})
assert resp.status_code == 200, resp.text
out = resp.json()
assert out["file_deleted"] is True and out["discarded_stagings"] == [open_st["id"]]
assert out["import"]["raw_state"] == "purged" and out["import"]["purged"]["signed_by"] == "合成署名丙"
assert not raw_store.raw_exists(aug.raw_sha256) and not stale.exists()
assert Path(build.db_path).is_file(), "构建库(数据)照旧,只清原件"
assert (await db_get(ImportStaging, open_st["id"])).status == "discarded"
resp = await client.post(url, json={"confirm": True, "reason": REASON})
assert resp.status_code == 409 and resp.json()["code"] == "raw_already_purged"
# 不在当前版本里的导入:替换掉 9 月之后,旧 9 月的原件照样能清
raw, fn = flow_workbook(SEP, 30, seed=304)
replaced = await add_period(client, sid, raw, fn)
assert (await db_get(TableImport, v["sep"])).status == "superseded"
resp = await client.post(f"{API}/{sid}/imports/{v['sep']}/purge-raw", json={"confirm": True, "reason": REASON})
assert resp.status_code == 200, resp.text
# 清除不影响累积
raw, fn = flow_workbook(OCT, 31, seed=303)
out = await add_period(client, sid, raw, fn)
assert out["parts"] == 3 and out["snapshot_id"] != replaced["snapshot_id"]
assert await count(client, v["name"], "日客流") == 92
# ==========================================================================
# 遗留项 L1:被改过的构建文件(能重建就重建,否则明确报错)
# ==========================================================================
def simple_sheet(seed: int) -> bytes:
"""一份最简单的表格(期 1 的简单上传用):表头加两行,数字随机。"""
import io
import random
rnd = random.Random(seed)
book = Workbook()
ws = book.active
ws.title = "甲表"
ws.append(["分区", "数量"])
ws.append(["分区甲", rnd.randint(1, 999)])
ws.append(["分区乙", rnd.randint(1, 999)])
buf = io.BytesIO()
book.save(buf)
return buf.getvalue()
def corrupt(path: Path, data: bytes = b"") -> None:
os.chmod(path, 0o644)
path.write_bytes(data)
def quarantined(store: Path, source_id: str) -> list[Path]:
return sorted((store / "uploads" / "quarantine" / source_id).rglob("*.db"))
async def test_l1_upload_restores_a_tampered_build_and_cleans_the_quarantine(client, store):
"""L1(/upload):同一文件重传、构建文件被改成 0 字节 → 201,build_restored=true,quarantine/<源 id>/ 里有原文件,
快照的文件哈希恢复;构建记录的哈希也被改掉时 → 409 build_conflict,指针不动;隔离文件超过 7 天被回收删除;删除
数据源后隔离目录被删。"""
name, raw = unique(), simple_sheet(201)
resp = await client.post(f"{API}/upload", files={"file": ("甲表.xlsx", raw, XLSX)}, data={"name": name})
assert resp.status_code == 201 and resp.json()["build_restored"] is False, resp.text
first = resp.json()
sid = first["source"]["id"]
snap = await db_get(SourceSnapshot, first["snapshot_id"])
corrupt(Path(snap.db_path))
resp = await client.post(f"{API}/upload", files={"file": ("甲表.xlsx", raw, XLSX)}, data={"name": name})
assert resp.status_code == 201, resp.text
assert resp.json()["build_restored"] is True
[moved] = quarantined(store, sid)
assert moved.stat().st_size == 0, "隔离的是被改过的那份"
assert raw_store.sha256_file(Path(snap.db_path)) == snap.db_sha256, "快照的文件哈希恢复"
# 记录的哈希也被改掉:无法用本次上传恢复 → 409 build_conflict,指针不动
imp = await db_get(TableImport, first["import_id"])
pointer = await current_id(sid)
corrupt(Path(snap.db_path), b"tampered again")
async with SessionLocal() as session:
await session.execute(update(TableBuild).where(TableBuild.id == imp.build_id).values(db_sha256="0" * 64))
await session.commit()
resp = await client.post(f"{API}/upload", files={"file": ("甲表.xlsx", raw, XLSX)}, data={"name": name})
assert resp.status_code == 409 and resp.json()["code"] == "build_conflict", resp.text
assert await current_id(sid) == pointer
# 隔离文件超过 QUARANTINE_DAYS 天:回收删掉
old = time.time() - (table_versions.QUARANTINE_DAYS + 1) * 86400
os.utime(moved, (old, old))
async with SessionLocal() as session:
await table_versions.gc(session)
assert not moved.exists()
# 删除数据源:隔离目录一并删掉
corrupt(Path(snap.db_path), b"")
async with SessionLocal() as session:
await session.execute(update(TableBuild).where(TableBuild.id == imp.build_id)
.values(db_sha256=snap.db_sha256))
await session.commit()
resp = await client.post(f"{API}/upload", files={"file": ("甲表.xlsx", raw, XLSX)}, data={"name": name})
assert resp.status_code == 201 and resp.json()["build_restored"] is True, resp.text
assert quarantined(store, sid)
resp = await client.delete(f"{API}/{sid}")
assert resp.status_code == 204, resp.text
assert not (store / "uploads" / "quarantine" / sid).exists()
async def test_l1_recipe_commit_restores_conflicts_and_quarantines_an_orphan(client, store):
"""L1(配方提交):8 月(每期替换)→ 9 月 → 8 月原文件再传(构建与第一次相同),第一次的构建文件被改成 0 字节 →
201、build_restored=true;构建记录的哈希也被改掉 → 409 build_conflict、指针不动;目标位置有一个没人登记、内容不同的
文件(孤儿)→ 隔离后 link,发布成功。"""
base = await first_import(client, seed=211, mode="replace")
sid = base["source_id"]
raw9, fn9 = flow_workbook(SEP, 30, seed=212)
await add_period(client, sid, raw9, fn9)
build = await db_get(TableBuild, (await db_get(TableImport, base["import_id"])).build_id)
corrupt(Path(build.db_path))
st = await reupload(client, sid, base["raw"], base["file_name"])
assert st["trial"]["status"] == "passed", st["trial"]["problems"]
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
assert resp.json()["build_restored"] is True and resp.json()["build_id"] == build.id
assert raw_store.sha256_file(Path(build.db_path)) == build.db_sha256
assert len(quarantined(store, sid)) == 1
# 记录的哈希也被改掉:9 月原文件再传,那份构建无法恢复
sep_build = await db_get(TableBuild, (await db_get(TableImport, (await snapshots(client, sid))[1]["parts"][0]
["import_id"])).build_id)
corrupt(Path(sep_build.db_path))
async with SessionLocal() as session:
await session.execute(update(TableBuild).where(TableBuild.id == sep_build.id).values(db_sha256="0" * 64))
await session.commit()
pointer = await current_id(sid)
st = await reupload(client, sid, raw9, fn9)
resp = await commit(client, st)
assert resp.status_code == 409 and resp.json()["code"] == "build_conflict", resp.text
assert await current_id(sid) == pointer
# 孤儿:10 月的构建位置上已经有一个没人登记的文件
raw10, fn10 = flow_workbook(OCT, 31, seed=213)
st = await reupload(client, sid, raw10, fn10)
build_id = (await db_get(ImportStaging, st["id"])).trial["build_id"]
orphan = table_versions.build_path(sid, build_id)
orphan.parent.mkdir(parents=True, exist_ok=True)
orphan.write_bytes(b"orphan nobody registered")
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
assert resp.json()["build_restored"] is False
assert raw_store.sha256_file(orphan) == (await db_get(TableBuild, build_id)).db_sha256
assert any(p.read_bytes() == b"orphan nobody registered" for p in quarantined(store, sid))
# ==========================================================================
# 遗留项 L2:改配方后回答延续
# ==========================================================================
async def test_l2_answers_survive_a_put_that_embodies_them(client):
"""L2:先回答 q_placeholder:·,再 PUT 一份体现了这个回答的配方 → answers 里仍有它,answers_dropped 为空;PUT 一份
与回答矛盾的配方(去掉占位符)→ answers_dropped 有它(带问题文字)。"""
raw, fn = flow_workbook(AUG, 31, seed=221)
st = await stage(client, raw, fn)
qid = "q_placeholder:·"
st = await answer(client, st["id"], {qid: {"value": "null"}})
st = await put_recipe(client, st["id"], st["recipe"])
assert st["answers"].get(qid, {}).get("value") == "null" and st["answers_dropped"] == []
rec = json.loads(json.dumps(st["recipe"], ensure_ascii=False))
for sheet in rec["sheets"]:
for block in sheet["blocks"]:
if isinstance(block.get("values"), dict):
block["values"]["placeholders"] = []
st = await put_recipe(client, st["id"], rec)
dropped = {d["id"]: d for d in st["answers_dropped"]}
assert qid in dropped and dropped[qid]["text"] and dropped[qid]["reason"] in ("changed", "gone"), dropped
assert qid not in st["answers"]
# ==========================================================================
# 遗留项 L3:沿用已确认的常量(3.3),经接口
# ==========================================================================
async def test_l3_keeping_a_confirmed_constant_follows_the_three_rules(client):
"""L3(3.3):D09 改了日间的分段标题。(b) 沿用:base 里同一分段的 pick「日间」仍是新标题的子串 → 静态校验通过;
反例:换成不在候选词里、base 里也没有的「时段」→ const_not_candidate;含数字的 pick → period_literal;(a) 候选词
「日间分」→ 通过(但 breaking);首次导入没有 base 时沿用不成立 → const_not_candidate;(c) 提交后下一期按重放校验
(origin=replay,没有 base),子串成立 → 通过。"""
base = await first_import(client, seed=231)
sid = base["source_id"]
raw, fn = DRIFT_CASES["D09"].build(232)
st = await reupload(client, sid, raw, fn)
def with_title(rec: dict[str, Any], pick: str) -> dict[str, Any]:
rec = json.loads(json.dumps(rec, ensure_ascii=False))
seg = next(s for s in rec["sheets"][0]["blocks"][0]["segments"] if s["id"] == "日间")
seg["locate"]["title"] = "日间分时段客流(人次)"
seg["const"]["时段类别"]["pick"] = pick
return rec
codes = lambda out: {p["code"] for p in out["recipe_problems"]} # noqa: E731
out = await put_recipe(client, st["id"], with_title(st["recipe"], "时段"))
assert "const_not_candidate" in codes(out), out["recipe_problems"]
out = await put_recipe(client, st["id"], with_title(st["recipe"], "9月"))
assert "period_literal" in codes(out), out["recipe_problems"]
out = await put_recipe(client, st["id"], with_title(st["recipe"], "日间分"))
assert out["recipe_problems"] == [], "(a) 候选词照旧放行"
kept = with_title(st["recipe"], "日间")
out = await put_recipe(client, st["id"], kept)
assert out["recipe_problems"] == [], "(b) 沿用:base 里同一分段的 pick,仍是新标题的子串"
st = await trial(client, out["id"])
assert st["trial"]["status"] == "passed", st["trial"]["problems"]
assert not [i for i in confirm_ids(st) if i.startswith("breaking:")], confirm_ids(st)
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
# 首次导入没有 base:沿用不成立
other = await stage(client, raw, fn)
out = await put_recipe(client, other["id"], rename_wide(kept, "日客流", "日客流"))
assert "const_not_candidate" in codes(out), out["recipe_problems"]
# (c) 下一期按重放校验:子串成立就放行
raw10, fn10 = flow_workbook(OCT, 31, seed=233, variant="D09")
st = await reupload(client, sid, raw10, fn10)
assert st["recipe_problems"] == [] and st["trial"]["status"] == "passed", (st["recipe_problems"],
st["trial"]["problems"])
# ==========================================================================
# 遗留项 L4:排除的行(rows_excluded);列表只剩表头必须确认(empty_block)
# ==========================================================================
def list_book(rows: list[tuple[Any, ...]], *, sheet: str = "明细", extra: list[tuple[Any, ...]] | None = None
) -> tuple[bytes, str]:
"""一张列表:表头「地区、产品、金额」加 rows(None 行即空行),再接 extra。"""
book = Workbook()
ws = book.active
ws.title = sheet
for row in [("地区", "产品", "金额"), *rows, *(extra or [])]:
ws.append(list(row))
return excel_saved(save(book), sheet, {}), f"{sheet}.xlsx"
def excluded_reasons(doc: dict[str, Any]) -> set[str]:
return {x["reason"] for x in doc["rows_excluded"]}
async def _commit_and_manifest(client, st: dict[str, Any]) -> dict[str, Any]:
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
imp = await db_get(TableImport, resp.json()["import_id"])
return artifact_store.load(imp.manifest_artifact)
async def test_l4_rows_excluded_reach_the_receipt_and_the_import_manifest(client):
"""L4:c06 exclude → hidden_excluded;c08 gap skip → blank_skipped;D21 忽略 → ignored_rows(A6b 已提交过一次,这里
再看清单);列表停止后的文字行 → after_stop。trial.receipt.rows_excluded 经接口给出,导入清单的 receipt 里也有。"""
# c06:隐藏行不导入
raw, fn = lab.c06()
st = await stage(client, raw, fn)
q = next(q for q in st["questions"] if q["id"].startswith("q_hidden:"))
st = await answer(client, st["id"], {q["id"]: {"value": "exclude"}})
st = await trial(client, st["id"])
assert st["trial"]["status"] == "passed", st["trial"]["problems"]
assert "hidden_excluded" in excluded_reasons(st["trial"]["receipt"])
assert "hidden_excluded" in excluded_reasons((await _commit_and_manifest(client, st))["receipt"])
# c08 gap:中间空行跳过
raw, fn = lab.c08("gap")
st = await trial(client, (await stage(client, raw, fn))["id"])
assert st["trial"]["status"] == "passed"
gap = [x for x in st["trial"]["receipt"]["rows_excluded"] if x["reason"] == "blank_skipped"]
assert gap and gap[0]["rows"] == [[4, 4]], st["trial"]["receipt"]["rows_excluded"]
assert "blank_skipped" in excluded_reasons((await _commit_and_manifest(client, st))["receipt"])
# 列表停止后的文字行:空行之后还有两格文字(像数据,没导入)→ rows_after_stop 需确认,记 after_stop
raw, fn = list_book([("分区甲", "产品甲", 10), ("分区乙", "产品乙", 20), (None, None, None),
("补充说明", "见附件甲", None)])
st = await stage(client, raw, fn)
rec = json.loads(json.dumps(st["recipe"], ensure_ascii=False))
rec["sheets"][0]["blocks"][0]["rows"]["blank_rows"] = "stop"
st = await trial(client, (await put_recipe(client, st["id"], rec))["id"])
assert st["trial"]["status"] == "passed", st["trial"]["problems"]
assert problems(st, "rows_after_stop") and "after_stop" in excluded_reasons(st["trial"]["receipt"])
assert "after_stop" in excluded_reasons((await _commit_and_manifest(client, st))["receipt"])
# D21:按行标签忽略
done = await _fix_case(client, "D21", seed=241)
assert "ignored_rows" in excluded_reasons(done["st"]["trial"]["receipt"])
assert "ignored_rows" in excluded_reasons((await _commit_and_manifest(client, done["st"]))["receipt"])
async def test_l4_a_list_that_drops_to_only_its_header_must_be_confirmed(client):
"""遗留项 4 原意(期 2 AU-1 修复③):上一期有数据行、本期只剩表头的列表重传,必须出 empty_block:<块> 必勾项,
不勾不能启用;勾上之后这张表是 0 行。"""
raw, fn = list_book([("分区甲", "产品甲", 10), ("分区乙", "产品乙", 20), ("分区丙", "产品丙", 30)])
st = await trial(client, (await stage(client, raw, fn))["id"])
assert st["trial"]["status"] == "passed", st["trial"]["problems"]
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
sid, name = resp.json()["source"]["id"], resp.json()["source"]["name"]
table = st["recipe"]["sheets"][0]["blocks"][0]["table"]
block = st["recipe"]["sheets"][0]["blocks"][0]["id"]
raw, fn = list_book([])
st = await reupload(client, sid, raw, fn)
t = st["trial"]
assert t["status"] == "passed", t["problems"]
assert problems(st, "list_empty"), t["problems"]
item = f"empty_block:{block}"
items = {i["id"]: i for i in t["confirm_items"]}
assert item in items and items[item]["required"] is True, list(items)
resp = await commit(client, st, confirmations=[i for i in confirm_ids(st) if i != item])
assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" and item in resp.json()["detail"]
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
assert await count(client, name, table) == 0
# ==========================================================================
# 遗留项 L5:另一个进程持有存储守卫
# ==========================================================================
_HOLD = """
import fcntl, os, sys
fd = os.open(sys.argv[1], os.O_RDWR | os.O_CREAT, 0o600)
fcntl.flock(fd, fcntl.LOCK_EX)
print("locked", flush=True)
sys.stdin.read()
"""
@pytest.mark.sqlite_only # 文件锁守卫只在 SQLite(单进程)时有:PostgreSQL 可以多进程,版本存储靠跨进程的锁(app/cluster/mutex)
async def test_l5_another_process_holding_the_store_guard(client, store):
"""L5:子进程先占住 .store.lock → 本进程 acquire_store_guard() 返回 False;本进程调 startup:事先放好的临时文件和孤儿
构建库一个都没删;/upload、stage、trial、commit、activate 返回 503 store_unavailable,GET 接口照常;过期处理不抛;
子进程退出后重新调用返回 True,startup 照常清理。"""
v = await august_september(client, seed=251)
sid = v["source_id"]
raw10, fn10 = flow_workbook(OCT, 31, seed=252)
st = await reupload(client, sid, raw10, fn10)
half = table_versions.build_path(sid, "1" * 64).with_name(f"{'1' * 64}.db{raw_store.TMP_MARK}zzz")
half.write_bytes(b"half written")
orphan = table_versions.build_path(sid, "2" * 64)
orphan.write_bytes(b"orphan build")
proc = subprocess.Popen([sys.executable, "-c", _HOLD, str(settings.uploads_dir / ".store.lock")],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True)
try:
assert proc.stdout.readline().strip() == "locked"
assert table_versions.acquire_store_guard() is False
ran: list[str] = []
real_gc = table_versions.gc
async def spy_gc(session):
ran.append("gc")
table_versions.gc = spy_gc
try:
async with SessionLocal() as session:
await table_versions.startup(session)
finally:
table_versions.gc = real_gc
assert ran == [] and half.exists() and orphan.exists(), "拿不到守卫:startup 一个文件都不删,不回收"
def refused(resp) -> None:
assert resp.status_code == 503, resp.text
assert resp.json()["code"] == "store_unavailable"
refused(await client.post(f"{API}/upload", files={"file": ("甲表.xlsx", simple_sheet(253), XLSX)},
data={"name": unique()}))
refused(await client.post(f"{API}/imports/stage", files={"file": (fn10, raw10, XLSX)},
data={"name": unique()}))
refused(await client.post(f"{API}/imports/{st['id']}/trial", json={}))
refused(await commit(client, st))
refused(await client.post(f"{API}/{sid}/snapshots/{v['s1']}/activate",
json={"confirm": True, "expected_current_snapshot_id": v["s2"]}))
assert (await client.get(API)).status_code == 200
assert (await client.get(f"{API}/{sid}/snapshots")).status_code == 200
assert (await client.get(f"{API}/{sid}/imports")).status_code == 200
assert (await client.get(f"{API}/imports/{st['id']}")).status_code == 200
assert await count(client, v["name"], "日客流") == 61
async with SessionLocal() as session:
assert await table_versions.expire_stagings(session) == []
assert await current_id(sid) == v["s2"]
finally:
proc.stdin.close()
proc.wait(timeout=10)
try:
assert table_versions.acquire_store_guard() is True
async with SessionLocal() as session:
await table_versions.startup(session)
assert not half.exists() and not orphan.exists()
resp = await commit(client, st)
assert resp.status_code == 201 and resp.json()["parts"] == 3, resp.text
finally:
table_versions._release_guard()
# ==========================================================================
# 11.4 数据变异
# ==========================================================================
def shifted_period(raw: bytes, start, days: int) -> bytes:
"""把 9 月 D01 的统计期(B2)和日期表头(第 4 行)一起改成从 start 起 days 天:数字照旧,只挪统计期。"""
import datetime as dt
from openpyxl.utils import get_column_letter
end = start + dt.timedelta(days=days - 1)
edits: dict[str, Any] = {"B2": f"统计时间范围:{start.year}年{start.month}月{start.day}日至"
f"{end.year}年{end.month}月{end.day}日"}
for i in range(days):
d = start + dt.timedelta(days=i)
edits[f"{get_column_letter(3 + i)}4"] = f"{d.month}月{d.day}日"
return edit_cells(raw, edits)
async def test_mutation_september_period_overlapping_august_by_one_day(client):
"""11.4 第 1 行:9 月 D01 的 B2 改成 08-31 至 09-29(与 8 月重叠一天)→ period_overlap,当前快照不动。日期表头跟着挪
(否则先撞上「日期不在统计期内」的结构问题,同样拒收,只是原因不同,见下半)。"""
base = await first_import(client, seed=261)
raw, fn = flow_workbook(SEP, 30, seed=262)
st = await reupload(client, base["source_id"], shifted_period(raw, date_of(2026, 8, 31), 30), fn)
assert st["trial"]["status"] == "rejected"
[overlap] = problems(st, "period_overlap")
assert "2026-08-31 至 2026-09-29" in overlap["message"] and "2026-08-01 至 2026-08-31" in overlap["message"]
resp = await commit(client, st)
assert resp.status_code == 409 and resp.json()["code"] == "trial_not_passed"
await client.delete(f"{API}/imports/{st['id']}")
# 只改 B2、表头不动:9 月 30 日落在统计期之外,执行器先拒收(axis_year_missing),同样不动当前快照
bad = edit_cells(raw, {"B2": "统计时间范围:2026年8月31日至2026年9月29日"})
st = await reupload(client, base["source_id"], bad, fn)
assert st["trial"]["status"] == "rejected" and problems(st, "axis_year_missing")
assert await current_id(base["source_id"]) == base["s1"]
async def test_mutation_trial_files_tampered_or_deleted_before_commit(client):
"""11.4 第 2、3 行:本期构建的试运行库在提交前被改字节 → 409 trial_tampered;并集试运行库被改 → 409 union_tampered;
并集试运行库被删 → 409 trial_required,暂存区退回 drafting。当前快照都不动。"""
base = await first_import(client, seed=271)
sid = base["source_id"]
raw, fn = flow_workbook(SEP, 30, seed=272)
st = await reupload(client, sid, raw, fn)
row = await db_get(ImportStaging, st["id"])
main = Path(row.trial_path)
with open(main, "ab") as f:
f.write(b"\0")
resp = await commit(client, st)
assert resp.status_code == 409 and resp.json()["code"] == "trial_tampered", resp.text
st = await trial(client, st["id"])
union = table_versions.union_trial_path(Path((await db_get(ImportStaging, st["id"])).trial_path))
with open(union, "ab") as f:
f.write(b"\0")
resp = await commit(client, st)
assert resp.status_code == 409 and resp.json()["code"] == "union_tampered", resp.text
st = await trial(client, st["id"])
union = table_versions.union_trial_path(Path((await db_get(ImportStaging, st["id"])).trial_path))
union.unlink()
resp = await commit(client, st)
assert resp.status_code == 409 and resp.json()["code"] == "trial_required", resp.text
assert (await db_get(ImportStaging, st["id"])).status == "drafting"
assert await current_id(sid) == base["s1"]
async def test_mutation_published_part_or_its_period_column_tampered(client):
"""11.4 第 4、5 行:已发布的某一期构建库被改字节,再上传下一期 → 试运行 rejected、part_tampered;某一期导入记录的
period_start 库列被改 → rejected、part_tampered(统计期取自导入清单)。"""
v = await august_september(client, seed=281)
sid = v["source_id"]
build = await db_get(TableBuild, (await db_get(TableImport, v["aug"])).build_id)
path = Path(build.db_path)
original = path.read_bytes()
corrupt(path, original[:-1] + bytes([original[-1] ^ 1]))
raw, fn = flow_workbook(OCT, 31, seed=282)
st = await reupload(client, sid, raw, fn)
assert st["trial"]["status"] == "rejected" and problems(st, "part_tampered"), st["trial"]["problems"]
corrupt(path, original)
os.chmod(path, 0o444)
async with SessionLocal() as session:
await session.execute(update(TableImport).where(TableImport.id == v["sep"]).values(period_start="2026-09-02"))
await session.commit()
st = await trial(client, st["id"])
assert st["trial"]["status"] == "rejected"
tampered = problems(st, "part_tampered")
assert tampered and "统计期" in tampered[0]["message"], st["trial"]["problems"]
assert await current_id(sid) == v["s2"]
async def test_mutation_snapshot_hash_column_rewritten_to_match_a_swapped_file(client):
"""11.4 第 6 行:并集快照的库文件被换掉,source_snapshots.db_sha256 库列也改成与新文件一致的假值(文件哈希核对照样
通过)→ 查询时拿快照清单交叉核对,报 snapshot_tampered(版本清单不一致),不会查到被换掉的数据。"""
v = await august_september(client, seed=291)
snap = await db_get(SourceSnapshot, v["s2"])
path = Path(snap.db_path)
os.chmod(path, 0o644)
with closing(sqlite3.connect(path)) as conn:
conn.execute('UPDATE "日客流" SET "全日客流" = 0')
conn.commit()
table_versions.settle_journal(path)
fake = raw_store.sha256_file(path)
async with SessionLocal() as session:
await session.execute(update(SourceSnapshot).where(SourceSnapshot.id == v["s2"]).values(db_sha256=fake))
await session.commit()
from app.data.engine import engines
await engines.invalidate(v["source_id"])
resp = await client.post(f"/api/tools/db_query__{v['name']}/run",
json={"args": {"sql": 'SELECT SUM("全日客流") FROM "日客流"'}})
assert resp.status_code == 200, resp.text
out = resp.json()
# 工具把数据不可用写成给模型看的结果(不抛),内容是拒绝查询的原因,没有数
assert "查询失败(数据不可用)" in out["result"] and "版本清单不一致" in out["result"], out
with pytest.raises(table_versions.SnapshotManifestMismatch) as err:
async with SessionLocal() as session:
await table_versions.resolve_source(session, await session.get(DataSource, v["source_id"]))
assert err.value.code == "snapshot_tampered"
# ==========================================================================
# WP-8 修补 3:跳过空行的列表,表下隔着空行的说明不是数据(经真管线)
# ==========================================================================
async def test_fix3_note_under_a_list_with_a_middle_gap_stays_outside(client):
"""列表中间有一个空行(起草成 blank_rows=skip),表下再隔一个空行有「注:……」:试运行 4 行,「注:……」进区域外文字,
跳过的空行只有中间那一行;提交后表里没有首列是「注:……」的记录。框选整块(含中间空行)重放与框一致。曾静默多读进
一行首列是说明、其余为空值的记录(p3w1a/fix-2.md「需要你决定的事项」第 5 条)。"""
raw, fn = list_book([("分区甲", "产品甲", 10), ("分区甲", "产品乙", 20), (None, None, None),
("分区乙", "产品甲", 30), ("分区乙", "产品乙", 40)],
extra=[(None, None, None), ("注:以上为初步统计,以终稿为准。", None, None)])
st = await stage(client, raw, fn)
block = st["recipe"]["sheets"][0]["blocks"][0]
assert block["rows"]["blank_rows"] == "skip"
sel = {"sheet": "明细", "ref": "A1:C6", "as": "list", "options": {"header_rows": 1, "bottom": "box"}}
pv = (await preview(client, st["id"], {"selection": sel})).json()
assert pv["ok"] is True and pv["replay"]["match"] is True, (pv["problems"], pv["replay"])
st = await trial(client, st["id"])
t = st["trial"]
assert t["status"] == "passed", t["problems"]
assert rows_of(st)[block["table"]] == 4
assert ("明细!A8", "注:以上为初步统计,以终稿为准。") in [(o["cell"], o["text"]) for o in t["receipt"]["outside_text"]]
# 列表停在哪一行、因为什么也看得出来(评审意见 4):说明那一行记进排除的行(after_stop,1 格,锚点是说明文字);
# 像说明,不出 rows_after_stop
note = "注:以上为初步统计,以终稿为准。"
assert [(x["reason"], x["rows"], x["cells"], x["anchor"]) for x in t["receipt"]["rows_excluded"]] == [
("blank_skipped", [[4, 4]], 0, None), ("after_stop", [[8, 8]], 1, note)]
assert t["receipt"]["blank_rows_skipped"] == 1
assert not problems(st, "rows_after_stop") and not [i for i in confirm_ids(st) if i.startswith("rows_after_stop")]
assert "空行之后" not in t["notes"][block["table"]]["comment"]
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
name = resp.json()["source"]["name"]
doc = artifact_store.load((await db_get(TableImport, resp.json()["import_id"])).manifest_artifact)
assert ("after_stop", [[8, 8]], note) in [(x["reason"], x["rows"], x["anchor"]) for x in doc["receipt"]["rows_excluded"]]
assert await count(client, name, block["table"]) == 4
assert await count(client, name, block["table"], ' WHERE "地区" LIKE \'注%\'') == 0
def _lone_tail_book(seed: int) -> tuple[bytes, str]:
"""起草成 blank_rows=skip 的列表:中间一个空行,表尾隔一个空行是一条只填了首列的「分区丁」(评审意见 4 的探针)。
金额随 seed 变,地区和产品不变。"""
import random
rnd = random.Random(seed)
return list_book([("分区甲", "产品甲", rnd.randint(10, 99)), ("分区乙", "产品甲", rnd.randint(10, 99)),
(None, None, None), ("分区丙", "产品甲", rnd.randint(10, 99)), (None, None, None),
("分区丁", None, None)])
async def test_fix3_a_lone_first_column_row_that_is_not_a_note_must_be_ticked_every_time(client):
"""评审意见 4:表尾只填了首列的「分区丁」照旧当表的下边界(与起草器一致,不导入),但它不像说明:出 confirm 类的
rows_after_stop(「第 7 行只有首列文字,已当作表下说明,没有导入」),必勾项 rows_after_stop:<块>,不勾不能提交;
排除的行记 after_stop(锚点「分区丁」)。下一期重放同样的表尾,照样要勾。曾只出现在区域外文字里:确认项只有
blank_skip 和 mode,排除的行里也没有它。"""
raw, fn = _lone_tail_book(711)
st = await stage(client, raw, fn)
block = st["recipe"]["sheets"][0]["blocks"][0]
assert block["rows"]["blank_rows"] == "skip"
st = await trial(client, st["id"])
t = st["trial"]
assert t["status"] == "passed", t["problems"]
assert rows_of(st)[block["table"]] == 3
found = problems(st, "rows_after_stop")
assert len(found) == 1 and found[0]["category"] == "confirm" and found[0]["cells"] == ["明细!A7"], found
assert "第 7 行只有首列文字" in found[0]["message"] and "已当作表下说明,没有导入" in found[0]["message"]
assert ("after_stop", [[7, 7]], 1, "分区丁") in [(x["reason"], x["rows"], x["cells"], x["anchor"])
for x in t["receipt"]["rows_excluded"]]
item = f"rows_after_stop:{block['id']}"
items = {i["id"]: i for i in t["confirm_items"]}
assert item in items and items[item]["required"] is True, list(items)
assert "第 7 行只有首列文字" in items[item]["label"]
assert "原表在空行之后另有未导入的文字行" in t["notes"][block["table"]]["comment"]
resp = await commit(client, st, confirmations=[i for i in confirm_ids(st) if i != item])
assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" and item in resp.json()["detail"]
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
sid, name = resp.json()["source"]["id"], resp.json()["source"]["name"]
assert await count(client, name, block["table"]) == 3
assert await count(client, name, block["table"], ' WHERE "地区" = \'分区丁\'') == 0
# 下一期(数字不同、表尾照旧):同样出这一项,不勾照样不能提交
raw, fn = _lone_tail_book(712)
st = await reupload(client, sid, raw, fn)
t = st["trial"]
assert t["status"] == "passed", t["problems"]
assert item in confirm_ids(st), confirm_ids(st)
resp = await commit(client, st, confirmations=[i for i in confirm_ids(st) if i != item])
assert resp.status_code == 422 and resp.json()["code"] == "confirm_required" and item in resp.json()["detail"]
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
assert await count(client, name, block["table"]) == 3
async def test_fix3_a_build_from_the_old_engine_is_not_reused_for_the_same_file(client, monkeypatch):
"""评审意见 1:修补 3 改了「列表 + 跳过空行」的执行结果,RECIPE_ENGINE_VER 没升(升了交叉表配方的构建全部换 id),
改为这类配方的构建 id 带执行器语义标记(table_versions.recipe_engine_semantics)。先用修补前的执行器(_note_after_blank
恒 False、构建 id 不带标记)发布一次:5 行,其中一条首列是「注:……」。恢复后把同一文件换个名字重传:试运行 4 行,
提交不复用旧构建(换了 id),发布出去的库就是这次试运行的库,表里 4 行。曾复用旧构建:核对的 4 行、发布的 5 行。"""
from app.data import recipe_engine
note = "注:以上为初步统计,以终稿为准。"
raw, fn = list_book([("分区甲", "产品甲", 31), ("分区甲", "产品乙", 47), (None, None, None),
("分区乙", "产品甲", 52), ("分区乙", "产品乙", 68)],
extra=[(None, None, None), (note, None, None)])
with monkeypatch.context() as old:
old.setattr(recipe_engine._ListRun, "_note_after_blank", lambda self, inb: False)
old.setattr(table_versions, "recipe_engine_semantics", lambda recipe: {})
st = await stage(client, raw, fn)
block = st["recipe"]["sheets"][0]["blocks"][0]
assert block["rows"]["blank_rows"] == "skip"
st = await trial(client, st["id"])
assert st["trial"]["status"] == "passed", st["trial"]["problems"]
assert rows_of(st)[block["table"]] == 5
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
first = resp.json()
sid, name, table = first["source"]["id"], first["source"]["name"], block["table"]
assert await count(client, name, table, ' WHERE "地区" LIKE \'注%\'') == 1
old_build = await db_get(TableBuild, first["build_id"])
assert "engine_semantics" not in old_build.options
st = await reupload(client, sid, raw, "另一个名字.xlsx")
t = st["trial"]
assert t["status"] == "passed", t["problems"]
assert rows_of(st)[table] == 4
resp = await commit(client, st)
assert resp.status_code == 201, resp.text
out = resp.json()
assert out["build_reused"] is False and out["build_id"] != first["build_id"]
build = await db_get(TableBuild, out["build_id"])
assert build.db_sha256 == t["receipt"]["db_sha256"], "发布出去的库就是这次试运行、用户核对过的那一份"
assert build.options["engine_semantics"] == {table_versions.LIST_NOTE_AFTER_BLANK: 1}
assert build.options_sha256 == table_versions.sha_json(build.options)
assert await count(client, name, table) == 4
assert await count(client, name, table, ' WHERE "地区" LIKE \'注%\'') == 0