AgentLab/backend/tests/test_recipe_accumulate.py

"""按期累积（期 3，WP-4 recipe_accumulate）：配方演进的分档、累积计划、并集的物化。

要守住的几件事（P3-SPEC 2.4–2.7、11.3 的 WP-4 部分）：
- 计划的判定表逐行成立：不兼容即 restart，而且优先于部分重叠的 rejected；相等的统计期是「替换该期」、要确认；
  模式切换时当前那一期有统计期、满足资格、配方兼容才作为第一期；
- 退役只对「这次才退役的」出确认项：连续三期只有第一次是 retired_new；退役名和新名只差大小写按不兼容；
  没给当前表结构时按当前各期推（新增列照样报得出）；restart、rejected 不带各期维度取值的差异；
- 并集 id 带上退役列的组成和顺序（顺序进库哈希）；retired_columns 先拼早先退役的、再拼这次才退役的；
- 物化：行数、rowid 区间、确定性、U1–U3 各自能失败、退役列、单期物化、组合表哈希；各期库被改过就 part_tampered，
  rowid 不连续就 union_failed；先写临时文件再 os.replace。

夹具：8 月、9 月用合成客流表（tests/fixtures/xlsx/flow.py，假名、随机数）按参考配方经执行器建库；其余变体的
期库用 sqlite3 直接建，数字是手写的假数。
"""
from __future__ import annotations

import copy
import datetime as dt
import json
import os
import shutil
import sqlite3
from functools import lru_cache
from pathlib import Path
from typing import Any

import pytest

from app.data import raw_store, recipe_accumulate as ra, recipe_engine, table_versions, xlsx_scan
from app.data.recipe_types import (
    UNION_VER, ColumnOut, PartInfo, Recipe, UnionPart, derive_tables,
)
from tests.fixtures.xlsx.flow import flow_workbook

FLOW = json.loads((Path(__file__).parent / "fixtures" / "recipes" / "flow_recipe.json").read_text(encoding="utf-8"))


def recipe(edit=None, *, mode: str = "accumulate") -> Recipe:
    data = copy.deepcopy(FLOW)
    data["mode"] = mode
    if edit is not None:
        edit(data)
    return Recipe.model_validate(data)


def _segments(data: dict) -> list[dict]:
    return data["sheets"][0]["blocks"][0]["segments"]


def no_zone_b(data: dict) -> None:
    """去掉 measures 的分区乙（R1 一并去掉：sum_eq 至少两个成员）。"""
    seg = _segments(data)[0]
    seg["labels"]["expect"].remove("分区乙（人次）")
    del seg["measures"]["分区乙（人次）"]
    del data["tables"][0]["units"]["分区乙"]
    data["relations"] = [r for r in data["relations"] if r["id"] != "R1"]


def with_zone_c(data: dict) -> None:
    seg = _segments(data)[0]
    seg["labels"]["expect"].append("分区丙（人次）")
    seg["measures"]["分区丙（人次）"] = "分区丙"
    data["tables"][0]["units"]["分区丙"] = "人次"


def unit_changed(data: dict) -> None:
    data["tables"][0]["units"]["分区甲"] = "万人次"


def no_seven(data: dict) -> None:
    """D04 的修法：日间去掉「7-8」。"""
    _segments(data)[1]["labels"]["expect"].remove("7-8")


def no_covers(data: dict) -> None:
    data["sheets"][0]["blocks"][0]["axis"]["checks"] = ["contiguous"]


REF = recipe()
AUG = ("2026-08-01", "2026-08-31")
SEP = ("2026-09-01", "2026-09-30")
OCT = ("2026-10-01", "2026-10-31")
NOV = ("2026-11-01", "2026-11-30")


def part(period: tuple[str | None, str | None], *, r: Recipe | None = REF, seq: int = 1, name: str = "",
         rows: dict[str, int] | None = None) -> PartInfo:
    return PartInfo(import_id=f"imp{seq}", seq=seq, start=period[0], end=period[1], build_id=f"b{seq}",
                    db_path="", db_sha256="", manifest_db_sha256=None, recipe_id=f"r{seq}", recipe=r,
                    recipe_sha256=None, raw_state="kept", file_name=name or f"第{seq}期.xlsx", manifest_artifact=None,
                    rows=rows or {})


def tables_of(r: Recipe) -> dict[str, list[ColumnOut]]:
    return derive_tables(r)[0]


def plan(new: Recipe, period, *, kind: str = "reupload", mode: str | None = "accumulate",
         snap: str | None = "recipe", parts: list[PartInfo] = (), current=None, retired_status=None,
         source: str = "cells"):
    return ra.plan_accumulate(new_recipe=new, new_period=period, period_source=source, staging_kind=kind,
                              current_mode=mode, current_snapshot_kind=snap, parts=list(parts),
                              current_tables=current if current is not None else tables_of(REF),
                              retired_status=retired_status)


# ==========================================================================
# 累积计划：判定表
# ==========================================================================


def test_replace_recipe_stays_replace_without_mode_switch():
    p = plan(recipe(mode="replace"), SEP, mode="replace", parts=[part(AUG)])
    assert p.action == "replace" and p.mode == "replace" and p.mode_switch is None
    assert [x["new"] for x in p.parts] == [True] and p.dropped == []


def test_accumulate_to_replace_drops_every_period():
    p = plan(recipe(mode="replace"), OCT, parts=[part(AUG), part(SEP, seq=2)])
    assert p.action == "replace" and p.mode_switch == "accumulate->replace"
    assert [(d["start"], d["import_id"]) for d in p.dropped] == [("2026-08-01", "imp1"), ("2026-09-01", "imp2")]
    assert [x["new"] for x in p.parts] == [True]


@pytest.mark.parametrize("kind,mode,snap,parts", [
    ("first", None, None, []),
    ("switch", None, "simple", [part((None, None))]),
    ("reupload", "replace", None, []),
])
def test_first_import_and_switch_from_simple_start_from_this_period(kind, mode, snap, parts):
    p = plan(REF, AUG, kind=kind, mode=mode, snap=snap, parts=parts)
    assert p.action == "first" and p.mode == "accumulate"
    assert [(x["start"], x["end"], x["new"], x["import_id"]) for x in p.parts] == [(*AUG, True, None)]
    assert p.union is None


def test_switch_from_replace_keeps_the_current_period_as_the_first():
    old = recipe(mode="replace")
    p = plan(REF, SEP, mode="replace", parts=[part(AUG, r=old, rows={"日客流": 31})])
    assert p.action == "append" and p.mode_switch == "replace->accumulate"
    assert [(x["start"], x["new"]) for x in p.parts] == [("2026-08-01", False), ("2026-09-01", True)]
    assert p.parts[0]["rows"] == {"日客流": 31} and p.parts[0]["blockers"] == []
    assert p.dropped == [] and p.reason is None


def test_switch_from_replace_without_a_period_starts_over():
    p = plan(REF, SEP, mode="replace", parts=[part((None, None), r=recipe(mode="replace"))])
    assert p.action == "first" and p.mode_switch == "replace->accumulate"
    assert "没有统计期" in p.reason and [d["import_id"] for d in p.dropped] == ["imp1"]


def test_switch_from_replace_whose_recipe_is_not_eligible_starts_over():
    p = plan(REF, SEP, mode="replace", parts=[part(AUG, r=recipe(no_covers, mode="replace"))])
    assert p.action == "first" and p.mode_switch == "replace->accumulate"
    assert "不满足按期累积的要求" in p.reason and "恰好覆盖统计期" in p.reason
    assert p.dropped[0]["blockers"]


def test_switch_from_replace_with_an_incompatible_recipe_restarts():
    p = plan(recipe(unit_changed), SEP, mode="replace", parts=[part(AUG, r=recipe(mode="replace"))])
    assert p.action == "restart" and p.mode_switch == "replace->accumulate"
    assert p.change == "semantic" and any("万人次" in s for s in p.semantic)
    assert [x["new"] for x in p.parts] == [True] and len(p.dropped) == 1


def test_switch_from_replace_on_the_same_period_replaces_it():
    p = plan(REF, AUG, mode="replace", parts=[part(AUG, r=recipe(mode="replace"))])
    assert p.action == "replace_period" and p.replaces["import_id"] == "imp1"
    assert [x["new"] for x in p.parts] == [True]


def test_append_a_later_period():
    p = plan(REF, OCT, parts=[part(AUG), part(SEP, seq=2)])
    assert p.action == "append" and p.change == "same" and p.mode_switch is None
    assert [x["start"] for x in p.parts] == ["2026-08-01", "2026-09-01", "2026-10-01"]
    assert [x["new"] for x in p.parts] == [False, False, True]
    assert p.gaps == [] and p.backfill is False and p.overlaps == [] and p.replaces is None


def test_backfill_an_earlier_period_is_sorted_in():
    p = plan(REF, SEP, parts=[part(AUG), part(OCT, seq=2)])
    assert p.action == "append" and p.backfill is True
    assert [x["start"] for x in p.parts] == ["2026-08-01", "2026-09-01", "2026-10-01"]
    assert p.gaps == [] and p.period == {"start": "2026-09-01", "end": "2026-09-30", "source": "cells"}


def test_backfill_with_a_period_typed_by_hand_carries_the_source():
    """补传早期、本期统计期是人工录入：差异卡要改为需确认（2.3），计划把来源带给 WP-3。"""
    p = plan(REF, AUG, parts=[part(SEP), part(OCT, seq=2)], source="human")
    assert p.action == "append" and p.backfill is True and p.period["source"] == "human"
    assert [x["new"] for x in p.parts] == [True, False, False]


def test_gap_between_periods_is_reported():
    p = plan(REF, NOV, parts=[part(AUG), part(SEP, seq=2)])
    assert p.action == "append" and p.gaps == [{"start": "2026-10-01", "end": "2026-10-31"}]


def test_same_period_is_a_replace_candidate():
    p = plan(REF, SEP, parts=[part(AUG), part(SEP, seq=2)])
    assert p.action == "replace_period" and p.replaces["import_id"] == "imp2"
    assert [(x["start"], x["new"]) for x in p.parts] == [("2026-08-01", False), ("2026-09-01", True)]


def test_partial_overlap_is_rejected_and_listed():
    p = plan(REF, ("2026-08-15", "2026-09-14"), parts=[part(AUG), part(SEP, seq=2)])
    assert p.action == "rejected"
    assert [o["import_id"] for o in p.overlaps] == ["imp1", "imp2"]
    # 当前版本原样不动
    assert [x["import_id"] for x in p.parts] == ["imp1", "imp2"]
    msg = ra.overlap_message(p.period, p.overlaps)
    assert "2026-08-15 至 2026-09-14" in msg and "2026-08-01 至 2026-08-31" in msg
    assert "2026-09-01 至 2026-09-30" in msg and "部分重叠" in msg


def test_one_day_overlap_is_rejected():
    p = plan(REF, ("2026-08-31", "2026-09-29"), parts=[part(AUG)])
    assert p.action == "rejected" and [o["import_id"] for o in p.overlaps] == ["imp1"]


def test_incompatible_change_restarts_even_when_periods_overlap():
    """restart 优先于 rejected（评审一-m12）：重叠的各期写进 overlaps 只作提示。"""
    p = plan(recipe(unit_changed), ("2026-08-15", "2026-09-14"), parts=[part(AUG), part(SEP, seq=2)])
    assert p.action == "restart" and [o["import_id"] for o in p.overlaps] == ["imp1", "imp2"]
    assert "不兼容" in p.reason and len(p.dropped) == 2
    assert [x["new"] for x in p.parts] == [True]


def test_incompatible_change_restarts():
    p = plan(recipe(unit_changed), OCT, parts=[part(AUG), part(SEP, seq=2)])
    assert p.action == "restart" and p.change == "semantic"
    assert [d["import_id"] for d in p.dropped] == ["imp1", "imp2"]
    assert p.added == [] and p.retired_new == [] and p.retired_existing == []


def test_a_kept_period_that_is_not_eligible_restarts():
    p = plan(REF, OCT, parts=[part(AUG, r=recipe(no_covers)), part(SEP, seq=2)])
    assert p.action == "restart" and "第 1 期" in p.reason
    assert p.dropped[0]["blockers"] and p.dropped[1]["blockers"] == []


def test_redraft_replaces_the_latest_period():
    p = plan(recipe(no_seven), SEP, kind="redraft", parts=[part(AUG), part(SEP, seq=2)])
    assert p.action == "replace_period" and p.replaces["import_id"] == "imp2"
    assert p.change == "compatible"


def test_redraft_with_an_incompatible_recipe_restarts():
    p = plan(recipe(unit_changed), SEP, kind="redraft", parts=[part(AUG), part(SEP, seq=2)])
    assert p.action == "restart" and len(p.dropped) == 2


def test_accumulate_needs_a_period():
    with pytest.raises(ValueError, match="统计期"):
        plan(REF, (None, None), parts=[part(AUG)])


def test_new_recipe_that_is_not_eligible_is_rejected():
    p = plan(recipe(no_covers), OCT, parts=[part(AUG)])
    assert p.action == "rejected" and "恰好覆盖统计期" in p.reason


# ==========================================================================
# 配方演进：兼容、退役、不兼容
# ==========================================================================


def test_identical_recipes_are_the_same():
    cc = ra.classify_changes([part(AUG), part(SEP, seq=2)], REF, current_tables=tables_of(REF))
    assert cc.change == "same" and not (cc.added or cc.retired_new or cc.retired_existing or cc.semantic)


def test_new_column_is_compatible():
    target = recipe(with_zone_c)
    cc = ra.classify_changes([part(AUG), part(SEP, r=target, seq=2)], target, current_tables=tables_of(REF))
    assert cc.change == "compatible" and cc.added == [{"table": "日客流", "column": "分区丙"}]
    assert cc.semantic == [] and cc.retired_new == []


def test_unit_change_is_semantic():
    cc = ra.classify_changes([part(AUG)], recipe(unit_changed), current_tables=tables_of(REF))
    assert cc.change == "semantic" and cc.semantic and "万人次" in cc.semantic[0]


def test_removed_column_retires_only_once_over_three_periods():
    """连续三期：去掉分区乙的那一期出 retired_new，之后各期是 retired_existing、不再出确认项（评审二-M3）。"""
    target = recipe(no_zone_b)
    first = plan(target, OCT, parts=[part(AUG), part(SEP, seq=2)])
    assert first.action == "append" and first.change == "retire"
    assert first.retired_new == [{"table": "日客流", "column": "分区乙"}] and first.retired_existing == []
    # 10 月提交后，当前版本的实际表结构仍含分区乙（保留在并集里），快照清单把它标成 retired
    current = tables_of(REF)
    status = {"日客流": {"分区乙": {"status": "retired", "since": "2026-10-01~2026-10-31", "unit": "人次"}}}
    second = plan(target, NOV, parts=[part(AUG), part(SEP, seq=2), part(OCT, r=target, seq=3)],
                  current=current, retired_status=status)
    assert second.action == "append" and second.change == "compatible"
    assert second.retired_new == [] and second.retired_existing == [{"table": "日客流", "column": "分区乙"}]


def test_retired_table_is_existing_only_when_every_column_is_marked():
    target = recipe()
    current = {**tables_of(REF), "旧表": [ColumnOut("日期", "TEXT", role="axis"), ColumnOut("值", "INTEGER")]}
    fresh = ra.classify_changes([part(AUG)], target, current_tables=current)
    assert {"table": "旧表", "column": None} in fresh.retired_new
    half = ra.classify_changes([part(AUG)], target, current_tables=current,
                               retired_status={"旧表": {"日期": {"status": "retired"}}})
    assert {"table": "旧表", "column": None} in half.retired_new
    done = ra.classify_changes([part(AUG)], target, current_tables=current, retired_status={
        "旧表": {"日期": {"status": "retired"}, "值": {"status": "retired"}}})
    assert done.retired_existing == [{"table": "旧表", "column": None}] and done.retired_new == []


def test_new_name_colliding_with_a_retired_name_is_semantic():
    def add_lower(data: dict) -> None:
        seg = _segments(data)[0]
        seg["labels"]["expect"].append("PM2.5（人次）")
        seg["measures"]["PM2.5（人次）"] = "pm2_5"
    current = copy.deepcopy(tables_of(REF))
    current["日客流"].append(ColumnOut("PM2_5", "INTEGER"))
    status = {"日客流": {"PM2_5": {"status": "retired"}}}
    cc = ra.classify_changes([part(AUG)], recipe(add_lower), current_tables=current, retired_status=status)
    assert cc.change == "semantic" and "只差大小写" in cc.semantic[0]
    p = plan(recipe(add_lower), SEP, parts=[part(AUG)], current=current, retired_status=status)
    assert p.action == "restart"


def test_label_sets_record_which_periods_lack_a_label():
    target = recipe(no_seven)
    cc = ra.classify_changes([part(AUG), part(SEP, r=target, seq=2)], target, current_tables=tables_of(REF))
    assert cc.change == "compatible" and cc.semantic == []
    [entry] = cc.label_sets
    assert (entry["table"], entry["column"], entry["segment"]) == ("时段客流", "时段", "日间")
    aug, sep = entry["periods"]
    assert (aug["start"], aug["missing"], aug["extra"]) == ("2026-08-01", [], ["7-8"])
    assert (sep["start"], sep["missing"], sep["extra"]) == ("2026-09-01", ["7-8"], [])
    p = plan(target, SEP, kind="redraft", parts=[part(AUG), part(SEP, seq=2)])
    assert p.label_sets and p.label_sets[0]["segment"] == "日间"


def test_label_sets_compare_canonical_writing():
    """全角、en-dash 写法按规范写法比：只是写法不同不算取值不同。"""
    def dashed(data: dict) -> None:
        seg = _segments(data)[1]
        seg["labels"]["expect"] = [x.replace("-", "–") for x in seg["labels"]["expect"]]
    cc = ra.classify_changes([part(AUG), part(SEP, r=recipe(dashed), seq=2)], REF, current_tables=tables_of(REF))
    assert cc.label_sets == []


def test_missing_current_tables_falls_back_to_the_parts():
    target = recipe(no_zone_b)
    cc = ra.classify_changes([part(AUG)], target, current_tables=None)
    assert cc.retired_new == [{"table": "日客流", "column": "分区乙"}]


def _raw_plan(new: Recipe, period, parts: list[PartInfo], *, kind: str = "reupload", current=None):
    return ra.plan_accumulate(new_recipe=new, new_period=period, period_source="cells", staging_kind=kind,
                              current_mode="accumulate", current_snapshot_kind="recipe", parts=parts,
                              current_tables=current)


def test_plan_without_current_tables_still_reports_added_columns():
    """current_tables 为 None 时按当前各期的配方推当前表结构，不能把本期也算进「当前已有」：那样新增列永远报不出来。"""
    target = recipe(with_zone_c)
    p = _raw_plan(target, SEP, [part(AUG)])
    assert p.action == "append" and p.added == [{"table": "日客流", "column": "分区丙"}]
    assert p.added == _raw_plan(target, SEP, [part(AUG)], current=tables_of(REF)).added
    # 修改配方重建最近一期：分区丙在要被替换的 9 月里已经有了，它此刻仍在当前版本中，不算新增
    redraft = _raw_plan(target, SEP, [part(AUG), part(SEP, r=target, seq=2)], kind="redraft")
    assert redraft.action == "replace_period" and redraft.added == []
    # 退役同样比得出来
    gone = _raw_plan(recipe(no_zone_b), SEP, [part(AUG)])
    assert gone.retired_new == [{"table": "日客流", "column": "分区乙"}]


def test_restart_and_rejected_plans_carry_no_label_sets():
    """restart 的结果只有本期、rejected 没有结果版本：各期维度取值的差异没有落点，不能让差异卡说「此前各期有」。"""
    def no_seven_and_unit(data: dict) -> None:
        no_seven(data)
        unit_changed(data)
    restart = plan(recipe(no_seven_and_unit), SEP, parts=[part(AUG)])
    assert restart.action == "restart" and [x["start"] for x in restart.parts] == ["2026-09-01"]
    assert restart.label_sets == []
    rejected = plan(recipe(no_seven), ("2026-08-15", "2026-09-14"), parts=[part(AUG)])
    assert rejected.action == "rejected" and rejected.label_sets == []
    # 对照：同样的取值差异在追加时照常记下
    assert plan(recipe(no_seven), SEP, parts=[part(AUG)]).label_sets


# ==========================================================================
# 并集 id
# ==========================================================================


def test_union_id_ignores_import_ids_and_input_order():
    a = ra.union_id("src1", [("2026-08-01", "2026-08-31", "b1"), ("2026-09-01", "2026-09-30", "b2")], "r" * 64)
    b = ra.union_id("src1", [("2026-09-01", "2026-09-30", "b2"), ("2026-08-01", "2026-08-31", "b1")], "r" * 64)
    assert a == b and len(a) == 64
    assert table_versions.sha_json({"source_id": "src1", "parts": [["2026-08-01", "2026-08-31", "b1"],
                                                                     ["2026-09-01", "2026-09-30", "b2"]],
                                    "target_recipe_sha256": "r" * 64, "union_ver": UNION_VER}) == a
    assert ra.union_id("src1", [("2026-08-01", "2026-08-31", "b1")], "r" * 64) != a
    assert ra.union_id("src1", [("2026-08-01", "2026-08-31", "b1"), ("2026-09-01", "2026-09-30", "b2")],
                       "s" * 64) != a
    assert ra.union_id("src2", [("2026-08-01", "2026-08-31", "b1"), ("2026-09-01", "2026-09-30", "b2")],
                       "r" * 64) != a


def test_union_id_commits_to_the_retired_layout():
    """退役列有历史：同样的各期、同样的目标配方，退役列的组成、顺序、类型不同就是结构不同的文件，id 必须跟着不同。"""
    parts = [("2026-08-01", "2026-08-31", "b1"), ("2026-09-01", "2026-09-30", "b2")]
    base = ra.union_id("src1", parts, "r" * 64)
    assert ra.union_id("src1", parts, "r" * 64, retired=None) == base
    assert ra.union_id("src1", parts, "r" * 64, retired={}) == base
    assert ra.union_id("src1", parts, "r" * 64, retired={"日客流": []}) == base
    d, e = ColumnOut("分区丁", "INTEGER"), ColumnOut("分区戊", "INTEGER")
    one = ra.union_id("src1", parts, "r" * 64, retired={"日客流": [d]})
    assert one != base
    assert one == table_versions.sha_json({
        "source_id": "src1", "parts": [list(x) for x in parts], "target_recipe_sha256": "r" * 64,
        "union_ver": UNION_VER, "retired": [["日客流", [["分区丁", "INTEGER"]]]]})
    de = ra.union_id("src1", parts, "r" * 64, retired={"日客流": [d, e]})
    ed = ra.union_id("src1", parts, "r" * 64, retired={"日客流": [e, d]})
    assert len({one, de, ed}) == 3
    assert ra.union_id("src1", parts, "r" * 64, retired={"日客流": [ColumnOut("分区丁", "REAL")]}) != one
    # 登记的 options 与 id 的输入一一对应
    assert ra.union_options(parts, "r" * 64) == {"kind": "union", "parts": [list(x) for x in parts],
                                                 "target_recipe_sha256": "r" * 64}
    assert ra.union_options(parts, "r" * 64, retired={"日客流": [d, e]})["retired"] == [
        ["日客流", [["分区丁", "INTEGER"], ["分区戊", "INTEGER"]]]]


# ==========================================================================
# 物化
# ==========================================================================


@lru_cache(maxsize=None)
def _built(start: str, days: int, seed: int = 0) -> tuple[str, str, str, str, tuple]:
    """按参考配方（累积）经执行器建一期的库，进程内只建一次。返回 (路径, 哈希, 起, 止, 表哈希)。"""
    import tempfile

    raw, name = flow_workbook(dt.date.fromisoformat(start), days, seed=seed)
    folder = Path(tempfile.mkdtemp(prefix="wp4-built-"))
    db = folder / f"{start}-{seed}.db"
    ex = recipe_engine.execute(REF, raw, name, xlsx_scan.scan(raw), str(db))
    assert ex.ok, ex.problems
    table_versions.settle_journal(db)
    os.chmod(db, 0o444)
    return str(db), raw_store.sha256_file(db), ex.period.start, ex.period.end, tuple(sorted(ex.table_hashes.items()))


def built_part(first_day: str, days: int, *, seed: int = 0, r: Recipe = REF, **kw: Any) -> UnionPart:
    """一期的 UnionPart；kw 可以改登记的统计期、路径、哈希、表哈希（造各种失败用）。"""
    path, sha, s, e, hashes = _built(first_day, days, seed)
    return UnionPart(import_id=kw.pop("import_id", None), start=kw.pop("start", s), end=kw.pop("end", e),
                     db_path=kw.pop("db_path", path), recipe=r, db_sha256=kw.pop("db_sha256", sha),
                     table_hashes=kw.pop("table_hashes", dict(hashes)), **kw)


def aug_sep() -> list[UnionPart]:
    return [built_part("2026-08-01", 31), built_part("2026-09-01", 30)]


def count(path: Path, sql: str, params=()) -> Any:
    conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
    try:
        return conn.execute(sql, params).fetchone()[0]
    finally:
        conn.close()


def test_two_periods_materialize_with_rows_ranges_and_checks(tmp_path):
    out = tmp_path / "u.db"
    rep = ra.materialize_union(out, target=REF, parts=aug_sep())
    assert rep.ok and [c.status for c in rep.checks] == ["passed"] * 3
    assert [(c.id, c.kind, c.category) for c in rep.checks] == [
        ("U1", "union_rows", "structure"), ("U2", "union_pk", "structure"), ("U3", "union_period", "structure")]
    assert rep.rows == {"日客流": 61, "时段客流": 1037, "时段客流_表内合计": 183}
    assert rep.part_rows[0]["日客流"] == {"union": [1, 31], "part": [1, 31]}
    assert rep.part_rows[1]["日客流"] == {"union": [32, 61], "part": [1, 30]}
    assert rep.part_rows[1]["时段客流"] == {"union": [528, 1037], "part": [1, 510]}
    assert rep.db_sha256 == raw_store.sha256_file(out)
    assert count(out, 'SELECT COUNT(*) FROM "日客流"') == 61
    assert count(out, 'SELECT COUNT(*) FROM "日客流" WHERE substr("日期", 1, 7) = ?', ("2026-09",)) == 30
    assert count(out, 'SELECT COUNT(*) FROM "时段客流" WHERE "时段" = ?', ("8-9",)) == 61
    # 表结构与执行器的写法相同：STRICT、主键
    ddl = count(out, "SELECT sql FROM sqlite_master WHERE name = '日客流'")
    assert "STRICT" in ddl and 'PRIMARY KEY ("日期")' in ddl
    assert count(out, "PRAGMA journal_mode") == "delete"
    assert not list(tmp_path.glob(f"*{raw_store.TMP_MARK}*"))
    assert rep.added == [] and rep.retired == []
    assert rep.part_null_counts[1]["时段客流"]["客流"] == count(
        out, 'SELECT COUNT(*) FROM "时段客流" WHERE rowid BETWEEN 528 AND 1037 AND "客流" IS NULL')
    assert rep.null_counts["时段客流"]["客流"] == count(out, 'SELECT COUNT(*) FROM "时段客流" WHERE "客流" IS NULL')


def test_materialize_is_deterministic_and_replaces_an_existing_file(tmp_path):
    first = ra.materialize_union(tmp_path / "a.db", target=REF, parts=aug_sep())
    out = tmp_path / "b.db"
    out.write_bytes(b"stale union from an older trial")
    second = ra.materialize_union(out, target=REF, parts=aug_sep())
    assert first.db_sha256 == second.db_sha256 == raw_store.sha256_file(out)


def test_composite_table_hash_uses_registered_part_hashes(tmp_path):
    parts = aug_sep()
    rep = ra.materialize_union(tmp_path / "u.db", target=REF, parts=parts)
    cols = [[c.name, c.type] for c in tables_of(REF)["日客流"]]
    want = table_versions.sha_json({"union_ver": UNION_VER, "columns": cols, "grain": ["日期"],
                                    "parts": [parts[0].table_hashes["日客流"], parts[1].table_hashes["日客流"]]})
    assert rep.table_hashes["日客流"] == want
    # 换一期登记的表哈希，组合哈希跟着变；导入记录 id 不参与
    other = aug_sep()
    other[1].table_hashes = {**other[1].table_hashes, "日客流": "0" * 64}
    other[0].import_id = "another-import"
    rep2 = ra.materialize_union(tmp_path / "v.db", target=REF, parts=other)
    assert rep2.table_hashes["日客流"] != want
    assert rep2.table_hashes["时段客流"] == rep.table_hashes["时段客流"]


def test_tampered_part_is_refused_before_attaching(tmp_path):
    parts = aug_sep()
    parts[1].db_sha256 = "0" * 64
    with pytest.raises(ra.UnionError) as err:
        ra.materialize_union(tmp_path / "u.db", target=REF, parts=parts)
    assert err.value.code == "part_tampered" and err.value.part == 2 and "第 2 期" in str(err.value)
    assert not (tmp_path / "u.db").exists()


def test_part_whose_bytes_changed_is_tampered(tmp_path):
    parts = aug_sep()
    copy_path = tmp_path / "aug.db"
    shutil.copyfile(parts[0].db_path, copy_path)
    with open(copy_path, "ab") as fh:
        fh.write(b"\0")
    parts[0].db_path = str(copy_path)           # 登记的哈希仍是原来的
    with pytest.raises(ra.UnionError) as err:
        ra.materialize_union(tmp_path / "u.db", target=REF, parts=parts)
    assert err.value.code == "part_tampered" and err.value.part == 1


def test_manifest_hash_must_match_too(tmp_path):
    parts = aug_sep()
    parts[0].manifest_db_sha256 = "1" * 64
    with pytest.raises(ra.UnionError) as err:
        ra.materialize_union(tmp_path / "u.db", target=REF, parts=parts)
    assert err.value.code == "part_tampered" and err.value.part == 1


def test_missing_part_file(tmp_path):
    parts = aug_sep()
    parts[1].db_path = str(tmp_path / "gone.db")
    with pytest.raises(ra.UnionError) as err:
        ra.materialize_union(tmp_path / "u.db", target=REF, parts=parts)
    assert err.value.code == "part_missing" and err.value.part == 2
    # 接口层直接转发这句，所以出路要写在里面（7.4）
    msg = str(err.value)
    assert "第 2 期（2026-09-01 至 2026-09-30）" in msg and "移除这一期" in msg and "启用更早的版本" in msg


def test_rowid_gap_in_a_part_fails(tmp_path):
    parts = aug_sep()
    holed = tmp_path / "holed.db"
    shutil.copyfile(parts[1].db_path, holed)
    os.chmod(holed, 0o644)
    conn = sqlite3.connect(holed)
    conn.execute('DELETE FROM "日客流" WHERE rowid = 5')
    conn.commit()
    conn.close()
    parts[1].db_path, parts[1].db_sha256 = str(holed), raw_store.sha256_file(holed)
    out = tmp_path / "u.db"
    with pytest.raises(ra.UnionError) as err:
        ra.materialize_union(out, target=REF, parts=parts)
    assert err.value.code == "union_failed" and err.value.part == 2 and "行号不连续" in str(err.value)
    assert not out.exists() and not list(tmp_path.glob(f"*{raw_store.TMP_MARK}*"))


def test_duplicate_primary_key_fails_u2_and_produces_no_file(tmp_path):
    """两期写了同一天（计划本该拒收的重叠）：写入撞上主键，U2 不通过并指出哪两期、哪个主键。"""
    aug = built_part("2026-08-01", 31)
    again = built_part("2026-08-01", 31, seed=0)
    out = tmp_path / "u.db"
    out.write_bytes(b"stale")
    rep = ra.materialize_union(out, target=REF, parts=[aug, again])
    assert rep.ok is False and rep.db_sha256 == ""
    u1, u2, u3 = rep.checks
    assert u2.status == "mismatch" and "第 1 期与第 2 期" in u2.details[0] and "日期=2026-08-01" in u2.details[0]
    assert u1.status == "mismatch"           # 回滚的那一期没写进去，行数也对不上
    assert u3.status == "passed"
    assert not out.exists() and not list(tmp_path.glob(f"*{raw_store.TMP_MARK}*"))


def test_dates_outside_the_period_fail_u3(tmp_path):
    aug = built_part("2026-08-01", 31, start="2026-09-01", end="2026-09-30")
    rep = ra.materialize_union(tmp_path / "u.db", target=REF, parts=[aug])
    assert rep.ok is False
    u3 = rep.checks[2]
    assert u3.status == "mismatch" and u3.failed == 3
    assert "第 1 期（2026-09-01 至 2026-09-30）的「日客流」有 31 行日期不在该期统计期内" in u3.details
    assert u3.sql and u3.params[2:] == ["2026-09-01", "2026-09-30"]


def _small_part(path: Path, r: Recipe, start: str, days: int, values: dict[str, list[int]]) -> UnionPart:
    """只含「日客流」一张表的期库（sqlite3 直接建，列按 r 推出的表结构、写法同执行器）。"""
    cols = tables_of(r)["日客流"]
    conn = sqlite3.connect(path)
    defs = ", ".join(f'"{c.name}" {c.type}' + (" NOT NULL" if c.name == "日期" else "") for c in cols)
    conn.execute(f'CREATE TABLE "日客流" ({defs}, PRIMARY KEY ("日期")) STRICT')
    first = dt.date.fromisoformat(start)
    for i in range(days):
        row = [(first + dt.timedelta(days=i)).isoformat()] + [values[c.name][i] for c in cols[1:]]
        conn.execute(f'INSERT INTO "日客流" VALUES ({", ".join("?" * len(cols))})', row)
    conn.commit()
    conn.close()
    table_versions.settle_journal(path)
    end = (first + dt.timedelta(days=days - 1)).isoformat()
    return UnionPart(import_id=None, start=start, end=end, db_path=str(path), recipe=r,
                     db_sha256=raw_store.sha256_file(path), table_hashes={"日客流": f"h-{start}"})


def test_retired_column_keeps_old_values_and_later_periods_are_null(tmp_path):
    target = recipe(no_zone_b)
    aug = built_part("2026-08-01", 31)
    sep = _small_part(tmp_path / "sep.db", target, "2026-09-01", 3,
                      {"全日客流": [30, 31, 32], "分区甲": [30, 31, 32]})
    retired = {"日客流": [ColumnOut("分区乙", "INTEGER")]}
    rep = ra.materialize_union(tmp_path / "u.db", target=target, parts=[aug, sep], retired=retired)
    assert rep.ok, rep.checks
    names = [c.name for c in rep.tables["日客流"]]
    assert names == ["日期", "全日客流", "分区甲", "分区乙"]       # 目标的列在前、退役的追加在后
    zone_b = rep.tables["日客流"][-1]
    assert zone_b.unit == "人次" and zone_b.header == "分区乙（人次）"   # 取最后一个含它的那一期
    out = tmp_path / "u.db"
    assert count(out, 'SELECT COUNT(*) FROM "日客流" WHERE "分区乙" IS NOT NULL AND rowid <= 31') == 31
    assert count(out, 'SELECT COUNT(*) FROM "日客流" WHERE "分区乙" IS NULL AND rowid > 31') == 3
    assert {"table": "日客流", "column": "分区乙", "periods": ["2026-08-01~2026-08-31"]} in rep.retired
    # 按期的空值数只记该期有的列：9 月没有分区乙，就没有这个键（结构性空值不能说成「原表为空格」）
    assert "分区乙" in rep.part_null_counts[0]["日客流"] and "分区乙" not in rep.part_null_counts[1]["日客流"]
    assert rep.null_counts["日客流"]["分区乙"] == 3
    # 9 月没有另外两张表：那几张表只有 8 月的行，记作只有部分期
    assert rep.rows["时段客流"] == 527 and "时段客流" not in rep.part_rows[1]
    assert {"table": "时段客流", "column": None, "periods": ["2026-08-01~2026-08-31"]} in rep.added


def test_require_union_ok_turns_a_failed_report_into_union_failed(tmp_path):
    """U1–U3 不通过时 materialize_union 只返回 ok=False（试运行要展示核对结果）；版本页的写操作用 require_union_ok
    换成 union_failed，消息带上不通过的细节。"""
    bad = ra.materialize_union(tmp_path / "bad.db", target=REF,
                               parts=[built_part("2026-08-01", 31, start="2026-09-01", end="2026-09-30")])
    assert bad.ok is False
    with pytest.raises(ra.UnionError) as err:
        ra.require_union_ok(bad)
    assert err.value.code == "union_failed" and err.value.part is None
    assert "日期不在该期统计期内" in str(err.value)
    ra.require_union_ok(ra.materialize_union(tmp_path / "good.db", target=REF, parts=aug_sep()))


def test_retired_columns_put_earlier_retirements_first(tmp_path):
    """先早先退役的（按当前表结构里的列序）、再这次才退役的（2.6「按退役先后追加」）；照 current_tables 直接拼会
    把新退役的排到前面。顺序进库哈希，所以 union_id 也要带上同一个 retired。"""
    current = copy.deepcopy(tables_of(REF))
    current["日客流"].append(ColumnOut("分区丁", "INTEGER", unit="人次"))
    current["旧表"] = [ColumnOut("日期", "TEXT", role="axis"), ColumnOut("值", "INTEGER")]
    status = {"日客流": {"分区丁": {"status": "retired"}}, "旧表": {"日期": {"status": "retired"},
                                                                  "值": {"status": "retired"}}}
    target = recipe(no_zone_b)
    p = plan(target, SEP, parts=[part(AUG)], current=current, retired_status=status)
    assert p.retired_existing == [{"table": "日客流", "column": "分区丁"}, {"table": "旧表", "column": None}]
    assert p.retired_new == [{"table": "日客流", "column": "分区乙"}]
    retired = ra.retired_columns(p, current)
    assert list(retired) == ["日客流", "旧表"]
    assert [c.name for c in retired["日客流"]] == ["分区丁", "分区乙"]
    assert [c.name for c in retired["旧表"]] == ["日期", "值"]
    cc = ra.classify_changes([part(AUG)], target, current_tables=current, retired_status=status)
    assert ra.retired_layout(ra.retired_columns(cc, current)) == ra.retired_layout(retired)
    rep = ra.materialize_union(tmp_path / "u.db", target=target, parts=[built_part("2026-08-01", 31)],
                               retired=retired)
    assert rep.ok, rep.checks
    assert [c.name for c in rep.tables["日客流"]] == ["日期", "全日客流", "分区甲", "分区丁", "分区乙"]
    assert rep.rows["旧表"] == 0
    # 换个顺序就是另一个文件，id 也跟着不同
    swapped = {"日客流": list(reversed(retired["日客流"])), "旧表": retired["旧表"]}
    rep2 = ra.materialize_union(tmp_path / "v.db", target=target, parts=[built_part("2026-08-01", 31)],
                                retired=swapped)
    assert rep2.db_sha256 != rep.db_sha256
    ids = [("2026-08-01", "2026-08-31", "b1")]
    assert ra.union_id("s", ids, "r" * 64, retired=retired) != ra.union_id("s", ids, "r" * 64, retired=swapped)
    assert ra.retired_columns(p, None) == {}


def test_part_column_without_a_place_in_the_union_fails(tmp_path):
    """目标去掉了分区乙、却没把它作为退役列传进来：那一列的数据没地方放，不能悄悄丢掉。"""
    with pytest.raises(ra.UnionError) as err:
        ra.materialize_union(tmp_path / "u.db", target=recipe(no_zone_b), parts=[built_part("2026-08-01", 31)])
    assert err.value.code == "union_failed" and "分区乙" in str(err.value)


def test_single_period_materialized_by_a_different_target(tmp_path):
    """移除、撤销之后只剩一期、而那一期的配方不是目标配方：按目标表结构物化成「一期的并集」。"""
    target = recipe(with_zone_c)
    rep = ra.materialize_union(tmp_path / "u.db", target=target, parts=[built_part("2026-08-01", 31)])
    assert rep.ok
    assert [c.name for c in rep.tables["日客流"]][-1] == "分区丙"
    assert rep.null_counts["日客流"]["分区丙"] == 31
    assert {"table": "日客流", "column": "分区丙", "periods": []} in rep.added
    assert rep.rows == {"日客流": 31, "时段客流": 527, "时段客流_表内合计": 93}


def test_missing_table_hash_is_refused(tmp_path):
    parts = aug_sep()
    parts[0].table_hashes = {}
    with pytest.raises(ra.UnionError) as err:
        ra.materialize_union(tmp_path / "u.db", target=REF, parts=parts)
    assert err.value.code == "union_failed" and "表哈希" in str(err.value)


def test_parts_must_be_sorted(tmp_path):
    with pytest.raises(ValueError, match="排序"):
        ra.materialize_union(tmp_path / "u.db", target=REF, parts=list(reversed(aug_sep())))