Files

2990 lines
163 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""按配方导入的流程(期 2,WP-5c;P2-SPEC 第 7 节):暂存 → 起草 → 回答问题 / 改配方 → 试运行 → 确认并启用,
以及「上传新一期」(按已确认的配方重放)和「修改配方」(不换文件)。
**暂存区是唯一的工作台。** 起草、回答、改配方、试运行都只改 import_stagings 里的一行,提交成功的那个事务
(table_versions.publish_build)才碰数据源、导入记录和快照。所以任何拒收、放弃、过期都不动当前快照,
暂存区结束即瘦身(table_versions.close_staging)。
**各模块经 Pipeline 接缝调用。** 静态校验、读取、执行、核对、说明、起草、差异卡、确认项都来自别的工作包,
这里只编排:default_pipeline() 在函数体里惰性导入真模块,测试替换 pipeline() 返回假实现,状态机的每条
分支(错误码、提交事务、并发)都能在不依赖真表格的情况下测到。
**几条贯穿全文件的规矩:**
- CPU 密集的步骤(扫描、读网格、干跑、试运行、核对、压缩表示)在线程池里,受 table_versions.parse_slot()
限并发;等模型回复期间不占名额(三次调用、每次 90 秒,占着名额会堵住所有上传和试运行)。
- 工作配方 = answers_base 按问题顺序重放全部回答。问题(含 effects)按 answers_base 生成,只在 answers_base
换掉(暂存、PUT、AI 草稿、上传新一期、修改配方)时重算;每次回答只重算卡片。这样先选 A 再改选 B 等于
直接选 B,effects 的路径也不会打在已经改过的配方上。
- 每次回答、PUT、修改配方之前用 detect_facts(grids, scan, recipe=工作配方) 取系统发现,静态校验带上它
(认领检查);上传新一期的现行配方原样重放时按 origin="replay" 校验、不查认领(配方已经确认过,本期的
数据变化由试运行报,不该在静态校验这一步就把 D26 这类数据质量问题挡成「配方不合法」)。
- 试运行把 asdict(Extraction) 和核对结果整份存进 trial,提交时据此还原、重算必勾确认项;差异卡的本期
一侧用这份完整回执,不用只截了前 50 条规范写法的 TrialOut.receipt。
- 发布前不改会话里的任何对象:对暂存区的修改一律放进 publish_build 的 before_commit(锁内、同一事务)。
**期 3(WP-5)补的几条:**
- 修复按钮与框选(edits/preview、apply、undo):补丁由服务端按封闭选项算,客户端只发提议 id、选项和理由;
应用时在同一个工作配方上重新换算,哈希对不上就是预览之后配方或理由变了(edit_stale)。每次修改的修改前起点
记在暂存区的 edits 里,撤销只撤最近一次,PUT 整份替换会清空撤销栈(之前的修改标 superseded);
- 改配方之后,回答在新起点上仍然成立的保留(carry_answers),不成立的进 answers_dropped;
- 静态校验的 base 一律取数据源**当前的**现行配方(3.3):启用旧版本之后,未结束暂存区的三处判断要一致;
- 按期累积:试运行里做累积计划、物化并集(提交时只 link),单期提交走 publish_build,多期走 publish_snapshot;
- 只读进程(没拿到版本存储的守卫)里,会删、写试运行库和原件的入口一律先 require_store_writable(503)。
"""
from __future__ import annotations
import asyncio
import copy
import dataclasses
import datetime as _dt
import hashlib
import json
import logging
import os
import secrets
from dataclasses import dataclass, field, fields
from pathlib import Path
from typing import Any, Awaitable, Callable
from sqlalchemy import func, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.data import raw_store, recipe_parsers, recipe_types, table_versions
from app.data.recipe_confirm import ConfirmContext
from app.data.recipe_types import (
REASON_SLOT,
Acceptance,
AiAvailability,
AxisOut,
CanonEntry,
Card,
CheckResult,
ColumnOut,
ConfirmItem,
DerivedItem,
DiffItem,
Draft,
DraftFacts,
EditResult,
ExcludedRows,
Extraction,
Fact,
Grid,
LedgerSheet,
OutsideText,
PeriodInput,
PeriodOut,
Problem,
Question,
Recipe,
RecipeProblem,
RegionMark,
SchemaNotes,
SegmentLabels,
Selection,
SheetsOut,
TableOut,
UnionPart,
)
from app.data.xlsx_scan import WorkbookScan
from app.db.base import new_id, utcnow
from app.db.models import DataSource, ImportStaging, SourceSnapshot, TableBuild, TableImport, TableRecipe
from app.db.portable import unique_violation
logger = logging.getLogger(__name__)
#: 导入清单的格式版本(7.7)
MANIFEST_FORMAT = "agentlab-import-manifest/1"
#: 回答里理由的字数上限(Question.options 的 needs_reason)
ANSWER_REASON_MAX = 200
#: 接受(Acceptance)理由的字数上限(5.3)
ACCEPT_REASON_MAX = 500
#: 试运行回执里规范写法对照给界面看几条(全部在导入清单里)
TRIAL_CANON_MAX = 50
#: trial 里只给服务端自己用的键:StagingOut 不带它们(整份 Extraction 很大,界面也不需要)。union 是物化好的
#: 并集(试运行文件路径、并集 id、回执),提交时用
TRIAL_INTERNAL = ("extraction", "null_counts", "inputs", "recipe_sha256", "build_id", "union", "prev_import_id")
#: TrialOut.receipt 给界面看的回执字段(7.4)。完整的回执在 trial["extraction"] 和导入清单里
_RECEIPT_VIEW = (
"ledger", "tables", "period", "axes", "placeholders", "labels", "outside_text", "hidden", "sheets",
"derived_form", "block_order", "ignored_columns", "full_calc_on_load", "formula_cells_accepted",
"blank_rows_skipped", "table_hashes", "timings", "rows_excluded",
)
#: 暂存区的 recipe_origin → 静态校验的 origin。manual / mixed 一律按 manual 校验;采用按规则重新起草的结果
#: (rules_redraft,6.3)按规则草稿的口径校验
_VALIDATE_ORIGIN = {"rules": "rules", "ai": "ai", "rules_redraft": "rules"}
#: 修改记录里只给服务端的键:撤销用的修改前起点,StagingOut、导入清单都不带(整份旧配方很大,界面也不需要)
EDIT_INTERNAL = ("before", "base_sha256_after")
#: 修改记录里只进导入清单、不给界面的键(StagingOut 的 edits 不带补丁:界面用不上,也免得回包随补丁变大)
EDIT_MANIFEST_ONLY = ("ops",)
#: 统计期上下文的 id(契约 ContextSpec.id 只能是它)
PERIOD_KEY = "统计期"
_ACTIVE_TRIAL = ("passed", "needs_decision")
# ==========================================================================
# 错误
# ==========================================================================
class ImportRefused(Exception):
"""业务上拒绝这次操作。status、code 给接口层原样转成 CodedHTTPException;message 给人看。"""
def __init__(self, status: int, code: str, message: str) -> None:
super().__init__(message)
self.status = status
self.code = code
self.message = message
class CommitConflict(ImportRefused):
"""提交时在版本存储锁内发现的冲突(base_changed / name_taken):409。"""
def __init__(self, code: str, message: str) -> None:
super().__init__(409, code, message)
_CLOSED = "这次导入已经结束(已提交、已放弃或已过期),不能再修改。请重新开始导入"
_NOT_RECIPE = "这个数据源不是按配方导入的。请用「上传表格」更新,或通过按配方导入重新开始"
_RAW_GONE = "这次导入的原件已不在服务端(可能已被清除),请重新上传文件"
_TRIAL_STALE = "试运行之后配方或录入有变化,请重新试运行"
_BASE_CHANGED = "试运行之后当前版本已被更新,请重新试运行后再提交"
#: AI 起草失败(502 ai_failed、413 ai_too_large)之后能做什么,取决于有没有工作配方:有的话配方面板里是表单,
#: 可以接着改;没有的话面板里只能粘贴一份完整的配方(没有表单可填)。recipe_ai 的失败原因只说出了什么事
_AI_NEXT = {
(True, True): "可以重试,或在配方面板中修改现有配方",
(True, False): "可以重试,或在配方面板中粘贴一份完整的配方",
(False, True): "请在配方面板中修改现有配方",
(False, False): "请在配方面板中粘贴一份完整的配方",
}
# ==========================================================================
# 接缝
# ==========================================================================
@dataclass
class Pipeline:
"""各工作包的函数。default_pipeline() 接真模块;测试替换 pipeline() 返回假实现。
在 P2-SPEC 10 节的清单上多了几项:scan(暂存、上传新一期要扫描)、column_null_counts(说明只给本期真有
空值的列写空值说明,WP-3 的要求)、first_call_text(同意框展示首次调用要发出的全文)、grid_max_cells /
draft_max_rows(大表部分干跑的阈值)、units(单位词表,StagingOut 给表单用)。
"""
validate: Callable[..., tuple[Recipe | None, list[RecipeProblem]]]
recipe_sha256: Callable[[Recipe], str]
canonical: Callable[[Recipe], dict[str, Any]]
apply_patch: Callable[[dict[str, Any], list[dict[str, Any]]], dict[str, Any]]
#: (原件, 文件名) → 扫描结果。文件名定按 CSV 读还是按工作簿读(csv_grid.scan_any)
scan: Callable[[bytes, str], WorkbookScan]
scan_to_json: Callable[[WorkbookScan], dict[str, Any]]
scan_from_json: Callable[[dict[str, Any]], WorkbookScan]
read_grid: Callable[..., Grid]
preview: Callable[..., dict[str, Any]]
execute: Callable[..., Extraction]
run_checks: Callable[[str, Extraction, Recipe], list[CheckResult]]
column_null_counts: Callable[[str, Recipe], dict[str, dict[str, int]]]
build_notes: Callable[..., SchemaNotes]
apply_notes: Callable[[dict[str, Any], SchemaNotes], dict[str, Any]]
detect_facts: Callable[..., DraftFacts]
draft: Callable[..., Draft]
questions_for: Callable[..., tuple[list[Card], list[Question]]]
ai_availability: Callable[..., Awaitable[AiAvailability]]
ai_draft: Callable[..., Awaitable[Any]]
compress: Callable[..., Any]
first_call_text: Callable[..., str]
diff_reports: Callable[..., list[DiffItem]]
confirm_items: Callable[[ConfirmContext], list[ConfirmItem]]
breaking_changes: Callable[..., dict[str, list[str]]]
grid_max_cells: int = 200_000
draft_max_rows: int = 500
units: tuple[str, ...] = ()
#: (模型对象, 模型 id, 可用性):同意与发送只解析一次模型(AU-7)。None 时退回 ai_availability,ai_draft 自己解析
resolve_model: Callable[..., Awaitable[tuple[Any, str, AiAvailability]]] | None = None
# ---- 期 3(P3-SPEC 12.6)。都给默认值 None:期 2 测试里全关键字构造的假 Pipeline 照旧成立,没接上的功能
# 按「没有」处理(没有修复提议、不做累积计划),状态机的其余分支照常测
#: WP-2 recipe_fixes:问题 → 修复提议;选中的选项 → 补丁
propose_fixes: Callable[..., list[Any]] | None = None
fix_ops: Callable[..., EditResult] | None = None
#: WP-2 recipe_select:框选 → 补丁;重放比对
selection_edit: Callable[..., EditResult] | None = None
replay_compare: Callable[..., Any] | None = None
#: WP-2 recipe_suggest:按规则重新起草的结果按来源对齐到现行配方的名字(6.3)
align_to_contract: Callable[..., tuple[dict[str, Any], list[str]]] | None = None
#: WP-3 recipe_compare:新旧配方对照
compare_recipes: Callable[..., dict[str, Any]] | None = None
#: WP-4 recipe_accumulate:累积计划、配方演进分档、并集物化
plan_accumulate: Callable[..., Any] | None = None
classify_changes: Callable[..., Any] | None = None
materialize_union: Callable[..., Any] | None = None
#: WP-3 recipe_notes:物化快照的说明
build_union_notes: Callable[..., SchemaNotes] | None = None
def default_pipeline() -> Pipeline:
"""接上真模块。函数体里惰性导入:这些模块(openpyxl、执行器、起草器)导入不便宜,接口模块加载时不必付。"""
from app.data import (
csv_grid, recipe, recipe_accumulate, recipe_ai, recipe_checks, recipe_compare, recipe_confirm, recipe_diff,
recipe_engine, recipe_fixes, recipe_notes, recipe_select, recipe_suggest, xlsx_cells, xlsx_scan,
)
return Pipeline(
validate=recipe.validate_recipe,
recipe_sha256=recipe.recipe_sha256,
canonical=recipe.canonical_recipe,
apply_patch=recipe.apply_patch,
# CSV / TSV 也走配方:csv_grid 适配成一张「工作表」,其余照旧交给 xlsx_scan(Excel 期 5)
scan=csv_grid.scan_any,
scan_to_json=xlsx_scan.scan_to_json,
scan_from_json=xlsx_scan.scan_from_json,
read_grid=xlsx_cells.read_grid,
preview=xlsx_cells.preview,
execute=recipe_engine.execute,
run_checks=recipe_checks.run_checks,
column_null_counts=recipe_checks.column_null_counts,
build_notes=recipe_notes.build_notes,
apply_notes=recipe_notes.apply_notes,
detect_facts=recipe_suggest.detect_facts,
draft=recipe_suggest.draft,
questions_for=recipe_suggest.questions_for,
ai_availability=recipe_ai.ai_availability,
ai_draft=recipe_ai.ai_draft,
compress=recipe_ai.compress,
first_call_text=recipe_ai.first_call_text,
diff_reports=recipe_diff.diff_reports,
confirm_items=recipe_confirm.confirm_items,
breaking_changes=recipe_confirm.breaking_changes,
grid_max_cells=xlsx_cells.GRID_MAX_CELLS,
draft_max_rows=recipe_suggest.DRAFT_MAX_ROWS,
units=tuple(recipe_parsers.UNITS),
resolve_model=recipe_ai.resolve_draft_model,
propose_fixes=recipe_fixes.propose_fixes,
fix_ops=recipe_fixes.fix_ops,
selection_edit=recipe_select.selection_edit,
replay_compare=recipe_select.replay_compare,
align_to_contract=recipe_suggest.align_to_contract,
compare_recipes=recipe_compare.compare_recipes,
plan_accumulate=recipe_accumulate.plan_accumulate,
classify_changes=recipe_accumulate.classify_changes,
materialize_union=recipe_accumulate.materialize_union,
build_union_notes=recipe_notes.build_union_notes,
)
def pipeline() -> Pipeline:
"""本模块实际用的接缝。测试 monkeypatch 它。"""
return default_pipeline()
@dataclass
class CommitResult:
"""提交的结果(CommitOut 除 source 以外的字段,接口层补上 source)。"""
import_id: str
snapshot_id: str
build_id: str
recipe_id: str
build_reused: bool
unchanged: bool
source_id: str = ""
#: 服务端已有的同版本数据文件被改过,这次用试运行的文件恢复了它(第 8 节遗留项 1)
build_restored: bool = False
#: 启用后当前版本含几期
parts: int = 1
#: 与此前某个版本内容相同,直接启用了那个版本(publish_snapshot 的复用、复活)
snapshot_reused: bool = False
# ==========================================================================
# JSON 往返
# ==========================================================================
def _jsonable(obj: Any) -> Any:
"""dataclass(含嵌套)、日期 → 能存进 JSON 列的形状。"""
if dataclasses.is_dataclass(obj) and not isinstance(obj, type):
obj = dataclasses.asdict(obj)
return json.loads(json.dumps(obj, ensure_ascii=False, default=str))
def _known(cls: type, data: Any) -> dict[str, Any]:
names = {f.name for f in fields(cls)}
return {k: v for k, v in (data or {}).items() if k in names}
def check_from_dict(data: dict[str, Any]) -> CheckResult:
"""试运行存下的核对结果 → CheckResult(提交时还原用,7.8 第 3 步)。"""
return CheckResult(**_known(CheckResult, data))
def _table_from(data: dict[str, Any]) -> TableOut:
d = _known(TableOut, data)
d["columns"] = [ColumnOut(**_known(ColumnOut, c)) for c in d.get("columns") or []]
return TableOut(**d)
#: Extraction 里需要还原成 dataclass 的字段
_EXTRACTION_PARTS: dict[str, Callable[[Any], Any]] = {
"problems": lambda v: [Problem(**_known(Problem, x)) for x in v or []],
"tables": lambda v: [_table_from(x) for x in v or []],
"ledger": lambda v: [LedgerSheet(**_known(LedgerSheet, x)) for x in v or []],
"regions": lambda v: [RegionMark(**_known(RegionMark, x)) for x in v or []],
"period": lambda v: PeriodOut(**_known(PeriodOut, v)) if v else None,
"context_checks": lambda v: [check_from_dict(x) for x in v or []],
"derived": lambda v: [DerivedItem(**_known(DerivedItem, x)) for x in v or []],
"canonicalized": lambda v: [CanonEntry(**_known(CanonEntry, x)) for x in v or []],
"labels": lambda v: [SegmentLabels(**_known(SegmentLabels, x)) for x in v or []],
"outside_text": lambda v: [OutsideText(**_known(OutsideText, x)) for x in v or []],
"sheets": lambda v: SheetsOut(**_known(SheetsOut, v)),
"axes": lambda v: [AxisOut(**_known(AxisOut, x)) for x in v or []],
# 期 3:排除的行(契约 ExcludedRows,第 8 节遗留项 4)。期 3 之前的回执没有这个键,按默认值(空列表)
"rows_excluded": lambda v: [ExcludedRows(**_known(ExcludedRows, x)) for x in v or [] if isinstance(x, dict)],
}
def extraction_from_dict(data: dict[str, Any]) -> Extraction:
"""试运行存下的 asdict(Extraction) → Extraction(提交时还原,说明生成要用 dataclass)。缺的字段取默认值。"""
kw: dict[str, Any] = {}
for f in fields(Extraction):
if f.name not in data:
continue
conv = _EXTRACTION_PARTS.get(f.name)
kw[f.name] = conv(data[f.name]) if conv is not None else copy.deepcopy(data[f.name])
kw.setdefault("ok", bool(data.get("ok")))
return Extraction(**kw)
def _facts_from(data: Any) -> DraftFacts:
data = data or {}
return DraftFacts(facts=[Fact(**_known(Fact, f)) for f in data.get("facts") or []],
candidates=dict(data.get("candidates") or {}),
nonnumeric=dict(data.get("nonnumeric") or {}))
def _question_from(data: dict[str, Any]) -> Question:
return Question(**_known(Question, data))
def _card_from(data: dict[str, Any]) -> Card:
return Card(**_known(Card, data))
def _draft_from(data: Any) -> Draft | None:
if not data:
return None
return Draft(recipe=data.get("recipe"), complete=bool(data.get("complete")), origin=data.get("origin") or "rules",
cards=[_card_from(c) for c in data.get("cards") or []],
questions=[_question_from(q) for q in data.get("questions") or []],
facts=_facts_from(data.get("facts")), failures=list(data.get("failures") or []),
failures_for_model=list(data.get("failures_for_model") or []))
def _draft_out(data: Any) -> dict[str, Any] | None:
"""Draft 给界面的形状:不含给模型看的那份原因(failures_for_model),也不重复带系统发现。"""
if not data:
return None
return {k: v for k, v in data.items() if k not in ("failures_for_model", "facts")}
def _full_form(recipe: Any) -> Any:
"""补全默认值的完整形式:问题的 effects 按它写路径、界面表单按它显示。schema 不过时原样返回。"""
if not isinstance(recipe, dict):
return recipe
try:
return Recipe.model_validate(recipe).model_dump(mode="json")
except Exception: # noqa: BLE001 - 不合法的配方照存,由静态校验报问题
return copy.deepcopy(recipe)
def _period_inputs(ctx: dict[str, Any] | None) -> dict[str, PeriodInput] | None:
"""暂存区存的人工录入 → 执行器要的 PeriodInput。"""
out: dict[str, PeriodInput] = {}
for key, v in (ctx or {}).items():
try:
out[key] = PeriodInput(start=_dt.date.fromisoformat(str(v["start"])),
end=_dt.date.fromisoformat(str(v["end"])), signed_by=v.get("signed_by"))
except (KeyError, TypeError, ValueError):
continue
return out or None
def _now_iso() -> str:
return utcnow().isoformat()
# ==========================================================================
# 读原件、扫描、网格(线程里跑)
# ==========================================================================
@dataclass
class _Book:
raw: bytes
scan: WorkbookScan
#: 每个有内容的可见工作表的前 draft_max_rows 行(起草、系统发现、压缩表示用)
grids: dict[str, Grid]
@dataclass
class _Snap:
"""暂存区在线程里要用的几项。ORM 对象不进线程:属性过期时会回库加载,线程里没有事件循环。"""
id: str
kind: str
raw_sha256: str
scan: dict[str, Any] | None
file_name: str
recipe_origin: str | None
#: 起点配方(上传新一期的现行配方)的哈希:工作配方与它相同时按重放校验
base_sha: str | None = None
context: dict[str, Any] = field(default_factory=dict)
#: 现行配方(canonical 形式):数据源**当前的** current_recipe_id 对应的那一版(3.3,评审一-m7)。kind 为
#: reupload / redraft 时给,静态校验据此放宽常量沿用、只查改动过的部分的认领;first / switch 为 None
base_recipe: dict[str, Any] | None = None
def _snap(staging: ImportStaging, *, base_sha: str | None = None, recipe_origin: str | None = None,
base_recipe: dict[str, Any] | None = None) -> _Snap:
return _Snap(id=staging.id, kind=staging.kind, raw_sha256=staging.raw_sha256,
scan=copy.deepcopy(staging.scan), file_name=staging.file_name or "",
recipe_origin=recipe_origin if recipe_origin is not None else staging.recipe_origin,
base_sha=base_sha, context=copy.deepcopy(staging.context_inputs or {}),
base_recipe=copy.deepcopy(base_recipe) if isinstance(base_recipe, dict) else None)
def _read_raw(sha: str) -> bytes:
try:
path = raw_store.raw_path(sha)
except ValueError:
raise ImportRefused(409, "raw_missing", _RAW_GONE) from None
if not path.is_file():
raise ImportRefused(409, "raw_missing", _RAW_GONE)
return path.read_bytes()
def _open_book(p: Pipeline, raw: bytes, scan: WorkbookScan) -> _Book:
grids: dict[str, Grid] = {}
for s in scan.sheets:
if s.state != "visible" or s.bounds is None or not s.nonempty:
continue
grids[s.name] = p.read_grid(raw, scan, s.name, max_rows=p.draft_max_rows)
return _Book(raw=raw, scan=scan, grids=grids)
def _book_of(p: Pipeline, snap: _Snap) -> _Book:
raw = _read_raw(snap.raw_sha256)
scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw, snap.file_name)
return _open_book(p, raw, scan)
async def _in_slot(fn: Callable[..., Any], *args: Any, **kw: Any) -> Any:
"""CPU 密集的一步:占一个解析名额、放进线程池。只包这一步,用完即还。"""
async with table_versions.parse_slot():
return await asyncio.to_thread(fn, *args, **kw)
class _Capture:
"""包一层干跑:记下最后一次的 Extraction(起草器内部干跑过,不必再跑一遍)。"""
def __init__(self, fn: Callable[[Recipe], Extraction]) -> None:
self.fn = fn
self.last: Extraction | None = None
def __call__(self, recipe: Recipe) -> Extraction:
self.last = self.fn(recipe)
return self.last
def _dry_fn(p: Pipeline, book: _Book, filename: str, ctx: dict[str, Any] | None) -> Callable[[Recipe], Extraction]:
"""交互时的干跑(不建库)。有工作表的非空格超过 GRID_MAX_CELLS 时只在前 draft_max_rows 行上部分干跑(4.9)。"""
big = any(s.state == "visible" and s.nonempty > p.grid_max_cells for s in book.scan.sheets)
inputs = _period_inputs(ctx)
def run(recipe: Recipe) -> Extraction:
return p.execute(recipe, book.raw, filename, book.scan, None, context_inputs=inputs,
max_rows=p.draft_max_rows if big else None)
return run
def _validate(p: Pipeline, snap: _Snap, work: Any, facts: DraftFacts | None) -> tuple[Recipe | None, list[Any]]:
"""静态校验。上传新一期、工作配方仍是现行配方时按重放校验(不查认领);其余带系统发现查认领。
有现行配方(kind 为 reupload / redraft)时把它作为 base 传下去(3.3):常量可以沿用已确认的 pick,认领检查
只查与现行配方相比改动过的部分。base 只在有值时传:期 2 测试里的假 validate 没有这个关键字(12.0)。"""
if not isinstance(work, dict):
return None, []
if snap.kind == "reupload" and snap.base_sha:
parsed, problems = p.validate(work, facts=None, origin="replay")
if parsed is not None and p.recipe_sha256(parsed) == snap.base_sha:
return parsed, problems
extra = {"base": snap.base_recipe} if snap.base_recipe is not None else {}
return p.validate(work, facts=facts, origin=_VALIDATE_ORIGIN.get(snap.recipe_origin or "", "manual"), **extra)
def _cards(p: Pipeline, recipe: Any, facts: DraftFacts, grids: dict[str, Grid],
ext: Extraction | None) -> tuple[list[Card], list[Question]]:
if not isinstance(recipe, dict):
return [], []
try:
return p.questions_for(recipe, facts, grids, extraction=ext)
except Exception as e: # noqa: BLE001 - 卡片和问题是辅助信息,出错不能拖垮回答、改配方
logger.warning("questions_for failed: %s", type(e).__name__)
return [], []
#: 交互时的干跑自己出错时给界面的问题(异常只进日志:消息里可能带着格子的值)
_DRY_FAILED = "按配方检查表格未能完成:检查时出错。请修改配方后重试,或直接试运行查看完整结果"
def _safe_dry(run: Callable[[Recipe], Extraction], recipe: Recipe) -> Extraction:
"""交互时的干跑只给着色和问题列表用:出错时变成一条问题,不让回答、改配方整个失败。试运行不走这里。"""
from app.data.xlsx_scan import UnsupportedTable
try:
return run(recipe)
except UnsupportedTable as e:
message = str(e) or _DRY_FAILED
except Exception as e: # noqa: BLE001
logger.warning("interactive dry run failed: %s", type(e).__name__)
message = _DRY_FAILED
return Extraction(ok=False, problems=[Problem("dry_run_failed", "structure", message,
model_message="dry_run_failed")])
@dataclass
class _Eval:
"""一次「静态校验 + 干跑 + 卡片」的结果。"""
recipe: Any
sha: str | None
parsed: Recipe | None
problems: list[Any]
facts: DraftFacts
extraction: Extraction | None
cards: list[Card]
questions: list[Question]
def _evaluate(p: Pipeline, book: _Book, snap: _Snap, work: Any, *, facts: DraftFacts | None = None) -> _Eval:
"""工作配方 → 系统发现(按这份配方的写法)、静态校验、干跑、卡片和问题(同步,线程里跑)。"""
if facts is None:
facts = p.detect_facts(book.grids, book.scan, recipe=work if isinstance(work, dict) else None)
parsed, problems = _validate(p, snap, work, facts)
ext = None
if parsed is not None and not problems:
ext = _safe_dry(_dry_fn(p, book, snap.file_name, snap.context), parsed)
cards, questions = _cards(p, work, facts, book.grids, ext)
stored = p.canonical(parsed) if parsed is not None else (copy.deepcopy(work) if isinstance(work, dict) else None)
sha = p.recipe_sha256(parsed) if parsed is not None else None
return _Eval(recipe=stored, sha=sha, parsed=parsed, problems=list(problems), facts=facts, extraction=ext,
cards=cards, questions=questions)
def _problem_json(p: Any) -> dict[str, Any]:
if isinstance(p, RecipeProblem):
return {"path": p.path, "code": p.code, "message": p.message}
return _jsonable(p)
#: 静态校验的问题存库、回显最多几条(SE-5:病态配方能产生几万条,整份存两份再回显)。多出的合成一条
RECIPE_PROBLEMS_MAX = 200
def _problems_json(problems: list[Any]) -> list[dict[str, Any]]:
out = [_problem_json(x) for x in problems[:RECIPE_PROBLEMS_MAX]]
if len(problems) > RECIPE_PROBLEMS_MAX:
out.append({"path": "", "code": "too_many_problems",
"message": f"另有 {len(problems) - RECIPE_PROBLEMS_MAX} 个问题没有列出(共 {len(problems)} 个),"
"请先按上面的提示修改"})
return out
def _regions(ext: Extraction | dict[str, Any] | None) -> list[dict[str, Any]]:
if ext is None:
return []
if isinstance(ext, dict):
return [dict(r) for r in ext.get("regions") or [] if isinstance(r, dict)]
return [_jsonable(r) for r in ext.regions]
def _ai_card(staging: ImportStaging) -> dict[str, Any] | None:
"""AI 草稿那张「请逐项核对」的卡片(ai_draft 生成,存在 drafts.ai 里)。工作配方仍出自 AI 时一直显示。"""
if staging.recipe_origin != "ai":
return None
draft = ((staging.drafts or {}).get("ai_adopted") or {}).get("draft") or {}
return next((c for c in draft.get("cards") or [] if c.get("id") == "ai_origin"), None)
def _invalidate_trial(staging: ImportStaging) -> str | None:
"""工作配方或录入变了:试运行作废(键和库清空、状态退回 drafting),回执留着给界面标「已失效」。
返回旧的试运行库路径:提交之后再删(remove_trial_file)。
"""
old = staging.trial_path
staging.trial_key = None
staging.trial_path = None
if staging.status in ("trialed", "rejected"):
staging.status = "drafting"
return old
def _apply_eval(staging: ImportStaging, ev: _Eval, *, questions: bool) -> str | None:
"""把一次评估写回暂存区(不提交)。配方变了就让试运行作废,返回旧的试运行库路径。"""
old_sha = staging.recipe_sha256
staging.recipe = ev.recipe
staging.recipe_sha256 = ev.sha
staging.recipe_problems = _problems_json(ev.problems)
staging.facts = _jsonable(ev.facts)
staging.draft_problems = [_jsonable(x) for x in ev.extraction.problems] if ev.extraction is not None else []
staging.draft_partial = bool(ev.extraction is not None and ev.extraction.partial)
cards = [_jsonable(c) for c in ev.cards]
card = _ai_card(staging)
if card is not None and not any(c.get("id") == "ai_origin" for c in cards):
cards.insert(0, card)
staging.cards = cards
if questions:
staging.questions = [_jsonable(q) for q in ev.questions]
drafts = dict(staging.drafts or {})
drafts["marks"] = _regions(ev.extraction)
staging.drafts = drafts
staging.updated_at = utcnow()
if ev.sha is None or ev.sha != old_sha:
return _invalidate_trial(staging)
return None
async def _remove_trial(path: str | None) -> None:
if path:
await asyncio.to_thread(table_versions.remove_trial_file, path)
def require_open(staging: ImportStaging) -> None:
if not table_versions.staging_is_open(staging):
raise ImportRefused(409, "staging_closed", _CLOSED)
async def _still_open(session: AsyncSession, staging: ImportStaging) -> None:
"""慢步骤之后再看一眼:期间被放弃、过期或提交了的,不再往上写。"""
await session.refresh(staging, ["status"])
require_open(staging)
# ==========================================================================
# 现行版本:配方、导入、上一期回执
# ==========================================================================
async def current_recipe(session: AsyncSession, source: DataSource | None) -> TableRecipe | None:
if source is None or not source.current_recipe_id:
return None
return await session.get(TableRecipe, source.current_recipe_id)
async def _require_recipe_source(session: AsyncSession, source: DataSource) -> TableRecipe:
row = await current_recipe(session, source)
if row is None:
raise ImportRefused(409, "not_recipe_source", _NOT_RECIPE)
return row
async def current_import(session: AsyncSession, source: DataSource | None) -> tuple[SourceSnapshot | None,
TableImport | None]:
"""源的当前快照和其中最近的那一期导入(期 2 替换模式下一个快照只有一期)。"""
if source is None or not source.current_snapshot_id:
return None, None
snap = await session.get(SourceSnapshot, source.current_snapshot_id)
if snap is None or not snap.imports:
return snap, None
return snap, await session.get(TableImport, str(snap.imports[-1]))
async def _prev_doc(session: AsyncSession, source: DataSource | None,
imp: TableImport | None) -> dict[str, Any] | None:
"""差异卡、确认项的「上一期导入回执」。
按配方导入的源取现行导入的导入清单(7.7:回执、核对含 C1 C2、统计期、配方哈希都在里面),不取
TableBuild.report——构建可能被复用,回执里没有 C1、C2,文件名也不是那一次的。简单导入的源(切换)
取期 1 的构建回执,没有格子账,差异卡只给 rows、period。
"""
if source is None or imp is None:
return None
if source.current_recipe_id:
from app.core import artifact_store
manifest = artifact_store.load(imp.manifest_artifact) if imp.manifest_artifact else None
if not isinstance(manifest, dict):
raise ImportRefused(500, "manifest_missing",
"现行版本的导入清单不在了,无法与上一期比较。请联系管理员检查数据目录")
return manifest
build = await session.get(TableBuild, imp.build_id) if imp.build_id else None
report = dict(build.report or {}) if build is not None else {}
report.setdefault("tables", [])
return {"receipt": report}
def effective_origin(staging: ImportStaging, base_origin: str | None) -> str:
"""提交时配方记的 origin,也是确认项的 recipe_origin。
人手改过(recipe_origin=manual)而起点出自 AI(这个暂存区采用过 AI 草稿,或现行配方本来就是 ai / mixed)
时记 mixed:AI 类确认项(ai_origin、rename、columns)照出。规则草稿改过的记 manual,不出 AI 类。
"""
origin = staging.recipe_origin or "manual"
if origin != "manual":
return origin
ai_used = bool(((staging.drafts or {}).get("ai_adopted") or {}).get("draft"))
return "mixed" if ai_used or base_origin in ("ai", "mixed") else "manual"
#: 配方记录(TableRecipe.origin)、导入清单只记这四种来源:界面按 RECIPE_ORIGIN_LABEL 显示,没有别的键
STORED_ORIGINS = ("rules", "ai", "manual", "mixed")
def stored_origin(origin: str | None, staging: ImportStaging | None = None) -> str:
"""配方入库(TableRecipe.origin、导入清单的 recipe.origin)时记的来源,只取 STORED_ORIGINS 之一。
rules_redraft 是**这个暂存区**采用了按规则重新起草的结果(6.3):只用来让本次的确认清单列出全部配方类项并出
redraft_adopted。它不能原样进配方记录:下个月上传新一期时,reupload 拿现行配方的 origin 给暂存区赋值,
recipe_confirm 见到 rules_redraft 就又出 redraft_adopted、又列全部配方类项,每个月都要再勾一遍并没有发生的
「采用重新起草」,违背「每月重传不打扰人」;卡片上还会露出英文键名(评审意见)。所以:没再改过的记 rules(内容
就是规则起草器给的,名字按来源对齐过);采用之后又用修复按钮、框选改过(还有未被覆盖的修改)的记 manual,
与首次导入时规则草稿加修复记 manual 一致。staging 为 None 时(给暂存区赋初值,防住已经入库的旧值)按没改过算。"""
if origin == "rules_redraft":
live = staging is not None and any(isinstance(e, dict) and not e.get("superseded")
for e in staging.edits or [])
return "manual" if live else "rules"
return origin if origin in STORED_ORIGINS else "manual"
# ==========================================================================
# 暂存、上传新一期、修改配方
# ==========================================================================
@dataclass
class _Staged:
scan_json: dict[str, Any]
previews: list[dict[str, Any]]
facts: DraftFacts
draft: Draft | None
ev: _Eval | None
def _stage_work(p: Pipeline, raw: bytes, filename: str) -> _Staged:
"""首次导入(或切换)的暂存:扫描、网格预览、规则起草(带干跑)、对草稿做一次静态校验。"""
scan = p.scan(raw, filename)
book = _open_book(p, raw, scan)
dry = _Capture(_dry_fn(p, book, filename, {}))
d = p.draft(scan, book.grids, filename, validate=lambda data, f, o: p.validate(data, facts=f, origin=o),
dry_run=dry)
previews = [p.preview(g) for g in book.grids.values()]
ev = None
if d.recipe is not None:
snap = _Snap(id="", kind="first", raw_sha256="", scan=None, file_name=filename, recipe_origin="rules")
parsed, problems = _validate(p, snap, d.recipe, d.facts)
ext = dry.last if parsed is not None and not problems else None
if parsed is not None and not problems and ext is None:
ext = _safe_dry(dry, parsed)
ev = _Eval(recipe=p.canonical(parsed) if parsed is not None else copy.deepcopy(d.recipe),
sha=p.recipe_sha256(parsed) if parsed is not None else None, parsed=parsed,
problems=list(problems), facts=d.facts, extraction=ext, cards=list(d.cards),
questions=list(d.questions))
return _Staged(scan_json=p.scan_to_json(scan), previews=previews, facts=d.facts, draft=d, ev=ev)
def _new_staging(*, source_id: str, name: str, description: str, kind: str, filename: str, raw: bytes,
signed_by: str | None, base_recipe_id: str | None = None) -> ImportStaging:
now = utcnow()
return ImportStaging(
id=new_id(), source_id=source_id, source_name=name, description=description or "", kind=kind,
status="drafting", raw_sha256=hashlib.sha256(raw).hexdigest(), file_name=(filename or "")[:500],
file_size=len(raw), answers={}, cards=[], questions=[], recipe_problems=[], draft_problems=[],
draft_partial=False, context_inputs={}, ai_usage=[], ai_consents=[], base_recipe_id=base_recipe_id,
signed_by=(signed_by or None), created_at=now, updated_at=now,
)
def _fill(staging: ImportStaging, staged: _Staged, ev: _Eval | None, *, answers_base: Any) -> None:
staging.scan = staged.scan_json
staging.grid_preview = staged.previews
staging.facts = _jsonable(staged.facts)
staging.answers_base = copy.deepcopy(answers_base)
if ev is not None:
staging.recipe = ev.recipe
staging.recipe_sha256 = ev.sha
staging.recipe_problems = _problems_json(ev.problems)
staging.facts = _jsonable(ev.facts)
staging.draft_problems = [_jsonable(x) for x in ev.extraction.problems] if ev.extraction else []
staging.draft_partial = bool(ev.extraction is not None and ev.extraction.partial)
staging.cards = [_jsonable(c) for c in ev.cards]
staging.questions = [_jsonable(q) for q in ev.questions]
drafts = dict(staging.drafts or {})
drafts["marks"] = _regions(ev.extraction if ev is not None else None)
staging.drafts = drafts
async def stage_upload(session: AsyncSession, *, raw: bytes, filename: str, name: str, description: str,
signed_by: str | None) -> ImportStaging:
"""POST imports/stage:原件存档、扫描、网格预览、系统发现、规则起草(加干跑)。
同名的简单导入源 → kind=switch(提交时原地切成按配方导入,源 id 和工具名不变);没有同名源 → first,
这时就分配源 id,提交时用它建 DataSource。名字格式由接口层查。扫描认不出时 UnsupportedTable 原样抛出。
"""
p = pipeline()
await table_versions.expire_stagings(session)
# 只读进程(没拿到版本存储的守卫)存不了原件:别先花时间扫描、起草(第 8 节遗留项 5)
table_versions.require_store_writable()
existing = (await session.execute(select(DataSource).where(DataSource.name == name))).scalar_one_or_none()
if existing is None:
kind, source_id = "first", new_id()
elif existing.current_recipe_id:
raise ImportRefused(409, "recipe_source_exists",
f"「{name}」已经按配方导入。更新数据请用「上传新一期」,调整取数规则请用「修改配方」")
elif (existing.origin or "manual") == "upload":
kind, source_id = "switch", existing.id
else:
raise ImportRefused(409, "name_taken", f"已存在名为「{name}」的数据源(非上传表格),请换一个名称")
staged = await _in_slot(_stage_work, p, raw, filename)
staging = _new_staging(source_id=source_id, name=name, description=description, kind=kind,
filename=filename, raw=raw, signed_by=signed_by)
staging.recipe_origin = "rules"
d = staged.draft
_fill(staging, staged, staged.ev, answers_base=_full_form(d.recipe) if d and d.recipe is not None else None)
drafts = dict(staging.drafts or {})
drafts.update({"rules": _jsonable(d) if d is not None else None, "ai": None})
staging.drafts = drafts
await table_versions.store_staging(session, staging, raw)
return staging
def _replay_work(p: Pipeline, raw: bytes, current: dict[str, Any], filename: str, snap: _Snap,
*, evaluate: bool) -> tuple[_Staged, _Eval | None]:
"""上传新一期 / 修改配方:扫描(或取存下的扫描)、按现行配方的写法取系统发现,修改配方时再评估一遍。"""
scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw, filename)
book = _open_book(p, raw, scan)
facts = p.detect_facts(book.grids, scan, recipe=current)
staged = _Staged(scan_json=p.scan_to_json(scan), previews=[p.preview(g) for g in book.grids.values()],
facts=facts, draft=None, ev=None)
if not evaluate:
# 上传新一期:现行配方原样重放(不查认领),试运行紧接着做,这里不另干跑
parsed, problems = _validate(p, snap, current, facts)
cards, questions = _cards(p, current, facts, book.grids, None)
return staged, _Eval(recipe=p.canonical(parsed) if parsed is not None else copy.deepcopy(current),
sha=p.recipe_sha256(parsed) if parsed is not None else None, parsed=parsed,
problems=list(problems), facts=facts, extraction=None, cards=cards,
questions=questions)
return staged, _evaluate(p, book, snap, current, facts=facts)
async def reupload(session: AsyncSession, source: DataSource, *, raw: bytes, filename: str,
signed_by: str | None) -> ImportStaging:
"""POST {source_id}/reupload:原件存档、扫描、系统发现,按当前配方自动试运行(不调模型)→ 回执 + 差异卡。
系统发现按现行配方的写法存下,用户随后改配方时静态校验用。现行配方原样重放不查认领(见模块 docstring)。
"""
p = pipeline()
await table_versions.expire_stagings(session)
table_versions.require_store_writable()
row = await _require_recipe_source(session, source)
full = _full_form(row.recipe)
staging = _new_staging(source_id=source.id, name=source.name, description=source.description or "",
kind="reupload", filename=filename, raw=raw, signed_by=signed_by, base_recipe_id=row.id)
staging.recipe_origin = stored_origin(row.origin or "manual")
snap = _snap(staging, base_sha=row.recipe_sha256, base_recipe=row.recipe)
staged, ev = await _in_slot(_replay_work, p, raw, full, filename, snap, evaluate=False)
_fill(staging, staged, ev, answers_base=full)
drafts = dict(staging.drafts or {})
drafts.update({"rules": None, "ai": None})
staging.drafts = drafts
await table_versions.store_staging(session, staging, raw)
if staging.recipe_sha256 and not staging.recipe_problems:
staging_id = staging.id
try:
staging = await run_trial(session, staging, context_inputs=None, signed_by=signed_by)
except Exception:
# 自动试运行没跑完(全量执行时文件读不了、上一期的清单不在了……):接口回的是错误,前端拿不到
# 这个暂存区的 id,留着它只会占着原件直到过期。放弃它(瘦身、原件没有别的引用就删),再抛原错误
await _discard_quietly(session, staging_id)
raise
return staging
async def _discard_quietly(session: AsyncSession, staging_id: str) -> None:
"""出错之后放弃刚建的暂存区。收拾本身出错只记日志,不盖住原来的错误。"""
try:
await session.rollback()
row = await session.get(ImportStaging, staging_id, populate_existing=True)
if row is not None and table_versions.staging_is_open(row):
await discard_staging(session, row)
except Exception: # noqa: BLE001
logger.exception("failed to discard staging %s after an error", staging_id)
async def redraft(session: AsyncSession, source: DataSource, *, signed_by: str | None) -> ImportStaging:
"""POST {source_id}/redraft:取当前导入的原件、以当前配方为工作配方,重做系统发现和干跑 → drafting。"""
p = pipeline()
await table_versions.expire_stagings(session)
table_versions.require_store_writable()
row = await _require_recipe_source(session, source)
_snapshot, imp = await current_import(session, source)
if imp is None or imp.raw_state != "kept" or not raw_store.raw_exists(imp.raw_sha256):
raise ImportRefused(409, "raw_missing",
"当前版本的原件已清除或没有保存,无法在原文件上修改配方。请用「上传新一期」带上文件进入修改")
raw = await asyncio.to_thread(_read_raw, imp.raw_sha256)
full = _full_form(row.recipe)
staging = _new_staging(source_id=source.id, name=source.name, description=source.description or "",
kind="redraft", filename=imp.file_name, raw=raw, signed_by=signed_by,
base_recipe_id=row.id)
staging.recipe_origin = stored_origin(row.origin or "manual")
snap = _snap(staging, base_recipe=row.recipe)
staged, ev = await _in_slot(_replay_work, p, raw, full, imp.file_name, snap, evaluate=True)
_fill(staging, staged, ev, answers_base=full)
drafts = dict(staging.drafts or {})
drafts.update({"rules": None, "ai": None})
staging.drafts = drafts
await table_versions.store_staging(session, staging, raw)
return staging
# ==========================================================================
# 回答问题、改配方
# ==========================================================================
def _fill_reason(node: Any, reason: str | None) -> Any:
"""effects 里整个字符串等于 REASON_SLOT 的值换成理由(不做子串替换,契约 Question.effects)。"""
if isinstance(node, str):
return (reason or "") if node == REASON_SLOT else node
if isinstance(node, list):
return [_fill_reason(x, reason) for x in node]
if isinstance(node, dict):
return {k: _fill_reason(v, reason) for k, v in node.items()}
return node
def replay_answers(p: Pipeline, base: Any, questions: list[Question], answers: dict[str, dict[str, Any]]) -> Any:
"""工作配方 = answers_base 按问题顺序重放全部回答的 effects(7.2)。effects 用不上时抛 ValueError。"""
doc = copy.deepcopy(base)
for q in questions:
a = answers.get(q.id)
if a is None:
continue
ops = q.effects.get(str(a.get("value")), [])
if ops:
doc = p.apply_patch(doc, _fill_reason(ops, a.get("reason")))
return doc
def _check_answers(questions: list[Question], answers: dict[str, Any]) -> dict[str, dict[str, Any]]:
"""回答必须指向存在的问题和选项;needs_reason 的选项要有 1–200 字的理由。返回规范后的回答。"""
by_id = {q.id: q for q in questions}
clean: dict[str, dict[str, Any]] = {}
for qid, a in answers.items():
q = by_id.get(qid)
if q is None:
raise ImportRefused(422, "answer_invalid", f"问题「{qid}」不存在,请刷新后重新回答")
if not isinstance(a, dict):
raise ImportRefused(422, "answer_invalid", f"问题「{q.text}」的回答格式不对,请重新选择")
value = a.get("value")
option = next((o for o in q.options if str(o.get("value")) == str(value)), None)
if option is None:
raise ImportRefused(422, "answer_invalid", f"问题「{q.text}」没有「{value}」这个选项,请重新选择")
reason = a.get("reason")
reason = str(reason).strip() if reason is not None else ""
if len(reason) > ANSWER_REASON_MAX:
raise ImportRefused(422, "answer_invalid",
f"问题「{q.text}」的理由超过 {ANSWER_REASON_MAX} 字,请精简后重新提交")
if option.get("needs_reason") and not reason:
raise ImportRefused(422, "answer_invalid",
f"问题「{q.text}」选择「{option.get('label') or value}」时需要写明理由")
clean[qid] = {"value": str(option.get("value")), "reason": reason or None}
return clean
async def _base_of(session: AsyncSession, staging: ImportStaging) -> tuple[str | None, dict[str, Any] | None]:
"""(按重放校验时比的现行配方哈希, 静态校验的 base)。一律取数据源**当前的** current_recipe_id(3.3,评审一-m7):
启用旧版本、移除、作废都会改它,静态校验、只列有变化的确认项、试运行三处取的「现行配方」必须一致。暂存区的
base_recipe_id 只是创建时的记录。哈希只给上传新一期(修改配方期 2 起就一律带系统发现校验)。"""
if staging.kind not in ("reupload", "redraft"):
return None, None
source = await session.get(DataSource, staging.source_id)
row = await current_recipe(session, source)
if row is None:
return None, None
return (row.recipe_sha256 if staging.kind == "reupload" else None), copy.deepcopy(row.recipe)
async def _snap_of(session: AsyncSession, staging: ImportStaging, *, recipe_origin: str | None = None) -> _Snap:
sha, base = await _base_of(session, staging)
return _snap(staging, base_sha=sha, recipe_origin=recipe_origin, base_recipe=base)
def _reeval_work(p: Pipeline, snap: _Snap, work: Any) -> _Eval:
return _evaluate(p, _book_of(p, snap), snap, work)
def _doc_sha(doc: Any) -> str:
"""一份起点配方的哈希(canonical 口径,与 recipe_sha256 同一种写法:去掉等于默认值的字段)。合 schema 的按
canonical 算,不合的(PUT 照存的半成品)按原样算。延续回答、撤销时比对「是不是同一份起点」用。"""
try:
return table_versions.sha_json(Recipe.model_validate(doc).model_dump(mode="json", exclude_defaults=True))
except Exception: # noqa: BLE001 - 不合 schema 的照样要能比
return table_versions.sha_json(doc)
def carry_answers(p: Pipeline, base: Any, questions: list[Question], old_questions: list[Question],
answers: dict[str, dict[str, Any]], *,
reapply: frozenset[str] | set[str] = frozenset()) -> tuple[dict[str, dict[str, Any]],
list[dict[str, Any]]]:
"""起点换了之后(PUT、应用修复或框选、撤销修改),旧回答逐个判断还成不成立(第 8 节遗留项 2)。
- 新问题里有同 id、同选项值的:把它的 effects(理由照旧填)应用到新起点上,结果与新起点的 canonical 形式
相同(新起点已经体现了这个回答),保留为已选;否则丢弃,reason=changed;
- 新问题里已经没有这个 id:丢弃,reason=gone。
返回 (保留的回答, answers_dropped)。answers_dropped 每项 {id, text, reason}:text 取自旧问题(gone 的问题在新
问题里已经没有了,界面只拿到 id 的话要么露出 id,要么什么都显示不了)。
为什么不清空(期 2 的写法):PUT 进来的配方本来就体现了之前的回答(占位符已经在列表里),清空之后看起来像没
答过,用户会以为要重答,或者以为自己的选择丢了。
reapply(只有撤销修改用):这几个回答是在那次修改**之后**才做的,撤销恢复的起点当然还没有体现它们。按上面的
规则它们一律判 changed,用户在修改之后做的选择就被撤销连带撤掉了(3.1「修改之后又回答过的问题不应该被撤
掉」)。所以对它们放宽成「effects 能干净地应用到恢复的起点上就保留」,等于在恢复的起点上把这道题重新答一遍。
"""
by_id = {q.id: q for q in questions}
old_by = {q.id: q for q in old_questions}
target = _doc_sha(base)
kept: dict[str, dict[str, Any]] = {}
dropped: list[dict[str, Any]] = []
for qid, a in (answers or {}).items():
old_q = old_by.get(qid)
text = old_q.text if old_q is not None else (by_id[qid].text if qid in by_id else qid)
q = by_id.get(qid)
if q is None:
dropped.append({"id": qid, "text": text, "reason": "gone"})
continue
value = str((a or {}).get("value"))
if not any(str(o.get("value")) == value for o in q.options):
dropped.append({"id": qid, "text": text, "reason": "changed"})
continue
ops = q.effects.get(value, [])
try:
doc = p.apply_patch(copy.deepcopy(base), _fill_reason(ops, (a or {}).get("reason"))) if ops else base
except (ValueError, TypeError, KeyError, IndexError):
doc = None
if doc is not None and (_doc_sha(doc) == target or qid in reapply):
kept[qid] = {"value": value, "reason": (a or {}).get("reason") or None}
else:
dropped.append({"id": qid, "text": text, "reason": "changed"})
return kept, dropped
@dataclass
class _Rebased:
"""起点换掉之后的一次评估:按新起点重出的问题、延续下来的回答、丢掉的回答。"""
ev: _Eval
answers: dict[str, dict[str, Any]]
dropped: list[dict[str, Any]]
def _rebase_on(p: Pipeline, book: _Book, snap: _Snap, base: Any, old_questions: list[Question],
answers: dict[str, dict[str, Any]], ev: _Eval | None = None, *,
reapply: frozenset[str] | set[str] = frozenset()) -> _Rebased:
"""新起点 → 评估(问题按新起点重出)→ 延续回答。保留下来的回答在新起点上一般都是无效果的,工作配方仍是新起点;
撤销修改时 reapply 的那几条(修改之后才做的回答)有效果,几条回答叠在一起也可能有效果:这两种情况按工作配方
重新评估,问题仍按起点出。重放失败时先丢掉 reapply 的那几条再试(其余各条单独看都是无效果的),还不行才全丢。"""
ev = ev or _evaluate(p, book, snap, base)
kept, dropped = carry_answers(p, base, ev.questions, old_questions, answers, reapply=reapply)
def drop(ids: list[str]) -> None:
for qid in ids:
dropped.append({"id": qid, "text": next((q.text for q in ev.questions if q.id == qid), qid),
"reason": "changed"})
kept.pop(qid, None)
if kept:
try:
work = replay_answers(p, base, ev.questions, kept)
except ValueError:
work = None
if work is None and any(qid in reapply for qid in kept):
drop([qid for qid in kept if qid in reapply])
try:
work = replay_answers(p, base, ev.questions, kept) if kept else base
except ValueError:
work = None
if work is None:
drop(list(kept))
work = base
if _doc_sha(work) != _doc_sha(base):
again = _evaluate(p, book, snap, work)
again.questions = ev.questions
ev = again
return _Rebased(ev=ev, answers=kept, dropped=dropped)
def _rebase_work(p: Pipeline, snap: _Snap, base: Any, old_questions: list[Question],
answers: dict[str, dict[str, Any]], *,
reapply: frozenset[str] | set[str] = frozenset()) -> _Rebased:
return _rebase_on(p, _book_of(p, snap), snap, base, old_questions, answers, reapply=reapply)
def _apply_rebase(staging: ImportStaging, base: Any, rebased: _Rebased, *, origin: str | None) -> str | None:
"""把换起点的结果写回暂存区(不提交)。返回旧的试运行库路径(配方变了才有)。"""
if origin is not None:
staging.recipe_origin = origin
staging.answers_base = copy.deepcopy(base)
staging.answers = dict(rebased.answers)
staging.answers_dropped = list(rebased.dropped)
return _apply_eval(staging, rebased.ev, questions=True)
def _supersede_edits(staging: ImportStaging) -> None:
"""整份替换工作配方(PUT、采用 AI 草稿):之前的修改标 superseded,不能再撤销,也不再出 fix:* / select:* 确认项
(它们的效果可能已经不在新配方里),只作为历史进导入清单(3.1,评审丙-M4)。"""
edits = [dict(e) for e in staging.edits or [] if isinstance(e, dict)]
if edits:
staging.edits = [{**e, "superseded": True} for e in edits]
async def answer(session: AsyncSession, staging: ImportStaging, answers: dict[str, dict[str, Any]]) -> ImportStaging:
"""POST imports/{id}/answers:工作配方 = answers_base 重放这一组回答(整组替换已存的回答),
重做系统发现、静态校验、干跑和卡片。问题集合不变(它按 answers_base 生成)。"""
p = pipeline()
require_open(staging)
questions = [_question_from(q) for q in staging.questions or []]
clean = _check_answers(questions, answers or {})
if staging.answers_base is None:
if clean:
raise ImportRefused(422, "answer_invalid", "还没有可回答的配方草稿")
return staging
# 回答会让旧的试运行库作废、被删掉:只读进程不写版本存储(第 8 节遗留项 5)
table_versions.require_store_writable()
try:
work = replay_answers(p, staging.answers_base, questions, clean)
except ValueError as e:
raise ImportRefused(422, "answer_invalid", f"回答没能应用到配方上:{e}") from e
snap = await _snap_of(session, staging)
ev = await _in_slot(_reeval_work, p, snap, work)
await _still_open(session, staging)
old = _apply_eval(staging, ev, questions=False)
staging.answers = clean
# 用户重新回答过了:「需要重新回答」的提示到此为止
staging.answers_dropped = []
await session.commit()
await _remove_trial(old)
return staging
_REDRAFT_MISMATCH = "采用的配方与重新起草的结果不同,请重新起草后再采用"
def _schema_sha(p: Pipeline, doc: Any) -> str | None:
"""配方的哈希(只看 schema,不做语义校验),与 EditResult.recipe_sha256_after、暂存区的哈希同一口径。"""
try:
return p.recipe_sha256(Recipe.model_validate(doc))
except Exception: # noqa: BLE001
return None
async def put_recipe(session: AsyncSession, staging: ImportStaging, recipe: dict[str, Any], *,
origin: str = "manual") -> ImportStaging:
"""PUT imports/{id}/recipe:整份配方作为新的起点(answers_base 换成它),照存;静态校验的问题写进
recipe_problems(有问题也存,只是不能试运行)。
期 3:回答在新起点上仍然成立的保留(carry_answers),不成立的进 answers_dropped;撤销栈清空,之前的修改标
superseded(3.1)。origin="rules_redraft"(采用按规则重新起草的结果,6.3)时配方的哈希必须等于最近一次
redraft-rules 给出的那份(redraft_mismatch):服务端据此记 recipe_origin=rules_redraft,确认清单按首次导入
列出全部配方类项并出 redraft_adopted,起草器识别出的定位规则才会逐项经过人的确认(H3)。"""
p = pipeline()
require_open(staging)
base = _full_form(recipe)
if origin == "rules_redraft":
sha = _schema_sha(p, base)
if not staging.redraft_sha256 or sha != staging.redraft_sha256:
raise ImportRefused(409, "redraft_mismatch", _REDRAFT_MISMATCH)
table_versions.require_store_writable()
old_questions = [_question_from(q) for q in staging.questions or []]
answers = dict(staging.answers or {})
snap = await _snap_of(session, staging, recipe_origin=origin)
rebased = await _in_slot(_rebase_work, p, snap, base, old_questions, answers)
await _still_open(session, staging)
_supersede_edits(staging)
old = _apply_rebase(staging, base, rebased, origin=origin)
await session.commit()
await _remove_trial(old)
return staging
# ==========================================================================
# 期 3:修复按钮、框选(edits/preview、apply、undo),按规则重新起草
# ==========================================================================
@dataclass
class EditRequest:
"""edits/preview、edits/apply 的请求(接口层解析好):修复给 fix_id、option、reason;框选给 selection。"""
kind: str
fix_id: str = ""
option: str = ""
reason: str | None = None
selection: Selection | None = None
def _fix_inputs(staging: ImportStaging) -> tuple[str, list[dict[str, Any]], list[dict[str, Any]], dict[str, str] | None]:
"""提议按哪些问题现算(3.1):当前试运行没失效且拒收时用 trial.problems,否则用 draft_problems;再加上
recipe_problems,以及当前试运行的 extraction.sheets.renamed。返回 (问题出自哪里, 问题, 配方问题, 改名)。"""
trial = staging.trial if isinstance(staging.trial, dict) else {}
if staging.trial_key and trial.get("status") == "rejected":
where, problems = "trial", [x for x in trial.get("problems") or [] if isinstance(x, dict)]
else:
where, problems = "draft", [x for x in staging.draft_problems or [] if isinstance(x, dict)]
renamed = None
if staging.trial_key:
sheets = ((trial.get("extraction") or {}).get("sheets") or {}) if isinstance(trial.get("extraction"), dict) else {}
got = sheets.get("renamed") if isinstance(sheets, dict) else None
renamed = {str(k): str(v) for k, v in got.items()} if isinstance(got, dict) and got else None
recipe_problems = [x for x in staging.recipe_problems or [] if isinstance(x, dict)]
return where, problems, recipe_problems, renamed
def proposals_for(p: Pipeline, staging: ImportStaging) -> tuple[list[Any], str]:
"""当前的修复提议(不存库,每次按暂存区里存的问题现算;同样的问题得到同样的 id)。没接上 propose_fixes、
暂存区已结束或还没有配方时为空。出错只记日志:提议是辅助信息,不能拖垮 StagingOut。"""
where, problems, recipe_problems, renamed = _fix_inputs(staging)
if p.propose_fixes is None or not table_versions.staging_is_open(staging) or not isinstance(staging.recipe, dict):
return [], where
try:
return list(p.propose_fixes(_full_form(staging.recipe), problems, recipe_problems, staging.facts or {},
renamed)), where
except Exception as e: # noqa: BLE001
logger.warning("propose_fixes failed: %s", type(e).__name__)
return [], where
@dataclass
class _EditRun:
edit: EditResult
new: Any = None
rebased: _Rebased | None = None
replay: Any = None
def _dry_before(p: Pipeline, book: _Book, snap: _Snap, full: Any) -> Extraction | None:
"""框选要的「当前工作配方最近一次干跑」:工作配方合 schema 才干跑(判断选区和已有的块是否重叠)。"""
if not isinstance(full, dict):
return None
try:
parsed = Recipe.model_validate(full)
except Exception: # noqa: BLE001
return None
return _safe_dry(_dry_fn(p, book, snap.file_name, snap.context), parsed)
def _edit_work(p: Pipeline, snap: _Snap, full: Any, req: EditRequest, proposal: Any,
retired: dict[str, list[str]] | None, old_questions: list[Question],
answers: dict[str, dict[str, Any]]) -> _EditRun:
"""(线程里跑)换算一次修改,ok 时把补丁应用到工作配方上、静态校验(带系统发现和 base)、干跑、重放比对,
再按新起点重出问题、延续回答(4.4)。预览和应用走同一段,应用时才把结果写回暂存区。"""
from app.data.recipe_fixes import EditRequestError
book = _book_of(p, snap)
if req.kind == "fix":
edit = p.fix_ops(full, proposal, req.option, req.reason)
else:
sel = req.selection
grid = book.grids.get(sel.sheet) if sel is not None else None
if grid is None:
raise EditRequestError("edit_invalid", f"这份文件里没有可以框选的工作表「{sel.sheet if sel else ''}」")
before = _dry_before(p, book, snap, full)
def draft_fn() -> Draft:
return p.draft(book.scan, book.grids, snap.file_name,
validate=lambda data, f, o: p.validate(data, facts=f, origin=o),
dry_run=_Capture(_dry_fn(p, book, snap.file_name, snap.context)))
edit = p.selection_edit(full if isinstance(full, dict) else None, grid, sel, extraction=before,
draft_fn=draft_fn, retired_names=retired or None)
if not edit.ok:
return _EditRun(edit=edit)
new = p.apply_patch(copy.deepcopy(full) if isinstance(full, dict) else {}, edit.ops)
rebased = _rebase_on(p, book, snap, new, old_questions, answers)
replay = None
ext = rebased.ev.extraction
if req.kind == "selection" and ext is not None and p.replay_compare is not None:
window = p.draft_max_rows if ext.partial else None
replay = p.replay_compare(edit, ext, window_rows=window)
return _EditRun(edit=edit, new=new, rebased=rebased, replay=replay)
async def _edit_context(session: AsyncSession, staging: ImportStaging, req: EditRequest
) -> tuple[Pipeline, _Snap, Any, Any, dict[str, list[str]] | None]:
"""预览、应用共用的准备:工作配方的完整形式、要用的提议(fix_stale)、退役名字(框选撞名检查用)。"""
p = pipeline()
require_open(staging)
full = _full_form(staging.recipe) if isinstance(staging.recipe, dict) else None
proposal = None
if req.kind == "fix":
if p.fix_ops is None:
raise ImportRefused(409, "fix_stale", "这条修复建议已不在当前的列表里,请刷新后重新查看")
proposal = next((x for x in proposals_for(p, staging)[0] if x.id == req.fix_id), None)
if proposal is None:
raise ImportRefused(409, "fix_stale", "这条修复建议已不在当前的列表里:问题在打开建议之后有了变化。"
"请重新查看问题和修复建议")
elif p.selection_edit is None:
raise ImportRefused(422, "edit_invalid", "这个服务端没有提供框选")
retired: dict[str, list[str]] | None = None
if req.kind == "selection":
from app.data import source_versions
source = await session.get(DataSource, staging.source_id)
snapshot = (await session.get(SourceSnapshot, source.current_snapshot_id)
if source is not None and source.current_snapshot_id else None)
try:
retired = source_versions.retired_names(source_versions.snapshot_manifest(snapshot))
except source_versions.PartProblem:
retired = None
snap = await _snap_of(session, staging)
return p, snap, full, proposal, retired
async def _run_edit(session: AsyncSession, staging: ImportStaging, req: EditRequest) -> tuple[Pipeline, Any, _EditRun]:
from app.data.recipe_fixes import EditRequestError
p, snap, full, proposal, retired = await _edit_context(session, staging, req)
old_questions = [_question_from(q) for q in staging.questions or []]
try:
run = await _in_slot(_edit_work, p, snap, full, req, proposal, retired, old_questions,
dict(staging.answers or {}))
except EditRequestError as e:
raise ImportRefused(422, e.code or "edit_invalid", str(e)) from e
except ValueError as e:
raise ImportRefused(422, "edit_invalid", str(e) or "修改请求的格式不对") from e
return p, full, run
async def _accumulate_change(session: AsyncSession, p: Pipeline, staging: ImportStaging, new: Any) -> str | None:
"""预览里的 accumulate_change:当前版本是按期累积时,修改后的配方对当前各期的分档(2.5);否则 None。"""
if p.classify_changes is None:
return None
from app.data import source_versions
source = await session.get(DataSource, staging.source_id)
snapshot = (await session.get(SourceSnapshot, source.current_snapshot_id)
if source is not None and source.current_snapshot_id else None)
if snapshot is None or (snapshot.mode or "replace") != "accumulate" or not source.current_recipe_id:
return None
try:
parts = await source_versions.load_parts(session, snapshot)
tables = await source_versions.current_tables(session, snapshot)
status = source_versions.retired_status(source_versions.snapshot_manifest(snapshot))
cc = p.classify_changes([x.info for x in parts], Recipe.model_validate(new), current_tables=tables,
retired_status=status)
except Exception as e: # noqa: BLE001 - 分档只是提示
logger.warning("classify_changes for preview failed: %s", type(e).__name__)
return None
return str(cc.change)
async def edit_preview(session: AsyncSession, staging: ImportStaging, req: EditRequest, *,
seq: Any = None) -> dict[str, Any]:
"""POST imports/{id}/edits/preview(4.4、9.4 EditPreview):换算、应用到工作配方的副本上、静态校验、干跑、重放
比对、对现行配方的破坏性变化、新旧对照。不改暂存区。换算不了时照样 200,ok=false、problems 写原因。"""
p, full, run = await _run_edit(session, staging, req)
edit = run.edit
out = _jsonable(edit)
out.update({"recipe_sha256_before": staging.recipe_sha256, "recipe_problems": [], "dry_run": None,
"replay": None, "breaking": {}, "accumulate_change": None, "compare": None, "seq": seq})
if not edit.ok:
return out
ev = run.rebased.ev
out["recipe_problems"] = _problems_json(ev.problems)
if ev.extraction is not None:
out["dry_run"] = {"problems": [_jsonable(x) for x in ev.extraction.problems],
"partial": bool(ev.extraction.partial), "marks": _regions(ev.extraction)}
out["replay"] = _jsonable(run.replay) if run.replay is not None else None
source = await session.get(DataSource, staging.source_id)
row = await current_recipe(session, source)
if row is not None and isinstance(run.new, dict):
# 9.4:breaking 是 table_changes 算出的「表名 → 人话列表」,不是 EditResult.breaking 那个布尔(WP-0 交接)
try:
out["breaking"] = dict(p.breaking_changes(row.recipe, run.new))
except Exception: # noqa: BLE001 - 修改后的配方不合 schema 时没有可比的表结构
out["breaking"] = {}
if p.compare_recipes is not None and isinstance(full, dict) and isinstance(run.new, dict):
try:
out["compare"] = p.compare_recipes(full, run.new)
except Exception: # noqa: BLE001
out["compare"] = None
out["accumulate_change"] = await _accumulate_change(session, p, staging, run.new)
return out
def _edit_out(entry: dict[str, Any], *, last: bool, open_: bool) -> dict[str, Any]:
"""修改记录给界面的形状:不带撤销用的修改前起点;undoable 只给最后一条、未被覆盖的。"""
out = {k: copy.deepcopy(v) for k, v in entry.items() if k not in EDIT_INTERNAL and k not in EDIT_MANIFEST_ONLY}
out["superseded"] = bool(entry.get("superseded"))
out["undoable"] = bool(last and open_ and not entry.get("superseded"))
return out
async def edit_apply(session: AsyncSession, staging: ImportStaging, req: EditRequest, *, expected_sha256: str | None,
signed_by: str | None) -> ImportStaging:
"""POST imports/{id}/edits/apply:在同一个工作配方上重新换算,哈希必须等于预览时的 expected_sha256(edit_stale:
预览之后配方、选项或理由变了;理由进配方哈希,界面必须原样提交预览时的理由)。之后新配方作为新的起点,回答按
carry_answers 延续,试运行作废。修改记录存修改前的工作配方和回答(撤销用)、修改后起点的哈希(undo_stale 的兜底),
以及补丁和所选的选项、理由或选区(进导入清单的 edits)。"""
p, full, run = await _run_edit(session, staging, req)
edit = run.edit
if not edit.ok:
why = ";".join(x.message for x in edit.problems[:3]) or "这次修改已无法应用到当前的配方上"
raise ImportRefused(422, "edit_not_applicable", why)
if not expected_sha256 or edit.recipe_sha256_after != expected_sha256:
raise ImportRefused(409, "edit_stale", "配方在预览之后有变化(或选项、理由与预览时不同),请重新预览")
table_versions.require_store_writable()
await _still_open(session, staging)
edits = [dict(e) for e in staging.edits or [] if isinstance(e, dict)]
# 撤销用的「修改前的起点」存修改前的**工作配方**(完整形式),不存 answers_base:补丁打在工作配方上(新起点 =
# 工作配方 + 补丁,已经体现了此前的回答),撤销时恢复的起点必须与它对称,此前的回答在恢复的起点里也都已体现,
# carry_answers 才会照常保留它们。存 answers_base 的话,有效果的回答(例如导入模式选了每期替换)在撤销时一律
# 判 changed 丢掉,配方悄悄退回草稿的默认值(评审意见:撤销丢回答)。修改前的回答另存一份,撤销时据此分出
# 哪些是修改之后才做的(见 edit_undo)
before_base = full if isinstance(full, dict) else copy.deepcopy(staging.answers_base)
entry = {
"seq": max((int(e.get("seq") or 0) for e in edits), default=0) + 1,
"kind": req.kind, "key": edit.key, "title": edit.title, "summary": list(edit.summary),
"at": _now_iso(), "signed_by": (signed_by or staging.signed_by or None),
"recipe_sha256_before": staging.recipe_sha256, "recipe_sha256_after": edit.recipe_sha256_after,
"superseded": False,
# 修改的内容本身(3.1「修改记录(谁、何时、补丁)……只进导入清单 edits」):补丁,以及人选的是什么——修复
# 记提议 id、选项和理由,框选记选区(Selection.to_json,键是 as,不用 asdict)。导入清单据此能复原这次
# 改了什么,不只剩人话摘要。StagingOut 不给 ops(_edit_out 去掉),界面用不上
"ops": copy.deepcopy(list(edit.ops)),
**({"fix": {"id": req.fix_id, "option": req.option, "reason": req.reason}} if req.kind == "fix" else {}),
**({"selection": req.selection.to_json()} if req.kind == "selection" and req.selection is not None else {}),
"before": {"answers_base": copy.deepcopy(before_base), "answers": copy.deepcopy(dict(staging.answers or {})),
"recipe_origin": staging.recipe_origin},
"base_sha256_after": _doc_sha(run.new),
}
# 修改是人做的:照 PUT 记 manual(采用过 AI 草稿的由 effective_origin 记 mixed)。采用按规则重新起草的结果之后
# 再用修复按钮,仍记 rules_redraft:确认清单照旧列出全部配方类项和 redraft_adopted,起草器识别出的定位规则不能
# 因为补了一个封闭补丁就免于逐项确认(H3)
origin = "rules_redraft" if staging.recipe_origin == "rules_redraft" else "manual"
staging.edits = [*edits, entry]
old = _apply_rebase(staging, run.new, run.rebased, origin=origin)
await session.commit()
await _remove_trial(old)
return staging
async def edit_undo(session: AsyncSession, staging: ImportStaging) -> ImportStaging:
"""POST imports/{id}/edits/undo:撤销最近一次修改(3.1)。先比对当前起点的哈希等于最后一条修改的
base_sha256_after(否则 undo_stale:之后又被改过;正常流程不会出现,PUT 会清空撤销栈,这是兜底),再恢复修改前的
起点(修改前的工作配方,见 edit_apply)和 recipe_origin。回答按 carry_answers 延续,不是整份恢复成修改前的回答:
修改前的回答已体现在恢复的起点里,照常保留;修改之后又回答过的问题不该被撤掉,在恢复的起点上重新应用(reapply)。"""
p = pipeline()
require_open(staging)
edits = [dict(e) for e in staging.edits or [] if isinstance(e, dict)]
last = edits[-1] if edits else None
if last is None or last.get("superseded") or not isinstance(last.get("before"), dict):
raise ImportRefused(409, "nothing_to_undo", "没有可以撤销的修改(整份替换配方之后,之前的修改不能再撤销)")
if _doc_sha(staging.answers_base) != last.get("base_sha256_after"):
raise ImportRefused(409, "undo_stale", "这次修改之后配方又被改过,无法直接撤销;请在配方面板中手工改回")
table_versions.require_store_writable()
before = last["before"]
base = copy.deepcopy(before.get("answers_base"))
origin = before.get("recipe_origin")
old_questions = [_question_from(q) for q in staging.questions or []]
# 恢复的起点是修改前的工作配方,修改前的回答都已体现在里面。要判断的回答 = 修改前的回答,再叠上现在的回答:
# 修改时因为补丁而丢掉的回答,撤销后又成立了,照常恢复为已选;修改之后才做(或改过)的回答,恢复的起点还没有
# 体现,按 reapply 在恢复的起点上重新应用,不跟着撤销一起撤掉(3.1)
prior = {str(k): v for k, v in (before.get("answers") or {}).items() if isinstance(v, dict)}
current = {str(k): v for k, v in (staging.answers or {}).items() if isinstance(v, dict)}
def same(a: Any, b: Any) -> bool:
return (isinstance(a, dict) and isinstance(b, dict) and str(a.get("value")) == str(b.get("value"))
and (a.get("reason") or None) == (b.get("reason") or None))
later = {qid for qid, a in current.items() if not same(prior.get(qid), a)}
snap = await _snap_of(session, staging, recipe_origin=origin)
rebased = await _in_slot(_rebase_work, p, snap, base, old_questions, {**prior, **current}, reapply=later)
# 只对现在的回答报「需要重新回答」:修改前的回答若在修改时已经丢掉、撤销后仍不成立,修改那一刻已经提示过
rebased.dropped = [d for d in rebased.dropped if d.get("id") in current]
await _still_open(session, staging)
staging.edits = edits[:-1]
old = _apply_rebase(staging, base, rebased, origin=origin)
await session.commit()
await _remove_trial(old)
return staging
@dataclass
class _Redrafted:
draft: Draft
aligned: dict[str, Any] | None
alignment: list[str]
compare: dict[str, Any] | None
sha: str | None
def _redraft_rules_work(p: Pipeline, snap: _Snap, current: dict[str, Any]) -> _Redrafted:
"""(线程里跑)对暂存区的文件跑规则起草(不调用模型),名字按来源对齐到现行配方,再和现行配方对照(6.3)。"""
book = _book_of(p, snap)
dry = _Capture(_dry_fn(p, book, snap.file_name, snap.context))
d = p.draft(book.scan, book.grids, snap.file_name, validate=lambda data, f, o: p.validate(data, facts=f, origin=o),
dry_run=dry)
if d.recipe is None:
# 失败原因只留在 draft.failures:alignment 是「名字对齐」的报告,界面把它放在那个标题下面;抄一份过去,同一句话
# 就在错误的标题下再出现一次(评审意见;check-manage 对这种情形伪造的也是 alignment: [])
return _Redrafted(draft=d, aligned=None, alignment=[], compare=None, sha=None)
candidate = _full_form(d.recipe)
if p.align_to_contract is not None:
aligned, report = p.align_to_contract(candidate, current)
else:
aligned, report = candidate, []
try:
parsed = Recipe.model_validate(aligned)
except Exception: # noqa: BLE001
return _Redrafted(draft=d, aligned=None, compare=None, sha=None,
alignment=[*report, "对齐后的配方格式不对,无法采用;请在配方面板中修改现行配方"])
full = parsed.model_dump(mode="json")
cmp = p.compare_recipes(current, full) if p.compare_recipes is not None else None
return _Redrafted(draft=d, aligned=full, alignment=list(report), compare=cmp, sha=p.recipe_sha256(parsed))
async def redraft_rules(session: AsyncSession, staging: ImportStaging) -> dict[str, Any]:
"""POST imports/{id}/redraft-rules(6.3):不改工作配方,只把对齐后配方的哈希记在 redraft_sha256,「采用」时
PUT recipe(origin=rules_redraft)核对它。首次导入、切换那里已有规则草稿,不提供(redraft_not_offered)。"""
p = pipeline()
require_open(staging)
if staging.kind not in ("reupload", "redraft"):
raise ImportRefused(409, "redraft_not_offered", "首次导入已有规则起草的结果,不需要重新起草")
table_versions.require_store_writable()
source = await session.get(DataSource, staging.source_id)
row = await _require_recipe_source(session, source) if source is not None else None
if row is None:
raise ImportRefused(409, "not_recipe_source", _NOT_RECIPE)
current = _full_form(row.recipe)
snap = await _snap_of(session, staging)
out = await _in_slot(_redraft_rules_work, p, snap, current)
await _still_open(session, staging)
staging.redraft_sha256 = out.sha
staging.updated_at = utcnow()
await session.commit()
return {"draft": _draft_out(_jsonable(out.draft)), "aligned_recipe": out.aligned, "alignment": out.alignment,
"compare": out.compare}
# ==========================================================================
# AI 起草
# ==========================================================================
def ai_offered(staging: ImportStaging) -> bool:
"""首次导入(含切换)、规则草稿不完整时才提供 AI 起草(6.2)。"""
if staging.kind not in ("first", "switch"):
return False
rules = (staging.drafts or {}).get("rules") or {}
return not rules.get("complete")
@dataclass
class _Gate:
avail: AiAvailability
#: 解析出的模型对象和 id(pipeline 没有 resolve_model 时为 None / "",由 ai_draft 自己解析)
model: Any = None
model_id: str = ""
async def _ai_gate(session: AsyncSession, staging: ImportStaging) -> _Gate:
require_open(staging)
if staging.kind not in ("first", "switch"):
raise ImportRefused(409, "ai_not_offered",
"上传新一期和修改配方时不提供 AI 起草:工作配方是已确认的现行配方。请在配方面板中修改")
if not ai_offered(staging):
raise ImportRefused(409, "ai_not_needed", "规则起草已经完整,不需要 AI 起草。请检查配方后试运行")
p = pipeline()
if p.resolve_model is not None:
model, model_id, avail = await p.resolve_model(session)
else:
model, model_id, avail = None, "", await p.ai_availability(session)
if not avail.available or (p.resolve_model is not None and model is None):
raise ImportRefused(409, "ai_unavailable", avail.reason or "AI 起草不可用。请到「设置 → 模型接入」检查")
return _Gate(avail=avail, model=model, model_id=model_id or avail.model or "")
def consent_token(text_sha256: str, avail: AiAvailability) -> str:
"""同意框绑定的令牌:发送的全文 + 接入名 + 模型 id(AU-7、SE-4)。
只按全文算的话,看完预览之后把助手的模型从本机换成外部服务,全文不变、令牌不变,内容就发给了同意框里
没写的那个接入。三者任一变了,POST draft-ai 就回 409 ai_preview_stale,要重新查看。
"""
return table_versions.sha_json({"text_sha256": text_sha256, "provider": avail.provider or "",
"model": avail.model or ""})
async def _mask_headers(session: AsyncSession, staging: ImportStaging) -> set[str]:
"""切换时,原数据源遮罩的列在原表里的表头(压缩表示里这些列只给画像)。首次导入没有遮罩列。"""
if staging.kind != "switch":
return set()
from app.data.engine import masked_columns
source = await session.get(DataSource, staging.source_id)
# 遮罩列名不区分大小写(与证据面板、裁判的遮罩一致,SE-9)
masked = {m.lower() for m in masked_columns(source.options if source is not None else None)}
if not masked:
return set()
_snapshot, imp = await current_import(session, source)
build = await session.get(TableBuild, imp.build_id) if imp is not None and imp.build_id else None
heads: set[str] = set()
for t in ((build.report or {}).get("tables") or []) if build is not None else []:
for c in t.get("columns") or [] if isinstance(t, dict) else []:
if isinstance(c, dict) and str(c.get("name") or "").lower() in masked and c.get("header"):
heads.add(str(c["header"]))
return heads
def _ai_failure(staging: ImportStaging, message: str, *, retry: bool = True) -> str:
"""AI 起草失败的原话补上下一步(见 _AI_NEXT)。"""
has_work = isinstance(staging.recipe, dict) and bool(staging.recipe)
return f"{message.rstrip('。')}。{_AI_NEXT[(retry, has_work)]}"
@dataclass
class _AiPrep:
book: _Book
facts: DraftFacts
compressed: Any
text: str
sha256: str
def _ai_prep_work(p: Pipeline, snap: _Snap, rules: Draft | None, mask: set[str]) -> _AiPrep:
"""读原件、系统发现(规则起草的分段 id,与半成品配方一致)、压缩表示、首次调用的全文。"""
book = _book_of(p, snap)
facts = p.detect_facts(book.grids, book.scan)
compressed = p.compress(book.grids, book.scan, facts, mask_headers=frozenset(mask))
text = p.first_call_text(compressed, grids=book.grids, scan=book.scan, rules_draft=rules)
return _AiPrep(book=book, facts=facts, compressed=compressed, text=text,
sha256=hashlib.sha256(text.encode("utf-8")).hexdigest())
async def _ai_prep(session: AsyncSession, staging: ImportStaging) -> _AiPrep:
from app.data.recipe_ai import AiTooLarge
p = pipeline()
rules = _draft_from((staging.drafts or {}).get("rules"))
mask = await _mask_headers(session, staging)
try:
return await _in_slot(_ai_prep_work, p, _snap(staging), rules, mask)
except AiTooLarge as e:
raise ImportRefused(413, "ai_too_large", _ai_failure(staging, str(e), retry=False)) from e
async def ai_preview(session: AsyncSession, staging: ImportStaging) -> dict[str, Any]:
"""GET imports/{id}/draft-ai/preview:首次调用要发给模型的全文(系统提示 + 首次提示词,含单位词表、
规则半成品、格式说明)。sha256 是同意令牌(全文 + 接入名 + 模型 id,见 consent_token),POST draft-ai 时
带回来比对;text_sha256 是全文本身的哈希,留痕用。"""
gate = await _ai_gate(session, staging)
prep = await _ai_prep(session, staging)
return {"text": prep.text, "chars": len(prep.text), "sha256": consent_token(prep.sha256, gate.avail),
"text_sha256": prep.sha256, "model": gate.avail.model, "provider": gate.avail.provider}
def _usage_total(usage: list[dict[str, Any]]) -> tuple[int, float]:
tokens = sum(int(u.get("input_tokens") or 0) + int(u.get("output_tokens") or 0) for u in usage)
return tokens, round(sum(float(u.get("cost_usd") or 0) for u in usage), 6)
def _adopt_ai_work(p: Pipeline, book: _Book, snap: _Snap, work: Any) -> _Eval:
return _evaluate(p, book, snap, work)
#: 正在 AI 起草的暂存区(本进程里):同一暂存区不并发起草,否则两次发送各花一份钱,同意记录、用量还会互相覆盖
#: (SE-11)。事件循环单线程,查和加之间没有 await,不会有两个请求同时通过;进程重启后自然清空。多进程时另取跨进程的锁
#: (run_ai_draft)
_AI_RUNNING: set[str] = set()
async def run_ai_draft(session: AsyncSession, staging: ImportStaging, *, consent: bool,
preview_sha256: str | None, signed_by: str | None) -> ImportStaging:
"""POST imports/{id}/draft-ai(流程见 _run_ai_draft)。同一暂存区同时只允许一次 AI 起草:已有一次在跑时回
409 ai_in_progress,不记同意、不调模型。"""
if consent is not True:
raise ImportRefused(422, "ai_consent_required", "需要先查看并同意发送的内容,才能让 AI 起草")
if staging.id in _AI_RUNNING:
raise ImportRefused(409, "ai_in_progress", "这次导入正在由 AI 起草,请等它完成后再试")
# 多进程:另一个进程上正在起草同一个暂存区的,跨进程的锁占不到,同样回 409(app/cluster/mutex)
from app.cluster import mutex
lock = mutex.lock(f"recipe-ai:{staging.id}")
_AI_RUNNING.add(staging.id)
try:
if not await lock.try_take():
raise ImportRefused(409, "ai_in_progress", "这次导入正在由 AI 起草,请等它完成后再试")
try:
return await _run_ai_draft(session, staging, preview_sha256=preview_sha256, signed_by=signed_by)
finally:
await lock.give_back()
finally:
_AI_RUNNING.discard(staging.id)
async def _run_ai_draft(session: AsyncSession, staging: ImportStaging, *,
preview_sha256: str | None, signed_by: str | None) -> ImportStaging:
"""POST imports/{id}/draft-ai:用户同意后才发送。
发送的全文必须就是预览给用户看的那份:服务端重算的 sha256 与 preview_sha256 不同 → 409 ai_preview_stale。
压缩表示在解析名额内先算好、经 compressed= 交给 ai_draft;等模型回复期间不占名额。同意记录在发送前落库;
用量(AiDraftResult.usage)拿到就单独提交,之后才看暂存区是否还开着(期间结束了 → 409 staging_closed,
用量照留)、草稿能不能用(不能 → 502 ai_failed)。成功时工作配方换成 AI 草稿(answers_base 换掉、
回答清空、recipe_origin=ai)。
"""
from app.data.recipe_ai import AiTooLarge, AiUnavailable
p = pipeline()
gate = await _ai_gate(session, staging)
avail = gate.avail
prep = await _ai_prep(session, staging)
token = consent_token(prep.sha256, avail)
if not preview_sha256 or preview_sha256 != token:
raise ImportRefused(409, "ai_preview_stale",
"要发送的内容或接收它的模型与查看时不同(原件、设置或助手的模型有变化),请重新查看后再同意")
await _still_open(session, staging)
consent_rec = {"signed_by": signed_by or None, "at": _now_iso(), "model": avail.model,
"provider": avail.provider, "compressed_sha256": prep.compressed.sha256,
"preview_sha256": token, "text_sha256": prep.sha256, "chars": len(prep.text)}
staging.ai_consents = [*(staging.ai_consents or []), consent_rec]
staging.updated_at = utcnow()
await session.commit()
rules = _draft_from((staging.drafts or {}).get("rules"))
# AI 每次尝试之后的干跑:协程函数,占一个解析名额、在线程池里执行(ai_draft 经 run_dry_async await 它)。
# 等模型回复期间不占名额,只有这一步占
dry_sync = _dry_fn(p, prep.book, staging.file_name or "", staging.context_inputs or {})
async def dry(recipe: Recipe) -> Extraction:
return await _in_slot(dry_sync, recipe)
# 同意记录里的模型就是实际调用的模型:把同意前解析出的那个交给 ai_draft,不让它再解析一次(AU-7)
model_kw: dict[str, Any] = {"model": gate.model, "model_id": gate.model_id} if gate.model is not None else {}
try:
result = await p.ai_draft(
session, grids=prep.book.grids, scan=prep.book.scan, facts=prep.facts, rules_draft=rules,
filename=staging.file_name or "", validate=lambda d, f, o: p.validate(d, facts=f, origin=o),
dry_run=dry, compressed=prep.compressed, **model_kw)
except AiUnavailable as e:
raise ImportRefused(409, "ai_unavailable", str(e) or "AI 起草不可用。请到「设置 → 模型接入」检查") from e
except AiTooLarge as e:
raise ImportRefused(413, "ai_too_large", _ai_failure(staging, str(e), retry=False)) from e
# 用量先落库,再看暂存区还开不开、草稿能不能用:等模型期间暂存区可能已被放弃、过期或提交了(瘦身后的
# 行也留着用量),采用草稿时也可能出错,这几次调用花的钱都要记下。已结束的暂存区只追加用量,不写草稿
await session.refresh(staging)
usage = [_jsonable(u) for u in result.usage or []]
tokens, cost = _usage_total(usage)
draft = result.draft
summary = {"draft": _jsonable(draft) if draft is not None else None, "usage": usage,
"attempts": int(result.attempts or 0), "total_tokens": tokens, "cost_usd": cost,
"error": result.error, "compressed_chars": result.compressed_chars,
"compressed_sha256": result.compressed_sha256}
staging.ai_usage = [*(staging.ai_usage or []), *usage]
if table_versions.staging_is_open(staging):
# ai:最近一次起草的摘要(成败都记,界面显示它);ai_adopted:被采用为工作配方的那一次
staging.drafts = {**(staging.drafts or {}), "ai": summary}
staging.updated_at = utcnow()
await session.commit()
require_open(staging)
if draft is None or draft.recipe is None:
# 工作配方不变
raise ImportRefused(502, "ai_failed", _ai_failure(staging, result.error or "AI 未能起草出合法的配方"))
base = _full_form(draft.recipe)
snap = _snap(staging, recipe_origin="ai")
ev = await _in_slot(_adopt_ai_work, p, prep.book, snap, base)
await _still_open(session, staging)
was_ai = effective_origin(staging, None) in ("ai", "mixed")
staging.drafts = {**(staging.drafts or {}), "ai_adopted": summary}
staging.recipe_origin = "ai"
staging.answers_base = copy.deepcopy(base)
staging.answers = {}
staging.answers_dropped = []
# 整份换成 AI 草稿:同 PUT,之前的修改不能再撤销、不再出确认项
_supersede_edits(staging)
old = _apply_eval(staging, ev, questions=True)
if old is None and not was_ai:
# AI 草稿与工作配方逐字相同(哈希没变)时 _apply_eval 不作废试运行,但来源从规则 / 手改变成了 AI,
# 提交时要多勾 ai_origin、rename、columns:存下的确认清单已经不全,试运行一并作废(AU-4)
old = _invalidate_trial(staging)
await session.commit()
await _remove_trial(old)
return staging
# ==========================================================================
# 试运行
# ==========================================================================
def trial_key(staging: ImportStaging, recipe_sha: str, ctx: dict[str, Any]) -> str:
"""试运行的键:工作配方、人工录入、文件名、解析器与执行器版本(外加原件和暂存区)。任何一项变了就要重新试运行。"""
return table_versions.sha_json({
"staging_id": staging.id, "raw_sha256": staging.raw_sha256, "recipe_sha256": recipe_sha,
"context": ctx or {}, "file_name": staging.file_name or "",
"parser_ver": recipe_parsers.PARSER_VER, "engine_ver": recipe_types.RECIPE_ENGINE_VER,
})
def _context_allowed(staging: ImportStaging) -> bool:
"""录入统计期只在需要时收:上一次试运行要求录入(needs_input),或者上一次用的就是人工录入。"""
trial = staging.trial or {}
if trial.get("status") == "needs_input":
return True
period = (trial.get("receipt") or {}).get("period") or {}
return period.get("source") == "human"
def _build_inputs(ex: Extraction | dict[str, Any]) -> dict[str, Any]:
"""进构建 id 的人工录入:统计期是人工录入时才有(署名不进)。"""
period = ex.get("period") if isinstance(ex, dict) else (dataclasses.asdict(ex.period) if ex.period else None)
if isinstance(period, dict) and period.get("source") == "human":
return {"context": {PERIOD_KEY: {"start": period.get("start"), "end": period.get("end")}}}
return {}
@dataclass
class _TrialRun:
parsed: Recipe | None
problems: list[Any]
facts: DraftFacts | None = None
extraction: Extraction | None = None
db: Path | None = None
db_sha256: str | None = None
checks: list[CheckResult] = field(default_factory=list)
null_counts: dict[str, dict[str, int]] | None = None
notes: SchemaNotes | None = None
def _trial_work(p: Pipeline, snap: _Snap, work: dict[str, Any], facts: DraftFacts | None, path: Path) -> _TrialRun:
"""执行(写临时库)→ settle → 改名 → 算库哈希 → 核对 → 各列空值数 → 说明预览(同步,线程里跑)。"""
parsed, problems = _validate(p, snap, work, facts)
if parsed is None or problems:
return _TrialRun(parsed=parsed, problems=list(problems))
raw = _read_raw(snap.raw_sha256)
scan = p.scan_from_json(snap.scan) if snap.scan else p.scan(raw, snap.file_name)
path.parent.mkdir(parents=True, exist_ok=True)
# 先写 .tmp- 再改名:半成品归启动清理(_sweep_tmp)管,发布、回收不会把半个库当成试运行库
tmp = path.parent / f"{path.name}{raw_store.TMP_MARK}{secrets.token_hex(6)}"
ex = p.execute(parsed, raw, snap.file_name, scan, str(tmp), context_inputs=_period_inputs(snap.context))
if not tmp.is_file():
return _TrialRun(parsed=parsed, problems=[], extraction=ex, checks=list(ex.context_checks))
try:
# 先 settle 再算哈希:发布时 _publish_file 还会 settle 一次,库已是回滚日志模式,文件头不会再变
table_versions.settle_journal(tmp)
os.replace(tmp, path)
except BaseException:
for suffix in ("", "-journal", "-wal", "-shm"):
Path(f"{tmp}{suffix}").unlink(missing_ok=True)
raise
db_sha = raw_store.sha256_file(path)
checks = p.run_checks(str(path), ex, parsed)
nulls = p.column_null_counts(str(path), parsed)
notes = p.build_notes(parsed, ex, checks, [], null_counts=nulls)
return _TrialRun(parsed=parsed, problems=[], extraction=ex, db=path, db_sha256=db_sha, checks=checks,
null_counts=nulls, notes=notes)
def trial_status(problems: list[Problem], checks: list[CheckResult], notes_problems: list[str]) -> str:
"""5.3:recipe / structure 类问题、不可接受的结构类核对没过 → rejected;input 类问题 → needs_input;
有可接受、没通过的核对 → needs_decision;否则 passed。confirm 类问题不影响状态。"""
if notes_problems or any(p.category in ("recipe", "structure") for p in problems):
return "rejected"
if any(c.status != "passed" and c.category == "structure" and not c.acceptable for c in checks):
return "rejected"
if any(p.category == "input" for p in problems):
return "needs_input"
if acceptable_ids(checks):
return "needs_decision"
return "passed"
def acceptable_ids(checks: list[CheckResult]) -> list[str]:
"""可以写理由接受、本次没通过的核对。"""
return [c.id for c in checks if c.acceptable and c.status in ("mismatch", "unverifiable")]
def _receipt_full(ex: Extraction) -> dict[str, Any]:
"""导入回执 = asdict(Extraction) 去掉 problems(7.7、7.6):axes、tables、ledger、period……一样不少。"""
data = _jsonable(ex)
data.pop("problems", None)
return data
def _receipt_view(full: dict[str, Any], db_sha: str | None) -> dict[str, Any]:
out = {k: copy.deepcopy(full.get(k)) for k in _RECEIPT_VIEW if k in full}
canon = list(full.get("canonicalized") or [])
out["canonicalized"] = canon[:TRIAL_CANON_MAX]
out["canonicalized_total"] = len(canon)
out["db_sha256"] = db_sha
return out
def _checks_summary(checks: list[dict[str, Any]]) -> list[dict[str, Any]]:
keys = ("id", "kind", "title", "status", "category", "checked", "failed", "unverifiable", "reasons", "acceptable")
return [{k: c.get(k) for k in keys} for c in checks]
@dataclass
class _Context:
"""试运行、提交都要的「现行版本」上下文。"""
source: DataSource | None
recipe: TableRecipe | None
snapshot: SourceSnapshot | None
imp: TableImport | None
prev: dict[str, Any] | None
async def _context(session: AsyncSession, staging: ImportStaging) -> _Context:
source = await session.get(DataSource, staging.source_id)
row = await current_recipe(session, source)
snapshot, imp = await current_import(session, source)
prev = await _prev_doc(session, source, imp)
if staging.kind in ("reupload", "redraft") and (row is None or prev is None):
raise ImportRefused(409, "not_recipe_source", _NOT_RECIPE)
return _Context(source=source, recipe=row, snapshot=snapshot, imp=imp, prev=prev)
def _active_edits(staging: ImportStaging) -> list[dict[str, Any]]:
"""确认项要的修改记录:只含未被覆盖的(3.1),不带撤销用的修改前起点。"""
return [{k: v for k, v in e.items() if k not in EDIT_INTERNAL}
for e in staging.edits or [] if isinstance(e, dict) and not e.get("superseded")]
def _confirm_context(staging: ImportStaging, cx: _Context, recipe: Recipe, ex_dict: dict[str, Any],
checks: list[Any], diff: list[Any] | None, *, prev: dict[str, Any] | None = None,
accumulate: dict[str, Any] | None = None) -> ConfirmContext:
"""确认项的输入。prev 给了就用它(按期累积时上一期是统计期最晚的那一期,替换该期时是被替换的那一期,9.5),
否则用现行导入的清单。"""
from app.data.engine import masked_columns
prev = prev if prev is not None else cx.prev
old = cx.recipe.recipe if cx.recipe is not None and staging.kind in ("reupload", "redraft") else None
return ConfirmContext(
kind=staging.kind, recipe=recipe, old_recipe=old,
old_schema_cache=(cx.snapshot.schema_cache or {}) if staging.kind == "switch" and cx.snapshot else None,
extraction=ex_dict, checks=checks, diff=(diff or []) if prev is not None else None, prev=prev,
recipe_origin=effective_origin(staging, cx.recipe.origin if cx.recipe is not None else None),
facts=_facts_from(staging.facts),
mask_columns=masked_columns(cx.source.options) if cx.source is not None else None,
accumulate=accumulate, edits=_active_edits(staging),
)
def _structure_problem(code: str, message: str) -> dict[str, Any]:
return {"code": code, "category": "structure", "message": message, "cells": [], "model_message": code,
"fix": None}
@dataclass
class _Acc:
"""试运行里按期累积的那一步(2.4、2.6、2.8):计划、物化好的并集、说明、额外的拒收问题。"""
plan: dict[str, Any] | None = None
#: 提交时用的并集(只给服务端,TRIAL_INTERNAL):试运行文件、并集 id、哈希、登记的选项和回执、各期顺序
union: dict[str, Any] | None = None
union_checks: list[dict[str, Any]] | None = None
notes: SchemaNotes | None = None
problems: list[dict[str, Any]] = field(default_factory=list)
#: 差异卡的上一期(替换该期时是被替换的那一期,9.5);None = 用现行导入
prev_import_id: str | None = None
def _union_note_new(parsed: Recipe, ex: Extraction, run: _TrialRun, start: str, end: str, acc: list[Acceptance]) -> Any:
from app.data.recipe_types import UnionNotePart
return UnionNotePart(recipe=parsed, extraction=ex, checks=list(run.checks), acceptances=list(acc),
null_counts=run.null_counts, start=start, end=end)
async def _accumulate_step(session: AsyncSession, p: Pipeline, staging: ImportStaging, cx: _Context, parsed: Recipe,
recipe_sha: str, run: _TrialRun, build_id: str) -> _Acc | None:
"""新配方是按期累积、或者当前版本是按期累积时做累积计划;结果有两期及以上时在这里物化并集(提交时只 link),
并按各期最弱的结论生成说明预览(本期按「全部未接受」)。不需要累积计划时返回 None。
当前快照某一期的导入清单被改过、统计期与库列对不上(part_tampered):拒收,当前快照不动。某一期的构建库不在了
(part_missing):409,消息指向数据源卡片的「版本」(可以先移除那一期,7.4)。"""
from app.data import recipe_accumulate as ra
from app.data import source_versions as sv
if p.plan_accumulate is None or run.extraction is None or run.db is None:
return None
ex = run.extraction
snapshot = cx.snapshot if cx.source is not None else None
cur_kind = None if snapshot is None else ("recipe" if cx.source.current_recipe_id else "simple")
cur_mode = (snapshot.mode or "replace") if cur_kind == "recipe" else None
if parsed.mode != "accumulate" and cur_mode != "accumulate":
return None
period = ex.period
new_period = (period.start, period.end) if period is not None and period.start and period.end else None
parts: list[sv.Part] = []
tables = status = None
try:
if cur_kind == "recipe":
parts = await sv.load_parts(session, snapshot)
# 退役只按结果各期有数据的列判定(WP-8 修补):当前版本的表结构里,只留结果各期(当前各期去掉本期要替换的
# 那一期)的配方里有、或者目标配方里有的列。否则两种情况都会出一条任何一期都没有数据的必勾 retire:*,
# 文案还写「早期各期保留原值」:移除唯一含某列的那一期之后(留下的全空目标列),以及替换唯一含某列的那一期
# (被替换掉的旧数据不在新版本里)。替换该期时这一列就是从当前配方里消失了,由 breaking:<表> 要人确认
gone = _replaced_part(parts, new_period, staging.kind)
tables = sv.tables_in_parts(await sv.current_tables(session, snapshot),
[x for x in parts if x is not gone], also=parsed)
status = sv.retired_status(sv.snapshot_manifest(snapshot))
except sv.PartProblem as e:
if e.code == "part_missing":
raise ImportRefused(409, "part_missing", e.message) from e
return _Acc(problems=[_structure_problem(e.code, e.message)])
try:
plan = p.plan_accumulate(new_recipe=parsed, new_period=new_period,
period_source=(period.source if period is not None else "cells"),
staging_kind=staging.kind, current_mode=cur_mode, current_snapshot_kind=cur_kind,
parts=[x.info for x in parts], current_tables=tables, retired_status=status)
except ValueError as e:
return _Acc(problems=[_structure_problem("accumulate_rejected", str(e))])
plan_json = _jsonable(plan)
rows = {t.name: int(t.rows) for t in ex.tables}
for entry in plan_json.get("parts") or []:
if entry.get("new"):
entry["file_name"], entry["rows"] = staging.file_name or "", dict(rows)
acc = _Acc(plan=plan_json)
if plan.action == "rejected":
if plan.overlaps:
acc.problems.append(_structure_problem("period_overlap", ra.overlap_message(plan.period, plan.overlaps)))
else:
acc.problems.append(_structure_problem("accumulate_rejected", plan.reason or "这份配方不能按期累积"))
return acc
if plan.action == "replace_period" and plan.replaces:
acc.prev_import_id = plan.replaces.get("import_id")
if len(plan.parts) < 2:
if plan.dropped:
# restart、模式切换后只剩本期:快照库就是本期的构建库,说明照单期写,另加「此前各期不在当前版本中」
# (2.8 开头,评审一-m10)
acc.notes = _add_single_after_drop(run.notes) if run.notes is not None else None
return acc
by_id = {x.info.import_id: x for x in parts}
start, end = new_period or ("", "")
union_parts, triples, order = [], [], []
try:
for entry in plan.parts:
if entry.get("new"):
union_parts.append(UnionPart(import_id=None, start=start, end=end, db_path=str(run.db), recipe=parsed,
db_sha256=run.db_sha256 or "", table_hashes=dict(ex.table_hashes)))
triples.append((start, end, build_id))
order.append(None)
else:
x = by_id[entry["import_id"]]
union_parts.append(x.union_part())
triples.append((x.info.start, x.info.end, x.info.build_id))
order.append(x.info.import_id)
except sv.PartProblem as e:
acc.problems.append(_structure_problem(e.code, e.message))
return acc
retired = ra.retired_columns(plan, tables)
uid = ra.union_id(staging.source_id, triples, recipe_sha, retired=retired)
out = table_versions.union_trial_path(run.db)
try:
report = await _in_slot(p.materialize_union, out, target=parsed, parts=union_parts, retired=retired)
except ra.UnionError as e:
if e.code == "part_missing":
raise ImportRefused(409, "part_missing", str(e)) from e
acc.problems.append(_structure_problem(e.code, str(e)))
return acc
acc.union_checks = [_jsonable(c) for c in report.checks]
plan_json["union"] = {"union_id": uid, "db_sha256": report.db_sha256, "rows": dict(report.rows)}
if not report.ok:
details = [d for c in report.checks if c.status != "passed" for d in (c.details or [c.title])]
acc.problems.append(_structure_problem("union_failed", "累积后的数据没有通过结构核对:"
+ ";".join(details[:3])))
return acc
note_parts = []
for entry in plan.parts:
if entry.get("new"):
note_parts.append(_union_note_new(parsed, ex, run, start, end, []))
else:
note_parts.append(await _in_slot(sv.note_part, by_id[entry["import_id"]]))
acc.notes = p.build_union_notes(
parsed, note_parts, union_tables=report.tables, null_counts=report.null_counts,
part_null_counts=report.part_null_counts, added=report.added, retired=report.retired,
label_sets=plan.label_sets, gaps=bool(plan.gaps), dropped=False)
acc.union = {"path": str(out), "union_id": uid, "db_sha256": report.db_sha256,
"options": ra.union_options(triples, recipe_sha, retired=retired), "report": _jsonable(report),
"imports": order}
return acc
def _replaced_part(parts: list[Any], new_period: tuple[str, str] | None, staging_kind: str) -> Any | None:
"""当前各期里本期要替换的那一期(与 recipe_accumulate.plan_accumulate 的判定一致):修改配方、不换文件(redraft)
时是最晚一期;否则是统计期起止与本期都相同的那一期。没有就 None(追加、部分重叠拒收)。"""
if not parts:
return None
if staging_kind == "redraft":
return max(parts, key=lambda x: (x.info.start or "", x.info.end or "", x.info.seq or 0))
if new_period is None:
return None
return next((x for x in parts if (x.info.start, x.info.end) == tuple(new_period)), None)
def _add_single_after_drop(notes: SchemaNotes) -> SchemaNotes:
from app.data.recipe_notes import add_single_after_drop
try:
return add_single_after_drop(notes)
except Exception: # noqa: BLE001 - 假说明(测试)没有片段时照原样
return notes
async def _prev_of(session: AsyncSession, cx: _Context, import_id: str | None) -> tuple[dict[str, Any] | None,
str | None]:
"""差异卡、确认项的上一期:给了导入 id(替换该期)就取那一期的导入清单,否则取现行导入的。"""
if not import_id or cx.imp is None or import_id == cx.imp.id:
return cx.prev, (cx.imp.file_name if cx.imp is not None else None)
imp = await session.get(TableImport, import_id)
doc = await _prev_doc(session, cx.source, imp)
return doc, (imp.file_name if imp is not None else None)
async def _prior_acceptances(session: AsyncSession, source_id: str, build_id: str) -> list[dict[str, Any]]:
"""同源、build_id 相同的导入记录以前的接受(第 5 节):只读显示「上次接受的理由」,不预填。
已作废(revoked 非空)的导入不算:那条理由已经被人判定为不成立,再摆在回执旁边当「上次接受的理由」,等于引导
用户照着再接受一次(H3);界面的 PriorAcceptance 也没有「已作废」的标记,用户分辨不出来(评审意见)。"""
rows = (await session.execute(
select(TableImport).where(TableImport.source_id == source_id, TableImport.build_id == build_id)
.order_by(TableImport.seq.desc()))).scalars()
out: list[dict[str, Any]] = []
for imp in rows:
if imp.revoked:
continue
for rec in [*(imp.overrides or []), *(imp.waivers or [])]:
if isinstance(rec, dict) and rec.get("check_id"):
out.append({"check_id": rec.get("check_id"), "reason": rec.get("reason"),
"signed_by": rec.get("signed_by"), "at": rec.get("at"), "import_seq": imp.seq})
return out
def _same_candidate(cx: _Context, acc: _Acc | None) -> str | None:
"""「与某次导入完全相同」看哪一期:替换该期时是被替换的那一期;当前版本只有一期时是它(期 2 的写法)。"""
if acc is not None and acc.plan is not None and acc.plan.get("action") == "replace_period":
return (acc.plan.get("replaces") or {}).get("import_id")
if cx.snapshot is not None and len(cx.snapshot.imports or []) == 1 and cx.imp is not None:
return cx.imp.id
return None
async def run_trial(session: AsyncSession, staging: ImportStaging, *,
context_inputs: dict[str, PeriodInput] | None, signed_by: str | None) -> ImportStaging:
"""POST imports/{id}/trial:执行 → 核对 → 说明预览 →(按期累积时)累积计划、物化并集 → 确认项 →(源有当前
版本时)差异卡。拒收也照常返回,看 trial.status。试运行库写好后删掉同一暂存区旧的试运行库。"""
p = pipeline()
require_open(staging)
if not isinstance(staging.recipe, dict):
raise ImportRefused(409, "recipe_invalid", "还没有可用的配方,请先在配方面板中填写")
# 试运行要写试运行库(和并集文件):只读进程不写版本存储(第 8 节遗留项 5)
table_versions.require_store_writable()
ctx = copy.deepcopy(staging.context_inputs or {})
if context_inputs:
if not _context_allowed(staging):
raise ImportRefused(422, "context_not_needed", "这份表格里能解析出统计期,不需要人工录入")
ctx = {key: {"start": v.start.isoformat(), "end": v.end.isoformat(),
"signed_by": (v.signed_by or signed_by or None)} for key, v in context_inputs.items()}
cx = await _context(session, staging)
base_sha = cx.recipe.recipe_sha256 if cx.recipe is not None and staging.kind == "reupload" else None
base_recipe = cx.recipe.recipe if cx.recipe is not None and staging.kind in ("reupload", "redraft") else None
snap = _snap(staging, base_sha=base_sha, base_recipe=base_recipe)
snap.context = ctx
work = _full_form(staging.recipe)
facts = _facts_from(staging.facts)
parsed, problems = _validate(p, snap, work, facts)
if parsed is None or problems:
raise ImportRefused(409, "recipe_invalid", "配方没有通过检查,请先按配方面板里的提示修改")
recipe_sha = p.recipe_sha256(parsed)
key = trial_key(staging, recipe_sha, ctx)
path = table_versions.trial_db_path(staging.source_id, staging.id, key)
from app.data.xlsx_scan import UnsupportedTable
prev_path = staging.trial_path
try:
run = await _in_slot(_trial_work, p, snap, work, facts, path)
except UnsupportedTable as e:
raise ImportRefused(400, "file_unreadable", str(e) or "无法读取这个文件,请确认是未加密的 Excel 文件") from e
# 从这里到提交之间出错(期间暂存区结束、差异卡或确认项报错、提交失败):这次新写的试运行库没有记录
# 指着,当场删掉(连同附属的并集文件)。路径与旧的相同(同一个键重跑)时不删,那是暂存区记着的库
try:
await _still_open(session, staging)
if run.extraction is None:
raise ImportRefused(409, "recipe_invalid", "配方没有通过检查,请先按配方面板里的提示修改")
ex = run.extraction
notes_problems = list(run.notes.problems) if run.notes is not None else []
problems_out = [_jsonable(x) for x in ex.problems]
status = trial_status(ex.problems, run.checks, notes_problems)
inputs = _build_inputs(ex)
# 执行器语义标记(table_versions.recipe_engine_semantics):只有受修补 3 影响的配方(列表跳过空行)才有,
# 这类配方的构建因此换 id,不会复用修补前的执行器建的旧库;提交时按同一份配方重算,与这里一致
build_id = table_versions.recipe_build_id(staging.source_id, staging.raw_sha256, recipe_sha, inputs,
semantics=table_versions.recipe_engine_semantics(parsed))
acc: _Acc | None = None
notes = run.notes
if status in _ACTIVE_TRIAL:
acc = await _accumulate_step(session, p, staging, cx, parsed, recipe_sha, run, build_id)
if acc is not None:
if acc.notes is not None:
notes = acc.notes
notes_problems = list(acc.notes.problems)
if acc.problems:
problems_out += acc.problems
status = "rejected"
elif notes_problems:
status = "rejected"
problems_out += [{"code": "notes_invalid", "category": "recipe", "message": f"说明没有通过检查:{m}",
"cells": [], "model_message": "", "fix": None} for m in notes_problems]
checks_json = [_jsonable(c) for c in run.checks]
full = _receipt_full(ex)
ex_dict = _jsonable(ex)
plan_json = acc.plan if acc is not None else None
prev, prev_name = await _prev_of(session, cx, acc.prev_import_id if acc is not None else None)
diff: list[Any] | None = None
items: list[ConfirmItem] = []
# 差异卡和确认项只在能提交的试运行(passed / needs_decision)上算。拒收时执行器在结构问题上停下,
# 表、格子账可能只写了一半,拿它比会冒出「行数 31 → 0」这类假变化,结构上的变化已经作为拒收问题
# 报了(7.6);needs_input 缺统计期,录入后再试运行就有。这两种状态 diff 为 null
if status in _ACTIVE_TRIAL:
if prev is not None:
cur = {"receipt": full, "checks": checks_json, "period": full.get("period"),
"recipe_sha256": recipe_sha}
# 配方变没变按上一期自己的配方比:按期累积时上一期的配方不一定是现行配方(移除之后剩下的期)
prev_sha = ((prev.get("recipe") or {}).get("sha256") if isinstance(prev.get("recipe"), dict)
else None) or (cx.recipe.recipe_sha256 if cx.recipe is not None else None)
kw: dict[str, Any] = {"accumulate": plan_json} if plan_json is not None else {}
diff = list(p.diff_reports(prev, cur, prev_file_name=prev_name,
cur_file_name=staging.file_name or "", recipe=parsed,
recipe_changed=prev_sha != recipe_sha, **kw))
items = list(p.confirm_items(_confirm_context(staging, cx, parsed, ex_dict, run.checks, diff, prev=prev,
accumulate=plan_json)))
same = None
cand = _same_candidate(cx, acc)
if cand is not None:
cand_imp = cx.imp if cx.imp is not None and cx.imp.id == cand else await session.get(TableImport, cand)
if cand_imp is not None and cand_imp.build_id == build_id:
same = {"id": cand_imp.id, "seq": cand_imp.seq}
notes_view = ({t: {"comment": n.comment, "columns": dict(n.columns)} for t, n in notes.tables.items()}
if notes is not None else {})
compare = None
if (p.compare_recipes is not None and cx.recipe is not None and staging.kind in ("reupload", "redraft")
and cx.recipe.recipe_sha256 != recipe_sha):
try:
compare = p.compare_recipes(cx.recipe.recipe, parsed, plan=plan_json)
except Exception as e: # noqa: BLE001 - 对照只是辅助信息
logger.warning("compare_recipes failed: %s", type(e).__name__)
base_snapshot = cx.source.current_snapshot_id if cx.source is not None else None
staging.trial = {
"trial_id": key[:16], "status": status, "receipt": _receipt_view(full, run.db_sha256),
"problems": problems_out, "checks": checks_json, "confirm_items": [_jsonable(i) for i in items],
"acceptable": acceptable_ids(run.checks), "notes": notes_view,
"diff": [_jsonable(d) for d in diff] if diff is not None else None,
"same_as_import": same, "base_snapshot_id": base_snapshot,
# 期 3(9.4 TrialOut)
"accumulate": plan_json, "recipe_compare": compare,
"union_checks": acc.union_checks if acc is not None else None,
"prior_acceptances": await _prior_acceptances(session, staging.source_id, build_id),
# 以下只给服务端:提交时还原 Extraction、重算必勾项、生成说明、发布并集
"extraction": ex_dict, "null_counts": run.null_counts, "inputs": inputs, "recipe_sha256": recipe_sha,
"build_id": build_id, "union": acc.union if acc is not None and status in _ACTIVE_TRIAL else None,
"prev_import_id": acc.prev_import_id if acc is not None else None,
}
old = staging.trial_path
staging.context_inputs = ctx
staging.trial_key = key
staging.trial_path = str(run.db) if run.db is not None else None
staging.base_snapshot_id = base_snapshot
staging.status = "rejected" if status == "rejected" else "trialed"
drafts = dict(staging.drafts or {})
drafts["marks"] = _regions(ex)
staging.drafts = drafts
if signed_by:
staging.signed_by = signed_by
staging.updated_at = utcnow()
await session.commit()
except Exception:
if run.db is not None and str(run.db) != prev_path:
await _remove_trial(str(run.db))
raise
if old and old != staging.trial_path:
await _remove_trial(old)
return staging
# ==========================================================================
# 提交
# ==========================================================================
async def _reopen_drafting(session: AsyncSession, staging: ImportStaging) -> None:
"""提交预检发现试运行用不了:试运行作废、退回 drafting,删掉试运行库。
用带状态条件的 UPDATE:预检期间(几次 await)暂存区可能已被放弃或过期,不能拿会话里过期的对象把它改回
drafting(SE-8)。已结束的回 409 staging_closed。"""
old = staging.trial_path
done = await session.execute(
update(ImportStaging)
.where(ImportStaging.id == staging.id, ImportStaging.status.in_(table_versions.STAGING_OPEN))
.values(trial_path=None, trial_key=None, status="drafting", updated_at=utcnow())
.execution_options(synchronize_session=False))
await session.commit()
if not done.rowcount:
raise ImportRefused(409, "staging_closed", _CLOSED)
await _remove_trial(old)
def _missing_text(items: list[ConfirmItem]) -> str:
shown = ";".join(f"{i.id}「{i.label}」" for i in items[:20])
more = f"等 {len(items)} 项" if len(items) > 20 else ""
return f"还有确认项没有勾选{more}:{shown}"
async def _back_to_drafting(session: AsyncSession, staging_id: str) -> None:
"""发布失败:试运行库已被发布流程消耗(或本来就不在),另开一个事务把暂存区退回 drafting。回执留着。"""
try:
await session.rollback()
except Exception: # noqa: BLE001
pass
row = await session.get(ImportStaging, staging_id, populate_existing=True)
if row is None or not table_versions.staging_is_open(row):
return
old = row.trial_path
row.trial_path = None
row.trial_key = None
row.status = "drafting"
row.updated_at = utcnow()
await session.commit()
await _remove_trial(old)
def _manifest(*, staging_id: str, kind: str, source_id: str, info: table_versions.PublishInfo, trial_sha: str | None,
ex_dict: dict[str, Any], file: dict[str, Any], recipe_meta: dict[str, Any], checks: list[dict[str, Any]],
overrides: list[dict[str, Any]], waivers: list[dict[str, Any]], confirmed: list[dict[str, Any]],
notes: SchemaNotes, ai: dict[str, Any], signed_by: str | None,
edits: list[dict[str, Any]] | None = None, signer_verified: bool = False) -> dict[str, Any]:
receipt = copy.deepcopy(ex_dict)
receipt.pop("problems", None)
return {
"format": MANIFEST_FORMAT,
"source_id": source_id, "import_id": info.import_id, "seq": info.seq, "build_id": info.build_id,
"build_reused": info.build_reused, "snapshot_id": info.snapshot_id,
"db_sha256": info.db_sha256, "trial_db_sha256": trial_sha,
"table_hashes": copy.deepcopy(ex_dict.get("table_hashes") or {}),
"file": file, "recipe": recipe_meta,
"engine_ver": recipe_types.RECIPE_ENGINE_VER, "parser_ver": recipe_parsers.PARSER_VER,
"period": copy.deepcopy(ex_dict.get("period")),
"checks": checks,
"acceptances": {"overrides": overrides, "waivers": waivers}, "confirmations": confirmed,
# receipt 照 asdict(Extraction) 去掉 problems:期 3 的 rows_excluded(排除的行,H2)随之写进来,只取本次
# 试运行的 Extraction,不取复用构建登记的回执(那里没有这个字段,第 8 节遗留项 4)
"receipt": receipt,
"notes": {"templates": dict(notes.templates),
"rendered": {t: {"comment": n.comment, "columns": dict(n.columns)} for t, n in notes.tables.items()}},
"ai": ai,
# 期 3:修复、框选的修改记录(含已被整份替换覆盖的,作为历史;不带撤销用的修改前起点)
"edits": copy.deepcopy(list(edits or [])),
# 是否认证照实记(05-已知遗留 (c)):开了登录、署名是登录的人时为真。以前写死 False,封存的证据记错了事实
"signed_by": {"name": signed_by, "verified": bool(signer_verified and signed_by)},
"staging_id": staging_id, "kind": kind,
"created_at": _now_iso(),
}
#: 发布原语的错误码 → 接口的状态和机读码(10.2 的错误映射表)
_PUBLISH_STATUS = {
"trial_missing": (409, "trial_required"), "trial_tampered": (409, "trial_tampered"),
"build_conflict": (409, "build_conflict"), "base_changed": (409, "base_changed"),
"part_missing": (409, "part_missing"), "part_tampered": (409, "part_tampered"),
"raw_missing": (409, "raw_missing"), "store_unavailable": (503, "store_unavailable"),
}
def _publish_refused(e: table_versions.PublishError) -> ImportRefused:
status, code = _PUBLISH_STATUS.get(e.code or "", (500, "publish_failed"))
return ImportRefused(status, code, str(e) or "导入失败,当前版本未受影响")
def _file_sha(path: Path) -> str | None:
try:
return raw_store.sha256_file(path) if path.is_file() and path.stat().st_size > 0 else None
except OSError:
return None
async def commit_staging(session: AsyncSession, staging: ImportStaging, *, trial_id: str, confirmations: list[str],
acceptances: list[Acceptance], signed_by: str | None,
signer_verified: bool = False) -> CommitResult:
"""POST imports/{id}/commit(7.8):预检 → (与现行导入相同时)不新建记录 → 发布构建、写配方、导入记录、
快照、清单,切指针。失败时指针不动,暂存区退回 drafting(试运行库已被发布流程消耗)。
期 3 的三条路径(12.6):
- 结果只有一期、快照库就是本期的构建库(替换模式,按期累积的 first / restart / 替换唯一的一期):照期 2 走
publish_build,带 snapshot_fields(mode、目标配方:累积快照的 id 要用);
- 结果有两期及以上:试运行时已物化好并集,这里走 publish_snapshot(本期构建、并集都 link,写导入清单和快照清单);
- 与当前版本里的某一期完全相同(同一构建、同一配方、同一文件名、没有接受、统计期不是人工录入):不新建任何记录。
按期累积时这一期是被替换的那一期(替换该期而构建相同)。"""
p = pipeline()
require_open(staging)
trial = staging.trial or {}
if not isinstance(staging.recipe, dict) or not trial:
raise ImportRefused(409, "trial_required", "还没有试运行,请先试运行")
# 试运行作废时回执留着(界面标「已失效」),键清空;这时工作配方可能已改成 schema 不过的样子
# (PUT 照存),先看键,别让 model_validate 的英文原文经全局 ValueError 处理器漏成 400
if not staging.trial_key:
raise ImportRefused(409, "trial_required", _TRIAL_STALE)
try:
parsed = Recipe.model_validate(staging.recipe)
except ValueError:
raise ImportRefused(409, "trial_required", _TRIAL_STALE) from None
recipe_sha = p.recipe_sha256(parsed)
ctx = staging.context_inputs or {}
key = trial_key(staging, recipe_sha, ctx)
if staging.trial_key != key or trial.get("trial_id") != key[:16] or trial_id != key[:16]:
raise ImportRefused(409, "trial_required", _TRIAL_STALE)
if trial.get("status") not in _ACTIVE_TRIAL:
raise ImportRefused(409, "trial_not_passed", "试运行没有通过(拒收或需要录入统计期),不能提交")
# 发布会消耗试运行库:只读进程在动任何东西之前就停下(503),别把用户的试运行白白删掉
table_versions.require_store_writable()
cx = await _context(session, staging)
source = cx.source
base = staging.base_snapshot_id
# 先看当前版本有没有被换掉:换了的话上一期回执也换了,按它重算的确认项没有意义(这里只是快速预检,
# 最终判定在 publish_build / publish_snapshot 的锁内)
if source is not None and source.current_snapshot_id != base:
raise ImportRefused(409, "base_changed", _BASE_CHANGED)
if source is None:
taken = (await session.execute(
select(DataSource.id).where(DataSource.name == staging.source_name))).first()
if taken is not None:
raise ImportRefused(409, "name_taken", f"名称「{staging.source_name}」已被占用,请放弃这次导入后换一个名称")
ex_dict = trial.get("extraction") or {}
checks = [check_from_dict(c) for c in trial.get("checks") or []]
plan = trial.get("accumulate") if isinstance(trial.get("accumulate"), dict) else None
union_info = trial.get("union") if isinstance(trial.get("union"), dict) else None
prev, _prev_name = await _prev_of(session, cx, trial.get("prev_import_id"))
items = list(p.confirm_items(_confirm_context(staging, cx, parsed, ex_dict, checks, trial.get("diff"), prev=prev,
accumulate=plan)))
shown = {str(i.get("id")) for i in trial.get("confirm_items") or [] if isinstance(i, dict)}
if any(i.required and i.id not in shown for i in items):
# 服务端此刻要求的确认项比试运行时给界面的多(配方来源、数据源的遮罩设置等在试运行之后变了):界面上的
# 清单已经不全,用户勾不到缺的那几项。试运行作废,请重新试运行,而不是回一个对不上清单的 422(AU-4)
await _reopen_drafting(session, staging)
raise ImportRefused(409, "trial_required", "试运行之后确认项有变化(配方来源或数据源设置变了),请重新试运行")
ticked = set(confirmations or [])
missing = [i for i in items if i.required and i.id not in ticked]
if missing:
raise ImportRefused(422, "confirm_required", _missing_text(missing))
acceptable = list(trial.get("acceptable") or [])
given: dict[str, Acceptance] = {}
for a in acceptances or []:
if a.check_id in acceptable and (a.reason or "").strip():
given[a.check_id] = a
too_long = [cid for cid, a in given.items() if len(a.reason.strip()) > ACCEPT_REASON_MAX]
if too_long:
raise ImportRefused(422, "acceptance_required",
f"接受理由不能超过 {ACCEPT_REASON_MAX} 字:{'、'.join(too_long)}")
lacking = [cid for cid in acceptable if cid not in given]
if lacking:
raise ImportRefused(422, "acceptance_required",
f"以下核对没有通过,接受前需要写明理由:{'、'.join(lacking)}")
path = Path(staging.trial_path) if staging.trial_path else None
try:
usable = path is not None and path.is_file() and path.stat().st_size > 0
except OSError:
usable = False
if not usable:
await _reopen_drafting(session, staging)
raise ImportRefused(409, "trial_required", "试运行的数据文件已不在,请重新试运行")
union_path: Path | None = None
if union_info is not None:
union_path = Path(str(union_info.get("path") or ""))
got = await asyncio.to_thread(_file_sha, union_path)
if got is None:
await _reopen_drafting(session, staging)
raise ImportRefused(409, "trial_required", "试运行时生成的累积数据文件已不在,请重新试运行")
if got != union_info.get("db_sha256"):
raise ImportRefused(409, "union_tampered", "试运行之后累积数据文件被改动过,请重新试运行")
human = bool(_build_inputs(ex_dict))
same = trial.get("same_as_import") or None
same_imp = await session.get(TableImport, str(same.get("id"))) if same and same.get("id") else None
current_ids = {str(x) for x in (cx.snapshot.imports or [])} if cx.snapshot is not None else set()
if (same_imp is not None and not given and not human and same_imp.id in current_ids
and same_imp.file_name == staging.file_name and source is not None and cx.recipe is not None
and cx.recipe.recipe_sha256 == recipe_sha):
# 与现行导入完全相同:不新建任何记录(final.md 2.6 第 5 条)。结束暂存区也在锁内、重读指针之后:
# 预检之后别的提交切走了指针的话,这里返回的「现行导入」已经不是现行的了
imp_id, imp_build, source_id = same_imp.id, same_imp.build_id, source.id
async with table_versions.store_lock():
cur = (await session.execute(
select(DataSource.current_snapshot_id, DataSource.current_recipe_id)
.where(DataSource.id == source_id))).first()
if cur is None or cur[0] != base:
raise ImportRefused(409, "base_changed", _BASE_CHANGED)
await _require_open_locked(session, staging.id)
old = table_versions.close_staging(staging, "committed")
staging.committed_import_id = imp_id
await session.commit()
await _remove_trial(old)
return CommitResult(import_id=imp_id, snapshot_id=cur[0] or "", build_id=imp_build,
recipe_id=cur[1] or "", build_reused=True, unchanged=True, source_id=source_id,
parts=max(len(current_ids), 1))
ex = extraction_from_dict(ex_dict)
who = signed_by or staging.signed_by
# 署名是这次请求的(signed_by)才谈得上认证;退回暂存区上记的那个,是否认证这里不知道,照实记未认证
who_verified = bool(signer_verified and signed_by)
acc = [Acceptance(check_id=a.check_id, reason=a.reason.strip(), signed_by=who) for a in given.values()]
# 这一期自己的说明(带实际的接受)进它的导入清单:版本页逐期显示「改写后的那一句」从这里取(第 5 节)。
# 快照冻结的说明在多期时另按各期最弱的结论合并(2.8),只剩本期时就是它
period_notes = p.build_notes(parsed, ex, checks, acc, null_counts=trial.get("null_counts"))
parts: list[Any] = []
if union_info is not None:
from app.data import source_versions as sv
try:
parts = await sv.load_parts(session, cx.snapshot)
except sv.PartProblem as e:
raise ImportRefused(409, e.code, e.message) from e
notes = await _union_notes_for_commit(p, parsed, ex, checks, acc, trial, union_info, plan or {}, parts)
else:
notes = period_notes
if plan is not None and plan.get("dropped"):
notes = _add_single_after_drop(notes)
problems = list(notes.problems) + [m for m in period_notes.problems if m not in notes.problems]
if problems:
raise ImportRefused(422, "notes_invalid", "说明没有通过检查:" + ";".join(problems[:5]))
# ---- 以下到 publish_build / publish_snapshot 之前,不改会话里的任何对象(7.2 第 1 条最后一项)
at = _now_iso()
by_id = {c.id: c for c in checks}
overrides, waivers = [], []
for a in acc:
rec = {"check_id": a.check_id, "reason": a.reason, "signed_by": a.signed_by, "at": at}
(overrides if by_id[a.check_id].category == "data_quality" else waivers).append(rec)
confirmed = [{"id": i.id, "label": i.label, "at": at} for i in items if i.id in ticked]
checks_full = [_jsonable(c) for c in checks]
period = ex_dict.get("period") or {}
inputs = trial.get("inputs") or {}
source_id, staging_id, kind = staging.source_id, staging.id, staging.kind
name, file_name, file_size, raw_sha = (staging.source_name, staging.file_name or "", int(staging.file_size or 0),
staging.raw_sha256)
base_origin = cx.recipe.origin if cx.recipe is not None else None
# 确认项按 effective_origin 算(上面已经算过);入库只记四种来源之一(rules_redraft → rules / manual)
origin = stored_origin(effective_origin(staging, base_origin), staging)
recipe_changed = cx.recipe is None or cx.recipe.recipe_sha256 != recipe_sha
recipe_id = new_id() if recipe_changed else cx.recipe.id
recipe_seq_now = cx.recipe.seq if cx.recipe is not None and not recipe_changed else None
canonical = p.canonical(parsed)
semantics = table_versions.recipe_engine_semantics(parsed)
build_id = table_versions.recipe_build_id(source_id, raw_sha, recipe_sha, inputs, semantics=semantics)
import_id = new_id()
ai = {"usage": copy.deepcopy(staging.ai_usage or []), "consents": copy.deepcopy(staging.ai_consents or [])}
edits = [{k: v for k, v in e.items() if k not in EDIT_INTERNAL} for e in staging.edits or [] if isinstance(e, dict)]
trial_sha = (trial.get("receipt") or {}).get("db_sha256")
report = copy.deepcopy(ex_dict)
report.pop("problems", None)
report.pop("context_checks", None)
import_fields = {
"recipe_id": recipe_id, "period_start": period.get("start"), "period_end": period.get("end"),
"context": copy.deepcopy(period) or None, "checks": _checks_summary(checks_full),
"overrides": overrides, "waivers": waivers, "confirmations": confirmed, "signed_by": who,
"staging_id": staging_id,
}
if source is None:
target = DataSource(id=source_id, name=name, kind="sqlite", readonly=True,
description=staging.description or "", origin="upload")
else:
target = source
manifest_ids: dict[str, str] = {}
activation = {"kind": "commit", "import_id": import_id, "signed_by": who}
def import_manifest(info: table_versions.PublishInfo) -> dict[str, Any]:
return _manifest(
staging_id=staging_id, kind=kind, source_id=source_id, info=info, trial_sha=trial_sha, ex_dict=ex_dict,
file={"name": file_name, "size": file_size, "raw_sha256": raw_sha},
recipe_meta={"id": recipe_id, "sha256": recipe_sha, "origin": origin, "canonical": canonical},
checks=checks_full, overrides=overrides, waivers=waivers, confirmed=confirmed,
notes=notes if union_info is None else period_notes, ai=ai, signed_by=who, edits=edits,
signer_verified=who_verified)
async def schema_cache_for(view: table_versions.SourceView, info: table_versions.PublishInfo) -> dict[str, Any]:
from app.core import artifact_store
cache = await table_versions.introspect_view(view, info.report)
cache = p.apply_notes(cache, notes)
if notes.problems:
raise table_versions.PublishError("导入失败:说明没能写进表结构(" + ";".join(notes.problems[:3])
+ "),当前版本未受影响")
table_versions.check_probe(cache, info.report, source_id)
mid = await artifact_store.put_json(import_manifest(info), kind="import_manifest",
meta={"source_id": source_id, "import_id": info.import_id})
manifest_ids["id"] = mid
cache["import_manifests"] = [mid]
return cache
async def union_cache_for(view: table_versions.SourceView, info: table_versions.SnapshotInfo) -> dict[str, Any]:
"""多期快照冻结的表结构:探查并集库、写说明,再写本期的导入清单和快照清单(2.9)。"""
from app.core import artifact_store
from app.data import source_versions as sv
union_report = info.union_report or union_info.get("report") or {}
probe = sv.union_table_report(union_report)
cache = await table_versions.introspect_view(view, probe)
cache = p.apply_notes(cache, notes)
if notes.problems:
raise table_versions.PublishError("导入失败:说明没能写进表结构(" + ";".join(notes.problems[:3])
+ "),当前版本未受影响")
table_versions.check_probe(cache, probe, source_id)
pinfo = table_versions.PublishInfo(
import_id=info.import_id or import_id, snapshot_id=info.snapshot_id, seq=int(info.seq or 0),
build_id=info.build_id or build_id, db_sha256=info.db_sha256 or "", build_reused=info.build_reused,
report=info.report, build_restored=info.build_restored)
mid = await artifact_store.put_json(import_manifest(pinfo), kind="import_manifest",
meta={"source_id": source_id, "import_id": pinfo.import_id})
manifest_ids["id"] = mid
by_imp = {x.info.import_id: x for x in parts}
ordered: list[Any] = []
for iid in union_info.get("imports") or []:
if iid is None:
ordered.append({"import_id": pinfo.import_id, "seq": pinfo.seq,
"period": {"start": period.get("start"), "end": period.get("end")},
"build_id": pinfo.build_id, "db_sha256": pinfo.db_sha256, "recipe_id": recipe_id,
"recipe_sha256": recipe_sha, "manifest_artifact": mid, "file_name": file_name,
"raw_sha256": raw_sha})
else:
ordered.append(by_imp[iid])
seq = recipe_seq_now
if seq is None:
last = (await session.execute(
select(func.max(TableRecipe.seq)).where(TableRecipe.source_id == source_id))).scalar() or 0
seq = last + 1
doc = sv.manifest_doc(
source_id=source_id, snapshot_id=info.snapshot_id, action=str((plan or {}).get("action") or "append"),
recipe={"id": recipe_id, "seq": seq, "sha256": recipe_sha},
union={"build_id": info.union_id, "db_sha256": info.union_db_sha256,
"table_hashes": dict(union_report.get("table_hashes") or {}), "union_ver": recipe_types.UNION_VER},
parts=ordered, report=union_report, label_sets=list((plan or {}).get("label_sets") or []),
gaps=list((plan or {}).get("gaps") or []), replaced=(plan or {}).get("replaces"), removed=None,
notes=notes, signed_by=who, signer_verified=who_verified)
smid = await artifact_store.put_json(doc, kind="snapshot_manifest",
meta={"source_id": source_id, "snapshot_id": info.snapshot_id})
cache["import_manifests"] = [mid if iid is None else by_imp[iid].info.manifest_artifact
for iid in union_info.get("imports") or []]
cache["snapshot_manifest"] = smid
return cache
async def before_commit(imp_id: str | None, snap_id: str) -> None:
# 锁内、同一事务:用 select 重读,不信会话里缓存的对象
now = utcnow()
cur = (await session.execute(
select(DataSource.current_snapshot_id).where(DataSource.id == source_id))).first()
if (cur[0] if cur is not None else None) != base:
raise CommitConflict("base_changed", _BASE_CHANGED)
# 暂存区本身也要在锁内重读:预检之后它可能已被放弃(放弃的请求先拿到了锁),已放弃的不能再发布(SE-8)
await _require_open_locked(session, staging_id)
if kind == "first" or source is None:
taken = (await session.execute(
select(DataSource.id).where(DataSource.name == name, DataSource.id != source_id))).first()
if taken is not None:
raise CommitConflict("name_taken", f"名称「{name}」已被占用,请放弃这次导入后换一个名称")
if recipe_changed:
last = (await session.execute(
select(func.max(TableRecipe.seq)).where(TableRecipe.source_id == source_id))).scalar() or 0
await session.execute(
update(TableRecipe).where(TableRecipe.source_id == source_id, TableRecipe.status == "active")
.values(status="superseded"))
session.add(TableRecipe(
id=recipe_id, source_id=source_id, seq=last + 1, recipe=canonical, recipe_sha256=recipe_sha,
recipe_format=recipe_types.RECIPE_FORMAT, origin=origin, status="active", confirmations=confirmed,
signed_by=who, ai_usage={"calls": ai["usage"], "consents": ai["consents"]}, staging_id=staging_id,
created_at=now, activated_at=now))
target.current_recipe_id = recipe_id
imp = await session.get(TableImport, imp_id) if imp_id else None
if imp is not None:
imp.manifest_artifact = manifest_ids.get("id")
staging.committed_import_id = imp_id
table_versions.close_staging(staging, "committed", now=now)
try:
if union_info is not None:
imports = [import_id if iid is None else str(iid) for iid in union_info.get("imports") or []]
pub = await table_versions.publish_snapshot(
session, target, imports=imports, mode="accumulate", recipe_id=recipe_id, recipe_sha256=recipe_sha,
expected_current=base,
period_build=table_versions.PeriodBuild(
tmp_db=path, build_id=build_id,
options=table_versions.recipe_build_options(recipe_sha, inputs, semantics=semantics),
report=report, expected_sha256=trial_sha or ""),
new_import=table_versions.NewImport(import_id=import_id, raw_sha=raw_sha, file_name=file_name,
file_size=file_size, fields=import_fields),
union=table_versions.UnionBuild(
tmp_db=union_path, union_id=str(union_info["union_id"]), options=dict(union_info.get("options") or {}),
report=dict(union_info.get("report") or {}), expected_sha256=str(union_info.get("db_sha256") or "")),
schema_cache_for=union_cache_for, before_commit=before_commit, activation=activation)
return CommitResult(import_id=pub.import_id or import_id, snapshot_id=pub.snapshot_id,
build_id=pub.build_id or build_id, recipe_id=recipe_id, build_reused=pub.build_reused,
unchanged=False, source_id=source_id, build_restored=pub.build_restored,
parts=len(imports), snapshot_reused=bool(pub.snapshot_reused or pub.snapshot_revived))
result = await table_versions.publish_build(
session, target, tmp_db=path, build_id=build_id,
build_options=table_versions.recipe_build_options(recipe_sha, inputs, semantics=semantics),
engine_ver=recipe_types.RECIPE_ENGINE_VER, report=report, raw=None, raw_sha=raw_sha,
file_name=file_name, file_size=file_size, schema_cache_for=schema_cache_for,
import_fields=import_fields, expected_db_sha256=trial_sha, before_commit=before_commit,
import_id=import_id,
snapshot_fields={"mode": parsed.mode, "recipe_id": recipe_id, "recipe_sha256": recipe_sha},
activation=activation)
except ImportRefused:
await _back_to_drafting(session, staging_id)
raise
except table_versions.PublishError as e:
await _back_to_drafting(session, staging_id)
raise _publish_refused(e) from e
except IntegrityError as e:
await _back_to_drafting(session, staging_id)
if unique_violation(e, "data_sources", "name"):
raise ImportRefused(409, "name_taken", f"名称「{name}」已被占用,请放弃这次导入后换一个名称") from e
raise ImportRefused(500, "publish_failed", "导入失败:写入版本记录时出错,当前版本未受影响") from e
except BaseException:
await _back_to_drafting(session, staging_id)
raise
finally:
if union_path is not None:
# publish_snapshot 无论成败都消耗并集的试运行文件;走 publish_build 的路径用不上它,也一并删掉
await asyncio.to_thread(_drop_file, union_path)
return CommitResult(import_id=result.import_id, snapshot_id=result.snapshot_id, build_id=result.build_id,
recipe_id=recipe_id, build_reused=result.build_reused, unchanged=False, source_id=source_id,
build_restored=result.build_restored, parts=1)
def _drop_file(path: Path) -> None:
if path.parent.name != table_versions.TRIALS_DIR:
return
for suffix in ("", "-journal", "-wal", "-shm"):
Path(f"{path}{suffix}").unlink(missing_ok=True)
async def _union_notes_for_commit(p: Pipeline, parsed: Recipe, ex: Extraction, checks: list[CheckResult],
acc: list[Acceptance], trial: dict[str, Any], union_info: dict[str, Any],
plan: dict[str, Any], parts: list[Any]) -> SchemaNotes:
"""提交时冻结的并集说明:与试运行预览同一套合并规则,只是本期带上实际的接受(2.8)。"""
from app.data import source_versions as sv
report = union_info.get("report") or {}
by_imp = {x.info.import_id: x for x in parts}
period = (trial.get("extraction") or {}).get("period") or {}
start, end = str(period.get("start") or ""), str(period.get("end") or "")
run = _TrialRun(parsed=parsed, problems=[], extraction=ex, checks=checks, null_counts=trial.get("null_counts"))
note_parts = []
for iid in union_info.get("imports") or []:
if iid is None:
note_parts.append(_union_note_new(parsed, ex, run, start, end, acc))
else:
x = by_imp.get(iid)
if x is None:
raise ImportRefused(409, "base_changed", _BASE_CHANGED)
note_parts.append(await _in_slot(sv.note_part, x))
tables = {str(k): [ColumnOut(**_known(ColumnOut, c)) for c in v or [] if isinstance(c, dict)]
for k, v in (report.get("tables") or {}).items()}
return p.build_union_notes(
parsed, note_parts, union_tables=tables, null_counts=report.get("null_counts"),
part_null_counts=report.get("part_null_counts"), added=list(report.get("added") or []),
retired=list(report.get("retired") or []), label_sets=list(plan.get("label_sets") or []),
gaps=bool(plan.get("gaps")), dropped=False)
# ==========================================================================
# 放弃
# ==========================================================================
async def _require_open_locked(session: AsyncSession, staging_id: str) -> None:
"""锁内按库里的状态判断暂存区是否还开着(select 重读,不信会话里缓存的对象)。"""
status = (await session.execute(
select(ImportStaging.status).where(ImportStaging.id == staging_id))).scalar_one_or_none()
if status not in table_versions.STAGING_OPEN:
raise CommitConflict("staging_closed", _CLOSED)
async def discard_staging(session: AsyncSession, staging: ImportStaging) -> None:
"""DELETE imports/{id}:status → discarded、瘦身、删试运行库;原件已没有别的引用时当场删除。
进锁之后重读状态:排在提交后面等锁的放弃,拿到锁时暂存区可能已经提交了,不能把 committed 改写成
discarded(SE-8)。只读进程不放弃:放弃要删试运行库和原件(第 8 节遗留项 5)。"""
require_open(staging)
table_versions.require_store_writable()
async with table_versions.store_lock():
await session.refresh(staging, ["status"])
require_open(staging)
old = table_versions.close_staging(staging, "discarded")
await session.commit()
await _remove_trial(old)
await table_versions.release_staging_raw(session, staging)
# ==========================================================================
# 响应形状(7.4 StagingOut)
# ==========================================================================
def _span_count(spans: Any) -> int:
total = 0
for s in spans or []:
try:
total += int(s[1]) - int(s[0]) + 1
except (TypeError, ValueError, IndexError):
continue
return total
def _bounds_a1(b: Any) -> str | None:
if not isinstance(b, dict):
return None
from app.data.xlsx_scan import cell_ref
try:
return f"{cell_ref(int(b['min_row']), int(b['min_col']))}:{cell_ref(int(b['max_row']), int(b['max_col']))}"
except (KeyError, TypeError, ValueError):
return None
def import_mode(source: DataSource | None) -> str | None:
if source is None:
return None
if source.current_recipe_id:
return "recipe"
return "simple" if (source.origin or "manual") == "upload" else None
def _iso(v: Any) -> str | None:
return v.isoformat() if v is not None else None
def staging_out(staging: ImportStaging, source: DataSource | None, *,
ai: AiAvailability | None = None, units: tuple[str, ...] | None = None) -> dict[str, Any]:
"""7.4 的 StagingOut。ai 是模型接入的可用性(只在提供 AI 起草时由接口层查好传进来)。结束(瘦身)后的
暂存区照样能给出:大字段都是空的。"""
scan = staging.scan or {}
sheets, skipped = [], []
for s in scan.get("sheets") or []:
if not isinstance(s, dict):
continue
if s.get("state") != "visible":
skipped.append({"sheet": s.get("name"), "state": s.get("state")})
continue
sheets.append({
"name": s.get("name"), "state": s.get("state"), "bounds": _bounds_a1(s.get("bounds")),
"nonempty": int(s.get("nonempty") or 0),
"merged": int(s.get("merged_total") or len(s.get("merged") or [])),
"formulas": int(s.get("formulas") or 0),
"formulas_uncached": int((s.get("formulas_uncached") or {}).get("total") or 0),
"hidden_rows": _span_count(s.get("hidden_rows")), "hidden_cols": _span_count(s.get("hidden_cols")),
})
drafts = staging.drafts or {}
trial = staging.trial
if isinstance(trial, dict):
trial = {k: copy.deepcopy(v) for k, v in trial.items() if k not in TRIAL_INTERNAL}
ai_sum = drafts.get("ai")
if isinstance(ai_sum, dict):
ai_sum = {**ai_sum, "draft": _draft_out(ai_sum.get("draft"))}
offered = table_versions.staging_is_open(staging) and ai_offered(staging)
recipe = _full_form(staging.recipe) if isinstance(staging.recipe, dict) else None
facts = staging.facts or {}
marks = drafts.get("marks") or []
if not marks and isinstance(staging.trial, dict):
marks = _regions(staging.trial.get("extraction"))
if units is None:
units = tuple(recipe_parsers.UNITS)
fixes, fix_where = proposals_for(pipeline(), staging)
trial, draft_problems, recipe_problems = _with_fix_ids(fixes, fix_where, trial, staging)
edits = [e for e in staging.edits or [] if isinstance(e, dict)]
open_ = table_versions.staging_is_open(staging)
return {
"id": staging.id, "kind": staging.kind, "status": staging.status,
"source": {"id": staging.source_id, "name": source.name if source is not None else staging.source_name,
"exists": source is not None, "import_mode": import_mode(source)},
"file": {"name": staging.file_name, "size": int(staging.file_size or 0),
"sha256_prefix": (staging.raw_sha256 or "")[:8]},
"sheets": sheets, "skipped_sheets": skipped, "full_calc_on_load": bool(scan.get("full_calc_on_load")),
"grids": list(staging.grid_preview or []),
"draft": _draft_out(drafts.get("rules")),
"ai_draft": ai_sum,
"ai": {"offered": offered, "available": bool(ai.available) if ai is not None else False,
"reason": ai.reason if ai is not None else "", "model": ai.model if ai is not None else "",
"provider": ai.provider if ai is not None else ""},
"ai_consents": list(staging.ai_consents or []),
"recipe": recipe, "recipe_origin": staging.recipe_origin,
"recipe_problems": recipe_problems,
"answers": dict(staging.answers or {}),
"cards": list(staging.cards or []), "questions": list(staging.questions or []),
"draft_problems": draft_problems, "draft_partial": bool(staging.draft_partial),
"candidates": dict(facts.get("candidates") or {}),
"units": list(units),
"marks": marks,
"trial": trial,
"context_inputs": dict(staging.context_inputs or {}),
"base_snapshot_id": staging.base_snapshot_id,
"committed_import_id": staging.committed_import_id,
"created_at": _iso(staging.created_at), "updated_at": _iso(staging.updated_at),
"expires_at": _iso(staging.expires_at),
# 期 3(9.4 StagingOut)
"fixes": [_jsonable(f) for f in fixes],
"recipe_sha256": staging.recipe_sha256,
"edits": [_edit_out(e, last=i == len(edits) - 1, open_=open_) for i, e in enumerate(edits)],
"answers_dropped": list(staging.answers_dropped or []),
}
def _with_fix_ids(fixes: list[Any], where: str, trial: Any, staging: ImportStaging
) -> tuple[Any, list[dict[str, Any]], list[dict[str, Any]]]:
"""按提议的 anchor 把 fix_ids 写到对应的问题上(3.1,评审丙-B2):界面只按 fix_ids 放按钮,不靠 code 加坐标去猜
(title_not_found 没有坐标,两个分段同时找不到标题时只有这样才分得清)。没有提议的问题 fix_ids 为 []。
返回 (trial, draft_problems, recipe_problems),都是新的副本。"""
by_problem: dict[int, list[str]] = {}
by_recipe: dict[int, list[str]] = {}
for f in fixes:
anchor = getattr(f, "anchor", None)
index = getattr(anchor, "index", None)
if index is None:
continue
if getattr(anchor, "kind", None) == "problem":
by_problem.setdefault(int(index), []).append(f.id)
elif getattr(anchor, "kind", None) == "recipe_problem":
by_recipe.setdefault(int(index), []).append(f.id)
def tag(items: Any, ids: dict[int, list[str]]) -> list[dict[str, Any]]:
return [{**x, "fix_ids": list(ids.get(i, []))} for i, x in enumerate(items or []) if isinstance(x, dict)]
draft_problems = tag(staging.draft_problems, by_problem if where == "draft" else {})
recipe_problems = tag(staging.recipe_problems, by_recipe)
if isinstance(trial, dict) and isinstance(trial.get("problems"), list):
trial = {**trial, "problems": tag(trial["problems"], by_problem if where == "trial" else {})}
return trial, draft_problems, recipe_problems
__all__ = [
"ACCEPT_REASON_MAX", "ANSWER_REASON_MAX", "CommitConflict", "CommitResult", "EditRequest", "ImportRefused",
"MANIFEST_FORMAT", "Pipeline", "acceptable_ids", "ai_offered", "ai_preview", "answer", "carry_answers",
"check_from_dict", "commit_staging", "consent_token", "edit_apply", "edit_preview", "edit_undo", "proposals_for",
"redraft_rules",
"current_import", "current_recipe", "default_pipeline", "discard_staging", "effective_origin",
"extraction_from_dict", "import_mode", "pipeline", "put_recipe", "redraft", "replay_answers", "reupload",
"run_ai_draft", "run_trial", "stage_upload", "staging_out", "stored_origin", "trial_key", "trial_status",
]