Excel 导入改造期 4:证据下钻——报告里引用上传表格的数,点开能看到它来自原表格的哪一格;裁判核对时看得到来源、口径、接受理由和区域外文字

推断的来源(新 engine/direct_select、data/provenance、data/provenance_db;api/evidence 新增推断来源接口)
- 只对直接引用一列的查询下钻:受限文法用和执行时相同的 SQLite 分词读法,别名互换、自连接、表达式、聚合、子查询、集合运算(含藏在注释、字符串、方括号、转义、BOM 之后的)一律只给表级来历;再用 EXPLAIN 字节码和授权回调兜底
- 结果带齐主键、值非空,按主键参数化回查只读快照、类型和值严格一致才给格子;不一致时整份重算库哈希,哈希不符标红
- 从导入清单和溯源找回格子:宽表、交叉表逆透视的长表(日期取自表头格、年份取自统计期、行标签、分段标题)、列表、合计表、按期累积的并集(换算到所属那一期);原文与规范写法不同时一并列出
- 清单读不到、登记哈希与清单不一致、快照已回收、遮罩列,都只给表级来历或不给,写明原因;原件已清除、接受已作废照实显示;未封存的运行照常推断并标明
- 相关核对:这一行、这一期的核对结论与接受理由

裁判摘录(engine/judge)
- 带期 4 标记的报告,上传表格的查询旁加来源行、口径行、接受理由、可见单元格里的区域外文字和人工录入的统计期;限定语写明这些数未经系统核对,长度有上限、截断时写明
- 隐藏行列里的区域外文字不送;设了遮罩的源只写条数

前端证据面板(run/EvidenceProvenance;ManifestBody 从版本页抽出复用)
- 「推断的来源」一节与已封存的证据分开标注:格子坐标、取自哪些格、年份来源、规范写法、回查、相关核对、数据版本与各期;标红与拒绝原因按状态显示

不变的部分
- 文档顶层加 provenance 标记,只有新报告走新路径;期 4 之前的运行、非上传源、手工源的面板、复核、裁判摘录逐字不变,由金样测试钉住
- 执行器给区域外文字记录是否在隐藏行列;不建「导入校验」系统表(D6)
This commit is contained in:
ThearchyHelios
2026-10-02 14:38:10 +08:00
parent 2cdbf9a733
commit e4f8ee5d08
35 changed files with 32542 additions and 272 deletions
+2 -1
View File
@@ -208,7 +208,8 @@ flowchart LR
要进口径卡。数据源可设 `mask_columns` 遮罩列(只减少暴露,不是安全边界)。
**点开看到什么。** 数字:指标的原式、代入式和复算结果,输入与快照是否一致,查询的 SQL、被引用的行加前后
各 2 行、高亮到格,最后是封存状态;只认封存范围内的事件交回过的工件。表名、字段名:出现在哪次查询、类型、
各 2 行、高亮到格,最后是封存状态;只认封存范围内的事件交回过的工件。上传表格的单元格可以追到原表格子
(推断的来源)。表名、字段名:出现在哪次查询、类型、
所属的表。表结构快照、SQL、结果列里都没有的名字标成可疑实体(「可能是编造的名字」,附最接近的 3 个);
快照被截断、核对不了的只说核对不了,不冤枉成编造。
+493 -24
View File
@@ -31,22 +31,49 @@ import asyncio
import contextlib
import csv
import io
import logging
import re
import sqlite3
from collections import OrderedDict
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any
from dataclasses import asdict, dataclass, field, replace
from pathlib import Path
from typing import Any, get_args
from fastapi import APIRouter, Body, Query
from fastapi.responses import JSONResponse, Response
from sqlalchemy import select, update
from sqlalchemy.exc import StatementError
from app.api.coded import CodedHTTPException
from app.core.artifact_store import load
from app.core.events import EventType
from app.data.engine import masked_columns
from app.data import provenance as prov
from app.data import provenance_db
from app.data.engine import SnapshotTampered, engines, masked_columns
from app.data.names import name_key
from app.data.provenance_types import (
DOC_PROVENANCE,
Alert,
CellRef,
Chain,
ChainProblem,
LocateProblem,
ProvenanceOut,
RawState,
Reason,
Recheck,
RelatedCheck,
ReportRef,
SelectRefusal,
VersionView,
alert,
refuse,
)
from app.data.table_versions import SnapshotManifestMismatch, SnapshotMissing
from app.db.base import SessionLocal
from app.db.models import DataSource, Run, RunEvent, Workflow
from app.engine import direct_select
from app.engine.evidence import (
ENTITY_KINDS,
GUESS_NOTE,
@@ -76,6 +103,7 @@ from app.engine.judge import FIELDS_MAX, SETTLED, UNSUPPORTED, VERDICTS, candida
from app.engine.toolcalls import QUERY_PREFIX
router = APIRouter(prefix="/api/runs", tags=["evidence"])
logger = logging.getLogger(__name__)
SCHEMA = "agentlab.evidence/1"
@@ -673,20 +701,8 @@ async def evidence_segment(run_id: str, segment_id: str, report: str | None = No
一次运行里有好几份报告时用 ?report=<节点 id> 指定;不指定取成果里标注的那一份。
"""
sealed = await _sealed(run_id)
reports = _reports(sealed)
if not reports:
raise CodedHTTPException(404, "本次运行没有封存的报告文档,没有可查看的片段", REPORT_NOT_FOUND)
chosen = _choose(reports, report, sealed)
if chosen.hash_ok is False:
raise CodedHTTPException(409, "报告文档与哈希不一致,疑似被修改,不能作为证据展示", DOC_TAMPERED)
if chosen.doc is None:
raise CodedHTTPException(404, "无法读取报告文档", DOC_MISSING)
found = find_segment(chosen.doc, segment_id)
if found is None:
raise CodedHTTPException(404, "报告中没有这个片段", SEGMENT_NOT_FOUND)
block, unit, seg = found
doc = chosen.doc
chosen, (block, unit, seg) = _open_segment(sealed, report, segment_id)
doc = chosen.doc or {}
markdown = str(doc.get("markdown") or "")
start, end = unit.get("span") or [0, 0]
chain = await _chain(seg, doc, sealed, chosen.node_id)
@@ -712,6 +728,23 @@ async def evidence_segment(run_id: str, segment_id: str, report: str | None = No
}
def _open_segment(sealed: _Sealed, wanted: str | None, segment_id: str
) -> tuple[_Report, tuple[dict[str, Any], dict[str, Any], dict[str, Any]]]:
"""选报告、确认文档完好、找到片段:(报告, (块, 句子, 片段))。片段接口和推断来源接口共用同一套错误码。"""
reports = _reports(sealed)
if not reports:
raise CodedHTTPException(404, "本次运行没有封存的报告文档,没有可查看的片段", REPORT_NOT_FOUND)
chosen = _choose(reports, wanted, sealed)
if chosen.hash_ok is False:
raise CodedHTTPException(409, "报告文档与哈希不一致,疑似被修改,不能作为证据展示", DOC_TAMPERED)
if chosen.doc is None:
raise CodedHTTPException(404, "无法读取报告文档", DOC_MISSING)
found = find_segment(chosen.doc, segment_id)
if found is None:
raise CodedHTTPException(404, "报告中没有这个片段", SEGMENT_NOT_FOUND)
return chosen, found
def _choose(reports: list[_Report], wanted: str | None, sealed: _Sealed) -> _Report:
if wanted:
chosen = next((r for r in reports if r.node_id == wanted), None)
@@ -743,8 +776,10 @@ async def _chain(seg: dict[str, Any], doc: dict[str, Any], sealed: _Sealed,
return []
if cite.get("kind") == "cell":
locator = cite.get("locator") or {}
# 只有直接引用的单元格才可能推断来源(P4-SPEC 2.8.2)。指标链里的查询步骤不带:指标片段的推断来源只会
# 回「不是单元格」,带上提示等于让界面白发一次请求
return [await _query_step(str(entry.get("artifact") or ""), [(locator.get("row"), locator.get("column"))],
doc, sealed, masks)]
doc, sealed, masks, hint=doc.get("provenance") == DOC_PROVENANCE)]
if cite.get("kind") == "input":
name = (cite.get("locator") or {}).get("field")
inputs = _input_payloads(sealed)
@@ -857,11 +892,15 @@ class _Masks:
async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str, Any], sealed: _Sealed,
masks: _Masks) -> dict[str, Any]:
masks: _Masks, *, hint: bool = False) -> dict[str, Any]:
"""查询步骤:被引用的格所在的行加前后各 WINDOW 行,高亮那几格,遮掉数据源设了遮罩的列。
快照只认封存范围内的事件交回过的那些(tool.end.query_artifact、node.finished.evidence 的
query 条目);目录里写着、事件里追不到的,照实说不认,一行都不给。
hint(期 4,P4-SPEC 2.8.2):调用方是带文档标记的单元格片段时为真。这时快照若是上传表格的(有 data_version),
步骤多一个 `provenance: true`,界面据此才去请求推断来源接口;其余情况一个键都不加,形状和期 4 之前完全相同
(期 4 之前的文档、手工源、指标链都不多发请求)。不看封存状态:未封存的运行照常推断,响应里照实标明。
"""
catalog = doc.get("catalog") or {}
alias = _query_alias(catalog, artifact)
@@ -905,7 +944,8 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str
step.update(sql=snap.get("sql"), columns=columns, total_rows=len(rows), truncated=bool(snap.get("truncated")),
masked=[columns[i] for i in sorted(masked)],
# 查询时从驱动的原始值记下的列类型(老快照没有):文本列里的 "2026" 按文本显示,不当数
column_types=snap.get("column_types") if isinstance(snap.get("column_types"), dict) else {})
column_types=snap.get("column_types") if isinstance(snap.get("column_types"), dict) else {},
**({"provenance": True} if hint and snap.get("data_version") else {}))
if len(index) > MAX_WINDOW_ROWS:
index, step["window_truncated"] = index[:MAX_WINDOW_ROWS], True
step["row_index"] = index
@@ -915,6 +955,395 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str
return step
# --------------------------------------------------------------------------
# GET /api/runs/{run_id}/evidence/segments/{segment_id}/provenance(期 4:推断的来源)
# --------------------------------------------------------------------------
@router.get("/{run_id}/evidence/segments/{segment_id}/provenance")
async def evidence_provenance(run_id: str, segment_id: str, report: str | None = None) -> dict[str, Any]:
"""点开报告里一个直接引用的查询单元格时,这一格来自原表的哪个工作表、哪一格(P4-SPEC 2.8.1)。
选报告、错误码和片段接口相同(run_not_found、evidence_report_not_found、evidence_doc_tampered、
evidence_doc_missing、evidence_segment_not_found)。其余情况一律 200:推断不出来不是错误,结论写在
status、reason、alert 里。按需算、不缓存(2.9):原件是否清除、数据文件是否被改都是当前状态,事后会变。
返回 `asdict(ProvenanceOut)`:CellSource 的「from」在 Python 里叫 from_,asdict 已经换回「from」;返回注解
写 dict,不写 ProvenanceOut,和其余证据路由一样不让框架按类型再校验、再序列化一遍(provenance_types 模块说明)。
"""
sealed = await _sealed(run_id)
chosen, (_block, _unit, seg) = _open_segment(sealed, report, segment_id)
out = await segment_provenance(chosen, seg, sealed, masks=_Masks())
return asdict(out)
#: R14 之后(绑定快照、取引擎、编译核对、回查、行级核对)会抛的三类异常,按 _snapshot_failure 的顺序映射
_SNAPSHOT_ERRORS: tuple[type[BaseException], ...] = (SnapshotMissing, SnapshotTampered)
def _snapshot_failure(exc: BaseException) -> tuple[Reason, Alert | None]:
"""R14 之后抛出的数据文件异常 → (原因, 标红提示)(P4-SPEC 2.3 的异常映射)。
**顺序是守卫点**:SnapshotManifestMismatch 同时继承 SnapshotMissing 和引擎的 SnapshotTampered
(table_versions.py 的定义),必须最先认,否则会被当成文件被改(db_tampered)或文件不在(snapshot_gone):
1. SnapshotManifestMismatch:登记的哈希与快照清单对不上 → chain_mismatch,标红;
2. SnapshotTampered(含借连接时指纹不符):数据文件与登记的版本不一致 → db_tampered,标红;
3. SnapshotMissing:绑定时文件没了、记录不全 → snapshot_gone,不标红(被运行引用的版本受保护,出现多半是
数据源被删或手工清理)。
三种都是 table_only:数据版本一节只靠清单,照样给;不给格子,已经算出的行级状态一律丢掉(调用方负责)。
不是这三类的异常原样抛出:不该被当成「推断不出来」吞掉。
"""
if isinstance(exc, SnapshotManifestMismatch):
return refuse("chain_mismatch"), alert("chain_mismatch")
if isinstance(exc, SnapshotTampered):
return refuse("db_tampered"), alert("db_tampered")
if isinstance(exc, SnapshotMissing):
return refuse("snapshot_gone"), None
raise exc
class _Artifacts:
"""一次请求里取过的工件(P4-SPEC 2.9「同一工件只取一次」)。取回的结果按 _fetch 的口径记 (内容, hash_ok)。
- `await get(id)`:在线程里取(快照、清单可能几十 KB 到几 MB,读文件、复验哈希不该卡事件循环,H9);
- `load(id)`:同步取,语义同 artifact_store.load(取不到 None,哈希不符抛 ValueError),给已经在线程里跑的
纯函数当 loader 用(WP-B 的 resolve_chain 等)。两条路共用同一份备忘。
"""
def __init__(self) -> None:
self._memo: dict[str, tuple[Any, bool | None]] = {}
async def get(self, artifact: str) -> tuple[Any, bool | None]:
key = str(artifact or "")
if not key:
return None, None
if key not in self._memo:
self._memo[key] = await asyncio.to_thread(_fetch, key)
return self._memo[key]
def load(self, artifact: str) -> Any:
key = str(artifact or "")
if not key:
return None
if key not in self._memo:
self._memo[key] = _fetch(key)
content, ok = self._memo[key]
if ok is False:
raise ValueError(f"工件 {key[:12]} 与哈希不一致")
return content
def _cell_ref(cite: dict[str, Any], entry: dict[str, Any]) -> CellRef | None:
"""片段本身是单元格引用(有 alias 和 row、column 齐全的定位)就给,与 status 无关(2.8.1「字段取值」)。"""
locator = cite.get("locator") or {}
row, column = locator.get("row"), locator.get("column")
if cite.get("kind") != "cell" or not cite.get("alias") or not isinstance(row, int) or isinstance(row, bool) \
or not isinstance(column, str) or not column:
return None
artifact = entry.get("artifact")
return CellRef(alias=str(cite["alias"]), row=row, column=column, artifact=str(artifact) if artifact else None)
async def segment_provenance(report: _Report, seg: dict[str, Any], sealed: _Sealed, *,
masks: _Masks) -> ProvenanceOut:
"""一个片段的推断来源:按 P4-SPEC 2.3 的规则表顺序判断,第一条不满足的就是结论。
**封存状态不进规则**(1.3 调整 11):未封存(停在审批上)、封存核对不通过的运行照常判断,`sealed` 照实给
sealed.trusted,界面另加一句。查询快照、表结构快照按「本次运行的事件交回过」认(sealed.queries、
sealed.schemas,_sealed 对未封存的运行按全部事件拼),不经 _sealed_loader:它在 trusted 为假时一律不给,
会把未封存的运行误判成清单无法读取(2.8.3)。
这里判 R0–R4(只看文档、目录和两份快照),R5 起交给 _drill:清单链(provenance)、SQL 判据(direct_select)、
遮罩、空值、数据文件(provenance_db)、找回格子和相关核对。
"""
doc = report.doc or {}
cite = seg.get("cite") or {}
entry = (doc.get("catalog") or {}).get(cite.get("alias")) or {}
base: dict[str, Any] = {
"report": ReportRef(node_id=report.node_id, doc_artifact=report.doc_artifact),
"segment": str(seg.get("id") or ""), "cell": _cell_ref(cite, entry), "sealed": sealed.trusted,
}
def none(code: Any) -> ProvenanceOut:
return ProvenanceOut(status="none", reason=refuse(code), **base)
# R0:期 4 之前组装的文档(没有标记)一律不推断,期 4 之前的面板不变(1.4)
if doc.get("provenance") != DOC_PROVENANCE:
return none("legacy_doc")
# R1:只对直接引用的查询单元格下钻;代码节点、Agent 字段的取数(工具名不是 db_query__ 开头)不算
if (cite.get("kind") != "cell" or cite.get("status") != "resolved" or seg.get("state") != "deterministic"
or entry.get("kind") != "query" or not str(entry.get("tool") or "").startswith(QUERY_PREFIX)):
return none("not_cell")
artifacts = _Artifacts()
# R2:查询快照是本次运行的事件交回过的,取回时哈希复验通过
artifact = str(entry.get("artifact") or "")
query, ok = await artifacts.get(artifact) if artifact in sealed.queries else (None, None)
if ok is not True or not isinstance(query, dict):
return none("not_sealed")
# R3:上传表格的查询快照才有 data_version(datasource.py 只给上传源写)
if not query.get("data_version"):
return none("not_upload")
# R4:表结构快照同样要是本次运行交回过、复验通过的;不是按配方导入的(简单导入、v0 迁移出的版本)没有溯源
schema_id = str(query.get("schema_artifact") or "")
schema, ok = await artifacts.get(schema_id) if schema_id in sealed.schemas else (None, None)
if ok is not True or not isinstance(schema, dict):
return none("not_sealed")
if schema.get("import_mode") != "recipe":
return none("simple_upload")
return await _drill(base, seg=seg, entry=entry, query=query, schema=schema, artifacts=artifacts,
masks=masks, sealed=sealed)
async def _drill(base: dict[str, Any], *, seg: dict[str, Any], entry: dict[str, Any], query: dict[str, Any],
schema: dict[str, Any], artifacts: _Artifacts, masks: _Masks, sealed: _Sealed) -> ProvenanceOut:
"""R5–R17(P4-SPEC 2.3):第一条不满足的就是结论。
- R5、R6 只看内容寻址的工件(resolve_chain,放线程:要逐份取回清单、复验哈希)。链断了就什么来历都不给:
status none、version 为 null,标红;
- 之后的每一条都照样给数据版本(它只靠清单),不给格子:R7–R11 只看查询快照和冻结表结构,R12、R13 对照数据库
里的数据文件记录,R14 起打开数据文件。R14 之后任何一步抛出的数据文件异常按 _snapshot_failure 的顺序映射,
已经算出的格子、行级状态一律丢掉(2.3 末尾);
- 全部满足才给格子(inferred)。
"""
# R5(取不回、哈希不符)、R6(链本身对不上):resolve_chain 不往外抛,结论在 ChainProblem 里
chain = await asyncio.to_thread(prov.resolve_chain, query, schema, artifacts.load)
if isinstance(chain, ChainProblem):
return ProvenanceOut(status="none", reason=refuse(chain.code, chain.detail), alert=alert(chain.code), **base)
frozen = _frozen_tables(schema)
tables = {name: _frozen_columns(meta) for name, meta in frozen.items()}
sql = str(query.get("sql") or "")
# 遮罩 = 数据源现在设的 + 查询当时记下的(同 _query_step 的口径:数据源改名、删掉了也照遮)
source_name = (sealed.queries.get(str(entry.get("artifact") or "")) or {}).get("source") \
or entry.get("source") or query.get("source")
current, _found = await masks.of(source_name)
hidden = current | _recorded_masks(query)
async with SessionLocal() as session:
states = await provenance_db.import_states(session, [p.import_id for p in chain.parts])
record = await provenance_db.snapshot_record(session, chain.snapshot_id)
# 版本页同一个口径的表:SQL 里出现的冻结表结构的表(tables_in 宁可多认,2.4)
version = _version_of(chain, direct_select.tables_in(sql, tables), states, manifest_view=not hidden)
def table_only(reason: Reason, red: Alert | None = None) -> ProvenanceOut:
return ProvenanceOut(status="table_only", reason=reason, alert=red, version=version, **base)
# R7:受限文法内的直接选取(识别器和执行这条 SQL 的 SQLite 读同一串记号)。传的是快照里原样存下的 SQL
columns = [str(c) for c in query.get("columns") or []]
picked = direct_select.recognize(sql, result_columns=columns, tables=tables)
if isinstance(picked, SelectRefusal):
return table_only(refuse(picked.code, picked.detail))
# R8:被引用的结果列按第一次出现的位置对应选取项(locate_cell 也按第一次出现取)。识别器已拒绝重名,
# 这里对不上只是防御
locator = (seg.get("cite") or {}).get("locator") or {}
row, column = locator.get("row"), locator.get("column")
if not isinstance(column, str) or column not in columns or columns.index(column) >= len(picked.columns):
return table_only(refuse("unparsed"))
at = columns.index(column)
target = picked.columns[at]
# R9:表有主键,主键各列都以直接引用出现在结果里;结果各行的主键两两不同(与解析无关的兜底:同一张表经集合
# 运算拼出来的结果可能重复主键)
meta = frozen.get(picked.table)
pk_names = [str(p) for p in (meta.get("primary_key") or [])] if isinstance(meta, dict) else []
if not pk_names:
return table_only(refuse("no_pk"))
where = {name_key(c): j for j, c in enumerate(picked.columns)}
missing = [p for p in pk_names if name_key(p) not in where]
if missing:
return table_only(refuse("pk_missing", 列=missing))
rows = query.get("rows")
if not isinstance(rows, list) or not all(isinstance(r, list) and len(r) == len(columns) for r in rows):
return table_only(refuse("unparsed"))
pk_at = [where[name_key(p)] for p in pk_names]
if _repeats([tuple(r[j] for j in pk_at) for r in rows]):
return table_only(refuse("multi_table", "duplicate_pk"))
# R10:被引用的列、主键列、推断要用到的键列(维度、派生、常量:行标签原文、分段标题的定位文字都是它们的值)
# 都不在遮罩里
needed = [column, target, *pk_names, *_key_columns(chain, picked.table)]
if any(str(n).lower() in hidden for n in needed):
return table_only(refuse("masked"))
# R11:被引用的值、主键各值都不是空值
if isinstance(row, bool) or not isinstance(row, int) or not 0 <= row < len(rows):
return table_only(refuse("unparsed"))
value = rows[row][at]
if value is None:
return table_only(refuse("null_value"))
pk = {p: rows[row][j] for p, j in zip(pk_names, pk_at)}
if any(v is None for v in pk.values()):
return table_only(refuse("null_pk"))
# R12:数据文件记录在、没回收、文件在(不标红:被运行引用的版本受保护,出现多半是数据源被删或手工清理)
if record is None or record.retired_at is not None or not record.db_path or not Path(record.db_path).is_file():
return table_only(refuse("snapshot_gone"))
# R13:登记哈希等于清单里期望的库哈希、source_id 一致(防「数据文件和登记哈希被一起改成一致的假值」:
# 清单是内容寻址的,又挂在封存的表结构快照下面,改不了)。登记哈希为空也算不等。
# Reason.detail 不带细分:契约只给 DETAILS 和 ChainProblem / LocateProblem 的细分留了位置,R13 规格只写
# chain_mismatch,和异常映射(SnapshotManifestMismatch)同一个口径;哪一项对不上写进日志
registered = str(record.db_sha256 or "").strip().lower()
if not registered or registered != chain.expected_db_sha256.strip().lower():
logger.warning("provenance: 快照 %s 的登记哈希与导入清单里的库哈希不一致", chain.snapshot_id)
return table_only(refuse("chain_mismatch"), alert("chain_mismatch"))
if str(record.source_id) != chain.source_id:
logger.warning("provenance: 快照 %s 的 source_id 与导入清单不一致", chain.snapshot_id)
return table_only(refuse("chain_mismatch"), alert("chain_mismatch"))
try:
return await _open_and_locate(base, table_only, chain=chain, record=record, version=version,
states=states, hidden=hidden, sql=sql, table=picked.table, target=target,
value=value, pk=pk)
except _SNAPSHOT_ERRORS as exc:
# R14 之后(绑定、取引擎、编译核对、回查、行级核对)任何一步:不给格子,已算出的行级状态一律丢掉
reason, red = _snapshot_failure(exc)
return table_only(reason, red)
async def _open_and_locate(base: dict[str, Any], table_only: Callable[..., ProvenanceOut], *, chain: Chain,
record: Any, version: VersionView, states: dict[str, dict[str, Any]], hidden: set[str],
sql: str, table: str, target: str, value: Any, pk: dict[str, Any]) -> ProvenanceOut:
"""R14–R17 和行级核对:要打开数据文件的几步。三类数据文件异常原样往外抛,由 _drill 统一映射。"""
# R14:绑定快照(累积快照首次绑定时和快照清单交叉核对),经 EngineCache 打开数据文件(只读、immutable,本进程
# 首次整份核对哈希)。之后的回查、行级核对都经同一个缓存键,读的就是查询读过的那个文件
view = provenance_db.view_of(record, source_name=chain.source)
await engines.get(view)
# R15:第二道核对,与解析无关。读到的表恰好只有识别器认出的那一张;字节码里 ResultRow 恰好 1 个、没有 Yield
# (UNION ALL 有多个 ResultRow,UNION、INTERSECT、EXCEPT 有 Yield)。编译不了(SQLite 自己报错)同样不下钻
try:
facts = await provenance_db.compile_facts(view, sql)
except sqlite3.Error:
logger.warning("provenance: 编译核对失败,只给表级来历", exc_info=True)
return table_only(refuse("unparsed"))
if {name_key(t) for t in facts.tables} != {name_key(table)}:
return table_only(refuse("multi_table", "authorizer"))
if facts.result_rows != 1 or facts.yields != 0:
return table_only(refuse("multi_table", "compound"))
# R16:按主键参数化回查,恰好一行,值和快照里那一格类型、值都相等
try:
found, recheck_sql, params = await provenance_db.recheck(view, table=table, column=target, pk=pk)
except (sqlite3.Error, StatementError) as e:
_reraise_snapshot_error(e)
# 细分同样不进 Reason.detail(契约外的值),回查出错和回查没找到行给同一个原因,区别只在日志
logger.warning("provenance: 回查失败,只给表级来历", exc_info=True)
return table_only(refuse("recheck_missing"))
if not found:
return table_only(refuse("recheck_missing"))
if len(found) > 1:
return table_only(refuse("recheck_multiple"))
rowid, now = found[0]
if not _same_cell(now, value):
# 值对不上先强制整份重算一次哈希(不走 EngineCache 的指纹捷径:原地改写同样字节数、再还原修改时间的改动,
# 指纹认不出,P4-SPEC 2.6)。文件确实变了 → 标红;没变 → 这条 SQL 的含义和判据认定的不同,不标红
if not await provenance_db.rehash(view):
return table_only(refuse("db_tampered"), alert("db_tampered"))
return table_only(refuse("recheck_mismatch"))
# R17:从 (表, 列, rowid) 找回格子(并集 rowid 先换算到某一期)。纯函数,但要解析配方、还原回执,放线程
located = await asyncio.to_thread(prov.locate, chain, table=table, column=target, rowid=rowid)
if isinstance(located, LocateProblem):
if located.code == "chain_mismatch":
return table_only(refuse("chain_mismatch", located.detail), alert("chain_mismatch"))
return table_only(refuse("no_lineage", located.detail))
# 附带的格写出的是哪几列的值(R10 已按回执里的键列判过,这里按实际给出的再核一次,防回执和推断不一致)
if any(str(f.column).lower() in hidden for f in located.cell.from_ if f.column):
return table_only(refuse("masked"))
plans = await asyncio.to_thread(prov.related_checks, chain, located, table=table, column=target)
checks: list[RelatedCheck] = []
for plan in plans:
if plan.rule is None:
checks.append(plan.check)
continue
# 行级状态按**快照库**的 rowid 跑(并集时是并集 rowid),条件和导入时的核对是同一段代码生成的
try:
status = await provenance_db.row_status(view, plan.rule, rowid)
except (sqlite3.Error, StatementError) as e:
_reraise_snapshot_error(e)
logger.warning("provenance: 行级核对失败,这一行不给结论", exc_info=True)
status = None
checks.append(replace(plan.check, row_status=status))
state = states.get(located.part.import_id) or {}
cell = replace(located.cell, rowid=rowid, pk=pk, raw_purged=state.get("raw_state") == "purged",
recheck=Recheck(sql=recheck_sql, params=list(params), ok=True))
version = replace(version, parts=[replace(p, has_row=p.import_id == located.part.import_id)
for p in version.parts])
return ProvenanceOut(status="inferred", version=version, cell_source=cell, checks=checks, **base)
def _reraise_snapshot_error(exc: BaseException) -> None:
"""SQLAlchemy 把连接池事件里抛出的异常包成 StatementError 时,数据文件的三类异常要原样交给 _drill 映射,
不能被当成「这条 SQL 执行失败」吞掉(借连接时指纹不符抛的 SnapshotTampered 就在这一路上)。"""
orig = getattr(exc, "orig", None)
if isinstance(orig, _SNAPSHOT_ERRORS):
raise orig from exc
def _frozen_tables(schema: dict[str, Any]) -> dict[str, Any]:
tables = schema.get("tables") if isinstance(schema, dict) else None
return tables if isinstance(tables, dict) else {}
def _frozen_columns(meta: Any) -> list[str]:
"""冻结表结构里一张表的列名,按定义顺序(`SELECT *` 要按这个顺序比对结果列)。"""
cols = meta.get("columns") if isinstance(meta, dict) else None
return [str(c["name"]) for c in cols or [] if isinstance(c, dict) and c.get("name") is not None]
def _recorded_masks(query: dict[str, Any]) -> set[str]:
recorded = query.get("mask_columns")
return {c.lower() for c in recorded if isinstance(c, str)} if isinstance(recorded, list) else set()
#: 推断要用到的键列(回执 tables[].columns[].role):维度列的行标签原文、派生列由行标签解析、常量列取分段标题的
#: 定位文字,展示出来就是这几列的值(R10)
_KEY_COLUMN_ROLES = frozenset({"dim", "derive", "const"})
def _key_columns(chain: Chain, table: str) -> list[str]:
"""各期回执里这张表的键列(维度、派生、常量)。取全部各期的并集:R10 在知道格子落在哪一期之前判,宁可多遮。"""
out: list[str] = []
for part in chain.parts:
receipt = part.manifest.get("receipt") if isinstance(part.manifest.get("receipt"), dict) else {}
for t in receipt.get("tables") or []:
if not isinstance(t, dict) or t.get("name") != table:
continue
out += [str(c["name"]) for c in t.get("columns") or []
if isinstance(c, dict) and c.get("role") in _KEY_COLUMN_ROLES and c.get("name")]
return list(dict.fromkeys(out))
def _repeats(keys: list[tuple[Any, ...]]) -> bool:
"""主键元组有没有重复。按 Python 的相等比(5 和 5.0 算同一个):宁可多判重复、少下钻。"""
seen: set[Any] = set()
for key in keys:
try:
marker: Any = key
hash(marker)
except TypeError:
marker = repr(key)
if marker in seen:
return True
seen.add(marker)
return False
def _same_cell(now: Any, then: Any) -> bool:
"""回查的值和快照里那一格:类型和值都相等(int、float、str 分开比,8754 和 8754.0 不算相等;bool 不算数)。
两边都经 engine._jsonable 同一口径规范化过(快照还经过一次 JSON 存取,int、float、str 原样回来),类型不同
只可能是文件或 SQL 的含义变了,不能按数值宽松地放过。"""
if isinstance(now, bool) or isinstance(then, bool):
return False
return type(now) is type(then) and now == then
def _version_of(chain: Chain, tables: list[str], states: dict[str, dict[str, Any]], *,
manifest_view: bool) -> VersionView:
"""数据版本:清单里的部分由 provenance.version_view 给(内容寻址),原件状态、清除、作废、配方第几版取自导入
记录(当前状态,界面标「当前状态」)。manifest_view:数据源设有任何遮罩时不在面板里出「查看导入清单」。"""
view = prov.version_view(chain, tables=tables)
parts = []
for p in view.parts:
state = states.get(p.import_id) or {}
raw = state.get("raw_state")
parts.append(replace(p, raw_state=raw if raw in get_args(RawState) else None, purged=state.get("purged"),
revoked=state.get("revoked"), recipe_seq=state.get("recipe_seq")))
return replace(view, parts=parts, manifest_view=manifest_view)
# --------------------------------------------------------------------------
# 表名、字段名、原话
# --------------------------------------------------------------------------
@@ -1340,17 +1769,56 @@ def _stale(verdict: Any) -> bool:
return isinstance(verdict, dict) and bool(verdict.get("post_seal")) and outdated(verdict)
def _sealed_loader(sealed: _Sealed) -> Callable[[str], Any]:
def _sealed_loader(sealed: _Sealed, *, chain: bool = False) -> Callable[[str], Any]:
"""给裁判取证据的 loader:只认封存范围内的事件交回过的工件(查询、表结构、检索快照和台账里记过的),
取回时照旧复验哈希。别的一律当取不到——人为插进工件表的行、封存之后才追加的事件引用的快照,
都不能送进裁判的摘录。报告文档里的目录也不例外:目录写着哪件工件,不等于它在封存链上。"""
都不能送进裁判的摘录。报告文档里的目录也不例外:目录写着哪件工件,不等于它在封存链上。
chain(期 4,P4-SPEC 2.8.3):只有带文档标记的文档传真。这时另外认封存范围内的表结构快照经哈希链列出的
导入清单、快照清单(以及快照清单各期的导入清单):这些 id 写在内容寻址、又在封存范围内的表结构快照里,属于
封存链的延伸,不是数据库列。扩展集合**惰性**计算:第一次被要一件不在基本范围里的工件时才算,同一个 loader
只算一次(一次请求一个 loader)。chain 为假时和期 4 之前完全相同,期 4 之前的文档不会因为任何一份清单出错。
"""
allowed = {*sealed.queries, *sealed.schemas, *sealed.retrievals, *(str(a) for _, a in sealed.ledger)}
extended: set[str] | None = None
def loader(artifact: str) -> Any:
return load(artifact) if sealed.trusted and str(artifact) in allowed else None
nonlocal extended
key = str(artifact)
if not sealed.trusted:
return None
if key in allowed:
return load(key)
if not chain:
return None
if extended is None:
extended = _chain_set(sealed)
return load(key) if key in extended else None
return loader
def _chain_set(sealed: _Sealed) -> set[str]:
"""封存范围内每份表结构快照经哈希链列出的清单 id 的并集。表结构快照取不回、哈希不符的跳过(不在链上)。"""
out: set[str] = set()
for schema_id in sealed.schemas:
try:
schema = load(schema_id)
except ValueError:
continue
if isinstance(schema, dict):
out |= _chain_artifacts(schema, load)
return out
def _chain_artifacts(schema: dict[str, Any], loader: Callable[[str], Any]) -> set[str]:
"""一份表结构快照经哈希链列出的清单 id:provenance.chain_artifacts(永不抛异常,坏清单视为不在链上)。
按名字现取(不用模块顶部绑定的 prov):测试可以在 sys.modules 里换掉这个模块,核对这里确实委托给它。"""
from app.data.provenance import chain_artifacts
return set(chain_artifacts(schema, loader))
def _node_judge(sealed: _Sealed, node_id: str) -> dict[str, Any]:
"""这次运行的图上报告节点写的 judge(裁判模型、句数和时长上限);图读不出来就当没写。"""
from app.engine.nodes.report import report_judge
@@ -1450,7 +1918,8 @@ async def _judge_now(sealed: _Sealed, report: _Report, units: list[str]) -> dict
event = None
lost = False
if pending:
loader = _sealed_loader(sealed)
# 带期 4 标记的文档才让裁判取得到哈希链上的清单(摘录的来历行要用);老文档的 loader 和以前完全相同
loader = _sealed_loader(sealed, chain=doc.get("provenance") == DOC_PROVENANCE)
masked = await judging.source_masks(catalog, loader=loader)
request = judging.prepare(doc, catalog, units=pending, loader=loader, masked=masked)
skipped = dict(request.skipped)
+22 -2
View File
@@ -249,7 +249,7 @@ def sqlite_dqs_self_check() -> None:
raise RuntimeError(f"SQLite 自检失败:无法确认写错的双引号列名会报错,服务不启动。{problem}")
def open_checked_sqlite(path: str, *, readonly: bool = True) -> sqlite3.Connection:
def open_checked_sqlite(path: str, *, readonly: bool = True, immutable: bool = False) -> sqlite3.Connection:
"""同步打开一个本地 SQLite 文件,DQS 已关(配方导入的建库、核对用)。
配方执行器建临时库、核对模块跑 SQL 都不经 SQLAlchemy,直接用 sqlite3。它们同样要关 DQS:
@@ -261,11 +261,20 @@ def open_checked_sqlite(path: str, *, readonly: bool = True) -> sqlite3.Connecti
readonly=False 以 mode=rwc 打开(执行器建库)。不设日志模式:沿用 SQLite 默认的回滚日志,
不开 WAL——WAL 会改写文件头,库文件哈希就对不上了(P2-SPEC 4.2 第 9 步)。
路径按 URI 转义:文件名里的空格、中文、「?」「#」都不会被当成 URI 的参数或片段。
immutable=True 另加 `immutable=1`,和 EngineCache 打开版本快照的方式一致(_sqlite_url):证据下钻在快照文件上
单开一个连接编译原 SQL(provenance_db.compile_facts,P4-SPEC 2.9),读的必须是查询走的同一种打开方式。
只许和 readonly 一起用:immutable 告诉 SQLite 文件不会变、不加锁,可写的连接这样开会写坏库。缺省 False,
现有调用方的行为不变。
"""
from urllib.parse import quote
if immutable and not readonly:
raise ValueError("immutable 只能用于只读连接")
mode = "ro" if readonly else "rwc"
return sqlite3.connect(f"file:{quote(os.fspath(path))}?mode={mode}", uri=True, factory=_DqsOffConnection)
extra = "&immutable=1" if immutable else ""
return sqlite3.connect(f"file:{quote(os.fspath(path))}?mode={mode}{extra}", uri=True,
factory=_DqsOffConnection)
def _sqlite_extra() -> dict[str, Any]:
@@ -569,6 +578,17 @@ class EngineCache:
# 正被借出的连接不受影响:dispose 只关池子里空闲的,借出的还回来时随旧池子回收
await old.dispose()
def pinned(self, source: Any) -> Fingerprint | None:
"""这个源(版本快照)的引擎核对过的文件指纹;不是快照引擎、或者引擎不在缓存里时为 None。
给证据下钻的编译核对用(provenance_db.compile_facts,P4-SPEC 2.9):它在同一个文件上单开一个连接,
开连接前、关连接后各拿当时的 stat 和这份指纹比,比得上才说明单开的连接读的是 get() 核对过的那个文件。
调用方要在 `await get(source)` 返回之后**同步**调它,中间不能有 await:否则这个键可能已经被 LRU 挤掉,
或者被别的请求摘掉重核。只读不改:已经标成不可信(stale)的也照样返回那份指纹,调用方比对时自然对不上。
"""
pin = self._pins.get(cache_key(source))
return pin.fingerprint if pin is not None else None
async def _verify_once(self, key: str, source: Any, expected: str) -> Fingerprint:
"""核对快照哈希;同一个键已经有一次在核对,就等那一次的结果。
File diff suppressed because it is too large Load Diff
+257
View File
@@ -0,0 +1,257 @@
"""证据下钻(期 4「推断的来源」)里要碰数据库和数据文件的几步(P4-SPEC 2.3 R12–R16、2.7、2.9)。
识别器(engine/direct_select.py)和清单、格子(data/provenance.py)都是纯函数,不连库;这里收拢剩下的:
取数据文件记录、绑定快照、按主键回查、编译核对、强制重算哈希、行级核对、导入记录上的当前状态。
**为什么回查、行级核对都经 EngineCache**:查询本身就是经它打开快照的(immutable 只读、本进程首次整份核对哈希、
之后每次取引擎和借连接都比文件指纹)。走同一条路,下钻核对过的就是查询读过的那个文件,保证范围也和查询相同
(P4-SPEC 2.6 照实写了这个范围的边界)。编译核对另开一个连接,开前关后各比一次 EngineCache 记下的指纹。
**异常**:SnapshotManifestMismatch、SnapshotTampered、SnapshotMissing 一律原样抛出,由接口一侧按 2.3 的
except 顺序映射成 chain_mismatch / db_tampered / snapshot_gone(api/evidence._snapshot_failure)。这里不吞、
不改判:同一类异常在接口的每一步都要得到同一个结论。
同步的文件操作(编译核对、整份重算哈希)一律放 asyncio.to_thread,不卡事件循环(H9)。
"""
from __future__ import annotations
import asyncio
import sqlite3
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.data import engine as engine_mod
from app.data.engine import SnapshotTampered, engines, open_checked_sqlite
from app.data.names import name_key
from app.data.provenance_types import CompileFacts, StateNote, SumEqRule
from app.data.recipe_checks import sum_eq_conditions
from app.data.table_versions import SnapshotMissing, SourceView, bind_snapshot
from app.db.models import SourceSnapshot, TableImport, TableRecipe
#: 数据文件记录在、却已回收或文件不在(绑定之前先认出来,免得取引擎时被当成「文件被改」标红)
_GONE = "数据版本 {sid} 的数据文件已回收或不存在,无法按主键回查"
#: EngineCache 里没有这个快照的指纹(不是快照引擎):不该出现,按文件不在处理(P4-SPEC 2.9 末尾)
_NOT_PINNED = "数据版本 {sid} 没有核对过的数据文件记录,无法做编译核对"
#: 编译核对前后,文件和 EngineCache 核对过的那份对不上:和借连接时指纹不符同一个说法
_REPLACED = engine_mod._REPLACED
def _q(name: str) -> str:
"""SQL 标识符:一律双引号、内部双引号写两个(和 recipe_checks._q、契约 Recheck 的写法一致)。"""
return '"' + str(name).replace('"', '""') + '"'
def _fingerprint(path: str) -> engine_mod.Fingerprint | None:
"""文件指纹,口径和 EngineCache 记下的那份完全相同(同一个函数)。单独包一层,测试可以只换这里。"""
return engine_mod._fingerprint(path)
# --------------------------------------------------------------------------
# 数据文件记录与绑定(R12、R14)
# --------------------------------------------------------------------------
async def snapshot_record(session: AsyncSession, snapshot_id: str) -> SourceSnapshot | None:
"""查询快照的 data_version 对应的数据文件记录(source_snapshots 一行);没有为 None(R12)。"""
if not snapshot_id:
return None
return await session.get(SourceSnapshot, str(snapshot_id))
def view_of(snapshot: SourceSnapshot, *, source_name: str) -> SourceView:
"""数据文件记录 → 绑定了这个快照的 SourceView,供回查、编译核对、行级核对打开数据文件(R14)。
用 table_versions.bind_snapshot,和查询解析源的那条路相同(登记哈希为空拒绝;有快照清单的首次绑定时交叉
核对)。传进去的不是数据源记录,而是一个替身:
- id = snapshot.source_id:缓存键(cache_key)和查询时相同,首次整份哈希只算一次;
- name = source_name(查询快照记的数据源名,只用在报错文字里);description、enabled、options 等给缺省值。
bind_snapshot 直接读 source.description、bool(source.enabled),少了会抛 AttributeError。
所以数据源被删了、改名了也能回查:回查只需要快照文件和它的登记哈希。options 为空,查询时限、遮罩都不进
这个视图——回查的 SQL 是这里拼的主键查找,遮罩由接口一侧按 R10 先判过。
记录已回收(retired_at)或文件不在时抛 SnapshotMissing(→ snapshot_gone,不标红):不先认出来的话,取引擎时
读不到文件会被 EngineCache 报成 SnapshotTampered,误标成「数据文件被改」。SnapshotManifestMismatch、
SnapshotMissing(登记哈希为空)原样抛出。
"""
path = str(snapshot.db_path or "")
if snapshot.retired_at is not None or not path or not Path(path).is_file():
raise SnapshotMissing(_GONE.format(sid=str(snapshot.id)[:12]))
stand_in = SimpleNamespace(
id=snapshot.source_id, name=source_name, description="", enabled=True, options={},
current_snapshot_id=None, last_check=None,
)
return bind_snapshot(stand_in, snapshot)
# --------------------------------------------------------------------------
# R15:编译核对
# --------------------------------------------------------------------------
async def compile_facts(view: SourceView, sql: str) -> CompileFacts:
"""在查询读过的同一个文件上编译原 SQL(EXPLAIN,不执行),给出与解析无关的三项事实(R15):
授权回调报出读了哪些表、字节码里 ResultRow 和 Yield 各有几个。
步骤:经 EngineCache 取引擎(首次时整份核对哈希)→ **同步**取 engines.pinned(view)(中间没有 await,不会被 LRU
挤掉)→ 在线程里:比指纹 → 单开 immutable 只读连接(和 EngineCache 打开快照的方式相同)→ 装授权回调、EXPLAIN
一次 → 关连接 → 再比指纹。开前关后的指纹都要等于 EngineCache 核对过的那份,单开的连接读的才是核对过的文件;
不等抛 SnapshotTampered。pinned 为 None(不是快照引擎)抛 SnapshotMissing。
授权回调只观察、不拦(一律放行):连接本身只读、immutable,EXPLAIN 不执行语句。结论怎么用由接口判(R15 的
①②③),这里只报事实。
"""
await engines.get(view)
pinned = engines.pinned(view)
if pinned is None:
raise SnapshotMissing(_NOT_PINNED.format(sid=str(view.snapshot_id or "")[:12]))
return await asyncio.to_thread(_compile, str(view.database or ""), pinned, sql)
def _compile(path: str, pinned: engine_mod.Fingerprint, sql: str) -> CompileFacts:
if _fingerprint(path) != pinned:
raise SnapshotTampered(_REPLACED)
reads: set[str] = set()
def observe(action: int, arg1: str | None, _arg2: str | None, _db: str | None, _trigger: str | None) -> int:
# SQLITE_READ 的 arg1 是表名(建表时的写法)。子查询、自连接读到的表都会报出来,改名、CTE 报的和合法写法
# 一样(P4-SPEC 2.4 末尾),所以这只是兜底,主判据仍是识别器
if action == sqlite3.SQLITE_READ and arg1:
reads.add(arg1)
return sqlite3.SQLITE_OK
conn = open_checked_sqlite(path, readonly=True, immutable=True)
try:
conn.set_authorizer(observe)
opcodes = [str(row[1]) for row in conn.execute("EXPLAIN " + sql)]
finally:
conn.close()
if _fingerprint(path) != pinned:
raise SnapshotTampered(_REPLACED)
return CompileFacts(tables=reads, result_rows=opcodes.count("ResultRow"), yields=opcodes.count("Yield"))
# --------------------------------------------------------------------------
# R16:按主键回查、强制重算哈希
# --------------------------------------------------------------------------
async def recheck(view: SourceView, *, table: str, column: str,
pk: dict[str, Any]) -> tuple[list[tuple[int, Any]], str, list[Any]]:
"""按主键参数化回查被引用的那一格(R16):返回 (行, sql, params),行是 [(rowid, 值), …]。
SQL 的写法以契约 Recheck 为准:`SELECT rowid, "列" FROM "表" WHERE "主键1" = ? AND "主键2" = ?`,标识符一律双引号,
`?` 位置占位符,WHERE 和 params 都按 pk 的键序(调用方按冻结表结构 primary_key 的顺序构造 pk)。面板「技术
细节」原样显示这里返回的 sql、params,所以不加 LIMIT:多出的行本身就是 recheck_multiple 的证据。
值经 engine._jsonable 规范化,和查询结果进快照时同一个口径(run_query 逐格调它),调用方直接按类型和值比较。
经 EngineCache 打开(同一个缓存键、同一份哈希核对),三类异常原样抛出。
"""
if not pk:
raise ValueError("回查需要主键")
names = list(pk)
sql = (f"SELECT rowid, {_q(column)} FROM {_q(table)} WHERE "
+ " AND ".join(f"{_q(name)} = ?" for name in names))
params = [pk[name] for name in names]
engine = await engines.get(view)
async with engine.connect() as conn:
result = await conn.exec_driver_sql(sql, tuple(params))
rows = [(int(r[0]), engine_mod._jsonable(r[1])) for r in result.fetchall()]
return rows, sql, params
async def rehash(view: SourceView) -> bool:
"""在线程里整份重算数据文件的 sha256,等于 view.expected_sha256 返回 True(R16 的强制重算)。
不经 EngineCache 的指纹捷径:回查的值和快照对不上时才调。EngineCache 只在首次和指纹变化时重算,原地改写同样
字节数、再还原修改时间的改动在指纹不变期间认不出(P4-SPEC 2.6);这里强制读一遍,才能把「文件确实被改了」
和「这条 SQL 的含义与判据认定的不同」分开。文件读不出来、没有登记哈希时返回 False:核对不了就不能当没变。
"""
expected = str(view.expected_sha256 or "").strip().lower()
if not expected:
return False
try:
actual = await asyncio.to_thread(engine_mod._file_sha256, str(view.database or ""))
except OSError:
return False
return actual.lower() == expected
# --------------------------------------------------------------------------
# 2.7:关系核对的行级状态
# --------------------------------------------------------------------------
async def row_status(view: SourceView, rule: SumEqRule, snapshot_rowid: int) -> str | None:
"""这一行的关系核对「total = parts 之和」成不成立:passed / mismatch / unverifiable;判不了为 None(2.7)。
条件取自 recipe_checks.sum_eq_conditions,和导入时的核对(_check_sum_eq)是同一段代码生成的:面板说「这一天
成立」,和导入时的结论是同一个判断。先判含空值(unverifiable),再判不等(mismatch),其余 passed,与导入时
「含空值的行不参与比较、计入无法核对」一致。
snapshot_rowid 是**快照库**的 rowid(cell_source.rowid,并集时是并集 rowid),不是该期构建库的 part_rowid:
SQL 跑在快照库上。各期的行是逐字节拷进并集库的,同一个条件照样成立(P3-SPEC 2.7)。
规则里的列在快照库这张表里不存在(不该出现:关系取自格子所在那一期的配方,目标表结构只会多出退役列),或者
没有这一行时返回 None,不报错。三类数据文件异常原样抛出。
"""
_nonnull, anynull, mismatch = sum_eq_conditions(rule.total, list(rule.parts), dict(rule.types))
sql = (f"SELECT CASE WHEN {anynull} THEN 'unverifiable' WHEN {mismatch} THEN 'mismatch' ELSE 'passed' END "
f"FROM {_q(rule.table)} WHERE rowid = ?")
engine = await engines.get(view)
async with engine.connect() as conn:
present = {name_key(str(r[0])) for r in
(await conn.exec_driver_sql("SELECT name FROM pragma_table_xinfo(?)", (rule.table,))).fetchall()}
if not present or any(name_key(c) not in present for c in [rule.total, *rule.parts]):
return None
row = (await conn.exec_driver_sql(sql, (int(snapshot_rowid),))).first()
return str(row[0]) if row is not None else None
# --------------------------------------------------------------------------
# 导入记录上的当前状态(数据版本一节标「当前状态」的那几项)
# --------------------------------------------------------------------------
async def import_states(session: AsyncSession, import_ids: list[str]) -> dict[str, dict[str, Any]]:
"""各期导入记录上的当前状态:{import_id: {raw_state, purged, revoked, recipe_seq}}。
- raw_state:kept / purged / absent(table_imports.raw_state);
- purged、revoked:StateNote{at, signed_by, reason},没有记录为 None。清除原件、作废接受都只记在导入记录上,
不在封存范围内,界面标「当前状态」(P4-SPEC 0.2 补充 2);
- recipe_seq:这一期配方记录的 seq(table_recipes.seq,经 table_imports.recipe_id 找),取不到为 None。
找不到的 import_id 不出现在结果里(调用方按 None 处理)。只取需要的几列:导入记录上还有核对摘要等大字段。
"""
ids = [str(i) for i in dict.fromkeys(import_ids) if i]
if not ids:
return {}
rows = (await session.execute(
select(TableImport.id, TableImport.raw_state, TableImport.purged, TableImport.revoked, TableImport.recipe_id)
.where(TableImport.id.in_(ids)))).all()
recipe_ids = sorted({str(r.recipe_id) for r in rows if r.recipe_id})
seqs: dict[str, int] = {}
if recipe_ids:
seqs = {str(rid): int(seq) for rid, seq in (await session.execute(
select(TableRecipe.id, TableRecipe.seq).where(TableRecipe.id.in_(recipe_ids)))).all()}
return {str(r.id): {
"raw_state": r.raw_state or None,
"purged": _note(r.purged),
"revoked": _note(r.revoked),
"recipe_seq": seqs.get(str(r.recipe_id)) if r.recipe_id else None,
} for r in rows}
def _note(record: Any) -> StateNote | None:
"""table_imports.purged / revoked 的记录 {at, reason, signed_by, …} → StateNote;空的、不是 dict 的为 None。"""
if not isinstance(record, dict) or not record:
return None
def text(key: str) -> str | None:
value = record.get(key)
return str(value) if value not in (None, "") else None
return StateNote(at=text("at"), signed_by=text("signed_by"), reason=text("reason"))
+730
View File
@@ -0,0 +1,730 @@
"""证据下钻(期 4「推断的来源」)的契约:常量、取值、服务端原文、响应形状和各工作包之间传递的中间结构。
由编排者在第 0 步(WP-0)提交,各工作包只通过这里的类型和 P4-SPEC.md 第 7 节写明的函数签名交互,**不改签名**;
需要改的写进交付说明,由编排者改。这里只有声明,业务逻辑只有三个小函数(refuse、alert、contract_problems):
接口、测试、前端夹具三处要用同一份原文和同一套不变式。
**为什么只用标准库**:识别器(engine/direct_select.py)只许导入守卫、names 和这个模块(P4-SPEC 7.2),这里一旦
导入 recipe_types、pydantic,判据就被拖进整套配方模块。和 recipe_types 重名的取值(核对状态)在契约测试里钉住一致。
**响应形状**(P4-SPEC 2.8.1):`dataclasses.asdict(ProvenanceOut(...))` 就是接口返回的 JSON,键名、顺序和规格逐字
一致。响应用的 dataclass 一律 kw_only:字段多、并行的几个包各自构造,按位置传参错一位不会报错,按名字传错了立刻报。
中间结构(Chain、DirectSelect 等)照规格的写法可以按位置构造。
**唯一的改名**:CellSource 的「from」是 Python 关键字,不能当属性名。Python 里写 `from_`(构造、读写都用它),
`asdict` / `fields` / `dataclasses.replace` / `typing.get_type_hints`、pydantic 和 JSON 里一律是「from」(见 _json_keys)。
构造时也认 `**{"from": …}`,从 JSON 读回来可以直接 `CellSource(**d)`。
**路由怎么返回**(WP-C):照现有证据路由的写法,`return dataclasses.asdict(out)`,返回注解写 `dict[str, Any]`。
把 ProvenanceOut 写成返回注解或 `response_model` 也得到同样的 JSON(pydantic 按 `__annotations__` 收集 dataclass
字段,_json_keys 把注解里的名字也改成了「from」,契约测试钉住 TypeAdapter 和 FastAPI 带类型路由的输出等于 asdict),
但不推荐:那样每次响应都要按类型再校验、再序列化一遍,与其余证据路由不一致。
**文档标记**(P4-SPEC 1.4):compose_doc 组装的新文档顶层写 `"provenance": DOC_PROVENANCE`。识别器的文法、
`tables_in` 的读法都绑在这个版本号上;以后放宽文法要升版本号,已有 `provenance: 1` 的文档仍按 1 的文法判。
"""
from __future__ import annotations
import logging
import re
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any, Literal, get_args
_log = logging.getLogger(__name__)
# ==========================================================================
# 版本与常量
# ==========================================================================
#: 文档标记的版本号:带这个标记的文档才走期 4 的行为(查询步骤的提示字段、推断来源接口、裁判摘录的新行)
DOC_PROVENANCE = 1
#: 推断来源接口响应的 schema
SCHEMA = "agentlab.provenance/1"
#: 裁判摘录(P4-SPEC 3.2):来源的分期行最多几期(取最近的几期,其余写「另有 n 期」)
PROV_PARTS = 3
#: 已接受的理由最多几条
PROV_ACCEPT = 4
#: 区域外文字最多几格(每条查询)
PROV_OUTSIDE = 4
#: 区域外文字每格最多几个字,超出写「…(已截断)」
PROV_OUTSIDE_CHARS = 200
#: 列的说明最多几行
PROV_COLUMN_NOTES = 6
#: 整块来历(来源到区域外文字,含口径)最多几个字,超出按 3.2 的五步顺序删
PROV_CHARS = 1200
# ==========================================================================
# 取值
# ==========================================================================
#: inferred:给出格子;table_only:只给表级来历(version 非空),附原因;none:什么都不画,或只画一句原因 / 红色提示
Status = Literal["inferred", "table_only", "none"]
#: 不下钻的原因(P4-SPEC 2.3 的规则表、2.4 的识别器分组)。同一个 code 可能出现在不同 status 下:
#: chain_mismatch 在 R6(清单链本身对不上)是 none,在 R13、R17 和异常映射里是 table_only
ReasonCode = Literal[
# R0:文档没有标记(期 4 之前组装的)
"legacy_doc",
# R1:不是直接引用的查询单元格(指标、实体、代码节点和 Agent 字段的取数)
"not_cell",
# R2 / R4:查询快照或表结构快照不在本次运行的事件里、取回复验不通过
"not_sealed",
# R3:查询快照没有 data_version(手工源)
"not_upload",
# R4:表结构快照不是按配方导入的(简单导入、v0 迁移出的版本)
"simple_upload",
# R5(红):清单取不回来或哈希不符
"manifest_unreadable",
# R6 / R13 / R17 / 异常映射第 1 条(红)
"chain_mismatch",
# R7 / R8 / R9 / R15:SQL 判据(2.4 的分组,细分写在 detail,取值见 DETAILS)
"expression", "alias", "multi_table", "unparsed",
# R9
"no_pk", "pk_missing",
# R10
"masked",
# R11
"null_value", "null_pk",
# R12 / 异常映射第 3 条
"snapshot_gone",
# 异常映射第 2 条、R16 强制重算后哈希不等(红)
"db_tampered",
# R16
"recheck_missing", "recheck_multiple", "recheck_mismatch",
# R17
"no_lineage",
]
#: 标红的提示。出现时 reason.code 与它相同,界面只画提示、不再画 reason.text(P4-SPEC 4.1、4.2 末尾)
AlertCode = Literal["db_tampered", "chain_mismatch", "manifest_unreadable"]
#: SQL 判据的拒绝分组(DirectSelect / SelectRefusal,P4-SPEC 2.4)
RefusalCode = Literal["expression", "alias", "multi_table", "unparsed"]
#: 每个分组允许的细分(Reason.detail / SelectRefusal.detail)。duplicate_pk(R9)、authorizer、compound(R15)
#: 由接口一侧补上,同样归 multi_table。界面只显示分组的原文,detail 给测试和日志用。
#:
#: 识别器(WP-A)的每个拒绝分支都必须落在这里,测试对每个拒绝用例断言 `detail in DETAILS[code]`。原型
#: p4-probe/probe_recognizer.py 返回的 expression_or_alias、subquery_or_compound、table_shape 之类是原型自己的名字,
#: 移植时要逐个映射,不能照抄。2.4 没逐条写的几种按文法归:
#: INDEXED BY(表名后面的词被当成省略 AS 的表别名,再往后不是 WHERE / ORDER / LIMIT)→ unparsed · tail_shape;
#: 表函数(表名紧跟「(」)→ unparsed · table_shape;FROM 后面直接是「(」(子查询)→ multi_table · subquery;
#: 带库名的表(main."日客流")→ unparsed · qualified_table;聚合、函数调用、运算 → expression · expression。
#: 漏了映射时 refuse 不抛异常(照样给分组的原文,细分原样留下、记一条警告),contract_problems 会报出来
DETAILS: dict[str, tuple[str, ...]] = {
"expression": ("expression", "string_literal", "literal_keyword", "window", "collate", "distinct"),
"alias": ("alias",),
"multi_table": ("join", "comma_join", "subquery", "cte", "compound", "authorizer", "duplicate_pk"),
"unparsed": ("not_select", "multi_statement", "table_shape", "qualified_table", "qualifier", "star_mismatch",
"unknown_table", "unknown_column", "ambiguous_column", "tail_shape"),
}
#: 单期是配方的 mode,并集是快照清单的 mode
Mode = Literal["replace", "accumulate"]
#: 表的种类:冻结表结构里 kind 缺省(None)就是 data,reported_total 是原表写明的合计
TableKind = Literal["data", "reported_total"]
#: 回执里列的角色(recipe_types.ColumnOut.role)
ColumnRole = Literal["axis", "dim", "derive", "const", "measure", "value", "text"]
#: 附带的格(FromCell.role):日期表头格、行标签、分段标题、列表头、合计标签(含回执里的 derived_label)
FromRole = Literal["axis_header", "row_label", "section_title", "col_header", "total_label"]
#: 原件状态(table_imports.raw_state,当前状态)
RawState = Literal["kept", "purged", "absent"]
#: 接受的种类:overrides(数据质量类不成立)→ override;waivers(无法核对)→ waiver
AcceptanceKind = Literal["override", "waiver"]
#: 本期结论:照搬清单里 CheckResult.status(与 recipe_types.CheckStatus 相同,契约测试钉住)
PartStatus = Literal["passed", "mismatch", "unverifiable", "info"]
#: 这一行的关系核对结果(只给关系核对)
RowStatus = Literal["passed", "mismatch", "unverifiable"]
#: 这一格的合计核对结果(只给 K、G、T)
CellStatus = Literal["ok", "unverifiable", "unknown", "not_formula"]
#: 年份取自哪里:统计期从格子解析(period)、人工录入(human)
YearFrom = Literal["period", "human"]
#: loader 的语义同 artifact_store.load:取不到返回 None,哈希不符抛 ValueError
Loader = Callable[[str], Any]
# ==========================================================================
# 服务端原文(P4-SPEC 4.2):界面原样显示 reason.text、alert.text,不另写
# ==========================================================================
_RECHECK_TEXT = "按主键回查数据文件,结果与查询结果不一致,不推断来源"
#: code → 原文。pk_missing 里的「{列}」由 refuse 用 fmt 填(没带齐的主键列,多列用「、」连)
REASON_TEXT: dict[str, str] = {
"legacy_doc": "这份报告生成于系统升级前,不推断原表格子",
# not_cell、not_upload 界面不画(P4-SPEC 4.1 渲染矩阵),原文只给日志和测试,规格的原文表里没有,由 WP-0 补
"not_cell": "这一处不是直接引用的查询单元格,不推断原表格子",
"not_sealed": "查询快照或表结构快照不在本次运行的记录里,不推断来源",
"not_upload": "这次查询的数据源不是上传的表格,不推断原表格子",
"simple_upload": "这份表格按表头行直接导入,没有记录原表格子的溯源",
"manifest_unreadable": "导入清单无法读取或与哈希不一致,无法给出来历",
# 规格原文是「…可能被修改过:只显示导入清单记录的来历」。chain_mismatch 也用于 R6(清单链本身对不上,
# version 为 null,什么来历都不显示),那句后半句在 R6 时不属实,所以去掉;table_only 时数据版本一节照样画出来,
# 不需要再说(交付说明里写明)
"chain_mismatch": "数据文件的登记与导入清单不一致,可能被修改过",
"db_tampered": "数据文件与登记的版本不一致,可能被修改过:只显示导入清单记录的来历",
"snapshot_gone": "这一版的数据文件已回收或不存在,无法按主键回查,只显示导入清单记录的来历",
"expression": "这一格是计算、聚合或改写的结果,只给出表级来历",
"alias": "查询给列改了名,无法确认这一格对应哪一列,只给出表级来历",
"multi_table": "查询涉及多张表、同一张表多次,或含子查询、公用表表达式、集合运算,只给出表级来历",
"unparsed": "查询的写法超出可追溯的范围,只给出表级来历",
"no_pk": "这张表没有主键,无法按主键回查,只给出表级来历",
"pk_missing": "查询结果没有带齐主键列({列}),无法按主键回查,只给出表级来历",
"masked": "这一格涉及设置了遮罩的列,不推断来源",
"null_value": "这一格是空值,不推断来源",
"null_pk": "这一行的主键是空值,无法按主键回查",
"recheck_missing": _RECHECK_TEXT,
"recheck_multiple": _RECHECK_TEXT,
"recheck_mismatch": _RECHECK_TEXT,
"no_lineage": "导入清单里没有这一格的溯源记录,只给出表级来历",
}
#: 标红提示的原文:与同名 reason 的原文相同(两处同时出现时界面只画提示)
ALERT_TEXT: dict[str, str] = {code: REASON_TEXT[code] for code in get_args(AlertCode)}
_PLACEHOLDER = re.compile(r"\{([^{}]+)\}")
#: 回查 SQL 里双引号括起的标识符(内部的双引号写两个):数 `?` 占位符之前先去掉,列名里的问号不算
_QUOTED = re.compile(r'"(?:[^"]|"")*"')
# ==========================================================================
# 响应(GET /api/runs/{run_id}/evidence/segments/{segment_id}/provenance,P4-SPEC 2.8.1)
# ==========================================================================
def _stored_as(py: str, js: str) -> property:
"""Python 名字 `py` 的属性:读写实例字典里的 `js` 键(_json_keys 用)。没有这个键时抛 AttributeError,hasattr 照常。"""
def get(self: Any) -> Any:
try:
return self.__dict__[js]
except KeyError:
raise AttributeError(py) from None
def set_(self: Any, value: Any) -> None:
self.__dict__[js] = value
return property(get, set_)
def _json_keys(**aliases: str) -> Callable[[type], type]:
"""dataclass 的字段在 Python 里叫 `py`,在 asdict / fields / replace / get_type_hints / pydantic / JSON 里叫 `js`
(只给 CellSource.from 用)。
做法:dataclass 照常按 `py` 生成 __init__、__repr__、__eq__;生成之后
- 字段表 `__dataclass_fields__` 和注解 `__annotations__` 里的名字都改成 `js`(顺序不变)。只改字段表的话,
pydantic 按注解收集字段,对不上的直接跳过:FastAPI 带类型的路由、TypeAdapter 的输出会悄悄少掉「from」;
- 值存在实例字典的 `js` 键下,`py` 是读写它的属性。pydantic 从 JSON 校验出实例时直接按字段名填实例字典,
不经过 __init__,存在 `py` 下的话这样得到的实例读 `from_` 会抛 AttributeError;
- __init__ 也认 `js` 这个关键字(dataclasses.replace、`Cls(**json)` 传的是它)。两个都传时以 `py` 为准:
`replace(obj, from_=新值)` 会同时带上旧的 `from`。
"""
def wrap(cls: type) -> type:
fields_: dict[str, Any] = cls.__dataclass_fields__ # type: ignore[attr-defined]
renamed: dict[str, Any] = {}
for name, f in fields_.items():
if name in aliases:
f.name = aliases[name]
renamed[f.name] = f
cls.__dataclass_fields__ = renamed # type: ignore[attr-defined]
cls.__annotations__ = {aliases.get(k, k): v for k, v in cls.__dict__["__annotations__"].items()}
for py, js in aliases.items():
setattr(cls, py, _stored_as(py, js))
generated = cls.__init__
def __init__(self: Any, *args: Any, **kwargs: Any) -> None:
for py, js in aliases.items():
if js in kwargs:
value = kwargs.pop(js)
kwargs.setdefault(py, value)
generated(self, *args, **kwargs)
__init__.__doc__ = generated.__doc__
__init__.__qualname__ = f"{cls.__qualname__}.__init__"
cls.__init__ = __init__ # type: ignore[method-assign]
return cls
return wrap
@dataclass(kw_only=True)
class ReportRef:
#: 报告节点 id
node_id: str
#: 报告文档的工件 id
doc_artifact: str
@dataclass(kw_only=True)
class CellRef:
"""片段本身是单元格引用(cite.kind == "cell",有 alias 和 locator)就给,与 status 无关(legacy_doc 时也给)。"""
#: 目录里的查询编号(Q3)
alias: str
#: 快照里的行号,从 0 数(locator.row)
row: int
#: 被引用的结果列名(locator.column)
column: str
#: 目录条目的 artifact(查询快照 id);条目没记时 None
artifact: str | None = None
@dataclass(kw_only=True)
class Reason:
"""status 不是 inferred 时必有。用 refuse() 构造,原文取自 REASON_TEXT。"""
code: ReasonCode
#: 细分(DETAILS 里的值、ChainProblem / LocateProblem 的 detail);没有细分为空串
detail: str = ""
text: str
@dataclass(kw_only=True)
class Alert:
"""标红的提示。用 alert() 构造,原文取自 ALERT_TEXT。"""
code: AlertCode
text: str
@dataclass(kw_only=True)
class TableRef:
"""SQL 用到的表:direct_select.tables_in 与冻结表结构的交集,按第一次出现的顺序。"""
name: str
kind: TableKind = "data"
@dataclass(kw_only=True)
class PeriodView:
"""一期的统计期(导入清单的 period)。"""
start: str
end: str
source: Literal["cells", "human"]
#: 解析出统计期的格子,带工作表名(「客流汇总!B2」);人工录入时为空
cells: list[str] = field(default_factory=list)
#: 人工录入时的署名(未认证)
signed_by: str | None = None
@dataclass(kw_only=True)
class AcceptanceView:
"""一条接受理由(导入清单 acceptances 的一项,内容寻址,是当时的事实)。
不叫 Acceptance:recipe_types.Acceptance 已有同名类,字段不同。
"""
check_id: str
#: 同一份清单 checks 里 id 相同的那条的 title;找不到时 None
title: str | None
kind: AcceptanceKind
reason: str
signed_by: str | None = None
#: 清单里这条接受的 at;老清单没有时 None
at: str | None = None
@dataclass(kw_only=True)
class StateNote:
"""导入记录上的当前状态(清除原件、作废接受):table_imports.purged / revoked 里的 at、signed_by、reason。"""
at: str | None = None
signed_by: str | None = None
reason: str | None = None
@dataclass(kw_only=True)
class PartView:
"""数据版本里的一期(按统计期升序)。前半来自导入清单(内容寻址),raw_state 起的三项来自导入记录(当前状态)。
version_view(WP-B)只填清单里的那部分:recipe_seq、raw_state、purged、revoked 留 None,has_row 全 False,
由接口一侧(WP-C)从数据库补、按格子所在的一期标 has_row。
"""
import_id: str
#: 导入记录的 seq(同 ChainPart.seq)
seq: int
#: 导入清单的工件 id(「查看导入清单」按它经工件接口取)
manifest: str
#: 清单没有统计期时 None(界面写「统计期未记录」)
period: PeriodView | None
file_name: str | None
raw_sha256: str | None
#: S9:回执 regions 里块内各角色区域的外接矩形,按工作表(「客流汇总!B4:AG30」),多张表用「、」连;取不到为 None
region: str | None = None
#: S9:回执 rows_excluded 的行数之和;期 3 之前的回执没有这个键时为 None
excluded_rows: int | None = None
recipe_sha256: str | None = None
#: 配方是第几版:取自配方记录(当前状态),取不到为 None
recipe_seq: int | None = None
#: 清单的 created_at
committed_at: str | None = None
#: 清单的 signed_by.name(署名,未认证)
signed_by: str | None = None
#: 这一期的全部接受理由(不只是相关的):先 overrides,再 waivers,各按清单里的顺序
acceptances: list[AcceptanceView] = field(default_factory=list)
raw_state: RawState | None = None
purged: StateNote | None = None
revoked: StateNote | None = None
#: 这一格所在的一期:inferred 时恰好一项为 True,其余 False
has_row: bool = False
@dataclass(kw_only=True)
class VersionView:
"""数据版本(表级来历):status 为 none 时整节为 null。"""
#: 查询快照记的数据源名
source: str
#: 查询快照的 data_version
snapshot_id: str
mode: Mode
#: 累积并集(表结构快照有 snapshot_manifest)
union: bool
tables: list[TableRef] = field(default_factory=list)
#: 能不能在面板里查看导入清单:数据源设有任何遮罩(当前设置 ∪ 快照记录)时 False。
#: version_view(WP-B)不知道遮罩,一律给 True,由接口一侧(WP-C)改
manifest_view: bool = True
parts: list[PartView] = field(default_factory=list)
@dataclass(kw_only=True)
class FromCell:
"""这一行其余键列各取自哪一格(日期表头格、行标签格、分段标题格、列表头、合计标签)。"""
role: FromRole
#: 这一格给出的是哪一列的值;宽表指标列的行标签(指标名取自哪一格)为 None
column: str | None
sheet: str
#: 不带工作表名的 A1 坐标(「G4」)
cell: str
#: 只放格子原文:行标签取 labels[].raw,合计标签取 DerivedItem.label_raw;拿不到为 None,不拿规范写法冒充
text: str | None = None
#: 只给 section_title:配方里的定位文字 locate.title(原表里的字可以不同)
locate_title: str | None = None
@dataclass(kw_only=True)
class YearSource:
"""年份取自哪里:交叉表的轴是文本日期(text / mixed)且配方 axis.year_from 有值时才有,其余为 null。"""
source: YearFrom
#: period:统计期的格子(带工作表名);human:空
cells: list[str] = field(default_factory=list)
#: human:录入统计期时的署名(未认证);period:None
signed_by: str | None = None
#: 这一块的日期表头有的是日期格、有的是文本(AxisOut.form == "mixed"),逐格判断不了
mixed: bool = False
@dataclass(kw_only=True)
class CanonicalView:
"""这一行的标签原文和数据库里的值(规范写法)不同时给出(P4-SPEC 2.5 dim 一行、6.2 A8、A18),否则为 null。
比的是这一行的**标签列**:交叉表逆透视出的 dim 列(原文取自 labels[].raw)、合计表的合计项列(原文取自
DerivedItem.label_raw)。raw 是那一格的原文,canonical 是这一行该列在数据库里的值(即 pk 里那一列的值)。
**被引用的是值列也给**:A8 合计表那一格是 {"raw": "18-22 时合计", "canonical": "18-22时合计"},累积并集里全角
标签那一期的值格是 {"raw": "8-9", "canonical": "8-9"};被引用的就是标签列时同样给(A18)。
宽表指标列的行标签是指标名,不是哪一列的值,不给;拿不到逐行原文的(列表形态没有 SegmentLabels)不给。
"""
raw: str
canonical: str
@dataclass(kw_only=True)
class Recheck:
"""按主键参数化回查数据文件的那一句(推断出格子时 ok 恒为 True,不一致的走 table_only)。
**占位符用 `?`(位置参数),不用 `:p0`**(规格 2.3 R16 写的 `:p0` 以这里和 2.8.1 为准):
`SELECT rowid, "列" FROM "表" WHERE "主键1" = ? AND "主键2" = ?`。标识符一律双引号、内部的双引号写两个;
WHERE 按冻结表结构 primary_key 的顺序逐列 `= ?` 用 AND 连;params 与占位符一一对应,同样按 primary_key 的顺序
(等于 CellSource.pk 的值的顺序)。面板「技术细节」原样显示 sql 和 params,两者必须对得上;provenance_db.recheck
返回的 (行, sql, params) 就是这里的 sql、params。
"""
sql: str
#: 与 sql 里的 `?` 按位置一一对应(primary_key 的顺序)
params: list[Any] = field(default_factory=list)
ok: bool
@_json_keys(from_="from")
@dataclass(kw_only=True)
class CellSource:
"""推断出的格子:只在 inferred 时有,否则 null。
locate(WP-B)填格子、附带的格、年份、规范写法、merged_fill、part_seq、part_rowid;pk、rowid、raw_purged、recheck
由接口一侧(WP-C)回查后填。Python 里「from」写成 from_,JSON 里是「from」(见模块说明)。
"""
table: str
column: str
column_role: ColumnRole
kind: TableKind = "data"
#: 主键列 → 值(查询快照里那一行的值),键按冻结表结构 primary_key 的顺序(Recheck.params 同序)
pk: dict[str, Any] = field(default_factory=dict)
#: 快照库的 rowid(并集时是并集 rowid,不是 part_rowid)
rowid: int | None = None
#: 格子所在的那一期的 seq(导入记录的 seq,同 ChainPart.seq)和该期构建库里的 rowid(单期时 part_rowid == rowid)
part_seq: int
part_rowid: int
sheet: str
#: 不带工作表名的 A1 坐标(「G5」)
cell: str
#: 取自 receipt.tables 里这一列的 header、unit;没有为 None
header: str | None = None
unit: str | None = None
from_: list[FromCell] = field(default_factory=list)
year: YearSource | None = None
canonical: CanonicalView | None = None
#: 这一期的原件已清除(导入记录的 raw_state == "purged",当前状态)
raw_purged: bool = False
#: 列表这一块的配方写了 merged_data == "fill":值可能取自合并区域的左上格
merged_fill: bool = False
recheck: Recheck | None = None
@dataclass(kw_only=True)
class RelatedCheck:
"""相关核对的一条(进响应,不含行级规则:规则只在服务端内部流转,见 RelatedCheckPlan)。"""
#: 核对 id(R1、K1、G1、T1、C1…)
id: str
#: CheckResult.kind(relation_sum_eq、relation_not_comparable、derived_sum、formula_refs、column_sum、context_agree…)
kind: str
title: str
part_status: PartStatus
#: 只给关系核对(relation_sum_eq);列在当前目标表结构里不存在时 None
row_status: RowStatus | None = None
#: 只给 K、G、T
cell_status: CellStatus | None = None
#: 挂在这条核对上的接受理由(同一期清单 acceptances 里 check_id 相同的那条)
acceptance: AcceptanceView | None = None
#: K 无法核对时的原因摘要:CheckResult.details 的第一条,可能带数字(系统呈现,不经模型转述)
detail: str | None = None
@dataclass(kw_only=True)
class ProvenanceOut:
"""推断来源接口的响应。asdict 之后就是 P4-SPEC 2.8.1 的 JSON;不变式见 contract_problems。"""
schema: str = SCHEMA
report: ReportRef
#: 片段 id
segment: str
cell: CellRef | None = None
status: Status
reason: Reason | None = None
alert: Alert | None = None
#: 运行已封存且封存核对通过(同 sealed.trusted)。False 时照常给结论,界面加一句未封存
sealed: bool
version: VersionView | None = None
cell_source: CellSource | None = None
checks: list[RelatedCheck] = field(default_factory=list)
# ==========================================================================
# 跨包的中间结构(P4-SPEC 7.2–7.4)
# ==========================================================================
@dataclass
class DirectSelect:
"""识别器认可的直接选取(WP-A recognize 的结果)。"""
#: 冻结表结构里的表名(原样)
table: str
alias: str | None
#: 第 i 个结果列对应的表列名(原样,已按 name_key 解析);* 已展开
columns: list[str]
@dataclass
class SelectRefusal:
"""识别器拒绝:code 是 2.4 的分组,detail 必须是 DETAILS[code] 里的细分值(映射见 DETAILS 上面的说明)。"""
code: RefusalCode
detail: str
@dataclass
class ChainPart:
#: 导入记录的 seq(清单里的 seq、快照清单 parts[].seq),不是在 parts 里的位置:每期替换的第二期只有一项,seq 是 2。
#: 界面和摘录里的「第 k 次导入」、CellSource.part_seq、PartView.seq 都是它
seq: int
import_id: str
manifest_id: str
#: 已取回的导入清单
manifest: dict[str, Any]
#: 表 → [union 起, 止, part 起, part 止](快照清单 parts[].rows 摊平);单期为 None
union_rows: dict[str, list[int]] | None
@dataclass
class Chain:
"""查询快照 → 表结构快照 → 清单(resolve_chain 的结果,P4-SPEC 2.2)。"""
#: 查询快照记的数据源名
source: str
#: 清单里的 source_id(与 source_snapshots.source_id 交叉核对,R13)
source_id: str
#: 查询快照的 data_version
snapshot_id: str
mode: Mode
union: bool
#: 单期:导入清单的 db_sha256;并集:快照清单的 union.db_sha256
expected_db_sha256: str
#: 按统计期升序
parts: list[ChainPart]
#: 冻结的表结构快照(已取回)
schema: dict[str, Any]
#: 快照清单(并集才有)
snapshot_manifest: dict[str, Any] | None
@dataclass
class ChainProblem:
"""resolve_chain 失败:取不回或哈希不符 → manifest_unreadable(R5);链本身对不上 → chain_mismatch(R6)。"""
code: Literal["manifest_unreadable", "chain_mismatch"]
#: 给测试和日志的细分;规格里写的是 ChainProblem("manifest_unreadable") 这种只给 code 的构造,所以有缺省
detail: str = ""
@dataclass
class Located:
"""locate 的结果:格子所在的一期、该期的 rowid、推断出的格子(rowid、pk、recheck 留给 WP-C 填)。"""
part: ChainPart
part_rowid: int
cell: CellSource
@dataclass
class LocateProblem:
"""locate 失败:找不到溯源段、锚格所在的块、轴或分段标题 → no_lineage;并集 rowid 越界 → chain_mismatch(红)。"""
code: Literal["no_lineage", "chain_mismatch"]
detail: str = ""
@dataclass
class SumEqRule:
"""关系核对的行级规则:只在服务端内部流转,不进响应。WP-C 据此用 sum_eq_conditions 拼 SQL、按快照库的 rowid 跑。"""
table: str
total: str
parts: list[str]
#: 列 → INTEGER / REAL / TEXT(取自格子所在那一期的 receipt.tables)
types: dict[str, str]
@dataclass
class RelatedCheckPlan:
"""related_checks 的一项:check 进响应(row_status 由 WP-C 填),rule 是关系核对的行级规则(其余核对为 None)。"""
check: RelatedCheck
rule: SumEqRule | None
@dataclass
class CompileFacts:
"""R15 的第二道核对(EXPLAIN 编译、不执行):授权回调读到的表、字节码里 ResultRow 和 Yield 的个数。"""
tables: set[str]
result_rows: int
yields: int
# ==========================================================================
# 小函数
# ==========================================================================
def refuse(code: ReasonCode, detail: str = "", **fmt: Any) -> Reason:
"""按 REASON_TEXT 生成 Reason。
pk_missing 的原文里有「{列}」,用 fmt 填:`refuse("pk_missing", 列=["日期", "时段"])` →「…主键列(日期、时段)…」,
列表、元组按「、」连。建议填没带齐的那几列。
code 不认识、原文里有占位符却没给对应的 fmt,抛 ValueError:两者都是调用处写死的常量,那个分支的任何一条测试都会
暴露,不该带着半截原文上界面。
SQL 判据的四个分组带了 DETAILS 之外的细分**不抛**:细分来自识别器对这条 SQL 的判断,取决于数据,没被测试覆盖
的写法(INDEXED BY、表函数之类)漏了映射时,抛异常会让推断来源接口回 500,而不是只给表级来历。这时照样按分组给
原文,细分原样留下,记一条警告;contract_problems 会把它报出来,接口和验收测试断言 contract_problems 为空就能抓到。
"""
if code not in REASON_TEXT:
raise ValueError(f"不认识的原因代码:{code!r}")
if code in DETAILS and detail and detail not in DETAILS[code]:
_log.warning("provenance: %s 不认识细分 %r,应为 %s 之一(照样按分组给原文)", code, detail, DETAILS[code])
template = REASON_TEXT[code]
values = {k: "、".join(str(x) for x in v) if isinstance(v, (list, tuple)) else str(v) for k, v in fmt.items()}
missing = [m for m in _PLACEHOLDER.findall(template) if m not in values]
if missing:
raise ValueError(f"{code} 的原文需要 {missing} 填值")
text = template.format_map(values) if _PLACEHOLDER.search(template) else template
return Reason(code=code, detail=detail, text=text)
def alert(code: AlertCode) -> Alert:
"""按 ALERT_TEXT 生成标红的提示。"""
if code not in ALERT_TEXT:
raise ValueError(f"不认识的提示代码:{code!r}")
return Alert(code=code, text=ALERT_TEXT[code])
def contract_problems(out: ProvenanceOut) -> list[str]:
"""响应的不变式(P4-SPEC 2.3、2.8.1、4.1 渲染矩阵),给接口和验收测试断言用;空列表表示通过。
不在构造时强制:违反不变式是实现的错,应该在测试里暴露,不该在线上把整个请求变成 500。
"""
out_problems: list[str] = []
bad = out_problems.append
if out.schema != SCHEMA:
bad(f"schema 应为 {SCHEMA}")
if out.status not in get_args(Status):
bad(f"status 不认识:{out.status!r}")
if out.reason is not None and out.reason.code not in get_args(ReasonCode):
bad(f"reason.code 不认识:{out.reason.code!r}")
if (out.reason is not None and out.reason.code in DETAILS and out.reason.detail
and out.reason.detail not in DETAILS[out.reason.code]):
# 空细分不报:R8 的防御分支规格没给细分,refuse 的缺省也是空串
bad(f"{out.reason.code} 的细分 {out.reason.detail!r} 不在 DETAILS 里(识别器漏了映射)")
if out.alert is not None:
if out.alert.code not in get_args(AlertCode):
bad(f"alert.code 不认识:{out.alert.code!r}")
if out.reason is None or out.reason.code != out.alert.code:
bad("有 alert 时 reason.code 必须与 alert.code 相同")
if out.reason is not None and out.reason.code in get_args(AlertCode) and out.alert is None:
bad(f"{out.reason.code} 必须同时给 alert(标红)")
if out.status == "inferred":
if out.reason is not None or out.alert is not None:
bad("inferred 时 reason、alert 都应为 null")
if out.version is None or out.cell_source is None:
bad("inferred 时 version、cell_source 都必须有")
else:
cs = out.cell_source
if cs.rowid is None or not cs.pk or cs.recheck is None or not cs.recheck.ok:
bad("inferred 时 cell_source 的 rowid、pk、recheck 必须填好,且 recheck.ok 为真")
elif cs.recheck.params != list(cs.pk.values()) or _QUOTED.sub("", cs.recheck.sql).count("?") != len(cs.pk):
bad("recheck 的 sql 用 ? 占位,个数和 params 都与 pk 一一对应(按 primary_key 的顺序)")
hits = [p for p in out.version.parts if p.has_row]
if len(hits) != 1:
bad(f"inferred 时 version.parts 恰好一项 has_row,现在是 {len(hits)} 项")
elif hits[0].seq != cs.part_seq:
bad("has_row 的那一期与 cell_source.part_seq 不一致")
else:
if out.reason is None:
bad(f"{out.status} 时必须给 reason")
if out.cell_source is not None or out.checks:
bad(f"{out.status} 时 cell_source 为 null、checks 为空")
if out.version is not None and any(p.has_row for p in out.version.parts):
bad("没有推断出格子时 has_row 全为 False")
if out.status == "table_only" and out.version is None:
bad("table_only 时 version 必须有")
if out.status == "none" and out.version is not None:
bad("none 时 version 为 null")
return out_problems
+24 -8
View File
@@ -446,20 +446,36 @@ def _check_t(sql: _Sql, cid: str, title: str, items: list[DerivedItem], full_cal
# ---------------------------------------------------------------- R:关系
def _check_sum_eq(sql: _Sql, rel: SumEq, title: str, columns: list[ColumnOut], grain: list[str],
lineage: dict[str, dict[str, list[list[Any]]]]) -> CheckResult:
table, total, parts = rel.table, rel.total, list(rel.parts)
def sum_eq_conditions(total: str, parts: list[str], types: dict[str, str]) -> tuple[str, str, str]:
"""关系核对「total = parts 之和」逐行判定用的三段 SQL 条件:(nonnull, anynull, mismatch)。
- nonnull:参与的列都不是空值(这一行比得上);
- anynull:任一列是空值(这一行无法核对);
- mismatch:合计与各部分之和不等。只在 nonnull 成立时有意义,调用方自己和 nonnull 组合(AND,或者 CASE 里
先判 anynull)。有一列是 REAL 时按相对误差 1e-9 比(浮点加法有舍入),全是整数时严格比。
为什么抽出来:导入时的核对(_check_sum_eq)和证据面板的行级核对(provenance_db.row_status,P4-SPEC 2.7)
必须用同一段代码生成的条件。面板说「这一天成立」,就得和导入时的结论是同一个判断,不能各写一份、日后改了一边。
types 是列名 → INTEGER / REAL / TEXT(回执里这一期的列类型),缺的列按非 REAL 处理。
"""
names = [total, *parts]
nonnull = " AND ".join(f"{_q(c)} IS NOT NULL" for c in names)
anynull = " OR ".join(f"{_q(c)} IS NULL" for c in names)
checked = sql.one(f"SELECT COUNT(*) FROM {_q(table)} WHERE {nonnull}")[0]
nulls = sql.one(f"SELECT COUNT(*) FROM {_q(table)} WHERE {anynull}")[0]
types = {c.name: c.type for c in columns}
added = " + ".join(_q(p) for p in parts)
if any(types.get(c) == "REAL" for c in names):
cond = f"ABS({_q(total)} - ({added})) > 1e-9 * MAX(1, ABS({_q(total)}))"
mismatch = f"ABS({_q(total)} - ({added})) > 1e-9 * MAX(1, ABS({_q(total)}))"
else:
cond = f"{_q(total)} <> {added}"
mismatch = f"{_q(total)} <> {added}"
return nonnull, anynull, mismatch
def _check_sum_eq(sql: _Sql, rel: SumEq, title: str, columns: list[ColumnOut], grain: list[str],
lineage: dict[str, dict[str, list[list[Any]]]]) -> CheckResult:
table, total, parts = rel.table, rel.total, list(rel.parts)
nonnull, anynull, cond = sum_eq_conditions(total, parts, {c.name: c.type for c in columns})
checked = sql.one(f"SELECT COUNT(*) FROM {_q(table)} WHERE {nonnull}")[0]
nulls = sql.one(f"SELECT COUNT(*) FROM {_q(table)} WHERE {anynull}")[0]
added = " + ".join(_q(p) for p in parts)
where = f"{nonnull} AND {cond}"
failed = sql.one(f"SELECT COUNT(*) FROM {_q(table)} WHERE {where}")[0]
picked = ", ".join(["rowid", *(_q(g) for g in grain), _q(total), f"({added})"])
+12 -2
View File
@@ -239,6 +239,11 @@ def _in_spans(spans: list[tuple[int, int]], starts: list[int], x: int) -> bool:
return i >= 0 and spans[i][0] <= x <= spans[i][1]
def _hidden_cell(sh: _Sheet, r: int, c: int) -> bool:
"""这一格所在的行或列是否被隐藏(OutsideText.hidden)。行列都看:隐藏列里的备注和隐藏行里的一样不外发。"""
return sh.hidden_row(r) or sh.hidden_col(c)
def _open_sqlite(path: str, *, readonly: bool) -> sqlite3.Connection:
"""关 DQS 的 sqlite3 连接,和核对模块共用 engine.open_checked_sqlite:关不掉就拒绝连接。"""
return _data_engine.open_checked_sqlite(path, readonly=readonly)
@@ -2475,7 +2480,10 @@ class _Engine:
coord = sh.coord(r, c)
sh.roles[role] += 1
sh.regions.feed_row(r, [(c, role)])
self.ex.outside_text.append(OutsideText(sh.name, coord, text, kind, period_source)) # type: ignore[arg-type]
# 期 4(P4-SPEC 3.2、调整 10):记下这格是否在隐藏行列里。裁判摘录只把确知可见的区域外文字全文外发,
# 隐藏行列里常是内部备注(H10 的依据正是隐藏内容里实测有口令),不能借裁判多一条外发通道
self.ex.outside_text.append(OutsideText(sh.name, coord, text, kind, period_source, # type: ignore[arg-type]
hidden=_hidden_cell(sh, r, c)))
if kind == "text_digits" and not period_source:
self.problems.add(
"outside_digits",
@@ -2551,7 +2559,9 @@ class _Engine:
sh.regions.feed_row(r, [(c, "context")])
if not is_pure_period_cell(text, prefer):
out.annotated[coord] = period_residue(text) or ""
self.ex.outside_text.append(OutsideText(sh.name, coord, text, "text_digits", True))
# 带附加文字的统计期格同样是区域外文字,同样记是否隐藏(同 _outside_text)
self.ex.outside_text.append(OutsideText(sh.name, coord, text, "text_digits", True,
hidden=_hidden_cell(sh, r, c)))
self.ex.period = out
cells = list(out.cells)
if failed:
+8 -1
View File
@@ -1,4 +1,5 @@
"""配方导入的契约(期 2,波次 0 由编排者提交;期 3 的改动见 P3-SPEC 9.2,同样由波次 0 提交):配方 JSON 的模型、
"""配方导入的契约(期 2,波次 0 由编排者提交;期 3 的改动见 P3-SPEC 9.2,同样由波次 0 提交;期 4 只给 OutsideText
加了 hidden,见 P4-SPEC 7.1):配方 JSON 的模型、
执行结果和各模块之间传递的数据结构。
各工作包之间只通过这里的类型和 P2-SPEC.md、P3-SPEC.md 里写明的函数签名交互。这里**只有声明**,没有业务逻辑
@@ -706,6 +707,12 @@ class OutsideText:
kind: Literal["text", "text_digits"]
#: 这格同时是统计期来源(带附加文字的统计期格,PeriodOut.annotated)
period_source: bool = False
#: 期 4(P4-SPEC 7.1,WP-0):这格所在的行或列是否被隐藏,执行器写入(_outside_text、统计期格附带的文字两处)。
#: True = 在隐藏行列里;False = 可见;None = 导入时没有记录(期 4 之前的清单没有这个键,
#: recipe_imports._known 读老清单时缺键就是 None,不用改读法)。
#: 裁判摘录只送 hidden is False 的项(P4-SPEC 3.2):隐藏行里的内部备注不能多一条经裁判外发的通道,
#: 不知道是否隐藏的(None)同样只计条数。差异卡、确认项只按坐标、文字、kind、period_source 比较,不看它
hidden: bool | None = None
@dataclass
+533
View File
@@ -0,0 +1,533 @@
"""期 4「推断的来源」的 SQL 判据:认出「从一张表里直接选取若干列」这一种形状(P4-SPEC 2.4、7.2)。
下钻要回答「查询结果的第 i 列是哪张表的哪一列」。只有一张表、每个结果列都是这张表某列的直接引用时,这个问题才有
唯一的答案;改名、表达式、多表、子查询、集合运算之后,结果列装的可能是另一列的值,值恰好相等时就会下钻到错误的
格子。所以这里只认一种形状,形状之外一律拒绝:**宁可不下钻,也不能下钻错**。
**读法和执行这条 SQL 的 SQLite 相同**:分词用守卫的 `guard._tokens(sql, guard._SQLITE)`。查询快照里存的 SQL 是
`guard.check(…, dialect="sqlite")` 的返回值,执行的也正是它(engine.run_query)。`_SQLITE` 读法:单引号字符串里
反斜杠不是转义;`[…]` 是标识符,里面没有转义,遇到第一个 `]` 就结束;`"…"`、`` `…` `` 是标识符,两个引号表示
一个引号。识别器和数据库读的是同一串记号,中间没有第二种读法。
**SQLite 分词的两处细节**(tokenize.c,实测 3.53):
- U+FEFF(字节 EF BB BF,CC_BOM)出现在记号开头时是空白,出现在名字中间时是名字的一部分。不能照「码位 ≥ 0x80
一律算名字」读:那样紧跟在 U+FEFF 后面的 UNION 成了一个普通的词,藏在它后面的 UNION ALL、GROUP BY 就看不见了,
SQLite 却照样执行,结果列里混着另一列的值;
- 竖制表符出现在记号开头时是非法字符,紧跟在其他空白后面时是空白(CC_SPACE 的循环用 sqlite3Isspace,含 \\v)。
这里一律当空白:SQLite 不当空白的位置整条 SQL 执行不了,不会留下查询快照,判据碰不到;能执行的位置读法相同。
**为什么不用 `evidence._sql_scan`**:它按多方言「宁可少认一张表」设计,单引号里认反斜杠转义、方括号里含引号时不当
标识符。藏在 `ESCAPE '\\'` 或 `AS [t']` 后面的 `UNION ALL` 在它眼里不存在,SQLite 却照样执行(P4-SPEC 附录 B7)。
对 `sql_tables` 那只是少认一张表,对下钻就是错格子。`_sql_scan`、`sql_tables` 本身不动(H7)。
**执行时与存下的字面不同的一处**:engine 经 SQLAlchemy 的 `text()` 执行,它把 `\\:` 改成 `:`、把 `:名字` 当成绑定
参数。后者没有给值,执行会失败,不会留下查询快照;前者只删掉冒号前的反斜杠,SQLite 读法里反斜杠不影响任何记号的
边界,只改字符串和带引号名字的内容,而配方导入的表名、列名不含反斜杠和冒号(names.name_problem),对不上任何表列。
**分组**(界面显示分组的原文,细分给测试和日志用,取值见 provenance_types.DETAILS):
- expression:表达式、聚合、字符串或字面量关键字、窗口、COLLATE、DISTINCT;
- alias:给列改名(含省略 AS 的改名);
- multi_table:JOIN、逗号连接、子查询、公用表表达式、集合运算;
- unparsed:不是 SELECT、多条语句、表的写法不认识、限定名不是本表、列或表找不到、重名、`*` 对不上、尾部写法不认识。
判据绑在文档标记 `DOC_PROVENANCE = 1` 上(P4-SPEC 1.4):以后放宽文法要升版本号、给 recognize 加 version 参数,
已有 `provenance: 1` 的文档仍按这一版判,结论不跟着变。
只许导入守卫、names 和 provenance_types(P4-SPEC 7.2):判据是纯函数,不连库,不依赖证据层和接口层。
"""
from __future__ import annotations
from collections.abc import Iterable
from dataclasses import dataclass
from app.data.guard import _SQLITE, _tokens
from app.data.names import name_key
from app.data.provenance_types import DirectSelect, SelectRefusal
# ==========================================================================
# 分词:守卫的 SQLite 读法之上,再按 SQLite 的规则把代码段切成词和标点
# ==========================================================================
#: SQLite 在记号之间认的空白:CC_SPACE 的五个,加竖制表符。不能用 str.isspace:全角空格、不换行空格在 SQLite 里是
#: 名字的一部分(除记号开头的 U+FEFF 之外,码位 ≥ 0x80 的字符一律算名字)。竖制表符出现在记号开头时 SQLite 报非法
#: 记号,紧跟其他空白时当空白;一律当空白是安全的:前一种位置执行不了,后一种读法相同(模块说明)
_SPACE = frozenset(" \t\n\v\f\r")
#: U+FEFF 只在记号开头是空白(CC_BOM),名字中间是名字的一部分(「日\ufeff期」是一个名字),所以不放进 _SPACE、
#: 只在 _split_code 判断记号开头的地方单独认
_BOM = "\ufeff"
_ASCII_LETTERS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
_ASCII_DIGITS = frozenset("0123456789")
def _name_start(ch: str) -> bool:
"""不加引号的名字的开头:ASCII 字母、下划线、码位 ≥ 0x80 的字符(同 SQLite,也同 2 版实体语法)。
U+FEFF 例外:出现在记号开头时 SQLite 当空白,调用处(_split_code)先把它判掉,轮不到这里。名字中间的 U+FEFF
仍算名字:_name_char 借这个函数认后续字符,所以这里不排除它。"""
return ch in _ASCII_LETTERS or ch == "_" or ord(ch) >= 0x80
def _name_char(ch: str) -> bool:
"""名字的后续字符:开头字符之外再加 ASCII 数字和 $(SQLite 的 IdChar)。不用 isalnum:「٣」这类非 ASCII
数字本来就 ≥ 0x80,算名字;ASCII 以外的 isdigit 不能决定切词。"""
return _name_start(ch) or ch in _ASCII_DIGITS or ch == "$"
@dataclass(frozen=True)
class _Lex:
#: word:不加引号的词;name:带引号的名字("…" `…` […]);str:字符串;blob:x'…';num:数字;punct:标点;
#: sep:分号
kind: str
#: word、num、punct 是原文;name、str 是去掉定界符、还原了成对引号的内容
text: str
#: 在原 SQL 里的结束位置:认 x'…' 时要知道 x 和引号之间有没有空白
end: int
def _split_code(text: str, base: int, out: list[_Lex]) -> None:
"""把一段代码(守卫已经把注释、字符串、带引号的名字、分号切出去了)按 SQLite 的规则切成词、数字和标点。
数字只吃数字和小数点:`1e5`、`0x1F` 会被切成「数字 + 词」,`1union` 也会切出 UNION。SQLite 遇到数字后紧跟
名字字符会报非法记号、根本不执行,所以这样切只会让判据多看见词、多拒,不会少看见。
循环顶层的每个位置都是一个记号的开头(代码段的开头紧跟在注释、字符串、带引号的名字或分号后面,也是):
U+FEFF 在这里当空白,同 SQLite 的 CC_BOM;进了名字以后由 _name_char 当名字字符吃掉。
"""
i, n = 0, len(text)
while i < n:
ch = text[i]
if ch in _SPACE or ch == _BOM:
i += 1
elif _name_start(ch):
j = i + 1
while j < n and _name_char(text[j]):
j += 1
out.append(_Lex("word", text[i:j], base + j))
i = j
elif ch in _ASCII_DIGITS or (ch == "." and i + 1 < n and text[i + 1] in _ASCII_DIGITS):
j = i + 1
while j < n and (text[j] in _ASCII_DIGITS or text[j] == "."):
j += 1
out.append(_Lex("num", text[i:j], base + j))
i = j
else:
out.append(_Lex("punct", ch, base + i + 1))
i += 1
def _lex(sql: str, *, strict: bool = True) -> list[_Lex] | None:
"""整条 SQL → 词序列。引号没闭合时 strict 返回 None(SQLite 会报非法记号,存下的 SQL 不该这样);
不 strict 时照样切,没闭合的那一段内容取到末尾(tables_in 宁可多认)。
注释当空白跳过:SQLite 也这样读,注释两边的代码各自成词(`SEL/**/ECT` 是两个词)。
"""
out: list[_Lex] = []
pos = 0
for tok in _tokens(sql, _SQLITE):
start, pos = pos, pos + len(tok.text)
if tok.kind == "comment":
continue
if tok.kind == "sep":
out.append(_Lex("sep", ";", pos))
elif tok.kind in ("string", "ident"):
if tok.close == 0 and strict:
return None
inner = tok.text[tok.open:len(tok.text) - tok.close]
quote = tok.text[0]
if tok.kind == "string":
inner = inner.replace("''", "'")
prev = out[-1] if out else None
# x'00':紧挨在单独一个 x 后面的单引号串是 BLOB 字面量,不是「名字 x 加字符串别名」
if prev is not None and prev.kind == "word" and prev.text in ("x", "X") and prev.end == start:
out[-1] = _Lex("blob", inner, pos)
else:
out.append(_Lex("str", inner, pos))
else:
# 方括号里没有转义;双引号、反引号里两个引号表示一个
if quote in "\"`":
inner = inner.replace(quote * 2, quote)
out.append(_Lex("name", inner, pos))
else:
_split_code(tok.text, start, out)
return out
def _kw(lex: _Lex | None) -> str | None:
"""不加引号的词按关键字比较时用的键(只对 ASCII 不区分大小写,同 SQLite);不是词返回 None。
带引号的名字永远不是关键字:`"union"` 是一个叫 union 的列。
"""
return name_key(lex.text) if lex is not None and lex.kind == "word" else None
def _is_name(lex: _Lex | None) -> bool:
return lex is not None and lex.kind in ("word", "name")
# ==========================================================================
# 关键字表
# ==========================================================================
#: P4-SPEC 2.4:尾部(以及整条语句里)不得出现的词,按细分归组。七组的并集就是规格的那张表,测试钉住
_COMPOUND = frozenset({"union", "intersect", "except"})
_CTE = frozenset({"with", "recursive"})
_SUBQUERY = frozenset({"select", "values"})
_JOIN = frozenset({"join", "natural", "cross", "inner", "left", "right", "full", "outer"})
_WINDOW = frozenset({"over", "window"})
#: GROUP BY / HAVING:结果是聚合出来的
_AGGREGATE = frozenset({"group", "having"})
#: 尾部的 FROM、DISTINCT 只会出现在 `IS [NOT] DISTINCT FROM` 这类写法里:不影响结果列,但超出这一版文法
_TAIL_ODD = frozenset({"from", "distinct"})
FORBIDDEN = _COMPOUND | _CTE | _SUBQUERY | _JOIN | _WINDOW | _AGGREGATE | _TAIL_ODD
#: 不加引号时是字面量的关键字(P4-SPEC 2.4)。CURRENT_* 永远是关键字;TRUE、FALSE 只在表里有同名列时才指那一列,
#: 读法取决于表结构,一律拒最保守
_LITERAL_KEYWORDS = frozenset({"null", "true", "false", "current_date", "current_time", "current_timestamp"})
#: SQLite 的保留字里不能当不加引号的名字用的那些:出现在选取项、表名的位置,这一项就是表达式(CASE …、NOT x、
#: EXISTS (…))或者写法不对。CAST、RAISE、MATCH、GLOB、END、FILTER 之类不在里面:SQLite 允许它们当名字(fallback),
#: 后面紧跟「(」时由下一步按函数调用拒,单独出现、又真有这一列时它就是那一列
_EXPR_WORDS = frozenset({
"case", "not", "exists", "when", "then", "else", "select", "distinct", "all",
"from", "where", "group", "having", "order", "limit", "as", "on", "using",
"and", "or", "is", "in", "between", "escape", "collate", "isnull", "notnull",
})
#: 选取项后面紧跟这些词:这一项是运算的一部分(`"日期" IS NULL`、`"a" BETWEEN …`),不是省略 AS 的改名
_OPERATOR_WORDS = frozenset({
"is", "isnull", "notnull", "not", "and", "or", "in", "like", "glob", "regexp", "match", "between",
"escape", "filter",
})
#: 表名后面紧跟这些词时它们不是表别名:子句开头、连接、`INDEXED BY` / `NOT INDEXED`、ON / USING
_AFTER_TABLE = frozenset({
"where", "order", "limit", "group", "having", "window", "indexed", "not", "on", "using", "as",
*_JOIN, *_COMPOUND,
})
#: 尾部允许的开头(P4-SPEC 2.4 文法):只过滤、排序、截取行,不改变结果列的来历
_TAIL_START = frozenset({"where", "order", "limit"})
def _refuse(code: str, detail: str) -> SelectRefusal:
return SelectRefusal(code, detail) # type: ignore[arg-type]
# ==========================================================================
# recognize
# ==========================================================================
def _statement(lexes: list[_Lex]) -> list[_Lex] | SelectRefusal:
"""去掉尾部的分号;分号后面还有东西就是多条语句。
存下的 SQL 是 guard.check 的返回值,本来就没有分号和注释。P4-SPEC 7.2 写的是「出现就按 multi_statement 拒」,
但 2.4 的文法写了 `[;]`,原型的 24 例(期望不变)和 6.3 要求注释里的 JOIN、尾分号照样认可,两处矛盾。这里取
后者,按 SQLite 的读法接受尾分号、把注释当空白:两者都不改变这条语句选了什么,读法又和执行时相同,不会因此
认错;真正的第二条语句照样拒。
"""
end = len(lexes)
while end and lexes[end - 1].kind == "sep":
end -= 1
body = lexes[:end]
if any(lx.kind == "sep" for lx in body):
return _refuse("unparsed", "multi_statement")
return body
def _global_shape(body: list[_Lex]) -> SelectRefusal | None:
"""整条语句里只要出现就不是「一张表的直接选取」的关键字:集合运算、公用表表达式、子查询、连接、窗口。
先于逐项的形状判断:B7 那类写法(`… ESCAPE '\\' UNION ALL …`)不管前面哪一项长什么样,都要报成集合运算。
词只取代码段里的:字符串、带引号的名字、注释里出现的不算,和 SQLite 的读法一样。
"""
words = {_kw(lx) for lx in body[1:] if lx.kind == "word"}
for group, detail, code in ((_COMPOUND, "compound", "multi_table"), (_CTE, "cte", "multi_table"),
(_SUBQUERY, "subquery", "multi_table"), (_JOIN, "join", "multi_table"),
(_WINDOW, "window", "expression")):
if words & group:
return _refuse(code, detail)
return None
@dataclass
class _Item:
qualifier: str | None
column: str
def _item_start(lex: _Lex | None) -> SelectRefusal | None:
"""选取项的第一个记号不是名字时,这一项是什么。是名字返回 None。"""
if lex is None:
return _refuse("unparsed", "not_select")
if lex.kind == "str":
return _refuse("expression", "string_literal")
kw = _kw(lex)
if kw in _LITERAL_KEYWORDS:
return _refuse("expression", "literal_keyword")
if kw in _EXPR_WORDS or not _is_name(lex):
# 数字、blob、括号、正负号、CASE / CAST / NOT …
return _refuse("expression", "expression")
return None
def _after_item(lex: _Lex | None) -> SelectRefusal:
"""选取项后面只能紧跟「,」或 FROM(P4-SPEC 2.4)。其余的分别是什么。"""
kw = _kw(lex)
if lex is None:
# 没有 FROM:不是从表里选
return _refuse("unparsed", "table_shape")
if kw == "as":
return _refuse("alias", "alias")
if kw == "collate":
return _refuse("expression", "collate")
if kw in _OPERATOR_WORDS:
return _refuse("expression", "expression")
if lex.kind in ("name", "str") or (lex.kind == "word" and kw not in _EXPR_WORDS):
# `"日期" "分区乙"`、`"日期" '分区乙'`、`"日期" 别名`:SQLite 都当省略 AS 的改名
return _refuse("alias", "alias")
# 「(」是函数调用,其余标点、数字是运算
return _refuse("expression", "expression")
def _select_list(body: list[_Lex], k: int) -> tuple[list[_Item] | None, int] | SelectRefusal:
"""解析选取项,返回 (项列表,FROM 的位置)。`*` 单独出现时项列表为 None。"""
if k < len(body) and body[k].kind == "punct" and body[k].text == "*":
nxt = body[k + 1] if k + 1 < len(body) else None
if _kw(nxt) != "from":
if nxt is not None and nxt.kind == "punct" and nxt.text == ",":
# `*, "日期"`:* 只许单独出现,否则结果列和表的列对不上位置
return _refuse("unparsed", "star_mismatch")
return _after_item(nxt)
return None, k + 1
items: list[_Item] = []
while True:
lex = body[k] if k < len(body) else None
if lex is not None and lex.kind == "punct" and lex.text == "*":
# `"日期", *`
return _refuse("unparsed", "star_mismatch")
refusal = _item_start(lex)
if refusal is not None:
return refusal
assert lex is not None
k += 1
qualifier: str | None = None
column = lex.text
if k < len(body) and body[k].kind == "punct" and body[k].text == ".":
target = body[k + 1] if k + 1 < len(body) else None
if target is not None and target.kind == "punct" and target.text == "*":
# `t.*`:不在这一版文法里
return _refuse("unparsed", "star_mismatch")
if not _is_name(target):
return _refuse("expression", "expression")
assert target is not None
qualifier, column = lex.text, target.text
k += 2
if k < len(body) and body[k].kind == "punct" and body[k].text == ".":
# `main.日客流.日期`:带库名的限定名
return _refuse("unparsed", "qualifier")
items.append(_Item(qualifier, column))
nxt = body[k] if k < len(body) else None
if nxt is not None and nxt.kind == "punct" and nxt.text == ",":
k += 1
continue
if _kw(nxt) == "from":
return items, k
return _after_item(nxt)
def _from_clause(body: list[_Lex], k: int) -> tuple[str, str | None, int] | SelectRefusal:
"""FROM 后面:表名、可选的表别名。返回 (表名,别名,尾部的起点)。"""
lex = body[k] if k < len(body) else None
if lex is not None and lex.kind == "punct" and lex.text == "(":
# FROM (SELECT …);`FROM (日客流)` 在 SQLite 里也能执行,一律不认
return _refuse("multi_table", "subquery")
if not _is_name(lex) or _kw(lex) in _AFTER_TABLE | _EXPR_WORDS:
# 字符串当表名(SQLite 的历史兼容)、数字、缺表名
return _refuse("unparsed", "table_shape")
assert lex is not None
table = lex.text
k += 1
nxt = body[k] if k < len(body) else None
if nxt is not None and nxt.kind == "punct" and nxt.text == ".":
return _refuse("unparsed", "qualified_table")
if nxt is not None and nxt.kind == "punct" and nxt.text == "(":
# 表函数(json_each(…))
return _refuse("unparsed", "table_shape")
alias: str | None = None
if _kw(nxt) == "as":
target = body[k + 1] if k + 1 < len(body) else None
if not _is_name(target) or _kw(target) in _AFTER_TABLE | _EXPR_WORDS:
# `AS '别名'`:SQLite 认字符串当别名,这一版文法不认
return _refuse("unparsed", "table_shape")
assert target is not None
alias = target.text
k += 2
elif nxt is not None and (nxt.kind == "name"
or (nxt.kind == "word" and _kw(nxt) not in _AFTER_TABLE | _EXPR_WORDS)):
alias = nxt.text
k += 1
return table, alias, k
def _tail(body: list[_Lex], k: int) -> SelectRefusal | None:
"""尾部只能以 WHERE / ORDER / LIMIT 开头,里面不得有 P4-SPEC 2.4 那张表里的词。
整条语句里的集合运算、子查询、连接、窗口已经在 _global_shape 拦了,这里只剩 GROUP / HAVING(聚合)和尾部的
FROM、DISTINCT。
"""
if k >= len(body):
return None
head = body[k]
kw = _kw(head)
if kw not in _TAIL_START:
if head.kind == "punct" and head.text == ",":
return _refuse("multi_table", "comma_join")
if kw in _AGGREGATE:
return _refuse("expression", "expression")
# INDEXED BY、NOT INDEXED、第二个别名、括号……
return _refuse("unparsed", "tail_shape")
words = {_kw(lx) for lx in body[k:] if lx.kind == "word"}
if words & _AGGREGATE:
return _refuse("expression", "expression")
if words & _TAIL_ODD:
return _refuse("unparsed", "tail_shape")
return None
def _index(names: Iterable[str]) -> dict[str, str | None]:
"""name_key → 原名。两个名字撞了键(冻结表结构里不该出现)时记 None:说不清是哪一个,按找不到处理。"""
out: dict[str, str | None] = {}
for name in names:
key = name_key(name)
out[key] = None if key in out else name
return out
def recognize(sql: str, *, result_columns: list[str], tables: dict[str, list[str]]
) -> DirectSelect | SelectRefusal:
"""tables:冻结表结构里的 {表名: [列名…]}(按定义顺序)。
认可(P4-SPEC 2.4 的文法)→ DirectSelect;否则 → SelectRefusal(code, detail)。纯函数,不连库,不抛异常。
文法对应 DOC_PROVENANCE = 1(1.4);以后放宽文法时加 version 参数,不改这一版的结论。
判断顺序:分词和语句个数 → 是不是 SELECT(WITH 开头是公用表表达式)→ 整条语句里的集合运算、公用表表达式、
子查询、连接、窗口 → 选取项 → FROM → 尾部 → 名字解析(表、限定名、列、重名)→ 结果列与选取项逐个对应。
"""
lexes = _lex(sql)
if lexes is None:
return _refuse("unparsed", "not_select")
body = _statement(lexes)
if isinstance(body, SelectRefusal):
return body
first = _kw(body[0]) if body else None
if first == "with":
return _refuse("multi_table", "cte")
if first != "select":
return _refuse("unparsed", "not_select")
refusal = _global_shape(body)
if refusal is not None:
return refusal
k = 1
if _kw(body[k] if k < len(body) else None) == "all":
k += 1
if _kw(body[k] if k < len(body) else None) == "distinct":
return _refuse("expression", "distinct")
parsed = _select_list(body, k)
if isinstance(parsed, SelectRefusal):
return parsed
items, k = parsed
shape = _from_clause(body, k + 1)
if isinstance(shape, SelectRefusal):
return shape
table_written, alias, k = shape
refusal = _tail(body, k)
if refusal is not None:
return refusal
# ---- 名字解析:一律按 name_key(只对 ASCII 不区分大小写,同 SQLite) ----
table = _index(tables).get(name_key(table_written))
if table is None:
return _refuse("unparsed", "unknown_table")
defined = list(tables[table])
if items is None:
# `*`:结果列必须和表的列按 name_key 逐个相等、顺序一致,才能按位置对应
if [name_key(c) for c in result_columns] != [name_key(c) for c in defined]:
return _refuse("unparsed", "star_mismatch")
return DirectSelect(table, alias, defined)
# 有别名时只认别名:SQLite 里表起了别名之后,原表名不能再当限定名(`SELECT 日客流.日期 FROM 日客流 AS t`
# 报 no such column),所以这一条比「表名或别名」严,严出来的只是执行不了的写法
owner = name_key(alias if alias is not None else table_written)
if any(it.qualifier is not None and name_key(it.qualifier) != owner for it in items):
return _refuse("unparsed", "qualifier")
columns_index = _index(defined)
columns: list[str] = []
for it in items:
column = columns_index.get(name_key(it.column))
if column is None:
return _refuse("unparsed", "unknown_column")
if column in columns:
# `SELECT "日期", "日期"`、`SELECT Key, key`:locate_cell 按第一次出现取,说不清是哪一个
return _refuse("unparsed", "ambiguous_column")
columns.append(column)
# 结果列按位置对应第 i 个选取项,不按名字认(SQLite 返回的列名有时是原写法、有时是定义名,P4-SPEC 2.4)。
# 个数必须相等;名字只作一致性核对:直接引用的列,SQLite 返回的名字按 name_key 一定等于那一列,
# 不等说明这份结果不是这条 SQL 的,不下钻
if len(result_columns) != len(columns) or any(
name_key(r) != name_key(c) for r, c in zip(result_columns, columns)):
return _refuse("unparsed", "unknown_column")
return DirectSelect(table, alias, columns)
# ==========================================================================
# tables_in
# ==========================================================================
#: 这些词后面紧跟的单引号串,SQLite 当表名读(`FROM '日客流'` 能执行,历史兼容)。IN:SQLite 文法 `expr IN nm`
#: 的 nm 可以是字符串,`WHERE 日期 IN '单列表'` 读的是那张表(列数和左边对得上就能执行,行值也行)
_TABLE_BEFORE_STRING = frozenset({"from", "join", "in"})
#: 逗号后面的单引号串当表名读的子句:FROM 列表里(含 ON / USING 之后接着逗号连接)
_TABLE_LIST_CLAUSES = frozenset({"from", "join", "on", "using"})
#: 切换子句的词:tables_in 据此判断逗号后面是不是还在 FROM 列表里
#: 不含 AS:`FROM a AS x, '时段客流'` 的逗号仍在 FROM 列表里
_CLAUSE_WORDS = frozenset({
"select", "from", "join", "where", "group", "having", "order", "limit", "on", "using", "window", "values",
"union", "intersect", "except", "with",
})
def tables_in(sql: str, tables: dict[str, list[str]]) -> list[str]:
"""SQL 里出现的冻结表结构的表名(原样),按第一次出现的顺序去重。读法同 recognize,宁可多认(P4-SPEC 2.4)。
每个带引号的名字、代码里的每个词都按 name_key 对表名:列名或别名恰好和某张表同名时会多出这张表,后果只是
摘录里多一行说明;漏认的后果是裁判拿不到「原表写明的合计,不要相加」。不用 2 版的 `sql_tables`:它的读法和
SQLite 不同,藏在 `ESCAPE '\\'` 后面的合计表会漏掉。
字符串一般不算(`WHERE 分区 = '日客流'` 不是读表),只有 SQLite 会把它当表名读的位置才算:紧跟 FROM / JOIN /
IN、紧跟「.」(`main.'日客流'`),以及 FROM 列表里的逗号、括号之后。「紧跟」按 SQLite 的分词算:中间隔着的空白
(含记号开头的 U+FEFF、空白后面的竖制表符)不算记号。注释里的不算:SQLite 不读注释。
"""
index = _index(tables)
found: dict[str, None] = {}
lexes = _lex(sql, strict=False) or []
clause: str | None = None
prev: _Lex | None = None
for lx in lexes:
candidate: str | None = None
if lx.kind in ("word", "name"):
candidate = lx.text
elif lx.kind == "str" and prev is not None:
pkw = _kw(prev)
if (pkw in _TABLE_BEFORE_STRING or (prev.kind == "punct" and prev.text == ".")
or (prev.kind == "punct" and prev.text in ",(" and clause in _TABLE_LIST_CLAUSES)):
candidate = lx.text
if candidate is not None:
table = index.get(name_key(candidate))
if table is not None:
found.setdefault(table, None)
kw = _kw(lx)
if kw in _CLAUSE_WORDS:
clause = kw
prev = lx
return list(found)
+5 -1
View File
@@ -54,6 +54,7 @@ from typing import Any, Literal, TypedDict
from app.core.artifact_store import canonical_json, content_hash
from app.core.artifact_store import load as load_artifact
from app.data.names import FILLERS, name_key
from app.data.provenance_types import DOC_PROVENANCE
from app.data.tabular import UNSHAPED_NOTE
from app.engine.expressions import CellError, cell_value, column_kind, locate_cell
from app.engine.issuance import _tolerance, extract_numbers, number_allowance
@@ -2600,8 +2601,11 @@ def compose_doc(
if c.get("status") == "resolved" and c.get("alias")}
kept = {a: e for a, e in catalog.items() if e.get("kind") not in ENTITY_KINDS or a in used}
# entity_syntax:这份文档按哪一版实体语法组装,复核照它走(verify_doc)。在文档顶层,参与内容哈希
# provenance:期 4 的文档标记(P4-SPEC 1.4)。带它的文档才有查询步骤的提示字段、推断来源、裁判摘录的新行;
# 没有它的(期 4 之前组装的)一律按原样展示。写在顶层、参与内容哈希,事后补不上:补了哈希就变,封存链就断
doc: dict[str, Any] = {
"schema": DOC_SCHEMA, "entity_syntax": snaps.syntax, "run_id": run_id, "node_id": node_id,
"schema": DOC_SCHEMA, "entity_syntax": snaps.syntax, "provenance": DOC_PROVENANCE,
"run_id": run_id, "node_id": node_id,
"markdown": markdown, "source": source, "catalog": kept, "blocks": out_blocks,
}
checked = verify_doc(doc, catalog, allow_numbers=allow_numbers, cells_allowed=cells_allowed, loader=snaps)
+54 -5
View File
@@ -9,7 +9,8 @@
哪一格和那次查询的 SQL;查询给 SQL 前 300 字和被引用的行(最多 20 行,数据源遮罩的列不给;宽表先给
被引用的列,再补满 12 列);表附上字段清单和类型(最多 60 个,遮罩的列不列,快照不完整要注明),
字段附上类型和所属的表,两样都附上出现过的查询 SQL;知识库片段最多 600 字。快照一律经 loader 取
(按需裁判传只认封存工件的那个)。规则里写明数字已经由系统核对过
(按需裁判传只认封存工件的那个)。规则里写明数字已经由系统核对过。带期 4 文档标记的报告,上传表格的
查询在 SQL 之后另有来历行(来源、口径、已接受、区域外文字,_provenance_lines)
3. 调用(judge_doc):一份报告合并成一次调用,超过每批的句数就分批;结构化输出失败退回 JSON 解析,
再失败就记未裁判。thinking 关掉,max_tokens 4096
4. 预算:每份报告的句数、金额、时长和全局每日金额,每一项都能是 None(不限)。调用前按目录价估算,
@@ -28,6 +29,7 @@ from __future__ import annotations
import asyncio
import json
import logging
import math
import re
import time
@@ -43,6 +45,8 @@ from app.core.artifact_store import canonical_json, content_hash
from app.core.artifact_store import load as load_artifact
from app.core.errors import describe_exception, not_configured
from app.core.events import EventType
# 只用标准库的契约模块:取文档标记的版本号。识别器、清单模块(direct_select、provenance)在函数体内导入(_provenance_lines)
from app.data.provenance_types import DOC_PROVENANCE
from app.data.tabular import UNSHAPED_NOTE
from app.db.base import SessionLocal
from app.db.models import Setting
@@ -66,6 +70,8 @@ from app.engine.state import message_text
from app.providers import catalog as pricing
from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model
logger = logging.getLogger(__name__)
# --------------------------------------------------------------------------
# 常量
# --------------------------------------------------------------------------
@@ -328,7 +334,12 @@ def _shown_columns(columns: list[str], cited: Iterable[str]) -> list[str]:
def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iterable[str], whole: bool,
fetch: _Fetch, masked: dict[str, Any] | None) -> str | None:
fetch: _Fetch, masked: dict[str, Any] | None, *, provenance_on: bool = False) -> str | None:
"""一条查询的摘录:头、SQL、(期 4)来历行、列、被引用的行。
provenance_on:文档带期 4 标记时为真(prepare 算)。这时上传表格的查询快照(有 data_version)在 SQL 之后、「列:」
之前多几行来历(_provenance_lines);其余情况一个字都不多,期 4 之前的文档、手工源的摘录和以前逐字相同。
"""
snapshot = fetch(entry.get("artifact"))
if not isinstance(snapshot, dict) or not isinstance(snapshot.get("rows"), list):
return None
@@ -339,6 +350,8 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter
sql = _clean(str(snapshot.get("sql") or ""))
lines = [head, f"SQL:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"]
hidden = _hidden(snapshot, source, masked)
if provenance_on and snapshot.get("data_version"):
lines += _provenance_lines(snapshot, fetch, hidden)
columns = [str(c) for c in snapshot.get("columns") or [] if str(c).lower() not in hidden]
shown_cols = _shown_columns(columns, cols)
# 点名引用的行先给,整份引用的再从头补满,一共不超过 EXCERPT_ROWS 行
@@ -361,6 +374,34 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter
return "\n".join(lines)
def _provenance_lines(snapshot: dict[str, Any], fetch: _Fetch, hidden: set[str]) -> list[str]:
"""上传表格的查询快照的来历行(P4-SPEC 3.2、3.3):provenance.judge_lines 生成,这里只负责取冻结的表结构快照、
算 SQL 用到的表(direct_select.tables_in,和执行这条 SQL 的 SQLite 同一种读法,宁可多认),把现成的遮罩交过去。
- 两个模块在函数体内导入:judge 被设置接口等地方导入,不能因此拉进整套配方模块;也只有带标记的文档、上传
表格的快照才走到这里,期 4 之前的文档一次都不碰它们(测试用桩钉住);
- 工件一律经同一个 fetch 取(按需裁判时是只认封存工件、带标记才扩展到哈希链上清单的那个 loader),取不到、
哈希不符的都当没有:judge_lines 据此写「导入清单无法读取…」那一行,不中断摘录;
- 确定:同样的工件永远得到同样的行(摘录进断点指纹)。生成过程中出了意外(工件形状不对)同样落到那一行,
报告节点的裁判不因一份坏清单整体失败。
"""
from app.data.provenance import SOURCE_BROKEN, judge_lines
from app.engine.direct_select import tables_in
schema_id = snapshot.get("schema_artifact")
schema = fetch(schema_id) if schema_id else None
schema = schema if isinstance(schema, dict) else None
frozen = schema.get("tables") if schema is not None else None
tables = {str(name): [str(c.get("name")) for c in (meta.get("columns") or []) if isinstance(c, dict)]
for name, meta in frozen.items() if isinstance(meta, dict)} if isinstance(frozen, dict) else {}
try:
used = tables_in(str(snapshot.get("sql") or ""), tables) if tables else []
return judge_lines(snapshot, schema, fetch, tables=used, hidden=hidden)
except Exception: # noqa: BLE001 - 坏形状的工件只让这条查询少了来历,不让整份报告判不了
logger.warning("judge: 生成来历行失败", exc_info=True)
return [SOURCE_BROKEN]
def _sql_line(label: str, sql: str) -> str:
return f"{label}:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"
@@ -557,7 +598,9 @@ def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], who
def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _Fetch,
masked: dict[str, Any] | None, catalog: dict[str, Any]) -> str | None:
masked: dict[str, Any] | None, catalog: dict[str, Any], *, provenance_on: bool = False) -> str | None:
"""一件证据的摘录。provenance_on 只交给查询摘录(_query_excerpt):指标摘录里「输入取自查询 Qn」的那几行不加
来历(P4-SPEC 调整 1),表名、字段名的摘录照旧。"""
kind = entry.get("kind")
if kind == "metric":
return _metric_excerpt(alias, entry, fetch, catalog, masked)
@@ -567,7 +610,8 @@ def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _
if kind == "query":
rows = {r for c in cands for a, r in c.rows if a == alias}
cols = [col for c in cands for a, col in c.cols if a == alias]
return _query_excerpt(alias, entry, rows, cols, any(alias in c.whole for c in cands), fetch, masked)
return _query_excerpt(alias, entry, rows, cols, any(alias in c.whole for c in cands), fetch, masked,
provenance_on=provenance_on)
if kind == "retrieval":
picked = [h for c in cands for a, h in c.hits if a == alias]
return _retrieval_excerpt(alias, entry, picked, any(alias in c.whole for c in cands), fetch)
@@ -666,14 +710,19 @@ def prepare(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, units
"""裁判的输入:候选句和证据摘录。纯函数(快照经 loader 取,缺省读工件库),不调模型。
catalog 缺省用文档自己存的目录;masked = {数据源名: 要遮的列},和快照里记下的 mask_columns 合在一起。
摘录按文档标记分版本(P4-SPEC 1.4、3.3):带 `provenance: DOC_PROVENANCE` 的文档,上传表格的查询摘录多几行
来历;没有标记的(期 4 之前组装的)和以前逐字相同,复用键、断点指纹都不变,已判过的句子照旧复用。
"""
catalog = catalog if catalog is not None else (doc.get("catalog") or {})
cands = candidates(doc, units)
fetch = _Fetch(loader)
provenance_on = doc.get("provenance") == DOC_PROVENANCE
excerpts: dict[str, str] = {}
for alias in dict.fromkeys(a for c in cands for a in c.aliases):
entry = catalog.get(alias)
if isinstance(entry, dict) and (text := _excerpt(alias, entry, cands, fetch, masked, catalog)):
if isinstance(entry, dict) and (text := _excerpt(alias, entry, cands, fetch, masked, catalog,
provenance_on=provenance_on)):
excerpts[alias] = text
picked = {c.unit for c in cands}
skipped: dict[str, str] = {}
File diff suppressed because it is too large Load Diff
+862
View File
@@ -0,0 +1,862 @@
"""期 4 的 SQL 判据(P4-SPEC 2.4、6.3、7.2,WP-A):engine/direct_select 的 recognize、tables_in。
判据决定一个结果格能不能下钻到原表的格子。放过一条不是「一张表的直接选取」的 SQL,结果列装的可能是另一列的值,
值恰好相等时界面就会指着错误的格子说「来自这里」——所以这里大半的用例是拒绝的写法,尤其是评审实测过的
B7(`ESCAPE '\\'`、`AS [t']` 后面藏的 UNION ALL)。最后一组拿真的 SQLite 执行随机拼出的 SQL,核对「认可的
每一列,值确实都来自它说的那一列」,不靠逐条想到的用例。
夹具一律假名(分区甲、分区乙、日客流、时段客流),数字随机。
"""
from __future__ import annotations
import ast
import itertools
import random
import sqlite3
from pathlib import Path
import pytest
from app.data import guard
from app.data import provenance_types as P
from app.data.provenance_types import DirectSelect, SelectRefusal
from app.engine import direct_select as D
from app.engine.direct_select import recognize, tables_in
TABLES: dict[str, list[str]] = {
"日客流": ["日期", "全日客流", "分区甲", "分区乙"],
"时段客流": ["日期", "时段", "时段类别", "客流"],
"时段客流_表内合计": ["日期", "合计项", "客流"],
# P4-SPEC 附录 B2:定义名是 Key,SQLite 对 `abc.key` 返回定义名
"abc": ["Key", "v"],
}
DAILY = TABLES["日客流"]
def ok(sql: str, result: list[str], *, tables: dict[str, list[str]] = TABLES) -> DirectSelect:
out = recognize(sql, result_columns=result, tables=tables)
assert isinstance(out, DirectSelect), f"应认可:{sql!r} → {out!r}"
return out
def no(sql: str, code: str, detail: str, *, result: list[str] | None = None,
tables: dict[str, list[str]] = TABLES) -> SelectRefusal:
out = recognize(sql, result_columns=result if result is not None else ["日期", "全日客流"], tables=tables)
assert isinstance(out, SelectRefusal), f"应拒绝:{sql!r} → {out!r}"
# 每个拒绝分支的细分都得在契约的 DETAILS 里:漏了映射,接口一侧的 contract_problems 会报
assert out.detail in P.DETAILS[out.code], out
assert (out.code, out.detail) == (code, detail), f"{sql!r} → {out!r}"
return out
def stored(sql: str) -> str:
"""查询快照里存的就是守卫的返回值(engine.run_query):用例照这个形态喂给判据。"""
return guard.check(sql, readonly=True, dialect="sqlite")
# ---------------------------------------------------------------------------
# 原型的 24 例(P4-SPEC 2.4、附录 B4):期望照原型,改成 code 和 detail
# ---------------------------------------------------------------------------
PROTOTYPE_REFUSED = [
('SELECT "日期", "分区甲" AS "分区乙" FROM "日客流"', "alias", "alias"),
('SELECT "日期" "分区乙" FROM "日客流"', "alias", "alias"),
('SELECT a."日期", b."全日客流" FROM "日客流" a JOIN "日客流" b ON a."日期" = b."日期"', "multi_table", "join"),
('SELECT a."日期", b."全日客流" FROM "日客流" a, "日客流" b WHERE a."日期" = b."日期"', "multi_table",
"comma_join"),
('WITH x AS (SELECT * FROM "日客流") SELECT "日期", "全日客流" FROM x', "multi_table", "cte"),
('SELECT "日期", "全日客流" FROM (SELECT * FROM "日客流")', "multi_table", "subquery"),
('SELECT "日期", "全日客流" FROM "日客流" UNION SELECT "日期", "分区甲" FROM "日客流"', "multi_table", "compound"),
('SELECT "日期", SUM("全日客流") OVER () FROM "日客流"', "expression", "window"),
('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" IN (SELECT "日期" FROM "时段客流")', "multi_table",
"subquery"),
('SELECT SUM("全日客流") AS "合计", COUNT(*) FROM "日客流"', "expression", "expression"),
('SELECT "日期", "全日客流" + 0 FROM "日客流"', "expression", "expression"),
("SELECT '日期', \"全日客流\" FROM \"日客流\"", "expression", "string_literal"),
('SELECT DISTINCT "日期", "全日客流" FROM "日客流"', "expression", "distinct"),
# 表名后面的 INDEXED 不是别名,再往后不是 WHERE / ORDER / LIMIT
('SELECT "日期", "全日客流" FROM "日客流" INDEXED BY x', "unparsed", "tail_shape"),
('SELECT "日期", "全日客流" COLLATE NOCASE FROM "日客流"', "expression", "collate"),
('SELECT "日期", "全日客流" FROM json_each(\'[]\')', "unparsed", "table_shape"),
('SELECT "日期", "全日客流" FROM main."日客流"', "unparsed", "qualified_table"),
('SELECT x."日期", x."全日客流" FROM "日客流" t', "unparsed", "qualifier"),
]
PROTOTYPE_ACCEPTED = [
('SELECT "日期", "全日客流" FROM "日客流" ORDER BY "全日客流" DESC LIMIT 1', ["日期", "全日客流"],
("日客流", None, ["日期", "全日客流"])),
("SELECT 日期, 全日客流 FROM 日客流 WHERE 日期 = '2026-08-05';", ["日期", "全日客流"],
("日客流", None, ["日期", "全日客流"])),
('SELECT t."日期", t."全日客流" FROM "日客流" AS t WHERE substr(t."日期", 1, 7) = \'2026-08\'',
["日期", "全日客流"], ("日客流", "t", ["日期", "全日客流"])),
('SELECT * FROM "日客流" WHERE "日期" = \'2026-08-05\'', DAILY, ("日客流", None, DAILY)),
('-- FROM 别的表\nSELECT "日期", "客流" FROM "时段客流" /* JOIN x */ WHERE "时段" = \'8-9\'', ["日期", "客流"],
("时段客流", None, ["日期", "客流"])),
('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" = \'x UNION SELECT\'', ["日期", "全日客流"],
("日客流", None, ["日期", "全日客流"])),
]
def test_the_prototype_has_24_cases():
assert len(PROTOTYPE_REFUSED) + len(PROTOTYPE_ACCEPTED) == 24
@pytest.mark.parametrize(("sql", "code", "detail"), PROTOTYPE_REFUSED)
def test_prototype_refusals(sql, code, detail):
no(sql, code, detail)
@pytest.mark.parametrize(("sql", "result", "expect"), PROTOTYPE_ACCEPTED)
def test_prototype_acceptances(sql, result, expect):
out = ok(sql, result)
assert (out.table, out.alias, out.columns) == expect
@pytest.mark.parametrize(("sql", "result", "expect"), PROTOTYPE_ACCEPTED)
def test_prototype_acceptances_in_the_stored_form(sql, result, expect):
"""守卫去掉注释和尾分号之后(存进查询快照的形态)结论相同。"""
out = ok(stored(sql), result)
assert (out.table, out.alias, out.columns) == expect
# ---------------------------------------------------------------------------
# B7:多方言扫描看不见、SQLite 照样执行的集合运算(评审甲的 bs.py、br.py)
# ---------------------------------------------------------------------------
B7_COMPOUND = [
# bs.py 的 3 条:ESCAPE '\' 在多方言扫描里是「转义了的引号」,后面整段被当成字符串
"""SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" LIKE '2026-08%' ESCAPE '\\' UNION ALL """
"""SELECT "日期", "分区甲" FROM "日客流" WHERE "日期" LIKE '2026-09%'""",
"""SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" LIKE '%\\_%' ESCAPE '\\' OR 1 UNION ALL """
"""SELECT "日期", "分区甲" FROM "日客流" WHERE '1' = '1'""",
"""SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" = '\\' UNION ALL """
"""SELECT "日期", "分区甲" FROM "日客流" WHERE '1'='1'""",
# br.py 的第 1 条:方括号里含引号,多方言扫描不当标识符,[t'] … [u'] 被当成一段
"""SELECT "日期", "全日客流" FROM "日客流" AS [t'] WHERE "日期" > '2026-09' UNION ALL """
"""SELECT "日期", "分区甲" FROM "日客流" AS [u']""",
# br.py 的第 2 条
"""SELECT "日期", "全日客流" FROM "日客流" AS t ORDER BY [x'] UNION ALL """
"""SELECT "日期","分区甲" FROM "日客流" ORDER BY [y']""",
]
@pytest.mark.parametrize("sql", B7_COMPOUND)
def test_b7_compound_hidden_behind_backslash_or_bracket_is_refused(sql):
no(stored(sql), "multi_table", "compound")
def test_b7_join_hidden_behind_escape_backslash_is_refused():
no(stored("""SELECT a."日期", a."全日客流" LIKE '%' ESCAPE '\\', a."分区甲" FROM "日客流" a """
"""JOIN "日客流" b ON a."日期" = b."日期\""""), "multi_table", "join")
no(stored("""SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" LIKE '%' ESCAPE '\\' """
"""CROSS JOIN "时段客流\""""), "multi_table", "join")
def test_b7_comma_join_after_escape_backslash_is_refused():
# br.py 的第 3 条
no(stored("""SELECT a."日期", b."全日客流" FROM "日客流" a, "日客流" b WHERE a."日期" = b."日期" """
"""AND a."日期" LIKE '%' ESCAPE '\\' """), "multi_table", "comma_join")
def test_b7_would_slip_through_the_multi_dialect_scan():
"""反证:同一串 SQL 在多方言扫描(evidence._sql_scan 的读法)里看不见 UNION,判据换回那种读法就会放过它。
这条钉住「判据必须用 SQLite 读法」的理由,换读法的变异由上面两条抓。"""
from app.engine.evidence import _SQL, _sql_scan
code, _ = _sql_scan(stored(B7_COMPOUND[0]), _SQL[2])
assert "UNION" not in code.upper()
seen = " ".join(t.text for t in guard._tokens(stored(B7_COMPOUND[0]), guard._SQLITE) if t.kind == "code")
assert "UNION" in seen.upper()
# ---------------------------------------------------------------------------
# 守卫读法的三个性质(P4-SPEC 2.4):守卫以后改读法时这里先失败
# ---------------------------------------------------------------------------
def _kinds(sql: str) -> list[tuple[str, str]]:
return [(t.kind, t.text) for t in guard._tokens(sql, guard._SQLITE)]
def test_sqlite_reading_backslash_does_not_escape_in_strings():
assert _kinds("'a\\' x") == [("string", "'a\\'"), ("code", " x")]
def test_sqlite_reading_brackets_are_identifiers():
assert _kinds("[t'] x") == [("ident", "[t']"), ("code", " x")]
def test_sqlite_reading_brackets_end_at_the_first_closing_bracket():
assert _kinds("[a]]b]") == [("ident", "[a]"), ("code", "]b]")]
assert _kinds('[a""b]') == [("ident", '[a""b]')]
def test_bracket_names_are_taken_verbatim_and_doubled_quotes_collapse_elsewhere():
tables = {'a""b': ["x"], 'c"d': ["y"], "e`f": ["z"]}
assert ok('SELECT x FROM [a""b]', ["x"], tables=tables).table == 'a""b'
assert ok('SELECT y FROM "c""d"', ["y"], tables=tables).table == 'c"d'
assert ok("SELECT z FROM `e``f`", ["z"], tables=tables).table == "e`f"
# ---------------------------------------------------------------------------
# 字面量关键字、重名、位置对应
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("word", ["CURRENT_DATE", "current_time", "Current_Timestamp", "TRUE", "false", "NULL"])
def test_literal_keywords_are_refused(word):
no(f'SELECT {word}, "全日客流" FROM "日客流"', "expression", "literal_keyword")
no(f'SELECT "日期", {word} FROM "日客流"', "expression", "literal_keyword")
def test_literal_keywords_are_refused_even_when_the_table_has_such_a_column():
# TRUE / FALSE 有同名列时才指那一列:读法取决于表结构,一律拒
tables = {"t": ["true", "x"]}
no("SELECT TRUE, x FROM t", "expression", "literal_keyword", result=["true", "x"], tables=tables)
# 加了引号就是名字
assert ok('SELECT "true", x FROM t', ["true", "x"], tables=tables).columns == ["true", "x"]
def test_two_items_naming_the_same_column_are_ambiguous():
no('SELECT "日期", "日期" FROM "日客流"', "unparsed", "ambiguous_column", result=["日期", "日期"])
no("SELECT Key, key FROM abc", "unparsed", "ambiguous_column", result=["Key", "key"])
no('SELECT t."日期", "日期" FROM "日客流" t', "unparsed", "ambiguous_column", result=["日期", "日期"])
def test_result_columns_follow_the_items_by_position():
"""第 i 个结果列就是第 i 个选取项(P4-SPEC 2.4)。SQLite 对 `abc.key` 返回定义名 Key(附录 B2)。"""
out = ok("SELECT abc.key, ABC.V FROM abc", ["Key", "v"])
assert out.columns == ["Key", "v"]
out = ok('SELECT "分区乙", "日期", "分区甲" FROM "日客流"', ["分区乙", "日期", "分区甲"])
assert out.columns == ["分区乙", "日期", "分区甲"]
def test_result_columns_that_do_not_line_up_with_the_items_are_refused():
"""名字只作一致性核对:结果列的名字和对应位置的列对不上,或者个数不等,说明结果不是这条 SQL 的。
按名字对应(变异)会把第一条认成 ['分区甲', '日期']。"""
no('SELECT "日期", "分区甲" FROM "日客流"', "unparsed", "unknown_column", result=["分区甲", "日期"])
no('SELECT "日期" FROM "日客流"', "unparsed", "unknown_column", result=["日期", "全日客流"])
no('SELECT "日期", "全日客流" FROM "日客流"', "unparsed", "unknown_column", result=["日期"])
def test_star_must_match_the_table_columns_in_order():
assert ok('SELECT * FROM "日客流"', DAILY).columns == DAILY
# 结果列的大小写不同也认(name_key),返回的是定义名
assert ok("SELECT * FROM abc", ["key", "V"]).columns == ["Key", "v"]
no('SELECT * FROM "日客流"', "unparsed", "star_mismatch", result=["全日客流", "日期", "分区甲", "分区乙"])
no('SELECT * FROM "日客流"', "unparsed", "star_mismatch", result=DAILY[:3])
no('SELECT * FROM "日客流"', "unparsed", "star_mismatch", result=[*DAILY, "多一列"])
def test_star_is_only_allowed_alone():
no('SELECT *, "日期" FROM "日客流"', "unparsed", "star_mismatch", result=[*DAILY, "日期"])
no('SELECT "日期", * FROM "日客流"', "unparsed", "star_mismatch", result=["日期", *DAILY])
no('SELECT t.* FROM "日客流" t', "unparsed", "star_mismatch", result=DAILY)
# ---------------------------------------------------------------------------
# 名字:引号、中文、大小写、空白
# ---------------------------------------------------------------------------
def test_unquoted_chinese_names():
out = ok("SELECT 日期, 分区甲 FROM 日客流 WHERE 分区甲 > 3 ORDER BY 日期", ["日期", "分区甲"])
assert (out.table, out.columns) == ("日客流", ["日期", "分区甲"])
def test_backticks_and_brackets():
out = ok("SELECT `日期`, [全日客流] FROM [日客流] AS `t` WHERE [t].`日期` = '2026-08-01'", ["日期", "全日客流"])
assert (out.table, out.alias, out.columns) == ("日客流", "t", ["日期", "全日客流"])
def test_names_are_case_insensitive_only_for_ascii():
out = ok("SeLeCt ABC.KEY, v FrOm Abc", ["Key", "v"])
assert (out.table, out.columns) == ("abc", ["Key", "v"])
# 别名同样只对 ASCII 不区分大小写;返回的别名是原写法
out = ok('SELECT "T".日期 FROM 日客流 AS t', ["日期"])
assert out.alias == "t"
cyr = {"Дата": ["Сумма", "Итог"]}
no("SELECT Сумма FROM дата", "unparsed", "unknown_table", result=["Сумма"], tables=cyr)
no("SELECT сумма FROM Дата", "unparsed", "unknown_column", result=["сумма"], tables=cyr)
assert ok("SELECT Сумма, Итог FROM Дата", ["Сумма", "Итог"], tables=cyr).columns == ["Сумма", "Итог"]
def test_fullwidth_names_are_not_folded():
# name_key 不做 NFKC:全角 ABC 和 abc 是两张表
no("SELECT Key FROM abc", "unparsed", "unknown_table", result=["Key"])
def test_newlines_tabs_and_form_feeds_are_whitespace():
sql = 'SELECT\n\t"日期",\r\n\t"全日客流"\fFROM\t"日客流"\nWHERE "日期" = \'2026-08-01\'\n'
assert ok(sql, ["日期", "全日客流"]).columns == ["日期", "全日客流"]
def test_fullwidth_space_is_part_of_a_name():
"""除记号开头的 U+FEFF 之外,SQLite 把码位 ≥ 0x80 的字符一律算名字(2 版实体语法同样如此,evidence._SqlSyntax):
`日客流\u3000t` 是一张叫这个名字的表,不是表名加别名;`日客流\u3000WHERE` 也不是表名加 WHERE。"""
no("SELECT 日期, 全日客流 FROM 日客流\u3000t", "unparsed", "unknown_table")
no("SELECT 日期, 全日客流 FROM 日客流\u3000WHERE", "unparsed", "unknown_table")
# 「日客流\u3000WHERE」是表名,「日期」成了它的别名,后面的「=」不是尾部的开头
no("SELECT 日期, 全日客流 FROM 日客流\u3000WHERE 日期 = 'x'", "unparsed", "tail_shape")
tables = {"日客流": ["日期\u3000x", "全日客流"]}
assert ok("SELECT 日期\u3000x FROM 日客流", ["日期\u3000x"], tables=tables).columns == ["日期\u3000x"]
# 不换行空格同理:「日期\u00a0FROM」是一个名字,「日客流」成了它的别名,整条语句没有 FROM
no("SELECT 日期\u00a0FROM 日客流", "alias", "alias", result=["日客流"])
def test_vertical_tab_reads_as_whitespace():
"""竖制表符出现在记号开头时 SQLite 报非法记号,紧跟其他空白时 SQLite 当空白(CC_SPACE 的循环用 sqlite3Isspace,
含 \\v)。判据一律当空白:前一种执行不了、不会留下查询快照;后一种读法相同,藏在它后面的 UNION ALL 照样看得见。
当成标点时,紧跟在它后面的单引号表名 tables_in 会漏(见 tables_in 一节)。"""
no(stored('SELECT "日期", "全日客流" FROM "日客流" WHERE 1 \x0bUNION ALL SELECT "日期", "分区甲" FROM "日客流"'),
"multi_table", "compound")
no(stored('SELECT "日期", "全日客流" FROM "日客流" WHERE 1 \x0bGROUP BY "日期"'), "expression", "expression")
out = ok(stored('SELECT \x0b"日期", \x0b"全日客流" FROM \x0b"日客流" \x0bWHERE 1'), ["日期", "全日客流"])
assert (out.table, out.alias, out.columns) == ("日客流", None, ["日期", "全日客流"])
conn = sqlite3.connect(":memory:")
with pytest.raises(sqlite3.Error):
conn.execute("SELECT\x0b1")
assert conn.execute("SELECT 1 \x0bUNION ALL SELECT 2").fetchall() == [(1,), (2,)]
BOM = "\ufeff"
#: 评审实测(bom.py、attacks.py):守卫照样放行,SQLite 照样执行,前两条的全日客流一列里混着分区甲的值
BOM_HIDDEN = [
(f'SELECT "日期", "全日客流" FROM "日客流" WHERE 1 {BOM}UNION ALL {BOM}SELECT "日期", "分区甲" {BOM}FROM "日客流"',
"multi_table", "compound"),
(f'SELECT "日期", "全日客流" FROM "日客流" WHERE 1 {BOM}UNION {BOM}SELECT "日期", "分区甲" {BOM}FROM "日客流"',
"multi_table", "compound"),
(f'SELECT "日期", "全日客流" FROM "日客流" WHERE 1 {BOM}GROUP BY "日期"', "expression", "expression"),
]
@pytest.mark.parametrize(("sql", "code", "detail"), BOM_HIDDEN)
def test_bom_at_a_token_start_is_whitespace_so_the_keyword_behind_it_counts(sql, code, detail):
"""U+FEFF 出现在记号开头时 SQLite 当空白(tokenize.c 的 CC_BOM)。照「码位 ≥ 0x80 一律算名字」读,紧跟在它后面的
UNION 就成了一个普通的词,判据把这几条认成直接选取。"""
no(stored(sql), code, detail)
def test_bom_between_tokens_reads_like_a_space():
"""记号开头指:紧跟在空白、标点、带引号的名字、字符串后面。紧跟在不加引号的词后面就进了那个词(`SELECT\ufeff`
是一个名字,SQLite 报语法错误),见下一条。"""
sql = stored(f'SELECT {BOM}"日期",{BOM}"全日客流" {BOM}FROM {BOM}"日客流"{BOM} WHERE 1')
out = ok(sql, ["日期", "全日客流"])
assert (out.table, out.alias, out.columns) == ("日客流", None, ["日期", "全日客流"])
conn = sqlite3.connect(":memory:")
conn.execute('CREATE TABLE "日客流" ("日期", "全日客流")')
assert [d[0] for d in conn.execute(sql).description] == ["日期", "全日客流"]
with pytest.raises(sqlite3.Error):
conn.execute(f'SELECT{BOM} "日期" FROM "日客流"')
# 紧跟在带引号的名字、标点后面,单独一个 U+FEFF 也是记号开头
no(stored(f'SELECT "日期", "全日客流" FROM "日客流"{BOM}UNION ALL SELECT "日期", "分区甲" FROM "日客流"'),
"multi_table", "compound")
no(stored(f'SELECT "日期",{BOM}"分区甲"{BOM}AS "全日客流" FROM "日客流"'), "alias", "alias")
def test_bom_inside_a_name_is_part_of_the_name():
"""名字中间(含末尾)的 U+FEFF 是名字的一部分,同 SQLite:`日客流\ufeff` 是另一张表,`UNION\ufeff` 不是关键字。"""
tables = {"日客流": [f"日{BOM}期", "全日客流"]}
assert ok(f"SELECT 日{BOM}期 FROM 日客流", [f"日{BOM}期"], tables=tables).columns == [f"日{BOM}期"]
no(f"SELECT 全日客流 FROM 日客流{BOM}", "unparsed", "unknown_table", result=["全日客流"], tables=tables)
# 「日期\ufeffFROM」是一个名字,「日客流」成了它的别名,整条语句没有 FROM
no(f"SELECT 日期{BOM}FROM 日客流", "alias", "alias", result=["日客流"])
conn = sqlite3.connect(":memory:")
conn.execute(f'CREATE TABLE "日客流" ("日{BOM}期", "全日客流")')
conn.execute("INSERT INTO 日客流 VALUES (1, 2)")
assert conn.execute(f"SELECT 日{BOM}期 FROM 日客流").fetchall() == [(1,)]
with pytest.raises(sqlite3.Error):
conn.execute(f"SELECT 全日客流 FROM 日客流{BOM}")
# ---------------------------------------------------------------------------
# 语句个数、注释、字符串
# ---------------------------------------------------------------------------
def test_trailing_semicolons_are_allowed():
assert ok('SELECT "日期" FROM "日客流";', ["日期"]).columns == ["日期"]
assert ok('SELECT "日期" FROM "日客流" ; ; ', ["日期"]).columns == ["日期"]
def test_two_statements_are_refused():
no('SELECT "日期" FROM "日客流"; SELECT "日期" FROM "时段客流"', "unparsed", "multi_statement", result=["日期"])
no('SELECT "日期" FROM "日客流"; DROP TABLE x', "unparsed", "multi_statement", result=["日期"])
def test_keywords_inside_comments_and_strings_do_not_count():
sql = ('SELECT "日期", "全日客流" /* JOIN "时段客流" */ FROM "日客流" -- UNION ALL SELECT\n'
"WHERE \"日期\" <> 'UNION SELECT x FROM y JOIN z' AND \"日期\" <> 'GROUP BY'")
assert ok(sql, ["日期", "全日客流"]).columns == ["日期", "全日客流"]
def test_keywords_inside_quoted_names_do_not_count():
tables = {"union": ["select", "join"]}
out = ok('SELECT "select", [join] FROM "union"', ["select", "join"], tables=tables)
assert (out.table, out.columns) == ("union", ["select", "join"])
def test_comments_split_words_like_sqlite_does():
# SQLite 把注释当空白:SEL/**/ECT 是两个词,不是 SELECT
no('SEL/**/ECT "日期" FROM "日客流"', "unparsed", "not_select", result=["日期"])
def test_unterminated_quotes_are_refused():
no('SELECT "日期" FROM "日客流', "unparsed", "not_select")
# 反例:`"日期 FROM "` 是一个闭合的名字,后面的「日客流」是它的别名
no('SELECT "日期 FROM "日客流', "alias", "alias")
no("SELECT 日期 FROM 日客流 WHERE 日期 = 'x", "unparsed", "not_select")
no("SELECT [日期 FROM 日客流", "unparsed", "not_select")
def test_empty_and_non_select_statements_are_refused():
for sql in ("", " ", ";", "VALUES (1)", "PRAGMA table_info(日客流)", "EXPLAIN SELECT 日期 FROM 日客流",
"(SELECT 日期 FROM 日客流)"):
no(sql, "unparsed", "not_select")
# ---------------------------------------------------------------------------
# 选取项、FROM、尾部的其余写法
# ---------------------------------------------------------------------------
def test_where_subquery_and_values_are_refused():
no('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" IN (SELECT "日期" FROM "日客流")', "multi_table",
"subquery")
no('SELECT "日期", "全日客流" FROM "日客流" WHERE EXISTS (SELECT 1)', "multi_table", "subquery")
no('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" IN (VALUES (\'a\'))', "multi_table", "subquery")
no('SELECT "日期", (SELECT 1) FROM "日客流"', "multi_table", "subquery")
def test_join_family_words_are_refused_wherever_they_appear():
for word in ("JOIN", "NATURAL JOIN", "CROSS JOIN", "INNER JOIN", "LEFT JOIN", "LEFT OUTER JOIN", "RIGHT JOIN",
"FULL JOIN"):
no(f'SELECT a."日期", a."全日客流" FROM "日客流" a {word} "时段客流" b', "multi_table", "join")
def test_compound_and_cte_words_are_refused():
for word in ("UNION", "UNION ALL", "INTERSECT", "EXCEPT"):
no(f'SELECT "日期", "全日客流" FROM "日客流" {word} SELECT "日期", "分区甲" FROM "日客流"', "multi_table",
"compound")
no('WITH RECURSIVE x(n) AS (SELECT 1) SELECT "日期", "全日客流" FROM "日客流"', "multi_table", "cte")
no('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" IN (WITH x AS (SELECT 1) SELECT * FROM x)',
"multi_table", "cte")
def test_window_clause_is_refused():
no('SELECT "日期", "全日客流" FROM "日客流" WINDOW w AS (ORDER BY "日期")', "expression", "window")
def test_aggregation_in_the_tail_is_refused():
no('SELECT "日期", "全日客流" FROM "日客流" GROUP BY "日期"', "expression", "expression")
no('SELECT "日期", "全日客流" FROM "日客流" WHERE 1 GROUP BY "日期" HAVING 1', "expression", "expression")
def test_is_distinct_from_in_the_tail_is_outside_this_grammar():
no('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" IS DISTINCT FROM NULL', "unparsed", "tail_shape")
@pytest.mark.parametrize(("item", "detail"), [
('CASE WHEN 1 THEN "全日客流" END', "expression"),
('CAST("全日客流" AS TEXT)', "expression"),
('"全日客流" IS NULL', "expression"),
('"全日客流" BETWEEN 1 AND 2', "expression"),
('-"全日客流"', "expression"),
("+全日客流", "expression"),
("(全日客流)", "expression"),
("全日客流 || ''", "expression"),
("x'00'", "expression"),
("1", "expression"),
("1.5", "expression"),
('abs("全日客流")', "expression"),
('"abs"("全日客流")', "expression"),
("'全日客流'", "string_literal"),
('"全日客流" COLLATE BINARY', "collate"),
])
def test_items_that_are_not_bare_columns(item, detail):
no(f'SELECT "日期", {item} FROM "日客流"', "expression", detail)
@pytest.mark.parametrize("item", [
'"全日客流" AS "分区甲"', '"全日客流" AS x', '"全日客流" "分区甲"', "全日客流 分区甲", "\"全日客流\" '分区甲'",
'"全日客流" [分区甲]', '"全日客流" `分区甲`', "t.全日客流 x",
])
def test_renamed_items_are_aliases(item):
no(f'SELECT "日期", {item} FROM "日客流" t', "alias", "alias")
def test_qualifier_must_name_this_table():
assert ok('SELECT "日客流"."日期", 日客流.全日客流 FROM "日客流"', ["日期", "全日客流"]).alias is None
no('SELECT "时段客流"."日期", "全日客流" FROM "日客流"', "unparsed", "qualifier")
no('SELECT a."日期", "全日客流" FROM "日客流" AS b', "unparsed", "qualifier")
# 起了别名之后原表名不能再当限定名(SQLite 报 no such column),只认别名
no('SELECT 日客流."日期", "全日客流" FROM "日客流" AS t', "unparsed", "qualifier")
# 带库名的限定名
no('SELECT main."日客流"."日期", "全日客流" FROM "日客流"', "unparsed", "qualifier")
def test_table_shapes():
no('SELECT "日期", "全日客流" FROM temp."日客流"', "unparsed", "qualified_table")
no('SELECT "日期", "全日客流" FROM pragma_table_info(\'日客流\')', "unparsed", "table_shape")
# FROM (日客流) 在 SQLite 里能执行(括起来的表列表),一律不认
no('SELECT "日期", "全日客流" FROM ("日客流")', "multi_table", "subquery")
# 字符串当表名、当表别名:SQLite 的历史兼容,这一版文法不认
no("SELECT \"日期\", \"全日客流\" FROM '日客流'", "unparsed", "table_shape")
no("SELECT \"日期\", \"全日客流\" FROM \"日客流\" AS 't'", "unparsed", "table_shape")
no('SELECT "日期", "全日客流"', "unparsed", "table_shape")
no('SELECT "日期", "全日客流" FROM', "unparsed", "table_shape")
no('SELECT "日期", "全日客流" FROM WHERE', "unparsed", "table_shape")
def test_tail_shapes():
no('SELECT "日期", "全日客流" FROM "日客流" NOT INDEXED', "unparsed", "tail_shape")
no('SELECT "日期", "全日客流" FROM "日客流" t u', "unparsed", "tail_shape")
no("SELECT \"日期\", \"全日客流\" FROM \"日客流\" 't'", "unparsed", "tail_shape")
no('SELECT "日期", "全日客流" FROM "日客流" t, "时段客流" u', "multi_table", "comma_join")
no('SELECT "日期", "全日客流" FROM "日客流", "时段客流"', "multi_table", "comma_join")
def test_accepted_tails():
for tail in ("", " WHERE 1", " ORDER BY 2 DESC, 1", " LIMIT 3 OFFSET 1", " LIMIT 1, 2",
" WHERE \"日期\" BETWEEN '2026-08-01' AND '2026-08-31' ORDER BY \"日期\" COLLATE NOCASE LIMIT 5",
" WHERE \"日期\" LIKE '2026\\_%' ESCAPE '\\' AND \"分区甲\" IN (1, 2, 3)",
" WHERE CAST(\"分区甲\" AS TEXT) GLOB '1*' OR \"分区乙\" IS NULL",
" WHERE (CASE WHEN \"分区甲\" > 1 THEN 1 ELSE 0 END) = 1"):
out = ok(f'SELECT "日期", "全日客流" FROM "日客流"{tail}', ["日期", "全日客流"])
assert out.columns == ["日期", "全日客流"], tail
def test_select_all_is_allowed():
assert ok('SELECT ALL "日期" FROM "日客流"', ["日期"]).columns == ["日期"]
assert ok('SELECT ALL * FROM "日客流"', DAILY).columns == DAILY
def test_unknown_table_and_column():
no('SELECT "日期" FROM "不存在"', "unparsed", "unknown_table", result=["日期"])
no('SELECT "不存在" FROM "日客流"', "unparsed", "unknown_column", result=["不存在"])
# rowid 不是冻结表结构里的列
no('SELECT rowid, "日期" FROM "日客流"', "unparsed", "unknown_column", result=["rowid", "日期"])
def test_colliding_names_in_the_schema_are_treated_as_unknown():
# 冻结表结构里不该出现:说不清是哪一个,按找不到处理
no("SELECT x FROM T", "unparsed", "unknown_table", result=["x"], tables={"t": ["x"], "T": ["x"]})
no("SELECT x FROM t", "unparsed", "unknown_column", result=["x"], tables={"t": ["x", "X"]})
def test_forbidden_words_are_the_spec_table():
assert D.FORBIDDEN == {
"select", "from", "join", "union", "intersect", "except", "with", "over", "window", "group", "having",
"values", "natural", "cross", "inner", "left", "right", "full", "outer", "distinct", "recursive"}
# ---------------------------------------------------------------------------
# tables_in(P4-SPEC 2.4)
# ---------------------------------------------------------------------------
def test_tables_in_sees_tables_hidden_behind_escape_backslash():
sql = stored("""SELECT "日期", "客流" FROM "时段客流" WHERE "时段" LIKE '%' ESCAPE '\\' UNION ALL """
"""SELECT "日期", "客流" FROM "时段客流_表内合计\"""")
assert tables_in(sql, TABLES) == ["时段客流", "时段客流_表内合计"]
# 2 版 sql_tables 用多方言读法,这里漏掉了合计表:这正是 tables_in 存在的理由
from app.engine.evidence import sql_tables
assert "时段客流_表内合计" not in sql_tables(sql)
def test_tables_in_bracket_backtick_and_unquoted_chinese_names():
assert tables_in("SELECT * FROM [时段客流] JOIN `日客流` USING (日期) JOIN 时段客流_表内合计", TABLES) == [
"时段客流", "日客流", "时段客流_表内合计"]
assert tables_in("SELECT * FROM [t'] UNION ALL SELECT * FROM 日客流", {**TABLES, "t'": ["x"]}) == [
"t'", "日客流"]
def test_tables_in_ignores_names_only_in_strings_and_comments():
assert tables_in("SELECT * FROM 日客流 WHERE 分区 = '时段客流' -- 时段客流_表内合计", TABLES) == ["日客流"]
assert tables_in("SELECT '时段客流', \"日期\" FROM 日客流 /* abc */", TABLES) == ["日客流"]
def test_tables_in_counts_strings_where_sqlite_reads_them_as_table_names():
# `FROM '时段客流'` 在 SQLite 里能执行(历史兼容):读到的就是这张表
assert tables_in("SELECT 日期 FROM 日客流 UNION ALL SELECT 日期 FROM '时段客流'", TABLES) == [
"日客流", "时段客流"]
assert tables_in("SELECT a.日期 FROM 日客流 AS a, '时段客流' b", TABLES) == ["日客流", "时段客流"]
assert tables_in("SELECT 日期 FROM main.'时段客流_表内合计'", TABLES) == ["时段客流_表内合计"]
assert tables_in("SELECT 1 FROM 日客流 a JOIN '时段客流' b ON 1", TABLES) == ["日客流", "时段客流"]
def test_tables_in_counts_a_string_right_after_in():
"""SQLite 文法 `expr IN nm` 的 nm 可以是字符串:`日期 IN '单列表'` 读的是那张表(评审 attacks.py 的 in table string)。
括号里的字符串是值列表,不算。"""
tables = {**TABLES, "单列": ["日期"]}
assert tables_in("SELECT 日期, 全日客流 FROM 日客流 WHERE 日期 IN '单列'", tables) == ["日客流", "单列"]
assert tables_in("SELECT 日期 FROM 日客流 WHERE 日期 NOT IN '单列'", tables) == ["日客流", "单列"]
assert tables_in("SELECT 日期 FROM 日客流 WHERE 日期 IN ('单列', '时段客流')", tables) == ["日客流"]
def test_tables_in_sees_tables_behind_bom_and_vertical_tab():
"""「紧跟」按 SQLite 的分词算:记号开头的 U+FEFF、紧跟其他空白的竖制表符都是空白,不是记号(评审 bom.py、vt.py)。
读错了,紧跟在后面的合计表就漏掉,裁判拿不到「不要相加」的口径行。"""
head = 'SELECT "日期", "全日客流" FROM "日客流" UNION ALL SELECT "日期", "客流" '
expect = ["日客流", "时段客流_表内合计"]
assert tables_in(stored(f"{head}{BOM}FROM {BOM}时段客流_表内合计"), TABLES) == expect
assert tables_in(stored(f"{head}FROM {BOM}'时段客流_表内合计'"), TABLES) == expect
assert tables_in(stored(f"{head}FROM \x0b'时段客流_表内合计'"), TABLES) == expect
assert tables_in(stored(f"{head}FROM main.{BOM}'时段客流_表内合计'"), TABLES) == expect
tables = {**TABLES, "单列": ["日期"]}
assert tables_in(stored(f"SELECT 日期 FROM 日客流 WHERE 日期 IN {BOM}'单列'"), tables) == ["日客流", "单列"]
# 名字中间的 U+FEFF 是名字的一部分:「时段客流_表内合计\ufeff」不是那张表,SQLite 也报找不到
assert tables_in(f"{head}FROM 时段客流_表内合计{BOM}", TABLES) == ["日客流"]
def test_tables_in_is_ordered_deduplicated_and_returns_schema_names():
assert tables_in("select * from ABC, 日客流, abc, \"Abc\"", TABLES) == ["abc", "日客流"]
assert tables_in("SELECT 1", TABLES) == []
assert tables_in("", TABLES) == []
def test_tables_in_errs_on_the_side_of_more():
# 列名恰好和某张表同名时多出这张表:后果只是多一行说明
tables = {"日客流": ["日期", "时段客流"], "时段客流": ["日期"]}
assert tables_in('SELECT "时段客流" FROM "日客流"', tables) == ["时段客流", "日客流"]
# 引号没闭合也照样认能切出来的部分
assert tables_in('SELECT * FROM 日客流 WHERE x = "时段客流', TABLES) == ["日客流", "时段客流"]
def test_tables_in_keeps_fullwidth_space_inside_the_name():
assert tables_in("SELECT * FROM 日客流\u3000WHERE 1", TABLES) == []
assert tables_in("SELECT * FROM 日客流 WHERE 1", TABLES) == ["日客流"]
# ---------------------------------------------------------------------------
# 模块边界、不抛异常
# ---------------------------------------------------------------------------
def test_module_imports_only_guard_names_and_the_contract():
"""P4-SPEC 7.2:判据是纯函数,只许导入守卫、names 和 provenance_types(加标准库),不依赖证据层、接口层。"""
tree = ast.parse(Path(D.__file__).read_text(encoding="utf-8"))
imported: set[str] = set()
for node in ast.walk(tree):
if isinstance(node, ast.Import):
imported |= {a.name for a in node.names}
elif isinstance(node, ast.ImportFrom):
imported.add(node.module or "")
assert imported <= {"__future__", "collections.abc", "dataclasses", "app.data.guard", "app.data.names",
"app.data.provenance_types"}, imported
_PIECES = [
"SELECT", "FROM", "WHERE", "ORDER", "BY", "LIMIT", "AS", "JOIN", "UNION", "ALL", "DISTINCT", "*", ",", ".", "(",
")", ";", "'", '"', "[", "]", "`", "\\", "--", "/*", "*/", "日期", "全日客流", '"日客流"', "日客流", "t", "x'0'",
"1", "NULL", "\u3000", "\n", " ", "COLLATE", "OVER", "abc", "KEY", "=", "'a''b'", '"a""b"', "[t']", "ESCAPE",
]
def test_recognize_never_raises_and_every_refusal_is_mapped():
rng = random.Random(20261002)
for _ in range(3000):
sql = " ".join(rng.choice(_PIECES) for _ in range(rng.randint(0, 14)))
if rng.random() < 0.5:
sql = "SELECT " + sql
result = rng.choice([["日期"], ["日期", "全日客流"], DAILY, [], ["Key", "v"]])
out = recognize(sql, result_columns=result, tables=TABLES)
assert isinstance(out, (DirectSelect, SelectRefusal)), sql
if isinstance(out, SelectRefusal):
assert out.detail in P.DETAILS[out.code], (sql, out)
else:
assert out.table in TABLES and all(c in TABLES[out.table] for c in out.columns), (sql, out)
assert isinstance(tables_in(sql, TABLES), list)
# ---------------------------------------------------------------------------
# 拿真的 SQLite 核对:认可的每一列,值都来自它说的那一列
# ---------------------------------------------------------------------------
_DB_TABLES = {
"日客流": ["日期", "全日客流", "分区甲", "分区乙"],
"时段客流": ["日期", "时段", "客流"],
"abc": ["Key", "v"],
}
def _database(rng: random.Random, tables: dict[str, list[str]] | None = None
) -> tuple[sqlite3.Connection, dict[object, tuple[str, str]]]:
"""每一格的值全库唯一:结果里的任何一个值都能反查出它来自哪张表的哪一列。DQS 关掉,同 engine。"""
conn = sqlite3.connect(":memory:")
if hasattr(conn, "setconfig"):
conn.setconfig(sqlite3.SQLITE_DBCONFIG_DQS_DML, False)
conn.setconfig(sqlite3.SQLITE_DBCONFIG_DQS_DDL, False)
origin: dict[object, tuple[str, str]] = {}
values = iter(rng.sample(range(100_000, 999_999), 200))
for table, cols in (tables or _DB_TABLES).items():
quoted = ", ".join(f'"{c}"' for c in cols)
conn.execute(f'CREATE TABLE "{table}" ({quoted})')
for day in range(1, 6):
row = []
for c in cols:
v: object = f"2026-08-{day:02d}#{table}" if c == "日期" else next(values)
origin[v] = (table, c)
row.append(v)
conn.execute(f'INSERT INTO "{table}" VALUES ({", ".join("?" * len(cols))})', row)
return conn, origin
_SELECTS = [
'"日期", "全日客流"', '"日期", "分区甲"', "日期, 分区乙", 't."日期", t."全日客流"', "*", '"全日客流"',
'"日期", "分区甲" AS "全日客流"', '"日期" "全日客流"', '"分区甲" "全日客流", "日期"', '"日期", "全日客流" + 0',
'"日期", 分区甲 AS 全日客流', 'T."日期", "全日客流"', '"全日客流", "日期"', "[日期], `全日客流`",
'"日期", (SELECT "分区甲")', '"日期", "全日客流" COLLATE NOCASE', '"日期", coalesce("全日客流", 0)',
]
_FROMS = [
'"日客流"', '"日客流" t', '"日客流" AS t', "日客流 AS [t']", '"日客流" AS "T"', '"日客流" AS t, "日客流" u',
'"日客流" t JOIN "日客流" u USING ("日期")', '(SELECT "日期", "分区甲" AS "全日客流", "分区乙", "分区甲" FROM "日客流")',
'"日客流" NOT INDEXED', "'日客流'",
]
_TAILS = [
"", " WHERE 1", " ORDER BY 1 DESC", " LIMIT 2", " WHERE \"日期\" LIKE '%' ESCAPE '\\'",
" WHERE \"日期\" = '\\'", " ORDER BY [x']", " WHERE '\\' = '\\'",
""" UNION ALL SELECT "日期", "分区甲" FROM "日客流\"""",
""" WHERE "日期" LIKE '%' ESCAPE '\\' UNION ALL SELECT "日期", "分区甲" FROM "日客流\"""",
""" AS [u'] UNION ALL SELECT "日期", "分区乙" FROM "日客流" AS [v']""",
""" UNION SELECT "日期", "分区甲" FROM "日客流\"""",
""" EXCEPT SELECT "日期", "分区甲" FROM "日客流" WHERE 0""",
""" WHERE "日期" IN (SELECT "日期" FROM "时段客流")""",
" GROUP BY \"日期\"", " /* UNION ALL SELECT 1 */", " -- JOIN x",
]
def _tables_read(conn: sqlite3.Connection, stmt: str) -> set[str]:
"""SQLite 的授权回调报出的、这条语句读到的表:tables_in 必须一张不漏(宁可多认)。"""
read: set[str] = set()
def authorize(action: int, table: str | None, *_: object) -> int:
if action == sqlite3.SQLITE_READ and table and not table.startswith("sqlite_"):
read.add(table)
return sqlite3.SQLITE_OK
conn.set_authorizer(authorize)
try:
conn.execute(stmt).fetchall()
finally:
conn.set_authorizer(None)
return read
def _check_accepted_values(out: DirectSelect, rows: list[tuple], origin: dict[object, tuple[str, str]],
stmt: str) -> None:
for row in rows:
for i, value in enumerate(row):
assert origin.get(value) == (out.table, out.columns[i]), (
f"{stmt!r}:第 {i} 列认作 {out.table}.{out.columns[i]},值 {value!r} 实际来自 {origin.get(value)}")
def test_every_accepted_column_really_comes_from_the_column_it_names():
"""随机拼 SQL,经守卫、在 SQLite 上真的执行;判据认可的,结果第 i 列的每个值都必须来自 DirectSelect 说的那张表的
第 i 列。B7 那类写法(集合运算拼进另一列的值)一旦被放过,这里就会报出值的真实来处。授权回调报出的表,
tables_in 也必须一张不漏。"""
rng = random.Random(4242)
conn, origin = _database(rng)
executed = accepted = 0
combos = list(itertools.product(_SELECTS, _FROMS, _TAILS))
rng.shuffle(combos)
for select, source, tail in combos:
sql = f"SELECT {select} FROM {source}{tail}"
try:
stmt = stored(sql)
cur = conn.execute(stmt)
except (guard.SqlRejected, sqlite3.Error):
continue
executed += 1
columns = [d[0] for d in cur.description]
rows = cur.fetchall()
read = _tables_read(conn, stmt)
assert read <= set(tables_in(stmt, _DB_TABLES)), (stmt, read)
out = recognize(stmt, result_columns=columns, tables=_DB_TABLES)
if isinstance(out, SelectRefusal):
assert out.detail in P.DETAILS[out.code], (stmt, out)
continue
accepted += 1
assert len(out.columns) == len(columns), stmt
_check_accepted_values(out, rows, origin, stmt)
# 拼出来的大多数都能执行,其中有认可的、也有被拒的:这条测试确实在比对
assert executed > 500 and 50 < accepted < executed, (executed, accepted)
def test_b7_writings_really_return_another_columns_values_in_sqlite():
"""反证上一条的比对有牙齿:B7 的写法在 SQLite 里确实把分区甲的值放进了全日客流那一列。"""
rng = random.Random(7)
conn, origin = _database(rng)
for sql in B7_COMPOUND[:1] + B7_COMPOUND[3:4]:
rows = conn.execute(stored(sql).replace("'2026-08%'", "'%'").replace("'2026-09%'", "'%'")
.replace("> '2026-09'", "> ''")).fetchall()
assert {origin[r[1]] for r in rows} == {("日客流", "全日客流"), ("日客流", "分区甲")}, sql
#: 记号之间的间隔:把模板里的 {g} 换成它。紧跟在关键字、数字后面的、紧跟在带引号的名字和标点后面的都有,
#: 记号开头和空白后面两种位置都覆盖到
_GAP_TEMPLATES = [
'SELECT "日期", "全日客流" FROM "日客流" WHERE "分区甲" > 0{g}UNION ALL{g}SELECT "日期", "分区甲"{g}FROM "日客流"',
'SELECT "日期", "全日客流" FROM "日客流" WHERE "分区甲" > 0{g}UNION{g}SELECT "日期", "分区甲"{g}FROM "日客流"',
# 限定名:间隔要是被 SQLite 读成了表别名,「"日客流"."日期"」就找不到列,不会被误当成空白
'SELECT "日客流"."日期", "全日客流" FROM "日客流"{g}UNION ALL SELECT "日期", "分区乙" FROM "日客流"',
'SELECT "日期", "全日客流" FROM "日客流" WHERE "分区甲" > 0{g}GROUP BY "日期"',
'SELECT "日期", "全日客流" FROM "日客流" UNION ALL SELECT "日期", "客流" FROM{g}时段客流',
"SELECT \"日期\", \"全日客流\" FROM \"日客流\" UNION ALL SELECT \"日期\", \"客流\" FROM{g}'时段客流'",
"SELECT \"日期\", \"全日客流\" FROM \"日客流\" WHERE \"全日客流\" NOT IN{g}'单列'",
"SELECT{g}\"日期\",{g}\"全日客流\"{g}FROM{g}\"日客流\"{g}WHERE{g}\"日期\" <> '2026-08-01#日客流'",
]
#: 候选字符:ASCII 控制字符和空格、str.isspace 认而 SQLite 不认的、U+FEFF 及其邻居
_GAP_CHARS = [chr(c) for c in (*range(0x01, 0x21), 0x7F, 0x85, 0xA0, 0x1680, 0x200B, 0x2028, 0x2029, 0x3000, 0xFEFF,
0xFFFE)]
#: SQLite 当空白的:记号开头只有这六个;竖制表符只在紧跟其他空白时才是
_SQLITE_SPACE = "\t\n\x0c\r \ufeff"
def test_gaps_that_sqlite_reads_as_whitespace_read_the_same_here():
"""拿真的 SQLite 核对分词:把记号之间的空格换成各种字符,SQLite 能执行的每一条——
- 认可的,值都来自它说的那一列;授权回调报出的表 tables_in 一张不漏;
- 结果、列名、字节码都和用空格的那条相同(SQLite 把它当空白读)时,判据的结论和 tables_in 也必须相同。
最后核对「SQLite 当空白的间隔」恰好是 U+FEFF、五个 ASCII 空白和空白后面的竖制表符:SQLite 升级改了读法,
这里先失败。评审的 U+FEFF、竖制表符两处都是这条能抓到的。"""
tables = {**_DB_TABLES, "单列": ["日期"]}
conn, origin = _database(random.Random(5), tables)
gaps = sorted({*_GAP_CHARS, *(" " + c for c in _GAP_CHARS), *(c + " " for c in _GAP_CHARS)})
def run(stmt: str) -> tuple[list[str], list[tuple], list[tuple]]:
cur = conn.execute(stmt)
columns, rows = [d[0] for d in cur.description], cur.fetchall()
# EXPLAIN 的最后一列是注释,会带出原文片段,只比操作码和五个操作数
program = [tuple(r[1:7]) for r in conn.execute("EXPLAIN " + stmt).fetchall()]
return columns, rows, program
whitespace: set[str] = set()
executed_with: dict[str, int] = {}
for g in gaps:
executed = same = 0
for template in _GAP_TEMPLATES:
base_stmt = stored(template.format(g=" "))
try:
stmt = stored(template.format(g=g))
columns, rows, program = run(stmt)
except (guard.SqlRejected, sqlite3.Error):
continue
executed += 1
out = recognize(stmt, result_columns=columns, tables=tables)
if isinstance(out, DirectSelect):
_check_accepted_values(out, rows, origin, stmt)
found = tables_in(stmt, tables)
assert _tables_read(conn, stmt) <= set(found), (g, stmt, found)
if (columns, rows, program) == run(base_stmt):
same += 1
base = recognize(base_stmt, result_columns=columns, tables=tables)
assert out == base, f"{g!r}:{stmt!r} → {out!r},用空格时 → {base!r}"
assert found == tables_in(base_stmt, tables), (g, stmt)
executed_with[g] = executed
if executed and same == executed:
whitespace.add(g)
expected = {c for c in _SQLITE_SPACE} | {" " + c for c in _SQLITE_SPACE + "\x0b"} | {c + " " for c in _SQLITE_SPACE}
assert whitespace == expected, (sorted(map(repr, whitespace - expected)), sorted(map(repr, expected - whitespace)))
# 有牙齿:空白后面的 U+FEFF、竖制表符在每个模板里都执行了(含藏 UNION ALL、GROUP BY 的那几条)
assert executed_with[" " + BOM] == executed_with[" \x0b"] == len(_GAP_TEMPLATES)
+284
View File
@@ -0,0 +1,284 @@
"""裁判摘录的期 4 来历行(P4-SPEC 3.2–3.4):按文档标记分版本、只对上传表格。
- 期 4 之前的文档(没有标记):摘录逐字等于金样(tests/fixtures/evidence/pre_p4_golden.json),provenance、
direct_select 两个模块一次都不碰(用桩钉住:桩一被访问就抛错);
- 带标记的文档:手工源的摘录不变;上传源的查询摘录只在 SQL 那一行之后、「列:」之前多出 judge_lines 的行,
去掉这些行就等于金样;同样的输入两次得到同样的字;
- 按需裁判的 loader:带标记的文档才取得到哈希链上的清单,老文档连坏掉的清单都碰不到(A26);
- 真导入的 D28(表下有含数字的说明):区域外文字带限定语、带统计期;遮罩时不送;单句的点击估价在上限以内。
金样的工件逐个 put_json 进临时数据目录,先断言返回的 id 等于金样记下的 id(内容寻址,对不上说明金样被改过)。
夹具全部合成、假名;不调用任何模型(只比摘录文字和估价)。
"""
from __future__ import annotations
import copy
import json
import sys
import types
from pathlib import Path
import pytest
from httpx import ASGITransport, AsyncClient
from app.api import evidence as api
from app.core import artifact_store
from app.core.config import settings
from app.data import provenance as prov
from app.data.provenance_types import DOC_PROVENANCE, PROV_OUTSIDE_CHARS
from app.data.tabular import UNSHAPED_NOTE
from app.engine import judge
from app.engine.direct_select import tables_in
from app.main import app
from tests.test_provenance_api import import_flow, run_sql, sealed_of
GOLDEN = Path(__file__).parent / "fixtures" / "evidence" / "pre_p4_golden.json"
#: 3.2 区域外文字的小标题(限定语每块只写一次)
OUTSIDE_HEAD = ("区域外文字(原表区域外的文字原文,未经系统核对;其中的数字不是查询结果,不能用来支持或否定报告中的"
"数字):")
# --------------------------------------------------------------------------
# 夹具
# --------------------------------------------------------------------------
@pytest.fixture
async def golden(monkeypatch, tmp_path) -> dict:
"""金样的输入工件写进一个临时数据目录(id 必须等于金样记下的),返回金样本身。"""
data = tmp_path / "data"
(data / "uploads").mkdir(parents=True)
monkeypatch.setattr(settings, "data_dir", data)
g = json.loads(GOLDEN.read_text("utf-8"))
for aid, content in g["artifacts"].items():
assert await artifact_store.put_json(content, kind="golden") == aid
return g
class _Forbidden(types.ModuleType):
"""桩模块:任何属性一被访问就记下来并抛错。期 4 之前的文档、手工源的摘录不该碰 provenance、direct_select。"""
def __init__(self, name: str, touched: list[str]) -> None:
super().__init__(name)
self._touched = touched
def __getattr__(self, attr: str):
if attr.startswith("__"):
raise AttributeError(attr)
self._touched.append(f"{self.__name__}.{attr}")
raise AssertionError(f"不该用到 {self.__name__}.{attr}")
@pytest.fixture
def forbidden(monkeypatch) -> list[str]:
touched: list[str] = []
for name in ("app.data.provenance", "app.engine.direct_select"):
monkeypatch.setitem(sys.modules, name, _Forbidden(name, touched))
return touched
def doc_of(g: dict, kind: str, *, marked: bool) -> tuple[dict, dict]:
report = g["reports"][kind]
doc = copy.deepcopy(report["outputs"]["doc"])
assert "provenance" not in doc
if marked:
doc["provenance"] = DOC_PROVENANCE
return doc, copy.deepcopy(report["catalog"])
def excerpts(doc: dict, catalog: dict, **kw) -> dict[str, str]:
return judge.prepare(doc, catalog, loader=kw.pop("loader", artifact_store.load), **kw).excerpts
def inserted(new: str, old: str) -> list[str]:
"""新摘录比金样多出的行:只许插在第 2 行(SQL)之后、「列:」之前,其余逐字相同。"""
a, b = new.split("\n"), old.split("\n")
extra = len(a) - len(b)
assert extra >= 0 and a[:2] == b[:2] and a[2 + extra:] == b[2:], (new, old)
assert b[2].startswith("列:")
return a[2:2 + extra]
# --------------------------------------------------------------------------
# 按文档标记分版本(3.3)
# --------------------------------------------------------------------------
@pytest.mark.parametrize("kind", ["recipe", "simple", "manual"])
async def test_unmarked_documents_are_unchanged_and_never_touch_the_new_modules(golden, forbidden, kind):
doc, catalog = doc_of(golden, kind, marked=False)
assert excerpts(doc, catalog) == golden["reports"][kind]["outputs"]["excerpts"]
assert forbidden == []
async def test_marked_manual_source_is_unchanged(golden, forbidden):
"""非上传源(查询快照没有 data_version):带标记也一字不差,两个模块照样不碰。"""
doc, catalog = doc_of(golden, "manual", marked=True)
assert excerpts(doc, catalog) == golden["reports"]["manual"]["outputs"]["excerpts"]
assert forbidden == []
async def test_marked_recipe_source_adds_lines_only_between_sql_and_columns(golden):
doc, catalog = doc_of(golden, "recipe", marked=True)
new, old = excerpts(doc, catalog), golden["reports"]["recipe"]["outputs"]["excerpts"]
assert set(new) == set(old)
# 表名实体的摘录不加来历(只改查询摘录)
assert new["t:日客流"] == old["t:日客流"]
for alias, table in (("Q1", "日客流"), ("Q2", "时段客流")):
lines = inserted(new[alias], old[alias])
snap = artifact_store.load(catalog[alias]["artifact"])
schema = artifact_store.load(snap["schema_artifact"])
tables = {t: [c["name"] for c in meta["columns"]] for t, meta in schema["tables"].items()}
assert tables_in(snap["sql"], tables) == [table]
assert lines == prov.judge_lines(snap, schema, artifact_store.load, tables=[table], hidden=set())
assert lines[0].startswith("来源:上传的表格,每期替换,第 1 次导入:统计期 2026-08-01 至 2026-08-31")
assert f"口径:表 {table} 的说明:" in "\n".join(lines)
async def test_marked_simple_upload_adds_only_caliber_lines(golden):
"""简单导入(没有导入清单)只多口径行(调整 6):未规整的表写 UNSHAPED_NOTE;没有来源、接受、区域外文字。"""
doc, catalog = doc_of(golden, "simple", marked=True)
new, old = excerpts(doc, catalog), golden["reports"]["simple"]["outputs"]["excerpts"]
lines = inserted(new["Q1"], old["Q1"])
assert lines and all(line.startswith("口径:") for line in lines)
assert f"口径:表 月报 的说明:{UNSHAPED_NOTE}" in lines
assert new["t:月报"] == old["t:月报"]
async def test_excerpts_are_deterministic(golden):
"""摘录进断点指纹和逐句复用键:同样的输入两次得到同样的字(不取当前时间、不取数据库列)。"""
doc, catalog = doc_of(golden, "recipe", marked=True)
assert excerpts(doc, catalog) == excerpts(copy.deepcopy(doc), copy.deepcopy(catalog))
async def test_masks_and_tables_are_handed_to_judge_lines(golden, monkeypatch):
"""hidden 就是查询摘录现成的那份遮罩(数据源现在设的 ∪ 快照记下的,小写);tables 由 tables_in 按冻结表结构算。"""
calls: list[dict] = []
real = prov.judge_lines
def spy(query, schema, load, *, tables, hidden):
calls.append({"sql": query["sql"], "tables": tables, "hidden": set(hidden), "schema": schema is not None})
return real(query, schema, load, tables=tables, hidden=hidden)
monkeypatch.setattr(prov, "judge_lines", spy)
doc, catalog = doc_of(golden, "recipe", marked=True)
out = excerpts(doc, catalog, masked={"flow_demo": ["分区甲"], "别的源": ["日期"]})
assert [(c["tables"], c["hidden"], c["schema"]) for c in calls] == [
(["日客流"], {"分区甲"}, True), (["时段客流"], {"分区甲"}, True)]
# 遮罩的列:结果里不给,列的说明也不写
assert "口径:列 分区甲 的说明" not in out["Q1"] and "分区甲" not in out["Q1"].split("\n")[-1]
async def test_a_broken_artifact_degrades_to_one_line(golden, monkeypatch):
"""生成来历行时出了意外(工件形状不对):这条查询只写「导入清单无法读取…」一行,整份报告照样能判。"""
def boom(*_a, **_kw):
raise TypeError("坏形状")
monkeypatch.setattr(prov, "judge_lines", boom)
doc, catalog = doc_of(golden, "recipe", marked=True)
new, old = excerpts(doc, catalog), golden["reports"]["recipe"]["outputs"]["excerpts"]
assert inserted(new["Q1"], old["Q1"]) == [prov.SOURCE_BROKEN]
# --------------------------------------------------------------------------
# 按需裁判的取证范围(3.4、2.8.3)
# --------------------------------------------------------------------------
def golden_sealed(g: dict, kind: str) -> api._Sealed:
info = g["reports"][kind]["sealed"]
sealed = sealed_of(queries=list(info["queries"]), schemas=list(info["schemas"]))
sealed.queries.update(info["queries"])
return sealed
async def test_on_demand_loader_reaches_the_chain_only_for_marked_docs(golden):
doc, catalog = doc_of(golden, "recipe", marked=True)
sealed = golden_sealed(golden, "recipe")
old = golden["reports"]["recipe"]["outputs"]["excerpts"]
# 带标记:loader 扩展到表结构快照经哈希链列出的导入清单,来源行照常写
chained = excerpts(doc, catalog, loader=api._sealed_loader(sealed, chain=True))
assert inserted(chained["Q1"], old["Q1"])[0].startswith("来源:上传的表格,每期替换")
assert chained == excerpts(doc, catalog)
# 不扩展的 loader 取不到清单:只写「导入清单无法读取…」一行,口径照写(来自封存范围内的表结构快照)
plain = excerpts(doc, catalog, loader=api._sealed_loader(sealed))
lines = inserted(plain["Q1"], old["Q1"])
assert lines[0] == prov.SOURCE_BROKEN and any(line.startswith("口径:表 日客流") for line in lines)
async def test_pre_p4_on_demand_judging_never_reads_a_broken_manifest(golden, forbidden):
"""A26:老文档(chain 为假)+ 表结构快照列出的导入清单文件被改坏:摘录与金样逐字相同,根本不去取清单。"""
report = golden["reports"]["recipe"]
snap = artifact_store.load(report["catalog"]["Q1"]["artifact"])
[mid] = artifact_store.load(snap["schema_artifact"])["import_manifests"]
path = settings.data_dir / "artifacts" / mid[:2] / f"{mid}.json"
path.write_text(path.read_text("utf-8").replace('"seq":1', '"seq":9', 1), encoding="utf-8")
with pytest.raises(ValueError):
artifact_store.load(mid)
doc, catalog = doc_of(golden, "recipe", marked=False)
sealed = golden_sealed(golden, "recipe")
loader = api._sealed_loader(sealed, chain=doc.get("provenance") == DOC_PROVENANCE)
assert excerpts(doc, catalog, loader=loader) == report["outputs"]["excerpts"]
assert forbidden == []
# --------------------------------------------------------------------------
# 真导入:区域外文字、遮罩、估价(3.2)
# --------------------------------------------------------------------------
@pytest.fixture
async def client():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
yield c
@pytest.fixture
def no_model(monkeypatch, tmp_path):
from app.data import recipe_ai
from app.data.recipe_types import AiAvailability
data = tmp_path / "data"
(data / "uploads").mkdir(parents=True)
monkeypatch.setattr(settings, "data_dir", data)
async def none(session):
return None, "", AiAvailability(False, reason="测试不接模型")
monkeypatch.setattr(recipe_ai, "resolve_draft_model", none)
async def test_outside_text_masks_and_cost_on_a_real_import(no_model, client):
"""D28(9 月,表下 B32 有一句含数字的说明,新导入的清单记了 hidden=False):摘录送全文、带限定语和统计期;
数据源设遮罩时一格都不送、只写条数;只判一句、引用这一条查询的点击估价不超过 0.01 美元。"""
from app.data.engine import engines
from tests.fixtures.xlsx.drift import SEP
up = await import_flow(client, seed=4201, start=SEP, days=30, variant="D28")
try:
qid, _sid = await run_sql(client, up, 'SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" = \'2026-09-15\'')
g = json.loads(GOLDEN.read_text("utf-8"))
doc = copy.deepcopy(g["reports"]["recipe"]["outputs"]["doc"])
doc["provenance"] = DOC_PROVENANCE
catalog = {"Q1": {**g["reports"]["recipe"]["catalog"]["Q1"], "artifact": qid, "source": up.name,
"tool": f"db_query__{up.name}"}}
request = judge.prepare(doc, catalog)
text = request.excerpts["Q1"]
lines = text.split("\n")
assert lines[2].startswith("来源:上传的表格,每期替换,第 1 次导入:统计期 2026-09-01 至 2026-09-30")
assert OUTSIDE_HEAD in lines
note = next(line for line in lines if "B32:" in line)
assert note.startswith(" 第 1 次导入(统计期 2026-09-01 至 2026-09-30)客流汇总!B32:「") and "9月15日" in note
assert len(note) < PROV_OUTSIDE_CHARS + 60
assert judge.prepare(doc, catalog).excerpts == request.excerpts # 确定
# 遮罩:区域外文字一格都不送,只写条数
masked = judge.prepare(doc, catalog, masked={up.name: ["分区甲"]}).excerpts["Q1"]
assert "9月15日" not in masked and OUTSIDE_HEAD not in masked
assert "(数据源设置了遮罩,未列出)" in masked
# 估价:只判一句、引用这一条查询(claude-sonnet-5 的目录价)
one = [c for c in request.cands if "Q1" in c.aliases][:1]
assert one
cost = request.cost("claude-sonnet-5", one)
assert cost is not None and cost <= 0.01, cost
finally:
await engines.invalidate(up.source_id)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+819
View File
@@ -0,0 +1,819 @@
"""期 4 证据下钻的契约(P4-SPEC 7.1,WP-0):provenance_types 的常量、取值、原文、响应形状和中间结构。
并行的几个工作包看不到彼此的代码,只靠这份契约对齐。这里钉住的是「改了就会合错」的东西:字段名、顺序、缺省值、
asdict 之后的 JSON 键(逐层对 P4-SPEC 2.8.1)、原文覆盖每个 code、原文守文案规范。另外两份 WP-0 交付物也在这里
做最低限度的自检:前端夹具里每一份推断来源的答复都能按契约读回、满足不变式;金样的工件 id 等于内容哈希。
"""
from __future__ import annotations
import ast
import dataclasses
import json
import re
import types
import typing
from dataclasses import MISSING, asdict
from pathlib import Path
from typing import Any, get_args, get_origin
import pydantic
import pytest
from app.data import provenance_types as P
from app.data import recipe_types as T
ROOT = Path(__file__).resolve().parents[2]
FIXTURE = ROOT / "frontend" / "src" / "run" / "__tests__" / "evidence-provenance.json"
GOLDEN = Path(__file__).resolve().parent / "fixtures" / "evidence" / "pre_p4_golden.json"
# ---------------------------------------------------------------------------
# 常量与取值
# ---------------------------------------------------------------------------
def test_constants():
assert P.DOC_PROVENANCE == 1
assert P.SCHEMA == "agentlab.provenance/1"
assert (P.PROV_PARTS, P.PROV_ACCEPT, P.PROV_OUTSIDE, P.PROV_OUTSIDE_CHARS, P.PROV_COLUMN_NOTES, P.PROV_CHARS) == (
3, 4, 4, 200, 6, 1200)
def test_literals():
assert get_args(P.Status) == ("inferred", "table_only", "none")
assert set(get_args(P.ReasonCode)) == {
"legacy_doc", "not_cell", "not_sealed", "not_upload", "simple_upload", "manifest_unreadable",
"chain_mismatch", "expression", "alias", "multi_table", "unparsed", "no_pk", "pk_missing", "masked",
"null_value", "null_pk", "snapshot_gone", "db_tampered", "recheck_missing", "recheck_multiple",
"recheck_mismatch", "no_lineage"}
assert len(get_args(P.ReasonCode)) == len(set(get_args(P.ReasonCode)))
assert set(get_args(P.AlertCode)) == {"db_tampered", "chain_mismatch", "manifest_unreadable"}
assert set(get_args(P.AlertCode)) <= set(get_args(P.ReasonCode))
assert get_args(P.RefusalCode) == ("expression", "alias", "multi_table", "unparsed")
assert set(P.DETAILS) == set(get_args(P.RefusalCode))
assert P.DETAILS["multi_table"] == ("join", "comma_join", "subquery", "cte", "compound", "authorizer",
"duplicate_pk")
assert "literal_keyword" in P.DETAILS["expression"] and "ambiguous_column" in P.DETAILS["unparsed"]
assert get_args(P.FromRole) == ("axis_header", "row_label", "section_title", "col_header", "total_label")
assert get_args(P.RowStatus) == ("passed", "mismatch", "unverifiable")
assert get_args(P.CellStatus) == ("ok", "unverifiable", "unknown", "not_formula")
assert get_args(P.RawState) == ("kept", "purged", "absent")
assert get_args(P.Mode) == ("replace", "accumulate")
assert get_args(P.AcceptanceKind) == ("override", "waiver")
assert get_args(P.YearFrom) == ("period", "human")
assert get_args(P.ColumnRole) == ("axis", "dim", "derive", "const", "measure", "value", "text")
# 和配方契约同名的取值:照搬 CheckResult.status、TableOut.kind、PeriodOut.source,不能各写一份漂开
assert get_args(P.PartStatus) == get_args(T.CheckStatus)
assert get_args(P.TableKind) == get_args(typing.get_type_hints(T.TableOut)["kind"])
assert get_args(typing.get_type_hints(P.PeriodView)["source"]) == get_args(
typing.get_type_hints(T.PeriodOut)["source"])
def test_module_imports_only_stdlib():
"""识别器只许导入守卫、names 和这个模块(P4-SPEC 7.2):这里不能把配方模块、pydantic 拖进来。"""
tree = ast.parse(Path(P.__file__).read_text("utf-8"))
mods = set()
for node in ast.walk(tree):
if isinstance(node, ast.Import):
mods |= {a.name.split(".")[0] for a in node.names}
elif isinstance(node, ast.ImportFrom):
mods.add((node.module or "").split(".")[0])
assert mods <= {"__future__", "logging", "re", "collections", "dataclasses", "typing"}, mods
# ---------------------------------------------------------------------------
# 每个 dataclass 的字段名、顺序和缺省值(防止实现者改名;REQ = 必填)
# ---------------------------------------------------------------------------
REQ = "<必填>"
FACTORY = "<默认工厂>"
def _shape(cls) -> list[tuple[str, object]]:
out = []
for f in dataclasses.fields(cls):
if f.default is not MISSING:
out.append((f.name, f.default))
elif f.default_factory is not MISSING:
out.append((f.name, FACTORY))
else:
out.append((f.name, REQ))
return out
SHAPES: dict[str, list[tuple[str, object]]] = {
# 响应(P4-SPEC 2.8.1)
"ReportRef": [("node_id", REQ), ("doc_artifact", REQ)],
"CellRef": [("alias", REQ), ("row", REQ), ("column", REQ), ("artifact", None)],
"Reason": [("code", REQ), ("detail", ""), ("text", REQ)],
"Alert": [("code", REQ), ("text", REQ)],
"TableRef": [("name", REQ), ("kind", "data")],
"PeriodView": [("start", REQ), ("end", REQ), ("source", REQ), ("cells", FACTORY), ("signed_by", None)],
"AcceptanceView": [("check_id", REQ), ("title", REQ), ("kind", REQ), ("reason", REQ), ("signed_by", None),
("at", None)],
"StateNote": [("at", None), ("signed_by", None), ("reason", None)],
"PartView": [("import_id", REQ), ("seq", REQ), ("manifest", REQ), ("period", REQ), ("file_name", REQ),
("raw_sha256", REQ), ("region", None), ("excluded_rows", None), ("recipe_sha256", None),
("recipe_seq", None), ("committed_at", None), ("signed_by", None), ("acceptances", FACTORY),
("raw_state", None), ("purged", None), ("revoked", None), ("has_row", False)],
"VersionView": [("source", REQ), ("snapshot_id", REQ), ("mode", REQ), ("union", REQ), ("tables", FACTORY),
("manifest_view", True), ("parts", FACTORY)],
"FromCell": [("role", REQ), ("column", REQ), ("sheet", REQ), ("cell", REQ), ("text", None),
("locate_title", None)],
"YearSource": [("source", REQ), ("cells", FACTORY), ("signed_by", None), ("mixed", False)],
"CanonicalView": [("raw", REQ), ("canonical", REQ)],
"Recheck": [("sql", REQ), ("params", FACTORY), ("ok", REQ)],
"CellSource": [("table", REQ), ("column", REQ), ("column_role", REQ), ("kind", "data"), ("pk", FACTORY),
("rowid", None), ("part_seq", REQ), ("part_rowid", REQ), ("sheet", REQ), ("cell", REQ),
("header", None), ("unit", None), ("from", FACTORY), ("year", None), ("canonical", None),
("raw_purged", False), ("merged_fill", False), ("recheck", None)],
"RelatedCheck": [("id", REQ), ("kind", REQ), ("title", REQ), ("part_status", REQ), ("row_status", None),
("cell_status", None), ("acceptance", None), ("detail", None)],
"ProvenanceOut": [("schema", P.SCHEMA), ("report", REQ), ("segment", REQ), ("cell", None), ("status", REQ),
("reason", None), ("alert", None), ("sealed", REQ), ("version", None),
("cell_source", None), ("checks", FACTORY)],
# 中间结构(P4-SPEC 7.2–7.4)
"DirectSelect": [("table", REQ), ("alias", REQ), ("columns", REQ)],
"SelectRefusal": [("code", REQ), ("detail", REQ)],
"ChainPart": [("seq", REQ), ("import_id", REQ), ("manifest_id", REQ), ("manifest", REQ), ("union_rows", REQ)],
"Chain": [("source", REQ), ("source_id", REQ), ("snapshot_id", REQ), ("mode", REQ), ("union", REQ),
("expected_db_sha256", REQ), ("parts", REQ), ("schema", REQ), ("snapshot_manifest", REQ)],
"ChainProblem": [("code", REQ), ("detail", "")],
"Located": [("part", REQ), ("part_rowid", REQ), ("cell", REQ)],
"LocateProblem": [("code", REQ), ("detail", "")],
"SumEqRule": [("table", REQ), ("total", REQ), ("parts", REQ), ("types", REQ)],
"RelatedCheckPlan": [("check", REQ), ("rule", REQ)],
"CompileFacts": [("tables", REQ), ("result_rows", REQ), ("yields", REQ)],
}
RESPONSE = ("ReportRef", "CellRef", "Reason", "Alert", "TableRef", "PeriodView", "AcceptanceView", "StateNote",
"PartView", "VersionView", "FromCell", "YearSource", "CanonicalView", "Recheck", "CellSource",
"RelatedCheck", "ProvenanceOut")
@pytest.mark.parametrize("name", sorted(SHAPES))
def test_dataclass_shape(name):
cls = getattr(P, name)
assert dataclasses.is_dataclass(cls)
assert _shape(cls) == SHAPES[name]
def test_every_dataclass_is_pinned():
declared = {n for n, v in vars(P).items() if isinstance(v, type) and dataclasses.is_dataclass(v)}
assert declared == set(SHAPES), declared ^ set(SHAPES)
@pytest.mark.parametrize("name", RESPONSE)
def test_response_types_are_keyword_only(name):
"""响应的字段多、几个包各自构造:按位置传参会被拒,传错名字立刻报错。"""
cls = getattr(P, name)
assert all(f.kw_only for f in dataclasses.fields(cls)), name
with pytest.raises(TypeError):
cls("x")
def test_intermediate_types_take_positional_args_as_the_spec_writes_them():
assert P.ChainProblem("manifest_unreadable").detail == ""
assert P.LocateProblem("no_lineage") == P.LocateProblem(code="no_lineage", detail="")
assert P.SelectRefusal("alias", "alias").code == "alias"
assert P.CompileFacts({"日客流"}, 1, 0).yields == 0
assert P.DirectSelect("日客流", None, ["日期", "全日客流"]).columns == ["日期", "全日客流"]
# ---------------------------------------------------------------------------
# asdict 之后逐层对 P4-SPEC 2.8.1 的 JSON
# ---------------------------------------------------------------------------
KEYS = {
"top": ["schema", "report", "segment", "cell", "status", "reason", "alert", "sealed", "version", "cell_source",
"checks"],
"report": ["node_id", "doc_artifact"],
"cell": ["alias", "row", "column", "artifact"],
"reason": ["code", "detail", "text"],
"alert": ["code", "text"],
"version": ["source", "snapshot_id", "mode", "union", "tables", "manifest_view", "parts"],
"table": ["name", "kind"],
"part": ["import_id", "seq", "manifest", "period", "file_name", "raw_sha256", "region", "excluded_rows",
"recipe_sha256", "recipe_seq", "committed_at", "signed_by", "acceptances", "raw_state", "purged",
"revoked", "has_row"],
"period": ["start", "end", "source", "cells", "signed_by"],
"acceptance": ["check_id", "title", "kind", "reason", "signed_by", "at"],
"state": ["at", "signed_by", "reason"],
"cell_source": ["table", "column", "column_role", "kind", "pk", "rowid", "part_seq", "part_rowid", "sheet",
"cell", "header", "unit", "from", "year", "canonical", "raw_purged", "merged_fill", "recheck"],
"from": ["role", "column", "sheet", "cell", "text", "locate_title"],
"year": ["source", "cells", "signed_by", "mixed"],
"canonical": ["raw", "canonical"],
"recheck": ["sql", "params", "ok"],
"check": ["id", "kind", "title", "part_status", "row_status", "cell_status", "acceptance", "detail"],
}
def _accept(**kw) -> P.AcceptanceView:
return P.AcceptanceView(**{"check_id": "R1", "title": "「日客流」:全日客流 = 分区甲 + 分区乙", "kind": "override",
"reason": "合成理由:分区合计口径调整", "signed_by": None,
"at": "2026-10-01T08:00:00+00:00", **kw})
def full_out() -> P.ProvenanceOut:
"""字段齐全的一份(每个可空的嵌套对象都非空、每个列表至少一项):空的 checks 会让键集合的比较空过。"""
return P.ProvenanceOut(
report=P.ReportRef(node_id="write", doc_artifact="d" * 64),
segment="s7",
cell=P.CellRef(alias="Q3", row=0, column="全日客流", artifact="a" * 64),
status="inferred",
sealed=True,
version=P.VersionView(
source="flow_demo", snapshot_id="b" * 64, mode="accumulate", union=True,
tables=[P.TableRef(name="日客流", kind="data")],
manifest_view=True,
parts=[P.PartView(
import_id="i1", seq=1, manifest="c" * 64,
period=P.PeriodView(start="2026-08-01", end="2026-08-31", source="cells", cells=["客流汇总!B2"]),
file_name="月报导出_2026-08-01_2026-08-31.xlsx", raw_sha256="e" * 64,
region="客流汇总!B4:AG30", excluded_rows=0, recipe_sha256="f" * 64, recipe_seq=1,
committed_at="2026-10-01T08:00:00+00:00", signed_by=None,
acceptances=[_accept()], raw_state="purged",
purged=P.StateNote(at="2026-10-02T08:00:00+00:00", signed_by="录入员甲", reason="合成理由"),
revoked=P.StateNote(at="2026-10-02T09:00:00+00:00", signed_by=None, reason="合成理由"),
has_row=True)]),
cell_source=P.CellSource(
table="日客流", column="全日客流", column_role="measure", kind="data", pk={"日期": "2026-08-05"},
rowid=5, part_seq=1, part_rowid=5, sheet="客流汇总", cell="G5", header="全日客流(人次)", unit="人次",
from_=[P.FromCell(role="axis_header", column="日期", sheet="客流汇总", cell="G4"),
P.FromCell(role="row_label", column=None, sheet="客流汇总", cell="B5", text="全日客流(人次)")],
year=P.YearSource(source="period", cells=["客流汇总!B2"]),
canonical=P.CanonicalView(raw="8–9", canonical="8-9"),
raw_purged=True, merged_fill=False,
recheck=P.Recheck(sql='SELECT rowid, "全日客流" FROM "日客流" WHERE "日期" = ?', params=["2026-08-05"],
ok=True)),
checks=[P.RelatedCheck(id="R1", kind="relation_sum_eq", title="「日客流」:全日客流 = 分区甲 + 分区乙",
part_status="mismatch", row_status="passed", cell_status=None,
acceptance=_accept(), detail="合成的原因摘要")],
)
def test_asdict_matches_the_spec_json_level_by_level():
d = asdict(full_out())
v, part, cs = d["version"], d["version"]["parts"][0], d["cell_source"]
levels = {
"top": d, "report": d["report"], "cell": d["cell"], "version": v, "table": v["tables"][0], "part": part,
"period": part["period"], "acceptance": part["acceptances"][0], "state": part["purged"],
"cell_source": cs, "from": cs["from"][0], "year": cs["year"], "canonical": cs["canonical"],
"recheck": cs["recheck"], "check": d["checks"][0],
}
for name, obj in levels.items():
assert isinstance(obj, dict), name
assert list(obj) == KEYS[name], name
assert list(part["revoked"]) == KEYS["state"]
assert list(d["checks"][0]["acceptance"]) == KEYS["acceptance"]
# 原因、提示:table_only 加红的一份
red = P.ProvenanceOut(report=P.ReportRef(node_id="write", doc_artifact="d" * 64), segment="s9",
status="table_only", reason=P.refuse("db_tampered"), alert=P.alert("db_tampered"),
sealed=True, version=full_out().version)
rd = asdict(red)
assert list(rd["reason"]) == KEYS["reason"] and list(rd["alert"]) == KEYS["alert"]
assert rd["cell_source"] is None and rd["checks"] == [] and rd["cell"] is None
json.dumps(d, ensure_ascii=False)
def test_null_and_empty_defaults_match_the_spec():
out = P.ProvenanceOut(report=P.ReportRef(node_id="write", doc_artifact="d"), segment="s1", status="none",
reason=P.refuse("legacy_doc"), sealed=False)
d = asdict(out)
assert d["schema"] == "agentlab.provenance/1"
assert (d["cell"], d["alert"], d["version"], d["cell_source"], d["checks"]) == (None, None, None, None, [])
assert d["reason"] == {"code": "legacy_doc", "detail": "", "text": P.REASON_TEXT["legacy_doc"]}
# ---------------------------------------------------------------------------
# CellSource 的「from」:Python 里是 from_,asdict / fields / replace / JSON 里是 from
# ---------------------------------------------------------------------------
def _cell(**kw) -> P.CellSource:
base = {"table": "日客流", "column": "全日客流", "column_role": "measure", "part_seq": 1, "part_rowid": 5,
"sheet": "客流汇总", "cell": "G5"}
return P.CellSource(**{**base, **kw})
def test_cell_source_from_key():
fc = P.FromCell(role="axis_header", column="日期", sheet="客流汇总", cell="G4")
a = _cell(from_=[fc])
assert a.from_ == [fc] and getattr(a, "from") == [fc]
assert [f.name for f in dataclasses.fields(P.CellSource)].count("from") == 1
assert "from_" not in {f.name for f in dataclasses.fields(P.CellSource)}
d = asdict(a)
assert "from" in d and "from_" not in d and d["from"][0]["cell"] == "G4"
# 从 JSON 读回来:键是 from
b = P.CellSource(**{**d, "from": [P.FromCell(**x) for x in d["from"]]})
assert b == a
# replace 不丢 from(WP-C 回查后用 replace 补 rowid、pk、recheck 是常见写法)
c = dataclasses.replace(a, rowid=5, pk={"日期": "2026-08-05"})
assert c.from_ == [fc] and c.rowid == 5
# replace 换 from:两种名字都认,Python 名字优先
fc2 = P.FromCell(role="row_label", column=None, sheet="客流汇总", cell="B5")
assert dataclasses.replace(a, from_=[fc2]).from_ == [fc2]
assert dataclasses.replace(a, **{"from": [fc2]}).from_ == [fc2]
# 属性写入两种名字指同一份
setattr(a, "from", [])
assert a.from_ == [] and asdict(a)["from"] == []
assert _cell().from_ == [] and _cell().from_ is not _cell().from_
assert "from_=" in repr(_cell())
# 注解里同样是 from(pydantic 按注解收集字段);值存在实例字典的 from 键下
hints = typing.get_type_hints(P.CellSource)
assert "from" in hints and "from_" not in hints
assert list(hints) == [f.name for f in dataclasses.fields(P.CellSource)]
assert "from" in vars(a) and "from_" not in vars(a)
assert not hasattr(object.__new__(P.CellSource), "from_")
def test_pydantic_sees_the_same_json_as_asdict():
"""WP-C 若把 ProvenanceOut 写成返回注解或 response_model,pydantic 会按类型再序列化一遍:输出必须等于 asdict,
不能悄悄少掉「from」(只改字段表、不改注解时就是这样)。从 JSON 校验回来的实例也要能照常读写 from_。"""
ta = pydantic.TypeAdapter(P.ProvenanceOut)
out = full_out()
want = asdict(out)
assert ta.dump_python(out) == want and "from" in ta.dump_python(out)["cell_source"]
assert json.loads(ta.dump_json(out)) == json.loads(json.dumps(want))
back = ta.validate_json(json.dumps(want), strict=True)
assert back == out and back.cell_source.from_ == out.cell_source.from_
assert asdict(back) == want and dataclasses.replace(back.cell_source, rowid=9).from_ == out.cell_source.from_
cs_ta = pydantic.TypeAdapter(P.CellSource)
assert cs_ta.dump_python(out.cell_source) == asdict(out.cell_source)
async def test_fastapi_routes_return_the_same_json_however_typed():
"""三种写法的路由(推荐的 asdict + dict 注解、带类型的返回注解、response_model)回的 JSON 逐字相同。"""
from fastapi import FastAPI
from httpx import ASGITransport, AsyncClient
app = FastAPI()
@app.get("/plain")
async def plain() -> dict[str, Any]:
return asdict(full_out())
@app.get("/typed")
async def typed() -> P.ProvenanceOut:
return full_out()
@app.get("/model", response_model=P.ProvenanceOut)
async def model() -> Any:
return asdict(full_out())
want = json.loads(json.dumps(asdict(full_out())))
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://t") as client:
for path in ("/plain", "/typed", "/model"):
r = await client.get(path)
assert r.status_code == 200, path
assert r.json() == want, path
assert "from" in r.json()["cell_source"], path
assert (await client.get("/openapi.json")).status_code == 200
# ---------------------------------------------------------------------------
# 原文
# ---------------------------------------------------------------------------
def test_reason_text_covers_every_code():
assert set(P.REASON_TEXT) == set(get_args(P.ReasonCode))
assert set(P.ALERT_TEXT) == set(get_args(P.AlertCode))
for code, text in P.ALERT_TEXT.items():
assert text == P.REASON_TEXT[code]
# 4.2 的原文(WP-0 写定,界面原样显示):抽几条与规格逐字对照
assert P.REASON_TEXT["simple_upload"] == "这份表格按表头行直接导入,没有记录原表格子的溯源"
assert P.REASON_TEXT["masked"] == "这一格涉及设置了遮罩的列,不推断来源"
assert P.REASON_TEXT["null_value"] == "这一格是空值,不推断来源"
assert P.REASON_TEXT["expression"] == "这一格是计算、聚合或改写的结果,只给出表级来历"
assert "可能被修改过" in P.REASON_TEXT["db_tampered"] and "可能被修改过" in P.REASON_TEXT["chain_mismatch"]
assert "改了名" in P.REASON_TEXT["alias"] and "同一张表多次" in P.REASON_TEXT["multi_table"]
assert "已回收或不存在" in P.REASON_TEXT["snapshot_gone"]
assert P.REASON_TEXT["recheck_missing"] == P.REASON_TEXT["recheck_multiple"] == P.REASON_TEXT["recheck_mismatch"]
_H = "[㐀-鿿]"
_HALF = re.compile(f"{_H}[,;!?]|[,;!?]{_H}|{_H}:(?![/\\d])|:{_H}|{_H}\\(|\\){_H}|{_H}\"|\"{_H}|[“”]|\\.\\.\\.")
_BANNED = re.compile("钉|放行|后端|跑|多半|照样|眼下|压根|取不到|没取回来|读不懂|连不上|写坏了|快照版本")
@pytest.mark.parametrize("code", sorted(P.REASON_TEXT))
def test_reason_text_follows_the_copy_rules(code):
"""原文上界面:不含数字、坐标(prose_problems),不写「钉」「放行」等禁用写法,全角标点、「」引号、不用「...」,
不以句号收尾(界面上是一行说明,和其余面板文字一致)。"""
text = P.REASON_TEXT[code]
assert T.prose_problems(text) == [], (code, T.prose_problems(text))
assert not _BANNED.search(text), code
assert not _HALF.search(text), (code, _HALF.search(text))
assert not text.endswith("。"), code
assert re.search(_H, text)
def test_refuse_and_alert():
r = P.refuse("pk_missing", 列=["日期", "时段"])
assert r == P.Reason(code="pk_missing", detail="",
text="查询结果没有带齐主键列(日期、时段),无法按主键回查,只给出表级来历")
assert "(日期)" in P.refuse("pk_missing", 列="日期").text
assert P.refuse("multi_table", "duplicate_pk").detail == "duplicate_pk"
assert P.refuse("multi_table", detail="compound").text == P.REASON_TEXT["multi_table"]
assert P.refuse("no_lineage", "regions 没有 block 键").detail == "regions 没有 block 键"
with pytest.raises(ValueError):
P.refuse("pk_missing")
with pytest.raises(ValueError):
P.refuse("no_such_code") # type: ignore[arg-type]
assert P.alert("chain_mismatch") == P.Alert(code="chain_mismatch", text=P.ALERT_TEXT["chain_mismatch"])
with pytest.raises(ValueError):
P.alert("no_lineage") # type: ignore[arg-type]
@pytest.mark.parametrize("code,detail", [
# 原型识别器自己的名字(p4-probe/probe_recognizer.py),WP-A 照抄就会走到这里;还有 2.4 没逐条写的写法
("unparsed", "indexed_by"), ("multi_table", "union"), ("expression", "aggregate"),
("alias", "expression_or_alias"), ("multi_table", "subquery_or_compound"), ("alias", "join"),
])
def test_refuse_with_an_unknown_detail_degrades_instead_of_raising(code, detail, caplog):
"""细分取决于识别器对这条 SQL 的判断:漏了映射时推断来源接口不能回 500。照样给分组的原文、记警告,
contract_problems 报出来(接口和验收测试据此抓到)。"""
with caplog.at_level("WARNING", logger=P.__name__):
r = P.refuse(code, detail)
assert r == P.Reason(code=code, detail=detail, text=P.REASON_TEXT[code])
assert detail in caplog.text
out = P.ProvenanceOut(report=P.ReportRef(node_id="write", doc_artifact="d"), segment="s1", status="table_only",
reason=r, sealed=True, version=_no_row_version())
assert any("DETAILS" in p for p in P.contract_problems(out))
def test_every_listed_detail_passes_and_empty_detail_is_allowed(caplog):
v = _no_row_version()
ref = P.ReportRef(node_id="write", doc_artifact="d")
with caplog.at_level("WARNING", logger=P.__name__):
for code, details in P.DETAILS.items():
for detail in (*details, ""):
out = P.ProvenanceOut(report=ref, segment="s1", status="table_only", reason=P.refuse(code, detail),
sealed=True, version=v)
assert P.contract_problems(out) == [], (code, detail)
assert caplog.text == ""
# 只有四个判据分组受 DETAILS 约束;其余 code 的细分是自由文字(ChainProblem / LocateProblem 的 detail)
assert P.contract_problems(P.ProvenanceOut(report=ref, segment="s1", status="table_only",
reason=P.refuse("no_lineage", "regions 没有 block 键"), sealed=True,
version=v)) == []
def _no_row_version() -> P.VersionView:
v = full_out().version
for part in v.parts:
part.has_row = False
return v
# ---------------------------------------------------------------------------
# 不变式
# ---------------------------------------------------------------------------
def test_contract_problems_accepts_the_valid_shapes():
assert P.contract_problems(full_out()) == []
v = full_out().version
for part in v.parts:
part.has_row = False
ref = P.ReportRef(node_id="write", doc_artifact="d")
ok = [
P.ProvenanceOut(report=ref, segment="s1", status="none", reason=P.refuse("legacy_doc"), sealed=True),
P.ProvenanceOut(report=ref, segment="s1", status="none", reason=P.refuse("chain_mismatch"),
alert=P.alert("chain_mismatch"), sealed=True),
P.ProvenanceOut(report=ref, segment="s1", status="table_only", reason=P.refuse("alias", "alias"),
sealed=False, version=v),
P.ProvenanceOut(report=ref, segment="s1", status="table_only", reason=P.refuse("chain_mismatch"),
alert=P.alert("chain_mismatch"), sealed=True, version=v),
]
for out in ok:
assert P.contract_problems(out) == [], out
def test_contract_problems_catches_broken_shapes():
ref = P.ReportRef(node_id="write", doc_artifact="d")
v = full_out().version
broken = [
dataclasses.replace(full_out(), reason=P.refuse("alias", "alias")),
dataclasses.replace(full_out(), cell_source=None),
dataclasses.replace(full_out(), cell_source=dataclasses.replace(full_out().cell_source, recheck=None)),
P.ProvenanceOut(report=ref, segment="s1", status="none", sealed=True),
P.ProvenanceOut(report=ref, segment="s1", status="none", reason=P.refuse("db_tampered"), sealed=True),
P.ProvenanceOut(report=ref, segment="s1", status="none", reason=P.refuse("legacy_doc"), sealed=True,
version=v),
P.ProvenanceOut(report=ref, segment="s1", status="table_only", reason=P.refuse("no_pk"), sealed=True),
P.ProvenanceOut(report=ref, segment="s1", status="table_only", reason=P.refuse("no_pk"), sealed=True,
version=v, checks=full_out().checks),
P.ProvenanceOut(report=ref, segment="s1", status="table_only", reason=P.refuse("no_pk"),
alert=P.alert("db_tampered"), sealed=True, version=v),
]
for out in broken:
assert P.contract_problems(out), out
def _with_recheck(sql: str, params: list[Any], pk: dict[str, Any]) -> P.ProvenanceOut:
out = full_out()
out.cell_source.pk = pk
out.cell_source.recheck = P.Recheck(sql=sql, params=params, ok=True)
return out
def test_recheck_uses_question_marks_in_primary_key_order():
"""回查 SQL 的占位符是 ?,params 与 pk 按 primary_key 的顺序一一对应(Recheck 的说明;规格 R16 的 :p0 以此为准)。"""
pk = {"日期": "2026-08-20", "时段": "13-14"}
good = 'SELECT rowid, "客流" FROM "时段客流" WHERE "日期" = ? AND "时段" = ?'
assert P.contract_problems(_with_recheck(good, ["2026-08-20", "13-14"], pk)) == []
# 列名里的问号不算占位符
assert P.contract_problems(_with_recheck('SELECT rowid, "是否?" FROM "表""甲" WHERE "日期" = ?',
["2026-08-05"], {"日期": "2026-08-05"})) == []
for sql, params in [
('SELECT rowid, "客流" FROM "时段客流" WHERE "日期" = :p0 AND "时段" = :p1', ["2026-08-20", "13-14"]),
(good, ["13-14", "2026-08-20"]),
(good, ["2026-08-20"]),
('SELECT rowid, "客流" FROM "时段客流" WHERE "日期" = ?', ["2026-08-20", "13-14"]),
]:
assert any("recheck" in p for p in P.contract_problems(_with_recheck(sql, params, pk))), (sql, params)
# ---------------------------------------------------------------------------
# recipe_types.OutsideText.hidden
# ---------------------------------------------------------------------------
def test_outside_text_hidden():
names = [f.name for f in dataclasses.fields(T.OutsideText)]
assert names == ["sheet", "cell", "text", "kind", "period_source", "hidden"]
assert T.OutsideText("客流汇总", "客流汇总!B3", "客流汇总表", "text").hidden is None
assert asdict(T.OutsideText("客流汇总", "客流汇总!B3", "注", "text", False, True))["hidden"] is True
from app.data.recipe_imports import extraction_from_dict
old = {"ok": True, "outside_text": [{"sheet": "客流汇总", "cell": "客流汇总!B3", "text": "客流汇总表", "kind": "text",
"period_source": False}]}
assert extraction_from_dict(old).outside_text[0].hidden is None
new = {"ok": True, "outside_text": [{**old["outside_text"][0], "hidden": False}]}
assert extraction_from_dict(new).outside_text[0].hidden is False
# ---------------------------------------------------------------------------
# 前端夹具:每一份推断来源的答复都按契约读得回、满足不变式
# ---------------------------------------------------------------------------
def _convert(tp: Any, value: Any) -> Any:
if value is None:
return None
if isinstance(tp, type) and dataclasses.is_dataclass(tp):
return _build(tp, value)
origin = get_origin(tp)
if origin is list:
(arg,) = get_args(tp)
return [_convert(arg, x) for x in value]
if origin in (typing.Union, types.UnionType):
for arg in get_args(tp):
if isinstance(arg, type) and dataclasses.is_dataclass(arg):
return _build(arg, value)
return value
def _build(cls: type, data: dict[str, Any]) -> Any:
"""按契约把 JSON 读回 dataclass。键必须一个不多、一个不少。"""
hints = typing.get_type_hints(cls)
names = [f.name for f in dataclasses.fields(cls)]
assert set(data) == set(names), (cls.__name__, set(data) ^ set(names))
return cls(**{n: _convert(hints[n], data[n]) for n in names})
def _fixture() -> dict[str, Any]:
return json.loads(FIXTURE.read_text("utf-8"))
def test_frontend_fixture_answers_follow_the_contract():
fx = _fixture()
assert fx["provenance"], "夹具里至少有一份推断来源的答复"
statuses = set()
strict = pydantic.TypeAdapter(P.ProvenanceOut)
for sid, body in fx["provenance"].items():
if not isinstance(body, dict) or body.get("schema") != P.SCHEMA:
continue # 模拟出错的答复({status, json})由检查脚本按原样回放
out = _build(P.ProvenanceOut, body)
assert asdict(out) == body, sid
# 键之外再按类型严格校验一遍:取值不在 Literal 里、数写成字符串、该有的对象是 null 都会报
assert asdict(strict.validate_json(json.dumps(body), strict=True)) == body, sid
assert P.contract_problems(out) == [], (sid, P.contract_problems(out))
assert out.segment == sid
if out.reason is not None:
assert out.reason.text == P.refuse(out.reason.code, out.reason.detail, 列=_pk_cols(out)).text, sid
if out.alert is not None:
assert out.alert.text == P.ALERT_TEXT[out.alert.code]
statuses.add(out.status)
assert statuses == {"inferred", "table_only", "none"}
def _pk_cols(out: P.ProvenanceOut) -> str:
m = re.search(r"主键列(([^)]*))", out.reason.text if out.reason else "")
return m.group(1) if m else ""
def test_frontend_fixture_canonical_follows_the_a8_rule():
"""CanonicalView 的口径(P4-SPEC 2.5、6.2 A8、A18):这一行标签列(dim、合计项)的原文与数据库里的值不同时给出,
被引用的是值列也给;相同、或这一行没有标签列(宽表的行标签是指标名,column 为 null)时为 null。"""
fx = _fixture()
given = set()
for sid, body in fx["provenance"].items():
cs = body.get("cell_source") if isinstance(body, dict) else None
if not cs:
continue
if cs["column_role"] == "dim":
# 被引用的就是标签格:原文不在 from 里(from 只列其余的格),只核对规范写法那一侧
if cs["canonical"] is not None:
assert cs["canonical"]["canonical"] == cs["pk"][cs["column"]], sid
given.add(sid)
continue
labels = [f for f in cs["from"] if f["role"] in ("row_label", "total_label") and f["column"] is not None]
assert len(labels) <= 1, sid
if labels and labels[0]["text"] is not None and labels[0]["text"] != cs["pk"][labels[0]["column"]]:
assert cs["canonical"] == {"raw": labels[0]["text"], "canonical": cs["pk"][labels[0]["column"]]}, sid
given.add(sid)
else:
assert cs["canonical"] is None, sid
# 合计表那一格(18-22 时合计)、累积并集里全角标签那一期的值格和标签格(8-9)
cases = fx["cases"]
assert {cases[k]["segment"] for k in ("total", "union", "fullwidth")} <= given, given
assert fx["provenance"][cases["total"]["segment"]]["cell_source"]["canonical"] == {
"raw": "18-22 时合计", "canonical": "18-22时合计"}
assert fx["provenance"][cases["union"]["segment"]]["cell_source"]["canonical"] == {"raw": "8-9", "canonical": "8-9"}
def test_frontend_fixture_cases_point_at_real_segments():
"""检查脚本按片段文字认片段:cases 里每个场景的文字在文档里恰好对应一个片段,编号与之一致。"""
fx = _fixture()
segs = [s for b in fx["doc"]["blocks"] for u in b["units"] for s in u["segments"]]
assert fx["doc"].get("provenance") == P.DOC_PROVENANCE
for key, case in fx["cases"].items():
hits = [s for s in segs if s["text"] == case["text"] and s.get("cite")]
assert len(hits) == 1, (key, case["text"], len(hits))
assert hits[0]["id"] == case["segment"], key
assert case["segment"] in fx["segments"], key
# ---------------------------------------------------------------------------
# 前端类型(frontend/src/types.ts)与契约逐个字段对齐:合并后 types.ts 归 WP-D,改名会在这里暴露
# ---------------------------------------------------------------------------
TYPES_TS = ROOT / "frontend" / "src" / "types.ts"
TS_INTERFACES = {
"EvidenceProvenance": "ProvenanceOut", "ProvenanceReportRef": "ReportRef", "ProvenanceCellRef": "CellRef",
"ProvenanceReason": "Reason", "ProvenanceAlert": "Alert", "ProvenanceTableRef": "TableRef",
"ProvenancePeriod": "PeriodView", "ProvenanceAcceptance": "AcceptanceView", "ProvenanceStateNote": "StateNote",
"ProvenancePart": "PartView", "ProvenanceVersion": "VersionView", "ProvenanceFromCell": "FromCell",
"ProvenanceYear": "YearSource", "ProvenanceCanonical": "CanonicalView", "ProvenanceRecheck": "Recheck",
"ProvenanceCellSource": "CellSource", "ProvenanceCheck": "RelatedCheck",
}
TS_UNIONS = {
"ProvenanceStatus": P.Status, "ProvenanceReasonCode": P.ReasonCode, "ProvenanceAlertCode": P.AlertCode,
"ProvenanceFromRole": P.FromRole, "ProvenancePartStatus": P.PartStatus, "ProvenanceRowStatus": P.RowStatus,
"ProvenanceCellStatus": P.CellStatus,
}
#: 有意与 Python 不同的几处(字段 → TS 里应写的联合成员):
#: - rowid、recheck 只在 inferred 时有、那时必填(contract_problems 钉住),TS 写成非空,界面不用到处判空;
#: - schema、column_role 带开放的尾巴 `(string & {})`:以后加取值时老界面照样能编译、能显示原值
TS_OVERRIDES: dict[tuple[str, str], tuple[str, ...]] = {
("ProvenanceCellSource", "rowid"): ("number",),
("ProvenanceCellSource", "recheck"): ("ProvenanceRecheck",),
("EvidenceProvenance", "schema"): (f"'{P.SCHEMA}'", "(string & {})"),
("ProvenanceCellSource", "column_role"): (*(f"'{x}'" for x in get_args(P.ColumnRole)), "(string & {})"),
}
def _ts_fields(src: str, name: str) -> list[str]:
return list(_ts_field_types(src, name))
def _ts_field_types(src: str, name: str) -> dict[str, str]:
"""接口的字段 → 类型原文(`name: type`、`name?: type`,可选的在键名后保留问号)。"""
m = re.search(rf"^export interface {name} \{{\n(.*?)^\}}", src, re.S | re.M)
assert m, name
return {f + q: t.strip() for f, q, t in re.findall(r"^ (\w+)(\??): (.+)$", m.group(1), re.M)}
def _ts_aliases(src: str) -> dict[str, tuple[str, ...]]:
"""TS_UNIONS 里的具名联合 → 字面量成员(带引号,同 _ts_members 的写法)。"""
out = {}
for ts in TS_UNIONS:
m = re.search(rf"^export type {ts} =((?:[^\n]*\n)(?:\s+\|[^\n]*\n)*)", src, re.M)
assert m, ts
out[ts] = tuple(f"'{x}'" for x in re.findall(r"'([^']+)'", m.group(1)))
return out
def _ts_union(text: str, aliases: dict[str, tuple[str, ...]]) -> tuple[str, ...]:
"""TS 类型原文 → 联合成员(顶层按「|」拆,具名联合展开成字面量)。这一节的类型里没有带「|」的泛型。"""
out: list[str] = []
for member in (m.strip() for m in text.split("|")):
out.extend(aliases.get(member, (member,)))
return tuple(out)
def _ts_members(tp: Any) -> tuple[str, ...]:
"""Python 注解 → 应写的 TS 联合成员:str/int/bool/Any → string/number/boolean/unknown,Literal → 带引号的取值,
X | None → X 的成员加 null,list[X] → X[],dict[str, X] → Record<string, X>,dataclass → 对应的 TS 接口名。"""
if tp is type(None):
return ("null",)
simple = {str: "string", bool: "boolean", int: "number", Any: "unknown"}
if tp in simple:
return (simple[tp],)
if isinstance(tp, type) and dataclasses.is_dataclass(tp):
return (TS_NAME[tp.__name__],)
origin = get_origin(tp)
if origin is typing.Literal:
return tuple(f"'{x}'" for x in get_args(tp))
if origin in (typing.Union, types.UnionType):
return tuple(m for arg in get_args(tp) for m in _ts_members(arg))
if origin is list:
(inner,) = (_ts_members(a) for a in get_args(tp))
assert len(inner) == 1, tp
return (f"{inner[0]}[]",)
if origin is dict:
key, value = get_args(tp)
assert key is str, tp
return (f"Record<string, {_ts_members(value)[0]}>",)
raise AssertionError(f"契约里出现了没教过的注解:{tp!r}")
TS_NAME = {py: ts for ts, py in TS_INTERFACES.items()}
def test_frontend_types_match_the_contract():
"""字段名、顺序之外,逐个字段比类型:取值联合(含行内的 'override' | 'waiver' 这种)、可空性、数组、嵌套接口。
合并后 types.ts 归 WP-D,这里是唯一能发现它和服务端契约漂开的地方。"""
_ts_alignment(TYPES_TS.read_text("utf-8"))
def _ts_alignment(src: str) -> None:
aliases = _ts_aliases(src)
for ts, lit in TS_UNIONS.items():
assert aliases[ts] == tuple(f"'{x}'" for x in get_args(lit)), ts
for ts, py in TS_INTERFACES.items():
cls = getattr(P, py)
got = _ts_field_types(src, ts)
names = [f.name for f in dataclasses.fields(cls)]
# 答复里每个键都在(值可以是 null),TS 一律不写成可选
assert list(got) == names, ts
hints = typing.get_type_hints(cls)
for name in names:
want = TS_OVERRIDES.get((ts, name), _ts_members(hints[name]))
assert _ts_union(got[name], aliases) == want, (ts, name, got[name], want)
# 白名单不能烂掉:有意写成非空的两项在 Python 里确实可空;开放尾巴的两项 Python 一侧分别是 str、ColumnRole
cs_hints = typing.get_type_hints(P.CellSource)
assert _ts_members(cs_hints["rowid"]) == ("number", "null")
assert _ts_members(cs_hints["recheck"]) == ("ProvenanceRecheck", "null")
assert typing.get_type_hints(P.ProvenanceOut)["schema"] is str
assert get_args(cs_hints["column_role"]) == get_args(P.ColumnRole)
step = re.search(r"^export interface EvidenceStep .*?^\}", src, re.S | re.M).group(0)
assert re.search(r"^ provenance\?: boolean$", step, re.M)
outside = re.search(r"^export interface OutsideText \{.*?^\}", src, re.S | re.M).group(0)
assert re.search(r"^ hidden\?: boolean \| null$", outside, re.M)
@pytest.mark.parametrize("old,new", [
# 评审实测过抓不到的两处,加上几种同类的改法:都必须让上面的对齐测试失败
(" kind: 'override' | 'waiver'\n", " kind: 'overrides' | 'waivers'\n"),
(" rowid: number\n", " rowid: string\n"),
(" raw_state: 'kept' | 'purged' | 'absent' | null\n", " raw_state: 'kept' | 'purged' | null\n"),
(" mode: 'replace' | 'accumulate'\n", " mode: 'replace' | 'accumulate' | 'append'\n"),
(" source: 'period' | 'human'\n", " source: 'cells' | 'human'\n"),
(" period: ProvenancePeriod | null\n", " period: ProvenancePeriod\n"),
(" title: string | null\n kind: 'override'", " title: string\n kind: 'override'"),
(" params: unknown[]\n", " params: string\n"),
(" row_status: ProvenanceRowStatus | null\n", " row_status: ProvenanceCellStatus | null\n"),
(" column_role: 'axis' | 'dim'", " column_role: 'axis' | 'dims'"),
])
def test_frontend_type_alignment_catches_drift(old, new):
src = TYPES_TS.read_text("utf-8")
start = src.index("// 期 4:推断的来源")
assert old in src[start:], old
with pytest.raises(AssertionError):
_ts_alignment(src[:start] + src[start:].replace(old, new, 1))
# ---------------------------------------------------------------------------
# 金样:工件 id 等于内容哈希(被人改过一个字就对不上)
# ---------------------------------------------------------------------------
def test_golden_is_content_addressed():
from app.core.artifact_store import canonical_json, content_hash
g = json.loads(GOLDEN.read_text("utf-8"))
assert "2cdbf9a" in g["_comment"] and g["commit"] == "2cdbf9a"
assert g["script"].endswith("p4-golden/gen.py")
for aid, content in g["artifacts"].items():
assert content_hash(canonical_json(content)) == aid
assert set(g["reports"]) == {"recipe", "simple", "manual"}
for rep in g["reports"].values():
doc = rep["outputs"]["doc"]
assert content_hash(canonical_json(doc)) == rep["doc_artifact"]
assert "provenance" not in doc
assert set(rep["artifacts"]) <= set(g["artifacts"])
assert rep["outputs"]["chain"] and rep["outputs"]["excerpts"]
+497
View File
@@ -0,0 +1,497 @@
"""证据下钻(期 4)里碰数据库和数据文件的几步:provenance_db 的各函数,engine.py 的两处小改,sum_eq_conditions。
数据文件都是测试里现造的小 SQLite 库(表名「日客流」「时段客流」,分区甲、分区乙,数字随机),数据文件记录
直接写进测试库的 source_snapshots,不经导入管线:这里要钉的是回查、编译核对、强制重算、行级核对各自的口径,
整条链(真导入、真运行)由 WP-E 的验收覆盖。
"""
from __future__ import annotations
import hashlib
import os
import random
import sqlite3
import threading
import uuid
from datetime import datetime, timezone
import pytest
from app.data import engine as engine_mod
from app.data import provenance_db
from app.data import recipe_checks
from app.data.engine import EngineCache, SnapshotTampered, cache_key, engines, open_checked_sqlite
from app.data.provenance_types import _QUOTED, CompileFacts, StateNote, SumEqRule
from app.data.recipe_checks import sum_eq_conditions
from app.data.recipe_types import ColumnOut, SumEq
from app.data.table_versions import SnapshotManifestMismatch, SnapshotMissing
from app.db.base import SessionLocal
from app.db.models import DataSource, SourceSnapshot, TableImport, TableRecipe
RNG = random.Random(4004)
# --------------------------------------------------------------------------
# 夹具
# --------------------------------------------------------------------------
def _sha(path) -> str:
with open(path, "rb") as f:
return hashlib.sha256(f.read()).hexdigest()
def _build(path, *, daily, hourly=(), real=()) -> str:
"""造一个快照库:日客流(单列主键,写成 PRIMARY KEY DESC,和执行器一样不让它成为 rowid 的别名)、
时段客流(两列主键)、分区占比(REAL 列)。"""
conn = sqlite3.connect(path)
conn.executescript("""
CREATE TABLE "日客流" ("日期" TEXT NOT NULL, "全日客流" INTEGER, "分区甲" INTEGER, "分区乙" INTEGER,
PRIMARY KEY ("日期" DESC));
CREATE TABLE "时段客流" ("日期" TEXT NOT NULL, "时段" TEXT NOT NULL, "客流" INTEGER,
PRIMARY KEY ("日期", "时段"));
CREATE TABLE "分区占比" ("日期" TEXT NOT NULL, "合计" REAL, "分区甲" REAL, "分区乙" REAL,
PRIMARY KEY ("日期" DESC));
""")
conn.executemany('INSERT INTO "日客流" VALUES (?, ?, ?, ?)', list(daily))
conn.executemany('INSERT INTO "时段客流" VALUES (?, ?, ?)', list(hourly))
conn.executemany('INSERT INTO "分区占比" VALUES (?, ?, ?, ?)', list(real))
conn.commit()
conn.close()
os.chmod(path, 0o444)
return str(path)
def _daily_rows(n=6):
"""前几行成立,第 3 行不成立(合计多 1),第 5 行分区乙为空。"""
rows = []
for i in range(n):
a, b = RNG.randint(1000, 9000), RNG.randint(1000, 9000)
total = a + b
if i == 2:
total += 1
rows.append((f"2026-08-{i + 1:02d}", total, a, None if i == 4 else b))
return rows
async def _register(path, *, source_id=None, retired=False, sha=None, manifest=None) -> SourceSnapshot:
snap = SourceSnapshot(
id=uuid.uuid4().hex + uuid.uuid4().hex, source_id=source_id or uuid.uuid4().hex,
db_path=str(path), db_sha256=_sha(path) if sha is None else sha, schema_cache={},
retired_at=datetime.now(timezone.utc) if retired else None, manifest_artifact=manifest,
)
async with SessionLocal() as session:
session.add(snap)
await session.commit()
await session.refresh(snap)
return snap
@pytest.fixture
async def snapshot(tmp_path):
"""一个登记好的快照库,数据源记录不存在(模拟数据源已删除)。用完清掉引擎缓存。"""
path = _build(tmp_path / "snap.db", daily=_daily_rows(),
hourly=[("2026-08-03", "7-8", None), ("2026-08-03", "8-9", 321)],
real=[("2026-08-01", 0.3, 0.1, 0.2), ("2026-08-02", 1.5, 0.5, 0.75)])
snap = await _register(path)
yield snap
await engines.invalidate(snap.source_id)
async def _view(snap: SourceSnapshot):
return provenance_db.view_of(snap, source_name="flow_demo")
def _poke(path: str, *, keep_mtime: bool) -> None:
"""原地改一个字节(大小不变)。keep_mtime 时再用 os.utime 还原修改时间:EngineCache 的指纹认不出这种改动。"""
st = os.stat(path)
os.chmod(path, 0o644)
with open(path, "r+b") as f:
f.seek(-1, os.SEEK_END)
last = f.read(1)
f.seek(-1, os.SEEK_END)
f.write(bytes([last[0] ^ 0x01]))
os.chmod(path, 0o444)
if keep_mtime:
os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns))
# --------------------------------------------------------------------------
# sum_eq_conditions:导入时的核对和行级核对共用一段条件
# --------------------------------------------------------------------------
def test_sum_eq_conditions_integer_and_real():
nonnull, anynull, mismatch = sum_eq_conditions("全日客流", ["分区甲", "分区乙"],
{"全日客流": "INTEGER", "分区甲": "INTEGER"})
assert nonnull == '"全日客流" IS NOT NULL AND "分区甲" IS NOT NULL AND "分区乙" IS NOT NULL'
assert anynull == '"全日客流" IS NULL OR "分区甲" IS NULL OR "分区乙" IS NULL'
assert mismatch == '"全日客流" <> "分区甲" + "分区乙"'
# 任一列是 REAL 就按相对误差比:0.1 + 0.2 不严格等于 0.3
_, _, real = sum_eq_conditions("合计", ["分区甲", "分区乙"], {"分区乙": "REAL"})
assert real == 'ABS("合计" - ("分区甲" + "分区乙")) > 1e-9 * MAX(1, ABS("合计"))'
# 名字里的双引号照样转义(DQS 关闭的连接上写错的名字直接报错)
assert sum_eq_conditions('a"b', ["c", "d"], {})[2] == '"a""b" <> "c" + "d"'
@pytest.mark.parametrize("table,total,parts,types", [
("日客流", "全日客流", ["分区甲", "分区乙"], {"全日客流": "INTEGER", "分区甲": "INTEGER", "分区乙": "INTEGER"}),
("分区占比", "合计", ["分区甲", "分区乙"], {"合计": "REAL", "分区甲": "REAL", "分区乙": "REAL"}),
])
async def test_row_status_agrees_with_the_import_check(snapshot, table, total, parts, types):
"""逐行的 row_status 和导入时的 _check_sum_eq 在同一个库上的结论对得上:成立、不成立、含空值的行数都相同。
REAL 表里 0.1 + 0.2 = 0.3 那一行两边都判成立(改成严格比较的条件会被这里抓到)。"""
view = await _view(snapshot)
rule = SumEqRule(table=table, total=total, parts=parts, types=types)
conn = open_checked_sqlite(snapshot.db_path)
try:
rowids = [r[0] for r in conn.execute(f'SELECT rowid FROM "{table}" ORDER BY rowid')]
result = recipe_checks._check_sum_eq(
recipe_checks._Sql(conn), SumEq(id="R1", kind="sum_eq", table=table, total=total, parts=parts), "核对",
[ColumnOut(name=n, type=t) for n, t in types.items()], ["日期"], {})
finally:
conn.close()
statuses = [await provenance_db.row_status(view, rule, rid) for rid in rowids]
assert statuses.count("mismatch") == result.failed
assert statuses.count("unverifiable") == result.unverifiable
assert statuses.count("passed") + statuses.count("mismatch") == result.checked
if table == "日客流":
assert statuses == ["passed", "passed", "mismatch", "passed", "unverifiable", "passed"]
else:
assert statuses == ["passed", "mismatch"]
async def test_row_status_follows_the_snapshot_rowid_it_is_given(snapshot):
"""按传进来的快照库 rowid 判,不按别的编号:同一条规则,第 3 行不成立、第 4 行成立。"""
view = await _view(snapshot)
rule = SumEqRule(table="日客流", total="全日客流", parts=["分区甲", "分区乙"], types={})
assert await provenance_db.row_status(view, rule, 3) == "mismatch"
assert await provenance_db.row_status(view, rule, 4) == "passed"
async def test_row_status_is_none_when_the_row_or_a_column_is_not_there(snapshot):
view = await _view(snapshot)
assert await provenance_db.row_status(
view, SumEqRule(table="日客流", total="全日客流", parts=["分区甲", "分区乙"], types={}), 999) is None
assert await provenance_db.row_status(
view, SumEqRule(table="日客流", total="全日客流", parts=["分区甲", "分区丙"], types={}), 1) is None
assert await provenance_db.row_status(
view, SumEqRule(table="没有这张表", total="a", parts=["b", "c"], types={}), 1) is None
# 名字只对 ASCII 不区分大小写(和 SQLite 一致),中文名原样比
assert await provenance_db.row_status(
view, SumEqRule(table="日客流", total="全日客流", parts=["分区甲", "分区乙"], types={}), 1) == "passed"
# --------------------------------------------------------------------------
# 数据文件记录与绑定
# --------------------------------------------------------------------------
async def test_snapshot_record(snapshot):
async with SessionLocal() as session:
got = await provenance_db.snapshot_record(session, snapshot.id)
assert got is not None and got.id == snapshot.id and got.db_path == snapshot.db_path
assert await provenance_db.snapshot_record(session, "f" * 64) is None
assert await provenance_db.snapshot_record(session, "") is None
async def test_view_of_uses_a_stand_in_keyed_like_the_query(snapshot):
"""数据源记录不存在(已删除)也能绑定;id 用 snapshot.source_id,缓存键和查询时相同,首次哈希只算一次。"""
async with SessionLocal() as session:
assert await session.get(DataSource, snapshot.source_id) is None
view = provenance_db.view_of(snapshot, source_name="flow_demo")
assert view.id == snapshot.source_id and view.name == "flow_demo"
assert view.snapshot_id == snapshot.id and view.database == snapshot.db_path
assert view.immutable is True and view.readonly is True
assert view.expected_sha256 == snapshot.db_sha256
assert view.options == {} and view.description == "" and view.enabled is True
assert cache_key(view) == f"{snapshot.source_id}:{snapshot.id}"
async def test_view_of_refuses_unregistered_hash_and_gone_files(tmp_path):
path = _build(tmp_path / "a.db", daily=_daily_rows(2))
with pytest.raises(SnapshotMissing):
provenance_db.view_of(await _register(path, sha=""), source_name="flow_demo")
with pytest.raises(SnapshotMissing):
provenance_db.view_of(await _register(path, retired=True), source_name="flow_demo")
gone = await _register(path)
os.chmod(path, 0o644)
os.remove(path)
# 文件不在:绑定前就认出来(snapshot_gone,不标红),不留给 EngineCache 报成「文件被改」
with pytest.raises(SnapshotMissing) as info:
provenance_db.view_of(gone, source_name="flow_demo")
assert not isinstance(info.value, SnapshotTampered)
async def test_view_of_raises_manifest_mismatch_untouched(tmp_path):
"""快照清单里的库哈希和登记的不一致:bind_snapshot 抛的 SnapshotManifestMismatch 原样抛出。"""
from app.core import artifact_store
path = _build(tmp_path / "u.db", daily=_daily_rows(2))
mid = await artifact_store.put_json({"snapshot_id": None, "union": {"db_sha256": "0" * 64}},
kind="snapshot_manifest")
snap = await _register(path, manifest=mid)
with pytest.raises(SnapshotManifestMismatch):
provenance_db.view_of(snap, source_name="flow_demo")
# --------------------------------------------------------------------------
# R16:回查
# --------------------------------------------------------------------------
async def test_recheck_sql_params_and_rows(snapshot):
view = await _view(snapshot)
rows, sql, params = await provenance_db.recheck(view, table="日客流", column="全日客流", pk={"日期": "2026-08-02"})
assert sql == 'SELECT rowid, "全日客流" FROM "日客流" WHERE "日期" = ?'
assert params == ["2026-08-02"]
conn = sqlite3.connect(snapshot.db_path)
want = conn.execute('SELECT rowid, "全日客流" FROM "日客流" WHERE "日期" = ?', ["2026-08-02"]).fetchall()
conn.close()
assert rows == want and isinstance(rows[0][1], int)
# 契约 Recheck 的口径:去掉双引号里的标识符后 ? 的个数等于主键列数
assert _QUOTED.sub("", sql).count("?") == 1
async def test_recheck_composite_key_keeps_primary_key_order_and_types(snapshot):
view = await _view(snapshot)
rows, sql, params = await provenance_db.recheck(view, table="时段客流", column="客流",
pk={"日期": "2026-08-03", "时段": "8-9"})
assert sql == 'SELECT rowid, "客流" FROM "时段客流" WHERE "日期" = ? AND "时段" = ?'
assert params == ["2026-08-03", "8-9"] and [v for _, v in rows] == [321]
# 空值照实回(R11 已拦下空值,这里只回事实);REAL 回 float,不和 int 混
rows, _, _ = await provenance_db.recheck(view, table="时段客流", column="客流", pk={"日期": "2026-08-03", "时段": "7-8"})
assert [v for _, v in rows] == [None]
rows, _, _ = await provenance_db.recheck(view, table="分区占比", column="合计", pk={"日期": "2026-08-02"})
assert rows[0][1] == 1.5 and isinstance(rows[0][1], float)
rows, _, _ = await provenance_db.recheck(view, table="日客流", column="全日客流", pk={"日期": "1999-01-01"})
assert rows == []
async def test_recheck_quotes_identifiers_with_double_quotes(tmp_path):
path = tmp_path / "q.db"
conn = sqlite3.connect(path)
conn.execute('CREATE TABLE "表""甲" ("键""一" TEXT NOT NULL, "值?" INTEGER, PRIMARY KEY ("键""一" DESC))')
conn.execute('INSERT INTO "表""甲" VALUES (?, ?)', ["k", 7])
conn.commit()
conn.close()
snap = await _register(path)
try:
rows, sql, params = await provenance_db.recheck(
await _view(snap), table='表"甲', column="值?", pk={'键"一': "k"})
assert sql == 'SELECT rowid, "值?" FROM "表""甲" WHERE "键""一" = ?'
assert [v for _, v in rows] == [7] and params == ["k"]
assert _QUOTED.sub("", sql).count("?") == 1
finally:
await engines.invalidate(snap.source_id)
async def test_recheck_raises_tampered_when_the_file_changed(snapshot):
"""改过一个字节(修改时间也变了):经 EngineCache 取引擎时重新核对哈希,SnapshotTampered 原样抛出。"""
view = await _view(snapshot)
await provenance_db.recheck(view, table="日客流", column="全日客流", pk={"日期": "2026-08-01"})
_poke(snapshot.db_path, keep_mtime=False)
with pytest.raises(SnapshotTampered):
await provenance_db.recheck(view, table="日客流", column="全日客流", pk={"日期": "2026-08-01"})
async def test_recheck_needs_a_primary_key(snapshot):
with pytest.raises(ValueError):
await provenance_db.recheck(await _view(snapshot), table="日客流", column="全日客流", pk={})
# --------------------------------------------------------------------------
# R16:强制重算
# --------------------------------------------------------------------------
async def test_rehash_catches_an_in_place_edit_that_keeps_the_fingerprint(snapshot, monkeypatch):
"""原地改一个字节、再还原修改时间:EngineCache 的指纹不变、照样给缓存的引擎,强制重算认得出(R16,A25②)。
重算放在线程里。"""
view = await _view(snapshot)
await engines.get(view)
assert await provenance_db.rehash(view) is True
threads: list[bool] = []
real = engine_mod._file_sha256
def spy(path):
threads.append(threading.current_thread() is threading.main_thread())
return real(path)
monkeypatch.setattr(engine_mod, "_file_sha256", spy)
before = engines.pinned(view)
_poke(snapshot.db_path, keep_mtime=True)
assert engine_mod._fingerprint(snapshot.db_path) == before
await engines.get(view) # 指纹没变,缓存命中,不重新核对
assert await provenance_db.rehash(view) is False
assert threads and not any(threads)
async def test_rehash_is_false_without_a_hash_or_a_file(snapshot):
view = await _view(snapshot)
view.expected_sha256 = None
assert await provenance_db.rehash(view) is False
view = await _view(snapshot)
view.database = str(snapshot.db_path) + ".missing"
assert await provenance_db.rehash(view) is False
# --------------------------------------------------------------------------
# R15:编译核对
# --------------------------------------------------------------------------
@pytest.mark.parametrize("sql,tables,rows,yields", [
('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" = \'2026-08-01\'', {"日客流"}, 1, 0),
('SELECT * FROM "日客流" ORDER BY "全日客流" DESC LIMIT 3', {"日客流"}, 1, 0),
('SELECT "日期","全日客流" FROM "日客流" UNION ALL SELECT "日期","分区甲" FROM "日客流"', {"日客流"}, 2, 0),
('SELECT "日期","全日客流" FROM "日客流" WHERE "日期" LIKE \'2026%\' ESCAPE \'\\\' '
'UNION ALL SELECT "日期","分区甲" FROM "日客流"', {"日客流"}, 2, 0),
('SELECT "日期","全日客流" FROM "日客流" WHERE "日期" IN (SELECT "日期" FROM "时段客流")',
{"日客流", "时段客流"}, 1, 0),
])
async def test_compile_facts_reports_tables_and_result_rows(snapshot, sql, tables, rows, yields):
facts = await provenance_db.compile_facts(await _view(snapshot), sql)
assert facts == CompileFacts(tables=tables, result_rows=rows, yields=yields)
@pytest.mark.parametrize("op", ["UNION", "INTERSECT", "EXCEPT"])
async def test_compile_facts_sees_yield_in_set_operations(snapshot, op):
facts = await provenance_db.compile_facts(
await _view(snapshot), f'SELECT "日期","全日客流" FROM "日客流" {op} SELECT "日期","分区甲" FROM "日客流"')
assert facts.yields >= 1
async def test_compile_facts_runs_in_a_thread_on_an_immutable_read_only_connection(snapshot, monkeypatch):
seen: list[dict] = []
real_open = provenance_db.open_checked_sqlite
def spy(path, **kw):
seen.append({"path": path, **kw, "main": threading.current_thread() is threading.main_thread()})
return real_open(path, **kw)
monkeypatch.setattr(provenance_db, "open_checked_sqlite", spy)
await provenance_db.compile_facts(await _view(snapshot), 'SELECT "日期" FROM "日客流"')
assert seen == [{"path": snapshot.db_path, "readonly": True, "immutable": True, "main": False}]
async def test_compile_facts_compares_against_the_engine_cache_fingerprint(snapshot, monkeypatch):
"""开连接前的指纹和 EngineCache 核对过的那份不等:SnapshotTampered(→ db_tampered)。"""
view = await _view(snapshot)
monkeypatch.setattr(engines, "pinned", lambda source: (0, 0, 0, 0))
with pytest.raises(SnapshotTampered):
await provenance_db.compile_facts(view, 'SELECT "日期" FROM "日客流"')
async def test_compile_facts_checks_the_fingerprint_again_after_closing(snapshot, monkeypatch):
"""编译期间文件被换(关连接后的指纹对不上):同样 SnapshotTampered,不交出编译结果。"""
view = await _view(snapshot)
real = provenance_db._fingerprint
calls: list[int] = []
def drifting(path):
calls.append(1)
fp = real(path)
return fp if len(calls) == 1 else (fp[0], fp[1] + 1, fp[2], fp[3])
monkeypatch.setattr(provenance_db, "_fingerprint", drifting)
with pytest.raises(SnapshotTampered):
await provenance_db.compile_facts(view, 'SELECT "日期" FROM "日客流"')
assert len(calls) == 2
async def test_compile_facts_without_a_pinned_engine_is_snapshot_missing(snapshot, monkeypatch):
monkeypatch.setattr(engines, "pinned", lambda source: None)
with pytest.raises(SnapshotMissing) as info:
await provenance_db.compile_facts(await _view(snapshot), 'SELECT "日期" FROM "日客流"')
assert not isinstance(info.value, SnapshotTampered)
async def test_compile_facts_refuses_a_replaced_file(snapshot):
"""引擎已在缓存里、文件随后被改(修改时间变了):取引擎时就重新核对,SnapshotTampered。"""
view = await _view(snapshot)
await provenance_db.compile_facts(view, 'SELECT "日期" FROM "日客流"')
_poke(snapshot.db_path, keep_mtime=False)
with pytest.raises(SnapshotTampered):
await provenance_db.compile_facts(view, 'SELECT "日期" FROM "日客流"')
# --------------------------------------------------------------------------
# 导入记录上的当前状态
# --------------------------------------------------------------------------
async def test_import_states():
sid = uuid.uuid4().hex
async with SessionLocal() as session:
recipe = TableRecipe(source_id=sid, seq=3, recipe={}, recipe_sha256="a" * 64)
session.add(recipe)
await session.flush()
kept = TableImport(source_id=sid, seq=1, raw_state="kept", recipe_id=recipe.id)
purged = TableImport(source_id=sid, seq=2, raw_state="purged", recipe_id="nosuchrecipe",
purged={"at": "2026-09-30T01:02:03+00:00", "reason": "合成理由:清除原件",
"signed_by": "录入员甲", "signed_by_verified": False})
revoked = TableImport(source_id=sid, seq=3, raw_state="kept",
revoked={"at": "2026-10-01T00:00:00+00:00", "reason": "合成理由:作废",
"signed_by": None, "signed_by_verified": False})
session.add_all([kept, purged, revoked])
await session.commit()
ids = [kept.id, purged.id, revoked.id]
got = await provenance_db.import_states(session, [*ids, "nosuchimport", kept.id])
assert set(got) == set(ids)
assert got[kept.id] == {"raw_state": "kept", "purged": None, "revoked": None, "recipe_seq": 3}
assert got[purged.id] == {"raw_state": "purged", "revoked": None, "recipe_seq": None, "purged": StateNote(
at="2026-09-30T01:02:03+00:00", signed_by="录入员甲", reason="合成理由:清除原件")}
assert got[revoked.id]["revoked"] == StateNote(at="2026-10-01T00:00:00+00:00", signed_by=None,
reason="合成理由:作废")
assert got[revoked.id]["recipe_seq"] is None
async with SessionLocal() as session:
assert await provenance_db.import_states(session, []) == {}
# --------------------------------------------------------------------------
# engine.py:EngineCache.pinned、open_checked_sqlite 的 immutable
# --------------------------------------------------------------------------
async def test_pinned_returns_the_verified_fingerprint(tmp_path):
path = _build(tmp_path / "p.db", daily=_daily_rows(2))
snap = await _register(path)
view = await _view(snap)
cache = EngineCache()
try:
assert cache.pinned(view) is None # 还没建引擎
await cache.get(view)
st = os.stat(path)
assert cache.pinned(view) == (st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns)
manual = type("S", (), {"id": "m1", "kind": "sqlite", "database": str(path), "readonly": True,
"options": {}, "name": "手工源"})()
await cache.get(manual)
assert cache.pinned(manual) is None # 手工源没有核对过的指纹
await cache.invalidate(snap.source_id)
assert cache.pinned(view) is None
finally:
await cache.close()
def test_open_checked_sqlite_immutable_flag(tmp_path, monkeypatch):
path = _build(tmp_path / "o.db", daily=_daily_rows(1))
uris: list[str] = []
real = sqlite3.connect
def spy(database, *a, **kw):
uris.append(database)
return real(database, *a, **kw)
monkeypatch.setattr(engine_mod.sqlite3, "connect", spy)
for kwargs in ({}, {"readonly": True}, {"readonly": True, "immutable": True}):
conn = open_checked_sqlite(path, **kwargs)
try:
assert conn.execute('SELECT COUNT(*) FROM "日客流"').fetchone() == (1,)
finally:
conn.close()
# 缺省行为不变:只读、不带 immutable
assert uris[0].endswith("?mode=ro") and uris[1].endswith("?mode=ro")
assert uris[2].endswith("?mode=ro&immutable=1")
with pytest.raises(ValueError):
open_checked_sqlite(path, readonly=False, immutable=True)
+394
View File
@@ -0,0 +1,394 @@
"""期 4「期 4 之前不变」的金样断言(P4-SPEC 6.6、1.4,WP-E)。
金样 `tests/fixtures/evidence/pre_p4_golden.json` 由 WP-0 在期 4 改动之前的提交(2cdbf9a)上真跑管线生成,自带三份
报告(配方源、简单导入、手工源)的全部输入工件和四项输出:verify_doc、裁判摘录、_chain(单元格片段的查询步骤)、
文档本身。这里不重跑管线(导入清单里有 created_at,重跑得到的 id 会不同),而是:
1. 把金样里的输入工件逐个 put_json 进本测试的临时数据目录,断言返回的 id 等于金样记下的键(内容寻址:对不上说明
金样被改过,后面的比较就没有意义);
2. 用新代码对同样的输入重算,逐字比金样。
为什么要这一层:期 4 的承诺是「期 4 之前的运行、非上传源的面板、复核、裁判摘录一个字都不变」(0.2 补充 1)。
文档标记 `provenance` 是唯一的开关(1.4):没有它的文档必须走和期 4 之前完全相同的路径,连 provenance、
direct_select 两个模块的函数都不许碰(3.3);带上它之后,非上传源仍然一字不差,上传源只在约定的位置多出约定的东西
(查询步骤只多 `provenance: true`;摘录只在 SQL 那一行之后、「列:」之前多出来历行)。
金样里的 `_chain` 输出是用预填 memo 的 `_Masks` 替身算的(不查库),这里用同样的替身;`_Sealed` 按金样记下的封存
范围在内存里拼(6.6)。夹具全部合成、假名,没有任何模型调用。
"""
from __future__ import annotations
import copy
import json
from pathlib import Path
from typing import Any
import pytest
from app.api import evidence as api
from app.core import artifact_store
from app.core.config import settings
from app.data import provenance as provenance_mod
from app.data.provenance_types import DOC_PROVENANCE, CellRef, contract_problems
from app.data.tabular import UNSHAPED_NOTE
from app.engine import direct_select as direct_select_mod
from app.engine import judge
from app.engine.evidence import iter_segments, verify_doc
GOLDEN_PATH = Path(__file__).parent / "fixtures" / "evidence" / "pre_p4_golden.json"
GOLDEN: dict[str, Any] = json.loads(GOLDEN_PATH.read_text("utf-8"))
REPORTS = ("recipe", "simple", "manual")
#: 上传表格的两份(查询快照有 data_version):带上标记后查询步骤多 provenance: true、摘录多来历行
UPLOADS = ("recipe", "simple")
#: 来历行(P4-SPEC 3.2)可能的行首:来源、来源的分期行(两个空格缩进)、口径、已接受、区域外文字、截断说明
NEW_LINE_HEADS = ("来源:", " ", "口径:", "已接受(", "区域外", "另有 ", "…另有部分来历信息未列出")
# ==========================================================================
# 夹具:临时数据目录里重建金样的输入工件
# ==========================================================================
@pytest.fixture(autouse=True)
async def rebuilt(monkeypatch, tmp_path) -> dict[str, str]:
"""每个用例一个独立的数据目录,金样的输入工件和三份文档逐个 put_json 进去,id 必须等于金样的键。
独立目录的理由:A26 那一例要把导入清单的文件改坏,不能影响别的用例;工件按内容寻址、不带运行 id,放在共用的
数据目录里会和别的测试串。"""
data = tmp_path / "data"
data.mkdir()
monkeypatch.setattr(settings, "data_dir", data)
ids: dict[str, str] = {}
for aid, content in GOLDEN["artifacts"].items():
got = await artifact_store.put_json(copy.deepcopy(content), kind="golden_input")
assert got == aid, f"金样里的工件 {aid[:12]}… 重建出来的 id 是 {got[:12]}…:金样被改过"
ids[aid] = got
for name in REPORTS:
rep = GOLDEN["reports"][name]
got = await artifact_store.put_json(copy.deepcopy(rep["outputs"]["doc"]), kind="report_doc")
assert got == rep["doc_artifact"], f"{name} 的文档重建出来的 id 不对:金样被改过"
return ids
@pytest.fixture
def tripwire(monkeypatch) -> list[str]:
"""把 provenance、direct_select 两个模块的公开函数全换成一调用就记下来并抛错的桩(P4-SPEC 3.3、6.6)。
老文档(没有标记)走的路径必须和期 4 之前完全相同,连这两个模块都不许碰:桩被调用一次,用例就失败。函数体内
`from app.data.provenance import judge_lines` 这种写法每次调用时现取模块属性,同样会撞上桩。"""
hits: list[str] = []
patched: list[str] = []
for mod in (provenance_mod, direct_select_mod):
for name, obj in vars(mod).items():
if name.startswith("_") or not callable(obj) or getattr(obj, "__module__", None) != mod.__name__ \
or isinstance(obj, type):
continue
def boom(*_a: Any, __name: str = f"{mod.__name__}.{name}", **_kw: Any) -> Any:
hits.append(__name)
raise AssertionError(f"期 4 之前的文档不该调用 {__name}")
monkeypatch.setattr(mod, name, boom)
patched.append(name)
# 桩真的装上了:入口函数一个不漏(漏了的话「没被调用」就是空话)
assert {"recognize", "tables_in", "judge_lines", "resolve_chain", "chain_artifacts", "version_view", "locate",
"related_checks"} <= set(patched), patched
return hits
# ==========================================================================
# 小工具
# ==========================================================================
def report(name: str) -> dict[str, Any]:
return GOLDEN["reports"][name]
def old_doc(name: str) -> dict[str, Any]:
doc = copy.deepcopy(report(name)["outputs"]["doc"])
assert "provenance" not in doc, "金样里的文档是期 4 之前组装的,不该有 provenance 键"
return doc
def marked_doc(name: str) -> dict[str, Any]:
return {**old_doc(name), "provenance": DOC_PROVENANCE}
def sealed_of(name: str) -> api._Sealed:
"""金样记下的封存范围(6.6):内存里拼,不读运行记录。"""
s = report(name)["sealed"]
return api._Sealed(run_id=report(name)["run_id"], graph={}, seal=dict(s["seal"]), events=[],
queries={k: dict(v) for k, v in s["queries"].items()},
schemas={k: dict(v) for k, v in s["schemas"].items()})
class _StubMasks:
"""金样生成时 `_Masks` 的替身:预填 memo(源名 → (遮罩列小写, 数据源还在)),不查库。"""
def __init__(self, memo: dict[str, tuple[set[str], bool]]) -> None:
self._memo = memo
async def of(self, source: Any) -> tuple[set[str], bool]:
name = str(source or "")
if not name:
return set(), True
return self._memo.get(name, (set(), True))
def stub_masks(monkeypatch, name: str) -> None:
memo = {k: ({c.lower() for c in v}, True) for k, v in report(name)["masks"].items()}
monkeypatch.setattr(api, "_Masks", lambda: _StubMasks(memo))
def cell_segments(doc: dict[str, Any]) -> list[dict[str, Any]]:
return [s for s in iter_segments(doc) if (s.get("cite") or {}).get("kind") == "cell"]
async def chains(doc: dict[str, Any], name: str) -> dict[str, list[dict[str, Any]]]:
sealed = sealed_of(name)
out = {}
for seg in cell_segments(doc):
out[seg["id"]] = await api._chain(seg, doc, sealed, report(name)["node_id"])
# 金样存的是 JSON:元组、浮点数都按 JSON 往返一次再比
return json.loads(json.dumps(out, ensure_ascii=False))
def excerpts(doc: dict[str, Any], name: str, *, loader: Any = None, masked: Any = None) -> dict[str, str]:
return judge.prepare(doc, report(name)["catalog"], loader=loader or artifact_store.load, masked=masked).excerpts
def verified(doc: dict[str, Any], name: str) -> dict[str, Any]:
v = verify_doc(doc, report(name)["catalog"], loader=artifact_store.load)
return {"ok": v["ok"], "stats": v["stats"], "violations": v["violations"]}
def state_mismatches(v: dict[str, Any]) -> list[dict[str, Any]]:
return [x for x in v["violations"] if x.get("code") == "state_mismatch"]
def split_new_lines(text: str) -> tuple[list[str], list[str]]:
"""查询摘录 → (去掉来历行之后的行, 来历行)。来历行只许出现在「SQL:」那一行之后、「列:」那一行之前(3.2)。"""
lines = text.split("\n")
sql = [i for i, line in enumerate(lines) if line.startswith("SQL:")]
cols = [i for i, line in enumerate(lines) if line.startswith("列:")]
assert len(sql) == 1 and len(cols) == 1 and sql[0] < cols[0], f"查询摘录的形状变了:\n{text}"
return lines[:sql[0] + 1] + lines[cols[0]:], lines[sql[0] + 1:cols[0]]
def golden_tables(schema: dict[str, Any]) -> dict[str, list[str]]:
"""冻结表结构 → {表: [列…]}(按定义顺序),direct_select.tables_in 的入参(3.3)。"""
return {name: [str(c.get("name")) for c in (meta.get("columns") or [])]
for name, meta in (schema.get("tables") or {}).items()}
def expected_judge_lines(name: str, alias: str) -> list[str]:
"""契约(3.3):_query_excerpt 先用 tables_in 算出 SQL 用到的表,再调 judge_lines,把返回的行原样插进摘录。
金样的报告没有遮罩(masks 为空、快照没有 mask_columns),hidden 为空集。"""
entry = report(name)["catalog"][alias]
query = artifact_store.load(entry["artifact"])
schema = artifact_store.load(query["schema_artifact"]) if query.get("schema_artifact") else None
tables = direct_select_mod.tables_in(query["sql"], golden_tables(schema or {}))
return provenance_mod.judge_lines(query, schema, artifact_store.load, tables=tables, hidden=set())
# ==========================================================================
# 金样本身
# ==========================================================================
def test_golden_records_three_kinds_of_sources(rebuilt):
"""金样的三份报告正好是规格要的三种源(6.6):配方源有 import_manifests、简单导入的表结构快照没有 import_mode、
手工源的查询快照没有 data_version。重建出的工件 id 全部等于金样的键(夹具里已逐个断言)。"""
arts = GOLDEN["artifacts"]
assert set(rebuilt) == set(arts) and all(k == v for k, v in rebuilt.items())
assert set(GOLDEN["reports"]) == set(REPORTS)
rq = arts[report("recipe")["artifacts"][0]]
rs = arts[rq["schema_artifact"]]
assert rq.get("data_version") and rs.get("import_mode") == "recipe" and len(rs["import_manifests"]) == 1
sq = arts[report("simple")["artifacts"][0]]
ss = arts[sq["schema_artifact"]]
assert sq.get("data_version") and "import_mode" not in ss and ss["tables"]["月报"]["comment"] == UNSHAPED_NOTE
mq = arts[report("manual")["artifacts"][0]]
assert "data_version" not in mq and "schema_artifact" not in mq
for name in REPORTS:
assert cell_segments(old_doc(name)), name
assert report(name)["outputs"]["excerpts"], name
assert report(name)["outputs"]["chain"], name
# ==========================================================================
# 期 4 之前的文档(没有标记):一个字都不变,也不碰新模块
# ==========================================================================
@pytest.mark.parametrize("name", REPORTS)
def test_old_documents_verify_exactly_as_before(name, tripwire):
"""① verify_doc 逐字等于金样,没有 state_mismatch(verify_doc 不看未知的顶层键,也不读裁判摘录,1.4)。"""
got = verified(old_doc(name), name)
assert got == report(name)["outputs"]["verify_doc"]
assert state_mismatches(got) == []
assert tripwire == []
@pytest.mark.parametrize("name", REPORTS)
def test_old_documents_give_the_same_judge_excerpts(name, tripwire):
"""② 裁判摘录逐字等于金样(摘录进断点指纹和逐句复用键,变一个字已判过的句子就不复用了,3.1)。"""
assert excerpts(old_doc(name), name) == report(name)["outputs"]["excerpts"]
assert tripwire == []
@pytest.mark.parametrize("name", REPORTS)
async def test_old_documents_give_the_same_query_steps(name, monkeypatch, tripwire):
"""③ 单元格片段的出处链(查询步骤)逐字等于金样:没有 provenance 键,界面不多发请求(2.8.2)。"""
stub_masks(monkeypatch, name)
got = await chains(old_doc(name), name)
assert got == report(name)["outputs"]["chain"]
assert not any("provenance" in step for steps in got.values() for step in steps)
assert tripwire == []
@pytest.mark.parametrize("name", REPORTS)
async def test_old_documents_answer_legacy_doc(name, tripwire):
"""推断来源接口对老文档一律回 none · legacy_doc(R0),不标红、没有数据版本;cell 照样给(与 status 无关,2.8.1)。
R0 在一切取证之前判,所以连查询快照都不用取,也不碰新模块。"""
doc = old_doc(name)
rep = report(name)
chosen = api._Report(node_id=rep["node_id"], doc_artifact=rep["doc_artifact"], ok=True, repairs=0, stats=None,
doc=doc, hash_ok=True, fields=[])
for seg in cell_segments(doc):
out = await api.segment_provenance(chosen, seg, sealed_of(name), masks=_StubMasks({}))
assert contract_problems(out) == []
assert (out.status, out.reason.code, out.alert, out.version, out.cell_source, out.checks) == \
("none", "legacy_doc", None, None, None, [])
assert out.sealed is True and out.segment == seg["id"]
loc = seg["cite"]["locator"]
assert out.cell == CellRef(alias=seg["cite"]["alias"], row=loc["row"], column=loc["column"],
artifact=rep["catalog"][seg["cite"]["alias"]]["artifact"])
assert out.report.node_id == rep["node_id"] and out.report.doc_artifact == rep["doc_artifact"]
assert tripwire == []
async def test_old_document_on_demand_judge_ignores_a_broken_manifest(tripwire):
"""A26(金样一侧):老文档走按需裁判的 loader(`_sealed_loader`,chain 按文档标记取,老文档为假),把表结构快照
列出的导入清单文件改坏一个字节,摘录仍逐字等于金样:chain 为假时根本不去取清单(2.8.3)。"""
name = "recipe"
doc = old_doc(name)
rep = report(name)
schema = artifact_store.load(artifact_store.load(rep["artifacts"][0])["schema_artifact"])
[manifest] = schema["import_manifests"]
path = artifact_store._path_of(manifest)
raw = path.read_bytes()
at = raw.index(b'"seq":')
path.write_bytes(raw[:at] + b'"seQ":' + raw[at + 6:])
with pytest.raises(ValueError):
artifact_store.load(manifest)
# _judge_now 的同一套取法:loader 按文档标记决定扩不扩展、遮罩按这个 loader 查
loader = api._sealed_loader(sealed_of(name), chain=doc.get("provenance") == DOC_PROVENANCE)
assert loader(manifest) is None
masked = await judge.source_masks(rep["catalog"], loader=loader)
assert not any(masked.values()), f"金样的数据源不该设遮罩:{masked}"
assert excerpts(doc, name, loader=loader, masked=masked) == rep["outputs"]["excerpts"]
assert tripwire == []
# ==========================================================================
# 带上标记之后:非上传源一字不差;上传源只在约定的位置多出约定的东西
# ==========================================================================
async def test_marked_manual_document_is_unchanged(monkeypatch):
"""手工源(查询快照没有 data_version)的文档加上标记:①②③仍逐字等于金样(1.4「非上传源」,3.3)。"""
name = "manual"
doc = marked_doc(name)
out = report(name)["outputs"]
assert verified(doc, name) == out["verify_doc"]
assert excerpts(doc, name) == out["excerpts"]
stub_masks(monkeypatch, name)
assert await chains(doc, name) == out["chain"]
@pytest.mark.parametrize("name", UPLOADS)
def test_marked_upload_documents_still_verify_the_same(name):
"""标记写在文档顶层、参与内容哈希,但 verify_doc 不看它:ok、stats、violations 与金样逐字相同,没有 state_mismatch。"""
got = verified(marked_doc(name), name)
assert got == report(name)["outputs"]["verify_doc"] and state_mismatches(got) == []
@pytest.mark.parametrize("name", UPLOADS)
async def test_marked_upload_query_steps_only_gain_the_hint(name, monkeypatch):
"""③ 上传表格的单元格片段:查询步骤只多一个 `provenance: true`,其余每个键逐字等于金样(2.8.2)。"""
stub_masks(monkeypatch, name)
got = await chains(marked_doc(name), name)
want = report(name)["outputs"]["chain"]
assert set(got) == set(want)
for sid, steps in got.items():
assert len(steps) == len(want[sid]) == 1, sid
for step, old in zip(steps, want[sid]):
assert set(step) - set(old) == {"provenance"} and set(old) <= set(step), (sid, set(step) ^ set(old))
assert step["provenance"] is True
assert {k: v for k, v in step.items() if k != "provenance"} == old, sid
@pytest.mark.parametrize("name", UPLOADS)
def test_marked_upload_excerpts_keep_the_golden_text_around_the_new_lines(name):
"""② 上传表格加上标记后,摘录里新增的东西只许插在查询摘录「SQL:」那一行之后、「列:」那一行之前:去掉这段之后
逐字等于金样;表名实体等其余摘录逐字等于金样(3.2 的位置约定)。新行一律以 3.2 的几种行首开头。"""
got = excerpts(marked_doc(name), name)
want = report(name)["outputs"]["excerpts"]
assert set(got) == set(want)
for alias, text in got.items():
if not alias.startswith("Q"):
assert text == want[alias], alias
continue
kept, new = split_new_lines(text)
golden_kept, golden_new = split_new_lines(want[alias])
assert golden_new == [], "金样的查询摘录在 SQL 与「列:」之间不该有别的行"
assert kept == golden_kept, alias
assert all(line.startswith(NEW_LINE_HEADS) for line in new), (alias, new)
def test_marked_recipe_excerpts_gain_the_provenance_lines():
"""② 配方源的文档加上标记:两条查询摘录都多出 3.2 的来历行,内容就是 judge_lines 对同样输入给出的行,原样插在
「SQL:」与「列:」之间(3.3)。断言几样确定的内容:
- 来源行:上传的表格、每期替换、第 1 次导入、统计期取自 客流汇总!B2、文件名与 sha256 前 12 位、区域(S9);
- 口径:SQL 用到的表的说明(冻结表结构的 comment 原文)、查询结果列的说明;
- 区域外文字:金样的清单是期 4 之前导入的,OutsideText 没有 hidden 字段,只计条数、不送全文(调整 10);
- 没有接受理由(D00 没有接受)。"""
name = "recipe"
got = excerpts(marked_doc(name), name)
rep = report(name)
query = artifact_store.load(rep["catalog"]["Q1"]["artifact"])
schema = artifact_store.load(query["schema_artifact"])
manifest = artifact_store.load(schema["import_manifests"][0])
for alias, table in (("Q1", "日客流"), ("Q2", "时段客流")):
_, new = split_new_lines(got[alias])
assert new, f"{alias} 的摘录没有来历行"
assert new == expected_judge_lines(name, alias), alias
source = new[0]
assert source.startswith("来源:上传的表格,每期替换,第 1 次导入:统计期 2026-08-01 至 2026-08-31(取自 客流汇总!B2)")
assert f"文件「{manifest['file']['name']}」(sha256 {manifest['file']['raw_sha256'][:12]})" in source
assert "区域 客流汇总!" in source
comment = schema["tables"][table]["comment"]
assert f"口径:表 {table} 的说明:{comment}" in new
assert not any(line.startswith("已接受") for line in new)
assert "另有 1 处区域外文字未列出(所在行列被隐藏,或导入时未记录是否隐藏),见证据面板" in new
assert not any("客流汇总表" in line for line in new), "隐藏与否未记录的区域外文字不该送全文"
_, new = split_new_lines(got["Q1"])
for col in ("日期", "全日客流", "分区甲"):
note = next(c["comment"] for c in schema["tables"]["日客流"]["columns"] if c["name"] == col)
assert f"口径:列 {col} 的说明:{note}" in new
assert len("\n".join(new)) <= 1200
def test_marked_simple_upload_excerpt_gains_only_the_caliber_line():
"""② 简单导入的文档加上标记:只多口径行(调整 6:未规整的表写 UNSHAPED_NOTE,裁判才知道不能直接对列求和);
没有导入清单,来源、已接受、区域外文字三类都不出。"""
name = "simple"
got = excerpts(marked_doc(name), name)
_, new = split_new_lines(got["Q1"])
assert new == expected_judge_lines(name, "Q1")
assert f"口径:表 月报 的说明:{UNSHAPED_NOTE}" in new
assert not any(line.startswith(("来源:", "已接受", "区域外")) for line in new), new
@@ -351,6 +351,36 @@ def test_outside_text_with_digits_needs_confirmation():
assert (f"{SHEET}!B32", "text_digits", False) in [(o.cell, o.kind, o.period_source) for o in ex.outside_text]
def test_outside_text_records_whether_its_row_or_column_is_hidden():
"""期 4(P4-SPEC 7.3、调整 10,评审 hid.py 的复现):区域外文字在隐藏行、隐藏列里时 hidden 为 True,可见的为
False;带附加文字的统计期格(period_source)同样写。裁判摘录只送 hidden is False 的项,隐藏行里的内部备注
不能多一条经裁判外发的通道。隐藏行列在认领区域之外,不触发 hidden_rows / hidden_cols。"""
def edit(ws, info):
ws["B32"] = "注:8月15日闸机故障,当日客流为估算值" # 可见、含数字
ws["B33"] = "内部:本月补贴 45678 元" # 隐藏行
ws.row_dimensions[33].hidden = True
ws["H34"] = "内部备注:分区甲口令 abc123" # 区域右侧、区域下方的隐藏列
ws.column_dimensions["H"].hidden = True
ws["B3"] = "客流汇总表(2026年8月)" # 可见的统计期附加文字
ws["B35"] = "注:2026年8月数据为初步统计" # 隐藏行里的统计期附加文字
ws.row_dimensions[35].hidden = True
ex = run(flow_book(days=3, edit=edit))
assert ex.ok, [(p.code, p.message) for p in ex.problems]
assert not {"hidden_rows", "hidden_cols"} & codes(ex)
got = {o.cell: (o.kind, o.period_source, o.hidden) for o in ex.outside_text}
assert got == {
f"{SHEET}!B3": ("text_digits", True, False),
f"{SHEET}!B32": ("text_digits", False, False),
f"{SHEET}!B33": ("text_digits", False, True),
f"{SHEET}!H34": ("text_digits", False, True),
f"{SHEET}!B35": ("text_digits", True, True),
}
# 回执(asdict)里一律带 hidden 键:导入清单按它判断,老清单没有这个键才是「未记录」
from dataclasses import asdict
assert all(isinstance(o["hidden"], bool) for o in asdict(ex)["outside_text"])
def test_outside_number_is_structure_and_merges_per_row():
def edit(ws, info):
ws["B1"] = 987654 # 日期表头以上:区域外
+10 -1
View File
@@ -5,7 +5,7 @@ import type {
} from '../types'
import type {
Approval, AutofixResult, Conversation, ConversationDetail, ConversationTurn, CustomTool, DataSource, EvidenceGraph,
EvidenceAudit, EvidenceJudgeResult, EvidenceSegmentDetail, GraphSpec, KbDocument, MemoryItem, Provider, PublishCheck, PublishLevel, ReviewResult, Run,
EvidenceAudit, EvidenceJudgeResult, EvidenceProvenance, EvidenceSegmentDetail, GraphSpec, KbDocument, MemoryItem, Provider, PublishCheck, PublishLevel, ReviewResult, Run,
RunEvent, RunStatus, Skill, ToolChange, ToolInfo, ToolTrust, UpgradeResult, UploadDecision, UploadMixedColumn, UploadResult,
UploadShapeReason, ValidationIssue, VarIssue, Variable, Workflow, WorkflowVersion,
} from '../types'
@@ -891,6 +891,15 @@ export const api = {
*/
judge: (runId: string, body: { units: string[]; report?: string }) =>
post<EvidenceJudgeResult>(`/runs/${encodeURIComponent(runId)}/evidence/judge`, body),
/**
* 推断的来源(期 4):上传表格的一个单元格来自原表的哪一格、这次查询查的是哪一版数据、相关核对。按需算,
* 推断不出来不是错误(200,结论在 status、reason、alert 里)。只在片段接口的查询步骤带 provenance: true 时请求;
* 老服务端没有这个接口,回 404 且没有 code。report 同片段接口
*/
provenance: (runId: string, segmentId: string, opts?: RequestOptions & { report?: string }) =>
get<EvidenceProvenance>(
`/runs/${encodeURIComponent(runId)}/evidence/segments/${encodeURIComponent(segmentId)}/provenance${qs({ report: opts?.report })}`,
opts),
/**
* 导出审计表:?format=json / csv 作为附件下载。回来的是文件本身和后端给的文件名
* (Content-Disposition;没给就用 evidence-<运行号前 8 位>.<格式>)
+20
View File
@@ -30,6 +30,7 @@ import evidenceQuery from '../run/__tests__/evidence-query.json'
import evidenceEntity from '../run/__tests__/evidence-entity.json'
import evidenceJudge from '../run/__tests__/evidence-judge.json'
import evidenceVerdict from '../run/__tests__/evidence-verdict.json'
import evidenceProvenance from '../run/__tests__/evidence-provenance.json'
import { EvidenceGuessView } from '../run/EvidenceGuess'
import { IssuanceBanner } from '../run/AssistantStream'
import mdPinned from '../run/__tests__/markdown-pinned.json'
@@ -207,6 +208,11 @@ const EV_JUDGE_PLAIN = evidenceJudge.plain.doc as unknown as EvidenceDocData
*/
const EV_VERDICT = evidenceVerdict.formal.doc as unknown as EvidenceDocData
const EV_VERDICT_EXPLORE = evidenceVerdict.explore.doc as unknown as EvidenceDocData
/**
* Excel 导入期 4:上传表格的单元格追到原表格子(evidence-provenance.json)。文档由 compose_doc 真跑、带文档标记;
* 片段接口、推断来源接口、导入清单的工件都由 check-evidence 用 page.route 伪造(按 runId 认出这一份夹具)
*/
const EV_PROVENANCE = evidenceProvenance.doc as unknown as EvidenceDocData
/** 同一份旧契约的出具,去掉位置:老运行就是这样,只能按字符串标 */
const LEGACY_LOOSE = {
matched: EV_LEGACY.matched.map(({ start: _s, end: _e, ...m }) => m),
@@ -431,6 +437,20 @@ function EvidenceOnly() {
))}
</div>
</div>
{/*
Excel 导入期 4:推断的来源。宽栏一份、360 宽的画布右栏一份(栏内展开,不出横向滚动)
*/}
<div className="flex flex-col gap-4" id="evidence-provenance-demo">
<div id="evidence-provenance" className="rounded-lg border p-3">
<div className="mb-1 text-2xs text-faint">上传表格的单元格:数据版本、推断的来源、相关核对</div>
<EvidenceDoc doc={EV_PROVENANCE} artifact={evidenceProvenance.doc_artifact} runId={evidenceProvenance.run_id}
label="answer" tally />
</div>
<div id="evidence-provenance-narrow" className="rounded-lg border bg-panel p-2.5" style={{ width: 360, maxWidth: '100%' }}>
<EvidenceDoc doc={EV_PROVENANCE} artifact={evidenceProvenance.doc_artifact} runId={evidenceProvenance.run_id}
label="answer" dense tally />
</div>
</div>
<div id="evidence-long" className="rounded-lg border p-3">
<div className="mb-1 text-2xs text-faint">长报告:{EV_LONG.blocks.length} 块,块级 content-visibility,超过 3000 字先折叠</div>
<EvidenceDoc doc={EV_LONG} label="长报告" tally />
+74 -1
View File
@@ -341,6 +341,78 @@ export const EVIDENCE_TEXT = {
guessNone: '未找到数值相同的单元格',
guessDiff: (d: string) => `相差 ${d}`,
guessCandidates: '可能来自',
// ---- 期 4:推断的来源(上传表格的单元格追到原表格子,P4-SPEC 4.2)----
// 用词同版本页:不写「快照」「构建」「并集」,写「数据版本」「这一期」「数据文件」;署名一律标「署名(未认证)」。
// 不下钻的原因、标红的提示由服务端给原文(provenance_types.REASON_TEXT / ALERT_TEXT),界面原样显示,不在这里另写
provenance: {
version: '数据版本',
/** 首行。每期替换只有一期,不写期数;导入模式的说法同版本页(VERSIONS_TEXT.modeLabel) */
versionHead: (source: string, mode: string, n: number) => (mode === 'accumulate'
? `数据源「${source}」· 按期累积 · 共 ${formatNumber(n)} 期`
: `数据源「${source}」· 每期替换`),
part: (seq: number, start: string, end: string) => `第 ${formatNumber(seq)} 次导入 · 统计期 ${start} 至 ${end}`,
partNoPeriod: (seq: number) => `第 ${formatNumber(seq)} 次导入 · 统计期未记录`,
/** sha 是原件 sha256 的前 12 位;清单没记哈希时只写文件名 */
file: (name: string, sha: string) => (sha ? `文件「${name}」· sha256 ${sha}` : `文件「${name}」`),
/** S9:回执里块内各区域的外接矩形和排除的行数。排除 0 行、没有记录时不写 */
region: (region: string, excluded: number | null) =>
(excluded ? `区域 ${region} · 排除 ${formatNumber(excluded)} 行` : `区域 ${region}`),
excluded: (n: number) => `排除 ${formatNumber(n)} 行`,
committedAt: (at: string) => `导入于 ${at}`,
/** 查询用到的表(与冻结表结构的交集)。原表写明的合计表另注一句:它不能和明细相加 */
tables: (names: string[]) => `涉及的表:${names.join('、')}`,
totalTable: (name: string) => `${name}(原表写明的合计)`,
hasRow: '这一格所在的一期',
current: '当前状态',
revoked: '这一期的接受已作废',
/** 清除原件、作废接受的记录:时间 · 理由 · 署名(未认证)。缺的不写(时间没记时调用方传空串),署名没填写「未填写」 */
stateDetail: (at: string, reason: string | null, name: string | null) =>
[at, reason ? `理由:${reason}` : '', `署名(未认证):${name || '未填写'}`].filter(Boolean).join(' · '),
/** 这一期写过的接受理由(导入清单里的,是当时的事实)。核对标题取不到时写核对编号 */
acceptedLine: (check: string, reason: string, name: string | null) =>
`已接受:${check} · ${reason} · 署名(未认证):${name || '未填写'}`,
manifest: '查看导入清单',
manifestMasked: '数据源设置了遮罩,导入清单请到数据源卡片上的「版本」查看',
unsealed: '本次运行尚未封存或封存核对未通过,以下内容不计入已封存的证据',
morePeriods: (n: number) => `另有 ${formatNumber(n)} 期`,
title: '推断的来源',
badge: '推断,不属于已封存的证据',
cell: (sheet: string, ref: string) => `工作表「${sheet}」${ref}`,
/** 多期时格子在哪一期的文件里:坐标是那一期原件的坐标,不是别的期的 */
fromPart: (seq: number, file: string | null) =>
(file ? `取自第 ${formatNumber(seq)} 次导入的文件「${file}」` : `取自第 ${formatNumber(seq)} 次导入`),
axisHeader: (ref: string) => `日期取自表头格 ${ref}`,
yearPeriod: (cells: string) => (cells ? `年份取自统计期(${cells})` : '年份取自统计期'),
yearHuman: (name: string) => `年份取自人工录入的统计期(署名(未认证):${name})`,
yearMixed: '这一块的日期表头有的是日期格(自带年份),有的是文本(年份取自统计期),无法确定这一格属于哪一种',
/** 格子原文拿不到时(text 为 null)只写坐标,不拿规范写法冒充原文 */
rowLabel: (ref: string, text?: string | null) => (text ? `行标签 ${ref}「${text}」` : `行标签 ${ref}`),
/** 清单里没有分段标题格的原文,只有配方里的定位文字:照实说是「按配方中的标题定位」 */
sectionTitle: (ref: string, title?: string | null) =>
(title ? `分段标题 ${ref}(按配方中的标题「${title}」定位)` : `分段标题 ${ref}`),
colHeader: (refs: string) => `列表头 ${refs}`,
totalLabel: (ref: string, text?: string | null) => (text ? `合计标签 ${ref}「${text}」` : `合计标签 ${ref}`),
canonical: (raw: string, canon: string) => `原文「${raw}」,按规范写法存为「${canon}」`,
reportedTotal: '原表写明的合计:不要彼此相加,也不要与明细相加',
rawPurged: '原件已清除:坐标来自导入时的记录,无法再对照原件',
mergedFill: '这一块按合并单元格的左上格填充:如果这一格在合并区域内,值取自该区域的左上格',
recheck: '已按主键回查数据文件,值一致',
details: '技术细节',
recheckSql: '回查 SQL',
checks: '相关核对',
row: { passed: '这一行成立', mismatch: '这一行不成立', unverifiable: '这一行含空值,未能核对' } as Record<string, string>,
cellStatus: {
ok: '这一格一致',
unverifiable: '这一格未能核对',
unknown: '本期有格子未能核对,这一格是否在其中无法确定',
not_formula: '这一格是写死的数,不做公式引用核对',
} as Record<string, string>,
partStatus: { passed: '本期通过', mismatch: '本期不成立', unverifiable: '本期未能核对', info: '口径说明' } as Record<string, string>,
accepted: (reason: string, name: string | null) => `已接受,理由:${reason}(署名(未认证):${name || '未填写'})`,
loading: '正在推断来源…',
failed: '未能获取推断的来源',
},
} as const
/** 结论句按裁判结论分成的几堆(lib/evidence 的 claimCounts 数出来的) */
@@ -1074,7 +1146,8 @@ export const VERSIONS_TEXT = {
/** tokens 是 formatTokens 的结果(带单位),cost 是 formatCost 的结果 */
aiUsage: (calls: number, tokens: string, cost: string) => `调用模型 ${formatNumber(calls)} 次,共 ${tokens},约 ${cost}`,
aiNone: '这次导入没有调用模型',
outsideTextNote: '区域外文字只在这里显示,不提供给模型',
/** 期 4 起,带来源标记的报告里,裁判摘录会带上可见单元格里的区域外文字(隐藏行列里的不送) */
outsideTextNote: '区域外文字不导入数据表;核对报告时,裁判可以看到其中可见单元格的文字',
revokedBadge: '接受已作废',
/** 当前版本或回滚目标被别人改过(base_changed、revoke_target_changed):服务端原话说了变的是什么,这里只说列表已刷新 */
refreshHint: '列表已刷新,请重新确认',
+237
View File
@@ -0,0 +1,237 @@
import type { ReactNode } from 'react'
import clsx from 'clsx'
import type { CheckResult, ImportAcceptance, ManifestOut } from '../../types'
import { CopyButton, Notice } from '../../components/ui'
import { formatBytes, formatCost, formatDateTime, formatTokens } from '../../lib/format'
import { RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, VERSIONS_TEXT as VT } from '../../lib/terms'
import { CheckStatusChip } from './ImportReceipt'
import { ReceiptSummary } from './ReceiptBlocks'
// ===========================================================================
// 导入清单的渲染(P3-SPEC 7.3):文件、配方、统计期、核对、接受、确认清单、修改记录、说明、回执摘要、AI 用量,
// 原始清单默认收起。
//
// 从版本页(VersionsDialog)抽出来,证据面板的「数据版本」一节也用它(P4-SPEC 4.1):面板里查看导入清单
// 不能退回整份 JSON 输出——那样遮罩不起作用,64 KB 的清单也没法读。两处的行为必须一致,check-versions 守版本页,
// check-evidence 的 provenance 段守面板
// ===========================================================================
function Block({ id, title, children }: { id: string; title: string; children: ReactNode }) {
return (
<section className="space-y-1.5 rounded-lg border bg-bg p-3 text-xs" data-manifest-block={id}>
<h4 className="font-semibold">{title}</h4>
{children}
</section>
)
}
const short = (sha: unknown) => (typeof sha === 'string' && sha ? sha.slice(0, 8) : '')
const cellsShort = (cells: string[]) => cells.map((c) => c.slice(c.lastIndexOf('!') + 1)).join('、')
const list = <T,>(v: unknown): T[] => (Array.isArray(v) ? (v as T[]) : [])
/**
* 导入清单的各块。content 是服务端存的原样(内容寻址,取回时复验了哈希),字段都按「可能缺」读:期 2 的清单
* 没有修改记录,简单导入只有导入回执。
*
* 清单之外的三样(文件名兜底、原件状态、配方第几版)由调用方给:版本页取自导入记录,证据面板取自推断来源接口的
* 那一期(PartView)。原件状态和配方版次是当前状态,不在清单里
*/
export function ManifestBody({ m, fileName, rawState, recipeSeq }: {
m: ManifestOut
/** 清单没记文件名时显示的名字 */
fileName?: string | null
/** 原件状态(kept / purged / absent),取不到为 null 时不显示 */
rawState?: string | null
/** 配方第几版,取不到为 null 时不显示 */
recipeSeq?: number | null
}) {
const c = m.content ?? {}
const B = VT.manifestBlocks
const simple = m.kind === 'build_report'
const receipt = simple ? c : c.receipt
const checks = list<CheckResult>(c.checks)
const titleOf = (id: string) => checks.find((x) => x.id === id)?.title
const accepts: ImportAcceptance[] = [
...list<ImportAcceptance>(c.acceptances?.overrides).map((a) => ({ ...a, kind: 'override' })),
...list<ImportAcceptance>(c.acceptances?.waivers).map((a) => ({ ...a, kind: 'waiver' })),
]
const confirmations = list<{ id: string; label: string; at?: string }>(c.confirmations)
const edits = Array.isArray(c.edits) ? list<Record<string, any>>(c.edits) : null
const rendered = Object.entries((c.notes?.rendered ?? {}) as Record<string, { comment?: string; columns?: Record<string, string> }>)
const usage = list<Record<string, any>>(c.ai?.usage)
const tokens = usage.reduce((n, u) => n + (Number(u.input_tokens) || 0) + (Number(u.output_tokens) || 0), 0)
const cost = usage.reduce((n, u) => n + (Number(u.cost_usd) || 0), 0)
const period = c.period as { start?: string; end?: string; source?: string; cells?: string[]; signed_by?: string | null } | null
const outside = list(receipt?.outside_text)
return (
<div className="space-y-2.5" data-manifest={m.kind} data-verified={m.verified ? 'true' : 'false'}>
{simple
? <Notice tone="info">{VT.manifestSimple}</Notice>
: !m.verified && <Notice tone="err" attr={{ 'data-manifest-unverified': '' }}>{VT.manifestUnverified}</Notice>}
{!simple && (
<>
<Block id="file" title={B.file}>
<div className="flex flex-wrap items-baseline gap-x-3 gap-y-1">
<span className="mono break-all">{c.file?.name || fileName}</span>
{c.file?.size != null && <span className="tnum text-dim">{formatBytes(Number(c.file.size))}</span>}
{short(c.file?.raw_sha256) && <span className="mono text-dim">{VT.sha(short(c.file?.raw_sha256))}</span>}
{rawState != null && <span className="chip">{RAW_STATE_LABEL[rawState] ?? rawState}</span>}
</div>
{c.signed_by?.name && <div className="text-2xs text-faint">{VT.signedBy(c.signed_by.name)}</div>}
</Block>
<Block id="recipe" title={B.recipe}>
<div className="flex flex-wrap items-baseline gap-x-3 gap-y-1">
{recipeSeq != null && <span>{VT.recipeSeq(recipeSeq)}</span>}
{c.recipe?.origin && <span className="text-dim">{RECIPE_ORIGIN_LABEL[c.recipe.origin] ?? c.recipe.origin}</span>}
{short(c.recipe?.sha256) && <span className="mono text-dim">{VT.sha(short(c.recipe?.sha256))}</span>}
</div>
{c.recipe?.canonical && <RawJson summary={VT.recipeJson} value={c.recipe.canonical} attr="data-recipe-json" />}
</Block>
<Block id="period" title={B.period}>
{period?.start && period?.end ? (
<div className="space-y-0.5">
<div className="tnum">{VT.period(period.start, period.end)}</div>
<div className="text-2xs text-faint">
{period.source === 'human'
? `${VT.periodHuman},${VT.signedBy(period.signed_by || VT.unsigned)}`
: period.cells?.length ? RECIPE_TEXT.periodFromCells(cellsShort(period.cells)) : ''}
</div>
</div>
) : <p className="text-faint">{VT.periodUnknown}</p>}
</Block>
<Block id="checks" title={B.checks}>
{checks.length ? (
<ul className="space-y-1.5">
{checks.map((k) => (
<li key={k.id} className="space-y-1 rounded-md border px-2.5 py-1.5" data-manifest-check={k.id} data-status={k.status}>
<div className="flex flex-wrap items-center gap-2">
<CheckStatusChip status={k.status} />
<span className="mono text-faint">{k.id}</span>
<span className="min-w-0 flex-1">{k.title}</span>
{/* 计数照试运行回执的规则(ImportReceipt 的 CheckRow):「说明」类的 failed 是预期内的不相等个数,不写 */}
{!!k.checked && k.status !== 'info' && (
<span className="tnum text-2xs text-faint" data-check-counts="">
{VT.checkCounts(k.checked, k.failed ?? 0, k.unverifiable ?? 0)}
</span>
)}
</div>
{/* 清单是当证据核对的,细节不截断:服务端每个核对最多逐条列 20 格,其余写成「另有 N 格不一致」
「N 格无法核对:…」,截掉的正是这几行 */}
{!!k.details?.length && (
<ul className="space-y-0.5 text-2xs leading-relaxed text-dim" data-check-details={k.details.length}>
{k.details.map((d, i) => <li key={i}>{d}</li>)}
</ul>
)}
{k.sql && <RawJson summary={VT.sql} value={k.params?.length ? `${k.sql}\n-- ${JSON.stringify(k.params)}` : k.sql} attr="data-check-sql" />}
</li>
))}
</ul>
) : <p className="text-faint">{VT.checksNone}</p>}
</Block>
<Block id="acceptances" title={B.acceptances}>
{accepts.length ? (
<ul className="space-y-1">
{accepts.map((a, i) => (
<li key={`${a.check_id}-${i}`} className="space-y-0.5 rounded-md border px-2.5 py-1.5" data-manifest-acceptance={a.check_id}>
<div className="flex flex-wrap items-baseline gap-x-2">
<span className="chip">{VT.acceptKind[a.kind] ?? VT.acceptKind.override}</span>
<span className="min-w-0 flex-1 break-words">
{VT.acceptRow(titleOf(a.check_id) ? `${a.check_id} ${titleOf(a.check_id)}` : a.check_id, a.reason)}
</span>
</div>
<div className="flex flex-wrap gap-x-2 text-2xs text-faint">
<span>{VT.signedBy(a.signed_by || VT.unsigned)}</span>
{a.at && <span className="tnum">{formatDateTime(a.at)}</span>}
</div>
</li>
))}
</ul>
) : <p className="text-faint">{VT.acceptancesNone}</p>}
</Block>
<Block id="confirmations" title={B.confirmations}>
{confirmations.length ? (
<ul className="list-inside list-disc space-y-0.5 text-dim">
{confirmations.map((x, i) => <li key={`${x.id}-${i}`}>{x.label}</li>)}
</ul>
) : <p className="text-faint">{VT.confirmationsNone}</p>}
</Block>
<Block id="edits" title={B.edits}>
{edits == null
? <p className="text-faint">{VT.editsUnrecorded}</p>
: edits.length ? (
<ul className="space-y-1">
{edits.map((e, i) => (
<li key={`${e.seq ?? i}`} className={clsx('flex flex-wrap items-baseline gap-x-2 rounded-md border px-2.5 py-1.5', e.superseded && 'opacity-60')}
data-manifest-edit={e.kind}>
<span className="chip">{VT.editKind[e.kind] ?? VT.editKind.fix}</span>
<span className="min-w-0 flex-1 break-words">{e.title}</span>
{e.superseded && <span className="text-2xs text-faint">{VT.editSuperseded}</span>}
<span className="text-2xs text-faint">{VT.signedBy(e.signed_by || VT.unsigned)}</span>
{e.at && <span className="tnum text-2xs text-faint">{formatDateTime(e.at)}</span>}
</li>
))}
</ul>
) : <p className="text-faint">{VT.editsNone}</p>}
</Block>
<Block id="notes" title={B.notes}>
{rendered.length ? (
<ul className="space-y-2">
{rendered.map(([table, n]) => (
<li key={table} className="space-y-0.5" data-manifest-note={table}>
<div className="font-medium">{table}</div>
{n.comment && <p className="leading-relaxed text-dim">{n.comment}</p>}
{Object.entries(n.columns ?? {}).length > 0 && (
<ul className="space-y-0.5 text-2xs text-dim">
{Object.entries(n.columns ?? {}).map(([col, text]) => <li key={col}>{col}:{text}</li>)}
</ul>
)}
</li>
))}
</ul>
) : <p className="text-faint">{VT.notesNone}</p>}
</Block>
</>
)}
<section className="space-y-1.5 rounded-lg border bg-bg p-3" data-manifest-block="receipt">
<ReceiptSummary receipt={receipt} />
{outside.length > 0 && <p className="text-2xs text-faint" data-outside-note="">{VT.outsideTextNote}</p>}
</section>
{!simple && (
<Block id="ai" title={B.ai}>
<p className={usage.length ? 'text-dim' : 'text-faint'}>
{usage.length ? VT.aiUsage(usage.length, formatTokens(tokens), formatCost(cost)) : VT.aiNone}
</p>
</Block>
)}
<RawJson summary={VT.manifestRaw} value={c} attr="data-manifest-raw" />
</div>
)
}
/** 等宽的原文(原始清单、配方 JSON、核对的 SQL),默认收起:给要逐字核对的人看,带复制 */
function RawJson({ summary, value, attr }: { summary: string; value: unknown; attr: string }) {
const text = typeof value === 'string' ? value : JSON.stringify(value, null, 2)
return (
<details className="text-2xs" {...{ [attr]: '' }}>
<summary className="cursor-pointer select-none text-faint hover:text-dim">{summary}</summary>
<div className="mt-1.5 flex items-start gap-1.5">
<pre className="mono max-h-80 min-w-0 flex-1 overflow-auto whitespace-pre-wrap break-all rounded-md border bg-[var(--bg)] p-2 leading-relaxed text-dim">
{text}
</pre>
<CopyButton text={text} />
</div>
</details>
)
}
+6 -216
View File
@@ -4,17 +4,16 @@ import { AlertTriangle, ArrowLeft, ChevronRight, FileText, Info, RotateCcw, Scis
import clsx from 'clsx'
import { ApiError, api } from '../../api/client'
import type {
CheckResult, DataSource, ImportAcceptance, ImportRecord, ManifestOut, PeriodRef, RevokePlan, SnapshotOut, SnapshotPart,
DataSource, ImportAcceptance, ImportRecord, ManifestOut, PeriodRef, RevokePlan, SnapshotOut, SnapshotPart,
} from '../../types'
import {
CopyButton, EmptyState, ErrorState, Modal, Notice, Skeleton, TabPanel, Tabs, promptDialog, toast,
EmptyState, ErrorState, Modal, Notice, Skeleton, TabPanel, Tabs, promptDialog, toast,
} from '../../components/ui'
import { localActor } from '../../lib/actor'
import { humanizeError } from '../../lib/errors'
import { formatBytes, formatCost, formatDateTime, formatTime, formatTokens, parseServerTime } from '../../lib/format'
import { RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, VERSIONS_TEXT as VT } from '../../lib/terms'
import { CheckStatusChip } from './ImportReceipt'
import { ReceiptSummary } from './ReceiptBlocks'
import { formatBytes, formatDateTime, formatTime, parseServerTime } from '../../lib/format'
import { RAW_STATE_LABEL, VERSIONS_TEXT as VT } from '../../lib/terms'
import { ManifestBody } from './ManifestBody'
// ===========================================================================
// 版本页(数据源卡片上的「版本」,P3-SPEC 7.8):当前版本的各期、历史版本、全部导入记录三个页签,
@@ -703,216 +702,7 @@ function ManifestView({ sourceId, rec, onBack }: { sourceId: string; rec: Import
? <ErrorState error={error} onRetry={() => setNonce((n) => n + 1)} />
: !m
? <Skeleton rows={5} height={36} gap={8} />
: <ManifestBody m={m} rec={rec} />}
: <ManifestBody m={m} fileName={rec.file_name} rawState={rec.raw_state} recipeSeq={rec.recipe_seq} />}
</div>
)
}
function Block({ id, title, children }: { id: string; title: string; children: ReactNode }) {
return (
<section className="space-y-1.5 rounded-lg border bg-bg p-3 text-xs" data-manifest-block={id}>
<h4 className="font-semibold">{title}</h4>
{children}
</section>
)
}
const short = (sha: unknown) => (typeof sha === 'string' && sha ? sha.slice(0, 8) : '')
const cellsShort = (cells: string[]) => cells.map((c) => c.slice(c.lastIndexOf('!') + 1)).join('、')
const list = <T,>(v: unknown): T[] => (Array.isArray(v) ? (v as T[]) : [])
/**
* 导入清单的各块。content 是服务端存的原样(内容寻址,取回时复验了哈希),字段都按「可能缺」读:期 2 的清单
* 没有修改记录,简单导入只有导入回执
*/
function ManifestBody({ m, rec }: { m: ManifestOut; rec: ImportRecord }) {
const c = m.content ?? {}
const B = VT.manifestBlocks
const simple = m.kind === 'build_report'
const receipt = simple ? c : c.receipt
const checks = list<CheckResult>(c.checks)
const titleOf = (id: string) => checks.find((x) => x.id === id)?.title
const accepts: ImportAcceptance[] = [
...list<ImportAcceptance>(c.acceptances?.overrides).map((a) => ({ ...a, kind: 'override' })),
...list<ImportAcceptance>(c.acceptances?.waivers).map((a) => ({ ...a, kind: 'waiver' })),
]
const confirmations = list<{ id: string; label: string; at?: string }>(c.confirmations)
const edits = Array.isArray(c.edits) ? list<Record<string, any>>(c.edits) : null
const rendered = Object.entries((c.notes?.rendered ?? {}) as Record<string, { comment?: string; columns?: Record<string, string> }>)
const usage = list<Record<string, any>>(c.ai?.usage)
const tokens = usage.reduce((n, u) => n + (Number(u.input_tokens) || 0) + (Number(u.output_tokens) || 0), 0)
const cost = usage.reduce((n, u) => n + (Number(u.cost_usd) || 0), 0)
const period = c.period as { start?: string; end?: string; source?: string; cells?: string[]; signed_by?: string | null } | null
const outside = list(receipt?.outside_text)
return (
<div className="space-y-2.5" data-manifest={m.kind} data-verified={m.verified ? 'true' : 'false'}>
{simple
? <Notice tone="info">{VT.manifestSimple}</Notice>
: !m.verified && <Notice tone="err" attr={{ 'data-manifest-unverified': '' }}>{VT.manifestUnverified}</Notice>}
{!simple && (
<>
<Block id="file" title={B.file}>
<div className="flex flex-wrap items-baseline gap-x-3 gap-y-1">
<span className="mono break-all">{c.file?.name || rec.file_name}</span>
{c.file?.size != null && <span className="tnum text-dim">{formatBytes(Number(c.file.size))}</span>}
{short(c.file?.raw_sha256) && <span className="mono text-dim">{VT.sha(short(c.file?.raw_sha256))}</span>}
<span className="chip">{RAW_STATE_LABEL[rec.raw_state] ?? rec.raw_state}</span>
</div>
{c.signed_by?.name && <div className="text-2xs text-faint">{VT.signedBy(c.signed_by.name)}</div>}
</Block>
<Block id="recipe" title={B.recipe}>
<div className="flex flex-wrap items-baseline gap-x-3 gap-y-1">
{rec.recipe_seq != null && <span>{VT.recipeSeq(rec.recipe_seq)}</span>}
{c.recipe?.origin && <span className="text-dim">{RECIPE_ORIGIN_LABEL[c.recipe.origin] ?? c.recipe.origin}</span>}
{short(c.recipe?.sha256) && <span className="mono text-dim">{VT.sha(short(c.recipe?.sha256))}</span>}
</div>
{c.recipe?.canonical && <RawJson summary={VT.recipeJson} value={c.recipe.canonical} attr="data-recipe-json" />}
</Block>
<Block id="period" title={B.period}>
{period?.start && period?.end ? (
<div className="space-y-0.5">
<div className="tnum">{VT.period(period.start, period.end)}</div>
<div className="text-2xs text-faint">
{period.source === 'human'
? `${VT.periodHuman},${VT.signedBy(period.signed_by || VT.unsigned)}`
: period.cells?.length ? RECIPE_TEXT.periodFromCells(cellsShort(period.cells)) : ''}
</div>
</div>
) : <p className="text-faint">{VT.periodUnknown}</p>}
</Block>
<Block id="checks" title={B.checks}>
{checks.length ? (
<ul className="space-y-1.5">
{checks.map((k) => (
<li key={k.id} className="space-y-1 rounded-md border px-2.5 py-1.5" data-manifest-check={k.id} data-status={k.status}>
<div className="flex flex-wrap items-center gap-2">
<CheckStatusChip status={k.status} />
<span className="mono text-faint">{k.id}</span>
<span className="min-w-0 flex-1">{k.title}</span>
{/* 计数照试运行回执的规则(ImportReceipt 的 CheckRow):「说明」类的 failed 是预期内的不相等个数,不写 */}
{!!k.checked && k.status !== 'info' && (
<span className="tnum text-2xs text-faint" data-check-counts="">
{VT.checkCounts(k.checked, k.failed ?? 0, k.unverifiable ?? 0)}
</span>
)}
</div>
{/* 清单是当证据核对的,细节不截断:服务端每个核对最多逐条列 20 格,其余写成「另有 N 格不一致」
「N 格无法核对:…」,截掉的正是这几行 */}
{!!k.details?.length && (
<ul className="space-y-0.5 text-2xs leading-relaxed text-dim" data-check-details={k.details.length}>
{k.details.map((d, i) => <li key={i}>{d}</li>)}
</ul>
)}
{k.sql && <RawJson summary={VT.sql} value={k.params?.length ? `${k.sql}\n-- ${JSON.stringify(k.params)}` : k.sql} attr="data-check-sql" />}
</li>
))}
</ul>
) : <p className="text-faint">{VT.checksNone}</p>}
</Block>
<Block id="acceptances" title={B.acceptances}>
{accepts.length ? (
<ul className="space-y-1">
{accepts.map((a, i) => (
<li key={`${a.check_id}-${i}`} className="space-y-0.5 rounded-md border px-2.5 py-1.5" data-manifest-acceptance={a.check_id}>
<div className="flex flex-wrap items-baseline gap-x-2">
<span className="chip">{VT.acceptKind[a.kind] ?? VT.acceptKind.override}</span>
<span className="min-w-0 flex-1 break-words">
{VT.acceptRow(titleOf(a.check_id) ? `${a.check_id} ${titleOf(a.check_id)}` : a.check_id, a.reason)}
</span>
</div>
<div className="flex flex-wrap gap-x-2 text-2xs text-faint">
<span>{VT.signedBy(a.signed_by || VT.unsigned)}</span>
{a.at && <span className="tnum">{formatDateTime(a.at)}</span>}
</div>
</li>
))}
</ul>
) : <p className="text-faint">{VT.acceptancesNone}</p>}
</Block>
<Block id="confirmations" title={B.confirmations}>
{confirmations.length ? (
<ul className="list-inside list-disc space-y-0.5 text-dim">
{confirmations.map((x, i) => <li key={`${x.id}-${i}`}>{x.label}</li>)}
</ul>
) : <p className="text-faint">{VT.confirmationsNone}</p>}
</Block>
<Block id="edits" title={B.edits}>
{edits == null
? <p className="text-faint">{VT.editsUnrecorded}</p>
: edits.length ? (
<ul className="space-y-1">
{edits.map((e, i) => (
<li key={`${e.seq ?? i}`} className={clsx('flex flex-wrap items-baseline gap-x-2 rounded-md border px-2.5 py-1.5', e.superseded && 'opacity-60')}
data-manifest-edit={e.kind}>
<span className="chip">{VT.editKind[e.kind] ?? VT.editKind.fix}</span>
<span className="min-w-0 flex-1 break-words">{e.title}</span>
{e.superseded && <span className="text-2xs text-faint">{VT.editSuperseded}</span>}
<span className="text-2xs text-faint">{VT.signedBy(e.signed_by || VT.unsigned)}</span>
{e.at && <span className="tnum text-2xs text-faint">{formatDateTime(e.at)}</span>}
</li>
))}
</ul>
) : <p className="text-faint">{VT.editsNone}</p>}
</Block>
<Block id="notes" title={B.notes}>
{rendered.length ? (
<ul className="space-y-2">
{rendered.map(([table, n]) => (
<li key={table} className="space-y-0.5" data-manifest-note={table}>
<div className="font-medium">{table}</div>
{n.comment && <p className="leading-relaxed text-dim">{n.comment}</p>}
{Object.entries(n.columns ?? {}).length > 0 && (
<ul className="space-y-0.5 text-2xs text-dim">
{Object.entries(n.columns ?? {}).map(([col, text]) => <li key={col}>{col}:{text}</li>)}
</ul>
)}
</li>
))}
</ul>
) : <p className="text-faint">{VT.notesNone}</p>}
</Block>
</>
)}
<section className="space-y-1.5 rounded-lg border bg-bg p-3" data-manifest-block="receipt">
<ReceiptSummary receipt={receipt} />
{outside.length > 0 && <p className="text-2xs text-faint" data-outside-note="">{VT.outsideTextNote}</p>}
</section>
{!simple && (
<Block id="ai" title={B.ai}>
<p className={usage.length ? 'text-dim' : 'text-faint'}>
{usage.length ? VT.aiUsage(usage.length, formatTokens(tokens), formatCost(cost)) : VT.aiNone}
</p>
</Block>
)}
<RawJson summary={VT.manifestRaw} value={c} attr="data-manifest-raw" />
</div>
)
}
/** 等宽的原文(原始清单、配方 JSON、核对的 SQL),默认收起:给要逐字核对的人看,带复制 */
function RawJson({ summary, value, attr }: { summary: string; value: unknown; attr: string }) {
const text = typeof value === 'string' ? value : JSON.stringify(value, null, 2)
return (
<details className="text-2xs" {...{ [attr]: '' }}>
<summary className="cursor-pointer select-none text-faint hover:text-dim">{summary}</summary>
<div className="mt-1.5 flex items-start gap-1.5">
<pre className="mono max-h-80 min-w-0 flex-1 overflow-auto whitespace-pre-wrap break-all rounded-md border bg-[var(--bg)] p-2 leading-relaxed text-dim">
{text}
</pre>
<CopyButton text={text} />
</div>
</details>
)
}
+14
View File
@@ -27,6 +27,7 @@ import type {
} from '../types'
import { ArtifactViewer, ResultTable } from './AssistantStream'
import { useEvidenceHost } from './evidenceHost'
import { EvidenceProvenance } from './EvidenceProvenance'
import { CopyChip } from './Markdown'
/**
@@ -58,6 +59,9 @@ import { CopyChip } from './Markdown'
* 裁判拆档(JR):证据不足的写明缺什么;裁判模型和写作模型相同、模型不在价格表里时在「模型的解释」下提醒;
* 表名实体步骤列出字段清单;讲方法、讲结构的句子在「挂的依据」下直接列出 SQL 和字段清单。
*
* Excel 导入期 4:上传表格的单元格多三节——数据版本、推断的来源、相关核对(EvidenceProvenance)。只在片段是单元格
* 引用、且片段接口的查询步骤带 provenance: true 时请求推断来源接口;推断来源的缓存只在面板开着期间有效。
*
* 四种摆法:side 从右侧弹出(宽屏),drawer 从底部抽出(窄屏),inline 在 360px 的
* 画布右栏里直接栏内展开,dock 放进页面给的一块常驻位置(记录页的「证据」页签)。
* 都不是模态的:开着面板照样能在正文里走。
@@ -82,6 +86,10 @@ export function EvidencePanel({ id, mode, doc, artifact, runId, runClass, view,
onViolations: () => void
}) {
const titleId = `${id}-title`
// 推断的来源只在面板这次打开期间缓存:回答里有当前状态(原件已清除、接受已作废)和取决于数据文件现状的结论,
// 关掉面板(卸载)或换了运行就作废,再打开重新取。同一次打开里来回点同一片段只取一次
const holdProvenance = useEvidence((s) => s.holdProvenance)
useEffect(() => (runId ? holdProvenance(runId) : undefined), [runId, holdProvenance])
const found = useMemo(() => (view.kind === 'seg' ? findSeg(doc, view.id) : null), [doc, view])
const claimAt = useMemo(() => (view.kind === 'unit' ? findUnit(doc, view.id) : null), [doc, view])
const overlay = useVerdicts(runId, doc.node_id || undefined)
@@ -247,6 +255,10 @@ function SegmentBody({ panelId, doc, artifact, runId, runClass, seg, unit, block
&& (cite?.kind === 'cell' || entry?.kind === 'query' || !!queries.length)
const bad = state === 'none' || state === 'suspect' || state === 'unverified'
const closest = closestNames(detail?.closest ?? entityStep?.closest)
// 期 4:推断的来源。条件写死成「片段是单元格引用、它的查询步骤带了提示」:期 4 之前的运行、手工源(查询步骤没有
// 提示)、指标片段(cite.kind 是 metric,哪怕它链上的查询步骤误带了提示)一个请求都不多发。接口答的是另一份报告
// (docForeign)时 detail 为空、链是空的,也就不请求
const provenanceOn = !!runId && cite?.kind === 'cell' && queries.some((q) => q.provenance === true)
// 画布右栏:点开片段时把证据路径交给画布(产出证据的节点实线、报告虚线),链取回来后再补全
const host = useEvidenceHost()
@@ -386,6 +398,8 @@ function SegmentBody({ panelId, doc, artifact, runId, runClass, seg, unit, block
<QueryPart key={`${q.artifact ?? ''}-${i}`} id={queryId(i)} step={q} seal={seal} masked={masked} />
))}
{provenanceOn && runId && <EvidenceProvenance runId={runId} segId={seg.id} report={report} />}
{state === 'deterministic' && !isMetric && !isInput && !isCell && !isEntity && !isQuote && (
<p className="text-dim">{sourceOf(seg, doc) || detail?.note}</p>
)}
+489
View File
@@ -0,0 +1,489 @@
import { useEffect, useId, useState, type ReactNode } from 'react'
import { createPortal } from 'react-dom'
import { AlertTriangle, ChevronRight, FileText } from 'lucide-react'
import clsx from 'clsx'
import { ApiError, api } from '../api/client'
import { ErrorState, Modal, Skeleton } from '../components/ui'
import { NONE, formatDateTime } from '../lib/format'
import { EVIDENCE_TEXT, RAW_STATE_LABEL, VERSIONS_TEXT } from '../lib/terms'
import { ManifestBody } from '../pages/import/ManifestBody'
import { segmentKey, useEvidence } from '../store/evidence'
import type {
EvidenceProvenance as Provenance, ManifestOut, ProvenanceCellSource, ProvenanceCheck, ProvenanceFromCell, ProvenancePart,
ProvenanceVersion,
} from '../types'
import { CopyChip } from './Markdown'
/**
* 证据面板里的「数据版本」「推断的来源」「相关核对」三节(Excel 导入期 4,P4-SPEC 4.1–4.3)。
*
* 上传表格的单元格:这次查询查的是哪一版数据(各期的统计期、文件、原件状态、写过的接受理由),这一格来自原表的
* 哪个工作表、哪一格(日期、年份、行标签、分段标题各取自哪里),以及这一行、这一格的核对结果。
*
* 数据都来自推断来源接口(按需算,面板打开那个片段时才算)。调用方只在片段是单元格引用、且片段接口的查询步骤
* 带 provenance: true 时挂这个组件,所以期 4 之前的运行、手工源、指标片段一个请求都不多发。
*
* 三节分两种性质,界面分开标注:
* - 数据版本来自封存链上的导入清单(内容寻址),不加「推断」标注;
* - 推断的来源、相关核对是先按主键认出这一行再推出来的,挂「推断,不属于已封存的证据」,用虚线左边框和
* 别的几节区分。封存那一行(verdict)不受它影响,推断的来源不计入「已封存 · 核对一致」。
*
* 标题用 <h4>(读屏按标题跳得到),视觉照抄面板里 Part 的小标题;Part 本身不动——改它会改变期 4 之前所有面板的
* DOM。标红的提示用自己的属性 data-ev-prov-alert,不用 data-ev-integrity:现有断言数过后者的个数。
* 折叠、展开不加动效(P4-SPEC 4.3)。
*/
const P = EVIDENCE_TEXT.provenance
/** 小标题的样式:同 EvidencePanel 的 Part(mb-1 text-2xs font-medium text-faint),只是换成真正的标题元素 */
const HEAD = 'text-2xs font-medium text-faint'
/** 期数多于这个数时折叠:只展开这一格所在的一期和最近一期 */
const FOLD_AT = 3
/** 原件 sha256 显示前几位(P4-SPEC 4.1:前 12 位) */
const SHA_PREFIX = 12
/**
* 导入、清除、作废记录的时间。没记(契约里 at 可以是 null)或认不出时给空串,由调用方略去这一项:formatDateTime
* 拿不到时间时返回「—」,它是真值,filter(Boolean) 去不掉,会在「原件已清除 · — · 理由:…」里留下一个孤立的「—」
*/
function stamp(at: string | null | undefined): string {
const text = formatDateTime(at)
return text === NONE ? '' : text
}
export function EvidenceProvenance({ runId, segId, report }: { runId: string; segId: string; report?: string }) {
const key = segmentKey(runId, segId, report)
const slot = useEvidence((s) => s.provenance[key])
const load = useEvidence((s) => s.loadProvenance)
// 槽没了就再取:面板开着时不会被清(清缓存只在最后一个面板关掉以后),这里只是不让一节永远停在「正在推断」
const missing = !slot
useEffect(() => {
if (missing) void load(runId, segId, report)
}, [missing, runId, segId, report, load])
if (!slot || slot.status === 'loading') {
// 取的过程中只画这一节的加载态,不先画三节空壳
return (
<SourceShell state="loading">
<p className="text-dim" data-ev-prov-loading="">{P.loading}</p>
</SourceShell>
)
}
if (slot.status === 'error' || !slot.data || typeof slot.data !== 'object') {
// 老服务端没有这个接口(404、405 且没有机读码):三节都不画,和期 4 之前一样
if (oldServer(slot.error)) return null
return (
<SourceShell state="error">
<p className="text-dim" data-ev-prov-failed="">{P.failed}</p>
</SourceShell>
)
}
// 按片段换实例:展开的「技术细节」「另有 n 期」是这一格的状态,点到别的格子时收起
return <Answer key={key} d={slot.data} />
}
function oldServer(error: unknown): boolean {
return error instanceof ApiError && (error.status === 404 || error.status === 405) && !error.code
}
/** 三节画不画、推断的来源那一节画什么(P4-SPEC 4.1 的渲染矩阵) */
interface Plan {
version: boolean
source: 'none' | 'reason' | 'alert' | 'cell'
checks: boolean
}
const NOTHING: Plan = { version: false, source: 'none', checks: false }
/**
* 渲染矩阵。表外的组合不该出现(服务端的契约检查会拦),出现时按「table_only、无 alert」画:给出数据版本(有的话)
* 和一句原因。alert 只在矩阵里列出的组合下画成红色
*/
function planOf(d: Provenance): Plan {
const code = d.reason?.code
const alert = d.alert?.code
if (d.status === 'none') {
if (!alert && (code === 'legacy_doc' || code === 'not_upload' || code === 'not_cell')) return NOTHING
if (!alert && (code === 'not_sealed' || code === 'simple_upload')) return { version: false, source: 'reason', checks: false }
if (alert === 'manifest_unreadable' || alert === 'chain_mismatch') return { version: false, source: 'alert', checks: false }
} else if (d.status === 'table_only') {
if (alert === 'db_tampered' || alert === 'chain_mismatch') return { version: true, source: 'alert', checks: false }
if (!alert) return { version: true, source: 'reason', checks: false }
} else if (d.status === 'inferred') {
if (!alert && d.cell_source) return { version: true, source: 'cell', checks: true }
}
return { version: true, source: 'reason', checks: false }
}
function Answer({ d }: { d: Provenance }) {
const plan = planOf(d)
const version = plan.version && d.version ? d.version : null
const [manifest, setManifest] = useState<ProvenancePart | null>(null)
if (plan.source === 'none') return null
return (
<>
{version && <VersionPart v={version} sealed={d.sealed !== false} onManifest={setManifest} />}
<SourceShell state={d.status}>
{plan.source === 'alert' && d.alert && (
// 标红:样式同 IntegrityList(失败色的框),放在这一节最前面;有它时不再画 reason.text
<p className="flex items-start gap-1.5 rounded border px-2 py-1 [overflow-wrap:anywhere]" data-ev-prov-alert={d.alert.code}
style={{ color: 'var(--st-failed)', borderColor: 'var(--st-failed)', background: 'var(--st-failed-soft)' }}>
<AlertTriangle size={12} aria-hidden className="mt-0.5 shrink-0" />
<span>{d.alert.text}</span>
</p>
)}
{plan.source === 'reason' && d.reason?.text && (
<p className="text-dim [overflow-wrap:anywhere]" data-ev-prov-reason={d.reason.code}>{d.reason.text}</p>
)}
{plan.source === 'cell' && d.cell_source && <CellPart cs={d.cell_source} version={d.version} />}
</SourceShell>
{plan.checks && Array.isArray(d.checks) && d.checks.length > 0 && <ChecksPart checks={d.checks} />}
{manifest && createPortal(
// 弹窗挂到 body 上:侧边面板是 fixed 的一层,里面的遮罩盖不住整页(同「打开完整快照」)
<ManifestDialog part={manifest} onClose={() => setManifest(null)} />,
document.body,
)}
</>
)
}
/** 「推断,不属于已封存的证据」:推断的来源、相关核对两节的标题旁都挂 */
function Badge() {
return (
<span className="chip shrink-0 text-faint" data-ev-prov-badge=""
style={{ borderColor: 'var(--border-strong)', borderStyle: 'dashed' }}>
{P.badge}
</span>
)
}
/** 推断出来的一节:虚线左边框,标题旁挂「推断」标签 */
function Inferred({ title, children, ...rest }: { title: string; children: ReactNode } & Record<`data-${string}`, string>) {
return (
<div className="border-l-2 border-dashed pl-2" style={{ borderColor: 'var(--border-strong)' }} {...rest}>
<div className="mb-1 flex flex-wrap items-center gap-x-1.5 gap-y-0.5">
<h4 className={HEAD}>{title}</h4>
<Badge />
</div>
{children}
</div>
)
}
/** 推断的来源一节的外壳:data-ev-provenance 写这一节的状态(inferred / table_only / none,取的过程中 loading,出错 error) */
function SourceShell({ state, children }: { state: string; children: ReactNode }) {
return <Inferred title={P.title} data-ev-provenance={state}>{children}</Inferred>
}
// ---------------------------------------------------------------------------
// 数据版本
// ---------------------------------------------------------------------------
function VersionPart({ v, sealed, onManifest }: {
v: ProvenanceVersion; sealed: boolean; onManifest: (part: ProvenancePart) => void
}) {
const [open, setOpen] = useState(false)
const moreId = useId()
const parts = Array.isArray(v.parts) ? v.parts : []
// 多于 3 期:这一格所在的一期和最近一期展开,其余收进「另有 n 期」
const fold = parts.length > FOLD_AT
const latest = parts[parts.length - 1]
const keep = parts.filter((p) => p.has_row || p === latest)
const folded = fold ? parts.length - keep.length : 0
const shown = fold && !open ? keep : parts
const tables = (Array.isArray(v.tables) ? v.tables : []).filter((t) => t && t.name)
return (
<div data-ev-prov-version="">
<h4 className={clsx('mb-1', HEAD)}>{P.version}</h4>
{!sealed && (
// 封存那一行已经照实说了状态,这里只是不让「来自封存链」的说法越界:dim 色,不标红
<p className="mb-1 text-2xs text-dim" data-ev-prov-unsealed="">{P.unsealed}</p>
)}
<p className="[overflow-wrap:anywhere]" data-ev-prov-head="">{P.versionHead(v.source, v.mode, parts.length)}</p>
{tables.length > 0 && (
<p className="mt-0.5 text-2xs text-dim [overflow-wrap:anywhere]" data-ev-prov-tables="">
{P.tables(tables.map((t) => (t.kind === 'reported_total' ? P.totalTable(t.name) : t.name)))}
</p>
)}
<ul className="mt-1 space-y-1.5" id={moreId}>
{shown.map((p) => (
<PartRow key={`${p.import_id}-${p.seq}`} p={p} manifest={v.manifest_view !== false} onManifest={onManifest} />
))}
</ul>
{fold && folded > 0 && (
// 展开后按钮还在、焦点留在它上面(同一个元素),再按一次收起
<button type="button" className="btn btn-xs btn-ghost -ml-1 mt-1" aria-expanded={open} aria-controls={moreId}
data-ev-prov-more="" onClick={() => setOpen((x) => !x)}>
<ChevronRight size={11} aria-hidden className={clsx(open && 'rotate-90')} />
{P.morePeriods(folded)}
</button>
)}
{v.manifest_view === false && (
// 清单里有核对细节里的数、标签原文、区域外文字:数据源设了遮罩时面板自己的入口不多暴露
<p className="mt-1 text-2xs text-dim" data-ev-prov-manifest-masked="">{P.manifestMasked}</p>
)}
</div>
)
}
/** 当前状态(取自导入记录,不在封存范围内):作废、清除后面加这枚小字 */
function Current() {
return <span className="chip ml-1.5 text-faint" data-ev-prov-current="">{P.current}</span>
}
function PartRow({ p, manifest, onManifest }: {
p: ProvenancePart; manifest: boolean; onManifest: (part: ProvenancePart) => void
}) {
const period = p.period && p.period.start && p.period.end ? p.period : null
const purged = p.raw_state === 'purged' || !!p.purged
const sha = typeof p.raw_sha256 === 'string' ? p.raw_sha256.slice(0, SHA_PREFIX) : ''
const committed = stamp(p.committed_at)
const facts = [
p.file_name ? P.file(p.file_name, sha) : '',
p.region ? P.region(p.region, p.excluded_rows) : p.excluded_rows ? P.excluded(p.excluded_rows) : '',
period?.source === 'human' ? `${VERSIONS_TEXT.periodHuman}(${VERSIONS_TEXT.signedBy(period.signed_by || VERSIONS_TEXT.unsigned)})` : '',
// 清除过的另起一行写清除记录和「当前状态」,这里不重复
!purged && p.raw_state ? RAW_STATE_LABEL[p.raw_state] ?? '' : '',
committed ? P.committedAt(committed) : '',
p.recipe_seq != null ? VERSIONS_TEXT.recipeSeq(p.recipe_seq) : '',
p.signed_by ? VERSIONS_TEXT.signedBy(p.signed_by) : '',
].filter(Boolean)
const accepts = Array.isArray(p.acceptances) ? p.acceptances : []
return (
<li className="min-w-0" data-ev-prov-part={String(p.seq)} data-ev-prov-has-row={p.has_row ? '' : undefined}>
<div className="flex flex-wrap items-baseline gap-x-2 gap-y-0.5">
<span className="tnum font-medium">{period ? P.part(p.seq, period.start, period.end) : P.partNoPeriod(p.seq)}</span>
{p.has_row && (
<span className="chip" style={{ color: 'var(--accent)', borderColor: 'var(--accent)' }} data-ev-prov-here="">{P.hasRow}</span>
)}
</div>
{facts.length > 0 && <p className="text-2xs text-dim [overflow-wrap:anywhere]">{facts.join(' · ')}</p>}
{purged && (
<p className="mt-0.5 text-2xs text-dim [overflow-wrap:anywhere]" data-ev-prov-purged="">
{RAW_STATE_LABEL.purged}
{p.purged ? ` · ${P.stateDetail(stamp(p.purged.at), p.purged.reason, p.purged.signed_by)}` : ''}
<Current />
</p>
)}
{p.revoked && (
<p className="mt-0.5 text-2xs [overflow-wrap:anywhere]" style={{ color: 'var(--st-waiting)' }} data-ev-prov-revoked="">
{P.revoked} · {P.stateDetail(stamp(p.revoked.at), p.revoked.reason, p.revoked.signed_by)}
<Current />
</p>
)}
{accepts.length > 0 && (
// 接受理由在清单里,是当时的事实:作废之后照旧列出
<ul className="ml-3 mt-0.5 space-y-0.5 text-2xs text-dim">
{accepts.map((a, i) => (
<li key={`${a.check_id}-${i}`} className="[overflow-wrap:anywhere]" data-ev-prov-acceptance={a.check_id}>
{P.acceptedLine(a.title || a.check_id, a.reason, a.signed_by)}
</li>
))}
</ul>
)}
{manifest && p.manifest && (
<button type="button" className="btn btn-xs btn-ghost -ml-1 mt-0.5" data-ev-prov-manifest={p.manifest}
onClick={() => onManifest(p)}>
<FileText size={11} aria-hidden /> {P.manifest}
</button>
)}
</li>
)
}
/**
* 查看导入清单:版本页的清单渲染(ManifestBody),内容按清单的工件 id 经工件接口取(内容寻址,取回时复验哈希)。
* 不用 ArtifactViewer:它对非表格内容整份 JSON 输出,遮罩不起作用,64 KB 的清单也没法读
*/
function ManifestDialog({ part, onClose }: { part: ProvenancePart; onClose: () => void }) {
const [state, setState] = useState<{ id: string; content?: unknown; error?: unknown } | null>(null)
useEffect(() => {
let alive = true
setState({ id: part.manifest })
api.artifact(part.manifest).then(
(res) => { if (alive) setState({ id: part.manifest, content: res?.content }) },
(error) => { if (alive) setState({ id: part.manifest, error }) },
)
return () => { alive = false }
}, [part.manifest])
const mine = state?.id === part.manifest ? state : null
const m: ManifestOut | null = mine?.content && typeof mine.content === 'object'
? { artifact_id: part.manifest, verified: true, kind: 'import_manifest', content: mine.content as Record<string, any> }
: null
return (
<Modal open onClose={onClose} width={920} title={VERSIONS_TEXT.manifestTitle(part.seq)}>
{mine?.error
? <ErrorState error={mine.error} />
: !m
? <Skeleton rows={5} height={36} gap={8} />
: (
<div data-ev-prov-manifest-view={part.manifest}>
<ManifestBody m={m} fileName={part.file_name} rawState={part.raw_state} recipeSeq={part.recipe_seq} />
</div>
)}
</Modal>
)
}
// ---------------------------------------------------------------------------
// 推断的来源:格子与逐条
// ---------------------------------------------------------------------------
function CellPart({ cs, version }: { cs: ProvenanceCellSource; version: ProvenanceVersion | null }) {
const [tech, setTech] = useState(false)
const techId = useId()
const parts = version?.parts ?? []
const part = parts.find((p) => p.has_row) ?? parts.find((p) => p.seq === cs.part_seq)
const items = fromItems(cs)
const recheck = cs.recheck
const sql = recheck?.sql ? (recheck.params?.length ? `${recheck.sql}\n-- ${JSON.stringify(recheck.params)}` : recheck.sql) : ''
return (
<>
<p className="text-sm font-semibold [overflow-wrap:anywhere]" data-ev-prov-cell="">{P.cell(cs.sheet, cs.cell)}</p>
{parts.length > 1 && (
// 多期时坐标是哪一期原件的坐标:同一个位置在别的期的文件里是另一个数
<p className="text-2xs text-dim [overflow-wrap:anywhere]" data-ev-prov-from-part="">
{P.fromPart(cs.part_seq, part?.file_name ?? null)}
</p>
)}
{items.length > 0 && (
<ul className="mt-1 space-y-0.5 [overflow-wrap:anywhere]" data-ev-prov-from="">
{items.map((it) => <li key={it.key} data-ev-prov-item={it.kind}>{it.text}</li>)}
</ul>
)}
{cs.merged_fill && (
<p className="mt-1 text-2xs [overflow-wrap:anywhere]" style={{ color: 'var(--st-waiting)' }} data-ev-prov-merged="">{P.mergedFill}</p>
)}
{cs.raw_purged && (
<p className="mt-1 text-2xs [overflow-wrap:anywhere]" style={{ color: 'var(--st-waiting)' }} data-ev-prov-raw="">{P.rawPurged}</p>
)}
{(recheck?.ok || sql) && (
<div className="mt-1 flex flex-wrap items-center gap-x-2 gap-y-0.5 text-2xs text-faint">
{recheck?.ok && <span data-ev-prov-recheck="">{P.recheck}</span>}
{sql && (
<button type="button" className="inline-flex items-center gap-0.5 rounded px-1 transition-colors hover:bg-hover hover:text-fg"
aria-expanded={tech} aria-controls={techId} data-ev-prov-details="" onClick={() => setTech((x) => !x)}>
<ChevronRight size={10} aria-hidden className={clsx(tech && 'rotate-90')} />
{P.details}
</button>
)}
</div>
)}
{sql && (
// 收起时留在 DOM 里(hidden):按钮的 aria-controls 始终指得到它
<div className="mt-1" id={techId} hidden={!tech} data-ev-prov-tech="">
<div className="mb-0.5 flex items-center gap-2">
<span className="min-w-0 flex-1 text-2xs font-medium text-faint">{P.recheckSql}</span>
<CopyChip label={EVIDENCE_TEXT.copySql} text={() => sql} />
</div>
<pre className="mono max-h-28 overflow-auto whitespace-pre-wrap rounded bg-bg px-1.5 py-1 text-2xs leading-relaxed text-dim [overflow-wrap:anywhere]">
{sql}
</pre>
</div>
)}
</>
)
}
interface Item { key: string; kind: string; text: string }
/**
* 逐条:这一行其余键列各取自哪一格,加上年份、规范写法、合计口径。顺序照 4.1:日期 → 年份 → 行标签 → 分段标题 →
* 规范写法。坐标和格子同一张工作表时只写坐标,不同时带上工作表名。列表的多行表头合成一条
*/
function fromItems(cs: ProvenanceCellSource): Item[] {
const from: ProvenanceFromCell[] = Array.isArray(cs.from) ? cs.from.filter((f) => f && f.cell) : []
const ref = (f: ProvenanceFromCell) => (f.sheet && f.sheet !== cs.sheet ? `${f.sheet}!${f.cell}` : f.cell)
const year = yearItem(cs)
const heads = from.filter((f) => f.role === 'col_header')
const out: Item[] = []
let yearDone = false
from.forEach((f, i) => {
const key = `${f.role}-${i}`
if (f.role === 'axis_header') {
out.push({ key, kind: f.role, text: P.axisHeader(ref(f)) })
if (year && !yearDone) { out.push(year); yearDone = true }
} else if (f.role === 'row_label') {
out.push({ key, kind: f.role, text: P.rowLabel(ref(f), f.text) })
} else if (f.role === 'section_title') {
out.push({ key, kind: f.role, text: P.sectionTitle(ref(f), f.locate_title) })
} else if (f.role === 'total_label') {
out.push({ key, kind: f.role, text: P.totalLabel(ref(f), f.text) })
} else if (f.role === 'col_header' && f === heads[0]) {
out.push({ key, kind: f.role, text: P.colHeader(heads.map(ref).join('、')) })
}
})
if (year && !yearDone) out.push(year)
if (cs.canonical?.raw && cs.canonical.canonical) {
out.push({ key: 'canonical', kind: 'canonical', text: P.canonical(cs.canonical.raw, cs.canonical.canonical) })
}
if (cs.kind === 'reported_total') out.push({ key: 'total', kind: 'reported_total', text: P.reportedTotal })
return out
}
function yearItem(cs: ProvenanceCellSource): Item | null {
const y = cs.year
if (!y) return null
// mixed:清单只记每块的表头形式,逐格判断不了,照实说,不断言这一格属于哪一种
const text = y.mixed ? P.yearMixed
: y.source === 'human' ? P.yearHuman(y.signed_by || VERSIONS_TEXT.unsigned)
: P.yearPeriod((Array.isArray(y.cells) ? y.cells : []).join('、'))
return { key: 'year', kind: 'year', text }
}
// ---------------------------------------------------------------------------
// 相关核对
// ---------------------------------------------------------------------------
/** 状态的颜色只用令牌:成立 / 一致是 done,不成立是 failed,未能核对、无法确定是 waiting,其余中性 */
const TONE: Record<string, string> = {
passed: 'var(--st-done)',
ok: 'var(--st-done)',
mismatch: 'var(--st-failed)',
unverifiable: 'var(--st-waiting)',
unknown: 'var(--st-waiting)',
}
function ChecksPart({ checks }: { checks: ProvenanceCheck[] }) {
return (
<Inferred title={P.checks} data-ev-prov-checks="">
<ul className="space-y-1">
{checks.map((c, i) => <CheckRow key={`${c.id}-${i}`} c={c} />)}
</ul>
</Inferred>
)
}
/**
* 一条核对:「编号 标题 — 这一行成立」(关系核对看行级结果,合计看格级结果,其余看本期结论)。本期结论不成立或未能
* 核对时另起一行写出来,写过理由的接在后面:「本期不成立 · 已接受,理由:…」——行级成立、本期不成立(别的日子不成立、
* 已接受)两件事都要看得到。口径不同(R2)不判行,本期结论就是「口径说明」,和别的核对同一种写法,扫读时对得齐
*/
function CheckRow({ c }: { c: ProvenanceCheck }) {
const main: { code: string; text: string } | null = c.row_status
? { code: c.row_status, text: P.row[c.row_status] ?? c.row_status }
: c.cell_status ? { code: c.cell_status, text: P.cellStatus[c.cell_status] ?? c.cell_status }
: null
const partText = P.partStatus[c.part_status] ?? c.part_status
// 没有行级、格级结果的(统计期、口径之外的核对)主行就写本期结论,理由接在同一行
const lead = main ?? { code: c.part_status, text: partText }
const accepted = c.acceptance ? P.accepted(c.acceptance.reason, c.acceptance.signed_by) : ''
const showPart = !!main && (c.part_status === 'mismatch' || c.part_status === 'unverifiable')
const second = main ? [showPart ? partText : '', accepted].filter(Boolean).join(' · ') : ''
return (
<li className="min-w-0 [overflow-wrap:anywhere]" data-ev-prov-check={c.id} data-ev-prov-part-status={c.part_status}
data-ev-prov-row={c.row_status ?? undefined} data-ev-prov-cell-status={c.cell_status ?? undefined}>
<p>
<span className="mono mr-1 text-faint">{c.id}</span>
<span>{c.title}</span>
{' — '}
<span style={{ color: TONE[lead.code] ?? 'var(--text-dim)' }} data-ev-prov-status="">{lead.text}</span>
{!main && accepted && <span className="text-dim"> · {accepted}</span>}
</p>
{second && <p className="text-2xs text-dim" data-ev-prov-period="">{second}</p>}
{c.detail && <p className="text-2xs text-faint" data-ev-prov-detail="">{c.detail}</p>}
</li>
)
}
File diff suppressed because it is too large Load Diff
+48 -2
View File
@@ -3,7 +3,8 @@ import { create } from 'zustand'
import { ApiError, api } from '../api/client'
import { graphVerdicts, judgedVerdicts, sqlOf } from '../lib/evidence'
import type {
EvidenceDocData, EvidenceGraph, EvidenceJudgeResult, EvidenceSegmentDetail, EvidenceStats, EvidenceVerdict,
EvidenceDocData, EvidenceGraph, EvidenceJudgeResult, EvidenceProvenance, EvidenceSegmentDetail, EvidenceStats,
EvidenceVerdict,
} from '../types'
/**
@@ -20,6 +21,11 @@ import type {
* 四期:封存之后按需追加的结论句判定按 `runId@报告` 记在 judged 里(按需裁判接口、片段接口叠上的
* 判定都记进来),正文的句末徽标、横幅的结论句计数据此盖过文档里的「未裁判 · 按需」。封存的文档本身
* 一个字不改。同一句正在判的不重复发;判过的(判定不是未裁判)不再给按钮,也就不会再发
*
* Excel 导入期 4:推断的来源按片段的缓存键记在 provenance 里,但**只在面板开着期间有效**。回答里有当前状态
* (原件已清除、接受已作废)和取决于数据文件现状的结论(被修改过、已回收),在版本页清除、作废、回滚之后,
* 关掉面板再打开就要看到新结论。所以面板挂载时 holdProvenance,最后一个面板关掉(或换了运行)时清掉这次
* 运行的槽;同一次打开里来回点同一片段只取一次
*/
export interface Slot<T> {
@@ -69,6 +75,14 @@ interface EvidenceState {
asks: Record<string, Slot<EvidenceJudgeResult>>
/** 运行类别(formal / exploratory):没开裁判的文档靠它认探索运行,按需取一次 */
runClasses: Record<string, Slot<string | null>>
/** 推断的来源(Excel 导入期 4),键同 segmentKey。只在面板开着期间有效,见 holdProvenance */
provenance: Record<string, Slot<EvidenceProvenance>>
loadProvenance: (runId: string, segmentId: string, report?: string) => Promise<void>
/**
* 证据面板挂载时调用,返回释放函数(面板卸载、换了运行时调)。这次运行最后一个面板释放后,清掉它的推断来源槽:
* 关掉面板再打开会重新取
*/
holdProvenance: (runId: string) => () => void
/** 记下几句封存后追加的判定(片段接口叠上的、证据图带来的) */
noteVerdicts: (runId: string, report: string | undefined, verdicts: Record<string, EvidenceVerdict>) => void
/** 请模型判断这几句(探索运行)。判定记进 judged;答复(含触顶、出错)记进 asks */
@@ -101,12 +115,17 @@ export class NotADocError extends Error {
}
}
/**
* 每次运行开着几个证据面板。放在模块里不放进 store:它只决定什么时候清缓存,界面不读它,变了也不该重渲染
*/
const provenanceHolds = new Map<string, number>()
export const useEvidence = create<EvidenceState>((set, get) => {
/**
* 取一次、记进 table[key];正在取或已经取到的不再发。取的途中这一格被清掉或换了(dropLatest 之后
* 又起了一次),回来的结果作废:不能让早先发出的请求把后来那份盖掉
*/
async function fill<K extends 'docs' | 'graphs' | 'segments' | 'latest' | 'sqls'>(
async function fill<K extends 'docs' | 'graphs' | 'segments' | 'latest' | 'sqls' | 'provenance'>(
table: K, key: string, fetcher: () => Promise<NonNullable<EvidenceState[K][string]['data']>>, force = false,
) {
const cur = get()[table][key]
@@ -190,6 +209,33 @@ export const useEvidence = create<EvidenceState>((set, get) => {
mark({ status: 'error', error })
}
},
provenance: {},
loadProvenance: (runId, segmentId, report) =>
fill('provenance', segmentKey(runId, segmentId, report), () => api.evidence.provenance(runId, segmentId, { report })),
holdProvenance: (runId) => {
provenanceHolds.set(runId, (provenanceHolds.get(runId) ?? 0) + 1)
let released = false
return () => {
if (released) return
released = true
const left = (provenanceHolds.get(runId) ?? 1) - 1
if (left > 0) {
provenanceHolds.set(runId, left)
return
}
provenanceHolds.delete(runId)
// 推到这一轮之后再清:开发期 StrictMode 把挂载的 effect 先清理、紧接着再跑一遍(又 hold 回来),
// 同一次提交里一个面板关、另一个面板开也是这样——这两种都不算「面板关了」
queueMicrotask(() => {
if (provenanceHolds.has(runId)) return
const prefix = `${runId}:`
const cur = get().provenance
if (!Object.keys(cur).some((k) => k.startsWith(prefix))) return
// 正在取的也清掉:fill 只在槽还是它自己放的那一份时才落结果,晚到的回包作废
set({ provenance: Object.fromEntries(Object.entries(cur).filter(([k]) => !k.startsWith(prefix))) })
})
}
},
loadRunClass: (runId) => {
const cur = get().runClasses[runId]
if (cur && cur.status !== 'error') return Promise.resolve()
+234
View File
@@ -550,6 +550,11 @@ export interface OutsideText {
text: string
kind: 'text' | 'text_digits' | (string & {})
period_source?: boolean
/**
* 期 4:所在行或列是否被隐藏(执行器写入)。true 在隐藏行列里;false 可见;null 或没有这个键:导入时没有记录
* (期 4 之前的清单)。裁判摘录只送 false 的项,界面照常显示全部区域外文字
*/
hidden?: boolean | null
}
/** 试运行回执(Extraction 去掉问题和核对之后给界面看的部分) */
@@ -2117,6 +2122,11 @@ export interface EvidenceStep extends Omit<EvidenceInput, 'status'> {
note?: string
/** 数据源改名或删掉了、按查询当时记下的遮罩处理时的那句说明。行照样有 */
mask_note?: string
/**
* 期 4:可以请求推断来源(GET …/segments/{sid}/provenance)。只有带文档标记的报告、单元格片段的查询步骤、
* 查询快照来自上传的表格时才有,且恒为 true;其余一个键都不加。前端只在 seg.cite?.kind === 'cell' 且它为 true 时请求
*/
provenance?: boolean
// ---- entity:表或字段(三期)----
/** entity:table / column */
kind?: string
@@ -2237,3 +2247,227 @@ export interface EvidenceSegmentDetail {
/** 可疑实体:最接近的已知名字(最多 3 个),给「是不是想写…」 */
closest?: (string | { alias?: string; name?: string; table?: string; kind?: string })[]
}
// -------------------------------------------------------------------------
// 期 4:推断的来源(GET /runs/{id}/evidence/segments/{sid}/provenance?report=…)
// 契约照 backend/app/data/provenance_types.py(asdict 之后的 JSON,P4-SPEC 2.8.1)。这是期 4 新加的接口,
// 答复里每个键都在(值可以是 null),所以这里不写成可选;老服务端没有这个接口时回 404 且没有 code,三节都不画。
// -------------------------------------------------------------------------
/** inferred 给出格子;table_only 只给表级来历(version 非空);none 不画,或只画一句原因 / 红色提示 */
export type ProvenanceStatus = 'inferred' | 'table_only' | 'none'
/** 不下钻的原因(provenance_types.ReasonCode)。界面不按它拼字,原样显示服务端给的 reason.text */
export type ProvenanceReasonCode =
| 'legacy_doc' | 'not_cell' | 'not_sealed' | 'not_upload' | 'simple_upload' | 'manifest_unreadable'
| 'chain_mismatch' | 'expression' | 'alias' | 'multi_table' | 'unparsed' | 'no_pk' | 'pk_missing' | 'masked'
| 'null_value' | 'null_pk' | 'snapshot_gone' | 'db_tampered' | 'recheck_missing' | 'recheck_multiple'
| 'recheck_mismatch' | 'no_lineage'
/** 标红的提示:出现时 reason.code 与它相同,界面只画提示 */
export type ProvenanceAlertCode = 'db_tampered' | 'chain_mismatch' | 'manifest_unreadable'
/** 附带的格:日期表头格、行标签、分段标题、列表头、合计标签 */
export type ProvenanceFromRole = 'axis_header' | 'row_label' | 'section_title' | 'col_header' | 'total_label'
/** 本期结论(照搬清单里 CheckResult.status) */
export type ProvenancePartStatus = 'passed' | 'mismatch' | 'unverifiable' | 'info'
/** 这一行的关系核对结果(只给关系核对) */
export type ProvenanceRowStatus = 'passed' | 'mismatch' | 'unverifiable'
/** 这一格的合计核对结果(只给 K、G、T) */
export type ProvenanceCellStatus = 'ok' | 'unverifiable' | 'unknown' | 'not_formula'
export interface ProvenanceReportRef {
node_id: string
doc_artifact: string
}
/** 片段本身是单元格引用就给(与 status 无关),不是时为 null */
export interface ProvenanceCellRef {
/** 目录里的查询编号(Q3) */
alias: string
row: number
column: string
/** 查询快照的工件 id */
artifact: string | null
}
export interface ProvenanceReason {
code: ProvenanceReasonCode
/** 细分,给测试和日志用,界面不显示 */
detail: string
/** 界面原样显示 */
text: string
}
export interface ProvenanceAlert {
code: ProvenanceAlertCode
text: string
}
export interface ProvenanceTableRef {
name: string
kind: 'data' | 'reported_total'
}
export interface ProvenancePeriod {
start: string
end: string
source: 'cells' | 'human'
/** 解析出统计期的格子,带工作表名(「客流汇总!B2」) */
cells: string[]
/** 人工录入时的署名(未认证) */
signed_by: string | null
}
/** 一条接受理由(导入清单里的,内容寻址,是当时的事实) */
export interface ProvenanceAcceptance {
check_id: string
title: string | null
kind: 'override' | 'waiver'
reason: string
signed_by: string | null
at: string | null
}
/** 导入记录上的当前状态(清除原件、作废接受) */
export interface ProvenanceStateNote {
at: string | null
signed_by: string | null
reason: string | null
}
/** 数据版本里的一期(按统计期升序)。raw_state、purged、revoked 是当前状态,界面标「当前状态」 */
export interface ProvenancePart {
import_id: string
seq: number
/** 导入清单的工件 id:「查看导入清单」按它经 /api/artifacts/{id} 取 */
manifest: string
/** 没有统计期时 null(「统计期未记录」) */
period: ProvenancePeriod | null
file_name: string | null
raw_sha256: string | null
/** 区域(「客流汇总!B4:AG30」,多张表用「、」连);取不到为 null */
region: string | null
/** 排除的行数之和;期 3 之前的回执没有记录时为 null */
excluded_rows: number | null
recipe_sha256: string | null
/** 配方第几版(当前状态);取不到为 null */
recipe_seq: number | null
/** 导入时间(清单的 created_at) */
committed_at: string | null
/** 署名(未认证) */
signed_by: string | null
acceptances: ProvenanceAcceptance[]
raw_state: 'kept' | 'purged' | 'absent' | null
purged: ProvenanceStateNote | null
revoked: ProvenanceStateNote | null
/** 这一格所在的一期:inferred 时恰好一项为 true */
has_row: boolean
}
/** 数据版本(表级来历),来自封存链上的清单,不加「推断」标注 */
export interface ProvenanceVersion {
source: string
snapshot_id: string
mode: 'replace' | 'accumulate'
union: boolean
tables: ProvenanceTableRef[]
/** false:数据源设有遮罩,不出「查看导入清单」 */
manifest_view: boolean
parts: ProvenancePart[]
}
export interface ProvenanceFromCell {
role: ProvenanceFromRole
/** 这一格给出的是哪一列的值;宽表指标列的行标签(指标名取自哪一格)为 null */
column: string | null
sheet: string
/** 不带工作表名的坐标(「G4」) */
cell: string
/** 格子原文;拿不到为 null */
text: string | null
/** 只给分段标题:配方中的定位文字 */
locate_title: string | null
}
export interface ProvenanceYear {
source: 'period' | 'human'
cells: string[]
signed_by: string | null
/** 这一块的日期表头有的是日期格、有的是文本,无法确定这一格属于哪一种 */
mixed: boolean
}
/**
* 这一行标签列(交叉表的维度、合计表的合计项)的原文与数据库里的规范写法不同时给出。被引用的是值列也给:
* 合计表那一格是「18-22 时合计」→「18-22时合计」,全角标签那一期的值格是「8-9」→「8-9」
*/
export interface ProvenanceCanonical {
raw: string
canonical: string
}
export interface ProvenanceRecheck {
/** 占位符一律是 ?,按主键的顺序与 params 一一对应 */
sql: string
params: unknown[]
ok: boolean
}
/** 推断出的格子(只在 inferred 时有) */
export interface ProvenanceCellSource {
table: string
column: string
column_role: 'axis' | 'dim' | 'derive' | 'const' | 'measure' | 'value' | 'text' | (string & {})
kind: 'data' | 'reported_total'
pk: Record<string, unknown>
/** 快照库的 rowid(累积时是合并后的行号) */
rowid: number
part_seq: number
part_rowid: number
sheet: string
/** 不带工作表名的坐标(「G5」) */
cell: string
header: string | null
unit: string | null
from: ProvenanceFromCell[]
year: ProvenanceYear | null
canonical: ProvenanceCanonical | null
raw_purged: boolean
/** 列表这一块按合并单元格的左上格填充 */
merged_fill: boolean
recheck: ProvenanceRecheck
}
/** 相关核对的一条(只在 inferred 时有) */
export interface ProvenanceCheck {
id: string
kind: string
title: string
part_status: ProvenancePartStatus
row_status: ProvenanceRowStatus | null
cell_status: ProvenanceCellStatus | null
acceptance: ProvenanceAcceptance | null
/** K 无法核对时的原因摘要(可能带数字,系统呈现) */
detail: string | null
}
/** GET /runs/{id}/evidence/segments/{sid}/provenance:推断的来源 */
export interface EvidenceProvenance {
schema: 'agentlab.provenance/1' | (string & {})
report: ProvenanceReportRef
segment: string
cell: ProvenanceCellRef | null
status: ProvenanceStatus
/** status 不是 inferred 时必有 */
reason: ProvenanceReason | null
/** 标红;有它时不再画 reason.text */
alert: ProvenanceAlert | null
/** 运行已封存且封存核对通过;false 时数据版本一节加一句未封存 */
sealed: boolean
/** status 为 none 时为 null */
version: ProvenanceVersion | null
cell_source: ProvenanceCellSource | null
checks: ProvenanceCheck[]
}
+1 -1
View File
@@ -85,7 +85,7 @@ const ORDER = [
// 拆开的两项看 SPLIT 里各份的 est。2026-09-29 八道并行、关浏览器不干等之后的实测
const ESTIMATE = {
stream: 173, chat: 160, studio: 149, manage: 145, runs: 112, publish: 92, shell: 87,
evidence: 85, versions: 60, 'ui-kit': 70, ui: 61, guards: 8, 'canvas-layout': 2, copy: 2, tokens: 1, trace: 1, decode: 1,
evidence: 91, versions: 60, 'ui-kit': 70, ui: 61, guards: 8, 'canvas-layout': 2, copy: 2, tokens: 1, trace: 1, decode: 1,
}
// 最慢的两项按段拆成几份。用的是它们自己的段过滤:最后一份用 *_SKIP 兜住「其余全部」,以后新加的段
// 不用改这里也跑得到(拆出去的那几份要是一段都没对上,那一份会报「一项都没跑」)。
+462 -5
View File
@@ -37,6 +37,11 @@ const fxj = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-
// 文档由 compose_doc 真跑、证据链由 api/evidence.py 的 _chain 真跑;判定和表名实体步骤的 fields 照前后端接口约定写
const fxv = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-verdict.json`, 'utf8'))
const JUDGE_SETS = [fxj.formal, fxj.explore, fxj.plain, fxv.formal, fxv.explore]
// Excel 导入期 4:推断的来源。文档由 compose_doc 真跑(带文档标记),片段答复由 _chain 真跑,推断来源的答复按契约
// (backend/app/data/provenance_types.py)拼;cases 写明每个场景是哪个片段、片段文字、覆盖 P4-SPEC 4.4 的哪几条
const fxp = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-provenance.json`, 'utf8'))
/** 推断来源夹具里各期导入清单的工件 id:夹具只收了两份清单,其余的按「工件不存在」答,不落到真服务端 */
const FXP_MANIFESTS = new Set(Object.values(fxp.provenance).flatMap((a) => a.version?.parts ?? []).map((p) => p.manifest))
/** 夹具里某段文字是哪个片段:检查按文字认片段,重新生成夹具时编号变了也不用改 */
const segOf = (doc, text) => doc.blocks.flatMap((b) => b.units.flatMap((u) => u.segments)).find((s) => s.text === text)?.id
@@ -75,12 +80,18 @@ function judgeReply(set, body) {
/**
* 伪造证据相关的 GET;片段请求记账(缓存:同一个片段点两次只取一次)。四期:按需裁判的 POST 也在这里认
* (记进 hits 的 judge:<运行>:<句子>),别的非 GET 一律拦掉。judge 给了就用它答(改坏验证、触顶、出错);
* segPatch 给了就改片段接口的答复(比如 on_demand 说运行还没封存)
* segPatch 给了就改片段接口的答复(比如 on_demand 说运行还没封存)。
*
* Excel 导入期 4:推断来源接口(…/segments/<id>/provenance)在片段正则之前单独认,不分运行、不分夹具,一律记成
* prov:<运行>:<片段>——片段正则以 $ 结尾认不出它,落到 r.continue() 会得到一个没有机读码的 404,界面按「老服务端」
* 三节都不画,只看 DOM 的断言不论前端发没发请求都会过。推断来源夹具的运行按 provenance[<片段>] 作答,其余一律回
* 可识别的哨兵(418,harness_unexpected_provenance),界面会画出「未能获取推断的来源」。prov 给了就由它改答复
* (prov(运行, 片段, 默认答复) → {status?, json} 或它的 Promise:延迟、出错、老服务端)
*/
async function routed(page, { judge = null, segPatch = null } = {}) {
async function routed(page, { judge = null, segPatch = null, prov = null } = {}) {
const hits = []
const judged = new Set()
await page.route((u) => new URL(u).pathname.startsWith('/api/'), (r) => {
await page.route((u) => new URL(u).pathname.startsWith('/api/'), async (r) => {
const req = r.request()
const url = new URL(req.url())
const jset = JUDGE_SETS.find((x) => url.pathname.includes(`/runs/${x.run_id}/`) || url.pathname.endsWith(`/runs/${x.run_id}`))
@@ -92,6 +103,33 @@ async function routed(page, { judge = null, segPatch = null } = {}) {
return r.fulfill({ status: reply.status ?? 200, json: reply.json })
}
if (req.method() !== 'GET') return r.abort()
const pm = url.pathname.match(/\/api\/runs\/([^/]+)\/evidence\/segments\/([^/]+)\/provenance$/)
if (pm) {
const [run, sid] = [decodeURIComponent(pm[1]), decodeURIComponent(pm[2])]
hits.push(`prov:${run}:${sid}`)
const own = run === fxp.run_id && fxp.provenance[sid]
let reply = own ? { json: structuredClone(own) } : fxp.errors.unexpected
if (prov) reply = await prov(run, sid, reply)
return r.fulfill({ status: reply.status ?? 200, json: reply.json })
}
if (url.pathname.includes(`/runs/${fxp.run_id}/`)) {
const seg = url.pathname.match(/\/evidence\/segments\/([^/]+)$/)
if (seg) {
const id = decodeURIComponent(seg[1])
hits.push(`p:${id}`)
const body = fxp.segments[id]
return body ? r.fulfill({ json: structuredClone(body) })
: r.fulfill({ status: 404, json: { detail: '报告中没有这个片段', code: 'evidence_segment_not_found' } })
}
if (/\/evidence$/.test(url.pathname)) return r.fulfill({ json: fxp.graph })
}
// 「查看导入清单」按清单的工件 id 取(内容寻址)
const art = url.pathname.match(/^\/api\/artifacts\/([^/]+)$/)?.[1]
if (art && fxp.artifacts[art]) {
hits.push(`artifact:P:${art}`)
return r.fulfill({ json: fxp.artifacts[art] })
}
if (art && FXP_MANIFESTS.has(art)) return r.fulfill({ status: 404, json: { detail: '工件不存在' } })
// 四期的报告工件:问数据页的答案(Output → EvidenceField)按工件 id 取文档
const jart = JUDGE_SETS.find((x) => url.pathname === `/api/artifacts/${x.doc_artifact}`)
if (jart) return r.fulfill({ json: { id: jart.doc_artifact, content: jart.doc } })
@@ -144,14 +182,14 @@ async function routed(page, { judge = null, segPatch = null } = {}) {
return hits
}
async function open(url, { w = 1280, h = 1000, reduced = false, patchQuery = null, judge = null, segPatch = null } = {}) {
async function open(url, { w = 1280, h = 1000, reduced = false, patchQuery = null, judge = null, segPatch = null, prov = null } = {}) {
const ctx = await browser.newContext({ viewport: { width: w, height: h }, ...(reduced ? { reducedMotion: 'reduce' } : {}) })
const page = await ctx.newPage()
// 找不到元素时 8 秒就报,不等默认的 30 秒:改坏验证时一节里十几处都找不到
page.setDefaultTimeout(8000)
const errors = []
page.on('pageerror', (e) => errors.push(e.message))
const hits = await routed(page, { judge, segPatch })
const hits = await routed(page, { judge, segPatch, prov })
// 预览页直接 import 第二期夹具(vite 把 JSON 当模块给):要一份改过的文档时把那个模块换掉
if (patchQuery) {
await page.route((u) => new URL(u).pathname.endsWith('/run/__tests__/evidence-query.json'), (r) =>
@@ -742,6 +780,12 @@ await section('query', '查询步骤:SQL 带复制、被引用的行和格高
return !!note && !!seal && !!(seal.compareDocumentPosition(note) & Node.DOCUMENT_POSITION_FOLLOWING)
}))
check('查询快照复验通过时不多说一句', await panel(page).locator('[data-ev-integrity]').count() === 0)
// Excel 导入期 4 之前的报告:查询步骤不带推断来源的提示,前端一个请求都不多发、三节都不画(P4-SPEC 4.4 第 14 条)。
// 断言请求记账,不只看 DOM:老服务端那种 404 也会让三节不画
check('期 4 之前的报告(查询步骤没有提示):不请求推断的来源,面板里没有数据版本、推断的来源',
!harness.hits.some((h) => h.startsWith('prov:'))
&& await panel(page).locator('[data-ev-provenance], [data-ev-prov-version], [data-ev-prov-checks]').count() === 0,
harness.hits.filter((h) => h.startsWith('prov:')).join(','))
const aria = await page.locator(`${Q} [data-seg="${qseg('1,288')}"]`).getAttribute('aria-label')
check('单元格的 aria-label 写明查询、第几行、哪一列', aria === '1,288,有出处:查询 Q3 · 第 6 行 · amount', aria)
@@ -2494,6 +2538,419 @@ await section('jr-banner', '裁判拆档:出具横幅的结论句计数带上
}
})
// ---------------------------------------------------------------------------
// Excel 导入期 4:推断的来源(夹具 evidence-provenance.json,P4-SPEC 4.4)
// ---------------------------------------------------------------------------
const PV = '#evidence-provenance'
const PVN = '#evidence-provenance-narrow'
/** 场景的片段 id:按夹具 cases 里记的片段文字认,重新生成夹具后编号挪了也不用改 */
const pseg = (name) => segOf(fxp.doc, fxp.cases[name].text)
/** 这个场景的片段请求过几次推断来源 */
const provHits = (hits, name) => hits.filter((h) => h === `prov:${fxp.run_id}:${pseg(name)}`).length
/** 点开一个场景,等推断的来源那一节定下来(不再是「正在推断」);不该有这一节的场景改等查询步骤、指标步骤 */
async function openP(p, name, { box = PV, wait = '[data-ev-provenance]:not([data-ev-provenance="loading"])' } = {}) {
await p.locator(`${box} [data-seg="${pseg(name)}"]`).click()
await p.waitForSelector(`[data-evidence-panel] ${wait}`, { timeout: 4000 }).catch(() => {})
await p.waitForTimeout(120)
}
const pv = (p, sel = '') => p.locator(`[data-evidence-panel] ${sel}`)
/** 元素的文字(空白合成一个空格);没有这个元素时是空串 */
const textOf = async (loc) => ((await loc.count()) ? (await loc.first().innerText()).replace(/\s+/g, ' ').trim() : '')
const fromOf = (p) => p.locator('[data-evidence-panel] [data-ev-provenance] [data-ev-prov-from] li').allInnerTexts()
const partsOf = (p) => p.locator('[data-evidence-panel] [data-ev-prov-version] [data-ev-prov-part]')
.evaluateAll((els) => els.map((e) => `${e.getAttribute('data-ev-prov-part')}${e.hasAttribute('data-ev-prov-has-row') ? '*' : ''}`))
/** 只给表级来历的场景:table_only、一句原因、没有格子和相关核对、数据版本照样有 */
async function tableOnly(p, name, want, what) {
await openP(p, name)
const reason = await textOf(pv(p, '[data-ev-provenance="table_only"] [data-ev-prov-reason]'))
check(what, (typeof want === 'string' ? reason === want : want.test(reason))
&& await pv(p, '[data-ev-prov-cell], [data-ev-prov-checks], [data-ev-prov-alert]').count() === 0
&& await pv(p, '[data-ev-prov-version]').count() === 1, reason)
}
const noSections = async (p) => (await pv(p, '[data-ev-provenance], [data-ev-prov-version], [data-ev-prov-checks]').count()) === 0
const activeHas = (p, attr) => p.evaluate((a) => document.activeElement?.hasAttribute(a) ?? false, attr)
await section('provenance', '期 4:推断的来源——数据版本、格子与逐条、相关核对,只在上传表格的单元格上请求', async () => {
const ids = Object.fromEntries(Object.keys(fxp.cases).map((k) => [k, pseg(k)]))
check('夹具的场景都能按片段文字认出来(和 cases 记的编号一致)', Object.entries(fxp.cases).every(([k, c]) => ids[k] === c.segment),
JSON.stringify(ids))
// 前面各节点开的都是期 4 之前的报告、没有提示的查询步骤:一个推断来源请求都不该有(4.4 第 14 条的全局版)
const stray = harness.hits.filter((h) => h.startsWith('prov:'))
check('其他夹具(期 4 之前的报告)点开过的片段一个推断来源请求都没发', stray.length === 0, stray.join(','))
// ---- 1. 宽表的一格:格子、逐条、相关核对、位置、标题
await openP(page, 'wide')
const sec = pv(page, '[data-ev-provenance="inferred"]')
check('满足条件:有「推断的来源」一节(inferred)', await sec.count() === 1, String(await pv(page, '[data-ev-provenance]').getAttribute('data-ev-provenance').catch(() => null)))
check('……标题「推断的来源」,旁边标「推断,不属于已封存的证据」', await textOf(sec.locator('h4')) === '推断的来源'
&& await textOf(sec.locator('[data-ev-prov-badge]')) === '推断,不属于已封存的证据')
const cell = await textOf(sec.locator('[data-ev-prov-cell]'))
check('……第一行醒目地写格子:工作表「客流汇总」G5', cell === '工作表「客流汇总」G5', cell)
const items = await fromOf(page)
check('……逐条:日期取自表头格 G4 → 年份取自统计期 → 行标签 B5(指标名)', items.join('|')
=== '日期取自表头格 G4|年份取自统计期(客流汇总!B2)|行标签 B5「全日客流(人次)」', items.join('|'))
check('相关核对:R1「这一行成立」', (await textOf(pv(page, '[data-ev-prov-checks] [data-ev-prov-check="R1"]'))).includes('这一行成立'))
// 规格 4.1 的散文写「口径说明:…」;R2 的标题本身是「口径不同:…」,再套一层冒号读不顺,所以和别的核对同一种
// 写法「R2 标题 — 口径说明」(口径说明是 4.2 partStatus.info 的原文)。改写法要连同 CheckRow 一起改
check('……口径不同(R2)只写口径说明,不判行', (await textOf(pv(page, '[data-ev-prov-check="R2"]'))).endsWith('— 口径说明')
&& await pv(page, '[data-ev-prov-check="R2"][data-ev-prov-row]').count() === 0)
const order = await page.evaluate(() => {
const p = document.querySelector('[data-evidence-panel]')
const q = p?.querySelector('[data-ev-query]')
const seq = ['[data-ev-prov-version]', '[data-ev-provenance]', '[data-ev-prov-checks]', '[data-ev-judge]', '[data-ev-seal]']
.map((s) => p?.querySelector(s)).filter(Boolean)
if (!q || seq.length < 4) return `缺:${seq.length}`
let prev = q
for (const el of seq) {
if (!(prev.compareDocumentPosition(el) & Node.DOCUMENT_POSITION_FOLLOWING)) return `顺序不对:${el.outerHTML.slice(0, 60)}`
prev = el
}
return 'ok'
})
check('三节在查询步骤之后、结论句裁判和封存之前:数据版本 → 推断的来源 → 相关核对', order === 'ok', order)
const heads = await Promise.all(['数据版本', '推断的来源', '相关核对']
.map((name) => panel(page).getByRole('heading', { name, exact: true }).evaluateAll((els) => els.map((e) => e.tagName).join(','))))
check('三节的标题都是真正的标题(getByRole heading,h4)', heads.every((t) => t === 'H4'), heads.join(' | '))
check('推断的来源用虚线左边框,和其余几节区分', await sec.evaluate((el) => getComputedStyle(el).borderLeftStyle) === 'dashed')
check('相关核对也挂「推断」标签;数据版本来自封存链上的清单,不挂', await pv(page, '[data-ev-prov-checks] [data-ev-prov-badge]').count() === 1
&& await pv(page, '[data-ev-prov-version] [data-ev-prov-badge]').count() === 0)
check('封存那一行照旧只按片段接口的链算:「已封存 · 核对一致」', (await textOf(pv(page, '[data-ev-seal]'))).includes('已封存 · 核对一致'))
const integrityBase = await pv(page, '[data-ev-integrity]').count()
const ver = pv(page, '[data-ev-prov-version]')
const vtext = await textOf(ver)
const p1 = fxp.provenance[ids.wide].version.parts[0]
check('数据版本:首行「数据源「flow_demo」· 每期替换」(每期替换不写期数)', await textOf(ver.locator('[data-ev-prov-head]')) === '数据源「flow_demo」· 每期替换',
vtext)
check('……每期一行:第几次导入 · 统计期;文件名、sha256 前 12 位、区域、原件状态、配方第几版',
vtext.includes('第 1 次导入 · 统计期 2026-08-01 至 2026-08-31') && vtext.includes(`文件「${p1.file_name}」· sha256 ${p1.raw_sha256.slice(0, 12)}`)
&& !vtext.includes(p1.raw_sha256.slice(0, 13)) && vtext.includes('区域 客流汇总!B4:AG30') && vtext.includes('原件已保存')
&& vtext.includes('配方第 1 版') && vtext.includes('涉及的表:日客流'), vtext)
check('……这一格所在的一期有标记;已封存的运行不写「尚未封存」', (await partsOf(page)).join(',') === '1*'
&& vtext.includes('这一格所在的一期') && await ver.locator('[data-ev-prov-unsealed]').count() === 0)
const det = sec.locator('[data-ev-prov-details]')
check('末尾一行小字「已按主键回查数据文件,值一致」,回查 SQL 收在「技术细节」里(默认收起)',
await textOf(sec.locator('[data-ev-prov-recheck]')) === '已按主键回查数据文件,值一致'
&& await det.getAttribute('aria-expanded') === 'false' && await sec.locator('[data-ev-prov-tech]').isHidden()
&& await det.getAttribute('aria-controls') === await sec.locator('[data-ev-prov-tech]').getAttribute('id'))
await det.focus()
await page.keyboard.press('Enter')
const tech = await textOf(sec.locator('[data-ev-prov-tech]'))
check('……「技术细节」是按钮,回车展开:回查 SQL 和参数,带复制', await det.getAttribute('aria-expanded') === 'true'
&& await sec.locator('[data-ev-prov-tech]').isVisible()
&& tech.includes('SELECT rowid, "全日客流" FROM "日客流" WHERE "日期" = ?') && tech.includes('2026-08-05')
&& await sec.locator('[data-ev-prov-tech] button', { hasText: '复制 SQL' }).count() === 1, tech)
// ---- 16、28:键盘打开导入清单,对话框是结构化的清单渲染,Esc 只关对话框
await pv(page, '[data-ev-query] [data-ev-query-head]').focus()
let reached = false
for (let i = 0; i < 30 && !reached; i++) {
await page.keyboard.press('Tab')
reached = await activeHas(page, 'data-ev-prov-manifest')
}
check('键盘:从查询步骤 Tab 走得到「查看导入清单」', reached)
const arts = harness.hits.filter((h) => h.startsWith('artifact:P:')).length
await page.keyboard.press('Enter')
await page.waitForSelector('[role="dialog"] [data-manifest-block]', { timeout: 4000 }).catch(() => {})
const dlg = page.locator('[role="dialog"]')
const dtext = await dlg.innerText().catch(() => '')
check('回车打开导入清单对话框,按清单的工件 id 经工件接口取', await dlg.count() === 1
&& harness.hits.filter((h) => h.startsWith('artifact:P:')).length === arts + 1 && harness.hits.includes(`artifact:P:${p1.manifest}`)
&& dtext.includes('第 1 次导入的导入清单'), dtext.slice(0, 80))
check('……正文是版本页的清单渲染(结构化区块),不是整份 JSON(展开的文字里没有键名 "acceptances")',
await dlg.locator('[data-manifest-block]').count() >= 5 && !dtext.includes('"acceptances"') && dtext.includes(p1.file_name)
&& await dlg.locator('[data-manifest-raw]').evaluate((el) => !el.open).catch(() => false), String(await dlg.locator('[data-manifest-block]').count()))
if (SHOTS) {
for (const theme of ['dark', 'light']) {
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
await page.waitForTimeout(250)
await page.screenshot({ path: `${SHOTS}/evidence-p4-manifest-${theme}.png` })
}
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
}
await page.keyboard.press('Escape')
await page.waitForTimeout(200)
check('Esc 只关清单对话框:证据面板还开着,焦点回到「查看导入清单」', await dlg.count() === 0 && await panel(page).count() === 1
&& await activeHas(page, 'data-ev-prov-manifest'))
if (SHOTS) {
for (const theme of ['dark', 'light']) {
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
await page.waitForTimeout(250)
await panel(page).screenshot({ path: `${SHOTS}/evidence-p4-inferred-${theme}.png` })
}
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
}
await page.keyboard.press('Escape')
await page.waitForTimeout(200)
// ---- 18:缓存只在面板这次打开期间有效
const c0 = provHits(harness.hits, 'wide')
await openP(page, 'wide')
await openP(page, 'long')
await openP(page, 'wide')
check('同一次打开里同一片段点两次,推断的来源只取一次', provHits(harness.hits, 'wide') === c0 + 1 && provHits(harness.hits, 'long') >= 1,
`${c0} → ${provHits(harness.hits, 'wide')}`)
await page.keyboard.press('Escape')
await page.waitForTimeout(200)
check('……面板关掉了', await panel(page).count() === 0)
await openP(page, 'wide')
check('关闭面板再打开、再点同一片段:重新取(回答里有当前状态)', provHits(harness.hits, 'wide') === c0 + 2, String(provHits(harness.hits, 'wide')))
// ---- 2、3:长表的分段标题、全角标签的规范写法、合计表
await openP(page, 'long')
const longItems = await fromOf(page)
check('长表一格:分段标题 B9(按配方中的标题「…」定位),不说它是格子原文;行标签 B16「13-14」',
longItems.includes('分段标题 B9(按配方中的标题「日间时段客流(人次)」定位)') && longItems.includes('行标签 B16「13-14」'), longItems.join('|'))
await openP(page, 'fullwidth')
const fw = await fromOf(page)
check('全角标签那一期的维度列:原文「8-9」,按规范写法存为「8-9」', fw.includes('原文「8-9」,按规范写法存为「8-9」'), fw.join('|'))
await openP(page, 'total')
const tItems = await fromOf(page)
check('合计表一格:合计标签 B28「18-22 时合计」(原文),口径「原表写明的合计:不要彼此相加…」', tItems.includes('合计标签 B28「18-22 时合计」')
&& tItems.includes('原表写明的合计:不要彼此相加,也不要与明细相加') && tItems.includes('原文「18-22 时合计」,按规范写法存为「18-22时合计」'),
tItems.join('|'))
check('……相关核对里 K1「这一格一致」', (await textOf(pv(page, '[data-ev-prov-check="K1"]'))).includes('这一格一致'))
// ---- 4:累积的两期
await openP(page, 'union')
const un = fxp.provenance[ids.union]
check('按期累积:首行写「按期累积 · 共 2 期」,两期都列出,第 2 期标「这一格所在的一期」',
await textOf(pv(page, '[data-ev-prov-head]')) === `数据源「${un.version.source}」· 按期累积 · 共 2 期`
&& (await partsOf(page)).join(',') === '1,2*', (await partsOf(page)).join(','))
const ufrom = await textOf(pv(page, '[data-ev-prov-from-part]'))
check('……格子是第 2 期(9 月)文件的坐标', await textOf(pv(page, '[data-ev-prov-cell]')) === `工作表「${un.cell_source.sheet}」${un.cell_source.cell}`
&& ufrom.includes('第 2 次导入') && ufrom.includes(un.version.parts[1].file_name), ufrom)
// ---- 5–8:只给表级来历
await tableOnly(page, 'alias', /改了名/, '改名:table_only,原因写「改了名」,没有格子,数据版本照样有')
await tableOnly(page, 'selfjoin', /同一张表多次/, '自连接:table_only,原因写「同一张表多次」')
await tableOnly(page, 'null', '这一格是空值,不推断来源', '空值:table_only,「这一格是空值,不推断来源」')
await tableOnly(page, 'aggregate', /计算、聚合/, '聚合:table_only,原因写「计算、聚合」')
check('……聚合的格子数据版本照样列出涉及的表', (await textOf(pv(page, '[data-ev-prov-tables]'))) === '涉及的表:日客流')
// ---- 9、10、21:标红
const failedColor = await tokenColor(page, 'color', 'var(--st-failed)')
const look = (code) => pv(page, `[data-ev-prov-alert="${code}"]`).evaluate((el) => {
const cs = getComputedStyle(el)
return { color: cs.color, border: cs.borderLeftColor, text: el.innerText }
}).catch(() => null)
await openP(page, 'tampered')
const tam = await look('db_tampered')
check('数据文件被改:标红(data-ev-prov-alert="db_tampered"),写「可能被修改过」,边框和字都是 --st-failed',
!!tam && tam.text.includes('可能被修改过') && tam.color === failedColor && tam.border === failedColor, JSON.stringify({ tam, failedColor }))
check('……不给格子、不写原因那一句(有 alert 时只画 alert),数据版本照样有', await pv(page, '[data-ev-prov-cell], [data-ev-prov-reason], [data-ev-prov-checks]').count() === 0
&& await pv(page, '[data-ev-prov-version]').count() === 1 && await pv(page, '[data-ev-provenance="table_only"]').count() === 1)
check('……标红用自己的属性:面板里 data-ev-integrity 的个数和没有 alert 时相同', await pv(page, '[data-ev-integrity]').count() === integrityBase)
if (SHOTS) {
for (const theme of ['dark', 'light']) {
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
await page.waitForTimeout(250)
await panel(page).screenshot({ path: `${SHOTS}/evidence-p4-tampered-${theme}.png` })
}
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
}
await openP(page, 'chain')
check('登记哈希与清单不符:table_only,标红 chain_mismatch,数据版本照样有,没有原因那一句',
await pv(page, '[data-ev-provenance="table_only"] [data-ev-prov-alert="chain_mismatch"]').count() === 1
&& await pv(page, '[data-ev-prov-version]').count() === 1 && await pv(page, '[data-ev-prov-reason]').count() === 0)
await openP(page, 'unreadable')
const unr = await look('manifest_unreadable')
check('清单无法读取:none 里只有红色的 manifest_unreadable,没有数据版本、没有原因那一句',
await pv(page, '[data-ev-provenance="none"] [data-ev-prov-alert="manifest_unreadable"]').count() === 1 && unr?.color === failedColor
&& await pv(page, '[data-ev-prov-version], [data-ev-prov-reason], [data-ev-prov-cell], [data-ev-prov-checks]').count() === 0, JSON.stringify(unr))
// ---- 11、12:当前状态
await openP(page, 'purged')
const pver = await textOf(pv(page, '[data-ev-prov-version]'))
const purgedLine = await textOf(pv(page, '[data-ev-prov-purged]'))
check('原件已清除:数据版本里写「原件已清除 · 清除时间 · 理由 · 署名(未认证)」,标「当前状态」',
/^原件已清除 · \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \(UTC[+-]\d{2}:\d{2}\) · 理由:合成理由:原件不再保留 · 署名(未认证):录入员甲/.test(purgedLine)
&& await pv(page, '[data-ev-prov-purged] [data-ev-prov-current]').count() === 1, purgedLine)
check('……格子照样有,旁边注明「原件已清除:坐标来自导入时的记录,无法再对照原件」', await pv(page, '[data-ev-prov-cell]').count() === 1
&& await textOf(pv(page, '[data-ev-prov-raw]')) === '原件已清除:坐标来自导入时的记录,无法再对照原件')
await openP(page, 'revoked')
const rver = await textOf(pv(page, '[data-ev-prov-version]'))
check('作废接受:数据版本里写「这一期的接受已作废 · 作废时间 · 理由」(当前状态),接受理由照旧列出',
/^这一期的接受已作废 · \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \(UTC[+-]\d{2}:\d{2}\) · 理由:合成理由:接受依据有误 · 署名(未认证):未填写/
.test(await textOf(pv(page, '[data-ev-prov-revoked]')))
&& await pv(page, '[data-ev-prov-revoked] [data-ev-prov-current]').count() === 1
&& rver.includes('已接受:「日客流」:全日客流 = 分区甲 + 分区乙 · 合成理由:分区合计口径调整 · 署名(未认证):未填写'), rver)
// ---- 13:简单导入
await openP(page, 'simple')
check('简单导入:none 里只有一句「这份表格按表头行直接导入…」,没有数据版本',
(await textOf(pv(page, '[data-ev-provenance="none"] [data-ev-prov-reason]'))).startsWith('这份表格按表头行直接导入')
&& await pv(page, '[data-ev-prov-version], [data-ev-prov-alert], [data-ev-prov-checks]').count() === 0)
// ---- 15、20:不该请求的两种
check('夹具自检:手工源那一格的查询步骤没有提示,指标片段链上的查询步骤误带了提示',
fxp.segments[ids.manual].chain.every((st) => st.provenance !== true)
&& fxp.segments[ids.metric].chain.some((st) => st.step === 'query' && st.provenance === true))
await openP(page, 'manual', { wait: '[data-ev-query]' })
await page.waitForTimeout(250)
check('手工源(查询步骤不带提示):不请求推断的来源,三节都不画', provHits(harness.hits, 'manual') === 0 && await noSections(page))
await openP(page, 'metric', { wait: '[data-ev-metric]' })
await page.waitForTimeout(250)
check('指标片段:链上的查询步骤带了提示也不请求(只认单元格引用)', provHits(harness.hits, 'metric') === 0 && await noSections(page)
&& await pv(page, '[data-ev-query]').count() >= 1)
// ---- 22、23、25、26:回收、遮罩、未封存、本期不成立并接受
await tableOnly(page, 'gone', /已回收或不存在/, '数据文件已回收:table_only,原因写「已回收或不存在」,不标红')
await tableOnly(page, 'masked', '这一格涉及设置了遮罩的列,不推断来源', '遮罩:table_only,「这一格涉及设置了遮罩的列,不推断来源」')
check('……数据源设有遮罩:数据版本里没有「查看导入清单」,改写一句去向', await pv(page, '[data-ev-prov-manifest]').count() === 0
&& await textOf(pv(page, '[data-ev-prov-manifest-masked]')) === '数据源设置了遮罩,导入清单请到数据源卡片上的「版本」查看')
await openP(page, 'unsealed')
check('运行未封存:照常推断,数据版本一节写「本次运行尚未封存…」,不标红', await pv(page, '[data-ev-provenance="inferred"]').count() === 1
&& (await textOf(pv(page, '[data-ev-prov-version] [data-ev-prov-unsealed]'))).startsWith('本次运行尚未封存或封存核对未通过')
&& await pv(page, '[data-ev-prov-alert]').count() === 0)
// 接受理由挂在本期结论上(接受针对的是这一期的核对结果,不是这一行),写法取 4.2 的 accepted 原文「已接受,理由:…」,
// 不是 4.1 散文里的「已接受:理由」;行级结果单独一行,本期结论另起一行——d26_other 要同时看得到「这一行成立」和
// 「本期不成立 · 已接受」(4.4 第 26 条)
await openP(page, 'd26_bad')
const bad = await textOf(pv(page, '[data-ev-prov-check="R1"]'))
check('不成立的那一天:R1「这一行不成立」,接着写「本期不成立 · 已接受,理由:…」', bad.includes('这一行不成立')
&& bad.includes('本期不成立 · 已接受,理由:合成理由:分区合计口径调整(署名(未认证):未填写)'), bad)
await openP(page, 'd26_other')
const other = await textOf(pv(page, '[data-ev-prov-check="R1"]'))
check('同一期的另一天:「这一行成立」,本期结论照样写「本期不成立 · 已接受」', other.includes('这一行成立') && other.includes('本期不成立 · 已接受'), other)
// ---- 人工录入的统计期、列表
await openP(page, 'human')
const hItems = await fromOf(page)
check('统计期人工录入:年份取自人工录入的统计期(署名(未认证):录入员甲)', hItems.includes('年份取自人工录入的统计期(署名(未认证):录入员甲)')
&& (await textOf(pv(page, '[data-ev-prov-version]'))).includes('人工录入(署名(未认证):录入员甲)'), hItems.join('|'))
await openP(page, 'list')
check('列表:格子在列表头下面(列表头 F5),统计期未记录', await textOf(pv(page, '[data-ev-prov-cell]')) === '工作表「月报」F8'
&& (await fromOf(page)).includes('列表头 F5') && (await textOf(pv(page, '[data-ev-prov-version]'))).includes('第 1 次导入 · 统计期未记录'))
// ---- 27:多于 3 期的折叠
await openP(page, 'many')
const more = pv(page, '[data-ev-prov-more]')
check('四期:首行「按期累积 · 共 4 期」,只展开这一格所在的一期和最近一期,其余收进「另有 2 期」',
(await textOf(pv(page, '[data-ev-prov-head]'))).endsWith('按期累积 · 共 4 期') && (await partsOf(page)).join(',') === '2*,4'
&& await textOf(more) === '另有 2 期' && await more.getAttribute('aria-expanded') === 'false', (await partsOf(page)).join(','))
await more.focus()
await page.keyboard.press('Enter')
check('……「另有 2 期」是带 aria-expanded 的按钮:回车展开(true),四期按统计期列全,焦点留在按钮上',
await more.getAttribute('aria-expanded') === 'true' && (await partsOf(page)).join(',') === '1,2*,3,4' && await activeHas(page, 'data-ev-prov-more'),
(await partsOf(page)).join(','))
if (SHOTS) {
for (const theme of ['dark', 'light']) {
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
await page.waitForTimeout(250)
await panel(page).screenshot({ path: `${SHOTS}/evidence-p4-many-${theme}.png` })
}
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
}
await page.keyboard.press('Escape')
await page.waitForTimeout(200)
// ---- 17:360 宽的画布右栏
const wrap = async () => page.evaluate((sel) => {
const box = document.querySelector(sel)
const p = box?.querySelector('[data-evidence-panel]')
return { mode: p?.getAttribute('data-evidence-panel') ?? null, box: [box?.scrollWidth, box?.clientWidth], panel: p ? [p.scrollWidth, p.clientWidth] : null }
}, PVN)
const fits = (w) => w.mode === 'inline' && w.box[0] <= w.box[1] && !!w.panel && w.panel[0] <= w.panel[1]
await openP(page, 'wide', { box: PVN })
await page.locator(`${PVN} [data-ev-prov-details]`).click().catch(() => {})
const w1 = await wrap()
await openP(page, 'many', { box: PVN })
await page.locator(`${PVN} [data-ev-prov-more]`).click().catch(() => {})
const w2 = await wrap()
await openP(page, 'tampered', { box: PVN })
const w3 = await wrap()
check('360 宽的画布右栏:栏内展开;宽表一格(技术细节展开)、四期(全部展开)、标红三种都不出横向滚动',
fits(w1) && fits(w2) && fits(w3), JSON.stringify({ w1, w2, w3 }))
if (SHOTS) {
await openP(page, 'wide', { box: PVN })
for (const theme of ['dark', 'light']) {
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
await page.waitForTimeout(250)
await page.locator(PVN).screenshot({ path: `${SHOTS}/evidence-p4-360-${theme}.png` })
}
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
}
await page.locator(`${PVN} [data-evidence-panel] button`, { hasText: '回到正文' }).click().catch(() => {})
await page.waitForTimeout(150)
// ---- 24:加载态
const wideId = ids.wide
const slow = await open('/ui-harness.html?evidence=1', {
prov: async (run, sid, reply) => {
if (sid === wideId) await new Promise((res) => setTimeout(res, 300))
return reply
},
})
await slow.page.locator(`${PV} [data-seg="${wideId}"]`).click()
await slow.page.waitForSelector('[data-evidence-panel] [data-ev-prov-loading]', { timeout: 4000 }).catch(() => {})
const loading = await textOf(pv(slow.page, '[data-ev-provenance="loading"] [data-ev-prov-loading]'))
check('取的过程中:「推断的来源」一节写「正在推断来源…」,三节不先画空壳', loading === '正在推断来源…'
&& await pv(slow.page, '[data-ev-prov-version], [data-ev-prov-checks], [data-ev-prov-cell]').count() === 0, loading)
await slow.page.waitForSelector('[data-evidence-panel] [data-ev-provenance="inferred"]', { timeout: 4000 }).catch(() => {})
check('……答复到了以后加载态消失,三节画出来', await pv(slow.page, '[data-ev-prov-loading]').count() === 0
&& await pv(slow.page, '[data-ev-prov-version]').count() === 1 && await pv(slow.page, '[data-ev-prov-checks]').count() === 1)
check('没有运行时报错(加载态)', slow.errors.length === 0, slow.errors.join(' | '))
await slow.ctx.close()
// ---- 19:出错、老服务端。另守渲染矩阵本身:服务端误把数据版本带在 none · manifest_unreadable、table_only 的答复里
// 时照样按矩阵画(前者不画数据版本,后者不画格子)——不能只靠夹具里那些字段恰好为空。
// 清除、作废、导入的时间没记(契约里 at 可以是 null,夹具里恰好都有):略去这一项,不留孤立的「—」
const longId = ids.long
const untimed = (ans) => ({
...ans,
version: {
...ans.version,
parts: ans.version.parts.map((pt) => ({
...pt,
committed_at: null,
purged: pt.purged && { ...pt.purged, at: null },
revoked: pt.revoked && { ...pt.revoked, at: null },
})),
},
})
const sloppy = {
[ids.unreadable]: { ...fxp.provenance[ids.unreadable], version: fxp.provenance[wideId].version },
[ids.alias]: { ...fxp.provenance[ids.alias], cell_source: fxp.provenance[wideId].cell_source, checks: fxp.provenance[wideId].checks },
[ids.purged]: untimed(fxp.provenance[ids.purged]),
[ids.revoked]: untimed(fxp.provenance[ids.revoked]),
}
const broken = await open('/ui-harness.html?evidence=1', {
prov: (run, sid, reply) => (sid === wideId ? fxp.errors.server : sid === longId ? fxp.errors.old_server
: sloppy[sid] ? { json: sloppy[sid] } : reply),
})
await openP(broken.page, 'wide')
check('推断来源接口回 500:「推断的来源」一节只写「未能获取推断的来源」', await textOf(pv(broken.page, '[data-ev-provenance="error"] [data-ev-prov-failed]'))
=== '未能获取推断的来源' && await pv(broken.page, '[data-ev-prov-version], [data-ev-prov-checks], [data-ev-prov-cell]').count() === 0)
check('……其余几节照常:查询步骤、封存都在', await pv(broken.page, '[data-ev-query]').count() === 1
&& (await textOf(pv(broken.page, '[data-ev-seal]'))).includes('已封存 · 核对一致'))
await openP(broken.page, 'long', { wait: '[data-ev-query]' })
await broken.page.waitForTimeout(300)
check('回 404 且没有机读码(老服务端):请求发了,三节都不画', provHits(broken.hits, 'long') === 1 && await noSections(broken.page),
String(provHits(broken.hits, 'long')))
await openP(broken.page, 'unreadable')
check('渲染矩阵:清单无法读取时,答复里哪怕带着数据版本也不画,只画红色提示',
await pv(broken.page, '[data-ev-prov-alert="manifest_unreadable"]').count() === 1 && await pv(broken.page, '[data-ev-prov-version]').count() === 0)
await openP(broken.page, 'alias')
check('渲染矩阵:只给表级来历时,答复里哪怕带着格子和核对也不画', await pv(broken.page, '[data-ev-prov-reason]').count() === 1
&& await pv(broken.page, '[data-ev-prov-cell], [data-ev-prov-checks]').count() === 0)
await openP(broken.page, 'purged')
const purgedUntimed = await textOf(pv(broken.page, '[data-ev-prov-purged]'))
const verUntimed = await textOf(pv(broken.page, '[data-ev-prov-version]'))
check('清除时间、导入时间没记:写「原件已清除 · 理由:… · 署名(未认证):…」,不留孤立的「—」,也不写「导入于」',
/^原件已清除 · 理由:合成理由:原件不再保留 · 署名(未认证):录入员甲/.test(purgedUntimed)
&& !verUntimed.includes('—') && !verUntimed.includes('导入于')
&& await pv(broken.page, '[data-ev-prov-purged] [data-ev-prov-current]').count() === 1, verUntimed)
await openP(broken.page, 'revoked')
const revokedUntimed = await textOf(pv(broken.page, '[data-ev-prov-revoked]'))
check('作废时间没记:写「这一期的接受已作废 · 理由:… · 署名(未认证):未填写」,不留孤立的「—」',
/^这一期的接受已作废 · 理由:合成理由:接受依据有误 · 署名(未认证):未填写/.test(revokedUntimed)
&& !(await textOf(pv(broken.page, '[data-ev-prov-version]'))).includes('—'), revokedUntimed)
check('没有运行时报错(出错、老服务端)', broken.errors.length === 0, broken.errors.join(' | '))
await broken.ctx.close()
})
if (SHOTS) {
for (const theme of ['dark', 'light']) {
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
+2 -2
View File
@@ -894,8 +894,8 @@ await section('版本 · 导入清单', async () => {
const excluded = await root.locator('[data-summary-excluded]').innerText().catch(() => '')
check('排除的行:原因、锚点、行号', excluded.includes('排除的行') && excluded.includes('只核对、不另存的合计行') && excluded.includes('18-22 时合计')
&& excluded.includes('第 28–30 行'), excluded.replace(/\s+/g, ' '))
check('回执摘要:区域外文字全文,注明不提供给模型', (await block('receipt')).includes('客流汇总表')
&& (await block('receipt')).includes('区域外文字只在这里显示,不提供给模型'))
check('回执摘要:区域外文字全文,注明不导入数据表、裁判可见', (await block('receipt')).includes('客流汇总表')
&& (await block('receipt')).includes('区域外文字不导入数据表;核对报告时,裁判可以看到其中可见单元格的文字'))
check('AI 用量块:没有调用模型', (await block('ai')).includes('这次导入没有调用模型'))
check('清单校验通过时不出哈希警告', await root.locator('[data-manifest-unverified]').count() === 0)
const raw = root.locator('[data-manifest-raw]')