mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-16 01:12:00 +00:00
Excel 导入改造期 4:证据下钻——报告里引用上传表格的数,点开能看到它来自原表格的哪一格;裁判核对时看得到来源、口径、接受理由和区域外文字
推断的来源(新 engine/direct_select、data/provenance、data/provenance_db;api/evidence 新增推断来源接口) - 只对直接引用一列的查询下钻:受限文法用和执行时相同的 SQLite 分词读法,别名互换、自连接、表达式、聚合、子查询、集合运算(含藏在注释、字符串、方括号、转义、BOM 之后的)一律只给表级来历;再用 EXPLAIN 字节码和授权回调兜底 - 结果带齐主键、值非空,按主键参数化回查只读快照、类型和值严格一致才给格子;不一致时整份重算库哈希,哈希不符标红 - 从导入清单和溯源找回格子:宽表、交叉表逆透视的长表(日期取自表头格、年份取自统计期、行标签、分段标题)、列表、合计表、按期累积的并集(换算到所属那一期);原文与规范写法不同时一并列出 - 清单读不到、登记哈希与清单不一致、快照已回收、遮罩列,都只给表级来历或不给,写明原因;原件已清除、接受已作废照实显示;未封存的运行照常推断并标明 - 相关核对:这一行、这一期的核对结论与接受理由 裁判摘录(engine/judge) - 带期 4 标记的报告,上传表格的查询旁加来源行、口径行、接受理由、可见单元格里的区域外文字和人工录入的统计期;限定语写明这些数未经系统核对,长度有上限、截断时写明 - 隐藏行列里的区域外文字不送;设了遮罩的源只写条数 前端证据面板(run/EvidenceProvenance;ManifestBody 从版本页抽出复用) - 「推断的来源」一节与已封存的证据分开标注:格子坐标、取自哪些格、年份来源、规范写法、回查、相关核对、数据版本与各期;标红与拒绝原因按状态显示 不变的部分 - 文档顶层加 provenance 标记,只有新报告走新路径;期 4 之前的运行、非上传源、手工源的面板、复核、裁判摘录逐字不变,由金样测试钉住 - 执行器给区域外文字记录是否在隐藏行列;不建「导入校验」系统表(D6)
This commit is contained in:
@@ -208,7 +208,8 @@ flowchart LR
|
||||
要进口径卡。数据源可设 `mask_columns` 遮罩列(只减少暴露,不是安全边界)。
|
||||
|
||||
**点开看到什么。** 数字:指标的原式、代入式和复算结果,输入与快照是否一致,查询的 SQL、被引用的行加前后
|
||||
各 2 行、高亮到格,最后是封存状态;只认封存范围内的事件交回过的工件。表名、字段名:出现在哪次查询、类型、
|
||||
各 2 行、高亮到格,最后是封存状态;只认封存范围内的事件交回过的工件。上传表格的单元格可以追到原表格子
|
||||
(推断的来源)。表名、字段名:出现在哪次查询、类型、
|
||||
所属的表。表结构快照、SQL、结果列里都没有的名字标成可疑实体(「可能是编造的名字」,附最接近的 3 个);
|
||||
快照被截断、核对不了的只说核对不了,不冤枉成编造。
|
||||
|
||||
|
||||
+493
-24
@@ -31,22 +31,49 @@ import asyncio
|
||||
import contextlib
|
||||
import csv
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
import sqlite3
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from dataclasses import asdict, dataclass, field, replace
|
||||
from pathlib import Path
|
||||
from typing import Any, get_args
|
||||
|
||||
from fastapi import APIRouter, Body, Query
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.exc import StatementError
|
||||
|
||||
from app.api.coded import CodedHTTPException
|
||||
from app.core.artifact_store import load
|
||||
from app.core.events import EventType
|
||||
from app.data.engine import masked_columns
|
||||
from app.data import provenance as prov
|
||||
from app.data import provenance_db
|
||||
from app.data.engine import SnapshotTampered, engines, masked_columns
|
||||
from app.data.names import name_key
|
||||
from app.data.provenance_types import (
|
||||
DOC_PROVENANCE,
|
||||
Alert,
|
||||
CellRef,
|
||||
Chain,
|
||||
ChainProblem,
|
||||
LocateProblem,
|
||||
ProvenanceOut,
|
||||
RawState,
|
||||
Reason,
|
||||
Recheck,
|
||||
RelatedCheck,
|
||||
ReportRef,
|
||||
SelectRefusal,
|
||||
VersionView,
|
||||
alert,
|
||||
refuse,
|
||||
)
|
||||
from app.data.table_versions import SnapshotManifestMismatch, SnapshotMissing
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource, Run, RunEvent, Workflow
|
||||
from app.engine import direct_select
|
||||
from app.engine.evidence import (
|
||||
ENTITY_KINDS,
|
||||
GUESS_NOTE,
|
||||
@@ -76,6 +103,7 @@ from app.engine.judge import FIELDS_MAX, SETTLED, UNSUPPORTED, VERDICTS, candida
|
||||
from app.engine.toolcalls import QUERY_PREFIX
|
||||
|
||||
router = APIRouter(prefix="/api/runs", tags=["evidence"])
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SCHEMA = "agentlab.evidence/1"
|
||||
|
||||
@@ -673,20 +701,8 @@ async def evidence_segment(run_id: str, segment_id: str, report: str | None = No
|
||||
一次运行里有好几份报告时用 ?report=<节点 id> 指定;不指定取成果里标注的那一份。
|
||||
"""
|
||||
sealed = await _sealed(run_id)
|
||||
reports = _reports(sealed)
|
||||
if not reports:
|
||||
raise CodedHTTPException(404, "本次运行没有封存的报告文档,没有可查看的片段", REPORT_NOT_FOUND)
|
||||
chosen = _choose(reports, report, sealed)
|
||||
if chosen.hash_ok is False:
|
||||
raise CodedHTTPException(409, "报告文档与哈希不一致,疑似被修改,不能作为证据展示", DOC_TAMPERED)
|
||||
if chosen.doc is None:
|
||||
raise CodedHTTPException(404, "无法读取报告文档", DOC_MISSING)
|
||||
found = find_segment(chosen.doc, segment_id)
|
||||
if found is None:
|
||||
raise CodedHTTPException(404, "报告中没有这个片段", SEGMENT_NOT_FOUND)
|
||||
|
||||
block, unit, seg = found
|
||||
doc = chosen.doc
|
||||
chosen, (block, unit, seg) = _open_segment(sealed, report, segment_id)
|
||||
doc = chosen.doc or {}
|
||||
markdown = str(doc.get("markdown") or "")
|
||||
start, end = unit.get("span") or [0, 0]
|
||||
chain = await _chain(seg, doc, sealed, chosen.node_id)
|
||||
@@ -712,6 +728,23 @@ async def evidence_segment(run_id: str, segment_id: str, report: str | None = No
|
||||
}
|
||||
|
||||
|
||||
def _open_segment(sealed: _Sealed, wanted: str | None, segment_id: str
|
||||
) -> tuple[_Report, tuple[dict[str, Any], dict[str, Any], dict[str, Any]]]:
|
||||
"""选报告、确认文档完好、找到片段:(报告, (块, 句子, 片段))。片段接口和推断来源接口共用同一套错误码。"""
|
||||
reports = _reports(sealed)
|
||||
if not reports:
|
||||
raise CodedHTTPException(404, "本次运行没有封存的报告文档,没有可查看的片段", REPORT_NOT_FOUND)
|
||||
chosen = _choose(reports, wanted, sealed)
|
||||
if chosen.hash_ok is False:
|
||||
raise CodedHTTPException(409, "报告文档与哈希不一致,疑似被修改,不能作为证据展示", DOC_TAMPERED)
|
||||
if chosen.doc is None:
|
||||
raise CodedHTTPException(404, "无法读取报告文档", DOC_MISSING)
|
||||
found = find_segment(chosen.doc, segment_id)
|
||||
if found is None:
|
||||
raise CodedHTTPException(404, "报告中没有这个片段", SEGMENT_NOT_FOUND)
|
||||
return chosen, found
|
||||
|
||||
|
||||
def _choose(reports: list[_Report], wanted: str | None, sealed: _Sealed) -> _Report:
|
||||
if wanted:
|
||||
chosen = next((r for r in reports if r.node_id == wanted), None)
|
||||
@@ -743,8 +776,10 @@ async def _chain(seg: dict[str, Any], doc: dict[str, Any], sealed: _Sealed,
|
||||
return []
|
||||
if cite.get("kind") == "cell":
|
||||
locator = cite.get("locator") or {}
|
||||
# 只有直接引用的单元格才可能推断来源(P4-SPEC 2.8.2)。指标链里的查询步骤不带:指标片段的推断来源只会
|
||||
# 回「不是单元格」,带上提示等于让界面白发一次请求
|
||||
return [await _query_step(str(entry.get("artifact") or ""), [(locator.get("row"), locator.get("column"))],
|
||||
doc, sealed, masks)]
|
||||
doc, sealed, masks, hint=doc.get("provenance") == DOC_PROVENANCE)]
|
||||
if cite.get("kind") == "input":
|
||||
name = (cite.get("locator") or {}).get("field")
|
||||
inputs = _input_payloads(sealed)
|
||||
@@ -857,11 +892,15 @@ class _Masks:
|
||||
|
||||
|
||||
async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str, Any], sealed: _Sealed,
|
||||
masks: _Masks) -> dict[str, Any]:
|
||||
masks: _Masks, *, hint: bool = False) -> dict[str, Any]:
|
||||
"""查询步骤:被引用的格所在的行加前后各 WINDOW 行,高亮那几格,遮掉数据源设了遮罩的列。
|
||||
|
||||
快照只认封存范围内的事件交回过的那些(tool.end.query_artifact、node.finished.evidence 的
|
||||
query 条目);目录里写着、事件里追不到的,照实说不认,一行都不给。
|
||||
|
||||
hint(期 4,P4-SPEC 2.8.2):调用方是带文档标记的单元格片段时为真。这时快照若是上传表格的(有 data_version),
|
||||
步骤多一个 `provenance: true`,界面据此才去请求推断来源接口;其余情况一个键都不加,形状和期 4 之前完全相同
|
||||
(期 4 之前的文档、手工源、指标链都不多发请求)。不看封存状态:未封存的运行照常推断,响应里照实标明。
|
||||
"""
|
||||
catalog = doc.get("catalog") or {}
|
||||
alias = _query_alias(catalog, artifact)
|
||||
@@ -905,7 +944,8 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str
|
||||
step.update(sql=snap.get("sql"), columns=columns, total_rows=len(rows), truncated=bool(snap.get("truncated")),
|
||||
masked=[columns[i] for i in sorted(masked)],
|
||||
# 查询时从驱动的原始值记下的列类型(老快照没有):文本列里的 "2026" 按文本显示,不当数
|
||||
column_types=snap.get("column_types") if isinstance(snap.get("column_types"), dict) else {})
|
||||
column_types=snap.get("column_types") if isinstance(snap.get("column_types"), dict) else {},
|
||||
**({"provenance": True} if hint and snap.get("data_version") else {}))
|
||||
if len(index) > MAX_WINDOW_ROWS:
|
||||
index, step["window_truncated"] = index[:MAX_WINDOW_ROWS], True
|
||||
step["row_index"] = index
|
||||
@@ -915,6 +955,395 @@ async def _query_step(artifact: str, cells: list[tuple[Any, Any]], doc: dict[str
|
||||
return step
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# GET /api/runs/{run_id}/evidence/segments/{segment_id}/provenance(期 4:推断的来源)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/{run_id}/evidence/segments/{segment_id}/provenance")
|
||||
async def evidence_provenance(run_id: str, segment_id: str, report: str | None = None) -> dict[str, Any]:
|
||||
"""点开报告里一个直接引用的查询单元格时,这一格来自原表的哪个工作表、哪一格(P4-SPEC 2.8.1)。
|
||||
|
||||
选报告、错误码和片段接口相同(run_not_found、evidence_report_not_found、evidence_doc_tampered、
|
||||
evidence_doc_missing、evidence_segment_not_found)。其余情况一律 200:推断不出来不是错误,结论写在
|
||||
status、reason、alert 里。按需算、不缓存(2.9):原件是否清除、数据文件是否被改都是当前状态,事后会变。
|
||||
|
||||
返回 `asdict(ProvenanceOut)`:CellSource 的「from」在 Python 里叫 from_,asdict 已经换回「from」;返回注解
|
||||
写 dict,不写 ProvenanceOut,和其余证据路由一样不让框架按类型再校验、再序列化一遍(provenance_types 模块说明)。
|
||||
"""
|
||||
sealed = await _sealed(run_id)
|
||||
chosen, (_block, _unit, seg) = _open_segment(sealed, report, segment_id)
|
||||
out = await segment_provenance(chosen, seg, sealed, masks=_Masks())
|
||||
return asdict(out)
|
||||
|
||||
|
||||
#: R14 之后(绑定快照、取引擎、编译核对、回查、行级核对)会抛的三类异常,按 _snapshot_failure 的顺序映射
|
||||
_SNAPSHOT_ERRORS: tuple[type[BaseException], ...] = (SnapshotMissing, SnapshotTampered)
|
||||
|
||||
|
||||
def _snapshot_failure(exc: BaseException) -> tuple[Reason, Alert | None]:
|
||||
"""R14 之后抛出的数据文件异常 → (原因, 标红提示)(P4-SPEC 2.3 的异常映射)。
|
||||
|
||||
**顺序是守卫点**:SnapshotManifestMismatch 同时继承 SnapshotMissing 和引擎的 SnapshotTampered
|
||||
(table_versions.py 的定义),必须最先认,否则会被当成文件被改(db_tampered)或文件不在(snapshot_gone):
|
||||
1. SnapshotManifestMismatch:登记的哈希与快照清单对不上 → chain_mismatch,标红;
|
||||
2. SnapshotTampered(含借连接时指纹不符):数据文件与登记的版本不一致 → db_tampered,标红;
|
||||
3. SnapshotMissing:绑定时文件没了、记录不全 → snapshot_gone,不标红(被运行引用的版本受保护,出现多半是
|
||||
数据源被删或手工清理)。
|
||||
三种都是 table_only:数据版本一节只靠清单,照样给;不给格子,已经算出的行级状态一律丢掉(调用方负责)。
|
||||
不是这三类的异常原样抛出:不该被当成「推断不出来」吞掉。
|
||||
"""
|
||||
if isinstance(exc, SnapshotManifestMismatch):
|
||||
return refuse("chain_mismatch"), alert("chain_mismatch")
|
||||
if isinstance(exc, SnapshotTampered):
|
||||
return refuse("db_tampered"), alert("db_tampered")
|
||||
if isinstance(exc, SnapshotMissing):
|
||||
return refuse("snapshot_gone"), None
|
||||
raise exc
|
||||
|
||||
|
||||
class _Artifacts:
|
||||
"""一次请求里取过的工件(P4-SPEC 2.9「同一工件只取一次」)。取回的结果按 _fetch 的口径记 (内容, hash_ok)。
|
||||
|
||||
- `await get(id)`:在线程里取(快照、清单可能几十 KB 到几 MB,读文件、复验哈希不该卡事件循环,H9);
|
||||
- `load(id)`:同步取,语义同 artifact_store.load(取不到 None,哈希不符抛 ValueError),给已经在线程里跑的
|
||||
纯函数当 loader 用(WP-B 的 resolve_chain 等)。两条路共用同一份备忘。
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._memo: dict[str, tuple[Any, bool | None]] = {}
|
||||
|
||||
async def get(self, artifact: str) -> tuple[Any, bool | None]:
|
||||
key = str(artifact or "")
|
||||
if not key:
|
||||
return None, None
|
||||
if key not in self._memo:
|
||||
self._memo[key] = await asyncio.to_thread(_fetch, key)
|
||||
return self._memo[key]
|
||||
|
||||
def load(self, artifact: str) -> Any:
|
||||
key = str(artifact or "")
|
||||
if not key:
|
||||
return None
|
||||
if key not in self._memo:
|
||||
self._memo[key] = _fetch(key)
|
||||
content, ok = self._memo[key]
|
||||
if ok is False:
|
||||
raise ValueError(f"工件 {key[:12]} 与哈希不一致")
|
||||
return content
|
||||
|
||||
|
||||
def _cell_ref(cite: dict[str, Any], entry: dict[str, Any]) -> CellRef | None:
|
||||
"""片段本身是单元格引用(有 alias 和 row、column 齐全的定位)就给,与 status 无关(2.8.1「字段取值」)。"""
|
||||
locator = cite.get("locator") or {}
|
||||
row, column = locator.get("row"), locator.get("column")
|
||||
if cite.get("kind") != "cell" or not cite.get("alias") or not isinstance(row, int) or isinstance(row, bool) \
|
||||
or not isinstance(column, str) or not column:
|
||||
return None
|
||||
artifact = entry.get("artifact")
|
||||
return CellRef(alias=str(cite["alias"]), row=row, column=column, artifact=str(artifact) if artifact else None)
|
||||
|
||||
|
||||
async def segment_provenance(report: _Report, seg: dict[str, Any], sealed: _Sealed, *,
|
||||
masks: _Masks) -> ProvenanceOut:
|
||||
"""一个片段的推断来源:按 P4-SPEC 2.3 的规则表顺序判断,第一条不满足的就是结论。
|
||||
|
||||
**封存状态不进规则**(1.3 调整 11):未封存(停在审批上)、封存核对不通过的运行照常判断,`sealed` 照实给
|
||||
sealed.trusted,界面另加一句。查询快照、表结构快照按「本次运行的事件交回过」认(sealed.queries、
|
||||
sealed.schemas,_sealed 对未封存的运行按全部事件拼),不经 _sealed_loader:它在 trusted 为假时一律不给,
|
||||
会把未封存的运行误判成清单无法读取(2.8.3)。
|
||||
|
||||
这里判 R0–R4(只看文档、目录和两份快照),R5 起交给 _drill:清单链(provenance)、SQL 判据(direct_select)、
|
||||
遮罩、空值、数据文件(provenance_db)、找回格子和相关核对。
|
||||
"""
|
||||
doc = report.doc or {}
|
||||
cite = seg.get("cite") or {}
|
||||
entry = (doc.get("catalog") or {}).get(cite.get("alias")) or {}
|
||||
base: dict[str, Any] = {
|
||||
"report": ReportRef(node_id=report.node_id, doc_artifact=report.doc_artifact),
|
||||
"segment": str(seg.get("id") or ""), "cell": _cell_ref(cite, entry), "sealed": sealed.trusted,
|
||||
}
|
||||
|
||||
def none(code: Any) -> ProvenanceOut:
|
||||
return ProvenanceOut(status="none", reason=refuse(code), **base)
|
||||
|
||||
# R0:期 4 之前组装的文档(没有标记)一律不推断,期 4 之前的面板不变(1.4)
|
||||
if doc.get("provenance") != DOC_PROVENANCE:
|
||||
return none("legacy_doc")
|
||||
# R1:只对直接引用的查询单元格下钻;代码节点、Agent 字段的取数(工具名不是 db_query__ 开头)不算
|
||||
if (cite.get("kind") != "cell" or cite.get("status") != "resolved" or seg.get("state") != "deterministic"
|
||||
or entry.get("kind") != "query" or not str(entry.get("tool") or "").startswith(QUERY_PREFIX)):
|
||||
return none("not_cell")
|
||||
artifacts = _Artifacts()
|
||||
# R2:查询快照是本次运行的事件交回过的,取回时哈希复验通过
|
||||
artifact = str(entry.get("artifact") or "")
|
||||
query, ok = await artifacts.get(artifact) if artifact in sealed.queries else (None, None)
|
||||
if ok is not True or not isinstance(query, dict):
|
||||
return none("not_sealed")
|
||||
# R3:上传表格的查询快照才有 data_version(datasource.py 只给上传源写)
|
||||
if not query.get("data_version"):
|
||||
return none("not_upload")
|
||||
# R4:表结构快照同样要是本次运行交回过、复验通过的;不是按配方导入的(简单导入、v0 迁移出的版本)没有溯源
|
||||
schema_id = str(query.get("schema_artifact") or "")
|
||||
schema, ok = await artifacts.get(schema_id) if schema_id in sealed.schemas else (None, None)
|
||||
if ok is not True or not isinstance(schema, dict):
|
||||
return none("not_sealed")
|
||||
if schema.get("import_mode") != "recipe":
|
||||
return none("simple_upload")
|
||||
return await _drill(base, seg=seg, entry=entry, query=query, schema=schema, artifacts=artifacts,
|
||||
masks=masks, sealed=sealed)
|
||||
|
||||
|
||||
async def _drill(base: dict[str, Any], *, seg: dict[str, Any], entry: dict[str, Any], query: dict[str, Any],
|
||||
schema: dict[str, Any], artifacts: _Artifacts, masks: _Masks, sealed: _Sealed) -> ProvenanceOut:
|
||||
"""R5–R17(P4-SPEC 2.3):第一条不满足的就是结论。
|
||||
|
||||
- R5、R6 只看内容寻址的工件(resolve_chain,放线程:要逐份取回清单、复验哈希)。链断了就什么来历都不给:
|
||||
status none、version 为 null,标红;
|
||||
- 之后的每一条都照样给数据版本(它只靠清单),不给格子:R7–R11 只看查询快照和冻结表结构,R12、R13 对照数据库
|
||||
里的数据文件记录,R14 起打开数据文件。R14 之后任何一步抛出的数据文件异常按 _snapshot_failure 的顺序映射,
|
||||
已经算出的格子、行级状态一律丢掉(2.3 末尾);
|
||||
- 全部满足才给格子(inferred)。
|
||||
"""
|
||||
# R5(取不回、哈希不符)、R6(链本身对不上):resolve_chain 不往外抛,结论在 ChainProblem 里
|
||||
chain = await asyncio.to_thread(prov.resolve_chain, query, schema, artifacts.load)
|
||||
if isinstance(chain, ChainProblem):
|
||||
return ProvenanceOut(status="none", reason=refuse(chain.code, chain.detail), alert=alert(chain.code), **base)
|
||||
|
||||
frozen = _frozen_tables(schema)
|
||||
tables = {name: _frozen_columns(meta) for name, meta in frozen.items()}
|
||||
sql = str(query.get("sql") or "")
|
||||
# 遮罩 = 数据源现在设的 + 查询当时记下的(同 _query_step 的口径:数据源改名、删掉了也照遮)
|
||||
source_name = (sealed.queries.get(str(entry.get("artifact") or "")) or {}).get("source") \
|
||||
or entry.get("source") or query.get("source")
|
||||
current, _found = await masks.of(source_name)
|
||||
hidden = current | _recorded_masks(query)
|
||||
async with SessionLocal() as session:
|
||||
states = await provenance_db.import_states(session, [p.import_id for p in chain.parts])
|
||||
record = await provenance_db.snapshot_record(session, chain.snapshot_id)
|
||||
# 版本页同一个口径的表:SQL 里出现的冻结表结构的表(tables_in 宁可多认,2.4)
|
||||
version = _version_of(chain, direct_select.tables_in(sql, tables), states, manifest_view=not hidden)
|
||||
|
||||
def table_only(reason: Reason, red: Alert | None = None) -> ProvenanceOut:
|
||||
return ProvenanceOut(status="table_only", reason=reason, alert=red, version=version, **base)
|
||||
|
||||
# R7:受限文法内的直接选取(识别器和执行这条 SQL 的 SQLite 读同一串记号)。传的是快照里原样存下的 SQL
|
||||
columns = [str(c) for c in query.get("columns") or []]
|
||||
picked = direct_select.recognize(sql, result_columns=columns, tables=tables)
|
||||
if isinstance(picked, SelectRefusal):
|
||||
return table_only(refuse(picked.code, picked.detail))
|
||||
# R8:被引用的结果列按第一次出现的位置对应选取项(locate_cell 也按第一次出现取)。识别器已拒绝重名,
|
||||
# 这里对不上只是防御
|
||||
locator = (seg.get("cite") or {}).get("locator") or {}
|
||||
row, column = locator.get("row"), locator.get("column")
|
||||
if not isinstance(column, str) or column not in columns or columns.index(column) >= len(picked.columns):
|
||||
return table_only(refuse("unparsed"))
|
||||
at = columns.index(column)
|
||||
target = picked.columns[at]
|
||||
# R9:表有主键,主键各列都以直接引用出现在结果里;结果各行的主键两两不同(与解析无关的兜底:同一张表经集合
|
||||
# 运算拼出来的结果可能重复主键)
|
||||
meta = frozen.get(picked.table)
|
||||
pk_names = [str(p) for p in (meta.get("primary_key") or [])] if isinstance(meta, dict) else []
|
||||
if not pk_names:
|
||||
return table_only(refuse("no_pk"))
|
||||
where = {name_key(c): j for j, c in enumerate(picked.columns)}
|
||||
missing = [p for p in pk_names if name_key(p) not in where]
|
||||
if missing:
|
||||
return table_only(refuse("pk_missing", 列=missing))
|
||||
rows = query.get("rows")
|
||||
if not isinstance(rows, list) or not all(isinstance(r, list) and len(r) == len(columns) for r in rows):
|
||||
return table_only(refuse("unparsed"))
|
||||
pk_at = [where[name_key(p)] for p in pk_names]
|
||||
if _repeats([tuple(r[j] for j in pk_at) for r in rows]):
|
||||
return table_only(refuse("multi_table", "duplicate_pk"))
|
||||
# R10:被引用的列、主键列、推断要用到的键列(维度、派生、常量:行标签原文、分段标题的定位文字都是它们的值)
|
||||
# 都不在遮罩里
|
||||
needed = [column, target, *pk_names, *_key_columns(chain, picked.table)]
|
||||
if any(str(n).lower() in hidden for n in needed):
|
||||
return table_only(refuse("masked"))
|
||||
# R11:被引用的值、主键各值都不是空值
|
||||
if isinstance(row, bool) or not isinstance(row, int) or not 0 <= row < len(rows):
|
||||
return table_only(refuse("unparsed"))
|
||||
value = rows[row][at]
|
||||
if value is None:
|
||||
return table_only(refuse("null_value"))
|
||||
pk = {p: rows[row][j] for p, j in zip(pk_names, pk_at)}
|
||||
if any(v is None for v in pk.values()):
|
||||
return table_only(refuse("null_pk"))
|
||||
# R12:数据文件记录在、没回收、文件在(不标红:被运行引用的版本受保护,出现多半是数据源被删或手工清理)
|
||||
if record is None or record.retired_at is not None or not record.db_path or not Path(record.db_path).is_file():
|
||||
return table_only(refuse("snapshot_gone"))
|
||||
# R13:登记哈希等于清单里期望的库哈希、source_id 一致(防「数据文件和登记哈希被一起改成一致的假值」:
|
||||
# 清单是内容寻址的,又挂在封存的表结构快照下面,改不了)。登记哈希为空也算不等。
|
||||
# Reason.detail 不带细分:契约只给 DETAILS 和 ChainProblem / LocateProblem 的细分留了位置,R13 规格只写
|
||||
# chain_mismatch,和异常映射(SnapshotManifestMismatch)同一个口径;哪一项对不上写进日志
|
||||
registered = str(record.db_sha256 or "").strip().lower()
|
||||
if not registered or registered != chain.expected_db_sha256.strip().lower():
|
||||
logger.warning("provenance: 快照 %s 的登记哈希与导入清单里的库哈希不一致", chain.snapshot_id)
|
||||
return table_only(refuse("chain_mismatch"), alert("chain_mismatch"))
|
||||
if str(record.source_id) != chain.source_id:
|
||||
logger.warning("provenance: 快照 %s 的 source_id 与导入清单不一致", chain.snapshot_id)
|
||||
return table_only(refuse("chain_mismatch"), alert("chain_mismatch"))
|
||||
try:
|
||||
return await _open_and_locate(base, table_only, chain=chain, record=record, version=version,
|
||||
states=states, hidden=hidden, sql=sql, table=picked.table, target=target,
|
||||
value=value, pk=pk)
|
||||
except _SNAPSHOT_ERRORS as exc:
|
||||
# R14 之后(绑定、取引擎、编译核对、回查、行级核对)任何一步:不给格子,已算出的行级状态一律丢掉
|
||||
reason, red = _snapshot_failure(exc)
|
||||
return table_only(reason, red)
|
||||
|
||||
|
||||
async def _open_and_locate(base: dict[str, Any], table_only: Callable[..., ProvenanceOut], *, chain: Chain,
|
||||
record: Any, version: VersionView, states: dict[str, dict[str, Any]], hidden: set[str],
|
||||
sql: str, table: str, target: str, value: Any, pk: dict[str, Any]) -> ProvenanceOut:
|
||||
"""R14–R17 和行级核对:要打开数据文件的几步。三类数据文件异常原样往外抛,由 _drill 统一映射。"""
|
||||
# R14:绑定快照(累积快照首次绑定时和快照清单交叉核对),经 EngineCache 打开数据文件(只读、immutable,本进程
|
||||
# 首次整份核对哈希)。之后的回查、行级核对都经同一个缓存键,读的就是查询读过的那个文件
|
||||
view = provenance_db.view_of(record, source_name=chain.source)
|
||||
await engines.get(view)
|
||||
# R15:第二道核对,与解析无关。读到的表恰好只有识别器认出的那一张;字节码里 ResultRow 恰好 1 个、没有 Yield
|
||||
# (UNION ALL 有多个 ResultRow,UNION、INTERSECT、EXCEPT 有 Yield)。编译不了(SQLite 自己报错)同样不下钻
|
||||
try:
|
||||
facts = await provenance_db.compile_facts(view, sql)
|
||||
except sqlite3.Error:
|
||||
logger.warning("provenance: 编译核对失败,只给表级来历", exc_info=True)
|
||||
return table_only(refuse("unparsed"))
|
||||
if {name_key(t) for t in facts.tables} != {name_key(table)}:
|
||||
return table_only(refuse("multi_table", "authorizer"))
|
||||
if facts.result_rows != 1 or facts.yields != 0:
|
||||
return table_only(refuse("multi_table", "compound"))
|
||||
# R16:按主键参数化回查,恰好一行,值和快照里那一格类型、值都相等
|
||||
try:
|
||||
found, recheck_sql, params = await provenance_db.recheck(view, table=table, column=target, pk=pk)
|
||||
except (sqlite3.Error, StatementError) as e:
|
||||
_reraise_snapshot_error(e)
|
||||
# 细分同样不进 Reason.detail(契约外的值),回查出错和回查没找到行给同一个原因,区别只在日志
|
||||
logger.warning("provenance: 回查失败,只给表级来历", exc_info=True)
|
||||
return table_only(refuse("recheck_missing"))
|
||||
if not found:
|
||||
return table_only(refuse("recheck_missing"))
|
||||
if len(found) > 1:
|
||||
return table_only(refuse("recheck_multiple"))
|
||||
rowid, now = found[0]
|
||||
if not _same_cell(now, value):
|
||||
# 值对不上先强制整份重算一次哈希(不走 EngineCache 的指纹捷径:原地改写同样字节数、再还原修改时间的改动,
|
||||
# 指纹认不出,P4-SPEC 2.6)。文件确实变了 → 标红;没变 → 这条 SQL 的含义和判据认定的不同,不标红
|
||||
if not await provenance_db.rehash(view):
|
||||
return table_only(refuse("db_tampered"), alert("db_tampered"))
|
||||
return table_only(refuse("recheck_mismatch"))
|
||||
# R17:从 (表, 列, rowid) 找回格子(并集 rowid 先换算到某一期)。纯函数,但要解析配方、还原回执,放线程
|
||||
located = await asyncio.to_thread(prov.locate, chain, table=table, column=target, rowid=rowid)
|
||||
if isinstance(located, LocateProblem):
|
||||
if located.code == "chain_mismatch":
|
||||
return table_only(refuse("chain_mismatch", located.detail), alert("chain_mismatch"))
|
||||
return table_only(refuse("no_lineage", located.detail))
|
||||
# 附带的格写出的是哪几列的值(R10 已按回执里的键列判过,这里按实际给出的再核一次,防回执和推断不一致)
|
||||
if any(str(f.column).lower() in hidden for f in located.cell.from_ if f.column):
|
||||
return table_only(refuse("masked"))
|
||||
plans = await asyncio.to_thread(prov.related_checks, chain, located, table=table, column=target)
|
||||
checks: list[RelatedCheck] = []
|
||||
for plan in plans:
|
||||
if plan.rule is None:
|
||||
checks.append(plan.check)
|
||||
continue
|
||||
# 行级状态按**快照库**的 rowid 跑(并集时是并集 rowid),条件和导入时的核对是同一段代码生成的
|
||||
try:
|
||||
status = await provenance_db.row_status(view, plan.rule, rowid)
|
||||
except (sqlite3.Error, StatementError) as e:
|
||||
_reraise_snapshot_error(e)
|
||||
logger.warning("provenance: 行级核对失败,这一行不给结论", exc_info=True)
|
||||
status = None
|
||||
checks.append(replace(plan.check, row_status=status))
|
||||
state = states.get(located.part.import_id) or {}
|
||||
cell = replace(located.cell, rowid=rowid, pk=pk, raw_purged=state.get("raw_state") == "purged",
|
||||
recheck=Recheck(sql=recheck_sql, params=list(params), ok=True))
|
||||
version = replace(version, parts=[replace(p, has_row=p.import_id == located.part.import_id)
|
||||
for p in version.parts])
|
||||
return ProvenanceOut(status="inferred", version=version, cell_source=cell, checks=checks, **base)
|
||||
|
||||
|
||||
def _reraise_snapshot_error(exc: BaseException) -> None:
|
||||
"""SQLAlchemy 把连接池事件里抛出的异常包成 StatementError 时,数据文件的三类异常要原样交给 _drill 映射,
|
||||
不能被当成「这条 SQL 执行失败」吞掉(借连接时指纹不符抛的 SnapshotTampered 就在这一路上)。"""
|
||||
orig = getattr(exc, "orig", None)
|
||||
if isinstance(orig, _SNAPSHOT_ERRORS):
|
||||
raise orig from exc
|
||||
|
||||
|
||||
def _frozen_tables(schema: dict[str, Any]) -> dict[str, Any]:
|
||||
tables = schema.get("tables") if isinstance(schema, dict) else None
|
||||
return tables if isinstance(tables, dict) else {}
|
||||
|
||||
|
||||
def _frozen_columns(meta: Any) -> list[str]:
|
||||
"""冻结表结构里一张表的列名,按定义顺序(`SELECT *` 要按这个顺序比对结果列)。"""
|
||||
cols = meta.get("columns") if isinstance(meta, dict) else None
|
||||
return [str(c["name"]) for c in cols or [] if isinstance(c, dict) and c.get("name") is not None]
|
||||
|
||||
|
||||
def _recorded_masks(query: dict[str, Any]) -> set[str]:
|
||||
recorded = query.get("mask_columns")
|
||||
return {c.lower() for c in recorded if isinstance(c, str)} if isinstance(recorded, list) else set()
|
||||
|
||||
|
||||
#: 推断要用到的键列(回执 tables[].columns[].role):维度列的行标签原文、派生列由行标签解析、常量列取分段标题的
|
||||
#: 定位文字,展示出来就是这几列的值(R10)
|
||||
_KEY_COLUMN_ROLES = frozenset({"dim", "derive", "const"})
|
||||
|
||||
|
||||
def _key_columns(chain: Chain, table: str) -> list[str]:
|
||||
"""各期回执里这张表的键列(维度、派生、常量)。取全部各期的并集:R10 在知道格子落在哪一期之前判,宁可多遮。"""
|
||||
out: list[str] = []
|
||||
for part in chain.parts:
|
||||
receipt = part.manifest.get("receipt") if isinstance(part.manifest.get("receipt"), dict) else {}
|
||||
for t in receipt.get("tables") or []:
|
||||
if not isinstance(t, dict) or t.get("name") != table:
|
||||
continue
|
||||
out += [str(c["name"]) for c in t.get("columns") or []
|
||||
if isinstance(c, dict) and c.get("role") in _KEY_COLUMN_ROLES and c.get("name")]
|
||||
return list(dict.fromkeys(out))
|
||||
|
||||
|
||||
def _repeats(keys: list[tuple[Any, ...]]) -> bool:
|
||||
"""主键元组有没有重复。按 Python 的相等比(5 和 5.0 算同一个):宁可多判重复、少下钻。"""
|
||||
seen: set[Any] = set()
|
||||
for key in keys:
|
||||
try:
|
||||
marker: Any = key
|
||||
hash(marker)
|
||||
except TypeError:
|
||||
marker = repr(key)
|
||||
if marker in seen:
|
||||
return True
|
||||
seen.add(marker)
|
||||
return False
|
||||
|
||||
|
||||
def _same_cell(now: Any, then: Any) -> bool:
|
||||
"""回查的值和快照里那一格:类型和值都相等(int、float、str 分开比,8754 和 8754.0 不算相等;bool 不算数)。
|
||||
|
||||
两边都经 engine._jsonable 同一口径规范化过(快照还经过一次 JSON 存取,int、float、str 原样回来),类型不同
|
||||
只可能是文件或 SQL 的含义变了,不能按数值宽松地放过。"""
|
||||
if isinstance(now, bool) or isinstance(then, bool):
|
||||
return False
|
||||
return type(now) is type(then) and now == then
|
||||
|
||||
|
||||
def _version_of(chain: Chain, tables: list[str], states: dict[str, dict[str, Any]], *,
|
||||
manifest_view: bool) -> VersionView:
|
||||
"""数据版本:清单里的部分由 provenance.version_view 给(内容寻址),原件状态、清除、作废、配方第几版取自导入
|
||||
记录(当前状态,界面标「当前状态」)。manifest_view:数据源设有任何遮罩时不在面板里出「查看导入清单」。"""
|
||||
view = prov.version_view(chain, tables=tables)
|
||||
parts = []
|
||||
for p in view.parts:
|
||||
state = states.get(p.import_id) or {}
|
||||
raw = state.get("raw_state")
|
||||
parts.append(replace(p, raw_state=raw if raw in get_args(RawState) else None, purged=state.get("purged"),
|
||||
revoked=state.get("revoked"), recipe_seq=state.get("recipe_seq")))
|
||||
return replace(view, parts=parts, manifest_view=manifest_view)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 表名、字段名、原话
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -1340,17 +1769,56 @@ def _stale(verdict: Any) -> bool:
|
||||
return isinstance(verdict, dict) and bool(verdict.get("post_seal")) and outdated(verdict)
|
||||
|
||||
|
||||
def _sealed_loader(sealed: _Sealed) -> Callable[[str], Any]:
|
||||
def _sealed_loader(sealed: _Sealed, *, chain: bool = False) -> Callable[[str], Any]:
|
||||
"""给裁判取证据的 loader:只认封存范围内的事件交回过的工件(查询、表结构、检索快照和台账里记过的),
|
||||
取回时照旧复验哈希。别的一律当取不到——人为插进工件表的行、封存之后才追加的事件引用的快照,
|
||||
都不能送进裁判的摘录。报告文档里的目录也不例外:目录写着哪件工件,不等于它在封存链上。"""
|
||||
都不能送进裁判的摘录。报告文档里的目录也不例外:目录写着哪件工件,不等于它在封存链上。
|
||||
|
||||
chain(期 4,P4-SPEC 2.8.3):只有带文档标记的文档传真。这时另外认封存范围内的表结构快照经哈希链列出的
|
||||
导入清单、快照清单(以及快照清单各期的导入清单):这些 id 写在内容寻址、又在封存范围内的表结构快照里,属于
|
||||
封存链的延伸,不是数据库列。扩展集合**惰性**计算:第一次被要一件不在基本范围里的工件时才算,同一个 loader
|
||||
只算一次(一次请求一个 loader)。chain 为假时和期 4 之前完全相同,期 4 之前的文档不会因为任何一份清单出错。
|
||||
"""
|
||||
allowed = {*sealed.queries, *sealed.schemas, *sealed.retrievals, *(str(a) for _, a in sealed.ledger)}
|
||||
extended: set[str] | None = None
|
||||
|
||||
def loader(artifact: str) -> Any:
|
||||
return load(artifact) if sealed.trusted and str(artifact) in allowed else None
|
||||
nonlocal extended
|
||||
key = str(artifact)
|
||||
if not sealed.trusted:
|
||||
return None
|
||||
if key in allowed:
|
||||
return load(key)
|
||||
if not chain:
|
||||
return None
|
||||
if extended is None:
|
||||
extended = _chain_set(sealed)
|
||||
return load(key) if key in extended else None
|
||||
return loader
|
||||
|
||||
|
||||
def _chain_set(sealed: _Sealed) -> set[str]:
|
||||
"""封存范围内每份表结构快照经哈希链列出的清单 id 的并集。表结构快照取不回、哈希不符的跳过(不在链上)。"""
|
||||
out: set[str] = set()
|
||||
for schema_id in sealed.schemas:
|
||||
try:
|
||||
schema = load(schema_id)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(schema, dict):
|
||||
out |= _chain_artifacts(schema, load)
|
||||
return out
|
||||
|
||||
|
||||
def _chain_artifacts(schema: dict[str, Any], loader: Callable[[str], Any]) -> set[str]:
|
||||
"""一份表结构快照经哈希链列出的清单 id:provenance.chain_artifacts(永不抛异常,坏清单视为不在链上)。
|
||||
|
||||
按名字现取(不用模块顶部绑定的 prov):测试可以在 sys.modules 里换掉这个模块,核对这里确实委托给它。"""
|
||||
from app.data.provenance import chain_artifacts
|
||||
|
||||
return set(chain_artifacts(schema, loader))
|
||||
|
||||
|
||||
def _node_judge(sealed: _Sealed, node_id: str) -> dict[str, Any]:
|
||||
"""这次运行的图上报告节点写的 judge(裁判模型、句数和时长上限);图读不出来就当没写。"""
|
||||
from app.engine.nodes.report import report_judge
|
||||
@@ -1450,7 +1918,8 @@ async def _judge_now(sealed: _Sealed, report: _Report, units: list[str]) -> dict
|
||||
event = None
|
||||
lost = False
|
||||
if pending:
|
||||
loader = _sealed_loader(sealed)
|
||||
# 带期 4 标记的文档才让裁判取得到哈希链上的清单(摘录的来历行要用);老文档的 loader 和以前完全相同
|
||||
loader = _sealed_loader(sealed, chain=doc.get("provenance") == DOC_PROVENANCE)
|
||||
masked = await judging.source_masks(catalog, loader=loader)
|
||||
request = judging.prepare(doc, catalog, units=pending, loader=loader, masked=masked)
|
||||
skipped = dict(request.skipped)
|
||||
|
||||
@@ -249,7 +249,7 @@ def sqlite_dqs_self_check() -> None:
|
||||
raise RuntimeError(f"SQLite 自检失败:无法确认写错的双引号列名会报错,服务不启动。{problem}")
|
||||
|
||||
|
||||
def open_checked_sqlite(path: str, *, readonly: bool = True) -> sqlite3.Connection:
|
||||
def open_checked_sqlite(path: str, *, readonly: bool = True, immutable: bool = False) -> sqlite3.Connection:
|
||||
"""同步打开一个本地 SQLite 文件,DQS 已关(配方导入的建库、核对用)。
|
||||
|
||||
配方执行器建临时库、核对模块跑 SQL 都不经 SQLAlchemy,直接用 sqlite3。它们同样要关 DQS:
|
||||
@@ -261,11 +261,20 @@ def open_checked_sqlite(path: str, *, readonly: bool = True) -> sqlite3.Connecti
|
||||
readonly=False 以 mode=rwc 打开(执行器建库)。不设日志模式:沿用 SQLite 默认的回滚日志,
|
||||
不开 WAL——WAL 会改写文件头,库文件哈希就对不上了(P2-SPEC 4.2 第 9 步)。
|
||||
路径按 URI 转义:文件名里的空格、中文、「?」「#」都不会被当成 URI 的参数或片段。
|
||||
|
||||
immutable=True 另加 `immutable=1`,和 EngineCache 打开版本快照的方式一致(_sqlite_url):证据下钻在快照文件上
|
||||
单开一个连接编译原 SQL(provenance_db.compile_facts,P4-SPEC 2.9),读的必须是查询走的同一种打开方式。
|
||||
只许和 readonly 一起用:immutable 告诉 SQLite 文件不会变、不加锁,可写的连接这样开会写坏库。缺省 False,
|
||||
现有调用方的行为不变。
|
||||
"""
|
||||
from urllib.parse import quote
|
||||
|
||||
if immutable and not readonly:
|
||||
raise ValueError("immutable 只能用于只读连接")
|
||||
mode = "ro" if readonly else "rwc"
|
||||
return sqlite3.connect(f"file:{quote(os.fspath(path))}?mode={mode}", uri=True, factory=_DqsOffConnection)
|
||||
extra = "&immutable=1" if immutable else ""
|
||||
return sqlite3.connect(f"file:{quote(os.fspath(path))}?mode={mode}{extra}", uri=True,
|
||||
factory=_DqsOffConnection)
|
||||
|
||||
|
||||
def _sqlite_extra() -> dict[str, Any]:
|
||||
@@ -569,6 +578,17 @@ class EngineCache:
|
||||
# 正被借出的连接不受影响:dispose 只关池子里空闲的,借出的还回来时随旧池子回收
|
||||
await old.dispose()
|
||||
|
||||
def pinned(self, source: Any) -> Fingerprint | None:
|
||||
"""这个源(版本快照)的引擎核对过的文件指纹;不是快照引擎、或者引擎不在缓存里时为 None。
|
||||
|
||||
给证据下钻的编译核对用(provenance_db.compile_facts,P4-SPEC 2.9):它在同一个文件上单开一个连接,
|
||||
开连接前、关连接后各拿当时的 stat 和这份指纹比,比得上才说明单开的连接读的是 get() 核对过的那个文件。
|
||||
调用方要在 `await get(source)` 返回之后**同步**调它,中间不能有 await:否则这个键可能已经被 LRU 挤掉,
|
||||
或者被别的请求摘掉重核。只读不改:已经标成不可信(stale)的也照样返回那份指纹,调用方比对时自然对不上。
|
||||
"""
|
||||
pin = self._pins.get(cache_key(source))
|
||||
return pin.fingerprint if pin is not None else None
|
||||
|
||||
async def _verify_once(self, key: str, source: Any, expected: str) -> Fingerprint:
|
||||
"""核对快照哈希;同一个键已经有一次在核对,就等那一次的结果。
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,257 @@
|
||||
"""证据下钻(期 4「推断的来源」)里要碰数据库和数据文件的几步(P4-SPEC 2.3 R12–R16、2.7、2.9)。
|
||||
|
||||
识别器(engine/direct_select.py)和清单、格子(data/provenance.py)都是纯函数,不连库;这里收拢剩下的:
|
||||
取数据文件记录、绑定快照、按主键回查、编译核对、强制重算哈希、行级核对、导入记录上的当前状态。
|
||||
|
||||
**为什么回查、行级核对都经 EngineCache**:查询本身就是经它打开快照的(immutable 只读、本进程首次整份核对哈希、
|
||||
之后每次取引擎和借连接都比文件指纹)。走同一条路,下钻核对过的就是查询读过的那个文件,保证范围也和查询相同
|
||||
(P4-SPEC 2.6 照实写了这个范围的边界)。编译核对另开一个连接,开前关后各比一次 EngineCache 记下的指纹。
|
||||
|
||||
**异常**:SnapshotManifestMismatch、SnapshotTampered、SnapshotMissing 一律原样抛出,由接口一侧按 2.3 的
|
||||
except 顺序映射成 chain_mismatch / db_tampered / snapshot_gone(api/evidence._snapshot_failure)。这里不吞、
|
||||
不改判:同一类异常在接口的每一步都要得到同一个结论。
|
||||
|
||||
同步的文件操作(编译核对、整份重算哈希)一律放 asyncio.to_thread,不卡事件循环(H9)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.data import engine as engine_mod
|
||||
from app.data.engine import SnapshotTampered, engines, open_checked_sqlite
|
||||
from app.data.names import name_key
|
||||
from app.data.provenance_types import CompileFacts, StateNote, SumEqRule
|
||||
from app.data.recipe_checks import sum_eq_conditions
|
||||
from app.data.table_versions import SnapshotMissing, SourceView, bind_snapshot
|
||||
from app.db.models import SourceSnapshot, TableImport, TableRecipe
|
||||
|
||||
#: 数据文件记录在、却已回收或文件不在(绑定之前先认出来,免得取引擎时被当成「文件被改」标红)
|
||||
_GONE = "数据版本 {sid} 的数据文件已回收或不存在,无法按主键回查"
|
||||
#: EngineCache 里没有这个快照的指纹(不是快照引擎):不该出现,按文件不在处理(P4-SPEC 2.9 末尾)
|
||||
_NOT_PINNED = "数据版本 {sid} 没有核对过的数据文件记录,无法做编译核对"
|
||||
#: 编译核对前后,文件和 EngineCache 核对过的那份对不上:和借连接时指纹不符同一个说法
|
||||
_REPLACED = engine_mod._REPLACED
|
||||
|
||||
|
||||
def _q(name: str) -> str:
|
||||
"""SQL 标识符:一律双引号、内部双引号写两个(和 recipe_checks._q、契约 Recheck 的写法一致)。"""
|
||||
return '"' + str(name).replace('"', '""') + '"'
|
||||
|
||||
|
||||
def _fingerprint(path: str) -> engine_mod.Fingerprint | None:
|
||||
"""文件指纹,口径和 EngineCache 记下的那份完全相同(同一个函数)。单独包一层,测试可以只换这里。"""
|
||||
return engine_mod._fingerprint(path)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 数据文件记录与绑定(R12、R14)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def snapshot_record(session: AsyncSession, snapshot_id: str) -> SourceSnapshot | None:
|
||||
"""查询快照的 data_version 对应的数据文件记录(source_snapshots 一行);没有为 None(R12)。"""
|
||||
if not snapshot_id:
|
||||
return None
|
||||
return await session.get(SourceSnapshot, str(snapshot_id))
|
||||
|
||||
|
||||
def view_of(snapshot: SourceSnapshot, *, source_name: str) -> SourceView:
|
||||
"""数据文件记录 → 绑定了这个快照的 SourceView,供回查、编译核对、行级核对打开数据文件(R14)。
|
||||
|
||||
用 table_versions.bind_snapshot,和查询解析源的那条路相同(登记哈希为空拒绝;有快照清单的首次绑定时交叉
|
||||
核对)。传进去的不是数据源记录,而是一个替身:
|
||||
- id = snapshot.source_id:缓存键(cache_key)和查询时相同,首次整份哈希只算一次;
|
||||
- name = source_name(查询快照记的数据源名,只用在报错文字里);description、enabled、options 等给缺省值。
|
||||
bind_snapshot 直接读 source.description、bool(source.enabled),少了会抛 AttributeError。
|
||||
所以数据源被删了、改名了也能回查:回查只需要快照文件和它的登记哈希。options 为空,查询时限、遮罩都不进
|
||||
这个视图——回查的 SQL 是这里拼的主键查找,遮罩由接口一侧按 R10 先判过。
|
||||
|
||||
记录已回收(retired_at)或文件不在时抛 SnapshotMissing(→ snapshot_gone,不标红):不先认出来的话,取引擎时
|
||||
读不到文件会被 EngineCache 报成 SnapshotTampered,误标成「数据文件被改」。SnapshotManifestMismatch、
|
||||
SnapshotMissing(登记哈希为空)原样抛出。
|
||||
"""
|
||||
path = str(snapshot.db_path or "")
|
||||
if snapshot.retired_at is not None or not path or not Path(path).is_file():
|
||||
raise SnapshotMissing(_GONE.format(sid=str(snapshot.id)[:12]))
|
||||
stand_in = SimpleNamespace(
|
||||
id=snapshot.source_id, name=source_name, description="", enabled=True, options={},
|
||||
current_snapshot_id=None, last_check=None,
|
||||
)
|
||||
return bind_snapshot(stand_in, snapshot)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# R15:编译核对
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def compile_facts(view: SourceView, sql: str) -> CompileFacts:
|
||||
"""在查询读过的同一个文件上编译原 SQL(EXPLAIN,不执行),给出与解析无关的三项事实(R15):
|
||||
授权回调报出读了哪些表、字节码里 ResultRow 和 Yield 各有几个。
|
||||
|
||||
步骤:经 EngineCache 取引擎(首次时整份核对哈希)→ **同步**取 engines.pinned(view)(中间没有 await,不会被 LRU
|
||||
挤掉)→ 在线程里:比指纹 → 单开 immutable 只读连接(和 EngineCache 打开快照的方式相同)→ 装授权回调、EXPLAIN
|
||||
一次 → 关连接 → 再比指纹。开前关后的指纹都要等于 EngineCache 核对过的那份,单开的连接读的才是核对过的文件;
|
||||
不等抛 SnapshotTampered。pinned 为 None(不是快照引擎)抛 SnapshotMissing。
|
||||
|
||||
授权回调只观察、不拦(一律放行):连接本身只读、immutable,EXPLAIN 不执行语句。结论怎么用由接口判(R15 的
|
||||
①②③),这里只报事实。
|
||||
"""
|
||||
await engines.get(view)
|
||||
pinned = engines.pinned(view)
|
||||
if pinned is None:
|
||||
raise SnapshotMissing(_NOT_PINNED.format(sid=str(view.snapshot_id or "")[:12]))
|
||||
return await asyncio.to_thread(_compile, str(view.database or ""), pinned, sql)
|
||||
|
||||
|
||||
def _compile(path: str, pinned: engine_mod.Fingerprint, sql: str) -> CompileFacts:
|
||||
if _fingerprint(path) != pinned:
|
||||
raise SnapshotTampered(_REPLACED)
|
||||
reads: set[str] = set()
|
||||
|
||||
def observe(action: int, arg1: str | None, _arg2: str | None, _db: str | None, _trigger: str | None) -> int:
|
||||
# SQLITE_READ 的 arg1 是表名(建表时的写法)。子查询、自连接读到的表都会报出来,改名、CTE 报的和合法写法
|
||||
# 一样(P4-SPEC 2.4 末尾),所以这只是兜底,主判据仍是识别器
|
||||
if action == sqlite3.SQLITE_READ and arg1:
|
||||
reads.add(arg1)
|
||||
return sqlite3.SQLITE_OK
|
||||
|
||||
conn = open_checked_sqlite(path, readonly=True, immutable=True)
|
||||
try:
|
||||
conn.set_authorizer(observe)
|
||||
opcodes = [str(row[1]) for row in conn.execute("EXPLAIN " + sql)]
|
||||
finally:
|
||||
conn.close()
|
||||
if _fingerprint(path) != pinned:
|
||||
raise SnapshotTampered(_REPLACED)
|
||||
return CompileFacts(tables=reads, result_rows=opcodes.count("ResultRow"), yields=opcodes.count("Yield"))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# R16:按主键回查、强制重算哈希
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def recheck(view: SourceView, *, table: str, column: str,
|
||||
pk: dict[str, Any]) -> tuple[list[tuple[int, Any]], str, list[Any]]:
|
||||
"""按主键参数化回查被引用的那一格(R16):返回 (行, sql, params),行是 [(rowid, 值), …]。
|
||||
|
||||
SQL 的写法以契约 Recheck 为准:`SELECT rowid, "列" FROM "表" WHERE "主键1" = ? AND "主键2" = ?`,标识符一律双引号,
|
||||
`?` 位置占位符,WHERE 和 params 都按 pk 的键序(调用方按冻结表结构 primary_key 的顺序构造 pk)。面板「技术
|
||||
细节」原样显示这里返回的 sql、params,所以不加 LIMIT:多出的行本身就是 recheck_multiple 的证据。
|
||||
|
||||
值经 engine._jsonable 规范化,和查询结果进快照时同一个口径(run_query 逐格调它),调用方直接按类型和值比较。
|
||||
经 EngineCache 打开(同一个缓存键、同一份哈希核对),三类异常原样抛出。
|
||||
"""
|
||||
if not pk:
|
||||
raise ValueError("回查需要主键")
|
||||
names = list(pk)
|
||||
sql = (f"SELECT rowid, {_q(column)} FROM {_q(table)} WHERE "
|
||||
+ " AND ".join(f"{_q(name)} = ?" for name in names))
|
||||
params = [pk[name] for name in names]
|
||||
engine = await engines.get(view)
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.exec_driver_sql(sql, tuple(params))
|
||||
rows = [(int(r[0]), engine_mod._jsonable(r[1])) for r in result.fetchall()]
|
||||
return rows, sql, params
|
||||
|
||||
|
||||
async def rehash(view: SourceView) -> bool:
|
||||
"""在线程里整份重算数据文件的 sha256,等于 view.expected_sha256 返回 True(R16 的强制重算)。
|
||||
|
||||
不经 EngineCache 的指纹捷径:回查的值和快照对不上时才调。EngineCache 只在首次和指纹变化时重算,原地改写同样
|
||||
字节数、再还原修改时间的改动在指纹不变期间认不出(P4-SPEC 2.6);这里强制读一遍,才能把「文件确实被改了」
|
||||
和「这条 SQL 的含义与判据认定的不同」分开。文件读不出来、没有登记哈希时返回 False:核对不了就不能当没变。
|
||||
"""
|
||||
expected = str(view.expected_sha256 or "").strip().lower()
|
||||
if not expected:
|
||||
return False
|
||||
try:
|
||||
actual = await asyncio.to_thread(engine_mod._file_sha256, str(view.database or ""))
|
||||
except OSError:
|
||||
return False
|
||||
return actual.lower() == expected
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 2.7:关系核对的行级状态
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def row_status(view: SourceView, rule: SumEqRule, snapshot_rowid: int) -> str | None:
|
||||
"""这一行的关系核对「total = parts 之和」成不成立:passed / mismatch / unverifiable;判不了为 None(2.7)。
|
||||
|
||||
条件取自 recipe_checks.sum_eq_conditions,和导入时的核对(_check_sum_eq)是同一段代码生成的:面板说「这一天
|
||||
成立」,和导入时的结论是同一个判断。先判含空值(unverifiable),再判不等(mismatch),其余 passed,与导入时
|
||||
「含空值的行不参与比较、计入无法核对」一致。
|
||||
|
||||
snapshot_rowid 是**快照库**的 rowid(cell_source.rowid,并集时是并集 rowid),不是该期构建库的 part_rowid:
|
||||
SQL 跑在快照库上。各期的行是逐字节拷进并集库的,同一个条件照样成立(P3-SPEC 2.7)。
|
||||
|
||||
规则里的列在快照库这张表里不存在(不该出现:关系取自格子所在那一期的配方,目标表结构只会多出退役列),或者
|
||||
没有这一行时返回 None,不报错。三类数据文件异常原样抛出。
|
||||
"""
|
||||
_nonnull, anynull, mismatch = sum_eq_conditions(rule.total, list(rule.parts), dict(rule.types))
|
||||
sql = (f"SELECT CASE WHEN {anynull} THEN 'unverifiable' WHEN {mismatch} THEN 'mismatch' ELSE 'passed' END "
|
||||
f"FROM {_q(rule.table)} WHERE rowid = ?")
|
||||
engine = await engines.get(view)
|
||||
async with engine.connect() as conn:
|
||||
present = {name_key(str(r[0])) for r in
|
||||
(await conn.exec_driver_sql("SELECT name FROM pragma_table_xinfo(?)", (rule.table,))).fetchall()}
|
||||
if not present or any(name_key(c) not in present for c in [rule.total, *rule.parts]):
|
||||
return None
|
||||
row = (await conn.exec_driver_sql(sql, (int(snapshot_rowid),))).first()
|
||||
return str(row[0]) if row is not None else None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 导入记录上的当前状态(数据版本一节标「当前状态」的那几项)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def import_states(session: AsyncSession, import_ids: list[str]) -> dict[str, dict[str, Any]]:
|
||||
"""各期导入记录上的当前状态:{import_id: {raw_state, purged, revoked, recipe_seq}}。
|
||||
|
||||
- raw_state:kept / purged / absent(table_imports.raw_state);
|
||||
- purged、revoked:StateNote{at, signed_by, reason},没有记录为 None。清除原件、作废接受都只记在导入记录上,
|
||||
不在封存范围内,界面标「当前状态」(P4-SPEC 0.2 补充 2);
|
||||
- recipe_seq:这一期配方记录的 seq(table_recipes.seq,经 table_imports.recipe_id 找),取不到为 None。
|
||||
|
||||
找不到的 import_id 不出现在结果里(调用方按 None 处理)。只取需要的几列:导入记录上还有核对摘要等大字段。
|
||||
"""
|
||||
ids = [str(i) for i in dict.fromkeys(import_ids) if i]
|
||||
if not ids:
|
||||
return {}
|
||||
rows = (await session.execute(
|
||||
select(TableImport.id, TableImport.raw_state, TableImport.purged, TableImport.revoked, TableImport.recipe_id)
|
||||
.where(TableImport.id.in_(ids)))).all()
|
||||
recipe_ids = sorted({str(r.recipe_id) for r in rows if r.recipe_id})
|
||||
seqs: dict[str, int] = {}
|
||||
if recipe_ids:
|
||||
seqs = {str(rid): int(seq) for rid, seq in (await session.execute(
|
||||
select(TableRecipe.id, TableRecipe.seq).where(TableRecipe.id.in_(recipe_ids)))).all()}
|
||||
return {str(r.id): {
|
||||
"raw_state": r.raw_state or None,
|
||||
"purged": _note(r.purged),
|
||||
"revoked": _note(r.revoked),
|
||||
"recipe_seq": seqs.get(str(r.recipe_id)) if r.recipe_id else None,
|
||||
} for r in rows}
|
||||
|
||||
|
||||
def _note(record: Any) -> StateNote | None:
|
||||
"""table_imports.purged / revoked 的记录 {at, reason, signed_by, …} → StateNote;空的、不是 dict 的为 None。"""
|
||||
if not isinstance(record, dict) or not record:
|
||||
return None
|
||||
|
||||
def text(key: str) -> str | None:
|
||||
value = record.get(key)
|
||||
return str(value) if value not in (None, "") else None
|
||||
|
||||
return StateNote(at=text("at"), signed_by=text("signed_by"), reason=text("reason"))
|
||||
@@ -0,0 +1,730 @@
|
||||
"""证据下钻(期 4「推断的来源」)的契约:常量、取值、服务端原文、响应形状和各工作包之间传递的中间结构。
|
||||
|
||||
由编排者在第 0 步(WP-0)提交,各工作包只通过这里的类型和 P4-SPEC.md 第 7 节写明的函数签名交互,**不改签名**;
|
||||
需要改的写进交付说明,由编排者改。这里只有声明,业务逻辑只有三个小函数(refuse、alert、contract_problems):
|
||||
接口、测试、前端夹具三处要用同一份原文和同一套不变式。
|
||||
|
||||
**为什么只用标准库**:识别器(engine/direct_select.py)只许导入守卫、names 和这个模块(P4-SPEC 7.2),这里一旦
|
||||
导入 recipe_types、pydantic,判据就被拖进整套配方模块。和 recipe_types 重名的取值(核对状态)在契约测试里钉住一致。
|
||||
|
||||
**响应形状**(P4-SPEC 2.8.1):`dataclasses.asdict(ProvenanceOut(...))` 就是接口返回的 JSON,键名、顺序和规格逐字
|
||||
一致。响应用的 dataclass 一律 kw_only:字段多、并行的几个包各自构造,按位置传参错一位不会报错,按名字传错了立刻报。
|
||||
中间结构(Chain、DirectSelect 等)照规格的写法可以按位置构造。
|
||||
|
||||
**唯一的改名**:CellSource 的「from」是 Python 关键字,不能当属性名。Python 里写 `from_`(构造、读写都用它),
|
||||
`asdict` / `fields` / `dataclasses.replace` / `typing.get_type_hints`、pydantic 和 JSON 里一律是「from」(见 _json_keys)。
|
||||
构造时也认 `**{"from": …}`,从 JSON 读回来可以直接 `CellSource(**d)`。
|
||||
|
||||
**路由怎么返回**(WP-C):照现有证据路由的写法,`return dataclasses.asdict(out)`,返回注解写 `dict[str, Any]`。
|
||||
把 ProvenanceOut 写成返回注解或 `response_model` 也得到同样的 JSON(pydantic 按 `__annotations__` 收集 dataclass
|
||||
字段,_json_keys 把注解里的名字也改成了「from」,契约测试钉住 TypeAdapter 和 FastAPI 带类型路由的输出等于 asdict),
|
||||
但不推荐:那样每次响应都要按类型再校验、再序列化一遍,与其余证据路由不一致。
|
||||
|
||||
**文档标记**(P4-SPEC 1.4):compose_doc 组装的新文档顶层写 `"provenance": DOC_PROVENANCE`。识别器的文法、
|
||||
`tables_in` 的读法都绑在这个版本号上;以后放宽文法要升版本号,已有 `provenance: 1` 的文档仍按 1 的文法判。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Literal, get_args
|
||||
|
||||
_log = logging.getLogger(__name__)
|
||||
|
||||
# ==========================================================================
|
||||
# 版本与常量
|
||||
# ==========================================================================
|
||||
|
||||
#: 文档标记的版本号:带这个标记的文档才走期 4 的行为(查询步骤的提示字段、推断来源接口、裁判摘录的新行)
|
||||
DOC_PROVENANCE = 1
|
||||
#: 推断来源接口响应的 schema
|
||||
SCHEMA = "agentlab.provenance/1"
|
||||
|
||||
#: 裁判摘录(P4-SPEC 3.2):来源的分期行最多几期(取最近的几期,其余写「另有 n 期」)
|
||||
PROV_PARTS = 3
|
||||
#: 已接受的理由最多几条
|
||||
PROV_ACCEPT = 4
|
||||
#: 区域外文字最多几格(每条查询)
|
||||
PROV_OUTSIDE = 4
|
||||
#: 区域外文字每格最多几个字,超出写「…(已截断)」
|
||||
PROV_OUTSIDE_CHARS = 200
|
||||
#: 列的说明最多几行
|
||||
PROV_COLUMN_NOTES = 6
|
||||
#: 整块来历(来源到区域外文字,含口径)最多几个字,超出按 3.2 的五步顺序删
|
||||
PROV_CHARS = 1200
|
||||
|
||||
# ==========================================================================
|
||||
# 取值
|
||||
# ==========================================================================
|
||||
|
||||
#: inferred:给出格子;table_only:只给表级来历(version 非空),附原因;none:什么都不画,或只画一句原因 / 红色提示
|
||||
Status = Literal["inferred", "table_only", "none"]
|
||||
|
||||
#: 不下钻的原因(P4-SPEC 2.3 的规则表、2.4 的识别器分组)。同一个 code 可能出现在不同 status 下:
|
||||
#: chain_mismatch 在 R6(清单链本身对不上)是 none,在 R13、R17 和异常映射里是 table_only
|
||||
ReasonCode = Literal[
|
||||
# R0:文档没有标记(期 4 之前组装的)
|
||||
"legacy_doc",
|
||||
# R1:不是直接引用的查询单元格(指标、实体、代码节点和 Agent 字段的取数)
|
||||
"not_cell",
|
||||
# R2 / R4:查询快照或表结构快照不在本次运行的事件里、取回复验不通过
|
||||
"not_sealed",
|
||||
# R3:查询快照没有 data_version(手工源)
|
||||
"not_upload",
|
||||
# R4:表结构快照不是按配方导入的(简单导入、v0 迁移出的版本)
|
||||
"simple_upload",
|
||||
# R5(红):清单取不回来或哈希不符
|
||||
"manifest_unreadable",
|
||||
# R6 / R13 / R17 / 异常映射第 1 条(红)
|
||||
"chain_mismatch",
|
||||
# R7 / R8 / R9 / R15:SQL 判据(2.4 的分组,细分写在 detail,取值见 DETAILS)
|
||||
"expression", "alias", "multi_table", "unparsed",
|
||||
# R9
|
||||
"no_pk", "pk_missing",
|
||||
# R10
|
||||
"masked",
|
||||
# R11
|
||||
"null_value", "null_pk",
|
||||
# R12 / 异常映射第 3 条
|
||||
"snapshot_gone",
|
||||
# 异常映射第 2 条、R16 强制重算后哈希不等(红)
|
||||
"db_tampered",
|
||||
# R16
|
||||
"recheck_missing", "recheck_multiple", "recheck_mismatch",
|
||||
# R17
|
||||
"no_lineage",
|
||||
]
|
||||
|
||||
#: 标红的提示。出现时 reason.code 与它相同,界面只画提示、不再画 reason.text(P4-SPEC 4.1、4.2 末尾)
|
||||
AlertCode = Literal["db_tampered", "chain_mismatch", "manifest_unreadable"]
|
||||
|
||||
#: SQL 判据的拒绝分组(DirectSelect / SelectRefusal,P4-SPEC 2.4)
|
||||
RefusalCode = Literal["expression", "alias", "multi_table", "unparsed"]
|
||||
|
||||
#: 每个分组允许的细分(Reason.detail / SelectRefusal.detail)。duplicate_pk(R9)、authorizer、compound(R15)
|
||||
#: 由接口一侧补上,同样归 multi_table。界面只显示分组的原文,detail 给测试和日志用。
|
||||
#:
|
||||
#: 识别器(WP-A)的每个拒绝分支都必须落在这里,测试对每个拒绝用例断言 `detail in DETAILS[code]`。原型
|
||||
#: p4-probe/probe_recognizer.py 返回的 expression_or_alias、subquery_or_compound、table_shape 之类是原型自己的名字,
|
||||
#: 移植时要逐个映射,不能照抄。2.4 没逐条写的几种按文法归:
|
||||
#: INDEXED BY(表名后面的词被当成省略 AS 的表别名,再往后不是 WHERE / ORDER / LIMIT)→ unparsed · tail_shape;
|
||||
#: 表函数(表名紧跟「(」)→ unparsed · table_shape;FROM 后面直接是「(」(子查询)→ multi_table · subquery;
|
||||
#: 带库名的表(main."日客流")→ unparsed · qualified_table;聚合、函数调用、运算 → expression · expression。
|
||||
#: 漏了映射时 refuse 不抛异常(照样给分组的原文,细分原样留下、记一条警告),contract_problems 会报出来
|
||||
DETAILS: dict[str, tuple[str, ...]] = {
|
||||
"expression": ("expression", "string_literal", "literal_keyword", "window", "collate", "distinct"),
|
||||
"alias": ("alias",),
|
||||
"multi_table": ("join", "comma_join", "subquery", "cte", "compound", "authorizer", "duplicate_pk"),
|
||||
"unparsed": ("not_select", "multi_statement", "table_shape", "qualified_table", "qualifier", "star_mismatch",
|
||||
"unknown_table", "unknown_column", "ambiguous_column", "tail_shape"),
|
||||
}
|
||||
|
||||
#: 单期是配方的 mode,并集是快照清单的 mode
|
||||
Mode = Literal["replace", "accumulate"]
|
||||
#: 表的种类:冻结表结构里 kind 缺省(None)就是 data,reported_total 是原表写明的合计
|
||||
TableKind = Literal["data", "reported_total"]
|
||||
#: 回执里列的角色(recipe_types.ColumnOut.role)
|
||||
ColumnRole = Literal["axis", "dim", "derive", "const", "measure", "value", "text"]
|
||||
#: 附带的格(FromCell.role):日期表头格、行标签、分段标题、列表头、合计标签(含回执里的 derived_label)
|
||||
FromRole = Literal["axis_header", "row_label", "section_title", "col_header", "total_label"]
|
||||
#: 原件状态(table_imports.raw_state,当前状态)
|
||||
RawState = Literal["kept", "purged", "absent"]
|
||||
#: 接受的种类:overrides(数据质量类不成立)→ override;waivers(无法核对)→ waiver
|
||||
AcceptanceKind = Literal["override", "waiver"]
|
||||
#: 本期结论:照搬清单里 CheckResult.status(与 recipe_types.CheckStatus 相同,契约测试钉住)
|
||||
PartStatus = Literal["passed", "mismatch", "unverifiable", "info"]
|
||||
#: 这一行的关系核对结果(只给关系核对)
|
||||
RowStatus = Literal["passed", "mismatch", "unverifiable"]
|
||||
#: 这一格的合计核对结果(只给 K、G、T)
|
||||
CellStatus = Literal["ok", "unverifiable", "unknown", "not_formula"]
|
||||
#: 年份取自哪里:统计期从格子解析(period)、人工录入(human)
|
||||
YearFrom = Literal["period", "human"]
|
||||
|
||||
#: loader 的语义同 artifact_store.load:取不到返回 None,哈希不符抛 ValueError
|
||||
Loader = Callable[[str], Any]
|
||||
|
||||
# ==========================================================================
|
||||
# 服务端原文(P4-SPEC 4.2):界面原样显示 reason.text、alert.text,不另写
|
||||
# ==========================================================================
|
||||
|
||||
_RECHECK_TEXT = "按主键回查数据文件,结果与查询结果不一致,不推断来源"
|
||||
|
||||
#: code → 原文。pk_missing 里的「{列}」由 refuse 用 fmt 填(没带齐的主键列,多列用「、」连)
|
||||
REASON_TEXT: dict[str, str] = {
|
||||
"legacy_doc": "这份报告生成于系统升级前,不推断原表格子",
|
||||
# not_cell、not_upload 界面不画(P4-SPEC 4.1 渲染矩阵),原文只给日志和测试,规格的原文表里没有,由 WP-0 补
|
||||
"not_cell": "这一处不是直接引用的查询单元格,不推断原表格子",
|
||||
"not_sealed": "查询快照或表结构快照不在本次运行的记录里,不推断来源",
|
||||
"not_upload": "这次查询的数据源不是上传的表格,不推断原表格子",
|
||||
"simple_upload": "这份表格按表头行直接导入,没有记录原表格子的溯源",
|
||||
"manifest_unreadable": "导入清单无法读取或与哈希不一致,无法给出来历",
|
||||
# 规格原文是「…可能被修改过:只显示导入清单记录的来历」。chain_mismatch 也用于 R6(清单链本身对不上,
|
||||
# version 为 null,什么来历都不显示),那句后半句在 R6 时不属实,所以去掉;table_only 时数据版本一节照样画出来,
|
||||
# 不需要再说(交付说明里写明)
|
||||
"chain_mismatch": "数据文件的登记与导入清单不一致,可能被修改过",
|
||||
"db_tampered": "数据文件与登记的版本不一致,可能被修改过:只显示导入清单记录的来历",
|
||||
"snapshot_gone": "这一版的数据文件已回收或不存在,无法按主键回查,只显示导入清单记录的来历",
|
||||
"expression": "这一格是计算、聚合或改写的结果,只给出表级来历",
|
||||
"alias": "查询给列改了名,无法确认这一格对应哪一列,只给出表级来历",
|
||||
"multi_table": "查询涉及多张表、同一张表多次,或含子查询、公用表表达式、集合运算,只给出表级来历",
|
||||
"unparsed": "查询的写法超出可追溯的范围,只给出表级来历",
|
||||
"no_pk": "这张表没有主键,无法按主键回查,只给出表级来历",
|
||||
"pk_missing": "查询结果没有带齐主键列({列}),无法按主键回查,只给出表级来历",
|
||||
"masked": "这一格涉及设置了遮罩的列,不推断来源",
|
||||
"null_value": "这一格是空值,不推断来源",
|
||||
"null_pk": "这一行的主键是空值,无法按主键回查",
|
||||
"recheck_missing": _RECHECK_TEXT,
|
||||
"recheck_multiple": _RECHECK_TEXT,
|
||||
"recheck_mismatch": _RECHECK_TEXT,
|
||||
"no_lineage": "导入清单里没有这一格的溯源记录,只给出表级来历",
|
||||
}
|
||||
|
||||
#: 标红提示的原文:与同名 reason 的原文相同(两处同时出现时界面只画提示)
|
||||
ALERT_TEXT: dict[str, str] = {code: REASON_TEXT[code] for code in get_args(AlertCode)}
|
||||
|
||||
_PLACEHOLDER = re.compile(r"\{([^{}]+)\}")
|
||||
#: 回查 SQL 里双引号括起的标识符(内部的双引号写两个):数 `?` 占位符之前先去掉,列名里的问号不算
|
||||
_QUOTED = re.compile(r'"(?:[^"]|"")*"')
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 响应(GET /api/runs/{run_id}/evidence/segments/{segment_id}/provenance,P4-SPEC 2.8.1)
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def _stored_as(py: str, js: str) -> property:
|
||||
"""Python 名字 `py` 的属性:读写实例字典里的 `js` 键(_json_keys 用)。没有这个键时抛 AttributeError,hasattr 照常。"""
|
||||
def get(self: Any) -> Any:
|
||||
try:
|
||||
return self.__dict__[js]
|
||||
except KeyError:
|
||||
raise AttributeError(py) from None
|
||||
|
||||
def set_(self: Any, value: Any) -> None:
|
||||
self.__dict__[js] = value
|
||||
|
||||
return property(get, set_)
|
||||
|
||||
|
||||
def _json_keys(**aliases: str) -> Callable[[type], type]:
|
||||
"""dataclass 的字段在 Python 里叫 `py`,在 asdict / fields / replace / get_type_hints / pydantic / JSON 里叫 `js`
|
||||
(只给 CellSource.from 用)。
|
||||
|
||||
做法:dataclass 照常按 `py` 生成 __init__、__repr__、__eq__;生成之后
|
||||
- 字段表 `__dataclass_fields__` 和注解 `__annotations__` 里的名字都改成 `js`(顺序不变)。只改字段表的话,
|
||||
pydantic 按注解收集字段,对不上的直接跳过:FastAPI 带类型的路由、TypeAdapter 的输出会悄悄少掉「from」;
|
||||
- 值存在实例字典的 `js` 键下,`py` 是读写它的属性。pydantic 从 JSON 校验出实例时直接按字段名填实例字典,
|
||||
不经过 __init__,存在 `py` 下的话这样得到的实例读 `from_` 会抛 AttributeError;
|
||||
- __init__ 也认 `js` 这个关键字(dataclasses.replace、`Cls(**json)` 传的是它)。两个都传时以 `py` 为准:
|
||||
`replace(obj, from_=新值)` 会同时带上旧的 `from`。
|
||||
"""
|
||||
def wrap(cls: type) -> type:
|
||||
fields_: dict[str, Any] = cls.__dataclass_fields__ # type: ignore[attr-defined]
|
||||
renamed: dict[str, Any] = {}
|
||||
for name, f in fields_.items():
|
||||
if name in aliases:
|
||||
f.name = aliases[name]
|
||||
renamed[f.name] = f
|
||||
cls.__dataclass_fields__ = renamed # type: ignore[attr-defined]
|
||||
cls.__annotations__ = {aliases.get(k, k): v for k, v in cls.__dict__["__annotations__"].items()}
|
||||
for py, js in aliases.items():
|
||||
setattr(cls, py, _stored_as(py, js))
|
||||
generated = cls.__init__
|
||||
|
||||
def __init__(self: Any, *args: Any, **kwargs: Any) -> None:
|
||||
for py, js in aliases.items():
|
||||
if js in kwargs:
|
||||
value = kwargs.pop(js)
|
||||
kwargs.setdefault(py, value)
|
||||
generated(self, *args, **kwargs)
|
||||
|
||||
__init__.__doc__ = generated.__doc__
|
||||
__init__.__qualname__ = f"{cls.__qualname__}.__init__"
|
||||
cls.__init__ = __init__ # type: ignore[method-assign]
|
||||
return cls
|
||||
return wrap
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class ReportRef:
|
||||
#: 报告节点 id
|
||||
node_id: str
|
||||
#: 报告文档的工件 id
|
||||
doc_artifact: str
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class CellRef:
|
||||
"""片段本身是单元格引用(cite.kind == "cell",有 alias 和 locator)就给,与 status 无关(legacy_doc 时也给)。"""
|
||||
|
||||
#: 目录里的查询编号(Q3)
|
||||
alias: str
|
||||
#: 快照里的行号,从 0 数(locator.row)
|
||||
row: int
|
||||
#: 被引用的结果列名(locator.column)
|
||||
column: str
|
||||
#: 目录条目的 artifact(查询快照 id);条目没记时 None
|
||||
artifact: str | None = None
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class Reason:
|
||||
"""status 不是 inferred 时必有。用 refuse() 构造,原文取自 REASON_TEXT。"""
|
||||
|
||||
code: ReasonCode
|
||||
#: 细分(DETAILS 里的值、ChainProblem / LocateProblem 的 detail);没有细分为空串
|
||||
detail: str = ""
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class Alert:
|
||||
"""标红的提示。用 alert() 构造,原文取自 ALERT_TEXT。"""
|
||||
|
||||
code: AlertCode
|
||||
text: str
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class TableRef:
|
||||
"""SQL 用到的表:direct_select.tables_in 与冻结表结构的交集,按第一次出现的顺序。"""
|
||||
|
||||
name: str
|
||||
kind: TableKind = "data"
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class PeriodView:
|
||||
"""一期的统计期(导入清单的 period)。"""
|
||||
|
||||
start: str
|
||||
end: str
|
||||
source: Literal["cells", "human"]
|
||||
#: 解析出统计期的格子,带工作表名(「客流汇总!B2」);人工录入时为空
|
||||
cells: list[str] = field(default_factory=list)
|
||||
#: 人工录入时的署名(未认证)
|
||||
signed_by: str | None = None
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class AcceptanceView:
|
||||
"""一条接受理由(导入清单 acceptances 的一项,内容寻址,是当时的事实)。
|
||||
|
||||
不叫 Acceptance:recipe_types.Acceptance 已有同名类,字段不同。
|
||||
"""
|
||||
|
||||
check_id: str
|
||||
#: 同一份清单 checks 里 id 相同的那条的 title;找不到时 None
|
||||
title: str | None
|
||||
kind: AcceptanceKind
|
||||
reason: str
|
||||
signed_by: str | None = None
|
||||
#: 清单里这条接受的 at;老清单没有时 None
|
||||
at: str | None = None
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class StateNote:
|
||||
"""导入记录上的当前状态(清除原件、作废接受):table_imports.purged / revoked 里的 at、signed_by、reason。"""
|
||||
|
||||
at: str | None = None
|
||||
signed_by: str | None = None
|
||||
reason: str | None = None
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class PartView:
|
||||
"""数据版本里的一期(按统计期升序)。前半来自导入清单(内容寻址),raw_state 起的三项来自导入记录(当前状态)。
|
||||
|
||||
version_view(WP-B)只填清单里的那部分:recipe_seq、raw_state、purged、revoked 留 None,has_row 全 False,
|
||||
由接口一侧(WP-C)从数据库补、按格子所在的一期标 has_row。
|
||||
"""
|
||||
|
||||
import_id: str
|
||||
#: 导入记录的 seq(同 ChainPart.seq)
|
||||
seq: int
|
||||
#: 导入清单的工件 id(「查看导入清单」按它经工件接口取)
|
||||
manifest: str
|
||||
#: 清单没有统计期时 None(界面写「统计期未记录」)
|
||||
period: PeriodView | None
|
||||
file_name: str | None
|
||||
raw_sha256: str | None
|
||||
#: S9:回执 regions 里块内各角色区域的外接矩形,按工作表(「客流汇总!B4:AG30」),多张表用「、」连;取不到为 None
|
||||
region: str | None = None
|
||||
#: S9:回执 rows_excluded 的行数之和;期 3 之前的回执没有这个键时为 None
|
||||
excluded_rows: int | None = None
|
||||
recipe_sha256: str | None = None
|
||||
#: 配方是第几版:取自配方记录(当前状态),取不到为 None
|
||||
recipe_seq: int | None = None
|
||||
#: 清单的 created_at
|
||||
committed_at: str | None = None
|
||||
#: 清单的 signed_by.name(署名,未认证)
|
||||
signed_by: str | None = None
|
||||
#: 这一期的全部接受理由(不只是相关的):先 overrides,再 waivers,各按清单里的顺序
|
||||
acceptances: list[AcceptanceView] = field(default_factory=list)
|
||||
raw_state: RawState | None = None
|
||||
purged: StateNote | None = None
|
||||
revoked: StateNote | None = None
|
||||
#: 这一格所在的一期:inferred 时恰好一项为 True,其余 False
|
||||
has_row: bool = False
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class VersionView:
|
||||
"""数据版本(表级来历):status 为 none 时整节为 null。"""
|
||||
|
||||
#: 查询快照记的数据源名
|
||||
source: str
|
||||
#: 查询快照的 data_version
|
||||
snapshot_id: str
|
||||
mode: Mode
|
||||
#: 累积并集(表结构快照有 snapshot_manifest)
|
||||
union: bool
|
||||
tables: list[TableRef] = field(default_factory=list)
|
||||
#: 能不能在面板里查看导入清单:数据源设有任何遮罩(当前设置 ∪ 快照记录)时 False。
|
||||
#: version_view(WP-B)不知道遮罩,一律给 True,由接口一侧(WP-C)改
|
||||
manifest_view: bool = True
|
||||
parts: list[PartView] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class FromCell:
|
||||
"""这一行其余键列各取自哪一格(日期表头格、行标签格、分段标题格、列表头、合计标签)。"""
|
||||
|
||||
role: FromRole
|
||||
#: 这一格给出的是哪一列的值;宽表指标列的行标签(指标名取自哪一格)为 None
|
||||
column: str | None
|
||||
sheet: str
|
||||
#: 不带工作表名的 A1 坐标(「G4」)
|
||||
cell: str
|
||||
#: 只放格子原文:行标签取 labels[].raw,合计标签取 DerivedItem.label_raw;拿不到为 None,不拿规范写法冒充
|
||||
text: str | None = None
|
||||
#: 只给 section_title:配方里的定位文字 locate.title(原表里的字可以不同)
|
||||
locate_title: str | None = None
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class YearSource:
|
||||
"""年份取自哪里:交叉表的轴是文本日期(text / mixed)且配方 axis.year_from 有值时才有,其余为 null。"""
|
||||
|
||||
source: YearFrom
|
||||
#: period:统计期的格子(带工作表名);human:空
|
||||
cells: list[str] = field(default_factory=list)
|
||||
#: human:录入统计期时的署名(未认证);period:None
|
||||
signed_by: str | None = None
|
||||
#: 这一块的日期表头有的是日期格、有的是文本(AxisOut.form == "mixed"),逐格判断不了
|
||||
mixed: bool = False
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class CanonicalView:
|
||||
"""这一行的标签原文和数据库里的值(规范写法)不同时给出(P4-SPEC 2.5 dim 一行、6.2 A8、A18),否则为 null。
|
||||
|
||||
比的是这一行的**标签列**:交叉表逆透视出的 dim 列(原文取自 labels[].raw)、合计表的合计项列(原文取自
|
||||
DerivedItem.label_raw)。raw 是那一格的原文,canonical 是这一行该列在数据库里的值(即 pk 里那一列的值)。
|
||||
**被引用的是值列也给**:A8 合计表那一格是 {"raw": "18-22 时合计", "canonical": "18-22时合计"},累积并集里全角
|
||||
标签那一期的值格是 {"raw": "8-9", "canonical": "8-9"};被引用的就是标签列时同样给(A18)。
|
||||
宽表指标列的行标签是指标名,不是哪一列的值,不给;拿不到逐行原文的(列表形态没有 SegmentLabels)不给。
|
||||
"""
|
||||
|
||||
raw: str
|
||||
canonical: str
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class Recheck:
|
||||
"""按主键参数化回查数据文件的那一句(推断出格子时 ok 恒为 True,不一致的走 table_only)。
|
||||
|
||||
**占位符用 `?`(位置参数),不用 `:p0`**(规格 2.3 R16 写的 `:p0` 以这里和 2.8.1 为准):
|
||||
`SELECT rowid, "列" FROM "表" WHERE "主键1" = ? AND "主键2" = ?`。标识符一律双引号、内部的双引号写两个;
|
||||
WHERE 按冻结表结构 primary_key 的顺序逐列 `= ?` 用 AND 连;params 与占位符一一对应,同样按 primary_key 的顺序
|
||||
(等于 CellSource.pk 的值的顺序)。面板「技术细节」原样显示 sql 和 params,两者必须对得上;provenance_db.recheck
|
||||
返回的 (行, sql, params) 就是这里的 sql、params。
|
||||
"""
|
||||
|
||||
sql: str
|
||||
#: 与 sql 里的 `?` 按位置一一对应(primary_key 的顺序)
|
||||
params: list[Any] = field(default_factory=list)
|
||||
ok: bool
|
||||
|
||||
|
||||
@_json_keys(from_="from")
|
||||
@dataclass(kw_only=True)
|
||||
class CellSource:
|
||||
"""推断出的格子:只在 inferred 时有,否则 null。
|
||||
|
||||
locate(WP-B)填格子、附带的格、年份、规范写法、merged_fill、part_seq、part_rowid;pk、rowid、raw_purged、recheck
|
||||
由接口一侧(WP-C)回查后填。Python 里「from」写成 from_,JSON 里是「from」(见模块说明)。
|
||||
"""
|
||||
|
||||
table: str
|
||||
column: str
|
||||
column_role: ColumnRole
|
||||
kind: TableKind = "data"
|
||||
#: 主键列 → 值(查询快照里那一行的值),键按冻结表结构 primary_key 的顺序(Recheck.params 同序)
|
||||
pk: dict[str, Any] = field(default_factory=dict)
|
||||
#: 快照库的 rowid(并集时是并集 rowid,不是 part_rowid)
|
||||
rowid: int | None = None
|
||||
#: 格子所在的那一期的 seq(导入记录的 seq,同 ChainPart.seq)和该期构建库里的 rowid(单期时 part_rowid == rowid)
|
||||
part_seq: int
|
||||
part_rowid: int
|
||||
sheet: str
|
||||
#: 不带工作表名的 A1 坐标(「G5」)
|
||||
cell: str
|
||||
#: 取自 receipt.tables 里这一列的 header、unit;没有为 None
|
||||
header: str | None = None
|
||||
unit: str | None = None
|
||||
from_: list[FromCell] = field(default_factory=list)
|
||||
year: YearSource | None = None
|
||||
canonical: CanonicalView | None = None
|
||||
#: 这一期的原件已清除(导入记录的 raw_state == "purged",当前状态)
|
||||
raw_purged: bool = False
|
||||
#: 列表这一块的配方写了 merged_data == "fill":值可能取自合并区域的左上格
|
||||
merged_fill: bool = False
|
||||
recheck: Recheck | None = None
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class RelatedCheck:
|
||||
"""相关核对的一条(进响应,不含行级规则:规则只在服务端内部流转,见 RelatedCheckPlan)。"""
|
||||
|
||||
#: 核对 id(R1、K1、G1、T1、C1…)
|
||||
id: str
|
||||
#: CheckResult.kind(relation_sum_eq、relation_not_comparable、derived_sum、formula_refs、column_sum、context_agree…)
|
||||
kind: str
|
||||
title: str
|
||||
part_status: PartStatus
|
||||
#: 只给关系核对(relation_sum_eq);列在当前目标表结构里不存在时 None
|
||||
row_status: RowStatus | None = None
|
||||
#: 只给 K、G、T
|
||||
cell_status: CellStatus | None = None
|
||||
#: 挂在这条核对上的接受理由(同一期清单 acceptances 里 check_id 相同的那条)
|
||||
acceptance: AcceptanceView | None = None
|
||||
#: K 无法核对时的原因摘要:CheckResult.details 的第一条,可能带数字(系统呈现,不经模型转述)
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class ProvenanceOut:
|
||||
"""推断来源接口的响应。asdict 之后就是 P4-SPEC 2.8.1 的 JSON;不变式见 contract_problems。"""
|
||||
|
||||
schema: str = SCHEMA
|
||||
report: ReportRef
|
||||
#: 片段 id
|
||||
segment: str
|
||||
cell: CellRef | None = None
|
||||
status: Status
|
||||
reason: Reason | None = None
|
||||
alert: Alert | None = None
|
||||
#: 运行已封存且封存核对通过(同 sealed.trusted)。False 时照常给结论,界面加一句未封存
|
||||
sealed: bool
|
||||
version: VersionView | None = None
|
||||
cell_source: CellSource | None = None
|
||||
checks: list[RelatedCheck] = field(default_factory=list)
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 跨包的中间结构(P4-SPEC 7.2–7.4)
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
@dataclass
|
||||
class DirectSelect:
|
||||
"""识别器认可的直接选取(WP-A recognize 的结果)。"""
|
||||
|
||||
#: 冻结表结构里的表名(原样)
|
||||
table: str
|
||||
alias: str | None
|
||||
#: 第 i 个结果列对应的表列名(原样,已按 name_key 解析);* 已展开
|
||||
columns: list[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SelectRefusal:
|
||||
"""识别器拒绝:code 是 2.4 的分组,detail 必须是 DETAILS[code] 里的细分值(映射见 DETAILS 上面的说明)。"""
|
||||
|
||||
code: RefusalCode
|
||||
detail: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChainPart:
|
||||
#: 导入记录的 seq(清单里的 seq、快照清单 parts[].seq),不是在 parts 里的位置:每期替换的第二期只有一项,seq 是 2。
|
||||
#: 界面和摘录里的「第 k 次导入」、CellSource.part_seq、PartView.seq 都是它
|
||||
seq: int
|
||||
import_id: str
|
||||
manifest_id: str
|
||||
#: 已取回的导入清单
|
||||
manifest: dict[str, Any]
|
||||
#: 表 → [union 起, 止, part 起, part 止](快照清单 parts[].rows 摊平);单期为 None
|
||||
union_rows: dict[str, list[int]] | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Chain:
|
||||
"""查询快照 → 表结构快照 → 清单(resolve_chain 的结果,P4-SPEC 2.2)。"""
|
||||
|
||||
#: 查询快照记的数据源名
|
||||
source: str
|
||||
#: 清单里的 source_id(与 source_snapshots.source_id 交叉核对,R13)
|
||||
source_id: str
|
||||
#: 查询快照的 data_version
|
||||
snapshot_id: str
|
||||
mode: Mode
|
||||
union: bool
|
||||
#: 单期:导入清单的 db_sha256;并集:快照清单的 union.db_sha256
|
||||
expected_db_sha256: str
|
||||
#: 按统计期升序
|
||||
parts: list[ChainPart]
|
||||
#: 冻结的表结构快照(已取回)
|
||||
schema: dict[str, Any]
|
||||
#: 快照清单(并集才有)
|
||||
snapshot_manifest: dict[str, Any] | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChainProblem:
|
||||
"""resolve_chain 失败:取不回或哈希不符 → manifest_unreadable(R5);链本身对不上 → chain_mismatch(R6)。"""
|
||||
|
||||
code: Literal["manifest_unreadable", "chain_mismatch"]
|
||||
#: 给测试和日志的细分;规格里写的是 ChainProblem("manifest_unreadable") 这种只给 code 的构造,所以有缺省
|
||||
detail: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Located:
|
||||
"""locate 的结果:格子所在的一期、该期的 rowid、推断出的格子(rowid、pk、recheck 留给 WP-C 填)。"""
|
||||
|
||||
part: ChainPart
|
||||
part_rowid: int
|
||||
cell: CellSource
|
||||
|
||||
|
||||
@dataclass
|
||||
class LocateProblem:
|
||||
"""locate 失败:找不到溯源段、锚格所在的块、轴或分段标题 → no_lineage;并集 rowid 越界 → chain_mismatch(红)。"""
|
||||
|
||||
code: Literal["no_lineage", "chain_mismatch"]
|
||||
detail: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SumEqRule:
|
||||
"""关系核对的行级规则:只在服务端内部流转,不进响应。WP-C 据此用 sum_eq_conditions 拼 SQL、按快照库的 rowid 跑。"""
|
||||
|
||||
table: str
|
||||
total: str
|
||||
parts: list[str]
|
||||
#: 列 → INTEGER / REAL / TEXT(取自格子所在那一期的 receipt.tables)
|
||||
types: dict[str, str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class RelatedCheckPlan:
|
||||
"""related_checks 的一项:check 进响应(row_status 由 WP-C 填),rule 是关系核对的行级规则(其余核对为 None)。"""
|
||||
|
||||
check: RelatedCheck
|
||||
rule: SumEqRule | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompileFacts:
|
||||
"""R15 的第二道核对(EXPLAIN 编译、不执行):授权回调读到的表、字节码里 ResultRow 和 Yield 的个数。"""
|
||||
|
||||
tables: set[str]
|
||||
result_rows: int
|
||||
yields: int
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 小函数
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def refuse(code: ReasonCode, detail: str = "", **fmt: Any) -> Reason:
|
||||
"""按 REASON_TEXT 生成 Reason。
|
||||
|
||||
pk_missing 的原文里有「{列}」,用 fmt 填:`refuse("pk_missing", 列=["日期", "时段"])` →「…主键列(日期、时段)…」,
|
||||
列表、元组按「、」连。建议填没带齐的那几列。
|
||||
|
||||
code 不认识、原文里有占位符却没给对应的 fmt,抛 ValueError:两者都是调用处写死的常量,那个分支的任何一条测试都会
|
||||
暴露,不该带着半截原文上界面。
|
||||
|
||||
SQL 判据的四个分组带了 DETAILS 之外的细分**不抛**:细分来自识别器对这条 SQL 的判断,取决于数据,没被测试覆盖
|
||||
的写法(INDEXED BY、表函数之类)漏了映射时,抛异常会让推断来源接口回 500,而不是只给表级来历。这时照样按分组给
|
||||
原文,细分原样留下,记一条警告;contract_problems 会把它报出来,接口和验收测试断言 contract_problems 为空就能抓到。
|
||||
"""
|
||||
if code not in REASON_TEXT:
|
||||
raise ValueError(f"不认识的原因代码:{code!r}")
|
||||
if code in DETAILS and detail and detail not in DETAILS[code]:
|
||||
_log.warning("provenance: %s 不认识细分 %r,应为 %s 之一(照样按分组给原文)", code, detail, DETAILS[code])
|
||||
template = REASON_TEXT[code]
|
||||
values = {k: "、".join(str(x) for x in v) if isinstance(v, (list, tuple)) else str(v) for k, v in fmt.items()}
|
||||
missing = [m for m in _PLACEHOLDER.findall(template) if m not in values]
|
||||
if missing:
|
||||
raise ValueError(f"{code} 的原文需要 {missing} 填值")
|
||||
text = template.format_map(values) if _PLACEHOLDER.search(template) else template
|
||||
return Reason(code=code, detail=detail, text=text)
|
||||
|
||||
|
||||
def alert(code: AlertCode) -> Alert:
|
||||
"""按 ALERT_TEXT 生成标红的提示。"""
|
||||
if code not in ALERT_TEXT:
|
||||
raise ValueError(f"不认识的提示代码:{code!r}")
|
||||
return Alert(code=code, text=ALERT_TEXT[code])
|
||||
|
||||
|
||||
def contract_problems(out: ProvenanceOut) -> list[str]:
|
||||
"""响应的不变式(P4-SPEC 2.3、2.8.1、4.1 渲染矩阵),给接口和验收测试断言用;空列表表示通过。
|
||||
|
||||
不在构造时强制:违反不变式是实现的错,应该在测试里暴露,不该在线上把整个请求变成 500。
|
||||
"""
|
||||
out_problems: list[str] = []
|
||||
bad = out_problems.append
|
||||
if out.schema != SCHEMA:
|
||||
bad(f"schema 应为 {SCHEMA}")
|
||||
if out.status not in get_args(Status):
|
||||
bad(f"status 不认识:{out.status!r}")
|
||||
if out.reason is not None and out.reason.code not in get_args(ReasonCode):
|
||||
bad(f"reason.code 不认识:{out.reason.code!r}")
|
||||
if (out.reason is not None and out.reason.code in DETAILS and out.reason.detail
|
||||
and out.reason.detail not in DETAILS[out.reason.code]):
|
||||
# 空细分不报:R8 的防御分支规格没给细分,refuse 的缺省也是空串
|
||||
bad(f"{out.reason.code} 的细分 {out.reason.detail!r} 不在 DETAILS 里(识别器漏了映射)")
|
||||
if out.alert is not None:
|
||||
if out.alert.code not in get_args(AlertCode):
|
||||
bad(f"alert.code 不认识:{out.alert.code!r}")
|
||||
if out.reason is None or out.reason.code != out.alert.code:
|
||||
bad("有 alert 时 reason.code 必须与 alert.code 相同")
|
||||
if out.reason is not None and out.reason.code in get_args(AlertCode) and out.alert is None:
|
||||
bad(f"{out.reason.code} 必须同时给 alert(标红)")
|
||||
if out.status == "inferred":
|
||||
if out.reason is not None or out.alert is not None:
|
||||
bad("inferred 时 reason、alert 都应为 null")
|
||||
if out.version is None or out.cell_source is None:
|
||||
bad("inferred 时 version、cell_source 都必须有")
|
||||
else:
|
||||
cs = out.cell_source
|
||||
if cs.rowid is None or not cs.pk or cs.recheck is None or not cs.recheck.ok:
|
||||
bad("inferred 时 cell_source 的 rowid、pk、recheck 必须填好,且 recheck.ok 为真")
|
||||
elif cs.recheck.params != list(cs.pk.values()) or _QUOTED.sub("", cs.recheck.sql).count("?") != len(cs.pk):
|
||||
bad("recheck 的 sql 用 ? 占位,个数和 params 都与 pk 一一对应(按 primary_key 的顺序)")
|
||||
hits = [p for p in out.version.parts if p.has_row]
|
||||
if len(hits) != 1:
|
||||
bad(f"inferred 时 version.parts 恰好一项 has_row,现在是 {len(hits)} 项")
|
||||
elif hits[0].seq != cs.part_seq:
|
||||
bad("has_row 的那一期与 cell_source.part_seq 不一致")
|
||||
else:
|
||||
if out.reason is None:
|
||||
bad(f"{out.status} 时必须给 reason")
|
||||
if out.cell_source is not None or out.checks:
|
||||
bad(f"{out.status} 时 cell_source 为 null、checks 为空")
|
||||
if out.version is not None and any(p.has_row for p in out.version.parts):
|
||||
bad("没有推断出格子时 has_row 全为 False")
|
||||
if out.status == "table_only" and out.version is None:
|
||||
bad("table_only 时 version 必须有")
|
||||
if out.status == "none" and out.version is not None:
|
||||
bad("none 时 version 为 null")
|
||||
return out_problems
|
||||
@@ -446,20 +446,36 @@ def _check_t(sql: _Sql, cid: str, title: str, items: list[DerivedItem], full_cal
|
||||
# ---------------------------------------------------------------- R:关系
|
||||
|
||||
|
||||
def _check_sum_eq(sql: _Sql, rel: SumEq, title: str, columns: list[ColumnOut], grain: list[str],
|
||||
lineage: dict[str, dict[str, list[list[Any]]]]) -> CheckResult:
|
||||
table, total, parts = rel.table, rel.total, list(rel.parts)
|
||||
def sum_eq_conditions(total: str, parts: list[str], types: dict[str, str]) -> tuple[str, str, str]:
|
||||
"""关系核对「total = parts 之和」逐行判定用的三段 SQL 条件:(nonnull, anynull, mismatch)。
|
||||
|
||||
- nonnull:参与的列都不是空值(这一行比得上);
|
||||
- anynull:任一列是空值(这一行无法核对);
|
||||
- mismatch:合计与各部分之和不等。只在 nonnull 成立时有意义,调用方自己和 nonnull 组合(AND,或者 CASE 里
|
||||
先判 anynull)。有一列是 REAL 时按相对误差 1e-9 比(浮点加法有舍入),全是整数时严格比。
|
||||
|
||||
为什么抽出来:导入时的核对(_check_sum_eq)和证据面板的行级核对(provenance_db.row_status,P4-SPEC 2.7)
|
||||
必须用同一段代码生成的条件。面板说「这一天成立」,就得和导入时的结论是同一个判断,不能各写一份、日后改了一边。
|
||||
types 是列名 → INTEGER / REAL / TEXT(回执里这一期的列类型),缺的列按非 REAL 处理。
|
||||
"""
|
||||
names = [total, *parts]
|
||||
nonnull = " AND ".join(f"{_q(c)} IS NOT NULL" for c in names)
|
||||
anynull = " OR ".join(f"{_q(c)} IS NULL" for c in names)
|
||||
checked = sql.one(f"SELECT COUNT(*) FROM {_q(table)} WHERE {nonnull}")[0]
|
||||
nulls = sql.one(f"SELECT COUNT(*) FROM {_q(table)} WHERE {anynull}")[0]
|
||||
types = {c.name: c.type for c in columns}
|
||||
added = " + ".join(_q(p) for p in parts)
|
||||
if any(types.get(c) == "REAL" for c in names):
|
||||
cond = f"ABS({_q(total)} - ({added})) > 1e-9 * MAX(1, ABS({_q(total)}))"
|
||||
mismatch = f"ABS({_q(total)} - ({added})) > 1e-9 * MAX(1, ABS({_q(total)}))"
|
||||
else:
|
||||
cond = f"{_q(total)} <> {added}"
|
||||
mismatch = f"{_q(total)} <> {added}"
|
||||
return nonnull, anynull, mismatch
|
||||
|
||||
|
||||
def _check_sum_eq(sql: _Sql, rel: SumEq, title: str, columns: list[ColumnOut], grain: list[str],
|
||||
lineage: dict[str, dict[str, list[list[Any]]]]) -> CheckResult:
|
||||
table, total, parts = rel.table, rel.total, list(rel.parts)
|
||||
nonnull, anynull, cond = sum_eq_conditions(total, parts, {c.name: c.type for c in columns})
|
||||
checked = sql.one(f"SELECT COUNT(*) FROM {_q(table)} WHERE {nonnull}")[0]
|
||||
nulls = sql.one(f"SELECT COUNT(*) FROM {_q(table)} WHERE {anynull}")[0]
|
||||
added = " + ".join(_q(p) for p in parts)
|
||||
where = f"{nonnull} AND {cond}"
|
||||
failed = sql.one(f"SELECT COUNT(*) FROM {_q(table)} WHERE {where}")[0]
|
||||
picked = ", ".join(["rowid", *(_q(g) for g in grain), _q(total), f"({added})"])
|
||||
|
||||
@@ -239,6 +239,11 @@ def _in_spans(spans: list[tuple[int, int]], starts: list[int], x: int) -> bool:
|
||||
return i >= 0 and spans[i][0] <= x <= spans[i][1]
|
||||
|
||||
|
||||
def _hidden_cell(sh: _Sheet, r: int, c: int) -> bool:
|
||||
"""这一格所在的行或列是否被隐藏(OutsideText.hidden)。行列都看:隐藏列里的备注和隐藏行里的一样不外发。"""
|
||||
return sh.hidden_row(r) or sh.hidden_col(c)
|
||||
|
||||
|
||||
def _open_sqlite(path: str, *, readonly: bool) -> sqlite3.Connection:
|
||||
"""关 DQS 的 sqlite3 连接,和核对模块共用 engine.open_checked_sqlite:关不掉就拒绝连接。"""
|
||||
return _data_engine.open_checked_sqlite(path, readonly=readonly)
|
||||
@@ -2475,7 +2480,10 @@ class _Engine:
|
||||
coord = sh.coord(r, c)
|
||||
sh.roles[role] += 1
|
||||
sh.regions.feed_row(r, [(c, role)])
|
||||
self.ex.outside_text.append(OutsideText(sh.name, coord, text, kind, period_source)) # type: ignore[arg-type]
|
||||
# 期 4(P4-SPEC 3.2、调整 10):记下这格是否在隐藏行列里。裁判摘录只把确知可见的区域外文字全文外发,
|
||||
# 隐藏行列里常是内部备注(H10 的依据正是隐藏内容里实测有口令),不能借裁判多一条外发通道
|
||||
self.ex.outside_text.append(OutsideText(sh.name, coord, text, kind, period_source, # type: ignore[arg-type]
|
||||
hidden=_hidden_cell(sh, r, c)))
|
||||
if kind == "text_digits" and not period_source:
|
||||
self.problems.add(
|
||||
"outside_digits",
|
||||
@@ -2551,7 +2559,9 @@ class _Engine:
|
||||
sh.regions.feed_row(r, [(c, "context")])
|
||||
if not is_pure_period_cell(text, prefer):
|
||||
out.annotated[coord] = period_residue(text) or ""
|
||||
self.ex.outside_text.append(OutsideText(sh.name, coord, text, "text_digits", True))
|
||||
# 带附加文字的统计期格同样是区域外文字,同样记是否隐藏(同 _outside_text)
|
||||
self.ex.outside_text.append(OutsideText(sh.name, coord, text, "text_digits", True,
|
||||
hidden=_hidden_cell(sh, r, c)))
|
||||
self.ex.period = out
|
||||
cells = list(out.cells)
|
||||
if failed:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""配方导入的契约(期 2,波次 0 由编排者提交;期 3 的改动见 P3-SPEC 9.2,同样由波次 0 提交):配方 JSON 的模型、
|
||||
"""配方导入的契约(期 2,波次 0 由编排者提交;期 3 的改动见 P3-SPEC 9.2,同样由波次 0 提交;期 4 只给 OutsideText
|
||||
加了 hidden,见 P4-SPEC 7.1):配方 JSON 的模型、
|
||||
执行结果和各模块之间传递的数据结构。
|
||||
|
||||
各工作包之间只通过这里的类型和 P2-SPEC.md、P3-SPEC.md 里写明的函数签名交互。这里**只有声明**,没有业务逻辑
|
||||
@@ -706,6 +707,12 @@ class OutsideText:
|
||||
kind: Literal["text", "text_digits"]
|
||||
#: 这格同时是统计期来源(带附加文字的统计期格,PeriodOut.annotated)
|
||||
period_source: bool = False
|
||||
#: 期 4(P4-SPEC 7.1,WP-0):这格所在的行或列是否被隐藏,执行器写入(_outside_text、统计期格附带的文字两处)。
|
||||
#: True = 在隐藏行列里;False = 可见;None = 导入时没有记录(期 4 之前的清单没有这个键,
|
||||
#: recipe_imports._known 读老清单时缺键就是 None,不用改读法)。
|
||||
#: 裁判摘录只送 hidden is False 的项(P4-SPEC 3.2):隐藏行里的内部备注不能多一条经裁判外发的通道,
|
||||
#: 不知道是否隐藏的(None)同样只计条数。差异卡、确认项只按坐标、文字、kind、period_source 比较,不看它
|
||||
hidden: bool | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -0,0 +1,533 @@
|
||||
"""期 4「推断的来源」的 SQL 判据:认出「从一张表里直接选取若干列」这一种形状(P4-SPEC 2.4、7.2)。
|
||||
|
||||
下钻要回答「查询结果的第 i 列是哪张表的哪一列」。只有一张表、每个结果列都是这张表某列的直接引用时,这个问题才有
|
||||
唯一的答案;改名、表达式、多表、子查询、集合运算之后,结果列装的可能是另一列的值,值恰好相等时就会下钻到错误的
|
||||
格子。所以这里只认一种形状,形状之外一律拒绝:**宁可不下钻,也不能下钻错**。
|
||||
|
||||
**读法和执行这条 SQL 的 SQLite 相同**:分词用守卫的 `guard._tokens(sql, guard._SQLITE)`。查询快照里存的 SQL 是
|
||||
`guard.check(…, dialect="sqlite")` 的返回值,执行的也正是它(engine.run_query)。`_SQLITE` 读法:单引号字符串里
|
||||
反斜杠不是转义;`[…]` 是标识符,里面没有转义,遇到第一个 `]` 就结束;`"…"`、`` `…` `` 是标识符,两个引号表示
|
||||
一个引号。识别器和数据库读的是同一串记号,中间没有第二种读法。
|
||||
|
||||
**SQLite 分词的两处细节**(tokenize.c,实测 3.53):
|
||||
- U+FEFF(字节 EF BB BF,CC_BOM)出现在记号开头时是空白,出现在名字中间时是名字的一部分。不能照「码位 ≥ 0x80
|
||||
一律算名字」读:那样紧跟在 U+FEFF 后面的 UNION 成了一个普通的词,藏在它后面的 UNION ALL、GROUP BY 就看不见了,
|
||||
SQLite 却照样执行,结果列里混着另一列的值;
|
||||
- 竖制表符出现在记号开头时是非法字符,紧跟在其他空白后面时是空白(CC_SPACE 的循环用 sqlite3Isspace,含 \\v)。
|
||||
这里一律当空白:SQLite 不当空白的位置整条 SQL 执行不了,不会留下查询快照,判据碰不到;能执行的位置读法相同。
|
||||
|
||||
**为什么不用 `evidence._sql_scan`**:它按多方言「宁可少认一张表」设计,单引号里认反斜杠转义、方括号里含引号时不当
|
||||
标识符。藏在 `ESCAPE '\\'` 或 `AS [t']` 后面的 `UNION ALL` 在它眼里不存在,SQLite 却照样执行(P4-SPEC 附录 B7)。
|
||||
对 `sql_tables` 那只是少认一张表,对下钻就是错格子。`_sql_scan`、`sql_tables` 本身不动(H7)。
|
||||
|
||||
**执行时与存下的字面不同的一处**:engine 经 SQLAlchemy 的 `text()` 执行,它把 `\\:` 改成 `:`、把 `:名字` 当成绑定
|
||||
参数。后者没有给值,执行会失败,不会留下查询快照;前者只删掉冒号前的反斜杠,SQLite 读法里反斜杠不影响任何记号的
|
||||
边界,只改字符串和带引号名字的内容,而配方导入的表名、列名不含反斜杠和冒号(names.name_problem),对不上任何表列。
|
||||
|
||||
**分组**(界面显示分组的原文,细分给测试和日志用,取值见 provenance_types.DETAILS):
|
||||
- expression:表达式、聚合、字符串或字面量关键字、窗口、COLLATE、DISTINCT;
|
||||
- alias:给列改名(含省略 AS 的改名);
|
||||
- multi_table:JOIN、逗号连接、子查询、公用表表达式、集合运算;
|
||||
- unparsed:不是 SELECT、多条语句、表的写法不认识、限定名不是本表、列或表找不到、重名、`*` 对不上、尾部写法不认识。
|
||||
|
||||
判据绑在文档标记 `DOC_PROVENANCE = 1` 上(P4-SPEC 1.4):以后放宽文法要升版本号、给 recognize 加 version 参数,
|
||||
已有 `provenance: 1` 的文档仍按这一版判,结论不跟着变。
|
||||
|
||||
只许导入守卫、names 和 provenance_types(P4-SPEC 7.2):判据是纯函数,不连库,不依赖证据层和接口层。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.data.guard import _SQLITE, _tokens
|
||||
from app.data.names import name_key
|
||||
from app.data.provenance_types import DirectSelect, SelectRefusal
|
||||
|
||||
# ==========================================================================
|
||||
# 分词:守卫的 SQLite 读法之上,再按 SQLite 的规则把代码段切成词和标点
|
||||
# ==========================================================================
|
||||
|
||||
#: SQLite 在记号之间认的空白:CC_SPACE 的五个,加竖制表符。不能用 str.isspace:全角空格、不换行空格在 SQLite 里是
|
||||
#: 名字的一部分(除记号开头的 U+FEFF 之外,码位 ≥ 0x80 的字符一律算名字)。竖制表符出现在记号开头时 SQLite 报非法
|
||||
#: 记号,紧跟其他空白时当空白;一律当空白是安全的:前一种位置执行不了,后一种读法相同(模块说明)
|
||||
_SPACE = frozenset(" \t\n\v\f\r")
|
||||
#: U+FEFF 只在记号开头是空白(CC_BOM),名字中间是名字的一部分(「日\ufeff期」是一个名字),所以不放进 _SPACE、
|
||||
#: 只在 _split_code 判断记号开头的地方单独认
|
||||
_BOM = "\ufeff"
|
||||
_ASCII_LETTERS = frozenset("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
|
||||
_ASCII_DIGITS = frozenset("0123456789")
|
||||
|
||||
|
||||
def _name_start(ch: str) -> bool:
|
||||
"""不加引号的名字的开头:ASCII 字母、下划线、码位 ≥ 0x80 的字符(同 SQLite,也同 2 版实体语法)。
|
||||
|
||||
U+FEFF 例外:出现在记号开头时 SQLite 当空白,调用处(_split_code)先把它判掉,轮不到这里。名字中间的 U+FEFF
|
||||
仍算名字:_name_char 借这个函数认后续字符,所以这里不排除它。"""
|
||||
return ch in _ASCII_LETTERS or ch == "_" or ord(ch) >= 0x80
|
||||
|
||||
|
||||
def _name_char(ch: str) -> bool:
|
||||
"""名字的后续字符:开头字符之外再加 ASCII 数字和 $(SQLite 的 IdChar)。不用 isalnum:「٣」这类非 ASCII
|
||||
数字本来就 ≥ 0x80,算名字;ASCII 以外的 isdigit 不能决定切词。"""
|
||||
return _name_start(ch) or ch in _ASCII_DIGITS or ch == "$"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Lex:
|
||||
#: word:不加引号的词;name:带引号的名字("…" `…` […]);str:字符串;blob:x'…';num:数字;punct:标点;
|
||||
#: sep:分号
|
||||
kind: str
|
||||
#: word、num、punct 是原文;name、str 是去掉定界符、还原了成对引号的内容
|
||||
text: str
|
||||
#: 在原 SQL 里的结束位置:认 x'…' 时要知道 x 和引号之间有没有空白
|
||||
end: int
|
||||
|
||||
|
||||
def _split_code(text: str, base: int, out: list[_Lex]) -> None:
|
||||
"""把一段代码(守卫已经把注释、字符串、带引号的名字、分号切出去了)按 SQLite 的规则切成词、数字和标点。
|
||||
|
||||
数字只吃数字和小数点:`1e5`、`0x1F` 会被切成「数字 + 词」,`1union` 也会切出 UNION。SQLite 遇到数字后紧跟
|
||||
名字字符会报非法记号、根本不执行,所以这样切只会让判据多看见词、多拒,不会少看见。
|
||||
|
||||
循环顶层的每个位置都是一个记号的开头(代码段的开头紧跟在注释、字符串、带引号的名字或分号后面,也是):
|
||||
U+FEFF 在这里当空白,同 SQLite 的 CC_BOM;进了名字以后由 _name_char 当名字字符吃掉。
|
||||
"""
|
||||
i, n = 0, len(text)
|
||||
while i < n:
|
||||
ch = text[i]
|
||||
if ch in _SPACE or ch == _BOM:
|
||||
i += 1
|
||||
elif _name_start(ch):
|
||||
j = i + 1
|
||||
while j < n and _name_char(text[j]):
|
||||
j += 1
|
||||
out.append(_Lex("word", text[i:j], base + j))
|
||||
i = j
|
||||
elif ch in _ASCII_DIGITS or (ch == "." and i + 1 < n and text[i + 1] in _ASCII_DIGITS):
|
||||
j = i + 1
|
||||
while j < n and (text[j] in _ASCII_DIGITS or text[j] == "."):
|
||||
j += 1
|
||||
out.append(_Lex("num", text[i:j], base + j))
|
||||
i = j
|
||||
else:
|
||||
out.append(_Lex("punct", ch, base + i + 1))
|
||||
i += 1
|
||||
|
||||
|
||||
def _lex(sql: str, *, strict: bool = True) -> list[_Lex] | None:
|
||||
"""整条 SQL → 词序列。引号没闭合时 strict 返回 None(SQLite 会报非法记号,存下的 SQL 不该这样);
|
||||
不 strict 时照样切,没闭合的那一段内容取到末尾(tables_in 宁可多认)。
|
||||
|
||||
注释当空白跳过:SQLite 也这样读,注释两边的代码各自成词(`SEL/**/ECT` 是两个词)。
|
||||
"""
|
||||
out: list[_Lex] = []
|
||||
pos = 0
|
||||
for tok in _tokens(sql, _SQLITE):
|
||||
start, pos = pos, pos + len(tok.text)
|
||||
if tok.kind == "comment":
|
||||
continue
|
||||
if tok.kind == "sep":
|
||||
out.append(_Lex("sep", ";", pos))
|
||||
elif tok.kind in ("string", "ident"):
|
||||
if tok.close == 0 and strict:
|
||||
return None
|
||||
inner = tok.text[tok.open:len(tok.text) - tok.close]
|
||||
quote = tok.text[0]
|
||||
if tok.kind == "string":
|
||||
inner = inner.replace("''", "'")
|
||||
prev = out[-1] if out else None
|
||||
# x'00':紧挨在单独一个 x 后面的单引号串是 BLOB 字面量,不是「名字 x 加字符串别名」
|
||||
if prev is not None and prev.kind == "word" and prev.text in ("x", "X") and prev.end == start:
|
||||
out[-1] = _Lex("blob", inner, pos)
|
||||
else:
|
||||
out.append(_Lex("str", inner, pos))
|
||||
else:
|
||||
# 方括号里没有转义;双引号、反引号里两个引号表示一个
|
||||
if quote in "\"`":
|
||||
inner = inner.replace(quote * 2, quote)
|
||||
out.append(_Lex("name", inner, pos))
|
||||
else:
|
||||
_split_code(tok.text, start, out)
|
||||
return out
|
||||
|
||||
|
||||
def _kw(lex: _Lex | None) -> str | None:
|
||||
"""不加引号的词按关键字比较时用的键(只对 ASCII 不区分大小写,同 SQLite);不是词返回 None。
|
||||
|
||||
带引号的名字永远不是关键字:`"union"` 是一个叫 union 的列。
|
||||
"""
|
||||
return name_key(lex.text) if lex is not None and lex.kind == "word" else None
|
||||
|
||||
|
||||
def _is_name(lex: _Lex | None) -> bool:
|
||||
return lex is not None and lex.kind in ("word", "name")
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 关键字表
|
||||
# ==========================================================================
|
||||
|
||||
#: P4-SPEC 2.4:尾部(以及整条语句里)不得出现的词,按细分归组。七组的并集就是规格的那张表,测试钉住
|
||||
_COMPOUND = frozenset({"union", "intersect", "except"})
|
||||
_CTE = frozenset({"with", "recursive"})
|
||||
_SUBQUERY = frozenset({"select", "values"})
|
||||
_JOIN = frozenset({"join", "natural", "cross", "inner", "left", "right", "full", "outer"})
|
||||
_WINDOW = frozenset({"over", "window"})
|
||||
#: GROUP BY / HAVING:结果是聚合出来的
|
||||
_AGGREGATE = frozenset({"group", "having"})
|
||||
#: 尾部的 FROM、DISTINCT 只会出现在 `IS [NOT] DISTINCT FROM` 这类写法里:不影响结果列,但超出这一版文法
|
||||
_TAIL_ODD = frozenset({"from", "distinct"})
|
||||
FORBIDDEN = _COMPOUND | _CTE | _SUBQUERY | _JOIN | _WINDOW | _AGGREGATE | _TAIL_ODD
|
||||
|
||||
#: 不加引号时是字面量的关键字(P4-SPEC 2.4)。CURRENT_* 永远是关键字;TRUE、FALSE 只在表里有同名列时才指那一列,
|
||||
#: 读法取决于表结构,一律拒最保守
|
||||
_LITERAL_KEYWORDS = frozenset({"null", "true", "false", "current_date", "current_time", "current_timestamp"})
|
||||
|
||||
#: SQLite 的保留字里不能当不加引号的名字用的那些:出现在选取项、表名的位置,这一项就是表达式(CASE …、NOT x、
|
||||
#: EXISTS (…))或者写法不对。CAST、RAISE、MATCH、GLOB、END、FILTER 之类不在里面:SQLite 允许它们当名字(fallback),
|
||||
#: 后面紧跟「(」时由下一步按函数调用拒,单独出现、又真有这一列时它就是那一列
|
||||
_EXPR_WORDS = frozenset({
|
||||
"case", "not", "exists", "when", "then", "else", "select", "distinct", "all",
|
||||
"from", "where", "group", "having", "order", "limit", "as", "on", "using",
|
||||
"and", "or", "is", "in", "between", "escape", "collate", "isnull", "notnull",
|
||||
})
|
||||
|
||||
#: 选取项后面紧跟这些词:这一项是运算的一部分(`"日期" IS NULL`、`"a" BETWEEN …`),不是省略 AS 的改名
|
||||
_OPERATOR_WORDS = frozenset({
|
||||
"is", "isnull", "notnull", "not", "and", "or", "in", "like", "glob", "regexp", "match", "between",
|
||||
"escape", "filter",
|
||||
})
|
||||
|
||||
#: 表名后面紧跟这些词时它们不是表别名:子句开头、连接、`INDEXED BY` / `NOT INDEXED`、ON / USING
|
||||
_AFTER_TABLE = frozenset({
|
||||
"where", "order", "limit", "group", "having", "window", "indexed", "not", "on", "using", "as",
|
||||
*_JOIN, *_COMPOUND,
|
||||
})
|
||||
|
||||
#: 尾部允许的开头(P4-SPEC 2.4 文法):只过滤、排序、截取行,不改变结果列的来历
|
||||
_TAIL_START = frozenset({"where", "order", "limit"})
|
||||
|
||||
|
||||
def _refuse(code: str, detail: str) -> SelectRefusal:
|
||||
return SelectRefusal(code, detail) # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# recognize
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def _statement(lexes: list[_Lex]) -> list[_Lex] | SelectRefusal:
|
||||
"""去掉尾部的分号;分号后面还有东西就是多条语句。
|
||||
|
||||
存下的 SQL 是 guard.check 的返回值,本来就没有分号和注释。P4-SPEC 7.2 写的是「出现就按 multi_statement 拒」,
|
||||
但 2.4 的文法写了 `[;]`,原型的 24 例(期望不变)和 6.3 要求注释里的 JOIN、尾分号照样认可,两处矛盾。这里取
|
||||
后者,按 SQLite 的读法接受尾分号、把注释当空白:两者都不改变这条语句选了什么,读法又和执行时相同,不会因此
|
||||
认错;真正的第二条语句照样拒。
|
||||
"""
|
||||
end = len(lexes)
|
||||
while end and lexes[end - 1].kind == "sep":
|
||||
end -= 1
|
||||
body = lexes[:end]
|
||||
if any(lx.kind == "sep" for lx in body):
|
||||
return _refuse("unparsed", "multi_statement")
|
||||
return body
|
||||
|
||||
|
||||
def _global_shape(body: list[_Lex]) -> SelectRefusal | None:
|
||||
"""整条语句里只要出现就不是「一张表的直接选取」的关键字:集合运算、公用表表达式、子查询、连接、窗口。
|
||||
|
||||
先于逐项的形状判断:B7 那类写法(`… ESCAPE '\\' UNION ALL …`)不管前面哪一项长什么样,都要报成集合运算。
|
||||
词只取代码段里的:字符串、带引号的名字、注释里出现的不算,和 SQLite 的读法一样。
|
||||
"""
|
||||
words = {_kw(lx) for lx in body[1:] if lx.kind == "word"}
|
||||
for group, detail, code in ((_COMPOUND, "compound", "multi_table"), (_CTE, "cte", "multi_table"),
|
||||
(_SUBQUERY, "subquery", "multi_table"), (_JOIN, "join", "multi_table"),
|
||||
(_WINDOW, "window", "expression")):
|
||||
if words & group:
|
||||
return _refuse(code, detail)
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Item:
|
||||
qualifier: str | None
|
||||
column: str
|
||||
|
||||
|
||||
def _item_start(lex: _Lex | None) -> SelectRefusal | None:
|
||||
"""选取项的第一个记号不是名字时,这一项是什么。是名字返回 None。"""
|
||||
if lex is None:
|
||||
return _refuse("unparsed", "not_select")
|
||||
if lex.kind == "str":
|
||||
return _refuse("expression", "string_literal")
|
||||
kw = _kw(lex)
|
||||
if kw in _LITERAL_KEYWORDS:
|
||||
return _refuse("expression", "literal_keyword")
|
||||
if kw in _EXPR_WORDS or not _is_name(lex):
|
||||
# 数字、blob、括号、正负号、CASE / CAST / NOT …
|
||||
return _refuse("expression", "expression")
|
||||
return None
|
||||
|
||||
|
||||
def _after_item(lex: _Lex | None) -> SelectRefusal:
|
||||
"""选取项后面只能紧跟「,」或 FROM(P4-SPEC 2.4)。其余的分别是什么。"""
|
||||
kw = _kw(lex)
|
||||
if lex is None:
|
||||
# 没有 FROM:不是从表里选
|
||||
return _refuse("unparsed", "table_shape")
|
||||
if kw == "as":
|
||||
return _refuse("alias", "alias")
|
||||
if kw == "collate":
|
||||
return _refuse("expression", "collate")
|
||||
if kw in _OPERATOR_WORDS:
|
||||
return _refuse("expression", "expression")
|
||||
if lex.kind in ("name", "str") or (lex.kind == "word" and kw not in _EXPR_WORDS):
|
||||
# `"日期" "分区乙"`、`"日期" '分区乙'`、`"日期" 别名`:SQLite 都当省略 AS 的改名
|
||||
return _refuse("alias", "alias")
|
||||
# 「(」是函数调用,其余标点、数字是运算
|
||||
return _refuse("expression", "expression")
|
||||
|
||||
|
||||
def _select_list(body: list[_Lex], k: int) -> tuple[list[_Item] | None, int] | SelectRefusal:
|
||||
"""解析选取项,返回 (项列表,FROM 的位置)。`*` 单独出现时项列表为 None。"""
|
||||
if k < len(body) and body[k].kind == "punct" and body[k].text == "*":
|
||||
nxt = body[k + 1] if k + 1 < len(body) else None
|
||||
if _kw(nxt) != "from":
|
||||
if nxt is not None and nxt.kind == "punct" and nxt.text == ",":
|
||||
# `*, "日期"`:* 只许单独出现,否则结果列和表的列对不上位置
|
||||
return _refuse("unparsed", "star_mismatch")
|
||||
return _after_item(nxt)
|
||||
return None, k + 1
|
||||
items: list[_Item] = []
|
||||
while True:
|
||||
lex = body[k] if k < len(body) else None
|
||||
if lex is not None and lex.kind == "punct" and lex.text == "*":
|
||||
# `"日期", *`
|
||||
return _refuse("unparsed", "star_mismatch")
|
||||
refusal = _item_start(lex)
|
||||
if refusal is not None:
|
||||
return refusal
|
||||
assert lex is not None
|
||||
k += 1
|
||||
qualifier: str | None = None
|
||||
column = lex.text
|
||||
if k < len(body) and body[k].kind == "punct" and body[k].text == ".":
|
||||
target = body[k + 1] if k + 1 < len(body) else None
|
||||
if target is not None and target.kind == "punct" and target.text == "*":
|
||||
# `t.*`:不在这一版文法里
|
||||
return _refuse("unparsed", "star_mismatch")
|
||||
if not _is_name(target):
|
||||
return _refuse("expression", "expression")
|
||||
assert target is not None
|
||||
qualifier, column = lex.text, target.text
|
||||
k += 2
|
||||
if k < len(body) and body[k].kind == "punct" and body[k].text == ".":
|
||||
# `main.日客流.日期`:带库名的限定名
|
||||
return _refuse("unparsed", "qualifier")
|
||||
items.append(_Item(qualifier, column))
|
||||
nxt = body[k] if k < len(body) else None
|
||||
if nxt is not None and nxt.kind == "punct" and nxt.text == ",":
|
||||
k += 1
|
||||
continue
|
||||
if _kw(nxt) == "from":
|
||||
return items, k
|
||||
return _after_item(nxt)
|
||||
|
||||
|
||||
def _from_clause(body: list[_Lex], k: int) -> tuple[str, str | None, int] | SelectRefusal:
|
||||
"""FROM 后面:表名、可选的表别名。返回 (表名,别名,尾部的起点)。"""
|
||||
lex = body[k] if k < len(body) else None
|
||||
if lex is not None and lex.kind == "punct" and lex.text == "(":
|
||||
# FROM (SELECT …);`FROM (日客流)` 在 SQLite 里也能执行,一律不认
|
||||
return _refuse("multi_table", "subquery")
|
||||
if not _is_name(lex) or _kw(lex) in _AFTER_TABLE | _EXPR_WORDS:
|
||||
# 字符串当表名(SQLite 的历史兼容)、数字、缺表名
|
||||
return _refuse("unparsed", "table_shape")
|
||||
assert lex is not None
|
||||
table = lex.text
|
||||
k += 1
|
||||
nxt = body[k] if k < len(body) else None
|
||||
if nxt is not None and nxt.kind == "punct" and nxt.text == ".":
|
||||
return _refuse("unparsed", "qualified_table")
|
||||
if nxt is not None and nxt.kind == "punct" and nxt.text == "(":
|
||||
# 表函数(json_each(…))
|
||||
return _refuse("unparsed", "table_shape")
|
||||
alias: str | None = None
|
||||
if _kw(nxt) == "as":
|
||||
target = body[k + 1] if k + 1 < len(body) else None
|
||||
if not _is_name(target) or _kw(target) in _AFTER_TABLE | _EXPR_WORDS:
|
||||
# `AS '别名'`:SQLite 认字符串当别名,这一版文法不认
|
||||
return _refuse("unparsed", "table_shape")
|
||||
assert target is not None
|
||||
alias = target.text
|
||||
k += 2
|
||||
elif nxt is not None and (nxt.kind == "name"
|
||||
or (nxt.kind == "word" and _kw(nxt) not in _AFTER_TABLE | _EXPR_WORDS)):
|
||||
alias = nxt.text
|
||||
k += 1
|
||||
return table, alias, k
|
||||
|
||||
|
||||
def _tail(body: list[_Lex], k: int) -> SelectRefusal | None:
|
||||
"""尾部只能以 WHERE / ORDER / LIMIT 开头,里面不得有 P4-SPEC 2.4 那张表里的词。
|
||||
|
||||
整条语句里的集合运算、子查询、连接、窗口已经在 _global_shape 拦了,这里只剩 GROUP / HAVING(聚合)和尾部的
|
||||
FROM、DISTINCT。
|
||||
"""
|
||||
if k >= len(body):
|
||||
return None
|
||||
head = body[k]
|
||||
kw = _kw(head)
|
||||
if kw not in _TAIL_START:
|
||||
if head.kind == "punct" and head.text == ",":
|
||||
return _refuse("multi_table", "comma_join")
|
||||
if kw in _AGGREGATE:
|
||||
return _refuse("expression", "expression")
|
||||
# INDEXED BY、NOT INDEXED、第二个别名、括号……
|
||||
return _refuse("unparsed", "tail_shape")
|
||||
words = {_kw(lx) for lx in body[k:] if lx.kind == "word"}
|
||||
if words & _AGGREGATE:
|
||||
return _refuse("expression", "expression")
|
||||
if words & _TAIL_ODD:
|
||||
return _refuse("unparsed", "tail_shape")
|
||||
return None
|
||||
|
||||
|
||||
def _index(names: Iterable[str]) -> dict[str, str | None]:
|
||||
"""name_key → 原名。两个名字撞了键(冻结表结构里不该出现)时记 None:说不清是哪一个,按找不到处理。"""
|
||||
out: dict[str, str | None] = {}
|
||||
for name in names:
|
||||
key = name_key(name)
|
||||
out[key] = None if key in out else name
|
||||
return out
|
||||
|
||||
|
||||
def recognize(sql: str, *, result_columns: list[str], tables: dict[str, list[str]]
|
||||
) -> DirectSelect | SelectRefusal:
|
||||
"""tables:冻结表结构里的 {表名: [列名…]}(按定义顺序)。
|
||||
|
||||
认可(P4-SPEC 2.4 的文法)→ DirectSelect;否则 → SelectRefusal(code, detail)。纯函数,不连库,不抛异常。
|
||||
文法对应 DOC_PROVENANCE = 1(1.4);以后放宽文法时加 version 参数,不改这一版的结论。
|
||||
|
||||
判断顺序:分词和语句个数 → 是不是 SELECT(WITH 开头是公用表表达式)→ 整条语句里的集合运算、公用表表达式、
|
||||
子查询、连接、窗口 → 选取项 → FROM → 尾部 → 名字解析(表、限定名、列、重名)→ 结果列与选取项逐个对应。
|
||||
"""
|
||||
lexes = _lex(sql)
|
||||
if lexes is None:
|
||||
return _refuse("unparsed", "not_select")
|
||||
body = _statement(lexes)
|
||||
if isinstance(body, SelectRefusal):
|
||||
return body
|
||||
first = _kw(body[0]) if body else None
|
||||
if first == "with":
|
||||
return _refuse("multi_table", "cte")
|
||||
if first != "select":
|
||||
return _refuse("unparsed", "not_select")
|
||||
refusal = _global_shape(body)
|
||||
if refusal is not None:
|
||||
return refusal
|
||||
|
||||
k = 1
|
||||
if _kw(body[k] if k < len(body) else None) == "all":
|
||||
k += 1
|
||||
if _kw(body[k] if k < len(body) else None) == "distinct":
|
||||
return _refuse("expression", "distinct")
|
||||
parsed = _select_list(body, k)
|
||||
if isinstance(parsed, SelectRefusal):
|
||||
return parsed
|
||||
items, k = parsed
|
||||
shape = _from_clause(body, k + 1)
|
||||
if isinstance(shape, SelectRefusal):
|
||||
return shape
|
||||
table_written, alias, k = shape
|
||||
refusal = _tail(body, k)
|
||||
if refusal is not None:
|
||||
return refusal
|
||||
|
||||
# ---- 名字解析:一律按 name_key(只对 ASCII 不区分大小写,同 SQLite) ----
|
||||
table = _index(tables).get(name_key(table_written))
|
||||
if table is None:
|
||||
return _refuse("unparsed", "unknown_table")
|
||||
defined = list(tables[table])
|
||||
if items is None:
|
||||
# `*`:结果列必须和表的列按 name_key 逐个相等、顺序一致,才能按位置对应
|
||||
if [name_key(c) for c in result_columns] != [name_key(c) for c in defined]:
|
||||
return _refuse("unparsed", "star_mismatch")
|
||||
return DirectSelect(table, alias, defined)
|
||||
# 有别名时只认别名:SQLite 里表起了别名之后,原表名不能再当限定名(`SELECT 日客流.日期 FROM 日客流 AS t`
|
||||
# 报 no such column),所以这一条比「表名或别名」严,严出来的只是执行不了的写法
|
||||
owner = name_key(alias if alias is not None else table_written)
|
||||
if any(it.qualifier is not None and name_key(it.qualifier) != owner for it in items):
|
||||
return _refuse("unparsed", "qualifier")
|
||||
columns_index = _index(defined)
|
||||
columns: list[str] = []
|
||||
for it in items:
|
||||
column = columns_index.get(name_key(it.column))
|
||||
if column is None:
|
||||
return _refuse("unparsed", "unknown_column")
|
||||
if column in columns:
|
||||
# `SELECT "日期", "日期"`、`SELECT Key, key`:locate_cell 按第一次出现取,说不清是哪一个
|
||||
return _refuse("unparsed", "ambiguous_column")
|
||||
columns.append(column)
|
||||
# 结果列按位置对应第 i 个选取项,不按名字认(SQLite 返回的列名有时是原写法、有时是定义名,P4-SPEC 2.4)。
|
||||
# 个数必须相等;名字只作一致性核对:直接引用的列,SQLite 返回的名字按 name_key 一定等于那一列,
|
||||
# 不等说明这份结果不是这条 SQL 的,不下钻
|
||||
if len(result_columns) != len(columns) or any(
|
||||
name_key(r) != name_key(c) for r, c in zip(result_columns, columns)):
|
||||
return _refuse("unparsed", "unknown_column")
|
||||
return DirectSelect(table, alias, columns)
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# tables_in
|
||||
# ==========================================================================
|
||||
|
||||
#: 这些词后面紧跟的单引号串,SQLite 当表名读(`FROM '日客流'` 能执行,历史兼容)。IN:SQLite 文法 `expr IN nm`
|
||||
#: 的 nm 可以是字符串,`WHERE 日期 IN '单列表'` 读的是那张表(列数和左边对得上就能执行,行值也行)
|
||||
_TABLE_BEFORE_STRING = frozenset({"from", "join", "in"})
|
||||
#: 逗号后面的单引号串当表名读的子句:FROM 列表里(含 ON / USING 之后接着逗号连接)
|
||||
_TABLE_LIST_CLAUSES = frozenset({"from", "join", "on", "using"})
|
||||
#: 切换子句的词:tables_in 据此判断逗号后面是不是还在 FROM 列表里
|
||||
#: 不含 AS:`FROM a AS x, '时段客流'` 的逗号仍在 FROM 列表里
|
||||
_CLAUSE_WORDS = frozenset({
|
||||
"select", "from", "join", "where", "group", "having", "order", "limit", "on", "using", "window", "values",
|
||||
"union", "intersect", "except", "with",
|
||||
})
|
||||
|
||||
|
||||
def tables_in(sql: str, tables: dict[str, list[str]]) -> list[str]:
|
||||
"""SQL 里出现的冻结表结构的表名(原样),按第一次出现的顺序去重。读法同 recognize,宁可多认(P4-SPEC 2.4)。
|
||||
|
||||
每个带引号的名字、代码里的每个词都按 name_key 对表名:列名或别名恰好和某张表同名时会多出这张表,后果只是
|
||||
摘录里多一行说明;漏认的后果是裁判拿不到「原表写明的合计,不要相加」。不用 2 版的 `sql_tables`:它的读法和
|
||||
SQLite 不同,藏在 `ESCAPE '\\'` 后面的合计表会漏掉。
|
||||
|
||||
字符串一般不算(`WHERE 分区 = '日客流'` 不是读表),只有 SQLite 会把它当表名读的位置才算:紧跟 FROM / JOIN /
|
||||
IN、紧跟「.」(`main.'日客流'`),以及 FROM 列表里的逗号、括号之后。「紧跟」按 SQLite 的分词算:中间隔着的空白
|
||||
(含记号开头的 U+FEFF、空白后面的竖制表符)不算记号。注释里的不算:SQLite 不读注释。
|
||||
"""
|
||||
index = _index(tables)
|
||||
found: dict[str, None] = {}
|
||||
lexes = _lex(sql, strict=False) or []
|
||||
clause: str | None = None
|
||||
prev: _Lex | None = None
|
||||
for lx in lexes:
|
||||
candidate: str | None = None
|
||||
if lx.kind in ("word", "name"):
|
||||
candidate = lx.text
|
||||
elif lx.kind == "str" and prev is not None:
|
||||
pkw = _kw(prev)
|
||||
if (pkw in _TABLE_BEFORE_STRING or (prev.kind == "punct" and prev.text == ".")
|
||||
or (prev.kind == "punct" and prev.text in ",(" and clause in _TABLE_LIST_CLAUSES)):
|
||||
candidate = lx.text
|
||||
if candidate is not None:
|
||||
table = index.get(name_key(candidate))
|
||||
if table is not None:
|
||||
found.setdefault(table, None)
|
||||
kw = _kw(lx)
|
||||
if kw in _CLAUSE_WORDS:
|
||||
clause = kw
|
||||
prev = lx
|
||||
return list(found)
|
||||
@@ -54,6 +54,7 @@ from typing import Any, Literal, TypedDict
|
||||
from app.core.artifact_store import canonical_json, content_hash
|
||||
from app.core.artifact_store import load as load_artifact
|
||||
from app.data.names import FILLERS, name_key
|
||||
from app.data.provenance_types import DOC_PROVENANCE
|
||||
from app.data.tabular import UNSHAPED_NOTE
|
||||
from app.engine.expressions import CellError, cell_value, column_kind, locate_cell
|
||||
from app.engine.issuance import _tolerance, extract_numbers, number_allowance
|
||||
@@ -2600,8 +2601,11 @@ def compose_doc(
|
||||
if c.get("status") == "resolved" and c.get("alias")}
|
||||
kept = {a: e for a, e in catalog.items() if e.get("kind") not in ENTITY_KINDS or a in used}
|
||||
# entity_syntax:这份文档按哪一版实体语法组装,复核照它走(verify_doc)。在文档顶层,参与内容哈希
|
||||
# provenance:期 4 的文档标记(P4-SPEC 1.4)。带它的文档才有查询步骤的提示字段、推断来源、裁判摘录的新行;
|
||||
# 没有它的(期 4 之前组装的)一律按原样展示。写在顶层、参与内容哈希,事后补不上:补了哈希就变,封存链就断
|
||||
doc: dict[str, Any] = {
|
||||
"schema": DOC_SCHEMA, "entity_syntax": snaps.syntax, "run_id": run_id, "node_id": node_id,
|
||||
"schema": DOC_SCHEMA, "entity_syntax": snaps.syntax, "provenance": DOC_PROVENANCE,
|
||||
"run_id": run_id, "node_id": node_id,
|
||||
"markdown": markdown, "source": source, "catalog": kept, "blocks": out_blocks,
|
||||
}
|
||||
checked = verify_doc(doc, catalog, allow_numbers=allow_numbers, cells_allowed=cells_allowed, loader=snaps)
|
||||
|
||||
@@ -9,7 +9,8 @@
|
||||
哪一格和那次查询的 SQL;查询给 SQL 前 300 字和被引用的行(最多 20 行,数据源遮罩的列不给;宽表先给
|
||||
被引用的列,再补满 12 列);表附上字段清单和类型(最多 60 个,遮罩的列不列,快照不完整要注明),
|
||||
字段附上类型和所属的表,两样都附上出现过的查询 SQL;知识库片段最多 600 字。快照一律经 loader 取
|
||||
(按需裁判传只认封存工件的那个)。规则里写明数字已经由系统核对过
|
||||
(按需裁判传只认封存工件的那个)。规则里写明数字已经由系统核对过。带期 4 文档标记的报告,上传表格的
|
||||
查询在 SQL 之后另有来历行(来源、口径、已接受、区域外文字,_provenance_lines)
|
||||
3. 调用(judge_doc):一份报告合并成一次调用,超过每批的句数就分批;结构化输出失败退回 JSON 解析,
|
||||
再失败就记未裁判。thinking 关掉,max_tokens 4096
|
||||
4. 预算:每份报告的句数、金额、时长和全局每日金额,每一项都能是 None(不限)。调用前按目录价估算,
|
||||
@@ -28,6 +29,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import time
|
||||
@@ -43,6 +45,8 @@ from app.core.artifact_store import canonical_json, content_hash
|
||||
from app.core.artifact_store import load as load_artifact
|
||||
from app.core.errors import describe_exception, not_configured
|
||||
from app.core.events import EventType
|
||||
# 只用标准库的契约模块:取文档标记的版本号。识别器、清单模块(direct_select、provenance)在函数体内导入(_provenance_lines)
|
||||
from app.data.provenance_types import DOC_PROVENANCE
|
||||
from app.data.tabular import UNSHAPED_NOTE
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Setting
|
||||
@@ -66,6 +70,8 @@ from app.engine.state import message_text
|
||||
from app.providers import catalog as pricing
|
||||
from app.providers.factory import ModelSpec, ProviderNotConfigured, get_chat_model
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 常量
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -328,7 +334,12 @@ def _shown_columns(columns: list[str], cited: Iterable[str]) -> list[str]:
|
||||
|
||||
|
||||
def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iterable[str], whole: bool,
|
||||
fetch: _Fetch, masked: dict[str, Any] | None) -> str | None:
|
||||
fetch: _Fetch, masked: dict[str, Any] | None, *, provenance_on: bool = False) -> str | None:
|
||||
"""一条查询的摘录:头、SQL、(期 4)来历行、列、被引用的行。
|
||||
|
||||
provenance_on:文档带期 4 标记时为真(prepare 算)。这时上传表格的查询快照(有 data_version)在 SQL 之后、「列:」
|
||||
之前多几行来历(_provenance_lines);其余情况一个字都不多,期 4 之前的文档、手工源的摘录和以前逐字相同。
|
||||
"""
|
||||
snapshot = fetch(entry.get("artifact"))
|
||||
if not isinstance(snapshot, dict) or not isinstance(snapshot.get("rows"), list):
|
||||
return None
|
||||
@@ -339,6 +350,8 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter
|
||||
sql = _clean(str(snapshot.get("sql") or ""))
|
||||
lines = [head, f"SQL:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"]
|
||||
hidden = _hidden(snapshot, source, masked)
|
||||
if provenance_on and snapshot.get("data_version"):
|
||||
lines += _provenance_lines(snapshot, fetch, hidden)
|
||||
columns = [str(c) for c in snapshot.get("columns") or [] if str(c).lower() not in hidden]
|
||||
shown_cols = _shown_columns(columns, cols)
|
||||
# 点名引用的行先给,整份引用的再从头补满,一共不超过 EXCERPT_ROWS 行
|
||||
@@ -361,6 +374,34 @@ def _query_excerpt(alias: str, entry: dict[str, Any], rows: set[int], cols: Iter
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _provenance_lines(snapshot: dict[str, Any], fetch: _Fetch, hidden: set[str]) -> list[str]:
|
||||
"""上传表格的查询快照的来历行(P4-SPEC 3.2、3.3):provenance.judge_lines 生成,这里只负责取冻结的表结构快照、
|
||||
算 SQL 用到的表(direct_select.tables_in,和执行这条 SQL 的 SQLite 同一种读法,宁可多认),把现成的遮罩交过去。
|
||||
|
||||
- 两个模块在函数体内导入:judge 被设置接口等地方导入,不能因此拉进整套配方模块;也只有带标记的文档、上传
|
||||
表格的快照才走到这里,期 4 之前的文档一次都不碰它们(测试用桩钉住);
|
||||
- 工件一律经同一个 fetch 取(按需裁判时是只认封存工件、带标记才扩展到哈希链上清单的那个 loader),取不到、
|
||||
哈希不符的都当没有:judge_lines 据此写「导入清单无法读取…」那一行,不中断摘录;
|
||||
- 确定:同样的工件永远得到同样的行(摘录进断点指纹)。生成过程中出了意外(工件形状不对)同样落到那一行,
|
||||
报告节点的裁判不因一份坏清单整体失败。
|
||||
"""
|
||||
from app.data.provenance import SOURCE_BROKEN, judge_lines
|
||||
from app.engine.direct_select import tables_in
|
||||
|
||||
schema_id = snapshot.get("schema_artifact")
|
||||
schema = fetch(schema_id) if schema_id else None
|
||||
schema = schema if isinstance(schema, dict) else None
|
||||
frozen = schema.get("tables") if schema is not None else None
|
||||
tables = {str(name): [str(c.get("name")) for c in (meta.get("columns") or []) if isinstance(c, dict)]
|
||||
for name, meta in frozen.items() if isinstance(meta, dict)} if isinstance(frozen, dict) else {}
|
||||
try:
|
||||
used = tables_in(str(snapshot.get("sql") or ""), tables) if tables else []
|
||||
return judge_lines(snapshot, schema, fetch, tables=used, hidden=hidden)
|
||||
except Exception: # noqa: BLE001 - 坏形状的工件只让这条查询少了来历,不让整份报告判不了
|
||||
logger.warning("judge: 生成来历行失败", exc_info=True)
|
||||
return [SOURCE_BROKEN]
|
||||
|
||||
|
||||
def _sql_line(label: str, sql: str) -> str:
|
||||
return f"{label}:{sql[:SQL_CHARS]}{'…' if len(sql) > SQL_CHARS else ''}"
|
||||
|
||||
@@ -557,7 +598,9 @@ def _retrieval_excerpt(alias: str, entry: dict[str, Any], picked: list[int], who
|
||||
|
||||
|
||||
def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _Fetch,
|
||||
masked: dict[str, Any] | None, catalog: dict[str, Any]) -> str | None:
|
||||
masked: dict[str, Any] | None, catalog: dict[str, Any], *, provenance_on: bool = False) -> str | None:
|
||||
"""一件证据的摘录。provenance_on 只交给查询摘录(_query_excerpt):指标摘录里「输入取自查询 Qn」的那几行不加
|
||||
来历(P4-SPEC 调整 1),表名、字段名的摘录照旧。"""
|
||||
kind = entry.get("kind")
|
||||
if kind == "metric":
|
||||
return _metric_excerpt(alias, entry, fetch, catalog, masked)
|
||||
@@ -567,7 +610,8 @@ def _excerpt(alias: str, entry: dict[str, Any], cands: list[Candidate], fetch: _
|
||||
if kind == "query":
|
||||
rows = {r for c in cands for a, r in c.rows if a == alias}
|
||||
cols = [col for c in cands for a, col in c.cols if a == alias]
|
||||
return _query_excerpt(alias, entry, rows, cols, any(alias in c.whole for c in cands), fetch, masked)
|
||||
return _query_excerpt(alias, entry, rows, cols, any(alias in c.whole for c in cands), fetch, masked,
|
||||
provenance_on=provenance_on)
|
||||
if kind == "retrieval":
|
||||
picked = [h for c in cands for a, h in c.hits if a == alias]
|
||||
return _retrieval_excerpt(alias, entry, picked, any(alias in c.whole for c in cands), fetch)
|
||||
@@ -666,14 +710,19 @@ def prepare(doc: dict[str, Any], catalog: dict[str, Any] | None = None, *, units
|
||||
"""裁判的输入:候选句和证据摘录。纯函数(快照经 loader 取,缺省读工件库),不调模型。
|
||||
|
||||
catalog 缺省用文档自己存的目录;masked = {数据源名: 要遮的列},和快照里记下的 mask_columns 合在一起。
|
||||
|
||||
摘录按文档标记分版本(P4-SPEC 1.4、3.3):带 `provenance: DOC_PROVENANCE` 的文档,上传表格的查询摘录多几行
|
||||
来历;没有标记的(期 4 之前组装的)和以前逐字相同,复用键、断点指纹都不变,已判过的句子照旧复用。
|
||||
"""
|
||||
catalog = catalog if catalog is not None else (doc.get("catalog") or {})
|
||||
cands = candidates(doc, units)
|
||||
fetch = _Fetch(loader)
|
||||
provenance_on = doc.get("provenance") == DOC_PROVENANCE
|
||||
excerpts: dict[str, str] = {}
|
||||
for alias in dict.fromkeys(a for c in cands for a in c.aliases):
|
||||
entry = catalog.get(alias)
|
||||
if isinstance(entry, dict) and (text := _excerpt(alias, entry, cands, fetch, masked, catalog)):
|
||||
if isinstance(entry, dict) and (text := _excerpt(alias, entry, cands, fetch, masked, catalog,
|
||||
provenance_on=provenance_on)):
|
||||
excerpts[alias] = text
|
||||
picked = {c.unit for c in cands}
|
||||
skipped: dict[str, str] = {}
|
||||
|
||||
+6201
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,862 @@
|
||||
"""期 4 的 SQL 判据(P4-SPEC 2.4、6.3、7.2,WP-A):engine/direct_select 的 recognize、tables_in。
|
||||
|
||||
判据决定一个结果格能不能下钻到原表的格子。放过一条不是「一张表的直接选取」的 SQL,结果列装的可能是另一列的值,
|
||||
值恰好相等时界面就会指着错误的格子说「来自这里」——所以这里大半的用例是拒绝的写法,尤其是评审实测过的
|
||||
B7(`ESCAPE '\\'`、`AS [t']` 后面藏的 UNION ALL)。最后一组拿真的 SQLite 执行随机拼出的 SQL,核对「认可的
|
||||
每一列,值确实都来自它说的那一列」,不靠逐条想到的用例。
|
||||
|
||||
夹具一律假名(分区甲、分区乙、日客流、时段客流),数字随机。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import itertools
|
||||
import random
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.data import guard
|
||||
from app.data import provenance_types as P
|
||||
from app.data.provenance_types import DirectSelect, SelectRefusal
|
||||
from app.engine import direct_select as D
|
||||
from app.engine.direct_select import recognize, tables_in
|
||||
|
||||
TABLES: dict[str, list[str]] = {
|
||||
"日客流": ["日期", "全日客流", "分区甲", "分区乙"],
|
||||
"时段客流": ["日期", "时段", "时段类别", "客流"],
|
||||
"时段客流_表内合计": ["日期", "合计项", "客流"],
|
||||
# P4-SPEC 附录 B2:定义名是 Key,SQLite 对 `abc.key` 返回定义名
|
||||
"abc": ["Key", "v"],
|
||||
}
|
||||
DAILY = TABLES["日客流"]
|
||||
|
||||
|
||||
def ok(sql: str, result: list[str], *, tables: dict[str, list[str]] = TABLES) -> DirectSelect:
|
||||
out = recognize(sql, result_columns=result, tables=tables)
|
||||
assert isinstance(out, DirectSelect), f"应认可:{sql!r} → {out!r}"
|
||||
return out
|
||||
|
||||
|
||||
def no(sql: str, code: str, detail: str, *, result: list[str] | None = None,
|
||||
tables: dict[str, list[str]] = TABLES) -> SelectRefusal:
|
||||
out = recognize(sql, result_columns=result if result is not None else ["日期", "全日客流"], tables=tables)
|
||||
assert isinstance(out, SelectRefusal), f"应拒绝:{sql!r} → {out!r}"
|
||||
# 每个拒绝分支的细分都得在契约的 DETAILS 里:漏了映射,接口一侧的 contract_problems 会报
|
||||
assert out.detail in P.DETAILS[out.code], out
|
||||
assert (out.code, out.detail) == (code, detail), f"{sql!r} → {out!r}"
|
||||
return out
|
||||
|
||||
|
||||
def stored(sql: str) -> str:
|
||||
"""查询快照里存的就是守卫的返回值(engine.run_query):用例照这个形态喂给判据。"""
|
||||
return guard.check(sql, readonly=True, dialect="sqlite")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 原型的 24 例(P4-SPEC 2.4、附录 B4):期望照原型,改成 code 和 detail
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
PROTOTYPE_REFUSED = [
|
||||
('SELECT "日期", "分区甲" AS "分区乙" FROM "日客流"', "alias", "alias"),
|
||||
('SELECT "日期" "分区乙" FROM "日客流"', "alias", "alias"),
|
||||
('SELECT a."日期", b."全日客流" FROM "日客流" a JOIN "日客流" b ON a."日期" = b."日期"', "multi_table", "join"),
|
||||
('SELECT a."日期", b."全日客流" FROM "日客流" a, "日客流" b WHERE a."日期" = b."日期"', "multi_table",
|
||||
"comma_join"),
|
||||
('WITH x AS (SELECT * FROM "日客流") SELECT "日期", "全日客流" FROM x', "multi_table", "cte"),
|
||||
('SELECT "日期", "全日客流" FROM (SELECT * FROM "日客流")', "multi_table", "subquery"),
|
||||
('SELECT "日期", "全日客流" FROM "日客流" UNION SELECT "日期", "分区甲" FROM "日客流"', "multi_table", "compound"),
|
||||
('SELECT "日期", SUM("全日客流") OVER () FROM "日客流"', "expression", "window"),
|
||||
('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" IN (SELECT "日期" FROM "时段客流")', "multi_table",
|
||||
"subquery"),
|
||||
('SELECT SUM("全日客流") AS "合计", COUNT(*) FROM "日客流"', "expression", "expression"),
|
||||
('SELECT "日期", "全日客流" + 0 FROM "日客流"', "expression", "expression"),
|
||||
("SELECT '日期', \"全日客流\" FROM \"日客流\"", "expression", "string_literal"),
|
||||
('SELECT DISTINCT "日期", "全日客流" FROM "日客流"', "expression", "distinct"),
|
||||
# 表名后面的 INDEXED 不是别名,再往后不是 WHERE / ORDER / LIMIT
|
||||
('SELECT "日期", "全日客流" FROM "日客流" INDEXED BY x', "unparsed", "tail_shape"),
|
||||
('SELECT "日期", "全日客流" COLLATE NOCASE FROM "日客流"', "expression", "collate"),
|
||||
('SELECT "日期", "全日客流" FROM json_each(\'[]\')', "unparsed", "table_shape"),
|
||||
('SELECT "日期", "全日客流" FROM main."日客流"', "unparsed", "qualified_table"),
|
||||
('SELECT x."日期", x."全日客流" FROM "日客流" t', "unparsed", "qualifier"),
|
||||
]
|
||||
|
||||
PROTOTYPE_ACCEPTED = [
|
||||
('SELECT "日期", "全日客流" FROM "日客流" ORDER BY "全日客流" DESC LIMIT 1', ["日期", "全日客流"],
|
||||
("日客流", None, ["日期", "全日客流"])),
|
||||
("SELECT 日期, 全日客流 FROM 日客流 WHERE 日期 = '2026-08-05';", ["日期", "全日客流"],
|
||||
("日客流", None, ["日期", "全日客流"])),
|
||||
('SELECT t."日期", t."全日客流" FROM "日客流" AS t WHERE substr(t."日期", 1, 7) = \'2026-08\'',
|
||||
["日期", "全日客流"], ("日客流", "t", ["日期", "全日客流"])),
|
||||
('SELECT * FROM "日客流" WHERE "日期" = \'2026-08-05\'', DAILY, ("日客流", None, DAILY)),
|
||||
('-- FROM 别的表\nSELECT "日期", "客流" FROM "时段客流" /* JOIN x */ WHERE "时段" = \'8-9\'', ["日期", "客流"],
|
||||
("时段客流", None, ["日期", "客流"])),
|
||||
('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" = \'x UNION SELECT\'', ["日期", "全日客流"],
|
||||
("日客流", None, ["日期", "全日客流"])),
|
||||
]
|
||||
|
||||
|
||||
def test_the_prototype_has_24_cases():
|
||||
assert len(PROTOTYPE_REFUSED) + len(PROTOTYPE_ACCEPTED) == 24
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("sql", "code", "detail"), PROTOTYPE_REFUSED)
|
||||
def test_prototype_refusals(sql, code, detail):
|
||||
no(sql, code, detail)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("sql", "result", "expect"), PROTOTYPE_ACCEPTED)
|
||||
def test_prototype_acceptances(sql, result, expect):
|
||||
out = ok(sql, result)
|
||||
assert (out.table, out.alias, out.columns) == expect
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("sql", "result", "expect"), PROTOTYPE_ACCEPTED)
|
||||
def test_prototype_acceptances_in_the_stored_form(sql, result, expect):
|
||||
"""守卫去掉注释和尾分号之后(存进查询快照的形态)结论相同。"""
|
||||
out = ok(stored(sql), result)
|
||||
assert (out.table, out.alias, out.columns) == expect
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# B7:多方言扫描看不见、SQLite 照样执行的集合运算(评审甲的 bs.py、br.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
B7_COMPOUND = [
|
||||
# bs.py 的 3 条:ESCAPE '\' 在多方言扫描里是「转义了的引号」,后面整段被当成字符串
|
||||
"""SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" LIKE '2026-08%' ESCAPE '\\' UNION ALL """
|
||||
"""SELECT "日期", "分区甲" FROM "日客流" WHERE "日期" LIKE '2026-09%'""",
|
||||
"""SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" LIKE '%\\_%' ESCAPE '\\' OR 1 UNION ALL """
|
||||
"""SELECT "日期", "分区甲" FROM "日客流" WHERE '1' = '1'""",
|
||||
"""SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" = '\\' UNION ALL """
|
||||
"""SELECT "日期", "分区甲" FROM "日客流" WHERE '1'='1'""",
|
||||
# br.py 的第 1 条:方括号里含引号,多方言扫描不当标识符,[t'] … [u'] 被当成一段
|
||||
"""SELECT "日期", "全日客流" FROM "日客流" AS [t'] WHERE "日期" > '2026-09' UNION ALL """
|
||||
"""SELECT "日期", "分区甲" FROM "日客流" AS [u']""",
|
||||
# br.py 的第 2 条
|
||||
"""SELECT "日期", "全日客流" FROM "日客流" AS t ORDER BY [x'] UNION ALL """
|
||||
"""SELECT "日期","分区甲" FROM "日客流" ORDER BY [y']""",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql", B7_COMPOUND)
|
||||
def test_b7_compound_hidden_behind_backslash_or_bracket_is_refused(sql):
|
||||
no(stored(sql), "multi_table", "compound")
|
||||
|
||||
|
||||
def test_b7_join_hidden_behind_escape_backslash_is_refused():
|
||||
no(stored("""SELECT a."日期", a."全日客流" LIKE '%' ESCAPE '\\', a."分区甲" FROM "日客流" a """
|
||||
"""JOIN "日客流" b ON a."日期" = b."日期\""""), "multi_table", "join")
|
||||
no(stored("""SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" LIKE '%' ESCAPE '\\' """
|
||||
"""CROSS JOIN "时段客流\""""), "multi_table", "join")
|
||||
|
||||
|
||||
def test_b7_comma_join_after_escape_backslash_is_refused():
|
||||
# br.py 的第 3 条
|
||||
no(stored("""SELECT a."日期", b."全日客流" FROM "日客流" a, "日客流" b WHERE a."日期" = b."日期" """
|
||||
"""AND a."日期" LIKE '%' ESCAPE '\\' """), "multi_table", "comma_join")
|
||||
|
||||
|
||||
def test_b7_would_slip_through_the_multi_dialect_scan():
|
||||
"""反证:同一串 SQL 在多方言扫描(evidence._sql_scan 的读法)里看不见 UNION,判据换回那种读法就会放过它。
|
||||
这条钉住「判据必须用 SQLite 读法」的理由,换读法的变异由上面两条抓。"""
|
||||
from app.engine.evidence import _SQL, _sql_scan
|
||||
|
||||
code, _ = _sql_scan(stored(B7_COMPOUND[0]), _SQL[2])
|
||||
assert "UNION" not in code.upper()
|
||||
seen = " ".join(t.text for t in guard._tokens(stored(B7_COMPOUND[0]), guard._SQLITE) if t.kind == "code")
|
||||
assert "UNION" in seen.upper()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 守卫读法的三个性质(P4-SPEC 2.4):守卫以后改读法时这里先失败
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _kinds(sql: str) -> list[tuple[str, str]]:
|
||||
return [(t.kind, t.text) for t in guard._tokens(sql, guard._SQLITE)]
|
||||
|
||||
|
||||
def test_sqlite_reading_backslash_does_not_escape_in_strings():
|
||||
assert _kinds("'a\\' x") == [("string", "'a\\'"), ("code", " x")]
|
||||
|
||||
|
||||
def test_sqlite_reading_brackets_are_identifiers():
|
||||
assert _kinds("[t'] x") == [("ident", "[t']"), ("code", " x")]
|
||||
|
||||
|
||||
def test_sqlite_reading_brackets_end_at_the_first_closing_bracket():
|
||||
assert _kinds("[a]]b]") == [("ident", "[a]"), ("code", "]b]")]
|
||||
assert _kinds('[a""b]') == [("ident", '[a""b]')]
|
||||
|
||||
|
||||
def test_bracket_names_are_taken_verbatim_and_doubled_quotes_collapse_elsewhere():
|
||||
tables = {'a""b': ["x"], 'c"d': ["y"], "e`f": ["z"]}
|
||||
assert ok('SELECT x FROM [a""b]', ["x"], tables=tables).table == 'a""b'
|
||||
assert ok('SELECT y FROM "c""d"', ["y"], tables=tables).table == 'c"d'
|
||||
assert ok("SELECT z FROM `e``f`", ["z"], tables=tables).table == "e`f"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 字面量关键字、重名、位置对应
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("word", ["CURRENT_DATE", "current_time", "Current_Timestamp", "TRUE", "false", "NULL"])
|
||||
def test_literal_keywords_are_refused(word):
|
||||
no(f'SELECT {word}, "全日客流" FROM "日客流"', "expression", "literal_keyword")
|
||||
no(f'SELECT "日期", {word} FROM "日客流"', "expression", "literal_keyword")
|
||||
|
||||
|
||||
def test_literal_keywords_are_refused_even_when_the_table_has_such_a_column():
|
||||
# TRUE / FALSE 有同名列时才指那一列:读法取决于表结构,一律拒
|
||||
tables = {"t": ["true", "x"]}
|
||||
no("SELECT TRUE, x FROM t", "expression", "literal_keyword", result=["true", "x"], tables=tables)
|
||||
# 加了引号就是名字
|
||||
assert ok('SELECT "true", x FROM t', ["true", "x"], tables=tables).columns == ["true", "x"]
|
||||
|
||||
|
||||
def test_two_items_naming_the_same_column_are_ambiguous():
|
||||
no('SELECT "日期", "日期" FROM "日客流"', "unparsed", "ambiguous_column", result=["日期", "日期"])
|
||||
no("SELECT Key, key FROM abc", "unparsed", "ambiguous_column", result=["Key", "key"])
|
||||
no('SELECT t."日期", "日期" FROM "日客流" t', "unparsed", "ambiguous_column", result=["日期", "日期"])
|
||||
|
||||
|
||||
def test_result_columns_follow_the_items_by_position():
|
||||
"""第 i 个结果列就是第 i 个选取项(P4-SPEC 2.4)。SQLite 对 `abc.key` 返回定义名 Key(附录 B2)。"""
|
||||
out = ok("SELECT abc.key, ABC.V FROM abc", ["Key", "v"])
|
||||
assert out.columns == ["Key", "v"]
|
||||
out = ok('SELECT "分区乙", "日期", "分区甲" FROM "日客流"', ["分区乙", "日期", "分区甲"])
|
||||
assert out.columns == ["分区乙", "日期", "分区甲"]
|
||||
|
||||
|
||||
def test_result_columns_that_do_not_line_up_with_the_items_are_refused():
|
||||
"""名字只作一致性核对:结果列的名字和对应位置的列对不上,或者个数不等,说明结果不是这条 SQL 的。
|
||||
按名字对应(变异)会把第一条认成 ['分区甲', '日期']。"""
|
||||
no('SELECT "日期", "分区甲" FROM "日客流"', "unparsed", "unknown_column", result=["分区甲", "日期"])
|
||||
no('SELECT "日期" FROM "日客流"', "unparsed", "unknown_column", result=["日期", "全日客流"])
|
||||
no('SELECT "日期", "全日客流" FROM "日客流"', "unparsed", "unknown_column", result=["日期"])
|
||||
|
||||
|
||||
def test_star_must_match_the_table_columns_in_order():
|
||||
assert ok('SELECT * FROM "日客流"', DAILY).columns == DAILY
|
||||
# 结果列的大小写不同也认(name_key),返回的是定义名
|
||||
assert ok("SELECT * FROM abc", ["key", "V"]).columns == ["Key", "v"]
|
||||
no('SELECT * FROM "日客流"', "unparsed", "star_mismatch", result=["全日客流", "日期", "分区甲", "分区乙"])
|
||||
no('SELECT * FROM "日客流"', "unparsed", "star_mismatch", result=DAILY[:3])
|
||||
no('SELECT * FROM "日客流"', "unparsed", "star_mismatch", result=[*DAILY, "多一列"])
|
||||
|
||||
|
||||
def test_star_is_only_allowed_alone():
|
||||
no('SELECT *, "日期" FROM "日客流"', "unparsed", "star_mismatch", result=[*DAILY, "日期"])
|
||||
no('SELECT "日期", * FROM "日客流"', "unparsed", "star_mismatch", result=["日期", *DAILY])
|
||||
no('SELECT t.* FROM "日客流" t', "unparsed", "star_mismatch", result=DAILY)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 名字:引号、中文、大小写、空白
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_unquoted_chinese_names():
|
||||
out = ok("SELECT 日期, 分区甲 FROM 日客流 WHERE 分区甲 > 3 ORDER BY 日期", ["日期", "分区甲"])
|
||||
assert (out.table, out.columns) == ("日客流", ["日期", "分区甲"])
|
||||
|
||||
|
||||
def test_backticks_and_brackets():
|
||||
out = ok("SELECT `日期`, [全日客流] FROM [日客流] AS `t` WHERE [t].`日期` = '2026-08-01'", ["日期", "全日客流"])
|
||||
assert (out.table, out.alias, out.columns) == ("日客流", "t", ["日期", "全日客流"])
|
||||
|
||||
|
||||
def test_names_are_case_insensitive_only_for_ascii():
|
||||
out = ok("SeLeCt ABC.KEY, v FrOm Abc", ["Key", "v"])
|
||||
assert (out.table, out.columns) == ("abc", ["Key", "v"])
|
||||
# 别名同样只对 ASCII 不区分大小写;返回的别名是原写法
|
||||
out = ok('SELECT "T".日期 FROM 日客流 AS t', ["日期"])
|
||||
assert out.alias == "t"
|
||||
cyr = {"Дата": ["Сумма", "Итог"]}
|
||||
no("SELECT Сумма FROM дата", "unparsed", "unknown_table", result=["Сумма"], tables=cyr)
|
||||
no("SELECT сумма FROM Дата", "unparsed", "unknown_column", result=["сумма"], tables=cyr)
|
||||
assert ok("SELECT Сумма, Итог FROM Дата", ["Сумма", "Итог"], tables=cyr).columns == ["Сумма", "Итог"]
|
||||
|
||||
|
||||
def test_fullwidth_names_are_not_folded():
|
||||
# name_key 不做 NFKC:全角 ABC 和 abc 是两张表
|
||||
no("SELECT Key FROM abc", "unparsed", "unknown_table", result=["Key"])
|
||||
|
||||
|
||||
def test_newlines_tabs_and_form_feeds_are_whitespace():
|
||||
sql = 'SELECT\n\t"日期",\r\n\t"全日客流"\fFROM\t"日客流"\nWHERE "日期" = \'2026-08-01\'\n'
|
||||
assert ok(sql, ["日期", "全日客流"]).columns == ["日期", "全日客流"]
|
||||
|
||||
|
||||
def test_fullwidth_space_is_part_of_a_name():
|
||||
"""除记号开头的 U+FEFF 之外,SQLite 把码位 ≥ 0x80 的字符一律算名字(2 版实体语法同样如此,evidence._SqlSyntax):
|
||||
`日客流\u3000t` 是一张叫这个名字的表,不是表名加别名;`日客流\u3000WHERE` 也不是表名加 WHERE。"""
|
||||
no("SELECT 日期, 全日客流 FROM 日客流\u3000t", "unparsed", "unknown_table")
|
||||
no("SELECT 日期, 全日客流 FROM 日客流\u3000WHERE", "unparsed", "unknown_table")
|
||||
# 「日客流\u3000WHERE」是表名,「日期」成了它的别名,后面的「=」不是尾部的开头
|
||||
no("SELECT 日期, 全日客流 FROM 日客流\u3000WHERE 日期 = 'x'", "unparsed", "tail_shape")
|
||||
tables = {"日客流": ["日期\u3000x", "全日客流"]}
|
||||
assert ok("SELECT 日期\u3000x FROM 日客流", ["日期\u3000x"], tables=tables).columns == ["日期\u3000x"]
|
||||
# 不换行空格同理:「日期\u00a0FROM」是一个名字,「日客流」成了它的别名,整条语句没有 FROM
|
||||
no("SELECT 日期\u00a0FROM 日客流", "alias", "alias", result=["日客流"])
|
||||
|
||||
|
||||
def test_vertical_tab_reads_as_whitespace():
|
||||
"""竖制表符出现在记号开头时 SQLite 报非法记号,紧跟其他空白时 SQLite 当空白(CC_SPACE 的循环用 sqlite3Isspace,
|
||||
含 \\v)。判据一律当空白:前一种执行不了、不会留下查询快照;后一种读法相同,藏在它后面的 UNION ALL 照样看得见。
|
||||
当成标点时,紧跟在它后面的单引号表名 tables_in 会漏(见 tables_in 一节)。"""
|
||||
no(stored('SELECT "日期", "全日客流" FROM "日客流" WHERE 1 \x0bUNION ALL SELECT "日期", "分区甲" FROM "日客流"'),
|
||||
"multi_table", "compound")
|
||||
no(stored('SELECT "日期", "全日客流" FROM "日客流" WHERE 1 \x0bGROUP BY "日期"'), "expression", "expression")
|
||||
out = ok(stored('SELECT \x0b"日期", \x0b"全日客流" FROM \x0b"日客流" \x0bWHERE 1'), ["日期", "全日客流"])
|
||||
assert (out.table, out.alias, out.columns) == ("日客流", None, ["日期", "全日客流"])
|
||||
conn = sqlite3.connect(":memory:")
|
||||
with pytest.raises(sqlite3.Error):
|
||||
conn.execute("SELECT\x0b1")
|
||||
assert conn.execute("SELECT 1 \x0bUNION ALL SELECT 2").fetchall() == [(1,), (2,)]
|
||||
|
||||
|
||||
BOM = "\ufeff"
|
||||
|
||||
#: 评审实测(bom.py、attacks.py):守卫照样放行,SQLite 照样执行,前两条的全日客流一列里混着分区甲的值
|
||||
BOM_HIDDEN = [
|
||||
(f'SELECT "日期", "全日客流" FROM "日客流" WHERE 1 {BOM}UNION ALL {BOM}SELECT "日期", "分区甲" {BOM}FROM "日客流"',
|
||||
"multi_table", "compound"),
|
||||
(f'SELECT "日期", "全日客流" FROM "日客流" WHERE 1 {BOM}UNION {BOM}SELECT "日期", "分区甲" {BOM}FROM "日客流"',
|
||||
"multi_table", "compound"),
|
||||
(f'SELECT "日期", "全日客流" FROM "日客流" WHERE 1 {BOM}GROUP BY "日期"', "expression", "expression"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("sql", "code", "detail"), BOM_HIDDEN)
|
||||
def test_bom_at_a_token_start_is_whitespace_so_the_keyword_behind_it_counts(sql, code, detail):
|
||||
"""U+FEFF 出现在记号开头时 SQLite 当空白(tokenize.c 的 CC_BOM)。照「码位 ≥ 0x80 一律算名字」读,紧跟在它后面的
|
||||
UNION 就成了一个普通的词,判据把这几条认成直接选取。"""
|
||||
no(stored(sql), code, detail)
|
||||
|
||||
|
||||
def test_bom_between_tokens_reads_like_a_space():
|
||||
"""记号开头指:紧跟在空白、标点、带引号的名字、字符串后面。紧跟在不加引号的词后面就进了那个词(`SELECT\ufeff`
|
||||
是一个名字,SQLite 报语法错误),见下一条。"""
|
||||
sql = stored(f'SELECT {BOM}"日期",{BOM}"全日客流" {BOM}FROM {BOM}"日客流"{BOM} WHERE 1')
|
||||
out = ok(sql, ["日期", "全日客流"])
|
||||
assert (out.table, out.alias, out.columns) == ("日客流", None, ["日期", "全日客流"])
|
||||
conn = sqlite3.connect(":memory:")
|
||||
conn.execute('CREATE TABLE "日客流" ("日期", "全日客流")')
|
||||
assert [d[0] for d in conn.execute(sql).description] == ["日期", "全日客流"]
|
||||
with pytest.raises(sqlite3.Error):
|
||||
conn.execute(f'SELECT{BOM} "日期" FROM "日客流"')
|
||||
# 紧跟在带引号的名字、标点后面,单独一个 U+FEFF 也是记号开头
|
||||
no(stored(f'SELECT "日期", "全日客流" FROM "日客流"{BOM}UNION ALL SELECT "日期", "分区甲" FROM "日客流"'),
|
||||
"multi_table", "compound")
|
||||
no(stored(f'SELECT "日期",{BOM}"分区甲"{BOM}AS "全日客流" FROM "日客流"'), "alias", "alias")
|
||||
|
||||
|
||||
def test_bom_inside_a_name_is_part_of_the_name():
|
||||
"""名字中间(含末尾)的 U+FEFF 是名字的一部分,同 SQLite:`日客流\ufeff` 是另一张表,`UNION\ufeff` 不是关键字。"""
|
||||
tables = {"日客流": [f"日{BOM}期", "全日客流"]}
|
||||
assert ok(f"SELECT 日{BOM}期 FROM 日客流", [f"日{BOM}期"], tables=tables).columns == [f"日{BOM}期"]
|
||||
no(f"SELECT 全日客流 FROM 日客流{BOM}", "unparsed", "unknown_table", result=["全日客流"], tables=tables)
|
||||
# 「日期\ufeffFROM」是一个名字,「日客流」成了它的别名,整条语句没有 FROM
|
||||
no(f"SELECT 日期{BOM}FROM 日客流", "alias", "alias", result=["日客流"])
|
||||
conn = sqlite3.connect(":memory:")
|
||||
conn.execute(f'CREATE TABLE "日客流" ("日{BOM}期", "全日客流")')
|
||||
conn.execute("INSERT INTO 日客流 VALUES (1, 2)")
|
||||
assert conn.execute(f"SELECT 日{BOM}期 FROM 日客流").fetchall() == [(1,)]
|
||||
with pytest.raises(sqlite3.Error):
|
||||
conn.execute(f"SELECT 全日客流 FROM 日客流{BOM}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 语句个数、注释、字符串
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_trailing_semicolons_are_allowed():
|
||||
assert ok('SELECT "日期" FROM "日客流";', ["日期"]).columns == ["日期"]
|
||||
assert ok('SELECT "日期" FROM "日客流" ; ; ', ["日期"]).columns == ["日期"]
|
||||
|
||||
|
||||
def test_two_statements_are_refused():
|
||||
no('SELECT "日期" FROM "日客流"; SELECT "日期" FROM "时段客流"', "unparsed", "multi_statement", result=["日期"])
|
||||
no('SELECT "日期" FROM "日客流"; DROP TABLE x', "unparsed", "multi_statement", result=["日期"])
|
||||
|
||||
|
||||
def test_keywords_inside_comments_and_strings_do_not_count():
|
||||
sql = ('SELECT "日期", "全日客流" /* JOIN "时段客流" */ FROM "日客流" -- UNION ALL SELECT\n'
|
||||
"WHERE \"日期\" <> 'UNION SELECT x FROM y JOIN z' AND \"日期\" <> 'GROUP BY'")
|
||||
assert ok(sql, ["日期", "全日客流"]).columns == ["日期", "全日客流"]
|
||||
|
||||
|
||||
def test_keywords_inside_quoted_names_do_not_count():
|
||||
tables = {"union": ["select", "join"]}
|
||||
out = ok('SELECT "select", [join] FROM "union"', ["select", "join"], tables=tables)
|
||||
assert (out.table, out.columns) == ("union", ["select", "join"])
|
||||
|
||||
|
||||
def test_comments_split_words_like_sqlite_does():
|
||||
# SQLite 把注释当空白:SEL/**/ECT 是两个词,不是 SELECT
|
||||
no('SEL/**/ECT "日期" FROM "日客流"', "unparsed", "not_select", result=["日期"])
|
||||
|
||||
|
||||
def test_unterminated_quotes_are_refused():
|
||||
no('SELECT "日期" FROM "日客流', "unparsed", "not_select")
|
||||
# 反例:`"日期 FROM "` 是一个闭合的名字,后面的「日客流」是它的别名
|
||||
no('SELECT "日期 FROM "日客流', "alias", "alias")
|
||||
no("SELECT 日期 FROM 日客流 WHERE 日期 = 'x", "unparsed", "not_select")
|
||||
no("SELECT [日期 FROM 日客流", "unparsed", "not_select")
|
||||
|
||||
|
||||
def test_empty_and_non_select_statements_are_refused():
|
||||
for sql in ("", " ", ";", "VALUES (1)", "PRAGMA table_info(日客流)", "EXPLAIN SELECT 日期 FROM 日客流",
|
||||
"(SELECT 日期 FROM 日客流)"):
|
||||
no(sql, "unparsed", "not_select")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 选取项、FROM、尾部的其余写法
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_where_subquery_and_values_are_refused():
|
||||
no('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" IN (SELECT "日期" FROM "日客流")', "multi_table",
|
||||
"subquery")
|
||||
no('SELECT "日期", "全日客流" FROM "日客流" WHERE EXISTS (SELECT 1)', "multi_table", "subquery")
|
||||
no('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" IN (VALUES (\'a\'))', "multi_table", "subquery")
|
||||
no('SELECT "日期", (SELECT 1) FROM "日客流"', "multi_table", "subquery")
|
||||
|
||||
|
||||
def test_join_family_words_are_refused_wherever_they_appear():
|
||||
for word in ("JOIN", "NATURAL JOIN", "CROSS JOIN", "INNER JOIN", "LEFT JOIN", "LEFT OUTER JOIN", "RIGHT JOIN",
|
||||
"FULL JOIN"):
|
||||
no(f'SELECT a."日期", a."全日客流" FROM "日客流" a {word} "时段客流" b', "multi_table", "join")
|
||||
|
||||
|
||||
def test_compound_and_cte_words_are_refused():
|
||||
for word in ("UNION", "UNION ALL", "INTERSECT", "EXCEPT"):
|
||||
no(f'SELECT "日期", "全日客流" FROM "日客流" {word} SELECT "日期", "分区甲" FROM "日客流"', "multi_table",
|
||||
"compound")
|
||||
no('WITH RECURSIVE x(n) AS (SELECT 1) SELECT "日期", "全日客流" FROM "日客流"', "multi_table", "cte")
|
||||
no('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" IN (WITH x AS (SELECT 1) SELECT * FROM x)',
|
||||
"multi_table", "cte")
|
||||
|
||||
|
||||
def test_window_clause_is_refused():
|
||||
no('SELECT "日期", "全日客流" FROM "日客流" WINDOW w AS (ORDER BY "日期")', "expression", "window")
|
||||
|
||||
|
||||
def test_aggregation_in_the_tail_is_refused():
|
||||
no('SELECT "日期", "全日客流" FROM "日客流" GROUP BY "日期"', "expression", "expression")
|
||||
no('SELECT "日期", "全日客流" FROM "日客流" WHERE 1 GROUP BY "日期" HAVING 1', "expression", "expression")
|
||||
|
||||
|
||||
def test_is_distinct_from_in_the_tail_is_outside_this_grammar():
|
||||
no('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" IS DISTINCT FROM NULL', "unparsed", "tail_shape")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("item", "detail"), [
|
||||
('CASE WHEN 1 THEN "全日客流" END', "expression"),
|
||||
('CAST("全日客流" AS TEXT)', "expression"),
|
||||
('"全日客流" IS NULL', "expression"),
|
||||
('"全日客流" BETWEEN 1 AND 2', "expression"),
|
||||
('-"全日客流"', "expression"),
|
||||
("+全日客流", "expression"),
|
||||
("(全日客流)", "expression"),
|
||||
("全日客流 || ''", "expression"),
|
||||
("x'00'", "expression"),
|
||||
("1", "expression"),
|
||||
("1.5", "expression"),
|
||||
('abs("全日客流")', "expression"),
|
||||
('"abs"("全日客流")', "expression"),
|
||||
("'全日客流'", "string_literal"),
|
||||
('"全日客流" COLLATE BINARY', "collate"),
|
||||
])
|
||||
def test_items_that_are_not_bare_columns(item, detail):
|
||||
no(f'SELECT "日期", {item} FROM "日客流"', "expression", detail)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("item", [
|
||||
'"全日客流" AS "分区甲"', '"全日客流" AS x', '"全日客流" "分区甲"', "全日客流 分区甲", "\"全日客流\" '分区甲'",
|
||||
'"全日客流" [分区甲]', '"全日客流" `分区甲`', "t.全日客流 x",
|
||||
])
|
||||
def test_renamed_items_are_aliases(item):
|
||||
no(f'SELECT "日期", {item} FROM "日客流" t', "alias", "alias")
|
||||
|
||||
|
||||
def test_qualifier_must_name_this_table():
|
||||
assert ok('SELECT "日客流"."日期", 日客流.全日客流 FROM "日客流"', ["日期", "全日客流"]).alias is None
|
||||
no('SELECT "时段客流"."日期", "全日客流" FROM "日客流"', "unparsed", "qualifier")
|
||||
no('SELECT a."日期", "全日客流" FROM "日客流" AS b', "unparsed", "qualifier")
|
||||
# 起了别名之后原表名不能再当限定名(SQLite 报 no such column),只认别名
|
||||
no('SELECT 日客流."日期", "全日客流" FROM "日客流" AS t', "unparsed", "qualifier")
|
||||
# 带库名的限定名
|
||||
no('SELECT main."日客流"."日期", "全日客流" FROM "日客流"', "unparsed", "qualifier")
|
||||
|
||||
|
||||
def test_table_shapes():
|
||||
no('SELECT "日期", "全日客流" FROM temp."日客流"', "unparsed", "qualified_table")
|
||||
no('SELECT "日期", "全日客流" FROM pragma_table_info(\'日客流\')', "unparsed", "table_shape")
|
||||
# FROM (日客流) 在 SQLite 里能执行(括起来的表列表),一律不认
|
||||
no('SELECT "日期", "全日客流" FROM ("日客流")', "multi_table", "subquery")
|
||||
# 字符串当表名、当表别名:SQLite 的历史兼容,这一版文法不认
|
||||
no("SELECT \"日期\", \"全日客流\" FROM '日客流'", "unparsed", "table_shape")
|
||||
no("SELECT \"日期\", \"全日客流\" FROM \"日客流\" AS 't'", "unparsed", "table_shape")
|
||||
no('SELECT "日期", "全日客流"', "unparsed", "table_shape")
|
||||
no('SELECT "日期", "全日客流" FROM', "unparsed", "table_shape")
|
||||
no('SELECT "日期", "全日客流" FROM WHERE', "unparsed", "table_shape")
|
||||
|
||||
|
||||
def test_tail_shapes():
|
||||
no('SELECT "日期", "全日客流" FROM "日客流" NOT INDEXED', "unparsed", "tail_shape")
|
||||
no('SELECT "日期", "全日客流" FROM "日客流" t u', "unparsed", "tail_shape")
|
||||
no("SELECT \"日期\", \"全日客流\" FROM \"日客流\" 't'", "unparsed", "tail_shape")
|
||||
no('SELECT "日期", "全日客流" FROM "日客流" t, "时段客流" u', "multi_table", "comma_join")
|
||||
no('SELECT "日期", "全日客流" FROM "日客流", "时段客流"', "multi_table", "comma_join")
|
||||
|
||||
|
||||
def test_accepted_tails():
|
||||
for tail in ("", " WHERE 1", " ORDER BY 2 DESC, 1", " LIMIT 3 OFFSET 1", " LIMIT 1, 2",
|
||||
" WHERE \"日期\" BETWEEN '2026-08-01' AND '2026-08-31' ORDER BY \"日期\" COLLATE NOCASE LIMIT 5",
|
||||
" WHERE \"日期\" LIKE '2026\\_%' ESCAPE '\\' AND \"分区甲\" IN (1, 2, 3)",
|
||||
" WHERE CAST(\"分区甲\" AS TEXT) GLOB '1*' OR \"分区乙\" IS NULL",
|
||||
" WHERE (CASE WHEN \"分区甲\" > 1 THEN 1 ELSE 0 END) = 1"):
|
||||
out = ok(f'SELECT "日期", "全日客流" FROM "日客流"{tail}', ["日期", "全日客流"])
|
||||
assert out.columns == ["日期", "全日客流"], tail
|
||||
|
||||
|
||||
def test_select_all_is_allowed():
|
||||
assert ok('SELECT ALL "日期" FROM "日客流"', ["日期"]).columns == ["日期"]
|
||||
assert ok('SELECT ALL * FROM "日客流"', DAILY).columns == DAILY
|
||||
|
||||
|
||||
def test_unknown_table_and_column():
|
||||
no('SELECT "日期" FROM "不存在"', "unparsed", "unknown_table", result=["日期"])
|
||||
no('SELECT "不存在" FROM "日客流"', "unparsed", "unknown_column", result=["不存在"])
|
||||
# rowid 不是冻结表结构里的列
|
||||
no('SELECT rowid, "日期" FROM "日客流"', "unparsed", "unknown_column", result=["rowid", "日期"])
|
||||
|
||||
|
||||
def test_colliding_names_in_the_schema_are_treated_as_unknown():
|
||||
# 冻结表结构里不该出现:说不清是哪一个,按找不到处理
|
||||
no("SELECT x FROM T", "unparsed", "unknown_table", result=["x"], tables={"t": ["x"], "T": ["x"]})
|
||||
no("SELECT x FROM t", "unparsed", "unknown_column", result=["x"], tables={"t": ["x", "X"]})
|
||||
|
||||
|
||||
def test_forbidden_words_are_the_spec_table():
|
||||
assert D.FORBIDDEN == {
|
||||
"select", "from", "join", "union", "intersect", "except", "with", "over", "window", "group", "having",
|
||||
"values", "natural", "cross", "inner", "left", "right", "full", "outer", "distinct", "recursive"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# tables_in(P4-SPEC 2.4)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_tables_in_sees_tables_hidden_behind_escape_backslash():
|
||||
sql = stored("""SELECT "日期", "客流" FROM "时段客流" WHERE "时段" LIKE '%' ESCAPE '\\' UNION ALL """
|
||||
"""SELECT "日期", "客流" FROM "时段客流_表内合计\"""")
|
||||
assert tables_in(sql, TABLES) == ["时段客流", "时段客流_表内合计"]
|
||||
# 2 版 sql_tables 用多方言读法,这里漏掉了合计表:这正是 tables_in 存在的理由
|
||||
from app.engine.evidence import sql_tables
|
||||
assert "时段客流_表内合计" not in sql_tables(sql)
|
||||
|
||||
|
||||
def test_tables_in_bracket_backtick_and_unquoted_chinese_names():
|
||||
assert tables_in("SELECT * FROM [时段客流] JOIN `日客流` USING (日期) JOIN 时段客流_表内合计", TABLES) == [
|
||||
"时段客流", "日客流", "时段客流_表内合计"]
|
||||
assert tables_in("SELECT * FROM [t'] UNION ALL SELECT * FROM 日客流", {**TABLES, "t'": ["x"]}) == [
|
||||
"t'", "日客流"]
|
||||
|
||||
|
||||
def test_tables_in_ignores_names_only_in_strings_and_comments():
|
||||
assert tables_in("SELECT * FROM 日客流 WHERE 分区 = '时段客流' -- 时段客流_表内合计", TABLES) == ["日客流"]
|
||||
assert tables_in("SELECT '时段客流', \"日期\" FROM 日客流 /* abc */", TABLES) == ["日客流"]
|
||||
|
||||
|
||||
def test_tables_in_counts_strings_where_sqlite_reads_them_as_table_names():
|
||||
# `FROM '时段客流'` 在 SQLite 里能执行(历史兼容):读到的就是这张表
|
||||
assert tables_in("SELECT 日期 FROM 日客流 UNION ALL SELECT 日期 FROM '时段客流'", TABLES) == [
|
||||
"日客流", "时段客流"]
|
||||
assert tables_in("SELECT a.日期 FROM 日客流 AS a, '时段客流' b", TABLES) == ["日客流", "时段客流"]
|
||||
assert tables_in("SELECT 日期 FROM main.'时段客流_表内合计'", TABLES) == ["时段客流_表内合计"]
|
||||
assert tables_in("SELECT 1 FROM 日客流 a JOIN '时段客流' b ON 1", TABLES) == ["日客流", "时段客流"]
|
||||
|
||||
|
||||
def test_tables_in_counts_a_string_right_after_in():
|
||||
"""SQLite 文法 `expr IN nm` 的 nm 可以是字符串:`日期 IN '单列表'` 读的是那张表(评审 attacks.py 的 in table string)。
|
||||
括号里的字符串是值列表,不算。"""
|
||||
tables = {**TABLES, "单列": ["日期"]}
|
||||
assert tables_in("SELECT 日期, 全日客流 FROM 日客流 WHERE 日期 IN '单列'", tables) == ["日客流", "单列"]
|
||||
assert tables_in("SELECT 日期 FROM 日客流 WHERE 日期 NOT IN '单列'", tables) == ["日客流", "单列"]
|
||||
assert tables_in("SELECT 日期 FROM 日客流 WHERE 日期 IN ('单列', '时段客流')", tables) == ["日客流"]
|
||||
|
||||
|
||||
def test_tables_in_sees_tables_behind_bom_and_vertical_tab():
|
||||
"""「紧跟」按 SQLite 的分词算:记号开头的 U+FEFF、紧跟其他空白的竖制表符都是空白,不是记号(评审 bom.py、vt.py)。
|
||||
读错了,紧跟在后面的合计表就漏掉,裁判拿不到「不要相加」的口径行。"""
|
||||
head = 'SELECT "日期", "全日客流" FROM "日客流" UNION ALL SELECT "日期", "客流" '
|
||||
expect = ["日客流", "时段客流_表内合计"]
|
||||
assert tables_in(stored(f"{head}{BOM}FROM {BOM}时段客流_表内合计"), TABLES) == expect
|
||||
assert tables_in(stored(f"{head}FROM {BOM}'时段客流_表内合计'"), TABLES) == expect
|
||||
assert tables_in(stored(f"{head}FROM \x0b'时段客流_表内合计'"), TABLES) == expect
|
||||
assert tables_in(stored(f"{head}FROM main.{BOM}'时段客流_表内合计'"), TABLES) == expect
|
||||
tables = {**TABLES, "单列": ["日期"]}
|
||||
assert tables_in(stored(f"SELECT 日期 FROM 日客流 WHERE 日期 IN {BOM}'单列'"), tables) == ["日客流", "单列"]
|
||||
# 名字中间的 U+FEFF 是名字的一部分:「时段客流_表内合计\ufeff」不是那张表,SQLite 也报找不到
|
||||
assert tables_in(f"{head}FROM 时段客流_表内合计{BOM}", TABLES) == ["日客流"]
|
||||
|
||||
|
||||
def test_tables_in_is_ordered_deduplicated_and_returns_schema_names():
|
||||
assert tables_in("select * from ABC, 日客流, abc, \"Abc\"", TABLES) == ["abc", "日客流"]
|
||||
assert tables_in("SELECT 1", TABLES) == []
|
||||
assert tables_in("", TABLES) == []
|
||||
|
||||
|
||||
def test_tables_in_errs_on_the_side_of_more():
|
||||
# 列名恰好和某张表同名时多出这张表:后果只是多一行说明
|
||||
tables = {"日客流": ["日期", "时段客流"], "时段客流": ["日期"]}
|
||||
assert tables_in('SELECT "时段客流" FROM "日客流"', tables) == ["时段客流", "日客流"]
|
||||
# 引号没闭合也照样认能切出来的部分
|
||||
assert tables_in('SELECT * FROM 日客流 WHERE x = "时段客流', TABLES) == ["日客流", "时段客流"]
|
||||
|
||||
|
||||
def test_tables_in_keeps_fullwidth_space_inside_the_name():
|
||||
assert tables_in("SELECT * FROM 日客流\u3000WHERE 1", TABLES) == []
|
||||
assert tables_in("SELECT * FROM 日客流 WHERE 1", TABLES) == ["日客流"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 模块边界、不抛异常
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_module_imports_only_guard_names_and_the_contract():
|
||||
"""P4-SPEC 7.2:判据是纯函数,只许导入守卫、names 和 provenance_types(加标准库),不依赖证据层、接口层。"""
|
||||
tree = ast.parse(Path(D.__file__).read_text(encoding="utf-8"))
|
||||
imported: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
imported |= {a.name for a in node.names}
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
imported.add(node.module or "")
|
||||
assert imported <= {"__future__", "collections.abc", "dataclasses", "app.data.guard", "app.data.names",
|
||||
"app.data.provenance_types"}, imported
|
||||
|
||||
|
||||
_PIECES = [
|
||||
"SELECT", "FROM", "WHERE", "ORDER", "BY", "LIMIT", "AS", "JOIN", "UNION", "ALL", "DISTINCT", "*", ",", ".", "(",
|
||||
")", ";", "'", '"', "[", "]", "`", "\\", "--", "/*", "*/", "日期", "全日客流", '"日客流"', "日客流", "t", "x'0'",
|
||||
"1", "NULL", "\u3000", "\n", " ", "COLLATE", "OVER", "abc", "KEY", "=", "'a''b'", '"a""b"', "[t']", "ESCAPE",
|
||||
]
|
||||
|
||||
|
||||
def test_recognize_never_raises_and_every_refusal_is_mapped():
|
||||
rng = random.Random(20261002)
|
||||
for _ in range(3000):
|
||||
sql = " ".join(rng.choice(_PIECES) for _ in range(rng.randint(0, 14)))
|
||||
if rng.random() < 0.5:
|
||||
sql = "SELECT " + sql
|
||||
result = rng.choice([["日期"], ["日期", "全日客流"], DAILY, [], ["Key", "v"]])
|
||||
out = recognize(sql, result_columns=result, tables=TABLES)
|
||||
assert isinstance(out, (DirectSelect, SelectRefusal)), sql
|
||||
if isinstance(out, SelectRefusal):
|
||||
assert out.detail in P.DETAILS[out.code], (sql, out)
|
||||
else:
|
||||
assert out.table in TABLES and all(c in TABLES[out.table] for c in out.columns), (sql, out)
|
||||
assert isinstance(tables_in(sql, TABLES), list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 拿真的 SQLite 核对:认可的每一列,值都来自它说的那一列
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DB_TABLES = {
|
||||
"日客流": ["日期", "全日客流", "分区甲", "分区乙"],
|
||||
"时段客流": ["日期", "时段", "客流"],
|
||||
"abc": ["Key", "v"],
|
||||
}
|
||||
|
||||
|
||||
def _database(rng: random.Random, tables: dict[str, list[str]] | None = None
|
||||
) -> tuple[sqlite3.Connection, dict[object, tuple[str, str]]]:
|
||||
"""每一格的值全库唯一:结果里的任何一个值都能反查出它来自哪张表的哪一列。DQS 关掉,同 engine。"""
|
||||
conn = sqlite3.connect(":memory:")
|
||||
if hasattr(conn, "setconfig"):
|
||||
conn.setconfig(sqlite3.SQLITE_DBCONFIG_DQS_DML, False)
|
||||
conn.setconfig(sqlite3.SQLITE_DBCONFIG_DQS_DDL, False)
|
||||
origin: dict[object, tuple[str, str]] = {}
|
||||
values = iter(rng.sample(range(100_000, 999_999), 200))
|
||||
for table, cols in (tables or _DB_TABLES).items():
|
||||
quoted = ", ".join(f'"{c}"' for c in cols)
|
||||
conn.execute(f'CREATE TABLE "{table}" ({quoted})')
|
||||
for day in range(1, 6):
|
||||
row = []
|
||||
for c in cols:
|
||||
v: object = f"2026-08-{day:02d}#{table}" if c == "日期" else next(values)
|
||||
origin[v] = (table, c)
|
||||
row.append(v)
|
||||
conn.execute(f'INSERT INTO "{table}" VALUES ({", ".join("?" * len(cols))})', row)
|
||||
return conn, origin
|
||||
|
||||
|
||||
_SELECTS = [
|
||||
'"日期", "全日客流"', '"日期", "分区甲"', "日期, 分区乙", 't."日期", t."全日客流"', "*", '"全日客流"',
|
||||
'"日期", "分区甲" AS "全日客流"', '"日期" "全日客流"', '"分区甲" "全日客流", "日期"', '"日期", "全日客流" + 0',
|
||||
'"日期", 分区甲 AS 全日客流', 'T."日期", "全日客流"', '"全日客流", "日期"', "[日期], `全日客流`",
|
||||
'"日期", (SELECT "分区甲")', '"日期", "全日客流" COLLATE NOCASE', '"日期", coalesce("全日客流", 0)',
|
||||
]
|
||||
_FROMS = [
|
||||
'"日客流"', '"日客流" t', '"日客流" AS t', "日客流 AS [t']", '"日客流" AS "T"', '"日客流" AS t, "日客流" u',
|
||||
'"日客流" t JOIN "日客流" u USING ("日期")', '(SELECT "日期", "分区甲" AS "全日客流", "分区乙", "分区甲" FROM "日客流")',
|
||||
'"日客流" NOT INDEXED', "'日客流'",
|
||||
]
|
||||
_TAILS = [
|
||||
"", " WHERE 1", " ORDER BY 1 DESC", " LIMIT 2", " WHERE \"日期\" LIKE '%' ESCAPE '\\'",
|
||||
" WHERE \"日期\" = '\\'", " ORDER BY [x']", " WHERE '\\' = '\\'",
|
||||
""" UNION ALL SELECT "日期", "分区甲" FROM "日客流\"""",
|
||||
""" WHERE "日期" LIKE '%' ESCAPE '\\' UNION ALL SELECT "日期", "分区甲" FROM "日客流\"""",
|
||||
""" AS [u'] UNION ALL SELECT "日期", "分区乙" FROM "日客流" AS [v']""",
|
||||
""" UNION SELECT "日期", "分区甲" FROM "日客流\"""",
|
||||
""" EXCEPT SELECT "日期", "分区甲" FROM "日客流" WHERE 0""",
|
||||
""" WHERE "日期" IN (SELECT "日期" FROM "时段客流")""",
|
||||
" GROUP BY \"日期\"", " /* UNION ALL SELECT 1 */", " -- JOIN x",
|
||||
]
|
||||
|
||||
|
||||
def _tables_read(conn: sqlite3.Connection, stmt: str) -> set[str]:
|
||||
"""SQLite 的授权回调报出的、这条语句读到的表:tables_in 必须一张不漏(宁可多认)。"""
|
||||
read: set[str] = set()
|
||||
|
||||
def authorize(action: int, table: str | None, *_: object) -> int:
|
||||
if action == sqlite3.SQLITE_READ and table and not table.startswith("sqlite_"):
|
||||
read.add(table)
|
||||
return sqlite3.SQLITE_OK
|
||||
|
||||
conn.set_authorizer(authorize)
|
||||
try:
|
||||
conn.execute(stmt).fetchall()
|
||||
finally:
|
||||
conn.set_authorizer(None)
|
||||
return read
|
||||
|
||||
|
||||
def _check_accepted_values(out: DirectSelect, rows: list[tuple], origin: dict[object, tuple[str, str]],
|
||||
stmt: str) -> None:
|
||||
for row in rows:
|
||||
for i, value in enumerate(row):
|
||||
assert origin.get(value) == (out.table, out.columns[i]), (
|
||||
f"{stmt!r}:第 {i} 列认作 {out.table}.{out.columns[i]},值 {value!r} 实际来自 {origin.get(value)}")
|
||||
|
||||
|
||||
def test_every_accepted_column_really_comes_from_the_column_it_names():
|
||||
"""随机拼 SQL,经守卫、在 SQLite 上真的执行;判据认可的,结果第 i 列的每个值都必须来自 DirectSelect 说的那张表的
|
||||
第 i 列。B7 那类写法(集合运算拼进另一列的值)一旦被放过,这里就会报出值的真实来处。授权回调报出的表,
|
||||
tables_in 也必须一张不漏。"""
|
||||
rng = random.Random(4242)
|
||||
conn, origin = _database(rng)
|
||||
executed = accepted = 0
|
||||
combos = list(itertools.product(_SELECTS, _FROMS, _TAILS))
|
||||
rng.shuffle(combos)
|
||||
for select, source, tail in combos:
|
||||
sql = f"SELECT {select} FROM {source}{tail}"
|
||||
try:
|
||||
stmt = stored(sql)
|
||||
cur = conn.execute(stmt)
|
||||
except (guard.SqlRejected, sqlite3.Error):
|
||||
continue
|
||||
executed += 1
|
||||
columns = [d[0] for d in cur.description]
|
||||
rows = cur.fetchall()
|
||||
read = _tables_read(conn, stmt)
|
||||
assert read <= set(tables_in(stmt, _DB_TABLES)), (stmt, read)
|
||||
out = recognize(stmt, result_columns=columns, tables=_DB_TABLES)
|
||||
if isinstance(out, SelectRefusal):
|
||||
assert out.detail in P.DETAILS[out.code], (stmt, out)
|
||||
continue
|
||||
accepted += 1
|
||||
assert len(out.columns) == len(columns), stmt
|
||||
_check_accepted_values(out, rows, origin, stmt)
|
||||
# 拼出来的大多数都能执行,其中有认可的、也有被拒的:这条测试确实在比对
|
||||
assert executed > 500 and 50 < accepted < executed, (executed, accepted)
|
||||
|
||||
|
||||
def test_b7_writings_really_return_another_columns_values_in_sqlite():
|
||||
"""反证上一条的比对有牙齿:B7 的写法在 SQLite 里确实把分区甲的值放进了全日客流那一列。"""
|
||||
rng = random.Random(7)
|
||||
conn, origin = _database(rng)
|
||||
for sql in B7_COMPOUND[:1] + B7_COMPOUND[3:4]:
|
||||
rows = conn.execute(stored(sql).replace("'2026-08%'", "'%'").replace("'2026-09%'", "'%'")
|
||||
.replace("> '2026-09'", "> ''")).fetchall()
|
||||
assert {origin[r[1]] for r in rows} == {("日客流", "全日客流"), ("日客流", "分区甲")}, sql
|
||||
|
||||
|
||||
#: 记号之间的间隔:把模板里的 {g} 换成它。紧跟在关键字、数字后面的、紧跟在带引号的名字和标点后面的都有,
|
||||
#: 记号开头和空白后面两种位置都覆盖到
|
||||
_GAP_TEMPLATES = [
|
||||
'SELECT "日期", "全日客流" FROM "日客流" WHERE "分区甲" > 0{g}UNION ALL{g}SELECT "日期", "分区甲"{g}FROM "日客流"',
|
||||
'SELECT "日期", "全日客流" FROM "日客流" WHERE "分区甲" > 0{g}UNION{g}SELECT "日期", "分区甲"{g}FROM "日客流"',
|
||||
# 限定名:间隔要是被 SQLite 读成了表别名,「"日客流"."日期"」就找不到列,不会被误当成空白
|
||||
'SELECT "日客流"."日期", "全日客流" FROM "日客流"{g}UNION ALL SELECT "日期", "分区乙" FROM "日客流"',
|
||||
'SELECT "日期", "全日客流" FROM "日客流" WHERE "分区甲" > 0{g}GROUP BY "日期"',
|
||||
'SELECT "日期", "全日客流" FROM "日客流" UNION ALL SELECT "日期", "客流" FROM{g}时段客流',
|
||||
"SELECT \"日期\", \"全日客流\" FROM \"日客流\" UNION ALL SELECT \"日期\", \"客流\" FROM{g}'时段客流'",
|
||||
"SELECT \"日期\", \"全日客流\" FROM \"日客流\" WHERE \"全日客流\" NOT IN{g}'单列'",
|
||||
"SELECT{g}\"日期\",{g}\"全日客流\"{g}FROM{g}\"日客流\"{g}WHERE{g}\"日期\" <> '2026-08-01#日客流'",
|
||||
]
|
||||
#: 候选字符:ASCII 控制字符和空格、str.isspace 认而 SQLite 不认的、U+FEFF 及其邻居
|
||||
_GAP_CHARS = [chr(c) for c in (*range(0x01, 0x21), 0x7F, 0x85, 0xA0, 0x1680, 0x200B, 0x2028, 0x2029, 0x3000, 0xFEFF,
|
||||
0xFFFE)]
|
||||
#: SQLite 当空白的:记号开头只有这六个;竖制表符只在紧跟其他空白时才是
|
||||
_SQLITE_SPACE = "\t\n\x0c\r \ufeff"
|
||||
|
||||
|
||||
def test_gaps_that_sqlite_reads_as_whitespace_read_the_same_here():
|
||||
"""拿真的 SQLite 核对分词:把记号之间的空格换成各种字符,SQLite 能执行的每一条——
|
||||
- 认可的,值都来自它说的那一列;授权回调报出的表 tables_in 一张不漏;
|
||||
- 结果、列名、字节码都和用空格的那条相同(SQLite 把它当空白读)时,判据的结论和 tables_in 也必须相同。
|
||||
最后核对「SQLite 当空白的间隔」恰好是 U+FEFF、五个 ASCII 空白和空白后面的竖制表符:SQLite 升级改了读法,
|
||||
这里先失败。评审的 U+FEFF、竖制表符两处都是这条能抓到的。"""
|
||||
tables = {**_DB_TABLES, "单列": ["日期"]}
|
||||
conn, origin = _database(random.Random(5), tables)
|
||||
gaps = sorted({*_GAP_CHARS, *(" " + c for c in _GAP_CHARS), *(c + " " for c in _GAP_CHARS)})
|
||||
|
||||
def run(stmt: str) -> tuple[list[str], list[tuple], list[tuple]]:
|
||||
cur = conn.execute(stmt)
|
||||
columns, rows = [d[0] for d in cur.description], cur.fetchall()
|
||||
# EXPLAIN 的最后一列是注释,会带出原文片段,只比操作码和五个操作数
|
||||
program = [tuple(r[1:7]) for r in conn.execute("EXPLAIN " + stmt).fetchall()]
|
||||
return columns, rows, program
|
||||
|
||||
whitespace: set[str] = set()
|
||||
executed_with: dict[str, int] = {}
|
||||
for g in gaps:
|
||||
executed = same = 0
|
||||
for template in _GAP_TEMPLATES:
|
||||
base_stmt = stored(template.format(g=" "))
|
||||
try:
|
||||
stmt = stored(template.format(g=g))
|
||||
columns, rows, program = run(stmt)
|
||||
except (guard.SqlRejected, sqlite3.Error):
|
||||
continue
|
||||
executed += 1
|
||||
out = recognize(stmt, result_columns=columns, tables=tables)
|
||||
if isinstance(out, DirectSelect):
|
||||
_check_accepted_values(out, rows, origin, stmt)
|
||||
found = tables_in(stmt, tables)
|
||||
assert _tables_read(conn, stmt) <= set(found), (g, stmt, found)
|
||||
if (columns, rows, program) == run(base_stmt):
|
||||
same += 1
|
||||
base = recognize(base_stmt, result_columns=columns, tables=tables)
|
||||
assert out == base, f"{g!r}:{stmt!r} → {out!r},用空格时 → {base!r}"
|
||||
assert found == tables_in(base_stmt, tables), (g, stmt)
|
||||
executed_with[g] = executed
|
||||
if executed and same == executed:
|
||||
whitespace.add(g)
|
||||
expected = {c for c in _SQLITE_SPACE} | {" " + c for c in _SQLITE_SPACE + "\x0b"} | {c + " " for c in _SQLITE_SPACE}
|
||||
assert whitespace == expected, (sorted(map(repr, whitespace - expected)), sorted(map(repr, expected - whitespace)))
|
||||
# 有牙齿:空白后面的 U+FEFF、竖制表符在每个模板里都执行了(含藏 UNION ALL、GROUP BY 的那几条)
|
||||
assert executed_with[" " + BOM] == executed_with[" \x0b"] == len(_GAP_TEMPLATES)
|
||||
@@ -0,0 +1,284 @@
|
||||
"""裁判摘录的期 4 来历行(P4-SPEC 3.2–3.4):按文档标记分版本、只对上传表格。
|
||||
|
||||
- 期 4 之前的文档(没有标记):摘录逐字等于金样(tests/fixtures/evidence/pre_p4_golden.json),provenance、
|
||||
direct_select 两个模块一次都不碰(用桩钉住:桩一被访问就抛错);
|
||||
- 带标记的文档:手工源的摘录不变;上传源的查询摘录只在 SQL 那一行之后、「列:」之前多出 judge_lines 的行,
|
||||
去掉这些行就等于金样;同样的输入两次得到同样的字;
|
||||
- 按需裁判的 loader:带标记的文档才取得到哈希链上的清单,老文档连坏掉的清单都碰不到(A26);
|
||||
- 真导入的 D28(表下有含数字的说明):区域外文字带限定语、带统计期;遮罩时不送;单句的点击估价在上限以内。
|
||||
|
||||
金样的工件逐个 put_json 进临时数据目录,先断言返回的 id 等于金样记下的 id(内容寻址,对不上说明金样被改过)。
|
||||
夹具全部合成、假名;不调用任何模型(只比摘录文字和估价)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from app.api import evidence as api
|
||||
from app.core import artifact_store
|
||||
from app.core.config import settings
|
||||
from app.data import provenance as prov
|
||||
from app.data.provenance_types import DOC_PROVENANCE, PROV_OUTSIDE_CHARS
|
||||
from app.data.tabular import UNSHAPED_NOTE
|
||||
from app.engine import judge
|
||||
from app.engine.direct_select import tables_in
|
||||
from app.main import app
|
||||
from tests.test_provenance_api import import_flow, run_sql, sealed_of
|
||||
|
||||
GOLDEN = Path(__file__).parent / "fixtures" / "evidence" / "pre_p4_golden.json"
|
||||
#: 3.2 区域外文字的小标题(限定语每块只写一次)
|
||||
OUTSIDE_HEAD = ("区域外文字(原表区域外的文字原文,未经系统核对;其中的数字不是查询结果,不能用来支持或否定报告中的"
|
||||
"数字):")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 夹具
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def golden(monkeypatch, tmp_path) -> dict:
|
||||
"""金样的输入工件写进一个临时数据目录(id 必须等于金样记下的),返回金样本身。"""
|
||||
data = tmp_path / "data"
|
||||
(data / "uploads").mkdir(parents=True)
|
||||
monkeypatch.setattr(settings, "data_dir", data)
|
||||
g = json.loads(GOLDEN.read_text("utf-8"))
|
||||
for aid, content in g["artifacts"].items():
|
||||
assert await artifact_store.put_json(content, kind="golden") == aid
|
||||
return g
|
||||
|
||||
|
||||
class _Forbidden(types.ModuleType):
|
||||
"""桩模块:任何属性一被访问就记下来并抛错。期 4 之前的文档、手工源的摘录不该碰 provenance、direct_select。"""
|
||||
|
||||
def __init__(self, name: str, touched: list[str]) -> None:
|
||||
super().__init__(name)
|
||||
self._touched = touched
|
||||
|
||||
def __getattr__(self, attr: str):
|
||||
if attr.startswith("__"):
|
||||
raise AttributeError(attr)
|
||||
self._touched.append(f"{self.__name__}.{attr}")
|
||||
raise AssertionError(f"不该用到 {self.__name__}.{attr}")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def forbidden(monkeypatch) -> list[str]:
|
||||
touched: list[str] = []
|
||||
for name in ("app.data.provenance", "app.engine.direct_select"):
|
||||
monkeypatch.setitem(sys.modules, name, _Forbidden(name, touched))
|
||||
return touched
|
||||
|
||||
|
||||
def doc_of(g: dict, kind: str, *, marked: bool) -> tuple[dict, dict]:
|
||||
report = g["reports"][kind]
|
||||
doc = copy.deepcopy(report["outputs"]["doc"])
|
||||
assert "provenance" not in doc
|
||||
if marked:
|
||||
doc["provenance"] = DOC_PROVENANCE
|
||||
return doc, copy.deepcopy(report["catalog"])
|
||||
|
||||
|
||||
def excerpts(doc: dict, catalog: dict, **kw) -> dict[str, str]:
|
||||
return judge.prepare(doc, catalog, loader=kw.pop("loader", artifact_store.load), **kw).excerpts
|
||||
|
||||
|
||||
def inserted(new: str, old: str) -> list[str]:
|
||||
"""新摘录比金样多出的行:只许插在第 2 行(SQL)之后、「列:」之前,其余逐字相同。"""
|
||||
a, b = new.split("\n"), old.split("\n")
|
||||
extra = len(a) - len(b)
|
||||
assert extra >= 0 and a[:2] == b[:2] and a[2 + extra:] == b[2:], (new, old)
|
||||
assert b[2].startswith("列:")
|
||||
return a[2:2 + extra]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 按文档标记分版本(3.3)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kind", ["recipe", "simple", "manual"])
|
||||
async def test_unmarked_documents_are_unchanged_and_never_touch_the_new_modules(golden, forbidden, kind):
|
||||
doc, catalog = doc_of(golden, kind, marked=False)
|
||||
assert excerpts(doc, catalog) == golden["reports"][kind]["outputs"]["excerpts"]
|
||||
assert forbidden == []
|
||||
|
||||
|
||||
async def test_marked_manual_source_is_unchanged(golden, forbidden):
|
||||
"""非上传源(查询快照没有 data_version):带标记也一字不差,两个模块照样不碰。"""
|
||||
doc, catalog = doc_of(golden, "manual", marked=True)
|
||||
assert excerpts(doc, catalog) == golden["reports"]["manual"]["outputs"]["excerpts"]
|
||||
assert forbidden == []
|
||||
|
||||
|
||||
async def test_marked_recipe_source_adds_lines_only_between_sql_and_columns(golden):
|
||||
doc, catalog = doc_of(golden, "recipe", marked=True)
|
||||
new, old = excerpts(doc, catalog), golden["reports"]["recipe"]["outputs"]["excerpts"]
|
||||
assert set(new) == set(old)
|
||||
# 表名实体的摘录不加来历(只改查询摘录)
|
||||
assert new["t:日客流"] == old["t:日客流"]
|
||||
for alias, table in (("Q1", "日客流"), ("Q2", "时段客流")):
|
||||
lines = inserted(new[alias], old[alias])
|
||||
snap = artifact_store.load(catalog[alias]["artifact"])
|
||||
schema = artifact_store.load(snap["schema_artifact"])
|
||||
tables = {t: [c["name"] for c in meta["columns"]] for t, meta in schema["tables"].items()}
|
||||
assert tables_in(snap["sql"], tables) == [table]
|
||||
assert lines == prov.judge_lines(snap, schema, artifact_store.load, tables=[table], hidden=set())
|
||||
assert lines[0].startswith("来源:上传的表格,每期替换,第 1 次导入:统计期 2026-08-01 至 2026-08-31")
|
||||
assert f"口径:表 {table} 的说明:" in "\n".join(lines)
|
||||
|
||||
|
||||
async def test_marked_simple_upload_adds_only_caliber_lines(golden):
|
||||
"""简单导入(没有导入清单)只多口径行(调整 6):未规整的表写 UNSHAPED_NOTE;没有来源、接受、区域外文字。"""
|
||||
doc, catalog = doc_of(golden, "simple", marked=True)
|
||||
new, old = excerpts(doc, catalog), golden["reports"]["simple"]["outputs"]["excerpts"]
|
||||
lines = inserted(new["Q1"], old["Q1"])
|
||||
assert lines and all(line.startswith("口径:") for line in lines)
|
||||
assert f"口径:表 月报 的说明:{UNSHAPED_NOTE}" in lines
|
||||
assert new["t:月报"] == old["t:月报"]
|
||||
|
||||
|
||||
async def test_excerpts_are_deterministic(golden):
|
||||
"""摘录进断点指纹和逐句复用键:同样的输入两次得到同样的字(不取当前时间、不取数据库列)。"""
|
||||
doc, catalog = doc_of(golden, "recipe", marked=True)
|
||||
assert excerpts(doc, catalog) == excerpts(copy.deepcopy(doc), copy.deepcopy(catalog))
|
||||
|
||||
|
||||
async def test_masks_and_tables_are_handed_to_judge_lines(golden, monkeypatch):
|
||||
"""hidden 就是查询摘录现成的那份遮罩(数据源现在设的 ∪ 快照记下的,小写);tables 由 tables_in 按冻结表结构算。"""
|
||||
calls: list[dict] = []
|
||||
real = prov.judge_lines
|
||||
|
||||
def spy(query, schema, load, *, tables, hidden):
|
||||
calls.append({"sql": query["sql"], "tables": tables, "hidden": set(hidden), "schema": schema is not None})
|
||||
return real(query, schema, load, tables=tables, hidden=hidden)
|
||||
|
||||
monkeypatch.setattr(prov, "judge_lines", spy)
|
||||
doc, catalog = doc_of(golden, "recipe", marked=True)
|
||||
out = excerpts(doc, catalog, masked={"flow_demo": ["分区甲"], "别的源": ["日期"]})
|
||||
assert [(c["tables"], c["hidden"], c["schema"]) for c in calls] == [
|
||||
(["日客流"], {"分区甲"}, True), (["时段客流"], {"分区甲"}, True)]
|
||||
# 遮罩的列:结果里不给,列的说明也不写
|
||||
assert "口径:列 分区甲 的说明" not in out["Q1"] and "分区甲" not in out["Q1"].split("\n")[-1]
|
||||
|
||||
|
||||
async def test_a_broken_artifact_degrades_to_one_line(golden, monkeypatch):
|
||||
"""生成来历行时出了意外(工件形状不对):这条查询只写「导入清单无法读取…」一行,整份报告照样能判。"""
|
||||
def boom(*_a, **_kw):
|
||||
raise TypeError("坏形状")
|
||||
|
||||
monkeypatch.setattr(prov, "judge_lines", boom)
|
||||
doc, catalog = doc_of(golden, "recipe", marked=True)
|
||||
new, old = excerpts(doc, catalog), golden["reports"]["recipe"]["outputs"]["excerpts"]
|
||||
assert inserted(new["Q1"], old["Q1"]) == [prov.SOURCE_BROKEN]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 按需裁判的取证范围(3.4、2.8.3)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def golden_sealed(g: dict, kind: str) -> api._Sealed:
|
||||
info = g["reports"][kind]["sealed"]
|
||||
sealed = sealed_of(queries=list(info["queries"]), schemas=list(info["schemas"]))
|
||||
sealed.queries.update(info["queries"])
|
||||
return sealed
|
||||
|
||||
|
||||
async def test_on_demand_loader_reaches_the_chain_only_for_marked_docs(golden):
|
||||
doc, catalog = doc_of(golden, "recipe", marked=True)
|
||||
sealed = golden_sealed(golden, "recipe")
|
||||
old = golden["reports"]["recipe"]["outputs"]["excerpts"]
|
||||
# 带标记:loader 扩展到表结构快照经哈希链列出的导入清单,来源行照常写
|
||||
chained = excerpts(doc, catalog, loader=api._sealed_loader(sealed, chain=True))
|
||||
assert inserted(chained["Q1"], old["Q1"])[0].startswith("来源:上传的表格,每期替换")
|
||||
assert chained == excerpts(doc, catalog)
|
||||
# 不扩展的 loader 取不到清单:只写「导入清单无法读取…」一行,口径照写(来自封存范围内的表结构快照)
|
||||
plain = excerpts(doc, catalog, loader=api._sealed_loader(sealed))
|
||||
lines = inserted(plain["Q1"], old["Q1"])
|
||||
assert lines[0] == prov.SOURCE_BROKEN and any(line.startswith("口径:表 日客流") for line in lines)
|
||||
|
||||
|
||||
async def test_pre_p4_on_demand_judging_never_reads_a_broken_manifest(golden, forbidden):
|
||||
"""A26:老文档(chain 为假)+ 表结构快照列出的导入清单文件被改坏:摘录与金样逐字相同,根本不去取清单。"""
|
||||
report = golden["reports"]["recipe"]
|
||||
snap = artifact_store.load(report["catalog"]["Q1"]["artifact"])
|
||||
[mid] = artifact_store.load(snap["schema_artifact"])["import_manifests"]
|
||||
path = settings.data_dir / "artifacts" / mid[:2] / f"{mid}.json"
|
||||
path.write_text(path.read_text("utf-8").replace('"seq":1', '"seq":9', 1), encoding="utf-8")
|
||||
with pytest.raises(ValueError):
|
||||
artifact_store.load(mid)
|
||||
doc, catalog = doc_of(golden, "recipe", marked=False)
|
||||
sealed = golden_sealed(golden, "recipe")
|
||||
loader = api._sealed_loader(sealed, chain=doc.get("provenance") == DOC_PROVENANCE)
|
||||
assert excerpts(doc, catalog, loader=loader) == report["outputs"]["excerpts"]
|
||||
assert forbidden == []
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 真导入:区域外文字、遮罩、估价(3.2)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def client():
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
|
||||
yield c
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_model(monkeypatch, tmp_path):
|
||||
from app.data import recipe_ai
|
||||
from app.data.recipe_types import AiAvailability
|
||||
|
||||
data = tmp_path / "data"
|
||||
(data / "uploads").mkdir(parents=True)
|
||||
monkeypatch.setattr(settings, "data_dir", data)
|
||||
|
||||
async def none(session):
|
||||
return None, "", AiAvailability(False, reason="测试不接模型")
|
||||
|
||||
monkeypatch.setattr(recipe_ai, "resolve_draft_model", none)
|
||||
|
||||
|
||||
async def test_outside_text_masks_and_cost_on_a_real_import(no_model, client):
|
||||
"""D28(9 月,表下 B32 有一句含数字的说明,新导入的清单记了 hidden=False):摘录送全文、带限定语和统计期;
|
||||
数据源设遮罩时一格都不送、只写条数;只判一句、引用这一条查询的点击估价不超过 0.01 美元。"""
|
||||
from app.data.engine import engines
|
||||
from tests.fixtures.xlsx.drift import SEP
|
||||
|
||||
up = await import_flow(client, seed=4201, start=SEP, days=30, variant="D28")
|
||||
try:
|
||||
qid, _sid = await run_sql(client, up, 'SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" = \'2026-09-15\'')
|
||||
g = json.loads(GOLDEN.read_text("utf-8"))
|
||||
doc = copy.deepcopy(g["reports"]["recipe"]["outputs"]["doc"])
|
||||
doc["provenance"] = DOC_PROVENANCE
|
||||
catalog = {"Q1": {**g["reports"]["recipe"]["catalog"]["Q1"], "artifact": qid, "source": up.name,
|
||||
"tool": f"db_query__{up.name}"}}
|
||||
request = judge.prepare(doc, catalog)
|
||||
text = request.excerpts["Q1"]
|
||||
lines = text.split("\n")
|
||||
assert lines[2].startswith("来源:上传的表格,每期替换,第 1 次导入:统计期 2026-09-01 至 2026-09-30")
|
||||
assert OUTSIDE_HEAD in lines
|
||||
note = next(line for line in lines if "B32:" in line)
|
||||
assert note.startswith(" 第 1 次导入(统计期 2026-09-01 至 2026-09-30)客流汇总!B32:「") and "9月15日" in note
|
||||
assert len(note) < PROV_OUTSIDE_CHARS + 60
|
||||
assert judge.prepare(doc, catalog).excerpts == request.excerpts # 确定
|
||||
# 遮罩:区域外文字一格都不送,只写条数
|
||||
masked = judge.prepare(doc, catalog, masked={up.name: ["分区甲"]}).excerpts["Q1"]
|
||||
assert "9月15日" not in masked and OUTSIDE_HEAD not in masked
|
||||
assert "(数据源设置了遮罩,未列出)" in masked
|
||||
# 估价:只判一句、引用这一条查询(claude-sonnet-5 的目录价)
|
||||
one = [c for c in request.cands if "Q1" in c.aliases][:1]
|
||||
assert one
|
||||
cost = request.cost("claude-sonnet-5", one)
|
||||
assert cost is not None and cost <= 0.01, cost
|
||||
finally:
|
||||
await engines.invalidate(up.source_id)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,819 @@
|
||||
"""期 4 证据下钻的契约(P4-SPEC 7.1,WP-0):provenance_types 的常量、取值、原文、响应形状和中间结构。
|
||||
|
||||
并行的几个工作包看不到彼此的代码,只靠这份契约对齐。这里钉住的是「改了就会合错」的东西:字段名、顺序、缺省值、
|
||||
asdict 之后的 JSON 键(逐层对 P4-SPEC 2.8.1)、原文覆盖每个 code、原文守文案规范。另外两份 WP-0 交付物也在这里
|
||||
做最低限度的自检:前端夹具里每一份推断来源的答复都能按契约读回、满足不变式;金样的工件 id 等于内容哈希。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import dataclasses
|
||||
import json
|
||||
import re
|
||||
import types
|
||||
import typing
|
||||
from dataclasses import MISSING, asdict
|
||||
from pathlib import Path
|
||||
from typing import Any, get_args, get_origin
|
||||
|
||||
import pydantic
|
||||
import pytest
|
||||
|
||||
from app.data import provenance_types as P
|
||||
from app.data import recipe_types as T
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
FIXTURE = ROOT / "frontend" / "src" / "run" / "__tests__" / "evidence-provenance.json"
|
||||
GOLDEN = Path(__file__).resolve().parent / "fixtures" / "evidence" / "pre_p4_golden.json"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 常量与取值
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_constants():
|
||||
assert P.DOC_PROVENANCE == 1
|
||||
assert P.SCHEMA == "agentlab.provenance/1"
|
||||
assert (P.PROV_PARTS, P.PROV_ACCEPT, P.PROV_OUTSIDE, P.PROV_OUTSIDE_CHARS, P.PROV_COLUMN_NOTES, P.PROV_CHARS) == (
|
||||
3, 4, 4, 200, 6, 1200)
|
||||
|
||||
|
||||
def test_literals():
|
||||
assert get_args(P.Status) == ("inferred", "table_only", "none")
|
||||
assert set(get_args(P.ReasonCode)) == {
|
||||
"legacy_doc", "not_cell", "not_sealed", "not_upload", "simple_upload", "manifest_unreadable",
|
||||
"chain_mismatch", "expression", "alias", "multi_table", "unparsed", "no_pk", "pk_missing", "masked",
|
||||
"null_value", "null_pk", "snapshot_gone", "db_tampered", "recheck_missing", "recheck_multiple",
|
||||
"recheck_mismatch", "no_lineage"}
|
||||
assert len(get_args(P.ReasonCode)) == len(set(get_args(P.ReasonCode)))
|
||||
assert set(get_args(P.AlertCode)) == {"db_tampered", "chain_mismatch", "manifest_unreadable"}
|
||||
assert set(get_args(P.AlertCode)) <= set(get_args(P.ReasonCode))
|
||||
assert get_args(P.RefusalCode) == ("expression", "alias", "multi_table", "unparsed")
|
||||
assert set(P.DETAILS) == set(get_args(P.RefusalCode))
|
||||
assert P.DETAILS["multi_table"] == ("join", "comma_join", "subquery", "cte", "compound", "authorizer",
|
||||
"duplicate_pk")
|
||||
assert "literal_keyword" in P.DETAILS["expression"] and "ambiguous_column" in P.DETAILS["unparsed"]
|
||||
assert get_args(P.FromRole) == ("axis_header", "row_label", "section_title", "col_header", "total_label")
|
||||
assert get_args(P.RowStatus) == ("passed", "mismatch", "unverifiable")
|
||||
assert get_args(P.CellStatus) == ("ok", "unverifiable", "unknown", "not_formula")
|
||||
assert get_args(P.RawState) == ("kept", "purged", "absent")
|
||||
assert get_args(P.Mode) == ("replace", "accumulate")
|
||||
assert get_args(P.AcceptanceKind) == ("override", "waiver")
|
||||
assert get_args(P.YearFrom) == ("period", "human")
|
||||
assert get_args(P.ColumnRole) == ("axis", "dim", "derive", "const", "measure", "value", "text")
|
||||
# 和配方契约同名的取值:照搬 CheckResult.status、TableOut.kind、PeriodOut.source,不能各写一份漂开
|
||||
assert get_args(P.PartStatus) == get_args(T.CheckStatus)
|
||||
assert get_args(P.TableKind) == get_args(typing.get_type_hints(T.TableOut)["kind"])
|
||||
assert get_args(typing.get_type_hints(P.PeriodView)["source"]) == get_args(
|
||||
typing.get_type_hints(T.PeriodOut)["source"])
|
||||
|
||||
|
||||
def test_module_imports_only_stdlib():
|
||||
"""识别器只许导入守卫、names 和这个模块(P4-SPEC 7.2):这里不能把配方模块、pydantic 拖进来。"""
|
||||
tree = ast.parse(Path(P.__file__).read_text("utf-8"))
|
||||
mods = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
mods |= {a.name.split(".")[0] for a in node.names}
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
mods.add((node.module or "").split(".")[0])
|
||||
assert mods <= {"__future__", "logging", "re", "collections", "dataclasses", "typing"}, mods
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 每个 dataclass 的字段名、顺序和缺省值(防止实现者改名;REQ = 必填)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
REQ = "<必填>"
|
||||
FACTORY = "<默认工厂>"
|
||||
|
||||
|
||||
def _shape(cls) -> list[tuple[str, object]]:
|
||||
out = []
|
||||
for f in dataclasses.fields(cls):
|
||||
if f.default is not MISSING:
|
||||
out.append((f.name, f.default))
|
||||
elif f.default_factory is not MISSING:
|
||||
out.append((f.name, FACTORY))
|
||||
else:
|
||||
out.append((f.name, REQ))
|
||||
return out
|
||||
|
||||
|
||||
SHAPES: dict[str, list[tuple[str, object]]] = {
|
||||
# 响应(P4-SPEC 2.8.1)
|
||||
"ReportRef": [("node_id", REQ), ("doc_artifact", REQ)],
|
||||
"CellRef": [("alias", REQ), ("row", REQ), ("column", REQ), ("artifact", None)],
|
||||
"Reason": [("code", REQ), ("detail", ""), ("text", REQ)],
|
||||
"Alert": [("code", REQ), ("text", REQ)],
|
||||
"TableRef": [("name", REQ), ("kind", "data")],
|
||||
"PeriodView": [("start", REQ), ("end", REQ), ("source", REQ), ("cells", FACTORY), ("signed_by", None)],
|
||||
"AcceptanceView": [("check_id", REQ), ("title", REQ), ("kind", REQ), ("reason", REQ), ("signed_by", None),
|
||||
("at", None)],
|
||||
"StateNote": [("at", None), ("signed_by", None), ("reason", None)],
|
||||
"PartView": [("import_id", REQ), ("seq", REQ), ("manifest", REQ), ("period", REQ), ("file_name", REQ),
|
||||
("raw_sha256", REQ), ("region", None), ("excluded_rows", None), ("recipe_sha256", None),
|
||||
("recipe_seq", None), ("committed_at", None), ("signed_by", None), ("acceptances", FACTORY),
|
||||
("raw_state", None), ("purged", None), ("revoked", None), ("has_row", False)],
|
||||
"VersionView": [("source", REQ), ("snapshot_id", REQ), ("mode", REQ), ("union", REQ), ("tables", FACTORY),
|
||||
("manifest_view", True), ("parts", FACTORY)],
|
||||
"FromCell": [("role", REQ), ("column", REQ), ("sheet", REQ), ("cell", REQ), ("text", None),
|
||||
("locate_title", None)],
|
||||
"YearSource": [("source", REQ), ("cells", FACTORY), ("signed_by", None), ("mixed", False)],
|
||||
"CanonicalView": [("raw", REQ), ("canonical", REQ)],
|
||||
"Recheck": [("sql", REQ), ("params", FACTORY), ("ok", REQ)],
|
||||
"CellSource": [("table", REQ), ("column", REQ), ("column_role", REQ), ("kind", "data"), ("pk", FACTORY),
|
||||
("rowid", None), ("part_seq", REQ), ("part_rowid", REQ), ("sheet", REQ), ("cell", REQ),
|
||||
("header", None), ("unit", None), ("from", FACTORY), ("year", None), ("canonical", None),
|
||||
("raw_purged", False), ("merged_fill", False), ("recheck", None)],
|
||||
"RelatedCheck": [("id", REQ), ("kind", REQ), ("title", REQ), ("part_status", REQ), ("row_status", None),
|
||||
("cell_status", None), ("acceptance", None), ("detail", None)],
|
||||
"ProvenanceOut": [("schema", P.SCHEMA), ("report", REQ), ("segment", REQ), ("cell", None), ("status", REQ),
|
||||
("reason", None), ("alert", None), ("sealed", REQ), ("version", None),
|
||||
("cell_source", None), ("checks", FACTORY)],
|
||||
# 中间结构(P4-SPEC 7.2–7.4)
|
||||
"DirectSelect": [("table", REQ), ("alias", REQ), ("columns", REQ)],
|
||||
"SelectRefusal": [("code", REQ), ("detail", REQ)],
|
||||
"ChainPart": [("seq", REQ), ("import_id", REQ), ("manifest_id", REQ), ("manifest", REQ), ("union_rows", REQ)],
|
||||
"Chain": [("source", REQ), ("source_id", REQ), ("snapshot_id", REQ), ("mode", REQ), ("union", REQ),
|
||||
("expected_db_sha256", REQ), ("parts", REQ), ("schema", REQ), ("snapshot_manifest", REQ)],
|
||||
"ChainProblem": [("code", REQ), ("detail", "")],
|
||||
"Located": [("part", REQ), ("part_rowid", REQ), ("cell", REQ)],
|
||||
"LocateProblem": [("code", REQ), ("detail", "")],
|
||||
"SumEqRule": [("table", REQ), ("total", REQ), ("parts", REQ), ("types", REQ)],
|
||||
"RelatedCheckPlan": [("check", REQ), ("rule", REQ)],
|
||||
"CompileFacts": [("tables", REQ), ("result_rows", REQ), ("yields", REQ)],
|
||||
}
|
||||
RESPONSE = ("ReportRef", "CellRef", "Reason", "Alert", "TableRef", "PeriodView", "AcceptanceView", "StateNote",
|
||||
"PartView", "VersionView", "FromCell", "YearSource", "CanonicalView", "Recheck", "CellSource",
|
||||
"RelatedCheck", "ProvenanceOut")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", sorted(SHAPES))
|
||||
def test_dataclass_shape(name):
|
||||
cls = getattr(P, name)
|
||||
assert dataclasses.is_dataclass(cls)
|
||||
assert _shape(cls) == SHAPES[name]
|
||||
|
||||
|
||||
def test_every_dataclass_is_pinned():
|
||||
declared = {n for n, v in vars(P).items() if isinstance(v, type) and dataclasses.is_dataclass(v)}
|
||||
assert declared == set(SHAPES), declared ^ set(SHAPES)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", RESPONSE)
|
||||
def test_response_types_are_keyword_only(name):
|
||||
"""响应的字段多、几个包各自构造:按位置传参会被拒,传错名字立刻报错。"""
|
||||
cls = getattr(P, name)
|
||||
assert all(f.kw_only for f in dataclasses.fields(cls)), name
|
||||
with pytest.raises(TypeError):
|
||||
cls("x")
|
||||
|
||||
|
||||
def test_intermediate_types_take_positional_args_as_the_spec_writes_them():
|
||||
assert P.ChainProblem("manifest_unreadable").detail == ""
|
||||
assert P.LocateProblem("no_lineage") == P.LocateProblem(code="no_lineage", detail="")
|
||||
assert P.SelectRefusal("alias", "alias").code == "alias"
|
||||
assert P.CompileFacts({"日客流"}, 1, 0).yields == 0
|
||||
assert P.DirectSelect("日客流", None, ["日期", "全日客流"]).columns == ["日期", "全日客流"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# asdict 之后逐层对 P4-SPEC 2.8.1 的 JSON
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
KEYS = {
|
||||
"top": ["schema", "report", "segment", "cell", "status", "reason", "alert", "sealed", "version", "cell_source",
|
||||
"checks"],
|
||||
"report": ["node_id", "doc_artifact"],
|
||||
"cell": ["alias", "row", "column", "artifact"],
|
||||
"reason": ["code", "detail", "text"],
|
||||
"alert": ["code", "text"],
|
||||
"version": ["source", "snapshot_id", "mode", "union", "tables", "manifest_view", "parts"],
|
||||
"table": ["name", "kind"],
|
||||
"part": ["import_id", "seq", "manifest", "period", "file_name", "raw_sha256", "region", "excluded_rows",
|
||||
"recipe_sha256", "recipe_seq", "committed_at", "signed_by", "acceptances", "raw_state", "purged",
|
||||
"revoked", "has_row"],
|
||||
"period": ["start", "end", "source", "cells", "signed_by"],
|
||||
"acceptance": ["check_id", "title", "kind", "reason", "signed_by", "at"],
|
||||
"state": ["at", "signed_by", "reason"],
|
||||
"cell_source": ["table", "column", "column_role", "kind", "pk", "rowid", "part_seq", "part_rowid", "sheet",
|
||||
"cell", "header", "unit", "from", "year", "canonical", "raw_purged", "merged_fill", "recheck"],
|
||||
"from": ["role", "column", "sheet", "cell", "text", "locate_title"],
|
||||
"year": ["source", "cells", "signed_by", "mixed"],
|
||||
"canonical": ["raw", "canonical"],
|
||||
"recheck": ["sql", "params", "ok"],
|
||||
"check": ["id", "kind", "title", "part_status", "row_status", "cell_status", "acceptance", "detail"],
|
||||
}
|
||||
|
||||
|
||||
def _accept(**kw) -> P.AcceptanceView:
|
||||
return P.AcceptanceView(**{"check_id": "R1", "title": "「日客流」:全日客流 = 分区甲 + 分区乙", "kind": "override",
|
||||
"reason": "合成理由:分区合计口径调整", "signed_by": None,
|
||||
"at": "2026-10-01T08:00:00+00:00", **kw})
|
||||
|
||||
|
||||
def full_out() -> P.ProvenanceOut:
|
||||
"""字段齐全的一份(每个可空的嵌套对象都非空、每个列表至少一项):空的 checks 会让键集合的比较空过。"""
|
||||
return P.ProvenanceOut(
|
||||
report=P.ReportRef(node_id="write", doc_artifact="d" * 64),
|
||||
segment="s7",
|
||||
cell=P.CellRef(alias="Q3", row=0, column="全日客流", artifact="a" * 64),
|
||||
status="inferred",
|
||||
sealed=True,
|
||||
version=P.VersionView(
|
||||
source="flow_demo", snapshot_id="b" * 64, mode="accumulate", union=True,
|
||||
tables=[P.TableRef(name="日客流", kind="data")],
|
||||
manifest_view=True,
|
||||
parts=[P.PartView(
|
||||
import_id="i1", seq=1, manifest="c" * 64,
|
||||
period=P.PeriodView(start="2026-08-01", end="2026-08-31", source="cells", cells=["客流汇总!B2"]),
|
||||
file_name="月报导出_2026-08-01_2026-08-31.xlsx", raw_sha256="e" * 64,
|
||||
region="客流汇总!B4:AG30", excluded_rows=0, recipe_sha256="f" * 64, recipe_seq=1,
|
||||
committed_at="2026-10-01T08:00:00+00:00", signed_by=None,
|
||||
acceptances=[_accept()], raw_state="purged",
|
||||
purged=P.StateNote(at="2026-10-02T08:00:00+00:00", signed_by="录入员甲", reason="合成理由"),
|
||||
revoked=P.StateNote(at="2026-10-02T09:00:00+00:00", signed_by=None, reason="合成理由"),
|
||||
has_row=True)]),
|
||||
cell_source=P.CellSource(
|
||||
table="日客流", column="全日客流", column_role="measure", kind="data", pk={"日期": "2026-08-05"},
|
||||
rowid=5, part_seq=1, part_rowid=5, sheet="客流汇总", cell="G5", header="全日客流(人次)", unit="人次",
|
||||
from_=[P.FromCell(role="axis_header", column="日期", sheet="客流汇总", cell="G4"),
|
||||
P.FromCell(role="row_label", column=None, sheet="客流汇总", cell="B5", text="全日客流(人次)")],
|
||||
year=P.YearSource(source="period", cells=["客流汇总!B2"]),
|
||||
canonical=P.CanonicalView(raw="8–9", canonical="8-9"),
|
||||
raw_purged=True, merged_fill=False,
|
||||
recheck=P.Recheck(sql='SELECT rowid, "全日客流" FROM "日客流" WHERE "日期" = ?', params=["2026-08-05"],
|
||||
ok=True)),
|
||||
checks=[P.RelatedCheck(id="R1", kind="relation_sum_eq", title="「日客流」:全日客流 = 分区甲 + 分区乙",
|
||||
part_status="mismatch", row_status="passed", cell_status=None,
|
||||
acceptance=_accept(), detail="合成的原因摘要")],
|
||||
)
|
||||
|
||||
|
||||
def test_asdict_matches_the_spec_json_level_by_level():
|
||||
d = asdict(full_out())
|
||||
v, part, cs = d["version"], d["version"]["parts"][0], d["cell_source"]
|
||||
levels = {
|
||||
"top": d, "report": d["report"], "cell": d["cell"], "version": v, "table": v["tables"][0], "part": part,
|
||||
"period": part["period"], "acceptance": part["acceptances"][0], "state": part["purged"],
|
||||
"cell_source": cs, "from": cs["from"][0], "year": cs["year"], "canonical": cs["canonical"],
|
||||
"recheck": cs["recheck"], "check": d["checks"][0],
|
||||
}
|
||||
for name, obj in levels.items():
|
||||
assert isinstance(obj, dict), name
|
||||
assert list(obj) == KEYS[name], name
|
||||
assert list(part["revoked"]) == KEYS["state"]
|
||||
assert list(d["checks"][0]["acceptance"]) == KEYS["acceptance"]
|
||||
# 原因、提示:table_only 加红的一份
|
||||
red = P.ProvenanceOut(report=P.ReportRef(node_id="write", doc_artifact="d" * 64), segment="s9",
|
||||
status="table_only", reason=P.refuse("db_tampered"), alert=P.alert("db_tampered"),
|
||||
sealed=True, version=full_out().version)
|
||||
rd = asdict(red)
|
||||
assert list(rd["reason"]) == KEYS["reason"] and list(rd["alert"]) == KEYS["alert"]
|
||||
assert rd["cell_source"] is None and rd["checks"] == [] and rd["cell"] is None
|
||||
json.dumps(d, ensure_ascii=False)
|
||||
|
||||
|
||||
def test_null_and_empty_defaults_match_the_spec():
|
||||
out = P.ProvenanceOut(report=P.ReportRef(node_id="write", doc_artifact="d"), segment="s1", status="none",
|
||||
reason=P.refuse("legacy_doc"), sealed=False)
|
||||
d = asdict(out)
|
||||
assert d["schema"] == "agentlab.provenance/1"
|
||||
assert (d["cell"], d["alert"], d["version"], d["cell_source"], d["checks"]) == (None, None, None, None, [])
|
||||
assert d["reason"] == {"code": "legacy_doc", "detail": "", "text": P.REASON_TEXT["legacy_doc"]}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CellSource 的「from」:Python 里是 from_,asdict / fields / replace / JSON 里是 from
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _cell(**kw) -> P.CellSource:
|
||||
base = {"table": "日客流", "column": "全日客流", "column_role": "measure", "part_seq": 1, "part_rowid": 5,
|
||||
"sheet": "客流汇总", "cell": "G5"}
|
||||
return P.CellSource(**{**base, **kw})
|
||||
|
||||
|
||||
def test_cell_source_from_key():
|
||||
fc = P.FromCell(role="axis_header", column="日期", sheet="客流汇总", cell="G4")
|
||||
a = _cell(from_=[fc])
|
||||
assert a.from_ == [fc] and getattr(a, "from") == [fc]
|
||||
assert [f.name for f in dataclasses.fields(P.CellSource)].count("from") == 1
|
||||
assert "from_" not in {f.name for f in dataclasses.fields(P.CellSource)}
|
||||
d = asdict(a)
|
||||
assert "from" in d and "from_" not in d and d["from"][0]["cell"] == "G4"
|
||||
# 从 JSON 读回来:键是 from
|
||||
b = P.CellSource(**{**d, "from": [P.FromCell(**x) for x in d["from"]]})
|
||||
assert b == a
|
||||
# replace 不丢 from(WP-C 回查后用 replace 补 rowid、pk、recheck 是常见写法)
|
||||
c = dataclasses.replace(a, rowid=5, pk={"日期": "2026-08-05"})
|
||||
assert c.from_ == [fc] and c.rowid == 5
|
||||
# replace 换 from:两种名字都认,Python 名字优先
|
||||
fc2 = P.FromCell(role="row_label", column=None, sheet="客流汇总", cell="B5")
|
||||
assert dataclasses.replace(a, from_=[fc2]).from_ == [fc2]
|
||||
assert dataclasses.replace(a, **{"from": [fc2]}).from_ == [fc2]
|
||||
# 属性写入两种名字指同一份
|
||||
setattr(a, "from", [])
|
||||
assert a.from_ == [] and asdict(a)["from"] == []
|
||||
assert _cell().from_ == [] and _cell().from_ is not _cell().from_
|
||||
assert "from_=" in repr(_cell())
|
||||
# 注解里同样是 from(pydantic 按注解收集字段);值存在实例字典的 from 键下
|
||||
hints = typing.get_type_hints(P.CellSource)
|
||||
assert "from" in hints and "from_" not in hints
|
||||
assert list(hints) == [f.name for f in dataclasses.fields(P.CellSource)]
|
||||
assert "from" in vars(a) and "from_" not in vars(a)
|
||||
assert not hasattr(object.__new__(P.CellSource), "from_")
|
||||
|
||||
|
||||
def test_pydantic_sees_the_same_json_as_asdict():
|
||||
"""WP-C 若把 ProvenanceOut 写成返回注解或 response_model,pydantic 会按类型再序列化一遍:输出必须等于 asdict,
|
||||
不能悄悄少掉「from」(只改字段表、不改注解时就是这样)。从 JSON 校验回来的实例也要能照常读写 from_。"""
|
||||
ta = pydantic.TypeAdapter(P.ProvenanceOut)
|
||||
out = full_out()
|
||||
want = asdict(out)
|
||||
assert ta.dump_python(out) == want and "from" in ta.dump_python(out)["cell_source"]
|
||||
assert json.loads(ta.dump_json(out)) == json.loads(json.dumps(want))
|
||||
back = ta.validate_json(json.dumps(want), strict=True)
|
||||
assert back == out and back.cell_source.from_ == out.cell_source.from_
|
||||
assert asdict(back) == want and dataclasses.replace(back.cell_source, rowid=9).from_ == out.cell_source.from_
|
||||
cs_ta = pydantic.TypeAdapter(P.CellSource)
|
||||
assert cs_ta.dump_python(out.cell_source) == asdict(out.cell_source)
|
||||
|
||||
|
||||
async def test_fastapi_routes_return_the_same_json_however_typed():
|
||||
"""三种写法的路由(推荐的 asdict + dict 注解、带类型的返回注解、response_model)回的 JSON 逐字相同。"""
|
||||
from fastapi import FastAPI
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@app.get("/plain")
|
||||
async def plain() -> dict[str, Any]:
|
||||
return asdict(full_out())
|
||||
|
||||
@app.get("/typed")
|
||||
async def typed() -> P.ProvenanceOut:
|
||||
return full_out()
|
||||
|
||||
@app.get("/model", response_model=P.ProvenanceOut)
|
||||
async def model() -> Any:
|
||||
return asdict(full_out())
|
||||
|
||||
want = json.loads(json.dumps(asdict(full_out())))
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://t") as client:
|
||||
for path in ("/plain", "/typed", "/model"):
|
||||
r = await client.get(path)
|
||||
assert r.status_code == 200, path
|
||||
assert r.json() == want, path
|
||||
assert "from" in r.json()["cell_source"], path
|
||||
assert (await client.get("/openapi.json")).status_code == 200
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 原文
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_reason_text_covers_every_code():
|
||||
assert set(P.REASON_TEXT) == set(get_args(P.ReasonCode))
|
||||
assert set(P.ALERT_TEXT) == set(get_args(P.AlertCode))
|
||||
for code, text in P.ALERT_TEXT.items():
|
||||
assert text == P.REASON_TEXT[code]
|
||||
# 4.2 的原文(WP-0 写定,界面原样显示):抽几条与规格逐字对照
|
||||
assert P.REASON_TEXT["simple_upload"] == "这份表格按表头行直接导入,没有记录原表格子的溯源"
|
||||
assert P.REASON_TEXT["masked"] == "这一格涉及设置了遮罩的列,不推断来源"
|
||||
assert P.REASON_TEXT["null_value"] == "这一格是空值,不推断来源"
|
||||
assert P.REASON_TEXT["expression"] == "这一格是计算、聚合或改写的结果,只给出表级来历"
|
||||
assert "可能被修改过" in P.REASON_TEXT["db_tampered"] and "可能被修改过" in P.REASON_TEXT["chain_mismatch"]
|
||||
assert "改了名" in P.REASON_TEXT["alias"] and "同一张表多次" in P.REASON_TEXT["multi_table"]
|
||||
assert "已回收或不存在" in P.REASON_TEXT["snapshot_gone"]
|
||||
assert P.REASON_TEXT["recheck_missing"] == P.REASON_TEXT["recheck_multiple"] == P.REASON_TEXT["recheck_mismatch"]
|
||||
|
||||
|
||||
_H = "[㐀-鿿]"
|
||||
_HALF = re.compile(f"{_H}[,;!?]|[,;!?]{_H}|{_H}:(?![/\\d])|:{_H}|{_H}\\(|\\){_H}|{_H}\"|\"{_H}|[“”]|\\.\\.\\.")
|
||||
_BANNED = re.compile("钉|放行|后端|跑|多半|照样|眼下|压根|取不到|没取回来|读不懂|连不上|写坏了|快照版本")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", sorted(P.REASON_TEXT))
|
||||
def test_reason_text_follows_the_copy_rules(code):
|
||||
"""原文上界面:不含数字、坐标(prose_problems),不写「钉」「放行」等禁用写法,全角标点、「」引号、不用「...」,
|
||||
不以句号收尾(界面上是一行说明,和其余面板文字一致)。"""
|
||||
text = P.REASON_TEXT[code]
|
||||
assert T.prose_problems(text) == [], (code, T.prose_problems(text))
|
||||
assert not _BANNED.search(text), code
|
||||
assert not _HALF.search(text), (code, _HALF.search(text))
|
||||
assert not text.endswith("。"), code
|
||||
assert re.search(_H, text)
|
||||
|
||||
|
||||
def test_refuse_and_alert():
|
||||
r = P.refuse("pk_missing", 列=["日期", "时段"])
|
||||
assert r == P.Reason(code="pk_missing", detail="",
|
||||
text="查询结果没有带齐主键列(日期、时段),无法按主键回查,只给出表级来历")
|
||||
assert "(日期)" in P.refuse("pk_missing", 列="日期").text
|
||||
assert P.refuse("multi_table", "duplicate_pk").detail == "duplicate_pk"
|
||||
assert P.refuse("multi_table", detail="compound").text == P.REASON_TEXT["multi_table"]
|
||||
assert P.refuse("no_lineage", "regions 没有 block 键").detail == "regions 没有 block 键"
|
||||
with pytest.raises(ValueError):
|
||||
P.refuse("pk_missing")
|
||||
with pytest.raises(ValueError):
|
||||
P.refuse("no_such_code") # type: ignore[arg-type]
|
||||
assert P.alert("chain_mismatch") == P.Alert(code="chain_mismatch", text=P.ALERT_TEXT["chain_mismatch"])
|
||||
with pytest.raises(ValueError):
|
||||
P.alert("no_lineage") # type: ignore[arg-type]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code,detail", [
|
||||
# 原型识别器自己的名字(p4-probe/probe_recognizer.py),WP-A 照抄就会走到这里;还有 2.4 没逐条写的写法
|
||||
("unparsed", "indexed_by"), ("multi_table", "union"), ("expression", "aggregate"),
|
||||
("alias", "expression_or_alias"), ("multi_table", "subquery_or_compound"), ("alias", "join"),
|
||||
])
|
||||
def test_refuse_with_an_unknown_detail_degrades_instead_of_raising(code, detail, caplog):
|
||||
"""细分取决于识别器对这条 SQL 的判断:漏了映射时推断来源接口不能回 500。照样给分组的原文、记警告,
|
||||
contract_problems 报出来(接口和验收测试据此抓到)。"""
|
||||
with caplog.at_level("WARNING", logger=P.__name__):
|
||||
r = P.refuse(code, detail)
|
||||
assert r == P.Reason(code=code, detail=detail, text=P.REASON_TEXT[code])
|
||||
assert detail in caplog.text
|
||||
out = P.ProvenanceOut(report=P.ReportRef(node_id="write", doc_artifact="d"), segment="s1", status="table_only",
|
||||
reason=r, sealed=True, version=_no_row_version())
|
||||
assert any("DETAILS" in p for p in P.contract_problems(out))
|
||||
|
||||
|
||||
def test_every_listed_detail_passes_and_empty_detail_is_allowed(caplog):
|
||||
v = _no_row_version()
|
||||
ref = P.ReportRef(node_id="write", doc_artifact="d")
|
||||
with caplog.at_level("WARNING", logger=P.__name__):
|
||||
for code, details in P.DETAILS.items():
|
||||
for detail in (*details, ""):
|
||||
out = P.ProvenanceOut(report=ref, segment="s1", status="table_only", reason=P.refuse(code, detail),
|
||||
sealed=True, version=v)
|
||||
assert P.contract_problems(out) == [], (code, detail)
|
||||
assert caplog.text == ""
|
||||
# 只有四个判据分组受 DETAILS 约束;其余 code 的细分是自由文字(ChainProblem / LocateProblem 的 detail)
|
||||
assert P.contract_problems(P.ProvenanceOut(report=ref, segment="s1", status="table_only",
|
||||
reason=P.refuse("no_lineage", "regions 没有 block 键"), sealed=True,
|
||||
version=v)) == []
|
||||
|
||||
|
||||
def _no_row_version() -> P.VersionView:
|
||||
v = full_out().version
|
||||
for part in v.parts:
|
||||
part.has_row = False
|
||||
return v
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 不变式
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_contract_problems_accepts_the_valid_shapes():
|
||||
assert P.contract_problems(full_out()) == []
|
||||
v = full_out().version
|
||||
for part in v.parts:
|
||||
part.has_row = False
|
||||
ref = P.ReportRef(node_id="write", doc_artifact="d")
|
||||
ok = [
|
||||
P.ProvenanceOut(report=ref, segment="s1", status="none", reason=P.refuse("legacy_doc"), sealed=True),
|
||||
P.ProvenanceOut(report=ref, segment="s1", status="none", reason=P.refuse("chain_mismatch"),
|
||||
alert=P.alert("chain_mismatch"), sealed=True),
|
||||
P.ProvenanceOut(report=ref, segment="s1", status="table_only", reason=P.refuse("alias", "alias"),
|
||||
sealed=False, version=v),
|
||||
P.ProvenanceOut(report=ref, segment="s1", status="table_only", reason=P.refuse("chain_mismatch"),
|
||||
alert=P.alert("chain_mismatch"), sealed=True, version=v),
|
||||
]
|
||||
for out in ok:
|
||||
assert P.contract_problems(out) == [], out
|
||||
|
||||
|
||||
def test_contract_problems_catches_broken_shapes():
|
||||
ref = P.ReportRef(node_id="write", doc_artifact="d")
|
||||
v = full_out().version
|
||||
broken = [
|
||||
dataclasses.replace(full_out(), reason=P.refuse("alias", "alias")),
|
||||
dataclasses.replace(full_out(), cell_source=None),
|
||||
dataclasses.replace(full_out(), cell_source=dataclasses.replace(full_out().cell_source, recheck=None)),
|
||||
P.ProvenanceOut(report=ref, segment="s1", status="none", sealed=True),
|
||||
P.ProvenanceOut(report=ref, segment="s1", status="none", reason=P.refuse("db_tampered"), sealed=True),
|
||||
P.ProvenanceOut(report=ref, segment="s1", status="none", reason=P.refuse("legacy_doc"), sealed=True,
|
||||
version=v),
|
||||
P.ProvenanceOut(report=ref, segment="s1", status="table_only", reason=P.refuse("no_pk"), sealed=True),
|
||||
P.ProvenanceOut(report=ref, segment="s1", status="table_only", reason=P.refuse("no_pk"), sealed=True,
|
||||
version=v, checks=full_out().checks),
|
||||
P.ProvenanceOut(report=ref, segment="s1", status="table_only", reason=P.refuse("no_pk"),
|
||||
alert=P.alert("db_tampered"), sealed=True, version=v),
|
||||
]
|
||||
for out in broken:
|
||||
assert P.contract_problems(out), out
|
||||
|
||||
|
||||
def _with_recheck(sql: str, params: list[Any], pk: dict[str, Any]) -> P.ProvenanceOut:
|
||||
out = full_out()
|
||||
out.cell_source.pk = pk
|
||||
out.cell_source.recheck = P.Recheck(sql=sql, params=params, ok=True)
|
||||
return out
|
||||
|
||||
|
||||
def test_recheck_uses_question_marks_in_primary_key_order():
|
||||
"""回查 SQL 的占位符是 ?,params 与 pk 按 primary_key 的顺序一一对应(Recheck 的说明;规格 R16 的 :p0 以此为准)。"""
|
||||
pk = {"日期": "2026-08-20", "时段": "13-14"}
|
||||
good = 'SELECT rowid, "客流" FROM "时段客流" WHERE "日期" = ? AND "时段" = ?'
|
||||
assert P.contract_problems(_with_recheck(good, ["2026-08-20", "13-14"], pk)) == []
|
||||
# 列名里的问号不算占位符
|
||||
assert P.contract_problems(_with_recheck('SELECT rowid, "是否?" FROM "表""甲" WHERE "日期" = ?',
|
||||
["2026-08-05"], {"日期": "2026-08-05"})) == []
|
||||
for sql, params in [
|
||||
('SELECT rowid, "客流" FROM "时段客流" WHERE "日期" = :p0 AND "时段" = :p1', ["2026-08-20", "13-14"]),
|
||||
(good, ["13-14", "2026-08-20"]),
|
||||
(good, ["2026-08-20"]),
|
||||
('SELECT rowid, "客流" FROM "时段客流" WHERE "日期" = ?', ["2026-08-20", "13-14"]),
|
||||
]:
|
||||
assert any("recheck" in p for p in P.contract_problems(_with_recheck(sql, params, pk))), (sql, params)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# recipe_types.OutsideText.hidden
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_outside_text_hidden():
|
||||
names = [f.name for f in dataclasses.fields(T.OutsideText)]
|
||||
assert names == ["sheet", "cell", "text", "kind", "period_source", "hidden"]
|
||||
assert T.OutsideText("客流汇总", "客流汇总!B3", "客流汇总表", "text").hidden is None
|
||||
assert asdict(T.OutsideText("客流汇总", "客流汇总!B3", "注", "text", False, True))["hidden"] is True
|
||||
from app.data.recipe_imports import extraction_from_dict
|
||||
|
||||
old = {"ok": True, "outside_text": [{"sheet": "客流汇总", "cell": "客流汇总!B3", "text": "客流汇总表", "kind": "text",
|
||||
"period_source": False}]}
|
||||
assert extraction_from_dict(old).outside_text[0].hidden is None
|
||||
new = {"ok": True, "outside_text": [{**old["outside_text"][0], "hidden": False}]}
|
||||
assert extraction_from_dict(new).outside_text[0].hidden is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 前端夹具:每一份推断来源的答复都按契约读得回、满足不变式
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _convert(tp: Any, value: Any) -> Any:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(tp, type) and dataclasses.is_dataclass(tp):
|
||||
return _build(tp, value)
|
||||
origin = get_origin(tp)
|
||||
if origin is list:
|
||||
(arg,) = get_args(tp)
|
||||
return [_convert(arg, x) for x in value]
|
||||
if origin in (typing.Union, types.UnionType):
|
||||
for arg in get_args(tp):
|
||||
if isinstance(arg, type) and dataclasses.is_dataclass(arg):
|
||||
return _build(arg, value)
|
||||
return value
|
||||
|
||||
|
||||
def _build(cls: type, data: dict[str, Any]) -> Any:
|
||||
"""按契约把 JSON 读回 dataclass。键必须一个不多、一个不少。"""
|
||||
hints = typing.get_type_hints(cls)
|
||||
names = [f.name for f in dataclasses.fields(cls)]
|
||||
assert set(data) == set(names), (cls.__name__, set(data) ^ set(names))
|
||||
return cls(**{n: _convert(hints[n], data[n]) for n in names})
|
||||
|
||||
|
||||
def _fixture() -> dict[str, Any]:
|
||||
return json.loads(FIXTURE.read_text("utf-8"))
|
||||
|
||||
|
||||
def test_frontend_fixture_answers_follow_the_contract():
|
||||
fx = _fixture()
|
||||
assert fx["provenance"], "夹具里至少有一份推断来源的答复"
|
||||
statuses = set()
|
||||
strict = pydantic.TypeAdapter(P.ProvenanceOut)
|
||||
for sid, body in fx["provenance"].items():
|
||||
if not isinstance(body, dict) or body.get("schema") != P.SCHEMA:
|
||||
continue # 模拟出错的答复({status, json})由检查脚本按原样回放
|
||||
out = _build(P.ProvenanceOut, body)
|
||||
assert asdict(out) == body, sid
|
||||
# 键之外再按类型严格校验一遍:取值不在 Literal 里、数写成字符串、该有的对象是 null 都会报
|
||||
assert asdict(strict.validate_json(json.dumps(body), strict=True)) == body, sid
|
||||
assert P.contract_problems(out) == [], (sid, P.contract_problems(out))
|
||||
assert out.segment == sid
|
||||
if out.reason is not None:
|
||||
assert out.reason.text == P.refuse(out.reason.code, out.reason.detail, 列=_pk_cols(out)).text, sid
|
||||
if out.alert is not None:
|
||||
assert out.alert.text == P.ALERT_TEXT[out.alert.code]
|
||||
statuses.add(out.status)
|
||||
assert statuses == {"inferred", "table_only", "none"}
|
||||
|
||||
|
||||
def _pk_cols(out: P.ProvenanceOut) -> str:
|
||||
m = re.search(r"主键列(([^)]*))", out.reason.text if out.reason else "")
|
||||
return m.group(1) if m else ""
|
||||
|
||||
|
||||
def test_frontend_fixture_canonical_follows_the_a8_rule():
|
||||
"""CanonicalView 的口径(P4-SPEC 2.5、6.2 A8、A18):这一行标签列(dim、合计项)的原文与数据库里的值不同时给出,
|
||||
被引用的是值列也给;相同、或这一行没有标签列(宽表的行标签是指标名,column 为 null)时为 null。"""
|
||||
fx = _fixture()
|
||||
given = set()
|
||||
for sid, body in fx["provenance"].items():
|
||||
cs = body.get("cell_source") if isinstance(body, dict) else None
|
||||
if not cs:
|
||||
continue
|
||||
if cs["column_role"] == "dim":
|
||||
# 被引用的就是标签格:原文不在 from 里(from 只列其余的格),只核对规范写法那一侧
|
||||
if cs["canonical"] is not None:
|
||||
assert cs["canonical"]["canonical"] == cs["pk"][cs["column"]], sid
|
||||
given.add(sid)
|
||||
continue
|
||||
labels = [f for f in cs["from"] if f["role"] in ("row_label", "total_label") and f["column"] is not None]
|
||||
assert len(labels) <= 1, sid
|
||||
if labels and labels[0]["text"] is not None and labels[0]["text"] != cs["pk"][labels[0]["column"]]:
|
||||
assert cs["canonical"] == {"raw": labels[0]["text"], "canonical": cs["pk"][labels[0]["column"]]}, sid
|
||||
given.add(sid)
|
||||
else:
|
||||
assert cs["canonical"] is None, sid
|
||||
# 合计表那一格(18-22 时合计)、累积并集里全角标签那一期的值格和标签格(8-9)
|
||||
cases = fx["cases"]
|
||||
assert {cases[k]["segment"] for k in ("total", "union", "fullwidth")} <= given, given
|
||||
assert fx["provenance"][cases["total"]["segment"]]["cell_source"]["canonical"] == {
|
||||
"raw": "18-22 时合计", "canonical": "18-22时合计"}
|
||||
assert fx["provenance"][cases["union"]["segment"]]["cell_source"]["canonical"] == {"raw": "8-9", "canonical": "8-9"}
|
||||
|
||||
|
||||
def test_frontend_fixture_cases_point_at_real_segments():
|
||||
"""检查脚本按片段文字认片段:cases 里每个场景的文字在文档里恰好对应一个片段,编号与之一致。"""
|
||||
fx = _fixture()
|
||||
segs = [s for b in fx["doc"]["blocks"] for u in b["units"] for s in u["segments"]]
|
||||
assert fx["doc"].get("provenance") == P.DOC_PROVENANCE
|
||||
for key, case in fx["cases"].items():
|
||||
hits = [s for s in segs if s["text"] == case["text"] and s.get("cite")]
|
||||
assert len(hits) == 1, (key, case["text"], len(hits))
|
||||
assert hits[0]["id"] == case["segment"], key
|
||||
assert case["segment"] in fx["segments"], key
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 前端类型(frontend/src/types.ts)与契约逐个字段对齐:合并后 types.ts 归 WP-D,改名会在这里暴露
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TYPES_TS = ROOT / "frontend" / "src" / "types.ts"
|
||||
TS_INTERFACES = {
|
||||
"EvidenceProvenance": "ProvenanceOut", "ProvenanceReportRef": "ReportRef", "ProvenanceCellRef": "CellRef",
|
||||
"ProvenanceReason": "Reason", "ProvenanceAlert": "Alert", "ProvenanceTableRef": "TableRef",
|
||||
"ProvenancePeriod": "PeriodView", "ProvenanceAcceptance": "AcceptanceView", "ProvenanceStateNote": "StateNote",
|
||||
"ProvenancePart": "PartView", "ProvenanceVersion": "VersionView", "ProvenanceFromCell": "FromCell",
|
||||
"ProvenanceYear": "YearSource", "ProvenanceCanonical": "CanonicalView", "ProvenanceRecheck": "Recheck",
|
||||
"ProvenanceCellSource": "CellSource", "ProvenanceCheck": "RelatedCheck",
|
||||
}
|
||||
TS_UNIONS = {
|
||||
"ProvenanceStatus": P.Status, "ProvenanceReasonCode": P.ReasonCode, "ProvenanceAlertCode": P.AlertCode,
|
||||
"ProvenanceFromRole": P.FromRole, "ProvenancePartStatus": P.PartStatus, "ProvenanceRowStatus": P.RowStatus,
|
||||
"ProvenanceCellStatus": P.CellStatus,
|
||||
}
|
||||
|
||||
|
||||
#: 有意与 Python 不同的几处(字段 → TS 里应写的联合成员):
|
||||
#: - rowid、recheck 只在 inferred 时有、那时必填(contract_problems 钉住),TS 写成非空,界面不用到处判空;
|
||||
#: - schema、column_role 带开放的尾巴 `(string & {})`:以后加取值时老界面照样能编译、能显示原值
|
||||
TS_OVERRIDES: dict[tuple[str, str], tuple[str, ...]] = {
|
||||
("ProvenanceCellSource", "rowid"): ("number",),
|
||||
("ProvenanceCellSource", "recheck"): ("ProvenanceRecheck",),
|
||||
("EvidenceProvenance", "schema"): (f"'{P.SCHEMA}'", "(string & {})"),
|
||||
("ProvenanceCellSource", "column_role"): (*(f"'{x}'" for x in get_args(P.ColumnRole)), "(string & {})"),
|
||||
}
|
||||
|
||||
|
||||
def _ts_fields(src: str, name: str) -> list[str]:
|
||||
return list(_ts_field_types(src, name))
|
||||
|
||||
|
||||
def _ts_field_types(src: str, name: str) -> dict[str, str]:
|
||||
"""接口的字段 → 类型原文(`name: type`、`name?: type`,可选的在键名后保留问号)。"""
|
||||
m = re.search(rf"^export interface {name} \{{\n(.*?)^\}}", src, re.S | re.M)
|
||||
assert m, name
|
||||
return {f + q: t.strip() for f, q, t in re.findall(r"^ (\w+)(\??): (.+)$", m.group(1), re.M)}
|
||||
|
||||
|
||||
def _ts_aliases(src: str) -> dict[str, tuple[str, ...]]:
|
||||
"""TS_UNIONS 里的具名联合 → 字面量成员(带引号,同 _ts_members 的写法)。"""
|
||||
out = {}
|
||||
for ts in TS_UNIONS:
|
||||
m = re.search(rf"^export type {ts} =((?:[^\n]*\n)(?:\s+\|[^\n]*\n)*)", src, re.M)
|
||||
assert m, ts
|
||||
out[ts] = tuple(f"'{x}'" for x in re.findall(r"'([^']+)'", m.group(1)))
|
||||
return out
|
||||
|
||||
|
||||
def _ts_union(text: str, aliases: dict[str, tuple[str, ...]]) -> tuple[str, ...]:
|
||||
"""TS 类型原文 → 联合成员(顶层按「|」拆,具名联合展开成字面量)。这一节的类型里没有带「|」的泛型。"""
|
||||
out: list[str] = []
|
||||
for member in (m.strip() for m in text.split("|")):
|
||||
out.extend(aliases.get(member, (member,)))
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _ts_members(tp: Any) -> tuple[str, ...]:
|
||||
"""Python 注解 → 应写的 TS 联合成员:str/int/bool/Any → string/number/boolean/unknown,Literal → 带引号的取值,
|
||||
X | None → X 的成员加 null,list[X] → X[],dict[str, X] → Record<string, X>,dataclass → 对应的 TS 接口名。"""
|
||||
if tp is type(None):
|
||||
return ("null",)
|
||||
simple = {str: "string", bool: "boolean", int: "number", Any: "unknown"}
|
||||
if tp in simple:
|
||||
return (simple[tp],)
|
||||
if isinstance(tp, type) and dataclasses.is_dataclass(tp):
|
||||
return (TS_NAME[tp.__name__],)
|
||||
origin = get_origin(tp)
|
||||
if origin is typing.Literal:
|
||||
return tuple(f"'{x}'" for x in get_args(tp))
|
||||
if origin in (typing.Union, types.UnionType):
|
||||
return tuple(m for arg in get_args(tp) for m in _ts_members(arg))
|
||||
if origin is list:
|
||||
(inner,) = (_ts_members(a) for a in get_args(tp))
|
||||
assert len(inner) == 1, tp
|
||||
return (f"{inner[0]}[]",)
|
||||
if origin is dict:
|
||||
key, value = get_args(tp)
|
||||
assert key is str, tp
|
||||
return (f"Record<string, {_ts_members(value)[0]}>",)
|
||||
raise AssertionError(f"契约里出现了没教过的注解:{tp!r}")
|
||||
|
||||
|
||||
TS_NAME = {py: ts for ts, py in TS_INTERFACES.items()}
|
||||
|
||||
|
||||
def test_frontend_types_match_the_contract():
|
||||
"""字段名、顺序之外,逐个字段比类型:取值联合(含行内的 'override' | 'waiver' 这种)、可空性、数组、嵌套接口。
|
||||
合并后 types.ts 归 WP-D,这里是唯一能发现它和服务端契约漂开的地方。"""
|
||||
_ts_alignment(TYPES_TS.read_text("utf-8"))
|
||||
|
||||
|
||||
def _ts_alignment(src: str) -> None:
|
||||
aliases = _ts_aliases(src)
|
||||
for ts, lit in TS_UNIONS.items():
|
||||
assert aliases[ts] == tuple(f"'{x}'" for x in get_args(lit)), ts
|
||||
for ts, py in TS_INTERFACES.items():
|
||||
cls = getattr(P, py)
|
||||
got = _ts_field_types(src, ts)
|
||||
names = [f.name for f in dataclasses.fields(cls)]
|
||||
# 答复里每个键都在(值可以是 null),TS 一律不写成可选
|
||||
assert list(got) == names, ts
|
||||
hints = typing.get_type_hints(cls)
|
||||
for name in names:
|
||||
want = TS_OVERRIDES.get((ts, name), _ts_members(hints[name]))
|
||||
assert _ts_union(got[name], aliases) == want, (ts, name, got[name], want)
|
||||
# 白名单不能烂掉:有意写成非空的两项在 Python 里确实可空;开放尾巴的两项 Python 一侧分别是 str、ColumnRole
|
||||
cs_hints = typing.get_type_hints(P.CellSource)
|
||||
assert _ts_members(cs_hints["rowid"]) == ("number", "null")
|
||||
assert _ts_members(cs_hints["recheck"]) == ("ProvenanceRecheck", "null")
|
||||
assert typing.get_type_hints(P.ProvenanceOut)["schema"] is str
|
||||
assert get_args(cs_hints["column_role"]) == get_args(P.ColumnRole)
|
||||
step = re.search(r"^export interface EvidenceStep .*?^\}", src, re.S | re.M).group(0)
|
||||
assert re.search(r"^ provenance\?: boolean$", step, re.M)
|
||||
outside = re.search(r"^export interface OutsideText \{.*?^\}", src, re.S | re.M).group(0)
|
||||
assert re.search(r"^ hidden\?: boolean \| null$", outside, re.M)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("old,new", [
|
||||
# 评审实测过抓不到的两处,加上几种同类的改法:都必须让上面的对齐测试失败
|
||||
(" kind: 'override' | 'waiver'\n", " kind: 'overrides' | 'waivers'\n"),
|
||||
(" rowid: number\n", " rowid: string\n"),
|
||||
(" raw_state: 'kept' | 'purged' | 'absent' | null\n", " raw_state: 'kept' | 'purged' | null\n"),
|
||||
(" mode: 'replace' | 'accumulate'\n", " mode: 'replace' | 'accumulate' | 'append'\n"),
|
||||
(" source: 'period' | 'human'\n", " source: 'cells' | 'human'\n"),
|
||||
(" period: ProvenancePeriod | null\n", " period: ProvenancePeriod\n"),
|
||||
(" title: string | null\n kind: 'override'", " title: string\n kind: 'override'"),
|
||||
(" params: unknown[]\n", " params: string\n"),
|
||||
(" row_status: ProvenanceRowStatus | null\n", " row_status: ProvenanceCellStatus | null\n"),
|
||||
(" column_role: 'axis' | 'dim'", " column_role: 'axis' | 'dims'"),
|
||||
])
|
||||
def test_frontend_type_alignment_catches_drift(old, new):
|
||||
src = TYPES_TS.read_text("utf-8")
|
||||
start = src.index("// 期 4:推断的来源")
|
||||
assert old in src[start:], old
|
||||
with pytest.raises(AssertionError):
|
||||
_ts_alignment(src[:start] + src[start:].replace(old, new, 1))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 金样:工件 id 等于内容哈希(被人改过一个字就对不上)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_golden_is_content_addressed():
|
||||
from app.core.artifact_store import canonical_json, content_hash
|
||||
|
||||
g = json.loads(GOLDEN.read_text("utf-8"))
|
||||
assert "2cdbf9a" in g["_comment"] and g["commit"] == "2cdbf9a"
|
||||
assert g["script"].endswith("p4-golden/gen.py")
|
||||
for aid, content in g["artifacts"].items():
|
||||
assert content_hash(canonical_json(content)) == aid
|
||||
assert set(g["reports"]) == {"recipe", "simple", "manual"}
|
||||
for rep in g["reports"].values():
|
||||
doc = rep["outputs"]["doc"]
|
||||
assert content_hash(canonical_json(doc)) == rep["doc_artifact"]
|
||||
assert "provenance" not in doc
|
||||
assert set(rep["artifacts"]) <= set(g["artifacts"])
|
||||
assert rep["outputs"]["chain"] and rep["outputs"]["excerpts"]
|
||||
@@ -0,0 +1,497 @@
|
||||
"""证据下钻(期 4)里碰数据库和数据文件的几步:provenance_db 的各函数,engine.py 的两处小改,sum_eq_conditions。
|
||||
|
||||
数据文件都是测试里现造的小 SQLite 库(表名「日客流」「时段客流」,分区甲、分区乙,数字随机),数据文件记录
|
||||
直接写进测试库的 source_snapshots,不经导入管线:这里要钉的是回查、编译核对、强制重算、行级核对各自的口径,
|
||||
整条链(真导入、真运行)由 WP-E 的验收覆盖。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import random
|
||||
import sqlite3
|
||||
import threading
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app.data import engine as engine_mod
|
||||
from app.data import provenance_db
|
||||
from app.data import recipe_checks
|
||||
from app.data.engine import EngineCache, SnapshotTampered, cache_key, engines, open_checked_sqlite
|
||||
from app.data.provenance_types import _QUOTED, CompileFacts, StateNote, SumEqRule
|
||||
from app.data.recipe_checks import sum_eq_conditions
|
||||
from app.data.recipe_types import ColumnOut, SumEq
|
||||
from app.data.table_versions import SnapshotManifestMismatch, SnapshotMissing
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import DataSource, SourceSnapshot, TableImport, TableRecipe
|
||||
|
||||
RNG = random.Random(4004)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 夹具
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _sha(path) -> str:
|
||||
with open(path, "rb") as f:
|
||||
return hashlib.sha256(f.read()).hexdigest()
|
||||
|
||||
|
||||
def _build(path, *, daily, hourly=(), real=()) -> str:
|
||||
"""造一个快照库:日客流(单列主键,写成 PRIMARY KEY DESC,和执行器一样不让它成为 rowid 的别名)、
|
||||
时段客流(两列主键)、分区占比(REAL 列)。"""
|
||||
conn = sqlite3.connect(path)
|
||||
conn.executescript("""
|
||||
CREATE TABLE "日客流" ("日期" TEXT NOT NULL, "全日客流" INTEGER, "分区甲" INTEGER, "分区乙" INTEGER,
|
||||
PRIMARY KEY ("日期" DESC));
|
||||
CREATE TABLE "时段客流" ("日期" TEXT NOT NULL, "时段" TEXT NOT NULL, "客流" INTEGER,
|
||||
PRIMARY KEY ("日期", "时段"));
|
||||
CREATE TABLE "分区占比" ("日期" TEXT NOT NULL, "合计" REAL, "分区甲" REAL, "分区乙" REAL,
|
||||
PRIMARY KEY ("日期" DESC));
|
||||
""")
|
||||
conn.executemany('INSERT INTO "日客流" VALUES (?, ?, ?, ?)', list(daily))
|
||||
conn.executemany('INSERT INTO "时段客流" VALUES (?, ?, ?)', list(hourly))
|
||||
conn.executemany('INSERT INTO "分区占比" VALUES (?, ?, ?, ?)', list(real))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
os.chmod(path, 0o444)
|
||||
return str(path)
|
||||
|
||||
|
||||
def _daily_rows(n=6):
|
||||
"""前几行成立,第 3 行不成立(合计多 1),第 5 行分区乙为空。"""
|
||||
rows = []
|
||||
for i in range(n):
|
||||
a, b = RNG.randint(1000, 9000), RNG.randint(1000, 9000)
|
||||
total = a + b
|
||||
if i == 2:
|
||||
total += 1
|
||||
rows.append((f"2026-08-{i + 1:02d}", total, a, None if i == 4 else b))
|
||||
return rows
|
||||
|
||||
|
||||
async def _register(path, *, source_id=None, retired=False, sha=None, manifest=None) -> SourceSnapshot:
|
||||
snap = SourceSnapshot(
|
||||
id=uuid.uuid4().hex + uuid.uuid4().hex, source_id=source_id or uuid.uuid4().hex,
|
||||
db_path=str(path), db_sha256=_sha(path) if sha is None else sha, schema_cache={},
|
||||
retired_at=datetime.now(timezone.utc) if retired else None, manifest_artifact=manifest,
|
||||
)
|
||||
async with SessionLocal() as session:
|
||||
session.add(snap)
|
||||
await session.commit()
|
||||
await session.refresh(snap)
|
||||
return snap
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def snapshot(tmp_path):
|
||||
"""一个登记好的快照库,数据源记录不存在(模拟数据源已删除)。用完清掉引擎缓存。"""
|
||||
path = _build(tmp_path / "snap.db", daily=_daily_rows(),
|
||||
hourly=[("2026-08-03", "7-8", None), ("2026-08-03", "8-9", 321)],
|
||||
real=[("2026-08-01", 0.3, 0.1, 0.2), ("2026-08-02", 1.5, 0.5, 0.75)])
|
||||
snap = await _register(path)
|
||||
yield snap
|
||||
await engines.invalidate(snap.source_id)
|
||||
|
||||
|
||||
async def _view(snap: SourceSnapshot):
|
||||
return provenance_db.view_of(snap, source_name="flow_demo")
|
||||
|
||||
|
||||
def _poke(path: str, *, keep_mtime: bool) -> None:
|
||||
"""原地改一个字节(大小不变)。keep_mtime 时再用 os.utime 还原修改时间:EngineCache 的指纹认不出这种改动。"""
|
||||
st = os.stat(path)
|
||||
os.chmod(path, 0o644)
|
||||
with open(path, "r+b") as f:
|
||||
f.seek(-1, os.SEEK_END)
|
||||
last = f.read(1)
|
||||
f.seek(-1, os.SEEK_END)
|
||||
f.write(bytes([last[0] ^ 0x01]))
|
||||
os.chmod(path, 0o444)
|
||||
if keep_mtime:
|
||||
os.utime(path, ns=(st.st_atime_ns, st.st_mtime_ns))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# sum_eq_conditions:导入时的核对和行级核对共用一段条件
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_sum_eq_conditions_integer_and_real():
|
||||
nonnull, anynull, mismatch = sum_eq_conditions("全日客流", ["分区甲", "分区乙"],
|
||||
{"全日客流": "INTEGER", "分区甲": "INTEGER"})
|
||||
assert nonnull == '"全日客流" IS NOT NULL AND "分区甲" IS NOT NULL AND "分区乙" IS NOT NULL'
|
||||
assert anynull == '"全日客流" IS NULL OR "分区甲" IS NULL OR "分区乙" IS NULL'
|
||||
assert mismatch == '"全日客流" <> "分区甲" + "分区乙"'
|
||||
# 任一列是 REAL 就按相对误差比:0.1 + 0.2 不严格等于 0.3
|
||||
_, _, real = sum_eq_conditions("合计", ["分区甲", "分区乙"], {"分区乙": "REAL"})
|
||||
assert real == 'ABS("合计" - ("分区甲" + "分区乙")) > 1e-9 * MAX(1, ABS("合计"))'
|
||||
# 名字里的双引号照样转义(DQS 关闭的连接上写错的名字直接报错)
|
||||
assert sum_eq_conditions('a"b', ["c", "d"], {})[2] == '"a""b" <> "c" + "d"'
|
||||
|
||||
|
||||
@pytest.mark.parametrize("table,total,parts,types", [
|
||||
("日客流", "全日客流", ["分区甲", "分区乙"], {"全日客流": "INTEGER", "分区甲": "INTEGER", "分区乙": "INTEGER"}),
|
||||
("分区占比", "合计", ["分区甲", "分区乙"], {"合计": "REAL", "分区甲": "REAL", "分区乙": "REAL"}),
|
||||
])
|
||||
async def test_row_status_agrees_with_the_import_check(snapshot, table, total, parts, types):
|
||||
"""逐行的 row_status 和导入时的 _check_sum_eq 在同一个库上的结论对得上:成立、不成立、含空值的行数都相同。
|
||||
REAL 表里 0.1 + 0.2 = 0.3 那一行两边都判成立(改成严格比较的条件会被这里抓到)。"""
|
||||
view = await _view(snapshot)
|
||||
rule = SumEqRule(table=table, total=total, parts=parts, types=types)
|
||||
conn = open_checked_sqlite(snapshot.db_path)
|
||||
try:
|
||||
rowids = [r[0] for r in conn.execute(f'SELECT rowid FROM "{table}" ORDER BY rowid')]
|
||||
result = recipe_checks._check_sum_eq(
|
||||
recipe_checks._Sql(conn), SumEq(id="R1", kind="sum_eq", table=table, total=total, parts=parts), "核对",
|
||||
[ColumnOut(name=n, type=t) for n, t in types.items()], ["日期"], {})
|
||||
finally:
|
||||
conn.close()
|
||||
statuses = [await provenance_db.row_status(view, rule, rid) for rid in rowids]
|
||||
assert statuses.count("mismatch") == result.failed
|
||||
assert statuses.count("unverifiable") == result.unverifiable
|
||||
assert statuses.count("passed") + statuses.count("mismatch") == result.checked
|
||||
if table == "日客流":
|
||||
assert statuses == ["passed", "passed", "mismatch", "passed", "unverifiable", "passed"]
|
||||
else:
|
||||
assert statuses == ["passed", "mismatch"]
|
||||
|
||||
|
||||
async def test_row_status_follows_the_snapshot_rowid_it_is_given(snapshot):
|
||||
"""按传进来的快照库 rowid 判,不按别的编号:同一条规则,第 3 行不成立、第 4 行成立。"""
|
||||
view = await _view(snapshot)
|
||||
rule = SumEqRule(table="日客流", total="全日客流", parts=["分区甲", "分区乙"], types={})
|
||||
assert await provenance_db.row_status(view, rule, 3) == "mismatch"
|
||||
assert await provenance_db.row_status(view, rule, 4) == "passed"
|
||||
|
||||
|
||||
async def test_row_status_is_none_when_the_row_or_a_column_is_not_there(snapshot):
|
||||
view = await _view(snapshot)
|
||||
assert await provenance_db.row_status(
|
||||
view, SumEqRule(table="日客流", total="全日客流", parts=["分区甲", "分区乙"], types={}), 999) is None
|
||||
assert await provenance_db.row_status(
|
||||
view, SumEqRule(table="日客流", total="全日客流", parts=["分区甲", "分区丙"], types={}), 1) is None
|
||||
assert await provenance_db.row_status(
|
||||
view, SumEqRule(table="没有这张表", total="a", parts=["b", "c"], types={}), 1) is None
|
||||
# 名字只对 ASCII 不区分大小写(和 SQLite 一致),中文名原样比
|
||||
assert await provenance_db.row_status(
|
||||
view, SumEqRule(table="日客流", total="全日客流", parts=["分区甲", "分区乙"], types={}), 1) == "passed"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 数据文件记录与绑定
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_snapshot_record(snapshot):
|
||||
async with SessionLocal() as session:
|
||||
got = await provenance_db.snapshot_record(session, snapshot.id)
|
||||
assert got is not None and got.id == snapshot.id and got.db_path == snapshot.db_path
|
||||
assert await provenance_db.snapshot_record(session, "f" * 64) is None
|
||||
assert await provenance_db.snapshot_record(session, "") is None
|
||||
|
||||
|
||||
async def test_view_of_uses_a_stand_in_keyed_like_the_query(snapshot):
|
||||
"""数据源记录不存在(已删除)也能绑定;id 用 snapshot.source_id,缓存键和查询时相同,首次哈希只算一次。"""
|
||||
async with SessionLocal() as session:
|
||||
assert await session.get(DataSource, snapshot.source_id) is None
|
||||
view = provenance_db.view_of(snapshot, source_name="flow_demo")
|
||||
assert view.id == snapshot.source_id and view.name == "flow_demo"
|
||||
assert view.snapshot_id == snapshot.id and view.database == snapshot.db_path
|
||||
assert view.immutable is True and view.readonly is True
|
||||
assert view.expected_sha256 == snapshot.db_sha256
|
||||
assert view.options == {} and view.description == "" and view.enabled is True
|
||||
assert cache_key(view) == f"{snapshot.source_id}:{snapshot.id}"
|
||||
|
||||
|
||||
async def test_view_of_refuses_unregistered_hash_and_gone_files(tmp_path):
|
||||
path = _build(tmp_path / "a.db", daily=_daily_rows(2))
|
||||
with pytest.raises(SnapshotMissing):
|
||||
provenance_db.view_of(await _register(path, sha=""), source_name="flow_demo")
|
||||
with pytest.raises(SnapshotMissing):
|
||||
provenance_db.view_of(await _register(path, retired=True), source_name="flow_demo")
|
||||
gone = await _register(path)
|
||||
os.chmod(path, 0o644)
|
||||
os.remove(path)
|
||||
# 文件不在:绑定前就认出来(snapshot_gone,不标红),不留给 EngineCache 报成「文件被改」
|
||||
with pytest.raises(SnapshotMissing) as info:
|
||||
provenance_db.view_of(gone, source_name="flow_demo")
|
||||
assert not isinstance(info.value, SnapshotTampered)
|
||||
|
||||
|
||||
async def test_view_of_raises_manifest_mismatch_untouched(tmp_path):
|
||||
"""快照清单里的库哈希和登记的不一致:bind_snapshot 抛的 SnapshotManifestMismatch 原样抛出。"""
|
||||
from app.core import artifact_store
|
||||
|
||||
path = _build(tmp_path / "u.db", daily=_daily_rows(2))
|
||||
mid = await artifact_store.put_json({"snapshot_id": None, "union": {"db_sha256": "0" * 64}},
|
||||
kind="snapshot_manifest")
|
||||
snap = await _register(path, manifest=mid)
|
||||
with pytest.raises(SnapshotManifestMismatch):
|
||||
provenance_db.view_of(snap, source_name="flow_demo")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# R16:回查
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_recheck_sql_params_and_rows(snapshot):
|
||||
view = await _view(snapshot)
|
||||
rows, sql, params = await provenance_db.recheck(view, table="日客流", column="全日客流", pk={"日期": "2026-08-02"})
|
||||
assert sql == 'SELECT rowid, "全日客流" FROM "日客流" WHERE "日期" = ?'
|
||||
assert params == ["2026-08-02"]
|
||||
conn = sqlite3.connect(snapshot.db_path)
|
||||
want = conn.execute('SELECT rowid, "全日客流" FROM "日客流" WHERE "日期" = ?', ["2026-08-02"]).fetchall()
|
||||
conn.close()
|
||||
assert rows == want and isinstance(rows[0][1], int)
|
||||
# 契约 Recheck 的口径:去掉双引号里的标识符后 ? 的个数等于主键列数
|
||||
assert _QUOTED.sub("", sql).count("?") == 1
|
||||
|
||||
|
||||
async def test_recheck_composite_key_keeps_primary_key_order_and_types(snapshot):
|
||||
view = await _view(snapshot)
|
||||
rows, sql, params = await provenance_db.recheck(view, table="时段客流", column="客流",
|
||||
pk={"日期": "2026-08-03", "时段": "8-9"})
|
||||
assert sql == 'SELECT rowid, "客流" FROM "时段客流" WHERE "日期" = ? AND "时段" = ?'
|
||||
assert params == ["2026-08-03", "8-9"] and [v for _, v in rows] == [321]
|
||||
# 空值照实回(R11 已拦下空值,这里只回事实);REAL 回 float,不和 int 混
|
||||
rows, _, _ = await provenance_db.recheck(view, table="时段客流", column="客流", pk={"日期": "2026-08-03", "时段": "7-8"})
|
||||
assert [v for _, v in rows] == [None]
|
||||
rows, _, _ = await provenance_db.recheck(view, table="分区占比", column="合计", pk={"日期": "2026-08-02"})
|
||||
assert rows[0][1] == 1.5 and isinstance(rows[0][1], float)
|
||||
rows, _, _ = await provenance_db.recheck(view, table="日客流", column="全日客流", pk={"日期": "1999-01-01"})
|
||||
assert rows == []
|
||||
|
||||
|
||||
async def test_recheck_quotes_identifiers_with_double_quotes(tmp_path):
|
||||
path = tmp_path / "q.db"
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute('CREATE TABLE "表""甲" ("键""一" TEXT NOT NULL, "值?" INTEGER, PRIMARY KEY ("键""一" DESC))')
|
||||
conn.execute('INSERT INTO "表""甲" VALUES (?, ?)', ["k", 7])
|
||||
conn.commit()
|
||||
conn.close()
|
||||
snap = await _register(path)
|
||||
try:
|
||||
rows, sql, params = await provenance_db.recheck(
|
||||
await _view(snap), table='表"甲', column="值?", pk={'键"一': "k"})
|
||||
assert sql == 'SELECT rowid, "值?" FROM "表""甲" WHERE "键""一" = ?'
|
||||
assert [v for _, v in rows] == [7] and params == ["k"]
|
||||
assert _QUOTED.sub("", sql).count("?") == 1
|
||||
finally:
|
||||
await engines.invalidate(snap.source_id)
|
||||
|
||||
|
||||
async def test_recheck_raises_tampered_when_the_file_changed(snapshot):
|
||||
"""改过一个字节(修改时间也变了):经 EngineCache 取引擎时重新核对哈希,SnapshotTampered 原样抛出。"""
|
||||
view = await _view(snapshot)
|
||||
await provenance_db.recheck(view, table="日客流", column="全日客流", pk={"日期": "2026-08-01"})
|
||||
_poke(snapshot.db_path, keep_mtime=False)
|
||||
with pytest.raises(SnapshotTampered):
|
||||
await provenance_db.recheck(view, table="日客流", column="全日客流", pk={"日期": "2026-08-01"})
|
||||
|
||||
|
||||
async def test_recheck_needs_a_primary_key(snapshot):
|
||||
with pytest.raises(ValueError):
|
||||
await provenance_db.recheck(await _view(snapshot), table="日客流", column="全日客流", pk={})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# R16:强制重算
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_rehash_catches_an_in_place_edit_that_keeps_the_fingerprint(snapshot, monkeypatch):
|
||||
"""原地改一个字节、再还原修改时间:EngineCache 的指纹不变、照样给缓存的引擎,强制重算认得出(R16,A25②)。
|
||||
重算放在线程里。"""
|
||||
view = await _view(snapshot)
|
||||
await engines.get(view)
|
||||
assert await provenance_db.rehash(view) is True
|
||||
threads: list[bool] = []
|
||||
real = engine_mod._file_sha256
|
||||
|
||||
def spy(path):
|
||||
threads.append(threading.current_thread() is threading.main_thread())
|
||||
return real(path)
|
||||
|
||||
monkeypatch.setattr(engine_mod, "_file_sha256", spy)
|
||||
before = engines.pinned(view)
|
||||
_poke(snapshot.db_path, keep_mtime=True)
|
||||
assert engine_mod._fingerprint(snapshot.db_path) == before
|
||||
await engines.get(view) # 指纹没变,缓存命中,不重新核对
|
||||
assert await provenance_db.rehash(view) is False
|
||||
assert threads and not any(threads)
|
||||
|
||||
|
||||
async def test_rehash_is_false_without_a_hash_or_a_file(snapshot):
|
||||
view = await _view(snapshot)
|
||||
view.expected_sha256 = None
|
||||
assert await provenance_db.rehash(view) is False
|
||||
view = await _view(snapshot)
|
||||
view.database = str(snapshot.db_path) + ".missing"
|
||||
assert await provenance_db.rehash(view) is False
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# R15:编译核对
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sql,tables,rows,yields", [
|
||||
('SELECT "日期", "全日客流" FROM "日客流" WHERE "日期" = \'2026-08-01\'', {"日客流"}, 1, 0),
|
||||
('SELECT * FROM "日客流" ORDER BY "全日客流" DESC LIMIT 3', {"日客流"}, 1, 0),
|
||||
('SELECT "日期","全日客流" FROM "日客流" UNION ALL SELECT "日期","分区甲" FROM "日客流"', {"日客流"}, 2, 0),
|
||||
('SELECT "日期","全日客流" FROM "日客流" WHERE "日期" LIKE \'2026%\' ESCAPE \'\\\' '
|
||||
'UNION ALL SELECT "日期","分区甲" FROM "日客流"', {"日客流"}, 2, 0),
|
||||
('SELECT "日期","全日客流" FROM "日客流" WHERE "日期" IN (SELECT "日期" FROM "时段客流")',
|
||||
{"日客流", "时段客流"}, 1, 0),
|
||||
])
|
||||
async def test_compile_facts_reports_tables_and_result_rows(snapshot, sql, tables, rows, yields):
|
||||
facts = await provenance_db.compile_facts(await _view(snapshot), sql)
|
||||
assert facts == CompileFacts(tables=tables, result_rows=rows, yields=yields)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("op", ["UNION", "INTERSECT", "EXCEPT"])
|
||||
async def test_compile_facts_sees_yield_in_set_operations(snapshot, op):
|
||||
facts = await provenance_db.compile_facts(
|
||||
await _view(snapshot), f'SELECT "日期","全日客流" FROM "日客流" {op} SELECT "日期","分区甲" FROM "日客流"')
|
||||
assert facts.yields >= 1
|
||||
|
||||
|
||||
async def test_compile_facts_runs_in_a_thread_on_an_immutable_read_only_connection(snapshot, monkeypatch):
|
||||
seen: list[dict] = []
|
||||
real_open = provenance_db.open_checked_sqlite
|
||||
|
||||
def spy(path, **kw):
|
||||
seen.append({"path": path, **kw, "main": threading.current_thread() is threading.main_thread()})
|
||||
return real_open(path, **kw)
|
||||
|
||||
monkeypatch.setattr(provenance_db, "open_checked_sqlite", spy)
|
||||
await provenance_db.compile_facts(await _view(snapshot), 'SELECT "日期" FROM "日客流"')
|
||||
assert seen == [{"path": snapshot.db_path, "readonly": True, "immutable": True, "main": False}]
|
||||
|
||||
|
||||
async def test_compile_facts_compares_against_the_engine_cache_fingerprint(snapshot, monkeypatch):
|
||||
"""开连接前的指纹和 EngineCache 核对过的那份不等:SnapshotTampered(→ db_tampered)。"""
|
||||
view = await _view(snapshot)
|
||||
monkeypatch.setattr(engines, "pinned", lambda source: (0, 0, 0, 0))
|
||||
with pytest.raises(SnapshotTampered):
|
||||
await provenance_db.compile_facts(view, 'SELECT "日期" FROM "日客流"')
|
||||
|
||||
|
||||
async def test_compile_facts_checks_the_fingerprint_again_after_closing(snapshot, monkeypatch):
|
||||
"""编译期间文件被换(关连接后的指纹对不上):同样 SnapshotTampered,不交出编译结果。"""
|
||||
view = await _view(snapshot)
|
||||
real = provenance_db._fingerprint
|
||||
calls: list[int] = []
|
||||
|
||||
def drifting(path):
|
||||
calls.append(1)
|
||||
fp = real(path)
|
||||
return fp if len(calls) == 1 else (fp[0], fp[1] + 1, fp[2], fp[3])
|
||||
|
||||
monkeypatch.setattr(provenance_db, "_fingerprint", drifting)
|
||||
with pytest.raises(SnapshotTampered):
|
||||
await provenance_db.compile_facts(view, 'SELECT "日期" FROM "日客流"')
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
async def test_compile_facts_without_a_pinned_engine_is_snapshot_missing(snapshot, monkeypatch):
|
||||
monkeypatch.setattr(engines, "pinned", lambda source: None)
|
||||
with pytest.raises(SnapshotMissing) as info:
|
||||
await provenance_db.compile_facts(await _view(snapshot), 'SELECT "日期" FROM "日客流"')
|
||||
assert not isinstance(info.value, SnapshotTampered)
|
||||
|
||||
|
||||
async def test_compile_facts_refuses_a_replaced_file(snapshot):
|
||||
"""引擎已在缓存里、文件随后被改(修改时间变了):取引擎时就重新核对,SnapshotTampered。"""
|
||||
view = await _view(snapshot)
|
||||
await provenance_db.compile_facts(view, 'SELECT "日期" FROM "日客流"')
|
||||
_poke(snapshot.db_path, keep_mtime=False)
|
||||
with pytest.raises(SnapshotTampered):
|
||||
await provenance_db.compile_facts(view, 'SELECT "日期" FROM "日客流"')
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 导入记录上的当前状态
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_import_states():
|
||||
sid = uuid.uuid4().hex
|
||||
async with SessionLocal() as session:
|
||||
recipe = TableRecipe(source_id=sid, seq=3, recipe={}, recipe_sha256="a" * 64)
|
||||
session.add(recipe)
|
||||
await session.flush()
|
||||
kept = TableImport(source_id=sid, seq=1, raw_state="kept", recipe_id=recipe.id)
|
||||
purged = TableImport(source_id=sid, seq=2, raw_state="purged", recipe_id="nosuchrecipe",
|
||||
purged={"at": "2026-09-30T01:02:03+00:00", "reason": "合成理由:清除原件",
|
||||
"signed_by": "录入员甲", "signed_by_verified": False})
|
||||
revoked = TableImport(source_id=sid, seq=3, raw_state="kept",
|
||||
revoked={"at": "2026-10-01T00:00:00+00:00", "reason": "合成理由:作废",
|
||||
"signed_by": None, "signed_by_verified": False})
|
||||
session.add_all([kept, purged, revoked])
|
||||
await session.commit()
|
||||
ids = [kept.id, purged.id, revoked.id]
|
||||
got = await provenance_db.import_states(session, [*ids, "nosuchimport", kept.id])
|
||||
assert set(got) == set(ids)
|
||||
assert got[kept.id] == {"raw_state": "kept", "purged": None, "revoked": None, "recipe_seq": 3}
|
||||
assert got[purged.id] == {"raw_state": "purged", "revoked": None, "recipe_seq": None, "purged": StateNote(
|
||||
at="2026-09-30T01:02:03+00:00", signed_by="录入员甲", reason="合成理由:清除原件")}
|
||||
assert got[revoked.id]["revoked"] == StateNote(at="2026-10-01T00:00:00+00:00", signed_by=None,
|
||||
reason="合成理由:作废")
|
||||
assert got[revoked.id]["recipe_seq"] is None
|
||||
async with SessionLocal() as session:
|
||||
assert await provenance_db.import_states(session, []) == {}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# engine.py:EngineCache.pinned、open_checked_sqlite 的 immutable
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_pinned_returns_the_verified_fingerprint(tmp_path):
|
||||
path = _build(tmp_path / "p.db", daily=_daily_rows(2))
|
||||
snap = await _register(path)
|
||||
view = await _view(snap)
|
||||
cache = EngineCache()
|
||||
try:
|
||||
assert cache.pinned(view) is None # 还没建引擎
|
||||
await cache.get(view)
|
||||
st = os.stat(path)
|
||||
assert cache.pinned(view) == (st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns)
|
||||
manual = type("S", (), {"id": "m1", "kind": "sqlite", "database": str(path), "readonly": True,
|
||||
"options": {}, "name": "手工源"})()
|
||||
await cache.get(manual)
|
||||
assert cache.pinned(manual) is None # 手工源没有核对过的指纹
|
||||
await cache.invalidate(snap.source_id)
|
||||
assert cache.pinned(view) is None
|
||||
finally:
|
||||
await cache.close()
|
||||
|
||||
|
||||
def test_open_checked_sqlite_immutable_flag(tmp_path, monkeypatch):
|
||||
path = _build(tmp_path / "o.db", daily=_daily_rows(1))
|
||||
uris: list[str] = []
|
||||
real = sqlite3.connect
|
||||
|
||||
def spy(database, *a, **kw):
|
||||
uris.append(database)
|
||||
return real(database, *a, **kw)
|
||||
|
||||
monkeypatch.setattr(engine_mod.sqlite3, "connect", spy)
|
||||
for kwargs in ({}, {"readonly": True}, {"readonly": True, "immutable": True}):
|
||||
conn = open_checked_sqlite(path, **kwargs)
|
||||
try:
|
||||
assert conn.execute('SELECT COUNT(*) FROM "日客流"').fetchone() == (1,)
|
||||
finally:
|
||||
conn.close()
|
||||
# 缺省行为不变:只读、不带 immutable
|
||||
assert uris[0].endswith("?mode=ro") and uris[1].endswith("?mode=ro")
|
||||
assert uris[2].endswith("?mode=ro&immutable=1")
|
||||
with pytest.raises(ValueError):
|
||||
open_checked_sqlite(path, readonly=False, immutable=True)
|
||||
|
||||
@@ -0,0 +1,394 @@
|
||||
"""期 4「期 4 之前不变」的金样断言(P4-SPEC 6.6、1.4,WP-E)。
|
||||
|
||||
金样 `tests/fixtures/evidence/pre_p4_golden.json` 由 WP-0 在期 4 改动之前的提交(2cdbf9a)上真跑管线生成,自带三份
|
||||
报告(配方源、简单导入、手工源)的全部输入工件和四项输出:verify_doc、裁判摘录、_chain(单元格片段的查询步骤)、
|
||||
文档本身。这里不重跑管线(导入清单里有 created_at,重跑得到的 id 会不同),而是:
|
||||
|
||||
1. 把金样里的输入工件逐个 put_json 进本测试的临时数据目录,断言返回的 id 等于金样记下的键(内容寻址:对不上说明
|
||||
金样被改过,后面的比较就没有意义);
|
||||
2. 用新代码对同样的输入重算,逐字比金样。
|
||||
|
||||
为什么要这一层:期 4 的承诺是「期 4 之前的运行、非上传源的面板、复核、裁判摘录一个字都不变」(0.2 补充 1)。
|
||||
文档标记 `provenance` 是唯一的开关(1.4):没有它的文档必须走和期 4 之前完全相同的路径,连 provenance、
|
||||
direct_select 两个模块的函数都不许碰(3.3);带上它之后,非上传源仍然一字不差,上传源只在约定的位置多出约定的东西
|
||||
(查询步骤只多 `provenance: true`;摘录只在 SQL 那一行之后、「列:」之前多出来历行)。
|
||||
|
||||
金样里的 `_chain` 输出是用预填 memo 的 `_Masks` 替身算的(不查库),这里用同样的替身;`_Sealed` 按金样记下的封存
|
||||
范围在内存里拼(6.6)。夹具全部合成、假名,没有任何模型调用。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from app.api import evidence as api
|
||||
from app.core import artifact_store
|
||||
from app.core.config import settings
|
||||
from app.data import provenance as provenance_mod
|
||||
from app.data.provenance_types import DOC_PROVENANCE, CellRef, contract_problems
|
||||
from app.data.tabular import UNSHAPED_NOTE
|
||||
from app.engine import direct_select as direct_select_mod
|
||||
from app.engine import judge
|
||||
from app.engine.evidence import iter_segments, verify_doc
|
||||
|
||||
GOLDEN_PATH = Path(__file__).parent / "fixtures" / "evidence" / "pre_p4_golden.json"
|
||||
GOLDEN: dict[str, Any] = json.loads(GOLDEN_PATH.read_text("utf-8"))
|
||||
REPORTS = ("recipe", "simple", "manual")
|
||||
#: 上传表格的两份(查询快照有 data_version):带上标记后查询步骤多 provenance: true、摘录多来历行
|
||||
UPLOADS = ("recipe", "simple")
|
||||
|
||||
#: 来历行(P4-SPEC 3.2)可能的行首:来源、来源的分期行(两个空格缩进)、口径、已接受、区域外文字、截断说明
|
||||
NEW_LINE_HEADS = ("来源:", " ", "口径:", "已接受(", "区域外", "另有 ", "…另有部分来历信息未列出")
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 夹具:临时数据目录里重建金样的输入工件
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def rebuilt(monkeypatch, tmp_path) -> dict[str, str]:
|
||||
"""每个用例一个独立的数据目录,金样的输入工件和三份文档逐个 put_json 进去,id 必须等于金样的键。
|
||||
|
||||
独立目录的理由:A26 那一例要把导入清单的文件改坏,不能影响别的用例;工件按内容寻址、不带运行 id,放在共用的
|
||||
数据目录里会和别的测试串。"""
|
||||
data = tmp_path / "data"
|
||||
data.mkdir()
|
||||
monkeypatch.setattr(settings, "data_dir", data)
|
||||
ids: dict[str, str] = {}
|
||||
for aid, content in GOLDEN["artifacts"].items():
|
||||
got = await artifact_store.put_json(copy.deepcopy(content), kind="golden_input")
|
||||
assert got == aid, f"金样里的工件 {aid[:12]}… 重建出来的 id 是 {got[:12]}…:金样被改过"
|
||||
ids[aid] = got
|
||||
for name in REPORTS:
|
||||
rep = GOLDEN["reports"][name]
|
||||
got = await artifact_store.put_json(copy.deepcopy(rep["outputs"]["doc"]), kind="report_doc")
|
||||
assert got == rep["doc_artifact"], f"{name} 的文档重建出来的 id 不对:金样被改过"
|
||||
return ids
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def tripwire(monkeypatch) -> list[str]:
|
||||
"""把 provenance、direct_select 两个模块的公开函数全换成一调用就记下来并抛错的桩(P4-SPEC 3.3、6.6)。
|
||||
|
||||
老文档(没有标记)走的路径必须和期 4 之前完全相同,连这两个模块都不许碰:桩被调用一次,用例就失败。函数体内
|
||||
`from app.data.provenance import judge_lines` 这种写法每次调用时现取模块属性,同样会撞上桩。"""
|
||||
hits: list[str] = []
|
||||
patched: list[str] = []
|
||||
for mod in (provenance_mod, direct_select_mod):
|
||||
for name, obj in vars(mod).items():
|
||||
if name.startswith("_") or not callable(obj) or getattr(obj, "__module__", None) != mod.__name__ \
|
||||
or isinstance(obj, type):
|
||||
continue
|
||||
|
||||
def boom(*_a: Any, __name: str = f"{mod.__name__}.{name}", **_kw: Any) -> Any:
|
||||
hits.append(__name)
|
||||
raise AssertionError(f"期 4 之前的文档不该调用 {__name}")
|
||||
|
||||
monkeypatch.setattr(mod, name, boom)
|
||||
patched.append(name)
|
||||
# 桩真的装上了:入口函数一个不漏(漏了的话「没被调用」就是空话)
|
||||
assert {"recognize", "tables_in", "judge_lines", "resolve_chain", "chain_artifacts", "version_view", "locate",
|
||||
"related_checks"} <= set(patched), patched
|
||||
return hits
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 小工具
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def report(name: str) -> dict[str, Any]:
|
||||
return GOLDEN["reports"][name]
|
||||
|
||||
|
||||
def old_doc(name: str) -> dict[str, Any]:
|
||||
doc = copy.deepcopy(report(name)["outputs"]["doc"])
|
||||
assert "provenance" not in doc, "金样里的文档是期 4 之前组装的,不该有 provenance 键"
|
||||
return doc
|
||||
|
||||
|
||||
def marked_doc(name: str) -> dict[str, Any]:
|
||||
return {**old_doc(name), "provenance": DOC_PROVENANCE}
|
||||
|
||||
|
||||
def sealed_of(name: str) -> api._Sealed:
|
||||
"""金样记下的封存范围(6.6):内存里拼,不读运行记录。"""
|
||||
s = report(name)["sealed"]
|
||||
return api._Sealed(run_id=report(name)["run_id"], graph={}, seal=dict(s["seal"]), events=[],
|
||||
queries={k: dict(v) for k, v in s["queries"].items()},
|
||||
schemas={k: dict(v) for k, v in s["schemas"].items()})
|
||||
|
||||
|
||||
class _StubMasks:
|
||||
"""金样生成时 `_Masks` 的替身:预填 memo(源名 → (遮罩列小写, 数据源还在)),不查库。"""
|
||||
|
||||
def __init__(self, memo: dict[str, tuple[set[str], bool]]) -> None:
|
||||
self._memo = memo
|
||||
|
||||
async def of(self, source: Any) -> tuple[set[str], bool]:
|
||||
name = str(source or "")
|
||||
if not name:
|
||||
return set(), True
|
||||
return self._memo.get(name, (set(), True))
|
||||
|
||||
|
||||
def stub_masks(monkeypatch, name: str) -> None:
|
||||
memo = {k: ({c.lower() for c in v}, True) for k, v in report(name)["masks"].items()}
|
||||
monkeypatch.setattr(api, "_Masks", lambda: _StubMasks(memo))
|
||||
|
||||
|
||||
def cell_segments(doc: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
return [s for s in iter_segments(doc) if (s.get("cite") or {}).get("kind") == "cell"]
|
||||
|
||||
|
||||
async def chains(doc: dict[str, Any], name: str) -> dict[str, list[dict[str, Any]]]:
|
||||
sealed = sealed_of(name)
|
||||
out = {}
|
||||
for seg in cell_segments(doc):
|
||||
out[seg["id"]] = await api._chain(seg, doc, sealed, report(name)["node_id"])
|
||||
# 金样存的是 JSON:元组、浮点数都按 JSON 往返一次再比
|
||||
return json.loads(json.dumps(out, ensure_ascii=False))
|
||||
|
||||
|
||||
def excerpts(doc: dict[str, Any], name: str, *, loader: Any = None, masked: Any = None) -> dict[str, str]:
|
||||
return judge.prepare(doc, report(name)["catalog"], loader=loader or artifact_store.load, masked=masked).excerpts
|
||||
|
||||
|
||||
def verified(doc: dict[str, Any], name: str) -> dict[str, Any]:
|
||||
v = verify_doc(doc, report(name)["catalog"], loader=artifact_store.load)
|
||||
return {"ok": v["ok"], "stats": v["stats"], "violations": v["violations"]}
|
||||
|
||||
|
||||
def state_mismatches(v: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
return [x for x in v["violations"] if x.get("code") == "state_mismatch"]
|
||||
|
||||
|
||||
def split_new_lines(text: str) -> tuple[list[str], list[str]]:
|
||||
"""查询摘录 → (去掉来历行之后的行, 来历行)。来历行只许出现在「SQL:」那一行之后、「列:」那一行之前(3.2)。"""
|
||||
lines = text.split("\n")
|
||||
sql = [i for i, line in enumerate(lines) if line.startswith("SQL:")]
|
||||
cols = [i for i, line in enumerate(lines) if line.startswith("列:")]
|
||||
assert len(sql) == 1 and len(cols) == 1 and sql[0] < cols[0], f"查询摘录的形状变了:\n{text}"
|
||||
return lines[:sql[0] + 1] + lines[cols[0]:], lines[sql[0] + 1:cols[0]]
|
||||
|
||||
|
||||
def golden_tables(schema: dict[str, Any]) -> dict[str, list[str]]:
|
||||
"""冻结表结构 → {表: [列…]}(按定义顺序),direct_select.tables_in 的入参(3.3)。"""
|
||||
return {name: [str(c.get("name")) for c in (meta.get("columns") or [])]
|
||||
for name, meta in (schema.get("tables") or {}).items()}
|
||||
|
||||
|
||||
def expected_judge_lines(name: str, alias: str) -> list[str]:
|
||||
"""契约(3.3):_query_excerpt 先用 tables_in 算出 SQL 用到的表,再调 judge_lines,把返回的行原样插进摘录。
|
||||
金样的报告没有遮罩(masks 为空、快照没有 mask_columns),hidden 为空集。"""
|
||||
entry = report(name)["catalog"][alias]
|
||||
query = artifact_store.load(entry["artifact"])
|
||||
schema = artifact_store.load(query["schema_artifact"]) if query.get("schema_artifact") else None
|
||||
tables = direct_select_mod.tables_in(query["sql"], golden_tables(schema or {}))
|
||||
return provenance_mod.judge_lines(query, schema, artifact_store.load, tables=tables, hidden=set())
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 金样本身
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def test_golden_records_three_kinds_of_sources(rebuilt):
|
||||
"""金样的三份报告正好是规格要的三种源(6.6):配方源有 import_manifests、简单导入的表结构快照没有 import_mode、
|
||||
手工源的查询快照没有 data_version。重建出的工件 id 全部等于金样的键(夹具里已逐个断言)。"""
|
||||
arts = GOLDEN["artifacts"]
|
||||
assert set(rebuilt) == set(arts) and all(k == v for k, v in rebuilt.items())
|
||||
assert set(GOLDEN["reports"]) == set(REPORTS)
|
||||
rq = arts[report("recipe")["artifacts"][0]]
|
||||
rs = arts[rq["schema_artifact"]]
|
||||
assert rq.get("data_version") and rs.get("import_mode") == "recipe" and len(rs["import_manifests"]) == 1
|
||||
sq = arts[report("simple")["artifacts"][0]]
|
||||
ss = arts[sq["schema_artifact"]]
|
||||
assert sq.get("data_version") and "import_mode" not in ss and ss["tables"]["月报"]["comment"] == UNSHAPED_NOTE
|
||||
mq = arts[report("manual")["artifacts"][0]]
|
||||
assert "data_version" not in mq and "schema_artifact" not in mq
|
||||
for name in REPORTS:
|
||||
assert cell_segments(old_doc(name)), name
|
||||
assert report(name)["outputs"]["excerpts"], name
|
||||
assert report(name)["outputs"]["chain"], name
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 期 4 之前的文档(没有标记):一个字都不变,也不碰新模块
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", REPORTS)
|
||||
def test_old_documents_verify_exactly_as_before(name, tripwire):
|
||||
"""① verify_doc 逐字等于金样,没有 state_mismatch(verify_doc 不看未知的顶层键,也不读裁判摘录,1.4)。"""
|
||||
got = verified(old_doc(name), name)
|
||||
assert got == report(name)["outputs"]["verify_doc"]
|
||||
assert state_mismatches(got) == []
|
||||
assert tripwire == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", REPORTS)
|
||||
def test_old_documents_give_the_same_judge_excerpts(name, tripwire):
|
||||
"""② 裁判摘录逐字等于金样(摘录进断点指纹和逐句复用键,变一个字已判过的句子就不复用了,3.1)。"""
|
||||
assert excerpts(old_doc(name), name) == report(name)["outputs"]["excerpts"]
|
||||
assert tripwire == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", REPORTS)
|
||||
async def test_old_documents_give_the_same_query_steps(name, monkeypatch, tripwire):
|
||||
"""③ 单元格片段的出处链(查询步骤)逐字等于金样:没有 provenance 键,界面不多发请求(2.8.2)。"""
|
||||
stub_masks(monkeypatch, name)
|
||||
got = await chains(old_doc(name), name)
|
||||
assert got == report(name)["outputs"]["chain"]
|
||||
assert not any("provenance" in step for steps in got.values() for step in steps)
|
||||
assert tripwire == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", REPORTS)
|
||||
async def test_old_documents_answer_legacy_doc(name, tripwire):
|
||||
"""推断来源接口对老文档一律回 none · legacy_doc(R0),不标红、没有数据版本;cell 照样给(与 status 无关,2.8.1)。
|
||||
R0 在一切取证之前判,所以连查询快照都不用取,也不碰新模块。"""
|
||||
doc = old_doc(name)
|
||||
rep = report(name)
|
||||
chosen = api._Report(node_id=rep["node_id"], doc_artifact=rep["doc_artifact"], ok=True, repairs=0, stats=None,
|
||||
doc=doc, hash_ok=True, fields=[])
|
||||
for seg in cell_segments(doc):
|
||||
out = await api.segment_provenance(chosen, seg, sealed_of(name), masks=_StubMasks({}))
|
||||
assert contract_problems(out) == []
|
||||
assert (out.status, out.reason.code, out.alert, out.version, out.cell_source, out.checks) == \
|
||||
("none", "legacy_doc", None, None, None, [])
|
||||
assert out.sealed is True and out.segment == seg["id"]
|
||||
loc = seg["cite"]["locator"]
|
||||
assert out.cell == CellRef(alias=seg["cite"]["alias"], row=loc["row"], column=loc["column"],
|
||||
artifact=rep["catalog"][seg["cite"]["alias"]]["artifact"])
|
||||
assert out.report.node_id == rep["node_id"] and out.report.doc_artifact == rep["doc_artifact"]
|
||||
assert tripwire == []
|
||||
|
||||
|
||||
async def test_old_document_on_demand_judge_ignores_a_broken_manifest(tripwire):
|
||||
"""A26(金样一侧):老文档走按需裁判的 loader(`_sealed_loader`,chain 按文档标记取,老文档为假),把表结构快照
|
||||
列出的导入清单文件改坏一个字节,摘录仍逐字等于金样:chain 为假时根本不去取清单(2.8.3)。"""
|
||||
name = "recipe"
|
||||
doc = old_doc(name)
|
||||
rep = report(name)
|
||||
schema = artifact_store.load(artifact_store.load(rep["artifacts"][0])["schema_artifact"])
|
||||
[manifest] = schema["import_manifests"]
|
||||
path = artifact_store._path_of(manifest)
|
||||
raw = path.read_bytes()
|
||||
at = raw.index(b'"seq":')
|
||||
path.write_bytes(raw[:at] + b'"seQ":' + raw[at + 6:])
|
||||
with pytest.raises(ValueError):
|
||||
artifact_store.load(manifest)
|
||||
# _judge_now 的同一套取法:loader 按文档标记决定扩不扩展、遮罩按这个 loader 查
|
||||
loader = api._sealed_loader(sealed_of(name), chain=doc.get("provenance") == DOC_PROVENANCE)
|
||||
assert loader(manifest) is None
|
||||
masked = await judge.source_masks(rep["catalog"], loader=loader)
|
||||
assert not any(masked.values()), f"金样的数据源不该设遮罩:{masked}"
|
||||
assert excerpts(doc, name, loader=loader, masked=masked) == rep["outputs"]["excerpts"]
|
||||
assert tripwire == []
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# 带上标记之后:非上传源一字不差;上传源只在约定的位置多出约定的东西
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
async def test_marked_manual_document_is_unchanged(monkeypatch):
|
||||
"""手工源(查询快照没有 data_version)的文档加上标记:①②③仍逐字等于金样(1.4「非上传源」,3.3)。"""
|
||||
name = "manual"
|
||||
doc = marked_doc(name)
|
||||
out = report(name)["outputs"]
|
||||
assert verified(doc, name) == out["verify_doc"]
|
||||
assert excerpts(doc, name) == out["excerpts"]
|
||||
stub_masks(monkeypatch, name)
|
||||
assert await chains(doc, name) == out["chain"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", UPLOADS)
|
||||
def test_marked_upload_documents_still_verify_the_same(name):
|
||||
"""标记写在文档顶层、参与内容哈希,但 verify_doc 不看它:ok、stats、violations 与金样逐字相同,没有 state_mismatch。"""
|
||||
got = verified(marked_doc(name), name)
|
||||
assert got == report(name)["outputs"]["verify_doc"] and state_mismatches(got) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", UPLOADS)
|
||||
async def test_marked_upload_query_steps_only_gain_the_hint(name, monkeypatch):
|
||||
"""③ 上传表格的单元格片段:查询步骤只多一个 `provenance: true`,其余每个键逐字等于金样(2.8.2)。"""
|
||||
stub_masks(monkeypatch, name)
|
||||
got = await chains(marked_doc(name), name)
|
||||
want = report(name)["outputs"]["chain"]
|
||||
assert set(got) == set(want)
|
||||
for sid, steps in got.items():
|
||||
assert len(steps) == len(want[sid]) == 1, sid
|
||||
for step, old in zip(steps, want[sid]):
|
||||
assert set(step) - set(old) == {"provenance"} and set(old) <= set(step), (sid, set(step) ^ set(old))
|
||||
assert step["provenance"] is True
|
||||
assert {k: v for k, v in step.items() if k != "provenance"} == old, sid
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", UPLOADS)
|
||||
def test_marked_upload_excerpts_keep_the_golden_text_around_the_new_lines(name):
|
||||
"""② 上传表格加上标记后,摘录里新增的东西只许插在查询摘录「SQL:」那一行之后、「列:」那一行之前:去掉这段之后
|
||||
逐字等于金样;表名实体等其余摘录逐字等于金样(3.2 的位置约定)。新行一律以 3.2 的几种行首开头。"""
|
||||
got = excerpts(marked_doc(name), name)
|
||||
want = report(name)["outputs"]["excerpts"]
|
||||
assert set(got) == set(want)
|
||||
for alias, text in got.items():
|
||||
if not alias.startswith("Q"):
|
||||
assert text == want[alias], alias
|
||||
continue
|
||||
kept, new = split_new_lines(text)
|
||||
golden_kept, golden_new = split_new_lines(want[alias])
|
||||
assert golden_new == [], "金样的查询摘录在 SQL 与「列:」之间不该有别的行"
|
||||
assert kept == golden_kept, alias
|
||||
assert all(line.startswith(NEW_LINE_HEADS) for line in new), (alias, new)
|
||||
|
||||
|
||||
def test_marked_recipe_excerpts_gain_the_provenance_lines():
|
||||
"""② 配方源的文档加上标记:两条查询摘录都多出 3.2 的来历行,内容就是 judge_lines 对同样输入给出的行,原样插在
|
||||
「SQL:」与「列:」之间(3.3)。断言几样确定的内容:
|
||||
|
||||
- 来源行:上传的表格、每期替换、第 1 次导入、统计期取自 客流汇总!B2、文件名与 sha256 前 12 位、区域(S9);
|
||||
- 口径:SQL 用到的表的说明(冻结表结构的 comment 原文)、查询结果列的说明;
|
||||
- 区域外文字:金样的清单是期 4 之前导入的,OutsideText 没有 hidden 字段,只计条数、不送全文(调整 10);
|
||||
- 没有接受理由(D00 没有接受)。"""
|
||||
name = "recipe"
|
||||
got = excerpts(marked_doc(name), name)
|
||||
rep = report(name)
|
||||
query = artifact_store.load(rep["catalog"]["Q1"]["artifact"])
|
||||
schema = artifact_store.load(query["schema_artifact"])
|
||||
manifest = artifact_store.load(schema["import_manifests"][0])
|
||||
for alias, table in (("Q1", "日客流"), ("Q2", "时段客流")):
|
||||
_, new = split_new_lines(got[alias])
|
||||
assert new, f"{alias} 的摘录没有来历行"
|
||||
assert new == expected_judge_lines(name, alias), alias
|
||||
source = new[0]
|
||||
assert source.startswith("来源:上传的表格,每期替换,第 1 次导入:统计期 2026-08-01 至 2026-08-31(取自 客流汇总!B2)")
|
||||
assert f"文件「{manifest['file']['name']}」(sha256 {manifest['file']['raw_sha256'][:12]})" in source
|
||||
assert "区域 客流汇总!" in source
|
||||
comment = schema["tables"][table]["comment"]
|
||||
assert f"口径:表 {table} 的说明:{comment}" in new
|
||||
assert not any(line.startswith("已接受") for line in new)
|
||||
assert "另有 1 处区域外文字未列出(所在行列被隐藏,或导入时未记录是否隐藏),见证据面板" in new
|
||||
assert not any("客流汇总表" in line for line in new), "隐藏与否未记录的区域外文字不该送全文"
|
||||
_, new = split_new_lines(got["Q1"])
|
||||
for col in ("日期", "全日客流", "分区甲"):
|
||||
note = next(c["comment"] for c in schema["tables"]["日客流"]["columns"] if c["name"] == col)
|
||||
assert f"口径:列 {col} 的说明:{note}" in new
|
||||
assert len("\n".join(new)) <= 1200
|
||||
|
||||
|
||||
def test_marked_simple_upload_excerpt_gains_only_the_caliber_line():
|
||||
"""② 简单导入的文档加上标记:只多口径行(调整 6:未规整的表写 UNSHAPED_NOTE,裁判才知道不能直接对列求和);
|
||||
没有导入清单,来源、已接受、区域外文字三类都不出。"""
|
||||
name = "simple"
|
||||
got = excerpts(marked_doc(name), name)
|
||||
_, new = split_new_lines(got["Q1"])
|
||||
assert new == expected_judge_lines(name, "Q1")
|
||||
assert f"口径:表 月报 的说明:{UNSHAPED_NOTE}" in new
|
||||
assert not any(line.startswith(("来源:", "已接受", "区域外")) for line in new), new
|
||||
@@ -351,6 +351,36 @@ def test_outside_text_with_digits_needs_confirmation():
|
||||
assert (f"{SHEET}!B32", "text_digits", False) in [(o.cell, o.kind, o.period_source) for o in ex.outside_text]
|
||||
|
||||
|
||||
def test_outside_text_records_whether_its_row_or_column_is_hidden():
|
||||
"""期 4(P4-SPEC 7.3、调整 10,评审 hid.py 的复现):区域外文字在隐藏行、隐藏列里时 hidden 为 True,可见的为
|
||||
False;带附加文字的统计期格(period_source)同样写。裁判摘录只送 hidden is False 的项,隐藏行里的内部备注
|
||||
不能多一条经裁判外发的通道。隐藏行列在认领区域之外,不触发 hidden_rows / hidden_cols。"""
|
||||
def edit(ws, info):
|
||||
ws["B32"] = "注:8月15日闸机故障,当日客流为估算值" # 可见、含数字
|
||||
ws["B33"] = "内部:本月补贴 45678 元" # 隐藏行
|
||||
ws.row_dimensions[33].hidden = True
|
||||
ws["H34"] = "内部备注:分区甲口令 abc123" # 区域右侧、区域下方的隐藏列
|
||||
ws.column_dimensions["H"].hidden = True
|
||||
ws["B3"] = "客流汇总表(2026年8月)" # 可见的统计期附加文字
|
||||
ws["B35"] = "注:2026年8月数据为初步统计" # 隐藏行里的统计期附加文字
|
||||
ws.row_dimensions[35].hidden = True
|
||||
ex = run(flow_book(days=3, edit=edit))
|
||||
assert ex.ok, [(p.code, p.message) for p in ex.problems]
|
||||
assert not {"hidden_rows", "hidden_cols"} & codes(ex)
|
||||
got = {o.cell: (o.kind, o.period_source, o.hidden) for o in ex.outside_text}
|
||||
assert got == {
|
||||
f"{SHEET}!B3": ("text_digits", True, False),
|
||||
f"{SHEET}!B32": ("text_digits", False, False),
|
||||
f"{SHEET}!B33": ("text_digits", False, True),
|
||||
f"{SHEET}!H34": ("text_digits", False, True),
|
||||
f"{SHEET}!B35": ("text_digits", True, True),
|
||||
}
|
||||
# 回执(asdict)里一律带 hidden 键:导入清单按它判断,老清单没有这个键才是「未记录」
|
||||
from dataclasses import asdict
|
||||
|
||||
assert all(isinstance(o["hidden"], bool) for o in asdict(ex)["outside_text"])
|
||||
|
||||
|
||||
def test_outside_number_is_structure_and_merges_per_row():
|
||||
def edit(ws, info):
|
||||
ws["B1"] = 987654 # 日期表头以上:区域外
|
||||
|
||||
@@ -5,7 +5,7 @@ import type {
|
||||
} from '../types'
|
||||
import type {
|
||||
Approval, AutofixResult, Conversation, ConversationDetail, ConversationTurn, CustomTool, DataSource, EvidenceGraph,
|
||||
EvidenceAudit, EvidenceJudgeResult, EvidenceSegmentDetail, GraphSpec, KbDocument, MemoryItem, Provider, PublishCheck, PublishLevel, ReviewResult, Run,
|
||||
EvidenceAudit, EvidenceJudgeResult, EvidenceProvenance, EvidenceSegmentDetail, GraphSpec, KbDocument, MemoryItem, Provider, PublishCheck, PublishLevel, ReviewResult, Run,
|
||||
RunEvent, RunStatus, Skill, ToolChange, ToolInfo, ToolTrust, UpgradeResult, UploadDecision, UploadMixedColumn, UploadResult,
|
||||
UploadShapeReason, ValidationIssue, VarIssue, Variable, Workflow, WorkflowVersion,
|
||||
} from '../types'
|
||||
@@ -891,6 +891,15 @@ export const api = {
|
||||
*/
|
||||
judge: (runId: string, body: { units: string[]; report?: string }) =>
|
||||
post<EvidenceJudgeResult>(`/runs/${encodeURIComponent(runId)}/evidence/judge`, body),
|
||||
/**
|
||||
* 推断的来源(期 4):上传表格的一个单元格来自原表的哪一格、这次查询查的是哪一版数据、相关核对。按需算,
|
||||
* 推断不出来不是错误(200,结论在 status、reason、alert 里)。只在片段接口的查询步骤带 provenance: true 时请求;
|
||||
* 老服务端没有这个接口,回 404 且没有 code。report 同片段接口
|
||||
*/
|
||||
provenance: (runId: string, segmentId: string, opts?: RequestOptions & { report?: string }) =>
|
||||
get<EvidenceProvenance>(
|
||||
`/runs/${encodeURIComponent(runId)}/evidence/segments/${encodeURIComponent(segmentId)}/provenance${qs({ report: opts?.report })}`,
|
||||
opts),
|
||||
/**
|
||||
* 导出审计表:?format=json / csv 作为附件下载。回来的是文件本身和后端给的文件名
|
||||
* (Content-Disposition;没给就用 evidence-<运行号前 8 位>.<格式>)
|
||||
|
||||
@@ -30,6 +30,7 @@ import evidenceQuery from '../run/__tests__/evidence-query.json'
|
||||
import evidenceEntity from '../run/__tests__/evidence-entity.json'
|
||||
import evidenceJudge from '../run/__tests__/evidence-judge.json'
|
||||
import evidenceVerdict from '../run/__tests__/evidence-verdict.json'
|
||||
import evidenceProvenance from '../run/__tests__/evidence-provenance.json'
|
||||
import { EvidenceGuessView } from '../run/EvidenceGuess'
|
||||
import { IssuanceBanner } from '../run/AssistantStream'
|
||||
import mdPinned from '../run/__tests__/markdown-pinned.json'
|
||||
@@ -207,6 +208,11 @@ const EV_JUDGE_PLAIN = evidenceJudge.plain.doc as unknown as EvidenceDocData
|
||||
*/
|
||||
const EV_VERDICT = evidenceVerdict.formal.doc as unknown as EvidenceDocData
|
||||
const EV_VERDICT_EXPLORE = evidenceVerdict.explore.doc as unknown as EvidenceDocData
|
||||
/**
|
||||
* Excel 导入期 4:上传表格的单元格追到原表格子(evidence-provenance.json)。文档由 compose_doc 真跑、带文档标记;
|
||||
* 片段接口、推断来源接口、导入清单的工件都由 check-evidence 用 page.route 伪造(按 runId 认出这一份夹具)
|
||||
*/
|
||||
const EV_PROVENANCE = evidenceProvenance.doc as unknown as EvidenceDocData
|
||||
/** 同一份旧契约的出具,去掉位置:老运行就是这样,只能按字符串标 */
|
||||
const LEGACY_LOOSE = {
|
||||
matched: EV_LEGACY.matched.map(({ start: _s, end: _e, ...m }) => m),
|
||||
@@ -431,6 +437,20 @@ function EvidenceOnly() {
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
{/*
|
||||
Excel 导入期 4:推断的来源。宽栏一份、360 宽的画布右栏一份(栏内展开,不出横向滚动)
|
||||
*/}
|
||||
<div className="flex flex-col gap-4" id="evidence-provenance-demo">
|
||||
<div id="evidence-provenance" className="rounded-lg border p-3">
|
||||
<div className="mb-1 text-2xs text-faint">上传表格的单元格:数据版本、推断的来源、相关核对</div>
|
||||
<EvidenceDoc doc={EV_PROVENANCE} artifact={evidenceProvenance.doc_artifact} runId={evidenceProvenance.run_id}
|
||||
label="answer" tally />
|
||||
</div>
|
||||
<div id="evidence-provenance-narrow" className="rounded-lg border bg-panel p-2.5" style={{ width: 360, maxWidth: '100%' }}>
|
||||
<EvidenceDoc doc={EV_PROVENANCE} artifact={evidenceProvenance.doc_artifact} runId={evidenceProvenance.run_id}
|
||||
label="answer" dense tally />
|
||||
</div>
|
||||
</div>
|
||||
<div id="evidence-long" className="rounded-lg border p-3">
|
||||
<div className="mb-1 text-2xs text-faint">长报告:{EV_LONG.blocks.length} 块,块级 content-visibility,超过 3000 字先折叠</div>
|
||||
<EvidenceDoc doc={EV_LONG} label="长报告" tally />
|
||||
|
||||
@@ -341,6 +341,78 @@ export const EVIDENCE_TEXT = {
|
||||
guessNone: '未找到数值相同的单元格',
|
||||
guessDiff: (d: string) => `相差 ${d}`,
|
||||
guessCandidates: '可能来自',
|
||||
|
||||
// ---- 期 4:推断的来源(上传表格的单元格追到原表格子,P4-SPEC 4.2)----
|
||||
// 用词同版本页:不写「快照」「构建」「并集」,写「数据版本」「这一期」「数据文件」;署名一律标「署名(未认证)」。
|
||||
// 不下钻的原因、标红的提示由服务端给原文(provenance_types.REASON_TEXT / ALERT_TEXT),界面原样显示,不在这里另写
|
||||
provenance: {
|
||||
version: '数据版本',
|
||||
/** 首行。每期替换只有一期,不写期数;导入模式的说法同版本页(VERSIONS_TEXT.modeLabel) */
|
||||
versionHead: (source: string, mode: string, n: number) => (mode === 'accumulate'
|
||||
? `数据源「${source}」· 按期累积 · 共 ${formatNumber(n)} 期`
|
||||
: `数据源「${source}」· 每期替换`),
|
||||
part: (seq: number, start: string, end: string) => `第 ${formatNumber(seq)} 次导入 · 统计期 ${start} 至 ${end}`,
|
||||
partNoPeriod: (seq: number) => `第 ${formatNumber(seq)} 次导入 · 统计期未记录`,
|
||||
/** sha 是原件 sha256 的前 12 位;清单没记哈希时只写文件名 */
|
||||
file: (name: string, sha: string) => (sha ? `文件「${name}」· sha256 ${sha}` : `文件「${name}」`),
|
||||
/** S9:回执里块内各区域的外接矩形和排除的行数。排除 0 行、没有记录时不写 */
|
||||
region: (region: string, excluded: number | null) =>
|
||||
(excluded ? `区域 ${region} · 排除 ${formatNumber(excluded)} 行` : `区域 ${region}`),
|
||||
excluded: (n: number) => `排除 ${formatNumber(n)} 行`,
|
||||
committedAt: (at: string) => `导入于 ${at}`,
|
||||
/** 查询用到的表(与冻结表结构的交集)。原表写明的合计表另注一句:它不能和明细相加 */
|
||||
tables: (names: string[]) => `涉及的表:${names.join('、')}`,
|
||||
totalTable: (name: string) => `${name}(原表写明的合计)`,
|
||||
hasRow: '这一格所在的一期',
|
||||
current: '当前状态',
|
||||
revoked: '这一期的接受已作废',
|
||||
/** 清除原件、作废接受的记录:时间 · 理由 · 署名(未认证)。缺的不写(时间没记时调用方传空串),署名没填写「未填写」 */
|
||||
stateDetail: (at: string, reason: string | null, name: string | null) =>
|
||||
[at, reason ? `理由:${reason}` : '', `署名(未认证):${name || '未填写'}`].filter(Boolean).join(' · '),
|
||||
/** 这一期写过的接受理由(导入清单里的,是当时的事实)。核对标题取不到时写核对编号 */
|
||||
acceptedLine: (check: string, reason: string, name: string | null) =>
|
||||
`已接受:${check} · ${reason} · 署名(未认证):${name || '未填写'}`,
|
||||
manifest: '查看导入清单',
|
||||
manifestMasked: '数据源设置了遮罩,导入清单请到数据源卡片上的「版本」查看',
|
||||
unsealed: '本次运行尚未封存或封存核对未通过,以下内容不计入已封存的证据',
|
||||
morePeriods: (n: number) => `另有 ${formatNumber(n)} 期`,
|
||||
title: '推断的来源',
|
||||
badge: '推断,不属于已封存的证据',
|
||||
cell: (sheet: string, ref: string) => `工作表「${sheet}」${ref}`,
|
||||
/** 多期时格子在哪一期的文件里:坐标是那一期原件的坐标,不是别的期的 */
|
||||
fromPart: (seq: number, file: string | null) =>
|
||||
(file ? `取自第 ${formatNumber(seq)} 次导入的文件「${file}」` : `取自第 ${formatNumber(seq)} 次导入`),
|
||||
axisHeader: (ref: string) => `日期取自表头格 ${ref}`,
|
||||
yearPeriod: (cells: string) => (cells ? `年份取自统计期(${cells})` : '年份取自统计期'),
|
||||
yearHuman: (name: string) => `年份取自人工录入的统计期(署名(未认证):${name})`,
|
||||
yearMixed: '这一块的日期表头有的是日期格(自带年份),有的是文本(年份取自统计期),无法确定这一格属于哪一种',
|
||||
/** 格子原文拿不到时(text 为 null)只写坐标,不拿规范写法冒充原文 */
|
||||
rowLabel: (ref: string, text?: string | null) => (text ? `行标签 ${ref}「${text}」` : `行标签 ${ref}`),
|
||||
/** 清单里没有分段标题格的原文,只有配方里的定位文字:照实说是「按配方中的标题定位」 */
|
||||
sectionTitle: (ref: string, title?: string | null) =>
|
||||
(title ? `分段标题 ${ref}(按配方中的标题「${title}」定位)` : `分段标题 ${ref}`),
|
||||
colHeader: (refs: string) => `列表头 ${refs}`,
|
||||
totalLabel: (ref: string, text?: string | null) => (text ? `合计标签 ${ref}「${text}」` : `合计标签 ${ref}`),
|
||||
canonical: (raw: string, canon: string) => `原文「${raw}」,按规范写法存为「${canon}」`,
|
||||
reportedTotal: '原表写明的合计:不要彼此相加,也不要与明细相加',
|
||||
rawPurged: '原件已清除:坐标来自导入时的记录,无法再对照原件',
|
||||
mergedFill: '这一块按合并单元格的左上格填充:如果这一格在合并区域内,值取自该区域的左上格',
|
||||
recheck: '已按主键回查数据文件,值一致',
|
||||
details: '技术细节',
|
||||
recheckSql: '回查 SQL',
|
||||
checks: '相关核对',
|
||||
row: { passed: '这一行成立', mismatch: '这一行不成立', unverifiable: '这一行含空值,未能核对' } as Record<string, string>,
|
||||
cellStatus: {
|
||||
ok: '这一格一致',
|
||||
unverifiable: '这一格未能核对',
|
||||
unknown: '本期有格子未能核对,这一格是否在其中无法确定',
|
||||
not_formula: '这一格是写死的数,不做公式引用核对',
|
||||
} as Record<string, string>,
|
||||
partStatus: { passed: '本期通过', mismatch: '本期不成立', unverifiable: '本期未能核对', info: '口径说明' } as Record<string, string>,
|
||||
accepted: (reason: string, name: string | null) => `已接受,理由:${reason}(署名(未认证):${name || '未填写'})`,
|
||||
loading: '正在推断来源…',
|
||||
failed: '未能获取推断的来源',
|
||||
},
|
||||
} as const
|
||||
|
||||
/** 结论句按裁判结论分成的几堆(lib/evidence 的 claimCounts 数出来的) */
|
||||
@@ -1074,7 +1146,8 @@ export const VERSIONS_TEXT = {
|
||||
/** tokens 是 formatTokens 的结果(带单位),cost 是 formatCost 的结果 */
|
||||
aiUsage: (calls: number, tokens: string, cost: string) => `调用模型 ${formatNumber(calls)} 次,共 ${tokens},约 ${cost}`,
|
||||
aiNone: '这次导入没有调用模型',
|
||||
outsideTextNote: '区域外文字只在这里显示,不提供给模型',
|
||||
/** 期 4 起,带来源标记的报告里,裁判摘录会带上可见单元格里的区域外文字(隐藏行列里的不送) */
|
||||
outsideTextNote: '区域外文字不导入数据表;核对报告时,裁判可以看到其中可见单元格的文字',
|
||||
revokedBadge: '接受已作废',
|
||||
/** 当前版本或回滚目标被别人改过(base_changed、revoke_target_changed):服务端原话说了变的是什么,这里只说列表已刷新 */
|
||||
refreshHint: '列表已刷新,请重新确认',
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
import type { ReactNode } from 'react'
|
||||
import clsx from 'clsx'
|
||||
import type { CheckResult, ImportAcceptance, ManifestOut } from '../../types'
|
||||
import { CopyButton, Notice } from '../../components/ui'
|
||||
import { formatBytes, formatCost, formatDateTime, formatTokens } from '../../lib/format'
|
||||
import { RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, VERSIONS_TEXT as VT } from '../../lib/terms'
|
||||
import { CheckStatusChip } from './ImportReceipt'
|
||||
import { ReceiptSummary } from './ReceiptBlocks'
|
||||
|
||||
// ===========================================================================
|
||||
// 导入清单的渲染(P3-SPEC 7.3):文件、配方、统计期、核对、接受、确认清单、修改记录、说明、回执摘要、AI 用量,
|
||||
// 原始清单默认收起。
|
||||
//
|
||||
// 从版本页(VersionsDialog)抽出来,证据面板的「数据版本」一节也用它(P4-SPEC 4.1):面板里查看导入清单
|
||||
// 不能退回整份 JSON 输出——那样遮罩不起作用,64 KB 的清单也没法读。两处的行为必须一致,check-versions 守版本页,
|
||||
// check-evidence 的 provenance 段守面板
|
||||
// ===========================================================================
|
||||
|
||||
function Block({ id, title, children }: { id: string; title: string; children: ReactNode }) {
|
||||
return (
|
||||
<section className="space-y-1.5 rounded-lg border bg-bg p-3 text-xs" data-manifest-block={id}>
|
||||
<h4 className="font-semibold">{title}</h4>
|
||||
{children}
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
const short = (sha: unknown) => (typeof sha === 'string' && sha ? sha.slice(0, 8) : '')
|
||||
const cellsShort = (cells: string[]) => cells.map((c) => c.slice(c.lastIndexOf('!') + 1)).join('、')
|
||||
const list = <T,>(v: unknown): T[] => (Array.isArray(v) ? (v as T[]) : [])
|
||||
|
||||
/**
|
||||
* 导入清单的各块。content 是服务端存的原样(内容寻址,取回时复验了哈希),字段都按「可能缺」读:期 2 的清单
|
||||
* 没有修改记录,简单导入只有导入回执。
|
||||
*
|
||||
* 清单之外的三样(文件名兜底、原件状态、配方第几版)由调用方给:版本页取自导入记录,证据面板取自推断来源接口的
|
||||
* 那一期(PartView)。原件状态和配方版次是当前状态,不在清单里
|
||||
*/
|
||||
export function ManifestBody({ m, fileName, rawState, recipeSeq }: {
|
||||
m: ManifestOut
|
||||
/** 清单没记文件名时显示的名字 */
|
||||
fileName?: string | null
|
||||
/** 原件状态(kept / purged / absent),取不到为 null 时不显示 */
|
||||
rawState?: string | null
|
||||
/** 配方第几版,取不到为 null 时不显示 */
|
||||
recipeSeq?: number | null
|
||||
}) {
|
||||
const c = m.content ?? {}
|
||||
const B = VT.manifestBlocks
|
||||
const simple = m.kind === 'build_report'
|
||||
const receipt = simple ? c : c.receipt
|
||||
const checks = list<CheckResult>(c.checks)
|
||||
const titleOf = (id: string) => checks.find((x) => x.id === id)?.title
|
||||
const accepts: ImportAcceptance[] = [
|
||||
...list<ImportAcceptance>(c.acceptances?.overrides).map((a) => ({ ...a, kind: 'override' })),
|
||||
...list<ImportAcceptance>(c.acceptances?.waivers).map((a) => ({ ...a, kind: 'waiver' })),
|
||||
]
|
||||
const confirmations = list<{ id: string; label: string; at?: string }>(c.confirmations)
|
||||
const edits = Array.isArray(c.edits) ? list<Record<string, any>>(c.edits) : null
|
||||
const rendered = Object.entries((c.notes?.rendered ?? {}) as Record<string, { comment?: string; columns?: Record<string, string> }>)
|
||||
const usage = list<Record<string, any>>(c.ai?.usage)
|
||||
const tokens = usage.reduce((n, u) => n + (Number(u.input_tokens) || 0) + (Number(u.output_tokens) || 0), 0)
|
||||
const cost = usage.reduce((n, u) => n + (Number(u.cost_usd) || 0), 0)
|
||||
const period = c.period as { start?: string; end?: string; source?: string; cells?: string[]; signed_by?: string | null } | null
|
||||
const outside = list(receipt?.outside_text)
|
||||
|
||||
return (
|
||||
<div className="space-y-2.5" data-manifest={m.kind} data-verified={m.verified ? 'true' : 'false'}>
|
||||
{simple
|
||||
? <Notice tone="info">{VT.manifestSimple}</Notice>
|
||||
: !m.verified && <Notice tone="err" attr={{ 'data-manifest-unverified': '' }}>{VT.manifestUnverified}</Notice>}
|
||||
|
||||
{!simple && (
|
||||
<>
|
||||
<Block id="file" title={B.file}>
|
||||
<div className="flex flex-wrap items-baseline gap-x-3 gap-y-1">
|
||||
<span className="mono break-all">{c.file?.name || fileName}</span>
|
||||
{c.file?.size != null && <span className="tnum text-dim">{formatBytes(Number(c.file.size))}</span>}
|
||||
{short(c.file?.raw_sha256) && <span className="mono text-dim">{VT.sha(short(c.file?.raw_sha256))}</span>}
|
||||
{rawState != null && <span className="chip">{RAW_STATE_LABEL[rawState] ?? rawState}</span>}
|
||||
</div>
|
||||
{c.signed_by?.name && <div className="text-2xs text-faint">{VT.signedBy(c.signed_by.name)}</div>}
|
||||
</Block>
|
||||
|
||||
<Block id="recipe" title={B.recipe}>
|
||||
<div className="flex flex-wrap items-baseline gap-x-3 gap-y-1">
|
||||
{recipeSeq != null && <span>{VT.recipeSeq(recipeSeq)}</span>}
|
||||
{c.recipe?.origin && <span className="text-dim">{RECIPE_ORIGIN_LABEL[c.recipe.origin] ?? c.recipe.origin}</span>}
|
||||
{short(c.recipe?.sha256) && <span className="mono text-dim">{VT.sha(short(c.recipe?.sha256))}</span>}
|
||||
</div>
|
||||
{c.recipe?.canonical && <RawJson summary={VT.recipeJson} value={c.recipe.canonical} attr="data-recipe-json" />}
|
||||
</Block>
|
||||
|
||||
<Block id="period" title={B.period}>
|
||||
{period?.start && period?.end ? (
|
||||
<div className="space-y-0.5">
|
||||
<div className="tnum">{VT.period(period.start, period.end)}</div>
|
||||
<div className="text-2xs text-faint">
|
||||
{period.source === 'human'
|
||||
? `${VT.periodHuman},${VT.signedBy(period.signed_by || VT.unsigned)}`
|
||||
: period.cells?.length ? RECIPE_TEXT.periodFromCells(cellsShort(period.cells)) : ''}
|
||||
</div>
|
||||
</div>
|
||||
) : <p className="text-faint">{VT.periodUnknown}</p>}
|
||||
</Block>
|
||||
|
||||
<Block id="checks" title={B.checks}>
|
||||
{checks.length ? (
|
||||
<ul className="space-y-1.5">
|
||||
{checks.map((k) => (
|
||||
<li key={k.id} className="space-y-1 rounded-md border px-2.5 py-1.5" data-manifest-check={k.id} data-status={k.status}>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<CheckStatusChip status={k.status} />
|
||||
<span className="mono text-faint">{k.id}</span>
|
||||
<span className="min-w-0 flex-1">{k.title}</span>
|
||||
{/* 计数照试运行回执的规则(ImportReceipt 的 CheckRow):「说明」类的 failed 是预期内的不相等个数,不写 */}
|
||||
{!!k.checked && k.status !== 'info' && (
|
||||
<span className="tnum text-2xs text-faint" data-check-counts="">
|
||||
{VT.checkCounts(k.checked, k.failed ?? 0, k.unverifiable ?? 0)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{/* 清单是当证据核对的,细节不截断:服务端每个核对最多逐条列 20 格,其余写成「另有 N 格不一致」
|
||||
「N 格无法核对:…」,截掉的正是这几行 */}
|
||||
{!!k.details?.length && (
|
||||
<ul className="space-y-0.5 text-2xs leading-relaxed text-dim" data-check-details={k.details.length}>
|
||||
{k.details.map((d, i) => <li key={i}>{d}</li>)}
|
||||
</ul>
|
||||
)}
|
||||
{k.sql && <RawJson summary={VT.sql} value={k.params?.length ? `${k.sql}\n-- ${JSON.stringify(k.params)}` : k.sql} attr="data-check-sql" />}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : <p className="text-faint">{VT.checksNone}</p>}
|
||||
</Block>
|
||||
|
||||
<Block id="acceptances" title={B.acceptances}>
|
||||
{accepts.length ? (
|
||||
<ul className="space-y-1">
|
||||
{accepts.map((a, i) => (
|
||||
<li key={`${a.check_id}-${i}`} className="space-y-0.5 rounded-md border px-2.5 py-1.5" data-manifest-acceptance={a.check_id}>
|
||||
<div className="flex flex-wrap items-baseline gap-x-2">
|
||||
<span className="chip">{VT.acceptKind[a.kind] ?? VT.acceptKind.override}</span>
|
||||
<span className="min-w-0 flex-1 break-words">
|
||||
{VT.acceptRow(titleOf(a.check_id) ? `${a.check_id} ${titleOf(a.check_id)}` : a.check_id, a.reason)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-x-2 text-2xs text-faint">
|
||||
<span>{VT.signedBy(a.signed_by || VT.unsigned)}</span>
|
||||
{a.at && <span className="tnum">{formatDateTime(a.at)}</span>}
|
||||
</div>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : <p className="text-faint">{VT.acceptancesNone}</p>}
|
||||
</Block>
|
||||
|
||||
<Block id="confirmations" title={B.confirmations}>
|
||||
{confirmations.length ? (
|
||||
<ul className="list-inside list-disc space-y-0.5 text-dim">
|
||||
{confirmations.map((x, i) => <li key={`${x.id}-${i}`}>{x.label}</li>)}
|
||||
</ul>
|
||||
) : <p className="text-faint">{VT.confirmationsNone}</p>}
|
||||
</Block>
|
||||
|
||||
<Block id="edits" title={B.edits}>
|
||||
{edits == null
|
||||
? <p className="text-faint">{VT.editsUnrecorded}</p>
|
||||
: edits.length ? (
|
||||
<ul className="space-y-1">
|
||||
{edits.map((e, i) => (
|
||||
<li key={`${e.seq ?? i}`} className={clsx('flex flex-wrap items-baseline gap-x-2 rounded-md border px-2.5 py-1.5', e.superseded && 'opacity-60')}
|
||||
data-manifest-edit={e.kind}>
|
||||
<span className="chip">{VT.editKind[e.kind] ?? VT.editKind.fix}</span>
|
||||
<span className="min-w-0 flex-1 break-words">{e.title}</span>
|
||||
{e.superseded && <span className="text-2xs text-faint">{VT.editSuperseded}</span>}
|
||||
<span className="text-2xs text-faint">{VT.signedBy(e.signed_by || VT.unsigned)}</span>
|
||||
{e.at && <span className="tnum text-2xs text-faint">{formatDateTime(e.at)}</span>}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : <p className="text-faint">{VT.editsNone}</p>}
|
||||
</Block>
|
||||
|
||||
<Block id="notes" title={B.notes}>
|
||||
{rendered.length ? (
|
||||
<ul className="space-y-2">
|
||||
{rendered.map(([table, n]) => (
|
||||
<li key={table} className="space-y-0.5" data-manifest-note={table}>
|
||||
<div className="font-medium">{table}</div>
|
||||
{n.comment && <p className="leading-relaxed text-dim">{n.comment}</p>}
|
||||
{Object.entries(n.columns ?? {}).length > 0 && (
|
||||
<ul className="space-y-0.5 text-2xs text-dim">
|
||||
{Object.entries(n.columns ?? {}).map(([col, text]) => <li key={col}>{col}:{text}</li>)}
|
||||
</ul>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : <p className="text-faint">{VT.notesNone}</p>}
|
||||
</Block>
|
||||
</>
|
||||
)}
|
||||
|
||||
<section className="space-y-1.5 rounded-lg border bg-bg p-3" data-manifest-block="receipt">
|
||||
<ReceiptSummary receipt={receipt} />
|
||||
{outside.length > 0 && <p className="text-2xs text-faint" data-outside-note="">{VT.outsideTextNote}</p>}
|
||||
</section>
|
||||
|
||||
{!simple && (
|
||||
<Block id="ai" title={B.ai}>
|
||||
<p className={usage.length ? 'text-dim' : 'text-faint'}>
|
||||
{usage.length ? VT.aiUsage(usage.length, formatTokens(tokens), formatCost(cost)) : VT.aiNone}
|
||||
</p>
|
||||
</Block>
|
||||
)}
|
||||
|
||||
<RawJson summary={VT.manifestRaw} value={c} attr="data-manifest-raw" />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** 等宽的原文(原始清单、配方 JSON、核对的 SQL),默认收起:给要逐字核对的人看,带复制 */
|
||||
function RawJson({ summary, value, attr }: { summary: string; value: unknown; attr: string }) {
|
||||
const text = typeof value === 'string' ? value : JSON.stringify(value, null, 2)
|
||||
return (
|
||||
<details className="text-2xs" {...{ [attr]: '' }}>
|
||||
<summary className="cursor-pointer select-none text-faint hover:text-dim">{summary}</summary>
|
||||
<div className="mt-1.5 flex items-start gap-1.5">
|
||||
<pre className="mono max-h-80 min-w-0 flex-1 overflow-auto whitespace-pre-wrap break-all rounded-md border bg-[var(--bg)] p-2 leading-relaxed text-dim">
|
||||
{text}
|
||||
</pre>
|
||||
<CopyButton text={text} />
|
||||
</div>
|
||||
</details>
|
||||
)
|
||||
}
|
||||
@@ -4,17 +4,16 @@ import { AlertTriangle, ArrowLeft, ChevronRight, FileText, Info, RotateCcw, Scis
|
||||
import clsx from 'clsx'
|
||||
import { ApiError, api } from '../../api/client'
|
||||
import type {
|
||||
CheckResult, DataSource, ImportAcceptance, ImportRecord, ManifestOut, PeriodRef, RevokePlan, SnapshotOut, SnapshotPart,
|
||||
DataSource, ImportAcceptance, ImportRecord, ManifestOut, PeriodRef, RevokePlan, SnapshotOut, SnapshotPart,
|
||||
} from '../../types'
|
||||
import {
|
||||
CopyButton, EmptyState, ErrorState, Modal, Notice, Skeleton, TabPanel, Tabs, promptDialog, toast,
|
||||
EmptyState, ErrorState, Modal, Notice, Skeleton, TabPanel, Tabs, promptDialog, toast,
|
||||
} from '../../components/ui'
|
||||
import { localActor } from '../../lib/actor'
|
||||
import { humanizeError } from '../../lib/errors'
|
||||
import { formatBytes, formatCost, formatDateTime, formatTime, formatTokens, parseServerTime } from '../../lib/format'
|
||||
import { RAW_STATE_LABEL, RECIPE_ORIGIN_LABEL, RECIPE_TEXT, VERSIONS_TEXT as VT } from '../../lib/terms'
|
||||
import { CheckStatusChip } from './ImportReceipt'
|
||||
import { ReceiptSummary } from './ReceiptBlocks'
|
||||
import { formatBytes, formatDateTime, formatTime, parseServerTime } from '../../lib/format'
|
||||
import { RAW_STATE_LABEL, VERSIONS_TEXT as VT } from '../../lib/terms'
|
||||
import { ManifestBody } from './ManifestBody'
|
||||
|
||||
// ===========================================================================
|
||||
// 版本页(数据源卡片上的「版本」,P3-SPEC 7.8):当前版本的各期、历史版本、全部导入记录三个页签,
|
||||
@@ -703,216 +702,7 @@ function ManifestView({ sourceId, rec, onBack }: { sourceId: string; rec: Import
|
||||
? <ErrorState error={error} onRetry={() => setNonce((n) => n + 1)} />
|
||||
: !m
|
||||
? <Skeleton rows={5} height={36} gap={8} />
|
||||
: <ManifestBody m={m} rec={rec} />}
|
||||
: <ManifestBody m={m} fileName={rec.file_name} rawState={rec.raw_state} recipeSeq={rec.recipe_seq} />}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Block({ id, title, children }: { id: string; title: string; children: ReactNode }) {
|
||||
return (
|
||||
<section className="space-y-1.5 rounded-lg border bg-bg p-3 text-xs" data-manifest-block={id}>
|
||||
<h4 className="font-semibold">{title}</h4>
|
||||
{children}
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
const short = (sha: unknown) => (typeof sha === 'string' && sha ? sha.slice(0, 8) : '')
|
||||
const cellsShort = (cells: string[]) => cells.map((c) => c.slice(c.lastIndexOf('!') + 1)).join('、')
|
||||
const list = <T,>(v: unknown): T[] => (Array.isArray(v) ? (v as T[]) : [])
|
||||
|
||||
/**
|
||||
* 导入清单的各块。content 是服务端存的原样(内容寻址,取回时复验了哈希),字段都按「可能缺」读:期 2 的清单
|
||||
* 没有修改记录,简单导入只有导入回执
|
||||
*/
|
||||
function ManifestBody({ m, rec }: { m: ManifestOut; rec: ImportRecord }) {
|
||||
const c = m.content ?? {}
|
||||
const B = VT.manifestBlocks
|
||||
const simple = m.kind === 'build_report'
|
||||
const receipt = simple ? c : c.receipt
|
||||
const checks = list<CheckResult>(c.checks)
|
||||
const titleOf = (id: string) => checks.find((x) => x.id === id)?.title
|
||||
const accepts: ImportAcceptance[] = [
|
||||
...list<ImportAcceptance>(c.acceptances?.overrides).map((a) => ({ ...a, kind: 'override' })),
|
||||
...list<ImportAcceptance>(c.acceptances?.waivers).map((a) => ({ ...a, kind: 'waiver' })),
|
||||
]
|
||||
const confirmations = list<{ id: string; label: string; at?: string }>(c.confirmations)
|
||||
const edits = Array.isArray(c.edits) ? list<Record<string, any>>(c.edits) : null
|
||||
const rendered = Object.entries((c.notes?.rendered ?? {}) as Record<string, { comment?: string; columns?: Record<string, string> }>)
|
||||
const usage = list<Record<string, any>>(c.ai?.usage)
|
||||
const tokens = usage.reduce((n, u) => n + (Number(u.input_tokens) || 0) + (Number(u.output_tokens) || 0), 0)
|
||||
const cost = usage.reduce((n, u) => n + (Number(u.cost_usd) || 0), 0)
|
||||
const period = c.period as { start?: string; end?: string; source?: string; cells?: string[]; signed_by?: string | null } | null
|
||||
const outside = list(receipt?.outside_text)
|
||||
|
||||
return (
|
||||
<div className="space-y-2.5" data-manifest={m.kind} data-verified={m.verified ? 'true' : 'false'}>
|
||||
{simple
|
||||
? <Notice tone="info">{VT.manifestSimple}</Notice>
|
||||
: !m.verified && <Notice tone="err" attr={{ 'data-manifest-unverified': '' }}>{VT.manifestUnverified}</Notice>}
|
||||
|
||||
{!simple && (
|
||||
<>
|
||||
<Block id="file" title={B.file}>
|
||||
<div className="flex flex-wrap items-baseline gap-x-3 gap-y-1">
|
||||
<span className="mono break-all">{c.file?.name || rec.file_name}</span>
|
||||
{c.file?.size != null && <span className="tnum text-dim">{formatBytes(Number(c.file.size))}</span>}
|
||||
{short(c.file?.raw_sha256) && <span className="mono text-dim">{VT.sha(short(c.file?.raw_sha256))}</span>}
|
||||
<span className="chip">{RAW_STATE_LABEL[rec.raw_state] ?? rec.raw_state}</span>
|
||||
</div>
|
||||
{c.signed_by?.name && <div className="text-2xs text-faint">{VT.signedBy(c.signed_by.name)}</div>}
|
||||
</Block>
|
||||
|
||||
<Block id="recipe" title={B.recipe}>
|
||||
<div className="flex flex-wrap items-baseline gap-x-3 gap-y-1">
|
||||
{rec.recipe_seq != null && <span>{VT.recipeSeq(rec.recipe_seq)}</span>}
|
||||
{c.recipe?.origin && <span className="text-dim">{RECIPE_ORIGIN_LABEL[c.recipe.origin] ?? c.recipe.origin}</span>}
|
||||
{short(c.recipe?.sha256) && <span className="mono text-dim">{VT.sha(short(c.recipe?.sha256))}</span>}
|
||||
</div>
|
||||
{c.recipe?.canonical && <RawJson summary={VT.recipeJson} value={c.recipe.canonical} attr="data-recipe-json" />}
|
||||
</Block>
|
||||
|
||||
<Block id="period" title={B.period}>
|
||||
{period?.start && period?.end ? (
|
||||
<div className="space-y-0.5">
|
||||
<div className="tnum">{VT.period(period.start, period.end)}</div>
|
||||
<div className="text-2xs text-faint">
|
||||
{period.source === 'human'
|
||||
? `${VT.periodHuman},${VT.signedBy(period.signed_by || VT.unsigned)}`
|
||||
: period.cells?.length ? RECIPE_TEXT.periodFromCells(cellsShort(period.cells)) : ''}
|
||||
</div>
|
||||
</div>
|
||||
) : <p className="text-faint">{VT.periodUnknown}</p>}
|
||||
</Block>
|
||||
|
||||
<Block id="checks" title={B.checks}>
|
||||
{checks.length ? (
|
||||
<ul className="space-y-1.5">
|
||||
{checks.map((k) => (
|
||||
<li key={k.id} className="space-y-1 rounded-md border px-2.5 py-1.5" data-manifest-check={k.id} data-status={k.status}>
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<CheckStatusChip status={k.status} />
|
||||
<span className="mono text-faint">{k.id}</span>
|
||||
<span className="min-w-0 flex-1">{k.title}</span>
|
||||
{/* 计数照试运行回执的规则(ImportReceipt 的 CheckRow):「说明」类的 failed 是预期内的不相等个数,不写 */}
|
||||
{!!k.checked && k.status !== 'info' && (
|
||||
<span className="tnum text-2xs text-faint" data-check-counts="">
|
||||
{VT.checkCounts(k.checked, k.failed ?? 0, k.unverifiable ?? 0)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
{/* 清单是当证据核对的,细节不截断:服务端每个核对最多逐条列 20 格,其余写成「另有 N 格不一致」
|
||||
「N 格无法核对:…」,截掉的正是这几行 */}
|
||||
{!!k.details?.length && (
|
||||
<ul className="space-y-0.5 text-2xs leading-relaxed text-dim" data-check-details={k.details.length}>
|
||||
{k.details.map((d, i) => <li key={i}>{d}</li>)}
|
||||
</ul>
|
||||
)}
|
||||
{k.sql && <RawJson summary={VT.sql} value={k.params?.length ? `${k.sql}\n-- ${JSON.stringify(k.params)}` : k.sql} attr="data-check-sql" />}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : <p className="text-faint">{VT.checksNone}</p>}
|
||||
</Block>
|
||||
|
||||
<Block id="acceptances" title={B.acceptances}>
|
||||
{accepts.length ? (
|
||||
<ul className="space-y-1">
|
||||
{accepts.map((a, i) => (
|
||||
<li key={`${a.check_id}-${i}`} className="space-y-0.5 rounded-md border px-2.5 py-1.5" data-manifest-acceptance={a.check_id}>
|
||||
<div className="flex flex-wrap items-baseline gap-x-2">
|
||||
<span className="chip">{VT.acceptKind[a.kind] ?? VT.acceptKind.override}</span>
|
||||
<span className="min-w-0 flex-1 break-words">
|
||||
{VT.acceptRow(titleOf(a.check_id) ? `${a.check_id} ${titleOf(a.check_id)}` : a.check_id, a.reason)}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-x-2 text-2xs text-faint">
|
||||
<span>{VT.signedBy(a.signed_by || VT.unsigned)}</span>
|
||||
{a.at && <span className="tnum">{formatDateTime(a.at)}</span>}
|
||||
</div>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : <p className="text-faint">{VT.acceptancesNone}</p>}
|
||||
</Block>
|
||||
|
||||
<Block id="confirmations" title={B.confirmations}>
|
||||
{confirmations.length ? (
|
||||
<ul className="list-inside list-disc space-y-0.5 text-dim">
|
||||
{confirmations.map((x, i) => <li key={`${x.id}-${i}`}>{x.label}</li>)}
|
||||
</ul>
|
||||
) : <p className="text-faint">{VT.confirmationsNone}</p>}
|
||||
</Block>
|
||||
|
||||
<Block id="edits" title={B.edits}>
|
||||
{edits == null
|
||||
? <p className="text-faint">{VT.editsUnrecorded}</p>
|
||||
: edits.length ? (
|
||||
<ul className="space-y-1">
|
||||
{edits.map((e, i) => (
|
||||
<li key={`${e.seq ?? i}`} className={clsx('flex flex-wrap items-baseline gap-x-2 rounded-md border px-2.5 py-1.5', e.superseded && 'opacity-60')}
|
||||
data-manifest-edit={e.kind}>
|
||||
<span className="chip">{VT.editKind[e.kind] ?? VT.editKind.fix}</span>
|
||||
<span className="min-w-0 flex-1 break-words">{e.title}</span>
|
||||
{e.superseded && <span className="text-2xs text-faint">{VT.editSuperseded}</span>}
|
||||
<span className="text-2xs text-faint">{VT.signedBy(e.signed_by || VT.unsigned)}</span>
|
||||
{e.at && <span className="tnum text-2xs text-faint">{formatDateTime(e.at)}</span>}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : <p className="text-faint">{VT.editsNone}</p>}
|
||||
</Block>
|
||||
|
||||
<Block id="notes" title={B.notes}>
|
||||
{rendered.length ? (
|
||||
<ul className="space-y-2">
|
||||
{rendered.map(([table, n]) => (
|
||||
<li key={table} className="space-y-0.5" data-manifest-note={table}>
|
||||
<div className="font-medium">{table}</div>
|
||||
{n.comment && <p className="leading-relaxed text-dim">{n.comment}</p>}
|
||||
{Object.entries(n.columns ?? {}).length > 0 && (
|
||||
<ul className="space-y-0.5 text-2xs text-dim">
|
||||
{Object.entries(n.columns ?? {}).map(([col, text]) => <li key={col}>{col}:{text}</li>)}
|
||||
</ul>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
) : <p className="text-faint">{VT.notesNone}</p>}
|
||||
</Block>
|
||||
</>
|
||||
)}
|
||||
|
||||
<section className="space-y-1.5 rounded-lg border bg-bg p-3" data-manifest-block="receipt">
|
||||
<ReceiptSummary receipt={receipt} />
|
||||
{outside.length > 0 && <p className="text-2xs text-faint" data-outside-note="">{VT.outsideTextNote}</p>}
|
||||
</section>
|
||||
|
||||
{!simple && (
|
||||
<Block id="ai" title={B.ai}>
|
||||
<p className={usage.length ? 'text-dim' : 'text-faint'}>
|
||||
{usage.length ? VT.aiUsage(usage.length, formatTokens(tokens), formatCost(cost)) : VT.aiNone}
|
||||
</p>
|
||||
</Block>
|
||||
)}
|
||||
|
||||
<RawJson summary={VT.manifestRaw} value={c} attr="data-manifest-raw" />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** 等宽的原文(原始清单、配方 JSON、核对的 SQL),默认收起:给要逐字核对的人看,带复制 */
|
||||
function RawJson({ summary, value, attr }: { summary: string; value: unknown; attr: string }) {
|
||||
const text = typeof value === 'string' ? value : JSON.stringify(value, null, 2)
|
||||
return (
|
||||
<details className="text-2xs" {...{ [attr]: '' }}>
|
||||
<summary className="cursor-pointer select-none text-faint hover:text-dim">{summary}</summary>
|
||||
<div className="mt-1.5 flex items-start gap-1.5">
|
||||
<pre className="mono max-h-80 min-w-0 flex-1 overflow-auto whitespace-pre-wrap break-all rounded-md border bg-[var(--bg)] p-2 leading-relaxed text-dim">
|
||||
{text}
|
||||
</pre>
|
||||
<CopyButton text={text} />
|
||||
</div>
|
||||
</details>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ import type {
|
||||
} from '../types'
|
||||
import { ArtifactViewer, ResultTable } from './AssistantStream'
|
||||
import { useEvidenceHost } from './evidenceHost'
|
||||
import { EvidenceProvenance } from './EvidenceProvenance'
|
||||
import { CopyChip } from './Markdown'
|
||||
|
||||
/**
|
||||
@@ -58,6 +59,9 @@ import { CopyChip } from './Markdown'
|
||||
* 裁判拆档(JR):证据不足的写明缺什么;裁判模型和写作模型相同、模型不在价格表里时在「模型的解释」下提醒;
|
||||
* 表名实体步骤列出字段清单;讲方法、讲结构的句子在「挂的依据」下直接列出 SQL 和字段清单。
|
||||
*
|
||||
* Excel 导入期 4:上传表格的单元格多三节——数据版本、推断的来源、相关核对(EvidenceProvenance)。只在片段是单元格
|
||||
* 引用、且片段接口的查询步骤带 provenance: true 时请求推断来源接口;推断来源的缓存只在面板开着期间有效。
|
||||
*
|
||||
* 四种摆法:side 从右侧弹出(宽屏),drawer 从底部抽出(窄屏),inline 在 360px 的
|
||||
* 画布右栏里直接栏内展开,dock 放进页面给的一块常驻位置(记录页的「证据」页签)。
|
||||
* 都不是模态的:开着面板照样能在正文里走。
|
||||
@@ -82,6 +86,10 @@ export function EvidencePanel({ id, mode, doc, artifact, runId, runClass, view,
|
||||
onViolations: () => void
|
||||
}) {
|
||||
const titleId = `${id}-title`
|
||||
// 推断的来源只在面板这次打开期间缓存:回答里有当前状态(原件已清除、接受已作废)和取决于数据文件现状的结论,
|
||||
// 关掉面板(卸载)或换了运行就作废,再打开重新取。同一次打开里来回点同一片段只取一次
|
||||
const holdProvenance = useEvidence((s) => s.holdProvenance)
|
||||
useEffect(() => (runId ? holdProvenance(runId) : undefined), [runId, holdProvenance])
|
||||
const found = useMemo(() => (view.kind === 'seg' ? findSeg(doc, view.id) : null), [doc, view])
|
||||
const claimAt = useMemo(() => (view.kind === 'unit' ? findUnit(doc, view.id) : null), [doc, view])
|
||||
const overlay = useVerdicts(runId, doc.node_id || undefined)
|
||||
@@ -247,6 +255,10 @@ function SegmentBody({ panelId, doc, artifact, runId, runClass, seg, unit, block
|
||||
&& (cite?.kind === 'cell' || entry?.kind === 'query' || !!queries.length)
|
||||
const bad = state === 'none' || state === 'suspect' || state === 'unverified'
|
||||
const closest = closestNames(detail?.closest ?? entityStep?.closest)
|
||||
// 期 4:推断的来源。条件写死成「片段是单元格引用、它的查询步骤带了提示」:期 4 之前的运行、手工源(查询步骤没有
|
||||
// 提示)、指标片段(cite.kind 是 metric,哪怕它链上的查询步骤误带了提示)一个请求都不多发。接口答的是另一份报告
|
||||
// (docForeign)时 detail 为空、链是空的,也就不请求
|
||||
const provenanceOn = !!runId && cite?.kind === 'cell' && queries.some((q) => q.provenance === true)
|
||||
|
||||
// 画布右栏:点开片段时把证据路径交给画布(产出证据的节点实线、报告虚线),链取回来后再补全
|
||||
const host = useEvidenceHost()
|
||||
@@ -386,6 +398,8 @@ function SegmentBody({ panelId, doc, artifact, runId, runClass, seg, unit, block
|
||||
<QueryPart key={`${q.artifact ?? ''}-${i}`} id={queryId(i)} step={q} seal={seal} masked={masked} />
|
||||
))}
|
||||
|
||||
{provenanceOn && runId && <EvidenceProvenance runId={runId} segId={seg.id} report={report} />}
|
||||
|
||||
{state === 'deterministic' && !isMetric && !isInput && !isCell && !isEntity && !isQuote && (
|
||||
<p className="text-dim">{sourceOf(seg, doc) || detail?.note}</p>
|
||||
)}
|
||||
|
||||
@@ -0,0 +1,489 @@
|
||||
import { useEffect, useId, useState, type ReactNode } from 'react'
|
||||
import { createPortal } from 'react-dom'
|
||||
import { AlertTriangle, ChevronRight, FileText } from 'lucide-react'
|
||||
import clsx from 'clsx'
|
||||
import { ApiError, api } from '../api/client'
|
||||
import { ErrorState, Modal, Skeleton } from '../components/ui'
|
||||
import { NONE, formatDateTime } from '../lib/format'
|
||||
import { EVIDENCE_TEXT, RAW_STATE_LABEL, VERSIONS_TEXT } from '../lib/terms'
|
||||
import { ManifestBody } from '../pages/import/ManifestBody'
|
||||
import { segmentKey, useEvidence } from '../store/evidence'
|
||||
import type {
|
||||
EvidenceProvenance as Provenance, ManifestOut, ProvenanceCellSource, ProvenanceCheck, ProvenanceFromCell, ProvenancePart,
|
||||
ProvenanceVersion,
|
||||
} from '../types'
|
||||
import { CopyChip } from './Markdown'
|
||||
|
||||
/**
|
||||
* 证据面板里的「数据版本」「推断的来源」「相关核对」三节(Excel 导入期 4,P4-SPEC 4.1–4.3)。
|
||||
*
|
||||
* 上传表格的单元格:这次查询查的是哪一版数据(各期的统计期、文件、原件状态、写过的接受理由),这一格来自原表的
|
||||
* 哪个工作表、哪一格(日期、年份、行标签、分段标题各取自哪里),以及这一行、这一格的核对结果。
|
||||
*
|
||||
* 数据都来自推断来源接口(按需算,面板打开那个片段时才算)。调用方只在片段是单元格引用、且片段接口的查询步骤
|
||||
* 带 provenance: true 时挂这个组件,所以期 4 之前的运行、手工源、指标片段一个请求都不多发。
|
||||
*
|
||||
* 三节分两种性质,界面分开标注:
|
||||
* - 数据版本来自封存链上的导入清单(内容寻址),不加「推断」标注;
|
||||
* - 推断的来源、相关核对是先按主键认出这一行再推出来的,挂「推断,不属于已封存的证据」,用虚线左边框和
|
||||
* 别的几节区分。封存那一行(verdict)不受它影响,推断的来源不计入「已封存 · 核对一致」。
|
||||
*
|
||||
* 标题用 <h4>(读屏按标题跳得到),视觉照抄面板里 Part 的小标题;Part 本身不动——改它会改变期 4 之前所有面板的
|
||||
* DOM。标红的提示用自己的属性 data-ev-prov-alert,不用 data-ev-integrity:现有断言数过后者的个数。
|
||||
* 折叠、展开不加动效(P4-SPEC 4.3)。
|
||||
*/
|
||||
|
||||
const P = EVIDENCE_TEXT.provenance
|
||||
|
||||
/** 小标题的样式:同 EvidencePanel 的 Part(mb-1 text-2xs font-medium text-faint),只是换成真正的标题元素 */
|
||||
const HEAD = 'text-2xs font-medium text-faint'
|
||||
|
||||
/** 期数多于这个数时折叠:只展开这一格所在的一期和最近一期 */
|
||||
const FOLD_AT = 3
|
||||
|
||||
/** 原件 sha256 显示前几位(P4-SPEC 4.1:前 12 位) */
|
||||
const SHA_PREFIX = 12
|
||||
|
||||
/**
|
||||
* 导入、清除、作废记录的时间。没记(契约里 at 可以是 null)或认不出时给空串,由调用方略去这一项:formatDateTime
|
||||
* 拿不到时间时返回「—」,它是真值,filter(Boolean) 去不掉,会在「原件已清除 · — · 理由:…」里留下一个孤立的「—」
|
||||
*/
|
||||
function stamp(at: string | null | undefined): string {
|
||||
const text = formatDateTime(at)
|
||||
return text === NONE ? '' : text
|
||||
}
|
||||
|
||||
export function EvidenceProvenance({ runId, segId, report }: { runId: string; segId: string; report?: string }) {
|
||||
const key = segmentKey(runId, segId, report)
|
||||
const slot = useEvidence((s) => s.provenance[key])
|
||||
const load = useEvidence((s) => s.loadProvenance)
|
||||
// 槽没了就再取:面板开着时不会被清(清缓存只在最后一个面板关掉以后),这里只是不让一节永远停在「正在推断」
|
||||
const missing = !slot
|
||||
useEffect(() => {
|
||||
if (missing) void load(runId, segId, report)
|
||||
}, [missing, runId, segId, report, load])
|
||||
|
||||
if (!slot || slot.status === 'loading') {
|
||||
// 取的过程中只画这一节的加载态,不先画三节空壳
|
||||
return (
|
||||
<SourceShell state="loading">
|
||||
<p className="text-dim" data-ev-prov-loading="">{P.loading}</p>
|
||||
</SourceShell>
|
||||
)
|
||||
}
|
||||
if (slot.status === 'error' || !slot.data || typeof slot.data !== 'object') {
|
||||
// 老服务端没有这个接口(404、405 且没有机读码):三节都不画,和期 4 之前一样
|
||||
if (oldServer(slot.error)) return null
|
||||
return (
|
||||
<SourceShell state="error">
|
||||
<p className="text-dim" data-ev-prov-failed="">{P.failed}</p>
|
||||
</SourceShell>
|
||||
)
|
||||
}
|
||||
// 按片段换实例:展开的「技术细节」「另有 n 期」是这一格的状态,点到别的格子时收起
|
||||
return <Answer key={key} d={slot.data} />
|
||||
}
|
||||
|
||||
function oldServer(error: unknown): boolean {
|
||||
return error instanceof ApiError && (error.status === 404 || error.status === 405) && !error.code
|
||||
}
|
||||
|
||||
/** 三节画不画、推断的来源那一节画什么(P4-SPEC 4.1 的渲染矩阵) */
|
||||
interface Plan {
|
||||
version: boolean
|
||||
source: 'none' | 'reason' | 'alert' | 'cell'
|
||||
checks: boolean
|
||||
}
|
||||
|
||||
const NOTHING: Plan = { version: false, source: 'none', checks: false }
|
||||
|
||||
/**
|
||||
* 渲染矩阵。表外的组合不该出现(服务端的契约检查会拦),出现时按「table_only、无 alert」画:给出数据版本(有的话)
|
||||
* 和一句原因。alert 只在矩阵里列出的组合下画成红色
|
||||
*/
|
||||
function planOf(d: Provenance): Plan {
|
||||
const code = d.reason?.code
|
||||
const alert = d.alert?.code
|
||||
if (d.status === 'none') {
|
||||
if (!alert && (code === 'legacy_doc' || code === 'not_upload' || code === 'not_cell')) return NOTHING
|
||||
if (!alert && (code === 'not_sealed' || code === 'simple_upload')) return { version: false, source: 'reason', checks: false }
|
||||
if (alert === 'manifest_unreadable' || alert === 'chain_mismatch') return { version: false, source: 'alert', checks: false }
|
||||
} else if (d.status === 'table_only') {
|
||||
if (alert === 'db_tampered' || alert === 'chain_mismatch') return { version: true, source: 'alert', checks: false }
|
||||
if (!alert) return { version: true, source: 'reason', checks: false }
|
||||
} else if (d.status === 'inferred') {
|
||||
if (!alert && d.cell_source) return { version: true, source: 'cell', checks: true }
|
||||
}
|
||||
return { version: true, source: 'reason', checks: false }
|
||||
}
|
||||
|
||||
function Answer({ d }: { d: Provenance }) {
|
||||
const plan = planOf(d)
|
||||
const version = plan.version && d.version ? d.version : null
|
||||
const [manifest, setManifest] = useState<ProvenancePart | null>(null)
|
||||
if (plan.source === 'none') return null
|
||||
return (
|
||||
<>
|
||||
{version && <VersionPart v={version} sealed={d.sealed !== false} onManifest={setManifest} />}
|
||||
<SourceShell state={d.status}>
|
||||
{plan.source === 'alert' && d.alert && (
|
||||
// 标红:样式同 IntegrityList(失败色的框),放在这一节最前面;有它时不再画 reason.text
|
||||
<p className="flex items-start gap-1.5 rounded border px-2 py-1 [overflow-wrap:anywhere]" data-ev-prov-alert={d.alert.code}
|
||||
style={{ color: 'var(--st-failed)', borderColor: 'var(--st-failed)', background: 'var(--st-failed-soft)' }}>
|
||||
<AlertTriangle size={12} aria-hidden className="mt-0.5 shrink-0" />
|
||||
<span>{d.alert.text}</span>
|
||||
</p>
|
||||
)}
|
||||
{plan.source === 'reason' && d.reason?.text && (
|
||||
<p className="text-dim [overflow-wrap:anywhere]" data-ev-prov-reason={d.reason.code}>{d.reason.text}</p>
|
||||
)}
|
||||
{plan.source === 'cell' && d.cell_source && <CellPart cs={d.cell_source} version={d.version} />}
|
||||
</SourceShell>
|
||||
{plan.checks && Array.isArray(d.checks) && d.checks.length > 0 && <ChecksPart checks={d.checks} />}
|
||||
{manifest && createPortal(
|
||||
// 弹窗挂到 body 上:侧边面板是 fixed 的一层,里面的遮罩盖不住整页(同「打开完整快照」)
|
||||
<ManifestDialog part={manifest} onClose={() => setManifest(null)} />,
|
||||
document.body,
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
/** 「推断,不属于已封存的证据」:推断的来源、相关核对两节的标题旁都挂 */
|
||||
function Badge() {
|
||||
return (
|
||||
<span className="chip shrink-0 text-faint" data-ev-prov-badge=""
|
||||
style={{ borderColor: 'var(--border-strong)', borderStyle: 'dashed' }}>
|
||||
{P.badge}
|
||||
</span>
|
||||
)
|
||||
}
|
||||
|
||||
/** 推断出来的一节:虚线左边框,标题旁挂「推断」标签 */
|
||||
function Inferred({ title, children, ...rest }: { title: string; children: ReactNode } & Record<`data-${string}`, string>) {
|
||||
return (
|
||||
<div className="border-l-2 border-dashed pl-2" style={{ borderColor: 'var(--border-strong)' }} {...rest}>
|
||||
<div className="mb-1 flex flex-wrap items-center gap-x-1.5 gap-y-0.5">
|
||||
<h4 className={HEAD}>{title}</h4>
|
||||
<Badge />
|
||||
</div>
|
||||
{children}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** 推断的来源一节的外壳:data-ev-provenance 写这一节的状态(inferred / table_only / none,取的过程中 loading,出错 error) */
|
||||
function SourceShell({ state, children }: { state: string; children: ReactNode }) {
|
||||
return <Inferred title={P.title} data-ev-provenance={state}>{children}</Inferred>
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 数据版本
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function VersionPart({ v, sealed, onManifest }: {
|
||||
v: ProvenanceVersion; sealed: boolean; onManifest: (part: ProvenancePart) => void
|
||||
}) {
|
||||
const [open, setOpen] = useState(false)
|
||||
const moreId = useId()
|
||||
const parts = Array.isArray(v.parts) ? v.parts : []
|
||||
// 多于 3 期:这一格所在的一期和最近一期展开,其余收进「另有 n 期」
|
||||
const fold = parts.length > FOLD_AT
|
||||
const latest = parts[parts.length - 1]
|
||||
const keep = parts.filter((p) => p.has_row || p === latest)
|
||||
const folded = fold ? parts.length - keep.length : 0
|
||||
const shown = fold && !open ? keep : parts
|
||||
const tables = (Array.isArray(v.tables) ? v.tables : []).filter((t) => t && t.name)
|
||||
return (
|
||||
<div data-ev-prov-version="">
|
||||
<h4 className={clsx('mb-1', HEAD)}>{P.version}</h4>
|
||||
{!sealed && (
|
||||
// 封存那一行已经照实说了状态,这里只是不让「来自封存链」的说法越界:dim 色,不标红
|
||||
<p className="mb-1 text-2xs text-dim" data-ev-prov-unsealed="">{P.unsealed}</p>
|
||||
)}
|
||||
<p className="[overflow-wrap:anywhere]" data-ev-prov-head="">{P.versionHead(v.source, v.mode, parts.length)}</p>
|
||||
{tables.length > 0 && (
|
||||
<p className="mt-0.5 text-2xs text-dim [overflow-wrap:anywhere]" data-ev-prov-tables="">
|
||||
{P.tables(tables.map((t) => (t.kind === 'reported_total' ? P.totalTable(t.name) : t.name)))}
|
||||
</p>
|
||||
)}
|
||||
<ul className="mt-1 space-y-1.5" id={moreId}>
|
||||
{shown.map((p) => (
|
||||
<PartRow key={`${p.import_id}-${p.seq}`} p={p} manifest={v.manifest_view !== false} onManifest={onManifest} />
|
||||
))}
|
||||
</ul>
|
||||
{fold && folded > 0 && (
|
||||
// 展开后按钮还在、焦点留在它上面(同一个元素),再按一次收起
|
||||
<button type="button" className="btn btn-xs btn-ghost -ml-1 mt-1" aria-expanded={open} aria-controls={moreId}
|
||||
data-ev-prov-more="" onClick={() => setOpen((x) => !x)}>
|
||||
<ChevronRight size={11} aria-hidden className={clsx(open && 'rotate-90')} />
|
||||
{P.morePeriods(folded)}
|
||||
</button>
|
||||
)}
|
||||
{v.manifest_view === false && (
|
||||
// 清单里有核对细节里的数、标签原文、区域外文字:数据源设了遮罩时面板自己的入口不多暴露
|
||||
<p className="mt-1 text-2xs text-dim" data-ev-prov-manifest-masked="">{P.manifestMasked}</p>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** 当前状态(取自导入记录,不在封存范围内):作废、清除后面加这枚小字 */
|
||||
function Current() {
|
||||
return <span className="chip ml-1.5 text-faint" data-ev-prov-current="">{P.current}</span>
|
||||
}
|
||||
|
||||
function PartRow({ p, manifest, onManifest }: {
|
||||
p: ProvenancePart; manifest: boolean; onManifest: (part: ProvenancePart) => void
|
||||
}) {
|
||||
const period = p.period && p.period.start && p.period.end ? p.period : null
|
||||
const purged = p.raw_state === 'purged' || !!p.purged
|
||||
const sha = typeof p.raw_sha256 === 'string' ? p.raw_sha256.slice(0, SHA_PREFIX) : ''
|
||||
const committed = stamp(p.committed_at)
|
||||
const facts = [
|
||||
p.file_name ? P.file(p.file_name, sha) : '',
|
||||
p.region ? P.region(p.region, p.excluded_rows) : p.excluded_rows ? P.excluded(p.excluded_rows) : '',
|
||||
period?.source === 'human' ? `${VERSIONS_TEXT.periodHuman}(${VERSIONS_TEXT.signedBy(period.signed_by || VERSIONS_TEXT.unsigned)})` : '',
|
||||
// 清除过的另起一行写清除记录和「当前状态」,这里不重复
|
||||
!purged && p.raw_state ? RAW_STATE_LABEL[p.raw_state] ?? '' : '',
|
||||
committed ? P.committedAt(committed) : '',
|
||||
p.recipe_seq != null ? VERSIONS_TEXT.recipeSeq(p.recipe_seq) : '',
|
||||
p.signed_by ? VERSIONS_TEXT.signedBy(p.signed_by) : '',
|
||||
].filter(Boolean)
|
||||
const accepts = Array.isArray(p.acceptances) ? p.acceptances : []
|
||||
return (
|
||||
<li className="min-w-0" data-ev-prov-part={String(p.seq)} data-ev-prov-has-row={p.has_row ? '' : undefined}>
|
||||
<div className="flex flex-wrap items-baseline gap-x-2 gap-y-0.5">
|
||||
<span className="tnum font-medium">{period ? P.part(p.seq, period.start, period.end) : P.partNoPeriod(p.seq)}</span>
|
||||
{p.has_row && (
|
||||
<span className="chip" style={{ color: 'var(--accent)', borderColor: 'var(--accent)' }} data-ev-prov-here="">{P.hasRow}</span>
|
||||
)}
|
||||
</div>
|
||||
{facts.length > 0 && <p className="text-2xs text-dim [overflow-wrap:anywhere]">{facts.join(' · ')}</p>}
|
||||
{purged && (
|
||||
<p className="mt-0.5 text-2xs text-dim [overflow-wrap:anywhere]" data-ev-prov-purged="">
|
||||
{RAW_STATE_LABEL.purged}
|
||||
{p.purged ? ` · ${P.stateDetail(stamp(p.purged.at), p.purged.reason, p.purged.signed_by)}` : ''}
|
||||
<Current />
|
||||
</p>
|
||||
)}
|
||||
{p.revoked && (
|
||||
<p className="mt-0.5 text-2xs [overflow-wrap:anywhere]" style={{ color: 'var(--st-waiting)' }} data-ev-prov-revoked="">
|
||||
{P.revoked} · {P.stateDetail(stamp(p.revoked.at), p.revoked.reason, p.revoked.signed_by)}
|
||||
<Current />
|
||||
</p>
|
||||
)}
|
||||
{accepts.length > 0 && (
|
||||
// 接受理由在清单里,是当时的事实:作废之后照旧列出
|
||||
<ul className="ml-3 mt-0.5 space-y-0.5 text-2xs text-dim">
|
||||
{accepts.map((a, i) => (
|
||||
<li key={`${a.check_id}-${i}`} className="[overflow-wrap:anywhere]" data-ev-prov-acceptance={a.check_id}>
|
||||
{P.acceptedLine(a.title || a.check_id, a.reason, a.signed_by)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
{manifest && p.manifest && (
|
||||
<button type="button" className="btn btn-xs btn-ghost -ml-1 mt-0.5" data-ev-prov-manifest={p.manifest}
|
||||
onClick={() => onManifest(p)}>
|
||||
<FileText size={11} aria-hidden /> {P.manifest}
|
||||
</button>
|
||||
)}
|
||||
</li>
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 查看导入清单:版本页的清单渲染(ManifestBody),内容按清单的工件 id 经工件接口取(内容寻址,取回时复验哈希)。
|
||||
* 不用 ArtifactViewer:它对非表格内容整份 JSON 输出,遮罩不起作用,64 KB 的清单也没法读
|
||||
*/
|
||||
function ManifestDialog({ part, onClose }: { part: ProvenancePart; onClose: () => void }) {
|
||||
const [state, setState] = useState<{ id: string; content?: unknown; error?: unknown } | null>(null)
|
||||
useEffect(() => {
|
||||
let alive = true
|
||||
setState({ id: part.manifest })
|
||||
api.artifact(part.manifest).then(
|
||||
(res) => { if (alive) setState({ id: part.manifest, content: res?.content }) },
|
||||
(error) => { if (alive) setState({ id: part.manifest, error }) },
|
||||
)
|
||||
return () => { alive = false }
|
||||
}, [part.manifest])
|
||||
const mine = state?.id === part.manifest ? state : null
|
||||
const m: ManifestOut | null = mine?.content && typeof mine.content === 'object'
|
||||
? { artifact_id: part.manifest, verified: true, kind: 'import_manifest', content: mine.content as Record<string, any> }
|
||||
: null
|
||||
return (
|
||||
<Modal open onClose={onClose} width={920} title={VERSIONS_TEXT.manifestTitle(part.seq)}>
|
||||
{mine?.error
|
||||
? <ErrorState error={mine.error} />
|
||||
: !m
|
||||
? <Skeleton rows={5} height={36} gap={8} />
|
||||
: (
|
||||
<div data-ev-prov-manifest-view={part.manifest}>
|
||||
<ManifestBody m={m} fileName={part.file_name} rawState={part.raw_state} recipeSeq={part.recipe_seq} />
|
||||
</div>
|
||||
)}
|
||||
</Modal>
|
||||
)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 推断的来源:格子与逐条
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function CellPart({ cs, version }: { cs: ProvenanceCellSource; version: ProvenanceVersion | null }) {
|
||||
const [tech, setTech] = useState(false)
|
||||
const techId = useId()
|
||||
const parts = version?.parts ?? []
|
||||
const part = parts.find((p) => p.has_row) ?? parts.find((p) => p.seq === cs.part_seq)
|
||||
const items = fromItems(cs)
|
||||
const recheck = cs.recheck
|
||||
const sql = recheck?.sql ? (recheck.params?.length ? `${recheck.sql}\n-- ${JSON.stringify(recheck.params)}` : recheck.sql) : ''
|
||||
return (
|
||||
<>
|
||||
<p className="text-sm font-semibold [overflow-wrap:anywhere]" data-ev-prov-cell="">{P.cell(cs.sheet, cs.cell)}</p>
|
||||
{parts.length > 1 && (
|
||||
// 多期时坐标是哪一期原件的坐标:同一个位置在别的期的文件里是另一个数
|
||||
<p className="text-2xs text-dim [overflow-wrap:anywhere]" data-ev-prov-from-part="">
|
||||
{P.fromPart(cs.part_seq, part?.file_name ?? null)}
|
||||
</p>
|
||||
)}
|
||||
{items.length > 0 && (
|
||||
<ul className="mt-1 space-y-0.5 [overflow-wrap:anywhere]" data-ev-prov-from="">
|
||||
{items.map((it) => <li key={it.key} data-ev-prov-item={it.kind}>{it.text}</li>)}
|
||||
</ul>
|
||||
)}
|
||||
{cs.merged_fill && (
|
||||
<p className="mt-1 text-2xs [overflow-wrap:anywhere]" style={{ color: 'var(--st-waiting)' }} data-ev-prov-merged="">{P.mergedFill}</p>
|
||||
)}
|
||||
{cs.raw_purged && (
|
||||
<p className="mt-1 text-2xs [overflow-wrap:anywhere]" style={{ color: 'var(--st-waiting)' }} data-ev-prov-raw="">{P.rawPurged}</p>
|
||||
)}
|
||||
{(recheck?.ok || sql) && (
|
||||
<div className="mt-1 flex flex-wrap items-center gap-x-2 gap-y-0.5 text-2xs text-faint">
|
||||
{recheck?.ok && <span data-ev-prov-recheck="">{P.recheck}</span>}
|
||||
{sql && (
|
||||
<button type="button" className="inline-flex items-center gap-0.5 rounded px-1 transition-colors hover:bg-hover hover:text-fg"
|
||||
aria-expanded={tech} aria-controls={techId} data-ev-prov-details="" onClick={() => setTech((x) => !x)}>
|
||||
<ChevronRight size={10} aria-hidden className={clsx(tech && 'rotate-90')} />
|
||||
{P.details}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{sql && (
|
||||
// 收起时留在 DOM 里(hidden):按钮的 aria-controls 始终指得到它
|
||||
<div className="mt-1" id={techId} hidden={!tech} data-ev-prov-tech="">
|
||||
<div className="mb-0.5 flex items-center gap-2">
|
||||
<span className="min-w-0 flex-1 text-2xs font-medium text-faint">{P.recheckSql}</span>
|
||||
<CopyChip label={EVIDENCE_TEXT.copySql} text={() => sql} />
|
||||
</div>
|
||||
<pre className="mono max-h-28 overflow-auto whitespace-pre-wrap rounded bg-bg px-1.5 py-1 text-2xs leading-relaxed text-dim [overflow-wrap:anywhere]">
|
||||
{sql}
|
||||
</pre>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
interface Item { key: string; kind: string; text: string }
|
||||
|
||||
/**
|
||||
* 逐条:这一行其余键列各取自哪一格,加上年份、规范写法、合计口径。顺序照 4.1:日期 → 年份 → 行标签 → 分段标题 →
|
||||
* 规范写法。坐标和格子同一张工作表时只写坐标,不同时带上工作表名。列表的多行表头合成一条
|
||||
*/
|
||||
function fromItems(cs: ProvenanceCellSource): Item[] {
|
||||
const from: ProvenanceFromCell[] = Array.isArray(cs.from) ? cs.from.filter((f) => f && f.cell) : []
|
||||
const ref = (f: ProvenanceFromCell) => (f.sheet && f.sheet !== cs.sheet ? `${f.sheet}!${f.cell}` : f.cell)
|
||||
const year = yearItem(cs)
|
||||
const heads = from.filter((f) => f.role === 'col_header')
|
||||
const out: Item[] = []
|
||||
let yearDone = false
|
||||
from.forEach((f, i) => {
|
||||
const key = `${f.role}-${i}`
|
||||
if (f.role === 'axis_header') {
|
||||
out.push({ key, kind: f.role, text: P.axisHeader(ref(f)) })
|
||||
if (year && !yearDone) { out.push(year); yearDone = true }
|
||||
} else if (f.role === 'row_label') {
|
||||
out.push({ key, kind: f.role, text: P.rowLabel(ref(f), f.text) })
|
||||
} else if (f.role === 'section_title') {
|
||||
out.push({ key, kind: f.role, text: P.sectionTitle(ref(f), f.locate_title) })
|
||||
} else if (f.role === 'total_label') {
|
||||
out.push({ key, kind: f.role, text: P.totalLabel(ref(f), f.text) })
|
||||
} else if (f.role === 'col_header' && f === heads[0]) {
|
||||
out.push({ key, kind: f.role, text: P.colHeader(heads.map(ref).join('、')) })
|
||||
}
|
||||
})
|
||||
if (year && !yearDone) out.push(year)
|
||||
if (cs.canonical?.raw && cs.canonical.canonical) {
|
||||
out.push({ key: 'canonical', kind: 'canonical', text: P.canonical(cs.canonical.raw, cs.canonical.canonical) })
|
||||
}
|
||||
if (cs.kind === 'reported_total') out.push({ key: 'total', kind: 'reported_total', text: P.reportedTotal })
|
||||
return out
|
||||
}
|
||||
|
||||
function yearItem(cs: ProvenanceCellSource): Item | null {
|
||||
const y = cs.year
|
||||
if (!y) return null
|
||||
// mixed:清单只记每块的表头形式,逐格判断不了,照实说,不断言这一格属于哪一种
|
||||
const text = y.mixed ? P.yearMixed
|
||||
: y.source === 'human' ? P.yearHuman(y.signed_by || VERSIONS_TEXT.unsigned)
|
||||
: P.yearPeriod((Array.isArray(y.cells) ? y.cells : []).join('、'))
|
||||
return { key: 'year', kind: 'year', text }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 相关核对
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 状态的颜色只用令牌:成立 / 一致是 done,不成立是 failed,未能核对、无法确定是 waiting,其余中性 */
|
||||
const TONE: Record<string, string> = {
|
||||
passed: 'var(--st-done)',
|
||||
ok: 'var(--st-done)',
|
||||
mismatch: 'var(--st-failed)',
|
||||
unverifiable: 'var(--st-waiting)',
|
||||
unknown: 'var(--st-waiting)',
|
||||
}
|
||||
|
||||
function ChecksPart({ checks }: { checks: ProvenanceCheck[] }) {
|
||||
return (
|
||||
<Inferred title={P.checks} data-ev-prov-checks="">
|
||||
<ul className="space-y-1">
|
||||
{checks.map((c, i) => <CheckRow key={`${c.id}-${i}`} c={c} />)}
|
||||
</ul>
|
||||
</Inferred>
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* 一条核对:「编号 标题 — 这一行成立」(关系核对看行级结果,合计看格级结果,其余看本期结论)。本期结论不成立或未能
|
||||
* 核对时另起一行写出来,写过理由的接在后面:「本期不成立 · 已接受,理由:…」——行级成立、本期不成立(别的日子不成立、
|
||||
* 已接受)两件事都要看得到。口径不同(R2)不判行,本期结论就是「口径说明」,和别的核对同一种写法,扫读时对得齐
|
||||
*/
|
||||
function CheckRow({ c }: { c: ProvenanceCheck }) {
|
||||
const main: { code: string; text: string } | null = c.row_status
|
||||
? { code: c.row_status, text: P.row[c.row_status] ?? c.row_status }
|
||||
: c.cell_status ? { code: c.cell_status, text: P.cellStatus[c.cell_status] ?? c.cell_status }
|
||||
: null
|
||||
const partText = P.partStatus[c.part_status] ?? c.part_status
|
||||
// 没有行级、格级结果的(统计期、口径之外的核对)主行就写本期结论,理由接在同一行
|
||||
const lead = main ?? { code: c.part_status, text: partText }
|
||||
const accepted = c.acceptance ? P.accepted(c.acceptance.reason, c.acceptance.signed_by) : ''
|
||||
const showPart = !!main && (c.part_status === 'mismatch' || c.part_status === 'unverifiable')
|
||||
const second = main ? [showPart ? partText : '', accepted].filter(Boolean).join(' · ') : ''
|
||||
return (
|
||||
<li className="min-w-0 [overflow-wrap:anywhere]" data-ev-prov-check={c.id} data-ev-prov-part-status={c.part_status}
|
||||
data-ev-prov-row={c.row_status ?? undefined} data-ev-prov-cell-status={c.cell_status ?? undefined}>
|
||||
<p>
|
||||
<span className="mono mr-1 text-faint">{c.id}</span>
|
||||
<span>{c.title}</span>
|
||||
{' — '}
|
||||
<span style={{ color: TONE[lead.code] ?? 'var(--text-dim)' }} data-ev-prov-status="">{lead.text}</span>
|
||||
{!main && accepted && <span className="text-dim"> · {accepted}</span>}
|
||||
</p>
|
||||
{second && <p className="text-2xs text-dim" data-ev-prov-period="">{second}</p>}
|
||||
{c.detail && <p className="text-2xs text-faint" data-ev-prov-detail="">{c.detail}</p>}
|
||||
</li>
|
||||
)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,7 +3,8 @@ import { create } from 'zustand'
|
||||
import { ApiError, api } from '../api/client'
|
||||
import { graphVerdicts, judgedVerdicts, sqlOf } from '../lib/evidence'
|
||||
import type {
|
||||
EvidenceDocData, EvidenceGraph, EvidenceJudgeResult, EvidenceSegmentDetail, EvidenceStats, EvidenceVerdict,
|
||||
EvidenceDocData, EvidenceGraph, EvidenceJudgeResult, EvidenceProvenance, EvidenceSegmentDetail, EvidenceStats,
|
||||
EvidenceVerdict,
|
||||
} from '../types'
|
||||
|
||||
/**
|
||||
@@ -20,6 +21,11 @@ import type {
|
||||
* 四期:封存之后按需追加的结论句判定按 `runId@报告` 记在 judged 里(按需裁判接口、片段接口叠上的
|
||||
* 判定都记进来),正文的句末徽标、横幅的结论句计数据此盖过文档里的「未裁判 · 按需」。封存的文档本身
|
||||
* 一个字不改。同一句正在判的不重复发;判过的(判定不是未裁判)不再给按钮,也就不会再发
|
||||
*
|
||||
* Excel 导入期 4:推断的来源按片段的缓存键记在 provenance 里,但**只在面板开着期间有效**。回答里有当前状态
|
||||
* (原件已清除、接受已作废)和取决于数据文件现状的结论(被修改过、已回收),在版本页清除、作废、回滚之后,
|
||||
* 关掉面板再打开就要看到新结论。所以面板挂载时 holdProvenance,最后一个面板关掉(或换了运行)时清掉这次
|
||||
* 运行的槽;同一次打开里来回点同一片段只取一次
|
||||
*/
|
||||
|
||||
export interface Slot<T> {
|
||||
@@ -69,6 +75,14 @@ interface EvidenceState {
|
||||
asks: Record<string, Slot<EvidenceJudgeResult>>
|
||||
/** 运行类别(formal / exploratory):没开裁判的文档靠它认探索运行,按需取一次 */
|
||||
runClasses: Record<string, Slot<string | null>>
|
||||
/** 推断的来源(Excel 导入期 4),键同 segmentKey。只在面板开着期间有效,见 holdProvenance */
|
||||
provenance: Record<string, Slot<EvidenceProvenance>>
|
||||
loadProvenance: (runId: string, segmentId: string, report?: string) => Promise<void>
|
||||
/**
|
||||
* 证据面板挂载时调用,返回释放函数(面板卸载、换了运行时调)。这次运行最后一个面板释放后,清掉它的推断来源槽:
|
||||
* 关掉面板再打开会重新取
|
||||
*/
|
||||
holdProvenance: (runId: string) => () => void
|
||||
/** 记下几句封存后追加的判定(片段接口叠上的、证据图带来的) */
|
||||
noteVerdicts: (runId: string, report: string | undefined, verdicts: Record<string, EvidenceVerdict>) => void
|
||||
/** 请模型判断这几句(探索运行)。判定记进 judged;答复(含触顶、出错)记进 asks */
|
||||
@@ -101,12 +115,17 @@ export class NotADocError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 每次运行开着几个证据面板。放在模块里不放进 store:它只决定什么时候清缓存,界面不读它,变了也不该重渲染
|
||||
*/
|
||||
const provenanceHolds = new Map<string, number>()
|
||||
|
||||
export const useEvidence = create<EvidenceState>((set, get) => {
|
||||
/**
|
||||
* 取一次、记进 table[key];正在取或已经取到的不再发。取的途中这一格被清掉或换了(dropLatest 之后
|
||||
* 又起了一次),回来的结果作废:不能让早先发出的请求把后来那份盖掉
|
||||
*/
|
||||
async function fill<K extends 'docs' | 'graphs' | 'segments' | 'latest' | 'sqls'>(
|
||||
async function fill<K extends 'docs' | 'graphs' | 'segments' | 'latest' | 'sqls' | 'provenance'>(
|
||||
table: K, key: string, fetcher: () => Promise<NonNullable<EvidenceState[K][string]['data']>>, force = false,
|
||||
) {
|
||||
const cur = get()[table][key]
|
||||
@@ -190,6 +209,33 @@ export const useEvidence = create<EvidenceState>((set, get) => {
|
||||
mark({ status: 'error', error })
|
||||
}
|
||||
},
|
||||
provenance: {},
|
||||
loadProvenance: (runId, segmentId, report) =>
|
||||
fill('provenance', segmentKey(runId, segmentId, report), () => api.evidence.provenance(runId, segmentId, { report })),
|
||||
holdProvenance: (runId) => {
|
||||
provenanceHolds.set(runId, (provenanceHolds.get(runId) ?? 0) + 1)
|
||||
let released = false
|
||||
return () => {
|
||||
if (released) return
|
||||
released = true
|
||||
const left = (provenanceHolds.get(runId) ?? 1) - 1
|
||||
if (left > 0) {
|
||||
provenanceHolds.set(runId, left)
|
||||
return
|
||||
}
|
||||
provenanceHolds.delete(runId)
|
||||
// 推到这一轮之后再清:开发期 StrictMode 把挂载的 effect 先清理、紧接着再跑一遍(又 hold 回来),
|
||||
// 同一次提交里一个面板关、另一个面板开也是这样——这两种都不算「面板关了」
|
||||
queueMicrotask(() => {
|
||||
if (provenanceHolds.has(runId)) return
|
||||
const prefix = `${runId}:`
|
||||
const cur = get().provenance
|
||||
if (!Object.keys(cur).some((k) => k.startsWith(prefix))) return
|
||||
// 正在取的也清掉:fill 只在槽还是它自己放的那一份时才落结果,晚到的回包作废
|
||||
set({ provenance: Object.fromEntries(Object.entries(cur).filter(([k]) => !k.startsWith(prefix))) })
|
||||
})
|
||||
}
|
||||
},
|
||||
loadRunClass: (runId) => {
|
||||
const cur = get().runClasses[runId]
|
||||
if (cur && cur.status !== 'error') return Promise.resolve()
|
||||
|
||||
@@ -550,6 +550,11 @@ export interface OutsideText {
|
||||
text: string
|
||||
kind: 'text' | 'text_digits' | (string & {})
|
||||
period_source?: boolean
|
||||
/**
|
||||
* 期 4:所在行或列是否被隐藏(执行器写入)。true 在隐藏行列里;false 可见;null 或没有这个键:导入时没有记录
|
||||
* (期 4 之前的清单)。裁判摘录只送 false 的项,界面照常显示全部区域外文字
|
||||
*/
|
||||
hidden?: boolean | null
|
||||
}
|
||||
|
||||
/** 试运行回执(Extraction 去掉问题和核对之后给界面看的部分) */
|
||||
@@ -2117,6 +2122,11 @@ export interface EvidenceStep extends Omit<EvidenceInput, 'status'> {
|
||||
note?: string
|
||||
/** 数据源改名或删掉了、按查询当时记下的遮罩处理时的那句说明。行照样有 */
|
||||
mask_note?: string
|
||||
/**
|
||||
* 期 4:可以请求推断来源(GET …/segments/{sid}/provenance)。只有带文档标记的报告、单元格片段的查询步骤、
|
||||
* 查询快照来自上传的表格时才有,且恒为 true;其余一个键都不加。前端只在 seg.cite?.kind === 'cell' 且它为 true 时请求
|
||||
*/
|
||||
provenance?: boolean
|
||||
// ---- entity:表或字段(三期)----
|
||||
/** entity:table / column */
|
||||
kind?: string
|
||||
@@ -2237,3 +2247,227 @@ export interface EvidenceSegmentDetail {
|
||||
/** 可疑实体:最接近的已知名字(最多 3 个),给「是不是想写…」 */
|
||||
closest?: (string | { alias?: string; name?: string; table?: string; kind?: string })[]
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// 期 4:推断的来源(GET /runs/{id}/evidence/segments/{sid}/provenance?report=…)
|
||||
// 契约照 backend/app/data/provenance_types.py(asdict 之后的 JSON,P4-SPEC 2.8.1)。这是期 4 新加的接口,
|
||||
// 答复里每个键都在(值可以是 null),所以这里不写成可选;老服务端没有这个接口时回 404 且没有 code,三节都不画。
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/** inferred 给出格子;table_only 只给表级来历(version 非空);none 不画,或只画一句原因 / 红色提示 */
|
||||
export type ProvenanceStatus = 'inferred' | 'table_only' | 'none'
|
||||
|
||||
/** 不下钻的原因(provenance_types.ReasonCode)。界面不按它拼字,原样显示服务端给的 reason.text */
|
||||
export type ProvenanceReasonCode =
|
||||
| 'legacy_doc' | 'not_cell' | 'not_sealed' | 'not_upload' | 'simple_upload' | 'manifest_unreadable'
|
||||
| 'chain_mismatch' | 'expression' | 'alias' | 'multi_table' | 'unparsed' | 'no_pk' | 'pk_missing' | 'masked'
|
||||
| 'null_value' | 'null_pk' | 'snapshot_gone' | 'db_tampered' | 'recheck_missing' | 'recheck_multiple'
|
||||
| 'recheck_mismatch' | 'no_lineage'
|
||||
|
||||
/** 标红的提示:出现时 reason.code 与它相同,界面只画提示 */
|
||||
export type ProvenanceAlertCode = 'db_tampered' | 'chain_mismatch' | 'manifest_unreadable'
|
||||
|
||||
/** 附带的格:日期表头格、行标签、分段标题、列表头、合计标签 */
|
||||
export type ProvenanceFromRole = 'axis_header' | 'row_label' | 'section_title' | 'col_header' | 'total_label'
|
||||
|
||||
/** 本期结论(照搬清单里 CheckResult.status) */
|
||||
export type ProvenancePartStatus = 'passed' | 'mismatch' | 'unverifiable' | 'info'
|
||||
/** 这一行的关系核对结果(只给关系核对) */
|
||||
export type ProvenanceRowStatus = 'passed' | 'mismatch' | 'unverifiable'
|
||||
/** 这一格的合计核对结果(只给 K、G、T) */
|
||||
export type ProvenanceCellStatus = 'ok' | 'unverifiable' | 'unknown' | 'not_formula'
|
||||
|
||||
export interface ProvenanceReportRef {
|
||||
node_id: string
|
||||
doc_artifact: string
|
||||
}
|
||||
|
||||
/** 片段本身是单元格引用就给(与 status 无关),不是时为 null */
|
||||
export interface ProvenanceCellRef {
|
||||
/** 目录里的查询编号(Q3) */
|
||||
alias: string
|
||||
row: number
|
||||
column: string
|
||||
/** 查询快照的工件 id */
|
||||
artifact: string | null
|
||||
}
|
||||
|
||||
export interface ProvenanceReason {
|
||||
code: ProvenanceReasonCode
|
||||
/** 细分,给测试和日志用,界面不显示 */
|
||||
detail: string
|
||||
/** 界面原样显示 */
|
||||
text: string
|
||||
}
|
||||
|
||||
export interface ProvenanceAlert {
|
||||
code: ProvenanceAlertCode
|
||||
text: string
|
||||
}
|
||||
|
||||
export interface ProvenanceTableRef {
|
||||
name: string
|
||||
kind: 'data' | 'reported_total'
|
||||
}
|
||||
|
||||
export interface ProvenancePeriod {
|
||||
start: string
|
||||
end: string
|
||||
source: 'cells' | 'human'
|
||||
/** 解析出统计期的格子,带工作表名(「客流汇总!B2」) */
|
||||
cells: string[]
|
||||
/** 人工录入时的署名(未认证) */
|
||||
signed_by: string | null
|
||||
}
|
||||
|
||||
/** 一条接受理由(导入清单里的,内容寻址,是当时的事实) */
|
||||
export interface ProvenanceAcceptance {
|
||||
check_id: string
|
||||
title: string | null
|
||||
kind: 'override' | 'waiver'
|
||||
reason: string
|
||||
signed_by: string | null
|
||||
at: string | null
|
||||
}
|
||||
|
||||
/** 导入记录上的当前状态(清除原件、作废接受) */
|
||||
export interface ProvenanceStateNote {
|
||||
at: string | null
|
||||
signed_by: string | null
|
||||
reason: string | null
|
||||
}
|
||||
|
||||
/** 数据版本里的一期(按统计期升序)。raw_state、purged、revoked 是当前状态,界面标「当前状态」 */
|
||||
export interface ProvenancePart {
|
||||
import_id: string
|
||||
seq: number
|
||||
/** 导入清单的工件 id:「查看导入清单」按它经 /api/artifacts/{id} 取 */
|
||||
manifest: string
|
||||
/** 没有统计期时 null(「统计期未记录」) */
|
||||
period: ProvenancePeriod | null
|
||||
file_name: string | null
|
||||
raw_sha256: string | null
|
||||
/** 区域(「客流汇总!B4:AG30」,多张表用「、」连);取不到为 null */
|
||||
region: string | null
|
||||
/** 排除的行数之和;期 3 之前的回执没有记录时为 null */
|
||||
excluded_rows: number | null
|
||||
recipe_sha256: string | null
|
||||
/** 配方第几版(当前状态);取不到为 null */
|
||||
recipe_seq: number | null
|
||||
/** 导入时间(清单的 created_at) */
|
||||
committed_at: string | null
|
||||
/** 署名(未认证) */
|
||||
signed_by: string | null
|
||||
acceptances: ProvenanceAcceptance[]
|
||||
raw_state: 'kept' | 'purged' | 'absent' | null
|
||||
purged: ProvenanceStateNote | null
|
||||
revoked: ProvenanceStateNote | null
|
||||
/** 这一格所在的一期:inferred 时恰好一项为 true */
|
||||
has_row: boolean
|
||||
}
|
||||
|
||||
/** 数据版本(表级来历),来自封存链上的清单,不加「推断」标注 */
|
||||
export interface ProvenanceVersion {
|
||||
source: string
|
||||
snapshot_id: string
|
||||
mode: 'replace' | 'accumulate'
|
||||
union: boolean
|
||||
tables: ProvenanceTableRef[]
|
||||
/** false:数据源设有遮罩,不出「查看导入清单」 */
|
||||
manifest_view: boolean
|
||||
parts: ProvenancePart[]
|
||||
}
|
||||
|
||||
export interface ProvenanceFromCell {
|
||||
role: ProvenanceFromRole
|
||||
/** 这一格给出的是哪一列的值;宽表指标列的行标签(指标名取自哪一格)为 null */
|
||||
column: string | null
|
||||
sheet: string
|
||||
/** 不带工作表名的坐标(「G4」) */
|
||||
cell: string
|
||||
/** 格子原文;拿不到为 null */
|
||||
text: string | null
|
||||
/** 只给分段标题:配方中的定位文字 */
|
||||
locate_title: string | null
|
||||
}
|
||||
|
||||
export interface ProvenanceYear {
|
||||
source: 'period' | 'human'
|
||||
cells: string[]
|
||||
signed_by: string | null
|
||||
/** 这一块的日期表头有的是日期格、有的是文本,无法确定这一格属于哪一种 */
|
||||
mixed: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* 这一行标签列(交叉表的维度、合计表的合计项)的原文与数据库里的规范写法不同时给出。被引用的是值列也给:
|
||||
* 合计表那一格是「18-22 时合计」→「18-22时合计」,全角标签那一期的值格是「8-9」→「8-9」
|
||||
*/
|
||||
export interface ProvenanceCanonical {
|
||||
raw: string
|
||||
canonical: string
|
||||
}
|
||||
|
||||
export interface ProvenanceRecheck {
|
||||
/** 占位符一律是 ?,按主键的顺序与 params 一一对应 */
|
||||
sql: string
|
||||
params: unknown[]
|
||||
ok: boolean
|
||||
}
|
||||
|
||||
/** 推断出的格子(只在 inferred 时有) */
|
||||
export interface ProvenanceCellSource {
|
||||
table: string
|
||||
column: string
|
||||
column_role: 'axis' | 'dim' | 'derive' | 'const' | 'measure' | 'value' | 'text' | (string & {})
|
||||
kind: 'data' | 'reported_total'
|
||||
pk: Record<string, unknown>
|
||||
/** 快照库的 rowid(累积时是合并后的行号) */
|
||||
rowid: number
|
||||
part_seq: number
|
||||
part_rowid: number
|
||||
sheet: string
|
||||
/** 不带工作表名的坐标(「G5」) */
|
||||
cell: string
|
||||
header: string | null
|
||||
unit: string | null
|
||||
from: ProvenanceFromCell[]
|
||||
year: ProvenanceYear | null
|
||||
canonical: ProvenanceCanonical | null
|
||||
raw_purged: boolean
|
||||
/** 列表这一块按合并单元格的左上格填充 */
|
||||
merged_fill: boolean
|
||||
recheck: ProvenanceRecheck
|
||||
}
|
||||
|
||||
/** 相关核对的一条(只在 inferred 时有) */
|
||||
export interface ProvenanceCheck {
|
||||
id: string
|
||||
kind: string
|
||||
title: string
|
||||
part_status: ProvenancePartStatus
|
||||
row_status: ProvenanceRowStatus | null
|
||||
cell_status: ProvenanceCellStatus | null
|
||||
acceptance: ProvenanceAcceptance | null
|
||||
/** K 无法核对时的原因摘要(可能带数字,系统呈现) */
|
||||
detail: string | null
|
||||
}
|
||||
|
||||
/** GET /runs/{id}/evidence/segments/{sid}/provenance:推断的来源 */
|
||||
export interface EvidenceProvenance {
|
||||
schema: 'agentlab.provenance/1' | (string & {})
|
||||
report: ProvenanceReportRef
|
||||
segment: string
|
||||
cell: ProvenanceCellRef | null
|
||||
status: ProvenanceStatus
|
||||
/** status 不是 inferred 时必有 */
|
||||
reason: ProvenanceReason | null
|
||||
/** 标红;有它时不再画 reason.text */
|
||||
alert: ProvenanceAlert | null
|
||||
/** 运行已封存且封存核对通过;false 时数据版本一节加一句未封存 */
|
||||
sealed: boolean
|
||||
/** status 为 none 时为 null */
|
||||
version: ProvenanceVersion | null
|
||||
cell_source: ProvenanceCellSource | null
|
||||
checks: ProvenanceCheck[]
|
||||
}
|
||||
|
||||
@@ -85,7 +85,7 @@ const ORDER = [
|
||||
// 拆开的两项看 SPLIT 里各份的 est。2026-09-29 八道并行、关浏览器不干等之后的实测
|
||||
const ESTIMATE = {
|
||||
stream: 173, chat: 160, studio: 149, manage: 145, runs: 112, publish: 92, shell: 87,
|
||||
evidence: 85, versions: 60, 'ui-kit': 70, ui: 61, guards: 8, 'canvas-layout': 2, copy: 2, tokens: 1, trace: 1, decode: 1,
|
||||
evidence: 91, versions: 60, 'ui-kit': 70, ui: 61, guards: 8, 'canvas-layout': 2, copy: 2, tokens: 1, trace: 1, decode: 1,
|
||||
}
|
||||
// 最慢的两项按段拆成几份。用的是它们自己的段过滤:最后一份用 *_SKIP 兜住「其余全部」,以后新加的段
|
||||
// 不用改这里也跑得到(拆出去的那几份要是一段都没对上,那一份会报「一项都没跑」)。
|
||||
|
||||
+462
-5
@@ -37,6 +37,11 @@ const fxj = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-
|
||||
// 文档由 compose_doc 真跑、证据链由 api/evidence.py 的 _chain 真跑;判定和表名实体步骤的 fields 照前后端接口约定写
|
||||
const fxv = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-verdict.json`, 'utf8'))
|
||||
const JUDGE_SETS = [fxj.formal, fxj.explore, fxj.plain, fxv.formal, fxv.explore]
|
||||
// Excel 导入期 4:推断的来源。文档由 compose_doc 真跑(带文档标记),片段答复由 _chain 真跑,推断来源的答复按契约
|
||||
// (backend/app/data/provenance_types.py)拼;cases 写明每个场景是哪个片段、片段文字、覆盖 P4-SPEC 4.4 的哪几条
|
||||
const fxp = JSON.parse(readFileSync(`${root}frontend/src/run/__tests__/evidence-provenance.json`, 'utf8'))
|
||||
/** 推断来源夹具里各期导入清单的工件 id:夹具只收了两份清单,其余的按「工件不存在」答,不落到真服务端 */
|
||||
const FXP_MANIFESTS = new Set(Object.values(fxp.provenance).flatMap((a) => a.version?.parts ?? []).map((p) => p.manifest))
|
||||
/** 夹具里某段文字是哪个片段:检查按文字认片段,重新生成夹具时编号变了也不用改 */
|
||||
const segOf = (doc, text) => doc.blocks.flatMap((b) => b.units.flatMap((u) => u.segments)).find((s) => s.text === text)?.id
|
||||
|
||||
@@ -75,12 +80,18 @@ function judgeReply(set, body) {
|
||||
/**
|
||||
* 伪造证据相关的 GET;片段请求记账(缓存:同一个片段点两次只取一次)。四期:按需裁判的 POST 也在这里认
|
||||
* (记进 hits 的 judge:<运行>:<句子>),别的非 GET 一律拦掉。judge 给了就用它答(改坏验证、触顶、出错);
|
||||
* segPatch 给了就改片段接口的答复(比如 on_demand 说运行还没封存)
|
||||
* segPatch 给了就改片段接口的答复(比如 on_demand 说运行还没封存)。
|
||||
*
|
||||
* Excel 导入期 4:推断来源接口(…/segments/<id>/provenance)在片段正则之前单独认,不分运行、不分夹具,一律记成
|
||||
* prov:<运行>:<片段>——片段正则以 $ 结尾认不出它,落到 r.continue() 会得到一个没有机读码的 404,界面按「老服务端」
|
||||
* 三节都不画,只看 DOM 的断言不论前端发没发请求都会过。推断来源夹具的运行按 provenance[<片段>] 作答,其余一律回
|
||||
* 可识别的哨兵(418,harness_unexpected_provenance),界面会画出「未能获取推断的来源」。prov 给了就由它改答复
|
||||
* (prov(运行, 片段, 默认答复) → {status?, json} 或它的 Promise:延迟、出错、老服务端)
|
||||
*/
|
||||
async function routed(page, { judge = null, segPatch = null } = {}) {
|
||||
async function routed(page, { judge = null, segPatch = null, prov = null } = {}) {
|
||||
const hits = []
|
||||
const judged = new Set()
|
||||
await page.route((u) => new URL(u).pathname.startsWith('/api/'), (r) => {
|
||||
await page.route((u) => new URL(u).pathname.startsWith('/api/'), async (r) => {
|
||||
const req = r.request()
|
||||
const url = new URL(req.url())
|
||||
const jset = JUDGE_SETS.find((x) => url.pathname.includes(`/runs/${x.run_id}/`) || url.pathname.endsWith(`/runs/${x.run_id}`))
|
||||
@@ -92,6 +103,33 @@ async function routed(page, { judge = null, segPatch = null } = {}) {
|
||||
return r.fulfill({ status: reply.status ?? 200, json: reply.json })
|
||||
}
|
||||
if (req.method() !== 'GET') return r.abort()
|
||||
const pm = url.pathname.match(/\/api\/runs\/([^/]+)\/evidence\/segments\/([^/]+)\/provenance$/)
|
||||
if (pm) {
|
||||
const [run, sid] = [decodeURIComponent(pm[1]), decodeURIComponent(pm[2])]
|
||||
hits.push(`prov:${run}:${sid}`)
|
||||
const own = run === fxp.run_id && fxp.provenance[sid]
|
||||
let reply = own ? { json: structuredClone(own) } : fxp.errors.unexpected
|
||||
if (prov) reply = await prov(run, sid, reply)
|
||||
return r.fulfill({ status: reply.status ?? 200, json: reply.json })
|
||||
}
|
||||
if (url.pathname.includes(`/runs/${fxp.run_id}/`)) {
|
||||
const seg = url.pathname.match(/\/evidence\/segments\/([^/]+)$/)
|
||||
if (seg) {
|
||||
const id = decodeURIComponent(seg[1])
|
||||
hits.push(`p:${id}`)
|
||||
const body = fxp.segments[id]
|
||||
return body ? r.fulfill({ json: structuredClone(body) })
|
||||
: r.fulfill({ status: 404, json: { detail: '报告中没有这个片段', code: 'evidence_segment_not_found' } })
|
||||
}
|
||||
if (/\/evidence$/.test(url.pathname)) return r.fulfill({ json: fxp.graph })
|
||||
}
|
||||
// 「查看导入清单」按清单的工件 id 取(内容寻址)
|
||||
const art = url.pathname.match(/^\/api\/artifacts\/([^/]+)$/)?.[1]
|
||||
if (art && fxp.artifacts[art]) {
|
||||
hits.push(`artifact:P:${art}`)
|
||||
return r.fulfill({ json: fxp.artifacts[art] })
|
||||
}
|
||||
if (art && FXP_MANIFESTS.has(art)) return r.fulfill({ status: 404, json: { detail: '工件不存在' } })
|
||||
// 四期的报告工件:问数据页的答案(Output → EvidenceField)按工件 id 取文档
|
||||
const jart = JUDGE_SETS.find((x) => url.pathname === `/api/artifacts/${x.doc_artifact}`)
|
||||
if (jart) return r.fulfill({ json: { id: jart.doc_artifact, content: jart.doc } })
|
||||
@@ -144,14 +182,14 @@ async function routed(page, { judge = null, segPatch = null } = {}) {
|
||||
return hits
|
||||
}
|
||||
|
||||
async function open(url, { w = 1280, h = 1000, reduced = false, patchQuery = null, judge = null, segPatch = null } = {}) {
|
||||
async function open(url, { w = 1280, h = 1000, reduced = false, patchQuery = null, judge = null, segPatch = null, prov = null } = {}) {
|
||||
const ctx = await browser.newContext({ viewport: { width: w, height: h }, ...(reduced ? { reducedMotion: 'reduce' } : {}) })
|
||||
const page = await ctx.newPage()
|
||||
// 找不到元素时 8 秒就报,不等默认的 30 秒:改坏验证时一节里十几处都找不到
|
||||
page.setDefaultTimeout(8000)
|
||||
const errors = []
|
||||
page.on('pageerror', (e) => errors.push(e.message))
|
||||
const hits = await routed(page, { judge, segPatch })
|
||||
const hits = await routed(page, { judge, segPatch, prov })
|
||||
// 预览页直接 import 第二期夹具(vite 把 JSON 当模块给):要一份改过的文档时把那个模块换掉
|
||||
if (patchQuery) {
|
||||
await page.route((u) => new URL(u).pathname.endsWith('/run/__tests__/evidence-query.json'), (r) =>
|
||||
@@ -742,6 +780,12 @@ await section('query', '查询步骤:SQL 带复制、被引用的行和格高
|
||||
return !!note && !!seal && !!(seal.compareDocumentPosition(note) & Node.DOCUMENT_POSITION_FOLLOWING)
|
||||
}))
|
||||
check('查询快照复验通过时不多说一句', await panel(page).locator('[data-ev-integrity]').count() === 0)
|
||||
// Excel 导入期 4 之前的报告:查询步骤不带推断来源的提示,前端一个请求都不多发、三节都不画(P4-SPEC 4.4 第 14 条)。
|
||||
// 断言请求记账,不只看 DOM:老服务端那种 404 也会让三节不画
|
||||
check('期 4 之前的报告(查询步骤没有提示):不请求推断的来源,面板里没有数据版本、推断的来源',
|
||||
!harness.hits.some((h) => h.startsWith('prov:'))
|
||||
&& await panel(page).locator('[data-ev-provenance], [data-ev-prov-version], [data-ev-prov-checks]').count() === 0,
|
||||
harness.hits.filter((h) => h.startsWith('prov:')).join(','))
|
||||
const aria = await page.locator(`${Q} [data-seg="${qseg('1,288')}"]`).getAttribute('aria-label')
|
||||
check('单元格的 aria-label 写明查询、第几行、哪一列', aria === '1,288,有出处:查询 Q3 · 第 6 行 · amount', aria)
|
||||
|
||||
@@ -2494,6 +2538,419 @@ await section('jr-banner', '裁判拆档:出具横幅的结论句计数带上
|
||||
}
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Excel 导入期 4:推断的来源(夹具 evidence-provenance.json,P4-SPEC 4.4)
|
||||
// ---------------------------------------------------------------------------
|
||||
const PV = '#evidence-provenance'
|
||||
const PVN = '#evidence-provenance-narrow'
|
||||
/** 场景的片段 id:按夹具 cases 里记的片段文字认,重新生成夹具后编号挪了也不用改 */
|
||||
const pseg = (name) => segOf(fxp.doc, fxp.cases[name].text)
|
||||
/** 这个场景的片段请求过几次推断来源 */
|
||||
const provHits = (hits, name) => hits.filter((h) => h === `prov:${fxp.run_id}:${pseg(name)}`).length
|
||||
/** 点开一个场景,等推断的来源那一节定下来(不再是「正在推断」);不该有这一节的场景改等查询步骤、指标步骤 */
|
||||
async function openP(p, name, { box = PV, wait = '[data-ev-provenance]:not([data-ev-provenance="loading"])' } = {}) {
|
||||
await p.locator(`${box} [data-seg="${pseg(name)}"]`).click()
|
||||
await p.waitForSelector(`[data-evidence-panel] ${wait}`, { timeout: 4000 }).catch(() => {})
|
||||
await p.waitForTimeout(120)
|
||||
}
|
||||
const pv = (p, sel = '') => p.locator(`[data-evidence-panel] ${sel}`)
|
||||
/** 元素的文字(空白合成一个空格);没有这个元素时是空串 */
|
||||
const textOf = async (loc) => ((await loc.count()) ? (await loc.first().innerText()).replace(/\s+/g, ' ').trim() : '')
|
||||
const fromOf = (p) => p.locator('[data-evidence-panel] [data-ev-provenance] [data-ev-prov-from] li').allInnerTexts()
|
||||
const partsOf = (p) => p.locator('[data-evidence-panel] [data-ev-prov-version] [data-ev-prov-part]')
|
||||
.evaluateAll((els) => els.map((e) => `${e.getAttribute('data-ev-prov-part')}${e.hasAttribute('data-ev-prov-has-row') ? '*' : ''}`))
|
||||
/** 只给表级来历的场景:table_only、一句原因、没有格子和相关核对、数据版本照样有 */
|
||||
async function tableOnly(p, name, want, what) {
|
||||
await openP(p, name)
|
||||
const reason = await textOf(pv(p, '[data-ev-provenance="table_only"] [data-ev-prov-reason]'))
|
||||
check(what, (typeof want === 'string' ? reason === want : want.test(reason))
|
||||
&& await pv(p, '[data-ev-prov-cell], [data-ev-prov-checks], [data-ev-prov-alert]').count() === 0
|
||||
&& await pv(p, '[data-ev-prov-version]').count() === 1, reason)
|
||||
}
|
||||
const noSections = async (p) => (await pv(p, '[data-ev-provenance], [data-ev-prov-version], [data-ev-prov-checks]').count()) === 0
|
||||
const activeHas = (p, attr) => p.evaluate((a) => document.activeElement?.hasAttribute(a) ?? false, attr)
|
||||
|
||||
await section('provenance', '期 4:推断的来源——数据版本、格子与逐条、相关核对,只在上传表格的单元格上请求', async () => {
|
||||
const ids = Object.fromEntries(Object.keys(fxp.cases).map((k) => [k, pseg(k)]))
|
||||
check('夹具的场景都能按片段文字认出来(和 cases 记的编号一致)', Object.entries(fxp.cases).every(([k, c]) => ids[k] === c.segment),
|
||||
JSON.stringify(ids))
|
||||
// 前面各节点开的都是期 4 之前的报告、没有提示的查询步骤:一个推断来源请求都不该有(4.4 第 14 条的全局版)
|
||||
const stray = harness.hits.filter((h) => h.startsWith('prov:'))
|
||||
check('其他夹具(期 4 之前的报告)点开过的片段一个推断来源请求都没发', stray.length === 0, stray.join(','))
|
||||
|
||||
// ---- 1. 宽表的一格:格子、逐条、相关核对、位置、标题
|
||||
await openP(page, 'wide')
|
||||
const sec = pv(page, '[data-ev-provenance="inferred"]')
|
||||
check('满足条件:有「推断的来源」一节(inferred)', await sec.count() === 1, String(await pv(page, '[data-ev-provenance]').getAttribute('data-ev-provenance').catch(() => null)))
|
||||
check('……标题「推断的来源」,旁边标「推断,不属于已封存的证据」', await textOf(sec.locator('h4')) === '推断的来源'
|
||||
&& await textOf(sec.locator('[data-ev-prov-badge]')) === '推断,不属于已封存的证据')
|
||||
const cell = await textOf(sec.locator('[data-ev-prov-cell]'))
|
||||
check('……第一行醒目地写格子:工作表「客流汇总」G5', cell === '工作表「客流汇总」G5', cell)
|
||||
const items = await fromOf(page)
|
||||
check('……逐条:日期取自表头格 G4 → 年份取自统计期 → 行标签 B5(指标名)', items.join('|')
|
||||
=== '日期取自表头格 G4|年份取自统计期(客流汇总!B2)|行标签 B5「全日客流(人次)」', items.join('|'))
|
||||
check('相关核对:R1「这一行成立」', (await textOf(pv(page, '[data-ev-prov-checks] [data-ev-prov-check="R1"]'))).includes('这一行成立'))
|
||||
// 规格 4.1 的散文写「口径说明:…」;R2 的标题本身是「口径不同:…」,再套一层冒号读不顺,所以和别的核对同一种
|
||||
// 写法「R2 标题 — 口径说明」(口径说明是 4.2 partStatus.info 的原文)。改写法要连同 CheckRow 一起改
|
||||
check('……口径不同(R2)只写口径说明,不判行', (await textOf(pv(page, '[data-ev-prov-check="R2"]'))).endsWith('— 口径说明')
|
||||
&& await pv(page, '[data-ev-prov-check="R2"][data-ev-prov-row]').count() === 0)
|
||||
const order = await page.evaluate(() => {
|
||||
const p = document.querySelector('[data-evidence-panel]')
|
||||
const q = p?.querySelector('[data-ev-query]')
|
||||
const seq = ['[data-ev-prov-version]', '[data-ev-provenance]', '[data-ev-prov-checks]', '[data-ev-judge]', '[data-ev-seal]']
|
||||
.map((s) => p?.querySelector(s)).filter(Boolean)
|
||||
if (!q || seq.length < 4) return `缺:${seq.length}`
|
||||
let prev = q
|
||||
for (const el of seq) {
|
||||
if (!(prev.compareDocumentPosition(el) & Node.DOCUMENT_POSITION_FOLLOWING)) return `顺序不对:${el.outerHTML.slice(0, 60)}`
|
||||
prev = el
|
||||
}
|
||||
return 'ok'
|
||||
})
|
||||
check('三节在查询步骤之后、结论句裁判和封存之前:数据版本 → 推断的来源 → 相关核对', order === 'ok', order)
|
||||
const heads = await Promise.all(['数据版本', '推断的来源', '相关核对']
|
||||
.map((name) => panel(page).getByRole('heading', { name, exact: true }).evaluateAll((els) => els.map((e) => e.tagName).join(','))))
|
||||
check('三节的标题都是真正的标题(getByRole heading,h4)', heads.every((t) => t === 'H4'), heads.join(' | '))
|
||||
check('推断的来源用虚线左边框,和其余几节区分', await sec.evaluate((el) => getComputedStyle(el).borderLeftStyle) === 'dashed')
|
||||
check('相关核对也挂「推断」标签;数据版本来自封存链上的清单,不挂', await pv(page, '[data-ev-prov-checks] [data-ev-prov-badge]').count() === 1
|
||||
&& await pv(page, '[data-ev-prov-version] [data-ev-prov-badge]').count() === 0)
|
||||
check('封存那一行照旧只按片段接口的链算:「已封存 · 核对一致」', (await textOf(pv(page, '[data-ev-seal]'))).includes('已封存 · 核对一致'))
|
||||
const integrityBase = await pv(page, '[data-ev-integrity]').count()
|
||||
|
||||
const ver = pv(page, '[data-ev-prov-version]')
|
||||
const vtext = await textOf(ver)
|
||||
const p1 = fxp.provenance[ids.wide].version.parts[0]
|
||||
check('数据版本:首行「数据源「flow_demo」· 每期替换」(每期替换不写期数)', await textOf(ver.locator('[data-ev-prov-head]')) === '数据源「flow_demo」· 每期替换',
|
||||
vtext)
|
||||
check('……每期一行:第几次导入 · 统计期;文件名、sha256 前 12 位、区域、原件状态、配方第几版',
|
||||
vtext.includes('第 1 次导入 · 统计期 2026-08-01 至 2026-08-31') && vtext.includes(`文件「${p1.file_name}」· sha256 ${p1.raw_sha256.slice(0, 12)}`)
|
||||
&& !vtext.includes(p1.raw_sha256.slice(0, 13)) && vtext.includes('区域 客流汇总!B4:AG30') && vtext.includes('原件已保存')
|
||||
&& vtext.includes('配方第 1 版') && vtext.includes('涉及的表:日客流'), vtext)
|
||||
check('……这一格所在的一期有标记;已封存的运行不写「尚未封存」', (await partsOf(page)).join(',') === '1*'
|
||||
&& vtext.includes('这一格所在的一期') && await ver.locator('[data-ev-prov-unsealed]').count() === 0)
|
||||
|
||||
const det = sec.locator('[data-ev-prov-details]')
|
||||
check('末尾一行小字「已按主键回查数据文件,值一致」,回查 SQL 收在「技术细节」里(默认收起)',
|
||||
await textOf(sec.locator('[data-ev-prov-recheck]')) === '已按主键回查数据文件,值一致'
|
||||
&& await det.getAttribute('aria-expanded') === 'false' && await sec.locator('[data-ev-prov-tech]').isHidden()
|
||||
&& await det.getAttribute('aria-controls') === await sec.locator('[data-ev-prov-tech]').getAttribute('id'))
|
||||
await det.focus()
|
||||
await page.keyboard.press('Enter')
|
||||
const tech = await textOf(sec.locator('[data-ev-prov-tech]'))
|
||||
check('……「技术细节」是按钮,回车展开:回查 SQL 和参数,带复制', await det.getAttribute('aria-expanded') === 'true'
|
||||
&& await sec.locator('[data-ev-prov-tech]').isVisible()
|
||||
&& tech.includes('SELECT rowid, "全日客流" FROM "日客流" WHERE "日期" = ?') && tech.includes('2026-08-05')
|
||||
&& await sec.locator('[data-ev-prov-tech] button', { hasText: '复制 SQL' }).count() === 1, tech)
|
||||
|
||||
// ---- 16、28:键盘打开导入清单,对话框是结构化的清单渲染,Esc 只关对话框
|
||||
await pv(page, '[data-ev-query] [data-ev-query-head]').focus()
|
||||
let reached = false
|
||||
for (let i = 0; i < 30 && !reached; i++) {
|
||||
await page.keyboard.press('Tab')
|
||||
reached = await activeHas(page, 'data-ev-prov-manifest')
|
||||
}
|
||||
check('键盘:从查询步骤 Tab 走得到「查看导入清单」', reached)
|
||||
const arts = harness.hits.filter((h) => h.startsWith('artifact:P:')).length
|
||||
await page.keyboard.press('Enter')
|
||||
await page.waitForSelector('[role="dialog"] [data-manifest-block]', { timeout: 4000 }).catch(() => {})
|
||||
const dlg = page.locator('[role="dialog"]')
|
||||
const dtext = await dlg.innerText().catch(() => '')
|
||||
check('回车打开导入清单对话框,按清单的工件 id 经工件接口取', await dlg.count() === 1
|
||||
&& harness.hits.filter((h) => h.startsWith('artifact:P:')).length === arts + 1 && harness.hits.includes(`artifact:P:${p1.manifest}`)
|
||||
&& dtext.includes('第 1 次导入的导入清单'), dtext.slice(0, 80))
|
||||
check('……正文是版本页的清单渲染(结构化区块),不是整份 JSON(展开的文字里没有键名 "acceptances")',
|
||||
await dlg.locator('[data-manifest-block]').count() >= 5 && !dtext.includes('"acceptances"') && dtext.includes(p1.file_name)
|
||||
&& await dlg.locator('[data-manifest-raw]').evaluate((el) => !el.open).catch(() => false), String(await dlg.locator('[data-manifest-block]').count()))
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await page.waitForTimeout(250)
|
||||
await page.screenshot({ path: `${SHOTS}/evidence-p4-manifest-${theme}.png` })
|
||||
}
|
||||
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
await page.keyboard.press('Escape')
|
||||
await page.waitForTimeout(200)
|
||||
check('Esc 只关清单对话框:证据面板还开着,焦点回到「查看导入清单」', await dlg.count() === 0 && await panel(page).count() === 1
|
||||
&& await activeHas(page, 'data-ev-prov-manifest'))
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await page.waitForTimeout(250)
|
||||
await panel(page).screenshot({ path: `${SHOTS}/evidence-p4-inferred-${theme}.png` })
|
||||
}
|
||||
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
await page.keyboard.press('Escape')
|
||||
await page.waitForTimeout(200)
|
||||
|
||||
// ---- 18:缓存只在面板这次打开期间有效
|
||||
const c0 = provHits(harness.hits, 'wide')
|
||||
await openP(page, 'wide')
|
||||
await openP(page, 'long')
|
||||
await openP(page, 'wide')
|
||||
check('同一次打开里同一片段点两次,推断的来源只取一次', provHits(harness.hits, 'wide') === c0 + 1 && provHits(harness.hits, 'long') >= 1,
|
||||
`${c0} → ${provHits(harness.hits, 'wide')}`)
|
||||
await page.keyboard.press('Escape')
|
||||
await page.waitForTimeout(200)
|
||||
check('……面板关掉了', await panel(page).count() === 0)
|
||||
await openP(page, 'wide')
|
||||
check('关闭面板再打开、再点同一片段:重新取(回答里有当前状态)', provHits(harness.hits, 'wide') === c0 + 2, String(provHits(harness.hits, 'wide')))
|
||||
|
||||
// ---- 2、3:长表的分段标题、全角标签的规范写法、合计表
|
||||
await openP(page, 'long')
|
||||
const longItems = await fromOf(page)
|
||||
check('长表一格:分段标题 B9(按配方中的标题「…」定位),不说它是格子原文;行标签 B16「13-14」',
|
||||
longItems.includes('分段标题 B9(按配方中的标题「日间时段客流(人次)」定位)') && longItems.includes('行标签 B16「13-14」'), longItems.join('|'))
|
||||
await openP(page, 'fullwidth')
|
||||
const fw = await fromOf(page)
|
||||
check('全角标签那一期的维度列:原文「8-9」,按规范写法存为「8-9」', fw.includes('原文「8-9」,按规范写法存为「8-9」'), fw.join('|'))
|
||||
await openP(page, 'total')
|
||||
const tItems = await fromOf(page)
|
||||
check('合计表一格:合计标签 B28「18-22 时合计」(原文),口径「原表写明的合计:不要彼此相加…」', tItems.includes('合计标签 B28「18-22 时合计」')
|
||||
&& tItems.includes('原表写明的合计:不要彼此相加,也不要与明细相加') && tItems.includes('原文「18-22 时合计」,按规范写法存为「18-22时合计」'),
|
||||
tItems.join('|'))
|
||||
check('……相关核对里 K1「这一格一致」', (await textOf(pv(page, '[data-ev-prov-check="K1"]'))).includes('这一格一致'))
|
||||
|
||||
// ---- 4:累积的两期
|
||||
await openP(page, 'union')
|
||||
const un = fxp.provenance[ids.union]
|
||||
check('按期累积:首行写「按期累积 · 共 2 期」,两期都列出,第 2 期标「这一格所在的一期」',
|
||||
await textOf(pv(page, '[data-ev-prov-head]')) === `数据源「${un.version.source}」· 按期累积 · 共 2 期`
|
||||
&& (await partsOf(page)).join(',') === '1,2*', (await partsOf(page)).join(','))
|
||||
const ufrom = await textOf(pv(page, '[data-ev-prov-from-part]'))
|
||||
check('……格子是第 2 期(9 月)文件的坐标', await textOf(pv(page, '[data-ev-prov-cell]')) === `工作表「${un.cell_source.sheet}」${un.cell_source.cell}`
|
||||
&& ufrom.includes('第 2 次导入') && ufrom.includes(un.version.parts[1].file_name), ufrom)
|
||||
|
||||
// ---- 5–8:只给表级来历
|
||||
await tableOnly(page, 'alias', /改了名/, '改名:table_only,原因写「改了名」,没有格子,数据版本照样有')
|
||||
await tableOnly(page, 'selfjoin', /同一张表多次/, '自连接:table_only,原因写「同一张表多次」')
|
||||
await tableOnly(page, 'null', '这一格是空值,不推断来源', '空值:table_only,「这一格是空值,不推断来源」')
|
||||
await tableOnly(page, 'aggregate', /计算、聚合/, '聚合:table_only,原因写「计算、聚合」')
|
||||
check('……聚合的格子数据版本照样列出涉及的表', (await textOf(pv(page, '[data-ev-prov-tables]'))) === '涉及的表:日客流')
|
||||
|
||||
// ---- 9、10、21:标红
|
||||
const failedColor = await tokenColor(page, 'color', 'var(--st-failed)')
|
||||
const look = (code) => pv(page, `[data-ev-prov-alert="${code}"]`).evaluate((el) => {
|
||||
const cs = getComputedStyle(el)
|
||||
return { color: cs.color, border: cs.borderLeftColor, text: el.innerText }
|
||||
}).catch(() => null)
|
||||
await openP(page, 'tampered')
|
||||
const tam = await look('db_tampered')
|
||||
check('数据文件被改:标红(data-ev-prov-alert="db_tampered"),写「可能被修改过」,边框和字都是 --st-failed',
|
||||
!!tam && tam.text.includes('可能被修改过') && tam.color === failedColor && tam.border === failedColor, JSON.stringify({ tam, failedColor }))
|
||||
check('……不给格子、不写原因那一句(有 alert 时只画 alert),数据版本照样有', await pv(page, '[data-ev-prov-cell], [data-ev-prov-reason], [data-ev-prov-checks]').count() === 0
|
||||
&& await pv(page, '[data-ev-prov-version]').count() === 1 && await pv(page, '[data-ev-provenance="table_only"]').count() === 1)
|
||||
check('……标红用自己的属性:面板里 data-ev-integrity 的个数和没有 alert 时相同', await pv(page, '[data-ev-integrity]').count() === integrityBase)
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await page.waitForTimeout(250)
|
||||
await panel(page).screenshot({ path: `${SHOTS}/evidence-p4-tampered-${theme}.png` })
|
||||
}
|
||||
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
await openP(page, 'chain')
|
||||
check('登记哈希与清单不符:table_only,标红 chain_mismatch,数据版本照样有,没有原因那一句',
|
||||
await pv(page, '[data-ev-provenance="table_only"] [data-ev-prov-alert="chain_mismatch"]').count() === 1
|
||||
&& await pv(page, '[data-ev-prov-version]').count() === 1 && await pv(page, '[data-ev-prov-reason]').count() === 0)
|
||||
await openP(page, 'unreadable')
|
||||
const unr = await look('manifest_unreadable')
|
||||
check('清单无法读取:none 里只有红色的 manifest_unreadable,没有数据版本、没有原因那一句',
|
||||
await pv(page, '[data-ev-provenance="none"] [data-ev-prov-alert="manifest_unreadable"]').count() === 1 && unr?.color === failedColor
|
||||
&& await pv(page, '[data-ev-prov-version], [data-ev-prov-reason], [data-ev-prov-cell], [data-ev-prov-checks]').count() === 0, JSON.stringify(unr))
|
||||
|
||||
// ---- 11、12:当前状态
|
||||
await openP(page, 'purged')
|
||||
const pver = await textOf(pv(page, '[data-ev-prov-version]'))
|
||||
const purgedLine = await textOf(pv(page, '[data-ev-prov-purged]'))
|
||||
check('原件已清除:数据版本里写「原件已清除 · 清除时间 · 理由 · 署名(未认证)」,标「当前状态」',
|
||||
/^原件已清除 · \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \(UTC[+-]\d{2}:\d{2}\) · 理由:合成理由:原件不再保留 · 署名(未认证):录入员甲/.test(purgedLine)
|
||||
&& await pv(page, '[data-ev-prov-purged] [data-ev-prov-current]').count() === 1, purgedLine)
|
||||
check('……格子照样有,旁边注明「原件已清除:坐标来自导入时的记录,无法再对照原件」', await pv(page, '[data-ev-prov-cell]').count() === 1
|
||||
&& await textOf(pv(page, '[data-ev-prov-raw]')) === '原件已清除:坐标来自导入时的记录,无法再对照原件')
|
||||
await openP(page, 'revoked')
|
||||
const rver = await textOf(pv(page, '[data-ev-prov-version]'))
|
||||
check('作废接受:数据版本里写「这一期的接受已作废 · 作废时间 · 理由」(当前状态),接受理由照旧列出',
|
||||
/^这一期的接受已作废 · \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \(UTC[+-]\d{2}:\d{2}\) · 理由:合成理由:接受依据有误 · 署名(未认证):未填写/
|
||||
.test(await textOf(pv(page, '[data-ev-prov-revoked]')))
|
||||
&& await pv(page, '[data-ev-prov-revoked] [data-ev-prov-current]').count() === 1
|
||||
&& rver.includes('已接受:「日客流」:全日客流 = 分区甲 + 分区乙 · 合成理由:分区合计口径调整 · 署名(未认证):未填写'), rver)
|
||||
|
||||
// ---- 13:简单导入
|
||||
await openP(page, 'simple')
|
||||
check('简单导入:none 里只有一句「这份表格按表头行直接导入…」,没有数据版本',
|
||||
(await textOf(pv(page, '[data-ev-provenance="none"] [data-ev-prov-reason]'))).startsWith('这份表格按表头行直接导入')
|
||||
&& await pv(page, '[data-ev-prov-version], [data-ev-prov-alert], [data-ev-prov-checks]').count() === 0)
|
||||
|
||||
// ---- 15、20:不该请求的两种
|
||||
check('夹具自检:手工源那一格的查询步骤没有提示,指标片段链上的查询步骤误带了提示',
|
||||
fxp.segments[ids.manual].chain.every((st) => st.provenance !== true)
|
||||
&& fxp.segments[ids.metric].chain.some((st) => st.step === 'query' && st.provenance === true))
|
||||
await openP(page, 'manual', { wait: '[data-ev-query]' })
|
||||
await page.waitForTimeout(250)
|
||||
check('手工源(查询步骤不带提示):不请求推断的来源,三节都不画', provHits(harness.hits, 'manual') === 0 && await noSections(page))
|
||||
await openP(page, 'metric', { wait: '[data-ev-metric]' })
|
||||
await page.waitForTimeout(250)
|
||||
check('指标片段:链上的查询步骤带了提示也不请求(只认单元格引用)', provHits(harness.hits, 'metric') === 0 && await noSections(page)
|
||||
&& await pv(page, '[data-ev-query]').count() >= 1)
|
||||
|
||||
// ---- 22、23、25、26:回收、遮罩、未封存、本期不成立并接受
|
||||
await tableOnly(page, 'gone', /已回收或不存在/, '数据文件已回收:table_only,原因写「已回收或不存在」,不标红')
|
||||
await tableOnly(page, 'masked', '这一格涉及设置了遮罩的列,不推断来源', '遮罩:table_only,「这一格涉及设置了遮罩的列,不推断来源」')
|
||||
check('……数据源设有遮罩:数据版本里没有「查看导入清单」,改写一句去向', await pv(page, '[data-ev-prov-manifest]').count() === 0
|
||||
&& await textOf(pv(page, '[data-ev-prov-manifest-masked]')) === '数据源设置了遮罩,导入清单请到数据源卡片上的「版本」查看')
|
||||
await openP(page, 'unsealed')
|
||||
check('运行未封存:照常推断,数据版本一节写「本次运行尚未封存…」,不标红', await pv(page, '[data-ev-provenance="inferred"]').count() === 1
|
||||
&& (await textOf(pv(page, '[data-ev-prov-version] [data-ev-prov-unsealed]'))).startsWith('本次运行尚未封存或封存核对未通过')
|
||||
&& await pv(page, '[data-ev-prov-alert]').count() === 0)
|
||||
// 接受理由挂在本期结论上(接受针对的是这一期的核对结果,不是这一行),写法取 4.2 的 accepted 原文「已接受,理由:…」,
|
||||
// 不是 4.1 散文里的「已接受:理由」;行级结果单独一行,本期结论另起一行——d26_other 要同时看得到「这一行成立」和
|
||||
// 「本期不成立 · 已接受」(4.4 第 26 条)
|
||||
await openP(page, 'd26_bad')
|
||||
const bad = await textOf(pv(page, '[data-ev-prov-check="R1"]'))
|
||||
check('不成立的那一天:R1「这一行不成立」,接着写「本期不成立 · 已接受,理由:…」', bad.includes('这一行不成立')
|
||||
&& bad.includes('本期不成立 · 已接受,理由:合成理由:分区合计口径调整(署名(未认证):未填写)'), bad)
|
||||
await openP(page, 'd26_other')
|
||||
const other = await textOf(pv(page, '[data-ev-prov-check="R1"]'))
|
||||
check('同一期的另一天:「这一行成立」,本期结论照样写「本期不成立 · 已接受」', other.includes('这一行成立') && other.includes('本期不成立 · 已接受'), other)
|
||||
|
||||
// ---- 人工录入的统计期、列表
|
||||
await openP(page, 'human')
|
||||
const hItems = await fromOf(page)
|
||||
check('统计期人工录入:年份取自人工录入的统计期(署名(未认证):录入员甲)', hItems.includes('年份取自人工录入的统计期(署名(未认证):录入员甲)')
|
||||
&& (await textOf(pv(page, '[data-ev-prov-version]'))).includes('人工录入(署名(未认证):录入员甲)'), hItems.join('|'))
|
||||
await openP(page, 'list')
|
||||
check('列表:格子在列表头下面(列表头 F5),统计期未记录', await textOf(pv(page, '[data-ev-prov-cell]')) === '工作表「月报」F8'
|
||||
&& (await fromOf(page)).includes('列表头 F5') && (await textOf(pv(page, '[data-ev-prov-version]'))).includes('第 1 次导入 · 统计期未记录'))
|
||||
|
||||
// ---- 27:多于 3 期的折叠
|
||||
await openP(page, 'many')
|
||||
const more = pv(page, '[data-ev-prov-more]')
|
||||
check('四期:首行「按期累积 · 共 4 期」,只展开这一格所在的一期和最近一期,其余收进「另有 2 期」',
|
||||
(await textOf(pv(page, '[data-ev-prov-head]'))).endsWith('按期累积 · 共 4 期') && (await partsOf(page)).join(',') === '2*,4'
|
||||
&& await textOf(more) === '另有 2 期' && await more.getAttribute('aria-expanded') === 'false', (await partsOf(page)).join(','))
|
||||
await more.focus()
|
||||
await page.keyboard.press('Enter')
|
||||
check('……「另有 2 期」是带 aria-expanded 的按钮:回车展开(true),四期按统计期列全,焦点留在按钮上',
|
||||
await more.getAttribute('aria-expanded') === 'true' && (await partsOf(page)).join(',') === '1,2*,3,4' && await activeHas(page, 'data-ev-prov-more'),
|
||||
(await partsOf(page)).join(','))
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await page.waitForTimeout(250)
|
||||
await panel(page).screenshot({ path: `${SHOTS}/evidence-p4-many-${theme}.png` })
|
||||
}
|
||||
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
await page.keyboard.press('Escape')
|
||||
await page.waitForTimeout(200)
|
||||
|
||||
// ---- 17:360 宽的画布右栏
|
||||
const wrap = async () => page.evaluate((sel) => {
|
||||
const box = document.querySelector(sel)
|
||||
const p = box?.querySelector('[data-evidence-panel]')
|
||||
return { mode: p?.getAttribute('data-evidence-panel') ?? null, box: [box?.scrollWidth, box?.clientWidth], panel: p ? [p.scrollWidth, p.clientWidth] : null }
|
||||
}, PVN)
|
||||
const fits = (w) => w.mode === 'inline' && w.box[0] <= w.box[1] && !!w.panel && w.panel[0] <= w.panel[1]
|
||||
await openP(page, 'wide', { box: PVN })
|
||||
await page.locator(`${PVN} [data-ev-prov-details]`).click().catch(() => {})
|
||||
const w1 = await wrap()
|
||||
await openP(page, 'many', { box: PVN })
|
||||
await page.locator(`${PVN} [data-ev-prov-more]`).click().catch(() => {})
|
||||
const w2 = await wrap()
|
||||
await openP(page, 'tampered', { box: PVN })
|
||||
const w3 = await wrap()
|
||||
check('360 宽的画布右栏:栏内展开;宽表一格(技术细节展开)、四期(全部展开)、标红三种都不出横向滚动',
|
||||
fits(w1) && fits(w2) && fits(w3), JSON.stringify({ w1, w2, w3 }))
|
||||
if (SHOTS) {
|
||||
await openP(page, 'wide', { box: PVN })
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
await page.waitForTimeout(250)
|
||||
await page.locator(PVN).screenshot({ path: `${SHOTS}/evidence-p4-360-${theme}.png` })
|
||||
}
|
||||
await page.evaluate(() => document.documentElement.setAttribute('data-theme', 'dark'))
|
||||
}
|
||||
await page.locator(`${PVN} [data-evidence-panel] button`, { hasText: '回到正文' }).click().catch(() => {})
|
||||
await page.waitForTimeout(150)
|
||||
|
||||
// ---- 24:加载态
|
||||
const wideId = ids.wide
|
||||
const slow = await open('/ui-harness.html?evidence=1', {
|
||||
prov: async (run, sid, reply) => {
|
||||
if (sid === wideId) await new Promise((res) => setTimeout(res, 300))
|
||||
return reply
|
||||
},
|
||||
})
|
||||
await slow.page.locator(`${PV} [data-seg="${wideId}"]`).click()
|
||||
await slow.page.waitForSelector('[data-evidence-panel] [data-ev-prov-loading]', { timeout: 4000 }).catch(() => {})
|
||||
const loading = await textOf(pv(slow.page, '[data-ev-provenance="loading"] [data-ev-prov-loading]'))
|
||||
check('取的过程中:「推断的来源」一节写「正在推断来源…」,三节不先画空壳', loading === '正在推断来源…'
|
||||
&& await pv(slow.page, '[data-ev-prov-version], [data-ev-prov-checks], [data-ev-prov-cell]').count() === 0, loading)
|
||||
await slow.page.waitForSelector('[data-evidence-panel] [data-ev-provenance="inferred"]', { timeout: 4000 }).catch(() => {})
|
||||
check('……答复到了以后加载态消失,三节画出来', await pv(slow.page, '[data-ev-prov-loading]').count() === 0
|
||||
&& await pv(slow.page, '[data-ev-prov-version]').count() === 1 && await pv(slow.page, '[data-ev-prov-checks]').count() === 1)
|
||||
check('没有运行时报错(加载态)', slow.errors.length === 0, slow.errors.join(' | '))
|
||||
await slow.ctx.close()
|
||||
|
||||
// ---- 19:出错、老服务端。另守渲染矩阵本身:服务端误把数据版本带在 none · manifest_unreadable、table_only 的答复里
|
||||
// 时照样按矩阵画(前者不画数据版本,后者不画格子)——不能只靠夹具里那些字段恰好为空。
|
||||
// 清除、作废、导入的时间没记(契约里 at 可以是 null,夹具里恰好都有):略去这一项,不留孤立的「—」
|
||||
const longId = ids.long
|
||||
const untimed = (ans) => ({
|
||||
...ans,
|
||||
version: {
|
||||
...ans.version,
|
||||
parts: ans.version.parts.map((pt) => ({
|
||||
...pt,
|
||||
committed_at: null,
|
||||
purged: pt.purged && { ...pt.purged, at: null },
|
||||
revoked: pt.revoked && { ...pt.revoked, at: null },
|
||||
})),
|
||||
},
|
||||
})
|
||||
const sloppy = {
|
||||
[ids.unreadable]: { ...fxp.provenance[ids.unreadable], version: fxp.provenance[wideId].version },
|
||||
[ids.alias]: { ...fxp.provenance[ids.alias], cell_source: fxp.provenance[wideId].cell_source, checks: fxp.provenance[wideId].checks },
|
||||
[ids.purged]: untimed(fxp.provenance[ids.purged]),
|
||||
[ids.revoked]: untimed(fxp.provenance[ids.revoked]),
|
||||
}
|
||||
const broken = await open('/ui-harness.html?evidence=1', {
|
||||
prov: (run, sid, reply) => (sid === wideId ? fxp.errors.server : sid === longId ? fxp.errors.old_server
|
||||
: sloppy[sid] ? { json: sloppy[sid] } : reply),
|
||||
})
|
||||
await openP(broken.page, 'wide')
|
||||
check('推断来源接口回 500:「推断的来源」一节只写「未能获取推断的来源」', await textOf(pv(broken.page, '[data-ev-provenance="error"] [data-ev-prov-failed]'))
|
||||
=== '未能获取推断的来源' && await pv(broken.page, '[data-ev-prov-version], [data-ev-prov-checks], [data-ev-prov-cell]').count() === 0)
|
||||
check('……其余几节照常:查询步骤、封存都在', await pv(broken.page, '[data-ev-query]').count() === 1
|
||||
&& (await textOf(pv(broken.page, '[data-ev-seal]'))).includes('已封存 · 核对一致'))
|
||||
await openP(broken.page, 'long', { wait: '[data-ev-query]' })
|
||||
await broken.page.waitForTimeout(300)
|
||||
check('回 404 且没有机读码(老服务端):请求发了,三节都不画', provHits(broken.hits, 'long') === 1 && await noSections(broken.page),
|
||||
String(provHits(broken.hits, 'long')))
|
||||
await openP(broken.page, 'unreadable')
|
||||
check('渲染矩阵:清单无法读取时,答复里哪怕带着数据版本也不画,只画红色提示',
|
||||
await pv(broken.page, '[data-ev-prov-alert="manifest_unreadable"]').count() === 1 && await pv(broken.page, '[data-ev-prov-version]').count() === 0)
|
||||
await openP(broken.page, 'alias')
|
||||
check('渲染矩阵:只给表级来历时,答复里哪怕带着格子和核对也不画', await pv(broken.page, '[data-ev-prov-reason]').count() === 1
|
||||
&& await pv(broken.page, '[data-ev-prov-cell], [data-ev-prov-checks]').count() === 0)
|
||||
await openP(broken.page, 'purged')
|
||||
const purgedUntimed = await textOf(pv(broken.page, '[data-ev-prov-purged]'))
|
||||
const verUntimed = await textOf(pv(broken.page, '[data-ev-prov-version]'))
|
||||
check('清除时间、导入时间没记:写「原件已清除 · 理由:… · 署名(未认证):…」,不留孤立的「—」,也不写「导入于」',
|
||||
/^原件已清除 · 理由:合成理由:原件不再保留 · 署名(未认证):录入员甲/.test(purgedUntimed)
|
||||
&& !verUntimed.includes('—') && !verUntimed.includes('导入于')
|
||||
&& await pv(broken.page, '[data-ev-prov-purged] [data-ev-prov-current]').count() === 1, verUntimed)
|
||||
await openP(broken.page, 'revoked')
|
||||
const revokedUntimed = await textOf(pv(broken.page, '[data-ev-prov-revoked]'))
|
||||
check('作废时间没记:写「这一期的接受已作废 · 理由:… · 署名(未认证):未填写」,不留孤立的「—」',
|
||||
/^这一期的接受已作废 · 理由:合成理由:接受依据有误 · 署名(未认证):未填写/.test(revokedUntimed)
|
||||
&& !(await textOf(pv(broken.page, '[data-ev-prov-version]'))).includes('—'), revokedUntimed)
|
||||
check('没有运行时报错(出错、老服务端)', broken.errors.length === 0, broken.errors.join(' | '))
|
||||
await broken.ctx.close()
|
||||
})
|
||||
|
||||
if (SHOTS) {
|
||||
for (const theme of ['dark', 'light']) {
|
||||
await page.evaluate((t) => document.documentElement.setAttribute('data-theme', t), theme)
|
||||
|
||||
@@ -894,8 +894,8 @@ await section('版本 · 导入清单', async () => {
|
||||
const excluded = await root.locator('[data-summary-excluded]').innerText().catch(() => '')
|
||||
check('排除的行:原因、锚点、行号', excluded.includes('排除的行') && excluded.includes('只核对、不另存的合计行') && excluded.includes('18-22 时合计')
|
||||
&& excluded.includes('第 28–30 行'), excluded.replace(/\s+/g, ' '))
|
||||
check('回执摘要:区域外文字全文,注明不提供给模型', (await block('receipt')).includes('客流汇总表')
|
||||
&& (await block('receipt')).includes('区域外文字只在这里显示,不提供给模型'))
|
||||
check('回执摘要:区域外文字全文,注明不导入数据表、裁判可见', (await block('receipt')).includes('客流汇总表')
|
||||
&& (await block('receipt')).includes('区域外文字不导入数据表;核对报告时,裁判可以看到其中可见单元格的文字'))
|
||||
check('AI 用量块:没有调用模型', (await block('ai')).includes('这次导入没有调用模型'))
|
||||
check('清单校验通过时不出哈希警告', await root.locator('[data-manifest-unverified]').count() === 0)
|
||||
const raw = root.locator('[data-manifest-raw]')
|
||||
|
||||
Reference in New Issue
Block a user