Files
AgentLab/backend/app/records/store.py
T
ThearchyHelios eddeded2a4 全栈安全加固、性能提速与架构演进:连接池与沙箱止血、双路混合检索、多版本归档及前端分包治理
- 第一批(安全加固与生命周期止血):隔离测试宿主机密钥与 .env 泄漏;修复 SQLite 取消查询挂死、:memory: 连接池报错与 EngineCache 世代竞态;补齐运行与评测删除时的 Checkpointer 线程、Artifact 磁盘文件与无主附件回收;加固三套沙箱路径越界、进程组清理与 Seatbelt 数据目录隔离,封堵 SSRF IPv4-mapped IPv6 元数据绕过与响应体 OOM;修复推理模型多轮 reasoning_content、结构化输出 Token 漏计与缓存折扣计费
- 第二批(数据库/集群/问数提速与首屏分包):补齐核心表索引与 SQLite WAL 调优,列表接口按需 defer 大列,收敛启动扫描并打通跨进程 slot 槽位唤醒;优化跨源全局选表、哈希桶数字绑定、流式 CSV 导入及多方言分析算子;剥离首屏静态引入的 @xyflow/react 与全局样式(首屏主包降至 461.59 kB),补充流式 rAF 攒批、Store 精细订阅与表单无障碍绑定
- 第三批(架构演进与工程自动化闭环):记录库超挂接阈值历史版本自动增量归档至单库多表(解除 SQLite 10 库 ATTACH 硬上限);重构知识库为词面 BM25 与全集合归一化向量矩阵真·双路独立召回并支持游标分批重建;升级多方言感知 SQL 模板词法分析(E'...' / $$...$$ / q'[...]' / # / 嵌套注释)与 Decimal / bool / NaN 参数校验,消除 ledger 层对 api 路由层的反向依赖;拆分 3101 行 CaliberDrafts 页面为 7 个领域子模块,消除检查脚本本机路径硬编码并新增 GitHub Actions CI 门禁与三批回归测试套件
2026-11-13 14:51:36 +08:00

664 lines
31 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""记录表的文件层(docs/design/normal-user-apps.md N8、§3.7):一个表版本一个 SQLite 文件,只许追加。
**文件。** data/records/<table_key>.v<version>.db,里面只有一张业务表 records:系统列(records.SYSTEM_COLUMNS)、
业务列(「填什么」卡片生成),外加一列不对外的 _action_key(幂等键,唯一)。查询走连接上的同名临时视图
(view_select),视图不露 _action_key。
**写。** 只有 append 一条路:写锁里开一个短事务(进程里的 _WRITE_LOCK;多进程时调用方另持库里的锁 write_lock(),几个进程
往同一个文件里写一个接一个——SQLite 文件自己的文件锁在网络文件系统上靠不住),先按 action_key 查——同一个动作第二次来交回
第一次写下的那一行,不再写;更正是一次新的追加,带 _supersedes 指向被更正的行,原行不改。_seq 是 SQLite 的 rowid
(INTEGER PRIMARY KEY),只增不减:从不删行,也就不会复用。
**读。** 高水位(high_water)是现在最大的 _seq。一次运行在开始时记下它(engine/runner._data_versions),之后这次
运行查到的都是 _seq ≤ 高水位、在那一刻没被更正过的行:行只追加、不改,所以同一次运行查几遍都是同一份。表单不兼容地改过
之后,临时视图把比这一版老的各版本(只读挂在连接上)按 UNION ALL 接上,各按自己的高水位(view_prelude 的 older)。
这里的函数都是同步的(sqlite3 直连、关 DQS),调用方放进线程。表结构的登记、数据源在 records/tables.py。
"""
from __future__ import annotations
import json
import math
import os
import re
import sqlite3
import threading
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from app.core.config import settings
from app.data.engine import open_checked_sqlite
from app.db.base import new_id
from app.records import SYSTEM_COLUMNS
#: 文件里那张业务表的名字。看板的 SQL 写 FROM records
TABLE = "records"
#: 不对外的幂等键列:视图不露它,写 main.records 的 SQL 被只读守卫拒掉
ACTION_KEY_COLUMN = "_action_key"
#: 表单字段类型 → 列类型。多选、文件存 JSON 文字
COLUMN_TYPES = {"string": "TEXT", "number": "REAL", "integer": "INTEGER", "boolean": "INTEGER", "array": "TEXT",
"file": "TEXT"}
#: 列名最长几个字
NAME_MAX = 64
_KEY = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
#: 列名里不许出现的字符:引号(拼进 SQL 的标识符)、控制字符
_BAD_NAME = re.compile(r"[\"`\[\]\x00-\x1f\x7f]")
#: 所有写入(建表、加列、追加)按文件路径/表键分锁;保留 _WRITE_LOCK 供直接引用方兜底
_WRITE_LOCK = threading.Lock()
_LOCKS_GUARD = threading.Lock()
_PATH_LOCKS: dict[str, threading.Lock] = {}
def _path_lock(path: Path) -> threading.Lock:
key = os.path.realpath(os.fspath(path))
with _LOCKS_GUARD:
lock = _PATH_LOCKS.get(key)
if lock is None:
lock = threading.Lock()
_PATH_LOCKS[key] = lock
return lock
def write_lock(table_key: str | None = None) -> Any:
"""跨进程的写锁(app/cluster/mutex:PostgreSQL 时是咨询锁,SQLite 只有一个进程、就是本进程的锁)。异步的调用方在把写入
放进线程之前持着它(async with)。给 table_key 时按表分锁,不同记录表的写入互不阻塞。"""
from app.cluster import mutex
if table_key and _KEY.match(table_key):
return mutex.lock(f"records-store:{table_key}")
return mutex.lock("records-store")
class RecordsRefused(ValueError):
"""这一次写入不收:code 是问题编号(records.*,白话在 app/plain.py),message 直接给人看。"""
def __init__(self, code: str, message: str) -> None:
super().__init__(message)
self.code = code
self.message = message
@dataclass(frozen=True)
class Column:
"""一个业务列:名字(就是表单字段名)、中文名、类型(COLUMN_TYPES 的键)、必填、敏感(进数据源的遮罩列)。"""
name: str
label: str = ""
type: str = "string"
required: bool = False
sensitive: bool = False
def out(self) -> dict[str, Any]:
return {"name": self.name, "label": self.label or self.name, "type": self.type, "required": self.required,
"sensitive": self.sensitive}
def column_problem(name: Any) -> str | None:
"""列名合不合规:返回原因,合规返回 None。下划线开头的留给系统列。"""
if not isinstance(name, str) or not name.strip():
return "字段名不能为空"
if name != name.strip():
return f"字段名「{name}」前后不能有空格"
if len(name) > NAME_MAX:
return f"字段名「{name[:20]}…」超过 {NAME_MAX} 个字"
if name.startswith("_"):
return f"字段名「{name}」不能以下划线开头:下划线开头的留给系统列"
if _BAD_NAME.search(name):
return f"字段名「{name}」里不能有引号、方括号或控制字符"
return None
def parse_columns(raw: Any) -> list[Column]:
"""records_tables.columns(或调用方交来的列定义)→ Column。类型不认识的、名字不合规的抛 RecordsRefused。"""
out: list[Column] = []
seen: set[str] = set()
for item in raw or []:
if not isinstance(item, dict):
raise RecordsRefused("records.column_invalid", "列定义需要是对象")
name = item.get("name")
if problem := column_problem(name):
raise RecordsRefused("records.column_invalid", problem)
if name.lower() in seen:
raise RecordsRefused("records.column_invalid", f"字段名「{name}」重复了")
seen.add(name.lower())
kind = str(item.get("type") or "string")
if kind not in COLUMN_TYPES:
raise RecordsRefused("records.column_invalid", f"字段「{name}」的类型「{kind}」不能存进记录表")
out.append(Column(name=name, label=str(item.get("label") or name), type=kind,
required=bool(item.get("required")), sensitive=bool(item.get("sensitive"))))
return out
# --------------------------------------------------------------------------
# 路径
# --------------------------------------------------------------------------
def records_root() -> Path:
"""记录表文件的根目录。按调用时的 settings 现算:测试会换数据目录。"""
return settings.data_dir / "records"
def table_path(table_key: str, version: int) -> Path:
"""data/records/<table_key>.v<version>.db。键只认字母、数字、下划线、连字符,挡住路径穿越。"""
if not _KEY.match(table_key or ""):
raise ValueError("记录表的键格式不对")
if not isinstance(version, int) or version < 1:
raise ValueError("记录表的版本号需要是正整数")
return records_root() / f"{table_key}.v{version}.db"
def archive_path(table_key: str) -> Path:
"""data/records/<table_key>.archive.db:超过 SQLite 挂库上限的早期版本按物理表 rows_v<version> 增量汇总在这里。"""
if not _KEY.match(table_key or ""):
raise ValueError("记录表的键格式不对")
return records_root() / f"{table_key}.archive.db"
def archive_table_name(version: int) -> str:
if not isinstance(version, int) or version < 1:
raise ValueError("记录表的版本号需要是正整数")
return f"rows_v{version}"
def _q(name: str) -> str:
return '"' + name.replace('"', '""') + '"'
def _lit(value: str) -> str:
"""SQL 文字字面量。只用于临时视图里拼看的人的名字:视图定义不能带参数。"""
return "'" + str(value).replace("'", "''") + "'"
def _enable_wal(conn: sqlite3.Connection) -> None:
try:
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA synchronous=NORMAL")
except sqlite3.Error:
pass
def _rw(path: Path) -> sqlite3.Connection:
if not path.is_file():
raise RecordsRefused("records.table_missing", "记录表的数据文件不见了,请联系管理员")
conn = open_checked_sqlite(str(path), readonly=False)
_enable_wal(conn)
return conn
def _ro(path: Path) -> sqlite3.Connection:
if not path.is_file():
raise RecordsRefused("records.table_missing", "记录表的数据文件不见了,请联系管理员")
return open_checked_sqlite(str(path), readonly=True)
# --------------------------------------------------------------------------
# 建表、加列
# --------------------------------------------------------------------------
def create_file(path: Path, columns: list[Column]) -> None:
"""新建一个表版本的文件。路径上已有文件时:没有任何行(上次建了一半、登记没提交)就重建,有行就拒绝——
有数据的文件绝不覆盖。"""
with _path_lock(path):
if path.exists():
if _row_count(path) > 0:
raise RecordsRefused("records.file_exists", f"记录表文件 {path.name} 已经有数据,不能重建")
path.unlink()
for suffix in ("-wal", "-shm"):
Path(f"{path}{suffix}").unlink(missing_ok=True)
path.parent.mkdir(parents=True, exist_ok=True)
tmp = path.with_name(f".{path.name}.tmp-{new_id()[:12]}")
conn = open_checked_sqlite(str(tmp), readonly=False)
try:
_enable_wal(conn)
business = "".join(f", {_q(c.name)} {COLUMN_TYPES[c.type]}" for c in columns)
conn.execute(
f"CREATE TABLE {_q(TABLE)} ("
"_seq INTEGER PRIMARY KEY, _id TEXT NOT NULL UNIQUE, _submitted_by TEXT, _submitted_at TEXT NOT NULL, "
"_run_id TEXT, _supersedes TEXT, _schema_version INTEGER NOT NULL, "
f"{ACTION_KEY_COLUMN} TEXT UNIQUE{business})")
conn.execute(f"CREATE INDEX ix_records_supersedes ON {_q(TABLE)} (_supersedes)")
conn.execute(f"CREATE INDEX ix_records_submitted_by ON {_q(TABLE)} (_submitted_by)")
conn.commit()
finally:
conn.close()
os.replace(tmp, path)
def add_columns(path: Path, columns: list[Column]) -> None:
"""兼容的改动(只加可选字段):在同一个版本上 ALTER TABLE ADD COLUMN。已有的列跳过(重做一次也没事)。"""
if not columns:
return
with _path_lock(path):
conn = _rw(path)
try:
have = {str(r[1]).lower() for r in conn.execute(f"PRAGMA table_info({_q(TABLE)})")}
for c in columns:
if c.name.lower() not in have:
conn.execute(f"ALTER TABLE {_q(TABLE)} ADD COLUMN {_q(c.name)} {COLUMN_TYPES[c.type]}")
conn.commit()
finally:
conn.close()
def _row_count(path: Path) -> int:
try:
conn = open_checked_sqlite(str(path), readonly=True)
except sqlite3.Error:
return 0
try:
return int(conn.execute(f"SELECT count(*) FROM {_q(TABLE)}").fetchone()[0])
except sqlite3.Error:
return 0
finally:
conn.close()
# --------------------------------------------------------------------------
# 追加
# --------------------------------------------------------------------------
def _now() -> str:
return datetime.now(timezone.utc).isoformat()
_TYPE_TEXT = {"number": "数", "integer": "整数", "boolean": "是或否", "array": "多选(列表)", "file": "文件"}
_INTEGER = re.compile(r"[+-]?\d+")
_NUMBER = re.compile(r"[+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?")
def coerce(column: Column, value: Any) -> Any:
"""一个值按列类型收好,能存进 SQLite 的样子。模板渲染出来的一律是文字("12.5"、"true"、JSON),在这里收;
收不成抛 RecordsRefused(records.value_invalid),说清是哪个字段。空值(None、空串)交回 None。"""
if value is None or (isinstance(value, str) and not value.strip() and column.type != "string"):
return None
if isinstance(value, str) and column.type == "string":
return value
bad = RecordsRefused("records.value_invalid",
f"字段「{column.label or column.name}」应为{_TYPE_TEXT.get(column.type, '文字')},"
f"实际是「{str(value)[:80]}」")
if column.type == "string":
if isinstance(value, (dict, list)):
return json.dumps(value, ensure_ascii=False)
return str(value)
if column.type in ("number", "integer"):
number: Any = None
if isinstance(value, (int, float)) and not isinstance(value, bool):
number = value
elif isinstance(value, str):
text = value.strip()
number = int(text) if _INTEGER.fullmatch(text) else float(text) if _NUMBER.fullmatch(text) else None
if number is None or (isinstance(number, float) and not math.isfinite(number)):
raise bad
if column.type == "integer":
if isinstance(number, float) and not number.is_integer():
raise bad
return int(number)
return number
if column.type == "boolean":
if isinstance(value, bool):
return int(value)
if isinstance(value, str) and value.strip().lower() in ("true", "false"):
return int(value.strip().lower() == "true")
raise bad
# 多选、文件:存 JSON 文字
parsed = value
if isinstance(value, str):
try:
parsed = json.loads(value)
except ValueError:
raise bad from None
if column.type == "array" and not isinstance(parsed, list):
raise bad
if column.type == "file":
from app.apps.spec import FILE_REF_KEYS
if not (isinstance(parsed, dict) and isinstance(parsed.get("file_id"), str)
and set(parsed) <= set(FILE_REF_KEYS)):
raise bad
return json.dumps(parsed, ensure_ascii=False, sort_keys=True)
@dataclass(frozen=True)
class Appended:
seq: int
row_id: str
submitted_at: str
#: 同一个动作键已经写过:交回的是第一次写下的那一行,这次没有再写
duplicate: bool
values: dict[str, Any]
#: 这一行记的提交人(更正行沿用原行的,见 append)
submitted_by: str | None = None
#: 更正行:动手更正的人(不记进行里,记进回执、动作记录和审计);不是更正为 None
corrected_by: str | None = None
def receipt(self) -> dict[str, Any]:
return {"_seq": self.seq, "_id": self.row_id, "_submitted_at": self.submitted_at,
"_submitted_by": self.submitted_by, "duplicate": self.duplicate,
**({"corrected_by": self.corrected_by} if self.corrected_by is not None else {})}
def append(path: Path, columns: list[Column], values: Any, *, version: int, run_id: str | None, actor: str | None,
action_key: str, supersedes: str | None = None) -> Appended:
"""写一行。同一个 action_key 第二次来交回第一次的那一行(duplicate=True),不再写。
values 只认表里的业务列:多出来的字段、必填没填、类型收不成都拒(RecordsRefused),一行都不写。
supersedes 是被更正的那一行的 _id:它得在这张表里、而且还没被更正过(更正要改最新的那一行)。
提交人(_submitted_by):新提交记 actor;更正行沿用原行的提交人——这一条还是他交的那一条,按「只看自己的」看的人
照样看得见改过的那一行,别人代为更正也不会让它从原提交人眼前消失。动手更正的是 actor,交回在 corrected_by 里,
由调用方记进动作记录和审计(records/actions.py)。"""
if not action_key:
raise RecordsRefused("records.action_key_missing", "写记录表需要动作键,只能经调用工具节点写入")
if not isinstance(values, dict):
raise RecordsRefused("records.value_invalid", "要写入的内容需要是一组字段和值")
by_name = {c.name: c for c in columns}
unknown = [k for k in values if k not in by_name]
if unknown:
raise RecordsRefused("records.column_unknown",
f"记录表里没有这些字段:{'、'.join(map(str, unknown[:5]))}。表单改过的话,请重新发布应用")
row = {c.name: coerce(c, values.get(c.name)) for c in columns}
missing = [c.label or c.name for c in columns if c.required and row[c.name] in (None, "")]
if missing:
raise RecordsRefused("records.required_missing", f"必填的字段没有值:{'、'.join(missing)}")
with _path_lock(path):
conn = _rw(path)
try:
conn.execute("BEGIN IMMEDIATE")
done = conn.execute(f"SELECT _seq, _id, _submitted_at, _submitted_by, _supersedes FROM {_q(TABLE)} "
f"WHERE {ACTION_KEY_COLUMN} = ?", (action_key,)).fetchone()
if done is not None:
conn.rollback()
return Appended(seq=int(done[0]), row_id=str(done[1]), submitted_at=str(done[2]), duplicate=True,
values=_row_values(path, columns, int(done[0])), submitted_by=done[3],
corrected_by=actor if done[4] else None)
submitted_by = actor
if supersedes:
target = conn.execute(f"SELECT _seq, _submitted_by FROM {_q(TABLE)} WHERE _id = ?",
(supersedes,)).fetchone()
if target is None:
raise RecordsRefused("records.supersede_missing", "要更正的那一行不在这张记录表里")
later = conn.execute(f"SELECT 1 FROM {_q(TABLE)} WHERE _supersedes = ? LIMIT 1",
(supersedes,)).fetchone()
if later is not None:
raise RecordsRefused("records.supersede_twice", "这一行已经被更正过了,请在最新的那一行上更正")
submitted_by = target[1]
row_id, at = new_id(), _now()
names = ["_id", "_submitted_by", "_submitted_at", "_run_id", "_supersedes", "_schema_version",
ACTION_KEY_COLUMN, *row]
params = [row_id, submitted_by, at, run_id, supersedes or None, version, action_key, *row.values()]
cur = conn.execute(f"INSERT INTO {_q(TABLE)} ({', '.join(_q(n) for n in names)}) "
f"VALUES ({', '.join('?' for _ in names)})", params)
seq = int(cur.lastrowid or 0)
conn.commit()
except BaseException:
conn.rollback()
raise
finally:
conn.close()
return Appended(seq=seq, row_id=row_id, submitted_at=at, duplicate=False, values=row, submitted_by=submitted_by,
corrected_by=actor if supersedes else None)
def _row_values(path: Path, columns: list[Column], seq: int) -> dict[str, Any]:
conn = _ro(path)
try:
names = [c.name for c in columns]
got = conn.execute(f"SELECT {', '.join(_q(n) for n in names) or '1'} FROM {_q(TABLE)} WHERE _seq = ?",
(seq,)).fetchone()
return dict(zip(names, got)) if got is not None and names else {}
finally:
conn.close()
# --------------------------------------------------------------------------
# 读
# --------------------------------------------------------------------------
def high_water(path: Path) -> int:
"""现在最大的 _seq;空表为 0。"""
conn = _ro(path)
try:
got = conn.execute(f"SELECT max(_seq) FROM {_q(TABLE)}").fetchone()
return int(got[0] or 0)
finally:
conn.close()
def lookup(path: Path, columns: list[Column], action_key: str) -> dict[str, Any] | None:
"""按动作键找写下的那一行(查询能力 records_lookup 用):没有为 None。"""
conn = _ro(path)
try:
got = conn.execute(f"SELECT _seq, _id, _submitted_at FROM {_q(TABLE)} WHERE {ACTION_KEY_COLUMN} = ?",
(action_key,)).fetchone()
finally:
conn.close()
if got is None:
return None
return {"_seq": int(got[0]), "_id": str(got[1]), "_submitted_at": str(got[2]), "duplicate": True}
def union_compatible(new: Column, old: Column) -> bool:
"""跨版本汇总时,旧版本的这一列能不能并进新版本的同名列:类型一样,或者整数和数之间(都是数)。别的类型改动(数改成
文字、文字改成是或否……)意思变了,旧版本那一列补空,不把两种东西混在一列里算。"""
return new.type == old.type or {new.type, old.type} <= {"number", "integer"}
def view_select(columns: list[Column], *, high_water: int | None, scope: str, viewer: str | None,
schema: str = "main", present: list[Column] | None = None, table: str = TABLE) -> str:
"""临时视图(和看板、记录列表)看到的那一份:
- _seq ≤ 高水位(None 为不限,现在有的都算);
- 没被更正过:在高水位之内没有一行指向它(_supersedes)。按高水位判,不靠可改的标记——原行永远不改;
- 行级过滤:scope=all 全部,mine 只看 _submitted_by = viewer 的,none 一行都不给。
不露 _action_key。列名照 SYSTEM_COLUMNS + 业务列的顺序。present 是 schema 那个文件里实际有的业务列(跨版本汇总时
旧版本的列):columns 里它没有的、类型对不上的(union_compatible)补 NULL;None 为 columns 都在。"""
have = None if present is None else {c.name.lower(): c for c in present}
def pick(c: Column) -> str:
if have is None:
return f"r.{_q(c.name)}"
old = have.get(c.name.lower())
return f"r.{_q(old.name)} AS {_q(c.name)}" if old is not None and union_compatible(c, old) \
else f"NULL AS {_q(c.name)}"
cols = ", ".join([*(f"r.{_q(n)}" for n in SYSTEM_COLUMNS), *(pick(c) for c in columns)])
src = f"{_q(schema)}.{_q(table)}"
where = []
if high_water is not None:
seq = int(high_water)
where.append(f"r._seq <= {seq}")
newer = f" AND s._seq <= {seq}"
else:
newer = ""
where.append(f"NOT EXISTS (SELECT 1 FROM {src} AS s WHERE s._supersedes = r._id{newer})")
if scope == "mine":
where.append(f"r._submitted_by = {_lit(viewer)}" if viewer else "0")
elif scope != "all":
where.append("0")
return f"SELECT {cols} FROM {src} AS r WHERE {' AND '.join(where)}"
@dataclass(frozen=True)
class OlderVersion:
"""跨版本汇总时并进来的一个旧表版本:挂在查询连接上的库名(ATTACH … AS schema)、那一版的业务列、这次查询的高水位
(None 为不限),以及该版本所在的物理表名(直接挂库时是 records,超挂库上限汇总进归档库时是 rows_v<version>)。"""
schema: str
columns: list[Column]
high_water: int | None
table: str = TABLE
@dataclass(frozen=True)
class ArchiveSpec:
"""需要增量同步进单库多物理表归档(<table_key>.archive.db)的旧版本描述。"""
version: int
path: str
columns: list[Column]
def sync_archive(table_key: str, specs: list[ArchiveSpec] | tuple[ArchiveSpec, ...]) -> Path:
"""把超挂库上限的旧版本增量同步进单库多物理表归档文件(rows_v<version>),规避 SQLite 最多 10 个 ATTACH 库的限制。
行只追加、不修改:已同步到相同列结构且 _seq 无新增的版本 O(1) 跳过;退役前发起的在途运行补写了新行时,只复制
_seq > meta_max_seq 的增量行。"""
dest = archive_path(table_key)
if not specs:
return dest
dest.parent.mkdir(parents=True, exist_ok=True)
with _path_lock(dest):
arch = open_checked_sqlite(str(dest), readonly=False)
try:
_enable_wal(arch)
arch.execute(
"CREATE TABLE IF NOT EXISTS _archive_meta ("
"version INTEGER PRIMARY KEY, max_seq INTEGER NOT NULL, col_sig TEXT NOT NULL)"
)
meta = {
int(r[0]): (int(r[1]), str(r[2]))
for r in arch.execute("SELECT version, max_seq, col_sig FROM _archive_meta").fetchall()
}
dirty = False
for spec in specs:
src_file = Path(spec.path)
if not src_file.is_file():
continue
src = _ro(src_file)
try:
info = [(str(r[1]), str(r[2] or "TEXT")) for r in src.execute(f"PRAGMA table_info({_q(TABLE)})")
if str(r[1]) != ACTION_KEY_COLUMN]
src_max = int(src.execute(f"SELECT coalesce(max(_seq), 0) FROM {_q(TABLE)}").fetchone()[0] or 0)
sig = json.dumps(info, ensure_ascii=False)
tbl = archive_table_name(spec.version)
prev = meta.get(spec.version)
if prev is not None and prev[1] == sig and prev[0] == src_max:
continue
col_names = [name for name, _ in info]
select_cols = ", ".join(_q(n) for n in col_names)
placeholders = ", ".join("?" for _ in col_names)
if prev is None or prev[1] != sig:
arch.execute(f"DROP TABLE IF EXISTS {_q(tbl)}")
ddl_cols = ", ".join(
"_seq INTEGER PRIMARY KEY" if name == "_seq" else f"{_q(name)} {col_type}"
for name, col_type in info
)
arch.execute(f"CREATE TABLE {_q(tbl)} ({ddl_cols})")
arch.execute(f"CREATE INDEX ix_{tbl}_supersedes ON {_q(tbl)} (_supersedes)")
arch.execute(f"CREATE INDEX ix_{tbl}_submitted_by ON {_q(tbl)} (_submitted_by)")
since_seq = 0
else:
since_seq = prev[0]
cur = src.execute(
f"SELECT {select_cols} FROM {_q(TABLE)} WHERE _seq > ? ORDER BY _seq ASC",
(since_seq,),
)
insert_sql = f"INSERT OR REPLACE INTO {_q(tbl)} ({select_cols}) VALUES ({placeholders})"
while batch := cur.fetchmany(500):
arch.executemany(insert_sql, batch)
arch.execute(
"INSERT OR REPLACE INTO _archive_meta (version, max_seq, col_sig) VALUES (?, ?, ?)",
(spec.version, src_max, sig),
)
dirty = True
finally:
src.close()
if dirty:
arch.commit()
except BaseException:
arch.rollback()
raise
finally:
arch.close()
return dest
def view_prelude(columns: list[Column], *, high_water: int | None, scope: str, viewer: str | None,
older: tuple[OlderVersion, ...] | list[OlderVersion] = ()) -> tuple[str, ...]:
"""查询连接上建同名临时视图的两条语句。SQLite 解析不带库名的表名时先找 temp 库,于是 FROM records 查到的就是
这份视图;直接写 main.records 的 SQL 由只读守卫拒掉(data/engine.run_query 的 forbid_schemas)。
older 是这张表比这一版老的各版本(只读挂在连接上,records/views.py):视图按 UNION ALL 把它们接在这一版后面,
列照这一版的来,旧版本没有的列补 NULL;每一版各按自己的高水位、自己文件里的更正判,行级过滤同一个。"""
parts = [view_select(columns, high_water=high_water, scope=scope, viewer=viewer)]
parts += [view_select(columns, high_water=o.high_water, scope=scope, viewer=viewer, schema=o.schema,
present=o.columns, table=o.table) for o in older]
return (f"DROP VIEW IF EXISTS temp.{_q(TABLE)}",
f"CREATE TEMP VIEW {_q(TABLE)} AS " + " UNION ALL ".join(parts))
def rows(path: Path, columns: list[Column], *, scope: str, viewer: str | None, history: bool = False,
before: int | None = None, limit: int = 50) -> tuple[list[dict[str, Any]], bool]:
"""记录列表(records_list、GET /api/apps/{id}/records):现在的数据,按看的人过滤,_seq 倒序。
history=False 只给当前有效的行;True 连被更正过的也给,标 superseded。before 是游标(只要 _seq 更小的)。
交回 (行, 后面还有没有)。"""
limit = max(1, min(int(limit), 200))
names = [c.name for c in columns]
src = f"main.{_q(TABLE)}"
where = ["1"]
params: list[Any] = []
if scope == "mine":
where.append("r._submitted_by = ?")
params.append(viewer)
elif scope != "all":
return [], False
if not history:
where.append(f"NOT EXISTS (SELECT 1 FROM {src} AS s WHERE s._supersedes = r._id)")
if before is not None:
where.append("r._seq < ?")
params.append(int(before))
cols = ", ".join(f"r.{_q(n)}" for n in (*SYSTEM_COLUMNS, *names))
sql = (f"SELECT {cols}, EXISTS (SELECT 1 FROM {src} AS s WHERE s._supersedes = r._id) "
f"FROM {src} AS r WHERE {' AND '.join(where)} ORDER BY r._seq DESC LIMIT ?")
conn = _ro(path)
try:
got = conn.execute(sql, [*params, limit + 1]).fetchall()
finally:
conn.close()
out: list[dict[str, Any]] = []
width = len(SYSTEM_COLUMNS)
for row in got[:limit]:
system = dict(zip(SYSTEM_COLUMNS, row[:width]))
values = {n: _shown(c, v) for n, c, v in zip(names, columns, row[width:width + len(names)])}
out.append({**system, "superseded": bool(row[-1]), "values": values})
return out, len(got) > limit
def row_by_id(path: Path, columns: list[Column], row_id: str) -> dict[str, Any] | None:
"""按 _id 取一行(下载附件时核对行级可见用)。"""
names = [c.name for c in columns]
cols = ", ".join(_q(n) for n in (*SYSTEM_COLUMNS, *names))
conn = _ro(path)
try:
got = conn.execute(f"SELECT {cols} FROM {_q(TABLE)} WHERE _id = ?", (row_id,)).fetchone()
finally:
conn.close()
if got is None:
return None
width = len(SYSTEM_COLUMNS)
return {**dict(zip(SYSTEM_COLUMNS, got[:width])),
"values": {n: _shown(c, v) for n, c, v in zip(names, columns, got[width:])}}
def _shown(column: Column, value: Any) -> Any:
"""存的样子 → 给界面的样子:是或否交回布尔,多选、文件解开 JSON。"""
if value is None:
return None
if column.type == "boolean":
return bool(value)
if column.type in ("array", "file") and isinstance(value, str):
try:
return json.loads(value)
except ValueError:
return value
return value