Files
AgentLab/backend/app/records/views.py
T
ThearchyHelios 2f90f0eb3a 口径台账:固定执行、验证审阅发布与增量重导入
- QueryCore 固定执行(K09):按已发布的不可变口径跑一次,交回类型化标量、发布/manifest 引用与逐查询证据
- 依赖闭包(K08):按选中的指标算出要拷的取数节点、要的入口字段,接进 derive / compose / metrics
- SQL 模板参数完整性(K01):试跑、组合问数、派生图、运行时取数共用一套语义,缺参数在渲染前就拒
- 受限字段变更服务(K03)与口径领域边界(K01):改动打在整图深拷贝上,身份与证据归属不可改
- 版本化受控模板绑定(K04):业务口径按显式绑定落到 SQL,对不上就报 unsupported / needs_update
- 验证记录(K05)与最小审阅:样例匹配不等于正式批准,审批人只从服务端身份取
- 发布依据清单(K07):发布时按不可变图物化内容寻址清单,正式用途按它判
- 增量重导入(K11)与 B/L/R 三方合并、受限自然语言修改提案(K12):只产候选,落图仍走既有写路径
- 接口与前端:口径草稿页接上字段变更、绑定状态、验证与重导入,新增 caliber-drafts / studio 检查脚本

验证:前端 tsc -b 通过。
2026-11-10 15:05:18 +08:00

248 lines
14 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""记录表的查询视图(docs/design/normal-user-apps.md §3.7「查询连接」)。
origin=records 的数据源查询时换成一个 RecordsView(字段和 DataSource 同名,同 table_versions.SourceView 的做法):
带着这次查询的高水位、看哪些行,以及连接上要先执行的两条语句(建同名临时视图)。data/engine.run_query 认这两样:
- prelude:借到连接后、执行查询前依次执行(DROP / CREATE TEMP VIEW records);
- forbid_schemas:SQL 里直接写 main.<表> 的拒掉——不然绕过临时视图就看到了底表(全部行、被更正的行、幂等键)。
高水位来自运行的 data_versions(发起时记下,engine/runner._data_versions),不在运行里(工具库试用、数据剖析)的查询
用现在的数据、看全部(这些入口只有搭建者以上进得来)。
**跨版本汇总。** 表单不兼容地改过之后,看板查的是新一版的数据源,旧记录在比它老的版本文件里。查询时把这张表比这一版老的
各版本只读挂到连接上(attach:ATTACH 'file:…?mode=ro' AS records_v<版本>,data/engine.run_query 执行),临时视图按
UNION ALL 接上,列照这一版的来、旧版本没有的(或类型改得对不上的)补 NULL(records/store.view_prelude)。各版本各有高水位:
发起时一并记进 data_versions 那一项的 older(停用之前发起的运行还能往停用的版本写完,records/actions.py,所以停用的版本
同样要按高水位固定)。挂上去的库名同样进 forbid_schemas:records_v1.records 和 main.records 一样绕过视图,一律拒。
用户 SQL 里写 ATTACH 照旧被只读守卫拒(data/guard.py),挂库只有平台自己做。
"""
from __future__ import annotations
import asyncio
import copy
import logging
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
from app.records import RECORDS_ORIGIN, store
logger = logging.getLogger(__name__)
#: data_versions 里记录表那一项的形状:{"name", "origin": "records", "table", "version", "seq", "older", "rows",
#: "viewer"?}。older 是比这一版老的各版本的高水位 [{"version", "seq"}](跨版本汇总);数据文件不在了的那一版记成
#: {"version", "seq": None, "missing": True}——汇总里没有它,印章(apps/resolve.stamp_of)、查询快照照实说
MISSING = "missing"
PIN_KEYS = ("name", "origin", "table", "version", "seq", "older", "rows", "viewer")
#: 旧版本挂在查询连接上的库名:records_v<版本>
ATTACH_PREFIX = "records_v"
@dataclass
class RecordsView:
"""绑定了高水位和行级过滤的记录表数据源。engine / introspect / guard / 工具只读这些字段,和 DataSource 同名。"""
id: str
name: str
kind: str = "sqlite"
database: str | None = None
options: dict[str, Any] = field(default_factory=dict)
readonly: bool = True
description: str = ""
schema_cache: dict[str, Any] = field(default_factory=dict)
schema_synced_at: datetime | None = None
enabled: bool = True
host: str | None = None
port: int | None = None
username: str | None = None
password: str | None = None
last_check: dict[str, Any] | None = None
origin: str = RECORDS_ORIGIN
current_snapshot_id: str | None = None
#: 记录表不分快照:引擎缓存按源 id 一个,临时视图每次查询现建
snapshot_id: str | None = None
#: 文件还在追加,不能以 immutable 打开
immutable: bool = False
expected_sha256: str | None = None
#: 这次查询的高水位(None:现在有的都算)
high_water: int | None = None
#: all / mine / none(records/access.py)
rows: str = "all"
viewer: str | None = None
#: 连接上先执行的语句(建同名临时视图)
prelude: tuple[str, ...] = ()
#: 先只读挂到连接上的旧版本:((库名, 文件路径), …),data/engine.run_query 在 prelude 之前挂好
attach: tuple[tuple[str, str], ...] = ()
#: SQL 里不许直接写的库名(main 和挂上去的各旧版本)
forbid_schemas: tuple[str, ...] = ("main",)
#: 并进来的旧版本和各自的高水位 [{"version", "seq"}],数据文件不在了、没并进来的记 {"version", "seq": None,
#: "missing": True}(查询快照里记下,照实说这次汇总含哪几版、缺哪几版)
older: tuple[dict[str, Any], ...] = ()
@dataclass(frozen=True)
class Older:
"""并进汇总的一个旧版本:版本号、文件、业务列、这次查询的高水位(None 为不限)。"""
version: int
path: str
columns: list[store.Column]
high_water: int | None
@property
def schema(self) -> str:
return f"{ATTACH_PREFIX}{int(self.version)}"
def bind_view(source: Any, columns: list[store.Column], *, high_water: int | None, rows: str = "all",
viewer: str | None = None, older: list[Older] | tuple[Older, ...] = (),
missing: list[int] | tuple[int, ...] = ()) -> RecordsView:
"""数据源行 + 高水位 + 行级过滤(+ 并进来的旧版本;missing 是数据文件不在了、并不进来的旧版本号)→ RecordsView。"""
rows = rows if rows in ("all", "mine", "none") else "none"
older = tuple(older)
return RecordsView(
id=source.id, name=source.name, database=source.database, options=dict(source.options or {}),
description=source.description or "", schema_cache=copy.deepcopy(source.schema_cache or {}),
schema_synced_at=getattr(source, "schema_synced_at", None), enabled=bool(source.enabled),
last_check=getattr(source, "last_check", None), high_water=high_water, rows=rows, viewer=viewer,
prelude=store.view_prelude(columns, high_water=high_water, scope=rows, viewer=viewer,
older=[store.OlderVersion(o.schema, o.columns, o.high_water) for o in older]),
attach=tuple((o.schema, o.path) for o in older),
forbid_schemas=("main", *(o.schema for o in older)),
older=(*({"version": o.version, "seq": o.high_water} for o in older),
*({"version": int(v), "seq": None, MISSING: True} for v in missing)))
def columns_of_source(source: Any) -> list[store.Column]:
"""数据源上登记的表结构里的业务列(不含系统列)。records_tables 的列定义是权威;这里只在拿不到登记时兜底。"""
from app.records import SYSTEM_COLUMNS
meta = ((getattr(source, "schema_cache", None) or {}).get("tables") or {}).get(store.TABLE) or {}
return [store.Column(name=str(c.get("name")), label=str(c.get("comment") or c.get("name")),
type=_type_of(str(c.get("type") or "")))
for c in meta.get("columns") or [] if isinstance(c, dict) and c.get("name") not in SYSTEM_COLUMNS]
def _type_of(sql_type: str) -> str:
return {"REAL": "number", "INTEGER": "integer"}.get(sql_type.upper(), "string")
async def _older_versions(session: AsyncSession, table: Any) -> list[Any]:
"""这张表比这一版老的各版本(新的在前):停用了的旧版本,以及起草出的新一版还没发布时线上在写的那一版。"""
from app.records.tables import versions_of
if table is None:
return []
return [row for row in await versions_of(session, table.table_key) if row.version < table.version]
async def resolve(session: AsyncSession, source: Any, entry: Any = None) -> RecordsView:
"""查询时实际该用的视图。entry 是这次运行 data_versions 里这张表的那一项;None 是不在运行里(现在的数据、看全部)。
比这一版老的各版本一起并进来(跨版本汇总):在运行里的,只并发起时固定了高水位的那几版(entry 的 older),各按自己的
高水位;不在运行里的,各版本现在的数据都算。"""
from app.records.tables import columns_of, path_of, table_of_source
table = await table_of_source(session, source.id)
columns = columns_of(table) if table is not None else columns_of_source(source)
older_rows = await _older_versions(session, table)
if isinstance(entry, dict):
seq = entry.get("seq")
marked = [o for o in entry.get("older") or [] if isinstance(o, dict) and isinstance(o.get("version"), int)]
# 发起时文件就不在的那几版:没有高水位,不挂;照原样带着,快照里照实说汇总缺了它们
missing = [int(o["version"]) for o in marked if o.get(MISSING)]
pinned = {int(o["version"]): o.get("seq") for o in marked if not o.get(MISSING)}
older = [Older(row.version, str(path_of(row)), columns_of(row),
int(pinned[row.version]) if isinstance(pinned[row.version], int) else 0)
for row in older_rows if row.version in pinned]
return bind_view(source, columns, high_water=int(seq) if isinstance(seq, int) else 0,
rows=str(entry.get("rows") or "none"), viewer=entry.get("viewer"), older=older,
missing=missing)
older = [Older(row.version, str(path_of(row)), columns_of(row), None) for row in older_rows
if path_of(row).is_file()]
missing = [row.version for row in older_rows if not path_of(row).is_file()]
if missing:
logger.warning("记录表 %s 第 %s 版的数据文件不在了,跨版本汇总里不含这几版", table.table_key,
"、".join(str(v) for v in missing))
return bind_view(source, columns, high_water=None, rows="all", older=older, missing=missing)
def default_view(source: Any) -> RecordsView:
"""没经解析就拿来查的记录表数据源(数据剖析这类只有搭建者进得来的入口):现在的数据、看全部,照样走临时视图——
底表里的幂等键、被更正的行不露出来。列取登记在数据源上的表结构。"""
return bind_view(source, columns_of_source(source), high_water=None, rows="all")
async def pin_entry(session: AsyncSession, source: Any, *, run_class: str | None = None,
started_by: str | None = None) -> dict[str, Any] | None:
"""发起运行时记进 data_versions 的那一项:高水位,以及(给了运行类别时)这次运行看哪些行。登记不在、文件不在的
返回 None(这个源不固定,查询时照实报错)。"""
from app.records.access import scope_of_run
from app.records.tables import path_of, table_of_source
table = await table_of_source(session, source.id)
if table is None:
return None
try:
seq = await asyncio.to_thread(store.high_water, path_of(table))
except store.RecordsRefused:
return None
older: list[dict[str, Any]] = []
for row in await _older_versions(session, table):
try:
older.append({"version": row.version, "seq": await asyncio.to_thread(store.high_water, path_of(row))})
except store.RecordsRefused:
# 旧版本的文件不在了(管理员手工删了文件才会这样):汇总里没有它。记进这一项(印章、查询快照照实说缺了哪一版,
# 看的人知道数不全),另记一条日志
older.append({"version": row.version, "seq": None, MISSING: True})
logger.warning("记录表 %s 第 %s 版的数据文件不在了,跨版本汇总里不含这一版", table.table_key, row.version)
entry: dict[str, Any] = {"name": source.name, "origin": RECORDS_ORIGIN, "table": table.table_key,
"version": table.version, "seq": seq, "older": older}
if run_class is not None:
entry.update(await scope_of_run(session, table.app_id, run_class=run_class, started_by=started_by))
return entry
async def scope_entries(session: AsyncSession, versions: dict[str, Any] | None, *, run_class: str | None,
started_by: str | None) -> None:
"""给 data_versions 里的记录表补上这次运行看哪些行(就地改):正式运行看全部,别的按发起人在应用上的授权。"""
from app.records.access import scope_of_run
from app.records.tables import table_of_source
for source_id, entry in (versions or {}).items():
if not isinstance(entry, dict) or entry.get("origin") != RECORDS_ORIGIN:
continue
table = await table_of_source(session, source_id)
entry.update(await scope_of_run(session, table.app_id if table is not None else None, run_class=run_class,
started_by=started_by))
def cache_token(entry: Any) -> Any:
"""按需视图缓存键里记录表那一项(apps/resolve.cache_key):这一版的高水位,并进来的旧版本有的话连同它们的高水位——
停用之前发起的运行还能往旧版本写(records/actions.py),旧版本多了一行,汇总就变了。"""
if not isinstance(entry, dict):
return None
older = [[o.get("version"), o.get("seq")] for o in entry.get("older") or [] if isinstance(o, dict)]
return [entry.get("seq"), older] if older else entry.get("seq")
def inherit_scope(versions: dict[str, Any] | None) -> None:
"""派生的子运行**继承**父 Run 已授权的 data_versions 时,记录表的行级范围原样带着,不重算(K10)。
重算是错的:子运行按 exploratory 重算,一个正式父 Run 的 all 会被收成 mine/none——父已授权、子看得更少,
同一次口径查询算出来的数就和父 Run 自己的不一样了;父 Run 发起之后管理员改了授权,同一次运行的前后两段
也会看到不同的行。范围在父 Run 发起时定下(records.access.scope_of_run),子运行既不放大也不缩小。
缺 rows 的项(升级前发起的运行、手工拼的)按最窄的 none 算,绝不当成 all。"""
from app.records.access import SCOPES
for entry in (versions or {}).values():
if not isinstance(entry, dict) or entry.get("origin") != RECORDS_ORIGIN:
continue
if entry.get("rows") not in SCOPES:
entry["rows"] = "none"
def is_records(source: Any) -> bool:
return (getattr(source, "origin", None) or "manual") == RECORDS_ORIGIN