全栈安全加固、性能提速与架构演进:连接池与沙箱止血、双路混合检索、多版本归档及前端分包治理

- 第一批(安全加固与生命周期止血):隔离测试宿主机密钥与 .env 泄漏;修复 SQLite 取消查询挂死、:memory: 连接池报错与 EngineCache 世代竞态;补齐运行与评测删除时的 Checkpointer 线程、Artifact 磁盘文件与无主附件回收;加固三套沙箱路径越界、进程组清理与 Seatbelt 数据目录隔离,封堵 SSRF IPv4-mapped IPv6 元数据绕过与响应体 OOM;修复推理模型多轮 reasoning_content、结构化输出 Token 漏计与缓存折扣计费
- 第二批(数据库/集群/问数提速与首屏分包):补齐核心表索引与 SQLite WAL 调优,列表接口按需 defer 大列,收敛启动扫描并打通跨进程 slot 槽位唤醒;优化跨源全局选表、哈希桶数字绑定、流式 CSV 导入及多方言分析算子;剥离首屏静态引入的 @xyflow/react 与全局样式(首屏主包降至 461.59 kB),补充流式 rAF 攒批、Store 精细订阅与表单无障碍绑定
- 第三批(架构演进与工程自动化闭环):记录库超挂接阈值历史版本自动增量归档至单库多表(解除 SQLite 10 库 ATTACH 硬上限);重构知识库为词面 BM25 与全集合归一化向量矩阵真·双路独立召回并支持游标分批重建;升级多方言感知 SQL 模板词法分析(E'...' / $$...$$ / q'[...]' / # / 嵌套注释)与 Decimal / bool / NaN 参数校验,消除 ledger 层对 api 路由层的反向依赖;拆分 3101 行 CaliberDrafts 页面为 7 个领域子模块,消除检查脚本本机路径硬编码并新增 GitHub Actions CI 门禁与三批回归测试套件
This commit is contained in:
ThearchyHelios
2026-11-13 14:51:36 +08:00
parent 09949fc8ac
commit eddeded2a4
106 changed files with 7178 additions and 3913 deletions
+158
View File
@@ -11,6 +11,10 @@ from app.engine.evidence import ledger_enabled
from app.engine.lineage import scoped_id as _scoped
from app.analysis.contract import LEDGER_KIND
MASKED = "已遮罩"
RUN_NOT_FOUND = "run_not_found"
REPORT_NOT_FOUND = "evidence_report_not_found"
def _fetch(artifact: str) -> tuple[Any, bool]:
try:
@@ -236,3 +240,157 @@ def metric_cards(sealed: Sealed) -> list[tuple[str, str, dict[str, Any]]]:
if ok is True and isinstance(content, dict) and isinstance(content.get("metrics"), list):
out.append((node_id, art, content))
return out
async def load_sealed(run_id: str) -> Sealed:
"""读取封存范围;运行记录不存在时抛 404 CodedHTTPException(供引擎与领域层共用,无需引入 api/evidence)。"""
from app.api.coded import CodedHTTPException
try:
return await read_sealed(run_id)
except LookupError:
raise CodedHTTPException(404, "运行记录不存在", RUN_NOT_FOUND) from None
@dataclass
class Report:
node_id: str
doc_artifact: str
ok: bool | None
repairs: int | None
stats: dict[str, Any] | None
doc: dict[str, Any] | None
hash_ok: bool | None
fields: list[str]
claims: str | None = None
def output_evidence(sealed: Sealed) -> list[dict[str, Any]]:
output = sealed.payload_of(sealed.nodes_of("output")) or {}
primary = output.get("_evidence")
if not isinstance(primary, dict):
return []
return [primary, *[r for r in primary.get("others") or [] if isinstance(r, dict)]]
def reports_of(sealed: Sealed, *, nested: bool = False) -> list[Report]:
"""封存范围内每个报告节点最后一次的自查结果,按第一次出现的顺序。"""
latest: dict[str, dict[str, Any]] = {}
for _, etype, node_id, data in sealed.events:
if (
etype == "report.checked"
and node_id
and isinstance(data, dict)
and data.get("doc_artifact")
and (nested or "/" not in node_id)
):
latest[node_id] = data
evidence = output_evidence(sealed)
out: list[Report] = []
for node_id, data in latest.items():
artifact = str(data["doc_artifact"])
try:
doc = load(artifact)
hash_ok: bool | None = True if doc is not None else None
except ValueError:
doc, hash_ok = None, False
if isinstance(doc, dict) and (doc.get("node_id") != node_id or doc.get("run_id") != sealed.run_id):
doc, hash_ok = None, False
fields = [
f
for r in evidence
if r.get("report_node") == node_id and r.get("doc_artifact") == artifact
for f in r.get("fields") or []
]
out.append(
Report(
node_id=node_id,
doc_artifact=artifact,
ok=data.get("ok"),
repairs=data.get("repairs"),
stats=(doc or {}).get("stats") or data.get("stats"),
doc=doc if isinstance(doc, dict) else None,
hash_ok=hash_ok,
fields=fields,
claims=data.get("claims") if isinstance(data.get("claims"), str) else None,
)
)
return out
def choose_report(reports: list[Report], wanted: str | None, sealed: Sealed) -> Report:
"""从封存报告列表里选出目标报告(指定 wanted 优先,否则取出口 _evidence 主报告,再退回第一份)。"""
from app.api.coded import CodedHTTPException
if wanted:
chosen = next((r for r in reports if r.node_id == wanted), None)
if chosen is None:
raise CodedHTTPException(404, "本次运行中没有所选报告撰写节点的文档", REPORT_NOT_FOUND)
return chosen
primary = next(iter(output_evidence(sealed)), {})
return next((r for r in reports if r.node_id == primary.get("report_node")), reports[0])
class ShownText:
"""把展示用 SQL、检查片段、合并警告里命中的运行遮罩值替换为「已遮罩」。"""
def __init__(self, secrets: Any) -> None:
self._secrets = secrets
self.changed = False
def text(self, value: Any) -> Any:
if self._secrets is None or not isinstance(value, str) or not value:
return value
out = self._secrets.scrub(value, loose=True)
self.changed = self.changed or out != value
return out
def checks(self, checks: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [
{**c, **{k: self.text(c[k]) for k in ("message", "sql_excerpt") if isinstance(c.get(k), str)}}
for c in checks
]
def merge(self, view: dict[str, Any]) -> dict[str, Any]:
view["sql"] = self.text(view.get("sql"))
view["warnings"] = [{**w, "message": self.text(w.get("message"))} for w in view.get("warnings") or []]
view["inputs"] = [
{**i, **({"checks": self.checks(i["checks"])} if i.get("checks") else {})}
for i in view.get("inputs") or []
]
return view
def open_masks(sealed: Sealed | None = None) -> Any:
"""获取请求级数据源遮罩上下文(通过统一边界暴露给领域层,兼容测试对证据遮罩构造器的桩替换)。"""
from app.api import evidence as _evidence_mod
return _evidence_mod._Masks(sealed)
async def resolve_metric_masks(
metric: dict[str, Any],
card_node: str,
artifact: str,
sealed: Sealed,
masks: Any,
*,
card: dict[str, Any] | None = None,
output: Any = None,
by_value: bool = True,
sure: bool = False,
) -> tuple[dict[str, Any], str | None]:
"""判断单个指标及其输入是否命中的数据源遮罩。"""
from app.api import evidence as _evidence_mod
return await _evidence_mod._metric_masks(
metric,
card_node,
artifact,
sealed,
masks,
card=card,
output=output,
by_value=by_value,
sure=sure,
)