全栈安全加固、性能提速与架构演进:连接池与沙箱止血、双路混合检索、多版本归档及前端分包治理

- 第一批(安全加固与生命周期止血):隔离测试宿主机密钥与 .env 泄漏;修复 SQLite 取消查询挂死、:memory: 连接池报错与 EngineCache 世代竞态;补齐运行与评测删除时的 Checkpointer 线程、Artifact 磁盘文件与无主附件回收;加固三套沙箱路径越界、进程组清理与 Seatbelt 数据目录隔离,封堵 SSRF IPv4-mapped IPv6 元数据绕过与响应体 OOM;修复推理模型多轮 reasoning_content、结构化输出 Token 漏计与缓存折扣计费
- 第二批(数据库/集群/问数提速与首屏分包):补齐核心表索引与 SQLite WAL 调优,列表接口按需 defer 大列,收敛启动扫描并打通跨进程 slot 槽位唤醒;优化跨源全局选表、哈希桶数字绑定、流式 CSV 导入及多方言分析算子;剥离首屏静态引入的 @xyflow/react 与全局样式(首屏主包降至 461.59 kB),补充流式 rAF 攒批、Store 精细订阅与表单无障碍绑定
- 第三批(架构演进与工程自动化闭环):记录库超挂接阈值历史版本自动增量归档至单库多表(解除 SQLite 10 库 ATTACH 硬上限);重构知识库为词面 BM25 与全集合归一化向量矩阵真·双路独立召回并支持游标分批重建;升级多方言感知 SQL 模板词法分析(E'...' / $$...$$ / q'[...]' / # / 嵌套注释)与 Decimal / bool / NaN 参数校验,消除 ledger 层对 api 路由层的反向依赖;拆分 3101 行 CaliberDrafts 页面为 7 个领域子模块,消除检查脚本本机路径硬编码并新增 GitHub Actions CI 门禁与三批回归测试套件
This commit is contained in:
ThearchyHelios
2026-11-13 14:51:36 +08:00
parent 09949fc8ac
commit eddeded2a4
106 changed files with 7178 additions and 3913 deletions
+1 -1
View File
@@ -41,7 +41,7 @@ ParentResolver = Callable[[str], AnalysisResult | None]
#: (row_out_of_range、headline_empty);要比的组这期没有(unknown_group、too_few_groups);要用的那一格这期为空
#: (empty_cell:乘积拆解的乘积、因子,差异检验的计数)
DATA_ERRORS = frozenset({"parent_refused", "parent_key_missing", "period_not_found", "row_out_of_range",
"headline_empty", "unknown_group", "too_few_groups", "empty_cell"})
"headline_empty", "unknown_group", "too_few_groups", "empty_cell", "too_many_rows"})
def _op_mode(request: Mapping[str, Any]) -> tuple[str, str | None]:
+6 -4
View File
@@ -724,11 +724,13 @@ def _half_ulp_ratio(raw: Any, value: Fraction) -> Fraction:
def lmdi(base: Sequence[float], target: Sequence[float], m_base: float, m_target: float) -> list[float]:
"""LMDI-I:contrib_k = L(M', M) · ln(F_k' / F_k),L(a, b) = (a − b) / (ln a − ln b),L(a, a) = a。因子按书写精度取过整、
乘积和乘积列差一点时,各因子按同一比例缩放到乘积恰好等于乘积列(每个因子的对数比各补上 (1/K)·差的对数),Σcontrib 在
浮点误差内等于 M' − M。"""
浮点误差内等于 M' − M。全程在对数域求和,避免多因子连乘 float64 上溢或下溢归零。"""
k = len(base)
fix = (math.log(m_target / math.prod(target)) - math.log(m_base / math.prod(base))) / k
weight = m_base if m_base == m_target else (m_target - m_base) / (math.log(m_target) - math.log(m_base))
return [weight * (math.log(t / b) + fix) for b, t in zip(base, target)]
log_m_base = math.log(m_base)
log_m_target = math.log(m_target)
fix = ((log_m_target - sum(math.log(t) for t in target)) - (log_m_base - sum(math.log(b) for b in base))) / k
weight = m_base if m_base == m_target else (m_target - m_base) / (log_m_target - log_m_base)
return [weight * (math.log(t) - math.log(b) + fix) for b, t in zip(base, target)]
def factor(raw: Mapping[str, Any], env: Env) -> Any:
+8
View File
@@ -38,6 +38,7 @@ from app.analysis.inputs import Snap, as_list, hashable, same_value
from app.analysis.numbers import canon, to_fraction
TESTS = {"z": "z 检验", "fisher": "Fisher 精确检验", "welch": "Welch t 检验"}
MAX_ROWS_MODE = 1000
def used_settings(mode: str | None) -> list[str]:
@@ -214,6 +215,13 @@ def differ(raw: Mapping[str, Any], env: Env) -> Any:
env.inputs.use(snap, "data", [group_col, *cols.values()])
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行,各组不完整",
criterion=criterion)
if mode == "rows" and len(snap.rows) > MAX_ROWS_MODE:
raise CallError(
"too_many_rows",
f"{snap.ref} 有 {len(snap.rows)} 行,超过逐行明细模式 {MAX_ROWS_MODE} 行的上限:"
f"请在 SQL 里按 {group_col} 汇总出 SUM、SUM(x*x)、COUNT(*),改用 mean 模式",
reason=f"{snap.ref} 有 {len(snap.rows)} 行,超过逐行明细 {MAX_ROWS_MODE} 行的上限",
)
groups = _groups(snap, group_col, cols, mode, by_period, params) # type: ignore[arg-type]
pairs = _pairs(raw, params, groups, mode, dim_label, by_period, group_col) # type: ignore[arg-type]
env.inputs.use(snap, "data", [group_col, *cols.values()],
+6
View File
@@ -286,6 +286,12 @@ def plain_day(sql: str, column: str, dialect: str | None) -> bool:
if isinstance(value, (exp.Date, exp.TsOrDsToDate)):
extra = [v for k, v in value.args.items() if k != "this" and v not in (None, [], False)]
value = cf._unwrap(value.this) if not extra else None
elif isinstance(value, (exp.Cast, exp.TryCast)):
to_type = value.args.get("to")
if isinstance(to_type, exp.DataType) and to_type.this == exp.DataType.Type.DATE:
value = cf._unwrap(value.this)
else:
value = None
return isinstance(value, exp.Column) and not isinstance(value.this, exp.Star)
+26 -5
View File
@@ -341,12 +341,33 @@ def _inversion_counts(n: int) -> list[int]:
return counts
_MAX_SCALE_BITS = 1024
def _safe_float(value: Any) -> float:
try:
return float(value)
except OverflowError:
return math.inf if value > 0 else -math.inf
def _safe_ratio_float(num: int, den: int) -> float:
try:
return num / den
except OverflowError:
return math.inf if (num > 0) == (den > 0) else -math.inf
def _scaled(values: Sequence[Any]) -> tuple[list[int], int] | None:
"""一组精确的数(整数、Fraction)乘上公分母换成整数:(整数, 公分母)。有浮点的为 None。两两相减、比大小在整数上做,
比 Fraction 快得多,结果一样。"""
"""一组精确的数(整数、Fraction)乘上公分母换成整数:(整数, 公分母)。有浮点或公分母位宽过大的为 None。两两相减、比大小
在整数上做,比 Fraction 快得多,结果一样。"""
if not all(isinstance(v, (int, Fraction)) and not isinstance(v, bool) for v in values):
return None
scale = math.lcm(*(Fraction(v).denominator for v in values)) if values else 1
scale = 1
for v in values:
scale = math.lcm(scale, Fraction(v).denominator)
if scale.bit_length() > _MAX_SCALE_BITS:
return None
return [int(Fraction(v) * scale) for v in values], scale
@@ -392,7 +413,7 @@ def theil_sen(xs: Sequence[Any], ys: Sequence[Any]) -> Any:
pairs.append((num, den) if den > 0 else (-num, -den))
if not pairs:
return None
keys = [num / den for num, den in pairs]
keys = [_safe_ratio_float(num, den) for num, den in pairs]
order = sorted(range(len(pairs)), key=keys.__getitem__)
def kth(k: int) -> Fraction:
@@ -421,7 +442,7 @@ _FAST_MEDIAN = 512
def _kth(values: Sequence[Any], k: int) -> Any:
"""从小到大第 k 个(从 0 数),精确:Fraction 换成浮点是正确舍入的、单调的——浮点小的精确值一定小,排错的只可能是浮点
相等的那几个,所以先按浮点排,再只把和第 k 个浮点相等的那一段按精确值排。"""
keyed = sorted((float(v), i) for i, v in enumerate(values))
keyed = sorted((_safe_float(v), i) for i, v in enumerate(values))
pivot = keyed[k][0]
lo, hi = k, k + 1
while lo > 0 and keyed[lo - 1][0] == pivot: