全栈安全加固、性能提速与架构演进:连接池与沙箱止血、双路混合检索、多版本归档及前端分包治理

- 第一批(安全加固与生命周期止血):隔离测试宿主机密钥与 .env 泄漏;修复 SQLite 取消查询挂死、:memory: 连接池报错与 EngineCache 世代竞态;补齐运行与评测删除时的 Checkpointer 线程、Artifact 磁盘文件与无主附件回收;加固三套沙箱路径越界、进程组清理与 Seatbelt 数据目录隔离,封堵 SSRF IPv4-mapped IPv6 元数据绕过与响应体 OOM;修复推理模型多轮 reasoning_content、结构化输出 Token 漏计与缓存折扣计费
- 第二批(数据库/集群/问数提速与首屏分包):补齐核心表索引与 SQLite WAL 调优,列表接口按需 defer 大列,收敛启动扫描并打通跨进程 slot 槽位唤醒;优化跨源全局选表、哈希桶数字绑定、流式 CSV 导入及多方言分析算子;剥离首屏静态引入的 @xyflow/react 与全局样式(首屏主包降至 461.59 kB),补充流式 rAF 攒批、Store 精细订阅与表单无障碍绑定
- 第三批(架构演进与工程自动化闭环):记录库超挂接阈值历史版本自动增量归档至单库多表(解除 SQLite 10 库 ATTACH 硬上限);重构知识库为词面 BM25 与全集合归一化向量矩阵真·双路独立召回并支持游标分批重建;升级多方言感知 SQL 模板词法分析(E'...' / $$...$$ / q'[...]' / # / 嵌套注释)与 Decimal / bool / NaN 参数校验,消除 ledger 层对 api 路由层的反向依赖;拆分 3101 行 CaliberDrafts 页面为 7 个领域子模块,消除检查脚本本机路径硬编码并新增 GitHub Actions CI 门禁与三批回归测试套件
This commit is contained in:
ThearchyHelios
2026-11-13 14:51:36 +08:00
parent 09949fc8ac
commit eddeded2a4
106 changed files with 7178 additions and 3913 deletions
+159 -24
View File
@@ -8496,47 +8496,182 @@ def _slot_of(expr: str, index: int, context: str) -> TemplateSlot:
return TemplateSlot(index=index, field=found.group(1), filters=filters, context=context)
def _template_slots(template: str) -> tuple[list[tuple[int, int, TemplateSlot]], str]:
"""模板里每一处 {{ … }}:[(起, 止, 槽位)],以及按哨兵渲染的 SQL。逐字看引号和注释,定下每一处在哪:单引号里是字,
双引号、反引号里是名字,-- 和 /* */ 里是注释,别处是裸写。{{ … }} 里面的字不参与判断。"""
_DOLLAR_QUOTE_RE = re.compile(r"\$(?:[^\W\d]\w*)?\$")
_ORACLE_Q_CLOSE = {"[": "]", "(": ")", "{": "}", "<": ">"}
def _is_word_char(ch: str) -> bool:
return ch.isalnum() or ch in "_$"
def _is_e_string_at(sql: str, i: int) -> bool:
return i >= 1 and sql[i - 1] in "eE" and not (i >= 2 and _is_word_char(sql[i - 2]))
def _oracle_q_close_at(sql: str, i: int) -> str | None:
if i + 1 >= len(sql) or sql[i + 1].isspace() or not (i >= 1 and sql[i - 1] in "qQ"):
return None
j = i - 2
if j >= 0 and sql[j] in "nN":
j -= 1
if j >= 0 and _is_word_char(sql[j]):
return None
opener = sql[i + 1]
return _ORACLE_Q_CLOSE.get(opener, opener)
def _is_bracket_ident_at(sql: str, i: int, dialect_key: str) -> bool:
if dialect_key == "sqlite":
return True
if dialect_key in ("postgres", "postgresql", "mysql", "mariadb", "oracle"):
return False
j = i - 1
while j >= 0 and sql[j].isspace():
j -= 1
return j < 0 or not (_is_word_char(sql[j]) or sql[j] in ")]")
def _template_slots(
template: str, *, dialect: str | None = None,
) -> tuple[list[tuple[int, int, TemplateSlot]], str]:
"""模板里每一处 {{ … }}:[(起, 止, 槽位)],以及按哨兵渲染的 SQL。
按方言词法逐字扫描引号与注释,定下每一处在哪:
- 单引号(含 PostgreSQL E'…'、MySQL 反斜杠转义、PostgreSQL $$…$$ / $tag$…$tag$、Oracle q'[…]')里是字(quoted);
- 双引号、反引号、SQLite 方括号 `[…]` 里是名字(name;MySQL 默认模式下双引号是字符串);
- `--`、`/* … */`(PostgreSQL 支持嵌套)与 MySQL `#` 里是注释(comment);
- 其余代码位置是裸写(bare)。`{{ … }}` 里面的字不参与外层引号/注释状态切换。
"""
dialect_key = (dialect or "").strip().lower()
is_mysql = dialect_key in ("mysql", "mariadb")
is_pg = dialect_key in ("postgres", "postgresql")
allow_dollar = dialect_key in ("", "postgres", "postgresql")
allow_q_quote = dialect_key in ("", "oracle")
spans: list[tuple[int, int, TemplateSlot]] = []
out: list[str] = []
state = "code"
quote = ""
backslash_escape = False
dollar_tag = ""
q_close = ""
block_depth = 0
i, n = 0, len(template)
while i < n:
if template.startswith("{{", i):
m = _SLOT_RE.match(template, i)
if m is not None:
context = {"code": "bare", "string": "quoted", "name": "name", "line": "comment",
"block": "comment"}[state]
context = {
"code": "bare",
"string": "quoted",
"dollar": "quoted",
"q_string": "quoted",
"name": "name",
"bracket_name": "name",
"line": "comment",
"block": "comment",
}[state]
slot = _slot_of(m.group(1).strip(), len(spans), context)
spans.append((m.start(), m.end(), slot))
out.append(_STRING_SENTINEL.format(slot.index) if context == "quoted"
else str(_NUMBER_SENTINEL + slot.index) if context == "bare" else "")
out.append(
_STRING_SENTINEL.format(slot.index)
if context == "quoted"
else str(_NUMBER_SENTINEL + slot.index)
if context == "bare"
else ""
)
i = m.end()
continue
ch = template[i]
if state == "code":
if ch == "'":
q_closer = _oracle_q_close_at(template, i) if allow_q_quote else None
if q_closer is not None:
state = "q_string"
q_close = q_closer
out.append(template[i:i + 2])
i += 2
continue
state = "string"
elif ch in "\"`":
quote = "'"
backslash_escape = is_mysql or _is_e_string_at(template, i)
elif ch == '"' and is_mysql:
state = "string"
quote = '"'
backslash_escape = True
elif ch in '"`':
state = "name"
quote = ch
elif ch == "[" and _is_bracket_ident_at(template, i, dialect_key):
state = "bracket_name"
elif allow_dollar and ch == "$" and (dm := _DOLLAR_QUOTE_RE.match(template, i)) is not None:
state = "dollar"
dollar_tag = dm.group(0)
out.append(dollar_tag)
i = dm.end()
continue
elif template.startswith("--", i):
state = "line"
out.append("--")
i += 2
continue
elif is_mysql and ch == "#":
state = "line"
elif template.startswith("/*", i):
state = "block"
elif state == "string" and ch == "'":
state = "code" # 成对的 '' 是转义:关上又马上打开,结果一样
elif state == "name" and ch == quote:
state = "code"
block_depth = 1
out.append("/*")
i += 2
continue
elif state == "string":
if backslash_escape and ch == "\\" and i + 1 < n:
out.append(template[i:i + 2])
i += 2
continue
if ch == quote:
if i + 1 < n and template[i + 1] == quote:
out.append(quote * 2)
i += 2
continue
state = "code"
elif state == "dollar":
if template.startswith(dollar_tag, i):
out.append(dollar_tag)
i += len(dollar_tag)
state = "code"
continue
elif state == "q_string":
if ch == q_close and i + 1 < n and template[i + 1] == "'":
out.append(template[i:i + 2])
i += 2
state = "code"
continue
elif state == "name":
if ch == quote:
if i + 1 < n and template[i + 1] == quote:
out.append(quote * 2)
i += 2
continue
state = "code"
elif state == "bracket_name":
if ch == "]":
state = "code"
elif state == "line" and ch == "\n":
state = "code"
elif state == "block" and template.startswith("*/", i):
out.append("*/")
i += 2
state = "code"
continue
elif state == "block":
if is_pg and template.startswith("/*", i):
block_depth += 1
out.append("/*")
i += 2
continue
if template.startswith("*/", i):
out.append("*/")
i += 2
block_depth -= 1
if block_depth <= 0:
state = "code"
continue
out.append(ch)
i += 1
return spans, "".join(out)
@@ -8734,7 +8869,7 @@ def _bindings(template_sql: str, sql: str, read: str | None, info: _Schema
) -> Iterator[tuple[str, dict[str, Any], tuple[str, ...], tuple[TemplateSlot, ...]]]:
"""按 BINDING_LEVELS 的先后给出配得上的绑定:(哪一级, {字段: 值}, 没取到值的字段, 槽位),同样的绑定只给一次。
模板没有槽位时只给一个空绑定(level 为 parsed:用不着配)。不可比抛 _NotComparable。"""
spans, sentinel_sql = _template_slots(template_sql)
spans, sentinel_sql = _template_slots(template_sql, dialect=read)
slots = tuple(slot for _, _, slot in spans)
fields = tuple(dict.fromkeys(s.field for s in slots))
if not slots:
@@ -8839,7 +8974,7 @@ def bind_template(template_sql: str, slots: Any, sql: str, dialect: str | None =
try:
for level, values, unbound, found in _bindings(template_sql or "", sql or "", read, info):
return TemplateBinding(True, values, unbound, level, None, found)
spans, _ = _template_slots(template_sql or "")
spans, _ = _template_slots(template_sql or "", dialect=read)
except _NotComparable as e:
return TemplateBinding(False, reason=str(e))
except Exception: # noqa: BLE001 - 绑定本身的缺陷不能连累调用方:当作没配上
@@ -8924,7 +9059,7 @@ def match_template(template_sql: str, slots: Any, sql: str, dialect: str | None
level, values, unbound, diff, rendered = tried[0]
return TemplateMatch(False, binding=values, unbound=unbound, level=level, diff=diff, rendered=rendered)
try:
spans, _ = _template_slots(template_sql)
spans, _ = _template_slots(template_sql, dialect=read)
except _NotComparable as e:
return TemplateMatch(False, comparable=False, reason=str(e))
fields = tuple(dict.fromkeys(slot.field for _, _, slot in spans))
@@ -8954,7 +9089,7 @@ def _now_slots(template_sql: str, sql: str, read: str | None) -> list[tuple[str,
from sqlglot import exp
try:
spans, sentinel = _template_slots(template_sql)
spans, sentinel = _template_slots(template_sql, dialect=read)
except _NotComparable:
return []
by_index = {slot.index: slot for _, _, slot in spans}
@@ -9344,7 +9479,7 @@ def project_template(template_sql: str, keep: Sequence[int], dialect: str | None
template = (template_sql or "").strip()
keep = tuple(keep)
try:
slots, sentinel = _template_slots(template)
slots, sentinel = _template_slots(template, dialect=read)
root, spans, expected = _plan_projection(sentinel, keep, read, None)
items = list(root.expressions)
names = tuple(_output_name(items[k], read) for k in keep)
@@ -9358,7 +9493,7 @@ def project_template(template_sql: str, keep: Sequence[int], dialect: str | None
raise _NotProjectable(_PROJ_TEXT)
out = _cut(template, mapped, keep) # type: ignore[arg-type]
# 换回模板之后再按哨兵渲染一遍:结构(字面量只看位置,槽位的编号会因为去掉了几处而变)要和投影后的一样
_, again = _template_slots(out)
_, again = _template_slots(out, dialect=read)
root_again, _ = _parse(again, read)
lits_a: list[Any] = []
lits_b: list[Any] = []
@@ -9420,7 +9555,7 @@ def output_names(sql: str, dialect: str | None = None, *, template: bool = False
text = (sql or "").strip()
try:
if template:
_, text = _template_slots(text)
_, text = _template_slots(text, dialect=read)
root, _ = _parse(_mysql_dashes(text) if read == "mysql" else text, read)
except _NotComparable:
return None