mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-21 20:12:00 +00:00
全栈安全加固、性能提速与架构演进:连接池与沙箱止血、双路混合检索、多版本归档及前端分包治理
- 第一批(安全加固与生命周期止血):隔离测试宿主机密钥与 .env 泄漏;修复 SQLite 取消查询挂死、:memory: 连接池报错与 EngineCache 世代竞态;补齐运行与评测删除时的 Checkpointer 线程、Artifact 磁盘文件与无主附件回收;加固三套沙箱路径越界、进程组清理与 Seatbelt 数据目录隔离,封堵 SSRF IPv4-mapped IPv6 元数据绕过与响应体 OOM;修复推理模型多轮 reasoning_content、结构化输出 Token 漏计与缓存折扣计费 - 第二批(数据库/集群/问数提速与首屏分包):补齐核心表索引与 SQLite WAL 调优,列表接口按需 defer 大列,收敛启动扫描并打通跨进程 slot 槽位唤醒;优化跨源全局选表、哈希桶数字绑定、流式 CSV 导入及多方言分析算子;剥离首屏静态引入的 @xyflow/react 与全局样式(首屏主包降至 461.59 kB),补充流式 rAF 攒批、Store 精细订阅与表单无障碍绑定 - 第三批(架构演进与工程自动化闭环):记录库超挂接阈值历史版本自动增量归档至单库多表(解除 SQLite 10 库 ATTACH 硬上限);重构知识库为词面 BM25 与全集合归一化向量矩阵真·双路独立召回并支持游标分批重建;升级多方言感知 SQL 模板词法分析(E'...' / $$...$$ / q'[...]' / # / 嵌套注释)与 Decimal / bool / NaN 参数校验,消除 ledger 层对 api 路由层的反向依赖;拆分 3101 行 CaliberDrafts 页面为 7 个领域子模块,消除检查脚本本机路径硬编码并新增 GitHub Actions CI 门禁与三批回归测试套件
This commit is contained in:
@@ -76,31 +76,38 @@ async def put_json(
|
||||
幂等:同内容重复写只落一次盘;同一工件被多个 run 引用时,
|
||||
数据库里各记一行引用(溯源需要知道"谁产出的"),文件只有一份。
|
||||
"""
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from app.cluster.mutex import lock
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact
|
||||
|
||||
text = canonical_json(obj)
|
||||
artifact_id = content_hash(text)
|
||||
path = _path_of(artifact_id)
|
||||
if not path.exists():
|
||||
# 原子落盘(core/atomic):写一半崩了不会留下半个工件;临时文件名带随机后缀,两个进程同时写同一份内容
|
||||
# 各写各的,不会一个把另一个写到一半的截断(以前共用 <id>.tmp,后到的会把先到的半成品改名成正式文件)
|
||||
atomic.write_text(path, text)
|
||||
async with lock(f"artifact:{artifact_id[:2]}"):
|
||||
if not path.exists():
|
||||
# 原子落盘(core/atomic):写一半崩了不会留下半个工件;临时文件名带随机后缀,两个进程同时写同一份内容
|
||||
# 各写各的,不会一个把另一个写到一半的截断;且与同分桶的 forget 互斥,不会刚写下就被并发回收删掉
|
||||
atomic.write_text(path, text)
|
||||
|
||||
async with SessionLocal() as session:
|
||||
existing = await session.get(Artifact, (artifact_id, run_id or ""))
|
||||
if existing is None:
|
||||
session.add(
|
||||
Artifact(
|
||||
id=artifact_id,
|
||||
run_id=run_id or "",
|
||||
kind=kind,
|
||||
node_id=node_id,
|
||||
size=len(text.encode()),
|
||||
meta=meta or {},
|
||||
async with SessionLocal() as session:
|
||||
existing = await session.get(Artifact, (artifact_id, run_id or ""))
|
||||
if existing is None:
|
||||
session.add(
|
||||
Artifact(
|
||||
id=artifact_id,
|
||||
run_id=run_id or "",
|
||||
kind=kind,
|
||||
node_id=node_id,
|
||||
size=len(text.encode()),
|
||||
meta=meta or {},
|
||||
)
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
return artifact_id
|
||||
|
||||
|
||||
@@ -148,27 +155,35 @@ async def forget(ids: Any) -> list[str]:
|
||||
读到错的内容。"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.cluster.mutex import lock
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact
|
||||
|
||||
wanted = sorted({i for i in ids or () if isinstance(i, str) and i and i.isalnum()})
|
||||
if not wanted:
|
||||
return []
|
||||
async with SessionLocal() as session:
|
||||
kept: set[str] = set()
|
||||
for start in range(0, len(wanted), 500):
|
||||
chunk = wanted[start:start + 500]
|
||||
kept |= set((await session.execute(select(Artifact.id).where(Artifact.id.in_(chunk)))).scalars())
|
||||
removed: list[str] = []
|
||||
by_bucket: dict[str, list[str]] = {}
|
||||
for artifact_id in wanted:
|
||||
if artifact_id in kept:
|
||||
continue
|
||||
path = _path_of(artifact_id)
|
||||
try:
|
||||
path.unlink()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
removed.append(artifact_id)
|
||||
by_bucket.setdefault(artifact_id[:2], []).append(artifact_id)
|
||||
|
||||
removed: list[str] = []
|
||||
for bucket in sorted(by_bucket):
|
||||
bucket_ids = by_bucket[bucket]
|
||||
async with lock(f"artifact:{bucket}"):
|
||||
async with SessionLocal() as session:
|
||||
kept: set[str] = set()
|
||||
for start in range(0, len(bucket_ids), 500):
|
||||
chunk = bucket_ids[start:start + 500]
|
||||
kept |= set((await session.execute(select(Artifact.id).where(Artifact.id.in_(chunk)))).scalars())
|
||||
for artifact_id in bucket_ids:
|
||||
if artifact_id in kept:
|
||||
continue
|
||||
path = _path_of(artifact_id)
|
||||
try:
|
||||
path.unlink()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
removed.append(artifact_id)
|
||||
return removed
|
||||
|
||||
|
||||
|
||||
+20
-5
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
from collections import defaultdict
|
||||
from collections import OrderedDict, defaultdict
|
||||
from collections.abc import Callable
|
||||
from typing import AsyncIterator
|
||||
|
||||
@@ -11,6 +11,7 @@ from app.core.events import RunEventModel
|
||||
# 单个订阅者的积压上限。慢消费者(比如卡住的浏览器标签)不应该拖垮整个 run,
|
||||
# 所以队列满了就丢最旧的事件并打标记,而不是阻塞发布方。
|
||||
_QUEUE_MAXSIZE = 2048
|
||||
_MAX_SEQ_ENTRIES = 4096
|
||||
|
||||
|
||||
class _Subscriber:
|
||||
@@ -31,17 +32,30 @@ class EventBus:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._subs: dict[str, set[_Subscriber]] = defaultdict(set)
|
||||
self._seq: dict[str, int] = defaultdict(int)
|
||||
self._seq: OrderedDict[str, int] = OrderedDict()
|
||||
self._lock = asyncio.Lock()
|
||||
self.tap: Callable[[RunEventModel], None] | None = None
|
||||
|
||||
def _trim_seq(self) -> None:
|
||||
while len(self._seq) > _MAX_SEQ_ENTRIES:
|
||||
oldest, _ = next(iter(self._seq.items()))
|
||||
if self._subs.get(oldest):
|
||||
self._seq.move_to_end(oldest)
|
||||
break
|
||||
self._seq.popitem(last=False)
|
||||
|
||||
def next_seq(self, run_id: str) -> int:
|
||||
self._seq[run_id] += 1
|
||||
return self._seq[run_id]
|
||||
nxt = self._seq.get(run_id, 0) + 1
|
||||
self._seq[run_id] = nxt
|
||||
self._seq.move_to_end(run_id)
|
||||
self._trim_seq()
|
||||
return nxt
|
||||
|
||||
def set_seq(self, run_id: str, value: int) -> None:
|
||||
"""恢复一个已存在的 run 时,把序号接上,保证前端排序不乱。"""
|
||||
self._seq[run_id] = max(self._seq[run_id], value)
|
||||
self._seq[run_id] = max(self._seq.get(run_id, 0), value)
|
||||
self._seq.move_to_end(run_id)
|
||||
self._trim_seq()
|
||||
|
||||
async def publish(self, event: RunEventModel, *, relayed: bool = False) -> None:
|
||||
if not relayed and self.tap is not None:
|
||||
@@ -61,6 +75,7 @@ class EventBus:
|
||||
|
||||
async def close(self, run_id: str) -> None:
|
||||
"""给所有订阅者发结束哨兵,让 WebSocket 干净地收尾。"""
|
||||
self._seq.pop(run_id, None)
|
||||
for sub in list(self._subs.get(run_id, ())):
|
||||
with contextlib.suppress(asyncio.QueueFull):
|
||||
sub.queue.put_nowait(None)
|
||||
|
||||
@@ -361,7 +361,10 @@ class Settings(BaseSettings):
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
s = Settings()
|
||||
import os
|
||||
|
||||
no_dotenv = os.environ.get("AGENTLAB_DISABLE_DOTENV", "").strip().lower() in ("1", "true", "yes", "on")
|
||||
s = Settings(_env_file=None) if no_dotenv else Settings()
|
||||
s.ensure_dirs()
|
||||
return s
|
||||
|
||||
|
||||
Reference in New Issue
Block a user