mirror of
https://github.com/ThearchyHelios/AgentLab.git
synced 2026-11-18 18:02:04 +00:00
全栈安全加固、性能提速与架构演进:连接池与沙箱止血、双路混合检索、多版本归档及前端分包治理
- 第一批(安全加固与生命周期止血):隔离测试宿主机密钥与 .env 泄漏;修复 SQLite 取消查询挂死、:memory: 连接池报错与 EngineCache 世代竞态;补齐运行与评测删除时的 Checkpointer 线程、Artifact 磁盘文件与无主附件回收;加固三套沙箱路径越界、进程组清理与 Seatbelt 数据目录隔离,封堵 SSRF IPv4-mapped IPv6 元数据绕过与响应体 OOM;修复推理模型多轮 reasoning_content、结构化输出 Token 漏计与缓存折扣计费 - 第二批(数据库/集群/问数提速与首屏分包):补齐核心表索引与 SQLite WAL 调优,列表接口按需 defer 大列,收敛启动扫描并打通跨进程 slot 槽位唤醒;优化跨源全局选表、哈希桶数字绑定、流式 CSV 导入及多方言分析算子;剥离首屏静态引入的 @xyflow/react 与全局样式(首屏主包降至 461.59 kB),补充流式 rAF 攒批、Store 精细订阅与表单无障碍绑定 - 第三批(架构演进与工程自动化闭环):记录库超挂接阈值历史版本自动增量归档至单库多表(解除 SQLite 10 库 ATTACH 硬上限);重构知识库为词面 BM25 与全集合归一化向量矩阵真·双路独立召回并支持游标分批重建;升级多方言感知 SQL 模板词法分析(E'...' / $$...$$ / q'[...]' / # / 嵌套注释)与 Decimal / bool / NaN 参数校验,消除 ledger 层对 api 路由层的反向依赖;拆分 3101 行 CaliberDrafts 页面为 7 个领域子模块,消除检查脚本本机路径硬编码并新增 GitHub Actions CI 门禁与三批回归测试套件
This commit is contained in:
@@ -41,7 +41,7 @@ ParentResolver = Callable[[str], AnalysisResult | None]
|
||||
#: (row_out_of_range、headline_empty);要比的组这期没有(unknown_group、too_few_groups);要用的那一格这期为空
|
||||
#: (empty_cell:乘积拆解的乘积、因子,差异检验的计数)
|
||||
DATA_ERRORS = frozenset({"parent_refused", "parent_key_missing", "period_not_found", "row_out_of_range",
|
||||
"headline_empty", "unknown_group", "too_few_groups", "empty_cell"})
|
||||
"headline_empty", "unknown_group", "too_few_groups", "empty_cell", "too_many_rows"})
|
||||
|
||||
|
||||
def _op_mode(request: Mapping[str, Any]) -> tuple[str, str | None]:
|
||||
|
||||
@@ -724,11 +724,13 @@ def _half_ulp_ratio(raw: Any, value: Fraction) -> Fraction:
|
||||
def lmdi(base: Sequence[float], target: Sequence[float], m_base: float, m_target: float) -> list[float]:
|
||||
"""LMDI-I:contrib_k = L(M', M) · ln(F_k' / F_k),L(a, b) = (a − b) / (ln a − ln b),L(a, a) = a。因子按书写精度取过整、
|
||||
乘积和乘积列差一点时,各因子按同一比例缩放到乘积恰好等于乘积列(每个因子的对数比各补上 (1/K)·差的对数),Σcontrib 在
|
||||
浮点误差内等于 M' − M。"""
|
||||
浮点误差内等于 M' − M。全程在对数域求和,避免多因子连乘 float64 上溢或下溢归零。"""
|
||||
k = len(base)
|
||||
fix = (math.log(m_target / math.prod(target)) - math.log(m_base / math.prod(base))) / k
|
||||
weight = m_base if m_base == m_target else (m_target - m_base) / (math.log(m_target) - math.log(m_base))
|
||||
return [weight * (math.log(t / b) + fix) for b, t in zip(base, target)]
|
||||
log_m_base = math.log(m_base)
|
||||
log_m_target = math.log(m_target)
|
||||
fix = ((log_m_target - sum(math.log(t) for t in target)) - (log_m_base - sum(math.log(b) for b in base))) / k
|
||||
weight = m_base if m_base == m_target else (m_target - m_base) / (log_m_target - log_m_base)
|
||||
return [weight * (math.log(t) - math.log(b) + fix) for b, t in zip(base, target)]
|
||||
|
||||
|
||||
def factor(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
|
||||
@@ -38,6 +38,7 @@ from app.analysis.inputs import Snap, as_list, hashable, same_value
|
||||
from app.analysis.numbers import canon, to_fraction
|
||||
|
||||
TESTS = {"z": "z 检验", "fisher": "Fisher 精确检验", "welch": "Welch t 检验"}
|
||||
MAX_ROWS_MODE = 1000
|
||||
|
||||
|
||||
def used_settings(mode: str | None) -> list[str]:
|
||||
@@ -214,6 +215,13 @@ def differ(raw: Mapping[str, Any], env: Env) -> Any:
|
||||
env.inputs.use(snap, "data", [group_col, *cols.values()])
|
||||
return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行,各组不完整",
|
||||
criterion=criterion)
|
||||
if mode == "rows" and len(snap.rows) > MAX_ROWS_MODE:
|
||||
raise CallError(
|
||||
"too_many_rows",
|
||||
f"{snap.ref} 有 {len(snap.rows)} 行,超过逐行明细模式 {MAX_ROWS_MODE} 行的上限:"
|
||||
f"请在 SQL 里按 {group_col} 汇总出 SUM、SUM(x*x)、COUNT(*),改用 mean 模式",
|
||||
reason=f"{snap.ref} 有 {len(snap.rows)} 行,超过逐行明细 {MAX_ROWS_MODE} 行的上限",
|
||||
)
|
||||
groups = _groups(snap, group_col, cols, mode, by_period, params) # type: ignore[arg-type]
|
||||
pairs = _pairs(raw, params, groups, mode, dim_label, by_period, group_col) # type: ignore[arg-type]
|
||||
env.inputs.use(snap, "data", [group_col, *cols.values()],
|
||||
|
||||
@@ -286,6 +286,12 @@ def plain_day(sql: str, column: str, dialect: str | None) -> bool:
|
||||
if isinstance(value, (exp.Date, exp.TsOrDsToDate)):
|
||||
extra = [v for k, v in value.args.items() if k != "this" and v not in (None, [], False)]
|
||||
value = cf._unwrap(value.this) if not extra else None
|
||||
elif isinstance(value, (exp.Cast, exp.TryCast)):
|
||||
to_type = value.args.get("to")
|
||||
if isinstance(to_type, exp.DataType) and to_type.this == exp.DataType.Type.DATE:
|
||||
value = cf._unwrap(value.this)
|
||||
else:
|
||||
value = None
|
||||
return isinstance(value, exp.Column) and not isinstance(value.this, exp.Star)
|
||||
|
||||
|
||||
|
||||
@@ -341,12 +341,33 @@ def _inversion_counts(n: int) -> list[int]:
|
||||
return counts
|
||||
|
||||
|
||||
_MAX_SCALE_BITS = 1024
|
||||
|
||||
|
||||
def _safe_float(value: Any) -> float:
|
||||
try:
|
||||
return float(value)
|
||||
except OverflowError:
|
||||
return math.inf if value > 0 else -math.inf
|
||||
|
||||
|
||||
def _safe_ratio_float(num: int, den: int) -> float:
|
||||
try:
|
||||
return num / den
|
||||
except OverflowError:
|
||||
return math.inf if (num > 0) == (den > 0) else -math.inf
|
||||
|
||||
|
||||
def _scaled(values: Sequence[Any]) -> tuple[list[int], int] | None:
|
||||
"""一组精确的数(整数、Fraction)乘上公分母换成整数:(整数, 公分母)。有浮点的为 None。两两相减、比大小在整数上做,
|
||||
比 Fraction 快得多,结果一样。"""
|
||||
"""一组精确的数(整数、Fraction)乘上公分母换成整数:(整数, 公分母)。有浮点或公分母位宽过大的为 None。两两相减、比大小
|
||||
在整数上做,比 Fraction 快得多,结果一样。"""
|
||||
if not all(isinstance(v, (int, Fraction)) and not isinstance(v, bool) for v in values):
|
||||
return None
|
||||
scale = math.lcm(*(Fraction(v).denominator for v in values)) if values else 1
|
||||
scale = 1
|
||||
for v in values:
|
||||
scale = math.lcm(scale, Fraction(v).denominator)
|
||||
if scale.bit_length() > _MAX_SCALE_BITS:
|
||||
return None
|
||||
return [int(Fraction(v) * scale) for v in values], scale
|
||||
|
||||
|
||||
@@ -392,7 +413,7 @@ def theil_sen(xs: Sequence[Any], ys: Sequence[Any]) -> Any:
|
||||
pairs.append((num, den) if den > 0 else (-num, -den))
|
||||
if not pairs:
|
||||
return None
|
||||
keys = [num / den for num, den in pairs]
|
||||
keys = [_safe_ratio_float(num, den) for num, den in pairs]
|
||||
order = sorted(range(len(pairs)), key=keys.__getitem__)
|
||||
|
||||
def kth(k: int) -> Fraction:
|
||||
@@ -421,7 +442,7 @@ _FAST_MEDIAN = 512
|
||||
def _kth(values: Sequence[Any], k: int) -> Any:
|
||||
"""从小到大第 k 个(从 0 数),精确:Fraction 换成浮点是正确舍入的、单调的——浮点小的精确值一定小,排错的只可能是浮点
|
||||
相等的那几个,所以先按浮点排,再只把和第 k 个浮点相等的那一段按精确值排。"""
|
||||
keyed = sorted((float(v), i) for i, v in enumerate(values))
|
||||
keyed = sorted((_safe_float(v), i) for i, v in enumerate(values))
|
||||
pivot = keyed[k][0]
|
||||
lo, hi = k, k + 1
|
||||
while lo > 0 and keyed[lo - 1][0] == pivot:
|
||||
|
||||
+17
-101
@@ -149,8 +149,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
SCHEMA = "agentlab.evidence/1"
|
||||
|
||||
RUN_NOT_FOUND = "run_not_found"
|
||||
REPORT_NOT_FOUND = "evidence_report_not_found"
|
||||
SEGMENT_NOT_FOUND = "evidence_segment_not_found"
|
||||
DOC_MISSING = "evidence_doc_missing"
|
||||
DOC_TAMPERED = "evidence_doc_tampered"
|
||||
@@ -159,17 +157,26 @@ LEGACY_NOTE = "旧版出具:按数值匹配,不是显式引用。同一个
|
||||
NONE_NOTE = "本次运行没有报告文档,也没有出具契约,没有可展示的证据"
|
||||
NO_DOC_NOTE = "本次运行的出具契约采用引用模式核对,但报告没有产出可核对的文档"
|
||||
|
||||
from app.engine.evidence_reader import Sealed as _Sealed, metric_cards as _sealed_cards, read_sealed
|
||||
from app.engine.evidence_reader import (
|
||||
MASKED,
|
||||
REPORT_NOT_FOUND,
|
||||
RUN_NOT_FOUND,
|
||||
Report as _Report,
|
||||
Sealed as _Sealed,
|
||||
ShownText as _Shown,
|
||||
choose_report as _choose,
|
||||
load_sealed,
|
||||
metric_cards as _sealed_cards,
|
||||
output_evidence as _output_evidence,
|
||||
read_sealed,
|
||||
reports_of as _reports,
|
||||
)
|
||||
|
||||
|
||||
async def _sealed(run_id: str) -> _Sealed:
|
||||
try:
|
||||
return await read_sealed(run_id)
|
||||
except LookupError:
|
||||
raise CodedHTTPException(404, "运行记录不存在", RUN_NOT_FOUND) from None
|
||||
return await load_sealed(run_id)
|
||||
|
||||
|
||||
MASKED = "已遮罩"
|
||||
#: 面板底下的遮罩说明。遮的列可能是遮罩列本身,也可能是 SQL 里给它起了别名、套了函数得来的结果列(c.phone AS tel)
|
||||
MASK_NOTE = "这些列取自数据源中设置了遮罩的列,面板上不显示原值。遮罩只减少暴露,不是安全边界"
|
||||
#: 遮没遮核对不了、不显示的值(指标步骤的值、推演的原值):和「已遮罩」分开,读的人不该以为这里设了遮罩
|
||||
@@ -198,66 +205,10 @@ MISSING_RETRIEVAL = "无法读取检索快照,或其中没有这条命中"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 报告文档
|
||||
# 报告文档(Report / reports_of / output_evidence 定义在 app/engine/evidence_reader.py)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Report:
|
||||
node_id: str
|
||||
doc_artifact: str
|
||||
ok: bool | None
|
||||
repairs: int | None
|
||||
stats: dict[str, Any] | None
|
||||
doc: dict[str, Any] | None
|
||||
#: True 取回并复验通过;False 内容和哈希对不上(或者不是这次运行这个节点写的);None 文件不在了
|
||||
hash_ok: bool | None
|
||||
fields: list[str]
|
||||
#: 写这份报告时的结论句策略(report.checked 里记的,升级前的运行没有)
|
||||
claims: str | None = None
|
||||
|
||||
|
||||
def _reports(sealed: _Sealed, *, nested: bool = False) -> list[_Report]:
|
||||
"""封存范围内每个报告节点最后一次的自查结果,按第一次出现的顺序。
|
||||
|
||||
默认只认这次运行的图上的报告节点:子工作流里的报告(节点 id「子工作流节点 id/…」,子图的事件和父运行在同一个流里)
|
||||
是子工作流那一步内部的东西,父图的答案是子工作流出口的值,不是那份报告本身——证据图、取答案的评测、按成果挑报告
|
||||
(_choose)都按父图自己的认。nested 为真时连子工作流里的一起给(按 ?report=sub/write 点开它们)。可验证报告另按全部
|
||||
report.checked 核对(bundle/build._report_events:自查过的文档一份都不能少)。"""
|
||||
latest: dict[str, dict[str, Any]] = {}
|
||||
for _, etype, node_id, data in sealed.events:
|
||||
if etype == "report.checked" and node_id and isinstance(data, dict) and data.get("doc_artifact") \
|
||||
and (nested or "/" not in node_id):
|
||||
latest[node_id] = data
|
||||
evidence = _output_evidence(sealed)
|
||||
out = []
|
||||
for node_id, data in latest.items():
|
||||
artifact = str(data["doc_artifact"])
|
||||
try:
|
||||
doc = load(artifact)
|
||||
hash_ok: bool | None = True if doc is not None else None
|
||||
except ValueError:
|
||||
doc, hash_ok = None, False
|
||||
# 文档要真是这次运行里这个节点写的:内容寻址只保证没被改,不保证没被张冠李戴
|
||||
if isinstance(doc, dict) and (doc.get("node_id") != node_id or doc.get("run_id") != sealed.run_id):
|
||||
doc, hash_ok = None, False
|
||||
fields = [f for r in evidence if r.get("report_node") == node_id and r.get("doc_artifact") == artifact
|
||||
for f in r.get("fields") or []]
|
||||
out.append(_Report(node_id=node_id, doc_artifact=artifact, ok=data.get("ok"), repairs=data.get("repairs"),
|
||||
stats=(doc or {}).get("stats") or data.get("stats"),
|
||||
doc=doc if isinstance(doc, dict) else None, hash_ok=hash_ok, fields=fields,
|
||||
claims=data.get("claims") if isinstance(data.get("claims"), str) else None))
|
||||
return out
|
||||
|
||||
|
||||
def _output_evidence(sealed: _Sealed) -> list[dict[str, Any]]:
|
||||
output = sealed.payload_of(sealed.nodes_of("output")) or {}
|
||||
primary = output.get("_evidence")
|
||||
if not isinstance(primary, dict):
|
||||
return []
|
||||
return [primary, *[r for r in primary.get("others") or [] if isinstance(r, dict)]]
|
||||
|
||||
|
||||
def _input_payloads(sealed: _Sealed) -> dict[str, Any]:
|
||||
return sealed.payload_of(sealed.nodes_of("input")) or {}
|
||||
|
||||
@@ -830,16 +781,6 @@ def _named_reports(sealed: _Sealed, wanted: str | None) -> list[_Report]:
|
||||
return _reports(sealed, nested=bool(wanted) and "/" in str(wanted))
|
||||
|
||||
|
||||
def _choose(reports: list[_Report], wanted: str | None, sealed: _Sealed) -> _Report:
|
||||
if wanted:
|
||||
chosen = next((r for r in reports if r.node_id == wanted), None)
|
||||
if chosen is None:
|
||||
raise CodedHTTPException(404, "本次运行中没有所选报告撰写节点的文档", REPORT_NOT_FOUND)
|
||||
return chosen
|
||||
primary = next(iter(_output_evidence(sealed)), {})
|
||||
return next((r for r in reports if r.node_id == primary.get("report_node")), reports[0])
|
||||
|
||||
|
||||
async def _chain(seg: dict[str, Any], doc: dict[str, Any], sealed: _Sealed,
|
||||
report_node: str) -> list[dict[str, Any]]:
|
||||
cite = seg.get("cite") or {}
|
||||
@@ -2977,32 +2918,7 @@ def _no_data_view(snap: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if isinstance(item, dict) and isinstance(item.get("message"), str)]
|
||||
|
||||
|
||||
class _Shown:
|
||||
"""查询步骤上要展示的 SQL、合并 SQL、SQL 检查的片段、合并警告:里面写着的遮罩值(secrets,这次运行查到过的,
|
||||
masking._Secrets)换成「已遮罩」,换了写法的长号码也认。secrets 为 None 时原样交回。changed:换过东西,查询步骤
|
||||
底下要说清这只是展示时遮的。"""
|
||||
|
||||
def __init__(self, secrets: Any) -> None:
|
||||
self._secrets = secrets
|
||||
self.changed = False
|
||||
|
||||
def text(self, value: Any) -> Any:
|
||||
if self._secrets is None or not isinstance(value, str) or not value:
|
||||
return value
|
||||
out = self._secrets.scrub(value, loose=True)
|
||||
self.changed = self.changed or out != value
|
||||
return out
|
||||
|
||||
def checks(self, checks: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [{**c, **{k: self.text(c[k]) for k in ("message", "sql_excerpt") if isinstance(c.get(k), str)}}
|
||||
for c in checks]
|
||||
|
||||
def merge(self, view: dict[str, Any]) -> dict[str, Any]:
|
||||
view["sql"] = self.text(view.get("sql"))
|
||||
view["warnings"] = [{**w, "message": self.text(w.get("message"))} for w in view.get("warnings") or []]
|
||||
view["inputs"] = [{**i, **({"checks": self.checks(i["checks"])} if i.get("checks") else {})}
|
||||
for i in view.get("inputs") or []]
|
||||
return view
|
||||
# _Shown 定义在 app.engine.evidence_reader.ShownText 并在上方导入为 _Shown
|
||||
|
||||
|
||||
def _has_values(snap: dict[str, Any]) -> bool:
|
||||
|
||||
@@ -9,6 +9,7 @@ from fastapi import (
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import defer
|
||||
|
||||
from app.core.errors import explain, raw
|
||||
from app.core.config import settings
|
||||
@@ -221,7 +222,12 @@ async def list_documents(
|
||||
collection: str | None = None,
|
||||
session: AsyncSession = Depends(get_session),
|
||||
) -> list[Document]:
|
||||
stmt = select(Document).order_by(Document.created_at.desc(), Document.id.desc()).limit(300)
|
||||
stmt = (
|
||||
select(Document)
|
||||
.options(defer(Document.content))
|
||||
.order_by(Document.created_at.desc(), Document.id.desc())
|
||||
.limit(300)
|
||||
)
|
||||
if collection:
|
||||
stmt = stmt.where(Document.collection == collection)
|
||||
return list((await session.execute(stmt)).scalars())
|
||||
|
||||
@@ -1387,11 +1387,9 @@ async def _draft_validation_summary(session: AsyncSession, draft: CaliberDraft,
|
||||
|
||||
def _graph_entry_fields(graph: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""图里入口节点(input)的字段;草稿页和试跑用它,画布改过的入口字段不会被 payload 的旧快照盖住。"""
|
||||
from app.apps.tree import node_map
|
||||
from app.ledger.fieldbaseline import graph_entry_fields
|
||||
|
||||
start = next((node for node in node_map(graph).values() if node.get("type") == "input"), None)
|
||||
return [dict(field) for field in ((start or {}).get("config") or {}).get("fields") or []
|
||||
if isinstance(field, dict)]
|
||||
return graph_entry_fields(graph)
|
||||
|
||||
|
||||
def _allowed_tables(source: DataSource | None) -> set[str]:
|
||||
@@ -1503,54 +1501,19 @@ async def _draft_graph_payload(session: AsyncSession, workflow: Workflow, versio
|
||||
|
||||
card_id 是这次要读的那张口径卡(写入时由 target.card_id 定);不给就按这张草稿的权威卡(draft_card)。
|
||||
"""
|
||||
from app.apps.tree import node_map
|
||||
from app.ledger.fieldbaseline import project_graph
|
||||
|
||||
row = (await session.execute(select(WorkflowVersion).where(
|
||||
WorkflowVersion.workflow_id == workflow.id,
|
||||
WorkflowVersion.version == int(version)))).scalar_one_or_none()
|
||||
if row is None or not isinstance(row.graph, dict):
|
||||
return None
|
||||
nodes = node_map(row.graph)
|
||||
wanted, cards = draft_card(row.graph)
|
||||
# 读的时候认不出唯一的权威卡(多张、契约也没说是哪张)就照旧显示第一张:显示不改任何东西,写入那条路上会拒
|
||||
wanted = card_id or wanted or (cards[0]["id"] if cards else None)
|
||||
card = nodes.get(wanted) if wanted else None
|
||||
metrics = [metric for metric in ((card or {}).get("config") or {}).get("metrics") or []
|
||||
if isinstance(metric, dict) and metric.get("id")]
|
||||
fields = _graph_entry_fields(row.graph)
|
||||
fetches: list[tuple[str, str]] = []
|
||||
for node_id, node in nodes.items():
|
||||
config = node.get("config") if isinstance(node.get("config"), dict) else {}
|
||||
args = config.get("args") if isinstance(config.get("args"), dict) else {}
|
||||
sql = args.get("sql") if isinstance(args, dict) else None
|
||||
if node.get("type") == "tool" and isinstance(sql, str) and sql.strip():
|
||||
fetches.append((str(node_id), sql))
|
||||
groups: list[dict[str, Any]] = []
|
||||
for node_id, sql in fetches:
|
||||
pattern = re.compile(rf"nodes\.{re.escape(node_id)}(?![\w])")
|
||||
from app.look.calibers import _metric_paths
|
||||
|
||||
ids = [str(metric["id"]) for metric in metrics
|
||||
if any(pattern.search(path) for path in _metric_paths(metric) or [])]
|
||||
groups.append({"sql": sql, "entry_fields": [dict(field) for field in fields], "metric_ids": ids})
|
||||
assigned = {metric_id for group in groups for metric_id in group["metric_ids"]}
|
||||
loose = [str(metric["id"]) for metric in metrics if str(metric["id"]) not in assigned]
|
||||
if loose:
|
||||
if groups:
|
||||
groups[0]["metric_ids"].extend(loose)
|
||||
else:
|
||||
groups = [{"sql": "", "entry_fields": [dict(field) for field in fields], "metric_ids": loose}]
|
||||
out = {
|
||||
"version": version,
|
||||
"metrics": metrics,
|
||||
"entry_fields": [dict(field) for field in fields],
|
||||
"sql": "\n\n".join(sql for _node, sql in fetches),
|
||||
"groups": groups,
|
||||
"blocks": len(fetches) or 1,
|
||||
}
|
||||
out = project_graph(row.graph, version, card_id=card_id)
|
||||
if draft is not None and payload is not None:
|
||||
out["trial"] = await _trial_view(session, payload.get("trial") or {}, version=version, graph=row.graph)
|
||||
out["checks"] = await _draft_checks(session, draft, {**payload, "trial": out["trial"]}, row.graph, metrics, groups)
|
||||
out["checks"] = await _draft_checks(
|
||||
session, draft, {**payload, "trial": out["trial"]}, row.graph, out["metrics"], out["groups"],
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@@ -1577,46 +1540,7 @@ CHANGE_UNAVAILABLE = "change_unavailable"
|
||||
DRAFT_CARD_GONE = "这份草稿的口径卡不见了(可能刚被删掉):请刷新页面再看一次"
|
||||
|
||||
|
||||
def editable_cards(graph: dict[str, Any]) -> list[dict[str, str]]:
|
||||
"""图里能写的一张张口径卡:[{id, title}],按节点出现的顺序。
|
||||
|
||||
能写的是源头口径卡(metrics 节点、没有 caliber_from):转抄别处的口径卡不是权威写入源(A01),改它等于在别处
|
||||
另存一份定义。
|
||||
"""
|
||||
from app.apps.tree import node_map
|
||||
|
||||
return [{"id": node_id, "title": str(node.get("title") or node_id)}
|
||||
for node_id, node in node_map(graph).items()
|
||||
if node.get("type") == "metrics" and not (node.get("config") or {}).get("caliber_from")]
|
||||
|
||||
|
||||
def contract_card(graph: dict[str, Any]) -> str | None:
|
||||
"""下游契约声明从哪张卡取指标(metrics_from 只写了一张时就是它):多张卡时按它认这张草稿的权威卡。"""
|
||||
for node in graph.get("nodes") or []:
|
||||
config = ((node or {}).get("data") or {}).get("config") if isinstance(node, dict) else None
|
||||
contract = config.get("contract") if isinstance(config, dict) else None
|
||||
if not isinstance(contract, dict):
|
||||
continue
|
||||
sources = contract.get("metrics_from")
|
||||
ids = [sources] if isinstance(sources, str) else [s for s in sources or [] if isinstance(s, str)]
|
||||
if len(ids) == 1:
|
||||
return ids[0]
|
||||
return None
|
||||
|
||||
|
||||
def draft_card(graph: dict[str, Any]) -> tuple[str | None, list[dict[str, str]]]:
|
||||
"""这份草稿的权威口径卡,另交回全部能写的卡。
|
||||
|
||||
一张能写的卡时就是它;多张时认下游契约声明的那一张——不是「第一张」。都认不出来时交回 None:写的时候必须显式
|
||||
给 target.card_id,读的时候页面也照实列候选。
|
||||
"""
|
||||
cards = editable_cards(graph)
|
||||
if len(cards) == 1:
|
||||
return cards[0]["id"], cards
|
||||
referenced = contract_card(graph)
|
||||
if referenced and any(card["id"] == referenced for card in cards):
|
||||
return referenced, cards
|
||||
return None, cards
|
||||
from app.ledger.fieldbaseline import contract_card, draft_card, editable_cards
|
||||
|
||||
|
||||
def _target_card(payload: "DraftPatchIn", workflow: Workflow) -> str | None:
|
||||
|
||||
+19
-1
@@ -12,6 +12,7 @@ from fastapi import APIRouter, Depends, Header, HTTPException, Query, WebSocket,
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import defer
|
||||
|
||||
from app.api.coded import CodedHTTPException
|
||||
# 「人」的唯一入口搬到了 api/deps.py(开登录时返回认证过的用户名)。别处照旧从这里导入 actor_of
|
||||
@@ -422,7 +423,7 @@ async def list_runs(
|
||||
没按来源筛时,系统派生的运行(评测、口径对账、口径召回:engine/origin.SYSTEM_ORIGINS)默认不列——几十道评测题的运行
|
||||
会把真实用户的运行挤出第一页;按来源点名就列。没列出几次由 GET /api/runs/unlisted 照实说,界面写在列表底下。
|
||||
"""
|
||||
stmt = _filtered(select(Run), workflow_id=workflow_id, status=status, run_class=run_class, q=q,
|
||||
stmt = _filtered(select(Run).options(defer(Run.graph)), workflow_id=workflow_id, status=status, run_class=run_class, q=q,
|
||||
schedule_id=schedule_id, origin_kind=origin_kind)
|
||||
stmt = stmt.order_by(Run.created_at.desc(), Run.id.desc()).limit(limit)
|
||||
if before is not None:
|
||||
@@ -817,8 +818,25 @@ async def delete_run(
|
||||
"这是一次已封存的正式运行,是出具结果的追溯凭证。删除后其事件和工件将一并删除,"
|
||||
"封存清单无法再核对。如仍要删除,请选择「强制删除」",
|
||||
)
|
||||
from sqlalchemy import delete as sql_delete
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.db.models import Artifact
|
||||
from app.engine import checkpoints
|
||||
from app.files import store as files_store
|
||||
|
||||
artifact_ids = list((await session.execute(select(Artifact.id).where(Artifact.run_id == run_id))).scalars())
|
||||
if artifact_ids:
|
||||
await session.execute(sql_delete(Artifact).where(Artifact.run_id == run_id)
|
||||
.execution_options(synchronize_session=False))
|
||||
released_shas = await files_store.release_runs(session, [run_id])
|
||||
await session.delete(run)
|
||||
await session.commit()
|
||||
if artifact_ids:
|
||||
await artifact_store.forget(artifact_ids)
|
||||
if released_shas:
|
||||
await files_store.drop_unreferenced(released_shas)
|
||||
await checkpoints.delete_threads([run_id])
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
+11
-64
@@ -24,6 +24,16 @@ from app.engine.judge import SETTING_GROUP as JUDGE_GROUP, SETTING_LABEL as JUDG
|
||||
from app.engine.judge import daily_spend, judge_model_spec, judge_settings
|
||||
from app.engine.judge import sanitize_settings as judge_values, settings_problem as judge_problem
|
||||
from app.engine.spend import SPEND_KEY as VERIFY_SPEND, daily_spend as verify_daily_spend
|
||||
from app.ledger.settings import (
|
||||
LEDGER_DEFAULTS,
|
||||
LEDGER_GROUP,
|
||||
LEDGER_RECALL_MAX,
|
||||
LEDGER_SETTING_LABEL,
|
||||
LEDGER_TIMEOUT_RANGE,
|
||||
ledger_problem,
|
||||
ledger_settings,
|
||||
ledger_values,
|
||||
)
|
||||
from app.providers import catalog
|
||||
from app.providers.factory import (
|
||||
ModelSpec, ProviderNotConfigured, _resolve_key, assistant_unavailable, build_chat_model,
|
||||
@@ -736,72 +746,9 @@ async def apps_settings(session: AsyncSession) -> dict[str, Any]:
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 口径账本:按口径自动正式出具的开关、每日上限,口径比对的时限(docs/design/next-round/02-口径账本.md 做法 5、2)
|
||||
# 口径账本:默认项、范围与校验定义在 app/ledger/settings.py(此处直接透出供设置接口使用)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
#: 设置里 ledger 这一组。auto_issue:问数据的答案整题等价于受管口径(Agent 独立写的 SQL 被证明等价、版本受管、图里没有人工
|
||||
#: 节点和写入)时自动按口径发起一次正式运行(拍板 L1;不满足的只给一键)。issue_daily_limit:每天最多按口径正式出具几次
|
||||
#: (自动的、一键的合计,0 表示不再出具):正式运行含报告撰写的模型费用。match_timeout_s:一次运行的口径比对最多算几秒,
|
||||
#: 超时的照实记没比完(ledger/match.py)。recall_limit:口径发布新版本后,一次最多自动重算几组受影响的答案(02 做法 7、拍板 L6,
|
||||
#: 缺省 50;超出的照实写「另有 M 组未重算」,可以手动接着算;0 表示发布后不自动重算,只列出受影响的答案)。只有管理员能改
|
||||
#: (整个设置接口的改动都要管理员)
|
||||
LEDGER_GROUP = "ledger"
|
||||
LEDGER_DEFAULTS: dict[str, Any] = {"auto_issue": True, "issue_daily_limit": 20, "match_timeout_s": 8, "recall_limit": 50}
|
||||
LEDGER_SETTING_LABEL = {"auto_issue": "自动按口径正式出具", "issue_daily_limit": "按口径正式出具每日上限",
|
||||
"match_timeout_s": "口径比对时限(秒)", "recall_limit": "口径升版后每次最多重算几组"}
|
||||
#: 一次发布最多自动重算几组:召回只跑口径的数据部分、不调模型,但每组都查业务库
|
||||
LEDGER_RECALL_MAX = 1000
|
||||
#: 时限的范围:太短比不完一份口径,太长占着比对的线程
|
||||
LEDGER_TIMEOUT_RANGE = (1, 60)
|
||||
|
||||
|
||||
def _timeout_ok(value: Any) -> bool:
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool) \
|
||||
and LEDGER_TIMEOUT_RANGE[0] <= value <= LEDGER_TIMEOUT_RANGE[1]
|
||||
|
||||
|
||||
def _recall_limit_ok(value: Any) -> bool:
|
||||
return isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= LEDGER_RECALL_MAX
|
||||
|
||||
|
||||
def ledger_problem(value: Any) -> str | None:
|
||||
"""设置页写进来的 ledger 这一组有什么问题,没问题返回 None。"""
|
||||
if not isinstance(value, dict):
|
||||
return f"「口径账本」设置格式有误,需要是一个对象,当前为「{value}」"
|
||||
for key, item in value.items():
|
||||
if key not in LEDGER_DEFAULTS:
|
||||
return (f"「口径账本」设置中不支持「{key}」,可设置的项:"
|
||||
+ "、".join(f"「{LEDGER_SETTING_LABEL[k]}」" for k in LEDGER_DEFAULTS))
|
||||
if key == "auto_issue" and not isinstance(item, bool):
|
||||
return f"「{LEDGER_SETTING_LABEL[key]}」只能是开或关,当前为「{item}」"
|
||||
if key == "issue_daily_limit" and not _daily_limit_ok(item):
|
||||
return (f"「{LEDGER_SETTING_LABEL[key]}」需要是 0 到 {APPS_DAILY_LIMIT_MAX:,} 之间的整数(0 表示不再按口径出具),"
|
||||
f"当前为「{item}」")
|
||||
if key == "recall_limit" and not _recall_limit_ok(item):
|
||||
return (f"「{LEDGER_SETTING_LABEL[key]}」需要是 0 到 {LEDGER_RECALL_MAX:,} 之间的整数(0 表示发布后不自动重算),"
|
||||
f"当前为「{item}」")
|
||||
if key == "match_timeout_s" and not _timeout_ok(item):
|
||||
return (f"「{LEDGER_SETTING_LABEL[key]}」需要是 {LEDGER_TIMEOUT_RANGE[0]} 到 {LEDGER_TIMEOUT_RANGE[1]} 之间的数,"
|
||||
f"当前为「{item}」")
|
||||
return None
|
||||
|
||||
|
||||
def ledger_values(stored: Any) -> dict[str, Any]:
|
||||
"""库里存的(或刚写进来的)ledger 一组 → 补齐默认;存坏了的项按默认(自动出具的开关存坏了按关,宁可少出具)。"""
|
||||
stored = stored if isinstance(stored, dict) else {}
|
||||
auto, limit, timeout = stored.get("auto_issue"), stored.get("issue_daily_limit"), stored.get("match_timeout_s")
|
||||
recall = stored.get("recall_limit")
|
||||
return {"auto_issue": auto if isinstance(auto, bool) else (LEDGER_DEFAULTS["auto_issue"] if auto is None else False),
|
||||
"issue_daily_limit": limit if _daily_limit_ok(limit) else LEDGER_DEFAULTS["issue_daily_limit"],
|
||||
"match_timeout_s": timeout if _timeout_ok(timeout) else LEDGER_DEFAULTS["match_timeout_s"],
|
||||
"recall_limit": recall if _recall_limit_ok(recall) else LEDGER_DEFAULTS["recall_limit"]}
|
||||
|
||||
|
||||
async def ledger_settings(session: AsyncSession) -> dict[str, Any]:
|
||||
"""设置里的 ledger 一组,没存过的补默认。按口径出具、比对从这里读。"""
|
||||
row = await session.get(Setting, LEDGER_GROUP)
|
||||
return ledger_values(row.value if row else None)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# 评测:并发、每题时限、单次和每日金额上限、保留几次、结论裁判的缺省(docs/design/next-round/01-效果度量与回归评测.md 做法 17)
|
||||
|
||||
@@ -45,14 +45,18 @@ def _shanghai_time_banner() -> str:
|
||||
)
|
||||
|
||||
|
||||
_METRIC_RULE_LINE = (
|
||||
" - 若下方「已命中的标准语义指标卡片」直接对应用户所问指标且口径完整,"
|
||||
"第 1 轮直接调用 `metric_query`(支持 `date_from`、`date_to`、`group_by_date`、`group_by`、`filters`)拿到数据后立即作答。\n"
|
||||
)
|
||||
|
||||
DIRECT_SYSTEM_PROMPT = """你是 AgentLab 的「快速直查数据分析助手」。你的任务是基于已预命中的指标卡片、标准业务实体与数据目录,以最少的工具轮次、最快速度给出准确、清晰、结构化的 Markdown 数据回答。
|
||||
|
||||
【核心取数与防错纪律】
|
||||
1. **{time_banner}**
|
||||
2. **严控轮次预算,杜绝多轮逆向摸底(最多 1~2 轮工具调用)**:
|
||||
- 快速直查的目标是秒级直取答案,**严禁**展开多轮跨表试探与字段考古。
|
||||
- 若下方「已命中的标准语义指标卡片」直接对应用户所问指标且口径完整,第 1 轮直接调用 `metric_query`(支持 `date_from`、`date_to`、`group_by_date`、`group_by`、`filters`)拿到数据后立即作答。
|
||||
- 若第 1 轮返回空集、全空分组或口径明显异常(如底表混存多种记录类型、汇总层级或分类口径而缺少筛选条件),**最多再做 1 轮并行 `db_query__` 补查**(在同一轮内同时核对关联维表或配置表映射并查询过滤后的目标汇总 SQL),拿到结果立即收口作答;若仍存在跨表复杂口径分歧,直接呈现已核实的事实并建议用户点击右上角「切换深度分析」。
|
||||
{metric_rule} - 若第 1 轮返回空集、全空分组或口径明显异常(如底表混存多种记录类型、汇总层级或分类口径而缺少筛选条件),**最多再做 1 轮并行 `db_query__` 补查**(在同一轮内同时核对关联维表或配置表映射并查询过滤后的目标汇总 SQL),拿到结果立即收口作答;若仍存在跨表复杂口径分歧,直接呈现已核实的事实并建议用户点击右上角「切换深度分析」。
|
||||
3. **合并查询,杜绝碎片化往返**:
|
||||
- 手写 `db_query__<源名>` 时,尽量在 **1 条(最多不超过 2 条)SQL** 内一次性查全所需指标与近期对比窗口(例如查「今天怎么样」时,可直接查近 7 天按日汇总 `WHERE stat_date >= '<7日前>' ORDER BY stat_date DESC`,一条 SQL 同时看清今天是否有数、最新到哪一天、以及相比前几天的走势!)。
|
||||
- 已开启并行工具调用(`parallel_tools=True`):确实需要同时看汇总与下钻明细、或查两张互不依赖的表时,必须在**同一轮**一次性并行发出工具调用,切勿拆成多轮串行调用。
|
||||
@@ -83,7 +87,10 @@ def build_direct_graph(
|
||||
tools.append(METRIC_QUERY)
|
||||
tools.extend(["calculator", "kb_search"])
|
||||
|
||||
base_sys = DIRECT_SYSTEM_PROMPT.format(time_banner=_shanghai_time_banner())
|
||||
base_sys = DIRECT_SYSTEM_PROMPT.format(
|
||||
time_banner=_shanghai_time_banner(),
|
||||
metric_rule=_METRIC_RULE_LINE if has_metrics else "",
|
||||
)
|
||||
if bundle.system_context:
|
||||
full_sys = f"{base_sys}\n\n---\n\n{_escape_template_braces(bundle.system_context)}"
|
||||
else:
|
||||
@@ -252,7 +259,7 @@ class DirectChatDriver(ChatDriver):
|
||||
self.grounding_summary = bundle.ui_summary
|
||||
|
||||
# 2. 构建确定性直查图(含零延迟证据核验)并落库、推送预命中事件
|
||||
direct_graph = build_direct_graph(bundle=bundle, has_metrics=True)
|
||||
direct_graph = build_direct_graph(bundle=bundle, has_metrics=bundle.has_enabled_metrics)
|
||||
if instruction and instruction.strip() and instruction.strip() != self.question.strip():
|
||||
# 用户点「重试本轮」且仍保持快速直查时,把重试提示附进用户 prompt
|
||||
agent_node = next((n for n in direct_graph["nodes"] if n.get("type") == "agent"), None)
|
||||
|
||||
@@ -473,6 +473,38 @@ def _auto_bind_bare_numbers(
|
||||
if not cells:
|
||||
return text
|
||||
|
||||
cell_order = {id(c): idx for idx, c in enumerate(cells)}
|
||||
text_code_by_raw: dict[str, list[_CellCandidate]] = {}
|
||||
numeric_cells: list[_CellCandidate] = []
|
||||
numeric_by_exact: dict[Decimal, list[_CellCandidate]] = {}
|
||||
for c in cells:
|
||||
if c.is_text_code:
|
||||
text_code_by_raw.setdefault(c.raw_text, []).append(c)
|
||||
else:
|
||||
numeric_cells.append(c)
|
||||
numeric_by_exact.setdefault(c.dec_value, []).append(c)
|
||||
|
||||
quant_cache: dict[tuple[str, int], dict[Decimal, list[_CellCandidate]]] = {}
|
||||
|
||||
def _quant_buckets(scale_kind: str, places: int) -> dict[Decimal, list[_CellCandidate]]:
|
||||
key = (scale_kind, places)
|
||||
cached = quant_cache.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
buckets: dict[Decimal, list[_CellCandidate]] = {}
|
||||
quant = Decimal(1).scaleb(-places)
|
||||
if scale_kind == "pct":
|
||||
for c in numeric_cells:
|
||||
q_val = (c.dec_value * Decimal(100)).quantize(quant, rounding=ROUND_HALF_UP)
|
||||
buckets.setdefault(q_val, []).append(c)
|
||||
else:
|
||||
divisor = Decimal(10000) if scale_kind == "万" else Decimal(100_000_000)
|
||||
for c in numeric_cells:
|
||||
q_val = (c.dec_value / divisor).quantize(quant, rounding=ROUND_HALF_UP)
|
||||
buckets.setdefault(q_val, []).append(c)
|
||||
quant_cache[key] = buckets
|
||||
return buckets
|
||||
|
||||
fences = _fenced_line_ranges(text)
|
||||
lines = text.splitlines(keepends=True)
|
||||
table_headers = _table_headers_by_line(lines)
|
||||
@@ -546,31 +578,31 @@ def _auto_bind_bare_numbers(
|
||||
continue
|
||||
|
||||
has_leading_zero = bool(_LEADING_ZERO_INT_RE.match(raw_clean))
|
||||
places = max(int(tok.decimals or 0), 0)
|
||||
matched: list[tuple[_CellCandidate, str | None]] = []
|
||||
for cand in cells:
|
||||
if has_leading_zero:
|
||||
if cand.is_text_code and cand.raw_text == raw_clean:
|
||||
matched.append((cand, None))
|
||||
continue
|
||||
if cand.is_text_code:
|
||||
if not tok.is_percent and not unit_conv and cand.raw_text == raw_clean:
|
||||
matched.append((cand, None))
|
||||
continue
|
||||
if unit_conv and unit_scale is not None:
|
||||
scaled_u = cand.dec_value / unit_scale
|
||||
places = max(int(tok.decimals or 0), 0)
|
||||
quant = Decimal(1).scaleb(-places)
|
||||
if scaled_u.quantize(quant, rounding=ROUND_HALF_UP) == target_dec:
|
||||
matched.append((cand, unit_conv))
|
||||
elif cand.dec_value == target_dec:
|
||||
matched.append((cand, None))
|
||||
elif tok.is_percent:
|
||||
# 单元格存 0.1258,正文写 12.6% 或 12.58%
|
||||
scaled = cand.dec_value * Decimal(100)
|
||||
places = max(int(tok.decimals or 0), 0)
|
||||
quant = Decimal(1).scaleb(-places)
|
||||
if scaled.quantize(quant, rounding=ROUND_HALF_UP) == target_dec:
|
||||
matched.append((cand, "pct"))
|
||||
if has_leading_zero:
|
||||
matched = [(c, None) for c in text_code_by_raw.get(raw_clean, ())]
|
||||
elif unit_conv and unit_scale is not None:
|
||||
matched = [(c, unit_conv) for c in _quant_buckets(unit_conv, places).get(target_dec, ())]
|
||||
elif not tok.is_percent:
|
||||
tc_hits = [(c, None) for c in text_code_by_raw.get(raw_clean, ())]
|
||||
num_hits = [(c, None) for c in numeric_by_exact.get(target_dec, ())]
|
||||
if tc_hits and num_hits:
|
||||
matched = sorted(tc_hits + num_hits, key=lambda pair: cell_order[id(pair[0])])
|
||||
else:
|
||||
matched = tc_hits or num_hits
|
||||
else:
|
||||
# 单元格存 12.6(直接相等)或存 0.1258(正文写 12.6% 或 12.58%)
|
||||
exact_hits = [(c, None) for c in numeric_by_exact.get(target_dec, ())]
|
||||
pct_hits = [
|
||||
(c, "pct")
|
||||
for c in _quant_buckets("pct", places).get(target_dec, ())
|
||||
if c.dec_value != target_dec
|
||||
]
|
||||
if exact_hits and pct_hits:
|
||||
matched = sorted(exact_hits + pct_hits, key=lambda pair: cell_order[id(pair[0])])
|
||||
else:
|
||||
matched = exact_hits or pct_hits
|
||||
|
||||
if not matched:
|
||||
continue
|
||||
|
||||
+101
-60
@@ -51,6 +51,7 @@ class GroundingBundle:
|
||||
guardrails: list[dict[str, str]] = field(default_factory=list)
|
||||
system_context: str = ""
|
||||
ui_summary: dict[str, Any] = field(default_factory=dict)
|
||||
has_enabled_metrics: bool = False
|
||||
|
||||
|
||||
def _term_in_text(term: str, text_lower: str, ascii_words: set[str]) -> bool:
|
||||
@@ -395,13 +396,14 @@ async def ground_question(
|
||||
else:
|
||||
narrowed_sources = list(candidate_sources)
|
||||
|
||||
# 5. 数据目录 BM25 直选 + 指标底表 + 实体绑定表 -> 组装高信噪比表结构与防错护栏
|
||||
# 5. 数据目录 BM25 直选 + 指标底表 + 实体绑定表 -> 跨源全局打分选表,避免首源独占全部槽位
|
||||
matched_tables: list[dict[str, Any]] = []
|
||||
guardrails: list[dict[str, str]] = []
|
||||
table_blocks: list[str] = []
|
||||
total_detail_chars = 0
|
||||
|
||||
for ds in narrowed_sources:
|
||||
ds_ctx_list: list[dict[str, Any]] = []
|
||||
for ds_idx, ds in enumerate(narrowed_sources):
|
||||
cache = ds.schema_cache if isinstance(ds.schema_cache, Mapping) else {}
|
||||
tables_map = (cache.get("tables") or {}) if isinstance(cache, Mapping) else {}
|
||||
if not isinstance(tables_map, Mapping) or not tables_map:
|
||||
@@ -414,74 +416,112 @@ async def ground_question(
|
||||
facts = await catalog_profile_jobs.last_facts(session, ds.id)
|
||||
except Exception: # noqa: BLE001
|
||||
facts = {}
|
||||
|
||||
picked_for_ds: list[str] = []
|
||||
reasons_for_table: dict[str, str] = {}
|
||||
|
||||
# (a) 命中指标的底表优先入选
|
||||
for mm in matched_metrics:
|
||||
if mm["source_id"] == ds.id:
|
||||
resolved = catalog.resolve_table_name(cache, mm["table_name"])
|
||||
if resolved and resolved not in picked_for_ds:
|
||||
picked_for_ds.append(resolved)
|
||||
reasons_for_table[resolved] = f"指标「{mm['name']}」底表"
|
||||
|
||||
# (b) 命中实体的绑定表入选
|
||||
for me in matched_entities:
|
||||
for b in me["bindings"]:
|
||||
if name_key(b["source_name"]) == name_key(ds.name):
|
||||
resolved = catalog.resolve_table_name(cache, b["table_name"])
|
||||
if resolved and resolved not in picked_for_ds and len(picked_for_ds) < MAX_GROUNDED_TABLES:
|
||||
picked_for_ds.append(resolved)
|
||||
reasons_for_table[resolved] = f"实体「{me['label']}」绑定表"
|
||||
|
||||
masked = schema_view.masked_set(ds)
|
||||
try:
|
||||
rel_graph = schema_view.relation_graph(ds, raw_notes)
|
||||
except Exception: # noqa: BLE001
|
||||
rel_graph = {}
|
||||
|
||||
def _include_companion_lookup(limit_cap: int) -> list[dict[str, Any]]:
|
||||
if not picked_for_ds:
|
||||
return []
|
||||
comps = find_companion_lookup_tables(
|
||||
tables_map,
|
||||
picked_for_ds,
|
||||
visible_by_table=visible_by_table,
|
||||
rel_graph=rel_graph,
|
||||
masked_set=masked,
|
||||
ds_ctx_list.append(
|
||||
{
|
||||
"idx": ds_idx,
|
||||
"ds": ds,
|
||||
"cache": cache,
|
||||
"tables_map": tables_map,
|
||||
"raw_notes": raw_notes,
|
||||
"visible_by_table": visible_by_table,
|
||||
"facts": facts,
|
||||
"masked": masked,
|
||||
"rel_graph": rel_graph,
|
||||
"picked": [],
|
||||
"reasons": {},
|
||||
}
|
||||
)
|
||||
|
||||
total_picked = 0
|
||||
|
||||
def _pick_table(ctx: dict[str, Any], tname: str, reason: str, *, cap: int = MAX_GROUNDED_TABLES) -> bool:
|
||||
nonlocal total_picked
|
||||
if total_picked >= cap or tname in ctx["picked"]:
|
||||
return False
|
||||
ctx["picked"].append(tname)
|
||||
ctx["reasons"][tname] = reason
|
||||
total_picked += 1
|
||||
return True
|
||||
|
||||
# (a) 命中指标的底表跨源最优先入选
|
||||
for ctx in ds_ctx_list:
|
||||
ds = ctx["ds"]
|
||||
for mm in matched_metrics:
|
||||
if mm["source_id"] == ds.id:
|
||||
resolved = catalog.resolve_table_name(ctx["cache"], mm["table_name"])
|
||||
if resolved:
|
||||
_pick_table(ctx, resolved, f"指标「{mm['name']}」底表", cap=MAX_GROUNDED_TABLES + 2)
|
||||
|
||||
# (b) 命中实体的绑定表跨源次优先入选
|
||||
for ctx in ds_ctx_list:
|
||||
ds = ctx["ds"]
|
||||
for me in matched_entities:
|
||||
for b in me["bindings"]:
|
||||
if name_key(b["source_name"]) == name_key(ds.name):
|
||||
resolved = catalog.resolve_table_name(ctx["cache"], b["table_name"])
|
||||
if resolved:
|
||||
_pick_table(ctx, resolved, f"实体「{me['label']}」绑定表")
|
||||
|
||||
# (c) 各候选源 BM25 检索统一收集并按得分全局排序选表(0 LLM 调用)
|
||||
if total_picked < MAX_GROUNDED_TABLES:
|
||||
global_bm25_hits: list[tuple[float, int, int, dict[str, Any], str]] = []
|
||||
for ctx in ds_ctx_list:
|
||||
hits = schema_view.search_tables(
|
||||
ctx["ds"],
|
||||
ctx["visible_by_table"],
|
||||
combined_q,
|
||||
limit=MAX_GROUNDED_TABLES,
|
||||
)
|
||||
for comp in comps:
|
||||
if comp["col_count"] > 12:
|
||||
continue
|
||||
ct_name = comp["table_name"]
|
||||
if ct_name not in picked_for_ds and len(picked_for_ds) + len(matched_tables) < limit_cap:
|
||||
picked_for_ds.append(ct_name)
|
||||
reasons_for_table[ct_name] = "关联维表与字典"
|
||||
for rank, h in enumerate(hits):
|
||||
if h.in_scope and h.name not in ctx["picked"]:
|
||||
global_bm25_hits.append((float(h.score), -ctx["idx"], -rank, ctx, h.name))
|
||||
global_bm25_hits.sort(key=lambda item: (item[0], item[1], item[2]), reverse=True)
|
||||
for _score, _neg_ds_idx, _neg_rank, ctx, tname in global_bm25_hits:
|
||||
if total_picked >= MAX_GROUNDED_TABLES:
|
||||
break
|
||||
_pick_table(ctx, tname, "目录语义检索命中")
|
||||
|
||||
# (d) 若用户明确指定了该小库(<= 5 张表)且尚未命中任何表,直接全给
|
||||
for ctx in ds_ctx_list:
|
||||
ds = ctx["ds"]
|
||||
if not ctx["picked"] and ds in explicit_sources and len(ctx["tables_map"]) <= 5:
|
||||
for tname in list(ctx["tables_map"].keys()):
|
||||
if not _pick_table(ctx, tname, "指定数据源核心表"):
|
||||
break
|
||||
return comps
|
||||
|
||||
_include_companion_lookup(MAX_GROUNDED_TABLES)
|
||||
|
||||
# (c) BM25 数据目录检索直选(0 LLM 调用)
|
||||
remaining_slots = MAX_GROUNDED_TABLES - len(matched_tables) - len(picked_for_ds)
|
||||
if remaining_slots > 0:
|
||||
hits = schema_view.search_tables(ds, visible_by_table, combined_q, limit=remaining_slots + 2)
|
||||
for h in hits:
|
||||
if h.in_scope and h.name not in picked_for_ds and len(picked_for_ds) + len(matched_tables) < MAX_GROUNDED_TABLES:
|
||||
picked_for_ds.append(h.name)
|
||||
reasons_for_table[h.name] = "目录语义检索命中"
|
||||
|
||||
# (d) 若用户明确指定了该小库(<= 5 张表)且尚未命中任何表,直接全给
|
||||
if not picked_for_ds and ds in explicit_sources and len(tables_map) <= 5:
|
||||
for tname in list(tables_map.keys())[:MAX_GROUNDED_TABLES - len(matched_tables)]:
|
||||
picked_for_ds.append(tname)
|
||||
reasons_for_table[tname] = "指定数据源核心表"
|
||||
|
||||
companion_links = _include_companion_lookup(MAX_GROUNDED_TABLES + 1)
|
||||
|
||||
# 按数据源顺序补齐关联维表与一跳连接表,渲染表结构与护栏
|
||||
for ctx in ds_ctx_list:
|
||||
ds = ctx["ds"]
|
||||
picked_for_ds: list[str] = ctx["picked"]
|
||||
if not picked_for_ds:
|
||||
continue
|
||||
reasons_for_table: dict[str, str] = ctx["reasons"]
|
||||
tables_map = ctx["tables_map"]
|
||||
raw_notes = ctx["raw_notes"]
|
||||
visible_by_table = ctx["visible_by_table"]
|
||||
facts = ctx["facts"]
|
||||
masked = ctx["masked"]
|
||||
rel_graph = ctx["rel_graph"]
|
||||
|
||||
companion_links = find_companion_lookup_tables(
|
||||
tables_map,
|
||||
picked_for_ds,
|
||||
visible_by_table=visible_by_table,
|
||||
rel_graph=rel_graph,
|
||||
masked_set=masked,
|
||||
)
|
||||
for comp in companion_links:
|
||||
if comp["col_count"] > 12:
|
||||
continue
|
||||
ct_name = comp["table_name"]
|
||||
if _pick_table(ctx, ct_name, "关联维表与字典", cap=MAX_GROUNDED_TABLES + 1):
|
||||
break
|
||||
|
||||
# 补一跳连接中间表(若有)
|
||||
try:
|
||||
@@ -546,7 +586,7 @@ async def ground_question(
|
||||
ds_sections.append("表间标准连接条件:\n" + "\n".join(join_lines[:12]))
|
||||
|
||||
table_blocks.append("\n".join(ds_sections))
|
||||
if len(matched_tables) >= MAX_GROUNDED_TABLES:
|
||||
if len(matched_tables) >= MAX_GROUNDED_TABLES + 1:
|
||||
break
|
||||
|
||||
elapsed_ms = max(1, int((time.perf_counter() - t0) * 1000))
|
||||
@@ -719,4 +759,5 @@ async def ground_question(
|
||||
guardrails=guardrails,
|
||||
system_context="\n\n".join(ctx_parts),
|
||||
ui_summary=ui_summary,
|
||||
has_enabled_metrics=bool(scoped_metrics),
|
||||
)
|
||||
|
||||
@@ -23,7 +23,7 @@ import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
from app.cluster import guard, identity, leader, reaper, requests, state, workers
|
||||
from app.cluster import guard, identity, leader, reaper, requests, slots, state, workers
|
||||
from app.cluster.signal import signals
|
||||
from app.core.config import settings
|
||||
|
||||
@@ -84,6 +84,7 @@ class _Service:
|
||||
await signals.start()
|
||||
signals.on("q", self._on_request)
|
||||
signals.on("gone", self._on_gone)
|
||||
signals.on("slot", self._on_slot)
|
||||
if not self.single:
|
||||
await signals.ready(timeout=5.0)
|
||||
self.wake = asyncio.Event()
|
||||
@@ -115,6 +116,9 @@ class _Service:
|
||||
if self.wake is not None:
|
||||
self.wake.set()
|
||||
|
||||
def _on_slot(self, _parts: list[str]) -> None:
|
||||
slots.wake_local()
|
||||
|
||||
# ---------------- 续租约:单独的任务、单独的连接 ----------------
|
||||
|
||||
async def _lease_conn_open(self) -> Any:
|
||||
|
||||
@@ -76,31 +76,38 @@ async def put_json(
|
||||
幂等:同内容重复写只落一次盘;同一工件被多个 run 引用时,
|
||||
数据库里各记一行引用(溯源需要知道"谁产出的"),文件只有一份。
|
||||
"""
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from app.cluster.mutex import lock
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact
|
||||
|
||||
text = canonical_json(obj)
|
||||
artifact_id = content_hash(text)
|
||||
path = _path_of(artifact_id)
|
||||
if not path.exists():
|
||||
# 原子落盘(core/atomic):写一半崩了不会留下半个工件;临时文件名带随机后缀,两个进程同时写同一份内容
|
||||
# 各写各的,不会一个把另一个写到一半的截断(以前共用 <id>.tmp,后到的会把先到的半成品改名成正式文件)
|
||||
atomic.write_text(path, text)
|
||||
async with lock(f"artifact:{artifact_id[:2]}"):
|
||||
if not path.exists():
|
||||
# 原子落盘(core/atomic):写一半崩了不会留下半个工件;临时文件名带随机后缀,两个进程同时写同一份内容
|
||||
# 各写各的,不会一个把另一个写到一半的截断;且与同分桶的 forget 互斥,不会刚写下就被并发回收删掉
|
||||
atomic.write_text(path, text)
|
||||
|
||||
async with SessionLocal() as session:
|
||||
existing = await session.get(Artifact, (artifact_id, run_id or ""))
|
||||
if existing is None:
|
||||
session.add(
|
||||
Artifact(
|
||||
id=artifact_id,
|
||||
run_id=run_id or "",
|
||||
kind=kind,
|
||||
node_id=node_id,
|
||||
size=len(text.encode()),
|
||||
meta=meta or {},
|
||||
async with SessionLocal() as session:
|
||||
existing = await session.get(Artifact, (artifact_id, run_id or ""))
|
||||
if existing is None:
|
||||
session.add(
|
||||
Artifact(
|
||||
id=artifact_id,
|
||||
run_id=run_id or "",
|
||||
kind=kind,
|
||||
node_id=node_id,
|
||||
size=len(text.encode()),
|
||||
meta=meta or {},
|
||||
)
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
try:
|
||||
await session.commit()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
return artifact_id
|
||||
|
||||
|
||||
@@ -148,27 +155,35 @@ async def forget(ids: Any) -> list[str]:
|
||||
读到错的内容。"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.cluster.mutex import lock
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact
|
||||
|
||||
wanted = sorted({i for i in ids or () if isinstance(i, str) and i and i.isalnum()})
|
||||
if not wanted:
|
||||
return []
|
||||
async with SessionLocal() as session:
|
||||
kept: set[str] = set()
|
||||
for start in range(0, len(wanted), 500):
|
||||
chunk = wanted[start:start + 500]
|
||||
kept |= set((await session.execute(select(Artifact.id).where(Artifact.id.in_(chunk)))).scalars())
|
||||
removed: list[str] = []
|
||||
by_bucket: dict[str, list[str]] = {}
|
||||
for artifact_id in wanted:
|
||||
if artifact_id in kept:
|
||||
continue
|
||||
path = _path_of(artifact_id)
|
||||
try:
|
||||
path.unlink()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
removed.append(artifact_id)
|
||||
by_bucket.setdefault(artifact_id[:2], []).append(artifact_id)
|
||||
|
||||
removed: list[str] = []
|
||||
for bucket in sorted(by_bucket):
|
||||
bucket_ids = by_bucket[bucket]
|
||||
async with lock(f"artifact:{bucket}"):
|
||||
async with SessionLocal() as session:
|
||||
kept: set[str] = set()
|
||||
for start in range(0, len(bucket_ids), 500):
|
||||
chunk = bucket_ids[start:start + 500]
|
||||
kept |= set((await session.execute(select(Artifact.id).where(Artifact.id.in_(chunk)))).scalars())
|
||||
for artifact_id in bucket_ids:
|
||||
if artifact_id in kept:
|
||||
continue
|
||||
path = _path_of(artifact_id)
|
||||
try:
|
||||
path.unlink()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
removed.append(artifact_id)
|
||||
return removed
|
||||
|
||||
|
||||
|
||||
+20
-5
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
from collections import defaultdict
|
||||
from collections import OrderedDict, defaultdict
|
||||
from collections.abc import Callable
|
||||
from typing import AsyncIterator
|
||||
|
||||
@@ -11,6 +11,7 @@ from app.core.events import RunEventModel
|
||||
# 单个订阅者的积压上限。慢消费者(比如卡住的浏览器标签)不应该拖垮整个 run,
|
||||
# 所以队列满了就丢最旧的事件并打标记,而不是阻塞发布方。
|
||||
_QUEUE_MAXSIZE = 2048
|
||||
_MAX_SEQ_ENTRIES = 4096
|
||||
|
||||
|
||||
class _Subscriber:
|
||||
@@ -31,17 +32,30 @@ class EventBus:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._subs: dict[str, set[_Subscriber]] = defaultdict(set)
|
||||
self._seq: dict[str, int] = defaultdict(int)
|
||||
self._seq: OrderedDict[str, int] = OrderedDict()
|
||||
self._lock = asyncio.Lock()
|
||||
self.tap: Callable[[RunEventModel], None] | None = None
|
||||
|
||||
def _trim_seq(self) -> None:
|
||||
while len(self._seq) > _MAX_SEQ_ENTRIES:
|
||||
oldest, _ = next(iter(self._seq.items()))
|
||||
if self._subs.get(oldest):
|
||||
self._seq.move_to_end(oldest)
|
||||
break
|
||||
self._seq.popitem(last=False)
|
||||
|
||||
def next_seq(self, run_id: str) -> int:
|
||||
self._seq[run_id] += 1
|
||||
return self._seq[run_id]
|
||||
nxt = self._seq.get(run_id, 0) + 1
|
||||
self._seq[run_id] = nxt
|
||||
self._seq.move_to_end(run_id)
|
||||
self._trim_seq()
|
||||
return nxt
|
||||
|
||||
def set_seq(self, run_id: str, value: int) -> None:
|
||||
"""恢复一个已存在的 run 时,把序号接上,保证前端排序不乱。"""
|
||||
self._seq[run_id] = max(self._seq[run_id], value)
|
||||
self._seq[run_id] = max(self._seq.get(run_id, 0), value)
|
||||
self._seq.move_to_end(run_id)
|
||||
self._trim_seq()
|
||||
|
||||
async def publish(self, event: RunEventModel, *, relayed: bool = False) -> None:
|
||||
if not relayed and self.tap is not None:
|
||||
@@ -61,6 +75,7 @@ class EventBus:
|
||||
|
||||
async def close(self, run_id: str) -> None:
|
||||
"""给所有订阅者发结束哨兵,让 WebSocket 干净地收尾。"""
|
||||
self._seq.pop(run_id, None)
|
||||
for sub in list(self._subs.get(run_id, ())):
|
||||
with contextlib.suppress(asyncio.QueueFull):
|
||||
sub.queue.put_nowait(None)
|
||||
|
||||
@@ -361,7 +361,10 @@ class Settings(BaseSettings):
|
||||
|
||||
@lru_cache
|
||||
def get_settings() -> Settings:
|
||||
s = Settings()
|
||||
import os
|
||||
|
||||
no_dotenv = os.environ.get("AGENTLAB_DISABLE_DOTENV", "").strip().lower() in ("1", "true", "yes", "on")
|
||||
s = Settings(_env_file=None) if no_dotenv else Settings()
|
||||
s.ensure_dirs()
|
||||
return s
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ import asyncio
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timedelta
|
||||
@@ -51,7 +52,9 @@ MAX_COLUMNS = 12
|
||||
#: 现查到的表的范围在进程里记多久(秒):同一张表一会儿内查好几次(一张图里几个取数节点、助手自查几轮)不再全表 MIN /
|
||||
#: MAX。滚动表一天才换一次数据,几分钟的旧不影响判断;查不成的、退回剖析的不记
|
||||
LIVE_TTL_S = 300.0
|
||||
_LIVE: dict[tuple[Any, ...], tuple[float, Any]] = {}
|
||||
SNAPSHOT_LIVE_TTL_S = 86400.0
|
||||
_LIVE_MAX = 512
|
||||
_LIVE: OrderedDict[tuple[Any, ...], tuple[float, Any]] = OrderedDict()
|
||||
#: 现查没成功(大表全表 MIN / MAX 超时、库一时连不上)也记 LIVE_TTL_S:这段时间里同一张表直接退回剖析、取值观察,
|
||||
#: 不然取数 agent 连查几次、每次结果里有整列 0,每次都要白等 PROBE_TIMEOUT_S 秒才退回
|
||||
_PROBE_FAILED = object()
|
||||
@@ -68,7 +71,7 @@ NO_DATA_SHORT = "查询区间里这张表没有数据"
|
||||
_COMPARE = {"GT": ">", "GTE": ">=", "LT": "<", "LTE": "<=", "EQ": "="}
|
||||
_FLIP = {">": "<", ">=": "<=", "<": ">", "<=": ">=", "=": "="}
|
||||
_ISO = re.compile(r"^(\d{4})-(\d{1,2})-(\d{1,2})(?:[ T](\d{1,2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?)?"
|
||||
r"(?:Z|[+-]\d{2}:?\d{2})?$")
|
||||
r"(?:Z|[+-]\d{2}(?::?\d{2})?)?$")
|
||||
_COMPACT = re.compile(r"^(\d{4})(\d{2})(\d{2})$")
|
||||
|
||||
|
||||
@@ -399,14 +402,19 @@ async def _range(source: Any, flt: _Filter, read: str | None, cache: Cache, fact
|
||||
key = (str(getattr(source, "name", "")), name_key(flt.table_name), name_key(flt.column_name))
|
||||
if key in cache.ranges:
|
||||
return cache.ranges[key]
|
||||
live = (getattr(source, "id", None), getattr(source, "snapshot_id", None), *key)
|
||||
snapshot_id = getattr(source, "snapshot_id", None)
|
||||
live = (getattr(source, "id", None), snapshot_id, *key)
|
||||
held = _LIVE.get(live)
|
||||
failed_before = False
|
||||
if held is not None and held[0] > time.monotonic():
|
||||
if held[1] is not _PROBE_FAILED:
|
||||
cache.ranges[key] = held[1]
|
||||
return held[1]
|
||||
failed_before = True
|
||||
if held is not None:
|
||||
if held[0] > time.monotonic():
|
||||
_LIVE.move_to_end(live)
|
||||
if held[1] is not _PROBE_FAILED:
|
||||
cache.ranges[key] = held[1]
|
||||
return held[1]
|
||||
failed_before = True
|
||||
else:
|
||||
_LIVE.pop(live, None)
|
||||
table = flt.table.copy()
|
||||
table.set("alias", None)
|
||||
ident = flt.column.this.copy() if isinstance(flt.column.this, exp.Identifier) \
|
||||
@@ -429,16 +437,18 @@ async def _range(source: Any, flt: _Filter, read: str | None, cache: Cache, fact
|
||||
out = await _fallback(flt, facts)
|
||||
cache.ranges[key] = out
|
||||
if found is None and not failed_before:
|
||||
_remember(live, _PROBE_FAILED)
|
||||
_remember(live, _PROBE_FAILED, ttl=LIVE_TTL_S)
|
||||
elif out is not None and out.basis == "live":
|
||||
_remember(live, out)
|
||||
ttl = SNAPSHOT_LIVE_TTL_S if snapshot_id else LIVE_TTL_S
|
||||
_remember(live, out, ttl=ttl)
|
||||
return out
|
||||
|
||||
|
||||
def _remember(live: tuple[Any, ...], value: Any) -> None:
|
||||
if len(_LIVE) > 512:
|
||||
_LIVE.clear()
|
||||
_LIVE[live] = (time.monotonic() + LIVE_TTL_S, value)
|
||||
def _remember(live: tuple[Any, ...], value: Any, *, ttl: float = LIVE_TTL_S) -> None:
|
||||
_LIVE.pop(live, None)
|
||||
while len(_LIVE) >= _LIVE_MAX:
|
||||
_LIVE.popitem(last=False)
|
||||
_LIVE[live] = (time.monotonic() + ttl, value)
|
||||
|
||||
|
||||
async def _fallback(flt: _Filter, facts: Facts) -> _Range | None:
|
||||
|
||||
@@ -110,29 +110,44 @@ class CsvTable:
|
||||
delimiter: str
|
||||
|
||||
|
||||
def _resolve_csv_meta(raw: bytes, delimiter: str | None) -> tuple[str, str, str]:
|
||||
from app.data import tabular
|
||||
|
||||
enc, errors = tabular._detect_csv_encoding(raw)
|
||||
if delimiter is None:
|
||||
with tabular._open_csv_text(raw, encoding=enc, errors=errors, newline="") as probe_stream:
|
||||
head = probe_stream.read(8192)
|
||||
try:
|
||||
delimiter = csv.Sniffer().sniff(head, delimiters=_DELIMITERS).delimiter
|
||||
except csv.Error:
|
||||
delimiter = "," # 嗅不出来就按最常见的来
|
||||
return enc, errors, delimiter
|
||||
|
||||
|
||||
def _iter_csv_records(raw: bytes, *, enc: str, errors: str, delimiter: str) -> Iterator[list[str]]:
|
||||
from app.data import tabular
|
||||
|
||||
row_count = 0
|
||||
try:
|
||||
with tabular._open_csv_text(raw, encoding=enc, errors=errors, newline="") as stream:
|
||||
for record in csv.reader(stream, delimiter=delimiter):
|
||||
if row_count >= MAX_ROWS:
|
||||
raise UnsupportedTable(f"CSV 文件超过 {MAX_ROWS:,} 行,超出可以导入的上限")
|
||||
if len(record) > MAX_COLS:
|
||||
raise UnsupportedTable(f"CSV 文件第 {row_count + 1} 行有 {len(record):,} 列,超过 {MAX_COLS:,} 列的上限")
|
||||
row_count += 1
|
||||
yield record
|
||||
except csv.Error as e:
|
||||
raise UnsupportedTable(f"无法按 CSV 读取这个文件:第 {row_count + 1} 行的写法不正确({e})。"
|
||||
"请在 Excel 中重新另存为 CSV 后上传") from e
|
||||
|
||||
|
||||
def parse(raw: bytes, delimiter: str | None = None) -> CsvTable:
|
||||
"""原件 → 记录。编码照 tabular 读 CSV 的做法;delimiter 不给时嗅探(同 tabular)。字段太长、引号不配对读不了时抛
|
||||
UnsupportedTable。"""
|
||||
from app.data import tabular
|
||||
|
||||
text = tabular._decode_csv(raw)
|
||||
if delimiter is None:
|
||||
try:
|
||||
delimiter = csv.Sniffer().sniff(text[:8192], delimiters=_DELIMITERS).delimiter
|
||||
except csv.Error:
|
||||
delimiter = "," # 嗅不出来就按最常见的来
|
||||
records: list[list[str]] = []
|
||||
try:
|
||||
for record in csv.reader(io.StringIO(text, newline=""), delimiter=delimiter):
|
||||
if len(records) >= MAX_ROWS:
|
||||
raise UnsupportedTable(f"CSV 文件超过 {MAX_ROWS:,} 行,超出可以导入的上限")
|
||||
if len(record) > MAX_COLS:
|
||||
raise UnsupportedTable(f"CSV 文件第 {len(records) + 1} 行有 {len(record):,} 列,超过 {MAX_COLS:,} 列的上限")
|
||||
records.append(record)
|
||||
except csv.Error as e:
|
||||
raise UnsupportedTable(f"无法按 CSV 读取这个文件:第 {len(records) + 1} 行的写法不正确({e})。"
|
||||
"请在 Excel 中重新另存为 CSV 后上传") from e
|
||||
return CsvTable(records=records, delimiter=delimiter)
|
||||
enc, errors, resolved_delim = _resolve_csv_meta(raw, delimiter)
|
||||
records = list(_iter_csv_records(raw, enc=enc, errors=errors, delimiter=resolved_delim))
|
||||
return CsvTable(records=records, delimiter=resolved_delim)
|
||||
|
||||
|
||||
def _cells(table: CsvTable) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]:
|
||||
@@ -155,15 +170,18 @@ def sheet_delimiter(sheet: Any) -> str | None:
|
||||
|
||||
def scan(raw: bytes, *, delimiter: str | None = None) -> WorkbookScan:
|
||||
"""CSV 原件 → 扫描结果:一张可见工作表(名字 CSV_SHEET、路径 CSV_PATH 加分隔符),边界、非空格数、逐行非空格数的
|
||||
游程。没有合并区、隐藏行列、公式、错误值、离群格(CSV 里每一格都是写出来的,没有残留格式)。"""
|
||||
table = parse(raw, delimiter)
|
||||
sheet = SheetScan(name=CSV_SHEET, state="visible", path=CSV_PATH + table.delimiter)
|
||||
游程。流式统计,不再先把整张 CSV 物化成全量字符串与二维列表。"""
|
||||
enc, errors, resolved_delim = _resolve_csv_meta(raw, delimiter)
|
||||
sheet = SheetScan(name=CSV_SHEET, state="visible", path=CSV_PATH + resolved_delim)
|
||||
lo_r = lo_c = hi_r = hi_c = 0
|
||||
runs: list[tuple[int, int, int]] = []
|
||||
for r, cells in _cells(table):
|
||||
n = len(cells)
|
||||
for r, record in enumerate(_iter_csv_records(raw, enc=enc, errors=errors, delimiter=resolved_delim), 1):
|
||||
nonempty_cols = [c for c, field in enumerate(record, 1) if cell_value(field) is not None]
|
||||
if not nonempty_cols:
|
||||
continue
|
||||
n = len(nonempty_cols)
|
||||
sheet.nonempty += n
|
||||
first, last = cells[0][0], cells[-1][0]
|
||||
first, last = nonempty_cols[0], nonempty_cols[-1]
|
||||
if not lo_r:
|
||||
lo_r, lo_c, hi_c = r, first, last
|
||||
lo_c, hi_c, hi_r = min(lo_c, first), max(hi_c, last), r
|
||||
|
||||
+108
-46
@@ -14,7 +14,7 @@ import re
|
||||
import sqlite3
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from contextlib import asynccontextmanager
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, AsyncIterator, Awaitable, Callable
|
||||
|
||||
@@ -297,7 +297,7 @@ def open_checked_sqlite(path: str, *, readonly: bool = True, immutable: bool = F
|
||||
mode = "ro" if readonly else "rwc"
|
||||
extra = "&immutable=1" if immutable else ""
|
||||
return sqlite3.connect(f"file:{quote(os.fspath(path))}?mode={mode}{extra}", uri=True,
|
||||
factory=_DqsOffConnection)
|
||||
timeout=5.0, factory=_DqsOffConnection)
|
||||
|
||||
|
||||
def open_memory_sqlite() -> sqlite3.Connection:
|
||||
@@ -549,6 +549,8 @@ class EngineCache:
|
||||
#: 正在进行的快照核对,按(缓存键, 文件, 登记的哈希)合并。核对结束(不论成败)即移除,
|
||||
#: 失败结果不留:下次照常重新核对
|
||||
self._verifying: dict[tuple[str, str, str], asyncio.Future[Fingerprint]] = {}
|
||||
#: 每个数据源的失效世代:get() 在锁外核对快照期间若赶上 invalidate(),进锁后看到世代变了就重来,绝不把旧连接装回缓存
|
||||
self._epochs: dict[str, int] = {}
|
||||
|
||||
def _hit(self, key: str, doomed: list[AsyncEngine]) -> AsyncEngine | None:
|
||||
"""缓存里还可信的 engine。快照文件和核对时的指纹对不上的,摘出来放进 doomed 交给调用方关掉。"""
|
||||
@@ -568,6 +570,7 @@ class EngineCache:
|
||||
|
||||
async def get(self, source: Any) -> AsyncEngine:
|
||||
key = cache_key(source)
|
||||
sid = _source_of(key)
|
||||
doomed: list[AsyncEngine] = []
|
||||
try:
|
||||
cached = self._hit(key, doomed)
|
||||
@@ -577,34 +580,44 @@ class EngineCache:
|
||||
if getattr(source, "immutable", False) and not expected:
|
||||
logger.warning("版本快照没有登记文件哈希,已拒绝:key=%s path=%s", key, source.database)
|
||||
raise SnapshotTampered(_NO_HASH)
|
||||
fingerprint: Fingerprint | None = None
|
||||
if expected:
|
||||
# 在锁外核对:哈希一个大文件要几秒,不能让别的数据源都排在它后面
|
||||
fingerprint = await self._verify_once(key, source, expected)
|
||||
async with self._lock:
|
||||
cached = self._hit(key, doomed)
|
||||
if cached is not None:
|
||||
return cached
|
||||
url, extra = engine_args(source)
|
||||
engine = create_async_engine(
|
||||
url,
|
||||
pool_size=3,
|
||||
max_overflow=2,
|
||||
pool_pre_ping=True, # 长时间空闲后连接会被数据库掐掉,先探活再用
|
||||
pool_recycle=1800,
|
||||
**extra,
|
||||
)
|
||||
if fingerprint is not None:
|
||||
pin = _Pin(str(source.database or ""), fingerprint)
|
||||
_guard_pool(engine, pin, key)
|
||||
self._pins[key] = pin
|
||||
self._engines[key] = engine
|
||||
limit = max(1, self._max if self._max is not None else MAX_ENGINES)
|
||||
while len(self._engines) > limit:
|
||||
old_key, old = self._engines.popitem(last=False)
|
||||
self._pins.pop(old_key, None)
|
||||
doomed.append(old)
|
||||
return engine
|
||||
while True:
|
||||
epoch = self._epochs.get(sid, 0)
|
||||
fingerprint: Fingerprint | None = None
|
||||
if expected:
|
||||
# 在锁外核对:哈希一个大文件要几秒,不能让别的数据源都排在它后面
|
||||
fingerprint = await self._verify_once(key, source, expected)
|
||||
async with self._lock:
|
||||
cached = self._hit(key, doomed)
|
||||
if cached is not None:
|
||||
return cached
|
||||
if self._epochs.get(sid, 0) != epoch:
|
||||
continue
|
||||
url, extra = engine_args(source)
|
||||
is_memory_sqlite = (source.kind or "").lower() == "sqlite" and (
|
||||
(source.database or "") == ":memory:" or url.endswith("/:memory:")
|
||||
)
|
||||
pool_kwargs: dict[str, Any] = {} if is_memory_sqlite else {
|
||||
"pool_size": 3,
|
||||
"max_overflow": 2,
|
||||
"pool_pre_ping": True, # 长时间空闲后连接会被数据库掐掉,先探活再用
|
||||
"pool_recycle": 1800,
|
||||
}
|
||||
engine = create_async_engine(
|
||||
url,
|
||||
**pool_kwargs,
|
||||
**extra,
|
||||
)
|
||||
if fingerprint is not None:
|
||||
pin = _Pin(str(source.database or ""), fingerprint)
|
||||
_guard_pool(engine, pin, key)
|
||||
self._pins[key] = pin
|
||||
self._engines[key] = engine
|
||||
limit = max(1, self._max if self._max is not None else MAX_ENGINES)
|
||||
while len(self._engines) > limit:
|
||||
old_key, old = self._engines.popitem(last=False)
|
||||
self._pins.pop(old_key, None)
|
||||
doomed.append(old)
|
||||
return engine
|
||||
finally:
|
||||
for old in doomed:
|
||||
# 正被借出的连接不受影响:dispose 只关池子里空闲的,借出的还回来时随旧池子回收
|
||||
@@ -653,22 +666,27 @@ class EngineCache:
|
||||
from app.cluster import caches
|
||||
|
||||
await caches.broadcast("datasource_engine", sid)
|
||||
# 正在进行的核对也撇开:失效之后来的请求重新核对,不搭失效之前那一次的便车
|
||||
for token in [t for t in self._verifying if _source_of(t[0]) == sid]:
|
||||
del self._verifying[token]
|
||||
keys = [key for key in list(self._engines) if _source_of(key) == sid]
|
||||
doomed = [self._engines.pop(key) for key in keys]
|
||||
for key in keys:
|
||||
self._pins.pop(key, None)
|
||||
async with self._lock:
|
||||
self._epochs[sid] = self._epochs.get(sid, 0) + 1
|
||||
# 正在进行的核对也撇开:失效之后来的请求重新核对,不搭失效之前那一次的便车
|
||||
for token in [t for t in self._verifying if _source_of(t[0]) == sid]:
|
||||
del self._verifying[token]
|
||||
keys = [key for key in list(self._engines) if _source_of(key) == sid]
|
||||
doomed = [self._engines.pop(key) for key in keys]
|
||||
for key in keys:
|
||||
self._pins.pop(key, None)
|
||||
for engine in doomed:
|
||||
await engine.dispose()
|
||||
|
||||
async def close(self) -> None:
|
||||
self._verifying.clear()
|
||||
for engine in list(self._engines.values()):
|
||||
async with self._lock:
|
||||
self._verifying.clear()
|
||||
doomed = list(self._engines.values())
|
||||
self._engines.clear()
|
||||
self._pins.clear()
|
||||
self._epochs.clear()
|
||||
for engine in doomed:
|
||||
await engine.dispose()
|
||||
self._engines.clear()
|
||||
self._pins.clear()
|
||||
|
||||
|
||||
engines = EngineCache()
|
||||
@@ -743,10 +761,20 @@ class _Deadline:
|
||||
def __init__(self, seconds: float) -> None:
|
||||
self.seconds = seconds
|
||||
self.ends = time.monotonic() + seconds
|
||||
#: SQLite 的进度回调真的掐过
|
||||
#: SQLite 的进度回调真的按时限掐过
|
||||
self.fired = False
|
||||
#: 调用方主动取消(CancelledError 等),进度回调同样立即中止工作线程里的语句
|
||||
self.cancelled = False
|
||||
#: 语句已经结束。回调这时要是还挂在连接上(摘除没成功),也不能再掐别人的语句
|
||||
self.over = False
|
||||
#: 由 _server_deadline 登记的底层驱动打断回调(如 sqlite3.Connection.interrupt / oracledb.cancel)
|
||||
self._aborter: Callable[[], None] | None = None
|
||||
|
||||
def abort(self) -> None:
|
||||
self.cancelled = True
|
||||
if self._aborter is not None:
|
||||
with suppress(Exception):
|
||||
self._aborter()
|
||||
|
||||
def remaining_ms(self) -> int:
|
||||
return max(1, int((self.ends - time.monotonic()) * 1000))
|
||||
@@ -759,7 +787,11 @@ class _Deadline:
|
||||
|
||||
def sqlite_progress(self) -> int:
|
||||
# 在 SQLite 执行语句的那个线程里被调用;返回非零,SQLite 就中止当前语句
|
||||
if not self.over and time.monotonic() >= self.ends:
|
||||
if self.over:
|
||||
return 0
|
||||
if self.cancelled:
|
||||
return 1
|
||||
if time.monotonic() >= self.ends:
|
||||
self.fired = True
|
||||
return 1
|
||||
return 0
|
||||
@@ -843,9 +875,16 @@ async def _server_deadline(conn: AsyncConnection, kind: str, deadline: _Deadline
|
||||
logger.warning("Oracle call_timeout 没设上(%s),只剩后端兜底", e)
|
||||
yield
|
||||
return
|
||||
cancel_fn = getattr(driver, "cancel", None) or getattr(driver, "breakexecution", None)
|
||||
if callable(cancel_fn):
|
||||
deadline._aborter = cancel_fn
|
||||
try:
|
||||
yield
|
||||
except asyncio.CancelledError:
|
||||
deadline.abort()
|
||||
raise
|
||||
finally:
|
||||
deadline._aborter = None
|
||||
try:
|
||||
driver.call_timeout = before
|
||||
except Exception: # noqa: BLE001 - 超时后连接可能已经被驱动关掉
|
||||
@@ -854,15 +893,23 @@ async def _server_deadline(conn: AsyncConnection, kind: str, deadline: _Deadline
|
||||
|
||||
if kind == "sqlite":
|
||||
driver = await _driver(conn)
|
||||
raw_conn = getattr(driver, "_conn", None)
|
||||
if raw_conn is not None:
|
||||
deadline._aborter = raw_conn.interrupt
|
||||
await driver.set_progress_handler(deadline.sqlite_progress, _SQLITE_STEPS)
|
||||
try:
|
||||
yield
|
||||
except BaseException:
|
||||
deadline.abort()
|
||||
raise
|
||||
finally:
|
||||
deadline.over = True
|
||||
deadline._aborter = None
|
||||
try:
|
||||
await driver.set_progress_handler(None, 0)
|
||||
except Exception: # noqa: BLE001 - over 已经让它不再掐人
|
||||
except Exception: # noqa: BLE001 - over 保底不再掐后面的语句
|
||||
pass
|
||||
finally:
|
||||
deadline.over = True
|
||||
return
|
||||
|
||||
yield
|
||||
@@ -887,7 +934,22 @@ async def _bounded(
|
||||
if deadline.expired():
|
||||
raise SqlRejected(over)
|
||||
async with _server_deadline(conn, kind, deadline):
|
||||
return await work(conn)
|
||||
# 用 shield 保护内层 greenlet:收到 CancelledError 时先经 deadline.abort() 通知底层驱动
|
||||
# (如 sqlite3_interrupt)正常中止查询,避免 CancelledError 直接打进 SQLAlchemy greenlet
|
||||
# 触发半关闭连接上的 rollback ValueError("Connection closed") 与连接池泄漏。
|
||||
work_task = asyncio.create_task(work(conn))
|
||||
try:
|
||||
return await asyncio.shield(work_task)
|
||||
except asyncio.CancelledError:
|
||||
deadline.abort()
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.shield(work_task), timeout=1.0)
|
||||
except BaseException:
|
||||
if not work_task.done():
|
||||
work_task.cancel()
|
||||
with suppress(BaseException):
|
||||
await work_task
|
||||
raise
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(_exec(), timeout=limits.timeout_seconds + BACKSTOP_S)
|
||||
|
||||
@@ -8496,47 +8496,182 @@ def _slot_of(expr: str, index: int, context: str) -> TemplateSlot:
|
||||
return TemplateSlot(index=index, field=found.group(1), filters=filters, context=context)
|
||||
|
||||
|
||||
def _template_slots(template: str) -> tuple[list[tuple[int, int, TemplateSlot]], str]:
|
||||
"""模板里每一处 {{ … }}:[(起, 止, 槽位)],以及按哨兵渲染的 SQL。逐字看引号和注释,定下每一处在哪:单引号里是字,
|
||||
双引号、反引号里是名字,-- 和 /* */ 里是注释,别处是裸写。{{ … }} 里面的字不参与判断。"""
|
||||
_DOLLAR_QUOTE_RE = re.compile(r"\$(?:[^\W\d]\w*)?\$")
|
||||
_ORACLE_Q_CLOSE = {"[": "]", "(": ")", "{": "}", "<": ">"}
|
||||
|
||||
|
||||
def _is_word_char(ch: str) -> bool:
|
||||
return ch.isalnum() or ch in "_$"
|
||||
|
||||
|
||||
def _is_e_string_at(sql: str, i: int) -> bool:
|
||||
return i >= 1 and sql[i - 1] in "eE" and not (i >= 2 and _is_word_char(sql[i - 2]))
|
||||
|
||||
|
||||
def _oracle_q_close_at(sql: str, i: int) -> str | None:
|
||||
if i + 1 >= len(sql) or sql[i + 1].isspace() or not (i >= 1 and sql[i - 1] in "qQ"):
|
||||
return None
|
||||
j = i - 2
|
||||
if j >= 0 and sql[j] in "nN":
|
||||
j -= 1
|
||||
if j >= 0 and _is_word_char(sql[j]):
|
||||
return None
|
||||
opener = sql[i + 1]
|
||||
return _ORACLE_Q_CLOSE.get(opener, opener)
|
||||
|
||||
|
||||
def _is_bracket_ident_at(sql: str, i: int, dialect_key: str) -> bool:
|
||||
if dialect_key == "sqlite":
|
||||
return True
|
||||
if dialect_key in ("postgres", "postgresql", "mysql", "mariadb", "oracle"):
|
||||
return False
|
||||
j = i - 1
|
||||
while j >= 0 and sql[j].isspace():
|
||||
j -= 1
|
||||
return j < 0 or not (_is_word_char(sql[j]) or sql[j] in ")]")
|
||||
|
||||
|
||||
def _template_slots(
|
||||
template: str, *, dialect: str | None = None,
|
||||
) -> tuple[list[tuple[int, int, TemplateSlot]], str]:
|
||||
"""模板里每一处 {{ … }}:[(起, 止, 槽位)],以及按哨兵渲染的 SQL。
|
||||
|
||||
按方言词法逐字扫描引号与注释,定下每一处在哪:
|
||||
- 单引号(含 PostgreSQL E'…'、MySQL 反斜杠转义、PostgreSQL $$…$$ / $tag$…$tag$、Oracle q'[…]')里是字(quoted);
|
||||
- 双引号、反引号、SQLite 方括号 `[…]` 里是名字(name;MySQL 默认模式下双引号是字符串);
|
||||
- `--`、`/* … */`(PostgreSQL 支持嵌套)与 MySQL `#` 里是注释(comment);
|
||||
- 其余代码位置是裸写(bare)。`{{ … }}` 里面的字不参与外层引号/注释状态切换。
|
||||
"""
|
||||
dialect_key = (dialect or "").strip().lower()
|
||||
is_mysql = dialect_key in ("mysql", "mariadb")
|
||||
is_pg = dialect_key in ("postgres", "postgresql")
|
||||
allow_dollar = dialect_key in ("", "postgres", "postgresql")
|
||||
allow_q_quote = dialect_key in ("", "oracle")
|
||||
|
||||
spans: list[tuple[int, int, TemplateSlot]] = []
|
||||
out: list[str] = []
|
||||
state = "code"
|
||||
quote = ""
|
||||
backslash_escape = False
|
||||
dollar_tag = ""
|
||||
q_close = ""
|
||||
block_depth = 0
|
||||
|
||||
i, n = 0, len(template)
|
||||
while i < n:
|
||||
if template.startswith("{{", i):
|
||||
m = _SLOT_RE.match(template, i)
|
||||
if m is not None:
|
||||
context = {"code": "bare", "string": "quoted", "name": "name", "line": "comment",
|
||||
"block": "comment"}[state]
|
||||
context = {
|
||||
"code": "bare",
|
||||
"string": "quoted",
|
||||
"dollar": "quoted",
|
||||
"q_string": "quoted",
|
||||
"name": "name",
|
||||
"bracket_name": "name",
|
||||
"line": "comment",
|
||||
"block": "comment",
|
||||
}[state]
|
||||
slot = _slot_of(m.group(1).strip(), len(spans), context)
|
||||
spans.append((m.start(), m.end(), slot))
|
||||
out.append(_STRING_SENTINEL.format(slot.index) if context == "quoted"
|
||||
else str(_NUMBER_SENTINEL + slot.index) if context == "bare" else "")
|
||||
out.append(
|
||||
_STRING_SENTINEL.format(slot.index)
|
||||
if context == "quoted"
|
||||
else str(_NUMBER_SENTINEL + slot.index)
|
||||
if context == "bare"
|
||||
else ""
|
||||
)
|
||||
i = m.end()
|
||||
continue
|
||||
ch = template[i]
|
||||
if state == "code":
|
||||
if ch == "'":
|
||||
q_closer = _oracle_q_close_at(template, i) if allow_q_quote else None
|
||||
if q_closer is not None:
|
||||
state = "q_string"
|
||||
q_close = q_closer
|
||||
out.append(template[i:i + 2])
|
||||
i += 2
|
||||
continue
|
||||
state = "string"
|
||||
elif ch in "\"`":
|
||||
quote = "'"
|
||||
backslash_escape = is_mysql or _is_e_string_at(template, i)
|
||||
elif ch == '"' and is_mysql:
|
||||
state = "string"
|
||||
quote = '"'
|
||||
backslash_escape = True
|
||||
elif ch in '"`':
|
||||
state = "name"
|
||||
quote = ch
|
||||
elif ch == "[" and _is_bracket_ident_at(template, i, dialect_key):
|
||||
state = "bracket_name"
|
||||
elif allow_dollar and ch == "$" and (dm := _DOLLAR_QUOTE_RE.match(template, i)) is not None:
|
||||
state = "dollar"
|
||||
dollar_tag = dm.group(0)
|
||||
out.append(dollar_tag)
|
||||
i = dm.end()
|
||||
continue
|
||||
elif template.startswith("--", i):
|
||||
state = "line"
|
||||
out.append("--")
|
||||
i += 2
|
||||
continue
|
||||
elif is_mysql and ch == "#":
|
||||
state = "line"
|
||||
elif template.startswith("/*", i):
|
||||
state = "block"
|
||||
elif state == "string" and ch == "'":
|
||||
state = "code" # 成对的 '' 是转义:关上又马上打开,结果一样
|
||||
elif state == "name" and ch == quote:
|
||||
state = "code"
|
||||
block_depth = 1
|
||||
out.append("/*")
|
||||
i += 2
|
||||
continue
|
||||
elif state == "string":
|
||||
if backslash_escape and ch == "\\" and i + 1 < n:
|
||||
out.append(template[i:i + 2])
|
||||
i += 2
|
||||
continue
|
||||
if ch == quote:
|
||||
if i + 1 < n and template[i + 1] == quote:
|
||||
out.append(quote * 2)
|
||||
i += 2
|
||||
continue
|
||||
state = "code"
|
||||
elif state == "dollar":
|
||||
if template.startswith(dollar_tag, i):
|
||||
out.append(dollar_tag)
|
||||
i += len(dollar_tag)
|
||||
state = "code"
|
||||
continue
|
||||
elif state == "q_string":
|
||||
if ch == q_close and i + 1 < n and template[i + 1] == "'":
|
||||
out.append(template[i:i + 2])
|
||||
i += 2
|
||||
state = "code"
|
||||
continue
|
||||
elif state == "name":
|
||||
if ch == quote:
|
||||
if i + 1 < n and template[i + 1] == quote:
|
||||
out.append(quote * 2)
|
||||
i += 2
|
||||
continue
|
||||
state = "code"
|
||||
elif state == "bracket_name":
|
||||
if ch == "]":
|
||||
state = "code"
|
||||
elif state == "line" and ch == "\n":
|
||||
state = "code"
|
||||
elif state == "block" and template.startswith("*/", i):
|
||||
out.append("*/")
|
||||
i += 2
|
||||
state = "code"
|
||||
continue
|
||||
elif state == "block":
|
||||
if is_pg and template.startswith("/*", i):
|
||||
block_depth += 1
|
||||
out.append("/*")
|
||||
i += 2
|
||||
continue
|
||||
if template.startswith("*/", i):
|
||||
out.append("*/")
|
||||
i += 2
|
||||
block_depth -= 1
|
||||
if block_depth <= 0:
|
||||
state = "code"
|
||||
continue
|
||||
out.append(ch)
|
||||
i += 1
|
||||
return spans, "".join(out)
|
||||
@@ -8734,7 +8869,7 @@ def _bindings(template_sql: str, sql: str, read: str | None, info: _Schema
|
||||
) -> Iterator[tuple[str, dict[str, Any], tuple[str, ...], tuple[TemplateSlot, ...]]]:
|
||||
"""按 BINDING_LEVELS 的先后给出配得上的绑定:(哪一级, {字段: 值}, 没取到值的字段, 槽位),同样的绑定只给一次。
|
||||
模板没有槽位时只给一个空绑定(level 为 parsed:用不着配)。不可比抛 _NotComparable。"""
|
||||
spans, sentinel_sql = _template_slots(template_sql)
|
||||
spans, sentinel_sql = _template_slots(template_sql, dialect=read)
|
||||
slots = tuple(slot for _, _, slot in spans)
|
||||
fields = tuple(dict.fromkeys(s.field for s in slots))
|
||||
if not slots:
|
||||
@@ -8839,7 +8974,7 @@ def bind_template(template_sql: str, slots: Any, sql: str, dialect: str | None =
|
||||
try:
|
||||
for level, values, unbound, found in _bindings(template_sql or "", sql or "", read, info):
|
||||
return TemplateBinding(True, values, unbound, level, None, found)
|
||||
spans, _ = _template_slots(template_sql or "")
|
||||
spans, _ = _template_slots(template_sql or "", dialect=read)
|
||||
except _NotComparable as e:
|
||||
return TemplateBinding(False, reason=str(e))
|
||||
except Exception: # noqa: BLE001 - 绑定本身的缺陷不能连累调用方:当作没配上
|
||||
@@ -8924,7 +9059,7 @@ def match_template(template_sql: str, slots: Any, sql: str, dialect: str | None
|
||||
level, values, unbound, diff, rendered = tried[0]
|
||||
return TemplateMatch(False, binding=values, unbound=unbound, level=level, diff=diff, rendered=rendered)
|
||||
try:
|
||||
spans, _ = _template_slots(template_sql)
|
||||
spans, _ = _template_slots(template_sql, dialect=read)
|
||||
except _NotComparable as e:
|
||||
return TemplateMatch(False, comparable=False, reason=str(e))
|
||||
fields = tuple(dict.fromkeys(slot.field for _, _, slot in spans))
|
||||
@@ -8954,7 +9089,7 @@ def _now_slots(template_sql: str, sql: str, read: str | None) -> list[tuple[str,
|
||||
from sqlglot import exp
|
||||
|
||||
try:
|
||||
spans, sentinel = _template_slots(template_sql)
|
||||
spans, sentinel = _template_slots(template_sql, dialect=read)
|
||||
except _NotComparable:
|
||||
return []
|
||||
by_index = {slot.index: slot for _, _, slot in spans}
|
||||
@@ -9344,7 +9479,7 @@ def project_template(template_sql: str, keep: Sequence[int], dialect: str | None
|
||||
template = (template_sql or "").strip()
|
||||
keep = tuple(keep)
|
||||
try:
|
||||
slots, sentinel = _template_slots(template)
|
||||
slots, sentinel = _template_slots(template, dialect=read)
|
||||
root, spans, expected = _plan_projection(sentinel, keep, read, None)
|
||||
items = list(root.expressions)
|
||||
names = tuple(_output_name(items[k], read) for k in keep)
|
||||
@@ -9358,7 +9493,7 @@ def project_template(template_sql: str, keep: Sequence[int], dialect: str | None
|
||||
raise _NotProjectable(_PROJ_TEXT)
|
||||
out = _cut(template, mapped, keep) # type: ignore[arg-type]
|
||||
# 换回模板之后再按哨兵渲染一遍:结构(字面量只看位置,槽位的编号会因为去掉了几处而变)要和投影后的一样
|
||||
_, again = _template_slots(out)
|
||||
_, again = _template_slots(out, dialect=read)
|
||||
root_again, _ = _parse(again, read)
|
||||
lits_a: list[Any] = []
|
||||
lits_b: list[Any] = []
|
||||
@@ -9420,7 +9555,7 @@ def output_names(sql: str, dialect: str | None = None, *, template: bool = False
|
||||
text = (sql or "").strip()
|
||||
try:
|
||||
if template:
|
||||
_, text = _template_slots(text)
|
||||
_, text = _template_slots(text, dialect=read)
|
||||
root, _ = _parse(_mysql_dashes(text) if read == "mysql" else text, read)
|
||||
except _NotComparable:
|
||||
return None
|
||||
|
||||
+48
-16
@@ -28,6 +28,7 @@ _ALWAYS_DENIED 里有 create,那是对的——模型不该能建表。而导
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import codecs
|
||||
import csv
|
||||
import datetime as _dt
|
||||
import io
|
||||
@@ -865,33 +866,64 @@ def _close(it: Any) -> None:
|
||||
_ENCODINGS = ("utf-8-sig", "utf-8", "gb18030", "latin-1")
|
||||
|
||||
|
||||
def _decode_csv(raw: bytes) -> str:
|
||||
# 带 BOM 的 UTF-16(Excel 的「Unicode 文本」):按 latin-1 读会混进 NUL,列名全乱
|
||||
if raw[:2] in (b"\xff\xfe", b"\xfe\xff"):
|
||||
try:
|
||||
return raw.decode("utf-16")
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
_CSV_DECODE_CHUNK = 1024 * 1024
|
||||
|
||||
|
||||
def _can_decode_csv(raw: bytes, enc: str) -> bool:
|
||||
dec = codecs.getincrementaldecoder(enc)()
|
||||
mv = memoryview(raw)
|
||||
try:
|
||||
for offset in range(0, len(mv), _CSV_DECODE_CHUNK):
|
||||
end = offset + _CSV_DECODE_CHUNK
|
||||
dec.decode(mv[offset:end], final=end >= len(mv))
|
||||
if not len(mv):
|
||||
dec.decode(b"", final=True)
|
||||
return True
|
||||
except UnicodeDecodeError:
|
||||
return False
|
||||
|
||||
|
||||
def _detect_csv_encoding(raw: bytes) -> tuple[str, str]:
|
||||
"""增量分块探测 CSV 编码,避免为判编码而一次性物化整份全量字符串。"""
|
||||
if raw[:2] in (b"\xff\xfe", b"\xfe\xff") and _can_decode_csv(raw, "utf-16"):
|
||||
return "utf-16", "strict"
|
||||
for enc in _ENCODINGS:
|
||||
try:
|
||||
return raw.decode(enc)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
return raw.decode("latin-1", errors="replace")
|
||||
if _can_decode_csv(raw, enc):
|
||||
return enc, "strict"
|
||||
return "latin-1", "replace"
|
||||
|
||||
|
||||
def _open_csv_text(
|
||||
raw: bytes,
|
||||
*,
|
||||
encoding: str | None = None,
|
||||
errors: str = "strict",
|
||||
newline: str | None = "",
|
||||
) -> io.TextIOWrapper:
|
||||
if encoding is None:
|
||||
encoding, errors = _detect_csv_encoding(raw)
|
||||
return io.TextIOWrapper(io.BytesIO(raw), encoding=encoding, errors=errors, newline=newline)
|
||||
|
||||
|
||||
def _decode_csv(raw: bytes) -> str:
|
||||
enc, errors = _detect_csv_encoding(raw)
|
||||
return raw.decode(enc, errors=errors)
|
||||
|
||||
|
||||
def _csv_sources(raw: bytes, filename: str, header_row: int) -> list[_Source]:
|
||||
text = _decode_csv(raw)
|
||||
enc, errors = _detect_csv_encoding(raw)
|
||||
delimiter = "\t" if _ext(filename) == ".tsv" else None
|
||||
if delimiter is None:
|
||||
with _open_csv_text(raw, encoding=enc, errors=errors, newline="") as probe_stream:
|
||||
head = probe_stream.read(8192)
|
||||
try:
|
||||
delimiter = csv.Sniffer().sniff(text[:8192], delimiters=",;\t|").delimiter
|
||||
delimiter = csv.Sniffer().sniff(head, delimiters=",;\t|").delimiter
|
||||
except csv.Error:
|
||||
delimiter = "," # 嗅不出来就按最常见的来
|
||||
|
||||
def rows() -> Iterator[Sequence[Any]]:
|
||||
reader = csv.reader(io.StringIO(text), delimiter=delimiter)
|
||||
return itertools.islice(reader, header_row - 1, None)
|
||||
with _open_csv_text(raw, encoding=enc, errors=errors, newline="") as stream:
|
||||
yield from itertools.islice(csv.reader(stream, delimiter=delimiter), header_row - 1, None)
|
||||
|
||||
stem = filename.rsplit("/", 1)[-1].rsplit(".", 1)[0]
|
||||
# 分号分隔是欧洲区域设置的 CSV:那里逗号是小数点,「1,500」是 1.5 而不是一千五
|
||||
|
||||
@@ -145,6 +145,8 @@ def _set_sqlite_pragma(dbapi_conn, _record): # pragma: no cover - 驱动回调
|
||||
"""WAL + 外键约束。WAL 让读写不互相阻塞,事件写入和前端轮询能并行。"""
|
||||
cur = dbapi_conn.cursor()
|
||||
cur.execute("PRAGMA journal_mode=WAL")
|
||||
cur.execute("PRAGMA synchronous=NORMAL")
|
||||
cur.execute("PRAGMA journal_size_limit=67108864")
|
||||
cur.execute("PRAGMA foreign_keys=ON")
|
||||
cur.execute("PRAGMA busy_timeout=5000")
|
||||
cur.close()
|
||||
@@ -514,6 +516,17 @@ _INDEX_MIGRATIONS: list[tuple[str, str, str | tuple[str, ...]]] = [
|
||||
("ix_artifacts_kind", "artifacts", "kind"),
|
||||
# 增量重导入的工作项按持久对齐键认(K11):候选顺序、条数变了也要认得出同一个工作项
|
||||
("ix_caliber_import_items_align_key", "caliber_import_items", "align_key"),
|
||||
# 热点翻页、跨库检索与级联外键索引:避免列表全表排序和删除父表时全表扫子表
|
||||
("ix_runs_created_id", "runs", ("created_at", "id")),
|
||||
("ix_documents_col_created", "documents", ("collection", "created_at", "id")),
|
||||
("ix_documents_created_id", "documents", ("created_at", "id")),
|
||||
("ix_chunks_document_id", "chunks", "document_id"),
|
||||
("ix_chunk_terms_term", "chunk_terms", "term"),
|
||||
("ix_conversation_turns_run_id", "conversation_turns", "run_id"),
|
||||
("ix_caliber_drafts_datasource_id", "caliber_drafts", "datasource_id"),
|
||||
("ix_caliber_drafts_workflow_id", "caliber_drafts", "workflow_id"),
|
||||
("ix_caliber_write_requests_workflow_id", "caliber_write_requests", "workflow_id"),
|
||||
("ix_trigger_deliveries_event_id", "trigger_deliveries", "event_id"),
|
||||
]
|
||||
|
||||
|
||||
|
||||
+23
-10
@@ -602,7 +602,7 @@ class Workflow(Base, TimestampMixin):
|
||||
origin: Mapped[dict[str, Any] | None] = mapped_column(default=None)
|
||||
|
||||
versions: Mapped[list["WorkflowVersion"]] = relationship(
|
||||
back_populates="workflow", cascade="all, delete-orphan", lazy="selectin"
|
||||
back_populates="workflow", cascade="all, delete-orphan", lazy="select", passive_deletes=True
|
||||
)
|
||||
|
||||
|
||||
@@ -649,8 +649,11 @@ class WorkflowVersion(Base, TimestampMixin):
|
||||
|
||||
class Run(Base, TimestampMixin):
|
||||
__tablename__ = "runs"
|
||||
# 找钉住版本最近一次封存的运行(应用的视图、口径对比)。存量库由 db/base.py 的 _INDEX_MIGRATIONS 补建
|
||||
__table_args__ = (Index("ix_runs_wf_ver_status", "workflow_id", "version", "status"),)
|
||||
# 找钉住版本最近一次封存的运行(应用的视图、口径对比)与列表游标翻页。存量库由 db/base.py 的 _INDEX_MIGRATIONS 补建
|
||||
__table_args__ = (
|
||||
Index("ix_runs_wf_ver_status", "workflow_id", "version", "status"),
|
||||
Index("ix_runs_created_id", "created_at", "id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id)
|
||||
workflow_id: Mapped[str | None] = mapped_column(
|
||||
@@ -765,7 +768,10 @@ class Artifact(Base, TimestampMixin):
|
||||
"""
|
||||
|
||||
__tablename__ = "artifacts"
|
||||
__table_args__ = (Index("ix_artifacts_run", "run_id"),)
|
||||
__table_args__ = (
|
||||
Index("ix_artifacts_run", "run_id"),
|
||||
Index("ix_artifacts_kind", "kind"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(32), primary_key=True, default="")
|
||||
@@ -806,6 +812,10 @@ class Document(Base, TimestampMixin):
|
||||
"""知识库里的一份原始文档。"""
|
||||
|
||||
__tablename__ = "documents"
|
||||
__table_args__ = (
|
||||
Index("ix_documents_col_created", "collection", "created_at", "id"),
|
||||
Index("ix_documents_created_id", "created_at", "id"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id)
|
||||
collection: Mapped[str] = mapped_column(String(100), default="default", index=True)
|
||||
@@ -833,7 +843,7 @@ class Chunk(Base):
|
||||
__table_args__ = (Index("ix_chunks_collection", "collection"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id)
|
||||
document_id: Mapped[str] = mapped_column(ForeignKey("documents.id", ondelete="CASCADE"))
|
||||
document_id: Mapped[str] = mapped_column(ForeignKey("documents.id", ondelete="CASCADE"), index=True)
|
||||
collection: Mapped[str] = mapped_column(String(100), default="default")
|
||||
ordinal: Mapped[int] = mapped_column(Integer, default=0)
|
||||
content: Mapped[str] = mapped_column(Text)
|
||||
@@ -863,7 +873,10 @@ class ChunkTerm(Base):
|
||||
"""
|
||||
|
||||
__tablename__ = "chunk_terms"
|
||||
__table_args__ = (Index("ix_chunk_terms_lookup", "collection", "term"),)
|
||||
__table_args__ = (
|
||||
Index("ix_chunk_terms_lookup", "collection", "term"),
|
||||
Index("ix_chunk_terms_term", "term"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
chunk_id: Mapped[str] = mapped_column(
|
||||
@@ -964,7 +977,7 @@ class ConversationTurn(Base, TimestampMixin):
|
||||
explanation: Mapped[str] = mapped_column(Text, default="")
|
||||
graph: Mapped[dict[str, Any] | None] = mapped_column(default=None)
|
||||
run_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("runs.id", ondelete="SET NULL"), default=None
|
||||
ForeignKey("runs.id", ondelete="SET NULL"), default=None, index=True
|
||||
)
|
||||
# running | done | error
|
||||
status: Mapped[str] = mapped_column(String(20), default="running")
|
||||
@@ -1123,7 +1136,7 @@ class CaliberDraft(Base, TimestampMixin):
|
||||
collection: Mapped[str] = mapped_column(String(100), default="default")
|
||||
# 起草时给模型看的数据源:表结构和数据目录是取数 SQL 的唯一依据;源删了草稿照旧在,只丢这一条线索
|
||||
datasource_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("data_sources.id", ondelete="SET NULL"), default=None
|
||||
ForeignKey("data_sources.id", ondelete="SET NULL"), default=None, index=True
|
||||
)
|
||||
# 同 docdraft.STATUSES:drafting | ready | questions_open | path_pending | attached | rejected | failed
|
||||
status: Mapped[str] = mapped_column(String(20), default="drafting", index=True)
|
||||
@@ -1131,7 +1144,7 @@ class CaliberDraft(Base, TimestampMixin):
|
||||
# 这份草稿从哪来:{"documents": [文档 id], "guidance": 用户补充要求}
|
||||
origin: Mapped[dict[str, Any]] = mapped_column(default=dict)
|
||||
workflow_id: Mapped[str | None] = mapped_column(
|
||||
ForeignKey("workflows.id", ondelete="SET NULL"), default=None
|
||||
ForeignKey("workflows.id", ondelete="SET NULL"), default=None, index=True
|
||||
)
|
||||
# 起草失败的说法(没有可用模型、模型回的东西取不出 JSON……)。草稿不置成 rejected:人改完还能再试
|
||||
error: Mapped[str | None] = mapped_column(Text, default=None)
|
||||
@@ -1153,7 +1166,7 @@ class CaliberWriteRequest(Base, TimestampMixin):
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id)
|
||||
#: 发起这次写入的主体(开登录时是认证过的用户名,关登录时是自报署名,照实记)
|
||||
actor: Mapped[str] = mapped_column(String(120), default="")
|
||||
workflow_id: Mapped[str] = mapped_column(ForeignKey("workflows.id", ondelete="CASCADE"))
|
||||
workflow_id: Mapped[str] = mapped_column(ForeignKey("workflows.id", ondelete="CASCADE"), index=True)
|
||||
#: 这次请求针对的那张口径卡;只改标题这类不碰图的请求记空串(列非空,唯一键才认得住)
|
||||
card_id: Mapped[str] = mapped_column(String(64), default="")
|
||||
request_id: Mapped[str] = mapped_column(String(64))
|
||||
|
||||
@@ -85,7 +85,7 @@ class TriggerDelivery(Base):
|
||||
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id)
|
||||
schedule_id: Mapped[str] = mapped_column(ForeignKey("schedules.id", ondelete="CASCADE"))
|
||||
event_id: Mapped[str] = mapped_column(ForeignKey("trigger_events.id", ondelete="CASCADE"))
|
||||
event_id: Mapped[str] = mapped_column(ForeignKey("trigger_events.id", ondelete="CASCADE"), index=True)
|
||||
#: 同事件的发生时刻(去抖窗口按它算)
|
||||
occurred_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow)
|
||||
status: Mapped[str] = mapped_column(String(20), default="pending")
|
||||
|
||||
@@ -11,7 +11,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Iterable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -64,6 +64,12 @@ async def open_sqlite_saver(path: Path | str) -> AsyncIterator[BaseCheckpointSav
|
||||
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
|
||||
|
||||
async with AsyncSqliteSaver.from_conn_string(str(path)) as saver:
|
||||
conn = getattr(saver, "conn", None)
|
||||
if conn is not None:
|
||||
await conn.execute("PRAGMA journal_mode=WAL")
|
||||
await conn.execute("PRAGMA synchronous=NORMAL")
|
||||
await conn.execute("PRAGMA journal_size_limit=67108864")
|
||||
await conn.execute("PRAGMA busy_timeout=5000")
|
||||
yield saver
|
||||
|
||||
|
||||
@@ -75,6 +81,39 @@ def open_saver() -> contextlib.AbstractAsyncContextManager[BaseCheckpointSaver]:
|
||||
return open_sqlite_saver(settings.checkpoint_path)
|
||||
|
||||
|
||||
async def _delete_on(saver: Any, ids: list[str]) -> None:
|
||||
adelete = getattr(saver, "adelete_thread", None)
|
||||
if not callable(adelete):
|
||||
return
|
||||
setup = getattr(saver, "setup", None)
|
||||
if callable(setup):
|
||||
with contextlib.suppress(Exception):
|
||||
await setup()
|
||||
for tid in ids:
|
||||
with contextlib.suppress(Exception):
|
||||
await adelete(tid)
|
||||
|
||||
|
||||
async def delete_threads(thread_ids: Iterable[str]) -> None:
|
||||
"""运行被彻底删除或评测明细被回收时,一并清理断点库里对应的 thread。
|
||||
|
||||
注意:正常运行结束(succeeded / cancelled)时不能删,因为 GET /api/runs/{id}/state 和 /checkpoints
|
||||
还要回看历史断点;只有在运行记录本身从数据库删除之后才调这里。
|
||||
"""
|
||||
ids = sorted({str(t) for t in thread_ids if t})
|
||||
if not ids:
|
||||
return
|
||||
from app.engine.runner import run_manager
|
||||
|
||||
saver = run_manager._checkpointer
|
||||
if saver is not None:
|
||||
await _delete_on(saver, ids)
|
||||
return
|
||||
if settings.checkpoint_conninfo or settings.checkpoint_path.is_file():
|
||||
async with open_saver() as opened:
|
||||
await _delete_on(opened, ids)
|
||||
|
||||
|
||||
async def count(saver: Any, thread_id: str | None = None) -> tuple[int, int]:
|
||||
"""(断点条数, pending writes 条数)。thread_id 为空数全部。copy-to-postgres 核对用,走公开接口。"""
|
||||
config = {"configurable": {"thread_id": thread_id}} if thread_id else None
|
||||
|
||||
@@ -11,6 +11,10 @@ from app.engine.evidence import ledger_enabled
|
||||
from app.engine.lineage import scoped_id as _scoped
|
||||
from app.analysis.contract import LEDGER_KIND
|
||||
|
||||
MASKED = "已遮罩"
|
||||
RUN_NOT_FOUND = "run_not_found"
|
||||
REPORT_NOT_FOUND = "evidence_report_not_found"
|
||||
|
||||
|
||||
def _fetch(artifact: str) -> tuple[Any, bool]:
|
||||
try:
|
||||
@@ -236,3 +240,157 @@ def metric_cards(sealed: Sealed) -> list[tuple[str, str, dict[str, Any]]]:
|
||||
if ok is True and isinstance(content, dict) and isinstance(content.get("metrics"), list):
|
||||
out.append((node_id, art, content))
|
||||
return out
|
||||
|
||||
|
||||
async def load_sealed(run_id: str) -> Sealed:
|
||||
"""读取封存范围;运行记录不存在时抛 404 CodedHTTPException(供引擎与领域层共用,无需引入 api/evidence)。"""
|
||||
from app.api.coded import CodedHTTPException
|
||||
|
||||
try:
|
||||
return await read_sealed(run_id)
|
||||
except LookupError:
|
||||
raise CodedHTTPException(404, "运行记录不存在", RUN_NOT_FOUND) from None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Report:
|
||||
node_id: str
|
||||
doc_artifact: str
|
||||
ok: bool | None
|
||||
repairs: int | None
|
||||
stats: dict[str, Any] | None
|
||||
doc: dict[str, Any] | None
|
||||
hash_ok: bool | None
|
||||
fields: list[str]
|
||||
claims: str | None = None
|
||||
|
||||
|
||||
def output_evidence(sealed: Sealed) -> list[dict[str, Any]]:
|
||||
output = sealed.payload_of(sealed.nodes_of("output")) or {}
|
||||
primary = output.get("_evidence")
|
||||
if not isinstance(primary, dict):
|
||||
return []
|
||||
return [primary, *[r for r in primary.get("others") or [] if isinstance(r, dict)]]
|
||||
|
||||
|
||||
def reports_of(sealed: Sealed, *, nested: bool = False) -> list[Report]:
|
||||
"""封存范围内每个报告节点最后一次的自查结果,按第一次出现的顺序。"""
|
||||
latest: dict[str, dict[str, Any]] = {}
|
||||
for _, etype, node_id, data in sealed.events:
|
||||
if (
|
||||
etype == "report.checked"
|
||||
and node_id
|
||||
and isinstance(data, dict)
|
||||
and data.get("doc_artifact")
|
||||
and (nested or "/" not in node_id)
|
||||
):
|
||||
latest[node_id] = data
|
||||
evidence = output_evidence(sealed)
|
||||
out: list[Report] = []
|
||||
for node_id, data in latest.items():
|
||||
artifact = str(data["doc_artifact"])
|
||||
try:
|
||||
doc = load(artifact)
|
||||
hash_ok: bool | None = True if doc is not None else None
|
||||
except ValueError:
|
||||
doc, hash_ok = None, False
|
||||
if isinstance(doc, dict) and (doc.get("node_id") != node_id or doc.get("run_id") != sealed.run_id):
|
||||
doc, hash_ok = None, False
|
||||
fields = [
|
||||
f
|
||||
for r in evidence
|
||||
if r.get("report_node") == node_id and r.get("doc_artifact") == artifact
|
||||
for f in r.get("fields") or []
|
||||
]
|
||||
out.append(
|
||||
Report(
|
||||
node_id=node_id,
|
||||
doc_artifact=artifact,
|
||||
ok=data.get("ok"),
|
||||
repairs=data.get("repairs"),
|
||||
stats=(doc or {}).get("stats") or data.get("stats"),
|
||||
doc=doc if isinstance(doc, dict) else None,
|
||||
hash_ok=hash_ok,
|
||||
fields=fields,
|
||||
claims=data.get("claims") if isinstance(data.get("claims"), str) else None,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def choose_report(reports: list[Report], wanted: str | None, sealed: Sealed) -> Report:
|
||||
"""从封存报告列表里选出目标报告(指定 wanted 优先,否则取出口 _evidence 主报告,再退回第一份)。"""
|
||||
from app.api.coded import CodedHTTPException
|
||||
|
||||
if wanted:
|
||||
chosen = next((r for r in reports if r.node_id == wanted), None)
|
||||
if chosen is None:
|
||||
raise CodedHTTPException(404, "本次运行中没有所选报告撰写节点的文档", REPORT_NOT_FOUND)
|
||||
return chosen
|
||||
primary = next(iter(output_evidence(sealed)), {})
|
||||
return next((r for r in reports if r.node_id == primary.get("report_node")), reports[0])
|
||||
|
||||
|
||||
class ShownText:
|
||||
"""把展示用 SQL、检查片段、合并警告里命中的运行遮罩值替换为「已遮罩」。"""
|
||||
|
||||
def __init__(self, secrets: Any) -> None:
|
||||
self._secrets = secrets
|
||||
self.changed = False
|
||||
|
||||
def text(self, value: Any) -> Any:
|
||||
if self._secrets is None or not isinstance(value, str) or not value:
|
||||
return value
|
||||
out = self._secrets.scrub(value, loose=True)
|
||||
self.changed = self.changed or out != value
|
||||
return out
|
||||
|
||||
def checks(self, checks: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{**c, **{k: self.text(c[k]) for k in ("message", "sql_excerpt") if isinstance(c.get(k), str)}}
|
||||
for c in checks
|
||||
]
|
||||
|
||||
def merge(self, view: dict[str, Any]) -> dict[str, Any]:
|
||||
view["sql"] = self.text(view.get("sql"))
|
||||
view["warnings"] = [{**w, "message": self.text(w.get("message"))} for w in view.get("warnings") or []]
|
||||
view["inputs"] = [
|
||||
{**i, **({"checks": self.checks(i["checks"])} if i.get("checks") else {})}
|
||||
for i in view.get("inputs") or []
|
||||
]
|
||||
return view
|
||||
|
||||
|
||||
def open_masks(sealed: Sealed | None = None) -> Any:
|
||||
"""获取请求级数据源遮罩上下文(通过统一边界暴露给领域层,兼容测试对证据遮罩构造器的桩替换)。"""
|
||||
from app.api import evidence as _evidence_mod
|
||||
|
||||
return _evidence_mod._Masks(sealed)
|
||||
|
||||
|
||||
async def resolve_metric_masks(
|
||||
metric: dict[str, Any],
|
||||
card_node: str,
|
||||
artifact: str,
|
||||
sealed: Sealed,
|
||||
masks: Any,
|
||||
*,
|
||||
card: dict[str, Any] | None = None,
|
||||
output: Any = None,
|
||||
by_value: bool = True,
|
||||
sure: bool = False,
|
||||
) -> tuple[dict[str, Any], str | None]:
|
||||
"""判断单个指标及其输入是否命中的数据源遮罩。"""
|
||||
from app.api import evidence as _evidence_mod
|
||||
|
||||
return await _evidence_mod._metric_masks(
|
||||
metric,
|
||||
card_node,
|
||||
artifact,
|
||||
sealed,
|
||||
masks,
|
||||
card=card,
|
||||
output=output,
|
||||
by_value=by_value,
|
||||
sure=sure,
|
||||
)
|
||||
|
||||
@@ -10,6 +10,7 @@ from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass, field
|
||||
from decimal import Decimal
|
||||
from typing import Any, Callable
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -36,6 +37,8 @@ def resolve_path(ctx: dict[str, Any], path: str) -> Any:
|
||||
if isinstance(cur, dict):
|
||||
cur = cur.get(name)
|
||||
else:
|
||||
if name.startswith("_"):
|
||||
return None
|
||||
cur = getattr(cur, name, None)
|
||||
if cur is None:
|
||||
return None
|
||||
@@ -65,11 +68,24 @@ _FILTERS = {
|
||||
}
|
||||
|
||||
|
||||
def _format_decimal(value: Decimal) -> str:
|
||||
if not value.is_finite():
|
||||
return str(value)
|
||||
text = str(value)
|
||||
if "e" in text or "E" in text:
|
||||
exp = value.as_tuple().exponent
|
||||
if isinstance(exp, int) and abs(exp) <= 100:
|
||||
return format(value, "f")
|
||||
return text
|
||||
|
||||
|
||||
def _stringify(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if isinstance(value, Decimal):
|
||||
return _format_decimal(value)
|
||||
if isinstance(value, (dict, list, tuple)):
|
||||
return json.dumps(value, ensure_ascii=False, default=str)
|
||||
return str(value)
|
||||
@@ -223,6 +239,7 @@ class ExpressionError(ValueError):
|
||||
# 一条画布上的分支条件就能把整个后端冻死。
|
||||
_MAX_POW_EXPONENT = 1024
|
||||
_MAX_POW_BASE = 1e15
|
||||
_MAX_SEQ_REPEAT = 1_000_000
|
||||
|
||||
|
||||
def _safe_pow(base: Any, exponent: Any) -> Any:
|
||||
@@ -237,10 +254,22 @@ def _safe_pow(base: Any, exponent: Any) -> Any:
|
||||
return operator.pow(base, exponent)
|
||||
|
||||
|
||||
def _safe_mul(a: Any, b: Any) -> Any:
|
||||
"""带上界的乘法:序列重复('x' * 10**9、[0] * 10**8)超界直接报错,防止在事件循环里把内存撑爆。"""
|
||||
seq_types = (str, bytes, bytearray, list, tuple, TruncatedRows)
|
||||
if isinstance(a, seq_types) and isinstance(b, int):
|
||||
if len(a) * max(0, b) > _MAX_SEQ_REPEAT:
|
||||
raise ExpressionError(f"序列重复长度超过上限 {_MAX_SEQ_REPEAT}")
|
||||
elif isinstance(b, seq_types) and isinstance(a, int):
|
||||
if len(b) * max(0, a) > _MAX_SEQ_REPEAT:
|
||||
raise ExpressionError(f"序列重复长度超过上限 {_MAX_SEQ_REPEAT}")
|
||||
return operator.mul(a, b)
|
||||
|
||||
|
||||
_BIN_OPS = {
|
||||
ast.Add: operator.add,
|
||||
ast.Sub: operator.sub,
|
||||
ast.Mult: operator.mul,
|
||||
ast.Mult: _safe_mul,
|
||||
ast.Div: operator.truediv,
|
||||
ast.FloorDiv: operator.floordiv,
|
||||
ast.Mod: operator.mod,
|
||||
|
||||
@@ -175,11 +175,23 @@ def _usage_of(message: BaseMessage, model_id: str) -> dict[str, Any]:
|
||||
meta = getattr(message, "usage_metadata", None) or {}
|
||||
inp = int(meta.get("input_tokens") or 0)
|
||||
out = int(meta.get("output_tokens") or 0)
|
||||
details = meta.get("input_token_details") or {}
|
||||
cache_read = int(details.get("cache_read") or 0) if isinstance(details, dict) else 0
|
||||
cache_create = int(details.get("cache_creation") or 0) if isinstance(details, dict) else 0
|
||||
if cache_read > 0 or cache_create > 0:
|
||||
try:
|
||||
cost = catalog.estimate_cost(
|
||||
model_id, inp, out, cache_read_tokens=cache_read, cache_creation_tokens=cache_create
|
||||
)
|
||||
except TypeError:
|
||||
cost = catalog.estimate_cost(model_id, inp, out)
|
||||
else:
|
||||
cost = catalog.estimate_cost(model_id, inp, out)
|
||||
return {
|
||||
"input_tokens": inp,
|
||||
"output_tokens": out,
|
||||
"total_tokens": inp + out,
|
||||
"cost_usd": round(catalog.estimate_cost(model_id, inp, out), 6),
|
||||
"cost_usd": round(cost, 6),
|
||||
"calls": 1,
|
||||
}
|
||||
|
||||
@@ -213,14 +225,31 @@ def explain_model_error(exc: Exception, model_id: str) -> str:
|
||||
"请稍后重试;如持续失败,请到「设置 → 模型接入」测试该模型")
|
||||
|
||||
|
||||
def _non_retriable_stream_error(exc: Exception) -> bool:
|
||||
"""鉴权失败、模型不存在、额度用尽、频率限制、上下文超长、输出额度被拒:非流式重发也必然因为同一原因失败,直接上报。"""
|
||||
from langchain_core.exceptions import ContextOverflowError
|
||||
|
||||
from app.providers.budget import budget_refused
|
||||
|
||||
if isinstance(exc, ContextOverflowError) or budget_refused(exc):
|
||||
return True
|
||||
status = getattr(exc, "status_code", None)
|
||||
if status in (401, 403, 413, 429):
|
||||
return True
|
||||
low = str(exc).lower()
|
||||
return any(k in low for k in (
|
||||
"invalid_model", "model does not exist", "insufficient", "quota",
|
||||
"rate limit", "unauthorized", "invalid api key",
|
||||
))
|
||||
|
||||
|
||||
async def _invoke_streaming(
|
||||
model: Any, messages: list[BaseMessage], ctx: NodeContext, model_id: str = ""
|
||||
) -> AIMessage:
|
||||
"""流式调用并把 token 实时推给前端。
|
||||
|
||||
失败时回退到非流式 —— 有些兼容网关不支持 SSE,不该因此让整个节点挂掉。
|
||||
但回退也失败的话,两次都是同一个原因,不能只把原始异常往上抛:用户看到的
|
||||
会是两段一模一样的供应商报错,而里面既没有模型名也没有下一步该干什么。
|
||||
但鉴权、模型不存在、限流、上下文超长这类请求本身就不成立的报错直接抛出,不再白发第二次请求。
|
||||
"""
|
||||
# 模型 id 要跟着 llm.start 走。以前这条事件只有 message_count,于是调用失败时
|
||||
# 整条轨迹里**没有任何地方**记着试的是哪个模型——排查只能靠翻配置猜。
|
||||
@@ -240,6 +269,8 @@ async def _invoke_streaming(
|
||||
except NotImplementedError:
|
||||
response = await model.ainvoke(messages)
|
||||
except Exception as e: # noqa: BLE001
|
||||
if _non_retriable_stream_error(e):
|
||||
raise NodeError(ctx.node.id, explain_model_error(e, model_id)) from e
|
||||
ctx.emit(EventType.LOG, level="warn",
|
||||
message=f"流式输出失败,已改用非流式调用:{explain_model_error(e, model_id)}",
|
||||
code="stream_fallback")
|
||||
@@ -290,15 +321,24 @@ async def run_llm(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
spent: list[dict[str, Any]] = []
|
||||
|
||||
if schema:
|
||||
# 要结构化结果就走原生 structured output,这条路径拿不到 token 流
|
||||
# 要结构化结果就走原生 structured output,带上 include_raw=True 把真实 token 用量一起拿回来
|
||||
ctx.emit(EventType.LLM_START, model=model_id, structured=True,
|
||||
message_count=len(messages))
|
||||
try:
|
||||
# langchain 靠 "title" 识别裸 JSON Schema dict,缺了会抛 Unsupported function
|
||||
if isinstance(schema, dict) and "title" not in schema:
|
||||
schema = {"title": "output", **schema}
|
||||
structured = model.with_structured_output(schema)
|
||||
value = await structured.ainvoke(messages)
|
||||
try:
|
||||
structured = model.with_structured_output(schema, include_raw=True)
|
||||
except TypeError:
|
||||
structured = model.with_structured_output(schema)
|
||||
got = await structured.ainvoke(messages)
|
||||
if isinstance(got, dict) and got.get("parsing_error"):
|
||||
err = got["parsing_error"]
|
||||
raise err if isinstance(err, BaseException) else ValueError(str(err))
|
||||
raw_msg, value = _split_structured(got)
|
||||
if value is None:
|
||||
raise ValueError("模型返回的结构化结果为空")
|
||||
except Exception as e: # noqa: BLE001
|
||||
# 模型压根不存在也会走到这里,那不是"结构化输出失败"
|
||||
if "invalid_model" in str(e) or "model does not exist" in str(e).lower():
|
||||
@@ -310,8 +350,12 @@ async def run_llm(state: GraphState, ctx: NodeContext) -> dict[str, Any]:
|
||||
) from e
|
||||
payload = value if isinstance(value, (dict, list)) else getattr(value, "model_dump", lambda: value)()
|
||||
output = {"data": payload, "text": json.dumps(payload, ensure_ascii=False, indent=2)}
|
||||
response: BaseMessage = AIMessage(content=output["text"])
|
||||
spent.append(_report_call(ctx, response, model_id, started))
|
||||
response = AIMessage(
|
||||
content=output["text"],
|
||||
usage_metadata=getattr(raw_msg, "usage_metadata", None),
|
||||
response_metadata=dict(getattr(raw_msg, "response_metadata", None) or {}),
|
||||
)
|
||||
spent.append(_report_call(ctx, raw_msg if raw_msg is not None else response, model_id, started))
|
||||
else:
|
||||
response = await _invoke_streaming(model, messages, ctx, model_id)
|
||||
spent.append(_report_call(ctx, response, model_id, started))
|
||||
|
||||
@@ -107,9 +107,14 @@ class RunManager:
|
||||
# 接下来各处都按 last_seq 接号:本模块的几处另外兜了底,按需裁判记判定(api/evidence.py 的
|
||||
# _append_judged)没有,不改正的话它重启后第一次追加就撞上已有的序号。有 (run_id, seq) 索引,
|
||||
# 每次运行一次索引查找。updated_at 不动:这是补记录,不是运行有了变化
|
||||
from sqlalchemy import exists
|
||||
|
||||
latest = select(func.max(RunEvent.seq)).where(RunEvent.run_id == Run.id).scalar_subquery()
|
||||
lagging = exists(
|
||||
select(1).where(RunEvent.run_id == Run.id, RunEvent.seq > func.coalesce(Run.last_seq, 0))
|
||||
)
|
||||
await session.execute(
|
||||
update(Run).where(func.coalesce(Run.last_seq, 0) < latest)
|
||||
update(Run).where(lagging)
|
||||
.values(last_seq=latest, updated_at=Run.updated_at)
|
||||
.execution_options(synchronize_session=False))
|
||||
await session.commit()
|
||||
@@ -195,6 +200,8 @@ class RunManager:
|
||||
if self._cm is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
await self._cm.__aexit__(None, None, None)
|
||||
self._cm = None
|
||||
self._checkpointer = None
|
||||
|
||||
@property
|
||||
def checkpointer(self) -> BaseCheckpointSaver:
|
||||
|
||||
+31
-13
@@ -40,12 +40,16 @@ def keep_set(runs: list[EvalRun], keep: int) -> set[str]:
|
||||
return kept
|
||||
|
||||
|
||||
async def _clean_one(session: AsyncSession, run: EvalRun) -> tuple[int, int, set[str]]:
|
||||
"""清一次评测的明细:(删了几项结果, 删了几次运行, 要回收的工件 id)。不提交。"""
|
||||
async def _clean_one(
|
||||
session: AsyncSession,
|
||||
run: EvalRun,
|
||||
origin_runs: dict[str, set[str]],
|
||||
) -> tuple[int, set[str], set[str]]:
|
||||
"""清一次评测的明细:(删了几项结果, 删了哪几次运行, 要回收的工件 id)。不提交。"""
|
||||
from app.db.models import Artifact, Run
|
||||
|
||||
results = list((await session.execute(select(EvalResult).where(EvalResult.eval_run_id == run.id))).scalars())
|
||||
run_ids: set[str] = set()
|
||||
run_ids: set[str] = set(origin_runs.get(run.id, ()))
|
||||
artifacts: set[str] = set()
|
||||
for r in results:
|
||||
if r.run_id:
|
||||
@@ -55,11 +59,7 @@ async def _clean_one(session: AsyncSession, run: EvalRun) -> tuple[int, int, set
|
||||
artifacts.add(r.details_artifact)
|
||||
await session.execute(delete(Artifact).where(Artifact.id == r.details_artifact, Artifact.run_id == "")
|
||||
.execution_options(synchronize_session=False))
|
||||
# 这次评测发起的、结果里没记下的运行(重来的前一次、停止时没写进结果的):按来源找
|
||||
for rid, origin in (await session.execute(select(Run.id, Run.origin).where(Run.origin_kind == "eval"))).all():
|
||||
if (origin or {}).get("eval_run_id") == run.id:
|
||||
run_ids.add(rid)
|
||||
removed = 0
|
||||
deleted_runs: set[str] = set()
|
||||
for rid in sorted(run_ids):
|
||||
row = await session.get(Run, rid)
|
||||
if row is None or row.origin_kind != "eval" or row.status in _LIVE:
|
||||
@@ -67,17 +67,20 @@ async def _clean_one(session: AsyncSession, run: EvalRun) -> tuple[int, int, set
|
||||
artifacts |= set((await session.execute(select(Artifact.id).where(Artifact.run_id == rid))).scalars())
|
||||
await session.execute(delete(Artifact).where(Artifact.run_id == rid).execution_options(synchronize_session=False))
|
||||
await session.delete(row)
|
||||
removed += 1
|
||||
deleted_runs.add(rid)
|
||||
for r in results:
|
||||
await session.delete(r)
|
||||
run.cleaned_at = utcnow()
|
||||
return len(results), removed, artifacts
|
||||
return len(results), deleted_runs, artifacts
|
||||
|
||||
|
||||
async def clean_suite(suite_id: str, *, keep: int | None = None) -> dict[str, Any]:
|
||||
"""按保留规则清一个题集:{runs: 清了几次评测, results, deleted_runs, files: 回收了几个工件文件}。"""
|
||||
from app.api.settings import eval_settings
|
||||
from app.core.artifact_store import forget
|
||||
from app.db.models import Run
|
||||
from app.engine import checkpoints
|
||||
from app.files import store as files_store
|
||||
|
||||
async with SessionLocal() as session:
|
||||
if keep is None:
|
||||
@@ -86,15 +89,30 @@ async def clean_suite(suite_id: str, *, keep: int | None = None) -> dict[str, An
|
||||
.order_by(EvalRun.created_at.desc(), EvalRun.id.desc()))).scalars())
|
||||
kept = keep_set(runs, keep)
|
||||
targets = [r for r in runs if r.id not in kept and r.cleaned_at is None]
|
||||
results = removed = 0
|
||||
origin_runs: dict[str, set[str]] = {}
|
||||
if targets:
|
||||
target_ids = {r.id for r in targets}
|
||||
# 评测发起的、结果里没记下的运行(重来的前一次、停止时没写进结果的):整批只扫一次按来源归组
|
||||
for rid, origin in (await session.execute(select(Run.id, Run.origin).where(Run.origin_kind == "eval"))).all():
|
||||
erid = (origin or {}).get("eval_run_id")
|
||||
if isinstance(erid, str) and erid in target_ids:
|
||||
origin_runs.setdefault(erid, set()).add(rid)
|
||||
results = 0
|
||||
deleted_run_ids: set[str] = set()
|
||||
artifacts: set[str] = set()
|
||||
for run in targets:
|
||||
n, m, found = await _clean_one(session, run)
|
||||
n, gone_runs, found = await _clean_one(session, run, origin_runs)
|
||||
results += n
|
||||
removed += m
|
||||
deleted_run_ids |= gone_runs
|
||||
artifacts |= found
|
||||
released_shas = await files_store.release_runs(session, deleted_run_ids)
|
||||
await session.commit()
|
||||
removed = len(deleted_run_ids)
|
||||
files = await forget(artifacts) if artifacts else []
|
||||
if released_shas:
|
||||
await files_store.drop_unreferenced(released_shas)
|
||||
if deleted_run_ids:
|
||||
await checkpoints.delete_threads(deleted_run_ids)
|
||||
if targets:
|
||||
logger.info("评测清理:题集 %s 清了 %d 次评测的明细(%d 项结果、%d 次运行、%d 个工件文件)", suite_id, len(targets),
|
||||
results, removed, len(files))
|
||||
|
||||
@@ -495,6 +495,34 @@ async def remove(session: AsyncSession, principal: Any, file_id: str) -> None:
|
||||
await _drop_unreferenced(session, {sha})
|
||||
|
||||
|
||||
async def release_runs(session: AsyncSession, run_ids: Any) -> set[str]:
|
||||
"""运行被删除时,解除没有绑定到记录表的上传文件登记,返回待检查回收的 sha256 集合(不提交;提交后调 drop_unreferenced)。"""
|
||||
ids = sorted({str(r) for r in run_ids or () if r})
|
||||
if not ids:
|
||||
return set()
|
||||
shas: set[str] = set()
|
||||
for start in range(0, len(ids), 500):
|
||||
chunk = ids[start:start + 500]
|
||||
rows = list((await session.execute(
|
||||
select(StoredFile.id, StoredFile.sha256).where(StoredFile.bound_run_id.in_(chunk), _NO_RECORD)
|
||||
)).all())
|
||||
if rows:
|
||||
shas |= {sha for _, sha in rows}
|
||||
await session.execute(
|
||||
delete(StoredFile).where(StoredFile.id.in_([fid for fid, _ in rows]))
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
return shas
|
||||
|
||||
|
||||
async def drop_unreferenced(shas: set[str]) -> int:
|
||||
"""在独立会话里检查并删除已无任何 StoredFile 引用的物理文件与文本缓存。"""
|
||||
if not shas:
|
||||
return 0
|
||||
async with SessionLocal() as session:
|
||||
return await _drop_unreferenced(session, shas)
|
||||
|
||||
|
||||
async def _drop_unreferenced(session: AsyncSession, shas: set[str]) -> int:
|
||||
"""这些内容没有任何登记引用了就删文件(连同抽出的文字缓存)。返回删了几个。"""
|
||||
if not shas:
|
||||
|
||||
@@ -570,9 +570,9 @@ async def freeze(session: Any, run_id: str, *, peers: Sequence[str] = (), name:
|
||||
"""把一次问数据运行固化成口径草稿(见模块说明),落库成新的草稿工作流,返回 {workflow_id, name, nodes, edges, params,
|
||||
sources, ...}。peers 是同一组(同一个 SQL 形状)的别的运行:认哪些字面量位置跨轮变了。不能固化抛 CertifyRefused。"""
|
||||
from app.api.deps import can_read_run, current_principal
|
||||
from app.api.evidence import _reports, _sealed
|
||||
from app.core.artifact_store import graph_hash, load
|
||||
from app.db.models import DataSource, Workflow, WorkflowVersion
|
||||
from app.engine.evidence_reader import load_sealed as _sealed, reports_of as _reports
|
||||
from app.ledger import store
|
||||
|
||||
run = await session.get(Run, run_id)
|
||||
|
||||
@@ -105,9 +105,9 @@ async def attach(session: Any, *, payload: dict, name: str, run_id: str, draft_i
|
||||
{question?, metrics: [docdraft 的指标对象]}。不发布、不发起运行、不写审计。
|
||||
"""
|
||||
from app.api.deps import can_read_run, current_principal, ensure_role
|
||||
from app.api.evidence import _sealed
|
||||
from app.core.artifact_store import graph_hash, load
|
||||
from app.db.models import DataSource, Workflow, WorkflowVersion
|
||||
from app.engine.evidence_reader import load_sealed as _sealed
|
||||
from app.ledger import certify, docdraft, store
|
||||
|
||||
ensure_role("builder")
|
||||
|
||||
@@ -28,6 +28,7 @@ readable=false:**有引用不等于真的有基线**,也不能在读草稿
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any
|
||||
|
||||
@@ -397,6 +398,115 @@ def status_of(payload: Any) -> dict[str, Any]:
|
||||
"source_snapshot_artifact_id": str(ref.get("source_snapshot_artifact_id") or "")}
|
||||
|
||||
|
||||
def editable_cards(graph: dict[str, Any]) -> list[dict[str, str]]:
|
||||
"""图里能写的一张张口径卡:[{id, title}],按节点出现的顺序。
|
||||
|
||||
能写的是源头口径卡(metrics 节点、没有 caliber_from):转抄别处的口径卡不是权威写入源(A01),改它等于在别处
|
||||
另存一份定义。
|
||||
"""
|
||||
from app.apps.tree import node_map
|
||||
|
||||
return [{"id": node_id, "title": str(node.get("title") or node_id)}
|
||||
for node_id, node in node_map(graph).items()
|
||||
if node.get("type") == "metrics" and not (node.get("config") or {}).get("caliber_from")]
|
||||
|
||||
|
||||
def contract_card(graph: dict[str, Any]) -> str | None:
|
||||
"""下游契约声明从哪张卡取指标(metrics_from 只写了一张时就是它):多张卡时按它认这张草稿的权威卡。"""
|
||||
for node in graph.get("nodes") or []:
|
||||
config = ((node or {}).get("data") or {}).get("config") if isinstance(node, dict) else None
|
||||
contract = config.get("contract") if isinstance(config, dict) else None
|
||||
if not isinstance(contract, dict):
|
||||
continue
|
||||
sources = contract.get("metrics_from")
|
||||
ids = [sources] if isinstance(sources, str) else [s for s in sources or [] if isinstance(s, str)]
|
||||
if len(ids) == 1:
|
||||
return ids[0]
|
||||
return None
|
||||
|
||||
|
||||
def draft_card(graph: dict[str, Any]) -> tuple[str | None, list[dict[str, str]]]:
|
||||
"""这份草稿的权威口径卡,另交回全部能写的卡。
|
||||
|
||||
一张能写的卡时就是它;多张时认下游契约声明的那一张——不是「第一张」。都认不出来时交回 None:写的时候必须显式
|
||||
给 target.card_id,读的时候页面也照实列候选。
|
||||
"""
|
||||
cards = editable_cards(graph)
|
||||
if len(cards) == 1:
|
||||
return cards[0]["id"], cards
|
||||
referenced = contract_card(graph)
|
||||
if referenced and any(card["id"] == referenced for card in cards):
|
||||
return referenced, cards
|
||||
return None, cards
|
||||
|
||||
|
||||
def graph_entry_fields(graph: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""图里入口节点(input)的字段;草稿页和试跑用它,画布改过的入口字段不会被 payload 的旧快照盖住。"""
|
||||
from app.apps.tree import node_map
|
||||
|
||||
start = next((node for node in node_map(graph).values() if node.get("type") == "input"), None)
|
||||
return [dict(field) for field in ((start or {}).get("config") or {}).get("fields") or []
|
||||
if isinstance(field, dict)]
|
||||
|
||||
|
||||
def project_graph(graph: dict[str, Any], version: int, *, card_id: str | None = None) -> dict[str, Any]:
|
||||
"""从草稿工作流的某一版图里投影出指标、入口字段、取数 SQL 与分组结构(纯函数)。"""
|
||||
from app.apps.tree import node_map
|
||||
from app.look.calibers import _metric_paths
|
||||
|
||||
nodes = node_map(graph)
|
||||
wanted, cards = draft_card(graph)
|
||||
wanted = card_id or wanted or (cards[0]["id"] if cards else None)
|
||||
card = nodes.get(wanted) if wanted else None
|
||||
metrics = [metric for metric in ((card or {}).get("config") or {}).get("metrics") or []
|
||||
if isinstance(metric, dict) and metric.get("id")]
|
||||
fields = graph_entry_fields(graph)
|
||||
fetches: list[tuple[str, str]] = []
|
||||
for node_id, node in nodes.items():
|
||||
config = node.get("config") if isinstance(node.get("config"), dict) else {}
|
||||
args = config.get("args") if isinstance(config.get("args"), dict) else {}
|
||||
sql = args.get("sql") if isinstance(args, dict) else None
|
||||
if node.get("type") == "tool" and isinstance(sql, str) and sql.strip():
|
||||
fetches.append((str(node_id), sql))
|
||||
groups: list[dict[str, Any]] = []
|
||||
for node_id, sql in fetches:
|
||||
pattern = re.compile(rf"nodes\.{re.escape(node_id)}(?![\w])")
|
||||
ids = [str(metric["id"]) for metric in metrics
|
||||
if any(pattern.search(path) for path in _metric_paths(metric) or [])]
|
||||
groups.append({"sql": sql, "entry_fields": [dict(field) for field in fields], "metric_ids": ids})
|
||||
assigned = {metric_id for group in groups for metric_id in group["metric_ids"]}
|
||||
loose = [str(metric["id"]) for metric in metrics if str(metric["id"]) not in assigned]
|
||||
if loose:
|
||||
if groups:
|
||||
groups[0]["metric_ids"].extend(loose)
|
||||
else:
|
||||
groups = [{"sql": "", "entry_fields": [dict(field) for field in fields], "metric_ids": loose}]
|
||||
return {
|
||||
"version": version,
|
||||
"metrics": metrics,
|
||||
"entry_fields": [dict(field) for field in fields],
|
||||
"sql": "\n\n".join(sql for _node, sql in fetches),
|
||||
"groups": groups,
|
||||
"blocks": len(fetches) or 1,
|
||||
}
|
||||
|
||||
|
||||
async def draft_graph_projection(session: Any, workflow: Any, version: int, *,
|
||||
card_id: str | None = None) -> dict[str, Any] | None:
|
||||
"""按 workflow_id + version 读取图版本并投影草稿视图所需字段(供重导入等领域层调用,不依赖 api/ledger)。"""
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db.models import WorkflowVersion
|
||||
|
||||
row = (await session.execute(select(WorkflowVersion).where(
|
||||
WorkflowVersion.workflow_id == workflow.id,
|
||||
WorkflowVersion.version == int(version)))).scalar_one_or_none()
|
||||
if row is None or not isinstance(row.graph, dict):
|
||||
return None
|
||||
return project_graph(row.graph, version, card_id=card_id)
|
||||
|
||||
|
||||
__all__ = ["CITATIONS_FIELD", "HISTORY_KEY", "KIND", "REF_KEY", "VERSION", "accepted_card", "advance",
|
||||
"coverage_of", "document", "field_paths", "fields_of", "history", "put_metrics", "read", "ref_of",
|
||||
"status_of", "value_at", "values"]
|
||||
"contract_card", "coverage_of", "document", "draft_card", "draft_graph_projection", "editable_cards",
|
||||
"field_paths", "fields_of", "graph_entry_fields", "history", "project_graph", "put_metrics", "read",
|
||||
"ref_of", "status_of", "value_at", "values"]
|
||||
|
||||
@@ -210,9 +210,9 @@ async def plan_issue(session: Any, run: Run | None, content: dict[str, Any] | No
|
||||
"""按 workflow_id@version 出具这次运行的打算(见模块说明)。content 是这次运行最新的比对工件内容(读不出为 None),
|
||||
hide 判一个绑定值是不是这次运行查到过的遮罩值。只读,不发起。"""
|
||||
from app.api.deps import can_read_run
|
||||
from app.api.settings import ledger_settings
|
||||
from app.ledger import match as ledger_match
|
||||
from app.ledger.labels import caliber_key
|
||||
from app.ledger.settings import ledger_settings
|
||||
|
||||
plan = Plan(run_id=run.id if run else "", workflow_id=workflow_id, version=int(version),
|
||||
caliber=caliber_key(workflow_id, int(version)), match_artifact=artifact)
|
||||
@@ -257,12 +257,12 @@ async def check_target(session: Any, plan: Plan, workflow: Any, *, principal: An
|
||||
(ledger/params:缺参数、未声明参数、承载不了的值、认不出参数写在哪的模板都拒);使用者只能按
|
||||
他有权使用的应用固定着的这一版(拍板 L5);同一个来源(source_run)、同一版、同一组参数发起过的交回那一次;当天没超
|
||||
上限。填好 plan 的 level、expect_hash、inputs、granted_by_app、existing;不能出具的写进 plan.refuse,交回 None。"""
|
||||
from app.api.settings import ledger_settings
|
||||
from app.apps import versions as app_versions
|
||||
from app.apps.backfill import graph_side_effect
|
||||
from app.apps.forms import input_schema, validate_form
|
||||
from app.apps.resolve import input_key
|
||||
from app.ledger.derive import entry_fields
|
||||
from app.ledger.settings import ledger_settings
|
||||
|
||||
workflow_id, version = plan.workflow_id, plan.version
|
||||
if workflow is None:
|
||||
@@ -362,7 +362,7 @@ QUOTA = "ledger_issue"
|
||||
|
||||
|
||||
async def _hider(run_id: str) -> Any:
|
||||
from app.api.evidence import _sealed
|
||||
from app.engine.evidence_reader import load_sealed as _sealed
|
||||
from app.ledger.view import _hider as hider_of
|
||||
|
||||
return (await hider_of(await _sealed(run_id))).hide
|
||||
@@ -397,8 +397,8 @@ async def issue(run_id: str, workflow_id: str, version: int, *, principal: Any,
|
||||
**({"publication": plan.publication} if plan.publication else {})}
|
||||
actor = run.started_by if auto else principal.username
|
||||
# 先占一份今天的额度(库里带条件的加一,几个进程同时出具也正好到上限为止),发起不成退回
|
||||
from app.api.settings import ledger_settings
|
||||
from app.cluster import quota
|
||||
from app.ledger.settings import ledger_settings
|
||||
|
||||
limit = int((await ledger_settings(session))["issue_daily_limit"])
|
||||
day = await quota.take(QUOTA, limit, seed=issued_today)
|
||||
|
||||
@@ -579,9 +579,15 @@ def compare_queries(queries: list[Query], calibers: list[Caliber], *, deadline:
|
||||
chosen = (composed or {}).get(str(query.node_id or ""))
|
||||
wanted = _wanted(query, referenced)
|
||||
pool = [(c, f) for c in calibers for f in c.fetches if f.source == query.source and f.tables & query.tables]
|
||||
if chosen is not None:
|
||||
# 助手选用的那一份排最前面:候选多的时候也一定比到它
|
||||
pool.sort(key=lambda cf: (cf[0].key, cf[1].node_id) != chosen)
|
||||
if pool:
|
||||
# 助手选用的排第一;其余按「共享表数越多越靠前、表集合规模越接近越靠前」排,同分保持工作流/版本/节点原序
|
||||
pool.sort(
|
||||
key=lambda cf: (
|
||||
(cf[0].key, cf[1].node_id) != chosen if chosen is not None else False,
|
||||
-len(cf[1].tables & query.tables),
|
||||
abs(len(cf[1].tables) - len(query.tables)),
|
||||
)
|
||||
)
|
||||
if not pool:
|
||||
out.append({**row, "status": "none", "candidates": []})
|
||||
continue
|
||||
@@ -649,7 +655,7 @@ def _card_loader(sealed: Any) -> Any:
|
||||
def referenced_of(sealed: Any) -> dict[str, set[str]]:
|
||||
"""这次运行封存的报告引用了哪条查询的哪几列(labels.referenced_columns,只读哈希核对一致的报告):compare_queries 的
|
||||
referenced。评测逐题比 SQL 时传它,就和账本一样按引用的列投影。在线程里调(读盘)。"""
|
||||
from app.api.evidence import _reports
|
||||
from app.engine.evidence_reader import reports_of as _reports
|
||||
|
||||
docs = [r.doc for r in _reports(sealed) if r.doc is not None and r.hash_ok is True]
|
||||
return ledger_labels.referenced_columns(docs, _card_loader(sealed))
|
||||
@@ -657,7 +663,7 @@ def referenced_of(sealed: Any) -> dict[str, set[str]]:
|
||||
|
||||
def compute(sealed: Any, calibers: list[Caliber], kinds: dict[str, str | None], *, timeout: float) -> dict[str, Any]:
|
||||
"""一次运行的比对工件内容(见 ledger/store 的模块说明)。在线程里调:读快照、比 SQL、标数都是同步的活。"""
|
||||
from app.api.evidence import _choose, _reports
|
||||
from app.engine.evidence_reader import choose_report as _choose, reports_of as _reports
|
||||
|
||||
deadline = time.monotonic() + timeout
|
||||
queries = read_queries(sealed, kinds)
|
||||
@@ -753,8 +759,8 @@ async def match_run(run_id: str, *, trigger: str = "sealed", timeout: float | No
|
||||
|
||||
|
||||
async def _match_run(run_id: str, *, trigger: str, timeout: float | None) -> str | None:
|
||||
from app.api.evidence import _sealed
|
||||
from app.api.settings import ledger_settings
|
||||
from app.engine.evidence_reader import load_sealed as _sealed
|
||||
from app.ledger.settings import ledger_settings
|
||||
|
||||
async with SessionLocal() as session:
|
||||
run = await session.get(Run, run_id)
|
||||
|
||||
@@ -69,9 +69,11 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
#: 参数问题的机器码(§4.4 的三条 + 值承载不了的一条)
|
||||
@@ -184,13 +186,13 @@ def describe(problems: Iterable[Problem]) -> str:
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
def slots_of(template: str) -> tuple[Slot, ...]:
|
||||
def slots_of(template: str, *, dialect: str | None = None) -> tuple[Slot, ...]:
|
||||
"""模板里每一处 `{{ input.字段 }}`(含写在哪儿、带了什么过滤器)。认不出写法、写在表名列名里的抛
|
||||
ParameterRefused(unsupported_parameter)——判断和带值比对那一套(data/sql_semantics._template_slots)是同一份。"""
|
||||
from app.data.sql_semantics import _NotComparable, _template_slots
|
||||
|
||||
try:
|
||||
spans, _ = _template_slots(template)
|
||||
spans, _ = _template_slots(template, dialect=dialect)
|
||||
except _NotComparable as e:
|
||||
raise ParameterRefused([Problem(UNSUPPORTED, str(e))]) from None
|
||||
return tuple(Slot(index=s.index, field=s.field, filters=tuple(s.filters), context=s.context) for _, _, s in spans)
|
||||
@@ -299,8 +301,8 @@ def declared(fields: Iterable[Mapping[str, Any]]) -> dict[str, dict[str, Any]]:
|
||||
|
||||
#: 声明类型 → 规范类型。认不出的一律 string(各入口同一套:类型判定不依赖字段先后,R02 的一半)
|
||||
_TYPES = {"string": "string", "text": "string", "integer": "integer", "int": "integer", "number": "number",
|
||||
"float": "number", "boolean": "boolean", "bool": "boolean", "date": "date", "datetime": "datetime",
|
||||
"time": "time", "file": "file", "array": "array", "object": "object"}
|
||||
"float": "number", "decimal": "number", "boolean": "boolean", "bool": "boolean", "date": "date",
|
||||
"datetime": "datetime", "time": "time", "file": "file", "array": "array", "object": "object"}
|
||||
|
||||
|
||||
def field_type(field: Mapping[str, Any]) -> str:
|
||||
@@ -314,7 +316,7 @@ def value_kind(value: Any) -> str:
|
||||
return "null"
|
||||
if isinstance(value, bool):
|
||||
return "boolean"
|
||||
if isinstance(value, (int, float)):
|
||||
if isinstance(value, (int, float, Decimal)):
|
||||
return "number"
|
||||
if isinstance(value, str):
|
||||
return "text"
|
||||
@@ -366,11 +368,29 @@ def resolve(fields: Iterable[Mapping[str, Any]], values: Mapping[str, Any]) -> d
|
||||
def _written(value: Any) -> str:
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if isinstance(value, Decimal):
|
||||
from app.engine.expressions import _format_decimal
|
||||
|
||||
return _format_decimal(value)
|
||||
if isinstance(value, (dict, list)):
|
||||
return json.dumps(value, ensure_ascii=False, default=str)
|
||||
return str(value)
|
||||
|
||||
|
||||
def _bare_scalar_ok(value: Any) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return True
|
||||
if isinstance(value, int):
|
||||
return True
|
||||
if isinstance(value, float):
|
||||
return math.isfinite(value)
|
||||
if isinstance(value, Decimal):
|
||||
return value.is_finite()
|
||||
if isinstance(value, str):
|
||||
return bool(_NUMBER.match(value))
|
||||
return False
|
||||
|
||||
|
||||
def _unresolved(name: str, field: Mapping[str, Any] | None, values: Mapping[str, Any]) -> bool:
|
||||
"""这个参数这次没有可用的值:没给、给了 None、或者给了空串(空串不是值,是省略——和入口收值 params.provided 同一套),
|
||||
而且声明的缺省值也是 None。
|
||||
@@ -390,14 +410,13 @@ def _unresolved(name: str, field: Mapping[str, Any] | None, values: Mapping[str,
|
||||
def literal_problem(name: str, value: Any, contexts: Iterable[str]) -> str | None:
|
||||
"""值写在 SQL 的哪个位置、旧适配器(内联文本替换)承载得了吗:承载不了给一句原因,承载得了为 None。
|
||||
|
||||
引号里的字里再出现引号 / 反斜杠 / 换行、裸写的位置上不是数、注释里带换行或注释结束符——这些都会让渲染出来的语句
|
||||
引号里的字里再出现引号 / 反斜杠 / 换行、裸写的位置上不是有限数值或布尔值、注释里带换行或注释结束符——这些都会让渲染出来的语句
|
||||
不再等于「那一版口径的 SQL」。真支持特殊字符要等驱动级参数绑定(见模块说明),那之前明确拒绝,不硬塞。"""
|
||||
text = _written(value)
|
||||
where = set(contexts)
|
||||
if "quoted" in where and any(ch in text for ch in ("'", "\\", "\n", "\r")):
|
||||
return QUOTED_TEXT.format(name=name)
|
||||
if "bare" in where and not (isinstance(value, (int, float)) and not isinstance(value, bool)) \
|
||||
and not (isinstance(value, str) and _NUMBER.match(value)):
|
||||
if "bare" in where and not _bare_scalar_ok(value):
|
||||
return BARE_TEXT.format(name=name, value=text[:40])
|
||||
if "comment" in where and ("\n" in text or "\r" in text or "*/" in text):
|
||||
return COMMENT_TEXT.format(name=name)
|
||||
@@ -405,13 +424,14 @@ def literal_problem(name: str, value: Any, contexts: Iterable[str]) -> str | Non
|
||||
|
||||
|
||||
def check_template(template: str, *, fields: Mapping[str, Mapping[str, Any]], values: Mapping[str, Any] | None = None,
|
||||
node_id: str | None = None, where: str = "入口字段") -> list[Problem]:
|
||||
node_id: str | None = None, where: str = "入口字段",
|
||||
dialect: str | None = None) -> list[Problem]:
|
||||
"""一条取数模板按声明的入口字段过一遍:未声明、认不出的写法;给了 values 的,再判缺值与值承载不了。
|
||||
|
||||
values 为 None 是「此刻还没有这次调用的值」(读一版口径时就是这样):只判模板和声明对不对得上。
|
||||
认不出写法(unsupported)时不接着往下判——值写在哪儿都还没定,别的结论不可靠。"""
|
||||
try:
|
||||
slots = slots_of(template)
|
||||
slots = slots_of(template, dialect=dialect)
|
||||
except ParameterRefused as e:
|
||||
return [Problem(UNSUPPORTED, UNREADABLE_TEXT.format(reason=p.message), node_id=node_id) for p in e.problems]
|
||||
problems: list[Problem] = []
|
||||
@@ -450,7 +470,8 @@ def _sql_of(node: Mapping[str, Any]) -> str | None:
|
||||
|
||||
|
||||
def check_nodes(nodes: Iterable[Mapping[str, Any]], *, fields: Iterable[Mapping[str, Any]],
|
||||
values: Mapping[str, Any] | None = None, where: str = "入口字段") -> list[Problem]:
|
||||
values: Mapping[str, Any] | None = None, where: str = "入口字段",
|
||||
dialect: str | None = None) -> list[Problem]:
|
||||
"""几个调用工具节点(原始 JSON)的 args.sql 各过一遍,问题按节点先后合起来。values 不给就只判模板与声明。"""
|
||||
declared_fields = declared(fields)
|
||||
out: list[Problem] = []
|
||||
@@ -461,7 +482,7 @@ def check_nodes(nodes: Iterable[Mapping[str, Any]], *, fields: Iterable[Mapping[
|
||||
if sql is None:
|
||||
continue
|
||||
out += check_template(sql, fields=declared_fields, values=values, node_id=str(node.get("id") or "") or None,
|
||||
where=where)
|
||||
where=where, dialect=dialect)
|
||||
return out
|
||||
|
||||
|
||||
@@ -515,7 +536,8 @@ def _hash(text: str) -> str:
|
||||
|
||||
|
||||
def material(template: str, *, values: Mapping[str, Any], declared_fields: Mapping[str, Mapping[str, Any]],
|
||||
source: Mapping[str, Any] | None = None, with_template: bool = True) -> dict[str, Any]:
|
||||
source: Mapping[str, Any] | None = None, with_template: bool = True,
|
||||
dialect: str | None = None) -> dict[str, Any]:
|
||||
"""查询快照要记的那一份:源模板、源模板哈希、规范化输入哈希、参数契约、每个参数的类型与写在哪儿、承载方式。
|
||||
|
||||
绑定值不在这里再存一份——内联方式下它就写在实际执行语句里,而实际语句本身在快照里(payload["sql"])。
|
||||
@@ -526,7 +548,7 @@ def material(template: str, *, values: Mapping[str, Any], declared_fields: Mappi
|
||||
names = input_refs(template)
|
||||
bound: list[dict[str, Any]] = []
|
||||
try:
|
||||
by_field = {s.field: s.context for s in slots_of(template)}
|
||||
by_field = {s.field: s.context for s in slots_of(template, dialect=dialect)}
|
||||
except ParameterRefused:
|
||||
by_field = {}
|
||||
for name in names:
|
||||
@@ -553,7 +575,7 @@ def material(template: str, *, values: Mapping[str, Any], declared_fields: Mappi
|
||||
|
||||
def plan(template: str, *, fields: Iterable[Mapping[str, Any]], values: Mapping[str, Any],
|
||||
source: Mapping[str, Any] | None = None, node_id: str | None = None,
|
||||
where: str = "入口字段") -> Statement:
|
||||
where: str = "入口字段", dialect: str | None = None) -> Statement:
|
||||
"""「现在就发一条 SQL」的入口用:先按声明的字段把这次的值定下来(resolve:显式非空值优先、否则声明缺省值),
|
||||
再判参数完整性,过了才渲染成字面语句,附快照材料——检查、渲染、材料里的输入哈希用的是**同一份 resolved values**。
|
||||
|
||||
@@ -567,11 +589,13 @@ def plan(template: str, *, fields: Iterable[Mapping[str, Any]], values: Mapping[
|
||||
check_nodes / check_template 判过)——它是给下一个入口(QueryCore 的固定执行)准备的共享层接口,行为由测试锁住。"""
|
||||
declared_fields = declared(fields)
|
||||
resolved = resolve(declared_fields.values(), values)
|
||||
problems = check_template(template, fields=declared_fields, values=resolved, node_id=node_id, where=where)
|
||||
problems = check_template(template, fields=declared_fields, values=resolved, node_id=node_id, where=where,
|
||||
dialect=dialect)
|
||||
if problems:
|
||||
raise ParameterRefused(problems)
|
||||
return Statement(statement=render(template, resolved),
|
||||
material=material(template, values=resolved, declared_fields=declared_fields, source=source))
|
||||
material=material(template, values=resolved, declared_fields=declared_fields, source=source,
|
||||
dialect=dialect))
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
|
||||
@@ -399,13 +399,12 @@ async def _metrics_from(sealed_run_id: str, *, card_id: str, metric_ids: Sequenc
|
||||
的时候,这里绝不能把值交出去。交回 (类型化指标, 原始指标),原始那一份给质量报告用。
|
||||
|
||||
memo 是这一次收尾里几处共用的封存证据(见 _sealed_of):给了就用那一份,没给就自己读一次。"""
|
||||
from app.api.evidence import _Masks, _metric_masks
|
||||
from app.engine.evidence_reader import metric_cards
|
||||
from app.engine.evidence_reader import metric_cards, open_masks, resolve_metric_masks
|
||||
|
||||
sealed = await _sealed_of(sealed_run_id, memo)
|
||||
if not sealed.trusted:
|
||||
return [], {"trusted": False}
|
||||
masks = _Masks(sealed)
|
||||
masks = open_masks(sealed)
|
||||
found: dict[str, dict[str, Any]] = {}
|
||||
raw: dict[str, dict[str, Any]] = {}
|
||||
for node_id, artifact, card in metric_cards(sealed):
|
||||
@@ -419,7 +418,7 @@ async def _metrics_from(sealed_run_id: str, *, card_id: str, metric_ids: Sequenc
|
||||
continue
|
||||
# 同一张卡执行过多次时按**最后一次**的产出算(和「节点最后一次产出」同一个认法),
|
||||
# 不拿早先那次的值
|
||||
_, hidden = await _metric_masks(metric, node_id, artifact, sealed, masks, card=card)
|
||||
_, hidden = await resolve_metric_masks(metric, node_id, artifact, sealed, masks, card=card)
|
||||
found[metric_id] = _typed_metric(metric, hidden)
|
||||
raw[metric_id] = dict(metric)
|
||||
ordered = [found[mid] for mid in metric_ids if mid in found]
|
||||
|
||||
@@ -495,7 +495,7 @@ def _lock(key: tuple[str, int]) -> Any:
|
||||
async def launch(workflow_id: str, new_version: int, *, batch: str = "auto", actor: str | None = None) -> list[str]:
|
||||
"""按计划发起一批召回运行,交回发起了的运行 id。auto:发布后自动的那一批,一次发布最多 recall_limit 组(已经自动发起
|
||||
过的扣掉,重复调幂等);manual:「接着算」,再发起一批(上限是 0 时按缺省的组数)。"""
|
||||
from app.api.settings import LEDGER_DEFAULTS, ledger_settings
|
||||
from app.ledger.settings import LEDGER_DEFAULTS, ledger_settings
|
||||
|
||||
if batch not in BATCHES:
|
||||
raise ValueError(f"不认识的召回批次:{batch}")
|
||||
@@ -534,7 +534,7 @@ def _counts(plan: Plan) -> dict[str, int]:
|
||||
async def preview(session: Any, workflow_id: str, version: int) -> dict[str, Any]:
|
||||
"""发布弹窗的召回预告:把 version 发布成新版本之后,旧版本被多少问数回答、导出、定时任务、应用用过,预计重算几组,超出
|
||||
上限的写明。只读、不发起(和发布之后真的召回走同一份计划,规模一致)。"""
|
||||
from app.api.settings import ledger_settings
|
||||
from app.ledger.settings import ledger_settings
|
||||
|
||||
plan = await build_plan(session, workflow_id, int(version))
|
||||
limit = int((await ledger_settings(session))["recall_limit"])
|
||||
@@ -598,7 +598,7 @@ async def batches(session: Any, workflow_id: str, version: int) -> dict[str, Any
|
||||
未重算」和能不能接着算(还有没算的组却不能接着算时 continue_reason 写为什么:之后又发布了更新的版本……)。
|
||||
接口只给搭建者(api/ledger 的 router):参数原样给,同他在召回运行的输入里读到的一样;使用者
|
||||
看的是 recall 待办(task_payload,按遮罩换过)。"""
|
||||
from app.api.settings import LEDGER_DEFAULTS, ledger_settings
|
||||
from app.ledger.settings import LEDGER_DEFAULTS, ledger_settings
|
||||
|
||||
plan = await build_plan(session, workflow_id, int(version))
|
||||
limit = int((await ledger_settings(session))["recall_limit"])
|
||||
@@ -664,7 +664,7 @@ async def _settled(session: Any, run_id: str) -> dict[str, Any] | None:
|
||||
|
||||
async def card_values(run_id: str) -> dict[str, dict[str, Any]]:
|
||||
"""召回运行封存范围内的召回卡:{指标 id: 指标}(value、rendered……)。读不出为空。"""
|
||||
from app.api.evidence import _sealed, _sealed_cards
|
||||
from app.engine.evidence_reader import load_sealed as _sealed, metric_cards as _sealed_cards
|
||||
from app.ledger import derive
|
||||
|
||||
sealed = await _sealed(run_id)
|
||||
@@ -878,7 +878,7 @@ async def task_payload(session: Any, task: Any, principal: Any) -> dict[str, Any
|
||||
"""recall 待办给处理人看的内容(待办箱、待办详情的 payload,process/tasks._describe 调):旧口径的值、新口径的值、两版之差
|
||||
和原答案当时的值(每个都带出处:哪次运行、哪张卡、哪个指标,点得开),这一组的参数,能做的动作。使用者看到的值照样
|
||||
遮罩(按那次运行查到过的遮罩值,同账本接口)。"""
|
||||
from app.api.evidence import _sealed
|
||||
from app.engine.evidence_reader import load_sealed as _sealed
|
||||
from app.ledger.view import _hider
|
||||
|
||||
run = await session.get(Run, task.run_id)
|
||||
@@ -946,7 +946,7 @@ async def original_values(use: Mapping[str, Any], meta: Sequence[Mapping[str, An
|
||||
"""原答案当时的值(每个比较一项):正式运行读它封存的口径卡(固定了这一版的读转抄来的那张);问数回答读等价的那条查询
|
||||
快照里那一格(口径卡的指标就是取数第一行某一列时,按列映射找这次的列)。读不出、原答案里没有单独这个数的为空,写为什么。
|
||||
hide_for:给使用者看,按那次运行查到过的遮罩值遮。"""
|
||||
from app.api.evidence import _sealed, _sealed_cards
|
||||
from app.engine.evidence_reader import load_sealed as _sealed, metric_cards as _sealed_cards
|
||||
from app.ledger.view import _hider
|
||||
|
||||
run_id = str(use.get("run_id") or "")
|
||||
|
||||
@@ -519,8 +519,8 @@ def _plan_rows(ours_columns: list[str], rows: list[Any], hidden: set[str], calib
|
||||
|
||||
def _referenced(sealed: Any, artifact: str) -> list[tuple[int, str]]:
|
||||
"""报告里引用了这条查询的哪几格:[(行, 列)],按正文的先后(单元格、表格里的格)。"""
|
||||
from app.api.evidence import _reports
|
||||
from app.engine.evidence import iter_units
|
||||
from app.engine.evidence_reader import reports_of as _reports
|
||||
|
||||
out: list[tuple[int, str]] = []
|
||||
for report in _reports(sealed):
|
||||
@@ -580,8 +580,8 @@ def pinned_effective(pinned: Any) -> list[dict[str, Any]]:
|
||||
async def reconcile(run_id: str, req: Request, *, principal: Any) -> Run:
|
||||
"""对这次运行的一条查询和一份相关口径对一次账:派生只读图、发起探索运行(来源 reconcile)。不能对的抛 ReconcileRefused。"""
|
||||
from app.api.deps import can_read_run
|
||||
from app.api.evidence import _Masks, _sealed
|
||||
from app.core.artifact_store import load
|
||||
from app.engine.evidence_reader import load_sealed as _sealed, open_masks as _Masks
|
||||
from app.engine.launch import LaunchRefused, launch_run
|
||||
from app.ledger import compose, derive, store
|
||||
from app.ledger.issue import _conversation_of
|
||||
|
||||
@@ -1208,8 +1208,6 @@ async def _refresh_run_items(session: Any, *, run: CaliberImportRun, draft: Cali
|
||||
没对齐上的工作项(mode 是 new / manual、metric_id 为空)没有可比的 L 一格,保持抽出来时的样子:
|
||||
要对齐得先走人工对齐(reimport.align 的入口在接口层)。
|
||||
"""
|
||||
from app.api.ledger import _draft_graph_payload
|
||||
|
||||
live_metrics = list(fieldbaseline.accepted_card(workflow.graph if workflow is not None else {})[1])
|
||||
live_by_id = {str(m.get("id")): m for m in live_metrics if isinstance(m, Mapping)}
|
||||
ref = fieldbaseline.ref_of(draft.payload)
|
||||
@@ -1228,7 +1226,7 @@ async def _refresh_run_items(session: Any, *, run: CaliberImportRun, draft: Cali
|
||||
item.fields, item.status, item.align = merged["fields"], merged["status"], merged["align"]
|
||||
await _refresh_counts(session, run)
|
||||
if workflow is not None:
|
||||
live = await _draft_graph_payload(session, workflow, workflow.version)
|
||||
live = await fieldbaseline.draft_graph_projection(session, workflow, workflow.version)
|
||||
if live is not None:
|
||||
draft.payload = {**(draft.payload or {}), **live}
|
||||
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
"""口径账本运行与治理设置:默认项、范围校验与持久化读取(供 ledger 领域层与设置接口共用)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Setting
|
||||
|
||||
LEDGER_GROUP = "ledger"
|
||||
LEDGER_DEFAULTS: dict[str, Any] = {
|
||||
"auto_issue": True,
|
||||
"issue_daily_limit": 20,
|
||||
"match_timeout_s": 8,
|
||||
"recall_limit": 50,
|
||||
}
|
||||
LEDGER_SETTING_LABEL = {
|
||||
"auto_issue": "自动按口径正式出具",
|
||||
"issue_daily_limit": "按口径正式出具每日上限",
|
||||
"match_timeout_s": "口径比对时限(秒)",
|
||||
"recall_limit": "口径升版后每次最多重算几组",
|
||||
}
|
||||
LEDGER_RECALL_MAX = 1000
|
||||
LEDGER_TIMEOUT_RANGE = (1, 60)
|
||||
LEDGER_DAILY_LIMIT_MAX = 100_000
|
||||
|
||||
|
||||
def _daily_limit_ok(value: Any) -> bool:
|
||||
return isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= LEDGER_DAILY_LIMIT_MAX
|
||||
|
||||
|
||||
def _timeout_ok(value: Any) -> bool:
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool) \
|
||||
and LEDGER_TIMEOUT_RANGE[0] <= value <= LEDGER_TIMEOUT_RANGE[1]
|
||||
|
||||
|
||||
def _recall_limit_ok(value: Any) -> bool:
|
||||
return isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= LEDGER_RECALL_MAX
|
||||
|
||||
|
||||
def ledger_problem(value: Any) -> str | None:
|
||||
"""设置页写进来的 ledger 这一组有什么问题,没问题返回 None。"""
|
||||
if not isinstance(value, dict):
|
||||
return f"「口径账本」设置格式有误,需要是一个对象,当前为「{value}」"
|
||||
for key, item in value.items():
|
||||
if key not in LEDGER_DEFAULTS:
|
||||
return (f"「口径账本」设置中不支持「{key}」,可设置的项:"
|
||||
+ "、".join(f"「{LEDGER_SETTING_LABEL[k]}」" for k in LEDGER_DEFAULTS))
|
||||
if key == "auto_issue" and not isinstance(item, bool):
|
||||
return f"「{LEDGER_SETTING_LABEL[key]}」只能是开或关,当前为「{item}」"
|
||||
if key == "issue_daily_limit" and not _daily_limit_ok(item):
|
||||
return (f"「{LEDGER_SETTING_LABEL[key]}」需要是 0 到 {LEDGER_DAILY_LIMIT_MAX:,} 之间的整数(0 表示不再按口径出具),"
|
||||
f"当前为「{item}」")
|
||||
if key == "recall_limit" and not _recall_limit_ok(item):
|
||||
return (f"「{LEDGER_SETTING_LABEL[key]}」需要是 0 到 {LEDGER_RECALL_MAX:,} 之间的整数(0 表示发布后不自动重算),"
|
||||
f"当前为「{item}」")
|
||||
if key == "match_timeout_s" and not _timeout_ok(item):
|
||||
return (f"「{LEDGER_SETTING_LABEL[key]}」需要是 {LEDGER_TIMEOUT_RANGE[0]} 到 {LEDGER_TIMEOUT_RANGE[1]} 之间的数,"
|
||||
f"当前为「{item}」")
|
||||
return None
|
||||
|
||||
|
||||
def ledger_values(stored: Any) -> dict[str, Any]:
|
||||
"""库里存的(或刚写进来的)ledger 一组 → 补齐默认;存坏了的项按默认(自动出具的开关存坏了按关,宁可少出具)。"""
|
||||
stored = stored if isinstance(stored, dict) else {}
|
||||
auto, limit, timeout = stored.get("auto_issue"), stored.get("issue_daily_limit"), stored.get("match_timeout_s")
|
||||
recall = stored.get("recall_limit")
|
||||
return {
|
||||
"auto_issue": auto if isinstance(auto, bool) else (LEDGER_DEFAULTS["auto_issue"] if auto is None else False),
|
||||
"issue_daily_limit": limit if _daily_limit_ok(limit) else LEDGER_DEFAULTS["issue_daily_limit"],
|
||||
"match_timeout_s": timeout if _timeout_ok(timeout) else LEDGER_DEFAULTS["match_timeout_s"],
|
||||
"recall_limit": recall if _recall_limit_ok(recall) else LEDGER_DEFAULTS["recall_limit"],
|
||||
}
|
||||
|
||||
|
||||
async def ledger_settings(session: AsyncSession) -> dict[str, Any]:
|
||||
"""设置里的 ledger 一组,没存过的补默认。按口径出具、比对从这里读。"""
|
||||
row = await session.get(Setting, LEDGER_GROUP)
|
||||
return ledger_values(row.value if row else None)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LEDGER_DAILY_LIMIT_MAX",
|
||||
"LEDGER_DEFAULTS",
|
||||
"LEDGER_GROUP",
|
||||
"LEDGER_RECALL_MAX",
|
||||
"LEDGER_SETTING_LABEL",
|
||||
"LEDGER_TIMEOUT_RANGE",
|
||||
"ledger_problem",
|
||||
"ledger_settings",
|
||||
"ledger_values",
|
||||
]
|
||||
@@ -46,10 +46,10 @@ class _Hide:
|
||||
"""按这次运行查到过的遮罩值遮:hide(值) 这个值给不给,shown(文字) 文字里的遮罩值换成「已遮罩」。"""
|
||||
|
||||
def __init__(self, secrets: Any) -> None:
|
||||
from app.api.evidence import _Shown
|
||||
from app.engine.evidence_reader import ShownText
|
||||
|
||||
self._secrets = secrets
|
||||
self._shown = _Shown(secrets)
|
||||
self._shown = ShownText(secrets)
|
||||
|
||||
def hide(self, value: Any) -> bool:
|
||||
if self._secrets is None or value is None or isinstance(value, bool):
|
||||
@@ -61,9 +61,9 @@ class _Hide:
|
||||
|
||||
|
||||
async def _hider(sealed: Any) -> _Hide:
|
||||
from app.api.evidence import _Masks
|
||||
from app.engine.evidence_reader import open_masks
|
||||
|
||||
return _Hide(await _Masks(sealed).run_secrets())
|
||||
return _Hide(await open_masks(sealed).run_secrets())
|
||||
|
||||
|
||||
async def _names(ids: set[str]) -> dict[str, str]:
|
||||
@@ -226,10 +226,8 @@ async def run_view(run_id: str) -> dict[str, Any]:
|
||||
"""GET /api/runs/{run_id}/ledger 的答复:标签、整题判断、每条查询的比对、能做的动作。没比对过、能比的运行这时排上按需比对
|
||||
(trigger=on_demand),答复 state=pending;不比的照实说为什么。"""
|
||||
from app.api.coded import CodedHTTPException
|
||||
from app.api.deps import restricted
|
||||
from app.api.evidence import RUN_NOT_FOUND, _sealed
|
||||
|
||||
from app.api.deps import current_principal
|
||||
from app.api.deps import current_principal, restricted
|
||||
from app.engine.evidence_reader import RUN_NOT_FOUND, load_sealed as _sealed
|
||||
from app.ledger.recall import answer_marks
|
||||
|
||||
ordinary = restricted()
|
||||
|
||||
+3
-1
@@ -171,9 +171,11 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]:
|
||||
await relay.stop()
|
||||
await cluster.stop()
|
||||
# 应用库的连接全关掉:SQLite 最后一个连接关掉时才把 -wal 并回、删掉(copy-to-postgres 按它认服务停没停),
|
||||
# PostgreSQL 的连接池也正常关
|
||||
# PostgreSQL 的连接池也正常关;外部数据源缓存的连接池一并释放
|
||||
from app.data.engine import engines
|
||||
from app.db.base import dispose_session_engine, engine as db_engine
|
||||
|
||||
await engines.close()
|
||||
await db_engine.dispose()
|
||||
await dispose_session_engine()
|
||||
logger.info("已停止")
|
||||
|
||||
@@ -22,6 +22,8 @@ from app.memory.embeddings import _tokens
|
||||
|
||||
_K1 = 1.5
|
||||
_B = 0.75
|
||||
_IN_BATCH = 500
|
||||
_REBUILD_BATCH = 256
|
||||
|
||||
|
||||
def term_freqs(text: str) -> dict[str, int]:
|
||||
@@ -47,11 +49,69 @@ async def drop_chunk_terms(session: AsyncSession, chunk_ids: list[str]) -> None:
|
||||
"""删片段时一起删倒排。
|
||||
|
||||
外键写了 ondelete=CASCADE,但 SQLite 的级联要连接级 PRAGMA 打开才生效
|
||||
(base.py 里是打开的)——索引一致性不该依赖一个开关,显式删。
|
||||
(base.py 里是打开的)——索引一致性不该依赖一个开关,显式按批删,避免超过
|
||||
SQLite / PostgreSQL 单条语句绑定参数上限(32,766)。
|
||||
"""
|
||||
if chunk_ids:
|
||||
await session.execute(
|
||||
delete(ChunkTerm).where(ChunkTerm.chunk_id.in_(chunk_ids)))
|
||||
for i in range(0, len(chunk_ids), _IN_BATCH):
|
||||
part = chunk_ids[i:i + _IN_BATCH]
|
||||
if part:
|
||||
await session.execute(
|
||||
delete(ChunkTerm).where(ChunkTerm.chunk_id.in_(part)))
|
||||
|
||||
|
||||
async def candidates_with_scores(
|
||||
session: AsyncSession, *, collection: str | None, query: str, limit: int,
|
||||
) -> tuple[list[tuple[str, float]], dict[str, float]]:
|
||||
"""按 BM25 取候选,返回 (前 limit 条 [(chunk_id, 分数)], 全部命中片段的精确 BM25 分数字典)。
|
||||
|
||||
直接通过 ChunkTerm JOIN Chunk 一次性取回 token_len,彻底消除大集合高频词命中数万条时
|
||||
`Chunk.id.in_(...)` 超出数据库 32,767 参数上限崩溃的问题。
|
||||
"""
|
||||
# 按查询词的出现次数加权,不能去重。原来的 BM25.score 遍历的是带重复的
|
||||
# 词表:「导出订单最多多少行」里「多」出现两次,它的贡献就加两次。去重会让
|
||||
# 同一个查询在倒排和全表扫两条路上给出不同的分——实测顶部那条差了 2.32
|
||||
qcounts = Counter(t for t in _tokens(query) if len(t) <= 64)
|
||||
terms = list(qcounts)
|
||||
if not terms:
|
||||
return [], {}
|
||||
|
||||
rows: list[Any] = []
|
||||
for i in range(0, len(terms), _IN_BATCH):
|
||||
part = terms[i:i + _IN_BATCH]
|
||||
stmt = (
|
||||
select(ChunkTerm.chunk_id, ChunkTerm.term, ChunkTerm.tf, Chunk.token_len)
|
||||
.join(Chunk, Chunk.id == ChunkTerm.chunk_id)
|
||||
.where(ChunkTerm.term.in_(part))
|
||||
)
|
||||
if collection:
|
||||
stmt = stmt.where(ChunkTerm.collection == collection)
|
||||
rows.extend((await session.execute(stmt)).all())
|
||||
if not rows:
|
||||
return [], {}
|
||||
|
||||
# df 直接从命中里数:含有该词的片段数,正是 BM25 要的那个 df
|
||||
df: dict[str, int] = defaultdict(int)
|
||||
for _chunk_id, term, _tf, _tlen in rows:
|
||||
df[term] += 1
|
||||
|
||||
stat = select(func.count(Chunk.id), func.sum(Chunk.token_len))
|
||||
if collection:
|
||||
stat = stat.where(Chunk.collection == collection)
|
||||
total, total_len = (await session.execute(stat)).one()
|
||||
total = int(total or 0)
|
||||
if not total:
|
||||
return [], {}
|
||||
avg_len = float(total_len or 0) / total or 1.0
|
||||
|
||||
scores: dict[str, float] = defaultdict(float)
|
||||
for chunk_id, term, tf, token_len in rows:
|
||||
idf = math.log(1 + (total - df[term] + 0.5) / (df[term] + 0.5))
|
||||
length = int(token_len or 0) or 1
|
||||
denom = tf + _K1 * (1 - _B + _B * length / avg_len)
|
||||
scores[chunk_id] += qcounts[term] * idf * (tf * (_K1 + 1)) / denom
|
||||
|
||||
ranked = sorted(scores.items(), key=lambda kv: (-kv[1], kv[0]))
|
||||
return ranked[:limit], dict(scores)
|
||||
|
||||
|
||||
async def candidates(
|
||||
@@ -61,51 +121,16 @@ async def candidates(
|
||||
|
||||
没有任何查询词命中时返回空——调用方据此决定是不是要退回全表。
|
||||
"""
|
||||
# 按查询词的出现次数加权,不能去重。原来的 BM25.score 遍历的是带重复的
|
||||
# 词表:「导出订单最多多少行」里「多」出现两次,它的贡献就加两次。去重会让
|
||||
# 同一个查询在倒排和全表扫两条路上给出不同的分——实测顶部那条差了 2.32
|
||||
qcounts = Counter(t for t in _tokens(query) if len(t) <= 64)
|
||||
terms = list(qcounts)
|
||||
if not terms:
|
||||
return []
|
||||
top, _ = await candidates_with_scores(session, collection=collection, query=query, limit=limit)
|
||||
return top
|
||||
|
||||
stmt = select(ChunkTerm.chunk_id, ChunkTerm.term, ChunkTerm.tf).where(
|
||||
ChunkTerm.term.in_(terms))
|
||||
|
||||
async def has_terms(session: AsyncSession, collection: str | None = None) -> bool:
|
||||
"""集合里是否已建有倒排词项(区分「真没命中查询词」与「存量升级尚未建倒排」)。"""
|
||||
stmt = select(ChunkTerm.id).limit(1)
|
||||
if collection:
|
||||
stmt = stmt.where(ChunkTerm.collection == collection)
|
||||
rows = list(await session.execute(stmt))
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
# df 直接从命中里数:含有该词的片段数,正是 BM25 要的那个 df
|
||||
df: dict[str, int] = defaultdict(int)
|
||||
for _chunk_id, term, _tf in rows:
|
||||
df[term] += 1
|
||||
|
||||
stat = select(func.count(Chunk.id), func.sum(Chunk.token_len))
|
||||
if collection:
|
||||
stat = stat.where(Chunk.collection == collection)
|
||||
total, total_len = (await session.execute(stat)).one()
|
||||
total = int(total or 0)
|
||||
if not total:
|
||||
return []
|
||||
avg_len = float(total_len or 0) / total or 1.0
|
||||
|
||||
# 要 .all():Result 自带 keys(),dict() 会当它是映射去下标取值,
|
||||
# 抛 'ChunkedIteratorResult' object is not subscriptable
|
||||
lengths = dict((await session.execute(
|
||||
select(Chunk.id, Chunk.token_len).where(
|
||||
Chunk.id.in_({r[0] for r in rows})))).all())
|
||||
|
||||
scores: dict[str, float] = defaultdict(float)
|
||||
for chunk_id, term, tf in rows:
|
||||
idf = math.log(1 + (total - df[term] + 0.5) / (df[term] + 0.5))
|
||||
length = lengths.get(chunk_id) or 1
|
||||
denom = tf + _K1 * (1 - _B + _B * length / avg_len)
|
||||
scores[chunk_id] += qcounts[term] * idf * (tf * (_K1 + 1)) / denom
|
||||
|
||||
ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)
|
||||
return ranked[:limit]
|
||||
return (await session.execute(stmt)).first() is not None
|
||||
|
||||
|
||||
async def missing_count(session: AsyncSession, collection: str | None = None) -> int:
|
||||
@@ -117,15 +142,20 @@ async def missing_count(session: AsyncSession, collection: str | None = None) ->
|
||||
|
||||
|
||||
async def rebuild(session: AsyncSession, collection: str | None = None) -> int:
|
||||
"""重建倒排。存量片段升级后要跑一次,改了分词器也要。"""
|
||||
stmt = select(Chunk)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
if not rows:
|
||||
return 0
|
||||
await drop_chunk_terms(session, [r.id for r in rows])
|
||||
for row in rows:
|
||||
await index_chunk(session, row)
|
||||
await session.commit()
|
||||
return len(rows)
|
||||
"""按游标分批重建倒排,避免一次性把全库片段加载进内存。"""
|
||||
total = 0
|
||||
last_id = ""
|
||||
while True:
|
||||
stmt = select(Chunk).where(Chunk.id > last_id).order_by(Chunk.id.asc()).limit(_REBUILD_BATCH)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
batch = list((await session.execute(stmt)).scalars())
|
||||
if not batch:
|
||||
break
|
||||
last_id = batch[-1].id
|
||||
await drop_chunk_terms(session, [r.id for r in batch])
|
||||
for row in batch:
|
||||
await index_chunk(session, row)
|
||||
await session.commit()
|
||||
total += len(batch)
|
||||
return total
|
||||
|
||||
+237
-78
@@ -1,9 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import OrderedDict
|
||||
from dataclasses import dataclass
|
||||
import heapq
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
import numpy as np
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db.models import Chunk, Document
|
||||
@@ -18,6 +22,29 @@ _OVERLAP = 120
|
||||
#: 每批送多少段去算向量。远端接口单次批量有上限(OpenAI 是 2048),而且
|
||||
#: 一次把上万段全发出去,要么超时要么被限流,中途失败还没有断点
|
||||
_EMBED_BATCH = 64
|
||||
_ID_FETCH_BATCH = 500
|
||||
_VEC_SCAN_BATCH = 512
|
||||
_VEC_CACHE_MAX_COLLECTIONS = 16
|
||||
_VEC_CACHE_MAX_CHUNKS = 200_000
|
||||
|
||||
|
||||
@dataclass
|
||||
class _VectorCacheEntry:
|
||||
stamp: tuple[int, str, str, int]
|
||||
chunk_ids: list[str]
|
||||
matrix: np.ndarray
|
||||
|
||||
|
||||
_VEC_CACHE: OrderedDict[str, _VectorCacheEntry] = OrderedDict()
|
||||
|
||||
|
||||
def invalidate_vector_cache(collection: str | None = None) -> None:
|
||||
"""文档增删或重建索引后失效进程内归一化向量矩阵缓存。"""
|
||||
if collection is None:
|
||||
_VEC_CACHE.clear()
|
||||
return
|
||||
_VEC_CACHE.pop(collection, None)
|
||||
_VEC_CACHE.pop("__all__", None)
|
||||
|
||||
|
||||
def chunk_text(text: str, target: int = _TARGET, overlap: int = _OVERLAP) -> list[str]:
|
||||
@@ -216,10 +243,137 @@ async def process_document(
|
||||
doc.status = "ready"
|
||||
doc.error = ""
|
||||
await session.commit()
|
||||
invalidate_vector_cache(collection)
|
||||
await session.refresh(doc)
|
||||
return doc
|
||||
|
||||
|
||||
async def _fetch_chunks_by_ids(session: AsyncSession, chunk_ids: list[str]) -> dict[str, Chunk]:
|
||||
found: dict[str, Chunk] = {}
|
||||
for i in range(0, len(chunk_ids), _ID_FETCH_BATCH):
|
||||
part = chunk_ids[i:i + _ID_FETCH_BATCH]
|
||||
if not part:
|
||||
continue
|
||||
for row in (await session.execute(select(Chunk).where(Chunk.id.in_(part)))).scalars():
|
||||
found[row.id] = row
|
||||
return found
|
||||
|
||||
|
||||
def _unit_vector(vec: np.ndarray) -> np.ndarray:
|
||||
arr = np.asarray(vec, dtype=np.float32)
|
||||
norm = float(np.linalg.norm(arr))
|
||||
return arr / norm if norm > 0 else np.zeros_like(arr, dtype=np.float32)
|
||||
|
||||
|
||||
async def _vector_top_ids(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
collection: str | None,
|
||||
query_vec: np.ndarray,
|
||||
limit: int,
|
||||
) -> list[str]:
|
||||
"""全集合语义向量路独立 Top-K 召回(不被 BM25 Top-50 前置截断)。
|
||||
|
||||
中小型集合缓存 L2 归一化 `float32` 矩阵,单次检索仅需一次 BLAS 矩阵乘法;
|
||||
超大集合(> 20 万块)自动退化为游标分批扫描 + 小顶堆,保证常驻内存有界。
|
||||
"""
|
||||
if limit <= 0:
|
||||
return []
|
||||
model_id, dim = embedder_id(), embedder_dim()
|
||||
stat_stmt = select(func.count(Chunk.id), func.max(Chunk.id))
|
||||
if collection:
|
||||
stat_stmt = stat_stmt.where(Chunk.collection == collection)
|
||||
total_raw, max_id_raw = (await session.execute(stat_stmt)).one()
|
||||
total = int(total_raw or 0)
|
||||
if total == 0:
|
||||
return []
|
||||
q_unit = _unit_vector(query_vec)
|
||||
if q_unit.shape[0] != dim:
|
||||
return []
|
||||
|
||||
cache_key = collection or "__all__"
|
||||
stamp = (total, str(max_id_raw or ""), model_id, dim)
|
||||
|
||||
if total <= _VEC_CACHE_MAX_CHUNKS:
|
||||
entry = _VEC_CACHE.get(cache_key)
|
||||
if entry is None or entry.stamp != stamp:
|
||||
ids: list[str] = []
|
||||
vecs: list[np.ndarray] = []
|
||||
last_id = ""
|
||||
while True:
|
||||
stmt = (
|
||||
select(Chunk.id, Chunk.embedding)
|
||||
.where(Chunk.id > last_id)
|
||||
.order_by(Chunk.id.asc())
|
||||
.limit(_VEC_SCAN_BATCH)
|
||||
)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
batch = (await session.execute(stmt)).all()
|
||||
if not batch:
|
||||
break
|
||||
last_id = str(batch[-1][0])
|
||||
for cid, blob in batch:
|
||||
v = from_blob(blob, dim)
|
||||
ids.append(str(cid))
|
||||
vecs.append(_unit_vector(v) if v is not None else np.zeros(dim, dtype=np.float32))
|
||||
matrix = np.vstack(vecs).astype(np.float32, copy=False) if vecs else np.zeros((0, dim), dtype=np.float32)
|
||||
entry = _VectorCacheEntry(stamp=stamp, chunk_ids=ids, matrix=matrix)
|
||||
_VEC_CACHE[cache_key] = entry
|
||||
_VEC_CACHE.move_to_end(cache_key)
|
||||
while len(_VEC_CACHE) > _VEC_CACHE_MAX_COLLECTIONS:
|
||||
_VEC_CACHE.popitem(last=False)
|
||||
else:
|
||||
_VEC_CACHE.move_to_end(cache_key)
|
||||
|
||||
if entry.matrix.shape[0] == 0:
|
||||
return []
|
||||
scores = entry.matrix @ q_unit
|
||||
n = scores.shape[0]
|
||||
if n <= limit:
|
||||
order = sorted(range(n), key=lambda idx: (-float(scores[idx]), entry.chunk_ids[idx]))
|
||||
return [entry.chunk_ids[idx] for idx in order]
|
||||
top_idx = np.argpartition(-scores, limit - 1)[:limit]
|
||||
ordered = sorted((int(idx) for idx in top_idx), key=lambda idx: (-float(scores[idx]), entry.chunk_ids[idx]))
|
||||
return [entry.chunk_ids[idx] for idx in ordered]
|
||||
|
||||
# 超大集合流式扫描兜底
|
||||
heap: list[tuple[float, str]] = []
|
||||
last_id = ""
|
||||
while True:
|
||||
stmt = (
|
||||
select(Chunk.id, Chunk.embedding)
|
||||
.where(Chunk.id > last_id)
|
||||
.order_by(Chunk.id.asc())
|
||||
.limit(_VEC_SCAN_BATCH)
|
||||
)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
batch = (await session.execute(stmt)).all()
|
||||
if not batch:
|
||||
break
|
||||
last_id = str(batch[-1][0])
|
||||
for cid, blob in batch:
|
||||
v = from_blob(blob, dim)
|
||||
sc = float(np.dot(_unit_vector(v), q_unit)) if v is not None else 0.0
|
||||
if len(heap) < limit:
|
||||
heapq.heappush(heap, (sc, str(cid)))
|
||||
elif sc > heap[0][0]:
|
||||
heapq.heapreplace(heap, (sc, str(cid)))
|
||||
heap.sort(key=lambda item: (-item[0], item[1]))
|
||||
return [cid for _, cid in heap]
|
||||
|
||||
|
||||
async def _stale_model_names(session: AsyncSession, collection: str | None) -> list[str]:
|
||||
stmt = select(Chunk.embed_model).distinct().where(
|
||||
or_(Chunk.embed_model != embedder_id(), Chunk.embed_dim != embedder_dim())
|
||||
)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
names = [str(m or "未知模型") for m in (await session.execute(stmt)).scalars()]
|
||||
return sorted(set(names)) or ["未知模型"]
|
||||
|
||||
|
||||
async def search(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
@@ -229,76 +383,79 @@ async def search(
|
||||
alpha: float | None = None,
|
||||
on_degrade: Any = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""混合检索。
|
||||
"""真·双路混合检索(词面 BM25 独立召回 + 全集合向量矩阵独立召回)。
|
||||
|
||||
存量向量和当前 embedder 对不上时**整个查询退回纯关键词**,并通过
|
||||
on_degrade 把原因交出去。不把对不上的那些当 0 分:那会让它们静默沉底,
|
||||
用户看到的是结果莫名其妙变差,而不是"这里有一批索引该重建了"。
|
||||
|
||||
在此之前这里连维度都不查,换个 embedder 直接 ValueError
|
||||
(shapes (1536,) and (512,) not aligned),整个检索 500。
|
||||
"""
|
||||
# 没显式指定就跟着 embedder 走——写死 0.5 是在给一个没有语义能力的信号
|
||||
# 一半权重,实测会把 hit@1 从 88% 拉到 81%
|
||||
if alpha is None:
|
||||
alpha = default_alpha()
|
||||
|
||||
# 先用倒排把候选收窄。以前这里是把整个集合载入内存、再在 Python 里重建
|
||||
# 一遍 BM25 倒排——每次查询都重新分词一遍所有片段。
|
||||
#
|
||||
# 多捞一些(limit 的 8 倍):倒排只按关键词收,最终名次还要混进向量那一路,
|
||||
# 卡得太紧会把"关键词一般但语义很近"的那些提前筛掉。
|
||||
rows: list[Chunk] = []
|
||||
kw_scores: list[float] | None = None
|
||||
picked = await inverted.candidates(
|
||||
session, collection=collection, query=query, limit=max(limit * 8, 50))
|
||||
if picked:
|
||||
found = dict(
|
||||
(c.id, c) for c in (await session.execute(
|
||||
select(Chunk).where(Chunk.id.in_([cid for cid, _ in picked])))).scalars())
|
||||
pairs = [(found[cid], sc) for cid, sc in picked if cid in found]
|
||||
rows = [c for c, _ in pairs]
|
||||
# 倒排算好的分要带下去。在候选子集上重算 BM25,df 和平均长度是另一套数
|
||||
kw_scores = [sc for _, sc in pairs]
|
||||
|
||||
if not rows:
|
||||
# 一个查询词都没命中,或者这批片段还没建倒排(存量数据升级上来就是这样)。
|
||||
# 退回全表扫,慢但不会漏——宁可慢,也不要因为索引没建好就说"没搜到"
|
||||
stmt = select(Chunk)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
kw_scores = None # 全表扫这条路照旧自己算
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
# 按 id 定序再排。hybrid_rank 用的是稳定排序,分数并列时保留输入顺序——
|
||||
# 而倒排那条路和全表扫那条路喂进去的顺序天生不同,同样的查询会因为走了
|
||||
# 哪条路给出不同的名次。并列本来就该有个确定的先后,用 id 就够了
|
||||
if kw_scores is not None:
|
||||
paired = sorted(zip(rows, kw_scores), key=lambda t: t[0].id)
|
||||
rows = [r for r, _ in paired]
|
||||
kw_scores = [s2 for _, s2 in paired]
|
||||
else:
|
||||
rows = sorted(rows, key=lambda r: r.id)
|
||||
|
||||
stale = [r for r in rows if not usable(r.embed_model, r.embed_dim)]
|
||||
if stale:
|
||||
total_stale = await stale_count(session, collection)
|
||||
if total_stale > 0:
|
||||
if on_degrade:
|
||||
names = sorted({r.embed_model or "未知模型" for r in stale})
|
||||
# 报的是整个集合里要重建多少条,不是本次候选里有几条——候选被倒排
|
||||
# 收窄过,用它会把问题说小,而用户要照着做的恰恰是"重建多少"
|
||||
total_stale = await stale_count(session, collection)
|
||||
names = await _stale_model_names(session, collection)
|
||||
on_degrade(
|
||||
f"{collection or '全部集合'}里有 {total_stale} 条向量由"
|
||||
f"「{'、'.join(names)}」建立,与当前的「{embedder_id()}」不一致,"
|
||||
f"本次已退回纯关键词检索。重建索引后恢复。"
|
||||
)
|
||||
alpha = 0.0 # 纯关键词
|
||||
vectors: list[Any] = [None] * len(rows)
|
||||
query_vec = await embed_text(query)
|
||||
alpha = 0.0
|
||||
|
||||
query_vec = await embed_text(query)
|
||||
cand_limit = max(limit * 8, 50)
|
||||
|
||||
# 第一路:倒排索引词面 BM25 Top-K(返回精确全集合 BM25 得分字典)
|
||||
lex_picked, all_kw_scores = await inverted.candidates_with_scores(
|
||||
session, collection=collection, query=query, limit=cand_limit,
|
||||
)
|
||||
lex_ids = [cid for cid, _ in lex_picked]
|
||||
|
||||
# 第二路:当启用语义权重时,独立从全集合归一化向量索引召回语义 Top-K,不再被 BM25 Top-50 卡死
|
||||
vec_ids = (
|
||||
await _vector_top_ids(session, collection=collection, query_vec=query_vec, limit=cand_limit)
|
||||
if alpha > 0.0
|
||||
else []
|
||||
)
|
||||
|
||||
merged_ids = sorted(set(lex_ids) | set(vec_ids))
|
||||
rows: list[Chunk] = []
|
||||
kw_scores: list[float] | None = None
|
||||
|
||||
if merged_ids:
|
||||
found = await _fetch_chunks_by_ids(session, merged_ids)
|
||||
rows = [found[cid] for cid in merged_ids if cid in found]
|
||||
if lex_picked or await inverted.has_terms(session, collection):
|
||||
kw_scores = [all_kw_scores.get(r.id, 0.0) for r in rows]
|
||||
else:
|
||||
kw_scores = None
|
||||
else:
|
||||
if await inverted.has_terms(session, collection):
|
||||
# 倒排已就绪但没有任何查询词命中且 alpha == 0:所有片段词面分均为 0,只需按 id 定序取前 limit 条
|
||||
stmt = select(Chunk).order_by(Chunk.id.asc()).limit(limit)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
kw_scores = [0.0] * len(rows)
|
||||
else:
|
||||
# 存量数据升级尚未建倒排:退回全表扫现算 BM25,保证不漏检
|
||||
stmt = select(Chunk).order_by(Chunk.id.asc())
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
kw_scores = None
|
||||
|
||||
if not rows:
|
||||
return []
|
||||
|
||||
if total_stale > 0 or alpha == 0.0:
|
||||
vectors: list[Any] = (
|
||||
[None] * len(rows)
|
||||
if total_stale > 0
|
||||
else [from_blob(r.embedding, embedder_dim()) for r in rows]
|
||||
)
|
||||
else:
|
||||
query_vec = await embed_text(query)
|
||||
vectors = [from_blob(r.embedding, embedder_dim()) for r in rows]
|
||||
|
||||
ranked = hybrid_contrib(
|
||||
@@ -313,7 +470,6 @@ async def search(
|
||||
for hit in ranked[:limit]:
|
||||
row = rows[hit.index]
|
||||
score = round(hit.score, 4)
|
||||
# 关键词那份用减法得出:各自四舍五入之后再相加,会和总分差出一个末位
|
||||
by_vec = min(round(hit.contrib["vector"], 4), score)
|
||||
out.append(
|
||||
{
|
||||
@@ -324,7 +480,6 @@ async def search(
|
||||
"content": row.content,
|
||||
"score": score,
|
||||
"signals": {k: round(v, 4) for k, v in hit.signals.items()},
|
||||
# 两路各贡献了多少分,两者之和等于 score。调试台据此画贡献条
|
||||
"contrib": {"vector": by_vec, "keyword": round(score - by_vec, 4)},
|
||||
}
|
||||
)
|
||||
@@ -334,34 +489,38 @@ async def search(
|
||||
async def reindex(
|
||||
session: AsyncSession, collection: str | None = None, *, on_progress: Any = None,
|
||||
) -> dict[str, Any]:
|
||||
"""用当前 embedder 重算向量。
|
||||
|
||||
换了 embedding 模型之后唯一的恢复手段。按批做:一次几千条 chunk 全塞给
|
||||
远端 embedding 接口,要么超时要么被限流,而中途失败又没有断点。
|
||||
on_progress(已完成, 总数) 每批提交之后回调一次。
|
||||
"""
|
||||
stmt = select(Chunk)
|
||||
"""用当前 embedder 按游标分批重算向量并重建倒排,避免全表一次性载入内存。"""
|
||||
count_stmt = select(func.count(Chunk.id))
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
if not rows:
|
||||
return {"collection": collection, "reindexed": 0, "embedder": embedder_id()}
|
||||
|
||||
count_stmt = count_stmt.where(Chunk.collection == collection)
|
||||
total = int((await session.execute(count_stmt)).scalar_one() or 0)
|
||||
model_id, dim = embedder_id(), embedder_dim()
|
||||
batch = _EMBED_BATCH
|
||||
for i in range(0, len(rows), batch):
|
||||
part = rows[i:i + batch]
|
||||
if total == 0:
|
||||
invalidate_vector_cache(collection)
|
||||
return {"collection": collection, "reindexed": 0, "embedder": model_id}
|
||||
|
||||
done = 0
|
||||
last_id = ""
|
||||
while True:
|
||||
stmt = select(Chunk).where(Chunk.id > last_id).order_by(Chunk.id.asc()).limit(_EMBED_BATCH)
|
||||
if collection:
|
||||
stmt = stmt.where(Chunk.collection == collection)
|
||||
part = list((await session.execute(stmt)).scalars())
|
||||
if not part:
|
||||
break
|
||||
last_id = part[-1].id
|
||||
vectors = await embed_texts([r.content for r in part])
|
||||
for row, vec in zip(part, vectors):
|
||||
row.embedding = to_blob(vec)
|
||||
row.embed_model = model_id
|
||||
row.embed_dim = dim
|
||||
await session.commit()
|
||||
done += len(part)
|
||||
if on_progress:
|
||||
on_progress(min(i + batch, len(rows)), len(rows))
|
||||
# 倒排一起重建:两者都是"存量数据升级"的恢复手段,分成两个按钮只会漏点一个
|
||||
on_progress(min(done, total), total)
|
||||
await inverted.rebuild(session, collection)
|
||||
return {"collection": collection, "reindexed": len(rows), "embedder": model_id}
|
||||
invalidate_vector_cache(collection)
|
||||
return {"collection": collection, "reindexed": done, "embedder": model_id}
|
||||
|
||||
|
||||
async def stale_count(session: AsyncSession, collection: str | None = None) -> int:
|
||||
@@ -370,8 +529,6 @@ async def stale_count(session: AsyncSession, collection: str | None = None) -> i
|
||||
用 SQL 数而不是把片段全载进来再在 Python 里过一遍——这个数在检索降级时
|
||||
也要取,那条路径上不该再来一次全表扫。
|
||||
"""
|
||||
from sqlalchemy import func, or_
|
||||
|
||||
stmt = select(func.count(Chunk.id)).where(
|
||||
or_(Chunk.embed_model != embedder_id(), Chunk.embed_dim != embedder_dim()))
|
||||
if collection:
|
||||
@@ -383,11 +540,13 @@ async def delete_document(session: AsyncSession, doc_id: str) -> bool:
|
||||
doc = await session.get(Document, doc_id)
|
||||
if not doc:
|
||||
return False
|
||||
collection = doc.collection
|
||||
chunk_ids = list((await session.execute(
|
||||
select(Chunk.id).where(Chunk.document_id == doc_id))).scalars())
|
||||
await inverted.drop_chunk_terms(session, chunk_ids)
|
||||
await session.delete(doc)
|
||||
await session.commit()
|
||||
invalidate_vector_cache(collection)
|
||||
return True
|
||||
|
||||
|
||||
|
||||
+22
-13
@@ -133,6 +133,7 @@ async def recall(
|
||||
)
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
picked_rows: list[MemoryItem] = []
|
||||
now = datetime.now(timezone.utc)
|
||||
for idx, score, parts in ranked:
|
||||
row = rows[idx]
|
||||
@@ -140,6 +141,7 @@ async def recall(
|
||||
boosted = score * (0.8 + 0.4 * row.importance) * (1 + min(row.use_count, 10) * 0.01)
|
||||
if boosted < min_score:
|
||||
continue
|
||||
picked_rows.append(row)
|
||||
results.append(
|
||||
{
|
||||
"id": row.id,
|
||||
@@ -158,11 +160,9 @@ async def recall(
|
||||
if not track:
|
||||
return results
|
||||
# 记一下被召回过,作为下次排序的信号
|
||||
for r in results:
|
||||
item = await session.get(MemoryItem, r["id"])
|
||||
if item:
|
||||
item.use_count += 1
|
||||
item.last_used_at = now
|
||||
for item in picked_rows:
|
||||
item.use_count += 1
|
||||
item.last_used_at = now
|
||||
await session.commit()
|
||||
return results
|
||||
|
||||
@@ -241,28 +241,37 @@ async def reindex(
|
||||
路径——记忆这边悄悄退回关键词,而且没有任何恢复手段。
|
||||
on_progress(已完成, 总数) 每批提交之后回调一次。
|
||||
"""
|
||||
stmt = select(MemoryItem)
|
||||
count_stmt = select(func.count(MemoryItem.id))
|
||||
if scope:
|
||||
stmt = stmt.where(MemoryItem.scope == scope)
|
||||
rows = list((await session.execute(stmt)).scalars())
|
||||
if not rows:
|
||||
count_stmt = count_stmt.where(MemoryItem.scope == scope)
|
||||
total = int((await session.execute(count_stmt)).scalar_one() or 0)
|
||||
if total == 0:
|
||||
return 0
|
||||
|
||||
model_id, dim = embedder_id(), embedder_dim()
|
||||
from app.memory import kb
|
||||
|
||||
batch = kb._EMBED_BATCH # 和 kb 那边统一:一次几千条全发给远端接口必被限流
|
||||
for i in range(0, len(rows), batch):
|
||||
part = rows[i:i + batch]
|
||||
done = 0
|
||||
last_id = ""
|
||||
while True:
|
||||
stmt = select(MemoryItem).where(MemoryItem.id > last_id).order_by(MemoryItem.id.asc()).limit(batch)
|
||||
if scope:
|
||||
stmt = stmt.where(MemoryItem.scope == scope)
|
||||
part = list((await session.execute(stmt)).scalars())
|
||||
if not part:
|
||||
break
|
||||
last_id = part[-1].id
|
||||
vectors = await embed_texts([r.content for r in part])
|
||||
for row, vec in zip(part, vectors):
|
||||
row.embedding = to_blob(vec)
|
||||
row.embed_model = model_id
|
||||
row.embed_dim = dim
|
||||
await session.commit()
|
||||
done += len(part)
|
||||
if on_progress:
|
||||
on_progress(min(i + batch, len(rows)), len(rows))
|
||||
return len(rows)
|
||||
on_progress(min(done, total), total)
|
||||
return done
|
||||
|
||||
|
||||
async def list_scopes(session: AsyncSession) -> list[dict[str, Any]]:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -187,14 +188,52 @@ def needs_interleaved_beta(model: str) -> bool:
|
||||
return name.startswith(_INTERLEAVED_BETA_PREFIXES)
|
||||
|
||||
|
||||
_OPENAI_REASONING_RE = re.compile(r"^o[134](?:$|[-_])")
|
||||
#: 用户在「设置 → 模型接入」里填了自定义价格或上下文窗口的模型(由 list_providers 同步)
|
||||
_CUSTOM_MODELS: dict[str, dict[str, Any]] = {}
|
||||
|
||||
|
||||
def sync_provider_models(providers: list[Any]) -> None:
|
||||
"""把当前启用的模型接入里带了自定义 pricing / context 的模型同步进内存表。"""
|
||||
custom: dict[str, dict[str, Any]] = {}
|
||||
for p in providers or ():
|
||||
if not getattr(p, "enabled", True):
|
||||
continue
|
||||
for entry in getattr(p, "models", None) or ():
|
||||
if not isinstance(entry, dict) or not entry.get("id"):
|
||||
continue
|
||||
key = _norm(str(entry["id"]))
|
||||
if not key:
|
||||
continue
|
||||
info: dict[str, Any] = {}
|
||||
pricing = entry.get("pricing")
|
||||
if isinstance(pricing, dict) and ("input" in pricing or "output" in pricing):
|
||||
info["pricing"] = {
|
||||
"input": float(pricing.get("input") or 0.0),
|
||||
"output": float(pricing.get("output") or 0.0),
|
||||
**({"cache_read": float(pricing["cache_read"])} if "cache_read" in pricing and pricing["cache_read"] is not None else {}),
|
||||
**({"cache_creation": float(pricing["cache_creation"])} if "cache_creation" in pricing and pricing["cache_creation"] is not None else {}),
|
||||
}
|
||||
ctx_val = entry.get("context")
|
||||
if isinstance(ctx_val, (int, float)) and not isinstance(ctx_val, bool) and ctx_val > 0:
|
||||
info["context"] = int(ctx_val)
|
||||
if info:
|
||||
custom[key] = info
|
||||
_CUSTOM_MODELS.clear()
|
||||
_CUSTOM_MODELS.update(custom)
|
||||
|
||||
|
||||
def _norm(model: str) -> str:
|
||||
"""去掉 openrouter 之类的前缀,只看模型名本身。"""
|
||||
return (model or "").split("/")[-1].strip()
|
||||
"""去掉 openrouter 之类的前缀并统一小写,只看模型名本身。"""
|
||||
return (model or "").split("/")[-1].strip().lower()
|
||||
|
||||
|
||||
def supports_sampling(model: str) -> bool:
|
||||
"""这个模型还能不能传 temperature / top_p。"""
|
||||
return not _norm(model).startswith(_NO_SAMPLING_PREFIXES)
|
||||
name = _norm(model)
|
||||
if name.startswith(_NO_SAMPLING_PREFIXES) or _OPENAI_REASONING_RE.match(name):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def supports_adaptive_thinking(model: str) -> bool:
|
||||
@@ -213,6 +252,8 @@ def supports_effort(model: str) -> bool:
|
||||
|
||||
def find_pricing(model: str) -> dict[str, float] | None:
|
||||
target = _norm(model)
|
||||
if target in _CUSTOM_MODELS and "pricing" in _CUSTOM_MODELS[target]:
|
||||
return _CUSTOM_MODELS[target]["pricing"]
|
||||
pools = [ANTHROPIC_MODELS, OPENAI_MODELS, MOCK_MODELS]
|
||||
pools += [p["models"] for p in COMPATIBLE_PRESETS]
|
||||
for pool in pools:
|
||||
@@ -225,6 +266,8 @@ def find_pricing(model: str) -> dict[str, float] | None:
|
||||
def find_context(model: str) -> int | None:
|
||||
"""目录里登记的上下文窗口(token)。没登记的返回 None,由调用方取保守值。"""
|
||||
target = _norm(model)
|
||||
if target in _CUSTOM_MODELS and "context" in _CUSTOM_MODELS[target]:
|
||||
return _CUSTOM_MODELS[target]["context"]
|
||||
pools = [ANTHROPIC_MODELS, OPENAI_MODELS, MOCK_MODELS]
|
||||
pools += [p["models"] for p in COMPATIBLE_PRESETS]
|
||||
for pool in pools:
|
||||
@@ -251,12 +294,30 @@ def unpriced_call(data: Any) -> bool:
|
||||
and find_pricing(model) is None
|
||||
|
||||
|
||||
def estimate_cost(model: str, input_tokens: int, output_tokens: int) -> float:
|
||||
"""按目录价估算一次调用的花费(美元)。"""
|
||||
def estimate_cost(
|
||||
model: str,
|
||||
input_tokens: int,
|
||||
output_tokens: int,
|
||||
*,
|
||||
cache_read_tokens: int = 0,
|
||||
cache_creation_tokens: int = 0,
|
||||
) -> float:
|
||||
"""按目录价估算一次调用的花费(美元)。带缓存读写明细时按缓存单价折算(默认命中 0.1x、建缓存 1.25x)。"""
|
||||
pricing = find_pricing(model)
|
||||
if not pricing:
|
||||
return 0.0
|
||||
inp_rate = pricing.get("input", 0.0)
|
||||
out_rate = pricing.get("output", 0.0)
|
||||
if cache_read_tokens <= 0 and cache_creation_tokens <= 0:
|
||||
return input_tokens / 1_000_000 * inp_rate + output_tokens / 1_000_000 * out_rate
|
||||
read_tokens = max(0, int(cache_read_tokens))
|
||||
write_tokens = max(0, int(cache_creation_tokens))
|
||||
uncached = max(0, int(input_tokens) - read_tokens - write_tokens)
|
||||
read_rate = pricing.get("cache_read", inp_rate * 0.1)
|
||||
write_rate = pricing.get("cache_creation", inp_rate * 1.25)
|
||||
return (
|
||||
input_tokens / 1_000_000 * pricing.get("input", 0.0)
|
||||
+ output_tokens / 1_000_000 * pricing.get("output", 0.0)
|
||||
)
|
||||
uncached * inp_rate
|
||||
+ read_tokens * read_rate
|
||||
+ write_tokens * write_rate
|
||||
+ output_tokens * out_rate
|
||||
) / 1_000_000
|
||||
|
||||
@@ -54,7 +54,9 @@ class ProviderNotConfigured(RuntimeError):
|
||||
|
||||
async def list_providers(session: AsyncSession) -> list[Provider]:
|
||||
rows = await session.execute(select(Provider).order_by(Provider.created_at, Provider.id))
|
||||
return list(rows.scalars())
|
||||
items = list(rows.scalars())
|
||||
catalog.sync_provider_models(items)
|
||||
return items
|
||||
|
||||
|
||||
#: 点名的接入不存在、已停用时,在「设置 → 模型接入」里能做什么。NAMED_FIX 是整句的前半句;后半句(去哪里换一个
|
||||
@@ -326,8 +328,16 @@ def build_chat_model(provider: Provider, spec: ModelSpec) -> BaseChatModel:
|
||||
# 且 api_key=None 时不会发出 x-api-key。
|
||||
import anthropic
|
||||
|
||||
client_args = {"base_url": base_url, "api_key": None, "auth_token": api_key,
|
||||
"max_retries": chat.max_retries}
|
||||
client_args: dict[str, Any] = {
|
||||
"base_url": base_url,
|
||||
"api_key": None,
|
||||
"auth_token": api_key,
|
||||
"max_retries": chat.max_retries,
|
||||
}
|
||||
if kwargs.get("timeout") is not None:
|
||||
client_args["timeout"] = kwargs["timeout"]
|
||||
if kwargs.get("default_headers"):
|
||||
client_args["default_headers"] = dict(kwargs["default_headers"])
|
||||
chat._client = anthropic.Anthropic(**client_args)
|
||||
chat._async_client = anthropic.AsyncAnthropic(**client_args)
|
||||
return chat
|
||||
@@ -414,12 +424,52 @@ def _reasoning_aware_openai() -> type:
|
||||
所以在这里把它捞回 additional_kwargs,thinking_text 再统一读出来。
|
||||
只加字段,不改 langchain 对正文的任何处理。
|
||||
"""
|
||||
from langchain_core.messages import AIMessage
|
||||
from langchain_openai import ChatOpenAI
|
||||
from pydantic import PrivateAttr
|
||||
|
||||
class ReasoningAwareChatOpenAI(ChatOpenAI): # type: ignore[misc]
|
||||
_reasoning_hint: dict[str, Any] = PrivateAttr(default_factory=dict)
|
||||
|
||||
def _get_request_payload(
|
||||
self, input_: Any, *, stop: list[str] | None = None, **kwargs: Any
|
||||
) -> dict:
|
||||
payload = super()._get_request_payload(input_, stop=stop, **kwargs)
|
||||
lc_msgs = self._convert_input(input_).to_messages()
|
||||
ai_reasonings = [
|
||||
(m.additional_kwargs or {}).get("reasoning_content")
|
||||
for m in lc_msgs
|
||||
if isinstance(m, AIMessage)
|
||||
]
|
||||
raw_msgs = payload.get("messages")
|
||||
if isinstance(raw_msgs, list) and ai_reasonings:
|
||||
ai_idx = 0
|
||||
for msg_dict in raw_msgs:
|
||||
if not isinstance(msg_dict, dict) or msg_dict.get("role") != "assistant":
|
||||
continue
|
||||
reasoning = ai_reasonings[ai_idx] if ai_idx < len(ai_reasonings) else None
|
||||
ai_idx += 1
|
||||
# DeepSeek Reasoner / Kimi K2 Thinking 等在工具调用多轮往返时要求携带了 tool_calls 的
|
||||
# assistant 消息必须原样回传 reasoning_content,否则直接报 400 Missing reasoning_content field
|
||||
if isinstance(reasoning, str) and reasoning and msg_dict.get("tool_calls"):
|
||||
msg_dict.setdefault("reasoning_content", reasoning)
|
||||
return payload
|
||||
|
||||
def _create_chat_result(self, response: Any, generation_info: dict | None = None):
|
||||
res = super()._create_chat_result(response, generation_info)
|
||||
raw_dict = response if isinstance(response, dict) else (
|
||||
response.model_dump() if hasattr(response, "model_dump") else {}
|
||||
)
|
||||
choices = raw_dict.get("choices") or []
|
||||
for idx, gen in enumerate(res.generations):
|
||||
if idx >= len(choices):
|
||||
break
|
||||
msg = (choices[idx] or {}).get("message") or {}
|
||||
text = msg.get("reasoning_content") or msg.get("reasoning")
|
||||
if isinstance(text, str) and text:
|
||||
gen.message.additional_kwargs["reasoning_content"] = text
|
||||
return res
|
||||
|
||||
def _convert_chunk_to_generation_chunk(
|
||||
self, chunk: dict, default_chunk_class: type, base_generation_info: dict | None
|
||||
):
|
||||
@@ -458,19 +508,19 @@ def _reasoning_aware_openai() -> type:
|
||||
raise e from first
|
||||
|
||||
async def _astream(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any):
|
||||
"""流式调用;服务方不认思考控制参数或 stream_options 时去掉它重发一次,之后这个接口不再带它。
|
||||
"""流式调用;服务方不认思考控制参数或 stream_options 时去掉它重发一次,之后这个接口+模型不再带它。
|
||||
|
||||
只在还没收到任何一块时重发:拒绝都发生在请求刚发出去的时候,重发不会重复输出。"""
|
||||
usage = not self._use_responses_api({**kwargs, **self.model_kwargs}) \
|
||||
and self._should_stream_usage(kwargs.get("stream_usage"), **kwargs)
|
||||
key = _endpoint_key(self)
|
||||
hint_key = (key, str(getattr(self, "model_name", "") or ""))
|
||||
use_hint = bool(self._reasoning_hint) and hint_key not in _NO_REASONING_HINT
|
||||
endpoint = _endpoint_key(self)
|
||||
model_key = (endpoint, str(getattr(self, "model_name", "") or ""))
|
||||
use_hint = bool(self._reasoning_hint) and model_key not in _NO_REASONING_HINT
|
||||
|
||||
if use_hint:
|
||||
started = False
|
||||
hint_kwargs = _merge_reasoning_hint(kwargs, self._reasoning_hint)
|
||||
if not usage or key in _NO_STREAM_USAGE:
|
||||
if not usage or model_key in _NO_STREAM_USAGE:
|
||||
hint_kwargs = {k: v for k, v in hint_kwargs.items() if k != "stream_options"} | {"stream_usage": False}
|
||||
try:
|
||||
async for chunk in super()._astream(messages, stop, run_manager, **hint_kwargs):
|
||||
@@ -480,17 +530,18 @@ def _reasoning_aware_openai() -> type:
|
||||
except Exception as e: # noqa: BLE001
|
||||
if started:
|
||||
raise
|
||||
if _rejects_stream_options_specifically(e) and usage and key not in _NO_STREAM_USAGE:
|
||||
if _rejects_stream_options_specifically(e) and usage and model_key not in _NO_STREAM_USAGE:
|
||||
# 明确点名拒绝 stream_options:剥离 stream_options 但保留思考控制参数再试
|
||||
no_usage_hint_kwargs = {
|
||||
k: v for k, v in hint_kwargs.items() if k != "stream_options"
|
||||
} | {"stream_usage": False}
|
||||
try:
|
||||
async for chunk in super()._astream(messages, stop, run_manager, **no_usage_hint_kwargs):
|
||||
if key not in _NO_STREAM_USAGE:
|
||||
_NO_STREAM_USAGE.add(key)
|
||||
if model_key not in _NO_STREAM_USAGE:
|
||||
_NO_STREAM_USAGE.add(model_key)
|
||||
logger.warning(
|
||||
"模型接入 %s 不支持流式用量回传(%s),已改为不请求用量", key, str(e)[:200]
|
||||
"模型接入 %s/%s 不支持流式用量回传(%s),已改为不请求用量",
|
||||
model_key[0], model_key[1], str(e)[:200],
|
||||
)
|
||||
yield chunk
|
||||
return
|
||||
@@ -500,24 +551,24 @@ def _reasoning_aware_openai() -> type:
|
||||
raise
|
||||
# 剥离思考控制参数后走下方常规流式逻辑;若成则记入 _NO_REASONING_HINT
|
||||
yielded_after_strip = False
|
||||
async for chunk in self._astream_without_hint(messages, stop, run_manager, usage=usage, key=key, **kwargs):
|
||||
if not yielded_after_strip and hint_key not in _NO_REASONING_HINT:
|
||||
_NO_REASONING_HINT.add(hint_key)
|
||||
async for chunk in self._astream_without_hint(messages, stop, run_manager, usage=usage, key=model_key, **kwargs):
|
||||
if not yielded_after_strip and model_key not in _NO_REASONING_HINT:
|
||||
_NO_REASONING_HINT.add(model_key)
|
||||
logger.warning(
|
||||
"模型接入 %s/%s 不支持思考控制参数(%s),已自动剥离",
|
||||
hint_key[0],
|
||||
hint_key[1],
|
||||
model_key[0],
|
||||
model_key[1],
|
||||
str(e)[:200],
|
||||
)
|
||||
yielded_after_strip = True
|
||||
yield chunk
|
||||
return
|
||||
|
||||
async for chunk in self._astream_without_hint(messages, stop, run_manager, usage=usage, key=key, **kwargs):
|
||||
async for chunk in self._astream_without_hint(messages, stop, run_manager, usage=usage, key=model_key, **kwargs):
|
||||
yield chunk
|
||||
|
||||
async def _astream_without_hint(
|
||||
self, messages: Any, stop: Any = None, run_manager: Any = None, *, usage: bool, key: str, **kwargs: Any
|
||||
self, messages: Any, stop: Any = None, run_manager: Any = None, *, usage: bool, key: tuple[str, str], **kwargs: Any
|
||||
):
|
||||
first: Exception | None = None
|
||||
if usage and key not in _NO_STREAM_USAGE:
|
||||
@@ -536,9 +587,12 @@ def _reasoning_aware_openai() -> type:
|
||||
try:
|
||||
async for chunk in super()._astream(messages, stop, run_manager, **kwargs):
|
||||
if first is not None and key not in _NO_STREAM_USAGE:
|
||||
# 去掉它就成了:记下这个接口不认,之后不再每次先失败一次
|
||||
# 去掉它就成了:记下这个接口+模型不认,之后不再每次先失败一次
|
||||
_NO_STREAM_USAGE.add(key)
|
||||
logger.warning("模型接入 %s 不支持流式用量回传(%s),已改为不请求用量", key, str(first)[:200])
|
||||
logger.warning(
|
||||
"模型接入 %s/%s 不支持流式用量回传(%s),已改为不请求用量",
|
||||
key[0], key[1], str(first)[:200],
|
||||
)
|
||||
yield chunk
|
||||
except Exception as e:
|
||||
if first is None:
|
||||
@@ -548,8 +602,8 @@ def _reasoning_aware_openai() -> type:
|
||||
return ReasoningAwareChatOpenAI
|
||||
|
||||
|
||||
#: 认定不支持 stream_options 的接口(base_url)。进程内有效:重启后再试一次,服务方可能已经升级
|
||||
_NO_STREAM_USAGE: set[str] = set()
|
||||
#: 认定不支持 stream_options 的 (base_url, model)。按接口+模型记忆,避免统一网关下单个旧模型连坐其它模型
|
||||
_NO_STREAM_USAGE: set[tuple[str, str]] = set()
|
||||
#: 认定不支持思考控制参数(reasoning_effort / enable_thinking / thinking)的 (base_url, model)
|
||||
_NO_REASONING_HINT: set[tuple[str, str]] = set()
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ from langchain_core.tracers.context import register_configure_hook
|
||||
#: 取不到模型名的调用记在这个名下
|
||||
UNKNOWN_MODEL = "unknown"
|
||||
#: 每个模型累计的几项(unreported 用量没报全的次数、unfinished 没做完的次数)
|
||||
_COUNTERS = ("input_tokens", "output_tokens", "calls", "unreported", "unfinished")
|
||||
_COUNTERS = ("input_tokens", "output_tokens", "cache_read_tokens", "cache_creation_tokens", "calls", "unreported", "unfinished")
|
||||
|
||||
|
||||
def _abandoned(error: BaseException) -> bool:
|
||||
@@ -112,6 +112,10 @@ class Meter(BaseCallbackHandler):
|
||||
if usage:
|
||||
row["input_tokens"] += int(usage.get("input_tokens") or 0)
|
||||
row["output_tokens"] += int(usage.get("output_tokens") or 0)
|
||||
details = usage.get("input_token_details") or {}
|
||||
if isinstance(details, dict):
|
||||
row["cache_read_tokens"] += int(details.get("cache_read") or 0)
|
||||
row["cache_creation_tokens"] += int(details.get("cache_creation") or 0)
|
||||
else:
|
||||
row["unreported"] += 1
|
||||
|
||||
@@ -141,6 +145,10 @@ class Meter(BaseCallbackHandler):
|
||||
reported = getattr(partial, "usage_metadata", None) or {}
|
||||
row["input_tokens"] += int(reported.get("input_tokens") or 0)
|
||||
row["output_tokens"] += int(reported.get("output_tokens") or 0)
|
||||
details = reported.get("input_token_details") or {}
|
||||
if isinstance(details, dict):
|
||||
row["cache_read_tokens"] += int(details.get("cache_read") or 0)
|
||||
row["cache_creation_tokens"] += int(details.get("cache_creation") or 0)
|
||||
|
||||
def _rows(self) -> dict[str, dict[str, int]]:
|
||||
"""累计的一份拷贝,开始了还没结束回调的调用算成没做完的(调用方持锁)。"""
|
||||
@@ -180,9 +188,21 @@ class Meter(BaseCallbackHandler):
|
||||
unpriced = row["calls"]
|
||||
else:
|
||||
unpriced = 0 if free else row["unreported"]
|
||||
cache_read = int(row.get("cache_read_tokens") or 0)
|
||||
cache_create = int(row.get("cache_creation_tokens") or 0)
|
||||
if cache_read > 0 or cache_create > 0:
|
||||
try:
|
||||
raw_cost = estimate_cost(
|
||||
model, row["input_tokens"], row["output_tokens"],
|
||||
cache_read_tokens=cache_read, cache_creation_tokens=cache_create,
|
||||
)
|
||||
except TypeError:
|
||||
raw_cost = estimate_cost(model, row["input_tokens"], row["output_tokens"])
|
||||
else:
|
||||
raw_cost = estimate_cost(model, row["input_tokens"], row["output_tokens"])
|
||||
by_model[model] = {"input_tokens": row["input_tokens"], "output_tokens": row["output_tokens"],
|
||||
"calls": row["calls"],
|
||||
"cost_usd": round(estimate_cost(model, row["input_tokens"], row["output_tokens"]), 6),
|
||||
"cost_usd": round(raw_cost, 6),
|
||||
"unpriced_calls": unpriced, "unfinished_calls": row["unfinished"]}
|
||||
total = {key: sum(r[key] for r in by_model.values())
|
||||
for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls", "unfinished_calls")}
|
||||
|
||||
@@ -79,8 +79,8 @@ async def append_action(args: dict[str, Any], *, idem_key: str | None, run_id: s
|
||||
environment=environment)
|
||||
try:
|
||||
columns = columns_of(table)
|
||||
# 多进程时几个进程会往同一个记录表文件里追加:库里的锁(store.write_lock)一个接一个,进程里的 _WRITE_LOCK 照旧
|
||||
async with store.write_lock():
|
||||
# 多进程时几个进程会往同一个记录表文件里追加:按表键持库里的锁(store.write_lock),进程里的文件锁照旧
|
||||
async with store.write_lock(table.table_key):
|
||||
done = await asyncio.to_thread(
|
||||
store.append, path_of(table), columns, values, version=table.version, run_id=run_id,
|
||||
actor=actor, action_key=action_key_of(idem_key), supersedes=supersedes)
|
||||
|
||||
@@ -37,7 +37,7 @@ async def _masked(session: AsyncSession, row: Any) -> set[str]:
|
||||
|
||||
def _shown(item: dict[str, Any], version: int, masked: set[str], viewer: str | None) -> dict[str, Any]:
|
||||
"""一行交给看的人的样子:别人提交的行,遮罩列有值的换成「已遮罩」(P4)。"""
|
||||
from app.api.evidence import MASKED
|
||||
from app.engine.evidence_reader import MASKED
|
||||
|
||||
out = {**item, "_version": version}
|
||||
if not masked or (viewer and item.get("_submitted_by") == viewer):
|
||||
|
||||
+136
-11
@@ -46,15 +46,29 @@ NAME_MAX = 64
|
||||
_KEY = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
|
||||
#: 列名里不许出现的字符:引号(拼进 SQL 的标识符)、控制字符
|
||||
_BAD_NAME = re.compile(r"[\"`\[\]\x00-\x1f\x7f]")
|
||||
#: 所有写入(建表、加列、追加)共用的锁。写都是几毫秒的短事务,不按表分锁也不会互相拖住
|
||||
#: 所有写入(建表、加列、追加)按文件路径/表键分锁;保留 _WRITE_LOCK 供直接引用方兜底
|
||||
_WRITE_LOCK = threading.Lock()
|
||||
_LOCKS_GUARD = threading.Lock()
|
||||
_PATH_LOCKS: dict[str, threading.Lock] = {}
|
||||
|
||||
|
||||
def write_lock() -> Any:
|
||||
def _path_lock(path: Path) -> threading.Lock:
|
||||
key = os.path.realpath(os.fspath(path))
|
||||
with _LOCKS_GUARD:
|
||||
lock = _PATH_LOCKS.get(key)
|
||||
if lock is None:
|
||||
lock = threading.Lock()
|
||||
_PATH_LOCKS[key] = lock
|
||||
return lock
|
||||
|
||||
|
||||
def write_lock(table_key: str | None = None) -> Any:
|
||||
"""跨进程的写锁(app/cluster/mutex:PostgreSQL 时是咨询锁,SQLite 只有一个进程、就是本进程的锁)。异步的调用方在把写入
|
||||
放进线程之前持着它(async with)。"""
|
||||
放进线程之前持着它(async with)。给 table_key 时按表分锁,不同记录表的写入互不阻塞。"""
|
||||
from app.cluster import mutex
|
||||
|
||||
if table_key and _KEY.match(table_key):
|
||||
return mutex.lock(f"records-store:{table_key}")
|
||||
return mutex.lock("records-store")
|
||||
|
||||
|
||||
@@ -137,6 +151,19 @@ def table_path(table_key: str, version: int) -> Path:
|
||||
return records_root() / f"{table_key}.v{version}.db"
|
||||
|
||||
|
||||
def archive_path(table_key: str) -> Path:
|
||||
"""data/records/<table_key>.archive.db:超过 SQLite 挂库上限的早期版本按物理表 rows_v<version> 增量汇总在这里。"""
|
||||
if not _KEY.match(table_key or ""):
|
||||
raise ValueError("记录表的键格式不对")
|
||||
return records_root() / f"{table_key}.archive.db"
|
||||
|
||||
|
||||
def archive_table_name(version: int) -> str:
|
||||
if not isinstance(version, int) or version < 1:
|
||||
raise ValueError("记录表的版本号需要是正整数")
|
||||
return f"rows_v{version}"
|
||||
|
||||
|
||||
def _q(name: str) -> str:
|
||||
return '"' + name.replace('"', '""') + '"'
|
||||
|
||||
@@ -146,10 +173,20 @@ def _lit(value: str) -> str:
|
||||
return "'" + str(value).replace("'", "''") + "'"
|
||||
|
||||
|
||||
def _enable_wal(conn: sqlite3.Connection) -> None:
|
||||
try:
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
except sqlite3.Error:
|
||||
pass
|
||||
|
||||
|
||||
def _rw(path: Path) -> sqlite3.Connection:
|
||||
if not path.is_file():
|
||||
raise RecordsRefused("records.table_missing", "记录表的数据文件不见了,请联系管理员")
|
||||
return open_checked_sqlite(str(path), readonly=False)
|
||||
conn = open_checked_sqlite(str(path), readonly=False)
|
||||
_enable_wal(conn)
|
||||
return conn
|
||||
|
||||
|
||||
def _ro(path: Path) -> sqlite3.Connection:
|
||||
@@ -166,15 +203,18 @@ def _ro(path: Path) -> sqlite3.Connection:
|
||||
def create_file(path: Path, columns: list[Column]) -> None:
|
||||
"""新建一个表版本的文件。路径上已有文件时:没有任何行(上次建了一半、登记没提交)就重建,有行就拒绝——
|
||||
有数据的文件绝不覆盖。"""
|
||||
with _WRITE_LOCK:
|
||||
with _path_lock(path):
|
||||
if path.exists():
|
||||
if _row_count(path) > 0:
|
||||
raise RecordsRefused("records.file_exists", f"记录表文件 {path.name} 已经有数据,不能重建")
|
||||
path.unlink()
|
||||
for suffix in ("-wal", "-shm"):
|
||||
Path(f"{path}{suffix}").unlink(missing_ok=True)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_name(f".{path.name}.tmp-{new_id()[:12]}")
|
||||
conn = open_checked_sqlite(str(tmp), readonly=False)
|
||||
try:
|
||||
_enable_wal(conn)
|
||||
business = "".join(f", {_q(c.name)} {COLUMN_TYPES[c.type]}" for c in columns)
|
||||
conn.execute(
|
||||
f"CREATE TABLE {_q(TABLE)} ("
|
||||
@@ -193,7 +233,7 @@ def add_columns(path: Path, columns: list[Column]) -> None:
|
||||
"""兼容的改动(只加可选字段):在同一个版本上 ALTER TABLE ADD COLUMN。已有的列跳过(重做一次也没事)。"""
|
||||
if not columns:
|
||||
return
|
||||
with _WRITE_LOCK:
|
||||
with _path_lock(path):
|
||||
conn = _rw(path)
|
||||
try:
|
||||
have = {str(r[1]).lower() for r in conn.execute(f"PRAGMA table_info({_q(TABLE)})")}
|
||||
@@ -326,7 +366,7 @@ def append(path: Path, columns: list[Column], values: Any, *, version: int, run_
|
||||
missing = [c.label or c.name for c in columns if c.required and row[c.name] in (None, "")]
|
||||
if missing:
|
||||
raise RecordsRefused("records.required_missing", f"必填的字段没有值:{'、'.join(missing)}")
|
||||
with _WRITE_LOCK:
|
||||
with _path_lock(path):
|
||||
conn = _rw(path)
|
||||
try:
|
||||
conn.execute("BEGIN IMMEDIATE")
|
||||
@@ -411,7 +451,7 @@ def union_compatible(new: Column, old: Column) -> bool:
|
||||
|
||||
|
||||
def view_select(columns: list[Column], *, high_water: int | None, scope: str, viewer: str | None,
|
||||
schema: str = "main", present: list[Column] | None = None) -> str:
|
||||
schema: str = "main", present: list[Column] | None = None, table: str = TABLE) -> str:
|
||||
"""临时视图(和看板、记录列表)看到的那一份:
|
||||
|
||||
- _seq ≤ 高水位(None 为不限,现在有的都算);
|
||||
@@ -430,7 +470,7 @@ def view_select(columns: list[Column], *, high_water: int | None, scope: str, vi
|
||||
else f"NULL AS {_q(c.name)}"
|
||||
|
||||
cols = ", ".join([*(f"r.{_q(n)}" for n in SYSTEM_COLUMNS), *(pick(c) for c in columns)])
|
||||
src = f"{_q(schema)}.{_q(TABLE)}"
|
||||
src = f"{_q(schema)}.{_q(table)}"
|
||||
where = []
|
||||
if high_water is not None:
|
||||
seq = int(high_water)
|
||||
@@ -449,11 +489,96 @@ def view_select(columns: list[Column], *, high_water: int | None, scope: str, vi
|
||||
@dataclass(frozen=True)
|
||||
class OlderVersion:
|
||||
"""跨版本汇总时并进来的一个旧表版本:挂在查询连接上的库名(ATTACH … AS schema)、那一版的业务列、这次查询的高水位
|
||||
(None 为不限)。"""
|
||||
(None 为不限),以及该版本所在的物理表名(直接挂库时是 records,超挂库上限汇总进归档库时是 rows_v<version>)。"""
|
||||
|
||||
schema: str
|
||||
columns: list[Column]
|
||||
high_water: int | None
|
||||
table: str = TABLE
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ArchiveSpec:
|
||||
"""需要增量同步进单库多物理表归档(<table_key>.archive.db)的旧版本描述。"""
|
||||
|
||||
version: int
|
||||
path: str
|
||||
columns: list[Column]
|
||||
|
||||
|
||||
def sync_archive(table_key: str, specs: list[ArchiveSpec] | tuple[ArchiveSpec, ...]) -> Path:
|
||||
"""把超挂库上限的旧版本增量同步进单库多物理表归档文件(rows_v<version>),规避 SQLite 最多 10 个 ATTACH 库的限制。
|
||||
|
||||
行只追加、不修改:已同步到相同列结构且 _seq 无新增的版本 O(1) 跳过;退役前发起的在途运行补写了新行时,只复制
|
||||
_seq > meta_max_seq 的增量行。"""
|
||||
dest = archive_path(table_key)
|
||||
if not specs:
|
||||
return dest
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
with _path_lock(dest):
|
||||
arch = open_checked_sqlite(str(dest), readonly=False)
|
||||
try:
|
||||
_enable_wal(arch)
|
||||
arch.execute(
|
||||
"CREATE TABLE IF NOT EXISTS _archive_meta ("
|
||||
"version INTEGER PRIMARY KEY, max_seq INTEGER NOT NULL, col_sig TEXT NOT NULL)"
|
||||
)
|
||||
meta = {
|
||||
int(r[0]): (int(r[1]), str(r[2]))
|
||||
for r in arch.execute("SELECT version, max_seq, col_sig FROM _archive_meta").fetchall()
|
||||
}
|
||||
dirty = False
|
||||
for spec in specs:
|
||||
src_file = Path(spec.path)
|
||||
if not src_file.is_file():
|
||||
continue
|
||||
src = _ro(src_file)
|
||||
try:
|
||||
info = [(str(r[1]), str(r[2] or "TEXT")) for r in src.execute(f"PRAGMA table_info({_q(TABLE)})")
|
||||
if str(r[1]) != ACTION_KEY_COLUMN]
|
||||
src_max = int(src.execute(f"SELECT coalesce(max(_seq), 0) FROM {_q(TABLE)}").fetchone()[0] or 0)
|
||||
sig = json.dumps(info, ensure_ascii=False)
|
||||
tbl = archive_table_name(spec.version)
|
||||
prev = meta.get(spec.version)
|
||||
if prev is not None and prev[1] == sig and prev[0] == src_max:
|
||||
continue
|
||||
col_names = [name for name, _ in info]
|
||||
select_cols = ", ".join(_q(n) for n in col_names)
|
||||
placeholders = ", ".join("?" for _ in col_names)
|
||||
if prev is None or prev[1] != sig:
|
||||
arch.execute(f"DROP TABLE IF EXISTS {_q(tbl)}")
|
||||
ddl_cols = ", ".join(
|
||||
"_seq INTEGER PRIMARY KEY" if name == "_seq" else f"{_q(name)} {col_type}"
|
||||
for name, col_type in info
|
||||
)
|
||||
arch.execute(f"CREATE TABLE {_q(tbl)} ({ddl_cols})")
|
||||
arch.execute(f"CREATE INDEX ix_{tbl}_supersedes ON {_q(tbl)} (_supersedes)")
|
||||
arch.execute(f"CREATE INDEX ix_{tbl}_submitted_by ON {_q(tbl)} (_submitted_by)")
|
||||
since_seq = 0
|
||||
else:
|
||||
since_seq = prev[0]
|
||||
cur = src.execute(
|
||||
f"SELECT {select_cols} FROM {_q(TABLE)} WHERE _seq > ? ORDER BY _seq ASC",
|
||||
(since_seq,),
|
||||
)
|
||||
insert_sql = f"INSERT OR REPLACE INTO {_q(tbl)} ({select_cols}) VALUES ({placeholders})"
|
||||
while batch := cur.fetchmany(500):
|
||||
arch.executemany(insert_sql, batch)
|
||||
arch.execute(
|
||||
"INSERT OR REPLACE INTO _archive_meta (version, max_seq, col_sig) VALUES (?, ?, ?)",
|
||||
(spec.version, src_max, sig),
|
||||
)
|
||||
dirty = True
|
||||
finally:
|
||||
src.close()
|
||||
if dirty:
|
||||
arch.commit()
|
||||
except BaseException:
|
||||
arch.rollback()
|
||||
raise
|
||||
finally:
|
||||
arch.close()
|
||||
return dest
|
||||
|
||||
|
||||
def view_prelude(columns: list[Column], *, high_water: int | None, scope: str, viewer: str | None,
|
||||
@@ -465,7 +590,7 @@ def view_prelude(columns: list[Column], *, high_water: int | None, scope: str, v
|
||||
列照这一版的来,旧版本没有的列补 NULL;每一版各按自己的高水位、自己文件里的更正判,行级过滤同一个。"""
|
||||
parts = [view_select(columns, high_water=high_water, scope=scope, viewer=viewer)]
|
||||
parts += [view_select(columns, high_water=o.high_water, scope=scope, viewer=viewer, schema=o.schema,
|
||||
present=o.columns) for o in older]
|
||||
present=o.columns, table=o.table) for o in older]
|
||||
return (f"DROP VIEW IF EXISTS temp.{_q(TABLE)}",
|
||||
f"CREATE TEMP VIEW {_q(TABLE)} AS " + " UNION ALL ".join(parts))
|
||||
|
||||
|
||||
@@ -188,7 +188,7 @@ async def _free_name(session: AsyncSession, base: str) -> str:
|
||||
async def _register(session: AsyncSession, *, key: str, version: int, columns: list[Column], label: str,
|
||||
app_id: str | None, blueprint_id: str | None) -> RecordsTable:
|
||||
path = store.table_path(key, version)
|
||||
async with store.write_lock():
|
||||
async with store.write_lock(key):
|
||||
await asyncio.to_thread(store.create_file, path, columns)
|
||||
base = f"{SOURCE_PREFIX}{re.sub(r'[^a-z0-9]', '', key.lower())[:12] or 'table'}"
|
||||
name = await _free_name(session, base if version == 1 else f"{base}_v{version}")
|
||||
@@ -373,7 +373,7 @@ async def _sync_registration(session: AsyncSession, row: RecordsTable, wanted: l
|
||||
have = columns_of(row)
|
||||
known = {h.name.lower() for h in have}
|
||||
added = [c for c in wanted if c.name.lower() not in known]
|
||||
async with store.write_lock():
|
||||
async with store.write_lock(row.table_key):
|
||||
await asyncio.to_thread(store.add_columns, store.table_path(row.table_key, row.version), added)
|
||||
by_name = {w.name.lower(): w for w in wanted}
|
||||
# 列的顺序:原有的在前(照原来的顺序),新加的在后——和文件里的列序一致。删掉的列(只在不兼容时出现)这里不会有
|
||||
|
||||
@@ -22,6 +22,7 @@ import copy
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
@@ -35,8 +36,10 @@ logger = logging.getLogger(__name__)
|
||||
#: {"version", "seq": None, "missing": True}——汇总里没有它,印章(apps/resolve.stamp_of)、查询快照照实说
|
||||
MISSING = "missing"
|
||||
PIN_KEYS = ("name", "origin", "table", "version", "seq", "older", "rows", "viewer")
|
||||
#: 旧版本挂在查询连接上的库名:records_v<版本>
|
||||
#: 旧版本挂在查询连接上的库名:records_v<版本>;超过 SQLite 10 库挂接上限时,早期版本汇总进单库多表 records_archive
|
||||
ATTACH_PREFIX = "records_v"
|
||||
ARCHIVE_SCHEMA = "records_archive"
|
||||
MAX_DIRECT_ATTACH = 8
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -89,27 +92,70 @@ class Older:
|
||||
path: str
|
||||
columns: list[store.Column]
|
||||
high_water: int | None
|
||||
schema_name: str | None = None
|
||||
table_name: str = store.TABLE
|
||||
|
||||
@property
|
||||
def schema(self) -> str:
|
||||
return f"{ATTACH_PREFIX}{int(self.version)}"
|
||||
return self.schema_name or f"{ATTACH_PREFIX}{int(self.version)}"
|
||||
|
||||
|
||||
def _infer_table_key(older: tuple[Older, ...] | list[Older]) -> str | None:
|
||||
for item in older:
|
||||
stem = Path(item.path).name
|
||||
if stem.endswith(".db") and ".v" in stem[:-3]:
|
||||
key, _, ver = stem[:-3].rpartition(".v")
|
||||
if key and ver.isdigit():
|
||||
return key
|
||||
return None
|
||||
|
||||
|
||||
def _materialize_older(older: tuple[Older, ...] | list[Older], *, table_key: str | None = None) -> tuple[Older, ...]:
|
||||
items = tuple(older)
|
||||
if len(items) <= MAX_DIRECT_ATTACH or any(o.schema_name is not None for o in items):
|
||||
return items
|
||||
key = table_key or _infer_table_key(items)
|
||||
if not key:
|
||||
return items
|
||||
direct = items[:MAX_DIRECT_ATTACH - 1]
|
||||
overflow = items[MAX_DIRECT_ATTACH - 1:]
|
||||
present = [o for o in overflow if Path(o.path).is_file()]
|
||||
if not present:
|
||||
return items
|
||||
arch = store.sync_archive(
|
||||
key,
|
||||
[store.ArchiveSpec(version=o.version, path=o.path, columns=o.columns) for o in present],
|
||||
)
|
||||
arch_str = str(arch)
|
||||
present_versions = {o.version for o in present}
|
||||
mapped = tuple(
|
||||
Older(o.version, arch_str, o.columns, o.high_water, schema_name=ARCHIVE_SCHEMA,
|
||||
table_name=store.archive_table_name(o.version))
|
||||
if o.version in present_versions else o
|
||||
for o in overflow
|
||||
)
|
||||
return (*direct, *mapped)
|
||||
|
||||
|
||||
def bind_view(source: Any, columns: list[store.Column], *, high_water: int | None, rows: str = "all",
|
||||
viewer: str | None = None, older: list[Older] | tuple[Older, ...] = (),
|
||||
missing: list[int] | tuple[int, ...] = ()) -> RecordsView:
|
||||
missing: list[int] | tuple[int, ...] = (), table_key: str | None = None) -> RecordsView:
|
||||
"""数据源行 + 高水位 + 行级过滤(+ 并进来的旧版本;missing 是数据文件不在了、并不进来的旧版本号)→ RecordsView。"""
|
||||
rows = rows if rows in ("all", "mine", "none") else "none"
|
||||
older = tuple(older)
|
||||
older = _materialize_older(older, table_key=table_key)
|
||||
attach_map: dict[str, str] = {}
|
||||
for o in older:
|
||||
attach_map.setdefault(o.schema, o.path)
|
||||
return RecordsView(
|
||||
id=source.id, name=source.name, database=source.database, options=dict(source.options or {}),
|
||||
description=source.description or "", schema_cache=copy.deepcopy(source.schema_cache or {}),
|
||||
schema_synced_at=getattr(source, "schema_synced_at", None), enabled=bool(source.enabled),
|
||||
last_check=getattr(source, "last_check", None), high_water=high_water, rows=rows, viewer=viewer,
|
||||
prelude=store.view_prelude(columns, high_water=high_water, scope=rows, viewer=viewer,
|
||||
older=[store.OlderVersion(o.schema, o.columns, o.high_water) for o in older]),
|
||||
attach=tuple((o.schema, o.path) for o in older),
|
||||
forbid_schemas=("main", *(o.schema for o in older)),
|
||||
older=[store.OlderVersion(o.schema, o.columns, o.high_water, o.table_name)
|
||||
for o in older]),
|
||||
attach=tuple(attach_map.items()),
|
||||
forbid_schemas=("main", *attach_map.keys()),
|
||||
older=(*({"version": o.version, "seq": o.high_water} for o in older),
|
||||
*({"version": int(v), "seq": None, MISSING: True} for v in missing)))
|
||||
|
||||
@@ -145,6 +191,7 @@ async def resolve(session: AsyncSession, source: Any, entry: Any = None) -> Reco
|
||||
from app.records.tables import columns_of, path_of, table_of_source
|
||||
|
||||
table = await table_of_source(session, source.id)
|
||||
table_key = table.table_key if table is not None else None
|
||||
columns = columns_of(table) if table is not None else columns_of_source(source)
|
||||
older_rows = await _older_versions(session, table)
|
||||
if isinstance(entry, dict):
|
||||
@@ -156,16 +203,21 @@ async def resolve(session: AsyncSession, source: Any, entry: Any = None) -> Reco
|
||||
older = [Older(row.version, str(path_of(row)), columns_of(row),
|
||||
int(pinned[row.version]) if isinstance(pinned[row.version], int) else 0)
|
||||
for row in older_rows if row.version in pinned]
|
||||
if len(older) > MAX_DIRECT_ATTACH:
|
||||
older = list(await asyncio.to_thread(_materialize_older, older, table_key=table_key))
|
||||
return bind_view(source, columns, high_water=int(seq) if isinstance(seq, int) else 0,
|
||||
rows=str(entry.get("rows") or "none"), viewer=entry.get("viewer"), older=older,
|
||||
missing=missing)
|
||||
missing=missing, table_key=table_key)
|
||||
older = [Older(row.version, str(path_of(row)), columns_of(row), None) for row in older_rows
|
||||
if path_of(row).is_file()]
|
||||
if len(older) > MAX_DIRECT_ATTACH:
|
||||
older = list(await asyncio.to_thread(_materialize_older, older, table_key=table_key))
|
||||
missing = [row.version for row in older_rows if not path_of(row).is_file()]
|
||||
if missing:
|
||||
logger.warning("记录表 %s 第 %s 版的数据文件不在了,跨版本汇总里不含这几版", table.table_key,
|
||||
"、".join(str(v) for v in missing))
|
||||
return bind_view(source, columns, high_water=None, rows="all", older=older, missing=missing)
|
||||
return bind_view(source, columns, high_water=None, rows="all", older=older, missing=missing,
|
||||
table_key=table_key)
|
||||
|
||||
|
||||
def default_view(source: Any) -> RecordsView:
|
||||
|
||||
@@ -184,58 +184,82 @@ class BubblewrapSandbox(Sandbox):
|
||||
cmd_prefix = cmd_prefix if cmd_prefix is not None else interpreter.python_argv()
|
||||
entry = entry_name(ext)
|
||||
cmd = [*cmd_prefix, f"/work/{entry}"]
|
||||
workdir = self._session_dir(session_id or f"tmp-{int(time.time() * 1000)}")
|
||||
atomic.write_text(workdir / entry, code)
|
||||
for rel, content in (files or {}).items():
|
||||
target = (workdir / rel).resolve()
|
||||
if not str(target).startswith(str(workdir.resolve())):
|
||||
return ExecResult(ok=False, backend=self.name, error=f"非法文件路径:{rel}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic.write_text(target, content)
|
||||
|
||||
argv = self._argv(workdir, limits, cmd, env)
|
||||
|
||||
started = time.perf_counter()
|
||||
ephemeral = not session_id
|
||||
workdir = self._session_dir(session_id or f"tmp-{int(time.time() * 1000)}-{os.urandom(4).hex()}")
|
||||
work_root = workdir.resolve()
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*argv,
|
||||
# bwrap 自己也只拿最小的环境:沙箱里的进程靠上面的 --setenv,不从这里继承
|
||||
env={"PATH": os.environ.get("PATH", "/usr/bin:/bin")},
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
preexec_fn=self._preexec(limits),
|
||||
)
|
||||
out, err = await asyncio.wait_for(proc.communicate(), timeout=limits.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), 9)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
return ExecResult(
|
||||
ok=False, backend=self.name, timed_out=True, exit_code=124,
|
||||
duration_ms=int((time.perf_counter() - started) * 1000),
|
||||
error=f"执行超过 {limits.timeout} 秒,已终止",
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
# 起进程和收输出在同一个 try 里。首行只说原因,类名和 errno 这些留给日志
|
||||
logger.warning("%s 沙箱进程出错:%s", self.name, raw(e))
|
||||
return ExecResult(ok=False, backend=self.name, error=f"沙箱进程出错:{describe_exception(e)}")
|
||||
atomic.write_text(workdir / entry, code)
|
||||
for rel, content in (files or {}).items():
|
||||
target = (workdir / rel).resolve()
|
||||
if not target.is_relative_to(work_root):
|
||||
return ExecResult(ok=False, backend=self.name, error=f"非法文件路径:{rel}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic.write_text(target, content)
|
||||
|
||||
stdout, t1 = truncate(out.decode(errors="replace"), limits.max_output)
|
||||
stderr, t2 = truncate(err.decode(errors="replace"), limits.max_output)
|
||||
code_ = proc.returncode or 0
|
||||
return ExecResult(
|
||||
ok=code_ == 0,
|
||||
exit_code=code_,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
truncated=t1 or t2,
|
||||
timed_out=code_ in (-24, 152),
|
||||
backend=self.name,
|
||||
duration_ms=int((time.perf_counter() - started) * 1000),
|
||||
files_written=[p.name for p in workdir.iterdir() if p.is_file() and not p.name.startswith(ENTRY_PREFIX)][:50],
|
||||
)
|
||||
argv = self._argv(workdir, limits, cmd, env)
|
||||
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*argv,
|
||||
# bwrap 自己也只拿最小的环境:沙箱里的进程靠上面的 --setenv,不从这里继承
|
||||
env={"PATH": os.environ.get("PATH", "/usr/bin:/bin")},
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
preexec_fn=self._preexec(limits),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning("%s 沙箱进程出错:%s", self.name, raw(e))
|
||||
return ExecResult(ok=False, backend=self.name, error=f"沙箱进程出错:{describe_exception(e)}")
|
||||
|
||||
try:
|
||||
out, err = await asyncio.wait_for(proc.communicate(), timeout=limits.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
self._kill(proc)
|
||||
await proc.wait()
|
||||
return ExecResult(
|
||||
ok=False, backend=self.name, timed_out=True, exit_code=124,
|
||||
duration_ms=int((time.perf_counter() - started) * 1000),
|
||||
error=f"执行超过 {limits.timeout} 秒,已终止",
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
if getattr(proc, "returncode", 0) is None:
|
||||
self._kill(proc)
|
||||
await proc.wait()
|
||||
logger.warning("%s 沙箱进程出错:%s", self.name, raw(e))
|
||||
return ExecResult(ok=False, backend=self.name, error=f"沙箱进程出错:{describe_exception(e)}")
|
||||
except BaseException:
|
||||
if getattr(proc, "returncode", 0) is None:
|
||||
self._kill(proc)
|
||||
await proc.wait()
|
||||
raise
|
||||
|
||||
stdout, t1 = truncate(out.decode(errors="replace"), limits.max_output)
|
||||
stderr, t2 = truncate(err.decode(errors="replace"), limits.max_output)
|
||||
code_ = proc.returncode or 0
|
||||
return ExecResult(
|
||||
ok=code_ == 0,
|
||||
exit_code=code_,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
truncated=t1 or t2,
|
||||
timed_out=code_ in (-24, 152),
|
||||
backend=self.name,
|
||||
duration_ms=int((time.perf_counter() - started) * 1000),
|
||||
files_written=[p.name for p in workdir.iterdir() if p.is_file() and not p.name.startswith(ENTRY_PREFIX)][:50],
|
||||
)
|
||||
finally:
|
||||
if ephemeral:
|
||||
root_resolved = self._root.resolve()
|
||||
if work_root != root_resolved and work_root.is_relative_to(root_resolved):
|
||||
shutil.rmtree(workdir, ignore_errors=True)
|
||||
|
||||
@staticmethod
|
||||
def _kill(proc: asyncio.subprocess.Process) -> None:
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), 9)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
proc.kill()
|
||||
|
||||
async def cleanup(self, session_id: str) -> None:
|
||||
await asyncio.to_thread(shutil.rmtree, self._session_dir(session_id), True)
|
||||
|
||||
@@ -107,73 +107,90 @@ class LocalSandbox(Sandbox):
|
||||
ok=False, backend=self.name, error=f"本机找不到 {argv[0]}"
|
||||
)
|
||||
|
||||
workdir = self._session_dir(session_id or f"tmp-{int(time.time()*1000)}")
|
||||
atomic.write_text(workdir / entry, code)
|
||||
for rel, content in (files or {}).items():
|
||||
target = (workdir / rel).resolve()
|
||||
if not str(target).startswith(str(workdir.resolve())):
|
||||
return ExecResult(ok=False, backend=self.name, error=f"非法文件路径:{rel}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic.write_text(target, content)
|
||||
|
||||
# 只透传最小环境变量,避免把宿主机的 API key 泄漏给被执行的代码
|
||||
child_env = {
|
||||
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
||||
"HOME": str(workdir),
|
||||
"LANG": "en_US.UTF-8",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
**(env or {}),
|
||||
}
|
||||
|
||||
started = time.perf_counter()
|
||||
ephemeral = not session_id
|
||||
workdir = self._session_dir(session_id or f"tmp-{int(time.time() * 1000)}-{os.urandom(4).hex()}")
|
||||
work_root = workdir.resolve()
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*argv,
|
||||
cwd=str(workdir),
|
||||
env=child_env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
preexec_fn=self._preexec(limits),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
# 首行只说原因,类名和 errno 这些留给日志
|
||||
logger.warning("%s 沙箱起进程失败:%s", self.name, raw(e))
|
||||
return ExecResult(ok=False, backend=self.name, error=f"无法启动进程:{describe_exception(e)}")
|
||||
atomic.write_text(workdir / entry, code)
|
||||
for rel, content in (files or {}).items():
|
||||
target = (workdir / rel).resolve()
|
||||
if not target.is_relative_to(work_root):
|
||||
return ExecResult(ok=False, backend=self.name, error=f"非法文件路径:{rel}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic.write_text(target, content)
|
||||
|
||||
try:
|
||||
out, err = await asyncio.wait_for(proc.communicate(), timeout=limits.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
# 只透传最小环境变量,避免把宿主机的 API key 泄漏给被执行的代码
|
||||
child_env = {
|
||||
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
||||
"HOME": str(workdir),
|
||||
"LANG": "en_US.UTF-8",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
**(env or {}),
|
||||
}
|
||||
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), 9)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
return ExecResult(
|
||||
ok=False,
|
||||
backend=self.name,
|
||||
timed_out=True,
|
||||
exit_code=124,
|
||||
duration_ms=int((time.perf_counter() - started) * 1000),
|
||||
error=f"执行超过 {limits.timeout} 秒,已终止",
|
||||
)
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*argv,
|
||||
cwd=str(workdir),
|
||||
env=child_env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
preexec_fn=self._preexec(limits),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
# 首行只说原因,类名和 errno 这些留给日志
|
||||
logger.warning("%s 沙箱起进程失败:%s", self.name, raw(e))
|
||||
return ExecResult(ok=False, backend=self.name, error=f"无法启动进程:{describe_exception(e)}")
|
||||
|
||||
stdout, t1 = truncate(out.decode(errors="replace"), limits.max_output)
|
||||
stderr, t2 = truncate(err.decode(errors="replace"), limits.max_output)
|
||||
written = [
|
||||
p.name
|
||||
for p in workdir.iterdir()
|
||||
if p.is_file() and not p.name.startswith(ENTRY_PREFIX)
|
||||
][:50]
|
||||
return ExecResult(
|
||||
ok=proc.returncode == 0,
|
||||
exit_code=proc.returncode or 0,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
truncated=t1 or t2,
|
||||
backend=self.name,
|
||||
duration_ms=int((time.perf_counter() - started) * 1000),
|
||||
files_written=written,
|
||||
)
|
||||
try:
|
||||
out, err = await asyncio.wait_for(proc.communicate(), timeout=limits.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
self._kill(proc)
|
||||
await proc.wait()
|
||||
return ExecResult(
|
||||
ok=False,
|
||||
backend=self.name,
|
||||
timed_out=True,
|
||||
exit_code=124,
|
||||
duration_ms=int((time.perf_counter() - started) * 1000),
|
||||
error=f"执行超过 {limits.timeout} 秒,已终止",
|
||||
)
|
||||
except BaseException:
|
||||
if proc.returncode is None:
|
||||
self._kill(proc)
|
||||
await proc.wait()
|
||||
raise
|
||||
|
||||
stdout, t1 = truncate(out.decode(errors="replace"), limits.max_output)
|
||||
stderr, t2 = truncate(err.decode(errors="replace"), limits.max_output)
|
||||
written = [
|
||||
p.name
|
||||
for p in workdir.iterdir()
|
||||
if p.is_file() and not p.name.startswith(ENTRY_PREFIX)
|
||||
][:50]
|
||||
return ExecResult(
|
||||
ok=proc.returncode == 0,
|
||||
exit_code=proc.returncode or 0,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
truncated=t1 or t2,
|
||||
backend=self.name,
|
||||
duration_ms=int((time.perf_counter() - started) * 1000),
|
||||
files_written=written,
|
||||
)
|
||||
finally:
|
||||
if ephemeral:
|
||||
root_resolved = self._root.resolve()
|
||||
if work_root != root_resolved and work_root.is_relative_to(root_resolved):
|
||||
shutil.rmtree(workdir, ignore_errors=True)
|
||||
|
||||
@staticmethod
|
||||
def _kill(proc: asyncio.subprocess.Process) -> None:
|
||||
try:
|
||||
os.killpg(os.getpgid(proc.pid), 9)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
proc.kill()
|
||||
|
||||
async def cleanup(self, session_id: str) -> None:
|
||||
path = self._session_dir(session_id)
|
||||
|
||||
@@ -78,6 +78,7 @@ def build_policy(workdir: Path, *, language: str, network: bool) -> str:
|
||||
"""
|
||||
work = _real(workdir)
|
||||
home = _real(Path.home())
|
||||
data = _real(settings.data_dir)
|
||||
lines = [
|
||||
"(version 1)",
|
||||
"(allow default)",
|
||||
@@ -98,8 +99,9 @@ def build_policy(workdir: Path, *, language: str, network: bool) -> str:
|
||||
' (literal "/dev/stderr") (literal "/dev/dtracehelper"))',
|
||||
'(allow file-write* (subpath "/private/var/folders"))', # 临时目录,Python 要用
|
||||
"",
|
||||
";; ---- 读:家目录挡住,放行工作区和解释器 ----",
|
||||
";; ---- 读:家目录与平台数据目录挡住,放行工作区和解释器 ----",
|
||||
f'(deny file-read* (subpath "{home}"))',
|
||||
*([f'(deny file-read* (subpath "{data}"))'] if data != home else []),
|
||||
f'(allow file-read* (subpath "{work}"))',
|
||||
]
|
||||
for root in _interpreter_roots(language):
|
||||
@@ -194,83 +196,99 @@ class SeatbeltSandbox(Sandbox):
|
||||
|
||||
ext, cmd_of = _LANG[language]
|
||||
entry = entry_name(ext)
|
||||
workdir = self._session_dir(session_id or f"tmp-{int(time.time() * 1000)}")
|
||||
atomic.write_text(workdir / entry, code)
|
||||
|
||||
for rel, content in (files or {}).items():
|
||||
target = (workdir / rel).resolve()
|
||||
if not str(target).startswith(str(workdir.resolve())):
|
||||
return ExecResult(ok=False, backend=self.name, error=f"非法文件路径:{rel}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic.write_text(target, content)
|
||||
|
||||
self._policies.mkdir(parents=True, exist_ok=True)
|
||||
policy_file = self._policies / f"{workdir.name}-{language}.sb"
|
||||
atomic.write_text(policy_file, build_policy(workdir, language=language, network=limits.network))
|
||||
|
||||
argv = [SANDBOX_EXEC, "-f", str(policy_file), *cmd_of(), entry]
|
||||
|
||||
# HOME 指向工作区:既挡住家目录,又让解释器的缓存有地方落
|
||||
child_env = {
|
||||
"PATH": "/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin",
|
||||
"HOME": str(workdir),
|
||||
"TMPDIR": str(workdir),
|
||||
"LANG": "en_US.UTF-8",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
**(env or {}),
|
||||
}
|
||||
|
||||
started = time.perf_counter()
|
||||
ephemeral = not session_id
|
||||
workdir = self._session_dir(session_id or f"tmp-{int(time.time() * 1000)}-{os.urandom(4).hex()}")
|
||||
work_root = workdir.resolve()
|
||||
policy_file: Path | None = None
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*argv,
|
||||
cwd=str(workdir),
|
||||
env=child_env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
preexec_fn=self._preexec(limits),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
# 首行只说原因,类名和 errno 这些留给日志
|
||||
logger.warning("%s 沙箱起进程失败:%s", self.name, raw(e))
|
||||
return ExecResult(ok=False, backend=self.name, error=f"没能启动进程:{describe_exception(e)}")
|
||||
atomic.write_text(workdir / entry, code)
|
||||
|
||||
for rel, content in (files or {}).items():
|
||||
target = (workdir / rel).resolve()
|
||||
if not target.is_relative_to(work_root):
|
||||
return ExecResult(ok=False, backend=self.name, error=f"非法文件路径:{rel}")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
atomic.write_text(target, content)
|
||||
|
||||
self._policies.mkdir(parents=True, exist_ok=True)
|
||||
policy_file = self._policies / f"{workdir.name}-{language}.sb"
|
||||
atomic.write_text(policy_file, build_policy(workdir, language=language, network=limits.network))
|
||||
|
||||
argv = [SANDBOX_EXEC, "-f", str(policy_file), *cmd_of(), entry]
|
||||
|
||||
# HOME 指向工作区:既挡住家目录,又让解释器的缓存有地方落
|
||||
child_env = {
|
||||
"PATH": "/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin",
|
||||
"HOME": str(workdir),
|
||||
"TMPDIR": str(workdir),
|
||||
"LANG": "en_US.UTF-8",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
**(env or {}),
|
||||
}
|
||||
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*argv,
|
||||
cwd=str(workdir),
|
||||
env=child_env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
preexec_fn=self._preexec(limits),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
# 首行只说原因,类名和 errno 这些留给日志
|
||||
logger.warning("%s 沙箱起进程失败:%s", self.name, raw(e))
|
||||
return ExecResult(ok=False, backend=self.name, error=f"没能启动进程:{describe_exception(e)}")
|
||||
|
||||
try:
|
||||
out, err = await asyncio.wait_for(proc.communicate(), timeout=limits.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
self._kill(proc)
|
||||
await proc.wait()
|
||||
return ExecResult(
|
||||
ok=False,
|
||||
backend=self.name,
|
||||
timed_out=True,
|
||||
exit_code=124,
|
||||
duration_ms=int((time.perf_counter() - started) * 1000),
|
||||
error=f"执行超过 {limits.timeout} 秒,已终止",
|
||||
)
|
||||
except BaseException:
|
||||
if proc.returncode is None:
|
||||
self._kill(proc)
|
||||
await proc.wait()
|
||||
raise
|
||||
|
||||
stdout, t1 = truncate(out.decode(errors="replace"), limits.max_output)
|
||||
stderr, t2 = truncate(err.decode(errors="replace"), limits.max_output)
|
||||
code_ = proc.returncode or 0
|
||||
# CPU 时间用尽是 SIGXCPU(24),归到超时里更好理解
|
||||
timed_out = code_ in (-24, 152)
|
||||
written = [
|
||||
p.name for p in workdir.iterdir()
|
||||
if p.is_file() and not p.name.startswith(ENTRY_PREFIX)
|
||||
][:50]
|
||||
|
||||
try:
|
||||
out, err = await asyncio.wait_for(proc.communicate(), timeout=limits.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
self._kill(proc)
|
||||
await proc.wait()
|
||||
return ExecResult(
|
||||
ok=False,
|
||||
ok=code_ == 0,
|
||||
exit_code=code_,
|
||||
stdout=stdout,
|
||||
stderr=stderr if not timed_out else (stderr or "CPU 时间超限,已终止"),
|
||||
truncated=t1 or t2,
|
||||
timed_out=timed_out,
|
||||
backend=self.name,
|
||||
timed_out=True,
|
||||
exit_code=124,
|
||||
duration_ms=int((time.perf_counter() - started) * 1000),
|
||||
error=f"执行超过 {limits.timeout} 秒,已终止",
|
||||
files_written=written,
|
||||
)
|
||||
|
||||
stdout, t1 = truncate(out.decode(errors="replace"), limits.max_output)
|
||||
stderr, t2 = truncate(err.decode(errors="replace"), limits.max_output)
|
||||
code_ = proc.returncode or 0
|
||||
# CPU 时间用尽是 SIGXCPU(24),归到超时里更好理解
|
||||
timed_out = code_ in (-24, 152)
|
||||
written = [
|
||||
p.name for p in workdir.iterdir()
|
||||
if p.is_file() and not p.name.startswith(ENTRY_PREFIX)
|
||||
][:50]
|
||||
|
||||
return ExecResult(
|
||||
ok=code_ == 0,
|
||||
exit_code=code_,
|
||||
stdout=stdout,
|
||||
stderr=stderr if not timed_out else (stderr or "CPU 时间超限,已终止"),
|
||||
truncated=t1 or t2,
|
||||
timed_out=timed_out,
|
||||
backend=self.name,
|
||||
duration_ms=int((time.perf_counter() - started) * 1000),
|
||||
files_written=written,
|
||||
)
|
||||
finally:
|
||||
if ephemeral:
|
||||
if policy_file is not None:
|
||||
policy_file.unlink(missing_ok=True)
|
||||
root_resolved = self._root.resolve()
|
||||
if work_root != root_resolved and work_root.is_relative_to(root_resolved):
|
||||
shutil.rmtree(workdir, ignore_errors=True)
|
||||
|
||||
@staticmethod
|
||||
def _kill(proc: asyncio.subprocess.Process) -> None:
|
||||
|
||||
+70
-33
@@ -21,10 +21,19 @@ _BLOCKED_PORTS = {22, 23, 25, 135, 139, 445, 3389}
|
||||
_METADATA_HOSTS = frozenset({"100.100.100.200", "fd00:ec2::254", "metadata.google.internal"})
|
||||
|
||||
|
||||
def _is_private(ip: str) -> bool:
|
||||
def _unwrap_ip(ip: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
except ValueError:
|
||||
return None
|
||||
if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None:
|
||||
return addr.ipv4_mapped
|
||||
return addr
|
||||
|
||||
|
||||
def _is_private(ip: str) -> bool:
|
||||
addr = _unwrap_ip(ip)
|
||||
if addr is None:
|
||||
return True
|
||||
return (
|
||||
addr.is_private
|
||||
@@ -48,9 +57,8 @@ def _fake_networks() -> list[ipaddress.IPv4Network | ipaddress.IPv6Network]:
|
||||
|
||||
|
||||
def _in(ip: str, networks) -> bool:
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
except ValueError:
|
||||
addr = _unwrap_ip(ip)
|
||||
if addr is None:
|
||||
return False
|
||||
return any(addr.version == n.version and addr in n for n in networks)
|
||||
|
||||
@@ -101,10 +109,11 @@ async def _real_ips(host: str) -> tuple[str, ...]:
|
||||
|
||||
def _is_metadata(ip: str) -> bool:
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip)
|
||||
raw_addr = ipaddress.ip_address(ip)
|
||||
except ValueError:
|
||||
return True
|
||||
return addr.is_link_local or str(addr) in _METADATA_HOSTS
|
||||
addr = raw_addr.ipv4_mapped if isinstance(raw_addr, ipaddress.IPv6Address) and raw_addr.ipv4_mapped is not None else raw_addr
|
||||
return raw_addr.is_link_local or addr.is_link_local or str(raw_addr) in _METADATA_HOSTS or str(addr) in _METADATA_HOSTS
|
||||
|
||||
|
||||
async def assert_safe_url(url: str, *, allow_intranet: bool = False) -> str:
|
||||
@@ -199,35 +208,63 @@ async def safe_request(
|
||||
max_bytes = max_bytes or settings.http_tool_max_bytes
|
||||
current = await assert_safe_url(url)
|
||||
|
||||
_SENSITIVE_HEADERS = frozenset({"authorization", "cookie", "proxy-authorization", "x-api-key"})
|
||||
cur_method = method.upper()
|
||||
cur_headers = dict(headers or {})
|
||||
cur_params: dict[str, str] | None = params
|
||||
cur_json: object | None = json_body
|
||||
cur_data: str | None = data
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
timeout=timeout, follow_redirects=False, trust_env=False
|
||||
) as client:
|
||||
for _ in range(5):
|
||||
resp = await client.request(
|
||||
method.upper(),
|
||||
for hop in range(6):
|
||||
async with client.stream(
|
||||
cur_method,
|
||||
current,
|
||||
headers=headers or {},
|
||||
params=params,
|
||||
json=json_body,
|
||||
content=data,
|
||||
)
|
||||
if resp.status_code in (301, 302, 303, 307, 308) and "location" in resp.headers:
|
||||
current = await assert_safe_url(
|
||||
str(httpx.URL(current).join(resp.headers["location"]))
|
||||
)
|
||||
continue
|
||||
break
|
||||
headers=cur_headers,
|
||||
params=cur_params,
|
||||
json=cur_json,
|
||||
content=cur_data,
|
||||
) as resp:
|
||||
if hop < 5 and resp.status_code in (301, 302, 303, 307, 308) and "location" in resp.headers:
|
||||
prev_url = httpx.URL(current)
|
||||
next_url = prev_url.join(resp.headers["location"])
|
||||
current = await assert_safe_url(str(next_url))
|
||||
cur_params = None
|
||||
if (next_url.scheme, next_url.host, next_url.port) != (prev_url.scheme, prev_url.host, prev_url.port):
|
||||
cur_headers = {k: v for k, v in cur_headers.items() if k.lower() not in _SENSITIVE_HEADERS}
|
||||
if resp.status_code == 303 or (resp.status_code in (301, 302) and cur_method == "POST"):
|
||||
cur_method = "GET"
|
||||
cur_json = None
|
||||
cur_data = None
|
||||
continue
|
||||
|
||||
body = resp.content[:max_bytes]
|
||||
truncated = len(resp.content) > max_bytes
|
||||
try:
|
||||
text = body.decode(resp.encoding or "utf-8", errors="replace")
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
text = body.decode("utf-8", errors="replace")
|
||||
return {
|
||||
"status": resp.status_code,
|
||||
"url": str(resp.url),
|
||||
"headers": dict(resp.headers),
|
||||
"body": text,
|
||||
"truncated": truncated,
|
||||
}
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
truncated = False
|
||||
async for chunk in resp.aiter_bytes():
|
||||
if not chunk:
|
||||
continue
|
||||
remaining = max_bytes - total
|
||||
if len(chunk) > remaining:
|
||||
if remaining > 0:
|
||||
chunks.append(chunk[:remaining])
|
||||
truncated = True
|
||||
break
|
||||
chunks.append(chunk)
|
||||
total += len(chunk)
|
||||
|
||||
body = b"".join(chunks)
|
||||
try:
|
||||
text = body.decode(resp.encoding or "utf-8", errors="replace")
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
text = body.decode("utf-8", errors="replace")
|
||||
return {
|
||||
"status": resp.status_code,
|
||||
"url": str(resp.url),
|
||||
"headers": dict(resp.headers),
|
||||
"body": text,
|
||||
"truncated": truncated,
|
||||
}
|
||||
raise UnsafeUrlError("重定向次数超过上限")
|
||||
|
||||
@@ -19,6 +19,17 @@ from pathlib import Path
|
||||
|
||||
_TMP = Path(tempfile.mkdtemp(prefix="agentlab-tests-"))
|
||||
os.environ["AGENTLAB_DATA_DIR"] = str(_TMP)
|
||||
os.environ["AGENTLAB_DISABLE_DOTENV"] = "1"
|
||||
# 剥离宿主机环境里可能携带的真实模型密钥与地址,防止测试或 seed 误调线上付费接口;需要测环境变量回退的用例用 monkeypatch.setenv 单独设
|
||||
for _env_key in (
|
||||
"ANTHROPIC_API_KEY",
|
||||
"ANTHROPIC_BASE_URL",
|
||||
"ANTHROPIC_AUTH_TOKEN",
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_BASE_URL",
|
||||
"AGENTLAB_USE_OPENAI_EMBEDDINGS",
|
||||
):
|
||||
os.environ.pop(_env_key, None)
|
||||
# 下面这几项也都要在任何 app.* 导入之前设(两种库同一个顺序):PostgreSQL 分支要先导入 app.db.base 装连接池,settings 在那一刻
|
||||
# 就按当时的环境定下来了——以前这两项写在那之后,PostgreSQL 上 AGENTLAB_SCHEDULER 没生效(scheduler_on 成了 True)
|
||||
# 别把真实的 key 带进测试环境
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
"""第一批 P0 止血修复、安全加固与多模型适配回归测试。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import types
|
||||
import uuid
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core import artifact_store
|
||||
from app.core.config import settings
|
||||
from app.data import engine as data_engine
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Artifact, DataSource, Provider, Run
|
||||
from app.db.models_blueprints import StoredFile
|
||||
from app.engine import checkpoints
|
||||
from app.engine.context import NodeContext, NodeError, RunContext
|
||||
from app.engine.expressions import ExpressionError, eval_expression, resolve_path
|
||||
from app.engine.nodes.llm import _invoke_streaming, run_llm
|
||||
from app.engine.runner import run_manager
|
||||
from app.engine.schema import GraphNode, GraphSpec
|
||||
from app.files import store as files_store
|
||||
from app.main import app
|
||||
from app.providers import catalog, factory
|
||||
from app.providers.factory import ModelSpec, _ReasoningAwareChatOpenAI, build_chat_model
|
||||
from app.sandbox.local_sandbox import LocalSandbox
|
||||
from app.sandbox.seatbelt_sandbox import SeatbeltSandbox, build_policy
|
||||
from app.tools import net
|
||||
|
||||
|
||||
async def test_memory_sqlite_engine_cache_and_epoch_invalidation(tmp_path):
|
||||
cache = data_engine.EngineCache()
|
||||
try:
|
||||
mem_source = DataSource(
|
||||
id=uuid.uuid4().hex,
|
||||
name="mem-src",
|
||||
kind="sqlite",
|
||||
database=":memory:",
|
||||
readonly=False,
|
||||
)
|
||||
eng = await cache.get(mem_source)
|
||||
async with eng.connect() as conn:
|
||||
val = await conn.scalar(data_engine.text("SELECT 42"))
|
||||
assert val == 42
|
||||
|
||||
# 模拟 get() 在锁外核对期间并发触发 invalidate(),旧世代不得写回缓存
|
||||
db_file = tmp_path / "snap.db"
|
||||
conn0 = data_engine.open_checked_sqlite(str(db_file), readonly=False)
|
||||
try:
|
||||
conn0.execute("CREATE TABLE t (x INT)")
|
||||
conn0.commit()
|
||||
finally:
|
||||
conn0.close()
|
||||
digest = data_engine._file_sha256(str(db_file))
|
||||
snap_view = types.SimpleNamespace(
|
||||
id="src-epoch",
|
||||
snapshot_id="s1",
|
||||
kind="sqlite",
|
||||
database=str(db_file),
|
||||
readonly=True,
|
||||
immutable=True,
|
||||
expected_sha256=digest,
|
||||
options={},
|
||||
)
|
||||
orig_verify = cache._verify_once
|
||||
invalidated_once = False
|
||||
|
||||
async def slow_verify(key: str, source: object, expected: str):
|
||||
nonlocal invalidated_once
|
||||
fp = await orig_verify(key, source, expected)
|
||||
if not invalidated_once:
|
||||
invalidated_once = True
|
||||
await cache.invalidate("src-epoch", broadcast=False)
|
||||
return fp
|
||||
|
||||
cache._verify_once = slow_verify # type: ignore[method-assign]
|
||||
eng2 = await cache.get(snap_view)
|
||||
assert eng2 is not None
|
||||
assert cache._epochs.get("src-epoch", 0) == 1
|
||||
finally:
|
||||
await cache.close()
|
||||
|
||||
|
||||
async def test_sqlite_cancelled_query_interrupts_immediately(tmp_path):
|
||||
db_file = tmp_path / "slow.db"
|
||||
conn0 = data_engine.open_checked_sqlite(str(db_file), readonly=False)
|
||||
try:
|
||||
conn0.execute("CREATE TABLE t (x INT)")
|
||||
conn0.commit()
|
||||
finally:
|
||||
conn0.close()
|
||||
|
||||
src = DataSource(
|
||||
id=uuid.uuid4().hex,
|
||||
name="slow-sqlite",
|
||||
kind="sqlite",
|
||||
database=str(db_file),
|
||||
readonly=True,
|
||||
)
|
||||
infinite_cte = (
|
||||
"WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r WHERE n < 100000000) "
|
||||
"SELECT SUM(n) FROM r"
|
||||
)
|
||||
task = asyncio.create_task(
|
||||
data_engine.run_query(src, infinite_cte, limits=data_engine.QueryLimits(timeout_seconds=30))
|
||||
)
|
||||
await asyncio.sleep(0.08)
|
||||
task.cancel()
|
||||
# 修复前 finally 先把 deadline.over 置 True,这里会挂满整条递归 CTE 甚至永久卡死
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
await data_engine.engines.invalidate(src.id, broadcast=False)
|
||||
|
||||
|
||||
async def test_delete_run_cleans_artifacts_files_and_checkpoints():
|
||||
await run_manager.setup()
|
||||
try:
|
||||
graph = {
|
||||
"nodes": [
|
||||
{"id": "in", "type": "input", "data": {"config": {}}},
|
||||
{"id": "out", "type": "output", "data": {"config": {"fields": [{"name": "ok", "value": "1"}]}}},
|
||||
],
|
||||
"edges": [{"source": "in", "target": "out"}],
|
||||
}
|
||||
started = await run_manager.start(graph=graph, input_payload={})
|
||||
for _ in range(100):
|
||||
await asyncio.sleep(0.03)
|
||||
async with SessionLocal() as session:
|
||||
row = await session.get(Run, started.id)
|
||||
if row and row.status in ("succeeded", "failed"):
|
||||
break
|
||||
assert row is not None and row.status == "succeeded"
|
||||
|
||||
art_id = await artifact_store.put_json({"unique_for_delete_run": started.id}, kind="test", run_id=started.id)
|
||||
async with SessionLocal() as session:
|
||||
saved_file = await files_store.save(
|
||||
session,
|
||||
f"run-bound-{started.id}".encode(),
|
||||
"note.txt",
|
||||
uploaded_by="tester",
|
||||
context={"blueprint_id": "b1"},
|
||||
)
|
||||
saved_file.bound_run_id = started.id
|
||||
await session.commit()
|
||||
file_id, file_sha = saved_file.id, saved_file.sha256
|
||||
|
||||
ckpt_before, _ = await checkpoints.count(run_manager.checkpointer, started.id)
|
||||
assert ckpt_before >= 1
|
||||
assert artifact_store.load(art_id) is not None
|
||||
assert files_store.blob_path(file_sha).exists()
|
||||
|
||||
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
|
||||
resp = await client.delete(f"/api/runs/{started.id}")
|
||||
assert resp.status_code == 204, resp.text
|
||||
|
||||
assert artifact_store.load(art_id) is None
|
||||
assert not files_store.blob_path(file_sha).exists()
|
||||
async with SessionLocal() as session:
|
||||
assert await session.get(StoredFile, file_id) is None
|
||||
rem_arts = list((await session.execute(select(Artifact).where(Artifact.run_id == started.id))).scalars())
|
||||
assert rem_arts == []
|
||||
ckpt_after, _ = await checkpoints.count(run_manager.checkpointer, started.id)
|
||||
assert ckpt_after == 0
|
||||
finally:
|
||||
await run_manager.shutdown()
|
||||
|
||||
|
||||
async def test_sandbox_path_prefix_and_ephemeral_cleanup():
|
||||
box = LocalSandbox()
|
||||
# workdir 同前缀兄弟目录逃逸(如 tmp-xxx -> ../tmp-xxx2/evil.txt)必须被 is_relative_to 拦下
|
||||
escaped = await box.run("print(1)", session_id="sess1", files={"../sess1_sibling/evil.txt": "bad"})
|
||||
assert not escaped.ok and "非法文件路径" in (escaped.error or "")
|
||||
await box.cleanup("sess1")
|
||||
|
||||
before_dirs = {p.name for p in box._root.iterdir()} if box._root.exists() else set()
|
||||
ok = await box.run("print('hi')")
|
||||
assert ok.ok and "hi" in ok.stdout
|
||||
after_dirs = {p.name for p in box._root.iterdir()} if box._root.exists() else set()
|
||||
assert after_dirs - before_dirs == set(), "无 session_id 的临时工作目录执行完必须回收"
|
||||
|
||||
|
||||
def test_seatbelt_policy_denies_data_dir_outside_home(tmp_path, monkeypatch):
|
||||
ext_data = tmp_path / "external_agentlab_data"
|
||||
work = ext_data / "workspace" / "seatbelt" / "s1"
|
||||
work.mkdir(parents=True)
|
||||
monkeypatch.setattr(settings, "data_dir", ext_data)
|
||||
policy = build_policy(work, language="python", network=False)
|
||||
real_data = str(ext_data.resolve())
|
||||
real_work = str(work.resolve())
|
||||
assert f'(deny file-read* (subpath "{real_data}"))' in policy
|
||||
assert policy.index(f'(deny file-read* (subpath "{real_data}"))') < policy.index(
|
||||
f'(allow file-read* (subpath "{real_work}"))'
|
||||
)
|
||||
assert SeatbeltSandbox is not None
|
||||
|
||||
|
||||
async def test_net_ipv4_mapped_metadata_and_streaming_redirect_sanitization(monkeypatch):
|
||||
assert net._is_metadata("::ffff:169.254.169.254") is True
|
||||
assert net._is_metadata("::ffff:100.100.100.200") is True
|
||||
assert net._is_private("::ffff:127.0.0.1") is True
|
||||
|
||||
async def fake_assert_safe(url: str, *, allow_intranet: bool = False) -> str:
|
||||
return url
|
||||
|
||||
monkeypatch.setattr(net, "assert_safe_url", fake_assert_safe)
|
||||
seen_requests: list[httpx.Request] = []
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen_requests.append(request)
|
||||
if request.url.host == "origin-a.example":
|
||||
return httpx.Response(302, headers={"location": "https://origin-b.example/final?x=1"})
|
||||
return httpx.Response(200, content=b"0123456789abcdef", headers={"content-type": "text/plain"})
|
||||
|
||||
orig_client = httpx.AsyncClient
|
||||
|
||||
def patched_client(**kwargs):
|
||||
return orig_client(transport=httpx.MockTransport(handler), **{k: v for k, v in kwargs.items() if k != "trust_env"})
|
||||
|
||||
monkeypatch.setattr(net.httpx, "AsyncClient", patched_client)
|
||||
res = await net.safe_request(
|
||||
"POST",
|
||||
"https://origin-a.example/start",
|
||||
headers={"Authorization": "Bearer secret", "Cookie": "sid=1", "X-Custom": "keep"},
|
||||
params={"init": "1"},
|
||||
json_body={"hello": "world"},
|
||||
max_bytes=10,
|
||||
)
|
||||
assert res["status"] == 200
|
||||
assert res["body"] == "0123456789"
|
||||
assert res["truncated"] is True
|
||||
assert len(seen_requests) == 2
|
||||
hop2 = seen_requests[1]
|
||||
assert hop2.method == "GET"
|
||||
assert "authorization" not in hop2.headers
|
||||
assert "cookie" not in hop2.headers
|
||||
assert hop2.headers.get("x-custom") == "keep"
|
||||
assert str(hop2.url) == "https://origin-b.example/final?x=1"
|
||||
|
||||
|
||||
def test_expressions_private_attr_and_seq_repeat_cap():
|
||||
class Dummy:
|
||||
public = "ok"
|
||||
_secret = "nope"
|
||||
|
||||
assert resolve_path({"obj": Dummy()}, "obj.public") == "ok"
|
||||
assert resolve_path({"obj": Dummy()}, "obj._secret") is None
|
||||
|
||||
assert eval_expression("'ab' * 3", {}) == "ababab"
|
||||
with pytest.raises(ExpressionError, match="序列重复长度超过上限"):
|
||||
eval_expression("'x' * 2000000", {})
|
||||
with pytest.raises(ExpressionError, match="序列重复长度超过上限"):
|
||||
eval_expression("[1, 2] * 600000", {})
|
||||
|
||||
|
||||
async def test_provider_reasoning_roundtrip_bearer_catalog_and_structured_usage(monkeypatch):
|
||||
# 1. o1 / o3 / o4 禁采样 + 大小写不敏感匹配 + 自定义模型价格 + 缓存折扣
|
||||
assert catalog.supports_sampling("o1") is False
|
||||
assert catalog.supports_sampling("openai/o3-mini") is False
|
||||
assert catalog.supports_sampling("O4-mini") is False
|
||||
assert catalog.supports_sampling("ollama") is True
|
||||
assert catalog.find_pricing("deepseek-ai/deepseek-v3") is not None
|
||||
|
||||
base_cost = catalog.estimate_cost("claude-sonnet-5", 10_000, 1_000)
|
||||
cached_cost = catalog.estimate_cost("claude-sonnet-5", 10_000, 1_000, cache_read_tokens=9_000)
|
||||
assert cached_cost < base_cost
|
||||
|
||||
# 2. Anthropic bearer 保留 timeout 与 default_headers
|
||||
p_bearer = Provider(
|
||||
name="anth-gw",
|
||||
kind="anthropic",
|
||||
base_url="https://gw.example.com",
|
||||
api_key="tok-123",
|
||||
default_model="claude-sonnet-5",
|
||||
enabled=True,
|
||||
extra={"auth_style": "bearer", "headers": {"X-Tenant": "t1"}},
|
||||
)
|
||||
chat_anth = build_chat_model(p_bearer, ModelSpec(timeout=42))
|
||||
assert chat_anth._async_client.timeout == 42
|
||||
assert chat_anth._async_client.default_headers.get("X-Tenant") == "t1"
|
||||
|
||||
# 3. ReasoningAwareChatOpenAI 在带 tool_calls 的多轮消息里回传 reasoning_content
|
||||
llm = _ReasoningAwareChatOpenAI(model="deepseek-reasoner", api_key="k", base_url="https://api.deepseek.com/v1")
|
||||
msgs = [
|
||||
HumanMessage(content="查销售额"),
|
||||
AIMessage(
|
||||
content="",
|
||||
tool_calls=[{"name": "sql_query", "args": {"sql": "SELECT 1"}, "id": "call_1", "type": "tool_call"}],
|
||||
additional_kwargs={"reasoning_content": "先查表确认销售额"},
|
||||
),
|
||||
ToolMessage(content="100", tool_call_id="call_1"),
|
||||
]
|
||||
payload = llm._get_request_payload(msgs)
|
||||
assistant_msg = next(m for m in payload["messages"] if m["role"] == "assistant")
|
||||
assert assistant_msg["reasoning_content"] == "先查表确认销售额"
|
||||
|
||||
# 4. _NO_STREAM_USAGE 按 (endpoint, model) 粒度隔离
|
||||
monkeypatch.setattr(factory, "_NO_STREAM_USAGE", {("https://gw.example.com/v1", "legacy-model")})
|
||||
assert ("https://gw.example.com/v1", "modern-model") not in factory._NO_STREAM_USAGE
|
||||
|
||||
# 5. run_llm 结构化输出通过 include_raw=True 记录真实 token 与费用
|
||||
class FakeStructuredModel:
|
||||
def with_structured_output(self, schema, include_raw: bool = False):
|
||||
assert include_raw is True
|
||||
|
||||
class _Runner:
|
||||
async def ainvoke(self_inner, _messages):
|
||||
return {
|
||||
"raw": AIMessage(
|
||||
content='{"answer": 42}',
|
||||
usage_metadata={"input_tokens": 500, "output_tokens": 100, "total_tokens": 600},
|
||||
),
|
||||
"parsed": {"answer": 42},
|
||||
"parsing_error": None,
|
||||
}
|
||||
|
||||
return _Runner()
|
||||
|
||||
async def fake_get_chat_model(_session, _spec):
|
||||
return FakeStructuredModel(), "gpt-5.2"
|
||||
|
||||
monkeypatch.setattr("app.engine.nodes.llm.get_chat_model", fake_get_chat_model)
|
||||
node = GraphNode(
|
||||
id="s1",
|
||||
type="llm",
|
||||
data={
|
||||
"label": "结构化",
|
||||
"config": {
|
||||
"prompt": "回答",
|
||||
"output_schema": {"type": "object", "properties": {"answer": {"type": "integer"}}},
|
||||
},
|
||||
},
|
||||
)
|
||||
events_seen: list[dict] = []
|
||||
ctx = NodeContext(
|
||||
node=node,
|
||||
run=RunContext(
|
||||
run_id="r-struct",
|
||||
thread_id="t-struct",
|
||||
spec=GraphSpec(nodes=[node], edges=[]),
|
||||
extra={"stream_writer": lambda ev: events_seen.append(ev)},
|
||||
),
|
||||
)
|
||||
out = await run_llm({"input": {}, "vars": {}, "nodes": {}, "messages": []}, ctx)
|
||||
assert out["nodes"]["s1"]["data"] == {"answer": 42}
|
||||
assert out["usage"]["input_tokens"] == 500 and out["usage"]["output_tokens"] == 100
|
||||
assert out["usage"]["cost_usd"] > 0
|
||||
|
||||
# 6. _invoke_streaming 遇到 401/429 等不可重试错误直接抛出,不触发二次非流式请求
|
||||
ainvoke_called = 0
|
||||
|
||||
class AuthFailModel:
|
||||
async def astream(self, _messages):
|
||||
if False:
|
||||
yield None
|
||||
err = RuntimeError("401 invalid api key")
|
||||
err.status_code = 401 # type: ignore[attr-defined]
|
||||
raise err
|
||||
|
||||
async def ainvoke(self, _messages):
|
||||
nonlocal ainvoke_called
|
||||
ainvoke_called += 1
|
||||
return AIMessage(content="should not happen")
|
||||
|
||||
with pytest.raises(NodeError):
|
||||
await _invoke_streaming(AuthFailModel(), [HumanMessage(content="hi")], ctx, "gpt-5.2")
|
||||
assert ainvoke_called == 0
|
||||
@@ -0,0 +1,256 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from fractions import Fraction
|
||||
import math
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import event, text
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
from app.analysis import DATA_ERRORS, stats as analysis_stats
|
||||
from app.analysis.contract import CallError
|
||||
from app.analysis.decompose import lmdi
|
||||
from app.analysis.differ import MAX_ROWS_MODE
|
||||
from app.analysis.periods import plain_day
|
||||
from app.analysis.stats import theil_sen
|
||||
from app.chat.direct import build_direct_graph
|
||||
from app.chat.direct_evidence import normalize_and_bind_fast_citations
|
||||
from app.chat.grounding import GroundingBundle
|
||||
from app.cluster.service import _Service
|
||||
from app.core.bus import EventBus, _MAX_SEQ_ENTRIES
|
||||
from app.data import coverage, csv_grid
|
||||
from app.data.tabular import _detect_csv_encoding
|
||||
from app.db.base import _INDEX_MIGRATIONS, _set_sqlite_pragma, create_schema
|
||||
from app.engine.schema import GraphSpec, validate_graph
|
||||
from app.ledger.match import Caliber, Fetch, Query, compare_queries
|
||||
from tests.analysis_samples import analyze, snap as make_snap
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sqlite_pragmas_and_index_migrations(tmp_path: Path) -> None:
|
||||
db_path = tmp_path / "idx_check.db"
|
||||
eng = create_async_engine(f"sqlite+aiosqlite:///{db_path}")
|
||||
event.listen(eng.sync_engine, "connect", _set_sqlite_pragma)
|
||||
try:
|
||||
await create_schema(eng)
|
||||
async with eng.connect() as conn:
|
||||
sync_mode = (await conn.execute(text("PRAGMA synchronous"))).scalar_one()
|
||||
# SQLite NORMAL = 1
|
||||
assert sync_mode == 1
|
||||
rows = (await conn.execute(text("SELECT name FROM sqlite_master WHERE type='index'"))).scalars().all()
|
||||
existing = set(rows)
|
||||
for idx_name, _, _ in _INDEX_MIGRATIONS:
|
||||
assert idx_name in existing
|
||||
finally:
|
||||
await eng.dispose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_event_bus_seq_lru_and_close_eviction() -> None:
|
||||
bus = EventBus()
|
||||
seq1 = bus.next_seq("run-1")
|
||||
assert seq1 == 1
|
||||
assert "run-1" in bus._seq
|
||||
await bus.close("run-1")
|
||||
assert "run-1" not in bus._seq
|
||||
|
||||
for i in range(_MAX_SEQ_ENTRIES + 25):
|
||||
bus.next_seq(f"r-{i}")
|
||||
assert len(bus._seq) <= _MAX_SEQ_ENTRIES
|
||||
assert "r-0" not in bus._seq
|
||||
|
||||
|
||||
def test_cluster_slot_signal_wakes_local_waiters(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
woken: list[bool] = []
|
||||
monkeypatch.setattr("app.cluster.service.slots.wake_local", lambda: woken.append(True))
|
||||
svc = _Service()
|
||||
svc._on_slot(["w1"])
|
||||
assert woken == [True]
|
||||
|
||||
|
||||
def test_coverage_parses_two_digit_tz_offset_and_lru_eviction() -> None:
|
||||
d1 = coverage.as_day("2026-08-31 23:30:00+08")
|
||||
d2 = coverage.as_day("2026-08-31T00:00:00+00")
|
||||
d3 = coverage.as_day("2026-08-31 12:00:00-0530")
|
||||
assert d1 is not None and d1.day == date(2026, 8, 31) and d1.timed is True
|
||||
assert d2 is not None and d2.day == date(2026, 8, 31) and d2.timed is False
|
||||
assert d3 is not None and d3.day == date(2026, 8, 31) and d3.timed is True
|
||||
|
||||
coverage._LIVE.clear()
|
||||
try:
|
||||
for i in range(coverage._LIVE_MAX + 10):
|
||||
coverage._LIVE[("src", f"t_{i}")] = (1e12, i) # type: ignore[assignment]
|
||||
if len(coverage._LIVE) > coverage._LIVE_MAX:
|
||||
coverage._LIVE.popitem(last=False)
|
||||
assert len(coverage._LIVE) == coverage._LIVE_MAX
|
||||
assert ("src", "t_0") not in coverage._LIVE
|
||||
assert ("src", f"t_{coverage._LIVE_MAX + 9}") in coverage._LIVE
|
||||
finally:
|
||||
coverage._LIVE.clear()
|
||||
|
||||
|
||||
def test_compare_queries_prioritizes_table_overlap_before_max_candidates() -> None:
|
||||
calibers: list[Caliber] = []
|
||||
for i in range(95):
|
||||
calibers.append(
|
||||
Caliber(
|
||||
workflow_id=f"wf_{i}",
|
||||
version=1,
|
||||
level="certified",
|
||||
graph_hash=f"h_{i}",
|
||||
fetches=(
|
||||
Fetch(
|
||||
node_id="q1",
|
||||
source="sales",
|
||||
sql=f"SELECT COUNT(*) AS cnt_{i} FROM orders",
|
||||
tables=frozenset({"orders"}),
|
||||
),
|
||||
),
|
||||
)
|
||||
)
|
||||
calibers.append(
|
||||
Caliber(
|
||||
workflow_id="wf_exact",
|
||||
version=1,
|
||||
level="certified",
|
||||
graph_hash="h_exact",
|
||||
fetches=(
|
||||
Fetch(
|
||||
node_id="q1",
|
||||
source="sales",
|
||||
sql="SELECT SUM(o.amount - COALESCE(r.refund_amt, 0)) AS net_sales "
|
||||
"FROM orders o LEFT JOIN refunds r ON r.order_id = o.id",
|
||||
tables=frozenset({"orders", "refunds"}),
|
||||
),
|
||||
),
|
||||
)
|
||||
)
|
||||
q = Query(
|
||||
artifact="sha_q1",
|
||||
node_id="fetch_1",
|
||||
source="sales",
|
||||
sql="SELECT SUM(o.amount - COALESCE(r.refund_amt, 0)) AS net_sales "
|
||||
"FROM orders o LEFT JOIN refunds r ON r.order_id = o.id",
|
||||
columns=["net_sales"],
|
||||
tables=frozenset({"orders", "refunds"}),
|
||||
dialect="sqlite",
|
||||
)
|
||||
res = compare_queries([q], calibers)
|
||||
assert len(res) == 1
|
||||
assert res[0]["status"] == "proven"
|
||||
assert res[0]["candidates"][0]["workflow_id"] == "wf_exact"
|
||||
|
||||
|
||||
def test_direct_evidence_indexed_auto_bind_preserves_semantics() -> None:
|
||||
snap = {
|
||||
"kind": "tool_output",
|
||||
"tool": "db_query__mes",
|
||||
"columns": ["line_code", "good_qty", "yield_rate", "output_amt"],
|
||||
"column_types": ["text", "integer", "real", "real"],
|
||||
"rows": [["L01", 69133, 0.985, 125000.0]],
|
||||
"truncated": False,
|
||||
}
|
||||
catalog = {
|
||||
"Q1": {
|
||||
"kind": "query",
|
||||
"artifact": "art_q1",
|
||||
"tool": "db_query__mes",
|
||||
"source": "mes",
|
||||
"columns": ["line_code", "good_qty", "yield_rate", "output_amt"],
|
||||
"row_count": 1,
|
||||
}
|
||||
}
|
||||
loader = lambda art: snap if art == "art_q1" else None
|
||||
raw = "产线 L01 良品数为 69,133 件,良率达到 98.5%,产值 12.5万 元。"
|
||||
bound = normalize_and_bind_fast_citations(raw, catalog, loader=loader)
|
||||
assert "[[v:Q1.r0.good_qty]]" in bound
|
||||
assert "[[v:Q1.r0.yield_rate|pct]]" in bound
|
||||
assert "[[v:Q1.r0.output_amt|万]]" in bound
|
||||
|
||||
|
||||
def test_plain_day_recognizes_cast_and_pg_double_colon() -> None:
|
||||
assert plain_day("SELECT CAST(stat_date AS DATE) AS dt, SUM(x) FROM t GROUP BY 1", "dt", "sqlite") is True
|
||||
assert plain_day("SELECT stat_date::date AS dt, SUM(x) FROM t GROUP BY 1", "dt", "postgres") is True
|
||||
assert plain_day("SELECT DATE_TRUNC('month', stat_date) AS dt, SUM(x) FROM t GROUP BY 1", "dt", "postgres") is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_differ_rows_mode_enforces_max_rows_ceiling() -> None:
|
||||
assert "too_many_rows" in DATA_ERRORS
|
||||
rows = [["A" if i % 2 == 0 else "B", float(i)] for i in range(MAX_ROWS_MODE + 5)]
|
||||
s = make_snap("Q1", "SELECT grp, val FROM t", ["grp", "val"], rows)
|
||||
with pytest.raises(CallError) as exc_info:
|
||||
await analyze("differ", "rows", snaps=[s], query="Q1", by="grp", value="val")
|
||||
assert exc_info.value.code == "too_many_rows"
|
||||
|
||||
refused = await analyze(
|
||||
"differ", "rows", snaps=[s], refuse_data_errors=True, query="Q1", by="grp", value="val"
|
||||
)
|
||||
assert refused["status"] == "refused"
|
||||
assert refused["reason_code"] == "input_unavailable"
|
||||
assert str(MAX_ROWS_MODE) in refused["reason"]
|
||||
|
||||
|
||||
def test_theil_sen_handles_large_denominator_lcm_and_extreme_floats() -> None:
|
||||
primes = [
|
||||
2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71,
|
||||
73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151,
|
||||
157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211, 223, 227, 229, 233,
|
||||
239, 241, 251, 257, 263, 269, 271, 277, 281,
|
||||
]
|
||||
xs = [Fraction(i + 1, p) for i, p in enumerate(primes)]
|
||||
ys = [x * 3 + Fraction(1, 7) for x in xs]
|
||||
slope = theil_sen(xs, ys)
|
||||
assert math.isclose(slope, 3.0, rel_tol=1e-9)
|
||||
|
||||
huge_xs = [Fraction(0), Fraction(1, 10**200), Fraction(2, 10**200)]
|
||||
huge_ys = [Fraction(0), Fraction(10**200, 1), Fraction(2 * 10**200, 1)]
|
||||
huge_slope = theil_sen(huge_xs, huge_ys)
|
||||
assert huge_slope == Fraction(10**400, 1)
|
||||
assert math.isinf(analysis_stats._safe_float(huge_slope))
|
||||
|
||||
|
||||
def test_lmdi_handles_many_factors_without_float_overflow() -> None:
|
||||
base = [10.0] * 300
|
||||
target = [20.0] * 300
|
||||
contribs = lmdi(base, target, 100.0, 250.0)
|
||||
assert len(contribs) == 300
|
||||
assert math.isclose(sum(contribs), 150.0, rel_tol=1e-9)
|
||||
|
||||
|
||||
def test_streaming_csv_encoding_detection_and_grid_scan() -> None:
|
||||
gbk_bytes = "产线,良品数,备注\nL01,1200,正常生产\nL02,980,换型调试\n".encode("gb18030")
|
||||
assert _detect_csv_encoding(gbk_bytes) == ("gb18030", "strict")
|
||||
|
||||
utf8_bom = b"\xef\xbb\xbfcol_a,col_b\n1,2\n"
|
||||
assert _detect_csv_encoding(utf8_bom) == ("utf-8-sig", "strict")
|
||||
|
||||
scanned = csv_grid.scan(gbk_bytes)
|
||||
assert len(scanned.sheets) == 1
|
||||
assert scanned.sheets[0].bounds is not None
|
||||
assert scanned.sheets[0].bounds.max_row == 3
|
||||
assert scanned.sheets[0].bounds.max_col == 3
|
||||
|
||||
parsed = csv_grid.parse(gbk_bytes)
|
||||
assert len(parsed.records) == 3
|
||||
assert parsed.records[0] == ["产线", "良品数", "备注"]
|
||||
assert parsed.records[1] == ["L01", "1200", "正常生产"]
|
||||
|
||||
streamed = list(csv_grid.rows(parsed, min_row=1, max_row=3, min_col=1, max_col=3))
|
||||
assert len(streamed) == 3
|
||||
|
||||
|
||||
def test_build_direct_graph_with_and_without_metrics_passes_validation() -> None:
|
||||
b = GroundingBundle(narrowed_sources=[SimpleNamespace(name="demo_db")]) # type: ignore[list-item]
|
||||
assert b.has_enabled_metrics is False
|
||||
|
||||
g_no_metrics = build_direct_graph(bundle=b, has_metrics=False)
|
||||
res_no = validate_graph(GraphSpec.model_validate(g_no_metrics))
|
||||
assert res_no.ok is True, res_no.issues
|
||||
|
||||
g_with_metrics = build_direct_graph(bundle=b, has_metrics=True)
|
||||
res_yes = validate_graph(GraphSpec.model_validate(g_with_metrics))
|
||||
assert res_yes.ok is True, res_yes.issues
|
||||
@@ -0,0 +1,388 @@
|
||||
"""第三批架构级演进回归测试(Step 3.1 – 3.4)。
|
||||
|
||||
覆盖:
|
||||
1. 记录库 `records` 多版本单库归档演进(突破 SQLite 10 库 `ATTACH` 上限、增量同步、WAL 与 `records_archive` 越权拦截);
|
||||
2. 知识库真·双路混合检索(零词面重合高语义命中不被 BM25 前置截断、向量缓存失效、>1000 块倒排分批更新);
|
||||
3. 方言感知 SQL 模板词法分析(MySQL `\\'`/`#`、PostgreSQL `E'...'`/`$$...$$`/嵌套注释/数组下标、Oracle `q'[...]'`、SQLite `[id]`)与 `Decimal`/`bool`/`NaN` 参数完整性;
|
||||
4. 分层依赖守卫(`app/ledger/*` 零依赖 `app.api.settings` / `app.api.ledger` / `app.api.evidence`)与前端组件拆分门禁。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import sqlite3
|
||||
import uuid
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.config import settings
|
||||
from app.data.engine import run_query
|
||||
from app.data.guard import SqlRejected
|
||||
from app.data.sql_semantics import _template_slots
|
||||
from app.db.base import SessionLocal
|
||||
from app.db.models import Chunk, DataSource, Document
|
||||
from app.db.models_apps import App
|
||||
from app.db.models_blueprints import RecordsTable
|
||||
from app.engine.expressions import render_template
|
||||
from app.ledger import params
|
||||
from app.memory import embeddings as emb
|
||||
from app.memory import inverted, kb
|
||||
from app.records import store
|
||||
from app.records.tables import columns_of, ensure_table, path_of
|
||||
from app.records.views import ARCHIVE_SCHEMA, resolve
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_data_dir(monkeypatch, tmp_path):
|
||||
data = tmp_path / "data"
|
||||
(data / "uploads").mkdir(parents=True)
|
||||
monkeypatch.setattr(settings, "data_dir", data)
|
||||
kb.invalidate_vector_cache()
|
||||
yield data
|
||||
kb.invalidate_vector_cache()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 3.1:记录库 >10 版本单库归档、增量同步与安全边界
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_records_scale_beyond_sqlite_attach_limit_with_incremental_archive(isolated_data_dir):
|
||||
async with SessionLocal() as session:
|
||||
owner = App(name="多版本报损台账", created_by="admin")
|
||||
session.add(owner)
|
||||
await session.commit()
|
||||
app_id = owner.id
|
||||
|
||||
versions: list[RecordsTable] = []
|
||||
inserted_row_ids: dict[int, str] = {}
|
||||
|
||||
# 连续演进 14 个不兼容版本(每次新增一个必填列),远超 SQLite 默认 SQLITE_MAX_ATTACHED=10 上限
|
||||
for idx in range(1, 15):
|
||||
cols = [
|
||||
{"name": "门店", "label": "门店", "type": "string", "required": True},
|
||||
{"name": "数量", "label": "数量", "type": "integer", "required": True},
|
||||
{"name": f"必填项_{idx}", "label": f"必填项_{idx}", "type": "string", "required": True},
|
||||
]
|
||||
async with SessionLocal() as session:
|
||||
owner = await session.get(App, app_id)
|
||||
tbl = await ensure_table(session, owner, cols)
|
||||
await session.commit()
|
||||
assert tbl.version == idx
|
||||
rec = store.append(
|
||||
path_of(tbl),
|
||||
columns_of(tbl),
|
||||
{"门店": f"门店_{idx:02d}", "数量": idx * 10, f"必填项_{idx}": f"v{idx}"},
|
||||
version=tbl.version,
|
||||
run_id=None,
|
||||
actor="amy",
|
||||
action_key=uuid.uuid4().hex,
|
||||
)
|
||||
inserted_row_ids[idx] = rec.row_id
|
||||
versions.append(tbl)
|
||||
|
||||
# 验证单表库开启了 WAL 模式
|
||||
with sqlite3.connect(path_of(versions[-1])) as conn:
|
||||
mode = conn.execute("PRAGMA journal_mode").fetchone()[0]
|
||||
assert str(mode).lower() == "wal"
|
||||
|
||||
# 在将被归档的第 2 版中做一次更正(supersedes),确保归档表保留 _supersedes 演进语义
|
||||
store.append(
|
||||
path_of(versions[1]),
|
||||
columns_of(versions[1]),
|
||||
{"门店": "门店_02", "数量": 25, "必填项_2": "v2_fixed"},
|
||||
version=2,
|
||||
run_id=None,
|
||||
actor="amy",
|
||||
action_key=uuid.uuid4().hex,
|
||||
supersedes=inserted_row_ids[2],
|
||||
)
|
||||
|
||||
latest = versions[-1]
|
||||
async with SessionLocal() as session:
|
||||
source = await session.get(DataSource, latest.source_id)
|
||||
resolved = await resolve(session, source)
|
||||
|
||||
# 13 个历史版本若逐库 ATTACH 会触发 OperationalError: too many attached databases (max 10)
|
||||
# 归档收敛后挂接数 <= 8(7 个近期直挂库 + 1 个归档库)
|
||||
assert len(resolved.attach) <= 8
|
||||
assert ARCHIVE_SCHEMA in dict(resolved.attach)
|
||||
assert store.archive_path(latest.table_key).exists()
|
||||
|
||||
dbs = await run_query(resolved, "PRAGMA database_list")
|
||||
assert len(dbs.rows) <= 10
|
||||
|
||||
summary = await run_query(resolved, 'SELECT count(*) AS n, sum("数量") AS total FROM records')
|
||||
expected_total = sum(i * 10 for i in range(1, 15)) - 20 + 25
|
||||
assert summary.rows == [[14, expected_total]]
|
||||
|
||||
# 模拟停用前已在途的动作向第 1 版追加一行,验证 sync_archive 按 _seq > since_seq 增量同步
|
||||
store.append(
|
||||
path_of(versions[0]),
|
||||
columns_of(versions[0]),
|
||||
{"门店": "门店_01_补单", "数量": 5, "必填项_1": "v1_late"},
|
||||
version=1,
|
||||
run_id=None,
|
||||
actor="bob",
|
||||
action_key=uuid.uuid4().hex,
|
||||
)
|
||||
async with SessionLocal() as session:
|
||||
refreshed = await resolve(session, source)
|
||||
summary_after = await run_query(refreshed, 'SELECT count(*) AS n, sum("数量") AS total FROM records')
|
||||
assert summary_after.rows == [[15, expected_total + 5]]
|
||||
|
||||
# 直接点名归档 schema 必须被 SQL 守卫拦截
|
||||
for forbidden_sql in (
|
||||
f"SELECT count(*) FROM {ARCHIVE_SCHEMA}.rows_v1",
|
||||
f'SELECT count(*) FROM "{ARCHIVE_SCHEMA}"."rows_v1"',
|
||||
f"SELECT count(*) FROM [{ARCHIVE_SCHEMA.upper()}].rows_v1",
|
||||
):
|
||||
with pytest.raises(SqlRejected):
|
||||
await run_query(refreshed, forbidden_sql)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 3.2:真·双路混合检索、缓存失效与超千块倒排分批处理
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_kb_true_dual_path_hybrid_search_finds_zero_lexical_overlap_hit(isolated_data_dir, monkeypatch):
|
||||
target_text = "超级管理员可调整他人角色与访问范围。"
|
||||
distractors = [f"审批规范第 {i} 条:普通报销单需部门主管签字归档。" for i in range(65)]
|
||||
|
||||
async def fake_embed_texts(texts: list[str]) -> list[np.ndarray]:
|
||||
vecs: list[np.ndarray] = []
|
||||
for t in texts:
|
||||
v = np.zeros(emb.embedder_dim(), dtype=np.float32)
|
||||
if "超级管理员" in t:
|
||||
v[0] = 1.0
|
||||
else:
|
||||
v[1] = 1.0
|
||||
vecs.append(v)
|
||||
return vecs
|
||||
|
||||
async def fake_embed_text(query: str) -> np.ndarray:
|
||||
v = np.zeros(emb.embedder_dim(), dtype=np.float32)
|
||||
v[0] = 1.0
|
||||
return v
|
||||
|
||||
monkeypatch.setattr(kb, "embed_texts", fake_embed_texts)
|
||||
monkeypatch.setattr(kb, "embed_text", fake_embed_text)
|
||||
|
||||
collection = f"dual_path_{uuid.uuid4().hex[:8]}"
|
||||
async with SessionLocal() as session:
|
||||
# 65 条词面强命中「审批规范」的干扰块(超过原 BM25 单路候选池上限 50)
|
||||
for idx, text in enumerate(distractors):
|
||||
await kb.ingest_document(session, collection=collection, title=f"干扰_{idx}", content=text)
|
||||
# 1 条零词面重合但语义完全一致的目标块
|
||||
target_doc = await kb.ingest_document(session, collection=collection, title="权限说明", content=target_text)
|
||||
|
||||
degrade_reasons: list[str] = []
|
||||
async with SessionLocal() as session:
|
||||
hits = await kb.search(
|
||||
session,
|
||||
collection=collection,
|
||||
query="审批规范里谁能改权限",
|
||||
limit=5,
|
||||
alpha=0.6,
|
||||
on_degrade=degrade_reasons.append,
|
||||
)
|
||||
|
||||
assert degrade_reasons == []
|
||||
assert hits, "双路召回应当返回结果"
|
||||
assert hits[0]["content"] == target_text, "零词面重合但余弦相似度为 1.0 的目标块在双路混合检索下应排第一"
|
||||
|
||||
# 删除目标文档后,向量矩阵缓存自动失效,不再返回已删块
|
||||
async with SessionLocal() as session:
|
||||
deleted = await kb.delete_document(session, target_doc.id)
|
||||
assert deleted is True
|
||||
after_delete = await kb.search(
|
||||
session,
|
||||
collection=collection,
|
||||
query="审批规范里谁能改权限",
|
||||
limit=5,
|
||||
alpha=0.6,
|
||||
)
|
||||
assert all(h["content"] != target_text for h in after_delete)
|
||||
|
||||
|
||||
async def test_inverted_index_handles_over_thousand_chunks_without_bind_limit(isolated_data_dir):
|
||||
collection = f"batch_inv_{uuid.uuid4().hex[:8]}"
|
||||
doc_id = f"doc_{uuid.uuid4().hex[:12]}"
|
||||
chunk_ids: list[str] = []
|
||||
made: list[Chunk] = []
|
||||
async with SessionLocal() as session:
|
||||
session.add(Document(id=doc_id, collection=collection, title="bulk", content="bulk"))
|
||||
await session.flush()
|
||||
for i in range(1100):
|
||||
cid = uuid.uuid4().hex
|
||||
chunk_ids.append(cid)
|
||||
chunk = Chunk(
|
||||
id=cid,
|
||||
document_id=doc_id,
|
||||
collection=collection,
|
||||
ordinal=i,
|
||||
content=f"统一结算批次规范条目 {i}",
|
||||
embedding=b"",
|
||||
embed_model="",
|
||||
embed_dim=0,
|
||||
meta={"title": "bulk"},
|
||||
)
|
||||
session.add(chunk)
|
||||
made.append(chunk)
|
||||
await session.flush()
|
||||
for chunk in made:
|
||||
await inverted.index_chunk(session, chunk)
|
||||
await session.commit()
|
||||
|
||||
async with SessionLocal() as session:
|
||||
scored, all_scores = await inverted.candidates_with_scores(
|
||||
session, collection=collection, query="统一结算批次规范", limit=25,
|
||||
)
|
||||
assert len(scored) == 25
|
||||
assert len(all_scores) == 1100
|
||||
assert scored[0][1] > 0
|
||||
|
||||
# 超 1000 个 chunk_id 删除时自动分批,不触发 SQLite 999 / PG 32767 绑定参数上限
|
||||
await inverted.drop_chunk_terms(session, chunk_ids)
|
||||
await session.commit()
|
||||
after, after_all = await inverted.candidates_with_scores(
|
||||
session, collection=collection, query="统一结算批次规范", limit=25,
|
||||
)
|
||||
assert after == []
|
||||
assert after_all == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 3.3:方言感知 SQL 模板词法分析与 Decimal / bool / NaN 参数完整性
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_dialect_aware_sql_template_slots():
|
||||
# MySQL 反斜杠转义单引号与 # 行注释:注释里的模板识别为 comment,真参数仍在 WHERE 后作为 bare
|
||||
mysql_sql = r"SELECT 'it\'s fine' AS tag # 忽略 {{ input.ignored }}" + "\nFROM orders WHERE id = {{ input.order_id }}"
|
||||
mysql_spans, _ = _template_slots(mysql_sql, dialect="mysql")
|
||||
assert [(s.field, s.context) for _, _, s in mysql_spans] == [
|
||||
("ignored", "comment"),
|
||||
("order_id", "bare"),
|
||||
]
|
||||
|
||||
# PostgreSQL $$...$$ 美元引号、E'...' 转义串、嵌套块注释与数组下标 arr[{{ input.idx }}]
|
||||
pg_sql = (
|
||||
"/* 外层 /* 内层 {{ input.in_comment }} */ 结束 */\n"
|
||||
"SELECT $$raw {{ input.in_dollar }}$$ AS d, E'escaped \\' {{ input.in_estr }}' AS e, "
|
||||
"tags[{{ input.idx }}] AS picked FROM items"
|
||||
)
|
||||
pg_spans, _ = _template_slots(pg_sql, dialect="postgres")
|
||||
assert [(s.field, s.context) for _, _, s in pg_spans] == [
|
||||
("in_comment", "comment"),
|
||||
("in_dollar", "quoted"),
|
||||
("in_estr", "quoted"),
|
||||
("idx", "bare"),
|
||||
]
|
||||
|
||||
# Oracle q'[...]' 替代引号字面量识别为 quoted 上下文
|
||||
oracle_sql = "SELECT q'[ Department's {{ input.dept }} ]' FROM dual"
|
||||
oracle_spans, _ = _template_slots(oracle_sql, dialect="oracle")
|
||||
assert [(s.field, s.context) for _, _, s in oracle_spans] == [("dept", "quoted")]
|
||||
|
||||
# SQLite 方括号标识符里的占位符属于不支持的标识符注入位,必须拒绝
|
||||
sqlite_sql = "SELECT * FROM [{{ input.table_name }}]"
|
||||
with pytest.raises(params.ParameterRefused) as exc_info:
|
||||
params.slots_of(sqlite_sql, dialect="sqlite")
|
||||
assert exc_info.value.code == params.UNSUPPORTED
|
||||
|
||||
|
||||
def test_decimal_boolean_and_non_finite_parameter_integrity():
|
||||
# 极小 Decimal 在表达式求值与参数写出时不退化为科学计数法 1E-7
|
||||
rendered = render_template("{{ input.rate }}", {"input": {"rate": Decimal("1E-7")}})
|
||||
assert rendered == "0.0000001"
|
||||
|
||||
fields = [
|
||||
{"name": "amount", "type": "decimal", "required": True},
|
||||
{"name": "active", "type": "boolean", "required": True},
|
||||
]
|
||||
template = "SELECT * FROM ledger_tx WHERE amount >= {{ input.amount }} AND is_active = {{ input.active }}"
|
||||
assert params.check_template(template, fields=params.declared(fields), dialect="postgres") == []
|
||||
|
||||
stmt = params.plan(
|
||||
template,
|
||||
fields=fields,
|
||||
values={"amount": Decimal("19.95"), "active": True},
|
||||
dialect="postgres",
|
||||
)
|
||||
assert "amount >= 19.95" in stmt.statement
|
||||
assert "is_active = True" in stmt.statement
|
||||
by_name = {entry["name"]: entry for entry in stmt.material["parameters"]}
|
||||
assert by_name["amount"]["kind"] == "number" and by_name["amount"]["declared"] == "number"
|
||||
assert by_name["active"]["kind"] == "boolean" and by_name["active"]["declared"] == "boolean"
|
||||
|
||||
# NaN / Inf 绝不能当作合法裸数拼入 SQL(否则会变成列名 NaN / Inf)
|
||||
for bad_number in (float("nan"), float("inf"), float("-inf"), Decimal("NaN"), Decimal("Infinity")):
|
||||
with pytest.raises(params.ParameterRefused) as exc_info:
|
||||
params.plan(
|
||||
"SELECT * FROM ledger_tx WHERE amount >= {{ input.amount }}",
|
||||
fields=[{"name": "amount", "type": "decimal", "required": True}],
|
||||
values={"amount": bad_number},
|
||||
dialect="postgres",
|
||||
)
|
||||
assert exc_info.value.code == params.LITERAL
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Step 3.3 & 3.4:分层依赖边界与前端超大文件拆分门禁
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_ledger_layer_has_no_upward_api_domain_imports():
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
ledger_dir = repo_root / "backend" / "app" / "ledger"
|
||||
forbidden_modules = {"app.api.settings", "app.api.ledger", "app.api.evidence"}
|
||||
|
||||
violations: list[str] = []
|
||||
for py_file in sorted(ledger_dir.rglob("*.py")):
|
||||
tree = ast.parse(py_file.read_text(encoding="utf-8"), filename=str(py_file))
|
||||
rel = py_file.relative_to(repo_root)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
if alias.name in forbidden_modules:
|
||||
violations.append(f"{rel}:{node.lineno} import {alias.name}")
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
if node.module in forbidden_modules:
|
||||
violations.append(f"{rel}:{node.lineno} from {node.module} import ...")
|
||||
elif node.module == "app.api":
|
||||
for alias in node.names:
|
||||
full = f"app.api.{alias.name}"
|
||||
if full in forbidden_modules:
|
||||
violations.append(f"{rel}:{node.lineno} from app.api import {alias.name}")
|
||||
|
||||
assert violations == [], f"ledger 领域层出现了对 api 路由层的反向依赖:{violations}"
|
||||
|
||||
listing_py = repo_root / "backend" / "app" / "records" / "listing.py"
|
||||
listing_src = listing_py.read_text(encoding="utf-8")
|
||||
assert "app.api.evidence" not in listing_src
|
||||
assert "app.engine.evidence_reader" in listing_src
|
||||
|
||||
|
||||
def test_caliber_drafts_page_is_modularized():
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
drafts_page = repo_root / "frontend" / "src" / "pages" / "ledger" / "CaliberDrafts.tsx"
|
||||
drafts_dir = repo_root / "frontend" / "src" / "pages" / "ledger" / "drafts"
|
||||
|
||||
line_count = len(drafts_page.read_text(encoding="utf-8").splitlines())
|
||||
assert line_count < 900, f"CaliberDrafts.tsx 仍超过 900 行(当前 {line_count} 行)"
|
||||
expected_modules = {
|
||||
"helpers.ts",
|
||||
"DraftSharedBlocks.tsx",
|
||||
"DraftTrialSection.tsx",
|
||||
"DraftRulesSection.tsx",
|
||||
"DraftImportsSection.tsx",
|
||||
"DraftNlSection.tsx",
|
||||
"DraftValidationSection.tsx",
|
||||
}
|
||||
actual_modules = {p.name for p in drafts_dir.glob("*.ts*")}
|
||||
assert expected_modules.issubset(actual_modules)
|
||||
@@ -63,13 +63,10 @@ def test_empty_reasoning_is_not_reported() -> None:
|
||||
|
||||
|
||||
def _anthropic(model: str, **spec_kw):
|
||||
import os
|
||||
|
||||
from app.db.models import Provider
|
||||
from app.providers.factory import ModelSpec, build_chat_model
|
||||
|
||||
os.environ.setdefault("ANTHROPIC_API_KEY", "test-only")
|
||||
provider = Provider(name="t", kind="anthropic", api_key=None, models=[],
|
||||
provider = Provider(name="t", kind="anthropic", api_key="test-only", models=[],
|
||||
extra=spec_kw.pop("extra", {}) or {})
|
||||
return build_chat_model(provider, ModelSpec(model=model, **spec_kw))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user