diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..1bd66e85 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,79 @@ +name: CI + +on: + push: + branches: + - main + - master + - 'feat/**' + - 'fix/**' + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + backend: + name: Backend Tests & Docs + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + cache-dependency-path: | + backend/requirements.lock + backend/pyproject.toml + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r backend/requirements.lock + python -m pip install -e "backend[dev,docs]" + + - name: Check bilingual docs & links + run: python scripts/check-docs.py + + - name: Run backend test suite + run: python -m pytest backend/tests -n auto -q + + frontend: + name: Frontend Build & Static Gates + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Setup pnpm + uses: pnpm/action-setup@v4 + with: + version: 12.3.4 + + - name: Setup Node 22 + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'pnpm' + cache-dependency-path: frontend/pnpm-lock.yaml + + - name: Install frontend dependencies + run: pnpm --dir frontend install --frozen-lockfile + + - name: Typecheck & build frontend bundle + run: pnpm --dir frontend build + + - name: Check shell script syntax + run: bash -n scripts/package-docker.sh + + - name: Run static & runtime unit checks (tokens, copy, trace, decode) + run: node scripts/check-all.mjs tokens copy trace decode diff --git a/.gitignore b/.gitignore index 840c4dd0..581d0ccb 100644 --- a/.gitignore +++ b/.gitignore @@ -19,10 +19,11 @@ dist/ .env.* !.env.example -# 编辑器 / 系统 +# 编辑器 / 系统 / AI 助手本地工作区 .DS_Store .idea/ .vscode/ +.claude/ *.swp # 已迁到 conda，不再用 uv diff --git a/backend/app/analysis/__init__.py b/backend/app/analysis/__init__.py index 472852b0..4dba0e32 100644 --- a/backend/app/analysis/__init__.py +++ b/backend/app/analysis/__init__.py @@ -41,7 +41,7 @@ ParentResolver = Callable[[str], AnalysisResult | None] #: （row_out_of_range、headline_empty）；要比的组这期没有（unknown_group、too_few_groups）；要用的那一格这期为空 #: （empty_cell：乘积拆解的乘积、因子，差异检验的计数） DATA_ERRORS = frozenset({"parent_refused", "parent_key_missing", "period_not_found", "row_out_of_range", - "headline_empty", "unknown_group", "too_few_groups", "empty_cell"}) + "headline_empty", "unknown_group", "too_few_groups", "empty_cell", "too_many_rows"}) def _op_mode(request: Mapping[str, Any]) -> tuple[str, str | None]: diff --git a/backend/app/analysis/decompose.py b/backend/app/analysis/decompose.py index a1933a2b..3a25accf 100644 --- a/backend/app/analysis/decompose.py +++ b/backend/app/analysis/decompose.py @@ -724,11 +724,13 @@ def _half_ulp_ratio(raw: Any, value: Fraction) -> Fraction: def lmdi(base: Sequence[float], target: Sequence[float], m_base: float, m_target: float) -> list[float]: """LMDI-I：contrib_k = L(M', M) · ln(F_k' / F_k)，L(a, b) = (a − b) / (ln a − ln b)，L(a, a) = a。因子按书写精度取过整、 乘积和乘积列差一点时，各因子按同一比例缩放到乘积恰好等于乘积列（每个因子的对数比各补上 (1/K)·差的对数），Σcontrib 在 - 浮点误差内等于 M' − M。""" + 浮点误差内等于 M' − M。全程在对数域求和，避免多因子连乘 float64 上溢或下溢归零。""" k = len(base) - fix = (math.log(m_target / math.prod(target)) - math.log(m_base / math.prod(base))) / k - weight = m_base if m_base == m_target else (m_target - m_base) / (math.log(m_target) - math.log(m_base)) - return [weight * (math.log(t / b) + fix) for b, t in zip(base, target)] + log_m_base = math.log(m_base) + log_m_target = math.log(m_target) + fix = ((log_m_target - sum(math.log(t) for t in target)) - (log_m_base - sum(math.log(b) for b in base))) / k + weight = m_base if m_base == m_target else (m_target - m_base) / (log_m_target - log_m_base) + return [weight * (math.log(t) - math.log(b) + fix) for b, t in zip(base, target)] def factor(raw: Mapping[str, Any], env: Env) -> Any: diff --git a/backend/app/analysis/differ.py b/backend/app/analysis/differ.py index 1b959b3b..abc97756 100644 --- a/backend/app/analysis/differ.py +++ b/backend/app/analysis/differ.py @@ -38,6 +38,7 @@ from app.analysis.inputs import Snap, as_list, hashable, same_value from app.analysis.numbers import canon, to_fraction TESTS = {"z": "z 检验", "fisher": "Fisher 精确检验", "welch": "Welch t 检验"} +MAX_ROWS_MODE = 1000 def used_settings(mode: str | None) -> list[str]: @@ -214,6 +215,13 @@ def differ(raw: Mapping[str, Any], env: Env) -> Any: env.inputs.use(snap, "data", [group_col, *cols.values()]) return refuse(env, params, used, "truncated", f"{snap.ref} 只取回了前 {len(snap.rows)} 行，各组不完整", criterion=criterion) + if mode == "rows" and len(snap.rows) > MAX_ROWS_MODE: + raise CallError( + "too_many_rows", + f"{snap.ref} 有 {len(snap.rows)} 行，超过逐行明细模式 {MAX_ROWS_MODE} 行的上限：" + f"请在 SQL 里按 {group_col} 汇总出 SUM、SUM(x*x)、COUNT(*)，改用 mean 模式", + reason=f"{snap.ref} 有 {len(snap.rows)} 行，超过逐行明细 {MAX_ROWS_MODE} 行的上限", + ) groups = _groups(snap, group_col, cols, mode, by_period, params) # type: ignore[arg-type] pairs = _pairs(raw, params, groups, mode, dim_label, by_period, group_col) # type: ignore[arg-type] env.inputs.use(snap, "data", [group_col, *cols.values()], diff --git a/backend/app/analysis/periods.py b/backend/app/analysis/periods.py index fd29fab1..7a981a50 100644 --- a/backend/app/analysis/periods.py +++ b/backend/app/analysis/periods.py @@ -286,6 +286,12 @@ def plain_day(sql: str, column: str, dialect: str | None) -> bool: if isinstance(value, (exp.Date, exp.TsOrDsToDate)): extra = [v for k, v in value.args.items() if k != "this" and v not in (None, [], False)] value = cf._unwrap(value.this) if not extra else None + elif isinstance(value, (exp.Cast, exp.TryCast)): + to_type = value.args.get("to") + if isinstance(to_type, exp.DataType) and to_type.this == exp.DataType.Type.DATE: + value = cf._unwrap(value.this) + else: + value = None return isinstance(value, exp.Column) and not isinstance(value.this, exp.Star) diff --git a/backend/app/analysis/stats.py b/backend/app/analysis/stats.py index 79780b22..1d48ac78 100644 --- a/backend/app/analysis/stats.py +++ b/backend/app/analysis/stats.py @@ -341,12 +341,33 @@ def _inversion_counts(n: int) -> list[int]: return counts +_MAX_SCALE_BITS = 1024 + + +def _safe_float(value: Any) -> float: + try: + return float(value) + except OverflowError: + return math.inf if value > 0 else -math.inf + + +def _safe_ratio_float(num: int, den: int) -> float: + try: + return num / den + except OverflowError: + return math.inf if (num > 0) == (den > 0) else -math.inf + + def _scaled(values: Sequence[Any]) -> tuple[list[int], int] | None: - """一组精确的数（整数、Fraction）乘上公分母换成整数：(整数, 公分母)。有浮点的为 None。两两相减、比大小在整数上做， - 比 Fraction 快得多，结果一样。""" + """一组精确的数（整数、Fraction）乘上公分母换成整数：(整数, 公分母)。有浮点或公分母位宽过大的为 None。两两相减、比大小 + 在整数上做，比 Fraction 快得多，结果一样。""" if not all(isinstance(v, (int, Fraction)) and not isinstance(v, bool) for v in values): return None - scale = math.lcm(*(Fraction(v).denominator for v in values)) if values else 1 + scale = 1 + for v in values: + scale = math.lcm(scale, Fraction(v).denominator) + if scale.bit_length() > _MAX_SCALE_BITS: + return None return [int(Fraction(v) * scale) for v in values], scale @@ -392,7 +413,7 @@ def theil_sen(xs: Sequence[Any], ys: Sequence[Any]) -> Any: pairs.append((num, den) if den > 0 else (-num, -den)) if not pairs: return None - keys = [num / den for num, den in pairs] + keys = [_safe_ratio_float(num, den) for num, den in pairs] order = sorted(range(len(pairs)), key=keys.__getitem__) def kth(k: int) -> Fraction: @@ -421,7 +442,7 @@ _FAST_MEDIAN = 512 def _kth(values: Sequence[Any], k: int) -> Any: """从小到大第 k 个（从 0 数），精确：Fraction 换成浮点是正确舍入的、单调的——浮点小的精确值一定小，排错的只可能是浮点 相等的那几个，所以先按浮点排，再只把和第 k 个浮点相等的那一段按精确值排。""" - keyed = sorted((float(v), i) for i, v in enumerate(values)) + keyed = sorted((_safe_float(v), i) for i, v in enumerate(values)) pivot = keyed[k][0] lo, hi = k, k + 1 while lo > 0 and keyed[lo - 1][0] == pivot: diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index 1c655746..beb5f63e 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -149,8 +149,6 @@ logger = logging.getLogger(__name__) SCHEMA = "agentlab.evidence/1" -RUN_NOT_FOUND = "run_not_found" -REPORT_NOT_FOUND = "evidence_report_not_found" SEGMENT_NOT_FOUND = "evidence_segment_not_found" DOC_MISSING = "evidence_doc_missing" DOC_TAMPERED = "evidence_doc_tampered" @@ -159,17 +157,26 @@ LEGACY_NOTE = "旧版出具：按数值匹配，不是显式引用。同一个 NONE_NOTE = "本次运行没有报告文档，也没有出具契约，没有可展示的证据" NO_DOC_NOTE = "本次运行的出具契约采用引用模式核对，但报告没有产出可核对的文档" -from app.engine.evidence_reader import Sealed as _Sealed, metric_cards as _sealed_cards, read_sealed +from app.engine.evidence_reader import ( + MASKED, + REPORT_NOT_FOUND, + RUN_NOT_FOUND, + Report as _Report, + Sealed as _Sealed, + ShownText as _Shown, + choose_report as _choose, + load_sealed, + metric_cards as _sealed_cards, + output_evidence as _output_evidence, + read_sealed, + reports_of as _reports, +) async def _sealed(run_id: str) -> _Sealed: - try: - return await read_sealed(run_id) - except LookupError: - raise CodedHTTPException(404, "运行记录不存在", RUN_NOT_FOUND) from None + return await load_sealed(run_id) -MASKED = "已遮罩" #: 面板底下的遮罩说明。遮的列可能是遮罩列本身，也可能是 SQL 里给它起了别名、套了函数得来的结果列（c.phone AS tel） MASK_NOTE = "这些列取自数据源中设置了遮罩的列，面板上不显示原值。遮罩只减少暴露，不是安全边界" #: 遮没遮核对不了、不显示的值（指标步骤的值、推演的原值）：和「已遮罩」分开，读的人不该以为这里设了遮罩 @@ -198,66 +205,10 @@ MISSING_RETRIEVAL = "无法读取检索快照，或其中没有这条命中" # -------------------------------------------------------------------------- -# 报告文档 +# 报告文档（Report / reports_of / output_evidence 定义在 app/engine/evidence_reader.py） # -------------------------------------------------------------------------- -@dataclass -class _Report: - node_id: str - doc_artifact: str - ok: bool | None - repairs: int | None - stats: dict[str, Any] | None - doc: dict[str, Any] | None - #: True 取回并复验通过；False 内容和哈希对不上（或者不是这次运行这个节点写的）；None 文件不在了 - hash_ok: bool | None - fields: list[str] - #: 写这份报告时的结论句策略（report.checked 里记的，升级前的运行没有） - claims: str | None = None - - -def _reports(sealed: _Sealed, *, nested: bool = False) -> list[_Report]: - """封存范围内每个报告节点最后一次的自查结果，按第一次出现的顺序。 - - 默认只认这次运行的图上的报告节点：子工作流里的报告（节点 id「子工作流节点 id/…」，子图的事件和父运行在同一个流里） - 是子工作流那一步内部的东西，父图的答案是子工作流出口的值，不是那份报告本身——证据图、取答案的评测、按成果挑报告 - （_choose）都按父图自己的认。nested 为真时连子工作流里的一起给（按 ?report=sub/write 点开它们）。可验证报告另按全部 - report.checked 核对（bundle/build._report_events：自查过的文档一份都不能少）。""" - latest: dict[str, dict[str, Any]] = {} - for _, etype, node_id, data in sealed.events: - if etype == "report.checked" and node_id and isinstance(data, dict) and data.get("doc_artifact") \ - and (nested or "/" not in node_id): - latest[node_id] = data - evidence = _output_evidence(sealed) - out = [] - for node_id, data in latest.items(): - artifact = str(data["doc_artifact"]) - try: - doc = load(artifact) - hash_ok: bool | None = True if doc is not None else None - except ValueError: - doc, hash_ok = None, False - # 文档要真是这次运行里这个节点写的：内容寻址只保证没被改，不保证没被张冠李戴 - if isinstance(doc, dict) and (doc.get("node_id") != node_id or doc.get("run_id") != sealed.run_id): - doc, hash_ok = None, False - fields = [f for r in evidence if r.get("report_node") == node_id and r.get("doc_artifact") == artifact - for f in r.get("fields") or []] - out.append(_Report(node_id=node_id, doc_artifact=artifact, ok=data.get("ok"), repairs=data.get("repairs"), - stats=(doc or {}).get("stats") or data.get("stats"), - doc=doc if isinstance(doc, dict) else None, hash_ok=hash_ok, fields=fields, - claims=data.get("claims") if isinstance(data.get("claims"), str) else None)) - return out - - -def _output_evidence(sealed: _Sealed) -> list[dict[str, Any]]: - output = sealed.payload_of(sealed.nodes_of("output")) or {} - primary = output.get("_evidence") - if not isinstance(primary, dict): - return [] - return [primary, *[r for r in primary.get("others") or [] if isinstance(r, dict)]] - - def _input_payloads(sealed: _Sealed) -> dict[str, Any]: return sealed.payload_of(sealed.nodes_of("input")) or {} @@ -830,16 +781,6 @@ def _named_reports(sealed: _Sealed, wanted: str | None) -> list[_Report]: return _reports(sealed, nested=bool(wanted) and "/" in str(wanted)) -def _choose(reports: list[_Report], wanted: str | None, sealed: _Sealed) -> _Report: - if wanted: - chosen = next((r for r in reports if r.node_id == wanted), None) - if chosen is None: - raise CodedHTTPException(404, "本次运行中没有所选报告撰写节点的文档", REPORT_NOT_FOUND) - return chosen - primary = next(iter(_output_evidence(sealed)), {}) - return next((r for r in reports if r.node_id == primary.get("report_node")), reports[0]) - - async def _chain(seg: dict[str, Any], doc: dict[str, Any], sealed: _Sealed, report_node: str) -> list[dict[str, Any]]: cite = seg.get("cite") or {} @@ -2977,32 +2918,7 @@ def _no_data_view(snap: dict[str, Any]) -> list[dict[str, Any]]: if isinstance(item, dict) and isinstance(item.get("message"), str)] -class _Shown: - """查询步骤上要展示的 SQL、合并 SQL、SQL 检查的片段、合并警告：里面写着的遮罩值（secrets，这次运行查到过的， - masking._Secrets）换成「已遮罩」，换了写法的长号码也认。secrets 为 None 时原样交回。changed：换过东西，查询步骤 - 底下要说清这只是展示时遮的。""" - - def __init__(self, secrets: Any) -> None: - self._secrets = secrets - self.changed = False - - def text(self, value: Any) -> Any: - if self._secrets is None or not isinstance(value, str) or not value: - return value - out = self._secrets.scrub(value, loose=True) - self.changed = self.changed or out != value - return out - - def checks(self, checks: list[dict[str, Any]]) -> list[dict[str, Any]]: - return [{**c, **{k: self.text(c[k]) for k in ("message", "sql_excerpt") if isinstance(c.get(k), str)}} - for c in checks] - - def merge(self, view: dict[str, Any]) -> dict[str, Any]: - view["sql"] = self.text(view.get("sql")) - view["warnings"] = [{**w, "message": self.text(w.get("message"))} for w in view.get("warnings") or []] - view["inputs"] = [{**i, **({"checks": self.checks(i["checks"])} if i.get("checks") else {})} - for i in view.get("inputs") or []] - return view +# _Shown 定义在 app.engine.evidence_reader.ShownText 并在上方导入为 _Shown def _has_values(snap: dict[str, Any]) -> bool: diff --git a/backend/app/api/knowledge.py b/backend/app/api/knowledge.py index 0f2bb700..f05093c1 100644 --- a/backend/app/api/knowledge.py +++ b/backend/app/api/knowledge.py @@ -9,6 +9,7 @@ from fastapi import ( from pydantic import BaseModel, Field from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import defer from app.core.errors import explain, raw from app.core.config import settings @@ -221,7 +222,12 @@ async def list_documents( collection: str | None = None, session: AsyncSession = Depends(get_session), ) -> list[Document]: - stmt = select(Document).order_by(Document.created_at.desc(), Document.id.desc()).limit(300) + stmt = ( + select(Document) + .options(defer(Document.content)) + .order_by(Document.created_at.desc(), Document.id.desc()) + .limit(300) + ) if collection: stmt = stmt.where(Document.collection == collection) return list((await session.execute(stmt)).scalars()) diff --git a/backend/app/api/ledger.py b/backend/app/api/ledger.py index 4b0f9b88..4a6e9d35 100644 --- a/backend/app/api/ledger.py +++ b/backend/app/api/ledger.py @@ -1387,11 +1387,9 @@ async def _draft_validation_summary(session: AsyncSession, draft: CaliberDraft, def _graph_entry_fields(graph: dict[str, Any]) -> list[dict[str, Any]]: """图里入口节点（input）的字段；草稿页和试跑用它，画布改过的入口字段不会被 payload 的旧快照盖住。""" - from app.apps.tree import node_map + from app.ledger.fieldbaseline import graph_entry_fields - start = next((node for node in node_map(graph).values() if node.get("type") == "input"), None) - return [dict(field) for field in ((start or {}).get("config") or {}).get("fields") or [] - if isinstance(field, dict)] + return graph_entry_fields(graph) def _allowed_tables(source: DataSource | None) -> set[str]: @@ -1503,54 +1501,19 @@ async def _draft_graph_payload(session: AsyncSession, workflow: Workflow, versio card_id 是这次要读的那张口径卡（写入时由 target.card_id 定）；不给就按这张草稿的权威卡（draft_card）。 """ - from app.apps.tree import node_map + from app.ledger.fieldbaseline import project_graph row = (await session.execute(select(WorkflowVersion).where( WorkflowVersion.workflow_id == workflow.id, WorkflowVersion.version == int(version)))).scalar_one_or_none() if row is None or not isinstance(row.graph, dict): return None - nodes = node_map(row.graph) - wanted, cards = draft_card(row.graph) - # 读的时候认不出唯一的权威卡（多张、契约也没说是哪张）就照旧显示第一张：显示不改任何东西，写入那条路上会拒 - wanted = card_id or wanted or (cards[0]["id"] if cards else None) - card = nodes.get(wanted) if wanted else None - metrics = [metric for metric in ((card or {}).get("config") or {}).get("metrics") or [] - if isinstance(metric, dict) and metric.get("id")] - fields = _graph_entry_fields(row.graph) - fetches: list[tuple[str, str]] = [] - for node_id, node in nodes.items(): - config = node.get("config") if isinstance(node.get("config"), dict) else {} - args = config.get("args") if isinstance(config.get("args"), dict) else {} - sql = args.get("sql") if isinstance(args, dict) else None - if node.get("type") == "tool" and isinstance(sql, str) and sql.strip(): - fetches.append((str(node_id), sql)) - groups: list[dict[str, Any]] = [] - for node_id, sql in fetches: - pattern = re.compile(rf"nodes\.{re.escape(node_id)}(?![\w])") - from app.look.calibers import _metric_paths - - ids = [str(metric["id"]) for metric in metrics - if any(pattern.search(path) for path in _metric_paths(metric) or [])] - groups.append({"sql": sql, "entry_fields": [dict(field) for field in fields], "metric_ids": ids}) - assigned = {metric_id for group in groups for metric_id in group["metric_ids"]} - loose = [str(metric["id"]) for metric in metrics if str(metric["id"]) not in assigned] - if loose: - if groups: - groups[0]["metric_ids"].extend(loose) - else: - groups = [{"sql": "", "entry_fields": [dict(field) for field in fields], "metric_ids": loose}] - out = { - "version": version, - "metrics": metrics, - "entry_fields": [dict(field) for field in fields], - "sql": "\n\n".join(sql for _node, sql in fetches), - "groups": groups, - "blocks": len(fetches) or 1, - } + out = project_graph(row.graph, version, card_id=card_id) if draft is not None and payload is not None: out["trial"] = await _trial_view(session, payload.get("trial") or {}, version=version, graph=row.graph) - out["checks"] = await _draft_checks(session, draft, {**payload, "trial": out["trial"]}, row.graph, metrics, groups) + out["checks"] = await _draft_checks( + session, draft, {**payload, "trial": out["trial"]}, row.graph, out["metrics"], out["groups"], + ) return out @@ -1577,46 +1540,7 @@ CHANGE_UNAVAILABLE = "change_unavailable" DRAFT_CARD_GONE = "这份草稿的口径卡不见了（可能刚被删掉）：请刷新页面再看一次" -def editable_cards(graph: dict[str, Any]) -> list[dict[str, str]]: - """图里能写的一张张口径卡：[{id, title}]，按节点出现的顺序。 - - 能写的是源头口径卡（metrics 节点、没有 caliber_from）：转抄别处的口径卡不是权威写入源（A01），改它等于在别处 - 另存一份定义。 - """ - from app.apps.tree import node_map - - return [{"id": node_id, "title": str(node.get("title") or node_id)} - for node_id, node in node_map(graph).items() - if node.get("type") == "metrics" and not (node.get("config") or {}).get("caliber_from")] - - -def contract_card(graph: dict[str, Any]) -> str | None: - """下游契约声明从哪张卡取指标（metrics_from 只写了一张时就是它）：多张卡时按它认这张草稿的权威卡。""" - for node in graph.get("nodes") or []: - config = ((node or {}).get("data") or {}).get("config") if isinstance(node, dict) else None - contract = config.get("contract") if isinstance(config, dict) else None - if not isinstance(contract, dict): - continue - sources = contract.get("metrics_from") - ids = [sources] if isinstance(sources, str) else [s for s in sources or [] if isinstance(s, str)] - if len(ids) == 1: - return ids[0] - return None - - -def draft_card(graph: dict[str, Any]) -> tuple[str | None, list[dict[str, str]]]: - """这份草稿的权威口径卡，另交回全部能写的卡。 - - 一张能写的卡时就是它；多张时认下游契约声明的那一张——不是「第一张」。都认不出来时交回 None：写的时候必须显式 - 给 target.card_id，读的时候页面也照实列候选。 - """ - cards = editable_cards(graph) - if len(cards) == 1: - return cards[0]["id"], cards - referenced = contract_card(graph) - if referenced and any(card["id"] == referenced for card in cards): - return referenced, cards - return None, cards +from app.ledger.fieldbaseline import contract_card, draft_card, editable_cards def _target_card(payload: "DraftPatchIn", workflow: Workflow) -> str | None: diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index 4712bc8d..3f99453f 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -12,6 +12,7 @@ from fastapi import APIRouter, Depends, Header, HTTPException, Query, WebSocket, from pydantic import BaseModel, Field from sqlalchemy import or_, select from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import defer from app.api.coded import CodedHTTPException # 「人」的唯一入口搬到了 api/deps.py（开登录时返回认证过的用户名）。别处照旧从这里导入 actor_of @@ -422,7 +423,7 @@ async def list_runs( 没按来源筛时，系统派生的运行（评测、口径对账、口径召回：engine/origin.SYSTEM_ORIGINS）默认不列——几十道评测题的运行 会把真实用户的运行挤出第一页；按来源点名就列。没列出几次由 GET /api/runs/unlisted 照实说，界面写在列表底下。 """ - stmt = _filtered(select(Run), workflow_id=workflow_id, status=status, run_class=run_class, q=q, + stmt = _filtered(select(Run).options(defer(Run.graph)), workflow_id=workflow_id, status=status, run_class=run_class, q=q, schedule_id=schedule_id, origin_kind=origin_kind) stmt = stmt.order_by(Run.created_at.desc(), Run.id.desc()).limit(limit) if before is not None: @@ -817,8 +818,25 @@ async def delete_run( "这是一次已封存的正式运行，是出具结果的追溯凭证。删除后其事件和工件将一并删除，" "封存清单无法再核对。如仍要删除，请选择「强制删除」", ) + from sqlalchemy import delete as sql_delete + + from app.core import artifact_store + from app.db.models import Artifact + from app.engine import checkpoints + from app.files import store as files_store + + artifact_ids = list((await session.execute(select(Artifact.id).where(Artifact.run_id == run_id))).scalars()) + if artifact_ids: + await session.execute(sql_delete(Artifact).where(Artifact.run_id == run_id) + .execution_options(synchronize_session=False)) + released_shas = await files_store.release_runs(session, [run_id]) await session.delete(run) await session.commit() + if artifact_ids: + await artifact_store.forget(artifact_ids) + if released_shas: + await files_store.drop_unreferenced(released_shas) + await checkpoints.delete_threads([run_id]) # -------------------------------------------------------------------------- diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index 601d5b4b..e5398a08 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -24,6 +24,16 @@ from app.engine.judge import SETTING_GROUP as JUDGE_GROUP, SETTING_LABEL as JUDG from app.engine.judge import daily_spend, judge_model_spec, judge_settings from app.engine.judge import sanitize_settings as judge_values, settings_problem as judge_problem from app.engine.spend import SPEND_KEY as VERIFY_SPEND, daily_spend as verify_daily_spend +from app.ledger.settings import ( + LEDGER_DEFAULTS, + LEDGER_GROUP, + LEDGER_RECALL_MAX, + LEDGER_SETTING_LABEL, + LEDGER_TIMEOUT_RANGE, + ledger_problem, + ledger_settings, + ledger_values, +) from app.providers import catalog from app.providers.factory import ( ModelSpec, ProviderNotConfigured, _resolve_key, assistant_unavailable, build_chat_model, @@ -736,72 +746,9 @@ async def apps_settings(session: AsyncSession) -> dict[str, Any]: # -------------------------------------------------------------------------- -# 口径账本：按口径自动正式出具的开关、每日上限，口径比对的时限（docs/design/next-round/02-口径账本.md 做法 5、2） +# 口径账本：默认项、范围与校验定义在 app/ledger/settings.py（此处直接透出供设置接口使用） # -------------------------------------------------------------------------- -#: 设置里 ledger 这一组。auto_issue：问数据的答案整题等价于受管口径（Agent 独立写的 SQL 被证明等价、版本受管、图里没有人工 -#: 节点和写入）时自动按口径发起一次正式运行（拍板 L1；不满足的只给一键）。issue_daily_limit：每天最多按口径正式出具几次 -#: （自动的、一键的合计，0 表示不再出具）：正式运行含报告撰写的模型费用。match_timeout_s：一次运行的口径比对最多算几秒， -#: 超时的照实记没比完（ledger/match.py）。recall_limit：口径发布新版本后，一次最多自动重算几组受影响的答案（02 做法 7、拍板 L6， -#: 缺省 50；超出的照实写「另有 M 组未重算」，可以手动接着算；0 表示发布后不自动重算，只列出受影响的答案）。只有管理员能改 -#: （整个设置接口的改动都要管理员） -LEDGER_GROUP = "ledger" -LEDGER_DEFAULTS: dict[str, Any] = {"auto_issue": True, "issue_daily_limit": 20, "match_timeout_s": 8, "recall_limit": 50} -LEDGER_SETTING_LABEL = {"auto_issue": "自动按口径正式出具", "issue_daily_limit": "按口径正式出具每日上限", - "match_timeout_s": "口径比对时限（秒）", "recall_limit": "口径升版后每次最多重算几组"} -#: 一次发布最多自动重算几组：召回只跑口径的数据部分、不调模型，但每组都查业务库 -LEDGER_RECALL_MAX = 1000 -#: 时限的范围：太短比不完一份口径，太长占着比对的线程 -LEDGER_TIMEOUT_RANGE = (1, 60) - - -def _timeout_ok(value: Any) -> bool: - return isinstance(value, (int, float)) and not isinstance(value, bool) \ - and LEDGER_TIMEOUT_RANGE[0] <= value <= LEDGER_TIMEOUT_RANGE[1] - - -def _recall_limit_ok(value: Any) -> bool: - return isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= LEDGER_RECALL_MAX - - -def ledger_problem(value: Any) -> str | None: - """设置页写进来的 ledger 这一组有什么问题，没问题返回 None。""" - if not isinstance(value, dict): - return f"「口径账本」设置格式有误，需要是一个对象，当前为「{value}」" - for key, item in value.items(): - if key not in LEDGER_DEFAULTS: - return (f"「口径账本」设置中不支持「{key}」，可设置的项：" - + "、".join(f"「{LEDGER_SETTING_LABEL[k]}」" for k in LEDGER_DEFAULTS)) - if key == "auto_issue" and not isinstance(item, bool): - return f"「{LEDGER_SETTING_LABEL[key]}」只能是开或关，当前为「{item}」" - if key == "issue_daily_limit" and not _daily_limit_ok(item): - return (f"「{LEDGER_SETTING_LABEL[key]}」需要是 0 到 {APPS_DAILY_LIMIT_MAX:,} 之间的整数（0 表示不再按口径出具），" - f"当前为「{item}」") - if key == "recall_limit" and not _recall_limit_ok(item): - return (f"「{LEDGER_SETTING_LABEL[key]}」需要是 0 到 {LEDGER_RECALL_MAX:,} 之间的整数（0 表示发布后不自动重算），" - f"当前为「{item}」") - if key == "match_timeout_s" and not _timeout_ok(item): - return (f"「{LEDGER_SETTING_LABEL[key]}」需要是 {LEDGER_TIMEOUT_RANGE[0]} 到 {LEDGER_TIMEOUT_RANGE[1]} 之间的数，" - f"当前为「{item}」") - return None - - -def ledger_values(stored: Any) -> dict[str, Any]: - """库里存的（或刚写进来的）ledger 一组 → 补齐默认；存坏了的项按默认（自动出具的开关存坏了按关，宁可少出具）。""" - stored = stored if isinstance(stored, dict) else {} - auto, limit, timeout = stored.get("auto_issue"), stored.get("issue_daily_limit"), stored.get("match_timeout_s") - recall = stored.get("recall_limit") - return {"auto_issue": auto if isinstance(auto, bool) else (LEDGER_DEFAULTS["auto_issue"] if auto is None else False), - "issue_daily_limit": limit if _daily_limit_ok(limit) else LEDGER_DEFAULTS["issue_daily_limit"], - "match_timeout_s": timeout if _timeout_ok(timeout) else LEDGER_DEFAULTS["match_timeout_s"], - "recall_limit": recall if _recall_limit_ok(recall) else LEDGER_DEFAULTS["recall_limit"]} - - -async def ledger_settings(session: AsyncSession) -> dict[str, Any]: - """设置里的 ledger 一组，没存过的补默认。按口径出具、比对从这里读。""" - row = await session.get(Setting, LEDGER_GROUP) - return ledger_values(row.value if row else None) - # -------------------------------------------------------------------------- # 评测：并发、每题时限、单次和每日金额上限、保留几次、结论裁判的缺省（docs/design/next-round/01-效果度量与回归评测.md 做法 17） diff --git a/backend/app/chat/direct.py b/backend/app/chat/direct.py index 9187900d..2dd0829b 100644 --- a/backend/app/chat/direct.py +++ b/backend/app/chat/direct.py @@ -45,14 +45,18 @@ def _shanghai_time_banner() -> str: ) +_METRIC_RULE_LINE = ( + " - 若下方「已命中的标准语义指标卡片」直接对应用户所问指标且口径完整，" + "第 1 轮直接调用 `metric_query`（支持 `date_from`、`date_to`、`group_by_date`、`group_by`、`filters`）拿到数据后立即作答。\n" +) + DIRECT_SYSTEM_PROMPT = """你是 AgentLab 的「快速直查数据分析助手」。你的任务是基于已预命中的指标卡片、标准业务实体与数据目录，以最少的工具轮次、最快速度给出准确、清晰、结构化的 Markdown 数据回答。 【核心取数与防错纪律】 1. **{time_banner}** 2. **严控轮次预算，杜绝多轮逆向摸底（最多 1～2 轮工具调用）**： - 快速直查的目标是秒级直取答案，**严禁**展开多轮跨表试探与字段考古。 - - 若下方「已命中的标准语义指标卡片」直接对应用户所问指标且口径完整，第 1 轮直接调用 `metric_query`（支持 `date_from`、`date_to`、`group_by_date`、`group_by`、`filters`）拿到数据后立即作答。 - - 若第 1 轮返回空集、全空分组或口径明显异常（如底表混存多种记录类型、汇总层级或分类口径而缺少筛选条件），**最多再做 1 轮并行 `db_query__` 补查**（在同一轮内同时核对关联维表或配置表映射并查询过滤后的目标汇总 SQL），拿到结果立即收口作答；若仍存在跨表复杂口径分歧，直接呈现已核实的事实并建议用户点击右上角「切换深度分析」。 +{metric_rule} - 若第 1 轮返回空集、全空分组或口径明显异常（如底表混存多种记录类型、汇总层级或分类口径而缺少筛选条件），**最多再做 1 轮并行 `db_query__` 补查**（在同一轮内同时核对关联维表或配置表映射并查询过滤后的目标汇总 SQL），拿到结果立即收口作答；若仍存在跨表复杂口径分歧，直接呈现已核实的事实并建议用户点击右上角「切换深度分析」。 3. **合并查询，杜绝碎片化往返**： - 手写 `db_query__<源名>` 时，尽量在 **1 条（最多不超过 2 条）SQL** 内一次性查全所需指标与近期对比窗口（例如查「今天怎么样」时，可直接查近 7 天按日汇总 `WHERE stat_date >= '<7日前>' ORDER BY stat_date DESC`，一条 SQL 同时看清今天是否有数、最新到哪一天、以及相比前几天的走势！）。 - 已开启并行工具调用（`parallel_tools=True`）：确实需要同时看汇总与下钻明细、或查两张互不依赖的表时，必须在**同一轮**一次性并行发出工具调用，切勿拆成多轮串行调用。 @@ -83,7 +87,10 @@ def build_direct_graph( tools.append(METRIC_QUERY) tools.extend(["calculator", "kb_search"]) - base_sys = DIRECT_SYSTEM_PROMPT.format(time_banner=_shanghai_time_banner()) + base_sys = DIRECT_SYSTEM_PROMPT.format( + time_banner=_shanghai_time_banner(), + metric_rule=_METRIC_RULE_LINE if has_metrics else "", + ) if bundle.system_context: full_sys = f"{base_sys}\n\n---\n\n{_escape_template_braces(bundle.system_context)}" else: @@ -252,7 +259,7 @@ class DirectChatDriver(ChatDriver): self.grounding_summary = bundle.ui_summary # 2. 构建确定性直查图（含零延迟证据核验）并落库、推送预命中事件 - direct_graph = build_direct_graph(bundle=bundle, has_metrics=True) + direct_graph = build_direct_graph(bundle=bundle, has_metrics=bundle.has_enabled_metrics) if instruction and instruction.strip() and instruction.strip() != self.question.strip(): # 用户点「重试本轮」且仍保持快速直查时，把重试提示附进用户 prompt agent_node = next((n for n in direct_graph["nodes"] if n.get("type") == "agent"), None) diff --git a/backend/app/chat/direct_evidence.py b/backend/app/chat/direct_evidence.py index 41b3dd3d..9e7de913 100644 --- a/backend/app/chat/direct_evidence.py +++ b/backend/app/chat/direct_evidence.py @@ -473,6 +473,38 @@ def _auto_bind_bare_numbers( if not cells: return text + cell_order = {id(c): idx for idx, c in enumerate(cells)} + text_code_by_raw: dict[str, list[_CellCandidate]] = {} + numeric_cells: list[_CellCandidate] = [] + numeric_by_exact: dict[Decimal, list[_CellCandidate]] = {} + for c in cells: + if c.is_text_code: + text_code_by_raw.setdefault(c.raw_text, []).append(c) + else: + numeric_cells.append(c) + numeric_by_exact.setdefault(c.dec_value, []).append(c) + + quant_cache: dict[tuple[str, int], dict[Decimal, list[_CellCandidate]]] = {} + + def _quant_buckets(scale_kind: str, places: int) -> dict[Decimal, list[_CellCandidate]]: + key = (scale_kind, places) + cached = quant_cache.get(key) + if cached is not None: + return cached + buckets: dict[Decimal, list[_CellCandidate]] = {} + quant = Decimal(1).scaleb(-places) + if scale_kind == "pct": + for c in numeric_cells: + q_val = (c.dec_value * Decimal(100)).quantize(quant, rounding=ROUND_HALF_UP) + buckets.setdefault(q_val, []).append(c) + else: + divisor = Decimal(10000) if scale_kind == "万" else Decimal(100_000_000) + for c in numeric_cells: + q_val = (c.dec_value / divisor).quantize(quant, rounding=ROUND_HALF_UP) + buckets.setdefault(q_val, []).append(c) + quant_cache[key] = buckets + return buckets + fences = _fenced_line_ranges(text) lines = text.splitlines(keepends=True) table_headers = _table_headers_by_line(lines) @@ -546,31 +578,31 @@ def _auto_bind_bare_numbers( continue has_leading_zero = bool(_LEADING_ZERO_INT_RE.match(raw_clean)) + places = max(int(tok.decimals or 0), 0) matched: list[tuple[_CellCandidate, str | None]] = [] - for cand in cells: - if has_leading_zero: - if cand.is_text_code and cand.raw_text == raw_clean: - matched.append((cand, None)) - continue - if cand.is_text_code: - if not tok.is_percent and not unit_conv and cand.raw_text == raw_clean: - matched.append((cand, None)) - continue - if unit_conv and unit_scale is not None: - scaled_u = cand.dec_value / unit_scale - places = max(int(tok.decimals or 0), 0) - quant = Decimal(1).scaleb(-places) - if scaled_u.quantize(quant, rounding=ROUND_HALF_UP) == target_dec: - matched.append((cand, unit_conv)) - elif cand.dec_value == target_dec: - matched.append((cand, None)) - elif tok.is_percent: - # 单元格存 0.1258，正文写 12.6% 或 12.58% - scaled = cand.dec_value * Decimal(100) - places = max(int(tok.decimals or 0), 0) - quant = Decimal(1).scaleb(-places) - if scaled.quantize(quant, rounding=ROUND_HALF_UP) == target_dec: - matched.append((cand, "pct")) + if has_leading_zero: + matched = [(c, None) for c in text_code_by_raw.get(raw_clean, ())] + elif unit_conv and unit_scale is not None: + matched = [(c, unit_conv) for c in _quant_buckets(unit_conv, places).get(target_dec, ())] + elif not tok.is_percent: + tc_hits = [(c, None) for c in text_code_by_raw.get(raw_clean, ())] + num_hits = [(c, None) for c in numeric_by_exact.get(target_dec, ())] + if tc_hits and num_hits: + matched = sorted(tc_hits + num_hits, key=lambda pair: cell_order[id(pair[0])]) + else: + matched = tc_hits or num_hits + else: + # 单元格存 12.6（直接相等）或存 0.1258（正文写 12.6% 或 12.58%） + exact_hits = [(c, None) for c in numeric_by_exact.get(target_dec, ())] + pct_hits = [ + (c, "pct") + for c in _quant_buckets("pct", places).get(target_dec, ()) + if c.dec_value != target_dec + ] + if exact_hits and pct_hits: + matched = sorted(exact_hits + pct_hits, key=lambda pair: cell_order[id(pair[0])]) + else: + matched = exact_hits or pct_hits if not matched: continue diff --git a/backend/app/chat/grounding.py b/backend/app/chat/grounding.py index e34f8ad5..4027c228 100644 --- a/backend/app/chat/grounding.py +++ b/backend/app/chat/grounding.py @@ -51,6 +51,7 @@ class GroundingBundle: guardrails: list[dict[str, str]] = field(default_factory=list) system_context: str = "" ui_summary: dict[str, Any] = field(default_factory=dict) + has_enabled_metrics: bool = False def _term_in_text(term: str, text_lower: str, ascii_words: set[str]) -> bool: @@ -395,13 +396,14 @@ async def ground_question( else: narrowed_sources = list(candidate_sources) - # 5. 数据目录 BM25 直选 + 指标底表 + 实体绑定表 -> 组装高信噪比表结构与防错护栏 + # 5. 数据目录 BM25 直选 + 指标底表 + 实体绑定表 -> 跨源全局打分选表，避免首源独占全部槽位 matched_tables: list[dict[str, Any]] = [] guardrails: list[dict[str, str]] = [] table_blocks: list[str] = [] total_detail_chars = 0 - for ds in narrowed_sources: + ds_ctx_list: list[dict[str, Any]] = [] + for ds_idx, ds in enumerate(narrowed_sources): cache = ds.schema_cache if isinstance(ds.schema_cache, Mapping) else {} tables_map = (cache.get("tables") or {}) if isinstance(cache, Mapping) else {} if not isinstance(tables_map, Mapping) or not tables_map: @@ -414,74 +416,112 @@ async def ground_question( facts = await catalog_profile_jobs.last_facts(session, ds.id) except Exception: # noqa: BLE001 facts = {} - - picked_for_ds: list[str] = [] - reasons_for_table: dict[str, str] = {} - - # (a) 命中指标的底表优先入选 - for mm in matched_metrics: - if mm["source_id"] == ds.id: - resolved = catalog.resolve_table_name(cache, mm["table_name"]) - if resolved and resolved not in picked_for_ds: - picked_for_ds.append(resolved) - reasons_for_table[resolved] = f"指标「{mm['name']}」底表" - - # (b) 命中实体的绑定表入选 - for me in matched_entities: - for b in me["bindings"]: - if name_key(b["source_name"]) == name_key(ds.name): - resolved = catalog.resolve_table_name(cache, b["table_name"]) - if resolved and resolved not in picked_for_ds and len(picked_for_ds) < MAX_GROUNDED_TABLES: - picked_for_ds.append(resolved) - reasons_for_table[resolved] = f"实体「{me['label']}」绑定表" - masked = schema_view.masked_set(ds) try: rel_graph = schema_view.relation_graph(ds, raw_notes) except Exception: # noqa: BLE001 rel_graph = {} - def _include_companion_lookup(limit_cap: int) -> list[dict[str, Any]]: - if not picked_for_ds: - return [] - comps = find_companion_lookup_tables( - tables_map, - picked_for_ds, - visible_by_table=visible_by_table, - rel_graph=rel_graph, - masked_set=masked, + ds_ctx_list.append( + { + "idx": ds_idx, + "ds": ds, + "cache": cache, + "tables_map": tables_map, + "raw_notes": raw_notes, + "visible_by_table": visible_by_table, + "facts": facts, + "masked": masked, + "rel_graph": rel_graph, + "picked": [], + "reasons": {}, + } + ) + + total_picked = 0 + + def _pick_table(ctx: dict[str, Any], tname: str, reason: str, *, cap: int = MAX_GROUNDED_TABLES) -> bool: + nonlocal total_picked + if total_picked >= cap or tname in ctx["picked"]: + return False + ctx["picked"].append(tname) + ctx["reasons"][tname] = reason + total_picked += 1 + return True + + # (a) 命中指标的底表跨源最优先入选 + for ctx in ds_ctx_list: + ds = ctx["ds"] + for mm in matched_metrics: + if mm["source_id"] == ds.id: + resolved = catalog.resolve_table_name(ctx["cache"], mm["table_name"]) + if resolved: + _pick_table(ctx, resolved, f"指标「{mm['name']}」底表", cap=MAX_GROUNDED_TABLES + 2) + + # (b) 命中实体的绑定表跨源次优先入选 + for ctx in ds_ctx_list: + ds = ctx["ds"] + for me in matched_entities: + for b in me["bindings"]: + if name_key(b["source_name"]) == name_key(ds.name): + resolved = catalog.resolve_table_name(ctx["cache"], b["table_name"]) + if resolved: + _pick_table(ctx, resolved, f"实体「{me['label']}」绑定表") + + # (c) 各候选源 BM25 检索统一收集并按得分全局排序选表（0 LLM 调用） + if total_picked < MAX_GROUNDED_TABLES: + global_bm25_hits: list[tuple[float, int, int, dict[str, Any], str]] = [] + for ctx in ds_ctx_list: + hits = schema_view.search_tables( + ctx["ds"], + ctx["visible_by_table"], + combined_q, + limit=MAX_GROUNDED_TABLES, ) - for comp in comps: - if comp["col_count"] > 12: - continue - ct_name = comp["table_name"] - if ct_name not in picked_for_ds and len(picked_for_ds) + len(matched_tables) < limit_cap: - picked_for_ds.append(ct_name) - reasons_for_table[ct_name] = "关联维表与字典" + for rank, h in enumerate(hits): + if h.in_scope and h.name not in ctx["picked"]: + global_bm25_hits.append((float(h.score), -ctx["idx"], -rank, ctx, h.name)) + global_bm25_hits.sort(key=lambda item: (item[0], item[1], item[2]), reverse=True) + for _score, _neg_ds_idx, _neg_rank, ctx, tname in global_bm25_hits: + if total_picked >= MAX_GROUNDED_TABLES: + break + _pick_table(ctx, tname, "目录语义检索命中") + + # (d) 若用户明确指定了该小库（<= 5 张表）且尚未命中任何表，直接全给 + for ctx in ds_ctx_list: + ds = ctx["ds"] + if not ctx["picked"] and ds in explicit_sources and len(ctx["tables_map"]) <= 5: + for tname in list(ctx["tables_map"].keys()): + if not _pick_table(ctx, tname, "指定数据源核心表"): break - return comps - - _include_companion_lookup(MAX_GROUNDED_TABLES) - - # (c) BM25 数据目录检索直选（0 LLM 调用） - remaining_slots = MAX_GROUNDED_TABLES - len(matched_tables) - len(picked_for_ds) - if remaining_slots > 0: - hits = schema_view.search_tables(ds, visible_by_table, combined_q, limit=remaining_slots + 2) - for h in hits: - if h.in_scope and h.name not in picked_for_ds and len(picked_for_ds) + len(matched_tables) < MAX_GROUNDED_TABLES: - picked_for_ds.append(h.name) - reasons_for_table[h.name] = "目录语义检索命中" - - # (d) 若用户明确指定了该小库（<= 5 张表）且尚未命中任何表，直接全给 - if not picked_for_ds and ds in explicit_sources and len(tables_map) <= 5: - for tname in list(tables_map.keys())[:MAX_GROUNDED_TABLES - len(matched_tables)]: - picked_for_ds.append(tname) - reasons_for_table[tname] = "指定数据源核心表" - - companion_links = _include_companion_lookup(MAX_GROUNDED_TABLES + 1) + # 按数据源顺序补齐关联维表与一跳连接表，渲染表结构与护栏 + for ctx in ds_ctx_list: + ds = ctx["ds"] + picked_for_ds: list[str] = ctx["picked"] if not picked_for_ds: continue + reasons_for_table: dict[str, str] = ctx["reasons"] + tables_map = ctx["tables_map"] + raw_notes = ctx["raw_notes"] + visible_by_table = ctx["visible_by_table"] + facts = ctx["facts"] + masked = ctx["masked"] + rel_graph = ctx["rel_graph"] + + companion_links = find_companion_lookup_tables( + tables_map, + picked_for_ds, + visible_by_table=visible_by_table, + rel_graph=rel_graph, + masked_set=masked, + ) + for comp in companion_links: + if comp["col_count"] > 12: + continue + ct_name = comp["table_name"] + if _pick_table(ctx, ct_name, "关联维表与字典", cap=MAX_GROUNDED_TABLES + 1): + break # 补一跳连接中间表（若有） try: @@ -546,7 +586,7 @@ async def ground_question( ds_sections.append("表间标准连接条件：\n" + "\n".join(join_lines[:12])) table_blocks.append("\n".join(ds_sections)) - if len(matched_tables) >= MAX_GROUNDED_TABLES: + if len(matched_tables) >= MAX_GROUNDED_TABLES + 1: break elapsed_ms = max(1, int((time.perf_counter() - t0) * 1000)) @@ -719,4 +759,5 @@ async def ground_question( guardrails=guardrails, system_context="\n\n".join(ctx_parts), ui_summary=ui_summary, + has_enabled_metrics=bool(scoped_metrics), ) diff --git a/backend/app/cluster/service.py b/backend/app/cluster/service.py index 2d40ed08..33ff110f 100644 --- a/backend/app/cluster/service.py +++ b/backend/app/cluster/service.py @@ -23,7 +23,7 @@ import time from collections.abc import Awaitable, Callable from typing import Any -from app.cluster import guard, identity, leader, reaper, requests, state, workers +from app.cluster import guard, identity, leader, reaper, requests, slots, state, workers from app.cluster.signal import signals from app.core.config import settings @@ -84,6 +84,7 @@ class _Service: await signals.start() signals.on("q", self._on_request) signals.on("gone", self._on_gone) + signals.on("slot", self._on_slot) if not self.single: await signals.ready(timeout=5.0) self.wake = asyncio.Event() @@ -115,6 +116,9 @@ class _Service: if self.wake is not None: self.wake.set() + def _on_slot(self, _parts: list[str]) -> None: + slots.wake_local() + # ---------------- 续租约：单独的任务、单独的连接 ---------------- async def _lease_conn_open(self) -> Any: diff --git a/backend/app/core/artifact_store.py b/backend/app/core/artifact_store.py index bf5ed29e..52dbdcdb 100644 --- a/backend/app/core/artifact_store.py +++ b/backend/app/core/artifact_store.py @@ -76,31 +76,38 @@ async def put_json( 幂等：同内容重复写只落一次盘；同一工件被多个 run 引用时， 数据库里各记一行引用（溯源需要知道"谁产出的"），文件只有一份。 """ + from sqlalchemy.exc import IntegrityError + + from app.cluster.mutex import lock from app.db.base import SessionLocal from app.db.models import Artifact text = canonical_json(obj) artifact_id = content_hash(text) path = _path_of(artifact_id) - if not path.exists(): - # 原子落盘（core/atomic）：写一半崩了不会留下半个工件；临时文件名带随机后缀，两个进程同时写同一份内容 - # 各写各的，不会一个把另一个写到一半的截断（以前共用 .tmp，后到的会把先到的半成品改名成正式文件） - atomic.write_text(path, text) + async with lock(f"artifact:{artifact_id[:2]}"): + if not path.exists(): + # 原子落盘（core/atomic）：写一半崩了不会留下半个工件；临时文件名带随机后缀，两个进程同时写同一份内容 + # 各写各的，不会一个把另一个写到一半的截断；且与同分桶的 forget 互斥，不会刚写下就被并发回收删掉 + atomic.write_text(path, text) - async with SessionLocal() as session: - existing = await session.get(Artifact, (artifact_id, run_id or "")) - if existing is None: - session.add( - Artifact( - id=artifact_id, - run_id=run_id or "", - kind=kind, - node_id=node_id, - size=len(text.encode()), - meta=meta or {}, + async with SessionLocal() as session: + existing = await session.get(Artifact, (artifact_id, run_id or "")) + if existing is None: + session.add( + Artifact( + id=artifact_id, + run_id=run_id or "", + kind=kind, + node_id=node_id, + size=len(text.encode()), + meta=meta or {}, + ) ) - ) - await session.commit() + try: + await session.commit() + except IntegrityError: + await session.rollback() return artifact_id @@ -148,27 +155,35 @@ async def forget(ids: Any) -> list[str]: 读到错的内容。""" from sqlalchemy import select + from app.cluster.mutex import lock from app.db.base import SessionLocal from app.db.models import Artifact wanted = sorted({i for i in ids or () if isinstance(i, str) and i and i.isalnum()}) if not wanted: return [] - async with SessionLocal() as session: - kept: set[str] = set() - for start in range(0, len(wanted), 500): - chunk = wanted[start:start + 500] - kept |= set((await session.execute(select(Artifact.id).where(Artifact.id.in_(chunk)))).scalars()) - removed: list[str] = [] + by_bucket: dict[str, list[str]] = {} for artifact_id in wanted: - if artifact_id in kept: - continue - path = _path_of(artifact_id) - try: - path.unlink() - except FileNotFoundError: - continue - removed.append(artifact_id) + by_bucket.setdefault(artifact_id[:2], []).append(artifact_id) + + removed: list[str] = [] + for bucket in sorted(by_bucket): + bucket_ids = by_bucket[bucket] + async with lock(f"artifact:{bucket}"): + async with SessionLocal() as session: + kept: set[str] = set() + for start in range(0, len(bucket_ids), 500): + chunk = bucket_ids[start:start + 500] + kept |= set((await session.execute(select(Artifact.id).where(Artifact.id.in_(chunk)))).scalars()) + for artifact_id in bucket_ids: + if artifact_id in kept: + continue + path = _path_of(artifact_id) + try: + path.unlink() + except FileNotFoundError: + continue + removed.append(artifact_id) return removed diff --git a/backend/app/core/bus.py b/backend/app/core/bus.py index 732e21d1..782ce3f0 100644 --- a/backend/app/core/bus.py +++ b/backend/app/core/bus.py @@ -2,7 +2,7 @@ from __future__ import annotations import asyncio import contextlib -from collections import defaultdict +from collections import OrderedDict, defaultdict from collections.abc import Callable from typing import AsyncIterator @@ -11,6 +11,7 @@ from app.core.events import RunEventModel # 单个订阅者的积压上限。慢消费者（比如卡住的浏览器标签）不应该拖垮整个 run， # 所以队列满了就丢最旧的事件并打标记，而不是阻塞发布方。 _QUEUE_MAXSIZE = 2048 +_MAX_SEQ_ENTRIES = 4096 class _Subscriber: @@ -31,17 +32,30 @@ class EventBus: def __init__(self) -> None: self._subs: dict[str, set[_Subscriber]] = defaultdict(set) - self._seq: dict[str, int] = defaultdict(int) + self._seq: OrderedDict[str, int] = OrderedDict() self._lock = asyncio.Lock() self.tap: Callable[[RunEventModel], None] | None = None + def _trim_seq(self) -> None: + while len(self._seq) > _MAX_SEQ_ENTRIES: + oldest, _ = next(iter(self._seq.items())) + if self._subs.get(oldest): + self._seq.move_to_end(oldest) + break + self._seq.popitem(last=False) + def next_seq(self, run_id: str) -> int: - self._seq[run_id] += 1 - return self._seq[run_id] + nxt = self._seq.get(run_id, 0) + 1 + self._seq[run_id] = nxt + self._seq.move_to_end(run_id) + self._trim_seq() + return nxt def set_seq(self, run_id: str, value: int) -> None: """恢复一个已存在的 run 时，把序号接上，保证前端排序不乱。""" - self._seq[run_id] = max(self._seq[run_id], value) + self._seq[run_id] = max(self._seq.get(run_id, 0), value) + self._seq.move_to_end(run_id) + self._trim_seq() async def publish(self, event: RunEventModel, *, relayed: bool = False) -> None: if not relayed and self.tap is not None: @@ -61,6 +75,7 @@ class EventBus: async def close(self, run_id: str) -> None: """给所有订阅者发结束哨兵，让 WebSocket 干净地收尾。""" + self._seq.pop(run_id, None) for sub in list(self._subs.get(run_id, ())): with contextlib.suppress(asyncio.QueueFull): sub.queue.put_nowait(None) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 3fce235d..84a40ff3 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -361,7 +361,10 @@ class Settings(BaseSettings): @lru_cache def get_settings() -> Settings: - s = Settings() + import os + + no_dotenv = os.environ.get("AGENTLAB_DISABLE_DOTENV", "").strip().lower() in ("1", "true", "yes", "on") + s = Settings(_env_file=None) if no_dotenv else Settings() s.ensure_dirs() return s diff --git a/backend/app/data/coverage.py b/backend/app/data/coverage.py index e30761b9..dd04a339 100644 --- a/backend/app/data/coverage.py +++ b/backend/app/data/coverage.py @@ -28,6 +28,7 @@ import asyncio import logging import re import time +from collections import OrderedDict from collections.abc import Awaitable, Callable, Iterable, Mapping from dataclasses import dataclass, field from datetime import date, datetime, timedelta @@ -51,7 +52,9 @@ MAX_COLUMNS = 12 #: 现查到的表的范围在进程里记多久（秒）：同一张表一会儿内查好几次（一张图里几个取数节点、助手自查几轮）不再全表 MIN / #: MAX。滚动表一天才换一次数据，几分钟的旧不影响判断；查不成的、退回剖析的不记 LIVE_TTL_S = 300.0 -_LIVE: dict[tuple[Any, ...], tuple[float, Any]] = {} +SNAPSHOT_LIVE_TTL_S = 86400.0 +_LIVE_MAX = 512 +_LIVE: OrderedDict[tuple[Any, ...], tuple[float, Any]] = OrderedDict() #: 现查没成功（大表全表 MIN / MAX 超时、库一时连不上）也记 LIVE_TTL_S：这段时间里同一张表直接退回剖析、取值观察， #: 不然取数 agent 连查几次、每次结果里有整列 0，每次都要白等 PROBE_TIMEOUT_S 秒才退回 _PROBE_FAILED = object() @@ -68,7 +71,7 @@ NO_DATA_SHORT = "查询区间里这张表没有数据" _COMPARE = {"GT": ">", "GTE": ">=", "LT": "<", "LTE": "<=", "EQ": "="} _FLIP = {">": "<", ">=": "<=", "<": ">", "<=": ">=", "=": "="} _ISO = re.compile(r"^(\d{4})-(\d{1,2})-(\d{1,2})(?:[ T](\d{1,2}):(\d{2})(?::(\d{2})(?:\.\d+)?)?)?" - r"(?:Z|[+-]\d{2}:?\d{2})?$") + r"(?:Z|[+-]\d{2}(?::?\d{2})?)?$") _COMPACT = re.compile(r"^(\d{4})(\d{2})(\d{2})$") @@ -399,14 +402,19 @@ async def _range(source: Any, flt: _Filter, read: str | None, cache: Cache, fact key = (str(getattr(source, "name", "")), name_key(flt.table_name), name_key(flt.column_name)) if key in cache.ranges: return cache.ranges[key] - live = (getattr(source, "id", None), getattr(source, "snapshot_id", None), *key) + snapshot_id = getattr(source, "snapshot_id", None) + live = (getattr(source, "id", None), snapshot_id, *key) held = _LIVE.get(live) failed_before = False - if held is not None and held[0] > time.monotonic(): - if held[1] is not _PROBE_FAILED: - cache.ranges[key] = held[1] - return held[1] - failed_before = True + if held is not None: + if held[0] > time.monotonic(): + _LIVE.move_to_end(live) + if held[1] is not _PROBE_FAILED: + cache.ranges[key] = held[1] + return held[1] + failed_before = True + else: + _LIVE.pop(live, None) table = flt.table.copy() table.set("alias", None) ident = flt.column.this.copy() if isinstance(flt.column.this, exp.Identifier) \ @@ -429,16 +437,18 @@ async def _range(source: Any, flt: _Filter, read: str | None, cache: Cache, fact out = await _fallback(flt, facts) cache.ranges[key] = out if found is None and not failed_before: - _remember(live, _PROBE_FAILED) + _remember(live, _PROBE_FAILED, ttl=LIVE_TTL_S) elif out is not None and out.basis == "live": - _remember(live, out) + ttl = SNAPSHOT_LIVE_TTL_S if snapshot_id else LIVE_TTL_S + _remember(live, out, ttl=ttl) return out -def _remember(live: tuple[Any, ...], value: Any) -> None: - if len(_LIVE) > 512: - _LIVE.clear() - _LIVE[live] = (time.monotonic() + LIVE_TTL_S, value) +def _remember(live: tuple[Any, ...], value: Any, *, ttl: float = LIVE_TTL_S) -> None: + _LIVE.pop(live, None) + while len(_LIVE) >= _LIVE_MAX: + _LIVE.popitem(last=False) + _LIVE[live] = (time.monotonic() + ttl, value) async def _fallback(flt: _Filter, facts: Facts) -> _Range | None: diff --git a/backend/app/data/csv_grid.py b/backend/app/data/csv_grid.py index 7fb8ece7..59aa3f6d 100644 --- a/backend/app/data/csv_grid.py +++ b/backend/app/data/csv_grid.py @@ -110,29 +110,44 @@ class CsvTable: delimiter: str +def _resolve_csv_meta(raw: bytes, delimiter: str | None) -> tuple[str, str, str]: + from app.data import tabular + + enc, errors = tabular._detect_csv_encoding(raw) + if delimiter is None: + with tabular._open_csv_text(raw, encoding=enc, errors=errors, newline="") as probe_stream: + head = probe_stream.read(8192) + try: + delimiter = csv.Sniffer().sniff(head, delimiters=_DELIMITERS).delimiter + except csv.Error: + delimiter = "," # 嗅不出来就按最常见的来 + return enc, errors, delimiter + + +def _iter_csv_records(raw: bytes, *, enc: str, errors: str, delimiter: str) -> Iterator[list[str]]: + from app.data import tabular + + row_count = 0 + try: + with tabular._open_csv_text(raw, encoding=enc, errors=errors, newline="") as stream: + for record in csv.reader(stream, delimiter=delimiter): + if row_count >= MAX_ROWS: + raise UnsupportedTable(f"CSV 文件超过 {MAX_ROWS:,} 行，超出可以导入的上限") + if len(record) > MAX_COLS: + raise UnsupportedTable(f"CSV 文件第 {row_count + 1} 行有 {len(record):,} 列，超过 {MAX_COLS:,} 列的上限") + row_count += 1 + yield record + except csv.Error as e: + raise UnsupportedTable(f"无法按 CSV 读取这个文件：第 {row_count + 1} 行的写法不正确（{e}）。" + "请在 Excel 中重新另存为 CSV 后上传") from e + + def parse(raw: bytes, delimiter: str | None = None) -> CsvTable: """原件 → 记录。编码照 tabular 读 CSV 的做法；delimiter 不给时嗅探（同 tabular）。字段太长、引号不配对读不了时抛 UnsupportedTable。""" - from app.data import tabular - - text = tabular._decode_csv(raw) - if delimiter is None: - try: - delimiter = csv.Sniffer().sniff(text[:8192], delimiters=_DELIMITERS).delimiter - except csv.Error: - delimiter = "," # 嗅不出来就按最常见的来 - records: list[list[str]] = [] - try: - for record in csv.reader(io.StringIO(text, newline=""), delimiter=delimiter): - if len(records) >= MAX_ROWS: - raise UnsupportedTable(f"CSV 文件超过 {MAX_ROWS:,} 行，超出可以导入的上限") - if len(record) > MAX_COLS: - raise UnsupportedTable(f"CSV 文件第 {len(records) + 1} 行有 {len(record):,} 列，超过 {MAX_COLS:,} 列的上限") - records.append(record) - except csv.Error as e: - raise UnsupportedTable(f"无法按 CSV 读取这个文件：第 {len(records) + 1} 行的写法不正确（{e}）。" - "请在 Excel 中重新另存为 CSV 后上传") from e - return CsvTable(records=records, delimiter=delimiter) + enc, errors, resolved_delim = _resolve_csv_meta(raw, delimiter) + records = list(_iter_csv_records(raw, enc=enc, errors=errors, delimiter=resolved_delim)) + return CsvTable(records=records, delimiter=resolved_delim) def _cells(table: CsvTable) -> Iterator[tuple[int, list[tuple[int, GridCell]]]]: @@ -155,15 +170,18 @@ def sheet_delimiter(sheet: Any) -> str | None: def scan(raw: bytes, *, delimiter: str | None = None) -> WorkbookScan: """CSV 原件 → 扫描结果：一张可见工作表（名字 CSV_SHEET、路径 CSV_PATH 加分隔符），边界、非空格数、逐行非空格数的 - 游程。没有合并区、隐藏行列、公式、错误值、离群格（CSV 里每一格都是写出来的，没有残留格式）。""" - table = parse(raw, delimiter) - sheet = SheetScan(name=CSV_SHEET, state="visible", path=CSV_PATH + table.delimiter) + 游程。流式统计，不再先把整张 CSV 物化成全量字符串与二维列表。""" + enc, errors, resolved_delim = _resolve_csv_meta(raw, delimiter) + sheet = SheetScan(name=CSV_SHEET, state="visible", path=CSV_PATH + resolved_delim) lo_r = lo_c = hi_r = hi_c = 0 runs: list[tuple[int, int, int]] = [] - for r, cells in _cells(table): - n = len(cells) + for r, record in enumerate(_iter_csv_records(raw, enc=enc, errors=errors, delimiter=resolved_delim), 1): + nonempty_cols = [c for c, field in enumerate(record, 1) if cell_value(field) is not None] + if not nonempty_cols: + continue + n = len(nonempty_cols) sheet.nonempty += n - first, last = cells[0][0], cells[-1][0] + first, last = nonempty_cols[0], nonempty_cols[-1] if not lo_r: lo_r, lo_c, hi_c = r, first, last lo_c, hi_c, hi_r = min(lo_c, first), max(hi_c, last), r diff --git a/backend/app/data/engine.py b/backend/app/data/engine.py index a5db6903..584f235f 100644 --- a/backend/app/data/engine.py +++ b/backend/app/data/engine.py @@ -14,7 +14,7 @@ import re import sqlite3 import time from collections import OrderedDict -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, suppress from dataclasses import dataclass from typing import Any, AsyncIterator, Awaitable, Callable @@ -297,7 +297,7 @@ def open_checked_sqlite(path: str, *, readonly: bool = True, immutable: bool = F mode = "ro" if readonly else "rwc" extra = "&immutable=1" if immutable else "" return sqlite3.connect(f"file:{quote(os.fspath(path))}?mode={mode}{extra}", uri=True, - factory=_DqsOffConnection) + timeout=5.0, factory=_DqsOffConnection) def open_memory_sqlite() -> sqlite3.Connection: @@ -549,6 +549,8 @@ class EngineCache: #: 正在进行的快照核对，按（缓存键, 文件, 登记的哈希）合并。核对结束（不论成败）即移除， #: 失败结果不留：下次照常重新核对 self._verifying: dict[tuple[str, str, str], asyncio.Future[Fingerprint]] = {} + #: 每个数据源的失效世代：get() 在锁外核对快照期间若赶上 invalidate()，进锁后看到世代变了就重来，绝不把旧连接装回缓存 + self._epochs: dict[str, int] = {} def _hit(self, key: str, doomed: list[AsyncEngine]) -> AsyncEngine | None: """缓存里还可信的 engine。快照文件和核对时的指纹对不上的，摘出来放进 doomed 交给调用方关掉。""" @@ -568,6 +570,7 @@ class EngineCache: async def get(self, source: Any) -> AsyncEngine: key = cache_key(source) + sid = _source_of(key) doomed: list[AsyncEngine] = [] try: cached = self._hit(key, doomed) @@ -577,34 +580,44 @@ class EngineCache: if getattr(source, "immutable", False) and not expected: logger.warning("版本快照没有登记文件哈希，已拒绝：key=%s path=%s", key, source.database) raise SnapshotTampered(_NO_HASH) - fingerprint: Fingerprint | None = None - if expected: - # 在锁外核对：哈希一个大文件要几秒，不能让别的数据源都排在它后面 - fingerprint = await self._verify_once(key, source, expected) - async with self._lock: - cached = self._hit(key, doomed) - if cached is not None: - return cached - url, extra = engine_args(source) - engine = create_async_engine( - url, - pool_size=3, - max_overflow=2, - pool_pre_ping=True, # 长时间空闲后连接会被数据库掐掉，先探活再用 - pool_recycle=1800, - **extra, - ) - if fingerprint is not None: - pin = _Pin(str(source.database or ""), fingerprint) - _guard_pool(engine, pin, key) - self._pins[key] = pin - self._engines[key] = engine - limit = max(1, self._max if self._max is not None else MAX_ENGINES) - while len(self._engines) > limit: - old_key, old = self._engines.popitem(last=False) - self._pins.pop(old_key, None) - doomed.append(old) - return engine + while True: + epoch = self._epochs.get(sid, 0) + fingerprint: Fingerprint | None = None + if expected: + # 在锁外核对：哈希一个大文件要几秒，不能让别的数据源都排在它后面 + fingerprint = await self._verify_once(key, source, expected) + async with self._lock: + cached = self._hit(key, doomed) + if cached is not None: + return cached + if self._epochs.get(sid, 0) != epoch: + continue + url, extra = engine_args(source) + is_memory_sqlite = (source.kind or "").lower() == "sqlite" and ( + (source.database or "") == ":memory:" or url.endswith("/:memory:") + ) + pool_kwargs: dict[str, Any] = {} if is_memory_sqlite else { + "pool_size": 3, + "max_overflow": 2, + "pool_pre_ping": True, # 长时间空闲后连接会被数据库掐掉，先探活再用 + "pool_recycle": 1800, + } + engine = create_async_engine( + url, + **pool_kwargs, + **extra, + ) + if fingerprint is not None: + pin = _Pin(str(source.database or ""), fingerprint) + _guard_pool(engine, pin, key) + self._pins[key] = pin + self._engines[key] = engine + limit = max(1, self._max if self._max is not None else MAX_ENGINES) + while len(self._engines) > limit: + old_key, old = self._engines.popitem(last=False) + self._pins.pop(old_key, None) + doomed.append(old) + return engine finally: for old in doomed: # 正被借出的连接不受影响：dispose 只关池子里空闲的，借出的还回来时随旧池子回收 @@ -653,22 +666,27 @@ class EngineCache: from app.cluster import caches await caches.broadcast("datasource_engine", sid) - # 正在进行的核对也撇开：失效之后来的请求重新核对，不搭失效之前那一次的便车 - for token in [t for t in self._verifying if _source_of(t[0]) == sid]: - del self._verifying[token] - keys = [key for key in list(self._engines) if _source_of(key) == sid] - doomed = [self._engines.pop(key) for key in keys] - for key in keys: - self._pins.pop(key, None) + async with self._lock: + self._epochs[sid] = self._epochs.get(sid, 0) + 1 + # 正在进行的核对也撇开：失效之后来的请求重新核对，不搭失效之前那一次的便车 + for token in [t for t in self._verifying if _source_of(t[0]) == sid]: + del self._verifying[token] + keys = [key for key in list(self._engines) if _source_of(key) == sid] + doomed = [self._engines.pop(key) for key in keys] + for key in keys: + self._pins.pop(key, None) for engine in doomed: await engine.dispose() async def close(self) -> None: - self._verifying.clear() - for engine in list(self._engines.values()): + async with self._lock: + self._verifying.clear() + doomed = list(self._engines.values()) + self._engines.clear() + self._pins.clear() + self._epochs.clear() + for engine in doomed: await engine.dispose() - self._engines.clear() - self._pins.clear() engines = EngineCache() @@ -743,10 +761,20 @@ class _Deadline: def __init__(self, seconds: float) -> None: self.seconds = seconds self.ends = time.monotonic() + seconds - #: SQLite 的进度回调真的掐过 + #: SQLite 的进度回调真的按时限掐过 self.fired = False + #: 调用方主动取消（CancelledError 等），进度回调同样立即中止工作线程里的语句 + self.cancelled = False #: 语句已经结束。回调这时要是还挂在连接上（摘除没成功），也不能再掐别人的语句 self.over = False + #: 由 _server_deadline 登记的底层驱动打断回调（如 sqlite3.Connection.interrupt / oracledb.cancel） + self._aborter: Callable[[], None] | None = None + + def abort(self) -> None: + self.cancelled = True + if self._aborter is not None: + with suppress(Exception): + self._aborter() def remaining_ms(self) -> int: return max(1, int((self.ends - time.monotonic()) * 1000)) @@ -759,7 +787,11 @@ class _Deadline: def sqlite_progress(self) -> int: # 在 SQLite 执行语句的那个线程里被调用；返回非零，SQLite 就中止当前语句 - if not self.over and time.monotonic() >= self.ends: + if self.over: + return 0 + if self.cancelled: + return 1 + if time.monotonic() >= self.ends: self.fired = True return 1 return 0 @@ -843,9 +875,16 @@ async def _server_deadline(conn: AsyncConnection, kind: str, deadline: _Deadline logger.warning("Oracle call_timeout 没设上（%s），只剩后端兜底", e) yield return + cancel_fn = getattr(driver, "cancel", None) or getattr(driver, "breakexecution", None) + if callable(cancel_fn): + deadline._aborter = cancel_fn try: yield + except asyncio.CancelledError: + deadline.abort() + raise finally: + deadline._aborter = None try: driver.call_timeout = before except Exception: # noqa: BLE001 - 超时后连接可能已经被驱动关掉 @@ -854,15 +893,23 @@ async def _server_deadline(conn: AsyncConnection, kind: str, deadline: _Deadline if kind == "sqlite": driver = await _driver(conn) + raw_conn = getattr(driver, "_conn", None) + if raw_conn is not None: + deadline._aborter = raw_conn.interrupt await driver.set_progress_handler(deadline.sqlite_progress, _SQLITE_STEPS) try: yield + except BaseException: + deadline.abort() + raise finally: - deadline.over = True + deadline._aborter = None try: await driver.set_progress_handler(None, 0) - except Exception: # noqa: BLE001 - over 已经让它不再掐人 + except Exception: # noqa: BLE001 - over 保底不再掐后面的语句 pass + finally: + deadline.over = True return yield @@ -887,7 +934,22 @@ async def _bounded( if deadline.expired(): raise SqlRejected(over) async with _server_deadline(conn, kind, deadline): - return await work(conn) + # 用 shield 保护内层 greenlet：收到 CancelledError 时先经 deadline.abort() 通知底层驱动 + # （如 sqlite3_interrupt）正常中止查询，避免 CancelledError 直接打进 SQLAlchemy greenlet + # 触发半关闭连接上的 rollback ValueError("Connection closed") 与连接池泄漏。 + work_task = asyncio.create_task(work(conn)) + try: + return await asyncio.shield(work_task) + except asyncio.CancelledError: + deadline.abort() + try: + await asyncio.wait_for(asyncio.shield(work_task), timeout=1.0) + except BaseException: + if not work_task.done(): + work_task.cancel() + with suppress(BaseException): + await work_task + raise try: return await asyncio.wait_for(_exec(), timeout=limits.timeout_seconds + BACKSTOP_S) diff --git a/backend/app/data/sql_semantics.py b/backend/app/data/sql_semantics.py index dd3ae597..745b4e6e 100644 --- a/backend/app/data/sql_semantics.py +++ b/backend/app/data/sql_semantics.py @@ -8496,47 +8496,182 @@ def _slot_of(expr: str, index: int, context: str) -> TemplateSlot: return TemplateSlot(index=index, field=found.group(1), filters=filters, context=context) -def _template_slots(template: str) -> tuple[list[tuple[int, int, TemplateSlot]], str]: - """模板里每一处 {{ … }}：[(起, 止, 槽位)]，以及按哨兵渲染的 SQL。逐字看引号和注释，定下每一处在哪：单引号里是字， - 双引号、反引号里是名字，-- 和 /* */ 里是注释，别处是裸写。{{ … }} 里面的字不参与判断。""" +_DOLLAR_QUOTE_RE = re.compile(r"\$(?:[^\W\d]\w*)?\$") +_ORACLE_Q_CLOSE = {"[": "]", "(": ")", "{": "}", "<": ">"} + + +def _is_word_char(ch: str) -> bool: + return ch.isalnum() or ch in "_$" + + +def _is_e_string_at(sql: str, i: int) -> bool: + return i >= 1 and sql[i - 1] in "eE" and not (i >= 2 and _is_word_char(sql[i - 2])) + + +def _oracle_q_close_at(sql: str, i: int) -> str | None: + if i + 1 >= len(sql) or sql[i + 1].isspace() or not (i >= 1 and sql[i - 1] in "qQ"): + return None + j = i - 2 + if j >= 0 and sql[j] in "nN": + j -= 1 + if j >= 0 and _is_word_char(sql[j]): + return None + opener = sql[i + 1] + return _ORACLE_Q_CLOSE.get(opener, opener) + + +def _is_bracket_ident_at(sql: str, i: int, dialect_key: str) -> bool: + if dialect_key == "sqlite": + return True + if dialect_key in ("postgres", "postgresql", "mysql", "mariadb", "oracle"): + return False + j = i - 1 + while j >= 0 and sql[j].isspace(): + j -= 1 + return j < 0 or not (_is_word_char(sql[j]) or sql[j] in ")]") + + +def _template_slots( + template: str, *, dialect: str | None = None, +) -> tuple[list[tuple[int, int, TemplateSlot]], str]: + """模板里每一处 {{ … }}：[(起, 止, 槽位)]，以及按哨兵渲染的 SQL。 + + 按方言词法逐字扫描引号与注释，定下每一处在哪： + - 单引号（含 PostgreSQL E'…'、MySQL 反斜杠转义、PostgreSQL $$…$$ / $tag$…$tag$、Oracle q'[…]'）里是字（quoted）； + - 双引号、反引号、SQLite 方括号 `[…]` 里是名字（name；MySQL 默认模式下双引号是字符串）； + - `--`、`/* … */`（PostgreSQL 支持嵌套）与 MySQL `#` 里是注释（comment）； + - 其余代码位置是裸写（bare）。`{{ … }}` 里面的字不参与外层引号/注释状态切换。 + """ + dialect_key = (dialect or "").strip().lower() + is_mysql = dialect_key in ("mysql", "mariadb") + is_pg = dialect_key in ("postgres", "postgresql") + allow_dollar = dialect_key in ("", "postgres", "postgresql") + allow_q_quote = dialect_key in ("", "oracle") + spans: list[tuple[int, int, TemplateSlot]] = [] out: list[str] = [] state = "code" + quote = "" + backslash_escape = False + dollar_tag = "" + q_close = "" + block_depth = 0 + i, n = 0, len(template) while i < n: if template.startswith("{{", i): m = _SLOT_RE.match(template, i) if m is not None: - context = {"code": "bare", "string": "quoted", "name": "name", "line": "comment", - "block": "comment"}[state] + context = { + "code": "bare", + "string": "quoted", + "dollar": "quoted", + "q_string": "quoted", + "name": "name", + "bracket_name": "name", + "line": "comment", + "block": "comment", + }[state] slot = _slot_of(m.group(1).strip(), len(spans), context) spans.append((m.start(), m.end(), slot)) - out.append(_STRING_SENTINEL.format(slot.index) if context == "quoted" - else str(_NUMBER_SENTINEL + slot.index) if context == "bare" else "") + out.append( + _STRING_SENTINEL.format(slot.index) + if context == "quoted" + else str(_NUMBER_SENTINEL + slot.index) + if context == "bare" + else "" + ) i = m.end() continue ch = template[i] if state == "code": if ch == "'": + q_closer = _oracle_q_close_at(template, i) if allow_q_quote else None + if q_closer is not None: + state = "q_string" + q_close = q_closer + out.append(template[i:i + 2]) + i += 2 + continue state = "string" - elif ch in "\"`": + quote = "'" + backslash_escape = is_mysql or _is_e_string_at(template, i) + elif ch == '"' and is_mysql: + state = "string" + quote = '"' + backslash_escape = True + elif ch in '"`': state = "name" quote = ch + elif ch == "[" and _is_bracket_ident_at(template, i, dialect_key): + state = "bracket_name" + elif allow_dollar and ch == "$" and (dm := _DOLLAR_QUOTE_RE.match(template, i)) is not None: + state = "dollar" + dollar_tag = dm.group(0) + out.append(dollar_tag) + i = dm.end() + continue elif template.startswith("--", i): state = "line" + out.append("--") + i += 2 + continue + elif is_mysql and ch == "#": + state = "line" elif template.startswith("/*", i): state = "block" - elif state == "string" and ch == "'": - state = "code" # 成对的 '' 是转义：关上又马上打开，结果一样 - elif state == "name" and ch == quote: - state = "code" + block_depth = 1 + out.append("/*") + i += 2 + continue + elif state == "string": + if backslash_escape and ch == "\\" and i + 1 < n: + out.append(template[i:i + 2]) + i += 2 + continue + if ch == quote: + if i + 1 < n and template[i + 1] == quote: + out.append(quote * 2) + i += 2 + continue + state = "code" + elif state == "dollar": + if template.startswith(dollar_tag, i): + out.append(dollar_tag) + i += len(dollar_tag) + state = "code" + continue + elif state == "q_string": + if ch == q_close and i + 1 < n and template[i + 1] == "'": + out.append(template[i:i + 2]) + i += 2 + state = "code" + continue + elif state == "name": + if ch == quote: + if i + 1 < n and template[i + 1] == quote: + out.append(quote * 2) + i += 2 + continue + state = "code" + elif state == "bracket_name": + if ch == "]": + state = "code" elif state == "line" and ch == "\n": state = "code" - elif state == "block" and template.startswith("*/", i): - out.append("*/") - i += 2 - state = "code" - continue + elif state == "block": + if is_pg and template.startswith("/*", i): + block_depth += 1 + out.append("/*") + i += 2 + continue + if template.startswith("*/", i): + out.append("*/") + i += 2 + block_depth -= 1 + if block_depth <= 0: + state = "code" + continue out.append(ch) i += 1 return spans, "".join(out) @@ -8734,7 +8869,7 @@ def _bindings(template_sql: str, sql: str, read: str | None, info: _Schema ) -> Iterator[tuple[str, dict[str, Any], tuple[str, ...], tuple[TemplateSlot, ...]]]: """按 BINDING_LEVELS 的先后给出配得上的绑定：(哪一级, {字段: 值}, 没取到值的字段, 槽位)，同样的绑定只给一次。 模板没有槽位时只给一个空绑定（level 为 parsed：用不着配）。不可比抛 _NotComparable。""" - spans, sentinel_sql = _template_slots(template_sql) + spans, sentinel_sql = _template_slots(template_sql, dialect=read) slots = tuple(slot for _, _, slot in spans) fields = tuple(dict.fromkeys(s.field for s in slots)) if not slots: @@ -8839,7 +8974,7 @@ def bind_template(template_sql: str, slots: Any, sql: str, dialect: str | None = try: for level, values, unbound, found in _bindings(template_sql or "", sql or "", read, info): return TemplateBinding(True, values, unbound, level, None, found) - spans, _ = _template_slots(template_sql or "") + spans, _ = _template_slots(template_sql or "", dialect=read) except _NotComparable as e: return TemplateBinding(False, reason=str(e)) except Exception: # noqa: BLE001 - 绑定本身的缺陷不能连累调用方：当作没配上 @@ -8924,7 +9059,7 @@ def match_template(template_sql: str, slots: Any, sql: str, dialect: str | None level, values, unbound, diff, rendered = tried[0] return TemplateMatch(False, binding=values, unbound=unbound, level=level, diff=diff, rendered=rendered) try: - spans, _ = _template_slots(template_sql) + spans, _ = _template_slots(template_sql, dialect=read) except _NotComparable as e: return TemplateMatch(False, comparable=False, reason=str(e)) fields = tuple(dict.fromkeys(slot.field for _, _, slot in spans)) @@ -8954,7 +9089,7 @@ def _now_slots(template_sql: str, sql: str, read: str | None) -> list[tuple[str, from sqlglot import exp try: - spans, sentinel = _template_slots(template_sql) + spans, sentinel = _template_slots(template_sql, dialect=read) except _NotComparable: return [] by_index = {slot.index: slot for _, _, slot in spans} @@ -9344,7 +9479,7 @@ def project_template(template_sql: str, keep: Sequence[int], dialect: str | None template = (template_sql or "").strip() keep = tuple(keep) try: - slots, sentinel = _template_slots(template) + slots, sentinel = _template_slots(template, dialect=read) root, spans, expected = _plan_projection(sentinel, keep, read, None) items = list(root.expressions) names = tuple(_output_name(items[k], read) for k in keep) @@ -9358,7 +9493,7 @@ def project_template(template_sql: str, keep: Sequence[int], dialect: str | None raise _NotProjectable(_PROJ_TEXT) out = _cut(template, mapped, keep) # type: ignore[arg-type] # 换回模板之后再按哨兵渲染一遍：结构（字面量只看位置，槽位的编号会因为去掉了几处而变）要和投影后的一样 - _, again = _template_slots(out) + _, again = _template_slots(out, dialect=read) root_again, _ = _parse(again, read) lits_a: list[Any] = [] lits_b: list[Any] = [] @@ -9420,7 +9555,7 @@ def output_names(sql: str, dialect: str | None = None, *, template: bool = False text = (sql or "").strip() try: if template: - _, text = _template_slots(text) + _, text = _template_slots(text, dialect=read) root, _ = _parse(_mysql_dashes(text) if read == "mysql" else text, read) except _NotComparable: return None diff --git a/backend/app/data/tabular.py b/backend/app/data/tabular.py index 512d9f1b..01cf5fc0 100644 --- a/backend/app/data/tabular.py +++ b/backend/app/data/tabular.py @@ -28,6 +28,7 @@ _ALWAYS_DENIED 里有 create，那是对的——模型不该能建表。而导 from __future__ import annotations +import codecs import csv import datetime as _dt import io @@ -865,33 +866,64 @@ def _close(it: Any) -> None: _ENCODINGS = ("utf-8-sig", "utf-8", "gb18030", "latin-1") -def _decode_csv(raw: bytes) -> str: - # 带 BOM 的 UTF-16（Excel 的「Unicode 文本」）：按 latin-1 读会混进 NUL，列名全乱 - if raw[:2] in (b"\xff\xfe", b"\xfe\xff"): - try: - return raw.decode("utf-16") - except UnicodeDecodeError: - pass +_CSV_DECODE_CHUNK = 1024 * 1024 + + +def _can_decode_csv(raw: bytes, enc: str) -> bool: + dec = codecs.getincrementaldecoder(enc)() + mv = memoryview(raw) + try: + for offset in range(0, len(mv), _CSV_DECODE_CHUNK): + end = offset + _CSV_DECODE_CHUNK + dec.decode(mv[offset:end], final=end >= len(mv)) + if not len(mv): + dec.decode(b"", final=True) + return True + except UnicodeDecodeError: + return False + + +def _detect_csv_encoding(raw: bytes) -> tuple[str, str]: + """增量分块探测 CSV 编码，避免为判编码而一次性物化整份全量字符串。""" + if raw[:2] in (b"\xff\xfe", b"\xfe\xff") and _can_decode_csv(raw, "utf-16"): + return "utf-16", "strict" for enc in _ENCODINGS: - try: - return raw.decode(enc) - except UnicodeDecodeError: - continue - return raw.decode("latin-1", errors="replace") + if _can_decode_csv(raw, enc): + return enc, "strict" + return "latin-1", "replace" + + +def _open_csv_text( + raw: bytes, + *, + encoding: str | None = None, + errors: str = "strict", + newline: str | None = "", +) -> io.TextIOWrapper: + if encoding is None: + encoding, errors = _detect_csv_encoding(raw) + return io.TextIOWrapper(io.BytesIO(raw), encoding=encoding, errors=errors, newline=newline) + + +def _decode_csv(raw: bytes) -> str: + enc, errors = _detect_csv_encoding(raw) + return raw.decode(enc, errors=errors) def _csv_sources(raw: bytes, filename: str, header_row: int) -> list[_Source]: - text = _decode_csv(raw) + enc, errors = _detect_csv_encoding(raw) delimiter = "\t" if _ext(filename) == ".tsv" else None if delimiter is None: + with _open_csv_text(raw, encoding=enc, errors=errors, newline="") as probe_stream: + head = probe_stream.read(8192) try: - delimiter = csv.Sniffer().sniff(text[:8192], delimiters=",;\t|").delimiter + delimiter = csv.Sniffer().sniff(head, delimiters=",;\t|").delimiter except csv.Error: delimiter = "," # 嗅不出来就按最常见的来 def rows() -> Iterator[Sequence[Any]]: - reader = csv.reader(io.StringIO(text), delimiter=delimiter) - return itertools.islice(reader, header_row - 1, None) + with _open_csv_text(raw, encoding=enc, errors=errors, newline="") as stream: + yield from itertools.islice(csv.reader(stream, delimiter=delimiter), header_row - 1, None) stem = filename.rsplit("/", 1)[-1].rsplit(".", 1)[0] # 分号分隔是欧洲区域设置的 CSV：那里逗号是小数点，「1,500」是 1.5 而不是一千五 diff --git a/backend/app/db/base.py b/backend/app/db/base.py index 3240545b..e63b5018 100644 --- a/backend/app/db/base.py +++ b/backend/app/db/base.py @@ -145,6 +145,8 @@ def _set_sqlite_pragma(dbapi_conn, _record): # pragma: no cover - 驱动回调 """WAL + 外键约束。WAL 让读写不互相阻塞，事件写入和前端轮询能并行。""" cur = dbapi_conn.cursor() cur.execute("PRAGMA journal_mode=WAL") + cur.execute("PRAGMA synchronous=NORMAL") + cur.execute("PRAGMA journal_size_limit=67108864") cur.execute("PRAGMA foreign_keys=ON") cur.execute("PRAGMA busy_timeout=5000") cur.close() @@ -514,6 +516,17 @@ _INDEX_MIGRATIONS: list[tuple[str, str, str | tuple[str, ...]]] = [ ("ix_artifacts_kind", "artifacts", "kind"), # 增量重导入的工作项按持久对齐键认（K11）：候选顺序、条数变了也要认得出同一个工作项 ("ix_caliber_import_items_align_key", "caliber_import_items", "align_key"), + # 热点翻页、跨库检索与级联外键索引：避免列表全表排序和删除父表时全表扫子表 + ("ix_runs_created_id", "runs", ("created_at", "id")), + ("ix_documents_col_created", "documents", ("collection", "created_at", "id")), + ("ix_documents_created_id", "documents", ("created_at", "id")), + ("ix_chunks_document_id", "chunks", "document_id"), + ("ix_chunk_terms_term", "chunk_terms", "term"), + ("ix_conversation_turns_run_id", "conversation_turns", "run_id"), + ("ix_caliber_drafts_datasource_id", "caliber_drafts", "datasource_id"), + ("ix_caliber_drafts_workflow_id", "caliber_drafts", "workflow_id"), + ("ix_caliber_write_requests_workflow_id", "caliber_write_requests", "workflow_id"), + ("ix_trigger_deliveries_event_id", "trigger_deliveries", "event_id"), ] diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 5147775f..86a564d0 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -602,7 +602,7 @@ class Workflow(Base, TimestampMixin): origin: Mapped[dict[str, Any] | None] = mapped_column(default=None) versions: Mapped[list["WorkflowVersion"]] = relationship( - back_populates="workflow", cascade="all, delete-orphan", lazy="selectin" + back_populates="workflow", cascade="all, delete-orphan", lazy="select", passive_deletes=True ) @@ -649,8 +649,11 @@ class WorkflowVersion(Base, TimestampMixin): class Run(Base, TimestampMixin): __tablename__ = "runs" - # 找钉住版本最近一次封存的运行（应用的视图、口径对比）。存量库由 db/base.py 的 _INDEX_MIGRATIONS 补建 - __table_args__ = (Index("ix_runs_wf_ver_status", "workflow_id", "version", "status"),) + # 找钉住版本最近一次封存的运行（应用的视图、口径对比）与列表游标翻页。存量库由 db/base.py 的 _INDEX_MIGRATIONS 补建 + __table_args__ = ( + Index("ix_runs_wf_ver_status", "workflow_id", "version", "status"), + Index("ix_runs_created_id", "created_at", "id"), + ) id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) workflow_id: Mapped[str | None] = mapped_column( @@ -765,7 +768,10 @@ class Artifact(Base, TimestampMixin): """ __tablename__ = "artifacts" - __table_args__ = (Index("ix_artifacts_run", "run_id"),) + __table_args__ = ( + Index("ix_artifacts_run", "run_id"), + Index("ix_artifacts_kind", "kind"), + ) id: Mapped[str] = mapped_column(String(64), primary_key=True) run_id: Mapped[str] = mapped_column(String(32), primary_key=True, default="") @@ -806,6 +812,10 @@ class Document(Base, TimestampMixin): """知识库里的一份原始文档。""" __tablename__ = "documents" + __table_args__ = ( + Index("ix_documents_col_created", "collection", "created_at", "id"), + Index("ix_documents_created_id", "created_at", "id"), + ) id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) collection: Mapped[str] = mapped_column(String(100), default="default", index=True) @@ -833,7 +843,7 @@ class Chunk(Base): __table_args__ = (Index("ix_chunks_collection", "collection"),) id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) - document_id: Mapped[str] = mapped_column(ForeignKey("documents.id", ondelete="CASCADE")) + document_id: Mapped[str] = mapped_column(ForeignKey("documents.id", ondelete="CASCADE"), index=True) collection: Mapped[str] = mapped_column(String(100), default="default") ordinal: Mapped[int] = mapped_column(Integer, default=0) content: Mapped[str] = mapped_column(Text) @@ -863,7 +873,10 @@ class ChunkTerm(Base): """ __tablename__ = "chunk_terms" - __table_args__ = (Index("ix_chunk_terms_lookup", "collection", "term"),) + __table_args__ = ( + Index("ix_chunk_terms_lookup", "collection", "term"), + Index("ix_chunk_terms_term", "term"), + ) id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) chunk_id: Mapped[str] = mapped_column( @@ -964,7 +977,7 @@ class ConversationTurn(Base, TimestampMixin): explanation: Mapped[str] = mapped_column(Text, default="") graph: Mapped[dict[str, Any] | None] = mapped_column(default=None) run_id: Mapped[str | None] = mapped_column( - ForeignKey("runs.id", ondelete="SET NULL"), default=None + ForeignKey("runs.id", ondelete="SET NULL"), default=None, index=True ) # running | done | error status: Mapped[str] = mapped_column(String(20), default="running") @@ -1123,7 +1136,7 @@ class CaliberDraft(Base, TimestampMixin): collection: Mapped[str] = mapped_column(String(100), default="default") # 起草时给模型看的数据源：表结构和数据目录是取数 SQL 的唯一依据；源删了草稿照旧在，只丢这一条线索 datasource_id: Mapped[str | None] = mapped_column( - ForeignKey("data_sources.id", ondelete="SET NULL"), default=None + ForeignKey("data_sources.id", ondelete="SET NULL"), default=None, index=True ) # 同 docdraft.STATUSES：drafting | ready | questions_open | path_pending | attached | rejected | failed status: Mapped[str] = mapped_column(String(20), default="drafting", index=True) @@ -1131,7 +1144,7 @@ class CaliberDraft(Base, TimestampMixin): # 这份草稿从哪来：{"documents": [文档 id], "guidance": 用户补充要求} origin: Mapped[dict[str, Any]] = mapped_column(default=dict) workflow_id: Mapped[str | None] = mapped_column( - ForeignKey("workflows.id", ondelete="SET NULL"), default=None + ForeignKey("workflows.id", ondelete="SET NULL"), default=None, index=True ) # 起草失败的说法（没有可用模型、模型回的东西取不出 JSON……）。草稿不置成 rejected：人改完还能再试 error: Mapped[str | None] = mapped_column(Text, default=None) @@ -1153,7 +1166,7 @@ class CaliberWriteRequest(Base, TimestampMixin): id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) #: 发起这次写入的主体（开登录时是认证过的用户名，关登录时是自报署名，照实记） actor: Mapped[str] = mapped_column(String(120), default="") - workflow_id: Mapped[str] = mapped_column(ForeignKey("workflows.id", ondelete="CASCADE")) + workflow_id: Mapped[str] = mapped_column(ForeignKey("workflows.id", ondelete="CASCADE"), index=True) #: 这次请求针对的那张口径卡；只改标题这类不碰图的请求记空串（列非空，唯一键才认得住） card_id: Mapped[str] = mapped_column(String(64), default="") request_id: Mapped[str] = mapped_column(String(64)) diff --git a/backend/app/db/models_triggers.py b/backend/app/db/models_triggers.py index 368eb36b..648b91c8 100644 --- a/backend/app/db/models_triggers.py +++ b/backend/app/db/models_triggers.py @@ -85,7 +85,7 @@ class TriggerDelivery(Base): id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) schedule_id: Mapped[str] = mapped_column(ForeignKey("schedules.id", ondelete="CASCADE")) - event_id: Mapped[str] = mapped_column(ForeignKey("trigger_events.id", ondelete="CASCADE")) + event_id: Mapped[str] = mapped_column(ForeignKey("trigger_events.id", ondelete="CASCADE"), index=True) #: 同事件的发生时刻（去抖窗口按它算） occurred_at: Mapped[datetime] = mapped_column(UTCDateTime, default=utcnow) status: Mapped[str] = mapped_column(String(20), default="pending") diff --git a/backend/app/engine/checkpoints.py b/backend/app/engine/checkpoints.py index 919ec46c..b6b98ae4 100644 --- a/backend/app/engine/checkpoints.py +++ b/backend/app/engine/checkpoints.py @@ -11,7 +11,7 @@ from __future__ import annotations import contextlib -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Iterable from pathlib import Path from typing import Any @@ -64,6 +64,12 @@ async def open_sqlite_saver(path: Path | str) -> AsyncIterator[BaseCheckpointSav from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver async with AsyncSqliteSaver.from_conn_string(str(path)) as saver: + conn = getattr(saver, "conn", None) + if conn is not None: + await conn.execute("PRAGMA journal_mode=WAL") + await conn.execute("PRAGMA synchronous=NORMAL") + await conn.execute("PRAGMA journal_size_limit=67108864") + await conn.execute("PRAGMA busy_timeout=5000") yield saver @@ -75,6 +81,39 @@ def open_saver() -> contextlib.AbstractAsyncContextManager[BaseCheckpointSaver]: return open_sqlite_saver(settings.checkpoint_path) +async def _delete_on(saver: Any, ids: list[str]) -> None: + adelete = getattr(saver, "adelete_thread", None) + if not callable(adelete): + return + setup = getattr(saver, "setup", None) + if callable(setup): + with contextlib.suppress(Exception): + await setup() + for tid in ids: + with contextlib.suppress(Exception): + await adelete(tid) + + +async def delete_threads(thread_ids: Iterable[str]) -> None: + """运行被彻底删除或评测明细被回收时，一并清理断点库里对应的 thread。 + + 注意：正常运行结束（succeeded / cancelled）时不能删，因为 GET /api/runs/{id}/state 和 /checkpoints + 还要回看历史断点；只有在运行记录本身从数据库删除之后才调这里。 + """ + ids = sorted({str(t) for t in thread_ids if t}) + if not ids: + return + from app.engine.runner import run_manager + + saver = run_manager._checkpointer + if saver is not None: + await _delete_on(saver, ids) + return + if settings.checkpoint_conninfo or settings.checkpoint_path.is_file(): + async with open_saver() as opened: + await _delete_on(opened, ids) + + async def count(saver: Any, thread_id: str | None = None) -> tuple[int, int]: """(断点条数, pending writes 条数)。thread_id 为空数全部。copy-to-postgres 核对用，走公开接口。""" config = {"configurable": {"thread_id": thread_id}} if thread_id else None diff --git a/backend/app/engine/evidence_reader.py b/backend/app/engine/evidence_reader.py index 7dc46eea..28a3ec91 100644 --- a/backend/app/engine/evidence_reader.py +++ b/backend/app/engine/evidence_reader.py @@ -11,6 +11,10 @@ from app.engine.evidence import ledger_enabled from app.engine.lineage import scoped_id as _scoped from app.analysis.contract import LEDGER_KIND +MASKED = "已遮罩" +RUN_NOT_FOUND = "run_not_found" +REPORT_NOT_FOUND = "evidence_report_not_found" + def _fetch(artifact: str) -> tuple[Any, bool]: try: @@ -236,3 +240,157 @@ def metric_cards(sealed: Sealed) -> list[tuple[str, str, dict[str, Any]]]: if ok is True and isinstance(content, dict) and isinstance(content.get("metrics"), list): out.append((node_id, art, content)) return out + + +async def load_sealed(run_id: str) -> Sealed: + """读取封存范围；运行记录不存在时抛 404 CodedHTTPException（供引擎与领域层共用，无需引入 api/evidence）。""" + from app.api.coded import CodedHTTPException + + try: + return await read_sealed(run_id) + except LookupError: + raise CodedHTTPException(404, "运行记录不存在", RUN_NOT_FOUND) from None + + +@dataclass +class Report: + node_id: str + doc_artifact: str + ok: bool | None + repairs: int | None + stats: dict[str, Any] | None + doc: dict[str, Any] | None + hash_ok: bool | None + fields: list[str] + claims: str | None = None + + +def output_evidence(sealed: Sealed) -> list[dict[str, Any]]: + output = sealed.payload_of(sealed.nodes_of("output")) or {} + primary = output.get("_evidence") + if not isinstance(primary, dict): + return [] + return [primary, *[r for r in primary.get("others") or [] if isinstance(r, dict)]] + + +def reports_of(sealed: Sealed, *, nested: bool = False) -> list[Report]: + """封存范围内每个报告节点最后一次的自查结果，按第一次出现的顺序。""" + latest: dict[str, dict[str, Any]] = {} + for _, etype, node_id, data in sealed.events: + if ( + etype == "report.checked" + and node_id + and isinstance(data, dict) + and data.get("doc_artifact") + and (nested or "/" not in node_id) + ): + latest[node_id] = data + evidence = output_evidence(sealed) + out: list[Report] = [] + for node_id, data in latest.items(): + artifact = str(data["doc_artifact"]) + try: + doc = load(artifact) + hash_ok: bool | None = True if doc is not None else None + except ValueError: + doc, hash_ok = None, False + if isinstance(doc, dict) and (doc.get("node_id") != node_id or doc.get("run_id") != sealed.run_id): + doc, hash_ok = None, False + fields = [ + f + for r in evidence + if r.get("report_node") == node_id and r.get("doc_artifact") == artifact + for f in r.get("fields") or [] + ] + out.append( + Report( + node_id=node_id, + doc_artifact=artifact, + ok=data.get("ok"), + repairs=data.get("repairs"), + stats=(doc or {}).get("stats") or data.get("stats"), + doc=doc if isinstance(doc, dict) else None, + hash_ok=hash_ok, + fields=fields, + claims=data.get("claims") if isinstance(data.get("claims"), str) else None, + ) + ) + return out + + +def choose_report(reports: list[Report], wanted: str | None, sealed: Sealed) -> Report: + """从封存报告列表里选出目标报告（指定 wanted 优先，否则取出口 _evidence 主报告，再退回第一份）。""" + from app.api.coded import CodedHTTPException + + if wanted: + chosen = next((r for r in reports if r.node_id == wanted), None) + if chosen is None: + raise CodedHTTPException(404, "本次运行中没有所选报告撰写节点的文档", REPORT_NOT_FOUND) + return chosen + primary = next(iter(output_evidence(sealed)), {}) + return next((r for r in reports if r.node_id == primary.get("report_node")), reports[0]) + + +class ShownText: + """把展示用 SQL、检查片段、合并警告里命中的运行遮罩值替换为「已遮罩」。""" + + def __init__(self, secrets: Any) -> None: + self._secrets = secrets + self.changed = False + + def text(self, value: Any) -> Any: + if self._secrets is None or not isinstance(value, str) or not value: + return value + out = self._secrets.scrub(value, loose=True) + self.changed = self.changed or out != value + return out + + def checks(self, checks: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + {**c, **{k: self.text(c[k]) for k in ("message", "sql_excerpt") if isinstance(c.get(k), str)}} + for c in checks + ] + + def merge(self, view: dict[str, Any]) -> dict[str, Any]: + view["sql"] = self.text(view.get("sql")) + view["warnings"] = [{**w, "message": self.text(w.get("message"))} for w in view.get("warnings") or []] + view["inputs"] = [ + {**i, **({"checks": self.checks(i["checks"])} if i.get("checks") else {})} + for i in view.get("inputs") or [] + ] + return view + + +def open_masks(sealed: Sealed | None = None) -> Any: + """获取请求级数据源遮罩上下文（通过统一边界暴露给领域层，兼容测试对证据遮罩构造器的桩替换）。""" + from app.api import evidence as _evidence_mod + + return _evidence_mod._Masks(sealed) + + +async def resolve_metric_masks( + metric: dict[str, Any], + card_node: str, + artifact: str, + sealed: Sealed, + masks: Any, + *, + card: dict[str, Any] | None = None, + output: Any = None, + by_value: bool = True, + sure: bool = False, +) -> tuple[dict[str, Any], str | None]: + """判断单个指标及其输入是否命中的数据源遮罩。""" + from app.api import evidence as _evidence_mod + + return await _evidence_mod._metric_masks( + metric, + card_node, + artifact, + sealed, + masks, + card=card, + output=output, + by_value=by_value, + sure=sure, + ) diff --git a/backend/app/engine/expressions.py b/backend/app/engine/expressions.py index 68c35138..b4488d82 100644 --- a/backend/app/engine/expressions.py +++ b/backend/app/engine/expressions.py @@ -10,6 +10,7 @@ from collections.abc import Iterator from contextlib import contextmanager from contextvars import ContextVar from dataclasses import dataclass, field +from decimal import Decimal from typing import Any, Callable # -------------------------------------------------------------------------- @@ -36,6 +37,8 @@ def resolve_path(ctx: dict[str, Any], path: str) -> Any: if isinstance(cur, dict): cur = cur.get(name) else: + if name.startswith("_"): + return None cur = getattr(cur, name, None) if cur is None: return None @@ -65,11 +68,24 @@ _FILTERS = { } +def _format_decimal(value: Decimal) -> str: + if not value.is_finite(): + return str(value) + text = str(value) + if "e" in text or "E" in text: + exp = value.as_tuple().exponent + if isinstance(exp, int) and abs(exp) <= 100: + return format(value, "f") + return text + + def _stringify(value: Any) -> str: if value is None: return "" if isinstance(value, str): return value + if isinstance(value, Decimal): + return _format_decimal(value) if isinstance(value, (dict, list, tuple)): return json.dumps(value, ensure_ascii=False, default=str) return str(value) @@ -223,6 +239,7 @@ class ExpressionError(ValueError): # 一条画布上的分支条件就能把整个后端冻死。 _MAX_POW_EXPONENT = 1024 _MAX_POW_BASE = 1e15 +_MAX_SEQ_REPEAT = 1_000_000 def _safe_pow(base: Any, exponent: Any) -> Any: @@ -237,10 +254,22 @@ def _safe_pow(base: Any, exponent: Any) -> Any: return operator.pow(base, exponent) +def _safe_mul(a: Any, b: Any) -> Any: + """带上界的乘法：序列重复（'x' * 10**9、[0] * 10**8）超界直接报错，防止在事件循环里把内存撑爆。""" + seq_types = (str, bytes, bytearray, list, tuple, TruncatedRows) + if isinstance(a, seq_types) and isinstance(b, int): + if len(a) * max(0, b) > _MAX_SEQ_REPEAT: + raise ExpressionError(f"序列重复长度超过上限 {_MAX_SEQ_REPEAT}") + elif isinstance(b, seq_types) and isinstance(a, int): + if len(b) * max(0, a) > _MAX_SEQ_REPEAT: + raise ExpressionError(f"序列重复长度超过上限 {_MAX_SEQ_REPEAT}") + return operator.mul(a, b) + + _BIN_OPS = { ast.Add: operator.add, ast.Sub: operator.sub, - ast.Mult: operator.mul, + ast.Mult: _safe_mul, ast.Div: operator.truediv, ast.FloorDiv: operator.floordiv, ast.Mod: operator.mod, diff --git a/backend/app/engine/nodes/llm.py b/backend/app/engine/nodes/llm.py index 054ed170..862d9d8e 100644 --- a/backend/app/engine/nodes/llm.py +++ b/backend/app/engine/nodes/llm.py @@ -175,11 +175,23 @@ def _usage_of(message: BaseMessage, model_id: str) -> dict[str, Any]: meta = getattr(message, "usage_metadata", None) or {} inp = int(meta.get("input_tokens") or 0) out = int(meta.get("output_tokens") or 0) + details = meta.get("input_token_details") or {} + cache_read = int(details.get("cache_read") or 0) if isinstance(details, dict) else 0 + cache_create = int(details.get("cache_creation") or 0) if isinstance(details, dict) else 0 + if cache_read > 0 or cache_create > 0: + try: + cost = catalog.estimate_cost( + model_id, inp, out, cache_read_tokens=cache_read, cache_creation_tokens=cache_create + ) + except TypeError: + cost = catalog.estimate_cost(model_id, inp, out) + else: + cost = catalog.estimate_cost(model_id, inp, out) return { "input_tokens": inp, "output_tokens": out, "total_tokens": inp + out, - "cost_usd": round(catalog.estimate_cost(model_id, inp, out), 6), + "cost_usd": round(cost, 6), "calls": 1, } @@ -213,14 +225,31 @@ def explain_model_error(exc: Exception, model_id: str) -> str: "请稍后重试；如持续失败，请到「设置 → 模型接入」测试该模型") +def _non_retriable_stream_error(exc: Exception) -> bool: + """鉴权失败、模型不存在、额度用尽、频率限制、上下文超长、输出额度被拒：非流式重发也必然因为同一原因失败，直接上报。""" + from langchain_core.exceptions import ContextOverflowError + + from app.providers.budget import budget_refused + + if isinstance(exc, ContextOverflowError) or budget_refused(exc): + return True + status = getattr(exc, "status_code", None) + if status in (401, 403, 413, 429): + return True + low = str(exc).lower() + return any(k in low for k in ( + "invalid_model", "model does not exist", "insufficient", "quota", + "rate limit", "unauthorized", "invalid api key", + )) + + async def _invoke_streaming( model: Any, messages: list[BaseMessage], ctx: NodeContext, model_id: str = "" ) -> AIMessage: """流式调用并把 token 实时推给前端。 失败时回退到非流式 —— 有些兼容网关不支持 SSE，不该因此让整个节点挂掉。 - 但回退也失败的话，两次都是同一个原因，不能只把原始异常往上抛：用户看到的 - 会是两段一模一样的供应商报错，而里面既没有模型名也没有下一步该干什么。 + 但鉴权、模型不存在、限流、上下文超长这类请求本身就不成立的报错直接抛出，不再白发第二次请求。 """ # 模型 id 要跟着 llm.start 走。以前这条事件只有 message_count，于是调用失败时 # 整条轨迹里**没有任何地方**记着试的是哪个模型——排查只能靠翻配置猜。 @@ -240,6 +269,8 @@ async def _invoke_streaming( except NotImplementedError: response = await model.ainvoke(messages) except Exception as e: # noqa: BLE001 + if _non_retriable_stream_error(e): + raise NodeError(ctx.node.id, explain_model_error(e, model_id)) from e ctx.emit(EventType.LOG, level="warn", message=f"流式输出失败，已改用非流式调用：{explain_model_error(e, model_id)}", code="stream_fallback") @@ -290,15 +321,24 @@ async def run_llm(state: GraphState, ctx: NodeContext) -> dict[str, Any]: spent: list[dict[str, Any]] = [] if schema: - # 要结构化结果就走原生 structured output，这条路径拿不到 token 流 + # 要结构化结果就走原生 structured output，带上 include_raw=True 把真实 token 用量一起拿回来 ctx.emit(EventType.LLM_START, model=model_id, structured=True, message_count=len(messages)) try: # langchain 靠 "title" 识别裸 JSON Schema dict，缺了会抛 Unsupported function if isinstance(schema, dict) and "title" not in schema: schema = {"title": "output", **schema} - structured = model.with_structured_output(schema) - value = await structured.ainvoke(messages) + try: + structured = model.with_structured_output(schema, include_raw=True) + except TypeError: + structured = model.with_structured_output(schema) + got = await structured.ainvoke(messages) + if isinstance(got, dict) and got.get("parsing_error"): + err = got["parsing_error"] + raise err if isinstance(err, BaseException) else ValueError(str(err)) + raw_msg, value = _split_structured(got) + if value is None: + raise ValueError("模型返回的结构化结果为空") except Exception as e: # noqa: BLE001 # 模型压根不存在也会走到这里，那不是"结构化输出失败" if "invalid_model" in str(e) or "model does not exist" in str(e).lower(): @@ -310,8 +350,12 @@ async def run_llm(state: GraphState, ctx: NodeContext) -> dict[str, Any]: ) from e payload = value if isinstance(value, (dict, list)) else getattr(value, "model_dump", lambda: value)() output = {"data": payload, "text": json.dumps(payload, ensure_ascii=False, indent=2)} - response: BaseMessage = AIMessage(content=output["text"]) - spent.append(_report_call(ctx, response, model_id, started)) + response = AIMessage( + content=output["text"], + usage_metadata=getattr(raw_msg, "usage_metadata", None), + response_metadata=dict(getattr(raw_msg, "response_metadata", None) or {}), + ) + spent.append(_report_call(ctx, raw_msg if raw_msg is not None else response, model_id, started)) else: response = await _invoke_streaming(model, messages, ctx, model_id) spent.append(_report_call(ctx, response, model_id, started)) diff --git a/backend/app/engine/runner.py b/backend/app/engine/runner.py index bfd012c3..86dc9165 100644 --- a/backend/app/engine/runner.py +++ b/backend/app/engine/runner.py @@ -107,9 +107,14 @@ class RunManager: # 接下来各处都按 last_seq 接号：本模块的几处另外兜了底，按需裁判记判定（api/evidence.py 的 # _append_judged）没有，不改正的话它重启后第一次追加就撞上已有的序号。有 (run_id, seq) 索引， # 每次运行一次索引查找。updated_at 不动：这是补记录，不是运行有了变化 + from sqlalchemy import exists + latest = select(func.max(RunEvent.seq)).where(RunEvent.run_id == Run.id).scalar_subquery() + lagging = exists( + select(1).where(RunEvent.run_id == Run.id, RunEvent.seq > func.coalesce(Run.last_seq, 0)) + ) await session.execute( - update(Run).where(func.coalesce(Run.last_seq, 0) < latest) + update(Run).where(lagging) .values(last_seq=latest, updated_at=Run.updated_at) .execution_options(synchronize_session=False)) await session.commit() @@ -195,6 +200,8 @@ class RunManager: if self._cm is not None: with contextlib.suppress(Exception): await self._cm.__aexit__(None, None, None) + self._cm = None + self._checkpointer = None @property def checkpointer(self) -> BaseCheckpointSaver: diff --git a/backend/app/eval/cleanup.py b/backend/app/eval/cleanup.py index 956b37f2..4ea98b0b 100644 --- a/backend/app/eval/cleanup.py +++ b/backend/app/eval/cleanup.py @@ -40,12 +40,16 @@ def keep_set(runs: list[EvalRun], keep: int) -> set[str]: return kept -async def _clean_one(session: AsyncSession, run: EvalRun) -> tuple[int, int, set[str]]: - """清一次评测的明细：(删了几项结果, 删了几次运行, 要回收的工件 id)。不提交。""" +async def _clean_one( + session: AsyncSession, + run: EvalRun, + origin_runs: dict[str, set[str]], +) -> tuple[int, set[str], set[str]]: + """清一次评测的明细：(删了几项结果, 删了哪几次运行, 要回收的工件 id)。不提交。""" from app.db.models import Artifact, Run results = list((await session.execute(select(EvalResult).where(EvalResult.eval_run_id == run.id))).scalars()) - run_ids: set[str] = set() + run_ids: set[str] = set(origin_runs.get(run.id, ())) artifacts: set[str] = set() for r in results: if r.run_id: @@ -55,11 +59,7 @@ async def _clean_one(session: AsyncSession, run: EvalRun) -> tuple[int, int, set artifacts.add(r.details_artifact) await session.execute(delete(Artifact).where(Artifact.id == r.details_artifact, Artifact.run_id == "") .execution_options(synchronize_session=False)) - # 这次评测发起的、结果里没记下的运行（重来的前一次、停止时没写进结果的）：按来源找 - for rid, origin in (await session.execute(select(Run.id, Run.origin).where(Run.origin_kind == "eval"))).all(): - if (origin or {}).get("eval_run_id") == run.id: - run_ids.add(rid) - removed = 0 + deleted_runs: set[str] = set() for rid in sorted(run_ids): row = await session.get(Run, rid) if row is None or row.origin_kind != "eval" or row.status in _LIVE: @@ -67,17 +67,20 @@ async def _clean_one(session: AsyncSession, run: EvalRun) -> tuple[int, int, set artifacts |= set((await session.execute(select(Artifact.id).where(Artifact.run_id == rid))).scalars()) await session.execute(delete(Artifact).where(Artifact.run_id == rid).execution_options(synchronize_session=False)) await session.delete(row) - removed += 1 + deleted_runs.add(rid) for r in results: await session.delete(r) run.cleaned_at = utcnow() - return len(results), removed, artifacts + return len(results), deleted_runs, artifacts async def clean_suite(suite_id: str, *, keep: int | None = None) -> dict[str, Any]: """按保留规则清一个题集：{runs: 清了几次评测, results, deleted_runs, files: 回收了几个工件文件}。""" from app.api.settings import eval_settings from app.core.artifact_store import forget + from app.db.models import Run + from app.engine import checkpoints + from app.files import store as files_store async with SessionLocal() as session: if keep is None: @@ -86,15 +89,30 @@ async def clean_suite(suite_id: str, *, keep: int | None = None) -> dict[str, An .order_by(EvalRun.created_at.desc(), EvalRun.id.desc()))).scalars()) kept = keep_set(runs, keep) targets = [r for r in runs if r.id not in kept and r.cleaned_at is None] - results = removed = 0 + origin_runs: dict[str, set[str]] = {} + if targets: + target_ids = {r.id for r in targets} + # 评测发起的、结果里没记下的运行（重来的前一次、停止时没写进结果的）：整批只扫一次按来源归组 + for rid, origin in (await session.execute(select(Run.id, Run.origin).where(Run.origin_kind == "eval"))).all(): + erid = (origin or {}).get("eval_run_id") + if isinstance(erid, str) and erid in target_ids: + origin_runs.setdefault(erid, set()).add(rid) + results = 0 + deleted_run_ids: set[str] = set() artifacts: set[str] = set() for run in targets: - n, m, found = await _clean_one(session, run) + n, gone_runs, found = await _clean_one(session, run, origin_runs) results += n - removed += m + deleted_run_ids |= gone_runs artifacts |= found + released_shas = await files_store.release_runs(session, deleted_run_ids) await session.commit() + removed = len(deleted_run_ids) files = await forget(artifacts) if artifacts else [] + if released_shas: + await files_store.drop_unreferenced(released_shas) + if deleted_run_ids: + await checkpoints.delete_threads(deleted_run_ids) if targets: logger.info("评测清理：题集 %s 清了 %d 次评测的明细（%d 项结果、%d 次运行、%d 个工件文件）", suite_id, len(targets), results, removed, len(files)) diff --git a/backend/app/files/store.py b/backend/app/files/store.py index 4b466005..38f96d30 100644 --- a/backend/app/files/store.py +++ b/backend/app/files/store.py @@ -495,6 +495,34 @@ async def remove(session: AsyncSession, principal: Any, file_id: str) -> None: await _drop_unreferenced(session, {sha}) +async def release_runs(session: AsyncSession, run_ids: Any) -> set[str]: + """运行被删除时，解除没有绑定到记录表的上传文件登记，返回待检查回收的 sha256 集合（不提交；提交后调 drop_unreferenced）。""" + ids = sorted({str(r) for r in run_ids or () if r}) + if not ids: + return set() + shas: set[str] = set() + for start in range(0, len(ids), 500): + chunk = ids[start:start + 500] + rows = list((await session.execute( + select(StoredFile.id, StoredFile.sha256).where(StoredFile.bound_run_id.in_(chunk), _NO_RECORD) + )).all()) + if rows: + shas |= {sha for _, sha in rows} + await session.execute( + delete(StoredFile).where(StoredFile.id.in_([fid for fid, _ in rows])) + .execution_options(synchronize_session=False) + ) + return shas + + +async def drop_unreferenced(shas: set[str]) -> int: + """在独立会话里检查并删除已无任何 StoredFile 引用的物理文件与文本缓存。""" + if not shas: + return 0 + async with SessionLocal() as session: + return await _drop_unreferenced(session, shas) + + async def _drop_unreferenced(session: AsyncSession, shas: set[str]) -> int: """这些内容没有任何登记引用了就删文件（连同抽出的文字缓存）。返回删了几个。""" if not shas: diff --git a/backend/app/ledger/certify.py b/backend/app/ledger/certify.py index d69562e8..659efc09 100644 --- a/backend/app/ledger/certify.py +++ b/backend/app/ledger/certify.py @@ -570,9 +570,9 @@ async def freeze(session: Any, run_id: str, *, peers: Sequence[str] = (), name: """把一次问数据运行固化成口径草稿（见模块说明），落库成新的草稿工作流，返回 {workflow_id, name, nodes, edges, params, sources, ...}。peers 是同一组（同一个 SQL 形状）的别的运行：认哪些字面量位置跨轮变了。不能固化抛 CertifyRefused。""" from app.api.deps import can_read_run, current_principal - from app.api.evidence import _reports, _sealed from app.core.artifact_store import graph_hash, load from app.db.models import DataSource, Workflow, WorkflowVersion + from app.engine.evidence_reader import load_sealed as _sealed, reports_of as _reports from app.ledger import store run = await session.get(Run, run_id) diff --git a/backend/app/ledger/docattach.py b/backend/app/ledger/docattach.py index cbe2138c..8ca7e374 100644 --- a/backend/app/ledger/docattach.py +++ b/backend/app/ledger/docattach.py @@ -105,9 +105,9 @@ async def attach(session: Any, *, payload: dict, name: str, run_id: str, draft_i {question?, metrics: [docdraft 的指标对象]}。不发布、不发起运行、不写审计。 """ from app.api.deps import can_read_run, current_principal, ensure_role - from app.api.evidence import _sealed from app.core.artifact_store import graph_hash, load from app.db.models import DataSource, Workflow, WorkflowVersion + from app.engine.evidence_reader import load_sealed as _sealed from app.ledger import certify, docdraft, store ensure_role("builder") diff --git a/backend/app/ledger/fieldbaseline.py b/backend/app/ledger/fieldbaseline.py index d21e449e..52cfb217 100644 --- a/backend/app/ledger/fieldbaseline.py +++ b/backend/app/ledger/fieldbaseline.py @@ -28,6 +28,7 @@ readable=false：**有引用不等于真的有基线**，也不能在读草稿 from __future__ import annotations import logging +import re from collections.abc import Iterable, Mapping from typing import Any @@ -397,6 +398,115 @@ def status_of(payload: Any) -> dict[str, Any]: "source_snapshot_artifact_id": str(ref.get("source_snapshot_artifact_id") or "")} +def editable_cards(graph: dict[str, Any]) -> list[dict[str, str]]: + """图里能写的一张张口径卡：[{id, title}]，按节点出现的顺序。 + + 能写的是源头口径卡（metrics 节点、没有 caliber_from）：转抄别处的口径卡不是权威写入源（A01），改它等于在别处 + 另存一份定义。 + """ + from app.apps.tree import node_map + + return [{"id": node_id, "title": str(node.get("title") or node_id)} + for node_id, node in node_map(graph).items() + if node.get("type") == "metrics" and not (node.get("config") or {}).get("caliber_from")] + + +def contract_card(graph: dict[str, Any]) -> str | None: + """下游契约声明从哪张卡取指标（metrics_from 只写了一张时就是它）：多张卡时按它认这张草稿的权威卡。""" + for node in graph.get("nodes") or []: + config = ((node or {}).get("data") or {}).get("config") if isinstance(node, dict) else None + contract = config.get("contract") if isinstance(config, dict) else None + if not isinstance(contract, dict): + continue + sources = contract.get("metrics_from") + ids = [sources] if isinstance(sources, str) else [s for s in sources or [] if isinstance(s, str)] + if len(ids) == 1: + return ids[0] + return None + + +def draft_card(graph: dict[str, Any]) -> tuple[str | None, list[dict[str, str]]]: + """这份草稿的权威口径卡，另交回全部能写的卡。 + + 一张能写的卡时就是它；多张时认下游契约声明的那一张——不是「第一张」。都认不出来时交回 None：写的时候必须显式 + 给 target.card_id，读的时候页面也照实列候选。 + """ + cards = editable_cards(graph) + if len(cards) == 1: + return cards[0]["id"], cards + referenced = contract_card(graph) + if referenced and any(card["id"] == referenced for card in cards): + return referenced, cards + return None, cards + + +def graph_entry_fields(graph: dict[str, Any]) -> list[dict[str, Any]]: + """图里入口节点（input）的字段；草稿页和试跑用它，画布改过的入口字段不会被 payload 的旧快照盖住。""" + from app.apps.tree import node_map + + start = next((node for node in node_map(graph).values() if node.get("type") == "input"), None) + return [dict(field) for field in ((start or {}).get("config") or {}).get("fields") or [] + if isinstance(field, dict)] + + +def project_graph(graph: dict[str, Any], version: int, *, card_id: str | None = None) -> dict[str, Any]: + """从草稿工作流的某一版图里投影出指标、入口字段、取数 SQL 与分组结构（纯函数）。""" + from app.apps.tree import node_map + from app.look.calibers import _metric_paths + + nodes = node_map(graph) + wanted, cards = draft_card(graph) + wanted = card_id or wanted or (cards[0]["id"] if cards else None) + card = nodes.get(wanted) if wanted else None + metrics = [metric for metric in ((card or {}).get("config") or {}).get("metrics") or [] + if isinstance(metric, dict) and metric.get("id")] + fields = graph_entry_fields(graph) + fetches: list[tuple[str, str]] = [] + for node_id, node in nodes.items(): + config = node.get("config") if isinstance(node.get("config"), dict) else {} + args = config.get("args") if isinstance(config.get("args"), dict) else {} + sql = args.get("sql") if isinstance(args, dict) else None + if node.get("type") == "tool" and isinstance(sql, str) and sql.strip(): + fetches.append((str(node_id), sql)) + groups: list[dict[str, Any]] = [] + for node_id, sql in fetches: + pattern = re.compile(rf"nodes\.{re.escape(node_id)}(?![\w])") + ids = [str(metric["id"]) for metric in metrics + if any(pattern.search(path) for path in _metric_paths(metric) or [])] + groups.append({"sql": sql, "entry_fields": [dict(field) for field in fields], "metric_ids": ids}) + assigned = {metric_id for group in groups for metric_id in group["metric_ids"]} + loose = [str(metric["id"]) for metric in metrics if str(metric["id"]) not in assigned] + if loose: + if groups: + groups[0]["metric_ids"].extend(loose) + else: + groups = [{"sql": "", "entry_fields": [dict(field) for field in fields], "metric_ids": loose}] + return { + "version": version, + "metrics": metrics, + "entry_fields": [dict(field) for field in fields], + "sql": "\n\n".join(sql for _node, sql in fetches), + "groups": groups, + "blocks": len(fetches) or 1, + } + + +async def draft_graph_projection(session: Any, workflow: Any, version: int, *, + card_id: str | None = None) -> dict[str, Any] | None: + """按 workflow_id + version 读取图版本并投影草稿视图所需字段（供重导入等领域层调用，不依赖 api/ledger）。""" + from sqlalchemy import select + + from app.db.models import WorkflowVersion + + row = (await session.execute(select(WorkflowVersion).where( + WorkflowVersion.workflow_id == workflow.id, + WorkflowVersion.version == int(version)))).scalar_one_or_none() + if row is None or not isinstance(row.graph, dict): + return None + return project_graph(row.graph, version, card_id=card_id) + + __all__ = ["CITATIONS_FIELD", "HISTORY_KEY", "KIND", "REF_KEY", "VERSION", "accepted_card", "advance", - "coverage_of", "document", "field_paths", "fields_of", "history", "put_metrics", "read", "ref_of", - "status_of", "value_at", "values"] + "contract_card", "coverage_of", "document", "draft_card", "draft_graph_projection", "editable_cards", + "field_paths", "fields_of", "graph_entry_fields", "history", "project_graph", "put_metrics", "read", + "ref_of", "status_of", "value_at", "values"] diff --git a/backend/app/ledger/issue.py b/backend/app/ledger/issue.py index 01c2843c..e07506a4 100644 --- a/backend/app/ledger/issue.py +++ b/backend/app/ledger/issue.py @@ -210,9 +210,9 @@ async def plan_issue(session: Any, run: Run | None, content: dict[str, Any] | No """按 workflow_id@version 出具这次运行的打算（见模块说明）。content 是这次运行最新的比对工件内容（读不出为 None）， hide 判一个绑定值是不是这次运行查到过的遮罩值。只读，不发起。""" from app.api.deps import can_read_run - from app.api.settings import ledger_settings from app.ledger import match as ledger_match from app.ledger.labels import caliber_key + from app.ledger.settings import ledger_settings plan = Plan(run_id=run.id if run else "", workflow_id=workflow_id, version=int(version), caliber=caliber_key(workflow_id, int(version)), match_artifact=artifact) @@ -257,12 +257,12 @@ async def check_target(session: Any, plan: Plan, workflow: Any, *, principal: An （ledger/params：缺参数、未声明参数、承载不了的值、认不出参数写在哪的模板都拒）；使用者只能按 他有权使用的应用固定着的这一版（拍板 L5）；同一个来源（source_run）、同一版、同一组参数发起过的交回那一次；当天没超 上限。填好 plan 的 level、expect_hash、inputs、granted_by_app、existing；不能出具的写进 plan.refuse，交回 None。""" - from app.api.settings import ledger_settings from app.apps import versions as app_versions from app.apps.backfill import graph_side_effect from app.apps.forms import input_schema, validate_form from app.apps.resolve import input_key from app.ledger.derive import entry_fields + from app.ledger.settings import ledger_settings workflow_id, version = plan.workflow_id, plan.version if workflow is None: @@ -362,7 +362,7 @@ QUOTA = "ledger_issue" async def _hider(run_id: str) -> Any: - from app.api.evidence import _sealed + from app.engine.evidence_reader import load_sealed as _sealed from app.ledger.view import _hider as hider_of return (await hider_of(await _sealed(run_id))).hide @@ -397,8 +397,8 @@ async def issue(run_id: str, workflow_id: str, version: int, *, principal: Any, **({"publication": plan.publication} if plan.publication else {})} actor = run.started_by if auto else principal.username # 先占一份今天的额度（库里带条件的加一，几个进程同时出具也正好到上限为止），发起不成退回 - from app.api.settings import ledger_settings from app.cluster import quota + from app.ledger.settings import ledger_settings limit = int((await ledger_settings(session))["issue_daily_limit"]) day = await quota.take(QUOTA, limit, seed=issued_today) diff --git a/backend/app/ledger/match.py b/backend/app/ledger/match.py index 25b14054..30e950d1 100644 --- a/backend/app/ledger/match.py +++ b/backend/app/ledger/match.py @@ -579,9 +579,15 @@ def compare_queries(queries: list[Query], calibers: list[Caliber], *, deadline: chosen = (composed or {}).get(str(query.node_id or "")) wanted = _wanted(query, referenced) pool = [(c, f) for c in calibers for f in c.fetches if f.source == query.source and f.tables & query.tables] - if chosen is not None: - # 助手选用的那一份排最前面：候选多的时候也一定比到它 - pool.sort(key=lambda cf: (cf[0].key, cf[1].node_id) != chosen) + if pool: + # 助手选用的排第一；其余按「共享表数越多越靠前、表集合规模越接近越靠前」排，同分保持工作流/版本/节点原序 + pool.sort( + key=lambda cf: ( + (cf[0].key, cf[1].node_id) != chosen if chosen is not None else False, + -len(cf[1].tables & query.tables), + abs(len(cf[1].tables) - len(query.tables)), + ) + ) if not pool: out.append({**row, "status": "none", "candidates": []}) continue @@ -649,7 +655,7 @@ def _card_loader(sealed: Any) -> Any: def referenced_of(sealed: Any) -> dict[str, set[str]]: """这次运行封存的报告引用了哪条查询的哪几列（labels.referenced_columns，只读哈希核对一致的报告）：compare_queries 的 referenced。评测逐题比 SQL 时传它，就和账本一样按引用的列投影。在线程里调（读盘）。""" - from app.api.evidence import _reports + from app.engine.evidence_reader import reports_of as _reports docs = [r.doc for r in _reports(sealed) if r.doc is not None and r.hash_ok is True] return ledger_labels.referenced_columns(docs, _card_loader(sealed)) @@ -657,7 +663,7 @@ def referenced_of(sealed: Any) -> dict[str, set[str]]: def compute(sealed: Any, calibers: list[Caliber], kinds: dict[str, str | None], *, timeout: float) -> dict[str, Any]: """一次运行的比对工件内容（见 ledger/store 的模块说明）。在线程里调：读快照、比 SQL、标数都是同步的活。""" - from app.api.evidence import _choose, _reports + from app.engine.evidence_reader import choose_report as _choose, reports_of as _reports deadline = time.monotonic() + timeout queries = read_queries(sealed, kinds) @@ -753,8 +759,8 @@ async def match_run(run_id: str, *, trigger: str = "sealed", timeout: float | No async def _match_run(run_id: str, *, trigger: str, timeout: float | None) -> str | None: - from app.api.evidence import _sealed - from app.api.settings import ledger_settings + from app.engine.evidence_reader import load_sealed as _sealed + from app.ledger.settings import ledger_settings async with SessionLocal() as session: run = await session.get(Run, run_id) diff --git a/backend/app/ledger/params.py b/backend/app/ledger/params.py index 01e7e61d..9d7d8be6 100644 --- a/backend/app/ledger/params.py +++ b/backend/app/ledger/params.py @@ -69,9 +69,11 @@ from __future__ import annotations import hashlib import json +import math import re from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field +from decimal import Decimal from typing import Any #: 参数问题的机器码（§4.4 的三条 + 值承载不了的一条） @@ -184,13 +186,13 @@ def describe(problems: Iterable[Problem]) -> str: # ========================================================================== -def slots_of(template: str) -> tuple[Slot, ...]: +def slots_of(template: str, *, dialect: str | None = None) -> tuple[Slot, ...]: """模板里每一处 `{{ input.字段 }}`（含写在哪儿、带了什么过滤器）。认不出写法、写在表名列名里的抛 ParameterRefused(unsupported_parameter)——判断和带值比对那一套（data/sql_semantics._template_slots）是同一份。""" from app.data.sql_semantics import _NotComparable, _template_slots try: - spans, _ = _template_slots(template) + spans, _ = _template_slots(template, dialect=dialect) except _NotComparable as e: raise ParameterRefused([Problem(UNSUPPORTED, str(e))]) from None return tuple(Slot(index=s.index, field=s.field, filters=tuple(s.filters), context=s.context) for _, _, s in spans) @@ -299,8 +301,8 @@ def declared(fields: Iterable[Mapping[str, Any]]) -> dict[str, dict[str, Any]]: #: 声明类型 → 规范类型。认不出的一律 string（各入口同一套：类型判定不依赖字段先后，R02 的一半） _TYPES = {"string": "string", "text": "string", "integer": "integer", "int": "integer", "number": "number", - "float": "number", "boolean": "boolean", "bool": "boolean", "date": "date", "datetime": "datetime", - "time": "time", "file": "file", "array": "array", "object": "object"} + "float": "number", "decimal": "number", "boolean": "boolean", "bool": "boolean", "date": "date", + "datetime": "datetime", "time": "time", "file": "file", "array": "array", "object": "object"} def field_type(field: Mapping[str, Any]) -> str: @@ -314,7 +316,7 @@ def value_kind(value: Any) -> str: return "null" if isinstance(value, bool): return "boolean" - if isinstance(value, (int, float)): + if isinstance(value, (int, float, Decimal)): return "number" if isinstance(value, str): return "text" @@ -366,11 +368,29 @@ def resolve(fields: Iterable[Mapping[str, Any]], values: Mapping[str, Any]) -> d def _written(value: Any) -> str: if isinstance(value, str): return value + if isinstance(value, Decimal): + from app.engine.expressions import _format_decimal + + return _format_decimal(value) if isinstance(value, (dict, list)): return json.dumps(value, ensure_ascii=False, default=str) return str(value) +def _bare_scalar_ok(value: Any) -> bool: + if isinstance(value, bool): + return True + if isinstance(value, int): + return True + if isinstance(value, float): + return math.isfinite(value) + if isinstance(value, Decimal): + return value.is_finite() + if isinstance(value, str): + return bool(_NUMBER.match(value)) + return False + + def _unresolved(name: str, field: Mapping[str, Any] | None, values: Mapping[str, Any]) -> bool: """这个参数这次没有可用的值：没给、给了 None、或者给了空串（空串不是值，是省略——和入口收值 params.provided 同一套）， 而且声明的缺省值也是 None。 @@ -390,14 +410,13 @@ def _unresolved(name: str, field: Mapping[str, Any] | None, values: Mapping[str, def literal_problem(name: str, value: Any, contexts: Iterable[str]) -> str | None: """值写在 SQL 的哪个位置、旧适配器（内联文本替换）承载得了吗：承载不了给一句原因，承载得了为 None。 - 引号里的字里再出现引号 / 反斜杠 / 换行、裸写的位置上不是数、注释里带换行或注释结束符——这些都会让渲染出来的语句 + 引号里的字里再出现引号 / 反斜杠 / 换行、裸写的位置上不是有限数值或布尔值、注释里带换行或注释结束符——这些都会让渲染出来的语句 不再等于「那一版口径的 SQL」。真支持特殊字符要等驱动级参数绑定（见模块说明），那之前明确拒绝，不硬塞。""" text = _written(value) where = set(contexts) if "quoted" in where and any(ch in text for ch in ("'", "\\", "\n", "\r")): return QUOTED_TEXT.format(name=name) - if "bare" in where and not (isinstance(value, (int, float)) and not isinstance(value, bool)) \ - and not (isinstance(value, str) and _NUMBER.match(value)): + if "bare" in where and not _bare_scalar_ok(value): return BARE_TEXT.format(name=name, value=text[:40]) if "comment" in where and ("\n" in text or "\r" in text or "*/" in text): return COMMENT_TEXT.format(name=name) @@ -405,13 +424,14 @@ def literal_problem(name: str, value: Any, contexts: Iterable[str]) -> str | Non def check_template(template: str, *, fields: Mapping[str, Mapping[str, Any]], values: Mapping[str, Any] | None = None, - node_id: str | None = None, where: str = "入口字段") -> list[Problem]: + node_id: str | None = None, where: str = "入口字段", + dialect: str | None = None) -> list[Problem]: """一条取数模板按声明的入口字段过一遍：未声明、认不出的写法；给了 values 的，再判缺值与值承载不了。 values 为 None 是「此刻还没有这次调用的值」（读一版口径时就是这样）：只判模板和声明对不对得上。 认不出写法（unsupported）时不接着往下判——值写在哪儿都还没定，别的结论不可靠。""" try: - slots = slots_of(template) + slots = slots_of(template, dialect=dialect) except ParameterRefused as e: return [Problem(UNSUPPORTED, UNREADABLE_TEXT.format(reason=p.message), node_id=node_id) for p in e.problems] problems: list[Problem] = [] @@ -450,7 +470,8 @@ def _sql_of(node: Mapping[str, Any]) -> str | None: def check_nodes(nodes: Iterable[Mapping[str, Any]], *, fields: Iterable[Mapping[str, Any]], - values: Mapping[str, Any] | None = None, where: str = "入口字段") -> list[Problem]: + values: Mapping[str, Any] | None = None, where: str = "入口字段", + dialect: str | None = None) -> list[Problem]: """几个调用工具节点（原始 JSON）的 args.sql 各过一遍，问题按节点先后合起来。values 不给就只判模板与声明。""" declared_fields = declared(fields) out: list[Problem] = [] @@ -461,7 +482,7 @@ def check_nodes(nodes: Iterable[Mapping[str, Any]], *, fields: Iterable[Mapping[ if sql is None: continue out += check_template(sql, fields=declared_fields, values=values, node_id=str(node.get("id") or "") or None, - where=where) + where=where, dialect=dialect) return out @@ -515,7 +536,8 @@ def _hash(text: str) -> str: def material(template: str, *, values: Mapping[str, Any], declared_fields: Mapping[str, Mapping[str, Any]], - source: Mapping[str, Any] | None = None, with_template: bool = True) -> dict[str, Any]: + source: Mapping[str, Any] | None = None, with_template: bool = True, + dialect: str | None = None) -> dict[str, Any]: """查询快照要记的那一份：源模板、源模板哈希、规范化输入哈希、参数契约、每个参数的类型与写在哪儿、承载方式。 绑定值不在这里再存一份——内联方式下它就写在实际执行语句里，而实际语句本身在快照里（payload["sql"]）。 @@ -526,7 +548,7 @@ def material(template: str, *, values: Mapping[str, Any], declared_fields: Mappi names = input_refs(template) bound: list[dict[str, Any]] = [] try: - by_field = {s.field: s.context for s in slots_of(template)} + by_field = {s.field: s.context for s in slots_of(template, dialect=dialect)} except ParameterRefused: by_field = {} for name in names: @@ -553,7 +575,7 @@ def material(template: str, *, values: Mapping[str, Any], declared_fields: Mappi def plan(template: str, *, fields: Iterable[Mapping[str, Any]], values: Mapping[str, Any], source: Mapping[str, Any] | None = None, node_id: str | None = None, - where: str = "入口字段") -> Statement: + where: str = "入口字段", dialect: str | None = None) -> Statement: """「现在就发一条 SQL」的入口用：先按声明的字段把这次的值定下来（resolve：显式非空值优先、否则声明缺省值）， 再判参数完整性，过了才渲染成字面语句，附快照材料——检查、渲染、材料里的输入哈希用的是**同一份 resolved values**。 @@ -567,11 +589,13 @@ def plan(template: str, *, fields: Iterable[Mapping[str, Any]], values: Mapping[ check_nodes / check_template 判过）——它是给下一个入口（QueryCore 的固定执行）准备的共享层接口，行为由测试锁住。""" declared_fields = declared(fields) resolved = resolve(declared_fields.values(), values) - problems = check_template(template, fields=declared_fields, values=resolved, node_id=node_id, where=where) + problems = check_template(template, fields=declared_fields, values=resolved, node_id=node_id, where=where, + dialect=dialect) if problems: raise ParameterRefused(problems) return Statement(statement=render(template, resolved), - material=material(template, values=resolved, declared_fields=declared_fields, source=source)) + material=material(template, values=resolved, declared_fields=declared_fields, source=source, + dialect=dialect)) # ========================================================================== diff --git a/backend/app/ledger/querycore.py b/backend/app/ledger/querycore.py index ca00ff16..44626a4a 100644 --- a/backend/app/ledger/querycore.py +++ b/backend/app/ledger/querycore.py @@ -399,13 +399,12 @@ async def _metrics_from(sealed_run_id: str, *, card_id: str, metric_ids: Sequenc 的时候，这里绝不能把值交出去。交回 (类型化指标, 原始指标)，原始那一份给质量报告用。 memo 是这一次收尾里几处共用的封存证据（见 _sealed_of）：给了就用那一份，没给就自己读一次。""" - from app.api.evidence import _Masks, _metric_masks - from app.engine.evidence_reader import metric_cards + from app.engine.evidence_reader import metric_cards, open_masks, resolve_metric_masks sealed = await _sealed_of(sealed_run_id, memo) if not sealed.trusted: return [], {"trusted": False} - masks = _Masks(sealed) + masks = open_masks(sealed) found: dict[str, dict[str, Any]] = {} raw: dict[str, dict[str, Any]] = {} for node_id, artifact, card in metric_cards(sealed): @@ -419,7 +418,7 @@ async def _metrics_from(sealed_run_id: str, *, card_id: str, metric_ids: Sequenc continue # 同一张卡执行过多次时按**最后一次**的产出算（和「节点最后一次产出」同一个认法）， # 不拿早先那次的值 - _, hidden = await _metric_masks(metric, node_id, artifact, sealed, masks, card=card) + _, hidden = await resolve_metric_masks(metric, node_id, artifact, sealed, masks, card=card) found[metric_id] = _typed_metric(metric, hidden) raw[metric_id] = dict(metric) ordered = [found[mid] for mid in metric_ids if mid in found] diff --git a/backend/app/ledger/recall.py b/backend/app/ledger/recall.py index 7c1755c8..ee04e096 100644 --- a/backend/app/ledger/recall.py +++ b/backend/app/ledger/recall.py @@ -495,7 +495,7 @@ def _lock(key: tuple[str, int]) -> Any: async def launch(workflow_id: str, new_version: int, *, batch: str = "auto", actor: str | None = None) -> list[str]: """按计划发起一批召回运行，交回发起了的运行 id。auto：发布后自动的那一批，一次发布最多 recall_limit 组（已经自动发起 过的扣掉，重复调幂等）；manual：「接着算」，再发起一批（上限是 0 时按缺省的组数）。""" - from app.api.settings import LEDGER_DEFAULTS, ledger_settings + from app.ledger.settings import LEDGER_DEFAULTS, ledger_settings if batch not in BATCHES: raise ValueError(f"不认识的召回批次：{batch}") @@ -534,7 +534,7 @@ def _counts(plan: Plan) -> dict[str, int]: async def preview(session: Any, workflow_id: str, version: int) -> dict[str, Any]: """发布弹窗的召回预告：把 version 发布成新版本之后，旧版本被多少问数回答、导出、定时任务、应用用过，预计重算几组，超出 上限的写明。只读、不发起（和发布之后真的召回走同一份计划，规模一致）。""" - from app.api.settings import ledger_settings + from app.ledger.settings import ledger_settings plan = await build_plan(session, workflow_id, int(version)) limit = int((await ledger_settings(session))["recall_limit"]) @@ -598,7 +598,7 @@ async def batches(session: Any, workflow_id: str, version: int) -> dict[str, Any 未重算」和能不能接着算（还有没算的组却不能接着算时 continue_reason 写为什么：之后又发布了更新的版本……）。 接口只给搭建者（api/ledger 的 router）：参数原样给，同他在召回运行的输入里读到的一样；使用者 看的是 recall 待办（task_payload，按遮罩换过）。""" - from app.api.settings import LEDGER_DEFAULTS, ledger_settings + from app.ledger.settings import LEDGER_DEFAULTS, ledger_settings plan = await build_plan(session, workflow_id, int(version)) limit = int((await ledger_settings(session))["recall_limit"]) @@ -664,7 +664,7 @@ async def _settled(session: Any, run_id: str) -> dict[str, Any] | None: async def card_values(run_id: str) -> dict[str, dict[str, Any]]: """召回运行封存范围内的召回卡：{指标 id: 指标}（value、rendered……）。读不出为空。""" - from app.api.evidence import _sealed, _sealed_cards + from app.engine.evidence_reader import load_sealed as _sealed, metric_cards as _sealed_cards from app.ledger import derive sealed = await _sealed(run_id) @@ -878,7 +878,7 @@ async def task_payload(session: Any, task: Any, principal: Any) -> dict[str, Any """recall 待办给处理人看的内容（待办箱、待办详情的 payload，process/tasks._describe 调）：旧口径的值、新口径的值、两版之差 和原答案当时的值（每个都带出处：哪次运行、哪张卡、哪个指标，点得开），这一组的参数，能做的动作。使用者看到的值照样 遮罩（按那次运行查到过的遮罩值，同账本接口）。""" - from app.api.evidence import _sealed + from app.engine.evidence_reader import load_sealed as _sealed from app.ledger.view import _hider run = await session.get(Run, task.run_id) @@ -946,7 +946,7 @@ async def original_values(use: Mapping[str, Any], meta: Sequence[Mapping[str, An """原答案当时的值（每个比较一项）：正式运行读它封存的口径卡（固定了这一版的读转抄来的那张）；问数回答读等价的那条查询 快照里那一格（口径卡的指标就是取数第一行某一列时，按列映射找这次的列）。读不出、原答案里没有单独这个数的为空，写为什么。 hide_for：给使用者看，按那次运行查到过的遮罩值遮。""" - from app.api.evidence import _sealed, _sealed_cards + from app.engine.evidence_reader import load_sealed as _sealed, metric_cards as _sealed_cards from app.ledger.view import _hider run_id = str(use.get("run_id") or "") diff --git a/backend/app/ledger/reconcile.py b/backend/app/ledger/reconcile.py index fbd87016..8c4c19d0 100644 --- a/backend/app/ledger/reconcile.py +++ b/backend/app/ledger/reconcile.py @@ -519,8 +519,8 @@ def _plan_rows(ours_columns: list[str], rows: list[Any], hidden: set[str], calib def _referenced(sealed: Any, artifact: str) -> list[tuple[int, str]]: """报告里引用了这条查询的哪几格：[(行, 列)]，按正文的先后（单元格、表格里的格）。""" - from app.api.evidence import _reports from app.engine.evidence import iter_units + from app.engine.evidence_reader import reports_of as _reports out: list[tuple[int, str]] = [] for report in _reports(sealed): @@ -580,8 +580,8 @@ def pinned_effective(pinned: Any) -> list[dict[str, Any]]: async def reconcile(run_id: str, req: Request, *, principal: Any) -> Run: """对这次运行的一条查询和一份相关口径对一次账：派生只读图、发起探索运行（来源 reconcile）。不能对的抛 ReconcileRefused。""" from app.api.deps import can_read_run - from app.api.evidence import _Masks, _sealed from app.core.artifact_store import load + from app.engine.evidence_reader import load_sealed as _sealed, open_masks as _Masks from app.engine.launch import LaunchRefused, launch_run from app.ledger import compose, derive, store from app.ledger.issue import _conversation_of diff --git a/backend/app/ledger/reimport.py b/backend/app/ledger/reimport.py index 8a62d152..8d773bb5 100644 --- a/backend/app/ledger/reimport.py +++ b/backend/app/ledger/reimport.py @@ -1208,8 +1208,6 @@ async def _refresh_run_items(session: Any, *, run: CaliberImportRun, draft: Cali 没对齐上的工作项（mode 是 new / manual、metric_id 为空）没有可比的 L 一格，保持抽出来时的样子： 要对齐得先走人工对齐（reimport.align 的入口在接口层）。 """ - from app.api.ledger import _draft_graph_payload - live_metrics = list(fieldbaseline.accepted_card(workflow.graph if workflow is not None else {})[1]) live_by_id = {str(m.get("id")): m for m in live_metrics if isinstance(m, Mapping)} ref = fieldbaseline.ref_of(draft.payload) @@ -1228,7 +1226,7 @@ async def _refresh_run_items(session: Any, *, run: CaliberImportRun, draft: Cali item.fields, item.status, item.align = merged["fields"], merged["status"], merged["align"] await _refresh_counts(session, run) if workflow is not None: - live = await _draft_graph_payload(session, workflow, workflow.version) + live = await fieldbaseline.draft_graph_projection(session, workflow, workflow.version) if live is not None: draft.payload = {**(draft.payload or {}), **live} diff --git a/backend/app/ledger/settings.py b/backend/app/ledger/settings.py new file mode 100644 index 00000000..01afb7f1 --- /dev/null +++ b/backend/app/ledger/settings.py @@ -0,0 +1,92 @@ +"""口径账本运行与治理设置：默认项、范围校验与持久化读取（供 ledger 领域层与设置接口共用）。""" +from __future__ import annotations + +from typing import Any + +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.models import Setting + +LEDGER_GROUP = "ledger" +LEDGER_DEFAULTS: dict[str, Any] = { + "auto_issue": True, + "issue_daily_limit": 20, + "match_timeout_s": 8, + "recall_limit": 50, +} +LEDGER_SETTING_LABEL = { + "auto_issue": "自动按口径正式出具", + "issue_daily_limit": "按口径正式出具每日上限", + "match_timeout_s": "口径比对时限（秒）", + "recall_limit": "口径升版后每次最多重算几组", +} +LEDGER_RECALL_MAX = 1000 +LEDGER_TIMEOUT_RANGE = (1, 60) +LEDGER_DAILY_LIMIT_MAX = 100_000 + + +def _daily_limit_ok(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= LEDGER_DAILY_LIMIT_MAX + + +def _timeout_ok(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) \ + and LEDGER_TIMEOUT_RANGE[0] <= value <= LEDGER_TIMEOUT_RANGE[1] + + +def _recall_limit_ok(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= LEDGER_RECALL_MAX + + +def ledger_problem(value: Any) -> str | None: + """设置页写进来的 ledger 这一组有什么问题，没问题返回 None。""" + if not isinstance(value, dict): + return f"「口径账本」设置格式有误，需要是一个对象，当前为「{value}」" + for key, item in value.items(): + if key not in LEDGER_DEFAULTS: + return (f"「口径账本」设置中不支持「{key}」，可设置的项：" + + "、".join(f"「{LEDGER_SETTING_LABEL[k]}」" for k in LEDGER_DEFAULTS)) + if key == "auto_issue" and not isinstance(item, bool): + return f"「{LEDGER_SETTING_LABEL[key]}」只能是开或关，当前为「{item}」" + if key == "issue_daily_limit" and not _daily_limit_ok(item): + return (f"「{LEDGER_SETTING_LABEL[key]}」需要是 0 到 {LEDGER_DAILY_LIMIT_MAX:,} 之间的整数（0 表示不再按口径出具），" + f"当前为「{item}」") + if key == "recall_limit" and not _recall_limit_ok(item): + return (f"「{LEDGER_SETTING_LABEL[key]}」需要是 0 到 {LEDGER_RECALL_MAX:,} 之间的整数（0 表示发布后不自动重算），" + f"当前为「{item}」") + if key == "match_timeout_s" and not _timeout_ok(item): + return (f"「{LEDGER_SETTING_LABEL[key]}」需要是 {LEDGER_TIMEOUT_RANGE[0]} 到 {LEDGER_TIMEOUT_RANGE[1]} 之间的数，" + f"当前为「{item}」") + return None + + +def ledger_values(stored: Any) -> dict[str, Any]: + """库里存的（或刚写进来的）ledger 一组 → 补齐默认；存坏了的项按默认（自动出具的开关存坏了按关，宁可少出具）。""" + stored = stored if isinstance(stored, dict) else {} + auto, limit, timeout = stored.get("auto_issue"), stored.get("issue_daily_limit"), stored.get("match_timeout_s") + recall = stored.get("recall_limit") + return { + "auto_issue": auto if isinstance(auto, bool) else (LEDGER_DEFAULTS["auto_issue"] if auto is None else False), + "issue_daily_limit": limit if _daily_limit_ok(limit) else LEDGER_DEFAULTS["issue_daily_limit"], + "match_timeout_s": timeout if _timeout_ok(timeout) else LEDGER_DEFAULTS["match_timeout_s"], + "recall_limit": recall if _recall_limit_ok(recall) else LEDGER_DEFAULTS["recall_limit"], + } + + +async def ledger_settings(session: AsyncSession) -> dict[str, Any]: + """设置里的 ledger 一组，没存过的补默认。按口径出具、比对从这里读。""" + row = await session.get(Setting, LEDGER_GROUP) + return ledger_values(row.value if row else None) + + +__all__ = [ + "LEDGER_DAILY_LIMIT_MAX", + "LEDGER_DEFAULTS", + "LEDGER_GROUP", + "LEDGER_RECALL_MAX", + "LEDGER_SETTING_LABEL", + "LEDGER_TIMEOUT_RANGE", + "ledger_problem", + "ledger_settings", + "ledger_values", +] diff --git a/backend/app/ledger/view.py b/backend/app/ledger/view.py index 8aec1ce6..8fa68684 100644 --- a/backend/app/ledger/view.py +++ b/backend/app/ledger/view.py @@ -46,10 +46,10 @@ class _Hide: """按这次运行查到过的遮罩值遮：hide(值) 这个值给不给，shown(文字) 文字里的遮罩值换成「已遮罩」。""" def __init__(self, secrets: Any) -> None: - from app.api.evidence import _Shown + from app.engine.evidence_reader import ShownText self._secrets = secrets - self._shown = _Shown(secrets) + self._shown = ShownText(secrets) def hide(self, value: Any) -> bool: if self._secrets is None or value is None or isinstance(value, bool): @@ -61,9 +61,9 @@ class _Hide: async def _hider(sealed: Any) -> _Hide: - from app.api.evidence import _Masks + from app.engine.evidence_reader import open_masks - return _Hide(await _Masks(sealed).run_secrets()) + return _Hide(await open_masks(sealed).run_secrets()) async def _names(ids: set[str]) -> dict[str, str]: @@ -226,10 +226,8 @@ async def run_view(run_id: str) -> dict[str, Any]: """GET /api/runs/{run_id}/ledger 的答复：标签、整题判断、每条查询的比对、能做的动作。没比对过、能比的运行这时排上按需比对 （trigger=on_demand），答复 state=pending；不比的照实说为什么。""" from app.api.coded import CodedHTTPException - from app.api.deps import restricted - from app.api.evidence import RUN_NOT_FOUND, _sealed - - from app.api.deps import current_principal + from app.api.deps import current_principal, restricted + from app.engine.evidence_reader import RUN_NOT_FOUND, load_sealed as _sealed from app.ledger.recall import answer_marks ordinary = restricted() diff --git a/backend/app/main.py b/backend/app/main.py index 47823f3c..f60d9e2b 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -171,9 +171,11 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await relay.stop() await cluster.stop() # 应用库的连接全关掉：SQLite 最后一个连接关掉时才把 -wal 并回、删掉（copy-to-postgres 按它认服务停没停）， - # PostgreSQL 的连接池也正常关 + # PostgreSQL 的连接池也正常关；外部数据源缓存的连接池一并释放 + from app.data.engine import engines from app.db.base import dispose_session_engine, engine as db_engine + await engines.close() await db_engine.dispose() await dispose_session_engine() logger.info("已停止") diff --git a/backend/app/memory/inverted.py b/backend/app/memory/inverted.py index 696c4363..3f623358 100644 --- a/backend/app/memory/inverted.py +++ b/backend/app/memory/inverted.py @@ -22,6 +22,8 @@ from app.memory.embeddings import _tokens _K1 = 1.5 _B = 0.75 +_IN_BATCH = 500 +_REBUILD_BATCH = 256 def term_freqs(text: str) -> dict[str, int]: @@ -47,11 +49,69 @@ async def drop_chunk_terms(session: AsyncSession, chunk_ids: list[str]) -> None: """删片段时一起删倒排。 外键写了 ondelete=CASCADE，但 SQLite 的级联要连接级 PRAGMA 打开才生效 - （base.py 里是打开的）——索引一致性不该依赖一个开关，显式删。 + （base.py 里是打开的）——索引一致性不该依赖一个开关，显式按批删，避免超过 + SQLite / PostgreSQL 单条语句绑定参数上限（32,766）。 """ - if chunk_ids: - await session.execute( - delete(ChunkTerm).where(ChunkTerm.chunk_id.in_(chunk_ids))) + for i in range(0, len(chunk_ids), _IN_BATCH): + part = chunk_ids[i:i + _IN_BATCH] + if part: + await session.execute( + delete(ChunkTerm).where(ChunkTerm.chunk_id.in_(part))) + + +async def candidates_with_scores( + session: AsyncSession, *, collection: str | None, query: str, limit: int, +) -> tuple[list[tuple[str, float]], dict[str, float]]: + """按 BM25 取候选，返回 (前 limit 条 [(chunk_id, 分数)], 全部命中片段的精确 BM25 分数字典)。 + + 直接通过 ChunkTerm JOIN Chunk 一次性取回 token_len，彻底消除大集合高频词命中数万条时 + `Chunk.id.in_(...)` 超出数据库 32,767 参数上限崩溃的问题。 + """ + # 按查询词的出现次数加权，不能去重。原来的 BM25.score 遍历的是带重复的 + # 词表：「导出订单最多多少行」里「多」出现两次，它的贡献就加两次。去重会让 + # 同一个查询在倒排和全表扫两条路上给出不同的分——实测顶部那条差了 2.32 + qcounts = Counter(t for t in _tokens(query) if len(t) <= 64) + terms = list(qcounts) + if not terms: + return [], {} + + rows: list[Any] = [] + for i in range(0, len(terms), _IN_BATCH): + part = terms[i:i + _IN_BATCH] + stmt = ( + select(ChunkTerm.chunk_id, ChunkTerm.term, ChunkTerm.tf, Chunk.token_len) + .join(Chunk, Chunk.id == ChunkTerm.chunk_id) + .where(ChunkTerm.term.in_(part)) + ) + if collection: + stmt = stmt.where(ChunkTerm.collection == collection) + rows.extend((await session.execute(stmt)).all()) + if not rows: + return [], {} + + # df 直接从命中里数：含有该词的片段数，正是 BM25 要的那个 df + df: dict[str, int] = defaultdict(int) + for _chunk_id, term, _tf, _tlen in rows: + df[term] += 1 + + stat = select(func.count(Chunk.id), func.sum(Chunk.token_len)) + if collection: + stat = stat.where(Chunk.collection == collection) + total, total_len = (await session.execute(stat)).one() + total = int(total or 0) + if not total: + return [], {} + avg_len = float(total_len or 0) / total or 1.0 + + scores: dict[str, float] = defaultdict(float) + for chunk_id, term, tf, token_len in rows: + idf = math.log(1 + (total - df[term] + 0.5) / (df[term] + 0.5)) + length = int(token_len or 0) or 1 + denom = tf + _K1 * (1 - _B + _B * length / avg_len) + scores[chunk_id] += qcounts[term] * idf * (tf * (_K1 + 1)) / denom + + ranked = sorted(scores.items(), key=lambda kv: (-kv[1], kv[0])) + return ranked[:limit], dict(scores) async def candidates( @@ -61,51 +121,16 @@ async def candidates( 没有任何查询词命中时返回空——调用方据此决定是不是要退回全表。 """ - # 按查询词的出现次数加权，不能去重。原来的 BM25.score 遍历的是带重复的 - # 词表：「导出订单最多多少行」里「多」出现两次，它的贡献就加两次。去重会让 - # 同一个查询在倒排和全表扫两条路上给出不同的分——实测顶部那条差了 2.32 - qcounts = Counter(t for t in _tokens(query) if len(t) <= 64) - terms = list(qcounts) - if not terms: - return [] + top, _ = await candidates_with_scores(session, collection=collection, query=query, limit=limit) + return top - stmt = select(ChunkTerm.chunk_id, ChunkTerm.term, ChunkTerm.tf).where( - ChunkTerm.term.in_(terms)) + +async def has_terms(session: AsyncSession, collection: str | None = None) -> bool: + """集合里是否已建有倒排词项（区分「真没命中查询词」与「存量升级尚未建倒排」）。""" + stmt = select(ChunkTerm.id).limit(1) if collection: stmt = stmt.where(ChunkTerm.collection == collection) - rows = list(await session.execute(stmt)) - if not rows: - return [] - - # df 直接从命中里数：含有该词的片段数，正是 BM25 要的那个 df - df: dict[str, int] = defaultdict(int) - for _chunk_id, term, _tf in rows: - df[term] += 1 - - stat = select(func.count(Chunk.id), func.sum(Chunk.token_len)) - if collection: - stat = stat.where(Chunk.collection == collection) - total, total_len = (await session.execute(stat)).one() - total = int(total or 0) - if not total: - return [] - avg_len = float(total_len or 0) / total or 1.0 - - # 要 .all()：Result 自带 keys()，dict() 会当它是映射去下标取值， - # 抛 'ChunkedIteratorResult' object is not subscriptable - lengths = dict((await session.execute( - select(Chunk.id, Chunk.token_len).where( - Chunk.id.in_({r[0] for r in rows})))).all()) - - scores: dict[str, float] = defaultdict(float) - for chunk_id, term, tf in rows: - idf = math.log(1 + (total - df[term] + 0.5) / (df[term] + 0.5)) - length = lengths.get(chunk_id) or 1 - denom = tf + _K1 * (1 - _B + _B * length / avg_len) - scores[chunk_id] += qcounts[term] * idf * (tf * (_K1 + 1)) / denom - - ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True) - return ranked[:limit] + return (await session.execute(stmt)).first() is not None async def missing_count(session: AsyncSession, collection: str | None = None) -> int: @@ -117,15 +142,20 @@ async def missing_count(session: AsyncSession, collection: str | None = None) -> async def rebuild(session: AsyncSession, collection: str | None = None) -> int: - """重建倒排。存量片段升级后要跑一次，改了分词器也要。""" - stmt = select(Chunk) - if collection: - stmt = stmt.where(Chunk.collection == collection) - rows = list((await session.execute(stmt)).scalars()) - if not rows: - return 0 - await drop_chunk_terms(session, [r.id for r in rows]) - for row in rows: - await index_chunk(session, row) - await session.commit() - return len(rows) + """按游标分批重建倒排，避免一次性把全库片段加载进内存。""" + total = 0 + last_id = "" + while True: + stmt = select(Chunk).where(Chunk.id > last_id).order_by(Chunk.id.asc()).limit(_REBUILD_BATCH) + if collection: + stmt = stmt.where(Chunk.collection == collection) + batch = list((await session.execute(stmt)).scalars()) + if not batch: + break + last_id = batch[-1].id + await drop_chunk_terms(session, [r.id for r in batch]) + for row in batch: + await index_chunk(session, row) + await session.commit() + total += len(batch) + return total diff --git a/backend/app/memory/kb.py b/backend/app/memory/kb.py index 9c976e63..d93de7a2 100644 --- a/backend/app/memory/kb.py +++ b/backend/app/memory/kb.py @@ -1,9 +1,13 @@ from __future__ import annotations +from collections import OrderedDict +from dataclasses import dataclass +import heapq import re from typing import Any -from sqlalchemy import func, select +import numpy as np +from sqlalchemy import func, or_, select from sqlalchemy.ext.asyncio import AsyncSession from app.db.models import Chunk, Document @@ -18,6 +22,29 @@ _OVERLAP = 120 #: 每批送多少段去算向量。远端接口单次批量有上限（OpenAI 是 2048），而且 #: 一次把上万段全发出去，要么超时要么被限流，中途失败还没有断点 _EMBED_BATCH = 64 +_ID_FETCH_BATCH = 500 +_VEC_SCAN_BATCH = 512 +_VEC_CACHE_MAX_COLLECTIONS = 16 +_VEC_CACHE_MAX_CHUNKS = 200_000 + + +@dataclass +class _VectorCacheEntry: + stamp: tuple[int, str, str, int] + chunk_ids: list[str] + matrix: np.ndarray + + +_VEC_CACHE: OrderedDict[str, _VectorCacheEntry] = OrderedDict() + + +def invalidate_vector_cache(collection: str | None = None) -> None: + """文档增删或重建索引后失效进程内归一化向量矩阵缓存。""" + if collection is None: + _VEC_CACHE.clear() + return + _VEC_CACHE.pop(collection, None) + _VEC_CACHE.pop("__all__", None) def chunk_text(text: str, target: int = _TARGET, overlap: int = _OVERLAP) -> list[str]: @@ -216,10 +243,137 @@ async def process_document( doc.status = "ready" doc.error = "" await session.commit() + invalidate_vector_cache(collection) await session.refresh(doc) return doc +async def _fetch_chunks_by_ids(session: AsyncSession, chunk_ids: list[str]) -> dict[str, Chunk]: + found: dict[str, Chunk] = {} + for i in range(0, len(chunk_ids), _ID_FETCH_BATCH): + part = chunk_ids[i:i + _ID_FETCH_BATCH] + if not part: + continue + for row in (await session.execute(select(Chunk).where(Chunk.id.in_(part)))).scalars(): + found[row.id] = row + return found + + +def _unit_vector(vec: np.ndarray) -> np.ndarray: + arr = np.asarray(vec, dtype=np.float32) + norm = float(np.linalg.norm(arr)) + return arr / norm if norm > 0 else np.zeros_like(arr, dtype=np.float32) + + +async def _vector_top_ids( + session: AsyncSession, + *, + collection: str | None, + query_vec: np.ndarray, + limit: int, +) -> list[str]: + """全集合语义向量路独立 Top-K 召回（不被 BM25 Top-50 前置截断）。 + + 中小型集合缓存 L2 归一化 `float32` 矩阵，单次检索仅需一次 BLAS 矩阵乘法； + 超大集合（> 20 万块）自动退化为游标分批扫描 + 小顶堆，保证常驻内存有界。 + """ + if limit <= 0: + return [] + model_id, dim = embedder_id(), embedder_dim() + stat_stmt = select(func.count(Chunk.id), func.max(Chunk.id)) + if collection: + stat_stmt = stat_stmt.where(Chunk.collection == collection) + total_raw, max_id_raw = (await session.execute(stat_stmt)).one() + total = int(total_raw or 0) + if total == 0: + return [] + q_unit = _unit_vector(query_vec) + if q_unit.shape[0] != dim: + return [] + + cache_key = collection or "__all__" + stamp = (total, str(max_id_raw or ""), model_id, dim) + + if total <= _VEC_CACHE_MAX_CHUNKS: + entry = _VEC_CACHE.get(cache_key) + if entry is None or entry.stamp != stamp: + ids: list[str] = [] + vecs: list[np.ndarray] = [] + last_id = "" + while True: + stmt = ( + select(Chunk.id, Chunk.embedding) + .where(Chunk.id > last_id) + .order_by(Chunk.id.asc()) + .limit(_VEC_SCAN_BATCH) + ) + if collection: + stmt = stmt.where(Chunk.collection == collection) + batch = (await session.execute(stmt)).all() + if not batch: + break + last_id = str(batch[-1][0]) + for cid, blob in batch: + v = from_blob(blob, dim) + ids.append(str(cid)) + vecs.append(_unit_vector(v) if v is not None else np.zeros(dim, dtype=np.float32)) + matrix = np.vstack(vecs).astype(np.float32, copy=False) if vecs else np.zeros((0, dim), dtype=np.float32) + entry = _VectorCacheEntry(stamp=stamp, chunk_ids=ids, matrix=matrix) + _VEC_CACHE[cache_key] = entry + _VEC_CACHE.move_to_end(cache_key) + while len(_VEC_CACHE) > _VEC_CACHE_MAX_COLLECTIONS: + _VEC_CACHE.popitem(last=False) + else: + _VEC_CACHE.move_to_end(cache_key) + + if entry.matrix.shape[0] == 0: + return [] + scores = entry.matrix @ q_unit + n = scores.shape[0] + if n <= limit: + order = sorted(range(n), key=lambda idx: (-float(scores[idx]), entry.chunk_ids[idx])) + return [entry.chunk_ids[idx] for idx in order] + top_idx = np.argpartition(-scores, limit - 1)[:limit] + ordered = sorted((int(idx) for idx in top_idx), key=lambda idx: (-float(scores[idx]), entry.chunk_ids[idx])) + return [entry.chunk_ids[idx] for idx in ordered] + + # 超大集合流式扫描兜底 + heap: list[tuple[float, str]] = [] + last_id = "" + while True: + stmt = ( + select(Chunk.id, Chunk.embedding) + .where(Chunk.id > last_id) + .order_by(Chunk.id.asc()) + .limit(_VEC_SCAN_BATCH) + ) + if collection: + stmt = stmt.where(Chunk.collection == collection) + batch = (await session.execute(stmt)).all() + if not batch: + break + last_id = str(batch[-1][0]) + for cid, blob in batch: + v = from_blob(blob, dim) + sc = float(np.dot(_unit_vector(v), q_unit)) if v is not None else 0.0 + if len(heap) < limit: + heapq.heappush(heap, (sc, str(cid))) + elif sc > heap[0][0]: + heapq.heapreplace(heap, (sc, str(cid))) + heap.sort(key=lambda item: (-item[0], item[1])) + return [cid for _, cid in heap] + + +async def _stale_model_names(session: AsyncSession, collection: str | None) -> list[str]: + stmt = select(Chunk.embed_model).distinct().where( + or_(Chunk.embed_model != embedder_id(), Chunk.embed_dim != embedder_dim()) + ) + if collection: + stmt = stmt.where(Chunk.collection == collection) + names = [str(m or "未知模型") for m in (await session.execute(stmt)).scalars()] + return sorted(set(names)) or ["未知模型"] + + async def search( session: AsyncSession, *, @@ -229,76 +383,79 @@ async def search( alpha: float | None = None, on_degrade: Any = None, ) -> list[dict[str, Any]]: - """混合检索。 + """真·双路混合检索（词面 BM25 独立召回 + 全集合向量矩阵独立召回）。 存量向量和当前 embedder 对不上时**整个查询退回纯关键词**，并通过 on_degrade 把原因交出去。不把对不上的那些当 0 分：那会让它们静默沉底， 用户看到的是结果莫名其妙变差，而不是"这里有一批索引该重建了"。 - - 在此之前这里连维度都不查，换个 embedder 直接 ValueError - （shapes (1536,) and (512,) not aligned），整个检索 500。 """ - # 没显式指定就跟着 embedder 走——写死 0.5 是在给一个没有语义能力的信号 - # 一半权重，实测会把 hit@1 从 88% 拉到 81% if alpha is None: alpha = default_alpha() - # 先用倒排把候选收窄。以前这里是把整个集合载入内存、再在 Python 里重建 - # 一遍 BM25 倒排——每次查询都重新分词一遍所有片段。 - # - # 多捞一些（limit 的 8 倍）：倒排只按关键词收，最终名次还要混进向量那一路， - # 卡得太紧会把"关键词一般但语义很近"的那些提前筛掉。 - rows: list[Chunk] = [] - kw_scores: list[float] | None = None - picked = await inverted.candidates( - session, collection=collection, query=query, limit=max(limit * 8, 50)) - if picked: - found = dict( - (c.id, c) for c in (await session.execute( - select(Chunk).where(Chunk.id.in_([cid for cid, _ in picked])))).scalars()) - pairs = [(found[cid], sc) for cid, sc in picked if cid in found] - rows = [c for c, _ in pairs] - # 倒排算好的分要带下去。在候选子集上重算 BM25，df 和平均长度是另一套数 - kw_scores = [sc for _, sc in pairs] - - if not rows: - # 一个查询词都没命中，或者这批片段还没建倒排（存量数据升级上来就是这样）。 - # 退回全表扫，慢但不会漏——宁可慢，也不要因为索引没建好就说"没搜到" - stmt = select(Chunk) - if collection: - stmt = stmt.where(Chunk.collection == collection) - rows = list((await session.execute(stmt)).scalars()) - kw_scores = None # 全表扫这条路照旧自己算 - if not rows: - return [] - - # 按 id 定序再排。hybrid_rank 用的是稳定排序，分数并列时保留输入顺序—— - # 而倒排那条路和全表扫那条路喂进去的顺序天生不同，同样的查询会因为走了 - # 哪条路给出不同的名次。并列本来就该有个确定的先后，用 id 就够了 - if kw_scores is not None: - paired = sorted(zip(rows, kw_scores), key=lambda t: t[0].id) - rows = [r for r, _ in paired] - kw_scores = [s2 for _, s2 in paired] - else: - rows = sorted(rows, key=lambda r: r.id) - - stale = [r for r in rows if not usable(r.embed_model, r.embed_dim)] - if stale: + total_stale = await stale_count(session, collection) + if total_stale > 0: if on_degrade: - names = sorted({r.embed_model or "未知模型" for r in stale}) - # 报的是整个集合里要重建多少条，不是本次候选里有几条——候选被倒排 - # 收窄过，用它会把问题说小，而用户要照着做的恰恰是"重建多少" - total_stale = await stale_count(session, collection) + names = await _stale_model_names(session, collection) on_degrade( f"{collection or '全部集合'}里有 {total_stale} 条向量由" f"「{'、'.join(names)}」建立，与当前的「{embedder_id()}」不一致，" f"本次已退回纯关键词检索。重建索引后恢复。" ) - alpha = 0.0 # 纯关键词 - vectors: list[Any] = [None] * len(rows) - query_vec = await embed_text(query) + alpha = 0.0 + + query_vec = await embed_text(query) + cand_limit = max(limit * 8, 50) + + # 第一路：倒排索引词面 BM25 Top-K（返回精确全集合 BM25 得分字典） + lex_picked, all_kw_scores = await inverted.candidates_with_scores( + session, collection=collection, query=query, limit=cand_limit, + ) + lex_ids = [cid for cid, _ in lex_picked] + + # 第二路：当启用语义权重时，独立从全集合归一化向量索引召回语义 Top-K，不再被 BM25 Top-50 卡死 + vec_ids = ( + await _vector_top_ids(session, collection=collection, query_vec=query_vec, limit=cand_limit) + if alpha > 0.0 + else [] + ) + + merged_ids = sorted(set(lex_ids) | set(vec_ids)) + rows: list[Chunk] = [] + kw_scores: list[float] | None = None + + if merged_ids: + found = await _fetch_chunks_by_ids(session, merged_ids) + rows = [found[cid] for cid in merged_ids if cid in found] + if lex_picked or await inverted.has_terms(session, collection): + kw_scores = [all_kw_scores.get(r.id, 0.0) for r in rows] + else: + kw_scores = None + else: + if await inverted.has_terms(session, collection): + # 倒排已就绪但没有任何查询词命中且 alpha == 0：所有片段词面分均为 0，只需按 id 定序取前 limit 条 + stmt = select(Chunk).order_by(Chunk.id.asc()).limit(limit) + if collection: + stmt = stmt.where(Chunk.collection == collection) + rows = list((await session.execute(stmt)).scalars()) + kw_scores = [0.0] * len(rows) + else: + # 存量数据升级尚未建倒排：退回全表扫现算 BM25，保证不漏检 + stmt = select(Chunk).order_by(Chunk.id.asc()) + if collection: + stmt = stmt.where(Chunk.collection == collection) + rows = list((await session.execute(stmt)).scalars()) + kw_scores = None + + if not rows: + return [] + + if total_stale > 0 or alpha == 0.0: + vectors: list[Any] = ( + [None] * len(rows) + if total_stale > 0 + else [from_blob(r.embedding, embedder_dim()) for r in rows] + ) else: - query_vec = await embed_text(query) vectors = [from_blob(r.embedding, embedder_dim()) for r in rows] ranked = hybrid_contrib( @@ -313,7 +470,6 @@ async def search( for hit in ranked[:limit]: row = rows[hit.index] score = round(hit.score, 4) - # 关键词那份用减法得出：各自四舍五入之后再相加，会和总分差出一个末位 by_vec = min(round(hit.contrib["vector"], 4), score) out.append( { @@ -324,7 +480,6 @@ async def search( "content": row.content, "score": score, "signals": {k: round(v, 4) for k, v in hit.signals.items()}, - # 两路各贡献了多少分，两者之和等于 score。调试台据此画贡献条 "contrib": {"vector": by_vec, "keyword": round(score - by_vec, 4)}, } ) @@ -334,34 +489,38 @@ async def search( async def reindex( session: AsyncSession, collection: str | None = None, *, on_progress: Any = None, ) -> dict[str, Any]: - """用当前 embedder 重算向量。 - - 换了 embedding 模型之后唯一的恢复手段。按批做：一次几千条 chunk 全塞给 - 远端 embedding 接口，要么超时要么被限流，而中途失败又没有断点。 - on_progress(已完成, 总数) 每批提交之后回调一次。 - """ - stmt = select(Chunk) + """用当前 embedder 按游标分批重算向量并重建倒排，避免全表一次性载入内存。""" + count_stmt = select(func.count(Chunk.id)) if collection: - stmt = stmt.where(Chunk.collection == collection) - rows = list((await session.execute(stmt)).scalars()) - if not rows: - return {"collection": collection, "reindexed": 0, "embedder": embedder_id()} - + count_stmt = count_stmt.where(Chunk.collection == collection) + total = int((await session.execute(count_stmt)).scalar_one() or 0) model_id, dim = embedder_id(), embedder_dim() - batch = _EMBED_BATCH - for i in range(0, len(rows), batch): - part = rows[i:i + batch] + if total == 0: + invalidate_vector_cache(collection) + return {"collection": collection, "reindexed": 0, "embedder": model_id} + + done = 0 + last_id = "" + while True: + stmt = select(Chunk).where(Chunk.id > last_id).order_by(Chunk.id.asc()).limit(_EMBED_BATCH) + if collection: + stmt = stmt.where(Chunk.collection == collection) + part = list((await session.execute(stmt)).scalars()) + if not part: + break + last_id = part[-1].id vectors = await embed_texts([r.content for r in part]) for row, vec in zip(part, vectors): row.embedding = to_blob(vec) row.embed_model = model_id row.embed_dim = dim await session.commit() + done += len(part) if on_progress: - on_progress(min(i + batch, len(rows)), len(rows)) - # 倒排一起重建：两者都是"存量数据升级"的恢复手段，分成两个按钮只会漏点一个 + on_progress(min(done, total), total) await inverted.rebuild(session, collection) - return {"collection": collection, "reindexed": len(rows), "embedder": model_id} + invalidate_vector_cache(collection) + return {"collection": collection, "reindexed": done, "embedder": model_id} async def stale_count(session: AsyncSession, collection: str | None = None) -> int: @@ -370,8 +529,6 @@ async def stale_count(session: AsyncSession, collection: str | None = None) -> i 用 SQL 数而不是把片段全载进来再在 Python 里过一遍——这个数在检索降级时 也要取，那条路径上不该再来一次全表扫。 """ - from sqlalchemy import func, or_ - stmt = select(func.count(Chunk.id)).where( or_(Chunk.embed_model != embedder_id(), Chunk.embed_dim != embedder_dim())) if collection: @@ -383,11 +540,13 @@ async def delete_document(session: AsyncSession, doc_id: str) -> bool: doc = await session.get(Document, doc_id) if not doc: return False + collection = doc.collection chunk_ids = list((await session.execute( select(Chunk.id).where(Chunk.document_id == doc_id))).scalars()) await inverted.drop_chunk_terms(session, chunk_ids) await session.delete(doc) await session.commit() + invalidate_vector_cache(collection) return True diff --git a/backend/app/memory/store.py b/backend/app/memory/store.py index 534d6f65..aebac1a1 100644 --- a/backend/app/memory/store.py +++ b/backend/app/memory/store.py @@ -133,6 +133,7 @@ async def recall( ) results: list[dict[str, Any]] = [] + picked_rows: list[MemoryItem] = [] now = datetime.now(timezone.utc) for idx, score, parts in ranked: row = rows[idx] @@ -140,6 +141,7 @@ async def recall( boosted = score * (0.8 + 0.4 * row.importance) * (1 + min(row.use_count, 10) * 0.01) if boosted < min_score: continue + picked_rows.append(row) results.append( { "id": row.id, @@ -158,11 +160,9 @@ async def recall( if not track: return results # 记一下被召回过，作为下次排序的信号 - for r in results: - item = await session.get(MemoryItem, r["id"]) - if item: - item.use_count += 1 - item.last_used_at = now + for item in picked_rows: + item.use_count += 1 + item.last_used_at = now await session.commit() return results @@ -241,28 +241,37 @@ async def reindex( 路径——记忆这边悄悄退回关键词，而且没有任何恢复手段。 on_progress(已完成, 总数) 每批提交之后回调一次。 """ - stmt = select(MemoryItem) + count_stmt = select(func.count(MemoryItem.id)) if scope: - stmt = stmt.where(MemoryItem.scope == scope) - rows = list((await session.execute(stmt)).scalars()) - if not rows: + count_stmt = count_stmt.where(MemoryItem.scope == scope) + total = int((await session.execute(count_stmt)).scalar_one() or 0) + if total == 0: return 0 model_id, dim = embedder_id(), embedder_dim() from app.memory import kb batch = kb._EMBED_BATCH # 和 kb 那边统一：一次几千条全发给远端接口必被限流 - for i in range(0, len(rows), batch): - part = rows[i:i + batch] + done = 0 + last_id = "" + while True: + stmt = select(MemoryItem).where(MemoryItem.id > last_id).order_by(MemoryItem.id.asc()).limit(batch) + if scope: + stmt = stmt.where(MemoryItem.scope == scope) + part = list((await session.execute(stmt)).scalars()) + if not part: + break + last_id = part[-1].id vectors = await embed_texts([r.content for r in part]) for row, vec in zip(part, vectors): row.embedding = to_blob(vec) row.embed_model = model_id row.embed_dim = dim await session.commit() + done += len(part) if on_progress: - on_progress(min(i + batch, len(rows)), len(rows)) - return len(rows) + on_progress(min(done, total), total) + return done async def list_scopes(session: AsyncSession) -> list[dict[str, Any]]: diff --git a/backend/app/providers/catalog.py b/backend/app/providers/catalog.py index b4f3fe40..7647ed15 100644 --- a/backend/app/providers/catalog.py +++ b/backend/app/providers/catalog.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from typing import Any # -------------------------------------------------------------------------- @@ -187,14 +188,52 @@ def needs_interleaved_beta(model: str) -> bool: return name.startswith(_INTERLEAVED_BETA_PREFIXES) +_OPENAI_REASONING_RE = re.compile(r"^o[134](?:$|[-_])") +#: 用户在「设置 → 模型接入」里填了自定义价格或上下文窗口的模型（由 list_providers 同步） +_CUSTOM_MODELS: dict[str, dict[str, Any]] = {} + + +def sync_provider_models(providers: list[Any]) -> None: + """把当前启用的模型接入里带了自定义 pricing / context 的模型同步进内存表。""" + custom: dict[str, dict[str, Any]] = {} + for p in providers or (): + if not getattr(p, "enabled", True): + continue + for entry in getattr(p, "models", None) or (): + if not isinstance(entry, dict) or not entry.get("id"): + continue + key = _norm(str(entry["id"])) + if not key: + continue + info: dict[str, Any] = {} + pricing = entry.get("pricing") + if isinstance(pricing, dict) and ("input" in pricing or "output" in pricing): + info["pricing"] = { + "input": float(pricing.get("input") or 0.0), + "output": float(pricing.get("output") or 0.0), + **({"cache_read": float(pricing["cache_read"])} if "cache_read" in pricing and pricing["cache_read"] is not None else {}), + **({"cache_creation": float(pricing["cache_creation"])} if "cache_creation" in pricing and pricing["cache_creation"] is not None else {}), + } + ctx_val = entry.get("context") + if isinstance(ctx_val, (int, float)) and not isinstance(ctx_val, bool) and ctx_val > 0: + info["context"] = int(ctx_val) + if info: + custom[key] = info + _CUSTOM_MODELS.clear() + _CUSTOM_MODELS.update(custom) + + def _norm(model: str) -> str: - """去掉 openrouter 之类的前缀，只看模型名本身。""" - return (model or "").split("/")[-1].strip() + """去掉 openrouter 之类的前缀并统一小写，只看模型名本身。""" + return (model or "").split("/")[-1].strip().lower() def supports_sampling(model: str) -> bool: """这个模型还能不能传 temperature / top_p。""" - return not _norm(model).startswith(_NO_SAMPLING_PREFIXES) + name = _norm(model) + if name.startswith(_NO_SAMPLING_PREFIXES) or _OPENAI_REASONING_RE.match(name): + return False + return True def supports_adaptive_thinking(model: str) -> bool: @@ -213,6 +252,8 @@ def supports_effort(model: str) -> bool: def find_pricing(model: str) -> dict[str, float] | None: target = _norm(model) + if target in _CUSTOM_MODELS and "pricing" in _CUSTOM_MODELS[target]: + return _CUSTOM_MODELS[target]["pricing"] pools = [ANTHROPIC_MODELS, OPENAI_MODELS, MOCK_MODELS] pools += [p["models"] for p in COMPATIBLE_PRESETS] for pool in pools: @@ -225,6 +266,8 @@ def find_pricing(model: str) -> dict[str, float] | None: def find_context(model: str) -> int | None: """目录里登记的上下文窗口（token）。没登记的返回 None，由调用方取保守值。""" target = _norm(model) + if target in _CUSTOM_MODELS and "context" in _CUSTOM_MODELS[target]: + return _CUSTOM_MODELS[target]["context"] pools = [ANTHROPIC_MODELS, OPENAI_MODELS, MOCK_MODELS] pools += [p["models"] for p in COMPATIBLE_PRESETS] for pool in pools: @@ -251,12 +294,30 @@ def unpriced_call(data: Any) -> bool: and find_pricing(model) is None -def estimate_cost(model: str, input_tokens: int, output_tokens: int) -> float: - """按目录价估算一次调用的花费（美元）。""" +def estimate_cost( + model: str, + input_tokens: int, + output_tokens: int, + *, + cache_read_tokens: int = 0, + cache_creation_tokens: int = 0, +) -> float: + """按目录价估算一次调用的花费（美元）。带缓存读写明细时按缓存单价折算（默认命中 0.1x、建缓存 1.25x）。""" pricing = find_pricing(model) if not pricing: return 0.0 + inp_rate = pricing.get("input", 0.0) + out_rate = pricing.get("output", 0.0) + if cache_read_tokens <= 0 and cache_creation_tokens <= 0: + return input_tokens / 1_000_000 * inp_rate + output_tokens / 1_000_000 * out_rate + read_tokens = max(0, int(cache_read_tokens)) + write_tokens = max(0, int(cache_creation_tokens)) + uncached = max(0, int(input_tokens) - read_tokens - write_tokens) + read_rate = pricing.get("cache_read", inp_rate * 0.1) + write_rate = pricing.get("cache_creation", inp_rate * 1.25) return ( - input_tokens / 1_000_000 * pricing.get("input", 0.0) - + output_tokens / 1_000_000 * pricing.get("output", 0.0) - ) + uncached * inp_rate + + read_tokens * read_rate + + write_tokens * write_rate + + output_tokens * out_rate + ) / 1_000_000 diff --git a/backend/app/providers/factory.py b/backend/app/providers/factory.py index 5ed3aa85..125fab00 100644 --- a/backend/app/providers/factory.py +++ b/backend/app/providers/factory.py @@ -54,7 +54,9 @@ class ProviderNotConfigured(RuntimeError): async def list_providers(session: AsyncSession) -> list[Provider]: rows = await session.execute(select(Provider).order_by(Provider.created_at, Provider.id)) - return list(rows.scalars()) + items = list(rows.scalars()) + catalog.sync_provider_models(items) + return items #: 点名的接入不存在、已停用时，在「设置 → 模型接入」里能做什么。NAMED_FIX 是整句的前半句；后半句（去哪里换一个 @@ -326,8 +328,16 @@ def build_chat_model(provider: Provider, spec: ModelSpec) -> BaseChatModel: # 且 api_key=None 时不会发出 x-api-key。 import anthropic - client_args = {"base_url": base_url, "api_key": None, "auth_token": api_key, - "max_retries": chat.max_retries} + client_args: dict[str, Any] = { + "base_url": base_url, + "api_key": None, + "auth_token": api_key, + "max_retries": chat.max_retries, + } + if kwargs.get("timeout") is not None: + client_args["timeout"] = kwargs["timeout"] + if kwargs.get("default_headers"): + client_args["default_headers"] = dict(kwargs["default_headers"]) chat._client = anthropic.Anthropic(**client_args) chat._async_client = anthropic.AsyncAnthropic(**client_args) return chat @@ -414,12 +424,52 @@ def _reasoning_aware_openai() -> type: 所以在这里把它捞回 additional_kwargs，thinking_text 再统一读出来。 只加字段，不改 langchain 对正文的任何处理。 """ + from langchain_core.messages import AIMessage from langchain_openai import ChatOpenAI from pydantic import PrivateAttr class ReasoningAwareChatOpenAI(ChatOpenAI): # type: ignore[misc] _reasoning_hint: dict[str, Any] = PrivateAttr(default_factory=dict) + def _get_request_payload( + self, input_: Any, *, stop: list[str] | None = None, **kwargs: Any + ) -> dict: + payload = super()._get_request_payload(input_, stop=stop, **kwargs) + lc_msgs = self._convert_input(input_).to_messages() + ai_reasonings = [ + (m.additional_kwargs or {}).get("reasoning_content") + for m in lc_msgs + if isinstance(m, AIMessage) + ] + raw_msgs = payload.get("messages") + if isinstance(raw_msgs, list) and ai_reasonings: + ai_idx = 0 + for msg_dict in raw_msgs: + if not isinstance(msg_dict, dict) or msg_dict.get("role") != "assistant": + continue + reasoning = ai_reasonings[ai_idx] if ai_idx < len(ai_reasonings) else None + ai_idx += 1 + # DeepSeek Reasoner / Kimi K2 Thinking 等在工具调用多轮往返时要求携带了 tool_calls 的 + # assistant 消息必须原样回传 reasoning_content，否则直接报 400 Missing reasoning_content field + if isinstance(reasoning, str) and reasoning and msg_dict.get("tool_calls"): + msg_dict.setdefault("reasoning_content", reasoning) + return payload + + def _create_chat_result(self, response: Any, generation_info: dict | None = None): + res = super()._create_chat_result(response, generation_info) + raw_dict = response if isinstance(response, dict) else ( + response.model_dump() if hasattr(response, "model_dump") else {} + ) + choices = raw_dict.get("choices") or [] + for idx, gen in enumerate(res.generations): + if idx >= len(choices): + break + msg = (choices[idx] or {}).get("message") or {} + text = msg.get("reasoning_content") or msg.get("reasoning") + if isinstance(text, str) and text: + gen.message.additional_kwargs["reasoning_content"] = text + return res + def _convert_chunk_to_generation_chunk( self, chunk: dict, default_chunk_class: type, base_generation_info: dict | None ): @@ -458,19 +508,19 @@ def _reasoning_aware_openai() -> type: raise e from first async def _astream(self, messages: Any, stop: Any = None, run_manager: Any = None, **kwargs: Any): - """流式调用；服务方不认思考控制参数或 stream_options 时去掉它重发一次，之后这个接口不再带它。 + """流式调用；服务方不认思考控制参数或 stream_options 时去掉它重发一次，之后这个接口+模型不再带它。 只在还没收到任何一块时重发：拒绝都发生在请求刚发出去的时候，重发不会重复输出。""" usage = not self._use_responses_api({**kwargs, **self.model_kwargs}) \ and self._should_stream_usage(kwargs.get("stream_usage"), **kwargs) - key = _endpoint_key(self) - hint_key = (key, str(getattr(self, "model_name", "") or "")) - use_hint = bool(self._reasoning_hint) and hint_key not in _NO_REASONING_HINT + endpoint = _endpoint_key(self) + model_key = (endpoint, str(getattr(self, "model_name", "") or "")) + use_hint = bool(self._reasoning_hint) and model_key not in _NO_REASONING_HINT if use_hint: started = False hint_kwargs = _merge_reasoning_hint(kwargs, self._reasoning_hint) - if not usage or key in _NO_STREAM_USAGE: + if not usage or model_key in _NO_STREAM_USAGE: hint_kwargs = {k: v for k, v in hint_kwargs.items() if k != "stream_options"} | {"stream_usage": False} try: async for chunk in super()._astream(messages, stop, run_manager, **hint_kwargs): @@ -480,17 +530,18 @@ def _reasoning_aware_openai() -> type: except Exception as e: # noqa: BLE001 if started: raise - if _rejects_stream_options_specifically(e) and usage and key not in _NO_STREAM_USAGE: + if _rejects_stream_options_specifically(e) and usage and model_key not in _NO_STREAM_USAGE: # 明确点名拒绝 stream_options：剥离 stream_options 但保留思考控制参数再试 no_usage_hint_kwargs = { k: v for k, v in hint_kwargs.items() if k != "stream_options" } | {"stream_usage": False} try: async for chunk in super()._astream(messages, stop, run_manager, **no_usage_hint_kwargs): - if key not in _NO_STREAM_USAGE: - _NO_STREAM_USAGE.add(key) + if model_key not in _NO_STREAM_USAGE: + _NO_STREAM_USAGE.add(model_key) logger.warning( - "模型接入 %s 不支持流式用量回传（%s），已改为不请求用量", key, str(e)[:200] + "模型接入 %s/%s 不支持流式用量回传（%s），已改为不请求用量", + model_key[0], model_key[1], str(e)[:200], ) yield chunk return @@ -500,24 +551,24 @@ def _reasoning_aware_openai() -> type: raise # 剥离思考控制参数后走下方常规流式逻辑；若成则记入 _NO_REASONING_HINT yielded_after_strip = False - async for chunk in self._astream_without_hint(messages, stop, run_manager, usage=usage, key=key, **kwargs): - if not yielded_after_strip and hint_key not in _NO_REASONING_HINT: - _NO_REASONING_HINT.add(hint_key) + async for chunk in self._astream_without_hint(messages, stop, run_manager, usage=usage, key=model_key, **kwargs): + if not yielded_after_strip and model_key not in _NO_REASONING_HINT: + _NO_REASONING_HINT.add(model_key) logger.warning( "模型接入 %s/%s 不支持思考控制参数（%s），已自动剥离", - hint_key[0], - hint_key[1], + model_key[0], + model_key[1], str(e)[:200], ) yielded_after_strip = True yield chunk return - async for chunk in self._astream_without_hint(messages, stop, run_manager, usage=usage, key=key, **kwargs): + async for chunk in self._astream_without_hint(messages, stop, run_manager, usage=usage, key=model_key, **kwargs): yield chunk async def _astream_without_hint( - self, messages: Any, stop: Any = None, run_manager: Any = None, *, usage: bool, key: str, **kwargs: Any + self, messages: Any, stop: Any = None, run_manager: Any = None, *, usage: bool, key: tuple[str, str], **kwargs: Any ): first: Exception | None = None if usage and key not in _NO_STREAM_USAGE: @@ -536,9 +587,12 @@ def _reasoning_aware_openai() -> type: try: async for chunk in super()._astream(messages, stop, run_manager, **kwargs): if first is not None and key not in _NO_STREAM_USAGE: - # 去掉它就成了：记下这个接口不认，之后不再每次先失败一次 + # 去掉它就成了：记下这个接口+模型不认，之后不再每次先失败一次 _NO_STREAM_USAGE.add(key) - logger.warning("模型接入 %s 不支持流式用量回传（%s），已改为不请求用量", key, str(first)[:200]) + logger.warning( + "模型接入 %s/%s 不支持流式用量回传（%s），已改为不请求用量", + key[0], key[1], str(first)[:200], + ) yield chunk except Exception as e: if first is None: @@ -548,8 +602,8 @@ def _reasoning_aware_openai() -> type: return ReasoningAwareChatOpenAI -#: 认定不支持 stream_options 的接口（base_url）。进程内有效：重启后再试一次，服务方可能已经升级 -_NO_STREAM_USAGE: set[str] = set() +#: 认定不支持 stream_options 的 (base_url, model)。按接口+模型记忆，避免统一网关下单个旧模型连坐其它模型 +_NO_STREAM_USAGE: set[tuple[str, str]] = set() #: 认定不支持思考控制参数（reasoning_effort / enable_thinking / thinking）的 (base_url, model) _NO_REASONING_HINT: set[tuple[str, str]] = set() diff --git a/backend/app/providers/usage.py b/backend/app/providers/usage.py index dbf029d6..ef570c20 100644 --- a/backend/app/providers/usage.py +++ b/backend/app/providers/usage.py @@ -47,7 +47,7 @@ from langchain_core.tracers.context import register_configure_hook #: 取不到模型名的调用记在这个名下 UNKNOWN_MODEL = "unknown" #: 每个模型累计的几项（unreported 用量没报全的次数、unfinished 没做完的次数） -_COUNTERS = ("input_tokens", "output_tokens", "calls", "unreported", "unfinished") +_COUNTERS = ("input_tokens", "output_tokens", "cache_read_tokens", "cache_creation_tokens", "calls", "unreported", "unfinished") def _abandoned(error: BaseException) -> bool: @@ -112,6 +112,10 @@ class Meter(BaseCallbackHandler): if usage: row["input_tokens"] += int(usage.get("input_tokens") or 0) row["output_tokens"] += int(usage.get("output_tokens") or 0) + details = usage.get("input_token_details") or {} + if isinstance(details, dict): + row["cache_read_tokens"] += int(details.get("cache_read") or 0) + row["cache_creation_tokens"] += int(details.get("cache_creation") or 0) else: row["unreported"] += 1 @@ -141,6 +145,10 @@ class Meter(BaseCallbackHandler): reported = getattr(partial, "usage_metadata", None) or {} row["input_tokens"] += int(reported.get("input_tokens") or 0) row["output_tokens"] += int(reported.get("output_tokens") or 0) + details = reported.get("input_token_details") or {} + if isinstance(details, dict): + row["cache_read_tokens"] += int(details.get("cache_read") or 0) + row["cache_creation_tokens"] += int(details.get("cache_creation") or 0) def _rows(self) -> dict[str, dict[str, int]]: """累计的一份拷贝，开始了还没结束回调的调用算成没做完的（调用方持锁）。""" @@ -180,9 +188,21 @@ class Meter(BaseCallbackHandler): unpriced = row["calls"] else: unpriced = 0 if free else row["unreported"] + cache_read = int(row.get("cache_read_tokens") or 0) + cache_create = int(row.get("cache_creation_tokens") or 0) + if cache_read > 0 or cache_create > 0: + try: + raw_cost = estimate_cost( + model, row["input_tokens"], row["output_tokens"], + cache_read_tokens=cache_read, cache_creation_tokens=cache_create, + ) + except TypeError: + raw_cost = estimate_cost(model, row["input_tokens"], row["output_tokens"]) + else: + raw_cost = estimate_cost(model, row["input_tokens"], row["output_tokens"]) by_model[model] = {"input_tokens": row["input_tokens"], "output_tokens": row["output_tokens"], "calls": row["calls"], - "cost_usd": round(estimate_cost(model, row["input_tokens"], row["output_tokens"]), 6), + "cost_usd": round(raw_cost, 6), "unpriced_calls": unpriced, "unfinished_calls": row["unfinished"]} total = {key: sum(r[key] for r in by_model.values()) for key in ("input_tokens", "output_tokens", "calls", "unpriced_calls", "unfinished_calls")} diff --git a/backend/app/records/actions.py b/backend/app/records/actions.py index aa4fb6da..c90f19c0 100644 --- a/backend/app/records/actions.py +++ b/backend/app/records/actions.py @@ -79,8 +79,8 @@ async def append_action(args: dict[str, Any], *, idem_key: str | None, run_id: s environment=environment) try: columns = columns_of(table) - # 多进程时几个进程会往同一个记录表文件里追加：库里的锁（store.write_lock）一个接一个，进程里的 _WRITE_LOCK 照旧 - async with store.write_lock(): + # 多进程时几个进程会往同一个记录表文件里追加：按表键持库里的锁（store.write_lock），进程里的文件锁照旧 + async with store.write_lock(table.table_key): done = await asyncio.to_thread( store.append, path_of(table), columns, values, version=table.version, run_id=run_id, actor=actor, action_key=action_key_of(idem_key), supersedes=supersedes) diff --git a/backend/app/records/listing.py b/backend/app/records/listing.py index 5e6562d5..8415c996 100644 --- a/backend/app/records/listing.py +++ b/backend/app/records/listing.py @@ -37,7 +37,7 @@ async def _masked(session: AsyncSession, row: Any) -> set[str]: def _shown(item: dict[str, Any], version: int, masked: set[str], viewer: str | None) -> dict[str, Any]: """一行交给看的人的样子：别人提交的行，遮罩列有值的换成「已遮罩」（P4）。""" - from app.api.evidence import MASKED + from app.engine.evidence_reader import MASKED out = {**item, "_version": version} if not masked or (viewer and item.get("_submitted_by") == viewer): diff --git a/backend/app/records/store.py b/backend/app/records/store.py index 5cd17f62..44afda35 100644 --- a/backend/app/records/store.py +++ b/backend/app/records/store.py @@ -46,15 +46,29 @@ NAME_MAX = 64 _KEY = re.compile(r"^[A-Za-z0-9_-]{1,64}$") #: 列名里不许出现的字符：引号（拼进 SQL 的标识符）、控制字符 _BAD_NAME = re.compile(r"[\"`\[\]\x00-\x1f\x7f]") -#: 所有写入（建表、加列、追加）共用的锁。写都是几毫秒的短事务，不按表分锁也不会互相拖住 +#: 所有写入（建表、加列、追加）按文件路径/表键分锁；保留 _WRITE_LOCK 供直接引用方兜底 _WRITE_LOCK = threading.Lock() +_LOCKS_GUARD = threading.Lock() +_PATH_LOCKS: dict[str, threading.Lock] = {} -def write_lock() -> Any: +def _path_lock(path: Path) -> threading.Lock: + key = os.path.realpath(os.fspath(path)) + with _LOCKS_GUARD: + lock = _PATH_LOCKS.get(key) + if lock is None: + lock = threading.Lock() + _PATH_LOCKS[key] = lock + return lock + + +def write_lock(table_key: str | None = None) -> Any: """跨进程的写锁（app/cluster/mutex：PostgreSQL 时是咨询锁，SQLite 只有一个进程、就是本进程的锁）。异步的调用方在把写入 - 放进线程之前持着它（async with）。""" + 放进线程之前持着它（async with）。给 table_key 时按表分锁，不同记录表的写入互不阻塞。""" from app.cluster import mutex + if table_key and _KEY.match(table_key): + return mutex.lock(f"records-store:{table_key}") return mutex.lock("records-store") @@ -137,6 +151,19 @@ def table_path(table_key: str, version: int) -> Path: return records_root() / f"{table_key}.v{version}.db" +def archive_path(table_key: str) -> Path: + """data/records/.archive.db：超过 SQLite 挂库上限的早期版本按物理表 rows_v 增量汇总在这里。""" + if not _KEY.match(table_key or ""): + raise ValueError("记录表的键格式不对") + return records_root() / f"{table_key}.archive.db" + + +def archive_table_name(version: int) -> str: + if not isinstance(version, int) or version < 1: + raise ValueError("记录表的版本号需要是正整数") + return f"rows_v{version}" + + def _q(name: str) -> str: return '"' + name.replace('"', '""') + '"' @@ -146,10 +173,20 @@ def _lit(value: str) -> str: return "'" + str(value).replace("'", "''") + "'" +def _enable_wal(conn: sqlite3.Connection) -> None: + try: + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=NORMAL") + except sqlite3.Error: + pass + + def _rw(path: Path) -> sqlite3.Connection: if not path.is_file(): raise RecordsRefused("records.table_missing", "记录表的数据文件不见了，请联系管理员") - return open_checked_sqlite(str(path), readonly=False) + conn = open_checked_sqlite(str(path), readonly=False) + _enable_wal(conn) + return conn def _ro(path: Path) -> sqlite3.Connection: @@ -166,15 +203,18 @@ def _ro(path: Path) -> sqlite3.Connection: def create_file(path: Path, columns: list[Column]) -> None: """新建一个表版本的文件。路径上已有文件时：没有任何行（上次建了一半、登记没提交）就重建，有行就拒绝—— 有数据的文件绝不覆盖。""" - with _WRITE_LOCK: + with _path_lock(path): if path.exists(): if _row_count(path) > 0: raise RecordsRefused("records.file_exists", f"记录表文件 {path.name} 已经有数据，不能重建") path.unlink() + for suffix in ("-wal", "-shm"): + Path(f"{path}{suffix}").unlink(missing_ok=True) path.parent.mkdir(parents=True, exist_ok=True) tmp = path.with_name(f".{path.name}.tmp-{new_id()[:12]}") conn = open_checked_sqlite(str(tmp), readonly=False) try: + _enable_wal(conn) business = "".join(f", {_q(c.name)} {COLUMN_TYPES[c.type]}" for c in columns) conn.execute( f"CREATE TABLE {_q(TABLE)} (" @@ -193,7 +233,7 @@ def add_columns(path: Path, columns: list[Column]) -> None: """兼容的改动（只加可选字段）：在同一个版本上 ALTER TABLE ADD COLUMN。已有的列跳过（重做一次也没事）。""" if not columns: return - with _WRITE_LOCK: + with _path_lock(path): conn = _rw(path) try: have = {str(r[1]).lower() for r in conn.execute(f"PRAGMA table_info({_q(TABLE)})")} @@ -326,7 +366,7 @@ def append(path: Path, columns: list[Column], values: Any, *, version: int, run_ missing = [c.label or c.name for c in columns if c.required and row[c.name] in (None, "")] if missing: raise RecordsRefused("records.required_missing", f"必填的字段没有值：{'、'.join(missing)}") - with _WRITE_LOCK: + with _path_lock(path): conn = _rw(path) try: conn.execute("BEGIN IMMEDIATE") @@ -411,7 +451,7 @@ def union_compatible(new: Column, old: Column) -> bool: def view_select(columns: list[Column], *, high_water: int | None, scope: str, viewer: str | None, - schema: str = "main", present: list[Column] | None = None) -> str: + schema: str = "main", present: list[Column] | None = None, table: str = TABLE) -> str: """临时视图（和看板、记录列表）看到的那一份： - _seq ≤ 高水位（None 为不限，现在有的都算）； @@ -430,7 +470,7 @@ def view_select(columns: list[Column], *, high_water: int | None, scope: str, vi else f"NULL AS {_q(c.name)}" cols = ", ".join([*(f"r.{_q(n)}" for n in SYSTEM_COLUMNS), *(pick(c) for c in columns)]) - src = f"{_q(schema)}.{_q(TABLE)}" + src = f"{_q(schema)}.{_q(table)}" where = [] if high_water is not None: seq = int(high_water) @@ -449,11 +489,96 @@ def view_select(columns: list[Column], *, high_water: int | None, scope: str, vi @dataclass(frozen=True) class OlderVersion: """跨版本汇总时并进来的一个旧表版本：挂在查询连接上的库名（ATTACH … AS schema）、那一版的业务列、这次查询的高水位 - （None 为不限）。""" + （None 为不限），以及该版本所在的物理表名（直接挂库时是 records，超挂库上限汇总进归档库时是 rows_v）。""" schema: str columns: list[Column] high_water: int | None + table: str = TABLE + + +@dataclass(frozen=True) +class ArchiveSpec: + """需要增量同步进单库多物理表归档（.archive.db）的旧版本描述。""" + + version: int + path: str + columns: list[Column] + + +def sync_archive(table_key: str, specs: list[ArchiveSpec] | tuple[ArchiveSpec, ...]) -> Path: + """把超挂库上限的旧版本增量同步进单库多物理表归档文件（rows_v），规避 SQLite 最多 10 个 ATTACH 库的限制。 + + 行只追加、不修改：已同步到相同列结构且 _seq 无新增的版本 O(1) 跳过；退役前发起的在途运行补写了新行时，只复制 + _seq > meta_max_seq 的增量行。""" + dest = archive_path(table_key) + if not specs: + return dest + dest.parent.mkdir(parents=True, exist_ok=True) + with _path_lock(dest): + arch = open_checked_sqlite(str(dest), readonly=False) + try: + _enable_wal(arch) + arch.execute( + "CREATE TABLE IF NOT EXISTS _archive_meta (" + "version INTEGER PRIMARY KEY, max_seq INTEGER NOT NULL, col_sig TEXT NOT NULL)" + ) + meta = { + int(r[0]): (int(r[1]), str(r[2])) + for r in arch.execute("SELECT version, max_seq, col_sig FROM _archive_meta").fetchall() + } + dirty = False + for spec in specs: + src_file = Path(spec.path) + if not src_file.is_file(): + continue + src = _ro(src_file) + try: + info = [(str(r[1]), str(r[2] or "TEXT")) for r in src.execute(f"PRAGMA table_info({_q(TABLE)})") + if str(r[1]) != ACTION_KEY_COLUMN] + src_max = int(src.execute(f"SELECT coalesce(max(_seq), 0) FROM {_q(TABLE)}").fetchone()[0] or 0) + sig = json.dumps(info, ensure_ascii=False) + tbl = archive_table_name(spec.version) + prev = meta.get(spec.version) + if prev is not None and prev[1] == sig and prev[0] == src_max: + continue + col_names = [name for name, _ in info] + select_cols = ", ".join(_q(n) for n in col_names) + placeholders = ", ".join("?" for _ in col_names) + if prev is None or prev[1] != sig: + arch.execute(f"DROP TABLE IF EXISTS {_q(tbl)}") + ddl_cols = ", ".join( + "_seq INTEGER PRIMARY KEY" if name == "_seq" else f"{_q(name)} {col_type}" + for name, col_type in info + ) + arch.execute(f"CREATE TABLE {_q(tbl)} ({ddl_cols})") + arch.execute(f"CREATE INDEX ix_{tbl}_supersedes ON {_q(tbl)} (_supersedes)") + arch.execute(f"CREATE INDEX ix_{tbl}_submitted_by ON {_q(tbl)} (_submitted_by)") + since_seq = 0 + else: + since_seq = prev[0] + cur = src.execute( + f"SELECT {select_cols} FROM {_q(TABLE)} WHERE _seq > ? ORDER BY _seq ASC", + (since_seq,), + ) + insert_sql = f"INSERT OR REPLACE INTO {_q(tbl)} ({select_cols}) VALUES ({placeholders})" + while batch := cur.fetchmany(500): + arch.executemany(insert_sql, batch) + arch.execute( + "INSERT OR REPLACE INTO _archive_meta (version, max_seq, col_sig) VALUES (?, ?, ?)", + (spec.version, src_max, sig), + ) + dirty = True + finally: + src.close() + if dirty: + arch.commit() + except BaseException: + arch.rollback() + raise + finally: + arch.close() + return dest def view_prelude(columns: list[Column], *, high_water: int | None, scope: str, viewer: str | None, @@ -465,7 +590,7 @@ def view_prelude(columns: list[Column], *, high_water: int | None, scope: str, v 列照这一版的来，旧版本没有的列补 NULL；每一版各按自己的高水位、自己文件里的更正判，行级过滤同一个。""" parts = [view_select(columns, high_water=high_water, scope=scope, viewer=viewer)] parts += [view_select(columns, high_water=o.high_water, scope=scope, viewer=viewer, schema=o.schema, - present=o.columns) for o in older] + present=o.columns, table=o.table) for o in older] return (f"DROP VIEW IF EXISTS temp.{_q(TABLE)}", f"CREATE TEMP VIEW {_q(TABLE)} AS " + " UNION ALL ".join(parts)) diff --git a/backend/app/records/tables.py b/backend/app/records/tables.py index d63d4c28..787dc8df 100644 --- a/backend/app/records/tables.py +++ b/backend/app/records/tables.py @@ -188,7 +188,7 @@ async def _free_name(session: AsyncSession, base: str) -> str: async def _register(session: AsyncSession, *, key: str, version: int, columns: list[Column], label: str, app_id: str | None, blueprint_id: str | None) -> RecordsTable: path = store.table_path(key, version) - async with store.write_lock(): + async with store.write_lock(key): await asyncio.to_thread(store.create_file, path, columns) base = f"{SOURCE_PREFIX}{re.sub(r'[^a-z0-9]', '', key.lower())[:12] or 'table'}" name = await _free_name(session, base if version == 1 else f"{base}_v{version}") @@ -373,7 +373,7 @@ async def _sync_registration(session: AsyncSession, row: RecordsTable, wanted: l have = columns_of(row) known = {h.name.lower() for h in have} added = [c for c in wanted if c.name.lower() not in known] - async with store.write_lock(): + async with store.write_lock(row.table_key): await asyncio.to_thread(store.add_columns, store.table_path(row.table_key, row.version), added) by_name = {w.name.lower(): w for w in wanted} # 列的顺序：原有的在前（照原来的顺序），新加的在后——和文件里的列序一致。删掉的列（只在不兼容时出现）这里不会有 diff --git a/backend/app/records/views.py b/backend/app/records/views.py index 51d9e255..abd231c1 100644 --- a/backend/app/records/views.py +++ b/backend/app/records/views.py @@ -22,6 +22,7 @@ import copy import logging from dataclasses import dataclass, field from datetime import datetime +from pathlib import Path from typing import Any from sqlalchemy.ext.asyncio import AsyncSession @@ -35,8 +36,10 @@ logger = logging.getLogger(__name__) #: {"version", "seq": None, "missing": True}——汇总里没有它，印章（apps/resolve.stamp_of）、查询快照照实说 MISSING = "missing" PIN_KEYS = ("name", "origin", "table", "version", "seq", "older", "rows", "viewer") -#: 旧版本挂在查询连接上的库名：records_v<版本> +#: 旧版本挂在查询连接上的库名：records_v<版本>；超过 SQLite 10 库挂接上限时，早期版本汇总进单库多表 records_archive ATTACH_PREFIX = "records_v" +ARCHIVE_SCHEMA = "records_archive" +MAX_DIRECT_ATTACH = 8 @dataclass @@ -89,27 +92,70 @@ class Older: path: str columns: list[store.Column] high_water: int | None + schema_name: str | None = None + table_name: str = store.TABLE @property def schema(self) -> str: - return f"{ATTACH_PREFIX}{int(self.version)}" + return self.schema_name or f"{ATTACH_PREFIX}{int(self.version)}" + + +def _infer_table_key(older: tuple[Older, ...] | list[Older]) -> str | None: + for item in older: + stem = Path(item.path).name + if stem.endswith(".db") and ".v" in stem[:-3]: + key, _, ver = stem[:-3].rpartition(".v") + if key and ver.isdigit(): + return key + return None + + +def _materialize_older(older: tuple[Older, ...] | list[Older], *, table_key: str | None = None) -> tuple[Older, ...]: + items = tuple(older) + if len(items) <= MAX_DIRECT_ATTACH or any(o.schema_name is not None for o in items): + return items + key = table_key or _infer_table_key(items) + if not key: + return items + direct = items[:MAX_DIRECT_ATTACH - 1] + overflow = items[MAX_DIRECT_ATTACH - 1:] + present = [o for o in overflow if Path(o.path).is_file()] + if not present: + return items + arch = store.sync_archive( + key, + [store.ArchiveSpec(version=o.version, path=o.path, columns=o.columns) for o in present], + ) + arch_str = str(arch) + present_versions = {o.version for o in present} + mapped = tuple( + Older(o.version, arch_str, o.columns, o.high_water, schema_name=ARCHIVE_SCHEMA, + table_name=store.archive_table_name(o.version)) + if o.version in present_versions else o + for o in overflow + ) + return (*direct, *mapped) def bind_view(source: Any, columns: list[store.Column], *, high_water: int | None, rows: str = "all", viewer: str | None = None, older: list[Older] | tuple[Older, ...] = (), - missing: list[int] | tuple[int, ...] = ()) -> RecordsView: + missing: list[int] | tuple[int, ...] = (), table_key: str | None = None) -> RecordsView: """数据源行 + 高水位 + 行级过滤（+ 并进来的旧版本；missing 是数据文件不在了、并不进来的旧版本号）→ RecordsView。""" rows = rows if rows in ("all", "mine", "none") else "none" - older = tuple(older) + older = _materialize_older(older, table_key=table_key) + attach_map: dict[str, str] = {} + for o in older: + attach_map.setdefault(o.schema, o.path) return RecordsView( id=source.id, name=source.name, database=source.database, options=dict(source.options or {}), description=source.description or "", schema_cache=copy.deepcopy(source.schema_cache or {}), schema_synced_at=getattr(source, "schema_synced_at", None), enabled=bool(source.enabled), last_check=getattr(source, "last_check", None), high_water=high_water, rows=rows, viewer=viewer, prelude=store.view_prelude(columns, high_water=high_water, scope=rows, viewer=viewer, - older=[store.OlderVersion(o.schema, o.columns, o.high_water) for o in older]), - attach=tuple((o.schema, o.path) for o in older), - forbid_schemas=("main", *(o.schema for o in older)), + older=[store.OlderVersion(o.schema, o.columns, o.high_water, o.table_name) + for o in older]), + attach=tuple(attach_map.items()), + forbid_schemas=("main", *attach_map.keys()), older=(*({"version": o.version, "seq": o.high_water} for o in older), *({"version": int(v), "seq": None, MISSING: True} for v in missing))) @@ -145,6 +191,7 @@ async def resolve(session: AsyncSession, source: Any, entry: Any = None) -> Reco from app.records.tables import columns_of, path_of, table_of_source table = await table_of_source(session, source.id) + table_key = table.table_key if table is not None else None columns = columns_of(table) if table is not None else columns_of_source(source) older_rows = await _older_versions(session, table) if isinstance(entry, dict): @@ -156,16 +203,21 @@ async def resolve(session: AsyncSession, source: Any, entry: Any = None) -> Reco older = [Older(row.version, str(path_of(row)), columns_of(row), int(pinned[row.version]) if isinstance(pinned[row.version], int) else 0) for row in older_rows if row.version in pinned] + if len(older) > MAX_DIRECT_ATTACH: + older = list(await asyncio.to_thread(_materialize_older, older, table_key=table_key)) return bind_view(source, columns, high_water=int(seq) if isinstance(seq, int) else 0, rows=str(entry.get("rows") or "none"), viewer=entry.get("viewer"), older=older, - missing=missing) + missing=missing, table_key=table_key) older = [Older(row.version, str(path_of(row)), columns_of(row), None) for row in older_rows if path_of(row).is_file()] + if len(older) > MAX_DIRECT_ATTACH: + older = list(await asyncio.to_thread(_materialize_older, older, table_key=table_key)) missing = [row.version for row in older_rows if not path_of(row).is_file()] if missing: logger.warning("记录表 %s 第 %s 版的数据文件不在了，跨版本汇总里不含这几版", table.table_key, "、".join(str(v) for v in missing)) - return bind_view(source, columns, high_water=None, rows="all", older=older, missing=missing) + return bind_view(source, columns, high_water=None, rows="all", older=older, missing=missing, + table_key=table_key) def default_view(source: Any) -> RecordsView: diff --git a/backend/app/sandbox/bubblewrap_sandbox.py b/backend/app/sandbox/bubblewrap_sandbox.py index 09f13be8..06986320 100644 --- a/backend/app/sandbox/bubblewrap_sandbox.py +++ b/backend/app/sandbox/bubblewrap_sandbox.py @@ -184,58 +184,82 @@ class BubblewrapSandbox(Sandbox): cmd_prefix = cmd_prefix if cmd_prefix is not None else interpreter.python_argv() entry = entry_name(ext) cmd = [*cmd_prefix, f"/work/{entry}"] - workdir = self._session_dir(session_id or f"tmp-{int(time.time() * 1000)}") - atomic.write_text(workdir / entry, code) - for rel, content in (files or {}).items(): - target = (workdir / rel).resolve() - if not str(target).startswith(str(workdir.resolve())): - return ExecResult(ok=False, backend=self.name, error=f"非法文件路径：{rel}") - target.parent.mkdir(parents=True, exist_ok=True) - atomic.write_text(target, content) - - argv = self._argv(workdir, limits, cmd, env) - - started = time.perf_counter() + ephemeral = not session_id + workdir = self._session_dir(session_id or f"tmp-{int(time.time() * 1000)}-{os.urandom(4).hex()}") + work_root = workdir.resolve() try: - proc = await asyncio.create_subprocess_exec( - *argv, - # bwrap 自己也只拿最小的环境：沙箱里的进程靠上面的 --setenv，不从这里继承 - env={"PATH": os.environ.get("PATH", "/usr/bin:/bin")}, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - preexec_fn=self._preexec(limits), - ) - out, err = await asyncio.wait_for(proc.communicate(), timeout=limits.timeout) - except asyncio.TimeoutError: - try: - os.killpg(os.getpgid(proc.pid), 9) - except (ProcessLookupError, PermissionError): - proc.kill() - await proc.wait() - return ExecResult( - ok=False, backend=self.name, timed_out=True, exit_code=124, - duration_ms=int((time.perf_counter() - started) * 1000), - error=f"执行超过 {limits.timeout} 秒，已终止", - ) - except Exception as e: # noqa: BLE001 - # 起进程和收输出在同一个 try 里。首行只说原因，类名和 errno 这些留给日志 - logger.warning("%s 沙箱进程出错：%s", self.name, raw(e)) - return ExecResult(ok=False, backend=self.name, error=f"沙箱进程出错：{describe_exception(e)}") + atomic.write_text(workdir / entry, code) + for rel, content in (files or {}).items(): + target = (workdir / rel).resolve() + if not target.is_relative_to(work_root): + return ExecResult(ok=False, backend=self.name, error=f"非法文件路径：{rel}") + target.parent.mkdir(parents=True, exist_ok=True) + atomic.write_text(target, content) - stdout, t1 = truncate(out.decode(errors="replace"), limits.max_output) - stderr, t2 = truncate(err.decode(errors="replace"), limits.max_output) - code_ = proc.returncode or 0 - return ExecResult( - ok=code_ == 0, - exit_code=code_, - stdout=stdout, - stderr=stderr, - truncated=t1 or t2, - timed_out=code_ in (-24, 152), - backend=self.name, - duration_ms=int((time.perf_counter() - started) * 1000), - files_written=[p.name for p in workdir.iterdir() if p.is_file() and not p.name.startswith(ENTRY_PREFIX)][:50], - ) + argv = self._argv(workdir, limits, cmd, env) + + started = time.perf_counter() + try: + proc = await asyncio.create_subprocess_exec( + *argv, + # bwrap 自己也只拿最小的环境：沙箱里的进程靠上面的 --setenv，不从这里继承 + env={"PATH": os.environ.get("PATH", "/usr/bin:/bin")}, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + preexec_fn=self._preexec(limits), + ) + except Exception as e: # noqa: BLE001 + logger.warning("%s 沙箱进程出错：%s", self.name, raw(e)) + return ExecResult(ok=False, backend=self.name, error=f"沙箱进程出错：{describe_exception(e)}") + + try: + out, err = await asyncio.wait_for(proc.communicate(), timeout=limits.timeout) + except asyncio.TimeoutError: + self._kill(proc) + await proc.wait() + return ExecResult( + ok=False, backend=self.name, timed_out=True, exit_code=124, + duration_ms=int((time.perf_counter() - started) * 1000), + error=f"执行超过 {limits.timeout} 秒，已终止", + ) + except Exception as e: # noqa: BLE001 + if getattr(proc, "returncode", 0) is None: + self._kill(proc) + await proc.wait() + logger.warning("%s 沙箱进程出错：%s", self.name, raw(e)) + return ExecResult(ok=False, backend=self.name, error=f"沙箱进程出错：{describe_exception(e)}") + except BaseException: + if getattr(proc, "returncode", 0) is None: + self._kill(proc) + await proc.wait() + raise + + stdout, t1 = truncate(out.decode(errors="replace"), limits.max_output) + stderr, t2 = truncate(err.decode(errors="replace"), limits.max_output) + code_ = proc.returncode or 0 + return ExecResult( + ok=code_ == 0, + exit_code=code_, + stdout=stdout, + stderr=stderr, + truncated=t1 or t2, + timed_out=code_ in (-24, 152), + backend=self.name, + duration_ms=int((time.perf_counter() - started) * 1000), + files_written=[p.name for p in workdir.iterdir() if p.is_file() and not p.name.startswith(ENTRY_PREFIX)][:50], + ) + finally: + if ephemeral: + root_resolved = self._root.resolve() + if work_root != root_resolved and work_root.is_relative_to(root_resolved): + shutil.rmtree(workdir, ignore_errors=True) + + @staticmethod + def _kill(proc: asyncio.subprocess.Process) -> None: + try: + os.killpg(os.getpgid(proc.pid), 9) + except (ProcessLookupError, PermissionError): + proc.kill() async def cleanup(self, session_id: str) -> None: await asyncio.to_thread(shutil.rmtree, self._session_dir(session_id), True) diff --git a/backend/app/sandbox/local_sandbox.py b/backend/app/sandbox/local_sandbox.py index 378e6682..b191d825 100644 --- a/backend/app/sandbox/local_sandbox.py +++ b/backend/app/sandbox/local_sandbox.py @@ -107,73 +107,90 @@ class LocalSandbox(Sandbox): ok=False, backend=self.name, error=f"本机找不到 {argv[0]}" ) - workdir = self._session_dir(session_id or f"tmp-{int(time.time()*1000)}") - atomic.write_text(workdir / entry, code) - for rel, content in (files or {}).items(): - target = (workdir / rel).resolve() - if not str(target).startswith(str(workdir.resolve())): - return ExecResult(ok=False, backend=self.name, error=f"非法文件路径：{rel}") - target.parent.mkdir(parents=True, exist_ok=True) - atomic.write_text(target, content) - - # 只透传最小环境变量，避免把宿主机的 API key 泄漏给被执行的代码 - child_env = { - "PATH": os.environ.get("PATH", "/usr/bin:/bin"), - "HOME": str(workdir), - "LANG": "en_US.UTF-8", - "PYTHONDONTWRITEBYTECODE": "1", - **(env or {}), - } - - started = time.perf_counter() + ephemeral = not session_id + workdir = self._session_dir(session_id or f"tmp-{int(time.time() * 1000)}-{os.urandom(4).hex()}") + work_root = workdir.resolve() try: - proc = await asyncio.create_subprocess_exec( - *argv, - cwd=str(workdir), - env=child_env, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - preexec_fn=self._preexec(limits), - ) - except Exception as e: # noqa: BLE001 - # 首行只说原因，类名和 errno 这些留给日志 - logger.warning("%s 沙箱起进程失败：%s", self.name, raw(e)) - return ExecResult(ok=False, backend=self.name, error=f"无法启动进程：{describe_exception(e)}") + atomic.write_text(workdir / entry, code) + for rel, content in (files or {}).items(): + target = (workdir / rel).resolve() + if not target.is_relative_to(work_root): + return ExecResult(ok=False, backend=self.name, error=f"非法文件路径：{rel}") + target.parent.mkdir(parents=True, exist_ok=True) + atomic.write_text(target, content) - try: - out, err = await asyncio.wait_for(proc.communicate(), timeout=limits.timeout) - except asyncio.TimeoutError: + # 只透传最小环境变量，避免把宿主机的 API key 泄漏给被执行的代码 + child_env = { + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "HOME": str(workdir), + "LANG": "en_US.UTF-8", + "PYTHONDONTWRITEBYTECODE": "1", + **(env or {}), + } + + started = time.perf_counter() try: - os.killpg(os.getpgid(proc.pid), 9) - except (ProcessLookupError, PermissionError): - proc.kill() - await proc.wait() - return ExecResult( - ok=False, - backend=self.name, - timed_out=True, - exit_code=124, - duration_ms=int((time.perf_counter() - started) * 1000), - error=f"执行超过 {limits.timeout} 秒，已终止", - ) + proc = await asyncio.create_subprocess_exec( + *argv, + cwd=str(workdir), + env=child_env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + preexec_fn=self._preexec(limits), + ) + except Exception as e: # noqa: BLE001 + # 首行只说原因，类名和 errno 这些留给日志 + logger.warning("%s 沙箱起进程失败：%s", self.name, raw(e)) + return ExecResult(ok=False, backend=self.name, error=f"无法启动进程：{describe_exception(e)}") - stdout, t1 = truncate(out.decode(errors="replace"), limits.max_output) - stderr, t2 = truncate(err.decode(errors="replace"), limits.max_output) - written = [ - p.name - for p in workdir.iterdir() - if p.is_file() and not p.name.startswith(ENTRY_PREFIX) - ][:50] - return ExecResult( - ok=proc.returncode == 0, - exit_code=proc.returncode or 0, - stdout=stdout, - stderr=stderr, - truncated=t1 or t2, - backend=self.name, - duration_ms=int((time.perf_counter() - started) * 1000), - files_written=written, - ) + try: + out, err = await asyncio.wait_for(proc.communicate(), timeout=limits.timeout) + except asyncio.TimeoutError: + self._kill(proc) + await proc.wait() + return ExecResult( + ok=False, + backend=self.name, + timed_out=True, + exit_code=124, + duration_ms=int((time.perf_counter() - started) * 1000), + error=f"执行超过 {limits.timeout} 秒，已终止", + ) + except BaseException: + if proc.returncode is None: + self._kill(proc) + await proc.wait() + raise + + stdout, t1 = truncate(out.decode(errors="replace"), limits.max_output) + stderr, t2 = truncate(err.decode(errors="replace"), limits.max_output) + written = [ + p.name + for p in workdir.iterdir() + if p.is_file() and not p.name.startswith(ENTRY_PREFIX) + ][:50] + return ExecResult( + ok=proc.returncode == 0, + exit_code=proc.returncode or 0, + stdout=stdout, + stderr=stderr, + truncated=t1 or t2, + backend=self.name, + duration_ms=int((time.perf_counter() - started) * 1000), + files_written=written, + ) + finally: + if ephemeral: + root_resolved = self._root.resolve() + if work_root != root_resolved and work_root.is_relative_to(root_resolved): + shutil.rmtree(workdir, ignore_errors=True) + + @staticmethod + def _kill(proc: asyncio.subprocess.Process) -> None: + try: + os.killpg(os.getpgid(proc.pid), 9) + except (ProcessLookupError, PermissionError): + proc.kill() async def cleanup(self, session_id: str) -> None: path = self._session_dir(session_id) diff --git a/backend/app/sandbox/seatbelt_sandbox.py b/backend/app/sandbox/seatbelt_sandbox.py index 221a1499..ff199c39 100644 --- a/backend/app/sandbox/seatbelt_sandbox.py +++ b/backend/app/sandbox/seatbelt_sandbox.py @@ -78,6 +78,7 @@ def build_policy(workdir: Path, *, language: str, network: bool) -> str: """ work = _real(workdir) home = _real(Path.home()) + data = _real(settings.data_dir) lines = [ "(version 1)", "(allow default)", @@ -98,8 +99,9 @@ def build_policy(workdir: Path, *, language: str, network: bool) -> str: ' (literal "/dev/stderr") (literal "/dev/dtracehelper"))', '(allow file-write* (subpath "/private/var/folders"))', # 临时目录，Python 要用 "", - ";; ---- 读：家目录挡住，放行工作区和解释器 ----", + ";; ---- 读：家目录与平台数据目录挡住，放行工作区和解释器 ----", f'(deny file-read* (subpath "{home}"))', + *([f'(deny file-read* (subpath "{data}"))'] if data != home else []), f'(allow file-read* (subpath "{work}"))', ] for root in _interpreter_roots(language): @@ -194,83 +196,99 @@ class SeatbeltSandbox(Sandbox): ext, cmd_of = _LANG[language] entry = entry_name(ext) - workdir = self._session_dir(session_id or f"tmp-{int(time.time() * 1000)}") - atomic.write_text(workdir / entry, code) - - for rel, content in (files or {}).items(): - target = (workdir / rel).resolve() - if not str(target).startswith(str(workdir.resolve())): - return ExecResult(ok=False, backend=self.name, error=f"非法文件路径：{rel}") - target.parent.mkdir(parents=True, exist_ok=True) - atomic.write_text(target, content) - - self._policies.mkdir(parents=True, exist_ok=True) - policy_file = self._policies / f"{workdir.name}-{language}.sb" - atomic.write_text(policy_file, build_policy(workdir, language=language, network=limits.network)) - - argv = [SANDBOX_EXEC, "-f", str(policy_file), *cmd_of(), entry] - - # HOME 指向工作区：既挡住家目录，又让解释器的缓存有地方落 - child_env = { - "PATH": "/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin", - "HOME": str(workdir), - "TMPDIR": str(workdir), - "LANG": "en_US.UTF-8", - "PYTHONDONTWRITEBYTECODE": "1", - "PYTHONUNBUFFERED": "1", - **(env or {}), - } - - started = time.perf_counter() + ephemeral = not session_id + workdir = self._session_dir(session_id or f"tmp-{int(time.time() * 1000)}-{os.urandom(4).hex()}") + work_root = workdir.resolve() + policy_file: Path | None = None try: - proc = await asyncio.create_subprocess_exec( - *argv, - cwd=str(workdir), - env=child_env, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - preexec_fn=self._preexec(limits), - ) - except Exception as e: # noqa: BLE001 - # 首行只说原因，类名和 errno 这些留给日志 - logger.warning("%s 沙箱起进程失败：%s", self.name, raw(e)) - return ExecResult(ok=False, backend=self.name, error=f"没能启动进程：{describe_exception(e)}") + atomic.write_text(workdir / entry, code) + + for rel, content in (files or {}).items(): + target = (workdir / rel).resolve() + if not target.is_relative_to(work_root): + return ExecResult(ok=False, backend=self.name, error=f"非法文件路径：{rel}") + target.parent.mkdir(parents=True, exist_ok=True) + atomic.write_text(target, content) + + self._policies.mkdir(parents=True, exist_ok=True) + policy_file = self._policies / f"{workdir.name}-{language}.sb" + atomic.write_text(policy_file, build_policy(workdir, language=language, network=limits.network)) + + argv = [SANDBOX_EXEC, "-f", str(policy_file), *cmd_of(), entry] + + # HOME 指向工作区：既挡住家目录，又让解释器的缓存有地方落 + child_env = { + "PATH": "/usr/bin:/bin:/usr/sbin:/sbin:/opt/homebrew/bin", + "HOME": str(workdir), + "TMPDIR": str(workdir), + "LANG": "en_US.UTF-8", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONUNBUFFERED": "1", + **(env or {}), + } + + started = time.perf_counter() + try: + proc = await asyncio.create_subprocess_exec( + *argv, + cwd=str(workdir), + env=child_env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + preexec_fn=self._preexec(limits), + ) + except Exception as e: # noqa: BLE001 + # 首行只说原因，类名和 errno 这些留给日志 + logger.warning("%s 沙箱起进程失败：%s", self.name, raw(e)) + return ExecResult(ok=False, backend=self.name, error=f"没能启动进程：{describe_exception(e)}") + + try: + out, err = await asyncio.wait_for(proc.communicate(), timeout=limits.timeout) + except asyncio.TimeoutError: + self._kill(proc) + await proc.wait() + return ExecResult( + ok=False, + backend=self.name, + timed_out=True, + exit_code=124, + duration_ms=int((time.perf_counter() - started) * 1000), + error=f"执行超过 {limits.timeout} 秒，已终止", + ) + except BaseException: + if proc.returncode is None: + self._kill(proc) + await proc.wait() + raise + + stdout, t1 = truncate(out.decode(errors="replace"), limits.max_output) + stderr, t2 = truncate(err.decode(errors="replace"), limits.max_output) + code_ = proc.returncode or 0 + # CPU 时间用尽是 SIGXCPU(24)，归到超时里更好理解 + timed_out = code_ in (-24, 152) + written = [ + p.name for p in workdir.iterdir() + if p.is_file() and not p.name.startswith(ENTRY_PREFIX) + ][:50] - try: - out, err = await asyncio.wait_for(proc.communicate(), timeout=limits.timeout) - except asyncio.TimeoutError: - self._kill(proc) - await proc.wait() return ExecResult( - ok=False, + ok=code_ == 0, + exit_code=code_, + stdout=stdout, + stderr=stderr if not timed_out else (stderr or "CPU 时间超限，已终止"), + truncated=t1 or t2, + timed_out=timed_out, backend=self.name, - timed_out=True, - exit_code=124, duration_ms=int((time.perf_counter() - started) * 1000), - error=f"执行超过 {limits.timeout} 秒，已终止", + files_written=written, ) - - stdout, t1 = truncate(out.decode(errors="replace"), limits.max_output) - stderr, t2 = truncate(err.decode(errors="replace"), limits.max_output) - code_ = proc.returncode or 0 - # CPU 时间用尽是 SIGXCPU(24)，归到超时里更好理解 - timed_out = code_ in (-24, 152) - written = [ - p.name for p in workdir.iterdir() - if p.is_file() and not p.name.startswith(ENTRY_PREFIX) - ][:50] - - return ExecResult( - ok=code_ == 0, - exit_code=code_, - stdout=stdout, - stderr=stderr if not timed_out else (stderr or "CPU 时间超限，已终止"), - truncated=t1 or t2, - timed_out=timed_out, - backend=self.name, - duration_ms=int((time.perf_counter() - started) * 1000), - files_written=written, - ) + finally: + if ephemeral: + if policy_file is not None: + policy_file.unlink(missing_ok=True) + root_resolved = self._root.resolve() + if work_root != root_resolved and work_root.is_relative_to(root_resolved): + shutil.rmtree(workdir, ignore_errors=True) @staticmethod def _kill(proc: asyncio.subprocess.Process) -> None: diff --git a/backend/app/tools/net.py b/backend/app/tools/net.py index c287d317..d3c932d3 100644 --- a/backend/app/tools/net.py +++ b/backend/app/tools/net.py @@ -21,10 +21,19 @@ _BLOCKED_PORTS = {22, 23, 25, 135, 139, 445, 3389} _METADATA_HOSTS = frozenset({"100.100.100.200", "fd00:ec2::254", "metadata.google.internal"}) -def _is_private(ip: str) -> bool: +def _unwrap_ip(ip: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None: try: addr = ipaddress.ip_address(ip) except ValueError: + return None + if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None: + return addr.ipv4_mapped + return addr + + +def _is_private(ip: str) -> bool: + addr = _unwrap_ip(ip) + if addr is None: return True return ( addr.is_private @@ -48,9 +57,8 @@ def _fake_networks() -> list[ipaddress.IPv4Network | ipaddress.IPv6Network]: def _in(ip: str, networks) -> bool: - try: - addr = ipaddress.ip_address(ip) - except ValueError: + addr = _unwrap_ip(ip) + if addr is None: return False return any(addr.version == n.version and addr in n for n in networks) @@ -101,10 +109,11 @@ async def _real_ips(host: str) -> tuple[str, ...]: def _is_metadata(ip: str) -> bool: try: - addr = ipaddress.ip_address(ip) + raw_addr = ipaddress.ip_address(ip) except ValueError: return True - return addr.is_link_local or str(addr) in _METADATA_HOSTS + addr = raw_addr.ipv4_mapped if isinstance(raw_addr, ipaddress.IPv6Address) and raw_addr.ipv4_mapped is not None else raw_addr + return raw_addr.is_link_local or addr.is_link_local or str(raw_addr) in _METADATA_HOSTS or str(addr) in _METADATA_HOSTS async def assert_safe_url(url: str, *, allow_intranet: bool = False) -> str: @@ -199,35 +208,63 @@ async def safe_request( max_bytes = max_bytes or settings.http_tool_max_bytes current = await assert_safe_url(url) + _SENSITIVE_HEADERS = frozenset({"authorization", "cookie", "proxy-authorization", "x-api-key"}) + cur_method = method.upper() + cur_headers = dict(headers or {}) + cur_params: dict[str, str] | None = params + cur_json: object | None = json_body + cur_data: str | None = data + async with httpx.AsyncClient( timeout=timeout, follow_redirects=False, trust_env=False ) as client: - for _ in range(5): - resp = await client.request( - method.upper(), + for hop in range(6): + async with client.stream( + cur_method, current, - headers=headers or {}, - params=params, - json=json_body, - content=data, - ) - if resp.status_code in (301, 302, 303, 307, 308) and "location" in resp.headers: - current = await assert_safe_url( - str(httpx.URL(current).join(resp.headers["location"])) - ) - continue - break + headers=cur_headers, + params=cur_params, + json=cur_json, + content=cur_data, + ) as resp: + if hop < 5 and resp.status_code in (301, 302, 303, 307, 308) and "location" in resp.headers: + prev_url = httpx.URL(current) + next_url = prev_url.join(resp.headers["location"]) + current = await assert_safe_url(str(next_url)) + cur_params = None + if (next_url.scheme, next_url.host, next_url.port) != (prev_url.scheme, prev_url.host, prev_url.port): + cur_headers = {k: v for k, v in cur_headers.items() if k.lower() not in _SENSITIVE_HEADERS} + if resp.status_code == 303 or (resp.status_code in (301, 302) and cur_method == "POST"): + cur_method = "GET" + cur_json = None + cur_data = None + continue - body = resp.content[:max_bytes] - truncated = len(resp.content) > max_bytes - try: - text = body.decode(resp.encoding or "utf-8", errors="replace") - except (LookupError, UnicodeDecodeError): - text = body.decode("utf-8", errors="replace") - return { - "status": resp.status_code, - "url": str(resp.url), - "headers": dict(resp.headers), - "body": text, - "truncated": truncated, - } + chunks: list[bytes] = [] + total = 0 + truncated = False + async for chunk in resp.aiter_bytes(): + if not chunk: + continue + remaining = max_bytes - total + if len(chunk) > remaining: + if remaining > 0: + chunks.append(chunk[:remaining]) + truncated = True + break + chunks.append(chunk) + total += len(chunk) + + body = b"".join(chunks) + try: + text = body.decode(resp.encoding or "utf-8", errors="replace") + except (LookupError, UnicodeDecodeError): + text = body.decode("utf-8", errors="replace") + return { + "status": resp.status_code, + "url": str(resp.url), + "headers": dict(resp.headers), + "body": text, + "truncated": truncated, + } + raise UnsafeUrlError("重定向次数超过上限") diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 88c1d8a8..20eaa186 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -19,6 +19,17 @@ from pathlib import Path _TMP = Path(tempfile.mkdtemp(prefix="agentlab-tests-")) os.environ["AGENTLAB_DATA_DIR"] = str(_TMP) +os.environ["AGENTLAB_DISABLE_DOTENV"] = "1" +# 剥离宿主机环境里可能携带的真实模型密钥与地址，防止测试或 seed 误调线上付费接口；需要测环境变量回退的用例用 monkeypatch.setenv 单独设 +for _env_key in ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_BASE_URL", + "ANTHROPIC_AUTH_TOKEN", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "AGENTLAB_USE_OPENAI_EMBEDDINGS", +): + os.environ.pop(_env_key, None) # 下面这几项也都要在任何 app.* 导入之前设（两种库同一个顺序）：PostgreSQL 分支要先导入 app.db.base 装连接池，settings 在那一刻 # 就按当时的环境定下来了——以前这两项写在那之后，PostgreSQL 上 AGENTLAB_SCHEDULER 没生效（scheduler_on 成了 True） # 别把真实的 key 带进测试环境 diff --git a/backend/tests/test_batch1_hardening.py b/backend/tests/test_batch1_hardening.py new file mode 100644 index 00000000..a365103c --- /dev/null +++ b/backend/tests/test_batch1_hardening.py @@ -0,0 +1,370 @@ +"""第一批 P0 止血修复、安全加固与多模型适配回归测试。""" +from __future__ import annotations + +import asyncio +import types +import uuid + +import httpx +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from sqlalchemy import select + +from app.core import artifact_store +from app.core.config import settings +from app.data import engine as data_engine +from app.db.base import SessionLocal +from app.db.models import Artifact, DataSource, Provider, Run +from app.db.models_blueprints import StoredFile +from app.engine import checkpoints +from app.engine.context import NodeContext, NodeError, RunContext +from app.engine.expressions import ExpressionError, eval_expression, resolve_path +from app.engine.nodes.llm import _invoke_streaming, run_llm +from app.engine.runner import run_manager +from app.engine.schema import GraphNode, GraphSpec +from app.files import store as files_store +from app.main import app +from app.providers import catalog, factory +from app.providers.factory import ModelSpec, _ReasoningAwareChatOpenAI, build_chat_model +from app.sandbox.local_sandbox import LocalSandbox +from app.sandbox.seatbelt_sandbox import SeatbeltSandbox, build_policy +from app.tools import net + + +async def test_memory_sqlite_engine_cache_and_epoch_invalidation(tmp_path): + cache = data_engine.EngineCache() + try: + mem_source = DataSource( + id=uuid.uuid4().hex, + name="mem-src", + kind="sqlite", + database=":memory:", + readonly=False, + ) + eng = await cache.get(mem_source) + async with eng.connect() as conn: + val = await conn.scalar(data_engine.text("SELECT 42")) + assert val == 42 + + # 模拟 get() 在锁外核对期间并发触发 invalidate()，旧世代不得写回缓存 + db_file = tmp_path / "snap.db" + conn0 = data_engine.open_checked_sqlite(str(db_file), readonly=False) + try: + conn0.execute("CREATE TABLE t (x INT)") + conn0.commit() + finally: + conn0.close() + digest = data_engine._file_sha256(str(db_file)) + snap_view = types.SimpleNamespace( + id="src-epoch", + snapshot_id="s1", + kind="sqlite", + database=str(db_file), + readonly=True, + immutable=True, + expected_sha256=digest, + options={}, + ) + orig_verify = cache._verify_once + invalidated_once = False + + async def slow_verify(key: str, source: object, expected: str): + nonlocal invalidated_once + fp = await orig_verify(key, source, expected) + if not invalidated_once: + invalidated_once = True + await cache.invalidate("src-epoch", broadcast=False) + return fp + + cache._verify_once = slow_verify # type: ignore[method-assign] + eng2 = await cache.get(snap_view) + assert eng2 is not None + assert cache._epochs.get("src-epoch", 0) == 1 + finally: + await cache.close() + + +async def test_sqlite_cancelled_query_interrupts_immediately(tmp_path): + db_file = tmp_path / "slow.db" + conn0 = data_engine.open_checked_sqlite(str(db_file), readonly=False) + try: + conn0.execute("CREATE TABLE t (x INT)") + conn0.commit() + finally: + conn0.close() + + src = DataSource( + id=uuid.uuid4().hex, + name="slow-sqlite", + kind="sqlite", + database=str(db_file), + readonly=True, + ) + infinite_cte = ( + "WITH RECURSIVE r(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM r WHERE n < 100000000) " + "SELECT SUM(n) FROM r" + ) + task = asyncio.create_task( + data_engine.run_query(src, infinite_cte, limits=data_engine.QueryLimits(timeout_seconds=30)) + ) + await asyncio.sleep(0.08) + task.cancel() + # 修复前 finally 先把 deadline.over 置 True，这里会挂满整条递归 CTE 甚至永久卡死 + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=2.0) + await data_engine.engines.invalidate(src.id, broadcast=False) + + +async def test_delete_run_cleans_artifacts_files_and_checkpoints(): + await run_manager.setup() + try: + graph = { + "nodes": [ + {"id": "in", "type": "input", "data": {"config": {}}}, + {"id": "out", "type": "output", "data": {"config": {"fields": [{"name": "ok", "value": "1"}]}}}, + ], + "edges": [{"source": "in", "target": "out"}], + } + started = await run_manager.start(graph=graph, input_payload={}) + for _ in range(100): + await asyncio.sleep(0.03) + async with SessionLocal() as session: + row = await session.get(Run, started.id) + if row and row.status in ("succeeded", "failed"): + break + assert row is not None and row.status == "succeeded" + + art_id = await artifact_store.put_json({"unique_for_delete_run": started.id}, kind="test", run_id=started.id) + async with SessionLocal() as session: + saved_file = await files_store.save( + session, + f"run-bound-{started.id}".encode(), + "note.txt", + uploaded_by="tester", + context={"blueprint_id": "b1"}, + ) + saved_file.bound_run_id = started.id + await session.commit() + file_id, file_sha = saved_file.id, saved_file.sha256 + + ckpt_before, _ = await checkpoints.count(run_manager.checkpointer, started.id) + assert ckpt_before >= 1 + assert artifact_store.load(art_id) is not None + assert files_store.blob_path(file_sha).exists() + + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client: + resp = await client.delete(f"/api/runs/{started.id}") + assert resp.status_code == 204, resp.text + + assert artifact_store.load(art_id) is None + assert not files_store.blob_path(file_sha).exists() + async with SessionLocal() as session: + assert await session.get(StoredFile, file_id) is None + rem_arts = list((await session.execute(select(Artifact).where(Artifact.run_id == started.id))).scalars()) + assert rem_arts == [] + ckpt_after, _ = await checkpoints.count(run_manager.checkpointer, started.id) + assert ckpt_after == 0 + finally: + await run_manager.shutdown() + + +async def test_sandbox_path_prefix_and_ephemeral_cleanup(): + box = LocalSandbox() + # workdir 同前缀兄弟目录逃逸（如 tmp-xxx -> ../tmp-xxx2/evil.txt）必须被 is_relative_to 拦下 + escaped = await box.run("print(1)", session_id="sess1", files={"../sess1_sibling/evil.txt": "bad"}) + assert not escaped.ok and "非法文件路径" in (escaped.error or "") + await box.cleanup("sess1") + + before_dirs = {p.name for p in box._root.iterdir()} if box._root.exists() else set() + ok = await box.run("print('hi')") + assert ok.ok and "hi" in ok.stdout + after_dirs = {p.name for p in box._root.iterdir()} if box._root.exists() else set() + assert after_dirs - before_dirs == set(), "无 session_id 的临时工作目录执行完必须回收" + + +def test_seatbelt_policy_denies_data_dir_outside_home(tmp_path, monkeypatch): + ext_data = tmp_path / "external_agentlab_data" + work = ext_data / "workspace" / "seatbelt" / "s1" + work.mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", ext_data) + policy = build_policy(work, language="python", network=False) + real_data = str(ext_data.resolve()) + real_work = str(work.resolve()) + assert f'(deny file-read* (subpath "{real_data}"))' in policy + assert policy.index(f'(deny file-read* (subpath "{real_data}"))') < policy.index( + f'(allow file-read* (subpath "{real_work}"))' + ) + assert SeatbeltSandbox is not None + + +async def test_net_ipv4_mapped_metadata_and_streaming_redirect_sanitization(monkeypatch): + assert net._is_metadata("::ffff:169.254.169.254") is True + assert net._is_metadata("::ffff:100.100.100.200") is True + assert net._is_private("::ffff:127.0.0.1") is True + + async def fake_assert_safe(url: str, *, allow_intranet: bool = False) -> str: + return url + + monkeypatch.setattr(net, "assert_safe_url", fake_assert_safe) + seen_requests: list[httpx.Request] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + seen_requests.append(request) + if request.url.host == "origin-a.example": + return httpx.Response(302, headers={"location": "https://origin-b.example/final?x=1"}) + return httpx.Response(200, content=b"0123456789abcdef", headers={"content-type": "text/plain"}) + + orig_client = httpx.AsyncClient + + def patched_client(**kwargs): + return orig_client(transport=httpx.MockTransport(handler), **{k: v for k, v in kwargs.items() if k != "trust_env"}) + + monkeypatch.setattr(net.httpx, "AsyncClient", patched_client) + res = await net.safe_request( + "POST", + "https://origin-a.example/start", + headers={"Authorization": "Bearer secret", "Cookie": "sid=1", "X-Custom": "keep"}, + params={"init": "1"}, + json_body={"hello": "world"}, + max_bytes=10, + ) + assert res["status"] == 200 + assert res["body"] == "0123456789" + assert res["truncated"] is True + assert len(seen_requests) == 2 + hop2 = seen_requests[1] + assert hop2.method == "GET" + assert "authorization" not in hop2.headers + assert "cookie" not in hop2.headers + assert hop2.headers.get("x-custom") == "keep" + assert str(hop2.url) == "https://origin-b.example/final?x=1" + + +def test_expressions_private_attr_and_seq_repeat_cap(): + class Dummy: + public = "ok" + _secret = "nope" + + assert resolve_path({"obj": Dummy()}, "obj.public") == "ok" + assert resolve_path({"obj": Dummy()}, "obj._secret") is None + + assert eval_expression("'ab' * 3", {}) == "ababab" + with pytest.raises(ExpressionError, match="序列重复长度超过上限"): + eval_expression("'x' * 2000000", {}) + with pytest.raises(ExpressionError, match="序列重复长度超过上限"): + eval_expression("[1, 2] * 600000", {}) + + +async def test_provider_reasoning_roundtrip_bearer_catalog_and_structured_usage(monkeypatch): + # 1. o1 / o3 / o4 禁采样 + 大小写不敏感匹配 + 自定义模型价格 + 缓存折扣 + assert catalog.supports_sampling("o1") is False + assert catalog.supports_sampling("openai/o3-mini") is False + assert catalog.supports_sampling("O4-mini") is False + assert catalog.supports_sampling("ollama") is True + assert catalog.find_pricing("deepseek-ai/deepseek-v3") is not None + + base_cost = catalog.estimate_cost("claude-sonnet-5", 10_000, 1_000) + cached_cost = catalog.estimate_cost("claude-sonnet-5", 10_000, 1_000, cache_read_tokens=9_000) + assert cached_cost < base_cost + + # 2. Anthropic bearer 保留 timeout 与 default_headers + p_bearer = Provider( + name="anth-gw", + kind="anthropic", + base_url="https://gw.example.com", + api_key="tok-123", + default_model="claude-sonnet-5", + enabled=True, + extra={"auth_style": "bearer", "headers": {"X-Tenant": "t1"}}, + ) + chat_anth = build_chat_model(p_bearer, ModelSpec(timeout=42)) + assert chat_anth._async_client.timeout == 42 + assert chat_anth._async_client.default_headers.get("X-Tenant") == "t1" + + # 3. ReasoningAwareChatOpenAI 在带 tool_calls 的多轮消息里回传 reasoning_content + llm = _ReasoningAwareChatOpenAI(model="deepseek-reasoner", api_key="k", base_url="https://api.deepseek.com/v1") + msgs = [ + HumanMessage(content="查销售额"), + AIMessage( + content="", + tool_calls=[{"name": "sql_query", "args": {"sql": "SELECT 1"}, "id": "call_1", "type": "tool_call"}], + additional_kwargs={"reasoning_content": "先查表确认销售额"}, + ), + ToolMessage(content="100", tool_call_id="call_1"), + ] + payload = llm._get_request_payload(msgs) + assistant_msg = next(m for m in payload["messages"] if m["role"] == "assistant") + assert assistant_msg["reasoning_content"] == "先查表确认销售额" + + # 4. _NO_STREAM_USAGE 按 (endpoint, model) 粒度隔离 + monkeypatch.setattr(factory, "_NO_STREAM_USAGE", {("https://gw.example.com/v1", "legacy-model")}) + assert ("https://gw.example.com/v1", "modern-model") not in factory._NO_STREAM_USAGE + + # 5. run_llm 结构化输出通过 include_raw=True 记录真实 token 与费用 + class FakeStructuredModel: + def with_structured_output(self, schema, include_raw: bool = False): + assert include_raw is True + + class _Runner: + async def ainvoke(self_inner, _messages): + return { + "raw": AIMessage( + content='{"answer": 42}', + usage_metadata={"input_tokens": 500, "output_tokens": 100, "total_tokens": 600}, + ), + "parsed": {"answer": 42}, + "parsing_error": None, + } + + return _Runner() + + async def fake_get_chat_model(_session, _spec): + return FakeStructuredModel(), "gpt-5.2" + + monkeypatch.setattr("app.engine.nodes.llm.get_chat_model", fake_get_chat_model) + node = GraphNode( + id="s1", + type="llm", + data={ + "label": "结构化", + "config": { + "prompt": "回答", + "output_schema": {"type": "object", "properties": {"answer": {"type": "integer"}}}, + }, + }, + ) + events_seen: list[dict] = [] + ctx = NodeContext( + node=node, + run=RunContext( + run_id="r-struct", + thread_id="t-struct", + spec=GraphSpec(nodes=[node], edges=[]), + extra={"stream_writer": lambda ev: events_seen.append(ev)}, + ), + ) + out = await run_llm({"input": {}, "vars": {}, "nodes": {}, "messages": []}, ctx) + assert out["nodes"]["s1"]["data"] == {"answer": 42} + assert out["usage"]["input_tokens"] == 500 and out["usage"]["output_tokens"] == 100 + assert out["usage"]["cost_usd"] > 0 + + # 6. _invoke_streaming 遇到 401/429 等不可重试错误直接抛出，不触发二次非流式请求 + ainvoke_called = 0 + + class AuthFailModel: + async def astream(self, _messages): + if False: + yield None + err = RuntimeError("401 invalid api key") + err.status_code = 401 # type: ignore[attr-defined] + raise err + + async def ainvoke(self, _messages): + nonlocal ainvoke_called + ainvoke_called += 1 + return AIMessage(content="should not happen") + + with pytest.raises(NodeError): + await _invoke_streaming(AuthFailModel(), [HumanMessage(content="hi")], ctx, "gpt-5.2") + assert ainvoke_called == 0 diff --git a/backend/tests/test_batch2_optimizations.py b/backend/tests/test_batch2_optimizations.py new file mode 100644 index 00000000..cc8af92c --- /dev/null +++ b/backend/tests/test_batch2_optimizations.py @@ -0,0 +1,256 @@ +from __future__ import annotations + +from datetime import date +from fractions import Fraction +import math +from pathlib import Path +from types import SimpleNamespace + +import pytest +from sqlalchemy import event, text +from sqlalchemy.ext.asyncio import create_async_engine + +from app.analysis import DATA_ERRORS, stats as analysis_stats +from app.analysis.contract import CallError +from app.analysis.decompose import lmdi +from app.analysis.differ import MAX_ROWS_MODE +from app.analysis.periods import plain_day +from app.analysis.stats import theil_sen +from app.chat.direct import build_direct_graph +from app.chat.direct_evidence import normalize_and_bind_fast_citations +from app.chat.grounding import GroundingBundle +from app.cluster.service import _Service +from app.core.bus import EventBus, _MAX_SEQ_ENTRIES +from app.data import coverage, csv_grid +from app.data.tabular import _detect_csv_encoding +from app.db.base import _INDEX_MIGRATIONS, _set_sqlite_pragma, create_schema +from app.engine.schema import GraphSpec, validate_graph +from app.ledger.match import Caliber, Fetch, Query, compare_queries +from tests.analysis_samples import analyze, snap as make_snap + + +@pytest.mark.asyncio +async def test_sqlite_pragmas_and_index_migrations(tmp_path: Path) -> None: + db_path = tmp_path / "idx_check.db" + eng = create_async_engine(f"sqlite+aiosqlite:///{db_path}") + event.listen(eng.sync_engine, "connect", _set_sqlite_pragma) + try: + await create_schema(eng) + async with eng.connect() as conn: + sync_mode = (await conn.execute(text("PRAGMA synchronous"))).scalar_one() + # SQLite NORMAL = 1 + assert sync_mode == 1 + rows = (await conn.execute(text("SELECT name FROM sqlite_master WHERE type='index'"))).scalars().all() + existing = set(rows) + for idx_name, _, _ in _INDEX_MIGRATIONS: + assert idx_name in existing + finally: + await eng.dispose() + + +@pytest.mark.asyncio +async def test_event_bus_seq_lru_and_close_eviction() -> None: + bus = EventBus() + seq1 = bus.next_seq("run-1") + assert seq1 == 1 + assert "run-1" in bus._seq + await bus.close("run-1") + assert "run-1" not in bus._seq + + for i in range(_MAX_SEQ_ENTRIES + 25): + bus.next_seq(f"r-{i}") + assert len(bus._seq) <= _MAX_SEQ_ENTRIES + assert "r-0" not in bus._seq + + +def test_cluster_slot_signal_wakes_local_waiters(monkeypatch: pytest.MonkeyPatch) -> None: + woken: list[bool] = [] + monkeypatch.setattr("app.cluster.service.slots.wake_local", lambda: woken.append(True)) + svc = _Service() + svc._on_slot(["w1"]) + assert woken == [True] + + +def test_coverage_parses_two_digit_tz_offset_and_lru_eviction() -> None: + d1 = coverage.as_day("2026-08-31 23:30:00+08") + d2 = coverage.as_day("2026-08-31T00:00:00+00") + d3 = coverage.as_day("2026-08-31 12:00:00-0530") + assert d1 is not None and d1.day == date(2026, 8, 31) and d1.timed is True + assert d2 is not None and d2.day == date(2026, 8, 31) and d2.timed is False + assert d3 is not None and d3.day == date(2026, 8, 31) and d3.timed is True + + coverage._LIVE.clear() + try: + for i in range(coverage._LIVE_MAX + 10): + coverage._LIVE[("src", f"t_{i}")] = (1e12, i) # type: ignore[assignment] + if len(coverage._LIVE) > coverage._LIVE_MAX: + coverage._LIVE.popitem(last=False) + assert len(coverage._LIVE) == coverage._LIVE_MAX + assert ("src", "t_0") not in coverage._LIVE + assert ("src", f"t_{coverage._LIVE_MAX + 9}") in coverage._LIVE + finally: + coverage._LIVE.clear() + + +def test_compare_queries_prioritizes_table_overlap_before_max_candidates() -> None: + calibers: list[Caliber] = [] + for i in range(95): + calibers.append( + Caliber( + workflow_id=f"wf_{i}", + version=1, + level="certified", + graph_hash=f"h_{i}", + fetches=( + Fetch( + node_id="q1", + source="sales", + sql=f"SELECT COUNT(*) AS cnt_{i} FROM orders", + tables=frozenset({"orders"}), + ), + ), + ) + ) + calibers.append( + Caliber( + workflow_id="wf_exact", + version=1, + level="certified", + graph_hash="h_exact", + fetches=( + Fetch( + node_id="q1", + source="sales", + sql="SELECT SUM(o.amount - COALESCE(r.refund_amt, 0)) AS net_sales " + "FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + tables=frozenset({"orders", "refunds"}), + ), + ), + ) + ) + q = Query( + artifact="sha_q1", + node_id="fetch_1", + source="sales", + sql="SELECT SUM(o.amount - COALESCE(r.refund_amt, 0)) AS net_sales " + "FROM orders o LEFT JOIN refunds r ON r.order_id = o.id", + columns=["net_sales"], + tables=frozenset({"orders", "refunds"}), + dialect="sqlite", + ) + res = compare_queries([q], calibers) + assert len(res) == 1 + assert res[0]["status"] == "proven" + assert res[0]["candidates"][0]["workflow_id"] == "wf_exact" + + +def test_direct_evidence_indexed_auto_bind_preserves_semantics() -> None: + snap = { + "kind": "tool_output", + "tool": "db_query__mes", + "columns": ["line_code", "good_qty", "yield_rate", "output_amt"], + "column_types": ["text", "integer", "real", "real"], + "rows": [["L01", 69133, 0.985, 125000.0]], + "truncated": False, + } + catalog = { + "Q1": { + "kind": "query", + "artifact": "art_q1", + "tool": "db_query__mes", + "source": "mes", + "columns": ["line_code", "good_qty", "yield_rate", "output_amt"], + "row_count": 1, + } + } + loader = lambda art: snap if art == "art_q1" else None + raw = "产线 L01 良品数为 69,133 件，良率达到 98.5%，产值 12.5万 元。" + bound = normalize_and_bind_fast_citations(raw, catalog, loader=loader) + assert "[[v:Q1.r0.good_qty]]" in bound + assert "[[v:Q1.r0.yield_rate|pct]]" in bound + assert "[[v:Q1.r0.output_amt|万]]" in bound + + +def test_plain_day_recognizes_cast_and_pg_double_colon() -> None: + assert plain_day("SELECT CAST(stat_date AS DATE) AS dt, SUM(x) FROM t GROUP BY 1", "dt", "sqlite") is True + assert plain_day("SELECT stat_date::date AS dt, SUM(x) FROM t GROUP BY 1", "dt", "postgres") is True + assert plain_day("SELECT DATE_TRUNC('month', stat_date) AS dt, SUM(x) FROM t GROUP BY 1", "dt", "postgres") is False + + +@pytest.mark.asyncio +async def test_differ_rows_mode_enforces_max_rows_ceiling() -> None: + assert "too_many_rows" in DATA_ERRORS + rows = [["A" if i % 2 == 0 else "B", float(i)] for i in range(MAX_ROWS_MODE + 5)] + s = make_snap("Q1", "SELECT grp, val FROM t", ["grp", "val"], rows) + with pytest.raises(CallError) as exc_info: + await analyze("differ", "rows", snaps=[s], query="Q1", by="grp", value="val") + assert exc_info.value.code == "too_many_rows" + + refused = await analyze( + "differ", "rows", snaps=[s], refuse_data_errors=True, query="Q1", by="grp", value="val" + ) + assert refused["status"] == "refused" + assert refused["reason_code"] == "input_unavailable" + assert str(MAX_ROWS_MODE) in refused["reason"] + + +def test_theil_sen_handles_large_denominator_lcm_and_extreme_floats() -> None: + primes = [ + 2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, + 73, 79, 83, 89, 97, 101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, + 157, 163, 167, 173, 179, 181, 191, 193, 197, 199, 211, 223, 227, 229, 233, + 239, 241, 251, 257, 263, 269, 271, 277, 281, + ] + xs = [Fraction(i + 1, p) for i, p in enumerate(primes)] + ys = [x * 3 + Fraction(1, 7) for x in xs] + slope = theil_sen(xs, ys) + assert math.isclose(slope, 3.0, rel_tol=1e-9) + + huge_xs = [Fraction(0), Fraction(1, 10**200), Fraction(2, 10**200)] + huge_ys = [Fraction(0), Fraction(10**200, 1), Fraction(2 * 10**200, 1)] + huge_slope = theil_sen(huge_xs, huge_ys) + assert huge_slope == Fraction(10**400, 1) + assert math.isinf(analysis_stats._safe_float(huge_slope)) + + +def test_lmdi_handles_many_factors_without_float_overflow() -> None: + base = [10.0] * 300 + target = [20.0] * 300 + contribs = lmdi(base, target, 100.0, 250.0) + assert len(contribs) == 300 + assert math.isclose(sum(contribs), 150.0, rel_tol=1e-9) + + +def test_streaming_csv_encoding_detection_and_grid_scan() -> None: + gbk_bytes = "产线,良品数,备注\nL01,1200,正常生产\nL02,980,换型调试\n".encode("gb18030") + assert _detect_csv_encoding(gbk_bytes) == ("gb18030", "strict") + + utf8_bom = b"\xef\xbb\xbfcol_a,col_b\n1,2\n" + assert _detect_csv_encoding(utf8_bom) == ("utf-8-sig", "strict") + + scanned = csv_grid.scan(gbk_bytes) + assert len(scanned.sheets) == 1 + assert scanned.sheets[0].bounds is not None + assert scanned.sheets[0].bounds.max_row == 3 + assert scanned.sheets[0].bounds.max_col == 3 + + parsed = csv_grid.parse(gbk_bytes) + assert len(parsed.records) == 3 + assert parsed.records[0] == ["产线", "良品数", "备注"] + assert parsed.records[1] == ["L01", "1200", "正常生产"] + + streamed = list(csv_grid.rows(parsed, min_row=1, max_row=3, min_col=1, max_col=3)) + assert len(streamed) == 3 + + +def test_build_direct_graph_with_and_without_metrics_passes_validation() -> None: + b = GroundingBundle(narrowed_sources=[SimpleNamespace(name="demo_db")]) # type: ignore[list-item] + assert b.has_enabled_metrics is False + + g_no_metrics = build_direct_graph(bundle=b, has_metrics=False) + res_no = validate_graph(GraphSpec.model_validate(g_no_metrics)) + assert res_no.ok is True, res_no.issues + + g_with_metrics = build_direct_graph(bundle=b, has_metrics=True) + res_yes = validate_graph(GraphSpec.model_validate(g_with_metrics)) + assert res_yes.ok is True, res_yes.issues diff --git a/backend/tests/test_batch3_architecture.py b/backend/tests/test_batch3_architecture.py new file mode 100644 index 00000000..65a70078 --- /dev/null +++ b/backend/tests/test_batch3_architecture.py @@ -0,0 +1,388 @@ +"""第三批架构级演进回归测试（Step 3.1 – 3.4）。 + +覆盖： +1. 记录库 `records` 多版本单库归档演进（突破 SQLite 10 库 `ATTACH` 上限、增量同步、WAL 与 `records_archive` 越权拦截）； +2. 知识库真·双路混合检索（零词面重合高语义命中不被 BM25 前置截断、向量缓存失效、>1000 块倒排分批更新）； +3. 方言感知 SQL 模板词法分析（MySQL `\\'`/`#`、PostgreSQL `E'...'`/`$$...$$`/嵌套注释/数组下标、Oracle `q'[...]'`、SQLite `[id]`）与 `Decimal`/`bool`/`NaN` 参数完整性； +4. 分层依赖守卫（`app/ledger/*` 零依赖 `app.api.settings` / `app.api.ledger` / `app.api.evidence`）与前端组件拆分门禁。 +""" +from __future__ import annotations + +import ast +import sqlite3 +import uuid +from decimal import Decimal +from pathlib import Path + +import numpy as np +import pytest +from sqlalchemy import select + +from app.core.config import settings +from app.data.engine import run_query +from app.data.guard import SqlRejected +from app.data.sql_semantics import _template_slots +from app.db.base import SessionLocal +from app.db.models import Chunk, DataSource, Document +from app.db.models_apps import App +from app.db.models_blueprints import RecordsTable +from app.engine.expressions import render_template +from app.ledger import params +from app.memory import embeddings as emb +from app.memory import inverted, kb +from app.records import store +from app.records.tables import columns_of, ensure_table, path_of +from app.records.views import ARCHIVE_SCHEMA, resolve + + +@pytest.fixture +def isolated_data_dir(monkeypatch, tmp_path): + data = tmp_path / "data" + (data / "uploads").mkdir(parents=True) + monkeypatch.setattr(settings, "data_dir", data) + kb.invalidate_vector_cache() + yield data + kb.invalidate_vector_cache() + + +# --------------------------------------------------------------------------- +# Step 3.1：记录库 >10 版本单库归档、增量同步与安全边界 +# --------------------------------------------------------------------------- + + +async def test_records_scale_beyond_sqlite_attach_limit_with_incremental_archive(isolated_data_dir): + async with SessionLocal() as session: + owner = App(name="多版本报损台账", created_by="admin") + session.add(owner) + await session.commit() + app_id = owner.id + + versions: list[RecordsTable] = [] + inserted_row_ids: dict[int, str] = {} + + # 连续演进 14 个不兼容版本（每次新增一个必填列），远超 SQLite 默认 SQLITE_MAX_ATTACHED=10 上限 + for idx in range(1, 15): + cols = [ + {"name": "门店", "label": "门店", "type": "string", "required": True}, + {"name": "数量", "label": "数量", "type": "integer", "required": True}, + {"name": f"必填项_{idx}", "label": f"必填项_{idx}", "type": "string", "required": True}, + ] + async with SessionLocal() as session: + owner = await session.get(App, app_id) + tbl = await ensure_table(session, owner, cols) + await session.commit() + assert tbl.version == idx + rec = store.append( + path_of(tbl), + columns_of(tbl), + {"门店": f"门店_{idx:02d}", "数量": idx * 10, f"必填项_{idx}": f"v{idx}"}, + version=tbl.version, + run_id=None, + actor="amy", + action_key=uuid.uuid4().hex, + ) + inserted_row_ids[idx] = rec.row_id + versions.append(tbl) + + # 验证单表库开启了 WAL 模式 + with sqlite3.connect(path_of(versions[-1])) as conn: + mode = conn.execute("PRAGMA journal_mode").fetchone()[0] + assert str(mode).lower() == "wal" + + # 在将被归档的第 2 版中做一次更正（supersedes），确保归档表保留 _supersedes 演进语义 + store.append( + path_of(versions[1]), + columns_of(versions[1]), + {"门店": "门店_02", "数量": 25, "必填项_2": "v2_fixed"}, + version=2, + run_id=None, + actor="amy", + action_key=uuid.uuid4().hex, + supersedes=inserted_row_ids[2], + ) + + latest = versions[-1] + async with SessionLocal() as session: + source = await session.get(DataSource, latest.source_id) + resolved = await resolve(session, source) + + # 13 个历史版本若逐库 ATTACH 会触发 OperationalError: too many attached databases (max 10) + # 归档收敛后挂接数 <= 8（7 个近期直挂库 + 1 个归档库） + assert len(resolved.attach) <= 8 + assert ARCHIVE_SCHEMA in dict(resolved.attach) + assert store.archive_path(latest.table_key).exists() + + dbs = await run_query(resolved, "PRAGMA database_list") + assert len(dbs.rows) <= 10 + + summary = await run_query(resolved, 'SELECT count(*) AS n, sum("数量") AS total FROM records') + expected_total = sum(i * 10 for i in range(1, 15)) - 20 + 25 + assert summary.rows == [[14, expected_total]] + + # 模拟停用前已在途的动作向第 1 版追加一行，验证 sync_archive 按 _seq > since_seq 增量同步 + store.append( + path_of(versions[0]), + columns_of(versions[0]), + {"门店": "门店_01_补单", "数量": 5, "必填项_1": "v1_late"}, + version=1, + run_id=None, + actor="bob", + action_key=uuid.uuid4().hex, + ) + async with SessionLocal() as session: + refreshed = await resolve(session, source) + summary_after = await run_query(refreshed, 'SELECT count(*) AS n, sum("数量") AS total FROM records') + assert summary_after.rows == [[15, expected_total + 5]] + + # 直接点名归档 schema 必须被 SQL 守卫拦截 + for forbidden_sql in ( + f"SELECT count(*) FROM {ARCHIVE_SCHEMA}.rows_v1", + f'SELECT count(*) FROM "{ARCHIVE_SCHEMA}"."rows_v1"', + f"SELECT count(*) FROM [{ARCHIVE_SCHEMA.upper()}].rows_v1", + ): + with pytest.raises(SqlRejected): + await run_query(refreshed, forbidden_sql) + + +# --------------------------------------------------------------------------- +# Step 3.2：真·双路混合检索、缓存失效与超千块倒排分批处理 +# --------------------------------------------------------------------------- + + +async def test_kb_true_dual_path_hybrid_search_finds_zero_lexical_overlap_hit(isolated_data_dir, monkeypatch): + target_text = "超级管理员可调整他人角色与访问范围。" + distractors = [f"审批规范第 {i} 条：普通报销单需部门主管签字归档。" for i in range(65)] + + async def fake_embed_texts(texts: list[str]) -> list[np.ndarray]: + vecs: list[np.ndarray] = [] + for t in texts: + v = np.zeros(emb.embedder_dim(), dtype=np.float32) + if "超级管理员" in t: + v[0] = 1.0 + else: + v[1] = 1.0 + vecs.append(v) + return vecs + + async def fake_embed_text(query: str) -> np.ndarray: + v = np.zeros(emb.embedder_dim(), dtype=np.float32) + v[0] = 1.0 + return v + + monkeypatch.setattr(kb, "embed_texts", fake_embed_texts) + monkeypatch.setattr(kb, "embed_text", fake_embed_text) + + collection = f"dual_path_{uuid.uuid4().hex[:8]}" + async with SessionLocal() as session: + # 65 条词面强命中「审批规范」的干扰块（超过原 BM25 单路候选池上限 50） + for idx, text in enumerate(distractors): + await kb.ingest_document(session, collection=collection, title=f"干扰_{idx}", content=text) + # 1 条零词面重合但语义完全一致的目标块 + target_doc = await kb.ingest_document(session, collection=collection, title="权限说明", content=target_text) + + degrade_reasons: list[str] = [] + async with SessionLocal() as session: + hits = await kb.search( + session, + collection=collection, + query="审批规范里谁能改权限", + limit=5, + alpha=0.6, + on_degrade=degrade_reasons.append, + ) + + assert degrade_reasons == [] + assert hits, "双路召回应当返回结果" + assert hits[0]["content"] == target_text, "零词面重合但余弦相似度为 1.0 的目标块在双路混合检索下应排第一" + + # 删除目标文档后，向量矩阵缓存自动失效，不再返回已删块 + async with SessionLocal() as session: + deleted = await kb.delete_document(session, target_doc.id) + assert deleted is True + after_delete = await kb.search( + session, + collection=collection, + query="审批规范里谁能改权限", + limit=5, + alpha=0.6, + ) + assert all(h["content"] != target_text for h in after_delete) + + +async def test_inverted_index_handles_over_thousand_chunks_without_bind_limit(isolated_data_dir): + collection = f"batch_inv_{uuid.uuid4().hex[:8]}" + doc_id = f"doc_{uuid.uuid4().hex[:12]}" + chunk_ids: list[str] = [] + made: list[Chunk] = [] + async with SessionLocal() as session: + session.add(Document(id=doc_id, collection=collection, title="bulk", content="bulk")) + await session.flush() + for i in range(1100): + cid = uuid.uuid4().hex + chunk_ids.append(cid) + chunk = Chunk( + id=cid, + document_id=doc_id, + collection=collection, + ordinal=i, + content=f"统一结算批次规范条目 {i}", + embedding=b"", + embed_model="", + embed_dim=0, + meta={"title": "bulk"}, + ) + session.add(chunk) + made.append(chunk) + await session.flush() + for chunk in made: + await inverted.index_chunk(session, chunk) + await session.commit() + + async with SessionLocal() as session: + scored, all_scores = await inverted.candidates_with_scores( + session, collection=collection, query="统一结算批次规范", limit=25, + ) + assert len(scored) == 25 + assert len(all_scores) == 1100 + assert scored[0][1] > 0 + + # 超 1000 个 chunk_id 删除时自动分批，不触发 SQLite 999 / PG 32767 绑定参数上限 + await inverted.drop_chunk_terms(session, chunk_ids) + await session.commit() + after, after_all = await inverted.candidates_with_scores( + session, collection=collection, query="统一结算批次规范", limit=25, + ) + assert after == [] + assert after_all == {} + + +# --------------------------------------------------------------------------- +# Step 3.3：方言感知 SQL 模板词法分析与 Decimal / bool / NaN 参数完整性 +# --------------------------------------------------------------------------- + + +def test_dialect_aware_sql_template_slots(): + # MySQL 反斜杠转义单引号与 # 行注释：注释里的模板识别为 comment，真参数仍在 WHERE 后作为 bare + mysql_sql = r"SELECT 'it\'s fine' AS tag # 忽略 {{ input.ignored }}" + "\nFROM orders WHERE id = {{ input.order_id }}" + mysql_spans, _ = _template_slots(mysql_sql, dialect="mysql") + assert [(s.field, s.context) for _, _, s in mysql_spans] == [ + ("ignored", "comment"), + ("order_id", "bare"), + ] + + # PostgreSQL $$...$$ 美元引号、E'...' 转义串、嵌套块注释与数组下标 arr[{{ input.idx }}] + pg_sql = ( + "/* 外层 /* 内层 {{ input.in_comment }} */ 结束 */\n" + "SELECT $$raw {{ input.in_dollar }}$$ AS d, E'escaped \\' {{ input.in_estr }}' AS e, " + "tags[{{ input.idx }}] AS picked FROM items" + ) + pg_spans, _ = _template_slots(pg_sql, dialect="postgres") + assert [(s.field, s.context) for _, _, s in pg_spans] == [ + ("in_comment", "comment"), + ("in_dollar", "quoted"), + ("in_estr", "quoted"), + ("idx", "bare"), + ] + + # Oracle q'[...]' 替代引号字面量识别为 quoted 上下文 + oracle_sql = "SELECT q'[ Department's {{ input.dept }} ]' FROM dual" + oracle_spans, _ = _template_slots(oracle_sql, dialect="oracle") + assert [(s.field, s.context) for _, _, s in oracle_spans] == [("dept", "quoted")] + + # SQLite 方括号标识符里的占位符属于不支持的标识符注入位，必须拒绝 + sqlite_sql = "SELECT * FROM [{{ input.table_name }}]" + with pytest.raises(params.ParameterRefused) as exc_info: + params.slots_of(sqlite_sql, dialect="sqlite") + assert exc_info.value.code == params.UNSUPPORTED + + +def test_decimal_boolean_and_non_finite_parameter_integrity(): + # 极小 Decimal 在表达式求值与参数写出时不退化为科学计数法 1E-7 + rendered = render_template("{{ input.rate }}", {"input": {"rate": Decimal("1E-7")}}) + assert rendered == "0.0000001" + + fields = [ + {"name": "amount", "type": "decimal", "required": True}, + {"name": "active", "type": "boolean", "required": True}, + ] + template = "SELECT * FROM ledger_tx WHERE amount >= {{ input.amount }} AND is_active = {{ input.active }}" + assert params.check_template(template, fields=params.declared(fields), dialect="postgres") == [] + + stmt = params.plan( + template, + fields=fields, + values={"amount": Decimal("19.95"), "active": True}, + dialect="postgres", + ) + assert "amount >= 19.95" in stmt.statement + assert "is_active = True" in stmt.statement + by_name = {entry["name"]: entry for entry in stmt.material["parameters"]} + assert by_name["amount"]["kind"] == "number" and by_name["amount"]["declared"] == "number" + assert by_name["active"]["kind"] == "boolean" and by_name["active"]["declared"] == "boolean" + + # NaN / Inf 绝不能当作合法裸数拼入 SQL（否则会变成列名 NaN / Inf） + for bad_number in (float("nan"), float("inf"), float("-inf"), Decimal("NaN"), Decimal("Infinity")): + with pytest.raises(params.ParameterRefused) as exc_info: + params.plan( + "SELECT * FROM ledger_tx WHERE amount >= {{ input.amount }}", + fields=[{"name": "amount", "type": "decimal", "required": True}], + values={"amount": bad_number}, + dialect="postgres", + ) + assert exc_info.value.code == params.LITERAL + + +# --------------------------------------------------------------------------- +# Step 3.3 & 3.4：分层依赖边界与前端超大文件拆分门禁 +# --------------------------------------------------------------------------- + + +def test_ledger_layer_has_no_upward_api_domain_imports(): + repo_root = Path(__file__).resolve().parents[2] + ledger_dir = repo_root / "backend" / "app" / "ledger" + forbidden_modules = {"app.api.settings", "app.api.ledger", "app.api.evidence"} + + violations: list[str] = [] + for py_file in sorted(ledger_dir.rglob("*.py")): + tree = ast.parse(py_file.read_text(encoding="utf-8"), filename=str(py_file)) + rel = py_file.relative_to(repo_root) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name in forbidden_modules: + violations.append(f"{rel}:{node.lineno} import {alias.name}") + elif isinstance(node, ast.ImportFrom) and node.module: + if node.module in forbidden_modules: + violations.append(f"{rel}:{node.lineno} from {node.module} import ...") + elif node.module == "app.api": + for alias in node.names: + full = f"app.api.{alias.name}" + if full in forbidden_modules: + violations.append(f"{rel}:{node.lineno} from app.api import {alias.name}") + + assert violations == [], f"ledger 领域层出现了对 api 路由层的反向依赖：{violations}" + + listing_py = repo_root / "backend" / "app" / "records" / "listing.py" + listing_src = listing_py.read_text(encoding="utf-8") + assert "app.api.evidence" not in listing_src + assert "app.engine.evidence_reader" in listing_src + + +def test_caliber_drafts_page_is_modularized(): + repo_root = Path(__file__).resolve().parents[2] + drafts_page = repo_root / "frontend" / "src" / "pages" / "ledger" / "CaliberDrafts.tsx" + drafts_dir = repo_root / "frontend" / "src" / "pages" / "ledger" / "drafts" + + line_count = len(drafts_page.read_text(encoding="utf-8").splitlines()) + assert line_count < 900, f"CaliberDrafts.tsx 仍超过 900 行（当前 {line_count} 行）" + expected_modules = { + "helpers.ts", + "DraftSharedBlocks.tsx", + "DraftTrialSection.tsx", + "DraftRulesSection.tsx", + "DraftImportsSection.tsx", + "DraftNlSection.tsx", + "DraftValidationSection.tsx", + } + actual_modules = {p.name for p in drafts_dir.glob("*.ts*")} + assert expected_modules.issubset(actual_modules) diff --git a/backend/tests/test_thinking.py b/backend/tests/test_thinking.py index 076df1df..cf25d6c0 100644 --- a/backend/tests/test_thinking.py +++ b/backend/tests/test_thinking.py @@ -63,13 +63,10 @@ def test_empty_reasoning_is_not_reported() -> None: def _anthropic(model: str, **spec_kw): - import os - from app.db.models import Provider from app.providers.factory import ModelSpec, build_chat_model - os.environ.setdefault("ANTHROPIC_API_KEY", "test-only") - provider = Provider(name="t", kind="anthropic", api_key=None, models=[], + provider = Provider(name="t", kind="anthropic", api_key="test-only", models=[], extra=spec_kw.pop("extra", {}) or {}) return build_chat_model(provider, ModelSpec(model=model, **spec_kw)) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 3996e09a..6d79a559 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -12,9 +12,9 @@ import { useConversations } from './store/conversations' import { personalAppsOn } from './pages/blueprints/shared' import { Logo } from './components/Logo' import { - CommandPalette, PAGES, ShortcutHelp, flushThemePref, openShortcutHelp, toggleCommandPalette, toggleNotify, - toggleTheme, usePages, useTheme, visiblePages, waitedFor, -} from './components/CommandPalette' + PAGES, flushThemePref, openShortcutHelp, toggleCommandPalette, toggleNotify, + toggleTheme, usePages, useShellOverlay, useTheme, visiblePages, waitedFor, +} from './components/shellCommands' import { atLeast, authSettled, loginOn, loginUrl, needsLogin, prefsLocalOnly, principalName, useAuth } from './store/auth' import { ErrorBoundary, Kbd, OfflineBanner, Spinner, StatusBadge, isComposing, toast, useTicker } from './components/ui' import { applyTheme, normalizeTheme } from './lib/theme' @@ -57,6 +57,16 @@ const NewAppStart = lazy(() => import('./pages/blueprints/NewAppStart').then((mo const BlueprintPage = lazy(() => import('./pages/blueprints/BlueprintPage').then((module) => ({ default: module.BlueprintPage }))) const LoginPage = lazy(() => import('./pages/LoginPage').then((module) => ({ default: module.LoginPage }))) const SharePage = lazy(() => import('./pages/apps/SharePage').then((module) => ({ default: module.SharePage }))) +const LazyShellOverlays = lazy(() => import('./components/CommandPalette').then((m) => ({ + default: function Overlays() { + return ( + <> + + + + ) + }, +}))) /** 启动时认人（/auth/me）最多等多久再取目录 */ const AUTH_WAIT_MS = 2500 @@ -230,8 +240,7 @@ function Shell() { - - + @@ -239,6 +248,20 @@ function Shell() { ) } +function ShellOverlays() { + const open = useShellOverlay() + const [everOpened, setEverOpened] = useState(false) + useEffect(() => { + if (open) setEverOpened(true) + }, [open]) + if (!open && !everOpened) return null + return ( + + + + ) +} + /** 首页：使用者（开了登录的 user 角色）落到应用首页，其余落到问数据 */ function HomeRedirect() { const me = useAuth((s) => s.me) diff --git a/frontend/src/api/client.ts b/frontend/src/api/client.ts index 3d11d180..190c696b 100644 --- a/frontend/src/api/client.ts +++ b/frontend/src/api/client.ts @@ -1205,6 +1205,7 @@ export function streamRun( /** 握手连续失败（一次都没打开过就断了）几次 */ let refused = 0 let pollTimer: ReturnType | undefined + let wsRetryTimer: ReturnType | undefined const finish = (status?: string) => { if (closed) return @@ -1253,7 +1254,8 @@ export function streamRun( socket = new WebSocket(`${proto}://${location.host}/api/runs/${runId}/stream?after=${lastSeq}`) socket.onmessage = (ev) => { - const event = parseBody(ev.data) as RunEvent & { type: string; status?: string } + const event = parseBody(ev.data) as (RunEvent & { type?: string; status?: string }) | null + if (!event || typeof event !== 'object' || typeof event.type !== 'string') return if (event.type === 'stream.end') { socket?.close() // 新后端放在 data.status，老后端放在顶层 status @@ -1261,7 +1263,7 @@ export function streamRun( return } if (event.seq) lastSeq = Math.max(lastSeq, event.seq) - onEvent(event) + onEvent(event as RunEvent) } socket.onclose = () => { if (closed) return @@ -1273,12 +1275,14 @@ export function streamRun( } // 指数退避重连，最多等 5 秒 retry += 1 - setTimeout(connect, Math.min(300 * 2 ** retry, 5000)) + clearTimeout(wsRetryTimer) + wsRetryTimer = setTimeout(connect, Math.min(300 * 2 ** retry, 5000)) } socket.onopen = () => { opened = true refused = 0 retry = 0 + clearTimeout(wsRetryTimer) report(true) } } @@ -1287,6 +1291,7 @@ export function streamRun( return () => { closed = true clearTimeout(pollTimer) + clearTimeout(wsRetryTimer) socket?.close() } } diff --git a/frontend/src/canvas/FlowCanvas.tsx b/frontend/src/canvas/FlowCanvas.tsx index 013f9108..1e46b7a8 100644 --- a/frontend/src/canvas/FlowCanvas.tsx +++ b/frontend/src/canvas/FlowCanvas.tsx @@ -1,11 +1,14 @@ +import './xyflow.css' import './surface.css' import { createContext, memo, useCallback, useContext, useEffect, useId, useMemo, useRef, useState, type CSSProperties, } from 'react' import { + addEdge, applyEdgeChanges, applyNodeChanges, Background, BackgroundVariant, ControlButton, Controls, MiniMap, ReactFlow, ReactFlowProvider, useReactFlow, useStoreApi, ViewportPortal, type AriaLabelConfig, type Edge, type MiniMapNodeProps, type Viewport, } from '@xyflow/react' +import { registerFlowHelpers } from '../store/flowChanges' import { Crosshair, Lock, PanelRightOpen, Route as RouteIcon, Search } from 'lucide-react' import clsx from 'clsx' import { FlowEdge, MomentContext, RoutingContext, type EdgeRunState, type EdgeView } from './FlowEdge' @@ -28,6 +31,8 @@ import { replayTraceCached } from '../run/useNodeView' import { EDIT_LOCK_TEXT, useStudio, type FlowNode } from '../store/studio' import type { NodeType } from '../types' +registerFlowHelpers({ addEdge, applyEdgeChanges, applyNodeChanges }) + const nodeTypes = { card: NodeCard } const edgeTypes = { flow: FlowEdge } diff --git a/frontend/src/canvas/xyflow.css b/frontend/src/canvas/xyflow.css new file mode 100644 index 00000000..1dcba9dd --- /dev/null +++ b/frontend/src/canvas/xyflow.css @@ -0,0 +1 @@ +@import '@xyflow/react/dist/style.css' layer(xyflow); diff --git a/frontend/src/components/CommandPalette.tsx b/frontend/src/components/CommandPalette.tsx index eba42481..0e29346d 100644 --- a/frontend/src/components/CommandPalette.tsx +++ b/frontend/src/components/CommandPalette.tsx @@ -1,237 +1,40 @@ -import { useEffect, useId, useMemo, useRef, useState, useSyncExternalStore } from 'react' -import type { ComponentType, KeyboardEvent as ReactKeyboardEvent, ReactNode } from 'react' +import { useEffect, useId, useMemo, useRef, useState } from 'react' +import type { KeyboardEvent as ReactKeyboardEvent, ReactNode } from 'react' import { useLocation, useNavigate } from 'react-router-dom' import type { NavigateFunction } from 'react-router-dom' -import { create } from 'zustand' import clsx from 'clsx' import { - Bell, BellOff, BookOpen, CalendarClock, Database, FlaskConical, Gauge, History, Hourglass, Inbox, Keyboard, Layers, - LayoutGrid, MessageSquare, MessageSquarePlus, Monitor, Plus, Search, Settings, SunMoon, Workflow as WorkflowIcon, Wrench, + Bell, BellOff, Hourglass, Inbox, Keyboard, + MessageSquare, MessageSquarePlus, Monitor, Plus, Search, SunMoon, Workflow as WorkflowIcon, } from 'lucide-react' -import { ApiError, api } from '../api/client' +import { api } from '../api/client' import { hasPendingApproval, useCatalog } from '../store/catalog' -import { atLeast, loginOn, prefsLocalOnly, useAuth } from '../store/auth' +import { atLeast, useAuth } from '../store/auth' import { useConversations } from '../store/conversations' import { createWorkflow } from '../canvas/WorkflowPicker' -import { Kbd, Modal, StatusBadge, isComposing, toast } from './ui' +import { Kbd, Modal, StatusBadge, isComposing } from './ui' import { Logo } from './Logo' -import { applyTheme, readThemePref, resolvedTheme } from '../lib/theme' -import type { ThemePref } from '../lib/theme' import { isMac } from '../lib/keys' import { formatDateTime, formatTime, parseServerTime, shortId } from '../lib/format' import { statusLabel } from '../lib/status' -import { APP_TEXT, SCHEDULE_TEXT, WORKFLOW_STATUS_LABEL, runClassLabel, runName } from '../lib/terms' -import { EVAL_TEXT } from '../lib/termsEval' -import { errorMessage } from '../lib/errors' -import { disableNotify, enableNotify, notifySupported, useSignals } from '../lib/notify' +import { APP_TEXT, WORKFLOW_STATUS_LABEL, runClassLabel, runName } from '../lib/terms' +import { notifySupported, useSignals } from '../lib/notify' import { canLeave, leavePass } from '../lib/leave' import type { Run } from '../types' -import type { Principal } from '../typesApp' +import { + closeCommandPalette, closeShortcutHelp, newConversation, openShortcutHelp, setThemePref, + toggleNotify, toggleTheme, usePages, useShell, useTheme, waitedFor, + type PageDef, +} from './shellCommands' -/** - * 外壳的命令表：导航、⌘K 命令面板、快捷键说明共用这一份。 - * - * 工作流、对话、运行都能深链，之前却只能点导航再翻列表去找；界面上还写着 - * 一个根本不存在的 ⌥V。命令面板把"去哪、做什么"收到一个键盘入口里，快捷键 - * 的显示和判定都出自同一份描述（lib/keys），提示写什么就真能按什么。 - */ +export * from './shellCommands' -// ------------------------------------------------------------------------- -// 页面表 -// ------------------------------------------------------------------------- - -export interface PageDef { - to: string - label: string - /** 一句话说这一页是干什么的：导航悬停提示和命令面板里都用 */ - hint: string - icon: ComponentType<{ size?: number; className?: string; strokeWidth?: number }> - shortcut: string - /** 命令面板的搜索词：英文路由名、旧叫法、同义词 */ - keywords: string -} - -// ⌥ + 数字切页：不和浏览器的 ⌘1–9（切标签页，页面拦不住）撞，也不和画布上的 -// Backspace / Delete 以及各处输入框打架——「g c」这类两键序列就是栽在这上面 -export const PAGES: PageDef[] = [ - { to: '/chat', label: '问数据', hint: '描述需求，自动接入数据源、构建工作流并给出结论', icon: MessageSquare, shortcut: 'Alt+1', keywords: 'chat ask 对话 提问 首页' }, - { to: '/studio', label: '编排', hint: '在画布上构建、调试和发布工作流', icon: FlaskConical, shortcut: 'Alt+2', keywords: 'studio canvas 画布 工作流 workflow' }, - // 应用是后加的，紧跟在「编排」后面（用的是编排发布出来的东西）；快捷键取下一个空着的 ⌥9，别的页一个都不动 - { to: '/apps', label: APP_TEXT.nav, hint: APP_TEXT.hint, icon: LayoutGrid, shortcut: 'Alt+9', keywords: 'apps app 应用 待办 inbox 流程 看板 发起' }, - { to: '/runs', label: '记录', hint: '每次运行的过程、成果和待审批', icon: History, shortcut: 'Alt+3', keywords: 'runs history 运行 运行记录 历史 审批' }, - { to: '/data', label: '数据', hint: '接入数据库和表格，供问数据和工作流查询', icon: Database, shortcut: 'Alt+4', keywords: 'data datasource 数据源 数据库 表格 excel csv' }, - { to: '/tools', label: '工具', hint: '工具库、沙箱试验台、自定义工具、MCP', icon: Wrench, shortcut: 'Alt+5', keywords: 'tools mcp sandbox 沙箱 函数' }, - { to: '/knowledge', label: '知识', hint: '知识库、长期记忆、方法论 Skill', icon: BookOpen, shortcut: 'Alt+6', keywords: 'knowledge kb memory skill 知识库 记忆 文档' }, - // 后加的页排在设置前面，设置留在最底下、快捷键也不动：⌥7 按了好几个版本，改了等于让人重新记 - { to: '/schedules', label: SCHEDULE_TEXT.title, hint: SCHEDULE_TEXT.hint, icon: CalendarClock, shortcut: 'Alt+8', keywords: 'schedules schedule cron 定时 计划 周期 自动运行' }, - // 评测（评测-D8）：搭建者以上可见（使用者的页面表在 visiblePages 里收窄，用不着另判），同样排在设置前面；快捷键用最后一个空着的 ⌥0 - { to: '/evals', label: EVAL_TEXT.nav, hint: EVAL_TEXT.hint, icon: Gauge, shortcut: 'Alt+0', keywords: 'evals eval evaluation 评测 回归 题集 准确率 A/B 口径命中 重复度' }, - // 口径目录：已发布口径的入口、依赖和试跑。后加的页，快捷键用没占用的 Alt+L - { to: '/ledger/calibers', label: '口径', hint: '查看已发布口径的入口、依赖，并按参数试运行', icon: Layers, shortcut: 'Alt+L', keywords: 'caliber ledger 口径 指标 试运行 入口' }, - { to: '/settings', label: '设置', hint: '模型接入、偏好、运行环境', icon: Settings, shortcut: 'Alt+7', keywords: 'settings provider 模型 署名 主题 偏好' }, -] - -/** - * 这个人看得到哪几页。开了登录的使用者（user 角色）只有「应用」和「设置」（个人那几项），管理员开了「使用者可以问数据」 - * 时加上问数据（Q2）；搭建者以上和关登录时全部都在 - */ -export function visiblePages(me: Principal | null | undefined): PageDef[] { - if (!loginOn(me) || atLeast(me, 'builder')) return PAGES - const keep = new Set(['/apps', '/settings', ...(me?.user_can_chat ? ['/chat'] : [])]) - return PAGES.filter((p) => keep.has(p.to)) -} - -export function usePages(): PageDef[] { - const me = useAuth((s) => s.me) - return useMemo(() => visiblePages(me), [me]) -} - -// ------------------------------------------------------------------------- -// 外壳浮层的开关 -// ------------------------------------------------------------------------- - -const useShell = create<{ palette: boolean; help: boolean }>(() => ({ palette: false, help: false })) - -export const openCommandPalette = () => useShell.setState({ palette: true, help: false }) -export const closeCommandPalette = () => useShell.setState({ palette: false }) -export const toggleCommandPalette = () => useShell.setState((s) => ({ palette: !s.palette, help: false })) -export const openShortcutHelp = () => useShell.setState({ help: true, palette: false }) -export const closeShortcutHelp = () => useShell.setState({ help: false }) -export const useShellOverlay = () => useShell((s) => s.palette || s.help) - -// ------------------------------------------------------------------------- -// 命令：导航和面板共用 -// ------------------------------------------------------------------------- - -function subscribeTheme(cb: () => void): () => void { - // 设置页的下拉、这里的快速切换、系统换深浅，三条路都会改画出来的主题 - const mo = new MutationObserver(cb) - mo.observe(document.documentElement, { attributes: true, attributeFilter: ['data-theme'] }) - const mq = window.matchMedia('(prefers-color-scheme: light)') - mq.addEventListener('change', cb) - return () => { - mo.disconnect() - mq.removeEventListener('change', cb) - } -} -const themeSnapshot = () => `${readThemePref()}:${resolvedTheme()}` - -/** 当前的主题偏好和实际画的那一套，谁改了都跟得上 */ -export function useTheme(): { pref: ThemePref; resolved: 'light' | 'dark' } { - const snap = useSyncExternalStore(subscribeTheme, themeSnapshot, () => 'system:dark') - const [pref, resolved] = snap.split(':') as [ThemePref, 'light' | 'dark'] - return { pref, resolved } -} - -/** 最后一次选的主题，和它还没存进设置时的那个值（断线时换的） */ -let latestTheme: ThemePref | null = null -let unsavedTheme: ThemePref | null = null - -/** - * 换主题并存进设置。主题的真源是后端 settings 的 ui.theme（App 启动时按它 - * 应用），只写 localStorage 的话下次打开又被改回去。 - * - * 开了登录、又不是管理员（D-s）：服务端的 ui 一组全站共用，改它要管理员，这里只存本机（applyTheme 已经写了 - * localStorage），不发请求、不提示失败 - */ -export async function setThemePref(pref: ThemePref): Promise { - applyTheme(pref) - latestTheme = pref - // 设置页的下拉若开着，听这个事件同步，免得它手上的旧值被再存回去；App 听它 - // 记下「人在这台设备上换过」，重连时就不再拿服务端的旧值翻回去 - window.dispatchEvent(new CustomEvent('agentlab:theme', { detail: pref })) - if (prefsLocalOnly(useAuth.getState().me)) { - unsavedTheme = null - return - } - try { - await saveTheme(pref) - } catch (e) { - const offline = e instanceof ApiError && e.kind === 'network' - toast.warn( - offline ? '主题已在本设备生效，连接服务端后将自动保存到设置' : '主题仅在本设备生效，未能保存到设置', - { detail: errorMessage(e), key: 'theme-save' }, - ) - } -} - -/** PUT 按组整体替换，所以先取回 ui 这一组、只改 theme，别把画布吸附、自动保存这些冲掉 */ -async function saveTheme(pref: ThemePref): Promise { - try { - const current = await api.settings.get() - await api.settings.put({ ui: { ...(current?.ui ?? {}), theme: pref } }) - if (latestTheme === pref) unsavedTheme = null - } catch (e) { - // 连点两下时先发的那次可能后失败：只记最后选的那个 - if (latestTheme === pref) unsavedTheme = pref - throw e - } -} - -/** - * 断线时换的主题当时没存上：后端恢复后补存。有要补的就返回 true，这时 App 不该 - * 再去读服务端——那里还是换之前的旧值。补存又失败就留到下一次恢复，不再弹提示： - * 这一下不是人点的。 - */ -export function flushThemePref(): boolean { - if (!unsavedTheme) return false - if (prefsLocalOnly(useAuth.getState().me)) { - unsavedTheme = null - return true - } - void saveTheme(unsavedTheme).catch(() => {}) - return true -} - -/** 在深浅之间翻：按眼下实际画的那一套翻，「跟随系统」时也一样 */ -export const toggleTheme = () => void setThemePref(resolvedTheme() === 'dark' ? 'light' : 'dark') - -export async function toggleNotify(): Promise { - if (useSignals.getState().notify) { - disableNotify() - toast.info('后台提醒已关闭') - return - } - const permission = await enableNotify() - if (permission === 'granted') toast.ok('后台提醒已开启：标签页在后台时，运行结束、失败或有新的待审批、待办时发送系统通知') - else if (permission === 'denied') { - toast.warn('浏览器已拒绝通知权限', { - detail: '请在地址栏左侧的站点设置中将「通知」改为允许，然后重新打开此开关。', - key: 'notify-denied', - }) - } else if (permission === 'unsupported') toast.warn('当前浏览器不支持系统通知') -} - -export async function newConversation(navigate: NavigateFunction): Promise { - // 先问再建：跳的时候才问的话，人点了「留下」，会话已经建出来了 - if (!(await canLeave())) return - try { - const id = await useConversations.getState().create() - navigate(`/chat/${id}`, leavePass()) - } catch (e) { - toast.error(e) - } -} - -/** 同上：先问离开，再问名字、建工作流 */ +/** 先问离开，再问名字、建工作流 */ async function newWorkflow(navigate: NavigateFunction): Promise { if (!(await canLeave())) return - // 和编排页的「新建工作流」同一条路：同名校验、画布上有没保存的改动先问 await createWorkflow((to, opts) => navigate(to, leavePass(opts)), () => useCatalog.getState().refresh()) } -/** 等了多久，给待审批用：「12 分钟」「3 小时」「8 天」。粗粒度就够，要的是一眼看出晾了多久 */ -export function waitedFor(createdAt: string | null | undefined, now = Date.now()): string | null { - const d = parseServerTime(createdAt) - if (!d) return null - const mins = Math.max(0, Math.floor((now - d.getTime()) / 60_000)) - if (mins < 1) return '不到 1 分钟' - if (mins < 60) return `${mins} 分钟` - const hours = Math.floor(mins / 60) - if (hours < 24) return `${hours} 小时` - return `${Math.floor(hours / 24)} 天` -} - // ------------------------------------------------------------------------- // ⌘K 命令面板 // ------------------------------------------------------------------------- diff --git a/frontend/src/components/shellCommands.ts b/frontend/src/components/shellCommands.ts new file mode 100644 index 00000000..76fded42 --- /dev/null +++ b/frontend/src/components/shellCommands.ts @@ -0,0 +1,186 @@ +import { useMemo, useSyncExternalStore } from 'react' +import type { ComponentType } from 'react' +import type { NavigateFunction } from 'react-router-dom' +import { create } from 'zustand' +import { + BookOpen, CalendarClock, Database, FlaskConical, Gauge, History, Layers, + LayoutGrid, MessageSquare, Settings, Wrench, +} from 'lucide-react' +import { ApiError, api } from '../api/client' +import { atLeast, loginOn, prefsLocalOnly, useAuth } from '../store/auth' +import { useConversations } from '../store/conversations' +import { toast } from './ui' +import { applyTheme, readThemePref, resolvedTheme } from '../lib/theme' +import type { ThemePref } from '../lib/theme' +import { parseServerTime } from '../lib/format' +import { APP_TEXT, SCHEDULE_TEXT } from '../lib/terms' +import { EVAL_TEXT } from '../lib/termsEval' +import { errorMessage } from '../lib/errors' +import { disableNotify, enableNotify, useSignals } from '../lib/notify' +import { canLeave, leavePass } from '../lib/leave' +import type { Principal } from '../typesApp' + +export interface PageDef { + to: string + label: string + /** 一句话说这一页是干什么的：导航悬停提示和命令面板里都用 */ + hint: string + icon: ComponentType<{ size?: number; className?: string; strokeWidth?: number }> + shortcut: string + /** 命令面板的搜索词：英文路由名、旧叫法、同义词 */ + keywords: string +} + +// ⌥ + 数字切页：不和浏览器的 ⌘1–9（切标签页，页面拦不住）撞，也不和画布上的 +// Backspace / Delete 以及各处输入框打架——「g c」这类两键序列就是栽在这上面 +export const PAGES: PageDef[] = [ + { to: '/chat', label: '问数据', hint: '描述需求，自动接入数据源、构建工作流并给出结论', icon: MessageSquare, shortcut: 'Alt+1', keywords: 'chat ask 对话 提问 首页' }, + { to: '/studio', label: '编排', hint: '在画布上构建、调试和发布工作流', icon: FlaskConical, shortcut: 'Alt+2', keywords: 'studio canvas 画布 工作流 workflow' }, + // 应用是后加的，紧跟在「编排」后面（用的是编排发布出来的东西）；快捷键取下一个空着的 ⌥9，别的页一个都不动 + { to: '/apps', label: APP_TEXT.nav, hint: APP_TEXT.hint, icon: LayoutGrid, shortcut: 'Alt+9', keywords: 'apps app 应用 待办 inbox 流程 看板 发起' }, + { to: '/runs', label: '记录', hint: '每次运行的过程、成果和待审批', icon: History, shortcut: 'Alt+3', keywords: 'runs history 运行 运行记录 历史 审批' }, + { to: '/data', label: '数据', hint: '接入数据库和表格，供问数据和工作流查询', icon: Database, shortcut: 'Alt+4', keywords: 'data datasource 数据源 数据库 表格 excel csv' }, + { to: '/tools', label: '工具', hint: '工具库、沙箱试验台、自定义工具、MCP', icon: Wrench, shortcut: 'Alt+5', keywords: 'tools mcp sandbox 沙箱 函数' }, + { to: '/knowledge', label: '知识', hint: '知识库、长期记忆、方法论 Skill', icon: BookOpen, shortcut: 'Alt+6', keywords: 'knowledge kb memory skill 知识库 记忆 文档' }, + // 后加的页排在设置前面，设置留在最底下、快捷键也不动：⌥7 按了好几个版本，改了等于让人重新记 + { to: '/schedules', label: SCHEDULE_TEXT.title, hint: SCHEDULE_TEXT.hint, icon: CalendarClock, shortcut: 'Alt+8', keywords: 'schedules schedule cron 定时 计划 周期 自动运行' }, + // 评测（评测-D8）：搭建者以上可见（使用者的页面表在 visiblePages 里收窄，用不着另判），同样排在设置前面；快捷键用最后一个空着的 ⌥0 + { to: '/evals', label: EVAL_TEXT.nav, hint: EVAL_TEXT.hint, icon: Gauge, shortcut: 'Alt+0', keywords: 'evals eval evaluation 评测 回归 题集 准确率 A/B 口径命中 重复度' }, + // 口径目录：已发布口径的入口、依赖和试跑。后加的页，快捷键用没占用的 Alt+L + { to: '/ledger/calibers', label: '口径', hint: '查看已发布口径的入口、依赖，并按参数试运行', icon: Layers, shortcut: 'Alt+L', keywords: 'caliber ledger 口径 指标 试运行 入口' }, + { to: '/settings', label: '设置', hint: '模型接入、偏好、运行环境', icon: Settings, shortcut: 'Alt+7', keywords: 'settings provider 模型 署名 主题 偏好' }, +] + +/** + * 这个人看得到哪几页。开了登录的使用者（user 角色）只有「应用」和「设置」（个人那几项），管理员开了「使用者可以问数据」 + * 时加上问数据（Q2）；搭建者以上和关登录时全部都在 + */ +export function visiblePages(me: Principal | null | undefined): PageDef[] { + if (!loginOn(me) || atLeast(me, 'builder')) return PAGES + const keep = new Set(['/apps', '/settings', ...(me?.user_can_chat ? ['/chat'] : [])]) + return PAGES.filter((p) => keep.has(p.to)) +} + +export function usePages(): PageDef[] { + const me = useAuth((s) => s.me) + return useMemo(() => visiblePages(me), [me]) +} + +// ------------------------------------------------------------------------- +// 外壳浮层的开关 +// ------------------------------------------------------------------------- + +export const useShell = create<{ palette: boolean; help: boolean }>(() => ({ palette: false, help: false })) + +export const openCommandPalette = () => useShell.setState({ palette: true, help: false }) +export const closeCommandPalette = () => useShell.setState({ palette: false }) +export const toggleCommandPalette = () => useShell.setState((s) => ({ palette: !s.palette, help: false })) +export const openShortcutHelp = () => useShell.setState({ help: true, palette: false }) +export const closeShortcutHelp = () => useShell.setState({ help: false }) +export const useShellOverlay = () => useShell((s) => s.palette || s.help) + +// ------------------------------------------------------------------------- +// 命令：导航和面板共用 +// ------------------------------------------------------------------------- + +function subscribeTheme(cb: () => void): () => void { + const mo = new MutationObserver(cb) + mo.observe(document.documentElement, { attributes: true, attributeFilter: ['data-theme'] }) + const mq = window.matchMedia('(prefers-color-scheme: light)') + mq.addEventListener('change', cb) + return () => { + mo.disconnect() + mq.removeEventListener('change', cb) + } +} +const themeSnapshot = () => `${readThemePref()}:${resolvedTheme()}` + +/** 当前的主题偏好和实际画的那一套，谁改了都跟得上 */ +export function useTheme(): { pref: ThemePref; resolved: 'light' | 'dark' } { + const snap = useSyncExternalStore(subscribeTheme, themeSnapshot, () => 'system:dark') + const [pref, resolved] = snap.split(':') as [ThemePref, 'light' | 'dark'] + return { pref, resolved } +} + +/** 最后一次选的主题，和它还没存进设置时的那个值（断线时换的） */ +let latestTheme: ThemePref | null = null +let unsavedTheme: ThemePref | null = null + +export async function setThemePref(pref: ThemePref): Promise { + applyTheme(pref) + latestTheme = pref + window.dispatchEvent(new CustomEvent('agentlab:theme', { detail: pref })) + if (prefsLocalOnly(useAuth.getState().me)) { + unsavedTheme = null + return + } + try { + await saveTheme(pref) + } catch (e) { + const offline = e instanceof ApiError && e.kind === 'network' + toast.warn( + offline ? '主题已在本设备生效，连接服务端后将自动保存到设置' : '主题仅在本设备生效，未能保存到设置', + { detail: errorMessage(e), key: 'theme-save' }, + ) + } +} + +async function saveTheme(pref: ThemePref): Promise { + try { + const current = await api.settings.get() + await api.settings.put({ ui: { ...(current?.ui ?? {}), theme: pref } }) + if (latestTheme === pref) unsavedTheme = null + } catch (e) { + if (latestTheme === pref) unsavedTheme = pref + throw e + } +} + +export function flushThemePref(): boolean { + if (!unsavedTheme) return false + if (prefsLocalOnly(useAuth.getState().me)) { + unsavedTheme = null + return true + } + void saveTheme(unsavedTheme).catch(() => {}) + return true +} + +export const toggleTheme = () => void setThemePref(resolvedTheme() === 'dark' ? 'light' : 'dark') + +export async function toggleNotify(): Promise { + if (useSignals.getState().notify) { + disableNotify() + toast.info('后台提醒已关闭') + return + } + const permission = await enableNotify() + if (permission === 'granted') toast.ok('后台提醒已开启：标签页在后台时，运行结束、失败或有新的待审批、待办时发送系统通知') + else if (permission === 'denied') { + toast.warn('浏览器已拒绝通知权限', { + detail: '请在地址栏左侧的站点设置中将「通知」改为允许，然后重新打开此开关。', + key: 'notify-denied', + }) + } else if (permission === 'unsupported') toast.warn('当前浏览器不支持系统通知') +} + +export async function newConversation(navigate: NavigateFunction): Promise { + if (!(await canLeave())) return + try { + const id = await useConversations.getState().create() + navigate(`/chat/${id}`, leavePass()) + } catch (e) { + toast.error(e) + } +} + +export function waitedFor(createdAt: string | null | undefined, now = Date.now()): string | null { + const d = parseServerTime(createdAt) + if (!d) return null + const mins = Math.max(0, Math.floor((now - d.getTime()) / 60_000)) + if (mins < 1) return '不到 1 分钟' + if (mins < 60) return `${mins} 分钟` + const hours = Math.floor(mins / 60) + if (hours < 24) return `${hours} 小时` + return `${Math.floor(hours / 24)} 天` +} diff --git a/frontend/src/components/ui.tsx b/frontend/src/components/ui.tsx index 0fdb9551..24fd4c38 100644 --- a/frontend/src/components/ui.tsx +++ b/frontend/src/components/ui.tsx @@ -1,4 +1,7 @@ -import { Component, useCallback, useEffect, useId, useLayoutEffect, useRef, useState, useSyncExternalStore } from 'react' +import { + cloneElement, Component, isValidElement, useCallback, useEffect, useId, useLayoutEffect, useRef, useState, + useSyncExternalStore, +} from 'react' import { useLocation, useNavigate, useParams } from 'react-router-dom' import type { ButtonHTMLAttributes, CSSProperties, ErrorInfo, KeyboardEvent as ReactKeyboardEvent, ReactNode, RefObject, @@ -1514,15 +1517,43 @@ export function Field({ label, htmlFor, hint, error, required, children, classNa const hintId = `${id}-hint` const errorId = `${id}-error` const describedBy = error ? errorId : hint ? hintId : undefined + const invalid = error ? true : undefined + const bindControl = (node: ReactNode): { node: ReactNode; bound: boolean } => { + if (isValidElement>(node) + && (node.type === 'input' || node.type === 'select' || node.type === 'textarea') + && node.props.id == null) { + return { + bound: true, + node: cloneElement(node, { + id, + 'aria-describedby': node.props['aria-describedby'] ?? describedBy, + 'aria-invalid': node.props['aria-invalid'] ?? invalid, + }), + } + } + return { node, bound: false } + } + let content: ReactNode + if (typeof children === 'function') { + content = children({ id, 'aria-describedby': describedBy, 'aria-invalid': invalid }) + } else if (Array.isArray(children)) { + let bound = false + content = children.map((child) => { + if (bound) return child + const res = bindControl(child) + bound = res.bound + return res.node + }) + } else { + content = bindControl(children).node + } return (
- {typeof children === 'function' - ? children({ id, 'aria-describedby': describedBy, 'aria-invalid': error ? true : undefined }) - : children} + {content} {error ?
{error}
: hint diff --git a/frontend/src/dev/fixtures/challenge.tsx b/frontend/src/dev/fixtures/challenge.tsx index b2c7528b..c0cd01c6 100644 --- a/frontend/src/dev/fixtures/challenge.tsx +++ b/frontend/src/dev/fixtures/challenge.tsx @@ -1,3 +1,4 @@ +import '../../canvas/xyflow.css' import { useState, type JSX } from 'react' import { ReactFlow, ReactFlowProvider } from '@xyflow/react' import { Inspector } from '../../canvas/Inspector' diff --git a/frontend/src/dev/fixtures/dual_check.tsx b/frontend/src/dev/fixtures/dual_check.tsx index 33383a10..14e37fc5 100644 --- a/frontend/src/dev/fixtures/dual_check.tsx +++ b/frontend/src/dev/fixtures/dual_check.tsx @@ -1,3 +1,4 @@ +import '../../canvas/xyflow.css' import { useState } from 'react' import type { JSX } from 'react' import { ReactFlow, ReactFlowProvider } from '@xyflow/react' diff --git a/frontend/src/index.css b/frontend/src/index.css index de79d03a..382a89d4 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -1,5 +1,5 @@ +@layer theme, base, xyflow, components, utilities; @import 'tailwindcss'; -@import '@xyflow/react/dist/style.css'; /* ========================================================================== 主题 token。深色是默认（长时间盯画布，深色更省眼），浅色由 data-theme 切换； diff --git a/frontend/src/lib/terms.ts b/frontend/src/lib/terms.ts index aee9c8f0..bc2c4ee8 100644 --- a/frontend/src/lib/terms.ts +++ b/frontend/src/lib/terms.ts @@ -6335,6 +6335,7 @@ export const TOOL_ENDPOINT_TEXT = { query: '查询参数（JSON，可插值）', body: '请求体（JSON，可插值）', noEndpoints: '还没有登记的端点，在「工具 → 端点」中登记', + loadFailed: '端点列表加载失败，已保留当前绑定的端点', } /** diff --git a/frontend/src/pages/ChatPage.tsx b/frontend/src/pages/ChatPage.tsx index 22fbc8a1..ffb025cf 100644 --- a/frontend/src/pages/ChatPage.tsx +++ b/frontend/src/pages/ChatPage.tsx @@ -48,7 +48,13 @@ import { * dense 一个开关。两边各写一套的话，同一次运行在两个页面会讲出不同的故事。 */ export function ChatPage() { - const { ask, stop, load, hydrate, recheck, setScope, setDepth } = useChat() + const ask = useChat((s) => s.ask) + const stop = useChat((s) => s.stop) + const load = useChat((s) => s.load) + const hydrate = useChat((s) => s.hydrate) + const recheck = useChat((s) => s.recheck) + const setScope = useChat((s) => s.setScope) + const setDepth = useChat((s) => s.setDepth) const depth = useChat((s) => s.depth) const { conversationId } = useParams() const navigate = useNavigate() @@ -60,7 +66,12 @@ export function ChatPage() { const turns = useChat((s) => (currentId ? s.byConversation[currentId] : undefined)) const loadState = useChat((s) => (currentId ? s.loadState[currentId] : undefined)) const loadError = useChat((s) => (currentId ? s.loadError[currentId] : undefined)) - const byConversation = useChat((s) => s.byConversation) + const elsewhereId = useChat((s) => { + for (const [id, ts] of Object.entries(s.byConversation)) { + if (id !== currentId && isBusy(ts)) return id + } + return null + }) const backend = useCatalog((s) => s.backend) // 在回收站里的对话可以点开看，但不能接着问：问了它也还在回收站里，左栏找不到 const trashed = useConversations((s) => (currentId ? s.trash.find((c) => c.id === currentId) : undefined)) @@ -147,13 +158,10 @@ export function ChatPage() { const trashBusy = !!trashed && stillRunning(trashed, turns) // 另一个会话在跑：这里照样能问（各跑各的），但得让人知道那边没停 const elsewhere = useMemo(() => { - for (const [id, ts] of Object.entries(byConversation)) { - if (id !== currentId && isBusy(ts)) { - return { id, title: list.find((c) => c.id === id)?.title || ts[ts.length - 1].question } - } - } - return null - }, [byConversation, currentId, list]) + if (!elsewhereId) return null + const ts = useChat.getState().byConversation[elsewhereId] + return { id: elsewhereId, title: list.find((c) => c.id === elsewhereId)?.title || ts?.[ts.length - 1]?.question || '' } + }, [elsewhereId, list]) const blocked = loadState === 'loading' ? '正在加载对话历史…' : loadState === 'error' ? '对话历史加载失败，请点击上方的「重试」' diff --git a/frontend/src/pages/ConversationList.tsx b/frontend/src/pages/ConversationList.tsx index df648109..4ba33116 100644 --- a/frontend/src/pages/ConversationList.tsx +++ b/frontend/src/pages/ConversationList.tsx @@ -1,5 +1,6 @@ import { useEffect, useMemo, useRef, useState, type ReactNode } from 'react' import { useNavigate, type NavigateFunction } from 'react-router-dom' +import { useShallow } from 'zustand/react/shallow' import { ArchiveRestore, ArrowLeft, Check, MessageSquarePlus, PanelLeftClose, PanelLeftOpen, Pencil, RotateCw, Search, Trash2, X, @@ -30,9 +31,19 @@ import type { Conversation } from '../types' * 对话，过了那几秒就该还有地方找回来。真正的删除只在回收站里做，而且要确认。 */ export function ConversationList() { - const { list, trash, showTrash, currentId, loading, error, load, create, rename, archive, restore, setShowTrash } = - useConversations() - const byConversation = useChat((s) => s.byConversation) + const list = useConversations((s) => s.list) + const trash = useConversations((s) => s.trash) + const showTrash = useConversations((s) => s.showTrash) + const currentId = useConversations((s) => s.currentId) + const loading = useConversations((s) => s.loading) + const error = useConversations((s) => s.error) + const load = useConversations((s) => s.load) + const create = useConversations((s) => s.create) + const rename = useConversations((s) => s.rename) + const archive = useConversations((s) => s.archive) + const restore = useConversations((s) => s.restore) + const setShowTrash = useConversations((s) => s.setShowTrash) + const runtimeByConv = useConversationRuntime() const forget = useChat((s) => s.forget) const navigate = useNavigate() // 相对时间和分组一分钟重算一次就够：「刚刚」「3 分钟前」不需要更细 @@ -164,10 +175,10 @@ export function ConversationList() {
{g.label}
{g.items.map((c) => { - const turns = byConversation[c.id] // 取回内存的以内存为准（它是这一刻的）；没打开过的看列表接口带来的最后一轮状态 - const live = turns ? liveStatus(turns) : listStatus(c) - const running = stillRunning(c, turns) + const mem = runtimeByConv[c.id] + const live = mem !== undefined ? (mem.split(':')[0] || null) : listStatus(c) + const running = mem !== undefined ? mem.endsWith(':1') : c.last_status === 'running' const active = c.id === currentId const when = c.last_active_at ?? c.created_at return ( @@ -312,9 +323,12 @@ function TrashView({ items, currentId, onBack }: { }) { const navigate = useNavigate() const emptyTrash = useConversations((s) => s.emptyTrash) - const byConversation = useChat((s) => s.byConversation) + const runtimeByConv = useConversationRuntime() const sorted = useMemo(() => [...items].sort((a, b) => stamp(b) - stamp(a)), [items]) - const idle = items.filter((c) => !stillRunning(c, byConversation[c.id])) + const idle = items.filter((c) => { + const mem = runtimeByConv[c.id] + return mem !== undefined ? !mem.endsWith(':1') : c.last_status !== 'running' + }) const emptyAll = async () => { // 确认框开着的时候谁又跑起来了，以点确认那一刻为准，所以挑两次 @@ -376,9 +390,9 @@ function TrashView({ items, currentId, onBack }: { const active = c.id === currentId const title = c.title || '新对话' const when = c.last_active_at ?? c.created_at - const turns = byConversation[c.id] - const live = turns ? liveStatus(turns) : listStatus(c) - const running = stillRunning(c, turns) + const mem = runtimeByConv[c.id] + const live = mem !== undefined ? (mem.split(':')[0] || null) : listStatus(c) + const running = mem !== undefined ? mem.endsWith(':1') : c.last_status === 'running' return (
{ + return useChat(useShallow((s) => { + const out: Record = {} + for (const [id, turns] of Object.entries(s.byConversation)) { + out[id] = `${liveStatus(turns) ?? ''}:${busyTurn(turns) ? 1 : 0}` + } + return out + })) +} + export const RUNNING_NO_DELETE = '该对话正在运行，请先停止再删除' const titles = (list: Conversation[]) => list.map((c) => `「${c.title || '新对话'}」`).join('、') diff --git a/frontend/src/pages/NotFound.tsx b/frontend/src/pages/NotFound.tsx index dda71497..5ff9276e 100644 --- a/frontend/src/pages/NotFound.tsx +++ b/frontend/src/pages/NotFound.tsx @@ -1,7 +1,7 @@ import { Link, useLocation, useNavigate } from 'react-router-dom' import { ArrowLeft, Search } from 'lucide-react' import { Kbd } from '../components/ui' -import { openCommandPalette } from '../components/CommandPalette' +import { openCommandPalette } from '../components/shellCommands' import { usePageTitle } from '../lib/notify' /** diff --git a/frontend/src/pages/SettingsPage.tsx b/frontend/src/pages/SettingsPage.tsx index 6ccf90dc..f7417c44 100644 --- a/frontend/src/pages/SettingsPage.tsx +++ b/frontend/src/pages/SettingsPage.tsx @@ -12,7 +12,7 @@ import { confirmDialog, DeleteButton, EmptyState, ErrorState, Field, HealthPill, isComposing, Modal, PageHeader, promptDialog, SectionBar, Skeleton, Spinner, TabPanel, Tabs, toast, useRadioGroup, useTabRoute, } from '../components/ui' -import { setThemePref, useTheme } from '../components/CommandPalette' +import { setThemePref, useTheme } from '../components/shellCommands' import { errorMessage } from '../lib/errors' import { formatNumber, shortLabel } from '../lib/format' import { checkHealth, forgetHealth, healthFromServer, setHealth, useHealth } from '../lib/health' diff --git a/frontend/src/pages/ToolsPage.tsx b/frontend/src/pages/ToolsPage.tsx index f80b7290..0479da15 100644 --- a/frontend/src/pages/ToolsPage.tsx +++ b/frontend/src/pages/ToolsPage.tsx @@ -631,7 +631,7 @@ function CustomTools() { const [rows, setRows] = useState(null) const [error, setError] = useState(null) const [editing, setEditing] = useState(null) - const endpoints = useEndpoints() + const { items: endpoints } = useEndpoints() // ?edit=<工具名>：运行失败说「参数定义写坏了」时直接打开那一个的编辑框，不用再翻列表。 // 读完就从地址里摘掉，免得关了弹窗、刷新一下又弹出来 @@ -774,7 +774,7 @@ function CustomToolEditor({ row, onClose, onSaved }: { row: any; onClose: () => const nameError = form.name && !TOOL_NAME_RE.test(form.name) ? '只能包含英文字母、数字和下划线，且不能以数字开头' : null const dirty = JSON.stringify(form) !== JSON.stringify(initial) // 绑定登记的端点（§3.9）：config 是 {endpoint, method, path, query, body}，只填路径，地址和认证头由端点按环境提供 - const endpoints = useEndpoints() + const { items: endpoints, failed: endpointsFailed } = useEndpoints() const bound = form.kind === 'http' && !!form.config.endpoint const missingConfig = [ form.kind === 'http' && !bound && !form.config.url && 'URL', @@ -892,13 +892,15 @@ function CustomToolEditor({ row, onClose, onSaved }: { row: any; onClose: () =>
{form.kind === 'http' && ( - {(p) => ( )} @@ -1019,11 +1021,16 @@ function mcpServerRecord(row: any): HealthRecord | undefined { const mcpConnection = (f: any) => JSON.stringify([f.transport, f.command.trim(), f.args.trim(), f.env ?? {}, f.url.trim(), f.headers ?? {}, f.endpoint_id ?? null]) -/** 登记的端点（绑定用）。取不到时为空列表，照样能不绑定 */ -function useEndpoints(): EndpointRow[] | null { - const [rows, setRows] = useState(null) - useEffect(() => { api.endpoints.list().then(({ items }) => setRows(items), () => setRows([])) }, []) - return rows +/** 登记的端点（绑定用）。加载失败时保留 null，避免把已绑定的端点误报为已删除 */ +function useEndpoints(): { items: EndpointRow[] | null; failed: boolean } { + const [state, setState] = useState<{ items: EndpointRow[] | null; failed: boolean }>({ items: null, failed: false }) + useEffect(() => { + api.endpoints.list().then( + ({ items }) => setState({ items, failed: false }), + () => setState({ items: null, failed: true }), + ) + }, []) + return state } /** @@ -1242,7 +1249,7 @@ function McpEditor({ row, onClose, onSaved }: { onSaved: (name: string, reconnected: boolean) => void }) { const admin = useAtLeast('admin') - const endpoints = useEndpoints() + const { items: endpoints, failed: endpointsFailed } = useEndpoints() const [initial] = useState(() => ({ name: row.name ?? '', transport: row.transport ?? 'stdio', command: row.command ?? '', args: (row.args ?? []).join(' '), env: row.env ?? {}, url: row.url ?? '', enabled: row.enabled ?? true, @@ -1326,12 +1333,15 @@ function McpEditor({ row, onClose, onSaved }: { ) : ( <> - + {(p) => ( setQuery(e.target.value)} + setQuery(e.target.value)} placeholder="按指标名、标识、物理表或工作流查找" aria-label="查找口径" />
diff --git a/frontend/src/pages/ledger/CaliberDrafts.tsx b/frontend/src/pages/ledger/CaliberDrafts.tsx index a17ddd68..a6748a91 100644 --- a/frontend/src/pages/ledger/CaliberDrafts.tsx +++ b/frontend/src/pages/ledger/CaliberDrafts.tsx @@ -1,492 +1,40 @@ /** * 从业务文档起草口径（/ledger/drafts；服务端 app/ledger/docdraft.py、docattach.py）。 * - * 主路径一次到位：选文档 → 模型读文档、对照数据目录产出定义 + 取数 + 入口字段 + 指标表达式 → 服务端校验后落成草稿图 - * → 后台自动只读试运行 → 这里展示「定义 + 数 + 原文引用」→ 人去画布审改并发布。发布才需要人签字。 - * - 发起：选知识库文档和**必选的数据源**（取数 SQL 只对照它的表和列写），可加一段起草要求；页面轮询详情直到状态离开 - * drafting（持续有进度时继续轮询，连续十五分钟无进展才暂停轮询；后台任务独立运行）； - * - 试运行：草稿顶部显示这次自动试运行的状态和每个指标的值，running 时每两秒取一次，最多等一分钟；必填入口没有值时停在 - * needs_input，在入口字段里填了再点「试运行」；失败照实写原因； - * - 审读：指标默认收成一行（名字、单位、值和未决问题数），展开看单位、粒度、指标表达式、定义和逐字核对过的引用原文 - * （第 N 段），逐条的未决问题可以勾掉；被丢弃的条目单独列出原因；入口字段列出名字、类型、必填和缺省值； - * - 发布：主按钮去画布审改并发布。「接上一次运行」只是辅助入口：已经有答案时让它认这份定义。 - * - * 业务规则（K06 的工作台）：服务端的能力描述（binding）说得出这张卡能按字段改哪几项业务写法（统计时间口径、 - * 退款处理）时，这里开出下拉；改完先把「哪几条指标、现在什么写法 → 保存后什么写法」列出来，再**一次**提交所有受 - * 影响指标的字段操作（PATCH 的 operations，带基准版本和请求键）。保存回执说清换成哪一版、绑定同步到哪一步、 - * 旧验证怎么失效，并按保存前的值给出「撤销这次修改」（逆操作 + 新请求键，再存一版，不整份覆盖）。写不进去 - * （基准版本过期）时保留这里的写法选择，刷新详情后由人显式再点一次；复杂或同步不了的照实写原因和技术待办。 - * 能力描述为空、认不出唯一的口径卡、没有版本号的资产保持只读。业务规则、变更预览和验证这几块只写业务说法， - * 不出现取数 SQL、节点 id 和字段路径。 - * - * 文档更新（K11 的增量重导入）：新一段材料重抽成候选 R，和现在图上人工维护的 L、上次被接受的导入值 B 逐字段比。 - * **R 只是候选，不改 L、不改 B**：落到图上只有「接受选定字段」这一次显式动作（一次提交里新增一版图 + 按被接受的那几格 - * 推进 B）。逐格写清 B / L / R 和状态：无冲突更新、人工修改冲突、新增候选、需要人工对齐（没有可比基线时必须人工 - * 对齐到某条指标）、**漏提取**（不当删除，L 上的原值原样留着，也不能接受）。没接受过的格子不推进 B，所以这块另写 - * 清字段基线覆盖到哪一步（有基线不等于整张都有：partial 要明确说出来）。被中断的导入由人点「恢复」显式续跑， - * 已接受 / 拒绝的不重复动。基准版本过期时保留选好的格子，刷新后由人再点一次。 - * - * 用一句话改口径（K12）：一句话只产出一份**只读提案**（K03 同一形状的 operations + 影响摘要 + 澄清项），页面上 - * 只写「哪条指标、哪一项写法、现在什么 → 保存后什么」，不摆 JSON、节点编号和字段路径。含糊的业务条件先显示澄清项、 - * 一条操作都不产出；确认（confirm=true）之后才按同一批操作落一版，失败时页面上什么都不改。 - * - * 验证与审阅（K05 的读法）：这份草稿的验证记录（状态、独立期望、实际值、覆盖判断、审阅决定）和总情况一句 - * 「结论」分开写——试运行成功、绑定对齐、样例对得上都不等于验证通过；没有独立期望就照实说只记了试运行结果。 - * 审阅只有调真实接口这一条路（审批人取服务端身份），结果照服务端交回的说。 - * - * 页面不显示执行值：取数 SQL 只作为模板只读展示，具体取数到画布上看。服务端把细状态折成四种阶段（起草中 / 待审 / - * 已丢弃 / 失败），页面按阶段筛、按阶段画；细状态放在标签的悬停提示里。 + * 页面作为编排外壳组装各子面板： + * - `drafts/helpers.ts`：阶段映射、幂等键、K05/K06/K11/K12 纯函数 + * - `drafts/DraftSharedBlocks.tsx`：发起导入栏、阶段徽标、指标列表、SQL 模板、发布前检查与接运行区 + * - `drafts/DraftTrialSection.tsx`：自动试运行状态轮询与入口字段表单 + * - `drafts/DraftRulesSection.tsx`：业务规则工作台（K06） + * - `drafts/DraftImportsSection.tsx`：文档更新与三方逐字段重导入（K11） + * - `drafts/DraftNlSection.tsx`：用一句话改口径预览与确认（K12） + * - `drafts/DraftValidationSection.tsx`：验证结论、独立期望与审阅记录（K05） */ import { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { Link } from 'react-router-dom' -import { - AlertTriangle, Check, ChevronLeft, FileSearch, FileText, PenLine, Play, RefreshCw, RotateCw, ScrollText, Sparkles, Undo2, X, -} from 'lucide-react' +import { ChevronLeft, FileText, PenLine, RefreshCw, ScrollText } from 'lucide-react' import clsx from 'clsx' import { api, ApiError } from '../../api/client' import { EmptyState, ErrorState, Notice, PageHeader, Skeleton, Spinner, promptDialog, toast } from '../../components/ui' import { formatDateTime, formatTime } from '../../lib/format' import { LEDGER_UI } from '../../lib/termsLedger' -import { useAtLeast } from '../../store/auth' -import { TrialResult, TrialValidation } from './TrialResult' import { ContinuationPanel, DefinitionInventory, QuerySemanticEditor } from './SemanticPanel' import { OntologyPanel } from './OntologyPanel' import { MigrationPanel } from './MigrationPanel' import type { DataSourceName, KbDocument, Run } from '../../types' -import type { - CaliberDraft, CaliberDraftStage, CaliberDraftStatus, CaliberDraftSummary, DraftAlignment, DraftBinding, DraftChangeReceipt, - DraftCitation, DraftEditableField, DraftEntryField, DraftExpectationSource, DraftImportAcceptOut, DraftImportField, - DraftImportItem, DraftImportRun, DraftMetric, DraftOperation, DraftTrialMetric, DraftTrialOut, DraftTrialValidation, - DraftValidationsOut, DraftValidationRecord, DraftValidationReview, NlChangeProposal, QualityReport, -} from '../../typesLedger' +import type { CaliberDraft, CaliberDraftSummary, DraftCitation } from '../../typesLedger' +import { DRAFT_POLL_LIMIT_MS, DRAFT_POLL_MS, FILTERS, newRequestId, stageOf, type Filter } from './drafts/helpers' +import { + AlignmentBlock, DraftAttachSection, DraftChecksSection, DraftMetricItem, DraftQuestionList, + DraftSqlSection, DraftStartSection, StatusPill, +} from './drafts/DraftSharedBlocks' +import { DraftEntryFieldsSection, DraftTrialSection, useDraftTrial } from './drafts/DraftTrialSection' +import { DraftRulesSection, useDraftRules } from './drafts/DraftRulesSection' +import { DraftImportsSection, useDraftImports } from './drafts/DraftImportsSection' +import { DraftNlSection, useDraftNlChange } from './drafts/DraftNlSection' +import { DraftValidationSection, useDraftValidations } from './drafts/DraftValidationSection' const D = LEDGER_UI.drafts -/** 业务规则（K04 的能力 + K06 的改法）的文案 */ -const R = LEDGER_UI.rules -/** 验证与审阅（K05 的记录）的文案 */ -const V = LEDGER_UI.validation -/** 文档更新 / 增量重导入（K11）的文案 */ -const I = LEDGER_UI.imports -/** 用一句话改口径（K12）的文案 */ -const N = LEDGER_UI.nl - -/** 列表按阶段筛：全部、起草中、待审、已丢弃、失败；细状态不再单独占筛选项 */ -type Filter = '' | CaliberDraftStage -const FILTERS: Filter[] = ['', 'drafting', 'review', 'discarded', 'failed'] - -/** 细状态 → 界面阶段；旧服务端没有 stage 时按 status 推，页面不会因为少一个字段就空 */ -const stageOf = (draft: CaliberDraftSummary | CaliberDraft): CaliberDraftStage => - draft.stage ?? (draft.status === 'drafting' ? 'drafting' : draft.status === 'rejected' ? 'discarded' - : draft.status === 'failed' ? 'failed' : 'review') - -/** 起草轮询：每两秒一次，多轮续跑要几分钟，最多等十分钟 */ -const DRAFT_POLL_MS = 2_000 -const DRAFT_POLL_LIMIT_MS = 15 * 60_000 - -/** 验证记录默认摆最近几条，其余收在「展开其余 N 条」后面 */ -const RECORDS_SHOWN = 3 - -/** - * 幂等键（K02）：服务端按「主体 + 工作流 + 目标卡 + request_id」认这一次修改。同一批勾选的问题反复提交（网络 - * 断了、手点重试）用同一个键，服务端认得出是同一件事、不会多存一版；勾选或草稿变了才换一个。 - */ -const newRequestId = () => - typeof crypto !== 'undefined' && 'randomUUID' in crypto - ? crypto.randomUUID() - : `patch-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}` - -/** 阶段标签：起草中 / 待审 / 已丢弃 / 起草失败；细状态放在 title 里，鼠标悬停能看到「待确认问题」这类差别 */ -function StatusPill({ stage, status }: { stage: CaliberDraftStage; status?: CaliberDraftStatus }) { - const color = stage === 'review' - ? (status === 'ready' || status === 'attached' ? 'var(--ok)' : 'var(--warn)') - : stage === 'drafting' ? 'var(--accent)' : stage === 'discarded' ? 'var(--text-faint)' : 'var(--err)' - return ( - - {D.stage[stage] ?? stage} - - ) -} - -/** span 是引用在段落原文里的 [start, end)：切成「前 / 引用 / 后」三段高亮 */ -function spanParts(content: string, span?: [number, number]): [string, string, string] | null { - if (!span) return null - const [start, end] = span - if (start < 0 || end > content.length || start >= end) return null - return [content.slice(0, start), content.slice(start, end), content.slice(end)] -} - -function QuoteBody({ content, span, quote }: { content: string; span?: [number, number]; quote: string }) { - const parts = spanParts(content, span) - if (!content) return <>{quote} - if (!parts) return <>{content} - return ( - <> - {parts[0]} - - {parts[1]} - - {parts[2]} - - ) -} - -/** 草稿和已发布口径的对齐：新的、已有出处的、定义不同的 */ -function AlignmentBlock({ alignment }: { alignment: DraftAlignment }) { - const added = alignment.new ?? [] - const duplicate = alignment.duplicate ?? [] - const conflict = alignment.conflict ?? [] - if (!added.length && !duplicate.length && !conflict.length) return null - const row = (label: string, items: { name: string; reason?: string }[]) => items.length > 0 && ( -
  • - {label} - {items.map((item) => ( - - {item.name} - {item.reason ? `（${item.reason}）` : ''} - - ))} -
    • - ) - return ( -
    -

    {D.alignment}

    -
      - {row(D.alignmentNew, added.map((name) => ({ name })))} - {row(D.alignmentDuplicate, duplicate)} - {row(D.alignmentConflict, conflict)} -
    -
    - ) -} - -/** 缺省值直接读：字符串照写，其他转成 JSON 文本 */ -function defaultText(value: unknown): string { - if (typeof value === 'string') return value - const text = JSON.stringify(value) - return text ?? String(value) -} - -/** 入口字段预填的日期：YYYY-MM-DD，按本地日期算 */ -function dayText(offsetDays: number): string { - const day = new Date() - day.setDate(day.getDate() + offsetDays) - const pad = (n: number) => String(n).padStart(2, '0') - return `${day.getFullYear()}-${pad(day.getMonth() + 1)}-${pad(day.getDate())}` -} - -/** - * 名字或说明里带「开始 / start」的日期字段预填七天前，带「结束 / end」的预填今天： - * 这两个值基本总是要填的，给个能改的范围比留空省一步。其余必填字段留空，让人自己写。 - */ -function datePrefill(field: DraftEntryField): string | null { - if ((field.type || 'string') !== 'string') return null - const hint = `${field.name} ${field.description ?? ''}`.toLowerCase() - if (hint.includes('开始') || /(^|[^a-z])start([^a-z]|$)/.test(hint)) return dayText(-7) - if (hint.includes('结束') || /(^|[^a-z])end([^a-z]|$)/.test(hint)) return dayText(0) - return null -} - -/** 入口字段的初值：有缺省值就用缺省值，日期字段按上面的规则预填，其余留空 */ -function entryInitialValues(fields: DraftEntryField[], prefilled: Record): Record { - const values: Record = {} - for (const field of fields) { - const hasDefault = field.default !== undefined && field.default !== null && field.default !== '' - if (hasDefault) { values[field.name] = defaultText(field.default); continue } - const guess = datePrefill(field) - values[field.name] = guess ?? '' - if (guess) prefilled[field.name] = true - } - return values -} - -/** 必填字段是不是都填了：布尔量总有值，数字要能读成数 */ -function entryReady(fields: DraftEntryField[], values: Record): boolean { - return fields.every((field) => { - if (!field.required) return true - const type = field.type || 'string' - if (type === 'boolean') return true - const text = (values[field.name] ?? '').trim() - if (!text) return false - return type === 'number' || type === 'integer' ? Number.isFinite(Number(text)) : true - }) -} - -/** 表单值换成接口要的 inputs：空字符串不传（服务端会用缺省值），数字转成数，布尔转成开关 */ -function entryInputs(fields: DraftEntryField[], values: Record): Record { - const inputs: Record = {} - for (const field of fields) { - const type = field.type || 'string' - const text = values[field.name] ?? '' - if (type === 'boolean') { inputs[field.name] = text === 'true'; continue } - if (!text.trim()) continue - if (type === 'number' || type === 'integer') { - const num = Number(text) - if (Number.isFinite(num)) inputs[field.name] = num - continue - } - inputs[field.name] = text - } - return inputs -} - -// ---------------------------------------------------------------- K06：业务规则、变更预览、验证记录 - -/** 业务值的说法：能力描述给了机器值 → 给人看的说法就用它，没有就照原样写 */ -const choiceLabel = (field: DraftEditableField | undefined, value: string): string => - field?.value_labels?.[value] ?? value - -/** 一个值转成页面上的一行文字：字符串照写，数字、布尔照写，对象转成 JSON 文本 */ -function valueText(value: unknown): string { - if (value == null) return '' - if (typeof value === 'string') return value - if (typeof value === 'number' || typeof value === 'boolean') return String(value) - const text = JSON.stringify(value) - return text ?? String(value) -} - -/** - * 表单初值：这一项在**所有**声明它的指标上写法一致时就是那个值，不一致（或没人声明）时留空。 - * 留空表示「保持各自现在的写法」——不替人选一个默认值，也就不会因为打开页面就凭空改掉谁。 - */ -function ruleInitialValues(binding?: DraftBinding): Record { - const out: Record = {} - if (!binding) return out - const declared = Object.values(binding.metric_values ?? {}) - for (const field of binding.editable_fields ?? []) { - const seen = new Set(declared.map((values) => values[field.field]).filter((value): value is string => !!value)) - out[field.field] = seen.size === 1 ? [...seen][0] : '' - } - return out -} - -/** 这一项这些指标里有没有人声明：一个都没有时这一项改不了（没有落点的取数不会被硬改） */ -const declaringMetrics = (binding: DraftBinding | undefined, field: string): string[] => - Object.entries(binding?.metric_values ?? {}) - .filter(([, values]) => !!values[field]).map(([metricId]) => metricId) - -/** 这次选中的写法会改到哪几条指标、每条从什么变成什么：只有声明了这一项、而且写法和选中的不一样才算 */ -function rulePlan(binding: DraftBinding | undefined, values: Record) { - const operations: DraftOperation[] = [] - const changes: { metricId: string; name: string; field: DraftEditableField; from: string; to: string }[] = [] - for (const field of binding?.editable_fields ?? []) { - const chosen = values[field.field] ?? '' - if (!chosen) continue - for (const [metricId, declared] of Object.entries(binding?.metric_values ?? {})) { - const now = declared[field.field] - if (!now || now === chosen) continue - operations.push({ op: 'set', metric_id: metricId, path: `/${field.field}`, value: chosen }) - changes.push({ metricId, name: '', field, from: now, to: chosen }) - } - } - return { operations, changes } -} - -/** - * 撤销这次修改要提交的逆操作：按**保存前**每一条指标声明的值构造——原来有值就设回去，原来没声明就删掉。 - * 撤销和保存一样是新增一版（当前版本 + 新的请求键），不整份覆盖、不改写已经存下的版本。 - */ -function undoOperations(binding: DraftBinding | undefined, operations: DraftOperation[]): DraftOperation[] { - return operations.map((op) => { - const field = op.path.replace(/^\//, '') - const before = binding?.metric_values?.[op.metric_id]?.[field] - return before - ? { op: 'set', metric_id: op.metric_id, path: op.path, value: before } - : { op: 'remove', metric_id: op.metric_id, path: op.path } - }) -} - -// ---------------------------------------------------------------- K11 / K12：字段的说法、三方值、一句话的影响 - -/** - * 字段路径 → 给人看的标签（K03 白名单和 app/ledger/changes.FIELDS 同一套，业务字段另有能力描述里的标签）。 - * 页面上只写这些业务说法，路径本身不上来。 - */ -const FIELD_LABELS: Record = { - '/synonyms': '同义词', - '/unit': '单位', - '/decimals': '小数位', - '/format': '展示格式', - '/expression': '取值表达式', - '/grain': '粒度', - '/time_field': '时间字段', - '/numerator': '分子', - '/denominator': '分母', - '/dimensions': '维度', - '/filters': '过滤条件', - '/definition/text': '口径说明', - '/definition/scope': '适用范围', - '/definition/exclusions': '不包含', - '/definition/citations': '引用原文', - '/time_role': '统计时间口径', - '/refund_rule': '退款处理', -} -/** 引用原文在 B / L / R 里是一段带段落 id 的结构化内容：只写有几段，不摆内容 */ -const CITATIONS_PATH = '/definition/citations' - -const fieldLabel = (path: string, binding?: DraftBinding): string => { - const name = path.replace(/^\//, '') - const editable = (binding?.editable_fields ?? []).find((field) => field.field === name) - return editable?.label ?? FIELD_LABELS[path] ?? I.otherField -} - -/** 业务字段的机器值 → 给人看的说法（能力描述里有就用它，没有就照原样写） */ -const valueLabelOf = (value: unknown, binding?: DraftBinding, path?: string): string => { - if (value == null) return I.noValue - if (typeof value === 'boolean') return value ? '是' : '否' - if (typeof value === 'number') return String(value) - if (typeof value === 'string') { - const name = String(path ?? '').replace(/^\//, '') - const editable = (binding?.editable_fields ?? []).find((field) => field.field === name) - return editable?.value_labels?.[value] ?? value - } - if (Array.isArray(value)) { - if (path === CITATIONS_PATH) return I.citationCount(value.length) - const parts = value.map((item) => valueLabelOf(item, binding, path)) - return parts.length ? parts.join('、') : I.emptyValue - } - if (typeof value === 'object') return I.objectValue - return String(value) -} - -/** 一格的一行字：这一格没有时返回 null（整行不画），不把「没有这一格」写成空值 */ -function cellText(cell: DraftImportField, which: 'b' | 'l' | 'r', binding?: DraftBinding): string | null { - const present = which === 'b' ? cell.b_present : which === 'l' ? cell.l_present : cell.r_present - if (!present) return null - return valueLabelOf(cell[which], binding, cell.path) -} - -/** 选中的格子：一条工作项 + 一条字段路径（未对齐的工作项没有可接受的字段，拒绝对齐前不能选） */ -const pickKey = (itemId: string, path: string): string => `${itemId}\u0000${path}` - -/** - * 这条工作项上「无冲突更新 + 新增候选」的格子：一次收掉的按钮按它算有几格，并把这几格**明确列给服务端** - * （不发 auto：页面列了几格就只接受几格）。已经接受 / 拒绝过的格子不再算进来，抽坏了的工作项一个也不收。 - */ -const cleanPaths = (item: DraftImportItem): string[] => - itemFailed(item) ? [] : item.fields.filter((cell) => (cell.state === 'update' || cell.state === 'new') - && cell.decided !== 'accepted' && cell.decided !== 'rejected').map((cell) => cell.path) - -/** 服务端标的「这条抽坏了」（引用没有给过模型的原文）：也可能是老服务端只给了状态 */ -const itemFailed = (item: DraftImportItem): boolean => item.failed ?? item.status === 'failed' - -/** 这个工作项现在能不能接受：抽坏了的（failed）、没对齐上的，一律不能 */ -const itemAccepts = (item: DraftImportItem): boolean => !itemFailed(item) && !!item.metric_id - -/** - * 按这一次导入**现在**的现状裁掉不再适用的选择：格子没了、已经决定过、不能接受、工作项抽坏了的都不留下。 - * stale 之后重新读回现状时用它——不拿旧的选择去提交（旧的选择可能对着已经变了的 L）。 - */ -function prunePicks(picks: Record, run: DraftImportRun): - Record { - const out: Record = {} - for (const item of run.items) { - for (const cell of item.fields) { - const choice = picks[pickKey(item.item_id, cell.path)] - if (!choice || cell.decided != null) continue - if (choice === 'accept' && (!itemAccepts(item) || !cell.acceptable)) continue - out[pickKey(item.item_id, cell.path)] = choice - } - } - return out -} - -/** 还没处置完的工作项：恢复只对这些有意义（已接受 / 拒绝 / 不变的不动） */ -const unfinishedItems = (run: DraftImportRun): number => - run.items.filter((item) => !['accepted', 'rejected', 'unchanged'].includes(item.status)).length - -/** 一句话提案里的一处影响：哪条指标、哪一项写法、现在什么 → 保存后什么（页面上只写业务说法） */ -interface NlChange { - metricId: string - name: string - label: string - from: string - to: string - remove: boolean -} - -/** 提案里的字段操作 → 人读得懂的影响：指标名 + 字段标签 + 前后值，不摆 path / metric_id / JSON */ -function nlChanges(draft: CaliberDraft, operations: DraftOperation[]): NlChange[] { - const metrics = new Map((draft.metrics ?? []).filter((metric) => metric.id).map((metric) => [metric.id as string, metric])) - return (operations ?? []).map((operation) => { - const name = operation.path.replace(/^\//, '') - const metric = metrics.get(operation.metric_id) - const declared = draft.binding?.metric_values?.[operation.metric_id]?.[name] - ?? (metric ? (metric as unknown as Record)[name] : undefined) - const remove = operation.op === 'remove' - return { - metricId: operation.metric_id, - name: metric?.name ?? N.unknownMetric, - label: fieldLabel(operation.path, draft.binding), - from: valueLabelOf(declared, draft.binding, operation.path), - to: remove ? N.removeValue : valueLabelOf(operation.value, draft.binding, operation.path), - remove, - } - }) -} - -/** - * 落版之后服务端交回的操作和预览里的是不是同一批（按「改哪条指标的哪个字段、怎么改」比）。 - * 同一段原文应当解析出同一批；不一样就说明页面显示的和实际落下的不是一回事，要照实提示。 - * 审计读不回来（老服务端不给这一段）时返回 null：判不了就说判不了，不假装一致。 - */ -function sameOperations(previewed: DraftOperation[], - applied: { op: string; metric_id: string; path: string }[] | undefined): boolean | null { - if (!applied) return null - const key = (op: { op: string; metric_id: string; path: string }) => `${op.op}|${op.metric_id}|${op.path}` - const before = [...new Set(previewed.map(key))].sort() - const after = [...new Set(applied.map(key))].sort() - return before.length === after.length && before.every((item, index) => item === after[index]) -} - -/** - * 独立期望里填的一格：整数、小数写成数，true / false 写成布尔，方括号花括号按 JSON 读，其余照字面写。 - * 服务端按**原值**比较（不做隐式舍入），这里也不替它换算。 - */ -function expectValue(text: string): unknown { - const trimmed = text.trim() - if (/^-?\d+(?:\.\d+)?$/.test(trimmed)) return Number(trimmed) - if (trimmed === 'true' || trimmed === 'false') return trimmed === 'true' - if (trimmed.startsWith('[') || trimmed.startsWith('{')) { - try { return JSON.parse(trimmed) } catch { return trimmed } - } - return trimmed -} - -/** - * 试跑响应里的质量结论：只有服务端给了认得的状态才算一份质量报告。 - * stale / needs_input / running 这些分支的 validation 可能只有验证记录那一段、甚至什么都没有—— - * 读不到状态就是「没有质量结论」，不能当成通过（也不能拿一次试跑成功代替验证）。 - */ -function qualityReportOf(validation?: DraftTrialValidation | null): QualityReport | null { - const status = validation?.status - if (status !== 'pass' && status !== 'fail' && status !== 'unknown') return null - return { status, checks: validation?.checks ?? [] } -} - -/** - * 覆盖判断：**只有 === true 才算覆盖**。false 是不覆盖，null / 缺字段是判不了（读不到当前依据）， - * 三者要在界面上分开写，不能把判不了默认成覆盖。 - */ -const coversText = (value: boolean | null | undefined): string => - value === true ? V.covers : value === false ? V.notCovers : V.coversUnknown - -/** - * 一条审阅的确认范围有没有盖住整条记录的全部指标：只确认了其中几条时不能写成整卡通过。 - * scope 空着（或者读数缺字段）一律按「没盖住」算——宁可不写通过，不替服务端把范围补全。 - */ -function scopeCovers(record: DraftValidationRecord, review: DraftValidationReview | null | undefined): boolean { - const scope = new Set(review?.scope ?? []) - const ids = record.metric_ids ?? [] - if (!ids.length) return scope.size > 0 - return ids.every((id) => scope.has(id)) -} - -/** 这条记录里没有被确认范围盖住的指标 */ -function uncoveredMetrics(record: DraftValidationRecord, review: DraftValidationReview | null | undefined): string[] { - const scope = new Set(review?.scope ?? []) - return (record.metric_ids ?? []).filter((id) => !scope.has(id)) -} export function CaliberDrafts() { const [drafts, setDrafts] = useState(null) @@ -504,7 +52,6 @@ export function CaliberDrafts() { const [pickedDocs, setPickedDocs] = useState([]) const [datasourceId, setDatasourceId] = useState('') const [guidance, setGuidance] = useState('') - /** 每次跑都不变的条件（工厂、产线、属性组）：要求写死在 SQL 里，不做成入口字段 */ const [fixedFilters, setFixedFilters] = useState('') const [submitting, setSubmitting] = useState(false) @@ -514,101 +61,17 @@ export function CaliberDrafts() { const [recentRuns, setRecentRuns] = useState(null) const [attachOut, setAttachOut] = useState<{ workflow_id: string; version: number; warnings: string[] } | null>(null) - /** 起草中的草稿：每两秒取一次详情，状态离开 drafting 或超过十分钟就停 */ const [pollingId, setPollingId] = useState(null) const [pollTimedOut, setPollTimedOut] = useState(false) - - /** 这次自动试运行的结果；running 时单独轮询，和草稿抽取的轮询互不相干 */ - const [trial, setTrial] = useState(null) - const trialRequestSeq = useRef(0) - const [trialPollId, setTrialPollId] = useState(null) - const [trialTimedOut, setTrialTimedOut] = useState(false) - const [trialError, setTrialError] = useState(null) const [jobError, setJobError] = useState(null) const [jobBusy, setJobBusy] = useState(null) + const [droppedExpanded, setDroppedExpanded] = useState(false) - /** 试运行的表单值：入口字段可改，必填没填满时按钮禁用；prefilled 记哪些是页面替人预填的 */ - const [entryValues, setEntryValues] = useState>({}) - const [prefilled, setPrefilled] = useState>({}) - const [trialStarting, setTrialStarting] = useState(false) - - /** 引用所在段落的正文和段号：按文档懒取，取过的记在这里 */ const [chunks, setChunks] = useState>({}) const asked = useRef(new Set()) - /** 连续点开两份草稿时，晚到的那次详情不覆盖现在打开的那份 */ const openSeq = useRef(0) - /** 这次「回答并移除」的幂等键：勾选或草稿变了才换（K02） */ const pendingWrite = useRef<{ signature: string; requestId: string }>({ signature: '', requestId: '' }) - /** K06 业务规则：表单里选中的写法（字段名 → 机器值；空串是「保持各自现在的写法」） */ - const [ruleValues, setRuleValues] = useState>({}) - /** 这次保存的幂等键：同一批字段操作反复提交用同一个键，选择变了才换 */ - const rulePending = useRef<{ signature: string; requestId: string }>({ signature: '', requestId: '' }) - const [ruleBusy, setRuleBusy] = useState<'save' | 'undo' | null>(null) - const [ruleError, setRuleError] = useState(null) - /** 写入被拒（基准版本过期）：说清你基于哪一版、服务端现在是哪一版，选择一个字不动 */ - const [ruleStale, setRuleStale] = useState<{ current: number | null; base: number | null } | null>(null) - /** 这次保存（或撤销）的回执：换成哪一版、绑定同步到哪一步、旧验证怎么失效 */ - const [ruleResult, setRuleResult] = useState(null) - const [ruleReplayed, setRuleReplayed] = useState(false) - /** 撤销要提交的逆操作（按保存前的值构造）和这次改了几处 */ - const [undoRecord, setUndoRecord] = useState<{ operations: DraftOperation[]; count: number } | null>(null) - - /** K11 增量重导入：这份草稿的导入记录、正在看的那一次、逐格的选择和人工对齐 */ - const [importRuns, setImportRuns] = useState(null) - const [importsError, setImportsError] = useState(null) - const [importId, setImportId] = useState(null) - const [importBusy, setImportBusy] = useState(null) - const [importError, setImportError] = useState(null) - /** 逐格的选择：`itemId\0path` → accept / reject（没选的格子不动） */ - const [importPicks, setImportPicks] = useState>({}) - /** 没对齐的工作项要人工指定对哪条指标：item_id → metric_id（空串是取消对齐） */ - const [alignPicks, setAlignPicks] = useState>({}) - /** 接受被拒（基准版本过期 / 这次导入比的还是旧的 L）：选好的格子一个不丢，说清你基于哪一版 */ - const [importStale, setImportStale] = useState<{ code: 'stale_revision' | 'import_stale_l' - current: number | null; base: number | null - /** 服务端点名的那几格（字段的给人看的说法），import_stale_l 时用来指名道姓 */ - names: string } | null>(null) - /** 这次接受 / 拒绝的回执 */ - const [importResult, setImportResult] = useState(null) - const [importAccepted, setImportAccepted] = useState(false) - const [importRejected, setImportRejected] = useState(false) - const [importNote, setImportNote] = useState('') - /** 接受的幂等键：同一批格子的接受反复提交用同一个键，选择变了才换 */ - const importRequest = useRef<{ signature: string; requestId: string }>({ signature: '', requestId: '' }) - /** 重新读取 / 恢复之后：queued、running 时每两秒取一次这一次导入 */ - const [importPollId, setImportPollId] = useState(null) - - /** K12 一句话改口径：输入、只读提案、这次落版的结果 */ - const [nlText, setNlText] = useState('') - const [nlPreview, setNlPreview] = useState(null) - /** 这份提案是哪一段原文产出来的：输入改过就必须重新预览，确认发的也是这一段 */ - const [nlPreviewOf, setNlPreviewOf] = useState('') - const [nlNotice, setNlNotice] = useState<'idle' | 'preview' | 'clarify' | 'applied'>('idle') - const [nlBusy, setNlBusy] = useState<'preview' | 'confirm' | null>(null) - const [nlError, setNlError] = useState(null) - const [nlResult, setNlResult] = useState<{ revision: number | null; invalidated: boolean; replayed: boolean - /** 服务端这次落下的操作和预览里的是不是同一批；null 是审计读不回来（判不了） */ - mismatch: boolean | null } | null>(null) - /** 确认落版用的幂等键：同一句话反复确认是同一件事（服务端认得出，不会多存一版） */ - const nlRequest = useRef<{ signature: string; requestId: string }>({ signature: '', requestId: '' }) - - /** K05 的验证记录：记录 + 总情况 + 字段基线状态，走单独那个接口取 */ - const [validations, setValidations] = useState(null) - const [validationsError, setValidationsError] = useState(null) - /** 刚做完的审阅：按记录 id 显示服务端交回的决定（谁、覆盖不覆盖），不等列表刷新 */ - const [reviewOut, setReviewOut] = useState>({}) - const [reviewNotes, setReviewNotes] = useState>({}) - const [reviewBusy, setReviewBusy] = useState(null) - const [reviewError, setReviewError] = useState(null) - /** 独立期望：每条指标一格，逐条给全才能提交 */ - const [expectValues, setExpectValues] = useState>({}) - const [expectSource, setExpectSource] = useState('human_confirmed') - /** 验证记录先摆最近几条，其余收起来 */ - const [recordsShown, setRecordsShown] = useState(RECORDS_SHOWN) - const [droppedExpanded, setDroppedExpanded] = useState(false) - const canReview = useAtLeast('builder') - const loadList = useCallback(() => { setListError(null) api.caliberLedger.drafts().then( @@ -618,39 +81,30 @@ export function CaliberDrafts() { }, []) useEffect(loadList, [loadList]) - /** K05 的验证记录：记录、总情况、字段基线状态。读不到照实写「无法读取」，不当成「还没有记录」 */ - const loadValidations = useCallback((id: string) => { + const refreshDetail = useCallback(async (id: string) => { const seq = openSeq.current - api.caliberLedger.draftValidations(id).then( - (out) => { if (seq === openSeq.current) { setValidations(out); setValidationsError(null) } }, - (e) => { if (seq === openSeq.current) { setValidations(null); setValidationsError(e) } }, - ) + try { + const fresh = await api.caliberLedger.draft(id) + if (seq === openSeq.current) setDetail((cur) => (cur && cur.id === fresh.id ? fresh : cur)) + } catch { + // 读不到就先保持现在这一份：人下次动手时还会再取 + } }, []) - /** 把一次导入的现状并回列表（新的在前）：重新读取 / 恢复 / 对齐 / 接受 / 拒绝之后都用它 */ - const putImport = useCallback((run: DraftImportRun) => { - setImportRuns((cur) => [run, ...(cur ?? []).filter((row) => row.run_id !== run.run_id)]) - }, []) + const validationsState = useDraftValidations({ openSeq, refreshDetail }) + const { loadValidations, resetValidations } = validationsState - /** - * K11 的导入记录：新的在前，每条都带工作项和逐字段三方。还有正在跑的那一次就跟着轮询它 - * （重新读取是后台任务，页面只管取它的现状） - */ - const loadImports = useCallback((id: string) => { - const seq = openSeq.current - api.caliberLedger.imports(id).then( - (out) => { - if (seq !== openSeq.current) return - setImportRuns(out.runs) - setImportsError(null) - // 默认摆最近那一次的明细：记录列表点哪一次看哪一次 - setImportId((cur) => (cur && out.runs.some((run) => run.run_id === cur) ? cur : (out.runs[0]?.run_id ?? null))) - const running = out.runs.find((run) => run.status === 'queued' || run.status === 'running') - if (running) setImportPollId(running.run_id) - }, - (e) => { if (seq === openSeq.current) { setImportRuns(null); setImportsError(e) } }, - ) - }, []) + const trialState = useDraftTrial({ detail, setDetail, openSeq, loadList, loadValidations }) + const { resetTrial, initEntryFields } = trialState + + const rulesState = useDraftRules({ setDetail, refreshDetail, loadValidations, loadList }) + const { resetRules, syncRulesFromBinding } = rulesState + + const importsState = useDraftImports({ openSeq, refreshDetail, loadValidations, loadList }) + const { resetImports, loadImports } = importsState + + const nlState = useDraftNlChange({ setDetail, syncRulesFromBinding, loadValidations, loadList }) + const { resetNl } = nlState useEffect(() => { api.kb.documents().then((rows) => { setDocs(rows); setDocsError(null) }, @@ -659,7 +113,6 @@ export function CaliberDrafts() { (all) => { const enabled = all.filter((s) => s.enabled !== false) setSources(enabled) - // 数据源必选（取数 SQL 要对照这个库的表和列写），默认选中第一个，别让人对着 422 才知道要填 setDatasourceId((cur) => cur || (enabled[0]?.id ?? '')) }, () => setSources([]), @@ -690,59 +143,19 @@ export function CaliberDrafts() { setRecentRuns(null) setPollingId(null) setPollTimedOut(false) - setTrial(null) - setTrialPollId(null) - setTrialTimedOut(false) - setTrialError(null) setJobError(null) - setRuleValues({}) - setRuleBusy(null) - setRuleError(null) - setRuleStale(null) - setRuleResult(null) - setRuleReplayed(false) - setUndoRecord(null) - rulePending.current = { signature: '', requestId: '' } - setValidations(null) - setValidationsError(null) - setReviewOut({}) - setReviewNotes({}) - setReviewError(null) - setExpectValues({}) - setExpectSource('human_confirmed') - setRecordsShown(RECORDS_SHOWN) setDroppedExpanded(false) - setImportRuns(null) - setImportsError(null) - setImportId(null) - setImportBusy(null) - setImportError(null) - setImportPicks({}) - setAlignPicks({}) - setImportStale(null) - setImportResult(null) - setImportAccepted(false) - setImportRejected(false) - setImportNote('') - setImportPollId(null) - importRequest.current = { signature: '', requestId: '' } - setNlText('') - setNlPreview(null) - setNlPreviewOf('') - setNlNotice('idle') - setNlBusy(null) - setNlError(null) - setNlResult(null) - nlRequest.current = { signature: '', requestId: '' } + resetTrial() + resetRules() + resetValidations() + resetImports() + resetNl() try { const draft = await api.caliberLedger.draft(id) if (seq !== openSeq.current) return setDetail(draft) - const marks: Record = {} - setEntryValues(entryInitialValues(draft.entry_fields ?? [], marks)) - setPrefilled(marks) - // 业务规则的初值按这次读到的能力描述定：写法一致的项直接显示，不一致的留空等人选 - setRuleValues(ruleInitialValues(draft.binding)) + initEntryFields(draft) + syncRulesFromBinding(draft.binding) if (draft.status === 'drafting') { setPollTimedOut(false); setPollingId(id) } } catch (e) { if (seq !== openSeq.current) return @@ -758,20 +171,6 @@ export function CaliberDrafts() { ) } - /** - * 只刷新详情，不重置页面（不清勾选、不动表单、不换打开的那份）。写入被拒之后用它：拿回服务端现在这一版 - * 当新的基准版本，人确认后再点一次就行（K02）。晚到的响应不覆盖已经切到别的草稿的那一份 - */ - const refreshDetail = useCallback(async (id: string) => { - const seq = openSeq.current - try { - const fresh = await api.caliberLedger.draft(id) - if (seq === openSeq.current) setDetail((cur) => (cur && cur.id === fresh.id ? fresh : cur)) - } catch { - // 读不到就先保持现在这一份：人下次动手时还会再取 - } - }, []) - useEffect(() => { if (!pollingId) return let until = Date.now() + DRAFT_POLL_LIMIT_MS @@ -813,101 +212,6 @@ export function CaliberDrafts() { return () => { alive = false; clearTimeout(timer) } }, [pollingId, loadList]) - /** 草稿有了试运行就取一次结果；还在 running 就交给下面的轮询 */ - useEffect(() => { - const id = detail?.id - const run = detail?.trial?.run_id - if (!id || !run) return - let alive = true - setTrial((cur) => (cur?.run_id === run && detail?.trial?.status !== 'stale' ? cur : { - run_id: run, status: detail?.trial?.status ?? 'running', metrics: [], error: detail?.trial?.error ?? '', - })) - const request = ++trialRequestSeq.current - api.caliberLedger.draftTrial(id).then((out) => { - if (!alive || request !== trialRequestSeq.current) return - setTrial(out) - setTrialError(null) - setTrialTimedOut(false) - if (out.status === 'running') setTrialPollId(id) - }, (e) => { - if (!alive || request !== trialRequestSeq.current) return - setTrialError(e) - setTrialPollId(null) - }) - return () => { alive = false } - }, [detail?.id, detail?.version, detail?.trial?.run_id, detail?.trial?.status]) - - /** 试运行还在跑：每两秒取一次，超过一分钟就停；收尾时顺带刷新详情（状态可能已变成待签字发布） */ - useEffect(() => { - if (!trialPollId) return - const until = Date.now() + 60_000 - let alive = true - let settled = false - const seq = openSeq.current - let timer: ReturnType - const poll = () => { - const request = ++trialRequestSeq.current - api.caliberLedger.draftTrial(trialPollId).then((out) => { - if (!alive || seq !== openSeq.current || request !== trialRequestSeq.current) return - setTrial(out) - setTrialError(null) - if (out.status === 'running') { - if (Date.now() > until) { - settled = true - setTrialPollId(null) - setTrialTimedOut(true) - } - return - } - settled = true - setTrialPollId(null) - setTrialTimedOut(false) - loadList() - // 试运行到终态时服务端会把这次跑完的验证记录补记一条：跟着取一次，结论按记录说 - loadValidations(trialPollId) - api.caliberLedger.draft(trialPollId).then((fresh) => { - if (seq !== openSeq.current) return - setDetail((cur) => (cur && cur.id === fresh.id && (cur.revision ?? 0) <= (fresh.revision ?? 0) ? fresh : cur)) - }, () => undefined) - }, (e) => { - if (!alive || seq !== openSeq.current || request !== trialRequestSeq.current) return - settled = true - setTrialPollId(null) - setTrialError(e) - }).finally(() => { - if (alive && !settled && seq === openSeq.current) timer = setTimeout(poll, 2000) - }) - } - timer = setTimeout(poll, 2000) - return () => { alive = false; clearTimeout(timer) } - }, [trialPollId, loadList, loadValidations]) - - /** - * 正在跑的那一次重导入：每两秒取一次现状（后台在抽候选、算三方），到终态就停。被中断的不会自动续跑， - * 页面上由人点「恢复」显式排回后台。 - */ - useEffect(() => { - if (!importPollId) return - const until = Date.now() + 5 * 60_000 - let alive = true - const timer = setInterval(() => { - api.caliberLedger.importRun(importPollId).then((run) => { - if (!alive) return - putImport(run) - if (run.status === 'queued' || run.status === 'running') { - if (Date.now() > until) setImportPollId(null) - return - } - setImportPollId(null) - }, (e) => { - if (!alive) return - setImportPollId(null) - setImportError(e) - }) - }, 2000) - return () => { alive = false; clearInterval(timer) } - }, [importPollId, putImport]) - useEffect(() => { for (const doc of detail?.documents ?? []) ensureChunks(doc.document_id) }, [detail, ensureChunks]) @@ -923,7 +227,7 @@ export function CaliberDrafts() { return map }, [detail]) - const paragraphOf = (citation: DraftCitation): number | null => { + const paragraphOf = useCallback((citation: DraftCitation): number | null => { const ordinal = chunks[citation.chunk]?.ordinal if (typeof ordinal === 'number') return ordinal if (typeof citation.ordinal === 'number') return citation.ordinal @@ -931,7 +235,7 @@ export function CaliberDrafts() { const doc = detail?.documents?.find((d) => d.document_id === documentId) const at = doc?.chunk_ids.indexOf(citation.chunk) ?? -1 return at >= 0 ? at + 1 : null - } + }, [chunkDoc, chunks, detail?.documents]) const submit = async (definitions = true) => { if (!pickedDocs.length) { toast.warn(D.needDoc); return } @@ -960,17 +264,12 @@ export function CaliberDrafts() { const resolve = async (draft: CaliberDraft) => { if (!pickedQuestions.length) { toast.warn(D.resolveNeed); return } - // 已接管的草稿：改哪张卡要明确（详情接口给的 card_id）；图上有多张可写的卡又认不出唯一目标时不给改， - // 免得按「第一张」猜（K02） if (draft.revision != null && draft.workflow_id && !draft.card_id) { toast.warn(draft.cards && draft.cards.length > 1 ? D.cardAmbiguous : D.cardMissing) return } setBusy(`resolve:${draft.id}`) - // 消问题是按集合做的：去空白、去重、排序之后再发，勾选顺序变一下不该被当成另一次请求（K02 的幂等指纹 - // 服务端也按同一份归一算） const questions = [...new Set(pickedQuestions.map((q) => q.trim()).filter(Boolean))].sort() - // 同一批勾选反复提交（网络失败后重试）用同一个幂等键：服务端认得出是同一件事，不会多存一版 const signature = `${draft.id}|${questions.join('\u0000')}` if (pendingWrite.current.signature !== signature) { pendingWrite.current = { signature, requestId: newRequestId() } @@ -985,16 +284,13 @@ export function CaliberDrafts() { setDetail(fresh) toast.ok(D.resolved(questions.length)) setPickedQuestions([]) - // 这一批已经落库：换一个键，别让下一次（重新勾选）落到同一个幂等键上 pendingWrite.current = { signature: '', requestId: '' } loadList() } catch (e) { const stale = e instanceof ApiError && e.status === 409 && e.code === 'stale_revision' if (stale) { - // 有人在别处先改过：这次没写进去，勾选一个不丢、也不拿新版本自动重发；说清现况，人确认后再点一次 const told = Number((e.body as { current_version?: unknown } | undefined)?.current_version) toast.warn(D.writeStale(Number.isFinite(told) ? told : null, draft.revision ?? null)) - // 只把详情刷到最新（拿新的基准版本），不清空勾选、不重置页面 void refreshDetail(draft.id) } else { toast.error(e) @@ -1043,455 +339,6 @@ export function CaliberDrafts() { setPickedQuestions((cur) => (checked ? [...cur, question] : cur.filter((q) => q !== question))) } - const questionList = (questions: string[] | undefined, prefix: string) => { - const list = (questions ?? []).filter((q) => q.trim()) - if (!list.length) return null - return ( -
      - {list.map((q, i) => ( -
    • - -
      • - ))} -
    - ) - } - - const citationRow = (citation: DraftCitation, key: string, metricName: string) => { - const chunk = chunks[citation.chunk] - const ordinal = paragraphOf(citation) - return ( -
  • -
    - - 「{citation.quote}」 - {ordinal != null && · {D.paragraph(ordinal)}} - -

    - {chunk - ? - : D.citationMissing} -

    -

    {metricName}

    -
    -
    • - ) - } - - /** - * 按填好的入口字段再试运行一次：自动试跑缺必填输入时停在 needs_input，人在这里补上。 - * 带 expect 时这一条会落成一条带独立期望的验证记录（对得上就是对得上，对不上照实记）；不带就是只记试跑结果。 - * 入口字段的写法变化只影响这一次请求的参数，不会创建口径定义版本。 - */ - const runTrial = async (draft: CaliberDraft, expect?: { metrics: Record; source: DraftExpectationSource }) => { - const fields = draft.entry_fields ?? [] - if (!entryReady(fields, entryValues)) return - setTrialStarting(true) - setTrialError(null) - try { - const out = await api.caliberLedger.runDraftTrial(draft.id, entryInputs(fields, entryValues), expect) - setTrial(out) - setTrialTimedOut(false) - if (out.status === 'running' && out.run_id) setTrialPollId(draft.id) - const fresh = await api.caliberLedger.draft(draft.id) - setDetail((cur) => cur?.id === fresh.id ? fresh : cur) - loadValidations(draft.id) - } catch (e) { - setTrialError(e) - } finally { - setTrialStarting(false) - } - } - - /** - * 保存这次业务规则的修改：把**所有**受影响指标的字段操作放在一次请求里（PATCH 的 operations），带上读到的 - * 基准版本、目标卡和幂等键。一次原子变更：全成或全不成；回执（change）说清落成哪一版、绑定同步到哪一步。 - * 被拒（基准版本过期）时什么都不改：这里的写法选择留着，刷新详情拿到新版本后由人显式再点一次。 - */ - const saveRules = async (draft: CaliberDraft, operations: DraftOperation[]) => { - if (!operations.length) { toast.warn(R.saveNeedChange); return } - if (!draft.card_id || draft.revision == null) { toast.warn(R.noCard); return } - setRuleBusy('save') - setRuleError(null) - // 同一批操作反复提交（网络断了、手点重试）用同一个幂等键：服务端认得出是同一件事，不会多存一版 - const signature = `${draft.id}|${JSON.stringify(operations)}` - if (rulePending.current.signature !== signature) { - rulePending.current = { signature, requestId: newRequestId() } - } - try { - const fresh = await api.caliberLedger.patchDraft(draft.id, { - operations, - expected_revision: draft.revision, - request_id: rulePending.current.requestId, - target: { workflow_id: draft.workflow_id ?? undefined, card_id: draft.card_id }, - }) - setDetail(fresh) - setRuleResult(fresh.change ?? null) - setRuleReplayed(!!fresh.replayed) - setRuleStale(null) - // 撤销按**保存前**那份能力描述里的值构造逆操作，所以用这一版的 binding，不用刚回来的 - setUndoRecord({ operations: undoOperations(draft.binding, operations), count: operations.length }) - setRuleValues(ruleInitialValues(fresh.binding)) - rulePending.current = { signature: '', requestId: '' } - toast.ok(R.saved(fresh.change?.revision ?? fresh.revision ?? null)) - loadValidations(draft.id) - loadList() - } catch (e) { - const stale = e instanceof ApiError && e.status === 409 && e.code === 'stale_revision' - if (stale) { - // 有人在别处先改过：这次没写进去，选择一个不丢、也不拿新版本自动重发；刷新详情只是为了拿到新的基准版本 - const told = Number((e.body as { current_version?: unknown } | undefined)?.current_version) - setRuleStale({ current: Number.isFinite(told) ? told : null, base: draft.revision ?? null }) - void refreshDetail(draft.id) - } else { - setRuleError(e) - } - } finally { - setRuleBusy(null) - } - } - - /** 撤销这次修改：逆操作 + 当前版本 + 新的请求键，再新增一版；不是整份覆盖，也不改已经存下的版本 */ - const undoRules = async (draft: CaliberDraft, record: { operations: DraftOperation[]; count: number }) => { - if (draft.revision == null || !record.operations.length) return - setRuleBusy('undo') - setRuleError(null) - // 撤销也按内容认一次：网络断了再点一次是同一件事，服务端认得出，不会多存一版 - const signature = `undo|${draft.id}|${JSON.stringify(record.operations)}` - if (rulePending.current.signature !== signature) { - rulePending.current = { signature, requestId: newRequestId() } - } - try { - const fresh = await api.caliberLedger.patchDraft(draft.id, { - operations: record.operations, - expected_revision: draft.revision, - request_id: rulePending.current.requestId, - target: { workflow_id: draft.workflow_id ?? undefined, card_id: draft.card_id ?? undefined }, - }) - setDetail(fresh) - setRuleResult(fresh.change ?? null) - setRuleReplayed(!!fresh.replayed) - setRuleStale(null) - setUndoRecord(null) - setRuleValues(ruleInitialValues(fresh.binding)) - rulePending.current = { signature: '', requestId: '' } - toast.ok(R.undone(fresh.change?.revision ?? fresh.revision ?? null)) - loadValidations(draft.id) - loadList() - } catch (e) { - const stale = e instanceof ApiError && e.status === 409 && e.code === 'stale_revision' - if (stale) { - const told = Number((e.body as { current_version?: unknown } | undefined)?.current_version) - setRuleStale({ current: Number.isFinite(told) ? told : null, base: draft.revision ?? null }) - void refreshDetail(draft.id) - } else { - setRuleError(e) - } - } finally { - setRuleBusy(null) - } - } - - /** - * 重新读一遍文档（K11）：在服务端重抽候选 R，和现在的内容 L、上次被接受的导入值 B 逐字段比。 - * **不改 L、不改 B**——真正落到图上的只有下面的「接受选定字段」。带幂等键：同一份草稿 + 同一个键只建一次。 - */ - const createImport = async (draft: CaliberDraft) => { - if (!draft.workflow_id) { toast.warn(I.readOnly); return } - setImportBusy('create') - setImportError(null) - setImportStale(null) - setImportResult(null) - setImportAccepted(false) - setImportRejected(false) - const signature = `import|${draft.id}|${draft.revision ?? ''}` - if (importRequest.current.signature !== signature) { - importRequest.current = { signature, requestId: newRequestId() } - } - try { - const run = await api.caliberLedger.createImport(draft.id, importRequest.current.requestId) - putImport(run) - setImportId(run.run_id) - setImportPicks({}) - setAlignPicks({}) - importRequest.current = { signature: '', requestId: '' } - if (run.status === 'queued' || run.status === 'running') setImportPollId(run.run_id) - toast.ok(I.started) - } catch (e) { - setImportError(e) - } finally { - setImportBusy(null) - } - } - - /** - * 人工把一条工作项对到图上某条指标（空串是取消对齐）。baseline_unknown 的每一条、还有同名不同范围（引的原文 - * 不是同一段）都要人在这里定：机器不按名字猜「这说的是同一条指标」。对齐只重算三方，不改 L、不改 B。 - */ - const alignImportItem = async (run: DraftImportRun, item: DraftImportItem, metricId: string) => { - setImportBusy(`align:${item.item_id}`) - setImportError(null) - try { - const fresh = await api.caliberLedger.alignImportItem(run.run_id, item.item_id, metricId) - putImport(fresh) - // 换了对哪一条，这条上旧的选择就不适用了：只清这一条的，别的地方不动 - setImportPicks((cur) => { - const next: Record = {} - for (const [key, value] of Object.entries(cur)) if (!key.startsWith(`${item.item_id}\u0000`)) next[key] = value - return next - }) - } catch (e) { - setImportError(e) - } finally { - setImportBusy(null) - } - } - - /** - * 重新读一次这一次导入的现状：逐字段的三方（B / L / R）是服务端按**现在**的图现算的，重新读才知道自己看的 - * 还是不是最新的。顺手把不再适用的选择裁掉——不拿旧的选择去提交。 - */ - const reloadImportRun = useCallback(async (runId: string): Promise => { - try { - const fresh = await api.caliberLedger.importRun(runId) - putImport(fresh) - setImportPicks((cur) => prunePicks(cur, fresh)) - return fresh - } catch (e) { - setImportError(e) - return null - } - }, [putImport]) - - /** - * 服务端说这次导入比对的还是旧的 L（409 import_stale_l）之后的恢复：按**现在的** L / B 重算这次导入还没决定的 - * 三方（人处置过的决定不动），把现状交回来。不能只把基准版本更新到最新一版就覆盖上去。 - */ - const refreshImportRun = useCallback(async (runId: string): Promise => { - try { - const fresh = await api.caliberLedger.refreshImport(runId) - putImport(fresh) - setImportPicks((cur) => prunePicks(cur, fresh)) - return fresh - } catch (e) { - setImportError(e) - return null - } - }, [putImport]) - - /** - * 接受选定的格子：一次提交（带读到的基准版本和幂等键）→ 图新增一版 + 按被接受的格子推进 B。 - * 每条按**工作项身份**（item_id）指认，带上要落的那条指标和**明确列出来的**字段路径；不是这里列出来的格子， - * 一个都不动。被拒（基准版本过期、或者这次比对的是旧的 L）时选好的格子原样留着，先把这一次导入的现状重新读 - * 回来再让人决定——不拿旧 L 直接覆盖。 - */ - const acceptImportFields = async (draft: CaliberDraft, run: DraftImportRun, - items: { item_id: string; metric_id?: string; fields: string[] }[]) => { - const picked = items.filter((item) => item.fields.length) - if (!picked.length) { toast.warn(I.nothingSelected); return } - if (draft.revision == null) { toast.warn(I.needRevision); return } - setImportBusy('accept') - setImportError(null) - setImportStale(null) - setImportResult(null) - setImportRejected(false) - const signature = `accept|${run.run_id}|${JSON.stringify(picked)}` - if (importRequest.current.signature !== signature) { - importRequest.current = { signature, requestId: newRequestId() } - } - try { - const out = await api.caliberLedger.acceptImport(run.run_id, { - items: picked, expected_revision: draft.revision, request_id: importRequest.current.requestId, - }) - putImport(out) - setImportResult(out) - setImportAccepted(true) - setImportPicks({}) - importRequest.current = { signature: '', requestId: '' } - toast.ok(I.accepted(out.revision)) - // 图和 B 是同一次提交里推进的：详情、验证状态和列表都跟着刷 - void refreshDetail(draft.id) - loadValidations(draft.id) - loadList() - } catch (e) { - const code = e instanceof ApiError && e.status === 409 ? e.code : null - if (code === 'stale_revision' || code === 'import_stale_l') { - const body = e instanceof ApiError ? (e.body as { current_version?: unknown - fields?: { path?: string }[] } | undefined) : undefined - const told = Number(body?.current_version) - // 服务端点名的就是这几格：把字段说法列出来，人一眼看出是哪一格对不上了 - const names = (body?.fields ?? []).map((row) => fieldLabel(String(row.path ?? ''), draft.binding)).join('、') - setImportStale({ - code: code === 'import_stale_l' ? 'import_stale_l' : 'stale_revision', - current: Number.isFinite(told) ? told : null, base: draft.revision ?? null, names, - }) - // 写不进去：先把这一次导入的现状重新读回来，再读一次详情拿新的基准版本。 - // import_stale_l 是「三方算的是旧的 L」：按现在的 L / B **重算三方**（refresh），不是只更新版本号就覆盖。 - if (code === 'import_stale_l') await refreshImportRun(run.run_id) - else await reloadImportRun(run.run_id) - void refreshDetail(draft.id) - } else { - setImportError(e) - } - } finally { - setImportBusy(null) - } - } - - /** 拒绝选定的格子：保持 L 不动，也不推进 B。同样按工作项身份指认，只拒这里明确列出来的格子 */ - const rejectImportFields = async (run: DraftImportRun, - items: { item_id: string; metric_id?: string; seq?: number; fields: string[] }[]) => { - const picked = items.filter((item) => item.fields.length) - if (!picked.length) { toast.warn(I.nothingSelected); return } - setImportBusy('reject') - setImportError(null) - setImportResult(null) - setImportAccepted(false) - try { - const fresh = await api.caliberLedger.rejectImport(run.run_id, { - items: picked, ...(importNote.trim() ? { note: importNote.trim() } : {}), - }) - putImport(fresh) - setImportRejected(true) - setImportPicks({}) - } catch (e) { - setImportError(e) - } finally { - setImportBusy(null) - } - } - - /** 显式恢复被中断的那一次导入：续做还没完成的工作项，已接受 / 拒绝的不动、不重复建版 */ - const resumeImportRun = async (run: DraftImportRun) => { - setImportBusy('resume') - setImportError(null) - try { - const fresh = await api.caliberLedger.resumeImport(run.run_id) - putImport(fresh) - setImportPollId(fresh.run_id) - toast.ok(I.resumed) - } catch (e) { - setImportError(e) - } finally { - setImportBusy(null) - } - } - - /** - * 用一句话看影响（K12）：只调**只读**的提案接口，不落库、不新增版本。认出来的条件落成 K03 同一形状的 operations， - * 这里只把它翻成「哪条指标、哪一项写法、现在什么 → 保存后什么」；说不清时只有澄清项，一条操作都不产出。 - * - * 提案和**产出它的那一段原文**一起记下来（nlPreviewOf）：确认时按那一段原文发、服务端再解析一次得到的还是同一批 - * 操作；页面显示的（A）和实际落下的（B）不会是两句不同的话。 - */ - const previewNl = async (draft: CaliberDraft) => { - const instruction = nlText.trim() - if (!instruction) { toast.warn(N.needText); return } - if (!draft.card_id) { toast.warn(draft.cards && draft.cards.length > 1 ? D.cardAmbiguous : D.cardMissing); return } - setNlBusy('preview') - setNlError(null) - setNlResult(null) - setNlPreview(null) - try { - const out = await api.caliberLedger.changeProposal(draft.id, { - instruction, - target: { workflow_id: draft.workflow_id ?? undefined, card_id: draft.card_id }, - }) - setNlPreview(out.proposal) - setNlPreviewOf(instruction) - setNlNotice(out.proposal.ready ? 'preview' : 'clarify') - // 同一句话反复确认是同一件事：请求键按内容定，重发不会多存一版 - const signature = `nl|${draft.id}|${instruction}` - if (nlRequest.current.signature !== signature) { - nlRequest.current = { signature, requestId: newRequestId() } - } - } catch (e) { - setNlNotice('idle') - setNlError(e) - } finally { - setNlBusy(null) - } - } - - /** - * 确认之后才落版（K12）：confirm=true 走服务端的薄适配，内部仍是现有变更服务（版本 CAS、幂等键、绑定同步、 - * 校验一条不少）。 - * - * 发出去的是**预览时那一段原文**，不是输入框里现在的内容：输入改过就必须重新预览（nlTouched），否则页面显示的 - * 是 A、落下的会是 B。落版之后另比一次服务端落下的操作和预览里的是不是同一批，不一样就照实提示。 - * 失败时页面上什么都不改——输入和预览都留着，由人决定下一步。 - */ - const confirmNl = async (draft: CaliberDraft) => { - if (!nlPreviewOf || !nlPreview?.ready) { toast.warn(N.needConfirm); return } - if (nlText.trim() !== nlPreviewOf) { toast.warn(N.needPreviewAgain); return } - if (draft.revision == null) { toast.warn(N.noRevision); return } - setNlBusy('confirm') - setNlError(null) - const signature = `nl|${draft.id}|${nlPreviewOf}` - if (nlRequest.current.signature !== signature) { - nlRequest.current = { signature, requestId: newRequestId() } - } - try { - const out = await api.caliberLedger.nlChange(draft.id, { - instruction: nlPreviewOf, - confirm: true, - expected_revision: draft.revision, - request_id: nlRequest.current.requestId, - target: { workflow_id: draft.workflow_id ?? undefined, card_id: draft.card_id ?? undefined }, - }) - const fresh = 'proposal' in out ? null : out - const revision = fresh?.change?.revision ?? fresh?.revision ?? null - // 服务端这次落下的操作和预览里的是不是同一批（同一段原文应当解析出同一批）；判不了就是 null - const same = sameOperations(nlPreview.operations, fresh?.change?.operations) - setNlResult({ - revision, - invalidated: fresh?.change?.validation?.invalidated ?? false, - replayed: !!fresh?.replayed, - mismatch: same === null ? null : !same, - }) - setNlNotice('applied') - nlRequest.current = { signature: '', requestId: '' } - toast.ok(N.applied(revision)) - if (fresh) { - setDetail((cur) => (cur && cur.id === fresh.id ? fresh : cur)) - setRuleValues(ruleInitialValues(fresh.binding)) - } - loadValidations(draft.id) - loadList() - } catch (e) { - setNlError(e) - } finally { - setNlBusy(null) - } - } - - /** - * 对一条验证记录做审阅：只调服务端那个接口（审批人由服务端身份定，界面代写不了），拿回来的决定照原样显示 - * ——谁、什么时候、覆盖不覆盖当前版本。样例对得上不会自动变成批准，这里也不在前端改结论。 - */ - const reviewRecord = async (draft: CaliberDraft, recordId: string, decision: 'confirmed' | 'rejected') => { - setReviewBusy(recordId) - setReviewError(null) - try { - const note = (reviewNotes[recordId] ?? '').trim() - const out = await api.caliberLedger.reviewValidation(draft.id, recordId, - { decision, ...(note ? { note } : {}) }) - setReviewOut((cur) => ({ ...cur, [recordId]: out.record.review ?? null })) - setReviewNotes((cur) => ({ ...cur, [recordId]: '' })) - toast.ok(V.reviewDone(decision === 'confirmed' ? V.reviewConfirmed : V.reviewRejected)) - loadValidations(draft.id) - void refreshDetail(draft.id) - } catch (e) { - setReviewError(e) - } finally { - setReviewBusy(null) - } - } - const changeDraftJob = async (draft: CaliberDraft, action: 'retry' | 'stop') => { const seq = openSeq.current setJobBusy(draft.id) @@ -1513,1191 +360,6 @@ export function CaliberDrafts() { } } - /** - * 业务规则（K06 的工作台）：服务端的能力描述说得出这张卡能按字段改哪几项时才开表单。改完先列「会改到哪几条指标、 - * 现在什么写法 → 保存后什么写法」，再把所有受影响指标的字段操作**一次**提交（原子变更）；回执说清落成哪一版、 - * 取数同步到哪一步、旧验证怎么失效，并按保存前的值给出「撤销这次修改」。这块只写业务说法：取数语句、内部编号、 - * 字段路径和 JSON 都不上来。 - */ - const renderRules = (draft: CaliberDraft) => { - const binding = draft.binding - if (!binding) return null - const fields = binding.editable_fields ?? [] - const metrics = (draft.metrics ?? []).filter((metric) => metric.id) - const names = new Map(metrics.map((metric) => [metric.id as string, metric.name])) - const declared = new Set(Object.keys(binding.metric_values ?? {})) - const plan = rulePlan(binding, ruleValues) - const changes = plan.changes.map((change) => ({ ...change, name: names.get(change.metricId) ?? change.metricId })) - const readOnly = !draft.workflow_id ? R.noWorkflow - : !draft.card_id ? R.noCard - : draft.revision == null ? R.noRevision - : declared.size === 0 ? R.unbound : null - const label = R.status[binding.status] ?? binding.status - const tone = binding.status === 'aligned' ? 'var(--ok)' - : binding.status === 'unbound' ? 'var(--text-faint)' : 'var(--warn)' - const saved = ruleResult?.binding - const untouched = metrics.length - declared.size - return ( -
    -
    -

    {R.title}

    - {label} - {R.hint} -
    - {binding.status === 'aligned' ? ( -

    {R.alignedNote}

    - ) : ( - <> -

    - {R.reason}：{binding.reason} -

    - {binding.fix && binding.fix !== 'none' && ( -

    - {R.fixLabel}：{R.fix[binding.fix] ?? binding.fix} -

    - )} - - )} - - {readOnly ? ( -

    {R.readOnly}：{readOnly}

    - ) : ( - <> -
      - {fields.map((field) => { - const declaring = declaringMetrics(binding, field.field) - const distinct = [...new Set(declaring.map((id) => binding.metric_values[id][field.field]))] - const chosen = ruleValues[field.field] ?? '' - return ( -
    • - {field.label} - - - {!declaring.length ? R.fieldNone - : distinct.length > 1 ? R.mixed - : `${R.before}：${choiceLabel(field, distinct[0])}`} - - {field.help} -
      • - ) - })} -
    - {untouched > 0 && ( -

    - {R.unaffected(untouched)}。{R.unaffectedHint} -

    - )} - -
    -
    - {R.previewTitle} - - {changes.length ? R.affected(changes.length) : R.noChange} - -
    - {changes.length > 0 && ( -
      - {changes.map((change) => ( -
    • - {change.name} - · {change.field.label}： - {choiceLabel(change.field, change.from)} - → - {choiceLabel(change.field, change.to)} -
      • - ))} -
    - )} -
    - -
    - - {undoRecord && ( - - )} -
    - - )} - - {ruleStale && ( -
    - {R.stale(ruleStale.current, ruleStale.base)} -
    - )} - {ruleError != null &&
    } - {ruleReplayed && !ruleResult && ( -
    {R.replayed}
    - )} - - {ruleResult && ( -
    -

    - {R.saved(ruleResult.revision)} - {ruleReplayed && · {R.replayed}} -

    - {ruleResult.applied ? ( - <> -

    - {saved?.synced ? R.savedAligned : R.savedNotSynced} - {!saved?.synced && saved?.reason ? `（${saved.reason}）` : ''} -

    -

    {R.invalidated}

    -

    {R.nextStep}

    - {!saved?.synced && saved?.fix && saved.fix !== 'none' && ( -

    - {R.fixLabel}：{R.fix[saved.fix] ?? saved.fix}。{R.fixNote} -

    - )} - - ) : ( -

    {R.savedNoChange}

    - )} -
    - )} -
    - ) - } - - /** - * 文档更新（K11 的增量重导入）：重抽候选 R，和现在的内容 L、上次被接受的导入值 B 逐字段比。R 只是候选—— - * 落到图上只有「接受选定字段」这一次显式动作（一次提交里新增一版图 + 按被接受的格子推进 B）。逐格写清三方 - * 的值、状态和为什么；漏提取不当删除（不能接受，L 上的原值原样留着）；没对齐上的工作项由人指定对到哪条指标。 - * 提交一律按**工作项身份**（item_id）指认、把要落的那几格**明确列出来**（不发 auto）；抽坏了的（failed）工作项 - * 一个接受都点不了，批量也不收它；已接受 / 拒绝过的格子不再被选。被中断的导入由人点「恢复」显式续跑。 - * 写不进去时（基准版本过期 stale_revision / 三方比的是旧的 L、import_stale_l）选好的格子一个不丢：前者重新读 - * 详情拿新版本，后者按现在的口径**重算三方**（refreshImport）再读回来，由人核对后再点一次——不拿旧 L 直接覆盖。 - * 这块只写业务说法，不出现取数语句、节点编号和字段路径。 - */ - const renderImports = (draft: CaliberDraft) => { - if (!draft.workflow_id) return null - const runs = importRuns ?? [] - const active = runs.find((run) => run.run_id === importId) ?? null - const metrics = (draft.metrics ?? []).filter((metric) => metric.id) - const nameOf = (id: string) => metrics.find((metric) => metric.id === id)?.name ?? I.unknownMetric - const baseline = validations?.field_baseline ?? draft.field_baseline - const coverage = baseline?.coverage - // 老工件没有 coverage 这一段：那时候 put_metrics 收的是整张卡，按 complete 处理（和 fieldbaseline.coverage_of 同一条规矩）； - // 没有基线（baseline_unknown）时一律是读不出来，不能拿现在的内容当作它 - const coverageStatus = coverage?.status ?? (baseline?.status === 'known' ? 'complete' : 'unknown') - const missingFields = Object.values(coverage?.metrics ?? {}) - .reduce((total, row) => total + (row.missing?.length ?? 0), 0) - const uncoveredMetrics = coverage?.uncovered_metric_ids?.length ?? 0 - const coverageText = coverageStatus === 'complete' ? I.coverageComplete - : coverageStatus === 'partial' ? I.coveragePartial(missingFields, uncoveredMetrics) - : I.coverageUnknown - // 这一次要提交的格子：按**现在**的现状现算——格子还在、还没决定过、能接受、工作项没抽坏才收； - // 每条都带上工作项身份（item_id），同一个 metric_id 上有重复项时也不会指错 - const picking = active ? active.items.map((item) => ({ - item, - paths: item.fields.filter((cell) => importPicks[pickKey(item.item_id, cell.path)] === 'accept' - && cell.decided == null && cell.acceptable && itemAccepts(item)).map((cell) => cell.path), - rejects: item.fields.filter((cell) => importPicks[pickKey(item.item_id, cell.path)] === 'reject' - && cell.decided == null).map((cell) => cell.path), - })) : [] - const acceptItems = picking.filter((row) => row.paths.length) - .map((row) => ({ item_id: row.item.item_id, metric_id: row.item.metric_id, fields: row.paths })) - const rejectItems = picking.filter((row) => row.rejects.length).map((row) => ({ - item_id: row.item.item_id, fields: row.rejects, - ...(row.item.metric_id ? { metric_id: row.item.metric_id } : { seq: row.item.seq }), - })) - const acceptCount = acceptItems.reduce((total, item) => total + item.fields.length, 0) - const rejectCount = rejectItems.reduce((total, item) => total + item.fields.length, 0) - const resumable = active != null && (active.status === 'interrupted' || active.status === 'failed') - const busy = importBusy != null - return ( -
    -
    -

    {I.title}

    - {I.hint} -
    -

    - {I.coverageLabel}：{coverageText}。{I.coverageHint} -

    - -
    - - {I.createHint} -
    - {importsError != null &&
    } - - {runs.length === 0 ? ( -

    - {I.runs}：{I.runsEmpty} -

    - ) : ( -
    -

    {I.runs}

    -
      - {runs.map((run) => ( -
    • - -
      • - ))} -
    -
    - )} - - {active && ( -
    -
    - - {I.status[active.status] ?? active.status} - - {(active.status === 'queued' || active.status === 'running') && } - {I.itemsTotal(active.items_total)} - {active.conflicts.length > 0 && ( - - {I.conflicts(active.conflicts.length)} - - )} - {active.created_at && {I.createdAt(formatDateTime(active.created_at))}} -
    - {active.error && {active.error}} - {resumable && ( -
    - - {I.resumeHint} - - {I.unfinished(unfinishedItems(active))} - -
    - )} - - {active.items.length === 0 ? ( -

    {I.noItems}

    - ) : ( -
      - {active.items.map((item) => { - const aligned = !!item.metric_id - const alignMode = String(item.align?.mode ?? '') - // 没对上要人指定；同名不同范围（身份对得上、引的原文不是同一段）也要人来定，所以 manual 一律开这个口 - const needsAlign = !aligned || alignMode === 'manual' - const clean = cleanPaths(item) - return ( -
    • -
      - {item.name || I.unknownMetric} - {aligned && · {nameOf(item.metric_id)}} - - {I.itemStatus[item.status] ?? I.state[item.status] ?? item.status} - -
      - - {needsAlign && ( -
      - {I.itemUnaligned} - - - {I.alignHint} -
      - )} - - {(alignMode === 'manual' || alignMode === 'new') && ( -

      - {I.reasonLabel}：{I.reason[alignMode] ?? alignMode} - {item.align?.reason ? ` · ${item.align.reason}` : ''} -

      - )} - {item.error &&

      {item.error}

      } - -
        - {item.fields.map((cell) => { - const choice = importPicks[pickKey(item.item_id, cell.path)] ?? '' - // 抽坏了的（failed）和没对齐上的一律不能接受；接受过的也不用再选 - const canAccept = itemAccepts(item) && cell.acceptable && cell.decided !== 'accepted' - const values: [string, string | null, string][] = [ - ['b', cellText(cell, 'b', draft.binding), I.b], - ['l', cellText(cell, 'l', draft.binding), I.l], - ['r', cellText(cell, 'r', draft.binding), I.r], - ] - return ( -
      • -
        - {fieldLabel(cell.path, draft.binding)} - - {I.state[cell.state] ?? cell.state} - - {cell.decided === 'accepted' && {I.accept}} - {cell.decided === 'rejected' && {I.reject}} - {cell.state === 'missing' && ( - - {I.missingNoDelete} - - )} -
        -
          - {values.map(([key, text, label]) => text != null && ( -
        • - {label}：{text} -
          • - ))} -
        -

        {I.stateHint[cell.state] ?? ''}

        - {cell.decided == null && ( -
        - - -
        - )} -
        • - ) - })} -
      - - {clean.length > 0 && ( -
      - {/* 一次收掉这条上无冲突的几格：把这几格**明确列出来**发（不发 auto），页面列了几格就只接受几格 */} - -
      - )} -
      • - ) - })} -
    - )} - - {active.items.length > 0 && ( -
    - - - {!acceptCount && !rejectCount && {I.nothingSelected}} - setImportNote(e.target.value)} /> -
    - )} - - {importStale && ( -
    - - {importStale.code === 'import_stale_l' - ? I.staleL(importStale.names) - : I.stale(importStale.current, importStale.base)} - -
    - )} - {importError != null &&
    } - {(importAccepted || importRejected) && ( -
    - {importAccepted && ( - <> -

    - {I.accepted(importResult?.revision ?? null)} - {importResult?.replayed && · {I.replayed}} -

    -

    {I.acceptedHint}

    - - )} - {importRejected &&

    {I.rejected}

    } -
    - )} -
    - )} -
    - ) - } - - /** - * 用一句话改口径（K12）：先调只读的提案接口，把 operations 翻成「哪条指标、哪一项写法、现在什么 → 保存后什么」， - * 不摆 JSON、节点编号和字段路径。含糊的业务条件先显示澄清项，一条操作都不产出——确认（confirm=true）之前， - * 服务端一个字节都不写；落版失败时这里的输入和预览原样留着。 - */ - const renderNl = (draft: CaliberDraft) => { - if (!draft.workflow_id) return null - const proposal = nlPreview - const changes = proposal?.ready ? nlChanges(draft, proposal.operations) : [] - const clarifications = proposal?.clarifications ?? [] - // 输入改过之后这份提案就不作数了：必须重新预览（确认的是预览时那一段原文，不是输入框里现在的字） - const touched = !!nlPreviewOf && nlText.trim() !== nlPreviewOf - const confirmable = !touched && proposal?.ready && changes.length > 0 && draft.revision != null - return ( -
    -
    -

    {N.title}

    - {N.hint} -
    -
    -