From 8d9a712eae17622b8bbefcf3aec9cb2957a073fe Mon Sep 17 00:00:00 2001 From: ThearchyHelios Date: Thu, 12 Nov 2026 12:57:53 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=9A=E5=8A=A1=E6=9C=AC=E4=BD=93=E4=B8=8E?= =?UTF-8?q?=E6=A0=87=E5=87=86=E5=AE=9E=E4=BD=93=EF=BC=9A=E8=B7=A8=E6=BA=90?= =?UTF-8?q?=E9=94=AE=E5=80=BC=E5=AF=B9=E9=BD=90=E3=80=81=E5=AF=B9=E8=B4=A6?= =?UTF-8?q?=E5=85=B1=E5=8F=98=E7=AE=97=E5=AD=90=E3=80=81=E5=81=A5=E5=BA=B7?= =?UTF-8?q?=E8=AF=81=E6=8D=AE=E9=93=BE=E4=B8=8E=E5=AE=9E=E4=BD=93=E5=8A=A9?= =?UTF-8?q?=E6=89=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增标准业务实体（CanonicalEntity）模型、确定性跨源编码翻译、后台候选实体扫描与文档驱动实体智能助手 - 新增指标本体图谱（factor / reconciles_with / upstream_driver）、DAG 防环校验以及 ontology_neighbors / source_health 内置工具 - 新增跨源对账（analysis_reconcile）与滞后共变（analysis_cochange）分析算子，打通 [[h:H1]] 数据源健康证据链、报告导出与本体消融评测 - 数据目录新增「标准业务实体」视图与候选审阅工作台，口径页集成指标本体拓扑面板与端到端浏览器检查 --- backend/app/analysis/__init__.py | 12 +- backend/app/analysis/cochange.py | 414 +++++ backend/app/analysis/contract.py | 9 +- backend/app/analysis/reconcile_op.py | 336 ++++ backend/app/analysis/testing.py | 55 +- backend/app/api/evals.py | 10 +- backend/app/api/evidence.py | 96 +- backend/app/api/ontology.py | 354 +++++ backend/app/api/runs.py | 26 +- backend/app/api/settings.py | 19 + backend/app/apps/publications.py | 7 +- backend/app/apps/validation.py | 8 +- backend/app/auth/chat_policy.py | 10 +- backend/app/bundle/build.py | 8 +- backend/app/bundle/verifier.js | 34 +- backend/app/capabilities/registry.py | 15 +- backend/app/data/caliber_identity.py | 2 +- backend/app/data/entities.py | 566 +++++++ backend/app/data/entity_assistant.py | 942 +++++++++++ backend/app/data/entity_discovery.py | 687 ++++++++ backend/app/data/entity_docs.py | 334 ++++ backend/app/data/entity_patch.py | 841 ++++++++++ backend/app/data/source_health.py | 463 ++++++ backend/app/db/models.py | 3 + backend/app/db/models_ontology.py | 63 + backend/app/engine/analysis_claims.py | 162 +- backend/app/engine/analysis_scope.py | 11 +- backend/app/engine/compare.py | 7 +- backend/app/engine/evidence.py | 163 +- backend/app/engine/evidence_reader.py | 6 + backend/app/engine/guards.py | 32 +- backend/app/engine/investigation.py | 144 +- backend/app/engine/labels.py | 1 + backend/app/engine/nodes/llm.py | 109 +- backend/app/engine/nodes/metrics.py | 24 + backend/app/engine/nodes/report.py | 185 ++- backend/app/engine/nodes/tools.py | 12 + backend/app/engine/review.py | 49 +- backend/app/engine/schema.py | 2 +- backend/app/eval/answer.py | 4 +- backend/app/eval/cases.py | 17 +- backend/app/eval/compare.py | 18 +- backend/app/eval/judgements.py | 10 +- backend/app/eval/runner.py | 8 +- backend/app/eval/score.py | 33 +- backend/app/eval/variants.py | 38 +- backend/app/ledger/ontology.py | 707 +++++++++ backend/app/ledger/querycore.py | 64 +- backend/app/main.py | 10 +- backend/app/report_export/build.py | 3 +- backend/app/seed.py | 45 + backend/app/tools/analysis.py | 218 ++- backend/app/tools/builtin/analysis.py | 56 +- backend/app/tools/ontology.py | 612 ++++++++ backend/app/tools/registry.py | 1 + .../acceptance/multi_source_ontology.py | 693 ++++++++ backend/tests/fixtures/analysis_ops.py | 2 +- .../tests/fixtures/eval/ontology_suite.json | 191 +++ backend/tests/test_entity_agent.py | 823 ++++++++++ backend/tests/test_frontend_eval_unions.py | 6 +- .../test_ontology_investigation_and_claims.py | 481 ++++++ backend/tests/test_ontology_model.py | 353 +++++ backend/tests/test_ontology_operators.py | 369 +++++ backend/tests/test_template_anomaly.py | 4 +- frontend/src/App.tsx | 89 +- frontend/src/api/client.ts | 7 + frontend/src/api/ontology.ts | 167 ++ frontend/src/canvas/Inspector.tsx | 7 +- frontend/src/canvas/analysisItems.ts | 10 +- frontend/src/components/CommandPalette.tsx | 7 +- frontend/src/lib/evidence.ts | 4 + frontend/src/lib/termsAnalysis.ts | 10 +- frontend/src/lib/termsEval.ts | 13 +- frontend/src/lib/termsLedger.ts | 9 +- frontend/src/pages/SettingsPage.tsx | 41 +- frontend/src/pages/apps/shared.tsx | 48 +- frontend/src/pages/blueprints/NewAppStart.tsx | 2 +- .../src/pages/blueprints/PublishSummary.tsx | 2 +- .../src/pages/blueprints/TrialSection.tsx | 2 +- .../pages/catalog/CanonicalEntitiesView.tsx | 1212 ++++++++++++++ frontend/src/pages/catalog/CatalogPage.tsx | 40 +- .../src/pages/catalog/EntityAssistant.tsx | 823 ++++++++++ .../pages/catalog/EntityDiscoverDialog.tsx | 339 ++++ frontend/src/pages/evals/StartDialog.tsx | 26 +- frontend/src/pages/evals/parts.tsx | 18 +- frontend/src/pages/ledger/OntologyPanel.tsx | 697 +++++++++ frontend/src/run/EvidencePanel.tsx | 119 +- frontend/src/run/LedgerTurn.tsx | 29 +- frontend/src/run/decode.ts | 83 +- frontend/src/store/catalog.ts | 6 +- frontend/src/store/evidence.ts | 7 + frontend/src/types.ts | 55 +- frontend/src/typesApp.ts | 1 + frontend/src/typesEval.ts | 21 +- frontend/src/typesLedger.ts | 279 +++- scripts/check-all.mjs | 2 + scripts/check-apps.mjs | 40 +- scripts/check-auth.mjs | 14 +- scripts/check-bundle.mjs | 2 +- scripts/check-caliber-drafts.mjs | 307 ++++ scripts/check-evals.mjs | 2 +- scripts/check-ledger.mjs | 9 + scripts/check-manage.mjs | 5 +- scripts/check-ontology.mjs | 1392 +++++++++++++++++ scripts/check-shell.mjs | 14 +- scripts/check-studio.mjs | 2 +- 106 files changed, 16654 insertions(+), 295 deletions(-) create mode 100644 backend/app/analysis/cochange.py create mode 100644 backend/app/analysis/reconcile_op.py create mode 100644 backend/app/api/ontology.py create mode 100644 backend/app/data/entities.py create mode 100644 backend/app/data/entity_assistant.py create mode 100644 backend/app/data/entity_discovery.py create mode 100644 backend/app/data/entity_docs.py create mode 100644 backend/app/data/entity_patch.py create mode 100644 backend/app/data/source_health.py create mode 100644 backend/app/db/models_ontology.py create mode 100644 backend/app/ledger/ontology.py create mode 100644 backend/app/tools/ontology.py create mode 100644 backend/tests/fixtures/acceptance/multi_source_ontology.py create mode 100644 backend/tests/fixtures/eval/ontology_suite.json create mode 100644 backend/tests/test_entity_agent.py create mode 100644 backend/tests/test_ontology_investigation_and_claims.py create mode 100644 backend/tests/test_ontology_model.py create mode 100644 backend/tests/test_ontology_operators.py create mode 100644 frontend/src/api/ontology.ts create mode 100644 frontend/src/pages/catalog/CanonicalEntitiesView.tsx create mode 100644 frontend/src/pages/catalog/EntityAssistant.tsx create mode 100644 frontend/src/pages/catalog/EntityDiscoverDialog.tsx create mode 100644 frontend/src/pages/ledger/OntologyPanel.tsx create mode 100644 scripts/check-ontology.mjs diff --git a/backend/app/analysis/__init__.py b/backend/app/analysis/__init__.py index ecdb0c02..472852b0 100644 --- a/backend/app/analysis/__init__.py +++ b/backend/app/analysis/__init__.py @@ -58,7 +58,7 @@ def _op_mode(request: Mapping[str, Any]) -> tuple[str, str | None]: def _compute(op: str, mode: str | None, raw: Mapping[str, Any], env: Any) -> AnalysisResult: - from app.analysis import comparable, decompose, differ, series + from app.analysis import cochange, comparable, decompose, differ, reconcile_op, series if op == "decompose": modes = {"additive": decompose.additive, "ratio": decompose.ratio, "factor": decompose.factor} @@ -69,6 +69,10 @@ def _compute(op: str, mode: str | None, raw: Mapping[str, Any], env: Any) -> Ana return series.trend(raw, env) if op == "differ": return differ.differ(raw, env) + if op == "reconcile": + return reconcile_op.reconcile(raw, env) + if op == "cochange": + return cochange.cochange(raw, env) return comparable.comparable(raw, env) @@ -83,7 +87,7 @@ def _env(op: str, mode: str | None, settings: Mapping[str, Any], inputs: Any, fa def _data_refusal(op: str, mode: str | None, raw: Mapping[str, Any], env: Any, error: CallError) -> AnalysisResult: """随数据变化的调用错误记成的算不了的结论。params 记调用原样（label 照样消毒；调用还没走完、规范化不完整，复算拿它重走一遍 走到同一处），取过的快照都记进输入（Inputs.settle），阈值和判定标准同这个算子自己算不了时那一份。""" - from app.analysis import decompose, differ, series + from app.analysis import cochange, decompose, differ, reconcile_op, series from app.analysis import phrases as ph from app.analysis.common import build @@ -97,6 +101,10 @@ def _data_refusal(op: str, mode: str | None, raw: Mapping[str, Any], env: Any, e used, criterion = series.refusal_basis(op, env.settings) elif op == "differ": used, criterion = differ.refusal_basis(mode, env.settings) + elif op == "reconcile": + used, criterion = reconcile_op.refusal_basis(env.settings) + elif op == "cochange": + used, criterion = cochange.refusal_basis(env.settings) else: used, criterion = [], ph.CRITERION_COMPARABLE code = "parent_refused" if error.code == "parent_refused" else "input_unavailable" diff --git a/backend/app/analysis/cochange.py b/backend/app/analysis/cochange.py new file mode 100644 index 00000000..b3fa8194 --- /dev/null +++ b/backend/app/analysis/cochange.py @@ -0,0 +1,414 @@ +"""跨源伴随异动算子（cochange，W1-b）：检验主指标异动时，上游关联指标（可跨数据源、可设领先滞后期 lag）是否在对应期次同步超出各自历史常规波动范围。 + +设计铁律（计划书决策 D6）： +- 只检验两条序列各自相对自身稳健历史基线（中位数 + 1.4826·MAD）是否同步超出常规波动带（`k_warn` / `k_alert`）， + 不需要在小样本下硬算不可靠的相关系数 p 值； +- 系统渲染的判定语严格表述为「同期/领先期统计伴随异动（属关联共变线索，不直接构成因果证明）」，绝不输出「导致/致使」等硬因果断言； +- 全程有理数（Fraction）确定性计算，支持 `recompute` 逐字节复算（`contract.same`）。 +""" +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from fractions import Fraction +from typing import Any + +from app.analysis import periods as pd +from app.analysis import phrases as ph +from app.analysis import stats +from app.analysis.common import Env, build, column_label, column_types, label_for, need, refuse, reserved_check +from app.analysis.contract import CONV_PCT, CallError +from app.analysis.inputs import Snap, same_value +from app.analysis.numbers import canon +from app.analysis.series import MAD_SCALE, unreadable_text + +USED = ("k_warn", "k_alert", "min_history", "max_history") +FIXED = ("driver_period_val", "primary_value", "primary_eval", "driver_value", "driver_eval", "role") +_LAG_MAX = 12 + + +def refusal_basis(settings: Mapping[str, Any]) -> tuple[list[str], str]: + k_w = ph.amount(settings["k_warn"]) + min_h = settings["min_history"] + return list(USED), f"两侧序列各取至少 {min_h} 期历史，按中位数 ± {k_w} 倍稳健标准差（1.4826·MAD）检验是否同步超出常规波动" + + +@dataclass +class _ParsedSeries: + snap: Snap + period_col: str + measure_col: str + grain: str + label: str + by_ordinal: dict[int, tuple[pd.Period, Fraction]] + + +@dataclass +class _EvalOutcome: + target_period: pd.Period + target_val: Fraction + evaluated: Fraction # change rate or level + median: Fraction + sigma: Fraction + z: Fraction | None # None when sigma == 0 + anomalous: bool + severe: bool + history_n: int + history_pairs: list[tuple[pd.Period, Fraction, Fraction]] # (period, raw_value, evaluated) + + +def _extract_series( + env: Env, + snap: Snap, + period_col: str, + measure_col: str, + *, + grain_hint: str | None, + label: str, + role_title: str, +) -> tuple[_ParsedSeries | None, tuple[str, str] | None]: + i_period = snap.index(period_col) + raw_periods = [row[i_period] if i_period < len(row) else None for row in snap.rows] + weeks = pd.week_scheme(snap.sql, period_col, snap.dialect) + parsed = pd.parse(raw_periods, grain_hint, weeks=weeks) + if parsed is None: + return None, ("period_unreadable", unreadable_text(period_col, raw_periods, grain_hint, weeks)) + items, detected_grain = parsed + seen: set[int] = set() + by_ord: dict[int, tuple[pd.Period, Fraction]] = {} + for r, item in enumerate(items): + if item.ordinal in seen: + raise CallError( + "duplicate_rows", + ph.say(f"{snap.ref} 里同一期有不止一行：请先在 SQL 中按 ", column_label(env, snap, period_col), " 汇总到每期一行"), + ) + seen.add(item.ordinal) + val = snap.number(r, measure_col, role_title) + if val is None: + continue + if pd.short_days(item, detected_grain) is not None: + continue + by_ord[item.ordinal] = (item, val) + return _ParsedSeries(snap, period_col, measure_col, detected_grain, label, by_ord), None + + +def _evaluate_at( + series: _ParsedSeries, + target_ord: int, + *, + basis: str, + min_history: int, + max_history: int, + k_warn: Fraction, + k_alert: Fraction, + who: str, +) -> tuple[_EvalOutcome | None, tuple[str, str] | None]: + if target_ord not in series.by_ordinal: + return None, ("target_gap", f"{who}缺少目标期次的数据，无法核对伴随波动") + + step = 1 + eval_points: list[tuple[pd.Period, Fraction, Fraction]] = [] + for ord_k in sorted(series.by_ordinal): + if ord_k > target_ord: + continue + p_item, val = series.by_ordinal[ord_k] + if basis == "level": + eval_points.append((p_item, val, val)) + else: + prev = series.by_ordinal.get(ord_k - step) + if prev is None: + if ord_k == target_ord: + return None, ("target_gap", f"{who}在目标期次「{p_item.raw}」的前一期缺失，无法计算环比变化率") + continue + if prev[1] == 0: + if ord_k == target_ord: + return None, ("zero_base", f"{who}在比较期「{prev[0].raw}」为 0，无法计算变化率（可改用 basis='level'）") + continue + eval_points.append((p_item, val, (val - prev[1]) / abs(prev[1]))) + + if not eval_points or eval_points[-1][0].ordinal != target_ord: + return None, ("target_gap", f"{who}在目标期次无可用评估值") + + target_tuple = eval_points[-1] + history = eval_points[:-1][-max_history:] + if len(history) < min_history: + return None, ( + "history_short", + f"{who}可用历史只有 {len(history)} 期（至少需要 {min_history} 期）", + ) + + hist_vals = [x[2] for x in history] + med = stats.median(hist_vals) + mad = stats.mad(hist_vals, med) + sigma = MAD_SCALE * mad + curr = target_tuple[2] + diff = curr - med + if sigma == 0: + z = None + anomalous = diff != 0 + severe = diff != 0 + else: + z = diff / sigma + anomalous = abs(z) >= k_warn + severe = abs(z) >= k_alert + + return _EvalOutcome( + target_period=target_tuple[0], + target_val=target_tuple[1], + evaluated=curr, + median=med, + sigma=sigma, + z=z, + anomalous=anomalous, + severe=severe, + history_n=len(history), + history_pairs=[*history, target_tuple], + ), None + + +def cochange(raw: Mapping[str, Any], env: Env) -> Any: + k_warn = env.threshold("k_warn") + k_alert = env.threshold("k_alert") + min_history = int(env.settings["min_history"]) + max_history = int(env.settings["max_history"]) + _, criterion = refusal_basis(env.settings) + + has_single = raw.get("query") not in (None, "") + has_two = raw.get("base_query") not in (None, "") or raw.get("target_query") not in (None, "") + if has_single and has_two: + raise CallError("conflicting_queries", "伴随异动检验不能同时写 query 和 base_query / target_query") + if not has_single and not has_two: + raise CallError("missing_query", "伴随异动检验请提供两份序列查询（base_query 与 target_query），或包含两列的单份查询（query）") + + basis = str(raw.get("basis") or "change").strip() + if basis not in ("change", "level"): + raise CallError("bad_basis", "比较方式（basis）只能是 change（变化率）或 level（水平值）") + grain_hint = raw.get("grain") + if grain_hint is not None and grain_hint not in (*pd.GRAINS, "seq"): + raise CallError("bad_grain", "期次粒度（grain）只能写 day、week、month、quarter 或留空") + + lag_raw = raw.get("lag") + if lag_raw in (None, ""): + lag = 0 + elif isinstance(lag_raw, bool) or not isinstance(lag_raw, int) or not (0 <= lag_raw <= _LAG_MAX): + raise CallError("bad_lag", f"上游领先期数（lag）须为 0 到 {_LAG_MAX} 之间的整数（0 表示同期）") + else: + lag = lag_raw + + params: dict[str, Any] = {"basis": basis} + if lag > 0: + params["lag"] = lag + + if has_single: + snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号", role="data") + p_col = snap.column(need(raw, "period", "期次列"), "期次列") + reserved_check(p_col, FIXED, "期次列") + if raw.get("target_period") not in (None, ""): + raise CallError("unexpected_target_period", "单份查询（query）共用期次列 period，不需要写 target_period") + m_col = snap.column(need(raw, "measure", "主指标度量列"), "主指标度量列") + d_col = snap.column(need(raw, "driver_measure", "驱动度量列"), "驱动度量列") + if m_col == d_col: + raise CallError("same_column", f"主指标度量列与驱动度量列不能是同一列「{m_col}」") + params["query"] = snap.ref + params["period"] = p_col + params["measure"] = m_col + params["driver_measure"] = d_col + p_label = label_for(env, raw.get("label"), snap, m_col) + d_label = column_label(env, snap, d_col) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + env.inputs.use(snap, "data", [p_col, m_col, d_col]) + if snap.truncated: + return refuse(env, params, USED, "truncated", f"{snap.ref} 结果已被截断，无法核对完整时间序列", criterion=criterion) + p_series, err = _extract_series(env, snap, p_col, m_col, grain_hint=grain_hint, label=p_label, role_title="主指标度量列") + if err is not None or p_series is None: + return refuse(env, params, USED, err[0], err[1], criterion=criterion) # type: ignore[index] + d_series, err = _extract_series(env, snap, p_col, d_col, grain_hint=p_series.grain, label=d_label, role_title="驱动度量列") + if err is not None or d_series is None: + return refuse(env, params, USED, err[0], err[1], criterion=criterion) # type: ignore[index] + period_dim_type = snap.kind(p_col) + else: + p_snap = env.inputs.snap(need(raw, "base_query", "主指标查询编号"), "主指标查询编号", role="base") + d_snap = env.inputs.snap(need(raw, "target_query", "驱动指标查询编号"), "驱动指标查询编号", role="target") + p_col = p_snap.column(need(raw, "period", "主指标期次列"), "主指标期次列") + reserved_check(p_col, FIXED, "主指标期次列") + d_p_col = d_snap.column(raw.get("target_period") or p_col, "驱动期次列") + m_col = p_snap.column(need(raw, "measure", "主指标度量列"), "主指标度量列") + d_col = d_snap.column(need(raw, "driver_measure", "驱动度量列"), "驱动度量列") + + params["base_query"] = p_snap.ref + params["target_query"] = d_snap.ref + params["period"] = p_col + if raw.get("target_period") not in (None, ""): + params["target_period"] = d_p_col + params["measure"] = m_col + params["driver_measure"] = d_col + p_label = label_for(env, raw.get("label"), p_snap, m_col) + d_label = column_label(env, d_snap, d_col) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + + env.inputs.use(p_snap, "base", [p_col, m_col]) + env.inputs.use(d_snap, "target", [d_p_col, d_col]) + if p_snap.truncated: + return refuse(env, params, USED, "truncated", f"{p_snap.ref} 结果已被截断，无法核对完整时间序列", criterion=criterion) + if d_snap.truncated: + return refuse(env, params, USED, "truncated", f"{d_snap.ref} 结果已被截断，无法核对完整时间序列", criterion=criterion) + + p_series, err = _extract_series(env, p_snap, p_col, m_col, grain_hint=grain_hint, label=p_label, role_title="主指标度量列") + if err is not None or p_series is None: + return refuse(env, params, USED, err[0], err[1], criterion=criterion) # type: ignore[index] + d_series, err = _extract_series(env, d_snap, d_p_col, d_col, grain_hint=p_series.grain, label=d_label, role_title="驱动度量列") + if err is not None or d_series is None: + return refuse(env, params, USED, err[0], err[1], criterion=criterion) # type: ignore[index] + if d_series.grain != p_series.grain: + return refuse( + env, + params, + USED, + "incomparable_sides", + f"主指标期次粒度（{p_series.grain}）与驱动指标期次粒度（{d_series.grain}）不一致，无法对齐检验", + criterion=criterion, + ) + period_dim_type = p_snap.kind(p_col) + + params["grain"] = p_series.grain + if not p_series.by_ordinal: + return refuse(env, params, USED, "history_short", f"主指标「{p_label}」没有有效数值行", criterion=criterion) + + wanted_target = raw.get("target") + if wanted_target in (None, ""): + target_ord = max(p_series.by_ordinal) + else: + matched_ords = [ord_k for ord_k, (p_it, _) in p_series.by_ordinal.items() if same_value(p_it.raw, wanted_target)] + if not matched_ords: + raise CallError( + "period_not_found", + f"主指标序列里没有期次「{wanted_target}」", + reason=f"主指标这次的数据里没有期次「{wanted_target}」", + ) + target_ord = matched_ords[0] + params["target"] = str(p_series.by_ordinal[target_ord][0].raw) + + p_eval, p_err = _evaluate_at( + p_series, + target_ord, + basis=basis, + min_history=min_history, + max_history=max_history, + k_warn=k_warn, + k_alert=k_alert, + who=f"主指标「{p_label}」", + ) + if p_err is not None or p_eval is None: + return refuse(env, params, USED, p_err[0], p_err[1], criterion=criterion) # type: ignore[index] + + driver_ord = target_ord - lag + d_eval, d_err = _evaluate_at( + d_series, + driver_ord, + basis=basis, + min_history=min_history, + max_history=max_history, + k_warn=k_warn, + k_alert=k_alert, + who=f"驱动指标「{d_label}」", + ) + if d_err is not None or d_eval is None: + return refuse(env, params, USED, d_err[0], d_err[1], criterion=criterion) # type: ignore[index] + + p_delta = p_eval.evaluated - p_eval.median + d_delta = d_eval.evaluated - d_eval.median + p_shown = ph.pct(p_eval.evaluated, signed=True) if basis == "change" else ph.amount(p_eval.evaluated) + d_shown = ph.pct(d_eval.evaluated, signed=True) if basis == "change" else ph.amount(d_eval.evaluated) + lag_word = f"领先 {lag} 期（{d_eval.target_period.raw}）" if lag > 0 else f"同期（{d_eval.target_period.raw}）" + + if not p_eval.anomalous: + verdict_code = "primary_normal" + direction = None + phrase = ph.say( + f"主指标「{p_label}」在 {p_eval.target_period.raw} 处于自身常规波动范围内（{p_shown}），无需触发跨源伴随异动归因。" + ) + elif d_eval.anomalous: + verdict_code = "co_anomalous" + direction = "up" if p_delta > 0 else ("down" if p_delta < 0 else None) + same_sign = (p_delta > 0 and d_delta > 0) or (p_delta < 0 and d_delta < 0) + rel_word = "同向伴随异动" if same_sign else "反向伴随异动" + pz_txt = f"，稳健 z = {ph.amount(p_eval.z, signed=True)}" if p_eval.z is not None else "" + dz_txt = f"，稳健 z = {ph.amount(d_eval.z, signed=True)}" if d_eval.z is not None else "" + phrase = ph.say( + f"主指标「{p_label}」在 {p_eval.target_period.raw} 超出常规波动范围（{p_shown}{pz_txt}），" + f"上游关联指标「{d_label}」在{lag_word}亦超出自身常规波动范围（{d_shown}{dz_txt}），" + f"两者呈{rel_word}（属统计共变线索，不直接构成因果证明）。" + ) + else: + verdict_code = "target_normal" + direction = "up" if p_delta > 0 else ("down" if p_delta < 0 else None) + phrase = ph.say( + f"主指标「{p_label}」在 {p_eval.target_period.raw} 超出常规波动范围（{p_shown}），" + f"但上游关联指标「{d_label}」在{lag_word}处于常规波动范围内（{d_shown}），未见伴随异动。" + ) + + # 组装对齐结果表（最近最多 12 期历史 + 目标期） + p_hist_map = {pt[0].ordinal: pt for pt in p_eval.history_pairs} + d_hist_map = {pt[0].ordinal: pt for pt in d_eval.history_pairs} + recent_ords = [pt[0].ordinal for pt in p_eval.history_pairs[-12:]] + rows_out: list[list[Any]] = [] + for ord_k in recent_ords: + p_pt = p_hist_map[ord_k] + d_pt = d_hist_map.get(ord_k - lag) + role_txt = "当期" if ord_k == target_ord else "历史基线" + rows_out.append([ + p_pt[0].raw, + d_pt[0].raw if d_pt else None, + canon(p_pt[1]), + canon(p_pt[2]), + canon(d_pt[1]) if d_pt else None, + canon(d_pt[2]) if d_pt else None, + role_txt, + ]) + + cols = [p_col, *FIXED] + labels_map = { + p_col: "主指标期次", + "driver_period_val": "驱动指标期次", + "primary_value": p_label, + "primary_eval": "主指标变化率" if basis == "change" else "主指标水平", + "driver_value": d_label, + "driver_eval": "驱动变化率" if basis == "change" else "驱动水平", + "role": "期次角色", + } + table_formats = {"primary_eval": CONV_PCT, "driver_eval": CONV_PCT} if basis == "change" else {} + named_formats = {"primary_evaluated": CONV_PCT, "driver_evaluated": CONV_PCT} if basis == "change" else {} + + table = { + "columns": cols, + "labels": labels_map, + "rows": rows_out, + "column_types": column_types(cols, dims={p_col: period_dim_type}, texts=("driver_period_val", "role")), + } + values: dict[str, Any] = { + "primary_value": canon(p_eval.target_val), + "primary_evaluated": canon(p_eval.evaluated), + "primary_median": canon(p_eval.median), + "primary_z": canon(p_eval.z), + "driver_value": canon(d_eval.target_val), + "driver_evaluated": canon(d_eval.evaluated), + "driver_median": canon(d_eval.median), + "driver_z": canon(d_eval.z), + "history_periods": str(min(p_eval.history_n, d_eval.history_n)), + "lag_periods": str(lag), + } + verdict = {"code": verdict_code, "subjects": [[d_label]] if verdict_code == "co_anomalous" else [], "direction": direction} + return build( + env, + params, + USED, + table=table, + values=values, + verdict=verdict, + criterion=criterion, + phrase=phrase, + formats=table_formats, + named=named_formats, + ) diff --git a/backend/app/analysis/contract.py b/backend/app/analysis/contract.py index 1da558d8..3634aa34 100644 --- a/backend/app/analysis/contract.py +++ b/backend/app/analysis/contract.py @@ -32,8 +32,8 @@ NODE_TYPE = "analysis" #: 设置分组的键：settings 里的 analysis 一组、run.options["analysis"] SETTINGS_KEY = "analysis" -Op = Literal["decompose", "baseline", "trend", "differ", "comparable"] -OPS: tuple[str, ...] = ("decompose", "baseline", "trend", "differ", "comparable") +Op = Literal["decompose", "baseline", "trend", "differ", "comparable", "reconcile", "cochange"] +OPS: tuple[str, ...] = ("decompose", "baseline", "trend", "differ", "comparable", "reconcile", "cochange") #: 有模式的算子和它们的模式；其余算子 mode 为 None MODES: dict[str, tuple[str, ...]] = { "decompose": ("additive", "ratio", "factor"), @@ -50,6 +50,8 @@ METHODS: dict[tuple[str, str | None], str] = { ("differ", "mean"): "differ.welch@2", ("differ", "rows"): "differ.welch_rows@2", ("comparable", None): "comparable@2", + ("reconcile", None): "reconcile@1", + ("cochange", None): "cochange.mad@1", } #: agent 分析工具的名字（tools/builtin 里注册），一个算子一个工具：参数各自聚焦，模型少写错 TOOL_NAMES: dict[str, str] = {op: f"analysis_{op}" for op in OPS} @@ -57,6 +59,7 @@ TOOL_NAMES: dict[str, str] = {op: f"analysis_{op}" for op in OPS} #: （engine/labels 的 ANALYSIS_OP_LABEL / ANALYSIS_MODE_LABEL 引用这里）、「{叫法}没有做成」的判定语都用它（method_label 拼成一句） OP_LABELS: dict[str, str] = { "decompose": "变化拆解", "baseline": "波动基线", "trend": "趋势判断", "differ": "差异检验", "comparable": "可比性检查", + "reconcile": "跨源对账", "cochange": "伴随异动", } MODE_LABELS: dict[str, str] = { "additive": "可加", "ratio": "比率", "factor": "乘积", "rate": "比率", "mean": "均值", "rows": "逐行明细", @@ -82,6 +85,8 @@ VERDICTS: dict[str, tuple[str, ...]] = { "trend": ("rising", "falling", "no_trend"), "differ": ("differs", "no_evidence"), "comparable": ("comparable", "conditional", "incomparable"), + "reconcile": ("aligned", "diverged", "missing_counterpart"), + "cochange": ("co_anomalous", "target_normal", "primary_normal"), } #: 算不了的原因（细案 01 §8.2）。算不了也记成结论：写作者可以照实写「历史只有 5 周，判断不了是否异常」。最后两种是分析卡 #: 里随数据变化的调用错误记成的（analysis.run 的 refuse_data_errors，见 app/analysis 的 DATA_ERRORS）：parent_refused 上一层 diff --git a/backend/app/analysis/reconcile_op.py b/backend/app/analysis/reconcile_op.py new file mode 100644 index 00000000..4e2f011e --- /dev/null +++ b/backend/app/analysis/reconcile_op.py @@ -0,0 +1,336 @@ +"""跨源对账算子（reconcile，W1-b）：对齐同一份或两份查询快照上的镜像指标，按容差判定对账一致、存在偏差或对侧缺失。 + +输入方式： +- 两份跨源快照：`base_query`（主源/左侧）+ `target_query`（对账源/右侧），`measure`（左侧度量列），可选 `target_measure` + （右侧度量列，缺省同 `measure`）、`by`（左侧对齐键列，0..2 列）、`target_by`（右侧对齐键列，缺省同 `by`）、 + `unit_factor`（物理计量单位换算系数，右侧乘以该系数后与左侧比较，缺省 1）； +- 单份已合并快照：`query` + `measure`（左侧列）+ `target_measure`（右侧列），可选 `by`、`unit_factor`。 + +阈值契约（D4）：请求参数不收容差阈值，统一使用平台分析设置里的 `flat_pct`（相对容差，缺省 0.5%）与 `table_top_k`。 +纯函数，全程精确有理数（Fraction），支持 `recompute` 逐字节复算核对（`contract.same`）。 +""" +from __future__ import annotations + +import math +from collections.abc import Mapping +from fractions import Fraction +from typing import Any + +from app.analysis import phrases as ph +from app.analysis.common import Env, build, column_label, column_types, label_for, need, refuse, reserved_check +from app.analysis.contract import CONV_PCT, CallError +from app.analysis.inputs import Snap, as_list, hashable +from app.analysis.numbers import canon, to_fraction + +USED = ("flat_pct", "table_top_k") +FIXED = ("left_value", "right_value", "diff", "rel_diff", "row_status") +_MAX_BY = 2 + + +def refusal_basis(settings: Mapping[str, Any]) -> tuple[list[str], str]: + tol = ph.pct(settings["flat_pct"], signed=False) + return list(USED), f"按相对误差不超过 {tol} 核对两侧口径数值是否对齐" + + +def _parse_unit_factor(raw_val: Any) -> tuple[Fraction, float | int | None]: + if raw_val in (None, ""): + return Fraction(1), None + if isinstance(raw_val, bool) or not isinstance(raw_val, (int, float)): + f = to_fraction(raw_val) + if f is None or f <= 0: + raise CallError("bad_unit_factor", "单位换算系数（unit_factor）须为大于 0 的有限数字（如吨转千克写 1000）") + as_num: float | int = int(f) if f.denominator == 1 else float(f) + return f, as_num + if not math.isfinite(float(raw_val)) or float(raw_val) <= 0: + raise CallError("bad_unit_factor", "单位换算系数（unit_factor）须为大于 0 的有限数字（如吨转千克写 1000）") + f = to_fraction(raw_val) + assert f is not None and f > 0 + as_num = int(raw_val) if isinstance(raw_val, int) else float(raw_val) + return f, as_num + + +def _norm_key(raw_key: tuple[Any, ...]) -> tuple[str, ...]: + return tuple("" if v is None else str(hashable(v)).strip() for v in raw_key) + + +def _extract_side( + env: Env, + snap: Snap, + by_cols: list[str], + measure_col: str, + *, + role_name: str, +) -> dict[tuple[str, ...], tuple[list[Any], Fraction | None]]: + out: dict[tuple[str, ...], tuple[list[Any], Fraction | None]] = {} + if not by_cols: + if len(snap.rows) > 1: + raise CallError( + "duplicate_rows", + f"{snap.ref} 有 {len(snap.rows)} 行，未指定对齐维度列（by）时每侧只能是汇总后的 1 行；请加上 by 列或在 SQL 中汇总", + ) + val = snap.number(0, measure_col, role_name) if snap.rows else None + out[()] = ([], val) + return out + + idx_by = [snap.index(c) for c in by_cols] + for r in range(len(snap.rows)): + row = snap.rows[r] + raw_vals = [row[i] if i < len(row) else None for i in idx_by] + key = _norm_key(tuple(raw_vals)) + if key in out: + shown = ph.key_text(raw_vals) + cols_label = "、".join(column_label(env, snap, c) for c in by_cols) + raise CallError( + "duplicate_rows", + f"{snap.ref} 在维度 {cols_label} = {shown} 上有多行：请先在 SQL 中按对齐维度汇总到唯一一行", + ) + val = snap.number(r, measure_col, role_name) + out[key] = (raw_vals, val) + return out + + +def reconcile(raw: Mapping[str, Any], env: Env) -> Any: + tol = env.threshold("flat_pct") + top_k = int(env.settings["table_top_k"]) + _, criterion = refusal_basis(env.settings) + + has_single = raw.get("query") not in (None, "") + has_two = raw.get("base_query") not in (None, "") or raw.get("target_query") not in (None, "") + if has_single and has_two: + raise CallError("conflicting_queries", "跨源对账不能同时写 query 和 base_query / target_query：单表对账写 query，两份快照对账写 base_query 与 target_query") + if not has_single and not has_two: + raise CallError("missing_query", "跨源对账请提供两份查询编号（base_query 与 target_query），或单份合并查询编号（query）") + + by_raw = [str(c).strip() for c in as_list(raw.get("by")) if str(c).strip()] + if len(by_raw) > _MAX_BY: + raise CallError("too_many_by", f"对账对齐维度最多支持 {_MAX_BY} 列") + + factor_frac, factor_param = _parse_unit_factor(raw.get("unit_factor")) + params: dict[str, Any] = {} + + if has_single: + snap = env.inputs.snap(need(raw, "query", "查询编号"), "查询编号", role="data") + by_cols = [snap.column(c, "对齐维度列") for c in by_raw] + for c in by_cols: + reserved_check(c, FIXED, "对齐维度列") + if raw.get("target_by") not in (None, [], ""): + raise CallError("unexpected_target_by", "单份查询（query）对账共用同一组维度列 by，不需要写 target_by") + left_col = snap.column(need(raw, "measure", "左侧度量列"), "左侧度量列") + right_col = snap.column(need(raw, "target_measure", "对侧度量列"), "对侧度量列") + if left_col == right_col and factor_frac == 1: + raise CallError("same_column", f"单表对账的左侧度量列与对侧度量列不能是同一列「{left_col}」") + params["query"] = snap.ref + if by_cols: + params["by"] = by_cols + params["measure"] = left_col + params["target_measure"] = right_col + if factor_param is not None: + params["unit_factor"] = factor_param + label = label_for(env, raw.get("label"), snap, left_col) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + env.inputs.use(snap, "data", [*by_cols, left_col, right_col]) + if snap.truncated: + return refuse(env, params, USED, "truncated", f"{snap.ref} 的查询结果已被截断，无法做完整对账", criterion=criterion) + left_map = _extract_side(env, snap, by_cols, left_col, role_name="左侧度量列") + right_map = _extract_side(env, snap, by_cols, right_col, role_name="对侧度量列") + dim_types = {c: snap.kind(c) for c in by_cols} + dim_labels = {c: column_label(env, snap, c) for c in by_cols} + else: + left_snap = env.inputs.snap(need(raw, "base_query", "主侧查询编号"), "主侧查询编号", role="base") + right_snap = env.inputs.snap(need(raw, "target_query", "对侧查询编号"), "对侧查询编号", role="target") + left_by = [left_snap.column(c, "对齐维度列") for c in by_raw] + for c in left_by: + reserved_check(c, FIXED, "对齐维度列") + t_by_raw = [str(c).strip() for c in as_list(raw.get("target_by")) if str(c).strip()] + if t_by_raw and len(t_by_raw) != len(left_by): + raise CallError("target_by_mismatch", f"对侧维度列（target_by，{len(t_by_raw)} 列）数量须与对齐维度列（by，{len(left_by)} 列）一致") + right_by = [right_snap.column(c, "对侧维度列") for c in (t_by_raw or by_raw)] + left_col = left_snap.column(need(raw, "measure", "左侧度量列"), "左侧度量列") + right_col = right_snap.column(raw.get("target_measure") or left_col, "对侧度量列") + + params["base_query"] = left_snap.ref + params["target_query"] = right_snap.ref + if left_by: + params["by"] = left_by + if t_by_raw: + params["target_by"] = right_by + params["measure"] = left_col + if raw.get("target_measure") not in (None, ""): + params["target_measure"] = right_col + if factor_param is not None: + params["unit_factor"] = factor_param + label = label_for(env, raw.get("label"), left_snap, left_col) + if raw.get("label") is not None: + params["label"] = ph.clean_label(raw.get("label")) or None + + env.inputs.use(left_snap, "base", [*left_by, left_col]) + env.inputs.use(right_snap, "target", [*right_by, right_col]) + if left_snap.truncated: + return refuse(env, params, USED, "truncated", f"{left_snap.ref} 的查询结果已被截断，无法做完整对账", criterion=criterion) + if right_snap.truncated: + return refuse(env, params, USED, "truncated", f"{right_snap.ref} 的查询结果已被截断，无法做完整对账", criterion=criterion) + left_map = _extract_side(env, left_snap, left_by, left_col, role_name="左侧度量列") + right_map = _extract_side(env, right_snap, right_by, right_col, role_name="对侧度量列") + by_cols = left_by + dim_types = {c: left_snap.kind(c) for c in left_by} + dim_labels = {c: column_label(env, left_snap, c) for c in left_by} + + all_keys = sorted(set(left_map) | set(right_map)) + paired_records: list[dict[str, Any]] = [] + left_sum = Fraction(0) + right_sum = Fraction(0) + matched_count = 0 + diverged_count = 0 + missing_count = 0 + max_abs = Fraction(0) + max_rel = Fraction(0) + + for key in all_keys: + l_entry = left_map.get(key) + r_entry = right_map.get(key) + raw_dims = (l_entry[0] if l_entry is not None else (r_entry[0] if r_entry is not None else [])) + l_val = l_entry[1] if l_entry is not None else None + r_raw = r_entry[1] if r_entry is not None else None + r_val = (r_raw * factor_frac) if r_raw is not None else None + + if l_val is None and r_val is None: + continue + if l_val is not None: + left_sum += l_val + if r_val is not None: + right_sum += r_val + + if l_val is None or r_val is None: + missing_count += 1 + status_code = "missing_right" if l_val is not None else "missing_left" + sort_gap = abs(l_val if l_val is not None else r_val) # type: ignore[arg-type] + paired_records.append({ + "dims": raw_dims, + "key": key, + "left": l_val, + "right": r_val, + "diff": None, + "rel": None, + "status": status_code, + "sort_rank": (2, sort_gap), + }) + continue + + matched_count += 1 + diff = l_val - r_val + abs_diff = abs(diff) + denom = max(abs(l_val), abs(r_val)) + rel = (abs_diff / denom) if denom != 0 else Fraction(0) + is_div = (abs_diff > tol * denom) if denom != 0 else False + if is_div: + diverged_count += 1 + if abs_diff > max_abs: + max_abs = abs_diff + if rel > max_rel: + max_rel = rel + paired_records.append({ + "dims": raw_dims, + "key": key, + "left": l_val, + "right": r_val, + "diff": diff, + "rel": rel, + "status": "diverged" if is_div else "aligned", + "sort_rank": (1 if is_div else 0, abs_diff), + }) + + if not paired_records: + return refuse(env, params, USED, "too_few_segments", "两侧查询均无有效数值行，无法对账", criterion=criterion) + + paired_records.sort(key=lambda item: (-item["sort_rank"][0], -item["sort_rank"][1], item["key"])) + net_diff = left_sum - right_sum + net_denom = max(abs(left_sum), abs(right_sum)) + net_rel = (abs(net_diff) / net_denom) if net_denom != 0 else Fraction(0) + + if diverged_count == 0 and missing_count > 0: + verdict_code = "missing_counterpart" + direction = None + missing_recs = [r for r in paired_records if r["status"] != "aligned"] + subjects = [[str(x) for x in r["dims"]] for r in missing_recs[:3] if r["dims"]] + matched_note = f"（已配对的 {matched_count} 项数值一致）" if matched_count > 0 else "" + phrase = ph.say( + f"「{label}」跨源对账发现对侧缺失（共 {missing_count} 项未能配对）{matched_note}，两侧无法完全闭合，请优先排查对侧数据源同步缺口或主数据编码未映射。" + ) + elif diverged_count == 0 and missing_count == 0: + verdict_code = "aligned" + direction = None + subjects = [] + phrase = ph.say( + f"「{label}」两侧对账一致（合计 {ph.amount(left_sum)} 对 {ph.amount(right_sum)}，最大相对偏差 {ph.pct(max_rel, signed=False)}，在容差 {ph.pct(tol, signed=False)} 以内）。" + ) + else: + verdict_code = "diverged" + direction = "up" if net_diff > 0 else ("down" if net_diff < 0 else None) + flagged = [r for r in paired_records if r["status"] != "aligned"] + subjects = [[str(x) for x in r["dims"]] for r in flagged[:3] if r["dims"]] + extra_missing = f"，另有 {missing_count} 项对侧缺失" if missing_count else "" + top_hint = "" + if subjects: + top_hint = f"（偏差集中在 {'、'.join(ph.key_text(s) for s in subjects)}）" + phrase = ph.say( + f"「{label}」跨源对账存在显著偏差：主侧合计 {ph.amount(left_sum)}，对侧合计 {ph.amount(right_sum)}，净差额 {ph.amount(net_diff, signed=True)}" + f"（最大相对偏差 {ph.pct(max_rel, signed=False)}，超出容差 {ph.pct(tol, signed=False)}，共 {diverged_count} 项超差{extra_missing}）{top_hint}。" + ) + + cols = [*by_cols, *FIXED] + labels_map = { + **dim_labels, + "left_value": "主侧数值", + "right_value": "对侧数值", + "diff": "差额", + "rel_diff": "相对偏差", + "row_status": "对账状态", + } + status_cn = { + "aligned": "一致", + "diverged": "超差", + "missing_right": "对侧缺失", + "missing_left": "主侧缺失", + } + shown_rows = [ + [ + *rec["dims"], + canon(rec["left"]), + canon(rec["right"]), + canon(rec["diff"]), + canon(rec["rel"]), + status_cn[rec["status"]], + ] + for rec in paired_records[:top_k] + ] + table = { + "columns": cols, + "labels": labels_map, + "rows": shown_rows, + "column_types": column_types(cols, dims=dim_types, texts=("row_status",)), + } + values = { + "left_total": canon(left_sum), + "right_total": canon(right_sum), + "net_diff": canon(net_diff), + "net_rel_diff": canon(net_rel), + "max_abs_diff": canon(max_abs), + "max_rel_diff": canon(max_rel), + "matched_rows": str(matched_count), + "diverged_rows": str(diverged_count), + "missing_counterpart_rows": str(missing_count), + } + verdict: dict[str, Any] = {"code": verdict_code, "subjects": subjects, "direction": direction} + return build( + env, + params, + USED, + table=table, + values=values, + verdict=verdict, + criterion=criterion, + phrase=phrase, + formats={"rel_diff": CONV_PCT}, + named={"net_rel_diff": CONV_PCT, "max_rel_diff": CONV_PCT}, + ) diff --git a/backend/app/analysis/testing.py b/backend/app/analysis/testing.py index 538c3792..4c64c8f6 100644 --- a/backend/app/analysis/testing.py +++ b/backend/app/analysis/testing.py @@ -19,6 +19,11 @@ _PHRASES = { "no_evidence": "西南和华东的退款率差异可能只是样本波动（样本量 40、230）", "comparable": "两期可比", "incomparable": "两期不可比：本期只有 3 天数据", + "aligned": "两侧报废数对账一致（主侧 120，对账侧 120，相差 0）", + "diverged": "两侧报废数存在显著偏差（主侧 120，对账侧 95，相差 +25，+26.3%），偏差集中在 L2", + "missing_counterpart": "两侧报废数对账发现对侧缺失记录（共 1 项缺失：L4）", + "co_anomalous": "产线效率本期变化 -14.2% 超出常规波动，关联的非计划停机在同期同样超出其常规波动（+185.0%，同期同步异动，统计伴随关系，非因果断言）", + "target_normal": "产线效率本期变化 -14.2% 超出常规波动，但关联的非计划停机在同期处于其常规波动范围内（+2.1%）", } _WEEKS = ["2026-07-20", "2026-07-27", "2026-08-03", "2026-08-10", "2026-08-17", "2026-08-24", "2026-08-31", @@ -102,13 +107,57 @@ def _comparable() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[ return table, values, params, ["gmv"] +def _reconcile() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]: + columns = ["line_code", "base", "target", "scaled_target", "diff", "rel_diff", "status"] + table = { + "columns": columns, + "labels": {"line_code": "产线", "base": "主侧", "target": "对账侧原值", "scaled_target": "换算后对账侧", + "diff": "差值", "rel_diff": "相对偏差", "status": "结果"}, + "rows": [["L2", "60", "35", "35", "25", "0.714285714286", "超差"], + ["L1", "60", "60", "60", "0", "0", "一致"]], + "column_types": {"line_code": "text", "base": "number", "target": "number", "scaled_target": "number", + "diff": "number", "rel_diff": "number", "status": "text"}, + } + values = {"base_total": "120", "target_total": "95", "diff_total": "25", "rel_diff_total": "0.263157894737", + "unit_factor": "1", "matched_count": "2", "diverged_count": "1", "missing_base_count": "0", + "missing_target_count": "0", "max_abs_diff": "25"} + params = {"base_query": "Q1", "target_query": "Q2", "by": ["line_code"], "measure": "scrap_qty", + "target_measure": "scrap_units", "unit_factor": "1", "label": "报废数"} + return table, values, params, ["line_code", "scrap_qty"] + + +def _cochange() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], list[str]]: + columns = ["day", "primary_value", "primary_change", "driver_value", "driver_change", "role"] + rows = [[w, "92", "0.01", "12", "0.02", "历史"] for w in _WEEKS[:-1]] + [ + [_WEEKS[-1], "79", "-0.141304347826", "34", "1.833333333333", "本期"] + ] + table = { + "columns": columns, + "labels": {"day": "日期", "primary_value": "产线效率", "primary_change": "主指标变化率", + "driver_value": "非计划停机", "driver_change": "关联指标变化率", "role": "说明"}, + "rows": rows, + "column_types": {"day": "text", "primary_value": "number", "primary_change": "number", + "driver_value": "number", "driver_change": "number", "role": "text"}, + } + values = {"primary_x": "-0.141304347826", "primary_z": "-4.2", "primary_band_lo": "-0.04", + "primary_band_hi": "0.04", "driver_x": "1.833333333333", "driver_z": "5.8", + "driver_band_lo": "-0.15", "driver_band_hi": "0.15", "lag": "0", "n_history": "12", + "co_direction": "opposite"} + params = {"base_query": "Q1", "target_query": "Q2", "period": "day", "measure": "efficiency", + "driver_measure": "unplanned_min", "lag": 0, "basis": "change", "target": _WEEKS[-1], + "label": "产线效率"} + return table, values, params, ["day", "efficiency"] + + _SHAPES = {"decompose": _decompose, "baseline": _baseline, "trend": _trend, "differ": _differ, - "comparable": _comparable} + "comparable": _comparable, "reconcile": _reconcile, "cochange": _cochange} _CRITERIA = {"decompose": "一段占总变化一半以上算主要来源", "baseline": "本期和过去 12 周的中位数比，差不到 2 倍稳健标准差算常规波动", "trend": "Mann–Kendall 检验的双侧 p 值小于 0.05 时按 Theil–Sen 斜率的正负判上升或下降", "differ": "双侧 p 值小于 0.05 算差异站得住", - "comparable": "只差日期、窗口一样长、是整期、星期构成相同才算可比"} + "comparable": "只差日期、窗口一样长、是整期、星期构成相同才算可比", + "reconcile": "两侧总量及各对齐切片相对偏差均不超过 1.0% 且无缺失项算对账一致", + "cochange": "主指标与关联指标在对齐期次上同按各自历史中位数 ± 2.0 倍稳健标准差检验是否同步超出常规波动"} def fake_result(op: str = "decompose", mode: str | None = "additive", verdict: str | None = "driver", *, @@ -121,7 +170,7 @@ def fake_result(op: str = "decompose", mode: str | None = "additive", verdict: s elif op == "differ" and mode not in ("rate", "mean", "rows"): mode = "rate" table, values, params, columns = _SHAPES.get(op, _decompose)() - if op == "comparable": + if op in ("comparable", "reconcile", "cochange"): default_inputs = [{"ref": ref, "artifact": "0" * 64, "role": role, "columns": columns, "rows": "all"} for ref, role in (("Q4", "base"), ("Q5", "target"))] else: diff --git a/backend/app/api/evals.py b/backend/app/api/evals.py index aacfb88a..c3fba0a8 100644 --- a/backend/app/api/evals.py +++ b/backend/app/api/evals.py @@ -855,8 +855,8 @@ EVALUATION_NOT_FOUND = "eval_evaluation_not_found" PAIR_NOT_FOUND = "eval_pair_not_found" RESULT_NOT_FOUND = "eval_result_not_found" CLEANED = "eval_details_cleaned" -#: 预设对照：目录贡献（当前目录 vs 关，可加只用已核实的项）、分析工具贡献（开 vs 关） -PRESETS = ("catalog", "catalog_reviewed", "analysis_tools") +#: 预设对照：目录贡献（当前目录 vs 关，可加只用已核实的项）、分析工具贡献（开 vs 关）、本体穿透贡献（开 vs 关） +PRESETS = ("catalog", "catalog_reviewed", "analysis_tools", "ontology_tools") def _refused(e: CaseInvalid) -> CodedHTTPException: @@ -865,8 +865,8 @@ def _refused(e: CaseInvalid) -> CodedHTTPException: async def _arms(session: AsyncSession, suite: EvalSuite, body: dict[str, Any]) -> list[dict[str, Any]]: """请求体 → 规范化的几臂变体：{variant}（一臂，不写用题集的缺省变体）、{arms: [变体…]}（A/B）或 {preset: catalog | - catalog_reviewed | analysis_tools}（目录贡献对照：当前目录 vs 关，可加只用已核实的项；分析工具贡献对照：开 vs 关；别的项 - 照 variant 或题集的缺省）。A/B 缺省每题 3 次。""" + catalog_reviewed | analysis_tools | ontology_tools}（目录贡献对照：当前目录 vs 关，可加只用已核实的项；分析工具贡献对照： + 开 vs 关；本体穿透贡献对照：开 vs 关；别的项照 variant 或题集的缺省）。A/B 缺省每题 3 次。""" from app.eval import variants as variant_rules judge = bool((await _site(session))["judge"]) @@ -880,6 +880,8 @@ async def _arms(session: AsyncSession, suite: EvalSuite, body: dict[str, Any]) - judge_default=judge, repeats_default=variant_rules.PAIR_REPEATS) if preset == "analysis_tools": return variant_rules.analysis_tools_preset(base) + if preset == "ontology_tools": + return variant_rules.ontology_tools_preset(base) return variant_rules.catalog_preset(base, reviewed=preset == "catalog_reviewed") if raw_arms is not None: if not isinstance(raw_arms, list) or not 2 <= len(raw_arms) <= variant_rules.ARMS_MAX: diff --git a/backend/app/api/evidence.py b/backend/app/api/evidence.py index eb5677fa..1c655746 100644 --- a/backend/app/api/evidence.py +++ b/backend/app/api/evidence.py @@ -72,6 +72,7 @@ from sqlalchemy import select, update from sqlalchemy.exc import StatementError from app.analysis.contract import CATALOG_PREFIX, LABELS as ANALYSIS_LABELS, LEDGER_KIND, is_result, method_label +from app.data.source_health import LEDGER_KIND as HEALTH_KIND, is_health_result from app.api.coded import CodedHTTPException from app.core.artifact_store import load from app.core.events import EventType @@ -411,9 +412,25 @@ def _graph_of(report: _Report, sealed: _Sealed, inputs: dict[str, Any]) -> tuple # 输入快照的 Q…，下钻的上一层 A…），图上看得出 A3 是从 Q2 算出来的 item.update({k: entry[k] for k in ("op", "mode", "status", "verdict", "item", "call_id") if entry.get(k) is not None}) + targets = [t for t in dict.fromkeys(_catalog_alias_of(catalog, a) for a in entry.get("inputs") or []) + if t and t != alias] + input_sources = { + str(catalog[t].get("source")) + for t in targets + if isinstance(catalog.get(t), dict) and catalog[t].get("source") + } + if len(input_sources) > 1 or entry.get("op") in ("reconcile", "cochange"): + item["cross_source"] = True edges.extend({"from": alias, "to": target, "rel": "derived_from", "report": report.node_id} - for target in dict.fromkeys(_catalog_alias_of(catalog, a) for a in entry.get("inputs") or []) - if target and target != alias) + for target in targets) + elif entry.get("kind") == HEALTH_KIND: + item.update({k: entry[k] for k in ("verdict", "sources", "issue_count", "call_id") + if entry.get(k) is not None}) + checked_sources = {str(s) for s in entry.get("sources") or [] if s} + edges.extend({"from": alias, "to": q_alias, "rel": "health_of", "report": report.node_id} + for q_alias, q_entry in catalog.items() + if isinstance(q_entry, dict) and q_entry.get("kind") == "query" + and str(q_entry.get("source") or "") in checked_sources) elif entry.get("kind") in ENTITY_KINDS: # 表和字段：叫什么、出现在哪几次查询里、来历（表结构快照 / SQL / 结果列）各自封存了没有 item.update({k: entry[k] for k in ("name", "table", "tables", "qualified", "source") if entry.get(k)}) @@ -845,6 +862,8 @@ async def _chain(seg: dict[str, Any], doc: dict[str, Any], sealed: _Sealed, if cite.get("kind") == LEDGER_KIND or (cite.get("kind") == "cell" and entry.get("kind") == LEDGER_KIND): # 分析结论：判定语、具名标量、结果表的一格都接同一条链（判定 → 方法 → 计算 → 输入 → 复算 → 封存） return await _analysis_chain(seg, cite, entry, doc, sealed, masks) + if cite.get("kind") == HEALTH_KIND: + return _health_chain(seg, cite, entry, sealed) if cite.get("kind") == "cell": locator = cite.get("locator") or {} # 只有直接引用的单元格才可能推断来源（P4-SPEC 2.8.2）。指标链里的查询步骤不带：指标片段的推断来源只会 @@ -890,10 +909,13 @@ _OP_SETTINGS: dict[str, tuple[str, ...]] = { "trend": ("trend_min_points", "trend_alpha", "min_history", "max_history"), "differ": ("differ_alpha", "min_n", "min_expected"), "comparable": (), + "reconcile": ("reconcile_rel_tol", "reconcile_abs_tol"), + "cochange": ("k_warn", "k_alert", "min_history", "max_history", "cochange_max_lag"), } #: 输入的角色（analysis.contract.AnalysisInput.role）给人看的叫法；比率拆解的 headline 是整体比率，不是总数 _INPUT_ROLES = {"data": "数据", "base": "基期", "target": "本期", "headline_base": "基期总数", - "headline_target": "本期总数", "parent": "上一层拆解"} + "headline_target": "本期总数", "parent": "上一层拆解", + "primary": "主侧数据", "counterpart": "对侧数据", "driver": "上游驱动"} _RATIO_ROLES = {"headline_base": "基期整体比率", "headline_target": "本期整体比率"} #: 闭合核对：headline 的那一格对着结论里的哪个数（细案 01 §2.1、§2.2）。可加拆解对各段之和（base_total / target_total）， #: 比率拆解对整体比率，即各段分子之和 ÷ 分母之和（r_base_total / r_target_total） @@ -911,6 +933,64 @@ def _analysis_of(artifact: str) -> tuple[dict[str, Any] | None, bool | None]: return (content, ok) if ok is True and is_result(content) else (None, False if ok is False else None) +def _health_of(artifact: str) -> tuple[dict[str, Any] | None, bool | None]: + """取回一份数据源健康结论：(结论, hash_ok)。""" + content, ok = _fetch(artifact) if artifact else (None, None) + return (content, ok) if ok is True and is_health_result(content) else (None, False if ok is False else None) + + +def _health_chain(seg: dict[str, Any], cite: dict[str, Any], entry: dict[str, Any], + sealed: _Sealed) -> list[dict[str, Any]]: + """数据源健康结论的出处链：健康核查判定 → 封存。""" + alias = str(cite.get("alias") or entry.get("alias") or "") + artifact = str(entry.get("artifact") or "") + result, hash_ok = _health_of(artifact) + info = sealed.source_healths.get(artifact) + in_ledger = sealed.trusted and (str(entry.get("node_id") or ""), artifact) in sealed.ledger + eid = make_eid(HEALTH_KIND, artifact or None, {}) + phrase = result.get("phrase") if isinstance(result, dict) else None + rendered = None + if isinstance(phrase, str): + with contextlib.suppress(RenderError): + rendered = render_cell(phrase, kind="text") + judged: dict[str, Any] = { + "step": HEALTH_KIND, + "alias": alias, + "node_id": entry.get("node_id"), + "label": _node_label(sealed, entry.get("node_id")) or None, + **({k: entry[k] for k in ("call_id",) if entry.get(k)}), + "artifact": artifact or None, + "eid": entry.get("eid"), + "eid_ok": entry.get("eid") == eid, + "rendered": seg.get("text"), + "render_ok": rendered is not None and rendered == seg.get("text"), + "hash_ok": hash_ok, + } + seal = { + "step": "health_seal", + "alias": alias, + "node_id": entry.get("node_id"), + "artifact": artifact or None, + "ledger": info is not None, + "hash_ok": hash_ok, + "eid_ok": judged["eid_ok"], + "sealed": in_ledger and hash_ok is True and judged["eid_ok"], + } + if info is None: + seal["note"] = "这份数据源健康核查不属于封存范围内的任何事件，不能作为证据展示" + if result is None: + judged["note"] = "数据源健康核查与哈希不一致，疑似被修改" if hash_ok is False else "无法读取数据源健康核查" + return [judged, seal] + judged.update( + verdict=result.get("verdict"), + phrase=result.get("phrase"), + sources=list(result.get("sources") or []), + issues=list(result.get("issues") or []), + tables=list(result.get("tables") or []), + ) + return [judged, seal] + + def _catalog_alias_of(catalog: dict[str, Any], artifact: Any) -> str | None: """这份工件在报告目录里的编号：查询快照是 Q…，分析结论是 A…。""" if not artifact: @@ -3202,7 +3282,7 @@ async def evidence_provenance(run_id: str, segment_id: str, report: str | None = def _export_cite_key(seg: dict[str, Any]) -> tuple | None: """数字 / 取值 / 判定语片段的引用键：同一处引用（同一个指标、同一格、同一条分析结论的同一项、同样的换算）共用一条 出处链。别的片段（表名、引文）的链跟着片段走，不归并。具名标量的键另接在后面：没有它的引用键和以前一模一样。""" - if seg.get("kind") not in ("number", "value", LEDGER_KIND): + if seg.get("kind") not in ("number", "value", LEDGER_KIND, HEALTH_KIND): return None cite = seg.get("cite") if not isinstance(cite, dict): @@ -3996,7 +4076,7 @@ def _scope_catalog(sealed: _Sealed, report_node: str) -> dict[str, Any]: from app.engine.nodes.report import report_catalog from app.engine.schema import GraphSpec - entries = [e for e in sealed.entries if e.get("kind") != LEDGER_KIND] + entries = [e for e in sealed.entries if e.get("kind") not in (LEDGER_KIND, HEALTH_KIND)] try: spec = GraphSpec.model_validate(sealed.graph) node = spec.node_map().get(report_node) @@ -4088,6 +4168,8 @@ def _note(seg: dict[str, Any], unit: dict[str, Any]) -> str: "仅核对它存在且本次用到，不核对其业务含义") if seg.get("state") == "deterministic" and seg.get("kind") == "quote": return "这段原话在知识库检索命中的片段中逐字出现（空白归一化后比对），已由系统核对" + if seg.get("state") == "deterministic" and cite.get("kind") == HEALTH_KIND: + return "判定语由系统按数据源健康核查规则算出、按固定模板写成，未经模型转写；各项核查与封存见出处链" if seg.get("state") == "deterministic" and cite.get("kind") == LEDGER_KIND: if (cite.get("locator") or {}).get("key"): return "这个数取自分析结论的具名的数，由系统按固定规则显示，未经模型转写" @@ -4666,7 +4748,7 @@ SEAL_NONE = "未封存：运行尚未结束，或正停在人工审批" _CSV_LABELS: dict[str, dict[Any, str]] = { "group": dict(GROUPS), "kind": {"number": "数字", "value": "值", "entity": "表名、字段名", "quote": "引文", "claim": "结论句", - "violation": "违规", LEDGER_KIND: "分析判定"}, + "violation": "违规", LEDGER_KIND: "分析判定", HEALTH_KIND: "数据源健康"}, "state": {"deterministic": "有出处", "none": "无证据", "candidate": "候选"}, "issue": {"uncited_number": "没有出处的数字", "unresolved_ref": "引用无法解析", "unknown_entity": "疑似不存在的名称", "unverified_entity": "无法核实", "uncited_claim": "结论句未附依据", "no_candidate": "未找到候选", @@ -4676,7 +4758,7 @@ _CSV_LABELS: dict[str, dict[Any, str]] = { "causal_unsupported": "因果说法没有依据", **{code: "文档不一致" for code in _INTEGRITY}}, "evidence_kind": {"metric": "口径卡指标", "cell": "查询单元格", "input": "运行输入", "table": "表", "column": "字段", - "quote": "知识库原话", "query": "查询结果", LEDGER_KIND: "分析结论"}, + "quote": "知识库原话", "query": "查询结果", LEDGER_KIND: "分析结论", HEALTH_KIND: "数据源健康"}, "sealed": {True: "是", False: "否", None: "—"}, } #: 电子表格会把以这些字符开头的格子当公式算：报告、答案里的字来自模型，照原样写进去就能被人利用 diff --git a/backend/app/api/ontology.py b/backend/app/api/ontology.py new file mode 100644 index 00000000..13cdf3f5 --- /dev/null +++ b/backend/app/api/ontology.py @@ -0,0 +1,354 @@ +"""标准业务实体与指标本体图谱接口：/api/ontology。 + +- GET /api/ontology/entities 列出标准业务实体（可按数据源过滤） +- POST /api/ontology/entities 新建标准业务实体 +- GET /api/ontology/entities/{entity_id} 读取单个标准业务实体 +- PATCH /api/ontology/entities/{entity_id} 更新标准业务实体（支持 version 乐观锁） +- DELETE /api/ontology/entities/{entity_id} 删除标准业务实体 +- POST /api/ontology/entities/{entity_id}/review 审阅单条跨源表列绑定或口径参数映射（确认 / 驳回） +- GET /api/ontology/graph 汇总已发布口径的指标关系网、反向表血缘与标准实体跨源桥 +- GET /api/ontology/workflows/{workflow_id} 读取单个工作流（当前草稿或指定版本）的指标关系与反向表血缘 +""" +from __future__ import annotations + +from typing import Any, Literal + +from fastapi import APIRouter, BackgroundTasks, Body, Depends, File, Form, Header, HTTPException, Query, UploadFile +from fastapi.responses import StreamingResponse +from pydantic import BaseModel, Field +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.coded import CATALOG_DRAFT_MODEL_UNAVAILABLE, CodedHTTPException +from app.api.deps import actor_of +from app.data import catalog, entity_assistant, entity_discovery, entity_docs, entity_patch +from app.data.entities import ( + EntityConflict, + EntityInvalid, + EntityNotFound, + create_entity, + delete_entity, + entity_to_dict, + get_entity, + list_entities, + review_entity_item, + update_entity, +) +from app.db.base import get_session +from app.db.models import Workflow, WorkflowVersion +from app.engine.schema import GraphSpec +from app.ledger.ontology import build_published_ontology_graph, ontology_publish_issues, workflow_ontology_index + +router = APIRouter(prefix="/api/ontology", tags=["ontology"]) + + +class EntityReviewIn(BaseModel): + target_kind: Literal["binding", "caliber_param"] + item_id: str + status: Literal["proposed", "verified", "confirmed", "rejected"] + reason: str = "" + version: int | None = None + + +class EntityAssistantMessageIn(BaseModel): + role: Literal["user", "assistant"] + content: str = Field(max_length=entity_assistant.MESSAGE_MAX) + + +class EntityAssistantIn(BaseModel): + messages: list[EntityAssistantMessageIn] = Field(default_factory=list, max_length=entity_assistant.MAX_MESSAGES) + source_ids: list[str] | None = Field(default=None, max_length=50) + focus_entity_key: str | None = None + attachment_ids: list[str] = Field(default_factory=list, max_length=20) + document_ids: list[str] = Field(default_factory=list, max_length=30) + pending_ops: list[dict[str, Any]] = Field(default_factory=list) + + +class EntityApplyPatchIn(BaseModel): + operations: list[dict[str, Any]] = Field(min_length=1, max_length=200) + expected_versions: dict[str, int] | None = None + + +class EntityDiscoveryJobIn(BaseModel): + source_ids: list[str] | None = None + document_ids: list[str] | None = None + attachment_ids: list[str] | None = None + note: str | None = None + + +@router.get("/entities") +async def api_list_entities( + source_name: str | None = Query(default=None), + only_active: bool = Query(default=False), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + rows = await list_entities(session, source_name=source_name, only_active_bindings=only_active) + return {"items": [entity_to_dict(r) for r in rows]} + + +@router.post("/entities", status_code=201) +async def api_create_entity( + payload: dict[str, Any] = Body(...), + x_actor: str | None = Header(default=None, alias="X-Actor"), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + try: + row = await create_entity(session, payload, actor=actor_of(x_actor)) + await session.commit() + except EntityInvalid as exc: + raise HTTPException(status_code=422, detail={"message": exc.message, "field": exc.field}) from None + except EntityConflict as exc: + raise HTTPException(status_code=409, detail={"message": exc.message, "field": exc.field}) from None + return entity_to_dict(row) + + +@router.post("/entities/attachments", status_code=201) +async def api_upload_entity_attachment( + background: BackgroundTasks, + file: UploadFile = File(...), + save_to_kb: bool = Form(default=False), + collection: str = Form(default="default"), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + """解析业务文档或主数据编码对照表附件，支持勾选同步存入知识库复用。""" + try: + raw_bytes = await entity_docs.read_upload_limited(file) + return await entity_docs.ingest_entity_attachment( + session, + raw_bytes=raw_bytes, + filename=file.filename or "上传文档", + mime=file.content_type or "text/plain", + save_to_kb=save_to_kb, + collection=collection, + background=background, + ) + except entity_docs.AttachmentTooLarge as exc: + raise HTTPException(status_code=413, detail=str(exc)) from None + except entity_docs.AttachmentInvalid as exc: + raise HTTPException(status_code=415, detail=str(exc)) from None + + +@router.post("/entities/assistant") +async def api_entity_assistant( + payload: EntityAssistantIn, + session: AsyncSession = Depends(get_session), +) -> Any: + """交互式实体助手（SSE）：跨数据源与业务文档对话对齐业务实体，只产出提案不直接写库。""" + messages = [m.model_dump() for m in payload.messages] + if problem := entity_assistant.conversation_problem(messages): + raise HTTPException(status_code=422, detail=problem) + try: + model, model_id = await entity_assistant.resolve_model(session) + except catalog.CatalogModelUnavailable as exc: + raise CodedHTTPException(409, str(exc), CATALOG_DRAFT_MODEL_UNAVAILABLE) from exc + ctx = await entity_assistant.prepare( + session, + source_ids=payload.source_ids, + focus_entity_key=payload.focus_entity_key, + attachment_ids=payload.attachment_ids, + document_ids=payload.document_ids, + pending_ops=payload.pending_ops, + ) + return StreamingResponse( + entity_assistant.stream(model, model_id, ctx, messages), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + +@router.post("/entities/apply-patch") +async def api_apply_entity_patch( + body: EntityApplyPatchIn, + x_actor: str | None = Header(default=None, alias="X-Actor"), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + """批量应用人工在实体助手审阅确认的修改提案（校验乐观锁版本）。""" + try: + result = await entity_patch.apply_entity_patch( + session, + body.operations, + mode="confirmed", + actor=actor_of(x_actor), + expected_versions=body.expected_versions, + ) + await session.commit() + return result + except EntityInvalid as exc: + raise HTTPException(status_code=422, detail={"message": exc.message, "field": exc.field}) from None + except EntityConflict as exc: + raise HTTPException(status_code=409, detail={"message": exc.message, "field": exc.field}) from None + + +@router.post("/discovery-jobs", status_code=202) +async def api_create_discovery_job( + body: EntityDiscoveryJobIn, + x_actor: str | None = Header(default=None, alias="X-Actor"), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + """创建后台扫描发现候选标准业务实体的任务（结果一律落为 proposed）。""" + try: + job = await entity_discovery.create_job( + session, + source_ids=body.source_ids, + document_ids=body.document_ids, + attachment_ids=body.attachment_ids, + note=body.note, + actor=actor_of(x_actor), + ) + return entity_discovery.job_to_dict(job) + except catalog.CatalogModelUnavailable as exc: + raise CodedHTTPException(409, str(exc), CATALOG_DRAFT_MODEL_UNAVAILABLE) from exc + except entity_discovery.DiscoveryBusy as exc: + raise CodedHTTPException(409, str(exc), entity_discovery.DISCOVERY_BUSY_CODE) from exc + except (entity_discovery.DiscoveryEmptyScope, ValueError) as exc: + raise HTTPException(status_code=422, detail=str(exc)) from None + + +@router.get("/discovery-jobs/active") +async def api_get_active_discovery_job( + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + return await entity_discovery.get_active_or_latest_job(session) + + +@router.get("/discovery-jobs/{job_id}") +async def api_get_discovery_job( + job_id: str, + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + row = await entity_discovery.get_job(session, job_id) + if row is None: + raise HTTPException(status_code=404, detail="候选实体扫描任务不存在") + return entity_discovery.job_to_dict(row) + + +@router.post("/discovery-jobs/{job_id}/cancel") +async def api_cancel_discovery_job( + job_id: str, + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + try: + row = await entity_discovery.cancel_job(session, job_id) + return entity_discovery.job_to_dict(row) + except LookupError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from None + except entity_discovery.DiscoveryFinished as exc: + raise CodedHTTPException(409, str(exc), entity_discovery.DISCOVERY_FINISHED_CODE) from exc + + +@router.get("/entities/{entity_id}") +async def api_get_entity( + entity_id: str, + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + row = await get_entity(session, entity_id) + if row is None: + raise HTTPException(status_code=404, detail="标准业务实体不存在") + return entity_to_dict(row) + + +@router.patch("/entities/{entity_id}") +async def api_update_entity( + entity_id: str, + payload: dict[str, Any] = Body(...), + x_actor: str | None = Header(default=None, alias="X-Actor"), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + try: + row = await update_entity(session, entity_id, payload, actor=actor_of(x_actor)) + await session.commit() + except EntityNotFound as exc: + raise HTTPException(status_code=404, detail=str(exc)) from None + except EntityInvalid as exc: + raise HTTPException(status_code=422, detail={"message": exc.message, "field": exc.field}) from None + except EntityConflict as exc: + raise HTTPException(status_code=409, detail={"message": exc.message, "field": exc.field}) from None + return entity_to_dict(row) + + +@router.post("/entities/{entity_id}/review") +async def api_review_entity_item( + entity_id: str, + body: EntityReviewIn, + x_actor: str | None = Header(default=None, alias="X-Actor"), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + try: + row = await review_entity_item( + session, + entity_id, + target_kind=body.target_kind, + item_id=body.item_id, + status=body.status, + reason=body.reason, + actor=actor_of(x_actor), + expected_version=body.version, + ) + await session.commit() + except EntityNotFound as exc: + raise HTTPException(status_code=404, detail=str(exc)) from None + except EntityInvalid as exc: + raise HTTPException(status_code=422, detail={"message": exc.message, "field": exc.field}) from None + except EntityConflict as exc: + raise HTTPException(status_code=409, detail={"message": exc.message, "field": exc.field}) from None + return entity_to_dict(row) + + +@router.delete("/entities/{entity_id}") +async def api_delete_entity( + entity_id: str, + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + deleted = await delete_entity(session, entity_id) + if not deleted: + raise HTTPException(status_code=404, detail="标准业务实体不存在") + await session.commit() + return {"ok": True} + + +@router.get("/graph") +async def api_get_ontology_graph( + include_proposed: bool = Query(default=True), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + return await build_published_ontology_graph(session, include_proposed=include_proposed) + + +@router.get("/workflows/{workflow_id}") +async def api_get_workflow_ontology( + workflow_id: str, + version: int | None = Query(default=None), + session: AsyncSession = Depends(get_session), +) -> dict[str, Any]: + workflow = await session.get(Workflow, workflow_id) + if workflow is None: + raise HTTPException(status_code=404, detail="工作流不存在") + if version is not None: + snap = ( + await session.execute( + select(WorkflowVersion).where( + WorkflowVersion.workflow_id == workflow_id, + WorkflowVersion.version == version, + ) + ) + ).scalar_one_or_none() + if snap is None: + raise HTTPException(status_code=404, detail=f"工作流不存在 v{version} 版本") + graph = snap.graph or {} + ver = int(snap.version) + else: + graph = workflow.graph or {} + ver = int(workflow.version or 1) + index = workflow_ontology_index( + workflow_id=workflow.id, + version=ver, + graph=graph, + workflow_name=workflow.name or workflow.id, + ) + issues: list[dict[str, Any]] = [] + try: + spec = GraphSpec.model_validate(graph) + issues = await ontology_publish_issues(session, spec, workflow_id=workflow.id) + except Exception: # noqa: BLE001 - 草稿图结构未成形时照常返回可提取的指标清单 + issues = [] + return {**index, "issues": issues} diff --git a/backend/app/api/runs.py b/backend/app/api/runs.py index 29bb9805..4712bc8d 100644 --- a/backend/app/api/runs.py +++ b/backend/app/api/runs.py @@ -102,10 +102,16 @@ class _Projection: return event node_id = str(event.get("node_id") or "") failure = str(event.get("type") or "") in FAILURE_EVENTS - if node_id in self.nodes or str(data.get("tool") or "") == CALIBER_QUERY \ + tool_name = str(data.get("tool") or "") + if node_id in self.nodes or tool_name in (CALIBER_QUERY, "ontology_neighbors", "source_health") \ or str(data.get("node_id") or "") in self.nodes \ or (failure and safe_failure(data.get("error")) is not None): safe = {k: v for k, v in data.items() if k in SAFE_EVENT_KEYS} + if isinstance(data.get("health_artifact"), Mapping): + safe["health_artifact"] = { + k: v for k, v in data["health_artifact"].items() + if k in ("ref", "verdict", "phrase") + } result = result_summary(data.get("artifact")) or safe_result(data.get("preview")) if result is not None: safe["result"] = result @@ -138,9 +144,12 @@ def _caliber_nodes(graph: Any) -> set[str]: def _touches_caliber(config: Mapping[str, Any]) -> bool: - """这个节点的配置里有没有口径的身份：绑了口径查询，或者是一张转抄卡。""" - bound = config.get("tools") if isinstance(config.get("tools"), list) else [] - if config.get("tool") == CALIBER_QUERY or CALIBER_QUERY in [t for t in bound if isinstance(t, str)]: + """这个节点的配置里有没有口径的身份：绑了口径查询/本体邻居，或者是一张转抄卡。""" + bound = [t for t in (config.get("tools") if isinstance(config.get("tools"), list) else []) if isinstance(t, str)] + caliber_tools = {CALIBER_QUERY, "ontology_neighbors"} + if str(config.get("tool") or "") in caliber_tools or any(t in caliber_tools for t in bound): + return True + if config.get("tool") == "source_health" and isinstance(config.get("args"), Mapping) and config["args"].get("workflow"): return True return config.get("caliber_from") not in (None, "", {}) @@ -151,10 +160,13 @@ def _safe_config(config: Mapping[str, Any]) -> dict[str, Any]: 别的键（节点自己的说明、inputs 的值、工具清单）一个字不动——「使用者的图长什么样」照旧看得见，看不见的只是 「调的是哪一份工作流的哪一版」。""" out = dict(config) - if out.get("tool") == CALIBER_QUERY and "args" in out: + caliber_tools = {CALIBER_QUERY, "ontology_neighbors"} + if (str(out.get("tool") or "") in caliber_tools or ( + out.get("tool") == "source_health" and isinstance(out.get("args"), Mapping) and out["args"].get("workflow") + )) and "args" in out: out["args"] = dict(CALIBER_ARGS_NOTE) - bound = out.get("tools") if isinstance(out.get("tools"), list) else [] - if out.get("tool") == CALIBER_QUERY or CALIBER_QUERY in [t for t in bound if isinstance(t, str)]: + bound = [t for t in (out.get("tools") if isinstance(out.get("tools"), list) else []) if isinstance(t, str)] + if str(out.get("tool") or "") in caliber_tools or any(t in caliber_tools for t in bound): for key in ("prompt", "instructions"): if key in out: out[key] = safe_call_text(out[key]) diff --git a/backend/app/api/settings.py b/backend/app/api/settings.py index eca23947..601d5b4b 100644 --- a/backend/app/api/settings.py +++ b/backend/app/api/settings.py @@ -988,6 +988,8 @@ DEFAULT_SETTINGS: dict[str, Any] = { "agent_max_steps": 100, "agent_budget_tokens": 2_000_000, "agent_budget_usd": None, + # 问数简单查询撞上硬异常信号时的处置策略（P1）：prompt（默认，正常交付并提示一键本体排查）/ auto（自动升级最多 1 轮） + "anomaly_escalation_mode": "prompt", }, "limits": { "max_graph_steps": app_settings.max_graph_steps, @@ -1018,6 +1020,9 @@ DEFAULT_SETTINGS: dict[str, Any] = { } +ANOMALY_ESCALATION_MODES = ("prompt", "auto") + + async def run_defaults(session: AsyncSession) -> dict[str, Any]: """设置里 run 这一组，没存过的键补内置默认。 @@ -1028,6 +1033,12 @@ async def run_defaults(session: AsyncSession) -> dict[str, Any]: return {**DEFAULT_SETTINGS["run"], **((row.value if row else None) or {})} +async def anomaly_escalation_mode(session: AsyncSession) -> str: + """问数简单查数命中硬异常时的本体排查升级策略：默认 prompt，明确设为 auto 时自动发起最多 1 轮诊断升级。""" + raw = (await run_defaults(session)).get("anomaly_escalation_mode") + return str(raw) if raw in ANOMALY_ESCALATION_MODES else "prompt" + + def tool_approval_default(run: dict[str, Any]) -> str: """「危险工具默认需要人工确认」换算成节点 approval 的取值，节点自己配了的不看它。 @@ -1163,6 +1174,14 @@ async def put_settings( from app.tools.trust import SETTING_KEY as TRUST_KEY # 先查后写：一组里有一项不对就整个请求不落库，免得存下半组 + if "run" in payload.values and isinstance(payload.values["run"], dict): + if "anomaly_escalation_mode" in payload.values["run"]: + mode = payload.values["run"]["anomaly_escalation_mode"] + if mode is not None and mode not in ANOMALY_ESCALATION_MODES: + raise HTTPException( + 422, + f"异常升级策略（run.anomaly_escalation_mode）只能是 {' 或 '.join(ANOMALY_ESCALATION_MODES)}，当前为「{mode}」", + ) if JUDGE_GROUP in payload.values and (problem := judge_problem(payload.values[JUDGE_GROUP])): raise HTTPException(422, problem) if VERIFY_GROUP in payload.values and (problem := verify_problem(payload.values[VERIFY_GROUP])): diff --git a/backend/app/apps/publications.py b/backend/app/apps/publications.py index 57e94a6a..96b9ba0b 100644 --- a/backend/app/apps/publications.py +++ b/backend/app/apps/publications.py @@ -690,9 +690,10 @@ def problem_key(problem: Mapping[str, Any]) -> tuple[str, str, tuple[tuple[str, def build_content(*, workflow_id: str, version: int, level: str, graph: Any, cards: Sequence[CardBasis]) -> dict[str, Any]: """这一版的清单内容：**只由原始输入决定**（这一版不可变的图 + 每张卡引用的那两行记录的值）。""" from app.core.artifact_store import execution_hash, graph_hash + from app.ledger.ontology import ontology_manifest_section sources, tables = fetch_dependencies(graph) - return { + out: dict[str, Any] = { "format": FORMAT, "workflow_id": str(workflow_id), "version": int(version), @@ -704,6 +705,10 @@ def build_content(*, workflow_id: str, version: int, level: str, graph: Any, car "parameters": parameters_of(graph), "dependencies": {"sources": sources, "tables": tables, "upstream": upstream_pins(graph)}, } + ontology = ontology_manifest_section(workflow_id=str(workflow_id), version=int(version), graph=graph or {}) + if ontology is not None: + out["ontology"] = ontology + return out def content_hash(content: Mapping[str, Any]) -> str: diff --git a/backend/app/apps/validation.py b/backend/app/apps/validation.py index 867d200b..aebb8129 100644 --- a/backend/app/apps/validation.py +++ b/backend/app/apps/validation.py @@ -6,12 +6,15 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.engine.schema import GraphSpec -async def gate_issues(session: AsyncSession, graph: Any, level: str) -> list[dict[str, Any]] | None: +async def gate_issues( + session: AsyncSession, graph: Any, level: str, *, workflow_id: str | None = None, +) -> list[dict[str, Any]] | None: from app.data.catalog_impact import open_structure_issues from app.data.sqlcheck import graph_sources, load_checkers_by_name from app.engine.governance import publish_issues from app.engine.lint_process import load_subgraphs, with_subgraphs from app.engine.provider_refs import load_providers + from app.ledger.ontology import ontology_publish_issues try: spec = GraphSpec.model_validate(graph) @@ -21,4 +24,5 @@ async def gate_issues(session: AsyncSession, graph: Any, level: str) -> list[dic providers = await load_providers(session, spec) with with_subgraphs(await load_subgraphs(session, spec)): issues = [i.model_dump() for i in publish_issues(spec, level=level, checkers=checkers, providers=providers)] - return issues + await open_structure_issues(session, spec) + ontology_issues = await ontology_publish_issues(session, spec, workflow_id=str(workflow_id or "")) + return issues + ontology_issues + await open_structure_issues(session, spec) diff --git a/backend/app/auth/chat_policy.py b/backend/app/auth/chat_policy.py index 3101dfde..03384d09 100644 --- a/backend/app/auth/chat_policy.py +++ b/backend/app/auth/chat_policy.py @@ -41,16 +41,18 @@ from typing import Any from app.analysis.contract import TOOL_NAMES from app.ledger.querycore import CALIBER_QUERY +from app.tools.ontology import ONTOLOGY_TOOLS #: 问数据运行里 user 角色能用的节点类型（engine/schema.py 的 NodeType 取值） CHAT_NODE_TYPES = frozenset({"input", "llm", "agent", "report", "output", "retrieve", "transform", "branch", "metrics", "merge", "analysis", "tool"}) -#: Agent 能绑的只读内置工具（tools/builtin/）：五个分析工具也在内（contract.TOOL_NAMES，不查库），以及口径查询 -#: （querycore.CALIBER_QUERY，只读、按资产授权收窄——见文件开头）。工具名只从登记处取常量，不在这里另写一个字符串 +#: Agent 能绑的只读内置工具（tools/builtin/）：分析工具也在内（contract.TOOL_NAMES，不查库），以及口径查询 +#: （querycore.CALIBER_QUERY，只读、按资产授权收窄——见文件开头） +#: 和本体邻居/源健康探针（ONTOLOGY_TOOLS，只读、按授权与遮罩收窄）。工具名只从登记处取常量，不在这里另写一个字符串 CHAT_BUILTIN_TOOLS = frozenset({"calculator", "current_time", "json_query", "kb_search", "memory_recall", - CALIBER_QUERY, *TOOL_NAMES.values()}) + CALIBER_QUERY, *ONTOLOGY_TOOLS, *TOOL_NAMES.values()}) #: 上面那几个内置工具给人看的说法（拒绝时的报错、提示词里都写它） -CHAT_BUILTIN_LABEL = "计算、当前时间、JSON 取值、知识库检索、记忆读取、分析、口径查询" +CHAT_BUILTIN_LABEL = "计算、当前时间、JSON 取值、知识库检索、记忆读取、分析、口径查询、本体邻居、数据源健康" #: 运行时改记忆范围、知识库的配置键（engine/nodes 的 ctx.cfg("memory_scope") / ctx.cfg("collection")，图级 defaults 同样生效） SCOPE_KEYS = ("memory_scope", "collection") SCOPE_REFUSED = "没有权限：使用者在问数据中不能指定记忆范围和知识库，使用的是管理员设好的默认值" diff --git a/backend/app/bundle/build.py b/backend/app/bundle/build.py index ac8a10a9..643f980d 100644 --- a/backend/app/bundle/build.py +++ b/backend/app/bundle/build.py @@ -93,7 +93,8 @@ _SHOWN_EVENTS = frozenset({EventType.RUN_STARTED, EventType.REPORT_CHECKED, Even #: node.finished 带规范字符串的节点：报告（预览就是嵌进来的那份文档的正文，文档只给哈希的不带）、口径卡（指标集的 id #: 记在台账里；卡上有值不给的指标、有输入取自遮罩列的不带，见 _card_shown）。成果节点在运行里没有带遮罩的快照时带 #: 目录条目的种类 → 嵌进来的工件种类。运行输入、检索、表和字段的条目不嵌：前者没有工件，后几样不在核对范围里 -_CITED_KINDS = {"metric": "metric_set", "query": "query", LEDGER_KIND: ARTIFACT_KIND} +_CITED_KINDS = {"metric": "metric_set", "query": "query", LEDGER_KIND: ARTIFACT_KIND, + "source_health": "source_health"} #: 下钻的上一层结论最多跟几层（同分析设置里下钻层数的上限） _PARENT_DEPTH = 6 @@ -1229,7 +1230,10 @@ def _shows_content(etype: str, node_id: str | None, data: Any, node_types: Mappi return not store.masked(artifact) # 分析工具：预览里是结论的前一截（判定语、具名的数、结果表），和结论本身同一个认法 found = data.get("analysis_artifact") if isinstance(data, dict) else None - return isinstance(found, str) and bool(found) and _result_shown(found, store) + if isinstance(found, str) and found: + return _result_shown(found, store) + health = data.get("health_artifact") if isinstance(data, dict) else None + return isinstance(health, str) and bool(health) and store.raw(health) is not None if etype == EventType.ANALYSIS_END: # 分析卡里一项做完的事件：带着判定、判定语和结论工件（analysis_artifact），结论随文件附带得了（输入没碰遮罩列）才带 found = data.get("analysis_artifact") if isinstance(data, dict) else None diff --git a/backend/app/bundle/verifier.js b/backend/app/bundle/verifier.js index b89065fb..0fdd5adf 100644 --- a/backend/app/bundle/verifier.js +++ b/backend/app/bundle/verifier.js @@ -51,9 +51,11 @@ const RESULT = { pass: '通过', fail: '不通过', na: '无法核对', partial: '部分无法核对' } const MARK = { pass: '✓', fail: '✗', na: '?', partial: '?' } - const KIND_LABEL = { report_doc: '报告文档', metric_set: '口径卡', query: '查询快照', analysis_result: '分析结论' } + const KIND_LABEL = { report_doc: '报告文档', metric_set: '口径卡', query: '查询快照', analysis_result: '分析结论', + source_health: '数据源健康' } /** 分析结论（app/analysis）：工件种类、目录条目种类、分析范围里的几组 */ const ANALYSIS = 'analysis_result' + const SOURCE_HEALTH = 'source_health' const SCOPE_GROUPS = [['phenomena', '现象'], ['decomposed', '拆过'], ['excluded', '排除'], ['refused', '算不了']] const TIER_LABEL = { formal: '完整出具', degraded: '降档出具', withheld: '不予出具' } const CLASS_LABEL = { formal: '正式运行', exploratory: '探索运行' } @@ -174,6 +176,7 @@ srcAnalysis: (alias) => `分析结论 ${alias} 的判定语`, srcAnalysisValue: (alias, key) => `分析结论 ${alias} 的具名的数「${key}」`, srcAnalysisCell: (alias, row, column) => `分析结论 ${alias} 的结果表 · 第 ${row + 1} 行 · ${column}`, + srcHealth: (alias) => `数据源健康 ${alias} 的判定语`, srcNone: '没有出处', srcOther: '其他引用', numUncited: '该数字未使用引用标记，无法核对其来源', @@ -191,6 +194,7 @@ numNoCell: (why) => `快照里找不到这一格：${why}`, numCellDiffers: '引用的值与快照里那一格的值不一致', numNoPhrase: '分析结论里没有判定语', + numNoHealthPhrase: '数据源健康结论里没有判定语', numNoValue: (key) => `分析结论里没有名为「${key}」的数`, numValueDiffers: '引用的值与分析结论里记的值不一致', numNoTable: '分析结论里没有结果表', @@ -228,7 +232,7 @@ shownLookalike: '紧挨着长得像数字的字母或符号（l、I、O、上标数字、圈码这类），读起来可能多出一位', shownOdd: '紧挨着这个数的字不在常见写法里，无法确认读者看到的数', numNotShown: '正文里没有显示这一处：导出时不会出现这种情况，文件可能被改过', - numAria: (kind, text, result) => `${kind === 'value' ? '取值' : kind === 'analysis' ? '判定语' : '数字'} ${text}，${result}，查看出处`, + numAria: (kind, text, result) => `${kind === 'value' ? '取值' : (kind === 'analysis' || kind === SOURCE_HEALTH) ? '判定语' : '数字'} ${text}，${result}，查看出处`, // 数字详情里口径卡一节：指标、计算方式（原式、代入式）、输入、复算。遮罩未附带的卡没有这一节（原因见核对明细 ③） dMetric: '指标', dUnit: '单位', dExpression: '原式（计算方式）', dSubstituted: '代入式（代入输入后的算式）', dInputs: '输入', dRecompute: '复算', dSourceName: '数据源', dInputValue: '取值', dFrom: '取自', @@ -258,14 +262,14 @@ dResult: '核对结果', dSource: '出处', dNode: '来源节点', dCaliber: '口径', dCited: '引用的值', dCard: '口径卡里的值', dCell: '快照里的值', dShown: '正文显示', dShownSame: '与按显示规则渲染的结果一致', dExpected: '按显示规则应为', dArtifact: '所在工件', dReport: '所在报告', - dPhrase: '判定语', dCriterion: '判定标准', dMethod: '方法与版本', dAnalysis: '分析结论里的值', dRecheck: '复算', + dPhrase: '判定语', dCriterion: '判定标准', dMethod: '方法与版本', dAnalysis: '分析结论里的值', dHealth: '数据源健康核查里的值', dRecheck: '复算', dAnalysisInputs: '输入', dHypothesis: 'Agent 写的假设', dCardHypothesis: '分析卡上写的『在验证什么』', dCaveats: '附带说明', recheckBy: '导出时服务端按封存的输入复算的结论；本页只核哈希，不在浏览器里重算统计', recheckNone: '导出时没有复算（结论未随文件附带）', analysisInput: (role, alias) => `${role}${alias ? ` · ${alias}` : ''}`, // 分析结论输入的角色（analysis.contract.AnalysisInput.role），同证据面板的叫法 roleLabel: { data: '数据', base: '基期', target: '本期', headline_base: '基期总数', headline_target: '本期总数', - parent: '上一层拆解' }, + parent: '上一层拆解', primary: '主侧数据', counterpart: '对侧数据', driver: '上游驱动' }, dComparability: '两期可比性', comparabilityLabel: { comparable: '两期可比', conditional: '两期有条件可比', incomparable: '两期不可比' }, dJudged: '判定', dChecks: '各项检查', comparabilityNotAttached: '可比性结论未随文件附带（见核对明细 ③ ④），这里按口径卡的记录展示', scopeTitle: '分析范围', scopeNote: '这一段由系统按这次运行的分析结论写成，写作者改不了；点开编号看判定语、判定标准、输入和复算。', @@ -1313,7 +1317,7 @@ // 内容不随文件附带，浏览器算不出它对不对得上：导出时服务端看到自己的文件和 id 对不上（mismatch）、解析不出来 // （unparsable）的照实说，对不上的判不通过——不然被改过的文件看起来和正常的遮罩一样，篡改的证据就丢了。报告文档 // 只给哈希时页面上没有正文可核，总览不是绿色的通过（overallOf 按报告文档没核对上算） - if (a.kind !== 'query' && a.kind !== 'metric_set' && a.kind !== 'report_doc' && a.kind !== ANALYSIS) { + if (a.kind !== 'query' && a.kind !== 'metric_set' && a.kind !== 'report_doc' && a.kind !== ANALYSIS && a.kind !== SOURCE_HEALTH) { item.result = FAIL item.reason = T.maskedOdd } else if (a.mismatch) { @@ -1453,7 +1457,7 @@ out.docs.push({ report, art, link, doc, trusted: trusted.has(report) }) if (!doc) continue for (const [seg, path] of segmentsOf(doc)) { - if (seg.kind !== 'number' && seg.kind !== 'value' && seg.kind !== 'analysis') continue + if (seg.kind !== 'number' && seg.kind !== 'value' && seg.kind !== 'analysis' && seg.kind !== SOURCE_HEALTH) continue // 引用的值在文档 JSON 里的原文：超过 2^53 的整数照原文写进原因和出处面板 const citedToken = () => tokenAt(tokensOf(art), [...path, 'cite', 'value']) const check = trusted.has(report) ? checkNumber(seg, doc, arts, chained, citedToken) @@ -1487,6 +1491,7 @@ if (cite.kind === 'analysis') { return loc.key != null ? T.srcAnalysisValue(cite.alias, String(loc.key)) : T.srcAnalysis(cite.alias) } + if (cite.kind === SOURCE_HEALTH) return T.srcHealth(cite.alias) return T.srcOther } @@ -1507,14 +1512,14 @@ { fmt: 'plain', conv: convOf(cite) })) return shown.result === FAIL ? shown : { ...shown, result: NA, reason: T.numInput } } - if (cite.kind !== 'metric' && cite.kind !== 'cell' && cite.kind !== 'analysis') { + if (cite.kind !== 'metric' && cite.kind !== 'cell' && cite.kind !== 'analysis' && cite.kind !== SOURCE_HEALTH) { return { result: NA, reason: T.numOther, source } } const entry = own(doc.catalog, cite.alias) const info = { source, kind: cite.kind, cite } // 分析结论：判定语、具名的数（cite.kind analysis），和它结果表里的一格（cite.kind cell，目录条目是 analysis） const analysis = cite.kind === 'analysis' || (cite.kind === 'cell' && own(entry, 'kind') === 'analysis') - const wanted = cite.kind === 'metric' ? 'metric' : analysis ? 'analysis' : 'query' + const wanted = cite.kind === 'metric' ? 'metric' : analysis ? 'analysis' : cite.kind === SOURCE_HEALTH ? SOURCE_HEALTH : 'query' if (!isObj(entry) || entry.kind !== wanted || typeof entry.artifact !== 'string') { return { ...info, result: FAIL, reason: T.numNoEntry } } @@ -1530,6 +1535,13 @@ if (!chained.has(entry.artifact)) return { ...info, result: NA, reason: T.numNotChained } info.citedToken = citedToken() if (analysis) return checkAnalysis(info, seg, doc, art) + if (cite.kind === SOURCE_HEALTH) { + const phrase = isObj(art.json) ? art.json.phrase : null + info.health = art.json + if (typeof phrase !== 'string' || !phrase.trim()) return { ...info, result: FAIL, reason: T.numNoHealthPhrase } + info.truth = phrase + return checkShown(info, seg, () => renderCell(phrase, null, null, 'text', 1)) + } if (cite.kind === 'metric') { const metricId = own(cite.locator, 'metric') ?? own(entry.locator, 'metric') const metrics = Array.isArray(art.json.metrics) ? art.json.metrics : [] @@ -1670,7 +1682,7 @@ info.expected = expected const shown = String(seg.text ?? '') if (pyStrip(shown) === pyStrip(expected)) return { ...info, result: PASS, shownOk: true } - if (info.cite.kind === 'analysis' && !Object.prototype.hasOwnProperty.call(info.cite, 'value')) { + if ((info.cite.kind === 'analysis' || info.cite.kind === SOURCE_HEALTH) && !Object.prototype.hasOwnProperty.call(info.cite, 'value')) { return { ...info, result: FAIL, reason: T.numPhraseDiffers(shown, expected) } } const value = info.cite.value @@ -1976,7 +1988,7 @@ metric ? [T.dMetric, [metric.name || metric.id || '—', metric.id && metric.name ? h('span', { class: 'muted' }, `（${metric.id}）`) : null]] : null, metric && metric.unit ? [T.dUnit, String(metric.unit)] : null, cite && cite.status === 'resolved' && 'value' in cite ? [T.dCited, exactText(cite.value, number.citedToken)] : null, - 'truth' in number ? [number.kind === 'metric' ? T.dCard : number.analysis ? T.dAnalysis : T.dCell, + 'truth' in number ? [number.kind === 'metric' ? T.dCard : number.analysis ? T.dAnalysis : number.health ? T.dHealth : T.dCell, number.truthText ?? exactText(number.truth, number.truthToken)] : null, checked && !same ? [T.dExpected, number.expected] : null, // 分析范围里的编号不是正文里的字：没有「正文显示」可核 @@ -2332,7 +2344,7 @@ refs.length ? ` ← ${T.chainFrom}${refs.join('、')}` : ''] })) const values = out.numbers.filter((n) => n.seg.kind === 'value').length - const phrases = out.numbers.filter((n) => n.seg.kind === 'analysis').length + const phrases = out.numbers.filter((n) => n.seg.kind === 'analysis' || n.seg.kind === SOURCE_HEALTH).length root.append(checkBlock('numbers', T.checkNumbers, T.sumNumbers(out.numbers.length - values - phrases, values, phrases), out.numbers, (n) => h('button', { type: 'button', 'data-num-item': n.key, onclick: () => { if (n.button) n.button.scrollIntoView({ block: 'center' }) diff --git a/backend/app/capabilities/registry.py b/backend/app/capabilities/registry.py index 16788b21..6be3b713 100644 --- a/backend/app/capabilities/registry.py +++ b/backend/app/capabilities/registry.py @@ -304,12 +304,14 @@ async def capability_versions_for(session: AsyncSession, tool_names: list[str]) def is_platform_query(name: str) -> bool: """这个工具的结果能不能当查询快照作证（替换 nodes/tools.py、nodes/llm.py 里按名字前缀的判断，F11）。 - 认的是「平台执行」这个身份：数据源查询工具、K10 的口径查询工具（`caliber_query`，由 QueryCore 自己执行取数）， + 认的是「平台执行」这个身份：数据源查询工具、K10 的口径查询工具（`caliber_query`，由 QueryCore 自己执行取数）、 + 独立语义指标查询工具（`metric_query`，委托数据源查询工具落快照）， 和登记里声明了 tabular、绑定端点、由平台执行的读取。不看结果长什么样。""" from app.ledger.querycore import CALIBER_QUERY from app.tools.datasource import QUERY_PREFIX + from app.tools.metrics import METRIC_QUERY - return name.startswith(QUERY_PREFIX) or name == CALIBER_QUERY or name in _platform_queries + return name.startswith(QUERY_PREFIX) or name in (CALIBER_QUERY, METRIC_QUERY) or name in _platform_queries def query_items(payload: Any, name: str) -> list[dict[str, Any]]: @@ -338,9 +340,18 @@ async def snapshot_owned(artifact_id: str, run_id: str, name: str) -> bool: 的工件一律拒（这就是父 Run 证据接入的那道门，不另建权限系统）； - 别的（登记里平台的端点读取）：工件得是这次运行落下的端点快照。""" from app.tools.datasource import QUERY_PREFIX + from app.tools.metrics import METRIC_QUERY if name.startswith(QUERY_PREFIX): return True + if name == METRIC_QUERY: + from app.data.catalog import QUERY_SNAPSHOT_KIND + from app.db.base import SessionLocal + from app.db.models import Artifact + + async with SessionLocal() as session: + row = await session.get(Artifact, (artifact_id, run_id or "")) + return row is not None and row.kind == QUERY_SNAPSHOT_KIND from app.ledger.querycore import CALIBER_QUERY, PARENT_RUN_KEY if name == CALIBER_QUERY: diff --git a/backend/app/data/caliber_identity.py b/backend/app/data/caliber_identity.py index 52608193..6c864ef5 100644 --- a/backend/app/data/caliber_identity.py +++ b/backend/app/data/caliber_identity.py @@ -29,7 +29,7 @@ from typing import Any #: time_role / refund_rule 是 K04 的业务定义字段（统计时间口径、退款处理）：改了定义身份就变，绑定同步状态跟着重判。 SEMANTIC_FIELDS = ("id", "name", "expression", "unit", "decimals", "format", "kind", "definition", "params", "external", "rule", "assessment", "grain", "time_field", "numerator", "denominator", "dimensions", "filters", - "contract", "time_role", "refund_rule") + "contract", "time_role", "refund_rule", "semantic", "links") def _blank(value: Any) -> bool: diff --git a/backend/app/data/entities.py b/backend/app/data/entities.py new file mode 100644 index 00000000..8c30c1e9 --- /dev/null +++ b/backend/app/data/entities.py @@ -0,0 +1,566 @@ +"""标准业务实体（CanonicalEntity）的校验、持久化与跨源键值对齐。 + +核心铁律： +1. 物理表列绑定（bindings）与口径参数映射（caliber_params）分设 proposed / verified / confirmed / rejected 四态； + 运行时自动穿透、跨源实体值翻译、参数传递**只认 verified / confirmed**，proposed 候选绝不自动参与穿透。 +2. 跨源实体值翻译（translate_entity_value）坚持确定性映射：有标准成员表（members）时只按显式 source_codes / + 标准值 / 别名精确匹配，映射不上返回 None，严禁靠子串猜测。 +3. 受限视角（restricted=True，如普通 user 角色）隐去物理表名、物理列名与跨源底层编码，只暴露业务实体名与标准值。 +""" +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.db.base import new_id, utcnow +from app.db.models_ontology import CanonicalEntity + +ENTITY_STATUSES = ("proposed", "verified", "confirmed", "rejected") +ACTIVE_ENTITY_STATUSES = ("verified", "confirmed") +ENTITY_SOURCES = ("human", "rule", "ai", "profile") + +_KEY_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") + + +class EntityInvalid(ValueError): + """标准实体请求格式或取值不合法。""" + + def __init__(self, message: str, *, field: str = "") -> None: + super().__init__(message) + self.message = message + self.field = field + + +class EntityConflict(ValueError): + """标准实体版本冲突或唯一键冲突。""" + + def __init__(self, message: str, *, field: str = "") -> None: + super().__init__(message) + self.message = message + self.field = field + + +class EntityNotFound(LookupError): + """标准实体或子项不存在。""" + + +def _norm_token(value: Any) -> str: + return str(value or "").strip().lower() + + +def normalize_entity_key(raw: Any, *, field: str = "key") -> str: + if not isinstance(raw, str) or not _KEY_RE.match(raw.strip()): + raise EntityInvalid("实体标识（key）须以小写字母开头，仅含小写字母、数字和下划线（最长 64 字符）", field=field) + return raw.strip() + + +def _clean_str_list(raw: Any, *, field: str) -> list[str]: + if raw in (None, ""): + return [] + if not isinstance(raw, list): + raise EntityInvalid(f"字段「{field}」需要是字符串列表", field=field) + seen: set[str] = set() + out: list[str] = [] + for idx, item in enumerate(raw): + if not isinstance(item, str): + raise EntityInvalid(f"字段「{field}[{idx}]」需要是字符串", field=f"{field}[{idx}]") + val = item.strip() + if val and val.lower() not in seen: + seen.add(val.lower()) + out.append(val) + return out + + +def normalize_member(raw: Any, *, index: int = 0) -> dict[str, Any]: + at = f"members[{index}]" + if not isinstance(raw, Mapping): + raise EntityInvalid("实体成员需要是对象", field=at) + value = str(raw.get("value") or "").strip() + if not value: + raise EntityInvalid("实体成员缺少标准值（value）", field=f"{at}.value") + label = str(raw.get("label") or value).strip() or value + aliases = _clean_str_list(raw.get("aliases"), field=f"{at}.aliases") + raw_codes = raw.get("source_codes") + source_codes: dict[str, str] = {} + if raw_codes not in (None, {}): + if not isinstance(raw_codes, Mapping): + raise EntityInvalid("跨源编码（source_codes）需要是「数据源名 → 编码」对象", field=f"{at}.source_codes") + for src, code in raw_codes.items(): + src_key = str(src or "").strip() + code_val = str(code or "").strip() + if src_key and code_val: + source_codes[src_key] = code_val + return {"value": value, "label": label, "aliases": aliases, "source_codes": source_codes} + + +def normalize_binding(raw: Any, *, index: int = 0, actor: str | None = None, + force_proposed: bool = False) -> dict[str, Any]: + at = f"bindings[{index}]" + if not isinstance(raw, Mapping): + raise EntityInvalid("物理表列绑定需要是对象", field=at) + source_name = str(raw.get("source_name") or "").strip() + source_id = str(raw.get("source_id") or "").strip() or None + table_name = str(raw.get("table_name") or "").strip() + column_name = str(raw.get("column_name") or "").strip() + if not source_name: + raise EntityInvalid("物理绑定缺少数据源名称（source_name）", field=f"{at}.source_name") + if not table_name: + raise EntityInvalid("物理绑定缺少表名（table_name）", field=f"{at}.table_name") + if not column_name: + raise EntityInvalid("物理绑定缺少列名（column_name）", field=f"{at}.column_name") + source = str(raw.get("source") or "human").strip() + if source not in ENTITY_SOURCES: + raise EntityInvalid(f"绑定来源「{source}」无效，可选：{', '.join(ENTITY_SOURCES)}", field=f"{at}.source") + status = "proposed" if (force_proposed or source in ("ai", "profile") and "status" not in raw) else str( + raw.get("status") or "confirmed" + ).strip() + if force_proposed: + status = "proposed" + if status not in ENTITY_STATUSES: + raise EntityInvalid(f"绑定状态「{status}」无效，可选：{', '.join(ENTITY_STATUSES)}", field=f"{at}.status") + return { + "id": str(raw.get("id") or "").strip() or new_id()[:12], + "source_id": source_id, + "source_name": source_name, + "table_name": table_name, + "column_name": column_name, + "status": status, + "source": source, + "note": str(raw.get("note") or "").strip(), + "reason": str(raw.get("reason") or "").strip(), + "updated_at": str(raw.get("updated_at") or utcnow().isoformat()), + "updated_by": str(raw.get("updated_by") or actor or "").strip() or None, + } + + +def normalize_caliber_param(raw: Any, *, index: int = 0, force_proposed: bool = False) -> dict[str, Any]: + at = f"caliber_params[{index}]" + if not isinstance(raw, Mapping): + raise EntityInvalid("口径参数映射需要是对象", field=at) + workflow_id = str(raw.get("workflow_id") or "").strip() + card_id = str(raw.get("card_id") or "").strip() + param_name = str(raw.get("param_name") or "").strip() + if not workflow_id or not param_name: + raise EntityInvalid("口径参数映射须写明工作流 ID（workflow_id）与参数名（param_name）", field=at) + source = str(raw.get("source") or "human").strip() + if source not in ENTITY_SOURCES: + raise EntityInvalid(f"口径参数映射来源「{source}」无效", field=f"{at}.source") + status = "proposed" if force_proposed else str(raw.get("status") or "confirmed").strip() + if status not in ENTITY_STATUSES: + raise EntityInvalid(f"口径参数映射状态「{status}」无效", field=f"{at}.status") + return { + "id": str(raw.get("id") or "").strip() or new_id()[:12], + "workflow_id": workflow_id, + "card_id": card_id, + "param_name": param_name, + "status": status, + "source": source, + "note": str(raw.get("note") or "").strip(), + } + + +def normalize_entity_payload(payload: Mapping[str, Any], *, partial: bool = False, + actor: str | None = None) -> dict[str, Any]: + if not isinstance(payload, Mapping): + raise EntityInvalid("请求内容需要是对象") + out: dict[str, Any] = {} + if not partial or "key" in payload: + out["key"] = normalize_entity_key(payload.get("key")) + if not partial or "label" in payload: + label = str(payload.get("label") or "").strip() + if not label: + raise EntityInvalid("实体名称（label）不能为空", field="label") + if len(label) > 100: + raise EntityInvalid("实体名称不能超过 100 个字符", field="label") + out["label"] = label + if not partial or "description" in payload: + out["description"] = str(payload.get("description") or "").strip() + if not partial or "aliases" in payload: + out["aliases"] = _clean_str_list(payload.get("aliases"), field="aliases") + if not partial or "members" in payload: + raw_members = payload.get("members") or [] + if not isinstance(raw_members, list): + raise EntityInvalid("成员列表（members）需要是数组", field="members") + seen_vals: set[str] = set() + members: list[dict[str, Any]] = [] + for idx, m in enumerate(raw_members): + norm = normalize_member(m, index=idx) + vkey = norm["value"].lower() + if vkey in seen_vals: + raise EntityInvalid(f"实体成员值「{norm['value']}」重复", field=f"members[{idx}].value") + seen_vals.add(vkey) + members.append(norm) + out["members"] = members + if not partial or "bindings" in payload: + raw_bindings = payload.get("bindings") or [] + if not isinstance(raw_bindings, list): + raise EntityInvalid("物理绑定列表（bindings）需要是数组", field="bindings") + seen_cols: set[tuple[str, str, str]] = set() + bindings: list[dict[str, Any]] = [] + for idx, b in enumerate(raw_bindings): + norm = normalize_binding(b, index=idx, actor=actor) + col_key = (norm["source_name"].lower(), norm["table_name"].lower(), norm["column_name"].lower()) + if col_key in seen_cols: + raise EntityInvalid( + f"数据源「{norm['source_name']}」表「{norm['table_name']}.{norm['column_name']}」重复绑定", + field=f"bindings[{idx}]", + ) + seen_cols.add(col_key) + bindings.append(norm) + out["bindings"] = bindings + if not partial or "caliber_params" in payload: + raw_params = payload.get("caliber_params") or [] + if not isinstance(raw_params, list): + raise EntityInvalid("口径参数映射（caliber_params）需要是数组", field="caliber_params") + out["caliber_params"] = [normalize_caliber_param(p, index=idx) for idx, p in enumerate(raw_params)] + return out + + +# -------------------------------------------------------------------------- +# 运行时对齐与状态过滤（只认 verified / confirmed） +# -------------------------------------------------------------------------- + + +def _as_dict(entity: CanonicalEntity | Mapping[str, Any]) -> dict[str, Any]: + if isinstance(entity, Mapping): + return dict(entity) + return { + "id": entity.id, + "key": entity.key, + "label": entity.label, + "description": entity.description or "", + "aliases": list(entity.aliases or []), + "members": list(entity.members or []), + "bindings": list(entity.bindings or []), + "caliber_params": list(entity.caliber_params or []), + "version": int(entity.version or 1), + "created_by": entity.created_by, + "updated_by": entity.updated_by, + "created_at": entity.created_at.isoformat() if getattr(entity, "created_at", None) else None, + "updated_at": entity.updated_at.isoformat() if getattr(entity, "updated_at", None) else None, + } + + +def entity_to_dict(entity: CanonicalEntity | Mapping[str, Any], *, restricted: bool = False) -> dict[str, Any]: + """序列化标准实体；受限视角（user 角色）隐去物理表名、列名、数据源内部 ID 与底层编码。""" + data = _as_dict(entity) + if not restricted: + return data + return { + "id": data["id"], + "key": data["key"], + "label": data["label"], + "description": data["description"], + "aliases": list(data.get("aliases") or []), + "members": [ + {"value": m.get("value"), "label": m.get("label"), "aliases": list(m.get("aliases") or [])} + for m in (data.get("members") or []) if isinstance(m, Mapping) + ], + "version": data["version"], + } + + +def confirmed_bindings( + entity: CanonicalEntity | Mapping[str, Any], + *, + source_id: str | None = None, + source_name: str | None = None, + table_name: str | None = None, +) -> list[dict[str, Any]]: + """返回已生效（verified / confirmed）的物理列绑定；proposed 与 rejected 一律排除。""" + data = _as_dict(entity) + out: list[dict[str, Any]] = [] + for item in data.get("bindings") or []: + if not isinstance(item, Mapping) or item.get("status") not in ACTIVE_ENTITY_STATUSES: + continue + if source_id and str(item.get("source_id") or "") != source_id: + continue + if source_name and _norm_token(item.get("source_name")) != _norm_token(source_name): + continue + if table_name and _norm_token(item.get("table_name")) != _norm_token(table_name): + continue + out.append(dict(item)) + return out + + +def confirmed_caliber_params( + entity: CanonicalEntity | Mapping[str, Any], + *, + workflow_id: str | None = None, + card_id: str | None = None, +) -> list[dict[str, Any]]: + """返回已生效（verified / confirmed）的口径输入参数映射。""" + data = _as_dict(entity) + out: list[dict[str, Any]] = [] + for item in data.get("caliber_params") or []: + if not isinstance(item, Mapping) or item.get("status") not in ACTIVE_ENTITY_STATUSES: + continue + if workflow_id and str(item.get("workflow_id") or "") != workflow_id: + continue + if card_id and item.get("card_id") and str(item.get("card_id") or "") != card_id: + continue + out.append(dict(item)) + return out + + +def resolve_caliber_param( + entity: CanonicalEntity | Mapping[str, Any], + *, + workflow_id: str, + card_id: str | None = None, +) -> str | None: + """把标准业务实体解析为目标口径的入口参数名（优先匹配指定 card_id，其次匹配同工作流通用参数）。""" + matches = confirmed_caliber_params(entity, workflow_id=workflow_id) + if card_id: + for item in matches: + if str(item.get("card_id") or "") == card_id: + return str(item["param_name"]) + for item in matches: + if not item.get("card_id") or card_id is None: + return str(item["param_name"]) + return str(matches[0]["param_name"]) if matches else None + + +def translate_entity_value( + entity: CanonicalEntity | Mapping[str, Any], + raw_value: Any, + *, + from_source: str | None = None, + to_source: str | None = None, +) -> dict[str, str] | None: + """把源侧实体值确定性地翻译成标准值与目标源编码。 + + - 有标准成员表（members 非空）时：严格按 from_source 的 source_codes、标准 value、label、aliases 精确匹配， + 再用 to_source 的 source_codes（缺省回退标准 value）给出 target_value；找不到返回 None，绝不模糊猜测。 + - 成员表为空（开放式编码域，如千万级订单号/批次号）时：仅当两侧数据源在该实体上均有已确认绑定时原样透传。 + """ + probe = str(raw_value or "").strip() + if not probe: + return None + data = _as_dict(entity) + members = [m for m in (data.get("members") or []) if isinstance(m, Mapping)] + probe_norm = probe.lower() + + if members: + matched: Mapping[str, Any] | None = None + if from_source: + src_norm = _norm_token(from_source) + for member in members: + codes = member.get("source_codes") + if isinstance(codes, Mapping): + for k, v in codes.items(): + if _norm_token(k) == src_norm and _norm_token(v) == probe_norm: + matched = member + break + if matched is not None: + break + if matched is None: + for member in members: + candidates = [member.get("value"), member.get("label"), *(member.get("aliases") or [])] + codes = member.get("source_codes") + if not from_source and isinstance(codes, Mapping): + candidates.extend(codes.values()) + if any(_norm_token(c) == probe_norm for c in candidates if c is not None): + matched = member + break + if matched is None: + return None + canonical = str(matched.get("value") or "").strip() + label = str(matched.get("label") or canonical).strip() + target_value = canonical + if to_source and isinstance(matched.get("source_codes"), Mapping): + dst_norm = _norm_token(to_source) + for k, v in matched["source_codes"].items(): + if _norm_token(k) == dst_norm and str(v or "").strip(): + target_value = str(v).strip() + break + return {"canonical": canonical, "label": label, "target_value": target_value} + + if from_source and not confirmed_bindings(data, source_name=from_source): + return None + if to_source and not confirmed_bindings(data, source_name=to_source): + return None + return {"canonical": probe, "label": probe, "target_value": probe} + + +def entities_bridging_sources( + entities: Iterable[CanonicalEntity | Mapping[str, Any]], + *, + source_name: str, + tables: Iterable[str] | None = None, +) -> list[dict[str, Any]]: + """找出在当前数据源（及可选表集合）上有已确认绑定、并且同时桥接到其他数据源的标准实体。""" + table_keys = {_norm_token(t) for t in tables} if tables is not None else None + out: list[dict[str, Any]] = [] + for entity in entities: + data = _as_dict(entity) + active = confirmed_bindings(data) + here = [ + b for b in active + if _norm_token(b.get("source_name")) == _norm_token(source_name) + and (table_keys is None or _norm_token(b.get("table_name")) in table_keys) + ] + if not here: + continue + peers = [b for b in active if _norm_token(b.get("source_name")) != _norm_token(source_name)] + out.append({ + "entity_id": data["id"], + "key": data["key"], + "label": data["label"], + "local_bindings": here, + "peer_bindings": peers, + "peer_sources": sorted({str(b["source_name"]) for b in peers}), + }) + return out + + +# -------------------------------------------------------------------------- +# 数据库读写与乐观锁审阅 +# -------------------------------------------------------------------------- + + +async def list_entities( + session: AsyncSession, + *, + source_name: str | None = None, + only_active_bindings: bool = False, +) -> list[CanonicalEntity]: + rows = list((await session.execute(select(CanonicalEntity).order_by(CanonicalEntity.key.asc()))).scalars()) + if not source_name: + return rows + src_norm = _norm_token(source_name) + matched: list[CanonicalEntity] = [] + for row in rows: + bindings = confirmed_bindings(row) if only_active_bindings else (row.bindings or []) + if any(isinstance(b, Mapping) and _norm_token(b.get("source_name")) == src_norm for b in bindings): + matched.append(row) + return matched + + +async def get_entity(session: AsyncSession, entity_id: str) -> CanonicalEntity | None: + return await session.get(CanonicalEntity, str(entity_id or "").strip()) + + +async def get_entity_by_key(session: AsyncSession, key: str) -> CanonicalEntity | None: + clean = str(key or "").strip().lower() + if not clean: + return None + return ( + await session.execute(select(CanonicalEntity).where(CanonicalEntity.key == clean).limit(1)) + ).scalar_one_or_none() + + +async def create_entity( + session: AsyncSession, + payload: Mapping[str, Any], + *, + actor: str | None = None, +) -> CanonicalEntity: + fields = normalize_entity_payload(payload, partial=False, actor=actor) + existing = await get_entity_by_key(session, fields["key"]) + if existing is not None: + raise EntityConflict(f"标准实体标识「{fields['key']}」已存在", field="key") + row = CanonicalEntity( + key=fields["key"], + label=fields["label"], + description=fields["description"], + aliases=fields["aliases"], + members=fields["members"], + bindings=fields["bindings"], + caliber_params=fields["caliber_params"], + version=1, + created_by=actor, + updated_by=actor, + ) + session.add(row) + await session.flush() + return row + + +async def update_entity( + session: AsyncSession, + entity_id: str, + payload: Mapping[str, Any], + *, + actor: str | None = None, +) -> CanonicalEntity: + row = await get_entity(session, entity_id) + if row is None: + raise EntityNotFound(f"标准实体「{entity_id}」不存在") + expected_version = payload.get("version") + if expected_version is not None: + if isinstance(expected_version, bool) or not isinstance(expected_version, int) or expected_version != row.version: + raise EntityConflict( + f"标准实体「{row.label}」已被其他人修改（当前版本 v{row.version}），请刷新后再保存", + field="version", + ) + fields = normalize_entity_payload(payload, partial=True, actor=actor) + if "key" in fields and fields["key"] != row.key: + clash = await get_entity_by_key(session, fields["key"]) + if clash is not None and clash.id != row.id: + raise EntityConflict(f"标准实体标识「{fields['key']}」已存在", field="key") + row.key = fields["key"] + for attr in ("label", "description", "aliases", "members", "bindings", "caliber_params"): + if attr in fields: + setattr(row, attr, fields[attr]) + row.version = int(row.version or 1) + 1 + row.updated_by = actor + await session.flush() + return row + + +async def review_entity_item( + session: AsyncSession, + entity_id: str, + *, + target_kind: str, + item_id: str, + status: str, + reason: str = "", + actor: str | None = None, + expected_version: int | None = None, +) -> CanonicalEntity: + """审阅单条物理表列绑定或口径参数映射（确认 confirmed / 驳回 rejected / 验证通过 verified）。""" + if target_kind not in ("binding", "caliber_param"): + raise EntityInvalid("审阅对象类型须为 binding 或 caliber_param", field="target_kind") + if status not in ENTITY_STATUSES: + raise EntityInvalid(f"状态「{status}」无效，可选：{', '.join(ENTITY_STATUSES)}", field="status") + row = await get_entity(session, entity_id) + if row is None: + raise EntityNotFound(f"标准实体「{entity_id}」不存在") + if expected_version is not None and int(expected_version) != int(row.version or 1): + raise EntityConflict(f"标准实体「{row.label}」版本已变更（当前 v{row.version}），请刷新后重试", field="version") + + attr = "bindings" if target_kind == "binding" else "caliber_params" + items = [dict(x) for x in (getattr(row, attr) or []) if isinstance(x, Mapping)] + found = False + now = utcnow().isoformat() + for item in items: + if str(item.get("id") or "") == str(item_id or "").strip(): + item["status"] = status + if reason.strip(): + item["reason"] = reason.strip() + item["updated_at"] = now + item["updated_by"] = actor + found = True + break + if not found: + raise EntityNotFound(f"标准实体「{row.label}」中不存在子项「{item_id}」") + setattr(row, attr, items) + row.version = int(row.version or 1) + 1 + row.updated_by = actor + await session.flush() + return row + + +async def delete_entity(session: AsyncSession, entity_id: str) -> bool: + row = await get_entity(session, entity_id) + if row is None: + return False + await session.delete(row) + await session.flush() + return True diff --git a/backend/app/data/entity_assistant.py b/backend/app/data/entity_assistant.py new file mode 100644 index 00000000..63ddb93a --- /dev/null +++ b/backend/app/data/entity_assistant.py @@ -0,0 +1,942 @@ +"""实体助手（E2）：在标准业务实体页与使用者流式对话，跨数据源与业务文档对齐维护 CanonicalEntity。 + +核心规矩： +1. 只提结构化提案（propose_entity_changes），绝不静默直接写库；使用者在界面看过后点击「保存修改」才经 + apply_entity_patch(mode="confirmed") 落库。 +2. 跨源只读探查（run_query）仅对已开启数据剖析的数据源放行，严格经 guard.check(readonly=True) 与遮罩列脱敏 + （mask_query_result）；遮罩列只给列名，不给注释或码值。 +3. 支持直接读取本次对话挂载的上传附件或知识库文档片段（read_document），并要求在提案 reason 中区分「使用者说：……」、 + 「文档《…》写着：……」与「数据探查显示：……」，同时主动向使用者汇报文档码表与实表取值的差异。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +from collections.abc import AsyncIterator, Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from datetime import date, datetime +from decimal import Decimal +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import catalog, catalog_assistant, catalog_profile, entities, entity_docs, entity_patch, introspect, table_versions +from app.data.draft_model import DRAFT_MODEL_PLACE +from app.data.engine import SnapshotTampered, masked_columns +from app.db.base import SessionLocal +from app.db.models import DataSource, Workflow, WorkflowVersion + +logger = logging.getLogger(__name__) + +MAX_MESSAGES = 40 +MESSAGE_MAX = 4000 +MAX_ROUNDS = 8 +MAX_QUERIES = 6 +QUERY_ROWS = 50 +SHOWN_ROWS = 10 +CELL_MAX = 300 +TOOL_CHARS = 12_000 +RETRY_DELAYS_S = (3.0, 10.0) +HEARTBEAT_S = 10.0 + +INSPECT_TOOL = "inspect_sources" +READ_DOC_TOOL = "read_document" +QUERY_TOOL = "run_query" +PROPOSE_TOOL = "propose_entity_changes" + +CODE_MODEL = "assistant_model_error" +CODE_TRUNCATED = "assistant_truncated" +CODE_TIMEOUT = "assistant_timeout" +CODE_ROUNDS = "assistant_rounds" +CODE_FAILED = "assistant_failed" + +# ---- 面向界面展示的错误文案 +ERROR_MODEL = ( + "调用模型「{model}」失败：{reason}。请稍后重试；如持续失败，请在{place}中换一个模型，" + "或到「设置 → 模型接入」测试该模型" +) +ERROR_TRUNCATED = "{cut}，回答没有写完。可以把问题拆小一点再问，或在{place}中换一个模型" +ERROR_TIMEOUT = "模型 {seconds} 秒内没有回答完，已停止这一轮。可以把问题拆小一点再问，或在{place}中换一个模型" +ERROR_ROUNDS = "这一轮已经调用了 {n} 次模型仍没有答完，已停止。可以换个问法，或者分几次问" +ERROR_FAILED = "实体助手出错：{line}" +LAST_NOT_USER = "对话的最后一条应当是使用者的话，请重新发送" +QUERY_TAMPERED = "数据源「{name}」的{error}。请联系管理员检查数据目录" + +# ---- 面向模型的提示词（命名含 _SYSTEM / _PROMPT） +ENTITY_ASSISTANT_SYSTEM = """你是 AgentLab 的「实体助手」，协助搭建者围绕当前选定范围的数据源与业务文档建立、完善「标准业务实体」（CanonicalEntity）： +包括实体的标准标识（key）、中文名称（label）、常用叫法（aliases）、物理表列绑定（bindings）、标准成员编码映射（members.source_codes）与口径入口参数映射（caliber_params）。 + +核心规矩： +1. 你不能修改业务数据库数据，也不直接写入标准业务实体：一切新增或修改都通过 propose_entity_changes 工具整理成结构化提案，由使用者在界面审阅后点击保存才生效。 +2. 严格限定在「当前分析范围的数据源」内工作：只探查、查询和绑定范围内的数据源，绝不提及、猜测或扩展到未纳入范围的其他数据源。若当前范围只有 1 个数据源，聚焦盘点该数据源内的核心业务实体、多表共享主外键、码值字典与口径入参；若选了多个数据源，则重点对齐这些选定数据源之间的同义实体与编码映射。 +3. 表名、列名、数据源名、工作流 ID、口径参数名必须先通过 inspect_sources 核实，严禁猜测不存在的表名或列名。 +4. 若挂载了业务文档或编码对照表，优先用 read_document 阅读原文定义或码值映射，并结合 inspect_sources / run_query 去真实数据源交叉核验： + - 如果文档里提到的编码与数据库实表抽样不一致（例如实表多出文档未登记的新编码，或文档编码在实表未出现），必须在回答正文中明确告知使用者，询问是否一并纳入。 + - 如果拿不准两套编码如何对应，先简短具体地向使用者提问，不要硬猜。 +5. 提案中每条操作的 reason 字段必须如实写明依据来源： + - 来自本次对话使用者亲口说明的，写「使用者说：……」； + - 来自挂载文档或知识库文档的，写「文档《文档名》写着：……」； + - 来自表目录或只读 SQL 核查的，写「数据探查显示：……」。 +6. 标着「已确认」或「已验证」的既有绑定，除非使用者明确要求调整，否则不要擅自改动或删除；标着「已驳回」的绑定使用者曾明确拒绝过，不要重复提议。 +7. 标着「遮罩列」的敏感字段禁止查询明细取值，也禁止为其生成成员编码映射（members.source_codes）。 +8. 回答用简短直接的中文。 + +propose_entity_changes 的 operations 数组支持以下 5 种增量操作： +- upsert_entity: {"op": "upsert_entity", "entity_key": "小写英文下划线标识", "label": "中文名称", "description": "业务粒度说明", "aliases": ["常用叫法"], "reason": "依据"} +- upsert_binding: {"op": "upsert_binding", "entity_key": "实体标识", "entity_label": "中文名（新建时必填）", "source_name": "数据源名", "table_name": "物理表名", "column_name": "物理列名", "note": "备注", "reason": "依据"} +- remove_binding: {"op": "remove_binding", "entity_key": "实体标识", "source_name": "数据源名", "table_name": "物理表名", "column_name": "物理列名", "reason": "依据"} +- upsert_member: {"op": "upsert_member", "entity_key": "实体标识", "entity_label": "中文名", "value": "标准值", "label": "标准展示名", "aliases": ["成员别名"], "source_codes": {"数据源名": "该源底层编码"}, "reason": "依据"} +- upsert_caliber_param: {"op": "upsert_caliber_param", "entity_key": "实体标识", "entity_label": "中文名", "workflow_id": "工作流ID", "card_id": "可选口径卡节点ID", "param_name": "入口参数名", "note": "备注", "reason": "依据"} +""" + +ENTITY_QUERY_PROMPT = ( + "跨源只读查询：可对开启了数据剖析的数据源调用 run_query 执行只读 SQL（如 SELECT DISTINCT 核对编码取值与行数分布），" + "单次最多返回 {rows} 行，本轮最多执行 {limit} 次。可查询的数据源：{sources}。" +) +ENTITY_NO_QUERY_PROMPT = "当前范围内没有开启数据剖析的数据源，不能执行 run_query 查实表数据，请根据表结构、数据目录与业务文档推断对齐。" +ENTITY_OPENING_PROMPT = ( + "（使用者刚打开实体助手，尚未发言。）请结合当前分析范围的数据源、既有实体与挂载文档概况，用两三句话指出当前范围内最值得优先建立或补齐的业务实体，并提出一两个具体问题；先不要调用 propose_entity_changes。" +) +ENTITY_OPENED_PROMPT = "（使用者打开了实体助手。）" + +ENTITY_TOOL_PROMPT = { + "query_disabled_source": "数据源「{source}」未开启数据剖析或不存在，无法执行只读查询。可查询的数据源：{allowed}", + "query_capped": "这一轮的只读查询次数已达上限（{n} 次），请根据已有信息回答或提出提案。", + "empty_sql": "请填写要执行的只读 SQL 语句。", + "query_failed": "查询未成功：{error}", + "no_ops": "operations 为空，请提供至少一条修改操作。", + "proposed": "已通过核验并向界面提交 {n} 项实体变更提案：{items}", + "nothing": "没有产生新的有效变更（可能与当前状态完全相同，或全部未通过核验）。", + "dropped": "以下项未通过平台核验，请核对表列名或参数后修正：{reasons}", + "no_docs": "当前对话未挂载任何业务文档或对照表附件。", + "unknown_tool": "未知工具「{name}」，可用工具：{tools}", + "last_round": "注意：下一轮是最后一次模型调用，请直接用中文向使用者总结结论，不要再调用工具。", +} + +INSPECT_TOOL_PROMPT: dict[str, Any] = { + "type": "function", + "function": { + "name": INSPECT_TOOL, + "description": "在已启用的数据源表结构、数据目录（列中文名、含义、码值）及已发布口径入口参数中检索与业务实体相关的表列和入参。", + "parameters": { + "type": "object", + "properties": { + "keyword": { + "type": "string", + "description": "检索关键词（如「产线」「line」「仓库」「region」）；留空则返回各数据源表概览与口径入参清单。", + }, + "source_name": { + "type": "string", + "description": "可选：限定只查看某一个数据源名称。", + }, + }, + }, + }, +} + +READ_DOC_TOOL_PROMPT: dict[str, Any] = { + "type": "function", + "function": { + "name": READ_DOC_TOOL, + "description": "读取本次对话挂载的业务文档、数据字典或编码对照表片段。", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "按关键词检索相关段落或表格行；留空则按顺序读取分段。", + }, + "doc_ref": { + "type": "string", + "description": "可选：限定某一份文档的 ID 或标题关键词。", + }, + "start_chunk": { + "type": "integer", + "description": "顺序读取时的起始段号（从 0 开始）。", + }, + "limit": { + "type": "integer", + "description": "最多返回几段（默认 6，最多 15）。", + }, + }, + }, + }, +} + +QUERY_TOOL_PROMPT: dict[str, Any] = { + "type": "function", + "function": { + "name": QUERY_TOOL, + "description": "在指定数据源上执行一条只读 SQL（如 SELECT DISTINCT code, name FROM ... LIMIT 50），用于核对跨源成员编码与分布。", + "parameters": { + "type": "object", + "properties": { + "source_name": {"type": "string", "description": "要查询的数据源名称"}, + "sql": {"type": "string", "description": "只读 SELECT 查询语句"}, + }, + "required": ["source_name", "sql"], + }, + }, +} + +PROPOSE_TOOL_PROMPT: dict[str, Any] = { + "type": "function", + "function": { + "name": PROPOSE_TOOL, + "description": "提交一批标准业务实体的新建或修改提案（经平台核验后展示在界面待保存区，不直接写库）。", + "parameters": { + "type": "object", + "properties": { + "operations": { + "type": "array", + "description": "增量操作数组（upsert_entity / upsert_binding / remove_binding / upsert_member / upsert_caliber_param）", + "items": {"type": "object"}, + }, + }, + "required": ["operations"], + }, + }, +} + + +@dataclass(slots=True) +class _SourceEntry: + row: DataSource + resolved: Any + kind: str + masked: set[str] + settings: catalog_profile.ProfileSettings + can_query: bool + catalog_notes: dict[str, Any] + + +@dataclass(slots=True) +class EntityAssistantContext: + sources: dict[str, _SourceEntry] + workflows: list[dict[str, Any]] + doc_bundles: list[entity_docs.EntityDocBundle] + pending_ops: list[dict[str, Any]] = field(default_factory=list) + system: str = "" + + @property + def queryable_sources(self) -> list[str]: + return [entry.row.name for entry in self.sources.values() if entry.can_query] + + def tools(self) -> list[dict[str, Any]]: + items = [INSPECT_TOOL_PROMPT] + if self.doc_bundles: + items.append(READ_DOC_TOOL_PROMPT) + if self.queryable_sources: + items.append(QUERY_TOOL_PROMPT) + items.append(PROPOSE_TOOL_PROMPT) + return items + + +resolve_model = catalog_assistant.resolve_model + + +def conversation_problem(messages: Iterable[Mapping[str, Any]]) -> str | None: + said = [m for m in messages if str(m.get("content") or "").strip()] + if any(m.get("role") == "user" for m in said) and said[-1].get("role") != "user": + return LAST_NOT_USER + return None + + +def _history(messages: Iterable[Mapping[str, Any]]) -> list[Any]: + from langchain_core.messages import AIMessage, HumanMessage + + merged: list[tuple[str, str]] = [] + for m in messages: + text = str(m.get("content") or "").strip() + if not text: + continue + role = "user" if m.get("role") == "user" else "assistant" + if merged and merged[-1][0] == role: + merged[-1] = (role, f"{merged[-1][1]}\n\n{text}") + else: + merged.append((role, text)) + out: list[Any] = [] + if merged and merged[0][0] == "assistant": + out.append(HumanMessage(content=ENTITY_OPENED_PROMPT)) + for role, text in merged: + out.append(HumanMessage(content=text) if role == "user" else AIMessage(content=text)) + if not any(role == "user" for role, _ in merged): + out.append(HumanMessage(content=ENTITY_OPENING_PROMPT)) + return out + + +def _extract_wf_summary( + wf_rows: Sequence[Workflow], + ver_rows: Sequence[WorkflowVersion], + *, + allowed_sources: set[str] | None = None, +) -> list[dict[str, Any]]: + from app.tools.datasource import source_of_tool + + published_by_wf: dict[str, WorkflowVersion] = {} + for ver in ver_rows: + cur = published_by_wf.get(ver.workflow_id) + if cur is None or int(ver.version or 0) > int(cur.version or 0): + published_by_wf[ver.workflow_id] = ver + + summaries: list[dict[str, Any]] = [] + for wf in wf_rows: + ver = published_by_wf.get(wf.id) + graph = (ver.graph if ver and isinstance(ver.graph, Mapping) else wf.graph) or {} + if not isinstance(graph, Mapping): + continue + cards: list[dict[str, Any]] = [] + inputs: list[dict[str, Any]] = [] + wf_sources: set[str] = set() + for node in graph.get("nodes") or []: + if not isinstance(node, Mapping): + continue + ntype = str(node.get("type") or "") + nid = str(node.get("id") or "") + data = node.get("data") if isinstance(node.get("data"), Mapping) else {} + cfg = data.get("config") if isinstance(data.get("config"), Mapping) else ( + node.get("config") if isinstance(node.get("config"), Mapping) else {} + ) + t_single = str(cfg.get("tool") or "").strip() + if s := source_of_tool(t_single): + if "{{" not in s: + wf_sources.add(s.lower()) + for t_item in cfg.get("tools") or []: + if isinstance(t_item, str) and (s := source_of_tool(t_item.strip())) and "{{" not in s: + wf_sources.add(s.lower()) + for skey in ("source", "source_name", "datasource", "source_id"): + sv = str(cfg.get(skey) or "").strip() + if sv and "{{" not in sv: + wf_sources.add(sv.lower()) + + if ntype == "metrics" and nid: + m_names = [ + str(m.get("name") or m.get("id") or "") + for m in (cfg.get("metrics") or []) + if isinstance(m, Mapping) + ] + cards.append({"card_id": nid, "metrics": [x for x in m_names if x][:8]}) + elif ntype == "input": + for f in cfg.get("fields") or []: + if isinstance(f, Mapping) and str(f.get("name") or "").strip(): + inputs.append({ + "name": str(f["name"]).strip(), + "label": str(f.get("label") or f.get("description") or "").strip(), + "type": str(f.get("type") or "string"), + }) + if allowed_sources is not None: + if wf_sources: + if not (wf_sources & allowed_sources): + continue + elif not allowed_sources: + continue + if cards and inputs: + summaries.append({ + "workflow_id": wf.id, + "name": wf.name, + "published": ver is not None, + "inputs": inputs, + "cards": cards, + "sources": sorted(wf_sources), + }) + return summaries + + +async def prepare( + session: AsyncSession, + *, + source_ids: Sequence[str] | None = None, + focus_entity_key: str | None = None, + attachment_ids: Sequence[str] = (), + document_ids: Sequence[str] = (), + pending_ops: Sequence[Mapping[str, Any]] = (), +) -> EntityAssistantContext: + """组装实体助手上下文：按选定范围过滤的数据源表目录、既有标准实体、口径入参、挂载文档摘要与待保存提案。""" + all_enabled = list( + (await session.execute(select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.name))).scalars() + ) + if source_ids is not None: + wanted = {str(s or "").strip().lower() for s in source_ids if str(s or "").strip()} + src_rows = [r for r in all_enabled if r.id.lower() in wanted or r.name.lower() in wanted] + else: + src_rows = all_enabled + + allowed_sources: set[str] | None = ( + {x for r in src_rows for x in (r.id.lower(), r.name.lower())} + if source_ids is not None + else None + ) + + sources: dict[str, _SourceEntry] = {} + source_lines: list[str] = ["当前分析范围的数据源："] + for row in src_rows: + try: + resolved = await table_versions.resolve_source(session, row) + except table_versions.SnapshotMissing: + resolved = row + opts = dict(row.options or {}) if isinstance(row.options, Mapping) else {} + masked = {c.lower() for c in masked_columns(opts)} + p_settings = catalog_profile.profile_settings(opts) + can_query = bool(p_settings.enabled and row.enabled) + cat_entries = await catalog.read_catalog(session, resolved.id) + notes_map = {tname: entry.notes for tname, entry in cat_entries.items()} + sources[row.name.lower()] = _SourceEntry( + row=row, + resolved=resolved, + kind=str(row.kind or ""), + masked=masked, + settings=p_settings, + can_query=can_query, + catalog_notes=notes_map, + ) + tables = ((resolved.schema_cache or {}).get("tables") or {}) if isinstance(resolved.schema_cache, Mapping) else {} + q_flag = "可只读查询" if can_query else "未开剖析" + source_lines.append(f"- {row.name}（{row.kind}，共 {len(tables)} 张表，{q_flag}）{('：' + row.description) if row.description else ''}") + if len(source_lines) == 1: + source_lines.append("- （当前未选定任何数据源，仅基于挂载文档与既有实体分析）") + + clean_pending = [dict(op) for op in (pending_ops or ()) if isinstance(op, Mapping)] + pending_keys = { + str(op.get("entity_key") or "").strip().lower() + for op in clean_pending + if str(op.get("entity_key") or "").strip() + } + + ent_rows = await entities.list_entities(session) + focus_norm = (focus_entity_key or "").strip().lower() + in_scope_entities: list[Any] = [] + other_domain_keys: list[str] = [] + for erow in ent_rows: + ekey_low = erow.key.lower() + b_list = [b for b in (erow.bindings or []) if isinstance(b, Mapping)] + if allowed_sources is None or ekey_low == focus_norm or ekey_low in pending_keys or not b_list: + in_scope_entities.append(erow) + continue + touches_scope = any( + str(b.get("source_name") or "").strip().lower() in allowed_sources + or str(b.get("source_id") or "").strip().lower() in allowed_sources + for b in b_list + ) + if touches_scope: + in_scope_entities.append(erow) + else: + other_domain_keys.append(f"{erow.key}({erow.label})") + + entity_lines: list[str] = ["", f"当前范围相关的标准业务实体（共 {len(in_scope_entities)} 个）："] + for erow in in_scope_entities[:60]: + b_list = [b for b in (erow.bindings or []) if isinstance(b, Mapping)] + if allowed_sources is not None and erow.key.lower() != focus_norm: + b_list = [ + b for b in b_list + if str(b.get("source_name") or "").strip().lower() in allowed_sources + or str(b.get("source_id") or "").strip().lower() in allowed_sources + ] + active_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") in ("confirmed", "verified")] + prop_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") == "proposed"] + rej_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") == "rejected"] + m_count = len(erow.members or []) + head = f"- {erow.label}（key={erow.key}，v{erow.version}，成员 {m_count} 项）" + if active_b: + head += f"｜已确认绑定：{', '.join(active_b[:8])}" + if prop_b: + head += f"｜待确认：{', '.join(prop_b[:6])}" + if rej_b: + head += f"｜已驳回（勿重提）：{', '.join(rej_b[:6])}" + entity_lines.append(head) + if focus_norm and erow.key.lower() == focus_norm: + entity_lines.append(f" 【当前使用者正在查看该实体】说明：{erow.description or '无'}；别名：{', '.join(erow.aliases or []) or '无'}") + for m in (erow.members or [])[:25]: + if isinstance(m, Mapping): + codes_txt = ", ".join(f"{k}={v}" for k, v in (m.get("source_codes") or {}).items()) + entity_lines.append(f" · 成员 {m.get('value')}（{m.get('label')}）：{codes_txt or '无跨源编码'}") + if other_domain_keys: + entity_lines.append( + f"（其他未选定数据源已占用的全局实体 key，避免重名冲突，无需分析未选定源：{', '.join(other_domain_keys[:30])}）" + ) + + wf_rows = list((await session.execute(select(Workflow))).scalars()) + ver_rows = list((await session.execute(select(WorkflowVersion).where(WorkflowVersion.published_at.is_not(None)))).scalars()) + workflows = _extract_wf_summary(wf_rows, ver_rows, allowed_sources=allowed_sources) + + doc_bundles = await entity_docs.load_doc_bundles(session, attachment_ids=attachment_ids, document_ids=document_ids) + doc_lines: list[str] = [] + if doc_bundles: + doc_lines.append("") + doc_lines.append(f"本次对话挂载的业务文档/对照表（共 {len(doc_bundles)} 份，可用 read_document 检索详情）：") + for b in doc_bundles: + preview = b.chunks[0][:300].replace("\n", " ") if b.chunks else "" + doc_lines.append(f"- 《{b.title}》（ref={b.ref_id}，共 {len(b.chunks)} 段，{b.char_count} 字）首段预览：{preview}…") + + pending_lines: list[str] = [] + if clean_pending: + pending_lines.append("") + pending_lines.append(f"界面待保存区已有的修改提案（共 {len(clean_pending)} 项，避免重复提议）：") + for pop in clean_pending[-40:]: + pending_lines.append(f"- {json.dumps(pop, ensure_ascii=False, default=str)}") + + queryable = [e.row.name for e in sources.values() if e.can_query] + query_prompt = ( + ENTITY_QUERY_PROMPT.format(rows=QUERY_ROWS, limit=MAX_QUERIES, sources="、".join(queryable)) + if queryable + else ENTITY_NO_QUERY_PROMPT + ) + + system = "\n\n".join( + part + for part in [ + ENTITY_ASSISTANT_SYSTEM, + query_prompt, + "\n".join([*source_lines, *entity_lines, *doc_lines, *pending_lines]), + ] + if part + ) + return EntityAssistantContext( + sources=sources, + workflows=workflows, + doc_bundles=doc_bundles, + pending_ops=clean_pending, + system=system, + ) + + +def _cell(value: Any, limit: int | None = None) -> Any: + if value is None or isinstance(value, (bool, int, float)): + return value + if isinstance(value, (datetime, date)): + text = value.isoformat() + elif isinstance(value, Decimal): + text = str(value) + elif isinstance(value, (bytes, bytearray)): + text = f"<{len(value)} 字节>" + else: + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, default=str) + if limit is not None and len(text) > limit: + return text[:limit] + "…" + return text + + +def _rows(rows: Any, n: int, limit: int | None) -> list[list[Any]]: + out: list[list[Any]] = [] + for row in (rows or [])[:n]: + cells = row if isinstance(row, list) else list(row.values()) if isinstance(row, dict) else [row] + out.append([_cell(v, limit) for v in cells]) + return out + + +class _EntityTools: + def __init__(self, ctx: EntityAssistantContext) -> None: + from app.engine.masking import SeenValues + + self.ctx = ctx + self.queries = 0 + self.seen_by_source: dict[str, Any] = {k: SeenValues() for k in ctx.sources} + + def inspect(self, keyword: Any, source_name: Any) -> str: + kw = str(keyword or "").strip().lower() + src_filter = str(source_name or "").strip().lower() + lines: list[str] = [] + + for skey, entry in self.ctx.sources.items(): + if src_filter and skey != src_filter and entry.row.id.lower() != src_filter: + continue + tables = ((entry.resolved.schema_cache or {}).get("tables") or {}) if isinstance(entry.resolved.schema_cache, Mapping) else {} + if not tables: + lines.append(f"数据源 {entry.row.name}：尚未探查表结构") + continue + matched_tables: list[str] = [] + for tname, tmeta in tables.items(): + if not isinstance(tmeta, Mapping): + continue + notes = entry.catalog_notes.get(tname) if isinstance(entry.catalog_notes.get(tname), Mapping) else {} + t_label_item = notes.get("label") if isinstance(notes.get("label"), Mapping) else {} + t_label = str(t_label_item.get("value") or "") if t_label_item.get("status") != "rejected" else "" + col_notes = notes.get("columns") if isinstance(notes.get("columns"), Mapping) else {} + cols = introspect.table_columns(dict(tmeta)) + + col_hits: list[str] = [] + for col in cols: + cname = str(col.get("name") or "") + is_masked = ( + cname.lower() in entry.masked + or f"{tname.lower()}.{cname.lower()}" in entry.masked + ) + cnote = col_notes.get(cname) if isinstance(col_notes.get(cname), Mapping) else {} + clabel = "" + cmeaning = "" + ccodes = "" + if not is_masked and isinstance(cnote, Mapping): + li = cnote.get("label") + if isinstance(li, Mapping) and li.get("status") != "rejected": + clabel = str(li.get("value") or "") + mi = cnote.get("meaning") + if isinstance(mi, Mapping) and mi.get("status") != "rejected": + cmeaning = str(mi.get("value") or "") + ci = cnote.get("codes") + if isinstance(ci, Mapping) and ci.get("status") != "rejected" and isinstance(ci.get("value"), Mapping): + pairs = [f"{k}={v}" for k, v in list(ci["value"].items())[:12]] + ccodes = f" 码值[{', '.join(pairs)}]" if pairs else "" + comment = "" if is_masked else str(col.get("comment") or "") + haystack = " ".join([tname, t_label, cname, clabel, cmeaning, comment, ccodes]).lower() + if not kw or kw in haystack: + desc_parts = [p for p in [clabel, cmeaning, comment] if p] + mask_tag = " [遮罩列]" if is_masked else "" + col_hits.append( + f" · {cname} ({col.get('type') or 'unknown'}){mask_tag}" + + (f"：{' / '.join(desc_parts)}" if desc_parts else "") + + ccodes + ) + if col_hits: + head = f" 表 {tname}" + (f"（{t_label}）" if t_label else "") + matched_tables.append(head + "\n" + "\n".join(col_hits[:25])) + if matched_tables: + lines.append(f"【数据源 {entry.row.name}】\n" + "\n".join(matched_tables[:15])) + + wf_hits: list[str] = [] + for wf in self.ctx.workflows: + hay = json.dumps(wf, ensure_ascii=False).lower() + if not kw or kw in hay: + in_txt = ", ".join(f"{i['name']}({i['label'] or i['type']})" for i in wf["inputs"]) + cd_txt = ", ".join(f"{c['card_id']}[{'/'.join(c['metrics'])}]" for c in wf["cards"]) + pub_tag = "已发布" if wf["published"] else "草稿" + wf_hits.append(f"- 工作流 {wf['name']}（workflow_id={wf['workflow_id']}，{pub_tag}）｜入口参数：{in_txt}｜口径卡：{cd_txt}") + if wf_hits: + lines.append("【口径工作流与入口参数】\n" + "\n".join(wf_hits[:20])) + + if not lines: + return f"未找到与「{keyword}」匹配的表列或口径参数，请尝试更短的词根或留空 keyword 查看全景概览。" + text = "\n\n".join(lines) + return text[:TOOL_CHARS] + ("…（已截断）" if len(text) > TOOL_CHARS else "") + + def read_doc(self, query: Any, doc_ref: Any, start_chunk: Any, limit: Any) -> str: + if not self.ctx.doc_bundles: + return ENTITY_TOOL_PROMPT["no_docs"] + hits = entity_docs.search_doc_bundles( + self.ctx.doc_bundles, + query=str(query or ""), + doc_ref=str(doc_ref or ""), + start_chunk=int(start_chunk) if isinstance(start_chunk, int) else 0, + limit=int(limit) if isinstance(limit, int) else 6, + ) + if not hits: + return "未检索到匹配的文档段落，可清空 query 按 start_chunk=0 顺序浏览。" + blocks = [ + f"【{h['citation']}（共 {h['total_chunks']} 段，ref={h['doc_ref']}）】\n{h['content']}" + for h in hits + ] + text = "\n\n".join(blocks) + return text[:TOOL_CHARS] + ("…（已截断）" if len(text) > TOOL_CHARS else "") + + async def query(self, source_name: Any, sql: Any) -> tuple[dict[str, Any] | None, str]: + prompts = ENTITY_TOOL_PROMPT + src_key = str(source_name or "").strip().lower() + entry = self.ctx.sources.get(src_key) + if entry is None or not entry.can_query: + allowed = "、".join(self.ctx.queryable_sources) or "无" + return None, prompts["query_disabled_source"].format(source=source_name or "", allowed=allowed) + if self.queries >= MAX_QUERIES: + return None, prompts["query_capped"].format(n=MAX_QUERIES) + clean_sql = sql.strip() if isinstance(sql, str) else "" + if not clean_sql: + return None, prompts["empty_sql"] + + self.queries += 1 + seen = self.seen_by_source[src_key] + mask = masked_columns(dict(entry.row.options or {}) if isinstance(entry.row.options, Mapping) else None) + known = await asyncio.to_thread(seen.pooled) + known = known if known.strong else None + error: str | None = None + result: Any = None + try: + result = await catalog_profile.run_single_query( + entry.resolved, + clean_sql, + settings=entry.settings, + max_rows=QUERY_ROWS, + kind=entry.kind, + ) + except catalog_profile.QueryFailed as exc: + error = exc.detail + except SnapshotTampered as exc: + error = QUERY_TAMPERED.format(name=entry.row.name, error=exc) + except ValueError as exc: + error = str(exc) + + if error is not None: + if mask or known is not None: + from app.engine.masking import mask_error_text + + error = mask_error_text(error, clean_sql, known) + event = { + "type": "query", + "source": entry.row.name, + "sql": clean_sql, + "columns": [], + "rows": [], + "row_count": 0, + "truncated": False, + "error": error, + } + return event, prompts["query_failed"].format(error=error) + + payload = result.to_payload() + if mask or known is not None: + from app.engine.masking import mask_query_result + + schema = getattr(entry.resolved, "schema_cache", None) + done = await asyncio.to_thread(mask_query_result, payload, mask, entry.kind, known, schema=schema) + payload = done.payload + await asyncio.to_thread(seen.add, done.secrets) + + columns = [str(c) for c in payload.get("columns") or []] + rows = payload.get("rows") or [] + event = { + "type": "query", + "source": entry.row.name, + "sql": str(payload.get("sql") or clean_sql), + "columns": columns, + "rows": _rows(rows, SHOWN_ROWS, None), + "row_count": int(payload.get("row_count") or len(rows)), + "truncated": bool(payload.get("truncated")), + "error": None, + } + given = { + "source": entry.row.name, + "columns": columns, + "rows": _rows(rows, QUERY_ROWS, CELL_MAX), + "row_count": event["row_count"], + "truncated": event["truncated"], + } + text = json.dumps(given, ensure_ascii=False, default=str) + while len(text) > TOOL_CHARS and given["rows"]: + given["rows"] = given["rows"][: max(1, len(given["rows"]) // 2)] if len(given["rows"]) > 1 else [] + text = json.dumps(given, ensure_ascii=False, default=str) + return event, text + + async def propose(self, operations: Any) -> tuple[dict[str, Any] | None, str]: + prompts = ENTITY_TOOL_PROMPT + if not isinstance(operations, list) or not operations: + return None, prompts["no_ops"] + async with SessionLocal() as session: + plan = await entity_patch.plan_entity_patch( + session, + operations, + pending_ops=self.ctx.pending_ops, + mode="confirmed", + ) + said: list[str] = [] + event: dict[str, Any] | None = None + if plan.changes: + for ch in plan.changes: + self.ctx.pending_ops.append(ch.op_payload) + event = { + "type": "proposal", + "changes": [ch.as_dict() for ch in plan.changes], + "expected_versions": plan.expected_versions, + } + said.append( + prompts["proposed"].format( + n=len(plan.changes), + items="；".join(ch.summary for ch in plan.changes), + ) + ) + else: + said.append(prompts["nothing"]) + if plan.problems: + said.append(prompts["dropped"].format(reasons="；".join(plan.problems))) + return event, "\n".join(said) + + +_SENTENCE_END = ("。", "！", "？", "；", "：", ".", "!", "?", ";", ":") + + +def _sse(obj: Mapping[str, Any]) -> str: + return f"data: {json.dumps(obj, ensure_ascii=False, default=str)}\n\n" + + +class _CallFailed(Exception): + def __init__(self, code: str, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +def _seconds(value: Any) -> str: + number = float(value) + return str(int(number)) if number.is_integer() else f"{number:g}" + + +def _transient(exc: BaseException) -> bool: + from app.core.errors import NETWORK, RATE_LIMIT, SERVER, classify + + return classify(exc) in (RATE_LIMIT, SERVER, NETWORK) + + +async def _pump(runner: Any, history: list[Any], queue: asyncio.Queue[tuple[str, Any]]) -> None: + try: + async for chunk in runner.astream(history): + await queue.put(("chunk", chunk)) + await queue.put(("end", None)) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 + await queue.put(("error", exc)) + + +async def _round(runner: Any, history: list[Any], model_id: str) -> AsyncIterator[tuple[str, Any]]: + from app.core.errors import TIMEOUT, classify, describe_exception + from app.engine.state import message_text + from app.providers.budget import cut_off + + loop = asyncio.get_running_loop() + attempt = 0 + while True: + queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue() + task = asyncio.ensure_future(_pump(runner, history, queue)) + deadline = loop.time() + float(catalog.MODEL_TIMEOUT_S) + message: Any = None + cut = False + started = False + failure: BaseException | None = None + try: + while True: + remaining = deadline - loop.time() + if remaining <= 0: + raise _CallFailed( + CODE_TIMEOUT, + ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S), place=DRAFT_MODEL_PLACE), + ) + try: + kind, value = await asyncio.wait_for(queue.get(), timeout=min(remaining, HEARTBEAT_S)) + except asyncio.TimeoutError: + yield "beat", None + continue + if kind == "end": + break + if kind == "error": + failure = value + break + message = value if message is None else message + value + cut = cut or bool(cut_off(value)) + text = message_text(value) + if text: + started = True + yield "text", text + finally: + if not task.done(): + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + if failure is None: + yield "message", (message, cut) + return + if not started and attempt < len(RETRY_DELAYS_S) and _transient(failure): + logger.info("实体助手：调用模型出错（%s），%s 秒后重试", type(failure).__name__, RETRY_DELAYS_S[attempt]) + await asyncio.sleep(RETRY_DELAYS_S[attempt]) + attempt += 1 + continue + if isinstance(failure, TimeoutError) or classify(failure) == TIMEOUT: + raise _CallFailed( + CODE_TIMEOUT, + ERROR_TIMEOUT.format(seconds=_seconds(catalog.MODEL_TIMEOUT_S), place=DRAFT_MODEL_PLACE), + ) from failure + raise _CallFailed( + CODE_MODEL, + ERROR_MODEL.format(model=model_id or "", reason=describe_exception(failure), place=DRAFT_MODEL_PLACE), + ) from failure + + +async def stream( + model: Any, + model_id: str, + ctx: EntityAssistantContext, + messages: Iterable[Mapping[str, Any]], +) -> AsyncIterator[str]: + """实体助手流式生成器：逐行产出 SSE data 事件与心跳注释。""" + from langchain_core.messages import SystemMessage, ToolMessage, message_chunk_to_message + + from app.providers.budget import cut_words + + history: list[Any] = [SystemMessage(content=ctx.system), *_history(messages)] + tools = _EntityTools(ctx) + usage = {"input_tokens": 0, "output_tokens": 0} + wrote = "" + tool_defs = ctx.tools() + names = [t["function"]["name"] for t in tool_defs] + try: + runner = model.bind_tools(tool_defs) + for round_no in range(MAX_ROUNDS): + fresh = True + message: Any = None + cut = False + async with contextlib.aclosing(_round(runner, history, model_id)) as rounds: + async for kind, value in rounds: + if kind == "beat": + yield ": keep-alive\n\n" + elif kind == "text": + if fresh and wrote.endswith(_SENTENCE_END): + yield _sse({"type": "text", "delta": "\n\n"}) + fresh = False + wrote = value + yield _sse({"type": "text", "delta": value}) + else: + message, cut = value + meta = getattr(message, "usage_metadata", None) or {} + usage["input_tokens"] += int(meta.get("input_tokens") or 0) + usage["output_tokens"] += int(meta.get("output_tokens") or 0) + if cut: + limit = getattr(model, "max_tokens", None) + yield _sse({ + "type": "error", + "code": CODE_TRUNCATED, + "message": ERROR_TRUNCATED.format(cut=cut_words("实体助手", limit), place=DRAFT_MODEL_PLACE), + }) + return + calls = list(getattr(message, "tool_calls", None) or []) if message is not None else [] + if not calls: + yield _sse({"type": "done", "usage": usage, "queries": tools.queries}) + return + if round_no == MAX_ROUNDS - 1: + yield _sse({"type": "error", "code": CODE_ROUNDS, "message": ERROR_ROUNDS.format(n=MAX_ROUNDS)}) + return + history.append(message_chunk_to_message(message)) + for call in calls: + name = str(call.get("name") or "") + args = call.get("args") if isinstance(call.get("args"), Mapping) else {} + call_id = str(call.get("id") or f"{name}-{round_no}-{len(history)}") + event: dict[str, Any] | None = None + if name == INSPECT_TOOL: + content = tools.inspect(args.get("keyword"), args.get("source_name")) + elif name == READ_DOC_TOOL: + content = tools.read_doc( + args.get("query"), + args.get("doc_ref"), + args.get("start_chunk"), + args.get("limit"), + ) + elif name == QUERY_TOOL: + event, content = await tools.query(args.get("source_name"), args.get("sql")) + elif name == PROPOSE_TOOL: + event, content = await tools.propose(args.get("operations")) + else: + content = ENTITY_TOOL_PROMPT["unknown_tool"].format(name=name, tools="、".join(names)) + if event is not None: + yield _sse(event) + if round_no == MAX_ROUNDS - 2: + content = f"{content}\n{ENTITY_TOOL_PROMPT['last_round']}" + history.append(ToolMessage(content=content, tool_call_id=call_id)) + except _CallFailed as exc: + yield _sse({"type": "error", "code": exc.code, "message": exc.message}) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 + from app.core.errors import describe_exception, first_line + + logger.exception("实体助手执行异常") + yield _sse({ + "type": "error", + "code": CODE_FAILED, + "message": ERROR_FAILED.format(line=first_line(exc) or describe_exception(exc)), + }) diff --git a/backend/app/data/entity_discovery.py b/backend/app/data/entity_discovery.py new file mode 100644 index 00000000..f32a6789 --- /dev/null +++ b/backend/app/data/entity_discovery.py @@ -0,0 +1,687 @@ +"""后台主动发现候选标准业务实体任务（E2：扫描发现候选实体）。 + +扫描范围： +1. 所选已启用数据源的表结构（主键、外键、标识/属性/状态列）与数据目录（中文名、含义、非遮罩码值）； +2. 已发布与草稿口径工作流的入口参数（entry_fields / input 节点）及口径卡； +3. 可选挂载的业务文档或跨系统编码对照表（知识库 Document 或临时上传附件）。 + +铁律： +- 发现结果一律经 entity_patch.apply_entity_patch(..., mode="proposed") 写入，所有表列绑定与口径入参映射强制标记为 + status="proposed", source="ai"，绝不直接参与线上跨源穿透，也绝不覆盖人工已确认、已验证或已驳回的项。 +""" +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +import re +import time +from collections.abc import Mapping, Sequence +from datetime import timezone +from typing import Any + +from sqlalchemy import select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.errors import describe_exception, first_line +from app.data import catalog, catalog_assistant, entities, entity_assistant, entity_docs, entity_patch, introspect, table_versions +from app.data.engine import masked_columns +from app.db.base import SessionLocal, utcnow +from app.db.models import DataSource, Workflow, WorkflowVersion +from app.db.models_ontology import EntityDiscoveryJob + +logger = logging.getLogger(__name__) + +OPEN_STATUSES = ("queued", "running") +NOTE_MAX = 1000 +MAX_DOC_CHARS_IN_DISCOVERY = 28_000 +MAX_SCHEMA_CHARS_IN_DISCOVERY = 36_000 + +DISCOVERY_BUSY_CODE = "discovery_job_busy" +DISCOVERY_FINISHED_CODE = "discovery_job_finished" + +INTERRUPTED_ERROR = "服务重启，候选实体扫描中断" +JOB_FAILED_TEXT = "扫描候选实体失败：{reason}" + +_FENCE_RE = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE) +_ACTIVE_TASKS: dict[str, asyncio.Task[Any]] = {} +_CREATE_LOCK = asyncio.Lock() + + +class DiscoveryBusy(Exception): + """已有正在排队或执行的候选实体扫描任务。""" + + +class DiscoveryFinished(Exception): + """要取消的候选实体扫描任务已经结束。""" + + +class DiscoveryEmptyScope(ValueError): + """扫描范围内没有任何可用的数据源表结构或业务文档。""" + + +DISCOVERY_SYSTEM_PROMPT = """你是 AgentLab 的「业务实体发现专家」。请根据给定的数据源表结构与数据目录、口径工作流入口参数以及业务文档/编码对照表，识别共用的「标准业务实体」（如产线、工厂、车间、仓库、门店、销售大区、渠道、客户等级、不良代码等），并输出增量操作 JSON。 + +硬性纪律： +1. 只输出一个合法 JSON 对象，格式为 {"operations": [...]}，不要输出任何额外解释文字。 +2. 若输入包含多个数据源，优先发现在多源之间形成桥接的核心业务对象；若仅包含单个数据源，则发现在该源多张业务表、口径入口参数或业务文档之间共用的核心业务维度实体。纯技术自增流水主键不要提议为业务实体。 +3. 表名、列名、数据源名、工作流 ID、参数名必须逐字取自输入材料，严禁编造不存在的字段。 +4. 标着「[遮罩列]」的敏感字段严禁为其生成 upsert_member 编码映射；标着「已驳回」的历史绑定严禁重复提议。 +5. 每条操作必须填写简明的中文 reason： + - 来自文档或对照表的写「文档《文档名》写着：……」； + - 来自表目录或字段语义对齐的写「目录语义对齐：……」。 + +operations 支持以下 4 种结构： +- {"op": "upsert_entity", "entity_key": "小写英文下划线标识（如 production_line）", "label": "中文名称", "description": "粒度与业务对齐说明", "aliases": ["同义词"], "reason": "依据"} +- {"op": "upsert_binding", "entity_key": "实体标识", "entity_label": "中文名称", "source_name": "数据源名称", "table_name": "物理表名", "column_name": "物理列名", "note": "字段角色备注", "reason": "依据"} +- {"op": "upsert_member", "entity_key": "实体标识", "entity_label": "中文名称", "value": "标准编码值", "label": "标准展示名", "aliases": ["别名"], "source_codes": {"数据源名称": "该源编码值"}, "reason": "依据"} +- {"op": "upsert_caliber_param", "entity_key": "实体标识", "entity_label": "中文名称", "workflow_id": "工作流ID", "card_id": "口径卡节点ID（可空串）", "param_name": "入口参数名", "note": "参数对齐说明", "reason": "依据"} +""" + +DISCOVERY_USER_PROMPT = """请扫描以下数据源结构、口径参数与业务文档，提取候选标准业务实体及其跨源绑定与编码映射，输出 {{"operations": [...]}} JSON： + +{body} +""" + + +def _iso(val: Any) -> str | None: + if val is None or not hasattr(val, "isoformat"): + return None + if getattr(val, "tzinfo", None) is None: + val = val.replace(tzinfo=timezone.utc) + return val.isoformat() + + +def job_to_dict(row: EntityDiscoveryJob) -> dict[str, Any]: + return { + "id": row.id, + "status": row.status, + "source_ids": list(row.source_ids or []), + "document_ids": list(row.document_ids or []), + "attachment_ids": list(row.attachment_ids or []), + "note": row.note or "", + "phase": row.phase or "", + "summary": dict(row.summary or {}), + "error": row.error, + "started_by": row.started_by, + "started_at": _iso(row.started_at), + "finished_at": _iso(row.finished_at), + "created_at": _iso(row.created_at), + "updated_at": _iso(row.updated_at), + } + + +def parse_discovery_operations(raw_text: str) -> list[dict[str, Any]]: + """从模型回复中提取 operations 列表。""" + text = (raw_text or "").strip() + if not text: + return [] + m = _FENCE_RE.search(text) + candidate = m.group(1).strip() if m else text + if not candidate.startswith(("{", "[")): + brace = candidate.find("{") + bracket = candidate.find("[") + starts = [i for i in (brace, bracket) if i >= 0] + if starts: + candidate = candidate[min(starts) :] + try: + parsed = json.loads(candidate) + except ValueError: + # 尝试截取最后一个匹配括号 + end_brace = candidate.rfind("}") + if end_brace > 0: + try: + parsed = json.loads(candidate[: end_brace + 1]) + except ValueError: + return [] + else: + return [] + if isinstance(parsed, list): + return [dict(x) for x in parsed if isinstance(x, Mapping)] + if isinstance(parsed, Mapping): + ops = parsed.get("operations") + if isinstance(ops, list): + return [dict(x) for x in ops if isinstance(x, Mapping)] + return [] + + +async def get_active_or_latest_job(session: AsyncSession) -> dict[str, Any]: + active = ( + await session.execute( + select(EntityDiscoveryJob) + .where(EntityDiscoveryJob.status.in_(OPEN_STATUSES)) + .order_by(EntityDiscoveryJob.created_at.desc()) + .limit(1) + ) + ).scalar_one_or_none() + latest = ( + await session.execute( + select(EntityDiscoveryJob) + .order_by(EntityDiscoveryJob.created_at.desc()) + .limit(1) + ) + ).scalar_one_or_none() + return { + "job": job_to_dict(active) if active is not None else None, + "latest": job_to_dict(latest) if latest is not None else None, + } + + +async def get_job(session: AsyncSession, job_id: str) -> EntityDiscoveryJob | None: + return await session.get(EntityDiscoveryJob, str(job_id or "").strip()) + + +async def _build_discovery_prompt_body( + session: AsyncSession, + *, + source_ids: Sequence[str], + document_ids: Sequence[str], + attachment_ids: Sequence[str], + note: str, +) -> tuple[str, dict[str, Any]]: + """返回 (prompt_body, 扫描范围量化统计)。""" + stmt = select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.name) + if source_ids: + stmt = stmt.where(DataSource.id.in_(list(source_ids))) + src_rows = list((await session.execute(stmt)).scalars()) + + schema_blocks: list[str] = [] + valid_sources = 0 + source_names: list[str] = [] + tables_scanned = 0 + columns_scanned = 0 + for row in src_rows: + try: + resolved = await table_versions.resolve_source(session, row) + except table_versions.SnapshotMissing: + resolved = row + tables = ((resolved.schema_cache or {}).get("tables") or {}) if isinstance(resolved.schema_cache, Mapping) else {} + if not tables: + continue + valid_sources += 1 + source_names.append(row.name) + opts = dict(row.options or {}) if isinstance(row.options, Mapping) else {} + masked = {c.lower() for c in masked_columns(opts)} + cat_entries = await catalog.read_catalog(session, resolved.id) + t_lines = [f"### 数据源：{row.name}（{row.kind}）"] + for tname, tmeta in list(tables.items())[:40]: + if not isinstance(tmeta, Mapping): + continue + entry = cat_entries.get(tname) + notes = entry.notes if entry else {} + t_label_item = notes.get("label") if isinstance(notes.get("label"), Mapping) else {} + t_label = str(t_label_item.get("value") or "") if t_label_item.get("status") != "rejected" else "" + col_notes = notes.get("columns") if isinstance(notes.get("columns"), Mapping) else {} + cols = introspect.table_columns(dict(tmeta)) + c_parts: list[str] = [] + for col in cols[:40]: + cname = str(col.get("name") or "") + is_masked = cname.lower() in masked or f"{tname.lower()}.{cname.lower()}" in masked + cnote = col_notes.get(cname) if isinstance(cnote := col_notes.get(cname), Mapping) else {} + measure = "" + clabel = "" + cmeaning = "" + ccodes = "" + if not is_masked and isinstance(cnote, Mapping): + mi = cnote.get("measure") + if isinstance(mi, Mapping) and mi.get("status") != "rejected": + measure = str(mi.get("value") or "") + # 跳过明显的纯度量浮点流量列，聚焦维度/标识/状态/外键或未分类列 + if measure in ("flow", "stock", "ratio"): + continue + li = cnote.get("label") + if isinstance(li, Mapping) and li.get("status") != "rejected": + clabel = str(li.get("value") or "") + mgi = cnote.get("meaning") + if isinstance(mgi, Mapping) and mgi.get("status") != "rejected": + cmeaning = str(mgi.get("value") or "") + ci = cnote.get("codes") + if isinstance(ci, Mapping) and ci.get("status") != "rejected" and isinstance(ci.get("value"), Mapping): + pairs = [f"{k}={v}" for k, v in list(ci["value"].items())[:15]] + ccodes = f" 码值[{', '.join(pairs)}]" if pairs else "" + comment = "" if is_masked else str(col.get("comment") or "") + tags = [p for p in [clabel, cmeaning, comment] if p] + pk_tag = "[PK]" if col.get("pk") else "" + mask_tag = "[遮罩列]" if is_masked else "" + c_parts.append( + f" - {cname} ({col.get('type') or ''}){pk_tag}{mask_tag}" + + (f"：{' / '.join(tags)}" if tags else "") + + ccodes + ) + if c_parts: + tables_scanned += 1 + columns_scanned += len(c_parts) + t_lines.append(f"- 表 {tname}" + (f"（{t_label}）" if t_label else "") + "\n" + "\n".join(c_parts)) + block = "\n".join(t_lines) + schema_blocks.append(block) + + schema_text = "\n\n".join(schema_blocks) + if len(schema_text) > MAX_SCHEMA_CHARS_IN_DISCOVERY: + schema_text = schema_text[:MAX_SCHEMA_CHARS_IN_DISCOVERY] + "\n…（表结构摘要已截断）" + + allowed_sources: set[str] | None = ( + {x for r in src_rows for x in (r.id.lower(), r.name.lower())} + if source_ids + else None + ) + + wf_rows = list((await session.execute(select(Workflow))).scalars()) + ver_rows = list((await session.execute(select(WorkflowVersion).where(WorkflowVersion.published_at.is_not(None)))).scalars()) + workflows = entity_assistant._extract_wf_summary(wf_rows, ver_rows, allowed_sources=allowed_sources) + wf_lines: list[str] = [] + if workflows: + wf_lines.append("## 口径工作流与入口参数") + for wf in workflows[:30]: + ins = ", ".join(f"{i['name']}({i['label'] or i['type']})" for i in wf["inputs"]) + cds = ", ".join(f"{c['card_id']}[{'/'.join(c['metrics'])}]" for c in wf["cards"]) + wf_lines.append(f"- {wf['name']}（workflow_id={wf['workflow_id']}）｜入口参数：{ins}｜口径卡：{cds}") + + ent_rows = await entities.list_entities(session) + ent_lines: list[str] = [] + if ent_rows: + ent_lines.append("## 既有标准业务实体（请复用同域已有 entity_key 增补绑定，勿重提已驳回项）") + other_keys: list[str] = [] + for erow in ent_rows[:60]: + b_list = [b for b in (erow.bindings or []) if isinstance(b, Mapping)] + if allowed_sources is not None and b_list: + b_scoped = [ + b for b in b_list + if str(b.get("source_name") or "").strip().lower() in allowed_sources + or str(b.get("source_id") or "").strip().lower() in allowed_sources + ] + if not b_scoped: + other_keys.append(f"{erow.key}({erow.label})") + continue + b_list = b_scoped + active_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") != "rejected"] + rej_b = [f"{b.get('source_name')}.{b.get('table_name')}.{b.get('column_name')}" for b in b_list if b.get("status") == "rejected"] + line = f"- {erow.label}（entity_key={erow.key}）｜已有绑定：{', '.join(active_b) or '无'}" + if rej_b: + line += f"｜已驳回：{', '.join(rej_b)}" + ent_lines.append(line) + if other_keys: + ent_lines.append(f"（其他未选定数据源已占用的实体标识：{', '.join(other_keys[:30])}）") + + bundles = await entity_docs.load_doc_bundles(session, attachment_ids=attachment_ids, document_ids=document_ids) + doc_blocks: list[str] = [] + used_doc_chars = 0 + for bundle in bundles: + piece_text = "\n\n".join(bundle.chunks) + remain = MAX_DOC_CHARS_IN_DISCOVERY - used_doc_chars + if remain <= 0: + break + clipped = piece_text[:remain] + used_doc_chars += len(clipped) + doc_blocks.append(f"### 文档《{bundle.title}》\n{clipped}") + + sections: list[str] = [] + if note.strip(): + sections.append(f"## 使用者补充的扫描重点\n{note.strip()}") + if schema_text: + sections.append(f"## 数据源与表目录\n{schema_text}") + if wf_lines: + sections.append("\n".join(wf_lines)) + if ent_lines: + sections.append("\n".join(ent_lines)) + if doc_blocks: + sections.append("## 业务文档与主数据对照表\n" + "\n\n".join(doc_blocks)) + + scope_stats: dict[str, Any] = { + "sources_scanned": valid_sources, + "source_names": source_names, + "tables_scanned": tables_scanned, + "columns_scanned": columns_scanned, + "workflows_scanned": len(workflows), + "documents_scanned": len(bundles), + "doc_titles": [b.title for b in bundles[:15]], + "chunks_scanned": sum(len(b.chunks) for b in bundles), + } + return "\n\n".join(sections), scope_stats + + +async def create_job( + session: AsyncSession, + *, + source_ids: Sequence[str] | None = None, + document_ids: Sequence[str] | None = None, + attachment_ids: Sequence[str] | None = None, + note: str | None = None, + actor: str | None = None, +) -> EntityDiscoveryJob: + clean_note = (note or "").strip() + if len(clean_note) > NOTE_MAX: + raise ValueError(f"扫描补充说明最多 {NOTE_MAX} 字，当前 {len(clean_note)} 字") + + # 建任务前先校验起草模型可用性 + await catalog_assistant.resolve_model(session) + + async with _CREATE_LOCK: + existing = ( + await session.execute( + select(EntityDiscoveryJob).where(EntityDiscoveryJob.status.in_(OPEN_STATUSES)).limit(1) + ) + ).scalar_one_or_none() + if existing is not None: + raise DiscoveryBusy("已有一个正在执行的候选实体扫描任务，请等待完成后再发起") + + clean_sources = list(dict.fromkeys(str(s).strip() for s in (source_ids or []) if str(s or "").strip())) + clean_docs = list(dict.fromkeys(str(d).strip() for d in (document_ids or []) if str(d or "").strip())) + clean_atts = list(dict.fromkeys(str(a).strip() for a in (attachment_ids or []) if str(a or "").strip())) + + prompt_body, scope_stats = await _build_discovery_prompt_body( + session, + source_ids=clean_sources, + document_ids=clean_docs, + attachment_ids=clean_atts, + note=clean_note, + ) + if scope_stats["sources_scanned"] == 0 and scope_stats["documents_scanned"] == 0 or not prompt_body.strip(): + raise DiscoveryEmptyScope("所选范围内没有已探查表结构的数据源或有效业务文档，请先探查表结构或添加文档") + + job = EntityDiscoveryJob( + status="queued", + source_ids=clean_sources, + document_ids=clean_docs, + attachment_ids=clean_atts, + note=clean_note, + phase="已加入扫描队列…", + summary={ + **scope_stats, + "step_index": 1, + "step_total": 4, + "progress_pct": 8, + }, + started_by=actor, + ) + session.add(job) + await session.commit() + await session.refresh(job) + + task = asyncio.create_task(_execute_job(job.id)) + _ACTIVE_TASKS[job.id] = task + task.add_done_callback(lambda _t, jid=job.id: _ACTIVE_TASKS.pop(jid, None)) + return job + + +async def _flush_job_progress(job_id: str, *, phase: str, summary: Mapping[str, Any]) -> None: + async with SessionLocal() as session: + await session.execute( + update(EntityDiscoveryJob) + .where(EntityDiscoveryJob.id == job_id, EntityDiscoveryJob.status.in_(OPEN_STATUSES)) + .values(phase=phase, summary=dict(summary)) + ) + await session.commit() + + +async def _execute_job(job_id: str) -> None: + from langchain_core.messages import HumanMessage, SystemMessage + + from app.engine.state import message_text + + try: + async with SessionLocal() as session: + job = await session.get(EntityDiscoveryJob, job_id) + if job is None or job.status not in OPEN_STATUSES: + return + job.status = "running" + job.started_at = utcnow() + base_summary = dict(job.summary or {}) + base_summary.update({"step_index": 1, "step_total": 4, "progress_pct": 18}) + job.summary = base_summary + job.phase = "步骤 1/4：正在读取数据源表目录与口径参数…" + await session.commit() + + prompt_body, scope_stats = await _build_discovery_prompt_body( + session, + source_ids=job.source_ids or [], + document_ids=job.document_ids or [], + attachment_ids=job.attachment_ids or [], + note=job.note or "", + ) + model, model_id = await catalog_assistant.resolve_model(session) + doc_note = ( + f"、{scope_stats['documents_scanned']} 份文档（{scope_stats['chunks_scanned']} 段）" + if scope_stats["documents_scanned"] > 0 + else "" + ) + live_summary: dict[str, Any] = { + **scope_stats, + "model_id": model_id, + "step_index": 3, + "step_total": 4, + "progress_pct": 38, + "model_chars": 0, + "ops_detected": 0, + "candidate_keys": [], + } + job.summary = live_summary + job.phase = ( + f"步骤 3/4：已汇总 {scope_stats['sources_scanned']} 个源（{scope_stats['tables_scanned']} 张表）{doc_note}，" + "正在调用模型识别候选实体…" + ) + await session.commit() + + msgs = [ + SystemMessage(content=DISCOVERY_SYSTEM_PROMPT), + HumanMessage(content=DISCOVERY_USER_PROMPT.format(body=prompt_body)), + ] + + first_token_seen = False + call_started = time.monotonic() + + async def _thinking_heartbeat() -> None: + nonlocal live_summary + while not first_token_seen: + await asyncio.sleep(2.0) + if first_token_seen: + break + waited = int(time.monotonic() - call_started) + pct = min(52, 38 + waited // 2) + live_summary = { + **live_summary, + "step_index": 3, + "step_total": 4, + "progress_pct": pct, + } + await _flush_job_progress( + job_id, + phase="步骤 3/4：模型正在深度分析表目录与业务关系，等待生成结果…", + summary=live_summary, + ) + + hb_task = asyncio.create_task(_thinking_heartbeat()) + try: + if hasattr(model, "astream") and not getattr(model, "invoke_reply", ""): + pieces: list[str] = [] + last_flush = time.monotonic() + async for chunk in model.astream(msgs): + piece = message_text(chunk) + if piece: + first_token_seen = True + pieces.append(piece) + now_mono = time.monotonic() + if now_mono - last_flush >= 0.8 and pieces: + last_flush = now_mono + so_far = "".join(pieces) + op_count = len(re.findall(r'"op"\s*:', so_far)) + found_keys = list(dict.fromkeys(re.findall(r'"entity_key"\s*:\s*"([a-z][a-z0-9_]{0,63})"', so_far))) + pct = min(86, 53 + min(33, len(so_far) // 80)) + live_summary = { + **live_summary, + "step_index": 3, + "step_total": 4, + "progress_pct": pct, + "model_chars": len(so_far), + "ops_detected": op_count, + "candidate_keys": found_keys[:15], + } + phase_txt = ( + f"步骤 3/4：模型正在识别候选实体（已识别 {op_count} 项操作，输出 {len(so_far)} 字）…" + if op_count > 0 + else f"步骤 3/4：模型正在流式输出候选结果（已输出 {len(so_far)} 字）…" + ) + await _flush_job_progress(job_id, phase=phase_txt, summary=live_summary) + raw_text = "".join(pieces) + else: + reply = await model.ainvoke(msgs) + first_token_seen = True + raw_text = message_text(reply) + finally: + first_token_seen = True + hb_task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await hb_task + + ops = parse_discovery_operations(raw_text) + found_keys_final = list( + dict.fromkeys( + str(op.get("entity_key") or "").strip().lower() + for op in ops + if isinstance(op, Mapping) and str(op.get("entity_key") or "").strip() + ) + ) + + async with SessionLocal() as session: + job = await session.get(EntityDiscoveryJob, job_id) + if job is None or job.status == "cancelled": + return + job.phase = f"步骤 4/4：正在核验 {len(ops)} 项候选的表列真实性并写入待确认列表…" + job.summary = { + **live_summary, + "step_index": 4, + "step_total": 4, + "progress_pct": 90, + "model_chars": len(raw_text), + "ops_detected": len(ops), + "candidate_keys": found_keys_final[:15], + } + await session.commit() + + if ops: + applied = await entity_patch.apply_entity_patch( + session, + ops, + mode="proposed", + actor=job.started_by, + ) + else: + applied = { + "applied_count": 0, + "created_entities": 0, + "updated_entities": 0, + "bindings_changed": 0, + "caliber_params_changed": 0, + "members_changed": 0, + "entities": [], + "problems": [], + } + touched_entities = [ + {"id": e["id"], "key": e["key"], "label": e["label"]} + for e in (applied.get("entities") or [])[:30] + if isinstance(e, Mapping) and e.get("id") and e.get("key") + ] + problems = [str(p) for p in (applied.get("problems") or [])] + job.status = "succeeded" + job.phase = "扫描完成" + job.finished_at = utcnow() + job.summary = { + **scope_stats, + "model_id": model_id, + "step_index": 4, + "step_total": 4, + "progress_pct": 100, + "model_chars": len(raw_text), + "ops_detected": len(ops), + "candidate_keys": found_keys_final[:15], + "applied_count": applied["applied_count"], + "created_entities": applied["created_entities"], + "updated_entities": applied["updated_entities"], + "proposed_bindings": applied["bindings_changed"], + "proposed_params": applied["caliber_params_changed"], + "added_members": applied["members_changed"], + "skipped_count": len(problems), + "skipped_reasons": problems[:15], + "touched_entities": touched_entities, + } + await session.commit() + except asyncio.CancelledError: + async with SessionLocal() as session: + job = await session.get(EntityDiscoveryJob, job_id) + if job is not None and job.status in OPEN_STATUSES: + job.status = "cancelled" + job.phase = "已停止扫描" + job.finished_at = utcnow() + await session.commit() + raise + except Exception as exc: # noqa: BLE001 + logger.exception("候选实体扫描任务 %s 失败", job_id) + async with SessionLocal() as session: + job = await session.get(EntityDiscoveryJob, job_id) + if job is not None and job.status in OPEN_STATUSES: + job.status = "failed" + job.phase = "扫描失败" + job.error = JOB_FAILED_TEXT.format(reason=first_line(exc) or describe_exception(exc)) + job.finished_at = utcnow() + await session.commit() + + +async def cancel_job(session: AsyncSession, job_id: str) -> EntityDiscoveryJob: + job = await session.get(EntityDiscoveryJob, str(job_id or "").strip()) + if job is None: + raise LookupError("候选实体扫描任务不存在") + if job.status not in OPEN_STATUSES: + raise DiscoveryFinished("该候选实体扫描任务已经结束") + task = _ACTIVE_TASKS.get(job.id) + if task is not None and not task.done(): + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await asyncio.wait_for(asyncio.shield(task), timeout=2.0) + await session.refresh(job) + if job.status in OPEN_STATUSES: + job.status = "cancelled" + job.phase = "已停止扫描" + job.finished_at = utcnow() + await session.commit() + await session.refresh(job) + return job + + +async def startup(*, live: Any = None) -> int: + """服务启动或清理孤儿任务时，将上次残留的未结束扫描任务标为 interrupted。""" + _ = live + running_ids = set(_ACTIVE_TASKS.keys()) + async with SessionLocal() as session: + rows = list( + ( + await session.execute( + select(EntityDiscoveryJob.id).where(EntityDiscoveryJob.status.in_(OPEN_STATUSES)) + ) + ).scalars() + ) + stale_ids = [jid for jid in rows if jid not in running_ids] + if stale_ids: + await session.execute( + update(EntityDiscoveryJob) + .where(EntityDiscoveryJob.id.in_(stale_ids), EntityDiscoveryJob.status.in_(OPEN_STATUSES)) + .values( + status="interrupted", + phase="扫描中断", + error=INTERRUPTED_ERROR, + finished_at=utcnow(), + ) + ) + await session.commit() + return len(stale_ids) + + +async def shutdown() -> None: + """服务关停时取消进程内正在执行的扫描任务并标为 interrupted。""" + tasks = list(_ACTIVE_TASKS.values()) + for task in tasks: + if not task.done(): + task.cancel() + for task in tasks: + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + await startup() diff --git a/backend/app/data/entity_docs.py b/backend/app/data/entity_docs.py new file mode 100644 index 00000000..31e3f705 --- /dev/null +++ b/backend/app/data/entity_docs.py @@ -0,0 +1,334 @@ +"""标准业务实体维护的业务文档与主数据对照表附件服务（E1）。 + +支持两类文档来源： +1. 临时上传附件（POST /api/ontology/entities/attachments）：复用 memory/parsing.extract(..., tables=True) + 提取 .md / .txt / .docx / .pdf / .pptx / .html 以及 .xlsx / .csv / .tsv 编码对照表正文， + 按 kb.chunk_text 切块（跨块自动保留 Markdown 表头）并存入内容寻址工件（entity_doc_attachment）； + 若勾选 save_to_kb=True，同步创建知识库 Document 并在后台完成切块向量索引，返回 document_id 供后续直接复用。 +2. 已有知识库文档（Document + Chunk）：若文档仍在后台切块中，自动回退按 doc.content 即时切块，无需等待向量计算。 +""" +from __future__ import annotations + +import asyncio +import logging +import re +from collections.abc import Sequence +from dataclasses import dataclass +from typing import Any + +from fastapi import BackgroundTasks, UploadFile +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core import artifact_store +from app.core.config import settings +from app.core.errors import explain, raw as raw_err +from app.db.base import SessionLocal +from app.db.models import Chunk, Document +from app.memory import kb +from app.memory.parsing import UnsupportedDocument, extract + +logger = logging.getLogger(__name__) + +ATTACHMENT_KIND = "entity_doc_attachment" +MAX_DOC_SEARCH_LIMIT = 15 + +#: 持有后台切块任务引用，防止事件循环提前回收 +_KB_TASKS: set[asyncio.Task[Any]] = set() + +_TOKEN_RE = re.compile(r"[A-Za-z0-9_]+|[\u4e00-\u9fff]{2,}") + + +class AttachmentTooLarge(ValueError): + """上传附件超过系统大小上限。""" + + +class AttachmentInvalid(ValueError): + """上传附件格式不支持或无有效文本。""" + + +@dataclass(slots=True) +class EntityDocBundle: + ref_id: str + kind: str # "attachment" | "kb" + title: str + mime: str + char_count: int + chunks: list[str] + document_id: str | None = None + + +async def _index_kb_document(doc_id: str) -> None: + """后台对新建的知识库文档切块并建立向量与倒排索引。""" + async with SessionLocal() as session: + doc = await session.get(Document, doc_id) + if not doc: + return + try: + await kb.process_document(session, doc) + except Exception as exc: # noqa: BLE001 + logger.warning("实体文档存入知识库索引失败（%s）：%s", doc_id, raw_err(exc)) + await session.rollback() + doc = await session.get(Document, doc_id) + if doc: + reason, hint = explain(exc) + doc.status = "failed" + doc.error = (f"文档分块或生成向量时出错：{reason}" + (f"。{hint}" if hint else ""))[:500] + await session.commit() + + +async def read_upload_limited(file: UploadFile) -> bytes: + limit = settings.max_upload_mb * 1024 * 1024 + pieces: list[bytes] = [] + total = 0 + while chunk := await file.read(1024 * 1024): + total += len(chunk) + if total > limit: + raise AttachmentTooLarge(f"文件超过大小上限（{settings.max_upload_mb} MB）") + pieces.append(chunk) + return b"".join(pieces) + + +async def ingest_entity_attachment( + session: AsyncSession, + *, + raw_bytes: bytes, + filename: str, + mime: str = "", + save_to_kb: bool = False, + collection: str = "default", + background: BackgroundTasks | None = None, +) -> dict[str, Any]: + """解析上传的业务文档或主数据表格，生成附件工件并按需存入知识库。""" + clean_name = (filename or "上传文档").strip() or "上传文档" + clean_mime = (mime or "text/plain").strip() or "text/plain" + if not raw_bytes: + raise AttachmentInvalid("上传文件为空，请选择包含有效内容的文件") + try: + content = extract(raw_bytes, clean_name, clean_mime, tables=True) + except UnsupportedDocument as exc: + raise AttachmentInvalid(str(exc)) from None + + content = (content or "").strip() + if not content: + raise AttachmentInvalid("未能从文件中提取出有效文字或表格内容") + + chunks = kb.chunk_text(content) + if not chunks: + chunks = [content[:4000]] + + doc_id: str | None = None + clean_collection = (collection or "default").strip() or "default" + if save_to_kb: + doc = await kb.create_document( + session, + collection=clean_collection, + title=clean_name, + content=content, + source=clean_name, + mime=clean_mime, + meta={"origin": "entity_agent"}, + status="processing", + ) + doc_id = doc.id + if background is not None: + background.add_task(_index_kb_document, doc.id) + else: + task = asyncio.create_task(_index_kb_document(doc.id)) + _KB_TASKS.add(task) + task.add_done_callback(_KB_TASKS.discard) + + payload = { + "title": clean_name, + "filename": clean_name, + "mime": clean_mime, + "char_count": len(content), + "chunks": chunks, + "document_id": doc_id, + } + attachment_id = await artifact_store.put_json( + payload, + kind=ATTACHMENT_KIND, + meta={"title": clean_name, "document_id": doc_id}, + ) + return { + "attachment_id": attachment_id, + "title": clean_name, + "mime": clean_mime, + "char_count": len(content), + "chunk_count": len(chunks), + "saved_to_kb": bool(doc_id), + "document_id": doc_id, + "collection": clean_collection if doc_id else None, + } + + +async def load_doc_bundles( + session: AsyncSession, + *, + attachment_ids: Sequence[str] = (), + document_ids: Sequence[str] = (), +) -> list[EntityDocBundle]: + """统一加载临时附件与知识库文档的切块，按引用去重。""" + bundles: list[EntityDocBundle] = [] + seen_doc_ids: set[str] = set() + seen_att_ids: set[str] = set() + + for raw_att in attachment_ids or (): + att_id = str(raw_att or "").strip() + if not att_id or att_id in seen_att_ids: + continue + seen_att_ids.add(att_id) + try: + loaded = artifact_store.load(att_id) + except ValueError: + loaded = None + if not isinstance(loaded, dict) or not isinstance(loaded.get("chunks"), list): + continue + chunks = [str(c) for c in loaded["chunks"] if isinstance(c, str) and c.strip()] + if not chunks: + continue + linked_doc_id = str(loaded.get("document_id") or "").strip() or None + if linked_doc_id: + seen_doc_ids.add(linked_doc_id) + bundles.append( + EntityDocBundle( + ref_id=att_id, + kind="attachment", + title=str(loaded.get("title") or "上传附件"), + mime=str(loaded.get("mime") or "text/plain"), + char_count=int(loaded.get("char_count") or sum(len(c) for c in chunks)), + chunks=chunks, + document_id=linked_doc_id, + ) + ) + + clean_doc_ids = [ + str(d).strip() + for d in (document_ids or ()) + if str(d or "").strip() and str(d).strip() not in seen_doc_ids + ] + if clean_doc_ids: + seen_doc_ids.update(clean_doc_ids) + docs = { + row.id: row + for row in (await session.execute(select(Document).where(Document.id.in_(clean_doc_ids)))).scalars() + } + chunk_rows = list( + ( + await session.execute( + select(Chunk) + .where(Chunk.document_id.in_(clean_doc_ids)) + .order_by(Chunk.document_id.asc(), Chunk.ordinal.asc()) + ) + ).scalars() + ) + by_doc: dict[str, list[str]] = {} + for crow in chunk_rows: + if crow.content and crow.content.strip(): + by_doc.setdefault(crow.document_id, []).append(crow.content) + + for doc_id in clean_doc_ids: + doc = docs.get(doc_id) + if doc is None: + continue + pieces = by_doc.get(doc_id) or kb.chunk_text(doc.content or "") + if not pieces and (doc.content or "").strip(): + pieces = [(doc.content or "").strip()[:4000]] + if not pieces: + continue + bundles.append( + EntityDocBundle( + ref_id=doc.id, + kind="kb", + title=doc.title or doc.source or "知识库文档", + mime=doc.mime or "text/plain", + char_count=len(doc.content or ""), + chunks=pieces, + document_id=doc.id, + ) + ) + return bundles + + +def _keywords(query: str) -> list[str]: + q = (query or "").strip().lower() + if not q: + return [] + tokens = [m.group(0) for m in _TOKEN_RE.finditer(q)] + out: list[str] = [] + seen: set[str] = set() + for tok in [q, *tokens]: + if tok and tok not in seen: + seen.add(tok) + out.append(tok) + return out + + +def search_doc_bundles( + bundles: Sequence[EntityDocBundle], + *, + query: str = "", + doc_ref: str = "", + start_chunk: int = 0, + limit: int = 6, +) -> list[dict[str, Any]]: + """按关键词或分段序号检索已挂载文档片段，返回带文档标题与段号的原文片段。""" + cap = max(1, min(int(limit or 6), MAX_DOC_SEARCH_LIMIT)) + offset = max(0, int(start_chunk or 0)) + ref_norm = (doc_ref or "").strip().lower() + + scoped = [ + b + for b in bundles + if not ref_norm + or b.ref_id.lower() == ref_norm + or (b.document_id and b.document_id.lower() == ref_norm) + or ref_norm in b.title.lower() + ] + if not scoped: + scoped = list(bundles) + + terms = _keywords(query) + if not terms: + out: list[dict[str, Any]] = [] + for bundle in scoped: + for idx in range(offset, len(bundle.chunks)): + out.append({ + "doc_ref": bundle.ref_id, + "title": bundle.title, + "chunk_index": idx, + "total_chunks": len(bundle.chunks), + "citation": f"文档《{bundle.title}》第 {idx + 1} 段", + "content": bundle.chunks[idx], + }) + if len(out) >= cap: + return out + return out + + scored: list[tuple[int, int, int, EntityDocBundle, str]] = [] + for b_idx, bundle in enumerate(scoped): + for c_idx, chunk in enumerate(bundle.chunks): + if c_idx < offset: + continue + low = chunk.lower() + score = 0 + for t_idx, term in enumerate(terms): + hits = low.count(term) + if hits: + weight = 5 if t_idx == 0 else (3 if len(term) >= 3 else 1) + score += hits * weight + if score > 0: + scored.append((-score, b_idx, c_idx, bundle, chunk)) + scored.sort(key=lambda item: (item[0], item[1], item[2])) + return [ + { + "doc_ref": bundle.ref_id, + "title": bundle.title, + "chunk_index": c_idx, + "total_chunks": len(bundle.chunks), + "citation": f"文档《{bundle.title}》第 {c_idx + 1} 段", + "content": chunk, + } + for _neg, _b, c_idx, bundle, chunk in scored[:cap] + ] diff --git a/backend/app/data/entity_patch.py b/backend/app/data/entity_patch.py new file mode 100644 index 00000000..aa99730f --- /dev/null +++ b/backend/app/data/entity_patch.py @@ -0,0 +1,841 @@ +"""标准业务实体的受控增量补丁规划器与落库器（E1）。 + +供交互式「实体助手」（Entity Copilot）与后台「扫描发现候选实体」共用同一套核验与合并内核： +1. 物理真实性硬校验：表列绑定必须在已启用数据源的 schema_cache 中真实存在；口径参数映射必须在对应工作流入口节点中真实存在。 +2. 敏感遮罩隔离：遮罩列（masked_columns）严禁写入或更新 members.source_codes 枚举字典。 +3. 剥离自封审批态：模型传入的 status / approved / verified / actor / version 一律忽略重算。 +4. 双模式落库： + - mode="confirmed"（人工在实体助手点「保存修改」）：严格校验 expected_versions CAS 乐观锁，绑定与参数落为 confirmed / human。 + - mode="proposed"（后台主动发现）：新增绑定与参数一律落为 proposed / ai，绝不覆盖人工已确认、已验证或已驳回的项。 +""" +from __future__ import annotations + +import copy +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Literal + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.artifact_store import canonical_json, content_hash +from app.data import entities, introspect +from app.data.engine import masked_columns +from app.db.base import new_id, utcnow +from app.db.models import DataSource, Workflow, WorkflowVersion +from app.db.models_ontology import CanonicalEntity + +OP_TYPES = ( + "upsert_entity", + "upsert_binding", + "remove_binding", + "upsert_member", + "upsert_caliber_param", +) + +_FORBIDDEN_KEYS = frozenset({ + "status", "approve", "approved", "approval", "approver", "verified", "verify", + "actor", "role", "review", "reviewed", "reviewed_by", "updated_by", "created_by", "version", +}) + + +@dataclass(slots=True) +class EntityPatchChange: + id: str + op: str + entity_key: str + entity_label: str + summary: str + before: Any + after: Any + reason: str + op_payload: dict[str, Any] + + def as_dict(self) -> dict[str, Any]: + return { + "id": self.id, + "op": self.op, + "entity_key": self.entity_key, + "entity_label": self.entity_label, + "summary": self.summary, + "before": self.before, + "after": self.after, + "reason": self.reason, + "op_payload": self.op_payload, + } + + +@dataclass(slots=True) +class EntityPatchPlan: + changes: list[EntityPatchChange] = field(default_factory=list) + problems: list[str] = field(default_factory=list) + expected_versions: dict[str, int] = field(default_factory=dict) + + +@dataclass(slots=True) +class _ValidationEnv: + sources_by_norm: dict[str, DataSource] + masked_by_source: dict[str, set[str]] + workflows_by_id: dict[str, Workflow] + workflows_by_name: dict[str, Workflow] + workflow_inputs: dict[str, set[str]] + workflow_cards: dict[str, set[str]] + entities_by_key: dict[str, CanonicalEntity] + + +def _strip_forbidden(raw: Mapping[str, Any]) -> dict[str, Any]: + return {k: v for k, v in raw.items() if k not in _FORBIDDEN_KEYS} + + +def _extract_graph_inputs_and_cards(graph: Mapping[str, Any] | None) -> tuple[set[str], set[str]]: + inputs: set[str] = set() + cards: set[str] = set() + if not isinstance(graph, Mapping): + return inputs, cards + for node in graph.get("nodes") or []: + if not isinstance(node, Mapping): + continue + ntype = str(node.get("type") or "").strip() + nid = str(node.get("id") or "").strip() + if ntype == "metrics" and nid: + cards.add(nid) + if ntype == "input": + data = node.get("data") if isinstance(node.get("data"), Mapping) else {} + cfg = data.get("config") if isinstance(data.get("config"), Mapping) else ( + node.get("config") if isinstance(node.get("config"), Mapping) else {} + ) + for f in cfg.get("fields") or []: + if isinstance(f, Mapping): + fname = str(f.get("name") or "").strip() + if fname: + inputs.add(fname) + return inputs, cards + + +async def _load_env(session: AsyncSession) -> _ValidationEnv: + from app.data import table_versions + + src_rows = list( + (await session.execute(select(DataSource).where(DataSource.enabled.is_(True)).order_by(DataSource.name))).scalars() + ) + sources_by_norm: dict[str, Any] = {} + masked_by_source: dict[str, set[str]] = {} + for row in src_rows: + try: + resolved = await table_versions.resolve_source(session, row) + except table_versions.SnapshotMissing: + resolved = row + key = row.name.strip().lower() + sources_by_norm[key] = resolved + sources_by_norm[row.id.strip().lower()] = resolved + masked_by_source[row.name.strip().lower()] = { + c.strip().lower() for c in masked_columns(row.options if isinstance(row.options, dict) else None) + } + + wf_rows = list((await session.execute(select(Workflow))).scalars()) + ver_rows = list( + ( + await session.execute( + select(WorkflowVersion).where(WorkflowVersion.published_at.is_not(None)) + ) + ).scalars() + ) + workflows_by_id: dict[str, Workflow] = {} + name_counts: dict[str, int] = {} + for wf in wf_rows: + workflows_by_id[wf.id] = wf + if wf.name: + nkey = wf.name.strip().lower() + name_counts[nkey] = name_counts.get(nkey, 0) + 1 + workflows_by_name = { + wf.name.strip().lower(): wf + for wf in wf_rows + if wf.name and name_counts.get(wf.name.strip().lower()) == 1 + } + + workflow_inputs: dict[str, set[str]] = {} + workflow_cards: dict[str, set[str]] = {} + for wf in wf_rows: + ins, cds = _extract_graph_inputs_and_cards(wf.graph if isinstance(wf.graph, dict) else None) + workflow_inputs.setdefault(wf.id, set()).update(ins) + workflow_cards.setdefault(wf.id, set()).update(cds) + for ver in ver_rows: + ins, cds = _extract_graph_inputs_and_cards(ver.graph if isinstance(ver.graph, dict) else None) + workflow_inputs.setdefault(ver.workflow_id, set()).update(ins) + workflow_cards.setdefault(ver.workflow_id, set()).update(cds) + + ent_rows = await entities.list_entities(session) + entities_by_key = {row.key.strip().lower(): row for row in ent_rows} + + return _ValidationEnv( + sources_by_norm=sources_by_norm, + masked_by_source=masked_by_source, + workflows_by_id=workflows_by_id, + workflows_by_name=workflows_by_name, + workflow_inputs=workflow_inputs, + workflow_cards=workflow_cards, + entities_by_key=entities_by_key, + ) + + +def _is_column_masked(masked_set: set[str], table_name: str, column_name: str) -> bool: + col_low = column_name.strip().lower() + tbl_low = table_name.strip().lower() + short_tbl = tbl_low.rsplit(".", 1)[-1] + return ( + col_low in masked_set + or f"{tbl_low}.{col_low}" in masked_set + or f"{short_tbl}.{col_low}" in masked_set + ) + + +def _resolve_table_and_column( + source: DataSource, + raw_table: str, + raw_column: str, +) -> tuple[str | None, str | None, str | None]: + """返回 (规范表名, 规范列名, 错误说明)。""" + cache = source.schema_cache if isinstance(source.schema_cache, Mapping) else {} + tables = cache.get("tables") if isinstance(cache.get("tables"), Mapping) else {} + if not tables: + return None, None, f"数据源「{source.name}」尚未探查表结构，无法核实表「{raw_table}」与列「{raw_column}」" + + t_probe = raw_table.strip() + matched_table: str | None = None + meta: Mapping[str, Any] | None = None + if t_probe in tables: + matched_table, meta = t_probe, tables[t_probe] + else: + t_low = t_probe.lower() + t_short = t_low.rsplit(".", 1)[-1] + for tname, tmeta in tables.items(): + if not isinstance(tmeta, Mapping): + continue + qual = str(tmeta.get("qualified") or tname).strip() + if tname.lower() == t_low or qual.lower() == t_low or tname.lower() == t_short: + matched_table, meta = tname, tmeta + break + if meta is None or matched_table is None: + sample = "、".join(list(tables.keys())[:15]) + return None, None, f"数据源「{source.name}」中不存在表「{raw_table}」（已探查表如：{sample}）" + + cols = introspect.table_columns(dict(meta)) + c_low = raw_column.strip().lower() + for col in cols: + cname = str(col.get("name") or "").strip() + if cname.lower() == c_low: + return matched_table, cname, None + col_sample = "、".join(str(c.get("name")) for c in cols[:20]) + return None, None, f"数据源「{source.name}」表「{matched_table}」中不存在列「{raw_column}」（可用列：{col_sample}）" + + +def _working_from_row(row: CanonicalEntity) -> dict[str, Any]: + return { + "id": row.id, + "key": row.key, + "label": row.label, + "description": str(row.description or ""), + "aliases": list(row.aliases or []), + "members": [copy.deepcopy(dict(m)) for m in (row.members or []) if isinstance(m, Mapping)], + "bindings": [copy.deepcopy(dict(b)) for b in (row.bindings or []) if isinstance(b, Mapping)], + "caliber_params": [copy.deepcopy(dict(p)) for p in (row.caliber_params or []) if isinstance(p, Mapping)], + "version": int(row.version or 1), + "is_new": False, + } + + +def _change_id(op: str, entity_key: str, identity: str) -> str: + return content_hash(canonical_json({"op": op, "entity_key": entity_key, "id": identity}))[:16] + + +def _normalize_single_op( + raw_op: Any, + *, + index: int, + env: _ValidationEnv, + working: dict[str, dict[str, Any]], + mode: Literal["confirmed", "proposed"], +) -> tuple[EntityPatchChange | None, str | None]: + if not isinstance(raw_op, Mapping): + return None, f"第 {index + 1} 项操作需要是对象" + cleaned = _strip_forbidden(raw_op) + op = str(cleaned.get("op") or "").strip() + if op not in OP_TYPES: + return None, f"第 {index + 1} 项操作类型「{op}」无效，支持：{', '.join(OP_TYPES)}" + if mode == "proposed" and op == "remove_binding": + return None, "后台发现模式不允许删除既有表列绑定（remove_binding）" + + raw_key = cleaned.get("entity_key") or cleaned.get("key") + try: + entity_key = entities.normalize_entity_key(raw_key, field="entity_key") + except entities.EntityInvalid as exc: + return None, f"第 {index + 1} 项：{exc.message}" + + reason = str(cleaned.get("reason") or "").strip()[:500] + note = str(cleaned.get("note") or "").strip()[:500] + + # 初始化当前实体工作副本 + state = working.get(entity_key) + if state is None and entity_key in env.entities_by_key: + state = _working_from_row(env.entities_by_key[entity_key]) + working[entity_key] = state + + if op == "upsert_entity": + raw_label = str(cleaned.get("label") or cleaned.get("entity_label") or "").strip() + if state is None and not raw_label: + return None, f"新建标准实体「{entity_key}」缺少中文名称（label）" + label = raw_label or (state["label"] if state else entity_key) + if len(label) > 100: + return None, f"标准实体「{entity_key}」名称不能超过 100 个字符" + desc = str(cleaned.get("description") if "description" in cleaned else (state["description"] if state else "") or "").strip() + raw_aliases = cleaned.get("aliases") + if raw_aliases is None: + new_aliases = list(state["aliases"]) if state else [] + elif isinstance(raw_aliases, list): + merged = list(state["aliases"]) if (state and mode == "proposed") else [] + seen_a = {a.lower() for a in merged} + for item in raw_aliases: + if isinstance(item, str) and item.strip() and item.strip().lower() not in seen_a: + seen_a.add(item.strip().lower()) + merged.append(item.strip()) + new_aliases = merged + else: + return None, f"标准实体「{entity_key}」的 aliases 需要是字符串数组" + + if state is None: + working[entity_key] = { + "id": None, + "key": entity_key, + "label": label, + "description": desc, + "aliases": new_aliases, + "members": [], + "bindings": [], + "caliber_params": [], + "version": 0, + "is_new": True, + } + payload = { + "op": "upsert_entity", + "entity_key": entity_key, + "label": label, + "description": desc, + "aliases": new_aliases, + "reason": reason, + } + return EntityPatchChange( + id=_change_id(op, entity_key, "meta"), + op=op, + entity_key=entity_key, + entity_label=label, + summary=f"新建标准业务实体「{label}（{entity_key}）」", + before=None, + after={"label": label, "description": desc, "aliases": new_aliases}, + reason=reason, + op_payload=payload, + ), None + + # 更新已有实体元信息（后台发现模式下不覆盖已有中文名与非空说明） + target_label = state["label"] if mode == "proposed" else label + target_desc = (state["description"] or desc) if mode == "proposed" else desc + before_meta = { + "label": state["label"], + "description": state["description"], + "aliases": list(state["aliases"]), + } + after_meta = { + "label": target_label, + "description": target_desc, + "aliases": new_aliases, + } + if before_meta == after_meta: + return None, None + state["label"] = target_label + state["description"] = target_desc + state["aliases"] = new_aliases + payload = { + "op": "upsert_entity", + "entity_key": entity_key, + "label": target_label, + "description": target_desc, + "aliases": new_aliases, + "reason": reason, + } + return EntityPatchChange( + id=_change_id(op, entity_key, "meta"), + op=op, + entity_key=entity_key, + entity_label=target_label, + summary=f"更新标准实体「{target_label}（{entity_key}）」基础信息", + before=before_meta, + after=after_meta, + reason=reason, + op_payload=payload, + ), None + + # 其余子项操作：若实体尚不存在，允许随操作附带 entity_label 自动初始化，否则明确报错 + if state is None: + fallback_label = str(cleaned.get("entity_label") or "").strip() + if not fallback_label: + return ( + None, + f"标准实体「{entity_key}」尚不存在，请先提供一条 upsert_entity（含 label 中文名称）或在操作中提供 entity_label", + ) + state = { + "id": None, + "key": entity_key, + "label": fallback_label[:100], + "description": "", + "aliases": [], + "members": [], + "bindings": [], + "caliber_params": [], + "version": 0, + "is_new": True, + } + working[entity_key] = state + + if op == "upsert_binding": + raw_src = str(cleaned.get("source_name") or "").strip() + raw_tbl = str(cleaned.get("table_name") or "").strip() + raw_col = str(cleaned.get("column_name") or "").strip() + if not raw_src or not raw_tbl or not raw_col: + return None, f"实体「{entity_key}」的表列绑定须完整填写 source_name、table_name、column_name" + src_row = env.sources_by_norm.get(raw_src.lower()) + if src_row is None: + avail_src = "、".join(sorted({r.name for r in env.sources_by_norm.values()})) or "无" + return None, f"未找到已启用的数据源「{raw_src}」（当前可用数据源：{avail_src}）" + canon_tbl, canon_col, problem = _resolve_table_and_column(src_row, raw_tbl, raw_col) + if problem or canon_tbl is None or canon_col is None: + return None, problem or "表列核验未通过" + + target_status = "confirmed" if mode == "confirmed" else "proposed" + target_source = "human" if mode == "confirmed" else "ai" + bkey = (src_row.name.lower(), canon_tbl.lower(), canon_col.lower()) + existing_idx = next( + ( + i + for i, b in enumerate(state["bindings"]) + if ( + str(b.get("source_name") or "").lower(), + str(b.get("table_name") or "").lower(), + str(b.get("column_name") or "").lower(), + ) + == bkey + ), + None, + ) + if existing_idx is not None: + cur = state["bindings"][existing_idx] + cur_status = str(cur.get("status") or "confirmed") + if mode == "proposed" and cur_status in ("confirmed", "verified", "rejected"): + return None, None + if ( + cur_status == target_status + and str(cur.get("note") or "") == (note or str(cur.get("note") or "")) + and str(cur.get("reason") or "") == (reason or str(cur.get("reason") or "")) + ): + return None, None + before_val = copy.deepcopy(cur) + updated = { + **cur, + "source_id": src_row.id, + "source_name": src_row.name, + "table_name": canon_tbl, + "column_name": canon_col, + "status": target_status, + "source": target_source, + "note": note or str(cur.get("note") or ""), + "reason": reason or str(cur.get("reason") or ""), + "updated_at": utcnow().isoformat(), + } + state["bindings"][existing_idx] = updated + else: + before_val = None + updated = { + "id": new_id()[:12], + "source_id": src_row.id, + "source_name": src_row.name, + "table_name": canon_tbl, + "column_name": canon_col, + "status": target_status, + "source": target_source, + "note": note, + "reason": reason, + "updated_at": utcnow().isoformat(), + } + state["bindings"].append(updated) + + payload = { + "op": "upsert_binding", + "entity_key": entity_key, + "entity_label": state["label"], + "source_name": src_row.name, + "table_name": canon_tbl, + "column_name": canon_col, + "note": note, + "reason": reason, + } + action_word = "新增" if before_val is None else "更新" + return EntityPatchChange( + id=_change_id(op, entity_key, f"{src_row.name}.{canon_tbl}.{canon_col}"), + op=op, + entity_key=entity_key, + entity_label=state["label"], + summary=f"{action_word}实体「{state['label']}」表列绑定：{src_row.name}.{canon_tbl}.{canon_col}", + before=before_val, + after=updated, + reason=reason, + op_payload=payload, + ), None + + if op == "remove_binding": + raw_src = str(cleaned.get("source_name") or "").strip() + raw_tbl = str(cleaned.get("table_name") or "").strip() + raw_col = str(cleaned.get("column_name") or "").strip() + bkey = (raw_src.lower(), raw_tbl.lower(), raw_col.lower()) + existing_idx = next( + ( + i + for i, b in enumerate(state["bindings"]) + if ( + str(b.get("source_name") or "").lower(), + str(b.get("table_name") or "").lower(), + str(b.get("column_name") or "").lower(), + ) + == bkey + ), + None, + ) + if existing_idx is None: + return None, f"实体「{entity_key}」中不存在待移除的绑定「{raw_src}.{raw_tbl}.{raw_col}」" + removed = state["bindings"].pop(existing_idx) + payload = { + "op": "remove_binding", + "entity_key": entity_key, + "source_name": removed.get("source_name"), + "table_name": removed.get("table_name"), + "column_name": removed.get("column_name"), + "reason": reason, + } + return EntityPatchChange( + id=_change_id(op, entity_key, f"{raw_src}.{raw_tbl}.{raw_col}"), + op=op, + entity_key=entity_key, + entity_label=state["label"], + summary=f"移除实体「{state['label']}」表列绑定：{removed.get('source_name')}.{removed.get('table_name')}.{removed.get('column_name')}", + before=removed, + after=None, + reason=reason, + op_payload=payload, + ), None + + if op == "upsert_member": + val = str(cleaned.get("value") or "").strip() + if not val: + return None, f"实体「{entity_key}」成员映射缺少标准值（value）" + label = str(cleaned.get("label") or val).strip() or val + raw_aliases = cleaned.get("aliases") + if raw_aliases is not None and not isinstance(raw_aliases, list): + return None, f"实体「{entity_key}」成员「{val}」的 aliases 需要是字符串数组" + raw_codes = cleaned.get("source_codes") + if raw_codes is not None and not isinstance(raw_codes, Mapping): + return None, f"实体「{entity_key}」成员「{val}」的 source_codes 需要是「数据源名 → 编码」对象" + + canon_codes: dict[str, str] = {} + for src_k, code_v in (raw_codes or {}).items(): + s_str = str(src_k or "").strip() + c_str = str(code_v or "").strip() + if not s_str or not c_str: + continue + src_row = env.sources_by_norm.get(s_str.lower()) + canon_src = src_row.name if src_row is not None else s_str + # 检查该实体在此数据源上的绑定是否涉及敏感遮罩列 + masked_set = env.masked_by_source.get(canon_src.lower(), set()) + if masked_set: + for b in state["bindings"]: + if str(b.get("source_name") or "").lower() == canon_src.lower(): + if _is_column_masked(masked_set, str(b.get("table_name") or ""), str(b.get("column_name") or "")): + return ( + None, + f"数据源「{canon_src}」表列「{b.get('table_name')}.{b.get('column_name')}」为敏感遮罩列，禁止为其配置成员编码映射", + ) + canon_codes[canon_src] = c_str + + m_idx = next( + (i for i, m in enumerate(state["members"]) if str(m.get("value") or "").strip().lower() == val.lower()), + None, + ) + if m_idx is not None: + cur_m = state["members"][m_idx] + before_m = copy.deepcopy(cur_m) + merged_aliases = list(cur_m.get("aliases") or []) + seen_ma = {a.lower() for a in merged_aliases if isinstance(a, str)} + for item in raw_aliases or []: + if isinstance(item, str) and item.strip() and item.strip().lower() not in seen_ma: + seen_ma.add(item.strip().lower()) + merged_aliases.append(item.strip()) + merged_codes = dict(cur_m.get("source_codes") or {}) + for cs, cv in canon_codes.items(): + if mode == "proposed" and cs in merged_codes and str(merged_codes[cs]).strip(): + continue + merged_codes[cs] = cv + target_mlabel = str(cur_m.get("label") or val) if mode == "proposed" else label + after_m = { + "value": str(cur_m.get("value") or val), + "label": target_mlabel, + "aliases": merged_aliases, + "source_codes": merged_codes, + } + if before_m == after_m: + return None, None + state["members"][m_idx] = after_m + else: + before_m = None + clean_aliases = [] + seen_ma = set() + for item in raw_aliases or []: + if isinstance(item, str) and item.strip() and item.strip().lower() not in seen_ma: + seen_ma.add(item.strip().lower()) + clean_aliases.append(item.strip()) + after_m = { + "value": val, + "label": label, + "aliases": clean_aliases, + "source_codes": canon_codes, + } + state["members"].append(after_m) + + payload = { + "op": "upsert_member", + "entity_key": entity_key, + "entity_label": state["label"], + "value": after_m["value"], + "label": after_m["label"], + "aliases": after_m["aliases"], + "source_codes": after_m["source_codes"], + "reason": reason, + } + code_desc = "，".join(f"{k}={v}" for k, v in after_m["source_codes"].items()) + suffix = f"（{code_desc}）" if code_desc else "" + return EntityPatchChange( + id=_change_id(op, entity_key, after_m["value"].lower()), + op=op, + entity_key=entity_key, + entity_label=state["label"], + summary=f"{'新增' if before_m is None else '更新'}实体「{state['label']}」成员「{after_m['label']}（{after_m['value']}）」{suffix}", + before=before_m, + after=after_m, + reason=reason, + op_payload=payload, + ), None + + # upsert_caliber_param + raw_wf = str(cleaned.get("workflow_id") or "").strip() + raw_card = str(cleaned.get("card_id") or "").strip() + raw_param = str(cleaned.get("param_name") or "").strip() + if not raw_wf or not raw_param: + return None, f"实体「{entity_key}」口径参数映射须提供 workflow_id 与 param_name" + wf_row = env.workflows_by_id.get(raw_wf) or env.workflows_by_name.get(raw_wf.lower()) + if wf_row is None: + return None, f"不存在工作流「{raw_wf}」，请核对已发布口径的工作流 ID" + valid_inputs = env.workflow_inputs.get(wf_row.id, set()) + canon_param = next((p for p in valid_inputs if p.lower() == raw_param.lower()), None) + if canon_param is None: + sample_in = "、".join(sorted(valid_inputs)) or "无入口参数" + return None, f"工作流「{wf_row.name}（{wf_row.id}）」入口节点不存在参数「{raw_param}」（可用参数：{sample_in}）" + valid_cards = env.workflow_cards.get(wf_row.id, set()) + canon_card = "" + if raw_card: + matched_card = next((c for c in valid_cards if c.lower() == raw_card.lower()), None) + if matched_card is None: + sample_cd = "、".join(sorted(valid_cards)) or "无口径卡节点" + return None, f"工作流「{wf_row.name}（{wf_row.id}）」中不存在口径卡「{raw_card}」（可用口径卡：{sample_cd}）" + canon_card = matched_card + + target_status = "confirmed" if mode == "confirmed" else "proposed" + target_source = "human" if mode == "confirmed" else "ai" + pkey = (wf_row.id, canon_card.lower(), canon_param.lower()) + p_idx = next( + ( + i + for i, p in enumerate(state["caliber_params"]) + if ( + str(p.get("workflow_id") or ""), + str(p.get("card_id") or "").lower(), + str(p.get("param_name") or "").lower(), + ) + == pkey + ), + None, + ) + if p_idx is not None: + cur_p = state["caliber_params"][p_idx] + cur_status = str(cur_p.get("status") or "confirmed") + if mode == "proposed" and cur_status in ("confirmed", "verified", "rejected"): + return None, None + if cur_status == target_status and str(cur_p.get("note") or "") == (note or str(cur_p.get("note") or "")): + return None, None + before_p = copy.deepcopy(cur_p) + after_p = { + **cur_p, + "workflow_id": wf_row.id, + "card_id": canon_card, + "param_name": canon_param, + "status": target_status, + "source": target_source, + "note": note or reason or str(cur_p.get("note") or ""), + } + state["caliber_params"][p_idx] = after_p + else: + before_p = None + after_p = { + "id": new_id()[:12], + "workflow_id": wf_row.id, + "card_id": canon_card, + "param_name": canon_param, + "status": target_status, + "source": target_source, + "note": note or reason, + } + state["caliber_params"].append(after_p) + + payload = { + "op": "upsert_caliber_param", + "entity_key": entity_key, + "entity_label": state["label"], + "workflow_id": wf_row.id, + "card_id": canon_card, + "param_name": canon_param, + "note": note or reason, + "reason": reason, + } + card_part = f" · {canon_card}" if canon_card else "" + return EntityPatchChange( + id=_change_id(op, entity_key, f"{wf_row.id}:{canon_card}:{canon_param}"), + op=op, + entity_key=entity_key, + entity_label=state["label"], + summary=f"{'新增' if before_p is None else '更新'}实体「{state['label']}」口径入参映射：{wf_row.name}{card_part} → {canon_param}", + before=before_p, + after=after_p, + reason=reason, + op_payload=payload, + ), None + + +async def plan_entity_patch( + session: AsyncSession, + raw_ops: Sequence[Any], + *, + pending_ops: Sequence[Any] = (), + mode: Literal["confirmed", "proposed"] = "confirmed", +) -> EntityPatchPlan: + """校验一批增量修改操作并生成结构化变更预览，不写入数据库。""" + env = await _load_env(session) + working: dict[str, dict[str, Any]] = {} + + # 先回放界面待保存的操作，保证后续新增实体/成员在此基础上叠加 + for p_idx, pop in enumerate(pending_ops or ()): + _normalize_single_op(pop, index=p_idx, env=env, working=working, mode=mode) + + plan = EntityPatchPlan() + for idx, item in enumerate(raw_ops or ()): + change, problem = _normalize_single_op(item, index=idx, env=env, working=working, mode=mode) + if problem: + plan.problems.append(problem) + continue + if change is not None: + plan.changes.append(change) + + touched_keys = {c.entity_key for c in plan.changes} + for ekey in sorted(touched_keys): + existing = env.entities_by_key.get(ekey) + plan.expected_versions[ekey] = int(existing.version or 1) if existing is not None else 0 + return plan + + +async def apply_entity_patch( + session: AsyncSession, + raw_ops: Sequence[Any], + *, + mode: Literal["confirmed", "proposed"], + actor: str | None = None, + expected_versions: Mapping[str, int] | None = None, +) -> dict[str, Any]: + """将一批增量操作合并并经 entities.create_entity / update_entity 统一落库。""" + if not raw_ops: + raise entities.EntityInvalid("没有可保存的实体修改项", field="operations") + + env = await _load_env(session) + working: dict[str, dict[str, Any]] = {} + changes: list[EntityPatchChange] = [] + problems: list[str] = [] + + for idx, item in enumerate(raw_ops): + change, problem = _normalize_single_op(item, index=idx, env=env, working=working, mode=mode) + if problem: + problems.append(problem) + continue + if change is not None: + changes.append(change) + + if mode == "confirmed" and problems: + raise entities.EntityInvalid("；".join(problems), field="operations") + + touched_keys = list(dict.fromkeys(c.entity_key for c in changes)) + if mode == "confirmed" and expected_versions is not None: + for ekey in touched_keys: + if ekey not in expected_versions: + continue + exp_ver = int(expected_versions[ekey]) + cur_row = env.entities_by_key.get(ekey) + cur_ver = int(cur_row.version or 1) if cur_row is not None else 0 + if exp_ver != cur_ver: + label = cur_row.label if cur_row is not None else ekey + raise entities.EntityConflict( + f"标准实体「{label}」在对话期间已被修改（基准 v{exp_ver}，当前 v{cur_ver}），请刷新后再应用修改", + field="expected_versions", + ) + + created_count = 0 + updated_count = 0 + saved_entities: list[dict[str, Any]] = [] + + for ekey in touched_keys: + state = working.get(ekey) + if state is None: + continue + payload: dict[str, Any] = { + "key": state["key"], + "label": state["label"], + "description": state["description"], + "aliases": state["aliases"], + "members": state["members"], + "bindings": state["bindings"], + "caliber_params": state["caliber_params"], + } + cur_row = env.entities_by_key.get(ekey) + if cur_row is None: + row = await entities.create_entity(session, payload, actor=actor) + created_count += 1 + else: + payload["version"] = int(cur_row.version or 1) + row = await entities.update_entity(session, cur_row.id, payload, actor=actor) + updated_count += 1 + saved_entities.append(entities.entity_to_dict(row)) + + proposed_bindings = sum(1 for c in changes if c.op == "upsert_binding") + proposed_params = sum(1 for c in changes if c.op == "upsert_caliber_param") + added_members = sum(1 for c in changes if c.op == "upsert_member") + + return { + "applied_count": len(changes), + "created_entities": created_count, + "updated_entities": updated_count, + "bindings_changed": proposed_bindings, + "caliber_params_changed": proposed_params, + "members_changed": added_members, + "entities": saved_entities, + "problems": problems, + } diff --git a/backend/app/data/source_health.py b/backend/app/data/source_health.py new file mode 100644 index 00000000..d0d46721 --- /dev/null +++ b/backend/app/data/source_health.py @@ -0,0 +1,463 @@ +"""跨数据源健康与漂移快速探针（source_health，W1-b）。 + +用于指标异动与跨源穿透时的第 1 顺位核查（先排数，后归因）： +1. 读取 `SourceChange` 待处理变化记录（`date_stalled`、`date_gap`、`volume_drop`、`rows_drop`、`codes_new`、 + `codes_shift`、`coverage_drop`、`table_removed`、`column_removed`、`column_type` 等）； +2. 对目标表非遮罩的时间列做有界只读鲜度与时间窗覆盖探查（`PROBE_TIMEOUT_S = 10.0s`，`guard.check + run_query`）， + 超时或失败优雅降级为 `unverified`，绝不拖垮主运行； +3. 严守遮罩铁律（铁律 3）：绝不探查或返回任何 `mask_columns` 列的值；`restricted=True`（普通 user 角色）时自动隐去 + 物理表名、物理列名与底层码值细节，只输出业务数据源说明与高层健康判定。 +""" +from __future__ import annotations + +import asyncio +from collections.abc import Iterable, Mapping +from datetime import date, datetime +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core import artifact_store +from app.data import schema_diff as sd +from app.data import source_changes as sc +from app.data.coverage import _as_day +from app.data.engine import QueryLimits, masked_columns, run_query +from app.data.guard import check, is_write +from app.data.names import name_key +from app.db.models import DataSource, Workflow, WorkflowVersion +from app.db.models_sources import SourceChange + +ARTIFACT_KIND = "source_health" +LEDGER_KIND = "source_health" +CATALOG_PREFIX = "H" +PROBE_TIMEOUT_S = 10.0 +MAX_TABLES_PROBED = 6 + +HEALTHY = "healthy" +DEGRADED = "degraded" +ANOMALOUS = "anomalous" +UNVERIFIED = "unverified" +VERDICTS = (HEALTHY, DEGRADED, ANOMALOUS, UNVERIFIED) + +_SEVERE_CHANGE_KINDS = frozenset({ + sd.TABLE_REMOVED, + sd.COLUMN_REMOVED, + sd.COLUMN_TYPE, + sc.DATE_STALLED, + sc.DATE_GAP, + sc.VOLUME_DROP, + sc.ROWS_DROP, + sc.COVERAGE_DROP, +}) +_WARN_CHANGE_KINDS = frozenset({ + sd.TABLE_REFLECT_FAILED, + sd.TABLE_LEFT_SCOPE, + sc.CODES_NEW, + sc.CODES_SHIFT, + sc.ROWS_JUMP, +}) + + +def _quote_ident(name: str, dialect: str) -> str: + clean = str(name) + if dialect in ("mysql", "mariadb"): + return "`" + clean.replace("`", "``") + "`" + return '"' + clean.replace('"', '""') + '"' + + +def _date_columns_of(table_meta: Mapping[str, Any], masked: set[str]) -> list[str]: + cols = table_meta.get("columns") + if not isinstance(cols, list): + return [] + out: list[str] = [] + for col in cols: + if not isinstance(col, Mapping): + continue + cname = str(col.get("name") or "").strip() + if not cname or cname.lower() in masked: + continue + ctype = str(col.get("type") or "").lower() + low = cname.lower() + if any(tok in ctype for tok in ("date", "time", "timestamp")) or any( + low == k or low.endswith(f"_{k}") or low.startswith(f"{k}_") + for k in ("date", "day", "dt", "shift_date", "production_day", "biz_date", "created_at") + ): + out.append(cname) + return out[:2] + + +async def _resolve_caliber_targets( + session: AsyncSession, + *, + workflow_id: str, + version: int | None, + card_id: str | None, + metric_id: str | None, +) -> list[tuple[str, list[str]]]: + """根据已发布（或指定版本）口径自动反查 (source_name, [table_names])。""" + from app.ledger.ontology import workflow_ontology_index + + wf = await session.get(Workflow, workflow_id) + if wf is None: + return [] + ver = version if version is not None else wf.published_version or wf.version + snap = ( + await session.execute( + select(WorkflowVersion).where( + WorkflowVersion.workflow_id == workflow_id, + WorkflowVersion.version == int(ver), + ) + ) + ).scalar_one_or_none() + graph = (snap.graph if snap is not None else wf.graph) or {} + idx = workflow_ontology_index(workflow_id=workflow.id if (workflow := wf) else workflow_id, version=int(ver), graph=graph) + by_src: dict[str, set[str]] = {} + for card in idx["cards"]: + if card_id and card["card_id"] != card_id: + continue + if metric_id: + for m in card["metrics"]: + if m["metric_id"] == metric_id: + for item in m["lineage"]: + by_src.setdefault(item["source"], set()).update(item["tables"]) + else: + for src, tbls in card["tables_by_source"].items(): + by_src.setdefault(src, set()).update(tbls) + return [(src, sorted(tbls)) for src, tbls in sorted(by_src.items())] + + +def _parse_window_day(raw: str | date | None) -> date | None: + if raw is None: + return None + if isinstance(raw, datetime): + return raw.date() + if isinstance(raw, date): + return raw + d = _as_day(str(raw).strip()) + return d.day if d is not None else None + + +async def _probe_table_coverage( + source: DataSource, + table_name: str, + table_meta: Mapping[str, Any], + *, + masked: set[str], + start_day: date | None, + end_day: date | None, +) -> dict[str, Any]: + """对单张表做只读行数与非遮罩日期边界探查，超时或报错返回 status='unverified'。""" + dialect = (source.kind or "sqlite").lower() + q_table = _quote_ident(table_name, dialect) + date_cols = _date_columns_of(table_meta, masked) + select_parts = ["COUNT(*) AS row_cnt"] + for idx, col in enumerate(date_cols): + qc = _quote_ident(col, dialect) + select_parts.append(f"MIN({qc}) AS min_d{idx}") + select_parts.append(f"MAX({qc}) AS max_d{idx}") + sql = f"SELECT {', '.join(select_parts)} FROM {q_table}" + if is_write(sql, dialect=dialect): + return {"table": table_name, "status": UNVERIFIED, "reason": "探查语句未通过只读校验"} + check(sql, readonly=True, source_name=source.name, dialect=dialect) + + try: + res = await asyncio.wait_for( + run_query(source, sql, limits=QueryLimits(max_rows=5, timeout_seconds=PROBE_TIMEOUT_S)), + timeout=PROBE_TIMEOUT_S + 1.0, + ) + except Exception: # noqa: BLE001 - 探查失败优雅降级，绝不中断主流程 + return {"table": table_name, "status": UNVERIFIED, "reason": "实时探查超时或暂时不可达"} + + if not res.rows: + return {"table": table_name, "status": ANOMALOUS, "issue": "empty_table", "row_count": 0} + first = res.rows[0] + row_cnt = int(first[0] or 0) if first and first[0] is not None else 0 + if row_cnt == 0: + return {"table": table_name, "status": ANOMALOUS, "issue": "empty_table", "row_count": 0} + + ranges: list[dict[str, Any]] = [] + window_issues: list[dict[str, Any]] = [] + for idx, col in enumerate(date_cols): + pos = 1 + idx * 2 + raw_min = first[pos] if pos < len(first) else None + raw_max = first[pos + 1] if (pos + 1) < len(first) else None + d_min = _as_day(raw_min) + d_max = _as_day(raw_max) + ranges.append({ + "column": col, + "min": d_min.day.isoformat() if d_min else (str(raw_min) if raw_min is not None else None), + "max": d_max.day.isoformat() if d_max else (str(raw_max) if raw_max is not None else None), + }) + if start_day and d_max and d_max.day < start_day: + window_issues.append({ + "kind": sc.DATE_STALLED, + "severity": "action", + "column": col, + "max_date": d_max.day.isoformat(), + "requested_start": start_day.isoformat(), + }) + elif end_day and d_min and d_min.day >= end_day: + window_issues.append({ + "kind": sc.DATE_GAP, + "severity": "action", + "column": col, + "min_date": d_min.day.isoformat(), + "requested_end": end_day.isoformat(), + }) + return { + "table": table_name, + "status": ANOMALOUS if window_issues else HEALTHY, + "row_count": row_cnt, + "ranges": ranges, + "window_issues": window_issues, + } + + +def _render_phrase(source_title: str, verdict: str, issues: list[dict[str, Any]], *, probe_unverified: bool) -> str: + if verdict == ANOMALOUS: + labels = list(dict.fromkeys(str(i.get("label") or i.get("kind") or "数据异常") for i in issues)) + shown = "、".join(labels[:3]) + return f"数据源「{source_title}」检测到数据健康异常（{shown}），当前异动优先归因于上游同步或主数据质量问题，不宜直接归结为业务实质变化。" + if verdict == DEGRADED: + labels = list(dict.fromkeys(str(i.get("label") or i.get("kind") or "取值变化") for i in issues)) + shown = "、".join(labels[:3]) + return f"数据源「{source_title}」存在待审阅的数据变化提醒（{shown}），跨源对比与归因时需留意码值或分布漂移。" + if verdict == UNVERIFIED or probe_unverified: + return f"数据源「{source_title}」未发现已登记的变化告警，但实时范围探查未完成（状态记为待核实）。" + return f"数据源「{source_title}」同步状态与查询区间覆盖正常，未发现断更、缺口或未处理的结构漂移。" + + +async def check_source_health( + session: AsyncSession, + *, + source_name: str | None = None, + tables: Iterable[str] | None = None, + workflow_id: str | None = None, + version: int | None = None, + card_id: str | None = None, + metric_id: str | None = None, + start: str | date | None = None, + end: str | date | None = None, + restricted: bool = False, + live_probe: bool = True, +) -> dict[str, Any]: + """检查目标数据源（或目标口径底层依赖源）的健康度与数据漂移，返回确定性结构化结论。""" + targets: list[tuple[str, list[str]]] = [] + if source_name and str(source_name).strip(): + explicit_tables = [str(t).strip() for t in (tables or ()) if str(t).strip()] + targets.append((str(source_name).strip(), explicit_tables)) + elif workflow_id and str(workflow_id).strip(): + targets = await _resolve_caliber_targets( + session, + workflow_id=str(workflow_id).strip(), + version=version, + card_id=card_id, + metric_id=metric_id, + ) + if not targets: + return { + "kind": ARTIFACT_KIND, + "verdict": UNVERIFIED, + "sources": [], + "issues": [], + "phrase": "未能定位目标数据源，健康状态记为未核实。", + } + + start_day = _parse_window_day(start) + end_day = _parse_window_day(end) + + source_reports: list[dict[str, Any]] = [] + all_issues: list[dict[str, Any]] = [] + overall_rank = 0 # 0 healthy, 1 unverified, 2 degraded, 3 anomalous + rank_to_verdict = {0: HEALTHY, 1: UNVERIFIED, 2: DEGRADED, 3: ANOMALOUS} + + for src_name, target_tables in targets: + src = ( + await session.execute(select(DataSource).where(DataSource.name == src_name).limit(1)) + ).scalar_one_or_none() + if src is None or not src.enabled: + overall_rank = max(overall_rank, 3 if src is not None and not src.enabled else 1) + status = ANOMALOUS if (src is not None and not src.enabled) else UNVERIFIED + issue_label = "数据源已停用" if status == ANOMALOUS else "数据源未找到" + issue = {"kind": "source_unavailable", "severity": "action", "label": issue_label} + if not restricted: + issue["source"] = src_name + all_issues.append(issue) + source_reports.append({ + "source_label": (src.description or src.name) if src else ("关联数据源" if restricted else src_name), + **({} if restricted else {"source": src_name}), + "verdict": status, + "issues": [issue], + }) + continue + + masked = {c.lower() for c in masked_columns(src.options)} + schema_tables = (src.schema_cache or {}).get("tables") if isinstance(src.schema_cache, Mapping) else {} + if not isinstance(schema_tables, Mapping): + schema_tables = {} + + wanted_keys = {name_key(t) for t in target_tables} if target_tables else None + change_rows = list( + ( + await session.execute( + select(SourceChange) + .where(SourceChange.source_id == src.id, SourceChange.status.in_(sc.PENDING)) + .order_by(SourceChange.seq.desc()) + ) + ).scalars() + ) + src_issues: list[dict[str, Any]] = [] + src_rank = 0 + for ch in change_rows: + if wanted_keys is not None and ch.table_name and name_key(ch.table_name) not in wanted_keys: + continue + # 遮罩列上的变化不暴露列名与取值细节（铁律 3） + if ch.column_name and ch.column_name.lower() in masked: + continue + label = sc.KIND_LABEL.get(ch.kind, ch.kind) + is_severe = ch.kind in _SEVERE_CHANGE_KINDS or ch.severity == "action" + src_rank = max(src_rank, 3 if is_severe else 2) + entry: dict[str, Any] = { + "kind": ch.kind, + "severity": ch.severity, + "status": ch.status, + "label": label, + } + if not restricted: + entry["source"] = src.name + if ch.table_name: + entry["table"] = ch.table_name + if ch.column_name: + entry["column"] = ch.column_name + entry["summary"] = sc.summary(ch) + src_issues.append(entry) + + probe_unverified = False + probed_tables: list[dict[str, Any]] = [] + if live_probe: + candidate_names = target_tables or list(schema_tables.keys())[:MAX_TABLES_PROBED] + for tbl_name in candidate_names[:MAX_TABLES_PROBED]: + # 找到 schema_cache 里精确的表名键 + matched_name = next((k for k in schema_tables if name_key(k) == name_key(tbl_name)), tbl_name) + meta = schema_tables.get(matched_name) if isinstance(schema_tables.get(matched_name), Mapping) else {} + probe = await _probe_table_coverage( + src, + matched_name, + meta, + masked=masked, + start_day=start_day, + end_day=end_day, + ) + if probe["status"] == UNVERIFIED: + probe_unverified = True + if src_rank == 0: + src_rank = 1 + elif probe["status"] == ANOMALOUS: + src_rank = max(src_rank, 3) + if probe.get("issue") == "empty_table": + item: dict[str, Any] = { + "kind": sc.ROWS_DROP, + "severity": "action", + "label": "目标表当前无数据（0 行）", + } + if not restricted: + item["source"] = src.name + item["table"] = matched_name + src_issues.append(item) + for wi in probe.get("window_issues") or []: + item = { + "kind": wi["kind"], + "severity": wi["severity"], + "label": sc.KIND_LABEL.get(wi["kind"], wi["kind"]), + } + if not restricted: + item["source"] = src.name + item["table"] = matched_name + item["column"] = wi["column"] + if "max_date" in wi: + item["max_date"] = wi["max_date"] + if "min_date" in wi: + item["min_date"] = wi["min_date"] + src_issues.append(item) + if not restricted: + probed_tables.append(probe) + + src_verdict = rank_to_verdict[src_rank] + overall_rank = max(overall_rank, src_rank) + all_issues.extend(src_issues) + display_title = (src.description or "关联业务源") if restricted else src.name + rep: dict[str, Any] = { + "source_label": display_title, + "verdict": src_verdict, + "issue_count": len(src_issues), + "issues": src_issues, + "phrase": _render_phrase(display_title, src_verdict, src_issues, probe_unverified=probe_unverified), + } + if not restricted: + rep["source"] = src.name + rep["tables_probed"] = probed_tables + source_reports.append(rep) + + overall_verdict = rank_to_verdict[overall_rank] + lead_phrase = source_reports[0]["phrase"] if len(source_reports) == 1 else ( + "；".join(r["phrase"] for r in source_reports) + ) + out: dict[str, Any] = { + "kind": ARTIFACT_KIND, + "verdict": overall_verdict, + "sources": source_reports, + "issues": all_issues, + "phrase": lead_phrase, + } + if workflow_id: + out["caliber_ref"] = { + "workflow_id": workflow_id, + **({"version": version} if version is not None else {}), + **({"card_id": card_id} if card_id else {}), + **({"metric_id": metric_id} if metric_id else {}), + } + return out + + +def is_health_result(content: Any) -> bool: + return ( + isinstance(content, Mapping) + and content.get("kind") == ARTIFACT_KIND + and content.get("verdict") in VERDICTS + and isinstance(content.get("sources"), list) + ) + + +async def store_source_health(result: Mapping[str, Any], *, run_id: str | None, node_id: str | None = None) -> str: + return await artifact_store.put_json(dict(result), kind=ARTIFACT_KIND, run_id=run_id, meta={ + "verdict": result.get("verdict"), + "node_id": node_id or "", + }) + + +def ledger_entry( + result: Mapping[str, Any], + *, + node_id: str, + exec_no: int, + artifact: str, + call_id: str | None = None, +) -> dict[str, Any]: + sources = [ + str(s.get("source") or s.get("source_label") or "") + for s in (result.get("sources") or []) + if isinstance(s, Mapping) + ] + entry: dict[str, Any] = { + "kind": LEDGER_KIND, + "node_id": node_id, + "exec": exec_no, + "artifact": artifact, + "verdict": result.get("verdict"), + "sources": [s for s in sources if s], + "issue_count": len(result.get("issues") or []), + } + if call_id is not None: + entry["call_id"] = call_id + return entry diff --git a/backend/app/db/models.py b/backend/app/db/models.py index 67ae8dee..161d814e 100644 --- a/backend/app/db/models.py +++ b/backend/app/db/models.py @@ -1429,3 +1429,6 @@ from app.db import models_triggers # noqa: E402,F401 # 编排完善第 2 波 W2-b（docs/design/orchestration-round/00-总方案.md §5）：多进程协调——工作进程的登记和租约、调度当班、 # 跨进程的请求、运行的并发名额、同源队、按时间窗的计数 from app.db import models_cluster # noqa: E402,F401 +# 指标本体穿透（W1-a）：标准业务实体与跨源映射 +from app.db import models_ontology # noqa: E402,F401 +from app.db.models_ontology import CanonicalEntity # noqa: E402,F401 diff --git a/backend/app/db/models_ontology.py b/backend/app/db/models_ontology.py new file mode 100644 index 00000000..b8153e5d --- /dev/null +++ b/backend/app/db/models_ontology.py @@ -0,0 +1,63 @@ +"""标准业务实体与跨源本体映射表。 + +由 app/db/models.py 末尾整模块导入，create_all 自动建表（兼容 SQLite 与 PostgreSQL）。 + +- CanonicalEntity：标准业务实体（如产线、仓库、门店、设备、区域），承接跨数据源的物理表列绑定（bindings）、 + 跨源编码映射（members.source_codes）与口径输入参数映射（caliber_params）。 +- 每一项物理列绑定、口径参数映射都带显式状态（proposed / verified / confirmed / rejected）： + 自动穿透与跨源对齐运行时只读取 verified / confirmed 的映射，proposed 候选绝不自动参与穿透（铁律 1）。 +""" +from __future__ import annotations + +from datetime import datetime +from typing import Any + +from sqlalchemy import Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base, TimestampMixin, UTCDateTime, new_id + + +class CanonicalEntity(Base, TimestampMixin): + """标准业务实体及跨源物理列、编码、口径参数映射。""" + + __tablename__ = "canonical_entities" + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + #: 机器稳定的标准键（小写字母、数字、下划线，如 production_line、warehouse） + key: Mapped[str] = mapped_column(String(64), unique=True, index=True) + #: 业务展示名称（如「产线」「仓库」） + label: Mapped[str] = mapped_column(String(100)) + description: Mapped[str] = mapped_column(Text, default="") + #: 业务同义词（如 ["生产线", "线体", "车间产线"]） + aliases: Mapped[list[Any]] = mapped_column(default=list) + #: 标准成员与跨源编码映射：[{value, label, aliases: [...], source_codes: {source_name: code}}] + members: Mapped[list[Any]] = mapped_column(default=list) + #: 跨源物理表列绑定：[{id, source_id, source_name, table_name, column_name, status, source, note, reason, updated_at, updated_by}] + bindings: Mapped[list[Any]] = mapped_column(default=list) + #: 口径输入参数映射：[{id, workflow_id, card_id, param_name, status, source, note}] + caliber_params: Mapped[list[Any]] = mapped_column(default=list) + #: 乐观锁版本号（从 1 起，每次修改加 1） + version: Mapped[int] = mapped_column(Integer, default=1) + created_by: Mapped[str | None] = mapped_column(String(100), default=None) + updated_by: Mapped[str | None] = mapped_column(String(100), default=None) + + +class EntityDiscoveryJob(Base, TimestampMixin): + """后台扫描发现候选标准业务实体的异步任务（E2）。""" + + __tablename__ = "entity_discovery_jobs" + + id: Mapped[str] = mapped_column(String(32), primary_key=True, default=new_id) + #: queued | running | succeeded | failed | cancelled | interrupted + status: Mapped[str] = mapped_column(String(24), default="queued", index=True) + source_ids: Mapped[list[Any]] = mapped_column(default=list) + document_ids: Mapped[list[Any]] = mapped_column(default=list) + attachment_ids: Mapped[list[Any]] = mapped_column(default=list) + note: Mapped[str] = mapped_column(Text, default="") + phase: Mapped[str] = mapped_column(String(120), default="正在准备扫描范围…") + summary: Mapped[dict[str, Any]] = mapped_column(default=dict) + error: Mapped[str | None] = mapped_column(Text, default=None) + started_by: Mapped[str | None] = mapped_column(String(100), default=None) + started_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) + finished_at: Mapped[datetime | None] = mapped_column(UTCDateTime, default=None) diff --git a/backend/app/engine/analysis_claims.py b/backend/app/engine/analysis_claims.py index e7692f95..2848d4a2 100644 --- a/backend/app/engine/analysis_claims.py +++ b/backend/app/engine/analysis_claims.py @@ -60,10 +60,11 @@ from decimal import Decimal, InvalidOperation from typing import Any from app.analysis.contract import LEDGER_KIND, OP_LABELS, catalog_result, method_label +from app.data.source_health import LEDGER_KIND as HEALTH_KIND, is_health_result from app.engine.evidence import _Reason, _violation, iter_units #: 说法表的版本：改了判断词、类、绑定规则就升版本，老版本留在 LEXICONS 里给老文档复核用 -LEXICON_VERSION = 1 +LEXICON_VERSION = 2 #: 报告节点 analysis_claims 的取值里要核的两种（off 不核、文档里也不记） MODES = ("check", "strict") #: 报告文档里记下核对处置和说法表版本的字段 @@ -278,7 +279,35 @@ _V1 = Lexicon( r"|转化率|退款率|退款额|退货率|毛利|利润|收入|营收|客流|流量|访客|GMV", re.I), ) -LEXICONS: dict[int, Lexicon] = {1: _V1} +_ONTOLOGY_CLAIMS: tuple[Claim, ...] = ( + Claim("reconcile_bad", "对账不一致", "reconcile", re.compile( + r"对账(?:不一致|差异|不平|偏离|异常)|勾稽(?:不一致|不平)|双源(?:不一致|对不上)|核对不平"), + flip="reconcile_ok"), + Claim("reconcile_ok", "对账一致", "reconcile", re.compile( + r"对账(?:一致|吻合|相符)|勾稽一致|双源一致|核对一致"), + flip="reconcile_bad", negative=True), + Claim("cochange", "跨源联动", "cochange", re.compile( + r"同步(?:异动|波动|上升|下降|增长|下滑)|联动(?:波动|异动|上升|下降)|同频(?:波动|异动)" + r"|同向(?:波动|异动|上升|下降)|反向(?:波动|异动|上升|下降)|伴随(?:出现)?(?:同步)?(?:波动|异动|上升|下降)"), + flip="no_cochange"), + Claim("no_cochange", "无跨源联动", "cochange", re.compile( + r"(?:无|没有|未见|并无)(?:明显|显著)?(?:的)?(?:同步异动|同步波动|联动波动|同频波动)|走势独立|独立波动"), + negative=True), + Claim("data_fault", "数据源异常", HEALTH_KIND, re.compile( + r"数据源(?:延迟|停更|断更|缺数|中断)|同步(?:中断|停滞|断流)|上游断流|表未更新|码值漂移|新增未知码值")), +) + +_V2 = replace( + _V1, + version=2, + claims=(*_ONTOLOGY_CLAIMS, *_V1.claims), + plan_ahead=re.compile( + r"建议|(? str: """这一类要挂的分析怎么叫：算子、模式的叫法从契约取（结构变化是「变化拆解（比率）」）。""" + if claim.op == HEALTH_KIND: + return "数据源健康核查" return method_label(claim.op, claim.mode) if claim.mode else OP_LABELS.get(claim.op or "", claim.op or "") @@ -826,12 +879,18 @@ def _snippet(shown: str, start: int, end: int, limit: int = 16) -> str: return piece or shown[start:end] +def _inline_marker_of(alias: str, op: str | None = None) -> str: + head = "h" if alias.startswith("H") or op == HEALTH_KIND else "a" + return f"[[{head}:{alias}]]" + + def _uncited(claim: Claim, word: str, available: Mapping[str, list[str]]) -> _Reason: op_text = _op_text(claim) pool = available.get(f"{claim.op}:{claim.mode}" if claim.mode else claim.op or "", []) - fallback = _FALLBACK["mix" if claim.mode == "ratio" else claim.op or "decompose"] + fallback = _FALLBACK.get("mix" if claim.mode == "ratio" else claim.op or "decompose", _FALLBACK["decompose"]) if pool: - model = (f"「{word}」要挂{op_text}的结论：照它的判定写、句末挂 [[see:{pool[0]}]]，或者直接引用判定 [[a:{pool[0]}]]；" + inline = _inline_marker_of(pool[0], claim.op) + model = (f"「{word}」要挂{op_text}的结论：照它的判定写、句末挂 [[see:{pool[0]}]]，或者直接引用判定 {inline}；" f"判定和你想写的不一样就照判定写，否则{fallback}") else: model = f"这次没有{op_text}的结论，不能写「{word}」：{fallback}" @@ -839,12 +898,19 @@ def _uncited(claim: Claim, word: str, available: Mapping[str, list[str]]) -> _Re def _mismatch(alias: str, snippet: str, miss: _Miss) -> _Reason: + inline = _inline_marker_of(alias) return _Reason(f"「{snippet}」与所挂的分析结论 {alias} 对不上：{miss.human}", - f"{alias} {miss.model}，不能写「{snippet}」：照判定改写（或者直接引用判定 [[a:{alias}]]），" + f"{alias} {miss.model}，不能写「{snippet}」：照判定改写（或者直接引用判定 {inline}），" "或者删掉这个判断") -def _causal(snippet: str) -> _Reason: +def _causal(snippet: str, *, cochange: bool = False) -> _Reason: + if cochange: + return _Reason( + f"「{snippet}」是原因判断：跨源联动检验只证明统计同步异动，不能直接写成因果", + f"「{snippet}」没有因果依据：跨源联动检验不能写成「导致/由于/造成」；请改写为「伴随…同步异动」，" + "或引用知识库原话，或写成「可能与…有关，待核实」", + ) return _Reason(f"「{snippet}」是原因判断，这句没有知识库原话或运行输入作依据，也没有写成推测", f"「{snippet}」没有依据：数据只能说明变化来自哪里；要写原因，引用知识库原话，或写成「可能与…有关，待核实」") @@ -895,10 +961,14 @@ def _said_cause(cause: str) -> bool: return bool(rest) and _POINTER.fullmatch(rest) is None +_CUTOFF_CAVEAT = re.compile(r"只到|截断|截止|未覆盖|不完整|样本|省略|篇幅") +_DATA_FAULT_CAUSE = re.compile(r"停更|断更|中断|断流|延迟|缺数|对账|码值|单位|换算") + + def check_sentence(sentence: Sentence, results: Mapping[str, Mapping[str, Any]], *, available: Mapping[str, list[str]] | None = None, version: int | None = None) -> list[dict[str, Any]]: - """一句话的说法违规。results 是写作目录里的分析结论 {A 编号: 结论}，available 是 {op: [这次有的 A 编号]}（没挂时 - 告诉写作者可以挂哪一份），不给就按 results 算。""" + """一句话的说法违规。results 是写作目录里的分析结论 {A 编号: 结论}（含数据源健康 H 编号），available 是 {op: [这次有的编号]} + （没挂时告诉写作者可以挂哪一份），不给就按 results 算。""" lex = lexicon(version) if available is None: available = _available(results) @@ -909,6 +979,12 @@ def check_sentence(sentence: Sentence, results: Mapping[str, Mapping[str, Any]], #: 这句挂的拆解（原因点名的是它的段就按来源核），和目录里所有的拆解（没挂时认原因是不是哪份拆解的段） decomposes = [r for _, r in mine if r.get("op") == "decompose"] known = [r for r in results.values() if r.get("op") == "decompose"] + has_cochange = any(r.get("op") == "cochange" for _, r in mine) + has_data_fault = any( + (r.get("op") == HEALTH_KIND and _verdict(r).get("code") == "anomalous") + or (r.get("op") == "reconcile" and _verdict(r).get("code") in ("diverged", "missing_counterpart")) + for _, r in mine + ) out: list[dict[str, Any]] = [] #: 同一分句里同一类只报一处（「由于促销结束导致下降」是一处没依据的原因，不是两处） seen: set[tuple[str, str, int]] = set() @@ -921,7 +997,11 @@ def check_sentence(sentence: Sentence, results: Mapping[str, Mapping[str, Any]], spans = _object(sentence, hit, left, right) if claim.key == "causal": clause = sentence.text[left:right] - if sentence.quoted or sentence.inputs or guessed or lex.method.search(clause): + method_exempt = bool( + (_CUTOFF_CAVEAT.search(clause) or (not has_cochange and lex.method.search(clause))) + or (has_data_fault and _DATA_FAULT_CAUSE.search(clause)) + ) + if sentence.quoted or sentence.inputs or guessed or method_exempt: continue cause = _cause(sentence, hit, left, right) if not _said_cause(cause): @@ -936,7 +1016,7 @@ def check_sentence(sentence: Sentence, results: Mapping[str, Mapping[str, Any]], key = (CAUSAL, claim.key, left) if key not in seen: seen.add(key) - out.append(_record(CAUSAL, _causal(snippet), sentence, hit, claim, mine)) + out.append(_record(CAUSAL, _causal(snippet, cochange=has_cochange), sentence, hit, claim, mine)) continue candidates = [(a, r) for a, r in mine if r.get("op") == claim.op] if not candidates: @@ -1002,9 +1082,32 @@ def results_of(catalog: Mapping[str, Any], load: Any = None) -> dict[str, dict[s return out +def healths_of(catalog: Mapping[str, Any], load: Any = None) -> dict[str, dict[str, Any]]: + """写作目录里的数据源健康结论：{H 编号: 归一化为分析结论形状的字典}。""" + out: dict[str, dict[str, Any]] = {} + for alias, entry in catalog.items(): + if not isinstance(entry, dict) or entry.get("kind") != HEALTH_KIND: + continue + cand = entry.get("result") if is_health_result(entry.get("result")) else entry.get("health") + raw = cand if is_health_result(cand) else None + if raw is None and load is not None: + loaded = load(str(alias), entry) + raw = loaded if is_health_result(loaded) else None + if isinstance(raw, dict): + out[str(alias)] = { + "status": "ok", + "op": HEALTH_KIND, + "mode": None, + "verdict": {"code": str(raw.get("verdict") or ""), "subjects": list(raw.get("sources") or [])}, + "phrase": str(raw.get("phrase") or ""), + "issues": list(raw.get("issues") or []), + } + return out + + def _sentence(unit: dict[str, Any], aliases: Iterable[str], results: Mapping[str, Any]) -> Sentence | None: """文档里的一个句子单元 → Sentence。写作者自己写的字（文字片段、没有出处的数字）留在 text；引用渲染出来的值、表名 - 字段名留在 names（点名用）；判定语（a:）、知识库原话（q:）、排版符号两边都涂掉，判定语另记下位置（phrases）。""" + 字段名留在 names（点名用）；判定语（a: / h:）、知识库原话（q:）、排版符号两边都涂掉，判定语另记下位置（phrases）。""" segments = [s for s in unit.get("segments") or [] if isinstance(s, dict)] if not segments: return None @@ -1023,11 +1126,11 @@ def _sentence(unit: dict[str, Any], aliases: Iterable[str], results: Mapping[str if kind == "structural": scan_parts.append(blank) name_parts.append(blank) - elif head in ("a", "q"): + elif head in ("a", "h", "q"): scan_parts.append(blank) name_parts.append(blank) quoted = quoted or (head == "q" and seg.get("state") == "deterministic") - if head == "a" and (phrase := _PHRASE_REF.match(ref)): + if head in ("a", "h") and (phrase := _PHRASE_REF.match(ref)): phrases.append((pos, pos + len(text), phrase.group(1))) elif ref or kind == "entity": scan_parts.append(blank) @@ -1050,8 +1153,8 @@ def _sentence(unit: dict[str, Any], aliases: Iterable[str], results: Mapping[str def check_doc(doc: dict[str, Any], catalog: Mapping[str, Any], *, cited: Mapping[int, list[str]] | None = None, version: int | None = None, load: Any = None) -> list[dict[str, Any]]: """一份报告文档的说法违规。cited 是 {id(unit): 这句按重新解析的引用算的依据}（verify_doc 交进来），不给就用文档里 - 记的 cites；load 取回只带摘要的条目的整份结论（results_of）。标题、代码块、表格（表头和正文的格）不核。""" - results = results_of(catalog, load) + 记的 cites；load 取回只带摘要的条目的整份结论（results_of / healths_of）。标题、代码块、表格（表头和正文的格）不核。""" + results = {**results_of(catalog, load), **healths_of(catalog, load)} available = _available(results) out: list[dict[str, Any]] = [] for block, unit in iter_units(doc): @@ -1146,14 +1249,29 @@ ANALYSIS_REPAIR_PROMPT = ("判断词照分析结论的判定写并挂上依据 "原因没有依据的写成待核实的推测。") +ONTOLOGY_CLAIMS_PROMPT = ( + "跨源对账、联动与数据源健康规则：\n" + "- 「对账一致」「对账不一致」「勾稽不平」只能挂跨源对账结论（reconcile），直接引用 [[a:An]] 或句末挂 [[see:An]]。\n" + "- 「同步异动」「同向波动」「反向波动」「伴随波动」只能挂跨源联动检验结论（cochange），且跨源联动检验仅代表统计同频，" + "严禁写成因果（「由于…导致」「造成」），必须写为「伴随…同步异动」或「可能与…有关，待核实」。\n" + "- 「数据源停更」「数据源延迟」「同步中断」「码值漂移」只能挂数据源健康核查结论（[[h:Hn]] 或 [[see:Hn]]）。" +) + + def rules_prompt(catalog: Mapping[str, Any]) -> str: - """写作规则里说判断词的那一段：目录里有分析结论和没有时各一种说法。""" - return ANALYSIS_RULES_PROMPT if results_of(catalog) else ANALYSIS_NONE_PROMPT + """写作规则里说判断词的那一段：目录里有分析结论和没有时各一种说法；含对账/联动/健康核查时追加本体规则。""" + results = results_of(catalog) + healths = healths_of(catalog) + if not results and not healths: + return ANALYSIS_NONE_PROMPT + if healths or any(r.get("op") in ("reconcile", "cochange") for r in results.values()): + return f"{ANALYSIS_RULES_PROMPT}\n{ONTOLOGY_CLAIMS_PROMPT}" + return ANALYSIS_RULES_PROMPT __all__ = [ "ANALYSIS_NONE_PROMPT", "ANALYSIS_REPAIR_PROMPT", "ANALYSIS_RULES_PROMPT", "CAUSAL", "CODES", "DOC_FIELD", - "LEXICONS", "LEXICON_VERSION", "MISMATCH", "MODES", "UNCITED", "Claim", "Hit", "Lexicon", "Sentence", - "check_doc", "check_sentence", "counts", "lexicon", "record", "recorded", "results_of", "rules_prompt", "scan", - "summary", "verify", + "LEXICONS", "LEXICON_VERSION", "MISMATCH", "MODES", "ONTOLOGY_CLAIMS_PROMPT", "UNCITED", "Claim", "Hit", "Lexicon", + "Sentence", "check_doc", "check_sentence", "counts", "healths_of", "lexicon", "record", "recorded", "results_of", + "rules_prompt", "scan", "summary", "verify", ] diff --git a/backend/app/engine/analysis_scope.py b/backend/app/engine/analysis_scope.py index c6626778..7fba0843 100644 --- a/backend/app/engine/analysis_scope.py +++ b/backend/app/engine/analysis_scope.py @@ -33,14 +33,14 @@ logger = logging.getLogger(__name__) #: 候选维度「取值不多」的上限：码值表里的取值不超过这么多（取值观察最多也只取这么多，data/catalog_profile_observe） CANDIDATE_MAX = 50 -#: 现象：可比性、基线、趋势 -_PHENOMENA = ("comparable", "baseline", "trend") +#: 现象：可比性、基线、趋势、跨源对账、跨源联动 +_PHENOMENA = ("comparable", "baseline", "trend", "reconcile", "cochange") #: 事实表的表类型（数据目录的 kind）：流水、日志也是一行一件事 _FACT_KINDS = ("fact", "log") #: 候选维度的度量类型 _DIMENSION_MEASURES = ("attribute", "status") -#: 拆解的输入快照（下钻的上一层不是快照） -_DATA_ROLES = ("data", "base", "target") +#: 分析算子的输入快照（下钻的上一层不是快照） +_DATA_ROLES = ("data", "base", "target", "primary", "counterpart", "driver") NOTE_NO_CATALOG = "这次运行读不到数据目录，候选维度核对不了" NOTE_NO_NOTES = "拆解所用的数据源还没有数据目录，候选维度核对不了" @@ -288,7 +288,8 @@ def build(catalog: Mapping[str, Any], *, sources: Mapping[str, Mapping[str, Any] decomposes = [r for _, r in ok if r.get("op") == "decompose"] excluded = [items[a] for a, r in ok if (r.get("op") == "decompose" and _code(r) == "dispersed") - or (r.get("op") == "differ" and _code(r) == "no_evidence")] + or (r.get("op") == "differ" and _code(r) == "no_evidence") + or (r.get("op") == "cochange" and _code(r) in ("target_normal", "primary_normal", "independent"))] tests = sum(1 for _, r in ok if r.get("op") == "differ") try: missing, note = _undecomposed([r for r in decomposes if _dims(r)], catalog, sources, load) diff --git a/backend/app/engine/compare.py b/backend/app/engine/compare.py index ad80bb25..57ba5409 100644 --- a/backend/app/engine/compare.py +++ b/backend/app/engine/compare.py @@ -115,10 +115,11 @@ _CFG_KEYS: dict[str, frozenset[str]] = { # 自己从不写这三项，默认设置改了就算它跟着变（报告节点自己写了的也算，宁可多算） "output": frozenset({"fields", "contract", "evidence_from", "metrics_from", "entities"}), "transform": frozenset({"mode", "expression", "template", "assign_to"}), - "llm": _MODEL_KEYS | {"system", "skills", "use_history", "prompt", "output_schema", "emit_message", "assign_to"}, + "llm": _MODEL_KEYS | {"system", "skills", "use_history", "prompt", "output_schema", "emit_message", "assign_to", + "ask_data_optimization"}, "agent": _MODEL_KEYS | {"system", "skills", "use_history", "prompt", "output_schema", "cite_fields", "tools", "memory_scope", "collection", "parallel_tools", "max_steps", "budget_tokens", "investigation", - "budget_usd", "approval", "emit_message", "assign_to"}, + "budget_usd", "approval", "emit_message", "assign_to", "ask_data_optimization"}, "supervisor": frozenset({"agents", "max_rounds", "on_exhausted", "goal", "provider", "model", "max_tokens", "max_parallel", "approval", "emit_message", "assign_to"}), "subgraph": frozenset({"workflow_id", "workflow_version", "input", "assign_to"}), @@ -135,7 +136,7 @@ _CFG_KEYS: dict[str, frozenset[str]] = { "metrics": frozenset({"metrics", "caliber", "caliber_version", "on_missing", "assign_to"}), "report": _MODEL_KEYS | {"numbers", "claims", "judge", "entities", "max_repairs", "system", "instructions", "on_violation", "evidence_from", "metrics_from", "emit_message", "assign_to", - "analysis_claims", "analysis_scope"}, + "analysis_claims", "analysis_scope", "ask_data_optimization", "reuse_answer_from"}, } #: 只在调模型时才读的（_calls_model）：条件分支让模型分类、结构校验让模型修复（默认开着）、知识检索用模型重排 _CFG_KEYS_IF: dict[str, frozenset[str]] = { diff --git a/backend/app/engine/evidence.py b/backend/app/engine/evidence.py index a52a3c70..a756f443 100644 --- a/backend/app/engine/evidence.py +++ b/backend/app/engine/evidence.py @@ -63,6 +63,10 @@ from decimal import ROUND_HALF_UP, Context, Decimal from typing import Any, Literal, TypedDict from app.analysis.contract import CATALOG_PREFIX, LEDGER_KIND, MARKER_KIND, catalog_result, is_result, method_label +from app.data.source_health import LEDGER_KIND as HEALTH_KIND, is_health_result + +HEALTH_MARKER = "h" +HEALTH_PREFIX = "H" from app.core.artifact_store import canonical_json, content_hash from app.core.artifact_store import load as load_artifact from app.data.names import FILLERS, name_key @@ -111,7 +115,7 @@ _ISO_DATETIME = re.compile(r"^(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}(?::\d{2}(?:\.\d+) class EvidenceEntry(TypedDict, total=False): """台账条目：state["evidence"] 里的一行，同时写进 node.finished.evidence。""" - kind: Literal["query", "metric_set", "retrieval", "tool", "node_output", "schema", "analysis"] + kind: Literal["query", "metric_set", "retrieval", "tool", "node_output", "schema", "analysis", "source_health"] node_id: str exec: int # 这个节点第几次执行（循环里会有多次），从 1 数 artifact: str # 被引用的那件工件 @@ -167,7 +171,7 @@ class Citation(TypedDict, total=False): class Segment(TypedDict, total=False): id: str #: analysis：[[a:A3]] 渲染出的判定语（一整句，系统按模板写的）；[[a:A3.键]] 的具名标量照数字、取值算 - kind: Literal["text", "number", "value", "entity", "quote", "structural", "analysis"] + kind: Literal["text", "number", "value", "entity", "quote", "structural", "analysis", "source_health"] text: str span: list[int] # [start, end)，在 doc.markdown 里的码点偏移 ref: str # 带前缀的原始引用：m:gmv、i:week、t:orders @@ -818,7 +822,7 @@ _MARKER_GUARD = rf"(?=[^\[\]\n]{{0,{MARKER_MAX - 4}}}\]\])" #: [[kind:body]]。body 里不许有方括号和换行——这样一个没闭合的 [[ 最多吞到行尾 MARKER_RE = re.compile( r"\[\[" + _MARKER_GUARD + r"[ \t]*(?P[A-Za-z]+)[ \t]*:(?P[^\[\]\n]*?)\]\]") -SUPPORTED_KINDS = frozenset({"m", "i", "see", "v", "table", "t", "c", "q", MARKER_KIND}) +SUPPORTED_KINDS = frozenset({"m", "i", "see", "v", "table", "t", "c", "q", MARKER_KIND, HEALTH_MARKER}) #: 三期起没有「以后才支持」的标记了；留着这个名字给老代码 import LATER_KINDS: frozenset[str] = frozenset() #: 这次运行没冻结表结构（没查过库、升级前的运行）时，t / c 仍按一期判为解析不了，原因是这一句 @@ -866,7 +870,7 @@ TABLE_DEFAULT_ROWS = 5 TABLE_MAX_ROWS = 20 TABLE_MAX_COLS = 12 _ROLE = {"t": "entity", "c": "entity", "q": "quote"} -_SEG_KIND = {"t": "entity", "c": "entity", "q": "quote"} +_SEG_KIND = {"t": "entity", "c": "entity", "q": "quote", HEALTH_MARKER: HEALTH_KIND} def _parse_body(kind: str, body: str) -> dict[str, Any]: @@ -1016,8 +1020,8 @@ def build_catalog( **({"comparability": metric["comparability"]} if isinstance(metric.get("comparability"), dict) else {}), } - queries = retrievals = analyses = 0 - snaps: _Snapshots | None = None # 取分析结论的工件用，有 A 条目才建 + queries = retrievals = analyses = healths = 0 + snaps: _Snapshots | None = None # 取分析结论或数据源健康工件用，有 A / H 条目才建 schemas: list[dict[str, Any]] = [] shaped: list[tuple[dict[str, Any], str]] = [] # 带表结构快照的查询条目和它的全局编号 for entry in entries: @@ -1043,6 +1047,12 @@ def build_catalog( snaps = snaps or _Snapshots(loader) catalog[alias] = _analysis_entry(entry, alias, snaps) continue + if entry.get("kind") == HEALTH_KIND: + healths += 1 + alias = f"{HEALTH_PREFIX}{healths}" + snaps = snaps or _Snapshots(loader) + catalog[alias] = _health_entry(entry, alias, snaps) + continue if entry.get("kind") == "query": queries += 1 alias, kind = f"Q{queries}", "query" @@ -1115,10 +1125,32 @@ def analysis_card(entry: dict[str, Any] | None) -> bool: return isinstance(entry, dict) and bool(entry.get("item")) and not entry.get("call_id") +_HEALTH_KEYS = ("node_id", "exec", "artifact", "call_id", "verdict", "sources", "issue_count") + + +def _health_entry(entry: dict[str, Any], alias: str, snaps: _Snapshots) -> dict[str, Any]: + """台账里的一条数据源健康核查 → 写作目录的 H 条目。""" + out: dict[str, Any] = { + "alias": alias, + "kind": HEALTH_KIND, + "eid": make_eid(HEALTH_KIND, entry.get("artifact"), {}), + "locator": {}, + **{k: entry[k] for k in _HEALTH_KEYS if k in entry}, + } + result, why = snaps.health(alias, {"artifact": entry.get("artifact")}) + if result is None: + out["unreadable"] = why + else: + out["result"] = result + phrase = _clean(str((result or {}).get("phrase") or "")) + out["label"] = f"数据源健康 · {phrase}" if phrase else "数据源健康" + return out + + def stored_entry(entry: dict[str, Any]) -> dict[str, Any]: - """存进报告文档的目录条目：分析结论只留摘要（编号、种类、判定码、工件、输入、行数……），不带整份结论（result）——它在 + """存进报告文档的目录条目：分析结论与数据源健康只留摘要（编号、种类、判定码、工件……），不带整份结论（result）——它在 工件里，取回时复验哈希；别的条目原样。""" - if entry.get("kind") == LEDGER_KIND and "result" in entry: + if entry.get("kind") in (LEDGER_KIND, HEALTH_KIND) and "result" in entry: return {k: v for k, v in entry.items() if k != "result"} return entry @@ -1524,6 +1556,18 @@ class _Snapshots: why = "的工件不是分析结论" return (None, f"{alias} {why}") if why else (content, None) + def health(self, alias: str, entry: dict[str, Any]) -> tuple[Any, str | None]: + """取回一份数据源健康度结论：(结论, 原因)。""" + cached = entry.get("result") + if is_health_result(cached): + return cached, None + if entry.get("unreadable"): + return None, str(entry["unreadable"]) + content, why = self._fetch(entry.get("artifact"), "数据源健康结论") + if why is None and not is_health_result(content): + why = "的工件不是数据源健康结论" + return (None, f"{alias} {why}") if why else (content, None) + def table(self, alias: str, entry: dict[str, Any]) -> tuple[Any, str | None]: """按单元格、整表取的那张表：(表, 取不到的原因)。查询结果是查询快照本身；分析结论是它的结果表（形状同查询快照： columns 是稳定的列 id，另带 labels 给人看的列名，整表的表头写它）。结果表另记了列类型（column_types）、各列的换算 @@ -1865,11 +1909,13 @@ def resolve_marker(marker: dict[str, Any], catalog: dict[str, Any], *, cells_all return _resolve_quote(marker, catalog, _snapshots(loader)) if kind == MARKER_KIND: return _resolve_analysis(marker, catalog, _snapshots(loader), cells_allowed) + if kind == HEALTH_MARKER: + return _resolve_health(marker, catalog, _snapshots(loader), cells_allowed) if kind not in ("m", "i"): return _unresolved(marker, f"{kind}:{ref}", kind, _Reason( f"引用类型「{kind}」无法识别", f"不认识的引用类型 {kind}：只支持 m（口径卡指标）、v（查询单元格）、table（整表）、" - "i（运行输入）、t / c（表 / 字段）、q（引文）、a（分析结论）、see（依据）")) + "i（运行输入）、t / c（表 / 字段）、q（引文）、a（分析结论）、h（数据源健康）、see（依据）")) conv = marker.get("conv") if kind == "m": @@ -1980,6 +2026,52 @@ def _resolve_analysis(marker: dict[str, Any], catalog: dict[str, Any], snaps: _S return cite +_HEALTH_REF = re.compile(rf"^(?P{HEALTH_PREFIX}\d+)$") +_HEALTH_FIX = f"数据源健康结论的判定语写 [[{HEALTH_MARKER}:H1]]，句末挂依据写 [[see:H1]]" + + +def _is_health_alias(alias: Any) -> bool: + return isinstance(alias, str) and alias.startswith(HEALTH_PREFIX) and alias[len(HEALTH_PREFIX):].isdigit() + + +def _resolve_health(marker: dict[str, Any], catalog: dict[str, Any], snaps: _Snapshots, + cells_allowed: bool) -> dict[str, Any]: + """[[h:H1]] → 数据源健康核查里系统渲染的判定语（整句涂白，核对封存哈希与显示字面）。""" + ref = (marker.get("ref") or "").strip() + conv = marker.get("conv") + parsed = _HEALTH_REF.match(ref) + if parsed is None: + return _unresolved(marker, ref or "H?", HEALTH_KIND, _Reason("数据源健康结论的引用格式无法识别", _HEALTH_FIX)) + alias = parsed.group("alias") + entry = catalog.get(alias) + if entry is None: + return _unresolved(marker, alias, HEALTH_KIND, _Reason( + f"引用的数据源健康结论 {alias} 不存在", f"目录里没有 {alias}（可用的健康核查见证据目录）")) + if entry.get("kind") != HEALTH_KIND: + return _unresolved(marker, alias, HEALTH_KIND, f"{alias} 不是数据源健康结论") + if not cells_allowed and entry.get("call_id"): + return _unresolved(marker, alias, HEALTH_KIND, ANALYSIS_CELLS_REASON) + if conv: + return _unresolved(marker, alias, HEALTH_KIND, _Reason( + "数据源健康判定语是一整句话，不能换算", f"判定语 [[{HEALTH_MARKER}:{alias}]] 不能加换算")) + result, why = snaps.health(alias, entry) + if result is None: + return _unresolved(marker, alias, HEALTH_KIND, why or f"{alias} 无法读取") + phrase = result.get("phrase") + if not isinstance(phrase, str) or not phrase.strip(): + return _unresolved(marker, alias, HEALTH_KIND, f"{alias} 没有判定语") + return { + "ref": ref, + "alias": alias, + "locator": {}, + "eid": make_eid(HEALTH_KIND, entry.get("artifact"), {}), + "kind": HEALTH_KIND, + "role": "value", + "status": "resolved", + "rendered": render_cell(phrase, kind="text"), + } + + def _resolve_entity(marker: dict[str, Any], catalog: dict[str, Any], snaps: _Snapshots) -> dict[str, Any]: """[[t:orders]] / [[c:orders.amount]]：显示名字本身（照写作者写的大小写），出处是目录里的表或字段。 @@ -2149,15 +2241,17 @@ def _resolve_quote(marker: dict[str, Any], catalog: dict[str, Any], snaps: _Snap "source": {k: v for k, v in source.items() if v is not None}} -_ROW_REF = re.compile(rf"^(?P[QK{CATALOG_PREFIX}]\d+)(?:\.r(?P\d+))?$") +_ROW_REF = re.compile(rf"^(?P[QK{CATALOG_PREFIX}{HEALTH_PREFIX}]\d+)(?:\.r(?P\d+))?$") def resolve_support(ref: str, catalog: dict[str, Any], *, loader: Loader | _Snapshots | None = None) -> dict[str, Any]: - """[[see:]] 里的一个依据：m:gmv、i:week、Q4、Q4.r0、K2、A3、A3.r0、t:orders、c:orders.amount。只核对「这件证据存在」。 - 分析结论当依据和查询编号一样写（[[see:A3]]），受管级别的出具里也照样可以写；顺手写成 a:A3 的照 A3 认。""" + """[[see:]] 里的一个依据：m:gmv、i:week、Q4、Q4.r0、K2、A3、A3.r0、H1、t:orders、c:orders.amount。只核对「这件证据存在」。 + 分析结论与数据源健康当依据和查询编号一样写（[[see:A3]] / [[see:H1]]），受管级别的出具里也照样可以写；顺手写成 a:A3 / h:H1 的照认。""" ref = ref.strip() if ref.startswith(f"{MARKER_KIND}:") and _is_analysis_alias(ref.partition(":")[2].strip()): ref = ref.partition(":")[2].strip() + elif ref.startswith(f"{HEALTH_MARKER}:") and _is_health_alias(ref.partition(":")[2].strip()): + ref = ref.partition(":")[2].strip() kind, sep, body = ref.partition(":") base = {"ref": ref, "role": "support"} if sep and kind in ("t", "c"): @@ -2178,15 +2272,18 @@ def resolve_support(ref: str, catalog: dict[str, Any], *, loader: Loader | _Snap why = f"运行输入中没有「{body}」" elif sep: why = _Reason(f"依据「{ref}」的写法无法识别", - f"不认识的依据写法 {ref}：写 m:指标、i:输入，或者 Q1 / K1 / A1 这样的编号") + f"不认识的依据写法 {ref}：写 m:指标、i:输入，或者 Q1 / K1 / A1 / H1 这样的编号") elif m := _ROW_REF.match(ref): alias = m.group("alias") if m.group("alias") in catalog else None if alias and m.group("row") is not None: - row, total = int(m.group("row")), catalog[alias].get("rows") - if isinstance(total, int) and row >= total: - alias, why = None, f"{m.group('alias')} 只有 {total} 行，没有第 {row} 行（行号从 0 开始）" + if catalog[alias].get("kind") == HEALTH_KIND: + alias, why = None, f"{m.group('alias')} 是数据源健康结论，没有行号，请直接写 [[see:{m.group('alias')}]]" else: - locator = {"row": row} + row, total = int(m.group("row")), catalog[alias].get("rows") + if isinstance(total, int) and row >= total: + alias, why = None, f"{m.group('alias')} 只有 {total} 行，没有第 {row} 行（行号从 0 开始）" + else: + locator = {"row": row} elif f"m:{ref}" in catalog: alias = f"m:{ref}" # 漏写了 m: 前缀的指标 id，意思没有歧义 if alias is None: @@ -3694,7 +3791,16 @@ def verify_doc( # 统计也不多这个键。目录条目只带摘要的（存进文档的那份）按工件取回整份结论 from app.engine import analysis_claims as said - said_found = said.verify(doc, catalog, cited=cited_by, load=lambda alias, entry: snaps.analysis(alias, entry)[0]) + said_found = said.verify( + doc, + catalog, + cited=cited_by, + load=lambda alias, entry: ( + snaps.health(alias, entry)[0] + if isinstance(entry, dict) and entry.get("kind") == HEALTH_KIND + else snaps.analysis(alias, entry)[0] + ), + ) if said_found is not None: violations.extend(said_found) stats["analysis_claims"] = sum(1 for v in said_found if v["code"] in said.CODES) @@ -4523,6 +4629,25 @@ def _cut_lines(lines: list[str], limit: int, note: str) -> list[str]: return out +def _health_prompt(catalog: dict[str, Any], snaps: _Snapshots, cells_allowed: bool) -> list[str]: + """数据源健康核查在写作目录里的文字（没有 H 条目时返回空列表，不影响既有目录）。""" + entries = [e for e in catalog.values() if isinstance(e, dict) and e.get("kind") == HEALTH_KIND] + if not entries: + return [] + lines = ["\n数据源健康核查（正文直接引用判定语写 [[h:H1]]，句末挂依据写 [[see:H1]]）："] + for e in entries: + alias = e["alias"] + result, why = snaps.health(alias, e) + if result is None: + lines.append(f"- {alias}：{why or '无法读取'}") + continue + phrase = _clean(str(result.get("phrase") or "")) + verdict = str(result.get("verdict") or "") + cite_tip = f"[[h:{alias}]]" if cells_allowed or not e.get("call_id") else f"[[see:{alias}]]（仅作句末依据）" + lines.append(f"- {alias}（判定={verdict}）：{phrase} —— 引用写 {cite_tip}") + return lines + + def writer_catalog(catalog: dict[str, Any], *, budget: int = DEFAULT_CATALOG_BUDGET, cells_allowed: bool = True, loader: Loader | None = None, fields: list[dict[str, Any]] | None = None, titles: dict[str, str] | None = None) -> WriterCatalog: @@ -4540,7 +4665,7 @@ def writer_catalog(catalog: dict[str, Any], *, budget: int = DEFAULT_CATALOG_BUD snaps = _Snapshots(loader) head = ["可引用的证据（写数字只能用下面的引用标记，系统会换成真实数值）："] field_lines, empty, cited = _fields_prompt(fields or [], catalog, snaps, cells_allowed) - first = [*_metric_prompt(catalog), *field_lines] + first = [*_metric_prompt(catalog), *field_lines, *_health_prompt(catalog, snaps, cells_allowed)] analyses = _AnalysisLines.of(catalog, snaps, cells_allowed) queries = [e for e in catalog.values() if e.get("kind") == "query"] rules = ([f"\n{CELL_RULES}" if cells_allowed else diff --git a/backend/app/engine/evidence_reader.py b/backend/app/engine/evidence_reader.py index 89b8e55c..7dc46eea 100644 --- a/backend/app/engine/evidence_reader.py +++ b/backend/app/engine/evidence_reader.py @@ -46,6 +46,8 @@ class Sealed: retrievals: dict[str, dict[str, Any]] = field(default_factory=dict) #: 封存范围内台账里记过的分析结论：工件 id → {node_id, exec, call_id / item, inputs} analyses: dict[str, dict[str, Any]] = field(default_factory=dict) + #: 封存范围内台账里记过的数据源健康度快照：工件 id → {node_id, exec, call_id, source_name, status} + source_healths: dict[str, dict[str, Any]] = field(default_factory=dict) #: 封存时记下的清单哈希（没封存的是 None）：旧答案的猜测按 (run_id, 它) 缓存 manifest_hash: str | None = None #: 运行记录上的 manifest_seq 原值（老运行是 None）：按需裁判追加判定前核对封存没换过 @@ -202,6 +204,10 @@ async def read_sealed(run_id: str) -> Sealed: out.analyses.setdefault(artifact, { "node_id": node_id, **{k: entry[k] for k in ("exec", "call_id", "item", "inputs") if entry.get(k) is not None}}) + elif entry.get("kind") == "source_health": + out.source_healths.setdefault(artifact, { + "node_id": node_id, **{k: entry[k] for k in ("exec", "call_id", "verdict", "sources", "issue_count") + if entry.get(k) is not None}}) return out diff --git a/backend/app/engine/guards.py b/backend/app/engine/guards.py index 8502f021..df4d4b95 100644 --- a/backend/app/engine/guards.py +++ b/backend/app/engine/guards.py @@ -89,7 +89,32 @@ def node_limits(config: dict[str, Any], snapshot: dict[str, Any], hard_cap: int) ) -def call_key(name: str, args: dict[str, Any]) -> str: +def call_key(name: str, args: dict[str, Any], *, canonical: bool = False) -> str: + if canonical: + args = dict(args) + if name.startswith("db_schema__") and "table" in args: + from app.tools.datasource import _schema_tables + + args["table"] = sorted(set(_schema_tables(args["table"]))) + if name.startswith("db_query__") and isinstance(args.get("sql"), str): + import sqlglot + + sql = args["sql"] + try: + tokens = sqlglot.tokenize(sql) + # Preserve every token verbatim, including literals, quoted identifiers and comments. + # Only collapse whitespace gaps: comments/non-whitespace gaps disqualify normalization. + end, pieces = 0, [] + for token in tokens: + if sql[end:token.start].strip(): + break + pieces.append(sql[token.start:token.end + 1]) + end = token.end + 1 + else: + if not sql[end:].strip(): + args["sql"] = " ".join(pieces) + except sqlglot.errors.TokenError: + pass return json.dumps({"tool": name, "args": args}, ensure_ascii=False, sort_keys=True, default=str) @@ -124,12 +149,13 @@ class Guard: last_input: int = 0 compressed: bool = False facts: dict[str, Any] = field(default_factory=dict) + canonical_calls: bool = False # ------------------------------------------------------------ 重复调用 def repeat(self, name: str, args: dict[str, Any]) -> str | None: """同一个工具、同样的参数已经成功跑过：返回要喂回去的那句话，不再执行。""" - hit = self.seen.get(call_key(name, args)) + hit = self.seen.get(call_key(name, args, canonical=self.canonical_calls)) if hit is None: return None step, content = hit @@ -138,7 +164,7 @@ class Guard: def record(self, step: int, name: str, args: dict[str, Any], ok: bool, content: str) -> None: if ok: - self.seen.setdefault(call_key(name, args), (step, content)) + self.seen.setdefault(call_key(name, args, canonical=self.canonical_calls), (step, content)) def step_done(self, progressed: bool) -> None: self.stalled = 0 if progressed else self.stalled + 1 diff --git a/backend/app/engine/investigation.py b/backend/app/engine/investigation.py index 02175371..881e4543 100644 --- a/backend/app/engine/investigation.py +++ b/backend/app/engine/investigation.py @@ -6,7 +6,7 @@ from typing import Any from pydantic import BaseModel, ConfigDict, Field -INVESTIGATION_VERSION = 1 +INVESTIGATION_VERSION = 2 class InvestigationPolicy(BaseModel): @@ -15,6 +15,9 @@ class InvestigationPolicy(BaseModel): max_rounds: int = Field(default=2, ge=0, le=5) minimum_depth: int = Field(default=1, ge=0, le=5) min_coverage: float = Field(default=0.8, ge=0, le=1) + max_ontology_hops: int = Field(default=2, ge=0, le=5) + max_cross_queries: int = Field(default=6, ge=0, le=20) + check_source_health_first: bool = True def _number(value: Any) -> float: @@ -25,24 +28,117 @@ def _number(value: Any) -> float: return 0.0 +def _neighbor_queried(neighbor: dict[str, Any], queried_calibers: set[str], queried_sources: set[str], + focal_sources: set[str], ontology_hops: int) -> bool: + if ontology_hops >= 1: + target = neighbor.get("target") or {} + wf_id = str(target.get("workflow_id") or "") + card_id = str(target.get("card_id") or "") + if wf_id and (f"{wf_id}:{card_id}" in queried_calibers or wf_id in queried_calibers): + return True + peer_sources = [str(s) for s in (target.get("sources") or []) if s] + if any(s in queried_sources and s not in focal_sources for s in peer_sources): + return True + return not wf_id and not peer_sources + return False + + def assess(scope: Any, question: str, policy: InvestigationPolicy) -> dict[str, Any]: from app.analysis.intent import analysis_intent - results = list(scope.results.values()) if scope is not None else [] + results = list(scope.results.values()) if scope is not None and hasattr(scope, "results") else [] + healths = [h for h in (getattr(scope, "source_healths", None) or {}).values() if isinstance(h, dict)] + neighbors = [n for n in (getattr(scope, "ontology_neighbors", None) or []) if isinstance(n, dict)] + contract_violations = list(getattr(scope, "contract_violations", None) or []) + ontology_hops = int(getattr(scope, "ontology_hops", 0) or 0) + cross_queries = int(getattr(scope, "cross_queries", 0) or 0) + queried_calibers = set(getattr(scope, "queried_calibers", None) or ()) + queried_sources = set(getattr(scope, "queried_sources", None) or ()) + focal_sources = set(getattr(scope, "focal_sources", None) or ()) + + has_health_anomaly = any(h.get("verdict") == "anomalous" for h in healths) + has_reconcile_fault = any( + r.get("status") == "ok" + and r.get("op") == "reconcile" + and (r.get("verdict") or {}).get("code") in ("diverged", "missing_counterpart") + for r in results + ) intent = analysis_intent(question) active = bool(policy.enabled and scope is not None and ( - (intent is not None and intent.kind == "why") or any( + (intent is not None and intent.kind == "why") + or any( r.get("status") == "ok" and (r.get("verdict") or {}).get("code") in ("outside_band", "far_outside_band") - for r in results))) - out: dict[str, Any] = {"version": INVESTIGATION_VERSION, "active": active, "status": "not_required", "depth": 0, "coverage": None, - "query_count": len(scope.queries) if scope is not None else 0, - "analysis_count": len(results), "reason": "", "next_step": ""} + for r in results + ) + or has_health_anomaly + or has_reconcile_fault + or bool(contract_violations) + )) + out: dict[str, Any] = { + "version": INVESTIGATION_VERSION, + "active": active, + "status": "not_required", + "depth": 0, + "coverage": None, + "query_count": len(scope.queries) if scope is not None and hasattr(scope, "queries") else 0, + "analysis_count": len(results), + "ontology_hops": ontology_hops, + "source_health_checked": bool(healths), + "reason": "", + "next_step": "", + } if not active: return out def pending(reason: str, next_step: str, stage: str) -> dict[str, Any]: return {**out, "status": "partial", "stage": stage, "reason": reason, "next_step": next_step} + # 第一阶段：数据源健康度、跨源对账与物理契约检查（发现源表故障或对账超差立即早停） + if has_health_anomaly: + return { + **out, + "status": "complete", + "stage": "health_and_contract", + "reason": "关联数据源健康度核查发现断更、断档或数据骤降异常，已锁定数据质量问题，停止业务归因拆解", + } + if has_reconcile_fault: + return { + **out, + "status": "complete", + "stage": "health_and_contract", + "reason": "跨源镜像口径对账超差或对侧缺失，已锁定跨源数据或单位不一致问题，停止业务归因拆解", + } + if contract_violations: + if not neighbors: + return pending( + "指标触发契约或值域越界，尚未查询本体构成因子与关联数据源", + "请先调用 ontology_neighbors 查看该指标的构成因子（factor）与跨源实体桥，再核查对侧基准值或数据源健康度。", + "health_and_contract", + ) + if ontology_hops < 1 and policy.max_ontology_hops > 0 and cross_queries < policy.max_cross_queries: + return pending( + "已发现构成因子或跨源实体桥，尚未穿透核对对侧主数据或因子口径", + "请按 ontology_neighbors 的 recommended_caliber_query 或标准实体桥查询关联因子与对侧主数据，核实是否存在跨库失配。", + "ontology_penetration", + ) + return { + **out, + "status": "complete", + "stage": "ontology_penetration", + "reason": "已完成契约越界指标的跨源本体穿透与因子核对", + } + if ( + policy.check_source_health_first + and any(n.get("kind") == "reconciles_with" for n in neighbors) + and not healths + and not any(r.get("op") == "reconcile" for r in results) + ): + return pending( + "发现已确认的跨源对账镜像口径，尚未核查对侧数据源健康度或对账一致性", + "请先调用 source_health 核查对侧数据源健康度，或调用 analysis_reconcile 核对镜像口径，排除断更与单位失配后再拆解业务维度。", + "health_and_contract", + ) + done = [r for r in results if r.get("status") == "ok"] decomposed = [r for r in done if r.get("op") == "decompose"] if not decomposed: @@ -99,4 +195,38 @@ def assess(scope: Any, question: str, policy: InvestigationPolicy) -> dict[str, return pending("主要贡献项尚未进一步下钻", "对主要贡献项选择更细的业务维度，用 parent 关联上层结果继续下钻；没有合适维度时说明限制。", "drill") if last.get("mode") == "additive" and coverage < policy.min_coverage: return pending("主要贡献项的解释覆盖程度不足", "检查其他候选维度和剩余差异，保留未解释的部分，不把贡献分解写成因果证明。", "coverage") + + # 第三阶段：同源切片定位完成后，对已发现的 1 跳 factor / upstream_driver 本体邻居执行有界跨源穿透核查 + driver_neighbors = [n for n in neighbors if n.get("kind") in ("upstream_driver", "factor")] + if driver_neighbors and policy.max_ontology_hops > 0: + within_budget = ontology_hops < policy.max_ontology_hops and cross_queries < policy.max_cross_queries + if not any(_neighbor_queried(n, queried_calibers, queried_sources, focal_sources, ontology_hops) + for n in driver_neighbors) and within_budget: + return pending( + "已完成同源切片定位，尚未沿本体穿透核查关联因子或上游驱动口径", + "请将定位到的异常切片通过标准实体桥对齐后调用 caliber_query 查询邻居指标，验证跨源表现。", + "ontology_penetration", + ) + if any(n.get("kind") == "upstream_driver" for n in driver_neighbors): + cochange_done = [r for r in done if r.get("op") == "cochange"] + if not cochange_done: + cochange_refused = [r for r in results if r.get("op") == "cochange" and r.get("status") == "refused"] + if cochange_refused: + return pending( + str(cochange_refused[-1].get("reason") or "上游共变检验样本不足"), + "请检查两序列的时间窗口与有效点数；无法补齐时明确说明统计共变尚未证实，不写因果结论。", + "ontology_penetration", + ) + if cross_queries <= policy.max_cross_queries: + return pending( + "已获取跨源上游驱动数据，尚未执行同步共变统计检验", + "请调用 analysis_cochange 检验主指标与上游驱动序列是否存在同窗或滞后共变，注意相关性不等于因果。", + "ontology_penetration", + ) + return { + **out, + "status": "complete", + "stage": "ontology_penetration", + "reason": "已完成同源切片定位与跨源本体穿透核查，相关性结论仍需独立业务证据确认因果", + } return {**out, "status": "complete", "stage": "contribution", "reason": "完成贡献分析，因果关系仍需独立证据"} diff --git a/backend/app/engine/labels.py b/backend/app/engine/labels.py index b16ccee2..d10a44f2 100644 --- a/backend/app/engine/labels.py +++ b/backend/app/engine/labels.py @@ -63,6 +63,7 @@ FIELD_LABEL: dict[str, str] = { "judge": "结论句裁判", "entities": "表名、字段名", # 分析能力一轮：判断词要挂对应的分析结论（engine/analysis_claims）、文档末尾系统写的分析范围（engine/analysis_scope） "analysis_claims": "分析说法核对", "analysis_scope": "分析范围", + "ask_data_optimization": "问数执行策略", "reuse_answer_from": "复用答案来源", # 口径卡 "caliber_from": "口径卡来源", "caliber": "口径名称", "caliber_version": "口径版本", "metrics": "指标定义", "on_missing": "缺输入时", diff --git a/backend/app/engine/nodes/llm.py b/backend/app/engine/nodes/llm.py index 4b036ba4..054ed170 100644 --- a/backend/app/engine/nodes/llm.py +++ b/backend/app/engine/nodes/llm.py @@ -49,18 +49,25 @@ from app.providers.factory import ( ) from app.engine.sql_problems import event_checks from app.data import catalog_view +from app.data.source_health import ledger_entry as health_ledger_entry +from app.ledger.querycore import CALIBER_QUERY from app.tools.analysis import ( REFUSED_CODE as ANALYSIS_REFUSED, SCOPE_KEY as ANALYSIS_SCOPE_KEY, AnalysisScope, call_error_text, event_view, + health_event_view, + health_payload_of, + health_reply_text, input_snapshot, is_analysis_tool, payload_of, + record_ontology_call, reply_text, run_snapshot, ) +from app.tools.ontology import ONTOLOGY_TOOLS, SOURCE_HEALTH from app.tools.datasource import QUERY_PREFIX, run_masks_results, run_reads_only, run_scope_fixed, run_versions from app.tools.registry import ( ToolArgsError, @@ -132,6 +139,10 @@ async def _build_messages(state: GraphState, ctx: NodeContext, tools: list[BaseT messages: list[BaseMessage] = [] system_parts: list[str] = [] + if ctx.cfg("ask_data_optimization") == 1: + from app.chat.optimization import CONCISE_RULE + + system_parts.append(CONCISE_RULE) system = ctx.render_str(ctx.cfg("system", ""), state) if system: system_parts.append(system) @@ -421,7 +432,8 @@ def _guard(ctx: NodeContext, model_id: str) -> Guard | None: {k: ctx.cfg(k) for k in ("max_steps", "budget_tokens", "budget_usd")}, ctx.run.agent_limits, settings.max_agent_steps, ) - return Guard(limits=limits, window=catalog.find_context(model_id) or FALLBACK_CONTEXT) + return Guard(limits=limits, window=catalog.find_context(model_id) or FALLBACK_CONTEXT, + canonical_calls=ctx.cfg("ask_data_optimization") in (1, 2)) def _policy(ctx: NodeContext, tool: BaseTool, args: dict[str, Any], granted: set[str]) -> str: @@ -456,9 +468,12 @@ async def run_agent(state: GraphState, ctx: NodeContext) -> dict[str, Any]: except ProviderNotConfigured as e: raise NodeError(ctx.node.id, not_configured(e)) from e - # 绑了分析工具才建分析容器：本节点查过的结果、做过的分析都放在这里（app/tools/analysis.py），编号和交回模型的一致 + # 绑了分析工具、本体工具、口径查询或显式开启调查策略时建分析容器：本节点查过的结果、做过的分析与健康核查都放在这里 + bound_names = ctx.cfg("tools", []) or [] scope = (AnalysisScope(node_id=ctx.node.id, snapshot=run_snapshot(ctx.run)) - if any(is_analysis_tool(n) for n in ctx.cfg("tools", []) or []) else None) + if (any(is_analysis_tool(n) or n in ONTOLOGY_TOOLS or n == CALIBER_QUERY for n in bound_names) + or bool(ctx.cfg("investigation"))) + else None) tools = await _resolve_tools(ctx, state, scope) tool_map = {t.name: t for t in tools} # 默认串行：一轮只调一个工具，拿到结果再想下一步。并行更快，但一批工具 @@ -493,12 +508,54 @@ async def run_agent(state: GraphState, ctx: NodeContext) -> dict[str, Any]: total_usage[key] += usage[key] total_usage["cost_usd"] = round(total_usage["cost_usd"] + usage["cost_usd"], 6) + inline_evidence = ctx.cfg("ask_data_optimization") == 2 and evidence_on + + def _live_catalog() -> dict[str, Any]: + from app.engine.evidence import build_catalog + + return build_catalog( + nodes=state.get("nodes") or {}, + ledger=[*(state.get("evidence") or []), *entries], + inputs=state.get("input") or {}, + ) + + class _InlineStreamProxy: + def __init__(self, inner_ctx: NodeContext, live_cat: dict[str, Any]) -> None: + from app.engine.evidence import StreamRenderer + + self._ctx = inner_ctx + self._cat = live_cat + self._renderer = StreamRenderer(live_cat, cells_allowed=True) + + def __getattr__(self, attr: str) -> Any: + return getattr(self._ctx, attr) + + def emit(self, event_type: EventType | str, **data: Any) -> None: + if event_type == EventType.LLM_TOKEN: + if txt := self._renderer.feed(str(data.get("delta") or "")): + self._ctx.emit(EventType.LLM_TOKEN, delta=txt) + return + if event_type == EventType.LOG and data.get("code") == "stream_fallback": + from app.engine.evidence import StreamRenderer + + self._renderer = StreamRenderer(self._cat, cells_allowed=True) + self._ctx.emit(event_type, **data) + + def flush(self) -> None: + if tail := self._renderer.flush(): + self._ctx.emit(EventType.LLM_TOKEN, delta=tail) + # @task 的返回值会进 checkpoint：人工审批导致节点重放时， # 之前已经完成的模型调用和工具执行不会重跑，也就不会重复计费。 @task async def llm_step(step: int, payload: list[BaseMessage]) -> BaseMessage: started = time.perf_counter() - response = await _invoke_streaming(model, payload, ctx, model_id) + if inline_evidence and any(e.get("kind") == "query" for e in entries): + proxy = _InlineStreamProxy(ctx, _live_catalog()) + response = await _invoke_streaming(model, payload, proxy, model_id) # type: ignore[arg-type] + proxy.flush() + else: + response = await _invoke_streaming(model, payload, ctx, model_id) _report(response, started) return response @@ -507,7 +564,12 @@ async def run_agent(state: GraphState, ctx: NodeContext) -> dict[str, Any]: # 用 base_model 而不是上面那个绑过工具的 model：收尾轮必须在**结构上** # 发不出工具调用，靠提示词说"别调工具"是约束不住的 started = time.perf_counter() - response = await _invoke_streaming(base_model, payload, ctx, model_id) + if inline_evidence and any(e.get("kind") == "query" for e in entries): + proxy = _InlineStreamProxy(ctx, _live_catalog()) + response = await _invoke_streaming(base_model, payload, proxy, model_id) # type: ignore[arg-type] + proxy.flush() + else: + response = await _invoke_streaming(base_model, payload, ctx, model_id) _report(response, started) return response @@ -570,6 +632,8 @@ async def run_agent(state: GraphState, ctx: NodeContext) -> dict[str, Any]: # 交回模型的「【分析 An】」是同一个（_record_call 紧接着按同一个容器编），不记台账的执行没有编号 analysis = payload_of(content) if ok and is_analysis_tool(name) else None alias = scope.next_alias() if analysis and scope is not None and evidence_on else None + health = health_payload_of(content) if ok and name == SOURCE_HEALTH else None + health_alias = scope.next_health_alias() if health and scope is not None and evidence_on else None ctx.emit( EventType.TOOL_END if ok else EventType.TOOL_ERROR, tool=name, @@ -582,6 +646,8 @@ async def run_agent(state: GraphState, ctx: NodeContext) -> dict[str, Any]: **({"query_artifacts": [q["artifact"] for q in queries]} if len(queries) > 1 else {}), **({"analysis_artifact": analysis["artifact"], "analysis": event_view(analysis["result"], alias)} if analysis else {}), + **({"health_artifact": health["artifact"], "source_health": health_event_view(health["result"], health_alias)} + if health else {}), # 查询当时数据源的表结构快照：和查询快照一样落在封存范围内，证据接口从这里认它 **({"schema_artifact": first["schema_artifact"]} if first and first.get("schema_artifact") else {}), # 这次查询的 SQL 检查结果单独带上：preview 截到前 2000 个字符，checks 通常在截掉的那段里 @@ -804,12 +870,17 @@ async def run_agent(state: GraphState, ctx: NodeContext) -> dict[str, Any]: outcome = await tool_step(step, name, args, call_id) content = outcome["content"] + if outcome["ok"]: + record_ontology_call(scope, name, content) record = {"tool": name, "args": args, "ok": outcome["ok"], "duration_ms": outcome["duration_ms"], "result": content[:4000]} if evidence_on: # 编号只取决于 tool_step 的返回值（重放时取自 checkpoint），重放出来的还是这一套 content = _record_call(ctx.node.id, exec_no, call_id, name, args, outcome, record, queries, entries, scope, run_id=ctx.run.run_id) + elif scope is not None and outcome["ok"] and name == SOURCE_HEALTH: + if found_health := health_payload_of(content): + scope.add_health(found_health["result"], found_health["artifact"]) transcript.append(record) messages.append(ToolMessage(content=content, tool_call_id=call_id)) if guard: @@ -925,9 +996,22 @@ async def run_agent(state: GraphState, ctx: NodeContext) -> dict[str, Any]: extra = {"max_tokens": extracted.get("max_tokens")} if code == "agent_field_truncated" else {} await once(ctx, EventType.LOG, level="warn", code=code, message=message, fields=names, **extra) + raw_final_text: str | None = None + shown_text = final_text + if inline_evidence and final_text: + raw_final_text = final_text + if any(e.get("kind") == "query" for e in entries): + from app.chat.direct_evidence import normalize_and_bind_fast_citations + from app.engine.evidence import render_markers + + live_cat = _live_catalog() + cleaned_raw = normalize_and_bind_fast_citations(final_text, live_cat) + shown_text = render_markers(cleaned_raw, live_cat, cells_allowed=True) + total_usage["total_tokens"] = total_usage["input_tokens"] + total_usage["output_tokens"] result = { - "text": final_text, + "text": shown_text, + **({"raw_text": raw_final_text} if raw_final_text is not None else {}), "steps": len(transcript), **({"investigation": investigation} if investigation["active"] else {}), "tool_calls": transcript, @@ -944,11 +1028,11 @@ async def run_agent(state: GraphState, ctx: NodeContext) -> dict[str, Any]: if entries: updates["evidence"] = entries if ctx.cfg("emit_message", True): - updates["messages"] = [AIMessage(content=final_text or "（空）")] + updates["messages"] = [AIMessage(content=shown_text or "（空）")] var_name = ctx.cfg("assign_to", "") if var_name: # 开了 cite_fields 的，变量里是核对过的 data（值以快照为准），下游口径卡按字段取 - updates["vars"] = {var_name: cited[0] if cited is not None else final_text} + updates["vars"] = {var_name: cited[0] if cited is not None else shown_text} return updates @@ -987,6 +1071,15 @@ def _record_call(node_id: str, exec_no: int, call_id: str, name: str, args: dict entries.append(ledger_entry(found["result"], node_id=node_id, exec_no=exec_no, artifact=found["artifact"], call_id=call_id)) return reply_text(alias, found["result"]) + if scope is not None and outcome.get("ok") and name == SOURCE_HEALTH: + found_health = health_payload_of(content) + if found_health is None: + return content + h_alias = scope.add_health(found_health["result"], found_health["artifact"]) + record.update(alias=h_alias, health_artifact=found_health["artifact"]) + entries.append(health_ledger_entry(found_health["result"], node_id=node_id, exec_no=exec_no, + artifact=found_health["artifact"], call_id=call_id)) + return health_reply_text(h_alias, found_health["result"]) items = _called_queries(outcome, content, name) if not items: return content diff --git a/backend/app/engine/nodes/metrics.py b/backend/app/engine/nodes/metrics.py index 154d2a44..2c95c05f 100644 --- a/backend/app/engine/nodes/metrics.py +++ b/backend/app/engine/nodes/metrics.py @@ -421,6 +421,30 @@ async def run_metrics(state: GraphState, ctx: NodeContext) -> dict[str, Any]: metrics=[m["id"] for m in lacking], metric_names=[m["name"] for m in lacking], reason=lacking[0]["no_data_reason"], message="指标" + "".join(f"「{m['name']}」" for m in lacking) + lacking[0]["no_data_reason"]) + if violated := [m for m in metrics if m.get("contract_findings")]: + raw_findings = [ + dict(f) if isinstance(f, Mapping) else str(f) + for m in violated + for f in (m.get("contract_findings") or []) + if f + ] + first_finding = raw_findings[0] if raw_findings else "" + detail = ( + str(first_finding.get("detail") or first_finding.get("code") or "触发指标数值契约异常") + if isinstance(first_finding, Mapping) + else (str(first_finding) or "触发指标数值契约异常") + ) + ctx.emit( + EventType.LOG, + level="warn", + code="metric_contract_violated", + metric=violated[0]["id"], + metrics=[m["id"] for m in violated], + metric_names=[m["name"] for m in violated], + findings=raw_findings, + reason=detail, + message="指标" + "".join(f"「{m['name']}」" for m in violated) + f"触发数值契约告警：{detail}", + ) card = { "kind": "metric_set", "validation_version": 1, diff --git a/backend/app/engine/nodes/report.py b/backend/app/engine/nodes/report.py index 84a9708b..56cd14bc 100644 --- a/backend/app/engine/nodes/report.py +++ b/backend/app/engine/nodes/report.py @@ -547,6 +547,15 @@ class _MaskedCells: snap = None self._memo[artifact] = await _cited_masks(snap, source, self._masks, artifact) if isinstance(snap, dict) else {} + async def all_query_masks(self) -> dict[str, dict[str, list[str]]]: + """预读当前目录全部查询快照的遮罩列映射，供快速直查在直通组装前过滤遮罩列。""" + for entry in self._catalog.values(): + if isinstance(entry, dict) and entry.get("kind") == "query": + artifact = entry.get("artifact") + if isinstance(artifact, str) and artifact and artifact not in self._memo: + await self._read(artifact, entry.get("source")) + return {a: m for a, m in self._memo.items() if m} + def _masked_said(hits: list[dict[str, Any]]) -> str: """报告引用了遮罩列的那几处在缺口、日志里怎么说：「引用了 2 处取自遮罩列的格子（「v:Q2.r0.tel」、…）」，引用了值取自 @@ -791,19 +800,155 @@ async def _store(doc: dict[str, Any], ctx: NodeContext) -> str | None: return None +async def _run_fast_inline_report(state: GraphState, ctx: NodeContext, on_violation: str) -> dict[str, Any]: + """快速直查专用零模型直通证据封装：0 次 LLM 调用、< 10ms 纯内存组装 report_doc 与封存事件。""" + from app.api.evidence import withheld_metrics + from app.chat.direct_evidence import normalize_and_bind_fast_citations + from app.engine.evidence import iter_segments + + started = time.perf_counter() + source = str(ctx.cfg("reuse_answer_from", "")) + previous = (state.get("nodes") or {}).get(source) or {} + raw_answer = str( + previous.get("raw_text") if isinstance(previous.get("raw_text"), str) else (previous.get("text") or "") + ).strip() + shown_fallback = str(previous.get("text") or raw_answer).strip() + evidence_on = ledger_enabled(ctx.run) + zero_usage = {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cost_usd": 0, "calls": 0} + upstream_model = str(previous.get("model") or "evidence") + + def _plain_pass(text_val: str) -> dict[str, Any]: + res = { + "text": text_val, + "doc_artifact": None, + "stats": {}, + "violations": [], + "repairs": 0, + "model": upstream_model, + "duration_ms": int((time.perf_counter() - started) * 1000), + "answer_reused": True, + "writer_calls": 0, + **({"claims": "off", "analysis_claims": "off"} if evidence_on else {}), + } + upd: dict[str, Any] = {"nodes": {ctx.node.id: res}, "usage": zero_usage} + if ctx.cfg("emit_message", True): + upd["messages"] = [AIMessage(content=text_val or "（空）")] + var_name = ctx.cfg("assign_to", "") + if var_name: + upd["vars"] = {var_name: text_val} + return upd + + if not evidence_on or not raw_answer: + return _plain_pass(shown_fallback) + + spec = ctx.run.spec + upstream = {**state, "nodes": {k: v for k, v in (state.get("nodes") or {}).items() if k != ctx.node.id}} + catalog = report_catalog(upstream, spec, ctx.node) # type: ignore[arg-type] + if not any(entry.get("kind") == "query" for entry in catalog.values()): + return _plain_pass(shown_fallback) + + withheld = await withheld_metrics(catalog, nodes=state.get("nodes") or {}, ledger=state.get("evidence") or []) + catalog = withhold_metrics(catalog, withheld, _masked_word()) + masked = _MaskedCells(catalog) + query_masks = await masked.all_query_masks() + + prepared = normalize_and_bind_fast_citations(_unfenced(raw_answer), catalog, masked_columns=query_masks) + entities = report_entities(spec, ctx.node) == "link" + allow = report_allowance(spec, ctx.node) + doc = await masked( + compose_doc( + prepared, + catalog, + node_id=ctx.run.global_id(ctx.node.id), + run_id=ctx.run.run_id, + allow_numbers=allow, + cells_allowed=True, + entities=entities, + absence=False, + analysis_claims=None, + ) + ) + text = doc["markdown"] + has_cited = doc["stats"].get("numbers_cited", 0) > 0 or any( + s.get("state") == "deterministic" for s in iter_segments(doc) + ) + if not has_cited: + return _plain_pass(text) + + doc_artifact = await _store(doc, ctx) + if doc_artifact: + ctx.emit( + EventType.REPORT_CHECKED, + doc_artifact=doc_artifact, + ok=not doc["violations"], + stats=doc["stats"], + violations=doc["violations"][:20], + repairs=0, + on_violation=on_violation, + failed=False, + claims="off", + analysis_claims="off", + ) + ctx.emit( + EventType.LOG, + level="info", + code="report_answer_reused", + message="已零延迟核验查询证据并生成可点击溯源文档", + ) + + result = { + "text": text, + "doc_artifact": doc_artifact, + "stats": doc["stats"], + "violations": doc["violations"][:20], + "repairs": 0, + "model": upstream_model, + "duration_ms": int((time.perf_counter() - started) * 1000), + "answer_reused": True, + "writer_calls": 0, + "claims": "off", + "analysis_claims": "off", + **({"masked_metrics": withheld} if withheld else {}), + } + updates: dict[str, Any] = {"nodes": {ctx.node.id: result}, "usage": zero_usage} + if ctx.cfg("emit_message", True): + updates["messages"] = [AIMessage(content=text or "（空）")] + var_name = ctx.cfg("assign_to", "") + if var_name: + updates["vars"] = {var_name: text} + return updates + + async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: numbers, max_repairs = _options(ctx) on_violation = str(ctx.cfg("on_violation", "") or "") or await _default_on_violation(ctx.run.run_id) if on_violation not in ON_VIOLATION: raise NodeError(ctx.node.id, _only("on_violation", ("flag", "fail"), on_violation)) - async with SessionLocal() as session: - try: - model, model_id = await get_chat_model(session, _model_spec(ctx)) - except ProviderNotConfigured as e: - raise NodeError(ctx.node.id, not_configured(e)) from e - # 写作目录的篇幅按这个模型的上下文窗口定（evidence.catalog_budget） - window = await model_window(session, _model_spec(ctx), model_id) + if ctx.cfg("ask_data_optimization") == 2: + return await _run_fast_inline_report(state, ctx, on_violation) + + candidate = "" + if ctx.cfg("ask_data_optimization") == 1 and ledger_enabled(ctx.run) \ + and await _run_class(ctx.run.run_id) == "exploratory": + source = str(ctx.cfg("reuse_answer_from", "")) + parents = {e.source for e in ctx.run.spec.edges if e.target == ctx.node.id} + agent_ids = {n.id for n in ctx.run.spec.nodes if n.type == NodeType.AGENT} + previous = (state.get("nodes") or {}).get(source) or {} + raw_answer = previous.get("text") + if source in parents & agent_ids and not previous.get("limited") and isinstance(raw_answer, str) \ + and 0 < len(raw_answer.strip()) <= 1200 and re.search(r"\[\[(?:v|m):", raw_answer) \ + and not leaked_markup(raw_answer): + candidate = raw_answer + + model, model_id, window = None, "evidence", 4096 + if not candidate: + async with SessionLocal() as session: + try: + model, model_id = await get_chat_model(session, _model_spec(ctx)) + except ProviderNotConfigured as e: + raise NodeError(ctx.node.id, not_configured(e)) from e + window = await model_window(session, _model_spec(ctx), model_id) spec = ctx.run.spec # 目录按上游的产出建：这个节点自己上一次的产出（循环里重跑时）不算，它记下的 masked_metrics 这一次重新认 @@ -866,7 +1011,13 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: nudged = False async def draft(payload: list[BaseMessage]) -> tuple[BaseMessage, str]: - nonlocal nudged + nonlocal nudged, model, model_id + if model is None: + async with SessionLocal() as session: + try: + model, model_id = await get_chat_model(session, _model_spec(ctx)) + except ProviderNotConfigured as e: + raise NodeError(ctx.node.id, not_configured(e)) from e stream = _RenderedStream(ctx, catalog, cells) began = time.perf_counter() response = await _invoke_streaming(model, payload, stream, model_id) # type: ignore[arg-type] @@ -904,8 +1055,18 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: doc["analysis_scope"] = scope return doc - response, raw = await draft(messages) - doc = await compose(raw) + reused = False + if candidate and claims != CLAIMS_JUDGE and not failed and not writer.trimmed \ + and not any(entry.get("truncated") for entry in catalog.values()): + doc = await compose(candidate) + reused = not doc["violations"] and doc["stats"].get("numbers_cited", 0) > 0 + if reused: + response = AIMessage(content=candidate) + ctx.emit(EventType.LOG, level="info", code="report_answer_reused", + message="已直接核验问数答案和引用证据，无需再次调用模型撰写") + else: + response, raw = await draft(messages) + doc = await compose(raw) repairs = 0 while (blocking := _blocking(doc["violations"], numbers)) and repairs < max_repairs \ and not (repairs and not _again(blocking, analysis)): @@ -986,7 +1147,8 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: f"报告有 {len(blocking)} 处未通过核对（已要求重写 {repairs} 次）：{head}{more}。{_fail_advice(blocking)}", ) - usage = {key: sum(one[key] for one in spent) for key in spent[0]} + usage = ({key: sum(one[key] for one in spent) for key in spent[0]} if spent else + {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "cost_usd": 0, "calls": 0}) usage["cost_usd"] = round(usage["cost_usd"], 6) text = doc["markdown"] result = { @@ -997,6 +1159,7 @@ async def run_report(state: GraphState, ctx: NodeContext) -> dict[str, Any]: "repairs": repairs, "model": model_id, "duration_ms": int((time.perf_counter() - started) * 1000), + **({"answer_reused": reused, "writer_calls": len(spent)} if ctx.cfg("ask_data_optimization") == 1 else {}), **({"claims": claims} if evidence_on else {}), # 分析说法核对的处置：出口按它和文档里记的那一项判 strict 要不要记缺口（升级前的运行没有这个键） **({"analysis_claims": analysis} if evidence_on else {}), diff --git a/backend/app/engine/nodes/tools.py b/backend/app/engine/nodes/tools.py index 7f681e24..a7317eae 100644 --- a/backend/app/engine/nodes/tools.py +++ b/backend/app/engine/nodes/tools.py @@ -298,11 +298,18 @@ async def run_tool(state: GraphState, ctx: NodeContext) -> dict[str, Any]: schema = first.get("schema_artifact") if first else None # 这次查询的 SQL 检查结果单独带上：preview 只有前 2000 个字符，checks 排在 rows 后面，通常被截掉 checks = event_checks(result) if is_platform_query(name) else [] + health: dict[str, Any] | None = None + if name == "source_health": + from app.tools.analysis import health_event_view, health_payload_of + + health = health_payload_of(result) ctx.emit(EventType.TOOL_END, tool=name, duration_ms=elapsed, preview=json.dumps(result, ensure_ascii=False, default=str)[:2000], artifact=snapshot_id, **({"query_artifact": first["artifact"]} if first else {}), # 一次调用交回多条查询（caliber_query）时把工件 id 全列出来：只给第一条，界面上剩下的就点不开 **({"query_artifacts": [q["artifact"] for q in queries]} if len(queries) > 1 else {}), + **({"health_artifact": health["artifact"], "source_health": health_event_view(health["result"])} + if health else {}), **({"schema_artifact": schema} if schema else {}), **({"checks": checks} if checks else {})) updates: dict[str, Any] = {"nodes": {ctx.node.id: result}} @@ -313,6 +320,11 @@ async def run_tool(state: GraphState, ctx: NodeContext) -> dict[str, Any]: if snapshot_id: entries.append({"kind": "tool", "node_id": ctx.node.id, "exec": exec_no, "artifact": snapshot_id, "tool": name}) + if health: + from app.data.source_health import ledger_entry as health_ledger_entry + + entries.append(health_ledger_entry(health["result"], node_id=ctx.node.id, exec_no=exec_no, + artifact=health["artifact"])) for item in queries: # 实际执行的执行身份（K10）：query_id 是逻辑查询、execution_id 是这一次、attempt 是第几次。 # 口径查询那条自己带着（K09 算好的），别的按这次节点执行的序号算 diff --git a/backend/app/engine/review.py b/backend/app/engine/review.py index cc82c7f2..be0f215f 100644 --- a/backend/app/engine/review.py +++ b/backend/app/engine/review.py @@ -153,10 +153,25 @@ WARN_CODES: dict[str, tuple[str, str]] = { REFUSED_CODE: (REFUSED_CODE, DEGRADED), # 口径卡上用到两次查询的指标没做成可比性检查（探查失败、算不了）：指标照常算，两期能不能直接比要人自己核对 "metric_comparability_skipped": ("metric_comparability_skipped", DEGRADED), + # 指标本体与跨源穿透异常信号（P1）：默认只说明、不进 RETRYABLE，由 anomaly_escalation 决定提示按钮或自动升级最多 1 轮 + "metric_contract_violated": ("metric_contract_violated", DEGRADED), + "ontology_reconcile_alert": ("ontology_reconcile_alert", DEGRADED), + "source_health_anomalous": ("source_health_anomalous", DEGRADED), } #: 分析调用写错了、后来同一个分析工具一次都没做成（tool.error 带 call_error，见 scan）：这几次分析没做成，同 analysis_refused #: 是缺口不是断裂。它不是日志代码，是 scan 从工具事件里认出来的信号种类 ANALYSIS_CALL_FAILED = "analysis_call_failed" +METRIC_CONTRACT_VIOLATED = "metric_contract_violated" +ONTOLOGY_RECONCILE_ALERT = "ontology_reconcile_alert" +SOURCE_HEALTH_ANOMALOUS = "source_health_anomalous" +#: 可触发「沿本体排查关联数据源」诊断升级提示（或 auto 模式下最多 1 轮自动升级）的硬异常信号（P1） +ESCALATION_SIGNALS = frozenset({ + METRIC_CONTRACT_VIOLATED, + "query_no_data", + "metric_no_data", + ONTOLOGY_RECONCILE_ALERT, + SOURCE_HEALTH_ANOMALOUS, +}) # 这些 kind 重跑一次大概率能好：原因说得清，且不是「问题本身没问明白」。 # 工具参数错了、步数不够、图没校验过，都属于改一下再来一遍就行的。 @@ -167,7 +182,19 @@ REBUILD = {"query_no_data", "metric_no_data"} #: 只写说明、不自动重来的信号（00-总方案 §3.7）：都是缺口（DEGRADED）、不进 RETRYABLE / REBUILD。一次运行的信号全是 #: 这几种时复核不叫模型，说明就照信号原话写（review）：答案照常、报告里已经标出，叫一次模型只是把同一句话换个说法 -NOTE_ONLY = frozenset({"investigation_incomplete", "report_analysis_claims", REFUSED_CODE, "metric_comparability_skipped", ANALYSIS_CALL_FAILED}) +NOTE_ONLY = frozenset({ + "investigation_incomplete", + "report_analysis_claims", + REFUSED_CODE, + "metric_comparability_skipped", + ANALYSIS_CALL_FAILED, +}) +#: 指标本体与数据源健康硬告警在默认 prompt 模式下同样只做说明、不触发自动复核模型改写 +ONTOLOGY_NOTE_ONLY = frozenset({ + METRIC_CONTRACT_VIOLATED, + ONTOLOGY_RECONCILE_ALERT, + SOURCE_HEALTH_ANOMALOUS, +}) #: 「有 n 次分析没做成」那条信号里最多点名几次，其余写「……」 _ANALYSIS_SHOWN = 3 @@ -221,6 +248,7 @@ def scan(events: Iterable[Any], output: Mapping[str, Any] | None = None) -> list # 不区分的话，参数自动纠正这条路每修一次就报一次「答案不可信」。 recovered: set[str] = set() failed_at: dict[str, int] = {} + used_ontology_neighbors = False #: 节点名：node.started 带着画布上的名字，报失败时写名字，不写节点 id titles: dict[str, str] = {} for i, e in enumerate(rows): @@ -229,6 +257,8 @@ def scan(events: Iterable[Any], output: Mapping[str, Any] | None = None) -> list if etype == EventType.NODE_STARTED and data.get("label"): titles.setdefault(str(_event_field(e, "node_id") or ""), str(data["label"])) tool = str(data.get("tool") or "") + if etype in (EventType.TOOL_START, EventType.TOOL_END) and tool == "ontology_neighbors": + used_ontology_neighbors = True if etype == EventType.TOOL_ERROR and tool: failed_at.setdefault(tool, i) elif etype == EventType.TOOL_END and tool and tool in failed_at: @@ -312,6 +342,19 @@ def scan(events: Iterable[Any], output: Mapping[str, Any] | None = None) -> list if message: add(kind, message[:300], severity, reason if isinstance(reason, str) and reason else None) + elif etype == EventType.TOOL_END: + tool = str(data.get("tool") or "") + art = data.get("analysis_artifact") + if isinstance(art, Mapping) and art.get("op") == "reconcile": + if art.get("verdict") in ("diverged", "missing_counterpart") and not used_ontology_neighbors: + phrase = str(art.get("phrase") or "").strip() or "跨源指标对账结果出现显著背离或缺对手方数据" + add(ONTOLOGY_RECONCILE_ALERT, phrase[:300], DEGRADED) + health_art = data.get("health_artifact") + if isinstance(health_art, Mapping) and health_art.get("verdict") == "anomalous": + if not (output and output.get("_evidence")): + phrase = str(health_art.get("phrase") or "").strip() or "数据源健康探查发现同步或码值异常" + add(SOURCE_HEALTH_ANOMALOUS, phrase[:300], DEGRADED) + # 先插靠后的那条（位置一样时先插后出现的），靠前那条的位置就不受影响 for kind in sorted(slots, key=lambda k: slots[k], reverse=True): items = refused if kind == REFUSED_CODE else unfixed @@ -599,8 +642,8 @@ async def review( fallback_note = "；".join(s.detail for s in signals if s.severity == BROKEN) or "；".join( s.detail for s in signals ) - if signals and all(s.kind in NOTE_ONLY for s in signals): - # 只写说明、不自动重来（NOTE_ONLY）：不叫模型，照信号原话说明 + if signals and all(s.kind in (NOTE_ONLY | ONTOLOGY_NOTE_ONLY) for s in signals): + # 只写说明、不自动重来（NOTE_ONLY / ONTOLOGY_NOTE_ONLY）：不叫模型，照信号原话说明 return ReviewResult(verdict="annotated", note=fallback_note, retry=False, signals=signals, severity=severity) messages = [("system", REVIEW_SYSTEM), diff --git a/backend/app/engine/schema.py b/backend/app/engine/schema.py index a20dbcdb..6613d1fa 100644 --- a/backend/app/engine/schema.py +++ b/backend/app/engine/schema.py @@ -757,7 +757,7 @@ def analysis_issues(spec: GraphSpec, node: GraphNode) -> list[ValidationIssue]: add(f"{who}的「在验证什么」需要是一段文字", f"{at}.hypothesis", "analysis.item_invalid") both = all(isinstance(params.get(k), str) and params[k].strip() for k in ANALYSIS_NODE_PARAMS) if not (isinstance(item.get("from"), str) and item["from"].strip()) and not both: - two = "；基期、本期是两份查询时，在参数里选上这两个查询节点" if op in ("decompose", "comparable") else "" + two = "；基期、本期是两份查询时，在参数里选上这两个查询节点" if op in ("decompose", "comparable", "reconcile", "cochange") else "" add(f"{who}还没有选择输入：选一个上游的查询节点（调用数据库查询工具的「调用工具」节点，或「合并查询」）{two}", f"{at}.from", "analysis.input_invalid") headline = params.get("headline") diff --git a/backend/app/eval/answer.py b/backend/app/eval/answer.py index c2816b7c..d623501f 100644 --- a/backend/app/eval/answer.py +++ b/backend/app/eval/answer.py @@ -74,6 +74,8 @@ class TurnContext: analysis: Mapping[str, Any] | None = None #: 变体：分析工具开不开（variants.analysis_tools）。关的那一臂建图时助手看不到分析工具（TurnDriver 的 analysis_tools） analysis_tools: bool = True + #: 变体：本体穿透开不开（variants.ontology_tools）。关的那一臂建图时助手看不到本体工具与邻居提示 + ontology_tools: bool = True async def writer_usage(run_ids: list[str]) -> dict[str, Any]: @@ -163,7 +165,7 @@ class Driver(TurnDriver): on_phase: Callable[[str], Awaitable[None]] | None = None) -> None: super().__init__(question, datasource_ids=datasource_ids, on_phase=on_phase, provider=ctx.assistant.get("provider"), model=ctx.assistant.get("model"), - analysis_tools=ctx.analysis_tools) + analysis_tools=ctx.analysis_tools, ontology_tools=ctx.ontology_tools) self.ctx = ctx def catalog_view(self) -> Any: diff --git a/backend/app/eval/cases.py b/backend/app/eval/cases.py index 4973ef25..28c9049c 100644 --- a/backend/app/eval/cases.py +++ b/backend/app/eval/cases.py @@ -64,6 +64,8 @@ GOLD_KINDS = ("sql", "caliber", "from_run") HUMAN_VALUE_KEYS = ("value", "values", "expected", "answer", "rows", "number", "segment", "segments", "subjects", "direction", "verdict", "judgement", "driver") +DIAGNOSIS_STAGES = ("health_and_contract", "slice_localization", "ontology_penetration") + #: 每种期望认的键（另有 id、kind、required、label） _EXPECTATION_KEYS: dict[str, tuple[str, ...]] = { "scalar": ("column", "metric", "row", "where", "tolerance"), @@ -74,7 +76,8 @@ _EXPECTATION_KEYS: dict[str, tuple[str, ...]] = { "text": ("points",), "contribution": ("op", "mode", "params", "top_n"), "verdict": ("op", "mode", "params"), - "investigation": ("minimum_depth", "max_queries", "status", "stop_reason"), + "investigation": ("minimum_depth", "max_queries", "status", "stop_reason", + "min_ontology_hops", "require_source_health", "diagnosis_stage"), } _COMMON_KEYS = ("id", "kind", "required", "label") #: 期望答案来源的键（客户端能写的）；from_run 只能由服务端建 @@ -292,12 +295,20 @@ def _expectation(raw: Any, i: int, *, gold_kind: str | None, taken: set[str]) -> if status not in ("complete", "partial"): raise CaseInvalid("调查期望 status 只能是 complete 或 partial", f"{path}.status") out["status"] = status - for key, low, high in (("minimum_depth", 0, 5), ("max_queries", 1, 1000)): - if key in raw: + for key, low, high in (("minimum_depth", 0, 5), ("max_queries", 1, 1000), ("min_ontology_hops", 0, 10)): + if key in raw and raw[key] is not None: value = raw[key] if isinstance(value, bool) or not isinstance(value, int) or not low <= value <= high: raise CaseInvalid(f"{key} 需要是 {low} 到 {high} 的整数", f"{path}.{key}") out[key] = value + if raw.get("require_source_health") is not None: + out["require_source_health"] = _bool(raw["require_source_health"], f"{where}的关联源健康检查要求", + f"{path}.require_source_health") + if raw.get("diagnosis_stage") is not None: + stage = raw["diagnosis_stage"] + if stage not in DIAGNOSIS_STAGES: + raise CaseInvalid(f"诊断阶段只能是 {'、'.join(DIAGNOSIS_STAGES)}", f"{path}.diagnosis_stage") + out["diagnosis_stage"] = stage if raw.get("stop_reason") is not None: reason = raw["stop_reason"] if not isinstance(reason, str) or not reason.strip() or len(reason) > 80: diff --git a/backend/app/eval/compare.py b/backend/app/eval/compare.py index 4347b3b4..847f9d2a 100644 --- a/backend/app/eval/compare.py +++ b/backend/app/eval/compare.py @@ -31,10 +31,10 @@ CAUSES = ("data", "method", "params", "path", "unknown") CAUSE_LABEL = {"data": "数据变", "method": "做法变", "params": "只参数变", "path": "路径变", "unknown": "说不清"} #: 指纹里比的几项和叫法 FINGERPRINT_LABEL = {"assistant_model": "助手模型", "writer_model": "写作模型", "catalog": "目录快照", - "analysis_tools": "分析工具", "prompt": "提示词", "schemas": "表结构", "cases": "题目", - "scorer_version": "评分规则", "analysis": "分析设置"} + "analysis_tools": "分析工具", "ontology_tools": "本体穿透", "prompt": "提示词", "schemas": "表结构", + "cases": "题目", "scorer_version": "评分规则", "analysis": "分析设置"} #: 指纹里照实写前后两边的几项（别的只说变没变） -_SHOWN = ("assistant_model", "writer_model", "catalog", "analysis_tools", "scorer_version") +_SHOWN = ("assistant_model", "writer_model", "catalog", "analysis_tools", "ontology_tools", "scorer_version") NO_DIFFERENCE = "这批题上看不出差别" SIGNIFICANT = 0.05 #: 基准能是哪几种状态 @@ -130,18 +130,20 @@ def transitions(base: Sequence[Mapping[str, Any]], target: Sequence[Mapping[str, return {"counts": counts, "causes": causes, "items": items} -def _tools(fingerprint: Mapping[str, Any]) -> str: - """分析工具这一项给人看的样子：开 / 关。加这一项之前的评测指纹里没有它，那时一直开着。""" +def _tools(fingerprint: Mapping[str, Any], key: str = "analysis_tools") -> str: + """分析工具、本体穿透这一项给人看的样子：开 / 关。加这一项之前的评测指纹里没有它，那时一直开着。""" from app.eval.variants import ANALYSIS_TOOLS_LABEL - return ANALYSIS_TOOLS_LABEL["off" if fingerprint.get("analysis_tools") == "off" else "on"] + return ANALYSIS_TOOLS_LABEL["off" if fingerprint.get(key) == "off" else "on"] def fingerprint_diff(base: Mapping[str, Any], target: Mapping[str, Any]) -> list[dict[str, Any]]: - """两次之间指纹里哪几项变了：[{key, label, before, after}]（表结构、题目只说变没变，不展开；分析工具写开、关）。""" + """两次之间指纹里哪几项变了：[{key, label, before, after}]（表结构、题目只说变没变，不展开；分析工具、本体穿透写开、关）。""" out = [] for key, label in FINGERPRINT_LABEL.items(): - a, b = (_tools(base), _tools(target)) if key == "analysis_tools" else (base.get(key), target.get(key)) + a, b = ((_tools(base, key), _tools(target, key)) + if key in ("analysis_tools", "ontology_tools") + else (base.get(key), target.get(key))) if a == b: continue item: dict[str, Any] = {"key": key, "label": label} diff --git a/backend/app/eval/judgements.py b/backend/app/eval/judgements.py index 73b79cd3..48fc8b34 100644 --- a/backend/app/eval/judgements.py +++ b/backend/app/eval/judgements.py @@ -43,11 +43,11 @@ from app.analysis.contract import OP_LABELS, method_label #: 两种分析期望 ANALYSIS_KINDS = ("contribution", "verdict") #: 判定期望能用的分析（贡献恒为变化拆解） -VERDICT_OPS = ("baseline", "trend", "differ", "comparable") -#: 判定类，见模块说明。前六个是贡献的，后九个是判定的 +VERDICT_OPS = ("baseline", "trend", "differ", "comparable", "reconcile", "cochange") +#: 判定类，见模块说明。前六个是贡献的，其余是判定的 JUDGEMENTS = ("source", "dispersed", "flat", "mix", "rate", "mixed", "normal", "anomaly", "rising", "falling", "no_trend", "differs", "no_evidence", "comparable", - "incomparable") + "incomparable", "aligned", "diverged", "co_anomalous", "target_normal") #: 分析的输入在算子里的名字（报错里照它说：「期望 SQL 里没有『地区』这一列」） GOLD_REF = "期望 SQL" #: 贡献期望最多点名前几段 @@ -66,6 +66,8 @@ _JUDGEMENT_OF: dict[tuple[str, str | None], dict[str, str]] = { ("trend", None): {"rising": "rising", "falling": "falling", "no_trend": "no_trend"}, ("differ", None): {"differs": "differs", "no_evidence": "no_evidence"}, ("comparable", None): {"comparable": "comparable", "conditional": "comparable", "incomparable": "incomparable"}, + ("reconcile", None): {"aligned": "aligned", "diverged": "diverged", "missing_counterpart": "diverged"}, + ("cochange", None): {"co_anomalous": "co_anomalous", "target_normal": "target_normal", "primary_normal": "target_normal"}, } #: 判定类要看方向的：主要来源、结构或自身为主、分散（总变化的涨跌）、超出常规波动（本期变化的涨跌）、差异站得住（A 相对 B） _DIRECTED = frozenset({"source", "mix", "rate", "mixed", "dispersed", "anomaly", "differs"}) @@ -79,6 +81,8 @@ _AFFIRMS: dict[str, frozenset[str]] = { "anomaly": frozenset({"anomaly"}), "rising": frozenset({"trend"}), "falling": frozenset({"trend"}), "no_trend": frozenset({"no_trend"}), "differs": frozenset({"differ"}), "no_evidence": frozenset({"no_diff"}), "comparable": frozenset(), "incomparable": frozenset(), + "aligned": frozenset(), "diverged": frozenset(), + "co_anomalous": frozenset({"anomaly"}), "target_normal": frozenset({"normal"}), } #: 判断词所在的句子最多留多少字给人看 _SNIPPET_MAX = 40 diff --git a/backend/app/eval/runner.py b/backend/app/eval/runner.py index 04380092..846377db 100644 --- a/backend/app/eval/runner.py +++ b/backend/app/eval/runner.py @@ -110,12 +110,14 @@ def prompt_fingerprint() -> str: from app.engine import analysis_claims, evidence, review from app.engine.investigation import INVESTIGATION_VERSION, InvestigationPolicy from app.tools.builtin.analysis import METHOD_GUIDE + from app.tools.ontology import ONTOLOGY_GUIDE parts = [copilot.NODE_REFERENCE, copilot._STREAM_PROTOCOL, copilot._user_message(copilot.GenerateIn(instruction="{question}", intent="answer"), patch=False), turn.RETRY_WITH_BASE, turn.RETRY_FROM_SCRATCH, str(getattr(review, "REVIEW_SYSTEM", "") or getattr(review, "SYSTEM_PROMPT", "")), - METHOD_GUIDE.id, METHOD_GUIDE.text, evidence.ANALYSIS_PROMPT, evidence.ANALYSIS_SEE_ONLY_PROMPT, + METHOD_GUIDE.id, METHOD_GUIDE.text, ONTOLOGY_GUIDE.id, ONTOLOGY_GUIDE.text, + evidence.ANALYSIS_PROMPT, evidence.ANALYSIS_SEE_ONLY_PROMPT, analysis_claims.ANALYSIS_RULES_PROMPT, analysis_claims.ANALYSIS_NONE_PROMPT, analysis_claims.ANALYSIS_REPAIR_PROMPT, analysis_claims.LEXICON_VERSION, INTENT_VERSION, analysis_gaps_fix(), INVESTIGATION_VERSION, InvestigationPolicy().model_dump()] @@ -245,6 +247,7 @@ async def start(session: AsyncSession, suite: EvalSuite, variants: Sequence[Mapp "assistant_model": models.get("assistant"), "writer_model": models.get("writer"), "catalog_mode": (variant.get("catalog") or {}).get("mode"), "analysis_tools": variant_rules.analysis_tools(variant), + "ontology_tools": variant_rules.ontology_tools(variant), "snapshot_artifact": snapshot, "cron": cron}, counts={"total": len(cases) * repeats, "done": 0}) session.add(run) @@ -554,7 +557,8 @@ class EvalExecution(runtime.JobRun): arm=arm.name, repeat=unit.repeat, assistant=arm.variant.get("assistant") or {}, writer=arm.variant.get("writer") or {}, view=arm.view, analysis=self.analysis, - analysis_tools=variant_rules.analysis_tools(arm.variant) == "on") + analysis_tools=variant_rules.analysis_tools(arm.variant) == "on", + ontology_tools=variant_rules.ontology_tools(arm.variant) == "on") driver = Driver(question, datasource_ids=list(case.get("datasource_ids") or []), ctx=ctx, on_phase=on_phase) try: if case.get("kind") == "workflow": diff --git a/backend/app/eval/score.py b/backend/app/eval/score.py index 894c31a8..bb59dc85 100644 --- a/backend/app/eval/score.py +++ b/backend/app/eval/score.py @@ -187,12 +187,14 @@ class Answer: analyses: list[dict[str, Any]] = field(default_factory=list) sentences: list[dict[str, Any]] = field(default_factory=list) investigations: list[dict[str, Any]] = field(default_factory=list) + source_healths: list[dict[str, Any]] = field(default_factory=list) def to_dict(self) -> dict[str, Any]: return {"run_id": self.run_id, "text": self.text, "values": self.values, "bare": self.bare, "queries": self.queries, "cited": self.cited, "stats": self.stats, "tier": self.tier, "report": self.report, "problem": self.problem, "analyses": self.analyses, - "sentences": self.sentences, "investigations": self.investigations} + "sentences": self.sentences, "investigations": self.investigations, + "source_healths": self.source_healths} @classmethod def from_dict(cls, raw: Mapping[str, Any]) -> Answer: @@ -200,7 +202,8 @@ class Answer: bare=list(raw.get("bare") or []), queries=list(raw.get("queries") or []), cited=list(raw.get("cited") or []), stats=dict(raw.get("stats") or {}), tier=raw.get("tier"), report=raw.get("report"), problem=raw.get("problem"), analyses=list(raw.get("analyses") or []), - sentences=list(raw.get("sentences") or []), investigations=list(raw.get("investigations") or [])) + sentences=list(raw.get("sentences") or []), investigations=list(raw.get("investigations") or []), + source_healths=list(raw.get("source_healths") or [])) def _output_text(output: Any) -> str: @@ -308,6 +311,21 @@ async def read_answer(run_id: str | None, *, reply: str | None = None) -> Answer "row_count": snap.get("row_count", len(rows)), "truncated": bool(snap.get("truncated")), "role": snap.get("role")}) found = _ledger_analyses(sealed, cache) + for artifact, info in (getattr(sealed, "source_healths", None) or {}).items(): + snap = _snapshot(artifact, cache) + if snap is None: + continue + answer.source_healths.append({ + "artifact": artifact, + "node_id": (info or {}).get("node_id"), + "verdict": snap.get("verdict") or (info or {}).get("verdict"), + "sources": list((info or {}).get("sources") or [ + s.get("source") or s.get("source_label") + for s in (snap.get("sources") or []) + if isinstance(s, Mapping) + ]), + "phrase": snap.get("phrase") or "", + }) from app.db.base import SessionLocal from app.db.models import Run @@ -636,6 +654,17 @@ def _expectation_scores(expectations: Sequence[Mapping[str, Any]], values: Mappi problems.append("未达到要求的下钻深度") if exp.get("stop_reason") and finding.get("stop_reason") != exp["stop_reason"]: problems.append("停止原因不符合期望") + if exp.get("min_ontology_hops") is not None: + best_hops = max((int(f.get("ontology_hops") or 0) for f in findings), default=0) + if best_hops < int(exp["min_ontology_hops"]): + problems.append("未达到要求的本体穿透跳数") + if exp.get("require_source_health"): + checked = any(bool(f.get("source_health_checked")) for f in findings) or bool(answer.source_healths) + if not checked: + problems.append("未执行关联数据源健康度检查") + if exp.get("diagnosis_stage") is not None: + if not findings or not any(f.get("diagnosis_stage") == exp["diagnosis_stage"] for f in findings): + problems.append("诊断阶段不符合期望") if exp.get("max_queries") is not None and sum(int(f.get("query_count") or 0) for f in findings) > exp["max_queries"]: problems.append("查询次数超出预算") out.append({**base, "outcome": "wrong" if problems else "traced", diff --git a/backend/app/eval/variants.py b/backend/app/eval/variants.py index 3ca58f93..ff1f830b 100644 --- a/backend/app/eval/variants.py +++ b/backend/app/eval/variants.py @@ -45,6 +45,9 @@ CATALOG_LABEL = {"live": "当前目录", "reviewed": "只用已核实和已确 #: 分析工具开、关（变体的 analysis.tools） ANALYSIS_TOOLS = ("on", "off") ANALYSIS_TOOLS_LABEL = {"on": "开", "off": "关"} +#: 本体穿透开、关（变体的 ontology.tools） +ONTOLOGY_TOOLS = ("on", "off") +ONTOLOGY_TOOLS_LABEL = {"on": "开", "off": "关"} REPEATS_MAX = 5 #: 定时跑 1 次，A/B 对照缺省 3 次（评测-D10） SCHEDULE_REPEATS = 1 @@ -52,7 +55,7 @@ PAIR_REPEATS = 3 #: 一次 A/B 最多几臂（目录贡献对照最多三臂） ARMS_MAX = 3 LABEL_MAX = 40 -_KEYS = ("assistant", "writer", "judge", "catalog", "analysis", "repeats", "label") +_KEYS = ("assistant", "writer", "judge", "catalog", "analysis", "ontology", "repeats", "label") def _invalid(message: str, field_name: str) -> CaseInvalid: @@ -104,6 +107,10 @@ def normalize_variant(raw: Any, *, judge_default: bool = False, repeats_default: if not isinstance(analysis, Mapping) or set(analysis) - {"tools"} \ or analysis.get("tools", "on") not in ANALYSIS_TOOLS: raise _invalid(f"变体的分析工具需要是 {{tools}}，tools 只能是 {'、'.join(ANALYSIS_TOOLS)}（开、关）", "variant.analysis") + ontology = {} if raw.get("ontology") is None else raw["ontology"] + if not isinstance(ontology, Mapping) or set(ontology) - {"tools"} \ + or ontology.get("tools", "on") not in ONTOLOGY_TOOLS: + raise _invalid(f"变体的本体穿透需要是 {{tools}}，tools 只能是 {'、'.join(ONTOLOGY_TOOLS)}（开、关）", "variant.ontology") repeats = raw.get("repeats", repeats_default) if isinstance(repeats, bool) or not isinstance(repeats, int) or not 1 <= repeats <= REPEATS_MAX: raise _invalid(f"每题重复的次数需要是 1 到 {REPEATS_MAX} 的整数", "variant.repeats") @@ -113,7 +120,8 @@ def normalize_variant(raw: Any, *, judge_default: bool = False, repeats_default: return {"assistant": _model_pair(raw.get("assistant"), "助手模型", "variant.assistant"), "writer": _model_pair(raw.get("writer"), "写作模型", "variant.writer"), "judge": judge, "catalog": {"mode": catalog["mode"], "snapshot": snapshot}, - "analysis": {"tools": analysis.get("tools", "on")}, "repeats": repeats, + "analysis": {"tools": analysis.get("tools", "on")}, + "ontology": {"tools": ontology.get("tools", "on")}, "repeats": repeats, "label": (label or "").strip()[:LABEL_MAX] or None} @@ -122,6 +130,11 @@ def analysis_tools(variant: Mapping[str, Any]) -> str: return "off" if (variant.get("analysis") or {}).get("tools") == "off" else "on" +def ontology_tools(variant: Mapping[str, Any]) -> str: + """这一臂的本体穿透开不开：on / off。加这一项之前的变体（存着的老评测）没有它，按开。""" + return "off" if (variant.get("ontology") or {}).get("tools") == "off" else "on" + + def catalog_preset(base: Mapping[str, Any], *, reviewed: bool = False) -> list[dict[str, Any]]: """「目录的贡献」预设对照：当前目录 vs 关（reviewed 时再加「只用已核实和已确认的项」）。模型、裁判、重复次数照 base。""" arms = [("live", CATALOG_LABEL["live"]), ("off", CATALOG_LABEL["off"])] @@ -137,6 +150,12 @@ def analysis_tools_preset(base: Mapping[str, Any]) -> list[dict[str, Any]]: for tools in ANALYSIS_TOOLS] +def ontology_tools_preset(base: Mapping[str, Any]) -> list[dict[str, Any]]: + """「本体穿透的贡献」预设对照：本体穿透开 vs 关。模型、目录、分析工具、裁判、重复次数照 base。""" + return [{**copy.deepcopy(dict(base)), "ontology": {"tools": tools}, "label": f"本体穿透{ONTOLOGY_TOOLS_LABEL[tools]}"} + for tools in ONTOLOGY_TOOLS] + + def arm_name(variant: Mapping[str, Any], index: int) -> str: """这一臂的名字：变体里写了就用，否则按目录（只有目录不同时一看就懂），再不然 A、B、C。""" return str(variant.get("label") or "") or "ABC"[index] @@ -221,6 +240,8 @@ def variant_key(variant: Mapping[str, Any], models: Mapping[str, Any]) -> str: "snapshot": catalog.get("snapshot") if catalog.get("mode") == "snapshot" else None} if analysis_tools(variant) == "off": body["analysis_tools"] = "off" + if ontology_tools(variant) == "off": + body["ontology_tools"] = "off" return content_hash(canonical_json(body))[:16] @@ -231,12 +252,15 @@ def describe(variant: Mapping[str, Any], models: Mapping[str, Any] | None = None parts = [f"助手 {models.get('assistant') or (variant.get('assistant') or {}).get('model') or '缺省'}", f"写作 {models.get('writer') or (variant.get('writer') or {}).get('model') or '各节点自己的'}", CATALOG_LABEL.get(catalog.get("mode") or "live", "当前目录"), - f"分析工具{ANALYSIS_TOOLS_LABEL[analysis_tools(variant)]}", - f"结论裁判{'开' if variant.get('judge') else '关'}", f"每题 {variant.get('repeats') or 1} 次"] + f"分析工具{ANALYSIS_TOOLS_LABEL[analysis_tools(variant)]}"] + if ontology_tools(variant) == "off": + parts.append(f"本体穿透{ONTOLOGY_TOOLS_LABEL['off']}") + parts += [f"结论裁判{'开' if variant.get('judge') else '关'}", f"每题 {variant.get('repeats') or 1} 次"] return " · ".join(parts) -__all__ = ["ANALYSIS_TOOLS", "ANALYSIS_TOOLS_LABEL", "ARMS_MAX", "CATALOG_LABEL", "CATALOG_MODES", "PAIR_REPEATS", +__all__ = ["ANALYSIS_TOOLS", "ANALYSIS_TOOLS_LABEL", "ARMS_MAX", "CATALOG_LABEL", "CATALOG_MODES", + "ONTOLOGY_TOOLS", "ONTOLOGY_TOOLS_LABEL", "PAIR_REPEATS", "REPEATS_MAX", "Resolved", "SCHEDULE_REPEATS", "VARIANT_INVALID", "VARIANT_UNAVAILABLE", "analysis_tools", - "analysis_tools_preset", "arm_name", "catalog_preset", "describe", "normalize_variant", "resolve_models", - "snapshot_artifact", "variant_key", "view_of"] + "analysis_tools_preset", "arm_name", "catalog_preset", "describe", "normalize_variant", "ontology_tools", + "ontology_tools_preset", "resolve_models", "snapshot_artifact", "variant_key", "view_of"] diff --git a/backend/app/ledger/ontology.py b/backend/app/ledger/ontology.py new file mode 100644 index 00000000..33caa678 --- /dev/null +++ b/backend/app/ledger/ontology.py @@ -0,0 +1,707 @@ +"""指标本体模型：跨口径语义关系（links）、DAG 无环门禁、自动反向数据血缘（metric -> source/table）。 + +设计原则（对应计划书 §2 与铁律 1-3）： +1. **三类显式指标关系**（挂在口径卡指标定义的 `links` 数组，参与 definition_hash）： + - `factor`：构成因子 / 公式拆解（有向无环 DAG，发布门禁强拒任何自环或跨口径成环）； + - `reconciles_with`：跨源镜像对账关系（可声明 abs_tol / rel_tol 对账容差，双向可对称）； + - `upstream_driver`：跨源上游业务/物理驱动关系（带 entity_key 对齐实体与可选 lag_periods 滞后期）。 +2. **建议态与已确认态分离**（P2）： + - `status` 取值 `proposed | verified | confirmed | rejected`； + - 文档起草（docdraft）或 AI 推荐产出的候选一律强制 `proposed`，**运行时自动穿透只遍历 `verified | confirmed`**。 +3. **自动反向血缘**： + - 不靠人工维护“指标用了哪张表”，直接复用 `ledger/closure.py` 依赖闭包与 `engine/evidence.sql_tables` + 静态提取 `(source_name, table_name)`，绝不把 SQL 正文或遮罩值写进本体索引或发布清单。 +""" +from __future__ import annotations + +import math +import re +from collections.abc import Iterable, Mapping +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.artifact_store import canonical_json, content_hash +from app.engine.schema import GraphSpec, NodeType + +LINK_KINDS = ("factor", "reconciles_with", "upstream_driver") +FACTOR_ROLES = ("numerator", "denominator", "multiplier", "addend", "subtrahend", "component") +LINK_STATUSES = ("proposed", "verified", "confirmed", "rejected") +ACTIVE_LINK_STATUSES = ("verified", "confirmed") +LINK_SOURCES = ("human", "rule", "docdraft", "ai") + +ONTOLOGY_CYCLE_CODE = "ontology.factor_cycle" +ONTOLOGY_LINK_INVALID_CODE = "ontology.link_invalid" + +_NODE_REF_RE = re.compile(r"\bnodes\.([A-Za-z_][\w-]*)|\bnodes\[['\"]([A-Za-z_][\w-]*)['\"]\]") +_SLUG_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") + + +class OntologyInvalid(ValueError): + """指标本体关系声明格式错误。""" + + def __init__(self, message: str, *, field: str = "", code: str = ONTOLOGY_LINK_INVALID_CODE) -> None: + super().__init__(message) + self.message = message + self.field = field + self.code = code + + +class OntologyCycleError(OntologyInvalid): + """指标 factor 构成边存在环。""" + + def __init__(self, cycle: list[str], *, field: str = "links") -> None: + rendered = " -> ".join(cycle) + super().__init__(f"指标构成因子（factor）不能形成循环依赖：{rendered}", field=field, code=ONTOLOGY_CYCLE_CODE) + self.cycle = cycle + + +def metric_node_key(workflow_id: str, card_id: str, metric_id: str) -> str: + return f"{workflow_id or 'self'}:{card_id}:{metric_id}" + + +def normalize_metric_target( + raw: Any, + *, + field: str = "target", + default_workflow_id: str = "", + default_card_id: str = "", +) -> dict[str, Any]: + if not isinstance(raw, Mapping): + raise OntologyInvalid("关联目标（target）需要是包含 workflow_id、card_id、metric_id 的对象", field=field) + workflow_id = str(raw.get("workflow_id") or default_workflow_id or "").strip() + card_id = str(raw.get("card_id") or default_card_id or "").strip() + metric_id = str(raw.get("metric_id") or "").strip() + if not metric_id: + raise OntologyInvalid("关联目标缺少指标标识（metric_id）", field=f"{field}.metric_id") + if not card_id: + raise OntologyInvalid("关联目标缺少口径卡节点标识（card_id）", field=f"{field}.card_id") + out: dict[str, Any] = { + "workflow_id": workflow_id, + "card_id": card_id, + "metric_id": metric_id, + } + version = raw.get("version") + if version not in (None, ""): + if isinstance(version, bool) or not isinstance(version, int) or version < 1: + raise OntologyInvalid("关联目标的固定版本（version）须为正整数", field=f"{field}.version") + out["version"] = version + return out + + +def normalize_link( + raw: Any, + *, + index: int = 0, + default_workflow_id: str = "", + default_card_id: str = "", + force_proposed: bool = False, + default_source: str = "human", +) -> dict[str, Any]: + at = f"links[{index}]" + if not isinstance(raw, Mapping): + raise OntologyInvalid("指标关联项需要是对象", field=at) + kind = str(raw.get("kind") or "").strip() + if kind not in LINK_KINDS: + raise OntologyInvalid(f"未知的指标关系类型「{kind}」，可选：{', '.join(LINK_KINDS)}", field=f"{at}.kind") + target = normalize_metric_target( + raw.get("target"), + field=f"{at}.target", + default_workflow_id=default_workflow_id, + default_card_id=default_card_id, + ) + source = str(raw.get("source") or default_source).strip() + if source not in LINK_SOURCES: + raise OntologyInvalid(f"关系来源「{source}」无效，可选：{', '.join(LINK_SOURCES)}", field=f"{at}.source") + status = "proposed" if (force_proposed or source in ("docdraft", "ai") and "status" not in raw) else str( + raw.get("status") or "confirmed" + ).strip() + if force_proposed: + status = "proposed" + if status not in LINK_STATUSES: + raise OntologyInvalid(f"关系状态「{status}」无效，可选：{', '.join(LINK_STATUSES)}", field=f"{at}.status") + + out: dict[str, Any] = { + "kind": kind, + "target": target, + "status": status, + "source": source, + } + if raw.get("role") not in (None, ""): + role = str(raw.get("role") or "").strip() + if role not in FACTOR_ROLES: + raise OntologyInvalid(f"因子角色「{role}」无效，可选：{', '.join(FACTOR_ROLES)}", field=f"{at}.role") + out["role"] = role + if raw.get("entity_key") not in (None, ""): + entity_key = str(raw.get("entity_key") or "").strip() + if not _SLUG_RE.match(entity_key): + raise OntologyInvalid("对齐实体键（entity_key）格式不合法", field=f"{at}.entity_key") + out["entity_key"] = entity_key + if raw.get("lag_periods") not in (None, ""): + lag = raw.get("lag_periods") + if isinstance(lag, bool) or not isinstance(lag, int) or lag < 0 or lag > 12: + raise OntologyInvalid("滞后期数（lag_periods）须为 0 到 12 之间的整数", field=f"{at}.lag_periods") + if lag > 0: + out["lag_periods"] = lag + for tol_key in ("abs_tol", "rel_tol"): + if raw.get(tol_key) not in (None, ""): + val = raw.get(tol_key) + if isinstance(val, bool) or not isinstance(val, (int, float)) or not math.isfinite(float(val)) or float(val) < 0: + raise OntologyInvalid(f"对账容差「{tol_key}」须为非负有限数字", field=f"{at}.{tol_key}") + out[tol_key] = float(val) + rationale = str(raw.get("rationale") or "").strip() + if rationale: + out["rationale"] = rationale + citations = raw.get("citations") + if isinstance(citations, list) and citations: + out["citations"] = [dict(c) for c in citations if isinstance(c, Mapping)] + # 稳定内容 id，便于审阅与去重 + link_id = str(raw.get("id") or "").strip() + if not link_id: + link_id = content_hash(canonical_json({"kind": kind, "target": target, "role": out.get("role")}))[:12] + out["id"] = link_id + return out + + +def normalize_links( + raw_links: Any, + *, + default_workflow_id: str = "", + default_card_id: str = "", + force_proposed: bool = False, + default_source: str = "human", +) -> list[dict[str, Any]]: + if raw_links in (None, ""): + return [] + if not isinstance(raw_links, list): + raise OntologyInvalid("指标关联列表（links）需要是数组", field="links") + seen: set[tuple[str, str, str, str, str]] = set() + out: list[dict[str, Any]] = [] + for idx, raw in enumerate(raw_links): + norm = normalize_link( + raw, + index=idx, + default_workflow_id=default_workflow_id, + default_card_id=default_card_id, + force_proposed=force_proposed, + default_source=default_source, + ) + t = norm["target"] + dedup_key = (norm["kind"], t["workflow_id"], t["card_id"], t["metric_id"], str(norm.get("role") or "")) + if dedup_key in seen: + continue + seen.add(dedup_key) + out.append(norm) + return out + + +def active_links(metric: Mapping[str, Any], *, kinds: Iterable[str] | None = None) -> list[dict[str, Any]]: + """取出单个指标定义上已生效（verified / confirmed）的关系边；proposed / rejected 不参与自动穿透。""" + raw = metric.get("links") if isinstance(metric, Mapping) else None + if not isinstance(raw, list) or not raw: + return [] + allowed_kinds = set(kinds) if kinds is not None else None + out: list[dict[str, Any]] = [] + for idx, item in enumerate(raw): + try: + norm = normalize_link(item, index=idx) + except OntologyInvalid: + continue + if norm["status"] not in ACTIVE_LINK_STATUSES: + continue + if allowed_kinds is not None and norm["kind"] not in allowed_kinds: + continue + out.append(norm) + return out + + +# -------------------------------------------------------------------------- +# DAG 无环检测（factor 构成边） +# -------------------------------------------------------------------------- + + +def find_factor_cycles(edges: Iterable[tuple[str, str]]) -> list[list[str]]: + """在 factor 有向边集合 [(u, v)] 中找出代表性环路径（如 ['a', 'b', 'a']）；无环返回 []。""" + adj: dict[str, list[str]] = {} + nodes: set[str] = set() + for src, dst in edges: + if not src or not dst: + continue + nodes.add(src) + nodes.add(dst) + adj.setdefault(src, []).append(dst) + for k in adj: + adj[k] = sorted(set(adj[k])) + + WHITE, GREY, BLACK = 0, 1, 2 + state: dict[str, int] = dict.fromkeys(sorted(nodes), WHITE) + stack: list[str] = [] + pos_in_stack: dict[str, int] = {} + cycles: list[list[str]] = [] + seen_signatures: set[tuple[str, ...]] = set() + + def _record_cycle(target: str) -> None: + start = pos_in_stack[target] + loop = [*stack[start:], target] + ring = loop[:-1] + min_idx = min(range(len(ring)), key=lambda i: ring[i]) + canonical = tuple(ring[min_idx:] + ring[:min_idx]) + if canonical not in seen_signatures: + seen_signatures.add(canonical) + cycles.append(loop) + + for root in sorted(nodes): + if state[root] != WHITE: + continue + state[root] = GREY + pos_in_stack[root] = 0 + stack = [root] + frames: list[tuple[str, int]] = [(root, 0)] + while frames: + curr, idx = frames[-1] + neighbors = adj.get(curr, ()) + if idx >= len(neighbors): + state[curr] = BLACK + pos_in_stack.pop(curr, None) + stack.pop() + frames.pop() + continue + frames[-1] = (curr, idx + 1) + nxt = neighbors[idx] + if state.get(nxt, WHITE) == GREY: + _record_cycle(nxt) + elif state.get(nxt, WHITE) == WHITE: + state[nxt] = GREY + pos_in_stack[nxt] = len(stack) + stack.append(nxt) + frames.append((nxt, 0)) + return cycles + + +def check_factor_dag(edges: Iterable[tuple[str, str]]) -> None: + cycles = find_factor_cycles(edges) + if cycles: + raise OntologyCycleError(cycles[0]) + + +# -------------------------------------------------------------------------- +# 自动反向血缘：Closure + sql_tables -> metric -> [(source, tables)] +# -------------------------------------------------------------------------- + + +def _referenced_nodes(expression: str) -> list[str]: + if not isinstance(expression, str) or not expression: + return [] + out: list[str] = [] + for m in _NODE_REF_RE.finditer(expression): + nid = m.group(1) or m.group(2) + if nid and nid not in out: + out.append(nid) + return out + + +def card_metric_lineage(graph: Mapping[str, Any], card_id: str) -> dict[str, Any]: + """从工作流图的依赖闭包静态推导一张源头口径卡各指标的数据源与物理表血缘。 + + 不存 SQL 原文、不读任何参数字面量或遮罩值，只记 (node_id, source_name, tables)。 + """ + from app.apps.tree import node_map + from app.engine.evidence import sql_tables + from app.ledger import closure as closure_mod + from app.tools.datasource import source_of_tool + + nmap = node_map(graph or {}) + card = nmap.get(card_id) or {} + cfg = card.get("config") if isinstance(card.get("config"), Mapping) else {} + metrics = [m for m in (cfg.get("metrics") or []) if isinstance(m, Mapping) and str(m.get("id") or "").strip()] + + try: + built = closure_mod.build(dict(graph or {}), "", 1, card_id) + closure_nodes = {q.node_id for q in built.queries} | set(built.excluded_query_ids) + if not closure_nodes: + closure_nodes = set(nmap.keys()) + except Exception: # noqa: BLE001 - 图不完整时退回整张图里的上游工具节点 + closure_nodes = set(nmap.keys()) + + query_lineage: dict[str, dict[str, Any]] = {} + for nid in sorted(closure_nodes): + node = nmap.get(nid) + if not isinstance(node, Mapping) or node.get("type") != "tool": + continue + ncfg = node.get("config") if isinstance(node.get("config"), Mapping) else {} + tool_name = str(ncfg.get("tool") or "").strip() + src = source_of_tool(tool_name) + if not src or "{{" in src: + continue + args = ncfg.get("args") if isinstance(ncfg.get("args"), Mapping) else {} + sql = args.get("sql") if isinstance(args.get("sql"), str) else "" + tables = sorted({t.rsplit(".", 1)[-1] for t in sql_tables(sql) if t}) if sql.strip() else [] + query_lineage[nid] = {"node_id": nid, "source": src, "tables": tables} + + tables_by_source: dict[str, set[str]] = {} + for q in query_lineage.values(): + tables_by_source.setdefault(q["source"], set()).update(q["tables"]) + + metric_map: dict[str, list[dict[str, Any]]] = {} + all_queries = list(query_lineage.values()) + for metric in metrics: + mid = str(metric["id"]).strip() + refs = _referenced_nodes(str(metric.get("expression") or "")) + direct = [query_lineage[r] for r in refs if r in query_lineage] + metric_map[mid] = direct if direct else all_queries + + return { + "card_id": card_id, + "sources": sorted(tables_by_source.keys()), + "tables_by_source": {src: sorted(tbls) for src, tbls in sorted(tables_by_source.items())}, + "metrics": metric_map, + } + + +def workflow_ontology_index( + *, + workflow_id: str, + version: int, + graph: Mapping[str, Any], + workflow_name: str = "", +) -> dict[str, Any]: + """提取一份工作流版本上全部源头口径卡的指标节点、本体关系边与自动反向表血缘。""" + from app.apps.publications import source_cards + from app.apps.tree import node_map + + nmap = node_map(graph or {}) + cards_out: list[dict[str, Any]] = [] + edges_out: list[dict[str, Any]] = [] + for card_meta in source_cards(graph): + cid = str(card_meta["id"]) + lineage = card_metric_lineage(graph, cid) + node = nmap.get(cid) or {} + cfg = node.get("config") if isinstance(node.get("config"), Mapping) else {} + metrics_out: list[dict[str, Any]] = [] + for raw_m in cfg.get("metrics") or []: + if not isinstance(raw_m, Mapping) or not str(raw_m.get("id") or "").strip(): + continue + mid = str(raw_m["id"]).strip() + mname = str(raw_m.get("name") or mid).strip() + raw_links = raw_m.get("links") + links: list[dict[str, Any]] = [] + if isinstance(raw_links, list) and raw_links: + for idx, item in enumerate(raw_links): + try: + norm = normalize_link(item, index=idx, default_workflow_id=workflow_id, default_card_id=cid) + except OntologyInvalid: + continue + links.append(norm) + target = norm["target"] + edges_out.append({ + "id": norm["id"], + "kind": norm["kind"], + "status": norm["status"], + "source": norm["source"], + "role": norm.get("role"), + "entity_key": norm.get("entity_key"), + "lag_periods": norm.get("lag_periods", 0), + "abs_tol": norm.get("abs_tol"), + "rel_tol": norm.get("rel_tol"), + "rationale": norm.get("rationale", ""), + "from": {"workflow_id": workflow_id, "version": version, "card_id": cid, "metric_id": mid}, + "to": { + "workflow_id": target["workflow_id"] or workflow_id, + "version": target.get("version"), + "card_id": target["card_id"], + "metric_id": target["metric_id"], + }, + }) + metrics_out.append({ + "metric_id": mid, + "name": mname, + "unit": str(raw_m.get("unit") or ""), + "contract": dict(raw_m["contract"]) if isinstance(raw_m.get("contract"), Mapping) else None, + "links": links, + "lineage": lineage["metrics"].get(mid, []), + }) + cards_out.append({ + "workflow_id": workflow_id, + "workflow_name": workflow_name, + "version": version, + "card_id": cid, + "title": card_meta["title"], + "caliber": card_meta["caliber"], + "sources": lineage["sources"], + "tables_by_source": lineage["tables_by_source"], + "metrics": metrics_out, + }) + return {"workflow_id": workflow_id, "version": version, "cards": cards_out, "edges": edges_out} + + +def ontology_manifest_section(*, workflow_id: str, version: int, graph: Mapping[str, Any]) -> dict[str, Any] | None: + """供 publications.build_content 物化进发布清单：仅当图上声明了指标本体 links 时写入，保证存量清单哈希一字不变。""" + index = workflow_ontology_index(workflow_id=workflow_id, version=version, graph=graph) + if not index["edges"]: + return None + lineage = { + card["card_id"]: { + "sources": card["sources"], + "tables_by_source": card["tables_by_source"], + "metrics": { + m["metric_id"]: [ + {"source": item["source"], "tables": item["tables"]} + for item in m["lineage"] + ] + for m in card["metrics"] + }, + } + for card in index["cards"] + } + return {"edges": index["edges"], "lineage": lineage} + + +# -------------------------------------------------------------------------- +# 发布预检与全站本体图构建 +# -------------------------------------------------------------------------- + + +def _graph_factor_edges(graph: Mapping[str, Any], *, workflow_id: str) -> tuple[list[tuple[str, str]], list[dict[str, Any]]]: + """返回 (factor 有向边列表, 语法校验问题列表)。只对非 rejected 的 factor 边判环（含 proposed，早发现早拦）。""" + spec_nodes = graph.get("nodes") if isinstance(graph, Mapping) else None + edges: list[tuple[str, str]] = [] + issues: list[dict[str, Any]] = [] + if not isinstance(spec_nodes, list): + return edges, issues + wf_key = workflow_id or "self" + for node in spec_nodes: + if not isinstance(node, Mapping) or node.get("type") != NodeType.METRICS: + continue + nid = str(node.get("id") or "") + cfg = node.get("data", {}).get("config") if isinstance(node.get("data"), Mapping) else node.get("config") + if not isinstance(cfg, Mapping) or cfg.get("caliber_from") not in (None, "", {}): + continue + metrics = cfg.get("metrics") + if not isinstance(metrics, list): + continue + for m_idx, metric in enumerate(metrics): + if not isinstance(metric, Mapping): + continue + mid = str(metric.get("id") or "").strip() + raw_links = metric.get("links") + if raw_links in (None, "", []): + continue + if not isinstance(raw_links, list): + issues.append({ + "level": "error", + "node_id": nid, + "field": f"metrics[{m_idx}].links", + "code": ONTOLOGY_LINK_INVALID_CODE, + "message": f"指标「{mid or m_idx + 1}」的本体关系（links）需要是数组", + }) + continue + for l_idx, raw_link in enumerate(raw_links): + try: + norm = normalize_link(raw_link, index=l_idx, default_workflow_id=wf_key, default_card_id=nid) + except OntologyInvalid as exc: + issues.append({ + "level": "error", + "node_id": nid, + "field": f"metrics[{m_idx}].{exc.field}", + "code": exc.code, + "message": f"指标「{mid or m_idx + 1}」：{exc.message}", + }) + continue + if norm["kind"] == "factor" and norm["status"] != "rejected": + target = norm["target"] + dst_wf = target["workflow_id"] or wf_key + src_key = metric_node_key(wf_key, nid, mid) + dst_key = metric_node_key(dst_wf, target["card_id"], target["metric_id"]) + edges.append((src_key, dst_key)) + return edges, issues + + +async def ontology_publish_issues( + session: AsyncSession, + spec: GraphSpec, + *, + workflow_id: str = "", +) -> list[dict[str, Any]]: + """发布门禁检查：校验指标 links 结构，并结合已发布口径检查跨工作流 factor 环依赖。""" + from app.db.models import Workflow, WorkflowVersion + + raw_graph = spec.model_dump() + local_edges, issues = _graph_factor_edges(raw_graph, workflow_id=workflow_id) + if issues: + return issues + + all_edges = list(local_edges) + if local_edges: + rows = ( + await session.execute( + select(Workflow.id, Workflow.published_version, WorkflowVersion.graph) + .join( + WorkflowVersion, + (WorkflowVersion.workflow_id == Workflow.id) + & (WorkflowVersion.version == Workflow.published_version), + ) + .where(Workflow.published_version.is_not(None)) + ) + ).all() + for pub_wf_id, _ver, pub_graph in rows: + if workflow_id and str(pub_wf_id) == str(workflow_id): + continue + peer_edges, _ = _graph_factor_edges(pub_graph or {}, workflow_id=str(pub_wf_id)) + all_edges.extend(peer_edges) + + for cycle in find_factor_cycles(all_edges): + issues.append({ + "level": "error", + "node_id": cycle[0].split(":")[1] if ":" in cycle[0] else None, + "field": "metrics.links", + "code": ONTOLOGY_CYCLE_CODE, + "message": f"指标构成因子（factor）不能形成循环依赖：{' -> '.join(cycle)}", + }) + return issues + + +def attach_docdraft_links(metric: Mapping[str, Any], raw_links: Any, chunks: Mapping[str, dict]) -> list[dict[str, Any]]: + """供 ledger/docdraft 规整模型从文档抽取的候选指标关系（P2）。 + + 只有至少带一条可逐字核对原文的引文（或者明确复用本指标已核对通过的引文）才保留，且状态一律锁为 `proposed`、 + 来源记为 `docdraft`，绝不直接变成运行时自动穿透的已确认边。 + """ + from app.ledger.docdraft import _check_citation + + if not isinstance(raw_links, list) or not raw_links: + return [] + fallback_citations = list(metric.get("citations") or []) + out: list[dict[str, Any]] = [] + for idx, item in enumerate(raw_links): + if not isinstance(item, Mapping): + continue + raw_cites = item.get("citations") + verified = [kept for kept in (_check_citation(c, chunks) for c in (raw_cites or [])) if kept] \ + if isinstance(raw_cites, list) and raw_cites else fallback_citations + if not verified: + continue + candidate = {**dict(item), "citations": verified, "source": "docdraft", "status": "proposed"} + try: + norm = normalize_link(candidate, index=idx, force_proposed=True, default_source="docdraft") + except OntologyInvalid: + continue + out.append(norm) + return out + + +async def build_published_ontology_graph( + session: AsyncSession, + *, + include_proposed: bool = False, + allowed_calibers: set[tuple[str, int, str]] | None = None, + restricted: bool = False, +) -> dict[str, Any]: + """汇总当前已发布口径的指标本体节点、关系边、自动表血缘及标准业务实体跨源桥。""" + from app.data.entities import confirmed_bindings, entity_to_dict, list_entities + from app.db.models import Workflow, WorkflowVersion + + rows = ( + await session.execute( + select(Workflow.id, Workflow.name, Workflow.published_version, WorkflowVersion.graph) + .join( + WorkflowVersion, + (WorkflowVersion.workflow_id == Workflow.id) + & (WorkflowVersion.version == Workflow.published_version), + ) + .where(Workflow.published_version.is_not(None)) + .order_by(Workflow.id.asc()) + ) + ).all() + + nodes: list[dict[str, Any]] = [] + edges: list[dict[str, Any]] = [] + node_keys: set[str] = set() + + for wf_id, wf_name, pub_ver, graph in rows: + ver = int(pub_ver) + idx = workflow_ontology_index( + workflow_id=str(wf_id), + version=ver, + graph=graph or {}, + workflow_name=str(wf_name or wf_id), + ) + for card in idx["cards"]: + cid = card["card_id"] + if allowed_calibers is not None and (str(wf_id), ver, cid) not in allowed_calibers: + continue + for m in card["metrics"]: + mid = m["metric_id"] + key = metric_node_key(str(wf_id), cid, mid) + node_keys.add(key) + node_entry: dict[str, Any] = { + "key": key, + "workflow_id": str(wf_id), + "workflow_name": str(wf_name or wf_id), + "version": ver, + "card_id": cid, + "card_title": card["title"], + "caliber": card["caliber"], + "metric_id": mid, + "name": m["name"], + "unit": m["unit"], + "contract": m["contract"], + "sources": card["sources"], + } + if not restricted: + node_entry["tables_by_source"] = card["tables_by_source"] + node_entry["lineage"] = m["lineage"] + nodes.append(node_entry) + + for edge in idx["edges"]: + if not include_proposed and edge["status"] not in ACTIVE_LINK_STATUSES: + continue + src = edge["from"] + dst = edge["to"] + if allowed_calibers is not None: + if (src["workflow_id"], ver, src["card_id"]) not in allowed_calibers: + continue + # 若受限用户无权访问目标口径，不暴露未授权邻接边（铁律 2） + dst_allowed = any( + w == dst["workflow_id"] and c == dst["card_id"] + and (dst.get("version") is None or v == dst.get("version")) + for w, v, c in allowed_calibers + ) + if not dst_allowed: + continue + edges.append(edge) + + entity_rows = await list_entities(session) + entities_out: list[dict[str, Any]] = [] + for row in entity_rows: + if not include_proposed and not confirmed_bindings(row) and not row.members: + continue + entities_out.append(entity_to_dict(row, restricted=restricted)) + + warnings: list[dict[str, Any]] = [] + factor_pairs = [ + ( + metric_node_key(e["from"]["workflow_id"], e["from"]["card_id"], e["from"]["metric_id"]), + metric_node_key(e["to"]["workflow_id"], e["to"]["card_id"], e["to"]["metric_id"]), + ) + for e in edges + if e["kind"] == "factor" and e["status"] != "rejected" + ] + for cycle in find_factor_cycles(factor_pairs): + warnings.append({"code": ONTOLOGY_CYCLE_CODE, "message": f"检测到 factor 循环：{' -> '.join(cycle)}"}) + for e in edges: + dst_key = metric_node_key(e["to"]["workflow_id"], e["to"]["card_id"], e["to"]["metric_id"]) + if allowed_calibers is None and dst_key not in node_keys: + warnings.append({ + "code": "ontology.dangling_target", + "edge_id": e["id"], + "target": dst_key, + "message": f"指标关联指向的已发布口径指标「{dst_key}」当前不存在或未发布", + }) + + return { + "nodes": nodes, + "edges": edges, + "entities": entities_out, + "warnings": warnings, + } diff --git a/backend/app/ledger/querycore.py b/backend/app/ledger/querycore.py index bbd664f3..ca00ff16 100644 --- a/backend/app/ledger/querycore.py +++ b/backend/app/ledger/querycore.py @@ -236,6 +236,12 @@ def _typed_metric(metric: Mapping[str, Any], hidden: str | None) -> dict[str, An value = metric.get("value") out["value"] = exact(value) out["value_type"] = typed(value) + if metric.get("contract_findings"): + out["contract_findings"] = [ + dict(f) if isinstance(f, Mapping) else str(f) + for f in metric["contract_findings"] + if f + ] out["quality"] = _metric_quality(metric) return out @@ -1586,8 +1592,10 @@ class CaliberQueryArgs(BaseModel): workflow: str = Field(description="口径所在的工作流：工作流 id，或唯一的工作流名字") card: str = Field(description="工作流里那张口径卡的节点 id：一次只算一张卡") metric_ids: list[str] = Field(min_length=1, description="这次要算的指标 id，可以多个；空的不当整卡") - inputs: dict[str, Any] = Field(default_factory=dict, - description="这次入口的取值：{字段：值}，按入口声明的类型校验（日期写 YYYY-MM-DD）") + inputs: dict[str, Any] = Field( + default_factory=dict, + description="这次入口的取值：{字段：值}，按入口声明的类型校验（日期写 YYYY-MM-DD；跨源穿透传入对侧编码或标准实体键时由平台自动对齐换算）", + ) @register( @@ -1607,6 +1615,8 @@ async def caliber_query(ctx: ToolContext, **kwargs: Any) -> dict[str, Any]: `parent_run_id`），不失去父关系；超时、失败、遮罩、空指标照实交回 `ok=False`，绝不冒充成功； - 交回的结果与交互工作台同一种形状：metrics + queries 数组（每条带 query_id / execution_id / attempt、工件、来源、 参数材料），节点据此把每条 query artifact 以合法引用记进父 Run 的证据。""" + extra_entity_inputs = kwargs.pop("entity_inputs", None) + extra_from_source = kwargs.pop("from_source", None) args = CaliberQueryArgs(**kwargs) from app.db.base import SessionLocal from app.db.models import Run @@ -1636,8 +1646,25 @@ async def caliber_query(ctx: ToolContext, **kwargs: Any) -> dict[str, Any]: purpose = HOST_PURPOSE if run_class == "formal" else DEFAULT_PURPOSE declared_budget = options.get(BUDGET_OPTION) workflow_id = await _workflow_id(session, args.workflow) + effective_inputs = dict(args.inputs or {}) + alignments: list[dict[str, Any]] = [] + if effective_inputs or extra_entity_inputs: + from app.auth.chat_policy import restricted_actor + from app.tools.ontology import align_caliber_inputs + + is_restricted = await restricted_actor(session, actor) + effective_inputs, alignments = await align_caliber_inputs( + session, + workflow_id=workflow_id, + card_id=args.card, + inputs=effective_inputs, + entity_inputs=extra_entity_inputs if isinstance(extra_entity_inputs, Mapping) else None, + from_source=str(extra_from_source) if extra_from_source else None, + restricted=is_restricted, + ) outcome = await execute( - session, workflow_id=workflow_id, card_id=args.card, metric_ids=args.metric_ids, inputs=args.inputs, + session, workflow_id=workflow_id, card_id=args.card, metric_ids=args.metric_ids, + inputs=effective_inputs, purpose=purpose, actor=actor, parent_run_id=parent, run_options=options, data_versions=versions if isinstance(versions, Mapping) else None, budget=declared_budget if isinstance(declared_budget, Mapping) else None, trace=trace, @@ -1647,7 +1674,36 @@ async def caliber_query(ctx: ToolContext, **kwargs: Any) -> dict[str, Any]: # 拒（没发起）在工具这一层是一条失败，不是一份结果：机器码带在报错里，节点、Agent 的失败信息按它认这一档， # 不用去解析中文 raise ValueError(f"{e.message}（{e.code}）") from None - return {"tool": CALIBER_QUERY, "kind": CALIBER_QUERY, **outcome.as_dict()} + out_payload: dict[str, Any] = {"tool": CALIBER_QUERY, "kind": CALIBER_QUERY, **outcome.as_dict()} + if alignments: + out_payload["ontology_alignment"] = alignments + violated = [m for m in out_payload.get("metrics") or [] if isinstance(m, Mapping) and m.get("contract_findings")] + emit_fn = getattr(ctx, "emit", None) + if violated and callable(emit_fn): + import inspect + from app.core.events import EventType + + first_finding = (violated[0].get("contract_findings") or [""])[0] + detail = ( + str(first_finding.get("detail") or first_finding.get("code") or "触发指标数值契约异常") + if isinstance(first_finding, Mapping) + else (str(first_finding) or "触发指标数值契约异常") + ) + emitted = emit_fn( + EventType.LOG, + level="warn", + code="metric_contract_violated", + workflow_id=out_payload.get("workflow_id"), + card_id=out_payload.get("card_id"), + metric=violated[0].get("id"), + metrics=[m.get("id") for m in violated], + metric_names=[m.get("name") or m.get("id") for m in violated], + reason=detail, + message="指标" + "".join(f"「{m.get('name') or m.get('id')}」" for m in violated) + f"触发数值契约告警：{detail}", + ) + if inspect.isawaitable(emitted): + await emitted + return out_payload async def _workflow_id(session: AsyncSession, ref: str) -> str: diff --git a/backend/app/main.py b/backend/app/main.py index 17d6f96e..ac4ae1d3 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -31,6 +31,8 @@ from app.api import ( from app.api import chat_turns as chat_turns_api # 增量重导入（K11）：建一次导入、看工作项和冲突、接受选定字段、拒绝、人工对齐、恢复 from app.api import caliber_import as caliber_import_api +# 指标本体穿透（W1-a）：标准业务实体与跨口径指标关系网 +from app.api import ontology as ontology_api from app.api.coded import CodedHTTPException, coded_handler from app.api.deps import chat_scope, evidence_scope, private_run_scope, require_role, role_by_method, run_scope from app.auth.keys import key_gate @@ -141,7 +143,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: finally: # 先停调度，再停运行：停运行的途中不该再有新运行被发起。起草、剖析任务停下来标成中断，下次启动不会再续 await scheduler.stop() - from app.data import catalog_jobs, catalog_profile_jobs, introspect_jobs + from app.data import catalog_jobs, catalog_profile_jobs, entity_discovery, introspect_jobs from app.chat import job as chat_job from app.eval import runner as eval_runner, snapshots as eval_snapshots @@ -152,6 +154,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: await catalog_jobs.manager.shutdown() await catalog_profile_jobs.manager.shutdown() await introspect_jobs.manager.shutdown() + await entity_discovery.shutdown() # 问数据的一轮停下不改那几行（下次启动、或者别的进程照「属主不在了」的规矩接手），也要在运行之前停 await chat_job.shutdown() # 评测要在运行之前停：它停下时会取消自己发起的作答、算期望的运行，再标中断 @@ -246,7 +249,7 @@ def _register_reapers() -> None: from app.blueprints import publish_plan from app.chat import job as chat_job from app.cluster import reaper - from app.data import catalog_jobs, catalog_profile_jobs, introspect_jobs + from app.data import catalog_jobs, catalog_profile_jobs, entity_discovery, introspect_jobs from app.eval import runner as eval_runner, snapshots as eval_snapshots from app.process import actions, tasks from app.ledger import jobs as draft_jobs, reimport as import_jobs @@ -262,6 +265,7 @@ def _register_reapers() -> None: ("数据目录起草任务", lambda live: catalog_jobs.startup(live=live)), ("数据剖析任务", lambda live: catalog_profile_jobs.startup(live=live)), ("探查任务", lambda live: introspect_jobs.startup(live=live)), + ("候选实体扫描任务", lambda live: entity_discovery.startup(live=live)), ("影子起草", lambda live: eval_snapshots.startup(live=live)), ("评测任务", lambda live: eval_runner.startup(live=live)), ("问数据的一轮", _chat_reaper(chat_job)), @@ -430,6 +434,8 @@ for router, guard in ( (ledger_api.router, _BUILDER), # 增量重导入（K11）：和 /api/ledger 其余写接口同一道门（搭建者）；访问密钥不开放（不登记 KEY_ROUTES） (caliber_import_api.router, _BUILDER), + # 指标本体穿透（W1-a）：标准业务实体与指标本体图谱（搭建者及以上） + (ontology_api.router, _BUILDER), # 下一轮第 2 波：评测只给搭建者；访问密钥登录即可，接口里再按本人 / 管理员判 (evals_api.router, _BUILDER), # 口径账本里按答案出具、对账的（第 3 波）：登录即可，接口里按运行和应用授权判 diff --git a/backend/app/report_export/build.py b/backend/app/report_export/build.py index 099748cf..adffda00 100644 --- a/backend/app/report_export/build.py +++ b/backend/app/report_export/build.py @@ -79,7 +79,8 @@ _STATE_CLASS = { "deterministic": "ok", "none": "na", "suspect": "bad", "unverified": "bad", "unjudged": "na", "candidate": "na", } -_KIND_LABEL = {"number": "数字", "value": "取值", "entity": "名称", "quote": "引文", "analysis": "判定"} +_KIND_LABEL = {"number": "数字", "value": "取值", "entity": "名称", "quote": "引文", "analysis": "判定", + "source_health": "健康"} def _esc(text: Any) -> str: diff --git a/backend/app/seed.py b/backend/app/seed.py index 478a6fb3..e9436200 100644 --- a/backend/app/seed.py +++ b/backend/app/seed.py @@ -396,6 +396,51 @@ TEMPLATES: list[dict[str, Any]] = [ _e("check", "write"), _e("write", "done"), ], }, + { + "name": "⑫ 跨源指标异动与对账诊断", + "description": "先核查数据源健康与跨源对账口径，再做同源维度定位与已确认本体穿透，出具带引用与非因果约束的诊断报告。", + "tags": ["分析", "本体", "对账", "证据"], + "nodes": [ + _n("start", "input", "诊断问题", fields=[ + {"name": "question", "required": True, + "default": "星河智造昨日设备有效运转率为何异常异动？请先核查数据源健康与跨源对账，再定位重点产线。", + "description": "要排查的指标异常或跨源对账问题"}, + {"name": "period", "default": "2026-10-21", "description": "目标统计周期"}]), + _n("investigate", "agent", "跨源本体排查 Agent", + system="你是跨源指标诊断研究员：严格遵循先核数据源健康与对账、再同源拆解切片、最后沿已确认本体穿透的三段式纪律；" + "一旦发现源表断更、断档或对账失配即停止业务下钻。", + prompt="围绕统计周期 {{ input.period }} 排查问题：{{ input.question }}\n" + "1. 先调用本体邻居查询查看目标口径已确认的构成因子、跨源对账镜像与上游驱动关系，并调用数据源健康核查检查" + "涉及的数据源是否存在断更、断档、行数骤降或新增未识别编码；若存在对账镜像，调用口径查询取数后做跨源对账。\n" + "2. 若数据源健康正常且主指标超出波动基线，先在同源做变化拆解锁定贡献最大的业务实体切片。\n" + "3. 将异常切片传入本体邻居查询获取对齐后的调用参数，调用口径查询拉取已确认邻居指标并检验伴随异动（最多穿透 2 跳）。", + tools=[ + "caliber_query", + "ontology_neighbors", + "source_health", + "analysis_comparable", + "analysis_baseline", + "analysis_decompose", + "analysis_reconcile", + "analysis_cochange", + ], + approval="dangerous", + assign_to="investigation_notes"), + _n("write", "report", "诊断报告撰写", numbers="strict", on_violation="fail", claims="require_citation", + analysis_claims="strict", + system="你是经营诊断撰写人：结论先行，判断严格引用分析结论与数据源健康判定语，统计伴随异动绝不写成因果定论。", + instructions="围绕统计周期 [[i:period]] 回答：{{ input.question }}（排查摘要：{{ vars.investigation_notes }}）。" + "按「核查结论 / 数据源健康与跨源对账 / 切片定位与伴随异动」三段组织正文：数据源健康结论引用 [[h:H1]]，" + "分析判定直接引用 [[a:A1]] 等判定语，口径与单元格按证据引用；伴随异动一律表述为统计同步异动，" + "没有明确规则依据时写成待核实的推测。"), + _n("done", "output", "出具", + fields=[{"name": "跨源诊断报告", "value": "{{ nodes.write.text }}"}], + contract={"report_from": "write", "cells": True, "strict": True}), + ], + "edges": [ + _e("start", "investigate"), _e("investigate", "write"), _e("write", "done"), + ], + }, ] diff --git a/backend/app/tools/analysis.py b/backend/app/tools/analysis.py index b70e0c99..2cc8cbff 100644 --- a/backend/app/tools/analysis.py +++ b/backend/app/tools/analysis.py @@ -38,6 +38,9 @@ SCOPE_KEY = "analysis_scope" ANALYSIS_TOOLS = frozenset(contract.TOOL_NAMES.values()) #: 交回 agent 的那一段开头的编号写法，和查询的「【证据 Qn】」一个样子 REPLY_HEAD = "【分析 {alias}】" +#: 数据源健康度工具交回 agent 的编号写法（证据目录里编成 H1、H2……） +HEALTH_PREFIX = "H" +HEALTH_REPLY_HEAD = "【健康 {alias}】" #: 调用写错时交回 agent 的那句的开头：说清没有记成结论，下一步照着改 CALL_ERROR_HEAD = "分析调用有误，没有记成结论：" #: agent 写的「这次在验证什么」最多留多少字（只在证据面板显示，不进正文） @@ -82,6 +85,19 @@ class AnalysisScope: probe_cache: dict[str, Any] = field(default_factory=dict) #: 已经换成工件库里那份完整快照的编号（prepare 每份只取回一次） prepared: set[str] = field(default_factory=set) + #: 数据源健康度核查结论：编号 Hn → 健康度结果 / 工件 id + source_healths: dict[str, dict[str, Any]] = field(default_factory=dict) + health_artifacts: dict[str, str] = field(default_factory=dict) + #: 本体穿透状态跟踪（重放时按 tool_step 返回值复现，保证确定性） + ontology_neighbors: list[dict[str, Any]] = field(default_factory=list) + focal_calibers: set[str] = field(default_factory=set) + focal_sources: set[str] = field(default_factory=set) + queried_calibers: set[str] = field(default_factory=set) + queried_sources: set[str] = field(default_factory=set) + traversed_targets: set[str] = field(default_factory=set) + contract_violations: list[dict[str, Any]] = field(default_factory=list) + ontology_hops: int = 0 + cross_queries: int = 0 def add_query(self, alias: str, snapshot: InputSnapshot) -> None: self.queries[alias] = snapshot @@ -97,6 +113,97 @@ class AnalysisScope: self.artifacts[alias] = artifact return alias + def next_health_alias(self) -> str: + """下一份数据源健康度核查在本节点的编号（H1、H2……）。""" + return f"{HEALTH_PREFIX}{len(self.source_healths) + 1}" + + def add_health(self, result: Mapping[str, Any], artifact: str) -> str: + """记一份数据源健康度结论，返回编号 H1、H2……。""" + alias = self.next_health_alias() + self.source_healths[alias] = dict(result) + self.health_artifacts[alias] = artifact + return alias + + def record_ontology_neighbors( + self, + neighbors: Iterable[Mapping[str, Any]], + focal_metrics: Iterable[Mapping[str, Any]] = (), + ) -> None: + """幂等记录 1 跳已确认本体邻居与起点口径/数据源。""" + if not self.focal_calibers and not self.focal_sources: + for fm in focal_metrics: + wf_id = str(fm.get("workflow_id") or "").strip() + card_id = str(fm.get("card_id") or "").strip() + if wf_id: + self.focal_calibers.add(f"{wf_id}:{card_id}" if card_id else wf_id) + for src in fm.get("sources") or []: + if src: + self.focal_sources.add(str(src)) + existing = { + ( + str(n.get("kind") or ""), + str((n.get("target") or {}).get("workflow_id") or ""), + str((n.get("target") or {}).get("card_id") or ""), + str((n.get("target") or {}).get("metric_id") or ""), + ) + for n in self.ontology_neighbors + } + for nb in neighbors: + target = nb.get("target") or {} + key = ( + str(nb.get("kind") or ""), + str(target.get("workflow_id") or ""), + str(target.get("card_id") or ""), + str(target.get("metric_id") or ""), + ) + if key not in existing: + existing.add(key) + self.ontology_neighbors.append(dict(nb)) + + def record_query_target( + self, + *, + sources: Iterable[str] = (), + workflow_id: str | None = None, + card_id: str | None = None, + violations: Iterable[Mapping[str, Any]] = (), + ) -> None: + """记录查询命中的口径或数据源，自动统计跨源查询数与本体跳数。""" + clean_sources = [str(s).strip() for s in sources if s and str(s).strip()] + cal_key = f"{workflow_id}:{card_id}" if workflow_id and card_id else (str(workflow_id) if workflow_id else "") + + first_touch = not self.focal_calibers and not self.focal_sources + if first_touch: + if cal_key: + self.focal_calibers.add(cal_key) + self.focal_sources.update(clean_sources) + else: + is_cross = False + if cal_key and cal_key not in self.focal_calibers: + is_cross = True + self.traversed_targets.add(f"caliber:{cal_key}") + for src in clean_sources: + if self.focal_sources and src not in self.focal_sources: + is_cross = True + if not cal_key: + self.traversed_targets.add(f"source:{src}") + if is_cross: + self.cross_queries += 1 + self.ontology_hops = len(self.traversed_targets) + + if cal_key: + self.queried_calibers.add(cal_key) + if workflow_id: + self.queried_calibers.add(str(workflow_id)) + self.queried_sources.update(clean_sources) + + seen_v = {(v.get("metric_id"), v.get("detail")) for v in self.contract_violations} + for v in violations: + vkey = (v.get("metric_id"), v.get("detail")) + if vkey not in seen_v: + seen_v.add(vkey) + self.contract_violations.append(dict(v)) + def resolve(self, name: str) -> InputSnapshot | None: return self.queries.get(str(name).strip()) @@ -384,8 +491,113 @@ def hypothesis_of(value: Any) -> str | None: return text[:HYPOTHESIS_MAX] if text else None +def health_payload_of(content: Any) -> dict[str, Any] | None: + """source_health 工具交回的 JSON → {artifact, result}；不是健康度结果的返回 None。""" + from app.data.source_health import is_health_result + + data = content + if isinstance(data, str): + if not data.lstrip().startswith("{"): + return None + try: + data = json.loads(data) + except ValueError: + return None + if not isinstance(data, dict): + return None + artifact = data.get("artifact") + result = data.get("result") + if not (isinstance(artifact, str) and artifact and is_health_result(result)): + return None + return {"artifact": artifact, "result": dict(result)} + + +def health_reply_text(alias: str, result: Mapping[str, Any]) -> str: + """交回模型的数据源健康度摘要：「【健康 Hn】」加紧凑 JSON。""" + body: dict[str, Any] = { + "verdict": result.get("verdict"), + "phrase": result.get("phrase"), + "sources": [ + { + k: s[k] + for k in ("source", "source_label", "verdict", "issue_count", "phrase") + if isinstance(s, Mapping) and k in s + } + for s in (result.get("sources") or []) + if isinstance(s, Mapping) + ], + "issues": list(result.get("issues") or []), + } + if result.get("caliber_ref"): + body["caliber_ref"] = result["caliber_ref"] + return HEALTH_REPLY_HEAD.format(alias=alias) + "\n" + json.dumps(body, ensure_ascii=False, default=str) + + +def health_event_view(result: Mapping[str, Any], alias: str | None = None) -> dict[str, Any]: + """source_health 在 tool.end 上附带的简要判定视图。""" + return { + "alias": alias, + "verdict": result.get("verdict"), + "phrase": result.get("phrase") or "", + "issue_count": len(result.get("issues") or []), + } + + +def record_ontology_call(scope: AnalysisScope | None, name: str, content: Any) -> None: + """在 tool_step 返回之后幂等记录 ontology_neighbors / caliber_query / db_query 的跨源穿透状态（支持节点重放）。""" + if scope is None: + return + data = content + if isinstance(data, str): + if not data.lstrip().startswith("{"): + return + try: + data = json.loads(data) + except ValueError: + return + if not isinstance(data, Mapping): + return + + if name == "ontology_neighbors" and data.get("kind") == "ontology_neighbors": + scope.record_ontology_neighbors( + data.get("neighbors") or [], + focal_metrics=data.get("focal_metrics") or [], + ) + return + + if name == "caliber_query" and data.get("workflow_id"): + q_sources = [ + str(q.get("data_source") or "") + for q in (data.get("queries") or []) + if isinstance(q, Mapping) and q.get("data_source") + ] + violations: list[dict[str, Any]] = [] + for m in data.get("metrics") or []: + if not isinstance(m, Mapping): + continue + for chk in (m.get("quality") or {}).get("checks") or []: + if isinstance(chk, Mapping) and chk.get("code") == "metric_contract": + violations.append({ + "metric_id": m.get("id"), + "name": m.get("name"), + "detail": chk.get("detail"), + }) + scope.record_query_target( + sources=q_sources, + workflow_id=str(data.get("workflow_id") or "") or None, + card_id=str(data.get("card_id") or "") or None, + violations=violations, + ) + return + + if name.startswith("db_query__") and data.get("source"): + scope.record_query_target(sources=[str(data["source"])]) + + __all__ = [ - "ANALYSIS_TOOLS", "CALL_ERROR_HEAD", "REFUSED_CODE", "REPLY_HEAD", "SCOPE_KEY", "AnalysisScope", "call_error_text", "context_of", - "event_view", "hypothesis_of", "input_snapshot", "is_analysis_tool", "masked_of", "payload_of", "prepare", "reply_text", - "run_snapshot", "scope_of", "settings_of", "store", "tool_context", + "ANALYSIS_TOOLS", "CALL_ERROR_HEAD", "HEALTH_PREFIX", "HEALTH_REPLY_HEAD", "REFUSED_CODE", "REPLY_HEAD", + "SCOPE_KEY", "AnalysisScope", "call_error_text", "context_of", "event_view", "health_event_view", + "health_payload_of", "health_reply_text", "hypothesis_of", "input_snapshot", "is_analysis_tool", "masked_of", + "payload_of", "prepare", "record_ontology_call", "reply_text", "run_snapshot", "scope_of", "settings_of", "store", + "tool_context", ] diff --git a/backend/app/tools/builtin/analysis.py b/backend/app/tools/builtin/analysis.py index 5f58b1d0..a2458cce 100644 --- a/backend/app/tools/builtin/analysis.py +++ b/backend/app/tools/builtin/analysis.py @@ -147,10 +147,40 @@ class ComparableArgs(BaseModel): hypothesis: str | None = Field(default=None, description=_HYPOTHESIS) +class ReconcileArgs(BaseModel): + base_query: str | None = Field(default=None, description="主侧（左侧数据源）查询结果的编号，如 Q1") + target_query: str | None = Field(default=None, description="对账侧（右侧镜像数据源）查询结果的编号，如 Q2") + query: str | None = Field(default=None, description="两侧已合并进同一份查询结果时的编号（这时不写 base_query、target_query）") + by: list[str] | None = Field(default=None, description="对齐维度列（如产线、日期），1 到 2 列；总量单行对账时留空") + target_by: list[str] | None = Field(default=None, description="两份查询且对账侧维度列名不同时，写对账侧的维度列名（与 by 顺序对应）") + measure: str = Field(description="主侧要核对的数值列名") + target_measure: str | None = Field(default=None, description="对账侧的数值列名（两份查询且列名与 measure 相同时可留空；单份查询时必填）") + unit_factor: float | None = Field(default=None, description="已知的物理/计量单位换算系数（对账侧数值乘以该系数后与主侧比，如吨转千克写 1000）") + label: str | None = Field(default=None, description=_LABEL) + hypothesis: str | None = Field(default=None, description=_HYPOTHESIS) + + +class CochangeArgs(BaseModel): + base_query: str | None = Field(default=None, description="主指标时间序列查询编号，如 Q1") + target_query: str | None = Field(default=None, description="上游驱动指标时间序列查询编号，如 Q2") + query: str | None = Field(default=None, description="主指标与驱动指标在同一份序列查询中时的编号（这时不写 base_query、target_query）") + period: str = Field(description="主指标期次列名") + target_period: str | None = Field(default=None, description="两份查询且驱动指标期次列名不同时，写驱动侧期次列名") + measure: str = Field(description="主指标数值列名") + driver_measure: str = Field(description="上游驱动指标数值列名") + target: str | None = Field(default=None, description="要检验异动的期次值；留空取主序列最后一期") + basis: Literal["change", "level"] | None = Field( + default=None, description="change：相邻期变化率是否同步超出常规波动（缺省）；level：水平值本身") + grain: Literal["day", "week", "month", "quarter"] | None = Field(default=None, description=_GRAIN) + lag: int | None = Field(default=None, description="上游驱动领先主指标的期数（0 表示同期，1 表示驱动早 1 期；缺省 0）") + label: str | None = Field(default=None, description=_LABEL) + hypothesis: str | None = Field(default=None, description=_HYPOTHESIS) + + #: 算子 → 参数模型。分析卡的一项按同一份参数模型校验参数（engine/nodes/analysis.py） ARGS: dict[str, type[BaseModel]] = { "decompose": DecomposeArgs, "baseline": BaselineArgs, "trend": TrendArgs, "differ": DifferArgs, - "comparable": ComparableArgs, + "comparable": ComparableArgs, "reconcile": ReconcileArgs, "cochange": CochangeArgs, } #: 不算「算什么」的参数：模式另放，这次在验证什么另放（contract.AnalysisRequest） NOT_PARAMS = frozenset({"mode", "hypothesis"}) @@ -262,4 +292,28 @@ async def analysis_comparable(ctx: ToolContext, **kwargs: Any) -> dict[str, Any] return await run_tool("comparable", ctx, ComparableArgs(**kwargs)) +@register( + name=contract.TOOL_NAMES["reconcile"], + category=CATEGORY, + description="跨源对账：核对两个镜像数据源（或两套口径快照）在总量或各实体切片上是否对齐，按平台容差判定对账一致、" + "存在显著偏差或对侧缺失，列出超差最大的实体项。", + args_schema=ReconcileArgs, + guide=METHOD_GUIDE, +) +async def analysis_reconcile(ctx: ToolContext, **kwargs: Any) -> dict[str, Any]: + return await run_tool("reconcile", ctx, ReconcileArgs(**kwargs)) + + +@register( + name=contract.TOOL_NAMES["cochange"], + category=CATEGORY, + description="伴随异动：检验主指标超出常规波动时，跨源上游关联指标在同期（或领先期 lag）是否也超出其自身稳健历史基线，" + "给出统计伴随异动或驱动未见异动的判定。", + args_schema=CochangeArgs, + guide=METHOD_GUIDE, +) +async def analysis_cochange(ctx: ToolContext, **kwargs: Any) -> dict[str, Any]: + return await run_tool("cochange", ctx, CochangeArgs(**kwargs)) + + __all__ = ["ANALYSIS_METHOD_PROMPT", "ARGS", "METHOD_GUIDE", "param_names", "request_of", "run_tool"] diff --git a/backend/app/tools/ontology.py b/backend/app/tools/ontology.py new file mode 100644 index 00000000..15f98ca9 --- /dev/null +++ b/backend/app/tools/ontology.py @@ -0,0 +1,612 @@ +"""指标本体与数据源健康内置工具（W2-a）：ontology_neighbors、source_health 与跨源实体参数对齐。 + +三条边界铁律： +1. **状态门禁（P2）**：运行期自动穿透只暴露与使用 `verified / confirmed` 状态的指标边（`links`）和标准实体绑定 + （`bindings` / `caliber_params`），`proposed` 候选一律不参与运行期穿透。 +2. **权限对齐（铁律 2 / K14）**：普通使用者（`restricted_actor == True`）调用 `ontology_neighbors` 时，仅返回已授权给其应用 + （`apps/access.usable_calibers`）的目标口径版本，无权邻居连名字、工作流 ID、表名都不暴露（仅计入 `omitted_unauthorized_count`）； + 越权点名未授权口径直接报 `querycore_not_authorized`。`source_health` 对普通使用者同样核查口径/数据源范围并开启 `restricted=True`。 +3. **确定性跨源键值翻译**：`align_caliber_inputs` 严格通过 `CanonicalEntity` 的已确认参数映射与成员编码表转换，绝不靠子串猜值。 +""" +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from pydantic import BaseModel, Field +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.data import source_health as sh +from app.data.entities import ( + ACTIVE_ENTITY_STATUSES, + confirmed_bindings, + confirmed_caliber_params, + entities_bridging_sources, + entity_to_dict, + list_entities, + resolve_caliber_param, + translate_entity_value, +) +from app.data.names import name_key +from app.ledger.ontology import ( + ACTIVE_LINK_STATUSES, + build_published_ontology_graph, + card_metric_lineage, + metric_node_key, + workflow_ontology_index, +) +from app.tools.registry import ToolContext, ToolGuide, register + +ONTOLOGY_NEIGHBORS = "ontology_neighbors" +SOURCE_HEALTH = "source_health" +ONTOLOGY_TOOLS = frozenset({ONTOLOGY_NEIGHBORS, SOURCE_HEALTH}) + +ONTOLOGY_GUIDE = ToolGuide( + id="ontology.method@1", + text=( + "【跨源本体穿透与异常排查纪律】\n" + "1. 先核健康与对账（第 1 阶段）：指标出现断崖下跌、无数据、契约越界（如比率 > 100%）或跨库口径分歧时，先调 " + "ontology_neighbors 查看已确认的 1 跳本体关系（factor 构成因子 / reconciles_with 对账镜像 / upstream_driver 上游驱动）" + "及跨源标准实体桥；涉及外部关联源时优先调 source_health 核查是否存在断更（date_stalled）、断档（date_gap）、" + "行数骤降（volume_drop）或新增未识别编码（codes_new）。若存在 reconciles_with 镜像口径，取数后调 analysis_reconcile " + "核对差额。一旦确认为源表断档或对账失配，立即停在数据故障结论（引用 [[h:H1]] 或 [[a:A1]]），不在问题数据上强拆业务维度。\n" + "2. 同源定切片再跨源（第 2 阶段）：源健康正常、业务统计波动超带（outside_band / far_outside_band）时，先在同源用 " + "analysis_decompose 锁定贡献最大的业务实体切片（如异常产线、仓库或渠道），再将该切片写入 ontology_neighbors 的 " + "entity_filters 获取对齐后的邻居口径调用参数。\n" + "3. 跨源穿透与非因果纪律（第 3 阶段）：按 recommended_caliber_query 调用 caliber_query（在 inputs 中传目标参数或标准实体键，" + "平台确定性翻译跨源编码）拉取邻居指标，最多穿透 2 跳。核对 upstream_driver 同步波动一律调用 analysis_cochange 并引用系统" + "判定语；统计伴随不等于因果证明，没有知识库明确规则依据时严禁写成「导致 / 引起 / 造成」。" + ), +) + + +class OntologyNeighborsArgs(BaseModel): + """查询指定口径指标的 1 跳已确认本体邻居、反向表血缘与跨源实体对齐参数。""" + + workflow: str = Field(description="当前口径所在的工作流 id 或唯一工作流名称") + card: str | None = Field(default=None, description="可选：限定口径卡节点 id；不填则查看该工作流全部源头口径卡") + metric_id: str | None = Field(default=None, description="可选：限定单个指标 id；不填则查看卡上全部指标") + entity_filters: dict[str, Any] = Field( + default_factory=dict, + description="可选：当前已定位的异常实体切片，如 {'production_line': 'L2'}，自动换算成各邻居口径的入口参数", + ) + from_source: str | None = Field( + default=None, + description="可选：entity_filters 取值来自哪个数据源名称，用于精确匹配该源的编码表", + ) + + +class SourceHealthArgs(BaseModel): + """快速核查数据源或口径底层表的健康度、新鲜度与待处理漂移告警。""" + + source_name: str | None = Field(default=None, description="要核查的数据源名称（与 workflow 二选一或同时提供）") + tables: list[str] | None = Field(default=None, description="可选：限定核查的表名列表；不填则按口径血缘或探查前若干张表") + workflow: str | None = Field(default=None, description="可选：按已发布口径工作流 id 或名称自动反查其底层数据源与表") + card: str | None = Field(default=None, description="可选：配合 workflow 限定某张口径卡") + metric_id: str | None = Field(default=None, description="可选：配合 workflow 限定某个指标") + start: str | None = Field(default=None, description="可选：分析时间窗起点 YYYY-MM-DD，用于核查窗口覆盖缺口") + end: str | None = Field(default=None, description="可选：分析时间窗终点 YYYY-MM-DD，用于核查断更或断档") + + +async def _resolve_actor_and_restriction(session: AsyncSession, ctx: ToolContext) -> tuple[Any, bool, dict[str, Any] | None]: + """与 QueryCore 同口径解析当前运行的真实主体与是否受限（普通 user 角色）。""" + from app.auth.chat_policy import restricted_actor + from app.db.models import Run + + run_id = str(getattr(ctx, "run_id", "") or "").strip() + parent_row = await session.get(Run, run_id) if run_id else None + if parent_row is not None: + actor = parent_row.started_by + graph = parent_row.graph if isinstance(parent_row.graph, dict) else None + else: + actor = getattr(ctx, "started_by", None) + graph = None + is_restricted = await restricted_actor(session, actor) + return actor, is_restricted, graph + + +def _input_field_names(graph: Mapping[str, Any] | None) -> set[str]: + out: set[str] = set() + for node in (graph or {}).get("nodes") or []: + if not isinstance(node, Mapping) or str(node.get("type") or "") != "input": + continue + cfg = node.get("data", {}).get("config") if isinstance(node.get("data"), Mapping) else node.get("config") + if not isinstance(cfg, Mapping): + continue + for f in cfg.get("fields") or []: + if isinstance(f, Mapping) and str(f.get("name") or "").strip(): + out.add(str(f["name"]).strip()) + return out + + +async def align_caliber_inputs( + session: AsyncSession, + *, + workflow_id: str, + card_id: str, + inputs: Mapping[str, Any] | None = None, + entity_inputs: Mapping[str, Any] | None = None, + from_source: str | None = None, + restricted: bool = False, +) -> tuple[dict[str, Any], list[dict[str, Any]]]: + """把业务实体切片值确定性地转换为目标口径（workflow_id + card_id）的入口参数与目标源编码。 + + 返回 (合并后的 inputs 字典, 对齐记录列表)。 + - 只认 `verified / confirmed` 状态的实体绑定与口径参数映射； + - 既支持显式传入 `entity_inputs={"production_line": "L2"}`，也支持调用方已在 `inputs={"equip_line_id": "L2"}` + 传入源侧别名/跨源编码时自动翻译为目标源编码（如 `"EQ-L2"`）。 + """ + from app.db.models import Workflow, WorkflowVersion + + merged: dict[str, Any] = dict(inputs or {}) + alignments: list[dict[str, Any]] = [] + + entities = await list_entities(session) + if not entities: + return merged, alignments + + wf = await session.get(Workflow, workflow_id) + target_graph: Mapping[str, Any] = {} + if wf is not None: + ver = wf.published_version or wf.version + snap = ( + await session.execute( + select(WorkflowVersion).where( + WorkflowVersion.workflow_id == workflow_id, + WorkflowVersion.version == int(ver), + ) + ) + ).scalar_one_or_none() + target_graph = (snap.graph if snap is not None else wf.graph) or {} + + lineage = card_metric_lineage(target_graph, card_id) if target_graph and card_id else {"sources": []} + target_sources: list[str] = list(lineage.get("sources") or []) + declared_fields = _input_field_names(target_graph) + + # 构建实体索引：key / label / aliases -> entity + by_token: dict[str, Any] = {} + by_param: dict[str, Any] = {} + for ent in entities: + has_active = bool(confirmed_bindings(ent) or confirmed_caliber_params(ent)) + if not has_active: + continue + for tok in (ent.key, ent.label, *(ent.aliases or [])): + clean = str(tok or "").strip().lower() + if clean and clean not in by_token: + by_token[clean] = ent + param_name = resolve_caliber_param(ent, workflow_id=workflow_id, card_id=card_id) + if param_name: + by_param[param_name] = ent + + def _pick_to_source(ent: Any) -> str | None: + if len(target_sources) == 1: + return target_sources[0] + for src in target_sources: + if confirmed_bindings(ent, source_name=src): + return src + return target_sources[0] if target_sources else None + + # 1) 先处理已经写在 inputs 里的口径参数值（如传了别名或跨源编码 'L2'，翻译为目标源编码 'EQ-L2'） + for param_key, raw_val in list(merged.items()): + ent = by_param.get(param_key) + if ent is None or raw_val in (None, "") or isinstance(raw_val, (dict, list, tuple, bool)): + continue + to_src = _pick_to_source(ent) + translated = translate_entity_value(ent, raw_val, from_source=from_source, to_source=to_src) + if translated is not None and str(translated["target_value"]) != str(raw_val): + merged[param_key] = translated["target_value"] + rec: dict[str, Any] = { + "entity_key": ent.key, + "entity_label": ent.label, + "param_name": param_key, + "canonical_value": translated["canonical"], + } + if not restricted: + rec["from_value"] = str(raw_val) + rec["target_value"] = translated["target_value"] + if to_src: + rec["to_source"] = to_src + alignments.append(rec) + + # 1b) 若调用方直接在 inputs 里写了标准实体标识键（如 {'production_line': 'L2'}），自动映射到目标口径入口参数名并换算编码 + for raw_key, raw_val in list(merged.items()): + if raw_key in declared_fields or raw_key in by_param: + continue + if raw_val in (None, "") or isinstance(raw_val, (dict, list, tuple, bool)): + continue + ent = by_token.get(str(raw_key or "").strip().lower()) + if ent is None: + continue + param_name = resolve_caliber_param(ent, workflow_id=workflow_id, card_id=card_id) + if not param_name and ent.key in declared_fields and confirmed_bindings(ent): + param_name = ent.key + if not param_name or param_name in merged: + continue + to_src = _pick_to_source(ent) + translated = translate_entity_value(ent, raw_val, from_source=from_source, to_source=to_src) + if translated is None: + continue + merged.pop(raw_key, None) + merged[param_name] = translated["target_value"] + rec = { + "entity_key": ent.key, + "entity_label": ent.label, + "param_name": param_name, + "canonical_value": translated["canonical"], + } + if not restricted: + rec["from_value"] = str(raw_val) + rec["target_value"] = translated["target_value"] + if to_src: + rec["to_source"] = to_src + alignments.append(rec) + + # 2) 再处理显式传入的 entity_inputs（如 {'production_line': 'L2'} -> {'equip_line_id': 'EQ-L2'}） + for raw_key, raw_val in (entity_inputs or {}).items(): + if raw_val in (None, "") or isinstance(raw_val, (dict, list, tuple, bool)): + continue + ent = by_token.get(str(raw_key or "").strip().lower()) + if ent is None: + continue + param_name = resolve_caliber_param(ent, workflow_id=workflow_id, card_id=card_id) + if not param_name and ent.key in declared_fields and confirmed_bindings(ent): + param_name = ent.key + if not param_name or param_name in merged: + continue + to_src = _pick_to_source(ent) + translated = translate_entity_value(ent, raw_val, from_source=from_source, to_source=to_src) + if translated is None: + continue + merged[param_name] = translated["target_value"] + rec = { + "entity_key": ent.key, + "entity_label": ent.label, + "param_name": param_name, + "canonical_value": translated["canonical"], + } + if not restricted: + rec["from_value"] = str(raw_val) + rec["target_value"] = translated["target_value"] + if to_src: + rec["to_source"] = to_src + alignments.append(rec) + + return merged, alignments + + +@register( + name=ONTOLOGY_NEIGHBORS, + category="口径", + effect="read", + description=( + "查询一份已发布口径或指标的 1 跳已确认业务本体邻居（构成因子 factor、跨源镜像对账 reconciles_with、" + "上游业务驱动 upstream_driver）、底层数据源与标准业务实体跨源桥；传入 entity_filters 时可自动把异常切片" + "转换为邻居口径可直接调用的入口参数。只读。" + ), + args_schema=OntologyNeighborsArgs, + guide=ONTOLOGY_GUIDE, +) +async def ontology_neighbors(ctx: ToolContext, **kwargs: Any) -> dict[str, Any]: + args = OntologyNeighborsArgs(**kwargs) + from app.apps.access import usable_calibers + from app.db.base import SessionLocal + from app.db.models import Workflow, WorkflowVersion + from app.ledger.querycore import NOT_AUTHORIZED, QueryCoreRefused, _workflow_id + from app.tools.analysis import scope_of + + async with SessionLocal() as session: + actor, is_restricted, _ = await _resolve_actor_and_restriction(session, ctx) + try: + wf_id = await _workflow_id(session, args.workflow) + except QueryCoreRefused as exc: + raise ValueError(f"{exc.message}（{exc.code}）") from None + + wf = await session.get(Workflow, wf_id) + if wf is None or wf.published_version is None: + raise ValueError(f"工作流「{args.workflow}」尚未发布，无法查询已生效本体关系") + pub_ver = int(wf.published_version) + + usable: dict[tuple[str, int], str] = {} + if is_restricted: + usable = await usable_calibers(session, actor) + if (wf_id, pub_ver) not in usable: + raise ValueError(f"没有权限查看口径「{args.workflow}」的本体关系（{NOT_AUTHORIZED}）") + + snap = ( + await session.execute( + select(WorkflowVersion).where( + WorkflowVersion.workflow_id == wf_id, + WorkflowVersion.version == pub_ver, + ) + ) + ).scalar_one_or_none() + local_idx = workflow_ontology_index( + workflow_id=wf_id, + version=pub_ver, + graph=(snap.graph if snap is not None else wf.graph) or {}, + workflow_name=wf.name, + ) + + # 全站已发布本体图（仅取 verified/confirmed 生效边） + full_graph = await build_published_ontology_graph(session, include_proposed=False, restricted=False) + node_by_key = {n["key"]: n for n in full_graph["nodes"]} + + focal_keys: set[str] = set() + focal_sources: set[str] = set() + focal_tables: dict[str, set[str]] = {} + focal_metrics: list[dict[str, Any]] = [] + for card in local_idx["cards"]: + if args.card and card["card_id"] != args.card: + continue + for m in card["metrics"]: + if args.metric_id and m["metric_id"] != args.metric_id: + continue + mkey = metric_node_key(wf_id, card["card_id"], m["metric_id"]) + focal_keys.add(mkey) + for src_name in card["sources"]: + focal_sources.add(src_name) + for src_name, tbls in (card.get("tables_by_source") or {}).items(): + focal_tables.setdefault(src_name, set()).update(tbls) + metric_summary: dict[str, Any] = { + "workflow_id": wf_id, + "workflow_name": wf.name, + "version": pub_ver, + "card_id": card["card_id"], + "card_title": card["title"], + "caliber": card["caliber"], + "metric_id": m["metric_id"], + "name": m["name"], + "unit": m["unit"], + "contract": m["contract"], + } + if not is_restricted: + metric_summary["sources"] = card["sources"] + metric_summary["tables_by_source"] = card["tables_by_source"] + focal_metrics.append(metric_summary) + + if not focal_keys: + raise ValueError("未找到匹配的已发布口径卡或指标，请检查 card / metric_id 参数") + + omitted_unauthorized = 0 + neighbors: list[dict[str, Any]] = [] + seen_edge_keys: set[tuple[str, str, str]] = set() + + for edge in full_graph["edges"]: + if edge.get("status") not in ACTIVE_LINK_STATUSES: + continue + src_ref = edge["from"] + dst_ref = edge["to"] + src_key = metric_node_key(src_ref["workflow_id"], src_ref["card_id"], src_ref["metric_id"]) + dst_key = metric_node_key(dst_ref["workflow_id"], dst_ref["card_id"], dst_ref["metric_id"]) + + direction: str | None = None + peer_key: str | None = None + if src_key in focal_keys and dst_key not in focal_keys: + direction, peer_key = "outbound", dst_key + elif src_key in focal_keys and dst_key in focal_keys and src_key != dst_key: + direction, peer_key = "outbound", dst_key + elif dst_key in focal_keys and src_key not in focal_keys and edge["kind"] == "reconciles_with": + # 对账镜像关系天然对称：反向声明的 reconciles_with 也作为 1 跳对账邻居发现 + direction, peer_key = "inbound_mirror", src_key + + if direction is None or peer_key is None: + continue + dedup = (edge["kind"], direction, peer_key) + if dedup in seen_edge_keys: + continue + seen_edge_keys.add(dedup) + + peer_node = node_by_key.get(peer_key) + if peer_node is None: + continue + peer_wf_id = str(peer_node["workflow_id"]) + peer_ver = int(peer_node["version"]) + + if is_restricted and (peer_wf_id, peer_ver) not in usable: + omitted_unauthorized += 1 + continue + + effective_from_src = args.from_source or (sorted(focal_sources)[0] if len(focal_sources) == 1 else None) + aligned_inputs, alignments = await align_caliber_inputs( + session, + workflow_id=peer_wf_id, + card_id=peer_node["card_id"], + inputs={}, + entity_inputs=args.entity_filters, + from_source=effective_from_src, + restricted=is_restricted, + ) + + recommended_call: dict[str, Any] = { + "workflow": peer_wf_id, + "card": peer_node["card_id"], + "metric_ids": [peer_node["metric_id"]], + "inputs": aligned_inputs, + } + neighbor_item: dict[str, Any] = { + "kind": edge["kind"], + "direction": direction, + "role": edge.get("role"), + "entity_key": edge.get("entity_key"), + "lag_periods": edge.get("lag_periods", 0), + "abs_tol": edge.get("abs_tol"), + "rel_tol": edge.get("rel_tol"), + "rationale": edge.get("rationale", ""), + "target": { + "workflow_id": peer_wf_id, + "workflow_name": peer_node["workflow_name"], + "version": peer_ver, + "card_id": peer_node["card_id"], + "card_title": peer_node["card_title"], + "caliber": peer_node["caliber"], + "metric_id": peer_node["metric_id"], + "name": peer_node["name"], + "unit": peer_node["unit"], + **({"sources": peer_node["sources"], "tables_by_source": peer_node.get("tables_by_source", {})} + if not is_restricted else {}), + }, + "recommended_caliber_query": recommended_call, + } + if alignments: + neighbor_item["entity_alignment"] = alignments + neighbors.append(neighbor_item) + + # 标准业务实体跨源桥（对 builder/admin 展示跨源物理表连接提示，对 user 隐去物理表列细节） + entity_rows = await list_entities(session) + bridges: list[dict[str, Any]] = [] + if not is_restricted: + seen_ent: set[str] = set() + for src_name in sorted(focal_sources): + for br in entities_bridging_sources( + entity_rows, + source_name=src_name, + tables=focal_tables.get(src_name), + ): + if br["key"] in seen_ent: + continue + seen_ent.add(br["key"]) + bridges.append(br) + else: + for row in entity_rows: + if any(b.get("source_name") in focal_sources for b in confirmed_bindings(row)): + bridges.append(entity_to_dict(row, restricted=True)) + + out: dict[str, Any] = { + "kind": ONTOLOGY_NEIGHBORS, + "focal_metrics": focal_metrics, + "neighbors": neighbors, + "entity_bridges": bridges, + } + if omitted_unauthorized: + out["omitted_unauthorized_count"] = omitted_unauthorized + + scope = scope_of(getattr(ctx, "extra", None)) + if scope is not None and hasattr(scope, "record_ontology_neighbors"): + scope.record_ontology_neighbors(neighbors, focal_metrics=focal_metrics) + return out + + +async def _permitted_sources_for_restricted_user( + session: AsyncSession, + actor: Any, + graph: Mapping[str, Any] | None, +) -> set[str]: + """普通使用者允许用 source_health 探查的数据源名集合：本轮图显式绑定的数据源 + 已授权口径用到的底层数据源。""" + from app.apps.access import usable_calibers + from app.db.models import Workflow, WorkflowVersion + from app.engine.launch import tool_bindings + from app.engine.schema import GraphSpec + from app.tools.datasource import source_of_tool + + allowed: set[str] = set() + if isinstance(graph, dict): + try: + spec = GraphSpec.model_validate(graph) + for _who, tname in tool_bindings(spec): + src = source_of_tool(tname) + if src: + allowed.add(name_key(src)) + except ValueError: + pass + + usable = await usable_calibers(session, actor) + for (wf_id, ver) in usable: + snap = ( + await session.execute( + select(WorkflowVersion).where( + WorkflowVersion.workflow_id == wf_id, + WorkflowVersion.version == int(ver), + ) + ) + ).scalar_one_or_none() + wf = await session.get(Workflow, wf_id) + g = (snap.graph if snap is not None else (wf.graph if wf is not None else None)) or {} + idx = workflow_ontology_index(workflow_id=wf_id, version=int(ver), graph=g) + for card in idx["cards"]: + for s in card["sources"]: + allowed.add(name_key(s)) + return allowed + + +@register( + name=SOURCE_HEALTH, + category="数据", + effect="read", + description=( + "快速核查指定数据源或已发布口径底层表的健康度、最新业务日期覆盖与待处理结构/数据漂移（断更 date_stalled、" + "断档 date_gap、行数骤降 volume_drop、新编码 codes_new 等），把结果存为可引用的健康度工件（H 编号）。只读。" + ), + args_schema=SourceHealthArgs, + guide=ONTOLOGY_GUIDE, +) +async def source_health_tool(ctx: ToolContext, **kwargs: Any) -> dict[str, Any]: + args = SourceHealthArgs(**kwargs) + if not (args.source_name or "").strip() and not (args.workflow or "").strip(): + raise ValueError("source_health 至少需要提供 source_name 或 workflow 之一") + + from app.apps.access import usable_calibers + from app.db.base import SessionLocal + from app.db.models import Workflow + from app.ledger.querycore import NOT_AUTHORIZED, QueryCoreRefused, _workflow_id + + async with SessionLocal() as session: + actor, is_restricted, graph = await _resolve_actor_and_restriction(session, ctx) + wf_id: str | None = None + pub_ver: int | None = None + + if (args.workflow or "").strip(): + try: + wf_id = await _workflow_id(session, str(args.workflow).strip()) + except QueryCoreRefused as exc: + raise ValueError(f"{exc.message}（{exc.code}）") from None + wf = await session.get(Workflow, wf_id) + pub_ver = int(wf.published_version) if wf is not None and wf.published_version is not None else None + if is_restricted: + usable = await usable_calibers(session, actor) + if pub_ver is None or (wf_id, pub_ver) not in usable: + raise ValueError(f"没有权限探查该口径关联的数据源健康度（{NOT_AUTHORIZED}）") + + if is_restricted and (args.source_name or "").strip(): + allowed_sources = await _permitted_sources_for_restricted_user(session, actor, graph) + if name_key(str(args.source_name).strip()) not in allowed_sources: + raise ValueError(f"没有权限探查数据源「{args.source_name}」的健康度（{NOT_AUTHORIZED}）") + + result = await sh.check_source_health( + session, + source_name=str(args.source_name).strip() if args.source_name else None, + tables=args.tables, + workflow_id=wf_id, + version=pub_ver, + card_id=args.card, + metric_id=args.metric_id, + start=args.start, + end=args.end, + restricted=is_restricted, + live_probe=True, + ) + + run_id = str(getattr(ctx, "run_id", "") or "").strip() or None + node_id = str(getattr(ctx, "node_id", "") or "").strip() or None + artifact = await sh.store_source_health(result, run_id=run_id, node_id=node_id) + return { + "tool": SOURCE_HEALTH, + "kind": sh.ARTIFACT_KIND, + "artifact": artifact, + "verdict": result.get("verdict"), + "phrase": result.get("phrase"), + "result": result, + } + + +__all__ = [ + "ONTOLOGY_GUIDE", + "ONTOLOGY_NEIGHBORS", + "ONTOLOGY_TOOLS", + "OntologyNeighborsArgs", + "SOURCE_HEALTH", + "SourceHealthArgs", + "align_caliber_inputs", + "ontology_neighbors", + "source_health_tool", +] diff --git a/backend/app/tools/registry.py b/backend/app/tools/registry.py index 08d8c0c8..1e353571 100644 --- a/backend/app/tools/registry.py +++ b/backend/app/tools/registry.py @@ -156,6 +156,7 @@ def register( def all_specs() -> dict[str, ToolSpec]: import app.tools.builtin # noqa: F401 触发内置工具注册 import app.ledger.querycore # noqa: F401 触发 caliber_query（K10 的父 Run 工具入口）注册 + import app.tools.ontology # noqa: F401 触发 ontology_neighbors / source_health 注册 return dict(_REGISTRY) diff --git a/backend/tests/fixtures/acceptance/multi_source_ontology.py b/backend/tests/fixtures/acceptance/multi_source_ontology.py new file mode 100644 index 00000000..1a53d341 --- /dev/null +++ b/backend/tests/fixtures/acceptance/multi_source_ontology.py @@ -0,0 +1,693 @@ +"""跨源指标本体穿透验收与回归评测用的三源虚构制造库（与任何真实业务无关）。 + + python -m tests.fixtures.acceptance.multi_source_ontology <输出目录> [--today YYYY-MM-DD] + +生成「今天」之前 24 天（截止昨天）的三份独立 SQLite 业务库，随机数种子固定：同一个「今天」生成的三份库逐字节一致。 +三个数据源使用异构物理列名与编码格式，只能经标准实体 `production_line`（产线）对齐： + +1. `ops_shift`（班次报工库）： + - `shift_output(id, shift_date, line_code, actual_qty, good_qty, scrap_qty, run_hours, operator_phone)` + 其中 `line_code` 取值 `L1..L4`，`operator_phone` 为遮罩敏感列。 + - `energy_readings(id, shift_date, line_code, meter_kwh, settled_mwh)` + 用于核对跨量纲换算（千瓦时 vs 兆瓦时，`unit_factor = 1000`）的对账场景。 +2. `equip_master`（设备与停机主数据库）： + - `line_specs(equip_line_id, line_name, rated_hourly_qty, updated_day)` + 其中 `equip_line_id` 取值 `EQ-L1..EQ-L4`。 + - `downtime_events(id, event_date, equip_line_id, net_run_ratio, unplanned_min, fault_category)` + 记录各设备产线每日有效运转率与非计划停机分钟数。 +3. `quality_scrap`（质量报废台账库）： + - `scrap_ledger(id, inspect_date, workcenter_code, scrap_units, defect_family, inspector_id)` + 其中 `workcenter_code` 取值 `WC-01..WC-04`，`inspector_id` 为遮罩敏感列。 + +埋入的确定性效应（互不干扰）： +- **S1（契约越界 → 跨库主数据过期失配）**：`ops_shift` 中 `L3` 昨日实际每小时产量 `108` 件（8 小时产 864 件），但 `equip_master.line_specs` + 中 `EQ-L3` 的 `rated_hourly_qty` 仍停留在旧版 `80` 件/小时，导致负荷率算得 `135%`（违反 `[0, 100]` 契约上界）。 +- **S2a（跨源对账断更 → 对侧源停更早停）**：`quality_scrap.scrap_ledger` 截止于「今天 - 3 天」（最近两天断更，0 行），而 `ops_shift` + 照常落库；健康检查应识别 `date_stalled` 并在阶段一停止，不误判为业务报废清零。 +- **S2b（跨源量纲与超差对账）**：`ops_shift.energy_readings` 昨日 `L1..L3` 按 `1000 kWh = 1 MWh` 完全对齐，`L4` 抄表 `14800 kWh` + 但结算导入记为 `12.5 MWh`（换算后相差 `+2300 kWh`，`+18.4%`），`reconcile` 应判 `diverged`。 +- **S3（统计异动 → 切片定位 `L2` → 跨源设备停机同步异动 `EQ-L2`）**：昨日 `L2` 合格产量从常态约 `900` 跌至 `612`（`-32%`）， + 穿透标准实体 `L2 -> EQ-L2` 后，`equip_master.downtime_events` 中 `EQ-L2` 非计划停机从常态 `11..14` 分钟飙升至 `96` 分钟， + `cochange` 判 `co_anomalous`（仅陈述同期同步异动，不下因果断言）。 +- **对照 1（常规波动不误报）**：`L1` 过去 24 天合格产量在 `892..916` 窄幅波动，昨日 `898` 相对前日 `908` 属常规波动（`within_band`）。 +- **对照 2（主指标异动但上游正常 → `target_normal`）**：`今天 - 3 天` 时 `EQ-L4` 有效运转率从 `0.90` 跌至 `0.73`，但同期非计划停机 + 仍为常态 `12` 分钟，`cochange` 判 `target_normal`，不强行归咎于设备非计划停机。 +""" +from __future__ import annotations + +import argparse +import random +import sqlite3 +from dataclasses import dataclass +from datetime import date, timedelta +from pathlib import Path +from typing import Any + +DAYS = 24 +SEED = 20261022 + +SOURCE_OPS = "ops_shift" +SOURCE_EQUIP = "equip_master" +SOURCE_QUALITY = "quality_scrap" +ALL_SOURCES = (SOURCE_OPS, SOURCE_EQUIP, SOURCE_QUALITY) + +LINES: tuple[tuple[str, str, str, str], ...] = ( + ("L1", "EQ-L1", "WC-01", "星梭一线"), + ("L2", "EQ-L2", "WC-02", "星梭二线"), + ("L3", "EQ-L3", "WC-03", "星梭三线"), + ("L4", "EQ-L4", "WC-04", "星梭四线"), +) + +OPS_SCHEMA = """ +CREATE TABLE shift_output ( + id INTEGER PRIMARY KEY, + shift_date DATE NOT NULL, + line_code TEXT NOT NULL, + actual_qty INTEGER NOT NULL, + good_qty INTEGER NOT NULL, + scrap_qty INTEGER NOT NULL, + run_hours REAL NOT NULL, + operator_phone TEXT NOT NULL +); +CREATE TABLE energy_readings ( + id INTEGER PRIMARY KEY, + shift_date DATE NOT NULL, + line_code TEXT NOT NULL, + meter_kwh REAL NOT NULL, + settled_mwh REAL NOT NULL +); +""" + +EQUIP_SCHEMA = """ +CREATE TABLE line_specs ( + equip_line_id TEXT PRIMARY KEY, + line_name TEXT NOT NULL, + rated_hourly_qty REAL NOT NULL, + updated_day DATE NOT NULL +); +CREATE TABLE downtime_events ( + id INTEGER PRIMARY KEY, + event_date DATE NOT NULL, + equip_line_id TEXT NOT NULL, + net_run_ratio REAL NOT NULL, + unplanned_min REAL NOT NULL, + fault_category TEXT NOT NULL +); +""" + +QUALITY_SCHEMA = """ +CREATE TABLE scrap_ledger ( + id INTEGER PRIMARY KEY, + inspect_date DATE NOT NULL, + workcenter_code TEXT NOT NULL, + scrap_units INTEGER NOT NULL, + defect_family TEXT NOT NULL, + inspector_id TEXT NOT NULL +); +""" + + +@dataclass(frozen=True) +class BuiltDatabases: + today: date + ops_path: Path + equip_path: Path + quality_path: Path + + +def build(out_dir: Path | str, today: date | None = None) -> BuiltDatabases: + """在 out_dir 下生成 ops_shift.db、equip_master.db、quality_scrap.db（已存在则覆盖）。""" + root = Path(out_dir) + root.mkdir(parents=True, exist_ok=True) + today = today or date(2026, 10, 22) + rng = random.Random(SEED) + + ops_path = root / f"{SOURCE_OPS}.db" + equip_path = root / f"{SOURCE_EQUIP}.db" + quality_path = root / f"{SOURCE_QUALITY}.db" + for p in (ops_path, equip_path, quality_path): + if p.exists(): + p.unlink() + + yesterday = today - timedelta(days=1) + control2_day = yesterday - timedelta(days=2) + stalled_after = yesterday - timedelta(days=2) # quality_scrap 停在 yesterday - 2 天，最近两天缺失 + days = [today - timedelta(days=offset) for offset in range(DAYS, 0, -1)] + + shift_rows: list[tuple[Any, ...]] = [] + energy_rows: list[tuple[Any, ...]] = [] + downtime_rows: list[tuple[Any, ...]] = [] + scrap_rows: list[tuple[Any, ...]] = [] + + base_good = {"L1": 904, "L2": 900, "L3": 844, "L4": 888} + base_scrap = {"L1": 18, "L2": 20, "L3": 19, "L4": 17} + phones = { + "L1": "138-0010-0101", + "L2": "138-0010-0102", + "L3": "138-0010-0103", + "L4": "138-0010-0104", + } + inspectors = { + "WC-01": "INSP-9001", + "WC-02": "INSP-9002", + "WC-03": "INSP-9003", + "WC-04": "INSP-9004", + } + + shift_id = 1 + energy_id = 1 + downtime_id = 1 + scrap_id = 1 + + for idx, d in enumerate(days): + ds = d.isoformat() + wave = (-1 if idx % 2 else 1) * (4 + (idx % 3) * 2) # 确定性小幅波动 ±4..±8 + for line_code, equip_id, wc_code, _ in LINES: + noise = rng.choice((-3, -1, 1, 3)) + good = base_good[line_code] + wave + noise + scrap = base_scrap[line_code] + (idx % 3) - 1 + run_hours = 8.0 + net_ratio = round(0.905 + ((idx % 4) - 1.5) * 0.006 + noise * 0.001, 4) + unplanned = float(12 + (-1 if idx % 2 else 1) * (1 + (idx % 3)) + (0.5 if noise > 0 else -0.5)) + fault_cat = "routine_check" + + # 对照 1：L1 末尾两日固定为平缓常规波动（前日 908 → 昨日 898，-1.1%） + if line_code == "L1" and d == yesterday - timedelta(days=1): + good = 908 + elif line_code == "L1" and d == yesterday: + good = 898 + + # S1：L3 昨日实际产率 108 件/小时（864 件 / 8 小时），对比 equip_master 过期额定 80 件/小时 = 135% + if line_code == "L3" and d == yesterday: + good = 844 + scrap = 20 + + # S3：L2 昨日产率显著下跌，同时 EQ-L2 非计划停机飙升至 96 分钟（统计伴随异动） + if line_code == "L2" and d == yesterday: + good = 612 + net_ratio = 0.6320 + unplanned = 96.0 + fault_cat = "feeder_jam" + + # 对照 2：EQ-L4 在 control2_day 有效运转率明显下跌，但非计划停机保持常态 12 分钟（target_normal） + if line_code == "L4" and d == control2_day: + good = 720 + net_ratio = 0.7280 + unplanned = 12.0 + fault_cat = "routine_check" + + actual = good + scrap + shift_rows.append((shift_id, ds, line_code, actual, good, scrap, run_hours, phones[line_code])) + shift_id += 1 + + # 能耗表：昨日 L4 出现结算导入偏差（14800 kWh vs 12.5 MWh），其余对齐 + kwh = round(actual * 14.0, 1) + mwh = round(kwh / 1000.0, 4) + if line_code == "L4" and d == yesterday: + kwh = 14800.0 + mwh = 12.5 + elif d == yesterday: + kwh = 12000.0 + (LINES.index((line_code, equip_id, wc_code, _)) * 500.0) + mwh = round(kwh / 1000.0, 4) + energy_rows.append((energy_id, ds, line_code, kwh, mwh)) + energy_id += 1 + + downtime_rows.append((downtime_id, ds, equip_id, net_ratio, unplanned, fault_cat)) + downtime_id += 1 + + # S2a：quality_scrap 在 stalled_after 之后断更（最近两天 0 行） + if d <= stalled_after: + scrap_rows.append((scrap_id, ds, wc_code, scrap, "surface_burr", inspectors[wc_code])) + scrap_id += 1 + + specs_rows = [ + ("EQ-L1", "星梭一线", 120.0, (today - timedelta(days=30)).isoformat()), + ("EQ-L2", "星梭二线", 120.0, (today - timedelta(days=30)).isoformat()), + # S1 根因：EQ-L3 技改提速后主数据未更新，仍为 80 件/小时 + ("EQ-L3", "星梭三线", 80.0, "2026-06-01"), + ("EQ-L4", "星梭四线", 120.0, (today - timedelta(days=30)).isoformat()), + ] + + ops_db = sqlite3.connect(ops_path) + ops_db.executescript(OPS_SCHEMA) + ops_db.executemany("INSERT INTO shift_output VALUES (?, ?, ?, ?, ?, ?, ?, ?)", shift_rows) + ops_db.executemany("INSERT INTO energy_readings VALUES (?, ?, ?, ?, ?)", energy_rows) + ops_db.commit() + ops_db.close() + + equip_db = sqlite3.connect(equip_path) + equip_db.executescript(EQUIP_SCHEMA) + equip_db.executemany("INSERT INTO line_specs VALUES (?, ?, ?, ?)", specs_rows) + equip_db.executemany("INSERT INTO downtime_events VALUES (?, ?, ?, ?, ?, ?)", downtime_rows) + equip_db.commit() + equip_db.close() + + quality_db = sqlite3.connect(quality_path) + quality_db.executescript(QUALITY_SCHEMA) + quality_db.executemany("INSERT INTO scrap_ledger VALUES (?, ?, ?, ?, ?, ?)", scrap_rows) + quality_db.commit() + quality_db.close() + + return BuiltDatabases(today=today, ops_path=ops_path, equip_path=equip_path, quality_path=quality_path) + + +async def seed_ontology_fixture( + tmp_path: Path, + scratch_source: Any, + *, + today: date | None = None, +) -> dict[str, Any]: + """登记三源 SQLite、标准实体 `production_line`、断更变更记录及三条跨源已发布口径工作流。""" + from app.api.workflows import commit_publish + from app.data import source_changes as sc + from app.data.entities import create_entity, entity_to_dict + from app.db.base import SessionLocal + from app.db.models import DataSource, Workflow, WorkflowVersion + + built = build(tmp_path, today=today) + ops_id = await scratch_source( + SOURCE_OPS, built.ops_path, options={"mask_columns": ["operator_phone"]}, schema=True + ) + equip_id = await scratch_source(SOURCE_EQUIP, built.equip_path, schema=True) + quality_id = await scratch_source( + SOURCE_QUALITY, built.quality_path, options={"mask_columns": ["inspector_id"]}, schema=True + ) + + stalled_day = (built.today - timedelta(days=3)).isoformat() + async with SessionLocal() as session: + quality_src = await session.get(DataSource, quality_id) + await sc.record( + session, + quality_src, + sc.NewChange( + kind=sc.DATE_STALLED, + table="scrap_ledger", + column="inspect_date", + severity="action", + before={"max_date": stalled_day}, + after={"max_date": stalled_day, "missing_days": 2}, + key=stalled_day, + ), + sc.Detected(job_kind="profile", trigger="manual", actor="fixture_builder"), + ) + await session.commit() + + async with SessionLocal() as session: + # 建三条分属不同数据源的口径工作流：先落基础工作流以拿到固定 workflow_id，再互相挂本体边 + wf_equip = Workflow(name="设备主数据与停机口径", description="设备额定产能与非计划停机口径", graph={"nodes": [], "edges": []}) + wf_quality = Workflow(name="质量报废台账口径", description="质量检验报废台账口径", graph={"nodes": [], "edges": []}) + wf_ops = Workflow(name="班次产率与负荷率口径", description="班次合格产率、负荷率与报工报废口径", graph={"nodes": [], "edges": []}) + session.add_all([wf_equip, wf_quality, wf_ops]) + await session.flush() + + equip_graph = { + "nodes": [ + { + "id": "in_eq", + "type": "input", + "position": {"x": 0, "y": 0}, + "data": { + "label": "入口", + "config": { + "fields": [ + {"name": "equip_line", "label": "设备产线", "type": "text", "default": "EQ-L3"}, + {"name": "stat_day", "label": "统计日期", "type": "text", "default": (built.today - timedelta(days=1)).isoformat()}, + ] + }, + }, + }, + { + "id": "q_spec", + "type": "tool", + "position": {"x": 220, "y": 0}, + "data": { + "label": "查额定产能", + "config": { + "tool": f"db_query__{SOURCE_EQUIP}", + "args": { + "sql": "SELECT equip_line_id, rated_hourly_qty FROM line_specs WHERE equip_line_id = '{{ input.equip_line }}'", + }, + }, + }, + }, + { + "id": "q_down", + "type": "tool", + "position": {"x": 220, "y": 140}, + "data": { + "label": "查停机时长", + "config": { + "tool": f"db_query__{SOURCE_EQUIP}", + "args": { + "sql": "SELECT event_date, unplanned_min FROM downtime_events WHERE equip_line_id = '{{ input.equip_line }}' AND event_date = '{{ input.stat_day }}'", + }, + }, + }, + }, + { + "id": "m_equip", + "type": "metrics", + "position": {"x": 480, "y": 60}, + "data": { + "label": "设备指标卡", + "config": { + "caliber": "设备主数据与非计划停机日口径", + "metrics": [ + { + "id": "rated_hourly_qty", + "name": "额定每小时产量", + "expression": "cell(nodes.q_spec, 0, 'rated_hourly_qty')", + "unit": "件/小时", + "semantic": { + "entities": [{"name": "产线", "key": "equip_line_id", "entity_key": "production_line"}] + }, + }, + { + "id": "unplanned_downtime_min", + "name": "非计划停机时长", + "expression": "cell(nodes.q_down, 0, 'unplanned_min')", + "unit": "分钟", + "semantic": { + "grain": "day", + "time": {"column": "event_date", "timezone": "Asia/Shanghai"}, + "entities": [{"name": "产线", "key": "equip_line_id", "entity_key": "production_line"}], + }, + }, + ], + }, + }, + }, + {"id": "out_eq", "type": "output", "position": {"x": 720, "y": 60}, "data": {"label": "出口", "config": {}}}, + ], + "edges": [ + {"source": "in_eq", "target": "q_spec"}, + {"source": "in_eq", "target": "q_down"}, + {"source": "q_spec", "target": "m_equip"}, + {"source": "q_down", "target": "m_equip"}, + {"source": "m_equip", "target": "out_eq"}, + ], + } + + quality_graph = { + "nodes": [ + { + "id": "in_q", + "type": "input", + "position": {"x": 0, "y": 0}, + "data": { + "label": "入口", + "config": { + "fields": [ + {"name": "workcenter", "label": "工作中心", "type": "text", "default": "WC-02"}, + {"name": "inspect_day", "label": "检验日期", "type": "text", "default": (built.today - timedelta(days=1)).isoformat()}, + ] + }, + }, + }, + { + "id": "q_scrap", + "type": "tool", + "position": {"x": 240, "y": 0}, + "data": { + "label": "查质量报废", + "config": { + "tool": f"db_query__{SOURCE_QUALITY}", + "args": { + "sql": "SELECT COALESCE(SUM(scrap_units), 0) AS scrap_units FROM scrap_ledger WHERE workcenter_code = '{{ input.workcenter }}' AND inspect_date = '{{ input.inspect_day }}'", + }, + }, + }, + }, + { + "id": "m_quality", + "type": "metrics", + "position": {"x": 480, "y": 0}, + "data": { + "label": "质量报废卡", + "config": { + "caliber": "质量检验台账日报废口径", + "metrics": [ + { + "id": "ledger_scrap_units", + "name": "台账报废件数", + "expression": "cell(nodes.q_scrap, 0, 'scrap_units')", + "unit": "件", + "semantic": { + "grain": "day", + "time": {"column": "inspect_date", "timezone": "Asia/Shanghai"}, + "entities": [{"name": "产线", "key": "workcenter_code", "entity_key": "production_line"}], + }, + } + ], + }, + }, + }, + {"id": "out_q", "type": "output", "position": {"x": 720, "y": 0}, "data": {"label": "出口", "config": {}}}, + ], + "edges": [ + {"source": "in_q", "target": "q_scrap"}, + {"source": "q_scrap", "target": "m_quality"}, + {"source": "m_quality", "target": "out_q"}, + ], + } + + ops_graph = { + "nodes": [ + { + "id": "in_ops", + "type": "input", + "position": {"x": 0, "y": 0}, + "data": { + "label": "入口", + "config": { + "fields": [ + {"name": "line_code", "label": "产线编码", "type": "text", "default": "L3"}, + {"name": "shift_day", "label": "班次日期", "type": "text", "default": (built.today - timedelta(days=1)).isoformat()}, + ] + }, + }, + }, + { + "id": "q_ops", + "type": "tool", + "position": {"x": 240, "y": 0}, + "data": { + "label": "查班次产出", + "config": { + "tool": f"db_query__{SOURCE_OPS}", + "args": { + "sql": "SELECT actual_qty, good_qty, scrap_qty, run_hours FROM shift_output WHERE line_code = '{{ input.line_code }}' AND shift_date = '{{ input.shift_day }}'", + }, + }, + }, + }, + { + "id": "m_ops", + "type": "metrics", + "position": {"x": 500, "y": 0}, + "data": { + "label": "班次指标卡", + "config": { + "caliber": "班次负荷率、合格产量与报工报废口径", + "metrics": [ + { + "id": "load_rate_pct", + "name": "产线负荷率", + "expression": "round((cell(nodes.q_ops, 0, 'actual_qty') / cell(nodes.q_ops, 0, 'run_hours')) / 80.0 * 100, 2)", + "unit": "%", + "contract": {"minimum": 0, "maximum": 100, "unit": "%"}, + "semantic": { + "grain": "day", + "time": {"column": "shift_date", "timezone": "Asia/Shanghai"}, + "entities": [{"name": "产线", "key": "line_code", "entity_key": "production_line"}], + }, + "links": [ + { + "kind": "factor", + "target": { + "workflow_id": wf_equip.id, + "card_id": "m_equip", + "metric_id": "rated_hourly_qty", + }, + "role": "denominator", + "entity_key": "production_line", + "status": "confirmed", + "rationale": "负荷率分母取设备主数据的额定每小时产量", + }, + { + "kind": "upstream_driver", + "target": { + "workflow_id": wf_equip.id, + "card_id": "m_equip", + "metric_id": "unplanned_downtime_min", + }, + "entity_key": "production_line", + "status": "proposed", + "rationale": "候选未确认边：验证运行期不会自动沿 proposed 边穿透", + }, + ], + }, + { + "id": "good_output_qty", + "name": "班次合格产量", + "expression": "cell(nodes.q_ops, 0, 'good_qty')", + "unit": "件", + "semantic": { + "grain": "day", + "time": {"column": "shift_date", "timezone": "Asia/Shanghai"}, + "entities": [{"name": "产线", "key": "line_code", "entity_key": "production_line"}], + }, + "links": [ + { + "kind": "upstream_driver", + "target": { + "workflow_id": wf_equip.id, + "card_id": "m_equip", + "metric_id": "unplanned_downtime_min", + }, + "entity_key": "production_line", + "lag_periods": 0, + "status": "confirmed", + "rationale": "设备非计划停机时长升高会同步压缩班次有效产出时间", + } + ], + }, + { + "id": "shift_scrap_qty", + "name": "报工报废件数", + "expression": "cell(nodes.q_ops, 0, 'scrap_qty')", + "unit": "件", + "semantic": { + "grain": "day", + "time": {"column": "shift_date", "timezone": "Asia/Shanghai"}, + "entities": [{"name": "产线", "key": "line_code", "entity_key": "production_line"}], + }, + "links": [ + { + "kind": "reconciles_with", + "target": { + "workflow_id": wf_quality.id, + "card_id": "m_quality", + "metric_id": "ledger_scrap_units", + }, + "entity_key": "production_line", + "rel_tol": 0.02, + "status": "confirmed", + "rationale": "班次报工报废数应与质量检验台账日报废件数在 2% 容差内对齐", + } + ], + }, + ], + }, + }, + }, + {"id": "out_ops", "type": "output", "position": {"x": 740, "y": 0}, "data": {"label": "出口", "config": {}}}, + ], + "edges": [ + {"source": "in_ops", "target": "q_ops"}, + {"source": "q_ops", "target": "m_ops"}, + {"source": "m_ops", "target": "out_ops"}, + ], + } + + snapshots: list[tuple[Workflow, WorkflowVersion]] = [] + for wf, graph in ((wf_equip, equip_graph), (wf_quality, quality_graph), (wf_ops, ops_graph)): + wf.graph = graph + wf.version = 1 + snap = WorkflowVersion(workflow_id=wf.id, version=1, graph=graph, note="初始版本") + session.add(snap) + snapshots.append((wf, snap)) + # 先提交未写锁的草稿行，再逐一走 commit_publish（避免 SQLite 单写锁下嵌套会话写工件抢锁） + await session.commit() + + for wf, snap in snapshots: + await session.refresh(wf) + await session.refresh(snap) + await commit_publish(session, wf, snap, "published", "fixture_builder") + + entity_row = await create_entity( + session, + { + "key": "production_line", + "label": "产线", + "description": "星梭示范装配厂标准产线实体，桥接班次报工、设备主数据与质量报废台账三套异构编码", + "aliases": ["装配产线", "车间产线", "工作中心"], + "bindings": [ + { + "source_name": SOURCE_OPS, + "table_name": "shift_output", + "column_name": "line_code", + "status": "confirmed", + }, + { + "source_name": SOURCE_OPS, + "table_name": "energy_readings", + "column_name": "line_code", + "status": "confirmed", + }, + { + "source_name": SOURCE_EQUIP, + "table_name": "line_specs", + "column_name": "equip_line_id", + "status": "confirmed", + }, + { + "source_name": SOURCE_EQUIP, + "table_name": "downtime_events", + "column_name": "equip_line_id", + "status": "confirmed", + }, + { + "source_name": SOURCE_QUALITY, + "table_name": "scrap_ledger", + "column_name": "workcenter_code", + "status": "confirmed", + }, + { + "source_name": SOURCE_QUALITY, + "table_name": "unverified_archive", + "column_name": "legacy_line", + "status": "proposed", + }, + ], + "caliber_params": [ + {"workflow_id": wf_ops.id, "param_name": "line_code", "status": "confirmed"}, + {"workflow_id": wf_equip.id, "param_name": "equip_line", "status": "confirmed"}, + {"workflow_id": wf_quality.id, "param_name": "workcenter", "status": "confirmed"}, + ], + "members": [ + { + "value": code, + "label": name, + "aliases": [code, equip_id, wc_code], + "source_codes": {SOURCE_OPS: code, SOURCE_EQUIP: equip_id, SOURCE_QUALITY: wc_code}, + } + for code, equip_id, wc_code, name in LINES + ], + }, + actor="fixture_builder", + ) + entity_dict = entity_to_dict(entity_row) + await session.commit() + + return { + "today": built.today, + "paths": {"ops": built.ops_path, "equip": built.equip_path, "quality": built.quality_path}, + "source_ids": {SOURCE_OPS: ops_id, SOURCE_EQUIP: equip_id, SOURCE_QUALITY: quality_id}, + "workflow_ids": {"ops": wf_ops.id, "equip": wf_equip.id, "quality": wf_quality.id}, + "entity_id": entity_dict["id"], + "entity": entity_dict, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="生成跨源指标本体穿透用的三源虚构制造库") + parser.add_argument("out_dir") + parser.add_argument("--today", type=date.fromisoformat, default=None) + args = parser.parse_args() + built = build(args.out_dir, args.today) + print(f"{built.ops_path}\n{built.equip_path}\n{built.quality_path}") + + +if __name__ == "__main__": + main() diff --git a/backend/tests/fixtures/analysis_ops.py b/backend/tests/fixtures/analysis_ops.py index 8b787901..0b1f3540 100644 --- a/backend/tests/fixtures/analysis_ops.py +++ b/backend/tests/fixtures/analysis_ops.py @@ -13,7 +13,7 @@ from app.analysis.testing import fake_result #: 每个算子缺省给的判定（fake_result 有现成判定语的那几个） VERDICT = {"decompose": "driver", "baseline": "outside_band", "trend": "rising", "differ": "no_evidence", - "comparable": "comparable"} + "comparable": "comparable", "reconcile": "diverged", "cochange": "co_anomalous"} class FakeOps: diff --git a/backend/tests/fixtures/eval/ontology_suite.json b/backend/tests/fixtures/eval/ontology_suite.json new file mode 100644 index 00000000..c1fa68f1 --- /dev/null +++ b/backend/tests/fixtures/eval/ontology_suite.json @@ -0,0 +1,191 @@ +{ + "format": "agentlab-eval-suite", + "version": 1, + "exported_at": "2026-10-22T00:00:00+00:00", + "suite": { + "name": "跨源指标本体穿透题集", + "description": "配 tests/fixtures/acceptance/multi_source_ontology.py 的三源虚构制造库（ops_shift、equip_master、quality_scrap）与标准实体 production_line：覆盖契约越界穿透主数据失配（S1）、对账源断更早停（S2a）、量纲换算与超差对账（S2b）、切片定位加跨源设备停机伴随异动（S3），以及常规波动不误报（对照 1）、主指标异动但上游正常不乱归因（对照 2）。全部期望由 SQL 探针与平台确定性算子重算，零人工填数", + "timezone": "Asia/Shanghai", + "budget": {} + }, + "cases": [ + { + "seq": 1, + "kind": "ask", + "question": "为什么昨天 L3 产线的负荷率超过了 100%？请沿指标本体排查是不是关联主数据出了问题。", + "datasources": ["ops_shift", "equip_master"], + "tags": ["本体穿透", "S1", "契约越界", "主数据失配"], + "gold": { + "kind": "sql", + "datasource": "equip_master", + "sql": "SELECT equip_line_id AS 设备产线, rated_hourly_qty AS 额定每小时产量 FROM line_specs WHERE equip_line_id = 'EQ-L3'" + }, + "expectations": [ + { + "id": "stale_spec", + "kind": "scalar", + "column": "额定每小时产量", + "row": 0, + "label": "穿透定位到 EQ-L3 在设备主数据中的过期额定每小时产量" + }, + { + "id": "inv_s1", + "kind": "investigation", + "status": "complete", + "minimum_depth": 1, + "min_ontology_hops": 1, + "diagnosis_stage": "ontology_penetration", + "label": "完成至少 1 跳跨源本体穿透定位因子分母" + } + ], + "enabled": true + }, + { + "seq": 2, + "kind": "ask", + "question": "对比班次报工和质量报废台账，昨天各产线的报废数为什么对不上？是业务真变了还是数据源异常？", + "datasources": ["ops_shift", "quality_scrap"], + "tags": ["本体穿透", "S2a", "跨源对账", "数据源断更"], + "gold": { + "kind": "sql", + "datasource": "quality_scrap", + "sql": "SELECT workcenter_code AS 工作中心, scrap_units AS 报废件数 FROM scrap_ledger WHERE inspect_date = '{{ schedule.yesterday }}'" + }, + "expectations": [ + { + "id": "stalled_zero_rows", + "kind": "none", + "label": "质量报废台账昨天断更无记录（0 行）" + }, + { + "id": "inv_s2a", + "kind": "investigation", + "status": "complete", + "require_source_health": true, + "diagnosis_stage": "health_and_contract", + "label": "先核查关联数据源健康度并在阶段一识别断更停止" + } + ], + "enabled": true + }, + { + "seq": 3, + "kind": "ask", + "question": "核对昨天各产线能耗的抄表千瓦时与结算兆瓦时（1 兆瓦时 = 1000 千瓦时），换算后两侧对账是否一致？", + "datasources": ["ops_shift"], + "tags": ["本体穿透", "S2b", "跨源对账", "量纲换算"], + "gold": { + "kind": "sql", + "datasource": "ops_shift", + "sql": "SELECT line_code AS 产线, meter_kwh AS 抄表千瓦时, settled_mwh AS 结算兆瓦时 FROM energy_readings WHERE shift_date = '{{ schedule.yesterday }}' ORDER BY line_code" + }, + "expectations": [ + { + "id": "reconcile_energy", + "kind": "verdict", + "op": "reconcile", + "params": { + "by": ["产线"], + "measure": "抄表千瓦时", + "target_measure": "结算兆瓦时", + "unit_factor": 1000, + "label": "产线能耗" + }, + "label": "按 1000 倍量纲换算后判定能耗对账存在显著超差" + } + ], + "enabled": true + }, + { + "seq": 4, + "kind": "ask", + "question": "昨天产线综合合格产量明显下滑，请先拆解定位哪条产线拖累，再沿本体穿透上游设备数据源检验是否有同步异动。", + "datasources": ["ops_shift", "equip_master"], + "tags": ["本体穿透", "S3", "切片定位", "伴随异动"], + "gold": { + "kind": "sql", + "datasource": "equip_master", + "sql": "SELECT event_date AS 日期, net_run_ratio AS 有效运转率, unplanned_min AS 非计划停机分钟 FROM downtime_events WHERE equip_line_id = 'EQ-L2' AND event_date BETWEEN date('{{ schedule.yesterday }}', '-19 days') AND '{{ schedule.yesterday }}' ORDER BY event_date" + }, + "expectations": [ + { + "id": "cochange_l2", + "kind": "verdict", + "op": "cochange", + "params": { + "period": "日期", + "measure": "有效运转率", + "driver_measure": "非计划停机分钟", + "target": "{{ schedule.yesterday }}", + "label": "有效运转率" + }, + "label": "EQ-L2 有效运转率下滑与非计划停机分钟飙升呈同期伴随异动" + }, + { + "id": "inv_s3", + "kind": "investigation", + "status": "complete", + "minimum_depth": 1, + "min_ontology_hops": 1, + "require_source_health": true, + "diagnosis_stage": "ontology_penetration", + "label": "经健康核查后完成跨源本体穿透检验" + } + ], + "enabled": true + }, + { + "seq": 5, + "kind": "ask", + "question": "昨天 L1 产线的合格产量比前一天略少一点，这算异常波动吗？", + "datasources": ["ops_shift"], + "tags": ["本体穿透", "对照", "常规波动"], + "gold": { + "kind": "sql", + "datasource": "ops_shift", + "sql": "SELECT shift_date AS 班次日期, good_qty AS 合格产量 FROM shift_output WHERE line_code = 'L1' AND shift_date BETWEEN date('{{ schedule.yesterday }}', '-19 days') AND '{{ schedule.yesterday }}' ORDER BY shift_date" + }, + "expectations": [ + { + "id": "normal_l1", + "kind": "verdict", + "op": "baseline", + "params": { + "period": "班次日期", + "measure": "合格产量", + "label": "L1合格产量" + }, + "label": "L1 昨日波动处于历史常规波动带内，不误报异常" + } + ], + "enabled": true + }, + { + "seq": 6, + "kind": "ask", + "question": "三天前 L4 产线的有效运转率出现明显下滑，是不是上游设备非计划停机异常引起的？", + "datasources": ["ops_shift", "equip_master"], + "tags": ["本体穿透", "对照", "排除上游"], + "gold": { + "kind": "sql", + "datasource": "equip_master", + "sql": "SELECT event_date AS 日期, net_run_ratio AS 有效运转率, unplanned_min AS 非计划停机分钟 FROM downtime_events WHERE equip_line_id = 'EQ-L4' AND event_date BETWEEN date('{{ schedule.yesterday }}', '-21 days') AND date('{{ schedule.yesterday }}', '-2 days') ORDER BY event_date" + }, + "expectations": [ + { + "id": "upstream_normal_l4", + "kind": "verdict", + "op": "cochange", + "params": { + "period": "日期", + "measure": "有效运转率", + "driver_measure": "非计划停机分钟", + "label": "有效运转率" + }, + "label": "主指标超出常规波动但同期设备非计划停机处于常规波动范围内（target_normal）" + } + ], + "enabled": true + } + ] +} diff --git a/backend/tests/test_entity_agent.py b/backend/tests/test_entity_agent.py new file mode 100644 index 00000000..87df2626 --- /dev/null +++ b/backend/tests/test_entity_agent.py @@ -0,0 +1,823 @@ +"""标准业务实体 Agent 化治理单元测试（E1 + E2）：附件/知识库联动、增量补丁护栏、交互式实体助手 SSE 与后台扫描发现任务。""" +from __future__ import annotations + +import asyncio +import json +import sqlite3 +import uuid +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient +from langchain_core.messages import AIMessage, AIMessageChunk +from sqlalchemy import delete, select + +from app.data import catalog_assistant, entities, entity_assistant, entity_discovery, entity_docs, entity_patch +from app.data.engine import CATALOG_PROFILE_OPTION, MASK_COLUMNS_OPTION +from app.db.base import SessionLocal +from app.db.models import Chunk, Document, Workflow +from app.db.models_ontology import CanonicalEntity, EntityDiscoveryJob +from app.main import app +from tests.fixtures.sources import forget_source, sqlite_source + + +@dataclass +class _Turn: + text: list[str] = field(default_factory=list) + calls: list[tuple[str, dict[str, Any]]] = field(default_factory=list) + usage: tuple[int, int] = (80, 30) + + +class _ScriptedModel: + def __init__(self, turns: list[_Turn] | None = None, *, invoke_reply: str = "") -> None: + self.turns = list(turns or []) + self.invoke_reply = invoke_reply + self.seen: list[list[Any]] = [] + self.tools: list[Any] | None = None + + def bind_tools(self, tools: list[Any], **_: Any) -> "_ScriptedModel": + self.tools = tools + return self + + async def astream(self, messages: list[Any]): + self.seen.append(list(messages)) + turn = self.turns.pop(0) + for piece in turn.text: + yield AIMessageChunk(content=piece) + for i, (name, args) in enumerate(turn.calls): + yield AIMessageChunk( + content="", + tool_call_chunks=[ + { + "name": name, + "args": json.dumps(args, ensure_ascii=False), + "id": f"tc_{len(self.seen)}_{i}", + "index": i, + } + ], + ) + yield AIMessageChunk( + content="", + usage_metadata={ + "input_tokens": turn.usage[0], + "output_tokens": turn.usage[1], + "total_tokens": sum(turn.usage), + }, + response_metadata={"finish_reason": "stop"}, + ) + + async def ainvoke(self, messages: list[Any]) -> AIMessage: + self.seen.append(list(messages)) + return AIMessage(content=self.invoke_reply) + + +async def _make_sqlite_source( + tmp_path: Path, + *, + name: str, + setup_sql: str, + profiling: bool = True, + mask_columns: str = "", +) -> str: + db_file = tmp_path / f"{name}_{uuid.uuid4().hex[:6]}.db" + conn = sqlite3.connect(db_file) + try: + conn.executescript(setup_sql) + conn.commit() + finally: + conn.close() + + options: dict[str, Any] = {CATALOG_PROFILE_OPTION: {"enabled": profiling}} + if mask_columns: + options[MASK_COLUMNS_OPTION] = mask_columns + + return await sqlite_source(name, db_file, options=options, schema=True) + + +@pytest.fixture +async def multi_source_env(tmp_path: Path): + mes_name = f"mes_ag_{uuid.uuid4().hex[:6]}" + erp_name = f"erp_ag_{uuid.uuid4().hex[:6]}" + wf_id = f"wf_ag_{uuid.uuid4().hex[:6]}" + + mes_id = await _make_sqlite_source( + tmp_path, + name=mes_name, + setup_sql=""" + CREATE TABLE dwd_workorder ( + line_code TEXT, + line_name TEXT, + operator_phone TEXT, + good_qty INTEGER + ); + INSERT INTO dwd_workorder VALUES + ('L1', '一号产线', '13900000011', 100), + ('L2', '二号产线', '13900000022', 95); + """, + profiling=True, + mask_columns="operator_phone", + ) + erp_id = await _make_sqlite_source( + tmp_path, + name=erp_name, + setup_sql=""" + CREATE TABLE dwd_settlement ( + workshop_line TEXT, + settled_qty INTEGER + ); + INSERT INTO dwd_settlement VALUES + ('LINE_01', 98), + ('LINE_02', 95); + """, + profiling=True, + ) + + async with SessionLocal() as session: + wf = Workflow( + id=wf_id, + name=f"良率核算_{wf_id}", + version=1, + graph={ + "nodes": [ + { + "id": "in_1", + "type": "input", + "data": {"config": {"fields": [{"name": "p_line", "label": "产线编码", "type": "string"}]}}, + }, + { + "id": "card_yield", + "type": "metrics", + "data": {"config": {"metrics": [{"id": "fpy", "name": "一次良率"}]}}, + }, + ], + "edges": [], + }, + ) + session.add(wf) + await session.commit() + + created_doc_ids: list[str] = [] + try: + yield { + "mes_id": mes_id, + "mes_name": mes_name, + "erp_id": erp_id, + "erp_name": erp_name, + "wf_id": wf_id, + "doc_ids": created_doc_ids, + } + finally: + await forget_source(mes_id) + await forget_source(erp_id) + async with SessionLocal() as session: + await session.execute(delete(CanonicalEntity)) + await session.execute(delete(EntityDiscoveryJob)) + await session.execute(delete(Workflow).where(Workflow.id == wf_id)) + if created_doc_ids: + await session.execute(delete(Chunk).where(Chunk.document_id.in_(created_doc_ids))) + await session.execute(delete(Document).where(Document.id.in_(created_doc_ids))) + await session.commit() + + +async def test_attachment_upload_and_optional_save_to_kb(multi_source_env): + csv_bytes = "标准编码,产线名称,MES编码,ERP编码\nL1,一号产线,L1,LINE_01\nL2,二号产线,L2,LINE_02\n".encode("utf-8") + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + # 1) 不勾选存入知识库：提取表格内容但不创建 Document + r1 = await client.post( + "/api/ontology/entities/attachments", + files={"file": ("line_mapping.csv", csv_bytes, "text/csv")}, + data={"save_to_kb": "false"}, + ) + assert r1.status_code == 201, r1.text + d1 = r1.json() + assert d1["saved_to_kb"] is False + assert d1["document_id"] is None + assert d1["chunk_count"] >= 1 + + # 2) 勾选存入知识库：同步创建 Document 并返回 document_id + r2 = await client.post( + "/api/ontology/entities/attachments", + files={"file": ("line_mapping_kb.csv", csv_bytes, "text/csv")}, + data={"save_to_kb": "true", "collection": "default"}, + ) + assert r2.status_code == 201, r2.text + d2 = r2.json() + assert d2["saved_to_kb"] is True + assert d2["document_id"] + multi_source_env["doc_ids"].append(d2["document_id"]) + + async with SessionLocal() as session: + doc_row = await session.get(Document, d2["document_id"]) + assert doc_row is not None + assert "LINE_01" in doc_row.content + + # 同时传 attachment_id 与对应 document_id 时自动去重为 1 份 + bundles = await entity_docs.load_doc_bundles( + session, + attachment_ids=[d2["attachment_id"]], + document_ids=[d2["document_id"]], + ) + assert len(bundles) == 1 + hits = entity_docs.search_doc_bundles(bundles, query="LINE_01") + assert len(hits) == 1 + assert "文档《line_mapping_kb.csv》" in hits[0]["citation"] + + +async def test_entity_patch_guardrails_masked_columns_and_cas(multi_source_env): + mes = multi_source_env["mes_name"] + erp = multi_source_env["erp_name"] + wf_id = multi_source_env["wf_id"] + + async with SessionLocal() as session: + # 1) 校验不存在的列、不存在的参数与遮罩列枚举阻断 + bad_plan = await entity_patch.plan_entity_patch( + session, + [ + {"op": "upsert_entity", "entity_key": "phone_ent", "label": "作业员号码"}, + { + "op": "upsert_binding", + "entity_key": "phone_ent", + "source_name": mes, + "table_name": "dwd_workorder", + "column_name": "operator_phone", + }, + { + "op": "upsert_member", + "entity_key": "phone_ent", + "value": "P1", + "label": "张三", + "source_codes": {mes: "13900000011"}, + }, + { + "op": "upsert_binding", + "entity_key": "phone_ent", + "source_name": mes, + "table_name": "dwd_workorder", + "column_name": "no_such_col", + }, + { + "op": "upsert_caliber_param", + "entity_key": "phone_ent", + "workflow_id": wf_id, + "param_name": "wrong_param", + }, + ], + mode="confirmed", + ) + assert len(bad_plan.problems) == 3 + assert any("遮罩列" in p for p in bad_plan.problems) + assert any("no_such_col" in p for p in bad_plan.problems) + assert any("wrong_param" in p for p in bad_plan.problems) + + # 2) mode="confirmed" 正常落库，随后测试 expected_versions CAS 乐观锁与 mode="proposed" 隔离 + res = await entity_patch.apply_entity_patch( + session, + [ + { + "op": "upsert_entity", + "entity_key": "prod_line_t", + "label": "产线", + "status": "proposed", # 模型自报状态将被忽略，confirmed 模式下按 confirmed 落库 + }, + { + "op": "upsert_binding", + "entity_key": "prod_line_t", + "source_name": mes, + "table_name": "dwd_workorder", + "column_name": "line_code", + "reason": "使用者说：工单产线列是 line_code", + }, + ], + mode="confirmed", + actor="builder_a", + ) + await session.commit() + assert res["created_entities"] == 1 + ent = res["entities"][0] + assert ent["bindings"][0]["status"] == "confirmed" + + # 将一条 ERP 绑定预先设为 rejected，验证 mode="proposed" 绝不覆盖 confirmed 或复活 rejected + ent_row = await entities.get_entity_by_key(session, "prod_line_t") + assert ent_row is not None + await entities.update_entity( + session, + ent_row.id, + { + "version": ent_row.version, + "bindings": [ + *ent_row.bindings, + { + "source_name": erp, + "table_name": "dwd_settlement", + "column_name": "settled_qty", + "status": "rejected", + "source": "human", + }, + ], + }, + ) + await session.commit() + + # 3) mode="proposed"：尝试重提 rejected 列 + 新增 workshop_line + 自封 status="confirmed" + prop_res = await entity_patch.apply_entity_patch( + session, + [ + { + "op": "upsert_binding", + "entity_key": "prod_line_t", + "source_name": erp, + "table_name": "dwd_settlement", + "column_name": "settled_qty", + "status": "confirmed", + }, + { + "op": "upsert_binding", + "entity_key": "prod_line_t", + "source_name": erp, + "table_name": "dwd_settlement", + "column_name": "workshop_line", + "status": "confirmed", + "reason": "跨源目录对齐：结算产线", + }, + { + "op": "upsert_caliber_param", + "entity_key": "prod_line_t", + "workflow_id": wf_id, + "card_id": "card_yield", + "param_name": "p_line", + "status": "confirmed", + }, + ], + mode="proposed", + ) + await session.commit() + assert prop_res["bindings_changed"] == 1 + assert prop_res["caliber_params_changed"] == 1 + updated_ent = prop_res["entities"][0] + by_col = {b["column_name"]: b["status"] for b in updated_ent["bindings"]} + assert by_col["line_code"] == "confirmed" + assert by_col["settled_qty"] == "rejected" + assert by_col["workshop_line"] == "proposed" + assert updated_ent["caliber_params"][0]["status"] == "proposed" + + # 4) CAS 版本冲突应抛 EntityConflict + with pytest.raises(entities.EntityConflict): + await entity_patch.apply_entity_patch( + session, + [{"op": "upsert_entity", "entity_key": "prod_line_t", "label": "新产线名"}], + mode="confirmed", + expected_versions={"prod_line_t": 1}, + ) + + +async def test_entity_assistant_sse_and_apply_patch(multi_source_env, monkeypatch): + mes = multi_source_env["mes_name"] + erp = multi_source_env["erp_name"] + wf_id = multi_source_env["wf_id"] + + scripted = _ScriptedModel( + turns=[ + _Turn( + calls=[ + ("inspect_sources", {"keyword": "line"}), + ("read_document", {"query": "LINE_01"}), + ("run_query", {"source_name": mes, "sql": "SELECT line_code, operator_phone FROM dwd_workorder"}), + ] + ), + _Turn( + calls=[ + ( + "propose_entity_changes", + { + "operations": [ + { + "op": "upsert_entity", + "entity_key": "workshop_line_ent", + "label": "车间产线", + "aliases": ["产线"], + "reason": "使用者说：建立车间产线标准实体", + }, + { + "op": "upsert_binding", + "entity_key": "workshop_line_ent", + "source_name": mes, + "table_name": "dwd_workorder", + "column_name": "line_code", + "reason": "数据探查显示：工单表产线列为 line_code", + }, + { + "op": "upsert_binding", + "entity_key": "workshop_line_ent", + "source_name": erp, + "table_name": "dwd_settlement", + "column_name": "workshop_line", + "reason": "文档《对照表.md》写着：ERP 结算表对应 workshop_line", + }, + { + "op": "upsert_member", + "entity_key": "workshop_line_ent", + "value": "L1", + "label": "一号产线", + "source_codes": {mes: "L1", erp: "LINE_01"}, + "reason": "文档《对照表.md》写着：L1 对应 LINE_01", + }, + { + "op": "upsert_caliber_param", + "entity_key": "workshop_line_ent", + "workflow_id": wf_id, + "card_id": "card_yield", + "param_name": "p_line", + "reason": "口径入口对齐", + }, + ] + }, + ) + ] + ), + _Turn(text=["已核对实表与对照文档，生成车间产线标准实体及跨源编码提案，请确认保存。"]), + ] + ) + + async def _fake_resolve(_session): + return scripted, "mock-draft-model" + + monkeypatch.setattr(entity_assistant, "resolve_model", _fake_resolve) + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + att_res = await client.post( + "/api/ontology/entities/attachments", + files={"file": ("对照表.md", "一号产线：MES 为 L1，ERP dwd_settlement.workshop_line 为 LINE_01".encode("utf-8"), "text/markdown")}, + data={"save_to_kb": "false"}, + ) + assert att_res.status_code == 201 + att_id = att_res.json()["attachment_id"] + + sse_res = await client.post( + "/api/ontology/entities/assistant", + json={ + "messages": [{"role": "user", "content": "请根据对照表帮我关联车间产线实体"}], + "attachment_ids": [att_id], + }, + ) + assert sse_res.status_code == 200 + events = [ + json.loads(line[5:].strip()) + for line in sse_res.text.splitlines() + if line.startswith("data:") + ] + types = [ev["type"] for ev in events] + assert "query" in types + assert "proposal" in types + assert types[-1] == "done" + + # 确认只读查询中的敏感列已被遮罩为「已遮罩」 + q_ev = next(ev for ev in events if ev["type"] == "query") + assert q_ev["source"] == mes + assert all(row[1] == "已遮罩" for row in q_ev["rows"]) + + # 提案尚未落库 + list_before = await client.get("/api/ontology/entities") + assert all(e["key"] != "workshop_line_ent" for e in list_before.json()["items"]) + + # 人工确认并应用提案 + prop_ev = next(ev for ev in events if ev["type"] == "proposal") + ops = [c["op_payload"] for c in prop_ev["changes"]] + apply_res = await client.post( + "/api/ontology/entities/apply-patch", + json={"operations": ops, "expected_versions": prop_ev["expected_versions"]}, + headers={"X-Actor": "builder_one"}, + ) + assert apply_res.status_code == 200, apply_res.text + applied = apply_res.json() + assert applied["created_entities"] == 1 + saved = applied["entities"][0] + assert saved["key"] == "workshop_line_ent" + assert all(b["status"] == "confirmed" for b in saved["bindings"]) + assert saved["members"][0]["source_codes"] == {mes: "L1", erp: "LINE_01"} + + +async def test_entity_discovery_job_lifecycle_and_proposed_isolation(multi_source_env, monkeypatch): + mes = multi_source_env["mes_name"] + erp = multi_source_env["erp_name"] + + discovery_json = json.dumps( + { + "operations": [ + { + "op": "upsert_entity", + "entity_key": "discovered_line", + "label": "自动发现产线", + "description": "跨 MES 与 ERP 的产线对象", + "aliases": ["产线"], + "reason": "跨源目录对齐：工单与结算产线", + }, + { + "op": "upsert_binding", + "entity_key": "discovered_line", + "entity_label": "自动发现产线", + "source_name": mes, + "table_name": "dwd_workorder", + "column_name": "line_code", + "status": "confirmed", # 后台发现必须强制降为 proposed + "reason": "跨源目录对齐：MES 工单产线", + }, + { + "op": "upsert_binding", + "entity_key": "discovered_line", + "entity_label": "自动发现产线", + "source_name": erp, + "table_name": "dwd_settlement", + "column_name": "workshop_line", + "reason": "跨源目录对齐：ERP 结算产线", + }, + { + "op": "upsert_member", + "entity_key": "discovered_line", + "entity_label": "自动发现产线", + "value": "L1", + "label": "一号产线", + "source_codes": {mes: "L1", erp: "LINE_01"}, + "reason": "文档对照", + }, + ] + }, + ensure_ascii=False, + ) + fake_model = _ScriptedModel(invoke_reply=f"```json\n{discovery_json}\n```") + + async def _fake_resolve(_session): + return fake_model, "mock-discovery-model" + + monkeypatch.setattr(catalog_assistant, "resolve_model", _fake_resolve) + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + r = await client.post( + "/api/ontology/discovery-jobs", + json={ + "source_ids": [multi_source_env["mes_id"], multi_source_env["erp_id"]], + "note": "重点扫描产线", + }, + headers={"X-Actor": "builder_scan"}, + ) + assert r.status_code == 202, r.text + job_id = r.json()["id"] + + # 轮询等待完成 + final_job = None + for _ in range(40): + jr = await client.get(f"/api/ontology/discovery-jobs/{job_id}") + assert jr.status_code == 200 + body = jr.json() + if body["status"] in ("succeeded", "failed", "cancelled"): + final_job = body + break + await asyncio.sleep(0.05) + + assert final_job is not None + assert final_job["status"] == "succeeded", final_job + assert final_job["summary"]["created_entities"] == 1 + assert final_job["summary"]["proposed_bindings"] == 2 + assert final_job["summary"]["added_members"] == 1 + + ents = (await client.get("/api/ontology/entities")).json()["items"] + disc = next(e for e in ents if e["key"] == "discovered_line") + assert all(b["status"] == "proposed" and b["source"] == "ai" for b in disc["bindings"]) + + # 运行期 confirmed_bindings 必须为 0（状态门禁隔离） + assert entities.confirmed_bindings(disc) == [] + + +async def test_entity_discovery_job_busy_and_cancel(multi_source_env, monkeypatch): + entered = asyncio.Event() + + class _SlowModel: + async def ainvoke(self, _messages): + entered.set() + await asyncio.sleep(30) + return AIMessage(content='{"operations": []}') + + async def _fake_resolve(_session): + return _SlowModel(), "mock-slow-model" + + monkeypatch.setattr(catalog_assistant, "resolve_model", _fake_resolve) + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + r1 = await client.post( + "/api/ontology/discovery-jobs", + json={"source_ids": [multi_source_env["mes_id"]]}, + ) + assert r1.status_code == 202 + job_id = r1.json()["id"] + + await asyncio.wait_for(entered.wait(), timeout=3.0) + + # 并发再次发起应返回 409 discovery_job_busy + r2 = await client.post( + "/api/ontology/discovery-jobs", + json={"source_ids": [multi_source_env["mes_id"]]}, + ) + assert r2.status_code == 409 + assert r2.json().get("code") == "discovery_job_busy" + + # 主动停止扫描 + rc = await client.post(f"/api/ontology/discovery-jobs/{job_id}/cancel") + assert rc.status_code == 200 + assert rc.json()["status"] == "cancelled" + + +async def test_entity_assistant_source_scope_isolates_unrelated_domains(multi_source_env): + """进入单个数据源目录页时，传入 source_ids 仅暴露所选数据源的表、查询权限与关联工作流，隔离其他业务域。""" + mes_id = multi_source_env["mes_id"] + mes_name = multi_source_env["mes_name"] + erp_id = multi_source_env["erp_id"] + erp_name = multi_source_env["erp_name"] + wf_mes_id = f"wf_mes_only_{uuid.uuid4().hex[:6]}" + wf_erp_id = f"wf_erp_only_{uuid.uuid4().hex[:6]}" + + async with SessionLocal() as session: + # 构造一个仅归属 MES 域的实体与两个分别绑定不同数据源工具的工作流 + await entities.create_entity( + session, + { + "key": "mes_only_line", + "label": "车间专属产线", + "bindings": [ + { + "source_id": mes_id, + "source_name": mes_name, + "table_name": "dwd_workorder", + "column_name": "line_code", + "status": "confirmed", + "source": "human", + } + ], + }, + actor="tester", + ) + session.add_all( + [ + Workflow( + id=wf_mes_id, + name="车间工单良率", + version=1, + graph={ + "nodes": [ + { + "id": "in_m", + "type": "input", + "data": {"config": {"fields": [{"name": "scx_code", "label": "产线", "type": "string"}]}}, + }, + { + "id": "q_m", + "type": "tool", + "data": {"config": {"tool": f"db_query__{mes_name}", "args": {"sql": "SELECT 1"}}}, + }, + { + "id": "c_m", + "type": "metrics", + "data": {"config": {"metrics": [{"id": "m1", "name": "良率"}]}}, + }, + ], + "edges": [], + }, + ), + Workflow( + id=wf_erp_id, + name="结算金额口径", + version=1, + graph={ + "nodes": [ + { + "id": "in_e", + "type": "input", + "data": {"config": {"fields": [{"name": "store_id", "label": "门店", "type": "string"}]}}, + }, + { + "id": "q_e", + "type": "tool", + "data": {"config": {"tool": f"db_query__{erp_name}", "args": {"sql": "SELECT 1"}}}, + }, + { + "id": "c_e", + "type": "metrics", + "data": {"config": {"metrics": [{"id": "e1", "name": "结算量"}]}}, + }, + ], + "edges": [], + }, + ), + ] + ) + await session.commit() + + try: + async with SessionLocal() as session: + ctx = await entity_assistant.prepare(session, source_ids=[erp_id]) + + # 1. 只加载 erp_name，不包含 mes_name + assert list(ctx.sources.keys()) == [erp_name.lower()] + assert ctx.queryable_sources == [erp_name] + + # 2. 系统提示词不泄露 mes_name 的物理表名 dwd_workorder + assert "dwd_workorder" not in ctx.system + assert erp_name in ctx.system + + # 3. inspect_sources 只返回 erp 的表与 erp 关联的口径工作流，不包含车间工单良率工作流 + tools = entity_assistant._EntityTools(ctx) + inspected = tools.inspect("", "") + assert "dwd_settlement" in inspected + assert "dwd_workorder" not in inspected + assert wf_erp_id in inspected + assert wf_mes_id not in inspected + + # 4. 对未纳入范围的 mes_name 发起只读查询会被立即拒绝，不尝试连接该库 + ev, msg = await tools.query(mes_name, "SELECT * FROM dwd_workorder LIMIT 5") + assert ev is None + assert erp_name in msg + finally: + async with SessionLocal() as session: + await session.execute(delete(Workflow).where(Workflow.id.in_([wf_mes_id, wf_erp_id]))) + await session.commit() + + +async def test_entity_discovery_streaming_progress_and_summary(multi_source_env, monkeypatch): + """扫描发现任务支持 astream 流式输出并在 summary 中记录量化扫描统计与新增实体清单。""" + mes_id = multi_source_env["mes_id"] + mes_name = multi_source_env["mes_name"] + stream_payload = json.dumps( + { + "operations": [ + { + "op": "upsert_entity", + "entity_key": "stream_line", + "label": "流式识别产线", + "reason": "目录语义对齐：产线编码", + }, + { + "op": "upsert_binding", + "entity_key": "stream_line", + "entity_label": "流式识别产线", + "source_name": mes_name, + "table_name": "dwd_workorder", + "column_name": "line_code", + "reason": "目录语义对齐：产线字段", + }, + ] + }, + ensure_ascii=False, + ) + + class _StreamingDiscoveryModel: + async def astream(self, _messages): + mid = len(stream_payload) // 2 + yield AIMessageChunk(content=stream_payload[:mid]) + await asyncio.sleep(0.02) + yield AIMessageChunk(content=stream_payload[mid:]) + + async def _fake_resolve(_session): + return _StreamingDiscoveryModel(), "mock-stream-model" + + monkeypatch.setattr(catalog_assistant, "resolve_model", _fake_resolve) + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + r = await client.post( + "/api/ontology/discovery-jobs", + json={"source_ids": [mes_id]}, + ) + assert r.status_code == 202 + job_id = r.json()["id"] + + final_job = None + for _ in range(40): + jr = await client.get(f"/api/ontology/discovery-jobs/{job_id}") + body = jr.json() + if body["status"] in ("succeeded", "failed", "cancelled"): + final_job = body + break + await asyncio.sleep(0.05) + + assert final_job is not None + assert final_job["status"] == "succeeded", final_job + assert isinstance(final_job["started_at"], str) and ( + final_job["started_at"].endswith("+00:00") or final_job["started_at"].endswith("Z") + ) + assert isinstance(final_job["finished_at"], str) and ( + final_job["finished_at"].endswith("+00:00") or final_job["finished_at"].endswith("Z") + ) + summary = final_job["summary"] + assert summary["progress_pct"] == 100 + assert summary["sources_scanned"] == 1 + assert summary["source_names"] == [mes_name] + assert summary["tables_scanned"] >= 1 + assert summary["ops_detected"] == 2 + assert any(t["key"] == "stream_line" for t in summary["touched_entities"]) + + + diff --git a/backend/tests/test_frontend_eval_unions.py b/backend/tests/test_frontend_eval_unions.py index 0ce6a1e0..bc6ea239 100644 --- a/backend/tests/test_frontend_eval_unions.py +++ b/backend/tests/test_frontend_eval_unions.py @@ -99,13 +99,15 @@ def test_scores_compare_and_catalog_modes(): def test_analysis_tools_and_presets_follow_the_server(): - """变体的分析工具开、关和预设对照（api/evals.PRESETS）：前端的取值和服务端一一对上。叫法写「开」「关」，和服务端 + """变体的分析工具与本体穿透开、关及预设对照（api/evals.PRESETS）：前端的取值和服务端一一对上。叫法写「开」「关」，和服务端 describe、对比里写的同一种说法。""" from app.api.evals import PRESETS - from app.eval.variants import ANALYSIS_TOOLS, ANALYSIS_TOOLS_LABEL + from app.eval.variants import ANALYSIS_TOOLS, ANALYSIS_TOOLS_LABEL, ONTOLOGY_TOOLS, ONTOLOGY_TOOLS_LABEL assert _union("EvalAnalysisTools") == list(ANALYSIS_TOOLS) assert _pairs("start.analysisToolsMode") == list(ANALYSIS_TOOLS_LABEL.items()) + assert _union("EvalOntologyTools") == list(ONTOLOGY_TOOLS) + assert _pairs("start.ontologyToolsMode") == list(ONTOLOGY_TOOLS_LABEL.items()) assert _union("EvalPreset") == list(PRESETS) diff --git a/backend/tests/test_ontology_investigation_and_claims.py b/backend/tests/test_ontology_investigation_and_claims.py new file mode 100644 index 00000000..9f2f7f75 --- /dev/null +++ b/backend/tests/test_ontology_investigation_and_claims.py @@ -0,0 +1,481 @@ +"""Wave 2-b 单元测试：三段式穿透充分性评估 investigation.assess、H{n} 源健康证据引用与离线包核验、analysis_claims V2 非因果护栏（D6）。""" +from __future__ import annotations + +import json +from datetime import date +from typing import Any + +import pytest + +from sqlalchemy import delete + +import app.analysis as analysis_ops +from app.analysis import contract as analysis_contract +from app.bundle.build import build_bundle +from app.core.artifact_store import canonical_json, content_hash, manifest_hash +from app.data import source_health as sh +from app.db.base import SessionLocal +from app.db.models_ontology import CanonicalEntity +from app.engine import analysis_claims, analysis_scope, evidence, investigation +from app.engine.investigation import InvestigationPolicy +from app.engine.runner import run_manager +from app.tools.analysis import AnalysisScope, input_snapshot +from tests.fixtures.acceptance import multi_source_ontology as mso +from tests.test_bundle_shown import NODE, doc_of, node, reseal + + +@pytest.fixture(autouse=True) +async def _setup_and_clean(): + await run_manager.setup() + yield + await run_manager.shutdown() + async with SessionLocal() as session: + await session.execute(delete(CanonicalEntity)) + await session.commit() + + +def _snap(ref: str, columns: list[str], rows: list[list], *, source: str = "ops_shift") -> dict[str, Any]: + return input_snapshot( + { + "columns": columns, + "rows": rows, + "sql": f"SELECT {', '.join(columns)} FROM t", + "source": source, + "truncated": False, + }, + ref=ref, + artifact=ref.lower().ljust(64, "0")[:64], + node_id="bot", + masked=[], + ) + + +async def _probe(source: str, table: str, column: str) -> dict[str, Any]: + return { + "source": source, + "table": table, + "column": column, + "min": "2026-10-01", + "max": "2026-10-21", + "basis": "live", + "at": "2026-10-22T00:00:00+00:00", + } + + +async def _run_op(req: dict[str, Any], scope: AnalysisScope, cfg: dict[str, Any]) -> dict[str, Any]: + return await analysis_ops.run( + req, + scope.resolve, + settings=cfg, + context={"today": date(2026, 10, 22), "probe": _probe}, + refuse_data_errors=False, + ) + + +async def test_investigation_three_stage_s1_s2_s3_and_early_stop(): + cfg = analysis_contract.effective_settings({}) + policy = InvestigationPolicy(minimum_depth=0) + + # --- S1：契约越界（load_rate_pct > 100%），尚未做 ontology_neighbors 排查 -> 要求沿本体排查构成因子 --- + s1_scope = AnalysisScope(node_id="bot", snapshot=cfg) + s1_scope.record_query_target( + sources=["ops_shift"], + workflow_id="wf_ops", + card_id="m_ops", + violations=[{"metric_id": "load_rate_pct", "detail": "指标值超出声明的范围"}], + ) + g_s1 = investigation.assess(s1_scope, "为什么昨天三线产线负荷率达到 135% 越界了？", policy) + assert g_s1["status"] == "partial" + assert "ontology_neighbors" in g_s1["next_step"] or "契约" in g_s1["reason"] + + # 发现 factor 邻居但尚未跨源核对 -> 提示按实体桥查询关联因子 + s1_scope.record_ontology_neighbors( + [ + { + "kind": "factor", + "role": "denominator", + "target": { + "workflow_id": "wf_equip", + "card_id": "m_equip", + "metric_id": "rated_hourly_qty", + "sources": ["equip_master"], + }, + } + ], + focal_metrics=[{"workflow_id": "wf_ops", "card_id": "m_ops", "sources": ["ops_shift"]}], + ) + g_s1_mid = investigation.assess(s1_scope, "为什么昨天三线产线负荷率达到 135% 越界了？", policy) + assert g_s1_mid["status"] == "partial" + assert g_s1_mid["stage"] == "ontology_penetration" + + # 完成 1 跳跨源核对后 -> S1 收口通过 + s1_scope.record_query_target(sources=["equip_master"], workflow_id="wf_equip", card_id="m_equip") + g_s1_done = investigation.assess(s1_scope, "为什么昨天三线产线负荷率达到 135% 越界了？", policy) + assert g_s1_done["status"] == "complete", g_s1_done + assert g_s1_done["stage"] == "ontology_penetration" + + # --- S2：数据源健康检查发现断更（anomalous） -> 立即早停（complete），绝不误导继续做业务归因拆解！ --- + s2_scope = AnalysisScope(node_id="bot", snapshot=cfg) + s2_scope.add_query("Q1", _snap("Q1", ["dt", "scrap"], [["2026-10-20", 18], ["2026-10-21", 20]], source="ops_shift")) + s2_scope.add_health( + { + "kind": "source_health", + "verdict": "anomalous", + "phrase": "数据源「quality_scrap」存在断更告警（最新日期停在 2026-10-19）", + "sources": [{"source": "quality_scrap", "verdict": "anomalous"}], + "issues": [{"kind": "date_stalled", "severity": "alert", "detail": "最新日期停在 2026-10-19"}], + }, + artifact="a" * 64, + ) + g_s2_health = investigation.assess(s2_scope, "为什么昨天质检台账报废数突然掉到 0，和车间对不上？", policy) + assert g_s2_health["status"] == "complete" + assert g_s2_health["stage"] == "health_and_contract" + + # --- S2 变体：reconcile 判定 diverged -> 同样在 health_and_contract 阶段立即早停收口 --- + s2_rec_scope = AnalysisScope(node_id="bot", snapshot=cfg) + q_left = _snap("Q1", ["line_code", "qty"], [["L1", 18], ["L2", 22]], source="ops_shift") + q_right = _snap("Q2", ["line_code", "qty"], [["L1", 18], ["L2", 2]], source="quality_scrap") + s2_rec_scope.add_query("Q1", q_left) + s2_rec_scope.add_query("Q2", q_right) + rec_res = await _run_op( + { + "op": "reconcile", + "mode": None, + "params": {"base_query": "Q1", "target_query": "Q2", "by": ["line_code"], "measure": "qty"}, + }, + s2_rec_scope, + cfg, + ) + assert rec_res["verdict"]["code"] == "diverged" + s2_rec_scope.add_result(rec_res, artifact="b" * 64) + g_s2_rec = investigation.assess(s2_rec_scope, "为什么两套报废数对不上？", policy) + assert g_s2_rec["status"] == "complete" + assert g_s2_rec["stage"] == "health_and_contract" + + # --- S3：源健康正常、同源基线异常 + 已拆解定位切片 L2，存在 upstream_driver 邻居未做跨源 cochange 验证 -> 提示跨源穿透 --- + s3_scope = AnalysisScope(node_id="bot", snapshot=cfg) + days = [f"2026-10-{d:02d}" for d in range(7, 22)] # 15 天序列 + agg_vals = [1800, 1804, 1796, 1802, 1800, 1798, 1806, 1802, 1796, 1804, 1800, 1796, 1802, 1800, 1512] + q_agg = _snap("Q1", ["dt", "good_qty"], [[d, v] for d, v in zip(days, agg_vals)], source="ops_shift") + q_slice = _snap( + "Q2", + ["dt", "line_code", "good_qty"], + [ + ["2026-10-20", "L1", 900], + ["2026-10-20", "L2", 900], + ["2026-10-21", "L1", 900], + ["2026-10-21", "L2", 612], + ], + source="ops_shift", + ) + s3_scope.add_query("Q1", q_agg) + s3_scope.add_query("Q2", q_slice) + s3_scope.record_query_target(sources=["ops_shift"], workflow_id="wf_ops", card_id="m_ops") + + base_res = await _run_op( + {"op": "baseline", "mode": None, "params": {"query": "Q1", "period": "dt", "measure": "good_qty", "lag": 1}}, + s3_scope, + cfg, + ) + assert base_res["verdict"]["code"] in ("outside_band", "far_outside_band") + decomp_res = await _run_op( + { + "op": "decompose", + "mode": "additive", + "params": { + "query": "Q2", + "period": "dt", + "by": ["line_code"], + "measure": "good_qty", + "base": "2026-10-20", + "target": "2026-10-21", + "headline": {"base": "Q1.r13.good_qty", "target": "Q1.r14.good_qty"}, + }, + }, + s3_scope, + cfg, + ) + assert decomp_res["status"] == "ok" + s3_scope.add_result(base_res, artifact="d" * 64) + s3_scope.add_result(decomp_res, artifact="e" * 64) + s3_scope.record_ontology_neighbors( + [ + { + "kind": "upstream_driver", + "entity_key": "production_line", + "target": { + "workflow_id": "wf_equip", + "card_id": "m_equip", + "metric_id": "unplanned_downtime_min", + "sources": ["equip_master"], + }, + } + ], + focal_metrics=[{"workflow_id": "wf_ops", "card_id": "m_ops", "sources": ["ops_shift"]}], + ) + g_s3_before = investigation.assess(s3_scope, "为什么昨天车间合格产量突然异常下跌？", policy) + assert g_s3_before["status"] == "partial" + assert g_s3_before["stage"] == "ontology_penetration" + + # 补上跨源驱动序列 + cochange 后 -> S3 完整收口！ + l2_vals = [900, 902, 898, 901, 900, 899, 903, 901, 898, 902, 900, 898, 901, 900, 612] + drv_vals = [12.0, 12.5, 11.8, 12.2, 12.0, 11.6, 12.4, 12.1, 11.9, 12.3, 12.0, 11.7, 12.2, 12.0, 96.0] + q_l2 = _snap("Q3", ["dt", "good_qty"], [[d, v] for d, v in zip(days, l2_vals)], source="ops_shift") + q_driver = _snap("Q4", ["dt", "downtime_min"], [[d, v] for d, v in zip(days, drv_vals)], source="equip_master") + s3_scope.add_query("Q3", q_l2) + s3_scope.add_query("Q4", q_driver) + s3_scope.record_query_target(sources=["equip_master"], workflow_id="wf_equip", card_id="m_equip") + co_res = await _run_op( + { + "op": "cochange", + "mode": None, + "params": { + "base_query": "Q3", + "target_query": "Q4", + "period": "dt", + "measure": "good_qty", + "driver_measure": "downtime_min", + }, + }, + s3_scope, + cfg, + ) + assert co_res["verdict"]["code"] == "co_anomalous" + s3_scope.add_result(co_res, artifact="f" * 64) + g_s3_after = investigation.assess(s3_scope, "为什么昨天车间合格产量突然异常下跌？", policy) + assert g_s3_after["status"] == "complete", g_s3_after + assert g_s3_after["stage"] == "ontology_penetration" + + +async def test_analysis_claims_v2_reconcile_cochange_data_fault_and_d6_non_causal(): + lex_v2 = analysis_claims.lexicon(2) + lex_v1 = analysis_claims.lexicon(1) + v2_keys = {c.key for c in lex_v2.claims} + v1_keys = {c.key for c in lex_v1.claims} + assert {"reconcile_bad", "cochange", "data_fault"}.issubset(v2_keys) + assert "reconcile_bad" not in v1_keys # 老封存报告按 v1 复核不漂移 + + cfg = analysis_contract.effective_settings({}) + scope = AnalysisScope(node_id="bot", snapshot=cfg) + days = [f"2026-10-{d:02d}" for d in range(7, 22)] + q_p = _snap( + "Q1", + ["dt", "good_qty"], + [[d, v] for d, v in zip(days, [900, 902, 898, 901, 900, 899, 903, 901, 898, 902, 900, 898, 901, 900, 612])], + source="ops_shift", + ) + q_d = _snap( + "Q2", + ["dt", "downtime_min"], + [[d, v] for d, v in zip(days, [12.0, 12.5, 11.8, 12.2, 12.0, 11.6, 12.4, 12.1, 11.9, 12.3, 12.0, 11.7, 12.2, 12.0, 96.0])], + source="equip_master", + ) + q_r = _snap("Q3", ["line_code", "scrap_qty"], [["L1", 18], ["L2", 22]], source="ops_shift") + q_c = _snap("Q4", ["line_code", "scrap_units"], [["L1", 18], ["L2", 2]], source="quality_scrap") + for alias, q in (("Q1", q_p), ("Q2", q_d), ("Q3", q_r), ("Q4", q_c)): + scope.add_query(alias, q) + + co_res = await _run_op( + { + "op": "cochange", + "mode": None, + "params": {"base_query": "Q1", "target_query": "Q2", "period": "dt", "measure": "good_qty", "driver_measure": "downtime_min"}, + }, + scope, + cfg, + ) + rec_res = await _run_op( + { + "op": "reconcile", + "mode": None, + "params": {"base_query": "Q3", "target_query": "Q4", "by": ["line_code"], "measure": "scrap_qty", "target_measure": "scrap_units"}, + }, + scope, + cfg, + ) + h_res = { + "kind": "source_health", + "verdict": "anomalous", + "phrase": "数据源「quality_scrap」存在断更告警（最新日期停在 2026-10-19）", + "sources": [{"source": "quality_scrap", "verdict": "anomalous", "issue_count": 1}], + "issues": [{"kind": "date_stalled", "severity": "alert", "detail": "最新日期停在 2026-10-19"}], + "tables": [{"table": "scrap_ledger", "max_date": "2026-10-19"}], + } + + store: dict[str, Any] = {} + + def put(obj: Any) -> str: + sha = content_hash(canonical_json(obj).encode()) + store[sha] = obj + return sha + + a1_art = put(co_res) + a2_art = put(rec_res) + h1_art = put(h_res) + ledger = [ + analysis_contract.ledger_entry(co_res, node_id="bot", exec_no=1, artifact=a1_art, call_id="c1"), + analysis_contract.ledger_entry(rec_res, node_id="bot", exec_no=1, artifact=a2_art, call_id="c2"), + sh.ledger_entry(h_res, node_id="bot", exec_no=1, artifact=h1_art, call_id="h1"), + ] + catalog = evidence.build_catalog(nodes={}, ledger=ledger, loader=store.get) + assert "H1" in catalog and catalog["H1"]["kind"] == sh.LEDGER_KIND + + # 1) D6 核心铁律：只挂 cochange（A1）、没有知识库/输入依据时，写「由于设备停机导致合格产量下降」必须被拦成 causal_unsupported！ + bad_doc = evidence.compose_doc( + "星梭二线由于非计划停机时长上升导致合格产量下降[[see:A1]]。", + catalog, + node_id="write", + run_id="r1", + loader=store.get, + analysis_claims="strict", + ) + v_bad = analysis_claims.check_doc(bad_doc, catalog) + assert any(v["code"] == analysis_claims.CAUSAL for v in v_bad), v_bad + + # 2) 合规写法：引用系统判定语 [[a:A1]] 并写统计伴随异动 -> 0 违规通过！ + good_cochange = "[[a:A1]]。两者在统计上伴随同步异动，具体工艺原因待结合现场检修记录核实[[see:A1]]。" + doc_good = evidence.compose_doc( + good_cochange, + catalog, + node_id="write", + run_id="r1", + loader=store.get, + analysis_claims="strict", + ) + assert analysis_claims.check_doc(doc_good, catalog) == [] + + # 3) 数据源断更/对账不一致：挂 H1 与 A2 -> 0 违规通过；无依据空口写「数据源断更」被拦成 analysis_claim_uncited + good_fault = "[[a:A2]]。[[h:H1]]，由于数据源断更导致双源报废数未对齐，不作业务滑坡归因[[see:A2,H1]]。" + doc_fault = evidence.compose_doc( + good_fault, + catalog, + node_id="write", + run_id="r1", + loader=store.get, + analysis_claims="strict", + ) + assert analysis_claims.check_doc(doc_fault, catalog) == [] + + doc_unbacked = evidence.compose_doc( + "质检台账昨天数据源断更，而且双源对账不一致。", + catalog, + node_id="write", + run_id="r1", + loader=store.get, + analysis_claims="strict", + ) + v_unbacked = analysis_claims.check_doc(doc_unbacked, catalog) + assert any(v["code"] == analysis_claims.UNCITED for v in v_unbacked), v_unbacked + + # 4) verify_doc 复核通过且 analysis_scope.build 把 reconcile / cochange 收入 phenomena + verify_res = evidence.verify_doc(doc_fault, catalog, loader=store.get) + assert verify_res["ok"] is True, verify_res + scope_sec = analysis_scope.build(catalog, sources={}, load=store.get) + assert scope_sec is not None + ops_in_phenomena = {item["op"] for item in scope_sec["phenomena"]} + assert {"reconcile", "cochange"}.issubset(ops_in_phenomena) + + +@pytest.mark.skipif(NODE is None, reason="没有 node，无法执行离线包 verifier.js") +async def test_health_citation_evidence_chain_and_offline_bundle_verifier(tmp_path, scratch_source): + seeded = await mso.seed_ontology_fixture(tmp_path, scratch_source, today=date(2026, 10, 22)) + yesterday = "2026-10-21" + + async with SessionLocal() as session: + health_res = await sh.check_source_health( + session, + workflow_id=seeded["workflow_ids"]["quality"], + card_id="m_quality", + metric_id="ledger_scrap_units", + start=yesterday, + end=yesterday, + ) + assert health_res["verdict"] == "anomalous" + + files: dict[str, bytes] = {} + + def put(obj: Any) -> str: + raw = canonical_json(obj).encode() + sha = content_hash(raw) + files[sha] = raw + return sha + + def load(sha: str) -> Any: + raw = files.get(sha) + return json.loads(raw) if raw is not None else None + + h_art = put(health_res) + h_entry = sh.ledger_entry(health_res, node_id="bot", exec_no=1, artifact=h_art, call_id="call_h1") + catalog = evidence.build_catalog(nodes={}, ledger=[h_entry], loader=load) + assert "H1" in catalog + + # 报告正文使用 [[h:H1]] 渲染源健康判定语，并用 [[see:H1]] 挂接句末依据 + report_md = "## 跨源对账核查\n\n[[h:H1]]。因此昨日台账缺失属于源表同步断更，不做业务归因[[see:H1]]。" + run_id = "9a1e0c3a5d9f4a2e8c6b4d2f0a1e3c5b" + doc = evidence.compose_doc(report_md, catalog, node_id="write", run_id=run_id, loader=load) + doc_id = put(doc) + + # 1) 服务端 verify_doc 校验通过 + checked = evidence.verify_doc(doc, catalog, loader=load) + assert checked["ok"] is True, checked + + # 2) 构建离线可验证报告包并用 node 执行 verifier.js 逐字节核验 + run_meta = { + "id": run_id, + "workflow_id": "wf-ontology-s2", + "workflow_name": "跨源健康排查", + "version": 1, + "version_hash": "ab" * 32, + "run_class": "formal", + "status": "succeeded", + "started_at": "2026-10-22T01:00:00+00:00", + "finished_at": "2026-10-22T01:01:00+00:00", + } + graph = { + "nodes": [ + {"id": "bot", "type": "agent", "data": {"label": "本体排查Agent"}}, + {"id": "write", "type": "report", "data": {"label": "排查报告"}}, + {"id": "out", "type": "output", "data": {"label": "成果"}}, + ] + } + events: list[tuple[int, str, str | None, Any]] = [ + (1, "run.started", None, {"nodes": 3, "resumed": False, "tool_trust": "formal", "replay_protocol": 2}), + ( + 2, + "tool.end", + "bot", + { + "tool": "source_health", + "duration_ms": 4, + "health_artifact": {"ref": "H1", "artifact": h_art, "verdict": "anomalous", "phrase": health_res["phrase"]}, + }, + ), + (3, "node.finished", "bot", {"duration_ms": 10, "attempt": 1, "evidence": [h_entry]}), + (4, "report.checked", "write", {"doc_artifact": doc_id, "ok": True}), + (5, "node.finished", "write", {"duration_ms": 8, "attempt": 1, "preview": {"text": "（报告正文）", "doc_artifact": doc_id}}), + (6, "issuance", "out", {"tier": "formal", "mode": "citations"}), + (7, "run.finished", None, {"output": {"报告": "（报告正文）"}}), + ] + bundle = build_bundle( + run_meta, + events, + manifest_seq=len(events), + manifest_hash=manifest_hash(events), + read=files.get, + exported_at="2026-10-22T02:00:00+00:00", + graph=graph, + source_masks={}, + ) + assert any(a["id"] == h_art and a["kind"] == sh.ARTIFACT_KIND for a in bundle["artifacts"]) + verdict = node("verify", data=bundle, seal=bundle["meta"]["manifest_hash"]) + assert verdict["overall"]["result"] == "pass", verdict + + # 3) 篡改报告里 [[h:H1]] 展示文字并重封事件哈希 -> verifier.js 必须精准检出健康判定语不一致！ + art, forged_doc = doc_of(bundle) + seg = next(s for s in evidence.iter_segments(forged_doc) if s.get("ref") == "h:H1") + fake_text = "数据源「quality_scrap」完全正常无任何延迟" + forged_doc["markdown"] = forged_doc["markdown"].replace(seg["text"], fake_text, 1) + seg["cite"]["rendered"] = seg["text"] = fake_text + resealed = reseal(bundle, art, forged_doc) + bad_verdict = node("verify", data=resealed, seal=resealed["meta"]["manifest_hash"]) + assert bad_verdict["overall"]["result"] == "fail", bad_verdict diff --git a/backend/tests/test_ontology_model.py b/backend/tests/test_ontology_model.py new file mode 100644 index 00000000..7142a082 --- /dev/null +++ b/backend/tests/test_ontology_model.py @@ -0,0 +1,353 @@ +"""Wave 1-a 单元测试：标准实体 CanonicalEntity、指标本体 links、DAG 无环门禁、指纹/发布清单后向兼容与 /api/ontology 接口。""" +from __future__ import annotations + +from datetime import date + +import pytest +from httpx import ASGITransport, AsyncClient +from sqlalchemy import delete + +from app.api.workflows import commit_publish +from app.apps import publications +from app.apps.validation import gate_issues +from app.core import artifact_store +from app.data import caliber_identity +from app.data.entities import ( + EntityConflict, + EntityInvalid, + confirmed_bindings, + create_entity, + delete_entity, + entities_bridging_sources, + entity_to_dict, + resolve_caliber_param, + review_entity_item, + translate_entity_value, + update_entity, +) +from app.db.base import SessionLocal +from app.db.models import Workflow, WorkflowPublication, WorkflowVersion +from app.db.models_ontology import CanonicalEntity +from app.ledger.docdraft import normalize_metrics +from app.ledger.ontology import ( + ONTOLOGY_CYCLE_CODE, + OntologyCycleError, + active_links, + build_published_ontology_graph, + card_metric_lineage, + check_factor_dag, + find_factor_cycles, + ontology_manifest_section, +) +from app.main import app +from tests.fixtures.acceptance import multi_source_ontology as mso + + +@pytest.fixture(autouse=True) +async def _clean_entities(): + yield + async with SessionLocal() as session: + await session.execute(delete(CanonicalEntity)) + await session.commit() + + +async def test_canonical_entity_crud_optimistic_lock_and_proposed_gating(): + async with SessionLocal() as session: + row = await create_entity( + session, + { + "key": "work_line_test", + "label": "示范产线", + "aliases": ["产线编码", "工位线"], + "bindings": [ + { + "source_name": "ops_shift", + "table_name": "shift_output", + "column_name": "line_code", + "status": "confirmed", + }, + { + "source_name": "equip_master", + "table_name": "line_specs", + "column_name": "equip_line_id", + "status": "confirmed", + }, + { + "source_name": "quality_scrap", + "table_name": "scrap_ledger", + "column_name": "workcenter_code", + "source": "ai", + }, + ], + "caliber_params": [ + {"workflow_id": "wf_eq", "param_name": "equip_line", "status": "confirmed"}, + {"workflow_id": "wf_q", "param_name": "wc_code", "status": "proposed"}, + ], + "members": [ + { + "value": "L1", + "label": "星梭一线", + "aliases": ["一线"], + "source_codes": {"ops_shift": "L1", "equip_master": "EQ-L1", "quality_scrap": "WC-01"}, + } + ], + }, + actor="builder_a", + ) + await session.commit() + entity_id = row.id + assert row.version == 1 + + # AI 来源未显式指定 status 时自动落为 proposed，不进 confirmed_bindings + active = confirmed_bindings(row) + assert {b["source_name"] for b in active} == {"ops_shift", "equip_master"} + + # 确定性跨源值翻译：精确命中 L1 -> EQ-L1，子串或未知编码返回 None（绝不模糊猜测） + translated = translate_entity_value(row, "L1", from_source="ops_shift", to_source="equip_master") + assert translated == {"canonical": "L1", "label": "星梭一线", "target_value": "EQ-L1"} + assert translate_entity_value(row, "L10", from_source="ops_shift", to_source="equip_master") is None + + # 口径参数解析：confirmed 可见，proposed 被拦下 + assert resolve_caliber_param(row, workflow_id="wf_eq") == "equip_line" + assert resolve_caliber_param(row, workflow_id="wf_q") is None + + # 跨源桥接：只列出 confirmed 绑定的对侧数据源 + bridges = entities_bridging_sources([row], source_name="ops_shift", tables=["shift_output"]) + assert len(bridges) == 1 and bridges[0]["peer_sources"] == ["equip_master"] + + # 受限视角（restricted=True）：隐去物理绑定、物理表列名与底层 source_codes + restricted_view = entity_to_dict(row, restricted=True) + assert "bindings" not in restricted_view + assert "shift_output" not in str(restricted_view) and "line_code" not in str(restricted_view) + for m in restricted_view["members"]: + assert "source_codes" not in m + + # 审阅确认 AI 候选绑定后，quality_scrap 进入生效绑定，版本号自增 + proposed_item = next(b for b in (row.bindings or []) if b["source_name"] == "quality_scrap") + updated = await review_entity_item( + session, + entity_id, + target_kind="binding", + item_id=proposed_item["id"], + status="confirmed", + actor="reviewer_b", + expected_version=1, + ) + await session.commit() + assert updated.version == 2 + assert {b["source_name"] for b in confirmed_bindings(updated)} == { + "ops_shift", + "equip_master", + "quality_scrap", + } + + # 乐观锁冲突：携带旧版本号 v1 更新或审阅必须抛 EntityConflict + with pytest.raises(EntityConflict): + await update_entity(session, entity_id, {"label": "过期写入", "version": 1}, actor="builder_a") + + # 唯一键冲突与非法 key 校验 + with pytest.raises(EntityConflict): + await create_entity(session, {"key": "work_line_test", "label": "重复"}) + with pytest.raises(EntityInvalid): + await create_entity(session, {"key": "Bad-Key!", "label": "非法"}) + + assert await delete_entity(session, entity_id) is True + await session.commit() + + +def test_factor_cycle_detection_and_docdraft_proposed_links(): + # 无环 factor 链放行 + check_factor_dag([("wf1:c1:oee", "wf1:c1:avail"), ("wf1:c1:avail", "wf2:c2:run_time")]) + + # 二跳环与三跳环均精准检出 + cycles_2 = find_factor_cycles([("a", "b"), ("b", "a")]) + assert len(cycles_2) == 1 + with pytest.raises(OntologyCycleError) as exc_info: + check_factor_dag([("m1", "m2"), ("m2", "m3"), ("m3", "m1")]) + assert exc_info.value.code == ONTOLOGY_CYCLE_CODE + + # 文档起草（docdraft）即使模型在草稿里写了 status="confirmed"，落库前一律强制降为 proposed 且带 source="docdraft" + draft_metrics, dropped = normalize_metrics( + [ + { + "id": "load_rate", + "name": "产线负荷率", + "unit": "%", + "citations": [{"chunk": "c0", "quote": "负荷率等于实际每小时产量除以额定每小时产量"}], + "links": [ + { + "kind": "factor", + "target": {"workflow_id": "wf_eq", "card_id": "m_eq", "metric_id": "rated_qty"}, + "role": "denominator", + "status": "confirmed", + } + ], + } + ], + chunks={"c0": {"content": "负荷率等于实际每小时产量除以额定每小时产量。", "document_id": "doc_1"}}, + ) + assert dropped == [] + assert len(draft_metrics) == 1 + links = draft_metrics[0]["links"] + assert len(links) == 1 + assert links[0]["status"] == "proposed" and links[0]["source"] == "docdraft" + assert active_links(links) == [] + + +async def test_definition_hash_and_publication_manifest_backward_compatibility(tmp_path, scratch_source): + await scratch_source("demo_ops", mso.build(tmp_path).ops_path) + + base_metric = { + "id": "good_qty", + "name": "合格产量", + "expression": "{{ nodes.q1.rows[0].good_qty }}", + "unit": "件", + } + h_plain = caliber_identity.definition_hash("日产率", "", [base_metric]) + h_empty = caliber_identity.definition_hash("日产率", "", [{**base_metric, "links": []}]) + assert h_plain == h_empty + + with_link = { + **base_metric, + "links": [ + { + "kind": "upstream_driver", + "target": {"workflow_id": "wf_up", "card_id": "c_up", "metric_id": "down_min"}, + "status": "confirmed", + } + ], + } + assert caliber_identity.definition_hash("日产率", "", [with_link]) != h_plain + + def make_graph(metrics_list): + return { + "nodes": [ + {"id": "in1", "type": "input", "position": {"x": 0, "y": 0}, "data": {"label": "入口", "config": {}}}, + { + "id": "q1", + "type": "tool", + "position": {"x": 200, "y": 0}, + "data": { + "label": "取数", + "config": { + "tool": "db_query__demo_ops", + "args": {"sql": "SELECT good_qty FROM shift_output WHERE operator_phone <> 'SECRET-PHONE'"}, + }, + }, + }, + { + "id": "m1", + "type": "metrics", + "position": {"x": 420, "y": 0}, + "data": {"label": "指标卡", "config": {"caliber": "日产率", "metrics": metrics_list}}, + }, + {"id": "out1", "type": "output", "position": {"x": 640, "y": 0}, "data": {"label": "出口", "config": {}}}, + ], + "edges": [ + {"source": "in1", "target": "q1"}, + {"source": "q1", "target": "m1"}, + {"source": "m1", "target": "out1"}, + ], + } + + # 1) 无 links 时：ontology_manifest_section 为 None，发布清单完全不含 "ontology" 键（存量哈希零变动） + plain_graph = make_graph([base_metric]) + assert ontology_manifest_section(workflow_id="wf_plain", version=1, graph=plain_graph) is None + + # 自动反向血缘只记 (source, tables)，绝不泄漏 SQL 字面量里的敏感字符串 + lineage = card_metric_lineage(plain_graph, "m1") + assert lineage["tables_by_source"] == {"demo_ops": ["shift_output"]} + assert "SECRET-PHONE" not in str(lineage) + + async with SessionLocal() as session: + wf1 = Workflow(name="无本体边工作流测试", description="", graph=plain_graph, version=1) + session.add(wf1) + await session.flush() + snap1 = WorkflowVersion(workflow_id=wf1.id, version=1, graph=plain_graph, note="v1") + session.add(snap1) + await session.commit() + await session.refresh(wf1) + await session.refresh(snap1) + pub1 = await commit_publish(session, wf1, snap1, "published", "tester") + manifest1 = artifact_store.load(pub1["manifest"]) + assert "ontology" not in manifest1 + + # 2) 含自环 factor 边的工作流在发布预检 gate_issues 阶段被直接拦下 + cyclic_graph = make_graph( + [ + { + **base_metric, + "links": [ + { + "kind": "factor", + "target": {"workflow_id": wf1.id, "card_id": "m1", "metric_id": "good_qty"}, + "status": "confirmed", + } + ], + } + ] + ) + issues = await gate_issues(session, cyclic_graph, "published", workflow_id=wf1.id) + assert issues is not None + assert any(i["level"] == "error" and i.get("code") == ONTOLOGY_CYCLE_CODE for i in issues) + + # 3) 合法本体边发布后物化进清单，且 publications.rebuild 重建结果逐字节一致 + linked_graph = make_graph([with_link]) + wf2 = Workflow(name="含本体边工作流测试", description="", graph=linked_graph, version=1) + session.add(wf2) + await session.flush() + snap2 = WorkflowVersion(workflow_id=wf2.id, version=1, graph=linked_graph, note="v1") + session.add(snap2) + await session.commit() + await session.refresh(wf2) + await session.refresh(snap2) + pub2 = await commit_publish(session, wf2, snap2, "published", "tester") + manifest2 = artifact_store.load(pub2["manifest"]) + assert "ontology" in manifest2 + assert "SECRET-PHONE" not in str(manifest2["ontology"]) + pub_row = await session.get(WorkflowPublication, pub2["publication"]["id"]) + assert pub_row is not None + rebuilt = await publications.rebuild(session, pub_row) + assert rebuilt["manifest"] == pub2["manifest"] + assert rebuilt["matches"] is True + + +async def test_ontology_http_endpoints_and_published_graph(tmp_path, scratch_source): + seeded = await mso.seed_ontology_fixture(tmp_path, scratch_source, today=date(2026, 10, 22)) + ops_wf = seeded["workflow_ids"]["ops"] + + async with SessionLocal() as session: + active_graph = await build_published_ontology_graph(session, include_proposed=False) + all_graph = await build_published_ontology_graph(session, include_proposed=True) + # ops 工作流里有 3 条 confirmed 边 + 1 条 proposed 边 + ops_active = [e for e in active_graph["edges"] if e["from"]["workflow_id"] == ops_wf] + ops_all = [e for e in all_graph["edges"] if e["from"]["workflow_id"] == ops_wf] + assert len(ops_active) == 3 + assert len(ops_all) == 4 + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + res_list = await client.get("/api/ontology/entities", params={"source_name": "ops_shift"}) + assert res_list.status_code == 200 + keys = [e["key"] for e in res_list.json()["items"]] + assert "production_line" in keys + + res_graph = await client.get("/api/ontology/graph") + assert res_graph.status_code == 200 + payload = res_graph.json() + assert any(e["key"] == "production_line" for e in payload["entities"]) + assert any(edge["kind"] == "factor" for edge in payload["edges"]) + + res_wf = await client.get(f"/api/ontology/workflows/{ops_wf}") + assert res_wf.status_code == 200 + wf_index = res_wf.json() + assert wf_index["workflow_id"] == ops_wf + assert wf_index["cards"][0]["tables_by_source"] == {"ops_shift": ["shift_output"]} + + # 乐观锁 409 验证 + entity_id = seeded["entity_id"] + res_conflict = await client.patch( + f"/api/ontology/entities/{entity_id}", + json={"label": "冲突测试", "version": 999}, + ) + assert res_conflict.status_code == 409 diff --git a/backend/tests/test_ontology_operators.py b/backend/tests/test_ontology_operators.py new file mode 100644 index 00000000..838489d9 --- /dev/null +++ b/backend/tests/test_ontology_operators.py @@ -0,0 +1,369 @@ +"""Wave 1-b 单元测试：数据源健康度检查 check_source_health、跨源对账算子 reconcile、伴随异动算子 cochange 及 Agent 工具接入。""" +from __future__ import annotations + +import asyncio +from datetime import date + +import pytest +from langchain_core.messages import AIMessage, ToolMessage +from sqlalchemy import delete, select + +import app.analysis as analysis_ops +from app.analysis import contract +from app.analysis.contract import CallError +from app.core import artifact_store +from app.data import source_changes as sc +from app.data import source_health as sh +from app.db.base import SessionLocal +from app.db.models import DataSource, Run, RunEvent +from app.db.models_ontology import CanonicalEntity +from app.engine.runner import run_manager +from app.tools.analysis import input_snapshot +from tests.fixtures.acceptance import multi_source_ontology as mso + + +@pytest.fixture(autouse=True) +async def _engine_lifecycle(): + await run_manager.setup() + yield + await run_manager.shutdown() + async with SessionLocal() as session: + await session.execute(delete(CanonicalEntity)) + await session.commit() + + +def _snap(ref: str, columns: list[str], rows: list[list], *, masked: list[str] | None = None, + truncated: bool = False, source: str = "ops_shift", artifact: str | None = None) -> dict: + return input_snapshot( + { + "columns": columns, + "rows": rows, + "sql": f"SELECT {', '.join(columns)} FROM t", + "source": source, + "truncated": truncated, + }, + ref=ref, + artifact=artifact or (ref.lower().ljust(64, "0")[:64]), + node_id="bot", + masked=masked or [], + ) + + +async def test_source_health_verdicts_masking_and_caliber_reverse_lineage(tmp_path, scratch_source, monkeypatch): + seeded = await mso.seed_ontology_fixture(tmp_path, scratch_source, today=date(2026, 10, 22)) + yesterday = "2026-10-21" + + async with SessionLocal() as session: + # 1) ops_shift 数据最新覆盖到昨天 -> healthy，且探查绝不触碰遮罩列 operator_phone + healthy = await sh.check_source_health( + session, + source_name=mso.SOURCE_OPS, + tables=["shift_output"], + start="2026-10-15", + end=yesterday, + ) + assert healthy["verdict"] == sh.HEALTHY + assert "operator_phone" not in str(healthy) + assert "138-0010" not in str(healthy) + + # 2) quality_scrap 在昨天断更（SourceChange date_stalled + 现查最大日期 2026-10-19 < 2026-10-21）-> anomalous + anomalous = await sh.check_source_health( + session, + source_name=mso.SOURCE_QUALITY, + tables=["scrap_ledger"], + start=yesterday, + end=yesterday, + ) + assert anomalous["verdict"] == sh.ANOMALOUS + assert any(i["kind"] == sc.DATE_STALLED for i in anomalous["issues"]) + assert "inspector_id" not in str(anomalous) + assert "INSP-900" not in str(anomalous) + + # 3) restricted=True 受限视角隐去物理表名、物理列名与底层数据源标识 + restricted_res = await sh.check_source_health( + session, + source_name=mso.SOURCE_QUALITY, + tables=["scrap_ledger"], + start=yesterday, + end=yesterday, + restricted=True, + ) + assert restricted_res["verdict"] == sh.ANOMALOUS + assert "scrap_ledger" not in str(restricted_res) + assert "inspect_date" not in str(restricted_res) + + # 4) 按口径卡反向血缘自动解析底层数据源与物理表 + via_caliber = await sh.check_source_health( + session, + workflow_id=seeded["workflow_ids"]["quality"], + card_id="m_quality", + metric_id="ledger_scrap_units", + start=yesterday, + end=yesterday, + ) + assert via_caliber["verdict"] == sh.ANOMALOUS + assert via_caliber["caliber_ref"]["workflow_id"] == seeded["workflow_ids"]["quality"] + + # 5) codes_new 新码值提醒 -> degraded + equip_src = await session.get(DataSource, seeded["source_ids"][mso.SOURCE_EQUIP]) + await sc.record( + session, + equip_src, + sc.NewChange(kind=sc.CODES_NEW, table="downtime_events", column="fault_category", severity="warning", key="new_code"), + sc.Detected(job_kind="profile", trigger="manual", actor="tester"), + ) + await session.commit() + degraded = await sh.check_source_health( + session, + source_name=mso.SOURCE_EQUIP, + tables=["downtime_events"], + start="2026-10-15", + end=yesterday, + ) + assert degraded["verdict"] == sh.DEGRADED + + # 6) 工件存储与台账条目形状校验 + art_id = await sh.store_source_health(anomalous, run_id=None, node_id="bot") + loaded = artifact_store.load(art_id) + assert sh.is_health_result(loaded) + entry = sh.ledger_entry(anomalous, node_id="bot", exec_no=1, artifact=art_id, call_id="c1") + assert entry["kind"] == "source_health" and entry["verdict"] == sh.ANOMALOUS + + # 7) 实时探查异常降级为 unverified 而不抛未捕获异常 + async def _boom(*_a, **_kw): + raise TimeoutError("simulated timeout") + + monkeypatch.setattr(sh, "run_query", _boom) + fallback = await sh.check_source_health(session, source_name=mso.SOURCE_OPS, tables=["shift_output"]) + assert fallback["verdict"] == sh.UNVERIFIED + + +async def _run_op(req: dict, resolve, *, refuse_data_errors: bool = False) -> dict: + return await analysis_ops.run( + req, + resolve, + settings=contract.DEFAULTS, + context={}, + refuse_data_errors=refuse_data_errors, + ) + + +async def test_reconcile_operator_verdicts_errors_and_byte_identical_recompute(): + q_left = _snap("Q1", ["line_code", "scrap_qty", "operator_phone"], [["L1", 20, "***"], ["L2", 50, "***"]], masked=["operator_phone"]) + q_right_ok = _snap("Q2", ["wc_line", "scrap_units"], [["L1", 20], ["L2", 50]], source="quality_scrap") + q_right_div = _snap("Q2", ["wc_line", "scrap_units"], [["L1", 20], ["L2", 35]], source="quality_scrap") + q_right_miss = _snap("Q2", ["wc_line", "scrap_units"], [["L1", 20]], source="quality_scrap") + + # 1) 完全对齐 -> aligned + res_aligned = await _run_op( + { + "op": "reconcile", + "mode": None, + "params": { + "base_query": "Q1", + "target_query": "Q2", + "by": ["line_code"], + "target_by": ["wc_line"], + "measure": "scrap_qty", + "target_measure": "scrap_units", + "label": "报废件数", + }, + }, + lambda name: {"Q1": q_left, "Q2": q_right_ok}.get(name), + ) + assert res_aligned["status"] == "ok" and res_aligned["verdict"]["code"] == "aligned" + + # 2) 超差 -> diverged，且双盲复算重算逐字节一致（contract.same） + req_div = { + "op": "reconcile", + "mode": None, + "params": { + "base_query": "Q1", + "target_query": "Q2", + "by": ["line_code"], + "target_by": ["wc_line"], + "measure": "scrap_qty", + "target_measure": "scrap_units", + "label": "报废件数", + }, + } + res_div = await _run_op(req_div, lambda name: {"Q1": q_left, "Q2": q_right_div}.get(name)) + res_div_again = await _run_op(req_div, lambda name: {"Q1": q_left, "Q2": q_right_div}.get(name)) + assert res_div["status"] == "ok" and res_div["verdict"]["code"] == "diverged" + assert res_div["verdict"]["subjects"] == [["L2"]] + assert contract.same(res_div, res_div_again) + + # 3) 对侧缺失 -> missing_counterpart + res_miss = await _run_op( + {**req_div}, + lambda name: {"Q1": q_left, "Q2": q_right_miss}.get(name), + ) + assert res_miss["status"] == "ok" and res_miss["verdict"]["code"] == "missing_counterpart" + + # 4) 单份查询 + unit_factor 换算（千瓦时 vs 兆瓦时 * 1000） + q_single = _snap("Q3", ["line_code", "kwh", "mwh"], [["L1", 12000, 12.0], ["L2", 13000, 13.0]]) + res_unit = await _run_op( + { + "op": "reconcile", + "mode": None, + "params": {"query": "Q3", "by": ["line_code"], "measure": "kwh", "target_measure": "mwh", "unit_factor": 1000}, + }, + lambda name: q_single if name == "Q3" else None, + ) + assert res_unit["status"] == "ok" and res_unit["verdict"]["code"] == "aligned" + + # 5) 遮罩列拒收 + 截断数据在 refuse_data_errors=True 时返回 refused + with pytest.raises(CallError): + await _run_op( + { + "op": "reconcile", + "mode": None, + "params": {"base_query": "Q1", "target_query": "Q2", "by": ["operator_phone"], "measure": "scrap_qty"}, + }, + lambda name: {"Q1": q_left, "Q2": q_right_ok}.get(name), + ) + q_trunc = _snap("QT", ["line_code", "kwh", "mwh"], [["L1", 12000, 10.0]], truncated=True) + res_trunc = await _run_op( + {"op": "reconcile", "mode": None, "params": {"query": "QT", "by": ["line_code"], "measure": "kwh", "target_measure": "mwh"}}, + lambda name: q_trunc if name == "QT" else None, + refuse_data_errors=True, + ) + assert res_trunc["status"] == "refused" + + +async def test_cochange_operator_verdicts_lag_non_causal_phrase_and_recompute(): + days = [f"2026-10-{d:02d}" for d in range(1, 16)] + # 前 14 期对称平稳小幅波动，第 15 期主指标暴跌、驱动指标暴涨 + p_base = [100, 102, 99, 101, 100, 98, 103, 101, 99, 102, 100, 98, 101, 100] + d_base = [12.0, 12.5, 11.8, 12.2, 12.0, 11.6, 12.4, 12.1, 11.9, 12.3, 12.0, 11.7, 12.2, 12.0] + p_vals = p_base + [68] + d_spike = d_base + [85.0] + d_flat = d_base + [12.1] + + q_primary = _snap("Q1", ["shift_date", "good_qty"], [[d, v] for d, v in zip(days, p_vals)]) + q_driver_spike = _snap("Q2", ["event_date", "unplanned_min"], [[d, v] for d, v in zip(days, d_spike)], source="equip_master") + q_driver_flat = _snap("Q2", ["event_date", "unplanned_min"], [[d, v] for d, v in zip(days, d_flat)], source="equip_master") + + req_co = { + "op": "cochange", + "mode": None, + "params": { + "base_query": "Q1", + "target_query": "Q2", + "period": "shift_date", + "target_period": "event_date", + "measure": "good_qty", + "driver_measure": "unplanned_min", + "label": "合格产量", + }, + } + res_co = await _run_op(req_co, lambda name: {"Q1": q_primary, "Q2": q_driver_spike}.get(name)) + res_co_2 = await _run_op(req_co, lambda name: {"Q1": q_primary, "Q2": q_driver_spike}.get(name)) + assert res_co["status"] == "ok" and res_co["verdict"]["code"] == "co_anomalous" + assert contract.same(res_co, res_co_2) + # 决策 D6：系统判定语严禁输出因果词（导致、引起、造成） + for forbidden in ("导致", "引起", "造成", "归因于"): + assert forbidden not in res_co["phrase"] + + # 上游平稳 -> target_normal + res_norm_target = await _run_op(req_co, lambda name: {"Q1": q_primary, "Q2": q_driver_flat}.get(name)) + assert res_norm_target["status"] == "ok" and res_norm_target["verdict"]["code"] == "target_normal" + + # 主指标平稳 -> primary_normal + q_primary_steady = _snap("Q1", ["shift_date", "good_qty"], [[d, v] for d, v in zip(days, p_base + [101])]) + res_norm_primary = await _run_op( + req_co, lambda name: {"Q1": q_primary_steady, "Q2": q_driver_spike}.get(name) + ) + assert res_norm_primary["status"] == "ok" and res_norm_primary["verdict"]["code"] == "primary_normal" + + # lag=1 领先期检验：驱动指标在第 14 期飙升，主指标在第 15 期下跌 -> co_anomalous + d_lead = d_base[:-1] + [88.0, 12.0] + q_driver_lead = _snap("Q2", ["event_date", "unplanned_min"], [[d, v] for d, v in zip(days, d_lead)], source="equip_master") + res_lag = await _run_op( + {**req_co, "params": {**req_co["params"], "lag": 1}}, + lambda name: {"Q1": q_primary, "Q2": q_driver_lead}.get(name), + ) + assert res_lag["status"] == "ok" and res_lag["verdict"]["code"] == "co_anomalous" + + # 历史不足 -> refused (history_short) + q_short = _snap("QS", ["shift_date", "good_qty", "unplanned_min"], [[days[i], p_vals[i], d_spike[i]] for i in range(5)]) + res_short = await _run_op( + { + "op": "cochange", + "mode": None, + "params": {"query": "QS", "period": "shift_date", "measure": "good_qty", "driver_measure": "unplanned_min"}, + }, + lambda name: q_short if name == "QS" else None, + ) + assert res_short["status"] == "refused" and res_short["reason_code"] == "history_short" + + +async def test_agent_executes_analysis_reconcile_and_cochange_and_records_ledger(tmp_path, scratch_source, monkeypatch): + from app.providers import mock_model + + await mso.seed_ontology_fixture(tmp_path, scratch_source, today=date(2026, 10, 22)) + calls = [ + ("db_query__ops_shift", {"sql": "SELECT line_code, meter_kwh, settled_mwh FROM energy_readings WHERE shift_date = '2026-10-21' ORDER BY line_code"}), + ( + "analysis_reconcile", + {"query": "Q1", "by": ["line_code"], "measure": "meter_kwh", "target_measure": "settled_mwh", "unit_factor": 1000, "label": "产线能耗"}, + ), + ("db_query__equip_master", {"sql": "SELECT event_date, net_run_ratio, unplanned_min FROM downtime_events WHERE equip_line_id = 'EQ-L2' ORDER BY event_date"}), + ( + "analysis_cochange", + {"query": "Q2", "period": "event_date", "measure": "net_run_ratio", "driver_measure": "unplanned_min", "target": "2026-10-21", "label": "有效运转率"}, + ), + ] + + def decide(model, messages): + done = sum(isinstance(m, ToolMessage) for m in messages) + if done < len(calls): + name, args = calls[done] + return AIMessage(content="", tool_calls=[{"name": name, "args": args, "id": f"call_{done}"}]) + return AIMessage(content="对账与伴随异动核查已完成。") + + monkeypatch.setattr(mock_model.MockChatModel, "_decide", decide) + + nodes = [ + {"id": "start", "type": "input", "position": {"x": 0, "y": 0}, "data": {"label": "入口", "config": {}}}, + { + "id": "bot", + "type": "agent", + "position": {"x": 220, "y": 0}, + "data": { + "label": "诊断助手", + "config": { + "prompt": "执行跨源能耗对账与设备停机伴随异动检验", + "tools": ["db_query__ops_shift", "db_query__equip_master", "analysis_reconcile", "analysis_cochange"], + "approval": "never", + }, + }, + }, + { + "id": "out", + "type": "output", + "position": {"x": 460, "y": 0}, + "data": {"label": "出口", "config": {"fields": [{"name": "answer", "value": "{{ nodes.bot.text }}"}]}}, + }, + ] + graph = {"nodes": nodes, "edges": [{"source": "start", "target": "bot"}, {"source": "bot", "target": "out"}]} + run = await run_manager.start(graph=graph, input_payload={}) + for _ in range(300): + await asyncio.sleep(0.05) + async with SessionLocal() as session: + row = await session.get(Run, run.id) + if row.status in ("succeeded", "failed") and row.manifest_seq is not None: + break + assert row.status == "succeeded", row.error + + async with SessionLocal() as session: + ev = ( + await session.execute( + select(RunEvent).where(RunEvent.run_id == run.id, RunEvent.type == "node.finished", RunEvent.node_id == "bot") + ) + ).scalar_one() + analysis_entries = [e for e in ev.data["evidence"] if e.get("kind") == "analysis"] + assert [(e["op"], e["verdict"]) for e in analysis_entries] == [ + ("reconcile", "diverged"), + ("cochange", "co_anomalous"), + ] diff --git a/backend/tests/test_template_anomaly.py b/backend/tests/test_template_anomaly.py index 7877941e..b8efa1d4 100644 --- a/backend/tests/test_template_anomaly.py +++ b/backend/tests/test_template_anomaly.py @@ -38,8 +38,8 @@ def graph() -> dict: def test_the_template_is_listed_once_after_the_others(): names = [t["name"] for t in TEMPLATES] - assert names.count(NAME) == 1 and names[-1] == NAME - assert any(n.startswith("⑨ 可追溯周报") for n in names) + nine = next(n for n in names if n.startswith("⑨ 可追溯周报")) + assert names.count(NAME) == 1 and names.index(NAME) > names.index(nine) def test_the_template_passes_validation(): diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index b78695d9..3996e09a 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState } from 'react' +import { lazy, Suspense, useEffect, useRef, useState } from 'react' import type { ReactNode, RefObject } from 'react' import { Link, NavLink, Navigate, Route, Routes, useLocation, useNavigate } from 'react-router-dom' import { Bell, BellOff, LogOut, Moon, RotateCw, Sun, UserRound } from 'lucide-react' @@ -9,34 +9,7 @@ import type { CatalogCheck } from './store/catalog' import { useStudio } from './store/studio' import { useChat } from './store/chat' import { useConversations } from './store/conversations' -import { ChatPage } from './pages/ChatPage' -import { StudioPage } from './pages/StudioPage' -import { RunsPage } from './pages/RunsPage' -import { ComparePage } from './pages/runs/ComparePage' -import { DataPage } from './pages/DataPage' -import { CatalogPage } from './pages/catalog/CatalogPage' -import { ToolsPage } from './pages/ToolsPage' -import { KnowledgePage } from './pages/KnowledgePage' -import { SettingsPage } from './pages/SettingsPage' -import { SchedulesPage } from './pages/schedules/SchedulesPage' -// 评测（下一轮第 4 波，评测-D8）：顶级导航，搭建者以上可见 -import { EvalsPage } from './pages/evals/EvalsPage' -import { CaliberCatalog } from './pages/ledger/CaliberCatalog' -import { CaliberDrafts } from './pages/ledger/CaliberDrafts' -import { NotFound } from './pages/NotFound' -// 应用平台：使用面（FE-5）、搭建面的应用编辑（FE-6） -import { AppsHome } from './pages/apps/AppsHome' -import { AppPage } from './pages/apps/AppPage' -import { EntryPage } from './pages/apps/EntryPage' -import { InstancePage } from './pages/apps/InstancePage' -import { TaskPage } from './pages/apps/TaskPage' -import { AppEditor } from './pages/apps/edit/AppEditor' -// 普通用户建应用：新建应用的起点、蓝图页（F5） -import { NewAppStart } from './pages/blueprints/NewAppStart' -import { BlueprintPage } from './pages/blueprints/BlueprintPage' import { personalAppsOn } from './pages/blueprints/shared' -import { LoginPage } from './pages/LoginPage' -import { SharePage } from './pages/apps/SharePage' import { Logo } from './components/Logo' import { CommandPalette, PAGES, ShortcutHelp, flushThemePref, openShortcutHelp, toggleCommandPalette, toggleNotify, @@ -58,6 +31,33 @@ import { useLeaveBlocker } from './lib/leave' import { useLocalActor } from './lib/actor' import { APP_TEXT, AUTH_MODE_LABEL, AUTH_TEXT, ROLE_LABEL, RUN_COMPARE_TEXT, TASK_NOTIFY_TEXT, runName } from './lib/terms' + +// 页面按需加载；外壳和导航始终可用，加载失败由现有错误边界提供恢复入口。 +const ChatPage = lazy(() => import('./pages/ChatPage').then((module) => ({ default: module.ChatPage }))) +const StudioPage = lazy(() => import('./pages/StudioPage').then((module) => ({ default: module.StudioPage }))) +const RunsPage = lazy(() => import('./pages/RunsPage').then((module) => ({ default: module.RunsPage }))) +const ComparePage = lazy(() => import('./pages/runs/ComparePage').then((module) => ({ default: module.ComparePage }))) +const DataPage = lazy(() => import('./pages/DataPage').then((module) => ({ default: module.DataPage }))) +const CatalogPage = lazy(() => import('./pages/catalog/CatalogPage').then((module) => ({ default: module.CatalogPage }))) +const ToolsPage = lazy(() => import('./pages/ToolsPage').then((module) => ({ default: module.ToolsPage }))) +const KnowledgePage = lazy(() => import('./pages/KnowledgePage').then((module) => ({ default: module.KnowledgePage }))) +const SettingsPage = lazy(() => import('./pages/SettingsPage').then((module) => ({ default: module.SettingsPage }))) +const SchedulesPage = lazy(() => import('./pages/schedules/SchedulesPage').then((module) => ({ default: module.SchedulesPage }))) +const EvalsPage = lazy(() => import('./pages/evals/EvalsPage').then((module) => ({ default: module.EvalsPage }))) +const CaliberCatalog = lazy(() => import('./pages/ledger/CaliberCatalog').then((module) => ({ default: module.CaliberCatalog }))) +const CaliberDrafts = lazy(() => import('./pages/ledger/CaliberDrafts').then((module) => ({ default: module.CaliberDrafts }))) +const NotFound = lazy(() => import('./pages/NotFound').then((module) => ({ default: module.NotFound }))) +const AppsHome = lazy(() => import('./pages/apps/AppsHome').then((module) => ({ default: module.AppsHome }))) +const AppPage = lazy(() => import('./pages/apps/AppPage').then((module) => ({ default: module.AppPage }))) +const EntryPage = lazy(() => import('./pages/apps/EntryPage').then((module) => ({ default: module.EntryPage }))) +const InstancePage = lazy(() => import('./pages/apps/InstancePage').then((module) => ({ default: module.InstancePage }))) +const TaskPage = lazy(() => import('./pages/apps/TaskPage').then((module) => ({ default: module.TaskPage }))) +const AppEditor = lazy(() => import('./pages/apps/edit/AppEditor').then((module) => ({ default: module.AppEditor }))) +const NewAppStart = lazy(() => import('./pages/blueprints/NewAppStart').then((module) => ({ default: module.NewAppStart }))) +const BlueprintPage = lazy(() => import('./pages/blueprints/BlueprintPage').then((module) => ({ default: module.BlueprintPage }))) +const LoginPage = lazy(() => import('./pages/LoginPage').then((module) => ({ default: module.LoginPage }))) +const SharePage = lazy(() => import('./pages/apps/SharePage').then((module) => ({ default: module.SharePage }))) + /** 启动时认人（/auth/me）最多等多久再取目录 */ const AUTH_WAIT_MS = 2500 @@ -68,7 +68,13 @@ const AUTH_WAIT_MS = 2500 const SHARE_PAGE = typeof window !== 'undefined' && window.location.pathname === '/s' export default function App() { - return SHARE_PAGE ? : + return }>{SHARE_PAGE ? : } +} + +function PageLoading() { + return
+ 正在加载页面… +
} function Shell() { @@ -168,7 +174,7 @@ function Shell() { {/* 开了登录的使用者只到得了「应用」和「设置」：别的地址（旧书签、手输）送回应用首页 */} {roleRedirect ? : ( - + }> {/* 默认落到对话页：多数人要的是答案，画布留给要自己编排的人。使用者落到应用首页 */} } /> } /> @@ -218,7 +224,7 @@ function Shell() { } /> } /> } /> - + )} @@ -358,17 +364,15 @@ function Nav({ stalled }: { stalled: CatalogCheck[] }) { const live = studio.startsWith('run') || chat === 'run' const pages = usePages() - // 整条导航约 690px 高（十个导航项），窗口再矮，最底下的遥测点和署名就被切掉、够不着。 - // 矮窗口（521–660px）按高度收紧到 540px 左右（导航项 36px），≤520px 再收一档到 470px 左右（导航项 30px、 - // 图标和字之间只留 2px：17px 图标 + 2px + 11px 字正好放下），而不是让导航滚动：导航一旦 overflow，挂在右侧的 - // 悬停提示和遥测浮层也会被一起裁掉。加导航项时在 600px、520px、480px 高的窗口里量一遍（check-shell 的「窗口高 …px」）。 - // 第十项「评测」加进来时量过：≤520px 那档还是 32px 的话，480px 高的窗口里遥测点超出视口 12px，所以收到 30px + // 11 项导航在中等高度用 36px，在 ≤520px 用 28px；最矮档图标和文字紧排，logo 同步收紧。 + // 保留所有入口，且不在 nav 上加 overflow，以免裁掉侧向提示和遥测浮层。 + // 增加入口后检查 700、600、520、480px：遥测和底部 padding 都必须留在视口内。 return (